LLVM 24.0.0git
DXILIntrinsicExpansion.cpp
Go to the documentation of this file.
1//===- DXILIntrinsicExpansion.cpp - Prepare LLVM Module for DXIL encoding--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file This file contains DXIL intrinsic expansions for those that don't have
10// opcodes in DirectX Intermediate Language (DXIL).
11//===----------------------------------------------------------------------===//
12
13#include "DirectX.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/STLExtras.h"
17#include "llvm/CodeGen/Passes.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/IRBuilder.h"
20#include "llvm/IR/InstrTypes.h"
21#include "llvm/IR/Instruction.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/IntrinsicsDirectX.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/PassManager.h"
28#include "llvm/IR/Type.h"
29#include "llvm/Pass.h"
33
34#define DEBUG_TYPE "dxil-intrinsic-expansion"
35
36using namespace llvm;
37
39
40public:
41 bool runOnModule(Module &M) override;
43
44 static char ID; // Pass identification.
45};
46
47static bool resourceAccessNeeds64BitExpansion(Module *M, Type *OverloadTy,
48 bool IsRaw) {
49 if (IsRaw && M->getTargetTriple().getDXILVersion() > VersionTuple(1, 2))
50 return false;
51
52 Type *ScalarTy = OverloadTy->getScalarType();
53 return ScalarTy->isDoubleTy() || ScalarTy->isIntegerTy(64);
54}
55
57 Module *M = Orig->getModule();
58 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))
59 return nullptr;
60
61 Value *Val = Orig->getOperand(0);
62 Type *ValTy = Val->getType();
63 if (!ValTy->getScalarType()->isHalfTy())
64 return nullptr;
65
66 IRBuilder<> Builder(Orig);
67 Type *IType = Type::getInt16Ty(M->getContext());
68 Constant *PosInf =
69 ValTy->isVectorTy()
72 cast<FixedVectorType>(ValTy)->getNumElements()),
73 ConstantInt::get(IType, 0x7c00))
74 : ConstantInt::get(IType, 0x7c00);
75
76 Constant *NegInf =
77 ValTy->isVectorTy()
80 cast<FixedVectorType>(ValTy)->getNumElements()),
81 ConstantInt::get(IType, 0xfc00))
82 : ConstantInt::get(IType, 0xfc00);
83
84 Value *IVal = Builder.CreateBitCast(Val, PosInf->getType());
85 Value *B1 = Builder.CreateICmpEQ(IVal, PosInf);
86 Value *B2 = Builder.CreateICmpEQ(IVal, NegInf);
87 Value *B3 = Builder.CreateOr(B1, B2);
88 return B3;
89}
90
92 Module *M = Orig->getModule();
93 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))
94 return nullptr;
95
96 Value *Val = Orig->getOperand(0);
97 Type *ValTy = Val->getType();
98 if (!ValTy->getScalarType()->isHalfTy())
99 return nullptr;
100
101 IRBuilder<> Builder(Orig);
102 Type *IType = Type::getInt16Ty(M->getContext());
103
104 Constant *ExpBitMask =
105 ValTy->isVectorTy()
108 cast<FixedVectorType>(ValTy)->getNumElements()),
109 ConstantInt::get(IType, 0x7c00))
110 : ConstantInt::get(IType, 0x7c00);
111 Constant *SigBitMask =
112 ValTy->isVectorTy()
115 cast<FixedVectorType>(ValTy)->getNumElements()),
116 ConstantInt::get(IType, 0x3ff))
117 : ConstantInt::get(IType, 0x3ff);
118
119 Constant *Zero =
120 ValTy->isVectorTy()
123 cast<FixedVectorType>(ValTy)->getNumElements()),
124 ConstantInt::get(IType, 0))
125 : ConstantInt::get(IType, 0);
126
127 Value *IVal = Builder.CreateBitCast(Val, ExpBitMask->getType());
128 Value *Exp = Builder.CreateAnd(IVal, ExpBitMask);
129 Value *B1 = Builder.CreateICmpEQ(Exp, ExpBitMask);
130
131 Value *Sig = Builder.CreateAnd(IVal, SigBitMask);
132 Value *B2 = Builder.CreateICmpNE(Sig, Zero);
133 Value *B3 = Builder.CreateAnd(B1, B2);
134 return B3;
135}
136
138 Module *M = Orig->getModule();
139 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))
140 return nullptr;
141
142 Value *Val = Orig->getOperand(0);
143 Type *ValTy = Val->getType();
144 if (!ValTy->getScalarType()->isHalfTy())
145 return nullptr;
146
147 IRBuilder<> Builder(Orig);
148 Type *IType = Type::getInt16Ty(M->getContext());
149
150 Constant *ExpBitMask =
151 ValTy->isVectorTy()
154 cast<FixedVectorType>(ValTy)->getNumElements()),
155 ConstantInt::get(IType, 0x7c00))
156 : ConstantInt::get(IType, 0x7c00);
157
158 Value *IVal = Builder.CreateBitCast(Val, ExpBitMask->getType());
159 Value *Exp = Builder.CreateAnd(IVal, ExpBitMask);
160 Value *B1 = Builder.CreateICmpNE(Exp, ExpBitMask);
161 return B1;
162}
163
165 Module *M = Orig->getModule();
166 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))
167 return nullptr;
168
169 Value *Val = Orig->getOperand(0);
170 Type *ValTy = Val->getType();
171 if (!ValTy->getScalarType()->isHalfTy())
172 return nullptr;
173
174 IRBuilder<> Builder(Orig);
175 Type *IType = Type::getInt16Ty(M->getContext());
176
177 Constant *ExpBitMask =
178 ValTy->isVectorTy()
181 cast<FixedVectorType>(ValTy)->getNumElements()),
182 ConstantInt::get(IType, 0x7c00))
183 : ConstantInt::get(IType, 0x7c00);
184 Constant *Zero =
185 ValTy->isVectorTy()
188 cast<FixedVectorType>(ValTy)->getNumElements()),
189 ConstantInt::get(IType, 0))
190 : ConstantInt::get(IType, 0);
191
192 Value *IVal = Builder.CreateBitCast(Val, ExpBitMask->getType());
193 Value *Exp = Builder.CreateAnd(IVal, ExpBitMask);
194 Value *NotAllZeroes = Builder.CreateICmpNE(Exp, Zero);
195 Value *NotAllOnes = Builder.CreateICmpNE(Exp, ExpBitMask);
196 Value *B1 = Builder.CreateAnd(NotAllZeroes, NotAllOnes);
197 return B1;
198}
199
201 assert(F.getIntrinsicID() == Intrinsic::dx_fdot &&
202 "Function is not a dx.fdot intrinsic");
203 auto *ParamTy = cast<FixedVectorType>(F.getFunctionType()->getParamType(0));
204 return ParamTy->getNumElements() <= 4 ||
205 F.getParent()->getTargetTriple().getOSVersion() < VersionTuple(6, 9);
206}
207
209 switch (F.getIntrinsicID()) {
210 case Intrinsic::assume:
211 case Intrinsic::abs:
212 case Intrinsic::atan2:
213 case Intrinsic::copysign:
214 case Intrinsic::fshl:
215 case Intrinsic::fshr:
216 case Intrinsic::exp:
217 case Intrinsic::is_fpclass:
218 case Intrinsic::log:
219 case Intrinsic::log10:
220 case Intrinsic::pow:
221 case Intrinsic::powi:
222 case Intrinsic::dx_uclamp:
223 case Intrinsic::dx_sclamp:
224 case Intrinsic::dx_nclamp:
225 case Intrinsic::dx_isfinite:
226 case Intrinsic::dx_isinf:
227 case Intrinsic::dx_isnan:
228 case Intrinsic::dx_sdot:
229 case Intrinsic::dx_udot:
230 case Intrinsic::dx_sign:
231 case Intrinsic::usub_sat:
232 case Intrinsic::vector_reduce_add:
233 case Intrinsic::vector_reduce_fadd:
234 case Intrinsic::matrix_multiply:
235 case Intrinsic::matrix_transpose:
236 case Intrinsic::umul_with_overflow:
237 case Intrinsic::smul_with_overflow:
238 case Intrinsic::dx_load_input:
239 case Intrinsic::dx_store_output:
240 return true;
241 case Intrinsic::vector_reduce_and:
242 case Intrinsic::vector_reduce_or:
243 return F.getParent()->getTargetTriple().getDXILVersion() <
244 VersionTuple(1, 9);
245 case Intrinsic::dx_quad_all:
246 return F.getParent()->getTargetTriple().getDXILVersion() <
247 VersionTuple(1, 7);
248 case Intrinsic::dx_fdot:
250 case Intrinsic::dx_resource_load_rawbuffer:
252 F.getParent(), F.getReturnType()->getStructElementType(0),
253 /*IsRaw*/ true);
254 case Intrinsic::dx_resource_load_typedbuffer:
256 F.getParent(), F.getReturnType()->getStructElementType(0),
257 /*IsRaw*/ false);
258 case Intrinsic::dx_resource_store_rawbuffer:
260 F.getParent(), F.getFunctionType()->getParamType(3), /*IsRaw*/ true);
261 case Intrinsic::dx_resource_store_typedbuffer:
263 F.getParent(), F.getFunctionType()->getParamType(2), /*IsRaw*/ false);
264 }
265 return false;
266}
267
269 Value *A = Orig->getArgOperand(0);
270 Value *B = Orig->getArgOperand(1);
271 Type *Ty = A->getType();
272
273 IRBuilder<> Builder(Orig);
274
275 Value *Cmp = Builder.CreateICmpULT(A, B, "usub.cmp");
276 Value *Sub = Builder.CreateSub(A, B, "usub.sub");
277 Value *Zero = ConstantInt::get(Ty, 0);
278 return Builder.CreateSelect(Cmp, Zero, Sub, "usub.sat");
279}
280
281// Compute the high N bits of the 2N-bit unsigned product of two N-bit values
282// using only N-bit arithmetic, so we don't introduce a wider integer type that
283// may be unsupported in DXIL.
285 Type *Ty, unsigned BW) {
286 assert(BW % 2 == 0 && "high-half split needs symmetric halves");
287 unsigned Half = BW / 2;
288 Value *HalfShift = ConstantInt::get(Ty, Half);
289 Value *LoMask = ConstantInt::get(Ty, APInt::getLowBitsSet(BW, Half));
290
291 Value *U0 = Builder.CreateAnd(A, LoMask);
292 Value *U1 = Builder.CreateLShr(A, HalfShift);
293 Value *V0 = Builder.CreateAnd(B, LoMask);
294 Value *V1 = Builder.CreateLShr(B, HalfShift);
295
296 Value *W0 = Builder.CreateMul(U0, V0);
297 Value *T = Builder.CreateAdd(Builder.CreateMul(U1, V0),
298 Builder.CreateLShr(W0, HalfShift));
299 Value *W1 = Builder.CreateAnd(T, LoMask);
300 Value *W2 = Builder.CreateLShr(T, HalfShift);
301 W1 = Builder.CreateAdd(Builder.CreateMul(U0, V1), W1);
302 return Builder.CreateAdd(Builder.CreateAdd(Builder.CreateMul(U1, V1), W2),
303 Builder.CreateLShr(W1, HalfShift));
304}
305
306// Expand a {u,s}mul.with.overflow intrinsic. The low half of the result is a
307// plain multiply; overflow is derived from the high half of the double-width
308// product.
310 IRBuilder<> Builder(Orig);
311 Value *A = Orig->getArgOperand(0);
312 Value *B = Orig->getArgOperand(1);
313 Type *Ty = A->getType();
314 unsigned BW = Ty->getScalarSizeInBits();
315
316 Value *Lo;
317 Value *Ov;
318
319 // A plain double-width multiply is simplest, but we avoid it once it would
320 // introduce a 64-bit (or wider) integer, which DXIL does not always support.
321 // For i32 we use the native DXIL IMul/UMul ops, which return the full product
322 // as two i32s; wider types fall back to a same-width high-half computation.
323 if (2 * BW <= 32) {
324 Lo = Builder.CreateMul(A, B);
325 Type *WideTy = Ty->getWithNewBitWidth(2 * BW);
326 Value *WideA =
327 Signed ? Builder.CreateSExt(A, WideTy) : Builder.CreateZExt(A, WideTy);
328 Value *WideB =
329 Signed ? Builder.CreateSExt(B, WideTy) : Builder.CreateZExt(B, WideTy);
330 Value *Wide = Builder.CreateMul(WideA, WideB);
331 if (Signed) {
332 // Overflow when the full product doesn't fit back into BW signed bits.
333 Ov = Builder.CreateICmpNE(Wide, Builder.CreateSExt(Lo, WideTy));
334 } else {
335 Value *Hi = Builder.CreateLShr(Wide, ConstantInt::get(WideTy, BW));
336 Ov = Builder.CreateICmpNE(Hi, ConstantInt::get(WideTy, 0));
337 }
338 } else if (BW == 32) {
339 // IMul/UMul return {high, low}; index 0 is the high 32 bits.
340 Type *ResTy = StructType::get(Ty, Ty);
341 Intrinsic::ID IntrinsicID =
342 Signed ? Intrinsic::dx_imul : Intrinsic::dx_umul;
343 Value *Mul = Builder.CreateIntrinsic(ResTy, IntrinsicID, {A, B});
344 Value *Hi = Builder.CreateExtractValue(Mul, 0);
345 Lo = Builder.CreateExtractValue(Mul, 1);
346 if (Signed)
347 Ov = Builder.CreateICmpNE(
348 Hi, Builder.CreateAShr(Lo, ConstantInt::get(Ty, BW - 1)));
349 else
350 Ov = Builder.CreateICmpNE(Hi, ConstantInt::get(Ty, 0));
351 } else {
352 Lo = Builder.CreateMul(A, B);
353 Value *Hi = createMulHighUnsigned(Builder, A, B, Ty, BW);
354 if (Signed) {
355 // Turn the unsigned high half into the signed one, then overflow means it
356 // isn't the sign extension of the low half.
357 Value *SignShift = ConstantInt::get(Ty, BW - 1);
358 Value *ASign = Builder.CreateAShr(A, SignShift);
359 Value *BSign = Builder.CreateAShr(B, SignShift);
360 Hi = Builder.CreateSub(Hi, Builder.CreateAnd(ASign, B));
361 Hi = Builder.CreateSub(Hi, Builder.CreateAnd(BSign, A));
362 Ov = Builder.CreateICmpNE(Hi, Builder.CreateAShr(Lo, SignShift));
363 } else {
364 Ov = Builder.CreateICmpNE(Hi, ConstantInt::get(Ty, 0));
365 }
366 }
367
368 Value *Agg = PoisonValue::get(Orig->getType());
369 Agg = Builder.CreateInsertValue(Agg, Lo, 0);
370 return Builder.CreateInsertValue(Agg, Ov, 1);
371}
372
373static Value *expandVecReduceAdd(CallInst *Orig, Intrinsic::ID IntrinsicId) {
374 assert(IntrinsicId == Intrinsic::vector_reduce_add ||
375 IntrinsicId == Intrinsic::vector_reduce_fadd);
376
377 IRBuilder<> Builder(Orig);
378 bool IsFAdd = (IntrinsicId == Intrinsic::vector_reduce_fadd);
379
380 Value *X = Orig->getOperand(IsFAdd ? 1 : 0);
381 Type *Ty = X->getType();
382 auto *XVec = dyn_cast<FixedVectorType>(Ty);
383 unsigned XVecSize = XVec->getNumElements();
384 Value *Sum = Builder.CreateExtractElement(X, static_cast<uint64_t>(0));
385
386 // Handle the initial start value for floating-point addition.
387 if (IsFAdd) {
388 Constant *StartValue = dyn_cast<Constant>(Orig->getOperand(0));
389 if (StartValue && !StartValue->isNullValue())
390 Sum = Builder.CreateFAdd(Sum, StartValue);
391 }
392
393 // Accumulate the remaining vector elements.
394 for (unsigned I = 1; I < XVecSize; I++) {
395 Value *Elt = Builder.CreateExtractElement(X, I);
396 if (IsFAdd)
397 Sum = Builder.CreateFAdd(Sum, Elt);
398 else
399 Sum = Builder.CreateAdd(Sum, Elt);
400 }
401
402 return Sum;
403}
404
406 assert(IntrinsicId == Intrinsic::vector_reduce_and ||
407 IntrinsicId == Intrinsic::vector_reduce_or);
408
409 IRBuilder<> Builder(Orig);
410 Value *X = Orig->getArgOperand(0);
411 auto *XVec = cast<FixedVectorType>(X->getType());
412 Value *Result = Builder.CreateExtractElement(X, static_cast<uint64_t>(0));
413 for (unsigned I = 1; I < XVec->getNumElements(); ++I) {
414 Value *Elt = Builder.CreateExtractElement(X, I);
415 Result = IntrinsicId == Intrinsic::vector_reduce_or
416 ? Builder.CreateOr(Result, Elt)
417 : Builder.CreateAnd(Result, Elt);
418 }
419 return Result;
420}
421
422static Value *expandAbs(CallInst *Orig) {
423 Value *X = Orig->getOperand(0);
424 IRBuilder<> Builder(Orig);
425 Type *Ty = X->getType();
426 Type *EltTy = Ty->getScalarType();
427 Constant *Zero = Ty->isVectorTy()
430 cast<FixedVectorType>(Ty)->getNumElements()),
431 ConstantInt::get(EltTy, 0))
432 : ConstantInt::get(EltTy, 0);
433 auto *V = Builder.CreateSub(Zero, X);
434 return Builder.CreateIntrinsic(Ty, Intrinsic::smax, {X, V}, nullptr,
435 "dx.max");
436}
437
438// Create a DXIL dot2, dot3, or dot4 for the given operands.
440 Type *ATy = A->getType();
441 [[maybe_unused]] Type *BTy = B->getType();
442 assert(ATy->isVectorTy() && BTy->isVectorTy());
443
444 IRBuilder<> Builder(Orig);
445
446 auto *AVec = dyn_cast<FixedVectorType>(ATy);
447
449
450 unsigned NumElts = AVec->getNumElements();
451 Intrinsic::ID DotIntrinsic;
452 switch (NumElts) {
453 case 2:
454 DotIntrinsic = Intrinsic::dx_dot2;
455 break;
456 case 3:
457 DotIntrinsic = Intrinsic::dx_dot3;
458 break;
459 case 4:
460 DotIntrinsic = Intrinsic::dx_dot4;
461 break;
462 default:
464 "Invalid dot product input vector: length is outside 2-4");
465 }
466
468 for (unsigned I = 0; I < NumElts; ++I)
469 Args.push_back(Builder.CreateExtractElement(A, Builder.getInt32(I)));
470 for (unsigned I = 0; I < NumElts; ++I)
471 Args.push_back(Builder.CreateExtractElement(B, Builder.getInt32(I)));
472 return Builder.CreateIntrinsic(ATy->getScalarType(), DotIntrinsic, Args,
473 nullptr, "dot");
474}
475
476// Expand an arbitrary-width float dot into the minimum number of legal DXIL
477// dot2, dot3, and dot4 operations.
479 Value *A = Orig->getOperand(0);
480 Value *B = Orig->getOperand(1);
481 unsigned NumElts = cast<FixedVectorType>(A->getType())->getNumElements();
482
483 // We return early here to avoid constructing unnecessary identity shuffles.
484 if (NumElts <= 4)
485 return expandFloatDotChunk(Orig, A, B);
486
488 VersionTuple(6, 9) &&
489 "long fdot must not be expanded for shader model 6.9 or later");
490
491 IRBuilder<> Builder(Orig);
492 Value *Result = nullptr;
493 for (unsigned Offset = 0; Offset < NumElts;) {
494 unsigned Remaining = NumElts - Offset;
495 // Taking four is optimal unless it would leave an illegal one-element
496 // tail. In that case, take three and finish with dot2.
497 unsigned ChunkSize = Remaining == 5 ? 3 : std::min(Remaining, 4u);
499 for (unsigned I = 0; I < ChunkSize; ++I)
500 Mask.push_back(Offset + I);
501 Value *AChunk = Builder.CreateShuffleVector(A, Mask);
502 Value *BChunk = Builder.CreateShuffleVector(B, Mask);
503 Value *Chunk = expandFloatDotChunk(Orig, AChunk, BChunk);
504 Result = Result ? Builder.CreateFAdd(Result, Chunk, "dot.add") : Chunk;
505 Offset += ChunkSize;
506 }
507 return Result;
508}
509
510// Expand integer dot product to multiply and add ops
512 Intrinsic::ID DotIntrinsic) {
513 assert(DotIntrinsic == Intrinsic::dx_sdot ||
514 DotIntrinsic == Intrinsic::dx_udot);
515 Value *A = Orig->getOperand(0);
516 Value *B = Orig->getOperand(1);
517 Type *ATy = A->getType();
518 [[maybe_unused]] Type *BTy = B->getType();
519 assert(ATy->isVectorTy() && BTy->isVectorTy());
520
521 IRBuilder<> Builder(Orig);
522
523 auto *AVec = dyn_cast<FixedVectorType>(ATy);
524
526
527 Value *Result;
528 Intrinsic::ID MadIntrinsic = DotIntrinsic == Intrinsic::dx_sdot
529 ? Intrinsic::dx_imad
530 : Intrinsic::dx_umad;
531 Value *Elt0 = Builder.CreateExtractElement(A, (uint64_t)0);
532 Value *Elt1 = Builder.CreateExtractElement(B, (uint64_t)0);
533 Result = Builder.CreateMul(Elt0, Elt1);
534 for (unsigned I = 1; I < AVec->getNumElements(); I++) {
535 Elt0 = Builder.CreateExtractElement(A, I);
536 Elt1 = Builder.CreateExtractElement(B, I);
537 Result = Builder.CreateIntrinsic(Result->getType(), MadIntrinsic,
538 ArrayRef<Value *>{Elt0, Elt1, Result},
539 nullptr, "dx.mad");
540 }
541 return Result;
542}
543
545 Value *X = Orig->getOperand(0);
546 IRBuilder<> Builder(Orig);
547 Type *Ty = X->getType();
548 Type *EltTy = Ty->getScalarType();
549 Constant *Log2eConst =
550 Ty->isVectorTy() ? ConstantVector::getSplat(
552 cast<FixedVectorType>(Ty)->getNumElements()),
553 ConstantFP::get(EltTy, numbers::log2ef))
554 : ConstantFP::get(EltTy, numbers::log2ef);
555 Value *NewX = Builder.CreateFMul(Log2eConst, X);
556 CallInst *Exp2Call = Builder.CreateIntrinsicWithoutFolding(
557 Ty, Intrinsic::exp2, {NewX}, nullptr, "dx.exp2");
558 Exp2Call->setTailCall(Orig->isTailCall());
559 Exp2Call->setAttributes(Orig->getAttributes());
560 return Exp2Call;
561}
562
564 Value *T = Orig->getArgOperand(1);
565 auto *TCI = dyn_cast<ConstantInt>(T);
566
567 // These FPClassTest cases have DXIL opcodes, so they will be handled in
568 // DXIL Op Lowering instead for all non f16 cases.
569 switch (TCI->getZExtValue()) {
571 return expand16BitIsInf(Orig);
573 return expand16BitIsNaN(Orig);
575 return expand16BitIsNormal(Orig);
577 return expand16BitIsFinite(Orig);
578 }
579
580 IRBuilder<> Builder(Orig);
581
582 Value *F = Orig->getArgOperand(0);
583 Type *FTy = F->getType();
584 unsigned FNumElem = 0; // 0 => F is not a vector
585
586 unsigned BitWidth; // Bit width of F or the ElemTy of F
587 Type *BitCastTy; // An IntNTy of the same bitwidth as F or ElemTy of F
588
589 if (auto *FVecTy = dyn_cast<FixedVectorType>(FTy)) {
590 Type *ElemTy = FVecTy->getElementType();
591 FNumElem = FVecTy->getNumElements();
592 BitWidth = ElemTy->getPrimitiveSizeInBits();
593 BitCastTy = FixedVectorType::get(Builder.getIntNTy(BitWidth), FNumElem);
594 } else {
596 BitCastTy = Builder.getIntNTy(BitWidth);
597 }
598
599 Value *FBitCast = Builder.CreateBitCast(F, BitCastTy);
600 switch (TCI->getZExtValue()) {
602 Value *NegZero = ConstantInt::get(Builder.getIntNTy(BitWidth),
604 Value *RetVal;
605 if (FNumElem) {
606 Value *NegZeroSplat = Builder.CreateVectorSplat(FNumElem, NegZero);
607 RetVal =
608 Builder.CreateICmpEQ(FBitCast, NegZeroSplat, "is.fpclass.negzero");
609 } else
610 RetVal = Builder.CreateICmpEQ(FBitCast, NegZero, "is.fpclass.negzero");
611 return RetVal;
612 }
613 default:
614 reportFatalUsageError("Unsupported FPClassTest");
615 }
616}
617
619 float LogConstVal = numbers::ln2f) {
620 Value *X = Orig->getOperand(0);
621 IRBuilder<> Builder(Orig);
622 Type *Ty = X->getType();
623 Type *EltTy = Ty->getScalarType();
624 Constant *Ln2Const =
625 Ty->isVectorTy() ? ConstantVector::getSplat(
627 cast<FixedVectorType>(Ty)->getNumElements()),
628 ConstantFP::get(EltTy, LogConstVal))
629 : ConstantFP::get(EltTy, LogConstVal);
630 CallInst *Log2Call = Builder.CreateIntrinsicWithoutFolding(
631 Ty, Intrinsic::log2, {X}, nullptr, "elt.log2");
632 Log2Call->setTailCall(Orig->isTailCall());
633 Log2Call->setAttributes(Orig->getAttributes());
634 return Builder.CreateFMul(Ln2Const, Log2Call);
635}
639
641 Value *Y = Orig->getOperand(0);
642 Value *X = Orig->getOperand(1);
643 Type *Ty = X->getType();
644 IRBuilder<> Builder(Orig);
645 Builder.setFastMathFlags(Orig->getFastMathFlags());
646
647 Value *Tan = Builder.CreateFDiv(Y, X);
648
649 CallInst *Atan = Builder.CreateIntrinsicWithoutFolding(
650 Ty, Intrinsic::atan, {Tan}, nullptr, "Elt.Atan");
651 Atan->setTailCall(Orig->isTailCall());
652 Atan->setAttributes(Orig->getAttributes());
653
654 // Modify atan result based on https://en.wikipedia.org/wiki/Atan2.
655 Constant *Pi = ConstantFP::get(Ty, llvm::numbers::pi);
656 Constant *HalfPi = ConstantFP::get(Ty, llvm::numbers::pi / 2);
657 Constant *NegHalfPi = ConstantFP::get(Ty, -llvm::numbers::pi / 2);
658 Constant *Zero = ConstantFP::get(Ty, 0);
659 Value *AtanAddPi = Builder.CreateFAdd(Atan, Pi);
660 Value *AtanSubPi = Builder.CreateFSub(Atan, Pi);
661
662 // x > 0 -> atan.
663 Value *Result = Atan;
664 Value *XLt0 = Builder.CreateFCmpOLT(X, Zero);
665 Value *XEq0 = Builder.CreateFCmpOEQ(X, Zero);
666 Value *YGe0 = Builder.CreateFCmpOGE(Y, Zero);
667 Value *YLt0 = Builder.CreateFCmpOLT(Y, Zero);
668
669 // x < 0, y >= 0 -> atan + pi.
670 Value *XLt0AndYGe0 = Builder.CreateAnd(XLt0, YGe0);
671 Result = Builder.CreateSelect(XLt0AndYGe0, AtanAddPi, Result);
672
673 // x < 0, y < 0 -> atan - pi.
674 Value *XLt0AndYLt0 = Builder.CreateAnd(XLt0, YLt0);
675 Result = Builder.CreateSelect(XLt0AndYLt0, AtanSubPi, Result);
676
677 // x == 0, y < 0 -> -pi/2
678 Value *XEq0AndYLt0 = Builder.CreateAnd(XEq0, YLt0);
679 Result = Builder.CreateSelect(XEq0AndYLt0, NegHalfPi, Result);
680
681 // x == 0, y > 0 -> pi/2
682 Value *XEq0AndYGe0 = Builder.CreateAnd(XEq0, YGe0);
683 Result = Builder.CreateSelect(XEq0AndYGe0, HalfPi, Result);
684
685 return Result;
686}
687
688template <bool LeftFunnel>
690 Type *Ty = Orig->getType();
691 Value *A = Orig->getOperand(0);
692 Value *B = Orig->getOperand(1);
693 Value *Shift = Orig->getOperand(2);
694
695 IRBuilder<> Builder(Orig);
696
697 assert(llvm::isPowerOf2_32(Ty->getScalarSizeInBits()) &&
698 "Can't use Mask to compute modulo and inverse");
699
700 // Note: if (Shift % BitWidth) == 0 then (BitWidth - Shift) == BitWidth,
701 // shifting by the bitwidth for shl/lshr returns a poisoned result. As such,
702 // we implement the same formula as LegalizerHelper::lowerFunnelShiftAsShifts.
703 //
704 // The funnel shift is expanded like so:
705 // fshl
706 // -> msb_extract((concat(A, B) << (Shift % BitWidth)), BitWidth)
707 // -> A << (Shift % BitWidth) | B >> 1 >> (BitWidth - 1 - (Shift % BitWidth))
708 // fshr
709 // -> lsb_extract((concat(A, B) >> (Shift % BitWidth), BitWidth))
710 // -> A << 1 << (BitWidth - 1 - (Shift % BitWidth)) | B >> (Shift % BitWidth)
711
712 // (BitWidth - 1) -> Mask
713 Constant *Mask = ConstantInt::get(Ty, Ty->getScalarSizeInBits() - 1);
714
715 // Shift % BitWidth
716 // -> Shift & (BitWidth - 1)
717 // -> Shift & Mask
718 Value *MaskedShift = Builder.CreateAnd(Shift, Mask);
719
720 // (BitWidth - 1) - (Shift % BitWidth)
721 // -> ~Shift & (BitWidth - 1)
722 // -> ~Shift & Mask
723 Value *NotShift = Builder.CreateNot(Shift);
724 Value *InverseShift = Builder.CreateAnd(NotShift, Mask);
725
726 Constant *One = ConstantInt::get(Ty, 1);
727 Value *ShiftedA;
728 Value *ShiftedB;
729
730 if (LeftFunnel) {
731 ShiftedA = Builder.CreateShl(A, MaskedShift);
732 Value *ShiftB1 = Builder.CreateLShr(B, One);
733 ShiftedB = Builder.CreateLShr(ShiftB1, InverseShift);
734 } else {
735 Value *ShiftA1 = Builder.CreateShl(A, One);
736 ShiftedA = Builder.CreateShl(ShiftA1, InverseShift);
737 ShiftedB = Builder.CreateLShr(B, MaskedShift);
738 }
739
740 Value *Result = Builder.CreateOr(ShiftedA, ShiftedB);
741 return Result;
742}
743
744static Value *expandPowIntrinsic(CallInst *Orig, Intrinsic::ID IntrinsicId) {
745
746 Value *X = Orig->getOperand(0);
747 Value *Y = Orig->getOperand(1);
748 Type *Ty = X->getType();
749 IRBuilder<> Builder(Orig);
750
751 if (IntrinsicId == Intrinsic::powi)
752 Y = Builder.CreateSIToFP(Y, Ty);
753
754 Value *Log2Call =
755 Builder.CreateIntrinsic(Ty, Intrinsic::log2, {X}, nullptr, "elt.log2");
756 auto *Mul = Builder.CreateFMul(Log2Call, Y);
757 CallInst *Exp2Call = Builder.CreateIntrinsicWithoutFolding(
758 Ty, Intrinsic::exp2, {Mul}, nullptr, "elt.exp2");
759 Exp2Call->setTailCall(Orig->isTailCall());
760 Exp2Call->setAttributes(Orig->getAttributes());
761 return Exp2Call;
762}
763
764static bool expandBufferLoadIntrinsic(CallInst *Orig, bool IsRaw) {
765 IRBuilder<> Builder(Orig);
766
767 Type *BufferTy = Orig->getType()->getStructElementType(0);
768 Type *ScalarTy = BufferTy->getScalarType();
769 bool IsDouble = ScalarTy->isDoubleTy();
770 assert(IsDouble || ScalarTy->isIntegerTy(64) &&
771 "Only expand double or int64 scalars or vectors");
772 bool IsVector = false;
773 unsigned ExtractNum = 2;
774 if (auto *VT = dyn_cast<FixedVectorType>(BufferTy)) {
775 ExtractNum = 2 * VT->getNumElements();
776 IsVector = true;
777 assert(IsRaw || ExtractNum == 4 && "TypedBufferLoad vector must be size 2");
778 }
779
781 Value *Result = PoisonValue::get(BufferTy);
782 unsigned Base = 0;
783 // If we need to extract more than 4 i32; we need to break it up into
784 // more than one load. LoadNum tells us how many i32s we are loading in
785 // each load
786 while (ExtractNum > 0) {
787 unsigned LoadNum = std::min(ExtractNum, 4u);
788 Type *Ty = VectorType::get(Builder.getInt32Ty(), LoadNum, false);
789
790 Type *LoadType = StructType::get(Ty, Builder.getInt1Ty());
791 Intrinsic::ID LoadIntrinsic = Intrinsic::dx_resource_load_typedbuffer;
792 SmallVector<Value *, 3> Args = {Orig->getOperand(0), Orig->getOperand(1)};
793 if (IsRaw) {
794 LoadIntrinsic = Intrinsic::dx_resource_load_rawbuffer;
795 Value *Tmp = Builder.getInt32(4 * Base * 2);
796 Value *Offset = Orig->getOperand(2);
797 Args.push_back(Offset);
798 unsigned AddressArg = isa<PoisonValue>(Offset) ? 1 : 2;
799 if (Base != 0)
800 Args[AddressArg] = Builder.CreateAdd(Args[AddressArg], Tmp);
801 }
802
803 Value *Load = Builder.CreateIntrinsic(LoadType, LoadIntrinsic, Args);
804 Loads.push_back(Load);
805
806 // extract the buffer load's result
807 Value *Extract = Builder.CreateExtractValue(Load, {0});
808
809 SmallVector<Value *> ExtractElements;
810 for (unsigned I = 0; I < LoadNum; ++I)
811 ExtractElements.push_back(
812 Builder.CreateExtractElement(Extract, Builder.getInt32(I)));
813
814 // combine into double(s) or int64(s)
815 for (unsigned I = 0; I < LoadNum; I += 2) {
816 Value *Combined = nullptr;
817 if (IsDouble)
818 // For doubles, use dx_asdouble intrinsic
819 Combined = Builder.CreateIntrinsic(
820 Builder.getDoubleTy(), Intrinsic::dx_asdouble,
821 {ExtractElements[I], ExtractElements[I + 1]});
822 else {
823 // For int64, manually combine two int32s
824 // First, zero-extend both values to i64
825 Value *Lo =
826 Builder.CreateZExt(ExtractElements[I], Builder.getInt64Ty());
827 Value *Hi =
828 Builder.CreateZExt(ExtractElements[I + 1], Builder.getInt64Ty());
829 // Shift the high bits left by 32 bits
830 Value *ShiftedHi = Builder.CreateShl(Hi, Builder.getInt64(32));
831 // OR the high and low bits together
832 Combined = Builder.CreateOr(Lo, ShiftedHi);
833 }
834
835 if (IsVector)
836 Result = Builder.CreateInsertElement(Result, Combined,
837 Builder.getInt32((I / 2) + Base));
838 else
839 Result = Combined;
840 }
841
842 ExtractNum -= LoadNum;
843 Base += LoadNum / 2;
844 }
845
846 Value *CheckBit = nullptr;
847 for (User *U : make_early_inc_range(Orig->users())) {
848 // If it's not a ExtractValueInst, we don't know how to
849 // handle it
850 auto *EVI = dyn_cast<ExtractValueInst>(U);
851 if (!EVI)
852 llvm_unreachable("Unexpected user of typedbufferload");
853
854 ArrayRef<unsigned> Indices = EVI->getIndices();
855 assert(Indices.size() == 1);
856
857 if (Indices[0] == 0) {
858 // Use of the value(s)
859 EVI->replaceAllUsesWith(Result);
860 } else {
861 // Use of the check bit
862 assert(Indices[0] == 1 && "Unexpected type for typedbufferload");
863 // Note: This does not always match the historical behaviour of DXC.
864 // See https://github.com/microsoft/DirectXShaderCompiler/issues/7622
865 if (!CheckBit) {
866 SmallVector<Value *, 2> CheckBits;
867 for (Value *L : Loads)
868 CheckBits.push_back(Builder.CreateExtractValue(L, {1}));
869 CheckBit = Builder.CreateAnd(CheckBits);
870 }
871 EVI->replaceAllUsesWith(CheckBit);
872 }
873 EVI->eraseFromParent();
874 }
875 Orig->eraseFromParent();
876 return true;
877}
878
879static bool expandBufferStoreIntrinsic(CallInst *Orig, bool IsRaw) {
880 IRBuilder<> Builder(Orig);
881
882 unsigned ValIndex = IsRaw ? 3 : 2;
883 Type *BufferTy = Orig->getFunctionType()->getParamType(ValIndex);
884 Type *ScalarTy = BufferTy->getScalarType();
885 bool IsDouble = ScalarTy->isDoubleTy();
886 assert((IsDouble || ScalarTy->isIntegerTy(64)) &&
887 "Only expand double or int64 scalars or vectors");
888
889 // Determine if we're dealing with a vector or scalar
890 bool IsVector = false;
891 unsigned ExtractNum = 2;
892 unsigned VecLen = 0;
893 if (auto *VT = dyn_cast<FixedVectorType>(BufferTy)) {
894 VecLen = VT->getNumElements();
895 assert(IsRaw || VecLen == 2 && "TypedBufferStore vector must be size 2");
896 ExtractNum = VecLen * 2;
897 IsVector = true;
898 }
899
900 // Create the appropriate vector type for the result
901 Type *Int32Ty = Builder.getInt32Ty();
902 Type *ResultTy = VectorType::get(Int32Ty, ExtractNum, false);
903 Value *Val = PoisonValue::get(ResultTy);
904
905 Type *SplitElementTy = Int32Ty;
906 if (IsVector)
907 SplitElementTy = VectorType::get(SplitElementTy, VecLen, false);
908
909 Value *LowBits = nullptr;
910 Value *HighBits = nullptr;
911 // Split the 64-bit values into 32-bit components
912 if (IsDouble) {
913 auto *SplitTy = llvm::StructType::get(SplitElementTy, SplitElementTy);
914 Value *Split = Builder.CreateIntrinsic(SplitTy, Intrinsic::dx_splitdouble,
915 {Orig->getOperand(ValIndex)});
916 LowBits = Builder.CreateExtractValue(Split, 0);
917 HighBits = Builder.CreateExtractValue(Split, 1);
918 } else {
919 // Handle int64 type(s)
920 Value *InputVal = Orig->getOperand(ValIndex);
921 Constant *ShiftAmt = Builder.getInt64(32);
922 if (IsVector)
923 ShiftAmt =
925
926 // Split into low and high 32-bit parts
927 LowBits = Builder.CreateTrunc(InputVal, SplitElementTy);
928 Value *ShiftedVal = Builder.CreateLShr(InputVal, ShiftAmt);
929 HighBits = Builder.CreateTrunc(ShiftedVal, SplitElementTy);
930 }
931
932 if (IsVector) {
934 for (unsigned I = 0; I < VecLen; ++I) {
935 Mask.push_back(I);
936 Mask.push_back(I + VecLen);
937 }
938 Val = Builder.CreateShuffleVector(LowBits, HighBits, Mask);
939 } else {
940 Val = Builder.CreateInsertElement(Val, LowBits, Builder.getInt32(0));
941 Val = Builder.CreateInsertElement(Val, HighBits, Builder.getInt32(1));
942 }
943
944 // If we need to extract more than 4 i32; we need to break it up into
945 // more than one store. StoreNum tells us how many i32s we are storing in
946 // each store
947 unsigned Base = 0;
948 while (ExtractNum > 0) {
949 unsigned StoreNum = std::min(ExtractNum, 4u);
950
951 Intrinsic::ID StoreIntrinsic = Intrinsic::dx_resource_store_typedbuffer;
952 SmallVector<Value *, 4> Args = {Orig->getOperand(0), Orig->getOperand(1)};
953 if (IsRaw) {
954 StoreIntrinsic = Intrinsic::dx_resource_store_rawbuffer;
955 Value *Tmp = Builder.getInt32(4 * Base);
956 Value *Offset = Orig->getOperand(2);
957 Args.push_back(Offset);
958 unsigned AddressArg = isa<PoisonValue>(Offset) ? 1 : 2;
959 if (Base != 0)
960 Args[AddressArg] = Builder.CreateAdd(Args[AddressArg], Tmp);
961 }
962
964 for (unsigned I = 0; I < StoreNum; ++I) {
965 Mask.push_back(Base + I);
966 }
967
968 Value *SubVal = Val;
969 if (VecLen > 2)
970 SubVal = Builder.CreateShuffleVector(Val, Mask);
971
972 Args.push_back(SubVal);
973 // Create the final intrinsic call
974 Builder.CreateIntrinsic(Builder.getVoidTy(), StoreIntrinsic, Args);
975
976 ExtractNum -= StoreNum;
977 Base += StoreNum;
978 }
979 Orig->eraseFromParent();
980 return true;
981}
982
984 if (ClampIntrinsic == Intrinsic::dx_uclamp)
985 return Intrinsic::umax;
986 if (ClampIntrinsic == Intrinsic::dx_sclamp)
987 return Intrinsic::smax;
988 assert(ClampIntrinsic == Intrinsic::dx_nclamp);
989 return Intrinsic::maxnum;
990}
991
993 if (ClampIntrinsic == Intrinsic::dx_uclamp)
994 return Intrinsic::umin;
995 if (ClampIntrinsic == Intrinsic::dx_sclamp)
996 return Intrinsic::smin;
997 assert(ClampIntrinsic == Intrinsic::dx_nclamp);
998 return Intrinsic::minnum;
999}
1000
1002 Intrinsic::ID ClampIntrinsic) {
1003 Value *X = Orig->getOperand(0);
1004 Value *Min = Orig->getOperand(1);
1005 Value *Max = Orig->getOperand(2);
1006 Type *Ty = X->getType();
1007 IRBuilder<> Builder(Orig);
1008 auto *MaxCall = Builder.CreateIntrinsic(Ty, getMaxForClamp(ClampIntrinsic),
1009 {X, Min}, nullptr, "dx.max");
1010 return Builder.CreateIntrinsic(Ty, getMinForClamp(ClampIntrinsic),
1011 {MaxCall, Max}, nullptr, "dx.min");
1012}
1013
1015 Value *X = Orig->getOperand(0);
1016 Type *Ty = X->getType();
1017 Type *ScalarTy = Ty->getScalarType();
1018 Type *RetTy = Orig->getType();
1020
1021 IRBuilder<> Builder(Orig);
1022
1023 Value *GT;
1024 Value *LT;
1025 if (ScalarTy->isFloatingPointTy()) {
1026 GT = Builder.CreateFCmpOLT(Zero, X);
1027 LT = Builder.CreateFCmpOLT(X, Zero);
1028 } else {
1029 assert(ScalarTy->isIntegerTy());
1030 GT = Builder.CreateICmpSLT(Zero, X);
1031 LT = Builder.CreateICmpSLT(X, Zero);
1032 }
1033
1034 Value *ZextGT = Builder.CreateZExt(GT, RetTy);
1035 Value *ZextLT = Builder.CreateZExt(LT, RetTy);
1036
1037 return Builder.CreateSub(ZextGT, ZextLT);
1038}
1039
1040// Expand llvm.copysign by combining the sign bit with the magnitude bits using
1041// bitwise operations.
1043 Value *Magnitude = Orig->getOperand(0);
1044 Value *Sign = Orig->getOperand(1);
1045 Type *Ty = Orig->getType();
1046
1047 IRBuilder<> Builder(Orig);
1048
1049 bool IsDouble = Ty->getScalarType()->isDoubleTy();
1050 unsigned BitWidth = IsDouble ? 32 : Ty->getScalarSizeInBits();
1051 Type *IntTy = Ty->getWithNewType(Builder.getIntNTy(BitWidth));
1052
1053 auto CopySignBit = [&](Value *MagnitudeInt, Value *SignInt) {
1054 APInt SignMaskVal = APInt::getSignMask(BitWidth);
1055 // `ConstantInt::get` broadcasts to a splat when `IntTy` is a vector.
1056 Constant *SignMask = ConstantInt::get(IntTy, SignMaskVal);
1057 Constant *NotSignMask = ConstantInt::get(IntTy, ~SignMaskVal);
1058
1059 Value *MagnitudeBits = Builder.CreateAnd(MagnitudeInt, NotSignMask);
1060 Value *SignBits = Builder.CreateAnd(SignInt, SignMask);
1061 return Builder.CreateOr(MagnitudeBits, SignBits);
1062 };
1063
1064 // Avoid i64 bitwise ops, which require the Int64Ops shader feature.
1065 if (IsDouble) {
1066 auto *SplitTy = StructType::get(IntTy, IntTy);
1067 Value *MagnitudeHalves = Builder.CreateIntrinsic(
1068 SplitTy, Intrinsic::dx_splitdouble, {Magnitude});
1069 Value *SignHalves =
1070 Builder.CreateIntrinsic(SplitTy, Intrinsic::dx_splitdouble, {Sign});
1071 Value *MagnitudeLow = Builder.CreateExtractValue(MagnitudeHalves, 0);
1072 Value *MagnitudeHigh = Builder.CreateExtractValue(MagnitudeHalves, 1);
1073 Value *SignHigh = Builder.CreateExtractValue(SignHalves, 1);
1074
1075 Value *CombinedHigh = CopySignBit(MagnitudeHigh, SignHigh);
1076 return Builder.CreateIntrinsic(Ty, Intrinsic::dx_asdouble,
1077 {MagnitudeLow, CombinedHigh});
1078 }
1079
1080 Value *MagnitudeInt = Builder.CreateBitCast(Magnitude, IntTy);
1081 Value *SignInt = Builder.CreateBitCast(Sign, IntTy);
1082 Value *CombinedInt = CopySignBit(MagnitudeInt, SignInt);
1083 return Builder.CreateBitCast(CombinedInt, Ty);
1084}
1085
1086// Expand llvm.matrix.multiply by extracting row/column vectors and computing
1087// dot products.
1088// Result[r,c] = dot(row_r(LHS), col_c(RHS))
1089// Element (r,c) is at index c*NumRows + r (column-major).
1091 Value *LHS = Orig->getArgOperand(0);
1092 Value *RHS = Orig->getArgOperand(1);
1093 unsigned LHSRows = cast<ConstantInt>(Orig->getArgOperand(2))->getZExtValue();
1094 unsigned LHSCols = cast<ConstantInt>(Orig->getArgOperand(3))->getZExtValue();
1095 unsigned RHSCols = cast<ConstantInt>(Orig->getArgOperand(4))->getZExtValue();
1096
1097 auto *RetTy = cast<FixedVectorType>(Orig->getType());
1098 Type *EltTy = RetTy->getElementType();
1099 bool IsFP = EltTy->isFloatingPointTy();
1100
1101 IRBuilder<> Builder(Orig);
1102
1103 // Column-major indexing:
1104 // LHS row R, element K: index = K * LHSRows + R
1105 // RHS col C, element K: index = C * LHSCols + K
1106 Value *Result = PoisonValue::get(RetTy);
1107
1108 // Extract all scalar elements from LHS and RHS once, then reuse them.
1109 unsigned LHSSize = LHSRows * LHSCols;
1110 unsigned RHSSize = LHSCols * RHSCols;
1111 SmallVector<Value *, 16> LHSElts(LHSSize);
1112 SmallVector<Value *, 16> RHSElts(RHSSize);
1113 for (unsigned I = 0; I < LHSSize; ++I)
1114 LHSElts[I] = Builder.CreateExtractElement(LHS, I);
1115 for (unsigned I = 0; I < RHSSize; ++I)
1116 RHSElts[I] = Builder.CreateExtractElement(RHS, I);
1117
1118 // Choose the appropriate scalar-arg dot intrinsic for floats.
1119 // K=1 and double types use scalar expansion instead.
1121 bool UseScalarFP = IsFP && (EltTy->isDoubleTy() || LHSCols == 1);
1122 if (IsFP && !UseScalarFP) {
1123 switch (LHSCols) {
1124 case 2:
1125 FloatDotID = Intrinsic::dx_dot2;
1126 break;
1127 case 3:
1128 FloatDotID = Intrinsic::dx_dot3;
1129 break;
1130 case 4:
1131 FloatDotID = Intrinsic::dx_dot4;
1132 break;
1133 default:
1135 "Invalid matrix inner dimension for dot product: must be 2-4");
1136 return nullptr;
1137 }
1138 }
1139
1140 for (unsigned C = 0; C < RHSCols; ++C) {
1141 for (unsigned R = 0; R < LHSRows; ++R) {
1142 // Gather row R from LHS and column C from RHS.
1143 SmallVector<Value *, 4> RowElts, ColElts;
1144 for (unsigned K = 0; K < LHSCols; ++K) {
1145 RowElts.push_back(LHSElts[K * LHSRows + R]);
1146 ColElts.push_back(RHSElts[C * LHSCols + K]);
1147 }
1148
1149 Value *Dot;
1150 if (UseScalarFP) {
1151 // Scalar fmul+fmuladd expansion for double types and K=1.
1152 Dot = Builder.CreateFMul(RowElts[0], ColElts[0]);
1153 for (unsigned K = 1; K < LHSCols; ++K)
1154 Dot = Builder.CreateIntrinsic(EltTy, Intrinsic::fmuladd,
1155 {RowElts[K], ColElts[K], Dot});
1156 } else if (IsFP) {
1157 // Emit scalar-arg DXIL dot directly (dx.dot2/dx.dot3/dx.dot4).
1159 Args.append(RowElts.begin(), RowElts.end());
1160 Args.append(ColElts.begin(), ColElts.end());
1161 Dot = Builder.CreateIntrinsic(EltTy, FloatDotID, Args);
1162 } else {
1163 // Integer: emit multiply + imad chain.
1164 Dot = Builder.CreateMul(RowElts[0], ColElts[0]);
1165 for (unsigned K = 1; K < LHSCols; ++K)
1166 Dot = Builder.CreateIntrinsic(EltTy, Intrinsic::dx_imad,
1167 {RowElts[K], ColElts[K], Dot});
1168 }
1169 unsigned ResIdx = C * LHSRows + R;
1170 Result = Builder.CreateInsertElement(Result, Dot, ResIdx);
1171 }
1172 }
1173 return Result;
1174}
1175
1176// Expand llvm.matrix.transpose as a shufflevector that permutes elements
1177// from column-major source to column-major transposed layout.
1178// Element (r,c) at index c*Rows + r moves to index r*Cols + c.
1180 Value *Mat = Orig->getArgOperand(0);
1181 unsigned Rows = cast<ConstantInt>(Orig->getArgOperand(1))->getZExtValue();
1182 unsigned Cols = cast<ConstantInt>(Orig->getArgOperand(2))->getZExtValue();
1183
1184 unsigned NumElts = Rows * Cols;
1185 SmallVector<int, 16> Mask(NumElts);
1186 for (unsigned I = 0; I < NumElts; ++I)
1187 Mask[I] = (I % Cols) * Rows + (I / Cols);
1188
1189 IRBuilder<> Builder(Orig);
1190 return Builder.CreateShuffleVector(Mat, Mask);
1191}
1192
1193// Scalarize a vector int_dx_store_output call into per-component scalar calls.
1194// The DXIL StoreOutput op is per-component; vector intrinsics are split here
1195// so that DXILOpLowering sees only scalar variants.
1196static bool expandStoreOutput(CallInst *Orig) {
1197 auto *VT = dyn_cast<FixedVectorType>(Orig->getArgOperand(3)->getType());
1198 if (!VT)
1199 return false; // already scalar, nothing to expand
1200
1201 IRBuilder<> Builder(Orig);
1202 Module *M = Orig->getModule();
1203 Type *Int8Ty = Builder.getInt8Ty();
1204 Type *Int32Ty = Builder.getInt32Ty();
1205 Type *ScalarTy = VT->getElementType();
1206 unsigned NumElems = VT->getNumElements();
1207
1208 Value *SigElementId = Orig->getArgOperand(0);
1209 Value *RowIndex = Orig->getArgOperand(1);
1210 Value *StartCol = Orig->getArgOperand(2); // i8
1211 Value *Data = Orig->getArgOperand(3);
1212 Value *StartColI32 = Builder.CreateZExt(StartCol, Int32Ty);
1213
1215 M, Intrinsic::dx_store_output, {ScalarTy});
1216
1217 for (unsigned I = 0; I < NumElems; ++I) {
1218 Value *Scalar =
1219 Builder.CreateExtractElement(Data, ConstantInt::get(Int32Ty, I));
1220 Value *ColIdx =
1221 Builder.CreateAdd(StartColI32, ConstantInt::get(Int32Ty, I));
1222 Value *ColI8 = Builder.CreateTrunc(ColIdx, Int8Ty);
1223 Builder.CreateCall(ScalarFn, {SigElementId, RowIndex, ColI8, Scalar});
1224 }
1225
1226 Orig->eraseFromParent();
1227 return true;
1228}
1229
1230// Scalarize a vector int_dx_load_input call into per-component scalar calls
1231// and reassemble the vector. The DXIL LoadInput op is per-component.
1233 auto *VT = dyn_cast<FixedVectorType>(Orig->getType());
1234 if (!VT)
1235 return nullptr; // already scalar, nothing to expand
1236
1237 IRBuilder<> Builder(Orig);
1238 Module *M = Orig->getModule();
1239 Type *Int8Ty = Builder.getInt8Ty();
1240 Type *Int32Ty = Builder.getInt32Ty();
1241 Type *ScalarTy = VT->getElementType();
1242 unsigned NumElems = VT->getNumElements();
1243
1244 Value *SigElementId = Orig->getArgOperand(0);
1245 Value *RowIndex = Orig->getArgOperand(1);
1246 Value *StartCol = Orig->getArgOperand(2); // i8
1247 Value *GsVertexOrPrimIndex = Orig->getArgOperand(3);
1248 Value *StartColI32 = Builder.CreateZExt(StartCol, Int32Ty);
1249
1251 M, Intrinsic::dx_load_input, {ScalarTy});
1252
1253 Value *Vec = PoisonValue::get(VT);
1254 for (unsigned I = 0; I < NumElems; ++I) {
1255 Value *ColIdx =
1256 Builder.CreateAdd(StartColI32, ConstantInt::get(Int32Ty, I));
1257 Value *ColI8 = Builder.CreateTrunc(ColIdx, Int8Ty);
1258 Value *Scalar = Builder.CreateCall(
1259 ScalarFn, {SigElementId, RowIndex, ColI8, GsVertexOrPrimIndex});
1260 Vec =
1261 Builder.CreateInsertElement(Vec, Scalar, ConstantInt::get(Int32Ty, I));
1262 }
1263
1264 return Vec;
1265}
1266
1268 IRBuilder<> Builder(Orig);
1270 Orig->getOperandBundlesAsDefs(Bundles);
1271
1272 Value *Cond =
1273 Builder.CreateSExt(Orig->getArgOperand(0), Builder.getInt32Ty());
1274 Value *Result = Cond;
1275 for (Intrinsic::ID ID :
1276 {Intrinsic::dx_quad_read_across_x, Intrinsic::dx_quad_read_across_y,
1277 Intrinsic::dx_quad_read_across_diagonal}) {
1278 Value *QuadValue = Builder.CreateIntrinsic(ID, {Builder.getInt32Ty()}, Cond,
1279 nullptr, "", Bundles);
1280 Result = Builder.CreateAnd(Result, QuadValue);
1281 }
1282 return Builder.CreateTrunc(Result, Builder.getInt1Ty());
1283}
1284
1285static bool expandIntrinsic(Function &F, CallInst *Orig) {
1286 Value *Result = nullptr;
1287 Intrinsic::ID IntrinsicId = F.getIntrinsicID();
1288 switch (IntrinsicId) {
1289 case Intrinsic::abs:
1290 Result = expandAbs(Orig);
1291 break;
1292 case Intrinsic::assume:
1293 Orig->eraseFromParent();
1294 return true;
1295 case Intrinsic::atan2:
1296 Result = expandAtan2Intrinsic(Orig);
1297 break;
1298 case Intrinsic::copysign:
1299 Result = expandCopySignIntrinsic(Orig);
1300 break;
1301 case Intrinsic::fshl:
1302 Result = expandFunnelShiftIntrinsic<true>(Orig);
1303 break;
1304 case Intrinsic::fshr:
1305 Result = expandFunnelShiftIntrinsic<false>(Orig);
1306 break;
1307 case Intrinsic::exp:
1308 Result = expandExpIntrinsic(Orig);
1309 break;
1310 case Intrinsic::is_fpclass:
1311 Result = expandIsFPClass(Orig);
1312 break;
1313 case Intrinsic::log:
1314 Result = expandLogIntrinsic(Orig);
1315 break;
1316 case Intrinsic::log10:
1317 Result = expandLog10Intrinsic(Orig);
1318 break;
1319 case Intrinsic::pow:
1320 case Intrinsic::powi:
1321 Result = expandPowIntrinsic(Orig, IntrinsicId);
1322 break;
1323 case Intrinsic::vector_reduce_and:
1324 case Intrinsic::vector_reduce_or:
1325 Result = expandVecReduceAndOr(Orig, IntrinsicId);
1326 break;
1327 case Intrinsic::dx_uclamp:
1328 case Intrinsic::dx_sclamp:
1329 case Intrinsic::dx_nclamp:
1330 Result = expandClampIntrinsic(Orig, IntrinsicId);
1331 break;
1332 case Intrinsic::dx_isfinite:
1333 Result = expand16BitIsFinite(Orig);
1334 break;
1335 case Intrinsic::dx_isinf:
1336 Result = expand16BitIsInf(Orig);
1337 break;
1338 case Intrinsic::dx_isnan:
1339 Result = expand16BitIsNaN(Orig);
1340 break;
1341 case Intrinsic::dx_fdot:
1342 Result = expandFloatDotIntrinsic(Orig);
1343 break;
1344 case Intrinsic::dx_sdot:
1345 case Intrinsic::dx_udot:
1346 Result = expandIntegerDotIntrinsic(Orig, IntrinsicId);
1347 break;
1348 case Intrinsic::dx_sign:
1349 Result = expandSignIntrinsic(Orig);
1350 break;
1351 case Intrinsic::dx_quad_all:
1352 Result = expandQuadAll(Orig);
1353 break;
1354 case Intrinsic::dx_load_input:
1355 Result = expandLoadInput(Orig);
1356 break;
1357 case Intrinsic::dx_store_output:
1358 if (expandStoreOutput(Orig))
1359 return true;
1360 break;
1361 case Intrinsic::dx_resource_load_rawbuffer:
1362 if (expandBufferLoadIntrinsic(Orig, /*IsRaw*/ true))
1363 return true;
1364 break;
1365 case Intrinsic::dx_resource_store_rawbuffer:
1366 if (expandBufferStoreIntrinsic(Orig, /*IsRaw*/ true))
1367 return true;
1368 break;
1369 case Intrinsic::dx_resource_load_typedbuffer:
1370 if (expandBufferLoadIntrinsic(Orig, /*IsRaw*/ false))
1371 return true;
1372 break;
1373 case Intrinsic::dx_resource_store_typedbuffer:
1374 if (expandBufferStoreIntrinsic(Orig, /*IsRaw*/ false))
1375 return true;
1376 break;
1377 case Intrinsic::usub_sat:
1378 Result = expandUsubSat(Orig);
1379 break;
1380 case Intrinsic::umul_with_overflow:
1381 case Intrinsic::smul_with_overflow:
1382 Result = expandMulWithOverflow(Orig, /*Signed=*/IntrinsicId ==
1383 Intrinsic::smul_with_overflow);
1384 break;
1385 case Intrinsic::vector_reduce_add:
1386 case Intrinsic::vector_reduce_fadd:
1387 Result = expandVecReduceAdd(Orig, IntrinsicId);
1388 break;
1389 case Intrinsic::matrix_multiply:
1390 Result = expandMatrixMultiply(Orig);
1391 break;
1392 case Intrinsic::matrix_transpose:
1393 Result = expandMatrixTranspose(Orig);
1394 break;
1395 }
1396 if (Result) {
1397 Orig->replaceAllUsesWith(Result);
1398 Orig->eraseFromParent();
1399 return true;
1400 }
1401 return false;
1402}
1403
1405 for (auto &F : make_early_inc_range(M.functions())) {
1406 if (!isIntrinsicExpansion(F))
1407 continue;
1408 bool IntrinsicExpanded = false;
1409 for (User *U : make_early_inc_range(F.users())) {
1410 auto *IntrinsicCall = dyn_cast<CallInst>(U);
1411 if (!IntrinsicCall)
1412 continue;
1413 IntrinsicExpanded = expandIntrinsic(F, IntrinsicCall);
1414 }
1415 if (F.user_empty() && IntrinsicExpanded)
1416 F.eraseFromParent();
1417 }
1418 return true;
1419}
1420
1427
1431
1433
1435 "DXIL Intrinsic Expansion", false, false)
1437 "DXIL Intrinsic Expansion", false, false)
1438
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
This file implements a class to represent arbitrary precision integral constant values and operations...
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static Value * expand16BitIsNormal(CallInst *Orig)
static Value * createMulHighUnsigned(IRBuilder<> &Builder, Value *A, Value *B, Type *Ty, unsigned BW)
static bool expandIntrinsic(Function &F, CallInst *Orig)
static Value * expandClampIntrinsic(CallInst *Orig, Intrinsic::ID ClampIntrinsic)
static Value * expand16BitIsInf(CallInst *Orig)
static bool expansionIntrinsics(Module &M)
static Value * expandCopySignIntrinsic(CallInst *Orig)
static Value * expand16BitIsFinite(CallInst *Orig)
static Value * expandLoadInput(CallInst *Orig)
static Value * expandUsubSat(CallInst *Orig)
static Value * expandFloatDotIntrinsic(CallInst *Orig)
static bool expandStoreOutput(CallInst *Orig)
static Value * expandMatrixTranspose(CallInst *Orig)
static Value * expandVecReduceAdd(CallInst *Orig, Intrinsic::ID IntrinsicId)
static Value * expandAtan2Intrinsic(CallInst *Orig)
static Value * expandQuadAll(CallInst *Orig)
static Value * expandLog10Intrinsic(CallInst *Orig)
static Intrinsic::ID getMinForClamp(Intrinsic::ID ClampIntrinsic)
static Value * expandIntegerDotIntrinsic(CallInst *Orig, Intrinsic::ID DotIntrinsic)
static Value * expandVecReduceAndOr(CallInst *Orig, Intrinsic::ID IntrinsicId)
static bool expandBufferStoreIntrinsic(CallInst *Orig, bool IsRaw)
static Value * expandLogIntrinsic(CallInst *Orig, float LogConstVal=numbers::ln2f)
static Value * expandMulWithOverflow(CallInst *Orig, bool Signed)
static Value * expandPowIntrinsic(CallInst *Orig, Intrinsic::ID IntrinsicId)
static bool resourceAccessNeeds64BitExpansion(Module *M, Type *OverloadTy, bool IsRaw)
static Value * expandExpIntrinsic(CallInst *Orig)
static Value * expand16BitIsNaN(CallInst *Orig)
static Value * expandSignIntrinsic(CallInst *Orig)
static Intrinsic::ID getMaxForClamp(Intrinsic::ID ClampIntrinsic)
static bool shouldExpandFloatDotIntrinsic(Function &F)
static Value * expandFloatDotChunk(CallInst *Orig, Value *A, Value *B)
static Value * expandAbs(CallInst *Orig)
static bool isIntrinsicExpansion(Function &F)
static bool expandBufferLoadIntrinsic(CallInst *Orig, bool IsRaw)
static Value * expandMatrixMultiply(CallInst *Orig)
static Value * expandIsFPClass(CallInst *Orig)
static Value * expandFunnelShiftIntrinsic(CallInst *Orig)
#define DEBUG_TYPE
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define T
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
SmallVector< MachineOperand, 4 > Cond
MachineInstr *& Lo
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
Value * RHS
Value * LHS
BinaryOperator * Mul
bool runOnModule(Module &M) override
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition APInt.h:225
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
void setAttributes(AttributeList A)
Set the attributes for this call.
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
AttributeList getAttributes() const
Return the attributes for this call.
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
void setTailCall(bool IsTc=true)
static LLVM_ABI Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
This is an important base class in LLVM.
Definition Constant.h:44
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition Constant.h:65
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &)
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
Type * getParamType(unsigned i) const
Parameter type accessors.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
iterator_range< user_iterator > users()
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition Pass.h:253
ModulePass(char &pid)
Definition Pass.h:255
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
const Triple & getTargetTriple() const
Get the target triple which is a string describing the target host.
Definition Module.h:328
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition Analysis.h:115
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:462
LLVM_ABI VersionTuple getOSVersion() const
Parse the version number from the OS name component of the triple, if present.
Definition Triple.cpp:1493
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
LLVM_ABI Type * getStructElementType(unsigned N) const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
Definition Type.cpp:294
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:292
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
LLVM_ABI Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
Definition Type.cpp:293
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
Definition Type.h:144
LLVM_ABI Type * getWithNewType(Type *EltTy) const
Given vector type, change the element type, whilst keeping the old number of elements.
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
Definition Type.h:158
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
Value * getOperand(unsigned i) const
Definition User.h:207
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Represents a version number in the form major[.minor[.subminor[.build]]].
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
constexpr float ln10f
Definition MathExtras.h:51
constexpr float log2ef
Definition MathExtras.h:52
constexpr double pi
constexpr float ln2f
Definition MathExtras.h:50
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Load
The value being inserted comes from a load (InsertElement only).
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
ModulePass * createDXILIntrinsicExpansionLegacyPass()
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
@ Sub
Subtraction of integers.
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
Definition MIRParser.h:39
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177