21#include "llvm/IR/IntrinsicsDirectX.h"
27#define DEBUG_TYPE "dxil-resource-access"
37 assert(
II->getIntrinsicID() == Intrinsic::dx_resource_getpointer &&
38 "Resource access through unexpected intrinsic");
39 return Offset ?
Offset : ConstantInt::get(Builder.getInt32Ty(), 0);
43 assert(
GEP &&
"Resource access through unexpected instruction");
45 unsigned NumIndices =
GEP->getNumIndices();
46 uint64_t IndexScale =
DL.getTypeAllocSize(
GEP->getSourceElementType());
47 APInt ConstantOffset(
DL.getIndexTypeSizeInBits(
GEP->getType()), 0);
49 if (
GEP->accumulateConstantOffset(
DL, ConstantOffset)) {
52 ConstantInt::get(
DL.getIndexType(
GEP->getType()), ConstantOffset);
54 }
else if (NumIndices == 1) {
57 GEPOffset = *
GEP->idx_begin();
58 }
else if (NumIndices == 2) {
60 auto IndexIt =
GEP->idx_begin();
62 "GEP is not indexing through pointer");
63 GEPOffset = *(++IndexIt);
68 if (!(IndexScale % ElemSize)) {
71 IndexScale /= ElemSize;
75 GEPOffset = Builder.CreateMul(
76 GEPOffset, ConstantInt::get(Builder.getInt32Ty(), IndexScale));
78 GEPOffset = Builder.CreateUDiv(
79 GEPOffset, ConstantInt::get(Builder.getInt32Ty(), ElemSize));
82 Ptr =
GEP->getPointerOperand();
96 Value *V =
SI->getValueOperand();
97 if (V->getType() == ContainedType) {
100 "Store of whole element has mismatched address to store to");
101 }
else if (V->getType() == ScalarType) {
104 auto *Load = Builder.CreateIntrinsic(
105 LoadType, Intrinsic::dx_resource_load_typedbuffer,
106 {
II->getOperand(0),
II->getOperand(1)});
107 auto *Struct = Builder.CreateExtractValue(Load, {0});
109 uint64_t AccessSize =
DL.getTypeSizeInBits(ScalarType) / 8;
112 V = Builder.CreateInsertElement(Struct, V,
Offset);
117 auto *Inst = Builder.CreateIntrinsic(
118 Builder.getVoidTy(), Intrinsic::dx_resource_store_typedbuffer,
119 {II->getOperand(0), II->getOperand(1), V});
120 SI->replaceAllUsesWith(Inst);
127 Value *V =
SI->getValueOperand();
133 auto *Inst = Builder.CreateIntrinsic(
134 Builder.getVoidTy(), Intrinsic::dx_resource_store_rawbuffer,
135 {II->getOperand(0), II->getOperand(1), Offset, V});
136 SI->replaceAllUsesWith(Inst);
179 Builder.CreateIntrinsic(LoadType, Intrinsic::dx_resource_load_typedbuffer,
180 {
II->getOperand(0),
II->getOperand(1)});
181 V = Builder.CreateExtractValue(V, {0});
184 uint64_t AccessSize =
DL.getTypeSizeInBits(ScalarType) / 8;
188 if (!ConstantOffset || !ConstantOffset->isZero())
189 V = Builder.CreateExtractElement(V,
Offset);
196 Builder.getInt32(0));
213 Builder.CreateIntrinsic(LoadType, Intrinsic::dx_resource_load_rawbuffer,
214 {
II->getOperand(0),
II->getOperand(1),
Offset});
215 V = Builder.CreateExtractValue(V, {0});
222struct CBufferRowIntrin {
225 unsigned int EltSize;
226 unsigned int NumElts;
228 CBufferRowIntrin(
const DataLayout &
DL,
Type *Ty) {
231 switch (
DL.getTypeSizeInBits(Ty)) {
233 IID = Intrinsic::dx_resource_load_cbufferrow_8;
239 IID = Intrinsic::dx_resource_load_cbufferrow_4;
245 IID = Intrinsic::dx_resource_load_cbufferrow_2;
263 CBufferRowIntrin Intrin(
DL, Ty->getScalarType());
266 Value *Handle =
II->getOperand(0);
271 assert(GlobalOffset &&
"CBuffer getpointer index must be constant");
274 Value *CurrentRow = ConstantInt::get(
276 unsigned int CurrentIndex;
285 "Unexpected indirect access to resource without GEP");
291 CurrentRow = Builder.CreateAdd(GEPOffset, CurrentRow);
293 APInt ConstantOffset(
DL.getIndexTypeSizeInBits(LastGEP->getType()), 0);
294 if (LastGEP->accumulateConstantOffset(
DL, ConstantOffset)) {
295 APInt Remainder(
DL.getIndexTypeSizeInBits(LastGEP->getType()),
297 APInt::udivrem(ConstantOffset, Remainder, ConstantOffset, Remainder);
298 CurrentRow = Builder.CreateAdd(
299 CurrentRow, ConstantInt::get(Builder.getInt32Ty(), ConstantOffset));
302 assert(LastGEP->getNumIndices() == 1 &&
303 "Last GEP of cbuffer access is not array or struct access");
309 ? *LastGEP->idx_begin()
310 : Builder.CreateAdd(CurrentRow, *LastGEP->idx_begin());
315 auto *CBufLoad = Builder.CreateIntrinsic(
316 Intrin.RetTy, Intrin.IID, {Handle, CurrentRow},
nullptr, Name +
".load");
318 Builder.CreateExtractValue(CBufLoad, {CurrentIndex++}, Name +
".extract");
322 unsigned int Remaining =
323 ((
DL.getTypeSizeInBits(Ty) / 8) / Intrin.EltSize) - 1;
324 if (Remaining == 0) {
330 assert(VT->getNumElements() == 1 &&
"Can't have multiple elements here");
332 Builder.getInt32(0), Name);
340 while (Remaining--) {
341 CurrentIndex %= Intrin.NumElts;
343 if (CurrentIndex == 0) {
344 CurrentRow = Builder.CreateAdd(CurrentRow,
345 ConstantInt::get(Builder.getInt32Ty(), 1));
346 CBufLoad = Builder.CreateIntrinsic(Intrin.RetTy, Intrin.IID,
347 {Handle, CurrentRow},
nullptr,
351 Extracts.
push_back(Builder.CreateExtractValue(CBufLoad, {CurrentIndex++},
357 for (
int I = 0,
E = Extracts.
size();
I <
E; ++
I)
358 Result = Builder.CreateInsertElement(
359 Result, Extracts[
I], Builder.getInt32(
I), Name +
formatv(
".upto{}",
I));
400 auto *BB = Start->getParent();
403 for (
User *U : Start->users()) {
405 if (
I->getParent() == BB)
411 while (!Worklist.
empty()) {
417 for (
User *U :
I->users()) {
419 if (J->getParent() == BB)
423 for (
Use &V :
I->operands()) {
425 if (J->getParent() == BB && V != Start)
449 Value *Val = Phi->getIncomingValueForBlock(BB);
451 Builder.SetInsertPoint(&BB->
back());
455 VMap[PhiNested] = PhiNested->getIncomingValueForBlock(BB);
461 Builder.Insert(Clone);
470 for (
User *U :
II->users()) {
479 for (
unsigned I = 0,
E = Phi->getNumIncomingValues();
I <
E;
I++) {
480 auto *CurrIncomingBB = Phi->getIncomingBlock(
I);
483 PrevBBDeadInsts.
push_back(&CurrIncomingBB->back());
498 Dead->eraseFromParent();
499 CurrBBDeadInsts.
clear();
504 for (
User *U :
II->users())
508 while (!Worklist.
empty()) {
518 assert(
SI->getValueOperand() !=
II &&
"Pointer escaped!");
531 Dead->eraseFromParent();
532 II->eraseFromParent();
542 if (
II->getIntrinsicID() == Intrinsic::dx_resource_getpointer)
547 if (
II->getIntrinsicID() == Intrinsic::dx_resource_getpointer) {
552 for (
auto *Dead : PrevBBDeadInsts)
553 Dead->eraseFromParent();
554 PrevBBDeadInsts.
clear();
555 for (
auto *Dead : DeadBB)
556 Dead->eraseFromParent();
559 for (
auto &[
II, RI] : Resources)
562 return !Resources.
empty();
570 assert(DRTM &&
"DXILResourceTypeAnalysis must be available");
587 getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();
590 StringRef getPassName()
const override {
return "DXIL Resource Access"; }
591 DXILResourceAccessLegacy() : FunctionPass(
ID) {}
594 void getAnalysisUsage(llvm::AnalysisUsage &AU)
const override {
599char DXILResourceAccessLegacy::ID = 0;
603 "DXIL Resource Access",
false,
false)
609 return new DXILResourceAccessLegacy();
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static void phiNodeRemapHelper(PHINode *Phi, BasicBlock *BB, IRBuilder<> &Builder, SmallVector< Instruction * > &UsesInBlock)
static void createLoadIntrinsic(IntrinsicInst *II, LoadInst *LI, dxil::ResourceTypeInfo &RTI)
static void createRawStore(IntrinsicInst *II, StoreInst *SI)
static void createTypedBufferLoad(IntrinsicInst *II, LoadInst *LI, dxil::ResourceTypeInfo &RTI)
static void createTypedBufferStore(IntrinsicInst *II, StoreInst *SI, dxil::ResourceTypeInfo &RTI)
static bool transformResourcePointers(Function &F, DXILResourceTypeMap &DRTM)
static Value * traverseGEPOffsets(const DataLayout &DL, IRBuilder<> &Builder, Value *Ptr, uint64_t AccessSize)
static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI, dxil::ResourceTypeInfo &RTI)
static void createCBufferLoad(IntrinsicInst *II, LoadInst *LI, dxil::ResourceTypeInfo &RTI)
static void createRawLoad(IntrinsicInst *II, LoadInst *LI)
static SmallVector< Instruction * > collectBlockUseDef(Instruction *Start)
static void phiNodeReplacement(IntrinsicInst *II, SmallVectorImpl< Instruction * > &PrevBBDeadInsts, SetVector< BasicBlock * > &DeadBB)
static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI)
static bool runOnFunction(Function &F, bool PostInlining)
uint64_t IntrinsicInst * II
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file implements a set that has insertion order iteration characteristics.
Class for arbitrary precision integers.
LLVM_ABI APInt udiv(const APInt &RHS) const
Unsigned division operation.
static LLVM_ABI void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
uint64_t getZExtValue() const
Get zero extended value.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
LLVM Basic Block Representation.
const Instruction & back() const
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
A parsed version of the target data layout string in and methods for querying it.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Analysis pass which computes a DominatorTree.
FunctionPass class - This class is used to implement most global optimizations.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
A wrapper class for inspecting calls to intrinsic functions.
An instruction for reading from memory.
Value * getPointerOperand()
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses & preserve()
Mark an analysis as preserved.
A vector that has set insertion semantics.
void clear()
Completely clear the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Type * getTypeParameter(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
TargetExtType * getHandleTy() const
dxil::ResourceKind getResourceKind() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ RTAccelerationStructure
const unsigned CBufferRowSizeInBytes
This is an optimization pass for GlobalISel generic memory operations.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
auto formatv(bool Validate, const char *Fmt, Ts &&...Vals)
auto reverse(ContainerTy &&C)
FunctionPass * createDXILResourceAccessLegacyPass()
Pass to update resource accesses to use load/store directly.
void sort(IteratorTy Start, IteratorTy End)
@ RF_IgnoreMissingLocals
If this flag is set, the remapper ignores missing function-local entries (Argument,...
@ RF_NoModuleLevelChanges
If this flag is set, the remapper knows that only local values within a function (such as an instruct...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr, const MetadataPredicate *IdentityMD=nullptr)
Convert the instruction operands from referencing the current values into those specified by VM.
ValueMap< const Value *, WeakTrackingVH > ValueToValueMapTy
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.