28constexpr unsigned CudaFatMagic = 0x466243b1;
29constexpr unsigned HIPFatMagic = 0x48495046;
32 return M.getDataLayout().getIntPtrType(
M.getContext());
47 PointerType::getUnqual(
C), PointerType::getUnqual(
C),
48 PointerType::getUnqual(
C));
53 return PointerType::getUnqual(
M.getContext());
68 PointerType::getUnqual(
C), PointerType::getUnqual(
C));
73 return PointerType::getUnqual(
M.getContext());
118 auto [EntriesB, EntriesE] = EntryArray;
125 ImagesInits.
reserve(Bufs.size());
130 GlobalVariable::InternalLinkage, Data,
131 ".omp_offloading.device_image" + Suffix);
132 Image->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
133 Image->setSection(Relocatable ?
".llvm.offloading.relocatable"
134 :
".llvm.offloading");
139 "Invalid binary format");
148 Binary.bytes_begin() + Header->EntryOffset);
162 ImageE, EntriesB, EntriesE));
167 ArrayType::get(getDeviceImageTy(M), ImagesInits.
size()), ImagesInits);
172 ".omp_offloading.device_images" + Suffix);
173 Images->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
186 ".omp_offloading.descriptor" + Suffix);
195 ".omp_offloading.descriptor_unreg" + Suffix, &M);
196 Func->setSection(
".text.startup");
199 auto *UnRegFuncTy = FunctionType::get(
Type::getVoidTy(
C), getBinDescPtrTy(M),
202 M.getOrInsertFunction(
"__tgt_unregister_lib", UnRegFuncTy);
206 Builder.CreateCall(UnRegFuncC, BinDesc);
207 Builder.CreateRetVoid();
217 ".omp_offloading.descriptor_reg" + Suffix, &M);
218 Func->setSection(
".text.startup");
224 M.getOrInsertFunction(
"__tgt_register_lib", RegFuncTy);
226 auto *AtExitTy = FunctionType::get(
230 Function *UnregFunc = createUnregisterFunction(M, BinDesc, Suffix);
235 Builder.CreateCall(RegFuncC, BinDesc);
241 Builder.CreateCall(AtExit, UnregFunc);
242 Builder.CreateRetVoid();
260 PointerType::getUnqual(
C), PointerType::getUnqual(
C));
274 IsHIP ?
".hip_fatbin"
278 GlobalVariable::InternalLinkage, Data,
279 ".fatbin_image" + Suffix);
280 Fatbin->setSection(FatbinConstantSection);
283 StringRef FatbinWrapperSection = IsHIP ?
".hipFatBinSegment"
285 :
".nvFatBinSegment";
298 FatbinInitializer,
".fatbin_wrapper" + Suffix);
299 FatbinDesc->setSection(FatbinWrapperSection);
300 FatbinDesc->setAlignment(
Align(8));
334 bool EmitSurfacesAndTextures) {
336 auto [EntriesB, EntriesE] = EntryArray;
342 auto *RegFuncTy = FunctionType::get(
345 Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int32PtrTy},
348 IsHIP ?
"__hipRegisterFunction" :
"__cudaRegisterFunction", RegFuncTy);
351 auto *RegVarTy = FunctionType::get(
357 IsHIP ?
"__hipRegisterVar" :
"__cudaRegisterVar", RegVarTy);
362 {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,
366 IsHIP ?
"__hipRegisterManagedVar" :
"__cudaRegisterManagedVar",
372 {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,
376 IsHIP ?
"__hipRegisterSurface" :
"__cudaRegisterSurface", RegSurfaceTy);
385 IsHIP ?
"__hipRegisterTexture" :
"__cudaRegisterTexture", RegTextureTy);
391 IsHIP ?
".hip.globals_reg" :
".cuda.globals_reg", &M);
392 RegGlobalsFn->setSection(
".text.startup");
407 auto *EntryCmp = Builder.CreateICmpNE(EntriesB, EntriesE);
408 Builder.CreateCondBr(EntryCmp, EntryBB, ExitBB);
409 Builder.SetInsertPoint(EntryBB);
410 auto *
Entry = Builder.CreatePHI(PointerType::getUnqual(
C), 2,
"entry");
415 auto *
Addr = Builder.CreateLoad(Int8PtrTy, AddrPtr,
"addr");
420 auto *AuxAddr = Builder.CreateLoad(Int8PtrTy, AuxAddrPtr,
"aux_addr");
430 auto *
Name = Builder.CreateLoad(Int8PtrTy, NamePtr,
"name");
445 auto *Data = Builder.CreateTrunc(
448 auto *
Type = Builder.CreateAnd(
452 auto *ExternBit = Builder.CreateAnd(
455 auto *
Extern = Builder.CreateLShr(
460 auto *
Const = Builder.CreateLShr(
462 auto *NormalizedBit = Builder.CreateAnd(
465 auto *Normalized = Builder.CreateLShr(
467 auto *KindCond = Builder.CreateICmpEQ(
469 IsHIP ? object::OffloadKind::OFK_HIP
470 : object::OffloadKind::OFK_Cuda));
471 Builder.CreateCondBr(KindCond, IfKindBB, IfEndBB);
472 Builder.SetInsertPoint(IfKindBB);
473 auto *FnCond = Builder.CreateICmpEQ(
475 Builder.CreateCondBr(FnCond, IfThenBB, IfElseBB);
478 Builder.SetInsertPoint(IfThenBB);
479 Builder.CreateCall(RegFunc, {RegGlobalsFn->arg_begin(),
Addr,
Name,
Name,
486 Builder.CreateBr(IfEndBB);
487 Builder.SetInsertPoint(IfElseBB);
489 auto *
Switch = Builder.CreateSwitch(
Type, IfEndBB);
491 Builder.SetInsertPoint(SwGlobalBB);
492 Builder.CreateCall(RegVar,
495 Builder.CreateBr(IfEndBB);
500 Builder.SetInsertPoint(SwManagedBB);
501 Builder.CreateCall(RegManagedVar, {RegGlobalsFn->arg_begin(), AuxAddr,
Addr,
503 Builder.CreateBr(IfEndBB);
507 Builder.SetInsertPoint(SwSurfaceBB);
508 if (EmitSurfacesAndTextures)
509 Builder.CreateCall(RegSurface, {RegGlobalsFn->arg_begin(),
Addr,
Name,
Name,
511 Builder.CreateBr(IfEndBB);
516 Builder.SetInsertPoint(SwTextureBB);
517 if (EmitSurfacesAndTextures)
518 Builder.CreateCall(RegTexture, {RegGlobalsFn->arg_begin(),
Addr,
Name,
Name,
519 Data, Normalized,
Extern});
520 Builder.CreateBr(IfEndBB);
524 Builder.SetInsertPoint(IfEndBB);
525 auto *NewEntry = Builder.CreateInBoundsGEP(
527 auto *
Cmp = Builder.CreateICmpEQ(
538 &RegGlobalsFn->getEntryBlock());
539 Entry->addIncoming(NewEntry, IfEndBB);
540 Builder.CreateCondBr(Cmp, ExitBB, EntryBB);
541 Builder.SetInsertPoint(ExitBB);
542 Builder.CreateRetVoid();
552 bool EmitSurfacesAndTextures) {
557 (IsHIP ?
".hip.fatbin_reg" :
".cuda.fatbin_reg") + Suffix, &M);
558 CtorFunc->setSection(
".text.startup");
563 (IsHIP ?
".hip.fatbin_unreg" :
".cuda.fatbin_unreg") + Suffix, &M);
564 DtorFunc->setSection(
".text.startup");
566 auto *PtrTy = PointerType::getUnqual(
C);
569 auto *RegFatTy = FunctionType::get(PtrTy, PtrTy,
false);
571 IsHIP ?
"__hipRegisterFatBinary" :
"__cudaRegisterFatBinary", RegFatTy);
576 M.getOrInsertFunction(
"__cudaRegisterFatBinaryEnd", RegFatEndTy);
581 IsHIP ?
"__hipUnregisterFatBinary" :
"__cudaUnregisterFatBinary",
591 (IsHIP ?
".hip.binary_handle" :
".cuda.binary_handle") + Suffix);
595 CallInst *Handle = CtorBuilder.CreateCall(
598 CtorBuilder.CreateAlignedStore(
599 Handle, BinaryHandleGlobal,
600 Align(
M.getDataLayout().getPointerTypeSize(PtrTy)));
601 CtorBuilder.CreateCall(createRegisterGlobalsFunction(M, IsHIP, EntryArray,
603 EmitSurfacesAndTextures),
606 CtorBuilder.CreateCall(RegFatbinEnd, Handle);
607 CtorBuilder.CreateCall(AtExit, DtorFunc);
608 CtorBuilder.CreateRetVoid();
614 LoadInst *BinaryHandle = DtorBuilder.CreateAlignedLoad(
615 PtrTy, BinaryHandleGlobal,
616 Align(
M.getDataLayout().getPointerTypeSize(PtrTy)));
617 DtorBuilder.CreateCall(UnregFatbin, BinaryHandle);
618 DtorBuilder.CreateRetVoid();
629 createBinDesc(M, Images, EntryArray, Suffix, Relocatable);
632 "No binary descriptors created.");
633 createRegisterFunction(M,
Desc, Suffix);
640 bool EmitSurfacesAndTextures) {
644 "No fatbin section created.");
646 createRegisterFatbinFunction(M,
Desc,
false, EntryArray, Suffix,
647 EmitSurfacesAndTextures);
653 bool EmitSurfacesAndTextures) {
657 "No fatbin section created.");
659 createRegisterFatbinFunction(M,
Desc,
true, EntryArray, Suffix,
660 EmitSurfacesAndTextures);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static IntegerType * getSizeTTy(IRBuilderBase &B, const TargetLibraryInfo *TLI)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Module.h This file contains the declarations for the Module class.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
This class represents a function call, abstracting a target machine's calling convention.
static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
static Constant * getInBoundsGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList)
Create an "inbounds" getelementptr.
static LLVM_ABI Constant * getPointerBitCastOrAddrSpaceCast(Constant *C, Type *Ty)
Create a BitCast or AddrSpaceCast for a pointer type depending on the address space.
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)
This is an important base class in LLVM.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
@ InternalLinkage
Rename collisions when linking (static functions).
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Class to represent integer types.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
A Module instance is used to store all the information related to an LLVM module.
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
Class to represent struct types.
static LLVM_ABI StructType * getTypeByName(LLVMContext &C, StringRef Name)
Return the type with the specified name, or null if there is none by that name.
static LLVM_ABI StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Triple - Helper class for working with autoconf configuration names.
bool isMacOSX() const
Is this a Mac OS X triple.
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
static uint64_t getAlignment()
@ C
The default llvm calling convention, compatible with C.
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
LLVM_ABI StructType * getEntryTy(Module &M)
Returns the type of the offloading entry we use to store kernels and globals that will be registered ...
@ OffloadGlobalSurfaceEntry
Mark the entry as a surface variable.
@ OffloadGlobalTextureEntry
Mark the entry as a texture variable.
@ OffloadGlobalNormalized
Mark the entry as being a normalized surface.
@ OffloadGlobalEntry
Mark the entry as a global entry.
@ OffloadGlobalManagedEntry
Mark the entry as a managed global variable.
@ OffloadGlobalExtern
Mark the entry as being extern.
@ OffloadGlobalConstant
Mark the entry as being constant.
LLVM_ABI llvm::Error wrapOpenMPBinaries(llvm::Module &M, llvm::ArrayRef< llvm::ArrayRef< char > > Images, EntryArrayTy EntryArray, llvm::StringRef Suffix="", bool Relocatable=false)
Wraps the input device images into the module M as global symbols and registers the images with the O...
std::pair< GlobalVariable *, GlobalVariable * > EntryArrayTy
LLVM_ABI llvm::Error wrapHIPBinary(llvm::Module &M, llvm::ArrayRef< char > Images, EntryArrayTy EntryArray, llvm::StringRef Suffix="", bool EmitSurfacesAndTextures=true)
Wraps the input bundled image into the module M as global symbols and registers the images with the H...
LLVM_ABI llvm::Error wrapCudaBinary(llvm::Module &M, llvm::ArrayRef< char > Images, EntryArrayTy EntryArray, llvm::StringRef Suffix="", bool EmitSurfacesAndTextures=true)
Wraps the input fatbinary image into the module M as global symbols and registers the images with the...
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI file_magic identify_magic(StringRef magic)
Identify the type of a binary file based on how magical it is.
LLVM_ABI std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
LLVM_ABI void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
@ Extern
Replace returns with jump to thunk, don't emit thunk.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Description of the encoding of one expression Op.
@ offload_binary
LLVM offload object file.