LLVM 20.0.0git
OffloadWrapper.cpp
Go to the documentation of this file.
1//===- OffloadWrapper.cpp ---------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10#include "llvm/ADT/ArrayRef.h"
13#include "llvm/IR/Constants.h"
15#include "llvm/IR/IRBuilder.h"
16#include "llvm/IR/LLVMContext.h"
17#include "llvm/IR/Module.h"
19#include "llvm/Support/Error.h"
22
23using namespace llvm;
24using namespace llvm::offloading;
25
26namespace {
27/// Magic number that begins the section containing the CUDA fatbinary.
28constexpr unsigned CudaFatMagic = 0x466243b1;
29constexpr unsigned HIPFatMagic = 0x48495046;
30
32 return M.getDataLayout().getIntPtrType(M.getContext());
33}
34
35// struct __tgt_device_image {
36// void *ImageStart;
37// void *ImageEnd;
38// __tgt_offload_entry *EntriesBegin;
39// __tgt_offload_entry *EntriesEnd;
40// };
41StructType *getDeviceImageTy(Module &M) {
42 LLVMContext &C = M.getContext();
43 StructType *ImageTy = StructType::getTypeByName(C, "__tgt_device_image");
44 if (!ImageTy)
45 ImageTy =
46 StructType::create("__tgt_device_image", PointerType::getUnqual(C),
47 PointerType::getUnqual(C), PointerType::getUnqual(C),
48 PointerType::getUnqual(C));
49 return ImageTy;
50}
51
52PointerType *getDeviceImagePtrTy(Module &M) {
53 return PointerType::getUnqual(getDeviceImageTy(M));
54}
55
56// struct __tgt_bin_desc {
57// int32_t NumDeviceImages;
58// __tgt_device_image *DeviceImages;
59// __tgt_offload_entry *HostEntriesBegin;
60// __tgt_offload_entry *HostEntriesEnd;
61// };
62StructType *getBinDescTy(Module &M) {
63 LLVMContext &C = M.getContext();
64 StructType *DescTy = StructType::getTypeByName(C, "__tgt_bin_desc");
65 if (!DescTy)
66 DescTy = StructType::create(
67 "__tgt_bin_desc", Type::getInt32Ty(C), getDeviceImagePtrTy(M),
68 PointerType::getUnqual(C), PointerType::getUnqual(C));
69 return DescTy;
70}
71
72PointerType *getBinDescPtrTy(Module &M) {
73 return PointerType::getUnqual(getBinDescTy(M));
74}
75
76/// Creates binary descriptor for the given device images. Binary descriptor
77/// is an object that is passed to the offloading runtime at program startup
78/// and it describes all device images available in the executable or shared
79/// library. It is defined as follows
80///
81/// __attribute__((visibility("hidden")))
82/// extern __tgt_offload_entry *__start_omp_offloading_entries;
83/// __attribute__((visibility("hidden")))
84/// extern __tgt_offload_entry *__stop_omp_offloading_entries;
85///
86/// static const char Image0[] = { <Bufs.front() contents> };
87/// ...
88/// static const char ImageN[] = { <Bufs.back() contents> };
89///
90/// static const __tgt_device_image Images[] = {
91/// {
92/// Image0, /*ImageStart*/
93/// Image0 + sizeof(Image0), /*ImageEnd*/
94/// __start_omp_offloading_entries, /*EntriesBegin*/
95/// __stop_omp_offloading_entries /*EntriesEnd*/
96/// },
97/// ...
98/// {
99/// ImageN, /*ImageStart*/
100/// ImageN + sizeof(ImageN), /*ImageEnd*/
101/// __start_omp_offloading_entries, /*EntriesBegin*/
102/// __stop_omp_offloading_entries /*EntriesEnd*/
103/// }
104/// };
105///
106/// static const __tgt_bin_desc BinDesc = {
107/// sizeof(Images) / sizeof(Images[0]), /*NumDeviceImages*/
108/// Images, /*DeviceImages*/
109/// __start_omp_offloading_entries, /*HostEntriesBegin*/
110/// __stop_omp_offloading_entries /*HostEntriesEnd*/
111/// };
112///
113/// Global variable that represents BinDesc is returned.
114GlobalVariable *createBinDesc(Module &M, ArrayRef<ArrayRef<char>> Bufs,
115 EntryArrayTy EntryArray, StringRef Suffix,
116 bool Relocatable) {
117 LLVMContext &C = M.getContext();
118 auto [EntriesB, EntriesE] = EntryArray;
119
120 auto *Zero = ConstantInt::get(getSizeTTy(M), 0u);
121 Constant *ZeroZero[] = {Zero, Zero};
122
123 // Create initializer for the images array.
124 SmallVector<Constant *, 4u> ImagesInits;
125 ImagesInits.reserve(Bufs.size());
126 for (ArrayRef<char> Buf : Bufs) {
127 // We embed the full offloading entry so the binary utilities can parse it.
128 auto *Data = ConstantDataArray::get(C, Buf);
129 auto *Image = new GlobalVariable(M, Data->getType(), /*isConstant=*/true,
130 GlobalVariable::InternalLinkage, Data,
131 ".omp_offloading.device_image" + Suffix);
132 Image->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
133 Image->setSection(Relocatable ? ".llvm.offloading.relocatable"
134 : ".llvm.offloading");
136
137 StringRef Binary(Buf.data(), Buf.size());
139 "Invalid binary format");
140
141 // The device image struct contains the pointer to the beginning and end of
142 // the image stored inside of the offload binary. There should only be one
143 // of these for each buffer so we parse it out manually.
144 const auto *Header =
145 reinterpret_cast<const object::OffloadBinary::Header *>(
146 Binary.bytes_begin());
147 const auto *Entry = reinterpret_cast<const object::OffloadBinary::Entry *>(
148 Binary.bytes_begin() + Header->EntryOffset);
149
150 auto *Begin = ConstantInt::get(getSizeTTy(M), Entry->ImageOffset);
151 auto *Size =
152 ConstantInt::get(getSizeTTy(M), Entry->ImageOffset + Entry->ImageSize);
153 Constant *ZeroBegin[] = {Zero, Begin};
154 Constant *ZeroSize[] = {Zero, Size};
155
156 auto *ImageB =
157 ConstantExpr::getGetElementPtr(Image->getValueType(), Image, ZeroBegin);
158 auto *ImageE =
159 ConstantExpr::getGetElementPtr(Image->getValueType(), Image, ZeroSize);
160
161 ImagesInits.push_back(ConstantStruct::get(getDeviceImageTy(M), ImageB,
162 ImageE, EntriesB, EntriesE));
163 }
164
165 // Then create images array.
166 auto *ImagesData = ConstantArray::get(
167 ArrayType::get(getDeviceImageTy(M), ImagesInits.size()), ImagesInits);
168
169 auto *Images =
170 new GlobalVariable(M, ImagesData->getType(), /*isConstant*/ true,
172 ".omp_offloading.device_images" + Suffix);
173 Images->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
174
175 auto *ImagesB =
176 ConstantExpr::getGetElementPtr(Images->getValueType(), Images, ZeroZero);
177
178 // And finally create the binary descriptor object.
179 auto *DescInit = ConstantStruct::get(
180 getBinDescTy(M),
181 ConstantInt::get(Type::getInt32Ty(C), ImagesInits.size()), ImagesB,
182 EntriesB, EntriesE);
183
184 return new GlobalVariable(M, DescInit->getType(), /*isConstant*/ true,
186 ".omp_offloading.descriptor" + Suffix);
187}
188
189Function *createUnregisterFunction(Module &M, GlobalVariable *BinDesc,
190 StringRef Suffix) {
191 LLVMContext &C = M.getContext();
192 auto *FuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);
193 auto *Func =
195 ".omp_offloading.descriptor_unreg" + Suffix, &M);
196 Func->setSection(".text.startup");
197
198 // Get __tgt_unregister_lib function declaration.
199 auto *UnRegFuncTy = FunctionType::get(Type::getVoidTy(C), getBinDescPtrTy(M),
200 /*isVarArg*/ false);
201 FunctionCallee UnRegFuncC =
202 M.getOrInsertFunction("__tgt_unregister_lib", UnRegFuncTy);
203
204 // Construct function body
205 IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));
206 Builder.CreateCall(UnRegFuncC, BinDesc);
207 Builder.CreateRetVoid();
208
209 return Func;
210}
211
212void createRegisterFunction(Module &M, GlobalVariable *BinDesc,
213 StringRef Suffix) {
214 LLVMContext &C = M.getContext();
215 auto *FuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);
217 ".omp_offloading.descriptor_reg" + Suffix, &M);
218 Func->setSection(".text.startup");
219
220 // Get __tgt_register_lib function declaration.
221 auto *RegFuncTy = FunctionType::get(Type::getVoidTy(C), getBinDescPtrTy(M),
222 /*isVarArg*/ false);
223 FunctionCallee RegFuncC =
224 M.getOrInsertFunction("__tgt_register_lib", RegFuncTy);
225
226 auto *AtExitTy = FunctionType::get(
227 Type::getInt32Ty(C), PointerType::getUnqual(C), /*isVarArg=*/false);
228 FunctionCallee AtExit = M.getOrInsertFunction("atexit", AtExitTy);
229
230 Function *UnregFunc = createUnregisterFunction(M, BinDesc, Suffix);
231
232 // Construct function body
233 IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));
234
235 Builder.CreateCall(RegFuncC, BinDesc);
236
237 // Register the destructors with 'atexit'. This is expected by the CUDA
238 // runtime and ensures that we clean up before dynamic objects are destroyed.
239 // This needs to be done after plugin initialization to ensure that it is
240 // called before the plugin runtime is destroyed.
241 Builder.CreateCall(AtExit, UnregFunc);
242 Builder.CreateRetVoid();
243
244 // Add this function to constructors.
245 appendToGlobalCtors(M, Func, /*Priority=*/101);
246}
247
248// struct fatbin_wrapper {
249// int32_t magic;
250// int32_t version;
251// void *image;
252// void *reserved;
253//};
254StructType *getFatbinWrapperTy(Module &M) {
255 LLVMContext &C = M.getContext();
256 StructType *FatbinTy = StructType::getTypeByName(C, "fatbin_wrapper");
257 if (!FatbinTy)
258 FatbinTy = StructType::create(
259 "fatbin_wrapper", Type::getInt32Ty(C), Type::getInt32Ty(C),
260 PointerType::getUnqual(C), PointerType::getUnqual(C));
261 return FatbinTy;
262}
263
264/// Embed the image \p Image into the module \p M so it can be found by the
265/// runtime.
266GlobalVariable *createFatbinDesc(Module &M, ArrayRef<char> Image, bool IsHIP,
267 StringRef Suffix) {
268 LLVMContext &C = M.getContext();
269 llvm::Type *Int8PtrTy = PointerType::getUnqual(C);
270 llvm::Triple Triple = llvm::Triple(M.getTargetTriple());
271
272 // Create the global string containing the fatbinary.
273 StringRef FatbinConstantSection =
274 IsHIP ? ".hip_fatbin"
275 : (Triple.isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin");
276 auto *Data = ConstantDataArray::get(C, Image);
277 auto *Fatbin = new GlobalVariable(M, Data->getType(), /*isConstant*/ true,
278 GlobalVariable::InternalLinkage, Data,
279 ".fatbin_image" + Suffix);
280 Fatbin->setSection(FatbinConstantSection);
281
282 // Create the fatbinary wrapper
283 StringRef FatbinWrapperSection = IsHIP ? ".hipFatBinSegment"
284 : Triple.isMacOSX() ? "__NV_CUDA,__fatbin"
285 : ".nvFatBinSegment";
286 Constant *FatbinWrapper[] = {
287 ConstantInt::get(Type::getInt32Ty(C), IsHIP ? HIPFatMagic : CudaFatMagic),
288 ConstantInt::get(Type::getInt32Ty(C), 1),
290 ConstantPointerNull::get(PointerType::getUnqual(C))};
291
292 Constant *FatbinInitializer =
293 ConstantStruct::get(getFatbinWrapperTy(M), FatbinWrapper);
294
295 auto *FatbinDesc =
296 new GlobalVariable(M, getFatbinWrapperTy(M),
297 /*isConstant*/ true, GlobalValue::InternalLinkage,
298 FatbinInitializer, ".fatbin_wrapper" + Suffix);
299 FatbinDesc->setSection(FatbinWrapperSection);
300 FatbinDesc->setAlignment(Align(8));
301
302 return FatbinDesc;
303}
304
305/// Create the register globals function. We will iterate all of the offloading
306/// entries stored at the begin / end symbols and register them according to
307/// their type. This creates the following function in IR:
308///
309/// extern struct __tgt_offload_entry __start_cuda_offloading_entries;
310/// extern struct __tgt_offload_entry __stop_cuda_offloading_entries;
311///
312/// extern void __cudaRegisterFunction(void **, void *, void *, void *, int,
313/// void *, void *, void *, void *, int *);
314/// extern void __cudaRegisterVar(void **, void *, void *, void *, int32_t,
315/// int64_t, int32_t, int32_t);
316///
317/// void __cudaRegisterTest(void **fatbinHandle) {
318/// for (struct __tgt_offload_entry *entry = &__start_cuda_offloading_entries;
319/// entry != &__stop_cuda_offloading_entries; ++entry) {
320/// if (!entry->size)
321/// __cudaRegisterFunction(fatbinHandle, entry->addr, entry->name,
322/// entry->name, -1, 0, 0, 0, 0, 0);
323/// else
324/// __cudaRegisterVar(fatbinHandle, entry->addr, entry->name, entry->name,
325/// 0, entry->size, 0, 0);
326/// }
327/// }
328Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,
329 EntryArrayTy EntryArray,
330 StringRef Suffix,
331 bool EmitSurfacesAndTextures) {
332 LLVMContext &C = M.getContext();
333 auto [EntriesB, EntriesE] = EntryArray;
334
335 // Get the __cudaRegisterFunction function declaration.
336 PointerType *Int8PtrTy = PointerType::get(C, 0);
337 PointerType *Int8PtrPtrTy = PointerType::get(C, 0);
338 PointerType *Int32PtrTy = PointerType::get(C, 0);
339 auto *RegFuncTy = FunctionType::get(
341 {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
342 Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int32PtrTy},
343 /*isVarArg*/ false);
344 FunctionCallee RegFunc = M.getOrInsertFunction(
345 IsHIP ? "__hipRegisterFunction" : "__cudaRegisterFunction", RegFuncTy);
346
347 // Get the __cudaRegisterVar function declaration.
348 auto *RegVarTy = FunctionType::get(
350 {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
352 /*isVarArg*/ false);
353 FunctionCallee RegVar = M.getOrInsertFunction(
354 IsHIP ? "__hipRegisterVar" : "__cudaRegisterVar", RegVarTy);
355
356 // Get the __cudaRegisterSurface function declaration.
357 FunctionType *RegManagedVarTy =
358 FunctionType::get(Type::getVoidTy(C),
359 {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,
361 /*isVarArg=*/false);
362 FunctionCallee RegManagedVar = M.getOrInsertFunction(
363 IsHIP ? "__hipRegisterManagedVar" : "__cudaRegisterManagedVar",
364 RegManagedVarTy);
365
366 // Get the __cudaRegisterSurface function declaration.
367 FunctionType *RegSurfaceTy =
368 FunctionType::get(Type::getVoidTy(C),
369 {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,
371 /*isVarArg=*/false);
372 FunctionCallee RegSurface = M.getOrInsertFunction(
373 IsHIP ? "__hipRegisterSurface" : "__cudaRegisterSurface", RegSurfaceTy);
374
375 // Get the __cudaRegisterTexture function declaration.
376 FunctionType *RegTextureTy = FunctionType::get(
378 {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
380 /*isVarArg=*/false);
381 FunctionCallee RegTexture = M.getOrInsertFunction(
382 IsHIP ? "__hipRegisterTexture" : "__cudaRegisterTexture", RegTextureTy);
383
384 auto *RegGlobalsTy = FunctionType::get(Type::getVoidTy(C), Int8PtrPtrTy,
385 /*isVarArg*/ false);
386 auto *RegGlobalsFn =
388 IsHIP ? ".hip.globals_reg" : ".cuda.globals_reg", &M);
389 RegGlobalsFn->setSection(".text.startup");
390
391 // Create the loop to register all the entries.
392 IRBuilder<> Builder(BasicBlock::Create(C, "entry", RegGlobalsFn));
393 auto *EntryBB = BasicBlock::Create(C, "while.entry", RegGlobalsFn);
394 auto *IfThenBB = BasicBlock::Create(C, "if.then", RegGlobalsFn);
395 auto *IfElseBB = BasicBlock::Create(C, "if.else", RegGlobalsFn);
396 auto *SwGlobalBB = BasicBlock::Create(C, "sw.global", RegGlobalsFn);
397 auto *SwManagedBB = BasicBlock::Create(C, "sw.managed", RegGlobalsFn);
398 auto *SwSurfaceBB = BasicBlock::Create(C, "sw.surface", RegGlobalsFn);
399 auto *SwTextureBB = BasicBlock::Create(C, "sw.texture", RegGlobalsFn);
400 auto *IfEndBB = BasicBlock::Create(C, "if.end", RegGlobalsFn);
401 auto *ExitBB = BasicBlock::Create(C, "while.end", RegGlobalsFn);
402
403 auto *EntryCmp = Builder.CreateICmpNE(EntriesB, EntriesE);
404 Builder.CreateCondBr(EntryCmp, EntryBB, ExitBB);
405 Builder.SetInsertPoint(EntryBB);
406 auto *Entry = Builder.CreatePHI(PointerType::getUnqual(C), 2, "entry");
407 auto *AddrPtr =
408 Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
409 {ConstantInt::get(getSizeTTy(M), 0),
410 ConstantInt::get(Type::getInt32Ty(C), 0)});
411 auto *Addr = Builder.CreateLoad(Int8PtrTy, AddrPtr, "addr");
412 auto *NamePtr =
413 Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
414 {ConstantInt::get(getSizeTTy(M), 0),
415 ConstantInt::get(Type::getInt32Ty(C), 1)});
416 auto *Name = Builder.CreateLoad(Int8PtrTy, NamePtr, "name");
417 auto *SizePtr =
418 Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
419 {ConstantInt::get(getSizeTTy(M), 0),
420 ConstantInt::get(Type::getInt32Ty(C), 2)});
421 auto *Size = Builder.CreateLoad(getSizeTTy(M), SizePtr, "size");
422 auto *FlagsPtr =
423 Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
424 {ConstantInt::get(getSizeTTy(M), 0),
425 ConstantInt::get(Type::getInt32Ty(C), 3)});
426 auto *Flags = Builder.CreateLoad(Type::getInt32Ty(C), FlagsPtr, "flags");
427 auto *DataPtr =
428 Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,
429 {ConstantInt::get(getSizeTTy(M), 0),
430 ConstantInt::get(Type::getInt32Ty(C), 4)});
431 auto *Data = Builder.CreateLoad(Type::getInt32Ty(C), DataPtr, "textype");
432 auto *Kind = Builder.CreateAnd(
433 Flags, ConstantInt::get(Type::getInt32Ty(C), 0x7), "type");
434
435 // Extract the flags stored in the bit-field and convert them to C booleans.
436 auto *ExternBit = Builder.CreateAnd(
437 Flags, ConstantInt::get(Type::getInt32Ty(C),
439 auto *Extern = Builder.CreateLShr(
440 ExternBit, ConstantInt::get(Type::getInt32Ty(C), 3), "extern");
441 auto *ConstantBit = Builder.CreateAnd(
442 Flags, ConstantInt::get(Type::getInt32Ty(C),
444 auto *Const = Builder.CreateLShr(
445 ConstantBit, ConstantInt::get(Type::getInt32Ty(C), 4), "constant");
446 auto *NormalizedBit = Builder.CreateAnd(
447 Flags, ConstantInt::get(Type::getInt32Ty(C),
449 auto *Normalized = Builder.CreateLShr(
450 NormalizedBit, ConstantInt::get(Type::getInt32Ty(C), 5), "normalized");
451 auto *FnCond =
452 Builder.CreateICmpEQ(Size, ConstantInt::getNullValue(getSizeTTy(M)));
453 Builder.CreateCondBr(FnCond, IfThenBB, IfElseBB);
454
455 // Create kernel registration code.
456 Builder.SetInsertPoint(IfThenBB);
457 Builder.CreateCall(RegFunc, {RegGlobalsFn->arg_begin(), Addr, Name, Name,
458 ConstantInt::get(Type::getInt32Ty(C), -1),
459 ConstantPointerNull::get(Int8PtrTy),
460 ConstantPointerNull::get(Int8PtrTy),
461 ConstantPointerNull::get(Int8PtrTy),
462 ConstantPointerNull::get(Int8PtrTy),
463 ConstantPointerNull::get(Int32PtrTy)});
464 Builder.CreateBr(IfEndBB);
465 Builder.SetInsertPoint(IfElseBB);
466
467 auto *Switch = Builder.CreateSwitch(Kind, IfEndBB);
468 // Create global variable registration code.
469 Builder.SetInsertPoint(SwGlobalBB);
470 Builder.CreateCall(RegVar,
471 {RegGlobalsFn->arg_begin(), Addr, Name, Name, Extern, Size,
472 Const, ConstantInt::get(Type::getInt32Ty(C), 0)});
473 Builder.CreateBr(IfEndBB);
474 Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalEntry),
475 SwGlobalBB);
476
477 // Create managed variable registration code.
478 Builder.SetInsertPoint(SwManagedBB);
479 auto *ManagedVar = Builder.CreateLoad(Int8PtrTy, Addr, "managed.addr");
480 auto *ManagedAddr = Builder.CreateInBoundsGEP(
481 Int8PtrTy, Addr, {ConstantInt::get(Builder.getInt64Ty(), 1)});
482 auto *Managed = Builder.CreateLoad(Int8PtrTy, ManagedAddr, "managed.addr");
483 Builder.CreateCall(RegManagedVar, {RegGlobalsFn->arg_begin(), ManagedVar,
484 Managed, Name, Size, Data});
485 Builder.CreateBr(IfEndBB);
486 Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalManagedEntry),
487 SwManagedBB);
488 // Create surface variable registration code.
489 Builder.SetInsertPoint(SwSurfaceBB);
490 if (EmitSurfacesAndTextures)
491 Builder.CreateCall(RegSurface, {RegGlobalsFn->arg_begin(), Addr, Name, Name,
492 Data, Extern});
493 Builder.CreateBr(IfEndBB);
494 Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalSurfaceEntry),
495 SwSurfaceBB);
496
497 // Create texture variable registration code.
498 Builder.SetInsertPoint(SwTextureBB);
499 if (EmitSurfacesAndTextures)
500 Builder.CreateCall(RegTexture, {RegGlobalsFn->arg_begin(), Addr, Name, Name,
501 Data, Normalized, Extern});
502 Builder.CreateBr(IfEndBB);
503 Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalTextureEntry),
504 SwTextureBB);
505
506 Builder.SetInsertPoint(IfEndBB);
507 auto *NewEntry = Builder.CreateInBoundsGEP(
508 offloading::getEntryTy(M), Entry, ConstantInt::get(getSizeTTy(M), 1));
509 auto *Cmp = Builder.CreateICmpEQ(
510 NewEntry,
512 ArrayType::get(offloading::getEntryTy(M), 0), EntriesE,
513 ArrayRef<Constant *>({ConstantInt::get(getSizeTTy(M), 0),
514 ConstantInt::get(getSizeTTy(M), 0)})));
515 Entry->addIncoming(
517 ArrayType::get(offloading::getEntryTy(M), 0), EntriesB,
518 ArrayRef<Constant *>({ConstantInt::get(getSizeTTy(M), 0),
519 ConstantInt::get(getSizeTTy(M), 0)})),
520 &RegGlobalsFn->getEntryBlock());
521 Entry->addIncoming(NewEntry, IfEndBB);
522 Builder.CreateCondBr(Cmp, ExitBB, EntryBB);
523 Builder.SetInsertPoint(ExitBB);
524 Builder.CreateRetVoid();
525
526 return RegGlobalsFn;
527}
528
529// Create the constructor and destructor to register the fatbinary with the CUDA
530// runtime.
531void createRegisterFatbinFunction(Module &M, GlobalVariable *FatbinDesc,
532 bool IsHIP, EntryArrayTy EntryArray,
533 StringRef Suffix,
534 bool EmitSurfacesAndTextures) {
535 LLVMContext &C = M.getContext();
536 auto *CtorFuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);
537 auto *CtorFunc = Function::Create(
539 (IsHIP ? ".hip.fatbin_reg" : ".cuda.fatbin_reg") + Suffix, &M);
540 CtorFunc->setSection(".text.startup");
541
542 auto *DtorFuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);
543 auto *DtorFunc = Function::Create(
545 (IsHIP ? ".hip.fatbin_unreg" : ".cuda.fatbin_unreg") + Suffix, &M);
546 DtorFunc->setSection(".text.startup");
547
548 auto *PtrTy = PointerType::getUnqual(C);
549
550 // Get the __cudaRegisterFatBinary function declaration.
551 auto *RegFatTy = FunctionType::get(PtrTy, PtrTy, /*isVarArg=*/false);
552 FunctionCallee RegFatbin = M.getOrInsertFunction(
553 IsHIP ? "__hipRegisterFatBinary" : "__cudaRegisterFatBinary", RegFatTy);
554 // Get the __cudaRegisterFatBinaryEnd function declaration.
555 auto *RegFatEndTy =
556 FunctionType::get(Type::getVoidTy(C), PtrTy, /*isVarArg=*/false);
557 FunctionCallee RegFatbinEnd =
558 M.getOrInsertFunction("__cudaRegisterFatBinaryEnd", RegFatEndTy);
559 // Get the __cudaUnregisterFatBinary function declaration.
560 auto *UnregFatTy =
561 FunctionType::get(Type::getVoidTy(C), PtrTy, /*isVarArg=*/false);
562 FunctionCallee UnregFatbin = M.getOrInsertFunction(
563 IsHIP ? "__hipUnregisterFatBinary" : "__cudaUnregisterFatBinary",
564 UnregFatTy);
565
566 auto *AtExitTy =
567 FunctionType::get(Type::getInt32Ty(C), PtrTy, /*isVarArg=*/false);
568 FunctionCallee AtExit = M.getOrInsertFunction("atexit", AtExitTy);
569
570 auto *BinaryHandleGlobal = new llvm::GlobalVariable(
571 M, PtrTy, false, llvm::GlobalValue::InternalLinkage,
573 (IsHIP ? ".hip.binary_handle" : ".cuda.binary_handle") + Suffix);
574
575 // Create the constructor to register this image with the runtime.
576 IRBuilder<> CtorBuilder(BasicBlock::Create(C, "entry", CtorFunc));
577 CallInst *Handle = CtorBuilder.CreateCall(
578 RegFatbin,
580 CtorBuilder.CreateAlignedStore(
581 Handle, BinaryHandleGlobal,
582 Align(M.getDataLayout().getPointerTypeSize(PtrTy)));
583 CtorBuilder.CreateCall(createRegisterGlobalsFunction(M, IsHIP, EntryArray,
584 Suffix,
585 EmitSurfacesAndTextures),
586 Handle);
587 if (!IsHIP)
588 CtorBuilder.CreateCall(RegFatbinEnd, Handle);
589 CtorBuilder.CreateCall(AtExit, DtorFunc);
590 CtorBuilder.CreateRetVoid();
591
592 // Create the destructor to unregister the image with the runtime. We cannot
593 // use a standard global destructor after CUDA 9.2 so this must be called by
594 // `atexit()` instead.
595 IRBuilder<> DtorBuilder(BasicBlock::Create(C, "entry", DtorFunc));
596 LoadInst *BinaryHandle = DtorBuilder.CreateAlignedLoad(
597 PtrTy, BinaryHandleGlobal,
598 Align(M.getDataLayout().getPointerTypeSize(PtrTy)));
599 DtorBuilder.CreateCall(UnregFatbin, BinaryHandle);
600 DtorBuilder.CreateRetVoid();
601
602 // Add this function to constructors.
603 appendToGlobalCtors(M, CtorFunc, /*Priority=*/101);
604}
605} // namespace
606
608 EntryArrayTy EntryArray,
609 llvm::StringRef Suffix, bool Relocatable) {
611 createBinDesc(M, Images, EntryArray, Suffix, Relocatable);
612 if (!Desc)
614 "No binary descriptors created.");
615 createRegisterFunction(M, Desc, Suffix);
616 return Error::success();
617}
618
620 EntryArrayTy EntryArray,
621 llvm::StringRef Suffix,
622 bool EmitSurfacesAndTextures) {
623 GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/false, Suffix);
624 if (!Desc)
626 "No fatbin section created.");
627
628 createRegisterFatbinFunction(M, Desc, /*IsHip=*/false, EntryArray, Suffix,
629 EmitSurfacesAndTextures);
630 return Error::success();
631}
632
634 EntryArrayTy EntryArray, llvm::StringRef Suffix,
635 bool EmitSurfacesAndTextures) {
636 GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/true, Suffix);
637 if (!Desc)
639 "No fatbin section created.");
640
641 createRegisterFatbinFunction(M, Desc, /*IsHip=*/true, EntryArray, Suffix,
642 EmitSurfacesAndTextures);
643 return Error::success();
644}
static IntegerType * getSizeTTy(IRBuilderBase &B, const TargetLibraryInfo *TLI)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
uint64_t Addr
std::string Name
uint64_t Size
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
@ ConstantBit
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:212
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1312
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
Definition: Constants.h:709
static Constant * getInBoundsGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList)
Create an "inbounds" getelementptr.
Definition: Constants.h:1294
static Constant * getPointerBitCastOrAddrSpaceCast(Constant *C, Type *Ty)
Create a BitCast or AddrSpaceCast for a pointer type depending on the address space.
Definition: Constants.cpp:2268
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
Definition: Constants.h:1267
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1826
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1378
This is an important base class in LLVM.
Definition: Constant.h:42
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:337
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:170
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:173
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:59
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2705
Class to represent integer types.
Definition: DerivedTypes.h:42
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:176
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
size_t size() const
Definition: SmallVector.h:78
void reserve(size_type N)
Definition: SmallVector.h:663
void push_back(const T &Elt)
Definition: SmallVector.h:413
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
Class to represent struct types.
Definition: DerivedTypes.h:218
static StructType * getTypeByName(LLVMContext &C, StringRef Name)
Return the type with the specified name, or null if there is none by that name.
Definition: Type.cpp:731
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Definition: Type.cpp:612
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool isMacOSX() const
Is this a Mac OS X triple.
Definition: Triple.h:544
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static uint64_t getAlignment()
Definition: OffloadBinary.h:84
@ Entry
Definition: COFF.h:844
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
StructType * getEntryTy(Module &M)
Returns the type of the offloading entry we use to store kernels and globals that will be registered ...
Definition: Utility.cpp:24
@ OffloadGlobalSurfaceEntry
Mark the entry as a surface variable.
Definition: Utility.h:43
@ OffloadGlobalTextureEntry
Mark the entry as a texture variable.
Definition: Utility.h:45
@ OffloadGlobalNormalized
Mark the entry as being a normalized surface.
Definition: Utility.h:51
@ OffloadGlobalEntry
Mark the entry as a global entry.
Definition: Utility.h:39
@ OffloadGlobalManagedEntry
Mark the entry as a managed global variable.
Definition: Utility.h:41
@ OffloadGlobalExtern
Mark the entry as being extern.
Definition: Utility.h:47
@ OffloadGlobalConstant
Mark the entry as being constant.
Definition: Utility.h:49
llvm::Error wrapOpenMPBinaries(llvm::Module &M, llvm::ArrayRef< llvm::ArrayRef< char > > Images, EntryArrayTy EntryArray, llvm::StringRef Suffix="", bool Relocatable=false)
Wraps the input device images into the module M as global symbols and registers the images with the O...
std::pair< GlobalVariable *, GlobalVariable * > EntryArrayTy
llvm::Error wrapHIPBinary(llvm::Module &M, llvm::ArrayRef< char > Images, EntryArrayTy EntryArray, llvm::StringRef Suffix="", bool EmitSurfacesAndTextures=true)
Wraps the input bundled image into the module M as global symbols and registers the images with the H...
llvm::Error wrapCudaBinary(llvm::Module &M, llvm::ArrayRef< char > Images, EntryArrayTy EntryArray, llvm::StringRef Suffix="", bool EmitSurfacesAndTextures=true)
Wraps the input fatbinary image into the module M as global symbols and registers the images with the...
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
file_magic identify_magic(StringRef magic)
Identify the type of a binary file based on how magical it is.
Definition: Magic.cpp:33
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:98
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
Definition: Error.h:1291
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:73
@ Extern
Replace returns with jump to thunk, don't emit thunk.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Description of the encoding of one expression Op.
@ offload_binary
LLVM offload object file.
Definition: Magic.h:57