1//===- OffloadWrapper.cpp ---------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Frontend/Offloading/OffloadWrapper.h"
10#include "llvm/ADT/ArrayRef.h"
11#include "llvm/BinaryFormat/Magic.h"
12#include "llvm/Frontend/Offloading/Utility.h"
13#include "llvm/IR/Constants.h"
14#include "llvm/IR/GlobalVariable.h"
15#include "llvm/IR/IRBuilder.h"
16#include "llvm/IR/LLVMContext.h"
17#include "llvm/IR/Module.h"
18#include "llvm/Object/OffloadBinary.h"
19#include "llvm/Support/Error.h"
20#include "llvm/TargetParser/Triple.h"
21#include "llvm/Transforms/Utils/ModuleUtils.h"
22
23using namespace llvm;
24using namespace llvm::offloading;
25
26namespace {
27/// Magic number that begins the section containing the CUDA fatbinary.
28constexpr unsigned CudaFatMagic = 0x466243b1;
29constexpr unsigned HIPFatMagic = 0x48495046;
30
31IntegerType *getSizeTTy(Module &M) {
32 return M.getDataLayout().getIntPtrType(C&: M.getContext());
33}
34
35// struct __tgt_device_image {
36// void *ImageStart;
37// void *ImageEnd;
38// __tgt_offload_entry *EntriesBegin;
39// __tgt_offload_entry *EntriesEnd;
40// };
41StructType *getDeviceImageTy(Module &M) {
42 LLVMContext &C = M.getContext();
43 StructType *ImageTy = StructType::getTypeByName(C, Name: "__tgt_device_image");
44 if (!ImageTy)
45 ImageTy =
46 StructType::create(Name: "__tgt_device_image", elt1: PointerType::getUnqual(C),
47 elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C),
48 elts: PointerType::getUnqual(C));
49 return ImageTy;
50}
51
52PointerType *getDeviceImagePtrTy(Module &M) {
53 return PointerType::getUnqual(ElementType: getDeviceImageTy(M));
54}
55
56// struct __tgt_bin_desc {
57// int32_t NumDeviceImages;
58// __tgt_device_image *DeviceImages;
59// __tgt_offload_entry *HostEntriesBegin;
60// __tgt_offload_entry *HostEntriesEnd;
61// };
62StructType *getBinDescTy(Module &M) {
63 LLVMContext &C = M.getContext();
64 StructType *DescTy = StructType::getTypeByName(C, Name: "__tgt_bin_desc");
65 if (!DescTy)
66 DescTy = StructType::create(
67 Name: "__tgt_bin_desc", elt1: Type::getInt32Ty(C), elts: getDeviceImagePtrTy(M),
68 elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C));
69 return DescTy;
70}
71
72PointerType *getBinDescPtrTy(Module &M) {
73 return PointerType::getUnqual(ElementType: getBinDescTy(M));
74}
75
76/// Creates binary descriptor for the given device images. Binary descriptor
77/// is an object that is passed to the offloading runtime at program startup
78/// and it describes all device images available in the executable or shared
79/// library. It is defined as follows
80///
81/// __attribute__((visibility("hidden")))
82/// extern __tgt_offload_entry *__start_omp_offloading_entries;
83/// __attribute__((visibility("hidden")))
84/// extern __tgt_offload_entry *__stop_omp_offloading_entries;
85///
86/// static const char Image0[] = { <Bufs.front() contents> };
87/// ...
88/// static const char ImageN[] = { <Bufs.back() contents> };
89///
90/// static const __tgt_device_image Images[] = {
91/// {
92/// Image0, /*ImageStart*/
93/// Image0 + sizeof(Image0), /*ImageEnd*/
94/// __start_omp_offloading_entries, /*EntriesBegin*/
95/// __stop_omp_offloading_entries /*EntriesEnd*/
96/// },
97/// ...
98/// {
99/// ImageN, /*ImageStart*/
100/// ImageN + sizeof(ImageN), /*ImageEnd*/
101/// __start_omp_offloading_entries, /*EntriesBegin*/
102/// __stop_omp_offloading_entries /*EntriesEnd*/
103/// }
104/// };
105///
106/// static const __tgt_bin_desc BinDesc = {
107/// sizeof(Images) / sizeof(Images[0]), /*NumDeviceImages*/
108/// Images, /*DeviceImages*/
109/// __start_omp_offloading_entries, /*HostEntriesBegin*/
110/// __stop_omp_offloading_entries /*HostEntriesEnd*/
111/// };
112///
113/// Global variable that represents BinDesc is returned.
114GlobalVariable *createBinDesc(Module &M, ArrayRef<ArrayRef<char>> Bufs,
115 EntryArrayTy EntryArray, StringRef Suffix,
116 bool Relocatable) {
117 LLVMContext &C = M.getContext();
118 auto [EntriesB, EntriesE] = EntryArray;
119
120 auto *Zero = ConstantInt::get(Ty: getSizeTTy(M), V: 0u);
121 Constant *ZeroZero[] = {Zero, Zero};
122
123 // Create initializer for the images array.
124 SmallVector<Constant *, 4u> ImagesInits;
125 ImagesInits.reserve(N: Bufs.size());
126 for (ArrayRef<char> Buf : Bufs) {
127 // We embed the full offloading entry so the binary utilities can parse it.
128 auto *Data = ConstantDataArray::get(Context&: C, Elts: Buf);
129 auto *Image = new GlobalVariable(M, Data->getType(), /*isConstant=*/true,
130 GlobalVariable::InternalLinkage, Data,
131 ".omp_offloading.device_image" + Suffix);
132 Image->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
133 Image->setSection(Relocatable ? ".llvm.offloading.relocatable"
134 : ".llvm.offloading");
135 Image->setAlignment(Align(object::OffloadBinary::getAlignment()));
136
137 StringRef Binary(Buf.data(), Buf.size());
138 assert(identify_magic(Binary) == file_magic::offload_binary &&
139 "Invalid binary format");
140
141 // The device image struct contains the pointer to the beginning and end of
142 // the image stored inside of the offload binary. There should only be one
143 // of these for each buffer so we parse it out manually.
144 const auto *Header =
145 reinterpret_cast<const object::OffloadBinary::Header *>(
146 Binary.bytes_begin());
147 const auto *Entry = reinterpret_cast<const object::OffloadBinary::Entry *>(
148 Binary.bytes_begin() + Header->EntryOffset);
149
150 auto *Begin = ConstantInt::get(Ty: getSizeTTy(M), V: Entry->ImageOffset);
151 auto *Size =
152 ConstantInt::get(Ty: getSizeTTy(M), V: Entry->ImageOffset + Entry->ImageSize);
153 Constant *ZeroBegin[] = {Zero, Begin};
154 Constant *ZeroSize[] = {Zero, Size};
155
156 auto *ImageB =
157 ConstantExpr::getGetElementPtr(Ty: Image->getValueType(), C: Image, IdxList: ZeroBegin);
158 auto *ImageE =
159 ConstantExpr::getGetElementPtr(Ty: Image->getValueType(), C: Image, IdxList: ZeroSize);
160
161 ImagesInits.push_back(Elt: ConstantStruct::get(T: getDeviceImageTy(M), Vs: ImageB,
162 Vs: ImageE, Vs: EntriesB, Vs: EntriesE));
163 }
164
165 // Then create images array.
166 auto *ImagesData = ConstantArray::get(
167 T: ArrayType::get(ElementType: getDeviceImageTy(M), NumElements: ImagesInits.size()), V: ImagesInits);
168
169 auto *Images =
170 new GlobalVariable(M, ImagesData->getType(), /*isConstant*/ true,
171 GlobalValue::InternalLinkage, ImagesData,
172 ".omp_offloading.device_images" + Suffix);
173 Images->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
174
175 auto *ImagesB =
176 ConstantExpr::getGetElementPtr(Ty: Images->getValueType(), C: Images, IdxList: ZeroZero);
177
178 // And finally create the binary descriptor object.
179 auto *DescInit = ConstantStruct::get(
180 T: getBinDescTy(M),
181 Vs: ConstantInt::get(Ty: Type::getInt32Ty(C), V: ImagesInits.size()), Vs: ImagesB,
182 Vs: EntriesB, Vs: EntriesE);
183
184 return new GlobalVariable(M, DescInit->getType(), /*isConstant*/ true,
185 GlobalValue::InternalLinkage, DescInit,
186 ".omp_offloading.descriptor" + Suffix);
187}
188
189Function *createUnregisterFunction(Module &M, GlobalVariable *BinDesc,
190 StringRef Suffix) {
191 LLVMContext &C = M.getContext();
192 auto *FuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
193 auto *Func =
194 Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
195 N: ".omp_offloading.descriptor_unreg" + Suffix, M: &M);
196 Func->setSection(".text.startup");
197
198 // Get __tgt_unregister_lib function declaration.
199 auto *UnRegFuncTy = FunctionType::get(Result: Type::getVoidTy(C), Params: getBinDescPtrTy(M),
200 /*isVarArg*/ false);
201 FunctionCallee UnRegFuncC =
202 M.getOrInsertFunction(Name: "__tgt_unregister_lib", T: UnRegFuncTy);
203
204 // Construct function body
205 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
206 Builder.CreateCall(Callee: UnRegFuncC, Args: BinDesc);
207 Builder.CreateRetVoid();
208
209 return Func;
210}
211
212void createRegisterFunction(Module &M, GlobalVariable *BinDesc,
213 StringRef Suffix) {
214 LLVMContext &C = M.getContext();
215 auto *FuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
216 auto *Func = Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage,
217 N: ".omp_offloading.descriptor_reg" + Suffix, M: &M);
218 Func->setSection(".text.startup");
219
220 // Get __tgt_register_lib function declaration.
221 auto *RegFuncTy = FunctionType::get(Result: Type::getVoidTy(C), Params: getBinDescPtrTy(M),
222 /*isVarArg*/ false);
223 FunctionCallee RegFuncC =
224 M.getOrInsertFunction(Name: "__tgt_register_lib", T: RegFuncTy);
225
226 auto *AtExitTy = FunctionType::get(
227 Result: Type::getInt32Ty(C), Params: PointerType::getUnqual(C), /*isVarArg=*/false);
228 FunctionCallee AtExit = M.getOrInsertFunction(Name: "atexit", T: AtExitTy);
229
230 Function *UnregFunc = createUnregisterFunction(M, BinDesc, Suffix);
231
232 // Construct function body
233 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: Func));
234
235 // Register the destructors with 'atexit'. This is expected by the CUDA
236 // runtime and ensures that we clean up before dynamic objects are destroyed.
237 // This needs to be done before the runtime is called and registers its own.
238 Builder.CreateCall(Callee: AtExit, Args: UnregFunc);
239
240 Builder.CreateCall(Callee: RegFuncC, Args: BinDesc);
241 Builder.CreateRetVoid();
242
243 // Add this function to constructors.
244 appendToGlobalCtors(M, F: Func, /*Priority=*/101);
245}
246
247// struct fatbin_wrapper {
248// int32_t magic;
249// int32_t version;
250// void *image;
251// void *reserved;
252//};
253StructType *getFatbinWrapperTy(Module &M) {
254 LLVMContext &C = M.getContext();
255 StructType *FatbinTy = StructType::getTypeByName(C, Name: "fatbin_wrapper");
256 if (!FatbinTy)
257 FatbinTy = StructType::create(
258 Name: "fatbin_wrapper", elt1: Type::getInt32Ty(C), elts: Type::getInt32Ty(C),
259 elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C));
260 return FatbinTy;
261}
262
263/// Embed the image \p Image into the module \p M so it can be found by the
264/// runtime.
265GlobalVariable *createFatbinDesc(Module &M, ArrayRef<char> Image, bool IsHIP,
266 StringRef Suffix) {
267 LLVMContext &C = M.getContext();
268 llvm::Type *Int8PtrTy = PointerType::getUnqual(C);
269 llvm::Triple Triple = llvm::Triple(M.getTargetTriple());
270
271 // Create the global string containing the fatbinary.
272 StringRef FatbinConstantSection =
273 IsHIP ? ".hip_fatbin"
274 : (Triple.isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin");
275 auto *Data = ConstantDataArray::get(Context&: C, Elts: Image);
276 auto *Fatbin = new GlobalVariable(M, Data->getType(), /*isConstant*/ true,
277 GlobalVariable::InternalLinkage, Data,
278 ".fatbin_image" + Suffix);
279 Fatbin->setSection(FatbinConstantSection);
280
281 // Create the fatbinary wrapper
282 StringRef FatbinWrapperSection = IsHIP ? ".hipFatBinSegment"
283 : Triple.isMacOSX() ? "__NV_CUDA,__fatbin"
284 : ".nvFatBinSegment";
285 Constant *FatbinWrapper[] = {
286 ConstantInt::get(Ty: Type::getInt32Ty(C), V: IsHIP ? HIPFatMagic : CudaFatMagic),
287 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1),
288 ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: Fatbin, Ty: Int8PtrTy),
289 ConstantPointerNull::get(T: PointerType::getUnqual(C))};
290
291 Constant *FatbinInitializer =
292 ConstantStruct::get(T: getFatbinWrapperTy(M), V: FatbinWrapper);
293
294 auto *FatbinDesc =
295 new GlobalVariable(M, getFatbinWrapperTy(M),
296 /*isConstant*/ true, GlobalValue::InternalLinkage,
297 FatbinInitializer, ".fatbin_wrapper" + Suffix);
298 FatbinDesc->setSection(FatbinWrapperSection);
299 FatbinDesc->setAlignment(Align(8));
300
301 return FatbinDesc;
302}
303
304/// Create the register globals function. We will iterate all of the offloading
305/// entries stored at the begin / end symbols and register them according to
306/// their type. This creates the following function in IR:
307///
308/// extern struct __tgt_offload_entry __start_cuda_offloading_entries;
309/// extern struct __tgt_offload_entry __stop_cuda_offloading_entries;
310///
311/// extern void __cudaRegisterFunction(void **, void *, void *, void *, int,
312/// void *, void *, void *, void *, int *);
313/// extern void __cudaRegisterVar(void **, void *, void *, void *, int32_t,
314/// int64_t, int32_t, int32_t);
315///
316/// void __cudaRegisterTest(void **fatbinHandle) {
317/// for (struct __tgt_offload_entry *entry = &__start_cuda_offloading_entries;
318/// entry != &__stop_cuda_offloading_entries; ++entry) {
319/// if (!entry->size)
320/// __cudaRegisterFunction(fatbinHandle, entry->addr, entry->name,
321/// entry->name, -1, 0, 0, 0, 0, 0);
322/// else
323/// __cudaRegisterVar(fatbinHandle, entry->addr, entry->name, entry->name,
324/// 0, entry->size, 0, 0);
325/// }
326/// }
327Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,
328 EntryArrayTy EntryArray,
329 StringRef Suffix,
330 bool EmitSurfacesAndTextures) {
331 LLVMContext &C = M.getContext();
332 auto [EntriesB, EntriesE] = EntryArray;
333
334 // Get the __cudaRegisterFunction function declaration.
335 PointerType *Int8PtrTy = PointerType::get(C, AddressSpace: 0);
336 PointerType *Int8PtrPtrTy = PointerType::get(C, AddressSpace: 0);
337 PointerType *Int32PtrTy = PointerType::get(C, AddressSpace: 0);
338 auto *RegFuncTy = FunctionType::get(
339 Result: Type::getInt32Ty(C),
340 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
341 Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int32PtrTy},
342 /*isVarArg*/ false);
343 FunctionCallee RegFunc = M.getOrInsertFunction(
344 Name: IsHIP ? "__hipRegisterFunction" : "__cudaRegisterFunction", T: RegFuncTy);
345
346 // Get the __cudaRegisterVar function declaration.
347 auto *RegVarTy = FunctionType::get(
348 Result: Type::getVoidTy(C),
349 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
350 getSizeTTy(M), Type::getInt32Ty(C), Type::getInt32Ty(C)},
351 /*isVarArg*/ false);
352 FunctionCallee RegVar = M.getOrInsertFunction(
353 Name: IsHIP ? "__hipRegisterVar" : "__cudaRegisterVar", T: RegVarTy);
354
355 // Get the __cudaRegisterSurface function declaration.
356 FunctionType *RegSurfaceTy =
357 FunctionType::get(Result: Type::getVoidTy(C),
358 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,
359 Type::getInt32Ty(C), Type::getInt32Ty(C)},
360 /*isVarArg=*/false);
361 FunctionCallee RegSurface = M.getOrInsertFunction(
362 Name: IsHIP ? "__hipRegisterSurface" : "__cudaRegisterSurface", T: RegSurfaceTy);
363
364 // Get the __cudaRegisterTexture function declaration.
365 FunctionType *RegTextureTy = FunctionType::get(
366 Result: Type::getVoidTy(C),
367 Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),
368 Type::getInt32Ty(C), Type::getInt32Ty(C)},
369 /*isVarArg=*/false);
370 FunctionCallee RegTexture = M.getOrInsertFunction(
371 Name: IsHIP ? "__hipRegisterTexture" : "__cudaRegisterTexture", T: RegTextureTy);
372
373 auto *RegGlobalsTy = FunctionType::get(Result: Type::getVoidTy(C), Params: Int8PtrPtrTy,
374 /*isVarArg*/ false);
375 auto *RegGlobalsFn =
376 Function::Create(Ty: RegGlobalsTy, Linkage: GlobalValue::InternalLinkage,
377 N: IsHIP ? ".hip.globals_reg" : ".cuda.globals_reg", M: &M);
378 RegGlobalsFn->setSection(".text.startup");
379
380 // Create the loop to register all the entries.
381 IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry", Parent: RegGlobalsFn));
382 auto *EntryBB = BasicBlock::Create(Context&: C, Name: "while.entry", Parent: RegGlobalsFn);
383 auto *IfThenBB = BasicBlock::Create(Context&: C, Name: "if.then", Parent: RegGlobalsFn);
384 auto *IfElseBB = BasicBlock::Create(Context&: C, Name: "if.else", Parent: RegGlobalsFn);
385 auto *SwGlobalBB = BasicBlock::Create(Context&: C, Name: "sw.global", Parent: RegGlobalsFn);
386 auto *SwManagedBB = BasicBlock::Create(Context&: C, Name: "sw.managed", Parent: RegGlobalsFn);
387 auto *SwSurfaceBB = BasicBlock::Create(Context&: C, Name: "sw.surface", Parent: RegGlobalsFn);
388 auto *SwTextureBB = BasicBlock::Create(Context&: C, Name: "sw.texture", Parent: RegGlobalsFn);
389 auto *IfEndBB = BasicBlock::Create(Context&: C, Name: "if.end", Parent: RegGlobalsFn);
390 auto *ExitBB = BasicBlock::Create(Context&: C, Name: "while.end", Parent: RegGlobalsFn);
391
392 auto *EntryCmp = Builder.CreateICmpNE(LHS: EntriesB, RHS: EntriesE);
393 Builder.CreateCondBr(Cond: EntryCmp, True: EntryBB, False: ExitBB);
394 Builder.SetInsertPoint(EntryBB);
395 auto *Entry = Builder.CreatePHI(Ty: PointerType::getUnqual(C), NumReservedValues: 2, Name: "entry");
396 auto *AddrPtr =
397 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
398 IdxList: {ConstantInt::get(Ty: getSizeTTy(M), V: 0),
399 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0)});
400 auto *Addr = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: AddrPtr, Name: "addr");
401 auto *NamePtr =
402 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
403 IdxList: {ConstantInt::get(Ty: getSizeTTy(M), V: 0),
404 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)});
405 auto *Name = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: NamePtr, Name: "name");
406 auto *SizePtr =
407 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
408 IdxList: {ConstantInt::get(Ty: getSizeTTy(M), V: 0),
409 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 2)});
410 auto *Size = Builder.CreateLoad(Ty: getSizeTTy(M), Ptr: SizePtr, Name: "size");
411 auto *FlagsPtr =
412 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
413 IdxList: {ConstantInt::get(Ty: getSizeTTy(M), V: 0),
414 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 3)});
415 auto *Flags = Builder.CreateLoad(Ty: Type::getInt32Ty(C), Ptr: FlagsPtr, Name: "flags");
416 auto *DataPtr =
417 Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry,
418 IdxList: {ConstantInt::get(Ty: getSizeTTy(M), V: 0),
419 ConstantInt::get(Ty: Type::getInt32Ty(C), V: 4)});
420 auto *Data = Builder.CreateLoad(Ty: Type::getInt32Ty(C), Ptr: DataPtr, Name: "textype");
421 auto *Kind = Builder.CreateAnd(
422 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0x7), Name: "type");
423
424 // Extract the flags stored in the bit-field and convert them to C booleans.
425 auto *ExternBit = Builder.CreateAnd(
426 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C),
427 V: llvm::offloading::OffloadGlobalExtern));
428 auto *Extern = Builder.CreateLShr(
429 LHS: ExternBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 3), Name: "extern");
430 auto *ConstantBit = Builder.CreateAnd(
431 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C),
432 V: llvm::offloading::OffloadGlobalConstant));
433 auto *Const = Builder.CreateLShr(
434 LHS: ConstantBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 4), Name: "constant");
435 auto *NormalizedBit = Builder.CreateAnd(
436 LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C),
437 V: llvm::offloading::OffloadGlobalNormalized));
438 auto *Normalized = Builder.CreateLShr(
439 LHS: NormalizedBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 5), Name: "normalized");
440 auto *FnCond =
441 Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::getNullValue(Ty: getSizeTTy(M)));
442 Builder.CreateCondBr(Cond: FnCond, True: IfThenBB, False: IfElseBB);
443
444 // Create kernel registration code.
445 Builder.SetInsertPoint(IfThenBB);
446 Builder.CreateCall(Callee: RegFunc, Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name,
447 ConstantInt::get(Ty: Type::getInt32Ty(C), V: -1),
448 ConstantPointerNull::get(T: Int8PtrTy),
449 ConstantPointerNull::get(T: Int8PtrTy),
450 ConstantPointerNull::get(T: Int8PtrTy),
451 ConstantPointerNull::get(T: Int8PtrTy),
452 ConstantPointerNull::get(T: Int32PtrTy)});
453 Builder.CreateBr(Dest: IfEndBB);
454 Builder.SetInsertPoint(IfElseBB);
455
456 auto *Switch = Builder.CreateSwitch(V: Kind, Dest: IfEndBB);
457 // Create global variable registration code.
458 Builder.SetInsertPoint(SwGlobalBB);
459 Builder.CreateCall(Callee: RegVar,
460 Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name, Extern, Size,
461 Const, ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0)});
462 Builder.CreateBr(Dest: IfEndBB);
463 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalEntry),
464 Dest: SwGlobalBB);
465
466 // Create managed variable registration code.
467 Builder.SetInsertPoint(SwManagedBB);
468 Builder.CreateBr(Dest: IfEndBB);
469 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalManagedEntry),
470 Dest: SwManagedBB);
471 // Create surface variable registration code.
472 Builder.SetInsertPoint(SwSurfaceBB);
473 if (EmitSurfacesAndTextures)
474 Builder.CreateCall(Callee: RegSurface, Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name,
475 Data, Extern});
476 Builder.CreateBr(Dest: IfEndBB);
477 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalSurfaceEntry),
478 Dest: SwSurfaceBB);
479
480 // Create texture variable registration code.
481 Builder.SetInsertPoint(SwTextureBB);
482 if (EmitSurfacesAndTextures)
483 Builder.CreateCall(Callee: RegTexture, Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name,
484 Data, Normalized, Extern});
485 Builder.CreateBr(Dest: IfEndBB);
486 Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalTextureEntry),
487 Dest: SwTextureBB);
488
489 Builder.SetInsertPoint(IfEndBB);
490 auto *NewEntry = Builder.CreateInBoundsGEP(
491 Ty: offloading::getEntryTy(M), Ptr: Entry, IdxList: ConstantInt::get(Ty: getSizeTTy(M), V: 1));
492 auto *Cmp = Builder.CreateICmpEQ(
493 LHS: NewEntry,
494 RHS: ConstantExpr::getInBoundsGetElementPtr(
495 Ty: ArrayType::get(ElementType: offloading::getEntryTy(M), NumElements: 0), C: EntriesE,
496 IdxList: ArrayRef<Constant *>({ConstantInt::get(Ty: getSizeTTy(M), V: 0),
497 ConstantInt::get(Ty: getSizeTTy(M), V: 0)})));
498 Entry->addIncoming(
499 V: ConstantExpr::getInBoundsGetElementPtr(
500 Ty: ArrayType::get(ElementType: offloading::getEntryTy(M), NumElements: 0), C: EntriesB,
501 IdxList: ArrayRef<Constant *>({ConstantInt::get(Ty: getSizeTTy(M), V: 0),
502 ConstantInt::get(Ty: getSizeTTy(M), V: 0)})),
503 BB: &RegGlobalsFn->getEntryBlock());
504 Entry->addIncoming(V: NewEntry, BB: IfEndBB);
505 Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: EntryBB);
506 Builder.SetInsertPoint(ExitBB);
507 Builder.CreateRetVoid();
508
509 return RegGlobalsFn;
510}
511
512// Create the constructor and destructor to register the fatbinary with the CUDA
513// runtime.
514void createRegisterFatbinFunction(Module &M, GlobalVariable *FatbinDesc,
515 bool IsHIP, EntryArrayTy EntryArray,
516 StringRef Suffix,
517 bool EmitSurfacesAndTextures) {
518 LLVMContext &C = M.getContext();
519 auto *CtorFuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
520 auto *CtorFunc = Function::Create(
521 Ty: CtorFuncTy, Linkage: GlobalValue::InternalLinkage,
522 N: (IsHIP ? ".hip.fatbin_reg" : ".cuda.fatbin_reg") + Suffix, M: &M);
523 CtorFunc->setSection(".text.startup");
524
525 auto *DtorFuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false);
526 auto *DtorFunc = Function::Create(
527 Ty: DtorFuncTy, Linkage: GlobalValue::InternalLinkage,
528 N: (IsHIP ? ".hip.fatbin_unreg" : ".cuda.fatbin_unreg") + Suffix, M: &M);
529 DtorFunc->setSection(".text.startup");
530
531 auto *PtrTy = PointerType::getUnqual(C);
532
533 // Get the __cudaRegisterFatBinary function declaration.
534 auto *RegFatTy = FunctionType::get(Result: PtrTy, Params: PtrTy, /*isVarArg=*/false);
535 FunctionCallee RegFatbin = M.getOrInsertFunction(
536 Name: IsHIP ? "__hipRegisterFatBinary" : "__cudaRegisterFatBinary", T: RegFatTy);
537 // Get the __cudaRegisterFatBinaryEnd function declaration.
538 auto *RegFatEndTy =
539 FunctionType::get(Result: Type::getVoidTy(C), Params: PtrTy, /*isVarArg=*/false);
540 FunctionCallee RegFatbinEnd =
541 M.getOrInsertFunction(Name: "__cudaRegisterFatBinaryEnd", T: RegFatEndTy);
542 // Get the __cudaUnregisterFatBinary function declaration.
543 auto *UnregFatTy =
544 FunctionType::get(Result: Type::getVoidTy(C), Params: PtrTy, /*isVarArg=*/false);
545 FunctionCallee UnregFatbin = M.getOrInsertFunction(
546 Name: IsHIP ? "__hipUnregisterFatBinary" : "__cudaUnregisterFatBinary",
547 T: UnregFatTy);
548
549 auto *AtExitTy =
550 FunctionType::get(Result: Type::getInt32Ty(C), Params: PtrTy, /*isVarArg=*/false);
551 FunctionCallee AtExit = M.getOrInsertFunction(Name: "atexit", T: AtExitTy);
552
553 auto *BinaryHandleGlobal = new llvm::GlobalVariable(
554 M, PtrTy, false, llvm::GlobalValue::InternalLinkage,
555 llvm::ConstantPointerNull::get(T: PtrTy),
556 (IsHIP ? ".hip.binary_handle" : ".cuda.binary_handle") + Suffix);
557
558 // Create the constructor to register this image with the runtime.
559 IRBuilder<> CtorBuilder(BasicBlock::Create(Context&: C, Name: "entry", Parent: CtorFunc));
560 CallInst *Handle = CtorBuilder.CreateCall(
561 Callee: RegFatbin,
562 Args: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: FatbinDesc, Ty: PtrTy));
563 CtorBuilder.CreateAlignedStore(
564 Val: Handle, Ptr: BinaryHandleGlobal,
565 Align: Align(M.getDataLayout().getPointerTypeSize(Ty: PtrTy)));
566 CtorBuilder.CreateCall(Callee: createRegisterGlobalsFunction(M, IsHIP, EntryArray,
567 Suffix,
568 EmitSurfacesAndTextures),
569 Args: Handle);
570 if (!IsHIP)
571 CtorBuilder.CreateCall(Callee: RegFatbinEnd, Args: Handle);
572 CtorBuilder.CreateCall(Callee: AtExit, Args: DtorFunc);
573 CtorBuilder.CreateRetVoid();
574
575 // Create the destructor to unregister the image with the runtime. We cannot
576 // use a standard global destructor after CUDA 9.2 so this must be called by
577 // `atexit()` intead.
578 IRBuilder<> DtorBuilder(BasicBlock::Create(Context&: C, Name: "entry", Parent: DtorFunc));
579 LoadInst *BinaryHandle = DtorBuilder.CreateAlignedLoad(
580 Ty: PtrTy, Ptr: BinaryHandleGlobal,
581 Align: Align(M.getDataLayout().getPointerTypeSize(Ty: PtrTy)));
582 DtorBuilder.CreateCall(Callee: UnregFatbin, Args: BinaryHandle);
583 DtorBuilder.CreateRetVoid();
584
585 // Add this function to constructors.
586 appendToGlobalCtors(M, F: CtorFunc, /*Priority=*/101);
587}
588} // namespace
589
590Error offloading::wrapOpenMPBinaries(Module &M, ArrayRef<ArrayRef<char>> Images,
591 EntryArrayTy EntryArray,
592 llvm::StringRef Suffix, bool Relocatable) {
593 GlobalVariable *Desc =
594 createBinDesc(M, Bufs: Images, EntryArray, Suffix, Relocatable);
595 if (!Desc)
596 return createStringError(EC: inconvertibleErrorCode(),
597 Msg: "No binary descriptors created.");
598 createRegisterFunction(M, BinDesc: Desc, Suffix);
599 return Error::success();
600}
601
602Error offloading::wrapCudaBinary(Module &M, ArrayRef<char> Image,
603 EntryArrayTy EntryArray,
604 llvm::StringRef Suffix,
605 bool EmitSurfacesAndTextures) {
606 GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/IsHIP: false, Suffix);
607 if (!Desc)
608 return createStringError(EC: inconvertibleErrorCode(),
609 Msg: "No fatbin section created.");
610
611 createRegisterFatbinFunction(M, FatbinDesc: Desc, /*IsHip=*/IsHIP: false, EntryArray, Suffix,
612 EmitSurfacesAndTextures);
613 return Error::success();
614}
615
616Error offloading::wrapHIPBinary(Module &M, ArrayRef<char> Image,
617 EntryArrayTy EntryArray, llvm::StringRef Suffix,
618 bool EmitSurfacesAndTextures) {
619 GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/IsHIP: true, Suffix);
620 if (!Desc)
621 return createStringError(EC: inconvertibleErrorCode(),
622 Msg: "No fatbin section created.");
623
624 createRegisterFatbinFunction(M, FatbinDesc: Desc, /*IsHip=*/IsHIP: true, EntryArray, Suffix,
625 EmitSurfacesAndTextures);
626 return Error::success();
627}
628

source code of llvm/lib/Frontend/Offloading/OffloadWrapper.cpp