1 | //===- OffloadWrapper.cpp ---------------------------------------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "llvm/Frontend/Offloading/OffloadWrapper.h" |
10 | #include "llvm/ADT/ArrayRef.h" |
11 | #include "llvm/BinaryFormat/Magic.h" |
12 | #include "llvm/Frontend/Offloading/Utility.h" |
13 | #include "llvm/IR/Constants.h" |
14 | #include "llvm/IR/GlobalVariable.h" |
15 | #include "llvm/IR/IRBuilder.h" |
16 | #include "llvm/IR/LLVMContext.h" |
17 | #include "llvm/IR/Module.h" |
18 | #include "llvm/Object/OffloadBinary.h" |
19 | #include "llvm/Support/Error.h" |
20 | #include "llvm/TargetParser/Triple.h" |
21 | #include "llvm/Transforms/Utils/ModuleUtils.h" |
22 | |
23 | using namespace llvm; |
24 | using namespace llvm::offloading; |
25 | |
26 | namespace { |
27 | /// Magic number that begins the section containing the CUDA fatbinary. |
28 | constexpr unsigned CudaFatMagic = 0x466243b1; |
29 | constexpr unsigned HIPFatMagic = 0x48495046; |
30 | |
31 | IntegerType *getSizeTTy(Module &M) { |
32 | return M.getDataLayout().getIntPtrType(C&: M.getContext()); |
33 | } |
34 | |
35 | // struct __tgt_device_image { |
36 | // void *ImageStart; |
37 | // void *ImageEnd; |
38 | // __tgt_offload_entry *EntriesBegin; |
39 | // __tgt_offload_entry *EntriesEnd; |
40 | // }; |
41 | StructType *getDeviceImageTy(Module &M) { |
42 | LLVMContext &C = M.getContext(); |
43 | StructType *ImageTy = StructType::getTypeByName(C, Name: "__tgt_device_image" ); |
44 | if (!ImageTy) |
45 | ImageTy = |
46 | StructType::create(Name: "__tgt_device_image" , elt1: PointerType::getUnqual(C), |
47 | elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C), |
48 | elts: PointerType::getUnqual(C)); |
49 | return ImageTy; |
50 | } |
51 | |
52 | PointerType *getDeviceImagePtrTy(Module &M) { |
53 | return PointerType::getUnqual(ElementType: getDeviceImageTy(M)); |
54 | } |
55 | |
56 | // struct __tgt_bin_desc { |
57 | // int32_t NumDeviceImages; |
58 | // __tgt_device_image *DeviceImages; |
59 | // __tgt_offload_entry *HostEntriesBegin; |
60 | // __tgt_offload_entry *HostEntriesEnd; |
61 | // }; |
62 | StructType *getBinDescTy(Module &M) { |
63 | LLVMContext &C = M.getContext(); |
64 | StructType *DescTy = StructType::getTypeByName(C, Name: "__tgt_bin_desc" ); |
65 | if (!DescTy) |
66 | DescTy = StructType::create( |
67 | Name: "__tgt_bin_desc" , elt1: Type::getInt32Ty(C), elts: getDeviceImagePtrTy(M), |
68 | elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C)); |
69 | return DescTy; |
70 | } |
71 | |
72 | PointerType *getBinDescPtrTy(Module &M) { |
73 | return PointerType::getUnqual(ElementType: getBinDescTy(M)); |
74 | } |
75 | |
76 | /// Creates binary descriptor for the given device images. Binary descriptor |
77 | /// is an object that is passed to the offloading runtime at program startup |
78 | /// and it describes all device images available in the executable or shared |
79 | /// library. It is defined as follows |
80 | /// |
81 | /// __attribute__((visibility("hidden"))) |
82 | /// extern __tgt_offload_entry *__start_omp_offloading_entries; |
83 | /// __attribute__((visibility("hidden"))) |
84 | /// extern __tgt_offload_entry *__stop_omp_offloading_entries; |
85 | /// |
86 | /// static const char Image0[] = { <Bufs.front() contents> }; |
87 | /// ... |
88 | /// static const char ImageN[] = { <Bufs.back() contents> }; |
89 | /// |
90 | /// static const __tgt_device_image Images[] = { |
91 | /// { |
92 | /// Image0, /*ImageStart*/ |
93 | /// Image0 + sizeof(Image0), /*ImageEnd*/ |
94 | /// __start_omp_offloading_entries, /*EntriesBegin*/ |
95 | /// __stop_omp_offloading_entries /*EntriesEnd*/ |
96 | /// }, |
97 | /// ... |
98 | /// { |
99 | /// ImageN, /*ImageStart*/ |
100 | /// ImageN + sizeof(ImageN), /*ImageEnd*/ |
101 | /// __start_omp_offloading_entries, /*EntriesBegin*/ |
102 | /// __stop_omp_offloading_entries /*EntriesEnd*/ |
103 | /// } |
104 | /// }; |
105 | /// |
106 | /// static const __tgt_bin_desc BinDesc = { |
107 | /// sizeof(Images) / sizeof(Images[0]), /*NumDeviceImages*/ |
108 | /// Images, /*DeviceImages*/ |
109 | /// __start_omp_offloading_entries, /*HostEntriesBegin*/ |
110 | /// __stop_omp_offloading_entries /*HostEntriesEnd*/ |
111 | /// }; |
112 | /// |
113 | /// Global variable that represents BinDesc is returned. |
114 | GlobalVariable *createBinDesc(Module &M, ArrayRef<ArrayRef<char>> Bufs, |
115 | EntryArrayTy EntryArray, StringRef Suffix, |
116 | bool Relocatable) { |
117 | LLVMContext &C = M.getContext(); |
118 | auto [EntriesB, EntriesE] = EntryArray; |
119 | |
120 | auto *Zero = ConstantInt::get(Ty: getSizeTTy(M), V: 0u); |
121 | Constant *ZeroZero[] = {Zero, Zero}; |
122 | |
123 | // Create initializer for the images array. |
124 | SmallVector<Constant *, 4u> ImagesInits; |
125 | ImagesInits.reserve(N: Bufs.size()); |
126 | for (ArrayRef<char> Buf : Bufs) { |
127 | // We embed the full offloading entry so the binary utilities can parse it. |
128 | auto *Data = ConstantDataArray::get(Context&: C, Elts: Buf); |
129 | auto *Image = new GlobalVariable(M, Data->getType(), /*isConstant=*/true, |
130 | GlobalVariable::InternalLinkage, Data, |
131 | ".omp_offloading.device_image" + Suffix); |
132 | Image->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); |
133 | Image->setSection(Relocatable ? ".llvm.offloading.relocatable" |
134 | : ".llvm.offloading" ); |
135 | Image->setAlignment(Align(object::OffloadBinary::getAlignment())); |
136 | |
137 | StringRef Binary(Buf.data(), Buf.size()); |
138 | assert(identify_magic(Binary) == file_magic::offload_binary && |
139 | "Invalid binary format" ); |
140 | |
141 | // The device image struct contains the pointer to the beginning and end of |
142 | // the image stored inside of the offload binary. There should only be one |
143 | // of these for each buffer so we parse it out manually. |
144 | const auto * = |
145 | reinterpret_cast<const object::OffloadBinary::Header *>( |
146 | Binary.bytes_begin()); |
147 | const auto *Entry = reinterpret_cast<const object::OffloadBinary::Entry *>( |
148 | Binary.bytes_begin() + Header->EntryOffset); |
149 | |
150 | auto *Begin = ConstantInt::get(Ty: getSizeTTy(M), V: Entry->ImageOffset); |
151 | auto *Size = |
152 | ConstantInt::get(Ty: getSizeTTy(M), V: Entry->ImageOffset + Entry->ImageSize); |
153 | Constant *ZeroBegin[] = {Zero, Begin}; |
154 | Constant *ZeroSize[] = {Zero, Size}; |
155 | |
156 | auto *ImageB = |
157 | ConstantExpr::getGetElementPtr(Ty: Image->getValueType(), C: Image, IdxList: ZeroBegin); |
158 | auto *ImageE = |
159 | ConstantExpr::getGetElementPtr(Ty: Image->getValueType(), C: Image, IdxList: ZeroSize); |
160 | |
161 | ImagesInits.push_back(Elt: ConstantStruct::get(T: getDeviceImageTy(M), Vs: ImageB, |
162 | Vs: ImageE, Vs: EntriesB, Vs: EntriesE)); |
163 | } |
164 | |
165 | // Then create images array. |
166 | auto *ImagesData = ConstantArray::get( |
167 | T: ArrayType::get(ElementType: getDeviceImageTy(M), NumElements: ImagesInits.size()), V: ImagesInits); |
168 | |
169 | auto *Images = |
170 | new GlobalVariable(M, ImagesData->getType(), /*isConstant*/ true, |
171 | GlobalValue::InternalLinkage, ImagesData, |
172 | ".omp_offloading.device_images" + Suffix); |
173 | Images->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); |
174 | |
175 | auto *ImagesB = |
176 | ConstantExpr::getGetElementPtr(Ty: Images->getValueType(), C: Images, IdxList: ZeroZero); |
177 | |
178 | // And finally create the binary descriptor object. |
179 | auto *DescInit = ConstantStruct::get( |
180 | T: getBinDescTy(M), |
181 | Vs: ConstantInt::get(Ty: Type::getInt32Ty(C), V: ImagesInits.size()), Vs: ImagesB, |
182 | Vs: EntriesB, Vs: EntriesE); |
183 | |
184 | return new GlobalVariable(M, DescInit->getType(), /*isConstant*/ true, |
185 | GlobalValue::InternalLinkage, DescInit, |
186 | ".omp_offloading.descriptor" + Suffix); |
187 | } |
188 | |
189 | Function *createUnregisterFunction(Module &M, GlobalVariable *BinDesc, |
190 | StringRef Suffix) { |
191 | LLVMContext &C = M.getContext(); |
192 | auto *FuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false); |
193 | auto *Func = |
194 | Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage, |
195 | N: ".omp_offloading.descriptor_unreg" + Suffix, M: &M); |
196 | Func->setSection(".text.startup" ); |
197 | |
198 | // Get __tgt_unregister_lib function declaration. |
199 | auto *UnRegFuncTy = FunctionType::get(Result: Type::getVoidTy(C), Params: getBinDescPtrTy(M), |
200 | /*isVarArg*/ false); |
201 | FunctionCallee UnRegFuncC = |
202 | M.getOrInsertFunction(Name: "__tgt_unregister_lib" , T: UnRegFuncTy); |
203 | |
204 | // Construct function body |
205 | IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry" , Parent: Func)); |
206 | Builder.CreateCall(Callee: UnRegFuncC, Args: BinDesc); |
207 | Builder.CreateRetVoid(); |
208 | |
209 | return Func; |
210 | } |
211 | |
212 | void createRegisterFunction(Module &M, GlobalVariable *BinDesc, |
213 | StringRef Suffix) { |
214 | LLVMContext &C = M.getContext(); |
215 | auto *FuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false); |
216 | auto *Func = Function::Create(Ty: FuncTy, Linkage: GlobalValue::InternalLinkage, |
217 | N: ".omp_offloading.descriptor_reg" + Suffix, M: &M); |
218 | Func->setSection(".text.startup" ); |
219 | |
220 | // Get __tgt_register_lib function declaration. |
221 | auto *RegFuncTy = FunctionType::get(Result: Type::getVoidTy(C), Params: getBinDescPtrTy(M), |
222 | /*isVarArg*/ false); |
223 | FunctionCallee RegFuncC = |
224 | M.getOrInsertFunction(Name: "__tgt_register_lib" , T: RegFuncTy); |
225 | |
226 | auto *AtExitTy = FunctionType::get( |
227 | Result: Type::getInt32Ty(C), Params: PointerType::getUnqual(C), /*isVarArg=*/false); |
228 | FunctionCallee AtExit = M.getOrInsertFunction(Name: "atexit" , T: AtExitTy); |
229 | |
230 | Function *UnregFunc = createUnregisterFunction(M, BinDesc, Suffix); |
231 | |
232 | // Construct function body |
233 | IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry" , Parent: Func)); |
234 | |
235 | // Register the destructors with 'atexit'. This is expected by the CUDA |
236 | // runtime and ensures that we clean up before dynamic objects are destroyed. |
237 | // This needs to be done before the runtime is called and registers its own. |
238 | Builder.CreateCall(Callee: AtExit, Args: UnregFunc); |
239 | |
240 | Builder.CreateCall(Callee: RegFuncC, Args: BinDesc); |
241 | Builder.CreateRetVoid(); |
242 | |
243 | // Add this function to constructors. |
244 | appendToGlobalCtors(M, F: Func, /*Priority=*/101); |
245 | } |
246 | |
247 | // struct fatbin_wrapper { |
248 | // int32_t magic; |
249 | // int32_t version; |
250 | // void *image; |
251 | // void *reserved; |
252 | //}; |
253 | StructType *getFatbinWrapperTy(Module &M) { |
254 | LLVMContext &C = M.getContext(); |
255 | StructType *FatbinTy = StructType::getTypeByName(C, Name: "fatbin_wrapper" ); |
256 | if (!FatbinTy) |
257 | FatbinTy = StructType::create( |
258 | Name: "fatbin_wrapper" , elt1: Type::getInt32Ty(C), elts: Type::getInt32Ty(C), |
259 | elts: PointerType::getUnqual(C), elts: PointerType::getUnqual(C)); |
260 | return FatbinTy; |
261 | } |
262 | |
263 | /// Embed the image \p Image into the module \p M so it can be found by the |
264 | /// runtime. |
265 | GlobalVariable *createFatbinDesc(Module &M, ArrayRef<char> Image, bool IsHIP, |
266 | StringRef Suffix) { |
267 | LLVMContext &C = M.getContext(); |
268 | llvm::Type *Int8PtrTy = PointerType::getUnqual(C); |
269 | llvm::Triple Triple = llvm::Triple(M.getTargetTriple()); |
270 | |
271 | // Create the global string containing the fatbinary. |
272 | StringRef FatbinConstantSection = |
273 | IsHIP ? ".hip_fatbin" |
274 | : (Triple.isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin" ); |
275 | auto *Data = ConstantDataArray::get(Context&: C, Elts: Image); |
276 | auto *Fatbin = new GlobalVariable(M, Data->getType(), /*isConstant*/ true, |
277 | GlobalVariable::InternalLinkage, Data, |
278 | ".fatbin_image" + Suffix); |
279 | Fatbin->setSection(FatbinConstantSection); |
280 | |
281 | // Create the fatbinary wrapper |
282 | StringRef FatbinWrapperSection = IsHIP ? ".hipFatBinSegment" |
283 | : Triple.isMacOSX() ? "__NV_CUDA,__fatbin" |
284 | : ".nvFatBinSegment" ; |
285 | Constant *FatbinWrapper[] = { |
286 | ConstantInt::get(Ty: Type::getInt32Ty(C), V: IsHIP ? HIPFatMagic : CudaFatMagic), |
287 | ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1), |
288 | ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: Fatbin, Ty: Int8PtrTy), |
289 | ConstantPointerNull::get(T: PointerType::getUnqual(C))}; |
290 | |
291 | Constant *FatbinInitializer = |
292 | ConstantStruct::get(T: getFatbinWrapperTy(M), V: FatbinWrapper); |
293 | |
294 | auto *FatbinDesc = |
295 | new GlobalVariable(M, getFatbinWrapperTy(M), |
296 | /*isConstant*/ true, GlobalValue::InternalLinkage, |
297 | FatbinInitializer, ".fatbin_wrapper" + Suffix); |
298 | FatbinDesc->setSection(FatbinWrapperSection); |
299 | FatbinDesc->setAlignment(Align(8)); |
300 | |
301 | return FatbinDesc; |
302 | } |
303 | |
304 | /// Create the register globals function. We will iterate all of the offloading |
305 | /// entries stored at the begin / end symbols and register them according to |
306 | /// their type. This creates the following function in IR: |
307 | /// |
308 | /// extern struct __tgt_offload_entry __start_cuda_offloading_entries; |
309 | /// extern struct __tgt_offload_entry __stop_cuda_offloading_entries; |
310 | /// |
311 | /// extern void __cudaRegisterFunction(void **, void *, void *, void *, int, |
312 | /// void *, void *, void *, void *, int *); |
313 | /// extern void __cudaRegisterVar(void **, void *, void *, void *, int32_t, |
314 | /// int64_t, int32_t, int32_t); |
315 | /// |
316 | /// void __cudaRegisterTest(void **fatbinHandle) { |
317 | /// for (struct __tgt_offload_entry *entry = &__start_cuda_offloading_entries; |
318 | /// entry != &__stop_cuda_offloading_entries; ++entry) { |
319 | /// if (!entry->size) |
320 | /// __cudaRegisterFunction(fatbinHandle, entry->addr, entry->name, |
321 | /// entry->name, -1, 0, 0, 0, 0, 0); |
322 | /// else |
323 | /// __cudaRegisterVar(fatbinHandle, entry->addr, entry->name, entry->name, |
324 | /// 0, entry->size, 0, 0); |
325 | /// } |
326 | /// } |
327 | Function *createRegisterGlobalsFunction(Module &M, bool IsHIP, |
328 | EntryArrayTy EntryArray, |
329 | StringRef Suffix, |
330 | bool EmitSurfacesAndTextures) { |
331 | LLVMContext &C = M.getContext(); |
332 | auto [EntriesB, EntriesE] = EntryArray; |
333 | |
334 | // Get the __cudaRegisterFunction function declaration. |
335 | PointerType *Int8PtrTy = PointerType::get(C, AddressSpace: 0); |
336 | PointerType *Int8PtrPtrTy = PointerType::get(C, AddressSpace: 0); |
337 | PointerType *Int32PtrTy = PointerType::get(C, AddressSpace: 0); |
338 | auto *RegFuncTy = FunctionType::get( |
339 | Result: Type::getInt32Ty(C), |
340 | Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C), |
341 | Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int32PtrTy}, |
342 | /*isVarArg*/ false); |
343 | FunctionCallee RegFunc = M.getOrInsertFunction( |
344 | Name: IsHIP ? "__hipRegisterFunction" : "__cudaRegisterFunction" , T: RegFuncTy); |
345 | |
346 | // Get the __cudaRegisterVar function declaration. |
347 | auto *RegVarTy = FunctionType::get( |
348 | Result: Type::getVoidTy(C), |
349 | Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C), |
350 | getSizeTTy(M), Type::getInt32Ty(C), Type::getInt32Ty(C)}, |
351 | /*isVarArg*/ false); |
352 | FunctionCallee RegVar = M.getOrInsertFunction( |
353 | Name: IsHIP ? "__hipRegisterVar" : "__cudaRegisterVar" , T: RegVarTy); |
354 | |
355 | // Get the __cudaRegisterSurface function declaration. |
356 | FunctionType *RegSurfaceTy = |
357 | FunctionType::get(Result: Type::getVoidTy(C), |
358 | Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, |
359 | Type::getInt32Ty(C), Type::getInt32Ty(C)}, |
360 | /*isVarArg=*/false); |
361 | FunctionCallee RegSurface = M.getOrInsertFunction( |
362 | Name: IsHIP ? "__hipRegisterSurface" : "__cudaRegisterSurface" , T: RegSurfaceTy); |
363 | |
364 | // Get the __cudaRegisterTexture function declaration. |
365 | FunctionType *RegTextureTy = FunctionType::get( |
366 | Result: Type::getVoidTy(C), |
367 | Params: {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C), |
368 | Type::getInt32Ty(C), Type::getInt32Ty(C)}, |
369 | /*isVarArg=*/false); |
370 | FunctionCallee RegTexture = M.getOrInsertFunction( |
371 | Name: IsHIP ? "__hipRegisterTexture" : "__cudaRegisterTexture" , T: RegTextureTy); |
372 | |
373 | auto *RegGlobalsTy = FunctionType::get(Result: Type::getVoidTy(C), Params: Int8PtrPtrTy, |
374 | /*isVarArg*/ false); |
375 | auto *RegGlobalsFn = |
376 | Function::Create(Ty: RegGlobalsTy, Linkage: GlobalValue::InternalLinkage, |
377 | N: IsHIP ? ".hip.globals_reg" : ".cuda.globals_reg" , M: &M); |
378 | RegGlobalsFn->setSection(".text.startup" ); |
379 | |
380 | // Create the loop to register all the entries. |
381 | IRBuilder<> Builder(BasicBlock::Create(Context&: C, Name: "entry" , Parent: RegGlobalsFn)); |
382 | auto *EntryBB = BasicBlock::Create(Context&: C, Name: "while.entry" , Parent: RegGlobalsFn); |
383 | auto *IfThenBB = BasicBlock::Create(Context&: C, Name: "if.then" , Parent: RegGlobalsFn); |
384 | auto *IfElseBB = BasicBlock::Create(Context&: C, Name: "if.else" , Parent: RegGlobalsFn); |
385 | auto *SwGlobalBB = BasicBlock::Create(Context&: C, Name: "sw.global" , Parent: RegGlobalsFn); |
386 | auto *SwManagedBB = BasicBlock::Create(Context&: C, Name: "sw.managed" , Parent: RegGlobalsFn); |
387 | auto *SwSurfaceBB = BasicBlock::Create(Context&: C, Name: "sw.surface" , Parent: RegGlobalsFn); |
388 | auto *SwTextureBB = BasicBlock::Create(Context&: C, Name: "sw.texture" , Parent: RegGlobalsFn); |
389 | auto *IfEndBB = BasicBlock::Create(Context&: C, Name: "if.end" , Parent: RegGlobalsFn); |
390 | auto *ExitBB = BasicBlock::Create(Context&: C, Name: "while.end" , Parent: RegGlobalsFn); |
391 | |
392 | auto *EntryCmp = Builder.CreateICmpNE(LHS: EntriesB, RHS: EntriesE); |
393 | Builder.CreateCondBr(Cond: EntryCmp, True: EntryBB, False: ExitBB); |
394 | Builder.SetInsertPoint(EntryBB); |
395 | auto *Entry = Builder.CreatePHI(Ty: PointerType::getUnqual(C), NumReservedValues: 2, Name: "entry" ); |
396 | auto *AddrPtr = |
397 | Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry, |
398 | IdxList: {ConstantInt::get(Ty: getSizeTTy(M), V: 0), |
399 | ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0)}); |
400 | auto *Addr = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: AddrPtr, Name: "addr" ); |
401 | auto *NamePtr = |
402 | Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry, |
403 | IdxList: {ConstantInt::get(Ty: getSizeTTy(M), V: 0), |
404 | ConstantInt::get(Ty: Type::getInt32Ty(C), V: 1)}); |
405 | auto *Name = Builder.CreateLoad(Ty: Int8PtrTy, Ptr: NamePtr, Name: "name" ); |
406 | auto *SizePtr = |
407 | Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry, |
408 | IdxList: {ConstantInt::get(Ty: getSizeTTy(M), V: 0), |
409 | ConstantInt::get(Ty: Type::getInt32Ty(C), V: 2)}); |
410 | auto *Size = Builder.CreateLoad(Ty: getSizeTTy(M), Ptr: SizePtr, Name: "size" ); |
411 | auto *FlagsPtr = |
412 | Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry, |
413 | IdxList: {ConstantInt::get(Ty: getSizeTTy(M), V: 0), |
414 | ConstantInt::get(Ty: Type::getInt32Ty(C), V: 3)}); |
415 | auto *Flags = Builder.CreateLoad(Ty: Type::getInt32Ty(C), Ptr: FlagsPtr, Name: "flags" ); |
416 | auto *DataPtr = |
417 | Builder.CreateInBoundsGEP(Ty: offloading::getEntryTy(M), Ptr: Entry, |
418 | IdxList: {ConstantInt::get(Ty: getSizeTTy(M), V: 0), |
419 | ConstantInt::get(Ty: Type::getInt32Ty(C), V: 4)}); |
420 | auto *Data = Builder.CreateLoad(Ty: Type::getInt32Ty(C), Ptr: DataPtr, Name: "textype" ); |
421 | auto *Kind = Builder.CreateAnd( |
422 | LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0x7), Name: "type" ); |
423 | |
424 | // Extract the flags stored in the bit-field and convert them to C booleans. |
425 | auto *ExternBit = Builder.CreateAnd( |
426 | LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), |
427 | V: llvm::offloading::OffloadGlobalExtern)); |
428 | auto *Extern = Builder.CreateLShr( |
429 | LHS: ExternBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 3), Name: "extern" ); |
430 | auto *ConstantBit = Builder.CreateAnd( |
431 | LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), |
432 | V: llvm::offloading::OffloadGlobalConstant)); |
433 | auto *Const = Builder.CreateLShr( |
434 | LHS: ConstantBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 4), Name: "constant" ); |
435 | auto *NormalizedBit = Builder.CreateAnd( |
436 | LHS: Flags, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), |
437 | V: llvm::offloading::OffloadGlobalNormalized)); |
438 | auto *Normalized = Builder.CreateLShr( |
439 | LHS: NormalizedBit, RHS: ConstantInt::get(Ty: Type::getInt32Ty(C), V: 5), Name: "normalized" ); |
440 | auto *FnCond = |
441 | Builder.CreateICmpEQ(LHS: Size, RHS: ConstantInt::getNullValue(Ty: getSizeTTy(M))); |
442 | Builder.CreateCondBr(Cond: FnCond, True: IfThenBB, False: IfElseBB); |
443 | |
444 | // Create kernel registration code. |
445 | Builder.SetInsertPoint(IfThenBB); |
446 | Builder.CreateCall(Callee: RegFunc, Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name, |
447 | ConstantInt::get(Ty: Type::getInt32Ty(C), V: -1), |
448 | ConstantPointerNull::get(T: Int8PtrTy), |
449 | ConstantPointerNull::get(T: Int8PtrTy), |
450 | ConstantPointerNull::get(T: Int8PtrTy), |
451 | ConstantPointerNull::get(T: Int8PtrTy), |
452 | ConstantPointerNull::get(T: Int32PtrTy)}); |
453 | Builder.CreateBr(Dest: IfEndBB); |
454 | Builder.SetInsertPoint(IfElseBB); |
455 | |
456 | auto *Switch = Builder.CreateSwitch(V: Kind, Dest: IfEndBB); |
457 | // Create global variable registration code. |
458 | Builder.SetInsertPoint(SwGlobalBB); |
459 | Builder.CreateCall(Callee: RegVar, |
460 | Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name, Extern, Size, |
461 | Const, ConstantInt::get(Ty: Type::getInt32Ty(C), V: 0)}); |
462 | Builder.CreateBr(Dest: IfEndBB); |
463 | Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalEntry), |
464 | Dest: SwGlobalBB); |
465 | |
466 | // Create managed variable registration code. |
467 | Builder.SetInsertPoint(SwManagedBB); |
468 | Builder.CreateBr(Dest: IfEndBB); |
469 | Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalManagedEntry), |
470 | Dest: SwManagedBB); |
471 | // Create surface variable registration code. |
472 | Builder.SetInsertPoint(SwSurfaceBB); |
473 | if (EmitSurfacesAndTextures) |
474 | Builder.CreateCall(Callee: RegSurface, Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name, |
475 | Data, Extern}); |
476 | Builder.CreateBr(Dest: IfEndBB); |
477 | Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalSurfaceEntry), |
478 | Dest: SwSurfaceBB); |
479 | |
480 | // Create texture variable registration code. |
481 | Builder.SetInsertPoint(SwTextureBB); |
482 | if (EmitSurfacesAndTextures) |
483 | Builder.CreateCall(Callee: RegTexture, Args: {RegGlobalsFn->arg_begin(), Addr, Name, Name, |
484 | Data, Normalized, Extern}); |
485 | Builder.CreateBr(Dest: IfEndBB); |
486 | Switch->addCase(OnVal: Builder.getInt32(C: llvm::offloading::OffloadGlobalTextureEntry), |
487 | Dest: SwTextureBB); |
488 | |
489 | Builder.SetInsertPoint(IfEndBB); |
490 | auto *NewEntry = Builder.CreateInBoundsGEP( |
491 | Ty: offloading::getEntryTy(M), Ptr: Entry, IdxList: ConstantInt::get(Ty: getSizeTTy(M), V: 1)); |
492 | auto *Cmp = Builder.CreateICmpEQ( |
493 | LHS: NewEntry, |
494 | RHS: ConstantExpr::getInBoundsGetElementPtr( |
495 | Ty: ArrayType::get(ElementType: offloading::getEntryTy(M), NumElements: 0), C: EntriesE, |
496 | IdxList: ArrayRef<Constant *>({ConstantInt::get(Ty: getSizeTTy(M), V: 0), |
497 | ConstantInt::get(Ty: getSizeTTy(M), V: 0)}))); |
498 | Entry->addIncoming( |
499 | V: ConstantExpr::getInBoundsGetElementPtr( |
500 | Ty: ArrayType::get(ElementType: offloading::getEntryTy(M), NumElements: 0), C: EntriesB, |
501 | IdxList: ArrayRef<Constant *>({ConstantInt::get(Ty: getSizeTTy(M), V: 0), |
502 | ConstantInt::get(Ty: getSizeTTy(M), V: 0)})), |
503 | BB: &RegGlobalsFn->getEntryBlock()); |
504 | Entry->addIncoming(V: NewEntry, BB: IfEndBB); |
505 | Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: EntryBB); |
506 | Builder.SetInsertPoint(ExitBB); |
507 | Builder.CreateRetVoid(); |
508 | |
509 | return RegGlobalsFn; |
510 | } |
511 | |
512 | // Create the constructor and destructor to register the fatbinary with the CUDA |
513 | // runtime. |
514 | void createRegisterFatbinFunction(Module &M, GlobalVariable *FatbinDesc, |
515 | bool IsHIP, EntryArrayTy EntryArray, |
516 | StringRef Suffix, |
517 | bool EmitSurfacesAndTextures) { |
518 | LLVMContext &C = M.getContext(); |
519 | auto *CtorFuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false); |
520 | auto *CtorFunc = Function::Create( |
521 | Ty: CtorFuncTy, Linkage: GlobalValue::InternalLinkage, |
522 | N: (IsHIP ? ".hip.fatbin_reg" : ".cuda.fatbin_reg" ) + Suffix, M: &M); |
523 | CtorFunc->setSection(".text.startup" ); |
524 | |
525 | auto *DtorFuncTy = FunctionType::get(Result: Type::getVoidTy(C), /*isVarArg*/ false); |
526 | auto *DtorFunc = Function::Create( |
527 | Ty: DtorFuncTy, Linkage: GlobalValue::InternalLinkage, |
528 | N: (IsHIP ? ".hip.fatbin_unreg" : ".cuda.fatbin_unreg" ) + Suffix, M: &M); |
529 | DtorFunc->setSection(".text.startup" ); |
530 | |
531 | auto *PtrTy = PointerType::getUnqual(C); |
532 | |
533 | // Get the __cudaRegisterFatBinary function declaration. |
534 | auto *RegFatTy = FunctionType::get(Result: PtrTy, Params: PtrTy, /*isVarArg=*/false); |
535 | FunctionCallee RegFatbin = M.getOrInsertFunction( |
536 | Name: IsHIP ? "__hipRegisterFatBinary" : "__cudaRegisterFatBinary" , T: RegFatTy); |
537 | // Get the __cudaRegisterFatBinaryEnd function declaration. |
538 | auto *RegFatEndTy = |
539 | FunctionType::get(Result: Type::getVoidTy(C), Params: PtrTy, /*isVarArg=*/false); |
540 | FunctionCallee RegFatbinEnd = |
541 | M.getOrInsertFunction(Name: "__cudaRegisterFatBinaryEnd" , T: RegFatEndTy); |
542 | // Get the __cudaUnregisterFatBinary function declaration. |
543 | auto *UnregFatTy = |
544 | FunctionType::get(Result: Type::getVoidTy(C), Params: PtrTy, /*isVarArg=*/false); |
545 | FunctionCallee UnregFatbin = M.getOrInsertFunction( |
546 | Name: IsHIP ? "__hipUnregisterFatBinary" : "__cudaUnregisterFatBinary" , |
547 | T: UnregFatTy); |
548 | |
549 | auto *AtExitTy = |
550 | FunctionType::get(Result: Type::getInt32Ty(C), Params: PtrTy, /*isVarArg=*/false); |
551 | FunctionCallee AtExit = M.getOrInsertFunction(Name: "atexit" , T: AtExitTy); |
552 | |
553 | auto *BinaryHandleGlobal = new llvm::GlobalVariable( |
554 | M, PtrTy, false, llvm::GlobalValue::InternalLinkage, |
555 | llvm::ConstantPointerNull::get(T: PtrTy), |
556 | (IsHIP ? ".hip.binary_handle" : ".cuda.binary_handle" ) + Suffix); |
557 | |
558 | // Create the constructor to register this image with the runtime. |
559 | IRBuilder<> CtorBuilder(BasicBlock::Create(Context&: C, Name: "entry" , Parent: CtorFunc)); |
560 | CallInst *Handle = CtorBuilder.CreateCall( |
561 | Callee: RegFatbin, |
562 | Args: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: FatbinDesc, Ty: PtrTy)); |
563 | CtorBuilder.CreateAlignedStore( |
564 | Val: Handle, Ptr: BinaryHandleGlobal, |
565 | Align: Align(M.getDataLayout().getPointerTypeSize(Ty: PtrTy))); |
566 | CtorBuilder.CreateCall(Callee: createRegisterGlobalsFunction(M, IsHIP, EntryArray, |
567 | Suffix, |
568 | EmitSurfacesAndTextures), |
569 | Args: Handle); |
570 | if (!IsHIP) |
571 | CtorBuilder.CreateCall(Callee: RegFatbinEnd, Args: Handle); |
572 | CtorBuilder.CreateCall(Callee: AtExit, Args: DtorFunc); |
573 | CtorBuilder.CreateRetVoid(); |
574 | |
575 | // Create the destructor to unregister the image with the runtime. We cannot |
576 | // use a standard global destructor after CUDA 9.2 so this must be called by |
577 | // `atexit()` intead. |
578 | IRBuilder<> DtorBuilder(BasicBlock::Create(Context&: C, Name: "entry" , Parent: DtorFunc)); |
579 | LoadInst *BinaryHandle = DtorBuilder.CreateAlignedLoad( |
580 | Ty: PtrTy, Ptr: BinaryHandleGlobal, |
581 | Align: Align(M.getDataLayout().getPointerTypeSize(Ty: PtrTy))); |
582 | DtorBuilder.CreateCall(Callee: UnregFatbin, Args: BinaryHandle); |
583 | DtorBuilder.CreateRetVoid(); |
584 | |
585 | // Add this function to constructors. |
586 | appendToGlobalCtors(M, F: CtorFunc, /*Priority=*/101); |
587 | } |
588 | } // namespace |
589 | |
590 | Error offloading::wrapOpenMPBinaries(Module &M, ArrayRef<ArrayRef<char>> Images, |
591 | EntryArrayTy EntryArray, |
592 | llvm::StringRef Suffix, bool Relocatable) { |
593 | GlobalVariable *Desc = |
594 | createBinDesc(M, Bufs: Images, EntryArray, Suffix, Relocatable); |
595 | if (!Desc) |
596 | return createStringError(EC: inconvertibleErrorCode(), |
597 | Msg: "No binary descriptors created." ); |
598 | createRegisterFunction(M, BinDesc: Desc, Suffix); |
599 | return Error::success(); |
600 | } |
601 | |
602 | Error offloading::wrapCudaBinary(Module &M, ArrayRef<char> Image, |
603 | EntryArrayTy EntryArray, |
604 | llvm::StringRef Suffix, |
605 | bool EmitSurfacesAndTextures) { |
606 | GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/IsHIP: false, Suffix); |
607 | if (!Desc) |
608 | return createStringError(EC: inconvertibleErrorCode(), |
609 | Msg: "No fatbin section created." ); |
610 | |
611 | createRegisterFatbinFunction(M, FatbinDesc: Desc, /*IsHip=*/IsHIP: false, EntryArray, Suffix, |
612 | EmitSurfacesAndTextures); |
613 | return Error::success(); |
614 | } |
615 | |
616 | Error offloading::wrapHIPBinary(Module &M, ArrayRef<char> Image, |
617 | EntryArrayTy EntryArray, llvm::StringRef Suffix, |
618 | bool EmitSurfacesAndTextures) { |
619 | GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/IsHIP: true, Suffix); |
620 | if (!Desc) |
621 | return createStringError(EC: inconvertibleErrorCode(), |
622 | Msg: "No fatbin section created." ); |
623 | |
624 | createRegisterFatbinFunction(M, FatbinDesc: Desc, /*IsHip=*/IsHIP: true, EntryArray, Suffix, |
625 | EmitSurfacesAndTextures); |
626 | return Error::success(); |
627 | } |
628 | |