1 | //===-- PluginManager.cpp - Plugin loading and communication API ---------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // Functionality for handling plugins. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "PluginManager.h" |
14 | #include "Shared/Debug.h" |
15 | #include "Shared/Profile.h" |
16 | |
17 | #include "llvm/Support/Error.h" |
18 | #include "llvm/Support/ErrorHandling.h" |
19 | #include <memory> |
20 | |
21 | using namespace llvm; |
22 | using namespace llvm::sys; |
23 | |
24 | PluginManager *PM; |
25 | |
26 | // List of all plugins that can support offloading. |
27 | static const char *RTLNames[] = {ENABLED_OFFLOAD_PLUGINS}; |
28 | |
29 | Expected<std::unique_ptr<PluginAdaptorTy>> |
30 | PluginAdaptorTy::create(const std::string &Name) { |
31 | DP("Attempting to load library '%s'...\n" , Name.c_str()); |
32 | TIMESCOPE_WITH_NAME_AND_IDENT(Name, (const ident_t *)nullptr); |
33 | |
34 | std::string ErrMsg; |
35 | auto LibraryHandler = std::make_unique<DynamicLibrary>( |
36 | args: DynamicLibrary::getPermanentLibrary(filename: Name.c_str(), errMsg: &ErrMsg)); |
37 | |
38 | if (!LibraryHandler->isValid()) { |
39 | // Library does not exist or cannot be found. |
40 | return createStringError(EC: inconvertibleErrorCode(), |
41 | Fmt: "Unable to load library '%s': %s!\n" , Vals: Name.c_str(), |
42 | Vals: ErrMsg.c_str()); |
43 | } |
44 | |
45 | DP("Successfully loaded library '%s'!\n" , Name.c_str()); |
46 | auto PluginAdaptor = std::unique_ptr<PluginAdaptorTy>( |
47 | new PluginAdaptorTy(Name, std::move(LibraryHandler))); |
48 | if (auto Err = PluginAdaptor->init()) |
49 | return Err; |
50 | return std::move(PluginAdaptor); |
51 | } |
52 | |
53 | PluginAdaptorTy::PluginAdaptorTy(const std::string &Name, |
54 | std::unique_ptr<llvm::sys::DynamicLibrary> DL) |
55 | : Name(Name), LibraryHandler(std::move(DL)) {} |
56 | |
57 | Error PluginAdaptorTy::init() { |
58 | |
59 | #define PLUGIN_API_HANDLE(NAME, MANDATORY) \ |
60 | NAME = reinterpret_cast<decltype(NAME)>( \ |
61 | LibraryHandler->getAddressOfSymbol(GETNAME(__tgt_rtl_##NAME))); \ |
62 | if (MANDATORY && !NAME) { \ |
63 | return createStringError(inconvertibleErrorCode(), \ |
64 | "Invalid plugin as necessary interface function " \ |
65 | "(%s) was not found.\n", \ |
66 | std::string(#NAME).c_str()); \ |
67 | } |
68 | |
69 | #include "Shared/PluginAPI.inc" |
70 | #undef PLUGIN_API_HANDLE |
71 | |
72 | // Remove plugin on failure to call optional init_plugin |
73 | int32_t Rc = init_plugin(); |
74 | if (Rc != OFFLOAD_SUCCESS) { |
75 | return createStringError(EC: inconvertibleErrorCode(), |
76 | Fmt: "Unable to initialize library '%s': %u!\n" , |
77 | Vals: Name.c_str(), Vals: Rc); |
78 | } |
79 | |
80 | // No devices are supported by this RTL? |
81 | NumberOfPluginDevices = number_of_devices(); |
82 | if (!NumberOfPluginDevices) { |
83 | return createStringError(EC: inconvertibleErrorCode(), |
84 | Msg: "No devices supported in this RTL\n" ); |
85 | } |
86 | |
87 | DP("Registered '%s' with %d plugin visible devices!\n" , Name.c_str(), |
88 | NumberOfPluginDevices); |
89 | return Error::success(); |
90 | } |
91 | |
92 | void PluginManager::init() { |
93 | TIMESCOPE(); |
94 | DP("Loading RTLs...\n" ); |
95 | |
96 | // Attempt to open all the plugins and, if they exist, check if the interface |
97 | // is correct and if they are supporting any devices. |
98 | for (const char *Name : RTLNames) { |
99 | auto PluginAdaptorOrErr = |
100 | PluginAdaptorTy::create(Name: std::string(Name) + ".so" ); |
101 | if (!PluginAdaptorOrErr) { |
102 | [[maybe_unused]] std::string InfoMsg = |
103 | toString(E: PluginAdaptorOrErr.takeError()); |
104 | DP("%s" , InfoMsg.c_str()); |
105 | } else { |
106 | PluginAdaptors.push_back(Elt: std::move(*PluginAdaptorOrErr)); |
107 | } |
108 | } |
109 | |
110 | DP("RTLs loaded!\n" ); |
111 | } |
112 | |
113 | void PluginAdaptorTy::initDevices(PluginManager &PM) { |
114 | if (isUsed()) |
115 | return; |
116 | TIMESCOPE(); |
117 | |
118 | // If this RTL is not already in use, initialize it. |
119 | assert(getNumberOfPluginDevices() > 0 && |
120 | "Tried to initialize useless plugin adaptor" ); |
121 | |
122 | // Initialize the device information for the RTL we are about to use. |
123 | auto ExclusiveDevicesAccessor = PM.getExclusiveDevicesAccessor(); |
124 | |
125 | // Initialize the index of this RTL and save it in the used RTLs. |
126 | DeviceOffset = ExclusiveDevicesAccessor->size(); |
127 | |
128 | // If possible, set the device identifier offset in the plugin. |
129 | if (set_device_offset) |
130 | set_device_offset(DeviceOffset); |
131 | |
132 | int32_t NumPD = getNumberOfPluginDevices(); |
133 | ExclusiveDevicesAccessor->reserve(N: DeviceOffset + NumPD); |
134 | // Auto zero-copy is a per-device property. We need to ensure |
135 | // that all devices are suggesting to use it. |
136 | bool UseAutoZeroCopy = !(NumPD == 0); |
137 | for (int32_t PDevI = 0, UserDevId = DeviceOffset; PDevI < NumPD; PDevI++) { |
138 | auto Device = std::make_unique<DeviceTy>(args: this, args&: UserDevId, args&: PDevI); |
139 | if (auto Err = Device->init()) { |
140 | DP("Skip plugin known device %d: %s\n" , PDevI, |
141 | toString(std::move(Err)).c_str()); |
142 | continue; |
143 | } |
144 | UseAutoZeroCopy = UseAutoZeroCopy && Device->useAutoZeroCopy(); |
145 | |
146 | ExclusiveDevicesAccessor->push_back(Elt: std::move(Device)); |
147 | ++NumberOfUserDevices; |
148 | ++UserDevId; |
149 | } |
150 | |
151 | // Auto Zero-Copy can only be currently triggered when the system is an |
152 | // homogeneous APU architecture without attached discrete GPUs. |
153 | // If all devices suggest to use it, change requirment flags to trigger |
154 | // zero-copy behavior when mapping memory. |
155 | if (UseAutoZeroCopy) |
156 | PM.addRequirements(Flags: OMPX_REQ_AUTO_ZERO_COPY); |
157 | |
158 | DP("Plugin adaptor " DPxMOD " has index %d, exposes %d out of %d devices!\n" , |
159 | DPxPTR(LibraryHandler.get()), DeviceOffset, NumberOfUserDevices, |
160 | NumberOfPluginDevices); |
161 | } |
162 | |
163 | void PluginManager::initAllPlugins() { |
164 | for (auto &R : PluginAdaptors) |
165 | R->initDevices(PM&: *this); |
166 | } |
167 | |
168 | static void registerImageIntoTranslationTable(TranslationTable &TT, |
169 | PluginAdaptorTy &RTL, |
170 | __tgt_device_image *Image) { |
171 | |
172 | // same size, as when we increase one, we also increase the other. |
173 | assert(TT.TargetsTable.size() == TT.TargetsImages.size() && |
174 | "We should have as many images as we have tables!" ); |
175 | |
176 | // Resize the Targets Table and Images to accommodate the new targets if |
177 | // required |
178 | unsigned TargetsTableMinimumSize = |
179 | RTL.DeviceOffset + RTL.getNumberOfUserDevices(); |
180 | |
181 | if (TT.TargetsTable.size() < TargetsTableMinimumSize) { |
182 | TT.DeviceTables.resize(N: TargetsTableMinimumSize, NV: {}); |
183 | TT.TargetsImages.resize(N: TargetsTableMinimumSize, NV: 0); |
184 | TT.TargetsEntries.resize(N: TargetsTableMinimumSize, NV: {}); |
185 | TT.TargetsTable.resize(N: TargetsTableMinimumSize, NV: 0); |
186 | } |
187 | |
188 | // Register the image in all devices for this target type. |
189 | for (int32_t I = 0; I < RTL.getNumberOfUserDevices(); ++I) { |
190 | // If we are changing the image we are also invalidating the target table. |
191 | if (TT.TargetsImages[RTL.DeviceOffset + I] != Image) { |
192 | TT.TargetsImages[RTL.DeviceOffset + I] = Image; |
193 | TT.TargetsTable[RTL.DeviceOffset + I] = |
194 | 0; // lazy initialization of target table. |
195 | } |
196 | } |
197 | } |
198 | |
199 | void PluginManager::registerLib(__tgt_bin_desc *Desc) { |
200 | PM->RTLsMtx.lock(); |
201 | |
202 | // Extract the exectuable image and extra information if availible. |
203 | for (int32_t i = 0; i < Desc->NumDeviceImages; ++i) |
204 | PM->addDeviceImage(TgtBinDesc&: *Desc, TgtDeviceImage&: Desc->DeviceImages[i]); |
205 | |
206 | // Register the images with the RTLs that understand them, if any. |
207 | for (DeviceImageTy &DI : PM->deviceImages()) { |
208 | // Obtain the image and information that was previously extracted. |
209 | __tgt_device_image *Img = &DI.getExecutableImage(); |
210 | |
211 | PluginAdaptorTy *FoundRTL = nullptr; |
212 | |
213 | // Scan the RTLs that have associated images until we find one that supports |
214 | // the current image. |
215 | for (auto &R : PM->pluginAdaptors()) { |
216 | if (!R.is_valid_binary(Img)) { |
217 | DP("Image " DPxMOD " is NOT compatible with RTL %s!\n" , |
218 | DPxPTR(Img->ImageStart), R.Name.c_str()); |
219 | continue; |
220 | } |
221 | |
222 | DP("Image " DPxMOD " is compatible with RTL %s!\n" , |
223 | DPxPTR(Img->ImageStart), R.Name.c_str()); |
224 | |
225 | R.initDevices(PM&: *this); |
226 | |
227 | // Initialize (if necessary) translation table for this library. |
228 | PM->TrlTblMtx.lock(); |
229 | if (!PM->HostEntriesBeginToTransTable.count(x: Desc->HostEntriesBegin)) { |
230 | PM->HostEntriesBeginRegistrationOrder.push_back(Elt: Desc->HostEntriesBegin); |
231 | TranslationTable &TransTable = |
232 | (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin]; |
233 | TransTable.HostTable.EntriesBegin = Desc->HostEntriesBegin; |
234 | TransTable.HostTable.EntriesEnd = Desc->HostEntriesEnd; |
235 | } |
236 | |
237 | // Retrieve translation table for this library. |
238 | TranslationTable &TransTable = |
239 | (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin]; |
240 | |
241 | DP("Registering image " DPxMOD " with RTL %s!\n" , DPxPTR(Img->ImageStart), |
242 | R.Name.c_str()); |
243 | registerImageIntoTranslationTable(TT&: TransTable, RTL&: R, Image: Img); |
244 | R.UsedImages.insert(V: Img); |
245 | |
246 | PM->TrlTblMtx.unlock(); |
247 | FoundRTL = &R; |
248 | |
249 | // if an RTL was found we are done - proceed to register the next image |
250 | break; |
251 | } |
252 | |
253 | if (!FoundRTL) { |
254 | DP("No RTL found for image " DPxMOD "!\n" , DPxPTR(Img->ImageStart)); |
255 | } |
256 | } |
257 | PM->RTLsMtx.unlock(); |
258 | |
259 | DP("Done registering entries!\n" ); |
260 | } |
261 | |
262 | // Temporary forward declaration, old style CTor/DTor handling is going away. |
263 | int target(ident_t *Loc, DeviceTy &Device, void *HostPtr, |
264 | KernelArgsTy &KernelArgs, AsyncInfoTy &AsyncInfo); |
265 | |
266 | void PluginManager::unregisterLib(__tgt_bin_desc *Desc) { |
267 | DP("Unloading target library!\n" ); |
268 | |
269 | PM->RTLsMtx.lock(); |
270 | // Find which RTL understands each image, if any. |
271 | for (DeviceImageTy &DI : PM->deviceImages()) { |
272 | // Obtain the image and information that was previously extracted. |
273 | __tgt_device_image *Img = &DI.getExecutableImage(); |
274 | |
275 | PluginAdaptorTy *FoundRTL = NULL; |
276 | |
277 | // Scan the RTLs that have associated images until we find one that supports |
278 | // the current image. We only need to scan RTLs that are already being used. |
279 | for (auto &R : PM->pluginAdaptors()) { |
280 | if (!R.isUsed()) |
281 | continue; |
282 | |
283 | // Ensure that we do not use any unused images associated with this RTL. |
284 | if (!R.UsedImages.contains(V: Img)) |
285 | continue; |
286 | |
287 | FoundRTL = &R; |
288 | |
289 | DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n" , |
290 | DPxPTR(Img->ImageStart), DPxPTR(R.LibraryHandler.get())); |
291 | |
292 | break; |
293 | } |
294 | |
295 | // if no RTL was found proceed to unregister the next image |
296 | if (!FoundRTL) { |
297 | DP("No RTLs in use support the image " DPxMOD "!\n" , |
298 | DPxPTR(Img->ImageStart)); |
299 | } |
300 | } |
301 | PM->RTLsMtx.unlock(); |
302 | DP("Done unregistering images!\n" ); |
303 | |
304 | // Remove entries from PM->HostPtrToTableMap |
305 | PM->TblMapMtx.lock(); |
306 | for (__tgt_offload_entry *Cur = Desc->HostEntriesBegin; |
307 | Cur < Desc->HostEntriesEnd; ++Cur) { |
308 | PM->HostPtrToTableMap.erase(x: Cur->addr); |
309 | } |
310 | |
311 | // Remove translation table for this descriptor. |
312 | auto TransTable = |
313 | PM->HostEntriesBeginToTransTable.find(x: Desc->HostEntriesBegin); |
314 | if (TransTable != PM->HostEntriesBeginToTransTable.end()) { |
315 | DP("Removing translation table for descriptor " DPxMOD "\n" , |
316 | DPxPTR(Desc->HostEntriesBegin)); |
317 | PM->HostEntriesBeginToTransTable.erase(position: TransTable); |
318 | } else { |
319 | DP("Translation table for descriptor " DPxMOD " cannot be found, probably " |
320 | "it has been already removed.\n" , |
321 | DPxPTR(Desc->HostEntriesBegin)); |
322 | } |
323 | |
324 | PM->TblMapMtx.unlock(); |
325 | |
326 | DP("Done unregistering library!\n" ); |
327 | } |
328 | |
329 | Expected<DeviceTy &> PluginManager::getDevice(uint32_t DeviceNo) { |
330 | auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor(); |
331 | if (DeviceNo >= ExclusiveDevicesAccessor->size()) |
332 | return createStringError( |
333 | EC: inconvertibleErrorCode(), |
334 | Fmt: "Device number '%i' out of range, only %i devices available" , Vals: DeviceNo, |
335 | Vals: ExclusiveDevicesAccessor->size()); |
336 | |
337 | return *(*ExclusiveDevicesAccessor)[DeviceNo]; |
338 | } |
339 | |