1 | //===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h" |
10 | |
11 | #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" |
12 | #include "llvm/Support/Process.h" |
13 | #include "llvm/Support/WindowsError.h" |
14 | |
15 | #include <sstream> |
16 | |
17 | #if defined(LLVM_ON_UNIX) |
18 | #include <errno.h> |
19 | #include <fcntl.h> |
20 | #include <sys/mman.h> |
21 | #include <unistd.h> |
22 | #endif |
23 | |
24 | namespace llvm { |
25 | namespace orc { |
26 | namespace rt_bootstrap { |
27 | |
28 | #if defined(_WIN32) |
29 | static DWORD getWindowsProtectionFlags(MemProt MP) { |
30 | if (MP == MemProt::Read) |
31 | return PAGE_READONLY; |
32 | if (MP == MemProt::Write || |
33 | MP == (MemProt::Write | MemProt::Read)) { |
34 | // Note: PAGE_WRITE is not supported by VirtualProtect |
35 | return PAGE_READWRITE; |
36 | } |
37 | if (MP == (MemProt::Read | MemProt::Exec)) |
38 | return PAGE_EXECUTE_READ; |
39 | if (MP == (MemProt::Read | MemProt::Write | MemProt::Exec)) |
40 | return PAGE_EXECUTE_READWRITE; |
41 | if (MP == MemProt::Exec) |
42 | return PAGE_EXECUTE; |
43 | |
44 | return PAGE_NOACCESS; |
45 | } |
46 | #endif |
47 | |
48 | Expected<std::pair<ExecutorAddr, std::string>> |
49 | ExecutorSharedMemoryMapperService::reserve(uint64_t Size) { |
50 | #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) |
51 | |
52 | #if defined(LLVM_ON_UNIX) |
53 | |
54 | std::string SharedMemoryName; |
55 | { |
56 | std::stringstream SharedMemoryNameStream; |
57 | SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_' |
58 | << (++SharedMemoryCount); |
59 | SharedMemoryName = SharedMemoryNameStream.str(); |
60 | } |
61 | |
62 | int SharedMemoryFile = |
63 | shm_open(name: SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, mode: 0700); |
64 | if (SharedMemoryFile < 0) |
65 | return errorCodeToError(EC: errnoAsErrorCode()); |
66 | |
67 | // by default size is 0 |
68 | if (ftruncate(fd: SharedMemoryFile, length: Size) < 0) |
69 | return errorCodeToError(EC: errnoAsErrorCode()); |
70 | |
71 | void *Addr = mmap(addr: nullptr, len: Size, PROT_NONE, MAP_SHARED, fd: SharedMemoryFile, offset: 0); |
72 | if (Addr == MAP_FAILED) |
73 | return errorCodeToError(EC: errnoAsErrorCode()); |
74 | |
75 | close(fd: SharedMemoryFile); |
76 | |
77 | #elif defined(_WIN32) |
78 | |
79 | std::string SharedMemoryName; |
80 | { |
81 | std::stringstream SharedMemoryNameStream; |
82 | SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_' |
83 | << (++SharedMemoryCount); |
84 | SharedMemoryName = SharedMemoryNameStream.str(); |
85 | } |
86 | |
87 | std::wstring WideSharedMemoryName(SharedMemoryName.begin(), |
88 | SharedMemoryName.end()); |
89 | HANDLE SharedMemoryFile = CreateFileMappingW( |
90 | INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32, |
91 | Size & 0xffffffff, WideSharedMemoryName.c_str()); |
92 | if (!SharedMemoryFile) |
93 | return errorCodeToError(mapWindowsError(GetLastError())); |
94 | |
95 | void *Addr = MapViewOfFile(SharedMemoryFile, |
96 | FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0); |
97 | if (!Addr) { |
98 | CloseHandle(SharedMemoryFile); |
99 | return errorCodeToError(mapWindowsError(GetLastError())); |
100 | } |
101 | |
102 | #endif |
103 | |
104 | { |
105 | std::lock_guard<std::mutex> Lock(Mutex); |
106 | Reservations[Addr].Size = Size; |
107 | #if defined(_WIN32) |
108 | Reservations[Addr].SharedMemoryFile = SharedMemoryFile; |
109 | #endif |
110 | } |
111 | |
112 | return std::make_pair(x: ExecutorAddr::fromPtr(Ptr: Addr), |
113 | y: std::move(SharedMemoryName)); |
114 | #else |
115 | return make_error<StringError>( |
116 | "SharedMemoryMapper is not supported on this platform yet" , |
117 | inconvertibleErrorCode()); |
118 | #endif |
119 | } |
120 | |
121 | Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize( |
122 | ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) { |
123 | #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) |
124 | |
125 | ExecutorAddr MinAddr(~0ULL); |
126 | |
127 | // Contents are already in place |
128 | for (auto &Segment : FR.Segments) { |
129 | if (Segment.Addr < MinAddr) |
130 | MinAddr = Segment.Addr; |
131 | |
132 | #if defined(LLVM_ON_UNIX) |
133 | |
134 | int NativeProt = 0; |
135 | if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read) |
136 | NativeProt |= PROT_READ; |
137 | if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write) |
138 | NativeProt |= PROT_WRITE; |
139 | if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec) |
140 | NativeProt |= PROT_EXEC; |
141 | |
142 | if (mprotect(addr: Segment.Addr.toPtr<void *>(), len: Segment.Size, prot: NativeProt)) |
143 | return errorCodeToError(EC: errnoAsErrorCode()); |
144 | |
145 | #elif defined(_WIN32) |
146 | |
147 | DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot); |
148 | |
149 | if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt, |
150 | &NativeProt)) |
151 | return errorCodeToError(mapWindowsError(GetLastError())); |
152 | |
153 | #endif |
154 | |
155 | if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec) |
156 | sys::Memory::InvalidateInstructionCache(Addr: Segment.Addr.toPtr<void *>(), |
157 | Len: Segment.Size); |
158 | } |
159 | |
160 | // Run finalization actions and get deinitlization action list. |
161 | auto DeinitializeActions = shared::runFinalizeActions(AAs&: FR.Actions); |
162 | if (!DeinitializeActions) { |
163 | return DeinitializeActions.takeError(); |
164 | } |
165 | |
166 | { |
167 | std::lock_guard<std::mutex> Lock(Mutex); |
168 | Allocations[MinAddr].DeinitializationActions = |
169 | std::move(*DeinitializeActions); |
170 | Reservations[Reservation.toPtr<void *>()].Allocations.push_back(x: MinAddr); |
171 | } |
172 | |
173 | return MinAddr; |
174 | |
175 | #else |
176 | return make_error<StringError>( |
177 | "SharedMemoryMapper is not supported on this platform yet" , |
178 | inconvertibleErrorCode()); |
179 | #endif |
180 | } |
181 | |
182 | Error ExecutorSharedMemoryMapperService::deinitialize( |
183 | const std::vector<ExecutorAddr> &Bases) { |
184 | Error AllErr = Error::success(); |
185 | |
186 | { |
187 | std::lock_guard<std::mutex> Lock(Mutex); |
188 | |
189 | for (auto Base : llvm::reverse(C: Bases)) { |
190 | if (Error Err = shared::runDeallocActions( |
191 | DAs: Allocations[Base].DeinitializationActions)) { |
192 | AllErr = joinErrors(E1: std::move(AllErr), E2: std::move(Err)); |
193 | } |
194 | |
195 | // Remove the allocation from the allocation list of its reservation |
196 | for (auto &Reservation : Reservations) { |
197 | auto AllocationIt = llvm::find(Range&: Reservation.second.Allocations, Val: Base); |
198 | if (AllocationIt != Reservation.second.Allocations.end()) { |
199 | Reservation.second.Allocations.erase(position: AllocationIt); |
200 | break; |
201 | } |
202 | } |
203 | |
204 | Allocations.erase(Val: Base); |
205 | } |
206 | } |
207 | |
208 | return AllErr; |
209 | } |
210 | |
211 | Error ExecutorSharedMemoryMapperService::release( |
212 | const std::vector<ExecutorAddr> &Bases) { |
213 | #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) |
214 | Error Err = Error::success(); |
215 | |
216 | for (auto Base : Bases) { |
217 | std::vector<ExecutorAddr> AllocAddrs; |
218 | size_t Size; |
219 | |
220 | #if defined(_WIN32) |
221 | HANDLE SharedMemoryFile; |
222 | #endif |
223 | |
224 | { |
225 | std::lock_guard<std::mutex> Lock(Mutex); |
226 | auto &R = Reservations[Base.toPtr<void *>()]; |
227 | Size = R.Size; |
228 | |
229 | #if defined(_WIN32) |
230 | SharedMemoryFile = R.SharedMemoryFile; |
231 | #endif |
232 | |
233 | AllocAddrs.swap(x&: R.Allocations); |
234 | } |
235 | |
236 | // deinitialize sub allocations |
237 | if (Error E = deinitialize(Bases: AllocAddrs)) |
238 | Err = joinErrors(E1: std::move(Err), E2: std::move(E)); |
239 | |
240 | #if defined(LLVM_ON_UNIX) |
241 | |
242 | if (munmap(addr: Base.toPtr<void *>(), len: Size) != 0) |
243 | Err = joinErrors(E1: std::move(Err), E2: errorCodeToError(EC: errnoAsErrorCode())); |
244 | |
245 | #elif defined(_WIN32) |
246 | (void)Size; |
247 | |
248 | if (!UnmapViewOfFile(Base.toPtr<void *>())) |
249 | Err = joinErrors(std::move(Err), |
250 | errorCodeToError(mapWindowsError(GetLastError()))); |
251 | |
252 | CloseHandle(SharedMemoryFile); |
253 | |
254 | #endif |
255 | |
256 | std::lock_guard<std::mutex> Lock(Mutex); |
257 | Reservations.erase(Val: Base.toPtr<void *>()); |
258 | } |
259 | |
260 | return Err; |
261 | #else |
262 | return make_error<StringError>( |
263 | "SharedMemoryMapper is not supported on this platform yet" , |
264 | inconvertibleErrorCode()); |
265 | #endif |
266 | } |
267 | |
268 | Error ExecutorSharedMemoryMapperService::shutdown() { |
269 | if (Reservations.empty()) |
270 | return Error::success(); |
271 | |
272 | std::vector<ExecutorAddr> ReservationAddrs; |
273 | ReservationAddrs.reserve(n: Reservations.size()); |
274 | for (const auto &R : Reservations) |
275 | ReservationAddrs.push_back(x: ExecutorAddr::fromPtr(Ptr: R.getFirst())); |
276 | |
277 | return release(Bases: std::move(ReservationAddrs)); |
278 | } |
279 | |
280 | void ExecutorSharedMemoryMapperService::addBootstrapSymbols( |
281 | StringMap<ExecutorAddr> &M) { |
282 | M[rt::ExecutorSharedMemoryMapperServiceInstanceName] = |
283 | ExecutorAddr::fromPtr(Ptr: this); |
284 | M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] = |
285 | ExecutorAddr::fromPtr(Ptr: &reserveWrapper); |
286 | M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] = |
287 | ExecutorAddr::fromPtr(Ptr: &initializeWrapper); |
288 | M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] = |
289 | ExecutorAddr::fromPtr(Ptr: &deinitializeWrapper); |
290 | M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] = |
291 | ExecutorAddr::fromPtr(Ptr: &releaseWrapper); |
292 | } |
293 | |
294 | llvm::orc::shared::CWrapperFunctionResult |
295 | ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData, |
296 | size_t ArgSize) { |
297 | return shared::WrapperFunction< |
298 | rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>:: |
299 | handle(ArgData, ArgSize, |
300 | Handler: shared::makeMethodWrapperHandler( |
301 | Method: &ExecutorSharedMemoryMapperService::reserve)) |
302 | .release(); |
303 | } |
304 | |
305 | llvm::orc::shared::CWrapperFunctionResult |
306 | ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData, |
307 | size_t ArgSize) { |
308 | return shared::WrapperFunction< |
309 | rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>:: |
310 | handle(ArgData, ArgSize, |
311 | Handler: shared::makeMethodWrapperHandler( |
312 | Method: &ExecutorSharedMemoryMapperService::initialize)) |
313 | .release(); |
314 | } |
315 | |
316 | llvm::orc::shared::CWrapperFunctionResult |
317 | ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData, |
318 | size_t ArgSize) { |
319 | return shared::WrapperFunction< |
320 | rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>:: |
321 | handle(ArgData, ArgSize, |
322 | Handler: shared::makeMethodWrapperHandler( |
323 | Method: &ExecutorSharedMemoryMapperService::deinitialize)) |
324 | .release(); |
325 | } |
326 | |
327 | llvm::orc::shared::CWrapperFunctionResult |
328 | ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData, |
329 | size_t ArgSize) { |
330 | return shared::WrapperFunction< |
331 | rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>:: |
332 | handle(ArgData, ArgSize, |
333 | Handler: shared::makeMethodWrapperHandler( |
334 | Method: &ExecutorSharedMemoryMapperService::release)) |
335 | .release(); |
336 | } |
337 | |
338 | } // namespace rt_bootstrap |
339 | } // end namespace orc |
340 | } // end namespace llvm |
341 | |