1//===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Contains utilities for adding indirections and breaking up modules.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
14#define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
15
16#include "llvm/ADT/StringMap.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ExecutionEngine/JITSymbol.h"
19#include "llvm/ExecutionEngine/Orc/Core.h"
20#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
21#include "llvm/Support/Error.h"
22#include "llvm/Support/Memory.h"
23#include "llvm/Support/Process.h"
24#include "llvm/Transforms/Utils/ValueMapper.h"
25#include <algorithm>
26#include <cassert>
27#include <cstdint>
28#include <functional>
29#include <future>
30#include <map>
31#include <memory>
32#include <system_error>
33#include <utility>
34#include <vector>
35
36namespace llvm {
37
38class Constant;
39class Function;
40class FunctionType;
41class GlobalAlias;
42class GlobalVariable;
43class Module;
44class PointerType;
45class Triple;
46class Twine;
47class Value;
48
49namespace orc {
50
51/// Base class for pools of compiler re-entry trampolines.
52/// These trampolines are callable addresses that save all register state
53/// before calling a supplied function to return the trampoline landing
54/// address, then restore all state before jumping to that address. They
55/// are used by various ORC APIs to support lazy compilation
56class TrampolinePool {
57public:
58 using NotifyLandingResolvedFunction =
59 unique_function<void(JITTargetAddress) const>;
60
61 using ResolveLandingFunction = unique_function<void(
62 JITTargetAddress TrampolineAddr,
63 NotifyLandingResolvedFunction OnLandingResolved) const>;
64
65 virtual ~TrampolinePool();
66
67 /// Get an available trampoline address.
68 /// Returns an error if no trampoline can be created.
69 Expected<JITTargetAddress> getTrampoline() {
70 std::lock_guard<std::mutex> Lock(TPMutex);
71 if (AvailableTrampolines.empty()) {
72 if (auto Err = grow())
73 return std::move(Err);
74 }
75 assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");
76 auto TrampolineAddr = AvailableTrampolines.back();
77 AvailableTrampolines.pop_back();
78 return TrampolineAddr;
79 }
80
81 /// Returns the given trampoline to the pool for re-use.
82 void releaseTrampoline(JITTargetAddress TrampolineAddr) {
83 std::lock_guard<std::mutex> Lock(TPMutex);
84 AvailableTrampolines.push_back(TrampolineAddr);
85 }
86
87protected:
88 virtual Error grow() = 0;
89
90 std::mutex TPMutex;
91 std::vector<JITTargetAddress> AvailableTrampolines;
92};
93
94/// A trampoline pool for trampolines within the current process.
95template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
96public:
97 /// Creates a LocalTrampolinePool with the given RunCallback function.
98 /// Returns an error if this function is unable to correctly allocate, write
99 /// and protect the resolver code block.
100 static Expected<std::unique_ptr<LocalTrampolinePool>>
101 Create(ResolveLandingFunction ResolveLanding) {
102 Error Err = Error::success();
103
104 auto LTP = std::unique_ptr<LocalTrampolinePool>(
105 new LocalTrampolinePool(std::move(ResolveLanding), Err));
106
107 if (Err)
108 return std::move(Err);
109 return std::move(LTP);
110 }
111
112private:
113 static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) {
114 LocalTrampolinePool<ORCABI> *TrampolinePool =
115 static_cast<LocalTrampolinePool *>(TrampolinePoolPtr);
116
117 std::promise<JITTargetAddress> LandingAddressP;
118 auto LandingAddressF = LandingAddressP.get_future();
119
120 TrampolinePool->ResolveLanding(pointerToJITTargetAddress(TrampolineId),
121 [&](JITTargetAddress LandingAddress) {
122 LandingAddressP.set_value(LandingAddress);
123 });
124 return LandingAddressF.get();
125 }
126
127 LocalTrampolinePool(ResolveLandingFunction ResolveLanding, Error &Err)
128 : ResolveLanding(std::move(ResolveLanding)) {
129
130 ErrorAsOutParameter _(&Err);
131
132 /// Try to set up the resolver block.
133 std::error_code EC;
134 ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
135 ORCABI::ResolverCodeSize, nullptr,
136 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
137 if (EC) {
138 Err = errorCodeToError(EC);
139 return;
140 }
141
142 ORCABI::writeResolverCode(static_cast<char *>(ResolverBlock.base()),
143 pointerToJITTargetAddress(ResolverBlock.base()),
144 pointerToJITTargetAddress(&reenter),
145 pointerToJITTargetAddress(this));
146
147 EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(),
148 sys::Memory::MF_READ |
149 sys::Memory::MF_EXEC);
150 if (EC) {
151 Err = errorCodeToError(EC);
152 return;
153 }
154 }
155
156 Error grow() override {
157 assert(AvailableTrampolines.empty() && "Growing prematurely?");
158
159 std::error_code EC;
160 auto TrampolineBlock =
161 sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
162 sys::Process::getPageSizeEstimate(), nullptr,
163 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
164 if (EC)
165 return errorCodeToError(EC);
166
167 unsigned NumTrampolines =
168 (sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) /
169 ORCABI::TrampolineSize;
170
171 char *TrampolineMem = static_cast<char *>(TrampolineBlock.base());
172 ORCABI::writeTrampolines(
173 TrampolineMem, pointerToJITTargetAddress(TrampolineMem),
174 pointerToJITTargetAddress(ResolverBlock.base()), NumTrampolines);
175
176 for (unsigned I = 0; I < NumTrampolines; ++I)
177 AvailableTrampolines.push_back(pointerToJITTargetAddress(
178 TrampolineMem + (I * ORCABI::TrampolineSize)));
179
180 if (auto EC = sys::Memory::protectMappedMemory(
181 TrampolineBlock.getMemoryBlock(),
182 sys::Memory::MF_READ | sys::Memory::MF_EXEC))
183 return errorCodeToError(EC);
184
185 TrampolineBlocks.push_back(std::move(TrampolineBlock));
186 return Error::success();
187 }
188
189 ResolveLandingFunction ResolveLanding;
190
191 sys::OwningMemoryBlock ResolverBlock;
192 std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
193};
194
195/// Target-independent base class for compile callback management.
196class JITCompileCallbackManager {
197public:
198 using CompileFunction = std::function<JITTargetAddress()>;
199
200 virtual ~JITCompileCallbackManager() = default;
201
202 /// Reserve a compile callback.
203 Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile);
204
205 /// Execute the callback for the given trampoline id. Called by the JIT
206 /// to compile functions on demand.
207 JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr);
208
209protected:
210 /// Construct a JITCompileCallbackManager.
211 JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP,
212 ExecutionSession &ES,
213 JITTargetAddress ErrorHandlerAddress)
214 : TP(std::move(TP)), ES(ES),
215 CallbacksJD(ES.createBareJITDylib("<Callbacks>")),
216 ErrorHandlerAddress(ErrorHandlerAddress) {}
217
218 void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) {
219 this->TP = std::move(TP);
220 }
221
222private:
223 std::mutex CCMgrMutex;
224 std::unique_ptr<TrampolinePool> TP;
225 ExecutionSession &ES;
226 JITDylib &CallbacksJD;
227 JITTargetAddress ErrorHandlerAddress;
228 std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol;
229 size_t NextCallbackId = 0;
230};
231
232/// Manage compile callbacks for in-process JITs.
233template <typename ORCABI>
234class LocalJITCompileCallbackManager : public JITCompileCallbackManager {
235public:
236 /// Create a new LocalJITCompileCallbackManager.
237 static Expected<std::unique_ptr<LocalJITCompileCallbackManager>>
238 Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) {
239 Error Err = Error::success();
240 auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>(
241 new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err));
242 if (Err)
243 return std::move(Err);
244 return std::move(CCMgr);
245 }
246
247private:
248 /// Construct a InProcessJITCompileCallbackManager.
249 /// @param ErrorHandlerAddress The address of an error handler in the target
250 /// process to be used if a compile callback fails.
251 LocalJITCompileCallbackManager(ExecutionSession &ES,
252 JITTargetAddress ErrorHandlerAddress,
253 Error &Err)
254 : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) {
255 using NotifyLandingResolvedFunction =
256 TrampolinePool::NotifyLandingResolvedFunction;
257
258 ErrorAsOutParameter _(&Err);
259 auto TP = LocalTrampolinePool<ORCABI>::Create(
260 [this](JITTargetAddress TrampolineAddr,
261 NotifyLandingResolvedFunction NotifyLandingResolved) {
262 NotifyLandingResolved(executeCompileCallback(TrampolineAddr));
263 });
264
265 if (!TP) {
266 Err = TP.takeError();
267 return;
268 }
269
270 setTrampolinePool(std::move(*TP));
271 }
272};
273
274/// Base class for managing collections of named indirect stubs.
275class IndirectStubsManager {
276public:
277 /// Map type for initializing the manager. See init.
278 using StubInitsMap = StringMap<std::pair<JITTargetAddress, JITSymbolFlags>>;
279
280 virtual ~IndirectStubsManager() = default;
281
282 /// Create a single stub with the given name, target address and flags.
283 virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr,
284 JITSymbolFlags StubFlags) = 0;
285
286 /// Create StubInits.size() stubs with the given names, target
287 /// addresses, and flags.
288 virtual Error createStubs(const StubInitsMap &StubInits) = 0;
289
290 /// Find the stub with the given name. If ExportedStubsOnly is true,
291 /// this will only return a result if the stub's flags indicate that it
292 /// is exported.
293 virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0;
294
295 /// Find the implementation-pointer for the stub.
296 virtual JITEvaluatedSymbol findPointer(StringRef Name) = 0;
297
298 /// Change the value of the implementation pointer for the stub.
299 virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0;
300
301private:
302 virtual void anchor();
303};
304
305template <typename ORCABI> class LocalIndirectStubsInfo {
306public:
307 LocalIndirectStubsInfo(unsigned NumStubs, sys::OwningMemoryBlock StubsMem)
308 : NumStubs(NumStubs), StubsMem(std::move(StubsMem)) {}
309
310 static Expected<LocalIndirectStubsInfo> create(unsigned MinStubs,
311 unsigned PageSize) {
312 auto ISAS = getIndirectStubsBlockSizes<ORCABI>(MinStubs, PageSize);
313
314 assert((ISAS.StubBytes % PageSize == 0) &&
315 "StubBytes is not a page size multiple");
316 uint64_t PointerAlloc = alignTo(ISAS.PointerBytes, PageSize);
317
318 // Allocate memory for stubs and pointers in one call.
319 std::error_code EC;
320 auto StubsAndPtrsMem =
321 sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
322 ISAS.StubBytes + PointerAlloc, nullptr,
323 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
324 if (EC)
325 return errorCodeToError(EC);
326
327 sys::MemoryBlock StubsBlock(StubsAndPtrsMem.base(), ISAS.StubBytes);
328 auto StubsBlockMem = static_cast<char *>(StubsAndPtrsMem.base());
329 auto PtrBlockAddress =
330 pointerToJITTargetAddress(StubsBlockMem) + ISAS.StubBytes;
331
332 ORCABI::writeIndirectStubsBlock(StubsBlockMem,
333 pointerToJITTargetAddress(StubsBlockMem),
334 PtrBlockAddress, ISAS.NumStubs);
335
336 if (auto EC = sys::Memory::protectMappedMemory(
337 StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC))
338 return errorCodeToError(EC);
339
340 return LocalIndirectStubsInfo(ISAS.NumStubs, std::move(StubsAndPtrsMem));
341 }
342
343 unsigned getNumStubs() const { return NumStubs; }
344
345 void *getStub(unsigned Idx) const {
346 return static_cast<char *>(StubsMem.base()) + Idx * ORCABI::StubSize;
347 }
348
349 void **getPtr(unsigned Idx) const {
350 char *PtrsBase =
351 static_cast<char *>(StubsMem.base()) + NumStubs * ORCABI::StubSize;
352 return reinterpret_cast<void **>(PtrsBase) + Idx;
353 }
354
355private:
356 unsigned NumStubs = 0;
357 sys::OwningMemoryBlock StubsMem;
358};
359
360/// IndirectStubsManager implementation for the host architecture, e.g.
361/// OrcX86_64. (See OrcArchitectureSupport.h).
362template <typename TargetT>
363class LocalIndirectStubsManager : public IndirectStubsManager {
364public:
365 Error createStub(StringRef StubName, JITTargetAddress StubAddr,
366 JITSymbolFlags StubFlags) override {
367 std::lock_guard<std::mutex> Lock(StubsMutex);
368 if (auto Err = reserveStubs(1))
369 return Err;
370
371 createStubInternal(StubName, StubAddr, StubFlags);
372
373 return Error::success();
374 }
375
376 Error createStubs(const StubInitsMap &StubInits) override {
377 std::lock_guard<std::mutex> Lock(StubsMutex);
378 if (auto Err = reserveStubs(StubInits.size()))
379 return Err;
380
381 for (auto &Entry : StubInits)
382 createStubInternal(Entry.first(), Entry.second.first,
383 Entry.second.second);
384
385 return Error::success();
386 }
387
388 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override {
389 std::lock_guard<std::mutex> Lock(StubsMutex);
390 auto I = StubIndexes.find(Name);
391 if (I == StubIndexes.end())
392 return nullptr;
393 auto Key = I->second.first;
394 void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second);
395 assert(StubAddr && "Missing stub address");
396 auto StubTargetAddr =
397 static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr));
398 auto StubSymbol = JITEvaluatedSymbol(StubTargetAddr, I->second.second);
399 if (ExportedStubsOnly && !StubSymbol.getFlags().isExported())
400 return nullptr;
401 return StubSymbol;
402 }
403
404 JITEvaluatedSymbol findPointer(StringRef Name) override {
405 std::lock_guard<std::mutex> Lock(StubsMutex);
406 auto I = StubIndexes.find(Name);
407 if (I == StubIndexes.end())
408 return nullptr;
409 auto Key = I->second.first;
410 void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second);
411 assert(PtrAddr && "Missing pointer address");
412 auto PtrTargetAddr =
413 static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr));
414 return JITEvaluatedSymbol(PtrTargetAddr, I->second.second);
415 }
416
417 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override {
418 using AtomicIntPtr = std::atomic<uintptr_t>;
419
420 std::lock_guard<std::mutex> Lock(StubsMutex);
421 auto I = StubIndexes.find(Name);
422 assert(I != StubIndexes.end() && "No stub pointer for symbol");
423 auto Key = I->second.first;
424 AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>(
425 IndirectStubsInfos[Key.first].getPtr(Key.second));
426 *AtomicStubPtr = static_cast<uintptr_t>(NewAddr);
427 return Error::success();
428 }
429
430private:
431 Error reserveStubs(unsigned NumStubs) {
432 if (NumStubs <= FreeStubs.size())
433 return Error::success();
434
435 unsigned NewStubsRequired = NumStubs - FreeStubs.size();
436 unsigned NewBlockId = IndirectStubsInfos.size();
437 auto ISI =
438 LocalIndirectStubsInfo<TargetT>::create(NewStubsRequired, PageSize);
439 if (!ISI)
440 return ISI.takeError();
441 for (unsigned I = 0; I < ISI->getNumStubs(); ++I)
442 FreeStubs.push_back(std::make_pair(NewBlockId, I));
443 IndirectStubsInfos.push_back(std::move(*ISI));
444 return Error::success();
445 }
446
447 void createStubInternal(StringRef StubName, JITTargetAddress InitAddr,
448 JITSymbolFlags StubFlags) {
449 auto Key = FreeStubs.back();
450 FreeStubs.pop_back();
451 *IndirectStubsInfos[Key.first].getPtr(Key.second) =
452 jitTargetAddressToPointer<void *>(InitAddr);
453 StubIndexes[StubName] = std::make_pair(Key, StubFlags);
454 }
455
456 unsigned PageSize = sys::Process::getPageSizeEstimate();
457 std::mutex StubsMutex;
458 std::vector<LocalIndirectStubsInfo<TargetT>> IndirectStubsInfos;
459 using StubKey = std::pair<uint16_t, uint16_t>;
460 std::vector<StubKey> FreeStubs;
461 StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes;
462};
463
464/// Create a local compile callback manager.
465///
466/// The given target triple will determine the ABI, and the given
467/// ErrorHandlerAddress will be used by the resulting compile callback
468/// manager if a compile callback fails.
469Expected<std::unique_ptr<JITCompileCallbackManager>>
470createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
471 JITTargetAddress ErrorHandlerAddress);
472
473/// Create a local indriect stubs manager builder.
474///
475/// The given target triple will determine the ABI.
476std::function<std::unique_ptr<IndirectStubsManager>()>
477createLocalIndirectStubsManagerBuilder(const Triple &T);
478
479/// Build a function pointer of FunctionType with the given constant
480/// address.
481///
482/// Usage example: Turn a trampoline address into a function pointer constant
483/// for use in a stub.
484Constant *createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr);
485
486/// Create a function pointer with the given type, name, and initializer
487/// in the given Module.
488GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name,
489 Constant *Initializer);
490
491/// Turn a function declaration into a stub function that makes an
492/// indirect call using the given function pointer.
493void makeStub(Function &F, Value &ImplPointer);
494
495/// Promotes private symbols to global hidden, and renames to prevent clashes
496/// with other promoted symbols. The same SymbolPromoter instance should be
497/// used for all symbols to be added to a single JITDylib.
498class SymbolLinkagePromoter {
499public:
500 /// Promote symbols in the given module. Returns the set of global values
501 /// that have been renamed/promoted.
502 std::vector<GlobalValue *> operator()(Module &M);
503
504private:
505 unsigned NextId = 0;
506};
507
508/// Clone a function declaration into a new module.
509///
510/// This function can be used as the first step towards creating a callback
511/// stub (see makeStub), or moving a function body (see moveFunctionBody).
512///
513/// If the VMap argument is non-null, a mapping will be added between F and
514/// the new declaration, and between each of F's arguments and the new
515/// declaration's arguments. This map can then be passed in to moveFunction to
516/// move the function body if required. Note: When moving functions between
517/// modules with these utilities, all decls should be cloned (and added to a
518/// single VMap) before any bodies are moved. This will ensure that references
519/// between functions all refer to the versions in the new module.
520Function *cloneFunctionDecl(Module &Dst, const Function &F,
521 ValueToValueMapTy *VMap = nullptr);
522
523/// Move the body of function 'F' to a cloned function declaration in a
524/// different module (See related cloneFunctionDecl).
525///
526/// If the target function declaration is not supplied via the NewF parameter
527/// then it will be looked up via the VMap.
528///
529/// This will delete the body of function 'F' from its original parent module,
530/// but leave its declaration.
531void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
532 ValueMaterializer *Materializer = nullptr,
533 Function *NewF = nullptr);
534
535/// Clone a global variable declaration into a new module.
536GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
537 ValueToValueMapTy *VMap = nullptr);
538
539/// Move global variable GV from its parent module to cloned global
540/// declaration in a different module.
541///
542/// If the target global declaration is not supplied via the NewGV parameter
543/// then it will be looked up via the VMap.
544///
545/// This will delete the initializer of GV from its original parent module,
546/// but leave its declaration.
547void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
548 ValueToValueMapTy &VMap,
549 ValueMaterializer *Materializer = nullptr,
550 GlobalVariable *NewGV = nullptr);
551
552/// Clone a global alias declaration into a new module.
553GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
554 ValueToValueMapTy &VMap);
555
556/// Clone module flags metadata into the destination module.
557void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
558 ValueToValueMapTy &VMap);
559
560} // end namespace orc
561
562} // end namespace llvm
563
564#endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
565