1//===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
10#include "llvm/ADT/STLExtras.h"
11#include "llvm/ExecutionEngine/JITLink/x86_64.h"
12#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
13#include "llvm/IR/IRBuilder.h"
14#include "llvm/MC/MCDisassembler/MCDisassembler.h"
15#include "llvm/MC/MCInstrAnalysis.h"
16#include "llvm/Support/Format.h"
17#include "llvm/TargetParser/Triple.h"
18#include "llvm/Transforms/Utils/Cloning.h"
19#include <sstream>
20
21#define DEBUG_TYPE "orc"
22
23using namespace llvm;
24using namespace llvm::orc;
25
26namespace {
27
28class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
29public:
30 using CompileFunction = JITCompileCallbackManager::CompileFunction;
31
32 CompileCallbackMaterializationUnit(SymbolStringPtr Name,
33 CompileFunction Compile)
34 : MaterializationUnit(Interface(
35 SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)),
36 Name(std::move(Name)), Compile(std::move(Compile)) {}
37
38 StringRef getName() const override { return "<Compile Callbacks>"; }
39
40private:
41 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
42 SymbolMap Result;
43 Result[Name] = {Compile(), JITSymbolFlags::Exported};
44 // No dependencies, so these calls cannot fail.
45 cantFail(Err: R->notifyResolved(Symbols: Result));
46 cantFail(Err: R->notifyEmitted(EmittedDeps: {}));
47 }
48
49 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
50 llvm_unreachable("Discard should never occur on a LMU?");
51 }
52
53 SymbolStringPtr Name;
54 CompileFunction Compile;
55};
56
57} // namespace
58
59namespace llvm {
60namespace orc {
61
62TrampolinePool::~TrampolinePool() = default;
63void IndirectStubsManager::anchor() {}
64
65Expected<ExecutorAddr>
66JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
67 if (auto TrampolineAddr = TP->getTrampoline()) {
68 auto CallbackName =
69 ES.intern(SymName: std::string("cc") + std::to_string(val: ++NextCallbackId));
70
71 std::lock_guard<std::mutex> Lock(CCMgrMutex);
72 AddrToSymbol[*TrampolineAddr] = CallbackName;
73 cantFail(
74 Err: CallbacksJD.define(MU: std::make_unique<CompileCallbackMaterializationUnit>(
75 args: std::move(CallbackName), args: std::move(Compile))));
76 return *TrampolineAddr;
77 } else
78 return TrampolineAddr.takeError();
79}
80
81ExecutorAddr
82JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) {
83 SymbolStringPtr Name;
84
85 {
86 std::unique_lock<std::mutex> Lock(CCMgrMutex);
87 auto I = AddrToSymbol.find(x: TrampolineAddr);
88
89 // If this address is not associated with a compile callback then report an
90 // error to the execution session and return ErrorHandlerAddress to the
91 // callee.
92 if (I == AddrToSymbol.end()) {
93 Lock.unlock();
94 ES.reportError(
95 Err: make_error<StringError>(Args: "No compile callback for trampoline at " +
96 formatv(Fmt: "{0:x}", Vals&: TrampolineAddr),
97 Args: inconvertibleErrorCode()));
98 return ErrorHandlerAddress;
99 } else
100 Name = I->second;
101 }
102
103 if (auto Sym =
104 ES.lookup(SearchOrder: makeJITDylibSearchOrder(
105 JDs: &CallbacksJD, Flags: JITDylibLookupFlags::MatchAllSymbols),
106 Symbol: Name))
107 return Sym->getAddress();
108 else {
109 llvm::dbgs() << "Didn't find callback.\n";
110 // If anything goes wrong materializing Sym then report it to the session
111 // and return the ErrorHandlerAddress;
112 ES.reportError(Err: Sym.takeError());
113 return ErrorHandlerAddress;
114 }
115}
116
117Expected<std::unique_ptr<JITCompileCallbackManager>>
118createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
119 ExecutorAddr ErrorHandlerAddress) {
120 switch (T.getArch()) {
121 default:
122 return make_error<StringError>(
123 Args: std::string("No callback manager available for ") + T.str(),
124 Args: inconvertibleErrorCode());
125 case Triple::aarch64:
126 case Triple::aarch64_32: {
127 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
128 return CCMgrT::Create(ES, ErrorHandlerAddress);
129 }
130
131 case Triple::x86: {
132 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
133 return CCMgrT::Create(ES, ErrorHandlerAddress);
134 }
135
136 case Triple::loongarch64: {
137 typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT;
138 return CCMgrT::Create(ES, ErrorHandlerAddress);
139 }
140
141 case Triple::mips: {
142 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
143 return CCMgrT::Create(ES, ErrorHandlerAddress);
144 }
145 case Triple::mipsel: {
146 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
147 return CCMgrT::Create(ES, ErrorHandlerAddress);
148 }
149
150 case Triple::mips64:
151 case Triple::mips64el: {
152 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
153 return CCMgrT::Create(ES, ErrorHandlerAddress);
154 }
155
156 case Triple::riscv64: {
157 typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT;
158 return CCMgrT::Create(ES, ErrorHandlerAddress);
159 }
160
161 case Triple::x86_64: {
162 if (T.getOS() == Triple::OSType::Win32) {
163 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
164 return CCMgrT::Create(ES, ErrorHandlerAddress);
165 } else {
166 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
167 return CCMgrT::Create(ES, ErrorHandlerAddress);
168 }
169 }
170
171 }
172}
173
174std::function<std::unique_ptr<IndirectStubsManager>()>
175createLocalIndirectStubsManagerBuilder(const Triple &T) {
176 switch (T.getArch()) {
177 default:
178 return [](){
179 return std::make_unique<
180 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
181 };
182
183 case Triple::aarch64:
184 case Triple::aarch64_32:
185 return [](){
186 return std::make_unique<
187 orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
188 };
189
190 case Triple::x86:
191 return [](){
192 return std::make_unique<
193 orc::LocalIndirectStubsManager<orc::OrcI386>>();
194 };
195
196 case Triple::loongarch64:
197 return []() {
198 return std::make_unique<
199 orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>();
200 };
201
202 case Triple::mips:
203 return [](){
204 return std::make_unique<
205 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
206 };
207
208 case Triple::mipsel:
209 return [](){
210 return std::make_unique<
211 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
212 };
213
214 case Triple::mips64:
215 case Triple::mips64el:
216 return [](){
217 return std::make_unique<
218 orc::LocalIndirectStubsManager<orc::OrcMips64>>();
219 };
220
221 case Triple::riscv64:
222 return []() {
223 return std::make_unique<
224 orc::LocalIndirectStubsManager<orc::OrcRiscv64>>();
225 };
226
227 case Triple::x86_64:
228 if (T.getOS() == Triple::OSType::Win32) {
229 return [](){
230 return std::make_unique<
231 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
232 };
233 } else {
234 return [](){
235 return std::make_unique<
236 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
237 };
238 }
239
240 }
241}
242
243Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) {
244 Constant *AddrIntVal =
245 ConstantInt::get(Ty: Type::getInt64Ty(C&: FT.getContext()), V: Addr.getValue());
246 Constant *AddrPtrVal =
247 ConstantExpr::getIntToPtr(C: AddrIntVal, Ty: PointerType::get(ElementType: &FT, AddressSpace: 0));
248 return AddrPtrVal;
249}
250
251GlobalVariable* createImplPointer(PointerType &PT, Module &M,
252 const Twine &Name, Constant *Initializer) {
253 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
254 Initializer, Name, nullptr,
255 GlobalValue::NotThreadLocal, 0, true);
256 IP->setVisibility(GlobalValue::HiddenVisibility);
257 return IP;
258}
259
260void makeStub(Function &F, Value &ImplPointer) {
261 assert(F.isDeclaration() && "Can't turn a definition into a stub.");
262 assert(F.getParent() && "Function isn't in a module.");
263 Module &M = *F.getParent();
264 BasicBlock *EntryBlock = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: &F);
265 IRBuilder<> Builder(EntryBlock);
266 LoadInst *ImplAddr = Builder.CreateLoad(Ty: F.getType(), Ptr: &ImplPointer);
267 std::vector<Value*> CallArgs;
268 for (auto &A : F.args())
269 CallArgs.push_back(x: &A);
270 CallInst *Call = Builder.CreateCall(FTy: F.getFunctionType(), Callee: ImplAddr, Args: CallArgs);
271 Call->setTailCall();
272 Call->setAttributes(F.getAttributes());
273 if (F.getReturnType()->isVoidTy())
274 Builder.CreateRetVoid();
275 else
276 Builder.CreateRet(V: Call);
277}
278
279std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
280 std::vector<GlobalValue *> PromotedGlobals;
281
282 for (auto &GV : M.global_values()) {
283 bool Promoted = true;
284
285 // Rename if necessary.
286 if (!GV.hasName())
287 GV.setName("__orc_anon." + Twine(NextId++));
288 else if (GV.getName().starts_with(Prefix: "\01L"))
289 GV.setName("__" + GV.getName().substr(Start: 1) + "." + Twine(NextId++));
290 else if (GV.hasLocalLinkage())
291 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
292 else
293 Promoted = false;
294
295 if (GV.hasLocalLinkage()) {
296 GV.setLinkage(GlobalValue::ExternalLinkage);
297 GV.setVisibility(GlobalValue::HiddenVisibility);
298 Promoted = true;
299 }
300 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
301
302 if (Promoted)
303 PromotedGlobals.push_back(x: &GV);
304 }
305
306 return PromotedGlobals;
307}
308
309Function* cloneFunctionDecl(Module &Dst, const Function &F,
310 ValueToValueMapTy *VMap) {
311 Function *NewF =
312 Function::Create(Ty: cast<FunctionType>(Val: F.getValueType()),
313 Linkage: F.getLinkage(), N: F.getName(), M: &Dst);
314 NewF->copyAttributesFrom(Src: &F);
315
316 if (VMap) {
317 (*VMap)[&F] = NewF;
318 auto NewArgI = NewF->arg_begin();
319 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
320 ++ArgI, ++NewArgI)
321 (*VMap)[&*ArgI] = &*NewArgI;
322 }
323
324 return NewF;
325}
326
327GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
328 ValueToValueMapTy *VMap) {
329 GlobalVariable *NewGV = new GlobalVariable(
330 Dst, GV.getValueType(), GV.isConstant(),
331 GV.getLinkage(), nullptr, GV.getName(), nullptr,
332 GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
333 NewGV->copyAttributesFrom(Src: &GV);
334 if (VMap)
335 (*VMap)[&GV] = NewGV;
336 return NewGV;
337}
338
339GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
340 ValueToValueMapTy &VMap) {
341 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
342 auto *NewA = GlobalAlias::create(Ty: OrigA.getValueType(),
343 AddressSpace: OrigA.getType()->getPointerAddressSpace(),
344 Linkage: OrigA.getLinkage(), Name: OrigA.getName(), Parent: &Dst);
345 NewA->copyAttributesFrom(Src: &OrigA);
346 VMap[&OrigA] = NewA;
347 return NewA;
348}
349
350Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
351 jitlink::LinkGraph &G,
352 MCDisassembler &Disassembler,
353 MCInstrAnalysis &MIA) {
354 // AArch64 appears to already come with the necessary relocations. Among other
355 // architectures, only x86_64 is currently implemented here.
356 if (G.getTargetTriple().getArch() != Triple::x86_64)
357 return Error::success();
358
359 raw_null_ostream CommentStream;
360 auto &STI = Disassembler.getSubtargetInfo();
361
362 // Determine the function bounds
363 auto &B = Sym.getBlock();
364 assert(!B.isZeroFill() && "expected content block");
365 auto SymAddress = Sym.getAddress();
366 auto SymStartInBlock =
367 (const uint8_t *)B.getContent().data() + Sym.getOffset();
368 auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset();
369 auto Content = ArrayRef(SymStartInBlock, SymSize);
370
371 LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n");
372
373 SmallDenseSet<uintptr_t, 8> ExistingRelocations;
374 for (auto &E : B.edges()) {
375 if (E.isRelocation())
376 ExistingRelocations.insert(V: E.getOffset());
377 }
378
379 size_t I = 0;
380 while (I < Content.size()) {
381 MCInst Instr;
382 uint64_t InstrSize = 0;
383 uint64_t InstrStart = SymAddress.getValue() + I;
384 auto DecodeStatus = Disassembler.getInstruction(
385 Instr, Size&: InstrSize, Bytes: Content.drop_front(N: I), Address: InstrStart, CStream&: CommentStream);
386 if (DecodeStatus != MCDisassembler::Success) {
387 LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address "
388 << InstrStart);
389 return make_error<StringError>(
390 Args: formatv(Fmt: "failed to disassemble at address {0:x16}", Vals&: InstrStart),
391 Args: inconvertibleErrorCode());
392 }
393 // Advance to the next instruction.
394 I += InstrSize;
395
396 // Check for a PC-relative address equal to the symbol itself.
397 auto PCRelAddr =
398 MIA.evaluateMemoryOperandAddress(Inst: Instr, STI: &STI, Addr: InstrStart, Size: InstrSize);
399 if (!PCRelAddr || *PCRelAddr != SymAddress.getValue())
400 continue;
401
402 auto RelocOffInInstr =
403 MIA.getMemoryOperandRelocationOffset(Inst: Instr, Size: InstrSize);
404 if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) {
405 LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at "
406 << InstrStart);
407 continue;
408 }
409
410 auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr -
411 SymAddress + Sym.getOffset();
412 if (ExistingRelocations.contains(V: RelocOffInBlock))
413 continue;
414
415 LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart);
416 B.addEdge(K: jitlink::x86_64::Delta32, Offset: RelocOffInBlock, Target&: Sym, /*Addend=*/-4);
417 }
418 return Error::success();
419}
420
421} // End namespace orc.
422} // End namespace llvm.
423

source code of llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp