1 | //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" |
10 | #include "llvm/ADT/STLExtras.h" |
11 | #include "llvm/ExecutionEngine/JITLink/x86_64.h" |
12 | #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" |
13 | #include "llvm/IR/IRBuilder.h" |
14 | #include "llvm/MC/MCDisassembler/MCDisassembler.h" |
15 | #include "llvm/MC/MCInstrAnalysis.h" |
16 | #include "llvm/Support/Format.h" |
17 | #include "llvm/TargetParser/Triple.h" |
18 | #include "llvm/Transforms/Utils/Cloning.h" |
19 | #include <sstream> |
20 | |
21 | #define DEBUG_TYPE "orc" |
22 | |
23 | using namespace llvm; |
24 | using namespace llvm::orc; |
25 | |
26 | namespace { |
27 | |
28 | class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { |
29 | public: |
30 | using CompileFunction = JITCompileCallbackManager::CompileFunction; |
31 | |
32 | CompileCallbackMaterializationUnit(SymbolStringPtr Name, |
33 | CompileFunction Compile) |
34 | : MaterializationUnit(Interface( |
35 | SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)), |
36 | Name(std::move(Name)), Compile(std::move(Compile)) {} |
37 | |
38 | StringRef getName() const override { return "<Compile Callbacks>" ; } |
39 | |
40 | private: |
41 | void materialize(std::unique_ptr<MaterializationResponsibility> R) override { |
42 | SymbolMap Result; |
43 | Result[Name] = {Compile(), JITSymbolFlags::Exported}; |
44 | // No dependencies, so these calls cannot fail. |
45 | cantFail(Err: R->notifyResolved(Symbols: Result)); |
46 | cantFail(Err: R->notifyEmitted(EmittedDeps: {})); |
47 | } |
48 | |
49 | void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { |
50 | llvm_unreachable("Discard should never occur on a LMU?" ); |
51 | } |
52 | |
53 | SymbolStringPtr Name; |
54 | CompileFunction Compile; |
55 | }; |
56 | |
57 | } // namespace |
58 | |
59 | namespace llvm { |
60 | namespace orc { |
61 | |
62 | TrampolinePool::~TrampolinePool() = default; |
63 | void IndirectStubsManager::anchor() {} |
64 | |
65 | Expected<ExecutorAddr> |
66 | JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { |
67 | if (auto TrampolineAddr = TP->getTrampoline()) { |
68 | auto CallbackName = |
69 | ES.intern(SymName: std::string("cc" ) + std::to_string(val: ++NextCallbackId)); |
70 | |
71 | std::lock_guard<std::mutex> Lock(CCMgrMutex); |
72 | AddrToSymbol[*TrampolineAddr] = CallbackName; |
73 | cantFail( |
74 | Err: CallbacksJD.define(MU: std::make_unique<CompileCallbackMaterializationUnit>( |
75 | args: std::move(CallbackName), args: std::move(Compile)))); |
76 | return *TrampolineAddr; |
77 | } else |
78 | return TrampolineAddr.takeError(); |
79 | } |
80 | |
81 | ExecutorAddr |
82 | JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) { |
83 | SymbolStringPtr Name; |
84 | |
85 | { |
86 | std::unique_lock<std::mutex> Lock(CCMgrMutex); |
87 | auto I = AddrToSymbol.find(x: TrampolineAddr); |
88 | |
89 | // If this address is not associated with a compile callback then report an |
90 | // error to the execution session and return ErrorHandlerAddress to the |
91 | // callee. |
92 | if (I == AddrToSymbol.end()) { |
93 | Lock.unlock(); |
94 | ES.reportError( |
95 | Err: make_error<StringError>(Args: "No compile callback for trampoline at " + |
96 | formatv(Fmt: "{0:x}" , Vals&: TrampolineAddr), |
97 | Args: inconvertibleErrorCode())); |
98 | return ErrorHandlerAddress; |
99 | } else |
100 | Name = I->second; |
101 | } |
102 | |
103 | if (auto Sym = |
104 | ES.lookup(SearchOrder: makeJITDylibSearchOrder( |
105 | JDs: &CallbacksJD, Flags: JITDylibLookupFlags::MatchAllSymbols), |
106 | Symbol: Name)) |
107 | return Sym->getAddress(); |
108 | else { |
109 | llvm::dbgs() << "Didn't find callback.\n" ; |
110 | // If anything goes wrong materializing Sym then report it to the session |
111 | // and return the ErrorHandlerAddress; |
112 | ES.reportError(Err: Sym.takeError()); |
113 | return ErrorHandlerAddress; |
114 | } |
115 | } |
116 | |
117 | Expected<std::unique_ptr<JITCompileCallbackManager>> |
118 | createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, |
119 | ExecutorAddr ErrorHandlerAddress) { |
120 | switch (T.getArch()) { |
121 | default: |
122 | return make_error<StringError>( |
123 | Args: std::string("No callback manager available for " ) + T.str(), |
124 | Args: inconvertibleErrorCode()); |
125 | case Triple::aarch64: |
126 | case Triple::aarch64_32: { |
127 | typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; |
128 | return CCMgrT::Create(ES, ErrorHandlerAddress); |
129 | } |
130 | |
131 | case Triple::x86: { |
132 | typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; |
133 | return CCMgrT::Create(ES, ErrorHandlerAddress); |
134 | } |
135 | |
136 | case Triple::loongarch64: { |
137 | typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT; |
138 | return CCMgrT::Create(ES, ErrorHandlerAddress); |
139 | } |
140 | |
141 | case Triple::mips: { |
142 | typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT; |
143 | return CCMgrT::Create(ES, ErrorHandlerAddress); |
144 | } |
145 | case Triple::mipsel: { |
146 | typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT; |
147 | return CCMgrT::Create(ES, ErrorHandlerAddress); |
148 | } |
149 | |
150 | case Triple::mips64: |
151 | case Triple::mips64el: { |
152 | typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT; |
153 | return CCMgrT::Create(ES, ErrorHandlerAddress); |
154 | } |
155 | |
156 | case Triple::riscv64: { |
157 | typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT; |
158 | return CCMgrT::Create(ES, ErrorHandlerAddress); |
159 | } |
160 | |
161 | case Triple::x86_64: { |
162 | if (T.getOS() == Triple::OSType::Win32) { |
163 | typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; |
164 | return CCMgrT::Create(ES, ErrorHandlerAddress); |
165 | } else { |
166 | typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; |
167 | return CCMgrT::Create(ES, ErrorHandlerAddress); |
168 | } |
169 | } |
170 | |
171 | } |
172 | } |
173 | |
174 | std::function<std::unique_ptr<IndirectStubsManager>()> |
175 | createLocalIndirectStubsManagerBuilder(const Triple &T) { |
176 | switch (T.getArch()) { |
177 | default: |
178 | return [](){ |
179 | return std::make_unique< |
180 | orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); |
181 | }; |
182 | |
183 | case Triple::aarch64: |
184 | case Triple::aarch64_32: |
185 | return [](){ |
186 | return std::make_unique< |
187 | orc::LocalIndirectStubsManager<orc::OrcAArch64>>(); |
188 | }; |
189 | |
190 | case Triple::x86: |
191 | return [](){ |
192 | return std::make_unique< |
193 | orc::LocalIndirectStubsManager<orc::OrcI386>>(); |
194 | }; |
195 | |
196 | case Triple::loongarch64: |
197 | return []() { |
198 | return std::make_unique< |
199 | orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>(); |
200 | }; |
201 | |
202 | case Triple::mips: |
203 | return [](){ |
204 | return std::make_unique< |
205 | orc::LocalIndirectStubsManager<orc::OrcMips32Be>>(); |
206 | }; |
207 | |
208 | case Triple::mipsel: |
209 | return [](){ |
210 | return std::make_unique< |
211 | orc::LocalIndirectStubsManager<orc::OrcMips32Le>>(); |
212 | }; |
213 | |
214 | case Triple::mips64: |
215 | case Triple::mips64el: |
216 | return [](){ |
217 | return std::make_unique< |
218 | orc::LocalIndirectStubsManager<orc::OrcMips64>>(); |
219 | }; |
220 | |
221 | case Triple::riscv64: |
222 | return []() { |
223 | return std::make_unique< |
224 | orc::LocalIndirectStubsManager<orc::OrcRiscv64>>(); |
225 | }; |
226 | |
227 | case Triple::x86_64: |
228 | if (T.getOS() == Triple::OSType::Win32) { |
229 | return [](){ |
230 | return std::make_unique< |
231 | orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>(); |
232 | }; |
233 | } else { |
234 | return [](){ |
235 | return std::make_unique< |
236 | orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>(); |
237 | }; |
238 | } |
239 | |
240 | } |
241 | } |
242 | |
243 | Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) { |
244 | Constant *AddrIntVal = |
245 | ConstantInt::get(Ty: Type::getInt64Ty(C&: FT.getContext()), V: Addr.getValue()); |
246 | Constant *AddrPtrVal = |
247 | ConstantExpr::getIntToPtr(C: AddrIntVal, Ty: PointerType::get(ElementType: &FT, AddressSpace: 0)); |
248 | return AddrPtrVal; |
249 | } |
250 | |
251 | GlobalVariable* createImplPointer(PointerType &PT, Module &M, |
252 | const Twine &Name, Constant *Initializer) { |
253 | auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, |
254 | Initializer, Name, nullptr, |
255 | GlobalValue::NotThreadLocal, 0, true); |
256 | IP->setVisibility(GlobalValue::HiddenVisibility); |
257 | return IP; |
258 | } |
259 | |
260 | void makeStub(Function &F, Value &ImplPointer) { |
261 | assert(F.isDeclaration() && "Can't turn a definition into a stub." ); |
262 | assert(F.getParent() && "Function isn't in a module." ); |
263 | Module &M = *F.getParent(); |
264 | BasicBlock *EntryBlock = BasicBlock::Create(Context&: M.getContext(), Name: "entry" , Parent: &F); |
265 | IRBuilder<> Builder(EntryBlock); |
266 | LoadInst *ImplAddr = Builder.CreateLoad(Ty: F.getType(), Ptr: &ImplPointer); |
267 | std::vector<Value*> CallArgs; |
268 | for (auto &A : F.args()) |
269 | CallArgs.push_back(x: &A); |
270 | CallInst *Call = Builder.CreateCall(FTy: F.getFunctionType(), Callee: ImplAddr, Args: CallArgs); |
271 | Call->setTailCall(); |
272 | Call->setAttributes(F.getAttributes()); |
273 | if (F.getReturnType()->isVoidTy()) |
274 | Builder.CreateRetVoid(); |
275 | else |
276 | Builder.CreateRet(V: Call); |
277 | } |
278 | |
279 | std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) { |
280 | std::vector<GlobalValue *> PromotedGlobals; |
281 | |
282 | for (auto &GV : M.global_values()) { |
283 | bool Promoted = true; |
284 | |
285 | // Rename if necessary. |
286 | if (!GV.hasName()) |
287 | GV.setName("__orc_anon." + Twine(NextId++)); |
288 | else if (GV.getName().starts_with(Prefix: "\01L" )) |
289 | GV.setName("__" + GV.getName().substr(Start: 1) + "." + Twine(NextId++)); |
290 | else if (GV.hasLocalLinkage()) |
291 | GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++)); |
292 | else |
293 | Promoted = false; |
294 | |
295 | if (GV.hasLocalLinkage()) { |
296 | GV.setLinkage(GlobalValue::ExternalLinkage); |
297 | GV.setVisibility(GlobalValue::HiddenVisibility); |
298 | Promoted = true; |
299 | } |
300 | GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None); |
301 | |
302 | if (Promoted) |
303 | PromotedGlobals.push_back(x: &GV); |
304 | } |
305 | |
306 | return PromotedGlobals; |
307 | } |
308 | |
309 | Function* cloneFunctionDecl(Module &Dst, const Function &F, |
310 | ValueToValueMapTy *VMap) { |
311 | Function *NewF = |
312 | Function::Create(Ty: cast<FunctionType>(Val: F.getValueType()), |
313 | Linkage: F.getLinkage(), N: F.getName(), M: &Dst); |
314 | NewF->copyAttributesFrom(Src: &F); |
315 | |
316 | if (VMap) { |
317 | (*VMap)[&F] = NewF; |
318 | auto NewArgI = NewF->arg_begin(); |
319 | for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE; |
320 | ++ArgI, ++NewArgI) |
321 | (*VMap)[&*ArgI] = &*NewArgI; |
322 | } |
323 | |
324 | return NewF; |
325 | } |
326 | |
327 | GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, |
328 | ValueToValueMapTy *VMap) { |
329 | GlobalVariable *NewGV = new GlobalVariable( |
330 | Dst, GV.getValueType(), GV.isConstant(), |
331 | GV.getLinkage(), nullptr, GV.getName(), nullptr, |
332 | GV.getThreadLocalMode(), GV.getType()->getAddressSpace()); |
333 | NewGV->copyAttributesFrom(Src: &GV); |
334 | if (VMap) |
335 | (*VMap)[&GV] = NewGV; |
336 | return NewGV; |
337 | } |
338 | |
339 | GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, |
340 | ValueToValueMapTy &VMap) { |
341 | assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?" ); |
342 | auto *NewA = GlobalAlias::create(Ty: OrigA.getValueType(), |
343 | AddressSpace: OrigA.getType()->getPointerAddressSpace(), |
344 | Linkage: OrigA.getLinkage(), Name: OrigA.getName(), Parent: &Dst); |
345 | NewA->copyAttributesFrom(Src: &OrigA); |
346 | VMap[&OrigA] = NewA; |
347 | return NewA; |
348 | } |
349 | |
350 | Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym, |
351 | jitlink::LinkGraph &G, |
352 | MCDisassembler &Disassembler, |
353 | MCInstrAnalysis &MIA) { |
354 | // AArch64 appears to already come with the necessary relocations. Among other |
355 | // architectures, only x86_64 is currently implemented here. |
356 | if (G.getTargetTriple().getArch() != Triple::x86_64) |
357 | return Error::success(); |
358 | |
359 | raw_null_ostream ; |
360 | auto &STI = Disassembler.getSubtargetInfo(); |
361 | |
362 | // Determine the function bounds |
363 | auto &B = Sym.getBlock(); |
364 | assert(!B.isZeroFill() && "expected content block" ); |
365 | auto SymAddress = Sym.getAddress(); |
366 | auto SymStartInBlock = |
367 | (const uint8_t *)B.getContent().data() + Sym.getOffset(); |
368 | auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset(); |
369 | auto Content = ArrayRef(SymStartInBlock, SymSize); |
370 | |
371 | LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n" ); |
372 | |
373 | SmallDenseSet<uintptr_t, 8> ExistingRelocations; |
374 | for (auto &E : B.edges()) { |
375 | if (E.isRelocation()) |
376 | ExistingRelocations.insert(V: E.getOffset()); |
377 | } |
378 | |
379 | size_t I = 0; |
380 | while (I < Content.size()) { |
381 | MCInst Instr; |
382 | uint64_t InstrSize = 0; |
383 | uint64_t InstrStart = SymAddress.getValue() + I; |
384 | auto DecodeStatus = Disassembler.getInstruction( |
385 | Instr, Size&: InstrSize, Bytes: Content.drop_front(N: I), Address: InstrStart, CStream&: CommentStream); |
386 | if (DecodeStatus != MCDisassembler::Success) { |
387 | LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address " |
388 | << InstrStart); |
389 | return make_error<StringError>( |
390 | Args: formatv(Fmt: "failed to disassemble at address {0:x16}" , Vals&: InstrStart), |
391 | Args: inconvertibleErrorCode()); |
392 | } |
393 | // Advance to the next instruction. |
394 | I += InstrSize; |
395 | |
396 | // Check for a PC-relative address equal to the symbol itself. |
397 | auto PCRelAddr = |
398 | MIA.evaluateMemoryOperandAddress(Inst: Instr, STI: &STI, Addr: InstrStart, Size: InstrSize); |
399 | if (!PCRelAddr || *PCRelAddr != SymAddress.getValue()) |
400 | continue; |
401 | |
402 | auto RelocOffInInstr = |
403 | MIA.getMemoryOperandRelocationOffset(Inst: Instr, Size: InstrSize); |
404 | if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) { |
405 | LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at " |
406 | << InstrStart); |
407 | continue; |
408 | } |
409 | |
410 | auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr - |
411 | SymAddress + Sym.getOffset(); |
412 | if (ExistingRelocations.contains(V: RelocOffInBlock)) |
413 | continue; |
414 | |
415 | LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart); |
416 | B.addEdge(K: jitlink::x86_64::Delta32, Offset: RelocOffInBlock, Target&: Sym, /*Addend=*/-4); |
417 | } |
418 | return Error::success(); |
419 | } |
420 | |
421 | } // End namespace orc. |
422 | } // End namespace llvm. |
423 | |