1 | //===------ MachOPlatform.cpp - Utilities for executing MachO in Orc ------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" |
10 | |
11 | #include "llvm/BinaryFormat/MachO.h" |
12 | #include "llvm/ExecutionEngine/JITLink/MachO.h" |
13 | #include "llvm/ExecutionEngine/JITLink/aarch64.h" |
14 | #include "llvm/ExecutionEngine/JITLink/x86_64.h" |
15 | #include "llvm/ExecutionEngine/Orc/DebugUtils.h" |
16 | #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" |
17 | #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" |
18 | #include "llvm/ExecutionEngine/Orc/MachOBuilder.h" |
19 | #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" |
20 | #include "llvm/Support/BinaryByteStream.h" |
21 | #include "llvm/Support/Debug.h" |
22 | #include <optional> |
23 | |
24 | #define DEBUG_TYPE "orc" |
25 | |
26 | using namespace llvm; |
27 | using namespace llvm::orc; |
28 | using namespace llvm::orc::shared; |
29 | |
30 | namespace llvm { |
31 | namespace orc { |
32 | namespace shared { |
33 | |
34 | using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>; |
35 | using SPSMachOJITDylibDepInfoMap = |
36 | SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>; |
37 | |
38 | class SPSMachOExecutorSymbolFlags; |
39 | |
40 | template <> |
41 | class SPSSerializationTraits<SPSMachOJITDylibDepInfo, |
42 | MachOPlatform::MachOJITDylibDepInfo> { |
43 | public: |
44 | static size_t size(const MachOPlatform::MachOJITDylibDepInfo &DDI) { |
45 | return SPSMachOJITDylibDepInfo::AsArgList::size(Arg: DDI.Sealed, Args: DDI.DepHeaders); |
46 | } |
47 | |
48 | static bool serialize(SPSOutputBuffer &OB, |
49 | const MachOPlatform::MachOJITDylibDepInfo &DDI) { |
50 | return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, Arg: DDI.Sealed, |
51 | Args: DDI.DepHeaders); |
52 | } |
53 | |
54 | static bool deserialize(SPSInputBuffer &IB, |
55 | MachOPlatform::MachOJITDylibDepInfo &DDI) { |
56 | return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, Arg&: DDI.Sealed, |
57 | Args&: DDI.DepHeaders); |
58 | } |
59 | }; |
60 | |
61 | template <> |
62 | class SPSSerializationTraits<SPSMachOExecutorSymbolFlags, |
63 | MachOPlatform::MachOExecutorSymbolFlags> { |
64 | private: |
65 | using UT = std::underlying_type_t<MachOPlatform::MachOExecutorSymbolFlags>; |
66 | |
67 | public: |
68 | static size_t size(const MachOPlatform::MachOExecutorSymbolFlags &SF) { |
69 | return sizeof(UT); |
70 | } |
71 | |
72 | static bool serialize(SPSOutputBuffer &OB, |
73 | const MachOPlatform::MachOExecutorSymbolFlags &SF) { |
74 | return SPSArgList<UT>::serialize(OB, Arg: static_cast<UT>(SF)); |
75 | } |
76 | |
77 | static bool deserialize(SPSInputBuffer &IB, |
78 | MachOPlatform::MachOExecutorSymbolFlags &SF) { |
79 | UT Tmp; |
80 | if (!SPSArgList<UT>::deserialize(IB, Arg&: Tmp)) |
81 | return false; |
82 | SF = static_cast<MachOPlatform::MachOExecutorSymbolFlags>(Tmp); |
83 | return true; |
84 | } |
85 | }; |
86 | |
87 | } // namespace shared |
88 | } // namespace orc |
89 | } // namespace llvm |
90 | |
91 | namespace { |
92 | |
93 | using SPSRegisterSymbolsArgs = |
94 | SPSArgList<SPSExecutorAddr, |
95 | SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr, |
96 | SPSMachOExecutorSymbolFlags>>>; |
97 | |
98 | std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(MachOPlatform &MOP, |
99 | std::string Name) { |
100 | unsigned PointerSize; |
101 | llvm::endianness Endianness; |
102 | const auto &TT = MOP.getExecutionSession().getTargetTriple(); |
103 | |
104 | switch (TT.getArch()) { |
105 | case Triple::aarch64: |
106 | case Triple::x86_64: |
107 | PointerSize = 8; |
108 | Endianness = llvm::endianness::little; |
109 | break; |
110 | default: |
111 | llvm_unreachable("Unrecognized architecture" ); |
112 | } |
113 | |
114 | return std::make_unique<jitlink::LinkGraph>(args: std::move(Name), args: TT, args&: PointerSize, |
115 | args&: Endianness, |
116 | args&: jitlink::getGenericEdgeKindName); |
117 | } |
118 | |
119 | // Creates a Bootstrap-Complete LinkGraph to run deferred actions. |
120 | class MachOPlatformCompleteBootstrapMaterializationUnit |
121 | : public MaterializationUnit { |
122 | public: |
123 | using SymbolTableVector = |
124 | SmallVector<std::tuple<ExecutorAddr, ExecutorAddr, |
125 | MachOPlatform::MachOExecutorSymbolFlags>>; |
126 | |
127 | MachOPlatformCompleteBootstrapMaterializationUnit( |
128 | MachOPlatform &MOP, StringRef PlatformJDName, |
129 | SymbolStringPtr CompleteBootstrapSymbol, SymbolTableVector SymTab, |
130 | shared::AllocActions DeferredAAs, ExecutorAddr , |
131 | ExecutorAddr PlatformBootstrap, ExecutorAddr PlatformShutdown, |
132 | ExecutorAddr RegisterJITDylib, ExecutorAddr DeregisterJITDylib, |
133 | ExecutorAddr RegisterObjectSymbolTable, |
134 | ExecutorAddr DeregisterObjectSymbolTable) |
135 | : MaterializationUnit( |
136 | {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}), |
137 | MOP(MOP), PlatformJDName(PlatformJDName), |
138 | CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)), |
139 | SymTab(std::move(SymTab)), DeferredAAs(std::move(DeferredAAs)), |
140 | MachOHeaderAddr(MachOHeaderAddr), PlatformBootstrap(PlatformBootstrap), |
141 | PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib), |
142 | DeregisterJITDylib(DeregisterJITDylib), |
143 | RegisterObjectSymbolTable(RegisterObjectSymbolTable), |
144 | DeregisterObjectSymbolTable(DeregisterObjectSymbolTable) {} |
145 | |
146 | StringRef getName() const override { |
147 | return "MachOPlatformCompleteBootstrap" ; |
148 | } |
149 | |
150 | void materialize(std::unique_ptr<MaterializationResponsibility> R) override { |
151 | using namespace jitlink; |
152 | auto G = createPlatformGraph(MOP, Name: "<OrcRTCompleteBootstrap>" ); |
153 | auto &PlaceholderSection = |
154 | G->createSection(Name: "__orc_rt_cplt_bs" , Prot: MemProt::Read); |
155 | auto &PlaceholderBlock = |
156 | G->createZeroFillBlock(Parent&: PlaceholderSection, Size: 1, Address: ExecutorAddr(), Alignment: 1, AlignmentOffset: 0); |
157 | G->addDefinedSymbol(Content&: PlaceholderBlock, Offset: 0, Name: *CompleteBootstrapSymbol, Size: 1, |
158 | L: Linkage::Strong, S: Scope::Hidden, IsCallable: false, IsLive: true); |
159 | |
160 | // Reserve space for the stolen actions, plus two extras. |
161 | G->allocActions().reserve(n: DeferredAAs.size() + 3); |
162 | |
163 | // 1. Bootstrap the platform support code. |
164 | G->allocActions().push_back( |
165 | x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<>>(FnAddr: PlatformBootstrap)), |
166 | .Dealloc: cantFail( |
167 | ValOrErr: WrapperFunctionCall::Create<SPSArgList<>>(FnAddr: PlatformShutdown))}); |
168 | |
169 | // 2. Register the platform JITDylib. |
170 | G->allocActions().push_back( |
171 | x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create< |
172 | SPSArgList<SPSString, SPSExecutorAddr>>( |
173 | FnAddr: RegisterJITDylib, Args: PlatformJDName, Args: MachOHeaderAddr)), |
174 | .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>( |
175 | FnAddr: DeregisterJITDylib, Args: MachOHeaderAddr))}); |
176 | |
177 | // 3. Register deferred symbols. |
178 | G->allocActions().push_back( |
179 | x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>( |
180 | FnAddr: RegisterObjectSymbolTable, Args: MachOHeaderAddr, Args: SymTab)), |
181 | .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>( |
182 | FnAddr: DeregisterObjectSymbolTable, Args: MachOHeaderAddr, Args: SymTab))}); |
183 | |
184 | // 4. Add the deferred actions to the graph. |
185 | std::move(first: DeferredAAs.begin(), last: DeferredAAs.end(), |
186 | result: std::back_inserter(x&: G->allocActions())); |
187 | |
188 | MOP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G)); |
189 | } |
190 | |
191 | void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} |
192 | |
193 | private: |
194 | MachOPlatform &MOP; |
195 | StringRef PlatformJDName; |
196 | SymbolStringPtr CompleteBootstrapSymbol; |
197 | SymbolTableVector SymTab; |
198 | shared::AllocActions DeferredAAs; |
199 | ExecutorAddr ; |
200 | ExecutorAddr PlatformBootstrap; |
201 | ExecutorAddr PlatformShutdown; |
202 | ExecutorAddr RegisterJITDylib; |
203 | ExecutorAddr DeregisterJITDylib; |
204 | ExecutorAddr RegisterObjectSymbolTable; |
205 | ExecutorAddr DeregisterObjectSymbolTable; |
206 | }; |
207 | |
208 | static StringRef ObjCRuntimeObjectSectionsData[] = { |
209 | MachOObjCCatListSectionName, MachOObjCClassListSectionName, |
210 | MachOObjCClassRefsSectionName, MachOObjCConstSectionName, |
211 | MachOObjCDataSectionName, MachOObjCSelRefsSectionName}; |
212 | |
213 | static StringRef ObjCRuntimeObjectSectionsText[] = { |
214 | MachOObjCClassNameSectionName, MachOObjCMethNameSectionName, |
215 | MachOObjCMethTypeSectionName, MachOSwift5TypesSectionName, |
216 | MachOSwift5TypeRefSectionName, MachOSwift5FieldMetadataSectionName, |
217 | MachOSwift5EntrySectionName, MachOSwift5ProtoSectionName, |
218 | MachOSwift5ProtosSectionName}; |
219 | |
220 | static StringRef ObjCRuntimeObjectSectionName = |
221 | "__llvm_jitlink_ObjCRuntimeRegistrationObject" ; |
222 | |
223 | static StringRef ObjCImageInfoSymbolName = |
224 | "__llvm_jitlink_macho_objc_imageinfo" ; |
225 | |
226 | struct ObjCImageInfoFlags { |
227 | uint16_t SwiftABIVersion; |
228 | uint16_t SwiftVersion; |
229 | bool HasCategoryClassProperties; |
230 | bool HasSignedObjCClassROs; |
231 | |
232 | static constexpr uint32_t SIGNED_CLASS_RO = (1 << 4); |
233 | static constexpr uint32_t HAS_CATEGORY_CLASS_PROPERTIES = (1 << 6); |
234 | |
235 | explicit ObjCImageInfoFlags(uint32_t RawFlags) { |
236 | HasSignedObjCClassROs = RawFlags & SIGNED_CLASS_RO; |
237 | HasCategoryClassProperties = RawFlags & HAS_CATEGORY_CLASS_PROPERTIES; |
238 | SwiftABIVersion = (RawFlags >> 8) & 0xFF; |
239 | SwiftVersion = (RawFlags >> 16) & 0xFFFF; |
240 | } |
241 | |
242 | uint32_t rawFlags() const { |
243 | uint32_t Result = 0; |
244 | if (HasCategoryClassProperties) |
245 | Result |= HAS_CATEGORY_CLASS_PROPERTIES; |
246 | if (HasSignedObjCClassROs) |
247 | Result |= SIGNED_CLASS_RO; |
248 | Result |= (SwiftABIVersion << 8); |
249 | Result |= (SwiftVersion << 16); |
250 | return Result; |
251 | } |
252 | }; |
253 | } // end anonymous namespace |
254 | |
255 | namespace llvm { |
256 | namespace orc { |
257 | |
258 | std::optional<MachOPlatform::HeaderOptions::BuildVersionOpts> |
259 | MachOPlatform::HeaderOptions::BuildVersionOpts::(const Triple &TT, |
260 | uint32_t MinOS, |
261 | uint32_t SDK) { |
262 | |
263 | uint32_t Platform; |
264 | switch (TT.getOS()) { |
265 | case Triple::IOS: |
266 | Platform = TT.isSimulatorEnvironment() ? MachO::PLATFORM_IOSSIMULATOR |
267 | : MachO::PLATFORM_IOS; |
268 | break; |
269 | case Triple::MacOSX: |
270 | Platform = MachO::PLATFORM_MACOS; |
271 | break; |
272 | case Triple::TvOS: |
273 | Platform = TT.isSimulatorEnvironment() ? MachO::PLATFORM_TVOSSIMULATOR |
274 | : MachO::PLATFORM_TVOS; |
275 | break; |
276 | case Triple::WatchOS: |
277 | Platform = TT.isSimulatorEnvironment() ? MachO::PLATFORM_WATCHOSSIMULATOR |
278 | : MachO::PLATFORM_WATCHOS; |
279 | break; |
280 | default: |
281 | return std::nullopt; |
282 | } |
283 | |
284 | return MachOPlatform::HeaderOptions::BuildVersionOpts{.Platform: Platform, .MinOS: MinOS, .SDK: SDK}; |
285 | } |
286 | |
287 | Expected<std::unique_ptr<MachOPlatform>> MachOPlatform::Create( |
288 | ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, |
289 | JITDylib &PlatformJD, std::unique_ptr<DefinitionGenerator> OrcRuntime, |
290 | HeaderOptions PlatformJDOpts, MachOHeaderMUBuilder , |
291 | std::optional<SymbolAliasMap> RuntimeAliases) { |
292 | |
293 | // If the target is not supported then bail out immediately. |
294 | if (!supportedTarget(TT: ES.getTargetTriple())) |
295 | return make_error<StringError>(Args: "Unsupported MachOPlatform triple: " + |
296 | ES.getTargetTriple().str(), |
297 | Args: inconvertibleErrorCode()); |
298 | |
299 | auto &EPC = ES.getExecutorProcessControl(); |
300 | |
301 | // Create default aliases if the caller didn't supply any. |
302 | if (!RuntimeAliases) |
303 | RuntimeAliases = standardPlatformAliases(ES); |
304 | |
305 | // Define the aliases. |
306 | if (auto Err = PlatformJD.define(MU: symbolAliases(Aliases: std::move(*RuntimeAliases)))) |
307 | return std::move(Err); |
308 | |
309 | // Add JIT-dispatch function support symbols. |
310 | if (auto Err = PlatformJD.define( |
311 | MU: absoluteSymbols(Symbols: {{ES.intern(SymName: "___orc_rt_jit_dispatch" ), |
312 | {EPC.getJITDispatchInfo().JITDispatchFunction, |
313 | JITSymbolFlags::Exported}}, |
314 | {ES.intern(SymName: "___orc_rt_jit_dispatch_ctx" ), |
315 | {EPC.getJITDispatchInfo().JITDispatchContext, |
316 | JITSymbolFlags::Exported}}}))) |
317 | return std::move(Err); |
318 | |
319 | // Create the instance. |
320 | Error Err = Error::success(); |
321 | auto P = std::unique_ptr<MachOPlatform>(new MachOPlatform( |
322 | ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), |
323 | std::move(PlatformJDOpts), std::move(BuildMachOHeaderMU), Err)); |
324 | if (Err) |
325 | return std::move(Err); |
326 | return std::move(P); |
327 | } |
328 | |
329 | Expected<std::unique_ptr<MachOPlatform>> |
330 | MachOPlatform::(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, |
331 | JITDylib &PlatformJD, const char *OrcRuntimePath, |
332 | HeaderOptions PlatformJDOpts, |
333 | MachOHeaderMUBuilder , |
334 | std::optional<SymbolAliasMap> RuntimeAliases) { |
335 | |
336 | // Create a generator for the ORC runtime archive. |
337 | auto OrcRuntimeArchiveGenerator = |
338 | StaticLibraryDefinitionGenerator::Load(L&: ObjLinkingLayer, FileName: OrcRuntimePath); |
339 | if (!OrcRuntimeArchiveGenerator) |
340 | return OrcRuntimeArchiveGenerator.takeError(); |
341 | |
342 | return Create(ES, ObjLinkingLayer, PlatformJD, |
343 | OrcRuntime: std::move(*OrcRuntimeArchiveGenerator), |
344 | PlatformJDOpts: std::move(PlatformJDOpts), BuildMachOHeaderMU: std::move(BuildMachOHeaderMU), |
345 | RuntimeAliases: std::move(RuntimeAliases)); |
346 | } |
347 | |
348 | Error MachOPlatform::setupJITDylib(JITDylib &JD) { |
349 | return setupJITDylib(JD, /*Opts=*/{}); |
350 | } |
351 | |
352 | Error MachOPlatform::(JITDylib &JD, HeaderOptions Opts) { |
353 | if (auto Err = JD.define(MU: BuildMachOHeaderMU(*this, std::move(Opts)))) |
354 | return Err; |
355 | |
356 | return ES.lookup(SearchOrder: {&JD}, Symbol: MachOHeaderStartSymbol).takeError(); |
357 | } |
358 | |
359 | Error MachOPlatform::teardownJITDylib(JITDylib &JD) { |
360 | std::lock_guard<std::mutex> Lock(PlatformMutex); |
361 | auto I = JITDylibToHeaderAddr.find(Val: &JD); |
362 | if (I != JITDylibToHeaderAddr.end()) { |
363 | assert(HeaderAddrToJITDylib.count(I->second) && |
364 | "HeaderAddrToJITDylib missing entry" ); |
365 | HeaderAddrToJITDylib.erase(Val: I->second); |
366 | JITDylibToHeaderAddr.erase(I); |
367 | } |
368 | JITDylibToPThreadKey.erase(Val: &JD); |
369 | return Error::success(); |
370 | } |
371 | |
372 | Error MachOPlatform::notifyAdding(ResourceTracker &RT, |
373 | const MaterializationUnit &MU) { |
374 | auto &JD = RT.getJITDylib(); |
375 | const auto &InitSym = MU.getInitializerSymbol(); |
376 | if (!InitSym) |
377 | return Error::success(); |
378 | |
379 | RegisteredInitSymbols[&JD].add(Name: InitSym, |
380 | Flags: SymbolLookupFlags::WeaklyReferencedSymbol); |
381 | LLVM_DEBUG({ |
382 | dbgs() << "MachOPlatform: Registered init symbol " << *InitSym << " for MU " |
383 | << MU.getName() << "\n" ; |
384 | }); |
385 | return Error::success(); |
386 | } |
387 | |
388 | Error MachOPlatform::notifyRemoving(ResourceTracker &RT) { |
389 | llvm_unreachable("Not supported yet" ); |
390 | } |
391 | |
392 | static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases, |
393 | ArrayRef<std::pair<const char *, const char *>> AL) { |
394 | for (auto &KV : AL) { |
395 | auto AliasName = ES.intern(SymName: KV.first); |
396 | assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map" ); |
397 | Aliases[std::move(AliasName)] = {ES.intern(SymName: KV.second), |
398 | JITSymbolFlags::Exported}; |
399 | } |
400 | } |
401 | |
402 | SymbolAliasMap MachOPlatform::standardPlatformAliases(ExecutionSession &ES) { |
403 | SymbolAliasMap Aliases; |
404 | addAliases(ES, Aliases, AL: requiredCXXAliases()); |
405 | addAliases(ES, Aliases, AL: standardRuntimeUtilityAliases()); |
406 | return Aliases; |
407 | } |
408 | |
409 | ArrayRef<std::pair<const char *, const char *>> |
410 | MachOPlatform::requiredCXXAliases() { |
411 | static const std::pair<const char *, const char *> RequiredCXXAliases[] = { |
412 | {"___cxa_atexit" , "___orc_rt_macho_cxa_atexit" }}; |
413 | |
414 | return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases); |
415 | } |
416 | |
417 | ArrayRef<std::pair<const char *, const char *>> |
418 | MachOPlatform::standardRuntimeUtilityAliases() { |
419 | static const std::pair<const char *, const char *> |
420 | StandardRuntimeUtilityAliases[] = { |
421 | {"___orc_rt_run_program" , "___orc_rt_macho_run_program" }, |
422 | {"___orc_rt_jit_dlerror" , "___orc_rt_macho_jit_dlerror" }, |
423 | {"___orc_rt_jit_dlopen" , "___orc_rt_macho_jit_dlopen" }, |
424 | {"___orc_rt_jit_dlclose" , "___orc_rt_macho_jit_dlclose" }, |
425 | {"___orc_rt_jit_dlsym" , "___orc_rt_macho_jit_dlsym" }, |
426 | {"___orc_rt_log_error" , "___orc_rt_log_error_to_stderr" }}; |
427 | |
428 | return ArrayRef<std::pair<const char *, const char *>>( |
429 | StandardRuntimeUtilityAliases); |
430 | } |
431 | |
432 | bool MachOPlatform::supportedTarget(const Triple &TT) { |
433 | switch (TT.getArch()) { |
434 | case Triple::aarch64: |
435 | case Triple::x86_64: |
436 | return true; |
437 | default: |
438 | return false; |
439 | } |
440 | } |
441 | |
442 | jitlink::Edge::Kind MachOPlatform::getPointerEdgeKind(jitlink::LinkGraph &G) { |
443 | switch (G.getTargetTriple().getArch()) { |
444 | case Triple::aarch64: |
445 | return jitlink::aarch64::Pointer64; |
446 | case Triple::x86_64: |
447 | return jitlink::x86_64::Pointer64; |
448 | default: |
449 | llvm_unreachable("Unsupported architecture" ); |
450 | } |
451 | } |
452 | |
453 | MachOPlatform::MachOExecutorSymbolFlags |
454 | MachOPlatform::flagsForSymbol(jitlink::Symbol &Sym) { |
455 | MachOPlatform::MachOExecutorSymbolFlags Flags{}; |
456 | if (Sym.getLinkage() == jitlink::Linkage::Weak) |
457 | Flags |= MachOExecutorSymbolFlags::Weak; |
458 | |
459 | if (Sym.isCallable()) |
460 | Flags |= MachOExecutorSymbolFlags::Callable; |
461 | |
462 | return Flags; |
463 | } |
464 | |
465 | MachOPlatform::MachOPlatform( |
466 | ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, |
467 | JITDylib &PlatformJD, |
468 | std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, |
469 | HeaderOptions PlatformJDOpts, MachOHeaderMUBuilder , |
470 | Error &Err) |
471 | : ES(ES), PlatformJD(PlatformJD), ObjLinkingLayer(ObjLinkingLayer), |
472 | BuildMachOHeaderMU(std::move(BuildMachOHeaderMU)) { |
473 | ErrorAsOutParameter _(&Err); |
474 | ObjLinkingLayer.addPlugin(P: std::make_unique<MachOPlatformPlugin>(args&: *this)); |
475 | PlatformJD.addGenerator(DefGenerator: std::move(OrcRuntimeGenerator)); |
476 | |
477 | BootstrapInfo BI; |
478 | Bootstrap = &BI; |
479 | |
480 | // Bootstrap process -- here be phase-ordering dragons. |
481 | // |
482 | // The MachOPlatform class uses allocation actions to register metadata |
483 | // sections with the ORC runtime, however the runtime contains metadata |
484 | // registration functions that have their own metadata that they need to |
485 | // register (e.g. the frame-info registration functions have frame-info). |
486 | // We can't use an ordinary lookup to find these registration functions |
487 | // because their address is needed during the link of the containing graph |
488 | // itself (to build the allocation actions that will call the registration |
489 | // functions). Further complicating the situation (a) the graph containing |
490 | // the registration functions is allowed to depend on other graphs (e.g. the |
491 | // graph containing the ORC runtime RTTI support) so we need to handle an |
492 | // unknown set of dependencies during bootstrap, and (b) these graphs may |
493 | // be linked concurrently if the user has installed a concurrent dispatcher. |
494 | // |
495 | // We satisfy these constraints by implementing a bootstrap phase during which |
496 | // allocation actions generated by MachOPlatform are appended to a list of |
497 | // deferred allocation actions, rather than to the graphs themselves. At the |
498 | // end of the bootstrap process the deferred actions are attached to a final |
499 | // "complete-bootstrap" graph that causes them to be run. |
500 | // |
501 | // The bootstrap steps are as follows: |
502 | // |
503 | // 1. Request the graph containing the mach header. This graph is guaranteed |
504 | // not to have any metadata so the fact that the registration functions |
505 | // are not available yet is not a problem. |
506 | // |
507 | // 2. Look up the registration functions and discard the results. This will |
508 | // trigger linking of the graph containing these functions, and |
509 | // consequently any graphs that it depends on. We do not use the lookup |
510 | // result to find the addresses of the functions requested (as described |
511 | // above the lookup will return too late for that), instead we capture the |
512 | // addresses in a post-allocation pass injected by the platform runtime |
513 | // during bootstrap only. |
514 | // |
515 | // 3. During bootstrap the MachOPlatformPlugin keeps a count of the number of |
516 | // graphs being linked (potentially concurrently), and we block until all |
517 | // of these graphs have completed linking. This is to avoid a race on the |
518 | // deferred-actions vector: the lookup for the runtime registration |
519 | // functions may return while some functions (those that are being |
520 | // incidentally linked in, but aren't reachable via the runtime functions) |
521 | // are still being linked, and we need to capture any allocation actions |
522 | // for this incidental code before we proceed. |
523 | // |
524 | // 4. Once all active links are complete we transfer the deferred actions to |
525 | // a newly added CompleteBootstrap graph and then request a symbol from |
526 | // the CompleteBootstrap graph to trigger materialization. This will cause |
527 | // all deferred actions to be run, and once this lookup returns we can |
528 | // proceed. |
529 | // |
530 | // 5. Finally, we associate runtime support methods in MachOPlatform with |
531 | // the corresponding jit-dispatch tag variables in the ORC runtime to make |
532 | // the support methods callable. The bootstrap is now complete. |
533 | |
534 | // Step (1) Add header materialization unit and request. |
535 | if ((Err = PlatformJD.define( |
536 | MU: this->BuildMachOHeaderMU(*this, std::move(PlatformJDOpts))))) |
537 | return; |
538 | if ((Err = ES.lookup(SearchOrder: &PlatformJD, Symbol: MachOHeaderStartSymbol).takeError())) |
539 | return; |
540 | |
541 | // Step (2) Request runtime registration functions to trigger |
542 | // materialization.. |
543 | if ((Err = ES.lookup(SearchOrder: makeJITDylibSearchOrder(JDs: &PlatformJD), |
544 | Symbols: SymbolLookupSet( |
545 | {PlatformBootstrap.Name, PlatformShutdown.Name, |
546 | RegisterJITDylib.Name, DeregisterJITDylib.Name, |
547 | RegisterObjectSymbolTable.Name, |
548 | DeregisterObjectSymbolTable.Name, |
549 | RegisterObjectPlatformSections.Name, |
550 | DeregisterObjectPlatformSections.Name, |
551 | CreatePThreadKey.Name})) |
552 | .takeError())) |
553 | return; |
554 | |
555 | // Step (3) Wait for any incidental linker work to complete. |
556 | { |
557 | std::unique_lock<std::mutex> Lock(BI.Mutex); |
558 | BI.CV.wait(lock&: Lock, p: [&]() { return BI.ActiveGraphs == 0; }); |
559 | Bootstrap = nullptr; |
560 | } |
561 | |
562 | // Step (4) Add complete-bootstrap materialization unit and request. |
563 | auto BootstrapCompleteSymbol = ES.intern(SymName: "__orc_rt_macho_complete_bootstrap" ); |
564 | if ((Err = PlatformJD.define( |
565 | MU: std::make_unique<MachOPlatformCompleteBootstrapMaterializationUnit>( |
566 | args&: *this, args: PlatformJD.getName(), args&: BootstrapCompleteSymbol, |
567 | args: std::move(BI.SymTab), args: std::move(BI.DeferredAAs), |
568 | args&: BI.MachOHeaderAddr, args&: PlatformBootstrap.Addr, |
569 | args&: PlatformShutdown.Addr, args&: RegisterJITDylib.Addr, |
570 | args&: DeregisterJITDylib.Addr, args&: RegisterObjectSymbolTable.Addr, |
571 | args&: DeregisterObjectSymbolTable.Addr)))) |
572 | return; |
573 | if ((Err = ES.lookup(SearchOrder: makeJITDylibSearchOrder( |
574 | JDs: &PlatformJD, Flags: JITDylibLookupFlags::MatchAllSymbols), |
575 | Symbol: std::move(BootstrapCompleteSymbol)) |
576 | .takeError())) |
577 | return; |
578 | |
579 | // (5) Associate runtime support functions. |
580 | if ((Err = associateRuntimeSupportFunctions())) |
581 | return; |
582 | } |
583 | |
584 | Error MachOPlatform::associateRuntimeSupportFunctions() { |
585 | ExecutionSession::JITDispatchHandlerAssociationMap WFs; |
586 | |
587 | using = |
588 | SPSExpected<SPSMachOJITDylibDepInfoMap>(SPSExecutorAddr); |
589 | WFs[ES.intern(SymName: "___orc_rt_macho_push_initializers_tag" )] = |
590 | ES.wrapAsyncWithSPS<PushInitializersSPSSig>( |
591 | Instance: this, Method: &MachOPlatform::rt_pushInitializers); |
592 | |
593 | using PushSymbolsSPSSig = |
594 | SPSError(SPSExecutorAddr, SPSSequence<SPSTuple<SPSString, bool>>); |
595 | WFs[ES.intern(SymName: "___orc_rt_macho_push_symbols_tag" )] = |
596 | ES.wrapAsyncWithSPS<PushSymbolsSPSSig>(Instance: this, |
597 | Method: &MachOPlatform::rt_pushSymbols); |
598 | |
599 | return ES.registerJITDispatchHandlers(JD&: PlatformJD, WFs: std::move(WFs)); |
600 | } |
601 | |
602 | void MachOPlatform::pushInitializersLoop( |
603 | PushInitializersSendResultFn SendResult, JITDylibSP JD) { |
604 | DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols; |
605 | DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap; |
606 | SmallVector<JITDylib *, 16> Worklist({JD.get()}); |
607 | |
608 | ES.runSessionLocked(F: [&]() { |
609 | while (!Worklist.empty()) { |
610 | // FIXME: Check for defunct dylibs. |
611 | |
612 | auto DepJD = Worklist.back(); |
613 | Worklist.pop_back(); |
614 | |
615 | // If we've already visited this JITDylib on this iteration then continue. |
616 | if (JDDepMap.count(Val: DepJD)) |
617 | continue; |
618 | |
619 | // Add dep info. |
620 | auto &DM = JDDepMap[DepJD]; |
621 | DepJD->withLinkOrderDo(F: [&](const JITDylibSearchOrder &O) { |
622 | for (auto &KV : O) { |
623 | if (KV.first == DepJD) |
624 | continue; |
625 | DM.push_back(Elt: KV.first); |
626 | Worklist.push_back(Elt: KV.first); |
627 | } |
628 | }); |
629 | |
630 | // Add any registered init symbols. |
631 | auto RISItr = RegisteredInitSymbols.find(Val: DepJD); |
632 | if (RISItr != RegisteredInitSymbols.end()) { |
633 | NewInitSymbols[DepJD] = std::move(RISItr->second); |
634 | RegisteredInitSymbols.erase(I: RISItr); |
635 | } |
636 | } |
637 | }); |
638 | |
639 | // If there are no further init symbols to look up then send the link order |
640 | // (as a list of header addresses) to the caller. |
641 | if (NewInitSymbols.empty()) { |
642 | |
643 | // To make the list intelligible to the runtime we need to convert all |
644 | // JITDylib pointers to their header addresses. Only include JITDylibs |
645 | // that appear in the JITDylibToHeaderAddr map (i.e. those that have been |
646 | // through setupJITDylib) -- bare JITDylibs aren't managed by the platform. |
647 | DenseMap<JITDylib *, ExecutorAddr> ; |
648 | HeaderAddrs.reserve(NumEntries: JDDepMap.size()); |
649 | { |
650 | std::lock_guard<std::mutex> Lock(PlatformMutex); |
651 | for (auto &KV : JDDepMap) { |
652 | auto I = JITDylibToHeaderAddr.find(Val: KV.first); |
653 | if (I != JITDylibToHeaderAddr.end()) |
654 | HeaderAddrs[KV.first] = I->second; |
655 | } |
656 | } |
657 | |
658 | // Build the dep info map to return. |
659 | MachOJITDylibDepInfoMap DIM; |
660 | DIM.reserve(n: JDDepMap.size()); |
661 | for (auto &KV : JDDepMap) { |
662 | auto HI = HeaderAddrs.find(Val: KV.first); |
663 | // Skip unmanaged JITDylibs. |
664 | if (HI == HeaderAddrs.end()) |
665 | continue; |
666 | auto H = HI->second; |
667 | MachOJITDylibDepInfo DepInfo; |
668 | for (auto &Dep : KV.second) { |
669 | auto HJ = HeaderAddrs.find(Val: Dep); |
670 | if (HJ != HeaderAddrs.end()) |
671 | DepInfo.DepHeaders.push_back(x: HJ->second); |
672 | } |
673 | DIM.push_back(x: std::make_pair(x&: H, y: std::move(DepInfo))); |
674 | } |
675 | SendResult(DIM); |
676 | return; |
677 | } |
678 | |
679 | // Otherwise issue a lookup and re-run this phase when it completes. |
680 | lookupInitSymbolsAsync( |
681 | OnComplete: [this, SendResult = std::move(SendResult), JD](Error Err) mutable { |
682 | if (Err) |
683 | SendResult(std::move(Err)); |
684 | else |
685 | pushInitializersLoop(SendResult: std::move(SendResult), JD); |
686 | }, |
687 | ES, InitSyms: std::move(NewInitSymbols)); |
688 | } |
689 | |
690 | void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, |
691 | ExecutorAddr ) { |
692 | JITDylibSP JD; |
693 | { |
694 | std::lock_guard<std::mutex> Lock(PlatformMutex); |
695 | auto I = HeaderAddrToJITDylib.find(Val: JDHeaderAddr); |
696 | if (I != HeaderAddrToJITDylib.end()) |
697 | JD = I->second; |
698 | } |
699 | |
700 | LLVM_DEBUG({ |
701 | dbgs() << "MachOPlatform::rt_pushInitializers(" << JDHeaderAddr << ") " ; |
702 | if (JD) |
703 | dbgs() << "pushing initializers for " << JD->getName() << "\n" ; |
704 | else |
705 | dbgs() << "No JITDylib for header address.\n" ; |
706 | }); |
707 | |
708 | if (!JD) { |
709 | SendResult(make_error<StringError>(Args: "No JITDylib with header addr " + |
710 | formatv(Fmt: "{0:x}" , Vals&: JDHeaderAddr), |
711 | Args: inconvertibleErrorCode())); |
712 | return; |
713 | } |
714 | |
715 | pushInitializersLoop(SendResult: std::move(SendResult), JD); |
716 | } |
717 | |
718 | void MachOPlatform::rt_pushSymbols( |
719 | PushSymbolsInSendResultFn SendResult, ExecutorAddr Handle, |
720 | const std::vector<std::pair<StringRef, bool>> &SymbolNames) { |
721 | |
722 | JITDylib *JD = nullptr; |
723 | |
724 | { |
725 | std::lock_guard<std::mutex> Lock(PlatformMutex); |
726 | auto I = HeaderAddrToJITDylib.find(Val: Handle); |
727 | if (I != HeaderAddrToJITDylib.end()) |
728 | JD = I->second; |
729 | } |
730 | LLVM_DEBUG({ |
731 | dbgs() << "MachOPlatform::rt_pushSymbols(" ; |
732 | if (JD) |
733 | dbgs() << "\"" << JD->getName() << "\", [ " ; |
734 | else |
735 | dbgs() << "<invalid handle " << Handle << ">, [ " ; |
736 | for (auto &Name : SymbolNames) |
737 | dbgs() << "\"" << Name.first << "\" " ; |
738 | dbgs() << "])\n" ; |
739 | }); |
740 | |
741 | if (!JD) { |
742 | SendResult(make_error<StringError>(Args: "No JITDylib associated with handle " + |
743 | formatv(Fmt: "{0:x}" , Vals&: Handle), |
744 | Args: inconvertibleErrorCode())); |
745 | return; |
746 | } |
747 | |
748 | SymbolLookupSet LS; |
749 | for (auto &[Name, Required] : SymbolNames) |
750 | LS.add(Name: ES.intern(SymName: Name), Flags: Required |
751 | ? SymbolLookupFlags::RequiredSymbol |
752 | : SymbolLookupFlags::WeaklyReferencedSymbol); |
753 | |
754 | ES.lookup( |
755 | K: LookupKind::DLSym, SearchOrder: {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}}, |
756 | Symbols: std::move(LS), RequiredState: SymbolState::Ready, |
757 | NotifyComplete: [SendResult = std::move(SendResult)](Expected<SymbolMap> Result) mutable { |
758 | SendResult(Result.takeError()); |
759 | }, |
760 | RegisterDependencies: NoDependenciesToRegister); |
761 | } |
762 | |
763 | Expected<uint64_t> MachOPlatform::createPThreadKey() { |
764 | if (!CreatePThreadKey.Addr) |
765 | return make_error<StringError>( |
766 | Args: "Attempting to create pthread key in target, but runtime support has " |
767 | "not been loaded yet" , |
768 | Args: inconvertibleErrorCode()); |
769 | |
770 | Expected<uint64_t> Result(0); |
771 | if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>( |
772 | WrapperFnAddr: CreatePThreadKey.Addr, WrapperCallArgs&: Result)) |
773 | return std::move(Err); |
774 | return Result; |
775 | } |
776 | |
777 | void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( |
778 | MaterializationResponsibility &MR, jitlink::LinkGraph &LG, |
779 | jitlink::PassConfiguration &Config) { |
780 | |
781 | using namespace jitlink; |
782 | |
783 | bool InBootstrapPhase = |
784 | &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap; |
785 | |
786 | // If we're in the bootstrap phase then increment the active graphs. |
787 | if (InBootstrapPhase) { |
788 | Config.PrePrunePasses.push_back( |
789 | x: [this](LinkGraph &G) { return bootstrapPipelineStart(G); }); |
790 | Config.PostAllocationPasses.push_back(x: [this](LinkGraph &G) { |
791 | return bootstrapPipelineRecordRuntimeFunctions(G); |
792 | }); |
793 | } |
794 | |
795 | // --- Handle Initializers --- |
796 | if (auto InitSymbol = MR.getInitializerSymbol()) { |
797 | |
798 | // If the initializer symbol is the MachOHeader start symbol then just |
799 | // register it and then bail out -- the header materialization unit |
800 | // definitely doesn't need any other passes. |
801 | if (InitSymbol == MP.MachOHeaderStartSymbol && !InBootstrapPhase) { |
802 | Config.PostAllocationPasses.push_back(x: [this, &MR](LinkGraph &G) { |
803 | return associateJITDylibHeaderSymbol(G, MR); |
804 | }); |
805 | return; |
806 | } |
807 | |
808 | // If the object contains an init symbol other than the header start symbol |
809 | // then add passes to preserve, process and register the init |
810 | // sections/symbols. |
811 | Config.PrePrunePasses.push_back(x: [this, &MR](LinkGraph &G) { |
812 | if (auto Err = preserveImportantSections(G, MR)) |
813 | return Err; |
814 | return processObjCImageInfo(G, MR); |
815 | }); |
816 | Config.PostPrunePasses.push_back( |
817 | x: [this](LinkGraph &G) { return createObjCRuntimeObject(G); }); |
818 | Config.PostAllocationPasses.push_back( |
819 | x: [this, &MR](LinkGraph &G) { return populateObjCRuntimeObject(G, MR); }); |
820 | } |
821 | |
822 | // Insert TLV lowering at the start of the PostPrunePasses, since we want |
823 | // it to run before GOT/PLT lowering. |
824 | Config.PostPrunePasses.insert( |
825 | position: Config.PostPrunePasses.begin(), |
826 | x: [this, &JD = MR.getTargetJITDylib()](LinkGraph &G) { |
827 | return fixTLVSectionsAndEdges(G, JD); |
828 | }); |
829 | |
830 | // Add symbol table prepare and register passes: These will add strings for |
831 | // all symbols to the c-strings section, and build a symbol table registration |
832 | // call. |
833 | auto JITSymTabInfo = std::make_shared<JITSymTabVector>(); |
834 | Config.PostPrunePasses.push_back(x: [this, JITSymTabInfo](LinkGraph &G) { |
835 | return prepareSymbolTableRegistration(G, JITSymTabInfo&: *JITSymTabInfo); |
836 | }); |
837 | Config.PostFixupPasses.push_back(x: [this, &MR, JITSymTabInfo, |
838 | InBootstrapPhase](LinkGraph &G) { |
839 | return addSymbolTableRegistration(G, MR, JITSymTabInfo&: *JITSymTabInfo, InBootstrapPhase); |
840 | }); |
841 | |
842 | // Add a pass to register the final addresses of any special sections in the |
843 | // object with the runtime. |
844 | Config.PostAllocationPasses.push_back( |
845 | x: [this, &JD = MR.getTargetJITDylib(), InBootstrapPhase](LinkGraph &G) { |
846 | return registerObjectPlatformSections(G, JD, InBootstrapPhase); |
847 | }); |
848 | |
849 | // If we're in the bootstrap phase then steal allocation actions and then |
850 | // decrement the active graphs. |
851 | if (InBootstrapPhase) |
852 | Config.PostFixupPasses.push_back( |
853 | x: [this](LinkGraph &G) { return bootstrapPipelineEnd(G); }); |
854 | } |
855 | |
856 | ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap |
857 | MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies( |
858 | MaterializationResponsibility &MR) { |
859 | std::lock_guard<std::mutex> Lock(PluginMutex); |
860 | auto I = InitSymbolDeps.find(Val: &MR); |
861 | if (I != InitSymbolDeps.end()) { |
862 | SyntheticSymbolDependenciesMap Result; |
863 | Result[MR.getInitializerSymbol()] = std::move(I->second); |
864 | InitSymbolDeps.erase(Val: &MR); |
865 | return Result; |
866 | } |
867 | return SyntheticSymbolDependenciesMap(); |
868 | } |
869 | |
870 | Error MachOPlatform::MachOPlatformPlugin::bootstrapPipelineStart( |
871 | jitlink::LinkGraph &G) { |
872 | // Increment the active graphs count in BootstrapInfo. |
873 | std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex); |
874 | ++MP.Bootstrap.load()->ActiveGraphs; |
875 | return Error::success(); |
876 | } |
877 | |
878 | Error MachOPlatform::MachOPlatformPlugin:: |
879 | bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) { |
880 | // Record bootstrap function names. |
881 | std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = { |
882 | {*MP.MachOHeaderStartSymbol, &MP.Bootstrap.load()->MachOHeaderAddr}, |
883 | {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr}, |
884 | {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr}, |
885 | {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr}, |
886 | {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr}, |
887 | {*MP.RegisterObjectSymbolTable.Name, &MP.RegisterObjectSymbolTable.Addr}, |
888 | {*MP.DeregisterObjectSymbolTable.Name, |
889 | &MP.DeregisterObjectSymbolTable.Addr}, |
890 | {*MP.RegisterObjectPlatformSections.Name, |
891 | &MP.RegisterObjectPlatformSections.Addr}, |
892 | {*MP.DeregisterObjectPlatformSections.Name, |
893 | &MP.DeregisterObjectPlatformSections.Addr}, |
894 | {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}, |
895 | {*MP.RegisterObjCRuntimeObject.Name, &MP.RegisterObjCRuntimeObject.Addr}, |
896 | {*MP.DeregisterObjCRuntimeObject.Name, |
897 | &MP.DeregisterObjCRuntimeObject.Addr}}; |
898 | |
899 | bool = false; |
900 | |
901 | for (auto *Sym : G.defined_symbols()) { |
902 | for (auto &RTSym : RuntimeSymbols) { |
903 | if (Sym->hasName() && Sym->getName() == RTSym.first) { |
904 | if (*RTSym.second) |
905 | return make_error<StringError>( |
906 | Args: "Duplicate " + RTSym.first + |
907 | " detected during MachOPlatform bootstrap" , |
908 | Args: inconvertibleErrorCode()); |
909 | |
910 | if (Sym->getName() == *MP.MachOHeaderStartSymbol) |
911 | RegisterMachOHeader = true; |
912 | |
913 | *RTSym.second = Sym->getAddress(); |
914 | } |
915 | } |
916 | } |
917 | |
918 | if (RegisterMachOHeader) { |
919 | // If this graph defines the macho header symbol then create the internal |
920 | // mapping between it and PlatformJD. |
921 | std::lock_guard<std::mutex> Lock(MP.PlatformMutex); |
922 | MP.JITDylibToHeaderAddr[&MP.PlatformJD] = |
923 | MP.Bootstrap.load()->MachOHeaderAddr; |
924 | MP.HeaderAddrToJITDylib[MP.Bootstrap.load()->MachOHeaderAddr] = |
925 | &MP.PlatformJD; |
926 | } |
927 | |
928 | return Error::success(); |
929 | } |
930 | |
931 | Error MachOPlatform::MachOPlatformPlugin::bootstrapPipelineEnd( |
932 | jitlink::LinkGraph &G) { |
933 | std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex); |
934 | assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed" ); |
935 | --MP.Bootstrap.load()->ActiveGraphs; |
936 | // Notify Bootstrap->CV while holding the mutex because the mutex is |
937 | // also keeping Bootstrap->CV alive. |
938 | if (MP.Bootstrap.load()->ActiveGraphs == 0) |
939 | MP.Bootstrap.load()->CV.notify_all(); |
940 | return Error::success(); |
941 | } |
942 | |
943 | Error MachOPlatform::MachOPlatformPlugin::( |
944 | jitlink::LinkGraph &G, MaterializationResponsibility &MR) { |
945 | auto I = llvm::find_if(Range: G.defined_symbols(), P: [this](jitlink::Symbol *Sym) { |
946 | return Sym->getName() == *MP.MachOHeaderStartSymbol; |
947 | }); |
948 | assert(I != G.defined_symbols().end() && "Missing MachO header start symbol" ); |
949 | |
950 | auto &JD = MR.getTargetJITDylib(); |
951 | std::lock_guard<std::mutex> Lock(MP.PlatformMutex); |
952 | auto = (*I)->getAddress(); |
953 | MP.JITDylibToHeaderAddr[&JD] = HeaderAddr; |
954 | MP.HeaderAddrToJITDylib[HeaderAddr] = &JD; |
955 | // We can unconditionally add these actions to the Graph because this pass |
956 | // isn't used during bootstrap. |
957 | G.allocActions().push_back( |
958 | x: {.Finalize: cantFail( |
959 | ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSString, SPSExecutorAddr>>( |
960 | FnAddr: MP.RegisterJITDylib.Addr, Args: JD.getName(), Args: HeaderAddr)), |
961 | .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>( |
962 | FnAddr: MP.DeregisterJITDylib.Addr, Args: HeaderAddr))}); |
963 | return Error::success(); |
964 | } |
965 | |
966 | Error MachOPlatform::MachOPlatformPlugin::preserveImportantSections( |
967 | jitlink::LinkGraph &G, MaterializationResponsibility &MR) { |
968 | // __objc_imageinfo is "important": we want to preserve it and record its |
969 | // address in the first graph that it appears in, then verify and discard it |
970 | // in all subsequent graphs. In this pass we preserve unconditionally -- we'll |
971 | // manually throw it away in the processObjCImageInfo pass. |
972 | if (auto *ObjCImageInfoSec = |
973 | G.findSectionByName(Name: MachOObjCImageInfoSectionName)) { |
974 | if (ObjCImageInfoSec->blocks_size() != 1) |
975 | return make_error<StringError>( |
976 | Args: "In " + G.getName() + |
977 | "__DATA,__objc_imageinfo contains multiple blocks" , |
978 | Args: inconvertibleErrorCode()); |
979 | G.addAnonymousSymbol(Content&: **ObjCImageInfoSec->blocks().begin(), Offset: 0, Size: 0, IsCallable: false, |
980 | IsLive: true); |
981 | |
982 | for (auto *B : ObjCImageInfoSec->blocks()) |
983 | if (!B->edges_empty()) |
984 | return make_error<StringError>(Args: "In " + G.getName() + ", " + |
985 | MachOObjCImageInfoSectionName + |
986 | " contains references to symbols" , |
987 | Args: inconvertibleErrorCode()); |
988 | } |
989 | |
990 | // Init sections are important: We need to preserve them and so that their |
991 | // addresses can be captured and reported to the ORC runtime in |
992 | // registerObjectPlatformSections. |
993 | JITLinkSymbolSet InitSectionSymbols; |
994 | for (auto &InitSectionName : MachOInitSectionNames) { |
995 | // Skip ObjCImageInfo -- this shouldn't have any dependencies, and we may |
996 | // remove it later. |
997 | if (InitSectionName == MachOObjCImageInfoSectionName) |
998 | continue; |
999 | |
1000 | // Skip non-init sections. |
1001 | auto *InitSection = G.findSectionByName(Name: InitSectionName); |
1002 | if (!InitSection) |
1003 | continue; |
1004 | |
1005 | // Make a pass over live symbols in the section: those blocks are already |
1006 | // preserved. |
1007 | DenseSet<jitlink::Block *> AlreadyLiveBlocks; |
1008 | for (auto &Sym : InitSection->symbols()) { |
1009 | auto &B = Sym->getBlock(); |
1010 | if (Sym->isLive() && Sym->getOffset() == 0 && |
1011 | Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(V: &B)) { |
1012 | InitSectionSymbols.insert(V: Sym); |
1013 | AlreadyLiveBlocks.insert(V: &B); |
1014 | } |
1015 | } |
1016 | |
1017 | // Add anonymous symbols to preserve any not-already-preserved blocks. |
1018 | for (auto *B : InitSection->blocks()) |
1019 | if (!AlreadyLiveBlocks.count(V: B)) |
1020 | InitSectionSymbols.insert( |
1021 | V: &G.addAnonymousSymbol(Content&: *B, Offset: 0, Size: B->getSize(), IsCallable: false, IsLive: true)); |
1022 | } |
1023 | |
1024 | if (!InitSectionSymbols.empty()) { |
1025 | std::lock_guard<std::mutex> Lock(PluginMutex); |
1026 | InitSymbolDeps[&MR] = std::move(InitSectionSymbols); |
1027 | } |
1028 | |
1029 | return Error::success(); |
1030 | } |
1031 | |
1032 | Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( |
1033 | jitlink::LinkGraph &G, MaterializationResponsibility &MR) { |
1034 | |
1035 | // If there's an ObjC imagine info then either |
1036 | // (1) It's the first __objc_imageinfo we've seen in this JITDylib. In |
1037 | // this case we name and record it. |
1038 | // OR |
1039 | // (2) We already have a recorded __objc_imageinfo for this JITDylib, |
1040 | // in which case we just verify it. |
1041 | auto *ObjCImageInfo = G.findSectionByName(Name: MachOObjCImageInfoSectionName); |
1042 | if (!ObjCImageInfo) |
1043 | return Error::success(); |
1044 | |
1045 | auto ObjCImageInfoBlocks = ObjCImageInfo->blocks(); |
1046 | |
1047 | // Check that the section is not empty if present. |
1048 | if (ObjCImageInfoBlocks.empty()) |
1049 | return make_error<StringError>(Args: "Empty " + MachOObjCImageInfoSectionName + |
1050 | " section in " + G.getName(), |
1051 | Args: inconvertibleErrorCode()); |
1052 | |
1053 | // Check that there's only one block in the section. |
1054 | if (std::next(x: ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end()) |
1055 | return make_error<StringError>(Args: "Multiple blocks in " + |
1056 | MachOObjCImageInfoSectionName + |
1057 | " section in " + G.getName(), |
1058 | Args: inconvertibleErrorCode()); |
1059 | |
1060 | // Check that the __objc_imageinfo section is unreferenced. |
1061 | // FIXME: We could optimize this check if Symbols had a ref-count. |
1062 | for (auto &Sec : G.sections()) { |
1063 | if (&Sec != ObjCImageInfo) |
1064 | for (auto *B : Sec.blocks()) |
1065 | for (auto &E : B->edges()) |
1066 | if (E.getTarget().isDefined() && |
1067 | &E.getTarget().getBlock().getSection() == ObjCImageInfo) |
1068 | return make_error<StringError>(Args: MachOObjCImageInfoSectionName + |
1069 | " is referenced within file " + |
1070 | G.getName(), |
1071 | Args: inconvertibleErrorCode()); |
1072 | } |
1073 | |
1074 | auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin(); |
1075 | auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data(); |
1076 | auto Version = support::endian::read32(P: ObjCImageInfoData, E: G.getEndianness()); |
1077 | auto Flags = |
1078 | support::endian::read32(P: ObjCImageInfoData + 4, E: G.getEndianness()); |
1079 | |
1080 | // Lock the mutex while we verify / update the ObjCImageInfos map. |
1081 | std::lock_guard<std::mutex> Lock(PluginMutex); |
1082 | |
1083 | auto ObjCImageInfoItr = ObjCImageInfos.find(Val: &MR.getTargetJITDylib()); |
1084 | if (ObjCImageInfoItr != ObjCImageInfos.end()) { |
1085 | // We've already registered an __objc_imageinfo section. Verify the |
1086 | // content of this new section matches, then delete it. |
1087 | if (ObjCImageInfoItr->second.Version != Version) |
1088 | return make_error<StringError>( |
1089 | Args: "ObjC version in " + G.getName() + |
1090 | " does not match first registered version" , |
1091 | Args: inconvertibleErrorCode()); |
1092 | if (ObjCImageInfoItr->second.Flags != Flags) |
1093 | if (Error E = mergeImageInfoFlags(G, MR, Info&: ObjCImageInfoItr->second, NewFlags: Flags)) |
1094 | return E; |
1095 | |
1096 | // __objc_imageinfo is valid. Delete the block. |
1097 | for (auto *S : ObjCImageInfo->symbols()) |
1098 | G.removeDefinedSymbol(Sym&: *S); |
1099 | G.removeBlock(B&: ObjCImageInfoBlock); |
1100 | } else { |
1101 | LLVM_DEBUG({ |
1102 | dbgs() << "MachOPlatform: Registered __objc_imageinfo for " |
1103 | << MR.getTargetJITDylib().getName() << " in " << G.getName() |
1104 | << "; flags = " << formatv("{0:x4}" , Flags) << "\n" ; |
1105 | }); |
1106 | // We haven't registered an __objc_imageinfo section yet. Register and |
1107 | // move on. The section should already be marked no-dead-strip. |
1108 | G.addDefinedSymbol(Content&: ObjCImageInfoBlock, Offset: 0, Name: ObjCImageInfoSymbolName, |
1109 | Size: ObjCImageInfoBlock.getSize(), L: jitlink::Linkage::Strong, |
1110 | S: jitlink::Scope::Hidden, IsCallable: false, IsLive: true); |
1111 | if (auto Err = MR.defineMaterializing( |
1112 | SymbolFlags: {{MR.getExecutionSession().intern(SymName: ObjCImageInfoSymbolName), |
1113 | JITSymbolFlags()}})) |
1114 | return Err; |
1115 | ObjCImageInfos[&MR.getTargetJITDylib()] = {.Version: Version, .Flags: Flags, .Finalized: false}; |
1116 | } |
1117 | |
1118 | return Error::success(); |
1119 | } |
1120 | |
1121 | Error MachOPlatform::MachOPlatformPlugin::mergeImageInfoFlags( |
1122 | jitlink::LinkGraph &G, MaterializationResponsibility &MR, |
1123 | ObjCImageInfo &Info, uint32_t NewFlags) { |
1124 | if (Info.Flags == NewFlags) |
1125 | return Error::success(); |
1126 | |
1127 | ObjCImageInfoFlags Old(Info.Flags); |
1128 | ObjCImageInfoFlags New(NewFlags); |
1129 | |
1130 | // Check for incompatible flags. |
1131 | if (Old.SwiftABIVersion && New.SwiftABIVersion && |
1132 | Old.SwiftABIVersion != New.SwiftABIVersion) |
1133 | return make_error<StringError>(Args: "Swift ABI version in " + G.getName() + |
1134 | " does not match first registered flags" , |
1135 | Args: inconvertibleErrorCode()); |
1136 | |
1137 | if (Old.HasCategoryClassProperties != New.HasCategoryClassProperties) |
1138 | return make_error<StringError>(Args: "ObjC category class property support in " + |
1139 | G.getName() + |
1140 | " does not match first registered flags" , |
1141 | Args: inconvertibleErrorCode()); |
1142 | if (Old.HasSignedObjCClassROs != New.HasSignedObjCClassROs) |
1143 | return make_error<StringError>(Args: "ObjC class_ro_t pointer signing in " + |
1144 | G.getName() + |
1145 | " does not match first registered flags" , |
1146 | Args: inconvertibleErrorCode()); |
1147 | |
1148 | // If we cannot change the flags, ignore any remaining differences. Adding |
1149 | // Swift or changing its version are unlikely to cause problems in practice. |
1150 | if (Info.Finalized) |
1151 | return Error::success(); |
1152 | |
1153 | // Use the minimum Swift version. |
1154 | if (Old.SwiftVersion && New.SwiftVersion) |
1155 | New.SwiftVersion = std::min(a: Old.SwiftVersion, b: New.SwiftVersion); |
1156 | else if (Old.SwiftVersion) |
1157 | New.SwiftVersion = Old.SwiftVersion; |
1158 | // Add a Swift ABI version if it was pure objc before. |
1159 | if (!New.SwiftABIVersion) |
1160 | New.SwiftABIVersion = Old.SwiftABIVersion; |
1161 | |
1162 | LLVM_DEBUG({ |
1163 | dbgs() << "MachOPlatform: Merging __objc_imageinfo flags for " |
1164 | << MR.getTargetJITDylib().getName() << " (was " |
1165 | << formatv("{0:x4}" , Old.rawFlags()) << ")" |
1166 | << " with " << G.getName() << " (" << formatv("{0:x4}" , NewFlags) |
1167 | << ")" |
1168 | << " -> " << formatv("{0:x4}" , New.rawFlags()) << "\n" ; |
1169 | }); |
1170 | |
1171 | Info.Flags = New.rawFlags(); |
1172 | return Error::success(); |
1173 | } |
1174 | |
1175 | Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( |
1176 | jitlink::LinkGraph &G, JITDylib &JD) { |
1177 | |
1178 | // Rename external references to __tlv_bootstrap to ___orc_rt_tlv_get_addr. |
1179 | for (auto *Sym : G.external_symbols()) |
1180 | if (Sym->getName() == "__tlv_bootstrap" ) { |
1181 | Sym->setName("___orc_rt_macho_tlv_get_addr" ); |
1182 | break; |
1183 | } |
1184 | |
1185 | // Store key in __thread_vars struct fields. |
1186 | if (auto *ThreadDataSec = G.findSectionByName(Name: MachOThreadVarsSectionName)) { |
1187 | std::optional<uint64_t> Key; |
1188 | { |
1189 | std::lock_guard<std::mutex> Lock(MP.PlatformMutex); |
1190 | auto I = MP.JITDylibToPThreadKey.find(Val: &JD); |
1191 | if (I != MP.JITDylibToPThreadKey.end()) |
1192 | Key = I->second; |
1193 | } |
1194 | |
1195 | if (!Key) { |
1196 | if (auto KeyOrErr = MP.createPThreadKey()) |
1197 | Key = *KeyOrErr; |
1198 | else |
1199 | return KeyOrErr.takeError(); |
1200 | } |
1201 | |
1202 | uint64_t PlatformKeyBits = |
1203 | support::endian::byte_swap(value: *Key, endian: G.getEndianness()); |
1204 | |
1205 | for (auto *B : ThreadDataSec->blocks()) { |
1206 | if (B->getSize() != 3 * G.getPointerSize()) |
1207 | return make_error<StringError>(Args: "__thread_vars block at " + |
1208 | formatv(Fmt: "{0:x}" , Vals: B->getAddress()) + |
1209 | " has unexpected size" , |
1210 | Args: inconvertibleErrorCode()); |
1211 | |
1212 | auto NewBlockContent = G.allocateBuffer(Size: B->getSize()); |
1213 | llvm::copy(Range: B->getContent(), Out: NewBlockContent.data()); |
1214 | memcpy(dest: NewBlockContent.data() + G.getPointerSize(), src: &PlatformKeyBits, |
1215 | n: G.getPointerSize()); |
1216 | B->setContent(NewBlockContent); |
1217 | } |
1218 | } |
1219 | |
1220 | // Transform any TLV edges into GOT edges. |
1221 | for (auto *B : G.blocks()) |
1222 | for (auto &E : B->edges()) |
1223 | if (E.getKind() == |
1224 | jitlink::x86_64::RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable) |
1225 | E.setKind(jitlink::x86_64:: |
1226 | RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable); |
1227 | |
1228 | return Error::success(); |
1229 | } |
1230 | |
1231 | std::optional<MachOPlatform::MachOPlatformPlugin::UnwindSections> |
1232 | MachOPlatform::MachOPlatformPlugin::findUnwindSectionInfo( |
1233 | jitlink::LinkGraph &G) { |
1234 | using namespace jitlink; |
1235 | |
1236 | UnwindSections US; |
1237 | |
1238 | // ScanSection records a section range and adds any executable blocks that |
1239 | // that section points to to the CodeBlocks vector. |
1240 | SmallVector<Block *> CodeBlocks; |
1241 | auto ScanUnwindInfoSection = [&](Section &Sec, ExecutorAddrRange &SecRange) { |
1242 | if (Sec.blocks().empty()) |
1243 | return; |
1244 | SecRange = (*Sec.blocks().begin())->getRange(); |
1245 | for (auto *B : Sec.blocks()) { |
1246 | auto R = B->getRange(); |
1247 | SecRange.Start = std::min(a: SecRange.Start, b: R.Start); |
1248 | SecRange.End = std::max(a: SecRange.End, b: R.End); |
1249 | for (auto &E : B->edges()) { |
1250 | if (!E.getTarget().isDefined()) |
1251 | continue; |
1252 | auto &TargetBlock = E.getTarget().getBlock(); |
1253 | auto &TargetSection = TargetBlock.getSection(); |
1254 | if ((TargetSection.getMemProt() & MemProt::Exec) == MemProt::Exec) |
1255 | CodeBlocks.push_back(Elt: &TargetBlock); |
1256 | } |
1257 | } |
1258 | }; |
1259 | |
1260 | if (Section *EHFrameSec = G.findSectionByName(Name: MachOEHFrameSectionName)) |
1261 | ScanUnwindInfoSection(*EHFrameSec, US.DwarfSection); |
1262 | |
1263 | if (Section *CUInfoSec = |
1264 | G.findSectionByName(Name: MachOCompactUnwindInfoSectionName)) |
1265 | ScanUnwindInfoSection(*CUInfoSec, US.CompactUnwindSection); |
1266 | |
1267 | // If we didn't find any pointed-to code-blocks then there's no need to |
1268 | // register any info. |
1269 | if (CodeBlocks.empty()) |
1270 | return std::nullopt; |
1271 | |
1272 | // We have info to register. Sort the code blocks into address order and |
1273 | // build a list of contiguous address ranges covering them all. |
1274 | llvm::sort(C&: CodeBlocks, Comp: [](const Block *LHS, const Block *RHS) { |
1275 | return LHS->getAddress() < RHS->getAddress(); |
1276 | }); |
1277 | for (auto *B : CodeBlocks) { |
1278 | if (US.CodeRanges.empty() || US.CodeRanges.back().End != B->getAddress()) |
1279 | US.CodeRanges.push_back(Elt: B->getRange()); |
1280 | else |
1281 | US.CodeRanges.back().End = B->getRange().End; |
1282 | } |
1283 | |
1284 | LLVM_DEBUG({ |
1285 | dbgs() << "MachOPlatform identified unwind info in " << G.getName() << ":\n" |
1286 | << " DWARF: " ; |
1287 | if (US.DwarfSection.Start) |
1288 | dbgs() << US.DwarfSection << "\n" ; |
1289 | else |
1290 | dbgs() << "none\n" ; |
1291 | dbgs() << " Compact-unwind: " ; |
1292 | if (US.CompactUnwindSection.Start) |
1293 | dbgs() << US.CompactUnwindSection << "\n" ; |
1294 | else |
1295 | dbgs() << "none\n" |
1296 | << "for code ranges:\n" ; |
1297 | for (auto &CR : US.CodeRanges) |
1298 | dbgs() << " " << CR << "\n" ; |
1299 | if (US.CodeRanges.size() >= G.sections_size()) |
1300 | dbgs() << "WARNING: High number of discontiguous code ranges! " |
1301 | "Padding may be interfering with coalescing.\n" ; |
1302 | }); |
1303 | |
1304 | return US; |
1305 | } |
1306 | |
1307 | Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( |
1308 | jitlink::LinkGraph &G, JITDylib &JD, bool InBootstrapPhase) { |
1309 | |
1310 | // Get a pointer to the thread data section if there is one. It will be used |
1311 | // below. |
1312 | jitlink::Section *ThreadDataSection = |
1313 | G.findSectionByName(Name: MachOThreadDataSectionName); |
1314 | |
1315 | // Handle thread BSS section if there is one. |
1316 | if (auto *ThreadBSSSection = G.findSectionByName(Name: MachOThreadBSSSectionName)) { |
1317 | // If there's already a thread data section in this graph then merge the |
1318 | // thread BSS section content into it, otherwise just treat the thread |
1319 | // BSS section as the thread data section. |
1320 | if (ThreadDataSection) |
1321 | G.mergeSections(DstSection&: *ThreadDataSection, SrcSection&: *ThreadBSSSection); |
1322 | else |
1323 | ThreadDataSection = ThreadBSSSection; |
1324 | } |
1325 | |
1326 | SmallVector<std::pair<StringRef, ExecutorAddrRange>, 8> MachOPlatformSecs; |
1327 | |
1328 | // Collect data sections to register. |
1329 | StringRef DataSections[] = {MachODataDataSectionName, |
1330 | MachODataCommonSectionName, |
1331 | MachOEHFrameSectionName}; |
1332 | for (auto &SecName : DataSections) { |
1333 | if (auto *Sec = G.findSectionByName(Name: SecName)) { |
1334 | jitlink::SectionRange R(*Sec); |
1335 | if (!R.empty()) |
1336 | MachOPlatformSecs.push_back(Elt: {SecName, R.getRange()}); |
1337 | } |
1338 | } |
1339 | |
1340 | // Having merged thread BSS (if present) and thread data (if present), |
1341 | // record the resulting section range. |
1342 | if (ThreadDataSection) { |
1343 | jitlink::SectionRange R(*ThreadDataSection); |
1344 | if (!R.empty()) |
1345 | MachOPlatformSecs.push_back(Elt: {MachOThreadDataSectionName, R.getRange()}); |
1346 | } |
1347 | |
1348 | // If any platform sections were found then add an allocation action to call |
1349 | // the registration function. |
1350 | StringRef PlatformSections[] = {MachOModInitFuncSectionName, |
1351 | ObjCRuntimeObjectSectionName}; |
1352 | |
1353 | for (auto &SecName : PlatformSections) { |
1354 | auto *Sec = G.findSectionByName(Name: SecName); |
1355 | if (!Sec) |
1356 | continue; |
1357 | jitlink::SectionRange R(*Sec); |
1358 | if (R.empty()) |
1359 | continue; |
1360 | |
1361 | MachOPlatformSecs.push_back(Elt: {SecName, R.getRange()}); |
1362 | } |
1363 | |
1364 | std::optional<std::tuple<SmallVector<ExecutorAddrRange>, ExecutorAddrRange, |
1365 | ExecutorAddrRange>> |
1366 | UnwindInfo; |
1367 | if (auto UI = findUnwindSectionInfo(G)) |
1368 | UnwindInfo = std::make_tuple(args: std::move(UI->CodeRanges), args&: UI->DwarfSection, |
1369 | args&: UI->CompactUnwindSection); |
1370 | |
1371 | if (!MachOPlatformSecs.empty() || UnwindInfo) { |
1372 | // Dump the scraped inits. |
1373 | LLVM_DEBUG({ |
1374 | dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n" ; |
1375 | for (auto &KV : MachOPlatformSecs) |
1376 | dbgs() << " " << KV.first << ": " << KV.second << "\n" ; |
1377 | }); |
1378 | |
1379 | using SPSRegisterObjectPlatformSectionsArgs = SPSArgList< |
1380 | SPSExecutorAddr, |
1381 | SPSOptional<SPSTuple<SPSSequence<SPSExecutorAddrRange>, |
1382 | SPSExecutorAddrRange, SPSExecutorAddrRange>>, |
1383 | SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>>; |
1384 | |
1385 | shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase) |
1386 | ? G.allocActions() |
1387 | : MP.Bootstrap.load()->DeferredAAs; |
1388 | |
1389 | ExecutorAddr ; |
1390 | { |
1391 | std::lock_guard<std::mutex> Lock(MP.PlatformMutex); |
1392 | auto I = MP.JITDylibToHeaderAddr.find(Val: &JD); |
1393 | assert(I != MP.JITDylibToHeaderAddr.end() && |
1394 | "No header registered for JD" ); |
1395 | assert(I->second && "Null header registered for JD" ); |
1396 | HeaderAddr = I->second; |
1397 | } |
1398 | allocActions.push_back( |
1399 | x: {.Finalize: cantFail( |
1400 | ValOrErr: WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>( |
1401 | FnAddr: MP.RegisterObjectPlatformSections.Addr, Args: HeaderAddr, Args: UnwindInfo, |
1402 | Args: MachOPlatformSecs)), |
1403 | .Dealloc: cantFail( |
1404 | ValOrErr: WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>( |
1405 | FnAddr: MP.DeregisterObjectPlatformSections.Addr, Args: HeaderAddr, |
1406 | Args: UnwindInfo, Args: MachOPlatformSecs))}); |
1407 | } |
1408 | |
1409 | return Error::success(); |
1410 | } |
1411 | |
1412 | Error MachOPlatform::MachOPlatformPlugin::createObjCRuntimeObject( |
1413 | jitlink::LinkGraph &G) { |
1414 | |
1415 | bool NeedTextSegment = false; |
1416 | size_t NumRuntimeSections = 0; |
1417 | |
1418 | for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsData) |
1419 | if (G.findSectionByName(Name: ObjCRuntimeSectionName)) |
1420 | ++NumRuntimeSections; |
1421 | |
1422 | for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsText) { |
1423 | if (G.findSectionByName(Name: ObjCRuntimeSectionName)) { |
1424 | ++NumRuntimeSections; |
1425 | NeedTextSegment = true; |
1426 | } |
1427 | } |
1428 | |
1429 | // Early out for no runtime sections. |
1430 | if (NumRuntimeSections == 0) |
1431 | return Error::success(); |
1432 | |
1433 | // If there were any runtime sections then we need to add an __objc_imageinfo |
1434 | // section. |
1435 | ++NumRuntimeSections; |
1436 | |
1437 | size_t MachOSize = sizeof(MachO::mach_header_64) + |
1438 | (NeedTextSegment + 1) * sizeof(MachO::segment_command_64) + |
1439 | NumRuntimeSections * sizeof(MachO::section_64); |
1440 | |
1441 | auto &Sec = G.createSection(Name: ObjCRuntimeObjectSectionName, |
1442 | Prot: MemProt::Read | MemProt::Write); |
1443 | G.createMutableContentBlock(Parent&: Sec, ContentSize: MachOSize, Address: ExecutorAddr(), Alignment: 16, AlignmentOffset: 0, ZeroInitialize: true); |
1444 | |
1445 | return Error::success(); |
1446 | } |
1447 | |
1448 | Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject( |
1449 | jitlink::LinkGraph &G, MaterializationResponsibility &MR) { |
1450 | |
1451 | auto *ObjCRuntimeObjectSec = |
1452 | G.findSectionByName(Name: ObjCRuntimeObjectSectionName); |
1453 | |
1454 | if (!ObjCRuntimeObjectSec) |
1455 | return Error::success(); |
1456 | |
1457 | switch (G.getTargetTriple().getArch()) { |
1458 | case Triple::aarch64: |
1459 | case Triple::x86_64: |
1460 | // Supported. |
1461 | break; |
1462 | default: |
1463 | return make_error<StringError>(Args: "Unrecognized MachO arch in triple " + |
1464 | G.getTargetTriple().str(), |
1465 | Args: inconvertibleErrorCode()); |
1466 | } |
1467 | |
1468 | auto &SecBlock = **ObjCRuntimeObjectSec->blocks().begin(); |
1469 | |
1470 | struct SecDesc { |
1471 | MachO::section_64 Sec; |
1472 | unique_function<void(size_t RecordOffset)> AddFixups; |
1473 | }; |
1474 | |
1475 | std::vector<SecDesc> TextSections, DataSections; |
1476 | auto AddSection = [&](SecDesc &SD, jitlink::Section &GraphSec) { |
1477 | jitlink::SectionRange SR(GraphSec); |
1478 | StringRef FQName = GraphSec.getName(); |
1479 | memset(s: &SD.Sec, c: 0, n: sizeof(MachO::section_64)); |
1480 | memcpy(dest: SD.Sec.sectname, src: FQName.drop_front(N: 7).data(), n: FQName.size() - 7); |
1481 | memcpy(dest: SD.Sec.segname, src: FQName.data(), n: 6); |
1482 | SD.Sec.addr = SR.getStart() - SecBlock.getAddress(); |
1483 | SD.Sec.size = SR.getSize(); |
1484 | SD.Sec.flags = MachO::S_REGULAR; |
1485 | }; |
1486 | |
1487 | // Add the __objc_imageinfo section. |
1488 | { |
1489 | DataSections.push_back(x: {}); |
1490 | auto &SD = DataSections.back(); |
1491 | memset(s: &SD.Sec, c: 0, n: sizeof(SD.Sec)); |
1492 | memcpy(dest: SD.Sec.sectname, src: "__objc_imageinfo" , n: 16); |
1493 | strcpy(dest: SD.Sec.segname, src: "__DATA" ); |
1494 | SD.Sec.size = 8; |
1495 | SD.AddFixups = [&](size_t RecordOffset) { |
1496 | auto PointerEdge = getPointerEdgeKind(G); |
1497 | |
1498 | // Look for an existing __objc_imageinfo symbol. |
1499 | jitlink::Symbol *ObjCImageInfoSym = nullptr; |
1500 | for (auto *Sym : G.external_symbols()) |
1501 | if (Sym->getName() == ObjCImageInfoSymbolName) { |
1502 | ObjCImageInfoSym = Sym; |
1503 | break; |
1504 | } |
1505 | if (!ObjCImageInfoSym) |
1506 | for (auto *Sym : G.absolute_symbols()) |
1507 | if (Sym->getName() == ObjCImageInfoSymbolName) { |
1508 | ObjCImageInfoSym = Sym; |
1509 | break; |
1510 | } |
1511 | if (!ObjCImageInfoSym) |
1512 | for (auto *Sym : G.defined_symbols()) |
1513 | if (Sym->hasName() && Sym->getName() == ObjCImageInfoSymbolName) { |
1514 | ObjCImageInfoSym = Sym; |
1515 | std::optional<uint32_t> Flags; |
1516 | { |
1517 | std::lock_guard<std::mutex> Lock(PluginMutex); |
1518 | auto It = ObjCImageInfos.find(Val: &MR.getTargetJITDylib()); |
1519 | if (It != ObjCImageInfos.end()) { |
1520 | It->second.Finalized = true; |
1521 | Flags = It->second.Flags; |
1522 | } |
1523 | } |
1524 | |
1525 | if (Flags) { |
1526 | // We own the definition of __objc_image_info; write the final |
1527 | // merged flags value. |
1528 | auto Content = Sym->getBlock().getMutableContent(G); |
1529 | assert(Content.size() == 8 && |
1530 | "__objc_image_info size should have been verified already" ); |
1531 | support::endian::write32(P: &Content[4], V: *Flags, E: G.getEndianness()); |
1532 | } |
1533 | break; |
1534 | } |
1535 | if (!ObjCImageInfoSym) |
1536 | ObjCImageInfoSym = |
1537 | &G.addExternalSymbol(Name: ObjCImageInfoSymbolName, Size: 8, IsWeaklyReferenced: false); |
1538 | |
1539 | SecBlock.addEdge(K: PointerEdge, |
1540 | Offset: RecordOffset + ((char *)&SD.Sec.addr - (char *)&SD.Sec), |
1541 | Target&: *ObjCImageInfoSym, Addend: -SecBlock.getAddress().getValue()); |
1542 | }; |
1543 | } |
1544 | |
1545 | for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsData) { |
1546 | if (auto *GraphSec = G.findSectionByName(Name: ObjCRuntimeSectionName)) { |
1547 | DataSections.push_back(x: {}); |
1548 | AddSection(DataSections.back(), *GraphSec); |
1549 | } |
1550 | } |
1551 | |
1552 | for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsText) { |
1553 | if (auto *GraphSec = G.findSectionByName(Name: ObjCRuntimeSectionName)) { |
1554 | TextSections.push_back(x: {}); |
1555 | AddSection(TextSections.back(), *GraphSec); |
1556 | } |
1557 | } |
1558 | |
1559 | assert(ObjCRuntimeObjectSec->blocks_size() == 1 && |
1560 | "Unexpected number of blocks in runtime sections object" ); |
1561 | |
1562 | // Build the header struct up-front. This also gives us a chance to check |
1563 | // that the triple is supported, which we'll assume below. |
1564 | MachO::mach_header_64 Hdr; |
1565 | Hdr.magic = MachO::MH_MAGIC_64; |
1566 | switch (G.getTargetTriple().getArch()) { |
1567 | case Triple::aarch64: |
1568 | Hdr.cputype = MachO::CPU_TYPE_ARM64; |
1569 | Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; |
1570 | break; |
1571 | case Triple::x86_64: |
1572 | Hdr.cputype = MachO::CPU_TYPE_X86_64; |
1573 | Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; |
1574 | break; |
1575 | default: |
1576 | llvm_unreachable("Unsupported architecture" ); |
1577 | } |
1578 | |
1579 | Hdr.filetype = MachO::MH_DYLIB; |
1580 | Hdr.ncmds = 1 + !TextSections.empty(); |
1581 | Hdr.sizeofcmds = |
1582 | Hdr.ncmds * sizeof(MachO::segment_command_64) + |
1583 | (TextSections.size() + DataSections.size()) * sizeof(MachO::section_64); |
1584 | Hdr.flags = 0; |
1585 | Hdr.reserved = 0; |
1586 | |
1587 | auto SecContent = SecBlock.getAlreadyMutableContent(); |
1588 | char *P = SecContent.data(); |
1589 | auto WriteMachOStruct = [&](auto S) { |
1590 | if (G.getEndianness() != llvm::endianness::native) |
1591 | MachO::swapStruct(S); |
1592 | memcpy(P, &S, sizeof(S)); |
1593 | P += sizeof(S); |
1594 | }; |
1595 | |
1596 | auto WriteSegment = [&](StringRef Name, std::vector<SecDesc> &Secs) { |
1597 | MachO::segment_command_64 SegLC; |
1598 | memset(s: &SegLC, c: 0, n: sizeof(SegLC)); |
1599 | memcpy(dest: SegLC.segname, src: Name.data(), n: Name.size()); |
1600 | SegLC.cmd = MachO::LC_SEGMENT_64; |
1601 | SegLC.cmdsize = sizeof(MachO::segment_command_64) + |
1602 | Secs.size() * sizeof(MachO::section_64); |
1603 | SegLC.nsects = Secs.size(); |
1604 | WriteMachOStruct(SegLC); |
1605 | for (auto &SD : Secs) { |
1606 | if (SD.AddFixups) |
1607 | SD.AddFixups(P - SecContent.data()); |
1608 | WriteMachOStruct(SD.Sec); |
1609 | } |
1610 | }; |
1611 | |
1612 | WriteMachOStruct(Hdr); |
1613 | if (!TextSections.empty()) |
1614 | WriteSegment("__TEXT" , TextSections); |
1615 | if (!DataSections.empty()) |
1616 | WriteSegment("__DATA" , DataSections); |
1617 | |
1618 | assert(P == SecContent.end() && "Underflow writing ObjC runtime object" ); |
1619 | return Error::success(); |
1620 | } |
1621 | |
1622 | Error MachOPlatform::MachOPlatformPlugin::prepareSymbolTableRegistration( |
1623 | jitlink::LinkGraph &G, JITSymTabVector &JITSymTabInfo) { |
1624 | |
1625 | auto *CStringSec = G.findSectionByName(Name: MachOCStringSectionName); |
1626 | if (!CStringSec) |
1627 | CStringSec = &G.createSection(Name: MachOCStringSectionName, |
1628 | Prot: MemProt::Read | MemProt::Exec); |
1629 | |
1630 | // Make a map of existing strings so that we can re-use them: |
1631 | DenseMap<StringRef, jitlink::Symbol *> ExistingStrings; |
1632 | for (auto *Sym : CStringSec->symbols()) { |
1633 | |
1634 | // The LinkGraph builder should have created single strings blocks, and all |
1635 | // plugins should have maintained this invariant. |
1636 | auto Content = Sym->getBlock().getContent(); |
1637 | ExistingStrings.insert( |
1638 | KV: std::make_pair(x: StringRef(Content.data(), Content.size()), y&: Sym)); |
1639 | } |
1640 | |
1641 | // Add all symbol names to the string section, and record the symbols for |
1642 | // those names. |
1643 | { |
1644 | SmallVector<jitlink::Symbol *> SymsToProcess; |
1645 | for (auto *Sym : G.defined_symbols()) |
1646 | SymsToProcess.push_back(Elt: Sym); |
1647 | for (auto *Sym : G.absolute_symbols()) |
1648 | SymsToProcess.push_back(Elt: Sym); |
1649 | |
1650 | for (auto *Sym : SymsToProcess) { |
1651 | if (!Sym->hasName()) |
1652 | continue; |
1653 | |
1654 | auto I = ExistingStrings.find(Val: Sym->getName()); |
1655 | if (I == ExistingStrings.end()) { |
1656 | auto &NameBlock = G.createMutableContentBlock( |
1657 | Parent&: *CStringSec, MutableContent: G.allocateCString(Source: Sym->getName()), Address: orc::ExecutorAddr(), |
1658 | Alignment: 1, AlignmentOffset: 0); |
1659 | auto &SymbolNameSym = G.addAnonymousSymbol( |
1660 | Content&: NameBlock, Offset: 0, Size: NameBlock.getSize(), IsCallable: false, IsLive: true); |
1661 | JITSymTabInfo.push_back(Elt: {.OriginalSym: Sym, .NameSym: &SymbolNameSym}); |
1662 | } else |
1663 | JITSymTabInfo.push_back(Elt: {.OriginalSym: Sym, .NameSym: I->second}); |
1664 | } |
1665 | } |
1666 | |
1667 | return Error::success(); |
1668 | } |
1669 | |
1670 | Error MachOPlatform::MachOPlatformPlugin::addSymbolTableRegistration( |
1671 | jitlink::LinkGraph &G, MaterializationResponsibility &MR, |
1672 | JITSymTabVector &JITSymTabInfo, bool InBootstrapPhase) { |
1673 | |
1674 | ExecutorAddr ; |
1675 | { |
1676 | std::lock_guard<std::mutex> Lock(MP.PlatformMutex); |
1677 | auto I = MP.JITDylibToHeaderAddr.find(Val: &MR.getTargetJITDylib()); |
1678 | assert(I != MP.JITDylibToHeaderAddr.end() && "No header registered for JD" ); |
1679 | assert(I->second && "Null header registered for JD" ); |
1680 | HeaderAddr = I->second; |
1681 | } |
1682 | |
1683 | SymbolTableVector LocalSymTab; |
1684 | auto &SymTab = LLVM_LIKELY(!InBootstrapPhase) ? LocalSymTab |
1685 | : MP.Bootstrap.load()->SymTab; |
1686 | for (auto &[OriginalSymbol, NameSym] : JITSymTabInfo) |
1687 | SymTab.push_back(Elt: {NameSym->getAddress(), OriginalSymbol->getAddress(), |
1688 | flagsForSymbol(Sym&: *OriginalSymbol)}); |
1689 | |
1690 | // Bail out if we're in the bootstrap phase -- registration of thees symbols |
1691 | // will be attached to the bootstrap graph. |
1692 | if (LLVM_UNLIKELY(InBootstrapPhase)) |
1693 | return Error::success(); |
1694 | |
1695 | shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase) |
1696 | ? G.allocActions() |
1697 | : MP.Bootstrap.load()->DeferredAAs; |
1698 | allocActions.push_back( |
1699 | x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>( |
1700 | FnAddr: MP.RegisterObjectSymbolTable.Addr, Args: HeaderAddr, Args: SymTab)), |
1701 | .Dealloc: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>( |
1702 | FnAddr: MP.DeregisterObjectSymbolTable.Addr, Args: HeaderAddr, Args: SymTab))}); |
1703 | |
1704 | return Error::success(); |
1705 | } |
1706 | |
1707 | template <typename MachOTraits> |
1708 | jitlink::Block &(MachOPlatform &MOP, |
1709 | const MachOPlatform::HeaderOptions &Opts, |
1710 | JITDylib &JD, jitlink::LinkGraph &G, |
1711 | jitlink::Section &) { |
1712 | auto HdrInfo = |
1713 | getMachOHeaderInfoFromTriple(TT: MOP.getExecutionSession().getTargetTriple()); |
1714 | MachOBuilder<MachOTraits> B(HdrInfo.PageSize); |
1715 | |
1716 | B.Header.filetype = MachO::MH_DYLIB; |
1717 | B.Header.cputype = HdrInfo.CPUType; |
1718 | B.Header.cpusubtype = HdrInfo.CPUSubType; |
1719 | |
1720 | if (Opts.IDDylib) |
1721 | B.template addLoadCommand<MachO::LC_ID_DYLIB>( |
1722 | Opts.IDDylib->Name, Opts.IDDylib->Timestamp, |
1723 | Opts.IDDylib->CurrentVersion, Opts.IDDylib->CompatibilityVersion); |
1724 | else |
1725 | B.template addLoadCommand<MachO::LC_ID_DYLIB>(JD.getName(), 0, 0, 0); |
1726 | |
1727 | for (auto &BV : Opts.BuildVersions) |
1728 | B.template addLoadCommand<MachO::LC_BUILD_VERSION>( |
1729 | BV.Platform, BV.MinOS, BV.SDK, static_cast<uint32_t>(0)); |
1730 | for (auto &D : Opts.LoadDylibs) |
1731 | B.template addLoadCommand<MachO::LC_LOAD_DYLIB>( |
1732 | D.Name, D.Timestamp, D.CurrentVersion, D.CompatibilityVersion); |
1733 | for (auto &P : Opts.RPaths) |
1734 | B.template addLoadCommand<MachO::LC_RPATH>(P); |
1735 | |
1736 | auto = G.allocateBuffer(Size: B.layout()); |
1737 | B.write(HeaderContent); |
1738 | |
1739 | return G.createContentBlock(Parent&: HeaderSection, Content: HeaderContent, Address: ExecutorAddr(), Alignment: 8, |
1740 | AlignmentOffset: 0); |
1741 | } |
1742 | |
1743 | SimpleMachOHeaderMU::(MachOPlatform &MOP, |
1744 | SymbolStringPtr , |
1745 | MachOPlatform::HeaderOptions Opts) |
1746 | : MaterializationUnit( |
1747 | createHeaderInterface(MOP, HeaderStartSymbol: std::move(HeaderStartSymbol))), |
1748 | MOP(MOP), Opts(std::move(Opts)) {} |
1749 | |
1750 | void SimpleMachOHeaderMU::( |
1751 | std::unique_ptr<MaterializationResponsibility> R) { |
1752 | auto G = createPlatformGraph(MOP, Name: "<MachOHeaderMU>" ); |
1753 | addMachOHeader(JD&: R->getTargetJITDylib(), G&: *G, InitializerSymbol: R->getInitializerSymbol()); |
1754 | MOP.getObjectLinkingLayer().emit(R: std::move(R), G: std::move(G)); |
1755 | } |
1756 | |
1757 | void SimpleMachOHeaderMU::(const JITDylib &JD, |
1758 | const SymbolStringPtr &Sym) {} |
1759 | |
1760 | void SimpleMachOHeaderMU::( |
1761 | JITDylib &JD, jitlink::LinkGraph &G, |
1762 | const SymbolStringPtr &InitializerSymbol) { |
1763 | auto & = G.createSection(Name: "__header" , Prot: MemProt::Read); |
1764 | auto & = createHeaderBlock(JD, G, HeaderSection); |
1765 | |
1766 | // Init symbol is header-start symbol. |
1767 | G.addDefinedSymbol(Content&: HeaderBlock, Offset: 0, Name: *InitializerSymbol, Size: HeaderBlock.getSize(), |
1768 | L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, |
1769 | IsLive: true); |
1770 | for (auto &HS : AdditionalHeaderSymbols) |
1771 | G.addDefinedSymbol(Content&: HeaderBlock, Offset: HS.Offset, Name: HS.Name, Size: HeaderBlock.getSize(), |
1772 | L: jitlink::Linkage::Strong, S: jitlink::Scope::Default, IsCallable: false, |
1773 | IsLive: true); |
1774 | } |
1775 | |
1776 | jitlink::Block & |
1777 | SimpleMachOHeaderMU::(JITDylib &JD, jitlink::LinkGraph &G, |
1778 | jitlink::Section &) { |
1779 | switch (MOP.getExecutionSession().getTargetTriple().getArch()) { |
1780 | case Triple::aarch64: |
1781 | case Triple::x86_64: |
1782 | return ::createHeaderBlock<MachO64LE>(MOP, Opts, JD, G, HeaderSection); |
1783 | default: |
1784 | llvm_unreachable("Unsupported architecture" ); |
1785 | } |
1786 | } |
1787 | |
1788 | MaterializationUnit::Interface SimpleMachOHeaderMU::( |
1789 | MachOPlatform &MOP, const SymbolStringPtr &) { |
1790 | SymbolFlagsMap ; |
1791 | |
1792 | HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; |
1793 | for (auto &HS : AdditionalHeaderSymbols) |
1794 | HeaderSymbolFlags[MOP.getExecutionSession().intern(SymName: HS.Name)] = |
1795 | JITSymbolFlags::Exported; |
1796 | |
1797 | return MaterializationUnit::Interface(std::move(HeaderSymbolFlags), |
1798 | HeaderStartSymbol); |
1799 | } |
1800 | |
1801 | MachOHeaderInfo (const Triple &TT) { |
1802 | switch (TT.getArch()) { |
1803 | case Triple::aarch64: |
1804 | return {/* PageSize = */ 16 * 1024, |
1805 | /* CPUType = */ MachO::CPU_TYPE_ARM64, |
1806 | /* CPUSubType = */ MachO::CPU_SUBTYPE_ARM64_ALL}; |
1807 | case Triple::x86_64: |
1808 | return {/* PageSize = */ 4 * 1024, |
1809 | /* CPUType = */ MachO::CPU_TYPE_X86_64, |
1810 | /* CPUSubType = */ MachO::CPU_SUBTYPE_X86_64_ALL}; |
1811 | default: |
1812 | llvm_unreachable("Unrecognized architecture" ); |
1813 | } |
1814 | } |
1815 | |
1816 | } // End namespace orc. |
1817 | } // End namespace llvm. |
1818 | |