1 | //===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" |
10 | #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" |
11 | #include "llvm/ExecutionEngine/JITLink/aarch32.h" |
12 | #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" |
13 | #include "llvm/ExecutionEngine/Orc/DebugUtils.h" |
14 | #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" |
15 | #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" |
16 | #include "llvm/Support/MemoryBuffer.h" |
17 | #include <string> |
18 | #include <vector> |
19 | |
20 | #define DEBUG_TYPE "orc" |
21 | |
22 | using namespace llvm; |
23 | using namespace llvm::jitlink; |
24 | using namespace llvm::orc; |
25 | |
26 | namespace { |
27 | |
28 | bool hasInitializerSection(jitlink::LinkGraph &G) { |
29 | bool IsMachO = G.getTargetTriple().isOSBinFormatMachO(); |
30 | bool IsElf = G.getTargetTriple().isOSBinFormatELF(); |
31 | if (!IsMachO && !IsElf) |
32 | return false; |
33 | |
34 | for (auto &Sec : G.sections()) { |
35 | if (IsMachO && isMachOInitializerSection(QualifiedName: Sec.getName())) |
36 | return true; |
37 | if (IsElf && isELFInitializerSection(SecName: Sec.getName())) |
38 | return true; |
39 | } |
40 | |
41 | return false; |
42 | } |
43 | |
44 | ExecutorAddr getJITSymbolPtrForSymbol(Symbol &Sym, const Triple &TT) { |
45 | switch (TT.getArch()) { |
46 | case Triple::arm: |
47 | case Triple::armeb: |
48 | case Triple::thumb: |
49 | case Triple::thumbeb: |
50 | if (hasTargetFlags(Sym, Flags: aarch32::ThumbSymbol)) { |
51 | // Set LSB to indicate thumb target |
52 | assert(Sym.isCallable() && "Only callable symbols can have thumb flag" ); |
53 | assert((Sym.getAddress().getValue() & 0x01) == 0 && "LSB is clear" ); |
54 | return Sym.getAddress() + 0x01; |
55 | } |
56 | return Sym.getAddress(); |
57 | default: |
58 | return Sym.getAddress(); |
59 | } |
60 | } |
61 | |
62 | JITSymbolFlags getJITSymbolFlagsForSymbol(Symbol &Sym) { |
63 | JITSymbolFlags Flags; |
64 | |
65 | if (Sym.getLinkage() == Linkage::Weak) |
66 | Flags |= JITSymbolFlags::Weak; |
67 | |
68 | if (Sym.getScope() == Scope::Default) |
69 | Flags |= JITSymbolFlags::Exported; |
70 | |
71 | if (Sym.isCallable()) |
72 | Flags |= JITSymbolFlags::Callable; |
73 | |
74 | return Flags; |
75 | } |
76 | |
77 | class LinkGraphMaterializationUnit : public MaterializationUnit { |
78 | public: |
79 | static std::unique_ptr<LinkGraphMaterializationUnit> |
80 | Create(ObjectLinkingLayer &ObjLinkingLayer, std::unique_ptr<LinkGraph> G) { |
81 | auto LGI = scanLinkGraph(ES&: ObjLinkingLayer.getExecutionSession(), G&: *G); |
82 | return std::unique_ptr<LinkGraphMaterializationUnit>( |
83 | new LinkGraphMaterializationUnit(ObjLinkingLayer, std::move(G), |
84 | std::move(LGI))); |
85 | } |
86 | |
87 | StringRef getName() const override { return G->getName(); } |
88 | void materialize(std::unique_ptr<MaterializationResponsibility> MR) override { |
89 | ObjLinkingLayer.emit(R: std::move(MR), G: std::move(G)); |
90 | } |
91 | |
92 | private: |
93 | static Interface scanLinkGraph(ExecutionSession &ES, LinkGraph &G) { |
94 | |
95 | Interface LGI; |
96 | |
97 | auto AddSymbol = [&](Symbol *Sym) { |
98 | // Skip local symbols. |
99 | if (Sym->getScope() == Scope::Local) |
100 | return; |
101 | assert(Sym->hasName() && "Anonymous non-local symbol?" ); |
102 | |
103 | LGI.SymbolFlags[ES.intern(SymName: Sym->getName())] = |
104 | getJITSymbolFlagsForSymbol(Sym&: *Sym); |
105 | }; |
106 | |
107 | for (auto *Sym : G.defined_symbols()) |
108 | AddSymbol(Sym); |
109 | for (auto *Sym : G.absolute_symbols()) |
110 | AddSymbol(Sym); |
111 | |
112 | if (hasInitializerSection(G)) |
113 | LGI.InitSymbol = makeInitSymbol(ES, G); |
114 | |
115 | return LGI; |
116 | } |
117 | |
118 | static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) { |
119 | std::string InitSymString; |
120 | raw_string_ostream(InitSymString) |
121 | << "$." << G.getName() << ".__inits" << Counter++; |
122 | return ES.intern(SymName: InitSymString); |
123 | } |
124 | |
125 | LinkGraphMaterializationUnit(ObjectLinkingLayer &ObjLinkingLayer, |
126 | std::unique_ptr<LinkGraph> G, Interface LGI) |
127 | : MaterializationUnit(std::move(LGI)), ObjLinkingLayer(ObjLinkingLayer), |
128 | G(std::move(G)) {} |
129 | |
130 | void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { |
131 | for (auto *Sym : G->defined_symbols()) |
132 | if (Sym->getName() == *Name) { |
133 | assert(Sym->getLinkage() == Linkage::Weak && |
134 | "Discarding non-weak definition" ); |
135 | G->makeExternal(Sym&: *Sym); |
136 | break; |
137 | } |
138 | } |
139 | |
140 | ObjectLinkingLayer &ObjLinkingLayer; |
141 | std::unique_ptr<LinkGraph> G; |
142 | static std::atomic<uint64_t> Counter; |
143 | }; |
144 | |
145 | std::atomic<uint64_t> LinkGraphMaterializationUnit::Counter{0}; |
146 | |
147 | } // end anonymous namespace |
148 | |
149 | namespace llvm { |
150 | namespace orc { |
151 | |
152 | class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { |
153 | public: |
154 | ObjectLinkingLayerJITLinkContext( |
155 | ObjectLinkingLayer &Layer, |
156 | std::unique_ptr<MaterializationResponsibility> MR, |
157 | std::unique_ptr<MemoryBuffer> ObjBuffer) |
158 | : JITLinkContext(&MR->getTargetJITDylib()), Layer(Layer), |
159 | MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} |
160 | |
161 | ~ObjectLinkingLayerJITLinkContext() { |
162 | // If there is an object buffer return function then use it to |
163 | // return ownership of the buffer. |
164 | if (Layer.ReturnObjectBuffer && ObjBuffer) |
165 | Layer.ReturnObjectBuffer(std::move(ObjBuffer)); |
166 | } |
167 | |
168 | JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } |
169 | |
170 | void notifyMaterializing(LinkGraph &G) { |
171 | for (auto &P : Layer.Plugins) |
172 | P->notifyMaterializing(MR&: *MR, G, Ctx&: *this, |
173 | InputObject: ObjBuffer ? ObjBuffer->getMemBufferRef() |
174 | : MemoryBufferRef()); |
175 | } |
176 | |
177 | void notifyFailed(Error Err) override { |
178 | for (auto &P : Layer.Plugins) |
179 | Err = joinErrors(E1: std::move(Err), E2: P->notifyFailed(MR&: *MR)); |
180 | Layer.getExecutionSession().reportError(Err: std::move(Err)); |
181 | MR->failMaterialization(); |
182 | } |
183 | |
184 | void lookup(const LookupMap &Symbols, |
185 | std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override { |
186 | |
187 | JITDylibSearchOrder LinkOrder; |
188 | MR->getTargetJITDylib().withLinkOrderDo( |
189 | F: [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; }); |
190 | |
191 | auto &ES = Layer.getExecutionSession(); |
192 | |
193 | SymbolLookupSet LookupSet; |
194 | for (auto &KV : Symbols) { |
195 | orc::SymbolLookupFlags LookupFlags; |
196 | switch (KV.second) { |
197 | case jitlink::SymbolLookupFlags::RequiredSymbol: |
198 | LookupFlags = orc::SymbolLookupFlags::RequiredSymbol; |
199 | break; |
200 | case jitlink::SymbolLookupFlags::WeaklyReferencedSymbol: |
201 | LookupFlags = orc::SymbolLookupFlags::WeaklyReferencedSymbol; |
202 | break; |
203 | } |
204 | LookupSet.add(Name: ES.intern(SymName: KV.first), Flags: LookupFlags); |
205 | } |
206 | |
207 | // OnResolve -- De-intern the symbols and pass the result to the linker. |
208 | auto OnResolve = [LookupContinuation = |
209 | std::move(LC)](Expected<SymbolMap> Result) mutable { |
210 | if (!Result) |
211 | LookupContinuation->run(LR: Result.takeError()); |
212 | else { |
213 | AsyncLookupResult LR; |
214 | for (auto &KV : *Result) |
215 | LR[*KV.first] = KV.second; |
216 | LookupContinuation->run(LR: std::move(LR)); |
217 | } |
218 | }; |
219 | |
220 | ES.lookup(K: LookupKind::Static, SearchOrder: LinkOrder, Symbols: std::move(LookupSet), |
221 | RequiredState: SymbolState::Resolved, NotifyComplete: std::move(OnResolve), |
222 | RegisterDependencies: [this](const SymbolDependenceMap &Deps) { |
223 | // Translate LookupDeps map to SymbolSourceJD. |
224 | for (auto &[DepJD, Deps] : Deps) |
225 | for (auto &DepSym : Deps) |
226 | SymbolSourceJDs[NonOwningSymbolStringPtr(DepSym)] = DepJD; |
227 | }); |
228 | } |
229 | |
230 | Error notifyResolved(LinkGraph &G) override { |
231 | auto &ES = Layer.getExecutionSession(); |
232 | |
233 | SymbolFlagsMap ; |
234 | bool AutoClaim = Layer.AutoClaimObjectSymbols; |
235 | |
236 | SymbolMap InternedResult; |
237 | for (auto *Sym : G.defined_symbols()) |
238 | if (Sym->hasName() && Sym->getScope() != Scope::Local) { |
239 | auto InternedName = ES.intern(SymName: Sym->getName()); |
240 | auto Ptr = getJITSymbolPtrForSymbol(Sym&: *Sym, TT: G.getTargetTriple()); |
241 | auto Flags = getJITSymbolFlagsForSymbol(Sym&: *Sym); |
242 | InternedResult[InternedName] = {Ptr, Flags}; |
243 | if (AutoClaim && !MR->getSymbols().count(Val: InternedName)) { |
244 | assert(!ExtraSymbolsToClaim.count(InternedName) && |
245 | "Duplicate symbol to claim?" ); |
246 | ExtraSymbolsToClaim[InternedName] = Flags; |
247 | } |
248 | } |
249 | |
250 | for (auto *Sym : G.absolute_symbols()) |
251 | if (Sym->hasName() && Sym->getScope() != Scope::Local) { |
252 | auto InternedName = ES.intern(SymName: Sym->getName()); |
253 | auto Ptr = getJITSymbolPtrForSymbol(Sym&: *Sym, TT: G.getTargetTriple()); |
254 | auto Flags = getJITSymbolFlagsForSymbol(Sym&: *Sym); |
255 | InternedResult[InternedName] = {Ptr, Flags}; |
256 | if (AutoClaim && !MR->getSymbols().count(Val: InternedName)) { |
257 | assert(!ExtraSymbolsToClaim.count(InternedName) && |
258 | "Duplicate symbol to claim?" ); |
259 | ExtraSymbolsToClaim[InternedName] = Flags; |
260 | } |
261 | } |
262 | |
263 | if (!ExtraSymbolsToClaim.empty()) |
264 | if (auto Err = MR->defineMaterializing(SymbolFlags: ExtraSymbolsToClaim)) |
265 | return Err; |
266 | |
267 | { |
268 | |
269 | // Check that InternedResult matches up with MR->getSymbols(), overriding |
270 | // flags if requested. |
271 | // This guards against faulty transformations / compilers / object caches. |
272 | |
273 | // First check that there aren't any missing symbols. |
274 | size_t NumMaterializationSideEffectsOnlySymbols = 0; |
275 | SymbolNameVector ; |
276 | SymbolNameVector MissingSymbols; |
277 | for (auto &KV : MR->getSymbols()) { |
278 | |
279 | auto I = InternedResult.find(Val: KV.first); |
280 | |
281 | // If this is a materialization-side-effects only symbol then bump |
282 | // the counter and make sure it's *not* defined, otherwise make |
283 | // sure that it is defined. |
284 | if (KV.second.hasMaterializationSideEffectsOnly()) { |
285 | ++NumMaterializationSideEffectsOnlySymbols; |
286 | if (I != InternedResult.end()) |
287 | ExtraSymbols.push_back(x: KV.first); |
288 | continue; |
289 | } else if (I == InternedResult.end()) |
290 | MissingSymbols.push_back(x: KV.first); |
291 | else if (Layer.OverrideObjectFlags) |
292 | I->second.setFlags(KV.second); |
293 | } |
294 | |
295 | // If there were missing symbols then report the error. |
296 | if (!MissingSymbols.empty()) |
297 | return make_error<MissingSymbolDefinitions>( |
298 | Args: Layer.getExecutionSession().getSymbolStringPool(), Args: G.getName(), |
299 | Args: std::move(MissingSymbols)); |
300 | |
301 | // If there are more definitions than expected, add them to the |
302 | // ExtraSymbols vector. |
303 | if (InternedResult.size() > |
304 | MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) { |
305 | for (auto &KV : InternedResult) |
306 | if (!MR->getSymbols().count(Val: KV.first)) |
307 | ExtraSymbols.push_back(x: KV.first); |
308 | } |
309 | |
310 | // If there were extra definitions then report the error. |
311 | if (!ExtraSymbols.empty()) |
312 | return make_error<UnexpectedSymbolDefinitions>( |
313 | Args: Layer.getExecutionSession().getSymbolStringPool(), Args: G.getName(), |
314 | Args: std::move(ExtraSymbols)); |
315 | } |
316 | |
317 | if (auto Err = MR->notifyResolved(Symbols: InternedResult)) |
318 | return Err; |
319 | |
320 | Layer.notifyLoaded(MR&: *MR); |
321 | return Error::success(); |
322 | } |
323 | |
324 | void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A) override { |
325 | if (auto Err = Layer.notifyEmitted(MR&: *MR, FA: std::move(A))) { |
326 | Layer.getExecutionSession().reportError(Err: std::move(Err)); |
327 | MR->failMaterialization(); |
328 | return; |
329 | } |
330 | if (auto Err = MR->notifyEmitted(EmittedDeps: SymbolDepGroups)) { |
331 | Layer.getExecutionSession().reportError(Err: std::move(Err)); |
332 | MR->failMaterialization(); |
333 | } |
334 | } |
335 | |
336 | LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override { |
337 | return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); }; |
338 | } |
339 | |
340 | Error modifyPassConfig(LinkGraph &LG, PassConfiguration &Config) override { |
341 | // Add passes to mark duplicate defs as should-discard, and to walk the |
342 | // link graph to build the symbol dependence graph. |
343 | Config.PrePrunePasses.push_back(x: [this](LinkGraph &G) { |
344 | return claimOrExternalizeWeakAndCommonSymbols(G); |
345 | }); |
346 | |
347 | Layer.modifyPassConfig(MR&: *MR, G&: LG, PassConfig&: Config); |
348 | |
349 | Config.PreFixupPasses.push_back( |
350 | x: [this](LinkGraph &G) { return registerDependencies(G); }); |
351 | |
352 | return Error::success(); |
353 | } |
354 | |
355 | private: |
356 | // Symbol name dependencies: |
357 | // Internal: Defined in this graph. |
358 | // External: Defined externally. |
359 | struct BlockSymbolDependencies { |
360 | SymbolNameSet Internal, External; |
361 | }; |
362 | |
363 | // Lazily populated map of blocks to BlockSymbolDependencies values. |
364 | class BlockDependenciesMap { |
365 | public: |
366 | BlockDependenciesMap(ExecutionSession &ES, |
367 | DenseMap<const Block *, DenseSet<Block *>> BlockDeps) |
368 | : ES(ES), BlockDeps(std::move(BlockDeps)) {} |
369 | |
370 | const BlockSymbolDependencies &operator[](const Block &B) { |
371 | // Check the cache first. |
372 | auto I = BlockTransitiveDepsCache.find(Val: &B); |
373 | if (I != BlockTransitiveDepsCache.end()) |
374 | return I->second; |
375 | |
376 | // No value. Populate the cache. |
377 | BlockSymbolDependencies BTDCacheVal; |
378 | auto BDI = BlockDeps.find(Val: &B); |
379 | assert(BDI != BlockDeps.end() && "No block dependencies" ); |
380 | |
381 | for (auto *BDep : BDI->second) { |
382 | auto &BID = getBlockImmediateDeps(B&: *BDep); |
383 | for (auto &ExternalDep : BID.External) |
384 | BTDCacheVal.External.insert(V: ExternalDep); |
385 | for (auto &InternalDep : BID.Internal) |
386 | BTDCacheVal.Internal.insert(V: InternalDep); |
387 | } |
388 | |
389 | return BlockTransitiveDepsCache |
390 | .insert(KV: std::make_pair(x: &B, y: std::move(BTDCacheVal))) |
391 | .first->second; |
392 | } |
393 | |
394 | SymbolStringPtr &getInternedName(Symbol &Sym) { |
395 | auto I = NameCache.find(Val: &Sym); |
396 | if (I != NameCache.end()) |
397 | return I->second; |
398 | |
399 | return NameCache.insert(KV: std::make_pair(x: &Sym, y: ES.intern(SymName: Sym.getName()))) |
400 | .first->second; |
401 | } |
402 | |
403 | private: |
404 | BlockSymbolDependencies &getBlockImmediateDeps(Block &B) { |
405 | // Check the cache first. |
406 | auto I = BlockImmediateDepsCache.find(Val: &B); |
407 | if (I != BlockImmediateDepsCache.end()) |
408 | return I->second; |
409 | |
410 | BlockSymbolDependencies BIDCacheVal; |
411 | for (auto &E : B.edges()) { |
412 | auto &Tgt = E.getTarget(); |
413 | if (Tgt.getScope() != Scope::Local) { |
414 | if (Tgt.isExternal()) { |
415 | if (Tgt.getAddress() || !Tgt.isWeaklyReferenced()) |
416 | BIDCacheVal.External.insert(V: getInternedName(Sym&: Tgt)); |
417 | } else |
418 | BIDCacheVal.Internal.insert(V: getInternedName(Sym&: Tgt)); |
419 | } |
420 | } |
421 | |
422 | return BlockImmediateDepsCache |
423 | .insert(KV: std::make_pair(x: &B, y: std::move(BIDCacheVal))) |
424 | .first->second; |
425 | } |
426 | |
427 | ExecutionSession &ES; |
428 | DenseMap<const Block *, DenseSet<Block *>> BlockDeps; |
429 | DenseMap<const Symbol *, SymbolStringPtr> NameCache; |
430 | DenseMap<const Block *, BlockSymbolDependencies> BlockImmediateDepsCache; |
431 | DenseMap<const Block *, BlockSymbolDependencies> BlockTransitiveDepsCache; |
432 | }; |
433 | |
434 | Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) { |
435 | auto &ES = Layer.getExecutionSession(); |
436 | |
437 | SymbolFlagsMap NewSymbolsToClaim; |
438 | std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym; |
439 | |
440 | auto ProcessSymbol = [&](Symbol *Sym) { |
441 | if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak && |
442 | Sym->getScope() != Scope::Local) { |
443 | auto Name = ES.intern(SymName: Sym->getName()); |
444 | if (!MR->getSymbols().count(Val: ES.intern(SymName: Sym->getName()))) { |
445 | NewSymbolsToClaim[Name] = |
446 | getJITSymbolFlagsForSymbol(Sym&: *Sym) | JITSymbolFlags::Weak; |
447 | NameToSym.push_back(x: std::make_pair(x: std::move(Name), y&: Sym)); |
448 | } |
449 | } |
450 | }; |
451 | |
452 | for (auto *Sym : G.defined_symbols()) |
453 | ProcessSymbol(Sym); |
454 | for (auto *Sym : G.absolute_symbols()) |
455 | ProcessSymbol(Sym); |
456 | |
457 | // Attempt to claim all weak defs that we're not already responsible for. |
458 | // This may fail if the resource tracker has become defunct, but should |
459 | // always succeed otherwise. |
460 | if (auto Err = MR->defineMaterializing(SymbolFlags: std::move(NewSymbolsToClaim))) |
461 | return Err; |
462 | |
463 | // Walk the list of symbols that we just tried to claim. Symbols that we're |
464 | // responsible for are marked live. Symbols that we're not responsible for |
465 | // are turned into external references. |
466 | for (auto &KV : NameToSym) { |
467 | if (MR->getSymbols().count(Val: KV.first)) |
468 | KV.second->setLive(true); |
469 | else |
470 | G.makeExternal(Sym&: *KV.second); |
471 | } |
472 | |
473 | return Error::success(); |
474 | } |
475 | |
476 | Error markResponsibilitySymbolsLive(LinkGraph &G) const { |
477 | auto &ES = Layer.getExecutionSession(); |
478 | for (auto *Sym : G.defined_symbols()) |
479 | if (Sym->hasName() && MR->getSymbols().count(Val: ES.intern(SymName: Sym->getName()))) |
480 | Sym->setLive(true); |
481 | return Error::success(); |
482 | } |
483 | |
484 | Error registerDependencies(LinkGraph &G) { |
485 | auto &TargetJD = MR->getTargetJITDylib(); |
486 | auto &ES = TargetJD.getExecutionSession(); |
487 | auto BlockDeps = computeBlockNonLocalDeps(G); |
488 | |
489 | DenseSet<Block *> BlockDepsProcessed; |
490 | DenseMap<Block *, SymbolDependenceGroup> DepGroupForBlock; |
491 | |
492 | // Compute dependencies for symbols defined in the JITLink graph. |
493 | for (auto *Sym : G.defined_symbols()) { |
494 | |
495 | // Skip local symbols. |
496 | if (Sym->getScope() == Scope::Local) |
497 | continue; |
498 | assert(Sym->hasName() && |
499 | "Defined non-local jitlink::Symbol should have a name" ); |
500 | |
501 | auto &BDeps = BlockDeps[Sym->getBlock()]; |
502 | |
503 | // Skip symbols in blocks that don't depend on anything. |
504 | if (BDeps.Internal.empty() && BDeps.External.empty()) |
505 | continue; |
506 | |
507 | SymbolDependenceGroup &SDG = DepGroupForBlock[&Sym->getBlock()]; |
508 | SDG.Symbols.insert(V: ES.intern(SymName: Sym->getName())); |
509 | |
510 | if (!BlockDepsProcessed.count(V: &Sym->getBlock())) { |
511 | BlockDepsProcessed.insert(V: &Sym->getBlock()); |
512 | |
513 | if (!BDeps.Internal.empty()) |
514 | SDG.Dependencies[&TargetJD] = BDeps.Internal; |
515 | for (auto &Dep : BDeps.External) { |
516 | auto DepSrcItr = SymbolSourceJDs.find(Val: NonOwningSymbolStringPtr(Dep)); |
517 | if (DepSrcItr != SymbolSourceJDs.end()) |
518 | SDG.Dependencies[DepSrcItr->second].insert(V: Dep); |
519 | } |
520 | } |
521 | } |
522 | |
523 | SymbolDependenceGroup SynthSDG; |
524 | |
525 | for (auto &P : Layer.Plugins) { |
526 | auto SynthDeps = P->getSyntheticSymbolDependencies(MR&: *MR); |
527 | if (SynthDeps.empty()) |
528 | continue; |
529 | |
530 | DenseSet<Block *> BlockVisited; |
531 | for (auto &[Name, DepSyms] : SynthDeps) { |
532 | SynthSDG.Symbols.insert(V: Name); |
533 | for (auto *Sym : DepSyms) { |
534 | if (Sym->getScope() == Scope::Local) { |
535 | auto &BDeps = BlockDeps[Sym->getBlock()]; |
536 | for (auto &S : BDeps.Internal) |
537 | SynthSDG.Dependencies[&TargetJD].insert(V: S); |
538 | for (auto &S : BDeps.External) { |
539 | auto DepSrcItr = |
540 | SymbolSourceJDs.find(Val: NonOwningSymbolStringPtr(S)); |
541 | if (DepSrcItr != SymbolSourceJDs.end()) |
542 | SynthSDG.Dependencies[DepSrcItr->second].insert(V: S); |
543 | } |
544 | } else { |
545 | auto SymName = ES.intern(SymName: Sym->getName()); |
546 | if (Sym->isExternal()) { |
547 | assert(SymbolSourceJDs.count(NonOwningSymbolStringPtr(SymName)) && |
548 | "External symbol source entry missing" ); |
549 | SynthSDG |
550 | .Dependencies[SymbolSourceJDs[NonOwningSymbolStringPtr( |
551 | SymName)]] |
552 | .insert(V: SymName); |
553 | } else |
554 | SynthSDG.Dependencies[&TargetJD].insert(V: SymName); |
555 | } |
556 | } |
557 | } |
558 | } |
559 | |
560 | // Transfer SDGs to SymbolDepGroups. |
561 | DepGroupForBlock.reserve(NumEntries: DepGroupForBlock.size() + 1); |
562 | for (auto &[B, SDG] : DepGroupForBlock) { |
563 | assert(!SDG.Symbols.empty() && "SymbolDependenceGroup covers no symbols" ); |
564 | if (!SDG.Dependencies.empty()) |
565 | SymbolDepGroups.push_back(x: std::move(SDG)); |
566 | } |
567 | if (!SynthSDG.Symbols.empty() && !SynthSDG.Dependencies.empty()) |
568 | SymbolDepGroups.push_back(x: std::move(SynthSDG)); |
569 | |
570 | return Error::success(); |
571 | } |
572 | |
573 | BlockDependenciesMap computeBlockNonLocalDeps(LinkGraph &G) { |
574 | // First calculate the reachable-via-non-local-symbol blocks for each block. |
575 | struct BlockInfo { |
576 | DenseSet<Block *> Dependencies; |
577 | DenseSet<Block *> Dependants; |
578 | bool DependenciesChanged = true; |
579 | }; |
580 | DenseMap<Block *, BlockInfo> BlockInfos; |
581 | SmallVector<Block *> WorkList; |
582 | |
583 | // Pre-allocate map entries. This prevents any iterator/reference |
584 | // invalidation in the next loop. |
585 | for (auto *B : G.blocks()) |
586 | (void)BlockInfos[B]; |
587 | |
588 | // Build initial worklist, record block dependencies/dependants and |
589 | // non-local symbol dependencies. |
590 | for (auto *B : G.blocks()) { |
591 | auto &BI = BlockInfos[B]; |
592 | for (auto &E : B->edges()) { |
593 | if (E.getTarget().getScope() == Scope::Local && |
594 | !E.getTarget().isAbsolute()) { |
595 | auto &TgtB = E.getTarget().getBlock(); |
596 | if (&TgtB != B) { |
597 | BI.Dependencies.insert(V: &TgtB); |
598 | BlockInfos[&TgtB].Dependants.insert(V: B); |
599 | } |
600 | } |
601 | } |
602 | |
603 | // If this node has both dependants and dependencies then add it to the |
604 | // worklist to propagate the dependencies to the dependants. |
605 | if (!BI.Dependants.empty() && !BI.Dependencies.empty()) |
606 | WorkList.push_back(Elt: B); |
607 | } |
608 | |
609 | // Propagate block-level dependencies through the block-dependence graph. |
610 | while (!WorkList.empty()) { |
611 | auto *B = WorkList.pop_back_val(); |
612 | |
613 | auto &BI = BlockInfos[B]; |
614 | assert(BI.DependenciesChanged && |
615 | "Block in worklist has unchanged dependencies" ); |
616 | BI.DependenciesChanged = false; |
617 | for (auto *Dependant : BI.Dependants) { |
618 | auto &DependantBI = BlockInfos[Dependant]; |
619 | for (auto *Dependency : BI.Dependencies) { |
620 | if (Dependant != Dependency && |
621 | DependantBI.Dependencies.insert(V: Dependency).second) |
622 | if (!DependantBI.DependenciesChanged) { |
623 | DependantBI.DependenciesChanged = true; |
624 | WorkList.push_back(Elt: Dependant); |
625 | } |
626 | } |
627 | } |
628 | } |
629 | |
630 | DenseMap<const Block *, DenseSet<Block *>> BlockDeps; |
631 | for (auto &KV : BlockInfos) |
632 | BlockDeps[KV.first] = std::move(KV.second.Dependencies); |
633 | |
634 | return BlockDependenciesMap(Layer.getExecutionSession(), |
635 | std::move(BlockDeps)); |
636 | } |
637 | |
638 | ObjectLinkingLayer &Layer; |
639 | std::unique_ptr<MaterializationResponsibility> MR; |
640 | std::unique_ptr<MemoryBuffer> ObjBuffer; |
641 | DenseMap<Block *, SymbolNameSet> ExternalBlockDeps; |
642 | DenseMap<Block *, SymbolNameSet> InternalBlockDeps; |
643 | DenseMap<NonOwningSymbolStringPtr, JITDylib *> SymbolSourceJDs; |
644 | std::vector<SymbolDependenceGroup> SymbolDepGroups; |
645 | }; |
646 | |
647 | ObjectLinkingLayer::Plugin::~Plugin() = default; |
648 | |
649 | char ObjectLinkingLayer::ID; |
650 | |
651 | using BaseT = RTTIExtends<ObjectLinkingLayer, ObjectLayer>; |
652 | |
653 | ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES) |
654 | : BaseT(ES), MemMgr(ES.getExecutorProcessControl().getMemMgr()) { |
655 | ES.registerResourceManager(RM&: *this); |
656 | } |
657 | |
658 | ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, |
659 | JITLinkMemoryManager &MemMgr) |
660 | : BaseT(ES), MemMgr(MemMgr) { |
661 | ES.registerResourceManager(RM&: *this); |
662 | } |
663 | |
664 | ObjectLinkingLayer::ObjectLinkingLayer( |
665 | ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr) |
666 | : BaseT(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) { |
667 | ES.registerResourceManager(RM&: *this); |
668 | } |
669 | |
670 | ObjectLinkingLayer::~ObjectLinkingLayer() { |
671 | assert(Allocs.empty() && "Layer destroyed with resources still attached" ); |
672 | getExecutionSession().deregisterResourceManager(RM&: *this); |
673 | } |
674 | |
675 | Error ObjectLinkingLayer::add(ResourceTrackerSP RT, |
676 | std::unique_ptr<LinkGraph> G) { |
677 | auto &JD = RT->getJITDylib(); |
678 | return JD.define(MU: LinkGraphMaterializationUnit::Create(ObjLinkingLayer&: *this, G: std::move(G)), |
679 | RT: std::move(RT)); |
680 | } |
681 | |
682 | void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, |
683 | std::unique_ptr<MemoryBuffer> O) { |
684 | assert(O && "Object must not be null" ); |
685 | MemoryBufferRef ObjBuffer = O->getMemBufferRef(); |
686 | |
687 | auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( |
688 | args&: *this, args: std::move(R), args: std::move(O)); |
689 | if (auto G = createLinkGraphFromObject(ObjectBuffer: ObjBuffer)) { |
690 | Ctx->notifyMaterializing(G&: **G); |
691 | link(G: std::move(*G), Ctx: std::move(Ctx)); |
692 | } else { |
693 | Ctx->notifyFailed(Err: G.takeError()); |
694 | } |
695 | } |
696 | |
697 | void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, |
698 | std::unique_ptr<LinkGraph> G) { |
699 | auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( |
700 | args&: *this, args: std::move(R), args: nullptr); |
701 | Ctx->notifyMaterializing(G&: *G); |
702 | link(G: std::move(G), Ctx: std::move(Ctx)); |
703 | } |
704 | |
705 | void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR, |
706 | LinkGraph &G, |
707 | PassConfiguration &PassConfig) { |
708 | for (auto &P : Plugins) |
709 | P->modifyPassConfig(MR, G, Config&: PassConfig); |
710 | } |
711 | |
712 | void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) { |
713 | for (auto &P : Plugins) |
714 | P->notifyLoaded(MR); |
715 | } |
716 | |
717 | Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, |
718 | FinalizedAlloc FA) { |
719 | Error Err = Error::success(); |
720 | for (auto &P : Plugins) |
721 | Err = joinErrors(E1: std::move(Err), E2: P->notifyEmitted(MR)); |
722 | |
723 | if (Err) { |
724 | if (FA) |
725 | Err = joinErrors(E1: std::move(Err), E2: MemMgr.deallocate(Alloc: std::move(FA))); |
726 | return Err; |
727 | } |
728 | |
729 | if (!FA) |
730 | return Error::success(); |
731 | |
732 | Err = MR.withResourceKeyDo( |
733 | F: [&](ResourceKey K) { Allocs[K].push_back(x: std::move(FA)); }); |
734 | |
735 | if (Err) |
736 | Err = joinErrors(E1: std::move(Err), E2: MemMgr.deallocate(Alloc: std::move(FA))); |
737 | |
738 | return Err; |
739 | } |
740 | |
741 | Error ObjectLinkingLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) { |
742 | |
743 | { |
744 | Error Err = Error::success(); |
745 | for (auto &P : Plugins) |
746 | Err = joinErrors(E1: std::move(Err), E2: P->notifyRemovingResources(JD, K)); |
747 | if (Err) |
748 | return Err; |
749 | } |
750 | |
751 | std::vector<FinalizedAlloc> AllocsToRemove; |
752 | getExecutionSession().runSessionLocked(F: [&] { |
753 | auto I = Allocs.find(Val: K); |
754 | if (I != Allocs.end()) { |
755 | std::swap(x&: AllocsToRemove, y&: I->second); |
756 | Allocs.erase(I); |
757 | } |
758 | }); |
759 | |
760 | if (AllocsToRemove.empty()) |
761 | return Error::success(); |
762 | |
763 | return MemMgr.deallocate(Allocs: std::move(AllocsToRemove)); |
764 | } |
765 | |
766 | void ObjectLinkingLayer::handleTransferResources(JITDylib &JD, |
767 | ResourceKey DstKey, |
768 | ResourceKey SrcKey) { |
769 | auto I = Allocs.find(Val: SrcKey); |
770 | if (I != Allocs.end()) { |
771 | auto &SrcAllocs = I->second; |
772 | auto &DstAllocs = Allocs[DstKey]; |
773 | DstAllocs.reserve(n: DstAllocs.size() + SrcAllocs.size()); |
774 | for (auto &Alloc : SrcAllocs) |
775 | DstAllocs.push_back(x: std::move(Alloc)); |
776 | |
777 | // Erase SrcKey entry using value rather than iterator I: I may have been |
778 | // invalidated when we looked up DstKey. |
779 | Allocs.erase(Val: SrcKey); |
780 | } |
781 | |
782 | for (auto &P : Plugins) |
783 | P->notifyTransferringResources(JD, DstKey, SrcKey); |
784 | } |
785 | |
786 | EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( |
787 | ExecutionSession &ES, std::unique_ptr<EHFrameRegistrar> Registrar) |
788 | : ES(ES), Registrar(std::move(Registrar)) {} |
789 | |
790 | void EHFrameRegistrationPlugin::modifyPassConfig( |
791 | MaterializationResponsibility &MR, LinkGraph &G, |
792 | PassConfiguration &PassConfig) { |
793 | |
794 | PassConfig.PostFixupPasses.push_back(x: createEHFrameRecorderPass( |
795 | TT: G.getTargetTriple(), StoreFrameRange: [this, &MR](ExecutorAddr Addr, size_t Size) { |
796 | if (Addr) { |
797 | std::lock_guard<std::mutex> Lock(EHFramePluginMutex); |
798 | assert(!InProcessLinks.count(&MR) && |
799 | "Link for MR already being tracked?" ); |
800 | InProcessLinks[&MR] = {Addr, Size}; |
801 | } |
802 | })); |
803 | } |
804 | |
805 | Error EHFrameRegistrationPlugin::notifyEmitted( |
806 | MaterializationResponsibility &MR) { |
807 | |
808 | ExecutorAddrRange EmittedRange; |
809 | { |
810 | std::lock_guard<std::mutex> Lock(EHFramePluginMutex); |
811 | |
812 | auto EHFrameRangeItr = InProcessLinks.find(Val: &MR); |
813 | if (EHFrameRangeItr == InProcessLinks.end()) |
814 | return Error::success(); |
815 | |
816 | EmittedRange = EHFrameRangeItr->second; |
817 | assert(EmittedRange.Start && "eh-frame addr to register can not be null" ); |
818 | InProcessLinks.erase(I: EHFrameRangeItr); |
819 | } |
820 | |
821 | if (auto Err = MR.withResourceKeyDo( |
822 | F: [&](ResourceKey K) { EHFrameRanges[K].push_back(x: EmittedRange); })) |
823 | return Err; |
824 | |
825 | return Registrar->registerEHFrames(EHFrameSection: EmittedRange); |
826 | } |
827 | |
828 | Error EHFrameRegistrationPlugin::notifyFailed( |
829 | MaterializationResponsibility &MR) { |
830 | std::lock_guard<std::mutex> Lock(EHFramePluginMutex); |
831 | InProcessLinks.erase(Val: &MR); |
832 | return Error::success(); |
833 | } |
834 | |
835 | Error EHFrameRegistrationPlugin::notifyRemovingResources(JITDylib &JD, |
836 | ResourceKey K) { |
837 | std::vector<ExecutorAddrRange> RangesToRemove; |
838 | |
839 | ES.runSessionLocked(F: [&] { |
840 | auto I = EHFrameRanges.find(Val: K); |
841 | if (I != EHFrameRanges.end()) { |
842 | RangesToRemove = std::move(I->second); |
843 | EHFrameRanges.erase(I); |
844 | } |
845 | }); |
846 | |
847 | Error Err = Error::success(); |
848 | while (!RangesToRemove.empty()) { |
849 | auto RangeToRemove = RangesToRemove.back(); |
850 | RangesToRemove.pop_back(); |
851 | assert(RangeToRemove.Start && "Untracked eh-frame range must not be null" ); |
852 | Err = joinErrors(E1: std::move(Err), |
853 | E2: Registrar->deregisterEHFrames(EHFrameSection: RangeToRemove)); |
854 | } |
855 | |
856 | return Err; |
857 | } |
858 | |
859 | void EHFrameRegistrationPlugin::notifyTransferringResources( |
860 | JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) { |
861 | auto SI = EHFrameRanges.find(Val: SrcKey); |
862 | if (SI == EHFrameRanges.end()) |
863 | return; |
864 | |
865 | auto DI = EHFrameRanges.find(Val: DstKey); |
866 | if (DI != EHFrameRanges.end()) { |
867 | auto &SrcRanges = SI->second; |
868 | auto &DstRanges = DI->second; |
869 | DstRanges.reserve(n: DstRanges.size() + SrcRanges.size()); |
870 | for (auto &SrcRange : SrcRanges) |
871 | DstRanges.push_back(x: std::move(SrcRange)); |
872 | EHFrameRanges.erase(I: SI); |
873 | } else { |
874 | // We need to move SrcKey's ranges over without invalidating the SI |
875 | // iterator. |
876 | auto Tmp = std::move(SI->second); |
877 | EHFrameRanges.erase(I: SI); |
878 | EHFrameRanges[DstKey] = std::move(Tmp); |
879 | } |
880 | } |
881 | |
882 | } // End namespace orc. |
883 | } // End namespace llvm. |
884 | |