1//===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
10#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
11#include "llvm/ExecutionEngine/JITLink/aarch32.h"
12#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h"
13#include "llvm/ExecutionEngine/Orc/DebugUtils.h"
14#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
15#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
16#include "llvm/Support/MemoryBuffer.h"
17#include <string>
18#include <vector>
19
20#define DEBUG_TYPE "orc"
21
22using namespace llvm;
23using namespace llvm::jitlink;
24using namespace llvm::orc;
25
26namespace {
27
28bool hasInitializerSection(jitlink::LinkGraph &G) {
29 bool IsMachO = G.getTargetTriple().isOSBinFormatMachO();
30 bool IsElf = G.getTargetTriple().isOSBinFormatELF();
31 if (!IsMachO && !IsElf)
32 return false;
33
34 for (auto &Sec : G.sections()) {
35 if (IsMachO && isMachOInitializerSection(QualifiedName: Sec.getName()))
36 return true;
37 if (IsElf && isELFInitializerSection(SecName: Sec.getName()))
38 return true;
39 }
40
41 return false;
42}
43
44ExecutorAddr getJITSymbolPtrForSymbol(Symbol &Sym, const Triple &TT) {
45 switch (TT.getArch()) {
46 case Triple::arm:
47 case Triple::armeb:
48 case Triple::thumb:
49 case Triple::thumbeb:
50 if (hasTargetFlags(Sym, Flags: aarch32::ThumbSymbol)) {
51 // Set LSB to indicate thumb target
52 assert(Sym.isCallable() && "Only callable symbols can have thumb flag");
53 assert((Sym.getAddress().getValue() & 0x01) == 0 && "LSB is clear");
54 return Sym.getAddress() + 0x01;
55 }
56 return Sym.getAddress();
57 default:
58 return Sym.getAddress();
59 }
60}
61
62JITSymbolFlags getJITSymbolFlagsForSymbol(Symbol &Sym) {
63 JITSymbolFlags Flags;
64
65 if (Sym.getLinkage() == Linkage::Weak)
66 Flags |= JITSymbolFlags::Weak;
67
68 if (Sym.getScope() == Scope::Default)
69 Flags |= JITSymbolFlags::Exported;
70
71 if (Sym.isCallable())
72 Flags |= JITSymbolFlags::Callable;
73
74 return Flags;
75}
76
77class LinkGraphMaterializationUnit : public MaterializationUnit {
78public:
79 static std::unique_ptr<LinkGraphMaterializationUnit>
80 Create(ObjectLinkingLayer &ObjLinkingLayer, std::unique_ptr<LinkGraph> G) {
81 auto LGI = scanLinkGraph(ES&: ObjLinkingLayer.getExecutionSession(), G&: *G);
82 return std::unique_ptr<LinkGraphMaterializationUnit>(
83 new LinkGraphMaterializationUnit(ObjLinkingLayer, std::move(G),
84 std::move(LGI)));
85 }
86
87 StringRef getName() const override { return G->getName(); }
88 void materialize(std::unique_ptr<MaterializationResponsibility> MR) override {
89 ObjLinkingLayer.emit(R: std::move(MR), G: std::move(G));
90 }
91
92private:
93 static Interface scanLinkGraph(ExecutionSession &ES, LinkGraph &G) {
94
95 Interface LGI;
96
97 auto AddSymbol = [&](Symbol *Sym) {
98 // Skip local symbols.
99 if (Sym->getScope() == Scope::Local)
100 return;
101 assert(Sym->hasName() && "Anonymous non-local symbol?");
102
103 LGI.SymbolFlags[ES.intern(SymName: Sym->getName())] =
104 getJITSymbolFlagsForSymbol(Sym&: *Sym);
105 };
106
107 for (auto *Sym : G.defined_symbols())
108 AddSymbol(Sym);
109 for (auto *Sym : G.absolute_symbols())
110 AddSymbol(Sym);
111
112 if (hasInitializerSection(G))
113 LGI.InitSymbol = makeInitSymbol(ES, G);
114
115 return LGI;
116 }
117
118 static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) {
119 std::string InitSymString;
120 raw_string_ostream(InitSymString)
121 << "$." << G.getName() << ".__inits" << Counter++;
122 return ES.intern(SymName: InitSymString);
123 }
124
125 LinkGraphMaterializationUnit(ObjectLinkingLayer &ObjLinkingLayer,
126 std::unique_ptr<LinkGraph> G, Interface LGI)
127 : MaterializationUnit(std::move(LGI)), ObjLinkingLayer(ObjLinkingLayer),
128 G(std::move(G)) {}
129
130 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
131 for (auto *Sym : G->defined_symbols())
132 if (Sym->getName() == *Name) {
133 assert(Sym->getLinkage() == Linkage::Weak &&
134 "Discarding non-weak definition");
135 G->makeExternal(Sym&: *Sym);
136 break;
137 }
138 }
139
140 ObjectLinkingLayer &ObjLinkingLayer;
141 std::unique_ptr<LinkGraph> G;
142 static std::atomic<uint64_t> Counter;
143};
144
145std::atomic<uint64_t> LinkGraphMaterializationUnit::Counter{0};
146
147} // end anonymous namespace
148
149namespace llvm {
150namespace orc {
151
152class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
153public:
154 ObjectLinkingLayerJITLinkContext(
155 ObjectLinkingLayer &Layer,
156 std::unique_ptr<MaterializationResponsibility> MR,
157 std::unique_ptr<MemoryBuffer> ObjBuffer)
158 : JITLinkContext(&MR->getTargetJITDylib()), Layer(Layer),
159 MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {}
160
161 ~ObjectLinkingLayerJITLinkContext() {
162 // If there is an object buffer return function then use it to
163 // return ownership of the buffer.
164 if (Layer.ReturnObjectBuffer && ObjBuffer)
165 Layer.ReturnObjectBuffer(std::move(ObjBuffer));
166 }
167
168 JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; }
169
170 void notifyMaterializing(LinkGraph &G) {
171 for (auto &P : Layer.Plugins)
172 P->notifyMaterializing(MR&: *MR, G, Ctx&: *this,
173 InputObject: ObjBuffer ? ObjBuffer->getMemBufferRef()
174 : MemoryBufferRef());
175 }
176
177 void notifyFailed(Error Err) override {
178 for (auto &P : Layer.Plugins)
179 Err = joinErrors(E1: std::move(Err), E2: P->notifyFailed(MR&: *MR));
180 Layer.getExecutionSession().reportError(Err: std::move(Err));
181 MR->failMaterialization();
182 }
183
184 void lookup(const LookupMap &Symbols,
185 std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override {
186
187 JITDylibSearchOrder LinkOrder;
188 MR->getTargetJITDylib().withLinkOrderDo(
189 F: [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; });
190
191 auto &ES = Layer.getExecutionSession();
192
193 SymbolLookupSet LookupSet;
194 for (auto &KV : Symbols) {
195 orc::SymbolLookupFlags LookupFlags;
196 switch (KV.second) {
197 case jitlink::SymbolLookupFlags::RequiredSymbol:
198 LookupFlags = orc::SymbolLookupFlags::RequiredSymbol;
199 break;
200 case jitlink::SymbolLookupFlags::WeaklyReferencedSymbol:
201 LookupFlags = orc::SymbolLookupFlags::WeaklyReferencedSymbol;
202 break;
203 }
204 LookupSet.add(Name: ES.intern(SymName: KV.first), Flags: LookupFlags);
205 }
206
207 // OnResolve -- De-intern the symbols and pass the result to the linker.
208 auto OnResolve = [LookupContinuation =
209 std::move(LC)](Expected<SymbolMap> Result) mutable {
210 if (!Result)
211 LookupContinuation->run(LR: Result.takeError());
212 else {
213 AsyncLookupResult LR;
214 for (auto &KV : *Result)
215 LR[*KV.first] = KV.second;
216 LookupContinuation->run(LR: std::move(LR));
217 }
218 };
219
220 ES.lookup(K: LookupKind::Static, SearchOrder: LinkOrder, Symbols: std::move(LookupSet),
221 RequiredState: SymbolState::Resolved, NotifyComplete: std::move(OnResolve),
222 RegisterDependencies: [this](const SymbolDependenceMap &Deps) {
223 // Translate LookupDeps map to SymbolSourceJD.
224 for (auto &[DepJD, Deps] : Deps)
225 for (auto &DepSym : Deps)
226 SymbolSourceJDs[NonOwningSymbolStringPtr(DepSym)] = DepJD;
227 });
228 }
229
230 Error notifyResolved(LinkGraph &G) override {
231 auto &ES = Layer.getExecutionSession();
232
233 SymbolFlagsMap ExtraSymbolsToClaim;
234 bool AutoClaim = Layer.AutoClaimObjectSymbols;
235
236 SymbolMap InternedResult;
237 for (auto *Sym : G.defined_symbols())
238 if (Sym->hasName() && Sym->getScope() != Scope::Local) {
239 auto InternedName = ES.intern(SymName: Sym->getName());
240 auto Ptr = getJITSymbolPtrForSymbol(Sym&: *Sym, TT: G.getTargetTriple());
241 auto Flags = getJITSymbolFlagsForSymbol(Sym&: *Sym);
242 InternedResult[InternedName] = {Ptr, Flags};
243 if (AutoClaim && !MR->getSymbols().count(Val: InternedName)) {
244 assert(!ExtraSymbolsToClaim.count(InternedName) &&
245 "Duplicate symbol to claim?");
246 ExtraSymbolsToClaim[InternedName] = Flags;
247 }
248 }
249
250 for (auto *Sym : G.absolute_symbols())
251 if (Sym->hasName() && Sym->getScope() != Scope::Local) {
252 auto InternedName = ES.intern(SymName: Sym->getName());
253 auto Ptr = getJITSymbolPtrForSymbol(Sym&: *Sym, TT: G.getTargetTriple());
254 auto Flags = getJITSymbolFlagsForSymbol(Sym&: *Sym);
255 InternedResult[InternedName] = {Ptr, Flags};
256 if (AutoClaim && !MR->getSymbols().count(Val: InternedName)) {
257 assert(!ExtraSymbolsToClaim.count(InternedName) &&
258 "Duplicate symbol to claim?");
259 ExtraSymbolsToClaim[InternedName] = Flags;
260 }
261 }
262
263 if (!ExtraSymbolsToClaim.empty())
264 if (auto Err = MR->defineMaterializing(SymbolFlags: ExtraSymbolsToClaim))
265 return Err;
266
267 {
268
269 // Check that InternedResult matches up with MR->getSymbols(), overriding
270 // flags if requested.
271 // This guards against faulty transformations / compilers / object caches.
272
273 // First check that there aren't any missing symbols.
274 size_t NumMaterializationSideEffectsOnlySymbols = 0;
275 SymbolNameVector ExtraSymbols;
276 SymbolNameVector MissingSymbols;
277 for (auto &KV : MR->getSymbols()) {
278
279 auto I = InternedResult.find(Val: KV.first);
280
281 // If this is a materialization-side-effects only symbol then bump
282 // the counter and make sure it's *not* defined, otherwise make
283 // sure that it is defined.
284 if (KV.second.hasMaterializationSideEffectsOnly()) {
285 ++NumMaterializationSideEffectsOnlySymbols;
286 if (I != InternedResult.end())
287 ExtraSymbols.push_back(x: KV.first);
288 continue;
289 } else if (I == InternedResult.end())
290 MissingSymbols.push_back(x: KV.first);
291 else if (Layer.OverrideObjectFlags)
292 I->second.setFlags(KV.second);
293 }
294
295 // If there were missing symbols then report the error.
296 if (!MissingSymbols.empty())
297 return make_error<MissingSymbolDefinitions>(
298 Args: Layer.getExecutionSession().getSymbolStringPool(), Args: G.getName(),
299 Args: std::move(MissingSymbols));
300
301 // If there are more definitions than expected, add them to the
302 // ExtraSymbols vector.
303 if (InternedResult.size() >
304 MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) {
305 for (auto &KV : InternedResult)
306 if (!MR->getSymbols().count(Val: KV.first))
307 ExtraSymbols.push_back(x: KV.first);
308 }
309
310 // If there were extra definitions then report the error.
311 if (!ExtraSymbols.empty())
312 return make_error<UnexpectedSymbolDefinitions>(
313 Args: Layer.getExecutionSession().getSymbolStringPool(), Args: G.getName(),
314 Args: std::move(ExtraSymbols));
315 }
316
317 if (auto Err = MR->notifyResolved(Symbols: InternedResult))
318 return Err;
319
320 Layer.notifyLoaded(MR&: *MR);
321 return Error::success();
322 }
323
324 void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A) override {
325 if (auto Err = Layer.notifyEmitted(MR&: *MR, FA: std::move(A))) {
326 Layer.getExecutionSession().reportError(Err: std::move(Err));
327 MR->failMaterialization();
328 return;
329 }
330 if (auto Err = MR->notifyEmitted(EmittedDeps: SymbolDepGroups)) {
331 Layer.getExecutionSession().reportError(Err: std::move(Err));
332 MR->failMaterialization();
333 }
334 }
335
336 LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override {
337 return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); };
338 }
339
340 Error modifyPassConfig(LinkGraph &LG, PassConfiguration &Config) override {
341 // Add passes to mark duplicate defs as should-discard, and to walk the
342 // link graph to build the symbol dependence graph.
343 Config.PrePrunePasses.push_back(x: [this](LinkGraph &G) {
344 return claimOrExternalizeWeakAndCommonSymbols(G);
345 });
346
347 Layer.modifyPassConfig(MR&: *MR, G&: LG, PassConfig&: Config);
348
349 Config.PreFixupPasses.push_back(
350 x: [this](LinkGraph &G) { return registerDependencies(G); });
351
352 return Error::success();
353 }
354
355private:
356 // Symbol name dependencies:
357 // Internal: Defined in this graph.
358 // External: Defined externally.
359 struct BlockSymbolDependencies {
360 SymbolNameSet Internal, External;
361 };
362
363 // Lazily populated map of blocks to BlockSymbolDependencies values.
364 class BlockDependenciesMap {
365 public:
366 BlockDependenciesMap(ExecutionSession &ES,
367 DenseMap<const Block *, DenseSet<Block *>> BlockDeps)
368 : ES(ES), BlockDeps(std::move(BlockDeps)) {}
369
370 const BlockSymbolDependencies &operator[](const Block &B) {
371 // Check the cache first.
372 auto I = BlockTransitiveDepsCache.find(Val: &B);
373 if (I != BlockTransitiveDepsCache.end())
374 return I->second;
375
376 // No value. Populate the cache.
377 BlockSymbolDependencies BTDCacheVal;
378 auto BDI = BlockDeps.find(Val: &B);
379 assert(BDI != BlockDeps.end() && "No block dependencies");
380
381 for (auto *BDep : BDI->second) {
382 auto &BID = getBlockImmediateDeps(B&: *BDep);
383 for (auto &ExternalDep : BID.External)
384 BTDCacheVal.External.insert(V: ExternalDep);
385 for (auto &InternalDep : BID.Internal)
386 BTDCacheVal.Internal.insert(V: InternalDep);
387 }
388
389 return BlockTransitiveDepsCache
390 .insert(KV: std::make_pair(x: &B, y: std::move(BTDCacheVal)))
391 .first->second;
392 }
393
394 SymbolStringPtr &getInternedName(Symbol &Sym) {
395 auto I = NameCache.find(Val: &Sym);
396 if (I != NameCache.end())
397 return I->second;
398
399 return NameCache.insert(KV: std::make_pair(x: &Sym, y: ES.intern(SymName: Sym.getName())))
400 .first->second;
401 }
402
403 private:
404 BlockSymbolDependencies &getBlockImmediateDeps(Block &B) {
405 // Check the cache first.
406 auto I = BlockImmediateDepsCache.find(Val: &B);
407 if (I != BlockImmediateDepsCache.end())
408 return I->second;
409
410 BlockSymbolDependencies BIDCacheVal;
411 for (auto &E : B.edges()) {
412 auto &Tgt = E.getTarget();
413 if (Tgt.getScope() != Scope::Local) {
414 if (Tgt.isExternal()) {
415 if (Tgt.getAddress() || !Tgt.isWeaklyReferenced())
416 BIDCacheVal.External.insert(V: getInternedName(Sym&: Tgt));
417 } else
418 BIDCacheVal.Internal.insert(V: getInternedName(Sym&: Tgt));
419 }
420 }
421
422 return BlockImmediateDepsCache
423 .insert(KV: std::make_pair(x: &B, y: std::move(BIDCacheVal)))
424 .first->second;
425 }
426
427 ExecutionSession &ES;
428 DenseMap<const Block *, DenseSet<Block *>> BlockDeps;
429 DenseMap<const Symbol *, SymbolStringPtr> NameCache;
430 DenseMap<const Block *, BlockSymbolDependencies> BlockImmediateDepsCache;
431 DenseMap<const Block *, BlockSymbolDependencies> BlockTransitiveDepsCache;
432 };
433
434 Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) {
435 auto &ES = Layer.getExecutionSession();
436
437 SymbolFlagsMap NewSymbolsToClaim;
438 std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym;
439
440 auto ProcessSymbol = [&](Symbol *Sym) {
441 if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak &&
442 Sym->getScope() != Scope::Local) {
443 auto Name = ES.intern(SymName: Sym->getName());
444 if (!MR->getSymbols().count(Val: ES.intern(SymName: Sym->getName()))) {
445 NewSymbolsToClaim[Name] =
446 getJITSymbolFlagsForSymbol(Sym&: *Sym) | JITSymbolFlags::Weak;
447 NameToSym.push_back(x: std::make_pair(x: std::move(Name), y&: Sym));
448 }
449 }
450 };
451
452 for (auto *Sym : G.defined_symbols())
453 ProcessSymbol(Sym);
454 for (auto *Sym : G.absolute_symbols())
455 ProcessSymbol(Sym);
456
457 // Attempt to claim all weak defs that we're not already responsible for.
458 // This may fail if the resource tracker has become defunct, but should
459 // always succeed otherwise.
460 if (auto Err = MR->defineMaterializing(SymbolFlags: std::move(NewSymbolsToClaim)))
461 return Err;
462
463 // Walk the list of symbols that we just tried to claim. Symbols that we're
464 // responsible for are marked live. Symbols that we're not responsible for
465 // are turned into external references.
466 for (auto &KV : NameToSym) {
467 if (MR->getSymbols().count(Val: KV.first))
468 KV.second->setLive(true);
469 else
470 G.makeExternal(Sym&: *KV.second);
471 }
472
473 return Error::success();
474 }
475
476 Error markResponsibilitySymbolsLive(LinkGraph &G) const {
477 auto &ES = Layer.getExecutionSession();
478 for (auto *Sym : G.defined_symbols())
479 if (Sym->hasName() && MR->getSymbols().count(Val: ES.intern(SymName: Sym->getName())))
480 Sym->setLive(true);
481 return Error::success();
482 }
483
484 Error registerDependencies(LinkGraph &G) {
485 auto &TargetJD = MR->getTargetJITDylib();
486 auto &ES = TargetJD.getExecutionSession();
487 auto BlockDeps = computeBlockNonLocalDeps(G);
488
489 DenseSet<Block *> BlockDepsProcessed;
490 DenseMap<Block *, SymbolDependenceGroup> DepGroupForBlock;
491
492 // Compute dependencies for symbols defined in the JITLink graph.
493 for (auto *Sym : G.defined_symbols()) {
494
495 // Skip local symbols.
496 if (Sym->getScope() == Scope::Local)
497 continue;
498 assert(Sym->hasName() &&
499 "Defined non-local jitlink::Symbol should have a name");
500
501 auto &BDeps = BlockDeps[Sym->getBlock()];
502
503 // Skip symbols in blocks that don't depend on anything.
504 if (BDeps.Internal.empty() && BDeps.External.empty())
505 continue;
506
507 SymbolDependenceGroup &SDG = DepGroupForBlock[&Sym->getBlock()];
508 SDG.Symbols.insert(V: ES.intern(SymName: Sym->getName()));
509
510 if (!BlockDepsProcessed.count(V: &Sym->getBlock())) {
511 BlockDepsProcessed.insert(V: &Sym->getBlock());
512
513 if (!BDeps.Internal.empty())
514 SDG.Dependencies[&TargetJD] = BDeps.Internal;
515 for (auto &Dep : BDeps.External) {
516 auto DepSrcItr = SymbolSourceJDs.find(Val: NonOwningSymbolStringPtr(Dep));
517 if (DepSrcItr != SymbolSourceJDs.end())
518 SDG.Dependencies[DepSrcItr->second].insert(V: Dep);
519 }
520 }
521 }
522
523 SymbolDependenceGroup SynthSDG;
524
525 for (auto &P : Layer.Plugins) {
526 auto SynthDeps = P->getSyntheticSymbolDependencies(MR&: *MR);
527 if (SynthDeps.empty())
528 continue;
529
530 DenseSet<Block *> BlockVisited;
531 for (auto &[Name, DepSyms] : SynthDeps) {
532 SynthSDG.Symbols.insert(V: Name);
533 for (auto *Sym : DepSyms) {
534 if (Sym->getScope() == Scope::Local) {
535 auto &BDeps = BlockDeps[Sym->getBlock()];
536 for (auto &S : BDeps.Internal)
537 SynthSDG.Dependencies[&TargetJD].insert(V: S);
538 for (auto &S : BDeps.External) {
539 auto DepSrcItr =
540 SymbolSourceJDs.find(Val: NonOwningSymbolStringPtr(S));
541 if (DepSrcItr != SymbolSourceJDs.end())
542 SynthSDG.Dependencies[DepSrcItr->second].insert(V: S);
543 }
544 } else {
545 auto SymName = ES.intern(SymName: Sym->getName());
546 if (Sym->isExternal()) {
547 assert(SymbolSourceJDs.count(NonOwningSymbolStringPtr(SymName)) &&
548 "External symbol source entry missing");
549 SynthSDG
550 .Dependencies[SymbolSourceJDs[NonOwningSymbolStringPtr(
551 SymName)]]
552 .insert(V: SymName);
553 } else
554 SynthSDG.Dependencies[&TargetJD].insert(V: SymName);
555 }
556 }
557 }
558 }
559
560 // Transfer SDGs to SymbolDepGroups.
561 DepGroupForBlock.reserve(NumEntries: DepGroupForBlock.size() + 1);
562 for (auto &[B, SDG] : DepGroupForBlock) {
563 assert(!SDG.Symbols.empty() && "SymbolDependenceGroup covers no symbols");
564 if (!SDG.Dependencies.empty())
565 SymbolDepGroups.push_back(x: std::move(SDG));
566 }
567 if (!SynthSDG.Symbols.empty() && !SynthSDG.Dependencies.empty())
568 SymbolDepGroups.push_back(x: std::move(SynthSDG));
569
570 return Error::success();
571 }
572
573 BlockDependenciesMap computeBlockNonLocalDeps(LinkGraph &G) {
574 // First calculate the reachable-via-non-local-symbol blocks for each block.
575 struct BlockInfo {
576 DenseSet<Block *> Dependencies;
577 DenseSet<Block *> Dependants;
578 bool DependenciesChanged = true;
579 };
580 DenseMap<Block *, BlockInfo> BlockInfos;
581 SmallVector<Block *> WorkList;
582
583 // Pre-allocate map entries. This prevents any iterator/reference
584 // invalidation in the next loop.
585 for (auto *B : G.blocks())
586 (void)BlockInfos[B];
587
588 // Build initial worklist, record block dependencies/dependants and
589 // non-local symbol dependencies.
590 for (auto *B : G.blocks()) {
591 auto &BI = BlockInfos[B];
592 for (auto &E : B->edges()) {
593 if (E.getTarget().getScope() == Scope::Local &&
594 !E.getTarget().isAbsolute()) {
595 auto &TgtB = E.getTarget().getBlock();
596 if (&TgtB != B) {
597 BI.Dependencies.insert(V: &TgtB);
598 BlockInfos[&TgtB].Dependants.insert(V: B);
599 }
600 }
601 }
602
603 // If this node has both dependants and dependencies then add it to the
604 // worklist to propagate the dependencies to the dependants.
605 if (!BI.Dependants.empty() && !BI.Dependencies.empty())
606 WorkList.push_back(Elt: B);
607 }
608
609 // Propagate block-level dependencies through the block-dependence graph.
610 while (!WorkList.empty()) {
611 auto *B = WorkList.pop_back_val();
612
613 auto &BI = BlockInfos[B];
614 assert(BI.DependenciesChanged &&
615 "Block in worklist has unchanged dependencies");
616 BI.DependenciesChanged = false;
617 for (auto *Dependant : BI.Dependants) {
618 auto &DependantBI = BlockInfos[Dependant];
619 for (auto *Dependency : BI.Dependencies) {
620 if (Dependant != Dependency &&
621 DependantBI.Dependencies.insert(V: Dependency).second)
622 if (!DependantBI.DependenciesChanged) {
623 DependantBI.DependenciesChanged = true;
624 WorkList.push_back(Elt: Dependant);
625 }
626 }
627 }
628 }
629
630 DenseMap<const Block *, DenseSet<Block *>> BlockDeps;
631 for (auto &KV : BlockInfos)
632 BlockDeps[KV.first] = std::move(KV.second.Dependencies);
633
634 return BlockDependenciesMap(Layer.getExecutionSession(),
635 std::move(BlockDeps));
636 }
637
638 ObjectLinkingLayer &Layer;
639 std::unique_ptr<MaterializationResponsibility> MR;
640 std::unique_ptr<MemoryBuffer> ObjBuffer;
641 DenseMap<Block *, SymbolNameSet> ExternalBlockDeps;
642 DenseMap<Block *, SymbolNameSet> InternalBlockDeps;
643 DenseMap<NonOwningSymbolStringPtr, JITDylib *> SymbolSourceJDs;
644 std::vector<SymbolDependenceGroup> SymbolDepGroups;
645};
646
647ObjectLinkingLayer::Plugin::~Plugin() = default;
648
649char ObjectLinkingLayer::ID;
650
651using BaseT = RTTIExtends<ObjectLinkingLayer, ObjectLayer>;
652
653ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES)
654 : BaseT(ES), MemMgr(ES.getExecutorProcessControl().getMemMgr()) {
655 ES.registerResourceManager(RM&: *this);
656}
657
658ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES,
659 JITLinkMemoryManager &MemMgr)
660 : BaseT(ES), MemMgr(MemMgr) {
661 ES.registerResourceManager(RM&: *this);
662}
663
664ObjectLinkingLayer::ObjectLinkingLayer(
665 ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr)
666 : BaseT(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) {
667 ES.registerResourceManager(RM&: *this);
668}
669
670ObjectLinkingLayer::~ObjectLinkingLayer() {
671 assert(Allocs.empty() && "Layer destroyed with resources still attached");
672 getExecutionSession().deregisterResourceManager(RM&: *this);
673}
674
675Error ObjectLinkingLayer::add(ResourceTrackerSP RT,
676 std::unique_ptr<LinkGraph> G) {
677 auto &JD = RT->getJITDylib();
678 return JD.define(MU: LinkGraphMaterializationUnit::Create(ObjLinkingLayer&: *this, G: std::move(G)),
679 RT: std::move(RT));
680}
681
682void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
683 std::unique_ptr<MemoryBuffer> O) {
684 assert(O && "Object must not be null");
685 MemoryBufferRef ObjBuffer = O->getMemBufferRef();
686
687 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>(
688 args&: *this, args: std::move(R), args: std::move(O));
689 if (auto G = createLinkGraphFromObject(ObjectBuffer: ObjBuffer)) {
690 Ctx->notifyMaterializing(G&: **G);
691 link(G: std::move(*G), Ctx: std::move(Ctx));
692 } else {
693 Ctx->notifyFailed(Err: G.takeError());
694 }
695}
696
697void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
698 std::unique_ptr<LinkGraph> G) {
699 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>(
700 args&: *this, args: std::move(R), args: nullptr);
701 Ctx->notifyMaterializing(G&: *G);
702 link(G: std::move(G), Ctx: std::move(Ctx));
703}
704
705void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR,
706 LinkGraph &G,
707 PassConfiguration &PassConfig) {
708 for (auto &P : Plugins)
709 P->modifyPassConfig(MR, G, Config&: PassConfig);
710}
711
712void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) {
713 for (auto &P : Plugins)
714 P->notifyLoaded(MR);
715}
716
717Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR,
718 FinalizedAlloc FA) {
719 Error Err = Error::success();
720 for (auto &P : Plugins)
721 Err = joinErrors(E1: std::move(Err), E2: P->notifyEmitted(MR));
722
723 if (Err) {
724 if (FA)
725 Err = joinErrors(E1: std::move(Err), E2: MemMgr.deallocate(Alloc: std::move(FA)));
726 return Err;
727 }
728
729 if (!FA)
730 return Error::success();
731
732 Err = MR.withResourceKeyDo(
733 F: [&](ResourceKey K) { Allocs[K].push_back(x: std::move(FA)); });
734
735 if (Err)
736 Err = joinErrors(E1: std::move(Err), E2: MemMgr.deallocate(Alloc: std::move(FA)));
737
738 return Err;
739}
740
741Error ObjectLinkingLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) {
742
743 {
744 Error Err = Error::success();
745 for (auto &P : Plugins)
746 Err = joinErrors(E1: std::move(Err), E2: P->notifyRemovingResources(JD, K));
747 if (Err)
748 return Err;
749 }
750
751 std::vector<FinalizedAlloc> AllocsToRemove;
752 getExecutionSession().runSessionLocked(F: [&] {
753 auto I = Allocs.find(Val: K);
754 if (I != Allocs.end()) {
755 std::swap(x&: AllocsToRemove, y&: I->second);
756 Allocs.erase(I);
757 }
758 });
759
760 if (AllocsToRemove.empty())
761 return Error::success();
762
763 return MemMgr.deallocate(Allocs: std::move(AllocsToRemove));
764}
765
766void ObjectLinkingLayer::handleTransferResources(JITDylib &JD,
767 ResourceKey DstKey,
768 ResourceKey SrcKey) {
769 auto I = Allocs.find(Val: SrcKey);
770 if (I != Allocs.end()) {
771 auto &SrcAllocs = I->second;
772 auto &DstAllocs = Allocs[DstKey];
773 DstAllocs.reserve(n: DstAllocs.size() + SrcAllocs.size());
774 for (auto &Alloc : SrcAllocs)
775 DstAllocs.push_back(x: std::move(Alloc));
776
777 // Erase SrcKey entry using value rather than iterator I: I may have been
778 // invalidated when we looked up DstKey.
779 Allocs.erase(Val: SrcKey);
780 }
781
782 for (auto &P : Plugins)
783 P->notifyTransferringResources(JD, DstKey, SrcKey);
784}
785
786EHFrameRegistrationPlugin::EHFrameRegistrationPlugin(
787 ExecutionSession &ES, std::unique_ptr<EHFrameRegistrar> Registrar)
788 : ES(ES), Registrar(std::move(Registrar)) {}
789
790void EHFrameRegistrationPlugin::modifyPassConfig(
791 MaterializationResponsibility &MR, LinkGraph &G,
792 PassConfiguration &PassConfig) {
793
794 PassConfig.PostFixupPasses.push_back(x: createEHFrameRecorderPass(
795 TT: G.getTargetTriple(), StoreFrameRange: [this, &MR](ExecutorAddr Addr, size_t Size) {
796 if (Addr) {
797 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
798 assert(!InProcessLinks.count(&MR) &&
799 "Link for MR already being tracked?");
800 InProcessLinks[&MR] = {Addr, Size};
801 }
802 }));
803}
804
805Error EHFrameRegistrationPlugin::notifyEmitted(
806 MaterializationResponsibility &MR) {
807
808 ExecutorAddrRange EmittedRange;
809 {
810 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
811
812 auto EHFrameRangeItr = InProcessLinks.find(Val: &MR);
813 if (EHFrameRangeItr == InProcessLinks.end())
814 return Error::success();
815
816 EmittedRange = EHFrameRangeItr->second;
817 assert(EmittedRange.Start && "eh-frame addr to register can not be null");
818 InProcessLinks.erase(I: EHFrameRangeItr);
819 }
820
821 if (auto Err = MR.withResourceKeyDo(
822 F: [&](ResourceKey K) { EHFrameRanges[K].push_back(x: EmittedRange); }))
823 return Err;
824
825 return Registrar->registerEHFrames(EHFrameSection: EmittedRange);
826}
827
828Error EHFrameRegistrationPlugin::notifyFailed(
829 MaterializationResponsibility &MR) {
830 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
831 InProcessLinks.erase(Val: &MR);
832 return Error::success();
833}
834
835Error EHFrameRegistrationPlugin::notifyRemovingResources(JITDylib &JD,
836 ResourceKey K) {
837 std::vector<ExecutorAddrRange> RangesToRemove;
838
839 ES.runSessionLocked(F: [&] {
840 auto I = EHFrameRanges.find(Val: K);
841 if (I != EHFrameRanges.end()) {
842 RangesToRemove = std::move(I->second);
843 EHFrameRanges.erase(I);
844 }
845 });
846
847 Error Err = Error::success();
848 while (!RangesToRemove.empty()) {
849 auto RangeToRemove = RangesToRemove.back();
850 RangesToRemove.pop_back();
851 assert(RangeToRemove.Start && "Untracked eh-frame range must not be null");
852 Err = joinErrors(E1: std::move(Err),
853 E2: Registrar->deregisterEHFrames(EHFrameSection: RangeToRemove));
854 }
855
856 return Err;
857}
858
859void EHFrameRegistrationPlugin::notifyTransferringResources(
860 JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) {
861 auto SI = EHFrameRanges.find(Val: SrcKey);
862 if (SI == EHFrameRanges.end())
863 return;
864
865 auto DI = EHFrameRanges.find(Val: DstKey);
866 if (DI != EHFrameRanges.end()) {
867 auto &SrcRanges = SI->second;
868 auto &DstRanges = DI->second;
869 DstRanges.reserve(n: DstRanges.size() + SrcRanges.size());
870 for (auto &SrcRange : SrcRanges)
871 DstRanges.push_back(x: std::move(SrcRange));
872 EHFrameRanges.erase(I: SI);
873 } else {
874 // We need to move SrcKey's ranges over without invalidating the SI
875 // iterator.
876 auto Tmp = std::move(SI->second);
877 EHFrameRanges.erase(I: SI);
878 EHFrameRanges[DstKey] = std::move(Tmp);
879 }
880}
881
882} // End namespace orc.
883} // End namespace llvm.
884

source code of llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp