1 | //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // ELF/x86-64 jit-link implementation. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" |
14 | #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" |
15 | #include "llvm/ExecutionEngine/JITLink/JITLink.h" |
16 | #include "llvm/ExecutionEngine/JITLink/TableManager.h" |
17 | #include "llvm/ExecutionEngine/JITLink/x86_64.h" |
18 | #include "llvm/Object/ELFObjectFile.h" |
19 | |
20 | #include "DefineExternalSectionStartAndEndSymbols.h" |
21 | #include "EHFrameSupportImpl.h" |
22 | #include "ELFLinkGraphBuilder.h" |
23 | #include "JITLinkGeneric.h" |
24 | |
25 | #define DEBUG_TYPE "jitlink" |
26 | |
27 | using namespace llvm; |
28 | using namespace llvm::jitlink; |
29 | |
30 | namespace { |
31 | |
32 | constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_" ; |
33 | constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO" ; |
34 | |
35 | class TLSInfoTableManager_ELF_x86_64 |
36 | : public TableManager<TLSInfoTableManager_ELF_x86_64> { |
37 | public: |
38 | static const uint8_t TLSInfoEntryContent[16]; |
39 | |
40 | static StringRef getSectionName() { return ELFTLSInfoSectionName; } |
41 | |
42 | bool visitEdge(LinkGraph &G, Block *B, Edge &E) { |
43 | if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) { |
44 | LLVM_DEBUG({ |
45 | dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " |
46 | << formatv("{0:x}" , B->getFixupAddress(E)) << " (" |
47 | << formatv("{0:x}" , B->getAddress()) << " + " |
48 | << formatv("{0:x}" , E.getOffset()) << ")\n" ; |
49 | }); |
50 | E.setKind(x86_64::Delta32); |
51 | E.setTarget(getEntryForTarget(G, Target&: E.getTarget())); |
52 | return true; |
53 | } |
54 | return false; |
55 | } |
56 | |
57 | Symbol &createEntry(LinkGraph &G, Symbol &Target) { |
58 | // the TLS Info entry's key value will be written by the fixTLVSectionByName |
59 | // pass, so create mutable content. |
60 | auto &TLSInfoEntry = G.createMutableContentBlock( |
61 | Parent&: getTLSInfoSection(G), MutableContent: G.allocateContent(Source: getTLSInfoEntryContent()), |
62 | Address: orc::ExecutorAddr(), Alignment: 8, AlignmentOffset: 0); |
63 | TLSInfoEntry.addEdge(K: x86_64::Pointer64, Offset: 8, Target, Addend: 0); |
64 | return G.addAnonymousSymbol(Content&: TLSInfoEntry, Offset: 0, Size: 16, IsCallable: false, IsLive: false); |
65 | } |
66 | |
67 | private: |
68 | Section &getTLSInfoSection(LinkGraph &G) { |
69 | if (!TLSInfoTable) |
70 | TLSInfoTable = |
71 | &G.createSection(Name: ELFTLSInfoSectionName, Prot: orc::MemProt::Read); |
72 | return *TLSInfoTable; |
73 | } |
74 | |
75 | ArrayRef<char> getTLSInfoEntryContent() const { |
76 | return {reinterpret_cast<const char *>(TLSInfoEntryContent), |
77 | sizeof(TLSInfoEntryContent)}; |
78 | } |
79 | |
80 | Section *TLSInfoTable = nullptr; |
81 | }; |
82 | |
83 | const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = { |
84 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */ |
85 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/ |
86 | }; |
87 | |
88 | Error buildTables_ELF_x86_64(LinkGraph &G) { |
89 | LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n" ); |
90 | |
91 | x86_64::GOTTableManager GOT; |
92 | x86_64::PLTTableManager PLT(GOT); |
93 | TLSInfoTableManager_ELF_x86_64 TLSInfo; |
94 | visitExistingEdges(G, Vs&: GOT, Vs&: PLT, Vs&: TLSInfo); |
95 | return Error::success(); |
96 | } |
97 | } // namespace |
98 | |
99 | namespace llvm { |
100 | namespace jitlink { |
101 | |
102 | class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> { |
103 | private: |
104 | using ELFT = object::ELF64LE; |
105 | |
106 | Error addRelocations() override { |
107 | LLVM_DEBUG(dbgs() << "Processing relocations:\n" ); |
108 | |
109 | using Base = ELFLinkGraphBuilder<ELFT>; |
110 | using Self = ELFLinkGraphBuilder_x86_64; |
111 | for (const auto &RelSect : Base::Sections) { |
112 | // Validate the section to read relocation entries from. |
113 | if (RelSect.sh_type == ELF::SHT_REL) |
114 | return make_error<StringError>( |
115 | Args: "No SHT_REL in valid x64 ELF object files" , |
116 | Args: inconvertibleErrorCode()); |
117 | |
118 | if (Error Err = Base::forEachRelaRelocation(RelSect, Instance: this, |
119 | Method: &Self::addSingleRelocation)) |
120 | return Err; |
121 | } |
122 | |
123 | return Error::success(); |
124 | } |
125 | |
126 | Error addSingleRelocation(const typename ELFT::Rela &Rel, |
127 | const typename ELFT::Shdr &FixupSection, |
128 | Block &BlockToFix) { |
129 | using Base = ELFLinkGraphBuilder<ELFT>; |
130 | |
131 | auto ELFReloc = Rel.getType(isMips64EL: false); |
132 | |
133 | // R_X86_64_NONE is a no-op. |
134 | if (LLVM_UNLIKELY(ELFReloc == ELF::R_X86_64_NONE)) |
135 | return Error::success(); |
136 | |
137 | uint32_t SymbolIndex = Rel.getSymbol(isMips64EL: false); |
138 | auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, SymTab: Base::SymTabSec); |
139 | if (!ObjSymbol) |
140 | return ObjSymbol.takeError(); |
141 | |
142 | Symbol *GraphSymbol = Base::getGraphSymbol(SymIndex: SymbolIndex); |
143 | if (!GraphSymbol) |
144 | return make_error<StringError>( |
145 | Args: formatv(Fmt: "Could not find symbol at given index, did you add it to " |
146 | "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}" , |
147 | Vals&: SymbolIndex, Vals: (*ObjSymbol)->st_shndx, |
148 | Vals: Base::GraphSymbols.size()), |
149 | Args: inconvertibleErrorCode()); |
150 | |
151 | // Validate the relocation kind. |
152 | int64_t Addend = Rel.r_addend; |
153 | Edge::Kind Kind = Edge::Invalid; |
154 | |
155 | switch (ELFReloc) { |
156 | case ELF::R_X86_64_PC32: |
157 | case ELF::R_X86_64_GOTPC32: |
158 | Kind = x86_64::Delta32; |
159 | break; |
160 | case ELF::R_X86_64_PC64: |
161 | case ELF::R_X86_64_GOTPC64: |
162 | Kind = x86_64::Delta64; |
163 | break; |
164 | case ELF::R_X86_64_32: |
165 | Kind = x86_64::Pointer32; |
166 | break; |
167 | case ELF::R_X86_64_16: |
168 | Kind = x86_64::Pointer16; |
169 | break; |
170 | case ELF::R_X86_64_8: |
171 | Kind = x86_64::Pointer8; |
172 | break; |
173 | case ELF::R_X86_64_32S: |
174 | Kind = x86_64::Pointer32Signed; |
175 | break; |
176 | case ELF::R_X86_64_64: |
177 | Kind = x86_64::Pointer64; |
178 | break; |
179 | case ELF::R_X86_64_GOTPCREL: |
180 | Kind = x86_64::RequestGOTAndTransformToDelta32; |
181 | break; |
182 | case ELF::R_X86_64_REX_GOTPCRELX: |
183 | Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable; |
184 | Addend = 0; |
185 | break; |
186 | case ELF::R_X86_64_TLSGD: |
187 | Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32; |
188 | break; |
189 | case ELF::R_X86_64_GOTPCRELX: |
190 | Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable; |
191 | Addend = 0; |
192 | break; |
193 | case ELF::R_X86_64_GOTPCREL64: |
194 | Kind = x86_64::RequestGOTAndTransformToDelta64; |
195 | break; |
196 | case ELF::R_X86_64_GOT64: |
197 | Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT; |
198 | break; |
199 | case ELF::R_X86_64_GOTOFF64: |
200 | Kind = x86_64::Delta64FromGOT; |
201 | break; |
202 | case ELF::R_X86_64_PLT32: |
203 | Kind = x86_64::BranchPCRel32; |
204 | // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to |
205 | // adjust the addend by '+4' to compensate. |
206 | Addend += 4; |
207 | break; |
208 | default: |
209 | return make_error<JITLinkError>( |
210 | Args: "In " + G->getName() + ": Unsupported x86-64 relocation type " + |
211 | object::getELFRelocationTypeName(Machine: ELF::EM_X86_64, Type: ELFReloc)); |
212 | } |
213 | |
214 | auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset; |
215 | Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); |
216 | Edge GE(Kind, Offset, *GraphSymbol, Addend); |
217 | LLVM_DEBUG({ |
218 | dbgs() << " " ; |
219 | printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind)); |
220 | dbgs() << "\n" ; |
221 | }); |
222 | |
223 | BlockToFix.addEdge(E: std::move(GE)); |
224 | return Error::success(); |
225 | } |
226 | |
227 | public: |
228 | ELFLinkGraphBuilder_x86_64(StringRef FileName, |
229 | const object::ELFFile<object::ELF64LE> &Obj, |
230 | SubtargetFeatures Features) |
231 | : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux" ), |
232 | std::move(Features), FileName, |
233 | x86_64::getEdgeKindName) {} |
234 | }; |
235 | |
236 | class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> { |
237 | friend class JITLinker<ELFJITLinker_x86_64>; |
238 | |
239 | public: |
240 | ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, |
241 | std::unique_ptr<LinkGraph> G, |
242 | PassConfiguration PassConfig) |
243 | : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) { |
244 | |
245 | if (shouldAddDefaultTargetPasses(TT: getGraph().getTargetTriple())) |
246 | getPassConfig().PostAllocationPasses.push_back( |
247 | x: [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); }); |
248 | } |
249 | |
250 | private: |
251 | Symbol *GOTSymbol = nullptr; |
252 | |
253 | Error getOrCreateGOTSymbol(LinkGraph &G) { |
254 | auto DefineExternalGOTSymbolIfPresent = |
255 | createDefineExternalSectionStartAndEndSymbolsPass( |
256 | F: [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc { |
257 | if (Sym.getName() == ELFGOTSymbolName) |
258 | if (auto *GOTSection = G.findSectionByName( |
259 | Name: x86_64::GOTTableManager::getSectionName())) { |
260 | GOTSymbol = &Sym; |
261 | return {*GOTSection, true}; |
262 | } |
263 | return {}; |
264 | }); |
265 | |
266 | // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an |
267 | // external. |
268 | if (auto Err = DefineExternalGOTSymbolIfPresent(G)) |
269 | return Err; |
270 | |
271 | // If we succeeded then we're done. |
272 | if (GOTSymbol) |
273 | return Error::success(); |
274 | |
275 | // Otherwise look for a GOT section: If it already has a start symbol we'll |
276 | // record it, otherwise we'll create our own. |
277 | // If there's a GOT section but we didn't find an external GOT symbol... |
278 | if (auto *GOTSection = |
279 | G.findSectionByName(Name: x86_64::GOTTableManager::getSectionName())) { |
280 | |
281 | // Check for an existing defined symbol. |
282 | for (auto *Sym : GOTSection->symbols()) |
283 | if (Sym->getName() == ELFGOTSymbolName) { |
284 | GOTSymbol = Sym; |
285 | return Error::success(); |
286 | } |
287 | |
288 | // If there's no defined symbol then create one. |
289 | SectionRange SR(*GOTSection); |
290 | if (SR.empty()) |
291 | GOTSymbol = |
292 | &G.addAbsoluteSymbol(Name: ELFGOTSymbolName, Address: orc::ExecutorAddr(), Size: 0, |
293 | L: Linkage::Strong, S: Scope::Local, IsLive: true); |
294 | else |
295 | GOTSymbol = |
296 | &G.addDefinedSymbol(Content&: *SR.getFirstBlock(), Offset: 0, Name: ELFGOTSymbolName, Size: 0, |
297 | L: Linkage::Strong, S: Scope::Local, IsCallable: false, IsLive: true); |
298 | } |
299 | |
300 | // If we still haven't found a GOT symbol then double check the externals. |
301 | // We may have a GOT-relative reference but no GOT section, in which case |
302 | // we just need to point the GOT symbol at some address in this graph. |
303 | if (!GOTSymbol) { |
304 | for (auto *Sym : G.external_symbols()) { |
305 | if (Sym->getName() == ELFGOTSymbolName) { |
306 | auto Blocks = G.blocks(); |
307 | if (!Blocks.empty()) { |
308 | G.makeAbsolute(Sym&: *Sym, Address: (*Blocks.begin())->getAddress()); |
309 | GOTSymbol = Sym; |
310 | break; |
311 | } |
312 | } |
313 | } |
314 | } |
315 | |
316 | return Error::success(); |
317 | } |
318 | |
319 | Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { |
320 | return x86_64::applyFixup(G, B, E, GOTSymbol); |
321 | } |
322 | }; |
323 | |
324 | Expected<std::unique_ptr<LinkGraph>> |
325 | createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { |
326 | LLVM_DEBUG({ |
327 | dbgs() << "Building jitlink graph for new input " |
328 | << ObjectBuffer.getBufferIdentifier() << "...\n" ; |
329 | }); |
330 | |
331 | auto ELFObj = object::ObjectFile::createELFObjectFile(Object: ObjectBuffer); |
332 | if (!ELFObj) |
333 | return ELFObj.takeError(); |
334 | |
335 | auto Features = (*ELFObj)->getFeatures(); |
336 | if (!Features) |
337 | return Features.takeError(); |
338 | |
339 | auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(Val&: **ELFObj); |
340 | return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(), |
341 | ELFObjFile.getELFFile(), |
342 | std::move(*Features)) |
343 | .buildGraph(); |
344 | } |
345 | |
346 | static SectionRangeSymbolDesc |
347 | identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { |
348 | constexpr StringRef StartSymbolPrefix = "__start" ; |
349 | constexpr StringRef EndSymbolPrefix = "__end" ; |
350 | |
351 | auto SymName = Sym.getName(); |
352 | if (SymName.starts_with(Prefix: StartSymbolPrefix)) { |
353 | if (auto *Sec = |
354 | G.findSectionByName(Name: SymName.drop_front(N: StartSymbolPrefix.size()))) |
355 | return {*Sec, true}; |
356 | } else if (SymName.starts_with(Prefix: EndSymbolPrefix)) { |
357 | if (auto *Sec = |
358 | G.findSectionByName(Name: SymName.drop_front(N: EndSymbolPrefix.size()))) |
359 | return {*Sec, false}; |
360 | } |
361 | return {}; |
362 | } |
363 | |
364 | void link_ELF_x86_64(std::unique_ptr<LinkGraph> G, |
365 | std::unique_ptr<JITLinkContext> Ctx) { |
366 | PassConfiguration Config; |
367 | |
368 | if (Ctx->shouldAddDefaultTargetPasses(TT: G->getTargetTriple())) { |
369 | |
370 | Config.PrePrunePasses.push_back(x: DWARFRecordSectionSplitter(".eh_frame" )); |
371 | Config.PrePrunePasses.push_back(x: EHFrameEdgeFixer( |
372 | ".eh_frame" , x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64, |
373 | x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32)); |
374 | Config.PrePrunePasses.push_back(x: EHFrameNullTerminator(".eh_frame" )); |
375 | |
376 | // Construct a JITLinker and run the link function. |
377 | // Add a mark-live pass. |
378 | if (auto MarkLive = Ctx->getMarkLivePass(TT: G->getTargetTriple())) |
379 | Config.PrePrunePasses.push_back(x: std::move(MarkLive)); |
380 | else |
381 | Config.PrePrunePasses.push_back(x: markAllSymbolsLive); |
382 | |
383 | // Add an in-place GOT/Stubs/TLSInfoEntry build pass. |
384 | Config.PostPrunePasses.push_back(x: buildTables_ELF_x86_64); |
385 | |
386 | // Resolve any external section start / end symbols. |
387 | Config.PostAllocationPasses.push_back( |
388 | x: createDefineExternalSectionStartAndEndSymbolsPass( |
389 | F&: identifyELFSectionStartAndEndSymbols)); |
390 | |
391 | // Add GOT/Stubs optimizer pass. |
392 | Config.PreFixupPasses.push_back(x: x86_64::optimizeGOTAndStubAccesses); |
393 | } |
394 | |
395 | if (auto Err = Ctx->modifyPassConfig(G&: *G, Config)) |
396 | return Ctx->notifyFailed(Err: std::move(Err)); |
397 | |
398 | ELFJITLinker_x86_64::link(Args: std::move(Ctx), Args: std::move(G), Args: std::move(Config)); |
399 | } |
400 | } // end namespace jitlink |
401 | } // end namespace llvm |
402 | |