1//===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// ELF/x86-64 jit-link implementation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
14#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
15#include "llvm/ExecutionEngine/JITLink/JITLink.h"
16#include "llvm/ExecutionEngine/JITLink/TableManager.h"
17#include "llvm/ExecutionEngine/JITLink/x86_64.h"
18#include "llvm/Object/ELFObjectFile.h"
19
20#include "DefineExternalSectionStartAndEndSymbols.h"
21#include "EHFrameSupportImpl.h"
22#include "ELFLinkGraphBuilder.h"
23#include "JITLinkGeneric.h"
24
25#define DEBUG_TYPE "jitlink"
26
27using namespace llvm;
28using namespace llvm::jitlink;
29
30namespace {
31
32constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
33constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";
34
35class TLSInfoTableManager_ELF_x86_64
36 : public TableManager<TLSInfoTableManager_ELF_x86_64> {
37public:
38 static const uint8_t TLSInfoEntryContent[16];
39
40 static StringRef getSectionName() { return ELFTLSInfoSectionName; }
41
42 bool visitEdge(LinkGraph &G, Block *B, Edge &E) {
43 if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) {
44 LLVM_DEBUG({
45 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
46 << formatv("{0:x}", B->getFixupAddress(E)) << " ("
47 << formatv("{0:x}", B->getAddress()) << " + "
48 << formatv("{0:x}", E.getOffset()) << ")\n";
49 });
50 E.setKind(x86_64::Delta32);
51 E.setTarget(getEntryForTarget(G, Target&: E.getTarget()));
52 return true;
53 }
54 return false;
55 }
56
57 Symbol &createEntry(LinkGraph &G, Symbol &Target) {
58 // the TLS Info entry's key value will be written by the fixTLVSectionByName
59 // pass, so create mutable content.
60 auto &TLSInfoEntry = G.createMutableContentBlock(
61 Parent&: getTLSInfoSection(G), MutableContent: G.allocateContent(Source: getTLSInfoEntryContent()),
62 Address: orc::ExecutorAddr(), Alignment: 8, AlignmentOffset: 0);
63 TLSInfoEntry.addEdge(K: x86_64::Pointer64, Offset: 8, Target, Addend: 0);
64 return G.addAnonymousSymbol(Content&: TLSInfoEntry, Offset: 0, Size: 16, IsCallable: false, IsLive: false);
65 }
66
67private:
68 Section &getTLSInfoSection(LinkGraph &G) {
69 if (!TLSInfoTable)
70 TLSInfoTable =
71 &G.createSection(Name: ELFTLSInfoSectionName, Prot: orc::MemProt::Read);
72 return *TLSInfoTable;
73 }
74
75 ArrayRef<char> getTLSInfoEntryContent() const {
76 return {reinterpret_cast<const char *>(TLSInfoEntryContent),
77 sizeof(TLSInfoEntryContent)};
78 }
79
80 Section *TLSInfoTable = nullptr;
81};
82
83const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = {
84 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
85 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/
86};
87
88Error buildTables_ELF_x86_64(LinkGraph &G) {
89 LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
90
91 x86_64::GOTTableManager GOT;
92 x86_64::PLTTableManager PLT(GOT);
93 TLSInfoTableManager_ELF_x86_64 TLSInfo;
94 visitExistingEdges(G, Vs&: GOT, Vs&: PLT, Vs&: TLSInfo);
95 return Error::success();
96}
97} // namespace
98
99namespace llvm {
100namespace jitlink {
101
102class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
103private:
104 using ELFT = object::ELF64LE;
105
106 Error addRelocations() override {
107 LLVM_DEBUG(dbgs() << "Processing relocations:\n");
108
109 using Base = ELFLinkGraphBuilder<ELFT>;
110 using Self = ELFLinkGraphBuilder_x86_64;
111 for (const auto &RelSect : Base::Sections) {
112 // Validate the section to read relocation entries from.
113 if (RelSect.sh_type == ELF::SHT_REL)
114 return make_error<StringError>(
115 Args: "No SHT_REL in valid x64 ELF object files",
116 Args: inconvertibleErrorCode());
117
118 if (Error Err = Base::forEachRelaRelocation(RelSect, Instance: this,
119 Method: &Self::addSingleRelocation))
120 return Err;
121 }
122
123 return Error::success();
124 }
125
126 Error addSingleRelocation(const typename ELFT::Rela &Rel,
127 const typename ELFT::Shdr &FixupSection,
128 Block &BlockToFix) {
129 using Base = ELFLinkGraphBuilder<ELFT>;
130
131 auto ELFReloc = Rel.getType(isMips64EL: false);
132
133 // R_X86_64_NONE is a no-op.
134 if (LLVM_UNLIKELY(ELFReloc == ELF::R_X86_64_NONE))
135 return Error::success();
136
137 uint32_t SymbolIndex = Rel.getSymbol(isMips64EL: false);
138 auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, SymTab: Base::SymTabSec);
139 if (!ObjSymbol)
140 return ObjSymbol.takeError();
141
142 Symbol *GraphSymbol = Base::getGraphSymbol(SymIndex: SymbolIndex);
143 if (!GraphSymbol)
144 return make_error<StringError>(
145 Args: formatv(Fmt: "Could not find symbol at given index, did you add it to "
146 "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
147 Vals&: SymbolIndex, Vals: (*ObjSymbol)->st_shndx,
148 Vals: Base::GraphSymbols.size()),
149 Args: inconvertibleErrorCode());
150
151 // Validate the relocation kind.
152 int64_t Addend = Rel.r_addend;
153 Edge::Kind Kind = Edge::Invalid;
154
155 switch (ELFReloc) {
156 case ELF::R_X86_64_PC32:
157 case ELF::R_X86_64_GOTPC32:
158 Kind = x86_64::Delta32;
159 break;
160 case ELF::R_X86_64_PC64:
161 case ELF::R_X86_64_GOTPC64:
162 Kind = x86_64::Delta64;
163 break;
164 case ELF::R_X86_64_32:
165 Kind = x86_64::Pointer32;
166 break;
167 case ELF::R_X86_64_16:
168 Kind = x86_64::Pointer16;
169 break;
170 case ELF::R_X86_64_8:
171 Kind = x86_64::Pointer8;
172 break;
173 case ELF::R_X86_64_32S:
174 Kind = x86_64::Pointer32Signed;
175 break;
176 case ELF::R_X86_64_64:
177 Kind = x86_64::Pointer64;
178 break;
179 case ELF::R_X86_64_GOTPCREL:
180 Kind = x86_64::RequestGOTAndTransformToDelta32;
181 break;
182 case ELF::R_X86_64_REX_GOTPCRELX:
183 Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
184 Addend = 0;
185 break;
186 case ELF::R_X86_64_TLSGD:
187 Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32;
188 break;
189 case ELF::R_X86_64_GOTPCRELX:
190 Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
191 Addend = 0;
192 break;
193 case ELF::R_X86_64_GOTPCREL64:
194 Kind = x86_64::RequestGOTAndTransformToDelta64;
195 break;
196 case ELF::R_X86_64_GOT64:
197 Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
198 break;
199 case ELF::R_X86_64_GOTOFF64:
200 Kind = x86_64::Delta64FromGOT;
201 break;
202 case ELF::R_X86_64_PLT32:
203 Kind = x86_64::BranchPCRel32;
204 // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to
205 // adjust the addend by '+4' to compensate.
206 Addend += 4;
207 break;
208 default:
209 return make_error<JITLinkError>(
210 Args: "In " + G->getName() + ": Unsupported x86-64 relocation type " +
211 object::getELFRelocationTypeName(Machine: ELF::EM_X86_64, Type: ELFReloc));
212 }
213
214 auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset;
215 Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
216 Edge GE(Kind, Offset, *GraphSymbol, Addend);
217 LLVM_DEBUG({
218 dbgs() << " ";
219 printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind));
220 dbgs() << "\n";
221 });
222
223 BlockToFix.addEdge(E: std::move(GE));
224 return Error::success();
225 }
226
227public:
228 ELFLinkGraphBuilder_x86_64(StringRef FileName,
229 const object::ELFFile<object::ELF64LE> &Obj,
230 SubtargetFeatures Features)
231 : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"),
232 std::move(Features), FileName,
233 x86_64::getEdgeKindName) {}
234};
235
236class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
237 friend class JITLinker<ELFJITLinker_x86_64>;
238
239public:
240 ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
241 std::unique_ptr<LinkGraph> G,
242 PassConfiguration PassConfig)
243 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
244
245 if (shouldAddDefaultTargetPasses(TT: getGraph().getTargetTriple()))
246 getPassConfig().PostAllocationPasses.push_back(
247 x: [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
248 }
249
250private:
251 Symbol *GOTSymbol = nullptr;
252
253 Error getOrCreateGOTSymbol(LinkGraph &G) {
254 auto DefineExternalGOTSymbolIfPresent =
255 createDefineExternalSectionStartAndEndSymbolsPass(
256 F: [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
257 if (Sym.getName() == ELFGOTSymbolName)
258 if (auto *GOTSection = G.findSectionByName(
259 Name: x86_64::GOTTableManager::getSectionName())) {
260 GOTSymbol = &Sym;
261 return {*GOTSection, true};
262 }
263 return {};
264 });
265
266 // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
267 // external.
268 if (auto Err = DefineExternalGOTSymbolIfPresent(G))
269 return Err;
270
271 // If we succeeded then we're done.
272 if (GOTSymbol)
273 return Error::success();
274
275 // Otherwise look for a GOT section: If it already has a start symbol we'll
276 // record it, otherwise we'll create our own.
277 // If there's a GOT section but we didn't find an external GOT symbol...
278 if (auto *GOTSection =
279 G.findSectionByName(Name: x86_64::GOTTableManager::getSectionName())) {
280
281 // Check for an existing defined symbol.
282 for (auto *Sym : GOTSection->symbols())
283 if (Sym->getName() == ELFGOTSymbolName) {
284 GOTSymbol = Sym;
285 return Error::success();
286 }
287
288 // If there's no defined symbol then create one.
289 SectionRange SR(*GOTSection);
290 if (SR.empty())
291 GOTSymbol =
292 &G.addAbsoluteSymbol(Name: ELFGOTSymbolName, Address: orc::ExecutorAddr(), Size: 0,
293 L: Linkage::Strong, S: Scope::Local, IsLive: true);
294 else
295 GOTSymbol =
296 &G.addDefinedSymbol(Content&: *SR.getFirstBlock(), Offset: 0, Name: ELFGOTSymbolName, Size: 0,
297 L: Linkage::Strong, S: Scope::Local, IsCallable: false, IsLive: true);
298 }
299
300 // If we still haven't found a GOT symbol then double check the externals.
301 // We may have a GOT-relative reference but no GOT section, in which case
302 // we just need to point the GOT symbol at some address in this graph.
303 if (!GOTSymbol) {
304 for (auto *Sym : G.external_symbols()) {
305 if (Sym->getName() == ELFGOTSymbolName) {
306 auto Blocks = G.blocks();
307 if (!Blocks.empty()) {
308 G.makeAbsolute(Sym&: *Sym, Address: (*Blocks.begin())->getAddress());
309 GOTSymbol = Sym;
310 break;
311 }
312 }
313 }
314 }
315
316 return Error::success();
317 }
318
319 Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
320 return x86_64::applyFixup(G, B, E, GOTSymbol);
321 }
322};
323
324Expected<std::unique_ptr<LinkGraph>>
325createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) {
326 LLVM_DEBUG({
327 dbgs() << "Building jitlink graph for new input "
328 << ObjectBuffer.getBufferIdentifier() << "...\n";
329 });
330
331 auto ELFObj = object::ObjectFile::createELFObjectFile(Object: ObjectBuffer);
332 if (!ELFObj)
333 return ELFObj.takeError();
334
335 auto Features = (*ELFObj)->getFeatures();
336 if (!Features)
337 return Features.takeError();
338
339 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(Val&: **ELFObj);
340 return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(),
341 ELFObjFile.getELFFile(),
342 std::move(*Features))
343 .buildGraph();
344}
345
346static SectionRangeSymbolDesc
347identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) {
348 constexpr StringRef StartSymbolPrefix = "__start";
349 constexpr StringRef EndSymbolPrefix = "__end";
350
351 auto SymName = Sym.getName();
352 if (SymName.starts_with(Prefix: StartSymbolPrefix)) {
353 if (auto *Sec =
354 G.findSectionByName(Name: SymName.drop_front(N: StartSymbolPrefix.size())))
355 return {*Sec, true};
356 } else if (SymName.starts_with(Prefix: EndSymbolPrefix)) {
357 if (auto *Sec =
358 G.findSectionByName(Name: SymName.drop_front(N: EndSymbolPrefix.size())))
359 return {*Sec, false};
360 }
361 return {};
362}
363
364void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
365 std::unique_ptr<JITLinkContext> Ctx) {
366 PassConfiguration Config;
367
368 if (Ctx->shouldAddDefaultTargetPasses(TT: G->getTargetTriple())) {
369
370 Config.PrePrunePasses.push_back(x: DWARFRecordSectionSplitter(".eh_frame"));
371 Config.PrePrunePasses.push_back(x: EHFrameEdgeFixer(
372 ".eh_frame", x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64,
373 x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32));
374 Config.PrePrunePasses.push_back(x: EHFrameNullTerminator(".eh_frame"));
375
376 // Construct a JITLinker and run the link function.
377 // Add a mark-live pass.
378 if (auto MarkLive = Ctx->getMarkLivePass(TT: G->getTargetTriple()))
379 Config.PrePrunePasses.push_back(x: std::move(MarkLive));
380 else
381 Config.PrePrunePasses.push_back(x: markAllSymbolsLive);
382
383 // Add an in-place GOT/Stubs/TLSInfoEntry build pass.
384 Config.PostPrunePasses.push_back(x: buildTables_ELF_x86_64);
385
386 // Resolve any external section start / end symbols.
387 Config.PostAllocationPasses.push_back(
388 x: createDefineExternalSectionStartAndEndSymbolsPass(
389 F&: identifyELFSectionStartAndEndSymbols));
390
391 // Add GOT/Stubs optimizer pass.
392 Config.PreFixupPasses.push_back(x: x86_64::optimizeGOTAndStubAccesses);
393 }
394
395 if (auto Err = Ctx->modifyPassConfig(G&: *G, Config))
396 return Ctx->notifyFailed(Err: std::move(Err));
397
398 ELFJITLinker_x86_64::link(Args: std::move(Ctx), Args: std::move(G), Args: std::move(Config));
399}
400} // end namespace jitlink
401} // end namespace llvm
402

source code of llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp