1 | // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) |
2 | /* |
3 | * BPF static linker |
4 | * |
5 | * Copyright (c) 2021 Facebook |
6 | */ |
7 | #include <stdbool.h> |
8 | #include <stddef.h> |
9 | #include <stdio.h> |
10 | #include <stdlib.h> |
11 | #include <string.h> |
12 | #include <unistd.h> |
13 | #include <errno.h> |
14 | #include <linux/err.h> |
15 | #include <linux/btf.h> |
16 | #include <elf.h> |
17 | #include <libelf.h> |
18 | #include <fcntl.h> |
19 | #include "libbpf.h" |
20 | #include "btf.h" |
21 | #include "libbpf_internal.h" |
22 | #include "strset.h" |
23 | |
24 | #define BTF_EXTERN_SEC ".extern" |
25 | |
26 | struct src_sec { |
27 | const char *sec_name; |
28 | /* positional (not necessarily ELF) index in an array of sections */ |
29 | int id; |
30 | /* positional (not necessarily ELF) index of a matching section in a final object file */ |
31 | int dst_id; |
32 | /* section data offset in a matching output section */ |
33 | int dst_off; |
34 | /* whether section is omitted from the final ELF file */ |
35 | bool skipped; |
36 | /* whether section is an ephemeral section, not mapped to an ELF section */ |
37 | bool ephemeral; |
38 | |
39 | /* ELF info */ |
40 | size_t sec_idx; |
41 | Elf_Scn *scn; |
42 | Elf64_Shdr *shdr; |
43 | Elf_Data *data; |
44 | |
45 | /* corresponding BTF DATASEC type ID */ |
46 | int sec_type_id; |
47 | }; |
48 | |
49 | struct src_obj { |
50 | const char *filename; |
51 | int fd; |
52 | Elf *elf; |
53 | /* Section header strings section index */ |
54 | size_t shstrs_sec_idx; |
55 | /* SYMTAB section index */ |
56 | size_t symtab_sec_idx; |
57 | |
58 | struct btf *btf; |
59 | struct btf_ext *btf_ext; |
60 | |
61 | /* List of sections (including ephemeral). Slot zero is unused. */ |
62 | struct src_sec *secs; |
63 | int sec_cnt; |
64 | |
65 | /* mapping of symbol indices from src to dst ELF */ |
66 | int *sym_map; |
67 | /* mapping from the src BTF type IDs to dst ones */ |
68 | int *btf_type_map; |
69 | }; |
70 | |
71 | /* single .BTF.ext data section */ |
72 | struct btf_ext_sec_data { |
73 | size_t rec_cnt; |
74 | __u32 rec_sz; |
75 | void *recs; |
76 | }; |
77 | |
78 | struct glob_sym { |
79 | /* ELF symbol index */ |
80 | int sym_idx; |
81 | /* associated section id for .ksyms, .kconfig, etc, but not .extern */ |
82 | int sec_id; |
83 | /* extern name offset in STRTAB */ |
84 | int name_off; |
85 | /* optional associated BTF type ID */ |
86 | int btf_id; |
87 | /* BTF type ID to which VAR/FUNC type is pointing to; used for |
88 | * rewriting types when extern VAR/FUNC is resolved to a concrete |
89 | * definition |
90 | */ |
91 | int underlying_btf_id; |
92 | /* sec_var index in the corresponding dst_sec, if exists */ |
93 | int var_idx; |
94 | |
95 | /* extern or resolved/global symbol */ |
96 | bool is_extern; |
97 | /* weak or strong symbol, never goes back from strong to weak */ |
98 | bool is_weak; |
99 | }; |
100 | |
101 | struct dst_sec { |
102 | char *sec_name; |
103 | /* positional (not necessarily ELF) index in an array of sections */ |
104 | int id; |
105 | |
106 | bool ephemeral; |
107 | |
108 | /* ELF info */ |
109 | size_t sec_idx; |
110 | Elf_Scn *scn; |
111 | Elf64_Shdr *shdr; |
112 | Elf_Data *data; |
113 | |
114 | /* final output section size */ |
115 | int sec_sz; |
116 | /* final output contents of the section */ |
117 | void *raw_data; |
118 | |
119 | /* corresponding STT_SECTION symbol index in SYMTAB */ |
120 | int sec_sym_idx; |
121 | |
122 | /* section's DATASEC variable info, emitted on BTF finalization */ |
123 | bool has_btf; |
124 | int sec_var_cnt; |
125 | struct btf_var_secinfo *sec_vars; |
126 | |
127 | /* section's .BTF.ext data */ |
128 | struct btf_ext_sec_data func_info; |
129 | struct btf_ext_sec_data line_info; |
130 | struct btf_ext_sec_data core_relo_info; |
131 | }; |
132 | |
133 | struct bpf_linker { |
134 | char *filename; |
135 | int fd; |
136 | Elf *elf; |
137 | Elf64_Ehdr *elf_hdr; |
138 | |
139 | /* Output sections metadata */ |
140 | struct dst_sec *secs; |
141 | int sec_cnt; |
142 | |
143 | struct strset *strtab_strs; /* STRTAB unique strings */ |
144 | size_t strtab_sec_idx; /* STRTAB section index */ |
145 | size_t symtab_sec_idx; /* SYMTAB section index */ |
146 | |
147 | struct btf *btf; |
148 | struct btf_ext *btf_ext; |
149 | |
150 | /* global (including extern) ELF symbols */ |
151 | int glob_sym_cnt; |
152 | struct glob_sym *glob_syms; |
153 | }; |
154 | |
155 | #define pr_warn_elf(fmt, ...) \ |
156 | libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1)) |
157 | |
158 | static int init_output_elf(struct bpf_linker *linker, const char *file); |
159 | |
160 | static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, |
161 | const struct bpf_linker_file_opts *opts, |
162 | struct src_obj *obj); |
163 | static int linker_sanity_check_elf(struct src_obj *obj); |
164 | static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec); |
165 | static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec); |
166 | static int linker_sanity_check_btf(struct src_obj *obj); |
167 | static int linker_sanity_check_btf_ext(struct src_obj *obj); |
168 | static int linker_fixup_btf(struct src_obj *obj); |
169 | static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj); |
170 | static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj); |
171 | static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj, |
172 | Elf64_Sym *sym, const char *sym_name, int src_sym_idx); |
173 | static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj); |
174 | static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj); |
175 | static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj); |
176 | |
177 | static int finalize_btf(struct bpf_linker *linker); |
178 | static int finalize_btf_ext(struct bpf_linker *linker); |
179 | |
180 | void bpf_linker__free(struct bpf_linker *linker) |
181 | { |
182 | int i; |
183 | |
184 | if (!linker) |
185 | return; |
186 | |
187 | free(linker->filename); |
188 | |
189 | if (linker->elf) |
190 | elf_end(linker->elf); |
191 | |
192 | if (linker->fd >= 0) |
193 | close(linker->fd); |
194 | |
195 | strset__free(set: linker->strtab_strs); |
196 | |
197 | btf__free(btf: linker->btf); |
198 | btf_ext__free(btf_ext: linker->btf_ext); |
199 | |
200 | for (i = 1; i < linker->sec_cnt; i++) { |
201 | struct dst_sec *sec = &linker->secs[i]; |
202 | |
203 | free(sec->sec_name); |
204 | free(sec->raw_data); |
205 | free(sec->sec_vars); |
206 | |
207 | free(sec->func_info.recs); |
208 | free(sec->line_info.recs); |
209 | free(sec->core_relo_info.recs); |
210 | } |
211 | free(linker->secs); |
212 | |
213 | free(linker->glob_syms); |
214 | free(linker); |
215 | } |
216 | |
217 | struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts) |
218 | { |
219 | struct bpf_linker *linker; |
220 | int err; |
221 | |
222 | if (!OPTS_VALID(opts, bpf_linker_opts)) |
223 | return errno = EINVAL, NULL; |
224 | |
225 | if (elf_version(EV_CURRENT) == EV_NONE) { |
226 | pr_warn_elf("libelf initialization failed" ); |
227 | return errno = EINVAL, NULL; |
228 | } |
229 | |
230 | linker = calloc(1, sizeof(*linker)); |
231 | if (!linker) |
232 | return errno = ENOMEM, NULL; |
233 | |
234 | linker->fd = -1; |
235 | |
236 | err = init_output_elf(linker, file: filename); |
237 | if (err) |
238 | goto err_out; |
239 | |
240 | return linker; |
241 | |
242 | err_out: |
243 | bpf_linker__free(linker); |
244 | return errno = -err, NULL; |
245 | } |
246 | |
247 | static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name) |
248 | { |
249 | struct dst_sec *secs = linker->secs, *sec; |
250 | size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2; |
251 | |
252 | secs = libbpf_reallocarray(ptr: secs, nmemb: new_cnt, size: sizeof(*secs)); |
253 | if (!secs) |
254 | return NULL; |
255 | |
256 | /* zero out newly allocated memory */ |
257 | memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs)); |
258 | |
259 | linker->secs = secs; |
260 | linker->sec_cnt = new_cnt; |
261 | |
262 | sec = &linker->secs[new_cnt - 1]; |
263 | sec->id = new_cnt - 1; |
264 | sec->sec_name = strdup(sec_name); |
265 | if (!sec->sec_name) |
266 | return NULL; |
267 | |
268 | return sec; |
269 | } |
270 | |
271 | static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx) |
272 | { |
273 | struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; |
274 | Elf64_Sym *syms, *sym; |
275 | size_t sym_cnt = symtab->sec_sz / sizeof(*sym); |
276 | |
277 | syms = libbpf_reallocarray(ptr: symtab->raw_data, nmemb: sym_cnt + 1, size: sizeof(*sym)); |
278 | if (!syms) |
279 | return NULL; |
280 | |
281 | sym = &syms[sym_cnt]; |
282 | memset(sym, 0, sizeof(*sym)); |
283 | |
284 | symtab->raw_data = syms; |
285 | symtab->sec_sz += sizeof(*sym); |
286 | symtab->shdr->sh_size += sizeof(*sym); |
287 | symtab->data->d_size += sizeof(*sym); |
288 | |
289 | if (sym_idx) |
290 | *sym_idx = sym_cnt; |
291 | |
292 | return sym; |
293 | } |
294 | |
295 | static int init_output_elf(struct bpf_linker *linker, const char *file) |
296 | { |
297 | int err, str_off; |
298 | Elf64_Sym *init_sym; |
299 | struct dst_sec *sec; |
300 | |
301 | linker->filename = strdup(file); |
302 | if (!linker->filename) |
303 | return -ENOMEM; |
304 | |
305 | linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); |
306 | if (linker->fd < 0) { |
307 | err = -errno; |
308 | pr_warn("failed to create '%s': %d\n" , file, err); |
309 | return err; |
310 | } |
311 | |
312 | linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL); |
313 | if (!linker->elf) { |
314 | pr_warn_elf("failed to create ELF object" ); |
315 | return -EINVAL; |
316 | } |
317 | |
318 | /* ELF header */ |
319 | linker->elf_hdr = elf64_newehdr(linker->elf); |
320 | if (!linker->elf_hdr) { |
321 | pr_warn_elf("failed to create ELF header" ); |
322 | return -EINVAL; |
323 | } |
324 | |
325 | linker->elf_hdr->e_machine = EM_BPF; |
326 | linker->elf_hdr->e_type = ET_REL; |
327 | #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ |
328 | linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB; |
329 | #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ |
330 | linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB; |
331 | #else |
332 | #error "Unknown __BYTE_ORDER__" |
333 | #endif |
334 | |
335 | /* STRTAB */ |
336 | /* initialize strset with an empty string to conform to ELF */ |
337 | linker->strtab_strs = strset__new(INT_MAX, init_data: "" , init_data_sz: sizeof("" )); |
338 | if (libbpf_get_error(ptr: linker->strtab_strs)) |
339 | return libbpf_get_error(ptr: linker->strtab_strs); |
340 | |
341 | sec = add_dst_sec(linker, sec_name: ".strtab" ); |
342 | if (!sec) |
343 | return -ENOMEM; |
344 | |
345 | sec->scn = elf_newscn(linker->elf); |
346 | if (!sec->scn) { |
347 | pr_warn_elf("failed to create STRTAB section" ); |
348 | return -EINVAL; |
349 | } |
350 | |
351 | sec->shdr = elf64_getshdr(sec->scn); |
352 | if (!sec->shdr) |
353 | return -EINVAL; |
354 | |
355 | sec->data = elf_newdata(sec->scn); |
356 | if (!sec->data) { |
357 | pr_warn_elf("failed to create STRTAB data" ); |
358 | return -EINVAL; |
359 | } |
360 | |
361 | str_off = strset__add_str(set: linker->strtab_strs, s: sec->sec_name); |
362 | if (str_off < 0) |
363 | return str_off; |
364 | |
365 | sec->sec_idx = elf_ndxscn(sec->scn); |
366 | linker->elf_hdr->e_shstrndx = sec->sec_idx; |
367 | linker->strtab_sec_idx = sec->sec_idx; |
368 | |
369 | sec->shdr->sh_name = str_off; |
370 | sec->shdr->sh_type = SHT_STRTAB; |
371 | sec->shdr->sh_flags = SHF_STRINGS; |
372 | sec->shdr->sh_offset = 0; |
373 | sec->shdr->sh_link = 0; |
374 | sec->shdr->sh_info = 0; |
375 | sec->shdr->sh_addralign = 1; |
376 | sec->shdr->sh_size = sec->sec_sz = 0; |
377 | sec->shdr->sh_entsize = 0; |
378 | |
379 | /* SYMTAB */ |
380 | sec = add_dst_sec(linker, sec_name: ".symtab" ); |
381 | if (!sec) |
382 | return -ENOMEM; |
383 | |
384 | sec->scn = elf_newscn(linker->elf); |
385 | if (!sec->scn) { |
386 | pr_warn_elf("failed to create SYMTAB section" ); |
387 | return -EINVAL; |
388 | } |
389 | |
390 | sec->shdr = elf64_getshdr(sec->scn); |
391 | if (!sec->shdr) |
392 | return -EINVAL; |
393 | |
394 | sec->data = elf_newdata(sec->scn); |
395 | if (!sec->data) { |
396 | pr_warn_elf("failed to create SYMTAB data" ); |
397 | return -EINVAL; |
398 | } |
399 | |
400 | str_off = strset__add_str(set: linker->strtab_strs, s: sec->sec_name); |
401 | if (str_off < 0) |
402 | return str_off; |
403 | |
404 | sec->sec_idx = elf_ndxscn(sec->scn); |
405 | linker->symtab_sec_idx = sec->sec_idx; |
406 | |
407 | sec->shdr->sh_name = str_off; |
408 | sec->shdr->sh_type = SHT_SYMTAB; |
409 | sec->shdr->sh_flags = 0; |
410 | sec->shdr->sh_offset = 0; |
411 | sec->shdr->sh_link = linker->strtab_sec_idx; |
412 | /* sh_info should be one greater than the index of the last local |
413 | * symbol (i.e., binding is STB_LOCAL). But why and who cares? |
414 | */ |
415 | sec->shdr->sh_info = 0; |
416 | sec->shdr->sh_addralign = 8; |
417 | sec->shdr->sh_entsize = sizeof(Elf64_Sym); |
418 | |
419 | /* .BTF */ |
420 | linker->btf = btf__new_empty(); |
421 | err = libbpf_get_error(ptr: linker->btf); |
422 | if (err) |
423 | return err; |
424 | |
425 | /* add the special all-zero symbol */ |
426 | init_sym = add_new_sym(linker, NULL); |
427 | if (!init_sym) |
428 | return -EINVAL; |
429 | |
430 | init_sym->st_name = 0; |
431 | init_sym->st_info = 0; |
432 | init_sym->st_other = 0; |
433 | init_sym->st_shndx = SHN_UNDEF; |
434 | init_sym->st_value = 0; |
435 | init_sym->st_size = 0; |
436 | |
437 | return 0; |
438 | } |
439 | |
440 | int bpf_linker__add_file(struct bpf_linker *linker, const char *filename, |
441 | const struct bpf_linker_file_opts *opts) |
442 | { |
443 | struct src_obj obj = {}; |
444 | int err = 0; |
445 | |
446 | if (!OPTS_VALID(opts, bpf_linker_file_opts)) |
447 | return libbpf_err(ret: -EINVAL); |
448 | |
449 | if (!linker->elf) |
450 | return libbpf_err(ret: -EINVAL); |
451 | |
452 | err = err ?: linker_load_obj_file(linker, filename, opts, obj: &obj); |
453 | err = err ?: linker_append_sec_data(linker, obj: &obj); |
454 | err = err ?: linker_append_elf_syms(linker, obj: &obj); |
455 | err = err ?: linker_append_elf_relos(linker, obj: &obj); |
456 | err = err ?: linker_append_btf(linker, obj: &obj); |
457 | err = err ?: linker_append_btf_ext(linker, obj: &obj); |
458 | |
459 | /* free up src_obj resources */ |
460 | free(obj.btf_type_map); |
461 | btf__free(btf: obj.btf); |
462 | btf_ext__free(btf_ext: obj.btf_ext); |
463 | free(obj.secs); |
464 | free(obj.sym_map); |
465 | if (obj.elf) |
466 | elf_end(obj.elf); |
467 | if (obj.fd >= 0) |
468 | close(obj.fd); |
469 | |
470 | return libbpf_err(ret: err); |
471 | } |
472 | |
473 | static bool is_dwarf_sec_name(const char *name) |
474 | { |
475 | /* approximation, but the actual list is too long */ |
476 | return strncmp(name, ".debug_" , sizeof(".debug_" ) - 1) == 0; |
477 | } |
478 | |
479 | static bool is_ignored_sec(struct src_sec *sec) |
480 | { |
481 | Elf64_Shdr *shdr = sec->shdr; |
482 | const char *name = sec->sec_name; |
483 | |
484 | /* no special handling of .strtab */ |
485 | if (shdr->sh_type == SHT_STRTAB) |
486 | return true; |
487 | |
488 | /* ignore .llvm_addrsig section as well */ |
489 | if (shdr->sh_type == SHT_LLVM_ADDRSIG) |
490 | return true; |
491 | |
492 | /* no subprograms will lead to an empty .text section, ignore it */ |
493 | if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 && |
494 | strcmp(sec->sec_name, ".text" ) == 0) |
495 | return true; |
496 | |
497 | /* DWARF sections */ |
498 | if (is_dwarf_sec_name(name: sec->sec_name)) |
499 | return true; |
500 | |
501 | if (strncmp(name, ".rel" , sizeof(".rel" ) - 1) == 0) { |
502 | name += sizeof(".rel" ) - 1; |
503 | /* DWARF section relocations */ |
504 | if (is_dwarf_sec_name(name)) |
505 | return true; |
506 | |
507 | /* .BTF and .BTF.ext don't need relocations */ |
508 | if (strcmp(name, BTF_ELF_SEC) == 0 || |
509 | strcmp(name, BTF_EXT_ELF_SEC) == 0) |
510 | return true; |
511 | } |
512 | |
513 | return false; |
514 | } |
515 | |
516 | static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name) |
517 | { |
518 | struct src_sec *secs = obj->secs, *sec; |
519 | size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2; |
520 | |
521 | secs = libbpf_reallocarray(ptr: secs, nmemb: new_cnt, size: sizeof(*secs)); |
522 | if (!secs) |
523 | return NULL; |
524 | |
525 | /* zero out newly allocated memory */ |
526 | memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs)); |
527 | |
528 | obj->secs = secs; |
529 | obj->sec_cnt = new_cnt; |
530 | |
531 | sec = &obj->secs[new_cnt - 1]; |
532 | sec->id = new_cnt - 1; |
533 | sec->sec_name = sec_name; |
534 | |
535 | return sec; |
536 | } |
537 | |
538 | static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, |
539 | const struct bpf_linker_file_opts *opts, |
540 | struct src_obj *obj) |
541 | { |
542 | #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ |
543 | const int host_endianness = ELFDATA2LSB; |
544 | #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ |
545 | const int host_endianness = ELFDATA2MSB; |
546 | #else |
547 | #error "Unknown __BYTE_ORDER__" |
548 | #endif |
549 | int err = 0; |
550 | Elf_Scn *scn; |
551 | Elf_Data *data; |
552 | Elf64_Ehdr *ehdr; |
553 | Elf64_Shdr *shdr; |
554 | struct src_sec *sec; |
555 | |
556 | pr_debug("linker: adding object file '%s'...\n" , filename); |
557 | |
558 | obj->filename = filename; |
559 | |
560 | obj->fd = open(filename, O_RDONLY | O_CLOEXEC); |
561 | if (obj->fd < 0) { |
562 | err = -errno; |
563 | pr_warn("failed to open file '%s': %d\n" , filename, err); |
564 | return err; |
565 | } |
566 | obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL); |
567 | if (!obj->elf) { |
568 | err = -errno; |
569 | pr_warn_elf("failed to parse ELF file '%s'" , filename); |
570 | return err; |
571 | } |
572 | |
573 | /* Sanity check ELF file high-level properties */ |
574 | ehdr = elf64_getehdr(obj->elf); |
575 | if (!ehdr) { |
576 | err = -errno; |
577 | pr_warn_elf("failed to get ELF header for %s" , filename); |
578 | return err; |
579 | } |
580 | if (ehdr->e_ident[EI_DATA] != host_endianness) { |
581 | err = -EOPNOTSUPP; |
582 | pr_warn_elf("unsupported byte order of ELF file %s" , filename); |
583 | return err; |
584 | } |
585 | if (ehdr->e_type != ET_REL |
586 | || ehdr->e_machine != EM_BPF |
587 | || ehdr->e_ident[EI_CLASS] != ELFCLASS64) { |
588 | err = -EOPNOTSUPP; |
589 | pr_warn_elf("unsupported kind of ELF file %s" , filename); |
590 | return err; |
591 | } |
592 | |
593 | if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) { |
594 | err = -errno; |
595 | pr_warn_elf("failed to get SHSTRTAB section index for %s" , filename); |
596 | return err; |
597 | } |
598 | |
599 | scn = NULL; |
600 | while ((scn = elf_nextscn(obj->elf, scn)) != NULL) { |
601 | size_t sec_idx = elf_ndxscn(scn); |
602 | const char *sec_name; |
603 | |
604 | shdr = elf64_getshdr(scn); |
605 | if (!shdr) { |
606 | err = -errno; |
607 | pr_warn_elf("failed to get section #%zu header for %s" , |
608 | sec_idx, filename); |
609 | return err; |
610 | } |
611 | |
612 | sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name); |
613 | if (!sec_name) { |
614 | err = -errno; |
615 | pr_warn_elf("failed to get section #%zu name for %s" , |
616 | sec_idx, filename); |
617 | return err; |
618 | } |
619 | |
620 | data = elf_getdata(scn, 0); |
621 | if (!data) { |
622 | err = -errno; |
623 | pr_warn_elf("failed to get section #%zu (%s) data from %s" , |
624 | sec_idx, sec_name, filename); |
625 | return err; |
626 | } |
627 | |
628 | sec = add_src_sec(obj, sec_name); |
629 | if (!sec) |
630 | return -ENOMEM; |
631 | |
632 | sec->scn = scn; |
633 | sec->shdr = shdr; |
634 | sec->data = data; |
635 | sec->sec_idx = elf_ndxscn(scn); |
636 | |
637 | if (is_ignored_sec(sec)) { |
638 | sec->skipped = true; |
639 | continue; |
640 | } |
641 | |
642 | switch (shdr->sh_type) { |
643 | case SHT_SYMTAB: |
644 | if (obj->symtab_sec_idx) { |
645 | err = -EOPNOTSUPP; |
646 | pr_warn("multiple SYMTAB sections found, not supported\n" ); |
647 | return err; |
648 | } |
649 | obj->symtab_sec_idx = sec_idx; |
650 | break; |
651 | case SHT_STRTAB: |
652 | /* we'll construct our own string table */ |
653 | break; |
654 | case SHT_PROGBITS: |
655 | if (strcmp(sec_name, BTF_ELF_SEC) == 0) { |
656 | obj->btf = btf__new(data->d_buf, shdr->sh_size); |
657 | err = libbpf_get_error(ptr: obj->btf); |
658 | if (err) { |
659 | pr_warn("failed to parse .BTF from %s: %d\n" , filename, err); |
660 | return err; |
661 | } |
662 | sec->skipped = true; |
663 | continue; |
664 | } |
665 | if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) { |
666 | obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size); |
667 | err = libbpf_get_error(ptr: obj->btf_ext); |
668 | if (err) { |
669 | pr_warn("failed to parse .BTF.ext from '%s': %d\n" , filename, err); |
670 | return err; |
671 | } |
672 | sec->skipped = true; |
673 | continue; |
674 | } |
675 | |
676 | /* data & code */ |
677 | break; |
678 | case SHT_NOBITS: |
679 | /* BSS */ |
680 | break; |
681 | case SHT_REL: |
682 | /* relocations */ |
683 | break; |
684 | default: |
685 | pr_warn("unrecognized section #%zu (%s) in %s\n" , |
686 | sec_idx, sec_name, filename); |
687 | err = -EINVAL; |
688 | return err; |
689 | } |
690 | } |
691 | |
692 | err = err ?: linker_sanity_check_elf(obj); |
693 | err = err ?: linker_sanity_check_btf(obj); |
694 | err = err ?: linker_sanity_check_btf_ext(obj); |
695 | err = err ?: linker_fixup_btf(obj); |
696 | |
697 | return err; |
698 | } |
699 | |
700 | static int linker_sanity_check_elf(struct src_obj *obj) |
701 | { |
702 | struct src_sec *sec; |
703 | int i, err; |
704 | |
705 | if (!obj->symtab_sec_idx) { |
706 | pr_warn("ELF is missing SYMTAB section in %s\n" , obj->filename); |
707 | return -EINVAL; |
708 | } |
709 | if (!obj->shstrs_sec_idx) { |
710 | pr_warn("ELF is missing section headers STRTAB section in %s\n" , obj->filename); |
711 | return -EINVAL; |
712 | } |
713 | |
714 | for (i = 1; i < obj->sec_cnt; i++) { |
715 | sec = &obj->secs[i]; |
716 | |
717 | if (sec->sec_name[0] == '\0') { |
718 | pr_warn("ELF section #%zu has empty name in %s\n" , sec->sec_idx, obj->filename); |
719 | return -EINVAL; |
720 | } |
721 | |
722 | if (is_dwarf_sec_name(name: sec->sec_name)) |
723 | continue; |
724 | |
725 | if (sec->shdr->sh_addralign && !is_pow_of_2(x: sec->shdr->sh_addralign)) { |
726 | pr_warn("ELF section #%zu alignment %llu is non pow-of-2 alignment in %s\n" , |
727 | sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign, |
728 | obj->filename); |
729 | return -EINVAL; |
730 | } |
731 | if (sec->shdr->sh_addralign != sec->data->d_align) { |
732 | pr_warn("ELF section #%zu has inconsistent alignment addr=%llu != d=%llu in %s\n" , |
733 | sec->sec_idx, (long long unsigned)sec->shdr->sh_addralign, |
734 | (long long unsigned)sec->data->d_align, obj->filename); |
735 | return -EINVAL; |
736 | } |
737 | |
738 | if (sec->shdr->sh_size != sec->data->d_size) { |
739 | pr_warn("ELF section #%zu has inconsistent section size sh=%llu != d=%llu in %s\n" , |
740 | sec->sec_idx, (long long unsigned)sec->shdr->sh_size, |
741 | (long long unsigned)sec->data->d_size, obj->filename); |
742 | return -EINVAL; |
743 | } |
744 | |
745 | switch (sec->shdr->sh_type) { |
746 | case SHT_SYMTAB: |
747 | err = linker_sanity_check_elf_symtab(obj, sec); |
748 | if (err) |
749 | return err; |
750 | break; |
751 | case SHT_STRTAB: |
752 | break; |
753 | case SHT_PROGBITS: |
754 | if (sec->shdr->sh_flags & SHF_EXECINSTR) { |
755 | if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) { |
756 | pr_warn("ELF section #%zu has unexpected size alignment %llu in %s\n" , |
757 | sec->sec_idx, (long long unsigned)sec->shdr->sh_size, |
758 | obj->filename); |
759 | return -EINVAL; |
760 | } |
761 | } |
762 | break; |
763 | case SHT_NOBITS: |
764 | break; |
765 | case SHT_REL: |
766 | err = linker_sanity_check_elf_relos(obj, sec); |
767 | if (err) |
768 | return err; |
769 | break; |
770 | case SHT_LLVM_ADDRSIG: |
771 | break; |
772 | default: |
773 | pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n" , |
774 | sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename); |
775 | return -EINVAL; |
776 | } |
777 | } |
778 | |
779 | return 0; |
780 | } |
781 | |
782 | static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec) |
783 | { |
784 | struct src_sec *link_sec; |
785 | Elf64_Sym *sym; |
786 | int i, n; |
787 | |
788 | if (sec->shdr->sh_entsize != sizeof(Elf64_Sym)) |
789 | return -EINVAL; |
790 | if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) |
791 | return -EINVAL; |
792 | |
793 | if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) { |
794 | pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n" , |
795 | sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); |
796 | return -EINVAL; |
797 | } |
798 | link_sec = &obj->secs[sec->shdr->sh_link]; |
799 | if (link_sec->shdr->sh_type != SHT_STRTAB) { |
800 | pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n" , |
801 | sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); |
802 | return -EINVAL; |
803 | } |
804 | |
805 | n = sec->shdr->sh_size / sec->shdr->sh_entsize; |
806 | sym = sec->data->d_buf; |
807 | for (i = 0; i < n; i++, sym++) { |
808 | int sym_type = ELF64_ST_TYPE(sym->st_info); |
809 | int sym_bind = ELF64_ST_BIND(sym->st_info); |
810 | int sym_vis = ELF64_ST_VISIBILITY(sym->st_other); |
811 | |
812 | if (i == 0) { |
813 | if (sym->st_name != 0 || sym->st_info != 0 |
814 | || sym->st_other != 0 || sym->st_shndx != 0 |
815 | || sym->st_value != 0 || sym->st_size != 0) { |
816 | pr_warn("ELF sym #0 is invalid in %s\n" , obj->filename); |
817 | return -EINVAL; |
818 | } |
819 | continue; |
820 | } |
821 | if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) { |
822 | pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n" , |
823 | i, sec->sec_idx, sym_bind); |
824 | return -EINVAL; |
825 | } |
826 | if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) { |
827 | pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n" , |
828 | i, sec->sec_idx, sym_vis); |
829 | return -EINVAL; |
830 | } |
831 | if (sym->st_shndx == 0) { |
832 | if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL |
833 | || sym->st_value != 0 || sym->st_size != 0) { |
834 | pr_warn("ELF sym #%d is invalid extern symbol in %s\n" , |
835 | i, obj->filename); |
836 | |
837 | return -EINVAL; |
838 | } |
839 | continue; |
840 | } |
841 | if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) { |
842 | pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n" , |
843 | i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename); |
844 | return -EINVAL; |
845 | } |
846 | if (sym_type == STT_SECTION) { |
847 | if (sym->st_value != 0) |
848 | return -EINVAL; |
849 | continue; |
850 | } |
851 | } |
852 | |
853 | return 0; |
854 | } |
855 | |
856 | static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec) |
857 | { |
858 | struct src_sec *link_sec, *sym_sec; |
859 | Elf64_Rel *relo; |
860 | int i, n; |
861 | |
862 | if (sec->shdr->sh_entsize != sizeof(Elf64_Rel)) |
863 | return -EINVAL; |
864 | if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) |
865 | return -EINVAL; |
866 | |
867 | /* SHT_REL's sh_link should point to SYMTAB */ |
868 | if (sec->shdr->sh_link != obj->symtab_sec_idx) { |
869 | pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n" , |
870 | sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); |
871 | return -EINVAL; |
872 | } |
873 | |
874 | /* SHT_REL's sh_info points to relocated section */ |
875 | if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) { |
876 | pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n" , |
877 | sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); |
878 | return -EINVAL; |
879 | } |
880 | link_sec = &obj->secs[sec->shdr->sh_info]; |
881 | |
882 | /* .rel<secname> -> <secname> pattern is followed */ |
883 | if (strncmp(sec->sec_name, ".rel" , sizeof(".rel" ) - 1) != 0 |
884 | || strcmp(sec->sec_name + sizeof(".rel" ) - 1, link_sec->sec_name) != 0) { |
885 | pr_warn("ELF relo section #%zu name has invalid name in %s\n" , |
886 | sec->sec_idx, obj->filename); |
887 | return -EINVAL; |
888 | } |
889 | |
890 | /* don't further validate relocations for ignored sections */ |
891 | if (link_sec->skipped) |
892 | return 0; |
893 | |
894 | /* relocatable section is data or instructions */ |
895 | if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) { |
896 | pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n" , |
897 | sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); |
898 | return -EINVAL; |
899 | } |
900 | |
901 | /* check sanity of each relocation */ |
902 | n = sec->shdr->sh_size / sec->shdr->sh_entsize; |
903 | relo = sec->data->d_buf; |
904 | sym_sec = &obj->secs[obj->symtab_sec_idx]; |
905 | for (i = 0; i < n; i++, relo++) { |
906 | size_t sym_idx = ELF64_R_SYM(relo->r_info); |
907 | size_t sym_type = ELF64_R_TYPE(relo->r_info); |
908 | |
909 | if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 && |
910 | sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) { |
911 | pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n" , |
912 | i, sec->sec_idx, sym_type, obj->filename); |
913 | return -EINVAL; |
914 | } |
915 | |
916 | if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) { |
917 | pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n" , |
918 | i, sec->sec_idx, sym_idx, obj->filename); |
919 | return -EINVAL; |
920 | } |
921 | |
922 | if (link_sec->shdr->sh_flags & SHF_EXECINSTR) { |
923 | if (relo->r_offset % sizeof(struct bpf_insn) != 0) { |
924 | pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n" , |
925 | i, sec->sec_idx, sym_idx, obj->filename); |
926 | return -EINVAL; |
927 | } |
928 | } |
929 | } |
930 | |
931 | return 0; |
932 | } |
933 | |
934 | static int check_btf_type_id(__u32 *type_id, void *ctx) |
935 | { |
936 | struct btf *btf = ctx; |
937 | |
938 | if (*type_id >= btf__type_cnt(btf)) |
939 | return -EINVAL; |
940 | |
941 | return 0; |
942 | } |
943 | |
944 | static int check_btf_str_off(__u32 *str_off, void *ctx) |
945 | { |
946 | struct btf *btf = ctx; |
947 | const char *s; |
948 | |
949 | s = btf__str_by_offset(btf, offset: *str_off); |
950 | |
951 | if (!s) |
952 | return -EINVAL; |
953 | |
954 | return 0; |
955 | } |
956 | |
957 | static int linker_sanity_check_btf(struct src_obj *obj) |
958 | { |
959 | struct btf_type *t; |
960 | int i, n, err = 0; |
961 | |
962 | if (!obj->btf) |
963 | return 0; |
964 | |
965 | n = btf__type_cnt(btf: obj->btf); |
966 | for (i = 1; i < n; i++) { |
967 | t = btf_type_by_id(btf: obj->btf, type_id: i); |
968 | |
969 | err = err ?: btf_type_visit_type_ids(t, visit: check_btf_type_id, ctx: obj->btf); |
970 | err = err ?: btf_type_visit_str_offs(t, visit: check_btf_str_off, ctx: obj->btf); |
971 | if (err) |
972 | return err; |
973 | } |
974 | |
975 | return 0; |
976 | } |
977 | |
978 | static int linker_sanity_check_btf_ext(struct src_obj *obj) |
979 | { |
980 | int err = 0; |
981 | |
982 | if (!obj->btf_ext) |
983 | return 0; |
984 | |
985 | /* can't use .BTF.ext without .BTF */ |
986 | if (!obj->btf) |
987 | return -EINVAL; |
988 | |
989 | err = err ?: btf_ext_visit_type_ids(btf_ext: obj->btf_ext, visit: check_btf_type_id, ctx: obj->btf); |
990 | err = err ?: btf_ext_visit_str_offs(btf_ext: obj->btf_ext, visit: check_btf_str_off, ctx: obj->btf); |
991 | if (err) |
992 | return err; |
993 | |
994 | return 0; |
995 | } |
996 | |
997 | static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec) |
998 | { |
999 | Elf_Scn *scn; |
1000 | Elf_Data *data; |
1001 | Elf64_Shdr *shdr; |
1002 | int name_off; |
1003 | |
1004 | dst_sec->sec_sz = 0; |
1005 | dst_sec->sec_idx = 0; |
1006 | dst_sec->ephemeral = src_sec->ephemeral; |
1007 | |
1008 | /* ephemeral sections are just thin section shells lacking most parts */ |
1009 | if (src_sec->ephemeral) |
1010 | return 0; |
1011 | |
1012 | scn = elf_newscn(linker->elf); |
1013 | if (!scn) |
1014 | return -ENOMEM; |
1015 | data = elf_newdata(scn); |
1016 | if (!data) |
1017 | return -ENOMEM; |
1018 | shdr = elf64_getshdr(scn); |
1019 | if (!shdr) |
1020 | return -ENOMEM; |
1021 | |
1022 | dst_sec->scn = scn; |
1023 | dst_sec->shdr = shdr; |
1024 | dst_sec->data = data; |
1025 | dst_sec->sec_idx = elf_ndxscn(scn); |
1026 | |
1027 | name_off = strset__add_str(set: linker->strtab_strs, s: src_sec->sec_name); |
1028 | if (name_off < 0) |
1029 | return name_off; |
1030 | |
1031 | shdr->sh_name = name_off; |
1032 | shdr->sh_type = src_sec->shdr->sh_type; |
1033 | shdr->sh_flags = src_sec->shdr->sh_flags; |
1034 | shdr->sh_size = 0; |
1035 | /* sh_link and sh_info have different meaning for different types of |
1036 | * sections, so we leave it up to the caller code to fill them in, if |
1037 | * necessary |
1038 | */ |
1039 | shdr->sh_link = 0; |
1040 | shdr->sh_info = 0; |
1041 | shdr->sh_addralign = src_sec->shdr->sh_addralign; |
1042 | shdr->sh_entsize = src_sec->shdr->sh_entsize; |
1043 | |
1044 | data->d_type = src_sec->data->d_type; |
1045 | data->d_size = 0; |
1046 | data->d_buf = NULL; |
1047 | data->d_align = src_sec->data->d_align; |
1048 | data->d_off = 0; |
1049 | |
1050 | return 0; |
1051 | } |
1052 | |
1053 | static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name) |
1054 | { |
1055 | struct dst_sec *sec; |
1056 | int i; |
1057 | |
1058 | for (i = 1; i < linker->sec_cnt; i++) { |
1059 | sec = &linker->secs[i]; |
1060 | |
1061 | if (strcmp(sec->sec_name, sec_name) == 0) |
1062 | return sec; |
1063 | } |
1064 | |
1065 | return NULL; |
1066 | } |
1067 | |
1068 | static bool secs_match(struct dst_sec *dst, struct src_sec *src) |
1069 | { |
1070 | if (dst->ephemeral || src->ephemeral) |
1071 | return true; |
1072 | |
1073 | if (dst->shdr->sh_type != src->shdr->sh_type) { |
1074 | pr_warn("sec %s types mismatch\n" , dst->sec_name); |
1075 | return false; |
1076 | } |
1077 | if (dst->shdr->sh_flags != src->shdr->sh_flags) { |
1078 | pr_warn("sec %s flags mismatch\n" , dst->sec_name); |
1079 | return false; |
1080 | } |
1081 | if (dst->shdr->sh_entsize != src->shdr->sh_entsize) { |
1082 | pr_warn("sec %s entsize mismatch\n" , dst->sec_name); |
1083 | return false; |
1084 | } |
1085 | |
1086 | return true; |
1087 | } |
1088 | |
1089 | static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec) |
1090 | { |
1091 | if (dst_sec->sec_sz != src_sec->shdr->sh_size) |
1092 | return false; |
1093 | if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0) |
1094 | return false; |
1095 | return true; |
1096 | } |
1097 | |
1098 | static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src) |
1099 | { |
1100 | void *tmp; |
1101 | size_t dst_align, src_align; |
1102 | size_t dst_align_sz, dst_final_sz; |
1103 | int err; |
1104 | |
1105 | /* Ephemeral source section doesn't contribute anything to ELF |
1106 | * section data. |
1107 | */ |
1108 | if (src->ephemeral) |
1109 | return 0; |
1110 | |
1111 | /* Some sections (like .maps) can contain both externs (and thus be |
1112 | * ephemeral) and non-externs (map definitions). So it's possible that |
1113 | * it has to be "upgraded" from ephemeral to non-ephemeral when the |
1114 | * first non-ephemeral entity appears. In such case, we add ELF |
1115 | * section, data, etc. |
1116 | */ |
1117 | if (dst->ephemeral) { |
1118 | err = init_sec(linker, dst_sec: dst, src_sec: src); |
1119 | if (err) |
1120 | return err; |
1121 | } |
1122 | |
1123 | dst_align = dst->shdr->sh_addralign; |
1124 | src_align = src->shdr->sh_addralign; |
1125 | if (dst_align == 0) |
1126 | dst_align = 1; |
1127 | if (dst_align < src_align) |
1128 | dst_align = src_align; |
1129 | |
1130 | dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align; |
1131 | |
1132 | /* no need to re-align final size */ |
1133 | dst_final_sz = dst_align_sz + src->shdr->sh_size; |
1134 | |
1135 | if (src->shdr->sh_type != SHT_NOBITS) { |
1136 | tmp = realloc(dst->raw_data, dst_final_sz); |
1137 | /* If dst_align_sz == 0, realloc() behaves in a special way: |
1138 | * 1. When dst->raw_data is NULL it returns: |
1139 | * "either NULL or a pointer suitable to be passed to free()" [1]. |
1140 | * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL, |
1141 | * thus invalidating any "pointer suitable to be passed to free()" obtained |
1142 | * at step (1). |
1143 | * |
1144 | * The dst_align_sz > 0 check avoids error exit after (2), otherwise |
1145 | * dst->raw_data would be freed again in bpf_linker__free(). |
1146 | * |
1147 | * [1] man 3 realloc |
1148 | */ |
1149 | if (!tmp && dst_align_sz > 0) |
1150 | return -ENOMEM; |
1151 | dst->raw_data = tmp; |
1152 | |
1153 | /* pad dst section, if it's alignment forced size increase */ |
1154 | memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz); |
1155 | /* now copy src data at a properly aligned offset */ |
1156 | memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size); |
1157 | } |
1158 | |
1159 | dst->sec_sz = dst_final_sz; |
1160 | dst->shdr->sh_size = dst_final_sz; |
1161 | dst->data->d_size = dst_final_sz; |
1162 | |
1163 | dst->shdr->sh_addralign = dst_align; |
1164 | dst->data->d_align = dst_align; |
1165 | |
1166 | src->dst_off = dst_align_sz; |
1167 | |
1168 | return 0; |
1169 | } |
1170 | |
1171 | static bool is_data_sec(struct src_sec *sec) |
1172 | { |
1173 | if (!sec || sec->skipped) |
1174 | return false; |
1175 | /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */ |
1176 | if (sec->ephemeral) |
1177 | return true; |
1178 | return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS; |
1179 | } |
1180 | |
1181 | static bool is_relo_sec(struct src_sec *sec) |
1182 | { |
1183 | if (!sec || sec->skipped || sec->ephemeral) |
1184 | return false; |
1185 | return sec->shdr->sh_type == SHT_REL; |
1186 | } |
1187 | |
1188 | static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj) |
1189 | { |
1190 | int i, err; |
1191 | |
1192 | for (i = 1; i < obj->sec_cnt; i++) { |
1193 | struct src_sec *src_sec; |
1194 | struct dst_sec *dst_sec; |
1195 | |
1196 | src_sec = &obj->secs[i]; |
1197 | if (!is_data_sec(sec: src_sec)) |
1198 | continue; |
1199 | |
1200 | dst_sec = find_dst_sec_by_name(linker, sec_name: src_sec->sec_name); |
1201 | if (!dst_sec) { |
1202 | dst_sec = add_dst_sec(linker, sec_name: src_sec->sec_name); |
1203 | if (!dst_sec) |
1204 | return -ENOMEM; |
1205 | err = init_sec(linker, dst_sec, src_sec); |
1206 | if (err) { |
1207 | pr_warn("failed to init section '%s'\n" , src_sec->sec_name); |
1208 | return err; |
1209 | } |
1210 | } else { |
1211 | if (!secs_match(dst: dst_sec, src: src_sec)) { |
1212 | pr_warn("ELF sections %s are incompatible\n" , src_sec->sec_name); |
1213 | return -1; |
1214 | } |
1215 | |
1216 | /* "license" and "version" sections are deduped */ |
1217 | if (strcmp(src_sec->sec_name, "license" ) == 0 |
1218 | || strcmp(src_sec->sec_name, "version" ) == 0) { |
1219 | if (!sec_content_is_same(dst_sec, src_sec)) { |
1220 | pr_warn("non-identical contents of section '%s' are not supported\n" , src_sec->sec_name); |
1221 | return -EINVAL; |
1222 | } |
1223 | src_sec->skipped = true; |
1224 | src_sec->dst_id = dst_sec->id; |
1225 | continue; |
1226 | } |
1227 | } |
1228 | |
1229 | /* record mapped section index */ |
1230 | src_sec->dst_id = dst_sec->id; |
1231 | |
1232 | err = extend_sec(linker, dst: dst_sec, src: src_sec); |
1233 | if (err) |
1234 | return err; |
1235 | } |
1236 | |
1237 | return 0; |
1238 | } |
1239 | |
1240 | static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj) |
1241 | { |
1242 | struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; |
1243 | Elf64_Sym *sym = symtab->data->d_buf; |
1244 | int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err; |
1245 | int str_sec_idx = symtab->shdr->sh_link; |
1246 | const char *sym_name; |
1247 | |
1248 | obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map)); |
1249 | if (!obj->sym_map) |
1250 | return -ENOMEM; |
1251 | |
1252 | for (i = 0; i < n; i++, sym++) { |
1253 | /* We already validated all-zero symbol #0 and we already |
1254 | * appended it preventively to the final SYMTAB, so skip it. |
1255 | */ |
1256 | if (i == 0) |
1257 | continue; |
1258 | |
1259 | sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); |
1260 | if (!sym_name) { |
1261 | pr_warn("can't fetch symbol name for symbol #%d in '%s'\n" , i, obj->filename); |
1262 | return -EINVAL; |
1263 | } |
1264 | |
1265 | err = linker_append_elf_sym(linker, obj, sym, sym_name, src_sym_idx: i); |
1266 | if (err) |
1267 | return err; |
1268 | } |
1269 | |
1270 | return 0; |
1271 | } |
1272 | |
1273 | static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx) |
1274 | { |
1275 | struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; |
1276 | Elf64_Sym *syms = symtab->raw_data; |
1277 | |
1278 | return &syms[sym_idx]; |
1279 | } |
1280 | |
1281 | static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name) |
1282 | { |
1283 | struct glob_sym *glob_sym; |
1284 | const char *name; |
1285 | int i; |
1286 | |
1287 | for (i = 0; i < linker->glob_sym_cnt; i++) { |
1288 | glob_sym = &linker->glob_syms[i]; |
1289 | name = strset__data(set: linker->strtab_strs) + glob_sym->name_off; |
1290 | |
1291 | if (strcmp(name, sym_name) == 0) |
1292 | return glob_sym; |
1293 | } |
1294 | |
1295 | return NULL; |
1296 | } |
1297 | |
1298 | static struct glob_sym *add_glob_sym(struct bpf_linker *linker) |
1299 | { |
1300 | struct glob_sym *syms, *sym; |
1301 | |
1302 | syms = libbpf_reallocarray(ptr: linker->glob_syms, nmemb: linker->glob_sym_cnt + 1, |
1303 | size: sizeof(*linker->glob_syms)); |
1304 | if (!syms) |
1305 | return NULL; |
1306 | |
1307 | sym = &syms[linker->glob_sym_cnt]; |
1308 | memset(sym, 0, sizeof(*sym)); |
1309 | sym->var_idx = -1; |
1310 | |
1311 | linker->glob_syms = syms; |
1312 | linker->glob_sym_cnt++; |
1313 | |
1314 | return sym; |
1315 | } |
1316 | |
1317 | static bool glob_sym_btf_matches(const char *sym_name, bool exact, |
1318 | const struct btf *btf1, __u32 id1, |
1319 | const struct btf *btf2, __u32 id2) |
1320 | { |
1321 | const struct btf_type *t1, *t2; |
1322 | bool is_static1, is_static2; |
1323 | const char *n1, *n2; |
1324 | int i, n; |
1325 | |
1326 | recur: |
1327 | n1 = n2 = NULL; |
1328 | t1 = skip_mods_and_typedefs(btf: btf1, id: id1, res_id: &id1); |
1329 | t2 = skip_mods_and_typedefs(btf: btf2, id: id2, res_id: &id2); |
1330 | |
1331 | /* check if only one side is FWD, otherwise handle with common logic */ |
1332 | if (!exact && btf_is_fwd(t: t1) != btf_is_fwd(t: t2)) { |
1333 | n1 = btf__str_by_offset(btf: btf1, offset: t1->name_off); |
1334 | n2 = btf__str_by_offset(btf: btf2, offset: t2->name_off); |
1335 | if (strcmp(n1, n2) != 0) { |
1336 | pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n" , |
1337 | sym_name, n1, n2); |
1338 | return false; |
1339 | } |
1340 | /* validate if FWD kind matches concrete kind */ |
1341 | if (btf_is_fwd(t: t1)) { |
1342 | if (btf_kflag(t: t1) && btf_is_union(t: t2)) |
1343 | return true; |
1344 | if (!btf_kflag(t: t1) && btf_is_struct(t: t2)) |
1345 | return true; |
1346 | pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n" , |
1347 | sym_name, btf_kflag(t1) ? "union" : "struct" , btf_kind_str(t2)); |
1348 | } else { |
1349 | if (btf_kflag(t: t2) && btf_is_union(t: t1)) |
1350 | return true; |
1351 | if (!btf_kflag(t: t2) && btf_is_struct(t: t1)) |
1352 | return true; |
1353 | pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n" , |
1354 | sym_name, btf_kflag(t2) ? "union" : "struct" , btf_kind_str(t1)); |
1355 | } |
1356 | return false; |
1357 | } |
1358 | |
1359 | if (btf_kind(t1) != btf_kind(t2)) { |
1360 | pr_warn("global '%s': incompatible BTF kinds %s and %s\n" , |
1361 | sym_name, btf_kind_str(t1), btf_kind_str(t2)); |
1362 | return false; |
1363 | } |
1364 | |
1365 | switch (btf_kind(t1)) { |
1366 | case BTF_KIND_STRUCT: |
1367 | case BTF_KIND_UNION: |
1368 | case BTF_KIND_ENUM: |
1369 | case BTF_KIND_ENUM64: |
1370 | case BTF_KIND_FWD: |
1371 | case BTF_KIND_FUNC: |
1372 | case BTF_KIND_VAR: |
1373 | n1 = btf__str_by_offset(btf: btf1, offset: t1->name_off); |
1374 | n2 = btf__str_by_offset(btf: btf2, offset: t2->name_off); |
1375 | if (strcmp(n1, n2) != 0) { |
1376 | pr_warn("global '%s': incompatible %s names '%s' and '%s'\n" , |
1377 | sym_name, btf_kind_str(t1), n1, n2); |
1378 | return false; |
1379 | } |
1380 | break; |
1381 | default: |
1382 | break; |
1383 | } |
1384 | |
1385 | switch (btf_kind(t1)) { |
1386 | case BTF_KIND_UNKN: /* void */ |
1387 | case BTF_KIND_FWD: |
1388 | return true; |
1389 | case BTF_KIND_INT: |
1390 | case BTF_KIND_FLOAT: |
1391 | case BTF_KIND_ENUM: |
1392 | case BTF_KIND_ENUM64: |
1393 | /* ignore encoding for int and enum values for enum */ |
1394 | if (t1->size != t2->size) { |
1395 | pr_warn("global '%s': incompatible %s '%s' size %u and %u\n" , |
1396 | sym_name, btf_kind_str(t1), n1, t1->size, t2->size); |
1397 | return false; |
1398 | } |
1399 | return true; |
1400 | case BTF_KIND_PTR: |
1401 | /* just validate overall shape of the referenced type, so no |
1402 | * contents comparison for struct/union, and allowd fwd vs |
1403 | * struct/union |
1404 | */ |
1405 | exact = false; |
1406 | id1 = t1->type; |
1407 | id2 = t2->type; |
1408 | goto recur; |
1409 | case BTF_KIND_ARRAY: |
1410 | /* ignore index type and array size */ |
1411 | id1 = btf_array(t1)->type; |
1412 | id2 = btf_array(t2)->type; |
1413 | goto recur; |
1414 | case BTF_KIND_FUNC: |
1415 | /* extern and global linkages are compatible */ |
1416 | is_static1 = btf_func_linkage(t: t1) == BTF_FUNC_STATIC; |
1417 | is_static2 = btf_func_linkage(t: t2) == BTF_FUNC_STATIC; |
1418 | if (is_static1 != is_static2) { |
1419 | pr_warn("global '%s': incompatible func '%s' linkage\n" , sym_name, n1); |
1420 | return false; |
1421 | } |
1422 | |
1423 | id1 = t1->type; |
1424 | id2 = t2->type; |
1425 | goto recur; |
1426 | case BTF_KIND_VAR: |
1427 | /* extern and global linkages are compatible */ |
1428 | is_static1 = btf_var(t: t1)->linkage == BTF_VAR_STATIC; |
1429 | is_static2 = btf_var(t: t2)->linkage == BTF_VAR_STATIC; |
1430 | if (is_static1 != is_static2) { |
1431 | pr_warn("global '%s': incompatible var '%s' linkage\n" , sym_name, n1); |
1432 | return false; |
1433 | } |
1434 | |
1435 | id1 = t1->type; |
1436 | id2 = t2->type; |
1437 | goto recur; |
1438 | case BTF_KIND_STRUCT: |
1439 | case BTF_KIND_UNION: { |
1440 | const struct btf_member *m1, *m2; |
1441 | |
1442 | if (!exact) |
1443 | return true; |
1444 | |
1445 | if (btf_vlen(t1) != btf_vlen(t2)) { |
1446 | pr_warn("global '%s': incompatible number of %s fields %u and %u\n" , |
1447 | sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2)); |
1448 | return false; |
1449 | } |
1450 | |
1451 | n = btf_vlen(t1); |
1452 | m1 = btf_members(t1); |
1453 | m2 = btf_members(t2); |
1454 | for (i = 0; i < n; i++, m1++, m2++) { |
1455 | n1 = btf__str_by_offset(btf: btf1, offset: m1->name_off); |
1456 | n2 = btf__str_by_offset(btf: btf2, offset: m2->name_off); |
1457 | if (strcmp(n1, n2) != 0) { |
1458 | pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n" , |
1459 | sym_name, i, n1, n2); |
1460 | return false; |
1461 | } |
1462 | if (m1->offset != m2->offset) { |
1463 | pr_warn("global '%s': incompatible field #%d ('%s') offsets\n" , |
1464 | sym_name, i, n1); |
1465 | return false; |
1466 | } |
1467 | if (!glob_sym_btf_matches(sym_name, exact, btf1, id1: m1->type, btf2, id2: m2->type)) |
1468 | return false; |
1469 | } |
1470 | |
1471 | return true; |
1472 | } |
1473 | case BTF_KIND_FUNC_PROTO: { |
1474 | const struct btf_param *m1, *m2; |
1475 | |
1476 | if (btf_vlen(t1) != btf_vlen(t2)) { |
1477 | pr_warn("global '%s': incompatible number of %s params %u and %u\n" , |
1478 | sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2)); |
1479 | return false; |
1480 | } |
1481 | |
1482 | n = btf_vlen(t1); |
1483 | m1 = btf_params(t1); |
1484 | m2 = btf_params(t2); |
1485 | for (i = 0; i < n; i++, m1++, m2++) { |
1486 | /* ignore func arg names */ |
1487 | if (!glob_sym_btf_matches(sym_name, exact, btf1, id1: m1->type, btf2, id2: m2->type)) |
1488 | return false; |
1489 | } |
1490 | |
1491 | /* now check return type as well */ |
1492 | id1 = t1->type; |
1493 | id2 = t2->type; |
1494 | goto recur; |
1495 | } |
1496 | |
1497 | /* skip_mods_and_typedefs() make this impossible */ |
1498 | case BTF_KIND_TYPEDEF: |
1499 | case BTF_KIND_VOLATILE: |
1500 | case BTF_KIND_CONST: |
1501 | case BTF_KIND_RESTRICT: |
1502 | /* DATASECs are never compared with each other */ |
1503 | case BTF_KIND_DATASEC: |
1504 | default: |
1505 | pr_warn("global '%s': unsupported BTF kind %s\n" , |
1506 | sym_name, btf_kind_str(t1)); |
1507 | return false; |
1508 | } |
1509 | } |
1510 | |
1511 | static bool map_defs_match(const char *sym_name, |
1512 | const struct btf *main_btf, |
1513 | const struct btf_map_def *main_def, |
1514 | const struct btf_map_def *main_inner_def, |
1515 | const struct btf *, |
1516 | const struct btf_map_def *, |
1517 | const struct btf_map_def *) |
1518 | { |
1519 | const char *reason; |
1520 | |
1521 | if (main_def->map_type != extra_def->map_type) { |
1522 | reason = "type" ; |
1523 | goto mismatch; |
1524 | } |
1525 | |
1526 | /* check key type/size match */ |
1527 | if (main_def->key_size != extra_def->key_size) { |
1528 | reason = "key_size" ; |
1529 | goto mismatch; |
1530 | } |
1531 | if (!!main_def->key_type_id != !!extra_def->key_type_id) { |
1532 | reason = "key type" ; |
1533 | goto mismatch; |
1534 | } |
1535 | if ((main_def->parts & MAP_DEF_KEY_TYPE) |
1536 | && !glob_sym_btf_matches(sym_name, exact: true /*exact*/, |
1537 | btf1: main_btf, id1: main_def->key_type_id, |
1538 | btf2: extra_btf, id2: extra_def->key_type_id)) { |
1539 | reason = "key type" ; |
1540 | goto mismatch; |
1541 | } |
1542 | |
1543 | /* validate value type/size match */ |
1544 | if (main_def->value_size != extra_def->value_size) { |
1545 | reason = "value_size" ; |
1546 | goto mismatch; |
1547 | } |
1548 | if (!!main_def->value_type_id != !!extra_def->value_type_id) { |
1549 | reason = "value type" ; |
1550 | goto mismatch; |
1551 | } |
1552 | if ((main_def->parts & MAP_DEF_VALUE_TYPE) |
1553 | && !glob_sym_btf_matches(sym_name, exact: true /*exact*/, |
1554 | btf1: main_btf, id1: main_def->value_type_id, |
1555 | btf2: extra_btf, id2: extra_def->value_type_id)) { |
1556 | reason = "key type" ; |
1557 | goto mismatch; |
1558 | } |
1559 | |
1560 | if (main_def->max_entries != extra_def->max_entries) { |
1561 | reason = "max_entries" ; |
1562 | goto mismatch; |
1563 | } |
1564 | if (main_def->map_flags != extra_def->map_flags) { |
1565 | reason = "map_flags" ; |
1566 | goto mismatch; |
1567 | } |
1568 | if (main_def->numa_node != extra_def->numa_node) { |
1569 | reason = "numa_node" ; |
1570 | goto mismatch; |
1571 | } |
1572 | if (main_def->pinning != extra_def->pinning) { |
1573 | reason = "pinning" ; |
1574 | goto mismatch; |
1575 | } |
1576 | |
1577 | if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) { |
1578 | reason = "inner map" ; |
1579 | goto mismatch; |
1580 | } |
1581 | |
1582 | if (main_def->parts & MAP_DEF_INNER_MAP) { |
1583 | char inner_map_name[128]; |
1584 | |
1585 | snprintf(buf: inner_map_name, size: sizeof(inner_map_name), fmt: "%s.inner" , sym_name); |
1586 | |
1587 | return map_defs_match(sym_name: inner_map_name, |
1588 | main_btf, main_def: main_inner_def, NULL, |
1589 | extra_btf, extra_def: extra_inner_def, NULL); |
1590 | } |
1591 | |
1592 | return true; |
1593 | |
1594 | mismatch: |
1595 | pr_warn("global '%s': map %s mismatch\n" , sym_name, reason); |
1596 | return false; |
1597 | } |
1598 | |
1599 | static bool glob_map_defs_match(const char *sym_name, |
1600 | struct bpf_linker *linker, struct glob_sym *glob_sym, |
1601 | struct src_obj *obj, Elf64_Sym *sym, int btf_id) |
1602 | { |
1603 | struct btf_map_def dst_def = {}, dst_inner_def = {}; |
1604 | struct btf_map_def src_def = {}, src_inner_def = {}; |
1605 | const struct btf_type *t; |
1606 | int err; |
1607 | |
1608 | t = btf__type_by_id(btf: obj->btf, id: btf_id); |
1609 | if (!btf_is_var(t)) { |
1610 | pr_warn("global '%s': invalid map definition type [%d]\n" , sym_name, btf_id); |
1611 | return false; |
1612 | } |
1613 | t = skip_mods_and_typedefs(btf: obj->btf, id: t->type, NULL); |
1614 | |
1615 | err = parse_btf_map_def(map_name: sym_name, btf: obj->btf, def_t: t, strict: true /*strict*/, map_def: &src_def, inner_def: &src_inner_def); |
1616 | if (err) { |
1617 | pr_warn("global '%s': invalid map definition\n" , sym_name); |
1618 | return false; |
1619 | } |
1620 | |
1621 | /* re-parse existing map definition */ |
1622 | t = btf__type_by_id(btf: linker->btf, id: glob_sym->btf_id); |
1623 | t = skip_mods_and_typedefs(btf: linker->btf, id: t->type, NULL); |
1624 | err = parse_btf_map_def(map_name: sym_name, btf: linker->btf, def_t: t, strict: true /*strict*/, map_def: &dst_def, inner_def: &dst_inner_def); |
1625 | if (err) { |
1626 | /* this should not happen, because we already validated it */ |
1627 | pr_warn("global '%s': invalid dst map definition\n" , sym_name); |
1628 | return false; |
1629 | } |
1630 | |
1631 | /* Currently extern map definition has to be complete and match |
1632 | * concrete map definition exactly. This restriction might be lifted |
1633 | * in the future. |
1634 | */ |
1635 | return map_defs_match(sym_name, main_btf: linker->btf, main_def: &dst_def, main_inner_def: &dst_inner_def, |
1636 | extra_btf: obj->btf, extra_def: &src_def, extra_inner_def: &src_inner_def); |
1637 | } |
1638 | |
1639 | static bool glob_syms_match(const char *sym_name, |
1640 | struct bpf_linker *linker, struct glob_sym *glob_sym, |
1641 | struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id) |
1642 | { |
1643 | const struct btf_type *src_t; |
1644 | |
1645 | /* if we are dealing with externs, BTF types describing both global |
1646 | * and extern VARs/FUNCs should be completely present in all files |
1647 | */ |
1648 | if (!glob_sym->btf_id || !btf_id) { |
1649 | pr_warn("BTF info is missing for global symbol '%s'\n" , sym_name); |
1650 | return false; |
1651 | } |
1652 | |
1653 | src_t = btf__type_by_id(btf: obj->btf, id: btf_id); |
1654 | if (!btf_is_var(t: src_t) && !btf_is_func(t: src_t)) { |
1655 | pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n" , |
1656 | btf_kind_str(src_t), sym_name); |
1657 | return false; |
1658 | } |
1659 | |
1660 | /* deal with .maps definitions specially */ |
1661 | if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0) |
1662 | return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id); |
1663 | |
1664 | if (!glob_sym_btf_matches(sym_name, exact: true /*exact*/, |
1665 | btf1: linker->btf, id1: glob_sym->btf_id, btf2: obj->btf, id2: btf_id)) |
1666 | return false; |
1667 | |
1668 | return true; |
1669 | } |
1670 | |
1671 | static bool btf_is_non_static(const struct btf_type *t) |
1672 | { |
1673 | return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC) |
1674 | || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC); |
1675 | } |
1676 | |
1677 | static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name, |
1678 | int *out_btf_sec_id, int *out_btf_id) |
1679 | { |
1680 | int i, j, n, m, btf_id = 0; |
1681 | const struct btf_type *t; |
1682 | const struct btf_var_secinfo *vi; |
1683 | const char *name; |
1684 | |
1685 | if (!obj->btf) { |
1686 | pr_warn("failed to find BTF info for object '%s'\n" , obj->filename); |
1687 | return -EINVAL; |
1688 | } |
1689 | |
1690 | n = btf__type_cnt(btf: obj->btf); |
1691 | for (i = 1; i < n; i++) { |
1692 | t = btf__type_by_id(btf: obj->btf, id: i); |
1693 | |
1694 | /* some global and extern FUNCs and VARs might not be associated with any |
1695 | * DATASEC, so try to detect them in the same pass |
1696 | */ |
1697 | if (btf_is_non_static(t)) { |
1698 | name = btf__str_by_offset(btf: obj->btf, offset: t->name_off); |
1699 | if (strcmp(name, sym_name) != 0) |
1700 | continue; |
1701 | |
1702 | /* remember and still try to find DATASEC */ |
1703 | btf_id = i; |
1704 | continue; |
1705 | } |
1706 | |
1707 | if (!btf_is_datasec(t)) |
1708 | continue; |
1709 | |
1710 | vi = btf_var_secinfos(t); |
1711 | for (j = 0, m = btf_vlen(t); j < m; j++, vi++) { |
1712 | t = btf__type_by_id(btf: obj->btf, id: vi->type); |
1713 | name = btf__str_by_offset(btf: obj->btf, offset: t->name_off); |
1714 | |
1715 | if (strcmp(name, sym_name) != 0) |
1716 | continue; |
1717 | if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC) |
1718 | continue; |
1719 | if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC) |
1720 | continue; |
1721 | |
1722 | if (btf_id && btf_id != vi->type) { |
1723 | pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n" , |
1724 | sym_name, btf_id, vi->type); |
1725 | return -EINVAL; |
1726 | } |
1727 | |
1728 | *out_btf_sec_id = i; |
1729 | *out_btf_id = vi->type; |
1730 | |
1731 | return 0; |
1732 | } |
1733 | } |
1734 | |
1735 | /* free-floating extern or global FUNC */ |
1736 | if (btf_id) { |
1737 | *out_btf_sec_id = 0; |
1738 | *out_btf_id = btf_id; |
1739 | return 0; |
1740 | } |
1741 | |
1742 | pr_warn("failed to find BTF info for global/extern symbol '%s'\n" , sym_name); |
1743 | return -ENOENT; |
1744 | } |
1745 | |
1746 | static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name) |
1747 | { |
1748 | struct src_sec *sec; |
1749 | int i; |
1750 | |
1751 | for (i = 1; i < obj->sec_cnt; i++) { |
1752 | sec = &obj->secs[i]; |
1753 | |
1754 | if (strcmp(sec->sec_name, sec_name) == 0) |
1755 | return sec; |
1756 | } |
1757 | |
1758 | return NULL; |
1759 | } |
1760 | |
1761 | static int complete_extern_btf_info(struct btf *dst_btf, int dst_id, |
1762 | struct btf *src_btf, int src_id) |
1763 | { |
1764 | struct btf_type *dst_t = btf_type_by_id(btf: dst_btf, type_id: dst_id); |
1765 | struct btf_type *src_t = btf_type_by_id(btf: src_btf, type_id: src_id); |
1766 | struct btf_param *src_p, *dst_p; |
1767 | const char *s; |
1768 | int i, n, off; |
1769 | |
1770 | /* We already made sure that source and destination types (FUNC or |
1771 | * VAR) match in terms of types and argument names. |
1772 | */ |
1773 | if (btf_is_var(t: dst_t)) { |
1774 | btf_var(t: dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; |
1775 | return 0; |
1776 | } |
1777 | |
1778 | dst_t->info = btf_type_info(BTF_KIND_FUNC, vlen: BTF_FUNC_GLOBAL, kflag: 0); |
1779 | |
1780 | /* now onto FUNC_PROTO types */ |
1781 | src_t = btf_type_by_id(btf: src_btf, type_id: src_t->type); |
1782 | dst_t = btf_type_by_id(btf: dst_btf, type_id: dst_t->type); |
1783 | |
1784 | /* Fill in all the argument names, which for extern FUNCs are missing. |
1785 | * We'll end up with two copies of FUNCs/VARs for externs, but that |
1786 | * will be taken care of by BTF dedup at the very end. |
1787 | * It might be that BTF types for extern in one file has less/more BTF |
1788 | * information (e.g., FWD instead of full STRUCT/UNION information), |
1789 | * but that should be (in most cases, subject to BTF dedup rules) |
1790 | * handled and resolved by BTF dedup algorithm as well, so we won't |
1791 | * worry about it. Our only job is to make sure that argument names |
1792 | * are populated on both sides, otherwise BTF dedup will pedantically |
1793 | * consider them different. |
1794 | */ |
1795 | src_p = btf_params(src_t); |
1796 | dst_p = btf_params(dst_t); |
1797 | for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) { |
1798 | if (!src_p->name_off) |
1799 | continue; |
1800 | |
1801 | /* src_btf has more complete info, so add name to dst_btf */ |
1802 | s = btf__str_by_offset(btf: src_btf, offset: src_p->name_off); |
1803 | off = btf__add_str(btf: dst_btf, s); |
1804 | if (off < 0) |
1805 | return off; |
1806 | dst_p->name_off = off; |
1807 | } |
1808 | return 0; |
1809 | } |
1810 | |
1811 | static void sym_update_bind(Elf64_Sym *sym, int sym_bind) |
1812 | { |
1813 | sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info)); |
1814 | } |
1815 | |
1816 | static void sym_update_type(Elf64_Sym *sym, int sym_type) |
1817 | { |
1818 | sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type); |
1819 | } |
1820 | |
1821 | static void sym_update_visibility(Elf64_Sym *sym, int sym_vis) |
1822 | { |
1823 | /* libelf doesn't provide setters for ST_VISIBILITY, |
1824 | * but it is stored in the lower 2 bits of st_other |
1825 | */ |
1826 | sym->st_other &= ~0x03; |
1827 | sym->st_other |= sym_vis; |
1828 | } |
1829 | |
1830 | static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj, |
1831 | Elf64_Sym *sym, const char *sym_name, int src_sym_idx) |
1832 | { |
1833 | struct src_sec *src_sec = NULL; |
1834 | struct dst_sec *dst_sec = NULL; |
1835 | struct glob_sym *glob_sym = NULL; |
1836 | int name_off, sym_type, sym_bind, sym_vis, err; |
1837 | int btf_sec_id = 0, btf_id = 0; |
1838 | size_t dst_sym_idx; |
1839 | Elf64_Sym *dst_sym; |
1840 | bool sym_is_extern; |
1841 | |
1842 | sym_type = ELF64_ST_TYPE(sym->st_info); |
1843 | sym_bind = ELF64_ST_BIND(sym->st_info); |
1844 | sym_vis = ELF64_ST_VISIBILITY(sym->st_other); |
1845 | sym_is_extern = sym->st_shndx == SHN_UNDEF; |
1846 | |
1847 | if (sym_is_extern) { |
1848 | if (!obj->btf) { |
1849 | pr_warn("externs without BTF info are not supported\n" ); |
1850 | return -ENOTSUP; |
1851 | } |
1852 | } else if (sym->st_shndx < SHN_LORESERVE) { |
1853 | src_sec = &obj->secs[sym->st_shndx]; |
1854 | if (src_sec->skipped) |
1855 | return 0; |
1856 | dst_sec = &linker->secs[src_sec->dst_id]; |
1857 | |
1858 | /* allow only one STT_SECTION symbol per section */ |
1859 | if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) { |
1860 | obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx; |
1861 | return 0; |
1862 | } |
1863 | } |
1864 | |
1865 | if (sym_bind == STB_LOCAL) |
1866 | goto add_sym; |
1867 | |
1868 | /* find matching BTF info */ |
1869 | err = find_glob_sym_btf(obj, sym, sym_name, out_btf_sec_id: &btf_sec_id, out_btf_id: &btf_id); |
1870 | if (err) |
1871 | return err; |
1872 | |
1873 | if (sym_is_extern && btf_sec_id) { |
1874 | const char *sec_name = NULL; |
1875 | const struct btf_type *t; |
1876 | |
1877 | t = btf__type_by_id(btf: obj->btf, id: btf_sec_id); |
1878 | sec_name = btf__str_by_offset(btf: obj->btf, offset: t->name_off); |
1879 | |
1880 | /* Clang puts unannotated extern vars into |
1881 | * '.extern' BTF DATASEC. Treat them the same |
1882 | * as unannotated extern funcs (which are |
1883 | * currently not put into any DATASECs). |
1884 | * Those don't have associated src_sec/dst_sec. |
1885 | */ |
1886 | if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) { |
1887 | src_sec = find_src_sec_by_name(obj, sec_name); |
1888 | if (!src_sec) { |
1889 | pr_warn("failed to find matching ELF sec '%s'\n" , sec_name); |
1890 | return -ENOENT; |
1891 | } |
1892 | dst_sec = &linker->secs[src_sec->dst_id]; |
1893 | } |
1894 | } |
1895 | |
1896 | glob_sym = find_glob_sym(linker, sym_name); |
1897 | if (glob_sym) { |
1898 | /* Preventively resolve to existing symbol. This is |
1899 | * needed for further relocation symbol remapping in |
1900 | * the next step of linking. |
1901 | */ |
1902 | obj->sym_map[src_sym_idx] = glob_sym->sym_idx; |
1903 | |
1904 | /* If both symbols are non-externs, at least one of |
1905 | * them has to be STB_WEAK, otherwise they are in |
1906 | * a conflict with each other. |
1907 | */ |
1908 | if (!sym_is_extern && !glob_sym->is_extern |
1909 | && !glob_sym->is_weak && sym_bind != STB_WEAK) { |
1910 | pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n" , |
1911 | src_sym_idx, sym_name, obj->filename); |
1912 | return -EINVAL; |
1913 | } |
1914 | |
1915 | if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, sym_idx: src_sym_idx, btf_id)) |
1916 | return -EINVAL; |
1917 | |
1918 | dst_sym = get_sym_by_idx(linker, sym_idx: glob_sym->sym_idx); |
1919 | |
1920 | /* If new symbol is strong, then force dst_sym to be strong as |
1921 | * well; this way a mix of weak and non-weak extern |
1922 | * definitions will end up being strong. |
1923 | */ |
1924 | if (sym_bind == STB_GLOBAL) { |
1925 | /* We still need to preserve type (NOTYPE or |
1926 | * OBJECT/FUNC, depending on whether the symbol is |
1927 | * extern or not) |
1928 | */ |
1929 | sym_update_bind(sym: dst_sym, STB_GLOBAL); |
1930 | glob_sym->is_weak = false; |
1931 | } |
1932 | |
1933 | /* Non-default visibility is "contaminating", with stricter |
1934 | * visibility overwriting more permissive ones, even if more |
1935 | * permissive visibility comes from just an extern definition. |
1936 | * Currently only STV_DEFAULT and STV_HIDDEN are allowed and |
1937 | * ensured by ELF symbol sanity checks above. |
1938 | */ |
1939 | if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other)) |
1940 | sym_update_visibility(sym: dst_sym, sym_vis); |
1941 | |
1942 | /* If the new symbol is extern, then regardless if |
1943 | * existing symbol is extern or resolved global, just |
1944 | * keep the existing one untouched. |
1945 | */ |
1946 | if (sym_is_extern) |
1947 | return 0; |
1948 | |
1949 | /* If existing symbol is a strong resolved symbol, bail out, |
1950 | * because we lost resolution battle have nothing to |
1951 | * contribute. We already checked abover that there is no |
1952 | * strong-strong conflict. We also already tightened binding |
1953 | * and visibility, so nothing else to contribute at that point. |
1954 | */ |
1955 | if (!glob_sym->is_extern && sym_bind == STB_WEAK) |
1956 | return 0; |
1957 | |
1958 | /* At this point, new symbol is strong non-extern, |
1959 | * so overwrite glob_sym with new symbol information. |
1960 | * Preserve binding and visibility. |
1961 | */ |
1962 | sym_update_type(sym: dst_sym, sym_type); |
1963 | dst_sym->st_shndx = dst_sec->sec_idx; |
1964 | dst_sym->st_value = src_sec->dst_off + sym->st_value; |
1965 | dst_sym->st_size = sym->st_size; |
1966 | |
1967 | /* see comment below about dst_sec->id vs dst_sec->sec_idx */ |
1968 | glob_sym->sec_id = dst_sec->id; |
1969 | glob_sym->is_extern = false; |
1970 | |
1971 | if (complete_extern_btf_info(dst_btf: linker->btf, dst_id: glob_sym->btf_id, |
1972 | src_btf: obj->btf, src_id: btf_id)) |
1973 | return -EINVAL; |
1974 | |
1975 | /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */ |
1976 | glob_sym->underlying_btf_id = 0; |
1977 | |
1978 | obj->sym_map[src_sym_idx] = glob_sym->sym_idx; |
1979 | return 0; |
1980 | } |
1981 | |
1982 | add_sym: |
1983 | name_off = strset__add_str(set: linker->strtab_strs, s: sym_name); |
1984 | if (name_off < 0) |
1985 | return name_off; |
1986 | |
1987 | dst_sym = add_new_sym(linker, sym_idx: &dst_sym_idx); |
1988 | if (!dst_sym) |
1989 | return -ENOMEM; |
1990 | |
1991 | dst_sym->st_name = name_off; |
1992 | dst_sym->st_info = sym->st_info; |
1993 | dst_sym->st_other = sym->st_other; |
1994 | dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx; |
1995 | dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; |
1996 | dst_sym->st_size = sym->st_size; |
1997 | |
1998 | obj->sym_map[src_sym_idx] = dst_sym_idx; |
1999 | |
2000 | if (sym_type == STT_SECTION && dst_sym) { |
2001 | dst_sec->sec_sym_idx = dst_sym_idx; |
2002 | dst_sym->st_value = 0; |
2003 | } |
2004 | |
2005 | if (sym_bind != STB_LOCAL) { |
2006 | glob_sym = add_glob_sym(linker); |
2007 | if (!glob_sym) |
2008 | return -ENOMEM; |
2009 | |
2010 | glob_sym->sym_idx = dst_sym_idx; |
2011 | /* we use dst_sec->id (and not dst_sec->sec_idx), because |
2012 | * ephemeral sections (.kconfig, .ksyms, etc) don't have |
2013 | * sec_idx (as they don't have corresponding ELF section), but |
2014 | * still have id. .extern doesn't have even ephemeral section |
2015 | * associated with it, so dst_sec->id == dst_sec->sec_idx == 0. |
2016 | */ |
2017 | glob_sym->sec_id = dst_sec ? dst_sec->id : 0; |
2018 | glob_sym->name_off = name_off; |
2019 | /* we will fill btf_id in during BTF merging step */ |
2020 | glob_sym->btf_id = 0; |
2021 | glob_sym->is_extern = sym_is_extern; |
2022 | glob_sym->is_weak = sym_bind == STB_WEAK; |
2023 | } |
2024 | |
2025 | return 0; |
2026 | } |
2027 | |
2028 | static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj) |
2029 | { |
2030 | struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx]; |
2031 | int i, err; |
2032 | |
2033 | for (i = 1; i < obj->sec_cnt; i++) { |
2034 | struct src_sec *src_sec, *src_linked_sec; |
2035 | struct dst_sec *dst_sec, *dst_linked_sec; |
2036 | Elf64_Rel *src_rel, *dst_rel; |
2037 | int j, n; |
2038 | |
2039 | src_sec = &obj->secs[i]; |
2040 | if (!is_relo_sec(sec: src_sec)) |
2041 | continue; |
2042 | |
2043 | /* shdr->sh_info points to relocatable section */ |
2044 | src_linked_sec = &obj->secs[src_sec->shdr->sh_info]; |
2045 | if (src_linked_sec->skipped) |
2046 | continue; |
2047 | |
2048 | dst_sec = find_dst_sec_by_name(linker, sec_name: src_sec->sec_name); |
2049 | if (!dst_sec) { |
2050 | dst_sec = add_dst_sec(linker, sec_name: src_sec->sec_name); |
2051 | if (!dst_sec) |
2052 | return -ENOMEM; |
2053 | err = init_sec(linker, dst_sec, src_sec); |
2054 | if (err) { |
2055 | pr_warn("failed to init section '%s'\n" , src_sec->sec_name); |
2056 | return err; |
2057 | } |
2058 | } else if (!secs_match(dst: dst_sec, src: src_sec)) { |
2059 | pr_warn("sections %s are not compatible\n" , src_sec->sec_name); |
2060 | return -1; |
2061 | } |
2062 | |
2063 | /* shdr->sh_link points to SYMTAB */ |
2064 | dst_sec->shdr->sh_link = linker->symtab_sec_idx; |
2065 | |
2066 | /* shdr->sh_info points to relocated section */ |
2067 | dst_linked_sec = &linker->secs[src_linked_sec->dst_id]; |
2068 | dst_sec->shdr->sh_info = dst_linked_sec->sec_idx; |
2069 | |
2070 | src_sec->dst_id = dst_sec->id; |
2071 | err = extend_sec(linker, dst: dst_sec, src: src_sec); |
2072 | if (err) |
2073 | return err; |
2074 | |
2075 | src_rel = src_sec->data->d_buf; |
2076 | dst_rel = dst_sec->raw_data + src_sec->dst_off; |
2077 | n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize; |
2078 | for (j = 0; j < n; j++, src_rel++, dst_rel++) { |
2079 | size_t src_sym_idx, dst_sym_idx, sym_type; |
2080 | Elf64_Sym *src_sym; |
2081 | |
2082 | src_sym_idx = ELF64_R_SYM(src_rel->r_info); |
2083 | src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx; |
2084 | |
2085 | dst_sym_idx = obj->sym_map[src_sym_idx]; |
2086 | dst_rel->r_offset += src_linked_sec->dst_off; |
2087 | sym_type = ELF64_R_TYPE(src_rel->r_info); |
2088 | dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type); |
2089 | |
2090 | if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) { |
2091 | struct src_sec *sec = &obj->secs[src_sym->st_shndx]; |
2092 | struct bpf_insn *insn; |
2093 | |
2094 | if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) { |
2095 | /* calls to the very first static function inside |
2096 | * .text section at offset 0 will |
2097 | * reference section symbol, not the |
2098 | * function symbol. Fix that up, |
2099 | * otherwise it won't be possible to |
2100 | * relocate calls to two different |
2101 | * static functions with the same name |
2102 | * (rom two different object files) |
2103 | */ |
2104 | insn = dst_linked_sec->raw_data + dst_rel->r_offset; |
2105 | if (insn->code == (BPF_JMP | BPF_CALL)) |
2106 | insn->imm += sec->dst_off / sizeof(struct bpf_insn); |
2107 | else |
2108 | insn->imm += sec->dst_off; |
2109 | } else { |
2110 | pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n" ); |
2111 | return -EINVAL; |
2112 | } |
2113 | } |
2114 | |
2115 | } |
2116 | } |
2117 | |
2118 | return 0; |
2119 | } |
2120 | |
2121 | static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx, |
2122 | int sym_type, const char *sym_name) |
2123 | { |
2124 | struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; |
2125 | Elf64_Sym *sym = symtab->data->d_buf; |
2126 | int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize; |
2127 | int str_sec_idx = symtab->shdr->sh_link; |
2128 | const char *name; |
2129 | |
2130 | for (i = 0; i < n; i++, sym++) { |
2131 | if (sym->st_shndx != sec_idx) |
2132 | continue; |
2133 | if (ELF64_ST_TYPE(sym->st_info) != sym_type) |
2134 | continue; |
2135 | |
2136 | name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); |
2137 | if (!name) |
2138 | return NULL; |
2139 | |
2140 | if (strcmp(sym_name, name) != 0) |
2141 | continue; |
2142 | |
2143 | return sym; |
2144 | } |
2145 | |
2146 | return NULL; |
2147 | } |
2148 | |
2149 | static int linker_fixup_btf(struct src_obj *obj) |
2150 | { |
2151 | const char *sec_name; |
2152 | struct src_sec *sec; |
2153 | int i, j, n, m; |
2154 | |
2155 | if (!obj->btf) |
2156 | return 0; |
2157 | |
2158 | n = btf__type_cnt(btf: obj->btf); |
2159 | for (i = 1; i < n; i++) { |
2160 | struct btf_var_secinfo *vi; |
2161 | struct btf_type *t; |
2162 | |
2163 | t = btf_type_by_id(btf: obj->btf, type_id: i); |
2164 | if (btf_kind(t) != BTF_KIND_DATASEC) |
2165 | continue; |
2166 | |
2167 | sec_name = btf__str_by_offset(btf: obj->btf, offset: t->name_off); |
2168 | sec = find_src_sec_by_name(obj, sec_name); |
2169 | if (sec) { |
2170 | /* record actual section size, unless ephemeral */ |
2171 | if (sec->shdr) |
2172 | t->size = sec->shdr->sh_size; |
2173 | } else { |
2174 | /* BTF can have some sections that are not represented |
2175 | * in ELF, e.g., .kconfig, .ksyms, .extern, which are used |
2176 | * for special extern variables. |
2177 | * |
2178 | * For all but one such special (ephemeral) |
2179 | * sections, we pre-create "section shells" to be able |
2180 | * to keep track of extra per-section metadata later |
2181 | * (e.g., those BTF extern variables). |
2182 | * |
2183 | * .extern is even more special, though, because it |
2184 | * contains extern variables that need to be resolved |
2185 | * by static linker, not libbpf and kernel. When such |
2186 | * externs are resolved, we are going to remove them |
2187 | * from .extern BTF section and might end up not |
2188 | * needing it at all. Each resolved extern should have |
2189 | * matching non-extern VAR/FUNC in other sections. |
2190 | * |
2191 | * We do support leaving some of the externs |
2192 | * unresolved, though, to support cases of building |
2193 | * libraries, which will later be linked against final |
2194 | * BPF applications. So if at finalization we still |
2195 | * see unresolved externs, we'll create .extern |
2196 | * section on our own. |
2197 | */ |
2198 | if (strcmp(sec_name, BTF_EXTERN_SEC) == 0) |
2199 | continue; |
2200 | |
2201 | sec = add_src_sec(obj, sec_name); |
2202 | if (!sec) |
2203 | return -ENOMEM; |
2204 | |
2205 | sec->ephemeral = true; |
2206 | sec->sec_idx = 0; /* will match UNDEF shndx in ELF */ |
2207 | } |
2208 | |
2209 | /* remember ELF section and its BTF type ID match */ |
2210 | sec->sec_type_id = i; |
2211 | |
2212 | /* fix up variable offsets */ |
2213 | vi = btf_var_secinfos(t); |
2214 | for (j = 0, m = btf_vlen(t); j < m; j++, vi++) { |
2215 | const struct btf_type *vt = btf__type_by_id(btf: obj->btf, id: vi->type); |
2216 | const char *var_name = btf__str_by_offset(btf: obj->btf, offset: vt->name_off); |
2217 | int var_linkage = btf_var(t: vt)->linkage; |
2218 | Elf64_Sym *sym; |
2219 | |
2220 | /* no need to patch up static or extern vars */ |
2221 | if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED) |
2222 | continue; |
2223 | |
2224 | sym = find_sym_by_name(obj, sec_idx: sec->sec_idx, STT_OBJECT, sym_name: var_name); |
2225 | if (!sym) { |
2226 | pr_warn("failed to find symbol for variable '%s' in section '%s'\n" , var_name, sec_name); |
2227 | return -ENOENT; |
2228 | } |
2229 | |
2230 | vi->offset = sym->st_value; |
2231 | } |
2232 | } |
2233 | |
2234 | return 0; |
2235 | } |
2236 | |
2237 | static int remap_type_id(__u32 *type_id, void *ctx) |
2238 | { |
2239 | int *id_map = ctx; |
2240 | int new_id = id_map[*type_id]; |
2241 | |
2242 | /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */ |
2243 | if (new_id == 0 && *type_id != 0) { |
2244 | pr_warn("failed to find new ID mapping for original BTF type ID %u\n" , *type_id); |
2245 | return -EINVAL; |
2246 | } |
2247 | |
2248 | *type_id = id_map[*type_id]; |
2249 | |
2250 | return 0; |
2251 | } |
2252 | |
2253 | static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj) |
2254 | { |
2255 | const struct btf_type *t; |
2256 | int i, j, n, start_id, id; |
2257 | const char *name; |
2258 | |
2259 | if (!obj->btf) |
2260 | return 0; |
2261 | |
2262 | start_id = btf__type_cnt(btf: linker->btf); |
2263 | n = btf__type_cnt(btf: obj->btf); |
2264 | |
2265 | obj->btf_type_map = calloc(n + 1, sizeof(int)); |
2266 | if (!obj->btf_type_map) |
2267 | return -ENOMEM; |
2268 | |
2269 | for (i = 1; i < n; i++) { |
2270 | struct glob_sym *glob_sym = NULL; |
2271 | |
2272 | t = btf__type_by_id(btf: obj->btf, id: i); |
2273 | |
2274 | /* DATASECs are handled specially below */ |
2275 | if (btf_kind(t) == BTF_KIND_DATASEC) |
2276 | continue; |
2277 | |
2278 | if (btf_is_non_static(t)) { |
2279 | /* there should be glob_sym already */ |
2280 | name = btf__str_by_offset(btf: obj->btf, offset: t->name_off); |
2281 | glob_sym = find_glob_sym(linker, sym_name: name); |
2282 | |
2283 | /* VARs without corresponding glob_sym are those that |
2284 | * belong to skipped/deduplicated sections (i.e., |
2285 | * license and version), so just skip them |
2286 | */ |
2287 | if (!glob_sym) |
2288 | continue; |
2289 | |
2290 | /* linker_append_elf_sym() might have requested |
2291 | * updating underlying type ID, if extern was resolved |
2292 | * to strong symbol or weak got upgraded to non-weak |
2293 | */ |
2294 | if (glob_sym->underlying_btf_id == 0) |
2295 | glob_sym->underlying_btf_id = -t->type; |
2296 | |
2297 | /* globals from previous object files that match our |
2298 | * VAR/FUNC already have a corresponding associated |
2299 | * BTF type, so just make sure to use it |
2300 | */ |
2301 | if (glob_sym->btf_id) { |
2302 | /* reuse existing BTF type for global var/func */ |
2303 | obj->btf_type_map[i] = glob_sym->btf_id; |
2304 | continue; |
2305 | } |
2306 | } |
2307 | |
2308 | id = btf__add_type(btf: linker->btf, src_btf: obj->btf, src_type: t); |
2309 | if (id < 0) { |
2310 | pr_warn("failed to append BTF type #%d from file '%s'\n" , i, obj->filename); |
2311 | return id; |
2312 | } |
2313 | |
2314 | obj->btf_type_map[i] = id; |
2315 | |
2316 | /* record just appended BTF type for var/func */ |
2317 | if (glob_sym) { |
2318 | glob_sym->btf_id = id; |
2319 | glob_sym->underlying_btf_id = -t->type; |
2320 | } |
2321 | } |
2322 | |
2323 | /* remap all the types except DATASECs */ |
2324 | n = btf__type_cnt(btf: linker->btf); |
2325 | for (i = start_id; i < n; i++) { |
2326 | struct btf_type *dst_t = btf_type_by_id(btf: linker->btf, type_id: i); |
2327 | |
2328 | if (btf_type_visit_type_ids(t: dst_t, visit: remap_type_id, ctx: obj->btf_type_map)) |
2329 | return -EINVAL; |
2330 | } |
2331 | |
2332 | /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's |
2333 | * actual type), if necessary |
2334 | */ |
2335 | for (i = 0; i < linker->glob_sym_cnt; i++) { |
2336 | struct glob_sym *glob_sym = &linker->glob_syms[i]; |
2337 | struct btf_type *glob_t; |
2338 | |
2339 | if (glob_sym->underlying_btf_id >= 0) |
2340 | continue; |
2341 | |
2342 | glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id]; |
2343 | |
2344 | glob_t = btf_type_by_id(btf: linker->btf, type_id: glob_sym->btf_id); |
2345 | glob_t->type = glob_sym->underlying_btf_id; |
2346 | } |
2347 | |
2348 | /* append DATASEC info */ |
2349 | for (i = 1; i < obj->sec_cnt; i++) { |
2350 | struct src_sec *src_sec; |
2351 | struct dst_sec *dst_sec; |
2352 | const struct btf_var_secinfo *src_var; |
2353 | struct btf_var_secinfo *dst_var; |
2354 | |
2355 | src_sec = &obj->secs[i]; |
2356 | if (!src_sec->sec_type_id || src_sec->skipped) |
2357 | continue; |
2358 | dst_sec = &linker->secs[src_sec->dst_id]; |
2359 | |
2360 | /* Mark section as having BTF regardless of the presence of |
2361 | * variables. In some cases compiler might generate empty BTF |
2362 | * with no variables information. E.g., when promoting local |
2363 | * array/structure variable initial values and BPF object |
2364 | * file otherwise has no read-only static variables in |
2365 | * .rodata. We need to preserve such empty BTF and just set |
2366 | * correct section size. |
2367 | */ |
2368 | dst_sec->has_btf = true; |
2369 | |
2370 | t = btf__type_by_id(btf: obj->btf, id: src_sec->sec_type_id); |
2371 | src_var = btf_var_secinfos(t); |
2372 | n = btf_vlen(t); |
2373 | for (j = 0; j < n; j++, src_var++) { |
2374 | void *sec_vars = dst_sec->sec_vars; |
2375 | int new_id = obj->btf_type_map[src_var->type]; |
2376 | struct glob_sym *glob_sym = NULL; |
2377 | |
2378 | t = btf_type_by_id(btf: linker->btf, type_id: new_id); |
2379 | if (btf_is_non_static(t)) { |
2380 | name = btf__str_by_offset(btf: linker->btf, offset: t->name_off); |
2381 | glob_sym = find_glob_sym(linker, sym_name: name); |
2382 | if (glob_sym->sec_id != dst_sec->id) { |
2383 | pr_warn("global '%s': section mismatch %d vs %d\n" , |
2384 | name, glob_sym->sec_id, dst_sec->id); |
2385 | return -EINVAL; |
2386 | } |
2387 | } |
2388 | |
2389 | /* If there is already a member (VAR or FUNC) mapped |
2390 | * to the same type, don't add a duplicate entry. |
2391 | * This will happen when multiple object files define |
2392 | * the same extern VARs/FUNCs. |
2393 | */ |
2394 | if (glob_sym && glob_sym->var_idx >= 0) { |
2395 | __s64 sz; |
2396 | |
2397 | dst_var = &dst_sec->sec_vars[glob_sym->var_idx]; |
2398 | /* Because underlying BTF type might have |
2399 | * changed, so might its size have changed, so |
2400 | * re-calculate and update it in sec_var. |
2401 | */ |
2402 | sz = btf__resolve_size(btf: linker->btf, type_id: glob_sym->underlying_btf_id); |
2403 | if (sz < 0) { |
2404 | pr_warn("global '%s': failed to resolve size of underlying type: %d\n" , |
2405 | name, (int)sz); |
2406 | return -EINVAL; |
2407 | } |
2408 | dst_var->size = sz; |
2409 | continue; |
2410 | } |
2411 | |
2412 | sec_vars = libbpf_reallocarray(ptr: sec_vars, |
2413 | nmemb: dst_sec->sec_var_cnt + 1, |
2414 | size: sizeof(*dst_sec->sec_vars)); |
2415 | if (!sec_vars) |
2416 | return -ENOMEM; |
2417 | |
2418 | dst_sec->sec_vars = sec_vars; |
2419 | dst_sec->sec_var_cnt++; |
2420 | |
2421 | dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1]; |
2422 | dst_var->type = obj->btf_type_map[src_var->type]; |
2423 | dst_var->size = src_var->size; |
2424 | dst_var->offset = src_sec->dst_off + src_var->offset; |
2425 | |
2426 | if (glob_sym) |
2427 | glob_sym->var_idx = dst_sec->sec_var_cnt - 1; |
2428 | } |
2429 | } |
2430 | |
2431 | return 0; |
2432 | } |
2433 | |
2434 | static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec) |
2435 | { |
2436 | void *tmp; |
2437 | |
2438 | tmp = libbpf_reallocarray(ptr: ext_data->recs, nmemb: ext_data->rec_cnt + 1, size: ext_data->rec_sz); |
2439 | if (!tmp) |
2440 | return NULL; |
2441 | ext_data->recs = tmp; |
2442 | |
2443 | tmp += ext_data->rec_cnt * ext_data->rec_sz; |
2444 | memcpy(tmp, src_rec, ext_data->rec_sz); |
2445 | |
2446 | ext_data->rec_cnt++; |
2447 | |
2448 | return tmp; |
2449 | } |
2450 | |
2451 | static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj) |
2452 | { |
2453 | const struct btf_ext_info_sec *ext_sec; |
2454 | const char *sec_name, *s; |
2455 | struct src_sec *src_sec; |
2456 | struct dst_sec *dst_sec; |
2457 | int rec_sz, str_off, i; |
2458 | |
2459 | if (!obj->btf_ext) |
2460 | return 0; |
2461 | |
2462 | rec_sz = obj->btf_ext->func_info.rec_size; |
2463 | for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) { |
2464 | struct bpf_func_info_min *src_rec, *dst_rec; |
2465 | |
2466 | sec_name = btf__name_by_offset(btf: obj->btf, offset: ext_sec->sec_name_off); |
2467 | src_sec = find_src_sec_by_name(obj, sec_name); |
2468 | if (!src_sec) { |
2469 | pr_warn("can't find section '%s' referenced from .BTF.ext\n" , sec_name); |
2470 | return -EINVAL; |
2471 | } |
2472 | dst_sec = &linker->secs[src_sec->dst_id]; |
2473 | |
2474 | if (dst_sec->func_info.rec_sz == 0) |
2475 | dst_sec->func_info.rec_sz = rec_sz; |
2476 | if (dst_sec->func_info.rec_sz != rec_sz) { |
2477 | pr_warn("incompatible .BTF.ext record sizes for section '%s'\n" , sec_name); |
2478 | return -EINVAL; |
2479 | } |
2480 | |
2481 | for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) { |
2482 | dst_rec = add_btf_ext_rec(ext_data: &dst_sec->func_info, src_rec); |
2483 | if (!dst_rec) |
2484 | return -ENOMEM; |
2485 | |
2486 | dst_rec->insn_off += src_sec->dst_off; |
2487 | dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; |
2488 | } |
2489 | } |
2490 | |
2491 | rec_sz = obj->btf_ext->line_info.rec_size; |
2492 | for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) { |
2493 | struct bpf_line_info_min *src_rec, *dst_rec; |
2494 | |
2495 | sec_name = btf__name_by_offset(btf: obj->btf, offset: ext_sec->sec_name_off); |
2496 | src_sec = find_src_sec_by_name(obj, sec_name); |
2497 | if (!src_sec) { |
2498 | pr_warn("can't find section '%s' referenced from .BTF.ext\n" , sec_name); |
2499 | return -EINVAL; |
2500 | } |
2501 | dst_sec = &linker->secs[src_sec->dst_id]; |
2502 | |
2503 | if (dst_sec->line_info.rec_sz == 0) |
2504 | dst_sec->line_info.rec_sz = rec_sz; |
2505 | if (dst_sec->line_info.rec_sz != rec_sz) { |
2506 | pr_warn("incompatible .BTF.ext record sizes for section '%s'\n" , sec_name); |
2507 | return -EINVAL; |
2508 | } |
2509 | |
2510 | for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) { |
2511 | dst_rec = add_btf_ext_rec(ext_data: &dst_sec->line_info, src_rec); |
2512 | if (!dst_rec) |
2513 | return -ENOMEM; |
2514 | |
2515 | dst_rec->insn_off += src_sec->dst_off; |
2516 | |
2517 | s = btf__str_by_offset(btf: obj->btf, offset: src_rec->file_name_off); |
2518 | str_off = btf__add_str(btf: linker->btf, s); |
2519 | if (str_off < 0) |
2520 | return -ENOMEM; |
2521 | dst_rec->file_name_off = str_off; |
2522 | |
2523 | s = btf__str_by_offset(btf: obj->btf, offset: src_rec->line_off); |
2524 | str_off = btf__add_str(btf: linker->btf, s); |
2525 | if (str_off < 0) |
2526 | return -ENOMEM; |
2527 | dst_rec->line_off = str_off; |
2528 | |
2529 | /* dst_rec->line_col is fine */ |
2530 | } |
2531 | } |
2532 | |
2533 | rec_sz = obj->btf_ext->core_relo_info.rec_size; |
2534 | for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) { |
2535 | struct bpf_core_relo *src_rec, *dst_rec; |
2536 | |
2537 | sec_name = btf__name_by_offset(btf: obj->btf, offset: ext_sec->sec_name_off); |
2538 | src_sec = find_src_sec_by_name(obj, sec_name); |
2539 | if (!src_sec) { |
2540 | pr_warn("can't find section '%s' referenced from .BTF.ext\n" , sec_name); |
2541 | return -EINVAL; |
2542 | } |
2543 | dst_sec = &linker->secs[src_sec->dst_id]; |
2544 | |
2545 | if (dst_sec->core_relo_info.rec_sz == 0) |
2546 | dst_sec->core_relo_info.rec_sz = rec_sz; |
2547 | if (dst_sec->core_relo_info.rec_sz != rec_sz) { |
2548 | pr_warn("incompatible .BTF.ext record sizes for section '%s'\n" , sec_name); |
2549 | return -EINVAL; |
2550 | } |
2551 | |
2552 | for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) { |
2553 | dst_rec = add_btf_ext_rec(ext_data: &dst_sec->core_relo_info, src_rec); |
2554 | if (!dst_rec) |
2555 | return -ENOMEM; |
2556 | |
2557 | dst_rec->insn_off += src_sec->dst_off; |
2558 | dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; |
2559 | |
2560 | s = btf__str_by_offset(btf: obj->btf, offset: src_rec->access_str_off); |
2561 | str_off = btf__add_str(btf: linker->btf, s); |
2562 | if (str_off < 0) |
2563 | return -ENOMEM; |
2564 | dst_rec->access_str_off = str_off; |
2565 | |
2566 | /* dst_rec->kind is fine */ |
2567 | } |
2568 | } |
2569 | |
2570 | return 0; |
2571 | } |
2572 | |
2573 | int bpf_linker__finalize(struct bpf_linker *linker) |
2574 | { |
2575 | struct dst_sec *sec; |
2576 | size_t strs_sz; |
2577 | const void *strs; |
2578 | int err, i; |
2579 | |
2580 | if (!linker->elf) |
2581 | return libbpf_err(ret: -EINVAL); |
2582 | |
2583 | err = finalize_btf(linker); |
2584 | if (err) |
2585 | return libbpf_err(ret: err); |
2586 | |
2587 | /* Finalize strings */ |
2588 | strs_sz = strset__data_size(set: linker->strtab_strs); |
2589 | strs = strset__data(set: linker->strtab_strs); |
2590 | |
2591 | sec = &linker->secs[linker->strtab_sec_idx]; |
2592 | sec->data->d_align = 1; |
2593 | sec->data->d_off = 0LL; |
2594 | sec->data->d_buf = (void *)strs; |
2595 | sec->data->d_type = ELF_T_BYTE; |
2596 | sec->data->d_size = strs_sz; |
2597 | sec->shdr->sh_size = strs_sz; |
2598 | |
2599 | for (i = 1; i < linker->sec_cnt; i++) { |
2600 | sec = &linker->secs[i]; |
2601 | |
2602 | /* STRTAB is handled specially above */ |
2603 | if (sec->sec_idx == linker->strtab_sec_idx) |
2604 | continue; |
2605 | |
2606 | /* special ephemeral sections (.ksyms, .kconfig, etc) */ |
2607 | if (!sec->scn) |
2608 | continue; |
2609 | |
2610 | sec->data->d_buf = sec->raw_data; |
2611 | } |
2612 | |
2613 | /* Finalize ELF layout */ |
2614 | if (elf_update(linker->elf, ELF_C_NULL) < 0) { |
2615 | err = -errno; |
2616 | pr_warn_elf("failed to finalize ELF layout" ); |
2617 | return libbpf_err(ret: err); |
2618 | } |
2619 | |
2620 | /* Write out final ELF contents */ |
2621 | if (elf_update(linker->elf, ELF_C_WRITE) < 0) { |
2622 | err = -errno; |
2623 | pr_warn_elf("failed to write ELF contents" ); |
2624 | return libbpf_err(ret: err); |
2625 | } |
2626 | |
2627 | elf_end(linker->elf); |
2628 | close(linker->fd); |
2629 | |
2630 | linker->elf = NULL; |
2631 | linker->fd = -1; |
2632 | |
2633 | return 0; |
2634 | } |
2635 | |
2636 | static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name, |
2637 | size_t align, const void *raw_data, size_t raw_sz) |
2638 | { |
2639 | Elf_Scn *scn; |
2640 | Elf_Data *data; |
2641 | Elf64_Shdr *shdr; |
2642 | int name_off; |
2643 | |
2644 | name_off = strset__add_str(set: linker->strtab_strs, s: sec_name); |
2645 | if (name_off < 0) |
2646 | return name_off; |
2647 | |
2648 | scn = elf_newscn(linker->elf); |
2649 | if (!scn) |
2650 | return -ENOMEM; |
2651 | data = elf_newdata(scn); |
2652 | if (!data) |
2653 | return -ENOMEM; |
2654 | shdr = elf64_getshdr(scn); |
2655 | if (!shdr) |
2656 | return -EINVAL; |
2657 | |
2658 | shdr->sh_name = name_off; |
2659 | shdr->sh_type = SHT_PROGBITS; |
2660 | shdr->sh_flags = 0; |
2661 | shdr->sh_size = raw_sz; |
2662 | shdr->sh_link = 0; |
2663 | shdr->sh_info = 0; |
2664 | shdr->sh_addralign = align; |
2665 | shdr->sh_entsize = 0; |
2666 | |
2667 | data->d_type = ELF_T_BYTE; |
2668 | data->d_size = raw_sz; |
2669 | data->d_buf = (void *)raw_data; |
2670 | data->d_align = align; |
2671 | data->d_off = 0; |
2672 | |
2673 | return 0; |
2674 | } |
2675 | |
2676 | static int finalize_btf(struct bpf_linker *linker) |
2677 | { |
2678 | LIBBPF_OPTS(btf_dedup_opts, opts); |
2679 | struct btf *btf = linker->btf; |
2680 | const void *raw_data; |
2681 | int i, j, id, err; |
2682 | __u32 raw_sz; |
2683 | |
2684 | /* bail out if no BTF data was produced */ |
2685 | if (btf__type_cnt(btf: linker->btf) == 1) |
2686 | return 0; |
2687 | |
2688 | for (i = 1; i < linker->sec_cnt; i++) { |
2689 | struct dst_sec *sec = &linker->secs[i]; |
2690 | |
2691 | if (!sec->has_btf) |
2692 | continue; |
2693 | |
2694 | id = btf__add_datasec(btf, name: sec->sec_name, byte_sz: sec->sec_sz); |
2695 | if (id < 0) { |
2696 | pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n" , |
2697 | sec->sec_name, id); |
2698 | return id; |
2699 | } |
2700 | |
2701 | for (j = 0; j < sec->sec_var_cnt; j++) { |
2702 | struct btf_var_secinfo *vi = &sec->sec_vars[j]; |
2703 | |
2704 | if (btf__add_datasec_var_info(btf, var_type_id: vi->type, offset: vi->offset, byte_sz: vi->size)) |
2705 | return -EINVAL; |
2706 | } |
2707 | } |
2708 | |
2709 | err = finalize_btf_ext(linker); |
2710 | if (err) { |
2711 | pr_warn(".BTF.ext generation failed: %d\n" , err); |
2712 | return err; |
2713 | } |
2714 | |
2715 | opts.btf_ext = linker->btf_ext; |
2716 | err = btf__dedup(btf: linker->btf, opts: &opts); |
2717 | if (err) { |
2718 | pr_warn("BTF dedup failed: %d\n" , err); |
2719 | return err; |
2720 | } |
2721 | |
2722 | /* Emit .BTF section */ |
2723 | raw_data = btf__raw_data(btf: linker->btf, size: &raw_sz); |
2724 | if (!raw_data) |
2725 | return -ENOMEM; |
2726 | |
2727 | err = emit_elf_data_sec(linker, BTF_ELF_SEC, align: 8, raw_data, raw_sz); |
2728 | if (err) { |
2729 | pr_warn("failed to write out .BTF ELF section: %d\n" , err); |
2730 | return err; |
2731 | } |
2732 | |
2733 | /* Emit .BTF.ext section */ |
2734 | if (linker->btf_ext) { |
2735 | raw_data = btf_ext__raw_data(btf_ext: linker->btf_ext, size: &raw_sz); |
2736 | if (!raw_data) |
2737 | return -ENOMEM; |
2738 | |
2739 | err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, align: 8, raw_data, raw_sz); |
2740 | if (err) { |
2741 | pr_warn("failed to write out .BTF.ext ELF section: %d\n" , err); |
2742 | return err; |
2743 | } |
2744 | } |
2745 | |
2746 | return 0; |
2747 | } |
2748 | |
2749 | static int emit_btf_ext_data(struct bpf_linker *linker, void *output, |
2750 | const char *sec_name, struct btf_ext_sec_data *sec_data) |
2751 | { |
2752 | struct btf_ext_info_sec *sec_info; |
2753 | void *cur = output; |
2754 | int str_off; |
2755 | size_t sz; |
2756 | |
2757 | if (!sec_data->rec_cnt) |
2758 | return 0; |
2759 | |
2760 | str_off = btf__add_str(btf: linker->btf, s: sec_name); |
2761 | if (str_off < 0) |
2762 | return -ENOMEM; |
2763 | |
2764 | sec_info = cur; |
2765 | sec_info->sec_name_off = str_off; |
2766 | sec_info->num_info = sec_data->rec_cnt; |
2767 | cur += sizeof(struct btf_ext_info_sec); |
2768 | |
2769 | sz = sec_data->rec_cnt * sec_data->rec_sz; |
2770 | memcpy(cur, sec_data->recs, sz); |
2771 | cur += sz; |
2772 | |
2773 | return cur - output; |
2774 | } |
2775 | |
2776 | static int finalize_btf_ext(struct bpf_linker *linker) |
2777 | { |
2778 | size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0; |
2779 | size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0; |
2780 | struct btf_ext_header *hdr; |
2781 | void *data, *cur; |
2782 | int i, err, sz; |
2783 | |
2784 | /* validate that all sections have the same .BTF.ext record sizes |
2785 | * and calculate total data size for each type of data (func info, |
2786 | * line info, core relos) |
2787 | */ |
2788 | for (i = 1; i < linker->sec_cnt; i++) { |
2789 | struct dst_sec *sec = &linker->secs[i]; |
2790 | |
2791 | if (sec->func_info.rec_cnt) { |
2792 | if (func_rec_sz == 0) |
2793 | func_rec_sz = sec->func_info.rec_sz; |
2794 | if (func_rec_sz != sec->func_info.rec_sz) { |
2795 | pr_warn("mismatch in func_info record size %zu != %u\n" , |
2796 | func_rec_sz, sec->func_info.rec_sz); |
2797 | return -EINVAL; |
2798 | } |
2799 | |
2800 | funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt; |
2801 | } |
2802 | if (sec->line_info.rec_cnt) { |
2803 | if (line_rec_sz == 0) |
2804 | line_rec_sz = sec->line_info.rec_sz; |
2805 | if (line_rec_sz != sec->line_info.rec_sz) { |
2806 | pr_warn("mismatch in line_info record size %zu != %u\n" , |
2807 | line_rec_sz, sec->line_info.rec_sz); |
2808 | return -EINVAL; |
2809 | } |
2810 | |
2811 | lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt; |
2812 | } |
2813 | if (sec->core_relo_info.rec_cnt) { |
2814 | if (core_relo_rec_sz == 0) |
2815 | core_relo_rec_sz = sec->core_relo_info.rec_sz; |
2816 | if (core_relo_rec_sz != sec->core_relo_info.rec_sz) { |
2817 | pr_warn("mismatch in core_relo_info record size %zu != %u\n" , |
2818 | core_relo_rec_sz, sec->core_relo_info.rec_sz); |
2819 | return -EINVAL; |
2820 | } |
2821 | |
2822 | core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt; |
2823 | } |
2824 | } |
2825 | |
2826 | if (!funcs_sz && !lines_sz && !core_relos_sz) |
2827 | return 0; |
2828 | |
2829 | total_sz += sizeof(struct btf_ext_header); |
2830 | if (funcs_sz) { |
2831 | funcs_sz += sizeof(__u32); /* record size prefix */ |
2832 | total_sz += funcs_sz; |
2833 | } |
2834 | if (lines_sz) { |
2835 | lines_sz += sizeof(__u32); /* record size prefix */ |
2836 | total_sz += lines_sz; |
2837 | } |
2838 | if (core_relos_sz) { |
2839 | core_relos_sz += sizeof(__u32); /* record size prefix */ |
2840 | total_sz += core_relos_sz; |
2841 | } |
2842 | |
2843 | cur = data = calloc(1, total_sz); |
2844 | if (!data) |
2845 | return -ENOMEM; |
2846 | |
2847 | hdr = cur; |
2848 | hdr->magic = BTF_MAGIC; |
2849 | hdr->version = BTF_VERSION; |
2850 | hdr->flags = 0; |
2851 | hdr->hdr_len = sizeof(struct btf_ext_header); |
2852 | cur += sizeof(struct btf_ext_header); |
2853 | |
2854 | /* All offsets are in bytes relative to the end of this header */ |
2855 | hdr->func_info_off = 0; |
2856 | hdr->func_info_len = funcs_sz; |
2857 | hdr->line_info_off = funcs_sz; |
2858 | hdr->line_info_len = lines_sz; |
2859 | hdr->core_relo_off = funcs_sz + lines_sz; |
2860 | hdr->core_relo_len = core_relos_sz; |
2861 | |
2862 | if (funcs_sz) { |
2863 | *(__u32 *)cur = func_rec_sz; |
2864 | cur += sizeof(__u32); |
2865 | |
2866 | for (i = 1; i < linker->sec_cnt; i++) { |
2867 | struct dst_sec *sec = &linker->secs[i]; |
2868 | |
2869 | sz = emit_btf_ext_data(linker, output: cur, sec_name: sec->sec_name, sec_data: &sec->func_info); |
2870 | if (sz < 0) { |
2871 | err = sz; |
2872 | goto out; |
2873 | } |
2874 | |
2875 | cur += sz; |
2876 | } |
2877 | } |
2878 | |
2879 | if (lines_sz) { |
2880 | *(__u32 *)cur = line_rec_sz; |
2881 | cur += sizeof(__u32); |
2882 | |
2883 | for (i = 1; i < linker->sec_cnt; i++) { |
2884 | struct dst_sec *sec = &linker->secs[i]; |
2885 | |
2886 | sz = emit_btf_ext_data(linker, output: cur, sec_name: sec->sec_name, sec_data: &sec->line_info); |
2887 | if (sz < 0) { |
2888 | err = sz; |
2889 | goto out; |
2890 | } |
2891 | |
2892 | cur += sz; |
2893 | } |
2894 | } |
2895 | |
2896 | if (core_relos_sz) { |
2897 | *(__u32 *)cur = core_relo_rec_sz; |
2898 | cur += sizeof(__u32); |
2899 | |
2900 | for (i = 1; i < linker->sec_cnt; i++) { |
2901 | struct dst_sec *sec = &linker->secs[i]; |
2902 | |
2903 | sz = emit_btf_ext_data(linker, output: cur, sec_name: sec->sec_name, sec_data: &sec->core_relo_info); |
2904 | if (sz < 0) { |
2905 | err = sz; |
2906 | goto out; |
2907 | } |
2908 | |
2909 | cur += sz; |
2910 | } |
2911 | } |
2912 | |
2913 | linker->btf_ext = btf_ext__new(data, size: total_sz); |
2914 | err = libbpf_get_error(ptr: linker->btf_ext); |
2915 | if (err) { |
2916 | linker->btf_ext = NULL; |
2917 | pr_warn("failed to parse final .BTF.ext data: %d\n" , err); |
2918 | goto out; |
2919 | } |
2920 | |
2921 | out: |
2922 | free(data); |
2923 | return err; |
2924 | } |
2925 | |