1 | /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ |
2 | |
3 | /* |
4 | * Internal libbpf helpers. |
5 | * |
6 | * Copyright (c) 2019 Facebook |
7 | */ |
8 | |
9 | #ifndef __LIBBPF_LIBBPF_INTERNAL_H |
10 | #define __LIBBPF_LIBBPF_INTERNAL_H |
11 | |
12 | #include <stdlib.h> |
13 | #include <limits.h> |
14 | #include <errno.h> |
15 | #include <linux/err.h> |
16 | #include <fcntl.h> |
17 | #include <unistd.h> |
18 | #include <sys/syscall.h> |
19 | #include <libelf.h> |
20 | #include "relo_core.h" |
21 | |
22 | /* Android's libc doesn't support AT_EACCESS in faccessat() implementation |
23 | * ([0]), and just returns -EINVAL even if file exists and is accessible. |
24 | * See [1] for issues caused by this. |
25 | * |
26 | * So just redefine it to 0 on Android. |
27 | * |
28 | * [0] https://android.googlesource.com/platform/bionic/+/refs/heads/android13-release/libc/bionic/faccessat.cpp#50 |
29 | * [1] https://github.com/libbpf/libbpf-bootstrap/issues/250#issuecomment-1911324250 |
30 | */ |
31 | #ifdef __ANDROID__ |
32 | #undef AT_EACCESS |
33 | #define AT_EACCESS 0 |
34 | #endif |
35 | |
36 | /* make sure libbpf doesn't use kernel-only integer typedefs */ |
37 | #pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 |
38 | |
39 | /* prevent accidental re-addition of reallocarray() */ |
40 | #pragma GCC poison reallocarray |
41 | |
42 | #include "libbpf.h" |
43 | #include "btf.h" |
44 | |
45 | #ifndef EM_BPF |
46 | #define EM_BPF 247 |
47 | #endif |
48 | |
49 | #ifndef R_BPF_64_64 |
50 | #define R_BPF_64_64 1 |
51 | #endif |
52 | #ifndef R_BPF_64_ABS64 |
53 | #define R_BPF_64_ABS64 2 |
54 | #endif |
55 | #ifndef R_BPF_64_ABS32 |
56 | #define R_BPF_64_ABS32 3 |
57 | #endif |
58 | #ifndef R_BPF_64_32 |
59 | #define R_BPF_64_32 10 |
60 | #endif |
61 | |
62 | #ifndef SHT_LLVM_ADDRSIG |
63 | #define SHT_LLVM_ADDRSIG 0x6FFF4C03 |
64 | #endif |
65 | |
66 | /* if libelf is old and doesn't support mmap(), fall back to read() */ |
67 | #ifndef ELF_C_READ_MMAP |
68 | #define ELF_C_READ_MMAP ELF_C_READ |
69 | #endif |
70 | |
71 | /* Older libelf all end up in this expression, for both 32 and 64 bit */ |
72 | #ifndef ELF64_ST_VISIBILITY |
73 | #define ELF64_ST_VISIBILITY(o) ((o) & 0x03) |
74 | #endif |
75 | |
76 | #define BTF_INFO_ENC(kind, kind_flag, vlen) \ |
77 | ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) |
78 | #define BTF_TYPE_ENC(name, info, size_or_type) (name), (info), (size_or_type) |
79 | #define BTF_INT_ENC(encoding, bits_offset, nr_bits) \ |
80 | ((encoding) << 24 | (bits_offset) << 16 | (nr_bits)) |
81 | #define BTF_TYPE_INT_ENC(name, encoding, bits_offset, bits, sz) \ |
82 | BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_INT, 0, 0), sz), \ |
83 | BTF_INT_ENC(encoding, bits_offset, bits) |
84 | #define BTF_MEMBER_ENC(name, type, bits_offset) (name), (type), (bits_offset) |
85 | #define BTF_PARAM_ENC(name, type) (name), (type) |
86 | #define BTF_VAR_SECINFO_ENC(type, offset, size) (type), (offset), (size) |
87 | #define BTF_TYPE_FLOAT_ENC(name, sz) \ |
88 | BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz) |
89 | #define BTF_TYPE_DECL_TAG_ENC(value, type, component_idx) \ |
90 | BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 0, 0), type), (component_idx) |
91 | #define BTF_TYPE_TYPE_TAG_ENC(value, type) \ |
92 | BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TYPE_TAG, 0, 0), type) |
93 | |
94 | #ifndef likely |
95 | #define likely(x) __builtin_expect(!!(x), 1) |
96 | #endif |
97 | #ifndef unlikely |
98 | #define unlikely(x) __builtin_expect(!!(x), 0) |
99 | #endif |
100 | #ifndef min |
101 | # define min(x, y) ((x) < (y) ? (x) : (y)) |
102 | #endif |
103 | #ifndef max |
104 | # define max(x, y) ((x) < (y) ? (y) : (x)) |
105 | #endif |
106 | #ifndef offsetofend |
107 | # define offsetofend(TYPE, FIELD) \ |
108 | (offsetof(TYPE, FIELD) + sizeof(((TYPE *)0)->FIELD)) |
109 | #endif |
110 | #ifndef __alias |
111 | #define __alias(symbol) __attribute__((alias(#symbol))) |
112 | #endif |
113 | |
114 | /* Check whether a string `str` has prefix `pfx`, regardless if `pfx` is |
115 | * a string literal known at compilation time or char * pointer known only at |
116 | * runtime. |
117 | */ |
118 | #define str_has_pfx(str, pfx) \ |
119 | (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0) |
120 | |
121 | /* suffix check */ |
122 | static inline bool str_has_sfx(const char *str, const char *sfx) |
123 | { |
124 | size_t str_len = strlen(str); |
125 | size_t sfx_len = strlen(sfx); |
126 | |
127 | if (sfx_len > str_len) |
128 | return false; |
129 | return strcmp(str + str_len - sfx_len, sfx) == 0; |
130 | } |
131 | |
132 | /* Symbol versioning is different between static and shared library. |
133 | * Properly versioned symbols are needed for shared library, but |
134 | * only the symbol of the new version is needed for static library. |
135 | * Starting with GNU C 10, use symver attribute instead of .symver assembler |
136 | * directive, which works better with GCC LTO builds. |
137 | */ |
138 | #if defined(SHARED) && defined(__GNUC__) && __GNUC__ >= 10 |
139 | |
140 | #define DEFAULT_VERSION(internal_name, api_name, version) \ |
141 | __attribute__((symver(#api_name "@@" #version))) |
142 | #define COMPAT_VERSION(internal_name, api_name, version) \ |
143 | __attribute__((symver(#api_name "@" #version))) |
144 | |
145 | #elif defined(SHARED) |
146 | |
147 | #define COMPAT_VERSION(internal_name, api_name, version) \ |
148 | asm(".symver " #internal_name "," #api_name "@" #version); |
149 | #define DEFAULT_VERSION(internal_name, api_name, version) \ |
150 | asm(".symver " #internal_name "," #api_name "@@" #version); |
151 | |
152 | #else /* !SHARED */ |
153 | |
154 | #define COMPAT_VERSION(internal_name, api_name, version) |
155 | #define DEFAULT_VERSION(internal_name, api_name, version) \ |
156 | extern typeof(internal_name) api_name \ |
157 | __attribute__((alias(#internal_name))); |
158 | |
159 | #endif |
160 | |
161 | extern void libbpf_print(enum libbpf_print_level level, |
162 | const char *format, ...) |
163 | __attribute__((format(printf, 2, 3))); |
164 | |
165 | #define __pr(level, fmt, ...) \ |
166 | do { \ |
167 | libbpf_print(level, "libbpf: " fmt, ##__VA_ARGS__); \ |
168 | } while (0) |
169 | |
170 | #define pr_warn(fmt, ...) __pr(LIBBPF_WARN, fmt, ##__VA_ARGS__) |
171 | #define pr_info(fmt, ...) __pr(LIBBPF_INFO, fmt, ##__VA_ARGS__) |
172 | #define pr_debug(fmt, ...) __pr(LIBBPF_DEBUG, fmt, ##__VA_ARGS__) |
173 | |
174 | #ifndef __has_builtin |
175 | #define __has_builtin(x) 0 |
176 | #endif |
177 | |
178 | struct bpf_link { |
179 | int (*detach)(struct bpf_link *link); |
180 | void (*dealloc)(struct bpf_link *link); |
181 | char *pin_path; /* NULL, if not pinned */ |
182 | int fd; /* hook FD, -1 if not applicable */ |
183 | bool disconnected; |
184 | }; |
185 | |
186 | /* |
187 | * Re-implement glibc's reallocarray() for libbpf internal-only use. |
188 | * reallocarray(), unfortunately, is not available in all versions of glibc, |
189 | * so requires extra feature detection and using reallocarray() stub from |
190 | * <tools/libc_compat.h> and COMPAT_NEED_REALLOCARRAY. All this complicates |
191 | * build of libbpf unnecessarily and is just a maintenance burden. Instead, |
192 | * it's trivial to implement libbpf-specific internal version and use it |
193 | * throughout libbpf. |
194 | */ |
195 | static inline void *libbpf_reallocarray(void *ptr, size_t nmemb, size_t size) |
196 | { |
197 | size_t total; |
198 | |
199 | #if __has_builtin(__builtin_mul_overflow) |
200 | if (unlikely(__builtin_mul_overflow(nmemb, size, &total))) |
201 | return NULL; |
202 | #else |
203 | if (size == 0 || nmemb > ULONG_MAX / size) |
204 | return NULL; |
205 | total = nmemb * size; |
206 | #endif |
207 | return realloc(ptr, total); |
208 | } |
209 | |
210 | /* Copy up to sz - 1 bytes from zero-terminated src string and ensure that dst |
211 | * is zero-terminated string no matter what (unless sz == 0, in which case |
212 | * it's a no-op). It's conceptually close to FreeBSD's strlcpy(), but differs |
213 | * in what is returned. Given this is internal helper, it's trivial to extend |
214 | * this, when necessary. Use this instead of strncpy inside libbpf source code. |
215 | */ |
216 | static inline void libbpf_strlcpy(char *dst, const char *src, size_t sz) |
217 | { |
218 | size_t i; |
219 | |
220 | if (sz == 0) |
221 | return; |
222 | |
223 | sz--; |
224 | for (i = 0; i < sz && src[i]; i++) |
225 | dst[i] = src[i]; |
226 | dst[i] = '\0'; |
227 | } |
228 | |
229 | __u32 get_kernel_version(void); |
230 | |
231 | struct btf; |
232 | struct btf_type; |
233 | |
234 | struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id); |
235 | const char *btf_kind_str(const struct btf_type *t); |
236 | const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id); |
237 | |
238 | static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t) |
239 | { |
240 | return (enum btf_func_linkage)(int)btf_vlen(t); |
241 | } |
242 | |
243 | static inline __u32 btf_type_info(int kind, int vlen, int kflag) |
244 | { |
245 | return (kflag << 31) | (kind << 24) | vlen; |
246 | } |
247 | |
248 | enum map_def_parts { |
249 | MAP_DEF_MAP_TYPE = 0x001, |
250 | MAP_DEF_KEY_TYPE = 0x002, |
251 | MAP_DEF_KEY_SIZE = 0x004, |
252 | MAP_DEF_VALUE_TYPE = 0x008, |
253 | MAP_DEF_VALUE_SIZE = 0x010, |
254 | MAP_DEF_MAX_ENTRIES = 0x020, |
255 | MAP_DEF_MAP_FLAGS = 0x040, |
256 | MAP_DEF_NUMA_NODE = 0x080, |
257 | MAP_DEF_PINNING = 0x100, |
258 | MAP_DEF_INNER_MAP = 0x200, |
259 | = 0x400, |
260 | |
261 | MAP_DEF_ALL = 0x7ff, /* combination of all above */ |
262 | }; |
263 | |
264 | struct btf_map_def { |
265 | enum map_def_parts parts; |
266 | __u32 map_type; |
267 | __u32 key_type_id; |
268 | __u32 key_size; |
269 | __u32 value_type_id; |
270 | __u32 value_size; |
271 | __u32 max_entries; |
272 | __u32 map_flags; |
273 | __u32 numa_node; |
274 | __u32 pinning; |
275 | __u64 ; |
276 | }; |
277 | |
278 | int parse_btf_map_def(const char *map_name, struct btf *btf, |
279 | const struct btf_type *def_t, bool strict, |
280 | struct btf_map_def *map_def, struct btf_map_def *inner_def); |
281 | |
282 | void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, |
283 | size_t cur_cnt, size_t max_cnt, size_t add_cnt); |
284 | int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt); |
285 | |
286 | static inline bool libbpf_is_mem_zeroed(const char *p, ssize_t len) |
287 | { |
288 | while (len > 0) { |
289 | if (*p) |
290 | return false; |
291 | p++; |
292 | len--; |
293 | } |
294 | return true; |
295 | } |
296 | |
297 | static inline bool libbpf_validate_opts(const char *opts, |
298 | size_t opts_sz, size_t user_sz, |
299 | const char *type_name) |
300 | { |
301 | if (user_sz < sizeof(size_t)) { |
302 | pr_warn("%s size (%zu) is too small\n" , type_name, user_sz); |
303 | return false; |
304 | } |
305 | if (!libbpf_is_mem_zeroed(p: opts + opts_sz, len: (ssize_t)user_sz - opts_sz)) { |
306 | pr_warn("%s has non-zero extra bytes\n" , type_name); |
307 | return false; |
308 | } |
309 | return true; |
310 | } |
311 | |
312 | #define OPTS_VALID(opts, type) \ |
313 | (!(opts) || libbpf_validate_opts((const char *)opts, \ |
314 | offsetofend(struct type, \ |
315 | type##__last_field), \ |
316 | (opts)->sz, #type)) |
317 | #define OPTS_HAS(opts, field) \ |
318 | ((opts) && opts->sz >= offsetofend(typeof(*(opts)), field)) |
319 | #define OPTS_GET(opts, field, fallback_value) \ |
320 | (OPTS_HAS(opts, field) ? (opts)->field : fallback_value) |
321 | #define OPTS_SET(opts, field, value) \ |
322 | do { \ |
323 | if (OPTS_HAS(opts, field)) \ |
324 | (opts)->field = value; \ |
325 | } while (0) |
326 | |
327 | #define OPTS_ZEROED(opts, last_nonzero_field) \ |
328 | ({ \ |
329 | ssize_t __off = offsetofend(typeof(*(opts)), last_nonzero_field); \ |
330 | !(opts) || libbpf_is_mem_zeroed((const void *)opts + __off, \ |
331 | (opts)->sz - __off); \ |
332 | }) |
333 | |
334 | enum kern_feature_id { |
335 | /* v4.14: kernel support for program & map names. */ |
336 | FEAT_PROG_NAME, |
337 | /* v5.2: kernel support for global data sections. */ |
338 | FEAT_GLOBAL_DATA, |
339 | /* BTF support */ |
340 | FEAT_BTF, |
341 | /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ |
342 | FEAT_BTF_FUNC, |
343 | /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ |
344 | FEAT_BTF_DATASEC, |
345 | /* BTF_FUNC_GLOBAL is supported */ |
346 | FEAT_BTF_GLOBAL_FUNC, |
347 | /* BPF_F_MMAPABLE is supported for arrays */ |
348 | FEAT_ARRAY_MMAP, |
349 | /* kernel support for expected_attach_type in BPF_PROG_LOAD */ |
350 | FEAT_EXP_ATTACH_TYPE, |
351 | /* bpf_probe_read_{kernel,user}[_str] helpers */ |
352 | FEAT_PROBE_READ_KERN, |
353 | /* BPF_PROG_BIND_MAP is supported */ |
354 | FEAT_PROG_BIND_MAP, |
355 | /* Kernel support for module BTFs */ |
356 | FEAT_MODULE_BTF, |
357 | /* BTF_KIND_FLOAT support */ |
358 | FEAT_BTF_FLOAT, |
359 | /* BPF perf link support */ |
360 | FEAT_PERF_LINK, |
361 | /* BTF_KIND_DECL_TAG support */ |
362 | FEAT_BTF_DECL_TAG, |
363 | /* BTF_KIND_TYPE_TAG support */ |
364 | FEAT_BTF_TYPE_TAG, |
365 | /* memcg-based accounting for BPF maps and progs */ |
366 | FEAT_MEMCG_ACCOUNT, |
367 | /* BPF cookie (bpf_get_attach_cookie() BPF helper) support */ |
368 | FEAT_BPF_COOKIE, |
369 | /* BTF_KIND_ENUM64 support and BTF_KIND_ENUM kflag support */ |
370 | FEAT_BTF_ENUM64, |
371 | /* Kernel uses syscall wrapper (CONFIG_ARCH_HAS_SYSCALL_WRAPPER) */ |
372 | FEAT_SYSCALL_WRAPPER, |
373 | /* BPF multi-uprobe link support */ |
374 | FEAT_UPROBE_MULTI_LINK, |
375 | /* Kernel supports arg:ctx tag (__arg_ctx) for global subprogs natively */ |
376 | FEAT_ARG_CTX_TAG, |
377 | /* Kernel supports '?' at the front of datasec names */ |
378 | FEAT_BTF_QMARK_DATASEC, |
379 | __FEAT_CNT, |
380 | }; |
381 | |
382 | enum kern_feature_result { |
383 | FEAT_UNKNOWN = 0, |
384 | FEAT_SUPPORTED = 1, |
385 | FEAT_MISSING = 2, |
386 | }; |
387 | |
388 | struct kern_feature_cache { |
389 | enum kern_feature_result res[__FEAT_CNT]; |
390 | int token_fd; |
391 | }; |
392 | |
393 | bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id); |
394 | bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id); |
395 | |
396 | int probe_kern_syscall_wrapper(int token_fd); |
397 | int probe_memcg_account(int token_fd); |
398 | int bump_rlimit_memlock(void); |
399 | |
400 | int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz); |
401 | int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz); |
402 | int libbpf__load_raw_btf(const char *raw_types, size_t types_len, |
403 | const char *str_sec, size_t str_len, |
404 | int token_fd); |
405 | int btf_load_into_kernel(struct btf *btf, |
406 | char *log_buf, size_t log_sz, __u32 log_level, |
407 | int token_fd); |
408 | |
409 | struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf); |
410 | void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, |
411 | const char **prefix, int *kind); |
412 | |
413 | struct btf_ext_info { |
414 | /* |
415 | * info points to the individual info section (e.g. func_info and |
416 | * line_info) from the .BTF.ext. It does not include the __u32 rec_size. |
417 | */ |
418 | void *info; |
419 | __u32 rec_size; |
420 | __u32 len; |
421 | /* optional (maintained internally by libbpf) mapping between .BTF.ext |
422 | * section and corresponding ELF section. This is used to join |
423 | * information like CO-RE relocation records with corresponding BPF |
424 | * programs defined in ELF sections |
425 | */ |
426 | __u32 *sec_idxs; |
427 | int sec_cnt; |
428 | }; |
429 | |
430 | #define for_each_btf_ext_sec(seg, sec) \ |
431 | for (sec = (seg)->info; \ |
432 | (void *)sec < (seg)->info + (seg)->len; \ |
433 | sec = (void *)sec + sizeof(struct btf_ext_info_sec) + \ |
434 | (seg)->rec_size * sec->num_info) |
435 | |
436 | #define for_each_btf_ext_rec(seg, sec, i, rec) \ |
437 | for (i = 0, rec = (void *)&(sec)->data; \ |
438 | i < (sec)->num_info; \ |
439 | i++, rec = (void *)rec + (seg)->rec_size) |
440 | |
441 | /* |
442 | * The .BTF.ext ELF section layout defined as |
443 | * struct btf_ext_header |
444 | * func_info subsection |
445 | * |
446 | * The func_info subsection layout: |
447 | * record size for struct bpf_func_info in the func_info subsection |
448 | * struct btf_sec_func_info for section #1 |
449 | * a list of bpf_func_info records for section #1 |
450 | * where struct bpf_func_info mimics one in include/uapi/linux/bpf.h |
451 | * but may not be identical |
452 | * struct btf_sec_func_info for section #2 |
453 | * a list of bpf_func_info records for section #2 |
454 | * ...... |
455 | * |
456 | * Note that the bpf_func_info record size in .BTF.ext may not |
457 | * be the same as the one defined in include/uapi/linux/bpf.h. |
458 | * The loader should ensure that record_size meets minimum |
459 | * requirement and pass the record as is to the kernel. The |
460 | * kernel will handle the func_info properly based on its contents. |
461 | */ |
462 | struct { |
463 | __u16 ; |
464 | __u8 ; |
465 | __u8 ; |
466 | __u32 ; |
467 | |
468 | /* All offsets are in bytes relative to the end of this header */ |
469 | __u32 ; |
470 | __u32 ; |
471 | __u32 ; |
472 | __u32 ; |
473 | |
474 | /* optional part of .BTF.ext header */ |
475 | __u32 ; |
476 | __u32 ; |
477 | }; |
478 | |
479 | struct btf_ext { |
480 | union { |
481 | struct btf_ext_header *hdr; |
482 | void *data; |
483 | }; |
484 | struct btf_ext_info func_info; |
485 | struct btf_ext_info line_info; |
486 | struct btf_ext_info core_relo_info; |
487 | __u32 data_size; |
488 | }; |
489 | |
490 | struct btf_ext_info_sec { |
491 | __u32 sec_name_off; |
492 | __u32 num_info; |
493 | /* Followed by num_info * record_size number of bytes */ |
494 | __u8 data[]; |
495 | }; |
496 | |
497 | /* The minimum bpf_func_info checked by the loader */ |
498 | struct bpf_func_info_min { |
499 | __u32 insn_off; |
500 | __u32 type_id; |
501 | }; |
502 | |
503 | /* The minimum bpf_line_info checked by the loader */ |
504 | struct bpf_line_info_min { |
505 | __u32 insn_off; |
506 | __u32 file_name_off; |
507 | __u32 line_off; |
508 | __u32 line_col; |
509 | }; |
510 | |
511 | |
512 | typedef int (*type_id_visit_fn)(__u32 *type_id, void *ctx); |
513 | typedef int (*str_off_visit_fn)(__u32 *str_off, void *ctx); |
514 | int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ctx); |
515 | int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ctx); |
516 | int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx); |
517 | int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx); |
518 | __s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name, |
519 | __u32 kind); |
520 | |
521 | typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type, |
522 | const char *sym_name, void *ctx); |
523 | |
524 | int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *arg); |
525 | |
526 | /* handle direct returned errors */ |
527 | static inline int libbpf_err(int ret) |
528 | { |
529 | if (ret < 0) |
530 | errno = -ret; |
531 | return ret; |
532 | } |
533 | |
534 | /* handle errno-based (e.g., syscall or libc) errors according to libbpf's |
535 | * strict mode settings |
536 | */ |
537 | static inline int libbpf_err_errno(int ret) |
538 | { |
539 | /* errno is already assumed to be set on error */ |
540 | return ret < 0 ? -errno : ret; |
541 | } |
542 | |
543 | /* handle error for pointer-returning APIs, err is assumed to be < 0 always */ |
544 | static inline void *libbpf_err_ptr(int err) |
545 | { |
546 | /* set errno on error, this doesn't break anything */ |
547 | errno = -err; |
548 | return NULL; |
549 | } |
550 | |
551 | /* handle pointer-returning APIs' error handling */ |
552 | static inline void *libbpf_ptr(void *ret) |
553 | { |
554 | /* set errno on error, this doesn't break anything */ |
555 | if (IS_ERR(ret)) |
556 | errno = -PTR_ERR(ret); |
557 | |
558 | return IS_ERR(ptr: ret) ? NULL : ret; |
559 | } |
560 | |
561 | static inline bool str_is_empty(const char *s) |
562 | { |
563 | return !s || !s[0]; |
564 | } |
565 | |
566 | static inline bool is_ldimm64_insn(struct bpf_insn *insn) |
567 | { |
568 | return insn->code == (BPF_LD | BPF_IMM | BPF_DW); |
569 | } |
570 | |
571 | /* Unconditionally dup FD, ensuring it doesn't use [0, 2] range. |
572 | * Original FD is not closed or altered in any other way. |
573 | * Preserves original FD value, if it's invalid (negative). |
574 | */ |
575 | static inline int dup_good_fd(int fd) |
576 | { |
577 | if (fd < 0) |
578 | return fd; |
579 | return fcntl(fd, F_DUPFD_CLOEXEC, 3); |
580 | } |
581 | |
582 | /* if fd is stdin, stdout, or stderr, dup to a fd greater than 2 |
583 | * Takes ownership of the fd passed in, and closes it if calling |
584 | * fcntl(fd, F_DUPFD_CLOEXEC, 3). |
585 | */ |
586 | static inline int ensure_good_fd(int fd) |
587 | { |
588 | int old_fd = fd, saved_errno; |
589 | |
590 | if (fd < 0) |
591 | return fd; |
592 | if (fd < 3) { |
593 | fd = dup_good_fd(fd); |
594 | saved_errno = errno; |
595 | close(old_fd); |
596 | errno = saved_errno; |
597 | if (fd < 0) { |
598 | pr_warn("failed to dup FD %d to FD > 2: %d\n" , old_fd, -saved_errno); |
599 | errno = saved_errno; |
600 | } |
601 | } |
602 | return fd; |
603 | } |
604 | |
605 | static inline int sys_dup2(int oldfd, int newfd) |
606 | { |
607 | #ifdef __NR_dup2 |
608 | return syscall(__NR_dup2, oldfd, newfd); |
609 | #else |
610 | return syscall(__NR_dup3, oldfd, newfd, 0); |
611 | #endif |
612 | } |
613 | |
614 | /* Point *fixed_fd* to the same file that *tmp_fd* points to. |
615 | * Regardless of success, *tmp_fd* is closed. |
616 | * Whatever *fixed_fd* pointed to is closed silently. |
617 | */ |
618 | static inline int reuse_fd(int fixed_fd, int tmp_fd) |
619 | { |
620 | int err; |
621 | |
622 | err = sys_dup2(oldfd: tmp_fd, newfd: fixed_fd); |
623 | err = err < 0 ? -errno : 0; |
624 | close(tmp_fd); /* clean up temporary FD */ |
625 | return err; |
626 | } |
627 | |
628 | /* The following two functions are exposed to bpftool */ |
629 | int bpf_core_add_cands(struct bpf_core_cand *local_cand, |
630 | size_t local_essent_len, |
631 | const struct btf *targ_btf, |
632 | const char *targ_btf_name, |
633 | int targ_start_id, |
634 | struct bpf_core_cand_list *cands); |
635 | void bpf_core_free_cands(struct bpf_core_cand_list *cands); |
636 | |
637 | struct usdt_manager *usdt_manager_new(struct bpf_object *obj); |
638 | void usdt_manager_free(struct usdt_manager *man); |
639 | struct bpf_link * usdt_manager_attach_usdt(struct usdt_manager *man, |
640 | const struct bpf_program *prog, |
641 | pid_t pid, const char *path, |
642 | const char *usdt_provider, const char *usdt_name, |
643 | __u64 usdt_cookie); |
644 | |
645 | static inline bool is_pow_of_2(size_t x) |
646 | { |
647 | return x && (x & (x - 1)) == 0; |
648 | } |
649 | |
650 | #define PROG_LOAD_ATTEMPTS 5 |
651 | int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts); |
652 | |
653 | bool glob_match(const char *str, const char *pat); |
654 | |
655 | long elf_find_func_offset(Elf *elf, const char *binary_path, const char *name); |
656 | long elf_find_func_offset_from_file(const char *binary_path, const char *name); |
657 | |
658 | struct elf_fd { |
659 | Elf *elf; |
660 | int fd; |
661 | }; |
662 | |
663 | int elf_open(const char *binary_path, struct elf_fd *elf_fd); |
664 | void elf_close(struct elf_fd *elf_fd); |
665 | |
666 | int elf_resolve_syms_offsets(const char *binary_path, int cnt, |
667 | const char **syms, unsigned long **poffsets, |
668 | int st_type); |
669 | int elf_resolve_pattern_offsets(const char *binary_path, const char *pattern, |
670 | unsigned long **poffsets, size_t *pcnt); |
671 | |
672 | int probe_fd(int fd); |
673 | |
674 | #endif /* __LIBBPF_LIBBPF_INTERNAL_H */ |
675 | |