1 | /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ |
2 | /* Copyright (c) 2021 Facebook */ |
3 | #ifndef __SKEL_INTERNAL_H |
4 | #define __SKEL_INTERNAL_H |
5 | |
6 | #ifdef __KERNEL__ |
7 | #include <linux/fdtable.h> |
8 | #include <linux/mm.h> |
9 | #include <linux/mman.h> |
10 | #include <linux/slab.h> |
11 | #include <linux/bpf.h> |
12 | #else |
13 | #include <unistd.h> |
14 | #include <sys/syscall.h> |
15 | #include <sys/mman.h> |
16 | #include <stdlib.h> |
17 | #include "bpf.h" |
18 | #endif |
19 | |
20 | #ifndef __NR_bpf |
21 | # if defined(__mips__) && defined(_ABIO32) |
22 | # define __NR_bpf 4355 |
23 | # elif defined(__mips__) && defined(_ABIN32) |
24 | # define __NR_bpf 6319 |
25 | # elif defined(__mips__) && defined(_ABI64) |
26 | # define __NR_bpf 5315 |
27 | # endif |
28 | #endif |
29 | |
30 | /* This file is a base header for auto-generated *.lskel.h files. |
31 | * Its contents will change and may become part of auto-generation in the future. |
32 | * |
33 | * The layout of bpf_[map|prog]_desc and bpf_loader_ctx is feature dependent |
34 | * and will change from one version of libbpf to another and features |
35 | * requested during loader program generation. |
36 | */ |
37 | struct bpf_map_desc { |
38 | /* output of the loader prog */ |
39 | int map_fd; |
40 | /* input for the loader prog */ |
41 | __u32 max_entries; |
42 | __aligned_u64 initial_value; |
43 | }; |
44 | struct bpf_prog_desc { |
45 | int prog_fd; |
46 | }; |
47 | |
48 | enum { |
49 | BPF_SKEL_KERNEL = (1ULL << 0), |
50 | }; |
51 | |
52 | struct bpf_loader_ctx { |
53 | __u32 sz; |
54 | __u32 flags; |
55 | __u32 log_level; |
56 | __u32 log_size; |
57 | __u64 log_buf; |
58 | }; |
59 | |
60 | struct bpf_load_and_run_opts { |
61 | struct bpf_loader_ctx *ctx; |
62 | const void *data; |
63 | const void *insns; |
64 | __u32 data_sz; |
65 | __u32 insns_sz; |
66 | const char *errstr; |
67 | }; |
68 | |
69 | long kern_sys_bpf(__u32 cmd, void *attr, __u32 attr_size); |
70 | |
71 | static inline int skel_sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, |
72 | unsigned int size) |
73 | { |
74 | #ifdef __KERNEL__ |
75 | return kern_sys_bpf(cmd, attr, attr_size: size); |
76 | #else |
77 | return syscall(__NR_bpf, cmd, attr, size); |
78 | #endif |
79 | } |
80 | |
81 | #ifdef __KERNEL__ |
82 | static inline int close(int fd) |
83 | { |
84 | return close_fd(fd); |
85 | } |
86 | |
87 | static inline void *skel_alloc(size_t size) |
88 | { |
89 | struct bpf_loader_ctx *ctx = kzalloc(size, GFP_KERNEL); |
90 | |
91 | if (!ctx) |
92 | return NULL; |
93 | ctx->flags |= BPF_SKEL_KERNEL; |
94 | return ctx; |
95 | } |
96 | |
97 | static inline void skel_free(const void *p) |
98 | { |
99 | kfree(objp: p); |
100 | } |
101 | |
102 | /* skel->bss/rodata maps are populated the following way: |
103 | * |
104 | * For kernel use: |
105 | * skel_prep_map_data() allocates kernel memory that kernel module can directly access. |
106 | * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value. |
107 | * The loader program will perform probe_read_kernel() from maps.rodata.initial_value. |
108 | * skel_finalize_map_data() sets skel->rodata to point to actual value in a bpf map and |
109 | * does maps.rodata.initial_value = ~0ULL to signal skel_free_map_data() that kvfree |
110 | * is not nessary. |
111 | * |
112 | * For user space: |
113 | * skel_prep_map_data() mmaps anon memory into skel->rodata that can be accessed directly. |
114 | * Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value. |
115 | * The loader program will perform copy_from_user() from maps.rodata.initial_value. |
116 | * skel_finalize_map_data() remaps bpf array map value from the kernel memory into |
117 | * skel->rodata address. |
118 | * |
119 | * The "bpftool gen skeleton -L" command generates lskel.h that is suitable for |
120 | * both kernel and user space. The generated loader program does |
121 | * either bpf_probe_read_kernel() or bpf_copy_from_user() from initial_value |
122 | * depending on bpf_loader_ctx->flags. |
123 | */ |
124 | static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) |
125 | { |
126 | if (addr != ~0ULL) |
127 | kvfree(addr: p); |
128 | /* When addr == ~0ULL the 'p' points to |
129 | * ((struct bpf_array *)map)->value. See skel_finalize_map_data. |
130 | */ |
131 | } |
132 | |
133 | static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) |
134 | { |
135 | void *addr; |
136 | |
137 | addr = kvmalloc(size: val_sz, GFP_KERNEL); |
138 | if (!addr) |
139 | return NULL; |
140 | memcpy(addr, val, val_sz); |
141 | return addr; |
142 | } |
143 | |
144 | static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) |
145 | { |
146 | struct bpf_map *map; |
147 | void *addr = NULL; |
148 | |
149 | kvfree(addr: (void *) (long) *init_val); |
150 | *init_val = ~0ULL; |
151 | |
152 | /* At this point bpf_load_and_run() finished without error and |
153 | * 'fd' is a valid bpf map FD. All sanity checks below should succeed. |
154 | */ |
155 | map = bpf_map_get(ufd: fd); |
156 | if (IS_ERR(ptr: map)) |
157 | return NULL; |
158 | if (map->map_type != BPF_MAP_TYPE_ARRAY) |
159 | goto out; |
160 | addr = ((struct bpf_array *)map)->value; |
161 | /* the addr stays valid, since FD is not closed */ |
162 | out: |
163 | bpf_map_put(map); |
164 | return addr; |
165 | } |
166 | |
167 | #else |
168 | |
169 | static inline void *skel_alloc(size_t size) |
170 | { |
171 | return calloc(1, size); |
172 | } |
173 | |
174 | static inline void skel_free(void *p) |
175 | { |
176 | free(p); |
177 | } |
178 | |
179 | static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) |
180 | { |
181 | munmap(p, sz); |
182 | } |
183 | |
184 | static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz) |
185 | { |
186 | void *addr; |
187 | |
188 | addr = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, |
189 | MAP_SHARED | MAP_ANONYMOUS, -1, 0); |
190 | if (addr == (void *) -1) |
191 | return NULL; |
192 | memcpy(addr, val, val_sz); |
193 | return addr; |
194 | } |
195 | |
196 | static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd) |
197 | { |
198 | void *addr; |
199 | |
200 | addr = mmap((void *) (long) *init_val, mmap_sz, flags, MAP_SHARED | MAP_FIXED, fd, 0); |
201 | if (addr == (void *) -1) |
202 | return NULL; |
203 | return addr; |
204 | } |
205 | #endif |
206 | |
207 | static inline int skel_closenz(int fd) |
208 | { |
209 | if (fd > 0) |
210 | return close(fd); |
211 | return -EINVAL; |
212 | } |
213 | |
214 | #ifndef offsetofend |
215 | #define offsetofend(TYPE, MEMBER) \ |
216 | (offsetof(TYPE, MEMBER) + sizeof((((TYPE *)0)->MEMBER))) |
217 | #endif |
218 | |
219 | static inline int skel_map_create(enum bpf_map_type map_type, |
220 | const char *map_name, |
221 | __u32 key_size, |
222 | __u32 value_size, |
223 | __u32 max_entries) |
224 | { |
225 | const size_t attr_sz = offsetofend(union bpf_attr, map_extra); |
226 | union bpf_attr attr; |
227 | |
228 | memset(&attr, 0, attr_sz); |
229 | |
230 | attr.map_type = map_type; |
231 | strncpy(p: attr.map_name, q: map_name, size: sizeof(attr.map_name)); |
232 | attr.key_size = key_size; |
233 | attr.value_size = value_size; |
234 | attr.max_entries = max_entries; |
235 | |
236 | return skel_sys_bpf(cmd: BPF_MAP_CREATE, attr: &attr, size: attr_sz); |
237 | } |
238 | |
239 | static inline int skel_map_update_elem(int fd, const void *key, |
240 | const void *value, __u64 flags) |
241 | { |
242 | const size_t attr_sz = offsetofend(union bpf_attr, flags); |
243 | union bpf_attr attr; |
244 | |
245 | memset(&attr, 0, attr_sz); |
246 | attr.map_fd = fd; |
247 | attr.key = (long) key; |
248 | attr.value = (long) value; |
249 | attr.flags = flags; |
250 | |
251 | return skel_sys_bpf(cmd: BPF_MAP_UPDATE_ELEM, attr: &attr, size: attr_sz); |
252 | } |
253 | |
254 | static inline int skel_map_delete_elem(int fd, const void *key) |
255 | { |
256 | const size_t attr_sz = offsetofend(union bpf_attr, flags); |
257 | union bpf_attr attr; |
258 | |
259 | memset(&attr, 0, attr_sz); |
260 | attr.map_fd = fd; |
261 | attr.key = (long)key; |
262 | |
263 | return skel_sys_bpf(cmd: BPF_MAP_DELETE_ELEM, attr: &attr, size: attr_sz); |
264 | } |
265 | |
266 | static inline int skel_map_get_fd_by_id(__u32 id) |
267 | { |
268 | const size_t attr_sz = offsetofend(union bpf_attr, flags); |
269 | union bpf_attr attr; |
270 | |
271 | memset(&attr, 0, attr_sz); |
272 | attr.map_id = id; |
273 | |
274 | return skel_sys_bpf(cmd: BPF_MAP_GET_FD_BY_ID, attr: &attr, size: attr_sz); |
275 | } |
276 | |
277 | static inline int skel_raw_tracepoint_open(const char *name, int prog_fd) |
278 | { |
279 | const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd); |
280 | union bpf_attr attr; |
281 | |
282 | memset(&attr, 0, attr_sz); |
283 | attr.raw_tracepoint.name = (long) name; |
284 | attr.raw_tracepoint.prog_fd = prog_fd; |
285 | |
286 | return skel_sys_bpf(cmd: BPF_RAW_TRACEPOINT_OPEN, attr: &attr, size: attr_sz); |
287 | } |
288 | |
289 | static inline int skel_link_create(int prog_fd, int target_fd, |
290 | enum bpf_attach_type attach_type) |
291 | { |
292 | const size_t attr_sz = offsetofend(union bpf_attr, link_create.iter_info_len); |
293 | union bpf_attr attr; |
294 | |
295 | memset(&attr, 0, attr_sz); |
296 | attr.link_create.prog_fd = prog_fd; |
297 | attr.link_create.target_fd = target_fd; |
298 | attr.link_create.attach_type = attach_type; |
299 | |
300 | return skel_sys_bpf(cmd: BPF_LINK_CREATE, attr: &attr, size: attr_sz); |
301 | } |
302 | |
303 | #ifdef __KERNEL__ |
304 | #define set_err |
305 | #else |
306 | #define set_err err = -errno |
307 | #endif |
308 | |
309 | static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) |
310 | { |
311 | const size_t prog_load_attr_sz = offsetofend(union bpf_attr, fd_array); |
312 | const size_t test_run_attr_sz = offsetofend(union bpf_attr, test); |
313 | int map_fd = -1, prog_fd = -1, key = 0, err; |
314 | union bpf_attr attr; |
315 | |
316 | err = map_fd = skel_map_create(map_type: BPF_MAP_TYPE_ARRAY, map_name: "__loader.map" , key_size: 4, value_size: opts->data_sz, max_entries: 1); |
317 | if (map_fd < 0) { |
318 | opts->errstr = "failed to create loader map" ; |
319 | set_err; |
320 | goto out; |
321 | } |
322 | |
323 | err = skel_map_update_elem(fd: map_fd, key: &key, value: opts->data, flags: 0); |
324 | if (err < 0) { |
325 | opts->errstr = "failed to update loader map" ; |
326 | set_err; |
327 | goto out; |
328 | } |
329 | |
330 | memset(&attr, 0, prog_load_attr_sz); |
331 | attr.prog_type = BPF_PROG_TYPE_SYSCALL; |
332 | attr.insns = (long) opts->insns; |
333 | attr.insn_cnt = opts->insns_sz / sizeof(struct bpf_insn); |
334 | attr.license = (long) "Dual BSD/GPL" ; |
335 | memcpy(attr.prog_name, "__loader.prog" , sizeof("__loader.prog" )); |
336 | attr.fd_array = (long) &map_fd; |
337 | attr.log_level = opts->ctx->log_level; |
338 | attr.log_size = opts->ctx->log_size; |
339 | attr.log_buf = opts->ctx->log_buf; |
340 | attr.prog_flags = BPF_F_SLEEPABLE; |
341 | err = prog_fd = skel_sys_bpf(cmd: BPF_PROG_LOAD, attr: &attr, size: prog_load_attr_sz); |
342 | if (prog_fd < 0) { |
343 | opts->errstr = "failed to load loader prog" ; |
344 | set_err; |
345 | goto out; |
346 | } |
347 | |
348 | memset(&attr, 0, test_run_attr_sz); |
349 | attr.test.prog_fd = prog_fd; |
350 | attr.test.ctx_in = (long) opts->ctx; |
351 | attr.test.ctx_size_in = opts->ctx->sz; |
352 | err = skel_sys_bpf(cmd: BPF_PROG_RUN, attr: &attr, size: test_run_attr_sz); |
353 | if (err < 0 || (int)attr.test.retval < 0) { |
354 | opts->errstr = "failed to execute loader prog" ; |
355 | if (err < 0) { |
356 | set_err; |
357 | } else { |
358 | err = (int)attr.test.retval; |
359 | #ifndef __KERNEL__ |
360 | errno = -err; |
361 | #endif |
362 | } |
363 | goto out; |
364 | } |
365 | err = 0; |
366 | out: |
367 | if (map_fd >= 0) |
368 | close(fd: map_fd); |
369 | if (prog_fd >= 0) |
370 | close(fd: prog_fd); |
371 | return err; |
372 | } |
373 | |
374 | #endif |
375 | |