1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (c) 2019 Facebook */ |
3 | |
4 | #include <linux/bpf.h> |
5 | #include <linux/bpf_verifier.h> |
6 | #include <linux/btf.h> |
7 | #include <linux/filter.h> |
8 | #include <linux/slab.h> |
9 | #include <linux/numa.h> |
10 | #include <linux/seq_file.h> |
11 | #include <linux/refcount.h> |
12 | #include <linux/mutex.h> |
13 | #include <linux/btf_ids.h> |
14 | #include <linux/rcupdate_wait.h> |
15 | |
16 | struct bpf_struct_ops_value { |
17 | struct bpf_struct_ops_common_value common; |
18 | char data[] ____cacheline_aligned_in_smp; |
19 | }; |
20 | |
21 | #define MAX_TRAMP_IMAGE_PAGES 8 |
22 | |
23 | struct bpf_struct_ops_map { |
24 | struct bpf_map map; |
25 | struct rcu_head rcu; |
26 | const struct bpf_struct_ops_desc *st_ops_desc; |
27 | /* protect map_update */ |
28 | struct mutex lock; |
29 | /* link has all the bpf_links that is populated |
30 | * to the func ptr of the kernel's struct |
31 | * (in kvalue.data). |
32 | */ |
33 | struct bpf_link **links; |
34 | u32 links_cnt; |
35 | u32 image_pages_cnt; |
36 | /* image_pages is an array of pages that has all the trampolines |
37 | * that stores the func args before calling the bpf_prog. |
38 | */ |
39 | void *image_pages[MAX_TRAMP_IMAGE_PAGES]; |
40 | /* The owner moduler's btf. */ |
41 | struct btf *btf; |
42 | /* uvalue->data stores the kernel struct |
43 | * (e.g. tcp_congestion_ops) that is more useful |
44 | * to userspace than the kvalue. For example, |
45 | * the bpf_prog's id is stored instead of the kernel |
46 | * address of a func ptr. |
47 | */ |
48 | struct bpf_struct_ops_value *uvalue; |
49 | /* kvalue.data stores the actual kernel's struct |
50 | * (e.g. tcp_congestion_ops) that will be |
51 | * registered to the kernel subsystem. |
52 | */ |
53 | struct bpf_struct_ops_value kvalue; |
54 | }; |
55 | |
56 | struct bpf_struct_ops_link { |
57 | struct bpf_link link; |
58 | struct bpf_map __rcu *map; |
59 | }; |
60 | |
61 | static DEFINE_MUTEX(update_mutex); |
62 | |
63 | #define VALUE_PREFIX "bpf_struct_ops_" |
64 | #define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1) |
65 | |
66 | const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = { |
67 | }; |
68 | |
69 | const struct bpf_prog_ops bpf_struct_ops_prog_ops = { |
70 | #ifdef CONFIG_NET |
71 | .test_run = bpf_struct_ops_test_run, |
72 | #endif |
73 | }; |
74 | |
75 | BTF_ID_LIST(st_ops_ids) |
76 | BTF_ID(struct, module) |
77 | BTF_ID(struct, bpf_struct_ops_common_value) |
78 | |
79 | enum { |
80 | IDX_MODULE_ID, |
81 | IDX_ST_OPS_COMMON_VALUE_ID, |
82 | }; |
83 | |
84 | extern struct btf *btf_vmlinux; |
85 | |
86 | static bool is_valid_value_type(struct btf *btf, s32 value_id, |
87 | const struct btf_type *type, |
88 | const char *value_name) |
89 | { |
90 | const struct btf_type *common_value_type; |
91 | const struct btf_member *member; |
92 | const struct btf_type *vt, *mt; |
93 | |
94 | vt = btf_type_by_id(btf, type_id: value_id); |
95 | if (btf_vlen(t: vt) != 2) { |
96 | pr_warn("The number of %s's members should be 2, but we get %d\n" , |
97 | value_name, btf_vlen(vt)); |
98 | return false; |
99 | } |
100 | member = btf_type_member(t: vt); |
101 | mt = btf_type_by_id(btf, type_id: member->type); |
102 | common_value_type = btf_type_by_id(btf: btf_vmlinux, |
103 | type_id: st_ops_ids[IDX_ST_OPS_COMMON_VALUE_ID]); |
104 | if (mt != common_value_type) { |
105 | pr_warn("The first member of %s should be bpf_struct_ops_common_value\n" , |
106 | value_name); |
107 | return false; |
108 | } |
109 | member++; |
110 | mt = btf_type_by_id(btf, type_id: member->type); |
111 | if (mt != type) { |
112 | pr_warn("The second member of %s should be %s\n" , |
113 | value_name, btf_name_by_offset(btf, type->name_off)); |
114 | return false; |
115 | } |
116 | |
117 | return true; |
118 | } |
119 | |
120 | static void *bpf_struct_ops_image_alloc(void) |
121 | { |
122 | void *image; |
123 | int err; |
124 | |
125 | err = bpf_jit_charge_modmem(PAGE_SIZE); |
126 | if (err) |
127 | return ERR_PTR(error: err); |
128 | image = arch_alloc_bpf_trampoline(PAGE_SIZE); |
129 | if (!image) { |
130 | bpf_jit_uncharge_modmem(PAGE_SIZE); |
131 | return ERR_PTR(error: -ENOMEM); |
132 | } |
133 | |
134 | return image; |
135 | } |
136 | |
137 | void bpf_struct_ops_image_free(void *image) |
138 | { |
139 | if (image) { |
140 | arch_free_bpf_trampoline(image, PAGE_SIZE); |
141 | bpf_jit_uncharge_modmem(PAGE_SIZE); |
142 | } |
143 | } |
144 | |
145 | #define MAYBE_NULL_SUFFIX "__nullable" |
146 | #define MAX_STUB_NAME 128 |
147 | |
148 | /* Return the type info of a stub function, if it exists. |
149 | * |
150 | * The name of a stub function is made up of the name of the struct_ops and |
151 | * the name of the function pointer member, separated by "__". For example, |
152 | * if the struct_ops type is named "foo_ops" and the function pointer |
153 | * member is named "bar", the stub function name would be "foo_ops__bar". |
154 | */ |
155 | static const struct btf_type * |
156 | find_stub_func_proto(const struct btf *btf, const char *st_op_name, |
157 | const char *member_name) |
158 | { |
159 | char stub_func_name[MAX_STUB_NAME]; |
160 | const struct btf_type *func_type; |
161 | s32 btf_id; |
162 | int cp; |
163 | |
164 | cp = snprintf(buf: stub_func_name, MAX_STUB_NAME, fmt: "%s__%s" , |
165 | st_op_name, member_name); |
166 | if (cp >= MAX_STUB_NAME) { |
167 | pr_warn("Stub function name too long\n" ); |
168 | return NULL; |
169 | } |
170 | btf_id = btf_find_by_name_kind(btf, name: stub_func_name, kind: BTF_KIND_FUNC); |
171 | if (btf_id < 0) |
172 | return NULL; |
173 | func_type = btf_type_by_id(btf, type_id: btf_id); |
174 | if (!func_type) |
175 | return NULL; |
176 | |
177 | return btf_type_by_id(btf, type_id: func_type->type); /* FUNC_PROTO */ |
178 | } |
179 | |
180 | /* Prepare argument info for every nullable argument of a member of a |
181 | * struct_ops type. |
182 | * |
183 | * Initialize a struct bpf_struct_ops_arg_info according to type info of |
184 | * the arguments of a stub function. (Check kCFI for more information about |
185 | * stub functions.) |
186 | * |
187 | * Each member in the struct_ops type has a struct bpf_struct_ops_arg_info |
188 | * to provide an array of struct bpf_ctx_arg_aux, which in turn provides |
189 | * the information that used by the verifier to check the arguments of the |
190 | * BPF struct_ops program assigned to the member. Here, we only care about |
191 | * the arguments that are marked as __nullable. |
192 | * |
193 | * The array of struct bpf_ctx_arg_aux is eventually assigned to |
194 | * prog->aux->ctx_arg_info of BPF struct_ops programs and passed to the |
195 | * verifier. (See check_struct_ops_btf_id()) |
196 | * |
197 | * arg_info->info will be the list of struct bpf_ctx_arg_aux if success. If |
198 | * fails, it will be kept untouched. |
199 | */ |
200 | static int prepare_arg_info(struct btf *btf, |
201 | const char *st_ops_name, |
202 | const char *member_name, |
203 | const struct btf_type *func_proto, |
204 | struct bpf_struct_ops_arg_info *arg_info) |
205 | { |
206 | const struct btf_type *stub_func_proto, *pointed_type; |
207 | const struct btf_param *stub_args, *args; |
208 | struct bpf_ctx_arg_aux *info, *info_buf; |
209 | u32 nargs, arg_no, info_cnt = 0; |
210 | u32 arg_btf_id; |
211 | int offset; |
212 | |
213 | stub_func_proto = find_stub_func_proto(btf, st_op_name: st_ops_name, member_name); |
214 | if (!stub_func_proto) |
215 | return 0; |
216 | |
217 | /* Check if the number of arguments of the stub function is the same |
218 | * as the number of arguments of the function pointer. |
219 | */ |
220 | nargs = btf_type_vlen(t: func_proto); |
221 | if (nargs != btf_type_vlen(t: stub_func_proto)) { |
222 | pr_warn("the number of arguments of the stub function %s__%s does not match the number of arguments of the member %s of struct %s\n" , |
223 | st_ops_name, member_name, member_name, st_ops_name); |
224 | return -EINVAL; |
225 | } |
226 | |
227 | if (!nargs) |
228 | return 0; |
229 | |
230 | args = btf_params(t: func_proto); |
231 | stub_args = btf_params(t: stub_func_proto); |
232 | |
233 | info_buf = kcalloc(n: nargs, size: sizeof(*info_buf), GFP_KERNEL); |
234 | if (!info_buf) |
235 | return -ENOMEM; |
236 | |
237 | /* Prepare info for every nullable argument */ |
238 | info = info_buf; |
239 | for (arg_no = 0; arg_no < nargs; arg_no++) { |
240 | /* Skip arguments that is not suffixed with |
241 | * "__nullable". |
242 | */ |
243 | if (!btf_param_match_suffix(btf, arg: &stub_args[arg_no], |
244 | MAYBE_NULL_SUFFIX)) |
245 | continue; |
246 | |
247 | /* Should be a pointer to struct */ |
248 | pointed_type = btf_type_resolve_ptr(btf, |
249 | id: args[arg_no].type, |
250 | res_id: &arg_btf_id); |
251 | if (!pointed_type || |
252 | !btf_type_is_struct(t: pointed_type)) { |
253 | pr_warn("stub function %s__%s has %s tagging to an unsupported type\n" , |
254 | st_ops_name, member_name, MAYBE_NULL_SUFFIX); |
255 | goto err_out; |
256 | } |
257 | |
258 | offset = btf_ctx_arg_offset(btf, func_proto, arg_no); |
259 | if (offset < 0) { |
260 | pr_warn("stub function %s__%s has an invalid trampoline ctx offset for arg#%u\n" , |
261 | st_ops_name, member_name, arg_no); |
262 | goto err_out; |
263 | } |
264 | |
265 | if (args[arg_no].type != stub_args[arg_no].type) { |
266 | pr_warn("arg#%u type in stub function %s__%s does not match with its original func_proto\n" , |
267 | arg_no, st_ops_name, member_name); |
268 | goto err_out; |
269 | } |
270 | |
271 | /* Fill the information of the new argument */ |
272 | info->reg_type = |
273 | PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL; |
274 | info->btf_id = arg_btf_id; |
275 | info->btf = btf; |
276 | info->offset = offset; |
277 | |
278 | info++; |
279 | info_cnt++; |
280 | } |
281 | |
282 | if (info_cnt) { |
283 | arg_info->info = info_buf; |
284 | arg_info->cnt = info_cnt; |
285 | } else { |
286 | kfree(objp: info_buf); |
287 | } |
288 | |
289 | return 0; |
290 | |
291 | err_out: |
292 | kfree(objp: info_buf); |
293 | |
294 | return -EINVAL; |
295 | } |
296 | |
297 | /* Clean up the arg_info in a struct bpf_struct_ops_desc. */ |
298 | void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc) |
299 | { |
300 | struct bpf_struct_ops_arg_info *arg_info; |
301 | int i; |
302 | |
303 | arg_info = st_ops_desc->arg_info; |
304 | for (i = 0; i < btf_type_vlen(t: st_ops_desc->type); i++) |
305 | kfree(objp: arg_info[i].info); |
306 | |
307 | kfree(objp: arg_info); |
308 | } |
309 | |
310 | int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, |
311 | struct btf *btf, |
312 | struct bpf_verifier_log *log) |
313 | { |
314 | struct bpf_struct_ops *st_ops = st_ops_desc->st_ops; |
315 | struct bpf_struct_ops_arg_info *arg_info; |
316 | const struct btf_member *member; |
317 | const struct btf_type *t; |
318 | s32 type_id, value_id; |
319 | char value_name[128]; |
320 | const char *mname; |
321 | int i, err; |
322 | |
323 | if (strlen(st_ops->name) + VALUE_PREFIX_LEN >= |
324 | sizeof(value_name)) { |
325 | pr_warn("struct_ops name %s is too long\n" , |
326 | st_ops->name); |
327 | return -EINVAL; |
328 | } |
329 | sprintf(buf: value_name, fmt: "%s%s" , VALUE_PREFIX, st_ops->name); |
330 | |
331 | if (!st_ops->cfi_stubs) { |
332 | pr_warn("struct_ops for %s has no cfi_stubs\n" , st_ops->name); |
333 | return -EINVAL; |
334 | } |
335 | |
336 | type_id = btf_find_by_name_kind(btf, name: st_ops->name, |
337 | kind: BTF_KIND_STRUCT); |
338 | if (type_id < 0) { |
339 | pr_warn("Cannot find struct %s in %s\n" , |
340 | st_ops->name, btf_get_name(btf)); |
341 | return -EINVAL; |
342 | } |
343 | t = btf_type_by_id(btf, type_id); |
344 | if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) { |
345 | pr_warn("Cannot support #%u members in struct %s\n" , |
346 | btf_type_vlen(t), st_ops->name); |
347 | return -EINVAL; |
348 | } |
349 | |
350 | value_id = btf_find_by_name_kind(btf, name: value_name, |
351 | kind: BTF_KIND_STRUCT); |
352 | if (value_id < 0) { |
353 | pr_warn("Cannot find struct %s in %s\n" , |
354 | value_name, btf_get_name(btf)); |
355 | return -EINVAL; |
356 | } |
357 | if (!is_valid_value_type(btf, value_id, type: t, value_name)) |
358 | return -EINVAL; |
359 | |
360 | arg_info = kcalloc(n: btf_type_vlen(t), size: sizeof(*arg_info), |
361 | GFP_KERNEL); |
362 | if (!arg_info) |
363 | return -ENOMEM; |
364 | |
365 | st_ops_desc->arg_info = arg_info; |
366 | st_ops_desc->type = t; |
367 | st_ops_desc->type_id = type_id; |
368 | st_ops_desc->value_id = value_id; |
369 | st_ops_desc->value_type = btf_type_by_id(btf, type_id: value_id); |
370 | |
371 | for_each_member(i, t, member) { |
372 | const struct btf_type *func_proto; |
373 | |
374 | mname = btf_name_by_offset(btf, offset: member->name_off); |
375 | if (!*mname) { |
376 | pr_warn("anon member in struct %s is not supported\n" , |
377 | st_ops->name); |
378 | err = -EOPNOTSUPP; |
379 | goto errout; |
380 | } |
381 | |
382 | if (__btf_member_bitfield_size(struct_type: t, member)) { |
383 | pr_warn("bit field member %s in struct %s is not supported\n" , |
384 | mname, st_ops->name); |
385 | err = -EOPNOTSUPP; |
386 | goto errout; |
387 | } |
388 | |
389 | func_proto = btf_type_resolve_func_ptr(btf, |
390 | id: member->type, |
391 | NULL); |
392 | if (!func_proto) |
393 | continue; |
394 | |
395 | if (btf_distill_func_proto(log, btf, |
396 | func_proto, func_name: mname, |
397 | m: &st_ops->func_models[i])) { |
398 | pr_warn("Error in parsing func ptr %s in struct %s\n" , |
399 | mname, st_ops->name); |
400 | err = -EINVAL; |
401 | goto errout; |
402 | } |
403 | |
404 | err = prepare_arg_info(btf, st_ops_name: st_ops->name, member_name: mname, |
405 | func_proto, |
406 | arg_info: arg_info + i); |
407 | if (err) |
408 | goto errout; |
409 | } |
410 | |
411 | if (st_ops->init(btf)) { |
412 | pr_warn("Error in init bpf_struct_ops %s\n" , |
413 | st_ops->name); |
414 | err = -EINVAL; |
415 | goto errout; |
416 | } |
417 | |
418 | return 0; |
419 | |
420 | errout: |
421 | bpf_struct_ops_desc_release(st_ops_desc); |
422 | |
423 | return err; |
424 | } |
425 | |
426 | static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key, |
427 | void *next_key) |
428 | { |
429 | if (key && *(u32 *)key == 0) |
430 | return -ENOENT; |
431 | |
432 | *(u32 *)next_key = 0; |
433 | return 0; |
434 | } |
435 | |
436 | int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, |
437 | void *value) |
438 | { |
439 | struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; |
440 | struct bpf_struct_ops_value *uvalue, *kvalue; |
441 | enum bpf_struct_ops_state state; |
442 | s64 refcnt; |
443 | |
444 | if (unlikely(*(u32 *)key != 0)) |
445 | return -ENOENT; |
446 | |
447 | kvalue = &st_map->kvalue; |
448 | /* Pair with smp_store_release() during map_update */ |
449 | state = smp_load_acquire(&kvalue->common.state); |
450 | if (state == BPF_STRUCT_OPS_STATE_INIT) { |
451 | memset(value, 0, map->value_size); |
452 | return 0; |
453 | } |
454 | |
455 | /* No lock is needed. state and refcnt do not need |
456 | * to be updated together under atomic context. |
457 | */ |
458 | uvalue = value; |
459 | memcpy(uvalue, st_map->uvalue, map->value_size); |
460 | uvalue->common.state = state; |
461 | |
462 | /* This value offers the user space a general estimate of how |
463 | * many sockets are still utilizing this struct_ops for TCP |
464 | * congestion control. The number might not be exact, but it |
465 | * should sufficiently meet our present goals. |
466 | */ |
467 | refcnt = atomic64_read(v: &map->refcnt) - atomic64_read(v: &map->usercnt); |
468 | refcount_set(r: &uvalue->common.refcnt, max_t(s64, refcnt, 0)); |
469 | |
470 | return 0; |
471 | } |
472 | |
473 | static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key) |
474 | { |
475 | return ERR_PTR(error: -EINVAL); |
476 | } |
477 | |
478 | static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map) |
479 | { |
480 | u32 i; |
481 | |
482 | for (i = 0; i < st_map->links_cnt; i++) { |
483 | if (st_map->links[i]) { |
484 | bpf_link_put(link: st_map->links[i]); |
485 | st_map->links[i] = NULL; |
486 | } |
487 | } |
488 | } |
489 | |
490 | static void bpf_struct_ops_map_free_image(struct bpf_struct_ops_map *st_map) |
491 | { |
492 | int i; |
493 | |
494 | for (i = 0; i < st_map->image_pages_cnt; i++) |
495 | bpf_struct_ops_image_free(image: st_map->image_pages[i]); |
496 | st_map->image_pages_cnt = 0; |
497 | } |
498 | |
499 | static int check_zero_holes(const struct btf *btf, const struct btf_type *t, void *data) |
500 | { |
501 | const struct btf_member *member; |
502 | u32 i, moff, msize, prev_mend = 0; |
503 | const struct btf_type *mtype; |
504 | |
505 | for_each_member(i, t, member) { |
506 | moff = __btf_member_bit_offset(struct_type: t, member) / 8; |
507 | if (moff > prev_mend && |
508 | memchr_inv(p: data + prev_mend, c: 0, size: moff - prev_mend)) |
509 | return -EINVAL; |
510 | |
511 | mtype = btf_type_by_id(btf, type_id: member->type); |
512 | mtype = btf_resolve_size(btf, type: mtype, type_size: &msize); |
513 | if (IS_ERR(ptr: mtype)) |
514 | return PTR_ERR(ptr: mtype); |
515 | prev_mend = moff + msize; |
516 | } |
517 | |
518 | if (t->size > prev_mend && |
519 | memchr_inv(p: data + prev_mend, c: 0, size: t->size - prev_mend)) |
520 | return -EINVAL; |
521 | |
522 | return 0; |
523 | } |
524 | |
525 | static void bpf_struct_ops_link_release(struct bpf_link *link) |
526 | { |
527 | } |
528 | |
529 | static void bpf_struct_ops_link_dealloc(struct bpf_link *link) |
530 | { |
531 | struct bpf_tramp_link *tlink = container_of(link, struct bpf_tramp_link, link); |
532 | |
533 | kfree(objp: tlink); |
534 | } |
535 | |
536 | const struct bpf_link_ops bpf_struct_ops_link_lops = { |
537 | .release = bpf_struct_ops_link_release, |
538 | .dealloc = bpf_struct_ops_link_dealloc, |
539 | }; |
540 | |
541 | int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, |
542 | struct bpf_tramp_link *link, |
543 | const struct btf_func_model *model, |
544 | void *stub_func, |
545 | void **_image, u32 *_image_off, |
546 | bool allow_alloc) |
547 | { |
548 | u32 image_off = *_image_off, flags = BPF_TRAMP_F_INDIRECT; |
549 | void *image = *_image; |
550 | int size; |
551 | |
552 | tlinks[BPF_TRAMP_FENTRY].links[0] = link; |
553 | tlinks[BPF_TRAMP_FENTRY].nr_links = 1; |
554 | |
555 | if (model->ret_size > 0) |
556 | flags |= BPF_TRAMP_F_RET_FENTRY_RET; |
557 | |
558 | size = arch_bpf_trampoline_size(m: model, flags, tlinks, NULL); |
559 | if (size <= 0) |
560 | return size ? : -EFAULT; |
561 | |
562 | /* Allocate image buffer if necessary */ |
563 | if (!image || size > PAGE_SIZE - image_off) { |
564 | if (!allow_alloc) |
565 | return -E2BIG; |
566 | |
567 | image = bpf_struct_ops_image_alloc(); |
568 | if (IS_ERR(ptr: image)) |
569 | return PTR_ERR(ptr: image); |
570 | image_off = 0; |
571 | } |
572 | |
573 | size = arch_prepare_bpf_trampoline(NULL, image: image + image_off, |
574 | image_end: image + PAGE_SIZE, |
575 | m: model, flags, tlinks, func_addr: stub_func); |
576 | if (size <= 0) { |
577 | if (image != *_image) |
578 | bpf_struct_ops_image_free(image); |
579 | return size ? : -EFAULT; |
580 | } |
581 | |
582 | *_image = image; |
583 | *_image_off = image_off + size; |
584 | return 0; |
585 | } |
586 | |
587 | static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, |
588 | void *value, u64 flags) |
589 | { |
590 | struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; |
591 | const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc; |
592 | const struct bpf_struct_ops *st_ops = st_ops_desc->st_ops; |
593 | struct bpf_struct_ops_value *uvalue, *kvalue; |
594 | const struct btf_type *module_type; |
595 | const struct btf_member *member; |
596 | const struct btf_type *t = st_ops_desc->type; |
597 | struct bpf_tramp_links *tlinks; |
598 | void *udata, *kdata; |
599 | int prog_fd, err; |
600 | u32 i, trampoline_start, image_off = 0; |
601 | void *cur_image = NULL, *image = NULL; |
602 | |
603 | if (flags) |
604 | return -EINVAL; |
605 | |
606 | if (*(u32 *)key != 0) |
607 | return -E2BIG; |
608 | |
609 | err = check_zero_holes(btf: st_map->btf, t: st_ops_desc->value_type, data: value); |
610 | if (err) |
611 | return err; |
612 | |
613 | uvalue = value; |
614 | err = check_zero_holes(btf: st_map->btf, t, data: uvalue->data); |
615 | if (err) |
616 | return err; |
617 | |
618 | if (uvalue->common.state || refcount_read(r: &uvalue->common.refcnt)) |
619 | return -EINVAL; |
620 | |
621 | tlinks = kcalloc(n: BPF_TRAMP_MAX, size: sizeof(*tlinks), GFP_KERNEL); |
622 | if (!tlinks) |
623 | return -ENOMEM; |
624 | |
625 | uvalue = (struct bpf_struct_ops_value *)st_map->uvalue; |
626 | kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue; |
627 | |
628 | mutex_lock(&st_map->lock); |
629 | |
630 | if (kvalue->common.state != BPF_STRUCT_OPS_STATE_INIT) { |
631 | err = -EBUSY; |
632 | goto unlock; |
633 | } |
634 | |
635 | memcpy(uvalue, value, map->value_size); |
636 | |
637 | udata = &uvalue->data; |
638 | kdata = &kvalue->data; |
639 | |
640 | module_type = btf_type_by_id(btf: btf_vmlinux, type_id: st_ops_ids[IDX_MODULE_ID]); |
641 | for_each_member(i, t, member) { |
642 | const struct btf_type *mtype, *ptype; |
643 | struct bpf_prog *prog; |
644 | struct bpf_tramp_link *link; |
645 | u32 moff; |
646 | |
647 | moff = __btf_member_bit_offset(struct_type: t, member) / 8; |
648 | ptype = btf_type_resolve_ptr(btf: st_map->btf, id: member->type, NULL); |
649 | if (ptype == module_type) { |
650 | if (*(void **)(udata + moff)) |
651 | goto reset_unlock; |
652 | *(void **)(kdata + moff) = BPF_MODULE_OWNER; |
653 | continue; |
654 | } |
655 | |
656 | err = st_ops->init_member(t, member, kdata, udata); |
657 | if (err < 0) |
658 | goto reset_unlock; |
659 | |
660 | /* The ->init_member() has handled this member */ |
661 | if (err > 0) |
662 | continue; |
663 | |
664 | /* If st_ops->init_member does not handle it, |
665 | * we will only handle func ptrs and zero-ed members |
666 | * here. Reject everything else. |
667 | */ |
668 | |
669 | /* All non func ptr member must be 0 */ |
670 | if (!ptype || !btf_type_is_func_proto(t: ptype)) { |
671 | u32 msize; |
672 | |
673 | mtype = btf_type_by_id(btf: st_map->btf, type_id: member->type); |
674 | mtype = btf_resolve_size(btf: st_map->btf, type: mtype, type_size: &msize); |
675 | if (IS_ERR(ptr: mtype)) { |
676 | err = PTR_ERR(ptr: mtype); |
677 | goto reset_unlock; |
678 | } |
679 | |
680 | if (memchr_inv(p: udata + moff, c: 0, size: msize)) { |
681 | err = -EINVAL; |
682 | goto reset_unlock; |
683 | } |
684 | |
685 | continue; |
686 | } |
687 | |
688 | prog_fd = (int)(*(unsigned long *)(udata + moff)); |
689 | /* Similar check as the attr->attach_prog_fd */ |
690 | if (!prog_fd) |
691 | continue; |
692 | |
693 | prog = bpf_prog_get(ufd: prog_fd); |
694 | if (IS_ERR(ptr: prog)) { |
695 | err = PTR_ERR(ptr: prog); |
696 | goto reset_unlock; |
697 | } |
698 | |
699 | if (prog->type != BPF_PROG_TYPE_STRUCT_OPS || |
700 | prog->aux->attach_btf_id != st_ops_desc->type_id || |
701 | prog->expected_attach_type != i) { |
702 | bpf_prog_put(prog); |
703 | err = -EINVAL; |
704 | goto reset_unlock; |
705 | } |
706 | |
707 | link = kzalloc(size: sizeof(*link), GFP_USER); |
708 | if (!link) { |
709 | bpf_prog_put(prog); |
710 | err = -ENOMEM; |
711 | goto reset_unlock; |
712 | } |
713 | bpf_link_init(link: &link->link, type: BPF_LINK_TYPE_STRUCT_OPS, |
714 | ops: &bpf_struct_ops_link_lops, prog); |
715 | st_map->links[i] = &link->link; |
716 | |
717 | trampoline_start = image_off; |
718 | err = bpf_struct_ops_prepare_trampoline(tlinks, link, |
719 | model: &st_ops->func_models[i], |
720 | stub_func: *(void **)(st_ops->cfi_stubs + moff), |
721 | image: &image, image_off: &image_off, |
722 | allow_alloc: st_map->image_pages_cnt < MAX_TRAMP_IMAGE_PAGES); |
723 | if (err) |
724 | goto reset_unlock; |
725 | |
726 | if (cur_image != image) { |
727 | st_map->image_pages[st_map->image_pages_cnt++] = image; |
728 | cur_image = image; |
729 | trampoline_start = 0; |
730 | } |
731 | if (err < 0) |
732 | goto reset_unlock; |
733 | |
734 | *(void **)(kdata + moff) = image + trampoline_start + cfi_get_offset(); |
735 | |
736 | /* put prog_id to udata */ |
737 | *(unsigned long *)(udata + moff) = prog->aux->id; |
738 | } |
739 | |
740 | if (st_ops->validate) { |
741 | err = st_ops->validate(kdata); |
742 | if (err) |
743 | goto reset_unlock; |
744 | } |
745 | for (i = 0; i < st_map->image_pages_cnt; i++) |
746 | arch_protect_bpf_trampoline(image: st_map->image_pages[i], PAGE_SIZE); |
747 | |
748 | if (st_map->map.map_flags & BPF_F_LINK) { |
749 | err = 0; |
750 | /* Let bpf_link handle registration & unregistration. |
751 | * |
752 | * Pair with smp_load_acquire() during lookup_elem(). |
753 | */ |
754 | smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_READY); |
755 | goto unlock; |
756 | } |
757 | |
758 | err = st_ops->reg(kdata); |
759 | if (likely(!err)) { |
760 | /* This refcnt increment on the map here after |
761 | * 'st_ops->reg()' is secure since the state of the |
762 | * map must be set to INIT at this moment, and thus |
763 | * bpf_struct_ops_map_delete_elem() can't unregister |
764 | * or transition it to TOBEFREE concurrently. |
765 | */ |
766 | bpf_map_inc(map); |
767 | /* Pair with smp_load_acquire() during lookup_elem(). |
768 | * It ensures the above udata updates (e.g. prog->aux->id) |
769 | * can be seen once BPF_STRUCT_OPS_STATE_INUSE is set. |
770 | */ |
771 | smp_store_release(&kvalue->common.state, BPF_STRUCT_OPS_STATE_INUSE); |
772 | goto unlock; |
773 | } |
774 | |
775 | /* Error during st_ops->reg(). Can happen if this struct_ops needs to be |
776 | * verified as a whole, after all init_member() calls. Can also happen if |
777 | * there was a race in registering the struct_ops (under the same name) to |
778 | * a sub-system through different struct_ops's maps. |
779 | */ |
780 | |
781 | reset_unlock: |
782 | bpf_struct_ops_map_free_image(st_map); |
783 | bpf_struct_ops_map_put_progs(st_map); |
784 | memset(uvalue, 0, map->value_size); |
785 | memset(kvalue, 0, map->value_size); |
786 | unlock: |
787 | kfree(objp: tlinks); |
788 | mutex_unlock(lock: &st_map->lock); |
789 | return err; |
790 | } |
791 | |
792 | static long bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key) |
793 | { |
794 | enum bpf_struct_ops_state prev_state; |
795 | struct bpf_struct_ops_map *st_map; |
796 | |
797 | st_map = (struct bpf_struct_ops_map *)map; |
798 | if (st_map->map.map_flags & BPF_F_LINK) |
799 | return -EOPNOTSUPP; |
800 | |
801 | prev_state = cmpxchg(&st_map->kvalue.common.state, |
802 | BPF_STRUCT_OPS_STATE_INUSE, |
803 | BPF_STRUCT_OPS_STATE_TOBEFREE); |
804 | switch (prev_state) { |
805 | case BPF_STRUCT_OPS_STATE_INUSE: |
806 | st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data); |
807 | bpf_map_put(map); |
808 | return 0; |
809 | case BPF_STRUCT_OPS_STATE_TOBEFREE: |
810 | return -EINPROGRESS; |
811 | case BPF_STRUCT_OPS_STATE_INIT: |
812 | return -ENOENT; |
813 | default: |
814 | WARN_ON_ONCE(1); |
815 | /* Should never happen. Treat it as not found. */ |
816 | return -ENOENT; |
817 | } |
818 | } |
819 | |
820 | static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key, |
821 | struct seq_file *m) |
822 | { |
823 | struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; |
824 | void *value; |
825 | int err; |
826 | |
827 | value = kmalloc(size: map->value_size, GFP_USER | __GFP_NOWARN); |
828 | if (!value) |
829 | return; |
830 | |
831 | err = bpf_struct_ops_map_sys_lookup_elem(map, key, value); |
832 | if (!err) { |
833 | btf_type_seq_show(btf: st_map->btf, |
834 | type_id: map->btf_vmlinux_value_type_id, |
835 | obj: value, m); |
836 | seq_puts(m, s: "\n" ); |
837 | } |
838 | |
839 | kfree(objp: value); |
840 | } |
841 | |
842 | static void __bpf_struct_ops_map_free(struct bpf_map *map) |
843 | { |
844 | struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; |
845 | |
846 | if (st_map->links) |
847 | bpf_struct_ops_map_put_progs(st_map); |
848 | bpf_map_area_free(base: st_map->links); |
849 | bpf_struct_ops_map_free_image(st_map); |
850 | bpf_map_area_free(base: st_map->uvalue); |
851 | bpf_map_area_free(base: st_map); |
852 | } |
853 | |
854 | static void bpf_struct_ops_map_free(struct bpf_map *map) |
855 | { |
856 | struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; |
857 | |
858 | /* st_ops->owner was acquired during map_alloc to implicitly holds |
859 | * the btf's refcnt. The acquire was only done when btf_is_module() |
860 | * st_map->btf cannot be NULL here. |
861 | */ |
862 | if (btf_is_module(btf: st_map->btf)) |
863 | module_put(module: st_map->st_ops_desc->st_ops->owner); |
864 | |
865 | /* The struct_ops's function may switch to another struct_ops. |
866 | * |
867 | * For example, bpf_tcp_cc_x->init() may switch to |
868 | * another tcp_cc_y by calling |
869 | * setsockopt(TCP_CONGESTION, "tcp_cc_y"). |
870 | * During the switch, bpf_struct_ops_put(tcp_cc_x) is called |
871 | * and its refcount may reach 0 which then free its |
872 | * trampoline image while tcp_cc_x is still running. |
873 | * |
874 | * A vanilla rcu gp is to wait for all bpf-tcp-cc prog |
875 | * to finish. bpf-tcp-cc prog is non sleepable. |
876 | * A rcu_tasks gp is to wait for the last few insn |
877 | * in the tramopline image to finish before releasing |
878 | * the trampoline image. |
879 | */ |
880 | synchronize_rcu_mult(call_rcu, call_rcu_tasks); |
881 | |
882 | __bpf_struct_ops_map_free(map); |
883 | } |
884 | |
885 | static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr) |
886 | { |
887 | if (attr->key_size != sizeof(unsigned int) || attr->max_entries != 1 || |
888 | (attr->map_flags & ~(BPF_F_LINK | BPF_F_VTYPE_BTF_OBJ_FD)) || |
889 | !attr->btf_vmlinux_value_type_id) |
890 | return -EINVAL; |
891 | return 0; |
892 | } |
893 | |
894 | static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) |
895 | { |
896 | const struct bpf_struct_ops_desc *st_ops_desc; |
897 | size_t st_map_size; |
898 | struct bpf_struct_ops_map *st_map; |
899 | const struct btf_type *t, *vt; |
900 | struct module *mod = NULL; |
901 | struct bpf_map *map; |
902 | struct btf *btf; |
903 | int ret; |
904 | |
905 | if (attr->map_flags & BPF_F_VTYPE_BTF_OBJ_FD) { |
906 | /* The map holds btf for its whole life time. */ |
907 | btf = btf_get_by_fd(fd: attr->value_type_btf_obj_fd); |
908 | if (IS_ERR(ptr: btf)) |
909 | return ERR_CAST(ptr: btf); |
910 | if (!btf_is_module(btf)) { |
911 | btf_put(btf); |
912 | return ERR_PTR(error: -EINVAL); |
913 | } |
914 | |
915 | mod = btf_try_get_module(btf); |
916 | /* mod holds a refcnt to btf. We don't need an extra refcnt |
917 | * here. |
918 | */ |
919 | btf_put(btf); |
920 | if (!mod) |
921 | return ERR_PTR(error: -EINVAL); |
922 | } else { |
923 | btf = bpf_get_btf_vmlinux(); |
924 | if (IS_ERR(ptr: btf)) |
925 | return ERR_CAST(ptr: btf); |
926 | if (!btf) |
927 | return ERR_PTR(error: -ENOTSUPP); |
928 | } |
929 | |
930 | st_ops_desc = bpf_struct_ops_find_value(btf, value_id: attr->btf_vmlinux_value_type_id); |
931 | if (!st_ops_desc) { |
932 | ret = -ENOTSUPP; |
933 | goto errout; |
934 | } |
935 | |
936 | vt = st_ops_desc->value_type; |
937 | if (attr->value_size != vt->size) { |
938 | ret = -EINVAL; |
939 | goto errout; |
940 | } |
941 | |
942 | t = st_ops_desc->type; |
943 | |
944 | st_map_size = sizeof(*st_map) + |
945 | /* kvalue stores the |
946 | * struct bpf_struct_ops_tcp_congestions_ops |
947 | */ |
948 | (vt->size - sizeof(struct bpf_struct_ops_value)); |
949 | |
950 | st_map = bpf_map_area_alloc(size: st_map_size, NUMA_NO_NODE); |
951 | if (!st_map) { |
952 | ret = -ENOMEM; |
953 | goto errout; |
954 | } |
955 | |
956 | st_map->st_ops_desc = st_ops_desc; |
957 | map = &st_map->map; |
958 | |
959 | st_map->uvalue = bpf_map_area_alloc(size: vt->size, NUMA_NO_NODE); |
960 | st_map->links_cnt = btf_type_vlen(t); |
961 | st_map->links = |
962 | bpf_map_area_alloc(size: st_map->links_cnt * sizeof(struct bpf_links *), |
963 | NUMA_NO_NODE); |
964 | if (!st_map->uvalue || !st_map->links) { |
965 | ret = -ENOMEM; |
966 | goto errout_free; |
967 | } |
968 | st_map->btf = btf; |
969 | |
970 | mutex_init(&st_map->lock); |
971 | bpf_map_init_from_attr(map, attr); |
972 | |
973 | return map; |
974 | |
975 | errout_free: |
976 | __bpf_struct_ops_map_free(map); |
977 | errout: |
978 | module_put(module: mod); |
979 | |
980 | return ERR_PTR(error: ret); |
981 | } |
982 | |
983 | static u64 bpf_struct_ops_map_mem_usage(const struct bpf_map *map) |
984 | { |
985 | struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; |
986 | const struct bpf_struct_ops_desc *st_ops_desc = st_map->st_ops_desc; |
987 | const struct btf_type *vt = st_ops_desc->value_type; |
988 | u64 usage; |
989 | |
990 | usage = sizeof(*st_map) + |
991 | vt->size - sizeof(struct bpf_struct_ops_value); |
992 | usage += vt->size; |
993 | usage += btf_type_vlen(t: vt) * sizeof(struct bpf_links *); |
994 | usage += PAGE_SIZE; |
995 | return usage; |
996 | } |
997 | |
998 | BTF_ID_LIST_SINGLE(bpf_struct_ops_map_btf_ids, struct, bpf_struct_ops_map) |
999 | const struct bpf_map_ops bpf_struct_ops_map_ops = { |
1000 | .map_alloc_check = bpf_struct_ops_map_alloc_check, |
1001 | .map_alloc = bpf_struct_ops_map_alloc, |
1002 | .map_free = bpf_struct_ops_map_free, |
1003 | .map_get_next_key = bpf_struct_ops_map_get_next_key, |
1004 | .map_lookup_elem = bpf_struct_ops_map_lookup_elem, |
1005 | .map_delete_elem = bpf_struct_ops_map_delete_elem, |
1006 | .map_update_elem = bpf_struct_ops_map_update_elem, |
1007 | .map_seq_show_elem = bpf_struct_ops_map_seq_show_elem, |
1008 | .map_mem_usage = bpf_struct_ops_map_mem_usage, |
1009 | .map_btf_id = &bpf_struct_ops_map_btf_ids[0], |
1010 | }; |
1011 | |
1012 | /* "const void *" because some subsystem is |
1013 | * passing a const (e.g. const struct tcp_congestion_ops *) |
1014 | */ |
1015 | bool bpf_struct_ops_get(const void *kdata) |
1016 | { |
1017 | struct bpf_struct_ops_value *kvalue; |
1018 | struct bpf_struct_ops_map *st_map; |
1019 | struct bpf_map *map; |
1020 | |
1021 | kvalue = container_of(kdata, struct bpf_struct_ops_value, data); |
1022 | st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue); |
1023 | |
1024 | map = __bpf_map_inc_not_zero(map: &st_map->map, uref: false); |
1025 | return !IS_ERR(ptr: map); |
1026 | } |
1027 | |
1028 | void bpf_struct_ops_put(const void *kdata) |
1029 | { |
1030 | struct bpf_struct_ops_value *kvalue; |
1031 | struct bpf_struct_ops_map *st_map; |
1032 | |
1033 | kvalue = container_of(kdata, struct bpf_struct_ops_value, data); |
1034 | st_map = container_of(kvalue, struct bpf_struct_ops_map, kvalue); |
1035 | |
1036 | bpf_map_put(map: &st_map->map); |
1037 | } |
1038 | |
1039 | static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) |
1040 | { |
1041 | struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; |
1042 | |
1043 | return map->map_type == BPF_MAP_TYPE_STRUCT_OPS && |
1044 | map->map_flags & BPF_F_LINK && |
1045 | /* Pair with smp_store_release() during map_update */ |
1046 | smp_load_acquire(&st_map->kvalue.common.state) == BPF_STRUCT_OPS_STATE_READY; |
1047 | } |
1048 | |
1049 | static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link) |
1050 | { |
1051 | struct bpf_struct_ops_link *st_link; |
1052 | struct bpf_struct_ops_map *st_map; |
1053 | |
1054 | st_link = container_of(link, struct bpf_struct_ops_link, link); |
1055 | st_map = (struct bpf_struct_ops_map *) |
1056 | rcu_dereference_protected(st_link->map, true); |
1057 | if (st_map) { |
1058 | /* st_link->map can be NULL if |
1059 | * bpf_struct_ops_link_create() fails to register. |
1060 | */ |
1061 | st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data); |
1062 | bpf_map_put(map: &st_map->map); |
1063 | } |
1064 | kfree(objp: st_link); |
1065 | } |
1066 | |
1067 | static void bpf_struct_ops_map_link_show_fdinfo(const struct bpf_link *link, |
1068 | struct seq_file *seq) |
1069 | { |
1070 | struct bpf_struct_ops_link *st_link; |
1071 | struct bpf_map *map; |
1072 | |
1073 | st_link = container_of(link, struct bpf_struct_ops_link, link); |
1074 | rcu_read_lock(); |
1075 | map = rcu_dereference(st_link->map); |
1076 | seq_printf(m: seq, fmt: "map_id:\t%d\n" , map->id); |
1077 | rcu_read_unlock(); |
1078 | } |
1079 | |
1080 | static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link, |
1081 | struct bpf_link_info *info) |
1082 | { |
1083 | struct bpf_struct_ops_link *st_link; |
1084 | struct bpf_map *map; |
1085 | |
1086 | st_link = container_of(link, struct bpf_struct_ops_link, link); |
1087 | rcu_read_lock(); |
1088 | map = rcu_dereference(st_link->map); |
1089 | info->struct_ops.map_id = map->id; |
1090 | rcu_read_unlock(); |
1091 | return 0; |
1092 | } |
1093 | |
1094 | static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map *new_map, |
1095 | struct bpf_map *expected_old_map) |
1096 | { |
1097 | struct bpf_struct_ops_map *st_map, *old_st_map; |
1098 | struct bpf_map *old_map; |
1099 | struct bpf_struct_ops_link *st_link; |
1100 | int err; |
1101 | |
1102 | st_link = container_of(link, struct bpf_struct_ops_link, link); |
1103 | st_map = container_of(new_map, struct bpf_struct_ops_map, map); |
1104 | |
1105 | if (!bpf_struct_ops_valid_to_reg(map: new_map)) |
1106 | return -EINVAL; |
1107 | |
1108 | if (!st_map->st_ops_desc->st_ops->update) |
1109 | return -EOPNOTSUPP; |
1110 | |
1111 | mutex_lock(&update_mutex); |
1112 | |
1113 | old_map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex)); |
1114 | if (expected_old_map && old_map != expected_old_map) { |
1115 | err = -EPERM; |
1116 | goto err_out; |
1117 | } |
1118 | |
1119 | old_st_map = container_of(old_map, struct bpf_struct_ops_map, map); |
1120 | /* The new and old struct_ops must be the same type. */ |
1121 | if (st_map->st_ops_desc != old_st_map->st_ops_desc) { |
1122 | err = -EINVAL; |
1123 | goto err_out; |
1124 | } |
1125 | |
1126 | err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data); |
1127 | if (err) |
1128 | goto err_out; |
1129 | |
1130 | bpf_map_inc(map: new_map); |
1131 | rcu_assign_pointer(st_link->map, new_map); |
1132 | bpf_map_put(map: old_map); |
1133 | |
1134 | err_out: |
1135 | mutex_unlock(lock: &update_mutex); |
1136 | |
1137 | return err; |
1138 | } |
1139 | |
1140 | static const struct bpf_link_ops bpf_struct_ops_map_lops = { |
1141 | .dealloc = bpf_struct_ops_map_link_dealloc, |
1142 | .show_fdinfo = bpf_struct_ops_map_link_show_fdinfo, |
1143 | .fill_link_info = bpf_struct_ops_map_link_fill_link_info, |
1144 | .update_map = bpf_struct_ops_map_link_update, |
1145 | }; |
1146 | |
1147 | int bpf_struct_ops_link_create(union bpf_attr *attr) |
1148 | { |
1149 | struct bpf_struct_ops_link *link = NULL; |
1150 | struct bpf_link_primer link_primer; |
1151 | struct bpf_struct_ops_map *st_map; |
1152 | struct bpf_map *map; |
1153 | int err; |
1154 | |
1155 | map = bpf_map_get(ufd: attr->link_create.map_fd); |
1156 | if (IS_ERR(ptr: map)) |
1157 | return PTR_ERR(ptr: map); |
1158 | |
1159 | st_map = (struct bpf_struct_ops_map *)map; |
1160 | |
1161 | if (!bpf_struct_ops_valid_to_reg(map)) { |
1162 | err = -EINVAL; |
1163 | goto err_out; |
1164 | } |
1165 | |
1166 | link = kzalloc(size: sizeof(*link), GFP_USER); |
1167 | if (!link) { |
1168 | err = -ENOMEM; |
1169 | goto err_out; |
1170 | } |
1171 | bpf_link_init(link: &link->link, type: BPF_LINK_TYPE_STRUCT_OPS, ops: &bpf_struct_ops_map_lops, NULL); |
1172 | |
1173 | err = bpf_link_prime(link: &link->link, primer: &link_primer); |
1174 | if (err) |
1175 | goto err_out; |
1176 | |
1177 | err = st_map->st_ops_desc->st_ops->reg(st_map->kvalue.data); |
1178 | if (err) { |
1179 | bpf_link_cleanup(primer: &link_primer); |
1180 | link = NULL; |
1181 | goto err_out; |
1182 | } |
1183 | RCU_INIT_POINTER(link->map, map); |
1184 | |
1185 | return bpf_link_settle(primer: &link_primer); |
1186 | |
1187 | err_out: |
1188 | bpf_map_put(map); |
1189 | kfree(objp: link); |
1190 | return err; |
1191 | } |
1192 | |
1193 | void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map) |
1194 | { |
1195 | struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; |
1196 | |
1197 | info->btf_vmlinux_id = btf_obj_id(btf: st_map->btf); |
1198 | } |
1199 | |