1	/* SPDX-License-Identifier: GPL-2.0-only */
2	/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3	*/
4	#ifndef _LINUX_BPF_H
5	#define _LINUX_BPF_H 1
6
7	#include <uapi/linux/bpf.h>
8	#include <uapi/linux/filter.h>
9
10	#include <linux/workqueue.h>
11	#include <linux/file.h>
12	#include <linux/percpu.h>
13	#include <linux/err.h>
14	#include <linux/rbtree_latch.h>
15	#include <linux/numa.h>
16	#include <linux/mm_types.h>
17	#include <linux/wait.h>
18	#include <linux/refcount.h>
19	#include <linux/mutex.h>
20	#include <linux/module.h>
21	#include <linux/kallsyms.h>
22	#include <linux/capability.h>
23	#include <linux/sched/mm.h>
24	#include <linux/slab.h>
25	#include <linux/percpu-refcount.h>
26	#include <linux/stddef.h>
27	#include <linux/bpfptr.h>
28	#include <linux/btf.h>
29	#include <linux/rcupdate_trace.h>
30	#include <linux/static_call.h>
31	#include <linux/memcontrol.h>
32
33	struct bpf_verifier_env;
34	struct bpf_verifier_log;
35	struct perf_event;
36	struct bpf_prog;
37	struct bpf_prog_aux;
38	struct bpf_map;
39	struct sock;
40	struct seq_file;
41	struct btf;
42	struct btf_type;
43	struct exception_table_entry;
44	struct seq_operations;
45	struct bpf_iter_aux_info;
46	struct bpf_local_storage;
47	struct bpf_local_storage_map;
48	struct kobject;
49	struct mem_cgroup;
50	struct module;
51	struct bpf_func_state;
52	struct ftrace_ops;
53	struct cgroup;
54
55	extern struct idr btf_idr;
56	extern spinlock_t btf_idr_lock;
57	extern struct kobject *btf_kobj;
58	extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma;
59	extern bool bpf_global_ma_set, bpf_global_percpu_ma_set;
60
61	typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64);
62	typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,
63	struct bpf_iter_aux_info *aux);
64	typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data);
65	typedef unsigned int (*bpf_func_t)(const void *,
66	const struct bpf_insn *);
67	struct bpf_iter_seq_info {
68	const struct seq_operations *seq_ops;
69	bpf_iter_init_seq_priv_t init_seq_private;
70	bpf_iter_fini_seq_priv_t fini_seq_private;
71	u32 seq_priv_size;
72	};
73
74	/* map is generic key/value storage optionally accessible by eBPF programs */
75	struct bpf_map_ops {
76	/* funcs callable from userspace (via syscall) */
77	int (*map_alloc_check)(union bpf_attr *attr);
78	struct bpf_map *(*map_alloc)(union bpf_attr *attr);
79	void (*map_release)(struct bpf_map *map, struct file *map_file);
80	void (*map_free)(struct bpf_map *map);
81	int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
82	void (*map_release_uref)(struct bpf_map *map);
83	void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);
84	int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,
85	union bpf_attr __user *uattr);
86	int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key,
87	void *value, u64 flags);
88	int (*map_lookup_and_delete_batch)(struct bpf_map *map,
89	const union bpf_attr *attr,
90	union bpf_attr __user *uattr);
91	int (*map_update_batch)(struct bpf_map *map, struct file *map_file,
92	const union bpf_attr *attr,
93	union bpf_attr __user *uattr);
94	int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr,
95	union bpf_attr __user *uattr);
96
97	/* funcs callable from userspace and from eBPF programs */
98	void *(*map_lookup_elem)(struct bpf_map *map, void *key);
99	long (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
100	long (*map_delete_elem)(struct bpf_map *map, void *key);
101	long (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);
102	long (*map_pop_elem)(struct bpf_map *map, void *value);
103	long (*map_peek_elem)(struct bpf_map *map, void *value);
104	void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu);
105
106	/* funcs called by prog_array and perf_event_array map */
107	void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
108	int fd);
109	void (*map_fd_put_ptr)(void *ptr);
110	int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
111	u32 (*map_fd_sys_lookup_elem)(void *ptr);
112	void (*map_seq_show_elem)(struct bpf_map *map, void *key,
113	struct seq_file *m);
114	int (*map_check_btf)(const struct bpf_map *map,
115	const struct btf *btf,
116	const struct btf_type *key_type,
117	const struct btf_type *value_type);
118
119	/* Prog poke tracking helpers. */
120	int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux);
121	void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux);
122	void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old,
123	struct bpf_prog *new);
124
125	/* Direct value access helpers. */
126	int (*map_direct_value_addr)(const struct bpf_map *map,
127	u64 *imm, u32 off);
128	int (*map_direct_value_meta)(const struct bpf_map *map,
129	u64 imm, u32 *off);
130	int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
131	__poll_t (*map_poll)(struct bpf_map *map, struct file *filp,
132	struct poll_table_struct *pts);
133
134	/* Functions called by bpf_local_storage maps */
135	int (*map_local_storage_charge)(struct bpf_local_storage_map *smap,
136	void *owner, u32 size);
137	void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap,
138	void *owner, u32 size);
139	struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner);
140
141	/* Misc helpers.*/
142	long (*map_redirect)(struct bpf_map *map, u64 key, u64 flags);
143
144	/* map_meta_equal must be implemented for maps that can be
145	* used as an inner map. It is a runtime check to ensure
146	* an inner map can be inserted to an outer map.
147	*
148	* Some properties of the inner map has been used during the
149	* verification time. When inserting an inner map at the runtime,
150	* map_meta_equal has to ensure the inserting map has the same
151	* properties that the verifier has used earlier.
152	*/
153	bool (*map_meta_equal)(const struct bpf_map *meta0,
154	const struct bpf_map *meta1);
155
156
157	int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env,
158	struct bpf_func_state *caller,
159	struct bpf_func_state *callee);
160	long (*map_for_each_callback)(struct bpf_map *map,
161	bpf_callback_t callback_fn,
162	void *callback_ctx, u64 flags);
163
164	u64 (*map_mem_usage)(const struct bpf_map *map);
165
166	/* BTF id of struct allocated by map_alloc */
167	int *map_btf_id;
168
169	/* bpf_iter info used to open a seq_file */
170	const struct bpf_iter_seq_info *iter_seq_info;
171	};
172
173	enum {
174	/* Support at most 10 fields in a BTF type */
175	BTF_FIELDS_MAX = 10,
176	};
177
178	enum btf_field_type {
179	BPF_SPIN_LOCK = (1 << 0),
180	BPF_TIMER = (1 << 1),
181	BPF_KPTR_UNREF = (1 << 2),
182	BPF_KPTR_REF = (1 << 3),
183	BPF_KPTR_PERCPU = (1 << 4),
184	BPF_KPTR = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU,
185	BPF_LIST_HEAD = (1 << 5),
186	BPF_LIST_NODE = (1 << 6),
187	BPF_RB_ROOT = (1 << 7),
188	BPF_RB_NODE = (1 << 8),
189	BPF_GRAPH_NODE_OR_ROOT = BPF_LIST_NODE | BPF_LIST_HEAD |
190	BPF_RB_NODE | BPF_RB_ROOT,
191	BPF_REFCOUNT = (1 << 9),
192	};
193
194	typedef void (*btf_dtor_kfunc_t)(void *);
195
196	struct btf_field_kptr {
197	struct btf *btf;
198	struct module *module;
199	/* dtor used if btf_is_kernel(btf), otherwise the type is
200	* program-allocated, dtor is NULL, and __bpf_obj_drop_impl is used
201	*/
202	btf_dtor_kfunc_t dtor;
203	u32 btf_id;
204	};
205
206	struct btf_field_graph_root {
207	struct btf *btf;
208	u32 value_btf_id;
209	u32 node_offset;
210	struct btf_record *value_rec;
211	};
212
213	struct btf_field {
214	u32 offset;
215	u32 size;
216	enum btf_field_type type;
217	union {
218	struct btf_field_kptr kptr;
219	struct btf_field_graph_root graph_root;
220	};
221	};
222
223	struct btf_record {
224	u32 cnt;
225	u32 field_mask;
226	int spin_lock_off;
227	int timer_off;
228	int refcount_off;
229	struct btf_field fields[];
230	};
231
232	/* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */
233	struct bpf_rb_node_kern {
234	struct rb_node rb_node;
235	void *owner;
236	} __attribute__((aligned(8)));
237
238	/* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */
239	struct bpf_list_node_kern {
240	struct list_head list_head;
241	void *owner;
242	} __attribute__((aligned(8)));
243
244	struct bpf_map {
245	/* The first two cachelines with read-mostly members of which some
246	* are also accessed in fast-path (e.g. ops, max_entries).
247	*/
248	const struct bpf_map_ops *ops ____cacheline_aligned;
249	struct bpf_map *inner_map_meta;
250	#ifdef CONFIG_SECURITY
251	void *security;
252	#endif
253	enum bpf_map_type map_type;
254	u32 key_size;
255	u32 value_size;
256	u32 max_entries;
257	u64 map_extra; /* any per-map-type extra fields */
258	u32 map_flags;
259	u32 id;
260	struct btf_record *record;
261	int numa_node;
262	u32 btf_key_type_id;
263	u32 btf_value_type_id;
264	u32 btf_vmlinux_value_type_id;
265	struct btf *btf;
266	#ifdef CONFIG_MEMCG_KMEM
267	struct obj_cgroup *objcg;
268	#endif
269	char name[BPF_OBJ_NAME_LEN];
270	/* The 3rd and 4th cacheline with misc members to avoid false sharing
271	* particularly with refcounting.
272	*/
273	atomic64_t refcnt ____cacheline_aligned;
274	atomic64_t usercnt;
275	struct work_struct work;
276	struct mutex freeze_mutex;
277	atomic64_t writecnt;
278	/* 'Ownership' of program-containing map is claimed by the first program
279	* that is going to use this map or by the first program which FD is
280	* stored in the map to make sure that all callers and callees have the
281	* same prog type, JITed flag and xdp_has_frags flag.
282	*/
283	struct {
284	spinlock_t lock;
285	enum bpf_prog_type type;
286	bool jited;
287	bool xdp_has_frags;
288	} owner;
289	bool bypass_spec_v1;
290	bool frozen; /* write-once; write-protected by freeze_mutex */
291	s64 __percpu *elem_count;
292	};
293
294	static inline const char *btf_field_type_name(enum btf_field_type type)
295	{
296	switch (type) {
297	case BPF_SPIN_LOCK:
298	return "bpf_spin_lock";
299	case BPF_TIMER:
300	return "bpf_timer";
301	case BPF_KPTR_UNREF:
302	case BPF_KPTR_REF:
303	return "kptr";
304	case BPF_KPTR_PERCPU:
305	return "percpu_kptr";
306	case BPF_LIST_HEAD:
307	return "bpf_list_head";
308	case BPF_LIST_NODE:
309	return "bpf_list_node";
310	case BPF_RB_ROOT:
311	return "bpf_rb_root";
312	case BPF_RB_NODE:
313	return "bpf_rb_node";
314	case BPF_REFCOUNT:
315	return "bpf_refcount";
316	default:
317	WARN_ON_ONCE(1);
318	return "unknown";
319	}
320	}
321
322	static inline u32 btf_field_type_size(enum btf_field_type type)
323	{
324	switch (type) {
325	case BPF_SPIN_LOCK:
326	return sizeof(struct bpf_spin_lock);
327	case BPF_TIMER:
328	return sizeof(struct bpf_timer);
329	case BPF_KPTR_UNREF:
330	case BPF_KPTR_REF:
331	case BPF_KPTR_PERCPU:
332	return sizeof(u64);
333	case BPF_LIST_HEAD:
334	return sizeof(struct bpf_list_head);
335	case BPF_LIST_NODE:
336	return sizeof(struct bpf_list_node);
337	case BPF_RB_ROOT:
338	return sizeof(struct bpf_rb_root);
339	case BPF_RB_NODE:
340	return sizeof(struct bpf_rb_node);
341	case BPF_REFCOUNT:
342	return sizeof(struct bpf_refcount);
343	default:
344	WARN_ON_ONCE(1);
345	return 0;
346	}
347	}
348
349	static inline u32 btf_field_type_align(enum btf_field_type type)
350	{
351	switch (type) {
352	case BPF_SPIN_LOCK:
353	return __alignof__(struct bpf_spin_lock);
354	case BPF_TIMER:
355	return __alignof__(struct bpf_timer);
356	case BPF_KPTR_UNREF:
357	case BPF_KPTR_REF:
358	case BPF_KPTR_PERCPU:
359	return __alignof__(u64);
360	case BPF_LIST_HEAD:
361	return __alignof__(struct bpf_list_head);
362	case BPF_LIST_NODE:
363	return __alignof__(struct bpf_list_node);
364	case BPF_RB_ROOT:
365	return __alignof__(struct bpf_rb_root);
366	case BPF_RB_NODE:
367	return __alignof__(struct bpf_rb_node);
368	case BPF_REFCOUNT:
369	return __alignof__(struct bpf_refcount);
370	default:
371	WARN_ON_ONCE(1);
372	return 0;
373	}
374	}
375
376	static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)
377	{
378	memset(addr, 0, field->size);
379
380	switch (field->type) {
381	case BPF_REFCOUNT:
382	refcount_set(r: (refcount_t *)addr, n: 1);
383	break;
384	case BPF_RB_NODE:
385	RB_CLEAR_NODE((struct rb_node *)addr);
386	break;
387	case BPF_LIST_HEAD:
388	case BPF_LIST_NODE:
389	INIT_LIST_HEAD(list: (struct list_head *)addr);
390	break;
391	case BPF_RB_ROOT:
392	/* RB_ROOT_CACHED 0-inits, no need to do anything after memset */
393	case BPF_SPIN_LOCK:
394	case BPF_TIMER:
395	case BPF_KPTR_UNREF:
396	case BPF_KPTR_REF:
397	case BPF_KPTR_PERCPU:
398	break;
399	default:
400	WARN_ON_ONCE(1);
401	return;
402	}
403	}
404
405	static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type)
406	{
407	if (IS_ERR_OR_NULL(ptr: rec))
408	return false;
409	return rec->field_mask & type;
410	}
411
412	static inline void bpf_obj_init(const struct btf_record *rec, void *obj)
413	{
414	int i;
415
416	if (IS_ERR_OR_NULL(ptr: rec))
417	return;
418	for (i = 0; i < rec->cnt; i++)
419	bpf_obj_init_field(field: &rec->fields[i], addr: obj + rec->fields[i].offset);
420	}
421
422	/* 'dst' must be a temporary buffer and should not point to memory that is being
423	* used in parallel by a bpf program or bpf syscall, otherwise the access from
424	* the bpf program or bpf syscall may be corrupted by the reinitialization,
425	* leading to weird problems. Even 'dst' is newly-allocated from bpf memory
426	* allocator, it is still possible for 'dst' to be used in parallel by a bpf
427	* program or bpf syscall.
428	*/
429	static inline void check_and_init_map_value(struct bpf_map *map, void *dst)
430	{
431	bpf_obj_init(rec: map->record, obj: dst);
432	}
433
434	/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
435	* forced to use 'long' read/writes to try to atomically copy long counters.
436	* Best-effort only. No barriers here, since it _will_ race with concurrent
437	* updates from BPF programs. Called from bpf syscall and mostly used with
438	* size 8 or 16 bytes, so ask compiler to inline it.
439	*/
440	static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
441	{
442	const long *lsrc = src;
443	long *ldst = dst;
444
445	size /= sizeof(long);
446	while (size--)
447	data_race(*ldst++ = *lsrc++);
448	}
449
450	/* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */
451	static inline void bpf_obj_memcpy(struct btf_record *rec,
452	void *dst, void *src, u32 size,
453	bool long_memcpy)
454	{
455	u32 curr_off = 0;
456	int i;
457
458	if (IS_ERR_OR_NULL(ptr: rec)) {
459	if (long_memcpy)
460	bpf_long_memcpy(dst, src, round_up(size, 8));
461	else
462	memcpy(dst, src, size);
463	return;
464	}
465
466	for (i = 0; i < rec->cnt; i++) {
467	u32 next_off = rec->fields[i].offset;
468	u32 sz = next_off - curr_off;
469
470	memcpy(dst + curr_off, src + curr_off, sz);
471	curr_off += rec->fields[i].size + sz;
472	}
473	memcpy(dst + curr_off, src + curr_off, size - curr_off);
474	}
475
476	static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
477	{
478	bpf_obj_memcpy(rec: map->record, dst, src, size: map->value_size, long_memcpy: false);
479	}
480
481	static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)
482	{
483	bpf_obj_memcpy(rec: map->record, dst, src, size: map->value_size, long_memcpy: true);
484	}
485
486	static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size)
487	{
488	u32 curr_off = 0;
489	int i;
490
491	if (IS_ERR_OR_NULL(ptr: rec)) {
492	memset(dst, 0, size);
493	return;
494	}
495
496	for (i = 0; i < rec->cnt; i++) {
497	u32 next_off = rec->fields[i].offset;
498	u32 sz = next_off - curr_off;
499
500	memset(dst + curr_off, 0, sz);
501	curr_off += rec->fields[i].size + sz;
502	}
503	memset(dst + curr_off, 0, size - curr_off);
504	}
505
506	static inline void zero_map_value(struct bpf_map *map, void *dst)
507	{
508	bpf_obj_memzero(rec: map->record, dst, size: map->value_size);
509	}
510
511	void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
512	bool lock_src);
513	void bpf_timer_cancel_and_free(void *timer);
514	void bpf_list_head_free(const struct btf_field *field, void *list_head,
515	struct bpf_spin_lock *spin_lock);
516	void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
517	struct bpf_spin_lock *spin_lock);
518
519
520	int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size);
521
522	struct bpf_offload_dev;
523	struct bpf_offloaded_map;
524
525	struct bpf_map_dev_ops {
526	int (*map_get_next_key)(struct bpf_offloaded_map *map,
527	void *key, void *next_key);
528	int (*map_lookup_elem)(struct bpf_offloaded_map *map,
529	void *key, void *value);
530	int (*map_update_elem)(struct bpf_offloaded_map *map,
531	void *key, void *value, u64 flags);
532	int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key);
533	};
534
535	struct bpf_offloaded_map {
536	struct bpf_map map;
537	struct net_device *netdev;
538	const struct bpf_map_dev_ops *dev_ops;
539	void *dev_priv;
540	struct list_head offloads;
541	};
542
543	static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map)
544	{
545	return container_of(map, struct bpf_offloaded_map, map);
546	}
547
548	static inline bool bpf_map_offload_neutral(const struct bpf_map *map)
549	{
550	return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
551	}
552
553	static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
554	{
555	return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) &&
556	map->ops->map_seq_show_elem;
557	}
558
559	int map_check_no_btf(const struct bpf_map *map,
560	const struct btf *btf,
561	const struct btf_type *key_type,
562	const struct btf_type *value_type);
563
564	bool bpf_map_meta_equal(const struct bpf_map *meta0,
565	const struct bpf_map *meta1);
566
567	extern const struct bpf_map_ops bpf_map_offload_ops;
568
569	/* bpf_type_flag contains a set of flags that are applicable to the values of
570	* arg_type, ret_type and reg_type. For example, a pointer value may be null,
571	* or a memory is read-only. We classify types into two categories: base types
572	* and extended types. Extended types are base types combined with a type flag.
573	*
574	* Currently there are no more than 32 base types in arg_type, ret_type and
575	* reg_types.
576	*/
577	#define BPF_BASE_TYPE_BITS 8
578
579	enum bpf_type_flag {
580	/* PTR may be NULL. */
581	PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS),
582
583	/* MEM is read-only. When applied on bpf_arg, it indicates the arg is
584	* compatible with both mutable and immutable memory.
585	*/
586	MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS),
587
588	/* MEM points to BPF ring buffer reservation. */
589	MEM_RINGBUF = BIT(2 + BPF_BASE_TYPE_BITS),
590
591	/* MEM is in user address space. */
592	MEM_USER = BIT(3 + BPF_BASE_TYPE_BITS),
593
594	/* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged
595	* with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In
596	* order to drop this tag, it must be passed into bpf_per_cpu_ptr()
597	* or bpf_this_cpu_ptr(), which will return the pointer corresponding
598	* to the specified cpu.
599	*/
600	MEM_PERCPU = BIT(4 + BPF_BASE_TYPE_BITS),
601
602	/* Indicates that the argument will be released. */
603	OBJ_RELEASE = BIT(5 + BPF_BASE_TYPE_BITS),
604
605	/* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark
606	* unreferenced and referenced kptr loaded from map value using a load
607	* instruction, so that they can only be dereferenced but not escape the
608	* BPF program into the kernel (i.e. cannot be passed as arguments to
609	* kfunc or bpf helpers).
610	*/
611	PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS),
612
613	MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS),
614
615	/* DYNPTR points to memory local to the bpf program. */
616	DYNPTR_TYPE_LOCAL = BIT(8 + BPF_BASE_TYPE_BITS),
617
618	/* DYNPTR points to a kernel-produced ringbuf record. */
619	DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS),
620
621	/* Size is known at compile time. */
622	MEM_FIXED_SIZE = BIT(10 + BPF_BASE_TYPE_BITS),
623
624	/* MEM is of an allocated object of type in program BTF. This is used to
625	* tag PTR_TO_BTF_ID allocated using bpf_obj_new.
626	*/
627	MEM_ALLOC = BIT(11 + BPF_BASE_TYPE_BITS),
628
629	/* PTR was passed from the kernel in a trusted context, and may be
630	* passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions.
631	* Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above.
632	* PTR_UNTRUSTED refers to a kptr that was read directly from a map
633	* without invoking bpf_kptr_xchg(). What we really need to know is
634	* whether a pointer is safe to pass to a kfunc or BPF helper function.
635	* While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF
636	* helpers, they do not cover all possible instances of unsafe
637	* pointers. For example, a pointer that was obtained from walking a
638	* struct will _not_ get the PTR_UNTRUSTED type modifier, despite the
639	* fact that it may be NULL, invalid, etc. This is due to backwards
640	* compatibility requirements, as this was the behavior that was first
641	* introduced when kptrs were added. The behavior is now considered
642	* deprecated, and PTR_UNTRUSTED will eventually be removed.
643	*
644	* PTR_TRUSTED, on the other hand, is a pointer that the kernel
645	* guarantees to be valid and safe to pass to kfuncs and BPF helpers.
646	* For example, pointers passed to tracepoint arguments are considered
647	* PTR_TRUSTED, as are pointers that are passed to struct_ops
648	* callbacks. As alluded to above, pointers that are obtained from
649	* walking PTR_TRUSTED pointers are _not_ trusted. For example, if a
650	* struct task_struct *task is PTR_TRUSTED, then accessing
651	* task->last_wakee will lose the PTR_TRUSTED modifier when it's stored
652	* in a BPF register. Similarly, pointers passed to certain programs
653	* types such as kretprobes are not guaranteed to be valid, as they may
654	* for example contain an object that was recently freed.
655	*/
656	PTR_TRUSTED = BIT(12 + BPF_BASE_TYPE_BITS),
657
658	/* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */
659	MEM_RCU = BIT(13 + BPF_BASE_TYPE_BITS),
660
661	/* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning.
662	* Currently only valid for linked-list and rbtree nodes. If the nodes
663	* have a bpf_refcount_field, they must be tagged MEM_RCU as well.
664	*/
665	NON_OWN_REF = BIT(14 + BPF_BASE_TYPE_BITS),
666
667	/* DYNPTR points to sk_buff */
668	DYNPTR_TYPE_SKB = BIT(15 + BPF_BASE_TYPE_BITS),
669
670	/* DYNPTR points to xdp_buff */
671	DYNPTR_TYPE_XDP = BIT(16 + BPF_BASE_TYPE_BITS),
672
673	__BPF_TYPE_FLAG_MAX,
674	__BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1,
675	};
676
677	#define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \
678	| DYNPTR_TYPE_XDP)
679
680	/* Max number of base types. */
681	#define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
682
683	/* Max number of all types. */
684	#define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1))
685
686	/* function argument constraints */
687	enum bpf_arg_type {
688	ARG_DONTCARE = 0, /* unused argument in helper function */
689
690	/* the following constraints used to prototype
691	* bpf_map_lookup/update/delete_elem() functions
692	*/
693	ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */
694	ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */
695	ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */
696
697	/* Used to prototype bpf_memcmp() and other functions that access data
698	* on eBPF program stack
699	*/
700	ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */
701
702	ARG_CONST_SIZE, /* number of bytes accessed from memory */
703	ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */
704
705	ARG_PTR_TO_CTX, /* pointer to context */
706	ARG_ANYTHING, /* any (initialized) argument is ok */
707	ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */
708	ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
709	ARG_PTR_TO_INT, /* pointer to int */
710	ARG_PTR_TO_LONG, /* pointer to long */
711	ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
712	ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */
713	ARG_PTR_TO_RINGBUF_MEM, /* pointer to dynamically reserved ringbuf memory */
714	ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */
715	ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
716	ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */
717	ARG_PTR_TO_FUNC, /* pointer to a bpf program function */
718	ARG_PTR_TO_STACK, /* pointer to stack */
719	ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */
720	ARG_PTR_TO_TIMER, /* pointer to bpf_timer */
721	ARG_PTR_TO_KPTR, /* pointer to referenced kptr */
722	ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */
723	__BPF_ARG_TYPE_MAX,
724
725	/* Extended arg_types. */
726	ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE,
727	ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MEM,
728	ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_CTX,
729	ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET,
730	ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_STACK,
731	ARG_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_BTF_ID,
732	/* pointer to memory does not need to be initialized, helper function must fill
733	* all bytes or clear them in error case.
734	*/
735	ARG_PTR_TO_UNINIT_MEM = MEM_UNINIT | ARG_PTR_TO_MEM,
736	/* Pointer to valid memory of size known at compile time. */
737	ARG_PTR_TO_FIXED_SIZE_MEM = MEM_FIXED_SIZE | ARG_PTR_TO_MEM,
738
739	/* This must be the last entry. Its purpose is to ensure the enum is
740	* wide enough to hold the higher bits reserved for bpf_type_flag.
741	*/
742	__BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT,
743	};
744	static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
745
746	/* type of values returned from helper functions */
747	enum bpf_return_type {
748	RET_INTEGER, /* function returns integer */
749	RET_VOID, /* function doesn't return anything */
750	RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */
751	RET_PTR_TO_SOCKET, /* returns a pointer to a socket */
752	RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */
753	RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */
754	RET_PTR_TO_MEM, /* returns a pointer to memory */
755	RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */
756	RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */
757	__BPF_RET_TYPE_MAX,
758
759	/* Extended ret_types. */
760	RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE,
761	RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET,
762	RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK,
763	RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON,
764	RET_PTR_TO_RINGBUF_MEM_OR_NULL = PTR_MAYBE_NULL | MEM_RINGBUF | RET_PTR_TO_MEM,
765	RET_PTR_TO_DYNPTR_MEM_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MEM,
766	RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID,
767	RET_PTR_TO_BTF_ID_TRUSTED = PTR_TRUSTED | RET_PTR_TO_BTF_ID,
768
769	/* This must be the last entry. Its purpose is to ensure the enum is
770	* wide enough to hold the higher bits reserved for bpf_type_flag.
771	*/
772	__BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT,
773	};
774	static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
775
776	/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
777	* to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
778	* instructions after verifying
779	*/
780	struct bpf_func_proto {
781	u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
782	bool gpl_only;
783	bool pkt_access;
784	bool might_sleep;
785	enum bpf_return_type ret_type;
786	union {
787	struct {
788	enum bpf_arg_type arg1_type;
789	enum bpf_arg_type arg2_type;
790	enum bpf_arg_type arg3_type;
791	enum bpf_arg_type arg4_type;
792	enum bpf_arg_type arg5_type;
793	};
794	enum bpf_arg_type arg_type[5];
795	};
796	union {
797	struct {
798	u32 *arg1_btf_id;
799	u32 *arg2_btf_id;
800	u32 *arg3_btf_id;
801	u32 *arg4_btf_id;
802	u32 *arg5_btf_id;
803	};
804	u32 *arg_btf_id[5];
805	struct {
806	size_t arg1_size;
807	size_t arg2_size;
808	size_t arg3_size;
809	size_t arg4_size;
810	size_t arg5_size;
811	};
812	size_t arg_size[5];
813	};
814	int *ret_btf_id; /* return value btf_id */
815	bool (*allowed)(const struct bpf_prog *prog);
816	};
817
818	/* bpf_context is intentionally undefined structure. Pointer to bpf_context is
819	* the first argument to eBPF programs.
820	* For socket filters: 'struct bpf_context *' == 'struct sk_buff *'
821	*/
822	struct bpf_context;
823
824	enum bpf_access_type {
825	BPF_READ = 1,
826	BPF_WRITE = 2
827	};
828
829	/* types of values stored in eBPF registers */
830	/* Pointer types represent:
831	* pointer
832	* pointer + imm
833	* pointer + (u16) var
834	* pointer + (u16) var + imm
835	* if (range > 0) then [ptr, ptr + range - off) is safe to access
836	* if (id > 0) means that some 'var' was added
837	* if (off > 0) means that 'imm' was added
838	*/
839	enum bpf_reg_type {
840	NOT_INIT = 0, /* nothing was written into register */
841	SCALAR_VALUE, /* reg doesn't contain a valid pointer */
842	PTR_TO_CTX, /* reg points to bpf_context */
843	CONST_PTR_TO_MAP, /* reg points to struct bpf_map */
844	PTR_TO_MAP_VALUE, /* reg points to map element value */
845	PTR_TO_MAP_KEY, /* reg points to a map element key */
846	PTR_TO_STACK, /* reg == frame_pointer + offset */
847	PTR_TO_PACKET_META, /* skb->data - meta_len */
848	PTR_TO_PACKET, /* reg points to skb->data */
849	PTR_TO_PACKET_END, /* skb->data + headlen */
850	PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */
851	PTR_TO_SOCKET, /* reg points to struct bpf_sock */
852	PTR_TO_SOCK_COMMON, /* reg points to sock_common */
853	PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */
854	PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */
855	PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */
856	/* PTR_TO_BTF_ID points to a kernel struct that does not need
857	* to be null checked by the BPF program. This does not imply the
858	* pointer is _not_ null and in practice this can easily be a null
859	* pointer when reading pointer chains. The assumption is program
860	* context will handle null pointer dereference typically via fault
861	* handling. The verifier must keep this in mind and can make no
862	* assumptions about null or non-null when doing branch analysis.
863	* Further, when passed into helpers the helpers can not, without
864	* additional context, assume the value is non-null.
865	*/
866	PTR_TO_BTF_ID,
867	/* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
868	* been checked for null. Used primarily to inform the verifier
869	* an explicit null check is required for this struct.
870	*/
871	PTR_TO_MEM, /* reg points to valid memory region */
872	PTR_TO_BUF, /* reg points to a read/write buffer */
873	PTR_TO_FUNC, /* reg points to a bpf program function */
874	CONST_PTR_TO_DYNPTR, /* reg points to a const struct bpf_dynptr */
875	__BPF_REG_TYPE_MAX,
876
877	/* Extended reg_types. */
878	PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MAP_VALUE,
879	PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET,
880	PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,
881	PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,
882	PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID,
883
884	/* This must be the last entry. Its purpose is to ensure the enum is
885	* wide enough to hold the higher bits reserved for bpf_type_flag.
886	*/
887	__BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT,
888	};
889	static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
890
891	/* The information passed from prog-specific *_is_valid_access
892	* back to the verifier.
893	*/
894	struct bpf_insn_access_aux {
895	enum bpf_reg_type reg_type;
896	union {
897	int ctx_field_size;
898	struct {
899	struct btf *btf;
900	u32 btf_id;
901	};
902	};
903	struct bpf_verifier_log *log; /* for verbose logs */
904	};
905
906	static inline void
907	bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
908	{
909	aux->ctx_field_size = size;
910	}
911
912	static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
913	{
914	return insn->code == (BPF_LD | BPF_IMM | BPF_DW) &&
915	insn->src_reg == BPF_PSEUDO_FUNC;
916	}
917
918	struct bpf_prog_ops {
919	int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
920	union bpf_attr __user *uattr);
921	};
922
923	struct bpf_reg_state;
924	struct bpf_verifier_ops {
925	/* return eBPF function prototype for verification */
926	const struct bpf_func_proto *
927	(*get_func_proto)(enum bpf_func_id func_id,
928	const struct bpf_prog *prog);
929
930	/* return true if 'size' wide access at offset 'off' within bpf_context
931	* with 'type' (read or write) is allowed
932	*/
933	bool (*is_valid_access)(int off, int size, enum bpf_access_type type,
934	const struct bpf_prog *prog,
935	struct bpf_insn_access_aux *info);
936	int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
937	const struct bpf_prog *prog);
938	int (*gen_ld_abs)(const struct bpf_insn *orig,
939	struct bpf_insn *insn_buf);
940	u32 (*convert_ctx_access)(enum bpf_access_type type,
941	const struct bpf_insn *src,
942	struct bpf_insn *dst,
943	struct bpf_prog *prog, u32 *target_size);
944	int (*btf_struct_access)(struct bpf_verifier_log *log,
945	const struct bpf_reg_state *reg,
946	int off, int size);
947	};
948
949	struct bpf_prog_offload_ops {
950	/* verifier basic callbacks */
951	int (*insn_hook)(struct bpf_verifier_env *env,
952	int insn_idx, int prev_insn_idx);
953	int (*finalize)(struct bpf_verifier_env *env);
954	/* verifier optimization callbacks (called after .finalize) */
955	int (*replace_insn)(struct bpf_verifier_env *env, u32 off,
956	struct bpf_insn *insn);
957	int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);
958	/* program management callbacks */
959	int (*prepare)(struct bpf_prog *prog);
960	int (*translate)(struct bpf_prog *prog);
961	void (*destroy)(struct bpf_prog *prog);
962	};
963
964	struct bpf_prog_offload {
965	struct bpf_prog *prog;
966	struct net_device *netdev;
967	struct bpf_offload_dev *offdev;
968	void *dev_priv;
969	struct list_head offloads;
970	bool dev_state;
971	bool opt_failed;
972	void *jited_image;
973	u32 jited_len;
974	};
975
976	enum bpf_cgroup_storage_type {
977	BPF_CGROUP_STORAGE_SHARED,
978	BPF_CGROUP_STORAGE_PERCPU,
979	__BPF_CGROUP_STORAGE_MAX
980	};
981
982	#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
983
984	/* The longest tracepoint has 12 args.
985	* See include/trace/bpf_probe.h
986	*/
987	#define MAX_BPF_FUNC_ARGS 12
988
989	/* The maximum number of arguments passed through registers
990	* a single function may have.
991	*/
992	#define MAX_BPF_FUNC_REG_ARGS 5
993
994	/* The argument is a structure. */
995	#define BTF_FMODEL_STRUCT_ARG BIT(0)
996
997	/* The argument is signed. */
998	#define BTF_FMODEL_SIGNED_ARG BIT(1)
999
1000	struct btf_func_model {
1001	u8 ret_size;
1002	u8 ret_flags;
1003	u8 nr_args;
1004	u8 arg_size[MAX_BPF_FUNC_ARGS];
1005	u8 arg_flags[MAX_BPF_FUNC_ARGS];
1006	};
1007
1008	/* Restore arguments before returning from trampoline to let original function
1009	* continue executing. This flag is used for fentry progs when there are no
1010	* fexit progs.
1011	*/
1012	#define BPF_TRAMP_F_RESTORE_REGS BIT(0)
1013	/* Call original function after fentry progs, but before fexit progs.
1014	* Makes sense for fentry/fexit, normal calls and indirect calls.
1015	*/
1016	#define BPF_TRAMP_F_CALL_ORIG BIT(1)
1017	/* Skip current frame and return to parent. Makes sense for fentry/fexit
1018	* programs only. Should not be used with normal calls and indirect calls.
1019	*/
1020	#define BPF_TRAMP_F_SKIP_FRAME BIT(2)
1021	/* Store IP address of the caller on the trampoline stack,
1022	* so it's available for trampoline's programs.
1023	*/
1024	#define BPF_TRAMP_F_IP_ARG BIT(3)
1025	/* Return the return value of fentry prog. Only used by bpf_struct_ops. */
1026	#define BPF_TRAMP_F_RET_FENTRY_RET BIT(4)
1027
1028	/* Get original function from stack instead of from provided direct address.
1029	* Makes sense for trampolines with fexit or fmod_ret programs.
1030	*/
1031	#define BPF_TRAMP_F_ORIG_STACK BIT(5)
1032
1033	/* This trampoline is on a function with another ftrace_ops with IPMODIFY,
1034	* e.g., a live patch. This flag is set and cleared by ftrace call backs,
1035	*/
1036	#define BPF_TRAMP_F_SHARE_IPMODIFY BIT(6)
1037
1038	/* Indicate that current trampoline is in a tail call context. Then, it has to
1039	* cache and restore tail_call_cnt to avoid infinite tail call loop.
1040	*/
1041	#define BPF_TRAMP_F_TAIL_CALL_CTX BIT(7)
1042
1043	/* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
1044	* bytes on x86.
1045	*/
1046	enum {
1047	#if defined(__s390x__)
1048	BPF_MAX_TRAMP_LINKS = 27,
1049	#else
1050	BPF_MAX_TRAMP_LINKS = 38,
1051	#endif
1052	};
1053
1054	struct bpf_tramp_links {
1055	struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];
1056	int nr_links;
1057	};
1058
1059	struct bpf_tramp_run_ctx;
1060
1061	/* Different use cases for BPF trampoline:
1062	* 1. replace nop at the function entry (kprobe equivalent)
1063	* flags = BPF_TRAMP_F_RESTORE_REGS
1064	* fentry = a set of programs to run before returning from trampoline
1065	*
1066	* 2. replace nop at the function entry (kprobe + kretprobe equivalent)
1067	* flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME
1068	* orig_call = fentry_ip + MCOUNT_INSN_SIZE
1069	* fentry = a set of program to run before calling original function
1070	* fexit = a set of program to run after original function
1071	*
1072	* 3. replace direct call instruction anywhere in the function body
1073	* or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid)
1074	* With flags = 0
1075	* fentry = a set of programs to run before returning from trampoline
1076	* With flags = BPF_TRAMP_F_CALL_ORIG
1077	* orig_call = original callback addr or direct function addr
1078	* fentry = a set of program to run before calling original function
1079	* fexit = a set of program to run after original function
1080	*/
1081	struct bpf_tramp_image;
1082	int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end,
1083	const struct btf_func_model *m, u32 flags,
1084	struct bpf_tramp_links *tlinks,
1085	void *orig_call);
1086	u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
1087	struct bpf_tramp_run_ctx *run_ctx);
1088	void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
1089	struct bpf_tramp_run_ctx *run_ctx);
1090	void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
1091	void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
1092	typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog,
1093	struct bpf_tramp_run_ctx *run_ctx);
1094	typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start,
1095	struct bpf_tramp_run_ctx *run_ctx);
1096	bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog);
1097	bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog);
1098
1099	struct bpf_ksym {
1100	unsigned long start;
1101	unsigned long end;
1102	char name[KSYM_NAME_LEN];
1103	struct list_head lnode;
1104	struct latch_tree_node tnode;
1105	bool prog;
1106	};
1107
1108	enum bpf_tramp_prog_type {
1109	BPF_TRAMP_FENTRY,
1110	BPF_TRAMP_FEXIT,
1111	BPF_TRAMP_MODIFY_RETURN,
1112	BPF_TRAMP_MAX,
1113	BPF_TRAMP_REPLACE, /* more than MAX */
1114	};
1115
1116	struct bpf_tramp_image {
1117	void *image;
1118	struct bpf_ksym ksym;
1119	struct percpu_ref pcref;
1120	void *ip_after_call;
1121	void *ip_epilogue;
1122	union {
1123	struct rcu_head rcu;
1124	struct work_struct work;
1125	};
1126	};
1127
1128	struct bpf_trampoline {
1129	/* hlist for trampoline_table */
1130	struct hlist_node hlist;
1131	struct ftrace_ops *fops;
1132	/* serializes access to fields of this trampoline */
1133	struct mutex mutex;
1134	refcount_t refcnt;
1135	u32 flags;
1136	u64 key;
1137	struct {
1138	struct btf_func_model model;
1139	void *addr;
1140	bool ftrace_managed;
1141	} func;
1142	/* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF
1143	* program by replacing one of its functions. func.addr is the address
1144	* of the function it replaced.
1145	*/
1146	struct bpf_prog *extension_prog;
1147	/* list of BPF programs using this trampoline */
1148	struct hlist_head progs_hlist[BPF_TRAMP_MAX];
1149	/* Number of attached programs. A counter per kind. */
1150	int progs_cnt[BPF_TRAMP_MAX];
1151	/* Executable image of trampoline */
1152	struct bpf_tramp_image *cur_image;
1153	struct module *mod;
1154	};
1155
1156	struct bpf_attach_target_info {
1157	struct btf_func_model fmodel;
1158	long tgt_addr;
1159	struct module *tgt_mod;
1160	const char *tgt_name;
1161	const struct btf_type *tgt_type;
1162	};
1163
1164	#define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */
1165
1166	struct bpf_dispatcher_prog {
1167	struct bpf_prog *prog;
1168	refcount_t users;
1169	};
1170
1171	struct bpf_dispatcher {
1172	/* dispatcher mutex */
1173	struct mutex mutex;
1174	void *func;
1175	struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX];
1176	int num_progs;
1177	void *image;
1178	void *rw_image;
1179	u32 image_off;
1180	struct bpf_ksym ksym;
1181	#ifdef CONFIG_HAVE_STATIC_CALL
1182	struct static_call_key *sc_key;
1183	void *sc_tramp;
1184	#endif
1185	};
1186
1187	static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
1188	const void *ctx,
1189	const struct bpf_insn *insnsi,
1190	bpf_func_t bpf_func)
1191	{
1192	return bpf_func(ctx, insnsi);
1193	}
1194
1195	/* the implementation of the opaque uapi struct bpf_dynptr */
1196	struct bpf_dynptr_kern {
1197	void *data;
1198	/* Size represents the number of usable bytes of dynptr data.
1199	* If for example the offset is at 4 for a local dynptr whose data is
1200	* of type u64, the number of usable bytes is 4.
1201	*
1202	* The upper 8 bits are reserved. It is as follows:
1203	* Bits 0 - 23 = size
1204	* Bits 24 - 30 = dynptr type
1205	* Bit 31 = whether dynptr is read-only
1206	*/
1207	u32 size;
1208	u32 offset;
1209	} __aligned(8);
1210
1211	enum bpf_dynptr_type {
1212	BPF_DYNPTR_TYPE_INVALID,
1213	/* Points to memory that is local to the bpf program */
1214	BPF_DYNPTR_TYPE_LOCAL,
1215	/* Underlying data is a ringbuf record */
1216	BPF_DYNPTR_TYPE_RINGBUF,
1217	/* Underlying data is a sk_buff */
1218	BPF_DYNPTR_TYPE_SKB,
1219	/* Underlying data is a xdp_buff */
1220	BPF_DYNPTR_TYPE_XDP,
1221	};
1222
1223	int bpf_dynptr_check_size(u32 size);
1224	u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr);
1225
1226	#ifdef CONFIG_BPF_JIT
1227	int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
1228	int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
1229	struct bpf_trampoline *bpf_trampoline_get(u64 key,
1230	struct bpf_attach_target_info *tgt_info);
1231	void bpf_trampoline_put(struct bpf_trampoline *tr);
1232	int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);
1233
1234	/*
1235	* When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
1236	* indirection with a direct call to the bpf program. If the architecture does
1237	* not have STATIC_CALL, avoid a double-indirection.
1238	*/
1239	#ifdef CONFIG_HAVE_STATIC_CALL
1240
1241	#define __BPF_DISPATCHER_SC_INIT(_name) \
1242	.sc_key = &STATIC_CALL_KEY(_name), \
1243	.sc_tramp = STATIC_CALL_TRAMP_ADDR(_name),
1244
1245	#define __BPF_DISPATCHER_SC(name) \
1246	DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func)
1247
1248	#define __BPF_DISPATCHER_CALL(name) \
1249	static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func)
1250
1251	#define __BPF_DISPATCHER_UPDATE(_d, _new) \
1252	__static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new))
1253
1254	#else
1255	#define __BPF_DISPATCHER_SC_INIT(name)
1256	#define __BPF_DISPATCHER_SC(name)
1257	#define __BPF_DISPATCHER_CALL(name) bpf_func(ctx, insnsi)
1258	#define __BPF_DISPATCHER_UPDATE(_d, _new)
1259	#endif
1260
1261	#define BPF_DISPATCHER_INIT(_name) { \
1262	.mutex = __MUTEX_INITIALIZER(_name.mutex), \
1263	.func = &_name##_func, \
1264	.progs = {}, \
1265	.num_progs = 0, \
1266	.image = NULL, \
1267	.image_off = 0, \
1268	.ksym = { \
1269	.name = #_name, \
1270	.lnode = LIST_HEAD_INIT(_name.ksym.lnode), \
1271	}, \
1272	__BPF_DISPATCHER_SC_INIT(_name##_call) \
1273	}
1274
1275	#define DEFINE_BPF_DISPATCHER(name) \
1276	__BPF_DISPATCHER_SC(name); \
1277	noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \
1278	const void *ctx, \
1279	const struct bpf_insn *insnsi, \
1280	bpf_func_t bpf_func) \
1281	{ \
1282	return __BPF_DISPATCHER_CALL(name); \
1283	} \
1284	EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \
1285	struct bpf_dispatcher bpf_dispatcher_##name = \
1286	BPF_DISPATCHER_INIT(bpf_dispatcher_##name);
1287
1288	#define DECLARE_BPF_DISPATCHER(name) \
1289	unsigned int bpf_dispatcher_##name##_func( \
1290	const void *ctx, \
1291	const struct bpf_insn *insnsi, \
1292	bpf_func_t bpf_func); \
1293	extern struct bpf_dispatcher bpf_dispatcher_##name;
1294
1295	#define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func
1296	#define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name)
1297	void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
1298	struct bpf_prog *to);
1299	/* Called only from JIT-enabled code, so there's no need for stubs. */
1300	void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym);
1301	void bpf_image_ksym_del(struct bpf_ksym *ksym);
1302	void bpf_ksym_add(struct bpf_ksym *ksym);
1303	void bpf_ksym_del(struct bpf_ksym *ksym);
1304	int bpf_jit_charge_modmem(u32 size);
1305	void bpf_jit_uncharge_modmem(u32 size);
1306	bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
1307	#else
1308	static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
1309	struct bpf_trampoline *tr)
1310	{
1311	return -ENOTSUPP;
1312	}
1313	static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
1314	struct bpf_trampoline *tr)
1315	{
1316	return -ENOTSUPP;
1317	}
1318	static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
1319	struct bpf_attach_target_info *tgt_info)
1320	{
1321	return NULL;
1322	}
1323	static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}
1324	#define DEFINE_BPF_DISPATCHER(name)
1325	#define DECLARE_BPF_DISPATCHER(name)
1326	#define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func
1327	#define BPF_DISPATCHER_PTR(name) NULL
1328	static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d,
1329	struct bpf_prog *from,
1330	struct bpf_prog *to) {}
1331	static inline bool is_bpf_image_address(unsigned long address)
1332	{
1333	return false;
1334	}
1335	static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
1336	{
1337	return false;
1338	}
1339	#endif
1340
1341	struct bpf_func_info_aux {
1342	u16 linkage;
1343	bool unreliable;
1344	};
1345
1346	enum bpf_jit_poke_reason {
1347	BPF_POKE_REASON_TAIL_CALL,
1348	};
1349
1350	/* Descriptor of pokes pointing /into/ the JITed image. */
1351	struct bpf_jit_poke_descriptor {
1352	void *tailcall_target;
1353	void *tailcall_bypass;
1354	void *bypass_addr;
1355	void *aux;
1356	union {
1357	struct {
1358	struct bpf_map *map;
1359	u32 key;
1360	} tail_call;
1361	};
1362	bool tailcall_target_stable;
1363	u8 adj_off;
1364	u16 reason;
1365	u32 insn_idx;
1366	};
1367
1368	/* reg_type info for ctx arguments */
1369	struct bpf_ctx_arg_aux {
1370	u32 offset;
1371	enum bpf_reg_type reg_type;
1372	u32 btf_id;
1373	};
1374
1375	struct btf_mod_pair {
1376	struct btf *btf;
1377	struct module *module;
1378	};
1379
1380	struct bpf_kfunc_desc_tab;
1381
1382	struct bpf_prog_aux {
1383	atomic64_t refcnt;
1384	u32 used_map_cnt;
1385	u32 used_btf_cnt;
1386	u32 max_ctx_offset;
1387	u32 max_pkt_offset;
1388	u32 max_tp_access;
1389	u32 stack_depth;
1390	u32 id;
1391	u32 func_cnt; /* used by non-func prog as the number of func progs */
1392	u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */
1393	u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
1394	u32 attach_btf_id; /* in-kernel BTF type id to attach to */
1395	u32 ctx_arg_info_size;
1396	u32 max_rdonly_access;
1397	u32 max_rdwr_access;
1398	struct btf *attach_btf;
1399	const struct bpf_ctx_arg_aux *ctx_arg_info;
1400	struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
1401	struct bpf_prog *dst_prog;
1402	struct bpf_trampoline *dst_trampoline;
1403	enum bpf_prog_type saved_dst_prog_type;
1404	enum bpf_attach_type saved_dst_attach_type;
1405	bool verifier_zext; /* Zero extensions has been inserted by verifier. */
1406	bool dev_bound; /* Program is bound to the netdev. */
1407	bool offload_requested; /* Program is bound and offloaded to the netdev. */
1408	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
1409	bool func_proto_unreliable;
1410	bool sleepable;
1411	bool tail_call_reachable;
1412	bool xdp_has_frags;
1413	bool exception_cb;
1414	bool exception_boundary;
1415	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
1416	const struct btf_type *attach_func_proto;
1417	/* function name for valid attach_btf_id */
1418	const char *attach_func_name;
1419	struct bpf_prog **func;
1420	void *jit_data; /* JIT specific data. arch dependent */
1421	struct bpf_jit_poke_descriptor *poke_tab;
1422	struct bpf_kfunc_desc_tab *kfunc_tab;
1423	struct bpf_kfunc_btf_tab *kfunc_btf_tab;
1424	u32 size_poke_tab;
1425	struct bpf_ksym ksym;
1426	const struct bpf_prog_ops *ops;
1427	struct bpf_map **used_maps;
1428	struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
1429	struct btf_mod_pair *used_btfs;
1430	struct bpf_prog *prog;
1431	struct user_struct *user;
1432	u64 load_time; /* ns since boottime */
1433	u32 verified_insns;
1434	int cgroup_atype; /* enum cgroup_bpf_attach_type */
1435	struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
1436	char name[BPF_OBJ_NAME_LEN];
1437	unsigned int (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp);
1438	#ifdef CONFIG_SECURITY
1439	void *security;
1440	#endif
1441	struct bpf_prog_offload *offload;
1442	struct btf *btf;
1443	struct bpf_func_info *func_info;
1444	struct bpf_func_info_aux *func_info_aux;
1445	/* bpf_line_info loaded from userspace. linfo->insn_off
1446	* has the xlated insn offset.
1447	* Both the main and sub prog share the same linfo.
1448	* The subprog can access its first linfo by
1449	* using the linfo_idx.
1450	*/
1451	struct bpf_line_info *linfo;
1452	/* jited_linfo is the jited addr of the linfo. It has a
1453	* one to one mapping to linfo:
1454	* jited_linfo[i] is the jited addr for the linfo[i]->insn_off.
1455	* Both the main and sub prog share the same jited_linfo.
1456	* The subprog can access its first jited_linfo by
1457	* using the linfo_idx.
1458	*/
1459	void **jited_linfo;
1460	u32 func_info_cnt;
1461	u32 nr_linfo;
1462	/* subprog can use linfo_idx to access its first linfo and
1463	* jited_linfo.
1464	* main prog always has linfo_idx == 0
1465	*/
1466	u32 linfo_idx;
1467	struct module *mod;
1468	u32 num_exentries;
1469	struct exception_table_entry *extable;
1470	union {
1471	struct work_struct work;
1472	struct rcu_head rcu;
1473	};
1474	};
1475
1476	struct bpf_prog {
1477	u16 pages; /* Number of allocated pages */
1478	u16 jited:1, /* Is our filter JIT'ed? */
1479	jit_requested:1,/* archs need to JIT the prog */
1480	gpl_compatible:1, /* Is filter GPL compatible? */
1481	cb_access:1, /* Is control block accessed? */
1482	dst_needed:1, /* Do we need dst entry? */
1483	blinding_requested:1, /* needs constant blinding */
1484	blinded:1, /* Was blinded */
1485	is_func:1, /* program is a bpf function */
1486	kprobe_override:1, /* Do we override a kprobe? */
1487	has_callchain_buf:1, /* callchain buffer allocated? */
1488	enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
1489	call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
1490	call_get_func_ip:1, /* Do we call get_func_ip() */
1491	tstamp_type_access:1; /* Accessed __sk_buff->tstamp_type */
1492	enum bpf_prog_type type; /* Type of BPF program */
1493	enum bpf_attach_type expected_attach_type; /* For some prog types */
1494	u32 len; /* Number of filter blocks */
1495	u32 jited_len; /* Size of jited insns in bytes */
1496	u8 tag[BPF_TAG_SIZE];
1497	struct bpf_prog_stats __percpu *stats;
1498	int __percpu *active;
1499	unsigned int (*bpf_func)(const void *ctx,
1500	const struct bpf_insn *insn);
1501	struct bpf_prog_aux *aux; /* Auxiliary fields */
1502	struct sock_fprog_kern *orig_prog; /* Original BPF program */
1503	/* Instructions for interpreter */
1504	union {
1505	DECLARE_FLEX_ARRAY(struct sock_filter, insns);
1506	DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi);
1507	};
1508	};
1509
1510	struct bpf_array_aux {
1511	/* Programs with direct jumps into programs part of this array. */
1512	struct list_head poke_progs;
1513	struct bpf_map *map;
1514	struct mutex poke_mutex;
1515	struct work_struct work;
1516	};
1517
1518	struct bpf_link {
1519	atomic64_t refcnt;
1520	u32 id;
1521	enum bpf_link_type type;
1522	const struct bpf_link_ops *ops;
1523	struct bpf_prog *prog;
1524	struct work_struct work;
1525	};
1526
1527	struct bpf_link_ops {
1528	void (*release)(struct bpf_link *link);
1529	void (*dealloc)(struct bpf_link *link);
1530	int (*detach)(struct bpf_link *link);
1531	int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog,
1532	struct bpf_prog *old_prog);
1533	void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq);
1534	int (*fill_link_info)(const struct bpf_link *link,
1535	struct bpf_link_info *info);
1536	int (*update_map)(struct bpf_link *link, struct bpf_map *new_map,
1537	struct bpf_map *old_map);
1538	};
1539
1540	struct bpf_tramp_link {
1541	struct bpf_link link;
1542	struct hlist_node tramp_hlist;
1543	u64 cookie;
1544	};
1545
1546	struct bpf_shim_tramp_link {
1547	struct bpf_tramp_link link;
1548	struct bpf_trampoline *trampoline;
1549	};
1550
1551	struct bpf_tracing_link {
1552	struct bpf_tramp_link link;
1553	enum bpf_attach_type attach_type;
1554	struct bpf_trampoline *trampoline;
1555	struct bpf_prog *tgt_prog;
1556	};
1557
1558	struct bpf_link_primer {
1559	struct bpf_link *link;
1560	struct file *file;
1561	int fd;
1562	u32 id;
1563	};
1564
1565	struct bpf_struct_ops_value;
1566	struct btf_member;
1567
1568	#define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
1569	/**
1570	* struct bpf_struct_ops - A structure of callbacks allowing a subsystem to
1571	* define a BPF_MAP_TYPE_STRUCT_OPS map type composed
1572	* of BPF_PROG_TYPE_STRUCT_OPS progs.
1573	* @verifier_ops: A structure of callbacks that are invoked by the verifier
1574	* when determining whether the struct_ops progs in the
1575	* struct_ops map are valid.
1576	* @init: A callback that is invoked a single time, and before any other
1577	* callback, to initialize the structure. A nonzero return value means
1578	* the subsystem could not be initialized.
1579	* @check_member: When defined, a callback invoked by the verifier to allow
1580	* the subsystem to determine if an entry in the struct_ops map
1581	* is valid. A nonzero return value means that the map is
1582	* invalid and should be rejected by the verifier.
1583	* @init_member: A callback that is invoked for each member of the struct_ops
1584	* map to allow the subsystem to initialize the member. A nonzero
1585	* value means the member could not be initialized. This callback
1586	* is exclusive with the @type, @type_id, @value_type, and
1587	* @value_id fields.
1588	* @reg: A callback that is invoked when the struct_ops map has been
1589	* initialized and is being attached to. Zero means the struct_ops map
1590	* has been successfully registered and is live. A nonzero return value
1591	* means the struct_ops map could not be registered.
1592	* @unreg: A callback that is invoked when the struct_ops map should be
1593	* unregistered.
1594	* @update: A callback that is invoked when the live struct_ops map is being
1595	* updated to contain new values. This callback is only invoked when
1596	* the struct_ops map is loaded with BPF_F_LINK. If not defined, the
1597	* it is assumed that the struct_ops map cannot be updated.
1598	* @validate: A callback that is invoked after all of the members have been
1599	* initialized. This callback should perform static checks on the
1600	* map, meaning that it should either fail or succeed
1601	* deterministically. A struct_ops map that has been validated may
1602	* not necessarily succeed in being registered if the call to @reg
1603	* fails. For example, a valid struct_ops map may be loaded, but
1604	* then fail to be registered due to there being another active
1605	* struct_ops map on the system in the subsystem already. For this
1606	* reason, if this callback is not defined, the check is skipped as
1607	* the struct_ops map will have final verification performed in
1608	* @reg.
1609	* @type: BTF type.
1610	* @value_type: Value type.
1611	* @name: The name of the struct bpf_struct_ops object.
1612	* @func_models: Func models
1613	* @type_id: BTF type id.
1614	* @value_id: BTF value id.
1615	*/
1616	struct bpf_struct_ops {
1617	const struct bpf_verifier_ops *verifier_ops;
1618	int (*init)(struct btf *btf);
1619	int (*check_member)(const struct btf_type *t,
1620	const struct btf_member *member,
1621	const struct bpf_prog *prog);
1622	int (*init_member)(const struct btf_type *t,
1623	const struct btf_member *member,
1624	void *kdata, const void *udata);
1625	int (*reg)(void *kdata);
1626	void (*unreg)(void *kdata);
1627	int (*update)(void *kdata, void *old_kdata);
1628	int (*validate)(void *kdata);
1629	const struct btf_type *type;
1630	const struct btf_type *value_type;
1631	const char *name;
1632	struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
1633	u32 type_id;
1634	u32 value_id;
1635	};
1636
1637	#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
1638	#define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
1639	const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id);
1640	void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log);
1641	bool bpf_struct_ops_get(const void *kdata);
1642	void bpf_struct_ops_put(const void *kdata);
1643	int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
1644	void *value);
1645	int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
1646	struct bpf_tramp_link *link,
1647	const struct btf_func_model *model,
1648	void *image, void *image_end);
1649	static inline bool bpf_try_module_get(const void *data, struct module *owner)
1650	{
1651	if (owner == BPF_MODULE_OWNER)
1652	return bpf_struct_ops_get(kdata: data);
1653	else
1654	return try_module_get(module: owner);
1655	}
1656	static inline void bpf_module_put(const void *data, struct module *owner)
1657	{
1658	if (owner == BPF_MODULE_OWNER)
1659	bpf_struct_ops_put(kdata: data);
1660	else
1661	module_put(module: owner);
1662	}
1663	int bpf_struct_ops_link_create(union bpf_attr *attr);
1664
1665	#ifdef CONFIG_NET
1666	/* Define it here to avoid the use of forward declaration */
1667	struct bpf_dummy_ops_state {
1668	int val;
1669	};
1670
1671	struct bpf_dummy_ops {
1672	int (*test_1)(struct bpf_dummy_ops_state *cb);
1673	int (*test_2)(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2,
1674	char a3, unsigned long a4);
1675	int (*test_sleepable)(struct bpf_dummy_ops_state *cb);
1676	};
1677
1678	int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
1679	union bpf_attr __user *uattr);
1680	#endif
1681	#else
1682	static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
1683	{
1684	return NULL;
1685	}
1686	static inline void bpf_struct_ops_init(struct btf *btf,
1687	struct bpf_verifier_log *log)
1688	{
1689	}
1690	static inline bool bpf_try_module_get(const void *data, struct module *owner)
1691	{
1692	return try_module_get(owner);
1693	}
1694	static inline void bpf_module_put(const void *data, struct module *owner)
1695	{
1696	module_put(owner);
1697	}
1698	static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,
1699	void *key,
1700	void *value)
1701	{
1702	return -EINVAL;
1703	}
1704	static inline int bpf_struct_ops_link_create(union bpf_attr *attr)
1705	{
1706	return -EOPNOTSUPP;
1707	}
1708
1709	#endif
1710
1711	#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
1712	int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
1713	int cgroup_atype);
1714	void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
1715	#else
1716	static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
1717	int cgroup_atype)
1718	{
1719	return -EOPNOTSUPP;
1720	}
1721	static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
1722	{
1723	}
1724	#endif
1725
1726	struct bpf_array {
1727	struct bpf_map map;
1728	u32 elem_size;
1729	u32 index_mask;
1730	struct bpf_array_aux *aux;
1731	union {
1732	DECLARE_FLEX_ARRAY(char, value) __aligned(8);
1733	DECLARE_FLEX_ARRAY(void *, ptrs) __aligned(8);
1734	DECLARE_FLEX_ARRAY(void __percpu *, pptrs) __aligned(8);
1735	};
1736	};
1737
1738	#define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */
1739	#define MAX_TAIL_CALL_CNT 33
1740
1741	/* Maximum number of loops for bpf_loop and bpf_iter_num.
1742	* It's enum to expose it (and thus make it discoverable) through BTF.
1743	*/
1744	enum {
1745	BPF_MAX_LOOPS = 8 * 1024 * 1024,
1746	};
1747
1748	#define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \
1749	BPF_F_RDONLY_PROG | \
1750	BPF_F_WRONLY | \
1751	BPF_F_WRONLY_PROG)
1752
1753	#define BPF_MAP_CAN_READ BIT(0)
1754	#define BPF_MAP_CAN_WRITE BIT(1)
1755
1756	/* Maximum number of user-producer ring buffer samples that can be drained in
1757	* a call to bpf_user_ringbuf_drain().
1758	*/
1759	#define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024)
1760
1761	static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)
1762	{
1763	u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
1764
1765	/* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is
1766	* not possible.
1767	*/
1768	if (access_flags & BPF_F_RDONLY_PROG)
1769	return BPF_MAP_CAN_READ;
1770	else if (access_flags & BPF_F_WRONLY_PROG)
1771	return BPF_MAP_CAN_WRITE;
1772	else
1773	return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE;
1774	}
1775
1776	static inline bool bpf_map_flags_access_ok(u32 access_flags)
1777	{
1778	return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) !=
1779	(BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
1780	}
1781
1782	struct bpf_event_entry {
1783	struct perf_event *event;
1784	struct file *perf_file;
1785	struct file *map_file;
1786	struct rcu_head rcu;
1787	};
1788
1789	static inline bool map_type_contains_progs(struct bpf_map *map)
1790	{
1791	return map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
1792	map->map_type == BPF_MAP_TYPE_DEVMAP ||
1793	map->map_type == BPF_MAP_TYPE_CPUMAP;
1794	}
1795
1796	bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp);
1797	int bpf_prog_calc_tag(struct bpf_prog *fp);
1798
1799	const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
1800	const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);
1801
1802	typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
1803	unsigned long off, unsigned long len);
1804	typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
1805	const struct bpf_insn *src,
1806	struct bpf_insn *dst,
1807	struct bpf_prog *prog,
1808	u32 *target_size);
1809
1810	u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
1811	void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);
1812
1813	/* an array of programs to be executed under rcu_lock.
1814	*
1815	* Typical usage:
1816	* ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run);
1817	*
1818	* the structure returned by bpf_prog_array_alloc() should be populated
1819	* with program pointers and the last pointer must be NULL.
1820	* The user has to keep refcnt on the program and make sure the program
1821	* is removed from the array before bpf_prog_put().
1822	* The 'struct bpf_prog_array *' should only be replaced with xchg()
1823	* since other cpus are walking the array of pointers in parallel.
1824	*/
1825	struct bpf_prog_array_item {
1826	struct bpf_prog *prog;
1827	union {
1828	struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
1829	u64 bpf_cookie;
1830	};
1831	};
1832
1833	struct bpf_prog_array {
1834	struct rcu_head rcu;
1835	struct bpf_prog_array_item items[];
1836	};
1837
1838	struct bpf_empty_prog_array {
1839	struct bpf_prog_array hdr;
1840	struct bpf_prog *null_prog;
1841	};
1842
1843	/* to avoid allocating empty bpf_prog_array for cgroups that
1844	* don't have bpf program attached use one global 'bpf_empty_prog_array'
1845	* It will not be modified the caller of bpf_prog_array_alloc()
1846	* (since caller requested prog_cnt == 0)
1847	* that pointer should be 'freed' by bpf_prog_array_free()
1848	*/
1849	extern struct bpf_empty_prog_array bpf_empty_prog_array;
1850
1851	struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
1852	void bpf_prog_array_free(struct bpf_prog_array *progs);
1853	/* Use when traversal over the bpf_prog_array uses tasks_trace rcu */
1854	void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs);
1855	int bpf_prog_array_length(struct bpf_prog_array *progs);
1856	bool bpf_prog_array_is_empty(struct bpf_prog_array *array);
1857	int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
1858	__u32 __user *prog_ids, u32 cnt);
1859
1860	void bpf_prog_array_delete_safe(struct bpf_prog_array *progs,
1861	struct bpf_prog *old_prog);
1862	int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index);
1863	int bpf_prog_array_update_at(struct bpf_prog_array *array, int index,
1864	struct bpf_prog *prog);
1865	int bpf_prog_array_copy_info(struct bpf_prog_array *array,
1866	u32 *prog_ids, u32 request_cnt,
1867	u32 *prog_cnt);
1868	int bpf_prog_array_copy(struct bpf_prog_array *old_array,
1869	struct bpf_prog *exclude_prog,
1870	struct bpf_prog *include_prog,
1871	u64 bpf_cookie,
1872	struct bpf_prog_array **new_array);
1873
1874	struct bpf_run_ctx {};
1875
1876	struct bpf_cg_run_ctx {
1877	struct bpf_run_ctx run_ctx;
1878	const struct bpf_prog_array_item *prog_item;
1879	int retval;
1880	};
1881
1882	struct bpf_trace_run_ctx {
1883	struct bpf_run_ctx run_ctx;
1884	u64 bpf_cookie;
1885	bool is_uprobe;
1886	};
1887
1888	struct bpf_tramp_run_ctx {
1889	struct bpf_run_ctx run_ctx;
1890	u64 bpf_cookie;
1891	struct bpf_run_ctx *saved_run_ctx;
1892	};
1893
1894	static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
1895	{
1896	struct bpf_run_ctx *old_ctx = NULL;
1897
1898	#ifdef CONFIG_BPF_SYSCALL
1899	old_ctx = current->bpf_ctx;
1900	current->bpf_ctx = new_ctx;
1901	#endif
1902	return old_ctx;
1903	}
1904
1905	static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx)
1906	{
1907	#ifdef CONFIG_BPF_SYSCALL
1908	current->bpf_ctx = old_ctx;
1909	#endif
1910	}
1911
1912	/* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */
1913	#define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0)
1914	/* BPF program asks to set CN on the packet. */
1915	#define BPF_RET_SET_CN (1 << 0)
1916
1917	typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx);
1918
1919	static __always_inline u32
1920	bpf_prog_run_array(const struct bpf_prog_array *array,
1921	const void *ctx, bpf_prog_run_fn run_prog)
1922	{
1923	const struct bpf_prog_array_item *item;
1924	const struct bpf_prog *prog;
1925	struct bpf_run_ctx *old_run_ctx;
1926	struct bpf_trace_run_ctx run_ctx;
1927	u32 ret = 1;
1928
1929	RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held");
1930
1931	if (unlikely(!array))
1932	return ret;
1933
1934	run_ctx.is_uprobe = false;
1935
1936	migrate_disable();
1937	old_run_ctx = bpf_set_run_ctx(new_ctx: &run_ctx.run_ctx);
1938	item = &array->items[0];
1939	while ((prog = READ_ONCE(item->prog))) {
1940	run_ctx.bpf_cookie = item->bpf_cookie;
1941	ret &= run_prog(prog, ctx);
1942	item++;
1943	}
1944	bpf_reset_run_ctx(old_ctx: old_run_ctx);
1945	migrate_enable();
1946	return ret;
1947	}
1948
1949	/* Notes on RCU design for bpf_prog_arrays containing sleepable programs:
1950	*
1951	* We use the tasks_trace rcu flavor read section to protect the bpf_prog_array
1952	* overall. As a result, we must use the bpf_prog_array_free_sleepable
1953	* in order to use the tasks_trace rcu grace period.
1954	*
1955	* When a non-sleepable program is inside the array, we take the rcu read
1956	* section and disable preemption for that program alone, so it can access
1957	* rcu-protected dynamically sized maps.
1958	*/
1959	static __always_inline u32
1960	bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu,
1961	const void *ctx, bpf_prog_run_fn run_prog)
1962	{
1963	const struct bpf_prog_array_item *item;
1964	const struct bpf_prog *prog;
1965	const struct bpf_prog_array *array;
1966	struct bpf_run_ctx *old_run_ctx;
1967	struct bpf_trace_run_ctx run_ctx;
1968	u32 ret = 1;
1969
1970	might_fault();
1971
1972	rcu_read_lock_trace();
1973	migrate_disable();
1974
1975	run_ctx.is_uprobe = true;
1976
1977	array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held());
1978	if (unlikely(!array))
1979	goto out;
1980	old_run_ctx = bpf_set_run_ctx(new_ctx: &run_ctx.run_ctx);
1981	item = &array->items[0];
1982	while ((prog = READ_ONCE(item->prog))) {
1983	if (!prog->aux->sleepable)
1984	rcu_read_lock();
1985
1986	run_ctx.bpf_cookie = item->bpf_cookie;
1987	ret &= run_prog(prog, ctx);
1988	item++;
1989
1990	if (!prog->aux->sleepable)
1991	rcu_read_unlock();
1992	}
1993	bpf_reset_run_ctx(old_ctx: old_run_ctx);
1994	out:
1995	migrate_enable();
1996	rcu_read_unlock_trace();
1997	return ret;
1998	}
1999
2000	#ifdef CONFIG_BPF_SYSCALL
2001	DECLARE_PER_CPU(int, bpf_prog_active);
2002	extern struct mutex bpf_stats_enabled_mutex;
2003
2004	/*
2005	* Block execution of BPF programs attached to instrumentation (perf,
2006	* kprobes, tracepoints) to prevent deadlocks on map operations as any of
2007	* these events can happen inside a region which holds a map bucket lock
2008	* and can deadlock on it.
2009	*/
2010	static inline void bpf_disable_instrumentation(void)
2011	{
2012	migrate_disable();
2013	this_cpu_inc(bpf_prog_active);
2014	}
2015
2016	static inline void bpf_enable_instrumentation(void)
2017	{
2018	this_cpu_dec(bpf_prog_active);
2019	migrate_enable();
2020	}
2021
2022	extern const struct file_operations bpf_map_fops;
2023	extern const struct file_operations bpf_prog_fops;
2024	extern const struct file_operations bpf_iter_fops;
2025
2026	#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
2027	extern const struct bpf_prog_ops _name ## _prog_ops; \
2028	extern const struct bpf_verifier_ops _name ## _verifier_ops;
2029	#define BPF_MAP_TYPE(_id, _ops) \
2030	extern const struct bpf_map_ops _ops;
2031	#define BPF_LINK_TYPE(_id, _name)
2032	#include <linux/bpf_types.h>
2033	#undef BPF_PROG_TYPE
2034	#undef BPF_MAP_TYPE
2035	#undef BPF_LINK_TYPE
2036
2037	extern const struct bpf_prog_ops bpf_offload_prog_ops;
2038	extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;
2039	extern const struct bpf_verifier_ops xdp_analyzer_ops;
2040
2041	struct bpf_prog *bpf_prog_get(u32 ufd);
2042	struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
2043	bool attach_drv);
2044	void bpf_prog_add(struct bpf_prog *prog, int i);
2045	void bpf_prog_sub(struct bpf_prog *prog, int i);
2046	void bpf_prog_inc(struct bpf_prog *prog);
2047	struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
2048	void bpf_prog_put(struct bpf_prog *prog);
2049
2050	void bpf_prog_free_id(struct bpf_prog *prog);
2051	void bpf_map_free_id(struct bpf_map *map);
2052
2053	struct btf_field *btf_record_find(const struct btf_record *rec,
2054	u32 offset, u32 field_mask);
2055	void btf_record_free(struct btf_record *rec);
2056	void bpf_map_free_record(struct bpf_map *map);
2057	struct btf_record *btf_record_dup(const struct btf_record *rec);
2058	bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b);
2059	void bpf_obj_free_timer(const struct btf_record *rec, void *obj);
2060	void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
2061	void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);
2062
2063	struct bpf_map *bpf_map_get(u32 ufd);
2064	struct bpf_map *bpf_map_get_with_uref(u32 ufd);
2065	struct bpf_map *__bpf_map_get(struct fd f);
2066	void bpf_map_inc(struct bpf_map *map);
2067	void bpf_map_inc_with_uref(struct bpf_map *map);
2068	struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref);
2069	struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map);
2070	void bpf_map_put_with_uref(struct bpf_map *map);
2071	void bpf_map_put(struct bpf_map *map);
2072	void *bpf_map_area_alloc(u64 size, int numa_node);
2073	void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);
2074	void bpf_map_area_free(void *base);
2075	bool bpf_map_write_active(const struct bpf_map *map);
2076	void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
2077	int generic_map_lookup_batch(struct bpf_map *map,
2078	const union bpf_attr *attr,
2079	union bpf_attr __user *uattr);
2080	int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
2081	const union bpf_attr *attr,
2082	union bpf_attr __user *uattr);
2083	int generic_map_delete_batch(struct bpf_map *map,
2084	const union bpf_attr *attr,
2085	union bpf_attr __user *uattr);
2086	struct bpf_map *bpf_map_get_curr_or_next(u32 *id);
2087	struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);
2088
2089	#ifdef CONFIG_MEMCG_KMEM
2090	void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
2091	int node);
2092	void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags);
2093	void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
2094	gfp_t flags);
2095	void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
2096	size_t align, gfp_t flags);
2097	#else
2098	static inline void *
2099	bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
2100	int node)
2101	{
2102	return kmalloc_node(size, flags, node);
2103	}
2104
2105	static inline void *
2106	bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
2107	{
2108	return kzalloc(size, flags);
2109	}
2110
2111	static inline void *
2112	bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, gfp_t flags)
2113	{
2114	return kvcalloc(n, size, flags);
2115	}
2116
2117	static inline void __percpu *
2118	bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align,
2119	gfp_t flags)
2120	{
2121	return __alloc_percpu_gfp(size, align, flags);
2122	}
2123	#endif
2124
2125	static inline int
2126	bpf_map_init_elem_count(struct bpf_map *map)
2127	{
2128	size_t size = sizeof(*map->elem_count), align = size;
2129	gfp_t flags = GFP_USER | __GFP_NOWARN;
2130
2131	map->elem_count = bpf_map_alloc_percpu(map, size, align, flags);
2132	if (!map->elem_count)
2133	return -ENOMEM;
2134
2135	return 0;
2136	}
2137
2138	static inline void
2139	bpf_map_free_elem_count(struct bpf_map *map)
2140	{
2141	free_percpu(pdata: map->elem_count);
2142	}
2143
2144	static inline void bpf_map_inc_elem_count(struct bpf_map *map)
2145	{
2146	this_cpu_inc(*map->elem_count);
2147	}
2148
2149	static inline void bpf_map_dec_elem_count(struct bpf_map *map)
2150	{
2151	this_cpu_dec(*map->elem_count);
2152	}
2153
2154	extern int sysctl_unprivileged_bpf_disabled;
2155
2156	static inline bool bpf_allow_ptr_leaks(void)
2157	{
2158	return perfmon_capable();
2159	}
2160
2161	static inline bool bpf_allow_uninit_stack(void)
2162	{
2163	return perfmon_capable();
2164	}
2165
2166	static inline bool bpf_bypass_spec_v1(void)
2167	{
2168	return cpu_mitigations_off() || perfmon_capable();
2169	}
2170
2171	static inline bool bpf_bypass_spec_v4(void)
2172	{
2173	return cpu_mitigations_off() || perfmon_capable();
2174	}
2175
2176	int bpf_map_new_fd(struct bpf_map *map, int flags);
2177	int bpf_prog_new_fd(struct bpf_prog *prog);
2178
2179	void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2180	const struct bpf_link_ops *ops, struct bpf_prog *prog);
2181	int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer);
2182	int bpf_link_settle(struct bpf_link_primer *primer);
2183	void bpf_link_cleanup(struct bpf_link_primer *primer);
2184	void bpf_link_inc(struct bpf_link *link);
2185	void bpf_link_put(struct bpf_link *link);
2186	int bpf_link_new_fd(struct bpf_link *link);
2187	struct bpf_link *bpf_link_get_from_fd(u32 ufd);
2188	struct bpf_link *bpf_link_get_curr_or_next(u32 *id);
2189
2190	int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname);
2191	int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags);
2192
2193	#define BPF_ITER_FUNC_PREFIX "bpf_iter_"
2194	#define DEFINE_BPF_ITER_FUNC(target, args...) \
2195	extern int bpf_iter_ ## target(args); \
2196	int __init bpf_iter_ ## target(args) { return 0; }
2197
2198	/*
2199	* The task type of iterators.
2200	*
2201	* For BPF task iterators, they can be parameterized with various
2202	* parameters to visit only some of tasks.
2203	*
2204	* BPF_TASK_ITER_ALL (default)
2205	* Iterate over resources of every task.
2206	*
2207	* BPF_TASK_ITER_TID
2208	* Iterate over resources of a task/tid.
2209	*
2210	* BPF_TASK_ITER_TGID
2211	* Iterate over resources of every task of a process / task group.
2212	*/
2213	enum bpf_iter_task_type {
2214	BPF_TASK_ITER_ALL = 0,
2215	BPF_TASK_ITER_TID,
2216	BPF_TASK_ITER_TGID,
2217	};
2218
2219	struct bpf_iter_aux_info {
2220	/* for map_elem iter */
2221	struct bpf_map *map;
2222
2223	/* for cgroup iter */
2224	struct {
2225	struct cgroup *start; /* starting cgroup */
2226	enum bpf_cgroup_iter_order order;
2227	} cgroup;
2228	struct {
2229	enum bpf_iter_task_type type;
2230	u32 pid;
2231	} task;
2232	};
2233
2234	typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog,
2235	union bpf_iter_link_info *linfo,
2236	struct bpf_iter_aux_info *aux);
2237	typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux);
2238	typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux,
2239	struct seq_file *seq);
2240	typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux,
2241	struct bpf_link_info *info);
2242	typedef const struct bpf_func_proto *
2243	(*bpf_iter_get_func_proto_t)(enum bpf_func_id func_id,
2244	const struct bpf_prog *prog);
2245
2246	enum bpf_iter_feature {
2247	BPF_ITER_RESCHED = BIT(0),
2248	};
2249
2250	#define BPF_ITER_CTX_ARG_MAX 2
2251	struct bpf_iter_reg {
2252	const char *target;
2253	bpf_iter_attach_target_t attach_target;
2254	bpf_iter_detach_target_t detach_target;
2255	bpf_iter_show_fdinfo_t show_fdinfo;
2256	bpf_iter_fill_link_info_t fill_link_info;
2257	bpf_iter_get_func_proto_t get_func_proto;
2258	u32 ctx_arg_info_size;
2259	u32 feature;
2260	struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX];
2261	const struct bpf_iter_seq_info *seq_info;
2262	};
2263
2264	struct bpf_iter_meta {
2265	__bpf_md_ptr(struct seq_file *, seq);
2266	u64 session_id;
2267	u64 seq_num;
2268	};
2269
2270	struct bpf_iter__bpf_map_elem {
2271	__bpf_md_ptr(struct bpf_iter_meta *, meta);
2272	__bpf_md_ptr(struct bpf_map *, map);
2273	__bpf_md_ptr(void *, key);
2274	__bpf_md_ptr(void *, value);
2275	};
2276
2277	int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
2278	void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
2279	bool bpf_iter_prog_supported(struct bpf_prog *prog);
2280	const struct bpf_func_proto *
2281	bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
2282	int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog);
2283	int bpf_iter_new_fd(struct bpf_link *link);
2284	bool bpf_link_is_iter(struct bpf_link *link);
2285	struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop);
2286	int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx);
2287	void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux,
2288	struct seq_file *seq);
2289	int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,
2290	struct bpf_link_info *info);
2291
2292	int map_set_for_each_callback_args(struct bpf_verifier_env *env,
2293	struct bpf_func_state *caller,
2294	struct bpf_func_state *callee);
2295
2296	int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
2297	int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
2298	int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
2299	u64 flags);
2300	int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
2301	u64 flags);
2302
2303	int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);
2304
2305	int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
2306	void *key, void *value, u64 map_flags);
2307	int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
2308	int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
2309	void *key, void *value, u64 map_flags);
2310	int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
2311
2312	int bpf_get_file_flag(int flags);
2313	int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
2314	size_t actual_size);
2315
2316	/* verify correctness of eBPF program */
2317	int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);
2318
2319	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
2320	void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
2321	#endif
2322
2323	struct btf *bpf_get_btf_vmlinux(void);
2324
2325	/* Map specifics */
2326	struct xdp_frame;
2327	struct sk_buff;
2328	struct bpf_dtab_netdev;
2329	struct bpf_cpu_map_entry;
2330
2331	void __dev_flush(void);
2332	int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
2333	struct net_device *dev_rx);
2334	int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
2335	struct net_device *dev_rx);
2336	int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
2337	struct bpf_map *map, bool exclude_ingress);
2338	int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
2339	struct bpf_prog *xdp_prog);
2340	int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
2341	struct bpf_prog *xdp_prog, struct bpf_map *map,
2342	bool exclude_ingress);
2343
2344	void __cpu_map_flush(void);
2345	int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
2346	struct net_device *dev_rx);
2347	int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
2348	struct sk_buff *skb);
2349
2350	/* Return map's numa specified by userspace */
2351	static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
2352	{
2353	return (attr->map_flags & BPF_F_NUMA_NODE) ?
2354	attr->numa_node : NUMA_NO_NODE;
2355	}
2356
2357	struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type);
2358	int array_map_alloc_check(union bpf_attr *attr);
2359
2360	int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
2361	union bpf_attr __user *uattr);
2362	int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
2363	union bpf_attr __user *uattr);
2364	int bpf_prog_test_run_tracing(struct bpf_prog *prog,
2365	const union bpf_attr *kattr,
2366	union bpf_attr __user *uattr);
2367	int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
2368	const union bpf_attr *kattr,
2369	union bpf_attr __user *uattr);
2370	int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
2371	const union bpf_attr *kattr,
2372	union bpf_attr __user *uattr);
2373	int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
2374	const union bpf_attr *kattr,
2375	union bpf_attr __user *uattr);
2376	int bpf_prog_test_run_nf(struct bpf_prog *prog,
2377	const union bpf_attr *kattr,
2378	union bpf_attr __user *uattr);
2379	bool btf_ctx_access(int off, int size, enum bpf_access_type type,
2380	const struct bpf_prog *prog,
2381	struct bpf_insn_access_aux *info);
2382
2383	static inline bool bpf_tracing_ctx_access(int off, int size,
2384	enum bpf_access_type type)
2385	{
2386	if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
2387	return false;
2388	if (type != BPF_READ)
2389	return false;
2390	if (off % size != 0)
2391	return false;
2392	return true;
2393	}
2394
2395	static inline bool bpf_tracing_btf_ctx_access(int off, int size,
2396	enum bpf_access_type type,
2397	const struct bpf_prog *prog,
2398	struct bpf_insn_access_aux *info)
2399	{
2400	if (!bpf_tracing_ctx_access(off, size, type))
2401	return false;
2402	return btf_ctx_access(off, size, type, prog, info);
2403	}
2404
2405	int btf_struct_access(struct bpf_verifier_log *log,
2406	const struct bpf_reg_state *reg,
2407	int off, int size, enum bpf_access_type atype,
2408	u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name);
2409	bool btf_struct_ids_match(struct bpf_verifier_log *log,
2410	const struct btf *btf, u32 id, int off,
2411	const struct btf *need_btf, u32 need_type_id,
2412	bool strict);
2413
2414	int btf_distill_func_proto(struct bpf_verifier_log *log,
2415	struct btf *btf,
2416	const struct btf_type *func_proto,
2417	const char *func_name,
2418	struct btf_func_model *m);
2419
2420	struct bpf_reg_state;
2421	int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
2422	struct bpf_reg_state *regs);
2423	int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
2424	struct bpf_reg_state *regs);
2425	int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
2426	struct bpf_reg_state *reg, bool is_ex_cb);
2427	int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
2428	struct btf *btf, const struct btf_type *t);
2429	const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt,
2430	int comp_idx, const char *tag_key);
2431
2432	struct bpf_prog *bpf_prog_by_id(u32 id);
2433	struct bpf_link *bpf_link_by_id(u32 id);
2434
2435	const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id);
2436	void bpf_task_storage_free(struct task_struct *task);
2437	void bpf_cgrp_storage_free(struct cgroup *cgroup);
2438	bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
2439	const struct btf_func_model *
2440	bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
2441	const struct bpf_insn *insn);
2442	int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2443	u16 btf_fd_idx, u8 **func_addr);
2444
2445	struct bpf_core_ctx {
2446	struct bpf_verifier_log *log;
2447	const struct btf *btf;
2448	};
2449
2450	bool btf_nested_type_is_trusted(struct bpf_verifier_log *log,
2451	const struct bpf_reg_state *reg,
2452	const char *field_name, u32 btf_id, const char *suffix);
2453
2454	bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log,
2455	const struct btf *reg_btf, u32 reg_id,
2456	const struct btf *arg_btf, u32 arg_id);
2457
2458	int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
2459	int relo_idx, void *insn);
2460
2461	static inline bool unprivileged_ebpf_enabled(void)
2462	{
2463	return !sysctl_unprivileged_bpf_disabled;
2464	}
2465
2466	/* Not all bpf prog type has the bpf_ctx.
2467	* For the bpf prog type that has initialized the bpf_ctx,
2468	* this function can be used to decide if a kernel function
2469	* is called by a bpf program.
2470	*/
2471	static inline bool has_current_bpf_ctx(void)
2472	{
2473	return !!current->bpf_ctx;
2474	}
2475
2476	void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
2477
2478	void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
2479	enum bpf_dynptr_type type, u32 offset, u32 size);
2480	void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
2481	void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
2482
2483	bool dev_check_flush(void);
2484	bool cpu_map_check_flush(void);
2485	#else /* !CONFIG_BPF_SYSCALL */
2486	static inline struct bpf_prog *bpf_prog_get(u32 ufd)
2487	{
2488	return ERR_PTR(-EOPNOTSUPP);
2489	}
2490
2491	static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd,
2492	enum bpf_prog_type type,
2493	bool attach_drv)
2494	{
2495	return ERR_PTR(-EOPNOTSUPP);
2496	}
2497
2498	static inline void bpf_prog_add(struct bpf_prog *prog, int i)
2499	{
2500	}
2501
2502	static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
2503	{
2504	}
2505
2506	static inline void bpf_prog_put(struct bpf_prog *prog)
2507	{
2508	}
2509
2510	static inline void bpf_prog_inc(struct bpf_prog *prog)
2511	{
2512	}
2513
2514	static inline struct bpf_prog *__must_check
2515	bpf_prog_inc_not_zero(struct bpf_prog *prog)
2516	{
2517	return ERR_PTR(-EOPNOTSUPP);
2518	}
2519
2520	static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2521	const struct bpf_link_ops *ops,
2522	struct bpf_prog *prog)
2523	{
2524	}
2525
2526	static inline int bpf_link_prime(struct bpf_link *link,
2527	struct bpf_link_primer *primer)
2528	{
2529	return -EOPNOTSUPP;
2530	}
2531
2532	static inline int bpf_link_settle(struct bpf_link_primer *primer)
2533	{
2534	return -EOPNOTSUPP;
2535	}
2536
2537	static inline void bpf_link_cleanup(struct bpf_link_primer *primer)
2538	{
2539	}
2540
2541	static inline void bpf_link_inc(struct bpf_link *link)
2542	{
2543	}
2544
2545	static inline void bpf_link_put(struct bpf_link *link)
2546	{
2547	}
2548
2549	static inline int bpf_obj_get_user(const char __user *pathname, int flags)
2550	{
2551	return -EOPNOTSUPP;
2552	}
2553
2554	static inline void __dev_flush(void)
2555	{
2556	}
2557
2558	struct xdp_frame;
2559	struct bpf_dtab_netdev;
2560	struct bpf_cpu_map_entry;
2561
2562	static inline
2563	int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
2564	struct net_device *dev_rx)
2565	{
2566	return 0;
2567	}
2568
2569	static inline
2570	int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
2571	struct net_device *dev_rx)
2572	{
2573	return 0;
2574	}
2575
2576	static inline
2577	int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
2578	struct bpf_map *map, bool exclude_ingress)
2579	{
2580	return 0;
2581	}
2582
2583	struct sk_buff;
2584
2585	static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,
2586	struct sk_buff *skb,
2587	struct bpf_prog *xdp_prog)
2588	{
2589	return 0;
2590	}
2591
2592	static inline
2593	int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
2594	struct bpf_prog *xdp_prog, struct bpf_map *map,
2595	bool exclude_ingress)
2596	{
2597	return 0;
2598	}
2599
2600	static inline void __cpu_map_flush(void)
2601	{
2602	}
2603
2604	static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
2605	struct xdp_frame *xdpf,
2606	struct net_device *dev_rx)
2607	{
2608	return 0;
2609	}
2610
2611	static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
2612	struct sk_buff *skb)
2613	{
2614	return -EOPNOTSUPP;
2615	}
2616
2617	static inline struct bpf_prog *bpf_prog_get_type_path(const char *name,
2618	enum bpf_prog_type type)
2619	{
2620	return ERR_PTR(-EOPNOTSUPP);
2621	}
2622
2623	static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog,
2624	const union bpf_attr *kattr,
2625	union bpf_attr __user *uattr)
2626	{
2627	return -ENOTSUPP;
2628	}
2629
2630	static inline int bpf_prog_test_run_skb(struct bpf_prog *prog,
2631	const union bpf_attr *kattr,
2632	union bpf_attr __user *uattr)
2633	{
2634	return -ENOTSUPP;
2635	}
2636
2637	static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog,
2638	const union bpf_attr *kattr,
2639	union bpf_attr __user *uattr)
2640	{
2641	return -ENOTSUPP;
2642	}
2643
2644	static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
2645	const union bpf_attr *kattr,
2646	union bpf_attr __user *uattr)
2647	{
2648	return -ENOTSUPP;
2649	}
2650
2651	static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
2652	const union bpf_attr *kattr,
2653	union bpf_attr __user *uattr)
2654	{
2655	return -ENOTSUPP;
2656	}
2657
2658	static inline void bpf_map_put(struct bpf_map *map)
2659	{
2660	}
2661
2662	static inline struct bpf_prog *bpf_prog_by_id(u32 id)
2663	{
2664	return ERR_PTR(-ENOTSUPP);
2665	}
2666
2667	static inline int btf_struct_access(struct bpf_verifier_log *log,
2668	const struct bpf_reg_state *reg,
2669	int off, int size, enum bpf_access_type atype,
2670	u32 *next_btf_id, enum bpf_type_flag *flag,
2671	const char **field_name)
2672	{
2673	return -EACCES;
2674	}
2675
2676	static inline const struct bpf_func_proto *
2677	bpf_base_func_proto(enum bpf_func_id func_id)
2678	{
2679	return NULL;
2680	}
2681
2682	static inline void bpf_task_storage_free(struct task_struct *task)
2683	{
2684	}
2685
2686	static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
2687	{
2688	return false;
2689	}
2690
2691	static inline const struct btf_func_model *
2692	bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
2693	const struct bpf_insn *insn)
2694	{
2695	return NULL;
2696	}
2697
2698	static inline int
2699	bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2700	u16 btf_fd_idx, u8 **func_addr)
2701	{
2702	return -ENOTSUPP;
2703	}
2704
2705	static inline bool unprivileged_ebpf_enabled(void)
2706	{
2707	return false;
2708	}
2709
2710	static inline bool has_current_bpf_ctx(void)
2711	{
2712	return false;
2713	}
2714
2715	static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
2716	{
2717	}
2718
2719	static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)
2720	{
2721	}
2722
2723	static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
2724	enum bpf_dynptr_type type, u32 offset, u32 size)
2725	{
2726	}
2727
2728	static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
2729	{
2730	}
2731
2732	static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
2733	{
2734	}
2735	#endif /* CONFIG_BPF_SYSCALL */
2736
2737	static __always_inline int
2738	bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr)
2739	{
2740	int ret = -EFAULT;
2741
2742	if (IS_ENABLED(CONFIG_BPF_EVENTS))
2743	ret = copy_from_kernel_nofault(dst, src: unsafe_ptr, size);
2744	if (unlikely(ret < 0))
2745	memset(dst, 0, size);
2746	return ret;
2747	}
2748
2749	void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
2750	struct btf_mod_pair *used_btfs, u32 len);
2751
2752	static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
2753	enum bpf_prog_type type)
2754	{
2755	return bpf_prog_get_type_dev(ufd, type, attach_drv: false);
2756	}
2757
2758	void __bpf_free_used_maps(struct bpf_prog_aux *aux,
2759	struct bpf_map **used_maps, u32 len);
2760
2761	bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool);
2762
2763	int bpf_prog_offload_compile(struct bpf_prog *prog);
2764	void bpf_prog_dev_bound_destroy(struct bpf_prog *prog);
2765	int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
2766	struct bpf_prog *prog);
2767
2768	int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map);
2769
2770	int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value);
2771	int bpf_map_offload_update_elem(struct bpf_map *map,
2772	void *key, void *value, u64 flags);
2773	int bpf_map_offload_delete_elem(struct bpf_map *map, void *key);
2774	int bpf_map_offload_get_next_key(struct bpf_map *map,
2775	void *key, void *next_key);
2776
2777	bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map);
2778
2779	struct bpf_offload_dev *
2780	bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv);
2781	void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);
2782	void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev);
2783	int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
2784	struct net_device *netdev);
2785	void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
2786	struct net_device *netdev);
2787	bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);
2788
2789	void unpriv_ebpf_notify(int new_state);
2790
2791	#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
2792	int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
2793	struct bpf_prog_aux *prog_aux);
2794	void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id);
2795	int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr);
2796	int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog);
2797	void bpf_dev_bound_netdev_unregister(struct net_device *dev);
2798
2799	static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
2800	{
2801	return aux->dev_bound;
2802	}
2803
2804	static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux *aux)
2805	{
2806	return aux->offload_requested;
2807	}
2808
2809	bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs);
2810
2811	static inline bool bpf_map_is_offloaded(struct bpf_map *map)
2812	{
2813	return unlikely(map->ops == &bpf_map_offload_ops);
2814	}
2815
2816	struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr);
2817	void bpf_map_offload_map_free(struct bpf_map *map);
2818	u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map);
2819	int bpf_prog_test_run_syscall(struct bpf_prog *prog,
2820	const union bpf_attr *kattr,
2821	union bpf_attr __user *uattr);
2822
2823	int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog);
2824	int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype);
2825	int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags);
2826	int sock_map_bpf_prog_query(const union bpf_attr *attr,
2827	union bpf_attr __user *uattr);
2828
2829	void sock_map_unhash(struct sock *sk);
2830	void sock_map_destroy(struct sock *sk);
2831	void sock_map_close(struct sock *sk, long timeout);
2832	#else
2833	static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
2834	struct bpf_prog_aux *prog_aux)
2835	{
2836	return -EOPNOTSUPP;
2837	}
2838
2839	static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog,
2840	u32 func_id)
2841	{
2842	return NULL;
2843	}
2844
2845	static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog,
2846	union bpf_attr *attr)
2847	{
2848	return -EOPNOTSUPP;
2849	}
2850
2851	static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog,
2852	struct bpf_prog *old_prog)
2853	{
2854	return -EOPNOTSUPP;
2855	}
2856
2857	static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev)
2858	{
2859	}
2860
2861	static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
2862	{
2863	return false;
2864	}
2865
2866	static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux *aux)
2867	{
2868	return false;
2869	}
2870
2871	static inline bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs)
2872	{
2873	return false;
2874	}
2875
2876	static inline bool bpf_map_is_offloaded(struct bpf_map *map)
2877	{
2878	return false;
2879	}
2880
2881	static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
2882	{
2883	return ERR_PTR(-EOPNOTSUPP);
2884	}
2885
2886	static inline void bpf_map_offload_map_free(struct bpf_map *map)
2887	{
2888	}
2889
2890	static inline u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map)
2891	{
2892	return 0;
2893	}
2894
2895	static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog,
2896	const union bpf_attr *kattr,
2897	union bpf_attr __user *uattr)
2898	{
2899	return -ENOTSUPP;
2900	}
2901
2902	#ifdef CONFIG_BPF_SYSCALL
2903	static inline int sock_map_get_from_fd(const union bpf_attr *attr,
2904	struct bpf_prog *prog)
2905	{
2906	return -EINVAL;
2907	}
2908
2909	static inline int sock_map_prog_detach(const union bpf_attr *attr,
2910	enum bpf_prog_type ptype)
2911	{
2912	return -EOPNOTSUPP;
2913	}
2914
2915	static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value,
2916	u64 flags)
2917	{
2918	return -EOPNOTSUPP;
2919	}
2920
2921	static inline int sock_map_bpf_prog_query(const union bpf_attr *attr,
2922	union bpf_attr __user *uattr)
2923	{
2924	return -EINVAL;
2925	}
2926	#endif /* CONFIG_BPF_SYSCALL */
2927	#endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */
2928
2929	static __always_inline void
2930	bpf_prog_inc_misses_counters(const struct bpf_prog_array *array)
2931	{
2932	const struct bpf_prog_array_item *item;
2933	struct bpf_prog *prog;
2934
2935	if (unlikely(!array))
2936	return;
2937
2938	item = &array->items[0];
2939	while ((prog = READ_ONCE(item->prog))) {
2940	bpf_prog_inc_misses_counter(prog);
2941	item++;
2942	}
2943	}
2944
2945	#if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
2946	void bpf_sk_reuseport_detach(struct sock *sk);
2947	int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,
2948	void *value);
2949	int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key,
2950	void *value, u64 map_flags);
2951	#else
2952	static inline void bpf_sk_reuseport_detach(struct sock *sk)
2953	{
2954	}
2955
2956	#ifdef CONFIG_BPF_SYSCALL
2957	static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map,
2958	void *key, void *value)
2959	{
2960	return -EOPNOTSUPP;
2961	}
2962
2963	static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
2964	void *key, void *value,
2965	u64 map_flags)
2966	{
2967	return -EOPNOTSUPP;
2968	}
2969	#endif /* CONFIG_BPF_SYSCALL */
2970	#endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */
2971
2972	/* verifier prototypes for helper functions called from eBPF programs */
2973	extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
2974	extern const struct bpf_func_proto bpf_map_update_elem_proto;
2975	extern const struct bpf_func_proto bpf_map_delete_elem_proto;
2976	extern const struct bpf_func_proto bpf_map_push_elem_proto;
2977	extern const struct bpf_func_proto bpf_map_pop_elem_proto;
2978	extern const struct bpf_func_proto bpf_map_peek_elem_proto;
2979	extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto;
2980
2981	extern const struct bpf_func_proto bpf_get_prandom_u32_proto;
2982	extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;
2983	extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
2984	extern const struct bpf_func_proto bpf_tail_call_proto;
2985	extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
2986	extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto;
2987	extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto;
2988	extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
2989	extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
2990	extern const struct bpf_func_proto bpf_get_current_comm_proto;
2991	extern const struct bpf_func_proto bpf_get_stackid_proto;
2992	extern const struct bpf_func_proto bpf_get_stack_proto;
2993	extern const struct bpf_func_proto bpf_get_task_stack_proto;
2994	extern const struct bpf_func_proto bpf_get_stackid_proto_pe;
2995	extern const struct bpf_func_proto bpf_get_stack_proto_pe;
2996	extern const struct bpf_func_proto bpf_sock_map_update_proto;
2997	extern const struct bpf_func_proto bpf_sock_hash_update_proto;
2998	extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
2999	extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto;
3000	extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto;
3001	extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
3002	extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
3003	extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;
3004	extern const struct bpf_func_proto bpf_sk_redirect_map_proto;
3005	extern const struct bpf_func_proto bpf_spin_lock_proto;
3006	extern const struct bpf_func_proto bpf_spin_unlock_proto;
3007	extern const struct bpf_func_proto bpf_get_local_storage_proto;
3008	extern const struct bpf_func_proto bpf_strtol_proto;
3009	extern const struct bpf_func_proto bpf_strtoul_proto;
3010	extern const struct bpf_func_proto bpf_tcp_sock_proto;
3011	extern const struct bpf_func_proto bpf_jiffies64_proto;
3012	extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto;
3013	extern const struct bpf_func_proto bpf_event_output_data_proto;
3014	extern const struct bpf_func_proto bpf_ringbuf_output_proto;
3015	extern const struct bpf_func_proto bpf_ringbuf_reserve_proto;
3016	extern const struct bpf_func_proto bpf_ringbuf_submit_proto;
3017	extern const struct bpf_func_proto bpf_ringbuf_discard_proto;
3018	extern const struct bpf_func_proto bpf_ringbuf_query_proto;
3019	extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto;
3020	extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto;
3021	extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto;
3022	extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto;
3023	extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto;
3024	extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto;
3025	extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto;
3026	extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;
3027	extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto;
3028	extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto;
3029	extern const struct bpf_func_proto bpf_copy_from_user_proto;
3030	extern const struct bpf_func_proto bpf_snprintf_btf_proto;
3031	extern const struct bpf_func_proto bpf_snprintf_proto;
3032	extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;
3033	extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
3034	extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;
3035	extern const struct bpf_func_proto bpf_sock_from_file_proto;
3036	extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;
3037	extern const struct bpf_func_proto bpf_task_storage_get_recur_proto;
3038	extern const struct bpf_func_proto bpf_task_storage_get_proto;
3039	extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto;
3040	extern const struct bpf_func_proto bpf_task_storage_delete_proto;
3041	extern const struct bpf_func_proto bpf_for_each_map_elem_proto;
3042	extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;
3043	extern const struct bpf_func_proto bpf_sk_setsockopt_proto;
3044	extern const struct bpf_func_proto bpf_sk_getsockopt_proto;
3045	extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto;
3046	extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto;
3047	extern const struct bpf_func_proto bpf_find_vma_proto;
3048	extern const struct bpf_func_proto bpf_loop_proto;
3049	extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
3050	extern const struct bpf_func_proto bpf_set_retval_proto;
3051	extern const struct bpf_func_proto bpf_get_retval_proto;
3052	extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto;
3053	extern const struct bpf_func_proto bpf_cgrp_storage_get_proto;
3054	extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto;
3055
3056	const struct bpf_func_proto *tracing_prog_func_proto(
3057	enum bpf_func_id func_id, const struct bpf_prog *prog);
3058
3059	/* Shared helpers among cBPF and eBPF. */
3060	void bpf_user_rnd_init_once(void);
3061	u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
3062	u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
3063
3064	#if defined(CONFIG_NET)
3065	bool bpf_sock_common_is_valid_access(int off, int size,
3066	enum bpf_access_type type,
3067	struct bpf_insn_access_aux *info);
3068	bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3069	struct bpf_insn_access_aux *info);
3070	u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
3071	const struct bpf_insn *si,
3072	struct bpf_insn *insn_buf,
3073	struct bpf_prog *prog,
3074	u32 *target_size);
3075	int bpf_dynptr_from_skb_rdonly(struct sk_buff *skb, u64 flags,
3076	struct bpf_dynptr_kern *ptr);
3077	#else
3078	static inline bool bpf_sock_common_is_valid_access(int off, int size,
3079	enum bpf_access_type type,
3080	struct bpf_insn_access_aux *info)
3081	{
3082	return false;
3083	}
3084	static inline bool bpf_sock_is_valid_access(int off, int size,
3085	enum bpf_access_type type,
3086	struct bpf_insn_access_aux *info)
3087	{
3088	return false;
3089	}
3090	static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
3091	const struct bpf_insn *si,
3092	struct bpf_insn *insn_buf,
3093	struct bpf_prog *prog,
3094	u32 *target_size)
3095	{
3096	return 0;
3097	}
3098	static inline int bpf_dynptr_from_skb_rdonly(struct sk_buff *skb, u64 flags,
3099	struct bpf_dynptr_kern *ptr)
3100	{
3101	return -EOPNOTSUPP;
3102	}
3103	#endif
3104
3105	#ifdef CONFIG_INET
3106	struct sk_reuseport_kern {
3107	struct sk_buff *skb;
3108	struct sock *sk;
3109	struct sock *selected_sk;
3110	struct sock *migrating_sk;
3111	void *data_end;
3112	u32 hash;
3113	u32 reuseport_id;
3114	bool bind_inany;
3115	};
3116	bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3117	struct bpf_insn_access_aux *info);
3118
3119	u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
3120	const struct bpf_insn *si,
3121	struct bpf_insn *insn_buf,
3122	struct bpf_prog *prog,
3123	u32 *target_size);
3124
3125	bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3126	struct bpf_insn_access_aux *info);
3127
3128	u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
3129	const struct bpf_insn *si,
3130	struct bpf_insn *insn_buf,
3131	struct bpf_prog *prog,
3132	u32 *target_size);
3133	#else
3134	static inline bool bpf_tcp_sock_is_valid_access(int off, int size,
3135	enum bpf_access_type type,
3136	struct bpf_insn_access_aux *info)
3137	{
3138	return false;
3139	}
3140
3141	static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
3142	const struct bpf_insn *si,
3143	struct bpf_insn *insn_buf,
3144	struct bpf_prog *prog,
3145	u32 *target_size)
3146	{
3147	return 0;
3148	}
3149	static inline bool bpf_xdp_sock_is_valid_access(int off, int size,
3150	enum bpf_access_type type,
3151	struct bpf_insn_access_aux *info)
3152	{
3153	return false;
3154	}
3155
3156	static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
3157	const struct bpf_insn *si,
3158	struct bpf_insn *insn_buf,
3159	struct bpf_prog *prog,
3160	u32 *target_size)
3161	{
3162	return 0;
3163	}
3164	#endif /* CONFIG_INET */
3165
3166	enum bpf_text_poke_type {
3167	BPF_MOD_CALL,
3168	BPF_MOD_JUMP,
3169	};
3170
3171	int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
3172	void *addr1, void *addr2);
3173
3174	void *bpf_arch_text_copy(void *dst, void *src, size_t len);
3175	int bpf_arch_text_invalidate(void *dst, size_t len);
3176
3177	struct btf_id_set;
3178	bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
3179
3180	#define MAX_BPRINTF_VARARGS 12
3181	#define MAX_BPRINTF_BUF 1024
3182
3183	struct bpf_bprintf_data {
3184	u32 *bin_args;
3185	char *buf;
3186	bool get_bin_args;
3187	bool get_buf;
3188	};
3189
3190	int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
3191	u32 num_args, struct bpf_bprintf_data *data);
3192	void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);
3193
3194	#ifdef CONFIG_BPF_LSM
3195	void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);
3196	void bpf_cgroup_atype_put(int cgroup_atype);
3197	#else
3198	static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {}
3199	static inline void bpf_cgroup_atype_put(int cgroup_atype) {}
3200	#endif /* CONFIG_BPF_LSM */
3201
3202	struct key;
3203
3204	#ifdef CONFIG_KEYS
3205	struct bpf_key {
3206	struct key *key;
3207	bool has_ref;
3208	};
3209	#endif /* CONFIG_KEYS */
3210
3211	static inline bool type_is_alloc(u32 type)
3212	{
3213	return type & MEM_ALLOC;
3214	}
3215
3216	static inline gfp_t bpf_memcg_flags(gfp_t flags)
3217	{
3218	if (memcg_bpf_enabled())
3219	return flags | __GFP_ACCOUNT;
3220	return flags;
3221	}
3222
3223	static inline bool bpf_is_subprog(const struct bpf_prog *prog)
3224	{
3225	return prog->aux->func_idx != 0;
3226	}
3227
3228	#endif /* _LINUX_BPF_H */
3229