1	// SPDX-License-Identifier: GPL-2.0-only
2	/* Copyright (c) 2017 Facebook
3	*/
4	#include <linux/bpf.h>
5	#include <linux/btf.h>
6	#include <linux/btf_ids.h>
7	#include <linux/slab.h>
8	#include <linux/init.h>
9	#include <linux/vmalloc.h>
10	#include <linux/etherdevice.h>
11	#include <linux/filter.h>
12	#include <linux/rcupdate_trace.h>
13	#include <linux/sched/signal.h>
14	#include <net/bpf_sk_storage.h>
15	#include <net/sock.h>
16	#include <net/tcp.h>
17	#include <net/net_namespace.h>
18	#include <net/page_pool/helpers.h>
19	#include <linux/error-injection.h>
20	#include <linux/smp.h>
21	#include <linux/sock_diag.h>
22	#include <linux/netfilter.h>
23	#include <net/netdev_rx_queue.h>
24	#include <net/xdp.h>
25	#include <net/netfilter/nf_bpf_link.h>
26
27	#define CREATE_TRACE_POINTS
28	#include <trace/events/bpf_test_run.h>
29
30	struct bpf_test_timer {
31	enum { NO_PREEMPT, NO_MIGRATE } mode;
32	u32 i;
33	u64 time_start, time_spent;
34	};
35
36	static void bpf_test_timer_enter(struct bpf_test_timer *t)
37	__acquires(rcu)
38	{
39	rcu_read_lock();
40	if (t->mode == NO_PREEMPT)
41	preempt_disable();
42	else
43	migrate_disable();
44
45	t->time_start = ktime_get_ns();
46	}
47
48	static void bpf_test_timer_leave(struct bpf_test_timer *t)
49	__releases(rcu)
50	{
51	t->time_start = 0;
52
53	if (t->mode == NO_PREEMPT)
54	preempt_enable();
55	else
56	migrate_enable();
57	rcu_read_unlock();
58	}
59
60	static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations,
61	u32 repeat, int *err, u32 *duration)
62	__must_hold(rcu)
63	{
64	t->i += iterations;
65	if (t->i >= repeat) {
66	/* We're done. */
67	t->time_spent += ktime_get_ns() - t->time_start;
68	do_div(t->time_spent, t->i);
69	*duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent;
70	*err = 0;
71	goto reset;
72	}
73
74	if (signal_pending(current)) {
75	/* During iteration: we've been cancelled, abort. */
76	*err = -EINTR;
77	goto reset;
78	}
79
80	if (need_resched()) {
81	/* During iteration: we need to reschedule between runs. */
82	t->time_spent += ktime_get_ns() - t->time_start;
83	bpf_test_timer_leave(t);
84	cond_resched();
85	bpf_test_timer_enter(t);
86	}
87
88	/* Do another round. */
89	return true;
90
91	reset:
92	t->i = 0;
93	return false;
94	}
95
96	/* We put this struct at the head of each page with a context and frame
97	* initialised when the page is allocated, so we don't have to do this on each
98	* repetition of the test run.
99	*/
100	struct xdp_page_head {
101	struct xdp_buff orig_ctx;
102	struct xdp_buff ctx;
103	union {
104	/* ::data_hard_start starts here */
105	DECLARE_FLEX_ARRAY(struct xdp_frame, frame);
106	DECLARE_FLEX_ARRAY(u8, data);
107	};
108	};
109
110	struct xdp_test_data {
111	struct xdp_buff *orig_ctx;
112	struct xdp_rxq_info rxq;
113	struct net_device *dev;
114	struct page_pool *pp;
115	struct xdp_frame **frames;
116	struct sk_buff **skbs;
117	struct xdp_mem_info mem;
118	u32 batch_size;
119	u32 frame_cnt;
120	};
121
122	/* tools/testing/selftests/bpf/prog_tests/xdp_do_redirect.c:%MAX_PKT_SIZE
123	* must be updated accordingly this gets changed, otherwise BPF selftests
124	* will fail.
125	*/
126	#define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))
127	#define TEST_XDP_MAX_BATCH 256
128
129	static void xdp_test_run_init_page(struct page *page, void *arg)
130	{
131	struct xdp_page_head *head = phys_to_virt(page_to_phys(page));
132	struct xdp_buff *new_ctx, *orig_ctx;
133	u32 headroom = XDP_PACKET_HEADROOM;
134	struct xdp_test_data *xdp = arg;
135	size_t frm_len, meta_len;
136	struct xdp_frame *frm;
137	void *data;
138
139	orig_ctx = xdp->orig_ctx;
140	frm_len = orig_ctx->data_end - orig_ctx->data_meta;
141	meta_len = orig_ctx->data - orig_ctx->data_meta;
142	headroom -= meta_len;
143
144	new_ctx = &head->ctx;
145	frm = head->frame;
146	data = head->data;
147	memcpy(data + headroom, orig_ctx->data_meta, frm_len);
148
149	xdp_init_buff(xdp: new_ctx, TEST_XDP_FRAME_SIZE, rxq: &xdp->rxq);
150	xdp_prepare_buff(xdp: new_ctx, hard_start: data, headroom, data_len: frm_len, meta_valid: true);
151	new_ctx->data = new_ctx->data_meta + meta_len;
152
153	xdp_update_frame_from_buff(xdp: new_ctx, xdp_frame: frm);
154	frm->mem = new_ctx->rxq->mem;
155
156	memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx));
157	}
158
159	static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx)
160	{
161	struct page_pool *pp;
162	int err = -ENOMEM;
163	struct page_pool_params pp_params = {
164	.order = 0,
165	.flags = 0,
166	.pool_size = xdp->batch_size,
167	.nid = NUMA_NO_NODE,
168	.init_callback = xdp_test_run_init_page,
169	.init_arg = xdp,
170	};
171
172	xdp->frames = kvmalloc_array(n: xdp->batch_size, size: sizeof(void *), GFP_KERNEL);
173	if (!xdp->frames)
174	return -ENOMEM;
175
176	xdp->skbs = kvmalloc_array(n: xdp->batch_size, size: sizeof(void *), GFP_KERNEL);
177	if (!xdp->skbs)
178	goto err_skbs;
179
180	pp = page_pool_create(params: &pp_params);
181	if (IS_ERR(ptr: pp)) {
182	err = PTR_ERR(ptr: pp);
183	goto err_pp;
184	}
185
186	/* will copy 'mem.id' into pp->xdp_mem_id */
187	err = xdp_reg_mem_model(mem: &xdp->mem, type: MEM_TYPE_PAGE_POOL, allocator: pp);
188	if (err)
189	goto err_mmodel;
190
191	xdp->pp = pp;
192
193	/* We create a 'fake' RXQ referencing the original dev, but with an
194	* xdp_mem_info pointing to our page_pool
195	*/
196	xdp_rxq_info_reg(xdp_rxq: &xdp->rxq, dev: orig_ctx->rxq->dev, queue_index: 0, napi_id: 0);
197	xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL;
198	xdp->rxq.mem.id = pp->xdp_mem_id;
199	xdp->dev = orig_ctx->rxq->dev;
200	xdp->orig_ctx = orig_ctx;
201
202	return 0;
203
204	err_mmodel:
205	page_pool_destroy(pool: pp);
206	err_pp:
207	kvfree(addr: xdp->skbs);
208	err_skbs:
209	kvfree(addr: xdp->frames);
210	return err;
211	}
212
213	static void xdp_test_run_teardown(struct xdp_test_data *xdp)
214	{
215	xdp_unreg_mem_model(mem: &xdp->mem);
216	page_pool_destroy(pool: xdp->pp);
217	kfree(objp: xdp->frames);
218	kfree(objp: xdp->skbs);
219	}
220
221	static bool frame_was_changed(const struct xdp_page_head *head)
222	{
223	/* xdp_scrub_frame() zeroes the data pointer, flags is the last field,
224	* i.e. has the highest chances to be overwritten. If those two are
225	* untouched, it's most likely safe to skip the context reset.
226	*/
227	return head->frame->data != head->orig_ctx.data ||
228	head->frame->flags != head->orig_ctx.flags;
229	}
230
231	static bool ctx_was_changed(struct xdp_page_head *head)
232	{
233	return head->orig_ctx.data != head->ctx.data ||
234	head->orig_ctx.data_meta != head->ctx.data_meta ||
235	head->orig_ctx.data_end != head->ctx.data_end;
236	}
237
238	static void reset_ctx(struct xdp_page_head *head)
239	{
240	if (likely(!frame_was_changed(head) && !ctx_was_changed(head)))
241	return;
242
243	head->ctx.data = head->orig_ctx.data;
244	head->ctx.data_meta = head->orig_ctx.data_meta;
245	head->ctx.data_end = head->orig_ctx.data_end;
246	xdp_update_frame_from_buff(xdp: &head->ctx, xdp_frame: head->frame);
247	}
248
249	static int xdp_recv_frames(struct xdp_frame **frames, int nframes,
250	struct sk_buff **skbs,
251	struct net_device *dev)
252	{
253	gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
254	int i, n;
255	LIST_HEAD(list);
256
257	n = kmem_cache_alloc_bulk(s: skbuff_cache, flags: gfp, size: nframes, p: (void **)skbs);
258	if (unlikely(n == 0)) {
259	for (i = 0; i < nframes; i++)
260	xdp_return_frame(xdpf: frames[i]);
261	return -ENOMEM;
262	}
263
264	for (i = 0; i < nframes; i++) {
265	struct xdp_frame *xdpf = frames[i];
266	struct sk_buff *skb = skbs[i];
267
268	skb = __xdp_build_skb_from_frame(xdpf, skb, dev);
269	if (!skb) {
270	xdp_return_frame(xdpf);
271	continue;
272	}
273
274	list_add_tail(new: &skb->list, head: &list);
275	}
276	netif_receive_skb_list(head: &list);
277
278	return 0;
279	}
280
281	static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog,
282	u32 repeat)
283	{
284	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
285	int err = 0, act, ret, i, nframes = 0, batch_sz;
286	struct xdp_frame **frames = xdp->frames;
287	struct xdp_page_head *head;
288	struct xdp_frame *frm;
289	bool redirect = false;
290	struct xdp_buff *ctx;
291	struct page *page;
292
293	batch_sz = min_t(u32, repeat, xdp->batch_size);
294
295	local_bh_disable();
296	xdp_set_return_frame_no_direct();
297
298	for (i = 0; i < batch_sz; i++) {
299	page = page_pool_dev_alloc_pages(pool: xdp->pp);
300	if (!page) {
301	err = -ENOMEM;
302	goto out;
303	}
304
305	head = phys_to_virt(page_to_phys(page));
306	reset_ctx(head);
307	ctx = &head->ctx;
308	frm = head->frame;
309	xdp->frame_cnt++;
310
311	act = bpf_prog_run_xdp(prog, xdp: ctx);
312
313	/* if program changed pkt bounds we need to update the xdp_frame */
314	if (unlikely(ctx_was_changed(head))) {
315	ret = xdp_update_frame_from_buff(xdp: ctx, xdp_frame: frm);
316	if (ret) {
317	xdp_return_buff(xdp: ctx);
318	continue;
319	}
320	}
321
322	switch (act) {
323	case XDP_TX:
324	/* we can't do a real XDP_TX since we're not in the
325	* driver, so turn it into a REDIRECT back to the same
326	* index
327	*/
328	ri->tgt_index = xdp->dev->ifindex;
329	ri->map_id = INT_MAX;
330	ri->map_type = BPF_MAP_TYPE_UNSPEC;
331	fallthrough;
332	case XDP_REDIRECT:
333	redirect = true;
334	ret = xdp_do_redirect_frame(dev: xdp->dev, xdp: ctx, xdpf: frm, prog);
335	if (ret)
336	xdp_return_buff(xdp: ctx);
337	break;
338	case XDP_PASS:
339	frames[nframes++] = frm;
340	break;
341	default:
342	bpf_warn_invalid_xdp_action(NULL, prog, act);
343	fallthrough;
344	case XDP_DROP:
345	xdp_return_buff(xdp: ctx);
346	break;
347	}
348	}
349
350	out:
351	if (redirect)
352	xdp_do_flush();
353	if (nframes) {
354	ret = xdp_recv_frames(frames, nframes, skbs: xdp->skbs, dev: xdp->dev);
355	if (ret)
356	err = ret;
357	}
358
359	xdp_clear_return_frame_no_direct();
360	local_bh_enable();
361	return err;
362	}
363
364	static int bpf_test_run_xdp_live(struct bpf_prog *prog, struct xdp_buff *ctx,
365	u32 repeat, u32 batch_size, u32 *time)
366
367	{
368	struct xdp_test_data xdp = { .batch_size = batch_size };
369	struct bpf_test_timer t = { .mode = NO_MIGRATE };
370	int ret;
371
372	if (!repeat)
373	repeat = 1;
374
375	ret = xdp_test_run_setup(xdp: &xdp, orig_ctx: ctx);
376	if (ret)
377	return ret;
378
379	bpf_test_timer_enter(t: &t);
380	do {
381	xdp.frame_cnt = 0;
382	ret = xdp_test_run_batch(xdp: &xdp, prog, repeat: repeat - t.i);
383	if (unlikely(ret < 0))
384	break;
385	} while (bpf_test_timer_continue(t: &t, iterations: xdp.frame_cnt, repeat, err: &ret, duration: time));
386	bpf_test_timer_leave(t: &t);
387
388	xdp_test_run_teardown(xdp: &xdp);
389	return ret;
390	}
391
392	static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
393	u32 *retval, u32 *time, bool xdp)
394	{
395	struct bpf_prog_array_item item = {.prog = prog};
396	struct bpf_run_ctx *old_ctx;
397	struct bpf_cg_run_ctx run_ctx;
398	struct bpf_test_timer t = { NO_MIGRATE };
399	enum bpf_cgroup_storage_type stype;
400	int ret;
401
402	for_each_cgroup_storage_type(stype) {
403	item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
404	if (IS_ERR(ptr: item.cgroup_storage[stype])) {
405	item.cgroup_storage[stype] = NULL;
406	for_each_cgroup_storage_type(stype)
407	bpf_cgroup_storage_free(storage: item.cgroup_storage[stype]);
408	return -ENOMEM;
409	}
410	}
411
412	if (!repeat)
413	repeat = 1;
414
415	bpf_test_timer_enter(t: &t);
416	old_ctx = bpf_set_run_ctx(new_ctx: &run_ctx.run_ctx);
417	do {
418	run_ctx.prog_item = &item;
419	local_bh_disable();
420	if (xdp)
421	*retval = bpf_prog_run_xdp(prog, xdp: ctx);
422	else
423	*retval = bpf_prog_run(prog, ctx);
424	local_bh_enable();
425	} while (bpf_test_timer_continue(t: &t, iterations: 1, repeat, err: &ret, duration: time));
426	bpf_reset_run_ctx(old_ctx);
427	bpf_test_timer_leave(t: &t);
428
429	for_each_cgroup_storage_type(stype)
430	bpf_cgroup_storage_free(storage: item.cgroup_storage[stype]);
431
432	return ret;
433	}
434
435	static int bpf_test_finish(const union bpf_attr *kattr,
436	union bpf_attr __user *uattr, const void *data,
437	struct skb_shared_info *sinfo, u32 size,
438	u32 retval, u32 duration)
439	{
440	void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
441	int err = -EFAULT;
442	u32 copy_size = size;
443
444	/* Clamp copy if the user has provided a size hint, but copy the full
445	* buffer if not to retain old behaviour.
446	*/
447	if (kattr->test.data_size_out &&
448	copy_size > kattr->test.data_size_out) {
449	copy_size = kattr->test.data_size_out;
450	err = -ENOSPC;
451	}
452
453	if (data_out) {
454	int len = sinfo ? copy_size - sinfo->xdp_frags_size : copy_size;
455
456	if (len < 0) {
457	err = -ENOSPC;
458	goto out;
459	}
460
461	if (copy_to_user(to: data_out, from: data, n: len))
462	goto out;
463
464	if (sinfo) {
465	int i, offset = len;
466	u32 data_len;
467
468	for (i = 0; i < sinfo->nr_frags; i++) {
469	skb_frag_t *frag = &sinfo->frags[i];
470
471	if (offset >= copy_size) {
472	err = -ENOSPC;
473	break;
474	}
475
476	data_len = min_t(u32, copy_size - offset,
477	skb_frag_size(frag));
478
479	if (copy_to_user(to: data_out + offset,
480	from: skb_frag_address(frag),
481	n: data_len))
482	goto out;
483
484	offset += data_len;
485	}
486	}
487	}
488
489	if (copy_to_user(to: &uattr->test.data_size_out, from: &size, n: sizeof(size)))
490	goto out;
491	if (copy_to_user(to: &uattr->test.retval, from: &retval, n: sizeof(retval)))
492	goto out;
493	if (copy_to_user(to: &uattr->test.duration, from: &duration, n: sizeof(duration)))
494	goto out;
495	if (err != -ENOSPC)
496	err = 0;
497	out:
498	trace_bpf_test_finish(err: &err);
499	return err;
500	}
501
502	/* Integer types of various sizes and pointer combinations cover variety of
503	* architecture dependent calling conventions. 7+ can be supported in the
504	* future.
505	*/
506	__diag_push();
507	__diag_ignore_all("-Wmissing-prototypes",
508	"Global functions as their definitions will be in vmlinux BTF");
509	__bpf_kfunc int bpf_fentry_test1(int a)
510	{
511	return a + 1;
512	}
513	EXPORT_SYMBOL_GPL(bpf_fentry_test1);
514
515	int noinline bpf_fentry_test2(int a, u64 b)
516	{
517	return a + b;
518	}
519
520	int noinline bpf_fentry_test3(char a, int b, u64 c)
521	{
522	return a + b + c;
523	}
524
525	int noinline bpf_fentry_test4(void *a, char b, int c, u64 d)
526	{
527	return (long)a + b + c + d;
528	}
529
530	int noinline bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e)
531	{
532	return a + (long)b + c + d + e;
533	}
534
535	int noinline bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f)
536	{
537	return a + (long)b + c + d + (long)e + f;
538	}
539
540	struct bpf_fentry_test_t {
541	struct bpf_fentry_test_t *a;
542	};
543
544	int noinline bpf_fentry_test7(struct bpf_fentry_test_t *arg)
545	{
546	asm volatile ("");
547	return (long)arg;
548	}
549
550	int noinline bpf_fentry_test8(struct bpf_fentry_test_t *arg)
551	{
552	return (long)arg->a;
553	}
554
555	__bpf_kfunc u32 bpf_fentry_test9(u32 *a)
556	{
557	return *a;
558	}
559
560	void noinline bpf_fentry_test_sinfo(struct skb_shared_info *sinfo)
561	{
562	}
563
564	__bpf_kfunc int bpf_modify_return_test(int a, int *b)
565	{
566	*b += 1;
567	return a + *b;
568	}
569
570	__bpf_kfunc int bpf_modify_return_test2(int a, int *b, short c, int d,
571	void *e, char f, int g)
572	{
573	*b += 1;
574	return a + *b + c + d + (long)e + f + g;
575	}
576
577	int noinline bpf_fentry_shadow_test(int a)
578	{
579	return a + 1;
580	}
581
582	struct prog_test_member1 {
583	int a;
584	};
585
586	struct prog_test_member {
587	struct prog_test_member1 m;
588	int c;
589	};
590
591	struct prog_test_ref_kfunc {
592	int a;
593	int b;
594	struct prog_test_member memb;
595	struct prog_test_ref_kfunc *next;
596	refcount_t cnt;
597	};
598
599	__bpf_kfunc void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p)
600	{
601	refcount_dec(r: &p->cnt);
602	}
603
604	__bpf_kfunc void bpf_kfunc_call_memb_release(struct prog_test_member *p)
605	{
606	}
607
608	__diag_pop();
609
610	BTF_SET8_START(bpf_test_modify_return_ids)
611	BTF_ID_FLAGS(func, bpf_modify_return_test)
612	BTF_ID_FLAGS(func, bpf_modify_return_test2)
613	BTF_ID_FLAGS(func, bpf_fentry_test1, KF_SLEEPABLE)
614	BTF_SET8_END(bpf_test_modify_return_ids)
615
616	static const struct btf_kfunc_id_set bpf_test_modify_return_set = {
617	.owner = THIS_MODULE,
618	.set = &bpf_test_modify_return_ids,
619	};
620
621	BTF_SET8_START(test_sk_check_kfunc_ids)
622	BTF_ID_FLAGS(func, bpf_kfunc_call_test_release, KF_RELEASE)
623	BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE)
624	BTF_SET8_END(test_sk_check_kfunc_ids)
625
626	static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size,
627	u32 size, u32 headroom, u32 tailroom)
628	{
629	void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
630	void *data;
631
632	if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom)
633	return ERR_PTR(error: -EINVAL);
634
635	if (user_size > size)
636	return ERR_PTR(error: -EMSGSIZE);
637
638	size = SKB_DATA_ALIGN(size);
639	data = kzalloc(size: size + headroom + tailroom, GFP_USER);
640	if (!data)
641	return ERR_PTR(error: -ENOMEM);
642
643	if (copy_from_user(to: data + headroom, from: data_in, n: user_size)) {
644	kfree(objp: data);
645	return ERR_PTR(error: -EFAULT);
646	}
647
648	return data;
649	}
650
651	int bpf_prog_test_run_tracing(struct bpf_prog *prog,
652	const union bpf_attr *kattr,
653	union bpf_attr __user *uattr)
654	{
655	struct bpf_fentry_test_t arg = {};
656	u16 side_effect = 0, ret = 0;
657	int b = 2, err = -EFAULT;
658	u32 retval = 0;
659
660	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
661	return -EINVAL;
662
663	switch (prog->expected_attach_type) {
664	case BPF_TRACE_FENTRY:
665	case BPF_TRACE_FEXIT:
666	if (bpf_fentry_test1(1) != 2 ||
667	bpf_fentry_test2(a: 2, b: 3) != 5 ||
668	bpf_fentry_test3(a: 4, b: 5, c: 6) != 15 ||
669	bpf_fentry_test4(a: (void *)7, b: 8, c: 9, d: 10) != 34 ||
670	bpf_fentry_test5(a: 11, b: (void *)12, c: 13, d: 14, e: 15) != 65 ||
671	bpf_fentry_test6(a: 16, b: (void *)17, c: 18, d: 19, e: (void *)20, f: 21) != 111 ||
672	bpf_fentry_test7(arg: (struct bpf_fentry_test_t *)0) != 0 ||
673	bpf_fentry_test8(arg: &arg) != 0 ||
674	bpf_fentry_test9(a: &retval) != 0)
675	goto out;
676	break;
677	case BPF_MODIFY_RETURN:
678	ret = bpf_modify_return_test(a: 1, b: &b);
679	if (b != 2)
680	side_effect++;
681	b = 2;
682	ret += bpf_modify_return_test2(a: 1, b: &b, c: 3, d: 4, e: (void *)5, f: 6, g: 7);
683	if (b != 2)
684	side_effect++;
685	break;
686	default:
687	goto out;
688	}
689
690	retval = ((u32)side_effect << 16) | ret;
691	if (copy_to_user(to: &uattr->test.retval, from: &retval, n: sizeof(retval)))
692	goto out;
693
694	err = 0;
695	out:
696	trace_bpf_test_finish(err: &err);
697	return err;
698	}
699
700	struct bpf_raw_tp_test_run_info {
701	struct bpf_prog *prog;
702	void *ctx;
703	u32 retval;
704	};
705
706	static void
707	__bpf_prog_test_run_raw_tp(void *data)
708	{
709	struct bpf_raw_tp_test_run_info *info = data;
710
711	rcu_read_lock();
712	info->retval = bpf_prog_run(prog: info->prog, ctx: info->ctx);
713	rcu_read_unlock();
714	}
715
716	int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
717	const union bpf_attr *kattr,
718	union bpf_attr __user *uattr)
719	{
720	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
721	__u32 ctx_size_in = kattr->test.ctx_size_in;
722	struct bpf_raw_tp_test_run_info info;
723	int cpu = kattr->test.cpu, err = 0;
724	int current_cpu;
725
726	/* doesn't support data_in/out, ctx_out, duration, or repeat */
727	if (kattr->test.data_in || kattr->test.data_out ||
728	kattr->test.ctx_out || kattr->test.duration ||
729	kattr->test.repeat || kattr->test.batch_size)
730	return -EINVAL;
731
732	if (ctx_size_in < prog->aux->max_ctx_offset ||
733	ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64))
734	return -EINVAL;
735
736	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)
737	return -EINVAL;
738
739	if (ctx_size_in) {
740	info.ctx = memdup_user(ctx_in, ctx_size_in);
741	if (IS_ERR(ptr: info.ctx))
742	return PTR_ERR(ptr: info.ctx);
743	} else {
744	info.ctx = NULL;
745	}
746
747	info.prog = prog;
748
749	current_cpu = get_cpu();
750	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||
751	cpu == current_cpu) {
752	__bpf_prog_test_run_raw_tp(data: &info);
753	} else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
754	/* smp_call_function_single() also checks cpu_online()
755	* after csd_lock(). However, since cpu is from user
756	* space, let's do an extra quick check to filter out
757	* invalid value before smp_call_function_single().
758	*/
759	err = -ENXIO;
760	} else {
761	err = smp_call_function_single(cpuid: cpu, func: __bpf_prog_test_run_raw_tp,
762	info: &info, wait: 1);
763	}
764	put_cpu();
765
766	if (!err &&
767	copy_to_user(to: &uattr->test.retval, from: &info.retval, n: sizeof(u32)))
768	err = -EFAULT;
769
770	kfree(objp: info.ctx);
771	return err;
772	}
773
774	static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
775	{
776	void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
777	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
778	u32 size = kattr->test.ctx_size_in;
779	void *data;
780	int err;
781
782	if (!data_in && !data_out)
783	return NULL;
784
785	data = kzalloc(size: max_size, GFP_USER);
786	if (!data)
787	return ERR_PTR(error: -ENOMEM);
788
789	if (data_in) {
790	err = bpf_check_uarg_tail_zero(uaddr: USER_BPFPTR(p: data_in), expected_size: max_size, actual_size: size);
791	if (err) {
792	kfree(objp: data);
793	return ERR_PTR(error: err);
794	}
795
796	size = min_t(u32, max_size, size);
797	if (copy_from_user(to: data, from: data_in, n: size)) {
798	kfree(objp: data);
799	return ERR_PTR(error: -EFAULT);
800	}
801	}
802	return data;
803	}
804
805	static int bpf_ctx_finish(const union bpf_attr *kattr,
806	union bpf_attr __user *uattr, const void *data,
807	u32 size)
808	{
809	void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
810	int err = -EFAULT;
811	u32 copy_size = size;
812
813	if (!data || !data_out)
814	return 0;
815
816	if (copy_size > kattr->test.ctx_size_out) {
817	copy_size = kattr->test.ctx_size_out;
818	err = -ENOSPC;
819	}
820
821	if (copy_to_user(to: data_out, from: data, n: copy_size))
822	goto out;
823	if (copy_to_user(to: &uattr->test.ctx_size_out, from: &size, n: sizeof(size)))
824	goto out;
825	if (err != -ENOSPC)
826	err = 0;
827	out:
828	return err;
829	}
830
831	/**
832	* range_is_zero - test whether buffer is initialized
833	* @buf: buffer to check
834	* @from: check from this position
835	* @to: check up until (excluding) this position
836	*
837	* This function returns true if the there is a non-zero byte
838	* in the buf in the range [from,to).
839	*/
840	static inline bool range_is_zero(void *buf, size_t from, size_t to)
841	{
842	return !memchr_inv(p: (u8 *)buf + from, c: 0, size: to - from);
843	}
844
845	static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
846	{
847	struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
848
849	if (!__skb)
850	return 0;
851
852	/* make sure the fields we don't use are zeroed */
853	if (!range_is_zero(buf: __skb, from: 0, offsetof(struct __sk_buff, mark)))
854	return -EINVAL;
855
856	/* mark is allowed */
857
858	if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, mark),
859	offsetof(struct __sk_buff, priority)))
860	return -EINVAL;
861
862	/* priority is allowed */
863	/* ingress_ifindex is allowed */
864	/* ifindex is allowed */
865
866	if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, ifindex),
867	offsetof(struct __sk_buff, cb)))
868	return -EINVAL;
869
870	/* cb is allowed */
871
872	if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, cb),
873	offsetof(struct __sk_buff, tstamp)))
874	return -EINVAL;
875
876	/* tstamp is allowed */
877	/* wire_len is allowed */
878	/* gso_segs is allowed */
879
880	if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, gso_segs),
881	offsetof(struct __sk_buff, gso_size)))
882	return -EINVAL;
883
884	/* gso_size is allowed */
885
886	if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, gso_size),
887	offsetof(struct __sk_buff, hwtstamp)))
888	return -EINVAL;
889
890	/* hwtstamp is allowed */
891
892	if (!range_is_zero(buf: __skb, offsetofend(struct __sk_buff, hwtstamp),
893	to: sizeof(struct __sk_buff)))
894	return -EINVAL;
895
896	skb->mark = __skb->mark;
897	skb->priority = __skb->priority;
898	skb->skb_iif = __skb->ingress_ifindex;
899	skb->tstamp = __skb->tstamp;
900	memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
901
902	if (__skb->wire_len == 0) {
903	cb->pkt_len = skb->len;
904	} else {
905	if (__skb->wire_len < skb->len ||
906	__skb->wire_len > GSO_LEGACY_MAX_SIZE)
907	return -EINVAL;
908	cb->pkt_len = __skb->wire_len;
909	}
910
911	if (__skb->gso_segs > GSO_MAX_SEGS)
912	return -EINVAL;
913	skb_shinfo(skb)->gso_segs = __skb->gso_segs;
914	skb_shinfo(skb)->gso_size = __skb->gso_size;
915	skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp;
916
917	return 0;
918	}
919
920	static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
921	{
922	struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;
923
924	if (!__skb)
925	return;
926
927	__skb->mark = skb->mark;
928	__skb->priority = skb->priority;
929	__skb->ingress_ifindex = skb->skb_iif;
930	__skb->ifindex = skb->dev->ifindex;
931	__skb->tstamp = skb->tstamp;
932	memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
933	__skb->wire_len = cb->pkt_len;
934	__skb->gso_segs = skb_shinfo(skb)->gso_segs;
935	__skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp;
936	}
937
938	static struct proto bpf_dummy_proto = {
939	.name = "bpf_dummy",
940	.owner = THIS_MODULE,
941	.obj_size = sizeof(struct sock),
942	};
943
944	int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
945	union bpf_attr __user *uattr)
946	{
947	bool is_l2 = false, is_direct_pkt_access = false;
948	struct net *net = current->nsproxy->net_ns;
949	struct net_device *dev = net->loopback_dev;
950	u32 size = kattr->test.data_size_in;
951	u32 repeat = kattr->test.repeat;
952	struct __sk_buff *ctx = NULL;
953	u32 retval, duration;
954	int hh_len = ETH_HLEN;
955	struct sk_buff *skb;
956	struct sock *sk;
957	void *data;
958	int ret;
959
960	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
961	return -EINVAL;
962
963	data = bpf_test_init(kattr, user_size: kattr->test.data_size_in,
964	size, NET_SKB_PAD + NET_IP_ALIGN,
965	SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
966	if (IS_ERR(ptr: data))
967	return PTR_ERR(ptr: data);
968
969	ctx = bpf_ctx_init(kattr, max_size: sizeof(struct __sk_buff));
970	if (IS_ERR(ptr: ctx)) {
971	kfree(objp: data);
972	return PTR_ERR(ptr: ctx);
973	}
974
975	switch (prog->type) {
976	case BPF_PROG_TYPE_SCHED_CLS:
977	case BPF_PROG_TYPE_SCHED_ACT:
978	is_l2 = true;
979	fallthrough;
980	case BPF_PROG_TYPE_LWT_IN:
981	case BPF_PROG_TYPE_LWT_OUT:
982	case BPF_PROG_TYPE_LWT_XMIT:
983	is_direct_pkt_access = true;
984	break;
985	default:
986	break;
987	}
988
989	sk = sk_alloc(net, AF_UNSPEC, GFP_USER, prot: &bpf_dummy_proto, kern: 1);
990	if (!sk) {
991	kfree(objp: data);
992	kfree(objp: ctx);
993	return -ENOMEM;
994	}
995	sock_init_data(NULL, sk);
996
997	skb = slab_build_skb(data);
998	if (!skb) {
999	kfree(objp: data);
1000	kfree(objp: ctx);
1001	sk_free(sk);
1002	return -ENOMEM;
1003	}
1004	skb->sk = sk;
1005
1006	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
1007	__skb_put(skb, len: size);
1008	if (ctx && ctx->ifindex > 1) {
1009	dev = dev_get_by_index(net, ifindex: ctx->ifindex);
1010	if (!dev) {
1011	ret = -ENODEV;
1012	goto out;
1013	}
1014	}
1015	skb->protocol = eth_type_trans(skb, dev);
1016	skb_reset_network_header(skb);
1017
1018	switch (skb->protocol) {
1019	case htons(ETH_P_IP):
1020	sk->sk_family = AF_INET;
1021	if (sizeof(struct iphdr) <= skb_headlen(skb)) {
1022	sk->sk_rcv_saddr = ip_hdr(skb)->saddr;
1023	sk->sk_daddr = ip_hdr(skb)->daddr;
1024	}
1025	break;
1026	#if IS_ENABLED(CONFIG_IPV6)
1027	case htons(ETH_P_IPV6):
1028	sk->sk_family = AF_INET6;
1029	if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) {
1030	sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr;
1031	sk->sk_v6_daddr = ipv6_hdr(skb)->daddr;
1032	}
1033	break;
1034	#endif
1035	default:
1036	break;
1037	}
1038
1039	if (is_l2)
1040	__skb_push(skb, len: hh_len);
1041	if (is_direct_pkt_access)
1042	bpf_compute_data_pointers(skb);
1043	ret = convert___skb_to_skb(skb, skb: ctx);
1044	if (ret)
1045	goto out;
1046	ret = bpf_test_run(prog, ctx: skb, repeat, retval: &retval, time: &duration, xdp: false);
1047	if (ret)
1048	goto out;
1049	if (!is_l2) {
1050	if (skb_headroom(skb) < hh_len) {
1051	int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));
1052
1053	if (pskb_expand_head(skb, nhead, ntail: 0, GFP_USER)) {
1054	ret = -ENOMEM;
1055	goto out;
1056	}
1057	}
1058	memset(__skb_push(skb, hh_len), 0, hh_len);
1059	}
1060	convert_skb_to___skb(skb, skb: ctx);
1061
1062	size = skb->len;
1063	/* bpf program can never convert linear skb to non-linear */
1064	if (WARN_ON_ONCE(skb_is_nonlinear(skb)))
1065	size = skb_headlen(skb);
1066	ret = bpf_test_finish(kattr, uattr, data: skb->data, NULL, size, retval,
1067	duration);
1068	if (!ret)
1069	ret = bpf_ctx_finish(kattr, uattr, data: ctx,
1070	size: sizeof(struct __sk_buff));
1071	out:
1072	if (dev && dev != net->loopback_dev)
1073	dev_put(dev);
1074	kfree_skb(skb);
1075	sk_free(sk);
1076	kfree(objp: ctx);
1077	return ret;
1078	}
1079
1080	static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp)
1081	{
1082	unsigned int ingress_ifindex, rx_queue_index;
1083	struct netdev_rx_queue *rxqueue;
1084	struct net_device *device;
1085
1086	if (!xdp_md)
1087	return 0;
1088
1089	if (xdp_md->egress_ifindex != 0)
1090	return -EINVAL;
1091
1092	ingress_ifindex = xdp_md->ingress_ifindex;
1093	rx_queue_index = xdp_md->rx_queue_index;
1094
1095	if (!ingress_ifindex && rx_queue_index)
1096	return -EINVAL;
1097
1098	if (ingress_ifindex) {
1099	device = dev_get_by_index(current->nsproxy->net_ns,
1100	ifindex: ingress_ifindex);
1101	if (!device)
1102	return -ENODEV;
1103
1104	if (rx_queue_index >= device->real_num_rx_queues)
1105	goto free_dev;
1106
1107	rxqueue = __netif_get_rx_queue(dev: device, rxq: rx_queue_index);
1108
1109	if (!xdp_rxq_info_is_reg(xdp_rxq: &rxqueue->xdp_rxq))
1110	goto free_dev;
1111
1112	xdp->rxq = &rxqueue->xdp_rxq;
1113	/* The device is now tracked in the xdp->rxq for later
1114	* dev_put()
1115	*/
1116	}
1117
1118	xdp->data = xdp->data_meta + xdp_md->data;
1119	return 0;
1120
1121	free_dev:
1122	dev_put(dev: device);
1123	return -EINVAL;
1124	}
1125
1126	static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md)
1127	{
1128	if (!xdp_md)
1129	return;
1130
1131	xdp_md->data = xdp->data - xdp->data_meta;
1132	xdp_md->data_end = xdp->data_end - xdp->data_meta;
1133
1134	if (xdp_md->ingress_ifindex)
1135	dev_put(dev: xdp->rxq->dev);
1136	}
1137
1138	int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
1139	union bpf_attr __user *uattr)
1140	{
1141	bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES);
1142	u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1143	u32 batch_size = kattr->test.batch_size;
1144	u32 retval = 0, duration, max_data_sz;
1145	u32 size = kattr->test.data_size_in;
1146	u32 headroom = XDP_PACKET_HEADROOM;
1147	u32 repeat = kattr->test.repeat;
1148	struct netdev_rx_queue *rxqueue;
1149	struct skb_shared_info *sinfo;
1150	struct xdp_buff xdp = {};
1151	int i, ret = -EINVAL;
1152	struct xdp_md *ctx;
1153	void *data;
1154
1155	if (prog->expected_attach_type == BPF_XDP_DEVMAP ||
1156	prog->expected_attach_type == BPF_XDP_CPUMAP)
1157	return -EINVAL;
1158
1159	if (kattr->test.flags & ~BPF_F_TEST_XDP_LIVE_FRAMES)
1160	return -EINVAL;
1161
1162	if (bpf_prog_is_dev_bound(aux: prog->aux))
1163	return -EINVAL;
1164
1165	if (do_live) {
1166	if (!batch_size)
1167	batch_size = NAPI_POLL_WEIGHT;
1168	else if (batch_size > TEST_XDP_MAX_BATCH)
1169	return -E2BIG;
1170
1171	headroom += sizeof(struct xdp_page_head);
1172	} else if (batch_size) {
1173	return -EINVAL;
1174	}
1175
1176	ctx = bpf_ctx_init(kattr, max_size: sizeof(struct xdp_md));
1177	if (IS_ERR(ptr: ctx))
1178	return PTR_ERR(ptr: ctx);
1179
1180	if (ctx) {
1181	/* There can't be user provided data before the meta data */
1182	if (ctx->data_meta || ctx->data_end != size ||
1183	ctx->data > ctx->data_end ||
1184	unlikely(xdp_metalen_invalid(ctx->data)) ||
1185	(do_live && (kattr->test.data_out || kattr->test.ctx_out)))
1186	goto free_ctx;
1187	/* Meta data is allocated from the headroom */
1188	headroom -= ctx->data;
1189	}
1190
1191	max_data_sz = 4096 - headroom - tailroom;
1192	if (size > max_data_sz) {
1193	/* disallow live data mode for jumbo frames */
1194	if (do_live)
1195	goto free_ctx;
1196	size = max_data_sz;
1197	}
1198
1199	data = bpf_test_init(kattr, user_size: size, size: max_data_sz, headroom, tailroom);
1200	if (IS_ERR(ptr: data)) {
1201	ret = PTR_ERR(ptr: data);
1202	goto free_ctx;
1203	}
1204
1205	rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, rxq: 0);
1206	rxqueue->xdp_rxq.frag_size = headroom + max_data_sz + tailroom;
1207	xdp_init_buff(xdp: &xdp, frame_sz: rxqueue->xdp_rxq.frag_size, rxq: &rxqueue->xdp_rxq);
1208	xdp_prepare_buff(xdp: &xdp, hard_start: data, headroom, data_len: size, meta_valid: true);
1209	sinfo = xdp_get_shared_info_from_buff(xdp: &xdp);
1210
1211	ret = xdp_convert_md_to_buff(xdp_md: ctx, xdp: &xdp);
1212	if (ret)
1213	goto free_data;
1214
1215	if (unlikely(kattr->test.data_size_in > size)) {
1216	void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
1217
1218	while (size < kattr->test.data_size_in) {
1219	struct page *page;
1220	skb_frag_t *frag;
1221	u32 data_len;
1222
1223	if (sinfo->nr_frags == MAX_SKB_FRAGS) {
1224	ret = -ENOMEM;
1225	goto out;
1226	}
1227
1228	page = alloc_page(GFP_KERNEL);
1229	if (!page) {
1230	ret = -ENOMEM;
1231	goto out;
1232	}
1233
1234	frag = &sinfo->frags[sinfo->nr_frags++];
1235
1236	data_len = min_t(u32, kattr->test.data_size_in - size,
1237	PAGE_SIZE);
1238	skb_frag_fill_page_desc(frag, page, off: 0, size: data_len);
1239
1240	if (copy_from_user(page_address(page), from: data_in + size,
1241	n: data_len)) {
1242	ret = -EFAULT;
1243	goto out;
1244	}
1245	sinfo->xdp_frags_size += data_len;
1246	size += data_len;
1247	}
1248	xdp_buff_set_frags_flag(xdp: &xdp);
1249	}
1250
1251	if (repeat > 1)
1252	bpf_prog_change_xdp(NULL, prog);
1253
1254	if (do_live)
1255	ret = bpf_test_run_xdp_live(prog, ctx: &xdp, repeat, batch_size, time: &duration);
1256	else
1257	ret = bpf_test_run(prog, ctx: &xdp, repeat, retval: &retval, time: &duration, xdp: true);
1258	/* We convert the xdp_buff back to an xdp_md before checking the return
1259	* code so the reference count of any held netdevice will be decremented
1260	* even if the test run failed.
1261	*/
1262	xdp_convert_buff_to_md(xdp: &xdp, xdp_md: ctx);
1263	if (ret)
1264	goto out;
1265
1266	size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size;
1267	ret = bpf_test_finish(kattr, uattr, data: xdp.data_meta, sinfo, size,
1268	retval, duration);
1269	if (!ret)
1270	ret = bpf_ctx_finish(kattr, uattr, data: ctx,
1271	size: sizeof(struct xdp_md));
1272
1273	out:
1274	if (repeat > 1)
1275	bpf_prog_change_xdp(prev_prog: prog, NULL);
1276	free_data:
1277	for (i = 0; i < sinfo->nr_frags; i++)
1278	__free_page(skb_frag_page(&sinfo->frags[i]));
1279	kfree(objp: data);
1280	free_ctx:
1281	kfree(objp: ctx);
1282	return ret;
1283	}
1284
1285	static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
1286	{
1287	/* make sure the fields we don't use are zeroed */
1288	if (!range_is_zero(buf: ctx, from: 0, offsetof(struct bpf_flow_keys, flags)))
1289	return -EINVAL;
1290
1291	/* flags is allowed */
1292
1293	if (!range_is_zero(buf: ctx, offsetofend(struct bpf_flow_keys, flags),
1294	to: sizeof(struct bpf_flow_keys)))
1295	return -EINVAL;
1296
1297	return 0;
1298	}
1299
1300	int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
1301	const union bpf_attr *kattr,
1302	union bpf_attr __user *uattr)
1303	{
1304	struct bpf_test_timer t = { NO_PREEMPT };
1305	u32 size = kattr->test.data_size_in;
1306	struct bpf_flow_dissector ctx = {};
1307	u32 repeat = kattr->test.repeat;
1308	struct bpf_flow_keys *user_ctx;
1309	struct bpf_flow_keys flow_keys;
1310	const struct ethhdr *eth;
1311	unsigned int flags = 0;
1312	u32 retval, duration;
1313	void *data;
1314	int ret;
1315
1316	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
1317	return -EINVAL;
1318
1319	if (size < ETH_HLEN)
1320	return -EINVAL;
1321
1322	data = bpf_test_init(kattr, user_size: kattr->test.data_size_in, size, headroom: 0, tailroom: 0);
1323	if (IS_ERR(ptr: data))
1324	return PTR_ERR(ptr: data);
1325
1326	eth = (struct ethhdr *)data;
1327
1328	if (!repeat)
1329	repeat = 1;
1330
1331	user_ctx = bpf_ctx_init(kattr, max_size: sizeof(struct bpf_flow_keys));
1332	if (IS_ERR(ptr: user_ctx)) {
1333	kfree(objp: data);
1334	return PTR_ERR(ptr: user_ctx);
1335	}
1336	if (user_ctx) {
1337	ret = verify_user_bpf_flow_keys(ctx: user_ctx);
1338	if (ret)
1339	goto out;
1340	flags = user_ctx->flags;
1341	}
1342
1343	ctx.flow_keys = &flow_keys;
1344	ctx.data = data;
1345	ctx.data_end = (__u8 *)data + size;
1346
1347	bpf_test_timer_enter(t: &t);
1348	do {
1349	retval = bpf_flow_dissect(prog, ctx: &ctx, proto: eth->h_proto, ETH_HLEN,
1350	hlen: size, flags);
1351	} while (bpf_test_timer_continue(t: &t, iterations: 1, repeat, err: &ret, duration: &duration));
1352	bpf_test_timer_leave(t: &t);
1353
1354	if (ret < 0)
1355	goto out;
1356
1357	ret = bpf_test_finish(kattr, uattr, data: &flow_keys, NULL,
1358	size: sizeof(flow_keys), retval, duration);
1359	if (!ret)
1360	ret = bpf_ctx_finish(kattr, uattr, data: user_ctx,
1361	size: sizeof(struct bpf_flow_keys));
1362
1363	out:
1364	kfree(objp: user_ctx);
1365	kfree(objp: data);
1366	return ret;
1367	}
1368
1369	int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,
1370	union bpf_attr __user *uattr)
1371	{
1372	struct bpf_test_timer t = { NO_PREEMPT };
1373	struct bpf_prog_array *progs = NULL;
1374	struct bpf_sk_lookup_kern ctx = {};
1375	u32 repeat = kattr->test.repeat;
1376	struct bpf_sk_lookup *user_ctx;
1377	u32 retval, duration;
1378	int ret = -EINVAL;
1379
1380	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
1381	return -EINVAL;
1382
1383	if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||
1384	kattr->test.data_size_out)
1385	return -EINVAL;
1386
1387	if (!repeat)
1388	repeat = 1;
1389
1390	user_ctx = bpf_ctx_init(kattr, max_size: sizeof(*user_ctx));
1391	if (IS_ERR(ptr: user_ctx))
1392	return PTR_ERR(ptr: user_ctx);
1393
1394	if (!user_ctx)
1395	return -EINVAL;
1396
1397	if (user_ctx->sk)
1398	goto out;
1399
1400	if (!range_is_zero(buf: user_ctx, offsetofend(typeof(*user_ctx), local_port), to: sizeof(*user_ctx)))
1401	goto out;
1402
1403	if (user_ctx->local_port > U16_MAX) {
1404	ret = -ERANGE;
1405	goto out;
1406	}
1407
1408	ctx.family = (u16)user_ctx->family;
1409	ctx.protocol = (u16)user_ctx->protocol;
1410	ctx.dport = (u16)user_ctx->local_port;
1411	ctx.sport = user_ctx->remote_port;
1412
1413	switch (ctx.family) {
1414	case AF_INET:
1415	ctx.v4.daddr = (__force __be32)user_ctx->local_ip4;
1416	ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4;
1417	break;
1418
1419	#if IS_ENABLED(CONFIG_IPV6)
1420	case AF_INET6:
1421	ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6;
1422	ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6;
1423	break;
1424	#endif
1425
1426	default:
1427	ret = -EAFNOSUPPORT;
1428	goto out;
1429	}
1430
1431	progs = bpf_prog_array_alloc(prog_cnt: 1, GFP_KERNEL);
1432	if (!progs) {
1433	ret = -ENOMEM;
1434	goto out;
1435	}
1436
1437	progs->items[0].prog = prog;
1438
1439	bpf_test_timer_enter(t: &t);
1440	do {
1441	ctx.selected_sk = NULL;
1442	retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run);
1443	} while (bpf_test_timer_continue(t: &t, iterations: 1, repeat, err: &ret, duration: &duration));
1444	bpf_test_timer_leave(t: &t);
1445
1446	if (ret < 0)
1447	goto out;
1448
1449	user_ctx->cookie = 0;
1450	if (ctx.selected_sk) {
1451	if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) {
1452	ret = -EOPNOTSUPP;
1453	goto out;
1454	}
1455
1456	user_ctx->cookie = sock_gen_cookie(sk: ctx.selected_sk);
1457	}
1458
1459	ret = bpf_test_finish(kattr, uattr, NULL, NULL, size: 0, retval, duration);
1460	if (!ret)
1461	ret = bpf_ctx_finish(kattr, uattr, data: user_ctx, size: sizeof(*user_ctx));
1462
1463	out:
1464	bpf_prog_array_free(progs);
1465	kfree(objp: user_ctx);
1466	return ret;
1467	}
1468
1469	int bpf_prog_test_run_syscall(struct bpf_prog *prog,
1470	const union bpf_attr *kattr,
1471	union bpf_attr __user *uattr)
1472	{
1473	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
1474	__u32 ctx_size_in = kattr->test.ctx_size_in;
1475	void *ctx = NULL;
1476	u32 retval;
1477	int err = 0;
1478
1479	/* doesn't support data_in/out, ctx_out, duration, or repeat or flags */
1480	if (kattr->test.data_in || kattr->test.data_out ||
1481	kattr->test.ctx_out || kattr->test.duration ||
1482	kattr->test.repeat || kattr->test.flags ||
1483	kattr->test.batch_size)
1484	return -EINVAL;
1485
1486	if (ctx_size_in < prog->aux->max_ctx_offset ||
1487	ctx_size_in > U16_MAX)
1488	return -EINVAL;
1489
1490	if (ctx_size_in) {
1491	ctx = memdup_user(ctx_in, ctx_size_in);
1492	if (IS_ERR(ptr: ctx))
1493	return PTR_ERR(ptr: ctx);
1494	}
1495
1496	rcu_read_lock_trace();
1497	retval = bpf_prog_run_pin_on_cpu(prog, ctx);
1498	rcu_read_unlock_trace();
1499
1500	if (copy_to_user(to: &uattr->test.retval, from: &retval, n: sizeof(u32))) {
1501	err = -EFAULT;
1502	goto out;
1503	}
1504	if (ctx_size_in)
1505	if (copy_to_user(to: ctx_in, from: ctx, n: ctx_size_in))
1506	err = -EFAULT;
1507	out:
1508	kfree(objp: ctx);
1509	return err;
1510	}
1511
1512	static int verify_and_copy_hook_state(struct nf_hook_state *state,
1513	const struct nf_hook_state *user,
1514	struct net_device *dev)
1515	{
1516	if (user->in || user->out)
1517	return -EINVAL;
1518
1519	if (user->net || user->sk || user->okfn)
1520	return -EINVAL;
1521
1522	switch (user->pf) {
1523	case NFPROTO_IPV4:
1524	case NFPROTO_IPV6:
1525	switch (state->hook) {
1526	case NF_INET_PRE_ROUTING:
1527	state->in = dev;
1528	break;
1529	case NF_INET_LOCAL_IN:
1530	state->in = dev;
1531	break;
1532	case NF_INET_FORWARD:
1533	state->in = dev;
1534	state->out = dev;
1535	break;
1536	case NF_INET_LOCAL_OUT:
1537	state->out = dev;
1538	break;
1539	case NF_INET_POST_ROUTING:
1540	state->out = dev;
1541	break;
1542	}
1543
1544	break;
1545	default:
1546	return -EINVAL;
1547	}
1548
1549	state->pf = user->pf;
1550	state->hook = user->hook;
1551
1552	return 0;
1553	}
1554
1555	static __be16 nfproto_eth(int nfproto)
1556	{
1557	switch (nfproto) {
1558	case NFPROTO_IPV4:
1559	return htons(ETH_P_IP);
1560	case NFPROTO_IPV6:
1561	break;
1562	}
1563
1564	return htons(ETH_P_IPV6);
1565	}
1566
1567	int bpf_prog_test_run_nf(struct bpf_prog *prog,
1568	const union bpf_attr *kattr,
1569	union bpf_attr __user *uattr)
1570	{
1571	struct net *net = current->nsproxy->net_ns;
1572	struct net_device *dev = net->loopback_dev;
1573	struct nf_hook_state *user_ctx, hook_state = {
1574	.pf = NFPROTO_IPV4,
1575	.hook = NF_INET_LOCAL_OUT,
1576	};
1577	u32 size = kattr->test.data_size_in;
1578	u32 repeat = kattr->test.repeat;
1579	struct bpf_nf_ctx ctx = {
1580	.state = &hook_state,
1581	};
1582	struct sk_buff *skb = NULL;
1583	u32 retval, duration;
1584	void *data;
1585	int ret;
1586
1587	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
1588	return -EINVAL;
1589
1590	if (size < sizeof(struct iphdr))
1591	return -EINVAL;
1592
1593	data = bpf_test_init(kattr, user_size: kattr->test.data_size_in, size,
1594	NET_SKB_PAD + NET_IP_ALIGN,
1595	SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
1596	if (IS_ERR(ptr: data))
1597	return PTR_ERR(ptr: data);
1598
1599	if (!repeat)
1600	repeat = 1;
1601
1602	user_ctx = bpf_ctx_init(kattr, max_size: sizeof(struct nf_hook_state));
1603	if (IS_ERR(ptr: user_ctx)) {
1604	kfree(objp: data);
1605	return PTR_ERR(ptr: user_ctx);
1606	}
1607
1608	if (user_ctx) {
1609	ret = verify_and_copy_hook_state(state: &hook_state, user: user_ctx, dev);
1610	if (ret)
1611	goto out;
1612	}
1613
1614	skb = slab_build_skb(data);
1615	if (!skb) {
1616	ret = -ENOMEM;
1617	goto out;
1618	}
1619
1620	data = NULL; /* data released via kfree_skb */
1621
1622	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
1623	__skb_put(skb, len: size);
1624
1625	ret = -EINVAL;
1626
1627	if (hook_state.hook != NF_INET_LOCAL_OUT) {
1628	if (size < ETH_HLEN + sizeof(struct iphdr))
1629	goto out;
1630
1631	skb->protocol = eth_type_trans(skb, dev);
1632	switch (skb->protocol) {
1633	case htons(ETH_P_IP):
1634	if (hook_state.pf == NFPROTO_IPV4)
1635	break;
1636	goto out;
1637	case htons(ETH_P_IPV6):
1638	if (size < ETH_HLEN + sizeof(struct ipv6hdr))
1639	goto out;
1640	if (hook_state.pf == NFPROTO_IPV6)
1641	break;
1642	goto out;
1643	default:
1644	ret = -EPROTO;
1645	goto out;
1646	}
1647
1648	skb_reset_network_header(skb);
1649	} else {
1650	skb->protocol = nfproto_eth(nfproto: hook_state.pf);
1651	}
1652
1653	ctx.skb = skb;
1654
1655	ret = bpf_test_run(prog, ctx: &ctx, repeat, retval: &retval, time: &duration, xdp: false);
1656	if (ret)
1657	goto out;
1658
1659	ret = bpf_test_finish(kattr, uattr, NULL, NULL, size: 0, retval, duration);
1660
1661	out:
1662	kfree(objp: user_ctx);
1663	kfree_skb(skb);
1664	kfree(objp: data);
1665	return ret;
1666	}
1667
1668	static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = {
1669	.owner = THIS_MODULE,
1670	.set = &test_sk_check_kfunc_ids,
1671	};
1672
1673	BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids)
1674	BTF_ID(struct, prog_test_ref_kfunc)
1675	BTF_ID(func, bpf_kfunc_call_test_release)
1676	BTF_ID(struct, prog_test_member)
1677	BTF_ID(func, bpf_kfunc_call_memb_release)
1678
1679	static int __init bpf_prog_test_run_init(void)
1680	{
1681	const struct btf_id_dtor_kfunc bpf_prog_test_dtor_kfunc[] = {
1682	{
1683	.btf_id = bpf_prog_test_dtor_kfunc_ids[0],
1684	.kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[1]
1685	},
1686	{
1687	.btf_id = bpf_prog_test_dtor_kfunc_ids[2],
1688	.kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[3],
1689	},
1690	};
1691	int ret;
1692
1693	ret = register_btf_fmodret_id_set(kset: &bpf_test_modify_return_set);
1694	ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SCHED_CLS, s: &bpf_prog_test_kfunc_set);
1695	ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_TRACING, s: &bpf_prog_test_kfunc_set);
1696	ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SYSCALL, s: &bpf_prog_test_kfunc_set);
1697	return ret ?: register_btf_id_dtor_kfuncs(dtors: bpf_prog_test_dtor_kfunc,
1698	ARRAY_SIZE(bpf_prog_test_dtor_kfunc),
1699	THIS_MODULE);
1700	}
1701	late_initcall(bpf_prog_test_run_init);
1702