1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* x_tables core - Backend for {ip,ip6,arp}_tables
4	*
5	* Copyright (C) 2006-2006 Harald Welte <laforge@netfilter.org>
6	* Copyright (C) 2006-2012 Patrick McHardy <kaber@trash.net>
7	*
8	* Based on existing ip_tables code which is
9	* Copyright (C) 1999 Paul `Rusty' Russell & Michael J. Neuling
10	* Copyright (C) 2000-2005 Netfilter Core Team <coreteam@netfilter.org>
11	*/
12	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13	#include <linux/kernel.h>
14	#include <linux/module.h>
15	#include <linux/socket.h>
16	#include <linux/net.h>
17	#include <linux/proc_fs.h>
18	#include <linux/seq_file.h>
19	#include <linux/string.h>
20	#include <linux/vmalloc.h>
21	#include <linux/mutex.h>
22	#include <linux/mm.h>
23	#include <linux/slab.h>
24	#include <linux/audit.h>
25	#include <linux/user_namespace.h>
26	#include <net/net_namespace.h>
27	#include <net/netns/generic.h>
28
29	#include <linux/netfilter/x_tables.h>
30	#include <linux/netfilter_arp.h>
31	#include <linux/netfilter_ipv4/ip_tables.h>
32	#include <linux/netfilter_ipv6/ip6_tables.h>
33	#include <linux/netfilter_arp/arp_tables.h>
34
35	MODULE_LICENSE("GPL");
36	MODULE_AUTHOR("Harald Welte <laforge@netfilter.org>");
37	MODULE_DESCRIPTION("{ip,ip6,arp,eb}_tables backend module");
38
39	#define XT_PCPU_BLOCK_SIZE 4096
40	#define XT_MAX_TABLE_SIZE (512 * 1024 * 1024)
41
42	struct xt_template {
43	struct list_head list;
44
45	/* called when table is needed in the given netns */
46	int (*table_init)(struct net *net);
47
48	struct module *me;
49
50	/* A unique name... */
51	char name[XT_TABLE_MAXNAMELEN];
52	};
53
54	static struct list_head xt_templates[NFPROTO_NUMPROTO];
55
56	struct xt_pernet {
57	struct list_head tables[NFPROTO_NUMPROTO];
58	};
59
60	struct compat_delta {
61	unsigned int offset; /* offset in kernel */
62	int delta; /* delta in 32bit user land */
63	};
64
65	struct xt_af {
66	struct mutex mutex;
67	struct list_head match;
68	struct list_head target;
69	#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
70	struct mutex compat_mutex;
71	struct compat_delta *compat_tab;
72	unsigned int number; /* number of slots in compat_tab[] */
73	unsigned int cur; /* number of used slots in compat_tab[] */
74	#endif
75	};
76
77	static unsigned int xt_pernet_id __read_mostly;
78	static struct xt_af *xt __read_mostly;
79
80	static const char *const xt_prefix[NFPROTO_NUMPROTO] = {
81	[NFPROTO_UNSPEC] = "x",
82	[NFPROTO_IPV4] = "ip",
83	[NFPROTO_ARP] = "arp",
84	[NFPROTO_BRIDGE] = "eb",
85	[NFPROTO_IPV6] = "ip6",
86	};
87
88	/* Registration hooks for targets. */
89	int xt_register_target(struct xt_target *target)
90	{
91	u_int8_t af = target->family;
92
93	mutex_lock(&xt[af].mutex);
94	list_add(new: &target->list, head: &xt[af].target);
95	mutex_unlock(lock: &xt[af].mutex);
96	return 0;
97	}
98	EXPORT_SYMBOL(xt_register_target);
99
100	void
101	xt_unregister_target(struct xt_target *target)
102	{
103	u_int8_t af = target->family;
104
105	mutex_lock(&xt[af].mutex);
106	list_del(entry: &target->list);
107	mutex_unlock(lock: &xt[af].mutex);
108	}
109	EXPORT_SYMBOL(xt_unregister_target);
110
111	int
112	xt_register_targets(struct xt_target *target, unsigned int n)
113	{
114	unsigned int i;
115	int err = 0;
116
117	for (i = 0; i < n; i++) {
118	err = xt_register_target(&target[i]);
119	if (err)
120	goto err;
121	}
122	return err;
123
124	err:
125	if (i > 0)
126	xt_unregister_targets(target, n: i);
127	return err;
128	}
129	EXPORT_SYMBOL(xt_register_targets);
130
131	void
132	xt_unregister_targets(struct xt_target *target, unsigned int n)
133	{
134	while (n-- > 0)
135	xt_unregister_target(&target[n]);
136	}
137	EXPORT_SYMBOL(xt_unregister_targets);
138
139	int xt_register_match(struct xt_match *match)
140	{
141	u_int8_t af = match->family;
142
143	mutex_lock(&xt[af].mutex);
144	list_add(new: &match->list, head: &xt[af].match);
145	mutex_unlock(lock: &xt[af].mutex);
146	return 0;
147	}
148	EXPORT_SYMBOL(xt_register_match);
149
150	void
151	xt_unregister_match(struct xt_match *match)
152	{
153	u_int8_t af = match->family;
154
155	mutex_lock(&xt[af].mutex);
156	list_del(entry: &match->list);
157	mutex_unlock(lock: &xt[af].mutex);
158	}
159	EXPORT_SYMBOL(xt_unregister_match);
160
161	int
162	xt_register_matches(struct xt_match *match, unsigned int n)
163	{
164	unsigned int i;
165	int err = 0;
166
167	for (i = 0; i < n; i++) {
168	err = xt_register_match(&match[i]);
169	if (err)
170	goto err;
171	}
172	return err;
173
174	err:
175	if (i > 0)
176	xt_unregister_matches(match, n: i);
177	return err;
178	}
179	EXPORT_SYMBOL(xt_register_matches);
180
181	void
182	xt_unregister_matches(struct xt_match *match, unsigned int n)
183	{
184	while (n-- > 0)
185	xt_unregister_match(&match[n]);
186	}
187	EXPORT_SYMBOL(xt_unregister_matches);
188
189
190	/*
191	* These are weird, but module loading must not be done with mutex
192	* held (since they will register), and we have to have a single
193	* function to use.
194	*/
195
196	/* Find match, grabs ref. Returns ERR_PTR() on error. */
197	struct xt_match *xt_find_match(u8 af, const char *name, u8 revision)
198	{
199	struct xt_match *m;
200	int err = -ENOENT;
201
202	if (strnlen(p: name, XT_EXTENSION_MAXNAMELEN) == XT_EXTENSION_MAXNAMELEN)
203	return ERR_PTR(error: -EINVAL);
204
205	mutex_lock(&xt[af].mutex);
206	list_for_each_entry(m, &xt[af].match, list) {
207	if (strcmp(m->name, name) == 0) {
208	if (m->revision == revision) {
209	if (try_module_get(module: m->me)) {
210	mutex_unlock(lock: &xt[af].mutex);
211	return m;
212	}
213	} else
214	err = -EPROTOTYPE; /* Found something. */
215	}
216	}
217	mutex_unlock(lock: &xt[af].mutex);
218
219	if (af != NFPROTO_UNSPEC)
220	/* Try searching again in the family-independent list */
221	return xt_find_match(af: NFPROTO_UNSPEC, name, revision);
222
223	return ERR_PTR(error: err);
224	}
225	EXPORT_SYMBOL(xt_find_match);
226
227	struct xt_match *
228	xt_request_find_match(uint8_t nfproto, const char *name, uint8_t revision)
229	{
230	struct xt_match *match;
231
232	if (strnlen(p: name, XT_EXTENSION_MAXNAMELEN) == XT_EXTENSION_MAXNAMELEN)
233	return ERR_PTR(error: -EINVAL);
234
235	match = xt_find_match(nfproto, name, revision);
236	if (IS_ERR(ptr: match)) {
237	request_module("%st_%s", xt_prefix[nfproto], name);
238	match = xt_find_match(nfproto, name, revision);
239	}
240
241	return match;
242	}
243	EXPORT_SYMBOL_GPL(xt_request_find_match);
244
245	/* Find target, grabs ref. Returns ERR_PTR() on error. */
246	static struct xt_target *xt_find_target(u8 af, const char *name, u8 revision)
247	{
248	struct xt_target *t;
249	int err = -ENOENT;
250
251	if (strnlen(p: name, XT_EXTENSION_MAXNAMELEN) == XT_EXTENSION_MAXNAMELEN)
252	return ERR_PTR(error: -EINVAL);
253
254	mutex_lock(&xt[af].mutex);
255	list_for_each_entry(t, &xt[af].target, list) {
256	if (strcmp(t->name, name) == 0) {
257	if (t->revision == revision) {
258	if (try_module_get(module: t->me)) {
259	mutex_unlock(lock: &xt[af].mutex);
260	return t;
261	}
262	} else
263	err = -EPROTOTYPE; /* Found something. */
264	}
265	}
266	mutex_unlock(lock: &xt[af].mutex);
267
268	if (af != NFPROTO_UNSPEC)
269	/* Try searching again in the family-independent list */
270	return xt_find_target(af: NFPROTO_UNSPEC, name, revision);
271
272	return ERR_PTR(error: err);
273	}
274
275	struct xt_target *xt_request_find_target(u8 af, const char *name, u8 revision)
276	{
277	struct xt_target *target;
278
279	if (strnlen(p: name, XT_EXTENSION_MAXNAMELEN) == XT_EXTENSION_MAXNAMELEN)
280	return ERR_PTR(error: -EINVAL);
281
282	target = xt_find_target(af, name, revision);
283	if (IS_ERR(ptr: target)) {
284	request_module("%st_%s", xt_prefix[af], name);
285	target = xt_find_target(af, name, revision);
286	}
287
288	return target;
289	}
290	EXPORT_SYMBOL_GPL(xt_request_find_target);
291
292
293	static int xt_obj_to_user(u16 __user *psize, u16 size,
294	void __user *pname, const char *name,
295	u8 __user *prev, u8 rev)
296	{
297	if (put_user(size, psize))
298	return -EFAULT;
299	if (copy_to_user(to: pname, from: name, strlen(name) + 1))
300	return -EFAULT;
301	if (put_user(rev, prev))
302	return -EFAULT;
303
304	return 0;
305	}
306
307	#define XT_OBJ_TO_USER(U, K, TYPE, C_SIZE) \
308	xt_obj_to_user(&U->u.TYPE##_size, C_SIZE ? : K->u.TYPE##_size, \
309	U->u.user.name, K->u.kernel.TYPE->name, \
310	&U->u.user.revision, K->u.kernel.TYPE->revision)
311
312	int xt_data_to_user(void __user *dst, const void *src,
313	int usersize, int size, int aligned_size)
314	{
315	usersize = usersize ? : size;
316	if (copy_to_user(to: dst, from: src, n: usersize))
317	return -EFAULT;
318	if (usersize != aligned_size &&
319	clear_user(to: dst + usersize, n: aligned_size - usersize))
320	return -EFAULT;
321
322	return 0;
323	}
324	EXPORT_SYMBOL_GPL(xt_data_to_user);
325
326	#define XT_DATA_TO_USER(U, K, TYPE) \
327	xt_data_to_user(U->data, K->data, \
328	K->u.kernel.TYPE->usersize, \
329	K->u.kernel.TYPE->TYPE##size, \
330	XT_ALIGN(K->u.kernel.TYPE->TYPE##size))
331
332	int xt_match_to_user(const struct xt_entry_match *m,
333	struct xt_entry_match __user *u)
334	{
335	return XT_OBJ_TO_USER(u, m, match, 0) ||
336	XT_DATA_TO_USER(u, m, match);
337	}
338	EXPORT_SYMBOL_GPL(xt_match_to_user);
339
340	int xt_target_to_user(const struct xt_entry_target *t,
341	struct xt_entry_target __user *u)
342	{
343	return XT_OBJ_TO_USER(u, t, target, 0) ||
344	XT_DATA_TO_USER(u, t, target);
345	}
346	EXPORT_SYMBOL_GPL(xt_target_to_user);
347
348	static int match_revfn(u8 af, const char *name, u8 revision, int *bestp)
349	{
350	const struct xt_match *m;
351	int have_rev = 0;
352
353	mutex_lock(&xt[af].mutex);
354	list_for_each_entry(m, &xt[af].match, list) {
355	if (strcmp(m->name, name) == 0) {
356	if (m->revision > *bestp)
357	*bestp = m->revision;
358	if (m->revision == revision)
359	have_rev = 1;
360	}
361	}
362	mutex_unlock(lock: &xt[af].mutex);
363
364	if (af != NFPROTO_UNSPEC && !have_rev)
365	return match_revfn(af: NFPROTO_UNSPEC, name, revision, bestp);
366
367	return have_rev;
368	}
369
370	static int target_revfn(u8 af, const char *name, u8 revision, int *bestp)
371	{
372	const struct xt_target *t;
373	int have_rev = 0;
374
375	mutex_lock(&xt[af].mutex);
376	list_for_each_entry(t, &xt[af].target, list) {
377	if (strcmp(t->name, name) == 0) {
378	if (t->revision > *bestp)
379	*bestp = t->revision;
380	if (t->revision == revision)
381	have_rev = 1;
382	}
383	}
384	mutex_unlock(lock: &xt[af].mutex);
385
386	if (af != NFPROTO_UNSPEC && !have_rev)
387	return target_revfn(af: NFPROTO_UNSPEC, name, revision, bestp);
388
389	return have_rev;
390	}
391
392	/* Returns true or false (if no such extension at all) */
393	int xt_find_revision(u8 af, const char *name, u8 revision, int target,
394	int *err)
395	{
396	int have_rev, best = -1;
397
398	if (target == 1)
399	have_rev = target_revfn(af, name, revision, bestp: &best);
400	else
401	have_rev = match_revfn(af, name, revision, bestp: &best);
402
403	/* Nothing at all? Return 0 to try loading module. */
404	if (best == -1) {
405	*err = -ENOENT;
406	return 0;
407	}
408
409	*err = best;
410	if (!have_rev)
411	*err = -EPROTONOSUPPORT;
412	return 1;
413	}
414	EXPORT_SYMBOL_GPL(xt_find_revision);
415
416	static char *
417	textify_hooks(char *buf, size_t size, unsigned int mask, uint8_t nfproto)
418	{
419	static const char *const inetbr_names[] = {
420	"PREROUTING", "INPUT", "FORWARD",
421	"OUTPUT", "POSTROUTING", "BROUTING",
422	};
423	static const char *const arp_names[] = {
424	"INPUT", "FORWARD", "OUTPUT",
425	};
426	const char *const *names;
427	unsigned int i, max;
428	char *p = buf;
429	bool np = false;
430	int res;
431
432	names = (nfproto == NFPROTO_ARP) ? arp_names : inetbr_names;
433	max = (nfproto == NFPROTO_ARP) ? ARRAY_SIZE(arp_names) :
434	ARRAY_SIZE(inetbr_names);
435	*p = '\0';
436	for (i = 0; i < max; ++i) {
437	if (!(mask & (1 << i)))
438	continue;
439	res = snprintf(buf: p, size, fmt: "%s%s", np ? "/" : "", names[i]);
440	if (res > 0) {
441	size -= res;
442	p += res;
443	}
444	np = true;
445	}
446
447	return buf;
448	}
449
450	/**
451	* xt_check_proc_name - check that name is suitable for /proc file creation
452	*
453	* @name: file name candidate
454	* @size: length of buffer
455	*
456	* some x_tables modules wish to create a file in /proc.
457	* This function makes sure that the name is suitable for this
458	* purpose, it checks that name is NUL terminated and isn't a 'special'
459	* name, like "..".
460	*
461	* returns negative number on error or 0 if name is useable.
462	*/
463	int xt_check_proc_name(const char *name, unsigned int size)
464	{
465	if (name[0] == '\0')
466	return -EINVAL;
467
468	if (strnlen(p: name, maxlen: size) == size)
469	return -ENAMETOOLONG;
470
471	if (strcmp(name, ".") == 0 ||
472	strcmp(name, "..") == 0 ||
473	strchr(name, '/'))
474	return -EINVAL;
475
476	return 0;
477	}
478	EXPORT_SYMBOL(xt_check_proc_name);
479
480	int xt_check_match(struct xt_mtchk_param *par,
481	unsigned int size, u16 proto, bool inv_proto)
482	{
483	int ret;
484
485	if (XT_ALIGN(par->match->matchsize) != size &&
486	par->match->matchsize != -1) {
487	/*
488	* ebt_among is exempt from centralized matchsize checking
489	* because it uses a dynamic-size data set.
490	*/
491	pr_err_ratelimited("%s_tables: %s.%u match: invalid size %u (kernel) != (user) %u\n",
492	xt_prefix[par->family], par->match->name,
493	par->match->revision,
494	XT_ALIGN(par->match->matchsize), size);
495	return -EINVAL;
496	}
497	if (par->match->table != NULL &&
498	strcmp(par->match->table, par->table) != 0) {
499	pr_info_ratelimited("%s_tables: %s match: only valid in %s table, not %s\n",
500	xt_prefix[par->family], par->match->name,
501	par->match->table, par->table);
502	return -EINVAL;
503	}
504	if (par->match->hooks && (par->hook_mask & ~par->match->hooks) != 0) {
505	char used[64], allow[64];
506
507	pr_info_ratelimited("%s_tables: %s match: used from hooks %s, but only valid from %s\n",
508	xt_prefix[par->family], par->match->name,
509	textify_hooks(used, sizeof(used),
510	par->hook_mask, par->family),
511	textify_hooks(allow, sizeof(allow),
512	par->match->hooks,
513	par->family));
514	return -EINVAL;
515	}
516	if (par->match->proto && (par->match->proto != proto || inv_proto)) {
517	pr_info_ratelimited("%s_tables: %s match: only valid for protocol %u\n",
518	xt_prefix[par->family], par->match->name,
519	par->match->proto);
520	return -EINVAL;
521	}
522	if (par->match->checkentry != NULL) {
523	ret = par->match->checkentry(par);
524	if (ret < 0)
525	return ret;
526	else if (ret > 0)
527	/* Flag up potential errors. */
528	return -EIO;
529	}
530	return 0;
531	}
532	EXPORT_SYMBOL_GPL(xt_check_match);
533
534	/** xt_check_entry_match - check that matches end before start of target
535	*
536	* @match: beginning of xt_entry_match
537	* @target: beginning of this rules target (alleged end of matches)
538	* @alignment: alignment requirement of match structures
539	*
540	* Validates that all matches add up to the beginning of the target,
541	* and that each match covers at least the base structure size.
542	*
543	* Return: 0 on success, negative errno on failure.
544	*/
545	static int xt_check_entry_match(const char *match, const char *target,
546	const size_t alignment)
547	{
548	const struct xt_entry_match *pos;
549	int length = target - match;
550
551	if (length == 0) /* no matches */
552	return 0;
553
554	pos = (struct xt_entry_match *)match;
555	do {
556	if ((unsigned long)pos % alignment)
557	return -EINVAL;
558
559	if (length < (int)sizeof(struct xt_entry_match))
560	return -EINVAL;
561
562	if (pos->u.match_size < sizeof(struct xt_entry_match))
563	return -EINVAL;
564
565	if (pos->u.match_size > length)
566	return -EINVAL;
567
568	length -= pos->u.match_size;
569	pos = ((void *)((char *)(pos) + (pos)->u.match_size));
570	} while (length > 0);
571
572	return 0;
573	}
574
575	/** xt_check_table_hooks - check hook entry points are sane
576	*
577	* @info xt_table_info to check
578	* @valid_hooks - hook entry points that we can enter from
579	*
580	* Validates that the hook entry and underflows points are set up.
581	*
582	* Return: 0 on success, negative errno on failure.
583	*/
584	int xt_check_table_hooks(const struct xt_table_info *info, unsigned int valid_hooks)
585	{
586	const char *err = "unsorted underflow";
587	unsigned int i, max_uflow, max_entry;
588	bool check_hooks = false;
589
590	BUILD_BUG_ON(ARRAY_SIZE(info->hook_entry) != ARRAY_SIZE(info->underflow));
591
592	max_entry = 0;
593	max_uflow = 0;
594
595	for (i = 0; i < ARRAY_SIZE(info->hook_entry); i++) {
596	if (!(valid_hooks & (1 << i)))
597	continue;
598
599	if (info->hook_entry[i] == 0xFFFFFFFF)
600	return -EINVAL;
601	if (info->underflow[i] == 0xFFFFFFFF)
602	return -EINVAL;
603
604	if (check_hooks) {
605	if (max_uflow > info->underflow[i])
606	goto error;
607
608	if (max_uflow == info->underflow[i]) {
609	err = "duplicate underflow";
610	goto error;
611	}
612	if (max_entry > info->hook_entry[i]) {
613	err = "unsorted entry";
614	goto error;
615	}
616	if (max_entry == info->hook_entry[i]) {
617	err = "duplicate entry";
618	goto error;
619	}
620	}
621	max_entry = info->hook_entry[i];
622	max_uflow = info->underflow[i];
623	check_hooks = true;
624	}
625
626	return 0;
627	error:
628	pr_err_ratelimited("%s at hook %d\n", err, i);
629	return -EINVAL;
630	}
631	EXPORT_SYMBOL(xt_check_table_hooks);
632
633	static bool verdict_ok(int verdict)
634	{
635	if (verdict > 0)
636	return true;
637
638	if (verdict < 0) {
639	int v = -verdict - 1;
640
641	if (verdict == XT_RETURN)
642	return true;
643
644	switch (v) {
645	case NF_ACCEPT: return true;
646	case NF_DROP: return true;
647	case NF_QUEUE: return true;
648	default:
649	break;
650	}
651
652	return false;
653	}
654
655	return false;
656	}
657
658	static bool error_tg_ok(unsigned int usersize, unsigned int kernsize,
659	const char *msg, unsigned int msglen)
660	{
661	return usersize == kernsize && strnlen(p: msg, maxlen: msglen) < msglen;
662	}
663
664	#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
665	int xt_compat_add_offset(u_int8_t af, unsigned int offset, int delta)
666	{
667	struct xt_af *xp = &xt[af];
668
669	WARN_ON(!mutex_is_locked(&xt[af].compat_mutex));
670
671	if (WARN_ON(!xp->compat_tab))
672	return -ENOMEM;
673
674	if (xp->cur >= xp->number)
675	return -EINVAL;
676
677	if (xp->cur)
678	delta += xp->compat_tab[xp->cur - 1].delta;
679	xp->compat_tab[xp->cur].offset = offset;
680	xp->compat_tab[xp->cur].delta = delta;
681	xp->cur++;
682	return 0;
683	}
684	EXPORT_SYMBOL_GPL(xt_compat_add_offset);
685
686	void xt_compat_flush_offsets(u_int8_t af)
687	{
688	WARN_ON(!mutex_is_locked(&xt[af].compat_mutex));
689
690	if (xt[af].compat_tab) {
691	vfree(addr: xt[af].compat_tab);
692	xt[af].compat_tab = NULL;
693	xt[af].number = 0;
694	xt[af].cur = 0;
695	}
696	}
697	EXPORT_SYMBOL_GPL(xt_compat_flush_offsets);
698
699	int xt_compat_calc_jump(u_int8_t af, unsigned int offset)
700	{
701	struct compat_delta *tmp = xt[af].compat_tab;
702	int mid, left = 0, right = xt[af].cur - 1;
703
704	while (left <= right) {
705	mid = (left + right) >> 1;
706	if (offset > tmp[mid].offset)
707	left = mid + 1;
708	else if (offset < tmp[mid].offset)
709	right = mid - 1;
710	else
711	return mid ? tmp[mid - 1].delta : 0;
712	}
713	return left ? tmp[left - 1].delta : 0;
714	}
715	EXPORT_SYMBOL_GPL(xt_compat_calc_jump);
716
717	int xt_compat_init_offsets(u8 af, unsigned int number)
718	{
719	size_t mem;
720
721	WARN_ON(!mutex_is_locked(&xt[af].compat_mutex));
722
723	if (!number || number > (INT_MAX / sizeof(struct compat_delta)))
724	return -EINVAL;
725
726	if (WARN_ON(xt[af].compat_tab))
727	return -EINVAL;
728
729	mem = sizeof(struct compat_delta) * number;
730	if (mem > XT_MAX_TABLE_SIZE)
731	return -ENOMEM;
732
733	xt[af].compat_tab = vmalloc(size: mem);
734	if (!xt[af].compat_tab)
735	return -ENOMEM;
736
737	xt[af].number = number;
738	xt[af].cur = 0;
739
740	return 0;
741	}
742	EXPORT_SYMBOL(xt_compat_init_offsets);
743
744	int xt_compat_match_offset(const struct xt_match *match)
745	{
746	u_int16_t csize = match->compatsize ? : match->matchsize;
747	return XT_ALIGN(match->matchsize) - COMPAT_XT_ALIGN(csize);
748	}
749	EXPORT_SYMBOL_GPL(xt_compat_match_offset);
750
751	void xt_compat_match_from_user(struct xt_entry_match *m, void **dstptr,
752	unsigned int *size)
753	{
754	const struct xt_match *match = m->u.kernel.match;
755	struct compat_xt_entry_match *cm = (struct compat_xt_entry_match *)m;
756	int off = xt_compat_match_offset(match);
757	u_int16_t msize = cm->u.user.match_size;
758	char name[sizeof(m->u.user.name)];
759
760	m = *dstptr;
761	memcpy(m, cm, sizeof(*cm));
762	if (match->compat_from_user)
763	match->compat_from_user(m->data, cm->data);
764	else
765	memcpy(m->data, cm->data, msize - sizeof(*cm));
766
767	msize += off;
768	m->u.user.match_size = msize;
769	strscpy(p: name, q: match->name, size: sizeof(name));
770	module_put(module: match->me);
771	strscpy_pad(dest: m->u.user.name, src: name, count: sizeof(m->u.user.name));
772
773	*size += off;
774	*dstptr += msize;
775	}
776	EXPORT_SYMBOL_GPL(xt_compat_match_from_user);
777
778	#define COMPAT_XT_DATA_TO_USER(U, K, TYPE, C_SIZE) \
779	xt_data_to_user(U->data, K->data, \
780	K->u.kernel.TYPE->usersize, \
781	C_SIZE, \
782	COMPAT_XT_ALIGN(C_SIZE))
783
784	int xt_compat_match_to_user(const struct xt_entry_match *m,
785	void __user **dstptr, unsigned int *size)
786	{
787	const struct xt_match *match = m->u.kernel.match;
788	struct compat_xt_entry_match __user *cm = *dstptr;
789	int off = xt_compat_match_offset(match);
790	u_int16_t msize = m->u.user.match_size - off;
791
792	if (XT_OBJ_TO_USER(cm, m, match, msize))
793	return -EFAULT;
794
795	if (match->compat_to_user) {
796	if (match->compat_to_user((void __user *)cm->data, m->data))
797	return -EFAULT;
798	} else {
799	if (COMPAT_XT_DATA_TO_USER(cm, m, match, msize - sizeof(*cm)))
800	return -EFAULT;
801	}
802
803	*size -= off;
804	*dstptr += msize;
805	return 0;
806	}
807	EXPORT_SYMBOL_GPL(xt_compat_match_to_user);
808
809	/* non-compat version may have padding after verdict */
810	struct compat_xt_standard_target {
811	struct compat_xt_entry_target t;
812	compat_uint_t verdict;
813	};
814
815	struct compat_xt_error_target {
816	struct compat_xt_entry_target t;
817	char errorname[XT_FUNCTION_MAXNAMELEN];
818	};
819
820	int xt_compat_check_entry_offsets(const void *base, const char *elems,
821	unsigned int target_offset,
822	unsigned int next_offset)
823	{
824	long size_of_base_struct = elems - (const char *)base;
825	const struct compat_xt_entry_target *t;
826	const char *e = base;
827
828	if (target_offset < size_of_base_struct)
829	return -EINVAL;
830
831	if (target_offset + sizeof(*t) > next_offset)
832	return -EINVAL;
833
834	t = (void *)(e + target_offset);
835	if (t->u.target_size < sizeof(*t))
836	return -EINVAL;
837
838	if (target_offset + t->u.target_size > next_offset)
839	return -EINVAL;
840
841	if (strcmp(t->u.user.name, XT_STANDARD_TARGET) == 0) {
842	const struct compat_xt_standard_target *st = (const void *)t;
843
844	if (COMPAT_XT_ALIGN(target_offset + sizeof(*st)) != next_offset)
845	return -EINVAL;
846
847	if (!verdict_ok(verdict: st->verdict))
848	return -EINVAL;
849	} else if (strcmp(t->u.user.name, XT_ERROR_TARGET) == 0) {
850	const struct compat_xt_error_target *et = (const void *)t;
851
852	if (!error_tg_ok(usersize: t->u.target_size, kernsize: sizeof(*et),
853	msg: et->errorname, msglen: sizeof(et->errorname)))
854	return -EINVAL;
855	}
856
857	/* compat_xt_entry match has less strict alignment requirements,
858	* otherwise they are identical. In case of padding differences
859	* we need to add compat version of xt_check_entry_match.
860	*/
861	BUILD_BUG_ON(sizeof(struct compat_xt_entry_match) != sizeof(struct xt_entry_match));
862
863	return xt_check_entry_match(match: elems, target: base + target_offset,
864	alignment: __alignof__(struct compat_xt_entry_match));
865	}
866	EXPORT_SYMBOL(xt_compat_check_entry_offsets);
867	#endif /* CONFIG_NETFILTER_XTABLES_COMPAT */
868
869	/**
870	* xt_check_entry_offsets - validate arp/ip/ip6t_entry
871	*
872	* @base: pointer to arp/ip/ip6t_entry
873	* @elems: pointer to first xt_entry_match, i.e. ip(6)t_entry->elems
874	* @target_offset: the arp/ip/ip6_t->target_offset
875	* @next_offset: the arp/ip/ip6_t->next_offset
876	*
877	* validates that target_offset and next_offset are sane and that all
878	* match sizes (if any) align with the target offset.
879	*
880	* This function does not validate the targets or matches themselves, it
881	* only tests that all the offsets and sizes are correct, that all
882	* match structures are aligned, and that the last structure ends where
883	* the target structure begins.
884	*
885	* Also see xt_compat_check_entry_offsets for CONFIG_NETFILTER_XTABLES_COMPAT version.
886	*
887	* The arp/ip/ip6t_entry structure @base must have passed following tests:
888	* - it must point to a valid memory location
889	* - base to base + next_offset must be accessible, i.e. not exceed allocated
890	* length.
891	*
892	* A well-formed entry looks like this:
893	*
894	* ip(6)t_entry match [mtdata] match [mtdata] target [tgdata] ip(6)t_entry
895	* e->elems[]-----' | |
896	* matchsize | |
897	* matchsize | |
898	* | |
899	* target_offset---------------------------------' |
900	* next_offset---------------------------------------------------'
901	*
902	* elems[]: flexible array member at end of ip(6)/arpt_entry struct.
903	* This is where matches (if any) and the target reside.
904	* target_offset: beginning of target.
905	* next_offset: start of the next rule; also: size of this rule.
906	* Since targets have a minimum size, target_offset + minlen <= next_offset.
907	*
908	* Every match stores its size, sum of sizes must not exceed target_offset.
909	*
910	* Return: 0 on success, negative errno on failure.
911	*/
912	int xt_check_entry_offsets(const void *base,
913	const char *elems,
914	unsigned int target_offset,
915	unsigned int next_offset)
916	{
917	long size_of_base_struct = elems - (const char *)base;
918	const struct xt_entry_target *t;
919	const char *e = base;
920
921	/* target start is within the ip/ip6/arpt_entry struct */
922	if (target_offset < size_of_base_struct)
923	return -EINVAL;
924
925	if (target_offset + sizeof(*t) > next_offset)
926	return -EINVAL;
927
928	t = (void *)(e + target_offset);
929	if (t->u.target_size < sizeof(*t))
930	return -EINVAL;
931
932	if (target_offset + t->u.target_size > next_offset)
933	return -EINVAL;
934
935	if (strcmp(t->u.user.name, XT_STANDARD_TARGET) == 0) {
936	const struct xt_standard_target *st = (const void *)t;
937
938	if (XT_ALIGN(target_offset + sizeof(*st)) != next_offset)
939	return -EINVAL;
940
941	if (!verdict_ok(verdict: st->verdict))
942	return -EINVAL;
943	} else if (strcmp(t->u.user.name, XT_ERROR_TARGET) == 0) {
944	const struct xt_error_target *et = (const void *)t;
945
946	if (!error_tg_ok(usersize: t->u.target_size, kernsize: sizeof(*et),
947	msg: et->errorname, msglen: sizeof(et->errorname)))
948	return -EINVAL;
949	}
950
951	return xt_check_entry_match(match: elems, target: base + target_offset,
952	alignment: __alignof__(struct xt_entry_match));
953	}
954	EXPORT_SYMBOL(xt_check_entry_offsets);
955
956	/**
957	* xt_alloc_entry_offsets - allocate array to store rule head offsets
958	*
959	* @size: number of entries
960	*
961	* Return: NULL or zeroed kmalloc'd or vmalloc'd array
962	*/
963	unsigned int *xt_alloc_entry_offsets(unsigned int size)
964	{
965	if (size > XT_MAX_TABLE_SIZE / sizeof(unsigned int))
966	return NULL;
967
968	return kvcalloc(n: size, size: sizeof(unsigned int), GFP_KERNEL);
969
970	}
971	EXPORT_SYMBOL(xt_alloc_entry_offsets);
972
973	/**
974	* xt_find_jump_offset - check if target is a valid jump offset
975	*
976	* @offsets: array containing all valid rule start offsets of a rule blob
977	* @target: the jump target to search for
978	* @size: entries in @offset
979	*/
980	bool xt_find_jump_offset(const unsigned int *offsets,
981	unsigned int target, unsigned int size)
982	{
983	int m, low = 0, hi = size;
984
985	while (hi > low) {
986	m = (low + hi) / 2u;
987
988	if (offsets[m] > target)
989	hi = m;
990	else if (offsets[m] < target)
991	low = m + 1;
992	else
993	return true;
994	}
995
996	return false;
997	}
998	EXPORT_SYMBOL(xt_find_jump_offset);
999
1000	int xt_check_target(struct xt_tgchk_param *par,
1001	unsigned int size, u16 proto, bool inv_proto)
1002	{
1003	int ret;
1004
1005	if (XT_ALIGN(par->target->targetsize) != size) {
1006	pr_err_ratelimited("%s_tables: %s.%u target: invalid size %u (kernel) != (user) %u\n",
1007	xt_prefix[par->family], par->target->name,
1008	par->target->revision,
1009	XT_ALIGN(par->target->targetsize), size);
1010	return -EINVAL;
1011	}
1012	if (par->target->table != NULL &&
1013	strcmp(par->target->table, par->table) != 0) {
1014	pr_info_ratelimited("%s_tables: %s target: only valid in %s table, not %s\n",
1015	xt_prefix[par->family], par->target->name,
1016	par->target->table, par->table);
1017	return -EINVAL;
1018	}
1019	if (par->target->hooks && (par->hook_mask & ~par->target->hooks) != 0) {
1020	char used[64], allow[64];
1021
1022	pr_info_ratelimited("%s_tables: %s target: used from hooks %s, but only usable from %s\n",
1023	xt_prefix[par->family], par->target->name,
1024	textify_hooks(used, sizeof(used),
1025	par->hook_mask, par->family),
1026	textify_hooks(allow, sizeof(allow),
1027	par->target->hooks,
1028	par->family));
1029	return -EINVAL;
1030	}
1031	if (par->target->proto && (par->target->proto != proto || inv_proto)) {
1032	pr_info_ratelimited("%s_tables: %s target: only valid for protocol %u\n",
1033	xt_prefix[par->family], par->target->name,
1034	par->target->proto);
1035	return -EINVAL;
1036	}
1037	if (par->target->checkentry != NULL) {
1038	ret = par->target->checkentry(par);
1039	if (ret < 0)
1040	return ret;
1041	else if (ret > 0)
1042	/* Flag up potential errors. */
1043	return -EIO;
1044	}
1045	return 0;
1046	}
1047	EXPORT_SYMBOL_GPL(xt_check_target);
1048
1049	/**
1050	* xt_copy_counters - copy counters and metadata from a sockptr_t
1051	*
1052	* @arg: src sockptr
1053	* @len: alleged size of userspace memory
1054	* @info: where to store the xt_counters_info metadata
1055	*
1056	* Copies counter meta data from @user and stores it in @info.
1057	*
1058	* vmallocs memory to hold the counters, then copies the counter data
1059	* from @user to the new memory and returns a pointer to it.
1060	*
1061	* If called from a compat syscall, @info gets converted automatically to the
1062	* 64bit representation.
1063	*
1064	* The metadata associated with the counters is stored in @info.
1065	*
1066	* Return: returns pointer that caller has to test via IS_ERR().
1067	* If IS_ERR is false, caller has to vfree the pointer.
1068	*/
1069	void *xt_copy_counters(sockptr_t arg, unsigned int len,
1070	struct xt_counters_info *info)
1071	{
1072	size_t offset;
1073	void *mem;
1074	u64 size;
1075
1076	#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
1077	if (in_compat_syscall()) {
1078	/* structures only differ in size due to alignment */
1079	struct compat_xt_counters_info compat_tmp;
1080
1081	if (len <= sizeof(compat_tmp))
1082	return ERR_PTR(error: -EINVAL);
1083
1084	len -= sizeof(compat_tmp);
1085	if (copy_from_sockptr(dst: &compat_tmp, src: arg, size: sizeof(compat_tmp)) != 0)
1086	return ERR_PTR(error: -EFAULT);
1087
1088	memcpy(info->name, compat_tmp.name, sizeof(info->name) - 1);
1089	info->num_counters = compat_tmp.num_counters;
1090	offset = sizeof(compat_tmp);
1091	} else
1092	#endif
1093	{
1094	if (len <= sizeof(*info))
1095	return ERR_PTR(error: -EINVAL);
1096
1097	len -= sizeof(*info);
1098	if (copy_from_sockptr(dst: info, src: arg, size: sizeof(*info)) != 0)
1099	return ERR_PTR(error: -EFAULT);
1100
1101	offset = sizeof(*info);
1102	}
1103	info->name[sizeof(info->name) - 1] = '\0';
1104
1105	size = sizeof(struct xt_counters);
1106	size *= info->num_counters;
1107
1108	if (size != (u64)len)
1109	return ERR_PTR(error: -EINVAL);
1110
1111	mem = vmalloc(size: len);
1112	if (!mem)
1113	return ERR_PTR(error: -ENOMEM);
1114
1115	if (copy_from_sockptr_offset(dst: mem, src: arg, offset, size: len) == 0)
1116	return mem;
1117
1118	vfree(addr: mem);
1119	return ERR_PTR(error: -EFAULT);
1120	}
1121	EXPORT_SYMBOL_GPL(xt_copy_counters);
1122
1123	#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
1124	int xt_compat_target_offset(const struct xt_target *target)
1125	{
1126	u_int16_t csize = target->compatsize ? : target->targetsize;
1127	return XT_ALIGN(target->targetsize) - COMPAT_XT_ALIGN(csize);
1128	}
1129	EXPORT_SYMBOL_GPL(xt_compat_target_offset);
1130
1131	void xt_compat_target_from_user(struct xt_entry_target *t, void **dstptr,
1132	unsigned int *size)
1133	{
1134	const struct xt_target *target = t->u.kernel.target;
1135	struct compat_xt_entry_target *ct = (struct compat_xt_entry_target *)t;
1136	int off = xt_compat_target_offset(target);
1137	u_int16_t tsize = ct->u.user.target_size;
1138	char name[sizeof(t->u.user.name)];
1139
1140	t = *dstptr;
1141	memcpy(t, ct, sizeof(*ct));
1142	if (target->compat_from_user)
1143	target->compat_from_user(t->data, ct->data);
1144	else
1145	memcpy(t->data, ct->data, tsize - sizeof(*ct));
1146
1147	tsize += off;
1148	t->u.user.target_size = tsize;
1149	strscpy(p: name, q: target->name, size: sizeof(name));
1150	module_put(module: target->me);
1151	strscpy_pad(dest: t->u.user.name, src: name, count: sizeof(t->u.user.name));
1152
1153	*size += off;
1154	*dstptr += tsize;
1155	}
1156	EXPORT_SYMBOL_GPL(xt_compat_target_from_user);
1157
1158	int xt_compat_target_to_user(const struct xt_entry_target *t,
1159	void __user **dstptr, unsigned int *size)
1160	{
1161	const struct xt_target *target = t->u.kernel.target;
1162	struct compat_xt_entry_target __user *ct = *dstptr;
1163	int off = xt_compat_target_offset(target);
1164	u_int16_t tsize = t->u.user.target_size - off;
1165
1166	if (XT_OBJ_TO_USER(ct, t, target, tsize))
1167	return -EFAULT;
1168
1169	if (target->compat_to_user) {
1170	if (target->compat_to_user((void __user *)ct->data, t->data))
1171	return -EFAULT;
1172	} else {
1173	if (COMPAT_XT_DATA_TO_USER(ct, t, target, tsize - sizeof(*ct)))
1174	return -EFAULT;
1175	}
1176
1177	*size -= off;
1178	*dstptr += tsize;
1179	return 0;
1180	}
1181	EXPORT_SYMBOL_GPL(xt_compat_target_to_user);
1182	#endif
1183
1184	struct xt_table_info *xt_alloc_table_info(unsigned int size)
1185	{
1186	struct xt_table_info *info = NULL;
1187	size_t sz = sizeof(*info) + size;
1188
1189	if (sz < sizeof(*info) || sz >= XT_MAX_TABLE_SIZE)
1190	return NULL;
1191
1192	info = kvmalloc(size: sz, GFP_KERNEL_ACCOUNT);
1193	if (!info)
1194	return NULL;
1195
1196	memset(info, 0, sizeof(*info));
1197	info->size = size;
1198	return info;
1199	}
1200	EXPORT_SYMBOL(xt_alloc_table_info);
1201
1202	void xt_free_table_info(struct xt_table_info *info)
1203	{
1204	int cpu;
1205
1206	if (info->jumpstack != NULL) {
1207	for_each_possible_cpu(cpu)
1208	kvfree(addr: info->jumpstack[cpu]);
1209	kvfree(addr: info->jumpstack);
1210	}
1211
1212	kvfree(addr: info);
1213	}
1214	EXPORT_SYMBOL(xt_free_table_info);
1215
1216	struct xt_table *xt_find_table(struct net *net, u8 af, const char *name)
1217	{
1218	struct xt_pernet *xt_net = net_generic(net, id: xt_pernet_id);
1219	struct xt_table *t;
1220
1221	mutex_lock(&xt[af].mutex);
1222	list_for_each_entry(t, &xt_net->tables[af], list) {
1223	if (strcmp(t->name, name) == 0) {
1224	mutex_unlock(lock: &xt[af].mutex);
1225	return t;
1226	}
1227	}
1228	mutex_unlock(lock: &xt[af].mutex);
1229	return NULL;
1230	}
1231	EXPORT_SYMBOL(xt_find_table);
1232
1233	/* Find table by name, grabs mutex & ref. Returns ERR_PTR on error. */
1234	struct xt_table *xt_find_table_lock(struct net *net, u_int8_t af,
1235	const char *name)
1236	{
1237	struct xt_pernet *xt_net = net_generic(net, id: xt_pernet_id);
1238	struct module *owner = NULL;
1239	struct xt_template *tmpl;
1240	struct xt_table *t;
1241
1242	mutex_lock(&xt[af].mutex);
1243	list_for_each_entry(t, &xt_net->tables[af], list)
1244	if (strcmp(t->name, name) == 0 && try_module_get(module: t->me))
1245	return t;
1246
1247	/* Table doesn't exist in this netns, check larval list */
1248	list_for_each_entry(tmpl, &xt_templates[af], list) {
1249	int err;
1250
1251	if (strcmp(tmpl->name, name))
1252	continue;
1253	if (!try_module_get(module: tmpl->me))
1254	goto out;
1255
1256	owner = tmpl->me;
1257
1258	mutex_unlock(lock: &xt[af].mutex);
1259	err = tmpl->table_init(net);
1260	if (err < 0) {
1261	module_put(module: owner);
1262	return ERR_PTR(error: err);
1263	}
1264
1265	mutex_lock(&xt[af].mutex);
1266	break;
1267	}
1268
1269	/* and once again: */
1270	list_for_each_entry(t, &xt_net->tables[af], list)
1271	if (strcmp(t->name, name) == 0)
1272	return t;
1273
1274	module_put(module: owner);
1275	out:
1276	mutex_unlock(lock: &xt[af].mutex);
1277	return ERR_PTR(error: -ENOENT);
1278	}
1279	EXPORT_SYMBOL_GPL(xt_find_table_lock);
1280
1281	struct xt_table *xt_request_find_table_lock(struct net *net, u_int8_t af,
1282	const char *name)
1283	{
1284	struct xt_table *t = xt_find_table_lock(net, af, name);
1285
1286	#ifdef CONFIG_MODULES
1287	if (IS_ERR(ptr: t)) {
1288	int err = request_module("%stable_%s", xt_prefix[af], name);
1289	if (err < 0)
1290	return ERR_PTR(error: err);
1291	t = xt_find_table_lock(net, af, name);
1292	}
1293	#endif
1294
1295	return t;
1296	}
1297	EXPORT_SYMBOL_GPL(xt_request_find_table_lock);
1298
1299	void xt_table_unlock(struct xt_table *table)
1300	{
1301	mutex_unlock(lock: &xt[table->af].mutex);
1302	}
1303	EXPORT_SYMBOL_GPL(xt_table_unlock);
1304
1305	#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
1306	void xt_compat_lock(u_int8_t af)
1307	{
1308	mutex_lock(&xt[af].compat_mutex);
1309	}
1310	EXPORT_SYMBOL_GPL(xt_compat_lock);
1311
1312	void xt_compat_unlock(u_int8_t af)
1313	{
1314	mutex_unlock(lock: &xt[af].compat_mutex);
1315	}
1316	EXPORT_SYMBOL_GPL(xt_compat_unlock);
1317	#endif
1318
1319	DEFINE_PER_CPU(seqcount_t, xt_recseq);
1320	EXPORT_PER_CPU_SYMBOL_GPL(xt_recseq);
1321
1322	struct static_key xt_tee_enabled __read_mostly;
1323	EXPORT_SYMBOL_GPL(xt_tee_enabled);
1324
1325	static int xt_jumpstack_alloc(struct xt_table_info *i)
1326	{
1327	unsigned int size;
1328	int cpu;
1329
1330	size = sizeof(void **) * nr_cpu_ids;
1331	if (size > PAGE_SIZE)
1332	i->jumpstack = kvzalloc(size, GFP_KERNEL);
1333	else
1334	i->jumpstack = kzalloc(size, GFP_KERNEL);
1335	if (i->jumpstack == NULL)
1336	return -ENOMEM;
1337
1338	/* ruleset without jumps -- no stack needed */
1339	if (i->stacksize == 0)
1340	return 0;
1341
1342	/* Jumpstack needs to be able to record two full callchains, one
1343	* from the first rule set traversal, plus one table reentrancy
1344	* via -j TEE without clobbering the callchain that brought us to
1345	* TEE target.
1346	*
1347	* This is done by allocating two jumpstacks per cpu, on reentry
1348	* the upper half of the stack is used.
1349	*
1350	* see the jumpstack setup in ipt_do_table() for more details.
1351	*/
1352	size = sizeof(void *) * i->stacksize * 2u;
1353	for_each_possible_cpu(cpu) {
1354	i->jumpstack[cpu] = kvmalloc_node(size, GFP_KERNEL,
1355	cpu_to_node(cpu));
1356	if (i->jumpstack[cpu] == NULL)
1357	/*
1358	* Freeing will be done later on by the callers. The
1359	* chain is: xt_replace_table -> __do_replace ->
1360	* do_replace -> xt_free_table_info.
1361	*/
1362	return -ENOMEM;
1363	}
1364
1365	return 0;
1366	}
1367
1368	struct xt_counters *xt_counters_alloc(unsigned int counters)
1369	{
1370	struct xt_counters *mem;
1371
1372	if (counters == 0 || counters > INT_MAX / sizeof(*mem))
1373	return NULL;
1374
1375	counters *= sizeof(*mem);
1376	if (counters > XT_MAX_TABLE_SIZE)
1377	return NULL;
1378
1379	return vzalloc(size: counters);
1380	}
1381	EXPORT_SYMBOL(xt_counters_alloc);
1382
1383	struct xt_table_info *
1384	xt_replace_table(struct xt_table *table,
1385	unsigned int num_counters,
1386	struct xt_table_info *newinfo,
1387	int *error)
1388	{
1389	struct xt_table_info *private;
1390	unsigned int cpu;
1391	int ret;
1392
1393	ret = xt_jumpstack_alloc(i: newinfo);
1394	if (ret < 0) {
1395	*error = ret;
1396	return NULL;
1397	}
1398
1399	/* Do the substitution. */
1400	local_bh_disable();
1401	private = table->private;
1402
1403	/* Check inside lock: is the old number correct? */
1404	if (num_counters != private->number) {
1405	pr_debug("num_counters != table->private->number (%u/%u)\n",
1406	num_counters, private->number);
1407	local_bh_enable();
1408	*error = -EAGAIN;
1409	return NULL;
1410	}
1411
1412	newinfo->initial_entries = private->initial_entries;
1413	/*
1414	* Ensure contents of newinfo are visible before assigning to
1415	* private.
1416	*/
1417	smp_wmb();
1418	table->private = newinfo;
1419
1420	/* make sure all cpus see new ->private value */
1421	smp_mb();
1422
1423	/*
1424	* Even though table entries have now been swapped, other CPU's
1425	* may still be using the old entries...
1426	*/
1427	local_bh_enable();
1428
1429	/* ... so wait for even xt_recseq on all cpus */
1430	for_each_possible_cpu(cpu) {
1431	seqcount_t *s = &per_cpu(xt_recseq, cpu);
1432	u32 seq = raw_read_seqcount(s);
1433
1434	if (seq & 1) {
1435	do {
1436	cond_resched();
1437	cpu_relax();
1438	} while (seq == raw_read_seqcount(s));
1439	}
1440	}
1441
1442	audit_log_nfcfg(name: table->name, af: table->af, nentries: private->number,
1443	op: !private->number ? AUDIT_XT_OP_REGISTER :
1444	AUDIT_XT_OP_REPLACE,
1445	GFP_KERNEL);
1446	return private;
1447	}
1448	EXPORT_SYMBOL_GPL(xt_replace_table);
1449
1450	struct xt_table *xt_register_table(struct net *net,
1451	const struct xt_table *input_table,
1452	struct xt_table_info *bootstrap,
1453	struct xt_table_info *newinfo)
1454	{
1455	struct xt_pernet *xt_net = net_generic(net, id: xt_pernet_id);
1456	struct xt_table_info *private;
1457	struct xt_table *t, *table;
1458	int ret;
1459
1460	/* Don't add one object to multiple lists. */
1461	table = kmemdup(p: input_table, size: sizeof(struct xt_table), GFP_KERNEL);
1462	if (!table) {
1463	ret = -ENOMEM;
1464	goto out;
1465	}
1466
1467	mutex_lock(&xt[table->af].mutex);
1468	/* Don't autoload: we'd eat our tail... */
1469	list_for_each_entry(t, &xt_net->tables[table->af], list) {
1470	if (strcmp(t->name, table->name) == 0) {
1471	ret = -EEXIST;
1472	goto unlock;
1473	}
1474	}
1475
1476	/* Simplifies replace_table code. */
1477	table->private = bootstrap;
1478
1479	if (!xt_replace_table(table, 0, newinfo, &ret))
1480	goto unlock;
1481
1482	private = table->private;
1483	pr_debug("table->private->number = %u\n", private->number);
1484
1485	/* save number of initial entries */
1486	private->initial_entries = private->number;
1487
1488	list_add(new: &table->list, head: &xt_net->tables[table->af]);
1489	mutex_unlock(lock: &xt[table->af].mutex);
1490	return table;
1491
1492	unlock:
1493	mutex_unlock(lock: &xt[table->af].mutex);
1494	kfree(objp: table);
1495	out:
1496	return ERR_PTR(error: ret);
1497	}
1498	EXPORT_SYMBOL_GPL(xt_register_table);
1499
1500	void *xt_unregister_table(struct xt_table *table)
1501	{
1502	struct xt_table_info *private;
1503
1504	mutex_lock(&xt[table->af].mutex);
1505	private = table->private;
1506	list_del(entry: &table->list);
1507	mutex_unlock(lock: &xt[table->af].mutex);
1508	audit_log_nfcfg(name: table->name, af: table->af, nentries: private->number,
1509	op: AUDIT_XT_OP_UNREGISTER, GFP_KERNEL);
1510	kfree(objp: table->ops);
1511	kfree(objp: table);
1512
1513	return private;
1514	}
1515	EXPORT_SYMBOL_GPL(xt_unregister_table);
1516
1517	#ifdef CONFIG_PROC_FS
1518	static void *xt_table_seq_start(struct seq_file *seq, loff_t *pos)
1519	{
1520	u8 af = (unsigned long)pde_data(inode: file_inode(f: seq->file));
1521	struct net *net = seq_file_net(seq);
1522	struct xt_pernet *xt_net;
1523
1524	xt_net = net_generic(net, id: xt_pernet_id);
1525
1526	mutex_lock(&xt[af].mutex);
1527	return seq_list_start(head: &xt_net->tables[af], pos: *pos);
1528	}
1529
1530	static void *xt_table_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1531	{
1532	u8 af = (unsigned long)pde_data(inode: file_inode(f: seq->file));
1533	struct net *net = seq_file_net(seq);
1534	struct xt_pernet *xt_net;
1535
1536	xt_net = net_generic(net, id: xt_pernet_id);
1537
1538	return seq_list_next(v, head: &xt_net->tables[af], ppos: pos);
1539	}
1540
1541	static void xt_table_seq_stop(struct seq_file *seq, void *v)
1542	{
1543	u_int8_t af = (unsigned long)pde_data(inode: file_inode(f: seq->file));
1544
1545	mutex_unlock(lock: &xt[af].mutex);
1546	}
1547
1548	static int xt_table_seq_show(struct seq_file *seq, void *v)
1549	{
1550	struct xt_table *table = list_entry(v, struct xt_table, list);
1551
1552	if (*table->name)
1553	seq_printf(m: seq, fmt: "%s\n", table->name);
1554	return 0;
1555	}
1556
1557	static const struct seq_operations xt_table_seq_ops = {
1558	.start = xt_table_seq_start,
1559	.next = xt_table_seq_next,
1560	.stop = xt_table_seq_stop,
1561	.show = xt_table_seq_show,
1562	};
1563
1564	/*
1565	* Traverse state for ip{,6}_{tables,matches} for helping crossing
1566	* the multi-AF mutexes.
1567	*/
1568	struct nf_mttg_trav {
1569	struct list_head *head, *curr;
1570	uint8_t class;
1571	};
1572
1573	enum {
1574	MTTG_TRAV_INIT,
1575	MTTG_TRAV_NFP_UNSPEC,
1576	MTTG_TRAV_NFP_SPEC,
1577	MTTG_TRAV_DONE,
1578	};
1579
1580	static void *xt_mttg_seq_next(struct seq_file *seq, void *v, loff_t *ppos,
1581	bool is_target)
1582	{
1583	static const uint8_t next_class[] = {
1584	[MTTG_TRAV_NFP_UNSPEC] = MTTG_TRAV_NFP_SPEC,
1585	[MTTG_TRAV_NFP_SPEC] = MTTG_TRAV_DONE,
1586	};
1587	uint8_t nfproto = (unsigned long)pde_data(inode: file_inode(f: seq->file));
1588	struct nf_mttg_trav *trav = seq->private;
1589
1590	if (ppos != NULL)
1591	++(*ppos);
1592
1593	switch (trav->class) {
1594	case MTTG_TRAV_INIT:
1595	trav->class = MTTG_TRAV_NFP_UNSPEC;
1596	mutex_lock(&xt[NFPROTO_UNSPEC].mutex);
1597	trav->head = trav->curr = is_target ?
1598	&xt[NFPROTO_UNSPEC].target : &xt[NFPROTO_UNSPEC].match;
1599	break;
1600	case MTTG_TRAV_NFP_UNSPEC:
1601	trav->curr = trav->curr->next;
1602	if (trav->curr != trav->head)
1603	break;
1604	mutex_unlock(lock: &xt[NFPROTO_UNSPEC].mutex);
1605	mutex_lock(&xt[nfproto].mutex);
1606	trav->head = trav->curr = is_target ?
1607	&xt[nfproto].target : &xt[nfproto].match;
1608	trav->class = next_class[trav->class];
1609	break;
1610	case MTTG_TRAV_NFP_SPEC:
1611	trav->curr = trav->curr->next;
1612	if (trav->curr != trav->head)
1613	break;
1614	fallthrough;
1615	default:
1616	return NULL;
1617	}
1618	return trav;
1619	}
1620
1621	static void *xt_mttg_seq_start(struct seq_file *seq, loff_t *pos,
1622	bool is_target)
1623	{
1624	struct nf_mttg_trav *trav = seq->private;
1625	unsigned int j;
1626
1627	trav->class = MTTG_TRAV_INIT;
1628	for (j = 0; j < *pos; ++j)
1629	if (xt_mttg_seq_next(seq, NULL, NULL, is_target) == NULL)
1630	return NULL;
1631	return trav;
1632	}
1633
1634	static void xt_mttg_seq_stop(struct seq_file *seq, void *v)
1635	{
1636	uint8_t nfproto = (unsigned long)pde_data(inode: file_inode(f: seq->file));
1637	struct nf_mttg_trav *trav = seq->private;
1638
1639	switch (trav->class) {
1640	case MTTG_TRAV_NFP_UNSPEC:
1641	mutex_unlock(lock: &xt[NFPROTO_UNSPEC].mutex);
1642	break;
1643	case MTTG_TRAV_NFP_SPEC:
1644	mutex_unlock(lock: &xt[nfproto].mutex);
1645	break;
1646	}
1647	}
1648
1649	static void *xt_match_seq_start(struct seq_file *seq, loff_t *pos)
1650	{
1651	return xt_mttg_seq_start(seq, pos, is_target: false);
1652	}
1653
1654	static void *xt_match_seq_next(struct seq_file *seq, void *v, loff_t *ppos)
1655	{
1656	return xt_mttg_seq_next(seq, v, ppos, is_target: false);
1657	}
1658
1659	static int xt_match_seq_show(struct seq_file *seq, void *v)
1660	{
1661	const struct nf_mttg_trav *trav = seq->private;
1662	const struct xt_match *match;
1663
1664	switch (trav->class) {
1665	case MTTG_TRAV_NFP_UNSPEC:
1666	case MTTG_TRAV_NFP_SPEC:
1667	if (trav->curr == trav->head)
1668	return 0;
1669	match = list_entry(trav->curr, struct xt_match, list);
1670	if (*match->name)
1671	seq_printf(m: seq, fmt: "%s\n", match->name);
1672	}
1673	return 0;
1674	}
1675
1676	static const struct seq_operations xt_match_seq_ops = {
1677	.start = xt_match_seq_start,
1678	.next = xt_match_seq_next,
1679	.stop = xt_mttg_seq_stop,
1680	.show = xt_match_seq_show,
1681	};
1682
1683	static void *xt_target_seq_start(struct seq_file *seq, loff_t *pos)
1684	{
1685	return xt_mttg_seq_start(seq, pos, is_target: true);
1686	}
1687
1688	static void *xt_target_seq_next(struct seq_file *seq, void *v, loff_t *ppos)
1689	{
1690	return xt_mttg_seq_next(seq, v, ppos, is_target: true);
1691	}
1692
1693	static int xt_target_seq_show(struct seq_file *seq, void *v)
1694	{
1695	const struct nf_mttg_trav *trav = seq->private;
1696	const struct xt_target *target;
1697
1698	switch (trav->class) {
1699	case MTTG_TRAV_NFP_UNSPEC:
1700	case MTTG_TRAV_NFP_SPEC:
1701	if (trav->curr == trav->head)
1702	return 0;
1703	target = list_entry(trav->curr, struct xt_target, list);
1704	if (*target->name)
1705	seq_printf(m: seq, fmt: "%s\n", target->name);
1706	}
1707	return 0;
1708	}
1709
1710	static const struct seq_operations xt_target_seq_ops = {
1711	.start = xt_target_seq_start,
1712	.next = xt_target_seq_next,
1713	.stop = xt_mttg_seq_stop,
1714	.show = xt_target_seq_show,
1715	};
1716
1717	#define FORMAT_TABLES "_tables_names"
1718	#define FORMAT_MATCHES "_tables_matches"
1719	#define FORMAT_TARGETS "_tables_targets"
1720
1721	#endif /* CONFIG_PROC_FS */
1722
1723	/**
1724	* xt_hook_ops_alloc - set up hooks for a new table
1725	* @table: table with metadata needed to set up hooks
1726	* @fn: Hook function
1727	*
1728	* This function will create the nf_hook_ops that the x_table needs
1729	* to hand to xt_hook_link_net().
1730	*/
1731	struct nf_hook_ops *
1732	xt_hook_ops_alloc(const struct xt_table *table, nf_hookfn *fn)
1733	{
1734	unsigned int hook_mask = table->valid_hooks;
1735	uint8_t i, num_hooks = hweight32(hook_mask);
1736	uint8_t hooknum;
1737	struct nf_hook_ops *ops;
1738
1739	if (!num_hooks)
1740	return ERR_PTR(error: -EINVAL);
1741
1742	ops = kcalloc(n: num_hooks, size: sizeof(*ops), GFP_KERNEL);
1743	if (ops == NULL)
1744	return ERR_PTR(error: -ENOMEM);
1745
1746	for (i = 0, hooknum = 0; i < num_hooks && hook_mask != 0;
1747	hook_mask >>= 1, ++hooknum) {
1748	if (!(hook_mask & 1))
1749	continue;
1750	ops[i].hook = fn;
1751	ops[i].pf = table->af;
1752	ops[i].hooknum = hooknum;
1753	ops[i].priority = table->priority;
1754	++i;
1755	}
1756
1757	return ops;
1758	}
1759	EXPORT_SYMBOL_GPL(xt_hook_ops_alloc);
1760
1761	int xt_register_template(const struct xt_table *table,
1762	int (*table_init)(struct net *net))
1763	{
1764	int ret = -EEXIST, af = table->af;
1765	struct xt_template *t;
1766
1767	mutex_lock(&xt[af].mutex);
1768
1769	list_for_each_entry(t, &xt_templates[af], list) {
1770	if (WARN_ON_ONCE(strcmp(table->name, t->name) == 0))
1771	goto out_unlock;
1772	}
1773
1774	ret = -ENOMEM;
1775	t = kzalloc(size: sizeof(*t), GFP_KERNEL);
1776	if (!t)
1777	goto out_unlock;
1778
1779	BUILD_BUG_ON(sizeof(t->name) != sizeof(table->name));
1780
1781	strscpy(p: t->name, q: table->name, size: sizeof(t->name));
1782	t->table_init = table_init;
1783	t->me = table->me;
1784	list_add(new: &t->list, head: &xt_templates[af]);
1785	ret = 0;
1786	out_unlock:
1787	mutex_unlock(lock: &xt[af].mutex);
1788	return ret;
1789	}
1790	EXPORT_SYMBOL_GPL(xt_register_template);
1791
1792	void xt_unregister_template(const struct xt_table *table)
1793	{
1794	struct xt_template *t;
1795	int af = table->af;
1796
1797	mutex_lock(&xt[af].mutex);
1798	list_for_each_entry(t, &xt_templates[af], list) {
1799	if (strcmp(table->name, t->name))
1800	continue;
1801
1802	list_del(entry: &t->list);
1803	mutex_unlock(lock: &xt[af].mutex);
1804	kfree(objp: t);
1805	return;
1806	}
1807
1808	mutex_unlock(lock: &xt[af].mutex);
1809	WARN_ON_ONCE(1);
1810	}
1811	EXPORT_SYMBOL_GPL(xt_unregister_template);
1812
1813	int xt_proto_init(struct net *net, u_int8_t af)
1814	{
1815	#ifdef CONFIG_PROC_FS
1816	char buf[XT_FUNCTION_MAXNAMELEN];
1817	struct proc_dir_entry *proc;
1818	kuid_t root_uid;
1819	kgid_t root_gid;
1820	#endif
1821
1822	if (af >= ARRAY_SIZE(xt_prefix))
1823	return -EINVAL;
1824
1825
1826	#ifdef CONFIG_PROC_FS
1827	root_uid = make_kuid(from: net->user_ns, uid: 0);
1828	root_gid = make_kgid(from: net->user_ns, gid: 0);
1829
1830	strscpy(p: buf, q: xt_prefix[af], size: sizeof(buf));
1831	strlcat(p: buf, FORMAT_TABLES, avail: sizeof(buf));
1832	proc = proc_create_net_data(name: buf, mode: 0440, parent: net->proc_net, ops: &xt_table_seq_ops,
1833	state_size: sizeof(struct seq_net_private),
1834	data: (void *)(unsigned long)af);
1835	if (!proc)
1836	goto out;
1837	if (uid_valid(uid: root_uid) && gid_valid(gid: root_gid))
1838	proc_set_user(proc, root_uid, root_gid);
1839
1840	strscpy(p: buf, q: xt_prefix[af], size: sizeof(buf));
1841	strlcat(p: buf, FORMAT_MATCHES, avail: sizeof(buf));
1842	proc = proc_create_seq_private(name: buf, mode: 0440, parent: net->proc_net,
1843	ops: &xt_match_seq_ops, state_size: sizeof(struct nf_mttg_trav),
1844	data: (void *)(unsigned long)af);
1845	if (!proc)
1846	goto out_remove_tables;
1847	if (uid_valid(uid: root_uid) && gid_valid(gid: root_gid))
1848	proc_set_user(proc, root_uid, root_gid);
1849
1850	strscpy(p: buf, q: xt_prefix[af], size: sizeof(buf));
1851	strlcat(p: buf, FORMAT_TARGETS, avail: sizeof(buf));
1852	proc = proc_create_seq_private(name: buf, mode: 0440, parent: net->proc_net,
1853	ops: &xt_target_seq_ops, state_size: sizeof(struct nf_mttg_trav),
1854	data: (void *)(unsigned long)af);
1855	if (!proc)
1856	goto out_remove_matches;
1857	if (uid_valid(uid: root_uid) && gid_valid(gid: root_gid))
1858	proc_set_user(proc, root_uid, root_gid);
1859	#endif
1860
1861	return 0;
1862
1863	#ifdef CONFIG_PROC_FS
1864	out_remove_matches:
1865	strscpy(p: buf, q: xt_prefix[af], size: sizeof(buf));
1866	strlcat(p: buf, FORMAT_MATCHES, avail: sizeof(buf));
1867	remove_proc_entry(buf, net->proc_net);
1868
1869	out_remove_tables:
1870	strscpy(p: buf, q: xt_prefix[af], size: sizeof(buf));
1871	strlcat(p: buf, FORMAT_TABLES, avail: sizeof(buf));
1872	remove_proc_entry(buf, net->proc_net);
1873	out:
1874	return -1;
1875	#endif
1876	}
1877	EXPORT_SYMBOL_GPL(xt_proto_init);
1878
1879	void xt_proto_fini(struct net *net, u_int8_t af)
1880	{
1881	#ifdef CONFIG_PROC_FS
1882	char buf[XT_FUNCTION_MAXNAMELEN];
1883
1884	strscpy(p: buf, q: xt_prefix[af], size: sizeof(buf));
1885	strlcat(p: buf, FORMAT_TABLES, avail: sizeof(buf));
1886	remove_proc_entry(buf, net->proc_net);
1887
1888	strscpy(p: buf, q: xt_prefix[af], size: sizeof(buf));
1889	strlcat(p: buf, FORMAT_TARGETS, avail: sizeof(buf));
1890	remove_proc_entry(buf, net->proc_net);
1891
1892	strscpy(p: buf, q: xt_prefix[af], size: sizeof(buf));
1893	strlcat(p: buf, FORMAT_MATCHES, avail: sizeof(buf));
1894	remove_proc_entry(buf, net->proc_net);
1895	#endif /*CONFIG_PROC_FS*/
1896	}
1897	EXPORT_SYMBOL_GPL(xt_proto_fini);
1898
1899	/**
1900	* xt_percpu_counter_alloc - allocate x_tables rule counter
1901	*
1902	* @state: pointer to xt_percpu allocation state
1903	* @counter: pointer to counter struct inside the ip(6)/arpt_entry struct
1904	*
1905	* On SMP, the packet counter [ ip(6)t_entry->counters.pcnt ] will then
1906	* contain the address of the real (percpu) counter.
1907	*
1908	* Rule evaluation needs to use xt_get_this_cpu_counter() helper
1909	* to fetch the real percpu counter.
1910	*
1911	* To speed up allocation and improve data locality, a 4kb block is
1912	* allocated. Freeing any counter may free an entire block, so all
1913	* counters allocated using the same state must be freed at the same
1914	* time.
1915	*
1916	* xt_percpu_counter_alloc_state contains the base address of the
1917	* allocated page and the current sub-offset.
1918	*
1919	* returns false on error.
1920	*/
1921	bool xt_percpu_counter_alloc(struct xt_percpu_counter_alloc_state *state,
1922	struct xt_counters *counter)
1923	{
1924	BUILD_BUG_ON(XT_PCPU_BLOCK_SIZE < (sizeof(*counter) * 2));
1925
1926	if (nr_cpu_ids <= 1)
1927	return true;
1928
1929	if (!state->mem) {
1930	state->mem = __alloc_percpu(XT_PCPU_BLOCK_SIZE,
1931	XT_PCPU_BLOCK_SIZE);
1932	if (!state->mem)
1933	return false;
1934	}
1935	counter->pcnt = (__force unsigned long)(state->mem + state->off);
1936	state->off += sizeof(*counter);
1937	if (state->off > (XT_PCPU_BLOCK_SIZE - sizeof(*counter))) {
1938	state->mem = NULL;
1939	state->off = 0;
1940	}
1941	return true;
1942	}
1943	EXPORT_SYMBOL_GPL(xt_percpu_counter_alloc);
1944
1945	void xt_percpu_counter_free(struct xt_counters *counters)
1946	{
1947	unsigned long pcnt = counters->pcnt;
1948
1949	if (nr_cpu_ids > 1 && (pcnt & (XT_PCPU_BLOCK_SIZE - 1)) == 0)
1950	free_percpu(pdata: (void __percpu *)pcnt);
1951	}
1952	EXPORT_SYMBOL_GPL(xt_percpu_counter_free);
1953
1954	static int __net_init xt_net_init(struct net *net)
1955	{
1956	struct xt_pernet *xt_net = net_generic(net, id: xt_pernet_id);
1957	int i;
1958
1959	for (i = 0; i < NFPROTO_NUMPROTO; i++)
1960	INIT_LIST_HEAD(list: &xt_net->tables[i]);
1961	return 0;
1962	}
1963
1964	static void __net_exit xt_net_exit(struct net *net)
1965	{
1966	struct xt_pernet *xt_net = net_generic(net, id: xt_pernet_id);
1967	int i;
1968
1969	for (i = 0; i < NFPROTO_NUMPROTO; i++)
1970	WARN_ON_ONCE(!list_empty(&xt_net->tables[i]));
1971	}
1972
1973	static struct pernet_operations xt_net_ops = {
1974	.init = xt_net_init,
1975	.exit = xt_net_exit,
1976	.id = &xt_pernet_id,
1977	.size = sizeof(struct xt_pernet),
1978	};
1979
1980	static int __init xt_init(void)
1981	{
1982	unsigned int i;
1983	int rv;
1984
1985	for_each_possible_cpu(i) {
1986	seqcount_init(&per_cpu(xt_recseq, i));
1987	}
1988
1989	xt = kcalloc(n: NFPROTO_NUMPROTO, size: sizeof(struct xt_af), GFP_KERNEL);
1990	if (!xt)
1991	return -ENOMEM;
1992
1993	for (i = 0; i < NFPROTO_NUMPROTO; i++) {
1994	mutex_init(&xt[i].mutex);
1995	#ifdef CONFIG_NETFILTER_XTABLES_COMPAT
1996	mutex_init(&xt[i].compat_mutex);
1997	xt[i].compat_tab = NULL;
1998	#endif
1999	INIT_LIST_HEAD(list: &xt[i].target);
2000	INIT_LIST_HEAD(list: &xt[i].match);
2001	INIT_LIST_HEAD(list: &xt_templates[i]);
2002	}
2003	rv = register_pernet_subsys(&xt_net_ops);
2004	if (rv < 0)
2005	kfree(objp: xt);
2006	return rv;
2007	}
2008
2009	static void __exit xt_fini(void)
2010	{
2011	unregister_pernet_subsys(&xt_net_ops);
2012	kfree(objp: xt);
2013	}
2014
2015	module_init(xt_init);
2016	module_exit(xt_fini);
2017