1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* xfrm_state.c
4	*
5	* Changes:
6	* Mitsuru KANDA @USAGI
7	* Kazunori MIYAZAWA @USAGI
8	* Kunihiro Ishiguro <kunihiro@ipinfusion.com>
9	* IPv6 support
10	* YOSHIFUJI Hideaki @USAGI
11	* Split up af-specific functions
12	* Derek Atkins <derek@ihtfp.com>
13	* Add UDP Encapsulation
14	*
15	*/
16
17	#include <linux/compat.h>
18	#include <linux/workqueue.h>
19	#include <net/xfrm.h>
20	#include <linux/pfkeyv2.h>
21	#include <linux/ipsec.h>
22	#include <linux/module.h>
23	#include <linux/cache.h>
24	#include <linux/audit.h>
25	#include <linux/uaccess.h>
26	#include <linux/ktime.h>
27	#include <linux/slab.h>
28	#include <linux/interrupt.h>
29	#include <linux/kernel.h>
30
31	#include <crypto/aead.h>
32
33	#include "xfrm_hash.h"
34
35	#define xfrm_state_deref_prot(table, net) \
36	rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
37
38	static void xfrm_state_gc_task(struct work_struct *work);
39
40	/* Each xfrm_state may be linked to two tables:
41
42	1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl)
43	2. Hash table by (daddr,family,reqid) to find what SAs exist for given
44	destination/tunnel endpoint. (output)
45	*/
46
47	static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024;
48	static struct kmem_cache *xfrm_state_cache __ro_after_init;
49
50	static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task);
51	static HLIST_HEAD(xfrm_state_gc_list);
52
53	static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x)
54	{
55	return refcount_inc_not_zero(r: &x->refcnt);
56	}
57
58	static inline unsigned int xfrm_dst_hash(struct net *net,
59	const xfrm_address_t *daddr,
60	const xfrm_address_t *saddr,
61	u32 reqid,
62	unsigned short family)
63	{
64	return __xfrm_dst_hash(daddr, saddr, reqid, family, hmask: net->xfrm.state_hmask);
65	}
66
67	static inline unsigned int xfrm_src_hash(struct net *net,
68	const xfrm_address_t *daddr,
69	const xfrm_address_t *saddr,
70	unsigned short family)
71	{
72	return __xfrm_src_hash(daddr, saddr, family, hmask: net->xfrm.state_hmask);
73	}
74
75	static inline unsigned int
76	xfrm_spi_hash(struct net *net, const xfrm_address_t *daddr,
77	__be32 spi, u8 proto, unsigned short family)
78	{
79	return __xfrm_spi_hash(daddr, spi, proto, family, hmask: net->xfrm.state_hmask);
80	}
81
82	static unsigned int xfrm_seq_hash(struct net *net, u32 seq)
83	{
84	return __xfrm_seq_hash(seq, hmask: net->xfrm.state_hmask);
85	}
86
87	#define XFRM_STATE_INSERT(by, _n, _h, _type) \
88	{ \
89	struct xfrm_state *_x = NULL; \
90	\
91	if (_type != XFRM_DEV_OFFLOAD_PACKET) { \
92	hlist_for_each_entry_rcu(_x, _h, by) { \
93	if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
94	continue; \
95	break; \
96	} \
97	} \
98	\
99	if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
100	/* SAD is empty or consist from HW SAs only */ \
101	hlist_add_head_rcu(_n, _h); \
102	else \
103	hlist_add_before_rcu(_n, &_x->by); \
104	}
105
106	static void xfrm_hash_transfer(struct hlist_head *list,
107	struct hlist_head *ndsttable,
108	struct hlist_head *nsrctable,
109	struct hlist_head *nspitable,
110	struct hlist_head *nseqtable,
111	unsigned int nhashmask)
112	{
113	struct hlist_node *tmp;
114	struct xfrm_state *x;
115
116	hlist_for_each_entry_safe(x, tmp, list, bydst) {
117	unsigned int h;
118
119	h = __xfrm_dst_hash(daddr: &x->id.daddr, saddr: &x->props.saddr,
120	reqid: x->props.reqid, family: x->props.family,
121	hmask: nhashmask);
122	XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type);
123
124	h = __xfrm_src_hash(daddr: &x->id.daddr, saddr: &x->props.saddr,
125	family: x->props.family,
126	hmask: nhashmask);
127	XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type);
128
129	if (x->id.spi) {
130	h = __xfrm_spi_hash(daddr: &x->id.daddr, spi: x->id.spi,
131	proto: x->id.proto, family: x->props.family,
132	hmask: nhashmask);
133	XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h,
134	x->xso.type);
135	}
136
137	if (x->km.seq) {
138	h = __xfrm_seq_hash(seq: x->km.seq, hmask: nhashmask);
139	XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h,
140	x->xso.type);
141	}
142	}
143	}
144
145	static unsigned long xfrm_hash_new_size(unsigned int state_hmask)
146	{
147	return ((state_hmask + 1) << 1) * sizeof(struct hlist_head);
148	}
149
150	static void xfrm_hash_resize(struct work_struct *work)
151	{
152	struct net *net = container_of(work, struct net, xfrm.state_hash_work);
153	struct hlist_head *ndst, *nsrc, *nspi, *nseq, *odst, *osrc, *ospi, *oseq;
154	unsigned long nsize, osize;
155	unsigned int nhashmask, ohashmask;
156	int i;
157
158	nsize = xfrm_hash_new_size(state_hmask: net->xfrm.state_hmask);
159	ndst = xfrm_hash_alloc(sz: nsize);
160	if (!ndst)
161	return;
162	nsrc = xfrm_hash_alloc(sz: nsize);
163	if (!nsrc) {
164	xfrm_hash_free(n: ndst, sz: nsize);
165	return;
166	}
167	nspi = xfrm_hash_alloc(sz: nsize);
168	if (!nspi) {
169	xfrm_hash_free(n: ndst, sz: nsize);
170	xfrm_hash_free(n: nsrc, sz: nsize);
171	return;
172	}
173	nseq = xfrm_hash_alloc(sz: nsize);
174	if (!nseq) {
175	xfrm_hash_free(n: ndst, sz: nsize);
176	xfrm_hash_free(n: nsrc, sz: nsize);
177	xfrm_hash_free(n: nspi, sz: nsize);
178	return;
179	}
180
181	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
182	write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
183
184	nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;
185	odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);
186	for (i = net->xfrm.state_hmask; i >= 0; i--)
187	xfrm_hash_transfer(list: odst + i, ndsttable: ndst, nsrctable: nsrc, nspitable: nspi, nseqtable: nseq, nhashmask);
188
189	osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);
190	ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);
191	oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net);
192	ohashmask = net->xfrm.state_hmask;
193
194	rcu_assign_pointer(net->xfrm.state_bydst, ndst);
195	rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);
196	rcu_assign_pointer(net->xfrm.state_byspi, nspi);
197	rcu_assign_pointer(net->xfrm.state_byseq, nseq);
198	net->xfrm.state_hmask = nhashmask;
199
200	write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);
201	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
202
203	osize = (ohashmask + 1) * sizeof(struct hlist_head);
204
205	synchronize_rcu();
206
207	xfrm_hash_free(n: odst, sz: osize);
208	xfrm_hash_free(n: osrc, sz: osize);
209	xfrm_hash_free(n: ospi, sz: osize);
210	xfrm_hash_free(n: oseq, sz: osize);
211	}
212
213	static DEFINE_SPINLOCK(xfrm_state_afinfo_lock);
214	static struct xfrm_state_afinfo __rcu *xfrm_state_afinfo[NPROTO];
215
216	static DEFINE_SPINLOCK(xfrm_state_gc_lock);
217
218	int __xfrm_state_delete(struct xfrm_state *x);
219
220	int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol);
221	static bool km_is_alive(const struct km_event *c);
222	void km_state_expired(struct xfrm_state *x, int hard, u32 portid);
223
224	int xfrm_register_type(const struct xfrm_type *type, unsigned short family)
225	{
226	struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
227	int err = 0;
228
229	if (!afinfo)
230	return -EAFNOSUPPORT;
231
232	#define X(afi, T, name) do { \
233	WARN_ON((afi)->type_ ## name); \
234	(afi)->type_ ## name = (T); \
235	} while (0)
236
237	switch (type->proto) {
238	case IPPROTO_COMP:
239	X(afinfo, type, comp);
240	break;
241	case IPPROTO_AH:
242	X(afinfo, type, ah);
243	break;
244	case IPPROTO_ESP:
245	X(afinfo, type, esp);
246	break;
247	case IPPROTO_IPIP:
248	X(afinfo, type, ipip);
249	break;
250	case IPPROTO_DSTOPTS:
251	X(afinfo, type, dstopts);
252	break;
253	case IPPROTO_ROUTING:
254	X(afinfo, type, routing);
255	break;
256	case IPPROTO_IPV6:
257	X(afinfo, type, ipip6);
258	break;
259	default:
260	WARN_ON(1);
261	err = -EPROTONOSUPPORT;
262	break;
263	}
264	#undef X
265	rcu_read_unlock();
266	return err;
267	}
268	EXPORT_SYMBOL(xfrm_register_type);
269
270	void xfrm_unregister_type(const struct xfrm_type *type, unsigned short family)
271	{
272	struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
273
274	if (unlikely(afinfo == NULL))
275	return;
276
277	#define X(afi, T, name) do { \
278	WARN_ON((afi)->type_ ## name != (T)); \
279	(afi)->type_ ## name = NULL; \
280	} while (0)
281
282	switch (type->proto) {
283	case IPPROTO_COMP:
284	X(afinfo, type, comp);
285	break;
286	case IPPROTO_AH:
287	X(afinfo, type, ah);
288	break;
289	case IPPROTO_ESP:
290	X(afinfo, type, esp);
291	break;
292	case IPPROTO_IPIP:
293	X(afinfo, type, ipip);
294	break;
295	case IPPROTO_DSTOPTS:
296	X(afinfo, type, dstopts);
297	break;
298	case IPPROTO_ROUTING:
299	X(afinfo, type, routing);
300	break;
301	case IPPROTO_IPV6:
302	X(afinfo, type, ipip6);
303	break;
304	default:
305	WARN_ON(1);
306	break;
307	}
308	#undef X
309	rcu_read_unlock();
310	}
311	EXPORT_SYMBOL(xfrm_unregister_type);
312
313	static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family)
314	{
315	const struct xfrm_type *type = NULL;
316	struct xfrm_state_afinfo *afinfo;
317	int modload_attempted = 0;
318
319	retry:
320	afinfo = xfrm_state_get_afinfo(family);
321	if (unlikely(afinfo == NULL))
322	return NULL;
323
324	switch (proto) {
325	case IPPROTO_COMP:
326	type = afinfo->type_comp;
327	break;
328	case IPPROTO_AH:
329	type = afinfo->type_ah;
330	break;
331	case IPPROTO_ESP:
332	type = afinfo->type_esp;
333	break;
334	case IPPROTO_IPIP:
335	type = afinfo->type_ipip;
336	break;
337	case IPPROTO_DSTOPTS:
338	type = afinfo->type_dstopts;
339	break;
340	case IPPROTO_ROUTING:
341	type = afinfo->type_routing;
342	break;
343	case IPPROTO_IPV6:
344	type = afinfo->type_ipip6;
345	break;
346	default:
347	break;
348	}
349
350	if (unlikely(type && !try_module_get(type->owner)))
351	type = NULL;
352
353	rcu_read_unlock();
354
355	if (!type && !modload_attempted) {
356	request_module("xfrm-type-%d-%d", family, proto);
357	modload_attempted = 1;
358	goto retry;
359	}
360
361	return type;
362	}
363
364	static void xfrm_put_type(const struct xfrm_type *type)
365	{
366	module_put(module: type->owner);
367	}
368
369	int xfrm_register_type_offload(const struct xfrm_type_offload *type,
370	unsigned short family)
371	{
372	struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
373	int err = 0;
374
375	if (unlikely(afinfo == NULL))
376	return -EAFNOSUPPORT;
377
378	switch (type->proto) {
379	case IPPROTO_ESP:
380	WARN_ON(afinfo->type_offload_esp);
381	afinfo->type_offload_esp = type;
382	break;
383	default:
384	WARN_ON(1);
385	err = -EPROTONOSUPPORT;
386	break;
387	}
388
389	rcu_read_unlock();
390	return err;
391	}
392	EXPORT_SYMBOL(xfrm_register_type_offload);
393
394	void xfrm_unregister_type_offload(const struct xfrm_type_offload *type,
395	unsigned short family)
396	{
397	struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
398
399	if (unlikely(afinfo == NULL))
400	return;
401
402	switch (type->proto) {
403	case IPPROTO_ESP:
404	WARN_ON(afinfo->type_offload_esp != type);
405	afinfo->type_offload_esp = NULL;
406	break;
407	default:
408	WARN_ON(1);
409	break;
410	}
411	rcu_read_unlock();
412	}
413	EXPORT_SYMBOL(xfrm_unregister_type_offload);
414
415	static const struct xfrm_type_offload *
416	xfrm_get_type_offload(u8 proto, unsigned short family, bool try_load)
417	{
418	const struct xfrm_type_offload *type = NULL;
419	struct xfrm_state_afinfo *afinfo;
420
421	retry:
422	afinfo = xfrm_state_get_afinfo(family);
423	if (unlikely(afinfo == NULL))
424	return NULL;
425
426	switch (proto) {
427	case IPPROTO_ESP:
428	type = afinfo->type_offload_esp;
429	break;
430	default:
431	break;
432	}
433
434	if ((type && !try_module_get(module: type->owner)))
435	type = NULL;
436
437	rcu_read_unlock();
438
439	if (!type && try_load) {
440	request_module("xfrm-offload-%d-%d", family, proto);
441	try_load = false;
442	goto retry;
443	}
444
445	return type;
446	}
447
448	static void xfrm_put_type_offload(const struct xfrm_type_offload *type)
449	{
450	module_put(module: type->owner);
451	}
452
453	static const struct xfrm_mode xfrm4_mode_map[XFRM_MODE_MAX] = {
454	[XFRM_MODE_BEET] = {
455	.encap = XFRM_MODE_BEET,
456	.flags = XFRM_MODE_FLAG_TUNNEL,
457	.family = AF_INET,
458	},
459	[XFRM_MODE_TRANSPORT] = {
460	.encap = XFRM_MODE_TRANSPORT,
461	.family = AF_INET,
462	},
463	[XFRM_MODE_TUNNEL] = {
464	.encap = XFRM_MODE_TUNNEL,
465	.flags = XFRM_MODE_FLAG_TUNNEL,
466	.family = AF_INET,
467	},
468	};
469
470	static const struct xfrm_mode xfrm6_mode_map[XFRM_MODE_MAX] = {
471	[XFRM_MODE_BEET] = {
472	.encap = XFRM_MODE_BEET,
473	.flags = XFRM_MODE_FLAG_TUNNEL,
474	.family = AF_INET6,
475	},
476	[XFRM_MODE_ROUTEOPTIMIZATION] = {
477	.encap = XFRM_MODE_ROUTEOPTIMIZATION,
478	.family = AF_INET6,
479	},
480	[XFRM_MODE_TRANSPORT] = {
481	.encap = XFRM_MODE_TRANSPORT,
482	.family = AF_INET6,
483	},
484	[XFRM_MODE_TUNNEL] = {
485	.encap = XFRM_MODE_TUNNEL,
486	.flags = XFRM_MODE_FLAG_TUNNEL,
487	.family = AF_INET6,
488	},
489	};
490
491	static const struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family)
492	{
493	const struct xfrm_mode *mode;
494
495	if (unlikely(encap >= XFRM_MODE_MAX))
496	return NULL;
497
498	switch (family) {
499	case AF_INET:
500	mode = &xfrm4_mode_map[encap];
501	if (mode->family == family)
502	return mode;
503	break;
504	case AF_INET6:
505	mode = &xfrm6_mode_map[encap];
506	if (mode->family == family)
507	return mode;
508	break;
509	default:
510	break;
511	}
512
513	return NULL;
514	}
515
516	void xfrm_state_free(struct xfrm_state *x)
517	{
518	kmem_cache_free(s: xfrm_state_cache, objp: x);
519	}
520	EXPORT_SYMBOL(xfrm_state_free);
521
522	static void ___xfrm_state_destroy(struct xfrm_state *x)
523	{
524	hrtimer_cancel(timer: &x->mtimer);
525	del_timer_sync(timer: &x->rtimer);
526	kfree(objp: x->aead);
527	kfree(objp: x->aalg);
528	kfree(objp: x->ealg);
529	kfree(objp: x->calg);
530	kfree(objp: x->encap);
531	kfree(objp: x->coaddr);
532	kfree(objp: x->replay_esn);
533	kfree(objp: x->preplay_esn);
534	if (x->type_offload)
535	xfrm_put_type_offload(type: x->type_offload);
536	if (x->type) {
537	x->type->destructor(x);
538	xfrm_put_type(type: x->type);
539	}
540	if (x->xfrag.page)
541	put_page(page: x->xfrag.page);
542	xfrm_dev_state_free(x);
543	security_xfrm_state_free(x);
544	xfrm_state_free(x);
545	}
546
547	static void xfrm_state_gc_task(struct work_struct *work)
548	{
549	struct xfrm_state *x;
550	struct hlist_node *tmp;
551	struct hlist_head gc_list;
552
553	spin_lock_bh(lock: &xfrm_state_gc_lock);
554	hlist_move_list(old: &xfrm_state_gc_list, new: &gc_list);
555	spin_unlock_bh(lock: &xfrm_state_gc_lock);
556
557	synchronize_rcu();
558
559	hlist_for_each_entry_safe(x, tmp, &gc_list, gclist)
560	___xfrm_state_destroy(x);
561	}
562
563	static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me)
564	{
565	struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer);
566	enum hrtimer_restart ret = HRTIMER_NORESTART;
567	time64_t now = ktime_get_real_seconds();
568	time64_t next = TIME64_MAX;
569	int warn = 0;
570	int err = 0;
571
572	spin_lock(lock: &x->lock);
573	xfrm_dev_state_update_curlft(x);
574
575	if (x->km.state == XFRM_STATE_DEAD)
576	goto out;
577	if (x->km.state == XFRM_STATE_EXPIRED)
578	goto expired;
579	if (x->lft.hard_add_expires_seconds) {
580	time64_t tmo = x->lft.hard_add_expires_seconds +
581	x->curlft.add_time - now;
582	if (tmo <= 0) {
583	if (x->xflags & XFRM_SOFT_EXPIRE) {
584	/* enter hard expire without soft expire first?!
585	* setting a new date could trigger this.
586	* workaround: fix x->curflt.add_time by below:
587	*/
588	x->curlft.add_time = now - x->saved_tmo - 1;
589	tmo = x->lft.hard_add_expires_seconds - x->saved_tmo;
590	} else
591	goto expired;
592	}
593	if (tmo < next)
594	next = tmo;
595	}
596	if (x->lft.hard_use_expires_seconds) {
597	time64_t tmo = x->lft.hard_use_expires_seconds +
598	(READ_ONCE(x->curlft.use_time) ? : now) - now;
599	if (tmo <= 0)
600	goto expired;
601	if (tmo < next)
602	next = tmo;
603	}
604	if (x->km.dying)
605	goto resched;
606	if (x->lft.soft_add_expires_seconds) {
607	time64_t tmo = x->lft.soft_add_expires_seconds +
608	x->curlft.add_time - now;
609	if (tmo <= 0) {
610	warn = 1;
611	x->xflags &= ~XFRM_SOFT_EXPIRE;
612	} else if (tmo < next) {
613	next = tmo;
614	x->xflags |= XFRM_SOFT_EXPIRE;
615	x->saved_tmo = tmo;
616	}
617	}
618	if (x->lft.soft_use_expires_seconds) {
619	time64_t tmo = x->lft.soft_use_expires_seconds +
620	(READ_ONCE(x->curlft.use_time) ? : now) - now;
621	if (tmo <= 0)
622	warn = 1;
623	else if (tmo < next)
624	next = tmo;
625	}
626
627	x->km.dying = warn;
628	if (warn)
629	km_state_expired(x, hard: 0, portid: 0);
630	resched:
631	if (next != TIME64_MAX) {
632	hrtimer_forward_now(timer: &x->mtimer, interval: ktime_set(secs: next, nsecs: 0));
633	ret = HRTIMER_RESTART;
634	}
635
636	goto out;
637
638	expired:
639	if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0)
640	x->km.state = XFRM_STATE_EXPIRED;
641
642	err = __xfrm_state_delete(x);
643	if (!err)
644	km_state_expired(x, hard: 1, portid: 0);
645
646	xfrm_audit_state_delete(x, result: err ? 0 : 1, task_valid: true);
647
648	out:
649	spin_unlock(lock: &x->lock);
650	return ret;
651	}
652
653	static void xfrm_replay_timer_handler(struct timer_list *t);
654
655	struct xfrm_state *xfrm_state_alloc(struct net *net)
656	{
657	struct xfrm_state *x;
658
659	x = kmem_cache_zalloc(k: xfrm_state_cache, GFP_ATOMIC);
660
661	if (x) {
662	write_pnet(pnet: &x->xs_net, net);
663	refcount_set(r: &x->refcnt, n: 1);
664	atomic_set(v: &x->tunnel_users, i: 0);
665	INIT_LIST_HEAD(list: &x->km.all);
666	INIT_HLIST_NODE(h: &x->bydst);
667	INIT_HLIST_NODE(h: &x->bysrc);
668	INIT_HLIST_NODE(h: &x->byspi);
669	INIT_HLIST_NODE(h: &x->byseq);
670	hrtimer_init(timer: &x->mtimer, CLOCK_BOOTTIME, mode: HRTIMER_MODE_ABS_SOFT);
671	x->mtimer.function = xfrm_timer_handler;
672	timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0);
673	x->curlft.add_time = ktime_get_real_seconds();
674	x->lft.soft_byte_limit = XFRM_INF;
675	x->lft.soft_packet_limit = XFRM_INF;
676	x->lft.hard_byte_limit = XFRM_INF;
677	x->lft.hard_packet_limit = XFRM_INF;
678	x->replay_maxage = 0;
679	x->replay_maxdiff = 0;
680	spin_lock_init(&x->lock);
681	}
682	return x;
683	}
684	EXPORT_SYMBOL(xfrm_state_alloc);
685
686	void __xfrm_state_destroy(struct xfrm_state *x, bool sync)
687	{
688	WARN_ON(x->km.state != XFRM_STATE_DEAD);
689
690	if (sync) {
691	synchronize_rcu();
692	___xfrm_state_destroy(x);
693	} else {
694	spin_lock_bh(lock: &xfrm_state_gc_lock);
695	hlist_add_head(n: &x->gclist, h: &xfrm_state_gc_list);
696	spin_unlock_bh(lock: &xfrm_state_gc_lock);
697	schedule_work(work: &xfrm_state_gc_work);
698	}
699	}
700	EXPORT_SYMBOL(__xfrm_state_destroy);
701
702	int __xfrm_state_delete(struct xfrm_state *x)
703	{
704	struct net *net = xs_net(x);
705	int err = -ESRCH;
706
707	if (x->km.state != XFRM_STATE_DEAD) {
708	x->km.state = XFRM_STATE_DEAD;
709	spin_lock(lock: &net->xfrm.xfrm_state_lock);
710	list_del(entry: &x->km.all);
711	hlist_del_rcu(n: &x->bydst);
712	hlist_del_rcu(n: &x->bysrc);
713	if (x->km.seq)
714	hlist_del_rcu(n: &x->byseq);
715	if (x->id.spi)
716	hlist_del_rcu(n: &x->byspi);
717	net->xfrm.state_num--;
718	spin_unlock(lock: &net->xfrm.xfrm_state_lock);
719
720	if (x->encap_sk)
721	sock_put(rcu_dereference_raw(x->encap_sk));
722
723	xfrm_dev_state_delete(x);
724
725	/* All xfrm_state objects are created by xfrm_state_alloc.
726	* The xfrm_state_alloc call gives a reference, and that
727	* is what we are dropping here.
728	*/
729	xfrm_state_put(x);
730	err = 0;
731	}
732
733	return err;
734	}
735	EXPORT_SYMBOL(__xfrm_state_delete);
736
737	int xfrm_state_delete(struct xfrm_state *x)
738	{
739	int err;
740
741	spin_lock_bh(lock: &x->lock);
742	err = __xfrm_state_delete(x);
743	spin_unlock_bh(lock: &x->lock);
744
745	return err;
746	}
747	EXPORT_SYMBOL(xfrm_state_delete);
748
749	#ifdef CONFIG_SECURITY_NETWORK_XFRM
750	static inline int
751	xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
752	{
753	int i, err = 0;
754
755	for (i = 0; i <= net->xfrm.state_hmask; i++) {
756	struct xfrm_state *x;
757
758	hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
759	if (xfrm_id_proto_match(proto: x->id.proto, userproto: proto) &&
760	(err = security_xfrm_state_delete(x)) != 0) {
761	xfrm_audit_state_delete(x, result: 0, task_valid);
762	return err;
763	}
764	}
765	}
766
767	return err;
768	}
769
770	static inline int
771	xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
772	{
773	int i, err = 0;
774
775	for (i = 0; i <= net->xfrm.state_hmask; i++) {
776	struct xfrm_state *x;
777	struct xfrm_dev_offload *xso;
778
779	hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
780	xso = &x->xso;
781
782	if (xso->dev == dev &&
783	(err = security_xfrm_state_delete(x)) != 0) {
784	xfrm_audit_state_delete(x, result: 0, task_valid);
785	return err;
786	}
787	}
788	}
789
790	return err;
791	}
792	#else
793	static inline int
794	xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
795	{
796	return 0;
797	}
798
799	static inline int
800	xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
801	{
802	return 0;
803	}
804	#endif
805
806	int xfrm_state_flush(struct net *net, u8 proto, bool task_valid, bool sync)
807	{
808	int i, err = 0, cnt = 0;
809
810	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
811	err = xfrm_state_flush_secctx_check(net, proto, task_valid);
812	if (err)
813	goto out;
814
815	err = -ESRCH;
816	for (i = 0; i <= net->xfrm.state_hmask; i++) {
817	struct xfrm_state *x;
818	restart:
819	hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
820	if (!xfrm_state_kern(x) &&
821	xfrm_id_proto_match(proto: x->id.proto, userproto: proto)) {
822	xfrm_state_hold(x);
823	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
824
825	err = xfrm_state_delete(x);
826	xfrm_audit_state_delete(x, result: err ? 0 : 1,
827	task_valid);
828	if (sync)
829	xfrm_state_put_sync(x);
830	else
831	xfrm_state_put(x);
832	if (!err)
833	cnt++;
834
835	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
836	goto restart;
837	}
838	}
839	}
840	out:
841	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
842	if (cnt)
843	err = 0;
844
845	return err;
846	}
847	EXPORT_SYMBOL(xfrm_state_flush);
848
849	int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid)
850	{
851	int i, err = 0, cnt = 0;
852
853	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
854	err = xfrm_dev_state_flush_secctx_check(net, dev, task_valid);
855	if (err)
856	goto out;
857
858	err = -ESRCH;
859	for (i = 0; i <= net->xfrm.state_hmask; i++) {
860	struct xfrm_state *x;
861	struct xfrm_dev_offload *xso;
862	restart:
863	hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {
864	xso = &x->xso;
865
866	if (!xfrm_state_kern(x) && xso->dev == dev) {
867	xfrm_state_hold(x);
868	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
869
870	err = xfrm_state_delete(x);
871	xfrm_audit_state_delete(x, result: err ? 0 : 1,
872	task_valid);
873	xfrm_state_put(x);
874	if (!err)
875	cnt++;
876
877	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
878	goto restart;
879	}
880	}
881	}
882	if (cnt)
883	err = 0;
884
885	out:
886	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
887	return err;
888	}
889	EXPORT_SYMBOL(xfrm_dev_state_flush);
890
891	void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si)
892	{
893	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
894	si->sadcnt = net->xfrm.state_num;
895	si->sadhcnt = net->xfrm.state_hmask + 1;
896	si->sadhmcnt = xfrm_state_hashmax;
897	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
898	}
899	EXPORT_SYMBOL(xfrm_sad_getinfo);
900
901	static void
902	__xfrm4_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl)
903	{
904	const struct flowi4 *fl4 = &fl->u.ip4;
905
906	sel->daddr.a4 = fl4->daddr;
907	sel->saddr.a4 = fl4->saddr;
908	sel->dport = xfrm_flowi_dport(fl, uli: &fl4->uli);
909	sel->dport_mask = htons(0xffff);
910	sel->sport = xfrm_flowi_sport(fl, uli: &fl4->uli);
911	sel->sport_mask = htons(0xffff);
912	sel->family = AF_INET;
913	sel->prefixlen_d = 32;
914	sel->prefixlen_s = 32;
915	sel->proto = fl4->flowi4_proto;
916	sel->ifindex = fl4->flowi4_oif;
917	}
918
919	static void
920	__xfrm6_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl)
921	{
922	const struct flowi6 *fl6 = &fl->u.ip6;
923
924	/* Initialize temporary selector matching only to current session. */
925	*(struct in6_addr *)&sel->daddr = fl6->daddr;
926	*(struct in6_addr *)&sel->saddr = fl6->saddr;
927	sel->dport = xfrm_flowi_dport(fl, uli: &fl6->uli);
928	sel->dport_mask = htons(0xffff);
929	sel->sport = xfrm_flowi_sport(fl, uli: &fl6->uli);
930	sel->sport_mask = htons(0xffff);
931	sel->family = AF_INET6;
932	sel->prefixlen_d = 128;
933	sel->prefixlen_s = 128;
934	sel->proto = fl6->flowi6_proto;
935	sel->ifindex = fl6->flowi6_oif;
936	}
937
938	static void
939	xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl,
940	const struct xfrm_tmpl *tmpl,
941	const xfrm_address_t *daddr, const xfrm_address_t *saddr,
942	unsigned short family)
943	{
944	switch (family) {
945	case AF_INET:
946	__xfrm4_init_tempsel(sel: &x->sel, fl);
947	break;
948	case AF_INET6:
949	__xfrm6_init_tempsel(sel: &x->sel, fl);
950	break;
951	}
952
953	x->id = tmpl->id;
954
955	switch (tmpl->encap_family) {
956	case AF_INET:
957	if (x->id.daddr.a4 == 0)
958	x->id.daddr.a4 = daddr->a4;
959	x->props.saddr = tmpl->saddr;
960	if (x->props.saddr.a4 == 0)
961	x->props.saddr.a4 = saddr->a4;
962	break;
963	case AF_INET6:
964	if (ipv6_addr_any(a: (struct in6_addr *)&x->id.daddr))
965	memcpy(&x->id.daddr, daddr, sizeof(x->sel.daddr));
966	memcpy(&x->props.saddr, &tmpl->saddr, sizeof(x->props.saddr));
967	if (ipv6_addr_any(a: (struct in6_addr *)&x->props.saddr))
968	memcpy(&x->props.saddr, saddr, sizeof(x->props.saddr));
969	break;
970	}
971
972	x->props.mode = tmpl->mode;
973	x->props.reqid = tmpl->reqid;
974	x->props.family = tmpl->encap_family;
975	}
976
977	static struct xfrm_state *__xfrm_state_lookup_all(struct net *net, u32 mark,
978	const xfrm_address_t *daddr,
979	__be32 spi, u8 proto,
980	unsigned short family,
981	struct xfrm_dev_offload *xdo)
982	{
983	unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family);
984	struct xfrm_state *x;
985
986	hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {
987	#ifdef CONFIG_XFRM_OFFLOAD
988	if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) {
989	if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
990	/* HW states are in the head of list, there is
991	* no need to iterate further.
992	*/
993	break;
994
995	/* Packet offload: both policy and SA should
996	* have same device.
997	*/
998	if (xdo->dev != x->xso.dev)
999	continue;
1000	} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1001	/* Skip HW policy for SW lookups */
1002	continue;
1003	#endif
1004	if (x->props.family != family ||
1005	x->id.spi != spi ||
1006	x->id.proto != proto ||
1007	!xfrm_addr_equal(a: &x->id.daddr, b: daddr, family))
1008	continue;
1009
1010	if ((mark & x->mark.m) != x->mark.v)
1011	continue;
1012	if (!xfrm_state_hold_rcu(x))
1013	continue;
1014	return x;
1015	}
1016
1017	return NULL;
1018	}
1019
1020	static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark,
1021	const xfrm_address_t *daddr,
1022	__be32 spi, u8 proto,
1023	unsigned short family)
1024	{
1025	unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family);
1026	struct xfrm_state *x;
1027
1028	hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {
1029	if (x->props.family != family ||
1030	x->id.spi != spi ||
1031	x->id.proto != proto ||
1032	!xfrm_addr_equal(a: &x->id.daddr, b: daddr, family))
1033	continue;
1034
1035	if ((mark & x->mark.m) != x->mark.v)
1036	continue;
1037	if (!xfrm_state_hold_rcu(x))
1038	continue;
1039	return x;
1040	}
1041
1042	return NULL;
1043	}
1044
1045	static struct xfrm_state *__xfrm_state_lookup_byaddr(struct net *net, u32 mark,
1046	const xfrm_address_t *daddr,
1047	const xfrm_address_t *saddr,
1048	u8 proto, unsigned short family)
1049	{
1050	unsigned int h = xfrm_src_hash(net, daddr, saddr, family);
1051	struct xfrm_state *x;
1052
1053	hlist_for_each_entry_rcu(x, net->xfrm.state_bysrc + h, bysrc) {
1054	if (x->props.family != family ||
1055	x->id.proto != proto ||
1056	!xfrm_addr_equal(a: &x->id.daddr, b: daddr, family) ||
1057	!xfrm_addr_equal(a: &x->props.saddr, b: saddr, family))
1058	continue;
1059
1060	if ((mark & x->mark.m) != x->mark.v)
1061	continue;
1062	if (!xfrm_state_hold_rcu(x))
1063	continue;
1064	return x;
1065	}
1066
1067	return NULL;
1068	}
1069
1070	static inline struct xfrm_state *
1071	__xfrm_state_locate(struct xfrm_state *x, int use_spi, int family)
1072	{
1073	struct net *net = xs_net(x);
1074	u32 mark = x->mark.v & x->mark.m;
1075
1076	if (use_spi)
1077	return __xfrm_state_lookup(net, mark, daddr: &x->id.daddr,
1078	spi: x->id.spi, proto: x->id.proto, family);
1079	else
1080	return __xfrm_state_lookup_byaddr(net, mark,
1081	daddr: &x->id.daddr,
1082	saddr: &x->props.saddr,
1083	proto: x->id.proto, family);
1084	}
1085
1086	static void xfrm_hash_grow_check(struct net *net, int have_hash_collision)
1087	{
1088	if (have_hash_collision &&
1089	(net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&
1090	net->xfrm.state_num > net->xfrm.state_hmask)
1091	schedule_work(work: &net->xfrm.state_hash_work);
1092	}
1093
1094	static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x,
1095	const struct flowi *fl, unsigned short family,
1096	struct xfrm_state **best, int *acq_in_progress,
1097	int *error)
1098	{
1099	/* Resolution logic:
1100	* 1. There is a valid state with matching selector. Done.
1101	* 2. Valid state with inappropriate selector. Skip.
1102	*
1103	* Entering area of "sysdeps".
1104	*
1105	* 3. If state is not valid, selector is temporary, it selects
1106	* only session which triggered previous resolution. Key
1107	* manager will do something to install a state with proper
1108	* selector.
1109	*/
1110	if (x->km.state == XFRM_STATE_VALID) {
1111	if ((x->sel.family &&
1112	(x->sel.family != family ||
1113	!xfrm_selector_match(sel: &x->sel, fl, family))) ||
1114	!security_xfrm_state_pol_flow_match(x, xp: pol,
1115	flic: &fl->u.__fl_common))
1116	return;
1117
1118	if (!*best ||
1119	(*best)->km.dying > x->km.dying ||
1120	((*best)->km.dying == x->km.dying &&
1121	(*best)->curlft.add_time < x->curlft.add_time))
1122	*best = x;
1123	} else if (x->km.state == XFRM_STATE_ACQ) {
1124	*acq_in_progress = 1;
1125	} else if (x->km.state == XFRM_STATE_ERROR ||
1126	x->km.state == XFRM_STATE_EXPIRED) {
1127	if ((!x->sel.family ||
1128	(x->sel.family == family &&
1129	xfrm_selector_match(sel: &x->sel, fl, family))) &&
1130	security_xfrm_state_pol_flow_match(x, xp: pol,
1131	flic: &fl->u.__fl_common))
1132	*error = -ESRCH;
1133	}
1134	}
1135
1136	struct xfrm_state *
1137	xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1138	const struct flowi *fl, struct xfrm_tmpl *tmpl,
1139	struct xfrm_policy *pol, int *err,
1140	unsigned short family, u32 if_id)
1141	{
1142	static xfrm_address_t saddr_wildcard = { };
1143	struct net *net = xp_net(xp: pol);
1144	unsigned int h, h_wildcard;
1145	struct xfrm_state *x, *x0, *to_put;
1146	int acquire_in_progress = 0;
1147	int error = 0;
1148	struct xfrm_state *best = NULL;
1149	u32 mark = pol->mark.v & pol->mark.m;
1150	unsigned short encap_family = tmpl->encap_family;
1151	unsigned int sequence;
1152	struct km_event c;
1153
1154	to_put = NULL;
1155
1156	sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
1157
1158	rcu_read_lock();
1159	h = xfrm_dst_hash(net, daddr, saddr, reqid: tmpl->reqid, family: encap_family);
1160	hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) {
1161	#ifdef CONFIG_XFRM_OFFLOAD
1162	if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1163	if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1164	/* HW states are in the head of list, there is
1165	* no need to iterate further.
1166	*/
1167	break;
1168
1169	/* Packet offload: both policy and SA should
1170	* have same device.
1171	*/
1172	if (pol->xdo.dev != x->xso.dev)
1173	continue;
1174	} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1175	/* Skip HW policy for SW lookups */
1176	continue;
1177	#endif
1178	if (x->props.family == encap_family &&
1179	x->props.reqid == tmpl->reqid &&
1180	(mark & x->mark.m) == x->mark.v &&
1181	x->if_id == if_id &&
1182	!(x->props.flags & XFRM_STATE_WILDRECV) &&
1183	xfrm_state_addr_check(x, daddr, saddr, family: encap_family) &&
1184	tmpl->mode == x->props.mode &&
1185	tmpl->id.proto == x->id.proto &&
1186	(tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1187	xfrm_state_look_at(pol, x, fl, family,
1188	best: &best, acq_in_progress: &acquire_in_progress, error: &error);
1189	}
1190	if (best || acquire_in_progress)
1191	goto found;
1192
1193	h_wildcard = xfrm_dst_hash(net, daddr, saddr: &saddr_wildcard, reqid: tmpl->reqid, family: encap_family);
1194	hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) {
1195	#ifdef CONFIG_XFRM_OFFLOAD
1196	if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1197	if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1198	/* HW states are in the head of list, there is
1199	* no need to iterate further.
1200	*/
1201	break;
1202
1203	/* Packet offload: both policy and SA should
1204	* have same device.
1205	*/
1206	if (pol->xdo.dev != x->xso.dev)
1207	continue;
1208	} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1209	/* Skip HW policy for SW lookups */
1210	continue;
1211	#endif
1212	if (x->props.family == encap_family &&
1213	x->props.reqid == tmpl->reqid &&
1214	(mark & x->mark.m) == x->mark.v &&
1215	x->if_id == if_id &&
1216	!(x->props.flags & XFRM_STATE_WILDRECV) &&
1217	xfrm_addr_equal(a: &x->id.daddr, b: daddr, family: encap_family) &&
1218	tmpl->mode == x->props.mode &&
1219	tmpl->id.proto == x->id.proto &&
1220	(tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1221	xfrm_state_look_at(pol, x, fl, family,
1222	best: &best, acq_in_progress: &acquire_in_progress, error: &error);
1223	}
1224
1225	found:
1226	x = best;
1227	if (!x && !error && !acquire_in_progress) {
1228	if (tmpl->id.spi &&
1229	(x0 = __xfrm_state_lookup_all(net, mark, daddr,
1230	spi: tmpl->id.spi, proto: tmpl->id.proto,
1231	family: encap_family,
1232	xdo: &pol->xdo)) != NULL) {
1233	to_put = x0;
1234	error = -EEXIST;
1235	goto out;
1236	}
1237
1238	c.net = net;
1239	/* If the KMs have no listeners (yet...), avoid allocating an SA
1240	* for each and every packet - garbage collection might not
1241	* handle the flood.
1242	*/
1243	if (!km_is_alive(c: &c)) {
1244	error = -ESRCH;
1245	goto out;
1246	}
1247
1248	x = xfrm_state_alloc(net);
1249	if (x == NULL) {
1250	error = -ENOMEM;
1251	goto out;
1252	}
1253	/* Initialize temporary state matching only
1254	* to current session. */
1255	xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family);
1256	memcpy(&x->mark, &pol->mark, sizeof(x->mark));
1257	x->if_id = if_id;
1258
1259	error = security_xfrm_state_alloc_acquire(x, polsec: pol->security, secid: fl->flowi_secid);
1260	if (error) {
1261	x->km.state = XFRM_STATE_DEAD;
1262	to_put = x;
1263	x = NULL;
1264	goto out;
1265	}
1266	#ifdef CONFIG_XFRM_OFFLOAD
1267	if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1268	struct xfrm_dev_offload *xdo = &pol->xdo;
1269	struct xfrm_dev_offload *xso = &x->xso;
1270
1271	xso->type = XFRM_DEV_OFFLOAD_PACKET;
1272	xso->dir = xdo->dir;
1273	xso->dev = xdo->dev;
1274	xso->real_dev = xdo->real_dev;
1275	xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ;
1276	netdev_tracker_alloc(dev: xso->dev, tracker: &xso->dev_tracker,
1277	GFP_ATOMIC);
1278	error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x, NULL);
1279	if (error) {
1280	xso->dir = 0;
1281	netdev_put(dev: xso->dev, tracker: &xso->dev_tracker);
1282	xso->dev = NULL;
1283	xso->real_dev = NULL;
1284	xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
1285	x->km.state = XFRM_STATE_DEAD;
1286	to_put = x;
1287	x = NULL;
1288	goto out;
1289	}
1290	}
1291	#endif
1292	if (km_query(x, t: tmpl, pol) == 0) {
1293	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1294	x->km.state = XFRM_STATE_ACQ;
1295	list_add(new: &x->km.all, head: &net->xfrm.state_all);
1296	XFRM_STATE_INSERT(bydst, &x->bydst,
1297	net->xfrm.state_bydst + h,
1298	x->xso.type);
1299	h = xfrm_src_hash(net, daddr, saddr, family: encap_family);
1300	XFRM_STATE_INSERT(bysrc, &x->bysrc,
1301	net->xfrm.state_bysrc + h,
1302	x->xso.type);
1303	if (x->id.spi) {
1304	h = xfrm_spi_hash(net, daddr: &x->id.daddr, spi: x->id.spi, proto: x->id.proto, family: encap_family);
1305	XFRM_STATE_INSERT(byspi, &x->byspi,
1306	net->xfrm.state_byspi + h,
1307	x->xso.type);
1308	}
1309	if (x->km.seq) {
1310	h = xfrm_seq_hash(net, seq: x->km.seq);
1311	XFRM_STATE_INSERT(byseq, &x->byseq,
1312	net->xfrm.state_byseq + h,
1313	x->xso.type);
1314	}
1315	x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1316	hrtimer_start(timer: &x->mtimer,
1317	tim: ktime_set(secs: net->xfrm.sysctl_acq_expires, nsecs: 0),
1318	mode: HRTIMER_MODE_REL_SOFT);
1319	net->xfrm.state_num++;
1320	xfrm_hash_grow_check(net, have_hash_collision: x->bydst.next != NULL);
1321	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1322	} else {
1323	#ifdef CONFIG_XFRM_OFFLOAD
1324	struct xfrm_dev_offload *xso = &x->xso;
1325
1326	if (xso->type == XFRM_DEV_OFFLOAD_PACKET) {
1327	xfrm_dev_state_delete(x);
1328	xfrm_dev_state_free(x);
1329	}
1330	#endif
1331	x->km.state = XFRM_STATE_DEAD;
1332	to_put = x;
1333	x = NULL;
1334	error = -ESRCH;
1335	}
1336	}
1337	out:
1338	if (x) {
1339	if (!xfrm_state_hold_rcu(x)) {
1340	*err = -EAGAIN;
1341	x = NULL;
1342	}
1343	} else {
1344	*err = acquire_in_progress ? -EAGAIN : error;
1345	}
1346	rcu_read_unlock();
1347	if (to_put)
1348	xfrm_state_put(x: to_put);
1349
1350	if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) {
1351	*err = -EAGAIN;
1352	if (x) {
1353	xfrm_state_put(x);
1354	x = NULL;
1355	}
1356	}
1357
1358	return x;
1359	}
1360
1361	struct xfrm_state *
1362	xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id,
1363	xfrm_address_t *daddr, xfrm_address_t *saddr,
1364	unsigned short family, u8 mode, u8 proto, u32 reqid)
1365	{
1366	unsigned int h;
1367	struct xfrm_state *rx = NULL, *x = NULL;
1368
1369	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1370	h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
1371	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1372	if (x->props.family == family &&
1373	x->props.reqid == reqid &&
1374	(mark & x->mark.m) == x->mark.v &&
1375	x->if_id == if_id &&
1376	!(x->props.flags & XFRM_STATE_WILDRECV) &&
1377	xfrm_state_addr_check(x, daddr, saddr, family) &&
1378	mode == x->props.mode &&
1379	proto == x->id.proto &&
1380	x->km.state == XFRM_STATE_VALID) {
1381	rx = x;
1382	break;
1383	}
1384	}
1385
1386	if (rx)
1387	xfrm_state_hold(x: rx);
1388	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1389
1390
1391	return rx;
1392	}
1393	EXPORT_SYMBOL(xfrm_stateonly_find);
1394
1395	struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi,
1396	unsigned short family)
1397	{
1398	struct xfrm_state *x;
1399	struct xfrm_state_walk *w;
1400
1401	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1402	list_for_each_entry(w, &net->xfrm.state_all, all) {
1403	x = container_of(w, struct xfrm_state, km);
1404	if (x->props.family != family ||
1405	x->id.spi != spi)
1406	continue;
1407
1408	xfrm_state_hold(x);
1409	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1410	return x;
1411	}
1412	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1413	return NULL;
1414	}
1415	EXPORT_SYMBOL(xfrm_state_lookup_byspi);
1416
1417	static void __xfrm_state_insert(struct xfrm_state *x)
1418	{
1419	struct net *net = xs_net(x);
1420	unsigned int h;
1421
1422	list_add(new: &x->km.all, head: &net->xfrm.state_all);
1423
1424	h = xfrm_dst_hash(net, daddr: &x->id.daddr, saddr: &x->props.saddr,
1425	reqid: x->props.reqid, family: x->props.family);
1426	XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,
1427	x->xso.type);
1428
1429	h = xfrm_src_hash(net, daddr: &x->id.daddr, saddr: &x->props.saddr, family: x->props.family);
1430	XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,
1431	x->xso.type);
1432
1433	if (x->id.spi) {
1434	h = xfrm_spi_hash(net, daddr: &x->id.daddr, spi: x->id.spi, proto: x->id.proto,
1435	family: x->props.family);
1436
1437	XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,
1438	x->xso.type);
1439	}
1440
1441	if (x->km.seq) {
1442	h = xfrm_seq_hash(net, seq: x->km.seq);
1443
1444	XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h,
1445	x->xso.type);
1446	}
1447
1448	hrtimer_start(timer: &x->mtimer, tim: ktime_set(secs: 1, nsecs: 0), mode: HRTIMER_MODE_REL_SOFT);
1449	if (x->replay_maxage)
1450	mod_timer(timer: &x->rtimer, expires: jiffies + x->replay_maxage);
1451
1452	net->xfrm.state_num++;
1453
1454	xfrm_hash_grow_check(net, have_hash_collision: x->bydst.next != NULL);
1455	}
1456
1457	/* net->xfrm.xfrm_state_lock is held */
1458	static void __xfrm_state_bump_genids(struct xfrm_state *xnew)
1459	{
1460	struct net *net = xs_net(x: xnew);
1461	unsigned short family = xnew->props.family;
1462	u32 reqid = xnew->props.reqid;
1463	struct xfrm_state *x;
1464	unsigned int h;
1465	u32 mark = xnew->mark.v & xnew->mark.m;
1466	u32 if_id = xnew->if_id;
1467
1468	h = xfrm_dst_hash(net, daddr: &xnew->id.daddr, saddr: &xnew->props.saddr, reqid, family);
1469	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1470	if (x->props.family == family &&
1471	x->props.reqid == reqid &&
1472	x->if_id == if_id &&
1473	(mark & x->mark.m) == x->mark.v &&
1474	xfrm_addr_equal(a: &x->id.daddr, b: &xnew->id.daddr, family) &&
1475	xfrm_addr_equal(a: &x->props.saddr, b: &xnew->props.saddr, family))
1476	x->genid++;
1477	}
1478	}
1479
1480	void xfrm_state_insert(struct xfrm_state *x)
1481	{
1482	struct net *net = xs_net(x);
1483
1484	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1485	__xfrm_state_bump_genids(xnew: x);
1486	__xfrm_state_insert(x);
1487	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1488	}
1489	EXPORT_SYMBOL(xfrm_state_insert);
1490
1491	/* net->xfrm.xfrm_state_lock is held */
1492	static struct xfrm_state *__find_acq_core(struct net *net,
1493	const struct xfrm_mark *m,
1494	unsigned short family, u8 mode,
1495	u32 reqid, u32 if_id, u8 proto,
1496	const xfrm_address_t *daddr,
1497	const xfrm_address_t *saddr,
1498	int create)
1499	{
1500	unsigned int h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
1501	struct xfrm_state *x;
1502	u32 mark = m->v & m->m;
1503
1504	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1505	if (x->props.reqid != reqid ||
1506	x->props.mode != mode ||
1507	x->props.family != family ||
1508	x->km.state != XFRM_STATE_ACQ ||
1509	x->id.spi != 0 ||
1510	x->id.proto != proto ||
1511	(mark & x->mark.m) != x->mark.v ||
1512	!xfrm_addr_equal(a: &x->id.daddr, b: daddr, family) ||
1513	!xfrm_addr_equal(a: &x->props.saddr, b: saddr, family))
1514	continue;
1515
1516	xfrm_state_hold(x);
1517	return x;
1518	}
1519
1520	if (!create)
1521	return NULL;
1522
1523	x = xfrm_state_alloc(net);
1524	if (likely(x)) {
1525	switch (family) {
1526	case AF_INET:
1527	x->sel.daddr.a4 = daddr->a4;
1528	x->sel.saddr.a4 = saddr->a4;
1529	x->sel.prefixlen_d = 32;
1530	x->sel.prefixlen_s = 32;
1531	x->props.saddr.a4 = saddr->a4;
1532	x->id.daddr.a4 = daddr->a4;
1533	break;
1534
1535	case AF_INET6:
1536	x->sel.daddr.in6 = daddr->in6;
1537	x->sel.saddr.in6 = saddr->in6;
1538	x->sel.prefixlen_d = 128;
1539	x->sel.prefixlen_s = 128;
1540	x->props.saddr.in6 = saddr->in6;
1541	x->id.daddr.in6 = daddr->in6;
1542	break;
1543	}
1544
1545	x->km.state = XFRM_STATE_ACQ;
1546	x->id.proto = proto;
1547	x->props.family = family;
1548	x->props.mode = mode;
1549	x->props.reqid = reqid;
1550	x->if_id = if_id;
1551	x->mark.v = m->v;
1552	x->mark.m = m->m;
1553	x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1554	xfrm_state_hold(x);
1555	hrtimer_start(timer: &x->mtimer,
1556	tim: ktime_set(secs: net->xfrm.sysctl_acq_expires, nsecs: 0),
1557	mode: HRTIMER_MODE_REL_SOFT);
1558	list_add(new: &x->km.all, head: &net->xfrm.state_all);
1559	XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,
1560	x->xso.type);
1561	h = xfrm_src_hash(net, daddr, saddr, family);
1562	XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,
1563	x->xso.type);
1564
1565	net->xfrm.state_num++;
1566
1567	xfrm_hash_grow_check(net, have_hash_collision: x->bydst.next != NULL);
1568	}
1569
1570	return x;
1571	}
1572
1573	static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq);
1574
1575	int xfrm_state_add(struct xfrm_state *x)
1576	{
1577	struct net *net = xs_net(x);
1578	struct xfrm_state *x1, *to_put;
1579	int family;
1580	int err;
1581	u32 mark = x->mark.v & x->mark.m;
1582	int use_spi = xfrm_id_proto_match(proto: x->id.proto, IPSEC_PROTO_ANY);
1583
1584	family = x->props.family;
1585
1586	to_put = NULL;
1587
1588	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1589
1590	x1 = __xfrm_state_locate(x, use_spi, family);
1591	if (x1) {
1592	to_put = x1;
1593	x1 = NULL;
1594	err = -EEXIST;
1595	goto out;
1596	}
1597
1598	if (use_spi && x->km.seq) {
1599	x1 = __xfrm_find_acq_byseq(net, mark, seq: x->km.seq);
1600	if (x1 && ((x1->id.proto != x->id.proto) ||
1601	!xfrm_addr_equal(a: &x1->id.daddr, b: &x->id.daddr, family))) {
1602	to_put = x1;
1603	x1 = NULL;
1604	}
1605	}
1606
1607	if (use_spi && !x1)
1608	x1 = __find_acq_core(net, m: &x->mark, family, mode: x->props.mode,
1609	reqid: x->props.reqid, if_id: x->if_id, proto: x->id.proto,
1610	daddr: &x->id.daddr, saddr: &x->props.saddr, create: 0);
1611
1612	__xfrm_state_bump_genids(xnew: x);
1613	__xfrm_state_insert(x);
1614	err = 0;
1615
1616	out:
1617	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1618
1619	if (x1) {
1620	xfrm_state_delete(x1);
1621	xfrm_state_put(x: x1);
1622	}
1623
1624	if (to_put)
1625	xfrm_state_put(x: to_put);
1626
1627	return err;
1628	}
1629	EXPORT_SYMBOL(xfrm_state_add);
1630
1631	#ifdef CONFIG_XFRM_MIGRATE
1632	static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *security)
1633	{
1634	struct xfrm_user_sec_ctx *uctx;
1635	int size = sizeof(*uctx) + security->ctx_len;
1636	int err;
1637
1638	uctx = kmalloc(size, GFP_KERNEL);
1639	if (!uctx)
1640	return -ENOMEM;
1641
1642	uctx->exttype = XFRMA_SEC_CTX;
1643	uctx->len = size;
1644	uctx->ctx_doi = security->ctx_doi;
1645	uctx->ctx_alg = security->ctx_alg;
1646	uctx->ctx_len = security->ctx_len;
1647	memcpy(uctx + 1, security->ctx_str, security->ctx_len);
1648	err = security_xfrm_state_alloc(x, sec_ctx: uctx);
1649	kfree(objp: uctx);
1650	if (err)
1651	return err;
1652
1653	return 0;
1654	}
1655
1656	static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig,
1657	struct xfrm_encap_tmpl *encap)
1658	{
1659	struct net *net = xs_net(x: orig);
1660	struct xfrm_state *x = xfrm_state_alloc(net);
1661	if (!x)
1662	goto out;
1663
1664	memcpy(&x->id, &orig->id, sizeof(x->id));
1665	memcpy(&x->sel, &orig->sel, sizeof(x->sel));
1666	memcpy(&x->lft, &orig->lft, sizeof(x->lft));
1667	x->props.mode = orig->props.mode;
1668	x->props.replay_window = orig->props.replay_window;
1669	x->props.reqid = orig->props.reqid;
1670	x->props.family = orig->props.family;
1671	x->props.saddr = orig->props.saddr;
1672
1673	if (orig->aalg) {
1674	x->aalg = xfrm_algo_auth_clone(orig: orig->aalg);
1675	if (!x->aalg)
1676	goto error;
1677	}
1678	x->props.aalgo = orig->props.aalgo;
1679
1680	if (orig->aead) {
1681	x->aead = xfrm_algo_aead_clone(orig: orig->aead);
1682	x->geniv = orig->geniv;
1683	if (!x->aead)
1684	goto error;
1685	}
1686	if (orig->ealg) {
1687	x->ealg = xfrm_algo_clone(orig: orig->ealg);
1688	if (!x->ealg)
1689	goto error;
1690	}
1691	x->props.ealgo = orig->props.ealgo;
1692
1693	if (orig->calg) {
1694	x->calg = xfrm_algo_clone(orig: orig->calg);
1695	if (!x->calg)
1696	goto error;
1697	}
1698	x->props.calgo = orig->props.calgo;
1699
1700	if (encap || orig->encap) {
1701	if (encap)
1702	x->encap = kmemdup(p: encap, size: sizeof(*x->encap),
1703	GFP_KERNEL);
1704	else
1705	x->encap = kmemdup(p: orig->encap, size: sizeof(*x->encap),
1706	GFP_KERNEL);
1707
1708	if (!x->encap)
1709	goto error;
1710	}
1711
1712	if (orig->security)
1713	if (clone_security(x, security: orig->security))
1714	goto error;
1715
1716	if (orig->coaddr) {
1717	x->coaddr = kmemdup(p: orig->coaddr, size: sizeof(*x->coaddr),
1718	GFP_KERNEL);
1719	if (!x->coaddr)
1720	goto error;
1721	}
1722
1723	if (orig->replay_esn) {
1724	if (xfrm_replay_clone(x, orig))
1725	goto error;
1726	}
1727
1728	memcpy(&x->mark, &orig->mark, sizeof(x->mark));
1729	memcpy(&x->props.smark, &orig->props.smark, sizeof(x->props.smark));
1730
1731	x->props.flags = orig->props.flags;
1732	x->props.extra_flags = orig->props.extra_flags;
1733
1734	x->if_id = orig->if_id;
1735	x->tfcpad = orig->tfcpad;
1736	x->replay_maxdiff = orig->replay_maxdiff;
1737	x->replay_maxage = orig->replay_maxage;
1738	memcpy(&x->curlft, &orig->curlft, sizeof(x->curlft));
1739	x->km.state = orig->km.state;
1740	x->km.seq = orig->km.seq;
1741	x->replay = orig->replay;
1742	x->preplay = orig->preplay;
1743	x->mapping_maxage = orig->mapping_maxage;
1744	x->lastused = orig->lastused;
1745	x->new_mapping = 0;
1746	x->new_mapping_sport = 0;
1747
1748	return x;
1749
1750	error:
1751	xfrm_state_put(x);
1752	out:
1753	return NULL;
1754	}
1755
1756	struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net,
1757	u32 if_id)
1758	{
1759	unsigned int h;
1760	struct xfrm_state *x = NULL;
1761
1762	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1763
1764	if (m->reqid) {
1765	h = xfrm_dst_hash(net, daddr: &m->old_daddr, saddr: &m->old_saddr,
1766	reqid: m->reqid, family: m->old_family);
1767	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
1768	if (x->props.mode != m->mode ||
1769	x->id.proto != m->proto)
1770	continue;
1771	if (m->reqid && x->props.reqid != m->reqid)
1772	continue;
1773	if (if_id != 0 && x->if_id != if_id)
1774	continue;
1775	if (!xfrm_addr_equal(a: &x->id.daddr, b: &m->old_daddr,
1776	family: m->old_family) ||
1777	!xfrm_addr_equal(a: &x->props.saddr, b: &m->old_saddr,
1778	family: m->old_family))
1779	continue;
1780	xfrm_state_hold(x);
1781	break;
1782	}
1783	} else {
1784	h = xfrm_src_hash(net, daddr: &m->old_daddr, saddr: &m->old_saddr,
1785	family: m->old_family);
1786	hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {
1787	if (x->props.mode != m->mode ||
1788	x->id.proto != m->proto)
1789	continue;
1790	if (if_id != 0 && x->if_id != if_id)
1791	continue;
1792	if (!xfrm_addr_equal(a: &x->id.daddr, b: &m->old_daddr,
1793	family: m->old_family) ||
1794	!xfrm_addr_equal(a: &x->props.saddr, b: &m->old_saddr,
1795	family: m->old_family))
1796	continue;
1797	xfrm_state_hold(x);
1798	break;
1799	}
1800	}
1801
1802	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1803
1804	return x;
1805	}
1806	EXPORT_SYMBOL(xfrm_migrate_state_find);
1807
1808	struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,
1809	struct xfrm_migrate *m,
1810	struct xfrm_encap_tmpl *encap)
1811	{
1812	struct xfrm_state *xc;
1813
1814	xc = xfrm_state_clone(orig: x, encap);
1815	if (!xc)
1816	return NULL;
1817
1818	xc->props.family = m->new_family;
1819
1820	if (xfrm_init_state(x: xc) < 0)
1821	goto error;
1822
1823	memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr));
1824	memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr));
1825
1826	/* add state */
1827	if (xfrm_addr_equal(a: &x->id.daddr, b: &m->new_daddr, family: m->new_family)) {
1828	/* a care is needed when the destination address of the
1829	state is to be updated as it is a part of triplet */
1830	xfrm_state_insert(xc);
1831	} else {
1832	if (xfrm_state_add(xc) < 0)
1833	goto error;
1834	}
1835
1836	return xc;
1837	error:
1838	xfrm_state_put(x: xc);
1839	return NULL;
1840	}
1841	EXPORT_SYMBOL(xfrm_state_migrate);
1842	#endif
1843
1844	int xfrm_state_update(struct xfrm_state *x)
1845	{
1846	struct xfrm_state *x1, *to_put;
1847	int err;
1848	int use_spi = xfrm_id_proto_match(proto: x->id.proto, IPSEC_PROTO_ANY);
1849	struct net *net = xs_net(x);
1850
1851	to_put = NULL;
1852
1853	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1854	x1 = __xfrm_state_locate(x, use_spi, family: x->props.family);
1855
1856	err = -ESRCH;
1857	if (!x1)
1858	goto out;
1859
1860	if (xfrm_state_kern(x: x1)) {
1861	to_put = x1;
1862	err = -EEXIST;
1863	goto out;
1864	}
1865
1866	if (x1->km.state == XFRM_STATE_ACQ) {
1867	__xfrm_state_insert(x);
1868	x = NULL;
1869	}
1870	err = 0;
1871
1872	out:
1873	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1874
1875	if (to_put)
1876	xfrm_state_put(x: to_put);
1877
1878	if (err)
1879	return err;
1880
1881	if (!x) {
1882	xfrm_state_delete(x1);
1883	xfrm_state_put(x: x1);
1884	return 0;
1885	}
1886
1887	err = -EINVAL;
1888	spin_lock_bh(lock: &x1->lock);
1889	if (likely(x1->km.state == XFRM_STATE_VALID)) {
1890	if (x->encap && x1->encap &&
1891	x->encap->encap_type == x1->encap->encap_type)
1892	memcpy(x1->encap, x->encap, sizeof(*x1->encap));
1893	else if (x->encap || x1->encap)
1894	goto fail;
1895
1896	if (x->coaddr && x1->coaddr) {
1897	memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr));
1898	}
1899	if (!use_spi && memcmp(p: &x1->sel, q: &x->sel, size: sizeof(x1->sel)))
1900	memcpy(&x1->sel, &x->sel, sizeof(x1->sel));
1901	memcpy(&x1->lft, &x->lft, sizeof(x1->lft));
1902	x1->km.dying = 0;
1903
1904	hrtimer_start(timer: &x1->mtimer, tim: ktime_set(secs: 1, nsecs: 0),
1905	mode: HRTIMER_MODE_REL_SOFT);
1906	if (READ_ONCE(x1->curlft.use_time))
1907	xfrm_state_check_expire(x: x1);
1908
1909	if (x->props.smark.m || x->props.smark.v || x->if_id) {
1910	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1911
1912	if (x->props.smark.m || x->props.smark.v)
1913	x1->props.smark = x->props.smark;
1914
1915	if (x->if_id)
1916	x1->if_id = x->if_id;
1917
1918	__xfrm_state_bump_genids(xnew: x1);
1919	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1920	}
1921
1922	err = 0;
1923	x->km.state = XFRM_STATE_DEAD;
1924	__xfrm_state_put(x);
1925	}
1926
1927	fail:
1928	spin_unlock_bh(lock: &x1->lock);
1929
1930	xfrm_state_put(x: x1);
1931
1932	return err;
1933	}
1934	EXPORT_SYMBOL(xfrm_state_update);
1935
1936	int xfrm_state_check_expire(struct xfrm_state *x)
1937	{
1938	xfrm_dev_state_update_curlft(x);
1939
1940	if (!READ_ONCE(x->curlft.use_time))
1941	WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds());
1942
1943	if (x->curlft.bytes >= x->lft.hard_byte_limit ||
1944	x->curlft.packets >= x->lft.hard_packet_limit) {
1945	x->km.state = XFRM_STATE_EXPIRED;
1946	hrtimer_start(timer: &x->mtimer, tim: 0, mode: HRTIMER_MODE_REL_SOFT);
1947	return -EINVAL;
1948	}
1949
1950	if (!x->km.dying &&
1951	(x->curlft.bytes >= x->lft.soft_byte_limit ||
1952	x->curlft.packets >= x->lft.soft_packet_limit)) {
1953	x->km.dying = 1;
1954	km_state_expired(x, hard: 0, portid: 0);
1955	}
1956	return 0;
1957	}
1958	EXPORT_SYMBOL(xfrm_state_check_expire);
1959
1960	struct xfrm_state *
1961	xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi,
1962	u8 proto, unsigned short family)
1963	{
1964	struct xfrm_state *x;
1965
1966	rcu_read_lock();
1967	x = __xfrm_state_lookup(net, mark, daddr, spi, proto, family);
1968	rcu_read_unlock();
1969	return x;
1970	}
1971	EXPORT_SYMBOL(xfrm_state_lookup);
1972
1973	struct xfrm_state *
1974	xfrm_state_lookup_byaddr(struct net *net, u32 mark,
1975	const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1976	u8 proto, unsigned short family)
1977	{
1978	struct xfrm_state *x;
1979
1980	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1981	x = __xfrm_state_lookup_byaddr(net, mark, daddr, saddr, proto, family);
1982	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1983	return x;
1984	}
1985	EXPORT_SYMBOL(xfrm_state_lookup_byaddr);
1986
1987	struct xfrm_state *
1988	xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid,
1989	u32 if_id, u8 proto, const xfrm_address_t *daddr,
1990	const xfrm_address_t *saddr, int create, unsigned short family)
1991	{
1992	struct xfrm_state *x;
1993
1994	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
1995	x = __find_acq_core(net, m: mark, family, mode, reqid, if_id, proto, daddr, saddr, create);
1996	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
1997
1998	return x;
1999	}
2000	EXPORT_SYMBOL(xfrm_find_acq);
2001
2002	#ifdef CONFIG_XFRM_SUB_POLICY
2003	#if IS_ENABLED(CONFIG_IPV6)
2004	/* distribution counting sort function for xfrm_state and xfrm_tmpl */
2005	static void
2006	__xfrm6_sort(void **dst, void **src, int n,
2007	int (*cmp)(const void *p), int maxclass)
2008	{
2009	int count[XFRM_MAX_DEPTH] = { };
2010	int class[XFRM_MAX_DEPTH];
2011	int i;
2012
2013	for (i = 0; i < n; i++) {
2014	int c = cmp(src[i]);
2015
2016	class[i] = c;
2017	count[c]++;
2018	}
2019
2020	for (i = 2; i < maxclass; i++)
2021	count[i] += count[i - 1];
2022
2023	for (i = 0; i < n; i++) {
2024	dst[count[class[i] - 1]++] = src[i];
2025	src[i] = NULL;
2026	}
2027	}
2028
2029	/* Rule for xfrm_state:
2030	*
2031	* rule 1: select IPsec transport except AH
2032	* rule 2: select MIPv6 RO or inbound trigger
2033	* rule 3: select IPsec transport AH
2034	* rule 4: select IPsec tunnel
2035	* rule 5: others
2036	*/
2037	static int __xfrm6_state_sort_cmp(const void *p)
2038	{
2039	const struct xfrm_state *v = p;
2040
2041	switch (v->props.mode) {
2042	case XFRM_MODE_TRANSPORT:
2043	if (v->id.proto != IPPROTO_AH)
2044	return 1;
2045	else
2046	return 3;
2047	#if IS_ENABLED(CONFIG_IPV6_MIP6)
2048	case XFRM_MODE_ROUTEOPTIMIZATION:
2049	case XFRM_MODE_IN_TRIGGER:
2050	return 2;
2051	#endif
2052	case XFRM_MODE_TUNNEL:
2053	case XFRM_MODE_BEET:
2054	return 4;
2055	}
2056	return 5;
2057	}
2058
2059	/* Rule for xfrm_tmpl:
2060	*
2061	* rule 1: select IPsec transport
2062	* rule 2: select MIPv6 RO or inbound trigger
2063	* rule 3: select IPsec tunnel
2064	* rule 4: others
2065	*/
2066	static int __xfrm6_tmpl_sort_cmp(const void *p)
2067	{
2068	const struct xfrm_tmpl *v = p;
2069
2070	switch (v->mode) {
2071	case XFRM_MODE_TRANSPORT:
2072	return 1;
2073	#if IS_ENABLED(CONFIG_IPV6_MIP6)
2074	case XFRM_MODE_ROUTEOPTIMIZATION:
2075	case XFRM_MODE_IN_TRIGGER:
2076	return 2;
2077	#endif
2078	case XFRM_MODE_TUNNEL:
2079	case XFRM_MODE_BEET:
2080	return 3;
2081	}
2082	return 4;
2083	}
2084	#else
2085	static inline int __xfrm6_state_sort_cmp(const void *p) { return 5; }
2086	static inline int __xfrm6_tmpl_sort_cmp(const void *p) { return 4; }
2087
2088	static inline void
2089	__xfrm6_sort(void **dst, void **src, int n,
2090	int (*cmp)(const void *p), int maxclass)
2091	{
2092	int i;
2093
2094	for (i = 0; i < n; i++)
2095	dst[i] = src[i];
2096	}
2097	#endif /* CONFIG_IPV6 */
2098
2099	void
2100	xfrm_tmpl_sort(struct xfrm_tmpl **dst, struct xfrm_tmpl **src, int n,
2101	unsigned short family)
2102	{
2103	int i;
2104
2105	if (family == AF_INET6)
2106	__xfrm6_sort(dst: (void **)dst, src: (void **)src, n,
2107	cmp: __xfrm6_tmpl_sort_cmp, maxclass: 5);
2108	else
2109	for (i = 0; i < n; i++)
2110	dst[i] = src[i];
2111	}
2112
2113	void
2114	xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n,
2115	unsigned short family)
2116	{
2117	int i;
2118
2119	if (family == AF_INET6)
2120	__xfrm6_sort(dst: (void **)dst, src: (void **)src, n,
2121	cmp: __xfrm6_state_sort_cmp, maxclass: 6);
2122	else
2123	for (i = 0; i < n; i++)
2124	dst[i] = src[i];
2125	}
2126	#endif
2127
2128	/* Silly enough, but I'm lazy to build resolution list */
2129
2130	static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq)
2131	{
2132	unsigned int h = xfrm_seq_hash(net, seq);
2133	struct xfrm_state *x;
2134
2135	hlist_for_each_entry_rcu(x, net->xfrm.state_byseq + h, byseq) {
2136	if (x->km.seq == seq &&
2137	(mark & x->mark.m) == x->mark.v &&
2138	x->km.state == XFRM_STATE_ACQ) {
2139	xfrm_state_hold(x);
2140	return x;
2141	}
2142	}
2143
2144	return NULL;
2145	}
2146
2147	struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq)
2148	{
2149	struct xfrm_state *x;
2150
2151	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
2152	x = __xfrm_find_acq_byseq(net, mark, seq);
2153	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
2154	return x;
2155	}
2156	EXPORT_SYMBOL(xfrm_find_acq_byseq);
2157
2158	u32 xfrm_get_acqseq(void)
2159	{
2160	u32 res;
2161	static atomic_t acqseq;
2162
2163	do {
2164	res = atomic_inc_return(v: &acqseq);
2165	} while (!res);
2166
2167	return res;
2168	}
2169	EXPORT_SYMBOL(xfrm_get_acqseq);
2170
2171	int verify_spi_info(u8 proto, u32 min, u32 max, struct netlink_ext_ack *extack)
2172	{
2173	switch (proto) {
2174	case IPPROTO_AH:
2175	case IPPROTO_ESP:
2176	break;
2177
2178	case IPPROTO_COMP:
2179	/* IPCOMP spi is 16-bits. */
2180	if (max >= 0x10000) {
2181	NL_SET_ERR_MSG(extack, "IPCOMP SPI must be <= 65535");
2182	return -EINVAL;
2183	}
2184	break;
2185
2186	default:
2187	NL_SET_ERR_MSG(extack, "Invalid protocol, must be one of AH, ESP, IPCOMP");
2188	return -EINVAL;
2189	}
2190
2191	if (min > max) {
2192	NL_SET_ERR_MSG(extack, "Invalid SPI range: min > max");
2193	return -EINVAL;
2194	}
2195
2196	return 0;
2197	}
2198	EXPORT_SYMBOL(verify_spi_info);
2199
2200	int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high,
2201	struct netlink_ext_ack *extack)
2202	{
2203	struct net *net = xs_net(x);
2204	unsigned int h;
2205	struct xfrm_state *x0;
2206	int err = -ENOENT;
2207	__be32 minspi = htonl(low);
2208	__be32 maxspi = htonl(high);
2209	__be32 newspi = 0;
2210	u32 mark = x->mark.v & x->mark.m;
2211
2212	spin_lock_bh(lock: &x->lock);
2213	if (x->km.state == XFRM_STATE_DEAD) {
2214	NL_SET_ERR_MSG(extack, "Target ACQUIRE is in DEAD state");
2215	goto unlock;
2216	}
2217
2218	err = 0;
2219	if (x->id.spi)
2220	goto unlock;
2221
2222	err = -ENOENT;
2223
2224	if (minspi == maxspi) {
2225	x0 = xfrm_state_lookup(net, mark, &x->id.daddr, minspi, x->id.proto, x->props.family);
2226	if (x0) {
2227	NL_SET_ERR_MSG(extack, "Requested SPI is already in use");
2228	xfrm_state_put(x: x0);
2229	goto unlock;
2230	}
2231	newspi = minspi;
2232	} else {
2233	u32 spi = 0;
2234	for (h = 0; h < high-low+1; h++) {
2235	spi = get_random_u32_inclusive(floor: low, ceil: high);
2236	x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family);
2237	if (x0 == NULL) {
2238	newspi = htonl(spi);
2239	break;
2240	}
2241	xfrm_state_put(x: x0);
2242	}
2243	}
2244	if (newspi) {
2245	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
2246	x->id.spi = newspi;
2247	h = xfrm_spi_hash(net, daddr: &x->id.daddr, spi: x->id.spi, proto: x->id.proto, family: x->props.family);
2248	XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,
2249	x->xso.type);
2250	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
2251
2252	err = 0;
2253	} else {
2254	NL_SET_ERR_MSG(extack, "No SPI available in the requested range");
2255	}
2256
2257	unlock:
2258	spin_unlock_bh(lock: &x->lock);
2259
2260	return err;
2261	}
2262	EXPORT_SYMBOL(xfrm_alloc_spi);
2263
2264	static bool __xfrm_state_filter_match(struct xfrm_state *x,
2265	struct xfrm_address_filter *filter)
2266	{
2267	if (filter) {
2268	if ((filter->family == AF_INET ||
2269	filter->family == AF_INET6) &&
2270	x->props.family != filter->family)
2271	return false;
2272
2273	return addr_match(token1: &x->props.saddr, token2: &filter->saddr,
2274	prefixlen: filter->splen) &&
2275	addr_match(token1: &x->id.daddr, token2: &filter->daddr,
2276	prefixlen: filter->dplen);
2277	}
2278	return true;
2279	}
2280
2281	int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk,
2282	int (*func)(struct xfrm_state *, int, void*),
2283	void *data)
2284	{
2285	struct xfrm_state *state;
2286	struct xfrm_state_walk *x;
2287	int err = 0;
2288
2289	if (walk->seq != 0 && list_empty(head: &walk->all))
2290	return 0;
2291
2292	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
2293	if (list_empty(head: &walk->all))
2294	x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);
2295	else
2296	x = list_first_entry(&walk->all, struct xfrm_state_walk, all);
2297	list_for_each_entry_from(x, &net->xfrm.state_all, all) {
2298	if (x->state == XFRM_STATE_DEAD)
2299	continue;
2300	state = container_of(x, struct xfrm_state, km);
2301	if (!xfrm_id_proto_match(proto: state->id.proto, userproto: walk->proto))
2302	continue;
2303	if (!__xfrm_state_filter_match(x: state, filter: walk->filter))
2304	continue;
2305	err = func(state, walk->seq, data);
2306	if (err) {
2307	list_move_tail(list: &walk->all, head: &x->all);
2308	goto out;
2309	}
2310	walk->seq++;
2311	}
2312	if (walk->seq == 0) {
2313	err = -ENOENT;
2314	goto out;
2315	}
2316	list_del_init(entry: &walk->all);
2317	out:
2318	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
2319	return err;
2320	}
2321	EXPORT_SYMBOL(xfrm_state_walk);
2322
2323	void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto,
2324	struct xfrm_address_filter *filter)
2325	{
2326	INIT_LIST_HEAD(list: &walk->all);
2327	walk->proto = proto;
2328	walk->state = XFRM_STATE_DEAD;
2329	walk->seq = 0;
2330	walk->filter = filter;
2331	}
2332	EXPORT_SYMBOL(xfrm_state_walk_init);
2333
2334	void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net)
2335	{
2336	kfree(objp: walk->filter);
2337
2338	if (list_empty(head: &walk->all))
2339	return;
2340
2341	spin_lock_bh(lock: &net->xfrm.xfrm_state_lock);
2342	list_del(entry: &walk->all);
2343	spin_unlock_bh(lock: &net->xfrm.xfrm_state_lock);
2344	}
2345	EXPORT_SYMBOL(xfrm_state_walk_done);
2346
2347	static void xfrm_replay_timer_handler(struct timer_list *t)
2348	{
2349	struct xfrm_state *x = from_timer(x, t, rtimer);
2350
2351	spin_lock(lock: &x->lock);
2352
2353	if (x->km.state == XFRM_STATE_VALID) {
2354	if (xfrm_aevent_is_on(net: xs_net(x)))
2355	xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT);
2356	else
2357	x->xflags |= XFRM_TIME_DEFER;
2358	}
2359
2360	spin_unlock(lock: &x->lock);
2361	}
2362
2363	static LIST_HEAD(xfrm_km_list);
2364
2365	void km_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c)
2366	{
2367	struct xfrm_mgr *km;
2368
2369	rcu_read_lock();
2370	list_for_each_entry_rcu(km, &xfrm_km_list, list)
2371	if (km->notify_policy)
2372	km->notify_policy(xp, dir, c);
2373	rcu_read_unlock();
2374	}
2375
2376	void km_state_notify(struct xfrm_state *x, const struct km_event *c)
2377	{
2378	struct xfrm_mgr *km;
2379	rcu_read_lock();
2380	list_for_each_entry_rcu(km, &xfrm_km_list, list)
2381	if (km->notify)
2382	km->notify(x, c);
2383	rcu_read_unlock();
2384	}
2385
2386	EXPORT_SYMBOL(km_policy_notify);
2387	EXPORT_SYMBOL(km_state_notify);
2388
2389	void km_state_expired(struct xfrm_state *x, int hard, u32 portid)
2390	{
2391	struct km_event c;
2392
2393	c.data.hard = hard;
2394	c.portid = portid;
2395	c.event = XFRM_MSG_EXPIRE;
2396	km_state_notify(x, &c);
2397	}
2398
2399	EXPORT_SYMBOL(km_state_expired);
2400	/*
2401	* We send to all registered managers regardless of failure
2402	* We are happy with one success
2403	*/
2404	int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol)
2405	{
2406	int err = -EINVAL, acqret;
2407	struct xfrm_mgr *km;
2408
2409	rcu_read_lock();
2410	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2411	acqret = km->acquire(x, t, pol);
2412	if (!acqret)
2413	err = acqret;
2414	}
2415	rcu_read_unlock();
2416	return err;
2417	}
2418	EXPORT_SYMBOL(km_query);
2419
2420	static int __km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
2421	{
2422	int err = -EINVAL;
2423	struct xfrm_mgr *km;
2424
2425	rcu_read_lock();
2426	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2427	if (km->new_mapping)
2428	err = km->new_mapping(x, ipaddr, sport);
2429	if (!err)
2430	break;
2431	}
2432	rcu_read_unlock();
2433	return err;
2434	}
2435
2436	int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
2437	{
2438	int ret = 0;
2439
2440	if (x->mapping_maxage) {
2441	if ((jiffies / HZ - x->new_mapping) > x->mapping_maxage ||
2442	x->new_mapping_sport != sport) {
2443	x->new_mapping_sport = sport;
2444	x->new_mapping = jiffies / HZ;
2445	ret = __km_new_mapping(x, ipaddr, sport);
2446	}
2447	} else {
2448	ret = __km_new_mapping(x, ipaddr, sport);
2449	}
2450
2451	return ret;
2452	}
2453	EXPORT_SYMBOL(km_new_mapping);
2454
2455	void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid)
2456	{
2457	struct km_event c;
2458
2459	c.data.hard = hard;
2460	c.portid = portid;
2461	c.event = XFRM_MSG_POLEXPIRE;
2462	km_policy_notify(pol, dir, &c);
2463	}
2464	EXPORT_SYMBOL(km_policy_expired);
2465
2466	#ifdef CONFIG_XFRM_MIGRATE
2467	int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
2468	const struct xfrm_migrate *m, int num_migrate,
2469	const struct xfrm_kmaddress *k,
2470	const struct xfrm_encap_tmpl *encap)
2471	{
2472	int err = -EINVAL;
2473	int ret;
2474	struct xfrm_mgr *km;
2475
2476	rcu_read_lock();
2477	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2478	if (km->migrate) {
2479	ret = km->migrate(sel, dir, type, m, num_migrate, k,
2480	encap);
2481	if (!ret)
2482	err = ret;
2483	}
2484	}
2485	rcu_read_unlock();
2486	return err;
2487	}
2488	EXPORT_SYMBOL(km_migrate);
2489	#endif
2490
2491	int km_report(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr)
2492	{
2493	int err = -EINVAL;
2494	int ret;
2495	struct xfrm_mgr *km;
2496
2497	rcu_read_lock();
2498	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2499	if (km->report) {
2500	ret = km->report(net, proto, sel, addr);
2501	if (!ret)
2502	err = ret;
2503	}
2504	}
2505	rcu_read_unlock();
2506	return err;
2507	}
2508	EXPORT_SYMBOL(km_report);
2509
2510	static bool km_is_alive(const struct km_event *c)
2511	{
2512	struct xfrm_mgr *km;
2513	bool is_alive = false;
2514
2515	rcu_read_lock();
2516	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2517	if (km->is_alive && km->is_alive(c)) {
2518	is_alive = true;
2519	break;
2520	}
2521	}
2522	rcu_read_unlock();
2523
2524	return is_alive;
2525	}
2526
2527	#if IS_ENABLED(CONFIG_XFRM_USER_COMPAT)
2528	static DEFINE_SPINLOCK(xfrm_translator_lock);
2529	static struct xfrm_translator __rcu *xfrm_translator;
2530
2531	struct xfrm_translator *xfrm_get_translator(void)
2532	{
2533	struct xfrm_translator *xtr;
2534
2535	rcu_read_lock();
2536	xtr = rcu_dereference(xfrm_translator);
2537	if (unlikely(!xtr))
2538	goto out;
2539	if (!try_module_get(module: xtr->owner))
2540	xtr = NULL;
2541	out:
2542	rcu_read_unlock();
2543	return xtr;
2544	}
2545	EXPORT_SYMBOL_GPL(xfrm_get_translator);
2546
2547	void xfrm_put_translator(struct xfrm_translator *xtr)
2548	{
2549	module_put(module: xtr->owner);
2550	}
2551	EXPORT_SYMBOL_GPL(xfrm_put_translator);
2552
2553	int xfrm_register_translator(struct xfrm_translator *xtr)
2554	{
2555	int err = 0;
2556
2557	spin_lock_bh(lock: &xfrm_translator_lock);
2558	if (unlikely(xfrm_translator != NULL))
2559	err = -EEXIST;
2560	else
2561	rcu_assign_pointer(xfrm_translator, xtr);
2562	spin_unlock_bh(lock: &xfrm_translator_lock);
2563
2564	return err;
2565	}
2566	EXPORT_SYMBOL_GPL(xfrm_register_translator);
2567
2568	int xfrm_unregister_translator(struct xfrm_translator *xtr)
2569	{
2570	int err = 0;
2571
2572	spin_lock_bh(lock: &xfrm_translator_lock);
2573	if (likely(xfrm_translator != NULL)) {
2574	if (rcu_access_pointer(xfrm_translator) != xtr)
2575	err = -EINVAL;
2576	else
2577	RCU_INIT_POINTER(xfrm_translator, NULL);
2578	}
2579	spin_unlock_bh(lock: &xfrm_translator_lock);
2580	synchronize_rcu();
2581
2582	return err;
2583	}
2584	EXPORT_SYMBOL_GPL(xfrm_unregister_translator);
2585	#endif
2586
2587	int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen)
2588	{
2589	int err;
2590	u8 *data;
2591	struct xfrm_mgr *km;
2592	struct xfrm_policy *pol = NULL;
2593
2594	if (sockptr_is_null(sockptr: optval) && !optlen) {
2595	xfrm_sk_policy_insert(sk, dir: XFRM_POLICY_IN, NULL);
2596	xfrm_sk_policy_insert(sk, dir: XFRM_POLICY_OUT, NULL);
2597	__sk_dst_reset(sk);
2598	return 0;
2599	}
2600
2601	if (optlen <= 0 || optlen > PAGE_SIZE)
2602	return -EMSGSIZE;
2603
2604	data = memdup_sockptr(src: optval, len: optlen);
2605	if (IS_ERR(ptr: data))
2606	return PTR_ERR(ptr: data);
2607
2608	if (in_compat_syscall()) {
2609	struct xfrm_translator *xtr = xfrm_get_translator();
2610
2611	if (!xtr) {
2612	kfree(objp: data);
2613	return -EOPNOTSUPP;
2614	}
2615
2616	err = xtr->xlate_user_policy_sockptr(&data, optlen);
2617	xfrm_put_translator(xtr);
2618	if (err) {
2619	kfree(objp: data);
2620	return err;
2621	}
2622	}
2623
2624	err = -EINVAL;
2625	rcu_read_lock();
2626	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2627	pol = km->compile_policy(sk, optname, data,
2628	optlen, &err);
2629	if (err >= 0)
2630	break;
2631	}
2632	rcu_read_unlock();
2633
2634	if (err >= 0) {
2635	xfrm_sk_policy_insert(sk, dir: err, pol);
2636	xfrm_pol_put(policy: pol);
2637	__sk_dst_reset(sk);
2638	err = 0;
2639	}
2640
2641	kfree(objp: data);
2642	return err;
2643	}
2644	EXPORT_SYMBOL(xfrm_user_policy);
2645
2646	static DEFINE_SPINLOCK(xfrm_km_lock);
2647
2648	void xfrm_register_km(struct xfrm_mgr *km)
2649	{
2650	spin_lock_bh(lock: &xfrm_km_lock);
2651	list_add_tail_rcu(new: &km->list, head: &xfrm_km_list);
2652	spin_unlock_bh(lock: &xfrm_km_lock);
2653	}
2654	EXPORT_SYMBOL(xfrm_register_km);
2655
2656	void xfrm_unregister_km(struct xfrm_mgr *km)
2657	{
2658	spin_lock_bh(lock: &xfrm_km_lock);
2659	list_del_rcu(entry: &km->list);
2660	spin_unlock_bh(lock: &xfrm_km_lock);
2661	synchronize_rcu();
2662	}
2663	EXPORT_SYMBOL(xfrm_unregister_km);
2664
2665	int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo)
2666	{
2667	int err = 0;
2668
2669	if (WARN_ON(afinfo->family >= NPROTO))
2670	return -EAFNOSUPPORT;
2671
2672	spin_lock_bh(lock: &xfrm_state_afinfo_lock);
2673	if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL))
2674	err = -EEXIST;
2675	else
2676	rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo);
2677	spin_unlock_bh(lock: &xfrm_state_afinfo_lock);
2678	return err;
2679	}
2680	EXPORT_SYMBOL(xfrm_state_register_afinfo);
2681
2682	int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo)
2683	{
2684	int err = 0, family = afinfo->family;
2685
2686	if (WARN_ON(family >= NPROTO))
2687	return -EAFNOSUPPORT;
2688
2689	spin_lock_bh(lock: &xfrm_state_afinfo_lock);
2690	if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) {
2691	if (rcu_access_pointer(xfrm_state_afinfo[family]) != afinfo)
2692	err = -EINVAL;
2693	else
2694	RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL);
2695	}
2696	spin_unlock_bh(lock: &xfrm_state_afinfo_lock);
2697	synchronize_rcu();
2698	return err;
2699	}
2700	EXPORT_SYMBOL(xfrm_state_unregister_afinfo);
2701
2702	struct xfrm_state_afinfo *xfrm_state_afinfo_get_rcu(unsigned int family)
2703	{
2704	if (unlikely(family >= NPROTO))
2705	return NULL;
2706
2707	return rcu_dereference(xfrm_state_afinfo[family]);
2708	}
2709	EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu);
2710
2711	struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family)
2712	{
2713	struct xfrm_state_afinfo *afinfo;
2714	if (unlikely(family >= NPROTO))
2715	return NULL;
2716	rcu_read_lock();
2717	afinfo = rcu_dereference(xfrm_state_afinfo[family]);
2718	if (unlikely(!afinfo))
2719	rcu_read_unlock();
2720	return afinfo;
2721	}
2722
2723	void xfrm_flush_gc(void)
2724	{
2725	flush_work(work: &xfrm_state_gc_work);
2726	}
2727	EXPORT_SYMBOL(xfrm_flush_gc);
2728
2729	/* Temporarily located here until net/xfrm/xfrm_tunnel.c is created */
2730	void xfrm_state_delete_tunnel(struct xfrm_state *x)
2731	{
2732	if (x->tunnel) {
2733	struct xfrm_state *t = x->tunnel;
2734
2735	if (atomic_read(v: &t->tunnel_users) == 2)
2736	xfrm_state_delete(t);
2737	atomic_dec(v: &t->tunnel_users);
2738	xfrm_state_put_sync(x: t);
2739	x->tunnel = NULL;
2740	}
2741	}
2742	EXPORT_SYMBOL(xfrm_state_delete_tunnel);
2743
2744	u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
2745	{
2746	const struct xfrm_type *type = READ_ONCE(x->type);
2747	struct crypto_aead *aead;
2748	u32 blksize, net_adj = 0;
2749
2750	if (x->km.state != XFRM_STATE_VALID ||
2751	!type || type->proto != IPPROTO_ESP)
2752	return mtu - x->props.header_len;
2753
2754	aead = x->data;
2755	blksize = ALIGN(crypto_aead_blocksize(aead), 4);
2756
2757	switch (x->props.mode) {
2758	case XFRM_MODE_TRANSPORT:
2759	case XFRM_MODE_BEET:
2760	if (x->props.family == AF_INET)
2761	net_adj = sizeof(struct iphdr);
2762	else if (x->props.family == AF_INET6)
2763	net_adj = sizeof(struct ipv6hdr);
2764	break;
2765	case XFRM_MODE_TUNNEL:
2766	break;
2767	default:
2768	WARN_ON_ONCE(1);
2769	break;
2770	}
2771
2772	return ((mtu - x->props.header_len - crypto_aead_authsize(tfm: aead) -
2773	net_adj) & ~(blksize - 1)) + net_adj - 2;
2774	}
2775	EXPORT_SYMBOL_GPL(xfrm_state_mtu);
2776
2777	int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload,
2778	struct netlink_ext_ack *extack)
2779	{
2780	const struct xfrm_mode *inner_mode;
2781	const struct xfrm_mode *outer_mode;
2782	int family = x->props.family;
2783	int err;
2784
2785	if (family == AF_INET &&
2786	READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc))
2787	x->props.flags |= XFRM_STATE_NOPMTUDISC;
2788
2789	err = -EPROTONOSUPPORT;
2790
2791	if (x->sel.family != AF_UNSPEC) {
2792	inner_mode = xfrm_get_mode(encap: x->props.mode, family: x->sel.family);
2793	if (inner_mode == NULL) {
2794	NL_SET_ERR_MSG(extack, "Requested mode not found");
2795	goto error;
2796	}
2797
2798	if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) &&
2799	family != x->sel.family) {
2800	NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate a change of family");
2801	goto error;
2802	}
2803
2804	x->inner_mode = *inner_mode;
2805	} else {
2806	const struct xfrm_mode *inner_mode_iaf;
2807	int iafamily = AF_INET;
2808
2809	inner_mode = xfrm_get_mode(encap: x->props.mode, family: x->props.family);
2810	if (inner_mode == NULL) {
2811	NL_SET_ERR_MSG(extack, "Requested mode not found");
2812	goto error;
2813	}
2814
2815	x->inner_mode = *inner_mode;
2816
2817	if (x->props.family == AF_INET)
2818	iafamily = AF_INET6;
2819
2820	inner_mode_iaf = xfrm_get_mode(encap: x->props.mode, family: iafamily);
2821	if (inner_mode_iaf) {
2822	if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL)
2823	x->inner_mode_iaf = *inner_mode_iaf;
2824	}
2825	}
2826
2827	x->type = xfrm_get_type(proto: x->id.proto, family);
2828	if (x->type == NULL) {
2829	NL_SET_ERR_MSG(extack, "Requested type not found");
2830	goto error;
2831	}
2832
2833	x->type_offload = xfrm_get_type_offload(proto: x->id.proto, family, try_load: offload);
2834
2835	err = x->type->init_state(x, extack);
2836	if (err)
2837	goto error;
2838
2839	outer_mode = xfrm_get_mode(encap: x->props.mode, family);
2840	if (!outer_mode) {
2841	NL_SET_ERR_MSG(extack, "Requested mode not found");
2842	err = -EPROTONOSUPPORT;
2843	goto error;
2844	}
2845
2846	x->outer_mode = *outer_mode;
2847	if (init_replay) {
2848	err = xfrm_init_replay(x, extack);
2849	if (err)
2850	goto error;
2851	}
2852
2853	error:
2854	return err;
2855	}
2856
2857	EXPORT_SYMBOL(__xfrm_init_state);
2858
2859	int xfrm_init_state(struct xfrm_state *x)
2860	{
2861	int err;
2862
2863	err = __xfrm_init_state(x, true, false, NULL);
2864	if (!err)
2865	x->km.state = XFRM_STATE_VALID;
2866
2867	return err;
2868	}
2869
2870	EXPORT_SYMBOL(xfrm_init_state);
2871
2872	int __net_init xfrm_state_init(struct net *net)
2873	{
2874	unsigned int sz;
2875
2876	if (net_eq(net1: net, net2: &init_net))
2877	xfrm_state_cache = KMEM_CACHE(xfrm_state,
2878	SLAB_HWCACHE_ALIGN | SLAB_PANIC);
2879
2880	INIT_LIST_HEAD(list: &net->xfrm.state_all);
2881
2882	sz = sizeof(struct hlist_head) * 8;
2883
2884	net->xfrm.state_bydst = xfrm_hash_alloc(sz);
2885	if (!net->xfrm.state_bydst)
2886	goto out_bydst;
2887	net->xfrm.state_bysrc = xfrm_hash_alloc(sz);
2888	if (!net->xfrm.state_bysrc)
2889	goto out_bysrc;
2890	net->xfrm.state_byspi = xfrm_hash_alloc(sz);
2891	if (!net->xfrm.state_byspi)
2892	goto out_byspi;
2893	net->xfrm.state_byseq = xfrm_hash_alloc(sz);
2894	if (!net->xfrm.state_byseq)
2895	goto out_byseq;
2896	net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);
2897
2898	net->xfrm.state_num = 0;
2899	INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);
2900	spin_lock_init(&net->xfrm.xfrm_state_lock);
2901	seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation,
2902	&net->xfrm.xfrm_state_lock);
2903	return 0;
2904
2905	out_byseq:
2906	xfrm_hash_free(n: net->xfrm.state_byspi, sz);
2907	out_byspi:
2908	xfrm_hash_free(n: net->xfrm.state_bysrc, sz);
2909	out_bysrc:
2910	xfrm_hash_free(n: net->xfrm.state_bydst, sz);
2911	out_bydst:
2912	return -ENOMEM;
2913	}
2914
2915	void xfrm_state_fini(struct net *net)
2916	{
2917	unsigned int sz;
2918
2919	flush_work(work: &net->xfrm.state_hash_work);
2920	flush_work(work: &xfrm_state_gc_work);
2921	xfrm_state_flush(net, 0, false, true);
2922
2923	WARN_ON(!list_empty(&net->xfrm.state_all));
2924
2925	sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);
2926	WARN_ON(!hlist_empty(net->xfrm.state_byseq));
2927	xfrm_hash_free(n: net->xfrm.state_byseq, sz);
2928	WARN_ON(!hlist_empty(net->xfrm.state_byspi));
2929	xfrm_hash_free(n: net->xfrm.state_byspi, sz);
2930	WARN_ON(!hlist_empty(net->xfrm.state_bysrc));
2931	xfrm_hash_free(n: net->xfrm.state_bysrc, sz);
2932	WARN_ON(!hlist_empty(net->xfrm.state_bydst));
2933	xfrm_hash_free(n: net->xfrm.state_bydst, sz);
2934	}
2935
2936	#ifdef CONFIG_AUDITSYSCALL
2937	static void xfrm_audit_helper_sainfo(struct xfrm_state *x,
2938	struct audit_buffer *audit_buf)
2939	{
2940	struct xfrm_sec_ctx *ctx = x->security;
2941	u32 spi = ntohl(x->id.spi);
2942
2943	if (ctx)
2944	audit_log_format(ab: audit_buf, fmt: " sec_alg=%u sec_doi=%u sec_obj=%s",
2945	ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
2946
2947	switch (x->props.family) {
2948	case AF_INET:
2949	audit_log_format(ab: audit_buf, fmt: " src=%pI4 dst=%pI4",
2950	&x->props.saddr.a4, &x->id.daddr.a4);
2951	break;
2952	case AF_INET6:
2953	audit_log_format(ab: audit_buf, fmt: " src=%pI6 dst=%pI6",
2954	x->props.saddr.a6, x->id.daddr.a6);
2955	break;
2956	}
2957
2958	audit_log_format(ab: audit_buf, fmt: " spi=%u(0x%x)", spi, spi);
2959	}
2960
2961	static void xfrm_audit_helper_pktinfo(struct sk_buff *skb, u16 family,
2962	struct audit_buffer *audit_buf)
2963	{
2964	const struct iphdr *iph4;
2965	const struct ipv6hdr *iph6;
2966
2967	switch (family) {
2968	case AF_INET:
2969	iph4 = ip_hdr(skb);
2970	audit_log_format(ab: audit_buf, fmt: " src=%pI4 dst=%pI4",
2971	&iph4->saddr, &iph4->daddr);
2972	break;
2973	case AF_INET6:
2974	iph6 = ipv6_hdr(skb);
2975	audit_log_format(ab: audit_buf,
2976	fmt: " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x",
2977	&iph6->saddr, &iph6->daddr,
2978	iph6->flow_lbl[0] & 0x0f,
2979	iph6->flow_lbl[1],
2980	iph6->flow_lbl[2]);
2981	break;
2982	}
2983	}
2984
2985	void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid)
2986	{
2987	struct audit_buffer *audit_buf;
2988
2989	audit_buf = xfrm_audit_start(op: "SAD-add");
2990	if (audit_buf == NULL)
2991	return;
2992	xfrm_audit_helper_usrinfo(task_valid, audit_buf);
2993	xfrm_audit_helper_sainfo(x, audit_buf);
2994	audit_log_format(ab: audit_buf, fmt: " res=%u", result);
2995	audit_log_end(ab: audit_buf);
2996	}
2997	EXPORT_SYMBOL_GPL(xfrm_audit_state_add);
2998
2999	void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid)
3000	{
3001	struct audit_buffer *audit_buf;
3002
3003	audit_buf = xfrm_audit_start(op: "SAD-delete");
3004	if (audit_buf == NULL)
3005	return;
3006	xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3007	xfrm_audit_helper_sainfo(x, audit_buf);
3008	audit_log_format(ab: audit_buf, fmt: " res=%u", result);
3009	audit_log_end(ab: audit_buf);
3010	}
3011	EXPORT_SYMBOL_GPL(xfrm_audit_state_delete);
3012
3013	void xfrm_audit_state_replay_overflow(struct xfrm_state *x,
3014	struct sk_buff *skb)
3015	{
3016	struct audit_buffer *audit_buf;
3017	u32 spi;
3018
3019	audit_buf = xfrm_audit_start(op: "SA-replay-overflow");
3020	if (audit_buf == NULL)
3021	return;
3022	xfrm_audit_helper_pktinfo(skb, family: x->props.family, audit_buf);
3023	/* don't record the sequence number because it's inherent in this kind
3024	* of audit message */
3025	spi = ntohl(x->id.spi);
3026	audit_log_format(ab: audit_buf, fmt: " spi=%u(0x%x)", spi, spi);
3027	audit_log_end(ab: audit_buf);
3028	}
3029	EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow);
3030
3031	void xfrm_audit_state_replay(struct xfrm_state *x,
3032	struct sk_buff *skb, __be32 net_seq)
3033	{
3034	struct audit_buffer *audit_buf;
3035	u32 spi;
3036
3037	audit_buf = xfrm_audit_start(op: "SA-replayed-pkt");
3038	if (audit_buf == NULL)
3039	return;
3040	xfrm_audit_helper_pktinfo(skb, family: x->props.family, audit_buf);
3041	spi = ntohl(x->id.spi);
3042	audit_log_format(ab: audit_buf, fmt: " spi=%u(0x%x) seqno=%u",
3043	spi, spi, ntohl(net_seq));
3044	audit_log_end(ab: audit_buf);
3045	}
3046	EXPORT_SYMBOL_GPL(xfrm_audit_state_replay);
3047
3048	void xfrm_audit_state_notfound_simple(struct sk_buff *skb, u16 family)
3049	{
3050	struct audit_buffer *audit_buf;
3051
3052	audit_buf = xfrm_audit_start(op: "SA-notfound");
3053	if (audit_buf == NULL)
3054	return;
3055	xfrm_audit_helper_pktinfo(skb, family, audit_buf);
3056	audit_log_end(ab: audit_buf);
3057	}
3058	EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple);
3059
3060	void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family,
3061	__be32 net_spi, __be32 net_seq)
3062	{
3063	struct audit_buffer *audit_buf;
3064	u32 spi;
3065
3066	audit_buf = xfrm_audit_start(op: "SA-notfound");
3067	if (audit_buf == NULL)
3068	return;
3069	xfrm_audit_helper_pktinfo(skb, family, audit_buf);
3070	spi = ntohl(net_spi);
3071	audit_log_format(ab: audit_buf, fmt: " spi=%u(0x%x) seqno=%u",
3072	spi, spi, ntohl(net_seq));
3073	audit_log_end(ab: audit_buf);
3074	}
3075	EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound);
3076
3077	void xfrm_audit_state_icvfail(struct xfrm_state *x,
3078	struct sk_buff *skb, u8 proto)
3079	{
3080	struct audit_buffer *audit_buf;
3081	__be32 net_spi;
3082	__be32 net_seq;
3083
3084	audit_buf = xfrm_audit_start(op: "SA-icv-failure");
3085	if (audit_buf == NULL)
3086	return;
3087	xfrm_audit_helper_pktinfo(skb, family: x->props.family, audit_buf);
3088	if (xfrm_parse_spi(skb, nexthdr: proto, spi: &net_spi, seq: &net_seq) == 0) {
3089	u32 spi = ntohl(net_spi);
3090	audit_log_format(ab: audit_buf, fmt: " spi=%u(0x%x) seqno=%u",
3091	spi, spi, ntohl(net_seq));
3092	}
3093	audit_log_end(ab: audit_buf);
3094	}
3095	EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail);
3096	#endif /* CONFIG_AUDITSYSCALL */
3097