1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* INET An implementation of the TCP Authentication Option (TCP-AO).
4	* See RFC5925.
5	*
6	* Authors: Dmitry Safonov <dima@arista.com>
7	* Francesco Ruggeri <fruggeri@arista.com>
8	* Salam Noureddine <noureddine@arista.com>
9	*/
10	#define pr_fmt(fmt) "TCP: " fmt
11
12	#include <crypto/hash.h>
13	#include <linux/inetdevice.h>
14	#include <linux/tcp.h>
15
16	#include <net/tcp.h>
17	#include <net/ipv6.h>
18	#include <net/icmp.h>
19
20	DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_ao_needed, HZ);
21
22	int tcp_ao_calc_traffic_key(struct tcp_ao_key *mkt, u8 *key, void *ctx,
23	unsigned int len, struct tcp_sigpool *hp)
24	{
25	struct scatterlist sg;
26	int ret;
27
28	if (crypto_ahash_setkey(tfm: crypto_ahash_reqtfm(req: hp->req),
29	key: mkt->key, keylen: mkt->keylen))
30	goto clear_hash;
31
32	ret = crypto_ahash_init(req: hp->req);
33	if (ret)
34	goto clear_hash;
35
36	sg_init_one(&sg, ctx, len);
37	ahash_request_set_crypt(req: hp->req, src: &sg, result: key, nbytes: len);
38	crypto_ahash_update(req: hp->req);
39
40	ret = crypto_ahash_final(req: hp->req);
41	if (ret)
42	goto clear_hash;
43
44	return 0;
45	clear_hash:
46	memset(key, 0, tcp_ao_digest_size(mkt));
47	return 1;
48	}
49
50	bool tcp_ao_ignore_icmp(const struct sock *sk, int family, int type, int code)
51	{
52	bool ignore_icmp = false;
53	struct tcp_ao_info *ao;
54
55	if (!static_branch_unlikely(&tcp_ao_needed.key))
56	return false;
57
58	/* RFC5925, 7.8:
59	* >> A TCP-AO implementation MUST default to ignore incoming ICMPv4
60	* messages of Type 3 (destination unreachable), Codes 2-4 (protocol
61	* unreachable, port unreachable, and fragmentation needed -- ’hard
62	* errors’), and ICMPv6 Type 1 (destination unreachable), Code 1
63	* (administratively prohibited) and Code 4 (port unreachable) intended
64	* for connections in synchronized states (ESTABLISHED, FIN-WAIT-1, FIN-
65	* WAIT-2, CLOSE-WAIT, CLOSING, LAST-ACK, TIME-WAIT) that match MKTs.
66	*/
67	if (family == AF_INET) {
68	if (type != ICMP_DEST_UNREACH)
69	return false;
70	if (code < ICMP_PROT_UNREACH || code > ICMP_FRAG_NEEDED)
71	return false;
72	} else {
73	if (type != ICMPV6_DEST_UNREACH)
74	return false;
75	if (code != ICMPV6_ADM_PROHIBITED && code != ICMPV6_PORT_UNREACH)
76	return false;
77	}
78
79	rcu_read_lock();
80	switch (sk->sk_state) {
81	case TCP_TIME_WAIT:
82	ao = rcu_dereference(tcp_twsk(sk)->ao_info);
83	break;
84	case TCP_SYN_SENT:
85	case TCP_SYN_RECV:
86	case TCP_LISTEN:
87	case TCP_NEW_SYN_RECV:
88	/* RFC5925 specifies to ignore ICMPs *only* on connections
89	* in synchronized states.
90	*/
91	rcu_read_unlock();
92	return false;
93	default:
94	ao = rcu_dereference(tcp_sk(sk)->ao_info);
95	}
96
97	if (ao && !ao->accept_icmps) {
98	ignore_icmp = true;
99	__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAODROPPEDICMPS);
100	atomic64_inc(v: &ao->counters.dropped_icmp);
101	}
102	rcu_read_unlock();
103
104	return ignore_icmp;
105	}
106
107	/* Optimized version of tcp_ao_do_lookup(): only for sockets for which
108	* it's known that the keys in ao_info are matching peer's
109	* family/address/VRF/etc.
110	*/
111	struct tcp_ao_key *tcp_ao_established_key(struct tcp_ao_info *ao,
112	int sndid, int rcvid)
113	{
114	struct tcp_ao_key *key;
115
116	hlist_for_each_entry_rcu(key, &ao->head, node) {
117	if ((sndid >= 0 && key->sndid != sndid) ||
118	(rcvid >= 0 && key->rcvid != rcvid))
119	continue;
120	return key;
121	}
122
123	return NULL;
124	}
125
126	static int ipv4_prefix_cmp(const struct in_addr *addr1,
127	const struct in_addr *addr2,
128	unsigned int prefixlen)
129	{
130	__be32 mask = inet_make_mask(logmask: prefixlen);
131	__be32 a1 = addr1->s_addr & mask;
132	__be32 a2 = addr2->s_addr & mask;
133
134	if (a1 == a2)
135	return 0;
136	return memcmp(p: &a1, q: &a2, size: sizeof(a1));
137	}
138
139	static int __tcp_ao_key_cmp(const struct tcp_ao_key *key, int l3index,
140	const union tcp_ao_addr *addr, u8 prefixlen,
141	int family, int sndid, int rcvid)
142	{
143	if (sndid >= 0 && key->sndid != sndid)
144	return (key->sndid > sndid) ? 1 : -1;
145	if (rcvid >= 0 && key->rcvid != rcvid)
146	return (key->rcvid > rcvid) ? 1 : -1;
147	if (l3index >= 0 && (key->keyflags & TCP_AO_KEYF_IFINDEX)) {
148	if (key->l3index != l3index)
149	return (key->l3index > l3index) ? 1 : -1;
150	}
151
152	if (family == AF_UNSPEC)
153	return 0;
154	if (key->family != family)
155	return (key->family > family) ? 1 : -1;
156
157	if (family == AF_INET) {
158	if (ntohl(key->addr.a4.s_addr) == INADDR_ANY)
159	return 0;
160	if (ntohl(addr->a4.s_addr) == INADDR_ANY)
161	return 0;
162	return ipv4_prefix_cmp(addr1: &key->addr.a4, addr2: &addr->a4, prefixlen);
163	#if IS_ENABLED(CONFIG_IPV6)
164	} else {
165	if (ipv6_addr_any(a: &key->addr.a6) || ipv6_addr_any(a: &addr->a6))
166	return 0;
167	if (ipv6_prefix_equal(addr1: &key->addr.a6, addr2: &addr->a6, prefixlen))
168	return 0;
169	return memcmp(p: &key->addr.a6, q: &addr->a6, size: sizeof(addr->a6));
170	#endif
171	}
172	return -1;
173	}
174
175	static int tcp_ao_key_cmp(const struct tcp_ao_key *key, int l3index,
176	const union tcp_ao_addr *addr, u8 prefixlen,
177	int family, int sndid, int rcvid)
178	{
179	#if IS_ENABLED(CONFIG_IPV6)
180	if (family == AF_INET6 && ipv6_addr_v4mapped(a: &addr->a6)) {
181	__be32 addr4 = addr->a6.s6_addr32[3];
182
183	return __tcp_ao_key_cmp(key, l3index,
184	addr: (union tcp_ao_addr *)&addr4,
185	prefixlen, AF_INET, sndid, rcvid);
186	}
187	#endif
188	return __tcp_ao_key_cmp(key, l3index, addr,
189	prefixlen, family, sndid, rcvid);
190	}
191
192	static struct tcp_ao_key *__tcp_ao_do_lookup(const struct sock *sk, int l3index,
193	const union tcp_ao_addr *addr, int family, u8 prefix,
194	int sndid, int rcvid)
195	{
196	struct tcp_ao_key *key;
197	struct tcp_ao_info *ao;
198
199	if (!static_branch_unlikely(&tcp_ao_needed.key))
200	return NULL;
201
202	ao = rcu_dereference_check(tcp_sk(sk)->ao_info,
203	lockdep_sock_is_held(sk));
204	if (!ao)
205	return NULL;
206
207	hlist_for_each_entry_rcu(key, &ao->head, node) {
208	u8 prefixlen = min(prefix, key->prefixlen);
209
210	if (!tcp_ao_key_cmp(key, l3index, addr, prefixlen,
211	family, sndid, rcvid))
212	return key;
213	}
214	return NULL;
215	}
216
217	struct tcp_ao_key *tcp_ao_do_lookup(const struct sock *sk, int l3index,
218	const union tcp_ao_addr *addr,
219	int family, int sndid, int rcvid)
220	{
221	return __tcp_ao_do_lookup(sk, l3index, addr, family, U8_MAX, sndid, rcvid);
222	}
223
224	static struct tcp_ao_info *tcp_ao_alloc_info(gfp_t flags)
225	{
226	struct tcp_ao_info *ao;
227
228	ao = kzalloc(size: sizeof(*ao), flags);
229	if (!ao)
230	return NULL;
231	INIT_HLIST_HEAD(&ao->head);
232	refcount_set(r: &ao->refcnt, n: 1);
233
234	return ao;
235	}
236
237	static void tcp_ao_link_mkt(struct tcp_ao_info *ao, struct tcp_ao_key *mkt)
238	{
239	hlist_add_head_rcu(n: &mkt->node, h: &ao->head);
240	}
241
242	static struct tcp_ao_key *tcp_ao_copy_key(struct sock *sk,
243	struct tcp_ao_key *key)
244	{
245	struct tcp_ao_key *new_key;
246
247	new_key = sock_kmalloc(sk, size: tcp_ao_sizeof_key(key),
248	GFP_ATOMIC);
249	if (!new_key)
250	return NULL;
251
252	*new_key = *key;
253	INIT_HLIST_NODE(h: &new_key->node);
254	tcp_sigpool_get(id: new_key->tcp_sigpool_id);
255	atomic64_set(v: &new_key->pkt_good, i: 0);
256	atomic64_set(v: &new_key->pkt_bad, i: 0);
257
258	return new_key;
259	}
260
261	static void tcp_ao_key_free_rcu(struct rcu_head *head)
262	{
263	struct tcp_ao_key *key = container_of(head, struct tcp_ao_key, rcu);
264
265	tcp_sigpool_release(id: key->tcp_sigpool_id);
266	kfree_sensitive(objp: key);
267	}
268
269	void tcp_ao_destroy_sock(struct sock *sk, bool twsk)
270	{
271	struct tcp_ao_info *ao;
272	struct tcp_ao_key *key;
273	struct hlist_node *n;
274
275	if (twsk) {
276	ao = rcu_dereference_protected(tcp_twsk(sk)->ao_info, 1);
277	tcp_twsk(sk)->ao_info = NULL;
278	} else {
279	ao = rcu_dereference_protected(tcp_sk(sk)->ao_info, 1);
280	tcp_sk(sk)->ao_info = NULL;
281	}
282
283	if (!ao || !refcount_dec_and_test(r: &ao->refcnt))
284	return;
285
286	hlist_for_each_entry_safe(key, n, &ao->head, node) {
287	hlist_del_rcu(n: &key->node);
288	if (!twsk)
289	atomic_sub(i: tcp_ao_sizeof_key(key), v: &sk->sk_omem_alloc);
290	call_rcu(head: &key->rcu, func: tcp_ao_key_free_rcu);
291	}
292
293	kfree_rcu(ao, rcu);
294	static_branch_slow_dec_deferred(&tcp_ao_needed);
295	}
296
297	void tcp_ao_time_wait(struct tcp_timewait_sock *tcptw, struct tcp_sock *tp)
298	{
299	struct tcp_ao_info *ao_info = rcu_dereference_protected(tp->ao_info, 1);
300
301	if (ao_info) {
302	struct tcp_ao_key *key;
303	struct hlist_node *n;
304	int omem = 0;
305
306	hlist_for_each_entry_safe(key, n, &ao_info->head, node) {
307	omem += tcp_ao_sizeof_key(key);
308	}
309
310	refcount_inc(r: &ao_info->refcnt);
311	atomic_sub(i: omem, v: &(((struct sock *)tp)->sk_omem_alloc));
312	rcu_assign_pointer(tcptw->ao_info, ao_info);
313	} else {
314	tcptw->ao_info = NULL;
315	}
316	}
317
318	/* 4 tuple and ISNs are expected in NBO */
319	static int tcp_v4_ao_calc_key(struct tcp_ao_key *mkt, u8 *key,
320	__be32 saddr, __be32 daddr,
321	__be16 sport, __be16 dport,
322	__be32 sisn, __be32 disn)
323	{
324	/* See RFC5926 3.1.1 */
325	struct kdf_input_block {
326	u8 counter;
327	u8 label[6];
328	struct tcp4_ao_context ctx;
329	__be16 outlen;
330	} __packed * tmp;
331	struct tcp_sigpool hp;
332	int err;
333
334	err = tcp_sigpool_start(id: mkt->tcp_sigpool_id, c: &hp);
335	if (err)
336	return err;
337
338	tmp = hp.scratch;
339	tmp->counter = 1;
340	memcpy(tmp->label, "TCP-AO", 6);
341	tmp->ctx.saddr = saddr;
342	tmp->ctx.daddr = daddr;
343	tmp->ctx.sport = sport;
344	tmp->ctx.dport = dport;
345	tmp->ctx.sisn = sisn;
346	tmp->ctx.disn = disn;
347	tmp->outlen = htons(tcp_ao_digest_size(mkt) * 8); /* in bits */
348
349	err = tcp_ao_calc_traffic_key(mkt, key, ctx: tmp, len: sizeof(*tmp), hp: &hp);
350	tcp_sigpool_end(c: &hp);
351
352	return err;
353	}
354
355	int tcp_v4_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
356	const struct sock *sk,
357	__be32 sisn, __be32 disn, bool send)
358	{
359	if (send)
360	return tcp_v4_ao_calc_key(mkt, key, saddr: sk->sk_rcv_saddr,
361	daddr: sk->sk_daddr, htons(sk->sk_num),
362	dport: sk->sk_dport, sisn, disn);
363	else
364	return tcp_v4_ao_calc_key(mkt, key, saddr: sk->sk_daddr,
365	daddr: sk->sk_rcv_saddr, sport: sk->sk_dport,
366	htons(sk->sk_num), sisn: disn, disn: sisn);
367	}
368
369	static int tcp_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
370	const struct sock *sk,
371	__be32 sisn, __be32 disn, bool send)
372	{
373	if (mkt->family == AF_INET)
374	return tcp_v4_ao_calc_key_sk(mkt, key, sk, sisn, disn, send);
375	#if IS_ENABLED(CONFIG_IPV6)
376	else if (mkt->family == AF_INET6)
377	return tcp_v6_ao_calc_key_sk(mkt, key, sk, sisn, disn, send);
378	#endif
379	else
380	return -EOPNOTSUPP;
381	}
382
383	int tcp_v4_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key,
384	struct request_sock *req)
385	{
386	struct inet_request_sock *ireq = inet_rsk(sk: req);
387
388	return tcp_v4_ao_calc_key(mkt, key,
389	saddr: ireq->ir_loc_addr, daddr: ireq->ir_rmt_addr,
390	htons(ireq->ir_num), dport: ireq->ir_rmt_port,
391	htonl(tcp_rsk(req)->snt_isn),
392	htonl(tcp_rsk(req)->rcv_isn));
393	}
394
395	static int tcp_v4_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
396	const struct sk_buff *skb,
397	__be32 sisn, __be32 disn)
398	{
399	const struct iphdr *iph = ip_hdr(skb);
400	const struct tcphdr *th = tcp_hdr(skb);
401
402	return tcp_v4_ao_calc_key(mkt, key, saddr: iph->saddr, daddr: iph->daddr,
403	sport: th->source, dport: th->dest, sisn, disn);
404	}
405
406	static int tcp_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
407	const struct sk_buff *skb,
408	__be32 sisn, __be32 disn, int family)
409	{
410	if (family == AF_INET)
411	return tcp_v4_ao_calc_key_skb(mkt, key, skb, sisn, disn);
412	#if IS_ENABLED(CONFIG_IPV6)
413	else if (family == AF_INET6)
414	return tcp_v6_ao_calc_key_skb(mkt, key, skb, sisn, disn);
415	#endif
416	return -EAFNOSUPPORT;
417	}
418
419	static int tcp_v4_ao_hash_pseudoheader(struct tcp_sigpool *hp,
420	__be32 daddr, __be32 saddr,
421	int nbytes)
422	{
423	struct tcp4_pseudohdr *bp;
424	struct scatterlist sg;
425
426	bp = hp->scratch;
427	bp->saddr = saddr;
428	bp->daddr = daddr;
429	bp->pad = 0;
430	bp->protocol = IPPROTO_TCP;
431	bp->len = cpu_to_be16(nbytes);
432
433	sg_init_one(&sg, bp, sizeof(*bp));
434	ahash_request_set_crypt(req: hp->req, src: &sg, NULL, nbytes: sizeof(*bp));
435	return crypto_ahash_update(req: hp->req);
436	}
437
438	static int tcp_ao_hash_pseudoheader(unsigned short int family,
439	const struct sock *sk,
440	const struct sk_buff *skb,
441	struct tcp_sigpool *hp, int nbytes)
442	{
443	const struct tcphdr *th = tcp_hdr(skb);
444
445	/* TODO: Can we rely on checksum being zero to mean outbound pkt? */
446	if (!th->check) {
447	if (family == AF_INET)
448	return tcp_v4_ao_hash_pseudoheader(hp, daddr: sk->sk_daddr,
449	saddr: sk->sk_rcv_saddr, nbytes: skb->len);
450	#if IS_ENABLED(CONFIG_IPV6)
451	else if (family == AF_INET6)
452	return tcp_v6_ao_hash_pseudoheader(hp, daddr: &sk->sk_v6_daddr,
453	saddr: &sk->sk_v6_rcv_saddr, nbytes: skb->len);
454	#endif
455	else
456	return -EAFNOSUPPORT;
457	}
458
459	if (family == AF_INET) {
460	const struct iphdr *iph = ip_hdr(skb);
461
462	return tcp_v4_ao_hash_pseudoheader(hp, daddr: iph->daddr,
463	saddr: iph->saddr, nbytes: skb->len);
464	#if IS_ENABLED(CONFIG_IPV6)
465	} else if (family == AF_INET6) {
466	const struct ipv6hdr *iph = ipv6_hdr(skb);
467
468	return tcp_v6_ao_hash_pseudoheader(hp, daddr: &iph->daddr,
469	saddr: &iph->saddr, nbytes: skb->len);
470	#endif
471	}
472	return -EAFNOSUPPORT;
473	}
474
475	u32 tcp_ao_compute_sne(u32 next_sne, u32 next_seq, u32 seq)
476	{
477	u32 sne = next_sne;
478
479	if (before(seq1: seq, seq2: next_seq)) {
480	if (seq > next_seq)
481	sne--;
482	} else {
483	if (seq < next_seq)
484	sne++;
485	}
486
487	return sne;
488	}
489
490	/* tcp_ao_hash_sne(struct tcp_sigpool *hp)
491	* @hp - used for hashing
492	* @sne - sne value
493	*/
494	static int tcp_ao_hash_sne(struct tcp_sigpool *hp, u32 sne)
495	{
496	struct scatterlist sg;
497	__be32 *bp;
498
499	bp = (__be32 *)hp->scratch;
500	*bp = htonl(sne);
501
502	sg_init_one(&sg, bp, sizeof(*bp));
503	ahash_request_set_crypt(req: hp->req, src: &sg, NULL, nbytes: sizeof(*bp));
504	return crypto_ahash_update(req: hp->req);
505	}
506
507	static int tcp_ao_hash_header(struct tcp_sigpool *hp,
508	const struct tcphdr *th,
509	bool exclude_options, u8 *hash,
510	int hash_offset, int hash_len)
511	{
512	struct scatterlist sg;
513	u8 *hdr = hp->scratch;
514	int err, len;
515
516	/* We are not allowed to change tcphdr, make a local copy */
517	if (exclude_options) {
518	len = sizeof(*th) + sizeof(struct tcp_ao_hdr) + hash_len;
519	memcpy(hdr, th, sizeof(*th));
520	memcpy(hdr + sizeof(*th),
521	(u8 *)th + hash_offset - sizeof(struct tcp_ao_hdr),
522	sizeof(struct tcp_ao_hdr));
523	memset(hdr + sizeof(*th) + sizeof(struct tcp_ao_hdr),
524	0, hash_len);
525	((struct tcphdr *)hdr)->check = 0;
526	} else {
527	len = th->doff << 2;
528	memcpy(hdr, th, len);
529	/* zero out tcp-ao hash */
530	((struct tcphdr *)hdr)->check = 0;
531	memset(hdr + hash_offset, 0, hash_len);
532	}
533
534	sg_init_one(&sg, hdr, len);
535	ahash_request_set_crypt(req: hp->req, src: &sg, NULL, nbytes: len);
536	err = crypto_ahash_update(req: hp->req);
537	WARN_ON_ONCE(err != 0);
538	return err;
539	}
540
541	int tcp_ao_hash_hdr(unsigned short int family, char *ao_hash,
542	struct tcp_ao_key *key, const u8 *tkey,
543	const union tcp_ao_addr *daddr,
544	const union tcp_ao_addr *saddr,
545	const struct tcphdr *th, u32 sne)
546	{
547	int tkey_len = tcp_ao_digest_size(key);
548	int hash_offset = ao_hash - (char *)th;
549	struct tcp_sigpool hp;
550	void *hash_buf = NULL;
551
552	hash_buf = kmalloc(size: tkey_len, GFP_ATOMIC);
553	if (!hash_buf)
554	goto clear_hash_noput;
555
556	if (tcp_sigpool_start(id: key->tcp_sigpool_id, c: &hp))
557	goto clear_hash_noput;
558
559	if (crypto_ahash_setkey(tfm: crypto_ahash_reqtfm(req: hp.req), key: tkey, keylen: tkey_len))
560	goto clear_hash;
561
562	if (crypto_ahash_init(req: hp.req))
563	goto clear_hash;
564
565	if (tcp_ao_hash_sne(hp: &hp, sne))
566	goto clear_hash;
567	if (family == AF_INET) {
568	if (tcp_v4_ao_hash_pseudoheader(hp: &hp, daddr: daddr->a4.s_addr,
569	saddr: saddr->a4.s_addr, nbytes: th->doff * 4))
570	goto clear_hash;
571	#if IS_ENABLED(CONFIG_IPV6)
572	} else if (family == AF_INET6) {
573	if (tcp_v6_ao_hash_pseudoheader(hp: &hp, daddr: &daddr->a6,
574	saddr: &saddr->a6, nbytes: th->doff * 4))
575	goto clear_hash;
576	#endif
577	} else {
578	WARN_ON_ONCE(1);
579	goto clear_hash;
580	}
581	if (tcp_ao_hash_header(hp: &hp, th,
582	exclude_options: !!(key->keyflags & TCP_AO_KEYF_EXCLUDE_OPT),
583	hash: ao_hash, hash_offset, hash_len: tcp_ao_maclen(key)))
584	goto clear_hash;
585	ahash_request_set_crypt(req: hp.req, NULL, result: hash_buf, nbytes: 0);
586	if (crypto_ahash_final(req: hp.req))
587	goto clear_hash;
588
589	memcpy(ao_hash, hash_buf, tcp_ao_maclen(key));
590	tcp_sigpool_end(c: &hp);
591	kfree(objp: hash_buf);
592	return 0;
593
594	clear_hash:
595	tcp_sigpool_end(c: &hp);
596	clear_hash_noput:
597	memset(ao_hash, 0, tcp_ao_maclen(key));
598	kfree(objp: hash_buf);
599	return 1;
600	}
601
602	int tcp_ao_hash_skb(unsigned short int family,
603	char *ao_hash, struct tcp_ao_key *key,
604	const struct sock *sk, const struct sk_buff *skb,
605	const u8 *tkey, int hash_offset, u32 sne)
606	{
607	const struct tcphdr *th = tcp_hdr(skb);
608	int tkey_len = tcp_ao_digest_size(key);
609	struct tcp_sigpool hp;
610	void *hash_buf = NULL;
611
612	hash_buf = kmalloc(size: tkey_len, GFP_ATOMIC);
613	if (!hash_buf)
614	goto clear_hash_noput;
615
616	if (tcp_sigpool_start(id: key->tcp_sigpool_id, c: &hp))
617	goto clear_hash_noput;
618
619	if (crypto_ahash_setkey(tfm: crypto_ahash_reqtfm(req: hp.req), key: tkey, keylen: tkey_len))
620	goto clear_hash;
621
622	/* For now use sha1 by default. Depends on alg in tcp_ao_key */
623	if (crypto_ahash_init(req: hp.req))
624	goto clear_hash;
625
626	if (tcp_ao_hash_sne(hp: &hp, sne))
627	goto clear_hash;
628	if (tcp_ao_hash_pseudoheader(family, sk, skb, hp: &hp, nbytes: skb->len))
629	goto clear_hash;
630	if (tcp_ao_hash_header(hp: &hp, th,
631	exclude_options: !!(key->keyflags & TCP_AO_KEYF_EXCLUDE_OPT),
632	hash: ao_hash, hash_offset, hash_len: tcp_ao_maclen(key)))
633	goto clear_hash;
634	if (tcp_sigpool_hash_skb_data(hp: &hp, skb, header_len: th->doff << 2))
635	goto clear_hash;
636	ahash_request_set_crypt(req: hp.req, NULL, result: hash_buf, nbytes: 0);
637	if (crypto_ahash_final(req: hp.req))
638	goto clear_hash;
639
640	memcpy(ao_hash, hash_buf, tcp_ao_maclen(key));
641	tcp_sigpool_end(c: &hp);
642	kfree(objp: hash_buf);
643	return 0;
644
645	clear_hash:
646	tcp_sigpool_end(c: &hp);
647	clear_hash_noput:
648	memset(ao_hash, 0, tcp_ao_maclen(key));
649	kfree(objp: hash_buf);
650	return 1;
651	}
652
653	int tcp_v4_ao_hash_skb(char *ao_hash, struct tcp_ao_key *key,
654	const struct sock *sk, const struct sk_buff *skb,
655	const u8 *tkey, int hash_offset, u32 sne)
656	{
657	return tcp_ao_hash_skb(AF_INET, ao_hash, key, sk, skb,
658	tkey, hash_offset, sne);
659	}
660
661	int tcp_v4_ao_synack_hash(char *ao_hash, struct tcp_ao_key *ao_key,
662	struct request_sock *req, const struct sk_buff *skb,
663	int hash_offset, u32 sne)
664	{
665	void *hash_buf = NULL;
666	int err;
667
668	hash_buf = kmalloc(size: tcp_ao_digest_size(key: ao_key), GFP_ATOMIC);
669	if (!hash_buf)
670	return -ENOMEM;
671
672	err = tcp_v4_ao_calc_key_rsk(mkt: ao_key, key: hash_buf, req);
673	if (err)
674	goto out;
675
676	err = tcp_ao_hash_skb(AF_INET, ao_hash, key: ao_key, sk: req_to_sk(req), skb,
677	tkey: hash_buf, hash_offset, sne);
678	out:
679	kfree(objp: hash_buf);
680	return err;
681	}
682
683	struct tcp_ao_key *tcp_v4_ao_lookup_rsk(const struct sock *sk,
684	struct request_sock *req,
685	int sndid, int rcvid)
686	{
687	struct inet_request_sock *ireq = inet_rsk(sk: req);
688	union tcp_ao_addr *addr = (union tcp_ao_addr *)&ireq->ir_rmt_addr;
689	int l3index;
690
691	l3index = l3mdev_master_ifindex_by_index(net: sock_net(sk), ifindex: ireq->ir_iif);
692	return tcp_ao_do_lookup(sk, l3index, addr, AF_INET, sndid, rcvid);
693	}
694
695	struct tcp_ao_key *tcp_v4_ao_lookup(const struct sock *sk, struct sock *addr_sk,
696	int sndid, int rcvid)
697	{
698	int l3index = l3mdev_master_ifindex_by_index(net: sock_net(sk),
699	ifindex: addr_sk->sk_bound_dev_if);
700	union tcp_ao_addr *addr = (union tcp_ao_addr *)&addr_sk->sk_daddr;
701
702	return tcp_ao_do_lookup(sk, l3index, addr, AF_INET, sndid, rcvid);
703	}
704
705	int tcp_ao_prepare_reset(const struct sock *sk, struct sk_buff *skb,
706	const struct tcp_ao_hdr *aoh, int l3index, u32 seq,
707	struct tcp_ao_key **key, char **traffic_key,
708	bool *allocated_traffic_key, u8 *keyid, u32 *sne)
709	{
710	const struct tcphdr *th = tcp_hdr(skb);
711	struct tcp_ao_info *ao_info;
712
713	*allocated_traffic_key = false;
714	/* If there's no socket - than initial sisn/disn are unknown.
715	* Drop the segment. RFC5925 (7.7) advises to require graceful
716	* restart [RFC4724]. Alternatively, the RFC5925 advises to
717	* save/restore traffic keys before/after reboot.
718	* Linux TCP-AO support provides TCP_AO_ADD_KEY and TCP_AO_REPAIR
719	* options to restore a socket post-reboot.
720	*/
721	if (!sk)
722	return -ENOTCONN;
723
724	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV)) {
725	unsigned int family = READ_ONCE(sk->sk_family);
726	union tcp_ao_addr *addr;
727	__be32 disn, sisn;
728
729	if (sk->sk_state == TCP_NEW_SYN_RECV) {
730	struct request_sock *req = inet_reqsk(sk);
731
732	sisn = htonl(tcp_rsk(req)->rcv_isn);
733	disn = htonl(tcp_rsk(req)->snt_isn);
734	*sne = tcp_ao_compute_sne(next_sne: 0, next_seq: tcp_rsk(req)->snt_isn, seq);
735	} else {
736	sisn = th->seq;
737	disn = 0;
738	}
739	if (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6)
740	addr = (union tcp_md5_addr *)&ipv6_hdr(skb)->saddr;
741	else
742	addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
743	#if IS_ENABLED(CONFIG_IPV6)
744	if (family == AF_INET6 && ipv6_addr_v4mapped(a: &sk->sk_v6_daddr))
745	family = AF_INET;
746	#endif
747
748	sk = sk_const_to_full_sk(sk);
749	ao_info = rcu_dereference(tcp_sk(sk)->ao_info);
750	if (!ao_info)
751	return -ENOENT;
752	*key = tcp_ao_do_lookup(sk, l3index, addr, family,
753	sndid: -1, rcvid: aoh->rnext_keyid);
754	if (!*key)
755	return -ENOENT;
756	*traffic_key = kmalloc(size: tcp_ao_digest_size(key: *key), GFP_ATOMIC);
757	if (!*traffic_key)
758	return -ENOMEM;
759	*allocated_traffic_key = true;
760	if (tcp_ao_calc_key_skb(mkt: *key, key: *traffic_key, skb,
761	sisn, disn, family))
762	return -1;
763	*keyid = (*key)->rcvid;
764	} else {
765	struct tcp_ao_key *rnext_key;
766	u32 snd_basis;
767
768	if (sk->sk_state == TCP_TIME_WAIT) {
769	ao_info = rcu_dereference(tcp_twsk(sk)->ao_info);
770	snd_basis = tcp_twsk(sk)->tw_snd_nxt;
771	} else {
772	ao_info = rcu_dereference(tcp_sk(sk)->ao_info);
773	snd_basis = tcp_sk(sk)->snd_una;
774	}
775	if (!ao_info)
776	return -ENOENT;
777
778	*key = tcp_ao_established_key(ao: ao_info, sndid: aoh->rnext_keyid, rcvid: -1);
779	if (!*key)
780	return -ENOENT;
781	*traffic_key = snd_other_key(key: *key);
782	rnext_key = READ_ONCE(ao_info->rnext_key);
783	*keyid = rnext_key->rcvid;
784	*sne = tcp_ao_compute_sne(READ_ONCE(ao_info->snd_sne),
785	next_seq: snd_basis, seq);
786	}
787	return 0;
788	}
789
790	int tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb,
791	struct tcp_ao_key *key, struct tcphdr *th,
792	__u8 *hash_location)
793	{
794	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
795	struct tcp_sock *tp = tcp_sk(sk);
796	struct tcp_ao_info *ao;
797	void *tkey_buf = NULL;
798	u8 *traffic_key;
799	u32 sne;
800
801	ao = rcu_dereference_protected(tcp_sk(sk)->ao_info,
802	lockdep_sock_is_held(sk));
803	traffic_key = snd_other_key(key);
804	if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
805	__be32 disn;
806
807	if (!(tcb->tcp_flags & TCPHDR_ACK)) {
808	disn = 0;
809	tkey_buf = kmalloc(size: tcp_ao_digest_size(key), GFP_ATOMIC);
810	if (!tkey_buf)
811	return -ENOMEM;
812	traffic_key = tkey_buf;
813	} else {
814	disn = ao->risn;
815	}
816	tp->af_specific->ao_calc_key_sk(key, traffic_key,
817	sk, ao->lisn, disn, true);
818	}
819	sne = tcp_ao_compute_sne(READ_ONCE(ao->snd_sne), READ_ONCE(tp->snd_una),
820	ntohl(th->seq));
821	tp->af_specific->calc_ao_hash(hash_location, key, sk, skb, traffic_key,
822	hash_location - (u8 *)th, sne);
823	kfree(objp: tkey_buf);
824	return 0;
825	}
826
827	static struct tcp_ao_key *tcp_ao_inbound_lookup(unsigned short int family,
828	const struct sock *sk, const struct sk_buff *skb,
829	int sndid, int rcvid, int l3index)
830	{
831	if (family == AF_INET) {
832	const struct iphdr *iph = ip_hdr(skb);
833
834	return tcp_ao_do_lookup(sk, l3index,
835	addr: (union tcp_ao_addr *)&iph->saddr,
836	AF_INET, sndid, rcvid);
837	} else {
838	const struct ipv6hdr *iph = ipv6_hdr(skb);
839
840	return tcp_ao_do_lookup(sk, l3index,
841	addr: (union tcp_ao_addr *)&iph->saddr,
842	AF_INET6, sndid, rcvid);
843	}
844	}
845
846	void tcp_ao_syncookie(struct sock *sk, const struct sk_buff *skb,
847	struct request_sock *req, unsigned short int family)
848	{
849	struct tcp_request_sock *treq = tcp_rsk(req);
850	const struct tcphdr *th = tcp_hdr(skb);
851	const struct tcp_ao_hdr *aoh;
852	struct tcp_ao_key *key;
853	int l3index;
854
855	/* treq->af_specific is used to perform TCP_AO lookup
856	* in tcp_create_openreq_child().
857	*/
858	#if IS_ENABLED(CONFIG_IPV6)
859	if (family == AF_INET6)
860	treq->af_specific = &tcp_request_sock_ipv6_ops;
861	else
862	#endif
863	treq->af_specific = &tcp_request_sock_ipv4_ops;
864
865	treq->used_tcp_ao = false;
866
867	if (tcp_parse_auth_options(th, NULL, aoh: &aoh) || !aoh)
868	return;
869
870	l3index = l3mdev_master_ifindex_by_index(net: sock_net(sk), ifindex: inet_rsk(sk: req)->ir_iif);
871	key = tcp_ao_inbound_lookup(family, sk, skb, sndid: -1, rcvid: aoh->keyid, l3index);
872	if (!key)
873	/* Key not found, continue without TCP-AO */
874	return;
875
876	treq->ao_rcv_next = aoh->keyid;
877	treq->ao_keyid = aoh->rnext_keyid;
878	treq->used_tcp_ao = true;
879	}
880
881	static enum skb_drop_reason
882	tcp_ao_verify_hash(const struct sock *sk, const struct sk_buff *skb,
883	unsigned short int family, struct tcp_ao_info *info,
884	const struct tcp_ao_hdr *aoh, struct tcp_ao_key *key,
885	u8 *traffic_key, u8 *phash, u32 sne, int l3index)
886	{
887	u8 maclen = aoh->length - sizeof(struct tcp_ao_hdr);
888	const struct tcphdr *th = tcp_hdr(skb);
889	void *hash_buf = NULL;
890
891	if (maclen != tcp_ao_maclen(key)) {
892	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
893	atomic64_inc(v: &info->counters.pkt_bad);
894	atomic64_inc(v: &key->pkt_bad);
895	tcp_hash_fail("AO hash wrong length", family, skb,
896	"%u != %d L3index: %d", maclen,
897	tcp_ao_maclen(key), l3index);
898	return SKB_DROP_REASON_TCP_AOFAILURE;
899	}
900
901	hash_buf = kmalloc(size: tcp_ao_digest_size(key), GFP_ATOMIC);
902	if (!hash_buf)
903	return SKB_DROP_REASON_NOT_SPECIFIED;
904
905	/* XXX: make it per-AF callback? */
906	tcp_ao_hash_skb(family, ao_hash: hash_buf, key, sk, skb, tkey: traffic_key,
907	hash_offset: (phash - (u8 *)th), sne);
908	if (memcmp(p: phash, q: hash_buf, size: maclen)) {
909	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
910	atomic64_inc(v: &info->counters.pkt_bad);
911	atomic64_inc(v: &key->pkt_bad);
912	tcp_hash_fail("AO hash mismatch", family, skb,
913	"L3index: %d", l3index);
914	kfree(objp: hash_buf);
915	return SKB_DROP_REASON_TCP_AOFAILURE;
916	}
917	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOGOOD);
918	atomic64_inc(v: &info->counters.pkt_good);
919	atomic64_inc(v: &key->pkt_good);
920	kfree(objp: hash_buf);
921	return SKB_NOT_DROPPED_YET;
922	}
923
924	enum skb_drop_reason
925	tcp_inbound_ao_hash(struct sock *sk, const struct sk_buff *skb,
926	unsigned short int family, const struct request_sock *req,
927	int l3index, const struct tcp_ao_hdr *aoh)
928	{
929	const struct tcphdr *th = tcp_hdr(skb);
930	u8 *phash = (u8 *)(aoh + 1); /* hash goes just after the header */
931	struct tcp_ao_info *info;
932	enum skb_drop_reason ret;
933	struct tcp_ao_key *key;
934	__be32 sisn, disn;
935	u8 *traffic_key;
936	u32 sne = 0;
937
938	info = rcu_dereference(tcp_sk(sk)->ao_info);
939	if (!info) {
940	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOKEYNOTFOUND);
941	tcp_hash_fail("AO key not found", family, skb,
942	"keyid: %u L3index: %d", aoh->keyid, l3index);
943	return SKB_DROP_REASON_TCP_AOUNEXPECTED;
944	}
945
946	if (unlikely(th->syn)) {
947	sisn = th->seq;
948	disn = 0;
949	}
950
951	/* Fast-path */
952	if (likely((1 << sk->sk_state) & TCP_AO_ESTABLISHED)) {
953	enum skb_drop_reason err;
954	struct tcp_ao_key *current_key;
955
956	/* Check if this socket's rnext_key matches the keyid in the
957	* packet. If not we lookup the key based on the keyid
958	* matching the rcvid in the mkt.
959	*/
960	key = READ_ONCE(info->rnext_key);
961	if (key->rcvid != aoh->keyid) {
962	key = tcp_ao_established_key(ao: info, sndid: -1, rcvid: aoh->keyid);
963	if (!key)
964	goto key_not_found;
965	}
966
967	/* Delayed retransmitted SYN */
968	if (unlikely(th->syn && !th->ack))
969	goto verify_hash;
970
971	sne = tcp_ao_compute_sne(next_sne: info->rcv_sne, tcp_sk(sk)->rcv_nxt,
972	ntohl(th->seq));
973	/* Established socket, traffic key are cached */
974	traffic_key = rcv_other_key(key);
975	err = tcp_ao_verify_hash(sk, skb, family, info, aoh, key,
976	traffic_key, phash, sne, l3index);
977	if (err)
978	return err;
979	current_key = READ_ONCE(info->current_key);
980	/* Key rotation: the peer asks us to use new key (RNext) */
981	if (unlikely(aoh->rnext_keyid != current_key->sndid)) {
982	/* If the key is not found we do nothing. */
983	key = tcp_ao_established_key(ao: info, sndid: aoh->rnext_keyid, rcvid: -1);
984	if (key)
985	/* pairs with tcp_ao_del_cmd */
986	WRITE_ONCE(info->current_key, key);
987	}
988	return SKB_NOT_DROPPED_YET;
989	}
990
991	/* Lookup key based on peer address and keyid.
992	* current_key and rnext_key must not be used on tcp listen
993	* sockets as otherwise:
994	* - request sockets would race on those key pointers
995	* - tcp_ao_del_cmd() allows async key removal
996	*/
997	key = tcp_ao_inbound_lookup(family, sk, skb, sndid: -1, rcvid: aoh->keyid, l3index);
998	if (!key)
999	goto key_not_found;
1000
1001	if (th->syn && !th->ack)
1002	goto verify_hash;
1003
1004	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV)) {
1005	/* Make the initial syn the likely case here */
1006	if (unlikely(req)) {
1007	sne = tcp_ao_compute_sne(next_sne: 0, next_seq: tcp_rsk(req)->rcv_isn,
1008	ntohl(th->seq));
1009	sisn = htonl(tcp_rsk(req)->rcv_isn);
1010	disn = htonl(tcp_rsk(req)->snt_isn);
1011	} else if (unlikely(th->ack && !th->syn)) {
1012	/* Possible syncookie packet */
1013	sisn = htonl(ntohl(th->seq) - 1);
1014	disn = htonl(ntohl(th->ack_seq) - 1);
1015	sne = tcp_ao_compute_sne(next_sne: 0, ntohl(sisn),
1016	ntohl(th->seq));
1017	} else if (unlikely(!th->syn)) {
1018	/* no way to figure out initial sisn/disn - drop */
1019	return SKB_DROP_REASON_TCP_FLAGS;
1020	}
1021	} else if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
1022	disn = info->lisn;
1023	if (th->syn || th->rst)
1024	sisn = th->seq;
1025	else
1026	sisn = info->risn;
1027	} else {
1028	WARN_ONCE(1, "TCP-AO: Unexpected sk_state %d", sk->sk_state);
1029	return SKB_DROP_REASON_TCP_AOFAILURE;
1030	}
1031	verify_hash:
1032	traffic_key = kmalloc(size: tcp_ao_digest_size(key), GFP_ATOMIC);
1033	if (!traffic_key)
1034	return SKB_DROP_REASON_NOT_SPECIFIED;
1035	tcp_ao_calc_key_skb(mkt: key, key: traffic_key, skb, sisn, disn, family);
1036	ret = tcp_ao_verify_hash(sk, skb, family, info, aoh, key,
1037	traffic_key, phash, sne, l3index);
1038	kfree(objp: traffic_key);
1039	return ret;
1040
1041	key_not_found:
1042	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOKEYNOTFOUND);
1043	atomic64_inc(v: &info->counters.key_not_found);
1044	tcp_hash_fail("Requested by the peer AO key id not found",
1045	family, skb, "L3index: %d", l3index);
1046	return SKB_DROP_REASON_TCP_AOKEYNOTFOUND;
1047	}
1048
1049	static int tcp_ao_cache_traffic_keys(const struct sock *sk,
1050	struct tcp_ao_info *ao,
1051	struct tcp_ao_key *ao_key)
1052	{
1053	u8 *traffic_key = snd_other_key(key: ao_key);
1054	int ret;
1055
1056	ret = tcp_ao_calc_key_sk(mkt: ao_key, key: traffic_key, sk,
1057	sisn: ao->lisn, disn: ao->risn, send: true);
1058	if (ret)
1059	return ret;
1060
1061	traffic_key = rcv_other_key(key: ao_key);
1062	ret = tcp_ao_calc_key_sk(mkt: ao_key, key: traffic_key, sk,
1063	sisn: ao->lisn, disn: ao->risn, send: false);
1064	return ret;
1065	}
1066
1067	void tcp_ao_connect_init(struct sock *sk)
1068	{
1069	struct tcp_sock *tp = tcp_sk(sk);
1070	struct tcp_ao_info *ao_info;
1071	union tcp_ao_addr *addr;
1072	struct tcp_ao_key *key;
1073	int family, l3index;
1074
1075	ao_info = rcu_dereference_protected(tp->ao_info,
1076	lockdep_sock_is_held(sk));
1077	if (!ao_info)
1078	return;
1079
1080	/* Remove all keys that don't match the peer */
1081	family = sk->sk_family;
1082	if (family == AF_INET)
1083	addr = (union tcp_ao_addr *)&sk->sk_daddr;
1084	#if IS_ENABLED(CONFIG_IPV6)
1085	else if (family == AF_INET6)
1086	addr = (union tcp_ao_addr *)&sk->sk_v6_daddr;
1087	#endif
1088	else
1089	return;
1090	l3index = l3mdev_master_ifindex_by_index(net: sock_net(sk),
1091	ifindex: sk->sk_bound_dev_if);
1092
1093	hlist_for_each_entry_rcu(key, &ao_info->head, node) {
1094	if (!tcp_ao_key_cmp(key, l3index, addr, prefixlen: key->prefixlen, family, sndid: -1, rcvid: -1))
1095	continue;
1096
1097	if (key == ao_info->current_key)
1098	ao_info->current_key = NULL;
1099	if (key == ao_info->rnext_key)
1100	ao_info->rnext_key = NULL;
1101	hlist_del_rcu(n: &key->node);
1102	atomic_sub(i: tcp_ao_sizeof_key(key), v: &sk->sk_omem_alloc);
1103	call_rcu(head: &key->rcu, func: tcp_ao_key_free_rcu);
1104	}
1105
1106	key = tp->af_specific->ao_lookup(sk, sk, -1, -1);
1107	if (key) {
1108	/* if current_key or rnext_key were not provided,
1109	* use the first key matching the peer
1110	*/
1111	if (!ao_info->current_key)
1112	ao_info->current_key = key;
1113	if (!ao_info->rnext_key)
1114	ao_info->rnext_key = key;
1115	tp->tcp_header_len += tcp_ao_len_aligned(key);
1116
1117	ao_info->lisn = htonl(tp->write_seq);
1118	ao_info->snd_sne = 0;
1119	} else {
1120	/* Can't happen: tcp_connect() verifies that there's
1121	* at least one tcp-ao key that matches the remote peer.
1122	*/
1123	WARN_ON_ONCE(1);
1124	rcu_assign_pointer(tp->ao_info, NULL);
1125	kfree(objp: ao_info);
1126	}
1127	}
1128
1129	void tcp_ao_established(struct sock *sk)
1130	{
1131	struct tcp_ao_info *ao;
1132	struct tcp_ao_key *key;
1133
1134	ao = rcu_dereference_protected(tcp_sk(sk)->ao_info,
1135	lockdep_sock_is_held(sk));
1136	if (!ao)
1137	return;
1138
1139	hlist_for_each_entry_rcu(key, &ao->head, node)
1140	tcp_ao_cache_traffic_keys(sk, ao, ao_key: key);
1141	}
1142
1143	void tcp_ao_finish_connect(struct sock *sk, struct sk_buff *skb)
1144	{
1145	struct tcp_ao_info *ao;
1146	struct tcp_ao_key *key;
1147
1148	ao = rcu_dereference_protected(tcp_sk(sk)->ao_info,
1149	lockdep_sock_is_held(sk));
1150	if (!ao)
1151	return;
1152
1153	WRITE_ONCE(ao->risn, tcp_hdr(skb)->seq);
1154	ao->rcv_sne = 0;
1155
1156	hlist_for_each_entry_rcu(key, &ao->head, node)
1157	tcp_ao_cache_traffic_keys(sk, ao, ao_key: key);
1158	}
1159
1160	int tcp_ao_copy_all_matching(const struct sock *sk, struct sock *newsk,
1161	struct request_sock *req, struct sk_buff *skb,
1162	int family)
1163	{
1164	struct tcp_ao_key *key, *new_key, *first_key;
1165	struct tcp_ao_info *new_ao, *ao;
1166	struct hlist_node *key_head;
1167	int l3index, ret = -ENOMEM;
1168	union tcp_ao_addr *addr;
1169	bool match = false;
1170
1171	ao = rcu_dereference(tcp_sk(sk)->ao_info);
1172	if (!ao)
1173	return 0;
1174
1175	/* New socket without TCP-AO on it */
1176	if (!tcp_rsk_used_ao(req))
1177	return 0;
1178
1179	new_ao = tcp_ao_alloc_info(GFP_ATOMIC);
1180	if (!new_ao)
1181	return -ENOMEM;
1182	new_ao->lisn = htonl(tcp_rsk(req)->snt_isn);
1183	new_ao->risn = htonl(tcp_rsk(req)->rcv_isn);
1184	new_ao->ao_required = ao->ao_required;
1185	new_ao->accept_icmps = ao->accept_icmps;
1186
1187	if (family == AF_INET) {
1188	addr = (union tcp_ao_addr *)&newsk->sk_daddr;
1189	#if IS_ENABLED(CONFIG_IPV6)
1190	} else if (family == AF_INET6) {
1191	addr = (union tcp_ao_addr *)&newsk->sk_v6_daddr;
1192	#endif
1193	} else {
1194	ret = -EAFNOSUPPORT;
1195	goto free_ao;
1196	}
1197	l3index = l3mdev_master_ifindex_by_index(net: sock_net(sk: newsk),
1198	ifindex: newsk->sk_bound_dev_if);
1199
1200	hlist_for_each_entry_rcu(key, &ao->head, node) {
1201	if (tcp_ao_key_cmp(key, l3index, addr, prefixlen: key->prefixlen, family, sndid: -1, rcvid: -1))
1202	continue;
1203
1204	new_key = tcp_ao_copy_key(sk: newsk, key);
1205	if (!new_key)
1206	goto free_and_exit;
1207
1208	tcp_ao_cache_traffic_keys(sk: newsk, ao: new_ao, ao_key: new_key);
1209	tcp_ao_link_mkt(ao: new_ao, mkt: new_key);
1210	match = true;
1211	}
1212
1213	if (!match) {
1214	/* RFC5925 (7.4.1) specifies that the TCP-AO status
1215	* of a connection is determined on the initial SYN.
1216	* At this point the connection was TCP-AO enabled, so
1217	* it can't switch to being unsigned if peer's key
1218	* disappears on the listening socket.
1219	*/
1220	ret = -EKEYREJECTED;
1221	goto free_and_exit;
1222	}
1223
1224	if (!static_key_fast_inc_not_disabled(key: &tcp_ao_needed.key.key)) {
1225	ret = -EUSERS;
1226	goto free_and_exit;
1227	}
1228
1229	key_head = rcu_dereference(hlist_first_rcu(&new_ao->head));
1230	first_key = hlist_entry_safe(key_head, struct tcp_ao_key, node);
1231
1232	key = tcp_ao_established_key(ao: new_ao, sndid: tcp_rsk(req)->ao_keyid, rcvid: -1);
1233	if (key)
1234	new_ao->current_key = key;
1235	else
1236	new_ao->current_key = first_key;
1237
1238	/* set rnext_key */
1239	key = tcp_ao_established_key(ao: new_ao, sndid: -1, rcvid: tcp_rsk(req)->ao_rcv_next);
1240	if (key)
1241	new_ao->rnext_key = key;
1242	else
1243	new_ao->rnext_key = first_key;
1244
1245	sk_gso_disable(sk: newsk);
1246	rcu_assign_pointer(tcp_sk(newsk)->ao_info, new_ao);
1247
1248	return 0;
1249
1250	free_and_exit:
1251	hlist_for_each_entry_safe(key, key_head, &new_ao->head, node) {
1252	hlist_del(n: &key->node);
1253	tcp_sigpool_release(id: key->tcp_sigpool_id);
1254	atomic_sub(i: tcp_ao_sizeof_key(key), v: &newsk->sk_omem_alloc);
1255	kfree_sensitive(objp: key);
1256	}
1257	free_ao:
1258	kfree(objp: new_ao);
1259	return ret;
1260	}
1261
1262	static bool tcp_ao_can_set_current_rnext(struct sock *sk)
1263	{
1264	/* There aren't current/rnext keys on TCP_LISTEN sockets */
1265	if (sk->sk_state == TCP_LISTEN)
1266	return false;
1267	return true;
1268	}
1269
1270	static int tcp_ao_verify_ipv4(struct sock *sk, struct tcp_ao_add *cmd,
1271	union tcp_ao_addr **addr)
1272	{
1273	struct sockaddr_in *sin = (struct sockaddr_in *)&cmd->addr;
1274	struct inet_sock *inet = inet_sk(sk);
1275
1276	if (sin->sin_family != AF_INET)
1277	return -EINVAL;
1278
1279	/* Currently matching is not performed on port (or port ranges) */
1280	if (sin->sin_port != 0)
1281	return -EINVAL;
1282
1283	/* Check prefix and trailing 0's in addr */
1284	if (cmd->prefix != 0) {
1285	__be32 mask;
1286
1287	if (ntohl(sin->sin_addr.s_addr) == INADDR_ANY)
1288	return -EINVAL;
1289	if (cmd->prefix > 32)
1290	return -EINVAL;
1291
1292	mask = inet_make_mask(logmask: cmd->prefix);
1293	if (sin->sin_addr.s_addr & ~mask)
1294	return -EINVAL;
1295
1296	/* Check that MKT address is consistent with socket */
1297	if (ntohl(inet->inet_daddr) != INADDR_ANY &&
1298	(inet->inet_daddr & mask) != sin->sin_addr.s_addr)
1299	return -EINVAL;
1300	} else {
1301	if (ntohl(sin->sin_addr.s_addr) != INADDR_ANY)
1302	return -EINVAL;
1303	}
1304
1305	*addr = (union tcp_ao_addr *)&sin->sin_addr;
1306	return 0;
1307	}
1308
1309	static int tcp_ao_parse_crypto(struct tcp_ao_add *cmd, struct tcp_ao_key *key)
1310	{
1311	unsigned int syn_tcp_option_space;
1312	bool is_kdf_aes_128_cmac = false;
1313	struct crypto_ahash *tfm;
1314	struct tcp_sigpool hp;
1315	void *tmp_key = NULL;
1316	int err;
1317
1318	/* RFC5926, 3.1.1.2. KDF_AES_128_CMAC */
1319	if (!strcmp("cmac(aes128)", cmd->alg_name)) {
1320	strscpy(cmd->alg_name, "cmac(aes)", sizeof(cmd->alg_name));
1321	is_kdf_aes_128_cmac = (cmd->keylen != 16);
1322	tmp_key = kmalloc(size: cmd->keylen, GFP_KERNEL);
1323	if (!tmp_key)
1324	return -ENOMEM;
1325	}
1326
1327	key->maclen = cmd->maclen ?: 12; /* 12 is the default in RFC5925 */
1328
1329	/* Check: maclen + tcp-ao header <= (MAX_TCP_OPTION_SPACE - mss
1330	* - tstamp (including sackperm)
1331	* - wscale),
1332	* see tcp_syn_options(), tcp_synack_options(), commit 33ad798c924b.
1333	*
1334	* In order to allow D-SACK with TCP-AO, the header size should be:
1335	* (MAX_TCP_OPTION_SPACE - TCPOLEN_TSTAMP_ALIGNED
1336	* - TCPOLEN_SACK_BASE_ALIGNED
1337	* - 2 * TCPOLEN_SACK_PERBLOCK) = 8 (maclen = 4),
1338	* see tcp_established_options().
1339	*
1340	* RFC5925, 2.2:
1341	* Typical MACs are 96-128 bits (12-16 bytes), but any length
1342	* that fits in the header of the segment being authenticated
1343	* is allowed.
1344	*
1345	* RFC5925, 7.6:
1346	* TCP-AO continues to consume 16 bytes in non-SYN segments,
1347	* leaving a total of 24 bytes for other options, of which
1348	* the timestamp consumes 10. This leaves 14 bytes, of which 10
1349	* are used for a single SACK block. When two SACK blocks are used,
1350	* such as to handle D-SACK, a smaller TCP-AO MAC would be required
1351	* to make room for the additional SACK block (i.e., to leave 18
1352	* bytes for the D-SACK variant of the SACK option) [RFC2883].
1353	* Note that D-SACK is not supportable in TCP MD5 in the presence
1354	* of timestamps, because TCP MD5’s MAC length is fixed and too
1355	* large to leave sufficient option space.
1356	*/
1357	syn_tcp_option_space = MAX_TCP_OPTION_SPACE;
1358	syn_tcp_option_space -= TCPOLEN_MSS_ALIGNED;
1359	syn_tcp_option_space -= TCPOLEN_TSTAMP_ALIGNED;
1360	syn_tcp_option_space -= TCPOLEN_WSCALE_ALIGNED;
1361	if (tcp_ao_len_aligned(key) > syn_tcp_option_space) {
1362	err = -EMSGSIZE;
1363	goto err_kfree;
1364	}
1365
1366	key->keylen = cmd->keylen;
1367	memcpy(key->key, cmd->key, cmd->keylen);
1368
1369	err = tcp_sigpool_start(id: key->tcp_sigpool_id, c: &hp);
1370	if (err)
1371	goto err_kfree;
1372
1373	tfm = crypto_ahash_reqtfm(req: hp.req);
1374	if (is_kdf_aes_128_cmac) {
1375	void *scratch = hp.scratch;
1376	struct scatterlist sg;
1377
1378	memcpy(tmp_key, cmd->key, cmd->keylen);
1379	sg_init_one(&sg, tmp_key, cmd->keylen);
1380
1381	/* Using zero-key of 16 bytes as described in RFC5926 */
1382	memset(scratch, 0, 16);
1383	err = crypto_ahash_setkey(tfm, key: scratch, keylen: 16);
1384	if (err)
1385	goto err_pool_end;
1386
1387	err = crypto_ahash_init(req: hp.req);
1388	if (err)
1389	goto err_pool_end;
1390
1391	ahash_request_set_crypt(req: hp.req, src: &sg, result: key->key, nbytes: cmd->keylen);
1392	err = crypto_ahash_update(req: hp.req);
1393	if (err)
1394	goto err_pool_end;
1395
1396	err |= crypto_ahash_final(req: hp.req);
1397	if (err)
1398	goto err_pool_end;
1399	key->keylen = 16;
1400	}
1401
1402	err = crypto_ahash_setkey(tfm, key: key->key, keylen: key->keylen);
1403	if (err)
1404	goto err_pool_end;
1405
1406	tcp_sigpool_end(c: &hp);
1407	kfree_sensitive(objp: tmp_key);
1408
1409	if (tcp_ao_maclen(key) > key->digest_size)
1410	return -EINVAL;
1411
1412	return 0;
1413
1414	err_pool_end:
1415	tcp_sigpool_end(c: &hp);
1416	err_kfree:
1417	kfree_sensitive(objp: tmp_key);
1418	return err;
1419	}
1420
1421	#if IS_ENABLED(CONFIG_IPV6)
1422	static int tcp_ao_verify_ipv6(struct sock *sk, struct tcp_ao_add *cmd,
1423	union tcp_ao_addr **paddr,
1424	unsigned short int *family)
1425	{
1426	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd->addr;
1427	struct in6_addr *addr = &sin6->sin6_addr;
1428	u8 prefix = cmd->prefix;
1429
1430	if (sin6->sin6_family != AF_INET6)
1431	return -EINVAL;
1432
1433	/* Currently matching is not performed on port (or port ranges) */
1434	if (sin6->sin6_port != 0)
1435	return -EINVAL;
1436
1437	/* Check prefix and trailing 0's in addr */
1438	if (cmd->prefix != 0 && ipv6_addr_v4mapped(a: addr)) {
1439	__be32 addr4 = addr->s6_addr32[3];
1440	__be32 mask;
1441
1442	if (prefix > 32 || ntohl(addr4) == INADDR_ANY)
1443	return -EINVAL;
1444
1445	mask = inet_make_mask(logmask: prefix);
1446	if (addr4 & ~mask)
1447	return -EINVAL;
1448
1449	/* Check that MKT address is consistent with socket */
1450	if (!ipv6_addr_any(a: &sk->sk_v6_daddr)) {
1451	__be32 daddr4 = sk->sk_v6_daddr.s6_addr32[3];
1452
1453	if (!ipv6_addr_v4mapped(a: &sk->sk_v6_daddr))
1454	return -EINVAL;
1455	if ((daddr4 & mask) != addr4)
1456	return -EINVAL;
1457	}
1458
1459	*paddr = (union tcp_ao_addr *)&addr->s6_addr32[3];
1460	*family = AF_INET;
1461	return 0;
1462	} else if (cmd->prefix != 0) {
1463	struct in6_addr pfx;
1464
1465	if (ipv6_addr_any(a: addr) || prefix > 128)
1466	return -EINVAL;
1467
1468	ipv6_addr_prefix(pfx: &pfx, addr, plen: prefix);
1469	if (ipv6_addr_cmp(a1: &pfx, a2: addr))
1470	return -EINVAL;
1471
1472	/* Check that MKT address is consistent with socket */
1473	if (!ipv6_addr_any(a: &sk->sk_v6_daddr) &&
1474	!ipv6_prefix_equal(addr1: &sk->sk_v6_daddr, addr2: addr, prefixlen: prefix))
1475
1476	return -EINVAL;
1477	} else {
1478	if (!ipv6_addr_any(a: addr))
1479	return -EINVAL;
1480	}
1481
1482	*paddr = (union tcp_ao_addr *)addr;
1483	return 0;
1484	}
1485	#else
1486	static int tcp_ao_verify_ipv6(struct sock *sk, struct tcp_ao_add *cmd,
1487	union tcp_ao_addr **paddr,
1488	unsigned short int *family)
1489	{
1490	return -EOPNOTSUPP;
1491	}
1492	#endif
1493
1494	static struct tcp_ao_info *setsockopt_ao_info(struct sock *sk)
1495	{
1496	if (sk_fullsock(sk)) {
1497	return rcu_dereference_protected(tcp_sk(sk)->ao_info,
1498	lockdep_sock_is_held(sk));
1499	} else if (sk->sk_state == TCP_TIME_WAIT) {
1500	return rcu_dereference_protected(tcp_twsk(sk)->ao_info,
1501	lockdep_sock_is_held(sk));
1502	}
1503	return ERR_PTR(error: -ESOCKTNOSUPPORT);
1504	}
1505
1506	static struct tcp_ao_info *getsockopt_ao_info(struct sock *sk)
1507	{
1508	if (sk_fullsock(sk))
1509	return rcu_dereference(tcp_sk(sk)->ao_info);
1510	else if (sk->sk_state == TCP_TIME_WAIT)
1511	return rcu_dereference(tcp_twsk(sk)->ao_info);
1512
1513	return ERR_PTR(error: -ESOCKTNOSUPPORT);
1514	}
1515
1516	#define TCP_AO_KEYF_ALL (TCP_AO_KEYF_IFINDEX | TCP_AO_KEYF_EXCLUDE_OPT)
1517	#define TCP_AO_GET_KEYF_VALID (TCP_AO_KEYF_IFINDEX)
1518
1519	static struct tcp_ao_key *tcp_ao_key_alloc(struct sock *sk,
1520	struct tcp_ao_add *cmd)
1521	{
1522	const char *algo = cmd->alg_name;
1523	unsigned int digest_size;
1524	struct crypto_ahash *tfm;
1525	struct tcp_ao_key *key;
1526	struct tcp_sigpool hp;
1527	int err, pool_id;
1528	size_t size;
1529
1530	/* Force null-termination of alg_name */
1531	cmd->alg_name[ARRAY_SIZE(cmd->alg_name) - 1] = '\0';
1532
1533	/* RFC5926, 3.1.1.2. KDF_AES_128_CMAC */
1534	if (!strcmp("cmac(aes128)", algo))
1535	algo = "cmac(aes)";
1536
1537	/* Full TCP header (th->doff << 2) should fit into scratch area,
1538	* see tcp_ao_hash_header().
1539	*/
1540	pool_id = tcp_sigpool_alloc_ahash(alg: algo, scratch_size: 60);
1541	if (pool_id < 0)
1542	return ERR_PTR(error: pool_id);
1543
1544	err = tcp_sigpool_start(id: pool_id, c: &hp);
1545	if (err)
1546	goto err_free_pool;
1547
1548	tfm = crypto_ahash_reqtfm(req: hp.req);
1549	digest_size = crypto_ahash_digestsize(tfm);
1550	tcp_sigpool_end(c: &hp);
1551
1552	size = sizeof(struct tcp_ao_key) + (digest_size << 1);
1553	key = sock_kmalloc(sk, size, GFP_KERNEL);
1554	if (!key) {
1555	err = -ENOMEM;
1556	goto err_free_pool;
1557	}
1558
1559	key->tcp_sigpool_id = pool_id;
1560	key->digest_size = digest_size;
1561	return key;
1562
1563	err_free_pool:
1564	tcp_sigpool_release(id: pool_id);
1565	return ERR_PTR(error: err);
1566	}
1567
1568	static int tcp_ao_add_cmd(struct sock *sk, unsigned short int family,
1569	sockptr_t optval, int optlen)
1570	{
1571	struct tcp_ao_info *ao_info;
1572	union tcp_ao_addr *addr;
1573	struct tcp_ao_key *key;
1574	struct tcp_ao_add cmd;
1575	int ret, l3index = 0;
1576	bool first = false;
1577
1578	if (optlen < sizeof(cmd))
1579	return -EINVAL;
1580
1581	ret = copy_struct_from_sockptr(dst: &cmd, ksize: sizeof(cmd), src: optval, usize: optlen);
1582	if (ret)
1583	return ret;
1584
1585	if (cmd.keylen > TCP_AO_MAXKEYLEN)
1586	return -EINVAL;
1587
1588	if (cmd.reserved != 0 || cmd.reserved2 != 0)
1589	return -EINVAL;
1590
1591	if (family == AF_INET)
1592	ret = tcp_ao_verify_ipv4(sk, cmd: &cmd, addr: &addr);
1593	else
1594	ret = tcp_ao_verify_ipv6(sk, cmd: &cmd, paddr: &addr, family: &family);
1595	if (ret)
1596	return ret;
1597
1598	if (cmd.keyflags & ~TCP_AO_KEYF_ALL)
1599	return -EINVAL;
1600
1601	if (cmd.set_current || cmd.set_rnext) {
1602	if (!tcp_ao_can_set_current_rnext(sk))
1603	return -EINVAL;
1604	}
1605
1606	if (cmd.ifindex && !(cmd.keyflags & TCP_AO_KEYF_IFINDEX))
1607	return -EINVAL;
1608
1609	/* For cmd.tcp_ifindex = 0 the key will apply to the default VRF */
1610	if (cmd.keyflags & TCP_AO_KEYF_IFINDEX && cmd.ifindex) {
1611	int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
1612	struct net_device *dev;
1613
1614	rcu_read_lock();
1615	dev = dev_get_by_index_rcu(net: sock_net(sk), ifindex: cmd.ifindex);
1616	if (dev && netif_is_l3_master(dev))
1617	l3index = dev->ifindex;
1618	rcu_read_unlock();
1619
1620	if (!dev || !l3index)
1621	return -EINVAL;
1622
1623	if (!bound_dev_if || bound_dev_if != cmd.ifindex) {
1624	/* tcp_ao_established_key() doesn't expect having
1625	* non peer-matching key on an established TCP-AO
1626	* connection.
1627	*/
1628	if (!((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)))
1629	return -EINVAL;
1630	}
1631
1632	/* It's still possible to bind after adding keys or even
1633	* re-bind to a different dev (with CAP_NET_RAW).
1634	* So, no reason to return error here, rather try to be
1635	* nice and warn the user.
1636	*/
1637	if (bound_dev_if && bound_dev_if != cmd.ifindex)
1638	net_warn_ratelimited("AO key ifindex %d != sk bound ifindex %d\n",
1639	cmd.ifindex, bound_dev_if);
1640	}
1641
1642	/* Don't allow keys for peers that have a matching TCP-MD5 key */
1643	if (cmd.keyflags & TCP_AO_KEYF_IFINDEX) {
1644	/* Non-_exact version of tcp_md5_do_lookup() will
1645	* as well match keys that aren't bound to a specific VRF
1646	* (that will make them match AO key with
1647	* sysctl_tcp_l3dev_accept = 1
1648	*/
1649	if (tcp_md5_do_lookup(sk, l3index, addr, family))
1650	return -EKEYREJECTED;
1651	} else {
1652	if (tcp_md5_do_lookup_any_l3index(sk, addr, family))
1653	return -EKEYREJECTED;
1654	}
1655
1656	ao_info = setsockopt_ao_info(sk);
1657	if (IS_ERR(ptr: ao_info))
1658	return PTR_ERR(ptr: ao_info);
1659
1660	if (!ao_info) {
1661	ao_info = tcp_ao_alloc_info(GFP_KERNEL);
1662	if (!ao_info)
1663	return -ENOMEM;
1664	first = true;
1665	} else {
1666	/* Check that neither RecvID nor SendID match any
1667	* existing key for the peer, RFC5925 3.1:
1668	* > The IDs of MKTs MUST NOT overlap where their
1669	* > TCP connection identifiers overlap.
1670	*/
1671	if (__tcp_ao_do_lookup(sk, l3index, addr, family, prefix: cmd.prefix, sndid: -1, rcvid: cmd.rcvid))
1672	return -EEXIST;
1673	if (__tcp_ao_do_lookup(sk, l3index, addr, family,
1674	prefix: cmd.prefix, sndid: cmd.sndid, rcvid: -1))
1675	return -EEXIST;
1676	}
1677
1678	key = tcp_ao_key_alloc(sk, cmd: &cmd);
1679	if (IS_ERR(ptr: key)) {
1680	ret = PTR_ERR(ptr: key);
1681	goto err_free_ao;
1682	}
1683
1684	INIT_HLIST_NODE(h: &key->node);
1685	memcpy(&key->addr, addr, (family == AF_INET) ? sizeof(struct in_addr) :
1686	sizeof(struct in6_addr));
1687	key->prefixlen = cmd.prefix;
1688	key->family = family;
1689	key->keyflags = cmd.keyflags;
1690	key->sndid = cmd.sndid;
1691	key->rcvid = cmd.rcvid;
1692	key->l3index = l3index;
1693	atomic64_set(v: &key->pkt_good, i: 0);
1694	atomic64_set(v: &key->pkt_bad, i: 0);
1695
1696	ret = tcp_ao_parse_crypto(cmd: &cmd, key);
1697	if (ret < 0)
1698	goto err_free_sock;
1699
1700	if (!((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))) {
1701	tcp_ao_cache_traffic_keys(sk, ao: ao_info, ao_key: key);
1702	if (first) {
1703	ao_info->current_key = key;
1704	ao_info->rnext_key = key;
1705	}
1706	}
1707
1708	tcp_ao_link_mkt(ao: ao_info, mkt: key);
1709	if (first) {
1710	if (!static_branch_inc(&tcp_ao_needed.key)) {
1711	ret = -EUSERS;
1712	goto err_free_sock;
1713	}
1714	sk_gso_disable(sk);
1715	rcu_assign_pointer(tcp_sk(sk)->ao_info, ao_info);
1716	}
1717
1718	if (cmd.set_current)
1719	WRITE_ONCE(ao_info->current_key, key);
1720	if (cmd.set_rnext)
1721	WRITE_ONCE(ao_info->rnext_key, key);
1722	return 0;
1723
1724	err_free_sock:
1725	atomic_sub(i: tcp_ao_sizeof_key(key), v: &sk->sk_omem_alloc);
1726	tcp_sigpool_release(id: key->tcp_sigpool_id);
1727	kfree_sensitive(objp: key);
1728	err_free_ao:
1729	if (first)
1730	kfree(objp: ao_info);
1731	return ret;
1732	}
1733
1734	static int tcp_ao_delete_key(struct sock *sk, struct tcp_ao_info *ao_info,
1735	bool del_async, struct tcp_ao_key *key,
1736	struct tcp_ao_key *new_current,
1737	struct tcp_ao_key *new_rnext)
1738	{
1739	int err;
1740
1741	hlist_del_rcu(n: &key->node);
1742
1743	/* Support for async delete on listening sockets: as they don't
1744	* need current_key/rnext_key maintaining, we don't need to check
1745	* them and we can just free all resources in RCU fashion.
1746	*/
1747	if (del_async) {
1748	atomic_sub(i: tcp_ao_sizeof_key(key), v: &sk->sk_omem_alloc);
1749	call_rcu(head: &key->rcu, func: tcp_ao_key_free_rcu);
1750	return 0;
1751	}
1752
1753	/* At this moment another CPU could have looked this key up
1754	* while it was unlinked from the list. Wait for RCU grace period,
1755	* after which the key is off-list and can't be looked up again;
1756	* the rx path [just before RCU came] might have used it and set it
1757	* as current_key (very unlikely).
1758	* Free the key with next RCU grace period (in case it was
1759	* current_key before tcp_ao_current_rnext() might have
1760	* changed it in forced-delete).
1761	*/
1762	synchronize_rcu();
1763	if (new_current)
1764	WRITE_ONCE(ao_info->current_key, new_current);
1765	if (new_rnext)
1766	WRITE_ONCE(ao_info->rnext_key, new_rnext);
1767
1768	if (unlikely(READ_ONCE(ao_info->current_key) == key ||
1769	READ_ONCE(ao_info->rnext_key) == key)) {
1770	err = -EBUSY;
1771	goto add_key;
1772	}
1773
1774	atomic_sub(i: tcp_ao_sizeof_key(key), v: &sk->sk_omem_alloc);
1775	call_rcu(head: &key->rcu, func: tcp_ao_key_free_rcu);
1776
1777	return 0;
1778	add_key:
1779	hlist_add_head_rcu(n: &key->node, h: &ao_info->head);
1780	return err;
1781	}
1782
1783	#define TCP_AO_DEL_KEYF_ALL (TCP_AO_KEYF_IFINDEX)
1784	static int tcp_ao_del_cmd(struct sock *sk, unsigned short int family,
1785	sockptr_t optval, int optlen)
1786	{
1787	struct tcp_ao_key *key, *new_current = NULL, *new_rnext = NULL;
1788	int err, addr_len, l3index = 0;
1789	struct tcp_ao_info *ao_info;
1790	union tcp_ao_addr *addr;
1791	struct tcp_ao_del cmd;
1792	__u8 prefix;
1793	u16 port;
1794
1795	if (optlen < sizeof(cmd))
1796	return -EINVAL;
1797
1798	err = copy_struct_from_sockptr(dst: &cmd, ksize: sizeof(cmd), src: optval, usize: optlen);
1799	if (err)
1800	return err;
1801
1802	if (cmd.reserved != 0 || cmd.reserved2 != 0)
1803	return -EINVAL;
1804
1805	if (cmd.set_current || cmd.set_rnext) {
1806	if (!tcp_ao_can_set_current_rnext(sk))
1807	return -EINVAL;
1808	}
1809
1810	if (cmd.keyflags & ~TCP_AO_DEL_KEYF_ALL)
1811	return -EINVAL;
1812
1813	/* No sanity check for TCP_AO_KEYF_IFINDEX as if a VRF
1814	* was destroyed, there still should be a way to delete keys,
1815	* that were bound to that l3intf. So, fail late at lookup stage
1816	* if there is no key for that ifindex.
1817	*/
1818	if (cmd.ifindex && !(cmd.keyflags & TCP_AO_KEYF_IFINDEX))
1819	return -EINVAL;
1820
1821	ao_info = setsockopt_ao_info(sk);
1822	if (IS_ERR(ptr: ao_info))
1823	return PTR_ERR(ptr: ao_info);
1824	if (!ao_info)
1825	return -ENOENT;
1826
1827	/* For sockets in TCP_CLOSED it's possible set keys that aren't
1828	* matching the future peer (address/VRF/etc),
1829	* tcp_ao_connect_init() will choose a correct matching MKT
1830	* if there's any.
1831	*/
1832	if (cmd.set_current) {
1833	new_current = tcp_ao_established_key(ao: ao_info, sndid: cmd.current_key, rcvid: -1);
1834	if (!new_current)
1835	return -ENOENT;
1836	}
1837	if (cmd.set_rnext) {
1838	new_rnext = tcp_ao_established_key(ao: ao_info, sndid: -1, rcvid: cmd.rnext);
1839	if (!new_rnext)
1840	return -ENOENT;
1841	}
1842	if (cmd.del_async && sk->sk_state != TCP_LISTEN)
1843	return -EINVAL;
1844
1845	if (family == AF_INET) {
1846	struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.addr;
1847
1848	addr = (union tcp_ao_addr *)&sin->sin_addr;
1849	addr_len = sizeof(struct in_addr);
1850	port = ntohs(sin->sin_port);
1851	} else {
1852	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.addr;
1853	struct in6_addr *addr6 = &sin6->sin6_addr;
1854
1855	if (ipv6_addr_v4mapped(a: addr6)) {
1856	addr = (union tcp_ao_addr *)&addr6->s6_addr32[3];
1857	addr_len = sizeof(struct in_addr);
1858	family = AF_INET;
1859	} else {
1860	addr = (union tcp_ao_addr *)addr6;
1861	addr_len = sizeof(struct in6_addr);
1862	}
1863	port = ntohs(sin6->sin6_port);
1864	}
1865	prefix = cmd.prefix;
1866
1867	/* Currently matching is not performed on port (or port ranges) */
1868	if (port != 0)
1869	return -EINVAL;
1870
1871	/* We could choose random present key here for current/rnext
1872	* but that's less predictable. Let's be strict and don't
1873	* allow removing a key that's in use. RFC5925 doesn't
1874	* specify how-to coordinate key removal, but says:
1875	* "It is presumed that an MKT affecting a particular
1876	* connection cannot be destroyed during an active connection"
1877	*/
1878	hlist_for_each_entry_rcu(key, &ao_info->head, node) {
1879	if (cmd.sndid != key->sndid ||
1880	cmd.rcvid != key->rcvid)
1881	continue;
1882
1883	if (family != key->family ||
1884	prefix != key->prefixlen ||
1885	memcmp(p: addr, q: &key->addr, size: addr_len))
1886	continue;
1887
1888	if ((cmd.keyflags & TCP_AO_KEYF_IFINDEX) !=
1889	(key->keyflags & TCP_AO_KEYF_IFINDEX))
1890	continue;
1891
1892	if (key->l3index != l3index)
1893	continue;
1894
1895	if (key == new_current || key == new_rnext)
1896	continue;
1897
1898	return tcp_ao_delete_key(sk, ao_info, del_async: cmd.del_async, key,
1899	new_current, new_rnext);
1900	}
1901	return -ENOENT;
1902	}
1903
1904	/* cmd.ao_required makes a socket TCP-AO only.
1905	* Don't allow any md5 keys for any l3intf on the socket together with it.
1906	* Restricting it early in setsockopt() removes a check for
1907	* ao_info->ao_required on inbound tcp segment fast-path.
1908	*/
1909	static int tcp_ao_required_verify(struct sock *sk)
1910	{
1911	#ifdef CONFIG_TCP_MD5SIG
1912	const struct tcp_md5sig_info *md5sig;
1913
1914	if (!static_branch_unlikely(&tcp_md5_needed.key))
1915	return 0;
1916
1917	md5sig = rcu_dereference_check(tcp_sk(sk)->md5sig_info,
1918	lockdep_sock_is_held(sk));
1919	if (!md5sig)
1920	return 0;
1921
1922	if (rcu_dereference_check(hlist_first_rcu(&md5sig->head),
1923	lockdep_sock_is_held(sk)))
1924	return 1;
1925	#endif
1926	return 0;
1927	}
1928
1929	static int tcp_ao_info_cmd(struct sock *sk, unsigned short int family,
1930	sockptr_t optval, int optlen)
1931	{
1932	struct tcp_ao_key *new_current = NULL, *new_rnext = NULL;
1933	struct tcp_ao_info *ao_info;
1934	struct tcp_ao_info_opt cmd;
1935	bool first = false;
1936	int err;
1937
1938	if (optlen < sizeof(cmd))
1939	return -EINVAL;
1940
1941	err = copy_struct_from_sockptr(dst: &cmd, ksize: sizeof(cmd), src: optval, usize: optlen);
1942	if (err)
1943	return err;
1944
1945	if (cmd.set_current || cmd.set_rnext) {
1946	if (!tcp_ao_can_set_current_rnext(sk))
1947	return -EINVAL;
1948	}
1949
1950	if (cmd.reserved != 0 || cmd.reserved2 != 0)
1951	return -EINVAL;
1952
1953	ao_info = setsockopt_ao_info(sk);
1954	if (IS_ERR(ptr: ao_info))
1955	return PTR_ERR(ptr: ao_info);
1956	if (!ao_info) {
1957	if (!((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)))
1958	return -EINVAL;
1959	ao_info = tcp_ao_alloc_info(GFP_KERNEL);
1960	if (!ao_info)
1961	return -ENOMEM;
1962	first = true;
1963	}
1964
1965	if (cmd.ao_required && tcp_ao_required_verify(sk))
1966	return -EKEYREJECTED;
1967
1968	/* For sockets in TCP_CLOSED it's possible set keys that aren't
1969	* matching the future peer (address/port/VRF/etc),
1970	* tcp_ao_connect_init() will choose a correct matching MKT
1971	* if there's any.
1972	*/
1973	if (cmd.set_current) {
1974	new_current = tcp_ao_established_key(ao: ao_info, sndid: cmd.current_key, rcvid: -1);
1975	if (!new_current) {
1976	err = -ENOENT;
1977	goto out;
1978	}
1979	}
1980	if (cmd.set_rnext) {
1981	new_rnext = tcp_ao_established_key(ao: ao_info, sndid: -1, rcvid: cmd.rnext);
1982	if (!new_rnext) {
1983	err = -ENOENT;
1984	goto out;
1985	}
1986	}
1987	if (cmd.set_counters) {
1988	atomic64_set(v: &ao_info->counters.pkt_good, i: cmd.pkt_good);
1989	atomic64_set(v: &ao_info->counters.pkt_bad, i: cmd.pkt_bad);
1990	atomic64_set(v: &ao_info->counters.key_not_found, i: cmd.pkt_key_not_found);
1991	atomic64_set(v: &ao_info->counters.ao_required, i: cmd.pkt_ao_required);
1992	atomic64_set(v: &ao_info->counters.dropped_icmp, i: cmd.pkt_dropped_icmp);
1993	}
1994
1995	ao_info->ao_required = cmd.ao_required;
1996	ao_info->accept_icmps = cmd.accept_icmps;
1997	if (new_current)
1998	WRITE_ONCE(ao_info->current_key, new_current);
1999	if (new_rnext)
2000	WRITE_ONCE(ao_info->rnext_key, new_rnext);
2001	if (first) {
2002	if (!static_branch_inc(&tcp_ao_needed.key)) {
2003	err = -EUSERS;
2004	goto out;
2005	}
2006	sk_gso_disable(sk);
2007	rcu_assign_pointer(tcp_sk(sk)->ao_info, ao_info);
2008	}
2009	return 0;
2010	out:
2011	if (first)
2012	kfree(objp: ao_info);
2013	return err;
2014	}
2015
2016	int tcp_parse_ao(struct sock *sk, int cmd, unsigned short int family,
2017	sockptr_t optval, int optlen)
2018	{
2019	if (WARN_ON_ONCE(family != AF_INET && family != AF_INET6))
2020	return -EAFNOSUPPORT;
2021
2022	switch (cmd) {
2023	case TCP_AO_ADD_KEY:
2024	return tcp_ao_add_cmd(sk, family, optval, optlen);
2025	case TCP_AO_DEL_KEY:
2026	return tcp_ao_del_cmd(sk, family, optval, optlen);
2027	case TCP_AO_INFO:
2028	return tcp_ao_info_cmd(sk, family, optval, optlen);
2029	default:
2030	WARN_ON_ONCE(1);
2031	return -EINVAL;
2032	}
2033	}
2034
2035	int tcp_v4_parse_ao(struct sock *sk, int cmd, sockptr_t optval, int optlen)
2036	{
2037	return tcp_parse_ao(sk, cmd, AF_INET, optval, optlen);
2038	}
2039
2040	/* tcp_ao_copy_mkts_to_user(ao_info, optval, optlen)
2041	*
2042	* @ao_info: struct tcp_ao_info on the socket that
2043	* socket getsockopt(TCP_AO_GET_KEYS) is executed on
2044	* @optval: pointer to array of tcp_ao_getsockopt structures in user space.
2045	* Must be != NULL.
2046	* @optlen: pointer to size of tcp_ao_getsockopt structure.
2047	* Must be != NULL.
2048	*
2049	* Return value: 0 on success, a negative error number otherwise.
2050	*
2051	* optval points to an array of tcp_ao_getsockopt structures in user space.
2052	* optval[0] is used as both input and output to getsockopt. It determines
2053	* which keys are returned by the kernel.
2054	* optval[0].nkeys is the size of the array in user space. On return it contains
2055	* the number of keys matching the search criteria.
2056	* If tcp_ao_getsockopt::get_all is set, then all keys in the socket are
2057	* returned, otherwise only keys matching <addr, prefix, sndid, rcvid>
2058	* in optval[0] are returned.
2059	* optlen is also used as both input and output. The user provides the size
2060	* of struct tcp_ao_getsockopt in user space, and the kernel returns the size
2061	* of the structure in kernel space.
2062	* The size of struct tcp_ao_getsockopt may differ between user and kernel.
2063	* There are three cases to consider:
2064	* * If usize == ksize, then keys are copied verbatim.
2065	* * If usize < ksize, then the userspace has passed an old struct to a
2066	* newer kernel. The rest of the trailing bytes in optval[0]
2067	* (ksize - usize) are interpreted as 0 by the kernel.
2068	* * If usize > ksize, then the userspace has passed a new struct to an
2069	* older kernel. The trailing bytes unknown to the kernel (usize - ksize)
2070	* are checked to ensure they are zeroed, otherwise -E2BIG is returned.
2071	* On return the kernel fills in min(usize, ksize) in each entry of the array.
2072	* The layout of the fields in the user and kernel structures is expected to
2073	* be the same (including in the 32bit vs 64bit case).
2074	*/
2075	static int tcp_ao_copy_mkts_to_user(struct tcp_ao_info *ao_info,
2076	sockptr_t optval, sockptr_t optlen)
2077	{
2078	struct tcp_ao_getsockopt opt_in, opt_out;
2079	struct tcp_ao_key *key, *current_key;
2080	bool do_address_matching = true;
2081	union tcp_ao_addr *addr = NULL;
2082	int err, l3index, user_len;
2083	unsigned int max_keys; /* maximum number of keys to copy to user */
2084	size_t out_offset = 0;
2085	size_t bytes_to_write; /* number of bytes to write to user level */
2086	u32 matched_keys; /* keys from ao_info matched so far */
2087	int optlen_out;
2088	__be16 port = 0;
2089
2090	if (copy_from_sockptr(dst: &user_len, src: optlen, size: sizeof(int)))
2091	return -EFAULT;
2092
2093	if (user_len <= 0)
2094	return -EINVAL;
2095
2096	memset(&opt_in, 0, sizeof(struct tcp_ao_getsockopt));
2097	err = copy_struct_from_sockptr(dst: &opt_in, ksize: sizeof(opt_in),
2098	src: optval, usize: user_len);
2099	if (err < 0)
2100	return err;
2101
2102	if (opt_in.pkt_good || opt_in.pkt_bad)
2103	return -EINVAL;
2104	if (opt_in.keyflags & ~TCP_AO_GET_KEYF_VALID)
2105	return -EINVAL;
2106	if (opt_in.ifindex && !(opt_in.keyflags & TCP_AO_KEYF_IFINDEX))
2107	return -EINVAL;
2108
2109	if (opt_in.reserved != 0)
2110	return -EINVAL;
2111
2112	max_keys = opt_in.nkeys;
2113	l3index = (opt_in.keyflags & TCP_AO_KEYF_IFINDEX) ? opt_in.ifindex : -1;
2114
2115	if (opt_in.get_all || opt_in.is_current || opt_in.is_rnext) {
2116	if (opt_in.get_all && (opt_in.is_current || opt_in.is_rnext))
2117	return -EINVAL;
2118	do_address_matching = false;
2119	}
2120
2121	switch (opt_in.addr.ss_family) {
2122	case AF_INET: {
2123	struct sockaddr_in *sin;
2124	__be32 mask;
2125
2126	sin = (struct sockaddr_in *)&opt_in.addr;
2127	port = sin->sin_port;
2128	addr = (union tcp_ao_addr *)&sin->sin_addr;
2129
2130	if (opt_in.prefix > 32)
2131	return -EINVAL;
2132
2133	if (ntohl(sin->sin_addr.s_addr) == INADDR_ANY &&
2134	opt_in.prefix != 0)
2135	return -EINVAL;
2136
2137	mask = inet_make_mask(logmask: opt_in.prefix);
2138	if (sin->sin_addr.s_addr & ~mask)
2139	return -EINVAL;
2140
2141	break;
2142	}
2143	case AF_INET6: {
2144	struct sockaddr_in6 *sin6;
2145	struct in6_addr *addr6;
2146
2147	sin6 = (struct sockaddr_in6 *)&opt_in.addr;
2148	addr = (union tcp_ao_addr *)&sin6->sin6_addr;
2149	addr6 = &sin6->sin6_addr;
2150	port = sin6->sin6_port;
2151
2152	/* We don't have to change family and @addr here if
2153	* ipv6_addr_v4mapped() like in key adding:
2154	* tcp_ao_key_cmp() does it. Do the sanity checks though.
2155	*/
2156	if (opt_in.prefix != 0) {
2157	if (ipv6_addr_v4mapped(a: addr6)) {
2158	__be32 mask, addr4 = addr6->s6_addr32[3];
2159
2160	if (opt_in.prefix > 32 ||
2161	ntohl(addr4) == INADDR_ANY)
2162	return -EINVAL;
2163	mask = inet_make_mask(logmask: opt_in.prefix);
2164	if (addr4 & ~mask)
2165	return -EINVAL;
2166	} else {
2167	struct in6_addr pfx;
2168
2169	if (ipv6_addr_any(a: addr6) ||
2170	opt_in.prefix > 128)
2171	return -EINVAL;
2172
2173	ipv6_addr_prefix(pfx: &pfx, addr: addr6, plen: opt_in.prefix);
2174	if (ipv6_addr_cmp(a1: &pfx, a2: addr6))
2175	return -EINVAL;
2176	}
2177	} else if (!ipv6_addr_any(a: addr6)) {
2178	return -EINVAL;
2179	}
2180	break;
2181	}
2182	case 0:
2183	if (!do_address_matching)
2184	break;
2185	fallthrough;
2186	default:
2187	return -EAFNOSUPPORT;
2188	}
2189
2190	if (!do_address_matching) {
2191	/* We could just ignore those, but let's do stricter checks */
2192	if (addr || port)
2193	return -EINVAL;
2194	if (opt_in.prefix || opt_in.sndid || opt_in.rcvid)
2195	return -EINVAL;
2196	}
2197
2198	bytes_to_write = min_t(int, user_len, sizeof(struct tcp_ao_getsockopt));
2199	matched_keys = 0;
2200	/* May change in RX, while we're dumping, pre-fetch it */
2201	current_key = READ_ONCE(ao_info->current_key);
2202
2203	hlist_for_each_entry_rcu(key, &ao_info->head, node) {
2204	if (opt_in.get_all)
2205	goto match;
2206
2207	if (opt_in.is_current || opt_in.is_rnext) {
2208	if (opt_in.is_current && key == current_key)
2209	goto match;
2210	if (opt_in.is_rnext && key == ao_info->rnext_key)
2211	goto match;
2212	continue;
2213	}
2214
2215	if (tcp_ao_key_cmp(key, l3index, addr, prefixlen: opt_in.prefix,
2216	family: opt_in.addr.ss_family,
2217	sndid: opt_in.sndid, rcvid: opt_in.rcvid) != 0)
2218	continue;
2219	match:
2220	matched_keys++;
2221	if (matched_keys > max_keys)
2222	continue;
2223
2224	memset(&opt_out, 0, sizeof(struct tcp_ao_getsockopt));
2225
2226	if (key->family == AF_INET) {
2227	struct sockaddr_in *sin_out = (struct sockaddr_in *)&opt_out.addr;
2228
2229	sin_out->sin_family = key->family;
2230	sin_out->sin_port = 0;
2231	memcpy(&sin_out->sin_addr, &key->addr, sizeof(struct in_addr));
2232	} else {
2233	struct sockaddr_in6 *sin6_out = (struct sockaddr_in6 *)&opt_out.addr;
2234
2235	sin6_out->sin6_family = key->family;
2236	sin6_out->sin6_port = 0;
2237	memcpy(&sin6_out->sin6_addr, &key->addr, sizeof(struct in6_addr));
2238	}
2239	opt_out.sndid = key->sndid;
2240	opt_out.rcvid = key->rcvid;
2241	opt_out.prefix = key->prefixlen;
2242	opt_out.keyflags = key->keyflags;
2243	opt_out.is_current = (key == current_key);
2244	opt_out.is_rnext = (key == ao_info->rnext_key);
2245	opt_out.nkeys = 0;
2246	opt_out.maclen = key->maclen;
2247	opt_out.keylen = key->keylen;
2248	opt_out.ifindex = key->l3index;
2249	opt_out.pkt_good = atomic64_read(v: &key->pkt_good);
2250	opt_out.pkt_bad = atomic64_read(v: &key->pkt_bad);
2251	memcpy(&opt_out.key, key->key, key->keylen);
2252	tcp_sigpool_algo(id: key->tcp_sigpool_id, buf: opt_out.alg_name, buf_len: 64);
2253
2254	/* Copy key to user */
2255	if (copy_to_sockptr_offset(dst: optval, offset: out_offset,
2256	src: &opt_out, size: bytes_to_write))
2257	return -EFAULT;
2258	out_offset += user_len;
2259	}
2260
2261	optlen_out = (int)sizeof(struct tcp_ao_getsockopt);
2262	if (copy_to_sockptr(dst: optlen, src: &optlen_out, size: sizeof(int)))
2263	return -EFAULT;
2264
2265	out_offset = offsetof(struct tcp_ao_getsockopt, nkeys);
2266	if (copy_to_sockptr_offset(dst: optval, offset: out_offset,
2267	src: &matched_keys, size: sizeof(u32)))
2268	return -EFAULT;
2269
2270	return 0;
2271	}
2272
2273	int tcp_ao_get_mkts(struct sock *sk, sockptr_t optval, sockptr_t optlen)
2274	{
2275	struct tcp_ao_info *ao_info;
2276
2277	ao_info = setsockopt_ao_info(sk);
2278	if (IS_ERR(ptr: ao_info))
2279	return PTR_ERR(ptr: ao_info);
2280	if (!ao_info)
2281	return -ENOENT;
2282
2283	return tcp_ao_copy_mkts_to_user(ao_info, optval, optlen);
2284	}
2285
2286	int tcp_ao_get_sock_info(struct sock *sk, sockptr_t optval, sockptr_t optlen)
2287	{
2288	struct tcp_ao_info_opt out, in = {};
2289	struct tcp_ao_key *current_key;
2290	struct tcp_ao_info *ao;
2291	int err, len;
2292
2293	if (copy_from_sockptr(dst: &len, src: optlen, size: sizeof(int)))
2294	return -EFAULT;
2295
2296	if (len <= 0)
2297	return -EINVAL;
2298
2299	/* Copying this "in" only to check ::reserved, ::reserved2,
2300	* that may be needed to extend (struct tcp_ao_info_opt) and
2301	* what getsockopt() provides in future.
2302	*/
2303	err = copy_struct_from_sockptr(dst: &in, ksize: sizeof(in), src: optval, usize: len);
2304	if (err)
2305	return err;
2306
2307	if (in.reserved != 0 || in.reserved2 != 0)
2308	return -EINVAL;
2309
2310	ao = setsockopt_ao_info(sk);
2311	if (IS_ERR(ptr: ao))
2312	return PTR_ERR(ptr: ao);
2313	if (!ao)
2314	return -ENOENT;
2315
2316	memset(&out, 0, sizeof(out));
2317	out.ao_required = ao->ao_required;
2318	out.accept_icmps = ao->accept_icmps;
2319	out.pkt_good = atomic64_read(v: &ao->counters.pkt_good);
2320	out.pkt_bad = atomic64_read(v: &ao->counters.pkt_bad);
2321	out.pkt_key_not_found = atomic64_read(v: &ao->counters.key_not_found);
2322	out.pkt_ao_required = atomic64_read(v: &ao->counters.ao_required);
2323	out.pkt_dropped_icmp = atomic64_read(v: &ao->counters.dropped_icmp);
2324
2325	current_key = READ_ONCE(ao->current_key);
2326	if (current_key) {
2327	out.set_current = 1;
2328	out.current_key = current_key->sndid;
2329	}
2330	if (ao->rnext_key) {
2331	out.set_rnext = 1;
2332	out.rnext = ao->rnext_key->rcvid;
2333	}
2334
2335	if (copy_to_sockptr(dst: optval, src: &out, min_t(int, len, sizeof(out))))
2336	return -EFAULT;
2337
2338	return 0;
2339	}
2340
2341	int tcp_ao_set_repair(struct sock *sk, sockptr_t optval, unsigned int optlen)
2342	{
2343	struct tcp_sock *tp = tcp_sk(sk);
2344	struct tcp_ao_repair cmd;
2345	struct tcp_ao_key *key;
2346	struct tcp_ao_info *ao;
2347	int err;
2348
2349	if (optlen < sizeof(cmd))
2350	return -EINVAL;
2351
2352	err = copy_struct_from_sockptr(dst: &cmd, ksize: sizeof(cmd), src: optval, usize: optlen);
2353	if (err)
2354	return err;
2355
2356	if (!tp->repair)
2357	return -EPERM;
2358
2359	ao = setsockopt_ao_info(sk);
2360	if (IS_ERR(ptr: ao))
2361	return PTR_ERR(ptr: ao);
2362	if (!ao)
2363	return -ENOENT;
2364
2365	WRITE_ONCE(ao->lisn, cmd.snt_isn);
2366	WRITE_ONCE(ao->risn, cmd.rcv_isn);
2367	WRITE_ONCE(ao->snd_sne, cmd.snd_sne);
2368	WRITE_ONCE(ao->rcv_sne, cmd.rcv_sne);
2369
2370	hlist_for_each_entry_rcu(key, &ao->head, node)
2371	tcp_ao_cache_traffic_keys(sk, ao, ao_key: key);
2372
2373	return 0;
2374	}
2375
2376	int tcp_ao_get_repair(struct sock *sk, sockptr_t optval, sockptr_t optlen)
2377	{
2378	struct tcp_sock *tp = tcp_sk(sk);
2379	struct tcp_ao_repair opt;
2380	struct tcp_ao_info *ao;
2381	int len;
2382
2383	if (copy_from_sockptr(dst: &len, src: optlen, size: sizeof(int)))
2384	return -EFAULT;
2385
2386	if (len <= 0)
2387	return -EINVAL;
2388
2389	if (!tp->repair)
2390	return -EPERM;
2391
2392	rcu_read_lock();
2393	ao = getsockopt_ao_info(sk);
2394	if (IS_ERR_OR_NULL(ptr: ao)) {
2395	rcu_read_unlock();
2396	return ao ? PTR_ERR(ptr: ao) : -ENOENT;
2397	}
2398
2399	opt.snt_isn = ao->lisn;
2400	opt.rcv_isn = ao->risn;
2401	opt.snd_sne = READ_ONCE(ao->snd_sne);
2402	opt.rcv_sne = READ_ONCE(ao->rcv_sne);
2403	rcu_read_unlock();
2404
2405	if (copy_to_sockptr(dst: optval, src: &opt, min_t(int, len, sizeof(opt))))
2406	return -EFAULT;
2407	return 0;
2408	}
2409