1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* INET An implementation of the TCP/IP protocol suite for the LINUX
4	* operating system. INET is implemented using the BSD Socket
5	* interface as the means of communication with the user level.
6	*
7	* The User Datagram Protocol (UDP).
8	*
9	* Authors: Ross Biro
10	* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11	* Arnt Gulbrandsen, <agulbra@nvg.unit.no>
12	* Alan Cox, <alan@lxorguk.ukuu.org.uk>
13	* Hirokazu Takahashi, <taka@valinux.co.jp>
14	*
15	* Fixes:
16	* Alan Cox : verify_area() calls
17	* Alan Cox : stopped close while in use off icmp
18	* messages. Not a fix but a botch that
19	* for udp at least is 'valid'.
20	* Alan Cox : Fixed icmp handling properly
21	* Alan Cox : Correct error for oversized datagrams
22	* Alan Cox : Tidied select() semantics.
23	* Alan Cox : udp_err() fixed properly, also now
24	* select and read wake correctly on errors
25	* Alan Cox : udp_send verify_area moved to avoid mem leak
26	* Alan Cox : UDP can count its memory
27	* Alan Cox : send to an unknown connection causes
28	* an ECONNREFUSED off the icmp, but
29	* does NOT close.
30	* Alan Cox : Switched to new sk_buff handlers. No more backlog!
31	* Alan Cox : Using generic datagram code. Even smaller and the PEEK
32	* bug no longer crashes it.
33	* Fred Van Kempen : Net2e support for sk->broadcast.
34	* Alan Cox : Uses skb_free_datagram
35	* Alan Cox : Added get/set sockopt support.
36	* Alan Cox : Broadcasting without option set returns EACCES.
37	* Alan Cox : No wakeup calls. Instead we now use the callbacks.
38	* Alan Cox : Use ip_tos and ip_ttl
39	* Alan Cox : SNMP Mibs
40	* Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support.
41	* Matt Dillon : UDP length checks.
42	* Alan Cox : Smarter af_inet used properly.
43	* Alan Cox : Use new kernel side addressing.
44	* Alan Cox : Incorrect return on truncated datagram receive.
45	* Arnt Gulbrandsen : New udp_send and stuff
46	* Alan Cox : Cache last socket
47	* Alan Cox : Route cache
48	* Jon Peatfield : Minor efficiency fix to sendto().
49	* Mike Shaver : RFC1122 checks.
50	* Alan Cox : Nonblocking error fix.
51	* Willy Konynenberg : Transparent proxying support.
52	* Mike McLagan : Routing by source
53	* David S. Miller : New socket lookup architecture.
54	* Last socket cache retained as it
55	* does have a high hit rate.
56	* Olaf Kirch : Don't linearise iovec on sendmsg.
57	* Andi Kleen : Some cleanups, cache destination entry
58	* for connect.
59	* Vitaly E. Lavrov : Transparent proxy revived after year coma.
60	* Melvin Smith : Check msg_name not msg_namelen in sendto(),
61	* return ENOTCONN for unconnected sockets (POSIX)
62	* Janos Farkas : don't deliver multi/broadcasts to a different
63	* bound-to-device socket
64	* Hirokazu Takahashi : HW checksumming for outgoing UDP
65	* datagrams.
66	* Hirokazu Takahashi : sendfile() on UDP works now.
67	* Arnaldo C. Melo : convert /proc/net/udp to seq_file
68	* YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
69	* Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind
70	* a single port at the same time.
71	* Derek Atkins <derek@ihtfp.com>: Add Encapulation Support
72	* James Chapman : Add L2TP encapsulation type.
73	*/
74
75	#define pr_fmt(fmt) "UDP: " fmt
76
77	#include <linux/bpf-cgroup.h>
78	#include <linux/uaccess.h>
79	#include <asm/ioctls.h>
80	#include <linux/memblock.h>
81	#include <linux/highmem.h>
82	#include <linux/types.h>
83	#include <linux/fcntl.h>
84	#include <linux/module.h>
85	#include <linux/socket.h>
86	#include <linux/sockios.h>
87	#include <linux/igmp.h>
88	#include <linux/inetdevice.h>
89	#include <linux/in.h>
90	#include <linux/errno.h>
91	#include <linux/timer.h>
92	#include <linux/mm.h>
93	#include <linux/inet.h>
94	#include <linux/netdevice.h>
95	#include <linux/slab.h>
96	#include <net/tcp_states.h>
97	#include <linux/skbuff.h>
98	#include <linux/proc_fs.h>
99	#include <linux/seq_file.h>
100	#include <net/net_namespace.h>
101	#include <net/icmp.h>
102	#include <net/inet_hashtables.h>
103	#include <net/ip_tunnels.h>
104	#include <net/route.h>
105	#include <net/checksum.h>
106	#include <net/gso.h>
107	#include <net/xfrm.h>
108	#include <trace/events/udp.h>
109	#include <linux/static_key.h>
110	#include <linux/btf_ids.h>
111	#include <trace/events/skb.h>
112	#include <net/busy_poll.h>
113	#include "udp_impl.h"
114	#include <net/sock_reuseport.h>
115	#include <net/addrconf.h>
116	#include <net/udp_tunnel.h>
117	#include <net/gro.h>
118	#if IS_ENABLED(CONFIG_IPV6)
119	#include <net/ipv6_stubs.h>
120	#endif
121
122	struct udp_table udp_table __read_mostly;
123	EXPORT_SYMBOL(udp_table);
124
125	long sysctl_udp_mem[3] __read_mostly;
126	EXPORT_SYMBOL(sysctl_udp_mem);
127
128	atomic_long_t udp_memory_allocated ____cacheline_aligned_in_smp;
129	EXPORT_SYMBOL(udp_memory_allocated);
130	DEFINE_PER_CPU(int, udp_memory_per_cpu_fw_alloc);
131	EXPORT_PER_CPU_SYMBOL_GPL(udp_memory_per_cpu_fw_alloc);
132
133	#define MAX_UDP_PORTS 65536
134	#define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN_PERNET)
135
136	static struct udp_table *udp_get_table_prot(struct sock *sk)
137	{
138	return sk->sk_prot->h.udp_table ? : sock_net(sk)->ipv4.udp_table;
139	}
140
141	static int udp_lib_lport_inuse(struct net *net, __u16 num,
142	const struct udp_hslot *hslot,
143	unsigned long *bitmap,
144	struct sock *sk, unsigned int log)
145	{
146	struct sock *sk2;
147	kuid_t uid = sock_i_uid(sk);
148
149	sk_for_each(sk2, &hslot->head) {
150	if (net_eq(net1: sock_net(sk: sk2), net2: net) &&
151	sk2 != sk &&
152	(bitmap || udp_sk(sk2)->udp_port_hash == num) &&
153	(!sk2->sk_reuse || !sk->sk_reuse) &&
154	(!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
155	sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
156	inet_rcv_saddr_equal(sk, sk2, match_wildcard: true)) {
157	if (sk2->sk_reuseport && sk->sk_reuseport &&
158	!rcu_access_pointer(sk->sk_reuseport_cb) &&
159	uid_eq(left: uid, right: sock_i_uid(sk: sk2))) {
160	if (!bitmap)
161	return 0;
162	} else {
163	if (!bitmap)
164	return 1;
165	__set_bit(udp_sk(sk2)->udp_port_hash >> log,
166	bitmap);
167	}
168	}
169	}
170	return 0;
171	}
172
173	/*
174	* Note: we still hold spinlock of primary hash chain, so no other writer
175	* can insert/delete a socket with local_port == num
176	*/
177	static int udp_lib_lport_inuse2(struct net *net, __u16 num,
178	struct udp_hslot *hslot2,
179	struct sock *sk)
180	{
181	struct sock *sk2;
182	kuid_t uid = sock_i_uid(sk);
183	int res = 0;
184
185	spin_lock(lock: &hslot2->lock);
186	udp_portaddr_for_each_entry(sk2, &hslot2->head) {
187	if (net_eq(net1: sock_net(sk: sk2), net2: net) &&
188	sk2 != sk &&
189	(udp_sk(sk2)->udp_port_hash == num) &&
190	(!sk2->sk_reuse || !sk->sk_reuse) &&
191	(!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
192	sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
193	inet_rcv_saddr_equal(sk, sk2, match_wildcard: true)) {
194	if (sk2->sk_reuseport && sk->sk_reuseport &&
195	!rcu_access_pointer(sk->sk_reuseport_cb) &&
196	uid_eq(left: uid, right: sock_i_uid(sk: sk2))) {
197	res = 0;
198	} else {
199	res = 1;
200	}
201	break;
202	}
203	}
204	spin_unlock(lock: &hslot2->lock);
205	return res;
206	}
207
208	static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot)
209	{
210	struct net *net = sock_net(sk);
211	kuid_t uid = sock_i_uid(sk);
212	struct sock *sk2;
213
214	sk_for_each(sk2, &hslot->head) {
215	if (net_eq(net1: sock_net(sk: sk2), net2: net) &&
216	sk2 != sk &&
217	sk2->sk_family == sk->sk_family &&
218	ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
219	(udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) &&
220	(sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
221	sk2->sk_reuseport && uid_eq(left: uid, right: sock_i_uid(sk: sk2)) &&
222	inet_rcv_saddr_equal(sk, sk2, match_wildcard: false)) {
223	return reuseport_add_sock(sk, sk2,
224	bind_inany: inet_rcv_saddr_any(sk));
225	}
226	}
227
228	return reuseport_alloc(sk, bind_inany: inet_rcv_saddr_any(sk));
229	}
230
231	/**
232	* udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6
233	*
234	* @sk: socket struct in question
235	* @snum: port number to look up
236	* @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
237	* with NULL address
238	*/
239	int udp_lib_get_port(struct sock *sk, unsigned short snum,
240	unsigned int hash2_nulladdr)
241	{
242	struct udp_table *udptable = udp_get_table_prot(sk);
243	struct udp_hslot *hslot, *hslot2;
244	struct net *net = sock_net(sk);
245	int error = -EADDRINUSE;
246
247	if (!snum) {
248	DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);
249	unsigned short first, last;
250	int low, high, remaining;
251	unsigned int rand;
252
253	inet_sk_get_local_port_range(sk, low: &low, high: &high);
254	remaining = (high - low) + 1;
255
256	rand = get_random_u32();
257	first = reciprocal_scale(val: rand, ep_ro: remaining) + low;
258	/*
259	* force rand to be an odd multiple of UDP_HTABLE_SIZE
260	*/
261	rand = (rand | 1) * (udptable->mask + 1);
262	last = first + udptable->mask + 1;
263	do {
264	hslot = udp_hashslot(table: udptable, net, num: first);
265	bitmap_zero(dst: bitmap, PORTS_PER_CHAIN);
266	spin_lock_bh(lock: &hslot->lock);
267	udp_lib_lport_inuse(net, num: snum, hslot, bitmap, sk,
268	log: udptable->log);
269
270	snum = first;
271	/*
272	* Iterate on all possible values of snum for this hash.
273	* Using steps of an odd multiple of UDP_HTABLE_SIZE
274	* give us randomization and full range coverage.
275	*/
276	do {
277	if (low <= snum && snum <= high &&
278	!test_bit(snum >> udptable->log, bitmap) &&
279	!inet_is_local_reserved_port(net, port: snum))
280	goto found;
281	snum += rand;
282	} while (snum != first);
283	spin_unlock_bh(lock: &hslot->lock);
284	cond_resched();
285	} while (++first != last);
286	goto fail;
287	} else {
288	hslot = udp_hashslot(table: udptable, net, num: snum);
289	spin_lock_bh(lock: &hslot->lock);
290	if (hslot->count > 10) {
291	int exist;
292	unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum;
293
294	slot2 &= udptable->mask;
295	hash2_nulladdr &= udptable->mask;
296
297	hslot2 = udp_hashslot2(table: udptable, hash: slot2);
298	if (hslot->count < hslot2->count)
299	goto scan_primary_hash;
300
301	exist = udp_lib_lport_inuse2(net, num: snum, hslot2, sk);
302	if (!exist && (hash2_nulladdr != slot2)) {
303	hslot2 = udp_hashslot2(table: udptable, hash: hash2_nulladdr);
304	exist = udp_lib_lport_inuse2(net, num: snum, hslot2,
305	sk);
306	}
307	if (exist)
308	goto fail_unlock;
309	else
310	goto found;
311	}
312	scan_primary_hash:
313	if (udp_lib_lport_inuse(net, num: snum, hslot, NULL, sk, log: 0))
314	goto fail_unlock;
315	}
316	found:
317	inet_sk(sk)->inet_num = snum;
318	udp_sk(sk)->udp_port_hash = snum;
319	udp_sk(sk)->udp_portaddr_hash ^= snum;
320	if (sk_unhashed(sk)) {
321	if (sk->sk_reuseport &&
322	udp_reuseport_add_sock(sk, hslot)) {
323	inet_sk(sk)->inet_num = 0;
324	udp_sk(sk)->udp_port_hash = 0;
325	udp_sk(sk)->udp_portaddr_hash ^= snum;
326	goto fail_unlock;
327	}
328
329	sk_add_node_rcu(sk, list: &hslot->head);
330	hslot->count++;
331	sock_prot_inuse_add(net: sock_net(sk), prot: sk->sk_prot, val: 1);
332
333	hslot2 = udp_hashslot2(table: udptable, udp_sk(sk)->udp_portaddr_hash);
334	spin_lock(lock: &hslot2->lock);
335	if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
336	sk->sk_family == AF_INET6)
337	hlist_add_tail_rcu(n: &udp_sk(sk)->udp_portaddr_node,
338	h: &hslot2->head);
339	else
340	hlist_add_head_rcu(n: &udp_sk(sk)->udp_portaddr_node,
341	h: &hslot2->head);
342	hslot2->count++;
343	spin_unlock(lock: &hslot2->lock);
344	}
345	sock_set_flag(sk, flag: SOCK_RCU_FREE);
346	error = 0;
347	fail_unlock:
348	spin_unlock_bh(lock: &hslot->lock);
349	fail:
350	return error;
351	}
352	EXPORT_SYMBOL(udp_lib_get_port);
353
354	int udp_v4_get_port(struct sock *sk, unsigned short snum)
355	{
356	unsigned int hash2_nulladdr =
357	ipv4_portaddr_hash(net: sock_net(sk), htonl(INADDR_ANY), port: snum);
358	unsigned int hash2_partial =
359	ipv4_portaddr_hash(net: sock_net(sk), inet_sk(sk)->inet_rcv_saddr, port: 0);
360
361	/* precompute partial secondary hash */
362	udp_sk(sk)->udp_portaddr_hash = hash2_partial;
363	return udp_lib_get_port(sk, snum, hash2_nulladdr);
364	}
365
366	static int compute_score(struct sock *sk, struct net *net,
367	__be32 saddr, __be16 sport,
368	__be32 daddr, unsigned short hnum,
369	int dif, int sdif)
370	{
371	int score;
372	struct inet_sock *inet;
373	bool dev_match;
374
375	if (!net_eq(net1: sock_net(sk), net2: net) ||
376	udp_sk(sk)->udp_port_hash != hnum ||
377	ipv6_only_sock(sk))
378	return -1;
379
380	if (sk->sk_rcv_saddr != daddr)
381	return -1;
382
383	score = (sk->sk_family == PF_INET) ? 2 : 1;
384
385	inet = inet_sk(sk);
386	if (inet->inet_daddr) {
387	if (inet->inet_daddr != saddr)
388	return -1;
389	score += 4;
390	}
391
392	if (inet->inet_dport) {
393	if (inet->inet_dport != sport)
394	return -1;
395	score += 4;
396	}
397
398	dev_match = udp_sk_bound_dev_eq(net, bound_dev_if: sk->sk_bound_dev_if,
399	dif, sdif);
400	if (!dev_match)
401	return -1;
402	if (sk->sk_bound_dev_if)
403	score += 4;
404
405	if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
406	score++;
407	return score;
408	}
409
410	INDIRECT_CALLABLE_SCOPE
411	u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
412	const __be32 faddr, const __be16 fport)
413	{
414	static u32 udp_ehash_secret __read_mostly;
415
416	net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
417
418	return __inet_ehashfn(laddr, lport, faddr, fport,
419	initval: udp_ehash_secret + net_hash_mix(net));
420	}
421
422	/* called with rcu_read_lock() */
423	static struct sock *udp4_lib_lookup2(struct net *net,
424	__be32 saddr, __be16 sport,
425	__be32 daddr, unsigned int hnum,
426	int dif, int sdif,
427	struct udp_hslot *hslot2,
428	struct sk_buff *skb)
429	{
430	struct sock *sk, *result;
431	int score, badness;
432
433	result = NULL;
434	badness = 0;
435	udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
436	score = compute_score(sk, net, saddr, sport,
437	daddr, hnum, dif, sdif);
438	if (score > badness) {
439	badness = score;
440
441	if (sk->sk_state == TCP_ESTABLISHED) {
442	result = sk;
443	continue;
444	}
445
446	result = inet_lookup_reuseport(net, sk, skb, doff: sizeof(struct udphdr),
447	saddr, sport, daddr, hnum, ehashfn: udp_ehashfn);
448	if (!result) {
449	result = sk;
450	continue;
451	}
452
453	/* Fall back to scoring if group has connections */
454	if (!reuseport_has_conns(sk))
455	return result;
456
457	/* Reuseport logic returned an error, keep original score. */
458	if (IS_ERR(ptr: result))
459	continue;
460
461	badness = compute_score(sk: result, net, saddr, sport,
462	daddr, hnum, dif, sdif);
463
464	}
465	}
466	return result;
467	}
468
469	/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
470	* harder than this. -DaveM
471	*/
472	struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
473	__be16 sport, __be32 daddr, __be16 dport, int dif,
474	int sdif, struct udp_table *udptable, struct sk_buff *skb)
475	{
476	unsigned short hnum = ntohs(dport);
477	unsigned int hash2, slot2;
478	struct udp_hslot *hslot2;
479	struct sock *result, *sk;
480
481	hash2 = ipv4_portaddr_hash(net, saddr: daddr, port: hnum);
482	slot2 = hash2 & udptable->mask;
483	hslot2 = &udptable->hash2[slot2];
484
485	/* Lookup connected or non-wildcard socket */
486	result = udp4_lib_lookup2(net, saddr, sport,
487	daddr, hnum, dif, sdif,
488	hslot2, skb);
489	if (!IS_ERR_OR_NULL(ptr: result) && result->sk_state == TCP_ESTABLISHED)
490	goto done;
491
492	/* Lookup redirect from BPF */
493	if (static_branch_unlikely(&bpf_sk_lookup_enabled) &&
494	udptable == net->ipv4.udp_table) {
495	sk = inet_lookup_run_sk_lookup(net, IPPROTO_UDP, skb, doff: sizeof(struct udphdr),
496	saddr, sport, daddr, hnum, dif,
497	ehashfn: udp_ehashfn);
498	if (sk) {
499	result = sk;
500	goto done;
501	}
502	}
503
504	/* Got non-wildcard socket or error on first lookup */
505	if (result)
506	goto done;
507
508	/* Lookup wildcard sockets */
509	hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), port: hnum);
510	slot2 = hash2 & udptable->mask;
511	hslot2 = &udptable->hash2[slot2];
512
513	result = udp4_lib_lookup2(net, saddr, sport,
514	htonl(INADDR_ANY), hnum, dif, sdif,
515	hslot2, skb);
516	done:
517	if (IS_ERR(ptr: result))
518	return NULL;
519	return result;
520	}
521	EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
522
523	static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
524	__be16 sport, __be16 dport,
525	struct udp_table *udptable)
526	{
527	const struct iphdr *iph = ip_hdr(skb);
528
529	return __udp4_lib_lookup(dev_net(dev: skb->dev), iph->saddr, sport,
530	iph->daddr, dport, inet_iif(skb),
531	inet_sdif(skb), udptable, skb);
532	}
533
534	struct sock *udp4_lib_lookup_skb(const struct sk_buff *skb,
535	__be16 sport, __be16 dport)
536	{
537	const struct iphdr *iph = ip_hdr(skb);
538	struct net *net = dev_net(dev: skb->dev);
539	int iif, sdif;
540
541	inet_get_iif_sdif(skb, iif: &iif, sdif: &sdif);
542
543	return __udp4_lib_lookup(net, iph->saddr, sport,
544	iph->daddr, dport, iif,
545	sdif, net->ipv4.udp_table, NULL);
546	}
547
548	/* Must be called under rcu_read_lock().
549	* Does increment socket refcount.
550	*/
551	#if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4)
552	struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport,
553	__be32 daddr, __be16 dport, int dif)
554	{
555	struct sock *sk;
556
557	sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport,
558	dif, 0, net->ipv4.udp_table, NULL);
559	if (sk && !refcount_inc_not_zero(r: &sk->sk_refcnt))
560	sk = NULL;
561	return sk;
562	}
563	EXPORT_SYMBOL_GPL(udp4_lib_lookup);
564	#endif
565
566	static inline bool __udp_is_mcast_sock(struct net *net, const struct sock *sk,
567	__be16 loc_port, __be32 loc_addr,
568	__be16 rmt_port, __be32 rmt_addr,
569	int dif, int sdif, unsigned short hnum)
570	{
571	const struct inet_sock *inet = inet_sk(sk);
572
573	if (!net_eq(net1: sock_net(sk), net2: net) ||
574	udp_sk(sk)->udp_port_hash != hnum ||
575	(inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
576	(inet->inet_dport != rmt_port && inet->inet_dport) ||
577	(inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
578	ipv6_only_sock(sk) ||
579	!udp_sk_bound_dev_eq(net, bound_dev_if: sk->sk_bound_dev_if, dif, sdif))
580	return false;
581	if (!ip_mc_sf_allow(sk, local: loc_addr, rmt: rmt_addr, dif, sdif))
582	return false;
583	return true;
584	}
585
586	DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key);
587	void udp_encap_enable(void)
588	{
589	static_branch_inc(&udp_encap_needed_key);
590	}
591	EXPORT_SYMBOL(udp_encap_enable);
592
593	void udp_encap_disable(void)
594	{
595	static_branch_dec(&udp_encap_needed_key);
596	}
597	EXPORT_SYMBOL(udp_encap_disable);
598
599	/* Handler for tunnels with arbitrary destination ports: no socket lookup, go
600	* through error handlers in encapsulations looking for a match.
601	*/
602	static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info)
603	{
604	int i;
605
606	for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) {
607	int (*handler)(struct sk_buff *skb, u32 info);
608	const struct ip_tunnel_encap_ops *encap;
609
610	encap = rcu_dereference(iptun_encaps[i]);
611	if (!encap)
612	continue;
613	handler = encap->err_handler;
614	if (handler && !handler(skb, info))
615	return 0;
616	}
617
618	return -ENOENT;
619	}
620
621	/* Try to match ICMP errors to UDP tunnels by looking up a socket without
622	* reversing source and destination port: this will match tunnels that force the
623	* same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that
624	* lwtunnels might actually break this assumption by being configured with
625	* different destination ports on endpoints, in this case we won't be able to
626	* trace ICMP messages back to them.
627	*
628	* If this doesn't match any socket, probe tunnels with arbitrary destination
629	* ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port
630	* we've sent packets to won't necessarily match the local destination port.
631	*
632	* Then ask the tunnel implementation to match the error against a valid
633	* association.
634	*
635	* Return an error if we can't find a match, the socket if we need further
636	* processing, zero otherwise.
637	*/
638	static struct sock *__udp4_lib_err_encap(struct net *net,
639	const struct iphdr *iph,
640	struct udphdr *uh,
641	struct udp_table *udptable,
642	struct sock *sk,
643	struct sk_buff *skb, u32 info)
644	{
645	int (*lookup)(struct sock *sk, struct sk_buff *skb);
646	int network_offset, transport_offset;
647	struct udp_sock *up;
648
649	network_offset = skb_network_offset(skb);
650	transport_offset = skb_transport_offset(skb);
651
652	/* Network header needs to point to the outer IPv4 header inside ICMP */
653	skb_reset_network_header(skb);
654
655	/* Transport header needs to point to the UDP header */
656	skb_set_transport_header(skb, offset: iph->ihl << 2);
657
658	if (sk) {
659	up = udp_sk(sk);
660
661	lookup = READ_ONCE(up->encap_err_lookup);
662	if (lookup && lookup(sk, skb))
663	sk = NULL;
664
665	goto out;
666	}
667
668	sk = __udp4_lib_lookup(net, iph->daddr, uh->source,
669	iph->saddr, uh->dest, skb->dev->ifindex, 0,
670	udptable, NULL);
671	if (sk) {
672	up = udp_sk(sk);
673
674	lookup = READ_ONCE(up->encap_err_lookup);
675	if (!lookup || lookup(sk, skb))
676	sk = NULL;
677	}
678
679	out:
680	if (!sk)
681	sk = ERR_PTR(error: __udp4_lib_err_encap_no_sk(skb, info));
682
683	skb_set_transport_header(skb, offset: transport_offset);
684	skb_set_network_header(skb, offset: network_offset);
685
686	return sk;
687	}
688
689	/*
690	* This routine is called by the ICMP module when it gets some
691	* sort of error condition. If err < 0 then the socket should
692	* be closed and the error returned to the user. If err > 0
693	* it's just the icmp type << 8 | icmp code.
694	* Header points to the ip header of the error packet. We move
695	* on past this. Then (as it used to claim before adjustment)
696	* header points to the first 8 bytes of the udp header. We need
697	* to find the appropriate port.
698	*/
699
700	int __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable)
701	{
702	struct inet_sock *inet;
703	const struct iphdr *iph = (const struct iphdr *)skb->data;
704	struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2));
705	const int type = icmp_hdr(skb)->type;
706	const int code = icmp_hdr(skb)->code;
707	bool tunnel = false;
708	struct sock *sk;
709	int harderr;
710	int err;
711	struct net *net = dev_net(dev: skb->dev);
712
713	sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
714	iph->saddr, uh->source, skb->dev->ifindex,
715	inet_sdif(skb), udptable, NULL);
716
717	if (!sk || READ_ONCE(udp_sk(sk)->encap_type)) {
718	/* No socket for error: try tunnels before discarding */
719	if (static_branch_unlikely(&udp_encap_needed_key)) {
720	sk = __udp4_lib_err_encap(net, iph, uh, udptable, sk, skb,
721	info);
722	if (!sk)
723	return 0;
724	} else
725	sk = ERR_PTR(error: -ENOENT);
726
727	if (IS_ERR(ptr: sk)) {
728	__ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
729	return PTR_ERR(ptr: sk);
730	}
731
732	tunnel = true;
733	}
734
735	err = 0;
736	harderr = 0;
737	inet = inet_sk(sk);
738
739	switch (type) {
740	default:
741	case ICMP_TIME_EXCEEDED:
742	err = EHOSTUNREACH;
743	break;
744	case ICMP_SOURCE_QUENCH:
745	goto out;
746	case ICMP_PARAMETERPROB:
747	err = EPROTO;
748	harderr = 1;
749	break;
750	case ICMP_DEST_UNREACH:
751	if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */
752	ipv4_sk_update_pmtu(skb, sk, mtu: info);
753	if (READ_ONCE(inet->pmtudisc) != IP_PMTUDISC_DONT) {
754	err = EMSGSIZE;
755	harderr = 1;
756	break;
757	}
758	goto out;
759	}
760	err = EHOSTUNREACH;
761	if (code <= NR_ICMP_UNREACH) {
762	harderr = icmp_err_convert[code].fatal;
763	err = icmp_err_convert[code].errno;
764	}
765	break;
766	case ICMP_REDIRECT:
767	ipv4_sk_redirect(skb, sk);
768	goto out;
769	}
770
771	/*
772	* RFC1122: OK. Passes ICMP errors back to application, as per
773	* 4.1.3.3.
774	*/
775	if (tunnel) {
776	/* ...not for tunnels though: we don't have a sending socket */
777	if (udp_sk(sk)->encap_err_rcv)
778	udp_sk(sk)->encap_err_rcv(sk, skb, err, uh->dest, info,
779	(u8 *)(uh+1));
780	goto out;
781	}
782	if (!inet_test_bit(RECVERR, sk)) {
783	if (!harderr || sk->sk_state != TCP_ESTABLISHED)
784	goto out;
785	} else
786	ip_icmp_error(sk, skb, err, port: uh->dest, info, payload: (u8 *)(uh+1));
787
788	sk->sk_err = err;
789	sk_error_report(sk);
790	out:
791	return 0;
792	}
793
794	int udp_err(struct sk_buff *skb, u32 info)
795	{
796	return __udp4_lib_err(skb, info, udptable: dev_net(dev: skb->dev)->ipv4.udp_table);
797	}
798
799	/*
800	* Throw away all pending data and cancel the corking. Socket is locked.
801	*/
802	void udp_flush_pending_frames(struct sock *sk)
803	{
804	struct udp_sock *up = udp_sk(sk);
805
806	if (up->pending) {
807	up->len = 0;
808	up->pending = 0;
809	ip_flush_pending_frames(sk);
810	}
811	}
812	EXPORT_SYMBOL(udp_flush_pending_frames);
813
814	/**
815	* udp4_hwcsum - handle outgoing HW checksumming
816	* @skb: sk_buff containing the filled-in UDP header
817	* (checksum field must be zeroed out)
818	* @src: source IP address
819	* @dst: destination IP address
820	*/
821	void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
822	{
823	struct udphdr *uh = udp_hdr(skb);
824	int offset = skb_transport_offset(skb);
825	int len = skb->len - offset;
826	int hlen = len;
827	__wsum csum = 0;
828
829	if (!skb_has_frag_list(skb)) {
830	/*
831	* Only one fragment on the socket.
832	*/
833	skb->csum_start = skb_transport_header(skb) - skb->head;
834	skb->csum_offset = offsetof(struct udphdr, check);
835	uh->check = ~csum_tcpudp_magic(saddr: src, daddr: dst, len,
836	IPPROTO_UDP, sum: 0);
837	} else {
838	struct sk_buff *frags;
839
840	/*
841	* HW-checksum won't work as there are two or more
842	* fragments on the socket so that all csums of sk_buffs
843	* should be together
844	*/
845	skb_walk_frags(skb, frags) {
846	csum = csum_add(csum, addend: frags->csum);
847	hlen -= frags->len;
848	}
849
850	csum = skb_checksum(skb, offset, len: hlen, csum);
851	skb->ip_summed = CHECKSUM_NONE;
852
853	uh->check = csum_tcpudp_magic(saddr: src, daddr: dst, len, IPPROTO_UDP, sum: csum);
854	if (uh->check == 0)
855	uh->check = CSUM_MANGLED_0;
856	}
857	}
858	EXPORT_SYMBOL_GPL(udp4_hwcsum);
859
860	/* Function to set UDP checksum for an IPv4 UDP packet. This is intended
861	* for the simple case like when setting the checksum for a UDP tunnel.
862	*/
863	void udp_set_csum(bool nocheck, struct sk_buff *skb,
864	__be32 saddr, __be32 daddr, int len)
865	{
866	struct udphdr *uh = udp_hdr(skb);
867
868	if (nocheck) {
869	uh->check = 0;
870	} else if (skb_is_gso(skb)) {
871	uh->check = ~udp_v4_check(len, saddr, daddr, base: 0);
872	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
873	uh->check = 0;
874	uh->check = udp_v4_check(len, saddr, daddr, base: lco_csum(skb));
875	if (uh->check == 0)
876	uh->check = CSUM_MANGLED_0;
877	} else {
878	skb->ip_summed = CHECKSUM_PARTIAL;
879	skb->csum_start = skb_transport_header(skb) - skb->head;
880	skb->csum_offset = offsetof(struct udphdr, check);
881	uh->check = ~udp_v4_check(len, saddr, daddr, base: 0);
882	}
883	}
884	EXPORT_SYMBOL(udp_set_csum);
885
886	static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,
887	struct inet_cork *cork)
888	{
889	struct sock *sk = skb->sk;
890	struct inet_sock *inet = inet_sk(sk);
891	struct udphdr *uh;
892	int err;
893	int is_udplite = IS_UDPLITE(sk);
894	int offset = skb_transport_offset(skb);
895	int len = skb->len - offset;
896	int datalen = len - sizeof(*uh);
897	__wsum csum = 0;
898
899	/*
900	* Create a UDP header
901	*/
902	uh = udp_hdr(skb);
903	uh->source = inet->inet_sport;
904	uh->dest = fl4->fl4_dport;
905	uh->len = htons(len);
906	uh->check = 0;
907
908	if (cork->gso_size) {
909	const int hlen = skb_network_header_len(skb) +
910	sizeof(struct udphdr);
911
912	if (hlen + cork->gso_size > cork->fragsize) {
913	kfree_skb(skb);
914	return -EINVAL;
915	}
916	if (datalen > cork->gso_size * UDP_MAX_SEGMENTS) {
917	kfree_skb(skb);
918	return -EINVAL;
919	}
920	if (sk->sk_no_check_tx) {
921	kfree_skb(skb);
922	return -EINVAL;
923	}
924	if (skb->ip_summed != CHECKSUM_PARTIAL || is_udplite ||
925	dst_xfrm(dst: skb_dst(skb))) {
926	kfree_skb(skb);
927	return -EIO;
928	}
929
930	if (datalen > cork->gso_size) {
931	skb_shinfo(skb)->gso_size = cork->gso_size;
932	skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
933	skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen,
934	cork->gso_size);
935	}
936	goto csum_partial;
937	}
938
939	if (is_udplite) /* UDP-Lite */
940	csum = udplite_csum(skb);
941
942	else if (sk->sk_no_check_tx) { /* UDP csum off */
943
944	skb->ip_summed = CHECKSUM_NONE;
945	goto send;
946
947	} else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
948	csum_partial:
949
950	udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
951	goto send;
952
953	} else
954	csum = udp_csum(skb);
955
956	/* add protocol-dependent pseudo-header */
957	uh->check = csum_tcpudp_magic(saddr: fl4->saddr, daddr: fl4->daddr, len,
958	proto: sk->sk_protocol, sum: csum);
959	if (uh->check == 0)
960	uh->check = CSUM_MANGLED_0;
961
962	send:
963	err = ip_send_skb(net: sock_net(sk), skb);
964	if (err) {
965	if (err == -ENOBUFS &&
966	!inet_test_bit(RECVERR, sk)) {
967	UDP_INC_STATS(sock_net(sk),
968	UDP_MIB_SNDBUFERRORS, is_udplite);
969	err = 0;
970	}
971	} else
972	UDP_INC_STATS(sock_net(sk),
973	UDP_MIB_OUTDATAGRAMS, is_udplite);
974	return err;
975	}
976
977	/*
978	* Push out all pending data as one UDP datagram. Socket is locked.
979	*/
980	int udp_push_pending_frames(struct sock *sk)
981	{
982	struct udp_sock *up = udp_sk(sk);
983	struct inet_sock *inet = inet_sk(sk);
984	struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
985	struct sk_buff *skb;
986	int err = 0;
987
988	skb = ip_finish_skb(sk, fl4);
989	if (!skb)
990	goto out;
991
992	err = udp_send_skb(skb, fl4, cork: &inet->cork.base);
993
994	out:
995	up->len = 0;
996	up->pending = 0;
997	return err;
998	}
999	EXPORT_SYMBOL(udp_push_pending_frames);
1000
1001	static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size)
1002	{
1003	switch (cmsg->cmsg_type) {
1004	case UDP_SEGMENT:
1005	if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16)))
1006	return -EINVAL;
1007	*gso_size = *(__u16 *)CMSG_DATA(cmsg);
1008	return 0;
1009	default:
1010	return -EINVAL;
1011	}
1012	}
1013
1014	int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size)
1015	{
1016	struct cmsghdr *cmsg;
1017	bool need_ip = false;
1018	int err;
1019
1020	for_each_cmsghdr(cmsg, msg) {
1021	if (!CMSG_OK(msg, cmsg))
1022	return -EINVAL;
1023
1024	if (cmsg->cmsg_level != SOL_UDP) {
1025	need_ip = true;
1026	continue;
1027	}
1028
1029	err = __udp_cmsg_send(cmsg, gso_size);
1030	if (err)
1031	return err;
1032	}
1033
1034	return need_ip;
1035	}
1036	EXPORT_SYMBOL_GPL(udp_cmsg_send);
1037
1038	int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
1039	{
1040	struct inet_sock *inet = inet_sk(sk);
1041	struct udp_sock *up = udp_sk(sk);
1042	DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
1043	struct flowi4 fl4_stack;
1044	struct flowi4 *fl4;
1045	int ulen = len;
1046	struct ipcm_cookie ipc;
1047	struct rtable *rt = NULL;
1048	int free = 0;
1049	int connected = 0;
1050	__be32 daddr, faddr, saddr;
1051	u8 tos, scope;
1052	__be16 dport;
1053	int err, is_udplite = IS_UDPLITE(sk);
1054	int corkreq = udp_test_bit(CORK, sk) || msg->msg_flags & MSG_MORE;
1055	int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
1056	struct sk_buff *skb;
1057	struct ip_options_data opt_copy;
1058	int uc_index;
1059
1060	if (len > 0xFFFF)
1061	return -EMSGSIZE;
1062
1063	/*
1064	* Check the flags.
1065	*/
1066
1067	if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
1068	return -EOPNOTSUPP;
1069
1070	getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
1071
1072	fl4 = &inet->cork.fl.u.ip4;
1073	if (up->pending) {
1074	/*
1075	* There are pending frames.
1076	* The socket lock must be held while it's corked.
1077	*/
1078	lock_sock(sk);
1079	if (likely(up->pending)) {
1080	if (unlikely(up->pending != AF_INET)) {
1081	release_sock(sk);
1082	return -EINVAL;
1083	}
1084	goto do_append_data;
1085	}
1086	release_sock(sk);
1087	}
1088	ulen += sizeof(struct udphdr);
1089
1090	/*
1091	* Get and verify the address.
1092	*/
1093	if (usin) {
1094	if (msg->msg_namelen < sizeof(*usin))
1095	return -EINVAL;
1096	if (usin->sin_family != AF_INET) {
1097	if (usin->sin_family != AF_UNSPEC)
1098	return -EAFNOSUPPORT;
1099	}
1100
1101	daddr = usin->sin_addr.s_addr;
1102	dport = usin->sin_port;
1103	if (dport == 0)
1104	return -EINVAL;
1105	} else {
1106	if (sk->sk_state != TCP_ESTABLISHED)
1107	return -EDESTADDRREQ;
1108	daddr = inet->inet_daddr;
1109	dport = inet->inet_dport;
1110	/* Open fast path for connected socket.
1111	Route will not be used, if at least one option is set.
1112	*/
1113	connected = 1;
1114	}
1115
1116	ipcm_init_sk(ipcm: &ipc, inet);
1117	ipc.gso_size = READ_ONCE(up->gso_size);
1118
1119	if (msg->msg_controllen) {
1120	err = udp_cmsg_send(sk, msg, &ipc.gso_size);
1121	if (err > 0)
1122	err = ip_cmsg_send(sk, msg, ipc: &ipc,
1123	allow_ipv6: sk->sk_family == AF_INET6);
1124	if (unlikely(err < 0)) {
1125	kfree(objp: ipc.opt);
1126	return err;
1127	}
1128	if (ipc.opt)
1129	free = 1;
1130	connected = 0;
1131	}
1132	if (!ipc.opt) {
1133	struct ip_options_rcu *inet_opt;
1134
1135	rcu_read_lock();
1136	inet_opt = rcu_dereference(inet->inet_opt);
1137	if (inet_opt) {
1138	memcpy(&opt_copy, inet_opt,
1139	sizeof(*inet_opt) + inet_opt->opt.optlen);
1140	ipc.opt = &opt_copy.opt;
1141	}
1142	rcu_read_unlock();
1143	}
1144
1145	if (cgroup_bpf_enabled(CGROUP_UDP4_SENDMSG) && !connected) {
1146	err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
1147	(struct sockaddr *)usin,
1148	&msg->msg_namelen,
1149	&ipc.addr);
1150	if (err)
1151	goto out_free;
1152	if (usin) {
1153	if (usin->sin_port == 0) {
1154	/* BPF program set invalid port. Reject it. */
1155	err = -EINVAL;
1156	goto out_free;
1157	}
1158	daddr = usin->sin_addr.s_addr;
1159	dport = usin->sin_port;
1160	}
1161	}
1162
1163	saddr = ipc.addr;
1164	ipc.addr = faddr = daddr;
1165
1166	if (ipc.opt && ipc.opt->opt.srr) {
1167	if (!daddr) {
1168	err = -EINVAL;
1169	goto out_free;
1170	}
1171	faddr = ipc.opt->opt.faddr;
1172	connected = 0;
1173	}
1174	tos = get_rttos(ipc: &ipc, inet);
1175	scope = ip_sendmsg_scope(inet, ipc: &ipc, msg);
1176	if (scope == RT_SCOPE_LINK)
1177	connected = 0;
1178
1179	uc_index = READ_ONCE(inet->uc_index);
1180	if (ipv4_is_multicast(addr: daddr)) {
1181	if (!ipc.oif || netif_index_is_l3_master(net: sock_net(sk), ifindex: ipc.oif))
1182	ipc.oif = READ_ONCE(inet->mc_index);
1183	if (!saddr)
1184	saddr = READ_ONCE(inet->mc_addr);
1185	connected = 0;
1186	} else if (!ipc.oif) {
1187	ipc.oif = uc_index;
1188	} else if (ipv4_is_lbcast(addr: daddr) && uc_index) {
1189	/* oif is set, packet is to local broadcast and
1190	* uc_index is set. oif is most likely set
1191	* by sk_bound_dev_if. If uc_index != oif check if the
1192	* oif is an L3 master and uc_index is an L3 slave.
1193	* If so, we want to allow the send using the uc_index.
1194	*/
1195	if (ipc.oif != uc_index &&
1196	ipc.oif == l3mdev_master_ifindex_by_index(net: sock_net(sk),
1197	ifindex: uc_index)) {
1198	ipc.oif = uc_index;
1199	}
1200	}
1201
1202	if (connected)
1203	rt = (struct rtable *)sk_dst_check(sk, cookie: 0);
1204
1205	if (!rt) {
1206	struct net *net = sock_net(sk);
1207	__u8 flow_flags = inet_sk_flowi_flags(sk);
1208
1209	fl4 = &fl4_stack;
1210
1211	flowi4_init_output(fl4, oif: ipc.oif, mark: ipc.sockc.mark, tos, scope,
1212	proto: sk->sk_protocol, flags: flow_flags, daddr: faddr, saddr,
1213	dport, sport: inet->inet_sport, uid: sk->sk_uid);
1214
1215	security_sk_classify_flow(sk, flic: flowi4_to_flowi_common(fl4));
1216	rt = ip_route_output_flow(net, flp: fl4, sk);
1217	if (IS_ERR(ptr: rt)) {
1218	err = PTR_ERR(ptr: rt);
1219	rt = NULL;
1220	if (err == -ENETUNREACH)
1221	IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
1222	goto out;
1223	}
1224
1225	err = -EACCES;
1226	if ((rt->rt_flags & RTCF_BROADCAST) &&
1227	!sock_flag(sk, flag: SOCK_BROADCAST))
1228	goto out;
1229	if (connected)
1230	sk_dst_set(sk, dst: dst_clone(dst: &rt->dst));
1231	}
1232
1233	if (msg->msg_flags&MSG_CONFIRM)
1234	goto do_confirm;
1235	back_from_confirm:
1236
1237	saddr = fl4->saddr;
1238	if (!ipc.addr)
1239	daddr = ipc.addr = fl4->daddr;
1240
1241	/* Lockless fast path for the non-corking case. */
1242	if (!corkreq) {
1243	struct inet_cork cork;
1244
1245	skb = ip_make_skb(sk, fl4, getfrag, from: msg, length: ulen,
1246	transhdrlen: sizeof(struct udphdr), ipc: &ipc, rtp: &rt,
1247	cork: &cork, flags: msg->msg_flags);
1248	err = PTR_ERR(ptr: skb);
1249	if (!IS_ERR_OR_NULL(ptr: skb))
1250	err = udp_send_skb(skb, fl4, cork: &cork);
1251	goto out;
1252	}
1253
1254	lock_sock(sk);
1255	if (unlikely(up->pending)) {
1256	/* The socket is already corked while preparing it. */
1257	/* ... which is an evident application bug. --ANK */
1258	release_sock(sk);
1259
1260	net_dbg_ratelimited("socket already corked\n");
1261	err = -EINVAL;
1262	goto out;
1263	}
1264	/*
1265	* Now cork the socket to pend data.
1266	*/
1267	fl4 = &inet->cork.fl.u.ip4;
1268	fl4->daddr = daddr;
1269	fl4->saddr = saddr;
1270	fl4->fl4_dport = dport;
1271	fl4->fl4_sport = inet->inet_sport;
1272	up->pending = AF_INET;
1273
1274	do_append_data:
1275	up->len += ulen;
1276	err = ip_append_data(sk, fl4, getfrag, from: msg, len: ulen,
1277	protolen: sizeof(struct udphdr), ipc: &ipc, rt: &rt,
1278	flags: corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
1279	if (err)
1280	udp_flush_pending_frames(sk);
1281	else if (!corkreq)
1282	err = udp_push_pending_frames(sk);
1283	else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
1284	up->pending = 0;
1285	release_sock(sk);
1286
1287	out:
1288	ip_rt_put(rt);
1289	out_free:
1290	if (free)
1291	kfree(objp: ipc.opt);
1292	if (!err)
1293	return len;
1294	/*
1295	* ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting
1296	* ENOBUFS might not be good (it's not tunable per se), but otherwise
1297	* we don't have a good statistic (IpOutDiscards but it can be too many
1298	* things). We could add another new stat but at least for now that
1299	* seems like overkill.
1300	*/
1301	if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1302	UDP_INC_STATS(sock_net(sk),
1303	UDP_MIB_SNDBUFERRORS, is_udplite);
1304	}
1305	return err;
1306
1307	do_confirm:
1308	if (msg->msg_flags & MSG_PROBE)
1309	dst_confirm_neigh(dst: &rt->dst, daddr: &fl4->daddr);
1310	if (!(msg->msg_flags&MSG_PROBE) || len)
1311	goto back_from_confirm;
1312	err = 0;
1313	goto out;
1314	}
1315	EXPORT_SYMBOL(udp_sendmsg);
1316
1317	void udp_splice_eof(struct socket *sock)
1318	{
1319	struct sock *sk = sock->sk;
1320	struct udp_sock *up = udp_sk(sk);
1321
1322	if (!up->pending || udp_test_bit(CORK, sk))
1323	return;
1324
1325	lock_sock(sk);
1326	if (up->pending && !udp_test_bit(CORK, sk))
1327	udp_push_pending_frames(sk);
1328	release_sock(sk);
1329	}
1330	EXPORT_SYMBOL_GPL(udp_splice_eof);
1331
1332	#define UDP_SKB_IS_STATELESS 0x80000000
1333
1334	/* all head states (dst, sk, nf conntrack) except skb extensions are
1335	* cleared by udp_rcv().
1336	*
1337	* We need to preserve secpath, if present, to eventually process
1338	* IP_CMSG_PASSSEC at recvmsg() time.
1339	*
1340	* Other extensions can be cleared.
1341	*/
1342	static bool udp_try_make_stateless(struct sk_buff *skb)
1343	{
1344	if (!skb_has_extensions(skb))
1345	return true;
1346
1347	if (!secpath_exists(skb)) {
1348	skb_ext_reset(skb);
1349	return true;
1350	}
1351
1352	return false;
1353	}
1354
1355	static void udp_set_dev_scratch(struct sk_buff *skb)
1356	{
1357	struct udp_dev_scratch *scratch = udp_skb_scratch(skb);
1358
1359	BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long));
1360	scratch->_tsize_state = skb->truesize;
1361	#if BITS_PER_LONG == 64
1362	scratch->len = skb->len;
1363	scratch->csum_unnecessary = !!skb_csum_unnecessary(skb);
1364	scratch->is_linear = !skb_is_nonlinear(skb);
1365	#endif
1366	if (udp_try_make_stateless(skb))
1367	scratch->_tsize_state |= UDP_SKB_IS_STATELESS;
1368	}
1369
1370	static void udp_skb_csum_unnecessary_set(struct sk_buff *skb)
1371	{
1372	/* We come here after udp_lib_checksum_complete() returned 0.
1373	* This means that __skb_checksum_complete() might have
1374	* set skb->csum_valid to 1.
1375	* On 64bit platforms, we can set csum_unnecessary
1376	* to true, but only if the skb is not shared.
1377	*/
1378	#if BITS_PER_LONG == 64
1379	if (!skb_shared(skb))
1380	udp_skb_scratch(skb)->csum_unnecessary = true;
1381	#endif
1382	}
1383
1384	static int udp_skb_truesize(struct sk_buff *skb)
1385	{
1386	return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS;
1387	}
1388
1389	static bool udp_skb_has_head_state(struct sk_buff *skb)
1390	{
1391	return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS);
1392	}
1393
1394	/* fully reclaim rmem/fwd memory allocated for skb */
1395	static void udp_rmem_release(struct sock *sk, int size, int partial,
1396	bool rx_queue_lock_held)
1397	{
1398	struct udp_sock *up = udp_sk(sk);
1399	struct sk_buff_head *sk_queue;
1400	int amt;
1401
1402	if (likely(partial)) {
1403	up->forward_deficit += size;
1404	size = up->forward_deficit;
1405	if (size < READ_ONCE(up->forward_threshold) &&
1406	!skb_queue_empty(list: &up->reader_queue))
1407	return;
1408	} else {
1409	size += up->forward_deficit;
1410	}
1411	up->forward_deficit = 0;
1412
1413	/* acquire the sk_receive_queue for fwd allocated memory scheduling,
1414	* if the called don't held it already
1415	*/
1416	sk_queue = &sk->sk_receive_queue;
1417	if (!rx_queue_lock_held)
1418	spin_lock(lock: &sk_queue->lock);
1419
1420
1421	sk_forward_alloc_add(sk, val: size);
1422	amt = (sk->sk_forward_alloc - partial) & ~(PAGE_SIZE - 1);
1423	sk_forward_alloc_add(sk, val: -amt);
1424
1425	if (amt)
1426	__sk_mem_reduce_allocated(sk, amount: amt >> PAGE_SHIFT);
1427
1428	atomic_sub(i: size, v: &sk->sk_rmem_alloc);
1429
1430	/* this can save us from acquiring the rx queue lock on next receive */
1431	skb_queue_splice_tail_init(list: sk_queue, head: &up->reader_queue);
1432
1433	if (!rx_queue_lock_held)
1434	spin_unlock(lock: &sk_queue->lock);
1435	}
1436
1437	/* Note: called with reader_queue.lock held.
1438	* Instead of using skb->truesize here, find a copy of it in skb->dev_scratch
1439	* This avoids a cache line miss while receive_queue lock is held.
1440	* Look at __udp_enqueue_schedule_skb() to find where this copy is done.
1441	*/
1442	void udp_skb_destructor(struct sock *sk, struct sk_buff *skb)
1443	{
1444	prefetch(&skb->data);
1445	udp_rmem_release(sk, size: udp_skb_truesize(skb), partial: 1, rx_queue_lock_held: false);
1446	}
1447	EXPORT_SYMBOL(udp_skb_destructor);
1448
1449	/* as above, but the caller held the rx queue lock, too */
1450	static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb)
1451	{
1452	prefetch(&skb->data);
1453	udp_rmem_release(sk, size: udp_skb_truesize(skb), partial: 1, rx_queue_lock_held: true);
1454	}
1455
1456	/* Idea of busylocks is to let producers grab an extra spinlock
1457	* to relieve pressure on the receive_queue spinlock shared by consumer.
1458	* Under flood, this means that only one producer can be in line
1459	* trying to acquire the receive_queue spinlock.
1460	* These busylock can be allocated on a per cpu manner, instead of a
1461	* per socket one (that would consume a cache line per socket)
1462	*/
1463	static int udp_busylocks_log __read_mostly;
1464	static spinlock_t *udp_busylocks __read_mostly;
1465
1466	static spinlock_t *busylock_acquire(void *ptr)
1467	{
1468	spinlock_t *busy;
1469
1470	busy = udp_busylocks + hash_ptr(ptr, bits: udp_busylocks_log);
1471	spin_lock(lock: busy);
1472	return busy;
1473	}
1474
1475	static void busylock_release(spinlock_t *busy)
1476	{
1477	if (busy)
1478	spin_unlock(lock: busy);
1479	}
1480
1481	static int udp_rmem_schedule(struct sock *sk, int size)
1482	{
1483	int delta;
1484
1485	delta = size - sk->sk_forward_alloc;
1486	if (delta > 0 && !__sk_mem_schedule(sk, size: delta, SK_MEM_RECV))
1487	return -ENOBUFS;
1488
1489	return 0;
1490	}
1491
1492	int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
1493	{
1494	struct sk_buff_head *list = &sk->sk_receive_queue;
1495	int rmem, err = -ENOMEM;
1496	spinlock_t *busy = NULL;
1497	int size;
1498
1499	/* try to avoid the costly atomic add/sub pair when the receive
1500	* queue is full; always allow at least a packet
1501	*/
1502	rmem = atomic_read(v: &sk->sk_rmem_alloc);
1503	if (rmem > sk->sk_rcvbuf)
1504	goto drop;
1505
1506	/* Under mem pressure, it might be helpful to help udp_recvmsg()
1507	* having linear skbs :
1508	* - Reduce memory overhead and thus increase receive queue capacity
1509	* - Less cache line misses at copyout() time
1510	* - Less work at consume_skb() (less alien page frag freeing)
1511	*/
1512	if (rmem > (sk->sk_rcvbuf >> 1)) {
1513	skb_condense(skb);
1514
1515	busy = busylock_acquire(ptr: sk);
1516	}
1517	size = skb->truesize;
1518	udp_set_dev_scratch(skb);
1519
1520	/* we drop only if the receive buf is full and the receive
1521	* queue contains some other skb
1522	*/
1523	rmem = atomic_add_return(i: size, v: &sk->sk_rmem_alloc);
1524	if (rmem > (size + (unsigned int)sk->sk_rcvbuf))
1525	goto uncharge_drop;
1526
1527	spin_lock(lock: &list->lock);
1528	err = udp_rmem_schedule(sk, size);
1529	if (err) {
1530	spin_unlock(lock: &list->lock);
1531	goto uncharge_drop;
1532	}
1533
1534	sk_forward_alloc_add(sk, val: -size);
1535
1536	/* no need to setup a destructor, we will explicitly release the
1537	* forward allocated memory on dequeue
1538	*/
1539	sock_skb_set_dropcount(sk, skb);
1540
1541	__skb_queue_tail(list, newsk: skb);
1542	spin_unlock(lock: &list->lock);
1543
1544	if (!sock_flag(sk, flag: SOCK_DEAD))
1545	INDIRECT_CALL_1(sk->sk_data_ready, sock_def_readable, sk);
1546
1547	busylock_release(busy);
1548	return 0;
1549
1550	uncharge_drop:
1551	atomic_sub(i: skb->truesize, v: &sk->sk_rmem_alloc);
1552
1553	drop:
1554	atomic_inc(v: &sk->sk_drops);
1555	busylock_release(busy);
1556	return err;
1557	}
1558	EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb);
1559
1560	void udp_destruct_common(struct sock *sk)
1561	{
1562	/* reclaim completely the forward allocated memory */
1563	struct udp_sock *up = udp_sk(sk);
1564	unsigned int total = 0;
1565	struct sk_buff *skb;
1566
1567	skb_queue_splice_tail_init(list: &sk->sk_receive_queue, head: &up->reader_queue);
1568	while ((skb = __skb_dequeue(list: &up->reader_queue)) != NULL) {
1569	total += skb->truesize;
1570	kfree_skb(skb);
1571	}
1572	udp_rmem_release(sk, size: total, partial: 0, rx_queue_lock_held: true);
1573	}
1574	EXPORT_SYMBOL_GPL(udp_destruct_common);
1575
1576	static void udp_destruct_sock(struct sock *sk)
1577	{
1578	udp_destruct_common(sk);
1579	inet_sock_destruct(sk);
1580	}
1581
1582	int udp_init_sock(struct sock *sk)
1583	{
1584	udp_lib_init_sock(sk);
1585	sk->sk_destruct = udp_destruct_sock;
1586	set_bit(SOCK_SUPPORT_ZC, addr: &sk->sk_socket->flags);
1587	return 0;
1588	}
1589
1590	void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
1591	{
1592	if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) {
1593	bool slow = lock_sock_fast(sk);
1594
1595	sk_peek_offset_bwd(sk, val: len);
1596	unlock_sock_fast(sk, slow);
1597	}
1598
1599	if (!skb_unref(skb))
1600	return;
1601
1602	/* In the more common cases we cleared the head states previously,
1603	* see __udp_queue_rcv_skb().
1604	*/
1605	if (unlikely(udp_skb_has_head_state(skb)))
1606	skb_release_head_state(skb);
1607	__consume_stateless_skb(skb);
1608	}
1609	EXPORT_SYMBOL_GPL(skb_consume_udp);
1610
1611	static struct sk_buff *__first_packet_length(struct sock *sk,
1612	struct sk_buff_head *rcvq,
1613	int *total)
1614	{
1615	struct sk_buff *skb;
1616
1617	while ((skb = skb_peek(list_: rcvq)) != NULL) {
1618	if (udp_lib_checksum_complete(skb)) {
1619	__UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS,
1620	IS_UDPLITE(sk));
1621	__UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS,
1622	IS_UDPLITE(sk));
1623	atomic_inc(v: &sk->sk_drops);
1624	__skb_unlink(skb, list: rcvq);
1625	*total += skb->truesize;
1626	kfree_skb(skb);
1627	} else {
1628	udp_skb_csum_unnecessary_set(skb);
1629	break;
1630	}
1631	}
1632	return skb;
1633	}
1634
1635	/**
1636	* first_packet_length - return length of first packet in receive queue
1637	* @sk: socket
1638	*
1639	* Drops all bad checksum frames, until a valid one is found.
1640	* Returns the length of found skb, or -1 if none is found.
1641	*/
1642	static int first_packet_length(struct sock *sk)
1643	{
1644	struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue;
1645	struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
1646	struct sk_buff *skb;
1647	int total = 0;
1648	int res;
1649
1650	spin_lock_bh(lock: &rcvq->lock);
1651	skb = __first_packet_length(sk, rcvq, total: &total);
1652	if (!skb && !skb_queue_empty_lockless(list: sk_queue)) {
1653	spin_lock(lock: &sk_queue->lock);
1654	skb_queue_splice_tail_init(list: sk_queue, head: rcvq);
1655	spin_unlock(lock: &sk_queue->lock);
1656
1657	skb = __first_packet_length(sk, rcvq, total: &total);
1658	}
1659	res = skb ? skb->len : -1;
1660	if (total)
1661	udp_rmem_release(sk, size: total, partial: 1, rx_queue_lock_held: false);
1662	spin_unlock_bh(lock: &rcvq->lock);
1663	return res;
1664	}
1665
1666	/*
1667	* IOCTL requests applicable to the UDP protocol
1668	*/
1669
1670	int udp_ioctl(struct sock *sk, int cmd, int *karg)
1671	{
1672	switch (cmd) {
1673	case SIOCOUTQ:
1674	{
1675	*karg = sk_wmem_alloc_get(sk);
1676	return 0;
1677	}
1678
1679	case SIOCINQ:
1680	{
1681	*karg = max_t(int, 0, first_packet_length(sk));
1682	return 0;
1683	}
1684
1685	default:
1686	return -ENOIOCTLCMD;
1687	}
1688
1689	return 0;
1690	}
1691	EXPORT_SYMBOL(udp_ioctl);
1692
1693	struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags,
1694	int *off, int *err)
1695	{
1696	struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
1697	struct sk_buff_head *queue;
1698	struct sk_buff *last;
1699	long timeo;
1700	int error;
1701
1702	queue = &udp_sk(sk)->reader_queue;
1703	timeo = sock_rcvtimeo(sk, noblock: flags & MSG_DONTWAIT);
1704	do {
1705	struct sk_buff *skb;
1706
1707	error = sock_error(sk);
1708	if (error)
1709	break;
1710
1711	error = -EAGAIN;
1712	do {
1713	spin_lock_bh(lock: &queue->lock);
1714	skb = __skb_try_recv_from_queue(sk, queue, flags, off,
1715	err, last: &last);
1716	if (skb) {
1717	if (!(flags & MSG_PEEK))
1718	udp_skb_destructor(sk, skb);
1719	spin_unlock_bh(lock: &queue->lock);
1720	return skb;
1721	}
1722
1723	if (skb_queue_empty_lockless(list: sk_queue)) {
1724	spin_unlock_bh(lock: &queue->lock);
1725	goto busy_check;
1726	}
1727
1728	/* refill the reader queue and walk it again
1729	* keep both queues locked to avoid re-acquiring
1730	* the sk_receive_queue lock if fwd memory scheduling
1731	* is needed.
1732	*/
1733	spin_lock(lock: &sk_queue->lock);
1734	skb_queue_splice_tail_init(list: sk_queue, head: queue);
1735
1736	skb = __skb_try_recv_from_queue(sk, queue, flags, off,
1737	err, last: &last);
1738	if (skb && !(flags & MSG_PEEK))
1739	udp_skb_dtor_locked(sk, skb);
1740	spin_unlock(lock: &sk_queue->lock);
1741	spin_unlock_bh(lock: &queue->lock);
1742	if (skb)
1743	return skb;
1744
1745	busy_check:
1746	if (!sk_can_busy_loop(sk))
1747	break;
1748
1749	sk_busy_loop(sk, nonblock: flags & MSG_DONTWAIT);
1750	} while (!skb_queue_empty_lockless(list: sk_queue));
1751
1752	/* sk_queue is empty, reader_queue may contain peeked packets */
1753	} while (timeo &&
1754	!__skb_wait_for_more_packets(sk, queue: &sk->sk_receive_queue,
1755	err: &error, timeo_p: &timeo,
1756	skb: (struct sk_buff *)sk_queue));
1757
1758	*err = error;
1759	return NULL;
1760	}
1761	EXPORT_SYMBOL(__skb_recv_udp);
1762
1763	int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
1764	{
1765	struct sk_buff *skb;
1766	int err;
1767
1768	try_again:
1769	skb = skb_recv_udp(sk, MSG_DONTWAIT, err: &err);
1770	if (!skb)
1771	return err;
1772
1773	if (udp_lib_checksum_complete(skb)) {
1774	int is_udplite = IS_UDPLITE(sk);
1775	struct net *net = sock_net(sk);
1776
1777	__UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, is_udplite);
1778	__UDP_INC_STATS(net, UDP_MIB_INERRORS, is_udplite);
1779	atomic_inc(v: &sk->sk_drops);
1780	kfree_skb(skb);
1781	goto try_again;
1782	}
1783
1784	WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
1785	return recv_actor(sk, skb);
1786	}
1787	EXPORT_SYMBOL(udp_read_skb);
1788
1789	/*
1790	* This should be easy, if there is something there we
1791	* return it, otherwise we block.
1792	*/
1793
1794	int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags,
1795	int *addr_len)
1796	{
1797	struct inet_sock *inet = inet_sk(sk);
1798	DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
1799	struct sk_buff *skb;
1800	unsigned int ulen, copied;
1801	int off, err, peeking = flags & MSG_PEEK;
1802	int is_udplite = IS_UDPLITE(sk);
1803	bool checksum_valid = false;
1804
1805	if (flags & MSG_ERRQUEUE)
1806	return ip_recv_error(sk, msg, len, addr_len);
1807
1808	try_again:
1809	off = sk_peek_offset(sk, flags);
1810	skb = __skb_recv_udp(sk, flags, &off, &err);
1811	if (!skb)
1812	return err;
1813
1814	ulen = udp_skb_len(skb);
1815	copied = len;
1816	if (copied > ulen - off)
1817	copied = ulen - off;
1818	else if (copied < ulen)
1819	msg->msg_flags |= MSG_TRUNC;
1820
1821	/*
1822	* If checksum is needed at all, try to do it while copying the
1823	* data. If the data is truncated, or if we only want a partial
1824	* coverage checksum (UDP-Lite), do it before the copy.
1825	*/
1826
1827	if (copied < ulen || peeking ||
1828	(is_udplite && UDP_SKB_CB(skb)->partial_cov)) {
1829	checksum_valid = udp_skb_csum_unnecessary(skb) ||
1830	!__udp_lib_checksum_complete(skb);
1831	if (!checksum_valid)
1832	goto csum_copy_err;
1833	}
1834
1835	if (checksum_valid || udp_skb_csum_unnecessary(skb)) {
1836	if (udp_skb_is_linear(skb))
1837	err = copy_linear_skb(skb, len: copied, off, to: &msg->msg_iter);
1838	else
1839	err = skb_copy_datagram_msg(from: skb, offset: off, msg, size: copied);
1840	} else {
1841	err = skb_copy_and_csum_datagram_msg(skb, hlen: off, msg);
1842
1843	if (err == -EINVAL)
1844	goto csum_copy_err;
1845	}
1846
1847	if (unlikely(err)) {
1848	if (!peeking) {
1849	atomic_inc(v: &sk->sk_drops);
1850	UDP_INC_STATS(sock_net(sk),
1851	UDP_MIB_INERRORS, is_udplite);
1852	}
1853	kfree_skb(skb);
1854	return err;
1855	}
1856
1857	if (!peeking)
1858	UDP_INC_STATS(sock_net(sk),
1859	UDP_MIB_INDATAGRAMS, is_udplite);
1860
1861	sock_recv_cmsgs(msg, sk, skb);
1862
1863	/* Copy the address. */
1864	if (sin) {
1865	sin->sin_family = AF_INET;
1866	sin->sin_port = udp_hdr(skb)->source;
1867	sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
1868	memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
1869	*addr_len = sizeof(*sin);
1870
1871	BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,
1872	(struct sockaddr *)sin,
1873	addr_len);
1874	}
1875
1876	if (udp_test_bit(GRO_ENABLED, sk))
1877	udp_cmsg_recv(msg, sk, skb);
1878
1879	if (inet_cmsg_flags(inet))
1880	ip_cmsg_recv_offset(msg, sk, skb, tlen: sizeof(struct udphdr), offset: off);
1881
1882	err = copied;
1883	if (flags & MSG_TRUNC)
1884	err = ulen;
1885
1886	skb_consume_udp(sk, skb, peeking ? -err : err);
1887	return err;
1888
1889	csum_copy_err:
1890	if (!__sk_queue_drop_skb(sk, sk_queue: &udp_sk(sk)->reader_queue, skb, flags,
1891	destructor: udp_skb_destructor)) {
1892	UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
1893	UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
1894	}
1895	kfree_skb(skb);
1896
1897	/* starting over for a new packet, but check if we need to yield */
1898	cond_resched();
1899	msg->msg_flags &= ~MSG_TRUNC;
1900	goto try_again;
1901	}
1902
1903	int udp_pre_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
1904	{
1905	/* This check is replicated from __ip4_datagram_connect() and
1906	* intended to prevent BPF program called below from accessing bytes
1907	* that are out of the bound specified by user in addr_len.
1908	*/
1909	if (addr_len < sizeof(struct sockaddr_in))
1910	return -EINVAL;
1911
1912	return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr, &addr_len);
1913	}
1914	EXPORT_SYMBOL(udp_pre_connect);
1915
1916	int __udp_disconnect(struct sock *sk, int flags)
1917	{
1918	struct inet_sock *inet = inet_sk(sk);
1919	/*
1920	* 1003.1g - break association.
1921	*/
1922
1923	sk->sk_state = TCP_CLOSE;
1924	inet->inet_daddr = 0;
1925	inet->inet_dport = 0;
1926	sock_rps_reset_rxhash(sk);
1927	sk->sk_bound_dev_if = 0;
1928	if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) {
1929	inet_reset_saddr(sk);
1930	if (sk->sk_prot->rehash &&
1931	(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
1932	sk->sk_prot->rehash(sk);
1933	}
1934
1935	if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
1936	sk->sk_prot->unhash(sk);
1937	inet->inet_sport = 0;
1938	}
1939	sk_dst_reset(sk);
1940	return 0;
1941	}
1942	EXPORT_SYMBOL(__udp_disconnect);
1943
1944	int udp_disconnect(struct sock *sk, int flags)
1945	{
1946	lock_sock(sk);
1947	__udp_disconnect(sk, flags);
1948	release_sock(sk);
1949	return 0;
1950	}
1951	EXPORT_SYMBOL(udp_disconnect);
1952
1953	void udp_lib_unhash(struct sock *sk)
1954	{
1955	if (sk_hashed(sk)) {
1956	struct udp_table *udptable = udp_get_table_prot(sk);
1957	struct udp_hslot *hslot, *hslot2;
1958
1959	hslot = udp_hashslot(table: udptable, net: sock_net(sk),
1960	udp_sk(sk)->udp_port_hash);
1961	hslot2 = udp_hashslot2(table: udptable, udp_sk(sk)->udp_portaddr_hash);
1962
1963	spin_lock_bh(lock: &hslot->lock);
1964	if (rcu_access_pointer(sk->sk_reuseport_cb))
1965	reuseport_detach_sock(sk);
1966	if (sk_del_node_init_rcu(sk)) {
1967	hslot->count--;
1968	inet_sk(sk)->inet_num = 0;
1969	sock_prot_inuse_add(net: sock_net(sk), prot: sk->sk_prot, val: -1);
1970
1971	spin_lock(lock: &hslot2->lock);
1972	hlist_del_init_rcu(n: &udp_sk(sk)->udp_portaddr_node);
1973	hslot2->count--;
1974	spin_unlock(lock: &hslot2->lock);
1975	}
1976	spin_unlock_bh(lock: &hslot->lock);
1977	}
1978	}
1979	EXPORT_SYMBOL(udp_lib_unhash);
1980
1981	/*
1982	* inet_rcv_saddr was changed, we must rehash secondary hash
1983	*/
1984	void udp_lib_rehash(struct sock *sk, u16 newhash)
1985	{
1986	if (sk_hashed(sk)) {
1987	struct udp_table *udptable = udp_get_table_prot(sk);
1988	struct udp_hslot *hslot, *hslot2, *nhslot2;
1989
1990	hslot2 = udp_hashslot2(table: udptable, udp_sk(sk)->udp_portaddr_hash);
1991	nhslot2 = udp_hashslot2(table: udptable, hash: newhash);
1992	udp_sk(sk)->udp_portaddr_hash = newhash;
1993
1994	if (hslot2 != nhslot2 ||
1995	rcu_access_pointer(sk->sk_reuseport_cb)) {
1996	hslot = udp_hashslot(table: udptable, net: sock_net(sk),
1997	udp_sk(sk)->udp_port_hash);
1998	/* we must lock primary chain too */
1999	spin_lock_bh(lock: &hslot->lock);
2000	if (rcu_access_pointer(sk->sk_reuseport_cb))
2001	reuseport_detach_sock(sk);
2002
2003	if (hslot2 != nhslot2) {
2004	spin_lock(lock: &hslot2->lock);
2005	hlist_del_init_rcu(n: &udp_sk(sk)->udp_portaddr_node);
2006	hslot2->count--;
2007	spin_unlock(lock: &hslot2->lock);
2008
2009	spin_lock(lock: &nhslot2->lock);
2010	hlist_add_head_rcu(n: &udp_sk(sk)->udp_portaddr_node,
2011	h: &nhslot2->head);
2012	nhslot2->count++;
2013	spin_unlock(lock: &nhslot2->lock);
2014	}
2015
2016	spin_unlock_bh(lock: &hslot->lock);
2017	}
2018	}
2019	}
2020	EXPORT_SYMBOL(udp_lib_rehash);
2021
2022	void udp_v4_rehash(struct sock *sk)
2023	{
2024	u16 new_hash = ipv4_portaddr_hash(net: sock_net(sk),
2025	inet_sk(sk)->inet_rcv_saddr,
2026	inet_sk(sk)->inet_num);
2027	udp_lib_rehash(sk, new_hash);
2028	}
2029
2030	static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
2031	{
2032	int rc;
2033
2034	if (inet_sk(sk)->inet_daddr) {
2035	sock_rps_save_rxhash(sk, skb);
2036	sk_mark_napi_id(sk, skb);
2037	sk_incoming_cpu_update(sk);
2038	} else {
2039	sk_mark_napi_id_once(sk, skb);
2040	}
2041
2042	rc = __udp_enqueue_schedule_skb(sk, skb);
2043	if (rc < 0) {
2044	int is_udplite = IS_UDPLITE(sk);
2045	int drop_reason;
2046
2047	/* Note that an ENOMEM error is charged twice */
2048	if (rc == -ENOMEM) {
2049	UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS,
2050	is_udplite);
2051	drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF;
2052	} else {
2053	UDP_INC_STATS(sock_net(sk), UDP_MIB_MEMERRORS,
2054	is_udplite);
2055	drop_reason = SKB_DROP_REASON_PROTO_MEM;
2056	}
2057	UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
2058	kfree_skb_reason(skb, reason: drop_reason);
2059	trace_udp_fail_queue_rcv_skb(rc, sk);
2060	return -1;
2061	}
2062
2063	return 0;
2064	}
2065
2066	/* returns:
2067	* -1: error
2068	* 0: success
2069	* >0: "udp encap" protocol resubmission
2070	*
2071	* Note that in the success and error cases, the skb is assumed to
2072	* have either been requeued or freed.
2073	*/
2074	static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
2075	{
2076	int drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
2077	struct udp_sock *up = udp_sk(sk);
2078	int is_udplite = IS_UDPLITE(sk);
2079
2080	/*
2081	* Charge it to the socket, dropping if the queue is full.
2082	*/
2083	if (!xfrm4_policy_check(sk, dir: XFRM_POLICY_IN, skb)) {
2084	drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2085	goto drop;
2086	}
2087	nf_reset_ct(skb);
2088
2089	if (static_branch_unlikely(&udp_encap_needed_key) &&
2090	READ_ONCE(up->encap_type)) {
2091	int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
2092
2093	/*
2094	* This is an encapsulation socket so pass the skb to
2095	* the socket's udp_encap_rcv() hook. Otherwise, just
2096	* fall through and pass this up the UDP socket.
2097	* up->encap_rcv() returns the following value:
2098	* =0 if skb was successfully passed to the encap
2099	* handler or was discarded by it.
2100	* >0 if skb should be passed on to UDP.
2101	* <0 if skb should be resubmitted as proto -N
2102	*/
2103
2104	/* if we're overly short, let UDP handle it */
2105	encap_rcv = READ_ONCE(up->encap_rcv);
2106	if (encap_rcv) {
2107	int ret;
2108
2109	/* Verify checksum before giving to encap */
2110	if (udp_lib_checksum_complete(skb))
2111	goto csum_error;
2112
2113	ret = encap_rcv(sk, skb);
2114	if (ret <= 0) {
2115	__UDP_INC_STATS(sock_net(sk),
2116	UDP_MIB_INDATAGRAMS,
2117	is_udplite);
2118	return -ret;
2119	}
2120	}
2121
2122	/* FALLTHROUGH -- it's a UDP Packet */
2123	}
2124
2125	/*
2126	* UDP-Lite specific tests, ignored on UDP sockets
2127	*/
2128	if (udp_test_bit(UDPLITE_RECV_CC, sk) && UDP_SKB_CB(skb)->partial_cov) {
2129	u16 pcrlen = READ_ONCE(up->pcrlen);
2130
2131	/*
2132	* MIB statistics other than incrementing the error count are
2133	* disabled for the following two types of errors: these depend
2134	* on the application settings, not on the functioning of the
2135	* protocol stack as such.
2136	*
2137	* RFC 3828 here recommends (sec 3.3): "There should also be a
2138	* way ... to ... at least let the receiving application block
2139	* delivery of packets with coverage values less than a value
2140	* provided by the application."
2141	*/
2142	if (pcrlen == 0) { /* full coverage was set */
2143	net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested\n",
2144	UDP_SKB_CB(skb)->cscov, skb->len);
2145	goto drop;
2146	}
2147	/* The next case involves violating the min. coverage requested
2148	* by the receiver. This is subtle: if receiver wants x and x is
2149	* greater than the buffersize/MTU then receiver will complain
2150	* that it wants x while sender emits packets of smaller size y.
2151	* Therefore the above ...()->partial_cov statement is essential.
2152	*/
2153	if (UDP_SKB_CB(skb)->cscov < pcrlen) {
2154	net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d\n",
2155	UDP_SKB_CB(skb)->cscov, pcrlen);
2156	goto drop;
2157	}
2158	}
2159
2160	prefetch(&sk->sk_rmem_alloc);
2161	if (rcu_access_pointer(sk->sk_filter) &&
2162	udp_lib_checksum_complete(skb))
2163	goto csum_error;
2164
2165	if (sk_filter_trim_cap(sk, skb, cap: sizeof(struct udphdr))) {
2166	drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
2167	goto drop;
2168	}
2169
2170	udp_csum_pull_header(skb);
2171
2172	ipv4_pktinfo_prepare(sk, skb);
2173	return __udp_queue_rcv_skb(sk, skb);
2174
2175	csum_error:
2176	drop_reason = SKB_DROP_REASON_UDP_CSUM;
2177	__UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
2178	drop:
2179	__UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
2180	atomic_inc(v: &sk->sk_drops);
2181	kfree_skb_reason(skb, reason: drop_reason);
2182	return -1;
2183	}
2184
2185	static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
2186	{
2187	struct sk_buff *next, *segs;
2188	int ret;
2189
2190	if (likely(!udp_unexpected_gso(sk, skb)))
2191	return udp_queue_rcv_one_skb(sk, skb);
2192
2193	BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_GSO_CB_OFFSET);
2194	__skb_push(skb, len: -skb_mac_offset(skb));
2195	segs = udp_rcv_segment(sk, skb, ipv4: true);
2196	skb_list_walk_safe(segs, skb, next) {
2197	__skb_pull(skb, len: skb_transport_offset(skb));
2198
2199	udp_post_segment_fix_csum(skb);
2200	ret = udp_queue_rcv_one_skb(sk, skb);
2201	if (ret > 0)
2202	ip_protocol_deliver_rcu(net: dev_net(dev: skb->dev), skb, proto: ret);
2203	}
2204	return 0;
2205	}
2206
2207	/* For TCP sockets, sk_rx_dst is protected by socket lock
2208	* For UDP, we use xchg() to guard against concurrent changes.
2209	*/
2210	bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
2211	{
2212	struct dst_entry *old;
2213
2214	if (dst_hold_safe(dst)) {
2215	old = xchg((__force struct dst_entry **)&sk->sk_rx_dst, dst);
2216	dst_release(dst: old);
2217	return old != dst;
2218	}
2219	return false;
2220	}
2221	EXPORT_SYMBOL(udp_sk_rx_dst_set);
2222
2223	/*
2224	* Multicasts and broadcasts go to each listener.
2225	*
2226	* Note: called only from the BH handler context.
2227	*/
2228	static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
2229	struct udphdr *uh,
2230	__be32 saddr, __be32 daddr,
2231	struct udp_table *udptable,
2232	int proto)
2233	{
2234	struct sock *sk, *first = NULL;
2235	unsigned short hnum = ntohs(uh->dest);
2236	struct udp_hslot *hslot = udp_hashslot(table: udptable, net, num: hnum);
2237	unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
2238	unsigned int offset = offsetof(typeof(*sk), sk_node);
2239	int dif = skb->dev->ifindex;
2240	int sdif = inet_sdif(skb);
2241	struct hlist_node *node;
2242	struct sk_buff *nskb;
2243
2244	if (use_hash2) {
2245	hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), port: hnum) &
2246	udptable->mask;
2247	hash2 = ipv4_portaddr_hash(net, saddr: daddr, port: hnum) & udptable->mask;
2248	start_lookup:
2249	hslot = &udptable->hash2[hash2];
2250	offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
2251	}
2252
2253	sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
2254	if (!__udp_is_mcast_sock(net, sk, loc_port: uh->dest, loc_addr: daddr,
2255	rmt_port: uh->source, rmt_addr: saddr, dif, sdif, hnum))
2256	continue;
2257
2258	if (!first) {
2259	first = sk;
2260	continue;
2261	}
2262	nskb = skb_clone(skb, GFP_ATOMIC);
2263
2264	if (unlikely(!nskb)) {
2265	atomic_inc(v: &sk->sk_drops);
2266	__UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS,
2267	IS_UDPLITE(sk));
2268	__UDP_INC_STATS(net, UDP_MIB_INERRORS,
2269	IS_UDPLITE(sk));
2270	continue;
2271	}
2272	if (udp_queue_rcv_skb(sk, skb: nskb) > 0)
2273	consume_skb(skb: nskb);
2274	}
2275
2276	/* Also lookup *:port if we are using hash2 and haven't done so yet. */
2277	if (use_hash2 && hash2 != hash2_any) {
2278	hash2 = hash2_any;
2279	goto start_lookup;
2280	}
2281
2282	if (first) {
2283	if (udp_queue_rcv_skb(sk: first, skb) > 0)
2284	consume_skb(skb);
2285	} else {
2286	kfree_skb(skb);
2287	__UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI,
2288	proto == IPPROTO_UDPLITE);
2289	}
2290	return 0;
2291	}
2292
2293	/* Initialize UDP checksum. If exited with zero value (success),
2294	* CHECKSUM_UNNECESSARY means, that no more checks are required.
2295	* Otherwise, csum completion requires checksumming packet body,
2296	* including udp header and folding it to skb->csum.
2297	*/
2298	static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh,
2299	int proto)
2300	{
2301	int err;
2302
2303	UDP_SKB_CB(skb)->partial_cov = 0;
2304	UDP_SKB_CB(skb)->cscov = skb->len;
2305
2306	if (proto == IPPROTO_UDPLITE) {
2307	err = udplite_checksum_init(skb, uh);
2308	if (err)
2309	return err;
2310
2311	if (UDP_SKB_CB(skb)->partial_cov) {
2312	skb->csum = inet_compute_pseudo(skb, proto);
2313	return 0;
2314	}
2315	}
2316
2317	/* Note, we are only interested in != 0 or == 0, thus the
2318	* force to int.
2319	*/
2320	err = (__force int)skb_checksum_init_zero_check(skb, proto, uh->check,
2321	inet_compute_pseudo);
2322	if (err)
2323	return err;
2324
2325	if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) {
2326	/* If SW calculated the value, we know it's bad */
2327	if (skb->csum_complete_sw)
2328	return 1;
2329
2330	/* HW says the value is bad. Let's validate that.
2331	* skb->csum is no longer the full packet checksum,
2332	* so don't treat it as such.
2333	*/
2334	skb_checksum_complete_unset(skb);
2335	}
2336
2337	return 0;
2338	}
2339
2340	/* wrapper for udp_queue_rcv_skb tacking care of csum conversion and
2341	* return code conversion for ip layer consumption
2342	*/
2343	static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb,
2344	struct udphdr *uh)
2345	{
2346	int ret;
2347
2348	if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))
2349	skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo);
2350
2351	ret = udp_queue_rcv_skb(sk, skb);
2352
2353	/* a return value > 0 means to resubmit the input, but
2354	* it wants the return to be -protocol, or 0
2355	*/
2356	if (ret > 0)
2357	return -ret;
2358	return 0;
2359	}
2360
2361	/*
2362	* All we need to do is get the socket, and then do a checksum.
2363	*/
2364
2365	int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
2366	int proto)
2367	{
2368	struct sock *sk;
2369	struct udphdr *uh;
2370	unsigned short ulen;
2371	struct rtable *rt = skb_rtable(skb);
2372	__be32 saddr, daddr;
2373	struct net *net = dev_net(dev: skb->dev);
2374	bool refcounted;
2375	int drop_reason;
2376
2377	drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
2378
2379	/*
2380	* Validate the packet.
2381	*/
2382	if (!pskb_may_pull(skb, len: sizeof(struct udphdr)))
2383	goto drop; /* No space for header. */
2384
2385	uh = udp_hdr(skb);
2386	ulen = ntohs(uh->len);
2387	saddr = ip_hdr(skb)->saddr;
2388	daddr = ip_hdr(skb)->daddr;
2389
2390	if (ulen > skb->len)
2391	goto short_packet;
2392
2393	if (proto == IPPROTO_UDP) {
2394	/* UDP validates ulen. */
2395	if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, len: ulen))
2396	goto short_packet;
2397	uh = udp_hdr(skb);
2398	}
2399
2400	if (udp4_csum_init(skb, uh, proto))
2401	goto csum_error;
2402
2403	sk = inet_steal_sock(net, skb, doff: sizeof(struct udphdr), saddr, sport: uh->source, daddr, dport: uh->dest,
2404	refcounted: &refcounted, ehashfn: udp_ehashfn);
2405	if (IS_ERR(ptr: sk))
2406	goto no_sk;
2407
2408	if (sk) {
2409	struct dst_entry *dst = skb_dst(skb);
2410	int ret;
2411
2412	if (unlikely(rcu_dereference(sk->sk_rx_dst) != dst))
2413	udp_sk_rx_dst_set(sk, dst);
2414
2415	ret = udp_unicast_rcv_skb(sk, skb, uh);
2416	if (refcounted)
2417	sock_put(sk);
2418	return ret;
2419	}
2420
2421	if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST))
2422	return __udp4_lib_mcast_deliver(net, skb, uh,
2423	saddr, daddr, udptable, proto);
2424
2425	sk = __udp4_lib_lookup_skb(skb, sport: uh->source, dport: uh->dest, udptable);
2426	if (sk)
2427	return udp_unicast_rcv_skb(sk, skb, uh);
2428	no_sk:
2429	if (!xfrm4_policy_check(NULL, dir: XFRM_POLICY_IN, skb))
2430	goto drop;
2431	nf_reset_ct(skb);
2432
2433	/* No socket. Drop packet silently, if checksum is wrong */
2434	if (udp_lib_checksum_complete(skb))
2435	goto csum_error;
2436
2437	drop_reason = SKB_DROP_REASON_NO_SOCKET;
2438	__UDP_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
2439	icmp_send(skb_in: skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, info: 0);
2440
2441	/*
2442	* Hmm. We got an UDP packet to a port to which we
2443	* don't wanna listen. Ignore it.
2444	*/
2445	kfree_skb_reason(skb, reason: drop_reason);
2446	return 0;
2447
2448	short_packet:
2449	drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
2450	net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
2451	proto == IPPROTO_UDPLITE ? "Lite" : "",
2452	&saddr, ntohs(uh->source),
2453	ulen, skb->len,
2454	&daddr, ntohs(uh->dest));
2455	goto drop;
2456
2457	csum_error:
2458	/*
2459	* RFC1122: OK. Discards the bad packet silently (as far as
2460	* the network is concerned, anyway) as per 4.1.3.4 (MUST).
2461	*/
2462	drop_reason = SKB_DROP_REASON_UDP_CSUM;
2463	net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
2464	proto == IPPROTO_UDPLITE ? "Lite" : "",
2465	&saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
2466	ulen);
2467	__UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
2468	drop:
2469	__UDP_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
2470	kfree_skb_reason(skb, reason: drop_reason);
2471	return 0;
2472	}
2473
2474	/* We can only early demux multicast if there is a single matching socket.
2475	* If more than one socket found returns NULL
2476	*/
2477	static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
2478	__be16 loc_port, __be32 loc_addr,
2479	__be16 rmt_port, __be32 rmt_addr,
2480	int dif, int sdif)
2481	{
2482	struct udp_table *udptable = net->ipv4.udp_table;
2483	unsigned short hnum = ntohs(loc_port);
2484	struct sock *sk, *result;
2485	struct udp_hslot *hslot;
2486	unsigned int slot;
2487
2488	slot = udp_hashfn(net, num: hnum, mask: udptable->mask);
2489	hslot = &udptable->hash[slot];
2490
2491	/* Do not bother scanning a too big list */
2492	if (hslot->count > 10)
2493	return NULL;
2494
2495	result = NULL;
2496	sk_for_each_rcu(sk, &hslot->head) {
2497	if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr,
2498	rmt_port, rmt_addr, dif, sdif, hnum)) {
2499	if (result)
2500	return NULL;
2501	result = sk;
2502	}
2503	}
2504
2505	return result;
2506	}
2507
2508	/* For unicast we should only early demux connected sockets or we can
2509	* break forwarding setups. The chains here can be long so only check
2510	* if the first socket is an exact match and if not move on.
2511	*/
2512	static struct sock *__udp4_lib_demux_lookup(struct net *net,
2513	__be16 loc_port, __be32 loc_addr,
2514	__be16 rmt_port, __be32 rmt_addr,
2515	int dif, int sdif)
2516	{
2517	struct udp_table *udptable = net->ipv4.udp_table;
2518	INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
2519	unsigned short hnum = ntohs(loc_port);
2520	unsigned int hash2, slot2;
2521	struct udp_hslot *hslot2;
2522	__portpair ports;
2523	struct sock *sk;
2524
2525	hash2 = ipv4_portaddr_hash(net, saddr: loc_addr, port: hnum);
2526	slot2 = hash2 & udptable->mask;
2527	hslot2 = &udptable->hash2[slot2];
2528	ports = INET_COMBINED_PORTS(rmt_port, hnum);
2529
2530	udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
2531	if (inet_match(net, sk, cookie: acookie, ports, dif, sdif))
2532	return sk;
2533	/* Only check first socket in chain */
2534	break;
2535	}
2536	return NULL;
2537	}
2538
2539	int udp_v4_early_demux(struct sk_buff *skb)
2540	{
2541	struct net *net = dev_net(dev: skb->dev);
2542	struct in_device *in_dev = NULL;
2543	const struct iphdr *iph;
2544	const struct udphdr *uh;
2545	struct sock *sk = NULL;
2546	struct dst_entry *dst;
2547	int dif = skb->dev->ifindex;
2548	int sdif = inet_sdif(skb);
2549	int ours;
2550
2551	/* validate the packet */
2552	if (!pskb_may_pull(skb, len: skb_transport_offset(skb) + sizeof(struct udphdr)))
2553	return 0;
2554
2555	iph = ip_hdr(skb);
2556	uh = udp_hdr(skb);
2557
2558	if (skb->pkt_type == PACKET_MULTICAST) {
2559	in_dev = __in_dev_get_rcu(dev: skb->dev);
2560
2561	if (!in_dev)
2562	return 0;
2563
2564	ours = ip_check_mc_rcu(dev: in_dev, mc_addr: iph->daddr, src_addr: iph->saddr,
2565	proto: iph->protocol);
2566	if (!ours)
2567	return 0;
2568
2569	sk = __udp4_lib_mcast_demux_lookup(net, loc_port: uh->dest, loc_addr: iph->daddr,
2570	rmt_port: uh->source, rmt_addr: iph->saddr,
2571	dif, sdif);
2572	} else if (skb->pkt_type == PACKET_HOST) {
2573	sk = __udp4_lib_demux_lookup(net, loc_port: uh->dest, loc_addr: iph->daddr,
2574	rmt_port: uh->source, rmt_addr: iph->saddr, dif, sdif);
2575	}
2576
2577	if (!sk || !refcount_inc_not_zero(r: &sk->sk_refcnt))
2578	return 0;
2579
2580	skb->sk = sk;
2581	skb->destructor = sock_efree;
2582	dst = rcu_dereference(sk->sk_rx_dst);
2583
2584	if (dst)
2585	dst = dst_check(dst, cookie: 0);
2586	if (dst) {
2587	u32 itag = 0;
2588
2589	/* set noref for now.
2590	* any place which wants to hold dst has to call
2591	* dst_hold_safe()
2592	*/
2593	skb_dst_set_noref(skb, dst);
2594
2595	/* for unconnected multicast sockets we need to validate
2596	* the source on each packet
2597	*/
2598	if (!inet_sk(sk)->inet_daddr && in_dev)
2599	return ip_mc_validate_source(skb, daddr: iph->daddr,
2600	saddr: iph->saddr,
2601	tos: iph->tos & IPTOS_RT_MASK,
2602	dev: skb->dev, in_dev, itag: &itag);
2603	}
2604	return 0;
2605	}
2606
2607	int udp_rcv(struct sk_buff *skb)
2608	{
2609	return __udp4_lib_rcv(skb, udptable: dev_net(dev: skb->dev)->ipv4.udp_table, IPPROTO_UDP);
2610	}
2611
2612	void udp_destroy_sock(struct sock *sk)
2613	{
2614	struct udp_sock *up = udp_sk(sk);
2615	bool slow = lock_sock_fast(sk);
2616
2617	/* protects from races with udp_abort() */
2618	sock_set_flag(sk, flag: SOCK_DEAD);
2619	udp_flush_pending_frames(sk);
2620	unlock_sock_fast(sk, slow);
2621	if (static_branch_unlikely(&udp_encap_needed_key)) {
2622	if (up->encap_type) {
2623	void (*encap_destroy)(struct sock *sk);
2624	encap_destroy = READ_ONCE(up->encap_destroy);
2625	if (encap_destroy)
2626	encap_destroy(sk);
2627	}
2628	if (udp_test_bit(ENCAP_ENABLED, sk))
2629	static_branch_dec(&udp_encap_needed_key);
2630	}
2631	}
2632
2633	static void set_xfrm_gro_udp_encap_rcv(__u16 encap_type, unsigned short family,
2634	struct sock *sk)
2635	{
2636	#ifdef CONFIG_XFRM
2637	if (udp_test_bit(GRO_ENABLED, sk) && encap_type == UDP_ENCAP_ESPINUDP) {
2638	if (family == AF_INET)
2639	WRITE_ONCE(udp_sk(sk)->gro_receive, xfrm4_gro_udp_encap_rcv);
2640	else if (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6)
2641	WRITE_ONCE(udp_sk(sk)->gro_receive, ipv6_stub->xfrm6_gro_udp_encap_rcv);
2642	}
2643	#endif
2644	}
2645
2646	/*
2647	* Socket option code for UDP
2648	*/
2649	int udp_lib_setsockopt(struct sock *sk, int level, int optname,
2650	sockptr_t optval, unsigned int optlen,
2651	int (*push_pending_frames)(struct sock *))
2652	{
2653	struct udp_sock *up = udp_sk(sk);
2654	int val, valbool;
2655	int err = 0;
2656	int is_udplite = IS_UDPLITE(sk);
2657
2658	if (level == SOL_SOCKET) {
2659	err = sk_setsockopt(sk, level, optname, optval, optlen);
2660
2661	if (optname == SO_RCVBUF || optname == SO_RCVBUFFORCE) {
2662	sockopt_lock_sock(sk);
2663	/* paired with READ_ONCE in udp_rmem_release() */
2664	WRITE_ONCE(up->forward_threshold, sk->sk_rcvbuf >> 2);
2665	sockopt_release_sock(sk);
2666	}
2667	return err;
2668	}
2669
2670	if (optlen < sizeof(int))
2671	return -EINVAL;
2672
2673	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
2674	return -EFAULT;
2675
2676	valbool = val ? 1 : 0;
2677
2678	switch (optname) {
2679	case UDP_CORK:
2680	if (val != 0) {
2681	udp_set_bit(CORK, sk);
2682	} else {
2683	udp_clear_bit(CORK, sk);
2684	lock_sock(sk);
2685	push_pending_frames(sk);
2686	release_sock(sk);
2687	}
2688	break;
2689
2690	case UDP_ENCAP:
2691	switch (val) {
2692	case 0:
2693	#ifdef CONFIG_XFRM
2694	case UDP_ENCAP_ESPINUDP:
2695	set_xfrm_gro_udp_encap_rcv(encap_type: val, family: sk->sk_family, sk);
2696	fallthrough;
2697	case UDP_ENCAP_ESPINUDP_NON_IKE:
2698	#if IS_ENABLED(CONFIG_IPV6)
2699	if (sk->sk_family == AF_INET6)
2700	WRITE_ONCE(up->encap_rcv,
2701	ipv6_stub->xfrm6_udp_encap_rcv);
2702	else
2703	#endif
2704	WRITE_ONCE(up->encap_rcv,
2705	xfrm4_udp_encap_rcv);
2706	#endif
2707	fallthrough;
2708	case UDP_ENCAP_L2TPINUDP:
2709	WRITE_ONCE(up->encap_type, val);
2710	udp_tunnel_encap_enable(sk);
2711	break;
2712	default:
2713	err = -ENOPROTOOPT;
2714	break;
2715	}
2716	break;
2717
2718	case UDP_NO_CHECK6_TX:
2719	udp_set_no_check6_tx(sk, val: valbool);
2720	break;
2721
2722	case UDP_NO_CHECK6_RX:
2723	udp_set_no_check6_rx(sk, val: valbool);
2724	break;
2725
2726	case UDP_SEGMENT:
2727	if (val < 0 || val > USHRT_MAX)
2728	return -EINVAL;
2729	WRITE_ONCE(up->gso_size, val);
2730	break;
2731
2732	case UDP_GRO:
2733
2734	/* when enabling GRO, accept the related GSO packet type */
2735	if (valbool)
2736	udp_tunnel_encap_enable(sk);
2737	udp_assign_bit(GRO_ENABLED, sk, valbool);
2738	udp_assign_bit(ACCEPT_L4, sk, valbool);
2739	set_xfrm_gro_udp_encap_rcv(encap_type: up->encap_type, family: sk->sk_family, sk);
2740	break;
2741
2742	/*
2743	* UDP-Lite's partial checksum coverage (RFC 3828).
2744	*/
2745	/* The sender sets actual checksum coverage length via this option.
2746	* The case coverage > packet length is handled by send module. */
2747	case UDPLITE_SEND_CSCOV:
2748	if (!is_udplite) /* Disable the option on UDP sockets */
2749	return -ENOPROTOOPT;
2750	if (val != 0 && val < 8) /* Illegal coverage: use default (8) */
2751	val = 8;
2752	else if (val > USHRT_MAX)
2753	val = USHRT_MAX;
2754	WRITE_ONCE(up->pcslen, val);
2755	udp_set_bit(UDPLITE_SEND_CC, sk);
2756	break;
2757
2758	/* The receiver specifies a minimum checksum coverage value. To make
2759	* sense, this should be set to at least 8 (as done below). If zero is
2760	* used, this again means full checksum coverage. */
2761	case UDPLITE_RECV_CSCOV:
2762	if (!is_udplite) /* Disable the option on UDP sockets */
2763	return -ENOPROTOOPT;
2764	if (val != 0 && val < 8) /* Avoid silly minimal values. */
2765	val = 8;
2766	else if (val > USHRT_MAX)
2767	val = USHRT_MAX;
2768	WRITE_ONCE(up->pcrlen, val);
2769	udp_set_bit(UDPLITE_RECV_CC, sk);
2770	break;
2771
2772	default:
2773	err = -ENOPROTOOPT;
2774	break;
2775	}
2776
2777	return err;
2778	}
2779	EXPORT_SYMBOL(udp_lib_setsockopt);
2780
2781	int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
2782	unsigned int optlen)
2783	{
2784	if (level == SOL_UDP || level == SOL_UDPLITE || level == SOL_SOCKET)
2785	return udp_lib_setsockopt(sk, level, optname,
2786	optval, optlen,
2787	udp_push_pending_frames);
2788	return ip_setsockopt(sk, level, optname, optval, optlen);
2789	}
2790
2791	int udp_lib_getsockopt(struct sock *sk, int level, int optname,
2792	char __user *optval, int __user *optlen)
2793	{
2794	struct udp_sock *up = udp_sk(sk);
2795	int val, len;
2796
2797	if (get_user(len, optlen))
2798	return -EFAULT;
2799
2800	len = min_t(unsigned int, len, sizeof(int));
2801
2802	if (len < 0)
2803	return -EINVAL;
2804
2805	switch (optname) {
2806	case UDP_CORK:
2807	val = udp_test_bit(CORK, sk);
2808	break;
2809
2810	case UDP_ENCAP:
2811	val = READ_ONCE(up->encap_type);
2812	break;
2813
2814	case UDP_NO_CHECK6_TX:
2815	val = udp_get_no_check6_tx(sk);
2816	break;
2817
2818	case UDP_NO_CHECK6_RX:
2819	val = udp_get_no_check6_rx(sk);
2820	break;
2821
2822	case UDP_SEGMENT:
2823	val = READ_ONCE(up->gso_size);
2824	break;
2825
2826	case UDP_GRO:
2827	val = udp_test_bit(GRO_ENABLED, sk);
2828	break;
2829
2830	/* The following two cannot be changed on UDP sockets, the return is
2831	* always 0 (which corresponds to the full checksum coverage of UDP). */
2832	case UDPLITE_SEND_CSCOV:
2833	val = READ_ONCE(up->pcslen);
2834	break;
2835
2836	case UDPLITE_RECV_CSCOV:
2837	val = READ_ONCE(up->pcrlen);
2838	break;
2839
2840	default:
2841	return -ENOPROTOOPT;
2842	}
2843
2844	if (put_user(len, optlen))
2845	return -EFAULT;
2846	if (copy_to_user(to: optval, from: &val, n: len))
2847	return -EFAULT;
2848	return 0;
2849	}
2850	EXPORT_SYMBOL(udp_lib_getsockopt);
2851
2852	int udp_getsockopt(struct sock *sk, int level, int optname,
2853	char __user *optval, int __user *optlen)
2854	{
2855	if (level == SOL_UDP || level == SOL_UDPLITE)
2856	return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2857	return ip_getsockopt(sk, level, optname, optval, optlen);
2858	}
2859
2860	/**
2861	* udp_poll - wait for a UDP event.
2862	* @file: - file struct
2863	* @sock: - socket
2864	* @wait: - poll table
2865	*
2866	* This is same as datagram poll, except for the special case of
2867	* blocking sockets. If application is using a blocking fd
2868	* and a packet with checksum error is in the queue;
2869	* then it could get return from select indicating data available
2870	* but then block when reading it. Add special case code
2871	* to work around these arguably broken applications.
2872	*/
2873	__poll_t udp_poll(struct file *file, struct socket *sock, poll_table *wait)
2874	{
2875	__poll_t mask = datagram_poll(file, sock, wait);
2876	struct sock *sk = sock->sk;
2877
2878	if (!skb_queue_empty_lockless(list: &udp_sk(sk)->reader_queue))
2879	mask |= EPOLLIN | EPOLLRDNORM;
2880
2881	/* Check for false positives due to checksum errors */
2882	if ((mask & EPOLLRDNORM) && !(file->f_flags & O_NONBLOCK) &&
2883	!(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1)
2884	mask &= ~(EPOLLIN | EPOLLRDNORM);
2885
2886	/* psock ingress_msg queue should not contain any bad checksum frames */
2887	if (sk_is_readable(sk))
2888	mask |= EPOLLIN | EPOLLRDNORM;
2889	return mask;
2890
2891	}
2892	EXPORT_SYMBOL(udp_poll);
2893
2894	int udp_abort(struct sock *sk, int err)
2895	{
2896	if (!has_current_bpf_ctx())
2897	lock_sock(sk);
2898
2899	/* udp{v6}_destroy_sock() sets it under the sk lock, avoid racing
2900	* with close()
2901	*/
2902	if (sock_flag(sk, flag: SOCK_DEAD))
2903	goto out;
2904
2905	sk->sk_err = err;
2906	sk_error_report(sk);
2907	__udp_disconnect(sk, 0);
2908
2909	out:
2910	if (!has_current_bpf_ctx())
2911	release_sock(sk);
2912
2913	return 0;
2914	}
2915	EXPORT_SYMBOL_GPL(udp_abort);
2916
2917	struct proto udp_prot = {
2918	.name = "UDP",
2919	.owner = THIS_MODULE,
2920	.close = udp_lib_close,
2921	.pre_connect = udp_pre_connect,
2922	.connect = ip4_datagram_connect,
2923	.disconnect = udp_disconnect,
2924	.ioctl = udp_ioctl,
2925	.init = udp_init_sock,
2926	.destroy = udp_destroy_sock,
2927	.setsockopt = udp_setsockopt,
2928	.getsockopt = udp_getsockopt,
2929	.sendmsg = udp_sendmsg,
2930	.recvmsg = udp_recvmsg,
2931	.splice_eof = udp_splice_eof,
2932	.release_cb = ip4_datagram_release_cb,
2933	.hash = udp_lib_hash,
2934	.unhash = udp_lib_unhash,
2935	.rehash = udp_v4_rehash,
2936	.get_port = udp_v4_get_port,
2937	.put_port = udp_lib_unhash,
2938	#ifdef CONFIG_BPF_SYSCALL
2939	.psock_update_sk_prot = udp_bpf_update_proto,
2940	#endif
2941	.memory_allocated = &udp_memory_allocated,
2942	.per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc,
2943
2944	.sysctl_mem = sysctl_udp_mem,
2945	.sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
2946	.sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min),
2947	.obj_size = sizeof(struct udp_sock),
2948	.h.udp_table = NULL,
2949	.diag_destroy = udp_abort,
2950	};
2951	EXPORT_SYMBOL(udp_prot);
2952
2953	/* ------------------------------------------------------------------------ */
2954	#ifdef CONFIG_PROC_FS
2955
2956	static unsigned short seq_file_family(const struct seq_file *seq);
2957	static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
2958	{
2959	unsigned short family = seq_file_family(seq);
2960
2961	/* AF_UNSPEC is used as a match all */
2962	return ((family == AF_UNSPEC || family == sk->sk_family) &&
2963	net_eq(net1: sock_net(sk), net2: seq_file_net(seq)));
2964	}
2965
2966	#ifdef CONFIG_BPF_SYSCALL
2967	static const struct seq_operations bpf_iter_udp_seq_ops;
2968	#endif
2969	static struct udp_table *udp_get_table_seq(struct seq_file *seq,
2970	struct net *net)
2971	{
2972	const struct udp_seq_afinfo *afinfo;
2973
2974	#ifdef CONFIG_BPF_SYSCALL
2975	if (seq->op == &bpf_iter_udp_seq_ops)
2976	return net->ipv4.udp_table;
2977	#endif
2978
2979	afinfo = pde_data(inode: file_inode(f: seq->file));
2980	return afinfo->udp_table ? : net->ipv4.udp_table;
2981	}
2982
2983	static struct sock *udp_get_first(struct seq_file *seq, int start)
2984	{
2985	struct udp_iter_state *state = seq->private;
2986	struct net *net = seq_file_net(seq);
2987	struct udp_table *udptable;
2988	struct sock *sk;
2989
2990	udptable = udp_get_table_seq(seq, net);
2991
2992	for (state->bucket = start; state->bucket <= udptable->mask;
2993	++state->bucket) {
2994	struct udp_hslot *hslot = &udptable->hash[state->bucket];
2995
2996	if (hlist_empty(h: &hslot->head))
2997	continue;
2998
2999	spin_lock_bh(lock: &hslot->lock);
3000	sk_for_each(sk, &hslot->head) {
3001	if (seq_sk_match(seq, sk))
3002	goto found;
3003	}
3004	spin_unlock_bh(lock: &hslot->lock);
3005	}
3006	sk = NULL;
3007	found:
3008	return sk;
3009	}
3010
3011	static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
3012	{
3013	struct udp_iter_state *state = seq->private;
3014	struct net *net = seq_file_net(seq);
3015	struct udp_table *udptable;
3016
3017	do {
3018	sk = sk_next(sk);
3019	} while (sk && !seq_sk_match(seq, sk));
3020
3021	if (!sk) {
3022	udptable = udp_get_table_seq(seq, net);
3023
3024	if (state->bucket <= udptable->mask)
3025	spin_unlock_bh(lock: &udptable->hash[state->bucket].lock);
3026
3027	return udp_get_first(seq, start: state->bucket + 1);
3028	}
3029	return sk;
3030	}
3031
3032	static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos)
3033	{
3034	struct sock *sk = udp_get_first(seq, start: 0);
3035
3036	if (sk)
3037	while (pos && (sk = udp_get_next(seq, sk)) != NULL)
3038	--pos;
3039	return pos ? NULL : sk;
3040	}
3041
3042	void *udp_seq_start(struct seq_file *seq, loff_t *pos)
3043	{
3044	struct udp_iter_state *state = seq->private;
3045	state->bucket = MAX_UDP_PORTS;
3046
3047	return *pos ? udp_get_idx(seq, pos: *pos-1) : SEQ_START_TOKEN;
3048	}
3049	EXPORT_SYMBOL(udp_seq_start);
3050
3051	void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3052	{
3053	struct sock *sk;
3054
3055	if (v == SEQ_START_TOKEN)
3056	sk = udp_get_idx(seq, pos: 0);
3057	else
3058	sk = udp_get_next(seq, sk: v);
3059
3060	++*pos;
3061	return sk;
3062	}
3063	EXPORT_SYMBOL(udp_seq_next);
3064
3065	void udp_seq_stop(struct seq_file *seq, void *v)
3066	{
3067	struct udp_iter_state *state = seq->private;
3068	struct udp_table *udptable;
3069
3070	udptable = udp_get_table_seq(seq, net: seq_file_net(seq));
3071
3072	if (state->bucket <= udptable->mask)
3073	spin_unlock_bh(lock: &udptable->hash[state->bucket].lock);
3074	}
3075	EXPORT_SYMBOL(udp_seq_stop);
3076
3077	/* ------------------------------------------------------------------------ */
3078	static void udp4_format_sock(struct sock *sp, struct seq_file *f,
3079	int bucket)
3080	{
3081	struct inet_sock *inet = inet_sk(sp);
3082	__be32 dest = inet->inet_daddr;
3083	__be32 src = inet->inet_rcv_saddr;
3084	__u16 destp = ntohs(inet->inet_dport);
3085	__u16 srcp = ntohs(inet->inet_sport);
3086
3087	seq_printf(m: f, fmt: "%5d: %08X:%04X %08X:%04X"
3088	" %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u",
3089	bucket, src, srcp, dest, destp, sp->sk_state,
3090	sk_wmem_alloc_get(sk: sp),
3091	udp_rqueue_get(sk: sp),
3092	0, 0L, 0,
3093	from_kuid_munged(to: seq_user_ns(seq: f), uid: sock_i_uid(sk: sp)),
3094	0, sock_i_ino(sk: sp),
3095	refcount_read(r: &sp->sk_refcnt), sp,
3096	atomic_read(v: &sp->sk_drops));
3097	}
3098
3099	int udp4_seq_show(struct seq_file *seq, void *v)
3100	{
3101	seq_setwidth(m: seq, size: 127);
3102	if (v == SEQ_START_TOKEN)
3103	seq_puts(m: seq, s: " sl local_address rem_address st tx_queue "
3104	"rx_queue tr tm->when retrnsmt uid timeout "
3105	"inode ref pointer drops");
3106	else {
3107	struct udp_iter_state *state = seq->private;
3108
3109	udp4_format_sock(sp: v, f: seq, bucket: state->bucket);
3110	}
3111	seq_pad(m: seq, c: '\n');
3112	return 0;
3113	}
3114
3115	#ifdef CONFIG_BPF_SYSCALL
3116	struct bpf_iter__udp {
3117	__bpf_md_ptr(struct bpf_iter_meta *, meta);
3118	__bpf_md_ptr(struct udp_sock *, udp_sk);
3119	uid_t uid __aligned(8);
3120	int bucket __aligned(8);
3121	};
3122
3123	struct bpf_udp_iter_state {
3124	struct udp_iter_state state;
3125	unsigned int cur_sk;
3126	unsigned int end_sk;
3127	unsigned int max_sk;
3128	int offset;
3129	struct sock **batch;
3130	bool st_bucket_done;
3131	};
3132
3133	static int bpf_iter_udp_realloc_batch(struct bpf_udp_iter_state *iter,
3134	unsigned int new_batch_sz);
3135	static struct sock *bpf_iter_udp_batch(struct seq_file *seq)
3136	{
3137	struct bpf_udp_iter_state *iter = seq->private;
3138	struct udp_iter_state *state = &iter->state;
3139	struct net *net = seq_file_net(seq);
3140	struct udp_table *udptable;
3141	unsigned int batch_sks = 0;
3142	bool resized = false;
3143	struct sock *sk;
3144
3145	/* The current batch is done, so advance the bucket. */
3146	if (iter->st_bucket_done) {
3147	state->bucket++;
3148	iter->offset = 0;
3149	}
3150
3151	udptable = udp_get_table_seq(seq, net);
3152
3153	again:
3154	/* New batch for the next bucket.
3155	* Iterate over the hash table to find a bucket with sockets matching
3156	* the iterator attributes, and return the first matching socket from
3157	* the bucket. The remaining matched sockets from the bucket are batched
3158	* before releasing the bucket lock. This allows BPF programs that are
3159	* called in seq_show to acquire the bucket lock if needed.
3160	*/
3161	iter->cur_sk = 0;
3162	iter->end_sk = 0;
3163	iter->st_bucket_done = false;
3164	batch_sks = 0;
3165
3166	for (; state->bucket <= udptable->mask; state->bucket++) {
3167	struct udp_hslot *hslot2 = &udptable->hash2[state->bucket];
3168
3169	if (hlist_empty(h: &hslot2->head)) {
3170	iter->offset = 0;
3171	continue;
3172	}
3173
3174	spin_lock_bh(lock: &hslot2->lock);
3175	udp_portaddr_for_each_entry(sk, &hslot2->head) {
3176	if (seq_sk_match(seq, sk)) {
3177	/* Resume from the last iterated socket at the
3178	* offset in the bucket before iterator was stopped.
3179	*/
3180	if (iter->offset) {
3181	--iter->offset;
3182	continue;
3183	}
3184	if (iter->end_sk < iter->max_sk) {
3185	sock_hold(sk);
3186	iter->batch[iter->end_sk++] = sk;
3187	}
3188	batch_sks++;
3189	}
3190	}
3191	spin_unlock_bh(lock: &hslot2->lock);
3192
3193	if (iter->end_sk)
3194	break;
3195
3196	/* Reset the current bucket's offset before moving to the next bucket. */
3197	iter->offset = 0;
3198	}
3199
3200	/* All done: no batch made. */
3201	if (!iter->end_sk)
3202	return NULL;
3203
3204	if (iter->end_sk == batch_sks) {
3205	/* Batching is done for the current bucket; return the first
3206	* socket to be iterated from the batch.
3207	*/
3208	iter->st_bucket_done = true;
3209	goto done;
3210	}
3211	if (!resized && !bpf_iter_udp_realloc_batch(iter, new_batch_sz: batch_sks * 3 / 2)) {
3212	resized = true;
3213	/* After allocating a larger batch, retry one more time to grab
3214	* the whole bucket.
3215	*/
3216	state->bucket--;
3217	goto again;
3218	}
3219	done:
3220	return iter->batch[0];
3221	}
3222
3223	static void *bpf_iter_udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3224	{
3225	struct bpf_udp_iter_state *iter = seq->private;
3226	struct sock *sk;
3227
3228	/* Whenever seq_next() is called, the iter->cur_sk is
3229	* done with seq_show(), so unref the iter->cur_sk.
3230	*/
3231	if (iter->cur_sk < iter->end_sk) {
3232	sock_put(sk: iter->batch[iter->cur_sk++]);
3233	++iter->offset;
3234	}
3235
3236	/* After updating iter->cur_sk, check if there are more sockets
3237	* available in the current bucket batch.
3238	*/
3239	if (iter->cur_sk < iter->end_sk)
3240	sk = iter->batch[iter->cur_sk];
3241	else
3242	/* Prepare a new batch. */
3243	sk = bpf_iter_udp_batch(seq);
3244
3245	++*pos;
3246	return sk;
3247	}
3248
3249	static void *bpf_iter_udp_seq_start(struct seq_file *seq, loff_t *pos)
3250	{
3251	/* bpf iter does not support lseek, so it always
3252	* continue from where it was stop()-ped.
3253	*/
3254	if (*pos)
3255	return bpf_iter_udp_batch(seq);
3256
3257	return SEQ_START_TOKEN;
3258	}
3259
3260	static int udp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
3261	struct udp_sock *udp_sk, uid_t uid, int bucket)
3262	{
3263	struct bpf_iter__udp ctx;
3264
3265	meta->seq_num--; /* skip SEQ_START_TOKEN */
3266	ctx.meta = meta;
3267	ctx.udp_sk = udp_sk;
3268	ctx.uid = uid;
3269	ctx.bucket = bucket;
3270	return bpf_iter_run_prog(prog, ctx: &ctx);
3271	}
3272
3273	static int bpf_iter_udp_seq_show(struct seq_file *seq, void *v)
3274	{
3275	struct udp_iter_state *state = seq->private;
3276	struct bpf_iter_meta meta;
3277	struct bpf_prog *prog;
3278	struct sock *sk = v;
3279	uid_t uid;
3280	int ret;
3281
3282	if (v == SEQ_START_TOKEN)
3283	return 0;
3284
3285	lock_sock(sk);
3286
3287	if (unlikely(sk_unhashed(sk))) {
3288	ret = SEQ_SKIP;
3289	goto unlock;
3290	}
3291
3292	uid = from_kuid_munged(to: seq_user_ns(seq), uid: sock_i_uid(sk));
3293	meta.seq = seq;
3294	prog = bpf_iter_get_info(meta: &meta, in_stop: false);
3295	ret = udp_prog_seq_show(prog, meta: &meta, udp_sk: v, uid, bucket: state->bucket);
3296
3297	unlock:
3298	release_sock(sk);
3299	return ret;
3300	}
3301
3302	static void bpf_iter_udp_put_batch(struct bpf_udp_iter_state *iter)
3303	{
3304	while (iter->cur_sk < iter->end_sk)
3305	sock_put(sk: iter->batch[iter->cur_sk++]);
3306	}
3307
3308	static void bpf_iter_udp_seq_stop(struct seq_file *seq, void *v)
3309	{
3310	struct bpf_udp_iter_state *iter = seq->private;
3311	struct bpf_iter_meta meta;
3312	struct bpf_prog *prog;
3313
3314	if (!v) {
3315	meta.seq = seq;
3316	prog = bpf_iter_get_info(meta: &meta, in_stop: true);
3317	if (prog)
3318	(void)udp_prog_seq_show(prog, meta: &meta, udp_sk: v, uid: 0, bucket: 0);
3319	}
3320
3321	if (iter->cur_sk < iter->end_sk) {
3322	bpf_iter_udp_put_batch(iter);
3323	iter->st_bucket_done = false;
3324	}
3325	}
3326
3327	static const struct seq_operations bpf_iter_udp_seq_ops = {
3328	.start = bpf_iter_udp_seq_start,
3329	.next = bpf_iter_udp_seq_next,
3330	.stop = bpf_iter_udp_seq_stop,
3331	.show = bpf_iter_udp_seq_show,
3332	};
3333	#endif
3334
3335	static unsigned short seq_file_family(const struct seq_file *seq)
3336	{
3337	const struct udp_seq_afinfo *afinfo;
3338
3339	#ifdef CONFIG_BPF_SYSCALL
3340	/* BPF iterator: bpf programs to filter sockets. */
3341	if (seq->op == &bpf_iter_udp_seq_ops)
3342	return AF_UNSPEC;
3343	#endif
3344
3345	/* Proc fs iterator */
3346	afinfo = pde_data(inode: file_inode(f: seq->file));
3347	return afinfo->family;
3348	}
3349
3350	const struct seq_operations udp_seq_ops = {
3351	.start = udp_seq_start,
3352	.next = udp_seq_next,
3353	.stop = udp_seq_stop,
3354	.show = udp4_seq_show,
3355	};
3356	EXPORT_SYMBOL(udp_seq_ops);
3357
3358	static struct udp_seq_afinfo udp4_seq_afinfo = {
3359	.family = AF_INET,
3360	.udp_table = NULL,
3361	};
3362
3363	static int __net_init udp4_proc_init_net(struct net *net)
3364	{
3365	if (!proc_create_net_data(name: "udp", mode: 0444, parent: net->proc_net, ops: &udp_seq_ops,
3366	state_size: sizeof(struct udp_iter_state), data: &udp4_seq_afinfo))
3367	return -ENOMEM;
3368	return 0;
3369	}
3370
3371	static void __net_exit udp4_proc_exit_net(struct net *net)
3372	{
3373	remove_proc_entry("udp", net->proc_net);
3374	}
3375
3376	static struct pernet_operations udp4_net_ops = {
3377	.init = udp4_proc_init_net,
3378	.exit = udp4_proc_exit_net,
3379	};
3380
3381	int __init udp4_proc_init(void)
3382	{
3383	return register_pernet_subsys(&udp4_net_ops);
3384	}
3385
3386	void udp4_proc_exit(void)
3387	{
3388	unregister_pernet_subsys(&udp4_net_ops);
3389	}
3390	#endif /* CONFIG_PROC_FS */
3391
3392	static __initdata unsigned long uhash_entries;
3393	static int __init set_uhash_entries(char *str)
3394	{
3395	ssize_t ret;
3396
3397	if (!str)
3398	return 0;
3399
3400	ret = kstrtoul(s: str, base: 0, res: &uhash_entries);
3401	if (ret)
3402	return 0;
3403
3404	if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN)
3405	uhash_entries = UDP_HTABLE_SIZE_MIN;
3406	return 1;
3407	}
3408	__setup("uhash_entries=", set_uhash_entries);
3409
3410	void __init udp_table_init(struct udp_table *table, const char *name)
3411	{
3412	unsigned int i;
3413
3414	table->hash = alloc_large_system_hash(tablename: name,
3415	bucketsize: 2 * sizeof(struct udp_hslot),
3416	numentries: uhash_entries,
3417	scale: 21, /* one slot per 2 MB */
3418	flags: 0,
3419	hash_shift: &table->log,
3420	hash_mask: &table->mask,
3421	UDP_HTABLE_SIZE_MIN,
3422	UDP_HTABLE_SIZE_MAX);
3423
3424	table->hash2 = table->hash + (table->mask + 1);
3425	for (i = 0; i <= table->mask; i++) {
3426	INIT_HLIST_HEAD(&table->hash[i].head);
3427	table->hash[i].count = 0;
3428	spin_lock_init(&table->hash[i].lock);
3429	}
3430	for (i = 0; i <= table->mask; i++) {
3431	INIT_HLIST_HEAD(&table->hash2[i].head);
3432	table->hash2[i].count = 0;
3433	spin_lock_init(&table->hash2[i].lock);
3434	}
3435	}
3436
3437	u32 udp_flow_hashrnd(void)
3438	{
3439	static u32 hashrnd __read_mostly;
3440
3441	net_get_random_once(&hashrnd, sizeof(hashrnd));
3442
3443	return hashrnd;
3444	}
3445	EXPORT_SYMBOL(udp_flow_hashrnd);
3446
3447	static void __net_init udp_sysctl_init(struct net *net)
3448	{
3449	net->ipv4.sysctl_udp_rmem_min = PAGE_SIZE;
3450	net->ipv4.sysctl_udp_wmem_min = PAGE_SIZE;
3451
3452	#ifdef CONFIG_NET_L3_MASTER_DEV
3453	net->ipv4.sysctl_udp_l3mdev_accept = 0;
3454	#endif
3455	}
3456
3457	static struct udp_table __net_init *udp_pernet_table_alloc(unsigned int hash_entries)
3458	{
3459	struct udp_table *udptable;
3460	int i;
3461
3462	udptable = kmalloc(size: sizeof(*udptable), GFP_KERNEL);
3463	if (!udptable)
3464	goto out;
3465
3466	udptable->hash = vmalloc_huge(size: hash_entries * 2 * sizeof(struct udp_hslot),
3467	GFP_KERNEL_ACCOUNT);
3468	if (!udptable->hash)
3469	goto free_table;
3470
3471	udptable->hash2 = udptable->hash + hash_entries;
3472	udptable->mask = hash_entries - 1;
3473	udptable->log = ilog2(hash_entries);
3474
3475	for (i = 0; i < hash_entries; i++) {
3476	INIT_HLIST_HEAD(&udptable->hash[i].head);
3477	udptable->hash[i].count = 0;
3478	spin_lock_init(&udptable->hash[i].lock);
3479
3480	INIT_HLIST_HEAD(&udptable->hash2[i].head);
3481	udptable->hash2[i].count = 0;
3482	spin_lock_init(&udptable->hash2[i].lock);
3483	}
3484
3485	return udptable;
3486
3487	free_table:
3488	kfree(objp: udptable);
3489	out:
3490	return NULL;
3491	}
3492
3493	static void __net_exit udp_pernet_table_free(struct net *net)
3494	{
3495	struct udp_table *udptable = net->ipv4.udp_table;
3496
3497	if (udptable == &udp_table)
3498	return;
3499
3500	kvfree(addr: udptable->hash);
3501	kfree(objp: udptable);
3502	}
3503
3504	static void __net_init udp_set_table(struct net *net)
3505	{
3506	struct udp_table *udptable;
3507	unsigned int hash_entries;
3508	struct net *old_net;
3509
3510	if (net_eq(net1: net, net2: &init_net))
3511	goto fallback;
3512
3513	old_net = current->nsproxy->net_ns;
3514	hash_entries = READ_ONCE(old_net->ipv4.sysctl_udp_child_hash_entries);
3515	if (!hash_entries)
3516	goto fallback;
3517
3518	/* Set min to keep the bitmap on stack in udp_lib_get_port() */
3519	if (hash_entries < UDP_HTABLE_SIZE_MIN_PERNET)
3520	hash_entries = UDP_HTABLE_SIZE_MIN_PERNET;
3521	else
3522	hash_entries = roundup_pow_of_two(hash_entries);
3523
3524	udptable = udp_pernet_table_alloc(hash_entries);
3525	if (udptable) {
3526	net->ipv4.udp_table = udptable;
3527	} else {
3528	pr_warn("Failed to allocate UDP hash table (entries: %u) "
3529	"for a netns, fallback to the global one\n",
3530	hash_entries);
3531	fallback:
3532	net->ipv4.udp_table = &udp_table;
3533	}
3534	}
3535
3536	static int __net_init udp_pernet_init(struct net *net)
3537	{
3538	udp_sysctl_init(net);
3539	udp_set_table(net);
3540
3541	return 0;
3542	}
3543
3544	static void __net_exit udp_pernet_exit(struct net *net)
3545	{
3546	udp_pernet_table_free(net);
3547	}
3548
3549	static struct pernet_operations __net_initdata udp_sysctl_ops = {
3550	.init = udp_pernet_init,
3551	.exit = udp_pernet_exit,
3552	};
3553
3554	#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3555	DEFINE_BPF_ITER_FUNC(udp, struct bpf_iter_meta *meta,
3556	struct udp_sock *udp_sk, uid_t uid, int bucket)
3557
3558	static int bpf_iter_udp_realloc_batch(struct bpf_udp_iter_state *iter,
3559	unsigned int new_batch_sz)
3560	{
3561	struct sock **new_batch;
3562
3563	new_batch = kvmalloc_array(n: new_batch_sz, size: sizeof(*new_batch),
3564	GFP_USER | __GFP_NOWARN);
3565	if (!new_batch)
3566	return -ENOMEM;
3567
3568	bpf_iter_udp_put_batch(iter);
3569	kvfree(addr: iter->batch);
3570	iter->batch = new_batch;
3571	iter->max_sk = new_batch_sz;
3572
3573	return 0;
3574	}
3575
3576	#define INIT_BATCH_SZ 16
3577
3578	static int bpf_iter_init_udp(void *priv_data, struct bpf_iter_aux_info *aux)
3579	{
3580	struct bpf_udp_iter_state *iter = priv_data;
3581	int ret;
3582
3583	ret = bpf_iter_init_seq_net(priv_data, aux);
3584	if (ret)
3585	return ret;
3586
3587	ret = bpf_iter_udp_realloc_batch(iter, INIT_BATCH_SZ);
3588	if (ret)
3589	bpf_iter_fini_seq_net(priv_data);
3590
3591	return ret;
3592	}
3593
3594	static void bpf_iter_fini_udp(void *priv_data)
3595	{
3596	struct bpf_udp_iter_state *iter = priv_data;
3597
3598	bpf_iter_fini_seq_net(priv_data);
3599	kvfree(addr: iter->batch);
3600	}
3601
3602	static const struct bpf_iter_seq_info udp_seq_info = {
3603	.seq_ops = &bpf_iter_udp_seq_ops,
3604	.init_seq_private = bpf_iter_init_udp,
3605	.fini_seq_private = bpf_iter_fini_udp,
3606	.seq_priv_size = sizeof(struct bpf_udp_iter_state),
3607	};
3608
3609	static struct bpf_iter_reg udp_reg_info = {
3610	.target = "udp",
3611	.ctx_arg_info_size = 1,
3612	.ctx_arg_info = {
3613	{ offsetof(struct bpf_iter__udp, udp_sk),
3614	.reg_type: PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
3615	},
3616	.seq_info = &udp_seq_info,
3617	};
3618
3619	static void __init bpf_iter_register(void)
3620	{
3621	udp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_UDP];
3622	if (bpf_iter_reg_target(reg_info: &udp_reg_info))
3623	pr_warn("Warning: could not register bpf iterator udp\n");
3624	}
3625	#endif
3626
3627	void __init udp_init(void)
3628	{
3629	unsigned long limit;
3630	unsigned int i;
3631
3632	udp_table_init(table: &udp_table, name: "UDP");
3633	limit = nr_free_buffer_pages() / 8;
3634	limit = max(limit, 128UL);
3635	sysctl_udp_mem[0] = limit / 4 * 3;
3636	sysctl_udp_mem[1] = limit;
3637	sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2;
3638
3639	/* 16 spinlocks per cpu */
3640	udp_busylocks_log = ilog2(nr_cpu_ids) + 4;
3641	udp_busylocks = kmalloc(size: sizeof(spinlock_t) << udp_busylocks_log,
3642	GFP_KERNEL);
3643	if (!udp_busylocks)
3644	panic(fmt: "UDP: failed to alloc udp_busylocks\n");
3645	for (i = 0; i < (1U << udp_busylocks_log); i++)
3646	spin_lock_init(udp_busylocks + i);
3647
3648	if (register_pernet_subsys(&udp_sysctl_ops))
3649	panic(fmt: "UDP: failed to init sysctl parameters.\n");
3650
3651	#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3652	bpf_iter_register();
3653	#endif
3654	}
3655