1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * The User Datagram Protocol (UDP).
7 *
8 * Authors: Ross Biro
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
11 * Alan Cox, <alan@lxorguk.ukuu.org.uk>
12 * Hirokazu Takahashi, <taka@valinux.co.jp>
13 *
14 * Fixes:
15 * Alan Cox : verify_area() calls
16 * Alan Cox : stopped close while in use off icmp
17 * messages. Not a fix but a botch that
18 * for udp at least is 'valid'.
19 * Alan Cox : Fixed icmp handling properly
20 * Alan Cox : Correct error for oversized datagrams
21 * Alan Cox : Tidied select() semantics.
22 * Alan Cox : udp_err() fixed properly, also now
23 * select and read wake correctly on errors
24 * Alan Cox : udp_send verify_area moved to avoid mem leak
25 * Alan Cox : UDP can count its memory
26 * Alan Cox : send to an unknown connection causes
27 * an ECONNREFUSED off the icmp, but
28 * does NOT close.
29 * Alan Cox : Switched to new sk_buff handlers. No more backlog!
30 * Alan Cox : Using generic datagram code. Even smaller and the PEEK
31 * bug no longer crashes it.
32 * Fred Van Kempen : Net2e support for sk->broadcast.
33 * Alan Cox : Uses skb_free_datagram
34 * Alan Cox : Added get/set sockopt support.
35 * Alan Cox : Broadcasting without option set returns EACCES.
36 * Alan Cox : No wakeup calls. Instead we now use the callbacks.
37 * Alan Cox : Use ip_tos and ip_ttl
38 * Alan Cox : SNMP Mibs
39 * Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support.
40 * Matt Dillon : UDP length checks.
41 * Alan Cox : Smarter af_inet used properly.
42 * Alan Cox : Use new kernel side addressing.
43 * Alan Cox : Incorrect return on truncated datagram receive.
44 * Arnt Gulbrandsen : New udp_send and stuff
45 * Alan Cox : Cache last socket
46 * Alan Cox : Route cache
47 * Jon Peatfield : Minor efficiency fix to sendto().
48 * Mike Shaver : RFC1122 checks.
49 * Alan Cox : Nonblocking error fix.
50 * Willy Konynenberg : Transparent proxying support.
51 * Mike McLagan : Routing by source
52 * David S. Miller : New socket lookup architecture.
53 * Last socket cache retained as it
54 * does have a high hit rate.
55 * Olaf Kirch : Don't linearise iovec on sendmsg.
56 * Andi Kleen : Some cleanups, cache destination entry
57 * for connect.
58 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
59 * Melvin Smith : Check msg_name not msg_namelen in sendto(),
60 * return ENOTCONN for unconnected sockets (POSIX)
61 * Janos Farkas : don't deliver multi/broadcasts to a different
62 * bound-to-device socket
63 * Hirokazu Takahashi : HW checksumming for outgoing UDP
64 * datagrams.
65 * Hirokazu Takahashi : sendfile() on UDP works now.
66 * Arnaldo C. Melo : convert /proc/net/udp to seq_file
67 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
68 * Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind
69 * a single port at the same time.
70 * Derek Atkins <derek@ihtfp.com>: Add Encapulation Support
71 * James Chapman : Add L2TP encapsulation type.
72 *
73 *
74 * This program is free software; you can redistribute it and/or
75 * modify it under the terms of the GNU General Public License
76 * as published by the Free Software Foundation; either version
77 * 2 of the License, or (at your option) any later version.
78 */
79
80#define pr_fmt(fmt) "UDP: " fmt
81
82#include <linux/uaccess.h>
83#include <asm/ioctls.h>
84#include <linux/memblock.h>
85#include <linux/highmem.h>
86#include <linux/swap.h>
87#include <linux/types.h>
88#include <linux/fcntl.h>
89#include <linux/module.h>
90#include <linux/socket.h>
91#include <linux/sockios.h>
92#include <linux/igmp.h>
93#include <linux/inetdevice.h>
94#include <linux/in.h>
95#include <linux/errno.h>
96#include <linux/timer.h>
97#include <linux/mm.h>
98#include <linux/inet.h>
99#include <linux/netdevice.h>
100#include <linux/slab.h>
101#include <net/tcp_states.h>
102#include <linux/skbuff.h>
103#include <linux/proc_fs.h>
104#include <linux/seq_file.h>
105#include <net/net_namespace.h>
106#include <net/icmp.h>
107#include <net/inet_hashtables.h>
108#include <net/ip_tunnels.h>
109#include <net/route.h>
110#include <net/checksum.h>
111#include <net/xfrm.h>
112#include <trace/events/udp.h>
113#include <linux/static_key.h>
114#include <trace/events/skb.h>
115#include <net/busy_poll.h>
116#include "udp_impl.h"
117#include <net/sock_reuseport.h>
118#include <net/addrconf.h>
119#include <net/udp_tunnel.h>
120
121struct udp_table udp_table __read_mostly;
122EXPORT_SYMBOL(udp_table);
123
124long sysctl_udp_mem[3] __read_mostly;
125EXPORT_SYMBOL(sysctl_udp_mem);
126
127atomic_long_t udp_memory_allocated;
128EXPORT_SYMBOL(udp_memory_allocated);
129
130#define MAX_UDP_PORTS 65536
131#define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN)
132
133/* IPCB reference means this can not be used from early demux */
134static bool udp_lib_exact_dif_match(struct net *net, struct sk_buff *skb)
135{
136#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
137 if (!net->ipv4.sysctl_udp_l3mdev_accept &&
138 skb && ipv4_l3mdev_skb(IPCB(skb)->flags))
139 return true;
140#endif
141 return false;
142}
143
144static int udp_lib_lport_inuse(struct net *net, __u16 num,
145 const struct udp_hslot *hslot,
146 unsigned long *bitmap,
147 struct sock *sk, unsigned int log)
148{
149 struct sock *sk2;
150 kuid_t uid = sock_i_uid(sk);
151
152 sk_for_each(sk2, &hslot->head) {
153 if (net_eq(sock_net(sk2), net) &&
154 sk2 != sk &&
155 (bitmap || udp_sk(sk2)->udp_port_hash == num) &&
156 (!sk2->sk_reuse || !sk->sk_reuse) &&
157 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
158 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
159 inet_rcv_saddr_equal(sk, sk2, true)) {
160 if (sk2->sk_reuseport && sk->sk_reuseport &&
161 !rcu_access_pointer(sk->sk_reuseport_cb) &&
162 uid_eq(uid, sock_i_uid(sk2))) {
163 if (!bitmap)
164 return 0;
165 } else {
166 if (!bitmap)
167 return 1;
168 __set_bit(udp_sk(sk2)->udp_port_hash >> log,
169 bitmap);
170 }
171 }
172 }
173 return 0;
174}
175
176/*
177 * Note: we still hold spinlock of primary hash chain, so no other writer
178 * can insert/delete a socket with local_port == num
179 */
180static int udp_lib_lport_inuse2(struct net *net, __u16 num,
181 struct udp_hslot *hslot2,
182 struct sock *sk)
183{
184 struct sock *sk2;
185 kuid_t uid = sock_i_uid(sk);
186 int res = 0;
187
188 spin_lock(&hslot2->lock);
189 udp_portaddr_for_each_entry(sk2, &hslot2->head) {
190 if (net_eq(sock_net(sk2), net) &&
191 sk2 != sk &&
192 (udp_sk(sk2)->udp_port_hash == num) &&
193 (!sk2->sk_reuse || !sk->sk_reuse) &&
194 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
195 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
196 inet_rcv_saddr_equal(sk, sk2, true)) {
197 if (sk2->sk_reuseport && sk->sk_reuseport &&
198 !rcu_access_pointer(sk->sk_reuseport_cb) &&
199 uid_eq(uid, sock_i_uid(sk2))) {
200 res = 0;
201 } else {
202 res = 1;
203 }
204 break;
205 }
206 }
207 spin_unlock(&hslot2->lock);
208 return res;
209}
210
211static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot)
212{
213 struct net *net = sock_net(sk);
214 kuid_t uid = sock_i_uid(sk);
215 struct sock *sk2;
216
217 sk_for_each(sk2, &hslot->head) {
218 if (net_eq(sock_net(sk2), net) &&
219 sk2 != sk &&
220 sk2->sk_family == sk->sk_family &&
221 ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
222 (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) &&
223 (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
224 sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) &&
225 inet_rcv_saddr_equal(sk, sk2, false)) {
226 return reuseport_add_sock(sk, sk2,
227 inet_rcv_saddr_any(sk));
228 }
229 }
230
231 return reuseport_alloc(sk, inet_rcv_saddr_any(sk));
232}
233
234/**
235 * udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6
236 *
237 * @sk: socket struct in question
238 * @snum: port number to look up
239 * @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
240 * with NULL address
241 */
242int udp_lib_get_port(struct sock *sk, unsigned short snum,
243 unsigned int hash2_nulladdr)
244{
245 struct udp_hslot *hslot, *hslot2;
246 struct udp_table *udptable = sk->sk_prot->h.udp_table;
247 int error = 1;
248 struct net *net = sock_net(sk);
249
250 if (!snum) {
251 int low, high, remaining;
252 unsigned int rand;
253 unsigned short first, last;
254 DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);
255
256 inet_get_local_port_range(net, &low, &high);
257 remaining = (high - low) + 1;
258
259 rand = prandom_u32();
260 first = reciprocal_scale(rand, remaining) + low;
261 /*
262 * force rand to be an odd multiple of UDP_HTABLE_SIZE
263 */
264 rand = (rand | 1) * (udptable->mask + 1);
265 last = first + udptable->mask + 1;
266 do {
267 hslot = udp_hashslot(udptable, net, first);
268 bitmap_zero(bitmap, PORTS_PER_CHAIN);
269 spin_lock_bh(&hslot->lock);
270 udp_lib_lport_inuse(net, snum, hslot, bitmap, sk,
271 udptable->log);
272
273 snum = first;
274 /*
275 * Iterate on all possible values of snum for this hash.
276 * Using steps of an odd multiple of UDP_HTABLE_SIZE
277 * give us randomization and full range coverage.
278 */
279 do {
280 if (low <= snum && snum <= high &&
281 !test_bit(snum >> udptable->log, bitmap) &&
282 !inet_is_local_reserved_port(net, snum))
283 goto found;
284 snum += rand;
285 } while (snum != first);
286 spin_unlock_bh(&hslot->lock);
287 cond_resched();
288 } while (++first != last);
289 goto fail;
290 } else {
291 hslot = udp_hashslot(udptable, net, snum);
292 spin_lock_bh(&hslot->lock);
293 if (hslot->count > 10) {
294 int exist;
295 unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum;
296
297 slot2 &= udptable->mask;
298 hash2_nulladdr &= udptable->mask;
299
300 hslot2 = udp_hashslot2(udptable, slot2);
301 if (hslot->count < hslot2->count)
302 goto scan_primary_hash;
303
304 exist = udp_lib_lport_inuse2(net, snum, hslot2, sk);
305 if (!exist && (hash2_nulladdr != slot2)) {
306 hslot2 = udp_hashslot2(udptable, hash2_nulladdr);
307 exist = udp_lib_lport_inuse2(net, snum, hslot2,
308 sk);
309 }
310 if (exist)
311 goto fail_unlock;
312 else
313 goto found;
314 }
315scan_primary_hash:
316 if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0))
317 goto fail_unlock;
318 }
319found:
320 inet_sk(sk)->inet_num = snum;
321 udp_sk(sk)->udp_port_hash = snum;
322 udp_sk(sk)->udp_portaddr_hash ^= snum;
323 if (sk_unhashed(sk)) {
324 if (sk->sk_reuseport &&
325 udp_reuseport_add_sock(sk, hslot)) {
326 inet_sk(sk)->inet_num = 0;
327 udp_sk(sk)->udp_port_hash = 0;
328 udp_sk(sk)->udp_portaddr_hash ^= snum;
329 goto fail_unlock;
330 }
331
332 sk_add_node_rcu(sk, &hslot->head);
333 hslot->count++;
334 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
335
336 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
337 spin_lock(&hslot2->lock);
338 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
339 sk->sk_family == AF_INET6)
340 hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node,
341 &hslot2->head);
342 else
343 hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
344 &hslot2->head);
345 hslot2->count++;
346 spin_unlock(&hslot2->lock);
347 }
348 sock_set_flag(sk, SOCK_RCU_FREE);
349 error = 0;
350fail_unlock:
351 spin_unlock_bh(&hslot->lock);
352fail:
353 return error;
354}
355EXPORT_SYMBOL(udp_lib_get_port);
356
357int udp_v4_get_port(struct sock *sk, unsigned short snum)
358{
359 unsigned int hash2_nulladdr =
360 ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
361 unsigned int hash2_partial =
362 ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
363
364 /* precompute partial secondary hash */
365 udp_sk(sk)->udp_portaddr_hash = hash2_partial;
366 return udp_lib_get_port(sk, snum, hash2_nulladdr);
367}
368
369static int compute_score(struct sock *sk, struct net *net,
370 __be32 saddr, __be16 sport,
371 __be32 daddr, unsigned short hnum,
372 int dif, int sdif, bool exact_dif)
373{
374 int score;
375 struct inet_sock *inet;
376 bool dev_match;
377
378 if (!net_eq(sock_net(sk), net) ||
379 udp_sk(sk)->udp_port_hash != hnum ||
380 ipv6_only_sock(sk))
381 return -1;
382
383 if (sk->sk_rcv_saddr != daddr)
384 return -1;
385
386 score = (sk->sk_family == PF_INET) ? 2 : 1;
387
388 inet = inet_sk(sk);
389 if (inet->inet_daddr) {
390 if (inet->inet_daddr != saddr)
391 return -1;
392 score += 4;
393 }
394
395 if (inet->inet_dport) {
396 if (inet->inet_dport != sport)
397 return -1;
398 score += 4;
399 }
400
401 dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if,
402 dif, sdif);
403 if (!dev_match)
404 return -1;
405 score += 4;
406
407 if (sk->sk_incoming_cpu == raw_smp_processor_id())
408 score++;
409 return score;
410}
411
412static u32 udp_ehashfn(const struct net *net, const __be32 laddr,
413 const __u16 lport, const __be32 faddr,
414 const __be16 fport)
415{
416 static u32 udp_ehash_secret __read_mostly;
417
418 net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
419
420 return __inet_ehashfn(laddr, lport, faddr, fport,
421 udp_ehash_secret + net_hash_mix(net));
422}
423
424/* called with rcu_read_lock() */
425static struct sock *udp4_lib_lookup2(struct net *net,
426 __be32 saddr, __be16 sport,
427 __be32 daddr, unsigned int hnum,
428 int dif, int sdif, bool exact_dif,
429 struct udp_hslot *hslot2,
430 struct sk_buff *skb)
431{
432 struct sock *sk, *result;
433 int score, badness;
434 u32 hash = 0;
435
436 result = NULL;
437 badness = 0;
438 udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
439 score = compute_score(sk, net, saddr, sport,
440 daddr, hnum, dif, sdif, exact_dif);
441 if (score > badness) {
442 if (sk->sk_reuseport) {
443 hash = udp_ehashfn(net, daddr, hnum,
444 saddr, sport);
445 result = reuseport_select_sock(sk, hash, skb,
446 sizeof(struct udphdr));
447 if (result)
448 return result;
449 }
450 badness = score;
451 result = sk;
452 }
453 }
454 return result;
455}
456
457/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
458 * harder than this. -DaveM
459 */
460struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
461 __be16 sport, __be32 daddr, __be16 dport, int dif,
462 int sdif, struct udp_table *udptable, struct sk_buff *skb)
463{
464 struct sock *result;
465 unsigned short hnum = ntohs(dport);
466 unsigned int hash2, slot2;
467 struct udp_hslot *hslot2;
468 bool exact_dif = udp_lib_exact_dif_match(net, skb);
469
470 hash2 = ipv4_portaddr_hash(net, daddr, hnum);
471 slot2 = hash2 & udptable->mask;
472 hslot2 = &udptable->hash2[slot2];
473
474 result = udp4_lib_lookup2(net, saddr, sport,
475 daddr, hnum, dif, sdif,
476 exact_dif, hslot2, skb);
477 if (!result) {
478 hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
479 slot2 = hash2 & udptable->mask;
480 hslot2 = &udptable->hash2[slot2];
481
482 result = udp4_lib_lookup2(net, saddr, sport,
483 htonl(INADDR_ANY), hnum, dif, sdif,
484 exact_dif, hslot2, skb);
485 }
486 if (unlikely(IS_ERR(result)))
487 return NULL;
488 return result;
489}
490EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
491
492static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
493 __be16 sport, __be16 dport,
494 struct udp_table *udptable)
495{
496 const struct iphdr *iph = ip_hdr(skb);
497
498 return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
499 iph->daddr, dport, inet_iif(skb),
500 inet_sdif(skb), udptable, skb);
501}
502
503struct sock *udp4_lib_lookup_skb(struct sk_buff *skb,
504 __be16 sport, __be16 dport)
505{
506 return __udp4_lib_lookup_skb(skb, sport, dport, &udp_table);
507}
508EXPORT_SYMBOL_GPL(udp4_lib_lookup_skb);
509
510/* Must be called under rcu_read_lock().
511 * Does increment socket refcount.
512 */
513#if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4)
514struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport,
515 __be32 daddr, __be16 dport, int dif)
516{
517 struct sock *sk;
518
519 sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport,
520 dif, 0, &udp_table, NULL);
521 if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
522 sk = NULL;
523 return sk;
524}
525EXPORT_SYMBOL_GPL(udp4_lib_lookup);
526#endif
527
528static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk,
529 __be16 loc_port, __be32 loc_addr,
530 __be16 rmt_port, __be32 rmt_addr,
531 int dif, int sdif, unsigned short hnum)
532{
533 struct inet_sock *inet = inet_sk(sk);
534
535 if (!net_eq(sock_net(sk), net) ||
536 udp_sk(sk)->udp_port_hash != hnum ||
537 (inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
538 (inet->inet_dport != rmt_port && inet->inet_dport) ||
539 (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
540 ipv6_only_sock(sk) ||
541 (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif &&
542 sk->sk_bound_dev_if != sdif))
543 return false;
544 if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif))
545 return false;
546 return true;
547}
548
549DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key);
550void udp_encap_enable(void)
551{
552 static_branch_inc(&udp_encap_needed_key);
553}
554EXPORT_SYMBOL(udp_encap_enable);
555
556/* Handler for tunnels with arbitrary destination ports: no socket lookup, go
557 * through error handlers in encapsulations looking for a match.
558 */
559static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info)
560{
561 int i;
562
563 for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) {
564 int (*handler)(struct sk_buff *skb, u32 info);
565 const struct ip_tunnel_encap_ops *encap;
566
567 encap = rcu_dereference(iptun_encaps[i]);
568 if (!encap)
569 continue;
570 handler = encap->err_handler;
571 if (handler && !handler(skb, info))
572 return 0;
573 }
574
575 return -ENOENT;
576}
577
578/* Try to match ICMP errors to UDP tunnels by looking up a socket without
579 * reversing source and destination port: this will match tunnels that force the
580 * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that
581 * lwtunnels might actually break this assumption by being configured with
582 * different destination ports on endpoints, in this case we won't be able to
583 * trace ICMP messages back to them.
584 *
585 * If this doesn't match any socket, probe tunnels with arbitrary destination
586 * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port
587 * we've sent packets to won't necessarily match the local destination port.
588 *
589 * Then ask the tunnel implementation to match the error against a valid
590 * association.
591 *
592 * Return an error if we can't find a match, the socket if we need further
593 * processing, zero otherwise.
594 */
595static struct sock *__udp4_lib_err_encap(struct net *net,
596 const struct iphdr *iph,
597 struct udphdr *uh,
598 struct udp_table *udptable,
599 struct sk_buff *skb, u32 info)
600{
601 int network_offset, transport_offset;
602 struct sock *sk;
603
604 network_offset = skb_network_offset(skb);
605 transport_offset = skb_transport_offset(skb);
606
607 /* Network header needs to point to the outer IPv4 header inside ICMP */
608 skb_reset_network_header(skb);
609
610 /* Transport header needs to point to the UDP header */
611 skb_set_transport_header(skb, iph->ihl << 2);
612
613 sk = __udp4_lib_lookup(net, iph->daddr, uh->source,
614 iph->saddr, uh->dest, skb->dev->ifindex, 0,
615 udptable, NULL);
616 if (sk) {
617 int (*lookup)(struct sock *sk, struct sk_buff *skb);
618 struct udp_sock *up = udp_sk(sk);
619
620 lookup = READ_ONCE(up->encap_err_lookup);
621 if (!lookup || lookup(sk, skb))
622 sk = NULL;
623 }
624
625 if (!sk)
626 sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info));
627
628 skb_set_transport_header(skb, transport_offset);
629 skb_set_network_header(skb, network_offset);
630
631 return sk;
632}
633
634/*
635 * This routine is called by the ICMP module when it gets some
636 * sort of error condition. If err < 0 then the socket should
637 * be closed and the error returned to the user. If err > 0
638 * it's just the icmp type << 8 | icmp code.
639 * Header points to the ip header of the error packet. We move
640 * on past this. Then (as it used to claim before adjustment)
641 * header points to the first 8 bytes of the udp header. We need
642 * to find the appropriate port.
643 */
644
645int __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable)
646{
647 struct inet_sock *inet;
648 const struct iphdr *iph = (const struct iphdr *)skb->data;
649 struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2));
650 const int type = icmp_hdr(skb)->type;
651 const int code = icmp_hdr(skb)->code;
652 bool tunnel = false;
653 struct sock *sk;
654 int harderr;
655 int err;
656 struct net *net = dev_net(skb->dev);
657
658 sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
659 iph->saddr, uh->source, skb->dev->ifindex,
660 inet_sdif(skb), udptable, NULL);
661 if (!sk) {
662 /* No socket for error: try tunnels before discarding */
663 sk = ERR_PTR(-ENOENT);
664 if (static_branch_unlikely(&udp_encap_needed_key)) {
665 sk = __udp4_lib_err_encap(net, iph, uh, udptable, skb,
666 info);
667 if (!sk)
668 return 0;
669 }
670
671 if (IS_ERR(sk)) {
672 __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
673 return PTR_ERR(sk);
674 }
675
676 tunnel = true;
677 }
678
679 err = 0;
680 harderr = 0;
681 inet = inet_sk(sk);
682
683 switch (type) {
684 default:
685 case ICMP_TIME_EXCEEDED:
686 err = EHOSTUNREACH;
687 break;
688 case ICMP_SOURCE_QUENCH:
689 goto out;
690 case ICMP_PARAMETERPROB:
691 err = EPROTO;
692 harderr = 1;
693 break;
694 case ICMP_DEST_UNREACH:
695 if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */
696 ipv4_sk_update_pmtu(skb, sk, info);
697 if (inet->pmtudisc != IP_PMTUDISC_DONT) {
698 err = EMSGSIZE;
699 harderr = 1;
700 break;
701 }
702 goto out;
703 }
704 err = EHOSTUNREACH;
705 if (code <= NR_ICMP_UNREACH) {
706 harderr = icmp_err_convert[code].fatal;
707 err = icmp_err_convert[code].errno;
708 }
709 break;
710 case ICMP_REDIRECT:
711 ipv4_sk_redirect(skb, sk);
712 goto out;
713 }
714
715 /*
716 * RFC1122: OK. Passes ICMP errors back to application, as per
717 * 4.1.3.3.
718 */
719 if (tunnel) {
720 /* ...not for tunnels though: we don't have a sending socket */
721 goto out;
722 }
723 if (!inet->recverr) {
724 if (!harderr || sk->sk_state != TCP_ESTABLISHED)
725 goto out;
726 } else
727 ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1));
728
729 sk->sk_err = err;
730 sk->sk_error_report(sk);
731out:
732 return 0;
733}
734
735int udp_err(struct sk_buff *skb, u32 info)
736{
737 return __udp4_lib_err(skb, info, &udp_table);
738}
739
740/*
741 * Throw away all pending data and cancel the corking. Socket is locked.
742 */
743void udp_flush_pending_frames(struct sock *sk)
744{
745 struct udp_sock *up = udp_sk(sk);
746
747 if (up->pending) {
748 up->len = 0;
749 up->pending = 0;
750 ip_flush_pending_frames(sk);
751 }
752}
753EXPORT_SYMBOL(udp_flush_pending_frames);
754
755/**
756 * udp4_hwcsum - handle outgoing HW checksumming
757 * @skb: sk_buff containing the filled-in UDP header
758 * (checksum field must be zeroed out)
759 * @src: source IP address
760 * @dst: destination IP address
761 */
762void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
763{
764 struct udphdr *uh = udp_hdr(skb);
765 int offset = skb_transport_offset(skb);
766 int len = skb->len - offset;
767 int hlen = len;
768 __wsum csum = 0;
769
770 if (!skb_has_frag_list(skb)) {
771 /*
772 * Only one fragment on the socket.
773 */
774 skb->csum_start = skb_transport_header(skb) - skb->head;
775 skb->csum_offset = offsetof(struct udphdr, check);
776 uh->check = ~csum_tcpudp_magic(src, dst, len,
777 IPPROTO_UDP, 0);
778 } else {
779 struct sk_buff *frags;
780
781 /*
782 * HW-checksum won't work as there are two or more
783 * fragments on the socket so that all csums of sk_buffs
784 * should be together
785 */
786 skb_walk_frags(skb, frags) {
787 csum = csum_add(csum, frags->csum);
788 hlen -= frags->len;
789 }
790
791 csum = skb_checksum(skb, offset, hlen, csum);
792 skb->ip_summed = CHECKSUM_NONE;
793
794 uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
795 if (uh->check == 0)
796 uh->check = CSUM_MANGLED_0;
797 }
798}
799EXPORT_SYMBOL_GPL(udp4_hwcsum);
800
801/* Function to set UDP checksum for an IPv4 UDP packet. This is intended
802 * for the simple case like when setting the checksum for a UDP tunnel.
803 */
804void udp_set_csum(bool nocheck, struct sk_buff *skb,
805 __be32 saddr, __be32 daddr, int len)
806{
807 struct udphdr *uh = udp_hdr(skb);
808
809 if (nocheck) {
810 uh->check = 0;
811 } else if (skb_is_gso(skb)) {
812 uh->check = ~udp_v4_check(len, saddr, daddr, 0);
813 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
814 uh->check = 0;
815 uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb));
816 if (uh->check == 0)
817 uh->check = CSUM_MANGLED_0;
818 } else {
819 skb->ip_summed = CHECKSUM_PARTIAL;
820 skb->csum_start = skb_transport_header(skb) - skb->head;
821 skb->csum_offset = offsetof(struct udphdr, check);
822 uh->check = ~udp_v4_check(len, saddr, daddr, 0);
823 }
824}
825EXPORT_SYMBOL(udp_set_csum);
826
827static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,
828 struct inet_cork *cork)
829{
830 struct sock *sk = skb->sk;
831 struct inet_sock *inet = inet_sk(sk);
832 struct udphdr *uh;
833 int err = 0;
834 int is_udplite = IS_UDPLITE(sk);
835 int offset = skb_transport_offset(skb);
836 int len = skb->len - offset;
837 __wsum csum = 0;
838
839 /*
840 * Create a UDP header
841 */
842 uh = udp_hdr(skb);
843 uh->source = inet->inet_sport;
844 uh->dest = fl4->fl4_dport;
845 uh->len = htons(len);
846 uh->check = 0;
847
848 if (cork->gso_size) {
849 const int hlen = skb_network_header_len(skb) +
850 sizeof(struct udphdr);
851
852 if (hlen + cork->gso_size > cork->fragsize) {
853 kfree_skb(skb);
854 return -EINVAL;
855 }
856 if (skb->len > cork->gso_size * UDP_MAX_SEGMENTS) {
857 kfree_skb(skb);
858 return -EINVAL;
859 }
860 if (sk->sk_no_check_tx) {
861 kfree_skb(skb);
862 return -EINVAL;
863 }
864 if (skb->ip_summed != CHECKSUM_PARTIAL || is_udplite ||
865 dst_xfrm(skb_dst(skb))) {
866 kfree_skb(skb);
867 return -EIO;
868 }
869
870 skb_shinfo(skb)->gso_size = cork->gso_size;
871 skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
872 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(len - sizeof(uh),
873 cork->gso_size);
874 goto csum_partial;
875 }
876
877 if (is_udplite) /* UDP-Lite */
878 csum = udplite_csum(skb);
879
880 else if (sk->sk_no_check_tx) { /* UDP csum off */
881
882 skb->ip_summed = CHECKSUM_NONE;
883 goto send;
884
885 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
886csum_partial:
887
888 udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
889 goto send;
890
891 } else
892 csum = udp_csum(skb);
893
894 /* add protocol-dependent pseudo-header */
895 uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
896 sk->sk_protocol, csum);
897 if (uh->check == 0)
898 uh->check = CSUM_MANGLED_0;
899
900send:
901 err = ip_send_skb(sock_net(sk), skb);
902 if (err) {
903 if (err == -ENOBUFS && !inet->recverr) {
904 UDP_INC_STATS(sock_net(sk),
905 UDP_MIB_SNDBUFERRORS, is_udplite);
906 err = 0;
907 }
908 } else
909 UDP_INC_STATS(sock_net(sk),
910 UDP_MIB_OUTDATAGRAMS, is_udplite);
911 return err;
912}
913
914/*
915 * Push out all pending data as one UDP datagram. Socket is locked.
916 */
917int udp_push_pending_frames(struct sock *sk)
918{
919 struct udp_sock *up = udp_sk(sk);
920 struct inet_sock *inet = inet_sk(sk);
921 struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
922 struct sk_buff *skb;
923 int err = 0;
924
925 skb = ip_finish_skb(sk, fl4);
926 if (!skb)
927 goto out;
928
929 err = udp_send_skb(skb, fl4, &inet->cork.base);
930
931out:
932 up->len = 0;
933 up->pending = 0;
934 return err;
935}
936EXPORT_SYMBOL(udp_push_pending_frames);
937
938static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size)
939{
940 switch (cmsg->cmsg_type) {
941 case UDP_SEGMENT:
942 if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16)))
943 return -EINVAL;
944 *gso_size = *(__u16 *)CMSG_DATA(cmsg);
945 return 0;
946 default:
947 return -EINVAL;
948 }
949}
950
951int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size)
952{
953 struct cmsghdr *cmsg;
954 bool need_ip = false;
955 int err;
956
957 for_each_cmsghdr(cmsg, msg) {
958 if (!CMSG_OK(msg, cmsg))
959 return -EINVAL;
960
961 if (cmsg->cmsg_level != SOL_UDP) {
962 need_ip = true;
963 continue;
964 }
965
966 err = __udp_cmsg_send(cmsg, gso_size);
967 if (err)
968 return err;
969 }
970
971 return need_ip;
972}
973EXPORT_SYMBOL_GPL(udp_cmsg_send);
974
975int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
976{
977 struct inet_sock *inet = inet_sk(sk);
978 struct udp_sock *up = udp_sk(sk);
979 DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
980 struct flowi4 fl4_stack;
981 struct flowi4 *fl4;
982 int ulen = len;
983 struct ipcm_cookie ipc;
984 struct rtable *rt = NULL;
985 int free = 0;
986 int connected = 0;
987 __be32 daddr, faddr, saddr;
988 __be16 dport;
989 u8 tos;
990 int err, is_udplite = IS_UDPLITE(sk);
991 int corkreq = up->corkflag || msg->msg_flags&MSG_MORE;
992 int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
993 struct sk_buff *skb;
994 struct ip_options_data opt_copy;
995
996 if (len > 0xFFFF)
997 return -EMSGSIZE;
998
999 /*
1000 * Check the flags.
1001 */
1002
1003 if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
1004 return -EOPNOTSUPP;
1005
1006 getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
1007
1008 fl4 = &inet->cork.fl.u.ip4;
1009 if (up->pending) {
1010 /*
1011 * There are pending frames.
1012 * The socket lock must be held while it's corked.
1013 */
1014 lock_sock(sk);
1015 if (likely(up->pending)) {
1016 if (unlikely(up->pending != AF_INET)) {
1017 release_sock(sk);
1018 return -EINVAL;
1019 }
1020 goto do_append_data;
1021 }
1022 release_sock(sk);
1023 }
1024 ulen += sizeof(struct udphdr);
1025
1026 /*
1027 * Get and verify the address.
1028 */
1029 if (usin) {
1030 if (msg->msg_namelen < sizeof(*usin))
1031 return -EINVAL;
1032 if (usin->sin_family != AF_INET) {
1033 if (usin->sin_family != AF_UNSPEC)
1034 return -EAFNOSUPPORT;
1035 }
1036
1037 daddr = usin->sin_addr.s_addr;
1038 dport = usin->sin_port;
1039 if (dport == 0)
1040 return -EINVAL;
1041 } else {
1042 if (sk->sk_state != TCP_ESTABLISHED)
1043 return -EDESTADDRREQ;
1044 daddr = inet->inet_daddr;
1045 dport = inet->inet_dport;
1046 /* Open fast path for connected socket.
1047 Route will not be used, if at least one option is set.
1048 */
1049 connected = 1;
1050 }
1051
1052 ipcm_init_sk(&ipc, inet);
1053 ipc.gso_size = up->gso_size;
1054
1055 if (msg->msg_controllen) {
1056 err = udp_cmsg_send(sk, msg, &ipc.gso_size);
1057 if (err > 0)
1058 err = ip_cmsg_send(sk, msg, &ipc,
1059 sk->sk_family == AF_INET6);
1060 if (unlikely(err < 0)) {
1061 kfree(ipc.opt);
1062 return err;
1063 }
1064 if (ipc.opt)
1065 free = 1;
1066 connected = 0;
1067 }
1068 if (!ipc.opt) {
1069 struct ip_options_rcu *inet_opt;
1070
1071 rcu_read_lock();
1072 inet_opt = rcu_dereference(inet->inet_opt);
1073 if (inet_opt) {
1074 memcpy(&opt_copy, inet_opt,
1075 sizeof(*inet_opt) + inet_opt->opt.optlen);
1076 ipc.opt = &opt_copy.opt;
1077 }
1078 rcu_read_unlock();
1079 }
1080
1081 if (cgroup_bpf_enabled && !connected) {
1082 err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
1083 (struct sockaddr *)usin, &ipc.addr);
1084 if (err)
1085 goto out_free;
1086 if (usin) {
1087 if (usin->sin_port == 0) {
1088 /* BPF program set invalid port. Reject it. */
1089 err = -EINVAL;
1090 goto out_free;
1091 }
1092 daddr = usin->sin_addr.s_addr;
1093 dport = usin->sin_port;
1094 }
1095 }
1096
1097 saddr = ipc.addr;
1098 ipc.addr = faddr = daddr;
1099
1100 if (ipc.opt && ipc.opt->opt.srr) {
1101 if (!daddr) {
1102 err = -EINVAL;
1103 goto out_free;
1104 }
1105 faddr = ipc.opt->opt.faddr;
1106 connected = 0;
1107 }
1108 tos = get_rttos(&ipc, inet);
1109 if (sock_flag(sk, SOCK_LOCALROUTE) ||
1110 (msg->msg_flags & MSG_DONTROUTE) ||
1111 (ipc.opt && ipc.opt->opt.is_strictroute)) {
1112 tos |= RTO_ONLINK;
1113 connected = 0;
1114 }
1115
1116 if (ipv4_is_multicast(daddr)) {
1117 if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif))
1118 ipc.oif = inet->mc_index;
1119 if (!saddr)
1120 saddr = inet->mc_addr;
1121 connected = 0;
1122 } else if (!ipc.oif) {
1123 ipc.oif = inet->uc_index;
1124 } else if (ipv4_is_lbcast(daddr) && inet->uc_index) {
1125 /* oif is set, packet is to local broadcast and
1126 * and uc_index is set. oif is most likely set
1127 * by sk_bound_dev_if. If uc_index != oif check if the
1128 * oif is an L3 master and uc_index is an L3 slave.
1129 * If so, we want to allow the send using the uc_index.
1130 */
1131 if (ipc.oif != inet->uc_index &&
1132 ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk),
1133 inet->uc_index)) {
1134 ipc.oif = inet->uc_index;
1135 }
1136 }
1137
1138 if (connected)
1139 rt = (struct rtable *)sk_dst_check(sk, 0);
1140
1141 if (!rt) {
1142 struct net *net = sock_net(sk);
1143 __u8 flow_flags = inet_sk_flowi_flags(sk);
1144
1145 fl4 = &fl4_stack;
1146
1147 flowi4_init_output(fl4, ipc.oif, sk->sk_mark, tos,
1148 RT_SCOPE_UNIVERSE, sk->sk_protocol,
1149 flow_flags,
1150 faddr, saddr, dport, inet->inet_sport,
1151 sk->sk_uid);
1152
1153 security_sk_classify_flow(sk, flowi4_to_flowi(fl4));
1154 rt = ip_route_output_flow(net, fl4, sk);
1155 if (IS_ERR(rt)) {
1156 err = PTR_ERR(rt);
1157 rt = NULL;
1158 if (err == -ENETUNREACH)
1159 IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
1160 goto out;
1161 }
1162
1163 err = -EACCES;
1164 if ((rt->rt_flags & RTCF_BROADCAST) &&
1165 !sock_flag(sk, SOCK_BROADCAST))
1166 goto out;
1167 if (connected)
1168 sk_dst_set(sk, dst_clone(&rt->dst));
1169 }
1170
1171 if (msg->msg_flags&MSG_CONFIRM)
1172 goto do_confirm;
1173back_from_confirm:
1174
1175 saddr = fl4->saddr;
1176 if (!ipc.addr)
1177 daddr = ipc.addr = fl4->daddr;
1178
1179 /* Lockless fast path for the non-corking case. */
1180 if (!corkreq) {
1181 struct inet_cork cork;
1182
1183 skb = ip_make_skb(sk, fl4, getfrag, msg, ulen,
1184 sizeof(struct udphdr), &ipc, &rt,
1185 &cork, msg->msg_flags);
1186 err = PTR_ERR(skb);
1187 if (!IS_ERR_OR_NULL(skb))
1188 err = udp_send_skb(skb, fl4, &cork);
1189 goto out;
1190 }
1191
1192 lock_sock(sk);
1193 if (unlikely(up->pending)) {
1194 /* The socket is already corked while preparing it. */
1195 /* ... which is an evident application bug. --ANK */
1196 release_sock(sk);
1197
1198 net_dbg_ratelimited("socket already corked\n");
1199 err = -EINVAL;
1200 goto out;
1201 }
1202 /*
1203 * Now cork the socket to pend data.
1204 */
1205 fl4 = &inet->cork.fl.u.ip4;
1206 fl4->daddr = daddr;
1207 fl4->saddr = saddr;
1208 fl4->fl4_dport = dport;
1209 fl4->fl4_sport = inet->inet_sport;
1210 up->pending = AF_INET;
1211
1212do_append_data:
1213 up->len += ulen;
1214 err = ip_append_data(sk, fl4, getfrag, msg, ulen,
1215 sizeof(struct udphdr), &ipc, &rt,
1216 corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
1217 if (err)
1218 udp_flush_pending_frames(sk);
1219 else if (!corkreq)
1220 err = udp_push_pending_frames(sk);
1221 else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
1222 up->pending = 0;
1223 release_sock(sk);
1224
1225out:
1226 ip_rt_put(rt);
1227out_free:
1228 if (free)
1229 kfree(ipc.opt);
1230 if (!err)
1231 return len;
1232 /*
1233 * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting
1234 * ENOBUFS might not be good (it's not tunable per se), but otherwise
1235 * we don't have a good statistic (IpOutDiscards but it can be too many
1236 * things). We could add another new stat but at least for now that
1237 * seems like overkill.
1238 */
1239 if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1240 UDP_INC_STATS(sock_net(sk),
1241 UDP_MIB_SNDBUFERRORS, is_udplite);
1242 }
1243 return err;
1244
1245do_confirm:
1246 if (msg->msg_flags & MSG_PROBE)
1247 dst_confirm_neigh(&rt->dst, &fl4->daddr);
1248 if (!(msg->msg_flags&MSG_PROBE) || len)
1249 goto back_from_confirm;
1250 err = 0;
1251 goto out;
1252}
1253EXPORT_SYMBOL(udp_sendmsg);
1254
1255int udp_sendpage(struct sock *sk, struct page *page, int offset,
1256 size_t size, int flags)
1257{
1258 struct inet_sock *inet = inet_sk(sk);
1259 struct udp_sock *up = udp_sk(sk);
1260 int ret;
1261
1262 if (flags & MSG_SENDPAGE_NOTLAST)
1263 flags |= MSG_MORE;
1264
1265 if (!up->pending) {
1266 struct msghdr msg = { .msg_flags = flags|MSG_MORE };
1267
1268 /* Call udp_sendmsg to specify destination address which
1269 * sendpage interface can't pass.
1270 * This will succeed only when the socket is connected.
1271 */
1272 ret = udp_sendmsg(sk, &msg, 0);
1273 if (ret < 0)
1274 return ret;
1275 }
1276
1277 lock_sock(sk);
1278
1279 if (unlikely(!up->pending)) {
1280 release_sock(sk);
1281
1282 net_dbg_ratelimited("cork failed\n");
1283 return -EINVAL;
1284 }
1285
1286 ret = ip_append_page(sk, &inet->cork.fl.u.ip4,
1287 page, offset, size, flags);
1288 if (ret == -EOPNOTSUPP) {
1289 release_sock(sk);
1290 return sock_no_sendpage(sk->sk_socket, page, offset,
1291 size, flags);
1292 }
1293 if (ret < 0) {
1294 udp_flush_pending_frames(sk);
1295 goto out;
1296 }
1297
1298 up->len += size;
1299 if (!(up->corkflag || (flags&MSG_MORE)))
1300 ret = udp_push_pending_frames(sk);
1301 if (!ret)
1302 ret = size;
1303out:
1304 release_sock(sk);
1305 return ret;
1306}
1307
1308#define UDP_SKB_IS_STATELESS 0x80000000
1309
1310static void udp_set_dev_scratch(struct sk_buff *skb)
1311{
1312 struct udp_dev_scratch *scratch = udp_skb_scratch(skb);
1313
1314 BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long));
1315 scratch->_tsize_state = skb->truesize;
1316#if BITS_PER_LONG == 64
1317 scratch->len = skb->len;
1318 scratch->csum_unnecessary = !!skb_csum_unnecessary(skb);
1319 scratch->is_linear = !skb_is_nonlinear(skb);
1320#endif
1321 /* all head states execept sp (dst, sk, nf) are always cleared by
1322 * udp_rcv() and we need to preserve secpath, if present, to eventually
1323 * process IP_CMSG_PASSSEC at recvmsg() time
1324 */
1325 if (likely(!skb_sec_path(skb)))
1326 scratch->_tsize_state |= UDP_SKB_IS_STATELESS;
1327}
1328
1329static int udp_skb_truesize(struct sk_buff *skb)
1330{
1331 return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS;
1332}
1333
1334static bool udp_skb_has_head_state(struct sk_buff *skb)
1335{
1336 return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS);
1337}
1338
1339/* fully reclaim rmem/fwd memory allocated for skb */
1340static void udp_rmem_release(struct sock *sk, int size, int partial,
1341 bool rx_queue_lock_held)
1342{
1343 struct udp_sock *up = udp_sk(sk);
1344 struct sk_buff_head *sk_queue;
1345 int amt;
1346
1347 if (likely(partial)) {
1348 up->forward_deficit += size;
1349 size = up->forward_deficit;
1350 if (size < (sk->sk_rcvbuf >> 2))
1351 return;
1352 } else {
1353 size += up->forward_deficit;
1354 }
1355 up->forward_deficit = 0;
1356
1357 /* acquire the sk_receive_queue for fwd allocated memory scheduling,
1358 * if the called don't held it already
1359 */
1360 sk_queue = &sk->sk_receive_queue;
1361 if (!rx_queue_lock_held)
1362 spin_lock(&sk_queue->lock);
1363
1364
1365 sk->sk_forward_alloc += size;
1366 amt = (sk->sk_forward_alloc - partial) & ~(SK_MEM_QUANTUM - 1);
1367 sk->sk_forward_alloc -= amt;
1368
1369 if (amt)
1370 __sk_mem_reduce_allocated(sk, amt >> SK_MEM_QUANTUM_SHIFT);
1371
1372 atomic_sub(size, &sk->sk_rmem_alloc);
1373
1374 /* this can save us from acquiring the rx queue lock on next receive */
1375 skb_queue_splice_tail_init(sk_queue, &up->reader_queue);
1376
1377 if (!rx_queue_lock_held)
1378 spin_unlock(&sk_queue->lock);
1379}
1380
1381/* Note: called with reader_queue.lock held.
1382 * Instead of using skb->truesize here, find a copy of it in skb->dev_scratch
1383 * This avoids a cache line miss while receive_queue lock is held.
1384 * Look at __udp_enqueue_schedule_skb() to find where this copy is done.
1385 */
1386void udp_skb_destructor(struct sock *sk, struct sk_buff *skb)
1387{
1388 prefetch(&skb->data);
1389 udp_rmem_release(sk, udp_skb_truesize(skb), 1, false);
1390}
1391EXPORT_SYMBOL(udp_skb_destructor);
1392
1393/* as above, but the caller held the rx queue lock, too */
1394static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb)
1395{
1396 prefetch(&skb->data);
1397 udp_rmem_release(sk, udp_skb_truesize(skb), 1, true);
1398}
1399
1400/* Idea of busylocks is to let producers grab an extra spinlock
1401 * to relieve pressure on the receive_queue spinlock shared by consumer.
1402 * Under flood, this means that only one producer can be in line
1403 * trying to acquire the receive_queue spinlock.
1404 * These busylock can be allocated on a per cpu manner, instead of a
1405 * per socket one (that would consume a cache line per socket)
1406 */
1407static int udp_busylocks_log __read_mostly;
1408static spinlock_t *udp_busylocks __read_mostly;
1409
1410static spinlock_t *busylock_acquire(void *ptr)
1411{
1412 spinlock_t *busy;
1413
1414 busy = udp_busylocks + hash_ptr(ptr, udp_busylocks_log);
1415 spin_lock(busy);
1416 return busy;
1417}
1418
1419static void busylock_release(spinlock_t *busy)
1420{
1421 if (busy)
1422 spin_unlock(busy);
1423}
1424
1425int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
1426{
1427 struct sk_buff_head *list = &sk->sk_receive_queue;
1428 int rmem, delta, amt, err = -ENOMEM;
1429 spinlock_t *busy = NULL;
1430 int size;
1431
1432 /* try to avoid the costly atomic add/sub pair when the receive
1433 * queue is full; always allow at least a packet
1434 */
1435 rmem = atomic_read(&sk->sk_rmem_alloc);
1436 if (rmem > sk->sk_rcvbuf)
1437 goto drop;
1438
1439 /* Under mem pressure, it might be helpful to help udp_recvmsg()
1440 * having linear skbs :
1441 * - Reduce memory overhead and thus increase receive queue capacity
1442 * - Less cache line misses at copyout() time
1443 * - Less work at consume_skb() (less alien page frag freeing)
1444 */
1445 if (rmem > (sk->sk_rcvbuf >> 1)) {
1446 skb_condense(skb);
1447
1448 busy = busylock_acquire(sk);
1449 }
1450 size = skb->truesize;
1451 udp_set_dev_scratch(skb);
1452
1453 /* we drop only if the receive buf is full and the receive
1454 * queue contains some other skb
1455 */
1456 rmem = atomic_add_return(size, &sk->sk_rmem_alloc);
1457 if (rmem > (size + sk->sk_rcvbuf))
1458 goto uncharge_drop;
1459
1460 spin_lock(&list->lock);
1461 if (size >= sk->sk_forward_alloc) {
1462 amt = sk_mem_pages(size);
1463 delta = amt << SK_MEM_QUANTUM_SHIFT;
1464 if (!__sk_mem_raise_allocated(sk, delta, amt, SK_MEM_RECV)) {
1465 err = -ENOBUFS;
1466 spin_unlock(&list->lock);
1467 goto uncharge_drop;
1468 }
1469
1470 sk->sk_forward_alloc += delta;
1471 }
1472
1473 sk->sk_forward_alloc -= size;
1474
1475 /* no need to setup a destructor, we will explicitly release the
1476 * forward allocated memory on dequeue
1477 */
1478 sock_skb_set_dropcount(sk, skb);
1479
1480 __skb_queue_tail(list, skb);
1481 spin_unlock(&list->lock);
1482
1483 if (!sock_flag(sk, SOCK_DEAD))
1484 sk->sk_data_ready(sk);
1485
1486 busylock_release(busy);
1487 return 0;
1488
1489uncharge_drop:
1490 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
1491
1492drop:
1493 atomic_inc(&sk->sk_drops);
1494 busylock_release(busy);
1495 return err;
1496}
1497EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb);
1498
1499void udp_destruct_sock(struct sock *sk)
1500{
1501 /* reclaim completely the forward allocated memory */
1502 struct udp_sock *up = udp_sk(sk);
1503 unsigned int total = 0;
1504 struct sk_buff *skb;
1505
1506 skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue);
1507 while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) {
1508 total += skb->truesize;
1509 kfree_skb(skb);
1510 }
1511 udp_rmem_release(sk, total, 0, true);
1512
1513 inet_sock_destruct(sk);
1514}
1515EXPORT_SYMBOL_GPL(udp_destruct_sock);
1516
1517int udp_init_sock(struct sock *sk)
1518{
1519 skb_queue_head_init(&udp_sk(sk)->reader_queue);
1520 sk->sk_destruct = udp_destruct_sock;
1521 return 0;
1522}
1523EXPORT_SYMBOL_GPL(udp_init_sock);
1524
1525void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
1526{
1527 if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) {
1528 bool slow = lock_sock_fast(sk);
1529
1530 sk_peek_offset_bwd(sk, len);
1531 unlock_sock_fast(sk, slow);
1532 }
1533
1534 if (!skb_unref(skb))
1535 return;
1536
1537 /* In the more common cases we cleared the head states previously,
1538 * see __udp_queue_rcv_skb().
1539 */
1540 if (unlikely(udp_skb_has_head_state(skb)))
1541 skb_release_head_state(skb);
1542 __consume_stateless_skb(skb);
1543}
1544EXPORT_SYMBOL_GPL(skb_consume_udp);
1545
1546static struct sk_buff *__first_packet_length(struct sock *sk,
1547 struct sk_buff_head *rcvq,
1548 int *total)
1549{
1550 struct sk_buff *skb;
1551
1552 while ((skb = skb_peek(rcvq)) != NULL) {
1553 if (udp_lib_checksum_complete(skb)) {
1554 __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS,
1555 IS_UDPLITE(sk));
1556 __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS,
1557 IS_UDPLITE(sk));
1558 atomic_inc(&sk->sk_drops);
1559 __skb_unlink(skb, rcvq);
1560 *total += skb->truesize;
1561 kfree_skb(skb);
1562 } else {
1563 /* the csum related bits could be changed, refresh
1564 * the scratch area
1565 */
1566 udp_set_dev_scratch(skb);
1567 break;
1568 }
1569 }
1570 return skb;
1571}
1572
1573/**
1574 * first_packet_length - return length of first packet in receive queue
1575 * @sk: socket
1576 *
1577 * Drops all bad checksum frames, until a valid one is found.
1578 * Returns the length of found skb, or -1 if none is found.
1579 */
1580static int first_packet_length(struct sock *sk)
1581{
1582 struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue;
1583 struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
1584 struct sk_buff *skb;
1585 int total = 0;
1586 int res;
1587
1588 spin_lock_bh(&rcvq->lock);
1589 skb = __first_packet_length(sk, rcvq, &total);
1590 if (!skb && !skb_queue_empty(sk_queue)) {
1591 spin_lock(&sk_queue->lock);
1592 skb_queue_splice_tail_init(sk_queue, rcvq);
1593 spin_unlock(&sk_queue->lock);
1594
1595 skb = __first_packet_length(sk, rcvq, &total);
1596 }
1597 res = skb ? skb->len : -1;
1598 if (total)
1599 udp_rmem_release(sk, total, 1, false);
1600 spin_unlock_bh(&rcvq->lock);
1601 return res;
1602}
1603
1604/*
1605 * IOCTL requests applicable to the UDP protocol
1606 */
1607
1608int udp_ioctl(struct sock *sk, int cmd, unsigned long arg)
1609{
1610 switch (cmd) {
1611 case SIOCOUTQ:
1612 {
1613 int amount = sk_wmem_alloc_get(sk);
1614
1615 return put_user(amount, (int __user *)arg);
1616 }
1617
1618 case SIOCINQ:
1619 {
1620 int amount = max_t(int, 0, first_packet_length(sk));
1621
1622 return put_user(amount, (int __user *)arg);
1623 }
1624
1625 default:
1626 return -ENOIOCTLCMD;
1627 }
1628
1629 return 0;
1630}
1631EXPORT_SYMBOL(udp_ioctl);
1632
1633struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags,
1634 int noblock, int *peeked, int *off, int *err)
1635{
1636 struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
1637 struct sk_buff_head *queue;
1638 struct sk_buff *last;
1639 long timeo;
1640 int error;
1641
1642 queue = &udp_sk(sk)->reader_queue;
1643 flags |= noblock ? MSG_DONTWAIT : 0;
1644 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1645 do {
1646 struct sk_buff *skb;
1647
1648 error = sock_error(sk);
1649 if (error)
1650 break;
1651
1652 error = -EAGAIN;
1653 *peeked = 0;
1654 do {
1655 spin_lock_bh(&queue->lock);
1656 skb = __skb_try_recv_from_queue(sk, queue, flags,
1657 udp_skb_destructor,
1658 peeked, off, err,
1659 &last);
1660 if (skb) {
1661 spin_unlock_bh(&queue->lock);
1662 return skb;
1663 }
1664
1665 if (skb_queue_empty(sk_queue)) {
1666 spin_unlock_bh(&queue->lock);
1667 goto busy_check;
1668 }
1669
1670 /* refill the reader queue and walk it again
1671 * keep both queues locked to avoid re-acquiring
1672 * the sk_receive_queue lock if fwd memory scheduling
1673 * is needed.
1674 */
1675 spin_lock(&sk_queue->lock);
1676 skb_queue_splice_tail_init(sk_queue, queue);
1677
1678 skb = __skb_try_recv_from_queue(sk, queue, flags,
1679 udp_skb_dtor_locked,
1680 peeked, off, err,
1681 &last);
1682 spin_unlock(&sk_queue->lock);
1683 spin_unlock_bh(&queue->lock);
1684 if (skb)
1685 return skb;
1686
1687busy_check:
1688 if (!sk_can_busy_loop(sk))
1689 break;
1690
1691 sk_busy_loop(sk, flags & MSG_DONTWAIT);
1692 } while (!skb_queue_empty(sk_queue));
1693
1694 /* sk_queue is empty, reader_queue may contain peeked packets */
1695 } while (timeo &&
1696 !__skb_wait_for_more_packets(sk, &error, &timeo,
1697 (struct sk_buff *)sk_queue));
1698
1699 *err = error;
1700 return NULL;
1701}
1702EXPORT_SYMBOL(__skb_recv_udp);
1703
1704/*
1705 * This should be easy, if there is something there we
1706 * return it, otherwise we block.
1707 */
1708
1709int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock,
1710 int flags, int *addr_len)
1711{
1712 struct inet_sock *inet = inet_sk(sk);
1713 DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
1714 struct sk_buff *skb;
1715 unsigned int ulen, copied;
1716 int peeked, peeking, off;
1717 int err;
1718 int is_udplite = IS_UDPLITE(sk);
1719 bool checksum_valid = false;
1720
1721 if (flags & MSG_ERRQUEUE)
1722 return ip_recv_error(sk, msg, len, addr_len);
1723
1724try_again:
1725 peeking = flags & MSG_PEEK;
1726 off = sk_peek_offset(sk, flags);
1727 skb = __skb_recv_udp(sk, flags, noblock, &peeked, &off, &err);
1728 if (!skb)
1729 return err;
1730
1731 ulen = udp_skb_len(skb);
1732 copied = len;
1733 if (copied > ulen - off)
1734 copied = ulen - off;
1735 else if (copied < ulen)
1736 msg->msg_flags |= MSG_TRUNC;
1737
1738 /*
1739 * If checksum is needed at all, try to do it while copying the
1740 * data. If the data is truncated, or if we only want a partial
1741 * coverage checksum (UDP-Lite), do it before the copy.
1742 */
1743
1744 if (copied < ulen || peeking ||
1745 (is_udplite && UDP_SKB_CB(skb)->partial_cov)) {
1746 checksum_valid = udp_skb_csum_unnecessary(skb) ||
1747 !__udp_lib_checksum_complete(skb);
1748 if (!checksum_valid)
1749 goto csum_copy_err;
1750 }
1751
1752 if (checksum_valid || udp_skb_csum_unnecessary(skb)) {
1753 if (udp_skb_is_linear(skb))
1754 err = copy_linear_skb(skb, copied, off, &msg->msg_iter);
1755 else
1756 err = skb_copy_datagram_msg(skb, off, msg, copied);
1757 } else {
1758 err = skb_copy_and_csum_datagram_msg(skb, off, msg);
1759
1760 if (err == -EINVAL)
1761 goto csum_copy_err;
1762 }
1763
1764 if (unlikely(err)) {
1765 if (!peeked) {
1766 atomic_inc(&sk->sk_drops);
1767 UDP_INC_STATS(sock_net(sk),
1768 UDP_MIB_INERRORS, is_udplite);
1769 }
1770 kfree_skb(skb);
1771 return err;
1772 }
1773
1774 if (!peeked)
1775 UDP_INC_STATS(sock_net(sk),
1776 UDP_MIB_INDATAGRAMS, is_udplite);
1777
1778 sock_recv_ts_and_drops(msg, sk, skb);
1779
1780 /* Copy the address. */
1781 if (sin) {
1782 sin->sin_family = AF_INET;
1783 sin->sin_port = udp_hdr(skb)->source;
1784 sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
1785 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
1786 *addr_len = sizeof(*sin);
1787 }
1788
1789 if (udp_sk(sk)->gro_enabled)
1790 udp_cmsg_recv(msg, sk, skb);
1791
1792 if (inet->cmsg_flags)
1793 ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off);
1794
1795 err = copied;
1796 if (flags & MSG_TRUNC)
1797 err = ulen;
1798
1799 skb_consume_udp(sk, skb, peeking ? -err : err);
1800 return err;
1801
1802csum_copy_err:
1803 if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags,
1804 udp_skb_destructor)) {
1805 UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
1806 UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
1807 }
1808 kfree_skb(skb);
1809
1810 /* starting over for a new packet, but check if we need to yield */
1811 cond_resched();
1812 msg->msg_flags &= ~MSG_TRUNC;
1813 goto try_again;
1814}
1815
1816int udp_pre_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
1817{
1818 /* This check is replicated from __ip4_datagram_connect() and
1819 * intended to prevent BPF program called below from accessing bytes
1820 * that are out of the bound specified by user in addr_len.
1821 */
1822 if (addr_len < sizeof(struct sockaddr_in))
1823 return -EINVAL;
1824
1825 return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr);
1826}
1827EXPORT_SYMBOL(udp_pre_connect);
1828
1829int __udp_disconnect(struct sock *sk, int flags)
1830{
1831 struct inet_sock *inet = inet_sk(sk);
1832 /*
1833 * 1003.1g - break association.
1834 */
1835
1836 sk->sk_state = TCP_CLOSE;
1837 inet->inet_daddr = 0;
1838 inet->inet_dport = 0;
1839 sock_rps_reset_rxhash(sk);
1840 sk->sk_bound_dev_if = 0;
1841 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
1842 inet_reset_saddr(sk);
1843
1844 if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
1845 sk->sk_prot->unhash(sk);
1846 inet->inet_sport = 0;
1847 }
1848 sk_dst_reset(sk);
1849 return 0;
1850}
1851EXPORT_SYMBOL(__udp_disconnect);
1852
1853int udp_disconnect(struct sock *sk, int flags)
1854{
1855 lock_sock(sk);
1856 __udp_disconnect(sk, flags);
1857 release_sock(sk);
1858 return 0;
1859}
1860EXPORT_SYMBOL(udp_disconnect);
1861
1862void udp_lib_unhash(struct sock *sk)
1863{
1864 if (sk_hashed(sk)) {
1865 struct udp_table *udptable = sk->sk_prot->h.udp_table;
1866 struct udp_hslot *hslot, *hslot2;
1867
1868 hslot = udp_hashslot(udptable, sock_net(sk),
1869 udp_sk(sk)->udp_port_hash);
1870 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
1871
1872 spin_lock_bh(&hslot->lock);
1873 if (rcu_access_pointer(sk->sk_reuseport_cb))
1874 reuseport_detach_sock(sk);
1875 if (sk_del_node_init_rcu(sk)) {
1876 hslot->count--;
1877 inet_sk(sk)->inet_num = 0;
1878 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
1879
1880 spin_lock(&hslot2->lock);
1881 hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
1882 hslot2->count--;
1883 spin_unlock(&hslot2->lock);
1884 }
1885 spin_unlock_bh(&hslot->lock);
1886 }
1887}
1888EXPORT_SYMBOL(udp_lib_unhash);
1889
1890/*
1891 * inet_rcv_saddr was changed, we must rehash secondary hash
1892 */
1893void udp_lib_rehash(struct sock *sk, u16 newhash)
1894{
1895 if (sk_hashed(sk)) {
1896 struct udp_table *udptable = sk->sk_prot->h.udp_table;
1897 struct udp_hslot *hslot, *hslot2, *nhslot2;
1898
1899 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
1900 nhslot2 = udp_hashslot2(udptable, newhash);
1901 udp_sk(sk)->udp_portaddr_hash = newhash;
1902
1903 if (hslot2 != nhslot2 ||
1904 rcu_access_pointer(sk->sk_reuseport_cb)) {
1905 hslot = udp_hashslot(udptable, sock_net(sk),
1906 udp_sk(sk)->udp_port_hash);
1907 /* we must lock primary chain too */
1908 spin_lock_bh(&hslot->lock);
1909 if (rcu_access_pointer(sk->sk_reuseport_cb))
1910 reuseport_detach_sock(sk);
1911
1912 if (hslot2 != nhslot2) {
1913 spin_lock(&hslot2->lock);
1914 hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
1915 hslot2->count--;
1916 spin_unlock(&hslot2->lock);
1917
1918 spin_lock(&nhslot2->lock);
1919 hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
1920 &nhslot2->head);
1921 nhslot2->count++;
1922 spin_unlock(&nhslot2->lock);
1923 }
1924
1925 spin_unlock_bh(&hslot->lock);
1926 }
1927 }
1928}
1929EXPORT_SYMBOL(udp_lib_rehash);
1930
1931void udp_v4_rehash(struct sock *sk)
1932{
1933 u16 new_hash = ipv4_portaddr_hash(sock_net(sk),
1934 inet_sk(sk)->inet_rcv_saddr,
1935 inet_sk(sk)->inet_num);
1936 udp_lib_rehash(sk, new_hash);
1937}
1938
1939static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
1940{
1941 int rc;
1942
1943 if (inet_sk(sk)->inet_daddr) {
1944 sock_rps_save_rxhash(sk, skb);
1945 sk_mark_napi_id(sk, skb);
1946 sk_incoming_cpu_update(sk);
1947 } else {
1948 sk_mark_napi_id_once(sk, skb);
1949 }
1950
1951 rc = __udp_enqueue_schedule_skb(sk, skb);
1952 if (rc < 0) {
1953 int is_udplite = IS_UDPLITE(sk);
1954
1955 /* Note that an ENOMEM error is charged twice */
1956 if (rc == -ENOMEM)
1957 UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS,
1958 is_udplite);
1959 UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
1960 kfree_skb(skb);
1961 trace_udp_fail_queue_rcv_skb(rc, sk);
1962 return -1;
1963 }
1964
1965 return 0;
1966}
1967
1968/* returns:
1969 * -1: error
1970 * 0: success
1971 * >0: "udp encap" protocol resubmission
1972 *
1973 * Note that in the success and error cases, the skb is assumed to
1974 * have either been requeued or freed.
1975 */
1976static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
1977{
1978 struct udp_sock *up = udp_sk(sk);
1979 int is_udplite = IS_UDPLITE(sk);
1980
1981 /*
1982 * Charge it to the socket, dropping if the queue is full.
1983 */
1984 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
1985 goto drop;
1986 nf_reset(skb);
1987
1988 if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) {
1989 int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
1990
1991 /*
1992 * This is an encapsulation socket so pass the skb to
1993 * the socket's udp_encap_rcv() hook. Otherwise, just
1994 * fall through and pass this up the UDP socket.
1995 * up->encap_rcv() returns the following value:
1996 * =0 if skb was successfully passed to the encap
1997 * handler or was discarded by it.
1998 * >0 if skb should be passed on to UDP.
1999 * <0 if skb should be resubmitted as proto -N
2000 */
2001
2002 /* if we're overly short, let UDP handle it */
2003 encap_rcv = READ_ONCE(up->encap_rcv);
2004 if (encap_rcv) {
2005 int ret;
2006
2007 /* Verify checksum before giving to encap */
2008 if (udp_lib_checksum_complete(skb))
2009 goto csum_error;
2010
2011 ret = encap_rcv(sk, skb);
2012 if (ret <= 0) {
2013 __UDP_INC_STATS(sock_net(sk),
2014 UDP_MIB_INDATAGRAMS,
2015 is_udplite);
2016 return -ret;
2017 }
2018 }
2019
2020 /* FALLTHROUGH -- it's a UDP Packet */
2021 }
2022
2023 /*
2024 * UDP-Lite specific tests, ignored on UDP sockets
2025 */
2026 if ((is_udplite & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) {
2027
2028 /*
2029 * MIB statistics other than incrementing the error count are
2030 * disabled for the following two types of errors: these depend
2031 * on the application settings, not on the functioning of the
2032 * protocol stack as such.
2033 *
2034 * RFC 3828 here recommends (sec 3.3): "There should also be a
2035 * way ... to ... at least let the receiving application block
2036 * delivery of packets with coverage values less than a value
2037 * provided by the application."
2038 */
2039 if (up->pcrlen == 0) { /* full coverage was set */
2040 net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested\n",
2041 UDP_SKB_CB(skb)->cscov, skb->len);
2042 goto drop;
2043 }
2044 /* The next case involves violating the min. coverage requested
2045 * by the receiver. This is subtle: if receiver wants x and x is
2046 * greater than the buffersize/MTU then receiver will complain
2047 * that it wants x while sender emits packets of smaller size y.
2048 * Therefore the above ...()->partial_cov statement is essential.
2049 */
2050 if (UDP_SKB_CB(skb)->cscov < up->pcrlen) {
2051 net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d\n",
2052 UDP_SKB_CB(skb)->cscov, up->pcrlen);
2053 goto drop;
2054 }
2055 }
2056
2057 prefetch(&sk->sk_rmem_alloc);
2058 if (rcu_access_pointer(sk->sk_filter) &&
2059 udp_lib_checksum_complete(skb))
2060 goto csum_error;
2061
2062 if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr)))
2063 goto drop;
2064
2065 udp_csum_pull_header(skb);
2066
2067 ipv4_pktinfo_prepare(sk, skb);
2068 return __udp_queue_rcv_skb(sk, skb);
2069
2070csum_error:
2071 __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
2072drop:
2073 __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
2074 atomic_inc(&sk->sk_drops);
2075 kfree_skb(skb);
2076 return -1;
2077}
2078
2079static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
2080{
2081 struct sk_buff *next, *segs;
2082 int ret;
2083
2084 if (likely(!udp_unexpected_gso(sk, skb)))
2085 return udp_queue_rcv_one_skb(sk, skb);
2086
2087 BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_SGO_CB_OFFSET);
2088 __skb_push(skb, -skb_mac_offset(skb));
2089 segs = udp_rcv_segment(sk, skb, true);
2090 for (skb = segs; skb; skb = next) {
2091 next = skb->next;
2092 __skb_pull(skb, skb_transport_offset(skb));
2093 ret = udp_queue_rcv_one_skb(sk, skb);
2094 if (ret > 0)
2095 ip_protocol_deliver_rcu(dev_net(skb->dev), skb, -ret);
2096 }
2097 return 0;
2098}
2099
2100/* For TCP sockets, sk_rx_dst is protected by socket lock
2101 * For UDP, we use xchg() to guard against concurrent changes.
2102 */
2103bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
2104{
2105 struct dst_entry *old;
2106
2107 if (dst_hold_safe(dst)) {
2108 old = xchg(&sk->sk_rx_dst, dst);
2109 dst_release(old);
2110 return old != dst;
2111 }
2112 return false;
2113}
2114EXPORT_SYMBOL(udp_sk_rx_dst_set);
2115
2116/*
2117 * Multicasts and broadcasts go to each listener.
2118 *
2119 * Note: called only from the BH handler context.
2120 */
2121static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
2122 struct udphdr *uh,
2123 __be32 saddr, __be32 daddr,
2124 struct