1/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
10 * Authors: Ross Biro
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <linux/uaccess.h>
76#include <linux/bitops.h>
77#include <linux/capability.h>
78#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
81#include <linux/hash.h>
82#include <linux/slab.h>
83#include <linux/sched.h>
84#include <linux/sched/mm.h>
85#include <linux/mutex.h>
86#include <linux/string.h>
87#include <linux/mm.h>
88#include <linux/socket.h>
89#include <linux/sockios.h>
90#include <linux/errno.h>
91#include <linux/interrupt.h>
92#include <linux/if_ether.h>
93#include <linux/netdevice.h>
94#include <linux/etherdevice.h>
95#include <linux/ethtool.h>
96#include <linux/notifier.h>
97#include <linux/skbuff.h>
98#include <linux/bpf.h>
99#include <linux/bpf_trace.h>
100#include <net/net_namespace.h>
101#include <net/sock.h>
102#include <net/busy_poll.h>
103#include <linux/rtnetlink.h>
104#include <linux/stat.h>
105#include <net/dst.h>
106#include <net/dst_metadata.h>
107#include <net/pkt_sched.h>
108#include <net/checksum.h>
109#include <net/xfrm.h>
110#include <linux/highmem.h>
111#include <linux/init.h>
112#include <linux/module.h>
113#include <linux/netpoll.h>
114#include <linux/rcupdate.h>
115#include <linux/delay.h>
116#include <net/iw_handler.h>
117#include <asm/current.h>
118#include <linux/audit.h>
119#include <linux/dmaengine.h>
120#include <linux/err.h>
121#include <linux/ctype.h>
122#include <linux/if_arp.h>
123#include <linux/if_vlan.h>
124#include <linux/ip.h>
125#include <net/ip.h>
126#include <net/mpls.h>
127#include <linux/ipv6.h>
128#include <linux/in.h>
129#include <linux/jhash.h>
130#include <linux/random.h>
131#include <trace/events/napi.h>
132#include <trace/events/net.h>
133#include <trace/events/skb.h>
134#include <linux/pci.h>
135#include <linux/inetdevice.h>
136#include <linux/cpu_rmap.h>
137#include <linux/static_key.h>
138#include <linux/hashtable.h>
139#include <linux/vmalloc.h>
140#include <linux/if_macvlan.h>
141#include <linux/errqueue.h>
142#include <linux/hrtimer.h>
143#include <linux/netfilter_ingress.h>
144#include <linux/crash_dump.h>
145
146#include "net-sysfs.h"
147
148/* Instead of increasing this, you should create a hash table. */
149#define MAX_GRO_SKBS 8
150
151/* This should be increased if a protocol with a bigger head is added. */
152#define GRO_MAX_HEAD (MAX_HEADER + 128)
153
154static DEFINE_SPINLOCK(ptype_lock);
155static DEFINE_SPINLOCK(offload_lock);
156struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
157struct list_head ptype_all __read_mostly; /* Taps */
158static struct list_head offload_base __read_mostly;
159
160static int netif_rx_internal(struct sk_buff *skb);
161static int call_netdevice_notifiers_info(unsigned long val,
162 struct net_device *dev,
163 struct netdev_notifier_info *info);
164
165/*
166 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
167 * semaphore.
168 *
169 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
170 *
171 * Writers must hold the rtnl semaphore while they loop through the
172 * dev_base_head list, and hold dev_base_lock for writing when they do the
173 * actual updates. This allows pure readers to access the list even
174 * while a writer is preparing to update it.
175 *
176 * To put it another way, dev_base_lock is held for writing only to
177 * protect against pure readers; the rtnl semaphore provides the
178 * protection against other writers.
179 *
180 * See, for example usages, register_netdevice() and
181 * unregister_netdevice(), which must be called with the rtnl
182 * semaphore held.
183 */
184DEFINE_RWLOCK(dev_base_lock);
185EXPORT_SYMBOL(dev_base_lock);
186
187/* protects napi_hash addition/deletion and napi_gen_id */
188static DEFINE_SPINLOCK(napi_hash_lock);
189
190static unsigned int napi_gen_id = NR_CPUS;
191static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
192
193static seqcount_t devnet_rename_seq;
194
195static inline void dev_base_seq_inc(struct net *net)
196{
197 while (++net->dev_base_seq == 0)
198 ;
199}
200
201static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
202{
203 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
204
205 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
206}
207
208static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
209{
210 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
211}
212
213static inline void rps_lock(struct softnet_data *sd)
214{
215#ifdef CONFIG_RPS
216 spin_lock(&sd->input_pkt_queue.lock);
217#endif
218}
219
220static inline void rps_unlock(struct softnet_data *sd)
221{
222#ifdef CONFIG_RPS
223 spin_unlock(&sd->input_pkt_queue.lock);
224#endif
225}
226
227/* Device list insertion */
228static void list_netdevice(struct net_device *dev)
229{
230 struct net *net = dev_net(dev);
231
232 ASSERT_RTNL();
233
234 write_lock_bh(&dev_base_lock);
235 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
236 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
237 hlist_add_head_rcu(&dev->index_hlist,
238 dev_index_hash(net, dev->ifindex));
239 write_unlock_bh(&dev_base_lock);
240
241 dev_base_seq_inc(net);
242}
243
244/* Device list removal
245 * caller must respect a RCU grace period before freeing/reusing dev
246 */
247static void unlist_netdevice(struct net_device *dev)
248{
249 ASSERT_RTNL();
250
251 /* Unlink dev from the device chain */
252 write_lock_bh(&dev_base_lock);
253 list_del_rcu(&dev->dev_list);
254 hlist_del_rcu(&dev->name_hlist);
255 hlist_del_rcu(&dev->index_hlist);
256 write_unlock_bh(&dev_base_lock);
257
258 dev_base_seq_inc(dev_net(dev));
259}
260
261/*
262 * Our notifier list
263 */
264
265static RAW_NOTIFIER_HEAD(netdev_chain);
266
267/*
268 * Device drivers call our routines to queue packets here. We empty the
269 * queue in the local softnet handler.
270 */
271
272DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
273EXPORT_PER_CPU_SYMBOL(softnet_data);
274
275#ifdef CONFIG_LOCKDEP
276/*
277 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
278 * according to dev->type
279 */
280static const unsigned short netdev_lock_type[] = {
281 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
282 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
283 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
284 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
285 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
286 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
287 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
288 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
289 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
290 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
291 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
292 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
293 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
294 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
295 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
296
297static const char *const netdev_lock_name[] = {
298 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
299 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
300 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
301 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
302 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
303 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
304 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
305 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
306 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
307 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
308 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
309 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
310 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
311 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
312 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
313
314static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
315static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
316
317static inline unsigned short netdev_lock_pos(unsigned short dev_type)
318{
319 int i;
320
321 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
322 if (netdev_lock_type[i] == dev_type)
323 return i;
324 /* the last key is used by default */
325 return ARRAY_SIZE(netdev_lock_type) - 1;
326}
327
328static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
329 unsigned short dev_type)
330{
331 int i;
332
333 i = netdev_lock_pos(dev_type);
334 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
335 netdev_lock_name[i]);
336}
337
338static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
339{
340 int i;
341
342 i = netdev_lock_pos(dev->type);
343 lockdep_set_class_and_name(&dev->addr_list_lock,
344 &netdev_addr_lock_key[i],
345 netdev_lock_name[i]);
346}
347#else
348static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
349 unsigned short dev_type)
350{
351}
352static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
353{
354}
355#endif
356
357/*******************************************************************************
358 *
359 * Protocol management and registration routines
360 *
361 *******************************************************************************/
362
363
364/*
365 * Add a protocol ID to the list. Now that the input handler is
366 * smarter we can dispense with all the messy stuff that used to be
367 * here.
368 *
369 * BEWARE!!! Protocol handlers, mangling input packets,
370 * MUST BE last in hash buckets and checking protocol handlers
371 * MUST start from promiscuous ptype_all chain in net_bh.
372 * It is true now, do not change it.
373 * Explanation follows: if protocol handler, mangling packet, will
374 * be the first on list, it is not able to sense, that packet
375 * is cloned and should be copied-on-write, so that it will
376 * change it and subsequent readers will get broken packet.
377 * --ANK (980803)
378 */
379
380static inline struct list_head *ptype_head(const struct packet_type *pt)
381{
382 if (pt->type == htons(ETH_P_ALL))
383 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
384 else
385 return pt->dev ? &pt->dev->ptype_specific :
386 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
387}
388
389/**
390 * dev_add_pack - add packet handler
391 * @pt: packet type declaration
392 *
393 * Add a protocol handler to the networking stack. The passed &packet_type
394 * is linked into kernel lists and may not be freed until it has been
395 * removed from the kernel lists.
396 *
397 * This call does not sleep therefore it can not
398 * guarantee all CPU's that are in middle of receiving packets
399 * will see the new packet type (until the next received packet).
400 */
401
402void dev_add_pack(struct packet_type *pt)
403{
404 struct list_head *head = ptype_head(pt);
405
406 spin_lock(&ptype_lock);
407 list_add_rcu(&pt->list, head);
408 spin_unlock(&ptype_lock);
409}
410EXPORT_SYMBOL(dev_add_pack);
411
412/**
413 * __dev_remove_pack - remove packet handler
414 * @pt: packet type declaration
415 *
416 * Remove a protocol handler that was previously added to the kernel
417 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
418 * from the kernel lists and can be freed or reused once this function
419 * returns.
420 *
421 * The packet type might still be in use by receivers
422 * and must not be freed until after all the CPU's have gone
423 * through a quiescent state.
424 */
425void __dev_remove_pack(struct packet_type *pt)
426{
427 struct list_head *head = ptype_head(pt);
428 struct packet_type *pt1;
429
430 spin_lock(&ptype_lock);
431
432 list_for_each_entry(pt1, head, list) {
433 if (pt == pt1) {
434 list_del_rcu(&pt->list);
435 goto out;
436 }
437 }
438
439 pr_warn("dev_remove_pack: %p not found\n", pt);
440out:
441 spin_unlock(&ptype_lock);
442}
443EXPORT_SYMBOL(__dev_remove_pack);
444
445/**
446 * dev_remove_pack - remove packet handler
447 * @pt: packet type declaration
448 *
449 * Remove a protocol handler that was previously added to the kernel
450 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
451 * from the kernel lists and can be freed or reused once this function
452 * returns.
453 *
454 * This call sleeps to guarantee that no CPU is looking at the packet
455 * type after return.
456 */
457void dev_remove_pack(struct packet_type *pt)
458{
459 __dev_remove_pack(pt);
460
461 synchronize_net();
462}
463EXPORT_SYMBOL(dev_remove_pack);
464
465
466/**
467 * dev_add_offload - register offload handlers
468 * @po: protocol offload declaration
469 *
470 * Add protocol offload handlers to the networking stack. The passed
471 * &proto_offload is linked into kernel lists and may not be freed until
472 * it has been removed from the kernel lists.
473 *
474 * This call does not sleep therefore it can not
475 * guarantee all CPU's that are in middle of receiving packets
476 * will see the new offload handlers (until the next received packet).
477 */
478void dev_add_offload(struct packet_offload *po)
479{
480 struct packet_offload *elem;
481
482 spin_lock(&offload_lock);
483 list_for_each_entry(elem, &offload_base, list) {
484 if (po->priority < elem->priority)
485 break;
486 }
487 list_add_rcu(&po->list, elem->list.prev);
488 spin_unlock(&offload_lock);
489}
490EXPORT_SYMBOL(dev_add_offload);
491
492/**
493 * __dev_remove_offload - remove offload handler
494 * @po: packet offload declaration
495 *
496 * Remove a protocol offload handler that was previously added to the
497 * kernel offload handlers by dev_add_offload(). The passed &offload_type
498 * is removed from the kernel lists and can be freed or reused once this
499 * function returns.
500 *
501 * The packet type might still be in use by receivers
502 * and must not be freed until after all the CPU's have gone
503 * through a quiescent state.
504 */
505static void __dev_remove_offload(struct packet_offload *po)
506{
507 struct list_head *head = &offload_base;
508 struct packet_offload *po1;
509
510 spin_lock(&offload_lock);
511
512 list_for_each_entry(po1, head, list) {
513 if (po == po1) {
514 list_del_rcu(&po->list);
515 goto out;
516 }
517 }
518
519 pr_warn("dev_remove_offload: %p not found\n", po);
520out:
521 spin_unlock(&offload_lock);
522}
523
524/**
525 * dev_remove_offload - remove packet offload handler
526 * @po: packet offload declaration
527 *
528 * Remove a packet offload handler that was previously added to the kernel
529 * offload handlers by dev_add_offload(). The passed &offload_type is
530 * removed from the kernel lists and can be freed or reused once this
531 * function returns.
532 *
533 * This call sleeps to guarantee that no CPU is looking at the packet
534 * type after return.
535 */
536void dev_remove_offload(struct packet_offload *po)
537{
538 __dev_remove_offload(po);
539
540 synchronize_net();
541}
542EXPORT_SYMBOL(dev_remove_offload);
543
544/******************************************************************************
545 *
546 * Device Boot-time Settings Routines
547 *
548 ******************************************************************************/
549
550/* Boot time configuration table */
551static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
552
553/**
554 * netdev_boot_setup_add - add new setup entry
555 * @name: name of the device
556 * @map: configured settings for the device
557 *
558 * Adds new setup entry to the dev_boot_setup list. The function
559 * returns 0 on error and 1 on success. This is a generic routine to
560 * all netdevices.
561 */
562static int netdev_boot_setup_add(char *name, struct ifmap *map)
563{
564 struct netdev_boot_setup *s;
565 int i;
566
567 s = dev_boot_setup;
568 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
569 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
570 memset(s[i].name, 0, sizeof(s[i].name));
571 strlcpy(s[i].name, name, IFNAMSIZ);
572 memcpy(&s[i].map, map, sizeof(s[i].map));
573 break;
574 }
575 }
576
577 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
578}
579
580/**
581 * netdev_boot_setup_check - check boot time settings
582 * @dev: the netdevice
583 *
584 * Check boot time settings for the device.
585 * The found settings are set for the device to be used
586 * later in the device probing.
587 * Returns 0 if no settings found, 1 if they are.
588 */
589int netdev_boot_setup_check(struct net_device *dev)
590{
591 struct netdev_boot_setup *s = dev_boot_setup;
592 int i;
593
594 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
595 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
596 !strcmp(dev->name, s[i].name)) {
597 dev->irq = s[i].map.irq;
598 dev->base_addr = s[i].map.base_addr;
599 dev->mem_start = s[i].map.mem_start;
600 dev->mem_end = s[i].map.mem_end;
601 return 1;
602 }
603 }
604 return 0;
605}
606EXPORT_SYMBOL(netdev_boot_setup_check);
607
608
609/**
610 * netdev_boot_base - get address from boot time settings
611 * @prefix: prefix for network device
612 * @unit: id for network device
613 *
614 * Check boot time settings for the base address of device.
615 * The found settings are set for the device to be used
616 * later in the device probing.
617 * Returns 0 if no settings found.
618 */
619unsigned long netdev_boot_base(const char *prefix, int unit)
620{
621 const struct netdev_boot_setup *s = dev_boot_setup;
622 char name[IFNAMSIZ];
623 int i;
624
625 sprintf(name, "%s%d", prefix, unit);
626
627 /*
628 * If device already registered then return base of 1
629 * to indicate not to probe for this interface
630 */
631 if (__dev_get_by_name(&init_net, name))
632 return 1;
633
634 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
635 if (!strcmp(name, s[i].name))
636 return s[i].map.base_addr;
637 return 0;
638}
639
640/*
641 * Saves at boot time configured settings for any netdevice.
642 */
643int __init netdev_boot_setup(char *str)
644{
645 int ints[5];
646 struct ifmap map;
647
648 str = get_options(str, ARRAY_SIZE(ints), ints);
649 if (!str || !*str)
650 return 0;
651
652 /* Save settings */
653 memset(&map, 0, sizeof(map));
654 if (ints[0] > 0)
655 map.irq = ints[1];
656 if (ints[0] > 1)
657 map.base_addr = ints[2];
658 if (ints[0] > 2)
659 map.mem_start = ints[3];
660 if (ints[0] > 3)
661 map.mem_end = ints[4];
662
663 /* Add new entry to the list */
664 return netdev_boot_setup_add(str, &map);
665}
666
667__setup("netdev=", netdev_boot_setup);
668
669/*******************************************************************************
670 *
671 * Device Interface Subroutines
672 *
673 *******************************************************************************/
674
675/**
676 * dev_get_iflink - get 'iflink' value of a interface
677 * @dev: targeted interface
678 *
679 * Indicates the ifindex the interface is linked to.
680 * Physical interfaces have the same 'ifindex' and 'iflink' values.
681 */
682
683int dev_get_iflink(const struct net_device *dev)
684{
685 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
686 return dev->netdev_ops->ndo_get_iflink(dev);
687
688 return dev->ifindex;
689}
690EXPORT_SYMBOL(dev_get_iflink);
691
692/**
693 * dev_fill_metadata_dst - Retrieve tunnel egress information.
694 * @dev: targeted interface
695 * @skb: The packet.
696 *
697 * For better visibility of tunnel traffic OVS needs to retrieve
698 * egress tunnel information for a packet. Following API allows
699 * user to get this info.
700 */
701int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
702{
703 struct ip_tunnel_info *info;
704
705 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
706 return -EINVAL;
707
708 info = skb_tunnel_info_unclone(skb);
709 if (!info)
710 return -ENOMEM;
711 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
712 return -EINVAL;
713
714 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
715}
716EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
717
718/**
719 * __dev_get_by_name - find a device by its name
720 * @net: the applicable net namespace
721 * @name: name to find
722 *
723 * Find an interface by name. Must be called under RTNL semaphore
724 * or @dev_base_lock. If the name is found a pointer to the device
725 * is returned. If the name is not found then %NULL is returned. The
726 * reference counters are not incremented so the caller must be
727 * careful with locks.
728 */
729
730struct net_device *__dev_get_by_name(struct net *net, const char *name)
731{
732 struct net_device *dev;
733 struct hlist_head *head = dev_name_hash(net, name);
734
735 hlist_for_each_entry(dev, head, name_hlist)
736 if (!strncmp(dev->name, name, IFNAMSIZ))
737 return dev;
738
739 return NULL;
740}
741EXPORT_SYMBOL(__dev_get_by_name);
742
743/**
744 * dev_get_by_name_rcu - find a device by its name
745 * @net: the applicable net namespace
746 * @name: name to find
747 *
748 * Find an interface by name.
749 * If the name is found a pointer to the device is returned.
750 * If the name is not found then %NULL is returned.
751 * The reference counters are not incremented so the caller must be
752 * careful with locks. The caller must hold RCU lock.
753 */
754
755struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
756{
757 struct net_device *dev;
758 struct hlist_head *head = dev_name_hash(net, name);
759
760 hlist_for_each_entry_rcu(dev, head, name_hlist)
761 if (!strncmp(dev->name, name, IFNAMSIZ))
762 return dev;
763
764 return NULL;
765}
766EXPORT_SYMBOL(dev_get_by_name_rcu);
767
768/**
769 * dev_get_by_name - find a device by its name
770 * @net: the applicable net namespace
771 * @name: name to find
772 *
773 * Find an interface by name. This can be called from any
774 * context and does its own locking. The returned handle has
775 * the usage count incremented and the caller must use dev_put() to
776 * release it when it is no longer needed. %NULL is returned if no
777 * matching device is found.
778 */
779
780struct net_device *dev_get_by_name(struct net *net, const char *name)
781{
782 struct net_device *dev;
783
784 rcu_read_lock();
785 dev = dev_get_by_name_rcu(net, name);
786 if (dev)
787 dev_hold(dev);
788 rcu_read_unlock();
789 return dev;
790}
791EXPORT_SYMBOL(dev_get_by_name);
792
793/**
794 * __dev_get_by_index - find a device by its ifindex
795 * @net: the applicable net namespace
796 * @ifindex: index of device
797 *
798 * Search for an interface by index. Returns %NULL if the device
799 * is not found or a pointer to the device. The device has not
800 * had its reference counter increased so the caller must be careful
801 * about locking. The caller must hold either the RTNL semaphore
802 * or @dev_base_lock.
803 */
804
805struct net_device *__dev_get_by_index(struct net *net, int ifindex)
806{
807 struct net_device *dev;
808 struct hlist_head *head = dev_index_hash(net, ifindex);
809
810 hlist_for_each_entry(dev, head, index_hlist)
811 if (dev->ifindex == ifindex)
812 return dev;
813
814 return NULL;
815}
816EXPORT_SYMBOL(__dev_get_by_index);
817
818/**
819 * dev_get_by_index_rcu - find a device by its ifindex
820 * @net: the applicable net namespace
821 * @ifindex: index of device
822 *
823 * Search for an interface by index. Returns %NULL if the device
824 * is not found or a pointer to the device. The device has not
825 * had its reference counter increased so the caller must be careful
826 * about locking. The caller must hold RCU lock.
827 */
828
829struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
830{
831 struct net_device *dev;
832 struct hlist_head *head = dev_index_hash(net, ifindex);
833
834 hlist_for_each_entry_rcu(dev, head, index_hlist)
835 if (dev->ifindex == ifindex)
836 return dev;
837
838 return NULL;
839}
840EXPORT_SYMBOL(dev_get_by_index_rcu);
841
842
843/**
844 * dev_get_by_index - find a device by its ifindex
845 * @net: the applicable net namespace
846 * @ifindex: index of device
847 *
848 * Search for an interface by index. Returns NULL if the device
849 * is not found or a pointer to the device. The device returned has
850 * had a reference added and the pointer is safe until the user calls
851 * dev_put to indicate they have finished with it.
852 */
853
854struct net_device *dev_get_by_index(struct net *net, int ifindex)
855{
856 struct net_device *dev;
857
858 rcu_read_lock();
859 dev = dev_get_by_index_rcu(net, ifindex);
860 if (dev)
861 dev_hold(dev);
862 rcu_read_unlock();
863 return dev;
864}
865EXPORT_SYMBOL(dev_get_by_index);
866
867/**
868 * netdev_get_name - get a netdevice name, knowing its ifindex.
869 * @net: network namespace
870 * @name: a pointer to the buffer where the name will be stored.
871 * @ifindex: the ifindex of the interface to get the name from.
872 *
873 * The use of raw_seqcount_begin() and cond_resched() before
874 * retrying is required as we want to give the writers a chance
875 * to complete when CONFIG_PREEMPT is not set.
876 */
877int netdev_get_name(struct net *net, char *name, int ifindex)
878{
879 struct net_device *dev;
880 unsigned int seq;
881
882retry:
883 seq = raw_seqcount_begin(&devnet_rename_seq);
884 rcu_read_lock();
885 dev = dev_get_by_index_rcu(net, ifindex);
886 if (!dev) {
887 rcu_read_unlock();
888 return -ENODEV;
889 }
890
891 strcpy(name, dev->name);
892 rcu_read_unlock();
893 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
894 cond_resched();
895 goto retry;
896 }
897
898 return 0;
899}
900
901/**
902 * dev_getbyhwaddr_rcu - find a device by its hardware address
903 * @net: the applicable net namespace
904 * @type: media type of device
905 * @ha: hardware address
906 *
907 * Search for an interface by MAC address. Returns NULL if the device
908 * is not found or a pointer to the device.
909 * The caller must hold RCU or RTNL.
910 * The returned device has not had its ref count increased
911 * and the caller must therefore be careful about locking
912 *
913 */
914
915struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
916 const char *ha)
917{
918 struct net_device *dev;
919
920 for_each_netdev_rcu(net, dev)
921 if (dev->type == type &&
922 !memcmp(dev->dev_addr, ha, dev->addr_len))
923 return dev;
924
925 return NULL;
926}
927EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
928
929struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
930{
931 struct net_device *dev;
932
933 ASSERT_RTNL();
934 for_each_netdev(net, dev)
935 if (dev->type == type)
936 return dev;
937
938 return NULL;
939}
940EXPORT_SYMBOL(__dev_getfirstbyhwtype);
941
942struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
943{
944 struct net_device *dev, *ret = NULL;
945
946 rcu_read_lock();
947 for_each_netdev_rcu(net, dev)
948 if (dev->type == type) {
949 dev_hold(dev);
950 ret = dev;
951 break;
952 }
953 rcu_read_unlock();
954 return ret;
955}
956EXPORT_SYMBOL(dev_getfirstbyhwtype);
957
958/**
959 * __dev_get_by_flags - find any device with given flags
960 * @net: the applicable net namespace
961 * @if_flags: IFF_* values
962 * @mask: bitmask of bits in if_flags to check
963 *
964 * Search for any interface with the given flags. Returns NULL if a device
965 * is not found or a pointer to the device. Must be called inside
966 * rtnl_lock(), and result refcount is unchanged.
967 */
968
969struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
970 unsigned short mask)
971{
972 struct net_device *dev, *ret;
973
974 ASSERT_RTNL();
975
976 ret = NULL;
977 for_each_netdev(net, dev) {
978 if (((dev->flags ^ if_flags) & mask) == 0) {
979 ret = dev;
980 break;
981 }
982 }
983 return ret;
984}
985EXPORT_SYMBOL(__dev_get_by_flags);
986
987/**
988 * dev_valid_name - check if name is okay for network device
989 * @name: name string
990 *
991 * Network device names need to be valid file names to
992 * to allow sysfs to work. We also disallow any kind of
993 * whitespace.
994 */
995bool dev_valid_name(const char *name)
996{
997 if (*name == '\0')
998 return false;
999 if (strlen(name) >= IFNAMSIZ)
1000 return false;
1001 if (!strcmp(name, ".") || !strcmp(name, ".."))
1002 return false;
1003
1004 while (*name) {
1005 if (*name == '/' || *name == ':' || isspace(*name))
1006 return false;
1007 name++;
1008 }
1009 return true;
1010}
1011EXPORT_SYMBOL(dev_valid_name);
1012
1013/**
1014 * __dev_alloc_name - allocate a name for a device
1015 * @net: network namespace to allocate the device name in
1016 * @name: name format string
1017 * @buf: scratch buffer and result name string
1018 *
1019 * Passed a format string - eg "lt%d" it will try and find a suitable
1020 * id. It scans list of devices to build up a free map, then chooses
1021 * the first empty slot. The caller must hold the dev_base or rtnl lock
1022 * while allocating the name and adding the device in order to avoid
1023 * duplicates.
1024 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1025 * Returns the number of the unit assigned or a negative errno code.
1026 */
1027
1028static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1029{
1030 int i = 0;
1031 const char *p;
1032 const int max_netdevices = 8*PAGE_SIZE;
1033 unsigned long *inuse;
1034 struct net_device *d;
1035
1036 p = strnchr(name, IFNAMSIZ-1, '%');
1037 if (p) {
1038 /*
1039 * Verify the string as this thing may have come from
1040 * the user. There must be either one "%d" and no other "%"
1041 * characters.
1042 */
1043 if (p[1] != 'd' || strchr(p + 2, '%'))
1044 return -EINVAL;
1045
1046 /* Use one page as a bit array of possible slots */
1047 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1048 if (!inuse)
1049 return -ENOMEM;
1050
1051 for_each_netdev(net, d) {
1052 if (!sscanf(d->name, name, &i))
1053 continue;
1054 if (i < 0 || i >= max_netdevices)
1055 continue;
1056
1057 /* avoid cases where sscanf is not exact inverse of printf */
1058 snprintf(buf, IFNAMSIZ, name, i);
1059 if (!strncmp(buf, d->name, IFNAMSIZ))
1060 set_bit(i, inuse);
1061 }
1062
1063 i = find_first_zero_bit(inuse, max_netdevices);
1064 free_page((unsigned long) inuse);
1065 }
1066
1067 if (buf != name)
1068 snprintf(buf, IFNAMSIZ, name, i);
1069 if (!__dev_get_by_name(net, buf))
1070 return i;
1071
1072 /* It is possible to run out of possible slots
1073 * when the name is long and there isn't enough space left
1074 * for the digits, or if all bits are used.
1075 */
1076 return -ENFILE;
1077}
1078
1079/**
1080 * dev_alloc_name - allocate a name for a device
1081 * @dev: device
1082 * @name: name format string
1083 *
1084 * Passed a format string - eg "lt%d" it will try and find a suitable
1085 * id. It scans list of devices to build up a free map, then chooses
1086 * the first empty slot. The caller must hold the dev_base or rtnl lock
1087 * while allocating the name and adding the device in order to avoid
1088 * duplicates.
1089 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1090 * Returns the number of the unit assigned or a negative errno code.
1091 */
1092
1093int dev_alloc_name(struct net_device *dev, const char *name)
1094{
1095 char buf[IFNAMSIZ];
1096 struct net *net;
1097 int ret;
1098
1099 BUG_ON(!dev_net(dev));
1100 net = dev_net(dev);
1101 ret = __dev_alloc_name(net, name, buf);
1102 if (ret >= 0)
1103 strlcpy(dev->name, buf, IFNAMSIZ);
1104 return ret;
1105}
1106EXPORT_SYMBOL(dev_alloc_name);
1107
1108static int dev_alloc_name_ns(struct net *net,
1109 struct net_device *dev,
1110 const char *name)
1111{
1112 char buf[IFNAMSIZ];
1113 int ret;
1114
1115 ret = __dev_alloc_name(net, name, buf);
1116 if (ret >= 0)
1117 strlcpy(dev->name, buf, IFNAMSIZ);
1118 return ret;
1119}
1120
1121static int dev_get_valid_name(struct net *net,
1122 struct net_device *dev,
1123 const char *name)
1124{
1125 BUG_ON(!net);
1126
1127 if (!dev_valid_name(name))
1128 return -EINVAL;
1129
1130 if (strchr(name, '%'))
1131 return dev_alloc_name_ns(net, dev, name);
1132 else if (__dev_get_by_name(net, name))
1133 return -EEXIST;
1134 else if (dev->name != name)
1135 strlcpy(dev->name, name, IFNAMSIZ);
1136
1137 return 0;
1138}
1139
1140/**
1141 * dev_change_name - change name of a device
1142 * @dev: device
1143 * @newname: name (or format string) must be at least IFNAMSIZ
1144 *
1145 * Change name of a device, can pass format strings "eth%d".
1146 * for wildcarding.
1147 */
1148int dev_change_name(struct net_device *dev, const char *newname)
1149{
1150 unsigned char old_assign_type;
1151 char oldname[IFNAMSIZ];
1152 int err = 0;
1153 int ret;
1154 struct net *net;
1155
1156 ASSERT_RTNL();
1157 BUG_ON(!dev_net(dev));
1158
1159 net = dev_net(dev);
1160 if (dev->flags & IFF_UP)
1161 return -EBUSY;
1162
1163 write_seqcount_begin(&devnet_rename_seq);
1164
1165 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1166 write_seqcount_end(&devnet_rename_seq);
1167 return 0;
1168 }
1169
1170 memcpy(oldname, dev->name, IFNAMSIZ);
1171
1172 err = dev_get_valid_name(net, dev, newname);
1173 if (err < 0) {
1174 write_seqcount_end(&devnet_rename_seq);
1175 return err;
1176 }
1177
1178 if (oldname[0] && !strchr(oldname, '%'))
1179 netdev_info(dev, "renamed from %s\n", oldname);
1180
1181 old_assign_type = dev->name_assign_type;
1182 dev->name_assign_type = NET_NAME_RENAMED;
1183
1184rollback:
1185 ret = device_rename(&dev->dev, dev->name);
1186 if (ret) {
1187 memcpy(dev->name, oldname, IFNAMSIZ);
1188 dev->name_assign_type = old_assign_type;
1189 write_seqcount_end(&devnet_rename_seq);
1190 return ret;
1191 }
1192
1193 write_seqcount_end(&devnet_rename_seq);
1194
1195 netdev_adjacent_rename_links(dev, oldname);
1196
1197 write_lock_bh(&dev_base_lock);
1198 hlist_del_rcu(&dev->name_hlist);
1199 write_unlock_bh(&dev_base_lock);
1200
1201 synchronize_rcu();
1202
1203 write_lock_bh(&dev_base_lock);
1204 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1205 write_unlock_bh(&dev_base_lock);
1206
1207 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1208 ret = notifier_to_errno(ret);
1209
1210 if (ret) {
1211 /* err >= 0 after dev_alloc_name() or stores the first errno */
1212 if (err >= 0) {
1213 err = ret;
1214 write_seqcount_begin(&devnet_rename_seq);
1215 memcpy(dev->name, oldname, IFNAMSIZ);
1216 memcpy(oldname, newname, IFNAMSIZ);
1217 dev->name_assign_type = old_assign_type;
1218 old_assign_type = NET_NAME_RENAMED;
1219 goto rollback;
1220 } else {
1221 pr_err("%s: name change rollback failed: %d\n",
1222 dev->name, ret);
1223 }
1224 }
1225
1226 return err;
1227}
1228
1229/**
1230 * dev_set_alias - change ifalias of a device
1231 * @dev: device
1232 * @alias: name up to IFALIASZ
1233 * @len: limit of bytes to copy from info
1234 *
1235 * Set ifalias for a device,
1236 */
1237int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1238{
1239 char *new_ifalias;
1240
1241 ASSERT_RTNL();
1242
1243 if (len >= IFALIASZ)
1244 return -EINVAL;
1245
1246 if (!len) {
1247 kfree(dev->ifalias);
1248 dev->ifalias = NULL;
1249 return 0;
1250 }
1251
1252 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1253 if (!new_ifalias)
1254 return -ENOMEM;
1255 dev->ifalias = new_ifalias;
1256 memcpy(dev->ifalias, alias, len);
1257 dev->ifalias[len] = 0;
1258
1259 return len;
1260}
1261
1262
1263/**
1264 * netdev_features_change - device changes features
1265 * @dev: device to cause notification
1266 *
1267 * Called to indicate a device has changed features.
1268 */
1269void netdev_features_change(struct net_device *dev)
1270{
1271 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1272}
1273EXPORT_SYMBOL(netdev_features_change);
1274
1275/**
1276 * netdev_state_change - device changes state
1277 * @dev: device to cause notification
1278 *
1279 * Called to indicate a device has changed state. This function calls
1280 * the notifier chains for netdev_chain and sends a NEWLINK message
1281 * to the routing socket.
1282 */
1283void netdev_state_change(struct net_device *dev)
1284{
1285 if (dev->flags & IFF_UP) {
1286 struct netdev_notifier_change_info change_info;
1287
1288 change_info.flags_changed = 0;
1289 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1290 &change_info.info);
1291 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1292 }
1293}
1294EXPORT_SYMBOL(netdev_state_change);
1295
1296/**
1297 * netdev_notify_peers - notify network peers about existence of @dev
1298 * @dev: network device
1299 *
1300 * Generate traffic such that interested network peers are aware of
1301 * @dev, such as by generating a gratuitous ARP. This may be used when
1302 * a device wants to inform the rest of the network about some sort of
1303 * reconfiguration such as a failover event or virtual machine
1304 * migration.
1305 */
1306void netdev_notify_peers(struct net_device *dev)
1307{
1308 rtnl_lock();
1309 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1310 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1311 rtnl_unlock();
1312}
1313EXPORT_SYMBOL(netdev_notify_peers);
1314
1315static int __dev_open(struct net_device *dev)
1316{
1317 const struct net_device_ops *ops = dev->netdev_ops;
1318 int ret;
1319
1320 ASSERT_RTNL();
1321
1322 if (!netif_device_present(dev))
1323 return -ENODEV;
1324
1325 /* Block netpoll from trying to do any rx path servicing.
1326 * If we don't do this there is a chance ndo_poll_controller
1327 * or ndo_poll may be running while we open the device
1328 */
1329 netpoll_poll_disable(dev);
1330
1331 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1332 ret = notifier_to_errno(ret);
1333 if (ret)
1334 return ret;
1335
1336 set_bit(__LINK_STATE_START, &dev->state);
1337
1338 if (ops->ndo_validate_addr)
1339 ret = ops->ndo_validate_addr(dev);
1340
1341 if (!ret && ops->ndo_open)
1342 ret = ops->ndo_open(dev);
1343
1344 netpoll_poll_enable(dev);
1345
1346 if (ret)
1347 clear_bit(__LINK_STATE_START, &dev->state);
1348 else {
1349 dev->flags |= IFF_UP;
1350 dev_set_rx_mode(dev);
1351 dev_activate(dev);
1352 add_device_randomness(dev->dev_addr, dev->addr_len);
1353 }
1354
1355 return ret;
1356}
1357
1358/**
1359 * dev_open - prepare an interface for use.
1360 * @dev: device to open
1361 *
1362 * Takes a device from down to up state. The device's private open
1363 * function is invoked and then the multicast lists are loaded. Finally
1364 * the device is moved into the up state and a %NETDEV_UP message is
1365 * sent to the netdev notifier chain.
1366 *
1367 * Calling this function on an active interface is a nop. On a failure
1368 * a negative errno code is returned.
1369 */
1370int dev_open(struct net_device *dev)
1371{
1372 int ret;
1373
1374 if (dev->flags & IFF_UP)
1375 return 0;
1376
1377 ret = __dev_open(dev);
1378 if (ret < 0)
1379 return ret;
1380
1381 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1382 call_netdevice_notifiers(NETDEV_UP, dev);
1383
1384 return ret;
1385}
1386EXPORT_SYMBOL(dev_open);
1387
1388static int __dev_close_many(struct list_head *head)
1389{
1390 struct net_device *dev;
1391
1392 ASSERT_RTNL();
1393 might_sleep();
1394
1395 list_for_each_entry(dev, head, close_list) {
1396 /* Temporarily disable netpoll until the interface is down */
1397 netpoll_poll_disable(dev);
1398
1399 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1400
1401 clear_bit(__LINK_STATE_START, &dev->state);
1402
1403 /* Synchronize to scheduled poll. We cannot touch poll list, it
1404 * can be even on different cpu. So just clear netif_running().
1405 *
1406 * dev->stop() will invoke napi_disable() on all of it's
1407 * napi_struct instances on this device.
1408 */
1409 smp_mb__after_atomic(); /* Commit netif_running(). */
1410 }
1411
1412 dev_deactivate_many(head);
1413
1414 list_for_each_entry(dev, head, close_list) {
1415 const struct net_device_ops *ops = dev->netdev_ops;
1416
1417 /*
1418 * Call the device specific close. This cannot fail.
1419 * Only if device is UP
1420 *
1421 * We allow it to be called even after a DETACH hot-plug
1422 * event.
1423 */
1424 if (ops->ndo_stop)
1425 ops->ndo_stop(dev);
1426
1427 dev->flags &= ~IFF_UP;
1428 netpoll_poll_enable(dev);
1429 }
1430
1431 return 0;
1432}
1433
1434static int __dev_close(struct net_device *dev)
1435{
1436 int retval;
1437 LIST_HEAD(single);
1438
1439 list_add(&dev->close_list, &single);
1440 retval = __dev_close_many(&single);
1441 list_del(&single);
1442
1443 return retval;
1444}
1445
1446int dev_close_many(struct list_head *head, bool unlink)
1447{
1448 struct net_device *dev, *tmp;
1449
1450 /* Remove the devices that don't need to be closed */
1451 list_for_each_entry_safe(dev, tmp, head, close_list)
1452 if (!(dev->flags & IFF_UP))
1453 list_del_init(&dev->close_list);
1454
1455 __dev_close_many(head);
1456
1457 list_for_each_entry_safe(dev, tmp, head, close_list) {
1458 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1459 call_netdevice_notifiers(NETDEV_DOWN, dev);
1460 if (unlink)
1461 list_del_init(&dev->close_list);
1462 }
1463
1464 return 0;
1465}
1466EXPORT_SYMBOL(dev_close_many);
1467
1468/**
1469 * dev_close - shutdown an interface.
1470 * @dev: device to shutdown
1471 *
1472 * This function moves an active device into down state. A
1473 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1474 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1475 * chain.
1476 */
1477int dev_close(struct net_device *dev)
1478{
1479 if (dev->flags & IFF_UP) {
1480 LIST_HEAD(single);
1481
1482 list_add(&dev->close_list, &single);
1483 dev_close_many(&single, true);
1484 list_del(&single);
1485 }
1486 return 0;
1487}
1488EXPORT_SYMBOL(dev_close);
1489
1490
1491/**
1492 * dev_disable_lro - disable Large Receive Offload on a device
1493 * @dev: device
1494 *
1495 * Disable Large Receive Offload (LRO) on a net device. Must be
1496 * called under RTNL. This is needed if received packets may be
1497 * forwarded to another interface.
1498 */
1499void dev_disable_lro(struct net_device *dev)
1500{
1501 struct net_device *lower_dev;
1502 struct list_head *iter;
1503
1504 dev->wanted_features &= ~NETIF_F_LRO;
1505 netdev_update_features(dev);
1506
1507 if (unlikely(dev->features & NETIF_F_LRO))
1508 netdev_WARN(dev, "failed to disable LRO!\n");
1509
1510 netdev_for_each_lower_dev(dev, lower_dev, iter)
1511 dev_disable_lro(lower_dev);
1512}
1513EXPORT_SYMBOL(dev_disable_lro);
1514
1515static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1516 struct net_device *dev)
1517{
1518 struct netdev_notifier_info info;
1519
1520 netdev_notifier_info_init(&info, dev);
1521 return nb->notifier_call(nb, val, &info);
1522}
1523
1524static int dev_boot_phase = 1;
1525
1526/**
1527 * register_netdevice_notifier - register a network notifier block
1528 * @nb: notifier
1529 *
1530 * Register a notifier to be called when network device events occur.
1531 * The notifier passed is linked into the kernel structures and must
1532 * not be reused until it has been unregistered. A negative errno code
1533 * is returned on a failure.
1534 *
1535 * When registered all registration and up events are replayed
1536 * to the new notifier to allow device to have a race free
1537 * view of the network device list.
1538 */
1539
1540int register_netdevice_notifier(struct notifier_block *nb)
1541{
1542 struct net_device *dev;
1543 struct net_device *last;
1544 struct net *net;
1545 int err;
1546
1547 rtnl_lock();
1548 err = raw_notifier_chain_register(&netdev_chain, nb);
1549 if (err)
1550 goto unlock;
1551 if (dev_boot_phase)
1552 goto unlock;
1553 for_each_net(net) {
1554 for_each_netdev(net, dev) {
1555 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1556 err = notifier_to_errno(err);
1557 if (err)
1558 goto rollback;
1559
1560 if (!(dev->flags & IFF_UP))
1561 continue;
1562
1563 call_netdevice_notifier(nb, NETDEV_UP, dev);
1564 }
1565 }
1566
1567unlock:
1568 rtnl_unlock();
1569 return err;
1570
1571rollback:
1572 last = dev;
1573 for_each_net(net) {
1574 for_each_netdev(net, dev) {
1575 if (dev == last)
1576 goto outroll;
1577
1578 if (dev->flags & IFF_UP) {
1579 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1580 dev);
1581 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1582 }
1583 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1584 }
1585 }
1586
1587outroll:
1588 raw_notifier_chain_unregister(&netdev_chain, nb);
1589 goto unlock;
1590}
1591EXPORT_SYMBOL(register_netdevice_notifier);
1592
1593/**
1594 * unregister_netdevice_notifier - unregister a network notifier block
1595 * @nb: notifier
1596 *
1597 * Unregister a notifier previously registered by
1598 * register_netdevice_notifier(). The notifier is unlinked into the
1599 * kernel structures and may then be reused. A negative errno code
1600 * is returned on a failure.
1601 *
1602 * After unregistering unregister and down device events are synthesized
1603 * for all devices on the device list to the removed notifier to remove
1604 * the need for special case cleanup code.
1605 */
1606
1607int unregister_netdevice_notifier(struct notifier_block *nb)
1608{
1609 struct net_device *dev;
1610 struct net *net;
1611 int err;
1612
1613 rtnl_lock();
1614 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1615 if (err)
1616 goto unlock;
1617
1618 for_each_net(net) {
1619 for_each_netdev(net, dev) {
1620 if (dev->flags & IFF_UP) {
1621 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1622 dev);
1623 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1624 }
1625 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1626 }
1627 }
1628unlock:
1629 rtnl_unlock();
1630 return err;
1631}
1632EXPORT_SYMBOL(unregister_netdevice_notifier);
1633
1634/**
1635 * call_netdevice_notifiers_info - call all network notifier blocks
1636 * @val: value passed unmodified to notifier function
1637 * @dev: net_device pointer passed unmodified to notifier function
1638 * @info: notifier information data
1639 *
1640 * Call all network notifier blocks. Parameters and return value
1641 * are as for raw_notifier_call_chain().
1642 */
1643
1644static int call_netdevice_notifiers_info(unsigned long val,
1645 struct net_device *dev,
1646 struct netdev_notifier_info *info)
1647{
1648 ASSERT_RTNL();
1649 netdev_notifier_info_init(info, dev);
1650 return raw_notifier_call_chain(&netdev_chain, val, info);
1651}
1652
1653/**
1654 * call_netdevice_notifiers - call all network notifier blocks
1655 * @val: value passed unmodified to notifier function
1656 * @dev: net_device pointer passed unmodified to notifier function
1657 *
1658 * Call all network notifier blocks. Parameters and return value
1659 * are as for raw_notifier_call_chain().
1660 */
1661
1662int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1663{
1664 struct netdev_notifier_info info;
1665
1666 return call_netdevice_notifiers_info(val, dev, &info);
1667}
1668EXPORT_SYMBOL(call_netdevice_notifiers);
1669
1670#ifdef CONFIG_NET_INGRESS
1671static struct static_key ingress_needed __read_mostly;
1672
1673void net_inc_ingress_queue(void)
1674{
1675 static_key_slow_inc(&ingress_needed);
1676}
1677EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1678
1679void net_dec_ingress_queue(void)
1680{
1681 static_key_slow_dec(&ingress_needed);
1682}
1683EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1684#endif
1685
1686#ifdef CONFIG_NET_EGRESS
1687static struct static_key egress_needed __read_mostly;
1688
1689void net_inc_egress_queue(void)
1690{
1691 static_key_slow_inc(&egress_needed);
1692}
1693EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1694
1695void net_dec_egress_queue(void)
1696{
1697 static_key_slow_dec(&egress_needed);
1698}
1699EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1700#endif
1701
1702static struct static_key netstamp_needed __read_mostly;
1703#ifdef HAVE_JUMP_LABEL
1704static atomic_t netstamp_needed_deferred;
1705static atomic_t netstamp_wanted;
1706static void netstamp_clear(struct work_struct *work)
1707{
1708 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1709 int wanted;
1710
1711 wanted = atomic_add_return(deferred, &netstamp_wanted);
1712 if (wanted > 0)
1713 static_key_enable(&netstamp_needed);
1714 else
1715 static_key_disable(&netstamp_needed);
1716}
1717static DECLARE_WORK(netstamp_work, netstamp_clear);
1718#endif
1719
1720void net_enable_timestamp(void)
1721{
1722#ifdef HAVE_JUMP_LABEL
1723 int wanted;
1724
1725 while (1) {
1726 wanted = atomic_read(&netstamp_wanted);
1727 if (wanted <= 0)
1728 break;
1729 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1730 return;
1731 }
1732 atomic_inc(&netstamp_needed_deferred);
1733 schedule_work(&netstamp_work);
1734#else
1735 static_key_slow_inc(&netstamp_needed);
1736#endif
1737}
1738EXPORT_SYMBOL(net_enable_timestamp);
1739
1740void net_disable_timestamp(void)
1741{
1742#ifdef HAVE_JUMP_LABEL
1743 int wanted;
1744
1745 while (1) {
1746 wanted = atomic_read(&netstamp_wanted);
1747 if (wanted <= 1)
1748 break;
1749 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1750 return;
1751 }
1752 atomic_dec(&netstamp_needed_deferred);
1753 schedule_work(&netstamp_work);
1754#else
1755 static_key_slow_dec(&netstamp_needed);
1756#endif
1757}
1758EXPORT_SYMBOL(net_disable_timestamp);
1759
1760static inline void net_timestamp_set(struct sk_buff *skb)
1761{
1762 skb->tstamp = 0;
1763 if (static_key_false(&netstamp_needed))
1764 __net_timestamp(skb);
1765}
1766
1767#define net_timestamp_check(COND, SKB) \
1768 if (static_key_false(&netstamp_needed)) { \
1769 if ((COND) && !(SKB)->tstamp) \
1770 __net_timestamp(SKB); \
1771 } \
1772
1773bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
1774{
1775 unsigned int len;
1776
1777 if (!(dev->flags & IFF_UP))
1778 return false;
1779
1780 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1781 if (skb->len <= len)
1782 return true;
1783
1784 /* if TSO is enabled, we don't care about the length as the packet
1785 * could be forwarded without being segmented before
1786 */
1787 if (skb_is_gso(skb))
1788 return true;
1789
1790 return false;
1791}
1792EXPORT_SYMBOL_GPL(is_skb_forwardable);
1793
1794int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1795{
1796 int ret = ____dev_forward_skb(dev, skb);
1797
1798 if (likely(!ret)) {
1799 skb->protocol = eth_type_trans(skb, dev);
1800 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1801 }
1802
1803 return ret;
1804}
1805EXPORT_SYMBOL_GPL(__dev_forward_skb);
1806
1807/**
1808 * dev_forward_skb - loopback an skb to another netif
1809 *
1810 * @dev: destination network device
1811 * @skb: buffer to forward
1812 *
1813 * return values:
1814 * NET_RX_SUCCESS (no congestion)
1815 * NET_RX_DROP (packet was dropped, but freed)
1816 *
1817 * dev_forward_skb can be used for injecting an skb from the
1818 * start_xmit function of one device into the receive queue
1819 * of another device.
1820 *
1821 * The receiving device may be in another namespace, so
1822 * we have to clear all information in the skb that could
1823 * impact namespace isolation.
1824 */
1825int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1826{
1827 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1828}
1829EXPORT_SYMBOL_GPL(dev_forward_skb);
1830
1831static inline int deliver_skb(struct sk_buff *skb,
1832 struct packet_type *pt_prev,
1833 struct net_device *orig_dev)
1834{
1835 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1836 return -ENOMEM;
1837 atomic_inc(&skb->users);
1838 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1839}
1840
1841static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1842 struct packet_type **pt,
1843 struct net_device *orig_dev,
1844 __be16 type,
1845 struct list_head *ptype_list)
1846{
1847 struct packet_type *ptype, *pt_prev = *pt;
1848
1849 list_for_each_entry_rcu(ptype, ptype_list, list) {
1850 if (ptype->type != type)
1851 continue;
1852 if (pt_prev)
1853 deliver_skb(skb, pt_prev, orig_dev);
1854 pt_prev = ptype;
1855 }
1856 *pt = pt_prev;
1857}
1858
1859static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1860{
1861 if (!ptype->af_packet_priv || !skb->sk)
1862 return false;
1863
1864 if (ptype->id_match)
1865 return ptype->id_match(ptype, skb->sk);
1866 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1867 return true;
1868
1869 return false;
1870}
1871
1872/*
1873 * Support routine. Sends outgoing frames to any network
1874 * taps currently in use.
1875 */
1876
1877void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1878{
1879 struct packet_type *ptype;
1880 struct sk_buff *skb2 = NULL;
1881 struct packet_type *pt_prev = NULL;
1882 struct list_head *ptype_list = &ptype_all;
1883
1884 rcu_read_lock();
1885again:
1886 list_for_each_entry_rcu(ptype, ptype_list, list) {
1887 /* Never send packets back to the socket
1888 * they originated from - MvS (miquels@drinkel.ow.org)
1889 */
1890 if (skb_loop_sk(ptype, skb))
1891 continue;
1892
1893 if (pt_prev) {
1894 deliver_skb(skb2, pt_prev, skb->dev);
1895 pt_prev = ptype;
1896 continue;
1897 }
1898
1899 /* need to clone skb, done only once */
1900 skb2 = skb_clone(skb, GFP_ATOMIC);
1901 if (!skb2)
1902 goto out_unlock;
1903
1904 net_timestamp_set(skb2);
1905
1906 /* skb->nh should be correctly
1907 * set by sender, so that the second statement is
1908 * just protection against buggy protocols.
1909 */
1910 skb_reset_mac_header(skb2);
1911
1912 if (skb_network_header(skb2) < skb2->data ||
1913 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1914 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1915 ntohs(skb2->protocol),
1916 dev->name);
1917 skb_reset_network_header(skb2);
1918 }
1919
1920 skb2->transport_header = skb2->network_header;
1921 skb2->pkt_type = PACKET_OUTGOING;
1922 pt_prev = ptype;
1923 }
1924
1925 if (ptype_list == &ptype_all) {
1926 ptype_list = &dev->ptype_all;
1927 goto again;
1928 }
1929out_unlock:
1930 if (pt_prev)
1931 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1932 rcu_read_unlock();
1933}
1934EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
1935
1936/**
1937 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1938 * @dev: Network device
1939 * @txq: number of queues available
1940 *
1941 * If real_num_tx_queues is changed the tc mappings may no longer be
1942 * valid. To resolve this verify the tc mapping remains valid and if
1943 * not NULL the mapping. With no priorities mapping to this
1944 * offset/count pair it will no longer be used. In the worst case TC0
1945 * is invalid nothing can be done so disable priority mappings. If is
1946 * expected that drivers will fix this mapping if they can before
1947 * calling netif_set_real_num_tx_queues.
1948 */
1949static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1950{
1951 int i;
1952 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1953
1954 /* If TC0 is invalidated disable TC mapping */
1955 if (tc->offset + tc->count > txq) {
1956 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1957 dev->num_tc = 0;
1958 return;
1959 }
1960
1961 /* Invalidated prio to tc mappings set to TC0 */
1962 for (i = 1; i < TC_BITMASK + 1; i++) {
1963 int q = netdev_get_prio_tc_map(dev, i);
1964
1965 tc = &dev->tc_to_txq[q];
1966 if (tc->offset + tc->count > txq) {
1967 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1968 i, q);
1969 netdev_set_prio_tc_map(dev, i, 0);
1970 }
1971 }
1972}
1973
1974int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
1975{
1976 if (dev->num_tc) {
1977 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1978 int i;
1979
1980 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
1981 if ((txq - tc->offset) < tc->count)
1982 return i;
1983 }
1984
1985 return -1;
1986 }
1987
1988 return 0;
1989}
1990
1991#ifdef CONFIG_XPS
1992static DEFINE_MUTEX(xps_map_mutex);
1993#define xmap_dereference(P) \
1994 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1995
1996static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
1997 int tci, u16 index)
1998{
1999 struct xps_map *map = NULL;
2000 int pos;
2001
2002 if (dev_maps)
2003 map = xmap_dereference(dev_maps->cpu_map[tci]);
2004 if (!map)
2005 return false;
2006
2007 for (pos = map->len; pos--;) {
2008 if (map->queues[pos] != index)
2009 continue;
2010
2011 if (map->len > 1) {
2012 map->queues[pos] = map->queues[--map->len];
2013 break;
2014 }
2015
2016 RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL);
2017 kfree_rcu(map, rcu);
2018 return false;
2019 }
2020
2021 return true;
2022}
2023
2024static bool remove_xps_queue_cpu(struct net_device *dev,
2025 struct xps_dev_maps *dev_maps,
2026 int cpu, u16 offset, u16 count)
2027{
2028 int num_tc = dev->num_tc ? : 1;
2029 bool active = false;
2030 int tci;
2031
2032 for (tci = cpu * num_tc; num_tc--; tci++) {
2033 int i, j;
2034
2035 for (i = count, j = offset; i--; j++) {
2036 if (!remove_xps_queue(dev_maps, cpu, j))
2037 break;
2038 }
2039
2040 active |= i < 0;
2041 }
2042
2043 return active;
2044}
2045
2046static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2047 u16 count)
2048{
2049 struct xps_dev_maps *dev_maps;
2050 int cpu, i;
2051 bool active = false;
2052
2053 mutex_lock(&xps_map_mutex);
2054 dev_maps = xmap_dereference(dev->xps_maps);
2055
2056 if (!dev_maps)
2057 goto out_no_maps;
2058
2059 for_each_possible_cpu(cpu)
2060 active |= remove_xps_queue_cpu(dev, dev_maps, cpu,
2061 offset, count);
2062
2063 if (!active) {
2064 RCU_INIT_POINTER(dev->xps_maps, NULL);
2065 kfree_rcu(dev_maps, rcu);
2066 }
2067
2068 for (i = offset + (count - 1); count--; i--)
2069 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2070 NUMA_NO_NODE);
2071
2072out_no_maps:
2073 mutex_unlock(&xps_map_mutex);
2074}
2075
2076static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2077{
2078 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2079}
2080
2081static struct xps_map *expand_xps_map(struct xps_map *map,
2082 int cpu, u16 index)
2083{
2084 struct xps_map *new_map;
2085 int alloc_len = XPS_MIN_MAP_ALLOC;
2086 int i, pos;
2087
2088 for (pos = 0; map && pos < map->len; pos++) {
2089 if (map->queues[pos] != index)
2090 continue;
2091 return map;
2092 }
2093
2094 /* Need to add queue to this CPU's existing map */
2095 if (map) {
2096 if (pos < map->alloc_len)
2097 return map;
2098
2099 alloc_len = map->alloc_len * 2;
2100 }
2101
2102 /* Need to allocate new map to store queue on this CPU's map */
2103 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2104 cpu_to_node(cpu));
2105 if (!new_map)
2106 return NULL;
2107
2108 for (i = 0; i < pos; i++)
2109 new_map->queues[i] = map->queues[i];
2110 new_map->alloc_len = alloc_len;
2111 new_map->len = pos;
2112
2113 return new_map;
2114}
2115
2116int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2117 u16 index)
2118{
2119 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2120 int i, cpu, tci, numa_node_id = -2;
2121 int maps_sz, num_tc = 1, tc = 0;
2122 struct xps_map *map, *new_map;
2123 bool active = false;
2124
2125 if (dev->num_tc) {
2126 num_tc = dev->num_tc;
2127 tc = netdev_txq_to_tc(dev, index);
2128 if (tc < 0)
2129 return -EINVAL;
2130 }
2131
2132 maps_sz = XPS_DEV_MAPS_SIZE(num_tc);
2133 if (maps_sz < L1_CACHE_BYTES)
2134 maps_sz = L1_CACHE_BYTES;
2135
2136 mutex_lock(&xps_map_mutex);
2137
2138 dev_maps = xmap_dereference(dev->xps_maps);
2139
2140 /* allocate memory for queue storage */
2141 for_each_cpu_and(cpu, cpu_online_mask, mask) {
2142 if (!new_dev_maps)
2143 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2144 if (!new_dev_maps) {
2145 mutex_unlock(&xps_map_mutex);
2146 return -ENOMEM;
2147 }
2148
2149 tci = cpu * num_tc + tc;
2150 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) :
2151 NULL;
2152
2153 map = expand_xps_map(map, cpu, index);
2154 if (!map)
2155 goto error;
2156
2157 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2158 }
2159
2160 if (!new_dev_maps)
2161 goto out_no_new_maps;
2162
2163 for_each_possible_cpu(cpu) {
2164 /* copy maps belonging to foreign traffic classes */
2165 for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) {
2166 /* fill in the new device map from the old device map */
2167 map = xmap_dereference(dev_maps->cpu_map[tci]);
2168 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2169 }
2170
2171 /* We need to explicitly update tci as prevous loop
2172 * could break out early if dev_maps is NULL.
2173 */
2174 tci = cpu * num_tc + tc;
2175
2176 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2177 /* add queue to CPU maps */
2178 int pos = 0;
2179
2180 map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2181 while ((pos < map->len) && (map->queues[pos] != index))
2182 pos++;
2183
2184 if (pos == map->len)
2185 map->queues[map->len++] = index;
2186#ifdef CONFIG_NUMA
2187 if (numa_node_id == -2)
2188 numa_node_id = cpu_to_node(cpu);
2189 else if (numa_node_id != cpu_to_node(cpu))
2190 numa_node_id = -1;
2191#endif
2192 } else if (dev_maps) {
2193 /* fill in the new device map from the old device map */
2194 map = xmap_dereference(dev_maps->cpu_map[tci]);
2195 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2196 }
2197
2198 /* copy maps belonging to foreign traffic classes */
2199 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2200 /* fill in the new device map from the old device map */
2201 map = xmap_dereference(dev_maps->cpu_map[tci]);
2202 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2203 }
2204 }
2205
2206 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2207
2208 /* Cleanup old maps */
2209 if (!dev_maps)
2210 goto out_no_old_maps;
2211
2212 for_each_possible_cpu(cpu) {
2213 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2214 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2215 map = xmap_dereference(dev_maps->cpu_map[tci]);
2216 if (map && map != new_map)
2217 kfree_rcu(map, rcu);
2218 }
2219 }
2220
2221 kfree_rcu(dev_maps, rcu);
2222
2223out_no_old_maps:
2224 dev_maps = new_dev_maps;
2225 active = true;
2226
2227out_no_new_maps:
2228 /* update Tx queue numa node */
2229 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2230 (numa_node_id >= 0) ? numa_node_id :
2231 NUMA_NO_NODE);
2232
2233 if (!dev_maps)
2234 goto out_no_maps;
2235
2236 /* removes queue from unused CPUs */
2237 for_each_possible_cpu(cpu) {
2238 for (i = tc, tci = cpu * num_tc; i--; tci++)
2239 active |= remove_xps_queue(dev_maps, tci, index);
2240 if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu))
2241 active |= remove_xps_queue(dev_maps, tci, index);
2242 for (i = num_tc - tc, tci++; --i; tci++)
2243 active |= remove_xps_queue(dev_maps, tci, index);
2244 }
2245
2246 /* free map if not active */
2247 if (!active) {
2248 RCU_INIT_POINTER(dev->xps_maps, NULL);
2249 kfree_rcu(dev_maps, rcu);
2250 }
2251
2252out_no_maps:
2253 mutex_unlock(&xps_map_mutex);
2254
2255 return 0;
2256error:
2257 /* remove any maps that we added */
2258 for_each_possible_cpu(cpu) {
2259 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2260 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2261 map = dev_maps ?
2262 xmap_dereference(dev_maps->cpu_map[tci]) :
2263 NULL;
2264 if (new_map && new_map != map)
2265 kfree(new_map);
2266 }
2267 }
2268
2269 mutex_unlock(&xps_map_mutex);
2270
2271 kfree(new_dev_maps);
2272 return -ENOMEM;
2273}
2274EXPORT_SYMBOL(netif_set_xps_queue);
2275
2276#endif
2277void netdev_reset_tc(struct net_device *dev)
2278{
2279#ifdef CONFIG_XPS
2280 netif_reset_xps_queues_gt(dev, 0);
2281#endif
2282 dev->num_tc = 0;
2283 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2284 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2285}
2286EXPORT_SYMBOL(netdev_reset_tc);
2287
2288int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2289{
2290 if (tc >= dev->num_tc)
2291 return -EINVAL;
2292
2293#ifdef CONFIG_XPS
2294 netif_reset_xps_queues(dev, offset, count);
2295#endif
2296 dev->tc_to_txq[tc].count = count;
2297 dev->tc_to_txq[tc].offset = offset;
2298 return 0;
2299}
2300EXPORT_SYMBOL(netdev_set_tc_queue);
2301
2302int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2303{
2304 if (num_tc > TC_MAX_QUEUE)
2305 return -EINVAL;
2306
2307#ifdef CONFIG_XPS
2308 netif_reset_xps_queues_gt(dev, 0);
2309#endif
2310 dev->num_tc = num_tc;
2311 return 0;
2312}
2313EXPORT_SYMBOL(netdev_set_num_tc);
2314
2315/*
2316 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2317 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2318 */
2319int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2320{
2321 int rc;
2322
2323 if (txq < 1 || txq > dev->num_tx_queues)
2324 return -EINVAL;
2325
2326 if (dev->reg_state == NETREG_REGISTERED ||
2327 dev->reg_state == NETREG_UNREGISTERING) {
2328 ASSERT_RTNL();
2329
2330 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2331 txq);
2332 if (rc)
2333 return rc;
2334
2335 if (dev->num_tc)
2336 netif_setup_tc(dev, txq);
2337
2338 if (txq < dev->real_num_tx_queues) {
2339 qdisc_reset_all_tx_gt(dev, txq);
2340#ifdef CONFIG_XPS
2341 netif_reset_xps_queues_gt(dev, txq);
2342#endif
2343 }
2344 }
2345
2346 dev->real_num_tx_queues = txq;
2347 return 0;
2348}
2349EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2350
2351#ifdef CONFIG_SYSFS
2352/**
2353 * netif_set_real_num_rx_queues - set actual number of RX queues used
2354 * @dev: Network device
2355 * @rxq: Actual number of RX queues
2356 *
2357 * This must be called either with the rtnl_lock held or before
2358 * registration of the net device. Returns 0 on success, or a
2359 * negative error code. If called before registration, it always
2360 * succeeds.
2361 */
2362int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2363{
2364 int rc;
2365
2366 if (rxq < 1 || rxq > dev->num_rx_queues)
2367 return -EINVAL;
2368
2369 if (dev->reg_state == NETREG_REGISTERED) {
2370 ASSERT_RTNL();
2371
2372 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2373 rxq);
2374 if (rc)
2375 return rc;
2376 }
2377
2378 dev->real_num_rx_queues = rxq;
2379 return 0;
2380}
2381EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2382#endif
2383
2384/**
2385 * netif_get_num_default_rss_queues - default number of RSS queues
2386 *
2387 * This routine should set an upper limit on the number of RSS queues
2388 * used by default by multiqueue devices.
2389 */
2390int netif_get_num_default_rss_queues(void)
2391{
2392 return is_kdump_kernel() ?
2393 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2394}
2395EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2396
2397static void __netif_reschedule(struct Qdisc *q)
2398{
2399 struct softnet_data *sd;
2400 unsigned long flags;
2401
2402 local_irq_save(flags);
2403 sd = this_cpu_ptr(&softnet_data);
2404 q->next_sched = NULL;
2405 *sd->output_queue_tailp = q;
2406 sd->output_queue_tailp = &q->next_sched;
2407 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2408 local_irq_restore(flags);
2409}
2410
2411void __netif_schedule(struct Qdisc *q)
2412{
2413 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2414 __netif_reschedule(q);
2415}
2416EXPORT_SYMBOL(__netif_schedule);
2417
2418struct dev_kfree_skb_cb {
2419 enum skb_free_reason reason;
2420};
2421
2422static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2423{
2424 return (struct dev_kfree_skb_cb *)skb->cb;
2425}
2426
2427void netif_schedule_queue(struct netdev_queue *txq)
2428{
2429 rcu_read_lock();
2430 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2431 struct Qdisc *q = rcu_dereference(txq->qdisc);
2432
2433 __netif_schedule(q);
2434 }
2435 rcu_read_unlock();
2436}
2437EXPORT_SYMBOL(netif_schedule_queue);
2438
2439void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2440{
2441 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2442 struct Qdisc *q;
2443
2444 rcu_read_lock();
2445 q = rcu_dereference(dev_queue->qdisc);
2446 __netif_schedule(q);
2447 rcu_read_unlock();
2448 }
2449}
2450EXPORT_SYMBOL(netif_tx_wake_queue);
2451
2452void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2453{
2454 unsigned long flags;
2455
2456 if (unlikely(!skb))
2457 return;
2458
2459 if (likely(atomic_read(&skb->users) == 1)) {
2460 smp_rmb();
2461 atomic_set(&skb->users, 0);
2462 } else if (likely(!atomic_dec_and_test(&skb->users))) {
2463 return;
2464 }
2465 get_kfree_skb_cb(skb)->reason = reason;
2466 local_irq_save(flags);
2467 skb->next = __this_cpu_read(softnet_data.completion_queue);
2468 __this_cpu_write(softnet_data.completion_queue, skb);
2469 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2470 local_irq_restore(flags);
2471}
2472EXPORT_SYMBOL(__dev_kfree_skb_irq);
2473
2474void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2475{
2476 if (in_irq() || irqs_disabled())
2477 __dev_kfree_skb_irq(skb, reason);
2478 else
2479 dev_kfree_skb(skb);
2480}
2481EXPORT_SYMBOL(__dev_kfree_skb_any);
2482
2483
2484/**
2485 * netif_device_detach - mark device as removed
2486 * @dev: network device
2487 *
2488 * Mark device as removed from system and therefore no longer available.
2489 */
2490void netif_device_detach(struct net_device *dev)
2491{
2492 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2493 netif_running(dev)) {
2494 netif_tx_stop_all_queues(dev);
2495 }
2496}
2497EXPORT_SYMBOL(netif_device_detach);
2498
2499/**
2500 * netif_device_attach - mark device as attached
2501 * @dev: network device
2502 *
2503 * Mark device as attached from system and restart if needed.
2504 */
2505void netif_device_attach(struct net_device *dev)
2506{
2507 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2508 netif_running(dev)) {
2509 netif_tx_wake_all_queues(dev);
2510 __netdev_watchdog_up(dev);
2511 }
2512}
2513EXPORT_SYMBOL(netif_device_attach);
2514
2515/*
2516 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2517 * to be used as a distribution range.
2518 */
2519u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2520 unsigned int num_tx_queues)
2521{
2522 u32 hash;
2523 u16 qoffset = 0;
2524 u16 qcount = num_tx_queues;
2525
2526 if (skb_rx_queue_recorded(skb)) {
2527 hash = skb_get_rx_queue(skb);
2528 while (unlikely(hash >= num_tx_queues))
2529 hash -= num_tx_queues;
2530 return hash;
2531 }
2532
2533 if (dev->num_tc) {
2534 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2535
2536 qoffset = dev->tc_to_txq[tc].offset;
2537 qcount = dev->tc_to_txq[tc].count;
2538 }
2539
2540 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2541}
2542EXPORT_SYMBOL(__skb_tx_hash);
2543
2544static void skb_warn_bad_offload(const struct sk_buff *skb)
2545{
2546 static const netdev_features_t null_features;
2547 struct net_device *dev = skb->dev;
2548 const char *name = "";
2549
2550 if (!net_ratelimit())
2551 return;
2552
2553 if (dev) {
2554 if (dev->dev.parent)
2555 name = dev_driver_string(dev->dev.parent);
2556 else
2557 name = netdev_name(dev);
2558 }
2559 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2560 "gso_type=%d ip_summed=%d\n",
2561 name, dev ? &dev->features : &null_features,
2562 skb->sk ? &skb->sk->sk_route_caps : &null_features,
2563 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2564 skb_shinfo(skb)->gso_type, skb->ip_summed);
2565}
2566
2567/*
2568 * Invalidate hardware checksum when packet is to be mangled, and
2569 * complete checksum manually on outgoing path.
2570 */
2571int skb_checksum_help(struct sk_buff *skb)
2572{
2573 __wsum csum;
2574 int ret = 0, offset;
2575
2576 if (skb->ip_summed == CHECKSUM_COMPLETE)
2577 goto out_set_summed;
2578
2579 if (unlikely(skb_shinfo(skb)->gso_size)) {
2580 skb_warn_bad_offload(skb);
2581 return -EINVAL;
2582 }
2583
2584 /* Before computing a checksum, we should make sure no frag could
2585 * be modified by an external entity : checksum could be wrong.
2586 */
2587 if (skb_has_shared_frag(skb)) {
2588 ret = __skb_linearize(skb);
2589 if (ret)
2590 goto out;
2591 }
2592
2593 offset = skb_checksum_start_offset(skb);
2594 BUG_ON(offset >= skb_headlen(skb));
2595 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2596
2597 offset += skb->csum_offset;
2598 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2599
2600 if (skb_cloned(skb) &&
2601 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2602 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2603 if (ret)
2604 goto out;
2605 }
2606
2607 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2608out_set_summed:
2609 skb->ip_summed = CHECKSUM_NONE;
2610out:
2611 return ret;
2612}
2613EXPORT_SYMBOL(skb_checksum_help);
2614
2615__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2616{
2617 __be16 type = skb->protocol;
2618
2619 /* Tunnel gso handlers can set protocol to ethernet. */
2620 if (type == htons(ETH_P_TEB)) {
2621 struct ethhdr *eth;
2622
2623 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2624 return 0;
2625
2626 eth = (struct ethhdr *)skb_mac_header(skb);
2627 type = eth->h_proto;
2628 }
2629
2630 return __vlan_get_protocol(skb, type, depth);
2631}
2632
2633/**
2634 * skb_mac_gso_segment - mac layer segmentation handler.
2635 * @skb: buffer to segment
2636 * @features: features for the output path (see dev->features)
2637 */
2638struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2639 netdev_features_t features)
2640{
2641 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2642 struct packet_offload *ptype;
2643 int vlan_depth = skb->mac_len;
2644 __be16 type = skb_network_protocol(skb, &vlan_depth);
2645
2646 if (unlikely(!type))
2647 return ERR_PTR(-EINVAL);
2648
2649 __skb_pull(skb, vlan_depth);
2650
2651 rcu_read_lock();
2652 list_for_each_entry_rcu(ptype, &offload_base, list) {
2653 if (ptype->type == type && ptype->callbacks.gso_segment) {
2654 segs = ptype->callbacks.gso_segment(skb, features);
2655 break;
2656 }
2657 }
2658 rcu_read_unlock();
2659
2660 __skb_push(skb, skb->data - skb_mac_header(skb));
2661
2662 return segs;
2663}
2664EXPORT_SYMBOL(skb_mac_gso_segment);
2665
2666
2667/* openvswitch calls this on rx path, so we need a different check.
2668 */
2669static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2670{
2671 if (tx_path)
2672 return skb->ip_summed != CHECKSUM_PARTIAL &&
2673 skb->ip_summed != CHECKSUM_NONE;
2674
2675 return skb->ip_summed == CHECKSUM_NONE;
2676}
2677
2678/**
2679 * __skb_gso_segment - Perform segmentation on skb.
2680 * @skb: buffer to segment
2681 * @features: features for the output path (see dev->features)
2682 * @tx_path: whether it is called in TX path
2683 *
2684 * This function segments the given skb and returns a list of segments.
2685 *
2686 * It may return NULL if the skb requires no segmentation. This is
2687 * only possible when GSO is used for verifying header integrity.
2688 *
2689 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
2690 */
2691struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2692 netdev_features_t features, bool tx_path)
2693{
2694 struct sk_buff *segs;
2695
2696 if (unlikely(skb_needs_check(skb, tx_path))) {
2697 int err;
2698
2699 /* We're going to init ->check field in TCP or UDP header */
2700 err = skb_cow_head(skb, 0);
2701 if (err < 0)
2702 return ERR_PTR(err);
2703 }
2704
2705 /* Only report GSO partial support if it will enable us to
2706 * support segmentation on this frame without needing additional
2707 * work.
2708 */
2709 if (features & NETIF_F_GSO_PARTIAL) {
2710 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
2711 struct net_device *dev = skb->dev;
2712
2713 partial_features |= dev->features & dev->gso_partial_features;
2714 if (!skb_gso_ok(skb, features | partial_features))
2715 features &= ~NETIF_F_GSO_PARTIAL;
2716 }
2717
2718 BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2719 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2720
2721 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2722 SKB_GSO_CB(skb)->encap_level = 0;
2723
2724 skb_reset_mac_header(skb);
2725 skb_reset_mac_len(skb);
2726
2727 segs = skb_mac_gso_segment(skb, features);
2728
2729 if (unlikely(skb_needs_check(skb, tx_path)))
2730 skb_warn_bad_offload(skb);
2731
2732 return segs;
2733}
2734EXPORT_SYMBOL(__skb_gso_segment);
2735
2736/* Take action when hardware reception checksum errors are detected. */
2737#ifdef CONFIG_BUG
2738void netdev_rx_csum_fault(struct net_device *dev)
2739{
2740 if (net_ratelimit()) {
2741 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2742 dump_stack();
2743 }
2744}
2745EXPORT_SYMBOL(netdev_rx_csum_fault);
2746#endif
2747
2748/* Actually, we should eliminate this check as soon as we know, that:
2749 * 1. IOMMU is present and allows to map all the memory.
2750 * 2. No high memory really exists on this machine.
2751 */
2752
2753static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2754{
2755#ifdef CONFIG_HIGHMEM
2756 int i;
2757
2758 if (!(dev->features & NETIF_F_HIGHDMA)) {
2759 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2760 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2761
2762 if (PageHighMem(skb_frag_page(frag)))
2763 return 1;
2764 }
2765 }
2766
2767 if (PCI_DMA_BUS_IS_PHYS) {
2768 struct device *pdev = dev->dev.parent;
2769
2770 if (!pdev)
2771 return 0;
2772 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2773 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2774 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2775
2776 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2777 return 1;
2778 }
2779 }
2780#endif
2781 return 0;
2782}
2783
2784/* If MPLS offload request, verify we are testing hardware MPLS features
2785 * instead of standard features for the netdev.
2786 */
2787#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2788static netdev_features_t net_mpls_features(struct sk_buff *skb,
2789 netdev_features_t features,
2790 __be16 type)
2791{
2792 if (eth_p_mpls(type))
2793 features &= skb->dev->mpls_features;
2794
2795 return features;
2796}
2797#else
2798static netdev_features_t net_mpls_features(struct sk_buff *skb,
2799 netdev_features_t features,
2800 __be16 type)
2801{
2802 return features;
2803}
2804#endif
2805
2806static netdev_features_t harmonize_features(struct sk_buff *skb,
2807 netdev_features_t features)
2808{
2809 int tmp;
2810 __be16 type;
2811
2812 type = skb_network_protocol(skb, &tmp);
2813 features = net_mpls_features(skb, features, type);
2814
2815 if (skb->ip_summed != CHECKSUM_NONE &&
2816 !can_checksum_protocol(features, type)) {
2817 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2818 }
2819 if (illegal_highdma(skb->dev, skb))
2820 features &= ~NETIF_F_SG;
2821
2822 return features;
2823}
2824
2825netdev_features_t passthru_features_check(struct sk_buff *skb,
2826 struct net_device *dev,
2827 netdev_features_t features)
2828{
2829 return features;
2830}
2831EXPORT_SYMBOL(passthru_features_check);
2832
2833static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2834 struct net_device *dev,
2835 netdev_features_t features)
2836{
2837 return vlan_features_check(skb, features);
2838}
2839
2840static netdev_features_t gso_features_check(const struct sk_buff *skb,
2841 struct net_device *dev,
2842 netdev_features_t features)
2843{
2844 u16 gso_segs = skb_shinfo(skb)->gso_segs;
2845
2846 if (gso_segs > dev->gso_max_segs)
2847 return features & ~NETIF_F_GSO_MASK;
2848
2849 /* Support for GSO partial features requires software
2850 * intervention before we can actually process the packets
2851 * so we need to strip support for any partial features now
2852 * and we can pull them back in after we have partially
2853 * segmented the frame.
2854 */
2855 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
2856 features &= ~dev->gso_partial_features;
2857
2858 /* Make sure to clear the IPv4 ID mangling feature if the
2859 * IPv4 header has the potential to be fragmented.
2860 */
2861 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
2862 struct iphdr *iph = skb->encapsulation ?
2863 inner_ip_hdr(skb) : ip_hdr(skb);
2864
2865 if (!(iph->frag_off & htons(IP_DF)))
2866 features &= ~NETIF_F_TSO_MANGLEID;
2867 }
2868
2869 return features;
2870}
2871
2872netdev_features_t netif_skb_features(struct sk_buff *skb)
2873{
2874 struct net_device *dev = skb->dev;
2875 netdev_features_t features = dev->features;
2876
2877 if (skb_is_gso(skb))
2878 features = gso_features_check(skb, dev, features);
2879
2880 /* If encapsulation offload request, verify we are testing
2881 * hardware encapsulation features instead of standard
2882 * features for the netdev
2883 */
2884 if (skb->encapsulation)
2885 features &= dev->hw_enc_features;
2886
2887 if (skb_vlan_tagged(skb))
2888 features = netdev_intersect_features(features,
2889 dev->vlan_features |
2890 NETIF_F_HW_VLAN_CTAG_TX |
2891 NETIF_F_HW_VLAN_STAG_TX);
2892
2893 if (dev->netdev_ops->ndo_features_check)
2894 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2895 features);
2896 else
2897 features &= dflt_features_check(skb, dev, features);
2898
2899 return harmonize_features(skb, features);
2900}
2901EXPORT_SYMBOL(netif_skb_features);
2902
2903static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2904 struct netdev_queue *txq, bool more)
2905{
2906 unsigned int len;
2907 int rc;
2908
2909 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
2910 dev_queue_xmit_nit(skb, dev);
2911
2912 len = skb->len;
2913 trace_net_dev_start_xmit(skb, dev);
2914 rc = netdev_start_xmit(skb, dev, txq, more);
2915 trace_net_dev_xmit(skb, rc, dev, len);
2916
2917 return rc;
2918}
2919
2920struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2921 struct netdev_queue *txq, int *ret)
2922{
2923 struct sk_buff *skb = first;
2924 int rc = NETDEV_TX_OK;
2925
2926 while (skb) {
2927 struct sk_buff *next = skb->next;
2928
2929 skb->next = NULL;
2930 rc = xmit_one(skb, dev, txq, next != NULL);
2931 if (unlikely(!dev_xmit_complete(rc))) {
2932 skb->next = next;
2933 goto out;
2934 }
2935
2936 skb = next;
2937 if (netif_xmit_stopped(txq) && skb) {
2938 rc = NETDEV_TX_BUSY;
2939 break;
2940 }
2941 }
2942
2943out:
2944 *ret = rc;
2945 return skb;
2946}
2947
2948static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2949 netdev_features_t features)
2950{
2951 if (skb_vlan_tag_present(skb) &&
2952 !vlan_hw_offload_capable(features, skb->vlan_proto))
2953 skb = __vlan_hwaccel_push_inside(skb);
2954 return skb;
2955}
2956
2957static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
2958{
2959 netdev_features_t features;
2960
2961 features = netif_skb_features(skb);
2962 skb = validate_xmit_vlan(skb, features);
2963 if (unlikely(!skb))
2964 goto out_null;
2965
2966 if (netif_needs_gso(skb, features)) {
2967 struct sk_buff *segs;
2968
2969 segs = skb_gso_segment(skb, features);
2970 if (IS_ERR(segs)) {
2971 goto out_kfree_skb;
2972 } else if (segs) {
2973 consume_skb(skb);
2974 skb = segs;
2975 }
2976 } else {
2977 if (skb_needs_linearize(skb, features) &&
2978 __skb_linearize(skb))
2979 goto out_kfree_skb;
2980
2981 if (validate_xmit_xfrm(skb, features))
2982 goto out_kfree_skb;
2983
2984 /* If packet is not checksummed and device does not
2985 * support checksumming for this protocol, complete
2986 * checksumming here.
2987 */
2988 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2989 if (skb->encapsulation)
2990 skb_set_inner_transport_header(skb,
2991 skb_checksum_start_offset(skb));
2992 else
2993 skb_set_transport_header(skb,
2994 skb_checksum_start_offset(skb));
2995 if (!(features & NETIF_F_CSUM_MASK) &&
2996 skb_checksum_help(skb))
2997 goto out_kfree_skb;
2998 }
2999 }
3000
3001 return skb;
3002
3003out_kfree_skb:
3004 kfree_skb(skb);
3005out_null:
3006 atomic_long_inc(&dev->tx_dropped);
3007 return NULL;
3008}
3009
3010struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
3011{
3012 struct sk_buff *next, *head = NULL, *tail;
3013
3014 for (; skb != NULL; skb = next) {
3015 next = skb->next;
3016 skb->next = NULL;
3017
3018 /* in case skb wont be segmented, point to itself */
3019 skb->prev = skb;
3020
3021 skb = validate_xmit_skb(skb, dev);
3022 if (!skb)
3023 continue;
3024
3025 if (!head)
3026 head = skb;
3027 else
3028 tail->next = skb;
3029 /* If skb was segmented, skb->prev points to
3030 * the last segment. If not, it still contains skb.
3031 */
3032 tail = skb->prev;
3033 }
3034 return head;
3035}
3036EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3037
3038static void qdisc_pkt_len_init(struct sk_buff *skb)
3039{
3040 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3041
3042 qdisc_skb_cb(skb)->pkt_len = skb->len;
3043
3044 /* To get more precise estimation of bytes sent on wire,
3045 * we add to pkt_len the headers size of all segments
3046 */
3047 if (shinfo->gso_size) {
3048 unsigned int hdr_len;
3049 u16 gso_segs = shinfo->gso_segs;
3050
3051 /* mac layer + network layer */
3052 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3053
3054 /* + transport layer */
3055 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3056 hdr_len += tcp_hdrlen(skb);
3057 else
3058 hdr_len += sizeof(struct udphdr);
3059
3060 if (shinfo->gso_type & SKB_GSO_DODGY)
3061 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3062 shinfo->gso_size);
3063
3064 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3065 }
3066}
3067
3068static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3069 struct net_device *dev,
3070 struct netdev_queue *txq)
3071{
3072 spinlock_t *root_lock = qdisc_lock(q);
3073 struct sk_buff *to_free = NULL;
3074 bool contended;
3075 int rc;
3076
3077 qdisc_calculate_pkt_len(skb, q);
3078 /*
3079 * Heuristic to force contended enqueues to serialize on a
3080 * separate lock before trying to get qdisc main lock.
3081 * This permits qdisc->running owner to get the lock more
3082 * often and dequeue packets faster.
3083 */
3084 contended = qdisc_is_running(q);
3085 if (unlikely(contended))
3086 spin_lock(&q->busylock);
3087
3088 spin_lock(root_lock);
3089 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3090 __qdisc_drop(skb, &to_free);
3091 rc = NET_XMIT_DROP;
3092 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3093 qdisc_run_begin(q)) {
3094 /*
3095 * This is a work-conserving queue; there are no old skbs
3096 * waiting to be sent out; and the qdisc is not running -
3097 * xmit the skb directly.
3098 */
3099
3100 qdisc_bstats_update(q, skb);
3101
3102 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3103 if (unlikely(contended)) {
3104 spin_unlock(&q->busylock);
3105 contended = false;
3106 }
3107 __qdisc_run(q);
3108 } else
3109 qdisc_run_end(q);
3110
3111 rc = NET_XMIT_SUCCESS;
3112 } else {
3113 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3114 if (qdisc_run_begin(q)) {
3115 if (unlikely(contended)) {
3116 spin_unlock(&q->busylock);
3117 contended = false;
3118 }
3119 __qdisc_run(q);
3120 }
3121 }
3122 spin_unlock(root_lock);
3123 if (unlikely(to_free))
3124 kfree_skb_list(to_free);
3125 if (unlikely(contended))
3126 spin_unlock(&q->busylock);
3127 return rc;
3128}
3129
3130#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3131static void skb_update_prio(struct sk_buff *skb)
3132{
3133 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
3134
3135 if (!skb->priority && skb->sk && map) {
3136 unsigned int prioidx =
3137 sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
3138
3139 if (prioidx < map->priomap_len)
3140 skb->priority = map->priomap[prioidx];
3141 }
3142}
3143#else
3144#define skb_update_prio(skb)
3145#endif
3146
3147DEFINE_PER_CPU(int, xmit_recursion);
3148EXPORT_SYMBOL(xmit_recursion);
3149
3150/**
3151 * dev_loopback_xmit - loop back @skb
3152 * @net: network namespace this loopback is happening in
3153 * @sk: sk needed to be a netfilter okfn
3154 * @skb: buffer to transmit
3155 */
3156int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3157{
3158 skb_reset_mac_header(skb);
3159 __skb_pull(skb, skb_network_offset(skb));
3160 skb->pkt_type = PACKET_LOOPBACK;
3161 skb->ip_summed = CHECKSUM_UNNECESSARY;
3162 WARN_ON(!skb_dst(skb));
3163 skb_dst_force(skb);
3164 netif_rx_ni(skb);
3165 return 0;
3166}
3167EXPORT_SYMBOL(dev_loopback_xmit);
3168
3169#ifdef CONFIG_NET_EGRESS
3170static struct sk_buff *
3171sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3172{
3173 struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3174 struct tcf_result cl_res;
3175
3176 if (!cl)
3177 return skb;
3178
3179 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3180 qdisc_bstats_cpu_update(cl->q, skb);
3181
3182 switch (tc_classify(skb, cl, &cl_res, false)) {
3183 case TC_ACT_OK:
3184 case TC_ACT_RECLASSIFY:
3185 skb->tc_index = TC_H_MIN(cl_res.classid);
3186 break;
3187 case TC_ACT_SHOT:
3188 qdisc_qstats_cpu_drop(cl->q);
3189 *ret = NET_XMIT_DROP;
3190 kfree_skb(skb);
3191 return NULL;
3192 case TC_ACT_STOLEN:
3193 case TC_ACT_QUEUED:
3194 *ret = NET_XMIT_SUCCESS;
3195 consume_skb(skb);
3196 return NULL;
3197 case TC_ACT_REDIRECT:
3198 /* No need to push/pop skb's mac_header here on egress! */
3199 skb_do_redirect(skb);
3200 *ret = NET_XMIT_SUCCESS;
3201 return NULL;
3202 default:
3203 break;
3204 }
3205
3206 return skb;
3207}
3208#endif /* CONFIG_NET_EGRESS */
3209
3210static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3211{
3212#ifdef CONFIG_XPS
3213 struct xps_dev_maps *dev_maps;
3214 struct xps_map *map;
3215 int queue_index = -1;
3216
3217 rcu_read_lock();
3218 dev_maps = rcu_dereference(dev->xps_maps);
3219 if (dev_maps) {
3220 unsigned int tci = skb->sender_cpu - 1;
3221
3222 if (dev->num_tc) {
3223 tci *= dev->num_tc;
3224 tci += netdev_get_prio_tc_map(dev, skb->priority);
3225 }
3226
3227 map = rcu_dereference(dev_maps->cpu_map[tci]);
3228 if (map) {
3229 if (map->len == 1)
3230 queue_index = map->queues[0];
3231 else
3232 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3233 map->len)];
3234 if (unlikely(queue_index >= dev->real_num_tx_queues))
3235 queue_index = -1;
3236 }
3237 }
3238 rcu_read_unlock();
3239
3240 return queue_index;
3241#else
3242 return -1;
3243#endif
3244}
3245
3246static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3247{
3248 struct sock *sk = skb->sk;
3249 int queue_index = sk_tx_queue_get(sk);
3250
3251 if (queue_index < 0 || skb->ooo_okay ||
3252 queue_index >= dev->real_num_tx_queues) {
3253 int new_index = get_xps_queue(dev, skb);
3254
3255 if (new_index < 0)
3256 new_index = skb_tx_hash(dev, skb);
3257
3258 if (queue_index != new_index && sk &&
3259 sk_fullsock(sk) &&
3260 rcu_access_pointer(sk->sk_dst_cache))
3261 sk_tx_queue_set(sk, new_index);
3262
3263 queue_index = new_index;
3264 }
3265
3266 return queue_index;
3267}
3268
3269struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3270 struct sk_buff *skb,
3271 void *accel_priv)
3272{
3273 int queue_index = 0;
3274
3275#ifdef CONFIG_XPS
3276 u32 sender_cpu = skb->sender_cpu - 1;
3277
3278 if (sender_cpu >= (u32)NR_CPUS)
3279 skb->sender_cpu = raw_smp_processor_id() + 1;
3280#endif
3281
3282 if (dev->real_num_tx_queues != 1) {
3283 const struct net_device_ops *ops = dev->netdev_ops;
3284
3285 if (ops->ndo_select_queue)
3286 queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3287 __netdev_pick_tx);
3288 else
3289 queue_index = __netdev_pick_tx(dev, skb);
3290
3291 if (!accel_priv)
3292 queue_index = netdev_cap_txqueue(dev, queue_index);
3293 }
3294
3295 skb_set_queue_mapping(skb, queue_index);
3296 return netdev_get_tx_queue(dev, queue_index);
3297}
3298
3299/**
3300 * __dev_queue_xmit - transmit a buffer
3301 * @skb: buffer to transmit
3302 * @accel_priv: private data used for L2 forwarding offload
3303 *
3304 * Queue a buffer for transmission to a network device. The caller must
3305 * have set the device and priority and built the buffer before calling
3306 * this function. The function can be called from an interrupt.
3307 *
3308 * A negative errno code is returned on a failure. A success does not
3309 * guarantee the frame will be transmitted as it may be dropped due
3310 * to congestion or traffic shaping.
3311 *
3312 * -----------------------------------------------------------------------------------
3313 * I notice this method can also return errors from the queue disciplines,
3314 * including NET_XMIT_DROP, which is a positive value. So, errors can also
3315 * be positive.
3316 *
3317 * Regardless of the return value, the skb is consumed, so it is currently
3318 * difficult to retry a send to this method. (You can bump the ref count
3319 * before sending to hold a reference for retry if you are careful.)
3320 *
3321 * When calling this method, interrupts MUST be enabled. This is because
3322 * the BH enable code must have IRQs enabled so that it will not deadlock.
3323 * --BLG
3324 */
3325static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3326{
3327 struct net_device *dev = skb->dev;
3328 struct netdev_queue *txq;
3329 struct Qdisc *q;
3330 int rc = -ENOMEM;
3331
3332 skb_reset_mac_header(skb);
3333
3334 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3335 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3336
3337 /* Disable soft irqs for various locks below. Also
3338 * stops preemption for RCU.
3339 */
3340 rcu_read_lock_bh();
3341
3342 skb_update_prio(skb);
3343
3344 qdisc_pkt_len_init(skb);
3345#ifdef CONFIG_NET_CLS_ACT
3346 skb->tc_at_ingress = 0;
3347# ifdef CONFIG_NET_EGRESS
3348 if (static_key_false(&egress_needed)) {
3349 skb = sch_handle_egress(skb, &rc, dev);
3350 if (!skb)
3351 goto out;
3352 }
3353# endif
3354#endif
3355 /* If device/qdisc don't need skb->dst, release it right now while
3356 * its hot in this cpu cache.
3357 */
3358 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3359 skb_dst_drop(skb);
3360 else
3361 skb_dst_force(skb);
3362
3363 txq = netdev_pick_tx(dev, skb, accel_priv);
3364 q = rcu_dereference_bh(txq->qdisc);
3365
3366 trace_net_dev_queue(skb);
3367 if (q->enqueue) {
3368 rc = __dev_xmit_skb(skb, q, dev, txq);
3369 goto out;
3370 }
3371
3372 /* The device has no queue. Common case for software devices:
3373 * loopback, all the sorts of tunnels...
3374
3375 * Really, it is unlikely that netif_tx_lock protection is necessary
3376 * here. (f.e. loopback and IP tunnels are clean ignoring statistics
3377 * counters.)
3378 * However, it is possible, that they rely on protection
3379 * made by us here.
3380
3381 * Check this and shot the lock. It is not prone from deadlocks.
3382 *Either shot noqueue qdisc, it is even simpler 8)
3383 */
3384 if (dev->flags & IFF_UP) {
3385 int cpu = smp_processor_id(); /* ok because BHs are off */
3386
3387 if (txq->xmit_lock_owner != cpu) {
3388 if (unlikely(__this_cpu_read(xmit_recursion) >
3389 XMIT_RECURSION_LIMIT))
3390 goto recursion_alert;
3391
3392 skb = validate_xmit_skb(skb, dev);
3393 if (!skb)
3394 goto out;
3395
3396 HARD_TX_LOCK(dev, txq, cpu);
3397
3398 if (!netif_xmit_stopped(txq)) {
3399 __this_cpu_inc(xmit_recursion);
3400 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3401 __this_cpu_dec(xmit_recursion);
3402 if (dev_xmit_complete(rc)) {
3403 HARD_TX_UNLOCK(dev, txq);
3404 goto out;
3405 }
3406 }
3407 HARD_TX_UNLOCK(dev, txq);
3408 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3409 dev->name);
3410 } else {
3411 /* Recursion is detected! It is possible,
3412 * unfortunately
3413 */
3414recursion_alert:
3415 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3416 dev->name);
3417 }
3418 }
3419
3420 rc = -ENETDOWN;
3421 rcu_read_unlock_bh();
3422
3423 atomic_long_inc(&dev->tx_dropped);
3424 kfree_skb_list(skb);
3425 return rc;
3426out:
3427 rcu_read_unlock_bh();
3428 return rc;
3429}
3430
3431int dev_queue_xmit(struct sk_buff *skb)
3432{
3433 return __dev_queue_xmit(skb, NULL);
3434}
3435EXPORT_SYMBOL(dev_queue_xmit);
3436
3437int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3438{
3439 return __dev_queue_xmit(skb, accel_priv);
3440}
3441EXPORT_SYMBOL(dev_queue_xmit_accel);
3442
3443
3444/*************************************************************************
3445 * Receiver routines
3446 *************************************************************************/
3447
3448int netdev_max_backlog __read_mostly = 1000;
3449EXPORT_SYMBOL(netdev_max_backlog);
3450
3451int netdev_tstamp_prequeue __read_mostly = 1;
3452int netdev_budget __read_mostly = 300;
3453unsigned int __read_mostly netdev_budget_usecs = 2000;
3454int weight_p __read_mostly = 64; /* old backlog weight */
3455int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
3456int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
3457int dev_rx_weight __read_mostly = 64;
3458int dev_tx_weight __read_mostly = 64;
3459
3460/* Called with irq disabled */
3461static inline void ____napi_schedule(struct softnet_data *sd,
3462 struct napi_struct *napi)
3463{
3464 list_add_tail(&napi->poll_list, &sd->poll_list);
3465 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3466}
3467
3468#ifdef CONFIG_RPS
3469
3470/* One global table that all flow-based protocols share. */
3471struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3472EXPORT_SYMBOL(rps_sock_flow_table);
3473u32 rps_cpu_mask __read_mostly;
3474EXPORT_SYMBOL(rps_cpu_mask);
3475
3476struct static_key rps_needed __read_mostly;
3477EXPORT_SYMBOL(rps_needed);
3478struct static_key rfs_needed __read_mostly;
3479EXPORT_SYMBOL(rfs_needed);
3480
3481static struct rps_dev_flow *
3482set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3483 struct rps_dev_flow *rflow, u16 next_cpu)
3484{
3485 if (next_cpu < nr_cpu_ids) {
3486#ifdef CONFIG_RFS_ACCEL
3487 struct netdev_rx_queue *rxqueue;
3488 struct rps_dev_flow_table *flow_table;
3489 struct rps_dev_flow *old_rflow;
3490 u32 flow_id;
3491 u16 rxq_index;
3492 int rc;
3493
3494 /* Should we steer this flow to a different hardware queue? */
3495 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3496 !(dev->features & NETIF_F_NTUPLE))
3497 goto out;
3498 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3499 if (rxq_index == skb_get_rx_queue(skb))
3500 goto out;
3501
3502 rxqueue = dev->_rx + rxq_index;
3503 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3504 if (!flow_table)
3505 goto out;
3506 flow_id = skb_get_hash(skb) & flow_table->mask;
3507 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3508 rxq_index, flow_id);
3509 if (rc < 0)
3510 goto out;
3511 old_rflow = rflow;
3512 rflow = &flow_table->flows[flow_id];
3513 rflow->filter = rc;
3514 if (old_rflow->filter == rflow->filter)
3515 old_rflow->filter = RPS_NO_FILTER;
3516 out:
3517#endif
3518 rflow->last_qtail =
3519 per_cpu(softnet_data, next_cpu).input_queue_head;
3520 }
3521
3522 rflow->cpu = next_cpu;
3523 return rflow;
3524}
3525
3526/*
3527 * get_rps_cpu is called from netif_receive_skb and returns the target
3528 * CPU from the RPS map of the receiving queue for a given skb.
3529 * rcu_read_lock must be held on entry.
3530 */
3531static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3532 struct rps_dev_flow **rflowp)
3533{
3534 const struct rps_sock_flow_table *sock_flow_table;
3535 struct netdev_rx_queue *rxqueue = dev->_rx;
3536 struct rps_dev_flow_table *flow_table;
3537 struct rps_map *map;
3538 int cpu = -1;
3539 u32 tcpu;
3540 u32 hash;
3541
3542 if (skb_rx_queue_recorded(skb)) {
3543 u16 index = skb_get_rx_queue(skb);
3544
3545 if (unlikely(index >= dev->real_num_rx_queues)) {
3546 WARN_ONCE(dev->real_num_rx_queues > 1,
3547 "%s received packet on queue %u, but number "
3548 "of RX queues is %u\n",
3549 dev->name, index, dev->real_num_rx_queues);
3550 goto done;
3551 }
3552 rxqueue += index;
3553 }
3554
3555 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3556
3557 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3558 map = rcu_dereference(rxqueue->rps_map);
3559 if (!flow_table && !map)
3560 goto done;
3561
3562 skb_reset_network_header(skb);
3563 hash = skb_get_hash(skb);
3564 if (!hash)
3565 goto done;
3566
3567 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3568 if (flow_table && sock_flow_table) {
3569 struct rps_dev_flow *rflow;
3570 u32 next_cpu;
3571 u32 ident;
3572
3573 /* First check into global flow table if there is a match */
3574 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3575 if ((ident ^ hash) & ~rps_cpu_mask)
3576 goto try_rps;
3577
3578 next_cpu = ident & rps_cpu_mask;
3579
3580 /* OK, now we know there is a match,
3581 * we can look at the local (per receive queue) flow table
3582 */
3583 rflow = &flow_table->flows[hash & flow_table->mask];
3584 tcpu = rflow->cpu;
3585
3586 /*
3587 * If the desired CPU (where last recvmsg was done) is
3588 * different from current CPU (one in the rx-queue flow
3589 * table entry), switch if one of the following holds:
3590 * - Current CPU is unset (>= nr_cpu_ids).
3591 * - Current CPU is offline.
3592 * - The current CPU's queue tail has advanced beyond the
3593 * last packet that was enqueued using this table entry.
3594 * This guarantees that all previous packets for the flow
3595 * have been dequeued, thus preserving in order delivery.
3596 */
3597 if (unlikely(tcpu != next_cpu) &&
3598 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3599 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3600 rflow->last_qtail)) >= 0)) {
3601 tcpu = next_cpu;
3602 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3603 }
3604
3605 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3606 *rflowp = rflow;
3607 cpu = tcpu;
3608 goto done;
3609 }
3610 }
3611
3612try_rps:
3613
3614 if (map) {
3615 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3616 if (cpu_online(tcpu)) {
3617 cpu = tcpu;
3618 goto done;
3619 }
3620 }
3621
3622done:
3623 return cpu;
3624}
3625
3626#ifdef CONFIG_RFS_ACCEL
3627
3628/**
3629 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3630 * @dev: Device on which the filter was set
3631 * @rxq_index: RX queue index
3632 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3633 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3634 *
3635 * Drivers that implement ndo_rx_flow_steer() should periodically call
3636 * this function for each installed filter and remove the filters for
3637 * which it returns %true.
3638 */
3639bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3640 u32 flow_id, u16 filter_id)
3641{
3642 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3643 struct rps_dev_flow_table *flow_table;
3644 struct rps_dev_flow *rflow;
3645 bool expire = true;
3646 unsigned int cpu;
3647
3648 rcu_read_lock();
3649 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3650 if (flow_table && flow_id <= flow_table->mask) {
3651 rflow = &flow_table->flows[flow_id];
3652 cpu = ACCESS_ONCE(rflow->cpu);
3653 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3654 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3655 rflow->last_qtail) <
3656 (int)(10 * flow_table->mask)))
3657 expire = false;
3658 }
3659 rcu_read_unlock();
3660 return expire;
3661}
3662EXPORT_SYMBOL(rps_may_expire_flow);
3663
3664#endif /* CONFIG_RFS_ACCEL */
3665
3666/* Called from hardirq (IPI) context */
3667static void rps_trigger_softirq(void *data)
3668{
3669 struct softnet_data *sd = data;
3670
3671 ____napi_schedule(sd, &sd->backlog);
3672 sd->received_rps++;
3673}
3674
3675#endif /* CONFIG_RPS */
3676
3677/*
3678 * Check if this softnet_data structure is another cpu one
3679 * If yes, queue it to our IPI list and return 1
3680 * If no, return 0
3681 */
3682static int rps_ipi_queued(struct softnet_data *sd)
3683{
3684#ifdef CONFIG_RPS
3685 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3686
3687 if (sd != mysd) {
3688 sd->rps_ipi_next = mysd->rps_ipi_list;
3689 mysd->rps_ipi_list = sd;
3690
3691 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3692 return 1;
3693 }
3694#endif /* CONFIG_RPS */
3695 return 0;
3696}
3697
3698#ifdef CONFIG_NET_FLOW_LIMIT
3699int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3700#endif
3701
3702static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3703{
3704#ifdef CONFIG_NET_FLOW_LIMIT
3705 struct sd_flow_limit *fl;
3706 struct softnet_data *sd;
3707 unsigned int old_flow, new_flow;
3708
3709 if (qlen < (netdev_max_backlog >> 1))
3710 return false;
3711
3712 sd = this_cpu_ptr(&softnet_data);
3713
3714 rcu_read_lock();
3715 fl = rcu_dereference(sd->flow_limit);
3716 if (fl) {
3717 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3718 old_flow = fl->history[fl->history_head];
3719 fl->history[fl->history_head] = new_flow;
3720
3721 fl->history_head++;
3722 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3723
3724 if (likely(fl->buckets[old_flow]))
3725 fl->buckets[old_flow]--;
3726
3727 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3728 fl->count++;
3729 rcu_read_unlock();
3730 return true;
3731 }
3732 }
3733 rcu_read_unlock();
3734#endif
3735 return false;
3736}
3737
3738/*
3739 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3740 * queue (may be a remote CPU queue).
3741 */
3742static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3743 unsigned int *qtail)
3744{
3745 struct softnet_data *sd;
3746 unsigned long flags;
3747 unsigned int qlen;
3748
3749 sd = &per_cpu(softnet_data, cpu);
3750
3751 local_irq_save(flags);
3752
3753 rps_lock(sd);
3754 if (!netif_running(skb->dev))
3755 goto drop;
3756 qlen = skb_queue_len(&sd->input_pkt_queue);
3757 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3758 if (qlen) {
3759enqueue:
3760 __skb_queue_tail(&sd->input_pkt_queue, skb);
3761 input_queue_tail_incr_save(sd, qtail);
3762 rps_unlock(sd);
3763 local_irq_restore(flags);
3764 return NET_RX_SUCCESS;
3765 }
3766
3767 /* Schedule NAPI for backlog device
3768 * We can use non atomic operation since we own the queue lock
3769 */
3770 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3771 if (!rps_ipi_queued(sd))
3772 ____napi_schedule(sd, &sd->backlog);
3773 }
3774 goto enqueue;
3775 }
3776
3777drop:
3778 sd->dropped++;
3779 rps_unlock(sd);
3780
3781 local_irq_restore(flags);
3782
3783 atomic_long_inc(&skb->dev->rx_dropped);
3784 kfree_skb(skb);
3785 return NET_RX_DROP;
3786}
3787
3788static int netif_rx_internal(struct sk_buff *skb)
3789{
3790 int ret;
3791
3792 net_timestamp_check(netdev_tstamp_prequeue, skb);
3793
3794 trace_netif_rx(skb);
3795#ifdef CONFIG_RPS
3796 if (static_key_false(&rps_needed)) {
3797 struct rps_dev_flow voidflow, *rflow = &voidflow;
3798 int cpu;
3799
3800 preempt_disable();
3801 rcu_read_lock();
3802
3803 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3804 if (cpu < 0)
3805 cpu = smp_processor_id();
3806
3807 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3808
3809 rcu_read_unlock();
3810 preempt_enable();
3811 } else
3812#endif
3813 {
3814 unsigned int qtail;
3815
3816 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3817 put_cpu();
3818 }
3819 return ret;
3820}
3821
3822/**
3823 * netif_rx - post buffer to the network code
3824 * @skb: buffer to post
3825 *
3826 * This function receives a packet from a device driver and queues it for
3827 * the upper (protocol) levels to process. It always succeeds. The buffer
3828 * may be dropped during processing for congestion control or by the
3829 * protocol layers.
3830 *
3831 * return values:
3832 * NET_RX_SUCCESS (no congestion)
3833 * NET_RX_DROP (packet was dropped)
3834 *
3835 */
3836
3837int netif_rx(struct sk_buff *skb)
3838{
3839 trace_netif_rx_entry(skb);
3840
3841 return netif_rx_internal(skb);
3842}
3843EXPORT_SYMBOL(netif_rx);
3844
3845int netif_rx_ni(struct sk_buff *skb)
3846{
3847 int err;
3848
3849 trace_netif_rx_ni_entry(skb);
3850
3851 preempt_disable();
3852 err = netif_rx_internal(skb);
3853 if (local_softirq_pending())
3854 do_softirq();
3855 preempt_enable();
3856
3857 return err;
3858}
3859EXPORT_SYMBOL(netif_rx_ni);
3860
3861static __latent_entropy void net_tx_action(struct softirq_action *h)
3862{
3863 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
3864
3865 if (sd->completion_queue) {
3866 struct sk_buff *clist;
3867
3868 local_irq_disable();
3869 clist = sd->completion_queue;
3870 sd->completion_queue = NULL;
3871 local_irq_enable();
3872
3873 while (clist) {
3874 struct sk_buff *skb = clist;
3875
3876 clist = clist->next;
3877
3878 WARN_ON(atomic_read(&skb->users));
3879 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3880 trace_consume_skb(skb);
3881 else
3882 trace_kfree_skb(skb, net_tx_action);
3883
3884 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
3885 __kfree_skb(skb);
3886 else
3887 __kfree_skb_defer(skb);
3888 }
3889
3890 __kfree_skb_flush();
3891 }
3892
3893 if (sd->output_queue) {
3894 struct Qdisc *head;
3895
3896 local_irq_disable();
3897 head = sd->output_queue;
3898 sd->output_queue = NULL;
3899 sd->output_queue_tailp = &sd->output_queue;
3900 local_irq_enable();
3901
3902 while (head) {
3903 struct Qdisc *q = head;
3904 spinlock_t *root_lock;
3905
3906 head = head->next_sched;
3907
3908 root_lock = qdisc_lock(q);
3909 spin_lock(root_lock);
3910 /* We need to make sure head->next_sched is read
3911 * before clearing __QDISC_STATE_SCHED
3912 */
3913 smp_mb__before_atomic();
3914 clear_bit(__QDISC_STATE_SCHED, &q->state);
3915 qdisc_run(q);
3916 spin_unlock(root_lock);
3917 }
3918 }
3919}
3920
3921#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
3922/* This hook is defined here for ATM LANE */
3923int (*br_fdb_test_addr_hook)(struct net_device *dev,
3924 unsigned char *addr) __read_mostly;
3925EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3926#endif
3927
3928static inline struct sk_buff *
3929sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
3930 struct net_device *orig_dev)
3931{
3932#ifdef CONFIG_NET_CLS_ACT
3933 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3934 struct tcf_result cl_res;
3935
3936 /* If there's at least one ingress present somewhere (so
3937 * we get here via enabled static key), remaining devices
3938 * that are not configured with an ingress qdisc will bail
3939 * out here.
3940 */
3941 if (!cl)
3942 return skb;
3943 if (*pt_prev) {
3944 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3945 *pt_prev = NULL;
3946 }
3947
3948 qdisc_skb_cb(skb)->pkt_len = skb->len;
3949 skb->tc_at_ingress = 1;
3950 qdisc_bstats_cpu_update(cl->q, skb);
3951
3952 switch (tc_classify(skb, cl, &cl_res, false)) {
3953 case TC_ACT_OK:
3954 case TC_ACT_RECLASSIFY:
3955 skb->tc_index = TC_H_MIN(cl_res.classid);
3956 break;
3957 case TC_ACT_SHOT:
3958 qdisc_qstats_cpu_drop(cl->q);
3959 kfree_skb(skb);
3960 return NULL;
3961 case TC_ACT_STOLEN:
3962 case TC_ACT_QUEUED:
3963 consume_skb(skb);
3964 return NULL;
3965 case TC_ACT_REDIRECT:
3966 /* skb_mac_header check was done by cls/act_bpf, so
3967 * we can safely push the L2 header back before
3968 * redirecting to another netdev
3969 */
3970 __skb_push(skb, skb->mac_len);
3971 skb_do_redirect(skb);
3972 return NULL;
3973 default:
3974 break;
3975 }
3976#endif /* CONFIG_NET_CLS_ACT */
3977 return skb;
3978}
3979
3980/**
3981 * netdev_is_rx_handler_busy - check if receive handler is registered
3982 * @dev: device to check
3983 *
3984 * Check if a receive handler is already registered for a given device.
3985 * Return true if there one.
3986 *
3987 * The caller must hold the rtnl_mutex.
3988 */
3989bool netdev_is_rx_handler_busy(struct net_device *dev)
3990{