1/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
10 * Authors: Ross Biro
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <linux/uaccess.h>
76#include <linux/bitops.h>
77#include <linux/capability.h>
78#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
81#include <linux/hash.h>
82#include <linux/slab.h>
83#include <linux/sched.h>
84#include <linux/mutex.h>
85#include <linux/string.h>
86#include <linux/mm.h>
87#include <linux/socket.h>
88#include <linux/sockios.h>
89#include <linux/errno.h>
90#include <linux/interrupt.h>
91#include <linux/if_ether.h>
92#include <linux/netdevice.h>
93#include <linux/etherdevice.h>
94#include <linux/ethtool.h>
95#include <linux/notifier.h>
96#include <linux/skbuff.h>
97#include <linux/bpf.h>
98#include <net/net_namespace.h>
99#include <net/sock.h>
100#include <net/busy_poll.h>
101#include <linux/rtnetlink.h>
102#include <linux/stat.h>
103#include <net/dst.h>
104#include <net/dst_metadata.h>
105#include <net/pkt_sched.h>
106#include <net/checksum.h>
107#include <net/xfrm.h>
108#include <linux/highmem.h>
109#include <linux/init.h>
110#include <linux/module.h>
111#include <linux/netpoll.h>
112#include <linux/rcupdate.h>
113#include <linux/delay.h>
114#include <net/iw_handler.h>
115#include <asm/current.h>
116#include <linux/audit.h>
117#include <linux/dmaengine.h>
118#include <linux/err.h>
119#include <linux/ctype.h>
120#include <linux/if_arp.h>
121#include <linux/if_vlan.h>
122#include <linux/ip.h>
123#include <net/ip.h>
124#include <net/mpls.h>
125#include <linux/ipv6.h>
126#include <linux/in.h>
127#include <linux/jhash.h>
128#include <linux/random.h>
129#include <trace/events/napi.h>
130#include <trace/events/net.h>
131#include <trace/events/skb.h>
132#include <linux/pci.h>
133#include <linux/inetdevice.h>
134#include <linux/cpu_rmap.h>
135#include <linux/static_key.h>
136#include <linux/hashtable.h>
137#include <linux/vmalloc.h>
138#include <linux/if_macvlan.h>
139#include <linux/errqueue.h>
140#include <linux/hrtimer.h>
141#include <linux/netfilter_ingress.h>
142#include <linux/crash_dump.h>
143
144#include "net-sysfs.h"
145
146/* Instead of increasing this, you should create a hash table. */
147#define MAX_GRO_SKBS 8
148
149/* This should be increased if a protocol with a bigger head is added. */
150#define GRO_MAX_HEAD (MAX_HEADER + 128)
151
152static DEFINE_SPINLOCK(ptype_lock);
153static DEFINE_SPINLOCK(offload_lock);
154struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
155struct list_head ptype_all __read_mostly; /* Taps */
156static struct list_head offload_base __read_mostly;
157
158static int netif_rx_internal(struct sk_buff *skb);
159static int call_netdevice_notifiers_info(unsigned long val,
160 struct net_device *dev,
161 struct netdev_notifier_info *info);
162
163/*
164 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
165 * semaphore.
166 *
167 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
168 *
169 * Writers must hold the rtnl semaphore while they loop through the
170 * dev_base_head list, and hold dev_base_lock for writing when they do the
171 * actual updates. This allows pure readers to access the list even
172 * while a writer is preparing to update it.
173 *
174 * To put it another way, dev_base_lock is held for writing only to
175 * protect against pure readers; the rtnl semaphore provides the
176 * protection against other writers.
177 *
178 * See, for example usages, register_netdevice() and
179 * unregister_netdevice(), which must be called with the rtnl
180 * semaphore held.
181 */
182DEFINE_RWLOCK(dev_base_lock);
183EXPORT_SYMBOL(dev_base_lock);
184
185/* protects napi_hash addition/deletion and napi_gen_id */
186static DEFINE_SPINLOCK(napi_hash_lock);
187
188static unsigned int napi_gen_id = NR_CPUS;
189static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
190
191static seqcount_t devnet_rename_seq;
192
193static inline void dev_base_seq_inc(struct net *net)
194{
195 while (++net->dev_base_seq == 0);
196}
197
198static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
199{
200 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
201
202 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
203}
204
205static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
206{
207 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
208}
209
210static inline void rps_lock(struct softnet_data *sd)
211{
212#ifdef CONFIG_RPS
213 spin_lock(&sd->input_pkt_queue.lock);
214#endif
215}
216
217static inline void rps_unlock(struct softnet_data *sd)
218{
219#ifdef CONFIG_RPS
220 spin_unlock(&sd->input_pkt_queue.lock);
221#endif
222}
223
224/* Device list insertion */
225static void list_netdevice(struct net_device *dev)
226{
227 struct net *net = dev_net(dev);
228
229 ASSERT_RTNL();
230
231 write_lock_bh(&dev_base_lock);
232 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
233 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
234 hlist_add_head_rcu(&dev->index_hlist,
235 dev_index_hash(net, dev->ifindex));
236 write_unlock_bh(&dev_base_lock);
237
238 dev_base_seq_inc(net);
239}
240
241/* Device list removal
242 * caller must respect a RCU grace period before freeing/reusing dev
243 */
244static void unlist_netdevice(struct net_device *dev)
245{
246 ASSERT_RTNL();
247
248 /* Unlink dev from the device chain */
249 write_lock_bh(&dev_base_lock);
250 list_del_rcu(&dev->dev_list);
251 hlist_del_rcu(&dev->name_hlist);
252 hlist_del_rcu(&dev->index_hlist);
253 write_unlock_bh(&dev_base_lock);
254
255 dev_base_seq_inc(dev_net(dev));
256}
257
258/*
259 * Our notifier list
260 */
261
262static RAW_NOTIFIER_HEAD(netdev_chain);
263
264/*
265 * Device drivers call our routines to queue packets here. We empty the
266 * queue in the local softnet handler.
267 */
268
269DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
270EXPORT_PER_CPU_SYMBOL(softnet_data);
271
272#ifdef CONFIG_LOCKDEP
273/*
274 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
275 * according to dev->type
276 */
277static const unsigned short netdev_lock_type[] =
278 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
279 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
280 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
281 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
282 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
283 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
284 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
285 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
286 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
287 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
288 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
289 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
290 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
291 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
292 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
293
294static const char *const netdev_lock_name[] =
295 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
296 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
297 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
298 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
299 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
300 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
301 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
302 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
303 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
304 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
305 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
306 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
307 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
308 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
309 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
310
311static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
312static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
313
314static inline unsigned short netdev_lock_pos(unsigned short dev_type)
315{
316 int i;
317
318 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
319 if (netdev_lock_type[i] == dev_type)
320 return i;
321 /* the last key is used by default */
322 return ARRAY_SIZE(netdev_lock_type) - 1;
323}
324
325static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
326 unsigned short dev_type)
327{
328 int i;
329
330 i = netdev_lock_pos(dev_type);
331 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
332 netdev_lock_name[i]);
333}
334
335static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
336{
337 int i;
338
339 i = netdev_lock_pos(dev->type);
340 lockdep_set_class_and_name(&dev->addr_list_lock,
341 &netdev_addr_lock_key[i],
342 netdev_lock_name[i]);
343}
344#else
345static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
346 unsigned short dev_type)
347{
348}
349static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
350{
351}
352#endif
353
354/*******************************************************************************
355
356 Protocol management and registration routines
357
358*******************************************************************************/
359
360/*
361 * Add a protocol ID to the list. Now that the input handler is
362 * smarter we can dispense with all the messy stuff that used to be
363 * here.
364 *
365 * BEWARE!!! Protocol handlers, mangling input packets,
366 * MUST BE last in hash buckets and checking protocol handlers
367 * MUST start from promiscuous ptype_all chain in net_bh.
368 * It is true now, do not change it.
369 * Explanation follows: if protocol handler, mangling packet, will
370 * be the first on list, it is not able to sense, that packet
371 * is cloned and should be copied-on-write, so that it will
372 * change it and subsequent readers will get broken packet.
373 * --ANK (980803)
374 */
375
376static inline struct list_head *ptype_head(const struct packet_type *pt)
377{
378 if (pt->type == htons(ETH_P_ALL))
379 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
380 else
381 return pt->dev ? &pt->dev->ptype_specific :
382 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
383}
384
385/**
386 * dev_add_pack - add packet handler
387 * @pt: packet type declaration
388 *
389 * Add a protocol handler to the networking stack. The passed &packet_type
390 * is linked into kernel lists and may not be freed until it has been
391 * removed from the kernel lists.
392 *
393 * This call does not sleep therefore it can not
394 * guarantee all CPU's that are in middle of receiving packets
395 * will see the new packet type (until the next received packet).
396 */
397
398void dev_add_pack(struct packet_type *pt)
399{
400 struct list_head *head = ptype_head(pt);
401
402 spin_lock(&ptype_lock);
403 list_add_rcu(&pt->list, head);
404 spin_unlock(&ptype_lock);
405}
406EXPORT_SYMBOL(dev_add_pack);
407
408/**
409 * __dev_remove_pack - remove packet handler
410 * @pt: packet type declaration
411 *
412 * Remove a protocol handler that was previously added to the kernel
413 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
414 * from the kernel lists and can be freed or reused once this function
415 * returns.
416 *
417 * The packet type might still be in use by receivers
418 * and must not be freed until after all the CPU's have gone
419 * through a quiescent state.
420 */
421void __dev_remove_pack(struct packet_type *pt)
422{
423 struct list_head *head = ptype_head(pt);
424 struct packet_type *pt1;
425
426 spin_lock(&ptype_lock);
427
428 list_for_each_entry(pt1, head, list) {
429 if (pt == pt1) {
430 list_del_rcu(&pt->list);
431 goto out;
432 }
433 }
434
435 pr_warn("dev_remove_pack: %p not found\n", pt);
436out:
437 spin_unlock(&ptype_lock);
438}
439EXPORT_SYMBOL(__dev_remove_pack);
440
441/**
442 * dev_remove_pack - remove packet handler
443 * @pt: packet type declaration
444 *
445 * Remove a protocol handler that was previously added to the kernel
446 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
447 * from the kernel lists and can be freed or reused once this function
448 * returns.
449 *
450 * This call sleeps to guarantee that no CPU is looking at the packet
451 * type after return.
452 */
453void dev_remove_pack(struct packet_type *pt)
454{
455 __dev_remove_pack(pt);
456
457 synchronize_net();
458}
459EXPORT_SYMBOL(dev_remove_pack);
460
461
462/**
463 * dev_add_offload - register offload handlers
464 * @po: protocol offload declaration
465 *
466 * Add protocol offload handlers to the networking stack. The passed
467 * &proto_offload is linked into kernel lists and may not be freed until
468 * it has been removed from the kernel lists.
469 *
470 * This call does not sleep therefore it can not
471 * guarantee all CPU's that are in middle of receiving packets
472 * will see the new offload handlers (until the next received packet).
473 */
474void dev_add_offload(struct packet_offload *po)
475{
476 struct packet_offload *elem;
477
478 spin_lock(&offload_lock);
479 list_for_each_entry(elem, &offload_base, list) {
480 if (po->priority < elem->priority)
481 break;
482 }
483 list_add_rcu(&po->list, elem->list.prev);
484 spin_unlock(&offload_lock);
485}
486EXPORT_SYMBOL(dev_add_offload);
487
488/**
489 * __dev_remove_offload - remove offload handler
490 * @po: packet offload declaration
491 *
492 * Remove a protocol offload handler that was previously added to the
493 * kernel offload handlers by dev_add_offload(). The passed &offload_type
494 * is removed from the kernel lists and can be freed or reused once this
495 * function returns.
496 *
497 * The packet type might still be in use by receivers
498 * and must not be freed until after all the CPU's have gone
499 * through a quiescent state.
500 */
501static void __dev_remove_offload(struct packet_offload *po)
502{
503 struct list_head *head = &offload_base;
504 struct packet_offload *po1;
505
506 spin_lock(&offload_lock);
507
508 list_for_each_entry(po1, head, list) {
509 if (po == po1) {
510 list_del_rcu(&po->list);
511 goto out;
512 }
513 }
514
515 pr_warn("dev_remove_offload: %p not found\n", po);
516out:
517 spin_unlock(&offload_lock);
518}
519
520/**
521 * dev_remove_offload - remove packet offload handler
522 * @po: packet offload declaration
523 *
524 * Remove a packet offload handler that was previously added to the kernel
525 * offload handlers by dev_add_offload(). The passed &offload_type is
526 * removed from the kernel lists and can be freed or reused once this
527 * function returns.
528 *
529 * This call sleeps to guarantee that no CPU is looking at the packet
530 * type after return.
531 */
532void dev_remove_offload(struct packet_offload *po)
533{
534 __dev_remove_offload(po);
535
536 synchronize_net();
537}
538EXPORT_SYMBOL(dev_remove_offload);
539
540/******************************************************************************
541
542 Device Boot-time Settings Routines
543
544*******************************************************************************/
545
546/* Boot time configuration table */
547static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
548
549/**
550 * netdev_boot_setup_add - add new setup entry
551 * @name: name of the device
552 * @map: configured settings for the device
553 *
554 * Adds new setup entry to the dev_boot_setup list. The function
555 * returns 0 on error and 1 on success. This is a generic routine to
556 * all netdevices.
557 */
558static int netdev_boot_setup_add(char *name, struct ifmap *map)
559{
560 struct netdev_boot_setup *s;
561 int i;
562
563 s = dev_boot_setup;
564 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
565 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
566 memset(s[i].name, 0, sizeof(s[i].name));
567 strlcpy(s[i].name, name, IFNAMSIZ);
568 memcpy(&s[i].map, map, sizeof(s[i].map));
569 break;
570 }
571 }
572
573 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
574}
575
576/**
577 * netdev_boot_setup_check - check boot time settings
578 * @dev: the netdevice
579 *
580 * Check boot time settings for the device.
581 * The found settings are set for the device to be used
582 * later in the device probing.
583 * Returns 0 if no settings found, 1 if they are.
584 */
585int netdev_boot_setup_check(struct net_device *dev)
586{
587 struct netdev_boot_setup *s = dev_boot_setup;
588 int i;
589
590 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
591 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
592 !strcmp(dev->name, s[i].name)) {
593 dev->irq = s[i].map.irq;
594 dev->base_addr = s[i].map.base_addr;
595 dev->mem_start = s[i].map.mem_start;
596 dev->mem_end = s[i].map.mem_end;
597 return 1;
598 }
599 }
600 return 0;
601}
602EXPORT_SYMBOL(netdev_boot_setup_check);
603
604
605/**
606 * netdev_boot_base - get address from boot time settings
607 * @prefix: prefix for network device
608 * @unit: id for network device
609 *
610 * Check boot time settings for the base address of device.
611 * The found settings are set for the device to be used
612 * later in the device probing.
613 * Returns 0 if no settings found.
614 */
615unsigned long netdev_boot_base(const char *prefix, int unit)
616{
617 const struct netdev_boot_setup *s = dev_boot_setup;
618 char name[IFNAMSIZ];
619 int i;
620
621 sprintf(name, "%s%d", prefix, unit);
622
623 /*
624 * If device already registered then return base of 1
625 * to indicate not to probe for this interface
626 */
627 if (__dev_get_by_name(&init_net, name))
628 return 1;
629
630 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
631 if (!strcmp(name, s[i].name))
632 return s[i].map.base_addr;
633 return 0;
634}
635
636/*
637 * Saves at boot time configured settings for any netdevice.
638 */
639int __init netdev_boot_setup(char *str)
640{
641 int ints[5];
642 struct ifmap map;
643
644 str = get_options(str, ARRAY_SIZE(ints), ints);
645 if (!str || !*str)
646 return 0;
647
648 /* Save settings */
649 memset(&map, 0, sizeof(map));
650 if (ints[0] > 0)
651 map.irq = ints[1];
652 if (ints[0] > 1)
653 map.base_addr = ints[2];
654 if (ints[0] > 2)
655 map.mem_start = ints[3];
656 if (ints[0] > 3)
657 map.mem_end = ints[4];
658
659 /* Add new entry to the list */
660 return netdev_boot_setup_add(str, &map);
661}
662
663__setup("netdev=", netdev_boot_setup);
664
665/*******************************************************************************
666
667 Device Interface Subroutines
668
669*******************************************************************************/
670
671/**
672 * dev_get_iflink - get 'iflink' value of a interface
673 * @dev: targeted interface
674 *
675 * Indicates the ifindex the interface is linked to.
676 * Physical interfaces have the same 'ifindex' and 'iflink' values.
677 */
678
679int dev_get_iflink(const struct net_device *dev)
680{
681 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
682 return dev->netdev_ops->ndo_get_iflink(dev);
683
684 return dev->ifindex;
685}
686EXPORT_SYMBOL(dev_get_iflink);
687
688/**
689 * dev_fill_metadata_dst - Retrieve tunnel egress information.
690 * @dev: targeted interface
691 * @skb: The packet.
692 *
693 * For better visibility of tunnel traffic OVS needs to retrieve
694 * egress tunnel information for a packet. Following API allows
695 * user to get this info.
696 */
697int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
698{
699 struct ip_tunnel_info *info;
700
701 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
702 return -EINVAL;
703
704 info = skb_tunnel_info_unclone(skb);
705 if (!info)
706 return -ENOMEM;
707 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
708 return -EINVAL;
709
710 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
711}
712EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
713
714/**
715 * __dev_get_by_name - find a device by its name
716 * @net: the applicable net namespace
717 * @name: name to find
718 *
719 * Find an interface by name. Must be called under RTNL semaphore
720 * or @dev_base_lock. If the name is found a pointer to the device
721 * is returned. If the name is not found then %NULL is returned. The
722 * reference counters are not incremented so the caller must be
723 * careful with locks.
724 */
725
726struct net_device *__dev_get_by_name(struct net *net, const char *name)
727{
728 struct net_device *dev;
729 struct hlist_head *head = dev_name_hash(net, name);
730
731 hlist_for_each_entry(dev, head, name_hlist)
732 if (!strncmp(dev->name, name, IFNAMSIZ))
733 return dev;
734
735 return NULL;
736}
737EXPORT_SYMBOL(__dev_get_by_name);
738
739/**
740 * dev_get_by_name_rcu - find a device by its name
741 * @net: the applicable net namespace
742 * @name: name to find
743 *
744 * Find an interface by name.
745 * If the name is found a pointer to the device is returned.
746 * If the name is not found then %NULL is returned.
747 * The reference counters are not incremented so the caller must be
748 * careful with locks. The caller must hold RCU lock.
749 */
750
751struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
752{
753 struct net_device *dev;
754 struct hlist_head *head = dev_name_hash(net, name);
755
756 hlist_for_each_entry_rcu(dev, head, name_hlist)
757 if (!strncmp(dev->name, name, IFNAMSIZ))
758 return dev;
759
760 return NULL;
761}
762EXPORT_SYMBOL(dev_get_by_name_rcu);
763
764/**
765 * dev_get_by_name - find a device by its name
766 * @net: the applicable net namespace
767 * @name: name to find
768 *
769 * Find an interface by name. This can be called from any
770 * context and does its own locking. The returned handle has
771 * the usage count incremented and the caller must use dev_put() to
772 * release it when it is no longer needed. %NULL is returned if no
773 * matching device is found.
774 */
775
776struct net_device *dev_get_by_name(struct net *net, const char *name)
777{
778 struct net_device *dev;
779
780 rcu_read_lock();
781 dev = dev_get_by_name_rcu(net, name);
782 if (dev)
783 dev_hold(dev);
784 rcu_read_unlock();
785 return dev;
786}
787EXPORT_SYMBOL(dev_get_by_name);
788
789/**
790 * __dev_get_by_index - find a device by its ifindex
791 * @net: the applicable net namespace
792 * @ifindex: index of device
793 *
794 * Search for an interface by index. Returns %NULL if the device
795 * is not found or a pointer to the device. The device has not
796 * had its reference counter increased so the caller must be careful
797 * about locking. The caller must hold either the RTNL semaphore
798 * or @dev_base_lock.
799 */
800
801struct net_device *__dev_get_by_index(struct net *net, int ifindex)
802{
803 struct net_device *dev;
804 struct hlist_head *head = dev_index_hash(net, ifindex);
805
806 hlist_for_each_entry(dev, head, index_hlist)
807 if (dev->ifindex == ifindex)
808 return dev;
809
810 return NULL;
811}
812EXPORT_SYMBOL(__dev_get_by_index);
813
814/**
815 * dev_get_by_index_rcu - find a device by its ifindex
816 * @net: the applicable net namespace
817 * @ifindex: index of device
818 *
819 * Search for an interface by index. Returns %NULL if the device
820 * is not found or a pointer to the device. The device has not
821 * had its reference counter increased so the caller must be careful
822 * about locking. The caller must hold RCU lock.
823 */
824
825struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
826{
827 struct net_device *dev;
828 struct hlist_head *head = dev_index_hash(net, ifindex);
829
830 hlist_for_each_entry_rcu(dev, head, index_hlist)
831 if (dev->ifindex == ifindex)
832 return dev;
833
834 return NULL;
835}
836EXPORT_SYMBOL(dev_get_by_index_rcu);
837
838
839/**
840 * dev_get_by_index - find a device by its ifindex
841 * @net: the applicable net namespace
842 * @ifindex: index of device
843 *
844 * Search for an interface by index. Returns NULL if the device
845 * is not found or a pointer to the device. The device returned has
846 * had a reference added and the pointer is safe until the user calls
847 * dev_put to indicate they have finished with it.
848 */
849
850struct net_device *dev_get_by_index(struct net *net, int ifindex)
851{
852 struct net_device *dev;
853
854 rcu_read_lock();
855 dev = dev_get_by_index_rcu(net, ifindex);
856 if (dev)
857 dev_hold(dev);
858 rcu_read_unlock();
859 return dev;
860}
861EXPORT_SYMBOL(dev_get_by_index);
862
863/**
864 * netdev_get_name - get a netdevice name, knowing its ifindex.
865 * @net: network namespace
866 * @name: a pointer to the buffer where the name will be stored.
867 * @ifindex: the ifindex of the interface to get the name from.
868 *
869 * The use of raw_seqcount_begin() and cond_resched() before
870 * retrying is required as we want to give the writers a chance
871 * to complete when CONFIG_PREEMPT is not set.
872 */
873int netdev_get_name(struct net *net, char *name, int ifindex)
874{
875 struct net_device *dev;
876 unsigned int seq;
877
878retry:
879 seq = raw_seqcount_begin(&devnet_rename_seq);
880 rcu_read_lock();
881 dev = dev_get_by_index_rcu(net, ifindex);
882 if (!dev) {
883 rcu_read_unlock();
884 return -ENODEV;
885 }
886
887 strcpy(name, dev->name);
888 rcu_read_unlock();
889 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
890 cond_resched();
891 goto retry;
892 }
893
894 return 0;
895}
896
897/**
898 * dev_getbyhwaddr_rcu - find a device by its hardware address
899 * @net: the applicable net namespace
900 * @type: media type of device
901 * @ha: hardware address
902 *
903 * Search for an interface by MAC address. Returns NULL if the device
904 * is not found or a pointer to the device.
905 * The caller must hold RCU or RTNL.
906 * The returned device has not had its ref count increased
907 * and the caller must therefore be careful about locking
908 *
909 */
910
911struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
912 const char *ha)
913{
914 struct net_device *dev;
915
916 for_each_netdev_rcu(net, dev)
917 if (dev->type == type &&
918 !memcmp(dev->dev_addr, ha, dev->addr_len))
919 return dev;
920
921 return NULL;
922}
923EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
924
925struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
926{
927 struct net_device *dev;
928
929 ASSERT_RTNL();
930 for_each_netdev(net, dev)
931 if (dev->type == type)
932 return dev;
933
934 return NULL;
935}
936EXPORT_SYMBOL(__dev_getfirstbyhwtype);
937
938struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
939{
940 struct net_device *dev, *ret = NULL;
941
942 rcu_read_lock();
943 for_each_netdev_rcu(net, dev)
944 if (dev->type == type) {
945 dev_hold(dev);
946 ret = dev;
947 break;
948 }
949 rcu_read_unlock();
950 return ret;
951}
952EXPORT_SYMBOL(dev_getfirstbyhwtype);
953
954/**
955 * __dev_get_by_flags - find any device with given flags
956 * @net: the applicable net namespace
957 * @if_flags: IFF_* values
958 * @mask: bitmask of bits in if_flags to check
959 *
960 * Search for any interface with the given flags. Returns NULL if a device
961 * is not found or a pointer to the device. Must be called inside
962 * rtnl_lock(), and result refcount is unchanged.
963 */
964
965struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
966 unsigned short mask)
967{
968 struct net_device *dev, *ret;
969
970 ASSERT_RTNL();
971
972 ret = NULL;
973 for_each_netdev(net, dev) {
974 if (((dev->flags ^ if_flags) & mask) == 0) {
975 ret = dev;
976 break;
977 }
978 }
979 return ret;
980}
981EXPORT_SYMBOL(__dev_get_by_flags);
982
983/**
984 * dev_valid_name - check if name is okay for network device
985 * @name: name string
986 *
987 * Network device names need to be valid file names to
988 * to allow sysfs to work. We also disallow any kind of
989 * whitespace.
990 */
991bool dev_valid_name(const char *name)
992{
993 if (*name == '\0')
994 return false;
995 if (strlen(name) >= IFNAMSIZ)
996 return false;
997 if (!strcmp(name, ".") || !strcmp(name, ".."))
998 return false;
999
1000 while (*name) {
1001 if (*name == '/' || *name == ':' || isspace(*name))
1002 return false;
1003 name++;
1004 }
1005 return true;
1006}
1007EXPORT_SYMBOL(dev_valid_name);
1008
1009/**
1010 * __dev_alloc_name - allocate a name for a device
1011 * @net: network namespace to allocate the device name in
1012 * @name: name format string
1013 * @buf: scratch buffer and result name string
1014 *
1015 * Passed a format string - eg "lt%d" it will try and find a suitable
1016 * id. It scans list of devices to build up a free map, then chooses
1017 * the first empty slot. The caller must hold the dev_base or rtnl lock
1018 * while allocating the name and adding the device in order to avoid
1019 * duplicates.
1020 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1021 * Returns the number of the unit assigned or a negative errno code.
1022 */
1023
1024static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1025{
1026 int i = 0;
1027 const char *p;
1028 const int max_netdevices = 8*PAGE_SIZE;
1029 unsigned long *inuse;
1030 struct net_device *d;
1031
1032 p = strnchr(name, IFNAMSIZ-1, '%');
1033 if (p) {
1034 /*
1035 * Verify the string as this thing may have come from
1036 * the user. There must be either one "%d" and no other "%"
1037 * characters.
1038 */
1039 if (p[1] != 'd' || strchr(p + 2, '%'))
1040 return -EINVAL;
1041
1042 /* Use one page as a bit array of possible slots */
1043 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1044 if (!inuse)
1045 return -ENOMEM;
1046
1047 for_each_netdev(net, d) {
1048 if (!sscanf(d->name, name, &i))
1049 continue;
1050 if (i < 0 || i >= max_netdevices)
1051 continue;
1052
1053 /* avoid cases where sscanf is not exact inverse of printf */
1054 snprintf(buf, IFNAMSIZ, name, i);
1055 if (!strncmp(buf, d->name, IFNAMSIZ))
1056 set_bit(i, inuse);
1057 }
1058
1059 i = find_first_zero_bit(inuse, max_netdevices);
1060 free_page((unsigned long) inuse);
1061 }
1062
1063 if (buf != name)
1064 snprintf(buf, IFNAMSIZ, name, i);
1065 if (!__dev_get_by_name(net, buf))
1066 return i;
1067
1068 /* It is possible to run out of possible slots
1069 * when the name is long and there isn't enough space left
1070 * for the digits, or if all bits are used.
1071 */
1072 return -ENFILE;
1073}
1074
1075/**
1076 * dev_alloc_name - allocate a name for a device
1077 * @dev: device
1078 * @name: name format string
1079 *
1080 * Passed a format string - eg "lt%d" it will try and find a suitable
1081 * id. It scans list of devices to build up a free map, then chooses
1082 * the first empty slot. The caller must hold the dev_base or rtnl lock
1083 * while allocating the name and adding the device in order to avoid
1084 * duplicates.
1085 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1086 * Returns the number of the unit assigned or a negative errno code.
1087 */
1088
1089int dev_alloc_name(struct net_device *dev, const char *name)
1090{
1091 char buf[IFNAMSIZ];
1092 struct net *net;
1093 int ret;
1094
1095 BUG_ON(!dev_net(dev));
1096 net = dev_net(dev);
1097 ret = __dev_alloc_name(net, name, buf);
1098 if (ret >= 0)
1099 strlcpy(dev->name, buf, IFNAMSIZ);
1100 return ret;
1101}
1102EXPORT_SYMBOL(dev_alloc_name);
1103
1104static int dev_alloc_name_ns(struct net *net,
1105 struct net_device *dev,
1106 const char *name)
1107{
1108 char buf[IFNAMSIZ];
1109 int ret;
1110
1111 ret = __dev_alloc_name(net, name, buf);
1112 if (ret >= 0)
1113 strlcpy(dev->name, buf, IFNAMSIZ);
1114 return ret;
1115}
1116
1117static int dev_get_valid_name(struct net *net,
1118 struct net_device *dev,
1119 const char *name)
1120{
1121 BUG_ON(!net);
1122
1123 if (!dev_valid_name(name))
1124 return -EINVAL;
1125
1126 if (strchr(name, '%'))
1127 return dev_alloc_name_ns(net, dev, name);
1128 else if (__dev_get_by_name(net, name))
1129 return -EEXIST;
1130 else if (dev->name != name)
1131 strlcpy(dev->name, name, IFNAMSIZ);
1132
1133 return 0;
1134}
1135
1136/**
1137 * dev_change_name - change name of a device
1138 * @dev: device
1139 * @newname: name (or format string) must be at least IFNAMSIZ
1140 *
1141 * Change name of a device, can pass format strings "eth%d".
1142 * for wildcarding.
1143 */
1144int dev_change_name(struct net_device *dev, const char *newname)
1145{
1146 unsigned char old_assign_type;
1147 char oldname[IFNAMSIZ];
1148 int err = 0;
1149 int ret;
1150 struct net *net;
1151
1152 ASSERT_RTNL();
1153 BUG_ON(!dev_net(dev));
1154
1155 net = dev_net(dev);
1156 if (dev->flags & IFF_UP)
1157 return -EBUSY;
1158
1159 write_seqcount_begin(&devnet_rename_seq);
1160
1161 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1162 write_seqcount_end(&devnet_rename_seq);
1163 return 0;
1164 }
1165
1166 memcpy(oldname, dev->name, IFNAMSIZ);
1167
1168 err = dev_get_valid_name(net, dev, newname);
1169 if (err < 0) {
1170 write_seqcount_end(&devnet_rename_seq);
1171 return err;
1172 }
1173
1174 if (oldname[0] && !strchr(oldname, '%'))
1175 netdev_info(dev, "renamed from %s\n", oldname);
1176
1177 old_assign_type = dev->name_assign_type;
1178 dev->name_assign_type = NET_NAME_RENAMED;
1179
1180rollback:
1181 ret = device_rename(&dev->dev, dev->name);
1182 if (ret) {
1183 memcpy(dev->name, oldname, IFNAMSIZ);
1184 dev->name_assign_type = old_assign_type;
1185 write_seqcount_end(&devnet_rename_seq);
1186 return ret;
1187 }
1188
1189 write_seqcount_end(&devnet_rename_seq);
1190
1191 netdev_adjacent_rename_links(dev, oldname);
1192
1193 write_lock_bh(&dev_base_lock);
1194 hlist_del_rcu(&dev->name_hlist);
1195 write_unlock_bh(&dev_base_lock);
1196
1197 synchronize_rcu();
1198
1199 write_lock_bh(&dev_base_lock);
1200 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1201 write_unlock_bh(&dev_base_lock);
1202
1203 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1204 ret = notifier_to_errno(ret);
1205
1206 if (ret) {
1207 /* err >= 0 after dev_alloc_name() or stores the first errno */
1208 if (err >= 0) {
1209 err = ret;
1210 write_seqcount_begin(&devnet_rename_seq);
1211 memcpy(dev->name, oldname, IFNAMSIZ);
1212 memcpy(oldname, newname, IFNAMSIZ);
1213 dev->name_assign_type = old_assign_type;
1214 old_assign_type = NET_NAME_RENAMED;
1215 goto rollback;
1216 } else {
1217 pr_err("%s: name change rollback failed: %d\n",
1218 dev->name, ret);
1219 }
1220 }
1221
1222 return err;
1223}
1224
1225/**
1226 * dev_set_alias - change ifalias of a device
1227 * @dev: device
1228 * @alias: name up to IFALIASZ
1229 * @len: limit of bytes to copy from info
1230 *
1231 * Set ifalias for a device,
1232 */
1233int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1234{
1235 char *new_ifalias;
1236
1237 ASSERT_RTNL();
1238
1239 if (len >= IFALIASZ)
1240 return -EINVAL;
1241
1242 if (!len) {
1243 kfree(dev->ifalias);
1244 dev->ifalias = NULL;
1245 return 0;
1246 }
1247
1248 new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1249 if (!new_ifalias)
1250 return -ENOMEM;
1251 dev->ifalias = new_ifalias;
1252
1253 strlcpy(dev->ifalias, alias, len+1);
1254 return len;
1255}
1256
1257
1258/**
1259 * netdev_features_change - device changes features
1260 * @dev: device to cause notification
1261 *
1262 * Called to indicate a device has changed features.
1263 */
1264void netdev_features_change(struct net_device *dev)
1265{
1266 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1267}
1268EXPORT_SYMBOL(netdev_features_change);
1269
1270/**
1271 * netdev_state_change - device changes state
1272 * @dev: device to cause notification
1273 *
1274 * Called to indicate a device has changed state. This function calls
1275 * the notifier chains for netdev_chain and sends a NEWLINK message
1276 * to the routing socket.
1277 */
1278void netdev_state_change(struct net_device *dev)
1279{
1280 if (dev->flags & IFF_UP) {
1281 struct netdev_notifier_change_info change_info;
1282
1283 change_info.flags_changed = 0;
1284 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1285 &change_info.info);
1286 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1287 }
1288}
1289EXPORT_SYMBOL(netdev_state_change);
1290
1291/**
1292 * netdev_notify_peers - notify network peers about existence of @dev
1293 * @dev: network device
1294 *
1295 * Generate traffic such that interested network peers are aware of
1296 * @dev, such as by generating a gratuitous ARP. This may be used when
1297 * a device wants to inform the rest of the network about some sort of
1298 * reconfiguration such as a failover event or virtual machine
1299 * migration.
1300 */
1301void netdev_notify_peers(struct net_device *dev)
1302{
1303 rtnl_lock();
1304 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1305 rtnl_unlock();
1306}
1307EXPORT_SYMBOL(netdev_notify_peers);
1308
1309static int __dev_open(struct net_device *dev)
1310{
1311 const struct net_device_ops *ops = dev->netdev_ops;
1312 int ret;
1313
1314 ASSERT_RTNL();
1315
1316 if (!netif_device_present(dev))
1317 return -ENODEV;
1318
1319 /* Block netpoll from trying to do any rx path servicing.
1320 * If we don't do this there is a chance ndo_poll_controller
1321 * or ndo_poll may be running while we open the device
1322 */
1323 netpoll_poll_disable(dev);
1324
1325 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1326 ret = notifier_to_errno(ret);
1327 if (ret)
1328 return ret;
1329
1330 set_bit(__LINK_STATE_START, &dev->state);
1331
1332 if (ops->ndo_validate_addr)
1333 ret = ops->ndo_validate_addr(dev);
1334
1335 if (!ret && ops->ndo_open)
1336 ret = ops->ndo_open(dev);
1337
1338 netpoll_poll_enable(dev);
1339
1340 if (ret)
1341 clear_bit(__LINK_STATE_START, &dev->state);
1342 else {
1343 dev->flags |= IFF_UP;
1344 dev_set_rx_mode(dev);
1345 dev_activate(dev);
1346 add_device_randomness(dev->dev_addr, dev->addr_len);
1347 }
1348
1349 return ret;
1350}
1351
1352/**
1353 * dev_open - prepare an interface for use.
1354 * @dev: device to open
1355 *
1356 * Takes a device from down to up state. The device's private open
1357 * function is invoked and then the multicast lists are loaded. Finally
1358 * the device is moved into the up state and a %NETDEV_UP message is
1359 * sent to the netdev notifier chain.
1360 *
1361 * Calling this function on an active interface is a nop. On a failure
1362 * a negative errno code is returned.
1363 */
1364int dev_open(struct net_device *dev)
1365{
1366 int ret;
1367
1368 if (dev->flags & IFF_UP)
1369 return 0;
1370
1371 ret = __dev_open(dev);
1372 if (ret < 0)
1373 return ret;
1374
1375 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1376 call_netdevice_notifiers(NETDEV_UP, dev);
1377
1378 return ret;
1379}
1380EXPORT_SYMBOL(dev_open);
1381
1382static int __dev_close_many(struct list_head *head)
1383{
1384 struct net_device *dev;
1385
1386 ASSERT_RTNL();
1387 might_sleep();
1388
1389 list_for_each_entry(dev, head, close_list) {
1390 /* Temporarily disable netpoll until the interface is down */
1391 netpoll_poll_disable(dev);
1392
1393 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1394
1395 clear_bit(__LINK_STATE_START, &dev->state);
1396
1397 /* Synchronize to scheduled poll. We cannot touch poll list, it
1398 * can be even on different cpu. So just clear netif_running().
1399 *
1400 * dev->stop() will invoke napi_disable() on all of it's
1401 * napi_struct instances on this device.
1402 */
1403 smp_mb__after_atomic(); /* Commit netif_running(). */
1404 }
1405
1406 dev_deactivate_many(head);
1407
1408 list_for_each_entry(dev, head, close_list) {
1409 const struct net_device_ops *ops = dev->netdev_ops;
1410
1411 /*
1412 * Call the device specific close. This cannot fail.
1413 * Only if device is UP
1414 *
1415 * We allow it to be called even after a DETACH hot-plug
1416 * event.
1417 */
1418 if (ops->ndo_stop)
1419 ops->ndo_stop(dev);
1420
1421 dev->flags &= ~IFF_UP;
1422 netpoll_poll_enable(dev);
1423 }
1424
1425 return 0;
1426}
1427
1428static int __dev_close(struct net_device *dev)
1429{
1430 int retval;
1431 LIST_HEAD(single);
1432
1433 list_add(&dev->close_list, &single);
1434 retval = __dev_close_many(&single);
1435 list_del(&single);
1436
1437 return retval;
1438}
1439
1440int dev_close_many(struct list_head *head, bool unlink)
1441{
1442 struct net_device *dev, *tmp;
1443
1444 /* Remove the devices that don't need to be closed */
1445 list_for_each_entry_safe(dev, tmp, head, close_list)
1446 if (!(dev->flags & IFF_UP))
1447 list_del_init(&dev->close_list);
1448
1449 __dev_close_many(head);
1450
1451 list_for_each_entry_safe(dev, tmp, head, close_list) {
1452 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1453 call_netdevice_notifiers(NETDEV_DOWN, dev);
1454 if (unlink)
1455 list_del_init(&dev->close_list);
1456 }
1457
1458 return 0;
1459}
1460EXPORT_SYMBOL(dev_close_many);
1461
1462/**
1463 * dev_close - shutdown an interface.
1464 * @dev: device to shutdown
1465 *
1466 * This function moves an active device into down state. A
1467 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1468 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1469 * chain.
1470 */
1471int dev_close(struct net_device *dev)
1472{
1473 if (dev->flags & IFF_UP) {
1474 LIST_HEAD(single);
1475
1476 list_add(&dev->close_list, &single);
1477 dev_close_many(&single, true);
1478 list_del(&single);
1479 }
1480 return 0;
1481}
1482EXPORT_SYMBOL(dev_close);
1483
1484
1485/**
1486 * dev_disable_lro - disable Large Receive Offload on a device
1487 * @dev: device
1488 *
1489 * Disable Large Receive Offload (LRO) on a net device. Must be
1490 * called under RTNL. This is needed if received packets may be
1491 * forwarded to another interface.
1492 */
1493void dev_disable_lro(struct net_device *dev)
1494{
1495 struct net_device *lower_dev;
1496 struct list_head *iter;
1497
1498 dev->wanted_features &= ~NETIF_F_LRO;
1499 netdev_update_features(dev);
1500
1501 if (unlikely(dev->features & NETIF_F_LRO))
1502 netdev_WARN(dev, "failed to disable LRO!\n");
1503
1504 netdev_for_each_lower_dev(dev, lower_dev, iter)
1505 dev_disable_lro(lower_dev);
1506}
1507EXPORT_SYMBOL(dev_disable_lro);
1508
1509static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1510 struct net_device *dev)
1511{
1512 struct netdev_notifier_info info;
1513
1514 netdev_notifier_info_init(&info, dev);
1515 return nb->notifier_call(nb, val, &info);
1516}
1517
1518static int dev_boot_phase = 1;
1519
1520/**
1521 * register_netdevice_notifier - register a network notifier block
1522 * @nb: notifier
1523 *
1524 * Register a notifier to be called when network device events occur.
1525 * The notifier passed is linked into the kernel structures and must
1526 * not be reused until it has been unregistered. A negative errno code
1527 * is returned on a failure.
1528 *
1529 * When registered all registration and up events are replayed
1530 * to the new notifier to allow device to have a race free
1531 * view of the network device list.
1532 */
1533
1534int register_netdevice_notifier(struct notifier_block *nb)
1535{
1536 struct net_device *dev;
1537 struct net_device *last;
1538 struct net *net;
1539 int err;
1540
1541 rtnl_lock();
1542 err = raw_notifier_chain_register(&netdev_chain, nb);
1543 if (err)
1544 goto unlock;
1545 if (dev_boot_phase)
1546 goto unlock;
1547 for_each_net(net) {
1548 for_each_netdev(net, dev) {
1549 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1550 err = notifier_to_errno(err);
1551 if (err)
1552 goto rollback;
1553
1554 if (!(dev->flags & IFF_UP))
1555 continue;
1556
1557 call_netdevice_notifier(nb, NETDEV_UP, dev);
1558 }
1559 }
1560
1561unlock:
1562 rtnl_unlock();
1563 return err;
1564
1565rollback:
1566 last = dev;
1567 for_each_net(net) {
1568 for_each_netdev(net, dev) {
1569 if (dev == last)
1570 goto outroll;
1571
1572 if (dev->flags & IFF_UP) {
1573 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1574 dev);
1575 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1576 }
1577 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1578 }
1579 }
1580
1581outroll:
1582 raw_notifier_chain_unregister(&netdev_chain, nb);
1583 goto unlock;
1584}
1585EXPORT_SYMBOL(register_netdevice_notifier);
1586
1587/**
1588 * unregister_netdevice_notifier - unregister a network notifier block
1589 * @nb: notifier
1590 *
1591 * Unregister a notifier previously registered by
1592 * register_netdevice_notifier(). The notifier is unlinked into the
1593 * kernel structures and may then be reused. A negative errno code
1594 * is returned on a failure.
1595 *
1596 * After unregistering unregister and down device events are synthesized
1597 * for all devices on the device list to the removed notifier to remove
1598 * the need for special case cleanup code.
1599 */
1600
1601int unregister_netdevice_notifier(struct notifier_block *nb)
1602{
1603 struct net_device *dev;
1604 struct net *net;
1605 int err;
1606
1607 rtnl_lock();
1608 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1609 if (err)
1610 goto unlock;
1611
1612 for_each_net(net) {
1613 for_each_netdev(net, dev) {
1614 if (dev->flags & IFF_UP) {
1615 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1616 dev);
1617 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1618 }
1619 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1620 }
1621 }
1622unlock:
1623 rtnl_unlock();
1624 return err;
1625}
1626EXPORT_SYMBOL(unregister_netdevice_notifier);
1627
1628/**
1629 * call_netdevice_notifiers_info - call all network notifier blocks
1630 * @val: value passed unmodified to notifier function
1631 * @dev: net_device pointer passed unmodified to notifier function
1632 * @info: notifier information data
1633 *
1634 * Call all network notifier blocks. Parameters and return value
1635 * are as for raw_notifier_call_chain().
1636 */
1637
1638static int call_netdevice_notifiers_info(unsigned long val,
1639 struct net_device *dev,
1640 struct netdev_notifier_info *info)
1641{
1642 ASSERT_RTNL();
1643 netdev_notifier_info_init(info, dev);
1644 return raw_notifier_call_chain(&netdev_chain, val, info);
1645}
1646
1647/**
1648 * call_netdevice_notifiers - call all network notifier blocks
1649 * @val: value passed unmodified to notifier function
1650 * @dev: net_device pointer passed unmodified to notifier function
1651 *
1652 * Call all network notifier blocks. Parameters and return value
1653 * are as for raw_notifier_call_chain().
1654 */
1655
1656int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1657{
1658 struct netdev_notifier_info info;
1659
1660 return call_netdevice_notifiers_info(val, dev, &info);
1661}
1662EXPORT_SYMBOL(call_netdevice_notifiers);
1663
1664#ifdef CONFIG_NET_INGRESS
1665static struct static_key ingress_needed __read_mostly;
1666
1667void net_inc_ingress_queue(void)
1668{
1669 static_key_slow_inc(&ingress_needed);
1670}
1671EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1672
1673void net_dec_ingress_queue(void)
1674{
1675 static_key_slow_dec(&ingress_needed);
1676}
1677EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1678#endif
1679
1680#ifdef CONFIG_NET_EGRESS
1681static struct static_key egress_needed __read_mostly;
1682
1683void net_inc_egress_queue(void)
1684{
1685 static_key_slow_inc(&egress_needed);
1686}
1687EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1688
1689void net_dec_egress_queue(void)
1690{
1691 static_key_slow_dec(&egress_needed);
1692}
1693EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1694#endif
1695
1696static struct static_key netstamp_needed __read_mostly;
1697#ifdef HAVE_JUMP_LABEL
1698/* We are not allowed to call static_key_slow_dec() from irq context
1699 * If net_disable_timestamp() is called from irq context, defer the
1700 * static_key_slow_dec() calls.
1701 */
1702static atomic_t netstamp_needed_deferred;
1703#endif
1704
1705void net_enable_timestamp(void)
1706{
1707#ifdef HAVE_JUMP_LABEL
1708 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1709
1710 if (deferred) {
1711 while (--deferred)
1712 static_key_slow_dec(&netstamp_needed);
1713 return;
1714 }
1715#endif
1716 static_key_slow_inc(&netstamp_needed);
1717}
1718EXPORT_SYMBOL(net_enable_timestamp);
1719
1720void net_disable_timestamp(void)
1721{
1722#ifdef HAVE_JUMP_LABEL
1723 if (in_interrupt()) {
1724 atomic_inc(&netstamp_needed_deferred);
1725 return;
1726 }
1727#endif
1728 static_key_slow_dec(&netstamp_needed);
1729}
1730EXPORT_SYMBOL(net_disable_timestamp);
1731
1732static inline void net_timestamp_set(struct sk_buff *skb)
1733{
1734 skb->tstamp = 0;
1735 if (static_key_false(&netstamp_needed))
1736 __net_timestamp(skb);
1737}
1738
1739#define net_timestamp_check(COND, SKB) \
1740 if (static_key_false(&netstamp_needed)) { \
1741 if ((COND) && !(SKB)->tstamp) \
1742 __net_timestamp(SKB); \
1743 } \
1744
1745bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
1746{
1747 unsigned int len;
1748
1749 if (!(dev->flags & IFF_UP))
1750 return false;
1751
1752 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1753 if (skb->len <= len)
1754 return true;
1755
1756 /* if TSO is enabled, we don't care about the length as the packet
1757 * could be forwarded without being segmented before
1758 */
1759 if (skb_is_gso(skb))
1760 return true;
1761
1762 return false;
1763}
1764EXPORT_SYMBOL_GPL(is_skb_forwardable);
1765
1766int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1767{
1768 int ret = ____dev_forward_skb(dev, skb);
1769
1770 if (likely(!ret)) {
1771 skb->protocol = eth_type_trans(skb, dev);
1772 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1773 }
1774
1775 return ret;
1776}
1777EXPORT_SYMBOL_GPL(__dev_forward_skb);
1778
1779/**
1780 * dev_forward_skb - loopback an skb to another netif
1781 *
1782 * @dev: destination network device
1783 * @skb: buffer to forward
1784 *
1785 * return values:
1786 * NET_RX_SUCCESS (no congestion)
1787 * NET_RX_DROP (packet was dropped, but freed)
1788 *
1789 * dev_forward_skb can be used for injecting an skb from the
1790 * start_xmit function of one device into the receive queue
1791 * of another device.
1792 *
1793 * The receiving device may be in another namespace, so
1794 * we have to clear all information in the skb that could
1795 * impact namespace isolation.
1796 */
1797int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1798{
1799 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1800}
1801EXPORT_SYMBOL_GPL(dev_forward_skb);
1802
1803static inline int deliver_skb(struct sk_buff *skb,
1804 struct packet_type *pt_prev,
1805 struct net_device *orig_dev)
1806{
1807 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1808 return -ENOMEM;
1809 atomic_inc(&skb->users);
1810 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1811}
1812
1813static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1814 struct packet_type **pt,
1815 struct net_device *orig_dev,
1816 __be16 type,
1817 struct list_head *ptype_list)
1818{
1819 struct packet_type *ptype, *pt_prev = *pt;
1820
1821 list_for_each_entry_rcu(ptype, ptype_list, list) {
1822 if (ptype->type != type)
1823 continue;
1824 if (pt_prev)
1825 deliver_skb(skb, pt_prev, orig_dev);
1826 pt_prev = ptype;
1827 }
1828 *pt = pt_prev;
1829}
1830
1831static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1832{
1833 if (!ptype->af_packet_priv || !skb->sk)
1834 return false;
1835
1836 if (ptype->id_match)
1837 return ptype->id_match(ptype, skb->sk);
1838 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1839 return true;
1840
1841 return false;
1842}
1843
1844/*
1845 * Support routine. Sends outgoing frames to any network
1846 * taps currently in use.
1847 */
1848
1849void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1850{
1851 struct packet_type *ptype;
1852 struct sk_buff *skb2 = NULL;
1853 struct packet_type *pt_prev = NULL;
1854 struct list_head *ptype_list = &ptype_all;
1855
1856 rcu_read_lock();
1857again:
1858 list_for_each_entry_rcu(ptype, ptype_list, list) {
1859 /* Never send packets back to the socket
1860 * they originated from - MvS (miquels@drinkel.ow.org)
1861 */
1862 if (skb_loop_sk(ptype, skb))
1863 continue;
1864
1865 if (pt_prev) {
1866 deliver_skb(skb2, pt_prev, skb->dev);
1867 pt_prev = ptype;
1868 continue;
1869 }
1870
1871 /* need to clone skb, done only once */
1872 skb2 = skb_clone(skb, GFP_ATOMIC);
1873 if (!skb2)
1874 goto out_unlock;
1875
1876 net_timestamp_set(skb2);
1877
1878 /* skb->nh should be correctly
1879 * set by sender, so that the second statement is
1880 * just protection against buggy protocols.
1881 */
1882 skb_reset_mac_header(skb2);
1883
1884 if (skb_network_header(skb2) < skb2->data ||
1885 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1886 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1887 ntohs(skb2->protocol),
1888 dev->name);
1889 skb_reset_network_header(skb2);
1890 }
1891
1892 skb2->transport_header = skb2->network_header;
1893 skb2->pkt_type = PACKET_OUTGOING;
1894 pt_prev = ptype;
1895 }
1896
1897 if (ptype_list == &ptype_all) {
1898 ptype_list = &dev->ptype_all;
1899 goto again;
1900 }
1901out_unlock:
1902 if (pt_prev)
1903 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1904 rcu_read_unlock();
1905}
1906EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
1907
1908/**
1909 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1910 * @dev: Network device
1911 * @txq: number of queues available
1912 *
1913 * If real_num_tx_queues is changed the tc mappings may no longer be
1914 * valid. To resolve this verify the tc mapping remains valid and if
1915 * not NULL the mapping. With no priorities mapping to this
1916 * offset/count pair it will no longer be used. In the worst case TC0
1917 * is invalid nothing can be done so disable priority mappings. If is
1918 * expected that drivers will fix this mapping if they can before
1919 * calling netif_set_real_num_tx_queues.
1920 */
1921static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1922{
1923 int i;
1924 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1925
1926 /* If TC0 is invalidated disable TC mapping */
1927 if (tc->offset + tc->count > txq) {
1928 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1929 dev->num_tc = 0;
1930 return;
1931 }
1932
1933 /* Invalidated prio to tc mappings set to TC0 */
1934 for (i = 1; i < TC_BITMASK + 1; i++) {
1935 int q = netdev_get_prio_tc_map(dev, i);
1936
1937 tc = &dev->tc_to_txq[q];
1938 if (tc->offset + tc->count > txq) {
1939 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1940 i, q);
1941 netdev_set_prio_tc_map(dev, i, 0);
1942 }
1943 }
1944}
1945
1946int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
1947{
1948 if (dev->num_tc) {
1949 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1950 int i;
1951
1952 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
1953 if ((txq - tc->offset) < tc->count)
1954 return i;
1955 }
1956
1957 return -1;
1958 }
1959
1960 return 0;
1961}
1962
1963#ifdef CONFIG_XPS
1964static DEFINE_MUTEX(xps_map_mutex);
1965#define xmap_dereference(P) \
1966 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1967
1968static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
1969 int tci, u16 index)
1970{
1971 struct xps_map *map = NULL;
1972 int pos;
1973
1974 if (dev_maps)
1975 map = xmap_dereference(dev_maps->cpu_map[tci]);
1976 if (!map)
1977 return false;
1978
1979 for (pos = map->len; pos--;) {
1980 if (map->queues[pos] != index)
1981 continue;
1982
1983 if (map->len > 1) {
1984 map->queues[pos] = map->queues[--map->len];
1985 break;
1986 }
1987
1988 RCU_INIT_POINTER(dev_maps->cpu_map[tci], NULL);
1989 kfree_rcu(map, rcu);
1990 return false;
1991 }
1992
1993 return true;
1994}
1995
1996static bool remove_xps_queue_cpu(struct net_device *dev,
1997 struct xps_dev_maps *dev_maps,
1998 int cpu, u16 offset, u16 count)
1999{
2000 int num_tc = dev->num_tc ? : 1;
2001 bool active = false;
2002 int tci;
2003
2004 for (tci = cpu * num_tc; num_tc--; tci++) {
2005 int i, j;
2006
2007 for (i = count, j = offset; i--; j++) {
2008 if (!remove_xps_queue(dev_maps, cpu, j))
2009 break;
2010 }
2011
2012 active |= i < 0;
2013 }
2014
2015 return active;
2016}
2017
2018static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2019 u16 count)
2020{
2021 struct xps_dev_maps *dev_maps;
2022 int cpu, i;
2023 bool active = false;
2024
2025 mutex_lock(&xps_map_mutex);
2026 dev_maps = xmap_dereference(dev->xps_maps);
2027
2028 if (!dev_maps)
2029 goto out_no_maps;
2030
2031 for_each_possible_cpu(cpu)
2032 active |= remove_xps_queue_cpu(dev, dev_maps, cpu,
2033 offset, count);
2034
2035 if (!active) {
2036 RCU_INIT_POINTER(dev->xps_maps, NULL);
2037 kfree_rcu(dev_maps, rcu);
2038 }
2039
2040 for (i = offset + (count - 1); count--; i--)
2041 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2042 NUMA_NO_NODE);
2043
2044out_no_maps:
2045 mutex_unlock(&xps_map_mutex);
2046}
2047
2048static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2049{
2050 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2051}
2052
2053static struct xps_map *expand_xps_map(struct xps_map *map,
2054 int cpu, u16 index)
2055{
2056 struct xps_map *new_map;
2057 int alloc_len = XPS_MIN_MAP_ALLOC;
2058 int i, pos;
2059
2060 for (pos = 0; map && pos < map->len; pos++) {
2061 if (map->queues[pos] != index)
2062 continue;
2063 return map;
2064 }
2065
2066 /* Need to add queue to this CPU's existing map */
2067 if (map) {
2068 if (pos < map->alloc_len)
2069 return map;
2070
2071 alloc_len = map->alloc_len * 2;
2072 }
2073
2074 /* Need to allocate new map to store queue on this CPU's map */
2075 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2076 cpu_to_node(cpu));
2077 if (!new_map)
2078 return NULL;
2079
2080 for (i = 0; i < pos; i++)
2081 new_map->queues[i] = map->queues[i];
2082 new_map->alloc_len = alloc_len;
2083 new_map->len = pos;
2084
2085 return new_map;
2086}
2087
2088int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2089 u16 index)
2090{
2091 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2092 int i, cpu, tci, numa_node_id = -2;
2093 int maps_sz, num_tc = 1, tc = 0;
2094 struct xps_map *map, *new_map;
2095 bool active = false;
2096
2097 if (dev->num_tc) {
2098 num_tc = dev->num_tc;
2099 tc = netdev_txq_to_tc(dev, index);
2100 if (tc < 0)
2101 return -EINVAL;
2102 }
2103
2104 maps_sz = XPS_DEV_MAPS_SIZE(num_tc);
2105 if (maps_sz < L1_CACHE_BYTES)
2106 maps_sz = L1_CACHE_BYTES;
2107
2108 mutex_lock(&xps_map_mutex);
2109
2110 dev_maps = xmap_dereference(dev->xps_maps);
2111
2112 /* allocate memory for queue storage */
2113 for_each_cpu_and(cpu, cpu_online_mask, mask) {
2114 if (!new_dev_maps)
2115 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2116 if (!new_dev_maps) {
2117 mutex_unlock(&xps_map_mutex);
2118 return -ENOMEM;
2119 }
2120
2121 tci = cpu * num_tc + tc;
2122 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[tci]) :
2123 NULL;
2124
2125 map = expand_xps_map(map, cpu, index);
2126 if (!map)
2127 goto error;
2128
2129 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2130 }
2131
2132 if (!new_dev_maps)
2133 goto out_no_new_maps;
2134
2135 for_each_possible_cpu(cpu) {
2136 /* copy maps belonging to foreign traffic classes */
2137 for (i = tc, tci = cpu * num_tc; dev_maps && i--; tci++) {
2138 /* fill in the new device map from the old device map */
2139 map = xmap_dereference(dev_maps->cpu_map[tci]);
2140 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2141 }
2142
2143 /* We need to explicitly update tci as prevous loop
2144 * could break out early if dev_maps is NULL.
2145 */
2146 tci = cpu * num_tc + tc;
2147
2148 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2149 /* add queue to CPU maps */
2150 int pos = 0;
2151
2152 map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2153 while ((pos < map->len) && (map->queues[pos] != index))
2154 pos++;
2155
2156 if (pos == map->len)
2157 map->queues[map->len++] = index;
2158#ifdef CONFIG_NUMA
2159 if (numa_node_id == -2)
2160 numa_node_id = cpu_to_node(cpu);
2161 else if (numa_node_id != cpu_to_node(cpu))
2162 numa_node_id = -1;
2163#endif
2164 } else if (dev_maps) {
2165 /* fill in the new device map from the old device map */
2166 map = xmap_dereference(dev_maps->cpu_map[tci]);
2167 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2168 }
2169
2170 /* copy maps belonging to foreign traffic classes */
2171 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2172 /* fill in the new device map from the old device map */
2173 map = xmap_dereference(dev_maps->cpu_map[tci]);
2174 RCU_INIT_POINTER(new_dev_maps->cpu_map[tci], map);
2175 }
2176 }
2177
2178 rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2179
2180 /* Cleanup old maps */
2181 if (!dev_maps)
2182 goto out_no_old_maps;
2183
2184 for_each_possible_cpu(cpu) {
2185 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2186 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2187 map = xmap_dereference(dev_maps->cpu_map[tci]);
2188 if (map && map != new_map)
2189 kfree_rcu(map, rcu);
2190 }
2191 }
2192
2193 kfree_rcu(dev_maps, rcu);
2194
2195out_no_old_maps:
2196 dev_maps = new_dev_maps;
2197 active = true;
2198
2199out_no_new_maps:
2200 /* update Tx queue numa node */
2201 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2202 (numa_node_id >= 0) ? numa_node_id :
2203 NUMA_NO_NODE);
2204
2205 if (!dev_maps)
2206 goto out_no_maps;
2207
2208 /* removes queue from unused CPUs */
2209 for_each_possible_cpu(cpu) {
2210 for (i = tc, tci = cpu * num_tc; i--; tci++)
2211 active |= remove_xps_queue(dev_maps, tci, index);
2212 if (!cpumask_test_cpu(cpu, mask) || !cpu_online(cpu))
2213 active |= remove_xps_queue(dev_maps, tci, index);
2214 for (i = num_tc - tc, tci++; --i; tci++)
2215 active |= remove_xps_queue(dev_maps, tci, index);
2216 }
2217
2218 /* free map if not active */
2219 if (!active) {
2220 RCU_INIT_POINTER(dev->xps_maps, NULL);
2221 kfree_rcu(dev_maps, rcu);
2222 }
2223
2224out_no_maps:
2225 mutex_unlock(&xps_map_mutex);
2226
2227 return 0;
2228error:
2229 /* remove any maps that we added */
2230 for_each_possible_cpu(cpu) {
2231 for (i = num_tc, tci = cpu * num_tc; i--; tci++) {
2232 new_map = xmap_dereference(new_dev_maps->cpu_map[tci]);
2233 map = dev_maps ?
2234 xmap_dereference(dev_maps->cpu_map[tci]) :
2235 NULL;
2236 if (new_map && new_map != map)
2237 kfree(new_map);
2238 }
2239 }
2240
2241 mutex_unlock(&xps_map_mutex);
2242
2243 kfree(new_dev_maps);
2244 return -ENOMEM;
2245}
2246EXPORT_SYMBOL(netif_set_xps_queue);
2247
2248#endif
2249void netdev_reset_tc(struct net_device *dev)
2250{
2251#ifdef CONFIG_XPS
2252 netif_reset_xps_queues_gt(dev, 0);
2253#endif
2254 dev->num_tc = 0;
2255 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2256 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2257}
2258EXPORT_SYMBOL(netdev_reset_tc);
2259
2260int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2261{
2262 if (tc >= dev->num_tc)
2263 return -EINVAL;
2264
2265#ifdef CONFIG_XPS
2266 netif_reset_xps_queues(dev, offset, count);
2267#endif
2268 dev->tc_to_txq[tc].count = count;
2269 dev->tc_to_txq[tc].offset = offset;
2270 return 0;
2271}
2272EXPORT_SYMBOL(netdev_set_tc_queue);
2273
2274int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2275{
2276 if (num_tc > TC_MAX_QUEUE)
2277 return -EINVAL;
2278
2279#ifdef CONFIG_XPS
2280 netif_reset_xps_queues_gt(dev, 0);
2281#endif
2282 dev->num_tc = num_tc;
2283 return 0;
2284}
2285EXPORT_SYMBOL(netdev_set_num_tc);
2286
2287/*
2288 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2289 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2290 */
2291int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2292{
2293 int rc;
2294
2295 if (txq < 1 || txq > dev->num_tx_queues)
2296 return -EINVAL;
2297
2298 if (dev->reg_state == NETREG_REGISTERED ||
2299 dev->reg_state == NETREG_UNREGISTERING) {
2300 ASSERT_RTNL();
2301
2302 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2303 txq);
2304 if (rc)
2305 return rc;
2306
2307 if (dev->num_tc)
2308 netif_setup_tc(dev, txq);
2309
2310 if (txq < dev->real_num_tx_queues) {
2311 qdisc_reset_all_tx_gt(dev, txq);
2312#ifdef CONFIG_XPS
2313 netif_reset_xps_queues_gt(dev, txq);
2314#endif
2315 }
2316 }
2317
2318 dev->real_num_tx_queues = txq;
2319 return 0;
2320}
2321EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2322
2323#ifdef CONFIG_SYSFS
2324/**
2325 * netif_set_real_num_rx_queues - set actual number of RX queues used
2326 * @dev: Network device
2327 * @rxq: Actual number of RX queues
2328 *
2329 * This must be called either with the rtnl_lock held or before
2330 * registration of the net device. Returns 0 on success, or a
2331 * negative error code. If called before registration, it always
2332 * succeeds.
2333 */
2334int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2335{
2336 int rc;
2337
2338 if (rxq < 1 || rxq > dev->num_rx_queues)
2339 return -EINVAL;
2340
2341 if (dev->reg_state == NETREG_REGISTERED) {
2342 ASSERT_RTNL();
2343
2344 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2345 rxq);
2346 if (rc)
2347 return rc;
2348 }
2349
2350 dev->real_num_rx_queues = rxq;
2351 return 0;
2352}
2353EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2354#endif
2355
2356/**
2357 * netif_get_num_default_rss_queues - default number of RSS queues
2358 *
2359 * This routine should set an upper limit on the number of RSS queues
2360 * used by default by multiqueue devices.
2361 */
2362int netif_get_num_default_rss_queues(void)
2363{
2364 return is_kdump_kernel() ?
2365 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2366}
2367EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2368
2369static void __netif_reschedule(struct Qdisc *q)
2370{
2371 struct softnet_data *sd;
2372 unsigned long flags;
2373
2374 local_irq_save(flags);
2375 sd = this_cpu_ptr(&softnet_data);
2376 q->next_sched = NULL;
2377 *sd->output_queue_tailp = q;
2378 sd->output_queue_tailp = &q->next_sched;
2379 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2380 local_irq_restore(flags);
2381}
2382
2383void __netif_schedule(struct Qdisc *q)
2384{
2385 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2386 __netif_reschedule(q);
2387}
2388EXPORT_SYMBOL(__netif_schedule);
2389
2390struct dev_kfree_skb_cb {
2391 enum skb_free_reason reason;
2392};
2393
2394static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2395{
2396 return (struct dev_kfree_skb_cb *)skb->cb;
2397}
2398
2399void netif_schedule_queue(struct netdev_queue *txq)
2400{
2401 rcu_read_lock();
2402 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2403 struct Qdisc *q = rcu_dereference(txq->qdisc);
2404
2405 __netif_schedule(q);
2406 }
2407 rcu_read_unlock();
2408}
2409EXPORT_SYMBOL(netif_schedule_queue);
2410
2411/**
2412 * netif_wake_subqueue - allow sending packets on subqueue
2413 * @dev: network device
2414 * @queue_index: sub queue index
2415 *
2416 * Resume individual transmit queue of a device with multiple transmit queues.
2417 */
2418void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2419{
2420 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2421
2422 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
2423 struct Qdisc *q;
2424
2425 rcu_read_lock();
2426 q = rcu_dereference(txq->qdisc);
2427 __netif_schedule(q);
2428 rcu_read_unlock();
2429 }
2430}
2431EXPORT_SYMBOL(netif_wake_subqueue);
2432
2433void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2434{
2435 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2436 struct Qdisc *q;
2437
2438 rcu_read_lock();
2439 q = rcu_dereference(dev_queue->qdisc);
2440 __netif_schedule(q);
2441 rcu_read_unlock();
2442 }
2443}
2444EXPORT_SYMBOL(netif_tx_wake_queue);
2445
2446void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2447{
2448 unsigned long flags;
2449
2450 if (likely(atomic_read(&skb->users) == 1)) {
2451 smp_rmb();
2452 atomic_set(&skb->users, 0);
2453 } else if (likely(!atomic_dec_and_test(&skb->users))) {
2454 return;
2455 }
2456 get_kfree_skb_cb(skb)->reason = reason;
2457 local_irq_save(flags);
2458 skb->next = __this_cpu_read(softnet_data.completion_queue);
2459 __this_cpu_write(softnet_data.completion_queue, skb);
2460 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2461 local_irq_restore(flags);
2462}
2463EXPORT_SYMBOL(__dev_kfree_skb_irq);
2464
2465void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2466{
2467 if (in_irq() || irqs_disabled())
2468 __dev_kfree_skb_irq(skb, reason);
2469 else
2470 dev_kfree_skb(skb);
2471}
2472EXPORT_SYMBOL(__dev_kfree_skb_any);
2473
2474
2475/**
2476 * netif_device_detach - mark device as removed
2477 * @dev: network device
2478 *
2479 * Mark device as removed from system and therefore no longer available.
2480 */
2481void netif_device_detach(struct net_device *dev)
2482{
2483 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2484 netif_running(dev)) {
2485 netif_tx_stop_all_queues(dev);
2486 }
2487}
2488EXPORT_SYMBOL(netif_device_detach);
2489
2490/**
2491 * netif_device_attach - mark device as attached
2492 * @dev: network device
2493 *
2494 * Mark device as attached from system and restart if needed.
2495 */
2496void netif_device_attach(struct net_device *dev)
2497{
2498 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2499 netif_running(dev)) {
2500 netif_tx_wake_all_queues(dev);
2501 __netdev_watchdog_up(dev);
2502 }
2503}
2504EXPORT_SYMBOL(netif_device_attach);
2505
2506/*
2507 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2508 * to be used as a distribution range.
2509 */
2510u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2511 unsigned int num_tx_queues)
2512{
2513 u32 hash;
2514 u16 qoffset = 0;
2515 u16 qcount = num_tx_queues;
2516
2517 if (skb_rx_queue_recorded(skb)) {
2518 hash = skb_get_rx_queue(skb);
2519 while (unlikely(hash >= num_tx_queues))
2520 hash -= num_tx_queues;
2521 return hash;
2522 }
2523
2524 if (dev->num_tc) {
2525 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2526 qoffset = dev->tc_to_txq[tc].offset;
2527 qcount = dev->tc_to_txq[tc].count;
2528 }
2529
2530 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2531}
2532EXPORT_SYMBOL(__skb_tx_hash);
2533
2534static void skb_warn_bad_offload(const struct sk_buff *skb)
2535{
2536 static const netdev_features_t null_features;
2537 struct net_device *dev = skb->dev;
2538 const char *name = "";
2539
2540 if (!net_ratelimit())
2541 return;
2542
2543 if (dev) {
2544 if (dev->dev.parent)
2545 name = dev_driver_string(dev->dev.parent);
2546 else
2547 name = netdev_name(dev);
2548 }
2549 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2550 "gso_type=%d ip_summed=%d\n",
2551 name, dev ? &dev->features : &null_features,
2552 skb->sk ? &skb->sk->sk_route_caps : &null_features,
2553 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2554 skb_shinfo(skb)->gso_type, skb->ip_summed);
2555}
2556
2557/*
2558 * Invalidate hardware checksum when packet is to be mangled, and
2559 * complete checksum manually on outgoing path.
2560 */
2561int skb_checksum_help(struct sk_buff *skb)
2562{
2563 __wsum csum;
2564 int ret = 0, offset;
2565
2566 if (skb->ip_summed == CHECKSUM_COMPLETE)
2567 goto out_set_summed;
2568
2569 if (unlikely(skb_shinfo(skb)->gso_size)) {
2570 skb_warn_bad_offload(skb);
2571 return -EINVAL;
2572 }
2573
2574 /* Before computing a checksum, we should make sure no frag could
2575 * be modified by an external entity : checksum could be wrong.
2576 */
2577 if (skb_has_shared_frag(skb)) {
2578 ret = __skb_linearize(skb);
2579 if (ret)
2580 goto out;
2581 }
2582
2583 offset = skb_checksum_start_offset(skb);
2584 BUG_ON(offset >= skb_headlen(skb));
2585 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2586
2587 offset += skb->csum_offset;
2588 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2589
2590 if (skb_cloned(skb) &&
2591 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2592 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2593 if (ret)
2594 goto out;
2595 }
2596
2597 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2598out_set_summed:
2599 skb->ip_summed = CHECKSUM_NONE;
2600out:
2601 return ret;
2602}
2603EXPORT_SYMBOL(skb_checksum_help);
2604
2605__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2606{
2607 __be16 type = skb->protocol;
2608
2609 /* Tunnel gso handlers can set protocol to ethernet. */
2610 if (type == htons(ETH_P_TEB)) {
2611 struct ethhdr *eth;
2612
2613 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2614 return 0;
2615
2616 eth = (struct ethhdr *)skb_mac_header(skb);
2617 type = eth->h_proto;
2618 }
2619
2620 return __vlan_get_protocol(skb, type, depth);
2621}
2622
2623/**
2624 * skb_mac_gso_segment - mac layer segmentation handler.
2625 * @skb: buffer to segment
2626 * @features: features for the output path (see dev->features)
2627 */
2628struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2629 netdev_features_t features)
2630{
2631 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2632 struct packet_offload *ptype;
2633 int vlan_depth = skb->mac_len;
2634 __be16 type = skb_network_protocol(skb, &vlan_depth);
2635
2636 if (unlikely(!type))
2637 return ERR_PTR(-EINVAL);
2638
2639 __skb_pull(skb, vlan_depth);
2640
2641 rcu_read_lock();
2642 list_for_each_entry_rcu(ptype, &offload_base, list) {
2643 if (ptype->type == type && ptype->callbacks.gso_segment) {
2644 segs = ptype->callbacks.gso_segment(skb, features);
2645 break;
2646 }
2647 }
2648 rcu_read_unlock();
2649
2650 __skb_push(skb, skb->data - skb_mac_header(skb));
2651
2652 return segs;
2653}
2654EXPORT_SYMBOL(skb_mac_gso_segment);
2655
2656
2657/* openvswitch calls this on rx path, so we need a different check.
2658 */
2659static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2660{
2661 if (tx_path)
2662 return skb->ip_summed != CHECKSUM_PARTIAL;
2663 else
2664 return skb->ip_summed == CHECKSUM_NONE;
2665}
2666
2667/**
2668 * __skb_gso_segment - Perform segmentation on skb.
2669 * @skb: buffer to segment
2670 * @features: features for the output path (see dev->features)
2671 * @tx_path: whether it is called in TX path
2672 *
2673 * This function segments the given skb and returns a list of segments.
2674 *
2675 * It may return NULL if the skb requires no segmentation. This is
2676 * only possible when GSO is used for verifying header integrity.
2677 *
2678 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
2679 */
2680struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2681 netdev_features_t features, bool tx_path)
2682{
2683 if (unlikely(skb_needs_check(skb, tx_path))) {
2684 int err;
2685
2686 skb_warn_bad_offload(skb);
2687
2688 err = skb_cow_head(skb, 0);
2689 if (err < 0)
2690 return ERR_PTR(err);
2691 }
2692
2693 /* Only report GSO partial support if it will enable us to
2694 * support segmentation on this frame without needing additional
2695 * work.
2696 */
2697 if (features & NETIF_F_GSO_PARTIAL) {
2698 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
2699 struct net_device *dev = skb->dev;
2700
2701 partial_features |= dev->features & dev->gso_partial_features;
2702 if (!skb_gso_ok(skb, features | partial_features))
2703 features &= ~NETIF_F_GSO_PARTIAL;
2704 }
2705
2706 BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2707 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2708
2709 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2710 SKB_GSO_CB(skb)->encap_level = 0;
2711
2712 skb_reset_mac_header(skb);
2713 skb_reset_mac_len(skb);
2714
2715 return skb_mac_gso_segment(skb, features);
2716}
2717EXPORT_SYMBOL(__skb_gso_segment);
2718
2719/* Take action when hardware reception checksum errors are detected. */
2720#ifdef CONFIG_BUG
2721void netdev_rx_csum_fault(struct net_device *dev)
2722{
2723 if (net_ratelimit()) {
2724 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2725 dump_stack();
2726 }
2727}
2728EXPORT_SYMBOL(netdev_rx_csum_fault);
2729#endif
2730
2731/* Actually, we should eliminate this check as soon as we know, that:
2732 * 1. IOMMU is present and allows to map all the memory.
2733 * 2. No high memory really exists on this machine.
2734 */
2735
2736static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2737{
2738#ifdef CONFIG_HIGHMEM
2739 int i;
2740 if (!(dev->features & NETIF_F_HIGHDMA)) {
2741 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2742 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2743 if (PageHighMem(skb_frag_page(frag)))
2744 return 1;
2745 }
2746 }
2747
2748 if (PCI_DMA_BUS_IS_PHYS) {
2749 struct device *pdev = dev->dev.parent;
2750
2751 if (!pdev)
2752 return 0;
2753 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2754 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2755 dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2756 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2757 return 1;
2758 }
2759 }
2760#endif
2761 return 0;
2762}
2763
2764/* If MPLS offload request, verify we are testing hardware MPLS features
2765 * instead of standard features for the netdev.
2766 */
2767#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2768static netdev_features_t net_mpls_features(struct sk_buff *skb,
2769 netdev_features_t features,
2770 __be16 type)
2771{
2772 if (eth_p_mpls(type))
2773 features &= skb->dev->mpls_features;
2774
2775 return features;
2776}
2777#else
2778static netdev_features_t net_mpls_features(struct sk_buff *skb,
2779 netdev_features_t features,
2780 __be16 type)
2781{
2782 return features;
2783}
2784#endif
2785
2786static netdev_features_t harmonize_features(struct sk_buff *skb,
2787 netdev_features_t features)
2788{
2789 int tmp;
2790 __be16 type;
2791
2792 type = skb_network_protocol(skb, &tmp);
2793 features = net_mpls_features(skb, features, type);
2794
2795 if (skb->ip_summed != CHECKSUM_NONE &&
2796 !can_checksum_protocol(features, type)) {
2797 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2798 }
2799 if (illegal_highdma(skb->dev, skb))
2800 features &= ~NETIF_F_SG;
2801
2802 return features;
2803}
2804
2805netdev_features_t passthru_features_check(struct sk_buff *skb,
2806 struct net_device *dev,
2807 netdev_features_t features)
2808{
2809 return features;
2810}
2811EXPORT_SYMBOL(passthru_features_check);
2812
2813static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2814 struct net_device *dev,
2815 netdev_features_t features)
2816{
2817 return vlan_features_check(skb, features);
2818}
2819
2820static netdev_features_t gso_features_check(const struct sk_buff *skb,
2821 struct net_device *dev,
2822 netdev_features_t features)
2823{
2824 u16 gso_segs = skb_shinfo(skb)->gso_segs;
2825
2826 if (gso_segs > dev->gso_max_segs)
2827 return features & ~NETIF_F_GSO_MASK;
2828
2829 /* Support for GSO partial features requires software
2830 * intervention before we can actually process the packets
2831 * so we need to strip support for any partial features now
2832 * and we can pull them back in after we have partially
2833 * segmented the frame.
2834 */
2835 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
2836 features &= ~dev->gso_partial_features;
2837
2838 /* Make sure to clear the IPv4 ID mangling feature if the
2839 * IPv4 header has the potential to be fragmented.
2840 */
2841 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
2842 struct iphdr *iph = skb->encapsulation ?
2843 inner_ip_hdr(skb) : ip_hdr(skb);
2844
2845 if (!(iph->frag_off & htons(IP_DF)))
2846 features &= ~NETIF_F_TSO_MANGLEID;
2847 }
2848
2849 return features;
2850}
2851
2852netdev_features_t netif_skb_features(struct sk_buff *skb)
2853{
2854 struct net_device *dev = skb->dev;
2855 netdev_features_t features = dev->features;
2856
2857 if (skb_is_gso(skb))
2858 features = gso_features_check(skb, dev, features);
2859
2860 /* If encapsulation offload request, verify we are testing
2861 * hardware encapsulation features instead of standard
2862 * features for the netdev
2863 */
2864 if (skb->encapsulation)
2865 features &= dev->hw_enc_features;
2866
2867 if (skb_vlan_tagged(skb))
2868 features = netdev_intersect_features(features,
2869 dev->vlan_features |
2870 NETIF_F_HW_VLAN_CTAG_TX |
2871 NETIF_F_HW_VLAN_STAG_TX);
2872
2873 if (dev->netdev_ops->ndo_features_check)
2874 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2875 features);
2876 else
2877 features &= dflt_features_check(skb, dev, features);
2878
2879 return harmonize_features(skb, features);
2880}
2881EXPORT_SYMBOL(netif_skb_features);
2882
2883static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2884 struct netdev_queue *txq, bool more)
2885{
2886 unsigned int len;
2887 int rc;
2888
2889 if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
2890 dev_queue_xmit_nit(skb, dev);
2891
2892 len = skb->len;
2893 trace_net_dev_start_xmit(skb, dev);
2894 rc = netdev_start_xmit(skb, dev, txq, more);
2895 trace_net_dev_xmit(skb, rc, dev, len);
2896
2897 return rc;
2898}
2899
2900struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2901 struct netdev_queue *txq, int *ret)
2902{
2903 struct sk_buff *skb = first;
2904 int rc = NETDEV_TX_OK;
2905
2906 while (skb) {
2907 struct sk_buff *next = skb->next;
2908
2909 skb->next = NULL;
2910 rc = xmit_one(skb, dev, txq, next != NULL);
2911 if (unlikely(!dev_xmit_complete(rc))) {
2912 skb->next = next;
2913 goto out;
2914 }
2915
2916 skb = next;
2917 if (netif_xmit_stopped(txq) && skb) {
2918 rc = NETDEV_TX_BUSY;
2919 break;
2920 }
2921 }
2922
2923out:
2924 *ret = rc;
2925 return skb;
2926}
2927
2928static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2929 netdev_features_t features)
2930{
2931 if (skb_vlan_tag_present(skb) &&
2932 !vlan_hw_offload_capable(features, skb->vlan_proto))
2933 skb = __vlan_hwaccel_push_inside(skb);
2934 return skb;
2935}
2936
2937static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
2938{
2939 netdev_features_t features;
2940
2941 features = netif_skb_features(skb);
2942 skb = validate_xmit_vlan(skb, features);
2943 if (unlikely(!skb))
2944 goto out_null;
2945
2946 if (netif_needs_gso(skb, features)) {
2947 struct sk_buff *segs;
2948
2949 segs = skb_gso_segment(skb, features);
2950 if (IS_ERR(segs)) {
2951 goto out_kfree_skb;
2952 } else if (segs) {
2953 consume_skb(skb);
2954 skb = segs;
2955 }
2956 } else {
2957 if (skb_needs_linearize(skb, features) &&
2958 __skb_linearize(skb))
2959 goto out_kfree_skb;
2960
2961 /* If packet is not checksummed and device does not
2962 * support checksumming for this protocol, complete
2963 * checksumming here.
2964 */
2965 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2966 if (skb->encapsulation)
2967 skb_set_inner_transport_header(skb,
2968 skb_checksum_start_offset(skb));
2969 else
2970 skb_set_transport_header(skb,
2971 skb_checksum_start_offset(skb));
2972 if (!(features & NETIF_F_CSUM_MASK) &&
2973 skb_checksum_help(skb))
2974 goto out_kfree_skb;
2975 }
2976 }
2977
2978 return skb;
2979
2980out_kfree_skb:
2981 kfree_skb(skb);
2982out_null:
2983 atomic_long_inc(&dev->tx_dropped);
2984 return NULL;
2985}
2986
2987struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
2988{
2989 struct sk_buff *next, *head = NULL, *tail;
2990
2991 for (; skb != NULL; skb = next) {
2992 next = skb->next;
2993 skb->next = NULL;
2994
2995 /* in case skb wont be segmented, point to itself */
2996 skb->prev = skb;
2997
2998 skb = validate_xmit_skb(skb, dev);
2999 if (!skb)
3000 continue;
3001
3002 if (!head)
3003 head = skb;
3004 else
3005 tail->next = skb;
3006 /* If skb was segmented, skb->prev points to
3007 * the last segment. If not, it still contains skb.
3008 */
3009 tail = skb->prev;
3010 }
3011 return head;
3012}
3013EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3014
3015static void qdisc_pkt_len_init(struct sk_buff *skb)
3016{
3017 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3018
3019 qdisc_skb_cb(skb)->pkt_len = skb->len;
3020
3021 /* To get more precise estimation of bytes sent on wire,
3022 * we add to pkt_len the headers size of all segments
3023 */
3024 if (shinfo->gso_size) {
3025 unsigned int hdr_len;
3026 u16 gso_segs = shinfo->gso_segs;
3027
3028 /* mac layer + network layer */
3029 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3030
3031 /* + transport layer */
3032 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3033 hdr_len += tcp_hdrlen(skb);
3034 else
3035 hdr_len += sizeof(struct udphdr);
3036
3037 if (shinfo->gso_type & SKB_GSO_DODGY)
3038 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3039 shinfo->gso_size);
3040
3041 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3042 }
3043}
3044
3045static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3046 struct net_device *dev,
3047 struct netdev_queue *txq)
3048{
3049 spinlock_t *root_lock = qdisc_lock(q);
3050 struct sk_buff *to_free = NULL;
3051 bool contended;
3052 int rc;
3053
3054 qdisc_calculate_pkt_len(skb, q);
3055 /*
3056 * Heuristic to force contended enqueues to serialize on a
3057 * separate lock before trying to get qdisc main lock.
3058 * This permits qdisc->running owner to get the lock more
3059 * often and dequeue packets faster.
3060 */
3061 contended = qdisc_is_running(q);
3062 if (unlikely(contended))
3063 spin_lock(&q->busylock);
3064
3065 spin_lock(root_lock);
3066 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3067 __qdisc_drop(skb, &to_free);
3068 rc = NET_XMIT_DROP;
3069 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3070 qdisc_run_begin(q)) {
3071 /*
3072 * This is a work-conserving queue; there are no old skbs
3073 * waiting to be sent out; and the qdisc is not running -
3074 * xmit the skb directly.
3075 */
3076
3077 qdisc_bstats_update(q, skb);
3078
3079 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3080 if (unlikely(contended)) {
3081 spin_unlock(&q->busylock);
3082 contended = false;
3083 }
3084 __qdisc_run(q);
3085 } else
3086 qdisc_run_end(q);
3087
3088 rc = NET_XMIT_SUCCESS;
3089 } else {
3090 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3091 if (qdisc_run_begin(q)) {
3092 if (unlikely(contended)) {
3093 spin_unlock(&q->busylock);
3094 contended = false;
3095 }
3096 __qdisc_run(q);
3097 }
3098 }
3099 spin_unlock(root_lock);
3100 if (unlikely(to_free))
3101 kfree_skb_list(to_free);
3102 if (unlikely(contended))
3103 spin_unlock(&q->busylock);
3104 return rc;
3105}
3106
3107#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3108static void skb_update_prio(struct sk_buff *skb)
3109{
3110 struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
3111
3112 if (!skb->priority && skb->sk && map) {
3113 unsigned int prioidx =
3114 sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
3115
3116 if (prioidx < map->priomap_len)
3117 skb->priority = map->priomap[prioidx];
3118 }
3119}
3120#else
3121#define skb_update_prio(skb)
3122#endif
3123
3124DEFINE_PER_CPU(int, xmit_recursion);
3125EXPORT_SYMBOL(xmit_recursion);
3126
3127/**
3128 * dev_loopback_xmit - loop back @skb
3129 * @net: network namespace this loopback is happening in
3130 * @sk: sk needed to be a netfilter okfn
3131 * @skb: buffer to transmit
3132 */
3133int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3134{
3135 skb_reset_mac_header(skb);
3136 __skb_pull(skb, skb_network_offset(skb));
3137 skb->pkt_type = PACKET_LOOPBACK;
3138 skb->ip_summed = CHECKSUM_UNNECESSARY;
3139 WARN_ON(!skb_dst(skb));
3140 skb_dst_force(skb);
3141 netif_rx_ni(skb);
3142 return 0;
3143}
3144EXPORT_SYMBOL(dev_loopback_xmit);
3145
3146#ifdef CONFIG_NET_EGRESS
3147static struct sk_buff *
3148sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3149{
3150 struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3151 struct tcf_result cl_res;
3152
3153 if (!cl)
3154 return skb;
3155
3156 /* skb->tc_verd and qdisc_skb_cb(skb)->pkt_len were already set
3157 * earlier by the caller.
3158 */
3159 qdisc_bstats_cpu_update(cl->q, skb);
3160
3161 switch (tc_classify(skb, cl, &cl_res, false)) {
3162 case TC_ACT_OK:
3163 case TC_ACT_RECLASSIFY:
3164 skb->tc_index = TC_H_MIN(cl_res.classid);
3165 break;
3166 case TC_ACT_SHOT:
3167 qdisc_qstats_cpu_drop(cl->q);
3168 *ret = NET_XMIT_DROP;
3169 kfree_skb(skb);
3170 return NULL;
3171 case TC_ACT_STOLEN:
3172 case TC_ACT_QUEUED:
3173 *ret = NET_XMIT_SUCCESS;
3174 consume_skb(skb);
3175 return NULL;
3176 case TC_ACT_REDIRECT:
3177 /* No need to push/pop skb's mac_header here on egress! */
3178 skb_do_redirect(skb);
3179 *ret = NET_XMIT_SUCCESS;
3180 return NULL;
3181 default:
3182 break;
3183 }
3184
3185 return skb;
3186}
3187#endif /* CONFIG_NET_EGRESS */
3188
3189static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3190{
3191#ifdef CONFIG_XPS
3192 struct xps_dev_maps *dev_maps;
3193 struct xps_map *map;
3194 int queue_index = -1;
3195
3196 rcu_read_lock();
3197 dev_maps = rcu_dereference(dev->xps_maps);
3198 if (dev_maps) {
3199 unsigned int tci = skb->sender_cpu - 1;
3200
3201 if (dev->num_tc) {
3202 tci *= dev->num_tc;
3203 tci += netdev_get_prio_tc_map(dev, skb->priority);
3204 }
3205
3206 map = rcu_dereference(dev_maps->cpu_map[tci]);
3207 if (map) {
3208 if (map->len == 1)
3209 queue_index = map->queues[0];
3210 else
3211 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3212 map->len)];
3213 if (unlikely(queue_index >= dev->real_num_tx_queues))
3214 queue_index = -1;
3215 }
3216 }
3217 rcu_read_unlock();
3218
3219 return queue_index;
3220#else
3221 return -1;
3222#endif
3223}
3224
3225static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3226{
3227 struct sock *sk = skb->sk;
3228 int queue_index = sk_tx_queue_get(sk);
3229
3230 if (queue_index < 0 || skb->ooo_okay ||
3231 queue_index >= dev->real_num_tx_queues) {
3232 int new_index = get_xps_queue(dev, skb);
3233 if (new_index < 0)
3234 new_index = skb_tx_hash(dev, skb);
3235
3236 if (queue_index != new_index && sk &&
3237 sk_fullsock(sk) &&
3238 rcu_access_pointer(sk->sk_dst_cache))
3239 sk_tx_queue_set(sk, new_index);
3240
3241 queue_index = new_index;
3242 }
3243
3244 return queue_index;
3245}
3246
3247struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3248 struct sk_buff *skb,
3249 void *accel_priv)
3250{
3251 int queue_index = 0;
3252
3253#ifdef CONFIG_XPS
3254 u32 sender_cpu = skb->sender_cpu - 1;
3255
3256 if (sender_cpu >= (u32)NR_CPUS)
3257 skb->sender_cpu = raw_smp_processor_id() + 1;
3258#endif
3259
3260 if (dev->real_num_tx_queues != 1) {
3261 const struct net_device_ops *ops = dev->netdev_ops;
3262 if (ops->ndo_select_queue)
3263 queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3264 __netdev_pick_tx);
3265 else
3266 queue_index = __netdev_pick_tx(dev, skb);
3267
3268 if (!accel_priv)
3269 queue_index = netdev_cap_txqueue(dev, queue_index);
3270 }
3271
3272 skb_set_queue_mapping(skb, queue_index);
3273 return netdev_get_tx_queue(dev, queue_index);
3274}
3275
3276/**
3277 * __dev_queue_xmit - transmit a buffer
3278 * @skb: buffer to transmit
3279 * @accel_priv: private data used for L2 forwarding offload
3280 *
3281 * Queue a buffer for transmission to a network device. The caller must
3282 * have set the device and priority and built the buffer before calling
3283 * this function. The function can be called from an interrupt.
3284 *
3285 * A negative errno code is returned on a failure. A success does not
3286 * guarantee the frame will be transmitted as it may be dropped due
3287 * to congestion or traffic shaping.
3288 *
3289 * -----------------------------------------------------------------------------------
3290 * I notice this method can also return errors from the queue disciplines,
3291 * including NET_XMIT_DROP, which is a positive value. So, errors can also
3292 * be positive.
3293 *
3294 * Regardless of the return value, the skb is consumed, so it is currently
3295 * difficult to retry a send to this method. (You can bump the ref count
3296 * before sending to hold a reference for retry if you are careful.)
3297 *
3298 * When calling this method, interrupts MUST be enabled. This is because
3299 * the BH enable code must have IRQs enabled so that it will not deadlock.
3300 * --BLG
3301 */
3302static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3303{
3304 struct net_device *dev = skb->dev;
3305 struct netdev_queue *txq;
3306 struct Qdisc *q;
3307 int rc = -ENOMEM;
3308
3309 skb_reset_mac_header(skb);
3310
3311 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3312 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3313
3314 /* Disable soft irqs for various locks below. Also
3315 * stops preemption for RCU.
3316 */
3317 rcu_read_lock_bh();
3318
3319 skb_update_prio(skb);
3320
3321 qdisc_pkt_len_init(skb);
3322#ifdef CONFIG_NET_CLS_ACT
3323 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
3324# ifdef CONFIG_NET_EGRESS
3325 if (static_key_false(&egress_needed)) {
3326 skb = sch_handle_egress(skb, &rc, dev);
3327 if (!skb)
3328 goto out;
3329 }
3330# endif
3331#endif
3332 /* If device/qdisc don't need skb->dst, release it right now while
3333 * its hot in this cpu cache.
3334 */
3335 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3336 skb_dst_drop(skb);
3337 else
3338 skb_dst_force(skb);
3339
3340 txq = netdev_pick_tx(dev, skb, accel_priv);
3341 q = rcu_dereference_bh(txq->qdisc);
3342
3343 trace_net_dev_queue(skb);
3344 if (q->enqueue) {
3345 rc = __dev_xmit_skb(skb, q, dev, txq);
3346 goto out;
3347 }
3348
3349 /* The device has no queue. Common case for software devices:
3350 loopback, all the sorts of tunnels...
3351
3352 Really, it is unlikely that netif_tx_lock protection is necessary
3353 here. (f.e. loopback and IP tunnels are clean ignoring statistics
3354 counters.)
3355 However, it is possible, that they rely on protection
3356 made by us here.
3357
3358 Check this and shot the lock. It is not prone from deadlocks.
3359 Either shot noqueue qdisc, it is even simpler 8)
3360 */
3361 if (dev->flags & IFF_UP) {
3362 int cpu = smp_processor_id(); /* ok because BHs are off */
3363
3364 if (txq->xmit_lock_owner != cpu) {
3365 if (unlikely(__this_cpu_read(xmit_recursion) >
3366 XMIT_RECURSION_LIMIT))
3367 goto recursion_alert;
3368
3369 skb = validate_xmit_skb(skb, dev);
3370 if (!skb)
3371 goto out;
3372
3373 HARD_TX_LOCK(dev, txq, cpu);
3374
3375 if (!netif_xmit_stopped(txq)) {
3376 __this_cpu_inc(xmit_recursion);
3377 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3378 __this_cpu_dec(xmit_recursion);
3379 if (dev_xmit_complete(rc)) {
3380 HARD_TX_UNLOCK(dev, txq);
3381 goto out;
3382 }
3383 }
3384 HARD_TX_UNLOCK(dev, txq);
3385 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3386 dev->name);
3387 } else {
3388 /* Recursion is detected! It is possible,
3389 * unfortunately
3390 */
3391recursion_alert:
3392 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3393 dev->name);
3394 }
3395 }
3396
3397 rc = -ENETDOWN;
3398 rcu_read_unlock_bh();
3399
3400 atomic_long_inc(&dev->tx_dropped);
3401 kfree_skb_list(skb);
3402 return rc;
3403out:
3404 rcu_read_unlock_bh();
3405 return rc;
3406}
3407
3408int dev_queue_xmit(struct sk_buff *skb)
3409{
3410 return __dev_queue_xmit(skb, NULL);
3411}
3412EXPORT_SYMBOL(dev_queue_xmit);
3413
3414int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3415{
3416 return __dev_queue_xmit(skb, accel_priv);
3417}
3418EXPORT_SYMBOL(dev_queue_xmit_accel);
3419
3420
3421/*=======================================================================
3422 Receiver routines
3423 =======================================================================*/
3424
3425int netdev_max_backlog __read_mostly = 1000;
3426EXPORT_SYMBOL(netdev_max_backlog);
3427
3428int netdev_tstamp_prequeue __read_mostly = 1;
3429int netdev_budget __read_mostly = 300;
3430int weight_p __read_mostly = 64; /* old backlog weight */
3431
3432/* Called with irq disabled */
3433static inline void ____napi_schedule(struct softnet_data *sd,
3434 struct napi_struct *napi)
3435{
3436 list_add_tail(&napi->poll_list, &sd->poll_list);
3437 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3438}
3439
3440#ifdef CONFIG_RPS
3441
3442/* One global table that all flow-based protocols share. */
3443struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3444EXPORT_SYMBOL(rps_sock_flow_table);
3445u32 rps_cpu_mask __read_mostly;
3446EXPORT_SYMBOL(rps_cpu_mask);
3447
3448struct static_key rps_needed __read_mostly;
3449EXPORT_SYMBOL(rps_needed);
3450struct static_key rfs_needed __read_mostly;
3451EXPORT_SYMBOL(rfs_needed);
3452
3453static struct rps_dev_flow *
3454set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3455 struct rps_dev_flow *rflow, u16 next_cpu)
3456{
3457 if (next_cpu < nr_cpu_ids) {
3458#ifdef CONFIG_RFS_ACCEL
3459 struct netdev_rx_queue *rxqueue;
3460 struct rps_dev_flow_table *flow_table;
3461 struct rps_dev_flow *old_rflow;
3462 u32 flow_id;
3463 u16 rxq_index;
3464 int rc;
3465
3466 /* Should we steer this flow to a different hardware queue? */
3467 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3468 !(dev->features & NETIF_F_NTUPLE))
3469 goto out;
3470 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3471 if (rxq_index == skb_get_rx_queue(skb))
3472 goto out;
3473
3474 rxqueue = dev->_rx + rxq_index;
3475 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3476 if (!flow_table)
3477 goto out;
3478 flow_id = skb_get_hash(skb) & flow_table->mask;
3479 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3480 rxq_index, flow_id);
3481 if (rc < 0)
3482 goto out;
3483 old_rflow = rflow;
3484 rflow = &flow_table->flows[flow_id];
3485 rflow->filter = rc;
3486 if (old_rflow->filter == rflow->filter)
3487 old_rflow->filter = RPS_NO_FILTER;
3488 out:
3489#endif
3490 rflow->last_qtail =
3491 per_cpu(softnet_data, next_cpu).input_queue_head;
3492 }
3493
3494 rflow->cpu = next_cpu;
3495 return rflow;
3496}
3497
3498/*
3499 * get_rps_cpu is called from netif_receive_skb and returns the target
3500 * CPU from the RPS map of the receiving queue for a given skb.
3501 * rcu_read_lock must be held on entry.
3502 */
3503static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3504 struct rps_dev_flow **rflowp)
3505{
3506 const struct rps_sock_flow_table *sock_flow_table;
3507 struct netdev_rx_queue *rxqueue = dev->_rx;
3508 struct rps_dev_flow_table *flow_table;
3509 struct rps_map *map;
3510 int cpu = -1;
3511 u32 tcpu;
3512 u32 hash;
3513
3514 if (skb_rx_queue_recorded(skb)) {
3515 u16 index = skb_get_rx_queue(skb);
3516
3517 if (unlikely(index >= dev->real_num_rx_queues)) {
3518 WARN_ONCE(dev->real_num_rx_queues > 1,
3519 "%s received packet on queue %u, but number "
3520 "of RX queues is %u\n",
3521 dev->name, index, dev->real_num_rx_queues);
3522 goto done;
3523 }
3524 rxqueue += index;
3525 }
3526
3527 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3528
3529 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3530 map = rcu_dereference(rxqueue->rps_map);
3531 if (!flow_table && !map)
3532 goto done;
3533
3534 skb_reset_network_header(skb);
3535 hash = skb_get_hash(skb);
3536 if (!hash)
3537 goto done;
3538
3539 sock_flow_table = rcu_dereference(rps_sock_flow_table);
3540 if (flow_table && sock_flow_table) {
3541 struct rps_dev_flow *rflow;
3542 u32 next_cpu;
3543 u32 ident;
3544
3545 /* First check into global flow table if there is a match */
3546 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3547 if ((ident ^ hash) & ~rps_cpu_mask)
3548 goto try_rps;
3549
3550 next_cpu = ident & rps_cpu_mask;
3551
3552 /* OK, now we know there is a match,
3553 * we can look at the local (per receive queue) flow table
3554 */
3555 rflow = &flow_table->flows[hash & flow_table->mask];
3556 tcpu = rflow->cpu;
3557
3558 /*
3559 * If the desired CPU (where last recvmsg was done) is
3560 * different from current CPU (one in the rx-queue flow
3561 * table entry), switch if one of the following holds:
3562 * - Current CPU is unset (>= nr_cpu_ids).
3563 * - Current CPU is offline.
3564 * - The current CPU's queue tail has advanced beyond the
3565 * last packet that was enqueued using this table entry.
3566 * This guarantees that all previous packets for the flow
3567 * have been dequeued, thus preserving in order delivery.
3568 */
3569 if (unlikely(tcpu != next_cpu) &&
3570 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3571 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3572 rflow->last_qtail)) >= 0)) {
3573 tcpu = next_cpu;
3574 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3575 }
3576
3577 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3578 *rflowp = rflow;
3579 cpu = tcpu;
3580 goto done;
3581 }
3582 }
3583
3584try_rps:
3585
3586 if (map) {
3587 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3588 if (cpu_online(tcpu)) {
3589 cpu = tcpu;
3590 goto done;
3591 }
3592 }
3593
3594done:
3595 return cpu;
3596}
3597
3598#ifdef CONFIG_RFS_ACCEL
3599
3600/**
3601 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3602 * @dev: Device on which the filter was set
3603 * @rxq_index: RX queue index
3604 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3605 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3606 *
3607 * Drivers that implement ndo_rx_flow_steer() should periodically call
3608 * this function for each installed filter and remove the filters for
3609 * which it returns %true.
3610 */
3611bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3612 u32 flow_id, u16 filter_id)
3613{
3614 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3615 struct rps_dev_flow_table *flow_table;
3616 struct rps_dev_flow *rflow;
3617 bool expire = true;
3618 unsigned int cpu;
3619
3620 rcu_read_lock();
3621 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3622 if (flow_table && flow_id <= flow_table->mask) {
3623 rflow = &flow_table->flows[flow_id];
3624 cpu = ACCESS_ONCE(rflow->cpu);
3625 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3626 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3627 rflow->last_qtail) <
3628 (int)(10 * flow_table->mask)))
3629 expire = false;
3630 }
3631 rcu_read_unlock();
3632 return expire;
3633}
3634EXPORT_SYMBOL(rps_may_expire_flow);
3635
3636#endif /* CONFIG_RFS_ACCEL */
3637
3638/* Called from hardirq (IPI) context */
3639static void rps_trigger_softirq(void *data)
3640{
3641 struct softnet_data *sd = data;
3642
3643 ____napi_schedule(sd, &sd->backlog);
3644 sd->received_rps++;
3645}
3646
3647#endif /* CONFIG_RPS */
3648
3649/*
3650 * Check if this softnet_data structure is another cpu one
3651 * If yes, queue it to our IPI list and return 1
3652 * If no, return 0
3653 */
3654static int rps_ipi_queued(struct softnet_data *sd)
3655{
3656#ifdef CONFIG_RPS
3657 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3658
3659 if (sd != mysd) {
3660 sd->rps_ipi_next = mysd->rps_ipi_list;
3661 mysd->rps_ipi_list = sd;
3662
3663 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3664 return 1;
3665 }
3666#endif /* CONFIG_RPS */
3667 return 0;
3668}
3669
3670#ifdef CONFIG_NET_FLOW_LIMIT
3671int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3672#endif
3673
3674static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3675{
3676#ifdef CONFIG_NET_FLOW_LIMIT
3677 struct sd_flow_limit *fl;
3678 struct softnet_data *sd;
3679 unsigned int old_flow, new_flow;
3680
3681 if (qlen < (netdev_max_backlog >> 1))
3682 return false;
3683
3684 sd = this_cpu_ptr(&softnet_data);
3685
3686 rcu_read_lock();
3687 fl = rcu_dereference(sd->flow_limit);
3688 if (fl) {
3689 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3690 old_flow = fl->history[fl->history_head];
3691 fl->history[fl->history_head] = new_flow;
3692
3693 fl->history_head++;
3694 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3695
3696 if (likely(fl->buckets[old_flow]))
3697 fl->buckets[old_flow]--;
3698
3699 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3700 fl->count++;
3701 rcu_read_unlock();
3702 return true;
3703 }
3704 }
3705 rcu_read_unlock();
3706#endif
3707 return false;
3708}
3709
3710/*
3711 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3712 * queue (may be a remote CPU queue).
3713 */
3714static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3715 unsigned int *qtail)
3716{
3717 struct softnet_data *sd;
3718 unsigned long flags;
3719 unsigned int qlen;
3720
3721 sd = &per_cpu(softnet_data, cpu);
3722
3723 local_irq_save(flags);
3724
3725 rps_lock(sd);
3726 if (!netif_running(skb->dev))
3727 goto drop;
3728 qlen = skb_queue_len(&sd->input_pkt_queue);
3729 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3730 if (qlen) {
3731enqueue:
3732 __skb_queue_tail(&sd->input_pkt_queue, skb);
3733 input_queue_tail_incr_save(sd, qtail);
3734 rps_unlock(sd);
3735 local_irq_restore(flags);
3736 return NET_RX_SUCCESS;
3737 }
3738
3739 /* Schedule NAPI for backlog device
3740 * We can use non atomic operation since we own the queue lock
3741 */
3742 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3743 if (!rps_ipi_queued(sd))
3744 ____napi_schedule(sd, &sd->backlog);
3745 }
3746 goto enqueue;
3747 }
3748
3749drop:
3750 sd->dropped++;
3751 rps_unlock(sd);
3752
3753 local_irq_restore(flags);
3754
3755 atomic_long_inc(&skb->dev->rx_dropped);
3756 kfree_skb(skb);
3757 return NET_RX_DROP;
3758}
3759
3760static int netif_rx_internal(struct sk_buff *skb)
3761{
3762 int ret;
3763
3764 net_timestamp_check(netdev_tstamp_prequeue, skb);
3765
3766 trace_netif_rx(skb);
3767#ifdef CONFIG_RPS
3768 if (static_key_false(&rps_needed)) {
3769 struct rps_dev_flow voidflow, *rflow = &voidflow;
3770 int cpu;
3771
3772 preempt_disable();
3773 rcu_read_lock();
3774
3775 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3776 if (cpu < 0)
3777 cpu = smp_processor_id();
3778
3779 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3780
3781 rcu_read_unlock();
3782 preempt_enable();
3783 } else
3784#endif
3785 {
3786 unsigned int qtail;
3787 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3788 put_cpu();
3789 }
3790 return ret;
3791}
3792
3793/**
3794 * netif_rx - post buffer to the network code
3795 * @skb: buffer to post
3796 *
3797 * This function receives a packet from a device driver and queues it for
3798 * the upper (protocol) levels to process. It always succeeds. The buffer
3799 * may be dropped during processing for congestion control or by the
3800 * protocol layers.
3801 *
3802 * return values:
3803 * NET_RX_SUCCESS (no congestion)
3804 * NET_RX_DROP (packet was dropped)
3805 *
3806 */
3807
3808int netif_rx(struct sk_buff *skb)
3809{
3810 trace_netif_rx_entry(skb);
3811
3812 return netif_rx_internal(skb);
3813}
3814EXPORT_SYMBOL(netif_rx);
3815
3816int netif_rx_ni(struct sk_buff *skb)
3817{
3818 int err;
3819
3820 trace_netif_rx_ni_entry(skb);
3821
3822 preempt_disable();
3823 err = netif_rx_internal(skb);
3824 if (local_softirq_pending())
3825 do_softirq();
3826 preempt_enable();
3827
3828 return err;
3829}
3830EXPORT_SYMBOL(netif_rx_ni);
3831
3832static __latent_entropy void net_tx_action(struct softirq_action *h)
3833{
3834 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
3835
3836 if (sd->completion_queue) {
3837 struct sk_buff *clist;
3838
3839 local_irq_disable();
3840 clist = sd->completion_queue;
3841 sd->completion_queue = NULL;
3842 local_irq_enable();
3843
3844 while (clist) {
3845 struct sk_buff *skb = clist;
3846 clist = clist->next;
3847
3848 WARN_ON(atomic_read(&skb->users));
3849 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3850 trace_consume_skb(skb);
3851 else
3852 trace_kfree_skb(skb, net_tx_action);
3853
3854 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
3855 __kfree_skb(skb);
3856 else
3857 __kfree_skb_defer(skb);
3858 }
3859
3860 __kfree_skb_flush();
3861 }
3862
3863 if (sd->output_queue) {
3864 struct Qdisc *head;
3865
3866 local_irq_disable();
3867 head = sd->output_queue;
3868 sd->output_queue = NULL;
3869 sd->output_queue_tailp = &sd->output_queue;
3870 local_irq_enable();
3871
3872 while (head) {
3873 struct Qdisc *q = head;
3874 spinlock_t *root_lock;
3875
3876 head = head->next_sched;
3877
3878 root_lock = qdisc_lock(q);
3879 spin_lock(root_lock);
3880 /* We need to make sure head->next_sched is read
3881 * before clearing __QDISC_STATE_SCHED
3882 */
3883 smp_mb__before_atomic();
3884 clear_bit(__QDISC_STATE_SCHED, &q->state);
3885 qdisc_run(q);
3886 spin_unlock(root_lock);
3887 }
3888 }
3889}
3890
3891#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
3892/* This hook is defined here for ATM LANE */
3893int (*br_fdb_test_addr_hook)(struct net_device *dev,
3894 unsigned char *addr) __read_mostly;
3895EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3896#endif
3897
3898static inline struct sk_buff *
3899sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
3900 struct net_device *orig_dev)
3901{
3902#ifdef CONFIG_NET_CLS_ACT
3903 struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3904 struct tcf_result cl_res;
3905
3906 /* If there's at least one ingress present somewhere (so
3907 * we get here via enabled static key), remaining devices
3908 * that are not configured with an ingress qdisc will bail
3909 * out here.
3910 */
3911 if (!cl)
3912 return skb;
3913 if (*pt_prev) {
3914 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3915 *pt_prev = NULL;
3916 }
3917
3918 qdisc_skb_cb(skb)->pkt_len = skb->len;
3919 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3920 qdisc_bstats_cpu_update(cl->q, skb);
3921
3922 switch (tc_classify(skb, cl, &cl_res, false)) {
3923 case TC_ACT_OK:
3924 case TC_ACT_RECLASSIFY:
3925 skb->tc_index = TC_H_MIN(cl_res.classid);
3926 break;
3927 case TC_ACT_SHOT:
3928 qdisc_qstats_cpu_drop(cl->q);
3929 kfree_skb(skb);
3930 return NULL;
3931 case TC_ACT_STOLEN:
3932 case TC_ACT_QUEUED:
3933 consume_skb(skb);
3934 return NULL;
3935 case TC_ACT_REDIRECT:
3936 /* skb_mac_header check was done by cls/act_bpf, so
3937 * we can safely push the L2 header back before
3938 * redirecting to another netdev
3939 */
3940 __skb_push(skb, skb->mac_len);
3941 skb_do_redirect(skb);
3942 return NULL;
3943 default:
3944 break;
3945 }
3946#endif /* CONFIG_NET_CLS_ACT */
3947 return skb;
3948}
3949
3950/**
3951 * netdev_is_rx_handler_busy - check if receive handler is registered
3952 * @dev: device to check
3953 *
3954 * Check if a receive handler is already registered for a given device.
3955 * Return true if there one.
3956 *
3957 * The caller must hold the rtnl_mutex.
3958 */
3959bool netdev_is_rx_handler_busy(struct net_device *dev)
3960{
3961 ASSERT_RTNL();
3962 return dev && rtnl_dereference(dev->rx_handler);
3963}
3964EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
3965
3966/**
3967 * netdev_rx_handler_register - register receive handler
3968 * @dev: device to register a handler for
3969 * @rx_handler: receive handler to register
3970 * @rx_handler_data: data pointer that is used by rx handler
3971 *
3972 * Register a receive handler for a device. This handler will then be
3973 * called from __netif_receive_skb. A negative errno code is returned
3974 * on a failure.
3975 *
3976 * The caller must hold the rtnl_mutex.
3977 *
3978 * For a general description of rx_handler, see enum rx_handler_result.
3979 */
3980int netdev_rx_handler_register(struct net_device *dev,
3981 rx_handler_func_t *rx_handler,
3982 void *rx_handler_data)
3983{
3984 ASSERT_RTNL();
3985
3986 if (dev->rx_handler)
3987 return -EBUSY;
3988
3989 /* Note: rx_handler_data must be set before rx_handler */
3990 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
3991 rcu_assign_pointer(dev->rx_handler, rx_handler);
3992
3993 return 0;
3994}
3995EXPORT_SYMBOL_GPL(