1/*
2 * NET3 Protocol independent device support routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *
9 * Derived from the non IP parts of dev.c 1.0.19
10 * Authors: Ross Biro
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 *
14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 *
22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
44 * call a packet.
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
73 */
74
75#include <linux/uaccess.h>
76#include <linux/bitops.h>
77#include <linux/capability.h>
78#include <linux/cpu.h>
79#include <linux/types.h>
80#include <linux/kernel.h>
81#include <linux/hash.h>
82#include <linux/slab.h>
83#include <linux/sched.h>
84#include <linux/sched/mm.h>
85#include <linux/mutex.h>
86#include <linux/string.h>
87#include <linux/mm.h>
88#include <linux/socket.h>
89#include <linux/sockios.h>
90#include <linux/errno.h>
91#include <linux/interrupt.h>
92#include <linux/if_ether.h>
93#include <linux/netdevice.h>
94#include <linux/etherdevice.h>
95#include <linux/ethtool.h>
96#include <linux/skbuff.h>
97#include <linux/bpf.h>
98#include <linux/bpf_trace.h>
99#include <net/net_namespace.h>
100#include <net/sock.h>
101#include <net/busy_poll.h>
102#include <linux/rtnetlink.h>
103#include <linux/stat.h>
104#include <net/dst.h>
105#include <net/dst_metadata.h>
106#include <net/pkt_sched.h>
107#include <net/pkt_cls.h>
108#include <net/checksum.h>
109#include <net/xfrm.h>
110#include <linux/highmem.h>
111#include <linux/init.h>
112#include <linux/module.h>
113#include <linux/netpoll.h>
114#include <linux/rcupdate.h>
115#include <linux/delay.h>
116#include <net/iw_handler.h>
117#include <asm/current.h>
118#include <linux/audit.h>
119#include <linux/dmaengine.h>
120#include <linux/err.h>
121#include <linux/ctype.h>
122#include <linux/if_arp.h>
123#include <linux/if_vlan.h>
124#include <linux/ip.h>
125#include <net/ip.h>
126#include <net/mpls.h>
127#include <linux/ipv6.h>
128#include <linux/in.h>
129#include <linux/jhash.h>
130#include <linux/random.h>
131#include <trace/events/napi.h>
132#include <trace/events/net.h>
133#include <trace/events/skb.h>
134#include <linux/pci.h>
135#include <linux/inetdevice.h>
136#include <linux/cpu_rmap.h>
137#include <linux/static_key.h>
138#include <linux/hashtable.h>
139#include <linux/vmalloc.h>
140#include <linux/if_macvlan.h>
141#include <linux/errqueue.h>
142#include <linux/hrtimer.h>
143#include <linux/netfilter_ingress.h>
144#include <linux/crash_dump.h>
145#include <linux/sctp.h>
146#include <net/udp_tunnel.h>
147#include <linux/net_namespace.h>
148#include <linux/indirect_call_wrapper.h>
149
150#include "net-sysfs.h"
151
152#define MAX_GRO_SKBS 8
153
154/* This should be increased if a protocol with a bigger head is added. */
155#define GRO_MAX_HEAD (MAX_HEADER + 128)
156
157static DEFINE_SPINLOCK(ptype_lock);
158static DEFINE_SPINLOCK(offload_lock);
159struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
160struct list_head ptype_all __read_mostly; /* Taps */
161static struct list_head offload_base __read_mostly;
162
163static int netif_rx_internal(struct sk_buff *skb);
164static int call_netdevice_notifiers_info(unsigned long val,
165 struct netdev_notifier_info *info);
166static int call_netdevice_notifiers_extack(unsigned long val,
167 struct net_device *dev,
168 struct netlink_ext_ack *extack);
169static struct napi_struct *napi_by_id(unsigned int napi_id);
170
171/*
172 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
173 * semaphore.
174 *
175 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
176 *
177 * Writers must hold the rtnl semaphore while they loop through the
178 * dev_base_head list, and hold dev_base_lock for writing when they do the
179 * actual updates. This allows pure readers to access the list even
180 * while a writer is preparing to update it.
181 *
182 * To put it another way, dev_base_lock is held for writing only to
183 * protect against pure readers; the rtnl semaphore provides the
184 * protection against other writers.
185 *
186 * See, for example usages, register_netdevice() and
187 * unregister_netdevice(), which must be called with the rtnl
188 * semaphore held.
189 */
190DEFINE_RWLOCK(dev_base_lock);
191EXPORT_SYMBOL(dev_base_lock);
192
193static DEFINE_MUTEX(ifalias_mutex);
194
195/* protects napi_hash addition/deletion and napi_gen_id */
196static DEFINE_SPINLOCK(napi_hash_lock);
197
198static unsigned int napi_gen_id = NR_CPUS;
199static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
200
201static seqcount_t devnet_rename_seq;
202
203static inline void dev_base_seq_inc(struct net *net)
204{
205 while (++net->dev_base_seq == 0)
206 ;
207}
208
209static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
210{
211 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
212
213 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
214}
215
216static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
217{
218 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
219}
220
221static inline void rps_lock(struct softnet_data *sd)
222{
223#ifdef CONFIG_RPS
224 spin_lock(&sd->input_pkt_queue.lock);
225#endif
226}
227
228static inline void rps_unlock(struct softnet_data *sd)
229{
230#ifdef CONFIG_RPS
231 spin_unlock(&sd->input_pkt_queue.lock);
232#endif
233}
234
235/* Device list insertion */
236static void list_netdevice(struct net_device *dev)
237{
238 struct net *net = dev_net(dev);
239
240 ASSERT_RTNL();
241
242 write_lock_bh(&dev_base_lock);
243 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
244 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
245 hlist_add_head_rcu(&dev->index_hlist,
246 dev_index_hash(net, dev->ifindex));
247 write_unlock_bh(&dev_base_lock);
248
249 dev_base_seq_inc(net);
250}
251
252/* Device list removal
253 * caller must respect a RCU grace period before freeing/reusing dev
254 */
255static void unlist_netdevice(struct net_device *dev)
256{
257 ASSERT_RTNL();
258
259 /* Unlink dev from the device chain */
260 write_lock_bh(&dev_base_lock);
261 list_del_rcu(&dev->dev_list);
262 hlist_del_rcu(&dev->name_hlist);
263 hlist_del_rcu(&dev->index_hlist);
264 write_unlock_bh(&dev_base_lock);
265
266 dev_base_seq_inc(dev_net(dev));
267}
268
269/*
270 * Our notifier list
271 */
272
273static RAW_NOTIFIER_HEAD(netdev_chain);
274
275/*
276 * Device drivers call our routines to queue packets here. We empty the
277 * queue in the local softnet handler.
278 */
279
280DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
281EXPORT_PER_CPU_SYMBOL(softnet_data);
282
283#ifdef CONFIG_LOCKDEP
284/*
285 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
286 * according to dev->type
287 */
288static const unsigned short netdev_lock_type[] = {
289 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
290 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
291 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
292 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
293 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
294 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
295 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
296 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
297 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
298 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
299 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
300 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
301 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
302 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
303 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
304
305static const char *const netdev_lock_name[] = {
306 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
307 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
308 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
309 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
310 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
311 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
312 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
313 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
314 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
315 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
316 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
317 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
318 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
319 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
320 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
321
322static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
323static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
324
325static inline unsigned short netdev_lock_pos(unsigned short dev_type)
326{
327 int i;
328
329 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
330 if (netdev_lock_type[i] == dev_type)
331 return i;
332 /* the last key is used by default */
333 return ARRAY_SIZE(netdev_lock_type) - 1;
334}
335
336static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
337 unsigned short dev_type)
338{
339 int i;
340
341 i = netdev_lock_pos(dev_type);
342 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
343 netdev_lock_name[i]);
344}
345
346static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
347{
348 int i;
349
350 i = netdev_lock_pos(dev->type);
351 lockdep_set_class_and_name(&dev->addr_list_lock,
352 &netdev_addr_lock_key[i],
353 netdev_lock_name[i]);
354}
355#else
356static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
357 unsigned short dev_type)
358{
359}
360static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
361{
362}
363#endif
364
365/*******************************************************************************
366 *
367 * Protocol management and registration routines
368 *
369 *******************************************************************************/
370
371
372/*
373 * Add a protocol ID to the list. Now that the input handler is
374 * smarter we can dispense with all the messy stuff that used to be
375 * here.
376 *
377 * BEWARE!!! Protocol handlers, mangling input packets,
378 * MUST BE last in hash buckets and checking protocol handlers
379 * MUST start from promiscuous ptype_all chain in net_bh.
380 * It is true now, do not change it.
381 * Explanation follows: if protocol handler, mangling packet, will
382 * be the first on list, it is not able to sense, that packet
383 * is cloned and should be copied-on-write, so that it will
384 * change it and subsequent readers will get broken packet.
385 * --ANK (980803)
386 */
387
388static inline struct list_head *ptype_head(const struct packet_type *pt)
389{
390 if (pt->type == htons(ETH_P_ALL))
391 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
392 else
393 return pt->dev ? &pt->dev->ptype_specific :
394 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
395}
396
397/**
398 * dev_add_pack - add packet handler
399 * @pt: packet type declaration
400 *
401 * Add a protocol handler to the networking stack. The passed &packet_type
402 * is linked into kernel lists and may not be freed until it has been
403 * removed from the kernel lists.
404 *
405 * This call does not sleep therefore it can not
406 * guarantee all CPU's that are in middle of receiving packets
407 * will see the new packet type (until the next received packet).
408 */
409
410void dev_add_pack(struct packet_type *pt)
411{
412 struct list_head *head = ptype_head(pt);
413
414 spin_lock(&ptype_lock);
415 list_add_rcu(&pt->list, head);
416 spin_unlock(&ptype_lock);
417}
418EXPORT_SYMBOL(dev_add_pack);
419
420/**
421 * __dev_remove_pack - remove packet handler
422 * @pt: packet type declaration
423 *
424 * Remove a protocol handler that was previously added to the kernel
425 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
426 * from the kernel lists and can be freed or reused once this function
427 * returns.
428 *
429 * The packet type might still be in use by receivers
430 * and must not be freed until after all the CPU's have gone
431 * through a quiescent state.
432 */
433void __dev_remove_pack(struct packet_type *pt)
434{
435 struct list_head *head = ptype_head(pt);
436 struct packet_type *pt1;
437
438 spin_lock(&ptype_lock);
439
440 list_for_each_entry(pt1, head, list) {
441 if (pt == pt1) {
442 list_del_rcu(&pt->list);
443 goto out;
444 }
445 }
446
447 pr_warn("dev_remove_pack: %p not found\n", pt);
448out:
449 spin_unlock(&ptype_lock);
450}
451EXPORT_SYMBOL(__dev_remove_pack);
452
453/**
454 * dev_remove_pack - remove packet handler
455 * @pt: packet type declaration
456 *
457 * Remove a protocol handler that was previously added to the kernel
458 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
459 * from the kernel lists and can be freed or reused once this function
460 * returns.
461 *
462 * This call sleeps to guarantee that no CPU is looking at the packet
463 * type after return.
464 */
465void dev_remove_pack(struct packet_type *pt)
466{
467 __dev_remove_pack(pt);
468
469 synchronize_net();
470}
471EXPORT_SYMBOL(dev_remove_pack);
472
473
474/**
475 * dev_add_offload - register offload handlers
476 * @po: protocol offload declaration
477 *
478 * Add protocol offload handlers to the networking stack. The passed
479 * &proto_offload is linked into kernel lists and may not be freed until
480 * it has been removed from the kernel lists.
481 *
482 * This call does not sleep therefore it can not
483 * guarantee all CPU's that are in middle of receiving packets
484 * will see the new offload handlers (until the next received packet).
485 */
486void dev_add_offload(struct packet_offload *po)
487{
488 struct packet_offload *elem;
489
490 spin_lock(&offload_lock);
491 list_for_each_entry(elem, &offload_base, list) {
492 if (po->priority < elem->priority)
493 break;
494 }
495 list_add_rcu(&po->list, elem->list.prev);
496 spin_unlock(&offload_lock);
497}
498EXPORT_SYMBOL(dev_add_offload);
499
500/**
501 * __dev_remove_offload - remove offload handler
502 * @po: packet offload declaration
503 *
504 * Remove a protocol offload handler that was previously added to the
505 * kernel offload handlers by dev_add_offload(). The passed &offload_type
506 * is removed from the kernel lists and can be freed or reused once this
507 * function returns.
508 *
509 * The packet type might still be in use by receivers
510 * and must not be freed until after all the CPU's have gone
511 * through a quiescent state.
512 */
513static void __dev_remove_offload(struct packet_offload *po)
514{
515 struct list_head *head = &offload_base;
516 struct packet_offload *po1;
517
518 spin_lock(&offload_lock);
519
520 list_for_each_entry(po1, head, list) {
521 if (po == po1) {
522 list_del_rcu(&po->list);
523 goto out;
524 }
525 }
526
527 pr_warn("dev_remove_offload: %p not found\n", po);
528out:
529 spin_unlock(&offload_lock);
530}
531
532/**
533 * dev_remove_offload - remove packet offload handler
534 * @po: packet offload declaration
535 *
536 * Remove a packet offload handler that was previously added to the kernel
537 * offload handlers by dev_add_offload(). The passed &offload_type is
538 * removed from the kernel lists and can be freed or reused once this
539 * function returns.
540 *
541 * This call sleeps to guarantee that no CPU is looking at the packet
542 * type after return.
543 */
544void dev_remove_offload(struct packet_offload *po)
545{
546 __dev_remove_offload(po);
547
548 synchronize_net();
549}
550EXPORT_SYMBOL(dev_remove_offload);
551
552/******************************************************************************
553 *
554 * Device Boot-time Settings Routines
555 *
556 ******************************************************************************/
557
558/* Boot time configuration table */
559static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
560
561/**
562 * netdev_boot_setup_add - add new setup entry
563 * @name: name of the device
564 * @map: configured settings for the device
565 *
566 * Adds new setup entry to the dev_boot_setup list. The function
567 * returns 0 on error and 1 on success. This is a generic routine to
568 * all netdevices.
569 */
570static int netdev_boot_setup_add(char *name, struct ifmap *map)
571{
572 struct netdev_boot_setup *s;
573 int i;
574
575 s = dev_boot_setup;
576 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
577 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
578 memset(s[i].name, 0, sizeof(s[i].name));
579 strlcpy(s[i].name, name, IFNAMSIZ);
580 memcpy(&s[i].map, map, sizeof(s[i].map));
581 break;
582 }
583 }
584
585 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
586}
587
588/**
589 * netdev_boot_setup_check - check boot time settings
590 * @dev: the netdevice
591 *
592 * Check boot time settings for the device.
593 * The found settings are set for the device to be used
594 * later in the device probing.
595 * Returns 0 if no settings found, 1 if they are.
596 */
597int netdev_boot_setup_check(struct net_device *dev)
598{
599 struct netdev_boot_setup *s = dev_boot_setup;
600 int i;
601
602 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
603 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
604 !strcmp(dev->name, s[i].name)) {
605 dev->irq = s[i].map.irq;
606 dev->base_addr = s[i].map.base_addr;
607 dev->mem_start = s[i].map.mem_start;
608 dev->mem_end = s[i].map.mem_end;
609 return 1;
610 }
611 }
612 return 0;
613}
614EXPORT_SYMBOL(netdev_boot_setup_check);
615
616
617/**
618 * netdev_boot_base - get address from boot time settings
619 * @prefix: prefix for network device
620 * @unit: id for network device
621 *
622 * Check boot time settings for the base address of device.
623 * The found settings are set for the device to be used
624 * later in the device probing.
625 * Returns 0 if no settings found.
626 */
627unsigned long netdev_boot_base(const char *prefix, int unit)
628{
629 const struct netdev_boot_setup *s = dev_boot_setup;
630 char name[IFNAMSIZ];
631 int i;
632
633 sprintf(name, "%s%d", prefix, unit);
634
635 /*
636 * If device already registered then return base of 1
637 * to indicate not to probe for this interface
638 */
639 if (__dev_get_by_name(&init_net, name))
640 return 1;
641
642 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
643 if (!strcmp(name, s[i].name))
644 return s[i].map.base_addr;
645 return 0;
646}
647
648/*
649 * Saves at boot time configured settings for any netdevice.
650 */
651int __init netdev_boot_setup(char *str)
652{
653 int ints[5];
654 struct ifmap map;
655
656 str = get_options(str, ARRAY_SIZE(ints), ints);
657 if (!str || !*str)
658 return 0;
659
660 /* Save settings */
661 memset(&map, 0, sizeof(map));
662 if (ints[0] > 0)
663 map.irq = ints[1];
664 if (ints[0] > 1)
665 map.base_addr = ints[2];
666 if (ints[0] > 2)
667 map.mem_start = ints[3];
668 if (ints[0] > 3)
669 map.mem_end = ints[4];
670
671 /* Add new entry to the list */
672 return netdev_boot_setup_add(str, &map);
673}
674
675__setup("netdev=", netdev_boot_setup);
676
677/*******************************************************************************
678 *
679 * Device Interface Subroutines
680 *
681 *******************************************************************************/
682
683/**
684 * dev_get_iflink - get 'iflink' value of a interface
685 * @dev: targeted interface
686 *
687 * Indicates the ifindex the interface is linked to.
688 * Physical interfaces have the same 'ifindex' and 'iflink' values.
689 */
690
691int dev_get_iflink(const struct net_device *dev)
692{
693 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
694 return dev->netdev_ops->ndo_get_iflink(dev);
695
696 return dev->ifindex;
697}
698EXPORT_SYMBOL(dev_get_iflink);
699
700/**
701 * dev_fill_metadata_dst - Retrieve tunnel egress information.
702 * @dev: targeted interface
703 * @skb: The packet.
704 *
705 * For better visibility of tunnel traffic OVS needs to retrieve
706 * egress tunnel information for a packet. Following API allows
707 * user to get this info.
708 */
709int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
710{
711 struct ip_tunnel_info *info;
712
713 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
714 return -EINVAL;
715
716 info = skb_tunnel_info_unclone(skb);
717 if (!info)
718 return -ENOMEM;
719 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
720 return -EINVAL;
721
722 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
723}
724EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
725
726/**
727 * __dev_get_by_name - find a device by its name
728 * @net: the applicable net namespace
729 * @name: name to find
730 *
731 * Find an interface by name. Must be called under RTNL semaphore
732 * or @dev_base_lock. If the name is found a pointer to the device
733 * is returned. If the name is not found then %NULL is returned. The
734 * reference counters are not incremented so the caller must be
735 * careful with locks.
736 */
737
738struct net_device *__dev_get_by_name(struct net *net, const char *name)
739{
740 struct net_device *dev;
741 struct hlist_head *head = dev_name_hash(net, name);
742
743 hlist_for_each_entry(dev, head, name_hlist)
744 if (!strncmp(dev->name, name, IFNAMSIZ))
745 return dev;
746
747 return NULL;
748}
749EXPORT_SYMBOL(__dev_get_by_name);
750
751/**
752 * dev_get_by_name_rcu - find a device by its name
753 * @net: the applicable net namespace
754 * @name: name to find
755 *
756 * Find an interface by name.
757 * If the name is found a pointer to the device is returned.
758 * If the name is not found then %NULL is returned.
759 * The reference counters are not incremented so the caller must be
760 * careful with locks. The caller must hold RCU lock.
761 */
762
763struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
764{
765 struct net_device *dev;
766 struct hlist_head *head = dev_name_hash(net, name);
767
768 hlist_for_each_entry_rcu(dev, head, name_hlist)
769 if (!strncmp(dev->name, name, IFNAMSIZ))
770 return dev;
771
772 return NULL;
773}
774EXPORT_SYMBOL(dev_get_by_name_rcu);
775
776/**
777 * dev_get_by_name - find a device by its name
778 * @net: the applicable net namespace
779 * @name: name to find
780 *
781 * Find an interface by name. This can be called from any
782 * context and does its own locking. The returned handle has
783 * the usage count incremented and the caller must use dev_put() to
784 * release it when it is no longer needed. %NULL is returned if no
785 * matching device is found.
786 */
787
788struct net_device *dev_get_by_name(struct net *net, const char *name)
789{
790 struct net_device *dev;
791
792 rcu_read_lock();
793 dev = dev_get_by_name_rcu(net, name);
794 if (dev)
795 dev_hold(dev);
796 rcu_read_unlock();
797 return dev;
798}
799EXPORT_SYMBOL(dev_get_by_name);
800
801/**
802 * __dev_get_by_index - find a device by its ifindex
803 * @net: the applicable net namespace
804 * @ifindex: index of device
805 *
806 * Search for an interface by index. Returns %NULL if the device
807 * is not found or a pointer to the device. The device has not
808 * had its reference counter increased so the caller must be careful
809 * about locking. The caller must hold either the RTNL semaphore
810 * or @dev_base_lock.
811 */
812
813struct net_device *__dev_get_by_index(struct net *net, int ifindex)
814{
815 struct net_device *dev;
816 struct hlist_head *head = dev_index_hash(net, ifindex);
817
818 hlist_for_each_entry(dev, head, index_hlist)
819 if (dev->ifindex == ifindex)
820 return dev;
821
822 return NULL;
823}
824EXPORT_SYMBOL(__dev_get_by_index);
825
826/**
827 * dev_get_by_index_rcu - find a device by its ifindex
828 * @net: the applicable net namespace
829 * @ifindex: index of device
830 *
831 * Search for an interface by index. Returns %NULL if the device
832 * is not found or a pointer to the device. The device has not
833 * had its reference counter increased so the caller must be careful
834 * about locking. The caller must hold RCU lock.
835 */
836
837struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
838{
839 struct net_device *dev;
840 struct hlist_head *head = dev_index_hash(net, ifindex);
841
842 hlist_for_each_entry_rcu(dev, head, index_hlist)
843 if (dev->ifindex == ifindex)
844 return dev;
845
846 return NULL;
847}
848EXPORT_SYMBOL(dev_get_by_index_rcu);
849
850
851/**
852 * dev_get_by_index - find a device by its ifindex
853 * @net: the applicable net namespace
854 * @ifindex: index of device
855 *
856 * Search for an interface by index. Returns NULL if the device
857 * is not found or a pointer to the device. The device returned has
858 * had a reference added and the pointer is safe until the user calls
859 * dev_put to indicate they have finished with it.
860 */
861
862struct net_device *dev_get_by_index(struct net *net, int ifindex)
863{
864 struct net_device *dev;
865
866 rcu_read_lock();
867 dev = dev_get_by_index_rcu(net, ifindex);
868 if (dev)
869 dev_hold(dev);
870 rcu_read_unlock();
871 return dev;
872}
873EXPORT_SYMBOL(dev_get_by_index);
874
875/**
876 * dev_get_by_napi_id - find a device by napi_id
877 * @napi_id: ID of the NAPI struct
878 *
879 * Search for an interface by NAPI ID. Returns %NULL if the device
880 * is not found or a pointer to the device. The device has not had
881 * its reference counter increased so the caller must be careful
882 * about locking. The caller must hold RCU lock.
883 */
884
885struct net_device *dev_get_by_napi_id(unsigned int napi_id)
886{
887 struct napi_struct *napi;
888
889 WARN_ON_ONCE(!rcu_read_lock_held());
890
891 if (napi_id < MIN_NAPI_ID)
892 return NULL;
893
894 napi = napi_by_id(napi_id);
895
896 return napi ? napi->dev : NULL;
897}
898EXPORT_SYMBOL(dev_get_by_napi_id);
899
900/**
901 * netdev_get_name - get a netdevice name, knowing its ifindex.
902 * @net: network namespace
903 * @name: a pointer to the buffer where the name will be stored.
904 * @ifindex: the ifindex of the interface to get the name from.
905 *
906 * The use of raw_seqcount_begin() and cond_resched() before
907 * retrying is required as we want to give the writers a chance
908 * to complete when CONFIG_PREEMPT is not set.
909 */
910int netdev_get_name(struct net *net, char *name, int ifindex)
911{
912 struct net_device *dev;
913 unsigned int seq;
914
915retry:
916 seq = raw_seqcount_begin(&devnet_rename_seq);
917 rcu_read_lock();
918 dev = dev_get_by_index_rcu(net, ifindex);
919 if (!dev) {
920 rcu_read_unlock();
921 return -ENODEV;
922 }
923
924 strcpy(name, dev->name);
925 rcu_read_unlock();
926 if (read_seqcount_retry(&devnet_rename_seq, seq)) {
927 cond_resched();
928 goto retry;
929 }
930
931 return 0;
932}
933
934/**
935 * dev_getbyhwaddr_rcu - find a device by its hardware address
936 * @net: the applicable net namespace
937 * @type: media type of device
938 * @ha: hardware address
939 *
940 * Search for an interface by MAC address. Returns NULL if the device
941 * is not found or a pointer to the device.
942 * The caller must hold RCU or RTNL.
943 * The returned device has not had its ref count increased
944 * and the caller must therefore be careful about locking
945 *
946 */
947
948struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
949 const char *ha)
950{
951 struct net_device *dev;
952
953 for_each_netdev_rcu(net, dev)
954 if (dev->type == type &&
955 !memcmp(dev->dev_addr, ha, dev->addr_len))
956 return dev;
957
958 return NULL;
959}
960EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
961
962struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
963{
964 struct net_device *dev;
965
966 ASSERT_RTNL();
967 for_each_netdev(net, dev)
968 if (dev->type == type)
969 return dev;
970
971 return NULL;
972}
973EXPORT_SYMBOL(__dev_getfirstbyhwtype);
974
975struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
976{
977 struct net_device *dev, *ret = NULL;
978
979 rcu_read_lock();
980 for_each_netdev_rcu(net, dev)
981 if (dev->type == type) {
982 dev_hold(dev);
983 ret = dev;
984 break;
985 }
986 rcu_read_unlock();
987 return ret;
988}
989EXPORT_SYMBOL(dev_getfirstbyhwtype);
990
991/**
992 * __dev_get_by_flags - find any device with given flags
993 * @net: the applicable net namespace
994 * @if_flags: IFF_* values
995 * @mask: bitmask of bits in if_flags to check
996 *
997 * Search for any interface with the given flags. Returns NULL if a device
998 * is not found or a pointer to the device. Must be called inside
999 * rtnl_lock(), and result refcount is unchanged.
1000 */
1001
1002struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
1003 unsigned short mask)
1004{
1005 struct net_device *dev, *ret;
1006
1007 ASSERT_RTNL();
1008
1009 ret = NULL;
1010 for_each_netdev(net, dev) {
1011 if (((dev->flags ^ if_flags) & mask) == 0) {
1012 ret = dev;
1013 break;
1014 }
1015 }
1016 return ret;
1017}
1018EXPORT_SYMBOL(__dev_get_by_flags);
1019
1020/**
1021 * dev_valid_name - check if name is okay for network device
1022 * @name: name string
1023 *
1024 * Network device names need to be valid file names to
1025 * to allow sysfs to work. We also disallow any kind of
1026 * whitespace.
1027 */
1028bool dev_valid_name(const char *name)
1029{
1030 if (*name == '\0')
1031 return false;
1032 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1033 return false;
1034 if (!strcmp(name, ".") || !strcmp(name, ".."))
1035 return false;
1036
1037 while (*name) {
1038 if (*name == '/' || *name == ':' || isspace(*name))
1039 return false;
1040 name++;
1041 }
1042 return true;
1043}
1044EXPORT_SYMBOL(dev_valid_name);
1045
1046/**
1047 * __dev_alloc_name - allocate a name for a device
1048 * @net: network namespace to allocate the device name in
1049 * @name: name format string
1050 * @buf: scratch buffer and result name string
1051 *
1052 * Passed a format string - eg "lt%d" it will try and find a suitable
1053 * id. It scans list of devices to build up a free map, then chooses
1054 * the first empty slot. The caller must hold the dev_base or rtnl lock
1055 * while allocating the name and adding the device in order to avoid
1056 * duplicates.
1057 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1058 * Returns the number of the unit assigned or a negative errno code.
1059 */
1060
1061static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1062{
1063 int i = 0;
1064 const char *p;
1065 const int max_netdevices = 8*PAGE_SIZE;
1066 unsigned long *inuse;
1067 struct net_device *d;
1068
1069 if (!dev_valid_name(name))
1070 return -EINVAL;
1071
1072 p = strchr(name, '%');
1073 if (p) {
1074 /*
1075 * Verify the string as this thing may have come from
1076 * the user. There must be either one "%d" and no other "%"
1077 * characters.
1078 */
1079 if (p[1] != 'd' || strchr(p + 2, '%'))
1080 return -EINVAL;
1081
1082 /* Use one page as a bit array of possible slots */
1083 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1084 if (!inuse)
1085 return -ENOMEM;
1086
1087 for_each_netdev(net, d) {
1088 if (!sscanf(d->name, name, &i))
1089 continue;
1090 if (i < 0 || i >= max_netdevices)
1091 continue;
1092
1093 /* avoid cases where sscanf is not exact inverse of printf */
1094 snprintf(buf, IFNAMSIZ, name, i);
1095 if (!strncmp(buf, d->name, IFNAMSIZ))
1096 set_bit(i, inuse);
1097 }
1098
1099 i = find_first_zero_bit(inuse, max_netdevices);
1100 free_page((unsigned long) inuse);
1101 }
1102
1103 snprintf(buf, IFNAMSIZ, name, i);
1104 if (!__dev_get_by_name(net, buf))
1105 return i;
1106
1107 /* It is possible to run out of possible slots
1108 * when the name is long and there isn't enough space left
1109 * for the digits, or if all bits are used.
1110 */
1111 return -ENFILE;
1112}
1113
1114static int dev_alloc_name_ns(struct net *net,
1115 struct net_device *dev,
1116 const char *name)
1117{
1118 char buf[IFNAMSIZ];
1119 int ret;
1120
1121 BUG_ON(!net);
1122 ret = __dev_alloc_name(net, name, buf);
1123 if (ret >= 0)
1124 strlcpy(dev->name, buf, IFNAMSIZ);
1125 return ret;
1126}
1127
1128/**
1129 * dev_alloc_name - allocate a name for a device
1130 * @dev: device
1131 * @name: name format string
1132 *
1133 * Passed a format string - eg "lt%d" it will try and find a suitable
1134 * id. It scans list of devices to build up a free map, then chooses
1135 * the first empty slot. The caller must hold the dev_base or rtnl lock
1136 * while allocating the name and adding the device in order to avoid
1137 * duplicates.
1138 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1139 * Returns the number of the unit assigned or a negative errno code.
1140 */
1141
1142int dev_alloc_name(struct net_device *dev, const char *name)
1143{
1144 return dev_alloc_name_ns(dev_net(dev), dev, name);
1145}
1146EXPORT_SYMBOL(dev_alloc_name);
1147
1148int dev_get_valid_name(struct net *net, struct net_device *dev,
1149 const char *name)
1150{
1151 BUG_ON(!net);
1152
1153 if (!dev_valid_name(name))
1154 return -EINVAL;
1155
1156 if (strchr(name, '%'))
1157 return dev_alloc_name_ns(net, dev, name);
1158 else if (__dev_get_by_name(net, name))
1159 return -EEXIST;
1160 else if (dev->name != name)
1161 strlcpy(dev->name, name, IFNAMSIZ);
1162
1163 return 0;
1164}
1165EXPORT_SYMBOL(dev_get_valid_name);
1166
1167/**
1168 * dev_change_name - change name of a device
1169 * @dev: device
1170 * @newname: name (or format string) must be at least IFNAMSIZ
1171 *
1172 * Change name of a device, can pass format strings "eth%d".
1173 * for wildcarding.
1174 */
1175int dev_change_name(struct net_device *dev, const char *newname)
1176{
1177 unsigned char old_assign_type;
1178 char oldname[IFNAMSIZ];
1179 int err = 0;
1180 int ret;
1181 struct net *net;
1182
1183 ASSERT_RTNL();
1184 BUG_ON(!dev_net(dev));
1185
1186 net = dev_net(dev);
1187 if (dev->flags & IFF_UP)
1188 return -EBUSY;
1189
1190 write_seqcount_begin(&devnet_rename_seq);
1191
1192 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1193 write_seqcount_end(&devnet_rename_seq);
1194 return 0;
1195 }
1196
1197 memcpy(oldname, dev->name, IFNAMSIZ);
1198
1199 err = dev_get_valid_name(net, dev, newname);
1200 if (err < 0) {
1201 write_seqcount_end(&devnet_rename_seq);
1202 return err;
1203 }
1204
1205 if (oldname[0] && !strchr(oldname, '%'))
1206 netdev_info(dev, "renamed from %s\n", oldname);
1207
1208 old_assign_type = dev->name_assign_type;
1209 dev->name_assign_type = NET_NAME_RENAMED;
1210
1211rollback:
1212 ret = device_rename(&dev->dev, dev->name);
1213 if (ret) {
1214 memcpy(dev->name, oldname, IFNAMSIZ);
1215 dev->name_assign_type = old_assign_type;
1216 write_seqcount_end(&devnet_rename_seq);
1217 return ret;
1218 }
1219
1220 write_seqcount_end(&devnet_rename_seq);
1221
1222 netdev_adjacent_rename_links(dev, oldname);
1223
1224 write_lock_bh(&dev_base_lock);
1225 hlist_del_rcu(&dev->name_hlist);
1226 write_unlock_bh(&dev_base_lock);
1227
1228 synchronize_rcu();
1229
1230 write_lock_bh(&dev_base_lock);
1231 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1232 write_unlock_bh(&dev_base_lock);
1233
1234 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1235 ret = notifier_to_errno(ret);
1236
1237 if (ret) {
1238 /* err >= 0 after dev_alloc_name() or stores the first errno */
1239 if (err >= 0) {
1240 err = ret;
1241 write_seqcount_begin(&devnet_rename_seq);
1242 memcpy(dev->name, oldname, IFNAMSIZ);
1243 memcpy(oldname, newname, IFNAMSIZ);
1244 dev->name_assign_type = old_assign_type;
1245 old_assign_type = NET_NAME_RENAMED;
1246 goto rollback;
1247 } else {
1248 pr_err("%s: name change rollback failed: %d\n",
1249 dev->name, ret);
1250 }
1251 }
1252
1253 return err;
1254}
1255
1256/**
1257 * dev_set_alias - change ifalias of a device
1258 * @dev: device
1259 * @alias: name up to IFALIASZ
1260 * @len: limit of bytes to copy from info
1261 *
1262 * Set ifalias for a device,
1263 */
1264int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1265{
1266 struct dev_ifalias *new_alias = NULL;
1267
1268 if (len >= IFALIASZ)
1269 return -EINVAL;
1270
1271 if (len) {
1272 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1273 if (!new_alias)
1274 return -ENOMEM;
1275
1276 memcpy(new_alias->ifalias, alias, len);
1277 new_alias->ifalias[len] = 0;
1278 }
1279
1280 mutex_lock(&ifalias_mutex);
1281 rcu_swap_protected(dev->ifalias, new_alias,
1282 mutex_is_locked(&ifalias_mutex));
1283 mutex_unlock(&ifalias_mutex);
1284
1285 if (new_alias)
1286 kfree_rcu(new_alias, rcuhead);
1287
1288 return len;
1289}
1290EXPORT_SYMBOL(dev_set_alias);
1291
1292/**
1293 * dev_get_alias - get ifalias of a device
1294 * @dev: device
1295 * @name: buffer to store name of ifalias
1296 * @len: size of buffer
1297 *
1298 * get ifalias for a device. Caller must make sure dev cannot go
1299 * away, e.g. rcu read lock or own a reference count to device.
1300 */
1301int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1302{
1303 const struct dev_ifalias *alias;
1304 int ret = 0;
1305
1306 rcu_read_lock();
1307 alias = rcu_dereference(dev->ifalias);
1308 if (alias)
1309 ret = snprintf(name, len, "%s", alias->ifalias);
1310 rcu_read_unlock();
1311
1312 return ret;
1313}
1314
1315/**
1316 * netdev_features_change - device changes features
1317 * @dev: device to cause notification
1318 *
1319 * Called to indicate a device has changed features.
1320 */
1321void netdev_features_change(struct net_device *dev)
1322{
1323 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1324}
1325EXPORT_SYMBOL(netdev_features_change);
1326
1327/**
1328 * netdev_state_change - device changes state
1329 * @dev: device to cause notification
1330 *
1331 * Called to indicate a device has changed state. This function calls
1332 * the notifier chains for netdev_chain and sends a NEWLINK message
1333 * to the routing socket.
1334 */
1335void netdev_state_change(struct net_device *dev)
1336{
1337 if (dev->flags & IFF_UP) {
1338 struct netdev_notifier_change_info change_info = {
1339 .info.dev = dev,
1340 };
1341
1342 call_netdevice_notifiers_info(NETDEV_CHANGE,
1343 &change_info.info);
1344 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1345 }
1346}
1347EXPORT_SYMBOL(netdev_state_change);
1348
1349/**
1350 * netdev_notify_peers - notify network peers about existence of @dev
1351 * @dev: network device
1352 *
1353 * Generate traffic such that interested network peers are aware of
1354 * @dev, such as by generating a gratuitous ARP. This may be used when
1355 * a device wants to inform the rest of the network about some sort of
1356 * reconfiguration such as a failover event or virtual machine
1357 * migration.
1358 */
1359void netdev_notify_peers(struct net_device *dev)
1360{
1361 rtnl_lock();
1362 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1363 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1364 rtnl_unlock();
1365}
1366EXPORT_SYMBOL(netdev_notify_peers);
1367
1368static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1369{
1370 const struct net_device_ops *ops = dev->netdev_ops;
1371 int ret;
1372
1373 ASSERT_RTNL();
1374
1375 if (!netif_device_present(dev))
1376 return -ENODEV;
1377
1378 /* Block netpoll from trying to do any rx path servicing.
1379 * If we don't do this there is a chance ndo_poll_controller
1380 * or ndo_poll may be running while we open the device
1381 */
1382 netpoll_poll_disable(dev);
1383
1384 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1385 ret = notifier_to_errno(ret);
1386 if (ret)
1387 return ret;
1388
1389 set_bit(__LINK_STATE_START, &dev->state);
1390
1391 if (ops->ndo_validate_addr)
1392 ret = ops->ndo_validate_addr(dev);
1393
1394 if (!ret && ops->ndo_open)
1395 ret = ops->ndo_open(dev);
1396
1397 netpoll_poll_enable(dev);
1398
1399 if (ret)
1400 clear_bit(__LINK_STATE_START, &dev->state);
1401 else {
1402 dev->flags |= IFF_UP;
1403 dev_set_rx_mode(dev);
1404 dev_activate(dev);
1405 add_device_randomness(dev->dev_addr, dev->addr_len);
1406 }
1407
1408 return ret;
1409}
1410
1411/**
1412 * dev_open - prepare an interface for use.
1413 * @dev: device to open
1414 * @extack: netlink extended ack
1415 *
1416 * Takes a device from down to up state. The device's private open
1417 * function is invoked and then the multicast lists are loaded. Finally
1418 * the device is moved into the up state and a %NETDEV_UP message is
1419 * sent to the netdev notifier chain.
1420 *
1421 * Calling this function on an active interface is a nop. On a failure
1422 * a negative errno code is returned.
1423 */
1424int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1425{
1426 int ret;
1427
1428 if (dev->flags & IFF_UP)
1429 return 0;
1430
1431 ret = __dev_open(dev, extack);
1432 if (ret < 0)
1433 return ret;
1434
1435 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1436 call_netdevice_notifiers(NETDEV_UP, dev);
1437
1438 return ret;
1439}
1440EXPORT_SYMBOL(dev_open);
1441
1442static void __dev_close_many(struct list_head *head)
1443{
1444 struct net_device *dev;
1445
1446 ASSERT_RTNL();
1447 might_sleep();
1448
1449 list_for_each_entry(dev, head, close_list) {
1450 /* Temporarily disable netpoll until the interface is down */
1451 netpoll_poll_disable(dev);
1452
1453 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1454
1455 clear_bit(__LINK_STATE_START, &dev->state);
1456
1457 /* Synchronize to scheduled poll. We cannot touch poll list, it
1458 * can be even on different cpu. So just clear netif_running().
1459 *
1460 * dev->stop() will invoke napi_disable() on all of it's
1461 * napi_struct instances on this device.
1462 */
1463 smp_mb__after_atomic(); /* Commit netif_running(). */
1464 }
1465
1466 dev_deactivate_many(head);
1467
1468 list_for_each_entry(dev, head, close_list) {
1469 const struct net_device_ops *ops = dev->netdev_ops;
1470
1471 /*
1472 * Call the device specific close. This cannot fail.
1473 * Only if device is UP
1474 *
1475 * We allow it to be called even after a DETACH hot-plug
1476 * event.
1477 */
1478 if (ops->ndo_stop)
1479 ops->ndo_stop(dev);
1480
1481 dev->flags &= ~IFF_UP;
1482 netpoll_poll_enable(dev);
1483 }
1484}
1485
1486static void __dev_close(struct net_device *dev)
1487{
1488 LIST_HEAD(single);
1489
1490 list_add(&dev->close_list, &single);
1491 __dev_close_many(&single);
1492 list_del(&single);
1493}
1494
1495void dev_close_many(struct list_head *head, bool unlink)
1496{
1497 struct net_device *dev, *tmp;
1498
1499 /* Remove the devices that don't need to be closed */
1500 list_for_each_entry_safe(dev, tmp, head, close_list)
1501 if (!(dev->flags & IFF_UP))
1502 list_del_init(&dev->close_list);
1503
1504 __dev_close_many(head);
1505
1506 list_for_each_entry_safe(dev, tmp, head, close_list) {
1507 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1508 call_netdevice_notifiers(NETDEV_DOWN, dev);
1509 if (unlink)
1510 list_del_init(&dev->close_list);
1511 }
1512}
1513EXPORT_SYMBOL(dev_close_many);
1514
1515/**
1516 * dev_close - shutdown an interface.
1517 * @dev: device to shutdown
1518 *
1519 * This function moves an active device into down state. A
1520 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1521 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1522 * chain.
1523 */
1524void dev_close(struct net_device *dev)
1525{
1526 if (dev->flags & IFF_UP) {
1527 LIST_HEAD(single);
1528
1529 list_add(&dev->close_list, &single);
1530 dev_close_many(&single, true);
1531 list_del(&single);
1532 }
1533}
1534EXPORT_SYMBOL(dev_close);
1535
1536
1537/**
1538 * dev_disable_lro - disable Large Receive Offload on a device
1539 * @dev: device
1540 *
1541 * Disable Large Receive Offload (LRO) on a net device. Must be
1542 * called under RTNL. This is needed if received packets may be
1543 * forwarded to another interface.
1544 */
1545void dev_disable_lro(struct net_device *dev)
1546{
1547 struct net_device *lower_dev;
1548 struct list_head *iter;
1549
1550 dev->wanted_features &= ~NETIF_F_LRO;
1551 netdev_update_features(dev);
1552
1553 if (unlikely(dev->features & NETIF_F_LRO))
1554 netdev_WARN(dev, "failed to disable LRO!\n");
1555
1556 netdev_for_each_lower_dev(dev, lower_dev, iter)
1557 dev_disable_lro(lower_dev);
1558}
1559EXPORT_SYMBOL(dev_disable_lro);
1560
1561/**
1562 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1563 * @dev: device
1564 *
1565 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
1566 * called under RTNL. This is needed if Generic XDP is installed on
1567 * the device.
1568 */
1569static void dev_disable_gro_hw(struct net_device *dev)
1570{
1571 dev->wanted_features &= ~NETIF_F_GRO_HW;
1572 netdev_update_features(dev);
1573
1574 if (unlikely(dev->features & NETIF_F_GRO_HW))
1575 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1576}
1577
1578const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1579{
1580#define N(val) \
1581 case NETDEV_##val: \
1582 return "NETDEV_" __stringify(val);
1583 switch (cmd) {
1584 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1585 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1586 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1587 N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1588 N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1589 N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1590 N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1591 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1592 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1593 N(PRE_CHANGEADDR)
1594 }
1595#undef N
1596 return "UNKNOWN_NETDEV_EVENT";
1597}
1598EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1599
1600static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1601 struct net_device *dev)
1602{
1603 struct netdev_notifier_info info = {
1604 .dev = dev,
1605 };
1606
1607 return nb->notifier_call(nb, val, &info);
1608}
1609
1610static int dev_boot_phase = 1;
1611
1612/**
1613 * register_netdevice_notifier - register a network notifier block
1614 * @nb: notifier
1615 *
1616 * Register a notifier to be called when network device events occur.
1617 * The notifier passed is linked into the kernel structures and must
1618 * not be reused until it has been unregistered. A negative errno code
1619 * is returned on a failure.
1620 *
1621 * When registered all registration and up events are replayed
1622 * to the new notifier to allow device to have a race free
1623 * view of the network device list.
1624 */
1625
1626int register_netdevice_notifier(struct notifier_block *nb)
1627{
1628 struct net_device *dev;
1629 struct net_device *last;
1630 struct net *net;
1631 int err;
1632
1633 /* Close race with setup_net() and cleanup_net() */
1634 down_write(&pernet_ops_rwsem);
1635 rtnl_lock();
1636 err = raw_notifier_chain_register(&netdev_chain, nb);
1637 if (err)
1638 goto unlock;
1639 if (dev_boot_phase)
1640 goto unlock;
1641 for_each_net(net) {
1642 for_each_netdev(net, dev) {
1643 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1644 err = notifier_to_errno(err);
1645 if (err)
1646 goto rollback;
1647
1648 if (!(dev->flags & IFF_UP))
1649 continue;
1650
1651 call_netdevice_notifier(nb, NETDEV_UP, dev);
1652 }
1653 }
1654
1655unlock:
1656 rtnl_unlock();
1657 up_write(&pernet_ops_rwsem);
1658 return err;
1659
1660rollback:
1661 last = dev;
1662 for_each_net(net) {
1663 for_each_netdev(net, dev) {
1664 if (dev == last)
1665 goto outroll;
1666
1667 if (dev->flags & IFF_UP) {
1668 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1669 dev);
1670 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1671 }
1672 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1673 }
1674 }
1675
1676outroll:
1677 raw_notifier_chain_unregister(&netdev_chain, nb);
1678 goto unlock;
1679}
1680EXPORT_SYMBOL(register_netdevice_notifier);
1681
1682/**
1683 * unregister_netdevice_notifier - unregister a network notifier block
1684 * @nb: notifier
1685 *
1686 * Unregister a notifier previously registered by
1687 * register_netdevice_notifier(). The notifier is unlinked into the
1688 * kernel structures and may then be reused. A negative errno code
1689 * is returned on a failure.
1690 *
1691 * After unregistering unregister and down device events are synthesized
1692 * for all devices on the device list to the removed notifier to remove
1693 * the need for special case cleanup code.
1694 */
1695
1696int unregister_netdevice_notifier(struct notifier_block *nb)
1697{
1698 struct net_device *dev;
1699 struct net *net;
1700 int err;
1701
1702 /* Close race with setup_net() and cleanup_net() */
1703 down_write(&pernet_ops_rwsem);
1704 rtnl_lock();
1705 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1706 if (err)
1707 goto unlock;
1708
1709 for_each_net(net) {
1710 for_each_netdev(net, dev) {
1711 if (dev->flags & IFF_UP) {
1712 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1713 dev);
1714 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1715 }
1716 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1717 }
1718 }
1719unlock:
1720 rtnl_unlock();
1721 up_write(&pernet_ops_rwsem);
1722 return err;
1723}
1724EXPORT_SYMBOL(unregister_netdevice_notifier);
1725
1726/**
1727 * call_netdevice_notifiers_info - call all network notifier blocks
1728 * @val: value passed unmodified to notifier function
1729 * @info: notifier information data
1730 *
1731 * Call all network notifier blocks. Parameters and return value
1732 * are as for raw_notifier_call_chain().
1733 */
1734
1735static int call_netdevice_notifiers_info(unsigned long val,
1736 struct netdev_notifier_info *info)
1737{
1738 ASSERT_RTNL();
1739 return raw_notifier_call_chain(&netdev_chain, val, info);
1740}
1741
1742static int call_netdevice_notifiers_extack(unsigned long val,
1743 struct net_device *dev,
1744 struct netlink_ext_ack *extack)
1745{
1746 struct netdev_notifier_info info = {
1747 .dev = dev,
1748 .extack = extack,
1749 };
1750
1751 return call_netdevice_notifiers_info(val, &info);
1752}
1753
1754/**
1755 * call_netdevice_notifiers - call all network notifier blocks
1756 * @val: value passed unmodified to notifier function
1757 * @dev: net_device pointer passed unmodified to notifier function
1758 *
1759 * Call all network notifier blocks. Parameters and return value
1760 * are as for raw_notifier_call_chain().
1761 */
1762
1763int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1764{
1765 return call_netdevice_notifiers_extack(val, dev, NULL);
1766}
1767EXPORT_SYMBOL(call_netdevice_notifiers);
1768
1769/**
1770 * call_netdevice_notifiers_mtu - call all network notifier blocks
1771 * @val: value passed unmodified to notifier function
1772 * @dev: net_device pointer passed unmodified to notifier function
1773 * @arg: additional u32 argument passed to the notifier function
1774 *
1775 * Call all network notifier blocks. Parameters and return value
1776 * are as for raw_notifier_call_chain().
1777 */
1778static int call_netdevice_notifiers_mtu(unsigned long val,
1779 struct net_device *dev, u32 arg)
1780{
1781 struct netdev_notifier_info_ext info = {
1782 .info.dev = dev,
1783 .ext.mtu = arg,
1784 };
1785
1786 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
1787
1788 return call_netdevice_notifiers_info(val, &info.info);
1789}
1790
1791#ifdef CONFIG_NET_INGRESS
1792static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
1793
1794void net_inc_ingress_queue(void)
1795{
1796 static_branch_inc(&ingress_needed_key);
1797}
1798EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1799
1800void net_dec_ingress_queue(void)
1801{
1802 static_branch_dec(&ingress_needed_key);
1803}
1804EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1805#endif
1806
1807#ifdef CONFIG_NET_EGRESS
1808static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
1809
1810void net_inc_egress_queue(void)
1811{
1812 static_branch_inc(&egress_needed_key);
1813}
1814EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1815
1816void net_dec_egress_queue(void)
1817{
1818 static_branch_dec(&egress_needed_key);
1819}
1820EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1821#endif
1822
1823static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
1824#ifdef CONFIG_JUMP_LABEL
1825static atomic_t netstamp_needed_deferred;
1826static atomic_t netstamp_wanted;
1827static void netstamp_clear(struct work_struct *work)
1828{
1829 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1830 int wanted;
1831
1832 wanted = atomic_add_return(deferred, &netstamp_wanted);
1833 if (wanted > 0)
1834 static_branch_enable(&netstamp_needed_key);
1835 else
1836 static_branch_disable(&netstamp_needed_key);
1837}
1838static DECLARE_WORK(netstamp_work, netstamp_clear);
1839#endif
1840
1841void net_enable_timestamp(void)
1842{
1843#ifdef CONFIG_JUMP_LABEL
1844 int wanted;
1845
1846 while (1) {
1847 wanted = atomic_read(&netstamp_wanted);
1848 if (wanted <= 0)
1849 break;
1850 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1851 return;
1852 }
1853 atomic_inc(&netstamp_needed_deferred);
1854 schedule_work(&netstamp_work);
1855#else
1856 static_branch_inc(&netstamp_needed_key);
1857#endif
1858}
1859EXPORT_SYMBOL(net_enable_timestamp);
1860
1861void net_disable_timestamp(void)
1862{
1863#ifdef CONFIG_JUMP_LABEL
1864 int wanted;
1865
1866 while (1) {
1867 wanted = atomic_read(&netstamp_wanted);
1868 if (wanted <= 1)
1869 break;
1870 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1871 return;
1872 }
1873 atomic_dec(&netstamp_needed_deferred);
1874 schedule_work(&netstamp_work);
1875#else
1876 static_branch_dec(&netstamp_needed_key);
1877#endif
1878}
1879EXPORT_SYMBOL(net_disable_timestamp);
1880
1881static inline void net_timestamp_set(struct sk_buff *skb)
1882{
1883 skb->tstamp = 0;
1884 if (static_branch_unlikely(&netstamp_needed_key))
1885 __net_timestamp(skb);
1886}
1887
1888#define net_timestamp_check(COND, SKB) \
1889 if (static_branch_unlikely(&netstamp_needed_key)) { \
1890 if ((COND) && !(SKB)->tstamp) \
1891 __net_timestamp(SKB); \
1892 } \
1893
1894bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
1895{
1896 unsigned int len;
1897
1898 if (!(dev->flags & IFF_UP))
1899 return false;
1900
1901 len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1902 if (skb->len <= len)
1903 return true;
1904
1905 /* if TSO is enabled, we don't care about the length as the packet
1906 * could be forwarded without being segmented before
1907 */
1908 if (skb_is_gso(skb))
1909 return true;
1910
1911 return false;
1912}
1913EXPORT_SYMBOL_GPL(is_skb_forwardable);
1914
1915int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1916{
1917 int ret = ____dev_forward_skb(dev, skb);
1918
1919 if (likely(!ret)) {
1920 skb->protocol = eth_type_trans(skb, dev);
1921 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1922 }
1923
1924 return ret;
1925}
1926EXPORT_SYMBOL_GPL(__dev_forward_skb);
1927
1928/**
1929 * dev_forward_skb - loopback an skb to another netif
1930 *
1931 * @dev: destination network device
1932 * @skb: buffer to forward
1933 *
1934 * return values:
1935 * NET_RX_SUCCESS (no congestion)
1936 * NET_RX_DROP (packet was dropped, but freed)
1937 *
1938 * dev_forward_skb can be used for injecting an skb from the
1939 * start_xmit function of one device into the receive queue
1940 * of another device.
1941 *
1942 * The receiving device may be in another namespace, so
1943 * we have to clear all information in the skb that could
1944 * impact namespace isolation.
1945 */
1946int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1947{
1948 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1949}
1950EXPORT_SYMBOL_GPL(dev_forward_skb);
1951
1952static inline int deliver_skb(struct sk_buff *skb,
1953 struct packet_type *pt_prev,
1954 struct net_device *orig_dev)
1955{
1956 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
1957 return -ENOMEM;
1958 refcount_inc(&skb->users);
1959 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1960}
1961
1962static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1963 struct packet_type **pt,
1964 struct net_device *orig_dev,
1965 __be16 type,
1966 struct list_head *ptype_list)
1967{
1968 struct packet_type *ptype, *pt_prev = *pt;
1969
1970 list_for_each_entry_rcu(ptype, ptype_list, list) {
1971 if (ptype->type != type)
1972 continue;
1973 if (pt_prev)
1974 deliver_skb(skb, pt_prev, orig_dev);
1975 pt_prev = ptype;
1976 }
1977 *pt = pt_prev;
1978}
1979
1980static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1981{
1982 if (!ptype->af_packet_priv || !skb->sk)
1983 return false;
1984
1985 if (ptype->id_match)
1986 return ptype->id_match(ptype, skb->sk);
1987 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1988 return true;
1989
1990 return false;
1991}
1992
1993/**
1994 * dev_nit_active - return true if any network interface taps are in use
1995 *
1996 * @dev: network device to check for the presence of taps
1997 */
1998bool dev_nit_active(struct net_device *dev)
1999{
2000 return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2001}
2002EXPORT_SYMBOL_GPL(dev_nit_active);
2003
2004/*
2005 * Support routine. Sends outgoing frames to any network
2006 * taps currently in use.
2007 */
2008
2009void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2010{
2011 struct packet_type *ptype;
2012 struct sk_buff *skb2 = NULL;
2013 struct packet_type *pt_prev = NULL;
2014 struct list_head *ptype_list = &ptype_all;
2015
2016 rcu_read_lock();
2017again:
2018 list_for_each_entry_rcu(ptype, ptype_list, list) {
2019 if (ptype->ignore_outgoing)
2020 continue;
2021
2022 /* Never send packets back to the socket
2023 * they originated from - MvS (miquels@drinkel.ow.org)
2024 */
2025 if (skb_loop_sk(ptype, skb))
2026 continue;
2027
2028 if (pt_prev) {
2029 deliver_skb(skb2, pt_prev, skb->dev);
2030 pt_prev = ptype;
2031 continue;
2032 }
2033
2034 /* need to clone skb, done only once */
2035 skb2 = skb_clone(skb, GFP_ATOMIC);
2036 if (!skb2)
2037 goto out_unlock;
2038
2039 net_timestamp_set(skb2);
2040
2041 /* skb->nh should be correctly
2042 * set by sender, so that the second statement is
2043 * just protection against buggy protocols.
2044 */
2045 skb_reset_mac_header(skb2);
2046
2047 if (skb_network_header(skb2) < skb2->data ||
2048 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2049 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2050 ntohs(skb2->protocol),
2051 dev->name);
2052 skb_reset_network_header(skb2);
2053 }
2054
2055 skb2->transport_header = skb2->network_header;
2056 skb2->pkt_type = PACKET_OUTGOING;
2057 pt_prev = ptype;
2058 }
2059
2060 if (ptype_list == &ptype_all) {
2061 ptype_list = &dev->ptype_all;
2062 goto again;
2063 }
2064out_unlock:
2065 if (pt_prev) {
2066 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2067 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2068 else
2069 kfree_skb(skb2);
2070 }
2071 rcu_read_unlock();
2072}
2073EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2074
2075/**
2076 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2077 * @dev: Network device
2078 * @txq: number of queues available
2079 *
2080 * If real_num_tx_queues is changed the tc mappings may no longer be
2081 * valid. To resolve this verify the tc mapping remains valid and if
2082 * not NULL the mapping. With no priorities mapping to this
2083 * offset/count pair it will no longer be used. In the worst case TC0
2084 * is invalid nothing can be done so disable priority mappings. If is
2085 * expected that drivers will fix this mapping if they can before
2086 * calling netif_set_real_num_tx_queues.
2087 */
2088static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2089{
2090 int i;
2091 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2092
2093 /* If TC0 is invalidated disable TC mapping */
2094 if (tc->offset + tc->count > txq) {
2095 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2096 dev->num_tc = 0;
2097 return;
2098 }
2099
2100 /* Invalidated prio to tc mappings set to TC0 */
2101 for (i = 1; i < TC_BITMASK + 1; i++) {
2102 int q = netdev_get_prio_tc_map(dev, i);
2103
2104 tc = &dev->tc_to_txq[q];
2105 if (tc->offset + tc->count > txq) {
2106 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2107 i, q);
2108 netdev_set_prio_tc_map(dev, i, 0);
2109 }
2110 }
2111}
2112
2113int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2114{
2115 if (dev->num_tc) {
2116 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2117 int i;
2118
2119 /* walk through the TCs and see if it falls into any of them */
2120 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2121 if ((txq - tc->offset) < tc->count)
2122 return i;
2123 }
2124
2125 /* didn't find it, just return -1 to indicate no match */
2126 return -1;
2127 }
2128
2129 return 0;
2130}
2131EXPORT_SYMBOL(netdev_txq_to_tc);
2132
2133#ifdef CONFIG_XPS
2134struct static_key xps_needed __read_mostly;
2135EXPORT_SYMBOL(xps_needed);
2136struct static_key xps_rxqs_needed __read_mostly;
2137EXPORT_SYMBOL(xps_rxqs_needed);
2138static DEFINE_MUTEX(xps_map_mutex);
2139#define xmap_dereference(P) \
2140 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2141
2142static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2143 int tci, u16 index)
2144{
2145 struct xps_map *map = NULL;
2146 int pos;
2147
2148 if (dev_maps)
2149 map = xmap_dereference(dev_maps->attr_map[tci]);
2150 if (!map)
2151 return false;
2152
2153 for (pos = map->len; pos--;) {
2154 if (map->queues[pos] != index)
2155 continue;
2156
2157 if (map->len > 1) {
2158 map->queues[pos] = map->queues[--map->len];
2159 break;
2160 }
2161
2162 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2163 kfree_rcu(map, rcu);
2164 return false;
2165 }
2166
2167 return true;
2168}
2169
2170static bool remove_xps_queue_cpu(struct net_device *dev,
2171 struct xps_dev_maps *dev_maps,
2172 int cpu, u16 offset, u16 count)
2173{
2174 int num_tc = dev->num_tc ? : 1;
2175 bool active = false;
2176 int tci;
2177
2178 for (tci = cpu * num_tc; num_tc--; tci++) {
2179 int i, j;
2180
2181 for (i = count, j = offset; i--; j++) {
2182 if (!remove_xps_queue(dev_maps, tci, j))
2183 break;
2184 }
2185
2186 active |= i < 0;
2187 }
2188
2189 return active;
2190}
2191
2192static void reset_xps_maps(struct net_device *dev,
2193 struct xps_dev_maps *dev_maps,
2194 bool is_rxqs_map)
2195{
2196 if (is_rxqs_map) {
2197 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2198 RCU_INIT_POINTER(dev->xps_rxqs_map, NULL);
2199 } else {
2200 RCU_INIT_POINTER(dev->xps_cpus_map, NULL);
2201 }
2202 static_key_slow_dec_cpuslocked(&xps_needed);
2203 kfree_rcu(dev_maps, rcu);
2204}
2205
2206static void clean_xps_maps(struct net_device *dev, const unsigned long *mask,
2207 struct xps_dev_maps *dev_maps, unsigned int nr_ids,
2208 u16 offset, u16 count, bool is_rxqs_map)
2209{
2210 bool active = false;
2211 int i, j;
2212
2213 for (j = -1; j = netif_attrmask_next(j, mask, nr_ids),
2214 j < nr_ids;)
2215 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset,
2216 count);
2217 if (!active)
2218 reset_xps_maps(dev, dev_maps, is_rxqs_map);
2219
2220 if (!is_rxqs_map) {
2221 for (i = offset + (count - 1); count--; i--) {
2222 netdev_queue_numa_node_write(
2223 netdev_get_tx_queue(dev, i),
2224 NUMA_NO_NODE);
2225 }
2226 }
2227}
2228
2229static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2230 u16 count)
2231{
2232 const unsigned long *possible_mask = NULL;
2233 struct xps_dev_maps *dev_maps;
2234 unsigned int nr_ids;
2235
2236 if (!static_key_false(&xps_needed))
2237 return;
2238
2239 cpus_read_lock();
2240 mutex_lock(&xps_map_mutex);
2241
2242 if (static_key_false(&xps_rxqs_needed)) {
2243 dev_maps = xmap_dereference(dev->xps_rxqs_map);
2244 if (dev_maps) {
2245 nr_ids = dev->num_rx_queues;
2246 clean_xps_maps(dev, possible_mask, dev_maps, nr_ids,
2247 offset, count, true);
2248 }
2249 }
2250
2251 dev_maps = xmap_dereference(dev->xps_cpus_map);
2252 if (!dev_maps)
2253 goto out_no_maps;
2254
2255 if (num_possible_cpus() > 1)
2256 possible_mask = cpumask_bits(cpu_possible_mask);
2257 nr_ids = nr_cpu_ids;
2258 clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, offset, count,
2259 false);
2260
2261out_no_maps:
2262 mutex_unlock(&xps_map_mutex);
2263 cpus_read_unlock();
2264}
2265
2266static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2267{
2268 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2269}
2270
2271static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2272 u16 index, bool is_rxqs_map)
2273{
2274 struct xps_map *new_map;
2275 int alloc_len = XPS_MIN_MAP_ALLOC;
2276 int i, pos;
2277
2278 for (pos = 0; map && pos < map->len; pos++) {
2279 if (map->queues[pos] != index)
2280 continue;
2281 return map;
2282 }
2283
2284 /* Need to add tx-queue to this CPU's/rx-queue's existing map */
2285 if (map) {
2286 if (pos < map->alloc_len)
2287 return map;
2288
2289 alloc_len = map->alloc_len * 2;
2290 }
2291
2292 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2293 * map
2294 */
2295 if (is_rxqs_map)
2296 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2297 else
2298 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2299 cpu_to_node(attr_index));
2300 if (!new_map)
2301 return NULL;
2302
2303 for (i = 0; i < pos; i++)
2304 new_map->queues[i] = map->queues[i];
2305 new_map->alloc_len = alloc_len;
2306 new_map->len = pos;
2307
2308 return new_map;
2309}
2310
2311/* Must be called under cpus_read_lock */
2312int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2313 u16 index, bool is_rxqs_map)
2314{
2315 const unsigned long *online_mask = NULL, *possible_mask = NULL;
2316 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2317 int i, j, tci, numa_node_id = -2;
2318 int maps_sz, num_tc = 1, tc = 0;
2319 struct xps_map *map, *new_map;
2320 bool active = false;
2321 unsigned int nr_ids;
2322
2323 if (dev->num_tc) {
2324 /* Do not allow XPS on subordinate device directly */
2325 num_tc = dev->num_tc;
2326 if (num_tc < 0)
2327 return -EINVAL;
2328
2329 /* If queue belongs to subordinate dev use its map */
2330 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2331
2332 tc = netdev_txq_to_tc(dev, index);
2333 if (tc < 0)
2334 return -EINVAL;
2335 }
2336
2337 mutex_lock(&xps_map_mutex);
2338 if (is_rxqs_map) {
2339 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2340 dev_maps = xmap_dereference(dev->xps_rxqs_map);
2341 nr_ids = dev->num_rx_queues;
2342 } else {
2343 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2344 if (num_possible_cpus() > 1) {
2345 online_mask = cpumask_bits(cpu_online_mask);
2346 possible_mask = cpumask_bits(cpu_possible_mask);
2347 }
2348 dev_maps = xmap_dereference(dev->xps_cpus_map);
2349 nr_ids = nr_cpu_ids;
2350 }
2351
2352 if (maps_sz < L1_CACHE_BYTES)
2353 maps_sz = L1_CACHE_BYTES;
2354
2355 /* allocate memory for queue storage */
2356 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2357 j < nr_ids;) {
2358 if (!new_dev_maps)
2359 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2360 if (!new_dev_maps) {
2361 mutex_unlock(&xps_map_mutex);
2362 return -ENOMEM;
2363 }
2364
2365 tci = j * num_tc + tc;
2366 map = dev_maps ? xmap_dereference(dev_maps->attr_map[tci]) :
2367 NULL;
2368
2369 map = expand_xps_map(map, j, index, is_rxqs_map);
2370 if (!map)
2371 goto error;
2372
2373 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2374 }
2375
2376 if (!new_dev_maps)
2377 goto out_no_new_maps;
2378
2379 if (!dev_maps) {
2380 /* Increment static keys at most once per type */
2381 static_key_slow_inc_cpuslocked(&xps_needed);
2382 if (is_rxqs_map)
2383 static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2384 }
2385
2386 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2387 j < nr_ids;) {
2388 /* copy maps belonging to foreign traffic classes */
2389 for (i = tc, tci = j * num_tc; dev_maps && i--; tci++) {
2390 /* fill in the new device map from the old device map */
2391 map = xmap_dereference(dev_maps->attr_map[tci]);
2392 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2393 }
2394
2395 /* We need to explicitly update tci as prevous loop
2396 * could break out early if dev_maps is NULL.
2397 */
2398 tci = j * num_tc + tc;
2399
2400 if (netif_attr_test_mask(j, mask, nr_ids) &&
2401 netif_attr_test_online(j, online_mask, nr_ids)) {
2402 /* add tx-queue to CPU/rx-queue maps */
2403 int pos = 0;
2404
2405 map = xmap_dereference(new_dev_maps->attr_map[tci]);
2406 while ((pos < map->len) && (map->queues[pos] != index))
2407 pos++;
2408
2409 if (pos == map->len)
2410 map->queues[map->len++] = index;
2411#ifdef CONFIG_NUMA
2412 if (!is_rxqs_map) {
2413 if (numa_node_id == -2)
2414 numa_node_id = cpu_to_node(j);
2415 else if (numa_node_id != cpu_to_node(j))
2416 numa_node_id = -1;
2417 }
2418#endif
2419 } else if (dev_maps) {
2420 /* fill in the new device map from the old device map */
2421 map = xmap_dereference(dev_maps->attr_map[tci]);
2422 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2423 }
2424
2425 /* copy maps belonging to foreign traffic classes */
2426 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2427 /* fill in the new device map from the old device map */
2428 map = xmap_dereference(dev_maps->attr_map[tci]);
2429 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2430 }
2431 }
2432
2433 if (is_rxqs_map)
2434 rcu_assign_pointer(dev->xps_rxqs_map, new_dev_maps);
2435 else
2436 rcu_assign_pointer(dev->xps_cpus_map, new_dev_maps);
2437
2438 /* Cleanup old maps */
2439 if (!dev_maps)
2440 goto out_no_old_maps;
2441
2442 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2443 j < nr_ids;) {
2444 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2445 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2446 map = xmap_dereference(dev_maps->attr_map[tci]);
2447 if (map && map != new_map)
2448 kfree_rcu(map, rcu);
2449 }
2450 }
2451
2452 kfree_rcu(dev_maps, rcu);
2453
2454out_no_old_maps:
2455 dev_maps = new_dev_maps;
2456 active = true;
2457
2458out_no_new_maps:
2459 if (!is_rxqs_map) {
2460 /* update Tx queue numa node */
2461 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2462 (numa_node_id >= 0) ?
2463 numa_node_id : NUMA_NO_NODE);
2464 }
2465
2466 if (!dev_maps)
2467 goto out_no_maps;
2468
2469 /* removes tx-queue from unused CPUs/rx-queues */
2470 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2471 j < nr_ids;) {
2472 for (i = tc, tci = j * num_tc; i--; tci++)
2473 active |= remove_xps_queue(dev_maps, tci, index);
2474 if (!netif_attr_test_mask(j, mask, nr_ids) ||
2475 !netif_attr_test_online(j, online_mask, nr_ids))
2476 active |= remove_xps_queue(dev_maps, tci, index);
2477 for (i = num_tc - tc, tci++; --i; tci++)
2478 active |= remove_xps_queue(dev_maps, tci, index);
2479 }
2480
2481 /* free map if not active */
2482 if (!active)
2483 reset_xps_maps(dev, dev_maps, is_rxqs_map);
2484
2485out_no_maps:
2486 mutex_unlock(&xps_map_mutex);
2487
2488 return 0;
2489error:
2490 /* remove any maps that we added */
2491 for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2492 j < nr_ids;) {
2493 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2494 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2495 map = dev_maps ?
2496 xmap_dereference(dev_maps->attr_map[tci]) :
2497 NULL;
2498 if (new_map && new_map != map)
2499 kfree(new_map);
2500 }
2501 }
2502
2503 mutex_unlock(&xps_map_mutex);
2504
2505 kfree(new_dev_maps);
2506 return -ENOMEM;
2507}
2508EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2509
2510int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2511 u16 index)
2512{
2513 int ret;
2514
2515 cpus_read_lock();
2516 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, false);
2517 cpus_read_unlock();
2518
2519 return ret;
2520}
2521EXPORT_SYMBOL(netif_set_xps_queue);
2522
2523#endif
2524static void netdev_unbind_all_sb_channels(struct net_device *dev)
2525{
2526 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2527
2528 /* Unbind any subordinate channels */
2529 while (txq-- != &dev->_tx[0]) {
2530 if (txq->sb_dev)
2531 netdev_unbind_sb_channel(dev, txq->sb_dev);
2532 }
2533}
2534
2535void netdev_reset_tc(struct net_device *dev)
2536{
2537#ifdef CONFIG_XPS
2538 netif_reset_xps_queues_gt(dev, 0);
2539#endif
2540 netdev_unbind_all_sb_channels(dev);
2541
2542 /* Reset TC configuration of device */
2543 dev->num_tc = 0;
2544 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2545 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2546}
2547EXPORT_SYMBOL(netdev_reset_tc);
2548
2549int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2550{
2551 if (tc >= dev->num_tc)
2552 return -EINVAL;
2553
2554#ifdef CONFIG_XPS
2555 netif_reset_xps_queues(dev, offset, count);
2556#endif
2557 dev->tc_to_txq[tc].count = count;
2558 dev->tc_to_txq[tc].offset = offset;
2559 return 0;
2560}
2561EXPORT_SYMBOL(netdev_set_tc_queue);
2562
2563int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2564{
2565 if (num_tc > TC_MAX_QUEUE)
2566 return -EINVAL;
2567
2568#ifdef CONFIG_XPS
2569 netif_reset_xps_queues_gt(dev, 0);
2570#endif
2571 netdev_unbind_all_sb_channels(dev);
2572
2573 dev->num_tc = num_tc;
2574 return 0;
2575}
2576EXPORT_SYMBOL(netdev_set_num_tc);
2577
2578void netdev_unbind_sb_channel(struct net_device *dev,
2579 struct net_device *sb_dev)
2580{
2581 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2582
2583#ifdef CONFIG_XPS
2584 netif_reset_xps_queues_gt(sb_dev, 0);
2585#endif
2586 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2587 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2588
2589 while (txq-- != &dev->_tx[0]) {
2590 if (txq->sb_dev == sb_dev)
2591 txq->sb_dev = NULL;
2592 }
2593}
2594EXPORT_SYMBOL(netdev_unbind_sb_channel);
2595
2596int netdev_bind_sb_channel_queue(struct net_device *dev,
2597 struct net_device *sb_dev,
2598 u8 tc, u16 count, u16 offset)
2599{
2600 /* Make certain the sb_dev and dev are already configured */
2601 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2602 return -EINVAL;
2603
2604 /* We cannot hand out queues we don't have */
2605 if ((offset + count) > dev->real_num_tx_queues)
2606 return -EINVAL;
2607
2608 /* Record the mapping */
2609 sb_dev->tc_to_txq[tc].count = count;
2610 sb_dev->tc_to_txq[tc].offset = offset;
2611
2612 /* Provide a way for Tx queue to find the tc_to_txq map or
2613 * XPS map for itself.
2614 */
2615 while (count--)
2616 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2617
2618 return 0;
2619}
2620EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2621
2622int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2623{
2624 /* Do not use a multiqueue device to represent a subordinate channel */
2625 if (netif_is_multiqueue(dev))
2626 return -ENODEV;
2627
2628 /* We allow channels 1 - 32767 to be used for subordinate channels.
2629 * Channel 0 is meant to be "native" mode and used only to represent
2630 * the main root device. We allow writing 0 to reset the device back
2631 * to normal mode after being used as a subordinate channel.
2632 */
2633 if (channel > S16_MAX)
2634 return -EINVAL;
2635
2636 dev->num_tc = -channel;
2637
2638 return 0;
2639}
2640EXPORT_SYMBOL(netdev_set_sb_channel);
2641
2642/*
2643 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2644 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2645 */
2646int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2647{
2648 bool disabling;
2649 int rc;
2650
2651 disabling = txq < dev->real_num_tx_queues;
2652
2653 if (txq < 1 || txq > dev->num_tx_queues)
2654 return -EINVAL;
2655
2656 if (dev->reg_state == NETREG_REGISTERED ||
2657 dev->reg_state == NETREG_UNREGISTERING) {
2658 ASSERT_RTNL();
2659
2660 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2661 txq);
2662 if (rc)
2663 return rc;
2664
2665 if (dev->num_tc)
2666 netif_setup_tc(dev, txq);
2667
2668 dev->real_num_tx_queues = txq;
2669
2670 if (disabling) {
2671 synchronize_net();
2672 qdisc_reset_all_tx_gt(dev, txq);
2673#ifdef CONFIG_XPS
2674 netif_reset_xps_queues_gt(dev, txq);
2675#endif
2676 }
2677 } else {
2678 dev->real_num_tx_queues = txq;
2679 }
2680
2681 return 0;
2682}
2683EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2684
2685#ifdef CONFIG_SYSFS
2686/**
2687 * netif_set_real_num_rx_queues - set actual number of RX queues used
2688 * @dev: Network device
2689 * @rxq: Actual number of RX queues
2690 *
2691 * This must be called either with the rtnl_lock held or before
2692 * registration of the net device. Returns 0 on success, or a
2693 * negative error code. If called before registration, it always
2694 * succeeds.
2695 */
2696int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2697{
2698 int rc;
2699
2700 if (rxq < 1 || rxq > dev->num_rx_queues)
2701 return -EINVAL;
2702
2703 if (dev->reg_state == NETREG_REGISTERED) {
2704 ASSERT_RTNL();
2705
2706 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2707 rxq);
2708 if (rc)
2709 return rc;
2710 }
2711
2712 dev->real_num_rx_queues = rxq;
2713 return 0;
2714}
2715EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2716#endif
2717
2718/**
2719 * netif_get_num_default_rss_queues - default number of RSS queues
2720 *
2721 * This routine should set an upper limit on the number of RSS queues
2722 * used by default by multiqueue devices.
2723 */
2724int netif_get_num_default_rss_queues(void)
2725{
2726 return is_kdump_kernel() ?
2727 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2728}
2729EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2730
2731static void __netif_reschedule(struct Qdisc *q)
2732{
2733 struct softnet_data *sd;
2734 unsigned long flags;
2735
2736 local_irq_save(flags);
2737 sd = this_cpu_ptr(&softnet_data);
2738 q->next_sched = NULL;
2739 *sd->output_queue_tailp = q;
2740 sd->output_queue_tailp = &q->next_sched;
2741 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2742 local_irq_restore(flags);
2743}
2744
2745void __netif_schedule(struct Qdisc *q)
2746{
2747 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2748 __netif_reschedule(q);
2749}
2750EXPORT_SYMBOL(__netif_schedule);
2751
2752struct dev_kfree_skb_cb {
2753 enum skb_free_reason reason;
2754};
2755
2756static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2757{
2758 return (struct dev_kfree_skb_cb *)skb->cb;
2759}
2760
2761void netif_schedule_queue(struct netdev_queue *txq)
2762{
2763 rcu_read_lock();
2764 if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2765 struct Qdisc *q = rcu_dereference(txq->qdisc);
2766
2767 __netif_schedule(q);
2768 }
2769 rcu_read_unlock();
2770}
2771EXPORT_SYMBOL(netif_schedule_queue);
2772
2773void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2774{
2775 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2776 struct Qdisc *q;
2777
2778 rcu_read_lock();
2779 q = rcu_dereference(dev_queue->qdisc);
2780 __netif_schedule(q);
2781 rcu_read_unlock();
2782 }
2783}
2784EXPORT_SYMBOL(netif_tx_wake_queue);
2785
2786void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2787{
2788 unsigned long flags;
2789
2790 if (unlikely(!skb))
2791 return;
2792
2793 if (likely(refcount_read(&skb->users) == 1)) {
2794 smp_rmb();
2795 refcount_set(&skb->users, 0);
2796 } else if (likely(!refcount_dec_and_test(&skb->users))) {
2797 return;
2798 }
2799 get_kfree_skb_cb(skb)->reason = reason;
2800 local_irq_save(flags);
2801 skb->next = __this_cpu_read(softnet_data.completion_queue);
2802 __this_cpu_write(softnet_data.completion_queue, skb);
2803 raise_softirq_irqoff(NET_TX_SOFTIRQ);
2804 local_irq_restore(flags);
2805}
2806EXPORT_SYMBOL(__dev_kfree_skb_irq);
2807
2808void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2809{
2810 if (in_irq() || irqs_disabled())
2811 __dev_kfree_skb_irq(skb, reason);
2812 else
2813 dev_kfree_skb(skb);
2814}
2815EXPORT_SYMBOL(__dev_kfree_skb_any);
2816
2817
2818/**
2819 * netif_device_detach - mark device as removed
2820 * @dev: network device
2821 *
2822 * Mark device as removed from system and therefore no longer available.
2823 */
2824void netif_device_detach(struct net_device *dev)
2825{
2826 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2827 netif_running(dev)) {
2828 netif_tx_stop_all_queues(dev);
2829 }
2830}
2831EXPORT_SYMBOL(netif_device_detach);
2832
2833/**
2834 * netif_device_attach - mark device as attached
2835 * @dev: network device
2836 *
2837 * Mark device as attached from system and restart if needed.
2838 */
2839void netif_device_attach(struct net_device *dev)
2840{
2841 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2842 netif_running(dev)) {
2843 netif_tx_wake_all_queues(dev);
2844 __netdev_watchdog_up(dev);
2845 }
2846}
2847EXPORT_SYMBOL(netif_device_attach);
2848
2849/*
2850 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2851 * to be used as a distribution range.
2852 */
2853static u16 skb_tx_hash(const struct net_device *dev,
2854 const struct net_device *sb_dev,
2855 struct sk_buff *skb)
2856{
2857 u32 hash;
2858 u16 qoffset = 0;
2859 u16 qcount = dev->real_num_tx_queues;
2860
2861 if (dev->num_tc) {
2862 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2863
2864 qoffset = sb_dev->tc_to_txq[tc].offset;
2865 qcount = sb_dev->tc_to_txq[tc].count;
2866 }
2867
2868 if (skb_rx_queue_recorded(skb)) {
2869 hash = skb_get_rx_queue(skb);
2870 while (unlikely(hash >= qcount))
2871 hash -= qcount;
2872 return hash + qoffset;
2873 }
2874
2875 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2876}
2877
2878static void skb_warn_bad_offload(const struct sk_buff *skb)
2879{
2880 static const netdev_features_t null_features;
2881 struct net_device *dev = skb->dev;
2882 const char *name = "";
2883
2884 if (!net_ratelimit())
2885 return;
2886
2887 if (dev) {
2888 if (dev->dev.parent)
2889 name = dev_driver_string(dev->dev.parent);
2890 else
2891 name = netdev_name(dev);
2892 }
2893 WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2894 "gso_type=%d ip_summed=%d\n",
2895 name, dev ? &dev->features : &null_features,
2896 skb->sk ? &skb->sk->sk_route_caps : &null_features,
2897 skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2898 skb_shinfo(skb)->gso_type, skb->ip_summed);
2899}
2900
2901/*
2902 * Invalidate hardware checksum when packet is to be mangled, and
2903 * complete checksum manually on outgoing path.
2904 */
2905int skb_checksum_help(struct sk_buff *skb)
2906{
2907 __wsum csum;
2908 int ret = 0, offset;
2909
2910 if (skb->ip_summed == CHECKSUM_COMPLETE)
2911 goto out_set_summed;
2912
2913 if (unlikely(skb_shinfo(skb)->gso_size)) {
2914 skb_warn_bad_offload(skb);
2915 return -EINVAL;
2916 }
2917
2918 /* Before computing a checksum, we should make sure no frag could
2919 * be modified by an external entity : checksum could be wrong.
2920 */
2921 if (skb_has_shared_frag(skb)) {
2922 ret = __skb_linearize(skb);
2923 if (ret)
2924 goto out;
2925 }
2926
2927 offset = skb_checksum_start_offset(skb);
2928 BUG_ON(offset >= skb_headlen(skb));
2929 csum = skb_checksum(skb, offset, skb->len - offset, 0);
2930
2931 offset += skb->csum_offset;
2932 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2933
2934 if (skb_cloned(skb) &&
2935 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2936 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2937 if (ret)
2938 goto out;
2939 }
2940
2941 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2942out_set_summed:
2943 skb->ip_summed = CHECKSUM_NONE;
2944out:
2945 return ret;
2946}
2947EXPORT_SYMBOL(skb_checksum_help);
2948
2949int skb_crc32c_csum_help(struct sk_buff *skb)
2950{
2951 __le32 crc32c_csum;
2952 int ret = 0, offset, start;
2953
2954 if (skb->ip_summed != CHECKSUM_PARTIAL)
2955 goto out;
2956
2957 if (unlikely(skb_is_gso(skb)))
2958 goto out;
2959
2960 /* Before computing a checksum, we should make sure no frag could
2961 * be modified by an external entity : checksum could be wrong.
2962 */
2963 if (unlikely(skb_has_shared_frag(skb))) {
2964 ret = __skb_linearize(skb);
2965 if (ret)
2966 goto out;
2967 }
2968 start = skb_checksum_start_offset(skb);
2969 offset = start + offsetof(struct sctphdr, checksum);
2970 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
2971 ret = -EINVAL;
2972 goto out;
2973 }
2974 if (skb_cloned(skb) &&
2975 !skb_clone_writable(skb, offset + sizeof(__le32))) {
2976 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2977 if (ret)
2978 goto out;
2979 }
2980 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
2981 skb->len - start, ~(__u32)0,
2982 crc32c_csum_stub));
2983 *(__le32 *)(skb->data + offset) = crc32c_csum;
2984 skb->ip_summed = CHECKSUM_NONE;
2985 skb->csum_not_inet = 0;
2986out:
2987 return ret;
2988}
2989
2990__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2991{
2992 __be16 type = skb->protocol;
2993
2994 /* Tunnel gso handlers can set protocol to ethernet. */
2995 if (type == htons(ETH_P_TEB)) {
2996 struct ethhdr *eth;
2997
2998 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2999 return 0;
3000
3001 eth = (struct ethhdr *)skb->data;
3002 type = eth->h_proto;
3003 }
3004
3005 return __vlan_get_protocol(skb, type, depth);
3006}
3007
3008/**
3009 * skb_mac_gso_segment - mac layer segmentation handler.
3010 * @skb: buffer to segment
3011 * @features: features for the output path (see dev->features)
3012 */
3013struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
3014 netdev_features_t features)
3015{
3016 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
3017 struct packet_offload *ptype;
3018 int vlan_depth = skb->mac_len;
3019 __be16 type = skb_network_protocol(skb, &vlan_depth);
3020
3021 if (unlikely(!type))
3022 return ERR_PTR(-EINVAL);
3023
3024 __skb_pull(skb, vlan_depth);
3025
3026 rcu_read_lock();
3027 list_for_each_entry_rcu(ptype, &offload_base, list) {
3028 if (ptype->type == type && ptype->callbacks.gso_segment) {
3029 segs = ptype->callbacks.gso_segment(skb, features);
3030 break;
3031 }
3032 }
3033 rcu_read_unlock();
3034
3035 __skb_push(skb, skb->data - skb_mac_header(skb));
3036
3037 return segs;
3038}
3039EXPORT_SYMBOL(skb_mac_gso_segment);
3040
3041
3042/* openvswitch calls this on rx path, so we need a different check.
3043 */
3044static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3045{
3046 if (tx_path)
3047 return skb->ip_summed != CHECKSUM_PARTIAL &&
3048 skb->ip_summed != CHECKSUM_UNNECESSARY;
3049
3050 return skb->ip_summed == CHECKSUM_NONE;
3051}
3052
3053/**
3054 * __skb_gso_segment - Perform segmentation on skb.
3055 * @skb: buffer to segment
3056 * @features: features for the output path (see dev->features)
3057 * @tx_path: whether it is called in TX path
3058 *
3059 * This function segments the given skb and returns a list of segments.
3060 *
3061 * It may return NULL if the skb requires no segmentation. This is
3062 * only possible when GSO is used for verifying header integrity.
3063 *
3064 * Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
3065 */
3066struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3067 netdev_features_t features, bool tx_path)
3068{
3069 struct sk_buff *segs;
3070
3071 if (unlikely(skb_needs_check(skb, tx_path))) {
3072 int err;
3073
3074 /* We're going to init ->check field in TCP or UDP header */
3075 err = skb_cow_head(skb, 0);
3076 if (err < 0)
3077 return ERR_PTR(err);
3078 }
3079
3080 /* Only report GSO partial support if it will enable us to
3081 * support segmentation on this frame without needing additional
3082 * work.
3083 */
3084 if (features & NETIF_F_GSO_PARTIAL) {
3085 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3086 struct net_device *dev = skb->dev;
3087
3088 partial_features |= dev->features & dev->gso_partial_features;
3089 if (!skb_gso_ok(skb, features | partial_features))
3090 features &= ~NETIF_F_GSO_PARTIAL;
3091 }
3092
3093 BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
3094 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3095
3096 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
3097 SKB_GSO_CB(skb)->encap_level = 0;
3098
3099 skb_reset_mac_header(skb);
3100 skb_reset_mac_len(skb);
3101
3102 segs = skb_mac_gso_segment(skb, features);
3103
3104 if (unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
3105 skb_warn_bad_offload(skb);
3106
3107 return segs;
3108}
3109EXPORT_SYMBOL(__skb_gso_segment);
3110
3111/* Take action when hardware reception checksum errors are detected. */
3112#ifdef CONFIG_BUG
3113void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3114{
3115 if (net_ratelimit()) {
3116 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
3117 if (dev)
3118 pr_err("dev features: %pNF\n", &dev->features);
3119 pr_err("skb len=%u data_len=%u pkt_type=%u gso_size=%u gso_type=%u nr_frags=%u ip_summed=%u csum=%x csum_complete_sw=%d csum_valid=%d csum_level=%u\n",
3120 skb->len, skb->data_len, skb->pkt_type,
3121 skb_shinfo(skb)->gso_size, skb_shinfo(skb)->gso_type,
3122 skb_shinfo(skb)->nr_frags, skb->ip_summed, skb->csum,
3123 skb->csum_complete_sw, skb->csum_valid, skb->csum_level);
3124 dump_stack();
3125 }
3126}
3127EXPORT_SYMBOL(netdev_rx_csum_fault);
3128#endif
3129
3130/* XXX: check that highmem exists at all on the given machine. */
3131static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3132{
3133#ifdef CONFIG_HIGHMEM
3134 int i;
3135
3136 if (!(dev->features & NETIF_F_HIGHDMA)) {
3137 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3138 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3139
3140 if (PageHighMem(skb_frag_page(frag)))
3141 return 1;
3142 }
3143 }
3144#endif
3145 return 0;
3146}
3147
3148/* If MPLS offload request, verify we are testing hardware MPLS features
3149 * instead of standard features for the netdev.
3150 */
3151#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3152static netdev_features_t net_mpls_features(struct sk_buff *skb,
3153 netdev_features_t features,
3154 __be16 type)
3155{
3156 if (eth_p_mpls(type))
3157 features &= skb->dev->mpls_features;
3158
3159 return features;
3160}
3161#else
3162static netdev_features_t net_mpls_features(struct sk_buff *skb,
3163 netdev_features_t features,
3164 __be16 type)
3165{
3166 return features;
3167}
3168#endif
3169
3170static netdev_features_t harmonize_features(struct sk_buff *skb,
3171 netdev_features_t features)
3172{
3173 int tmp;
3174 __be16 type;
3175
3176 type = skb_network_protocol(skb, &tmp);
3177 features = net_mpls_features(skb, features, type);
3178
3179 if (skb->ip_summed != CHECKSUM_NONE &&
3180 !can_checksum_protocol(features, type)) {
3181 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3182 }
3183 if (illegal_highdma(skb->dev, skb))
3184 features &= ~NETIF_F_SG;
3185
3186 return features;
3187}
3188
3189netdev_features_t passthru_features_check(struct sk_buff *skb,
3190 struct net_device *dev,
3191 netdev_features_t features)
3192{
3193 return features;
3194}
3195EXPORT_SYMBOL(passthru_features_check);
3196
3197static netdev_features_t dflt_features_check(struct sk_buff *skb,
3198 struct net_device *dev,
3199 netdev_features_t features)
3200{
3201 return vlan_features_check(skb, features);
3202}
3203
3204static netdev_features_t gso_features_check(const struct sk_buff *skb,
3205 struct net_device *dev,
3206 netdev_features_t features)
3207{
3208 u16 gso_segs = skb_shinfo(skb)->gso_segs;
3209
3210 if (gso_segs > dev->gso_max_segs)
3211 return features & ~NETIF_F_GSO_MASK;
3212
3213 /* Support for GSO partial features requires software
3214 * intervention before we can actually process the packets
3215 * so we need to strip support for any partial features now
3216 * and we can pull them back in after we have partially
3217 * segmented the frame.
3218 */
3219 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3220 features &= ~dev->gso_partial_features;
3221
3222 /* Make sure to clear the IPv4 ID mangling feature if the
3223 * IPv4 header has the potential to be fragmented.
3224 */
3225 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3226 struct iphdr *iph = skb->encapsulation ?
3227 inner_ip_hdr(skb) : ip_hdr(skb);
3228
3229 if (!(iph->frag_off & htons(IP_DF)))
3230 features &= ~NETIF_F_TSO_MANGLEID;
3231 }
3232
3233 return features;
3234}
3235
3236netdev_features_t netif_skb_features(struct sk_buff *skb)
3237{
3238 struct net_device *dev = skb->dev;
3239 netdev_features_t features = dev->features;
3240
3241 if (skb_is_gso(skb))
3242 features = gso_features_check(skb, dev, features);
3243
3244 /* If encapsulation offload request, verify we are testing
3245 * hardware encapsulation features instead of standard
3246 * features for the netdev
3247 */
3248 if (skb->encapsulation)
3249 features &= dev->hw_enc_features;
3250
3251 if (skb_vlan_tagged(skb))
3252 features = netdev_intersect_features(features,
3253 dev->vlan_features |
3254 NETIF_F_HW_VLAN_CTAG_TX |
3255 NETIF_F_HW_VLAN_STAG_TX);
3256
3257 if (dev->netdev_ops->ndo_features_check)
3258 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3259 features);
3260 else
3261 features &= dflt_features_check(skb, dev, features);
3262
3263 return harmonize_features(skb, features);
3264}
3265EXPORT_SYMBOL(netif_skb_features);
3266
3267static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3268 struct netdev_queue *txq, bool more)
3269{
3270 unsigned int len;
3271 int rc;
3272
3273 if (dev_nit_active(dev))
3274 dev_queue_xmit_nit(skb, dev);
3275
3276 len = skb->len;
3277 trace_net_dev_start_xmit(skb, dev);
3278 rc = netdev_start_xmit(skb, dev, txq, more);
3279 trace_net_dev_xmit(skb, rc, dev, len);
3280
3281 return rc;
3282}
3283
3284struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3285 struct netdev_queue *txq, int *ret)
3286{
3287 struct sk_buff *skb = first;
3288 int rc = NETDEV_TX_OK;
3289
3290 while (skb) {
3291 struct sk_buff *next = skb->next;
3292
3293 skb_mark_not_on_list(skb);
3294 rc = xmit_one(skb, dev, txq, next != NULL);
3295 if (unlikely(!dev_xmit_complete(rc))) {
3296 skb->next = next;
3297 goto out;
3298 }
3299
3300 skb = next;
3301 if (netif_tx_queue_stopped(txq) && skb) {
3302 rc = NETDEV_TX_BUSY;
3303 break;
3304 }
3305 }
3306
3307out:
3308 *ret = rc;
3309 return skb;
3310}
3311
3312static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3313 netdev_features_t features)
3314{
3315 if (skb_vlan_tag_present(skb) &&
3316 !vlan_hw_offload_capable(features, skb->vlan_proto))
3317 skb = __vlan_hwaccel_push_inside(skb);
3318 return skb;
3319}
3320
3321int skb_csum_hwoffload_help(struct sk_buff *skb,
3322 const netdev_features_t features)
3323{
3324 if (unlikely(skb->csum_not_inet))
3325 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3326 skb_crc32c_csum_help(skb);
3327
3328 return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb);
3329}
3330EXPORT_SYMBOL(skb_csum_hwoffload_help);
3331
3332static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3333{
3334 netdev_features_t features;
3335
3336 features = netif_skb_features(skb);
3337 skb = validate_xmit_vlan(skb, features);
3338 if (unlikely(!skb))
3339 goto out_null;
3340
3341 skb = sk_validate_xmit_skb(skb, dev);
3342 if (unlikely(!skb))
3343 goto out_null;
3344
3345 if (netif_needs_gso(skb, features)) {
3346 struct sk_buff *segs;
3347
3348 segs = skb_gso_segment(skb, features);
3349 if (IS_ERR(segs)) {
3350 goto out_kfree_skb;
3351 } else if (segs) {
3352 consume_skb(skb);
3353 skb = segs;
3354 }
3355 } else {
3356 if (skb_needs_linearize(skb, features) &&
3357 __skb_linearize(skb))
3358 goto out_kfree_skb;
3359
3360 /* If packet is not checksummed and device does not
3361 * support checksumming for this protocol, complete
3362 * checksumming here.
3363 */
3364 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3365 if (skb->encapsulation)
3366 skb_set_inner_transport_header(skb,
3367 skb_checksum_start_offset(skb));
3368 else
3369 skb_set_transport_header(skb,
3370 skb_checksum_start_offset(skb));
3371 if (skb_csum_hwoffload_help(skb, features))
3372 goto out_kfree_skb;
3373 }
3374 }
3375
3376 skb = validate_xmit_xfrm(skb, features, again);
3377
3378 return skb;
3379
3380out_kfree_skb:
3381 kfree_skb(skb);
3382out_null:
3383 atomic_long_inc(&dev->tx_dropped);
3384 return NULL;
3385}
3386
3387struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3388{
3389 struct sk_buff *next, *head = NULL, *tail;
3390
3391 for (; skb != NULL; skb = next) {
3392 next = skb->next;
3393 skb_mark_not_on_list(skb);
3394
3395 /* in case skb wont be segmented, point to itself */
3396 skb->prev = skb;
3397
3398 skb = validate_xmit_skb(skb, dev, again);
3399 if (!skb)
3400 continue;
3401
3402 if (!head)
3403 head = skb;
3404 else
3405 tail->next = skb;
3406 /* If skb was segmented, skb->prev points to
3407 * the last segment. If not, it still contains skb.
3408 */
3409 tail = skb->prev;
3410 }
3411 return head;
3412}
3413EXPORT_SYMBOL_GPL(