1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io |
3 | */ |
4 | |
5 | /* Devmaps primary use is as a backend map for XDP BPF helper call |
6 | * bpf_redirect_map(). Because XDP is mostly concerned with performance we |
7 | * spent some effort to ensure the datapath with redirect maps does not use |
8 | * any locking. This is a quick note on the details. |
9 | * |
10 | * We have three possible paths to get into the devmap control plane bpf |
11 | * syscalls, bpf programs, and driver side xmit/flush operations. A bpf syscall |
12 | * will invoke an update, delete, or lookup operation. To ensure updates and |
13 | * deletes appear atomic from the datapath side xchg() is used to modify the |
14 | * netdev_map array. Then because the datapath does a lookup into the netdev_map |
15 | * array (read-only) from an RCU critical section we use call_rcu() to wait for |
16 | * an rcu grace period before free'ing the old data structures. This ensures the |
17 | * datapath always has a valid copy. However, the datapath does a "flush" |
18 | * operation that pushes any pending packets in the driver outside the RCU |
19 | * critical section. Each bpf_dtab_netdev tracks these pending operations using |
20 | * a per-cpu flush list. The bpf_dtab_netdev object will not be destroyed until |
21 | * this list is empty, indicating outstanding flush operations have completed. |
22 | * |
23 | * BPF syscalls may race with BPF program calls on any of the update, delete |
24 | * or lookup operations. As noted above the xchg() operation also keep the |
25 | * netdev_map consistent in this case. From the devmap side BPF programs |
26 | * calling into these operations are the same as multiple user space threads |
27 | * making system calls. |
28 | * |
29 | * Finally, any of the above may race with a netdev_unregister notifier. The |
30 | * unregister notifier must search for net devices in the map structure that |
31 | * contain a reference to the net device and remove them. This is a two step |
32 | * process (a) dereference the bpf_dtab_netdev object in netdev_map and (b) |
33 | * check to see if the ifindex is the same as the net_device being removed. |
34 | * When removing the dev a cmpxchg() is used to ensure the correct dev is |
35 | * removed, in the case of a concurrent update or delete operation it is |
36 | * possible that the initially referenced dev is no longer in the map. As the |
37 | * notifier hook walks the map we know that new dev references can not be |
38 | * added by the user because core infrastructure ensures dev_get_by_index() |
39 | * calls will fail at this point. |
40 | * |
41 | * The devmap_hash type is a map type which interprets keys as ifindexes and |
42 | * indexes these using a hashmap. This allows maps that use ifindex as key to be |
43 | * densely packed instead of having holes in the lookup array for unused |
44 | * ifindexes. The setup and packet enqueue/send code is shared between the two |
45 | * types of devmap; only the lookup and insertion is different. |
46 | */ |
47 | #include <linux/bpf.h> |
48 | #include <net/xdp.h> |
49 | #include <linux/filter.h> |
50 | #include <trace/events/xdp.h> |
51 | #include <linux/btf_ids.h> |
52 | |
53 | #define DEV_CREATE_FLAG_MASK \ |
54 | (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY) |
55 | |
56 | struct xdp_dev_bulk_queue { |
57 | struct xdp_frame *q[DEV_MAP_BULK_SIZE]; |
58 | struct list_head flush_node; |
59 | struct net_device *dev; |
60 | struct net_device *dev_rx; |
61 | struct bpf_prog *xdp_prog; |
62 | unsigned int count; |
63 | }; |
64 | |
65 | struct bpf_dtab_netdev { |
66 | struct net_device *dev; /* must be first member, due to tracepoint */ |
67 | struct hlist_node index_hlist; |
68 | struct bpf_prog *xdp_prog; |
69 | struct rcu_head rcu; |
70 | unsigned int idx; |
71 | struct bpf_devmap_val val; |
72 | }; |
73 | |
74 | struct bpf_dtab { |
75 | struct bpf_map map; |
76 | struct bpf_dtab_netdev __rcu **netdev_map; /* DEVMAP type only */ |
77 | struct list_head list; |
78 | |
79 | /* these are only used for DEVMAP_HASH type maps */ |
80 | struct hlist_head *dev_index_head; |
81 | spinlock_t index_lock; |
82 | unsigned int items; |
83 | u32 n_buckets; |
84 | }; |
85 | |
86 | static DEFINE_PER_CPU(struct list_head, dev_flush_list); |
87 | static DEFINE_SPINLOCK(dev_map_lock); |
88 | static LIST_HEAD(dev_map_list); |
89 | |
90 | static struct hlist_head *dev_map_create_hash(unsigned int entries, |
91 | int numa_node) |
92 | { |
93 | int i; |
94 | struct hlist_head *hash; |
95 | |
96 | hash = bpf_map_area_alloc(size: (u64) entries * sizeof(*hash), numa_node); |
97 | if (hash != NULL) |
98 | for (i = 0; i < entries; i++) |
99 | INIT_HLIST_HEAD(&hash[i]); |
100 | |
101 | return hash; |
102 | } |
103 | |
104 | static inline struct hlist_head *dev_map_index_hash(struct bpf_dtab *dtab, |
105 | int idx) |
106 | { |
107 | return &dtab->dev_index_head[idx & (dtab->n_buckets - 1)]; |
108 | } |
109 | |
110 | static int dev_map_init_map(struct bpf_dtab *dtab, union bpf_attr *attr) |
111 | { |
112 | u32 valsize = attr->value_size; |
113 | |
114 | /* check sanity of attributes. 2 value sizes supported: |
115 | * 4 bytes: ifindex |
116 | * 8 bytes: ifindex + prog fd |
117 | */ |
118 | if (attr->max_entries == 0 || attr->key_size != 4 || |
119 | (valsize != offsetofend(struct bpf_devmap_val, ifindex) && |
120 | valsize != offsetofend(struct bpf_devmap_val, bpf_prog.fd)) || |
121 | attr->map_flags & ~DEV_CREATE_FLAG_MASK) |
122 | return -EINVAL; |
123 | |
124 | /* Lookup returns a pointer straight to dev->ifindex, so make sure the |
125 | * verifier prevents writes from the BPF side |
126 | */ |
127 | attr->map_flags |= BPF_F_RDONLY_PROG; |
128 | |
129 | |
130 | bpf_map_init_from_attr(map: &dtab->map, attr); |
131 | |
132 | if (attr->map_type == BPF_MAP_TYPE_DEVMAP_HASH) { |
133 | dtab->n_buckets = roundup_pow_of_two(dtab->map.max_entries); |
134 | |
135 | if (!dtab->n_buckets) /* Overflow check */ |
136 | return -EINVAL; |
137 | } |
138 | |
139 | if (attr->map_type == BPF_MAP_TYPE_DEVMAP_HASH) { |
140 | dtab->dev_index_head = dev_map_create_hash(entries: dtab->n_buckets, |
141 | numa_node: dtab->map.numa_node); |
142 | if (!dtab->dev_index_head) |
143 | return -ENOMEM; |
144 | |
145 | spin_lock_init(&dtab->index_lock); |
146 | } else { |
147 | dtab->netdev_map = bpf_map_area_alloc(size: (u64) dtab->map.max_entries * |
148 | sizeof(struct bpf_dtab_netdev *), |
149 | numa_node: dtab->map.numa_node); |
150 | if (!dtab->netdev_map) |
151 | return -ENOMEM; |
152 | } |
153 | |
154 | return 0; |
155 | } |
156 | |
157 | static struct bpf_map *dev_map_alloc(union bpf_attr *attr) |
158 | { |
159 | struct bpf_dtab *dtab; |
160 | int err; |
161 | |
162 | dtab = bpf_map_area_alloc(size: sizeof(*dtab), NUMA_NO_NODE); |
163 | if (!dtab) |
164 | return ERR_PTR(error: -ENOMEM); |
165 | |
166 | err = dev_map_init_map(dtab, attr); |
167 | if (err) { |
168 | bpf_map_area_free(base: dtab); |
169 | return ERR_PTR(error: err); |
170 | } |
171 | |
172 | spin_lock(lock: &dev_map_lock); |
173 | list_add_tail_rcu(new: &dtab->list, head: &dev_map_list); |
174 | spin_unlock(lock: &dev_map_lock); |
175 | |
176 | return &dtab->map; |
177 | } |
178 | |
179 | static void dev_map_free(struct bpf_map *map) |
180 | { |
181 | struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); |
182 | int i; |
183 | |
184 | /* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0, |
185 | * so the programs (can be more than one that used this map) were |
186 | * disconnected from events. The following synchronize_rcu() guarantees |
187 | * both rcu read critical sections complete and waits for |
188 | * preempt-disable regions (NAPI being the relevant context here) so we |
189 | * are certain there will be no further reads against the netdev_map and |
190 | * all flush operations are complete. Flush operations can only be done |
191 | * from NAPI context for this reason. |
192 | */ |
193 | |
194 | spin_lock(lock: &dev_map_lock); |
195 | list_del_rcu(entry: &dtab->list); |
196 | spin_unlock(lock: &dev_map_lock); |
197 | |
198 | bpf_clear_redirect_map(map); |
199 | synchronize_rcu(); |
200 | |
201 | /* Make sure prior __dev_map_entry_free() have completed. */ |
202 | rcu_barrier(); |
203 | |
204 | if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) { |
205 | for (i = 0; i < dtab->n_buckets; i++) { |
206 | struct bpf_dtab_netdev *dev; |
207 | struct hlist_head *head; |
208 | struct hlist_node *next; |
209 | |
210 | head = dev_map_index_hash(dtab, idx: i); |
211 | |
212 | hlist_for_each_entry_safe(dev, next, head, index_hlist) { |
213 | hlist_del_rcu(n: &dev->index_hlist); |
214 | if (dev->xdp_prog) |
215 | bpf_prog_put(prog: dev->xdp_prog); |
216 | dev_put(dev: dev->dev); |
217 | kfree(objp: dev); |
218 | } |
219 | } |
220 | |
221 | bpf_map_area_free(base: dtab->dev_index_head); |
222 | } else { |
223 | for (i = 0; i < dtab->map.max_entries; i++) { |
224 | struct bpf_dtab_netdev *dev; |
225 | |
226 | dev = rcu_dereference_raw(dtab->netdev_map[i]); |
227 | if (!dev) |
228 | continue; |
229 | |
230 | if (dev->xdp_prog) |
231 | bpf_prog_put(prog: dev->xdp_prog); |
232 | dev_put(dev: dev->dev); |
233 | kfree(objp: dev); |
234 | } |
235 | |
236 | bpf_map_area_free(base: dtab->netdev_map); |
237 | } |
238 | |
239 | bpf_map_area_free(base: dtab); |
240 | } |
241 | |
242 | static int dev_map_get_next_key(struct bpf_map *map, void *key, void *next_key) |
243 | { |
244 | struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); |
245 | u32 index = key ? *(u32 *)key : U32_MAX; |
246 | u32 *next = next_key; |
247 | |
248 | if (index >= dtab->map.max_entries) { |
249 | *next = 0; |
250 | return 0; |
251 | } |
252 | |
253 | if (index == dtab->map.max_entries - 1) |
254 | return -ENOENT; |
255 | *next = index + 1; |
256 | return 0; |
257 | } |
258 | |
259 | /* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or |
260 | * by local_bh_disable() (from XDP calls inside NAPI). The |
261 | * rcu_read_lock_bh_held() below makes lockdep accept both. |
262 | */ |
263 | static void *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key) |
264 | { |
265 | struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); |
266 | struct hlist_head *head = dev_map_index_hash(dtab, idx: key); |
267 | struct bpf_dtab_netdev *dev; |
268 | |
269 | hlist_for_each_entry_rcu(dev, head, index_hlist, |
270 | lockdep_is_held(&dtab->index_lock)) |
271 | if (dev->idx == key) |
272 | return dev; |
273 | |
274 | return NULL; |
275 | } |
276 | |
277 | static int dev_map_hash_get_next_key(struct bpf_map *map, void *key, |
278 | void *next_key) |
279 | { |
280 | struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); |
281 | u32 idx, *next = next_key; |
282 | struct bpf_dtab_netdev *dev, *next_dev; |
283 | struct hlist_head *head; |
284 | int i = 0; |
285 | |
286 | if (!key) |
287 | goto find_first; |
288 | |
289 | idx = *(u32 *)key; |
290 | |
291 | dev = __dev_map_hash_lookup_elem(map, key: idx); |
292 | if (!dev) |
293 | goto find_first; |
294 | |
295 | next_dev = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&dev->index_hlist)), |
296 | struct bpf_dtab_netdev, index_hlist); |
297 | |
298 | if (next_dev) { |
299 | *next = next_dev->idx; |
300 | return 0; |
301 | } |
302 | |
303 | i = idx & (dtab->n_buckets - 1); |
304 | i++; |
305 | |
306 | find_first: |
307 | for (; i < dtab->n_buckets; i++) { |
308 | head = dev_map_index_hash(dtab, idx: i); |
309 | |
310 | next_dev = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)), |
311 | struct bpf_dtab_netdev, |
312 | index_hlist); |
313 | if (next_dev) { |
314 | *next = next_dev->idx; |
315 | return 0; |
316 | } |
317 | } |
318 | |
319 | return -ENOENT; |
320 | } |
321 | |
322 | static int dev_map_bpf_prog_run(struct bpf_prog *xdp_prog, |
323 | struct xdp_frame **frames, int n, |
324 | struct net_device *dev) |
325 | { |
326 | struct xdp_txq_info txq = { .dev = dev }; |
327 | struct xdp_buff xdp; |
328 | int i, nframes = 0; |
329 | |
330 | for (i = 0; i < n; i++) { |
331 | struct xdp_frame *xdpf = frames[i]; |
332 | u32 act; |
333 | int err; |
334 | |
335 | xdp_convert_frame_to_buff(frame: xdpf, xdp: &xdp); |
336 | xdp.txq = &txq; |
337 | |
338 | act = bpf_prog_run_xdp(prog: xdp_prog, xdp: &xdp); |
339 | switch (act) { |
340 | case XDP_PASS: |
341 | err = xdp_update_frame_from_buff(xdp: &xdp, xdp_frame: xdpf); |
342 | if (unlikely(err < 0)) |
343 | xdp_return_frame_rx_napi(xdpf); |
344 | else |
345 | frames[nframes++] = xdpf; |
346 | break; |
347 | default: |
348 | bpf_warn_invalid_xdp_action(NULL, prog: xdp_prog, act); |
349 | fallthrough; |
350 | case XDP_ABORTED: |
351 | trace_xdp_exception(dev, xdp: xdp_prog, act); |
352 | fallthrough; |
353 | case XDP_DROP: |
354 | xdp_return_frame_rx_napi(xdpf); |
355 | break; |
356 | } |
357 | } |
358 | return nframes; /* sent frames count */ |
359 | } |
360 | |
361 | static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags) |
362 | { |
363 | struct net_device *dev = bq->dev; |
364 | unsigned int cnt = bq->count; |
365 | int sent = 0, err = 0; |
366 | int to_send = cnt; |
367 | int i; |
368 | |
369 | if (unlikely(!cnt)) |
370 | return; |
371 | |
372 | for (i = 0; i < cnt; i++) { |
373 | struct xdp_frame *xdpf = bq->q[i]; |
374 | |
375 | prefetch(xdpf); |
376 | } |
377 | |
378 | if (bq->xdp_prog) { |
379 | to_send = dev_map_bpf_prog_run(xdp_prog: bq->xdp_prog, frames: bq->q, n: cnt, dev); |
380 | if (!to_send) |
381 | goto out; |
382 | } |
383 | |
384 | sent = dev->netdev_ops->ndo_xdp_xmit(dev, to_send, bq->q, flags); |
385 | if (sent < 0) { |
386 | /* If ndo_xdp_xmit fails with an errno, no frames have |
387 | * been xmit'ed. |
388 | */ |
389 | err = sent; |
390 | sent = 0; |
391 | } |
392 | |
393 | /* If not all frames have been transmitted, it is our |
394 | * responsibility to free them |
395 | */ |
396 | for (i = sent; unlikely(i < to_send); i++) |
397 | xdp_return_frame_rx_napi(xdpf: bq->q[i]); |
398 | |
399 | out: |
400 | bq->count = 0; |
401 | trace_xdp_devmap_xmit(from_dev: bq->dev_rx, to_dev: dev, sent, drops: cnt - sent, err); |
402 | } |
403 | |
404 | /* __dev_flush is called from xdp_do_flush() which _must_ be signalled from the |
405 | * driver before returning from its napi->poll() routine. See the comment above |
406 | * xdp_do_flush() in filter.c. |
407 | */ |
408 | void __dev_flush(void) |
409 | { |
410 | struct list_head *flush_list = this_cpu_ptr(&dev_flush_list); |
411 | struct xdp_dev_bulk_queue *bq, *tmp; |
412 | |
413 | list_for_each_entry_safe(bq, tmp, flush_list, flush_node) { |
414 | bq_xmit_all(bq, XDP_XMIT_FLUSH); |
415 | bq->dev_rx = NULL; |
416 | bq->xdp_prog = NULL; |
417 | __list_del_clearprev(entry: &bq->flush_node); |
418 | } |
419 | } |
420 | |
421 | #ifdef CONFIG_DEBUG_NET |
422 | bool dev_check_flush(void) |
423 | { |
424 | if (list_empty(this_cpu_ptr(&dev_flush_list))) |
425 | return false; |
426 | __dev_flush(); |
427 | return true; |
428 | } |
429 | #endif |
430 | |
431 | /* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or |
432 | * by local_bh_disable() (from XDP calls inside NAPI). The |
433 | * rcu_read_lock_bh_held() below makes lockdep accept both. |
434 | */ |
435 | static void *__dev_map_lookup_elem(struct bpf_map *map, u32 key) |
436 | { |
437 | struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); |
438 | struct bpf_dtab_netdev *obj; |
439 | |
440 | if (key >= map->max_entries) |
441 | return NULL; |
442 | |
443 | obj = rcu_dereference_check(dtab->netdev_map[key], |
444 | rcu_read_lock_bh_held()); |
445 | return obj; |
446 | } |
447 | |
448 | /* Runs in NAPI, i.e., softirq under local_bh_disable(). Thus, safe percpu |
449 | * variable access, and map elements stick around. See comment above |
450 | * xdp_do_flush() in filter.c. |
451 | */ |
452 | static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf, |
453 | struct net_device *dev_rx, struct bpf_prog *xdp_prog) |
454 | { |
455 | struct list_head *flush_list = this_cpu_ptr(&dev_flush_list); |
456 | struct xdp_dev_bulk_queue *bq = this_cpu_ptr(dev->xdp_bulkq); |
457 | |
458 | if (unlikely(bq->count == DEV_MAP_BULK_SIZE)) |
459 | bq_xmit_all(bq, flags: 0); |
460 | |
461 | /* Ingress dev_rx will be the same for all xdp_frame's in |
462 | * bulk_queue, because bq stored per-CPU and must be flushed |
463 | * from net_device drivers NAPI func end. |
464 | * |
465 | * Do the same with xdp_prog and flush_list since these fields |
466 | * are only ever modified together. |
467 | */ |
468 | if (!bq->dev_rx) { |
469 | bq->dev_rx = dev_rx; |
470 | bq->xdp_prog = xdp_prog; |
471 | list_add(new: &bq->flush_node, head: flush_list); |
472 | } |
473 | |
474 | bq->q[bq->count++] = xdpf; |
475 | } |
476 | |
477 | static inline int __xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, |
478 | struct net_device *dev_rx, |
479 | struct bpf_prog *xdp_prog) |
480 | { |
481 | int err; |
482 | |
483 | if (!(dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT)) |
484 | return -EOPNOTSUPP; |
485 | |
486 | if (unlikely(!(dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT_SG) && |
487 | xdp_frame_has_frags(xdpf))) |
488 | return -EOPNOTSUPP; |
489 | |
490 | err = xdp_ok_fwd_dev(fwd: dev, pktlen: xdp_get_frame_len(xdpf)); |
491 | if (unlikely(err)) |
492 | return err; |
493 | |
494 | bq_enqueue(dev, xdpf, dev_rx, xdp_prog); |
495 | return 0; |
496 | } |
497 | |
498 | static u32 dev_map_bpf_prog_run_skb(struct sk_buff *skb, struct bpf_dtab_netdev *dst) |
499 | { |
500 | struct xdp_txq_info txq = { .dev = dst->dev }; |
501 | struct xdp_buff xdp; |
502 | u32 act; |
503 | |
504 | if (!dst->xdp_prog) |
505 | return XDP_PASS; |
506 | |
507 | __skb_pull(skb, len: skb->mac_len); |
508 | xdp.txq = &txq; |
509 | |
510 | act = bpf_prog_run_generic_xdp(skb, xdp: &xdp, xdp_prog: dst->xdp_prog); |
511 | switch (act) { |
512 | case XDP_PASS: |
513 | __skb_push(skb, len: skb->mac_len); |
514 | break; |
515 | default: |
516 | bpf_warn_invalid_xdp_action(NULL, prog: dst->xdp_prog, act); |
517 | fallthrough; |
518 | case XDP_ABORTED: |
519 | trace_xdp_exception(dev: dst->dev, xdp: dst->xdp_prog, act); |
520 | fallthrough; |
521 | case XDP_DROP: |
522 | kfree_skb(skb); |
523 | break; |
524 | } |
525 | |
526 | return act; |
527 | } |
528 | |
529 | int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, |
530 | struct net_device *dev_rx) |
531 | { |
532 | return __xdp_enqueue(dev, xdpf, dev_rx, NULL); |
533 | } |
534 | |
535 | int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, |
536 | struct net_device *dev_rx) |
537 | { |
538 | struct net_device *dev = dst->dev; |
539 | |
540 | return __xdp_enqueue(dev, xdpf, dev_rx, xdp_prog: dst->xdp_prog); |
541 | } |
542 | |
543 | static bool is_valid_dst(struct bpf_dtab_netdev *obj, struct xdp_frame *xdpf) |
544 | { |
545 | if (!obj) |
546 | return false; |
547 | |
548 | if (!(obj->dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT)) |
549 | return false; |
550 | |
551 | if (unlikely(!(obj->dev->xdp_features & NETDEV_XDP_ACT_NDO_XMIT_SG) && |
552 | xdp_frame_has_frags(xdpf))) |
553 | return false; |
554 | |
555 | if (xdp_ok_fwd_dev(fwd: obj->dev, pktlen: xdp_get_frame_len(xdpf))) |
556 | return false; |
557 | |
558 | return true; |
559 | } |
560 | |
561 | static int dev_map_enqueue_clone(struct bpf_dtab_netdev *obj, |
562 | struct net_device *dev_rx, |
563 | struct xdp_frame *xdpf) |
564 | { |
565 | struct xdp_frame *nxdpf; |
566 | |
567 | nxdpf = xdpf_clone(xdpf); |
568 | if (!nxdpf) |
569 | return -ENOMEM; |
570 | |
571 | bq_enqueue(dev: obj->dev, xdpf: nxdpf, dev_rx, xdp_prog: obj->xdp_prog); |
572 | |
573 | return 0; |
574 | } |
575 | |
576 | static inline bool is_ifindex_excluded(int *excluded, int num_excluded, int ifindex) |
577 | { |
578 | while (num_excluded--) { |
579 | if (ifindex == excluded[num_excluded]) |
580 | return true; |
581 | } |
582 | return false; |
583 | } |
584 | |
585 | /* Get ifindex of each upper device. 'indexes' must be able to hold at |
586 | * least MAX_NEST_DEV elements. |
587 | * Returns the number of ifindexes added. |
588 | */ |
589 | static int get_upper_ifindexes(struct net_device *dev, int *indexes) |
590 | { |
591 | struct net_device *upper; |
592 | struct list_head *iter; |
593 | int n = 0; |
594 | |
595 | netdev_for_each_upper_dev_rcu(dev, upper, iter) { |
596 | indexes[n++] = upper->ifindex; |
597 | } |
598 | return n; |
599 | } |
600 | |
601 | int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, |
602 | struct bpf_map *map, bool exclude_ingress) |
603 | { |
604 | struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); |
605 | struct bpf_dtab_netdev *dst, *last_dst = NULL; |
606 | int excluded_devices[1+MAX_NEST_DEV]; |
607 | struct hlist_head *head; |
608 | int num_excluded = 0; |
609 | unsigned int i; |
610 | int err; |
611 | |
612 | if (exclude_ingress) { |
613 | num_excluded = get_upper_ifindexes(dev: dev_rx, indexes: excluded_devices); |
614 | excluded_devices[num_excluded++] = dev_rx->ifindex; |
615 | } |
616 | |
617 | if (map->map_type == BPF_MAP_TYPE_DEVMAP) { |
618 | for (i = 0; i < map->max_entries; i++) { |
619 | dst = rcu_dereference_check(dtab->netdev_map[i], |
620 | rcu_read_lock_bh_held()); |
621 | if (!is_valid_dst(obj: dst, xdpf)) |
622 | continue; |
623 | |
624 | if (is_ifindex_excluded(excluded: excluded_devices, num_excluded, ifindex: dst->dev->ifindex)) |
625 | continue; |
626 | |
627 | /* we only need n-1 clones; last_dst enqueued below */ |
628 | if (!last_dst) { |
629 | last_dst = dst; |
630 | continue; |
631 | } |
632 | |
633 | err = dev_map_enqueue_clone(obj: last_dst, dev_rx, xdpf); |
634 | if (err) |
635 | return err; |
636 | |
637 | last_dst = dst; |
638 | } |
639 | } else { /* BPF_MAP_TYPE_DEVMAP_HASH */ |
640 | for (i = 0; i < dtab->n_buckets; i++) { |
641 | head = dev_map_index_hash(dtab, idx: i); |
642 | hlist_for_each_entry_rcu(dst, head, index_hlist, |
643 | lockdep_is_held(&dtab->index_lock)) { |
644 | if (!is_valid_dst(obj: dst, xdpf)) |
645 | continue; |
646 | |
647 | if (is_ifindex_excluded(excluded: excluded_devices, num_excluded, |
648 | ifindex: dst->dev->ifindex)) |
649 | continue; |
650 | |
651 | /* we only need n-1 clones; last_dst enqueued below */ |
652 | if (!last_dst) { |
653 | last_dst = dst; |
654 | continue; |
655 | } |
656 | |
657 | err = dev_map_enqueue_clone(obj: last_dst, dev_rx, xdpf); |
658 | if (err) |
659 | return err; |
660 | |
661 | last_dst = dst; |
662 | } |
663 | } |
664 | } |
665 | |
666 | /* consume the last copy of the frame */ |
667 | if (last_dst) |
668 | bq_enqueue(dev: last_dst->dev, xdpf, dev_rx, xdp_prog: last_dst->xdp_prog); |
669 | else |
670 | xdp_return_frame_rx_napi(xdpf); /* dtab is empty */ |
671 | |
672 | return 0; |
673 | } |
674 | |
675 | int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, |
676 | struct bpf_prog *xdp_prog) |
677 | { |
678 | int err; |
679 | |
680 | err = xdp_ok_fwd_dev(fwd: dst->dev, pktlen: skb->len); |
681 | if (unlikely(err)) |
682 | return err; |
683 | |
684 | /* Redirect has already succeeded semantically at this point, so we just |
685 | * return 0 even if packet is dropped. Helper below takes care of |
686 | * freeing skb. |
687 | */ |
688 | if (dev_map_bpf_prog_run_skb(skb, dst) != XDP_PASS) |
689 | return 0; |
690 | |
691 | skb->dev = dst->dev; |
692 | generic_xdp_tx(skb, xdp_prog); |
693 | |
694 | return 0; |
695 | } |
696 | |
697 | static int dev_map_redirect_clone(struct bpf_dtab_netdev *dst, |
698 | struct sk_buff *skb, |
699 | struct bpf_prog *xdp_prog) |
700 | { |
701 | struct sk_buff *nskb; |
702 | int err; |
703 | |
704 | nskb = skb_clone(skb, GFP_ATOMIC); |
705 | if (!nskb) |
706 | return -ENOMEM; |
707 | |
708 | err = dev_map_generic_redirect(dst, skb: nskb, xdp_prog); |
709 | if (unlikely(err)) { |
710 | consume_skb(skb: nskb); |
711 | return err; |
712 | } |
713 | |
714 | return 0; |
715 | } |
716 | |
717 | int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, |
718 | struct bpf_prog *xdp_prog, struct bpf_map *map, |
719 | bool exclude_ingress) |
720 | { |
721 | struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); |
722 | struct bpf_dtab_netdev *dst, *last_dst = NULL; |
723 | int excluded_devices[1+MAX_NEST_DEV]; |
724 | struct hlist_head *head; |
725 | struct hlist_node *next; |
726 | int num_excluded = 0; |
727 | unsigned int i; |
728 | int err; |
729 | |
730 | if (exclude_ingress) { |
731 | num_excluded = get_upper_ifindexes(dev, indexes: excluded_devices); |
732 | excluded_devices[num_excluded++] = dev->ifindex; |
733 | } |
734 | |
735 | if (map->map_type == BPF_MAP_TYPE_DEVMAP) { |
736 | for (i = 0; i < map->max_entries; i++) { |
737 | dst = rcu_dereference_check(dtab->netdev_map[i], |
738 | rcu_read_lock_bh_held()); |
739 | if (!dst) |
740 | continue; |
741 | |
742 | if (is_ifindex_excluded(excluded: excluded_devices, num_excluded, ifindex: dst->dev->ifindex)) |
743 | continue; |
744 | |
745 | /* we only need n-1 clones; last_dst enqueued below */ |
746 | if (!last_dst) { |
747 | last_dst = dst; |
748 | continue; |
749 | } |
750 | |
751 | err = dev_map_redirect_clone(dst: last_dst, skb, xdp_prog); |
752 | if (err) |
753 | return err; |
754 | |
755 | last_dst = dst; |
756 | |
757 | } |
758 | } else { /* BPF_MAP_TYPE_DEVMAP_HASH */ |
759 | for (i = 0; i < dtab->n_buckets; i++) { |
760 | head = dev_map_index_hash(dtab, idx: i); |
761 | hlist_for_each_entry_safe(dst, next, head, index_hlist) { |
762 | if (!dst) |
763 | continue; |
764 | |
765 | if (is_ifindex_excluded(excluded: excluded_devices, num_excluded, |
766 | ifindex: dst->dev->ifindex)) |
767 | continue; |
768 | |
769 | /* we only need n-1 clones; last_dst enqueued below */ |
770 | if (!last_dst) { |
771 | last_dst = dst; |
772 | continue; |
773 | } |
774 | |
775 | err = dev_map_redirect_clone(dst: last_dst, skb, xdp_prog); |
776 | if (err) |
777 | return err; |
778 | |
779 | last_dst = dst; |
780 | } |
781 | } |
782 | } |
783 | |
784 | /* consume the first skb and return */ |
785 | if (last_dst) |
786 | return dev_map_generic_redirect(dst: last_dst, skb, xdp_prog); |
787 | |
788 | /* dtab is empty */ |
789 | consume_skb(skb); |
790 | return 0; |
791 | } |
792 | |
793 | static void *dev_map_lookup_elem(struct bpf_map *map, void *key) |
794 | { |
795 | struct bpf_dtab_netdev *obj = __dev_map_lookup_elem(map, key: *(u32 *)key); |
796 | |
797 | return obj ? &obj->val : NULL; |
798 | } |
799 | |
800 | static void *dev_map_hash_lookup_elem(struct bpf_map *map, void *key) |
801 | { |
802 | struct bpf_dtab_netdev *obj = __dev_map_hash_lookup_elem(map, |
803 | key: *(u32 *)key); |
804 | return obj ? &obj->val : NULL; |
805 | } |
806 | |
807 | static void __dev_map_entry_free(struct rcu_head *rcu) |
808 | { |
809 | struct bpf_dtab_netdev *dev; |
810 | |
811 | dev = container_of(rcu, struct bpf_dtab_netdev, rcu); |
812 | if (dev->xdp_prog) |
813 | bpf_prog_put(prog: dev->xdp_prog); |
814 | dev_put(dev: dev->dev); |
815 | kfree(objp: dev); |
816 | } |
817 | |
818 | static long dev_map_delete_elem(struct bpf_map *map, void *key) |
819 | { |
820 | struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); |
821 | struct bpf_dtab_netdev *old_dev; |
822 | int k = *(u32 *)key; |
823 | |
824 | if (k >= map->max_entries) |
825 | return -EINVAL; |
826 | |
827 | old_dev = unrcu_pointer(xchg(&dtab->netdev_map[k], NULL)); |
828 | if (old_dev) { |
829 | call_rcu(head: &old_dev->rcu, func: __dev_map_entry_free); |
830 | atomic_dec(v: (atomic_t *)&dtab->items); |
831 | } |
832 | return 0; |
833 | } |
834 | |
835 | static long dev_map_hash_delete_elem(struct bpf_map *map, void *key) |
836 | { |
837 | struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); |
838 | struct bpf_dtab_netdev *old_dev; |
839 | int k = *(u32 *)key; |
840 | unsigned long flags; |
841 | int ret = -ENOENT; |
842 | |
843 | spin_lock_irqsave(&dtab->index_lock, flags); |
844 | |
845 | old_dev = __dev_map_hash_lookup_elem(map, key: k); |
846 | if (old_dev) { |
847 | dtab->items--; |
848 | hlist_del_init_rcu(n: &old_dev->index_hlist); |
849 | call_rcu(head: &old_dev->rcu, func: __dev_map_entry_free); |
850 | ret = 0; |
851 | } |
852 | spin_unlock_irqrestore(lock: &dtab->index_lock, flags); |
853 | |
854 | return ret; |
855 | } |
856 | |
857 | static struct bpf_dtab_netdev *__dev_map_alloc_node(struct net *net, |
858 | struct bpf_dtab *dtab, |
859 | struct bpf_devmap_val *val, |
860 | unsigned int idx) |
861 | { |
862 | struct bpf_prog *prog = NULL; |
863 | struct bpf_dtab_netdev *dev; |
864 | |
865 | dev = bpf_map_kmalloc_node(map: &dtab->map, size: sizeof(*dev), |
866 | GFP_NOWAIT | __GFP_NOWARN, |
867 | node: dtab->map.numa_node); |
868 | if (!dev) |
869 | return ERR_PTR(error: -ENOMEM); |
870 | |
871 | dev->dev = dev_get_by_index(net, ifindex: val->ifindex); |
872 | if (!dev->dev) |
873 | goto err_out; |
874 | |
875 | if (val->bpf_prog.fd > 0) { |
876 | prog = bpf_prog_get_type_dev(ufd: val->bpf_prog.fd, |
877 | type: BPF_PROG_TYPE_XDP, attach_drv: false); |
878 | if (IS_ERR(ptr: prog)) |
879 | goto err_put_dev; |
880 | if (prog->expected_attach_type != BPF_XDP_DEVMAP || |
881 | !bpf_prog_map_compatible(map: &dtab->map, fp: prog)) |
882 | goto err_put_prog; |
883 | } |
884 | |
885 | dev->idx = idx; |
886 | if (prog) { |
887 | dev->xdp_prog = prog; |
888 | dev->val.bpf_prog.id = prog->aux->id; |
889 | } else { |
890 | dev->xdp_prog = NULL; |
891 | dev->val.bpf_prog.id = 0; |
892 | } |
893 | dev->val.ifindex = val->ifindex; |
894 | |
895 | return dev; |
896 | err_put_prog: |
897 | bpf_prog_put(prog); |
898 | err_put_dev: |
899 | dev_put(dev: dev->dev); |
900 | err_out: |
901 | kfree(objp: dev); |
902 | return ERR_PTR(error: -EINVAL); |
903 | } |
904 | |
905 | static long __dev_map_update_elem(struct net *net, struct bpf_map *map, |
906 | void *key, void *value, u64 map_flags) |
907 | { |
908 | struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); |
909 | struct bpf_dtab_netdev *dev, *old_dev; |
910 | struct bpf_devmap_val val = {}; |
911 | u32 i = *(u32 *)key; |
912 | |
913 | if (unlikely(map_flags > BPF_EXIST)) |
914 | return -EINVAL; |
915 | if (unlikely(i >= dtab->map.max_entries)) |
916 | return -E2BIG; |
917 | if (unlikely(map_flags == BPF_NOEXIST)) |
918 | return -EEXIST; |
919 | |
920 | /* already verified value_size <= sizeof val */ |
921 | memcpy(&val, value, map->value_size); |
922 | |
923 | if (!val.ifindex) { |
924 | dev = NULL; |
925 | /* can not specify fd if ifindex is 0 */ |
926 | if (val.bpf_prog.fd > 0) |
927 | return -EINVAL; |
928 | } else { |
929 | dev = __dev_map_alloc_node(net, dtab, val: &val, idx: i); |
930 | if (IS_ERR(ptr: dev)) |
931 | return PTR_ERR(ptr: dev); |
932 | } |
933 | |
934 | /* Use call_rcu() here to ensure rcu critical sections have completed |
935 | * Remembering the driver side flush operation will happen before the |
936 | * net device is removed. |
937 | */ |
938 | old_dev = unrcu_pointer(xchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev))); |
939 | if (old_dev) |
940 | call_rcu(head: &old_dev->rcu, func: __dev_map_entry_free); |
941 | else |
942 | atomic_inc(v: (atomic_t *)&dtab->items); |
943 | |
944 | return 0; |
945 | } |
946 | |
947 | static long dev_map_update_elem(struct bpf_map *map, void *key, void *value, |
948 | u64 map_flags) |
949 | { |
950 | return __dev_map_update_elem(current->nsproxy->net_ns, |
951 | map, key, value, map_flags); |
952 | } |
953 | |
954 | static long __dev_map_hash_update_elem(struct net *net, struct bpf_map *map, |
955 | void *key, void *value, u64 map_flags) |
956 | { |
957 | struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); |
958 | struct bpf_dtab_netdev *dev, *old_dev; |
959 | struct bpf_devmap_val val = {}; |
960 | u32 idx = *(u32 *)key; |
961 | unsigned long flags; |
962 | int err = -EEXIST; |
963 | |
964 | /* already verified value_size <= sizeof val */ |
965 | memcpy(&val, value, map->value_size); |
966 | |
967 | if (unlikely(map_flags > BPF_EXIST || !val.ifindex)) |
968 | return -EINVAL; |
969 | |
970 | spin_lock_irqsave(&dtab->index_lock, flags); |
971 | |
972 | old_dev = __dev_map_hash_lookup_elem(map, key: idx); |
973 | if (old_dev && (map_flags & BPF_NOEXIST)) |
974 | goto out_err; |
975 | |
976 | dev = __dev_map_alloc_node(net, dtab, val: &val, idx); |
977 | if (IS_ERR(ptr: dev)) { |
978 | err = PTR_ERR(ptr: dev); |
979 | goto out_err; |
980 | } |
981 | |
982 | if (old_dev) { |
983 | hlist_del_rcu(n: &old_dev->index_hlist); |
984 | } else { |
985 | if (dtab->items >= dtab->map.max_entries) { |
986 | spin_unlock_irqrestore(lock: &dtab->index_lock, flags); |
987 | call_rcu(head: &dev->rcu, func: __dev_map_entry_free); |
988 | return -E2BIG; |
989 | } |
990 | dtab->items++; |
991 | } |
992 | |
993 | hlist_add_head_rcu(n: &dev->index_hlist, |
994 | h: dev_map_index_hash(dtab, idx)); |
995 | spin_unlock_irqrestore(lock: &dtab->index_lock, flags); |
996 | |
997 | if (old_dev) |
998 | call_rcu(head: &old_dev->rcu, func: __dev_map_entry_free); |
999 | |
1000 | return 0; |
1001 | |
1002 | out_err: |
1003 | spin_unlock_irqrestore(lock: &dtab->index_lock, flags); |
1004 | return err; |
1005 | } |
1006 | |
1007 | static long dev_map_hash_update_elem(struct bpf_map *map, void *key, void *value, |
1008 | u64 map_flags) |
1009 | { |
1010 | return __dev_map_hash_update_elem(current->nsproxy->net_ns, |
1011 | map, key, value, map_flags); |
1012 | } |
1013 | |
1014 | static long dev_map_redirect(struct bpf_map *map, u64 ifindex, u64 flags) |
1015 | { |
1016 | return __bpf_xdp_redirect_map(map, index: ifindex, flags, |
1017 | flag_mask: BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS, |
1018 | lookup_elem: __dev_map_lookup_elem); |
1019 | } |
1020 | |
1021 | static long dev_hash_map_redirect(struct bpf_map *map, u64 ifindex, u64 flags) |
1022 | { |
1023 | return __bpf_xdp_redirect_map(map, index: ifindex, flags, |
1024 | flag_mask: BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS, |
1025 | lookup_elem: __dev_map_hash_lookup_elem); |
1026 | } |
1027 | |
1028 | static u64 dev_map_mem_usage(const struct bpf_map *map) |
1029 | { |
1030 | struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); |
1031 | u64 usage = sizeof(struct bpf_dtab); |
1032 | |
1033 | if (map->map_type == BPF_MAP_TYPE_DEVMAP_HASH) |
1034 | usage += (u64)dtab->n_buckets * sizeof(struct hlist_head); |
1035 | else |
1036 | usage += (u64)map->max_entries * sizeof(struct bpf_dtab_netdev *); |
1037 | usage += atomic_read(v: (atomic_t *)&dtab->items) * |
1038 | (u64)sizeof(struct bpf_dtab_netdev); |
1039 | return usage; |
1040 | } |
1041 | |
1042 | BTF_ID_LIST_SINGLE(dev_map_btf_ids, struct, bpf_dtab) |
1043 | const struct bpf_map_ops dev_map_ops = { |
1044 | .map_meta_equal = bpf_map_meta_equal, |
1045 | .map_alloc = dev_map_alloc, |
1046 | .map_free = dev_map_free, |
1047 | .map_get_next_key = dev_map_get_next_key, |
1048 | .map_lookup_elem = dev_map_lookup_elem, |
1049 | .map_update_elem = dev_map_update_elem, |
1050 | .map_delete_elem = dev_map_delete_elem, |
1051 | .map_check_btf = map_check_no_btf, |
1052 | .map_mem_usage = dev_map_mem_usage, |
1053 | .map_btf_id = &dev_map_btf_ids[0], |
1054 | .map_redirect = dev_map_redirect, |
1055 | }; |
1056 | |
1057 | const struct bpf_map_ops dev_map_hash_ops = { |
1058 | .map_meta_equal = bpf_map_meta_equal, |
1059 | .map_alloc = dev_map_alloc, |
1060 | .map_free = dev_map_free, |
1061 | .map_get_next_key = dev_map_hash_get_next_key, |
1062 | .map_lookup_elem = dev_map_hash_lookup_elem, |
1063 | .map_update_elem = dev_map_hash_update_elem, |
1064 | .map_delete_elem = dev_map_hash_delete_elem, |
1065 | .map_check_btf = map_check_no_btf, |
1066 | .map_mem_usage = dev_map_mem_usage, |
1067 | .map_btf_id = &dev_map_btf_ids[0], |
1068 | .map_redirect = dev_hash_map_redirect, |
1069 | }; |
1070 | |
1071 | static void dev_map_hash_remove_netdev(struct bpf_dtab *dtab, |
1072 | struct net_device *netdev) |
1073 | { |
1074 | unsigned long flags; |
1075 | u32 i; |
1076 | |
1077 | spin_lock_irqsave(&dtab->index_lock, flags); |
1078 | for (i = 0; i < dtab->n_buckets; i++) { |
1079 | struct bpf_dtab_netdev *dev; |
1080 | struct hlist_head *head; |
1081 | struct hlist_node *next; |
1082 | |
1083 | head = dev_map_index_hash(dtab, idx: i); |
1084 | |
1085 | hlist_for_each_entry_safe(dev, next, head, index_hlist) { |
1086 | if (netdev != dev->dev) |
1087 | continue; |
1088 | |
1089 | dtab->items--; |
1090 | hlist_del_rcu(n: &dev->index_hlist); |
1091 | call_rcu(head: &dev->rcu, func: __dev_map_entry_free); |
1092 | } |
1093 | } |
1094 | spin_unlock_irqrestore(lock: &dtab->index_lock, flags); |
1095 | } |
1096 | |
1097 | static int dev_map_notification(struct notifier_block *notifier, |
1098 | ulong event, void *ptr) |
1099 | { |
1100 | struct net_device *netdev = netdev_notifier_info_to_dev(info: ptr); |
1101 | struct bpf_dtab *dtab; |
1102 | int i, cpu; |
1103 | |
1104 | switch (event) { |
1105 | case NETDEV_REGISTER: |
1106 | if (!netdev->netdev_ops->ndo_xdp_xmit || netdev->xdp_bulkq) |
1107 | break; |
1108 | |
1109 | /* will be freed in free_netdev() */ |
1110 | netdev->xdp_bulkq = alloc_percpu(struct xdp_dev_bulk_queue); |
1111 | if (!netdev->xdp_bulkq) |
1112 | return NOTIFY_BAD; |
1113 | |
1114 | for_each_possible_cpu(cpu) |
1115 | per_cpu_ptr(netdev->xdp_bulkq, cpu)->dev = netdev; |
1116 | break; |
1117 | case NETDEV_UNREGISTER: |
1118 | /* This rcu_read_lock/unlock pair is needed because |
1119 | * dev_map_list is an RCU list AND to ensure a delete |
1120 | * operation does not free a netdev_map entry while we |
1121 | * are comparing it against the netdev being unregistered. |
1122 | */ |
1123 | rcu_read_lock(); |
1124 | list_for_each_entry_rcu(dtab, &dev_map_list, list) { |
1125 | if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) { |
1126 | dev_map_hash_remove_netdev(dtab, netdev); |
1127 | continue; |
1128 | } |
1129 | |
1130 | for (i = 0; i < dtab->map.max_entries; i++) { |
1131 | struct bpf_dtab_netdev *dev, *odev; |
1132 | |
1133 | dev = rcu_dereference(dtab->netdev_map[i]); |
1134 | if (!dev || netdev != dev->dev) |
1135 | continue; |
1136 | odev = unrcu_pointer(cmpxchg(&dtab->netdev_map[i], RCU_INITIALIZER(dev), NULL)); |
1137 | if (dev == odev) { |
1138 | call_rcu(head: &dev->rcu, |
1139 | func: __dev_map_entry_free); |
1140 | atomic_dec(v: (atomic_t *)&dtab->items); |
1141 | } |
1142 | } |
1143 | } |
1144 | rcu_read_unlock(); |
1145 | break; |
1146 | default: |
1147 | break; |
1148 | } |
1149 | return NOTIFY_OK; |
1150 | } |
1151 | |
1152 | static struct notifier_block dev_map_notifier = { |
1153 | .notifier_call = dev_map_notification, |
1154 | }; |
1155 | |
1156 | static int __init dev_map_init(void) |
1157 | { |
1158 | int cpu; |
1159 | |
1160 | /* Assure tracepoint shadow struct _bpf_dtab_netdev is in sync */ |
1161 | BUILD_BUG_ON(offsetof(struct bpf_dtab_netdev, dev) != |
1162 | offsetof(struct _bpf_dtab_netdev, dev)); |
1163 | register_netdevice_notifier(nb: &dev_map_notifier); |
1164 | |
1165 | for_each_possible_cpu(cpu) |
1166 | INIT_LIST_HEAD(list: &per_cpu(dev_flush_list, cpu)); |
1167 | return 0; |
1168 | } |
1169 | |
1170 | subsys_initcall(dev_map_init); |
1171 | |