1 | /* Broadcom NetXtreme-C/E network driver. |
2 | * |
3 | * Copyright (c) 2016-2017 Broadcom Limited |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify |
6 | * it under the terms of the GNU General Public License as published by |
7 | * the Free Software Foundation. |
8 | */ |
9 | #include <linux/kernel.h> |
10 | #include <linux/errno.h> |
11 | #include <linux/pci.h> |
12 | #include <linux/netdevice.h> |
13 | #include <linux/etherdevice.h> |
14 | #include <linux/if_vlan.h> |
15 | #include <linux/bpf.h> |
16 | #include <linux/bpf_trace.h> |
17 | #include <linux/filter.h> |
18 | #include <net/page_pool/helpers.h> |
19 | #include "bnxt_hsi.h" |
20 | #include "bnxt.h" |
21 | #include "bnxt_xdp.h" |
22 | |
23 | DEFINE_STATIC_KEY_FALSE(bnxt_xdp_locking_key); |
24 | |
25 | struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp, |
26 | struct bnxt_tx_ring_info *txr, |
27 | dma_addr_t mapping, u32 len, |
28 | struct xdp_buff *xdp) |
29 | { |
30 | struct skb_shared_info *sinfo; |
31 | struct bnxt_sw_tx_bd *tx_buf; |
32 | struct tx_bd *txbd; |
33 | int num_frags = 0; |
34 | u32 flags; |
35 | u16 prod; |
36 | int i; |
37 | |
38 | if (xdp && xdp_buff_has_frags(xdp)) { |
39 | sinfo = xdp_get_shared_info_from_buff(xdp); |
40 | num_frags = sinfo->nr_frags; |
41 | } |
42 | |
43 | /* fill up the first buffer */ |
44 | prod = txr->tx_prod; |
45 | tx_buf = &txr->tx_buf_ring[prod]; |
46 | tx_buf->nr_frags = num_frags; |
47 | if (xdp) |
48 | tx_buf->page = virt_to_head_page(x: xdp->data); |
49 | |
50 | txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; |
51 | flags = (len << TX_BD_LEN_SHIFT) | |
52 | ((num_frags + 1) << TX_BD_FLAGS_BD_CNT_SHIFT) | |
53 | bnxt_lhint_arr[len >> 9]; |
54 | txbd->tx_bd_len_flags_type = cpu_to_le32(flags); |
55 | txbd->tx_bd_opaque = prod; |
56 | txbd->tx_bd_haddr = cpu_to_le64(mapping); |
57 | |
58 | /* now let us fill up the frags into the next buffers */ |
59 | for (i = 0; i < num_frags ; i++) { |
60 | skb_frag_t *frag = &sinfo->frags[i]; |
61 | struct bnxt_sw_tx_bd *frag_tx_buf; |
62 | struct pci_dev *pdev = bp->pdev; |
63 | dma_addr_t frag_mapping; |
64 | int frag_len; |
65 | |
66 | prod = NEXT_TX(prod); |
67 | WRITE_ONCE(txr->tx_prod, prod); |
68 | |
69 | /* first fill up the first buffer */ |
70 | frag_tx_buf = &txr->tx_buf_ring[prod]; |
71 | frag_tx_buf->page = skb_frag_page(frag); |
72 | |
73 | txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; |
74 | |
75 | frag_len = skb_frag_size(frag); |
76 | frag_mapping = skb_frag_dma_map(dev: &pdev->dev, frag, offset: 0, |
77 | size: frag_len, dir: DMA_TO_DEVICE); |
78 | |
79 | if (unlikely(dma_mapping_error(&pdev->dev, frag_mapping))) |
80 | return NULL; |
81 | |
82 | dma_unmap_addr_set(frag_tx_buf, mapping, frag_mapping); |
83 | |
84 | flags = frag_len << TX_BD_LEN_SHIFT; |
85 | txbd->tx_bd_len_flags_type = cpu_to_le32(flags); |
86 | txbd->tx_bd_haddr = cpu_to_le64(frag_mapping); |
87 | |
88 | len = frag_len; |
89 | } |
90 | |
91 | flags &= ~TX_BD_LEN; |
92 | txbd->tx_bd_len_flags_type = cpu_to_le32(((len) << TX_BD_LEN_SHIFT) | flags | |
93 | TX_BD_FLAGS_PACKET_END); |
94 | /* Sync TX BD */ |
95 | wmb(); |
96 | prod = NEXT_TX(prod); |
97 | WRITE_ONCE(txr->tx_prod, prod); |
98 | |
99 | return tx_buf; |
100 | } |
101 | |
102 | static void __bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, |
103 | dma_addr_t mapping, u32 len, u16 rx_prod, |
104 | struct xdp_buff *xdp) |
105 | { |
106 | struct bnxt_sw_tx_bd *tx_buf; |
107 | |
108 | tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp); |
109 | tx_buf->rx_prod = rx_prod; |
110 | tx_buf->action = XDP_TX; |
111 | |
112 | } |
113 | |
114 | static void __bnxt_xmit_xdp_redirect(struct bnxt *bp, |
115 | struct bnxt_tx_ring_info *txr, |
116 | dma_addr_t mapping, u32 len, |
117 | struct xdp_frame *xdpf) |
118 | { |
119 | struct bnxt_sw_tx_bd *tx_buf; |
120 | |
121 | tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, NULL); |
122 | tx_buf->action = XDP_REDIRECT; |
123 | tx_buf->xdpf = xdpf; |
124 | dma_unmap_addr_set(tx_buf, mapping, mapping); |
125 | dma_unmap_len_set(tx_buf, len, 0); |
126 | } |
127 | |
128 | void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) |
129 | { |
130 | struct bnxt_tx_ring_info *txr = bnapi->tx_ring; |
131 | struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; |
132 | bool rx_doorbell_needed = false; |
133 | int nr_pkts = bnapi->tx_pkts; |
134 | struct bnxt_sw_tx_bd *tx_buf; |
135 | u16 tx_cons = txr->tx_cons; |
136 | u16 last_tx_cons = tx_cons; |
137 | int i, j, frags; |
138 | |
139 | if (!budget) |
140 | return; |
141 | |
142 | for (i = 0; i < nr_pkts; i++) { |
143 | tx_buf = &txr->tx_buf_ring[tx_cons]; |
144 | |
145 | if (tx_buf->action == XDP_REDIRECT) { |
146 | struct pci_dev *pdev = bp->pdev; |
147 | |
148 | dma_unmap_single(&pdev->dev, |
149 | dma_unmap_addr(tx_buf, mapping), |
150 | dma_unmap_len(tx_buf, len), |
151 | DMA_TO_DEVICE); |
152 | xdp_return_frame(xdpf: tx_buf->xdpf); |
153 | tx_buf->action = 0; |
154 | tx_buf->xdpf = NULL; |
155 | } else if (tx_buf->action == XDP_TX) { |
156 | tx_buf->action = 0; |
157 | rx_doorbell_needed = true; |
158 | last_tx_cons = tx_cons; |
159 | |
160 | frags = tx_buf->nr_frags; |
161 | for (j = 0; j < frags; j++) { |
162 | tx_cons = NEXT_TX(tx_cons); |
163 | tx_buf = &txr->tx_buf_ring[tx_cons]; |
164 | page_pool_recycle_direct(pool: rxr->page_pool, page: tx_buf->page); |
165 | } |
166 | } else { |
167 | bnxt_sched_reset_txr(bp, txr, idx: i); |
168 | return; |
169 | } |
170 | tx_cons = NEXT_TX(tx_cons); |
171 | } |
172 | |
173 | bnapi->tx_pkts = 0; |
174 | WRITE_ONCE(txr->tx_cons, tx_cons); |
175 | if (rx_doorbell_needed) { |
176 | tx_buf = &txr->tx_buf_ring[last_tx_cons]; |
177 | bnxt_db_write(bp, db: &rxr->rx_db, idx: tx_buf->rx_prod); |
178 | |
179 | } |
180 | } |
181 | |
182 | bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) |
183 | { |
184 | struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); |
185 | |
186 | return !!xdp_prog; |
187 | } |
188 | |
189 | void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, |
190 | u16 cons, u8 *data_ptr, unsigned int len, |
191 | struct xdp_buff *xdp) |
192 | { |
193 | u32 buflen = BNXT_RX_PAGE_SIZE; |
194 | struct bnxt_sw_rx_bd *rx_buf; |
195 | struct pci_dev *pdev; |
196 | dma_addr_t mapping; |
197 | u32 offset; |
198 | |
199 | pdev = bp->pdev; |
200 | rx_buf = &rxr->rx_buf_ring[cons]; |
201 | offset = bp->rx_offset; |
202 | |
203 | mapping = rx_buf->mapping - bp->rx_dma_offset; |
204 | dma_sync_single_for_cpu(dev: &pdev->dev, addr: mapping + offset, size: len, dir: bp->rx_dir); |
205 | |
206 | xdp_init_buff(xdp, frame_sz: buflen, rxq: &rxr->xdp_rxq); |
207 | xdp_prepare_buff(xdp, hard_start: data_ptr - offset, headroom: offset, data_len: len, meta_valid: false); |
208 | } |
209 | |
210 | void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr, |
211 | struct xdp_buff *xdp) |
212 | { |
213 | struct skb_shared_info *shinfo; |
214 | int i; |
215 | |
216 | if (!xdp || !xdp_buff_has_frags(xdp)) |
217 | return; |
218 | shinfo = xdp_get_shared_info_from_buff(xdp); |
219 | for (i = 0; i < shinfo->nr_frags; i++) { |
220 | struct page *page = skb_frag_page(frag: &shinfo->frags[i]); |
221 | |
222 | page_pool_recycle_direct(pool: rxr->page_pool, page); |
223 | } |
224 | shinfo->nr_frags = 0; |
225 | } |
226 | |
227 | /* returns the following: |
228 | * true - packet consumed by XDP and new buffer is allocated. |
229 | * false - packet should be passed to the stack. |
230 | */ |
231 | bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, |
232 | struct xdp_buff xdp, struct page *page, u8 **data_ptr, |
233 | unsigned int *len, u8 *event) |
234 | { |
235 | struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); |
236 | struct bnxt_tx_ring_info *txr; |
237 | struct bnxt_sw_rx_bd *rx_buf; |
238 | struct pci_dev *pdev; |
239 | dma_addr_t mapping; |
240 | u32 tx_needed = 1; |
241 | void *orig_data; |
242 | u32 tx_avail; |
243 | u32 offset; |
244 | u32 act; |
245 | |
246 | if (!xdp_prog) |
247 | return false; |
248 | |
249 | pdev = bp->pdev; |
250 | offset = bp->rx_offset; |
251 | |
252 | txr = rxr->bnapi->tx_ring; |
253 | /* BNXT_RX_PAGE_MODE(bp) when XDP enabled */ |
254 | orig_data = xdp.data; |
255 | |
256 | act = bpf_prog_run_xdp(prog: xdp_prog, xdp: &xdp); |
257 | |
258 | tx_avail = bnxt_tx_avail(bp, txr); |
259 | /* If the tx ring is not full, we must not update the rx producer yet |
260 | * because we may still be transmitting on some BDs. |
261 | */ |
262 | if (tx_avail != bp->tx_ring_size) |
263 | *event &= ~BNXT_RX_EVENT; |
264 | |
265 | *len = xdp.data_end - xdp.data; |
266 | if (orig_data != xdp.data) { |
267 | offset = xdp.data - xdp.data_hard_start; |
268 | *data_ptr = xdp.data_hard_start + offset; |
269 | } |
270 | |
271 | switch (act) { |
272 | case XDP_PASS: |
273 | return false; |
274 | |
275 | case XDP_TX: |
276 | rx_buf = &rxr->rx_buf_ring[cons]; |
277 | mapping = rx_buf->mapping - bp->rx_dma_offset; |
278 | *event = 0; |
279 | |
280 | if (unlikely(xdp_buff_has_frags(&xdp))) { |
281 | struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp: &xdp); |
282 | |
283 | tx_needed += sinfo->nr_frags; |
284 | *event = BNXT_AGG_EVENT; |
285 | } |
286 | |
287 | if (tx_avail < tx_needed) { |
288 | trace_xdp_exception(dev: bp->dev, xdp: xdp_prog, act); |
289 | bnxt_xdp_buff_frags_free(rxr, xdp: &xdp); |
290 | bnxt_reuse_rx_data(rxr, cons, data: page); |
291 | return true; |
292 | } |
293 | |
294 | dma_sync_single_for_device(dev: &pdev->dev, addr: mapping + offset, size: *len, |
295 | dir: bp->rx_dir); |
296 | |
297 | *event |= BNXT_TX_EVENT; |
298 | __bnxt_xmit_xdp(bp, txr, mapping: mapping + offset, len: *len, |
299 | NEXT_RX(rxr->rx_prod), xdp: &xdp); |
300 | bnxt_reuse_rx_data(rxr, cons, data: page); |
301 | return true; |
302 | case XDP_REDIRECT: |
303 | /* if we are calling this here then we know that the |
304 | * redirect is coming from a frame received by the |
305 | * bnxt_en driver. |
306 | */ |
307 | rx_buf = &rxr->rx_buf_ring[cons]; |
308 | mapping = rx_buf->mapping - bp->rx_dma_offset; |
309 | dma_unmap_page_attrs(dev: &pdev->dev, addr: mapping, |
310 | BNXT_RX_PAGE_SIZE, dir: bp->rx_dir, |
311 | DMA_ATTR_WEAK_ORDERING); |
312 | |
313 | /* if we are unable to allocate a new buffer, abort and reuse */ |
314 | if (bnxt_alloc_rx_data(bp, rxr, prod: rxr->rx_prod, GFP_ATOMIC)) { |
315 | trace_xdp_exception(dev: bp->dev, xdp: xdp_prog, act); |
316 | bnxt_xdp_buff_frags_free(rxr, xdp: &xdp); |
317 | bnxt_reuse_rx_data(rxr, cons, data: page); |
318 | return true; |
319 | } |
320 | |
321 | if (xdp_do_redirect(dev: bp->dev, xdp: &xdp, prog: xdp_prog)) { |
322 | trace_xdp_exception(dev: bp->dev, xdp: xdp_prog, act); |
323 | page_pool_recycle_direct(pool: rxr->page_pool, page); |
324 | return true; |
325 | } |
326 | |
327 | *event |= BNXT_REDIRECT_EVENT; |
328 | break; |
329 | default: |
330 | bpf_warn_invalid_xdp_action(dev: bp->dev, prog: xdp_prog, act); |
331 | fallthrough; |
332 | case XDP_ABORTED: |
333 | trace_xdp_exception(dev: bp->dev, xdp: xdp_prog, act); |
334 | fallthrough; |
335 | case XDP_DROP: |
336 | bnxt_xdp_buff_frags_free(rxr, xdp: &xdp); |
337 | bnxt_reuse_rx_data(rxr, cons, data: page); |
338 | break; |
339 | } |
340 | return true; |
341 | } |
342 | |
343 | int bnxt_xdp_xmit(struct net_device *dev, int num_frames, |
344 | struct xdp_frame **frames, u32 flags) |
345 | { |
346 | struct bnxt *bp = netdev_priv(dev); |
347 | struct bpf_prog *xdp_prog = READ_ONCE(bp->xdp_prog); |
348 | struct pci_dev *pdev = bp->pdev; |
349 | struct bnxt_tx_ring_info *txr; |
350 | dma_addr_t mapping; |
351 | int nxmit = 0; |
352 | int ring; |
353 | int i; |
354 | |
355 | if (!test_bit(BNXT_STATE_OPEN, &bp->state) || |
356 | !bp->tx_nr_rings_xdp || |
357 | !xdp_prog) |
358 | return -EINVAL; |
359 | |
360 | ring = smp_processor_id() % bp->tx_nr_rings_xdp; |
361 | txr = &bp->tx_ring[ring]; |
362 | |
363 | if (READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING) |
364 | return -EINVAL; |
365 | |
366 | if (static_branch_unlikely(&bnxt_xdp_locking_key)) |
367 | spin_lock(lock: &txr->xdp_tx_lock); |
368 | |
369 | for (i = 0; i < num_frames; i++) { |
370 | struct xdp_frame *xdp = frames[i]; |
371 | |
372 | if (!bnxt_tx_avail(bp, txr)) |
373 | break; |
374 | |
375 | mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len, |
376 | DMA_TO_DEVICE); |
377 | |
378 | if (dma_mapping_error(dev: &pdev->dev, dma_addr: mapping)) |
379 | break; |
380 | |
381 | __bnxt_xmit_xdp_redirect(bp, txr, mapping, len: xdp->len, xdpf: xdp); |
382 | nxmit++; |
383 | } |
384 | |
385 | if (flags & XDP_XMIT_FLUSH) { |
386 | /* Sync BD data before updating doorbell */ |
387 | wmb(); |
388 | bnxt_db_write(bp, db: &txr->tx_db, idx: txr->tx_prod); |
389 | } |
390 | |
391 | if (static_branch_unlikely(&bnxt_xdp_locking_key)) |
392 | spin_unlock(lock: &txr->xdp_tx_lock); |
393 | |
394 | return nxmit; |
395 | } |
396 | |
397 | /* Under rtnl_lock */ |
398 | static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog) |
399 | { |
400 | struct net_device *dev = bp->dev; |
401 | int tx_xdp = 0, rc, tc; |
402 | struct bpf_prog *old; |
403 | |
404 | if (prog && !prog->aux->xdp_has_frags && |
405 | bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) { |
406 | netdev_warn(dev, format: "MTU %d larger than %d without XDP frag support.\n" , |
407 | bp->dev->mtu, BNXT_MAX_PAGE_MODE_MTU); |
408 | return -EOPNOTSUPP; |
409 | } |
410 | if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) { |
411 | netdev_warn(dev, format: "ethtool rx/tx channels must be combined to support XDP.\n" ); |
412 | return -EOPNOTSUPP; |
413 | } |
414 | if (prog) |
415 | tx_xdp = bp->rx_nr_rings; |
416 | |
417 | tc = netdev_get_num_tc(dev); |
418 | if (!tc) |
419 | tc = 1; |
420 | rc = bnxt_check_rings(bp, tx: bp->tx_nr_rings_per_tc, rx: bp->rx_nr_rings, |
421 | sh: true, tcs: tc, tx_xdp); |
422 | if (rc) { |
423 | netdev_warn(dev, format: "Unable to reserve enough TX rings to support XDP.\n" ); |
424 | return rc; |
425 | } |
426 | if (netif_running(dev)) |
427 | bnxt_close_nic(bp, true, false); |
428 | |
429 | old = xchg(&bp->xdp_prog, prog); |
430 | if (old) |
431 | bpf_prog_put(prog: old); |
432 | |
433 | if (prog) { |
434 | bnxt_set_rx_skb_mode(bp, page_mode: true); |
435 | xdp_features_set_redirect_target(dev, support_sg: true); |
436 | } else { |
437 | int rx, tx; |
438 | |
439 | xdp_features_clear_redirect_target(dev); |
440 | bnxt_set_rx_skb_mode(bp, page_mode: false); |
441 | bnxt_get_max_rings(bp, &rx, &tx, true); |
442 | if (rx > 1) { |
443 | bp->flags &= ~BNXT_FLAG_NO_AGG_RINGS; |
444 | bp->dev->hw_features |= NETIF_F_LRO; |
445 | } |
446 | } |
447 | bp->tx_nr_rings_xdp = tx_xdp; |
448 | bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp; |
449 | bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings); |
450 | bnxt_set_tpa_flags(bp); |
451 | bnxt_set_ring_params(bp); |
452 | |
453 | if (netif_running(dev)) |
454 | return bnxt_open_nic(bp, true, false); |
455 | |
456 | return 0; |
457 | } |
458 | |
459 | int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp) |
460 | { |
461 | struct bnxt *bp = netdev_priv(dev); |
462 | int rc; |
463 | |
464 | switch (xdp->command) { |
465 | case XDP_SETUP_PROG: |
466 | rc = bnxt_xdp_set(bp, prog: xdp->prog); |
467 | break; |
468 | default: |
469 | rc = -EINVAL; |
470 | break; |
471 | } |
472 | return rc; |
473 | } |
474 | |
475 | struct sk_buff * |
476 | bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags, |
477 | struct page_pool *pool, struct xdp_buff *xdp, |
478 | struct rx_cmp_ext *rxcmp1) |
479 | { |
480 | struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); |
481 | |
482 | if (!skb) |
483 | return NULL; |
484 | skb_checksum_none_assert(skb); |
485 | if (RX_CMP_L4_CS_OK(rxcmp1)) { |
486 | if (bp->dev->features & NETIF_F_RXCSUM) { |
487 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
488 | skb->csum_level = RX_CMP_ENCAP(rxcmp1); |
489 | } |
490 | } |
491 | xdp_update_skb_shared_info(skb, nr_frags: num_frags, |
492 | size: sinfo->xdp_frags_size, |
493 | BNXT_RX_PAGE_SIZE * sinfo->nr_frags, |
494 | pfmemalloc: xdp_buff_is_frag_pfmemalloc(xdp)); |
495 | return skb; |
496 | } |
497 | |