1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Atlantic Network Driver |
3 | * |
4 | * Copyright (C) 2014-2019 aQuantia Corporation |
5 | * Copyright (C) 2019-2020 Marvell International Ltd. |
6 | */ |
7 | |
8 | /* File aq_ring.c: Definition of functions for Rx/Tx rings. */ |
9 | |
10 | #include "aq_nic.h" |
11 | #include "aq_hw.h" |
12 | #include "aq_hw_utils.h" |
13 | #include "aq_ptp.h" |
14 | #include "aq_vec.h" |
15 | #include "aq_main.h" |
16 | |
17 | #include <net/xdp.h> |
18 | #include <linux/filter.h> |
19 | #include <linux/bpf_trace.h> |
20 | #include <linux/netdevice.h> |
21 | #include <linux/etherdevice.h> |
22 | |
23 | static void aq_get_rxpages_xdp(struct aq_ring_buff_s *buff, |
24 | struct xdp_buff *xdp) |
25 | { |
26 | struct skb_shared_info *sinfo; |
27 | int i; |
28 | |
29 | if (xdp_buff_has_frags(xdp)) { |
30 | sinfo = xdp_get_shared_info_from_buff(xdp); |
31 | |
32 | for (i = 0; i < sinfo->nr_frags; i++) { |
33 | skb_frag_t *frag = &sinfo->frags[i]; |
34 | |
35 | page_ref_inc(page: skb_frag_page(frag)); |
36 | } |
37 | } |
38 | page_ref_inc(page: buff->rxdata.page); |
39 | } |
40 | |
41 | static inline void aq_free_rxpage(struct aq_rxpage *rxpage, struct device *dev) |
42 | { |
43 | unsigned int len = PAGE_SIZE << rxpage->order; |
44 | |
45 | dma_unmap_page(dev, rxpage->daddr, len, DMA_FROM_DEVICE); |
46 | |
47 | /* Drop the ref for being in the ring. */ |
48 | __free_pages(page: rxpage->page, order: rxpage->order); |
49 | rxpage->page = NULL; |
50 | } |
51 | |
52 | static int aq_alloc_rxpages(struct aq_rxpage *rxpage, struct aq_ring_s *rx_ring) |
53 | { |
54 | struct device *dev = aq_nic_get_dev(self: rx_ring->aq_nic); |
55 | unsigned int order = rx_ring->page_order; |
56 | struct page *page; |
57 | int ret = -ENOMEM; |
58 | dma_addr_t daddr; |
59 | |
60 | page = dev_alloc_pages(order); |
61 | if (unlikely(!page)) |
62 | goto err_exit; |
63 | |
64 | daddr = dma_map_page(dev, page, 0, PAGE_SIZE << order, |
65 | DMA_FROM_DEVICE); |
66 | |
67 | if (unlikely(dma_mapping_error(dev, daddr))) |
68 | goto free_page; |
69 | |
70 | rxpage->page = page; |
71 | rxpage->daddr = daddr; |
72 | rxpage->order = order; |
73 | rxpage->pg_off = rx_ring->page_offset; |
74 | |
75 | return 0; |
76 | |
77 | free_page: |
78 | __free_pages(page, order); |
79 | |
80 | err_exit: |
81 | return ret; |
82 | } |
83 | |
84 | static int aq_get_rxpages(struct aq_ring_s *self, struct aq_ring_buff_s *rxbuf) |
85 | { |
86 | unsigned int order = self->page_order; |
87 | u16 page_offset = self->page_offset; |
88 | u16 frame_max = self->frame_max; |
89 | u16 tail_size = self->tail_size; |
90 | int ret; |
91 | |
92 | if (rxbuf->rxdata.page) { |
93 | /* One means ring is the only user and can reuse */ |
94 | if (page_ref_count(page: rxbuf->rxdata.page) > 1) { |
95 | /* Try reuse buffer */ |
96 | rxbuf->rxdata.pg_off += frame_max + page_offset + |
97 | tail_size; |
98 | if (rxbuf->rxdata.pg_off + frame_max + tail_size <= |
99 | (PAGE_SIZE << order)) { |
100 | u64_stats_update_begin(syncp: &self->stats.rx.syncp); |
101 | self->stats.rx.pg_flips++; |
102 | u64_stats_update_end(syncp: &self->stats.rx.syncp); |
103 | |
104 | } else { |
105 | /* Buffer exhausted. We have other users and |
106 | * should release this page and realloc |
107 | */ |
108 | aq_free_rxpage(rxpage: &rxbuf->rxdata, |
109 | dev: aq_nic_get_dev(self: self->aq_nic)); |
110 | u64_stats_update_begin(syncp: &self->stats.rx.syncp); |
111 | self->stats.rx.pg_losts++; |
112 | u64_stats_update_end(syncp: &self->stats.rx.syncp); |
113 | } |
114 | } else { |
115 | rxbuf->rxdata.pg_off = page_offset; |
116 | u64_stats_update_begin(syncp: &self->stats.rx.syncp); |
117 | self->stats.rx.pg_reuses++; |
118 | u64_stats_update_end(syncp: &self->stats.rx.syncp); |
119 | } |
120 | } |
121 | |
122 | if (!rxbuf->rxdata.page) { |
123 | ret = aq_alloc_rxpages(rxpage: &rxbuf->rxdata, rx_ring: self); |
124 | if (ret) { |
125 | u64_stats_update_begin(syncp: &self->stats.rx.syncp); |
126 | self->stats.rx.alloc_fails++; |
127 | u64_stats_update_end(syncp: &self->stats.rx.syncp); |
128 | } |
129 | return ret; |
130 | } |
131 | |
132 | return 0; |
133 | } |
134 | |
135 | static int aq_ring_alloc(struct aq_ring_s *self, |
136 | struct aq_nic_s *aq_nic) |
137 | { |
138 | int err = 0; |
139 | |
140 | self->buff_ring = |
141 | kcalloc(n: self->size, size: sizeof(struct aq_ring_buff_s), GFP_KERNEL); |
142 | |
143 | if (!self->buff_ring) { |
144 | err = -ENOMEM; |
145 | goto err_exit; |
146 | } |
147 | |
148 | self->dx_ring = dma_alloc_coherent(dev: aq_nic_get_dev(self: aq_nic), |
149 | size: self->size * self->dx_size, |
150 | dma_handle: &self->dx_ring_pa, GFP_KERNEL); |
151 | if (!self->dx_ring) { |
152 | err = -ENOMEM; |
153 | goto err_exit; |
154 | } |
155 | |
156 | err_exit: |
157 | if (err < 0) { |
158 | aq_ring_free(self); |
159 | } |
160 | |
161 | return err; |
162 | } |
163 | |
164 | int aq_ring_tx_alloc(struct aq_ring_s *self, |
165 | struct aq_nic_s *aq_nic, |
166 | unsigned int idx, |
167 | struct aq_nic_cfg_s *aq_nic_cfg) |
168 | { |
169 | self->aq_nic = aq_nic; |
170 | self->idx = idx; |
171 | self->size = aq_nic_cfg->txds; |
172 | self->dx_size = aq_nic_cfg->aq_hw_caps->txd_size; |
173 | |
174 | return aq_ring_alloc(self, aq_nic); |
175 | } |
176 | |
177 | int aq_ring_rx_alloc(struct aq_ring_s *self, |
178 | struct aq_nic_s *aq_nic, |
179 | unsigned int idx, |
180 | struct aq_nic_cfg_s *aq_nic_cfg) |
181 | { |
182 | self->aq_nic = aq_nic; |
183 | self->idx = idx; |
184 | self->size = aq_nic_cfg->rxds; |
185 | self->dx_size = aq_nic_cfg->aq_hw_caps->rxd_size; |
186 | self->xdp_prog = aq_nic->xdp_prog; |
187 | self->frame_max = AQ_CFG_RX_FRAME_MAX; |
188 | |
189 | /* Only order-2 is allowed if XDP is enabled */ |
190 | if (READ_ONCE(self->xdp_prog)) { |
191 | self->page_offset = AQ_XDP_HEADROOM; |
192 | self->page_order = AQ_CFG_XDP_PAGEORDER; |
193 | self->tail_size = AQ_XDP_TAILROOM; |
194 | } else { |
195 | self->page_offset = 0; |
196 | self->page_order = fls(x: self->frame_max / PAGE_SIZE + |
197 | (self->frame_max % PAGE_SIZE ? 1 : 0)) - 1; |
198 | if (aq_nic_cfg->rxpageorder > self->page_order) |
199 | self->page_order = aq_nic_cfg->rxpageorder; |
200 | self->tail_size = 0; |
201 | } |
202 | |
203 | return aq_ring_alloc(self, aq_nic); |
204 | } |
205 | |
206 | int |
207 | aq_ring_hwts_rx_alloc(struct aq_ring_s *self, struct aq_nic_s *aq_nic, |
208 | unsigned int idx, unsigned int size, unsigned int dx_size) |
209 | { |
210 | struct device *dev = aq_nic_get_dev(self: aq_nic); |
211 | size_t sz = size * dx_size + AQ_CFG_RXDS_DEF; |
212 | |
213 | memset(self, 0, sizeof(*self)); |
214 | |
215 | self->aq_nic = aq_nic; |
216 | self->idx = idx; |
217 | self->size = size; |
218 | self->dx_size = dx_size; |
219 | |
220 | self->dx_ring = dma_alloc_coherent(dev, size: sz, dma_handle: &self->dx_ring_pa, |
221 | GFP_KERNEL); |
222 | if (!self->dx_ring) { |
223 | aq_ring_free(self); |
224 | return -ENOMEM; |
225 | } |
226 | |
227 | return 0; |
228 | } |
229 | |
230 | int aq_ring_init(struct aq_ring_s *self, const enum atl_ring_type ring_type) |
231 | { |
232 | self->hw_head = 0; |
233 | self->sw_head = 0; |
234 | self->sw_tail = 0; |
235 | self->ring_type = ring_type; |
236 | |
237 | if (self->ring_type == ATL_RING_RX) |
238 | u64_stats_init(syncp: &self->stats.rx.syncp); |
239 | else |
240 | u64_stats_init(syncp: &self->stats.tx.syncp); |
241 | |
242 | return 0; |
243 | } |
244 | |
245 | static inline bool aq_ring_dx_in_range(unsigned int h, unsigned int i, |
246 | unsigned int t) |
247 | { |
248 | return (h < t) ? ((h < i) && (i < t)) : ((h < i) || (i < t)); |
249 | } |
250 | |
251 | void aq_ring_update_queue_state(struct aq_ring_s *ring) |
252 | { |
253 | if (aq_ring_avail_dx(self: ring) <= AQ_CFG_SKB_FRAGS_MAX) |
254 | aq_ring_queue_stop(ring); |
255 | else if (aq_ring_avail_dx(self: ring) > AQ_CFG_RESTART_DESC_THRES) |
256 | aq_ring_queue_wake(ring); |
257 | } |
258 | |
259 | void aq_ring_queue_wake(struct aq_ring_s *ring) |
260 | { |
261 | struct net_device *ndev = aq_nic_get_ndev(self: ring->aq_nic); |
262 | |
263 | if (__netif_subqueue_stopped(dev: ndev, |
264 | AQ_NIC_RING2QMAP(ring->aq_nic, |
265 | ring->idx))) { |
266 | netif_wake_subqueue(dev: ndev, |
267 | AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); |
268 | u64_stats_update_begin(syncp: &ring->stats.tx.syncp); |
269 | ring->stats.tx.queue_restarts++; |
270 | u64_stats_update_end(syncp: &ring->stats.tx.syncp); |
271 | } |
272 | } |
273 | |
274 | void aq_ring_queue_stop(struct aq_ring_s *ring) |
275 | { |
276 | struct net_device *ndev = aq_nic_get_ndev(self: ring->aq_nic); |
277 | |
278 | if (!__netif_subqueue_stopped(dev: ndev, |
279 | AQ_NIC_RING2QMAP(ring->aq_nic, |
280 | ring->idx))) |
281 | netif_stop_subqueue(dev: ndev, |
282 | AQ_NIC_RING2QMAP(ring->aq_nic, ring->idx)); |
283 | } |
284 | |
285 | bool aq_ring_tx_clean(struct aq_ring_s *self) |
286 | { |
287 | struct device *dev = aq_nic_get_dev(self: self->aq_nic); |
288 | unsigned int budget; |
289 | |
290 | for (budget = AQ_CFG_TX_CLEAN_BUDGET; |
291 | budget && self->sw_head != self->hw_head; budget--) { |
292 | struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; |
293 | |
294 | if (likely(buff->is_mapped)) { |
295 | if (unlikely(buff->is_sop)) { |
296 | if (!buff->is_eop && |
297 | buff->eop_index != 0xffffU && |
298 | (!aq_ring_dx_in_range(h: self->sw_head, |
299 | i: buff->eop_index, |
300 | t: self->hw_head))) |
301 | break; |
302 | |
303 | dma_unmap_single(dev, buff->pa, buff->len, |
304 | DMA_TO_DEVICE); |
305 | } else { |
306 | dma_unmap_page(dev, buff->pa, buff->len, |
307 | DMA_TO_DEVICE); |
308 | } |
309 | } |
310 | |
311 | if (likely(!buff->is_eop)) |
312 | goto out; |
313 | |
314 | if (buff->skb) { |
315 | u64_stats_update_begin(syncp: &self->stats.tx.syncp); |
316 | ++self->stats.tx.packets; |
317 | self->stats.tx.bytes += buff->skb->len; |
318 | u64_stats_update_end(syncp: &self->stats.tx.syncp); |
319 | dev_kfree_skb_any(skb: buff->skb); |
320 | } else if (buff->xdpf) { |
321 | u64_stats_update_begin(syncp: &self->stats.tx.syncp); |
322 | ++self->stats.tx.packets; |
323 | self->stats.tx.bytes += xdp_get_frame_len(xdpf: buff->xdpf); |
324 | u64_stats_update_end(syncp: &self->stats.tx.syncp); |
325 | xdp_return_frame_rx_napi(xdpf: buff->xdpf); |
326 | } |
327 | |
328 | out: |
329 | buff->skb = NULL; |
330 | buff->xdpf = NULL; |
331 | buff->pa = 0U; |
332 | buff->eop_index = 0xffffU; |
333 | self->sw_head = aq_ring_next_dx(self, dx: self->sw_head); |
334 | } |
335 | |
336 | return !!budget; |
337 | } |
338 | |
339 | static void aq_rx_checksum(struct aq_ring_s *self, |
340 | struct aq_ring_buff_s *buff, |
341 | struct sk_buff *skb) |
342 | { |
343 | if (!(self->aq_nic->ndev->features & NETIF_F_RXCSUM)) |
344 | return; |
345 | |
346 | if (unlikely(buff->is_cso_err)) { |
347 | u64_stats_update_begin(syncp: &self->stats.rx.syncp); |
348 | ++self->stats.rx.errors; |
349 | u64_stats_update_end(syncp: &self->stats.rx.syncp); |
350 | skb->ip_summed = CHECKSUM_NONE; |
351 | return; |
352 | } |
353 | if (buff->is_ip_cso) { |
354 | __skb_incr_checksum_unnecessary(skb); |
355 | } else { |
356 | skb->ip_summed = CHECKSUM_NONE; |
357 | } |
358 | |
359 | if (buff->is_udp_cso || buff->is_tcp_cso) |
360 | __skb_incr_checksum_unnecessary(skb); |
361 | } |
362 | |
363 | int aq_xdp_xmit(struct net_device *dev, int num_frames, |
364 | struct xdp_frame **frames, u32 flags) |
365 | { |
366 | struct aq_nic_s *aq_nic = netdev_priv(dev); |
367 | unsigned int vec, i, drop = 0; |
368 | int cpu = smp_processor_id(); |
369 | struct aq_nic_cfg_s *aq_cfg; |
370 | struct aq_ring_s *ring; |
371 | |
372 | aq_cfg = aq_nic_get_cfg(self: aq_nic); |
373 | vec = cpu % aq_cfg->vecs; |
374 | ring = aq_nic->aq_ring_tx[AQ_NIC_CFG_TCVEC2RING(aq_cfg, 0, vec)]; |
375 | |
376 | for (i = 0; i < num_frames; i++) { |
377 | struct xdp_frame *xdpf = frames[i]; |
378 | |
379 | if (aq_nic_xmit_xdpf(aq_nic, tx_ring: ring, xdpf) == NETDEV_TX_BUSY) |
380 | drop++; |
381 | } |
382 | |
383 | return num_frames - drop; |
384 | } |
385 | |
386 | static struct sk_buff *aq_xdp_build_skb(struct xdp_buff *xdp, |
387 | struct net_device *dev, |
388 | struct aq_ring_buff_s *buff) |
389 | { |
390 | struct xdp_frame *xdpf; |
391 | struct sk_buff *skb; |
392 | |
393 | xdpf = xdp_convert_buff_to_frame(xdp); |
394 | if (unlikely(!xdpf)) |
395 | return NULL; |
396 | |
397 | skb = xdp_build_skb_from_frame(xdpf, dev); |
398 | if (!skb) |
399 | return NULL; |
400 | |
401 | aq_get_rxpages_xdp(buff, xdp); |
402 | return skb; |
403 | } |
404 | |
405 | static struct sk_buff *aq_xdp_run_prog(struct aq_nic_s *aq_nic, |
406 | struct xdp_buff *xdp, |
407 | struct aq_ring_s *rx_ring, |
408 | struct aq_ring_buff_s *buff) |
409 | { |
410 | int result = NETDEV_TX_BUSY; |
411 | struct aq_ring_s *tx_ring; |
412 | struct xdp_frame *xdpf; |
413 | struct bpf_prog *prog; |
414 | u32 act = XDP_ABORTED; |
415 | struct sk_buff *skb; |
416 | |
417 | u64_stats_update_begin(syncp: &rx_ring->stats.rx.syncp); |
418 | ++rx_ring->stats.rx.packets; |
419 | rx_ring->stats.rx.bytes += xdp_get_buff_len(xdp); |
420 | u64_stats_update_end(syncp: &rx_ring->stats.rx.syncp); |
421 | |
422 | prog = READ_ONCE(rx_ring->xdp_prog); |
423 | if (!prog) |
424 | return aq_xdp_build_skb(xdp, dev: aq_nic->ndev, buff); |
425 | |
426 | prefetchw(x: xdp->data_hard_start); /* xdp_frame write */ |
427 | |
428 | /* single buffer XDP program, but packet is multi buffer, aborted */ |
429 | if (xdp_buff_has_frags(xdp) && !prog->aux->xdp_has_frags) |
430 | goto out_aborted; |
431 | |
432 | act = bpf_prog_run_xdp(prog, xdp); |
433 | switch (act) { |
434 | case XDP_PASS: |
435 | skb = aq_xdp_build_skb(xdp, dev: aq_nic->ndev, buff); |
436 | if (!skb) |
437 | goto out_aborted; |
438 | u64_stats_update_begin(syncp: &rx_ring->stats.rx.syncp); |
439 | ++rx_ring->stats.rx.xdp_pass; |
440 | u64_stats_update_end(syncp: &rx_ring->stats.rx.syncp); |
441 | return skb; |
442 | case XDP_TX: |
443 | xdpf = xdp_convert_buff_to_frame(xdp); |
444 | if (unlikely(!xdpf)) |
445 | goto out_aborted; |
446 | tx_ring = aq_nic->aq_ring_tx[rx_ring->idx]; |
447 | result = aq_nic_xmit_xdpf(aq_nic, tx_ring, xdpf); |
448 | if (result == NETDEV_TX_BUSY) |
449 | goto out_aborted; |
450 | u64_stats_update_begin(syncp: &rx_ring->stats.rx.syncp); |
451 | ++rx_ring->stats.rx.xdp_tx; |
452 | u64_stats_update_end(syncp: &rx_ring->stats.rx.syncp); |
453 | aq_get_rxpages_xdp(buff, xdp); |
454 | break; |
455 | case XDP_REDIRECT: |
456 | if (xdp_do_redirect(dev: aq_nic->ndev, xdp, prog) < 0) |
457 | goto out_aborted; |
458 | xdp_do_flush(); |
459 | u64_stats_update_begin(syncp: &rx_ring->stats.rx.syncp); |
460 | ++rx_ring->stats.rx.xdp_redirect; |
461 | u64_stats_update_end(syncp: &rx_ring->stats.rx.syncp); |
462 | aq_get_rxpages_xdp(buff, xdp); |
463 | break; |
464 | default: |
465 | fallthrough; |
466 | case XDP_ABORTED: |
467 | out_aborted: |
468 | u64_stats_update_begin(syncp: &rx_ring->stats.rx.syncp); |
469 | ++rx_ring->stats.rx.xdp_aborted; |
470 | u64_stats_update_end(syncp: &rx_ring->stats.rx.syncp); |
471 | trace_xdp_exception(dev: aq_nic->ndev, xdp: prog, act); |
472 | bpf_warn_invalid_xdp_action(dev: aq_nic->ndev, prog, act); |
473 | break; |
474 | case XDP_DROP: |
475 | u64_stats_update_begin(syncp: &rx_ring->stats.rx.syncp); |
476 | ++rx_ring->stats.rx.xdp_drop; |
477 | u64_stats_update_end(syncp: &rx_ring->stats.rx.syncp); |
478 | break; |
479 | } |
480 | |
481 | return ERR_PTR(error: -result); |
482 | } |
483 | |
484 | static bool aq_add_rx_fragment(struct device *dev, |
485 | struct aq_ring_s *ring, |
486 | struct aq_ring_buff_s *buff, |
487 | struct xdp_buff *xdp) |
488 | { |
489 | struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); |
490 | struct aq_ring_buff_s *buff_ = buff; |
491 | |
492 | memset(sinfo, 0, sizeof(*sinfo)); |
493 | do { |
494 | skb_frag_t *frag; |
495 | |
496 | if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) |
497 | return true; |
498 | |
499 | frag = &sinfo->frags[sinfo->nr_frags++]; |
500 | buff_ = &ring->buff_ring[buff_->next]; |
501 | dma_sync_single_range_for_cpu(dev, |
502 | addr: buff_->rxdata.daddr, |
503 | offset: buff_->rxdata.pg_off, |
504 | size: buff_->len, |
505 | dir: DMA_FROM_DEVICE); |
506 | sinfo->xdp_frags_size += buff_->len; |
507 | skb_frag_fill_page_desc(frag, page: buff_->rxdata.page, |
508 | off: buff_->rxdata.pg_off, |
509 | size: buff_->len); |
510 | |
511 | buff_->is_cleaned = 1; |
512 | |
513 | buff->is_ip_cso &= buff_->is_ip_cso; |
514 | buff->is_udp_cso &= buff_->is_udp_cso; |
515 | buff->is_tcp_cso &= buff_->is_tcp_cso; |
516 | buff->is_cso_err |= buff_->is_cso_err; |
517 | |
518 | if (page_is_pfmemalloc(page: buff_->rxdata.page)) |
519 | xdp_buff_set_frag_pfmemalloc(xdp); |
520 | |
521 | } while (!buff_->is_eop); |
522 | |
523 | xdp_buff_set_frags_flag(xdp); |
524 | |
525 | return false; |
526 | } |
527 | |
528 | static int __aq_ring_rx_clean(struct aq_ring_s *self, struct napi_struct *napi, |
529 | int *work_done, int budget) |
530 | { |
531 | struct net_device *ndev = aq_nic_get_ndev(self: self->aq_nic); |
532 | int err = 0; |
533 | |
534 | for (; (self->sw_head != self->hw_head) && budget; |
535 | self->sw_head = aq_ring_next_dx(self, dx: self->sw_head), |
536 | --budget, ++(*work_done)) { |
537 | struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; |
538 | bool is_ptp_ring = aq_ptp_ring(aq_nic: self->aq_nic, ring: self); |
539 | struct aq_ring_buff_s *buff_ = NULL; |
540 | struct sk_buff *skb = NULL; |
541 | unsigned int next_ = 0U; |
542 | unsigned int i = 0U; |
543 | u16 hdr_len; |
544 | |
545 | if (buff->is_cleaned) |
546 | continue; |
547 | |
548 | if (!buff->is_eop) { |
549 | unsigned int frag_cnt = 0U; |
550 | buff_ = buff; |
551 | do { |
552 | bool is_rsc_completed = true; |
553 | |
554 | if (buff_->next >= self->size) { |
555 | err = -EIO; |
556 | goto err_exit; |
557 | } |
558 | |
559 | frag_cnt++; |
560 | next_ = buff_->next, |
561 | buff_ = &self->buff_ring[next_]; |
562 | is_rsc_completed = |
563 | aq_ring_dx_in_range(h: self->sw_head, |
564 | i: next_, |
565 | t: self->hw_head); |
566 | |
567 | if (unlikely(!is_rsc_completed) || |
568 | frag_cnt > MAX_SKB_FRAGS) { |
569 | err = 0; |
570 | goto err_exit; |
571 | } |
572 | |
573 | buff->is_error |= buff_->is_error; |
574 | buff->is_cso_err |= buff_->is_cso_err; |
575 | |
576 | } while (!buff_->is_eop); |
577 | |
578 | if (buff->is_error || |
579 | (buff->is_lro && buff->is_cso_err)) { |
580 | buff_ = buff; |
581 | do { |
582 | if (buff_->next >= self->size) { |
583 | err = -EIO; |
584 | goto err_exit; |
585 | } |
586 | next_ = buff_->next, |
587 | buff_ = &self->buff_ring[next_]; |
588 | |
589 | buff_->is_cleaned = true; |
590 | } while (!buff_->is_eop); |
591 | |
592 | u64_stats_update_begin(syncp: &self->stats.rx.syncp); |
593 | ++self->stats.rx.errors; |
594 | u64_stats_update_end(syncp: &self->stats.rx.syncp); |
595 | continue; |
596 | } |
597 | } |
598 | |
599 | if (buff->is_error) { |
600 | u64_stats_update_begin(syncp: &self->stats.rx.syncp); |
601 | ++self->stats.rx.errors; |
602 | u64_stats_update_end(syncp: &self->stats.rx.syncp); |
603 | continue; |
604 | } |
605 | |
606 | dma_sync_single_range_for_cpu(dev: aq_nic_get_dev(self: self->aq_nic), |
607 | addr: buff->rxdata.daddr, |
608 | offset: buff->rxdata.pg_off, |
609 | size: buff->len, dir: DMA_FROM_DEVICE); |
610 | |
611 | skb = napi_alloc_skb(napi, AQ_CFG_RX_HDR_SIZE); |
612 | if (unlikely(!skb)) { |
613 | u64_stats_update_begin(syncp: &self->stats.rx.syncp); |
614 | self->stats.rx.skb_alloc_fails++; |
615 | u64_stats_update_end(syncp: &self->stats.rx.syncp); |
616 | err = -ENOMEM; |
617 | goto err_exit; |
618 | } |
619 | if (is_ptp_ring) |
620 | buff->len -= |
621 | aq_ptp_extract_ts(aq_nic: self->aq_nic, shhwtstamps: skb_hwtstamps(skb), |
622 | p: aq_buf_vaddr(rxpage: &buff->rxdata), |
623 | len: buff->len); |
624 | |
625 | hdr_len = buff->len; |
626 | if (hdr_len > AQ_CFG_RX_HDR_SIZE) |
627 | hdr_len = eth_get_headlen(dev: skb->dev, |
628 | data: aq_buf_vaddr(rxpage: &buff->rxdata), |
629 | AQ_CFG_RX_HDR_SIZE); |
630 | |
631 | memcpy(__skb_put(skb, hdr_len), aq_buf_vaddr(&buff->rxdata), |
632 | ALIGN(hdr_len, sizeof(long))); |
633 | |
634 | if (buff->len - hdr_len > 0) { |
635 | skb_add_rx_frag(skb, i: i++, page: buff->rxdata.page, |
636 | off: buff->rxdata.pg_off + hdr_len, |
637 | size: buff->len - hdr_len, |
638 | truesize: self->frame_max); |
639 | page_ref_inc(page: buff->rxdata.page); |
640 | } |
641 | |
642 | if (!buff->is_eop) { |
643 | buff_ = buff; |
644 | do { |
645 | next_ = buff_->next; |
646 | buff_ = &self->buff_ring[next_]; |
647 | |
648 | dma_sync_single_range_for_cpu(dev: aq_nic_get_dev(self: self->aq_nic), |
649 | addr: buff_->rxdata.daddr, |
650 | offset: buff_->rxdata.pg_off, |
651 | size: buff_->len, |
652 | dir: DMA_FROM_DEVICE); |
653 | skb_add_rx_frag(skb, i: i++, |
654 | page: buff_->rxdata.page, |
655 | off: buff_->rxdata.pg_off, |
656 | size: buff_->len, |
657 | truesize: self->frame_max); |
658 | page_ref_inc(page: buff_->rxdata.page); |
659 | buff_->is_cleaned = 1; |
660 | |
661 | buff->is_ip_cso &= buff_->is_ip_cso; |
662 | buff->is_udp_cso &= buff_->is_udp_cso; |
663 | buff->is_tcp_cso &= buff_->is_tcp_cso; |
664 | buff->is_cso_err |= buff_->is_cso_err; |
665 | |
666 | } while (!buff_->is_eop); |
667 | } |
668 | |
669 | if (buff->is_vlan) |
670 | __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), |
671 | vlan_tci: buff->vlan_rx_tag); |
672 | |
673 | skb->protocol = eth_type_trans(skb, dev: ndev); |
674 | |
675 | aq_rx_checksum(self, buff, skb); |
676 | |
677 | skb_set_hash(skb, hash: buff->rss_hash, |
678 | type: buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : |
679 | PKT_HASH_TYPE_NONE); |
680 | /* Send all PTP traffic to 0 queue */ |
681 | skb_record_rx_queue(skb, |
682 | rx_queue: is_ptp_ring ? 0 |
683 | : AQ_NIC_RING2QMAP(self->aq_nic, |
684 | self->idx)); |
685 | |
686 | u64_stats_update_begin(syncp: &self->stats.rx.syncp); |
687 | ++self->stats.rx.packets; |
688 | self->stats.rx.bytes += skb->len; |
689 | u64_stats_update_end(syncp: &self->stats.rx.syncp); |
690 | |
691 | napi_gro_receive(napi, skb); |
692 | } |
693 | |
694 | err_exit: |
695 | return err; |
696 | } |
697 | |
698 | static int __aq_ring_xdp_clean(struct aq_ring_s *rx_ring, |
699 | struct napi_struct *napi, int *work_done, |
700 | int budget) |
701 | { |
702 | int frame_sz = rx_ring->page_offset + rx_ring->frame_max + |
703 | rx_ring->tail_size; |
704 | struct aq_nic_s *aq_nic = rx_ring->aq_nic; |
705 | bool is_rsc_completed = true; |
706 | struct device *dev; |
707 | int err = 0; |
708 | |
709 | dev = aq_nic_get_dev(self: aq_nic); |
710 | for (; (rx_ring->sw_head != rx_ring->hw_head) && budget; |
711 | rx_ring->sw_head = aq_ring_next_dx(self: rx_ring, dx: rx_ring->sw_head), |
712 | --budget, ++(*work_done)) { |
713 | struct aq_ring_buff_s *buff = &rx_ring->buff_ring[rx_ring->sw_head]; |
714 | bool is_ptp_ring = aq_ptp_ring(aq_nic: rx_ring->aq_nic, ring: rx_ring); |
715 | struct aq_ring_buff_s *buff_ = NULL; |
716 | u16 ptp_hwtstamp_len = 0; |
717 | struct skb_shared_hwtstamps shhwtstamps; |
718 | struct sk_buff *skb = NULL; |
719 | unsigned int next_ = 0U; |
720 | struct xdp_buff xdp; |
721 | void *hard_start; |
722 | |
723 | if (buff->is_cleaned) |
724 | continue; |
725 | |
726 | if (!buff->is_eop) { |
727 | buff_ = buff; |
728 | do { |
729 | if (buff_->next >= rx_ring->size) { |
730 | err = -EIO; |
731 | goto err_exit; |
732 | } |
733 | next_ = buff_->next; |
734 | buff_ = &rx_ring->buff_ring[next_]; |
735 | is_rsc_completed = |
736 | aq_ring_dx_in_range(h: rx_ring->sw_head, |
737 | i: next_, |
738 | t: rx_ring->hw_head); |
739 | |
740 | if (unlikely(!is_rsc_completed)) |
741 | break; |
742 | |
743 | buff->is_error |= buff_->is_error; |
744 | buff->is_cso_err |= buff_->is_cso_err; |
745 | } while (!buff_->is_eop); |
746 | |
747 | if (!is_rsc_completed) { |
748 | err = 0; |
749 | goto err_exit; |
750 | } |
751 | if (buff->is_error || |
752 | (buff->is_lro && buff->is_cso_err)) { |
753 | buff_ = buff; |
754 | do { |
755 | if (buff_->next >= rx_ring->size) { |
756 | err = -EIO; |
757 | goto err_exit; |
758 | } |
759 | next_ = buff_->next; |
760 | buff_ = &rx_ring->buff_ring[next_]; |
761 | |
762 | buff_->is_cleaned = true; |
763 | } while (!buff_->is_eop); |
764 | |
765 | u64_stats_update_begin(syncp: &rx_ring->stats.rx.syncp); |
766 | ++rx_ring->stats.rx.errors; |
767 | u64_stats_update_end(syncp: &rx_ring->stats.rx.syncp); |
768 | continue; |
769 | } |
770 | } |
771 | |
772 | if (buff->is_error) { |
773 | u64_stats_update_begin(syncp: &rx_ring->stats.rx.syncp); |
774 | ++rx_ring->stats.rx.errors; |
775 | u64_stats_update_end(syncp: &rx_ring->stats.rx.syncp); |
776 | continue; |
777 | } |
778 | |
779 | dma_sync_single_range_for_cpu(dev, |
780 | addr: buff->rxdata.daddr, |
781 | offset: buff->rxdata.pg_off, |
782 | size: buff->len, dir: DMA_FROM_DEVICE); |
783 | hard_start = page_address(buff->rxdata.page) + |
784 | buff->rxdata.pg_off - rx_ring->page_offset; |
785 | |
786 | if (is_ptp_ring) { |
787 | ptp_hwtstamp_len = aq_ptp_extract_ts(aq_nic: rx_ring->aq_nic, shhwtstamps: &shhwtstamps, |
788 | p: aq_buf_vaddr(rxpage: &buff->rxdata), |
789 | len: buff->len); |
790 | buff->len -= ptp_hwtstamp_len; |
791 | } |
792 | |
793 | xdp_init_buff(xdp: &xdp, frame_sz, rxq: &rx_ring->xdp_rxq); |
794 | xdp_prepare_buff(xdp: &xdp, hard_start, headroom: rx_ring->page_offset, |
795 | data_len: buff->len, meta_valid: false); |
796 | if (!buff->is_eop) { |
797 | if (aq_add_rx_fragment(dev, ring: rx_ring, buff, xdp: &xdp)) { |
798 | u64_stats_update_begin(syncp: &rx_ring->stats.rx.syncp); |
799 | ++rx_ring->stats.rx.packets; |
800 | rx_ring->stats.rx.bytes += xdp_get_buff_len(xdp: &xdp); |
801 | ++rx_ring->stats.rx.xdp_aborted; |
802 | u64_stats_update_end(syncp: &rx_ring->stats.rx.syncp); |
803 | continue; |
804 | } |
805 | } |
806 | |
807 | skb = aq_xdp_run_prog(aq_nic, xdp: &xdp, rx_ring, buff); |
808 | if (IS_ERR(ptr: skb) || !skb) |
809 | continue; |
810 | |
811 | if (ptp_hwtstamp_len > 0) |
812 | *skb_hwtstamps(skb) = shhwtstamps; |
813 | |
814 | if (buff->is_vlan) |
815 | __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), |
816 | vlan_tci: buff->vlan_rx_tag); |
817 | |
818 | aq_rx_checksum(self: rx_ring, buff, skb); |
819 | |
820 | skb_set_hash(skb, hash: buff->rss_hash, |
821 | type: buff->is_hash_l4 ? PKT_HASH_TYPE_L4 : |
822 | PKT_HASH_TYPE_NONE); |
823 | /* Send all PTP traffic to 0 queue */ |
824 | skb_record_rx_queue(skb, |
825 | rx_queue: is_ptp_ring ? 0 |
826 | : AQ_NIC_RING2QMAP(rx_ring->aq_nic, |
827 | rx_ring->idx)); |
828 | |
829 | napi_gro_receive(napi, skb); |
830 | } |
831 | |
832 | err_exit: |
833 | return err; |
834 | } |
835 | |
836 | int aq_ring_rx_clean(struct aq_ring_s *self, |
837 | struct napi_struct *napi, |
838 | int *work_done, |
839 | int budget) |
840 | { |
841 | if (static_branch_unlikely(&aq_xdp_locking_key)) |
842 | return __aq_ring_xdp_clean(rx_ring: self, napi, work_done, budget); |
843 | else |
844 | return __aq_ring_rx_clean(self, napi, work_done, budget); |
845 | } |
846 | |
847 | void aq_ring_hwts_rx_clean(struct aq_ring_s *self, struct aq_nic_s *aq_nic) |
848 | { |
849 | #if IS_REACHABLE(CONFIG_PTP_1588_CLOCK) |
850 | while (self->sw_head != self->hw_head) { |
851 | u64 ns; |
852 | |
853 | aq_nic->aq_hw_ops->extract_hwts(aq_nic->aq_hw, |
854 | self->dx_ring + |
855 | (self->sw_head * self->dx_size), |
856 | self->dx_size, &ns); |
857 | aq_ptp_tx_hwtstamp(aq_nic, timestamp: ns); |
858 | |
859 | self->sw_head = aq_ring_next_dx(self, dx: self->sw_head); |
860 | } |
861 | #endif |
862 | } |
863 | |
864 | int aq_ring_rx_fill(struct aq_ring_s *self) |
865 | { |
866 | struct aq_ring_buff_s *buff = NULL; |
867 | int err = 0; |
868 | int i = 0; |
869 | |
870 | if (aq_ring_avail_dx(self) < min_t(unsigned int, AQ_CFG_RX_REFILL_THRES, |
871 | self->size / 2)) |
872 | return err; |
873 | |
874 | for (i = aq_ring_avail_dx(self); i--; |
875 | self->sw_tail = aq_ring_next_dx(self, dx: self->sw_tail)) { |
876 | buff = &self->buff_ring[self->sw_tail]; |
877 | |
878 | buff->flags = 0U; |
879 | buff->len = self->frame_max; |
880 | |
881 | err = aq_get_rxpages(self, rxbuf: buff); |
882 | if (err) |
883 | goto err_exit; |
884 | |
885 | buff->pa = aq_buf_daddr(rxpage: &buff->rxdata); |
886 | buff = NULL; |
887 | } |
888 | |
889 | err_exit: |
890 | return err; |
891 | } |
892 | |
893 | void aq_ring_rx_deinit(struct aq_ring_s *self) |
894 | { |
895 | if (!self) |
896 | return; |
897 | |
898 | for (; self->sw_head != self->sw_tail; |
899 | self->sw_head = aq_ring_next_dx(self, dx: self->sw_head)) { |
900 | struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; |
901 | |
902 | aq_free_rxpage(rxpage: &buff->rxdata, dev: aq_nic_get_dev(self: self->aq_nic)); |
903 | } |
904 | } |
905 | |
906 | void aq_ring_free(struct aq_ring_s *self) |
907 | { |
908 | if (!self) |
909 | return; |
910 | |
911 | kfree(objp: self->buff_ring); |
912 | self->buff_ring = NULL; |
913 | |
914 | if (self->dx_ring) { |
915 | dma_free_coherent(dev: aq_nic_get_dev(self: self->aq_nic), |
916 | size: self->size * self->dx_size, cpu_addr: self->dx_ring, |
917 | dma_handle: self->dx_ring_pa); |
918 | self->dx_ring = NULL; |
919 | } |
920 | } |
921 | |
922 | void aq_ring_hwts_rx_free(struct aq_ring_s *self) |
923 | { |
924 | if (!self) |
925 | return; |
926 | |
927 | if (self->dx_ring) { |
928 | dma_free_coherent(dev: aq_nic_get_dev(self: self->aq_nic), |
929 | size: self->size * self->dx_size + AQ_CFG_RXDS_DEF, |
930 | cpu_addr: self->dx_ring, dma_handle: self->dx_ring_pa); |
931 | self->dx_ring = NULL; |
932 | } |
933 | } |
934 | |
935 | unsigned int aq_ring_fill_stats_data(struct aq_ring_s *self, u64 *data) |
936 | { |
937 | unsigned int count; |
938 | unsigned int start; |
939 | |
940 | if (self->ring_type == ATL_RING_RX) { |
941 | /* This data should mimic aq_ethtool_queue_rx_stat_names structure */ |
942 | do { |
943 | count = 0; |
944 | start = u64_stats_fetch_begin(syncp: &self->stats.rx.syncp); |
945 | data[count] = self->stats.rx.packets; |
946 | data[++count] = self->stats.rx.jumbo_packets; |
947 | data[++count] = self->stats.rx.lro_packets; |
948 | data[++count] = self->stats.rx.errors; |
949 | data[++count] = self->stats.rx.alloc_fails; |
950 | data[++count] = self->stats.rx.skb_alloc_fails; |
951 | data[++count] = self->stats.rx.polls; |
952 | data[++count] = self->stats.rx.pg_flips; |
953 | data[++count] = self->stats.rx.pg_reuses; |
954 | data[++count] = self->stats.rx.pg_losts; |
955 | data[++count] = self->stats.rx.xdp_aborted; |
956 | data[++count] = self->stats.rx.xdp_drop; |
957 | data[++count] = self->stats.rx.xdp_pass; |
958 | data[++count] = self->stats.rx.xdp_tx; |
959 | data[++count] = self->stats.rx.xdp_invalid; |
960 | data[++count] = self->stats.rx.xdp_redirect; |
961 | } while (u64_stats_fetch_retry(syncp: &self->stats.rx.syncp, start)); |
962 | } else { |
963 | /* This data should mimic aq_ethtool_queue_tx_stat_names structure */ |
964 | do { |
965 | count = 0; |
966 | start = u64_stats_fetch_begin(syncp: &self->stats.tx.syncp); |
967 | data[count] = self->stats.tx.packets; |
968 | data[++count] = self->stats.tx.queue_restarts; |
969 | } while (u64_stats_fetch_retry(syncp: &self->stats.tx.syncp, start)); |
970 | } |
971 | |
972 | return ++count; |
973 | } |
974 | |