1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright (c) 2019, Intel Corporation. */ |
3 | |
4 | #include <linux/bpf_trace.h> |
5 | #include <net/xdp_sock_drv.h> |
6 | #include <net/xdp.h> |
7 | #include "ice.h" |
8 | #include "ice_base.h" |
9 | #include "ice_type.h" |
10 | #include "ice_xsk.h" |
11 | #include "ice_txrx.h" |
12 | #include "ice_txrx_lib.h" |
13 | #include "ice_lib.h" |
14 | |
15 | static struct xdp_buff **ice_xdp_buf(struct ice_rx_ring *rx_ring, u32 idx) |
16 | { |
17 | return &rx_ring->xdp_buf[idx]; |
18 | } |
19 | |
20 | /** |
21 | * ice_qp_reset_stats - Resets all stats for rings of given index |
22 | * @vsi: VSI that contains rings of interest |
23 | * @q_idx: ring index in array |
24 | */ |
25 | static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx) |
26 | { |
27 | struct ice_vsi_stats *vsi_stat; |
28 | struct ice_pf *pf; |
29 | |
30 | pf = vsi->back; |
31 | if (!pf->vsi_stats) |
32 | return; |
33 | |
34 | vsi_stat = pf->vsi_stats[vsi->idx]; |
35 | if (!vsi_stat) |
36 | return; |
37 | |
38 | memset(&vsi_stat->rx_ring_stats[q_idx]->rx_stats, 0, |
39 | sizeof(vsi_stat->rx_ring_stats[q_idx]->rx_stats)); |
40 | memset(&vsi_stat->tx_ring_stats[q_idx]->stats, 0, |
41 | sizeof(vsi_stat->tx_ring_stats[q_idx]->stats)); |
42 | if (ice_is_xdp_ena_vsi(vsi)) |
43 | memset(&vsi->xdp_rings[q_idx]->ring_stats->stats, 0, |
44 | sizeof(vsi->xdp_rings[q_idx]->ring_stats->stats)); |
45 | } |
46 | |
47 | /** |
48 | * ice_qp_clean_rings - Cleans all the rings of a given index |
49 | * @vsi: VSI that contains rings of interest |
50 | * @q_idx: ring index in array |
51 | */ |
52 | static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx) |
53 | { |
54 | ice_clean_tx_ring(tx_ring: vsi->tx_rings[q_idx]); |
55 | if (ice_is_xdp_ena_vsi(vsi)) { |
56 | synchronize_rcu(); |
57 | ice_clean_tx_ring(tx_ring: vsi->xdp_rings[q_idx]); |
58 | } |
59 | ice_clean_rx_ring(rx_ring: vsi->rx_rings[q_idx]); |
60 | } |
61 | |
62 | /** |
63 | * ice_qvec_toggle_napi - Enables/disables NAPI for a given q_vector |
64 | * @vsi: VSI that has netdev |
65 | * @q_vector: q_vector that has NAPI context |
66 | * @enable: true for enable, false for disable |
67 | */ |
68 | static void |
69 | ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector, |
70 | bool enable) |
71 | { |
72 | if (!vsi->netdev || !q_vector) |
73 | return; |
74 | |
75 | if (enable) |
76 | napi_enable(n: &q_vector->napi); |
77 | else |
78 | napi_disable(n: &q_vector->napi); |
79 | } |
80 | |
81 | /** |
82 | * ice_qvec_dis_irq - Mask off queue interrupt generation on given ring |
83 | * @vsi: the VSI that contains queue vector being un-configured |
84 | * @rx_ring: Rx ring that will have its IRQ disabled |
85 | * @q_vector: queue vector |
86 | */ |
87 | static void |
88 | ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_rx_ring *rx_ring, |
89 | struct ice_q_vector *q_vector) |
90 | { |
91 | struct ice_pf *pf = vsi->back; |
92 | struct ice_hw *hw = &pf->hw; |
93 | u16 reg; |
94 | u32 val; |
95 | |
96 | /* QINT_TQCTL is being cleared in ice_vsi_stop_tx_ring, so handle |
97 | * here only QINT_RQCTL |
98 | */ |
99 | reg = rx_ring->reg_idx; |
100 | val = rd32(hw, QINT_RQCTL(reg)); |
101 | val &= ~QINT_RQCTL_CAUSE_ENA_M; |
102 | wr32(hw, QINT_RQCTL(reg), val); |
103 | |
104 | if (q_vector) { |
105 | wr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), 0); |
106 | ice_flush(hw); |
107 | synchronize_irq(irq: q_vector->irq.virq); |
108 | } |
109 | } |
110 | |
111 | /** |
112 | * ice_qvec_cfg_msix - Enable IRQ for given queue vector |
113 | * @vsi: the VSI that contains queue vector |
114 | * @q_vector: queue vector |
115 | */ |
116 | static void |
117 | ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector) |
118 | { |
119 | u16 reg_idx = q_vector->reg_idx; |
120 | struct ice_pf *pf = vsi->back; |
121 | struct ice_hw *hw = &pf->hw; |
122 | struct ice_tx_ring *tx_ring; |
123 | struct ice_rx_ring *rx_ring; |
124 | |
125 | ice_cfg_itr(hw, q_vector); |
126 | |
127 | ice_for_each_tx_ring(tx_ring, q_vector->tx) |
128 | ice_cfg_txq_interrupt(vsi, txq: tx_ring->reg_idx, msix_idx: reg_idx, |
129 | itr_idx: q_vector->tx.itr_idx); |
130 | |
131 | ice_for_each_rx_ring(rx_ring, q_vector->rx) |
132 | ice_cfg_rxq_interrupt(vsi, rxq: rx_ring->reg_idx, msix_idx: reg_idx, |
133 | itr_idx: q_vector->rx.itr_idx); |
134 | |
135 | ice_flush(hw); |
136 | } |
137 | |
138 | /** |
139 | * ice_qvec_ena_irq - Enable IRQ for given queue vector |
140 | * @vsi: the VSI that contains queue vector |
141 | * @q_vector: queue vector |
142 | */ |
143 | static void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector) |
144 | { |
145 | struct ice_pf *pf = vsi->back; |
146 | struct ice_hw *hw = &pf->hw; |
147 | |
148 | ice_irq_dynamic_ena(hw, vsi, q_vector); |
149 | |
150 | ice_flush(hw); |
151 | } |
152 | |
153 | /** |
154 | * ice_qp_dis - Disables a queue pair |
155 | * @vsi: VSI of interest |
156 | * @q_idx: ring index in array |
157 | * |
158 | * Returns 0 on success, negative on failure. |
159 | */ |
160 | static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx) |
161 | { |
162 | struct ice_txq_meta txq_meta = { }; |
163 | struct ice_q_vector *q_vector; |
164 | struct ice_tx_ring *tx_ring; |
165 | struct ice_rx_ring *rx_ring; |
166 | int timeout = 50; |
167 | int err; |
168 | |
169 | if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq) |
170 | return -EINVAL; |
171 | |
172 | tx_ring = vsi->tx_rings[q_idx]; |
173 | rx_ring = vsi->rx_rings[q_idx]; |
174 | q_vector = rx_ring->q_vector; |
175 | |
176 | while (test_and_set_bit(nr: ICE_CFG_BUSY, addr: vsi->state)) { |
177 | timeout--; |
178 | if (!timeout) |
179 | return -EBUSY; |
180 | usleep_range(min: 1000, max: 2000); |
181 | } |
182 | |
183 | ice_qvec_dis_irq(vsi, rx_ring, q_vector); |
184 | ice_qvec_toggle_napi(vsi, q_vector, enable: false); |
185 | |
186 | netif_tx_stop_queue(dev_queue: netdev_get_tx_queue(dev: vsi->netdev, index: q_idx)); |
187 | |
188 | ice_fill_txq_meta(vsi, ring: tx_ring, txq_meta: &txq_meta); |
189 | err = ice_vsi_stop_tx_ring(vsi, rst_src: ICE_NO_RESET, rel_vmvf_num: 0, ring: tx_ring, txq_meta: &txq_meta); |
190 | if (err) |
191 | return err; |
192 | if (ice_is_xdp_ena_vsi(vsi)) { |
193 | struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; |
194 | |
195 | memset(&txq_meta, 0, sizeof(txq_meta)); |
196 | ice_fill_txq_meta(vsi, ring: xdp_ring, txq_meta: &txq_meta); |
197 | err = ice_vsi_stop_tx_ring(vsi, rst_src: ICE_NO_RESET, rel_vmvf_num: 0, ring: xdp_ring, |
198 | txq_meta: &txq_meta); |
199 | if (err) |
200 | return err; |
201 | } |
202 | err = ice_vsi_ctrl_one_rx_ring(vsi, ena: false, rxq_idx: q_idx, wait: true); |
203 | if (err) |
204 | return err; |
205 | |
206 | ice_qp_clean_rings(vsi, q_idx); |
207 | ice_qp_reset_stats(vsi, q_idx); |
208 | |
209 | return 0; |
210 | } |
211 | |
212 | /** |
213 | * ice_qp_ena - Enables a queue pair |
214 | * @vsi: VSI of interest |
215 | * @q_idx: ring index in array |
216 | * |
217 | * Returns 0 on success, negative on failure. |
218 | */ |
219 | static int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx) |
220 | { |
221 | struct ice_q_vector *q_vector; |
222 | int err; |
223 | |
224 | err = ice_vsi_cfg_single_txq(vsi, tx_rings: vsi->tx_rings, q_idx); |
225 | if (err) |
226 | return err; |
227 | |
228 | if (ice_is_xdp_ena_vsi(vsi)) { |
229 | struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; |
230 | |
231 | err = ice_vsi_cfg_single_txq(vsi, tx_rings: vsi->xdp_rings, q_idx); |
232 | if (err) |
233 | return err; |
234 | ice_set_ring_xdp(ring: xdp_ring); |
235 | ice_tx_xsk_pool(vsi, qid: q_idx); |
236 | } |
237 | |
238 | err = ice_vsi_cfg_single_rxq(vsi, q_idx); |
239 | if (err) |
240 | return err; |
241 | |
242 | q_vector = vsi->rx_rings[q_idx]->q_vector; |
243 | ice_qvec_cfg_msix(vsi, q_vector); |
244 | |
245 | err = ice_vsi_ctrl_one_rx_ring(vsi, ena: true, rxq_idx: q_idx, wait: true); |
246 | if (err) |
247 | return err; |
248 | |
249 | ice_qvec_toggle_napi(vsi, q_vector, enable: true); |
250 | ice_qvec_ena_irq(vsi, q_vector); |
251 | |
252 | netif_tx_start_queue(dev_queue: netdev_get_tx_queue(dev: vsi->netdev, index: q_idx)); |
253 | clear_bit(nr: ICE_CFG_BUSY, addr: vsi->state); |
254 | |
255 | return 0; |
256 | } |
257 | |
258 | /** |
259 | * ice_xsk_pool_disable - disable a buffer pool region |
260 | * @vsi: Current VSI |
261 | * @qid: queue ID |
262 | * |
263 | * Returns 0 on success, negative on failure |
264 | */ |
265 | static int ice_xsk_pool_disable(struct ice_vsi *vsi, u16 qid) |
266 | { |
267 | struct xsk_buff_pool *pool = xsk_get_pool_from_qid(dev: vsi->netdev, queue_id: qid); |
268 | |
269 | if (!pool) |
270 | return -EINVAL; |
271 | |
272 | clear_bit(nr: qid, addr: vsi->af_xdp_zc_qps); |
273 | xsk_pool_dma_unmap(pool, ICE_RX_DMA_ATTR); |
274 | |
275 | return 0; |
276 | } |
277 | |
278 | /** |
279 | * ice_xsk_pool_enable - enable a buffer pool region |
280 | * @vsi: Current VSI |
281 | * @pool: pointer to a requested buffer pool region |
282 | * @qid: queue ID |
283 | * |
284 | * Returns 0 on success, negative on failure |
285 | */ |
286 | static int |
287 | ice_xsk_pool_enable(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) |
288 | { |
289 | int err; |
290 | |
291 | if (vsi->type != ICE_VSI_PF) |
292 | return -EINVAL; |
293 | |
294 | if (qid >= vsi->netdev->real_num_rx_queues || |
295 | qid >= vsi->netdev->real_num_tx_queues) |
296 | return -EINVAL; |
297 | |
298 | err = xsk_pool_dma_map(pool, ice_pf_to_dev(vsi->back), |
299 | ICE_RX_DMA_ATTR); |
300 | if (err) |
301 | return err; |
302 | |
303 | set_bit(nr: qid, addr: vsi->af_xdp_zc_qps); |
304 | |
305 | return 0; |
306 | } |
307 | |
308 | /** |
309 | * ice_realloc_rx_xdp_bufs - reallocate for either XSK or normal buffer |
310 | * @rx_ring: Rx ring |
311 | * @pool_present: is pool for XSK present |
312 | * |
313 | * Try allocating memory and return ENOMEM, if failed to allocate. |
314 | * If allocation was successful, substitute buffer with allocated one. |
315 | * Returns 0 on success, negative on failure |
316 | */ |
317 | static int |
318 | ice_realloc_rx_xdp_bufs(struct ice_rx_ring *rx_ring, bool pool_present) |
319 | { |
320 | size_t elem_size = pool_present ? sizeof(*rx_ring->xdp_buf) : |
321 | sizeof(*rx_ring->rx_buf); |
322 | void *sw_ring = kcalloc(n: rx_ring->count, size: elem_size, GFP_KERNEL); |
323 | |
324 | if (!sw_ring) |
325 | return -ENOMEM; |
326 | |
327 | if (pool_present) { |
328 | kfree(objp: rx_ring->rx_buf); |
329 | rx_ring->rx_buf = NULL; |
330 | rx_ring->xdp_buf = sw_ring; |
331 | } else { |
332 | kfree(objp: rx_ring->xdp_buf); |
333 | rx_ring->xdp_buf = NULL; |
334 | rx_ring->rx_buf = sw_ring; |
335 | } |
336 | |
337 | return 0; |
338 | } |
339 | |
340 | /** |
341 | * ice_realloc_zc_buf - reallocate XDP ZC queue pairs |
342 | * @vsi: Current VSI |
343 | * @zc: is zero copy set |
344 | * |
345 | * Reallocate buffer for rx_rings that might be used by XSK. |
346 | * XDP requires more memory, than rx_buf provides. |
347 | * Returns 0 on success, negative on failure |
348 | */ |
349 | int ice_realloc_zc_buf(struct ice_vsi *vsi, bool zc) |
350 | { |
351 | struct ice_rx_ring *rx_ring; |
352 | unsigned long q; |
353 | |
354 | for_each_set_bit(q, vsi->af_xdp_zc_qps, |
355 | max_t(int, vsi->alloc_txq, vsi->alloc_rxq)) { |
356 | rx_ring = vsi->rx_rings[q]; |
357 | if (ice_realloc_rx_xdp_bufs(rx_ring, pool_present: zc)) |
358 | return -ENOMEM; |
359 | } |
360 | |
361 | return 0; |
362 | } |
363 | |
364 | /** |
365 | * ice_xsk_pool_setup - enable/disable a buffer pool region depending on its state |
366 | * @vsi: Current VSI |
367 | * @pool: buffer pool to enable/associate to a ring, NULL to disable |
368 | * @qid: queue ID |
369 | * |
370 | * Returns 0 on success, negative on failure |
371 | */ |
372 | int ice_xsk_pool_setup(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) |
373 | { |
374 | bool if_running, pool_present = !!pool; |
375 | int ret = 0, pool_failure = 0; |
376 | |
377 | if (qid >= vsi->num_rxq || qid >= vsi->num_txq) { |
378 | netdev_err(dev: vsi->netdev, format: "Please use queue id in scope of combined queues count\n" ); |
379 | pool_failure = -EINVAL; |
380 | goto failure; |
381 | } |
382 | |
383 | if_running = netif_running(dev: vsi->netdev) && ice_is_xdp_ena_vsi(vsi); |
384 | |
385 | if (if_running) { |
386 | struct ice_rx_ring *rx_ring = vsi->rx_rings[qid]; |
387 | |
388 | ret = ice_qp_dis(vsi, q_idx: qid); |
389 | if (ret) { |
390 | netdev_err(dev: vsi->netdev, format: "ice_qp_dis error = %d\n" , ret); |
391 | goto xsk_pool_if_up; |
392 | } |
393 | |
394 | ret = ice_realloc_rx_xdp_bufs(rx_ring, pool_present); |
395 | if (ret) |
396 | goto xsk_pool_if_up; |
397 | } |
398 | |
399 | pool_failure = pool_present ? ice_xsk_pool_enable(vsi, pool, qid) : |
400 | ice_xsk_pool_disable(vsi, qid); |
401 | |
402 | xsk_pool_if_up: |
403 | if (if_running) { |
404 | ret = ice_qp_ena(vsi, q_idx: qid); |
405 | if (!ret && pool_present) |
406 | napi_schedule(n: &vsi->rx_rings[qid]->xdp_ring->q_vector->napi); |
407 | else if (ret) |
408 | netdev_err(dev: vsi->netdev, format: "ice_qp_ena error = %d\n" , ret); |
409 | } |
410 | |
411 | failure: |
412 | if (pool_failure) { |
413 | netdev_err(dev: vsi->netdev, format: "Could not %sable buffer pool, error = %d\n" , |
414 | pool_present ? "en" : "dis" , pool_failure); |
415 | return pool_failure; |
416 | } |
417 | |
418 | return ret; |
419 | } |
420 | |
421 | /** |
422 | * ice_fill_rx_descs - pick buffers from XSK buffer pool and use it |
423 | * @pool: XSK Buffer pool to pull the buffers from |
424 | * @xdp: SW ring of xdp_buff that will hold the buffers |
425 | * @rx_desc: Pointer to Rx descriptors that will be filled |
426 | * @count: The number of buffers to allocate |
427 | * |
428 | * This function allocates a number of Rx buffers from the fill ring |
429 | * or the internal recycle mechanism and places them on the Rx ring. |
430 | * |
431 | * Note that ring wrap should be handled by caller of this function. |
432 | * |
433 | * Returns the amount of allocated Rx descriptors |
434 | */ |
435 | static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp, |
436 | union ice_32b_rx_flex_desc *rx_desc, u16 count) |
437 | { |
438 | dma_addr_t dma; |
439 | u16 buffs; |
440 | int i; |
441 | |
442 | buffs = xsk_buff_alloc_batch(pool, xdp, max: count); |
443 | for (i = 0; i < buffs; i++) { |
444 | dma = xsk_buff_xdp_get_dma(xdp: *xdp); |
445 | rx_desc->read.pkt_addr = cpu_to_le64(dma); |
446 | rx_desc->wb.status_error0 = 0; |
447 | |
448 | /* Put private info that changes on a per-packet basis |
449 | * into xdp_buff_xsk->cb. |
450 | */ |
451 | ice_xdp_meta_set_desc(xdp: *xdp, eop_desc: rx_desc); |
452 | |
453 | rx_desc++; |
454 | xdp++; |
455 | } |
456 | |
457 | return buffs; |
458 | } |
459 | |
460 | /** |
461 | * __ice_alloc_rx_bufs_zc - allocate a number of Rx buffers |
462 | * @rx_ring: Rx ring |
463 | * @count: The number of buffers to allocate |
464 | * |
465 | * Place the @count of descriptors onto Rx ring. Handle the ring wrap |
466 | * for case where space from next_to_use up to the end of ring is less |
467 | * than @count. Finally do a tail bump. |
468 | * |
469 | * Returns true if all allocations were successful, false if any fail. |
470 | */ |
471 | static bool __ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count) |
472 | { |
473 | u32 = 0, nb_buffs = 0; |
474 | union ice_32b_rx_flex_desc *rx_desc; |
475 | u16 ntu = rx_ring->next_to_use; |
476 | u16 total_count = count; |
477 | struct xdp_buff **xdp; |
478 | |
479 | rx_desc = ICE_RX_DESC(rx_ring, ntu); |
480 | xdp = ice_xdp_buf(rx_ring, idx: ntu); |
481 | |
482 | if (ntu + count >= rx_ring->count) { |
483 | nb_buffs_extra = ice_fill_rx_descs(pool: rx_ring->xsk_pool, xdp, |
484 | rx_desc, |
485 | count: rx_ring->count - ntu); |
486 | if (nb_buffs_extra != rx_ring->count - ntu) { |
487 | ntu += nb_buffs_extra; |
488 | goto exit; |
489 | } |
490 | rx_desc = ICE_RX_DESC(rx_ring, 0); |
491 | xdp = ice_xdp_buf(rx_ring, idx: 0); |
492 | ntu = 0; |
493 | count -= nb_buffs_extra; |
494 | ice_release_rx_desc(rx_ring, val: 0); |
495 | } |
496 | |
497 | nb_buffs = ice_fill_rx_descs(pool: rx_ring->xsk_pool, xdp, rx_desc, count); |
498 | |
499 | ntu += nb_buffs; |
500 | if (ntu == rx_ring->count) |
501 | ntu = 0; |
502 | |
503 | exit: |
504 | if (rx_ring->next_to_use != ntu) |
505 | ice_release_rx_desc(rx_ring, val: ntu); |
506 | |
507 | return total_count == (nb_buffs_extra + nb_buffs); |
508 | } |
509 | |
510 | /** |
511 | * ice_alloc_rx_bufs_zc - allocate a number of Rx buffers |
512 | * @rx_ring: Rx ring |
513 | * @count: The number of buffers to allocate |
514 | * |
515 | * Wrapper for internal allocation routine; figure out how many tail |
516 | * bumps should take place based on the given threshold |
517 | * |
518 | * Returns true if all calls to internal alloc routine succeeded |
519 | */ |
520 | bool ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count) |
521 | { |
522 | u16 rx_thresh = ICE_RING_QUARTER(rx_ring); |
523 | u16 leftover, i, tail_bumps; |
524 | |
525 | tail_bumps = count / rx_thresh; |
526 | leftover = count - (tail_bumps * rx_thresh); |
527 | |
528 | for (i = 0; i < tail_bumps; i++) |
529 | if (!__ice_alloc_rx_bufs_zc(rx_ring, count: rx_thresh)) |
530 | return false; |
531 | return __ice_alloc_rx_bufs_zc(rx_ring, count: leftover); |
532 | } |
533 | |
534 | /** |
535 | * ice_construct_skb_zc - Create an sk_buff from zero-copy buffer |
536 | * @rx_ring: Rx ring |
537 | * @xdp: Pointer to XDP buffer |
538 | * |
539 | * This function allocates a new skb from a zero-copy Rx buffer. |
540 | * |
541 | * Returns the skb on success, NULL on failure. |
542 | */ |
543 | static struct sk_buff * |
544 | ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) |
545 | { |
546 | unsigned int totalsize = xdp->data_end - xdp->data_meta; |
547 | unsigned int metasize = xdp->data - xdp->data_meta; |
548 | struct skb_shared_info *sinfo = NULL; |
549 | struct sk_buff *skb; |
550 | u32 nr_frags = 0; |
551 | |
552 | if (unlikely(xdp_buff_has_frags(xdp))) { |
553 | sinfo = xdp_get_shared_info_from_buff(xdp); |
554 | nr_frags = sinfo->nr_frags; |
555 | } |
556 | net_prefetch(p: xdp->data_meta); |
557 | |
558 | skb = __napi_alloc_skb(napi: &rx_ring->q_vector->napi, length: totalsize, |
559 | GFP_ATOMIC | __GFP_NOWARN); |
560 | if (unlikely(!skb)) |
561 | return NULL; |
562 | |
563 | memcpy(__skb_put(skb, totalsize), xdp->data_meta, |
564 | ALIGN(totalsize, sizeof(long))); |
565 | |
566 | if (metasize) { |
567 | skb_metadata_set(skb, meta_len: metasize); |
568 | __skb_pull(skb, len: metasize); |
569 | } |
570 | |
571 | if (likely(!xdp_buff_has_frags(xdp))) |
572 | goto out; |
573 | |
574 | for (int i = 0; i < nr_frags; i++) { |
575 | struct skb_shared_info *skinfo = skb_shinfo(skb); |
576 | skb_frag_t *frag = &sinfo->frags[i]; |
577 | struct page *page; |
578 | void *addr; |
579 | |
580 | page = dev_alloc_page(); |
581 | if (!page) { |
582 | dev_kfree_skb(skb); |
583 | return NULL; |
584 | } |
585 | addr = page_to_virt(page); |
586 | |
587 | memcpy(addr, skb_frag_page(frag), skb_frag_size(frag)); |
588 | |
589 | __skb_fill_page_desc_noacc(shinfo: skinfo, i: skinfo->nr_frags++, |
590 | page: addr, off: 0, size: skb_frag_size(frag)); |
591 | } |
592 | |
593 | out: |
594 | xsk_buff_free(xdp); |
595 | return skb; |
596 | } |
597 | |
598 | /** |
599 | * ice_clean_xdp_irq_zc - produce AF_XDP descriptors to CQ |
600 | * @xdp_ring: XDP Tx ring |
601 | */ |
602 | static u32 ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring) |
603 | { |
604 | u16 ntc = xdp_ring->next_to_clean; |
605 | struct ice_tx_desc *tx_desc; |
606 | u16 cnt = xdp_ring->count; |
607 | struct ice_tx_buf *tx_buf; |
608 | u16 completed_frames = 0; |
609 | u16 xsk_frames = 0; |
610 | u16 last_rs; |
611 | int i; |
612 | |
613 | last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1; |
614 | tx_desc = ICE_TX_DESC(xdp_ring, last_rs); |
615 | if (tx_desc->cmd_type_offset_bsz & |
616 | cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)) { |
617 | if (last_rs >= ntc) |
618 | completed_frames = last_rs - ntc + 1; |
619 | else |
620 | completed_frames = last_rs + cnt - ntc + 1; |
621 | } |
622 | |
623 | if (!completed_frames) |
624 | return 0; |
625 | |
626 | if (likely(!xdp_ring->xdp_tx_active)) { |
627 | xsk_frames = completed_frames; |
628 | goto skip; |
629 | } |
630 | |
631 | ntc = xdp_ring->next_to_clean; |
632 | for (i = 0; i < completed_frames; i++) { |
633 | tx_buf = &xdp_ring->tx_buf[ntc]; |
634 | |
635 | if (tx_buf->type == ICE_TX_BUF_XSK_TX) { |
636 | tx_buf->type = ICE_TX_BUF_EMPTY; |
637 | xsk_buff_free(xdp: tx_buf->xdp); |
638 | xdp_ring->xdp_tx_active--; |
639 | } else { |
640 | xsk_frames++; |
641 | } |
642 | |
643 | ntc++; |
644 | if (ntc >= xdp_ring->count) |
645 | ntc = 0; |
646 | } |
647 | skip: |
648 | tx_desc->cmd_type_offset_bsz = 0; |
649 | xdp_ring->next_to_clean += completed_frames; |
650 | if (xdp_ring->next_to_clean >= cnt) |
651 | xdp_ring->next_to_clean -= cnt; |
652 | if (xsk_frames) |
653 | xsk_tx_completed(pool: xdp_ring->xsk_pool, nb_entries: xsk_frames); |
654 | |
655 | return completed_frames; |
656 | } |
657 | |
658 | /** |
659 | * ice_xmit_xdp_tx_zc - AF_XDP ZC handler for XDP_TX |
660 | * @xdp: XDP buffer to xmit |
661 | * @xdp_ring: XDP ring to produce descriptor onto |
662 | * |
663 | * note that this function works directly on xdp_buff, no need to convert |
664 | * it to xdp_frame. xdp_buff pointer is stored to ice_tx_buf so that cleaning |
665 | * side will be able to xsk_buff_free() it. |
666 | * |
667 | * Returns ICE_XDP_TX for successfully produced desc, ICE_XDP_CONSUMED if there |
668 | * was not enough space on XDP ring |
669 | */ |
670 | static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp, |
671 | struct ice_tx_ring *xdp_ring) |
672 | { |
673 | struct skb_shared_info *sinfo = NULL; |
674 | u32 size = xdp->data_end - xdp->data; |
675 | u32 ntu = xdp_ring->next_to_use; |
676 | struct ice_tx_desc *tx_desc; |
677 | struct ice_tx_buf *tx_buf; |
678 | struct xdp_buff *head; |
679 | u32 nr_frags = 0; |
680 | u32 free_space; |
681 | u32 frag = 0; |
682 | |
683 | free_space = ICE_DESC_UNUSED(xdp_ring); |
684 | if (free_space < ICE_RING_QUARTER(xdp_ring)) |
685 | free_space += ice_clean_xdp_irq_zc(xdp_ring); |
686 | |
687 | if (unlikely(!free_space)) |
688 | goto busy; |
689 | |
690 | if (unlikely(xdp_buff_has_frags(xdp))) { |
691 | sinfo = xdp_get_shared_info_from_buff(xdp); |
692 | nr_frags = sinfo->nr_frags; |
693 | if (free_space < nr_frags + 1) |
694 | goto busy; |
695 | } |
696 | |
697 | tx_desc = ICE_TX_DESC(xdp_ring, ntu); |
698 | tx_buf = &xdp_ring->tx_buf[ntu]; |
699 | head = xdp; |
700 | |
701 | for (;;) { |
702 | dma_addr_t dma; |
703 | |
704 | dma = xsk_buff_xdp_get_dma(xdp); |
705 | xsk_buff_raw_dma_sync_for_device(pool: xdp_ring->xsk_pool, dma, size); |
706 | |
707 | tx_buf->xdp = xdp; |
708 | tx_buf->type = ICE_TX_BUF_XSK_TX; |
709 | tx_desc->buf_addr = cpu_to_le64(dma); |
710 | tx_desc->cmd_type_offset_bsz = ice_build_ctob(td_cmd: 0, td_offset: 0, size, td_tag: 0); |
711 | /* account for each xdp_buff from xsk_buff_pool */ |
712 | xdp_ring->xdp_tx_active++; |
713 | |
714 | if (++ntu == xdp_ring->count) |
715 | ntu = 0; |
716 | |
717 | if (frag == nr_frags) |
718 | break; |
719 | |
720 | tx_desc = ICE_TX_DESC(xdp_ring, ntu); |
721 | tx_buf = &xdp_ring->tx_buf[ntu]; |
722 | |
723 | xdp = xsk_buff_get_frag(first: head); |
724 | size = skb_frag_size(frag: &sinfo->frags[frag]); |
725 | frag++; |
726 | } |
727 | |
728 | xdp_ring->next_to_use = ntu; |
729 | /* update last descriptor from a frame with EOP */ |
730 | tx_desc->cmd_type_offset_bsz |= |
731 | cpu_to_le64(ICE_TX_DESC_CMD_EOP << ICE_TXD_QW1_CMD_S); |
732 | |
733 | return ICE_XDP_TX; |
734 | |
735 | busy: |
736 | xdp_ring->ring_stats->tx_stats.tx_busy++; |
737 | |
738 | return ICE_XDP_CONSUMED; |
739 | } |
740 | |
741 | /** |
742 | * ice_run_xdp_zc - Executes an XDP program in zero-copy path |
743 | * @rx_ring: Rx ring |
744 | * @xdp: xdp_buff used as input to the XDP program |
745 | * @xdp_prog: XDP program to run |
746 | * @xdp_ring: ring to be used for XDP_TX action |
747 | * |
748 | * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR} |
749 | */ |
750 | static int |
751 | ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp, |
752 | struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring) |
753 | { |
754 | int err, result = ICE_XDP_PASS; |
755 | u32 act; |
756 | |
757 | act = bpf_prog_run_xdp(prog: xdp_prog, xdp); |
758 | |
759 | if (likely(act == XDP_REDIRECT)) { |
760 | err = xdp_do_redirect(dev: rx_ring->netdev, xdp, prog: xdp_prog); |
761 | if (!err) |
762 | return ICE_XDP_REDIR; |
763 | if (xsk_uses_need_wakeup(pool: rx_ring->xsk_pool) && err == -ENOBUFS) |
764 | result = ICE_XDP_EXIT; |
765 | else |
766 | result = ICE_XDP_CONSUMED; |
767 | goto out_failure; |
768 | } |
769 | |
770 | switch (act) { |
771 | case XDP_PASS: |
772 | break; |
773 | case XDP_TX: |
774 | result = ice_xmit_xdp_tx_zc(xdp, xdp_ring); |
775 | if (result == ICE_XDP_CONSUMED) |
776 | goto out_failure; |
777 | break; |
778 | case XDP_DROP: |
779 | result = ICE_XDP_CONSUMED; |
780 | break; |
781 | default: |
782 | bpf_warn_invalid_xdp_action(dev: rx_ring->netdev, prog: xdp_prog, act); |
783 | fallthrough; |
784 | case XDP_ABORTED: |
785 | result = ICE_XDP_CONSUMED; |
786 | out_failure: |
787 | trace_xdp_exception(dev: rx_ring->netdev, xdp: xdp_prog, act); |
788 | break; |
789 | } |
790 | |
791 | return result; |
792 | } |
793 | |
794 | static int |
795 | ice_add_xsk_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *first, |
796 | struct xdp_buff *xdp, const unsigned int size) |
797 | { |
798 | struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp: first); |
799 | |
800 | if (!size) |
801 | return 0; |
802 | |
803 | if (!xdp_buff_has_frags(xdp: first)) { |
804 | sinfo->nr_frags = 0; |
805 | sinfo->xdp_frags_size = 0; |
806 | xdp_buff_set_frags_flag(xdp: first); |
807 | } |
808 | |
809 | if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) { |
810 | xsk_buff_free(xdp: first); |
811 | return -ENOMEM; |
812 | } |
813 | |
814 | __skb_fill_page_desc_noacc(shinfo: sinfo, i: sinfo->nr_frags++, |
815 | virt_to_page(xdp->data_hard_start), |
816 | XDP_PACKET_HEADROOM, size); |
817 | sinfo->xdp_frags_size += size; |
818 | xsk_buff_add_frag(xdp); |
819 | |
820 | return 0; |
821 | } |
822 | |
823 | /** |
824 | * ice_clean_rx_irq_zc - consumes packets from the hardware ring |
825 | * @rx_ring: AF_XDP Rx ring |
826 | * @budget: NAPI budget |
827 | * |
828 | * Returns number of processed packets on success, remaining budget on failure. |
829 | */ |
830 | int ice_clean_rx_irq_zc(struct ice_rx_ring *rx_ring, int budget) |
831 | { |
832 | unsigned int total_rx_bytes = 0, total_rx_packets = 0; |
833 | struct xsk_buff_pool *xsk_pool = rx_ring->xsk_pool; |
834 | u32 ntc = rx_ring->next_to_clean; |
835 | u32 ntu = rx_ring->next_to_use; |
836 | struct xdp_buff *first = NULL; |
837 | struct ice_tx_ring *xdp_ring; |
838 | unsigned int xdp_xmit = 0; |
839 | struct bpf_prog *xdp_prog; |
840 | u32 cnt = rx_ring->count; |
841 | bool failure = false; |
842 | int entries_to_alloc; |
843 | |
844 | /* ZC patch is enabled only when XDP program is set, |
845 | * so here it can not be NULL |
846 | */ |
847 | xdp_prog = READ_ONCE(rx_ring->xdp_prog); |
848 | xdp_ring = rx_ring->xdp_ring; |
849 | |
850 | if (ntc != rx_ring->first_desc) |
851 | first = *ice_xdp_buf(rx_ring, idx: rx_ring->first_desc); |
852 | |
853 | while (likely(total_rx_packets < (unsigned int)budget)) { |
854 | union ice_32b_rx_flex_desc *rx_desc; |
855 | unsigned int size, xdp_res = 0; |
856 | struct xdp_buff *xdp; |
857 | struct sk_buff *skb; |
858 | u16 stat_err_bits; |
859 | u16 vlan_tci; |
860 | |
861 | rx_desc = ICE_RX_DESC(rx_ring, ntc); |
862 | |
863 | stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S); |
864 | if (!ice_test_staterr(status_err_n: rx_desc->wb.status_error0, stat_err_bits)) |
865 | break; |
866 | |
867 | /* This memory barrier is needed to keep us from reading |
868 | * any other fields out of the rx_desc until we have |
869 | * verified the descriptor has been written back. |
870 | */ |
871 | dma_rmb(); |
872 | |
873 | if (unlikely(ntc == ntu)) |
874 | break; |
875 | |
876 | xdp = *ice_xdp_buf(rx_ring, idx: ntc); |
877 | |
878 | size = le16_to_cpu(rx_desc->wb.pkt_len) & |
879 | ICE_RX_FLX_DESC_PKT_LEN_M; |
880 | |
881 | xsk_buff_set_size(xdp, size); |
882 | xsk_buff_dma_sync_for_cpu(xdp, pool: xsk_pool); |
883 | |
884 | if (!first) { |
885 | first = xdp; |
886 | } else if (ice_add_xsk_frag(rx_ring, first, xdp, size)) { |
887 | break; |
888 | } |
889 | |
890 | if (++ntc == cnt) |
891 | ntc = 0; |
892 | |
893 | if (ice_is_non_eop(rx_ring, rx_desc)) |
894 | continue; |
895 | |
896 | xdp_res = ice_run_xdp_zc(rx_ring, xdp: first, xdp_prog, xdp_ring); |
897 | if (likely(xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR))) { |
898 | xdp_xmit |= xdp_res; |
899 | } else if (xdp_res == ICE_XDP_EXIT) { |
900 | failure = true; |
901 | first = NULL; |
902 | rx_ring->first_desc = ntc; |
903 | break; |
904 | } else if (xdp_res == ICE_XDP_CONSUMED) { |
905 | xsk_buff_free(xdp: first); |
906 | } else if (xdp_res == ICE_XDP_PASS) { |
907 | goto construct_skb; |
908 | } |
909 | |
910 | total_rx_bytes += xdp_get_buff_len(xdp: first); |
911 | total_rx_packets++; |
912 | |
913 | first = NULL; |
914 | rx_ring->first_desc = ntc; |
915 | continue; |
916 | |
917 | construct_skb: |
918 | /* XDP_PASS path */ |
919 | skb = ice_construct_skb_zc(rx_ring, xdp: first); |
920 | if (!skb) { |
921 | rx_ring->ring_stats->rx_stats.alloc_buf_failed++; |
922 | break; |
923 | } |
924 | |
925 | first = NULL; |
926 | rx_ring->first_desc = ntc; |
927 | |
928 | if (eth_skb_pad(skb)) { |
929 | skb = NULL; |
930 | continue; |
931 | } |
932 | |
933 | total_rx_bytes += skb->len; |
934 | total_rx_packets++; |
935 | |
936 | vlan_tci = ice_get_vlan_tci(rx_desc); |
937 | |
938 | ice_process_skb_fields(rx_ring, rx_desc, skb); |
939 | ice_receive_skb(rx_ring, skb, vlan_tci); |
940 | } |
941 | |
942 | rx_ring->next_to_clean = ntc; |
943 | entries_to_alloc = ICE_RX_DESC_UNUSED(rx_ring); |
944 | if (entries_to_alloc > ICE_RING_QUARTER(rx_ring)) |
945 | failure |= !ice_alloc_rx_bufs_zc(rx_ring, count: entries_to_alloc); |
946 | |
947 | ice_finalize_xdp_rx(xdp_ring, xdp_res: xdp_xmit, first_idx: 0); |
948 | ice_update_rx_ring_stats(ring: rx_ring, pkts: total_rx_packets, bytes: total_rx_bytes); |
949 | |
950 | if (xsk_uses_need_wakeup(pool: xsk_pool)) { |
951 | /* ntu could have changed when allocating entries above, so |
952 | * use rx_ring value instead of stack based one |
953 | */ |
954 | if (failure || ntc == rx_ring->next_to_use) |
955 | xsk_set_rx_need_wakeup(pool: xsk_pool); |
956 | else |
957 | xsk_clear_rx_need_wakeup(pool: xsk_pool); |
958 | |
959 | return (int)total_rx_packets; |
960 | } |
961 | |
962 | return failure ? budget : (int)total_rx_packets; |
963 | } |
964 | |
965 | /** |
966 | * ice_xmit_pkt - produce a single HW Tx descriptor out of AF_XDP descriptor |
967 | * @xdp_ring: XDP ring to produce the HW Tx descriptor on |
968 | * @desc: AF_XDP descriptor to pull the DMA address and length from |
969 | * @total_bytes: bytes accumulator that will be used for stats update |
970 | */ |
971 | static void ice_xmit_pkt(struct ice_tx_ring *xdp_ring, struct xdp_desc *desc, |
972 | unsigned int *total_bytes) |
973 | { |
974 | struct ice_tx_desc *tx_desc; |
975 | dma_addr_t dma; |
976 | |
977 | dma = xsk_buff_raw_get_dma(pool: xdp_ring->xsk_pool, addr: desc->addr); |
978 | xsk_buff_raw_dma_sync_for_device(pool: xdp_ring->xsk_pool, dma, size: desc->len); |
979 | |
980 | tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_to_use++); |
981 | tx_desc->buf_addr = cpu_to_le64(dma); |
982 | tx_desc->cmd_type_offset_bsz = ice_build_ctob(td_cmd: xsk_is_eop_desc(desc), |
983 | td_offset: 0, size: desc->len, td_tag: 0); |
984 | |
985 | *total_bytes += desc->len; |
986 | } |
987 | |
988 | /** |
989 | * ice_xmit_pkt_batch - produce a batch of HW Tx descriptors out of AF_XDP descriptors |
990 | * @xdp_ring: XDP ring to produce the HW Tx descriptors on |
991 | * @descs: AF_XDP descriptors to pull the DMA addresses and lengths from |
992 | * @total_bytes: bytes accumulator that will be used for stats update |
993 | */ |
994 | static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring, struct xdp_desc *descs, |
995 | unsigned int *total_bytes) |
996 | { |
997 | u16 ntu = xdp_ring->next_to_use; |
998 | struct ice_tx_desc *tx_desc; |
999 | u32 i; |
1000 | |
1001 | loop_unrolled_for(i = 0; i < PKTS_PER_BATCH; i++) { |
1002 | dma_addr_t dma; |
1003 | |
1004 | dma = xsk_buff_raw_get_dma(pool: xdp_ring->xsk_pool, addr: descs[i].addr); |
1005 | xsk_buff_raw_dma_sync_for_device(pool: xdp_ring->xsk_pool, dma, size: descs[i].len); |
1006 | |
1007 | tx_desc = ICE_TX_DESC(xdp_ring, ntu++); |
1008 | tx_desc->buf_addr = cpu_to_le64(dma); |
1009 | tx_desc->cmd_type_offset_bsz = ice_build_ctob(td_cmd: xsk_is_eop_desc(desc: &descs[i]), |
1010 | td_offset: 0, size: descs[i].len, td_tag: 0); |
1011 | |
1012 | *total_bytes += descs[i].len; |
1013 | } |
1014 | |
1015 | xdp_ring->next_to_use = ntu; |
1016 | } |
1017 | |
1018 | /** |
1019 | * ice_fill_tx_hw_ring - produce the number of Tx descriptors onto ring |
1020 | * @xdp_ring: XDP ring to produce the HW Tx descriptors on |
1021 | * @descs: AF_XDP descriptors to pull the DMA addresses and lengths from |
1022 | * @nb_pkts: count of packets to be send |
1023 | * @total_bytes: bytes accumulator that will be used for stats update |
1024 | */ |
1025 | static void ice_fill_tx_hw_ring(struct ice_tx_ring *xdp_ring, struct xdp_desc *descs, |
1026 | u32 nb_pkts, unsigned int *total_bytes) |
1027 | { |
1028 | u32 batched, leftover, i; |
1029 | |
1030 | batched = ALIGN_DOWN(nb_pkts, PKTS_PER_BATCH); |
1031 | leftover = nb_pkts & (PKTS_PER_BATCH - 1); |
1032 | for (i = 0; i < batched; i += PKTS_PER_BATCH) |
1033 | ice_xmit_pkt_batch(xdp_ring, descs: &descs[i], total_bytes); |
1034 | for (; i < batched + leftover; i++) |
1035 | ice_xmit_pkt(xdp_ring, desc: &descs[i], total_bytes); |
1036 | } |
1037 | |
1038 | /** |
1039 | * ice_xmit_zc - take entries from XSK Tx ring and place them onto HW Tx ring |
1040 | * @xdp_ring: XDP ring to produce the HW Tx descriptors on |
1041 | * |
1042 | * Returns true if there is no more work that needs to be done, false otherwise |
1043 | */ |
1044 | bool ice_xmit_zc(struct ice_tx_ring *xdp_ring) |
1045 | { |
1046 | struct xdp_desc *descs = xdp_ring->xsk_pool->tx_descs; |
1047 | u32 nb_pkts, nb_processed = 0; |
1048 | unsigned int total_bytes = 0; |
1049 | int budget; |
1050 | |
1051 | ice_clean_xdp_irq_zc(xdp_ring); |
1052 | |
1053 | budget = ICE_DESC_UNUSED(xdp_ring); |
1054 | budget = min_t(u16, budget, ICE_RING_QUARTER(xdp_ring)); |
1055 | |
1056 | nb_pkts = xsk_tx_peek_release_desc_batch(pool: xdp_ring->xsk_pool, max: budget); |
1057 | if (!nb_pkts) |
1058 | return true; |
1059 | |
1060 | if (xdp_ring->next_to_use + nb_pkts >= xdp_ring->count) { |
1061 | nb_processed = xdp_ring->count - xdp_ring->next_to_use; |
1062 | ice_fill_tx_hw_ring(xdp_ring, descs, nb_pkts: nb_processed, total_bytes: &total_bytes); |
1063 | xdp_ring->next_to_use = 0; |
1064 | } |
1065 | |
1066 | ice_fill_tx_hw_ring(xdp_ring, descs: &descs[nb_processed], nb_pkts: nb_pkts - nb_processed, |
1067 | total_bytes: &total_bytes); |
1068 | |
1069 | ice_set_rs_bit(xdp_ring); |
1070 | ice_xdp_ring_update_tail(xdp_ring); |
1071 | ice_update_tx_ring_stats(ring: xdp_ring, pkts: nb_pkts, bytes: total_bytes); |
1072 | |
1073 | if (xsk_uses_need_wakeup(pool: xdp_ring->xsk_pool)) |
1074 | xsk_set_tx_need_wakeup(pool: xdp_ring->xsk_pool); |
1075 | |
1076 | return nb_pkts < budget; |
1077 | } |
1078 | |
1079 | /** |
1080 | * ice_xsk_wakeup - Implements ndo_xsk_wakeup |
1081 | * @netdev: net_device |
1082 | * @queue_id: queue to wake up |
1083 | * @flags: ignored in our case, since we have Rx and Tx in the same NAPI |
1084 | * |
1085 | * Returns negative on error, zero otherwise. |
1086 | */ |
1087 | int |
1088 | ice_xsk_wakeup(struct net_device *netdev, u32 queue_id, |
1089 | u32 __always_unused flags) |
1090 | { |
1091 | struct ice_netdev_priv *np = netdev_priv(dev: netdev); |
1092 | struct ice_q_vector *q_vector; |
1093 | struct ice_vsi *vsi = np->vsi; |
1094 | struct ice_tx_ring *ring; |
1095 | |
1096 | if (test_bit(ICE_VSI_DOWN, vsi->state)) |
1097 | return -ENETDOWN; |
1098 | |
1099 | if (!ice_is_xdp_ena_vsi(vsi)) |
1100 | return -EINVAL; |
1101 | |
1102 | if (queue_id >= vsi->num_txq || queue_id >= vsi->num_rxq) |
1103 | return -EINVAL; |
1104 | |
1105 | ring = vsi->rx_rings[queue_id]->xdp_ring; |
1106 | |
1107 | if (!ring->xsk_pool) |
1108 | return -EINVAL; |
1109 | |
1110 | /* The idea here is that if NAPI is running, mark a miss, so |
1111 | * it will run again. If not, trigger an interrupt and |
1112 | * schedule the NAPI from interrupt context. If NAPI would be |
1113 | * scheduled here, the interrupt affinity would not be |
1114 | * honored. |
1115 | */ |
1116 | q_vector = ring->q_vector; |
1117 | if (!napi_if_scheduled_mark_missed(n: &q_vector->napi)) |
1118 | ice_trigger_sw_intr(hw: &vsi->back->hw, q_vector); |
1119 | |
1120 | return 0; |
1121 | } |
1122 | |
1123 | /** |
1124 | * ice_xsk_any_rx_ring_ena - Checks if Rx rings have AF_XDP buff pool attached |
1125 | * @vsi: VSI to be checked |
1126 | * |
1127 | * Returns true if any of the Rx rings has an AF_XDP buff pool attached |
1128 | */ |
1129 | bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi) |
1130 | { |
1131 | int i; |
1132 | |
1133 | ice_for_each_rxq(vsi, i) { |
1134 | if (xsk_get_pool_from_qid(dev: vsi->netdev, queue_id: i)) |
1135 | return true; |
1136 | } |
1137 | |
1138 | return false; |
1139 | } |
1140 | |
1141 | /** |
1142 | * ice_xsk_clean_rx_ring - clean buffer pool queues connected to a given Rx ring |
1143 | * @rx_ring: ring to be cleaned |
1144 | */ |
1145 | void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring) |
1146 | { |
1147 | u16 ntc = rx_ring->next_to_clean; |
1148 | u16 ntu = rx_ring->next_to_use; |
1149 | |
1150 | while (ntc != ntu) { |
1151 | struct xdp_buff *xdp = *ice_xdp_buf(rx_ring, idx: ntc); |
1152 | |
1153 | xsk_buff_free(xdp); |
1154 | ntc++; |
1155 | if (ntc >= rx_ring->count) |
1156 | ntc = 0; |
1157 | } |
1158 | } |
1159 | |
1160 | /** |
1161 | * ice_xsk_clean_xdp_ring - Clean the XDP Tx ring and its buffer pool queues |
1162 | * @xdp_ring: XDP_Tx ring |
1163 | */ |
1164 | void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring) |
1165 | { |
1166 | u16 ntc = xdp_ring->next_to_clean, ntu = xdp_ring->next_to_use; |
1167 | u32 xsk_frames = 0; |
1168 | |
1169 | while (ntc != ntu) { |
1170 | struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc]; |
1171 | |
1172 | if (tx_buf->type == ICE_TX_BUF_XSK_TX) { |
1173 | tx_buf->type = ICE_TX_BUF_EMPTY; |
1174 | xsk_buff_free(xdp: tx_buf->xdp); |
1175 | } else { |
1176 | xsk_frames++; |
1177 | } |
1178 | |
1179 | ntc++; |
1180 | if (ntc >= xdp_ring->count) |
1181 | ntc = 0; |
1182 | } |
1183 | |
1184 | if (xsk_frames) |
1185 | xsk_tx_completed(pool: xdp_ring->xsk_pool, nb_entries: xsk_frames); |
1186 | } |
1187 | |