1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2019 Mellanox Technologies. */
3
4#include "tx.h"
5#include "pool.h"
6#include "en/xdp.h"
7#include "en/params.h"
8#include <net/xdp_sock_drv.h>
9
10int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
11{
12 struct mlx5e_priv *priv = netdev_priv(dev);
13 struct mlx5e_params *params = &priv->channels.params;
14 struct mlx5e_channel *c;
15
16 if (unlikely(!mlx5e_xdp_is_active(priv)))
17 return -ENETDOWN;
18
19 if (unlikely(qid >= params->num_channels))
20 return -EINVAL;
21
22 c = priv->channels.c[qid];
23
24 if (!napi_if_scheduled_mark_missed(n: &c->napi)) {
25 /* To avoid WQE overrun, don't post a NOP if async_icosq is not
26 * active and not polled by NAPI. Return 0, because the upcoming
27 * activate will trigger the IRQ for us.
28 */
29 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->async_icosq.state)))
30 return 0;
31
32 if (test_and_set_bit(nr: MLX5E_SQ_STATE_PENDING_XSK_TX, addr: &c->async_icosq.state))
33 return 0;
34
35 mlx5e_trigger_napi_icosq(c);
36 }
37
38 return 0;
39}
40
41/* When TX fails (because of the size of the packet), we need to get completions
42 * in order, so post a NOP to get a CQE. Since AF_XDP doesn't distinguish
43 * between successful TX and errors, handling in mlx5e_poll_xdpsq_cq is the
44 * same.
45 */
46static void mlx5e_xsk_tx_post_err(struct mlx5e_xdpsq *sq,
47 union mlx5e_xdp_info *xdpi)
48{
49 u16 pi = mlx5_wq_cyc_ctr2ix(wq: &sq->wq, ctr: sq->pc);
50 struct mlx5e_xdp_wqe_info *wi = &sq->db.wqe_info[pi];
51 struct mlx5e_tx_wqe *nopwqe;
52
53 wi->num_wqebbs = 1;
54 wi->num_pkts = 1;
55
56 nopwqe = mlx5e_post_nop(wq: &sq->wq, sqn: sq->sqn, pc: &sq->pc);
57 mlx5e_xdpi_fifo_push(fifo: &sq->db.xdpi_fifo, xi: *xdpi);
58 if (xp_tx_metadata_enabled(pool: sq->xsk_pool))
59 mlx5e_xdpi_fifo_push(fifo: &sq->db.xdpi_fifo,
60 xi: (union mlx5e_xdp_info) { .xsk_meta = {} });
61 sq->doorbell_cseg = &nopwqe->ctrl;
62}
63
64bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
65{
66 struct xsk_buff_pool *pool = sq->xsk_pool;
67 struct xsk_tx_metadata *meta = NULL;
68 union mlx5e_xdp_info xdpi;
69 bool work_done = true;
70 bool flush = false;
71
72 xdpi.mode = MLX5E_XDP_XMIT_MODE_XSK;
73
74 for (; budget; budget--) {
75 int check_result = INDIRECT_CALL_2(sq->xmit_xdp_frame_check,
76 mlx5e_xmit_xdp_frame_check_mpwqe,
77 mlx5e_xmit_xdp_frame_check,
78 sq);
79 struct mlx5e_xmit_data xdptxd = {};
80 struct xdp_desc desc;
81 bool ret;
82
83 if (unlikely(check_result < 0)) {
84 work_done = false;
85 break;
86 }
87
88 if (!xsk_tx_peek_desc(pool, desc: &desc)) {
89 /* TX will get stuck until something wakes it up by
90 * triggering NAPI. Currently it's expected that the
91 * application calls sendto() if there are consumed, but
92 * not completed frames.
93 */
94 break;
95 }
96
97 xdptxd.dma_addr = xsk_buff_raw_get_dma(pool, addr: desc.addr);
98 xdptxd.data = xsk_buff_raw_get_data(pool, addr: desc.addr);
99 xdptxd.len = desc.len;
100 meta = xsk_buff_get_metadata(pool, addr: desc.addr);
101
102 xsk_buff_raw_dma_sync_for_device(pool, dma: xdptxd.dma_addr, size: xdptxd.len);
103
104 ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,
105 mlx5e_xmit_xdp_frame, sq, &xdptxd,
106 check_result, meta);
107 if (unlikely(!ret)) {
108 if (sq->mpwqe.wqe)
109 mlx5e_xdp_mpwqe_complete(sq);
110
111 mlx5e_xsk_tx_post_err(sq, xdpi: &xdpi);
112 } else {
113 mlx5e_xdpi_fifo_push(fifo: &sq->db.xdpi_fifo, xi: xdpi);
114 if (xp_tx_metadata_enabled(pool: sq->xsk_pool)) {
115 struct xsk_tx_metadata_compl compl;
116
117 xsk_tx_metadata_to_compl(meta, compl: &compl);
118 XSK_TX_COMPL_FITS(void *);
119
120 mlx5e_xdpi_fifo_push(fifo: &sq->db.xdpi_fifo,
121 xi: (union mlx5e_xdp_info)
122 { .xsk_meta = compl });
123 }
124 }
125
126 flush = true;
127 }
128
129 if (flush) {
130 if (sq->mpwqe.wqe)
131 mlx5e_xdp_mpwqe_complete(sq);
132 mlx5e_xmit_xdp_doorbell(sq);
133
134 xsk_tx_release(pool);
135 }
136
137 return !(budget && work_done);
138}
139

source code of linux/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c