1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2019 Mellanox Technologies. */ |
3 | |
4 | #include "tx.h" |
5 | #include "pool.h" |
6 | #include "en/xdp.h" |
7 | #include "en/params.h" |
8 | #include <net/xdp_sock_drv.h> |
9 | |
10 | int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) |
11 | { |
12 | struct mlx5e_priv *priv = netdev_priv(dev); |
13 | struct mlx5e_params *params = &priv->channels.params; |
14 | struct mlx5e_channel *c; |
15 | |
16 | if (unlikely(!mlx5e_xdp_is_active(priv))) |
17 | return -ENETDOWN; |
18 | |
19 | if (unlikely(qid >= params->num_channels)) |
20 | return -EINVAL; |
21 | |
22 | c = priv->channels.c[qid]; |
23 | |
24 | if (!napi_if_scheduled_mark_missed(n: &c->napi)) { |
25 | /* To avoid WQE overrun, don't post a NOP if async_icosq is not |
26 | * active and not polled by NAPI. Return 0, because the upcoming |
27 | * activate will trigger the IRQ for us. |
28 | */ |
29 | if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->async_icosq.state))) |
30 | return 0; |
31 | |
32 | if (test_and_set_bit(nr: MLX5E_SQ_STATE_PENDING_XSK_TX, addr: &c->async_icosq.state)) |
33 | return 0; |
34 | |
35 | mlx5e_trigger_napi_icosq(c); |
36 | } |
37 | |
38 | return 0; |
39 | } |
40 | |
41 | /* When TX fails (because of the size of the packet), we need to get completions |
42 | * in order, so post a NOP to get a CQE. Since AF_XDP doesn't distinguish |
43 | * between successful TX and errors, handling in mlx5e_poll_xdpsq_cq is the |
44 | * same. |
45 | */ |
46 | static void mlx5e_xsk_tx_post_err(struct mlx5e_xdpsq *sq, |
47 | union mlx5e_xdp_info *xdpi) |
48 | { |
49 | u16 pi = mlx5_wq_cyc_ctr2ix(wq: &sq->wq, ctr: sq->pc); |
50 | struct mlx5e_xdp_wqe_info *wi = &sq->db.wqe_info[pi]; |
51 | struct mlx5e_tx_wqe *nopwqe; |
52 | |
53 | wi->num_wqebbs = 1; |
54 | wi->num_pkts = 1; |
55 | |
56 | nopwqe = mlx5e_post_nop(wq: &sq->wq, sqn: sq->sqn, pc: &sq->pc); |
57 | mlx5e_xdpi_fifo_push(fifo: &sq->db.xdpi_fifo, xi: *xdpi); |
58 | if (xp_tx_metadata_enabled(pool: sq->xsk_pool)) |
59 | mlx5e_xdpi_fifo_push(fifo: &sq->db.xdpi_fifo, |
60 | xi: (union mlx5e_xdp_info) { .xsk_meta = {} }); |
61 | sq->doorbell_cseg = &nopwqe->ctrl; |
62 | } |
63 | |
64 | bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget) |
65 | { |
66 | struct xsk_buff_pool *pool = sq->xsk_pool; |
67 | struct xsk_tx_metadata *meta = NULL; |
68 | union mlx5e_xdp_info xdpi; |
69 | bool work_done = true; |
70 | bool flush = false; |
71 | |
72 | xdpi.mode = MLX5E_XDP_XMIT_MODE_XSK; |
73 | |
74 | for (; budget; budget--) { |
75 | int check_result = INDIRECT_CALL_2(sq->xmit_xdp_frame_check, |
76 | mlx5e_xmit_xdp_frame_check_mpwqe, |
77 | mlx5e_xmit_xdp_frame_check, |
78 | sq); |
79 | struct mlx5e_xmit_data xdptxd = {}; |
80 | struct xdp_desc desc; |
81 | bool ret; |
82 | |
83 | if (unlikely(check_result < 0)) { |
84 | work_done = false; |
85 | break; |
86 | } |
87 | |
88 | if (!xsk_tx_peek_desc(pool, desc: &desc)) { |
89 | /* TX will get stuck until something wakes it up by |
90 | * triggering NAPI. Currently it's expected that the |
91 | * application calls sendto() if there are consumed, but |
92 | * not completed frames. |
93 | */ |
94 | break; |
95 | } |
96 | |
97 | xdptxd.dma_addr = xsk_buff_raw_get_dma(pool, addr: desc.addr); |
98 | xdptxd.data = xsk_buff_raw_get_data(pool, addr: desc.addr); |
99 | xdptxd.len = desc.len; |
100 | meta = xsk_buff_get_metadata(pool, addr: desc.addr); |
101 | |
102 | xsk_buff_raw_dma_sync_for_device(pool, dma: xdptxd.dma_addr, size: xdptxd.len); |
103 | |
104 | ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe, |
105 | mlx5e_xmit_xdp_frame, sq, &xdptxd, |
106 | check_result, meta); |
107 | if (unlikely(!ret)) { |
108 | if (sq->mpwqe.wqe) |
109 | mlx5e_xdp_mpwqe_complete(sq); |
110 | |
111 | mlx5e_xsk_tx_post_err(sq, xdpi: &xdpi); |
112 | } else { |
113 | mlx5e_xdpi_fifo_push(fifo: &sq->db.xdpi_fifo, xi: xdpi); |
114 | if (xp_tx_metadata_enabled(pool: sq->xsk_pool)) { |
115 | struct xsk_tx_metadata_compl compl; |
116 | |
117 | xsk_tx_metadata_to_compl(meta, compl: &compl); |
118 | XSK_TX_COMPL_FITS(void *); |
119 | |
120 | mlx5e_xdpi_fifo_push(fifo: &sq->db.xdpi_fifo, |
121 | xi: (union mlx5e_xdp_info) |
122 | { .xsk_meta = compl }); |
123 | } |
124 | } |
125 | |
126 | flush = true; |
127 | } |
128 | |
129 | if (flush) { |
130 | if (sq->mpwqe.wqe) |
131 | mlx5e_xdp_mpwqe_complete(sq); |
132 | mlx5e_xmit_xdp_doorbell(sq); |
133 | |
134 | xsk_tx_release(pool); |
135 | } |
136 | |
137 | return !(budget && work_done); |
138 | } |
139 | |