1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2019-2020, Mellanox Technologies inc. All rights reserved. */
3
4#include <net/xdp_sock_drv.h>
5#include "pool.h"
6#include "setup.h"
7#include "en/params.h"
8
9static int mlx5e_xsk_map_pool(struct mlx5_core_dev *mdev,
10 struct xsk_buff_pool *pool)
11{
12 struct device *dev = mlx5_core_dma_dev(dev: mdev);
13
14 return xsk_pool_dma_map(pool, dev, DMA_ATTR_SKIP_CPU_SYNC);
15}
16
17static void mlx5e_xsk_unmap_pool(struct mlx5e_priv *priv,
18 struct xsk_buff_pool *pool)
19{
20 return xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC);
21}
22
23static int mlx5e_xsk_get_pools(struct mlx5e_xsk *xsk)
24{
25 if (!xsk->pools) {
26 xsk->pools = kcalloc(MLX5E_MAX_NUM_CHANNELS,
27 size: sizeof(*xsk->pools), GFP_KERNEL);
28 if (unlikely(!xsk->pools))
29 return -ENOMEM;
30 }
31
32 xsk->refcnt++;
33 xsk->ever_used = true;
34
35 return 0;
36}
37
38static void mlx5e_xsk_put_pools(struct mlx5e_xsk *xsk)
39{
40 if (!--xsk->refcnt) {
41 kfree(objp: xsk->pools);
42 xsk->pools = NULL;
43 }
44}
45
46static int mlx5e_xsk_add_pool(struct mlx5e_xsk *xsk, struct xsk_buff_pool *pool, u16 ix)
47{
48 int err;
49
50 err = mlx5e_xsk_get_pools(xsk);
51 if (unlikely(err))
52 return err;
53
54 xsk->pools[ix] = pool;
55 return 0;
56}
57
58static void mlx5e_xsk_remove_pool(struct mlx5e_xsk *xsk, u16 ix)
59{
60 xsk->pools[ix] = NULL;
61
62 mlx5e_xsk_put_pools(xsk);
63}
64
65static bool mlx5e_xsk_is_pool_sane(struct xsk_buff_pool *pool)
66{
67 return xsk_pool_get_headroom(pool) <= 0xffff &&
68 xsk_pool_get_chunk_size(pool) <= 0xffff;
69}
70
71void mlx5e_build_xsk_param(struct xsk_buff_pool *pool, struct mlx5e_xsk_param *xsk)
72{
73 xsk->headroom = xsk_pool_get_headroom(pool);
74 xsk->chunk_size = xsk_pool_get_chunk_size(pool);
75 xsk->unaligned = pool->unaligned;
76}
77
78static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
79 struct xsk_buff_pool *pool, u16 ix)
80{
81 struct mlx5e_params *params = &priv->channels.params;
82 struct mlx5e_xsk_param xsk;
83 struct mlx5e_channel *c;
84 int err;
85
86 if (unlikely(mlx5e_xsk_get_pool(&priv->channels.params, &priv->xsk, ix)))
87 return -EBUSY;
88
89 if (unlikely(!mlx5e_xsk_is_pool_sane(pool)))
90 return -EINVAL;
91
92 err = mlx5e_xsk_map_pool(mdev: mlx5_sd_ch_ix_get_dev(primary: priv->mdev, ch_ix: ix), pool);
93 if (unlikely(err))
94 return err;
95
96 err = mlx5e_xsk_add_pool(xsk: &priv->xsk, pool, ix);
97 if (unlikely(err))
98 goto err_unmap_pool;
99
100 mlx5e_build_xsk_param(pool, xsk: &xsk);
101
102 if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
103 mlx5e_mpwrq_umr_mode(mdev: priv->mdev, xsk: &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) {
104 const char *recommendation = is_power_of_2(n: xsk.chunk_size) ?
105 "Upgrade firmware" : "Disable striding RQ";
106
107 mlx5_core_warn(priv->mdev, "Expected slowdown with XSK frame size %u. %s for better performance.\n",
108 xsk.chunk_size, recommendation);
109 }
110
111 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
112 /* XSK objects will be created on open. */
113 goto validate_closed;
114 }
115
116 if (!params->xdp_prog) {
117 /* XSK objects will be created when an XDP program is set,
118 * and the channels are reopened.
119 */
120 goto validate_closed;
121 }
122
123 c = priv->channels.c[ix];
124
125 err = mlx5e_open_xsk(priv, params, xsk: &xsk, pool, c);
126 if (unlikely(err))
127 goto err_remove_pool;
128
129 mlx5e_activate_xsk(c);
130 mlx5e_trigger_napi_icosq(c);
131
132 /* Don't wait for WQEs, because the newer xdpsock sample doesn't provide
133 * any Fill Ring entries at the setup stage.
134 */
135
136 mlx5e_rx_res_xsk_update(res: priv->rx_res, chs: &priv->channels, ix, xsk: true);
137
138 mlx5e_deactivate_rq(rq: &c->rq);
139 mlx5e_flush_rq(rq: &c->rq, curr_state: MLX5_RQC_STATE_RDY);
140
141 return 0;
142
143err_remove_pool:
144 mlx5e_xsk_remove_pool(xsk: &priv->xsk, ix);
145
146err_unmap_pool:
147 mlx5e_xsk_unmap_pool(priv, pool);
148
149 return err;
150
151validate_closed:
152 /* Check the configuration in advance, rather than fail at a later stage
153 * (in mlx5e_xdp_set or on open) and end up with no channels.
154 */
155 if (!mlx5e_validate_xsk_param(params, xsk: &xsk, mdev: priv->mdev)) {
156 err = -EINVAL;
157 goto err_remove_pool;
158 }
159
160 return 0;
161}
162
163static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix)
164{
165 struct xsk_buff_pool *pool = mlx5e_xsk_get_pool(params: &priv->channels.params,
166 xsk: &priv->xsk, ix);
167 struct mlx5e_channel *c;
168
169 if (unlikely(!pool))
170 return -EINVAL;
171
172 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
173 goto remove_pool;
174
175 /* XSK RQ and SQ are only created if XDP program is set. */
176 if (!priv->channels.params.xdp_prog)
177 goto remove_pool;
178
179 c = priv->channels.c[ix];
180
181 mlx5e_activate_rq(rq: &c->rq);
182 mlx5e_trigger_napi_icosq(c);
183 mlx5e_wait_for_min_rx_wqes(rq: &c->rq, MLX5E_RQ_WQES_TIMEOUT);
184
185 mlx5e_rx_res_xsk_update(res: priv->rx_res, chs: &priv->channels, ix, xsk: false);
186
187 mlx5e_deactivate_xsk(c);
188 mlx5e_close_xsk(c);
189
190remove_pool:
191 mlx5e_xsk_remove_pool(xsk: &priv->xsk, ix);
192 mlx5e_xsk_unmap_pool(priv, pool);
193
194 return 0;
195}
196
197static int mlx5e_xsk_enable_pool(struct mlx5e_priv *priv, struct xsk_buff_pool *pool,
198 u16 ix)
199{
200 int err;
201
202 mutex_lock(&priv->state_lock);
203 err = mlx5e_xsk_enable_locked(priv, pool, ix);
204 mutex_unlock(lock: &priv->state_lock);
205
206 return err;
207}
208
209static int mlx5e_xsk_disable_pool(struct mlx5e_priv *priv, u16 ix)
210{
211 int err;
212
213 mutex_lock(&priv->state_lock);
214 err = mlx5e_xsk_disable_locked(priv, ix);
215 mutex_unlock(lock: &priv->state_lock);
216
217 return err;
218}
219
220int mlx5e_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16 qid)
221{
222 struct mlx5e_priv *priv = netdev_priv(dev);
223 struct mlx5e_params *params = &priv->channels.params;
224
225 if (unlikely(qid >= params->num_channels))
226 return -EINVAL;
227
228 return pool ? mlx5e_xsk_enable_pool(priv, pool, ix: qid) :
229 mlx5e_xsk_disable_pool(priv, ix: qid);
230}
231

source code of linux/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c