1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2019-2020, Mellanox Technologies inc. All rights reserved. */ |
3 | |
4 | #include <net/xdp_sock_drv.h> |
5 | #include "pool.h" |
6 | #include "setup.h" |
7 | #include "en/params.h" |
8 | |
9 | static int mlx5e_xsk_map_pool(struct mlx5_core_dev *mdev, |
10 | struct xsk_buff_pool *pool) |
11 | { |
12 | struct device *dev = mlx5_core_dma_dev(dev: mdev); |
13 | |
14 | return xsk_pool_dma_map(pool, dev, DMA_ATTR_SKIP_CPU_SYNC); |
15 | } |
16 | |
17 | static void mlx5e_xsk_unmap_pool(struct mlx5e_priv *priv, |
18 | struct xsk_buff_pool *pool) |
19 | { |
20 | return xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC); |
21 | } |
22 | |
23 | static int mlx5e_xsk_get_pools(struct mlx5e_xsk *xsk) |
24 | { |
25 | if (!xsk->pools) { |
26 | xsk->pools = kcalloc(MLX5E_MAX_NUM_CHANNELS, |
27 | size: sizeof(*xsk->pools), GFP_KERNEL); |
28 | if (unlikely(!xsk->pools)) |
29 | return -ENOMEM; |
30 | } |
31 | |
32 | xsk->refcnt++; |
33 | xsk->ever_used = true; |
34 | |
35 | return 0; |
36 | } |
37 | |
38 | static void mlx5e_xsk_put_pools(struct mlx5e_xsk *xsk) |
39 | { |
40 | if (!--xsk->refcnt) { |
41 | kfree(objp: xsk->pools); |
42 | xsk->pools = NULL; |
43 | } |
44 | } |
45 | |
46 | static int mlx5e_xsk_add_pool(struct mlx5e_xsk *xsk, struct xsk_buff_pool *pool, u16 ix) |
47 | { |
48 | int err; |
49 | |
50 | err = mlx5e_xsk_get_pools(xsk); |
51 | if (unlikely(err)) |
52 | return err; |
53 | |
54 | xsk->pools[ix] = pool; |
55 | return 0; |
56 | } |
57 | |
58 | static void mlx5e_xsk_remove_pool(struct mlx5e_xsk *xsk, u16 ix) |
59 | { |
60 | xsk->pools[ix] = NULL; |
61 | |
62 | mlx5e_xsk_put_pools(xsk); |
63 | } |
64 | |
65 | static bool mlx5e_xsk_is_pool_sane(struct xsk_buff_pool *pool) |
66 | { |
67 | return xsk_pool_get_headroom(pool) <= 0xffff && |
68 | xsk_pool_get_chunk_size(pool) <= 0xffff; |
69 | } |
70 | |
71 | void mlx5e_build_xsk_param(struct xsk_buff_pool *pool, struct mlx5e_xsk_param *xsk) |
72 | { |
73 | xsk->headroom = xsk_pool_get_headroom(pool); |
74 | xsk->chunk_size = xsk_pool_get_chunk_size(pool); |
75 | xsk->unaligned = pool->unaligned; |
76 | } |
77 | |
78 | static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, |
79 | struct xsk_buff_pool *pool, u16 ix) |
80 | { |
81 | struct mlx5e_params *params = &priv->channels.params; |
82 | struct mlx5e_xsk_param xsk; |
83 | struct mlx5e_channel *c; |
84 | int err; |
85 | |
86 | if (unlikely(mlx5e_xsk_get_pool(&priv->channels.params, &priv->xsk, ix))) |
87 | return -EBUSY; |
88 | |
89 | if (unlikely(!mlx5e_xsk_is_pool_sane(pool))) |
90 | return -EINVAL; |
91 | |
92 | err = mlx5e_xsk_map_pool(mdev: mlx5_sd_ch_ix_get_dev(primary: priv->mdev, ch_ix: ix), pool); |
93 | if (unlikely(err)) |
94 | return err; |
95 | |
96 | err = mlx5e_xsk_add_pool(xsk: &priv->xsk, pool, ix); |
97 | if (unlikely(err)) |
98 | goto err_unmap_pool; |
99 | |
100 | mlx5e_build_xsk_param(pool, xsk: &xsk); |
101 | |
102 | if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ && |
103 | mlx5e_mpwrq_umr_mode(mdev: priv->mdev, xsk: &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) { |
104 | const char *recommendation = is_power_of_2(n: xsk.chunk_size) ? |
105 | "Upgrade firmware" : "Disable striding RQ" ; |
106 | |
107 | mlx5_core_warn(priv->mdev, "Expected slowdown with XSK frame size %u. %s for better performance.\n" , |
108 | xsk.chunk_size, recommendation); |
109 | } |
110 | |
111 | if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { |
112 | /* XSK objects will be created on open. */ |
113 | goto validate_closed; |
114 | } |
115 | |
116 | if (!params->xdp_prog) { |
117 | /* XSK objects will be created when an XDP program is set, |
118 | * and the channels are reopened. |
119 | */ |
120 | goto validate_closed; |
121 | } |
122 | |
123 | c = priv->channels.c[ix]; |
124 | |
125 | err = mlx5e_open_xsk(priv, params, xsk: &xsk, pool, c); |
126 | if (unlikely(err)) |
127 | goto err_remove_pool; |
128 | |
129 | mlx5e_activate_xsk(c); |
130 | mlx5e_trigger_napi_icosq(c); |
131 | |
132 | /* Don't wait for WQEs, because the newer xdpsock sample doesn't provide |
133 | * any Fill Ring entries at the setup stage. |
134 | */ |
135 | |
136 | mlx5e_rx_res_xsk_update(res: priv->rx_res, chs: &priv->channels, ix, xsk: true); |
137 | |
138 | mlx5e_deactivate_rq(rq: &c->rq); |
139 | mlx5e_flush_rq(rq: &c->rq, curr_state: MLX5_RQC_STATE_RDY); |
140 | |
141 | return 0; |
142 | |
143 | err_remove_pool: |
144 | mlx5e_xsk_remove_pool(xsk: &priv->xsk, ix); |
145 | |
146 | err_unmap_pool: |
147 | mlx5e_xsk_unmap_pool(priv, pool); |
148 | |
149 | return err; |
150 | |
151 | validate_closed: |
152 | /* Check the configuration in advance, rather than fail at a later stage |
153 | * (in mlx5e_xdp_set or on open) and end up with no channels. |
154 | */ |
155 | if (!mlx5e_validate_xsk_param(params, xsk: &xsk, mdev: priv->mdev)) { |
156 | err = -EINVAL; |
157 | goto err_remove_pool; |
158 | } |
159 | |
160 | return 0; |
161 | } |
162 | |
163 | static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix) |
164 | { |
165 | struct xsk_buff_pool *pool = mlx5e_xsk_get_pool(params: &priv->channels.params, |
166 | xsk: &priv->xsk, ix); |
167 | struct mlx5e_channel *c; |
168 | |
169 | if (unlikely(!pool)) |
170 | return -EINVAL; |
171 | |
172 | if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) |
173 | goto remove_pool; |
174 | |
175 | /* XSK RQ and SQ are only created if XDP program is set. */ |
176 | if (!priv->channels.params.xdp_prog) |
177 | goto remove_pool; |
178 | |
179 | c = priv->channels.c[ix]; |
180 | |
181 | mlx5e_activate_rq(rq: &c->rq); |
182 | mlx5e_trigger_napi_icosq(c); |
183 | mlx5e_wait_for_min_rx_wqes(rq: &c->rq, MLX5E_RQ_WQES_TIMEOUT); |
184 | |
185 | mlx5e_rx_res_xsk_update(res: priv->rx_res, chs: &priv->channels, ix, xsk: false); |
186 | |
187 | mlx5e_deactivate_xsk(c); |
188 | mlx5e_close_xsk(c); |
189 | |
190 | remove_pool: |
191 | mlx5e_xsk_remove_pool(xsk: &priv->xsk, ix); |
192 | mlx5e_xsk_unmap_pool(priv, pool); |
193 | |
194 | return 0; |
195 | } |
196 | |
197 | static int mlx5e_xsk_enable_pool(struct mlx5e_priv *priv, struct xsk_buff_pool *pool, |
198 | u16 ix) |
199 | { |
200 | int err; |
201 | |
202 | mutex_lock(&priv->state_lock); |
203 | err = mlx5e_xsk_enable_locked(priv, pool, ix); |
204 | mutex_unlock(lock: &priv->state_lock); |
205 | |
206 | return err; |
207 | } |
208 | |
209 | static int mlx5e_xsk_disable_pool(struct mlx5e_priv *priv, u16 ix) |
210 | { |
211 | int err; |
212 | |
213 | mutex_lock(&priv->state_lock); |
214 | err = mlx5e_xsk_disable_locked(priv, ix); |
215 | mutex_unlock(lock: &priv->state_lock); |
216 | |
217 | return err; |
218 | } |
219 | |
220 | int mlx5e_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16 qid) |
221 | { |
222 | struct mlx5e_priv *priv = netdev_priv(dev); |
223 | struct mlx5e_params *params = &priv->channels.params; |
224 | |
225 | if (unlikely(qid >= params->num_channels)) |
226 | return -EINVAL; |
227 | |
228 | return pool ? mlx5e_xsk_enable_pool(priv, pool, ix: qid) : |
229 | mlx5e_xsk_disable_pool(priv, ix: qid); |
230 | } |
231 | |