1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2015-2019 Netronome Systems, Inc. */ |
3 | |
4 | #include "nfp_app.h" |
5 | #include "nfp_net_dp.h" |
6 | #include "nfp_net_xsk.h" |
7 | |
8 | /** |
9 | * nfp_net_rx_alloc_one() - Allocate and map page frag for RX |
10 | * @dp: NFP Net data path struct |
11 | * @dma_addr: Pointer to storage for DMA address (output param) |
12 | * |
13 | * This function will allcate a new page frag, map it for DMA. |
14 | * |
15 | * Return: allocated page frag or NULL on failure. |
16 | */ |
17 | void *nfp_net_rx_alloc_one(struct nfp_net_dp *dp, dma_addr_t *dma_addr) |
18 | { |
19 | void *frag; |
20 | |
21 | if (!dp->xdp_prog) { |
22 | frag = netdev_alloc_frag(fragsz: dp->fl_bufsz); |
23 | } else { |
24 | struct page *page; |
25 | |
26 | page = alloc_page(GFP_KERNEL); |
27 | frag = page ? page_address(page) : NULL; |
28 | } |
29 | if (!frag) { |
30 | nn_dp_warn(dp, "Failed to alloc receive page frag\n" ); |
31 | return NULL; |
32 | } |
33 | |
34 | *dma_addr = nfp_net_dma_map_rx(dp, frag); |
35 | if (dma_mapping_error(dev: dp->dev, dma_addr: *dma_addr)) { |
36 | nfp_net_free_frag(frag, xdp: dp->xdp_prog); |
37 | nn_dp_warn(dp, "Failed to map DMA RX buffer\n" ); |
38 | return NULL; |
39 | } |
40 | |
41 | return frag; |
42 | } |
43 | |
44 | /** |
45 | * nfp_net_tx_ring_init() - Fill in the boilerplate for a TX ring |
46 | * @tx_ring: TX ring structure |
47 | * @dp: NFP Net data path struct |
48 | * @r_vec: IRQ vector servicing this ring |
49 | * @idx: Ring index |
50 | * @is_xdp: Is this an XDP TX ring? |
51 | */ |
52 | static void |
53 | nfp_net_tx_ring_init(struct nfp_net_tx_ring *tx_ring, struct nfp_net_dp *dp, |
54 | struct nfp_net_r_vector *r_vec, unsigned int idx, |
55 | bool is_xdp) |
56 | { |
57 | struct nfp_net *nn = r_vec->nfp_net; |
58 | |
59 | tx_ring->idx = idx; |
60 | tx_ring->r_vec = r_vec; |
61 | tx_ring->is_xdp = is_xdp; |
62 | u64_stats_init(syncp: &tx_ring->r_vec->tx_sync); |
63 | |
64 | tx_ring->qcidx = tx_ring->idx * nn->stride_tx; |
65 | tx_ring->txrwb = dp->txrwb ? &dp->txrwb[idx] : NULL; |
66 | tx_ring->qcp_q = nn->tx_bar + NFP_QCP_QUEUE_OFF(tx_ring->qcidx); |
67 | } |
68 | |
69 | /** |
70 | * nfp_net_rx_ring_init() - Fill in the boilerplate for a RX ring |
71 | * @rx_ring: RX ring structure |
72 | * @r_vec: IRQ vector servicing this ring |
73 | * @idx: Ring index |
74 | */ |
75 | static void |
76 | nfp_net_rx_ring_init(struct nfp_net_rx_ring *rx_ring, |
77 | struct nfp_net_r_vector *r_vec, unsigned int idx) |
78 | { |
79 | struct nfp_net *nn = r_vec->nfp_net; |
80 | |
81 | rx_ring->idx = idx; |
82 | rx_ring->r_vec = r_vec; |
83 | u64_stats_init(syncp: &rx_ring->r_vec->rx_sync); |
84 | |
85 | rx_ring->fl_qcidx = rx_ring->idx * nn->stride_rx; |
86 | rx_ring->qcp_fl = nn->rx_bar + NFP_QCP_QUEUE_OFF(rx_ring->fl_qcidx); |
87 | } |
88 | |
89 | /** |
90 | * nfp_net_rx_ring_reset() - Reflect in SW state of freelist after disable |
91 | * @rx_ring: RX ring structure |
92 | * |
93 | * Assumes that the device is stopped, must be idempotent. |
94 | */ |
95 | void nfp_net_rx_ring_reset(struct nfp_net_rx_ring *rx_ring) |
96 | { |
97 | unsigned int wr_idx, last_idx; |
98 | |
99 | /* wr_p == rd_p means ring was never fed FL bufs. RX rings are always |
100 | * kept at cnt - 1 FL bufs. |
101 | */ |
102 | if (rx_ring->wr_p == 0 && rx_ring->rd_p == 0) |
103 | return; |
104 | |
105 | /* Move the empty entry to the end of the list */ |
106 | wr_idx = D_IDX(rx_ring, rx_ring->wr_p); |
107 | last_idx = rx_ring->cnt - 1; |
108 | if (rx_ring->r_vec->xsk_pool) { |
109 | rx_ring->xsk_rxbufs[wr_idx] = rx_ring->xsk_rxbufs[last_idx]; |
110 | memset(&rx_ring->xsk_rxbufs[last_idx], 0, |
111 | sizeof(*rx_ring->xsk_rxbufs)); |
112 | } else { |
113 | rx_ring->rxbufs[wr_idx] = rx_ring->rxbufs[last_idx]; |
114 | memset(&rx_ring->rxbufs[last_idx], 0, sizeof(*rx_ring->rxbufs)); |
115 | } |
116 | |
117 | memset(rx_ring->rxds, 0, rx_ring->size); |
118 | rx_ring->wr_p = 0; |
119 | rx_ring->rd_p = 0; |
120 | } |
121 | |
122 | /** |
123 | * nfp_net_rx_ring_bufs_free() - Free any buffers currently on the RX ring |
124 | * @dp: NFP Net data path struct |
125 | * @rx_ring: RX ring to remove buffers from |
126 | * |
127 | * Assumes that the device is stopped and buffers are in [0, ring->cnt - 1) |
128 | * entries. After device is disabled nfp_net_rx_ring_reset() must be called |
129 | * to restore required ring geometry. |
130 | */ |
131 | static void |
132 | nfp_net_rx_ring_bufs_free(struct nfp_net_dp *dp, |
133 | struct nfp_net_rx_ring *rx_ring) |
134 | { |
135 | unsigned int i; |
136 | |
137 | if (nfp_net_has_xsk_pool_slow(dp, qid: rx_ring->idx)) |
138 | return; |
139 | |
140 | for (i = 0; i < rx_ring->cnt - 1; i++) { |
141 | /* NULL skb can only happen when initial filling of the ring |
142 | * fails to allocate enough buffers and calls here to free |
143 | * already allocated ones. |
144 | */ |
145 | if (!rx_ring->rxbufs[i].frag) |
146 | continue; |
147 | |
148 | nfp_net_dma_unmap_rx(dp, dma_addr: rx_ring->rxbufs[i].dma_addr); |
149 | nfp_net_free_frag(frag: rx_ring->rxbufs[i].frag, xdp: dp->xdp_prog); |
150 | rx_ring->rxbufs[i].dma_addr = 0; |
151 | rx_ring->rxbufs[i].frag = NULL; |
152 | } |
153 | } |
154 | |
155 | /** |
156 | * nfp_net_rx_ring_bufs_alloc() - Fill RX ring with buffers (don't give to FW) |
157 | * @dp: NFP Net data path struct |
158 | * @rx_ring: RX ring to remove buffers from |
159 | */ |
160 | static int |
161 | nfp_net_rx_ring_bufs_alloc(struct nfp_net_dp *dp, |
162 | struct nfp_net_rx_ring *rx_ring) |
163 | { |
164 | struct nfp_net_rx_buf *rxbufs; |
165 | unsigned int i; |
166 | |
167 | if (nfp_net_has_xsk_pool_slow(dp, qid: rx_ring->idx)) |
168 | return 0; |
169 | |
170 | rxbufs = rx_ring->rxbufs; |
171 | |
172 | for (i = 0; i < rx_ring->cnt - 1; i++) { |
173 | rxbufs[i].frag = nfp_net_rx_alloc_one(dp, dma_addr: &rxbufs[i].dma_addr); |
174 | if (!rxbufs[i].frag) { |
175 | nfp_net_rx_ring_bufs_free(dp, rx_ring); |
176 | return -ENOMEM; |
177 | } |
178 | } |
179 | |
180 | return 0; |
181 | } |
182 | |
183 | int nfp_net_tx_rings_prepare(struct nfp_net *nn, struct nfp_net_dp *dp) |
184 | { |
185 | unsigned int r; |
186 | |
187 | dp->tx_rings = kcalloc(n: dp->num_tx_rings, size: sizeof(*dp->tx_rings), |
188 | GFP_KERNEL); |
189 | if (!dp->tx_rings) |
190 | return -ENOMEM; |
191 | |
192 | if (dp->ctrl & NFP_NET_CFG_CTRL_TXRWB) { |
193 | dp->txrwb = dma_alloc_coherent(dev: dp->dev, |
194 | size: dp->num_tx_rings * sizeof(u64), |
195 | dma_handle: &dp->txrwb_dma, GFP_KERNEL); |
196 | if (!dp->txrwb) |
197 | goto err_free_rings; |
198 | } |
199 | |
200 | for (r = 0; r < dp->num_tx_rings; r++) { |
201 | int bias = 0; |
202 | |
203 | if (r >= dp->num_stack_tx_rings) |
204 | bias = dp->num_stack_tx_rings; |
205 | |
206 | nfp_net_tx_ring_init(tx_ring: &dp->tx_rings[r], dp, |
207 | r_vec: &nn->r_vecs[r - bias], idx: r, is_xdp: bias); |
208 | |
209 | if (nfp_net_tx_ring_alloc(dp, tx_ring: &dp->tx_rings[r])) |
210 | goto err_free_prev; |
211 | |
212 | if (nfp_net_tx_ring_bufs_alloc(dp, tx_ring: &dp->tx_rings[r])) |
213 | goto err_free_ring; |
214 | } |
215 | |
216 | return 0; |
217 | |
218 | err_free_prev: |
219 | while (r--) { |
220 | nfp_net_tx_ring_bufs_free(dp, tx_ring: &dp->tx_rings[r]); |
221 | err_free_ring: |
222 | nfp_net_tx_ring_free(dp, tx_ring: &dp->tx_rings[r]); |
223 | } |
224 | if (dp->txrwb) |
225 | dma_free_coherent(dev: dp->dev, size: dp->num_tx_rings * sizeof(u64), |
226 | cpu_addr: dp->txrwb, dma_handle: dp->txrwb_dma); |
227 | err_free_rings: |
228 | kfree(objp: dp->tx_rings); |
229 | return -ENOMEM; |
230 | } |
231 | |
232 | void nfp_net_tx_rings_free(struct nfp_net_dp *dp) |
233 | { |
234 | unsigned int r; |
235 | |
236 | for (r = 0; r < dp->num_tx_rings; r++) { |
237 | nfp_net_tx_ring_bufs_free(dp, tx_ring: &dp->tx_rings[r]); |
238 | nfp_net_tx_ring_free(dp, tx_ring: &dp->tx_rings[r]); |
239 | } |
240 | |
241 | if (dp->txrwb) |
242 | dma_free_coherent(dev: dp->dev, size: dp->num_tx_rings * sizeof(u64), |
243 | cpu_addr: dp->txrwb, dma_handle: dp->txrwb_dma); |
244 | kfree(objp: dp->tx_rings); |
245 | } |
246 | |
247 | /** |
248 | * nfp_net_rx_ring_free() - Free resources allocated to a RX ring |
249 | * @rx_ring: RX ring to free |
250 | */ |
251 | static void nfp_net_rx_ring_free(struct nfp_net_rx_ring *rx_ring) |
252 | { |
253 | struct nfp_net_r_vector *r_vec = rx_ring->r_vec; |
254 | struct nfp_net_dp *dp = &r_vec->nfp_net->dp; |
255 | |
256 | if (dp->netdev) |
257 | xdp_rxq_info_unreg(xdp_rxq: &rx_ring->xdp_rxq); |
258 | |
259 | if (nfp_net_has_xsk_pool_slow(dp, qid: rx_ring->idx)) |
260 | kvfree(addr: rx_ring->xsk_rxbufs); |
261 | else |
262 | kvfree(addr: rx_ring->rxbufs); |
263 | |
264 | if (rx_ring->rxds) |
265 | dma_free_coherent(dev: dp->dev, size: rx_ring->size, |
266 | cpu_addr: rx_ring->rxds, dma_handle: rx_ring->dma); |
267 | |
268 | rx_ring->cnt = 0; |
269 | rx_ring->rxbufs = NULL; |
270 | rx_ring->xsk_rxbufs = NULL; |
271 | rx_ring->rxds = NULL; |
272 | rx_ring->dma = 0; |
273 | rx_ring->size = 0; |
274 | } |
275 | |
276 | /** |
277 | * nfp_net_rx_ring_alloc() - Allocate resource for a RX ring |
278 | * @dp: NFP Net data path struct |
279 | * @rx_ring: RX ring to allocate |
280 | * |
281 | * Return: 0 on success, negative errno otherwise. |
282 | */ |
283 | static int |
284 | nfp_net_rx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_rx_ring *rx_ring) |
285 | { |
286 | enum xdp_mem_type mem_type; |
287 | size_t rxbuf_sw_desc_sz; |
288 | int err; |
289 | |
290 | if (nfp_net_has_xsk_pool_slow(dp, qid: rx_ring->idx)) { |
291 | mem_type = MEM_TYPE_XSK_BUFF_POOL; |
292 | rxbuf_sw_desc_sz = sizeof(*rx_ring->xsk_rxbufs); |
293 | } else { |
294 | mem_type = MEM_TYPE_PAGE_ORDER0; |
295 | rxbuf_sw_desc_sz = sizeof(*rx_ring->rxbufs); |
296 | } |
297 | |
298 | if (dp->netdev) { |
299 | err = xdp_rxq_info_reg(xdp_rxq: &rx_ring->xdp_rxq, dev: dp->netdev, |
300 | queue_index: rx_ring->idx, napi_id: rx_ring->r_vec->napi.napi_id); |
301 | if (err < 0) |
302 | return err; |
303 | |
304 | err = xdp_rxq_info_reg_mem_model(xdp_rxq: &rx_ring->xdp_rxq, type: mem_type, NULL); |
305 | if (err) |
306 | goto err_alloc; |
307 | } |
308 | |
309 | rx_ring->cnt = dp->rxd_cnt; |
310 | rx_ring->size = array_size(rx_ring->cnt, sizeof(*rx_ring->rxds)); |
311 | rx_ring->rxds = dma_alloc_coherent(dev: dp->dev, size: rx_ring->size, |
312 | dma_handle: &rx_ring->dma, |
313 | GFP_KERNEL | __GFP_NOWARN); |
314 | if (!rx_ring->rxds) { |
315 | netdev_warn(dev: dp->netdev, format: "failed to allocate RX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n" , |
316 | rx_ring->cnt); |
317 | goto err_alloc; |
318 | } |
319 | |
320 | if (nfp_net_has_xsk_pool_slow(dp, qid: rx_ring->idx)) { |
321 | rx_ring->xsk_rxbufs = kvcalloc(n: rx_ring->cnt, size: rxbuf_sw_desc_sz, |
322 | GFP_KERNEL); |
323 | if (!rx_ring->xsk_rxbufs) |
324 | goto err_alloc; |
325 | } else { |
326 | rx_ring->rxbufs = kvcalloc(n: rx_ring->cnt, size: rxbuf_sw_desc_sz, |
327 | GFP_KERNEL); |
328 | if (!rx_ring->rxbufs) |
329 | goto err_alloc; |
330 | } |
331 | |
332 | return 0; |
333 | |
334 | err_alloc: |
335 | nfp_net_rx_ring_free(rx_ring); |
336 | return -ENOMEM; |
337 | } |
338 | |
339 | int nfp_net_rx_rings_prepare(struct nfp_net *nn, struct nfp_net_dp *dp) |
340 | { |
341 | unsigned int r; |
342 | |
343 | dp->rx_rings = kcalloc(n: dp->num_rx_rings, size: sizeof(*dp->rx_rings), |
344 | GFP_KERNEL); |
345 | if (!dp->rx_rings) |
346 | return -ENOMEM; |
347 | |
348 | for (r = 0; r < dp->num_rx_rings; r++) { |
349 | nfp_net_rx_ring_init(rx_ring: &dp->rx_rings[r], r_vec: &nn->r_vecs[r], idx: r); |
350 | |
351 | if (nfp_net_rx_ring_alloc(dp, rx_ring: &dp->rx_rings[r])) |
352 | goto err_free_prev; |
353 | |
354 | if (nfp_net_rx_ring_bufs_alloc(dp, rx_ring: &dp->rx_rings[r])) |
355 | goto err_free_ring; |
356 | } |
357 | |
358 | return 0; |
359 | |
360 | err_free_prev: |
361 | while (r--) { |
362 | nfp_net_rx_ring_bufs_free(dp, rx_ring: &dp->rx_rings[r]); |
363 | err_free_ring: |
364 | nfp_net_rx_ring_free(rx_ring: &dp->rx_rings[r]); |
365 | } |
366 | kfree(objp: dp->rx_rings); |
367 | return -ENOMEM; |
368 | } |
369 | |
370 | void nfp_net_rx_rings_free(struct nfp_net_dp *dp) |
371 | { |
372 | unsigned int r; |
373 | |
374 | for (r = 0; r < dp->num_rx_rings; r++) { |
375 | nfp_net_rx_ring_bufs_free(dp, rx_ring: &dp->rx_rings[r]); |
376 | nfp_net_rx_ring_free(rx_ring: &dp->rx_rings[r]); |
377 | } |
378 | |
379 | kfree(objp: dp->rx_rings); |
380 | } |
381 | |
382 | void |
383 | nfp_net_rx_ring_hw_cfg_write(struct nfp_net *nn, |
384 | struct nfp_net_rx_ring *rx_ring, unsigned int idx) |
385 | { |
386 | /* Write the DMA address, size and MSI-X info to the device */ |
387 | nn_writeq(nn, NFP_NET_CFG_RXR_ADDR(idx), val: rx_ring->dma); |
388 | nn_writeb(nn, NFP_NET_CFG_RXR_SZ(idx), ilog2(rx_ring->cnt)); |
389 | nn_writeb(nn, NFP_NET_CFG_RXR_VEC(idx), val: rx_ring->r_vec->irq_entry); |
390 | } |
391 | |
392 | void |
393 | nfp_net_tx_ring_hw_cfg_write(struct nfp_net *nn, |
394 | struct nfp_net_tx_ring *tx_ring, unsigned int idx) |
395 | { |
396 | nn_writeq(nn, NFP_NET_CFG_TXR_ADDR(idx), val: tx_ring->dma); |
397 | if (tx_ring->txrwb) { |
398 | *tx_ring->txrwb = 0; |
399 | nn_writeq(nn, NFP_NET_CFG_TXR_WB_ADDR(idx), |
400 | val: nn->dp.txrwb_dma + idx * sizeof(u64)); |
401 | } |
402 | nn_writeb(nn, NFP_NET_CFG_TXR_SZ(idx), ilog2(tx_ring->cnt)); |
403 | nn_writeb(nn, NFP_NET_CFG_TXR_VEC(idx), val: tx_ring->r_vec->irq_entry); |
404 | } |
405 | |
406 | void nfp_net_vec_clear_ring_data(struct nfp_net *nn, unsigned int idx) |
407 | { |
408 | nn_writeq(nn, NFP_NET_CFG_RXR_ADDR(idx), val: 0); |
409 | nn_writeb(nn, NFP_NET_CFG_RXR_SZ(idx), val: 0); |
410 | nn_writeb(nn, NFP_NET_CFG_RXR_VEC(idx), val: 0); |
411 | |
412 | nn_writeq(nn, NFP_NET_CFG_TXR_ADDR(idx), val: 0); |
413 | nn_writeq(nn, NFP_NET_CFG_TXR_WB_ADDR(idx), val: 0); |
414 | nn_writeb(nn, NFP_NET_CFG_TXR_SZ(idx), val: 0); |
415 | nn_writeb(nn, NFP_NET_CFG_TXR_VEC(idx), val: 0); |
416 | } |
417 | |
418 | netdev_tx_t nfp_net_tx(struct sk_buff *skb, struct net_device *netdev) |
419 | { |
420 | struct nfp_net *nn = netdev_priv(dev: netdev); |
421 | |
422 | return nn->dp.ops->xmit(skb, netdev); |
423 | } |
424 | |
425 | bool __nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb) |
426 | { |
427 | struct nfp_net_r_vector *r_vec = &nn->r_vecs[0]; |
428 | |
429 | return nn->dp.ops->ctrl_tx_one(nn, r_vec, skb, false); |
430 | } |
431 | |
432 | bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb) |
433 | { |
434 | struct nfp_net_r_vector *r_vec = &nn->r_vecs[0]; |
435 | bool ret; |
436 | |
437 | spin_lock_bh(lock: &r_vec->lock); |
438 | ret = nn->dp.ops->ctrl_tx_one(nn, r_vec, skb, false); |
439 | spin_unlock_bh(lock: &r_vec->lock); |
440 | |
441 | return ret; |
442 | } |
443 | |
444 | bool nfp_net_vlan_strip(struct sk_buff *skb, const struct nfp_net_rx_desc *rxd, |
445 | const struct nfp_meta_parsed *meta) |
446 | { |
447 | u16 tpid = 0, tci = 0; |
448 | |
449 | if (rxd->rxd.flags & PCIE_DESC_RX_VLAN) { |
450 | tpid = ETH_P_8021Q; |
451 | tci = le16_to_cpu(rxd->rxd.vlan); |
452 | } else if (meta->vlan.stripped) { |
453 | if (meta->vlan.tpid == NFP_NET_VLAN_CTAG) |
454 | tpid = ETH_P_8021Q; |
455 | else if (meta->vlan.tpid == NFP_NET_VLAN_STAG) |
456 | tpid = ETH_P_8021AD; |
457 | else |
458 | return false; |
459 | |
460 | tci = meta->vlan.tci; |
461 | } |
462 | if (tpid) |
463 | __vlan_hwaccel_put_tag(skb, htons(tpid), vlan_tci: tci); |
464 | |
465 | return true; |
466 | } |
467 | |