1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2/* Copyright (C) 2018 Netronome Systems, Inc */
3/* Copyright (C) 2021 Corigine, Inc */
4
5#include <linux/dma-direction.h>
6#include <linux/dma-mapping.h>
7#include <linux/slab.h>
8#include <net/xdp_sock_drv.h>
9#include <trace/events/xdp.h>
10
11#include "nfp_app.h"
12#include "nfp_net.h"
13#include "nfp_net_dp.h"
14#include "nfp_net_xsk.h"
15
16static void
17nfp_net_xsk_rx_bufs_stash(struct nfp_net_rx_ring *rx_ring, unsigned int idx,
18 struct xdp_buff *xdp)
19{
20 unsigned int headroom;
21
22 headroom = xsk_pool_get_headroom(pool: rx_ring->r_vec->xsk_pool);
23
24 rx_ring->rxds[idx].fld.reserved = 0;
25 rx_ring->rxds[idx].fld.meta_len_dd = 0;
26
27 rx_ring->xsk_rxbufs[idx].xdp = xdp;
28 rx_ring->xsk_rxbufs[idx].dma_addr =
29 xsk_buff_xdp_get_frame_dma(xdp) + headroom;
30}
31
32void nfp_net_xsk_rx_unstash(struct nfp_net_xsk_rx_buf *rxbuf)
33{
34 rxbuf->dma_addr = 0;
35 rxbuf->xdp = NULL;
36}
37
38void nfp_net_xsk_rx_free(struct nfp_net_xsk_rx_buf *rxbuf)
39{
40 if (rxbuf->xdp)
41 xsk_buff_free(xdp: rxbuf->xdp);
42
43 nfp_net_xsk_rx_unstash(rxbuf);
44}
45
46void nfp_net_xsk_rx_bufs_free(struct nfp_net_rx_ring *rx_ring)
47{
48 unsigned int i;
49
50 if (!rx_ring->cnt)
51 return;
52
53 for (i = 0; i < rx_ring->cnt - 1; i++)
54 nfp_net_xsk_rx_free(rxbuf: &rx_ring->xsk_rxbufs[i]);
55}
56
57void nfp_net_xsk_rx_ring_fill_freelist(struct nfp_net_rx_ring *rx_ring)
58{
59 struct nfp_net_r_vector *r_vec = rx_ring->r_vec;
60 struct xsk_buff_pool *pool = r_vec->xsk_pool;
61 unsigned int wr_idx, wr_ptr_add = 0;
62 struct xdp_buff *xdp;
63
64 while (nfp_net_rx_space(rx_ring)) {
65 wr_idx = D_IDX(rx_ring, rx_ring->wr_p);
66
67 xdp = xsk_buff_alloc(pool);
68 if (!xdp)
69 break;
70
71 nfp_net_xsk_rx_bufs_stash(rx_ring, idx: wr_idx, xdp);
72
73 /* DMA address is expanded to 48-bit width in freelist for NFP3800,
74 * so the *_48b macro is used accordingly, it's also OK to fill
75 * a 40-bit address since the top 8 bits are get set to 0.
76 */
77 nfp_desc_set_dma_addr_48b(&rx_ring->rxds[wr_idx].fld,
78 rx_ring->xsk_rxbufs[wr_idx].dma_addr);
79
80 rx_ring->wr_p++;
81 wr_ptr_add++;
82 }
83
84 /* Ensure all records are visible before incrementing write counter. */
85 wmb();
86 nfp_qcp_wr_ptr_add(q: rx_ring->qcp_fl, val: wr_ptr_add);
87}
88
89void nfp_net_xsk_rx_drop(struct nfp_net_r_vector *r_vec,
90 struct nfp_net_xsk_rx_buf *xrxbuf)
91{
92 u64_stats_update_begin(syncp: &r_vec->rx_sync);
93 r_vec->rx_drops++;
94 u64_stats_update_end(syncp: &r_vec->rx_sync);
95
96 nfp_net_xsk_rx_free(rxbuf: xrxbuf);
97}
98
99static void nfp_net_xsk_pool_unmap(struct device *dev,
100 struct xsk_buff_pool *pool)
101{
102 return xsk_pool_dma_unmap(pool, attrs: 0);
103}
104
105static int nfp_net_xsk_pool_map(struct device *dev, struct xsk_buff_pool *pool)
106{
107 return xsk_pool_dma_map(pool, dev, attrs: 0);
108}
109
110int nfp_net_xsk_setup_pool(struct net_device *netdev,
111 struct xsk_buff_pool *pool, u16 queue_id)
112{
113 struct nfp_net *nn = netdev_priv(dev: netdev);
114
115 struct xsk_buff_pool *prev_pool;
116 struct nfp_net_dp *dp;
117 int err;
118
119 /* NFDK doesn't implement xsk yet. */
120 if (nn->dp.ops->version == NFP_NFD_VER_NFDK)
121 return -EOPNOTSUPP;
122
123 /* Reject on old FWs so we can drop some checks on datapath. */
124 if (nn->dp.rx_offset != NFP_NET_CFG_RX_OFFSET_DYNAMIC)
125 return -EOPNOTSUPP;
126 if (!nn->dp.chained_metadata_format)
127 return -EOPNOTSUPP;
128
129 /* Install */
130 if (pool) {
131 err = nfp_net_xsk_pool_map(dev: nn->dp.dev, pool);
132 if (err)
133 return err;
134 }
135
136 /* Reconfig/swap */
137 dp = nfp_net_clone_dp(nn);
138 if (!dp) {
139 err = -ENOMEM;
140 goto err_unmap;
141 }
142
143 prev_pool = dp->xsk_pools[queue_id];
144 dp->xsk_pools[queue_id] = pool;
145
146 err = nfp_net_ring_reconfig(nn, new: dp, NULL);
147 if (err)
148 goto err_unmap;
149
150 /* Uninstall */
151 if (prev_pool)
152 nfp_net_xsk_pool_unmap(dev: nn->dp.dev, pool: prev_pool);
153
154 return 0;
155err_unmap:
156 if (pool)
157 nfp_net_xsk_pool_unmap(dev: nn->dp.dev, pool);
158
159 return err;
160}
161
162int nfp_net_xsk_wakeup(struct net_device *netdev, u32 queue_id, u32 flags)
163{
164 struct nfp_net *nn = netdev_priv(dev: netdev);
165
166 /* queue_id comes from a zero-copy socket, installed with XDP_SETUP_XSK_POOL,
167 * so it must be within our vector range. Moreover, our napi structs
168 * are statically allocated, so we can always kick them without worrying
169 * if reconfig is in progress or interface down.
170 */
171 napi_schedule(n: &nn->r_vecs[queue_id].napi);
172
173 return 0;
174}
175

source code of linux/drivers/net/ethernet/netronome/nfp/nfp_net_xsk.c