1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2018 Netronome Systems, Inc */ |
3 | /* Copyright (C) 2021 Corigine, Inc */ |
4 | |
5 | #include <linux/dma-direction.h> |
6 | #include <linux/dma-mapping.h> |
7 | #include <linux/slab.h> |
8 | #include <net/xdp_sock_drv.h> |
9 | #include <trace/events/xdp.h> |
10 | |
11 | #include "nfp_app.h" |
12 | #include "nfp_net.h" |
13 | #include "nfp_net_dp.h" |
14 | #include "nfp_net_xsk.h" |
15 | |
16 | static void |
17 | nfp_net_xsk_rx_bufs_stash(struct nfp_net_rx_ring *rx_ring, unsigned int idx, |
18 | struct xdp_buff *xdp) |
19 | { |
20 | unsigned int headroom; |
21 | |
22 | headroom = xsk_pool_get_headroom(pool: rx_ring->r_vec->xsk_pool); |
23 | |
24 | rx_ring->rxds[idx].fld.reserved = 0; |
25 | rx_ring->rxds[idx].fld.meta_len_dd = 0; |
26 | |
27 | rx_ring->xsk_rxbufs[idx].xdp = xdp; |
28 | rx_ring->xsk_rxbufs[idx].dma_addr = |
29 | xsk_buff_xdp_get_frame_dma(xdp) + headroom; |
30 | } |
31 | |
32 | void nfp_net_xsk_rx_unstash(struct nfp_net_xsk_rx_buf *rxbuf) |
33 | { |
34 | rxbuf->dma_addr = 0; |
35 | rxbuf->xdp = NULL; |
36 | } |
37 | |
38 | void nfp_net_xsk_rx_free(struct nfp_net_xsk_rx_buf *rxbuf) |
39 | { |
40 | if (rxbuf->xdp) |
41 | xsk_buff_free(xdp: rxbuf->xdp); |
42 | |
43 | nfp_net_xsk_rx_unstash(rxbuf); |
44 | } |
45 | |
46 | void nfp_net_xsk_rx_bufs_free(struct nfp_net_rx_ring *rx_ring) |
47 | { |
48 | unsigned int i; |
49 | |
50 | if (!rx_ring->cnt) |
51 | return; |
52 | |
53 | for (i = 0; i < rx_ring->cnt - 1; i++) |
54 | nfp_net_xsk_rx_free(rxbuf: &rx_ring->xsk_rxbufs[i]); |
55 | } |
56 | |
57 | void nfp_net_xsk_rx_ring_fill_freelist(struct nfp_net_rx_ring *rx_ring) |
58 | { |
59 | struct nfp_net_r_vector *r_vec = rx_ring->r_vec; |
60 | struct xsk_buff_pool *pool = r_vec->xsk_pool; |
61 | unsigned int wr_idx, wr_ptr_add = 0; |
62 | struct xdp_buff *xdp; |
63 | |
64 | while (nfp_net_rx_space(rx_ring)) { |
65 | wr_idx = D_IDX(rx_ring, rx_ring->wr_p); |
66 | |
67 | xdp = xsk_buff_alloc(pool); |
68 | if (!xdp) |
69 | break; |
70 | |
71 | nfp_net_xsk_rx_bufs_stash(rx_ring, idx: wr_idx, xdp); |
72 | |
73 | /* DMA address is expanded to 48-bit width in freelist for NFP3800, |
74 | * so the *_48b macro is used accordingly, it's also OK to fill |
75 | * a 40-bit address since the top 8 bits are get set to 0. |
76 | */ |
77 | nfp_desc_set_dma_addr_48b(&rx_ring->rxds[wr_idx].fld, |
78 | rx_ring->xsk_rxbufs[wr_idx].dma_addr); |
79 | |
80 | rx_ring->wr_p++; |
81 | wr_ptr_add++; |
82 | } |
83 | |
84 | /* Ensure all records are visible before incrementing write counter. */ |
85 | wmb(); |
86 | nfp_qcp_wr_ptr_add(q: rx_ring->qcp_fl, val: wr_ptr_add); |
87 | } |
88 | |
89 | void nfp_net_xsk_rx_drop(struct nfp_net_r_vector *r_vec, |
90 | struct nfp_net_xsk_rx_buf *xrxbuf) |
91 | { |
92 | u64_stats_update_begin(syncp: &r_vec->rx_sync); |
93 | r_vec->rx_drops++; |
94 | u64_stats_update_end(syncp: &r_vec->rx_sync); |
95 | |
96 | nfp_net_xsk_rx_free(rxbuf: xrxbuf); |
97 | } |
98 | |
99 | static void nfp_net_xsk_pool_unmap(struct device *dev, |
100 | struct xsk_buff_pool *pool) |
101 | { |
102 | return xsk_pool_dma_unmap(pool, attrs: 0); |
103 | } |
104 | |
105 | static int nfp_net_xsk_pool_map(struct device *dev, struct xsk_buff_pool *pool) |
106 | { |
107 | return xsk_pool_dma_map(pool, dev, attrs: 0); |
108 | } |
109 | |
110 | int nfp_net_xsk_setup_pool(struct net_device *netdev, |
111 | struct xsk_buff_pool *pool, u16 queue_id) |
112 | { |
113 | struct nfp_net *nn = netdev_priv(dev: netdev); |
114 | |
115 | struct xsk_buff_pool *prev_pool; |
116 | struct nfp_net_dp *dp; |
117 | int err; |
118 | |
119 | /* NFDK doesn't implement xsk yet. */ |
120 | if (nn->dp.ops->version == NFP_NFD_VER_NFDK) |
121 | return -EOPNOTSUPP; |
122 | |
123 | /* Reject on old FWs so we can drop some checks on datapath. */ |
124 | if (nn->dp.rx_offset != NFP_NET_CFG_RX_OFFSET_DYNAMIC) |
125 | return -EOPNOTSUPP; |
126 | if (!nn->dp.chained_metadata_format) |
127 | return -EOPNOTSUPP; |
128 | |
129 | /* Install */ |
130 | if (pool) { |
131 | err = nfp_net_xsk_pool_map(dev: nn->dp.dev, pool); |
132 | if (err) |
133 | return err; |
134 | } |
135 | |
136 | /* Reconfig/swap */ |
137 | dp = nfp_net_clone_dp(nn); |
138 | if (!dp) { |
139 | err = -ENOMEM; |
140 | goto err_unmap; |
141 | } |
142 | |
143 | prev_pool = dp->xsk_pools[queue_id]; |
144 | dp->xsk_pools[queue_id] = pool; |
145 | |
146 | err = nfp_net_ring_reconfig(nn, new: dp, NULL); |
147 | if (err) |
148 | goto err_unmap; |
149 | |
150 | /* Uninstall */ |
151 | if (prev_pool) |
152 | nfp_net_xsk_pool_unmap(dev: nn->dp.dev, pool: prev_pool); |
153 | |
154 | return 0; |
155 | err_unmap: |
156 | if (pool) |
157 | nfp_net_xsk_pool_unmap(dev: nn->dp.dev, pool); |
158 | |
159 | return err; |
160 | } |
161 | |
162 | int nfp_net_xsk_wakeup(struct net_device *netdev, u32 queue_id, u32 flags) |
163 | { |
164 | struct nfp_net *nn = netdev_priv(dev: netdev); |
165 | |
166 | /* queue_id comes from a zero-copy socket, installed with XDP_SETUP_XSK_POOL, |
167 | * so it must be within our vector range. Moreover, our napi structs |
168 | * are statically allocated, so we can always kick them without worrying |
169 | * if reconfig is in progress or interface down. |
170 | */ |
171 | napi_schedule(n: &nn->r_vecs[queue_id].napi); |
172 | |
173 | return 0; |
174 | } |
175 | |