1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2015-2019 Netronome Systems, Inc. */ |
3 | |
4 | #include <linux/seq_file.h> |
5 | |
6 | #include "../nfp_net.h" |
7 | #include "../nfp_net_dp.h" |
8 | #include "../nfp_net_xsk.h" |
9 | #include "nfd3.h" |
10 | |
11 | static void nfp_nfd3_xsk_tx_bufs_free(struct nfp_net_tx_ring *tx_ring) |
12 | { |
13 | struct nfp_nfd3_tx_buf *txbuf; |
14 | unsigned int idx; |
15 | |
16 | while (tx_ring->rd_p != tx_ring->wr_p) { |
17 | idx = D_IDX(tx_ring, tx_ring->rd_p); |
18 | txbuf = &tx_ring->txbufs[idx]; |
19 | |
20 | txbuf->real_len = 0; |
21 | |
22 | tx_ring->qcp_rd_p++; |
23 | tx_ring->rd_p++; |
24 | |
25 | if (tx_ring->r_vec->xsk_pool) { |
26 | if (txbuf->is_xsk_tx) |
27 | nfp_nfd3_xsk_tx_free(txbuf); |
28 | |
29 | xsk_tx_completed(pool: tx_ring->r_vec->xsk_pool, nb_entries: 1); |
30 | } |
31 | } |
32 | } |
33 | |
34 | /** |
35 | * nfp_nfd3_tx_ring_reset() - Free any untransmitted buffers and reset pointers |
36 | * @dp: NFP Net data path struct |
37 | * @tx_ring: TX ring structure |
38 | * |
39 | * Assumes that the device is stopped, must be idempotent. |
40 | */ |
41 | static void |
42 | nfp_nfd3_tx_ring_reset(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring) |
43 | { |
44 | struct netdev_queue *nd_q; |
45 | const skb_frag_t *frag; |
46 | |
47 | while (!tx_ring->is_xdp && tx_ring->rd_p != tx_ring->wr_p) { |
48 | struct nfp_nfd3_tx_buf *tx_buf; |
49 | struct sk_buff *skb; |
50 | int idx, nr_frags; |
51 | |
52 | idx = D_IDX(tx_ring, tx_ring->rd_p); |
53 | tx_buf = &tx_ring->txbufs[idx]; |
54 | |
55 | skb = tx_ring->txbufs[idx].skb; |
56 | nr_frags = skb_shinfo(skb)->nr_frags; |
57 | |
58 | if (tx_buf->fidx == -1) { |
59 | /* unmap head */ |
60 | dma_unmap_single(dp->dev, tx_buf->dma_addr, |
61 | skb_headlen(skb), DMA_TO_DEVICE); |
62 | } else { |
63 | /* unmap fragment */ |
64 | frag = &skb_shinfo(skb)->frags[tx_buf->fidx]; |
65 | dma_unmap_page(dp->dev, tx_buf->dma_addr, |
66 | skb_frag_size(frag), DMA_TO_DEVICE); |
67 | } |
68 | |
69 | /* check for last gather fragment */ |
70 | if (tx_buf->fidx == nr_frags - 1) |
71 | dev_kfree_skb_any(skb); |
72 | |
73 | tx_buf->dma_addr = 0; |
74 | tx_buf->skb = NULL; |
75 | tx_buf->fidx = -2; |
76 | |
77 | tx_ring->qcp_rd_p++; |
78 | tx_ring->rd_p++; |
79 | } |
80 | |
81 | if (tx_ring->is_xdp) |
82 | nfp_nfd3_xsk_tx_bufs_free(tx_ring); |
83 | |
84 | memset(tx_ring->txds, 0, tx_ring->size); |
85 | tx_ring->wr_p = 0; |
86 | tx_ring->rd_p = 0; |
87 | tx_ring->qcp_rd_p = 0; |
88 | tx_ring->wr_ptr_add = 0; |
89 | |
90 | if (tx_ring->is_xdp || !dp->netdev) |
91 | return; |
92 | |
93 | nd_q = netdev_get_tx_queue(dev: dp->netdev, index: tx_ring->idx); |
94 | netdev_tx_reset_queue(q: nd_q); |
95 | } |
96 | |
97 | /** |
98 | * nfp_nfd3_tx_ring_free() - Free resources allocated to a TX ring |
99 | * @tx_ring: TX ring to free |
100 | */ |
101 | static void nfp_nfd3_tx_ring_free(struct nfp_net_tx_ring *tx_ring) |
102 | { |
103 | struct nfp_net_r_vector *r_vec = tx_ring->r_vec; |
104 | struct nfp_net_dp *dp = &r_vec->nfp_net->dp; |
105 | |
106 | kvfree(addr: tx_ring->txbufs); |
107 | |
108 | if (tx_ring->txds) |
109 | dma_free_coherent(dev: dp->dev, size: tx_ring->size, |
110 | cpu_addr: tx_ring->txds, dma_handle: tx_ring->dma); |
111 | |
112 | tx_ring->cnt = 0; |
113 | tx_ring->txbufs = NULL; |
114 | tx_ring->txds = NULL; |
115 | tx_ring->dma = 0; |
116 | tx_ring->size = 0; |
117 | } |
118 | |
119 | /** |
120 | * nfp_nfd3_tx_ring_alloc() - Allocate resource for a TX ring |
121 | * @dp: NFP Net data path struct |
122 | * @tx_ring: TX Ring structure to allocate |
123 | * |
124 | * Return: 0 on success, negative errno otherwise. |
125 | */ |
126 | static int |
127 | nfp_nfd3_tx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring) |
128 | { |
129 | struct nfp_net_r_vector *r_vec = tx_ring->r_vec; |
130 | |
131 | tx_ring->cnt = dp->txd_cnt; |
132 | |
133 | tx_ring->size = array_size(tx_ring->cnt, sizeof(*tx_ring->txds)); |
134 | tx_ring->txds = dma_alloc_coherent(dev: dp->dev, size: tx_ring->size, |
135 | dma_handle: &tx_ring->dma, |
136 | GFP_KERNEL | __GFP_NOWARN); |
137 | if (!tx_ring->txds) { |
138 | netdev_warn(dev: dp->netdev, format: "failed to allocate TX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n" , |
139 | tx_ring->cnt); |
140 | goto err_alloc; |
141 | } |
142 | |
143 | tx_ring->txbufs = kvcalloc(n: tx_ring->cnt, size: sizeof(*tx_ring->txbufs), |
144 | GFP_KERNEL); |
145 | if (!tx_ring->txbufs) |
146 | goto err_alloc; |
147 | |
148 | if (!tx_ring->is_xdp && dp->netdev) |
149 | netif_set_xps_queue(dev: dp->netdev, mask: &r_vec->affinity_mask, |
150 | index: tx_ring->idx); |
151 | |
152 | return 0; |
153 | |
154 | err_alloc: |
155 | nfp_nfd3_tx_ring_free(tx_ring); |
156 | return -ENOMEM; |
157 | } |
158 | |
159 | static void |
160 | nfp_nfd3_tx_ring_bufs_free(struct nfp_net_dp *dp, |
161 | struct nfp_net_tx_ring *tx_ring) |
162 | { |
163 | unsigned int i; |
164 | |
165 | if (!tx_ring->is_xdp) |
166 | return; |
167 | |
168 | for (i = 0; i < tx_ring->cnt; i++) { |
169 | if (!tx_ring->txbufs[i].frag) |
170 | return; |
171 | |
172 | nfp_net_dma_unmap_rx(dp, dma_addr: tx_ring->txbufs[i].dma_addr); |
173 | __free_page(virt_to_page(tx_ring->txbufs[i].frag)); |
174 | } |
175 | } |
176 | |
177 | static int |
178 | nfp_nfd3_tx_ring_bufs_alloc(struct nfp_net_dp *dp, |
179 | struct nfp_net_tx_ring *tx_ring) |
180 | { |
181 | struct nfp_nfd3_tx_buf *txbufs = tx_ring->txbufs; |
182 | unsigned int i; |
183 | |
184 | if (!tx_ring->is_xdp) |
185 | return 0; |
186 | |
187 | for (i = 0; i < tx_ring->cnt; i++) { |
188 | txbufs[i].frag = nfp_net_rx_alloc_one(dp, dma_addr: &txbufs[i].dma_addr); |
189 | if (!txbufs[i].frag) { |
190 | nfp_nfd3_tx_ring_bufs_free(dp, tx_ring); |
191 | return -ENOMEM; |
192 | } |
193 | } |
194 | |
195 | return 0; |
196 | } |
197 | |
198 | static void |
199 | nfp_nfd3_print_tx_descs(struct seq_file *file, |
200 | struct nfp_net_r_vector *r_vec, |
201 | struct nfp_net_tx_ring *tx_ring, |
202 | u32 d_rd_p, u32 d_wr_p) |
203 | { |
204 | struct nfp_nfd3_tx_desc *txd; |
205 | u32 txd_cnt = tx_ring->cnt; |
206 | int i; |
207 | |
208 | for (i = 0; i < txd_cnt; i++) { |
209 | struct xdp_buff *xdp; |
210 | struct sk_buff *skb; |
211 | |
212 | txd = &tx_ring->txds[i]; |
213 | seq_printf(m: file, fmt: "%04d: 0x%08x 0x%08x 0x%08x 0x%08x" , i, |
214 | txd->vals[0], txd->vals[1], |
215 | txd->vals[2], txd->vals[3]); |
216 | |
217 | if (!tx_ring->is_xdp) { |
218 | skb = READ_ONCE(tx_ring->txbufs[i].skb); |
219 | if (skb) |
220 | seq_printf(m: file, fmt: " skb->head=%p skb->data=%p" , |
221 | skb->head, skb->data); |
222 | } else { |
223 | xdp = READ_ONCE(tx_ring->txbufs[i].xdp); |
224 | if (xdp) |
225 | seq_printf(m: file, fmt: " xdp->data=%p" , xdp->data); |
226 | } |
227 | |
228 | if (tx_ring->txbufs[i].dma_addr) |
229 | seq_printf(m: file, fmt: " dma_addr=%pad" , |
230 | &tx_ring->txbufs[i].dma_addr); |
231 | |
232 | if (i == tx_ring->rd_p % txd_cnt) |
233 | seq_puts(m: file, s: " H_RD" ); |
234 | if (i == tx_ring->wr_p % txd_cnt) |
235 | seq_puts(m: file, s: " H_WR" ); |
236 | if (i == d_rd_p % txd_cnt) |
237 | seq_puts(m: file, s: " D_RD" ); |
238 | if (i == d_wr_p % txd_cnt) |
239 | seq_puts(m: file, s: " D_WR" ); |
240 | |
241 | seq_putc(m: file, c: '\n'); |
242 | } |
243 | } |
244 | |
245 | #define NFP_NFD3_CFG_CTRL_SUPPORTED \ |
246 | (NFP_NET_CFG_CTRL_ENABLE | NFP_NET_CFG_CTRL_PROMISC | \ |
247 | NFP_NET_CFG_CTRL_L2BC | NFP_NET_CFG_CTRL_L2MC | \ |
248 | NFP_NET_CFG_CTRL_RXCSUM | NFP_NET_CFG_CTRL_TXCSUM | \ |
249 | NFP_NET_CFG_CTRL_RXVLAN | NFP_NET_CFG_CTRL_TXVLAN | \ |
250 | NFP_NET_CFG_CTRL_RXVLAN_V2 | NFP_NET_CFG_CTRL_RXQINQ | \ |
251 | NFP_NET_CFG_CTRL_TXVLAN_V2 | \ |
252 | NFP_NET_CFG_CTRL_GATHER | NFP_NET_CFG_CTRL_LSO | \ |
253 | NFP_NET_CFG_CTRL_CTAG_FILTER | NFP_NET_CFG_CTRL_CMSG_DATA | \ |
254 | NFP_NET_CFG_CTRL_RINGCFG | NFP_NET_CFG_CTRL_RSS | \ |
255 | NFP_NET_CFG_CTRL_IRQMOD | NFP_NET_CFG_CTRL_TXRWB | \ |
256 | NFP_NET_CFG_CTRL_VEPA | \ |
257 | NFP_NET_CFG_CTRL_VXLAN | NFP_NET_CFG_CTRL_NVGRE | \ |
258 | NFP_NET_CFG_CTRL_BPF | NFP_NET_CFG_CTRL_LSO2 | \ |
259 | NFP_NET_CFG_CTRL_RSS2 | NFP_NET_CFG_CTRL_CSUM_COMPLETE | \ |
260 | NFP_NET_CFG_CTRL_LIVE_ADDR) |
261 | |
262 | const struct nfp_dp_ops nfp_nfd3_ops = { |
263 | .version = NFP_NFD_VER_NFD3, |
264 | .tx_min_desc_per_pkt = 1, |
265 | .cap_mask = NFP_NFD3_CFG_CTRL_SUPPORTED, |
266 | .dma_mask = DMA_BIT_MASK(40), |
267 | .poll = nfp_nfd3_poll, |
268 | .xsk_poll = nfp_nfd3_xsk_poll, |
269 | .ctrl_poll = nfp_nfd3_ctrl_poll, |
270 | .xmit = nfp_nfd3_tx, |
271 | .ctrl_tx_one = nfp_nfd3_ctrl_tx_one, |
272 | .rx_ring_fill_freelist = nfp_nfd3_rx_ring_fill_freelist, |
273 | .tx_ring_alloc = nfp_nfd3_tx_ring_alloc, |
274 | .tx_ring_reset = nfp_nfd3_tx_ring_reset, |
275 | .tx_ring_free = nfp_nfd3_tx_ring_free, |
276 | .tx_ring_bufs_alloc = nfp_nfd3_tx_ring_bufs_alloc, |
277 | .tx_ring_bufs_free = nfp_nfd3_tx_ring_bufs_free, |
278 | .print_tx_descs = nfp_nfd3_print_tx_descs |
279 | }; |
280 | |