1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Marvell Octeon EP (EndPoint) Ethernet Driver |
3 | * |
4 | * Copyright (C) 2020 Marvell. |
5 | * |
6 | */ |
7 | |
8 | #include <linux/pci.h> |
9 | #include <linux/etherdevice.h> |
10 | #include <linux/vmalloc.h> |
11 | |
12 | #include "octep_config.h" |
13 | #include "octep_main.h" |
14 | |
15 | static void octep_oq_reset_indices(struct octep_oq *oq) |
16 | { |
17 | oq->host_read_idx = 0; |
18 | oq->host_refill_idx = 0; |
19 | oq->refill_count = 0; |
20 | oq->last_pkt_count = 0; |
21 | oq->pkts_pending = 0; |
22 | } |
23 | |
24 | /** |
25 | * octep_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring. |
26 | * |
27 | * @oq: Octeon Rx queue data structure. |
28 | * |
29 | * Return: 0, if successfully filled receive buffers for all descriptors. |
30 | * -1, if failed to allocate a buffer or failed to map for DMA. |
31 | */ |
32 | static int octep_oq_fill_ring_buffers(struct octep_oq *oq) |
33 | { |
34 | struct octep_oq_desc_hw *desc_ring = oq->desc_ring; |
35 | struct page *page; |
36 | u32 i; |
37 | |
38 | for (i = 0; i < oq->max_count; i++) { |
39 | page = dev_alloc_page(); |
40 | if (unlikely(!page)) { |
41 | dev_err(oq->dev, "Rx buffer alloc failed\n" ); |
42 | goto rx_buf_alloc_err; |
43 | } |
44 | desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0, |
45 | PAGE_SIZE, |
46 | DMA_FROM_DEVICE); |
47 | if (dma_mapping_error(dev: oq->dev, dma_addr: desc_ring[i].buffer_ptr)) { |
48 | dev_err(oq->dev, |
49 | "OQ-%d buffer alloc: DMA mapping error!\n" , |
50 | oq->q_no); |
51 | put_page(page); |
52 | goto dma_map_err; |
53 | } |
54 | oq->buff_info[i].page = page; |
55 | } |
56 | |
57 | return 0; |
58 | |
59 | dma_map_err: |
60 | rx_buf_alloc_err: |
61 | while (i) { |
62 | i--; |
63 | dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE); |
64 | put_page(page: oq->buff_info[i].page); |
65 | oq->buff_info[i].page = NULL; |
66 | } |
67 | |
68 | return -1; |
69 | } |
70 | |
71 | /** |
72 | * octep_oq_refill() - refill buffers for used Rx ring descriptors. |
73 | * |
74 | * @oct: Octeon device private data structure. |
75 | * @oq: Octeon Rx queue data structure. |
76 | * |
77 | * Return: number of descriptors successfully refilled with receive buffers. |
78 | */ |
79 | static int octep_oq_refill(struct octep_device *oct, struct octep_oq *oq) |
80 | { |
81 | struct octep_oq_desc_hw *desc_ring = oq->desc_ring; |
82 | struct page *page; |
83 | u32 refill_idx, i; |
84 | |
85 | refill_idx = oq->host_refill_idx; |
86 | for (i = 0; i < oq->refill_count; i++) { |
87 | page = dev_alloc_page(); |
88 | if (unlikely(!page)) { |
89 | dev_err(oq->dev, "refill: rx buffer alloc failed\n" ); |
90 | oq->stats.alloc_failures++; |
91 | break; |
92 | } |
93 | |
94 | desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0, |
95 | PAGE_SIZE, DMA_FROM_DEVICE); |
96 | if (dma_mapping_error(dev: oq->dev, dma_addr: desc_ring[refill_idx].buffer_ptr)) { |
97 | dev_err(oq->dev, |
98 | "OQ-%d buffer refill: DMA mapping error!\n" , |
99 | oq->q_no); |
100 | put_page(page); |
101 | oq->stats.alloc_failures++; |
102 | break; |
103 | } |
104 | oq->buff_info[refill_idx].page = page; |
105 | refill_idx++; |
106 | if (refill_idx == oq->max_count) |
107 | refill_idx = 0; |
108 | } |
109 | oq->host_refill_idx = refill_idx; |
110 | oq->refill_count -= i; |
111 | |
112 | return i; |
113 | } |
114 | |
115 | /** |
116 | * octep_setup_oq() - Setup a Rx queue. |
117 | * |
118 | * @oct: Octeon device private data structure. |
119 | * @q_no: Rx queue number to be setup. |
120 | * |
121 | * Allocate resources for a Rx queue. |
122 | */ |
123 | static int octep_setup_oq(struct octep_device *oct, int q_no) |
124 | { |
125 | struct octep_oq *oq; |
126 | u32 desc_ring_size; |
127 | |
128 | oq = vzalloc(size: sizeof(*oq)); |
129 | if (!oq) |
130 | goto create_oq_fail; |
131 | oct->oq[q_no] = oq; |
132 | |
133 | oq->octep_dev = oct; |
134 | oq->netdev = oct->netdev; |
135 | oq->dev = &oct->pdev->dev; |
136 | oq->q_no = q_no; |
137 | oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf); |
138 | oq->ring_size_mask = oq->max_count - 1; |
139 | oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf); |
140 | oq->max_single_buffer_size = oq->buffer_size - OCTEP_OQ_RESP_HW_SIZE; |
141 | |
142 | /* When the hardware/firmware supports additional capabilities, |
143 | * additional header is filled-in by Octeon after length field in |
144 | * Rx packets. this header contains additional packet information. |
145 | */ |
146 | if (oct->conf->fw_info.rx_ol_flags) |
147 | oq->max_single_buffer_size -= OCTEP_OQ_RESP_HW_EXT_SIZE; |
148 | |
149 | oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf); |
150 | |
151 | desc_ring_size = oq->max_count * OCTEP_OQ_DESC_SIZE; |
152 | oq->desc_ring = dma_alloc_coherent(dev: oq->dev, size: desc_ring_size, |
153 | dma_handle: &oq->desc_ring_dma, GFP_KERNEL); |
154 | |
155 | if (unlikely(!oq->desc_ring)) { |
156 | dev_err(oq->dev, |
157 | "Failed to allocate DMA memory for OQ-%d !!\n" , q_no); |
158 | goto desc_dma_alloc_err; |
159 | } |
160 | |
161 | oq->buff_info = vcalloc(n: oq->max_count, OCTEP_OQ_RECVBUF_SIZE); |
162 | if (unlikely(!oq->buff_info)) { |
163 | dev_err(&oct->pdev->dev, |
164 | "Failed to allocate buffer info for OQ-%d\n" , q_no); |
165 | goto buf_list_err; |
166 | } |
167 | |
168 | if (octep_oq_fill_ring_buffers(oq)) |
169 | goto oq_fill_buff_err; |
170 | |
171 | octep_oq_reset_indices(oq); |
172 | oct->hw_ops.setup_oq_regs(oct, q_no); |
173 | oct->num_oqs++; |
174 | |
175 | return 0; |
176 | |
177 | oq_fill_buff_err: |
178 | vfree(addr: oq->buff_info); |
179 | oq->buff_info = NULL; |
180 | buf_list_err: |
181 | dma_free_coherent(dev: oq->dev, size: desc_ring_size, |
182 | cpu_addr: oq->desc_ring, dma_handle: oq->desc_ring_dma); |
183 | oq->desc_ring = NULL; |
184 | desc_dma_alloc_err: |
185 | vfree(addr: oq); |
186 | oct->oq[q_no] = NULL; |
187 | create_oq_fail: |
188 | return -1; |
189 | } |
190 | |
191 | /** |
192 | * octep_oq_free_ring_buffers() - Free ring buffers. |
193 | * |
194 | * @oq: Octeon Rx queue data structure. |
195 | * |
196 | * Free receive buffers in unused Rx queue descriptors. |
197 | */ |
198 | static void octep_oq_free_ring_buffers(struct octep_oq *oq) |
199 | { |
200 | struct octep_oq_desc_hw *desc_ring = oq->desc_ring; |
201 | int i; |
202 | |
203 | if (!oq->desc_ring || !oq->buff_info) |
204 | return; |
205 | |
206 | for (i = 0; i < oq->max_count; i++) { |
207 | if (oq->buff_info[i].page) { |
208 | dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, |
209 | PAGE_SIZE, DMA_FROM_DEVICE); |
210 | put_page(page: oq->buff_info[i].page); |
211 | oq->buff_info[i].page = NULL; |
212 | desc_ring[i].buffer_ptr = 0; |
213 | } |
214 | } |
215 | octep_oq_reset_indices(oq); |
216 | } |
217 | |
218 | /** |
219 | * octep_free_oq() - Free Rx queue resources. |
220 | * |
221 | * @oq: Octeon Rx queue data structure. |
222 | * |
223 | * Free all resources of a Rx queue. |
224 | */ |
225 | static int octep_free_oq(struct octep_oq *oq) |
226 | { |
227 | struct octep_device *oct = oq->octep_dev; |
228 | int q_no = oq->q_no; |
229 | |
230 | octep_oq_free_ring_buffers(oq); |
231 | |
232 | vfree(addr: oq->buff_info); |
233 | |
234 | if (oq->desc_ring) |
235 | dma_free_coherent(dev: oq->dev, |
236 | size: oq->max_count * OCTEP_OQ_DESC_SIZE, |
237 | cpu_addr: oq->desc_ring, dma_handle: oq->desc_ring_dma); |
238 | |
239 | vfree(addr: oq); |
240 | oct->oq[q_no] = NULL; |
241 | oct->num_oqs--; |
242 | return 0; |
243 | } |
244 | |
245 | /** |
246 | * octep_setup_oqs() - setup resources for all Rx queues. |
247 | * |
248 | * @oct: Octeon device private data structure. |
249 | */ |
250 | int octep_setup_oqs(struct octep_device *oct) |
251 | { |
252 | int i, retval = 0; |
253 | |
254 | oct->num_oqs = 0; |
255 | for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { |
256 | retval = octep_setup_oq(oct, q_no: i); |
257 | if (retval) { |
258 | dev_err(&oct->pdev->dev, |
259 | "Failed to setup OQ(RxQ)-%d.\n" , i); |
260 | goto oq_setup_err; |
261 | } |
262 | dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n" , i); |
263 | } |
264 | |
265 | return 0; |
266 | |
267 | oq_setup_err: |
268 | while (i) { |
269 | i--; |
270 | octep_free_oq(oq: oct->oq[i]); |
271 | } |
272 | return -1; |
273 | } |
274 | |
275 | /** |
276 | * octep_oq_dbell_init() - Initialize Rx queue doorbell. |
277 | * |
278 | * @oct: Octeon device private data structure. |
279 | * |
280 | * Write number of descriptors to Rx queue doorbell register. |
281 | */ |
282 | void octep_oq_dbell_init(struct octep_device *oct) |
283 | { |
284 | int i; |
285 | |
286 | for (i = 0; i < oct->num_oqs; i++) |
287 | writel(val: oct->oq[i]->max_count, addr: oct->oq[i]->pkts_credit_reg); |
288 | } |
289 | |
290 | /** |
291 | * octep_free_oqs() - Free resources of all Rx queues. |
292 | * |
293 | * @oct: Octeon device private data structure. |
294 | */ |
295 | void octep_free_oqs(struct octep_device *oct) |
296 | { |
297 | int i; |
298 | |
299 | for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { |
300 | if (!oct->oq[i]) |
301 | continue; |
302 | octep_free_oq(oq: oct->oq[i]); |
303 | dev_dbg(&oct->pdev->dev, |
304 | "Successfully freed OQ(RxQ)-%d.\n" , i); |
305 | } |
306 | } |
307 | |
308 | /** |
309 | * octep_oq_check_hw_for_pkts() - Check for new Rx packets. |
310 | * |
311 | * @oct: Octeon device private data structure. |
312 | * @oq: Octeon Rx queue data structure. |
313 | * |
314 | * Return: packets received after previous check. |
315 | */ |
316 | static int octep_oq_check_hw_for_pkts(struct octep_device *oct, |
317 | struct octep_oq *oq) |
318 | { |
319 | u32 pkt_count, new_pkts; |
320 | |
321 | pkt_count = readl(addr: oq->pkts_sent_reg); |
322 | new_pkts = pkt_count - oq->last_pkt_count; |
323 | |
324 | /* Clear the hardware packets counter register if the rx queue is |
325 | * being processed continuously with-in a single interrupt and |
326 | * reached half its max value. |
327 | * this counter is not cleared every time read, to save write cycles. |
328 | */ |
329 | if (unlikely(pkt_count > 0xF0000000U)) { |
330 | writel(val: pkt_count, addr: oq->pkts_sent_reg); |
331 | pkt_count = readl(addr: oq->pkts_sent_reg); |
332 | new_pkts += pkt_count; |
333 | } |
334 | oq->last_pkt_count = pkt_count; |
335 | oq->pkts_pending += new_pkts; |
336 | return new_pkts; |
337 | } |
338 | |
339 | /** |
340 | * __octep_oq_process_rx() - Process hardware Rx queue and push to stack. |
341 | * |
342 | * @oct: Octeon device private data structure. |
343 | * @oq: Octeon Rx queue data structure. |
344 | * @pkts_to_process: number of packets to be processed. |
345 | * |
346 | * Process the new packets in Rx queue. |
347 | * Packets larger than single Rx buffer arrive in consecutive descriptors. |
348 | * But, count returned by the API only accounts full packets, not fragments. |
349 | * |
350 | * Return: number of packets processed and pushed to stack. |
351 | */ |
352 | static int __octep_oq_process_rx(struct octep_device *oct, |
353 | struct octep_oq *oq, u16 pkts_to_process) |
354 | { |
355 | struct octep_oq_resp_hw_ext *resp_hw_ext = NULL; |
356 | netdev_features_t feat = oq->netdev->features; |
357 | struct octep_rx_buffer *buff_info; |
358 | struct octep_oq_resp_hw *resp_hw; |
359 | u32 pkt, rx_bytes, desc_used; |
360 | struct sk_buff *skb; |
361 | u16 data_offset; |
362 | u16 rx_ol_flags; |
363 | u32 read_idx; |
364 | |
365 | read_idx = oq->host_read_idx; |
366 | rx_bytes = 0; |
367 | desc_used = 0; |
368 | for (pkt = 0; pkt < pkts_to_process; pkt++) { |
369 | buff_info = (struct octep_rx_buffer *)&oq->buff_info[read_idx]; |
370 | dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr, |
371 | PAGE_SIZE, DMA_FROM_DEVICE); |
372 | resp_hw = page_address(buff_info->page); |
373 | buff_info->page = NULL; |
374 | |
375 | /* Swap the length field that is in Big-Endian to CPU */ |
376 | buff_info->len = be64_to_cpu(resp_hw->length); |
377 | if (oct->conf->fw_info.rx_ol_flags) { |
378 | /* Extended response header is immediately after |
379 | * response header (resp_hw) |
380 | */ |
381 | resp_hw_ext = (struct octep_oq_resp_hw_ext *) |
382 | (resp_hw + 1); |
383 | buff_info->len -= OCTEP_OQ_RESP_HW_EXT_SIZE; |
384 | /* Packet Data is immediately after |
385 | * extended response header. |
386 | */ |
387 | data_offset = OCTEP_OQ_RESP_HW_SIZE + |
388 | OCTEP_OQ_RESP_HW_EXT_SIZE; |
389 | rx_ol_flags = resp_hw_ext->rx_ol_flags; |
390 | } else { |
391 | /* Data is immediately after |
392 | * Hardware Rx response header. |
393 | */ |
394 | data_offset = OCTEP_OQ_RESP_HW_SIZE; |
395 | rx_ol_flags = 0; |
396 | } |
397 | rx_bytes += buff_info->len; |
398 | |
399 | if (buff_info->len <= oq->max_single_buffer_size) { |
400 | skb = build_skb(data: (void *)resp_hw, PAGE_SIZE); |
401 | skb_reserve(skb, len: data_offset); |
402 | skb_put(skb, len: buff_info->len); |
403 | read_idx++; |
404 | desc_used++; |
405 | if (read_idx == oq->max_count) |
406 | read_idx = 0; |
407 | } else { |
408 | struct skb_shared_info *shinfo; |
409 | u16 data_len; |
410 | |
411 | skb = build_skb(data: (void *)resp_hw, PAGE_SIZE); |
412 | skb_reserve(skb, len: data_offset); |
413 | /* Head fragment includes response header(s); |
414 | * subsequent fragments contains only data. |
415 | */ |
416 | skb_put(skb, len: oq->max_single_buffer_size); |
417 | read_idx++; |
418 | desc_used++; |
419 | if (read_idx == oq->max_count) |
420 | read_idx = 0; |
421 | |
422 | shinfo = skb_shinfo(skb); |
423 | data_len = buff_info->len - oq->max_single_buffer_size; |
424 | while (data_len) { |
425 | dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr, |
426 | PAGE_SIZE, DMA_FROM_DEVICE); |
427 | buff_info = (struct octep_rx_buffer *) |
428 | &oq->buff_info[read_idx]; |
429 | if (data_len < oq->buffer_size) { |
430 | buff_info->len = data_len; |
431 | data_len = 0; |
432 | } else { |
433 | buff_info->len = oq->buffer_size; |
434 | data_len -= oq->buffer_size; |
435 | } |
436 | |
437 | skb_add_rx_frag(skb, i: shinfo->nr_frags, |
438 | page: buff_info->page, off: 0, |
439 | size: buff_info->len, |
440 | truesize: buff_info->len); |
441 | buff_info->page = NULL; |
442 | read_idx++; |
443 | desc_used++; |
444 | if (read_idx == oq->max_count) |
445 | read_idx = 0; |
446 | } |
447 | } |
448 | |
449 | skb->dev = oq->netdev; |
450 | skb->protocol = eth_type_trans(skb, dev: skb->dev); |
451 | if (feat & NETIF_F_RXCSUM && |
452 | OCTEP_RX_CSUM_VERIFIED(rx_ol_flags)) |
453 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
454 | else |
455 | skb->ip_summed = CHECKSUM_NONE; |
456 | napi_gro_receive(napi: oq->napi, skb); |
457 | } |
458 | |
459 | oq->host_read_idx = read_idx; |
460 | oq->refill_count += desc_used; |
461 | oq->stats.packets += pkt; |
462 | oq->stats.bytes += rx_bytes; |
463 | |
464 | return pkt; |
465 | } |
466 | |
467 | /** |
468 | * octep_oq_process_rx() - Process Rx queue. |
469 | * |
470 | * @oq: Octeon Rx queue data structure. |
471 | * @budget: max number of packets can be processed in one invocation. |
472 | * |
473 | * Check for newly received packets and process them. |
474 | * Keeps checking for new packets until budget is used or no new packets seen. |
475 | * |
476 | * Return: number of packets processed. |
477 | */ |
478 | int octep_oq_process_rx(struct octep_oq *oq, int budget) |
479 | { |
480 | u32 pkts_available, pkts_processed, total_pkts_processed; |
481 | struct octep_device *oct = oq->octep_dev; |
482 | |
483 | pkts_available = 0; |
484 | pkts_processed = 0; |
485 | total_pkts_processed = 0; |
486 | while (total_pkts_processed < budget) { |
487 | /* update pending count only when current one exhausted */ |
488 | if (oq->pkts_pending == 0) |
489 | octep_oq_check_hw_for_pkts(oct, oq); |
490 | pkts_available = min(budget - total_pkts_processed, |
491 | oq->pkts_pending); |
492 | if (!pkts_available) |
493 | break; |
494 | |
495 | pkts_processed = __octep_oq_process_rx(oct, oq, |
496 | pkts_to_process: pkts_available); |
497 | oq->pkts_pending -= pkts_processed; |
498 | total_pkts_processed += pkts_processed; |
499 | } |
500 | |
501 | if (oq->refill_count >= oq->refill_threshold) { |
502 | u32 desc_refilled = octep_oq_refill(oct, oq); |
503 | |
504 | /* flush pending writes before updating credits */ |
505 | wmb(); |
506 | writel(val: desc_refilled, addr: oq->pkts_credit_reg); |
507 | } |
508 | |
509 | return total_pkts_processed; |
510 | } |
511 | |