1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * This file is based on code from OCTEON SDK by Cavium Networks. |
4 | * |
5 | * Copyright (c) 2003-2010 Cavium Networks |
6 | */ |
7 | |
8 | #include <linux/module.h> |
9 | #include <linux/kernel.h> |
10 | #include <linux/cache.h> |
11 | #include <linux/cpumask.h> |
12 | #include <linux/netdevice.h> |
13 | #include <linux/etherdevice.h> |
14 | #include <linux/ip.h> |
15 | #include <linux/string.h> |
16 | #include <linux/prefetch.h> |
17 | #include <linux/ratelimit.h> |
18 | #include <linux/smp.h> |
19 | #include <linux/interrupt.h> |
20 | #include <net/dst.h> |
21 | #ifdef CONFIG_XFRM |
22 | #include <linux/xfrm.h> |
23 | #include <net/xfrm.h> |
24 | #endif /* CONFIG_XFRM */ |
25 | |
26 | #include "octeon-ethernet.h" |
27 | #include "ethernet-defines.h" |
28 | #include "ethernet-mem.h" |
29 | #include "ethernet-rx.h" |
30 | #include "ethernet-util.h" |
31 | |
32 | static atomic_t oct_rx_ready = ATOMIC_INIT(0); |
33 | |
34 | static struct oct_rx_group { |
35 | int irq; |
36 | int group; |
37 | struct napi_struct napi; |
38 | } oct_rx_group[16]; |
39 | |
40 | /** |
41 | * cvm_oct_do_interrupt - interrupt handler. |
42 | * @irq: Interrupt number. |
43 | * @napi_id: Cookie to identify the NAPI instance. |
44 | * |
45 | * The interrupt occurs whenever the POW has packets in our group. |
46 | * |
47 | */ |
48 | static irqreturn_t cvm_oct_do_interrupt(int irq, void *napi_id) |
49 | { |
50 | /* Disable the IRQ and start napi_poll. */ |
51 | disable_irq_nosync(irq); |
52 | napi_schedule(n: napi_id); |
53 | |
54 | return IRQ_HANDLED; |
55 | } |
56 | |
57 | /** |
58 | * cvm_oct_check_rcv_error - process receive errors |
59 | * @work: Work queue entry pointing to the packet. |
60 | * |
61 | * Returns Non-zero if the packet can be dropped, zero otherwise. |
62 | */ |
63 | static inline int cvm_oct_check_rcv_error(struct cvmx_wqe *work) |
64 | { |
65 | int port; |
66 | |
67 | if (octeon_has_feature(OCTEON_FEATURE_PKND)) |
68 | port = work->word0.pip.cn68xx.pknd; |
69 | else |
70 | port = work->word1.cn38xx.ipprt; |
71 | |
72 | if ((work->word2.snoip.err_code == 10) && (work->word1.len <= 64)) |
73 | /* |
74 | * Ignore length errors on min size packets. Some |
75 | * equipment incorrectly pads packets to 64+4FCS |
76 | * instead of 60+4FCS. Note these packets still get |
77 | * counted as frame errors. |
78 | */ |
79 | return 0; |
80 | |
81 | if (work->word2.snoip.err_code == 5 || |
82 | work->word2.snoip.err_code == 7) { |
83 | /* |
84 | * We received a packet with either an alignment error |
85 | * or a FCS error. This may be signalling that we are |
86 | * running 10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK] |
87 | * off. If this is the case we need to parse the |
88 | * packet to determine if we can remove a non spec |
89 | * preamble and generate a correct packet. |
90 | */ |
91 | int interface = cvmx_helper_get_interface_num(ipd_port: port); |
92 | int index = cvmx_helper_get_interface_index_num(ipd_port: port); |
93 | union cvmx_gmxx_rxx_frm_ctl gmxx_rxx_frm_ctl; |
94 | |
95 | gmxx_rxx_frm_ctl.u64 = |
96 | cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface)); |
97 | if (gmxx_rxx_frm_ctl.s.pre_chk == 0) { |
98 | u8 *ptr = |
99 | cvmx_phys_to_ptr(physical_address: work->packet_ptr.s.addr); |
100 | int i = 0; |
101 | |
102 | while (i < work->word1.len - 1) { |
103 | if (*ptr != 0x55) |
104 | break; |
105 | ptr++; |
106 | i++; |
107 | } |
108 | |
109 | if (*ptr == 0xd5) { |
110 | /* Port received 0xd5 preamble */ |
111 | work->packet_ptr.s.addr += i + 1; |
112 | work->word1.len -= i + 5; |
113 | return 0; |
114 | } |
115 | |
116 | if ((*ptr & 0xf) == 0xd) { |
117 | /* Port received 0xd preamble */ |
118 | work->packet_ptr.s.addr += i; |
119 | work->word1.len -= i + 4; |
120 | for (i = 0; i < work->word1.len; i++) { |
121 | *ptr = |
122 | ((*ptr & 0xf0) >> 4) | |
123 | ((*(ptr + 1) & 0xf) << 4); |
124 | ptr++; |
125 | } |
126 | return 0; |
127 | } |
128 | |
129 | printk_ratelimited("Port %d unknown preamble, packet dropped\n" , |
130 | port); |
131 | cvm_oct_free_work(work_queue_entry: work); |
132 | return 1; |
133 | } |
134 | } |
135 | |
136 | printk_ratelimited("Port %d receive error code %d, packet dropped\n" , |
137 | port, work->word2.snoip.err_code); |
138 | cvm_oct_free_work(work_queue_entry: work); |
139 | return 1; |
140 | } |
141 | |
142 | static void copy_segments_to_skb(struct cvmx_wqe *work, struct sk_buff *skb) |
143 | { |
144 | int segments = work->word2.s.bufs; |
145 | union cvmx_buf_ptr segment_ptr = work->packet_ptr; |
146 | int len = work->word1.len; |
147 | int segment_size; |
148 | |
149 | while (segments--) { |
150 | union cvmx_buf_ptr next_ptr; |
151 | |
152 | next_ptr = *(union cvmx_buf_ptr *) |
153 | cvmx_phys_to_ptr(physical_address: segment_ptr.s.addr - 8); |
154 | |
155 | /* |
156 | * Octeon Errata PKI-100: The segment size is wrong. |
157 | * |
158 | * Until it is fixed, calculate the segment size based on |
159 | * the packet pool buffer size. |
160 | * When it is fixed, the following line should be replaced |
161 | * with this one: |
162 | * int segment_size = segment_ptr.s.size; |
163 | */ |
164 | segment_size = |
165 | CVMX_FPA_PACKET_POOL_SIZE - |
166 | (segment_ptr.s.addr - |
167 | (((segment_ptr.s.addr >> 7) - |
168 | segment_ptr.s.back) << 7)); |
169 | |
170 | /* Don't copy more than what is left in the packet */ |
171 | if (segment_size > len) |
172 | segment_size = len; |
173 | |
174 | /* Copy the data into the packet */ |
175 | skb_put_data(skb, data: cvmx_phys_to_ptr(physical_address: segment_ptr.s.addr), |
176 | len: segment_size); |
177 | len -= segment_size; |
178 | segment_ptr = next_ptr; |
179 | } |
180 | } |
181 | |
182 | static int cvm_oct_poll(struct oct_rx_group *rx_group, int budget) |
183 | { |
184 | const int coreid = cvmx_get_core_num(); |
185 | u64 old_group_mask; |
186 | u64 old_scratch; |
187 | int rx_count = 0; |
188 | int did_work_request = 0; |
189 | int packet_not_copied; |
190 | |
191 | /* Prefetch cvm_oct_device since we know we need it soon */ |
192 | prefetch(cvm_oct_device); |
193 | |
194 | if (USE_ASYNC_IOBDMA) { |
195 | /* Save scratch in case userspace is using it */ |
196 | CVMX_SYNCIOBDMA; |
197 | old_scratch = cvmx_scratch_read64(CVMX_SCR_SCRATCH); |
198 | } |
199 | |
200 | /* Only allow work for our group (and preserve priorities) */ |
201 | if (OCTEON_IS_MODEL(OCTEON_CN68XX)) { |
202 | old_group_mask = cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid)); |
203 | cvmx_write_csr(CVMX_SSO_PPX_GRP_MSK(coreid), |
204 | BIT(rx_group->group)); |
205 | cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid)); /* Flush */ |
206 | } else { |
207 | old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid)); |
208 | cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), |
209 | val: (old_group_mask & ~0xFFFFull) | |
210 | BIT(rx_group->group)); |
211 | } |
212 | |
213 | if (USE_ASYNC_IOBDMA) { |
214 | cvmx_pow_work_request_async(CVMX_SCR_SCRATCH, wait: CVMX_POW_NO_WAIT); |
215 | did_work_request = 1; |
216 | } |
217 | |
218 | while (rx_count < budget) { |
219 | struct sk_buff *skb = NULL; |
220 | struct sk_buff **pskb = NULL; |
221 | int skb_in_hw; |
222 | struct cvmx_wqe *work; |
223 | int port; |
224 | |
225 | if (USE_ASYNC_IOBDMA && did_work_request) |
226 | work = cvmx_pow_work_response_async(CVMX_SCR_SCRATCH); |
227 | else |
228 | work = cvmx_pow_work_request_sync(wait: CVMX_POW_NO_WAIT); |
229 | |
230 | prefetch(work); |
231 | did_work_request = 0; |
232 | if (!work) { |
233 | if (OCTEON_IS_MODEL(OCTEON_CN68XX)) { |
234 | cvmx_write_csr(CVMX_SSO_WQ_IQ_DIS, |
235 | BIT(rx_group->group)); |
236 | cvmx_write_csr(CVMX_SSO_WQ_INT, |
237 | BIT(rx_group->group)); |
238 | } else { |
239 | union cvmx_pow_wq_int wq_int; |
240 | |
241 | wq_int.u64 = 0; |
242 | wq_int.s.iq_dis = BIT(rx_group->group); |
243 | wq_int.s.wq_int = BIT(rx_group->group); |
244 | cvmx_write_csr(CVMX_POW_WQ_INT, val: wq_int.u64); |
245 | } |
246 | break; |
247 | } |
248 | pskb = (struct sk_buff **) |
249 | (cvm_oct_get_buffer_ptr(packet_ptr: work->packet_ptr) - |
250 | sizeof(void *)); |
251 | prefetch(pskb); |
252 | |
253 | if (USE_ASYNC_IOBDMA && rx_count < (budget - 1)) { |
254 | cvmx_pow_work_request_async_nocheck(CVMX_SCR_SCRATCH, |
255 | wait: CVMX_POW_NO_WAIT); |
256 | did_work_request = 1; |
257 | } |
258 | rx_count++; |
259 | |
260 | skb_in_hw = work->word2.s.bufs == 1; |
261 | if (likely(skb_in_hw)) { |
262 | skb = *pskb; |
263 | prefetch(&skb->head); |
264 | prefetch(&skb->len); |
265 | } |
266 | |
267 | if (octeon_has_feature(OCTEON_FEATURE_PKND)) |
268 | port = work->word0.pip.cn68xx.pknd; |
269 | else |
270 | port = work->word1.cn38xx.ipprt; |
271 | |
272 | prefetch(cvm_oct_device[port]); |
273 | |
274 | /* Immediately throw away all packets with receive errors */ |
275 | if (unlikely(work->word2.snoip.rcv_error)) { |
276 | if (cvm_oct_check_rcv_error(work)) |
277 | continue; |
278 | } |
279 | |
280 | /* |
281 | * We can only use the zero copy path if skbuffs are |
282 | * in the FPA pool and the packet fits in a single |
283 | * buffer. |
284 | */ |
285 | if (likely(skb_in_hw)) { |
286 | skb->data = skb->head + work->packet_ptr.s.addr - |
287 | cvmx_ptr_to_phys(ptr: skb->head); |
288 | prefetch(skb->data); |
289 | skb->len = work->word1.len; |
290 | skb_set_tail_pointer(skb, offset: skb->len); |
291 | packet_not_copied = 1; |
292 | } else { |
293 | /* |
294 | * We have to copy the packet. First allocate |
295 | * an skbuff for it. |
296 | */ |
297 | skb = dev_alloc_skb(length: work->word1.len); |
298 | if (!skb) { |
299 | cvm_oct_free_work(work_queue_entry: work); |
300 | continue; |
301 | } |
302 | |
303 | /* |
304 | * Check if we've received a packet that was |
305 | * entirely stored in the work entry. |
306 | */ |
307 | if (unlikely(work->word2.s.bufs == 0)) { |
308 | u8 *ptr = work->packet_data; |
309 | |
310 | if (likely(!work->word2.s.not_IP)) { |
311 | /* |
312 | * The beginning of the packet |
313 | * moves for IP packets. |
314 | */ |
315 | if (work->word2.s.is_v6) |
316 | ptr += 2; |
317 | else |
318 | ptr += 6; |
319 | } |
320 | skb_put_data(skb, data: ptr, len: work->word1.len); |
321 | /* No packet buffers to free */ |
322 | } else { |
323 | copy_segments_to_skb(work, skb); |
324 | } |
325 | packet_not_copied = 0; |
326 | } |
327 | if (likely((port < TOTAL_NUMBER_OF_PORTS) && |
328 | cvm_oct_device[port])) { |
329 | struct net_device *dev = cvm_oct_device[port]; |
330 | |
331 | /* |
332 | * Only accept packets for devices that are |
333 | * currently up. |
334 | */ |
335 | if (likely(dev->flags & IFF_UP)) { |
336 | skb->protocol = eth_type_trans(skb, dev); |
337 | skb->dev = dev; |
338 | |
339 | if (unlikely(work->word2.s.not_IP || |
340 | work->word2.s.IP_exc || |
341 | work->word2.s.L4_error || |
342 | !work->word2.s.tcp_or_udp)) |
343 | skb->ip_summed = CHECKSUM_NONE; |
344 | else |
345 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
346 | |
347 | /* Increment RX stats for virtual ports */ |
348 | if (port >= CVMX_PIP_NUM_INPUT_PORTS) { |
349 | dev->stats.rx_packets++; |
350 | dev->stats.rx_bytes += skb->len; |
351 | } |
352 | netif_receive_skb(skb); |
353 | } else { |
354 | /* |
355 | * Drop any packet received for a device that |
356 | * isn't up. |
357 | */ |
358 | dev->stats.rx_dropped++; |
359 | dev_kfree_skb_irq(skb); |
360 | } |
361 | } else { |
362 | /* |
363 | * Drop any packet received for a device that |
364 | * doesn't exist. |
365 | */ |
366 | printk_ratelimited("Port %d not controlled by Linux, packet dropped\n" , |
367 | port); |
368 | dev_kfree_skb_irq(skb); |
369 | } |
370 | /* |
371 | * Check to see if the skbuff and work share the same |
372 | * packet buffer. |
373 | */ |
374 | if (likely(packet_not_copied)) { |
375 | /* |
376 | * This buffer needs to be replaced, increment |
377 | * the number of buffers we need to free by |
378 | * one. |
379 | */ |
380 | cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, |
381 | value: 1); |
382 | |
383 | cvmx_fpa_free(ptr: work, CVMX_FPA_WQE_POOL, num_cache_lines: 1); |
384 | } else { |
385 | cvm_oct_free_work(work_queue_entry: work); |
386 | } |
387 | } |
388 | /* Restore the original POW group mask */ |
389 | if (OCTEON_IS_MODEL(OCTEON_CN68XX)) { |
390 | cvmx_write_csr(CVMX_SSO_PPX_GRP_MSK(coreid), val: old_group_mask); |
391 | cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid)); /* Flush */ |
392 | } else { |
393 | cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), val: old_group_mask); |
394 | } |
395 | |
396 | if (USE_ASYNC_IOBDMA) { |
397 | /* Restore the scratch area */ |
398 | cvmx_scratch_write64(CVMX_SCR_SCRATCH, value: old_scratch); |
399 | } |
400 | cvm_oct_rx_refill_pool(fill_threshold: 0); |
401 | |
402 | return rx_count; |
403 | } |
404 | |
405 | /** |
406 | * cvm_oct_napi_poll - the NAPI poll function. |
407 | * @napi: The NAPI instance. |
408 | * @budget: Maximum number of packets to receive. |
409 | * |
410 | * Returns the number of packets processed. |
411 | */ |
412 | static int cvm_oct_napi_poll(struct napi_struct *napi, int budget) |
413 | { |
414 | struct oct_rx_group *rx_group = container_of(napi, struct oct_rx_group, |
415 | napi); |
416 | int rx_count; |
417 | |
418 | rx_count = cvm_oct_poll(rx_group, budget); |
419 | |
420 | if (rx_count < budget) { |
421 | /* No more work */ |
422 | napi_complete_done(n: napi, work_done: rx_count); |
423 | enable_irq(irq: rx_group->irq); |
424 | } |
425 | return rx_count; |
426 | } |
427 | |
428 | #ifdef CONFIG_NET_POLL_CONTROLLER |
429 | /** |
430 | * cvm_oct_poll_controller - poll for receive packets |
431 | * device. |
432 | * |
433 | * @dev: Device to poll. Unused |
434 | */ |
435 | void cvm_oct_poll_controller(struct net_device *dev) |
436 | { |
437 | int i; |
438 | |
439 | if (!atomic_read(v: &oct_rx_ready)) |
440 | return; |
441 | |
442 | for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) { |
443 | if (!(pow_receive_groups & BIT(i))) |
444 | continue; |
445 | |
446 | cvm_oct_poll(rx_group: &oct_rx_group[i], budget: 16); |
447 | } |
448 | } |
449 | #endif |
450 | |
451 | void cvm_oct_rx_initialize(void) |
452 | { |
453 | int i; |
454 | struct net_device *dev_for_napi = NULL; |
455 | |
456 | for (i = 0; i < TOTAL_NUMBER_OF_PORTS; i++) { |
457 | if (cvm_oct_device[i]) { |
458 | dev_for_napi = cvm_oct_device[i]; |
459 | break; |
460 | } |
461 | } |
462 | |
463 | if (!dev_for_napi) |
464 | panic(fmt: "No net_devices were allocated." ); |
465 | |
466 | for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) { |
467 | int ret; |
468 | |
469 | if (!(pow_receive_groups & BIT(i))) |
470 | continue; |
471 | |
472 | netif_napi_add_weight(dev: dev_for_napi, napi: &oct_rx_group[i].napi, |
473 | poll: cvm_oct_napi_poll, weight: rx_napi_weight); |
474 | napi_enable(n: &oct_rx_group[i].napi); |
475 | |
476 | oct_rx_group[i].irq = OCTEON_IRQ_WORKQ0 + i; |
477 | oct_rx_group[i].group = i; |
478 | |
479 | /* Register an IRQ handler to receive POW interrupts */ |
480 | ret = request_irq(irq: oct_rx_group[i].irq, handler: cvm_oct_do_interrupt, flags: 0, |
481 | name: "Ethernet" , dev: &oct_rx_group[i].napi); |
482 | if (ret) |
483 | panic(fmt: "Could not acquire Ethernet IRQ %d\n" , |
484 | oct_rx_group[i].irq); |
485 | |
486 | disable_irq_nosync(irq: oct_rx_group[i].irq); |
487 | |
488 | /* Enable POW interrupt when our port has at least one packet */ |
489 | if (OCTEON_IS_MODEL(OCTEON_CN68XX)) { |
490 | union cvmx_sso_wq_int_thrx int_thr; |
491 | union cvmx_pow_wq_int_pc int_pc; |
492 | |
493 | int_thr.u64 = 0; |
494 | int_thr.s.tc_en = 1; |
495 | int_thr.s.tc_thr = 1; |
496 | cvmx_write_csr(CVMX_SSO_WQ_INT_THRX(i), val: int_thr.u64); |
497 | |
498 | int_pc.u64 = 0; |
499 | int_pc.s.pc_thr = 5; |
500 | cvmx_write_csr(CVMX_SSO_WQ_INT_PC, val: int_pc.u64); |
501 | } else { |
502 | union cvmx_pow_wq_int_thrx int_thr; |
503 | union cvmx_pow_wq_int_pc int_pc; |
504 | |
505 | int_thr.u64 = 0; |
506 | int_thr.s.tc_en = 1; |
507 | int_thr.s.tc_thr = 1; |
508 | cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), val: int_thr.u64); |
509 | |
510 | int_pc.u64 = 0; |
511 | int_pc.s.pc_thr = 5; |
512 | cvmx_write_csr(CVMX_POW_WQ_INT_PC, val: int_pc.u64); |
513 | } |
514 | |
515 | /* Schedule NAPI now. This will indirectly enable the |
516 | * interrupt. |
517 | */ |
518 | napi_schedule(n: &oct_rx_group[i].napi); |
519 | } |
520 | atomic_inc(v: &oct_rx_ready); |
521 | } |
522 | |
523 | void cvm_oct_rx_shutdown(void) |
524 | { |
525 | int i; |
526 | |
527 | for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) { |
528 | if (!(pow_receive_groups & BIT(i))) |
529 | continue; |
530 | |
531 | /* Disable POW interrupt */ |
532 | if (OCTEON_IS_MODEL(OCTEON_CN68XX)) |
533 | cvmx_write_csr(CVMX_SSO_WQ_INT_THRX(i), val: 0); |
534 | else |
535 | cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), val: 0); |
536 | |
537 | /* Free the interrupt handler */ |
538 | free_irq(oct_rx_group[i].irq, cvm_oct_device); |
539 | |
540 | netif_napi_del(napi: &oct_rx_group[i].napi); |
541 | } |
542 | } |
543 | |