1 | /********************************************************************** |
2 | * Author: Cavium, Inc. |
3 | * |
4 | * Contact: support@cavium.com |
5 | * Please include "LiquidIO" in the subject. |
6 | * |
7 | * Copyright (c) 2003-2016 Cavium, Inc. |
8 | * |
9 | * This file is free software; you can redistribute it and/or modify |
10 | * it under the terms of the GNU General Public License, Version 2, as |
11 | * published by the Free Software Foundation. |
12 | * |
13 | * This file is distributed in the hope that it will be useful, but |
14 | * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty |
15 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or |
16 | * NONINFRINGEMENT. See the GNU General Public License for more details. |
17 | ***********************************************************************/ |
18 | #include <linux/pci.h> |
19 | #include <linux/netdevice.h> |
20 | #include <linux/vmalloc.h> |
21 | #include "liquidio_common.h" |
22 | #include "octeon_droq.h" |
23 | #include "octeon_iq.h" |
24 | #include "response_manager.h" |
25 | #include "octeon_device.h" |
26 | #include "octeon_main.h" |
27 | #include "octeon_network.h" |
28 | #include "cn66xx_regs.h" |
29 | #include "cn66xx_device.h" |
30 | #include "cn23xx_pf_device.h" |
31 | #include "cn23xx_vf_device.h" |
32 | |
33 | struct niclist { |
34 | struct list_head list; |
35 | void *ptr; |
36 | }; |
37 | |
38 | struct __dispatch { |
39 | struct list_head list; |
40 | struct octeon_recv_info *rinfo; |
41 | octeon_dispatch_fn_t disp_fn; |
42 | }; |
43 | |
44 | /** Get the argument that the user set when registering dispatch |
45 | * function for a given opcode/subcode. |
46 | * @param octeon_dev - the octeon device pointer. |
47 | * @param opcode - the opcode for which the dispatch argument |
48 | * is to be checked. |
49 | * @param subcode - the subcode for which the dispatch argument |
50 | * is to be checked. |
51 | * @return Success: void * (argument to the dispatch function) |
52 | * @return Failure: NULL |
53 | * |
54 | */ |
55 | void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev, |
56 | u16 opcode, u16 subcode) |
57 | { |
58 | int idx; |
59 | struct list_head *dispatch; |
60 | void *fn_arg = NULL; |
61 | u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode); |
62 | |
63 | idx = combined_opcode & OCTEON_OPCODE_MASK; |
64 | |
65 | spin_lock_bh(lock: &octeon_dev->dispatch.lock); |
66 | |
67 | if (octeon_dev->dispatch.count == 0) { |
68 | spin_unlock_bh(lock: &octeon_dev->dispatch.lock); |
69 | return NULL; |
70 | } |
71 | |
72 | if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { |
73 | fn_arg = octeon_dev->dispatch.dlist[idx].arg; |
74 | } else { |
75 | list_for_each(dispatch, |
76 | &octeon_dev->dispatch.dlist[idx].list) { |
77 | if (((struct octeon_dispatch *)dispatch)->opcode == |
78 | combined_opcode) { |
79 | fn_arg = ((struct octeon_dispatch *) |
80 | dispatch)->arg; |
81 | break; |
82 | } |
83 | } |
84 | } |
85 | |
86 | spin_unlock_bh(lock: &octeon_dev->dispatch.lock); |
87 | return fn_arg; |
88 | } |
89 | |
90 | /** Check for packets on Droq. This function should be called with lock held. |
91 | * @param droq - Droq on which count is checked. |
92 | * @return Returns packet count. |
93 | */ |
94 | u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq) |
95 | { |
96 | u32 pkt_count = 0; |
97 | u32 last_count; |
98 | |
99 | pkt_count = readl(addr: droq->pkts_sent_reg); |
100 | |
101 | last_count = pkt_count - droq->pkt_count; |
102 | droq->pkt_count = pkt_count; |
103 | |
104 | /* we shall write to cnts at napi irq enable or end of droq tasklet */ |
105 | if (last_count) |
106 | atomic_add(i: last_count, v: &droq->pkts_pending); |
107 | |
108 | return last_count; |
109 | } |
110 | EXPORT_SYMBOL_GPL(octeon_droq_check_hw_for_pkts); |
111 | |
112 | static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq) |
113 | { |
114 | u32 count = 0; |
115 | |
116 | /* max_empty_descs is the max. no. of descs that can have no buffers. |
117 | * If the empty desc count goes beyond this value, we cannot safely |
118 | * read in a 64K packet sent by Octeon |
119 | * (64K is max pkt size from Octeon) |
120 | */ |
121 | droq->max_empty_descs = 0; |
122 | |
123 | do { |
124 | droq->max_empty_descs++; |
125 | count += droq->buffer_size; |
126 | } while (count < (64 * 1024)); |
127 | |
128 | droq->max_empty_descs = droq->max_count - droq->max_empty_descs; |
129 | } |
130 | |
131 | static void octeon_droq_reset_indices(struct octeon_droq *droq) |
132 | { |
133 | droq->read_idx = 0; |
134 | droq->write_idx = 0; |
135 | droq->refill_idx = 0; |
136 | droq->refill_count = 0; |
137 | atomic_set(v: &droq->pkts_pending, i: 0); |
138 | } |
139 | |
140 | static void |
141 | octeon_droq_destroy_ring_buffers(struct octeon_device *oct, |
142 | struct octeon_droq *droq) |
143 | { |
144 | u32 i; |
145 | struct octeon_skb_page_info *pg_info; |
146 | |
147 | for (i = 0; i < droq->max_count; i++) { |
148 | pg_info = &droq->recv_buf_list[i].pg_info; |
149 | if (!pg_info) |
150 | continue; |
151 | |
152 | if (pg_info->dma) |
153 | lio_unmap_ring(pci_dev: oct->pci_dev, |
154 | buf_ptr: (u64)pg_info->dma); |
155 | pg_info->dma = 0; |
156 | |
157 | if (pg_info->page) |
158 | recv_buffer_destroy(buffer: droq->recv_buf_list[i].buffer, |
159 | pg_info); |
160 | |
161 | droq->recv_buf_list[i].buffer = NULL; |
162 | } |
163 | |
164 | octeon_droq_reset_indices(droq); |
165 | } |
166 | |
167 | static int |
168 | octeon_droq_setup_ring_buffers(struct octeon_device *oct, |
169 | struct octeon_droq *droq) |
170 | { |
171 | u32 i; |
172 | void *buf; |
173 | struct octeon_droq_desc *desc_ring = droq->desc_ring; |
174 | |
175 | for (i = 0; i < droq->max_count; i++) { |
176 | buf = recv_buffer_alloc(oct, pg_info: &droq->recv_buf_list[i].pg_info); |
177 | |
178 | if (!buf) { |
179 | dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n" , |
180 | __func__); |
181 | droq->stats.rx_alloc_failure++; |
182 | return -ENOMEM; |
183 | } |
184 | |
185 | droq->recv_buf_list[i].buffer = buf; |
186 | droq->recv_buf_list[i].data = get_rbd(skb: buf); |
187 | desc_ring[i].info_ptr = 0; |
188 | desc_ring[i].buffer_ptr = |
189 | lio_map_ring(buf: droq->recv_buf_list[i].buffer); |
190 | } |
191 | |
192 | octeon_droq_reset_indices(droq); |
193 | |
194 | octeon_droq_compute_max_packet_bufs(droq); |
195 | |
196 | return 0; |
197 | } |
198 | |
199 | int octeon_delete_droq(struct octeon_device *oct, u32 q_no) |
200 | { |
201 | struct octeon_droq *droq = oct->droq[q_no]; |
202 | |
203 | dev_dbg(&oct->pci_dev->dev, "%s[%d]\n" , __func__, q_no); |
204 | |
205 | octeon_droq_destroy_ring_buffers(oct, droq); |
206 | vfree(addr: droq->recv_buf_list); |
207 | |
208 | if (droq->desc_ring) |
209 | lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE), |
210 | droq->desc_ring, droq->desc_ring_dma); |
211 | |
212 | memset(droq, 0, OCT_DROQ_SIZE); |
213 | oct->io_qmask.oq &= ~(1ULL << q_no); |
214 | vfree(addr: oct->droq[q_no]); |
215 | oct->droq[q_no] = NULL; |
216 | oct->num_oqs--; |
217 | |
218 | return 0; |
219 | } |
220 | EXPORT_SYMBOL_GPL(octeon_delete_droq); |
221 | |
222 | int octeon_init_droq(struct octeon_device *oct, |
223 | u32 q_no, |
224 | u32 num_descs, |
225 | u32 desc_size, |
226 | void *app_ctx) |
227 | { |
228 | struct octeon_droq *droq; |
229 | u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0; |
230 | u32 c_pkts_per_intr = 0, c_refill_threshold = 0; |
231 | int numa_node = dev_to_node(dev: &oct->pci_dev->dev); |
232 | |
233 | dev_dbg(&oct->pci_dev->dev, "%s[%d]\n" , __func__, q_no); |
234 | |
235 | droq = oct->droq[q_no]; |
236 | memset(droq, 0, OCT_DROQ_SIZE); |
237 | |
238 | droq->oct_dev = oct; |
239 | droq->q_no = q_no; |
240 | if (app_ctx) |
241 | droq->app_ctx = app_ctx; |
242 | else |
243 | droq->app_ctx = (void *)(size_t)q_no; |
244 | |
245 | c_num_descs = num_descs; |
246 | c_buf_size = desc_size; |
247 | if (OCTEON_CN6XXX(oct)) { |
248 | struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx); |
249 | |
250 | c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x); |
251 | c_refill_threshold = |
252 | (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x); |
253 | } else if (OCTEON_CN23XX_PF(oct)) { |
254 | struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf); |
255 | |
256 | c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); |
257 | c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); |
258 | } else if (OCTEON_CN23XX_VF(oct)) { |
259 | struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf); |
260 | |
261 | c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23); |
262 | c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23); |
263 | } else { |
264 | return 1; |
265 | } |
266 | |
267 | droq->max_count = c_num_descs; |
268 | droq->buffer_size = c_buf_size; |
269 | |
270 | desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE; |
271 | droq->desc_ring = lio_dma_alloc(oct, desc_ring_size, |
272 | (dma_addr_t *)&droq->desc_ring_dma); |
273 | |
274 | if (!droq->desc_ring) { |
275 | dev_err(&oct->pci_dev->dev, |
276 | "Output queue %d ring alloc failed\n" , q_no); |
277 | return 1; |
278 | } |
279 | |
280 | dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n" , |
281 | q_no, droq->desc_ring, droq->desc_ring_dma); |
282 | dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n" , q_no, |
283 | droq->max_count); |
284 | |
285 | droq->recv_buf_list = vzalloc_node(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE), |
286 | node: numa_node); |
287 | if (!droq->recv_buf_list) |
288 | droq->recv_buf_list = vzalloc(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE)); |
289 | if (!droq->recv_buf_list) { |
290 | dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n" ); |
291 | goto init_droq_fail; |
292 | } |
293 | |
294 | if (octeon_droq_setup_ring_buffers(oct, droq)) |
295 | goto init_droq_fail; |
296 | |
297 | droq->pkts_per_intr = c_pkts_per_intr; |
298 | droq->refill_threshold = c_refill_threshold; |
299 | |
300 | dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n" , |
301 | droq->max_empty_descs); |
302 | |
303 | INIT_LIST_HEAD(list: &droq->dispatch_list); |
304 | |
305 | /* For 56xx Pass1, this function won't be called, so no checks. */ |
306 | oct->fn_list.setup_oq_regs(oct, q_no); |
307 | |
308 | oct->io_qmask.oq |= BIT_ULL(q_no); |
309 | |
310 | return 0; |
311 | |
312 | init_droq_fail: |
313 | octeon_delete_droq(oct, q_no); |
314 | return 1; |
315 | } |
316 | |
317 | /* octeon_create_recv_info |
318 | * Parameters: |
319 | * octeon_dev - pointer to the octeon device structure |
320 | * droq - droq in which the packet arrived. |
321 | * buf_cnt - no. of buffers used by the packet. |
322 | * idx - index in the descriptor for the first buffer in the packet. |
323 | * Description: |
324 | * Allocates a recv_info_t and copies the buffer addresses for packet data |
325 | * into the recv_pkt space which starts at an 8B offset from recv_info_t. |
326 | * Flags the descriptors for refill later. If available descriptors go |
327 | * below the threshold to receive a 64K pkt, new buffers are first allocated |
328 | * before the recv_pkt_t is created. |
329 | * This routine will be called in interrupt context. |
330 | * Returns: |
331 | * Success: Pointer to recv_info_t |
332 | * Failure: NULL. |
333 | */ |
334 | static inline struct octeon_recv_info *octeon_create_recv_info( |
335 | struct octeon_device *octeon_dev, |
336 | struct octeon_droq *droq, |
337 | u32 buf_cnt, |
338 | u32 idx) |
339 | { |
340 | struct octeon_droq_info *info; |
341 | struct octeon_recv_pkt *recv_pkt; |
342 | struct octeon_recv_info *recv_info; |
343 | u32 i, bytes_left; |
344 | struct octeon_skb_page_info *pg_info; |
345 | |
346 | info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data; |
347 | |
348 | recv_info = octeon_alloc_recv_info(extra_bytes: sizeof(struct __dispatch)); |
349 | if (!recv_info) |
350 | return NULL; |
351 | |
352 | recv_pkt = recv_info->recv_pkt; |
353 | recv_pkt->rh = info->rh; |
354 | recv_pkt->length = (u32)info->length; |
355 | recv_pkt->buffer_count = (u16)buf_cnt; |
356 | recv_pkt->octeon_id = (u16)octeon_dev->octeon_id; |
357 | |
358 | i = 0; |
359 | bytes_left = (u32)info->length; |
360 | |
361 | while (buf_cnt) { |
362 | { |
363 | pg_info = &droq->recv_buf_list[idx].pg_info; |
364 | |
365 | lio_unmap_ring(pci_dev: octeon_dev->pci_dev, |
366 | buf_ptr: (u64)pg_info->dma); |
367 | pg_info->page = NULL; |
368 | pg_info->dma = 0; |
369 | } |
370 | |
371 | recv_pkt->buffer_size[i] = |
372 | (bytes_left >= |
373 | droq->buffer_size) ? droq->buffer_size : bytes_left; |
374 | |
375 | recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; |
376 | droq->recv_buf_list[idx].buffer = NULL; |
377 | |
378 | idx = incr_index(index: idx, count: 1, max: droq->max_count); |
379 | bytes_left -= droq->buffer_size; |
380 | i++; |
381 | buf_cnt--; |
382 | } |
383 | |
384 | return recv_info; |
385 | } |
386 | |
387 | /* If we were not able to refill all buffers, try to move around |
388 | * the buffers that were not dispatched. |
389 | */ |
390 | static inline u32 |
391 | octeon_droq_refill_pullup_descs(struct octeon_droq *droq, |
392 | struct octeon_droq_desc *desc_ring) |
393 | { |
394 | u32 desc_refilled = 0; |
395 | |
396 | u32 refill_index = droq->refill_idx; |
397 | |
398 | while (refill_index != droq->read_idx) { |
399 | if (droq->recv_buf_list[refill_index].buffer) { |
400 | droq->recv_buf_list[droq->refill_idx].buffer = |
401 | droq->recv_buf_list[refill_index].buffer; |
402 | droq->recv_buf_list[droq->refill_idx].data = |
403 | droq->recv_buf_list[refill_index].data; |
404 | desc_ring[droq->refill_idx].buffer_ptr = |
405 | desc_ring[refill_index].buffer_ptr; |
406 | droq->recv_buf_list[refill_index].buffer = NULL; |
407 | desc_ring[refill_index].buffer_ptr = 0; |
408 | do { |
409 | droq->refill_idx = incr_index(index: droq->refill_idx, |
410 | count: 1, |
411 | max: droq->max_count); |
412 | desc_refilled++; |
413 | droq->refill_count--; |
414 | } while (droq->recv_buf_list[droq->refill_idx].buffer); |
415 | } |
416 | refill_index = incr_index(index: refill_index, count: 1, max: droq->max_count); |
417 | } /* while */ |
418 | return desc_refilled; |
419 | } |
420 | |
421 | /* octeon_droq_refill |
422 | * Parameters: |
423 | * droq - droq in which descriptors require new buffers. |
424 | * Description: |
425 | * Called during normal DROQ processing in interrupt mode or by the poll |
426 | * thread to refill the descriptors from which buffers were dispatched |
427 | * to upper layers. Attempts to allocate new buffers. If that fails, moves |
428 | * up buffers (that were not dispatched) to form a contiguous ring. |
429 | * Returns: |
430 | * No of descriptors refilled. |
431 | */ |
432 | static u32 |
433 | octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq) |
434 | { |
435 | struct octeon_droq_desc *desc_ring; |
436 | void *buf = NULL; |
437 | u8 *data; |
438 | u32 desc_refilled = 0; |
439 | struct octeon_skb_page_info *pg_info; |
440 | |
441 | desc_ring = droq->desc_ring; |
442 | |
443 | while (droq->refill_count && (desc_refilled < droq->max_count)) { |
444 | /* If a valid buffer exists (happens if there is no dispatch), |
445 | * reuse the buffer, else allocate. |
446 | */ |
447 | if (!droq->recv_buf_list[droq->refill_idx].buffer) { |
448 | pg_info = |
449 | &droq->recv_buf_list[droq->refill_idx].pg_info; |
450 | /* Either recycle the existing pages or go for |
451 | * new page alloc |
452 | */ |
453 | if (pg_info->page) |
454 | buf = recv_buffer_reuse(oct: octeon_dev, buf: pg_info); |
455 | else |
456 | buf = recv_buffer_alloc(oct: octeon_dev, pg_info); |
457 | /* If a buffer could not be allocated, no point in |
458 | * continuing |
459 | */ |
460 | if (!buf) { |
461 | droq->stats.rx_alloc_failure++; |
462 | break; |
463 | } |
464 | droq->recv_buf_list[droq->refill_idx].buffer = |
465 | buf; |
466 | data = get_rbd(skb: buf); |
467 | } else { |
468 | data = get_rbd(skb: droq->recv_buf_list |
469 | [droq->refill_idx].buffer); |
470 | } |
471 | |
472 | droq->recv_buf_list[droq->refill_idx].data = data; |
473 | |
474 | desc_ring[droq->refill_idx].buffer_ptr = |
475 | lio_map_ring(buf: droq->recv_buf_list[ |
476 | droq->refill_idx].buffer); |
477 | |
478 | droq->refill_idx = incr_index(index: droq->refill_idx, count: 1, |
479 | max: droq->max_count); |
480 | desc_refilled++; |
481 | droq->refill_count--; |
482 | } |
483 | |
484 | if (droq->refill_count) |
485 | desc_refilled += |
486 | octeon_droq_refill_pullup_descs(droq, desc_ring); |
487 | |
488 | /* if droq->refill_count |
489 | * The refill count would not change in pass two. We only moved buffers |
490 | * to close the gap in the ring, but we would still have the same no. of |
491 | * buffers to refill. |
492 | */ |
493 | return desc_refilled; |
494 | } |
495 | |
496 | /** check if we can allocate packets to get out of oom. |
497 | * @param droq - Droq being checked. |
498 | * @return 1 if fails to refill minimum |
499 | */ |
500 | int octeon_retry_droq_refill(struct octeon_droq *droq) |
501 | { |
502 | struct octeon_device *oct = droq->oct_dev; |
503 | int desc_refilled, reschedule = 1; |
504 | u32 pkts_credit; |
505 | |
506 | pkts_credit = readl(addr: droq->pkts_credit_reg); |
507 | desc_refilled = octeon_droq_refill(octeon_dev: oct, droq); |
508 | if (desc_refilled) { |
509 | /* Flush the droq descriptor data to memory to be sure |
510 | * that when we update the credits the data in memory |
511 | * is accurate. |
512 | */ |
513 | wmb(); |
514 | writel(val: desc_refilled, addr: droq->pkts_credit_reg); |
515 | |
516 | if (pkts_credit + desc_refilled >= CN23XX_SLI_DEF_BP) |
517 | reschedule = 0; |
518 | } |
519 | |
520 | return reschedule; |
521 | } |
522 | |
523 | static inline u32 |
524 | octeon_droq_get_bufcount(u32 buf_size, u32 total_len) |
525 | { |
526 | return DIV_ROUND_UP(total_len, buf_size); |
527 | } |
528 | |
529 | static int |
530 | octeon_droq_dispatch_pkt(struct octeon_device *oct, |
531 | struct octeon_droq *droq, |
532 | union octeon_rh *rh, |
533 | struct octeon_droq_info *info) |
534 | { |
535 | u32 cnt; |
536 | octeon_dispatch_fn_t disp_fn; |
537 | struct octeon_recv_info *rinfo; |
538 | |
539 | cnt = octeon_droq_get_bufcount(buf_size: droq->buffer_size, total_len: (u32)info->length); |
540 | |
541 | disp_fn = octeon_get_dispatch(octeon_dev: oct, opcode: (u16)rh->r.opcode, |
542 | subcode: (u16)rh->r.subcode); |
543 | if (disp_fn) { |
544 | rinfo = octeon_create_recv_info(octeon_dev: oct, droq, buf_cnt: cnt, idx: droq->read_idx); |
545 | if (rinfo) { |
546 | struct __dispatch *rdisp = rinfo->rsvd; |
547 | |
548 | rdisp->rinfo = rinfo; |
549 | rdisp->disp_fn = disp_fn; |
550 | rinfo->recv_pkt->rh = *rh; |
551 | list_add_tail(new: &rdisp->list, |
552 | head: &droq->dispatch_list); |
553 | } else { |
554 | droq->stats.dropped_nomem++; |
555 | } |
556 | } else { |
557 | dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n" , |
558 | (unsigned int)rh->r.opcode, |
559 | (unsigned int)rh->r.subcode); |
560 | droq->stats.dropped_nodispatch++; |
561 | } |
562 | |
563 | return cnt; |
564 | } |
565 | |
566 | static inline void octeon_droq_drop_packets(struct octeon_device *oct, |
567 | struct octeon_droq *droq, |
568 | u32 cnt) |
569 | { |
570 | u32 i = 0, buf_cnt; |
571 | struct octeon_droq_info *info; |
572 | |
573 | for (i = 0; i < cnt; i++) { |
574 | info = (struct octeon_droq_info *) |
575 | droq->recv_buf_list[droq->read_idx].data; |
576 | octeon_swap_8B_data(data: (u64 *)info, blocks: 2); |
577 | |
578 | if (info->length) { |
579 | info->length += OCTNET_FRM_LENGTH_SIZE; |
580 | droq->stats.bytes_received += info->length; |
581 | buf_cnt = octeon_droq_get_bufcount(buf_size: droq->buffer_size, |
582 | total_len: (u32)info->length); |
583 | } else { |
584 | dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n" ); |
585 | buf_cnt = 1; |
586 | } |
587 | |
588 | droq->read_idx = incr_index(index: droq->read_idx, count: buf_cnt, |
589 | max: droq->max_count); |
590 | droq->refill_count += buf_cnt; |
591 | } |
592 | } |
593 | |
594 | static u32 |
595 | octeon_droq_fast_process_packets(struct octeon_device *oct, |
596 | struct octeon_droq *droq, |
597 | u32 pkts_to_process) |
598 | { |
599 | u32 pkt, total_len = 0, pkt_count, retval; |
600 | struct octeon_droq_info *info; |
601 | union octeon_rh *rh; |
602 | |
603 | pkt_count = pkts_to_process; |
604 | |
605 | for (pkt = 0; pkt < pkt_count; pkt++) { |
606 | u32 pkt_len = 0; |
607 | struct sk_buff *nicbuf = NULL; |
608 | struct octeon_skb_page_info *pg_info; |
609 | void *buf; |
610 | |
611 | info = (struct octeon_droq_info *) |
612 | droq->recv_buf_list[droq->read_idx].data; |
613 | octeon_swap_8B_data(data: (u64 *)info, blocks: 2); |
614 | |
615 | if (!info->length) { |
616 | dev_err(&oct->pci_dev->dev, |
617 | "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n" , |
618 | droq->q_no, droq->read_idx, pkt_count); |
619 | print_hex_dump_bytes("" , DUMP_PREFIX_ADDRESS, |
620 | (u8 *)info, |
621 | OCT_DROQ_INFO_SIZE); |
622 | break; |
623 | } |
624 | |
625 | /* Len of resp hdr in included in the received data len. */ |
626 | rh = &info->rh; |
627 | |
628 | info->length += OCTNET_FRM_LENGTH_SIZE; |
629 | rh->r_dh.len += (ROUNDUP8(OCT_DROQ_INFO_SIZE) / sizeof(u64)); |
630 | total_len += (u32)info->length; |
631 | if (opcode_slow_path(rh)) { |
632 | u32 buf_cnt; |
633 | |
634 | buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info); |
635 | droq->read_idx = incr_index(index: droq->read_idx, |
636 | count: buf_cnt, max: droq->max_count); |
637 | droq->refill_count += buf_cnt; |
638 | } else { |
639 | if (info->length <= droq->buffer_size) { |
640 | pkt_len = (u32)info->length; |
641 | nicbuf = droq->recv_buf_list[ |
642 | droq->read_idx].buffer; |
643 | pg_info = &droq->recv_buf_list[ |
644 | droq->read_idx].pg_info; |
645 | if (recv_buffer_recycle(oct, buf: pg_info)) |
646 | pg_info->page = NULL; |
647 | droq->recv_buf_list[droq->read_idx].buffer = |
648 | NULL; |
649 | |
650 | droq->read_idx = incr_index(index: droq->read_idx, count: 1, |
651 | max: droq->max_count); |
652 | droq->refill_count++; |
653 | } else { |
654 | nicbuf = octeon_fast_packet_alloc(size: (u32) |
655 | info->length); |
656 | pkt_len = 0; |
657 | /* nicbuf allocation can fail. We'll handle it |
658 | * inside the loop. |
659 | */ |
660 | while (pkt_len < info->length) { |
661 | int cpy_len, idx = droq->read_idx; |
662 | |
663 | cpy_len = ((pkt_len + droq->buffer_size) |
664 | > info->length) ? |
665 | ((u32)info->length - pkt_len) : |
666 | droq->buffer_size; |
667 | |
668 | if (nicbuf) { |
669 | octeon_fast_packet_next(droq, |
670 | nicbuf, |
671 | copy_len: cpy_len, |
672 | idx); |
673 | buf = droq->recv_buf_list[ |
674 | idx].buffer; |
675 | recv_buffer_fast_free(buffer: buf); |
676 | droq->recv_buf_list[idx].buffer |
677 | = NULL; |
678 | } else { |
679 | droq->stats.rx_alloc_failure++; |
680 | } |
681 | |
682 | pkt_len += cpy_len; |
683 | droq->read_idx = |
684 | incr_index(index: droq->read_idx, count: 1, |
685 | max: droq->max_count); |
686 | droq->refill_count++; |
687 | } |
688 | } |
689 | |
690 | if (nicbuf) { |
691 | if (droq->ops.fptr) { |
692 | droq->ops.fptr(oct->octeon_id, |
693 | nicbuf, pkt_len, |
694 | rh, &droq->napi, |
695 | droq->ops.farg); |
696 | } else { |
697 | recv_buffer_free(buffer: nicbuf); |
698 | } |
699 | } |
700 | } |
701 | |
702 | if (droq->refill_count >= droq->refill_threshold) { |
703 | int desc_refilled = octeon_droq_refill(octeon_dev: oct, droq); |
704 | |
705 | if (desc_refilled) { |
706 | /* Flush the droq descriptor data to memory to |
707 | * be sure that when we update the credits the |
708 | * data in memory is accurate. |
709 | */ |
710 | wmb(); |
711 | writel(val: desc_refilled, addr: droq->pkts_credit_reg); |
712 | } |
713 | } |
714 | } /* for (each packet)... */ |
715 | |
716 | /* Increment refill_count by the number of buffers processed. */ |
717 | droq->stats.pkts_received += pkt; |
718 | droq->stats.bytes_received += total_len; |
719 | |
720 | retval = pkt; |
721 | if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { |
722 | octeon_droq_drop_packets(oct, droq, cnt: (pkts_to_process - pkt)); |
723 | |
724 | droq->stats.dropped_toomany += (pkts_to_process - pkt); |
725 | retval = pkts_to_process; |
726 | } |
727 | |
728 | atomic_sub(i: retval, v: &droq->pkts_pending); |
729 | |
730 | if (droq->refill_count >= droq->refill_threshold && |
731 | readl(addr: droq->pkts_credit_reg) < CN23XX_SLI_DEF_BP) { |
732 | octeon_droq_check_hw_for_pkts(droq); |
733 | |
734 | /* Make sure there are no pkts_pending */ |
735 | if (!atomic_read(v: &droq->pkts_pending)) |
736 | octeon_schedule_rxq_oom_work(oct, droq); |
737 | } |
738 | |
739 | return retval; |
740 | } |
741 | |
742 | int |
743 | octeon_droq_process_packets(struct octeon_device *oct, |
744 | struct octeon_droq *droq, |
745 | u32 budget) |
746 | { |
747 | u32 pkt_count = 0; |
748 | struct list_head *tmp, *tmp2; |
749 | |
750 | octeon_droq_check_hw_for_pkts(droq); |
751 | pkt_count = atomic_read(v: &droq->pkts_pending); |
752 | |
753 | if (!pkt_count) |
754 | return 0; |
755 | |
756 | if (pkt_count > budget) |
757 | pkt_count = budget; |
758 | |
759 | octeon_droq_fast_process_packets(oct, droq, pkts_to_process: pkt_count); |
760 | |
761 | list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { |
762 | struct __dispatch *rdisp = (struct __dispatch *)tmp; |
763 | |
764 | list_del(entry: tmp); |
765 | rdisp->disp_fn(rdisp->rinfo, |
766 | octeon_get_dispatch_arg |
767 | (octeon_dev: oct, |
768 | opcode: (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, |
769 | subcode: (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); |
770 | } |
771 | |
772 | /* If there are packets pending. schedule tasklet again */ |
773 | if (atomic_read(v: &droq->pkts_pending)) |
774 | return 1; |
775 | |
776 | return 0; |
777 | } |
778 | EXPORT_SYMBOL_GPL(octeon_droq_process_packets); |
779 | |
780 | /* |
781 | * Utility function to poll for packets. check_hw_for_packets must be |
782 | * called before calling this routine. |
783 | */ |
784 | |
785 | int |
786 | octeon_droq_process_poll_pkts(struct octeon_device *oct, |
787 | struct octeon_droq *droq, u32 budget) |
788 | { |
789 | struct list_head *tmp, *tmp2; |
790 | u32 pkts_available = 0, pkts_processed = 0; |
791 | u32 total_pkts_processed = 0; |
792 | |
793 | if (budget > droq->max_count) |
794 | budget = droq->max_count; |
795 | |
796 | while (total_pkts_processed < budget) { |
797 | octeon_droq_check_hw_for_pkts(droq); |
798 | |
799 | pkts_available = min((budget - total_pkts_processed), |
800 | (u32)(atomic_read(&droq->pkts_pending))); |
801 | |
802 | if (pkts_available == 0) |
803 | break; |
804 | |
805 | pkts_processed = |
806 | octeon_droq_fast_process_packets(oct, droq, |
807 | pkts_to_process: pkts_available); |
808 | |
809 | total_pkts_processed += pkts_processed; |
810 | } |
811 | |
812 | list_for_each_safe(tmp, tmp2, &droq->dispatch_list) { |
813 | struct __dispatch *rdisp = (struct __dispatch *)tmp; |
814 | |
815 | list_del(entry: tmp); |
816 | rdisp->disp_fn(rdisp->rinfo, |
817 | octeon_get_dispatch_arg |
818 | (octeon_dev: oct, |
819 | opcode: (u16)rdisp->rinfo->recv_pkt->rh.r.opcode, |
820 | subcode: (u16)rdisp->rinfo->recv_pkt->rh.r.subcode)); |
821 | } |
822 | |
823 | return total_pkts_processed; |
824 | } |
825 | |
826 | /* Enable Pkt Interrupt */ |
827 | int |
828 | octeon_enable_irq(struct octeon_device *oct, u32 q_no) |
829 | { |
830 | switch (oct->chip_id) { |
831 | case OCTEON_CN66XX: |
832 | case OCTEON_CN68XX: { |
833 | struct octeon_cn6xxx *cn6xxx = |
834 | (struct octeon_cn6xxx *)oct->chip; |
835 | unsigned long flags; |
836 | u32 value; |
837 | |
838 | spin_lock_irqsave |
839 | (&cn6xxx->lock_for_droq_int_enb_reg, flags); |
840 | value = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB); |
841 | value |= (1 << q_no); |
842 | octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, value); |
843 | value = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB); |
844 | value |= (1 << q_no); |
845 | octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, value); |
846 | |
847 | /* don't bother flushing the enables */ |
848 | |
849 | spin_unlock_irqrestore |
850 | (lock: &cn6xxx->lock_for_droq_int_enb_reg, flags); |
851 | } |
852 | break; |
853 | case OCTEON_CN23XX_PF_VID: |
854 | lio_enable_irq(droq: oct->droq[q_no], iq: oct->instr_queue[q_no]); |
855 | break; |
856 | |
857 | case OCTEON_CN23XX_VF_VID: |
858 | lio_enable_irq(droq: oct->droq[q_no], iq: oct->instr_queue[q_no]); |
859 | break; |
860 | default: |
861 | dev_err(&oct->pci_dev->dev, "%s Unknown Chip\n" , __func__); |
862 | return 1; |
863 | } |
864 | |
865 | return 0; |
866 | } |
867 | |
868 | int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no, |
869 | struct octeon_droq_ops *ops) |
870 | { |
871 | struct octeon_config *oct_cfg = NULL; |
872 | struct octeon_droq *droq; |
873 | |
874 | oct_cfg = octeon_get_conf(oct); |
875 | |
876 | if (!oct_cfg) |
877 | return -EINVAL; |
878 | |
879 | if (!(ops)) { |
880 | dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n" , |
881 | __func__); |
882 | return -EINVAL; |
883 | } |
884 | |
885 | if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { |
886 | dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n" , |
887 | __func__, q_no, (oct->num_oqs - 1)); |
888 | return -EINVAL; |
889 | } |
890 | |
891 | droq = oct->droq[q_no]; |
892 | memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops)); |
893 | |
894 | return 0; |
895 | } |
896 | |
897 | int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no) |
898 | { |
899 | struct octeon_config *oct_cfg = NULL; |
900 | struct octeon_droq *droq; |
901 | |
902 | oct_cfg = octeon_get_conf(oct); |
903 | |
904 | if (!oct_cfg) |
905 | return -EINVAL; |
906 | |
907 | if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) { |
908 | dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n" , |
909 | __func__, q_no, oct->num_oqs - 1); |
910 | return -EINVAL; |
911 | } |
912 | |
913 | droq = oct->droq[q_no]; |
914 | |
915 | if (!droq) { |
916 | dev_info(&oct->pci_dev->dev, |
917 | "Droq id (%d) not available.\n" , q_no); |
918 | return 0; |
919 | } |
920 | |
921 | droq->ops.fptr = NULL; |
922 | droq->ops.farg = NULL; |
923 | droq->ops.drop_on_max = 0; |
924 | |
925 | return 0; |
926 | } |
927 | EXPORT_SYMBOL_GPL(octeon_unregister_droq_ops); |
928 | |
929 | int octeon_create_droq(struct octeon_device *oct, |
930 | u32 q_no, u32 num_descs, |
931 | u32 desc_size, void *app_ctx) |
932 | { |
933 | struct octeon_droq *droq; |
934 | int numa_node = dev_to_node(dev: &oct->pci_dev->dev); |
935 | |
936 | if (oct->droq[q_no]) { |
937 | dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n" , |
938 | q_no); |
939 | return 1; |
940 | } |
941 | |
942 | /* Allocate the DS for the new droq. */ |
943 | droq = vmalloc_node(size: sizeof(*droq), node: numa_node); |
944 | if (!droq) |
945 | droq = vmalloc(size: sizeof(*droq)); |
946 | if (!droq) |
947 | return -1; |
948 | |
949 | memset(droq, 0, sizeof(struct octeon_droq)); |
950 | |
951 | /*Disable the pkt o/p for this Q */ |
952 | octeon_set_droq_pkt_op(oct, q_no, enable: 0); |
953 | oct->droq[q_no] = droq; |
954 | |
955 | /* Initialize the Droq */ |
956 | if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) { |
957 | vfree(addr: oct->droq[q_no]); |
958 | oct->droq[q_no] = NULL; |
959 | return -1; |
960 | } |
961 | |
962 | oct->num_oqs++; |
963 | |
964 | dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n" , __func__, |
965 | oct->num_oqs); |
966 | |
967 | /* Global Droq register settings */ |
968 | |
969 | /* As of now not required, as setting are done for all 32 Droqs at |
970 | * the same time. |
971 | */ |
972 | return 0; |
973 | } |
974 | |