1 | /* |
2 | * This file is part of the Chelsio FCoE driver for Linux. |
3 | * |
4 | * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved. |
5 | * |
6 | * This software is available to you under a choice of one of two |
7 | * licenses. You may choose to be licensed under the terms of the GNU |
8 | * General Public License (GPL) Version 2, available from the file |
9 | * COPYING in the main directory of this source tree, or the |
10 | * OpenIB.org BSD license below: |
11 | * |
12 | * Redistribution and use in source and binary forms, with or |
13 | * without modification, are permitted provided that the following |
14 | * conditions are met: |
15 | * |
16 | * - Redistributions of source code must retain the above |
17 | * copyright notice, this list of conditions and the following |
18 | * disclaimer. |
19 | * |
20 | * - Redistributions in binary form must reproduce the above |
21 | * copyright notice, this list of conditions and the following |
22 | * disclaimer in the documentation and/or other materials |
23 | * provided with the distribution. |
24 | * |
25 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
26 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
27 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
28 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
29 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
30 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
31 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
32 | * SOFTWARE. |
33 | */ |
34 | |
35 | #include <linux/kernel.h> |
36 | #include <linux/string.h> |
37 | #include <linux/compiler.h> |
38 | #include <linux/slab.h> |
39 | #include <asm/page.h> |
40 | #include <linux/cache.h> |
41 | |
42 | #include "t4_values.h" |
43 | #include "csio_hw.h" |
44 | #include "csio_wr.h" |
45 | #include "csio_mb.h" |
46 | #include "csio_defs.h" |
47 | |
48 | int csio_intr_coalesce_cnt; /* value:SGE_INGRESS_RX_THRESHOLD[0] */ |
49 | static int csio_sge_thresh_reg; /* SGE_INGRESS_RX_THRESHOLD[0] */ |
50 | |
51 | int csio_intr_coalesce_time = 10; /* value:SGE_TIMER_VALUE_1 */ |
52 | static int csio_sge_timer_reg = 1; |
53 | |
54 | #define CSIO_SET_FLBUF_SIZE(_hw, _reg, _val) \ |
55 | csio_wr_reg32((_hw), (_val), SGE_FL_BUFFER_SIZE##_reg##_A) |
56 | |
57 | static void |
58 | csio_get_flbuf_size(struct csio_hw *hw, struct csio_sge *sge, uint32_t reg) |
59 | { |
60 | sge->sge_fl_buf_size[reg] = csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE0_A + |
61 | reg * sizeof(uint32_t)); |
62 | } |
63 | |
64 | /* Free list buffer size */ |
65 | static inline uint32_t |
66 | csio_wr_fl_bufsz(struct csio_sge *sge, struct csio_dma_buf *buf) |
67 | { |
68 | return sge->sge_fl_buf_size[buf->paddr & 0xF]; |
69 | } |
70 | |
71 | /* Size of the egress queue status page */ |
72 | static inline uint32_t |
73 | csio_wr_qstat_pgsz(struct csio_hw *hw) |
74 | { |
75 | return (hw->wrm.sge.sge_control & EGRSTATUSPAGESIZE_F) ? 128 : 64; |
76 | } |
77 | |
78 | /* Ring freelist doorbell */ |
79 | static inline void |
80 | csio_wr_ring_fldb(struct csio_hw *hw, struct csio_q *flq) |
81 | { |
82 | /* |
83 | * Ring the doorbell only when we have atleast CSIO_QCREDIT_SZ |
84 | * number of bytes in the freelist queue. This translates to atleast |
85 | * 8 freelist buffer pointers (since each pointer is 8 bytes). |
86 | */ |
87 | if (flq->inc_idx >= 8) { |
88 | csio_wr_reg32(hw, DBPRIO_F | QID_V(flq->un.fl.flid) | |
89 | PIDX_T5_V(flq->inc_idx / 8) | DBTYPE_F, |
90 | MYPF_REG(SGE_PF_KDOORBELL_A)); |
91 | flq->inc_idx &= 7; |
92 | } |
93 | } |
94 | |
95 | /* Write a 0 cidx increment value to enable SGE interrupts for this queue */ |
96 | static void |
97 | csio_wr_sge_intr_enable(struct csio_hw *hw, uint16_t iqid) |
98 | { |
99 | csio_wr_reg32(hw, CIDXINC_V(0) | |
100 | INGRESSQID_V(iqid) | |
101 | TIMERREG_V(X_TIMERREG_RESTART_COUNTER), |
102 | MYPF_REG(SGE_PF_GTS_A)); |
103 | } |
104 | |
105 | /* |
106 | * csio_wr_fill_fl - Populate the FL buffers of a FL queue. |
107 | * @hw: HW module. |
108 | * @flq: Freelist queue. |
109 | * |
110 | * Fill up freelist buffer entries with buffers of size specified |
111 | * in the size register. |
112 | * |
113 | */ |
114 | static int |
115 | csio_wr_fill_fl(struct csio_hw *hw, struct csio_q *flq) |
116 | { |
117 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
118 | struct csio_sge *sge = &wrm->sge; |
119 | __be64 *d = (__be64 *)(flq->vstart); |
120 | struct csio_dma_buf *buf = &flq->un.fl.bufs[0]; |
121 | uint64_t paddr; |
122 | int sreg = flq->un.fl.sreg; |
123 | int n = flq->credits; |
124 | |
125 | while (n--) { |
126 | buf->len = sge->sge_fl_buf_size[sreg]; |
127 | buf->vaddr = dma_alloc_coherent(dev: &hw->pdev->dev, size: buf->len, |
128 | dma_handle: &buf->paddr, GFP_KERNEL); |
129 | if (!buf->vaddr) { |
130 | csio_err(hw, "Could only fill %d buffers!\n" , n + 1); |
131 | return -ENOMEM; |
132 | } |
133 | |
134 | paddr = buf->paddr | (sreg & 0xF); |
135 | |
136 | *d++ = cpu_to_be64(paddr); |
137 | buf++; |
138 | } |
139 | |
140 | return 0; |
141 | } |
142 | |
143 | /* |
144 | * csio_wr_update_fl - |
145 | * @hw: HW module. |
146 | * @flq: Freelist queue. |
147 | * |
148 | * |
149 | */ |
150 | static inline void |
151 | csio_wr_update_fl(struct csio_hw *hw, struct csio_q *flq, uint16_t n) |
152 | { |
153 | |
154 | flq->inc_idx += n; |
155 | flq->pidx += n; |
156 | if (unlikely(flq->pidx >= flq->credits)) |
157 | flq->pidx -= (uint16_t)flq->credits; |
158 | |
159 | CSIO_INC_STATS(flq, n_flq_refill); |
160 | } |
161 | |
162 | /* |
163 | * csio_wr_alloc_q - Allocate a WR queue and initialize it. |
164 | * @hw: HW module |
165 | * @qsize: Size of the queue in bytes |
166 | * @wrsize: Since of WR in this queue, if fixed. |
167 | * @type: Type of queue (Ingress/Egress/Freelist) |
168 | * @owner: Module that owns this queue. |
169 | * @nflb: Number of freelist buffers for FL. |
170 | * @sreg: What is the FL buffer size register? |
171 | * @iq_int_handler: Ingress queue handler in INTx mode. |
172 | * |
173 | * This function allocates and sets up a queue for the caller |
174 | * of size qsize, aligned at the required boundary. This is subject to |
175 | * be free entries being available in the queue array. If one is found, |
176 | * it is initialized with the allocated queue, marked as being used (owner), |
177 | * and a handle returned to the caller in form of the queue's index |
178 | * into the q_arr array. |
179 | * If user has indicated a freelist (by specifying nflb > 0), create |
180 | * another queue (with its own index into q_arr) for the freelist. Allocate |
181 | * memory for DMA buffer metadata (vaddr, len etc). Save off the freelist |
182 | * idx in the ingress queue's flq.idx. This is how a Freelist is associated |
183 | * with its owning ingress queue. |
184 | */ |
185 | int |
186 | csio_wr_alloc_q(struct csio_hw *hw, uint32_t qsize, uint32_t wrsize, |
187 | uint16_t type, void *owner, uint32_t nflb, int sreg, |
188 | iq_handler_t iq_intx_handler) |
189 | { |
190 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
191 | struct csio_q *q, *flq; |
192 | int free_idx = wrm->free_qidx; |
193 | int ret_idx = free_idx; |
194 | uint32_t qsz; |
195 | int flq_idx; |
196 | |
197 | if (free_idx >= wrm->num_q) { |
198 | csio_err(hw, "No more free queues.\n" ); |
199 | return -1; |
200 | } |
201 | |
202 | switch (type) { |
203 | case CSIO_EGRESS: |
204 | qsz = ALIGN(qsize, CSIO_QCREDIT_SZ) + csio_wr_qstat_pgsz(hw); |
205 | break; |
206 | case CSIO_INGRESS: |
207 | switch (wrsize) { |
208 | case 16: |
209 | case 32: |
210 | case 64: |
211 | case 128: |
212 | break; |
213 | default: |
214 | csio_err(hw, "Invalid Ingress queue WR size:%d\n" , |
215 | wrsize); |
216 | return -1; |
217 | } |
218 | |
219 | /* |
220 | * Number of elements must be a multiple of 16 |
221 | * So this includes status page size |
222 | */ |
223 | qsz = ALIGN(qsize/wrsize, 16) * wrsize; |
224 | |
225 | break; |
226 | case CSIO_FREELIST: |
227 | qsz = ALIGN(qsize/wrsize, 8) * wrsize + csio_wr_qstat_pgsz(hw); |
228 | break; |
229 | default: |
230 | csio_err(hw, "Invalid queue type: 0x%x\n" , type); |
231 | return -1; |
232 | } |
233 | |
234 | q = wrm->q_arr[free_idx]; |
235 | |
236 | q->vstart = dma_alloc_coherent(dev: &hw->pdev->dev, size: qsz, dma_handle: &q->pstart, |
237 | GFP_KERNEL); |
238 | if (!q->vstart) { |
239 | csio_err(hw, |
240 | "Failed to allocate DMA memory for " |
241 | "queue at id: %d size: %d\n" , free_idx, qsize); |
242 | return -1; |
243 | } |
244 | |
245 | q->type = type; |
246 | q->owner = owner; |
247 | q->pidx = q->cidx = q->inc_idx = 0; |
248 | q->size = qsz; |
249 | q->wr_sz = wrsize; /* If using fixed size WRs */ |
250 | |
251 | wrm->free_qidx++; |
252 | |
253 | if (type == CSIO_INGRESS) { |
254 | /* Since queue area is set to zero */ |
255 | q->un.iq.genbit = 1; |
256 | |
257 | /* |
258 | * Ingress queue status page size is always the size of |
259 | * the ingress queue entry. |
260 | */ |
261 | q->credits = (qsz - q->wr_sz) / q->wr_sz; |
262 | q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz |
263 | - q->wr_sz); |
264 | |
265 | /* Allocate memory for FL if requested */ |
266 | if (nflb > 0) { |
267 | flq_idx = csio_wr_alloc_q(hw, qsize: nflb * sizeof(__be64), |
268 | wrsize: sizeof(__be64), type: CSIO_FREELIST, |
269 | owner, nflb: 0, sreg, NULL); |
270 | if (flq_idx == -1) { |
271 | csio_err(hw, |
272 | "Failed to allocate FL queue" |
273 | " for IQ idx:%d\n" , free_idx); |
274 | return -1; |
275 | } |
276 | |
277 | /* Associate the new FL with the Ingress quue */ |
278 | q->un.iq.flq_idx = flq_idx; |
279 | |
280 | flq = wrm->q_arr[q->un.iq.flq_idx]; |
281 | flq->un.fl.bufs = kcalloc(n: flq->credits, |
282 | size: sizeof(struct csio_dma_buf), |
283 | GFP_KERNEL); |
284 | if (!flq->un.fl.bufs) { |
285 | csio_err(hw, |
286 | "Failed to allocate FL queue bufs" |
287 | " for IQ idx:%d\n" , free_idx); |
288 | return -1; |
289 | } |
290 | |
291 | flq->un.fl.packen = 0; |
292 | flq->un.fl.offset = 0; |
293 | flq->un.fl.sreg = sreg; |
294 | |
295 | /* Fill up the free list buffers */ |
296 | if (csio_wr_fill_fl(hw, flq)) |
297 | return -1; |
298 | |
299 | /* |
300 | * Make sure in a FLQ, atleast 1 credit (8 FL buffers) |
301 | * remains unpopulated,otherwise HW thinks |
302 | * FLQ is empty. |
303 | */ |
304 | flq->pidx = flq->inc_idx = flq->credits - 8; |
305 | } else { |
306 | q->un.iq.flq_idx = -1; |
307 | } |
308 | |
309 | /* Associate the IQ INTx handler. */ |
310 | q->un.iq.iq_intx_handler = iq_intx_handler; |
311 | |
312 | csio_q_iqid(hw, ret_idx) = CSIO_MAX_QID; |
313 | |
314 | } else if (type == CSIO_EGRESS) { |
315 | q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / CSIO_QCREDIT_SZ; |
316 | q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz |
317 | - csio_wr_qstat_pgsz(hw)); |
318 | csio_q_eqid(hw, ret_idx) = CSIO_MAX_QID; |
319 | } else { /* Freelist */ |
320 | q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / sizeof(__be64); |
321 | q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz |
322 | - csio_wr_qstat_pgsz(hw)); |
323 | csio_q_flid(hw, ret_idx) = CSIO_MAX_QID; |
324 | } |
325 | |
326 | return ret_idx; |
327 | } |
328 | |
329 | /* |
330 | * csio_wr_iq_create_rsp - Response handler for IQ creation. |
331 | * @hw: The HW module. |
332 | * @mbp: Mailbox. |
333 | * @iq_idx: Ingress queue that got created. |
334 | * |
335 | * Handle FW_IQ_CMD mailbox completion. Save off the assigned IQ/FL ids. |
336 | */ |
337 | static int |
338 | csio_wr_iq_create_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx) |
339 | { |
340 | struct csio_iq_params iqp; |
341 | enum fw_retval retval; |
342 | uint32_t iq_id; |
343 | int flq_idx; |
344 | |
345 | memset(&iqp, 0, sizeof(struct csio_iq_params)); |
346 | |
347 | csio_mb_iq_alloc_write_rsp(hw, mbp, &retval, &iqp); |
348 | |
349 | if (retval != FW_SUCCESS) { |
350 | csio_err(hw, "IQ cmd returned 0x%x!\n" , retval); |
351 | mempool_free(element: mbp, pool: hw->mb_mempool); |
352 | return -EINVAL; |
353 | } |
354 | |
355 | csio_q_iqid(hw, iq_idx) = iqp.iqid; |
356 | csio_q_physiqid(hw, iq_idx) = iqp.physiqid; |
357 | csio_q_pidx(hw, iq_idx) = csio_q_cidx(hw, iq_idx) = 0; |
358 | csio_q_inc_idx(hw, iq_idx) = 0; |
359 | |
360 | /* Actual iq-id. */ |
361 | iq_id = iqp.iqid - hw->wrm.fw_iq_start; |
362 | |
363 | /* Set the iq-id to iq map table. */ |
364 | if (iq_id >= CSIO_MAX_IQ) { |
365 | csio_err(hw, |
366 | "Exceeding MAX_IQ(%d) supported!" |
367 | " iqid:%d rel_iqid:%d FW iq_start:%d\n" , |
368 | CSIO_MAX_IQ, iq_id, iqp.iqid, hw->wrm.fw_iq_start); |
369 | mempool_free(element: mbp, pool: hw->mb_mempool); |
370 | return -EINVAL; |
371 | } |
372 | csio_q_set_intr_map(hw, iq_idx, iq_id); |
373 | |
374 | /* |
375 | * During FW_IQ_CMD, FW sets interrupt_sent bit to 1 in the SGE |
376 | * ingress context of this queue. This will block interrupts to |
377 | * this queue until the next GTS write. Therefore, we do a |
378 | * 0-cidx increment GTS write for this queue just to clear the |
379 | * interrupt_sent bit. This will re-enable interrupts to this |
380 | * queue. |
381 | */ |
382 | csio_wr_sge_intr_enable(hw, iqid: iqp.physiqid); |
383 | |
384 | flq_idx = csio_q_iq_flq_idx(hw, iq_idx); |
385 | if (flq_idx != -1) { |
386 | struct csio_q *flq = hw->wrm.q_arr[flq_idx]; |
387 | |
388 | csio_q_flid(hw, flq_idx) = iqp.fl0id; |
389 | csio_q_cidx(hw, flq_idx) = 0; |
390 | csio_q_pidx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8; |
391 | csio_q_inc_idx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8; |
392 | |
393 | /* Now update SGE about the buffers allocated during init */ |
394 | csio_wr_ring_fldb(hw, flq); |
395 | } |
396 | |
397 | mempool_free(element: mbp, pool: hw->mb_mempool); |
398 | |
399 | return 0; |
400 | } |
401 | |
402 | /* |
403 | * csio_wr_iq_create - Configure an Ingress queue with FW. |
404 | * @hw: The HW module. |
405 | * @priv: Private data object. |
406 | * @iq_idx: Ingress queue index in the WR module. |
407 | * @vec: MSIX vector. |
408 | * @portid: PCIE Channel to be associated with this queue. |
409 | * @async: Is this a FW asynchronous message handling queue? |
410 | * @cbfn: Completion callback. |
411 | * |
412 | * This API configures an ingress queue with FW by issuing a FW_IQ_CMD mailbox |
413 | * with alloc/write bits set. |
414 | */ |
415 | int |
416 | csio_wr_iq_create(struct csio_hw *hw, void *priv, int iq_idx, |
417 | uint32_t vec, uint8_t portid, bool async, |
418 | void (*cbfn) (struct csio_hw *, struct csio_mb *)) |
419 | { |
420 | struct csio_mb *mbp; |
421 | struct csio_iq_params iqp; |
422 | int flq_idx; |
423 | |
424 | memset(&iqp, 0, sizeof(struct csio_iq_params)); |
425 | csio_q_portid(hw, iq_idx) = portid; |
426 | |
427 | mbp = mempool_alloc(pool: hw->mb_mempool, GFP_ATOMIC); |
428 | if (!mbp) { |
429 | csio_err(hw, "IQ command out of memory!\n" ); |
430 | return -ENOMEM; |
431 | } |
432 | |
433 | switch (hw->intr_mode) { |
434 | case CSIO_IM_INTX: |
435 | case CSIO_IM_MSI: |
436 | /* For interrupt forwarding queue only */ |
437 | if (hw->intr_iq_idx == iq_idx) |
438 | iqp.iqandst = X_INTERRUPTDESTINATION_PCIE; |
439 | else |
440 | iqp.iqandst = X_INTERRUPTDESTINATION_IQ; |
441 | iqp.iqandstindex = |
442 | csio_q_physiqid(hw, hw->intr_iq_idx); |
443 | break; |
444 | case CSIO_IM_MSIX: |
445 | iqp.iqandst = X_INTERRUPTDESTINATION_PCIE; |
446 | iqp.iqandstindex = (uint16_t)vec; |
447 | break; |
448 | case CSIO_IM_NONE: |
449 | mempool_free(element: mbp, pool: hw->mb_mempool); |
450 | return -EINVAL; |
451 | } |
452 | |
453 | /* Pass in the ingress queue cmd parameters */ |
454 | iqp.pfn = hw->pfn; |
455 | iqp.vfn = 0; |
456 | iqp.iq_start = 1; |
457 | iqp.viid = 0; |
458 | iqp.type = FW_IQ_TYPE_FL_INT_CAP; |
459 | iqp.iqasynch = async; |
460 | if (csio_intr_coalesce_cnt) |
461 | iqp.iqanus = X_UPDATESCHEDULING_COUNTER_OPTTIMER; |
462 | else |
463 | iqp.iqanus = X_UPDATESCHEDULING_TIMER; |
464 | iqp.iqanud = X_UPDATEDELIVERY_INTERRUPT; |
465 | iqp.iqpciech = portid; |
466 | iqp.iqintcntthresh = (uint8_t)csio_sge_thresh_reg; |
467 | |
468 | switch (csio_q_wr_sz(hw, iq_idx)) { |
469 | case 16: |
470 | iqp.iqesize = 0; break; |
471 | case 32: |
472 | iqp.iqesize = 1; break; |
473 | case 64: |
474 | iqp.iqesize = 2; break; |
475 | case 128: |
476 | iqp.iqesize = 3; break; |
477 | } |
478 | |
479 | iqp.iqsize = csio_q_size(hw, iq_idx) / |
480 | csio_q_wr_sz(hw, iq_idx); |
481 | iqp.iqaddr = csio_q_pstart(hw, iq_idx); |
482 | |
483 | flq_idx = csio_q_iq_flq_idx(hw, iq_idx); |
484 | if (flq_idx != -1) { |
485 | enum chip_type chip = CHELSIO_CHIP_VERSION(hw->chip_id); |
486 | struct csio_q *flq = hw->wrm.q_arr[flq_idx]; |
487 | |
488 | iqp.fl0paden = 1; |
489 | iqp.fl0packen = flq->un.fl.packen ? 1 : 0; |
490 | iqp.fl0fbmin = X_FETCHBURSTMIN_64B; |
491 | iqp.fl0fbmax = ((chip == CHELSIO_T5) ? |
492 | X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B); |
493 | iqp.fl0size = csio_q_size(hw, flq_idx) / CSIO_QCREDIT_SZ; |
494 | iqp.fl0addr = csio_q_pstart(hw, flq_idx); |
495 | } |
496 | |
497 | csio_mb_iq_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn); |
498 | |
499 | if (csio_mb_issue(hw, mbp)) { |
500 | csio_err(hw, "Issue of IQ cmd failed!\n" ); |
501 | mempool_free(element: mbp, pool: hw->mb_mempool); |
502 | return -EINVAL; |
503 | } |
504 | |
505 | if (cbfn != NULL) |
506 | return 0; |
507 | |
508 | return csio_wr_iq_create_rsp(hw, mbp, iq_idx); |
509 | } |
510 | |
511 | /* |
512 | * csio_wr_eq_create_rsp - Response handler for EQ creation. |
513 | * @hw: The HW module. |
514 | * @mbp: Mailbox. |
515 | * @eq_idx: Egress queue that got created. |
516 | * |
517 | * Handle FW_EQ_OFLD_CMD mailbox completion. Save off the assigned EQ ids. |
518 | */ |
519 | static int |
520 | csio_wr_eq_cfg_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx) |
521 | { |
522 | struct csio_eq_params eqp; |
523 | enum fw_retval retval; |
524 | |
525 | memset(&eqp, 0, sizeof(struct csio_eq_params)); |
526 | |
527 | csio_mb_eq_ofld_alloc_write_rsp(hw, mbp, &retval, &eqp); |
528 | |
529 | if (retval != FW_SUCCESS) { |
530 | csio_err(hw, "EQ OFLD cmd returned 0x%x!\n" , retval); |
531 | mempool_free(element: mbp, pool: hw->mb_mempool); |
532 | return -EINVAL; |
533 | } |
534 | |
535 | csio_q_eqid(hw, eq_idx) = (uint16_t)eqp.eqid; |
536 | csio_q_physeqid(hw, eq_idx) = (uint16_t)eqp.physeqid; |
537 | csio_q_pidx(hw, eq_idx) = csio_q_cidx(hw, eq_idx) = 0; |
538 | csio_q_inc_idx(hw, eq_idx) = 0; |
539 | |
540 | mempool_free(element: mbp, pool: hw->mb_mempool); |
541 | |
542 | return 0; |
543 | } |
544 | |
545 | /* |
546 | * csio_wr_eq_create - Configure an Egress queue with FW. |
547 | * @hw: HW module. |
548 | * @priv: Private data. |
549 | * @eq_idx: Egress queue index in the WR module. |
550 | * @iq_idx: Associated ingress queue index. |
551 | * @cbfn: Completion callback. |
552 | * |
553 | * This API configures a offload egress queue with FW by issuing a |
554 | * FW_EQ_OFLD_CMD (with alloc + write ) mailbox. |
555 | */ |
556 | int |
557 | csio_wr_eq_create(struct csio_hw *hw, void *priv, int eq_idx, |
558 | int iq_idx, uint8_t portid, |
559 | void (*cbfn) (struct csio_hw *, struct csio_mb *)) |
560 | { |
561 | struct csio_mb *mbp; |
562 | struct csio_eq_params eqp; |
563 | |
564 | memset(&eqp, 0, sizeof(struct csio_eq_params)); |
565 | |
566 | mbp = mempool_alloc(pool: hw->mb_mempool, GFP_ATOMIC); |
567 | if (!mbp) { |
568 | csio_err(hw, "EQ command out of memory!\n" ); |
569 | return -ENOMEM; |
570 | } |
571 | |
572 | eqp.pfn = hw->pfn; |
573 | eqp.vfn = 0; |
574 | eqp.eqstart = 1; |
575 | eqp.hostfcmode = X_HOSTFCMODE_STATUS_PAGE; |
576 | eqp.iqid = csio_q_iqid(hw, iq_idx); |
577 | eqp.fbmin = X_FETCHBURSTMIN_64B; |
578 | eqp.fbmax = X_FETCHBURSTMAX_512B; |
579 | eqp.cidxfthresh = 0; |
580 | eqp.pciechn = portid; |
581 | eqp.eqsize = csio_q_size(hw, eq_idx) / CSIO_QCREDIT_SZ; |
582 | eqp.eqaddr = csio_q_pstart(hw, eq_idx); |
583 | |
584 | csio_mb_eq_ofld_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, |
585 | &eqp, cbfn); |
586 | |
587 | if (csio_mb_issue(hw, mbp)) { |
588 | csio_err(hw, "Issue of EQ OFLD cmd failed!\n" ); |
589 | mempool_free(element: mbp, pool: hw->mb_mempool); |
590 | return -EINVAL; |
591 | } |
592 | |
593 | if (cbfn != NULL) |
594 | return 0; |
595 | |
596 | return csio_wr_eq_cfg_rsp(hw, mbp, eq_idx); |
597 | } |
598 | |
599 | /* |
600 | * csio_wr_iq_destroy_rsp - Response handler for IQ removal. |
601 | * @hw: The HW module. |
602 | * @mbp: Mailbox. |
603 | * @iq_idx: Ingress queue that was freed. |
604 | * |
605 | * Handle FW_IQ_CMD (free) mailbox completion. |
606 | */ |
607 | static int |
608 | csio_wr_iq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx) |
609 | { |
610 | enum fw_retval retval = csio_mb_fw_retval(mbp); |
611 | int rv = 0; |
612 | |
613 | if (retval != FW_SUCCESS) |
614 | rv = -EINVAL; |
615 | |
616 | mempool_free(element: mbp, pool: hw->mb_mempool); |
617 | |
618 | return rv; |
619 | } |
620 | |
621 | /* |
622 | * csio_wr_iq_destroy - Free an ingress queue. |
623 | * @hw: The HW module. |
624 | * @priv: Private data object. |
625 | * @iq_idx: Ingress queue index to destroy |
626 | * @cbfn: Completion callback. |
627 | * |
628 | * This API frees an ingress queue by issuing the FW_IQ_CMD |
629 | * with the free bit set. |
630 | */ |
631 | static int |
632 | csio_wr_iq_destroy(struct csio_hw *hw, void *priv, int iq_idx, |
633 | void (*cbfn)(struct csio_hw *, struct csio_mb *)) |
634 | { |
635 | int rv = 0; |
636 | struct csio_mb *mbp; |
637 | struct csio_iq_params iqp; |
638 | int flq_idx; |
639 | |
640 | memset(&iqp, 0, sizeof(struct csio_iq_params)); |
641 | |
642 | mbp = mempool_alloc(pool: hw->mb_mempool, GFP_ATOMIC); |
643 | if (!mbp) |
644 | return -ENOMEM; |
645 | |
646 | iqp.pfn = hw->pfn; |
647 | iqp.vfn = 0; |
648 | iqp.iqid = csio_q_iqid(hw, iq_idx); |
649 | iqp.type = FW_IQ_TYPE_FL_INT_CAP; |
650 | |
651 | flq_idx = csio_q_iq_flq_idx(hw, iq_idx); |
652 | if (flq_idx != -1) |
653 | iqp.fl0id = csio_q_flid(hw, flq_idx); |
654 | else |
655 | iqp.fl0id = 0xFFFF; |
656 | |
657 | iqp.fl1id = 0xFFFF; |
658 | |
659 | csio_mb_iq_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn); |
660 | |
661 | rv = csio_mb_issue(hw, mbp); |
662 | if (rv != 0) { |
663 | mempool_free(element: mbp, pool: hw->mb_mempool); |
664 | return rv; |
665 | } |
666 | |
667 | if (cbfn != NULL) |
668 | return 0; |
669 | |
670 | return csio_wr_iq_destroy_rsp(hw, mbp, iq_idx); |
671 | } |
672 | |
673 | /* |
674 | * csio_wr_eq_destroy_rsp - Response handler for OFLD EQ creation. |
675 | * @hw: The HW module. |
676 | * @mbp: Mailbox. |
677 | * @eq_idx: Egress queue that was freed. |
678 | * |
679 | * Handle FW_OFLD_EQ_CMD (free) mailbox completion. |
680 | */ |
681 | static int |
682 | csio_wr_eq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx) |
683 | { |
684 | enum fw_retval retval = csio_mb_fw_retval(mbp); |
685 | int rv = 0; |
686 | |
687 | if (retval != FW_SUCCESS) |
688 | rv = -EINVAL; |
689 | |
690 | mempool_free(element: mbp, pool: hw->mb_mempool); |
691 | |
692 | return rv; |
693 | } |
694 | |
695 | /* |
696 | * csio_wr_eq_destroy - Free an Egress queue. |
697 | * @hw: The HW module. |
698 | * @priv: Private data object. |
699 | * @eq_idx: Egress queue index to destroy |
700 | * @cbfn: Completion callback. |
701 | * |
702 | * This API frees an Egress queue by issuing the FW_EQ_OFLD_CMD |
703 | * with the free bit set. |
704 | */ |
705 | static int |
706 | csio_wr_eq_destroy(struct csio_hw *hw, void *priv, int eq_idx, |
707 | void (*cbfn) (struct csio_hw *, struct csio_mb *)) |
708 | { |
709 | int rv = 0; |
710 | struct csio_mb *mbp; |
711 | struct csio_eq_params eqp; |
712 | |
713 | memset(&eqp, 0, sizeof(struct csio_eq_params)); |
714 | |
715 | mbp = mempool_alloc(pool: hw->mb_mempool, GFP_ATOMIC); |
716 | if (!mbp) |
717 | return -ENOMEM; |
718 | |
719 | eqp.pfn = hw->pfn; |
720 | eqp.vfn = 0; |
721 | eqp.eqid = csio_q_eqid(hw, eq_idx); |
722 | |
723 | csio_mb_eq_ofld_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &eqp, cbfn); |
724 | |
725 | rv = csio_mb_issue(hw, mbp); |
726 | if (rv != 0) { |
727 | mempool_free(element: mbp, pool: hw->mb_mempool); |
728 | return rv; |
729 | } |
730 | |
731 | if (cbfn != NULL) |
732 | return 0; |
733 | |
734 | return csio_wr_eq_destroy_rsp(hw, mbp, eq_idx); |
735 | } |
736 | |
737 | /* |
738 | * csio_wr_cleanup_eq_stpg - Cleanup Egress queue status page |
739 | * @hw: HW module |
740 | * @qidx: Egress queue index |
741 | * |
742 | * Cleanup the Egress queue status page. |
743 | */ |
744 | static void |
745 | csio_wr_cleanup_eq_stpg(struct csio_hw *hw, int qidx) |
746 | { |
747 | struct csio_q *q = csio_hw_to_wrm(hw)->q_arr[qidx]; |
748 | struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap; |
749 | |
750 | memset(stp, 0, sizeof(*stp)); |
751 | } |
752 | |
753 | /* |
754 | * csio_wr_cleanup_iq_ftr - Cleanup Footer entries in IQ |
755 | * @hw: HW module |
756 | * @qidx: Ingress queue index |
757 | * |
758 | * Cleanup the footer entries in the given ingress queue, |
759 | * set to 1 the internal copy of genbit. |
760 | */ |
761 | static void |
762 | csio_wr_cleanup_iq_ftr(struct csio_hw *hw, int qidx) |
763 | { |
764 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
765 | struct csio_q *q = wrm->q_arr[qidx]; |
766 | void *wr; |
767 | struct csio_iqwr_footer *ftr; |
768 | uint32_t i = 0; |
769 | |
770 | /* set to 1 since we are just about zero out genbit */ |
771 | q->un.iq.genbit = 1; |
772 | |
773 | for (i = 0; i < q->credits; i++) { |
774 | /* Get the WR */ |
775 | wr = (void *)((uintptr_t)q->vstart + |
776 | (i * q->wr_sz)); |
777 | /* Get the footer */ |
778 | ftr = (struct csio_iqwr_footer *)((uintptr_t)wr + |
779 | (q->wr_sz - sizeof(*ftr))); |
780 | /* Zero out footer */ |
781 | memset(ftr, 0, sizeof(*ftr)); |
782 | } |
783 | } |
784 | |
785 | int |
786 | csio_wr_destroy_queues(struct csio_hw *hw, bool cmd) |
787 | { |
788 | int i, flq_idx; |
789 | struct csio_q *q; |
790 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
791 | int rv; |
792 | |
793 | for (i = 0; i < wrm->free_qidx; i++) { |
794 | q = wrm->q_arr[i]; |
795 | |
796 | switch (q->type) { |
797 | case CSIO_EGRESS: |
798 | if (csio_q_eqid(hw, i) != CSIO_MAX_QID) { |
799 | csio_wr_cleanup_eq_stpg(hw, qidx: i); |
800 | if (!cmd) { |
801 | csio_q_eqid(hw, i) = CSIO_MAX_QID; |
802 | continue; |
803 | } |
804 | |
805 | rv = csio_wr_eq_destroy(hw, NULL, eq_idx: i, NULL); |
806 | if ((rv == -EBUSY) || (rv == -ETIMEDOUT)) |
807 | cmd = false; |
808 | |
809 | csio_q_eqid(hw, i) = CSIO_MAX_QID; |
810 | } |
811 | fallthrough; |
812 | case CSIO_INGRESS: |
813 | if (csio_q_iqid(hw, i) != CSIO_MAX_QID) { |
814 | csio_wr_cleanup_iq_ftr(hw, qidx: i); |
815 | if (!cmd) { |
816 | csio_q_iqid(hw, i) = CSIO_MAX_QID; |
817 | flq_idx = csio_q_iq_flq_idx(hw, i); |
818 | if (flq_idx != -1) |
819 | csio_q_flid(hw, flq_idx) = |
820 | CSIO_MAX_QID; |
821 | continue; |
822 | } |
823 | |
824 | rv = csio_wr_iq_destroy(hw, NULL, iq_idx: i, NULL); |
825 | if ((rv == -EBUSY) || (rv == -ETIMEDOUT)) |
826 | cmd = false; |
827 | |
828 | csio_q_iqid(hw, i) = CSIO_MAX_QID; |
829 | flq_idx = csio_q_iq_flq_idx(hw, i); |
830 | if (flq_idx != -1) |
831 | csio_q_flid(hw, flq_idx) = CSIO_MAX_QID; |
832 | } |
833 | break; |
834 | default: |
835 | break; |
836 | } |
837 | } |
838 | |
839 | hw->flags &= ~CSIO_HWF_Q_FW_ALLOCED; |
840 | |
841 | return 0; |
842 | } |
843 | |
844 | /* |
845 | * csio_wr_get - Get requested size of WR entry/entries from queue. |
846 | * @hw: HW module. |
847 | * @qidx: Index of queue. |
848 | * @size: Cumulative size of Work request(s). |
849 | * @wrp: Work request pair. |
850 | * |
851 | * If requested credits are available, return the start address of the |
852 | * work request in the work request pair. Set pidx accordingly and |
853 | * return. |
854 | * |
855 | * NOTE about WR pair: |
856 | * ================== |
857 | * A WR can start towards the end of a queue, and then continue at the |
858 | * beginning, since the queue is considered to be circular. This will |
859 | * require a pair of address/size to be passed back to the caller - |
860 | * hence Work request pair format. |
861 | */ |
862 | int |
863 | csio_wr_get(struct csio_hw *hw, int qidx, uint32_t size, |
864 | struct csio_wr_pair *wrp) |
865 | { |
866 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
867 | struct csio_q *q = wrm->q_arr[qidx]; |
868 | void *cwr = (void *)((uintptr_t)(q->vstart) + |
869 | (q->pidx * CSIO_QCREDIT_SZ)); |
870 | struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap; |
871 | uint16_t cidx = q->cidx = ntohs(stp->cidx); |
872 | uint16_t pidx = q->pidx; |
873 | uint32_t req_sz = ALIGN(size, CSIO_QCREDIT_SZ); |
874 | int req_credits = req_sz / CSIO_QCREDIT_SZ; |
875 | int credits; |
876 | |
877 | CSIO_DB_ASSERT(q->owner != NULL); |
878 | CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx)); |
879 | CSIO_DB_ASSERT(cidx <= q->credits); |
880 | |
881 | /* Calculate credits */ |
882 | if (pidx > cidx) { |
883 | credits = q->credits - (pidx - cidx) - 1; |
884 | } else if (cidx > pidx) { |
885 | credits = cidx - pidx - 1; |
886 | } else { |
887 | /* cidx == pidx, empty queue */ |
888 | credits = q->credits; |
889 | CSIO_INC_STATS(q, n_qempty); |
890 | } |
891 | |
892 | /* |
893 | * Check if we have enough credits. |
894 | * credits = 1 implies queue is full. |
895 | */ |
896 | if (!credits || (req_credits > credits)) { |
897 | CSIO_INC_STATS(q, n_qfull); |
898 | return -EBUSY; |
899 | } |
900 | |
901 | /* |
902 | * If we are here, we have enough credits to satisfy the |
903 | * request. Check if we are near the end of q, and if WR spills over. |
904 | * If it does, use the first addr/size to cover the queue until |
905 | * the end. Fit the remainder portion of the request at the top |
906 | * of queue and return it in the second addr/len. Set pidx |
907 | * accordingly. |
908 | */ |
909 | if (unlikely(((uintptr_t)cwr + req_sz) > (uintptr_t)(q->vwrap))) { |
910 | wrp->addr1 = cwr; |
911 | wrp->size1 = (uint32_t)((uintptr_t)q->vwrap - (uintptr_t)cwr); |
912 | wrp->addr2 = q->vstart; |
913 | wrp->size2 = req_sz - wrp->size1; |
914 | q->pidx = (uint16_t)(ALIGN(wrp->size2, CSIO_QCREDIT_SZ) / |
915 | CSIO_QCREDIT_SZ); |
916 | CSIO_INC_STATS(q, n_qwrap); |
917 | CSIO_INC_STATS(q, n_eq_wr_split); |
918 | } else { |
919 | wrp->addr1 = cwr; |
920 | wrp->size1 = req_sz; |
921 | wrp->addr2 = NULL; |
922 | wrp->size2 = 0; |
923 | q->pidx += (uint16_t)req_credits; |
924 | |
925 | /* We are the end of queue, roll back pidx to top of queue */ |
926 | if (unlikely(q->pidx == q->credits)) { |
927 | q->pidx = 0; |
928 | CSIO_INC_STATS(q, n_qwrap); |
929 | } |
930 | } |
931 | |
932 | q->inc_idx = (uint16_t)req_credits; |
933 | |
934 | CSIO_INC_STATS(q, n_tot_reqs); |
935 | |
936 | return 0; |
937 | } |
938 | |
939 | /* |
940 | * csio_wr_copy_to_wrp - Copies given data into WR. |
941 | * @data_buf - Data buffer |
942 | * @wrp - Work request pair. |
943 | * @wr_off - Work request offset. |
944 | * @data_len - Data length. |
945 | * |
946 | * Copies the given data in Work Request. Work request pair(wrp) specifies |
947 | * address information of Work request. |
948 | * Returns: none |
949 | */ |
950 | void |
951 | csio_wr_copy_to_wrp(void *data_buf, struct csio_wr_pair *wrp, |
952 | uint32_t wr_off, uint32_t data_len) |
953 | { |
954 | uint32_t nbytes; |
955 | |
956 | /* Number of space available in buffer addr1 of WRP */ |
957 | nbytes = ((wrp->size1 - wr_off) >= data_len) ? |
958 | data_len : (wrp->size1 - wr_off); |
959 | |
960 | memcpy((uint8_t *) wrp->addr1 + wr_off, data_buf, nbytes); |
961 | data_len -= nbytes; |
962 | |
963 | /* Write the remaining data from the begining of circular buffer */ |
964 | if (data_len) { |
965 | CSIO_DB_ASSERT(data_len <= wrp->size2); |
966 | CSIO_DB_ASSERT(wrp->addr2 != NULL); |
967 | memcpy(wrp->addr2, (uint8_t *) data_buf + nbytes, data_len); |
968 | } |
969 | } |
970 | |
971 | /* |
972 | * csio_wr_issue - Notify chip of Work request. |
973 | * @hw: HW module. |
974 | * @qidx: Index of queue. |
975 | * @prio: 0: Low priority, 1: High priority |
976 | * |
977 | * Rings the SGE Doorbell by writing the current producer index of the passed |
978 | * in queue into the register. |
979 | * |
980 | */ |
981 | int |
982 | csio_wr_issue(struct csio_hw *hw, int qidx, bool prio) |
983 | { |
984 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
985 | struct csio_q *q = wrm->q_arr[qidx]; |
986 | |
987 | CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx)); |
988 | |
989 | wmb(); |
990 | /* Ring SGE Doorbell writing q->pidx into it */ |
991 | csio_wr_reg32(hw, DBPRIO_V(prio) | QID_V(q->un.eq.physeqid) | |
992 | PIDX_T5_V(q->inc_idx) | DBTYPE_F, |
993 | MYPF_REG(SGE_PF_KDOORBELL_A)); |
994 | q->inc_idx = 0; |
995 | |
996 | return 0; |
997 | } |
998 | |
999 | static inline uint32_t |
1000 | csio_wr_avail_qcredits(struct csio_q *q) |
1001 | { |
1002 | if (q->pidx > q->cidx) |
1003 | return q->pidx - q->cidx; |
1004 | else if (q->cidx > q->pidx) |
1005 | return q->credits - (q->cidx - q->pidx); |
1006 | else |
1007 | return 0; /* cidx == pidx, empty queue */ |
1008 | } |
1009 | |
1010 | /* |
1011 | * csio_wr_inval_flq_buf - Invalidate a free list buffer entry. |
1012 | * @hw: HW module. |
1013 | * @flq: The freelist queue. |
1014 | * |
1015 | * Invalidate the driver's version of a freelist buffer entry, |
1016 | * without freeing the associated the DMA memory. The entry |
1017 | * to be invalidated is picked up from the current Free list |
1018 | * queue cidx. |
1019 | * |
1020 | */ |
1021 | static inline void |
1022 | csio_wr_inval_flq_buf(struct csio_hw *hw, struct csio_q *flq) |
1023 | { |
1024 | flq->cidx++; |
1025 | if (flq->cidx == flq->credits) { |
1026 | flq->cidx = 0; |
1027 | CSIO_INC_STATS(flq, n_qwrap); |
1028 | } |
1029 | } |
1030 | |
1031 | /* |
1032 | * csio_wr_process_fl - Process a freelist completion. |
1033 | * @hw: HW module. |
1034 | * @q: The ingress queue attached to the Freelist. |
1035 | * @wr: The freelist completion WR in the ingress queue. |
1036 | * @len_to_qid: The lower 32-bits of the first flit of the RSP footer |
1037 | * @iq_handler: Caller's handler for this completion. |
1038 | * @priv: Private pointer of caller |
1039 | * |
1040 | */ |
1041 | static inline void |
1042 | csio_wr_process_fl(struct csio_hw *hw, struct csio_q *q, |
1043 | void *wr, uint32_t len_to_qid, |
1044 | void (*iq_handler)(struct csio_hw *, void *, |
1045 | uint32_t, struct csio_fl_dma_buf *, |
1046 | void *), |
1047 | void *priv) |
1048 | { |
1049 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
1050 | struct csio_sge *sge = &wrm->sge; |
1051 | struct csio_fl_dma_buf flb; |
1052 | struct csio_dma_buf *buf, *fbuf; |
1053 | uint32_t bufsz, len, lastlen = 0; |
1054 | struct csio_q *flq = hw->wrm.q_arr[q->un.iq.flq_idx]; |
1055 | |
1056 | CSIO_DB_ASSERT(flq != NULL); |
1057 | |
1058 | len = len_to_qid; |
1059 | |
1060 | if (len & IQWRF_NEWBUF) { |
1061 | if (flq->un.fl.offset > 0) { |
1062 | csio_wr_inval_flq_buf(hw, flq); |
1063 | flq->un.fl.offset = 0; |
1064 | } |
1065 | len = IQWRF_LEN_GET(len); |
1066 | } |
1067 | |
1068 | CSIO_DB_ASSERT(len != 0); |
1069 | |
1070 | flb.totlen = len; |
1071 | |
1072 | /* Consume all freelist buffers used for len bytes */ |
1073 | for (fbuf = flb.flbufs; ; fbuf++) { |
1074 | buf = &flq->un.fl.bufs[flq->cidx]; |
1075 | bufsz = csio_wr_fl_bufsz(sge, buf); |
1076 | |
1077 | fbuf->paddr = buf->paddr; |
1078 | fbuf->vaddr = buf->vaddr; |
1079 | |
1080 | flb.offset = flq->un.fl.offset; |
1081 | lastlen = min(bufsz, len); |
1082 | fbuf->len = lastlen; |
1083 | |
1084 | len -= lastlen; |
1085 | if (!len) |
1086 | break; |
1087 | csio_wr_inval_flq_buf(hw, flq); |
1088 | } |
1089 | |
1090 | flb.defer_free = flq->un.fl.packen ? 0 : 1; |
1091 | |
1092 | iq_handler(hw, wr, q->wr_sz - sizeof(struct csio_iqwr_footer), |
1093 | &flb, priv); |
1094 | |
1095 | if (flq->un.fl.packen) |
1096 | flq->un.fl.offset += ALIGN(lastlen, sge->csio_fl_align); |
1097 | else |
1098 | csio_wr_inval_flq_buf(hw, flq); |
1099 | |
1100 | } |
1101 | |
1102 | /* |
1103 | * csio_is_new_iqwr - Is this a new Ingress queue entry ? |
1104 | * @q: Ingress quueue. |
1105 | * @ftr: Ingress queue WR SGE footer. |
1106 | * |
1107 | * The entry is new if our generation bit matches the corresponding |
1108 | * bit in the footer of the current WR. |
1109 | */ |
1110 | static inline bool |
1111 | csio_is_new_iqwr(struct csio_q *q, struct csio_iqwr_footer *ftr) |
1112 | { |
1113 | return (q->un.iq.genbit == (ftr->u.type_gen >> IQWRF_GEN_SHIFT)); |
1114 | } |
1115 | |
1116 | /* |
1117 | * csio_wr_process_iq - Process elements in Ingress queue. |
1118 | * @hw: HW pointer |
1119 | * @qidx: Index of queue |
1120 | * @iq_handler: Handler for this queue |
1121 | * @priv: Caller's private pointer |
1122 | * |
1123 | * This routine walks through every entry of the ingress queue, calling |
1124 | * the provided iq_handler with the entry, until the generation bit |
1125 | * flips. |
1126 | */ |
1127 | int |
1128 | csio_wr_process_iq(struct csio_hw *hw, struct csio_q *q, |
1129 | void (*iq_handler)(struct csio_hw *, void *, |
1130 | uint32_t, struct csio_fl_dma_buf *, |
1131 | void *), |
1132 | void *priv) |
1133 | { |
1134 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
1135 | void *wr = (void *)((uintptr_t)q->vstart + (q->cidx * q->wr_sz)); |
1136 | struct csio_iqwr_footer *ftr; |
1137 | uint32_t wr_type, fw_qid, qid; |
1138 | struct csio_q *q_completed; |
1139 | struct csio_q *flq = csio_iq_has_fl(q) ? |
1140 | wrm->q_arr[q->un.iq.flq_idx] : NULL; |
1141 | int rv = 0; |
1142 | |
1143 | /* Get the footer */ |
1144 | ftr = (struct csio_iqwr_footer *)((uintptr_t)wr + |
1145 | (q->wr_sz - sizeof(*ftr))); |
1146 | |
1147 | /* |
1148 | * When q wrapped around last time, driver should have inverted |
1149 | * ic.genbit as well. |
1150 | */ |
1151 | while (csio_is_new_iqwr(q, ftr)) { |
1152 | |
1153 | CSIO_DB_ASSERT(((uintptr_t)wr + q->wr_sz) <= |
1154 | (uintptr_t)q->vwrap); |
1155 | rmb(); |
1156 | wr_type = IQWRF_TYPE_GET(ftr->u.type_gen); |
1157 | |
1158 | switch (wr_type) { |
1159 | case X_RSPD_TYPE_CPL: |
1160 | /* Subtract footer from WR len */ |
1161 | iq_handler(hw, wr, q->wr_sz - sizeof(*ftr), NULL, priv); |
1162 | break; |
1163 | case X_RSPD_TYPE_FLBUF: |
1164 | csio_wr_process_fl(hw, q, wr, |
1165 | ntohl(ftr->pldbuflen_qid), |
1166 | iq_handler, priv); |
1167 | break; |
1168 | case X_RSPD_TYPE_INTR: |
1169 | fw_qid = ntohl(ftr->pldbuflen_qid); |
1170 | qid = fw_qid - wrm->fw_iq_start; |
1171 | q_completed = hw->wrm.intr_map[qid]; |
1172 | |
1173 | if (unlikely(qid == |
1174 | csio_q_physiqid(hw, hw->intr_iq_idx))) { |
1175 | /* |
1176 | * We are already in the Forward Interrupt |
1177 | * Interrupt Queue Service! Do-not service |
1178 | * again! |
1179 | * |
1180 | */ |
1181 | } else { |
1182 | CSIO_DB_ASSERT(q_completed); |
1183 | CSIO_DB_ASSERT( |
1184 | q_completed->un.iq.iq_intx_handler); |
1185 | |
1186 | /* Call the queue handler. */ |
1187 | q_completed->un.iq.iq_intx_handler(hw, NULL, |
1188 | 0, NULL, (void *)q_completed); |
1189 | } |
1190 | break; |
1191 | default: |
1192 | csio_warn(hw, "Unknown resp type 0x%x received\n" , |
1193 | wr_type); |
1194 | CSIO_INC_STATS(q, n_rsp_unknown); |
1195 | break; |
1196 | } |
1197 | |
1198 | /* |
1199 | * Ingress *always* has fixed size WR entries. Therefore, |
1200 | * there should always be complete WRs towards the end of |
1201 | * queue. |
1202 | */ |
1203 | if (((uintptr_t)wr + q->wr_sz) == (uintptr_t)q->vwrap) { |
1204 | |
1205 | /* Roll over to start of queue */ |
1206 | q->cidx = 0; |
1207 | wr = q->vstart; |
1208 | |
1209 | /* Toggle genbit */ |
1210 | q->un.iq.genbit ^= 0x1; |
1211 | |
1212 | CSIO_INC_STATS(q, n_qwrap); |
1213 | } else { |
1214 | q->cidx++; |
1215 | wr = (void *)((uintptr_t)(q->vstart) + |
1216 | (q->cidx * q->wr_sz)); |
1217 | } |
1218 | |
1219 | ftr = (struct csio_iqwr_footer *)((uintptr_t)wr + |
1220 | (q->wr_sz - sizeof(*ftr))); |
1221 | q->inc_idx++; |
1222 | |
1223 | } /* while (q->un.iq.genbit == hdr->genbit) */ |
1224 | |
1225 | /* |
1226 | * We need to re-arm SGE interrupts in case we got a stray interrupt, |
1227 | * especially in msix mode. With INTx, this may be a common occurence. |
1228 | */ |
1229 | if (unlikely(!q->inc_idx)) { |
1230 | CSIO_INC_STATS(q, n_stray_comp); |
1231 | rv = -EINVAL; |
1232 | goto restart; |
1233 | } |
1234 | |
1235 | /* Replenish free list buffers if pending falls below low water mark */ |
1236 | if (flq) { |
1237 | uint32_t avail = csio_wr_avail_qcredits(q: flq); |
1238 | if (avail <= 16) { |
1239 | /* Make sure in FLQ, atleast 1 credit (8 FL buffers) |
1240 | * remains unpopulated otherwise HW thinks |
1241 | * FLQ is empty. |
1242 | */ |
1243 | csio_wr_update_fl(hw, flq, n: (flq->credits - 8) - avail); |
1244 | csio_wr_ring_fldb(hw, flq); |
1245 | } |
1246 | } |
1247 | |
1248 | restart: |
1249 | /* Now inform SGE about our incremental index value */ |
1250 | csio_wr_reg32(hw, CIDXINC_V(q->inc_idx) | |
1251 | INGRESSQID_V(q->un.iq.physiqid) | |
1252 | TIMERREG_V(csio_sge_timer_reg), |
1253 | MYPF_REG(SGE_PF_GTS_A)); |
1254 | q->stats.n_tot_rsps += q->inc_idx; |
1255 | |
1256 | q->inc_idx = 0; |
1257 | |
1258 | return rv; |
1259 | } |
1260 | |
1261 | int |
1262 | csio_wr_process_iq_idx(struct csio_hw *hw, int qidx, |
1263 | void (*iq_handler)(struct csio_hw *, void *, |
1264 | uint32_t, struct csio_fl_dma_buf *, |
1265 | void *), |
1266 | void *priv) |
1267 | { |
1268 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
1269 | struct csio_q *iq = wrm->q_arr[qidx]; |
1270 | |
1271 | return csio_wr_process_iq(hw, q: iq, iq_handler, priv); |
1272 | } |
1273 | |
1274 | static int |
1275 | csio_closest_timer(struct csio_sge *s, int time) |
1276 | { |
1277 | int i, delta, match = 0, min_delta = INT_MAX; |
1278 | |
1279 | for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { |
1280 | delta = time - s->timer_val[i]; |
1281 | if (delta < 0) |
1282 | delta = -delta; |
1283 | if (delta < min_delta) { |
1284 | min_delta = delta; |
1285 | match = i; |
1286 | } |
1287 | } |
1288 | return match; |
1289 | } |
1290 | |
1291 | static int |
1292 | csio_closest_thresh(struct csio_sge *s, int cnt) |
1293 | { |
1294 | int i, delta, match = 0, min_delta = INT_MAX; |
1295 | |
1296 | for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { |
1297 | delta = cnt - s->counter_val[i]; |
1298 | if (delta < 0) |
1299 | delta = -delta; |
1300 | if (delta < min_delta) { |
1301 | min_delta = delta; |
1302 | match = i; |
1303 | } |
1304 | } |
1305 | return match; |
1306 | } |
1307 | |
1308 | static void |
1309 | csio_wr_fixup_host_params(struct csio_hw *hw) |
1310 | { |
1311 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
1312 | struct csio_sge *sge = &wrm->sge; |
1313 | uint32_t clsz = L1_CACHE_BYTES; |
1314 | uint32_t s_hps = PAGE_SHIFT - 10; |
1315 | uint32_t stat_len = clsz > 64 ? 128 : 64; |
1316 | u32 fl_align = clsz < 32 ? 32 : clsz; |
1317 | u32 pack_align; |
1318 | u32 ingpad, ingpack; |
1319 | |
1320 | csio_wr_reg32(hw, HOSTPAGESIZEPF0_V(s_hps) | HOSTPAGESIZEPF1_V(s_hps) | |
1321 | HOSTPAGESIZEPF2_V(s_hps) | HOSTPAGESIZEPF3_V(s_hps) | |
1322 | HOSTPAGESIZEPF4_V(s_hps) | HOSTPAGESIZEPF5_V(s_hps) | |
1323 | HOSTPAGESIZEPF6_V(s_hps) | HOSTPAGESIZEPF7_V(s_hps), |
1324 | SGE_HOST_PAGE_SIZE_A); |
1325 | |
1326 | /* T5 introduced the separation of the Free List Padding and |
1327 | * Packing Boundaries. Thus, we can select a smaller Padding |
1328 | * Boundary to avoid uselessly chewing up PCIe Link and Memory |
1329 | * Bandwidth, and use a Packing Boundary which is large enough |
1330 | * to avoid false sharing between CPUs, etc. |
1331 | * |
1332 | * For the PCI Link, the smaller the Padding Boundary the |
1333 | * better. For the Memory Controller, a smaller Padding |
1334 | * Boundary is better until we cross under the Memory Line |
1335 | * Size (the minimum unit of transfer to/from Memory). If we |
1336 | * have a Padding Boundary which is smaller than the Memory |
1337 | * Line Size, that'll involve a Read-Modify-Write cycle on the |
1338 | * Memory Controller which is never good. |
1339 | */ |
1340 | |
1341 | /* We want the Packing Boundary to be based on the Cache Line |
1342 | * Size in order to help avoid False Sharing performance |
1343 | * issues between CPUs, etc. We also want the Packing |
1344 | * Boundary to incorporate the PCI-E Maximum Payload Size. We |
1345 | * get best performance when the Packing Boundary is a |
1346 | * multiple of the Maximum Payload Size. |
1347 | */ |
1348 | pack_align = fl_align; |
1349 | if (pci_is_pcie(dev: hw->pdev)) { |
1350 | u32 mps, mps_log; |
1351 | u16 devctl; |
1352 | |
1353 | /* The PCIe Device Control Maximum Payload Size field |
1354 | * [bits 7:5] encodes sizes as powers of 2 starting at |
1355 | * 128 bytes. |
1356 | */ |
1357 | pcie_capability_read_word(dev: hw->pdev, PCI_EXP_DEVCTL, val: &devctl); |
1358 | mps_log = ((devctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5) + 7; |
1359 | mps = 1 << mps_log; |
1360 | if (mps > pack_align) |
1361 | pack_align = mps; |
1362 | } |
1363 | |
1364 | /* T5/T6 have a special interpretation of the "0" |
1365 | * value for the Packing Boundary. This corresponds to 16 |
1366 | * bytes instead of the expected 32 bytes. |
1367 | */ |
1368 | if (pack_align <= 16) { |
1369 | ingpack = INGPACKBOUNDARY_16B_X; |
1370 | fl_align = 16; |
1371 | } else if (pack_align == 32) { |
1372 | ingpack = INGPACKBOUNDARY_64B_X; |
1373 | fl_align = 64; |
1374 | } else { |
1375 | u32 pack_align_log = fls(x: pack_align) - 1; |
1376 | |
1377 | ingpack = pack_align_log - INGPACKBOUNDARY_SHIFT_X; |
1378 | fl_align = pack_align; |
1379 | } |
1380 | |
1381 | /* Use the smallest Ingress Padding which isn't smaller than |
1382 | * the Memory Controller Read/Write Size. We'll take that as |
1383 | * being 8 bytes since we don't know of any system with a |
1384 | * wider Memory Controller Bus Width. |
1385 | */ |
1386 | if (csio_is_t5(chip: hw->pdev->device & CSIO_HW_CHIP_MASK)) |
1387 | ingpad = INGPADBOUNDARY_32B_X; |
1388 | else |
1389 | ingpad = T6_INGPADBOUNDARY_8B_X; |
1390 | |
1391 | csio_set_reg_field(hw, SGE_CONTROL_A, |
1392 | INGPADBOUNDARY_V(INGPADBOUNDARY_M) | |
1393 | EGRSTATUSPAGESIZE_F, |
1394 | INGPADBOUNDARY_V(ingpad) | |
1395 | EGRSTATUSPAGESIZE_V(stat_len != 64)); |
1396 | csio_set_reg_field(hw, SGE_CONTROL2_A, |
1397 | INGPACKBOUNDARY_V(INGPACKBOUNDARY_M), |
1398 | INGPACKBOUNDARY_V(ingpack)); |
1399 | |
1400 | /* FL BUFFER SIZE#0 is Page size i,e already aligned to cache line */ |
1401 | csio_wr_reg32(hw, PAGE_SIZE, SGE_FL_BUFFER_SIZE0_A); |
1402 | |
1403 | /* |
1404 | * If using hard params, the following will get set correctly |
1405 | * in csio_wr_set_sge(). |
1406 | */ |
1407 | if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) { |
1408 | csio_wr_reg32(hw, |
1409 | (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE2_A) + |
1410 | fl_align - 1) & ~(fl_align - 1), |
1411 | SGE_FL_BUFFER_SIZE2_A); |
1412 | csio_wr_reg32(hw, |
1413 | (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE3_A) + |
1414 | fl_align - 1) & ~(fl_align - 1), |
1415 | SGE_FL_BUFFER_SIZE3_A); |
1416 | } |
1417 | |
1418 | sge->csio_fl_align = fl_align; |
1419 | |
1420 | csio_wr_reg32(hw, HPZ0_V(PAGE_SHIFT - 12), ULP_RX_TDDP_PSZ_A); |
1421 | |
1422 | /* default value of rx_dma_offset of the NIC driver */ |
1423 | csio_set_reg_field(hw, SGE_CONTROL_A, |
1424 | PKTSHIFT_V(PKTSHIFT_M), |
1425 | PKTSHIFT_V(CSIO_SGE_RX_DMA_OFFSET)); |
1426 | |
1427 | csio_hw_tp_wr_bits_indirect(hw, TP_INGRESS_CONFIG_A, |
1428 | CSUM_HAS_PSEUDO_HDR_F, 0); |
1429 | } |
1430 | |
1431 | static void |
1432 | csio_init_intr_coalesce_parms(struct csio_hw *hw) |
1433 | { |
1434 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
1435 | struct csio_sge *sge = &wrm->sge; |
1436 | |
1437 | csio_sge_thresh_reg = csio_closest_thresh(s: sge, cnt: csio_intr_coalesce_cnt); |
1438 | if (csio_intr_coalesce_cnt) { |
1439 | csio_sge_thresh_reg = 0; |
1440 | csio_sge_timer_reg = X_TIMERREG_RESTART_COUNTER; |
1441 | return; |
1442 | } |
1443 | |
1444 | csio_sge_timer_reg = csio_closest_timer(s: sge, time: csio_intr_coalesce_time); |
1445 | } |
1446 | |
1447 | /* |
1448 | * csio_wr_get_sge - Get SGE register values. |
1449 | * @hw: HW module. |
1450 | * |
1451 | * Used by non-master functions and by master-functions relying on config file. |
1452 | */ |
1453 | static void |
1454 | csio_wr_get_sge(struct csio_hw *hw) |
1455 | { |
1456 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
1457 | struct csio_sge *sge = &wrm->sge; |
1458 | uint32_t ingpad; |
1459 | int i; |
1460 | u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5; |
1461 | u32 ingress_rx_threshold; |
1462 | |
1463 | sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A); |
1464 | |
1465 | ingpad = INGPADBOUNDARY_G(sge->sge_control); |
1466 | |
1467 | switch (ingpad) { |
1468 | case X_INGPCIEBOUNDARY_32B: |
1469 | sge->csio_fl_align = 32; break; |
1470 | case X_INGPCIEBOUNDARY_64B: |
1471 | sge->csio_fl_align = 64; break; |
1472 | case X_INGPCIEBOUNDARY_128B: |
1473 | sge->csio_fl_align = 128; break; |
1474 | case X_INGPCIEBOUNDARY_256B: |
1475 | sge->csio_fl_align = 256; break; |
1476 | case X_INGPCIEBOUNDARY_512B: |
1477 | sge->csio_fl_align = 512; break; |
1478 | case X_INGPCIEBOUNDARY_1024B: |
1479 | sge->csio_fl_align = 1024; break; |
1480 | case X_INGPCIEBOUNDARY_2048B: |
1481 | sge->csio_fl_align = 2048; break; |
1482 | case X_INGPCIEBOUNDARY_4096B: |
1483 | sge->csio_fl_align = 4096; break; |
1484 | } |
1485 | |
1486 | for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++) |
1487 | csio_get_flbuf_size(hw, sge, reg: i); |
1488 | |
1489 | timer_value_0_and_1 = csio_rd_reg32(hw, SGE_TIMER_VALUE_0_AND_1_A); |
1490 | timer_value_2_and_3 = csio_rd_reg32(hw, SGE_TIMER_VALUE_2_AND_3_A); |
1491 | timer_value_4_and_5 = csio_rd_reg32(hw, SGE_TIMER_VALUE_4_AND_5_A); |
1492 | |
1493 | sge->timer_val[0] = (uint16_t)csio_core_ticks_to_us(hw, |
1494 | TIMERVALUE0_G(timer_value_0_and_1)); |
1495 | sge->timer_val[1] = (uint16_t)csio_core_ticks_to_us(hw, |
1496 | TIMERVALUE1_G(timer_value_0_and_1)); |
1497 | sge->timer_val[2] = (uint16_t)csio_core_ticks_to_us(hw, |
1498 | TIMERVALUE2_G(timer_value_2_and_3)); |
1499 | sge->timer_val[3] = (uint16_t)csio_core_ticks_to_us(hw, |
1500 | TIMERVALUE3_G(timer_value_2_and_3)); |
1501 | sge->timer_val[4] = (uint16_t)csio_core_ticks_to_us(hw, |
1502 | TIMERVALUE4_G(timer_value_4_and_5)); |
1503 | sge->timer_val[5] = (uint16_t)csio_core_ticks_to_us(hw, |
1504 | TIMERVALUE5_G(timer_value_4_and_5)); |
1505 | |
1506 | ingress_rx_threshold = csio_rd_reg32(hw, SGE_INGRESS_RX_THRESHOLD_A); |
1507 | sge->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold); |
1508 | sge->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold); |
1509 | sge->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold); |
1510 | sge->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold); |
1511 | |
1512 | csio_init_intr_coalesce_parms(hw); |
1513 | } |
1514 | |
1515 | /* |
1516 | * csio_wr_set_sge - Initialize SGE registers |
1517 | * @hw: HW module. |
1518 | * |
1519 | * Used by Master function to initialize SGE registers in the absence |
1520 | * of a config file. |
1521 | */ |
1522 | static void |
1523 | csio_wr_set_sge(struct csio_hw *hw) |
1524 | { |
1525 | struct csio_wrm *wrm = csio_hw_to_wrm(hw); |
1526 | struct csio_sge *sge = &wrm->sge; |
1527 | int i; |
1528 | |
1529 | /* |
1530 | * Set up our basic SGE mode to deliver CPL messages to our Ingress |
1531 | * Queue and Packet Date to the Free List. |
1532 | */ |
1533 | csio_set_reg_field(hw, SGE_CONTROL_A, RXPKTCPLMODE_F, RXPKTCPLMODE_F); |
1534 | |
1535 | sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A); |
1536 | |
1537 | /* sge->csio_fl_align is set up by csio_wr_fixup_host_params(). */ |
1538 | |
1539 | /* |
1540 | * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows |
1541 | * and generate an interrupt when this occurs so we can recover. |
1542 | */ |
1543 | csio_set_reg_field(hw, SGE_DBFIFO_STATUS_A, |
1544 | LP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M), |
1545 | LP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH)); |
1546 | csio_set_reg_field(hw, SGE_DBFIFO_STATUS2_A, |
1547 | HP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M), |
1548 | HP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH)); |
1549 | |
1550 | csio_set_reg_field(hw, SGE_DOORBELL_CONTROL_A, ENABLE_DROP_F, |
1551 | ENABLE_DROP_F); |
1552 | |
1553 | /* SGE_FL_BUFFER_SIZE0 is set up by csio_wr_fixup_host_params(). */ |
1554 | |
1555 | CSIO_SET_FLBUF_SIZE(hw, 1, CSIO_SGE_FLBUF_SIZE1); |
1556 | csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE2 + sge->csio_fl_align - 1) |
1557 | & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE2_A); |
1558 | csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE3 + sge->csio_fl_align - 1) |
1559 | & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE3_A); |
1560 | CSIO_SET_FLBUF_SIZE(hw, 4, CSIO_SGE_FLBUF_SIZE4); |
1561 | CSIO_SET_FLBUF_SIZE(hw, 5, CSIO_SGE_FLBUF_SIZE5); |
1562 | CSIO_SET_FLBUF_SIZE(hw, 6, CSIO_SGE_FLBUF_SIZE6); |
1563 | CSIO_SET_FLBUF_SIZE(hw, 7, CSIO_SGE_FLBUF_SIZE7); |
1564 | CSIO_SET_FLBUF_SIZE(hw, 8, CSIO_SGE_FLBUF_SIZE8); |
1565 | |
1566 | for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++) |
1567 | csio_get_flbuf_size(hw, sge, reg: i); |
1568 | |
1569 | /* Initialize interrupt coalescing attributes */ |
1570 | sge->timer_val[0] = CSIO_SGE_TIMER_VAL_0; |
1571 | sge->timer_val[1] = CSIO_SGE_TIMER_VAL_1; |
1572 | sge->timer_val[2] = CSIO_SGE_TIMER_VAL_2; |
1573 | sge->timer_val[3] = CSIO_SGE_TIMER_VAL_3; |
1574 | sge->timer_val[4] = CSIO_SGE_TIMER_VAL_4; |
1575 | sge->timer_val[5] = CSIO_SGE_TIMER_VAL_5; |
1576 | |
1577 | sge->counter_val[0] = CSIO_SGE_INT_CNT_VAL_0; |
1578 | sge->counter_val[1] = CSIO_SGE_INT_CNT_VAL_1; |
1579 | sge->counter_val[2] = CSIO_SGE_INT_CNT_VAL_2; |
1580 | sge->counter_val[3] = CSIO_SGE_INT_CNT_VAL_3; |
1581 | |
1582 | csio_wr_reg32(hw, THRESHOLD_0_V(sge->counter_val[0]) | |
1583 | THRESHOLD_1_V(sge->counter_val[1]) | |
1584 | THRESHOLD_2_V(sge->counter_val[2]) | |
1585 | THRESHOLD_3_V(sge->counter_val[3]), |
1586 | SGE_INGRESS_RX_THRESHOLD_A); |
1587 | |
1588 | csio_wr_reg32(hw, |
1589 | TIMERVALUE0_V(csio_us_to_core_ticks(hw, sge->timer_val[0])) | |
1590 | TIMERVALUE1_V(csio_us_to_core_ticks(hw, sge->timer_val[1])), |
1591 | SGE_TIMER_VALUE_0_AND_1_A); |
1592 | |
1593 | csio_wr_reg32(hw, |
1594 | TIMERVALUE2_V(csio_us_to_core_ticks(hw, sge->timer_val[2])) | |
1595 | TIMERVALUE3_V(csio_us_to_core_ticks(hw, sge->timer_val[3])), |
1596 | SGE_TIMER_VALUE_2_AND_3_A); |
1597 | |
1598 | csio_wr_reg32(hw, |
1599 | TIMERVALUE4_V(csio_us_to_core_ticks(hw, sge->timer_val[4])) | |
1600 | TIMERVALUE5_V(csio_us_to_core_ticks(hw, sge->timer_val[5])), |
1601 | SGE_TIMER_VALUE_4_AND_5_A); |
1602 | |
1603 | csio_init_intr_coalesce_parms(hw); |
1604 | } |
1605 | |
1606 | void |
1607 | csio_wr_sge_init(struct csio_hw *hw) |
1608 | { |
1609 | /* |
1610 | * If we are master and chip is not initialized: |
1611 | * - If we plan to use the config file, we need to fixup some |
1612 | * host specific registers, and read the rest of the SGE |
1613 | * configuration. |
1614 | * - If we dont plan to use the config file, we need to initialize |
1615 | * SGE entirely, including fixing the host specific registers. |
1616 | * If we are master and chip is initialized, just read and work off of |
1617 | * the already initialized SGE values. |
1618 | * If we arent the master, we are only allowed to read and work off of |
1619 | * the already initialized SGE values. |
1620 | * |
1621 | * Therefore, before calling this function, we assume that the master- |
1622 | * ship of the card, state and whether to use config file or not, have |
1623 | * already been decided. |
1624 | */ |
1625 | if (csio_is_hw_master(hw)) { |
1626 | if (hw->fw_state != CSIO_DEV_STATE_INIT) |
1627 | csio_wr_fixup_host_params(hw); |
1628 | |
1629 | if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) |
1630 | csio_wr_get_sge(hw); |
1631 | else |
1632 | csio_wr_set_sge(hw); |
1633 | } else |
1634 | csio_wr_get_sge(hw); |
1635 | } |
1636 | |
1637 | /* |
1638 | * csio_wrm_init - Initialize Work request module. |
1639 | * @wrm: WR module |
1640 | * @hw: HW pointer |
1641 | * |
1642 | * Allocates memory for an array of queue pointers starting at q_arr. |
1643 | */ |
1644 | int |
1645 | csio_wrm_init(struct csio_wrm *wrm, struct csio_hw *hw) |
1646 | { |
1647 | int i; |
1648 | |
1649 | if (!wrm->num_q) { |
1650 | csio_err(hw, "Num queues is not set\n" ); |
1651 | return -EINVAL; |
1652 | } |
1653 | |
1654 | wrm->q_arr = kcalloc(n: wrm->num_q, size: sizeof(struct csio_q *), GFP_KERNEL); |
1655 | if (!wrm->q_arr) |
1656 | goto err; |
1657 | |
1658 | for (i = 0; i < wrm->num_q; i++) { |
1659 | wrm->q_arr[i] = kzalloc(size: sizeof(struct csio_q), GFP_KERNEL); |
1660 | if (!wrm->q_arr[i]) { |
1661 | while (--i >= 0) |
1662 | kfree(objp: wrm->q_arr[i]); |
1663 | goto err_free_arr; |
1664 | } |
1665 | } |
1666 | wrm->free_qidx = 0; |
1667 | |
1668 | return 0; |
1669 | |
1670 | err_free_arr: |
1671 | kfree(objp: wrm->q_arr); |
1672 | err: |
1673 | return -ENOMEM; |
1674 | } |
1675 | |
1676 | /* |
1677 | * csio_wrm_exit - Initialize Work request module. |
1678 | * @wrm: WR module |
1679 | * @hw: HW module |
1680 | * |
1681 | * Uninitialize WR module. Free q_arr and pointers in it. |
1682 | * We have the additional job of freeing the DMA memory associated |
1683 | * with the queues. |
1684 | */ |
1685 | void |
1686 | csio_wrm_exit(struct csio_wrm *wrm, struct csio_hw *hw) |
1687 | { |
1688 | int i; |
1689 | uint32_t j; |
1690 | struct csio_q *q; |
1691 | struct csio_dma_buf *buf; |
1692 | |
1693 | for (i = 0; i < wrm->num_q; i++) { |
1694 | q = wrm->q_arr[i]; |
1695 | |
1696 | if (wrm->free_qidx && (i < wrm->free_qidx)) { |
1697 | if (q->type == CSIO_FREELIST) { |
1698 | if (!q->un.fl.bufs) |
1699 | continue; |
1700 | for (j = 0; j < q->credits; j++) { |
1701 | buf = &q->un.fl.bufs[j]; |
1702 | if (!buf->vaddr) |
1703 | continue; |
1704 | dma_free_coherent(dev: &hw->pdev->dev, |
1705 | size: buf->len, cpu_addr: buf->vaddr, |
1706 | dma_handle: buf->paddr); |
1707 | } |
1708 | kfree(objp: q->un.fl.bufs); |
1709 | } |
1710 | dma_free_coherent(dev: &hw->pdev->dev, size: q->size, |
1711 | cpu_addr: q->vstart, dma_handle: q->pstart); |
1712 | } |
1713 | kfree(objp: q); |
1714 | } |
1715 | |
1716 | hw->flags &= ~CSIO_HWF_Q_MEM_ALLOCED; |
1717 | |
1718 | kfree(objp: wrm->q_arr); |
1719 | } |
1720 | |