1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Huawei HiNIC PCI Express Linux driver |
4 | * Copyright(c) 2017 Huawei Technologies Co., Ltd |
5 | */ |
6 | |
7 | #include <linux/kernel.h> |
8 | #include <linux/types.h> |
9 | #include <linux/pci.h> |
10 | #include <linux/device.h> |
11 | #include <linux/dma-mapping.h> |
12 | #include <linux/vmalloc.h> |
13 | #include <linux/errno.h> |
14 | #include <linux/sizes.h> |
15 | #include <linux/atomic.h> |
16 | #include <linux/skbuff.h> |
17 | #include <linux/io.h> |
18 | #include <asm/barrier.h> |
19 | #include <asm/byteorder.h> |
20 | |
21 | #include "hinic_common.h" |
22 | #include "hinic_hw_if.h" |
23 | #include "hinic_hw_wqe.h" |
24 | #include "hinic_hw_wq.h" |
25 | #include "hinic_hw_qp_ctxt.h" |
26 | #include "hinic_hw_qp.h" |
27 | #include "hinic_hw_io.h" |
28 | |
29 | #define SQ_DB_OFF SZ_2K |
30 | |
31 | /* The number of cache line to prefetch Until threshold state */ |
32 | #define WQ_PREFETCH_MAX 2 |
33 | /* The number of cache line to prefetch After threshold state */ |
34 | #define WQ_PREFETCH_MIN 1 |
35 | /* Threshold state */ |
36 | #define WQ_PREFETCH_THRESHOLD 256 |
37 | |
38 | /* sizes of the SQ/RQ ctxt */ |
39 | #define Q_CTXT_SIZE 48 |
40 | #define CTXT_RSVD 240 |
41 | |
42 | #define SQ_CTXT_OFFSET(max_sqs, max_rqs, q_id) \ |
43 | (((max_rqs) + (max_sqs)) * CTXT_RSVD + (q_id) * Q_CTXT_SIZE) |
44 | |
45 | #define RQ_CTXT_OFFSET(max_sqs, max_rqs, q_id) \ |
46 | (((max_rqs) + (max_sqs)) * CTXT_RSVD + \ |
47 | (max_sqs + (q_id)) * Q_CTXT_SIZE) |
48 | |
49 | #define SIZE_16BYTES(size) (ALIGN(size, 16) >> 4) |
50 | #define SIZE_8BYTES(size) (ALIGN(size, 8) >> 3) |
51 | #define SECT_SIZE_FROM_8BYTES(size) ((size) << 3) |
52 | |
53 | #define SQ_DB_PI_HI_SHIFT 8 |
54 | #define SQ_DB_PI_HI(prod_idx) ((prod_idx) >> SQ_DB_PI_HI_SHIFT) |
55 | |
56 | #define SQ_DB_PI_LOW_MASK 0xFF |
57 | #define SQ_DB_PI_LOW(prod_idx) ((prod_idx) & SQ_DB_PI_LOW_MASK) |
58 | |
59 | #define SQ_DB_ADDR(sq, pi) ((u64 *)((sq)->db_base) + SQ_DB_PI_LOW(pi)) |
60 | |
61 | #define SQ_MASKED_IDX(sq, idx) ((idx) & (sq)->wq->mask) |
62 | #define RQ_MASKED_IDX(rq, idx) ((idx) & (rq)->wq->mask) |
63 | |
64 | enum sq_wqe_type { |
65 | SQ_NORMAL_WQE = 0, |
66 | }; |
67 | |
68 | enum rq_completion_fmt { |
69 | RQ_COMPLETE_SGE = 1 |
70 | }; |
71 | |
72 | void (struct hinic_qp_ctxt_header *qp_ctxt_hdr, |
73 | enum hinic_qp_ctxt_type ctxt_type, |
74 | u16 num_queues, u16 max_queues) |
75 | { |
76 | u16 max_sqs = max_queues; |
77 | u16 max_rqs = max_queues; |
78 | |
79 | qp_ctxt_hdr->num_queues = num_queues; |
80 | qp_ctxt_hdr->queue_type = ctxt_type; |
81 | |
82 | if (ctxt_type == HINIC_QP_CTXT_TYPE_SQ) |
83 | qp_ctxt_hdr->addr_offset = SQ_CTXT_OFFSET(max_sqs, max_rqs, 0); |
84 | else |
85 | qp_ctxt_hdr->addr_offset = RQ_CTXT_OFFSET(max_sqs, max_rqs, 0); |
86 | |
87 | qp_ctxt_hdr->addr_offset = SIZE_16BYTES(qp_ctxt_hdr->addr_offset); |
88 | |
89 | hinic_cpu_to_be32(data: qp_ctxt_hdr, len: sizeof(*qp_ctxt_hdr)); |
90 | } |
91 | |
92 | void hinic_sq_prepare_ctxt(struct hinic_sq_ctxt *sq_ctxt, |
93 | struct hinic_sq *sq, u16 global_qid) |
94 | { |
95 | u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo; |
96 | u64 wq_page_addr, wq_page_pfn, wq_block_pfn; |
97 | u16 pi_start, ci_start; |
98 | struct hinic_wq *wq; |
99 | |
100 | wq = sq->wq; |
101 | ci_start = atomic_read(v: &wq->cons_idx); |
102 | pi_start = atomic_read(v: &wq->prod_idx); |
103 | |
104 | /* Read the first page paddr from the WQ page paddr ptrs */ |
105 | wq_page_addr = be64_to_cpu(*wq->block_vaddr); |
106 | |
107 | wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr); |
108 | wq_page_pfn_hi = upper_32_bits(wq_page_pfn); |
109 | wq_page_pfn_lo = lower_32_bits(wq_page_pfn); |
110 | |
111 | /* If only one page, use 0-level CLA */ |
112 | if (wq->num_q_pages == 1) |
113 | wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq_page_addr); |
114 | else |
115 | wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr); |
116 | |
117 | wq_block_pfn_hi = upper_32_bits(wq_block_pfn); |
118 | wq_block_pfn_lo = lower_32_bits(wq_block_pfn); |
119 | |
120 | sq_ctxt->ceq_attr = HINIC_SQ_CTXT_CEQ_ATTR_SET(global_qid, |
121 | GLOBAL_SQ_ID) | |
122 | HINIC_SQ_CTXT_CEQ_ATTR_SET(0, EN); |
123 | |
124 | sq_ctxt->ci_wrapped = HINIC_SQ_CTXT_CI_SET(ci_start, IDX) | |
125 | HINIC_SQ_CTXT_CI_SET(1, WRAPPED); |
126 | |
127 | sq_ctxt->wq_hi_pfn_pi = |
128 | HINIC_SQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) | |
129 | HINIC_SQ_CTXT_WQ_PAGE_SET(pi_start, PI); |
130 | |
131 | sq_ctxt->wq_lo_pfn = wq_page_pfn_lo; |
132 | |
133 | sq_ctxt->pref_cache = |
134 | HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) | |
135 | HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) | |
136 | HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD); |
137 | |
138 | sq_ctxt->pref_wrapped = 1; |
139 | |
140 | sq_ctxt->pref_wq_hi_pfn_ci = |
141 | HINIC_SQ_CTXT_PREF_SET(ci_start, CI) | |
142 | HINIC_SQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN); |
143 | |
144 | sq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo; |
145 | |
146 | sq_ctxt->wq_block_hi_pfn = |
147 | HINIC_SQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN); |
148 | |
149 | sq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo; |
150 | |
151 | hinic_cpu_to_be32(data: sq_ctxt, len: sizeof(*sq_ctxt)); |
152 | } |
153 | |
154 | void hinic_rq_prepare_ctxt(struct hinic_rq_ctxt *rq_ctxt, |
155 | struct hinic_rq *rq, u16 global_qid) |
156 | { |
157 | u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo; |
158 | u64 wq_page_addr, wq_page_pfn, wq_block_pfn; |
159 | u16 pi_start, ci_start; |
160 | struct hinic_wq *wq; |
161 | |
162 | wq = rq->wq; |
163 | ci_start = atomic_read(v: &wq->cons_idx); |
164 | pi_start = atomic_read(v: &wq->prod_idx); |
165 | |
166 | /* Read the first page paddr from the WQ page paddr ptrs */ |
167 | wq_page_addr = be64_to_cpu(*wq->block_vaddr); |
168 | |
169 | wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr); |
170 | wq_page_pfn_hi = upper_32_bits(wq_page_pfn); |
171 | wq_page_pfn_lo = lower_32_bits(wq_page_pfn); |
172 | |
173 | wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr); |
174 | wq_block_pfn_hi = upper_32_bits(wq_block_pfn); |
175 | wq_block_pfn_lo = lower_32_bits(wq_block_pfn); |
176 | |
177 | rq_ctxt->ceq_attr = HINIC_RQ_CTXT_CEQ_ATTR_SET(0, EN) | |
178 | HINIC_RQ_CTXT_CEQ_ATTR_SET(1, WRAPPED); |
179 | |
180 | rq_ctxt->pi_intr_attr = HINIC_RQ_CTXT_PI_SET(pi_start, IDX) | |
181 | HINIC_RQ_CTXT_PI_SET(rq->msix_entry, INTR); |
182 | |
183 | rq_ctxt->wq_hi_pfn_ci = HINIC_RQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, |
184 | HI_PFN) | |
185 | HINIC_RQ_CTXT_WQ_PAGE_SET(ci_start, CI); |
186 | |
187 | rq_ctxt->wq_lo_pfn = wq_page_pfn_lo; |
188 | |
189 | rq_ctxt->pref_cache = |
190 | HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) | |
191 | HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) | |
192 | HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD); |
193 | |
194 | rq_ctxt->pref_wrapped = 1; |
195 | |
196 | rq_ctxt->pref_wq_hi_pfn_ci = |
197 | HINIC_RQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN) | |
198 | HINIC_RQ_CTXT_PREF_SET(ci_start, CI); |
199 | |
200 | rq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo; |
201 | |
202 | rq_ctxt->pi_paddr_hi = upper_32_bits(rq->pi_dma_addr); |
203 | rq_ctxt->pi_paddr_lo = lower_32_bits(rq->pi_dma_addr); |
204 | |
205 | rq_ctxt->wq_block_hi_pfn = |
206 | HINIC_RQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN); |
207 | |
208 | rq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo; |
209 | |
210 | hinic_cpu_to_be32(data: rq_ctxt, len: sizeof(*rq_ctxt)); |
211 | } |
212 | |
213 | /** |
214 | * alloc_sq_skb_arr - allocate sq array for saved skb |
215 | * @sq: HW Send Queue |
216 | * |
217 | * Return 0 - Success, negative - Failure |
218 | **/ |
219 | static int alloc_sq_skb_arr(struct hinic_sq *sq) |
220 | { |
221 | struct hinic_wq *wq = sq->wq; |
222 | size_t skb_arr_size; |
223 | |
224 | skb_arr_size = wq->q_depth * sizeof(*sq->saved_skb); |
225 | sq->saved_skb = vzalloc(size: skb_arr_size); |
226 | if (!sq->saved_skb) |
227 | return -ENOMEM; |
228 | |
229 | return 0; |
230 | } |
231 | |
232 | /** |
233 | * free_sq_skb_arr - free sq array for saved skb |
234 | * @sq: HW Send Queue |
235 | **/ |
236 | static void free_sq_skb_arr(struct hinic_sq *sq) |
237 | { |
238 | vfree(addr: sq->saved_skb); |
239 | } |
240 | |
241 | /** |
242 | * alloc_rq_skb_arr - allocate rq array for saved skb |
243 | * @rq: HW Receive Queue |
244 | * |
245 | * Return 0 - Success, negative - Failure |
246 | **/ |
247 | static int alloc_rq_skb_arr(struct hinic_rq *rq) |
248 | { |
249 | struct hinic_wq *wq = rq->wq; |
250 | size_t skb_arr_size; |
251 | |
252 | skb_arr_size = wq->q_depth * sizeof(*rq->saved_skb); |
253 | rq->saved_skb = vzalloc(size: skb_arr_size); |
254 | if (!rq->saved_skb) |
255 | return -ENOMEM; |
256 | |
257 | return 0; |
258 | } |
259 | |
260 | /** |
261 | * free_rq_skb_arr - free rq array for saved skb |
262 | * @rq: HW Receive Queue |
263 | **/ |
264 | static void free_rq_skb_arr(struct hinic_rq *rq) |
265 | { |
266 | vfree(addr: rq->saved_skb); |
267 | } |
268 | |
269 | /** |
270 | * hinic_init_sq - Initialize HW Send Queue |
271 | * @sq: HW Send Queue |
272 | * @hwif: HW Interface for accessing HW |
273 | * @wq: Work Queue for the data of the SQ |
274 | * @entry: msix entry for sq |
275 | * @ci_addr: address for reading the current HW consumer index |
276 | * @ci_dma_addr: dma address for reading the current HW consumer index |
277 | * @db_base: doorbell base address |
278 | * |
279 | * Return 0 - Success, negative - Failure |
280 | **/ |
281 | int hinic_init_sq(struct hinic_sq *sq, struct hinic_hwif *hwif, |
282 | struct hinic_wq *wq, struct msix_entry *entry, |
283 | void *ci_addr, dma_addr_t ci_dma_addr, |
284 | void __iomem *db_base) |
285 | { |
286 | sq->hwif = hwif; |
287 | |
288 | sq->wq = wq; |
289 | |
290 | sq->irq = entry->vector; |
291 | sq->msix_entry = entry->entry; |
292 | |
293 | sq->hw_ci_addr = ci_addr; |
294 | sq->hw_ci_dma_addr = ci_dma_addr; |
295 | |
296 | sq->db_base = db_base + SQ_DB_OFF; |
297 | |
298 | return alloc_sq_skb_arr(sq); |
299 | } |
300 | |
301 | /** |
302 | * hinic_clean_sq - Clean HW Send Queue's Resources |
303 | * @sq: Send Queue |
304 | **/ |
305 | void hinic_clean_sq(struct hinic_sq *sq) |
306 | { |
307 | free_sq_skb_arr(sq); |
308 | } |
309 | |
310 | /** |
311 | * alloc_rq_cqe - allocate rq completion queue elements |
312 | * @rq: HW Receive Queue |
313 | * |
314 | * Return 0 - Success, negative - Failure |
315 | **/ |
316 | static int alloc_rq_cqe(struct hinic_rq *rq) |
317 | { |
318 | struct hinic_hwif *hwif = rq->hwif; |
319 | struct pci_dev *pdev = hwif->pdev; |
320 | size_t cqe_dma_size, cqe_size; |
321 | struct hinic_wq *wq = rq->wq; |
322 | int j, i; |
323 | |
324 | cqe_size = wq->q_depth * sizeof(*rq->cqe); |
325 | rq->cqe = vzalloc(size: cqe_size); |
326 | if (!rq->cqe) |
327 | return -ENOMEM; |
328 | |
329 | cqe_dma_size = wq->q_depth * sizeof(*rq->cqe_dma); |
330 | rq->cqe_dma = vzalloc(size: cqe_dma_size); |
331 | if (!rq->cqe_dma) |
332 | goto err_cqe_dma_arr_alloc; |
333 | |
334 | for (i = 0; i < wq->q_depth; i++) { |
335 | rq->cqe[i] = dma_alloc_coherent(dev: &pdev->dev, |
336 | size: sizeof(*rq->cqe[i]), |
337 | dma_handle: &rq->cqe_dma[i], GFP_KERNEL); |
338 | if (!rq->cqe[i]) |
339 | goto err_cqe_alloc; |
340 | } |
341 | |
342 | return 0; |
343 | |
344 | err_cqe_alloc: |
345 | for (j = 0; j < i; j++) |
346 | dma_free_coherent(dev: &pdev->dev, size: sizeof(*rq->cqe[j]), cpu_addr: rq->cqe[j], |
347 | dma_handle: rq->cqe_dma[j]); |
348 | |
349 | vfree(addr: rq->cqe_dma); |
350 | |
351 | err_cqe_dma_arr_alloc: |
352 | vfree(addr: rq->cqe); |
353 | return -ENOMEM; |
354 | } |
355 | |
356 | /** |
357 | * free_rq_cqe - free rq completion queue elements |
358 | * @rq: HW Receive Queue |
359 | **/ |
360 | static void free_rq_cqe(struct hinic_rq *rq) |
361 | { |
362 | struct hinic_hwif *hwif = rq->hwif; |
363 | struct pci_dev *pdev = hwif->pdev; |
364 | struct hinic_wq *wq = rq->wq; |
365 | int i; |
366 | |
367 | for (i = 0; i < wq->q_depth; i++) |
368 | dma_free_coherent(dev: &pdev->dev, size: sizeof(*rq->cqe[i]), cpu_addr: rq->cqe[i], |
369 | dma_handle: rq->cqe_dma[i]); |
370 | |
371 | vfree(addr: rq->cqe_dma); |
372 | vfree(addr: rq->cqe); |
373 | } |
374 | |
375 | /** |
376 | * hinic_init_rq - Initialize HW Receive Queue |
377 | * @rq: HW Receive Queue |
378 | * @hwif: HW Interface for accessing HW |
379 | * @wq: Work Queue for the data of the RQ |
380 | * @entry: msix entry for rq |
381 | * |
382 | * Return 0 - Success, negative - Failure |
383 | **/ |
384 | int hinic_init_rq(struct hinic_rq *rq, struct hinic_hwif *hwif, |
385 | struct hinic_wq *wq, struct msix_entry *entry) |
386 | { |
387 | struct pci_dev *pdev = hwif->pdev; |
388 | size_t pi_size; |
389 | int err; |
390 | |
391 | rq->hwif = hwif; |
392 | |
393 | rq->wq = wq; |
394 | |
395 | rq->irq = entry->vector; |
396 | rq->msix_entry = entry->entry; |
397 | |
398 | rq->buf_sz = HINIC_RX_BUF_SZ; |
399 | |
400 | err = alloc_rq_skb_arr(rq); |
401 | if (err) { |
402 | dev_err(&pdev->dev, "Failed to allocate rq priv data\n" ); |
403 | return err; |
404 | } |
405 | |
406 | err = alloc_rq_cqe(rq); |
407 | if (err) { |
408 | dev_err(&pdev->dev, "Failed to allocate rq cqe\n" ); |
409 | goto err_alloc_rq_cqe; |
410 | } |
411 | |
412 | /* HW requirements: Must be at least 32 bit */ |
413 | pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32)); |
414 | rq->pi_virt_addr = dma_alloc_coherent(dev: &pdev->dev, size: pi_size, |
415 | dma_handle: &rq->pi_dma_addr, GFP_KERNEL); |
416 | if (!rq->pi_virt_addr) { |
417 | err = -ENOMEM; |
418 | goto err_pi_virt; |
419 | } |
420 | |
421 | return 0; |
422 | |
423 | err_pi_virt: |
424 | free_rq_cqe(rq); |
425 | |
426 | err_alloc_rq_cqe: |
427 | free_rq_skb_arr(rq); |
428 | return err; |
429 | } |
430 | |
431 | /** |
432 | * hinic_clean_rq - Clean HW Receive Queue's Resources |
433 | * @rq: HW Receive Queue |
434 | **/ |
435 | void hinic_clean_rq(struct hinic_rq *rq) |
436 | { |
437 | struct hinic_hwif *hwif = rq->hwif; |
438 | struct pci_dev *pdev = hwif->pdev; |
439 | size_t pi_size; |
440 | |
441 | pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32)); |
442 | dma_free_coherent(dev: &pdev->dev, size: pi_size, cpu_addr: rq->pi_virt_addr, |
443 | dma_handle: rq->pi_dma_addr); |
444 | |
445 | free_rq_cqe(rq); |
446 | free_rq_skb_arr(rq); |
447 | } |
448 | |
449 | /** |
450 | * hinic_get_sq_free_wqebbs - return number of free wqebbs for use |
451 | * @sq: send queue |
452 | * |
453 | * Return number of free wqebbs |
454 | **/ |
455 | int hinic_get_sq_free_wqebbs(struct hinic_sq *sq) |
456 | { |
457 | struct hinic_wq *wq = sq->wq; |
458 | |
459 | return atomic_read(v: &wq->delta) - 1; |
460 | } |
461 | |
462 | /** |
463 | * hinic_get_rq_free_wqebbs - return number of free wqebbs for use |
464 | * @rq: recv queue |
465 | * |
466 | * Return number of free wqebbs |
467 | **/ |
468 | int hinic_get_rq_free_wqebbs(struct hinic_rq *rq) |
469 | { |
470 | struct hinic_wq *wq = rq->wq; |
471 | |
472 | return atomic_read(v: &wq->delta) - 1; |
473 | } |
474 | |
475 | static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, int nr_descs) |
476 | { |
477 | u32 ctrl_size, task_size, bufdesc_size; |
478 | |
479 | ctrl_size = SIZE_8BYTES(sizeof(struct hinic_sq_ctrl)); |
480 | task_size = SIZE_8BYTES(sizeof(struct hinic_sq_task)); |
481 | bufdesc_size = nr_descs * sizeof(struct hinic_sq_bufdesc); |
482 | bufdesc_size = SIZE_8BYTES(bufdesc_size); |
483 | |
484 | ctrl->ctrl_info = HINIC_SQ_CTRL_SET(bufdesc_size, BUFDESC_SECT_LEN) | |
485 | HINIC_SQ_CTRL_SET(task_size, TASKSECT_LEN) | |
486 | HINIC_SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) | |
487 | HINIC_SQ_CTRL_SET(ctrl_size, LEN); |
488 | |
489 | ctrl->queue_info = HINIC_SQ_CTRL_SET(HINIC_MSS_DEFAULT, |
490 | QUEUE_INFO_MSS) | |
491 | HINIC_SQ_CTRL_SET(1, QUEUE_INFO_UC); |
492 | } |
493 | |
494 | static void sq_prepare_task(struct hinic_sq_task *task) |
495 | { |
496 | task->pkt_info0 = 0; |
497 | task->pkt_info1 = 0; |
498 | task->pkt_info2 = 0; |
499 | |
500 | task->ufo_v6_identify = 0; |
501 | |
502 | task->pkt_info4 = HINIC_SQ_TASK_INFO4_SET(HINIC_L2TYPE_ETH, L2TYPE); |
503 | |
504 | task->zero_pad = 0; |
505 | } |
506 | |
507 | void hinic_task_set_l2hdr(struct hinic_sq_task *task, u32 len) |
508 | { |
509 | task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(len, L2HDR_LEN); |
510 | } |
511 | |
512 | void hinic_task_set_outter_l3(struct hinic_sq_task *task, |
513 | enum hinic_l3_offload_type l3_type, |
514 | u32 network_len) |
515 | { |
516 | task->pkt_info2 |= HINIC_SQ_TASK_INFO2_SET(l3_type, OUTER_L3TYPE) | |
517 | HINIC_SQ_TASK_INFO2_SET(network_len, OUTER_L3LEN); |
518 | } |
519 | |
520 | void hinic_task_set_inner_l3(struct hinic_sq_task *task, |
521 | enum hinic_l3_offload_type l3_type, |
522 | u32 network_len) |
523 | { |
524 | task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l3_type, INNER_L3TYPE); |
525 | task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(network_len, INNER_L3LEN); |
526 | } |
527 | |
528 | void hinic_task_set_tunnel_l4(struct hinic_sq_task *task, |
529 | enum hinic_l4_tunnel_type l4_type, |
530 | u32 tunnel_len) |
531 | { |
532 | task->pkt_info2 |= HINIC_SQ_TASK_INFO2_SET(l4_type, TUNNEL_L4TYPE) | |
533 | HINIC_SQ_TASK_INFO2_SET(tunnel_len, TUNNEL_L4LEN); |
534 | } |
535 | |
536 | void hinic_set_cs_inner_l4(struct hinic_sq_task *task, u32 *queue_info, |
537 | enum hinic_l4_offload_type l4_offload, |
538 | u32 l4_len, u32 offset) |
539 | { |
540 | u32 tcp_udp_cs = 0, sctp = 0; |
541 | u32 mss = HINIC_MSS_DEFAULT; |
542 | |
543 | if (l4_offload == TCP_OFFLOAD_ENABLE || |
544 | l4_offload == UDP_OFFLOAD_ENABLE) |
545 | tcp_udp_cs = 1; |
546 | else if (l4_offload == SCTP_OFFLOAD_ENABLE) |
547 | sctp = 1; |
548 | |
549 | task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l4_offload, L4_OFFLOAD); |
550 | task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(l4_len, INNER_L4LEN); |
551 | |
552 | *queue_info |= HINIC_SQ_CTRL_SET(offset, QUEUE_INFO_PLDOFF) | |
553 | HINIC_SQ_CTRL_SET(tcp_udp_cs, QUEUE_INFO_TCPUDP_CS) | |
554 | HINIC_SQ_CTRL_SET(sctp, QUEUE_INFO_SCTP); |
555 | |
556 | *queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS); |
557 | *queue_info |= HINIC_SQ_CTRL_SET(mss, QUEUE_INFO_MSS); |
558 | } |
559 | |
560 | void hinic_set_tso_inner_l4(struct hinic_sq_task *task, u32 *queue_info, |
561 | enum hinic_l4_offload_type l4_offload, |
562 | u32 l4_len, u32 offset, u32 ip_ident, u32 mss) |
563 | { |
564 | u32 tso = 0, ufo = 0; |
565 | |
566 | if (l4_offload == TCP_OFFLOAD_ENABLE) |
567 | tso = 1; |
568 | else if (l4_offload == UDP_OFFLOAD_ENABLE) |
569 | ufo = 1; |
570 | |
571 | task->ufo_v6_identify = ip_ident; |
572 | |
573 | task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l4_offload, L4_OFFLOAD); |
574 | task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(tso || ufo, TSO_FLAG); |
575 | task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(l4_len, INNER_L4LEN); |
576 | |
577 | *queue_info |= HINIC_SQ_CTRL_SET(offset, QUEUE_INFO_PLDOFF) | |
578 | HINIC_SQ_CTRL_SET(tso, QUEUE_INFO_TSO) | |
579 | HINIC_SQ_CTRL_SET(ufo, QUEUE_INFO_UFO) | |
580 | HINIC_SQ_CTRL_SET(!!l4_offload, QUEUE_INFO_TCPUDP_CS); |
581 | |
582 | /* set MSS value */ |
583 | *queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS); |
584 | *queue_info |= HINIC_SQ_CTRL_SET(mss, QUEUE_INFO_MSS); |
585 | } |
586 | |
587 | /** |
588 | * hinic_sq_prepare_wqe - prepare wqe before insert to the queue |
589 | * @sq: send queue |
590 | * @sq_wqe: wqe to prepare |
591 | * @sges: sges for use by the wqe for send for buf addresses |
592 | * @nr_sges: number of sges |
593 | **/ |
594 | void hinic_sq_prepare_wqe(struct hinic_sq *sq, struct hinic_sq_wqe *sq_wqe, |
595 | struct hinic_sge *sges, int nr_sges) |
596 | { |
597 | int i; |
598 | |
599 | sq_prepare_ctrl(ctrl: &sq_wqe->ctrl, nr_descs: nr_sges); |
600 | |
601 | sq_prepare_task(task: &sq_wqe->task); |
602 | |
603 | for (i = 0; i < nr_sges; i++) |
604 | sq_wqe->buf_descs[i].sge = sges[i]; |
605 | } |
606 | |
607 | /** |
608 | * sq_prepare_db - prepare doorbell to write |
609 | * @sq: send queue |
610 | * @prod_idx: pi value for the doorbell |
611 | * @cos: cos of the doorbell |
612 | * |
613 | * Return db value |
614 | **/ |
615 | static u32 sq_prepare_db(struct hinic_sq *sq, u16 prod_idx, unsigned int cos) |
616 | { |
617 | struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq); |
618 | u8 hi_prod_idx = SQ_DB_PI_HI(SQ_MASKED_IDX(sq, prod_idx)); |
619 | |
620 | /* Data should be written to HW in Big Endian Format */ |
621 | return cpu_to_be32(HINIC_SQ_DB_INFO_SET(hi_prod_idx, PI_HI) | |
622 | HINIC_SQ_DB_INFO_SET(HINIC_DB_SQ_TYPE, TYPE) | |
623 | HINIC_SQ_DB_INFO_SET(HINIC_DATA_PATH, PATH) | |
624 | HINIC_SQ_DB_INFO_SET(cos, COS) | |
625 | HINIC_SQ_DB_INFO_SET(qp->q_id, QID)); |
626 | } |
627 | |
628 | /** |
629 | * hinic_sq_write_db- write doorbell |
630 | * @sq: send queue |
631 | * @prod_idx: pi value for the doorbell |
632 | * @wqe_size: wqe size |
633 | * @cos: cos of the wqe |
634 | **/ |
635 | void hinic_sq_write_db(struct hinic_sq *sq, u16 prod_idx, unsigned int wqe_size, |
636 | unsigned int cos) |
637 | { |
638 | struct hinic_wq *wq = sq->wq; |
639 | |
640 | /* increment prod_idx to the next */ |
641 | prod_idx += ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; |
642 | prod_idx = SQ_MASKED_IDX(sq, prod_idx); |
643 | |
644 | wmb(); /* Write all before the doorbell */ |
645 | |
646 | writel(val: sq_prepare_db(sq, prod_idx, cos), SQ_DB_ADDR(sq, prod_idx)); |
647 | } |
648 | |
649 | /** |
650 | * hinic_sq_get_wqe - get wqe ptr in the current pi and update the pi |
651 | * @sq: sq to get wqe from |
652 | * @wqe_size: wqe size |
653 | * @prod_idx: returned pi |
654 | * |
655 | * Return wqe pointer |
656 | **/ |
657 | struct hinic_sq_wqe *hinic_sq_get_wqe(struct hinic_sq *sq, |
658 | unsigned int wqe_size, u16 *prod_idx) |
659 | { |
660 | struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(wq: sq->wq, wqe_size, |
661 | prod_idx); |
662 | |
663 | if (IS_ERR(ptr: hw_wqe)) |
664 | return NULL; |
665 | |
666 | return &hw_wqe->sq_wqe; |
667 | } |
668 | |
669 | /** |
670 | * hinic_sq_return_wqe - return the wqe to the sq |
671 | * @sq: send queue |
672 | * @wqe_size: the size of the wqe |
673 | **/ |
674 | void hinic_sq_return_wqe(struct hinic_sq *sq, unsigned int wqe_size) |
675 | { |
676 | hinic_return_wqe(wq: sq->wq, wqe_size); |
677 | } |
678 | |
679 | /** |
680 | * hinic_sq_write_wqe - write the wqe to the sq |
681 | * @sq: send queue |
682 | * @prod_idx: pi of the wqe |
683 | * @sq_wqe: the wqe to write |
684 | * @skb: skb to save |
685 | * @wqe_size: the size of the wqe |
686 | **/ |
687 | void hinic_sq_write_wqe(struct hinic_sq *sq, u16 prod_idx, |
688 | struct hinic_sq_wqe *sq_wqe, |
689 | struct sk_buff *skb, unsigned int wqe_size) |
690 | { |
691 | struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)sq_wqe; |
692 | |
693 | sq->saved_skb[prod_idx] = skb; |
694 | |
695 | /* The data in the HW should be in Big Endian Format */ |
696 | hinic_cpu_to_be32(data: sq_wqe, len: wqe_size); |
697 | |
698 | hinic_write_wqe(wq: sq->wq, wqe: hw_wqe, wqe_size); |
699 | } |
700 | |
701 | /** |
702 | * hinic_sq_read_wqebb - read wqe ptr in the current ci and update the ci, the |
703 | * wqe only have one wqebb |
704 | * @sq: send queue |
705 | * @skb: return skb that was saved |
706 | * @wqe_size: the wqe size ptr |
707 | * @cons_idx: consumer index of the wqe |
708 | * |
709 | * Return wqe in ci position |
710 | **/ |
711 | struct hinic_sq_wqe *hinic_sq_read_wqebb(struct hinic_sq *sq, |
712 | struct sk_buff **skb, |
713 | unsigned int *wqe_size, u16 *cons_idx) |
714 | { |
715 | struct hinic_hw_wqe *hw_wqe; |
716 | struct hinic_sq_wqe *sq_wqe; |
717 | struct hinic_sq_ctrl *ctrl; |
718 | unsigned int buf_sect_len; |
719 | u32 ctrl_info; |
720 | |
721 | /* read the ctrl section for getting wqe size */ |
722 | hw_wqe = hinic_read_wqe(wq: sq->wq, wqe_size: sizeof(*ctrl), cons_idx); |
723 | if (IS_ERR(ptr: hw_wqe)) |
724 | return NULL; |
725 | |
726 | *skb = sq->saved_skb[*cons_idx]; |
727 | |
728 | sq_wqe = &hw_wqe->sq_wqe; |
729 | ctrl = &sq_wqe->ctrl; |
730 | ctrl_info = be32_to_cpu(ctrl->ctrl_info); |
731 | buf_sect_len = HINIC_SQ_CTRL_GET(ctrl_info, BUFDESC_SECT_LEN); |
732 | |
733 | *wqe_size = sizeof(*ctrl) + sizeof(sq_wqe->task); |
734 | *wqe_size += SECT_SIZE_FROM_8BYTES(buf_sect_len); |
735 | *wqe_size = ALIGN(*wqe_size, sq->wq->wqebb_size); |
736 | |
737 | return &hw_wqe->sq_wqe; |
738 | } |
739 | |
740 | /** |
741 | * hinic_sq_read_wqe - read wqe ptr in the current ci and update the ci |
742 | * @sq: send queue |
743 | * @skb: return skb that was saved |
744 | * @wqe_size: the size of the wqe |
745 | * @cons_idx: consumer index of the wqe |
746 | * |
747 | * Return wqe in ci position |
748 | **/ |
749 | struct hinic_sq_wqe *hinic_sq_read_wqe(struct hinic_sq *sq, |
750 | struct sk_buff **skb, |
751 | unsigned int wqe_size, u16 *cons_idx) |
752 | { |
753 | struct hinic_hw_wqe *hw_wqe; |
754 | |
755 | hw_wqe = hinic_read_wqe(wq: sq->wq, wqe_size, cons_idx); |
756 | *skb = sq->saved_skb[*cons_idx]; |
757 | |
758 | return &hw_wqe->sq_wqe; |
759 | } |
760 | |
761 | /** |
762 | * hinic_sq_put_wqe - release the ci for new wqes |
763 | * @sq: send queue |
764 | * @wqe_size: the size of the wqe |
765 | **/ |
766 | void hinic_sq_put_wqe(struct hinic_sq *sq, unsigned int wqe_size) |
767 | { |
768 | hinic_put_wqe(wq: sq->wq, wqe_size); |
769 | } |
770 | |
771 | /** |
772 | * hinic_sq_get_sges - get sges from the wqe |
773 | * @sq_wqe: wqe to get the sges from its buffer addresses |
774 | * @sges: returned sges |
775 | * @nr_sges: number sges to return |
776 | **/ |
777 | void hinic_sq_get_sges(struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges, |
778 | int nr_sges) |
779 | { |
780 | int i; |
781 | |
782 | for (i = 0; i < nr_sges && i < HINIC_MAX_SQ_BUFDESCS; i++) { |
783 | sges[i] = sq_wqe->buf_descs[i].sge; |
784 | hinic_be32_to_cpu(data: &sges[i], len: sizeof(sges[i])); |
785 | } |
786 | } |
787 | |
788 | /** |
789 | * hinic_rq_get_wqe - get wqe ptr in the current pi and update the pi |
790 | * @rq: rq to get wqe from |
791 | * @wqe_size: wqe size |
792 | * @prod_idx: returned pi |
793 | * |
794 | * Return wqe pointer |
795 | **/ |
796 | struct hinic_rq_wqe *hinic_rq_get_wqe(struct hinic_rq *rq, |
797 | unsigned int wqe_size, u16 *prod_idx) |
798 | { |
799 | struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(wq: rq->wq, wqe_size, |
800 | prod_idx); |
801 | |
802 | if (IS_ERR(ptr: hw_wqe)) |
803 | return NULL; |
804 | |
805 | return &hw_wqe->rq_wqe; |
806 | } |
807 | |
808 | /** |
809 | * hinic_rq_write_wqe - write the wqe to the rq |
810 | * @rq: recv queue |
811 | * @prod_idx: pi of the wqe |
812 | * @rq_wqe: the wqe to write |
813 | * @skb: skb to save |
814 | **/ |
815 | void hinic_rq_write_wqe(struct hinic_rq *rq, u16 prod_idx, |
816 | struct hinic_rq_wqe *rq_wqe, struct sk_buff *skb) |
817 | { |
818 | struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)rq_wqe; |
819 | |
820 | rq->saved_skb[prod_idx] = skb; |
821 | |
822 | /* The data in the HW should be in Big Endian Format */ |
823 | hinic_cpu_to_be32(data: rq_wqe, len: sizeof(*rq_wqe)); |
824 | |
825 | hinic_write_wqe(wq: rq->wq, wqe: hw_wqe, wqe_size: sizeof(*rq_wqe)); |
826 | } |
827 | |
828 | /** |
829 | * hinic_rq_read_wqe - read wqe ptr in the current ci and update the ci |
830 | * @rq: recv queue |
831 | * @wqe_size: the size of the wqe |
832 | * @skb: return saved skb |
833 | * @cons_idx: consumer index of the wqe |
834 | * |
835 | * Return wqe in ci position |
836 | **/ |
837 | struct hinic_rq_wqe *hinic_rq_read_wqe(struct hinic_rq *rq, |
838 | unsigned int wqe_size, |
839 | struct sk_buff **skb, u16 *cons_idx) |
840 | { |
841 | struct hinic_hw_wqe *hw_wqe; |
842 | struct hinic_rq_cqe *cqe; |
843 | int rx_done; |
844 | u32 status; |
845 | |
846 | hw_wqe = hinic_read_wqe(wq: rq->wq, wqe_size, cons_idx); |
847 | if (IS_ERR(ptr: hw_wqe)) |
848 | return NULL; |
849 | |
850 | cqe = rq->cqe[*cons_idx]; |
851 | |
852 | status = be32_to_cpu(cqe->status); |
853 | |
854 | rx_done = HINIC_RQ_CQE_STATUS_GET(status, RXDONE); |
855 | if (!rx_done) |
856 | return NULL; |
857 | |
858 | *skb = rq->saved_skb[*cons_idx]; |
859 | |
860 | return &hw_wqe->rq_wqe; |
861 | } |
862 | |
863 | /** |
864 | * hinic_rq_read_next_wqe - increment ci and read the wqe in ci position |
865 | * @rq: recv queue |
866 | * @wqe_size: the size of the wqe |
867 | * @skb: return saved skb |
868 | * @cons_idx: consumer index in the wq |
869 | * |
870 | * Return wqe in incremented ci position |
871 | **/ |
872 | struct hinic_rq_wqe *hinic_rq_read_next_wqe(struct hinic_rq *rq, |
873 | unsigned int wqe_size, |
874 | struct sk_buff **skb, |
875 | u16 *cons_idx) |
876 | { |
877 | struct hinic_wq *wq = rq->wq; |
878 | struct hinic_hw_wqe *hw_wqe; |
879 | unsigned int num_wqebbs; |
880 | |
881 | wqe_size = ALIGN(wqe_size, wq->wqebb_size); |
882 | num_wqebbs = wqe_size / wq->wqebb_size; |
883 | |
884 | *cons_idx = RQ_MASKED_IDX(rq, *cons_idx + num_wqebbs); |
885 | |
886 | *skb = rq->saved_skb[*cons_idx]; |
887 | |
888 | hw_wqe = hinic_read_wqe_direct(wq, cons_idx: *cons_idx); |
889 | |
890 | return &hw_wqe->rq_wqe; |
891 | } |
892 | |
893 | /** |
894 | * hinic_rq_put_wqe - release the ci for new wqes |
895 | * @rq: recv queue |
896 | * @cons_idx: consumer index of the wqe |
897 | * @wqe_size: the size of the wqe |
898 | **/ |
899 | void hinic_rq_put_wqe(struct hinic_rq *rq, u16 cons_idx, |
900 | unsigned int wqe_size) |
901 | { |
902 | struct hinic_rq_cqe *cqe = rq->cqe[cons_idx]; |
903 | u32 status = be32_to_cpu(cqe->status); |
904 | |
905 | status = HINIC_RQ_CQE_STATUS_CLEAR(status, RXDONE); |
906 | |
907 | /* Rx WQE size is 1 WQEBB, no wq shadow*/ |
908 | cqe->status = cpu_to_be32(status); |
909 | |
910 | wmb(); /* clear done flag */ |
911 | |
912 | hinic_put_wqe(wq: rq->wq, wqe_size); |
913 | } |
914 | |
915 | /** |
916 | * hinic_rq_get_sge - get sge from the wqe |
917 | * @rq: recv queue |
918 | * @rq_wqe: wqe to get the sge from its buf address |
919 | * @cons_idx: consumer index |
920 | * @sge: returned sge |
921 | **/ |
922 | void hinic_rq_get_sge(struct hinic_rq *rq, struct hinic_rq_wqe *rq_wqe, |
923 | u16 cons_idx, struct hinic_sge *sge) |
924 | { |
925 | struct hinic_rq_cqe *cqe = rq->cqe[cons_idx]; |
926 | u32 len = be32_to_cpu(cqe->len); |
927 | |
928 | sge->hi_addr = be32_to_cpu(rq_wqe->buf_desc.hi_addr); |
929 | sge->lo_addr = be32_to_cpu(rq_wqe->buf_desc.lo_addr); |
930 | sge->len = HINIC_RQ_CQE_SGE_GET(len, LEN); |
931 | } |
932 | |
933 | /** |
934 | * hinic_rq_prepare_wqe - prepare wqe before insert to the queue |
935 | * @rq: recv queue |
936 | * @prod_idx: pi value |
937 | * @rq_wqe: the wqe |
938 | * @sge: sge for use by the wqe for recv buf address |
939 | **/ |
940 | void hinic_rq_prepare_wqe(struct hinic_rq *rq, u16 prod_idx, |
941 | struct hinic_rq_wqe *rq_wqe, struct hinic_sge *sge) |
942 | { |
943 | struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect; |
944 | struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc; |
945 | struct hinic_rq_cqe *cqe = rq->cqe[prod_idx]; |
946 | struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl; |
947 | dma_addr_t cqe_dma = rq->cqe_dma[prod_idx]; |
948 | |
949 | ctrl->ctrl_info = |
950 | HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)), LEN) | |
951 | HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)), |
952 | COMPLETE_LEN) | |
953 | HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)), |
954 | BUFDESC_SECT_LEN) | |
955 | HINIC_RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT); |
956 | |
957 | hinic_set_sge(sge: &cqe_sect->sge, addr: cqe_dma, len: sizeof(*cqe)); |
958 | |
959 | buf_desc->hi_addr = sge->hi_addr; |
960 | buf_desc->lo_addr = sge->lo_addr; |
961 | } |
962 | |
963 | /** |
964 | * hinic_rq_update - update pi of the rq |
965 | * @rq: recv queue |
966 | * @prod_idx: pi value |
967 | **/ |
968 | void hinic_rq_update(struct hinic_rq *rq, u16 prod_idx) |
969 | { |
970 | *rq->pi_virt_addr = cpu_to_be16(RQ_MASKED_IDX(rq, prod_idx + 1)); |
971 | } |
972 | |