1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Huawei HiNIC PCI Express Linux driver |
4 | * Copyright(c) 2017 Huawei Technologies Co., Ltd |
5 | */ |
6 | |
7 | #include <linux/kernel.h> |
8 | #include <linux/types.h> |
9 | #include <linux/pci.h> |
10 | #include <linux/device.h> |
11 | #include <linux/dma-mapping.h> |
12 | #include <linux/slab.h> |
13 | #include <linux/atomic.h> |
14 | #include <linux/semaphore.h> |
15 | #include <linux/errno.h> |
16 | #include <linux/vmalloc.h> |
17 | #include <linux/err.h> |
18 | #include <asm/byteorder.h> |
19 | |
20 | #include "hinic_hw_if.h" |
21 | #include "hinic_hw_wqe.h" |
22 | #include "hinic_hw_wq.h" |
23 | #include "hinic_hw_cmdq.h" |
24 | |
25 | #define WQS_BLOCKS_PER_PAGE 4 |
26 | |
27 | #define WQ_BLOCK_SIZE 4096 |
28 | #define WQS_PAGE_SIZE (WQS_BLOCKS_PER_PAGE * WQ_BLOCK_SIZE) |
29 | |
30 | #define WQS_MAX_NUM_BLOCKS 128 |
31 | #define WQS_FREE_BLOCKS_SIZE(wqs) (WQS_MAX_NUM_BLOCKS * \ |
32 | sizeof((wqs)->free_blocks[0])) |
33 | |
34 | #define WQ_SIZE(wq) ((wq)->q_depth * (wq)->wqebb_size) |
35 | |
36 | #define WQ_PAGE_ADDR_SIZE sizeof(u64) |
37 | #define WQ_MAX_PAGES (WQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE) |
38 | |
39 | #define CMDQ_BLOCK_SIZE 512 |
40 | #define CMDQ_PAGE_SIZE 4096 |
41 | |
42 | #define CMDQ_WQ_MAX_PAGES (CMDQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE) |
43 | |
44 | #define WQ_BASE_VADDR(wqs, wq) \ |
45 | ((void *)((wqs)->page_vaddr[(wq)->page_idx]) \ |
46 | + (wq)->block_idx * WQ_BLOCK_SIZE) |
47 | |
48 | #define WQ_BASE_PADDR(wqs, wq) \ |
49 | ((wqs)->page_paddr[(wq)->page_idx] \ |
50 | + (wq)->block_idx * WQ_BLOCK_SIZE) |
51 | |
52 | #define WQ_BASE_ADDR(wqs, wq) \ |
53 | ((void *)((wqs)->shadow_page_vaddr[(wq)->page_idx]) \ |
54 | + (wq)->block_idx * WQ_BLOCK_SIZE) |
55 | |
56 | #define CMDQ_BASE_VADDR(cmdq_pages, wq) \ |
57 | ((void *)((cmdq_pages)->page_vaddr) \ |
58 | + (wq)->block_idx * CMDQ_BLOCK_SIZE) |
59 | |
60 | #define CMDQ_BASE_PADDR(cmdq_pages, wq) \ |
61 | ((cmdq_pages)->page_paddr \ |
62 | + (wq)->block_idx * CMDQ_BLOCK_SIZE) |
63 | |
64 | #define CMDQ_BASE_ADDR(cmdq_pages, wq) \ |
65 | ((void *)((cmdq_pages)->shadow_page_vaddr) \ |
66 | + (wq)->block_idx * CMDQ_BLOCK_SIZE) |
67 | |
68 | #define WQ_PAGE_ADDR(wq, idx) \ |
69 | ((wq)->shadow_block_vaddr[WQE_PAGE_NUM(wq, idx)]) |
70 | |
71 | #define MASKED_WQE_IDX(wq, idx) ((idx) & (wq)->mask) |
72 | |
73 | #define WQE_IN_RANGE(wqe, start, end) \ |
74 | (((unsigned long)(wqe) >= (unsigned long)(start)) && \ |
75 | ((unsigned long)(wqe) < (unsigned long)(end))) |
76 | |
77 | #define WQE_SHADOW_PAGE(wq, wqe) \ |
78 | (((unsigned long)(wqe) - (unsigned long)(wq)->shadow_wqe) \ |
79 | / (wq)->max_wqe_size) |
80 | |
81 | static inline int WQE_PAGE_OFF(struct hinic_wq *wq, u16 idx) |
82 | { |
83 | return (((idx) & ((wq)->num_wqebbs_per_page - 1)) |
84 | << (wq)->wqebb_size_shift); |
85 | } |
86 | |
87 | static inline int WQE_PAGE_NUM(struct hinic_wq *wq, u16 idx) |
88 | { |
89 | return (((idx) >> ((wq)->wqebbs_per_page_shift)) |
90 | & ((wq)->num_q_pages - 1)); |
91 | } |
92 | |
93 | /** |
94 | * queue_alloc_page - allocate page for Queue |
95 | * @hwif: HW interface for allocating DMA |
96 | * @vaddr: virtual address will be returned in this address |
97 | * @paddr: physical address will be returned in this address |
98 | * @shadow_vaddr: VM area will be return here for holding WQ page addresses |
99 | * @page_sz: page size of each WQ page |
100 | * |
101 | * Return 0 - Success, negative - Failure |
102 | **/ |
103 | static int queue_alloc_page(struct hinic_hwif *hwif, u64 **vaddr, u64 *paddr, |
104 | void ***shadow_vaddr, size_t page_sz) |
105 | { |
106 | struct pci_dev *pdev = hwif->pdev; |
107 | dma_addr_t dma_addr; |
108 | |
109 | *vaddr = dma_alloc_coherent(dev: &pdev->dev, size: page_sz, dma_handle: &dma_addr, |
110 | GFP_KERNEL); |
111 | if (!*vaddr) { |
112 | dev_err(&pdev->dev, "Failed to allocate dma for wqs page\n" ); |
113 | return -ENOMEM; |
114 | } |
115 | |
116 | *paddr = (u64)dma_addr; |
117 | |
118 | /* use vzalloc for big mem */ |
119 | *shadow_vaddr = vzalloc(size: page_sz); |
120 | if (!*shadow_vaddr) |
121 | goto err_shadow_vaddr; |
122 | |
123 | return 0; |
124 | |
125 | err_shadow_vaddr: |
126 | dma_free_coherent(dev: &pdev->dev, size: page_sz, cpu_addr: *vaddr, dma_handle: dma_addr); |
127 | return -ENOMEM; |
128 | } |
129 | |
130 | /** |
131 | * wqs_allocate_page - allocate page for WQ set |
132 | * @wqs: Work Queue Set |
133 | * @page_idx: the page index of the page will be allocated |
134 | * |
135 | * Return 0 - Success, negative - Failure |
136 | **/ |
137 | static int wqs_allocate_page(struct hinic_wqs *wqs, int page_idx) |
138 | { |
139 | return queue_alloc_page(hwif: wqs->hwif, vaddr: &wqs->page_vaddr[page_idx], |
140 | paddr: &wqs->page_paddr[page_idx], |
141 | shadow_vaddr: &wqs->shadow_page_vaddr[page_idx], |
142 | WQS_PAGE_SIZE); |
143 | } |
144 | |
145 | /** |
146 | * wqs_free_page - free page of WQ set |
147 | * @wqs: Work Queue Set |
148 | * @page_idx: the page index of the page will be freed |
149 | **/ |
150 | static void wqs_free_page(struct hinic_wqs *wqs, int page_idx) |
151 | { |
152 | struct hinic_hwif *hwif = wqs->hwif; |
153 | struct pci_dev *pdev = hwif->pdev; |
154 | |
155 | dma_free_coherent(dev: &pdev->dev, WQS_PAGE_SIZE, |
156 | cpu_addr: wqs->page_vaddr[page_idx], |
157 | dma_handle: (dma_addr_t)wqs->page_paddr[page_idx]); |
158 | vfree(addr: wqs->shadow_page_vaddr[page_idx]); |
159 | } |
160 | |
161 | /** |
162 | * cmdq_allocate_page - allocate page for cmdq |
163 | * @cmdq_pages: the pages of the cmdq queue struct to hold the page |
164 | * |
165 | * Return 0 - Success, negative - Failure |
166 | **/ |
167 | static int cmdq_allocate_page(struct hinic_cmdq_pages *cmdq_pages) |
168 | { |
169 | return queue_alloc_page(hwif: cmdq_pages->hwif, vaddr: &cmdq_pages->page_vaddr, |
170 | paddr: &cmdq_pages->page_paddr, |
171 | shadow_vaddr: &cmdq_pages->shadow_page_vaddr, |
172 | CMDQ_PAGE_SIZE); |
173 | } |
174 | |
175 | /** |
176 | * cmdq_free_page - free page from cmdq |
177 | * @cmdq_pages: the pages of the cmdq queue struct that hold the page |
178 | **/ |
179 | static void cmdq_free_page(struct hinic_cmdq_pages *cmdq_pages) |
180 | { |
181 | struct hinic_hwif *hwif = cmdq_pages->hwif; |
182 | struct pci_dev *pdev = hwif->pdev; |
183 | |
184 | dma_free_coherent(dev: &pdev->dev, CMDQ_PAGE_SIZE, |
185 | cpu_addr: cmdq_pages->page_vaddr, |
186 | dma_handle: (dma_addr_t)cmdq_pages->page_paddr); |
187 | vfree(addr: cmdq_pages->shadow_page_vaddr); |
188 | } |
189 | |
190 | static int alloc_page_arrays(struct hinic_wqs *wqs) |
191 | { |
192 | struct hinic_hwif *hwif = wqs->hwif; |
193 | struct pci_dev *pdev = hwif->pdev; |
194 | |
195 | wqs->page_paddr = devm_kcalloc(dev: &pdev->dev, n: wqs->num_pages, |
196 | size: sizeof(*wqs->page_paddr), GFP_KERNEL); |
197 | if (!wqs->page_paddr) |
198 | return -ENOMEM; |
199 | |
200 | wqs->page_vaddr = devm_kcalloc(dev: &pdev->dev, n: wqs->num_pages, |
201 | size: sizeof(*wqs->page_vaddr), GFP_KERNEL); |
202 | if (!wqs->page_vaddr) |
203 | goto err_page_vaddr; |
204 | |
205 | wqs->shadow_page_vaddr = devm_kcalloc(dev: &pdev->dev, n: wqs->num_pages, |
206 | size: sizeof(*wqs->shadow_page_vaddr), |
207 | GFP_KERNEL); |
208 | if (!wqs->shadow_page_vaddr) |
209 | goto err_page_shadow_vaddr; |
210 | |
211 | return 0; |
212 | |
213 | err_page_shadow_vaddr: |
214 | devm_kfree(dev: &pdev->dev, p: wqs->page_vaddr); |
215 | |
216 | err_page_vaddr: |
217 | devm_kfree(dev: &pdev->dev, p: wqs->page_paddr); |
218 | return -ENOMEM; |
219 | } |
220 | |
221 | static void free_page_arrays(struct hinic_wqs *wqs) |
222 | { |
223 | struct hinic_hwif *hwif = wqs->hwif; |
224 | struct pci_dev *pdev = hwif->pdev; |
225 | |
226 | devm_kfree(dev: &pdev->dev, p: wqs->shadow_page_vaddr); |
227 | devm_kfree(dev: &pdev->dev, p: wqs->page_vaddr); |
228 | devm_kfree(dev: &pdev->dev, p: wqs->page_paddr); |
229 | } |
230 | |
231 | static int wqs_next_block(struct hinic_wqs *wqs, int *page_idx, |
232 | int *block_idx) |
233 | { |
234 | int pos; |
235 | |
236 | down(sem: &wqs->alloc_blocks_lock); |
237 | |
238 | wqs->num_free_blks--; |
239 | |
240 | if (wqs->num_free_blks < 0) { |
241 | wqs->num_free_blks++; |
242 | up(sem: &wqs->alloc_blocks_lock); |
243 | return -ENOMEM; |
244 | } |
245 | |
246 | pos = wqs->alloc_blk_pos++; |
247 | pos &= WQS_MAX_NUM_BLOCKS - 1; |
248 | |
249 | *page_idx = wqs->free_blocks[pos].page_idx; |
250 | *block_idx = wqs->free_blocks[pos].block_idx; |
251 | |
252 | wqs->free_blocks[pos].page_idx = -1; |
253 | wqs->free_blocks[pos].block_idx = -1; |
254 | |
255 | up(sem: &wqs->alloc_blocks_lock); |
256 | return 0; |
257 | } |
258 | |
259 | static void wqs_return_block(struct hinic_wqs *wqs, int page_idx, |
260 | int block_idx) |
261 | { |
262 | int pos; |
263 | |
264 | down(sem: &wqs->alloc_blocks_lock); |
265 | |
266 | pos = wqs->return_blk_pos++; |
267 | pos &= WQS_MAX_NUM_BLOCKS - 1; |
268 | |
269 | wqs->free_blocks[pos].page_idx = page_idx; |
270 | wqs->free_blocks[pos].block_idx = block_idx; |
271 | |
272 | wqs->num_free_blks++; |
273 | |
274 | up(sem: &wqs->alloc_blocks_lock); |
275 | } |
276 | |
277 | static void init_wqs_blocks_arr(struct hinic_wqs *wqs) |
278 | { |
279 | int page_idx, blk_idx, pos = 0; |
280 | |
281 | for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) { |
282 | for (blk_idx = 0; blk_idx < WQS_BLOCKS_PER_PAGE; blk_idx++) { |
283 | wqs->free_blocks[pos].page_idx = page_idx; |
284 | wqs->free_blocks[pos].block_idx = blk_idx; |
285 | pos++; |
286 | } |
287 | } |
288 | |
289 | wqs->alloc_blk_pos = 0; |
290 | wqs->return_blk_pos = pos; |
291 | wqs->num_free_blks = pos; |
292 | |
293 | sema_init(sem: &wqs->alloc_blocks_lock, val: 1); |
294 | } |
295 | |
296 | /** |
297 | * hinic_wqs_alloc - allocate Work Queues set |
298 | * @wqs: Work Queue Set |
299 | * @max_wqs: maximum wqs to allocate |
300 | * @hwif: HW interface for use for the allocation |
301 | * |
302 | * Return 0 - Success, negative - Failure |
303 | **/ |
304 | int hinic_wqs_alloc(struct hinic_wqs *wqs, int max_wqs, |
305 | struct hinic_hwif *hwif) |
306 | { |
307 | struct pci_dev *pdev = hwif->pdev; |
308 | int err, i, page_idx; |
309 | |
310 | max_wqs = ALIGN(max_wqs, WQS_BLOCKS_PER_PAGE); |
311 | if (max_wqs > WQS_MAX_NUM_BLOCKS) { |
312 | dev_err(&pdev->dev, "Invalid max_wqs = %d\n" , max_wqs); |
313 | return -EINVAL; |
314 | } |
315 | |
316 | wqs->hwif = hwif; |
317 | wqs->num_pages = max_wqs / WQS_BLOCKS_PER_PAGE; |
318 | |
319 | if (alloc_page_arrays(wqs)) { |
320 | dev_err(&pdev->dev, |
321 | "Failed to allocate mem for page addresses\n" ); |
322 | return -ENOMEM; |
323 | } |
324 | |
325 | for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) { |
326 | err = wqs_allocate_page(wqs, page_idx); |
327 | if (err) { |
328 | dev_err(&pdev->dev, "Failed wq page allocation\n" ); |
329 | goto err_wq_allocate_page; |
330 | } |
331 | } |
332 | |
333 | wqs->free_blocks = devm_kzalloc(dev: &pdev->dev, WQS_FREE_BLOCKS_SIZE(wqs), |
334 | GFP_KERNEL); |
335 | if (!wqs->free_blocks) { |
336 | err = -ENOMEM; |
337 | goto err_alloc_blocks; |
338 | } |
339 | |
340 | init_wqs_blocks_arr(wqs); |
341 | return 0; |
342 | |
343 | err_alloc_blocks: |
344 | err_wq_allocate_page: |
345 | for (i = 0; i < page_idx; i++) |
346 | wqs_free_page(wqs, page_idx: i); |
347 | |
348 | free_page_arrays(wqs); |
349 | return err; |
350 | } |
351 | |
352 | /** |
353 | * hinic_wqs_free - free Work Queues set |
354 | * @wqs: Work Queue Set |
355 | **/ |
356 | void hinic_wqs_free(struct hinic_wqs *wqs) |
357 | { |
358 | struct hinic_hwif *hwif = wqs->hwif; |
359 | struct pci_dev *pdev = hwif->pdev; |
360 | int page_idx; |
361 | |
362 | devm_kfree(dev: &pdev->dev, p: wqs->free_blocks); |
363 | |
364 | for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) |
365 | wqs_free_page(wqs, page_idx); |
366 | |
367 | free_page_arrays(wqs); |
368 | } |
369 | |
370 | /** |
371 | * alloc_wqes_shadow - allocate WQE shadows for WQ |
372 | * @wq: WQ to allocate shadows for |
373 | * |
374 | * Return 0 - Success, negative - Failure |
375 | **/ |
376 | static int alloc_wqes_shadow(struct hinic_wq *wq) |
377 | { |
378 | struct hinic_hwif *hwif = wq->hwif; |
379 | struct pci_dev *pdev = hwif->pdev; |
380 | |
381 | wq->shadow_wqe = devm_kcalloc(dev: &pdev->dev, n: wq->num_q_pages, |
382 | size: wq->max_wqe_size, GFP_KERNEL); |
383 | if (!wq->shadow_wqe) |
384 | return -ENOMEM; |
385 | |
386 | wq->shadow_idx = devm_kcalloc(dev: &pdev->dev, n: wq->num_q_pages, |
387 | size: sizeof(*wq->shadow_idx), GFP_KERNEL); |
388 | if (!wq->shadow_idx) |
389 | goto err_shadow_idx; |
390 | |
391 | return 0; |
392 | |
393 | err_shadow_idx: |
394 | devm_kfree(dev: &pdev->dev, p: wq->shadow_wqe); |
395 | return -ENOMEM; |
396 | } |
397 | |
398 | /** |
399 | * free_wqes_shadow - free WQE shadows of WQ |
400 | * @wq: WQ to free shadows from |
401 | **/ |
402 | static void free_wqes_shadow(struct hinic_wq *wq) |
403 | { |
404 | struct hinic_hwif *hwif = wq->hwif; |
405 | struct pci_dev *pdev = hwif->pdev; |
406 | |
407 | devm_kfree(dev: &pdev->dev, p: wq->shadow_idx); |
408 | devm_kfree(dev: &pdev->dev, p: wq->shadow_wqe); |
409 | } |
410 | |
411 | /** |
412 | * free_wq_pages - free pages of WQ |
413 | * @hwif: HW interface for releasing dma addresses |
414 | * @wq: WQ to free pages from |
415 | * @num_q_pages: number pages to free |
416 | **/ |
417 | static void free_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif, |
418 | int num_q_pages) |
419 | { |
420 | struct pci_dev *pdev = hwif->pdev; |
421 | int i; |
422 | |
423 | for (i = 0; i < num_q_pages; i++) { |
424 | void **vaddr = &wq->shadow_block_vaddr[i]; |
425 | u64 *paddr = &wq->block_vaddr[i]; |
426 | dma_addr_t dma_addr; |
427 | |
428 | dma_addr = (dma_addr_t)be64_to_cpu(*paddr); |
429 | dma_free_coherent(dev: &pdev->dev, size: wq->wq_page_size, cpu_addr: *vaddr, |
430 | dma_handle: dma_addr); |
431 | } |
432 | |
433 | free_wqes_shadow(wq); |
434 | } |
435 | |
436 | /** |
437 | * alloc_wq_pages - alloc pages for WQ |
438 | * @hwif: HW interface for allocating dma addresses |
439 | * @wq: WQ to allocate pages for |
440 | * @max_pages: maximum pages allowed |
441 | * |
442 | * Return 0 - Success, negative - Failure |
443 | **/ |
444 | static int alloc_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif, |
445 | int max_pages) |
446 | { |
447 | struct pci_dev *pdev = hwif->pdev; |
448 | int i, err, num_q_pages; |
449 | |
450 | num_q_pages = ALIGN(WQ_SIZE(wq), wq->wq_page_size) / wq->wq_page_size; |
451 | if (num_q_pages > max_pages) { |
452 | dev_err(&pdev->dev, "Number wq pages exceeds the limit\n" ); |
453 | return -EINVAL; |
454 | } |
455 | |
456 | if (num_q_pages & (num_q_pages - 1)) { |
457 | dev_err(&pdev->dev, "Number wq pages must be power of 2\n" ); |
458 | return -EINVAL; |
459 | } |
460 | |
461 | wq->num_q_pages = num_q_pages; |
462 | |
463 | err = alloc_wqes_shadow(wq); |
464 | if (err) { |
465 | dev_err(&pdev->dev, "Failed to allocate wqe shadow\n" ); |
466 | return err; |
467 | } |
468 | |
469 | for (i = 0; i < num_q_pages; i++) { |
470 | void **vaddr = &wq->shadow_block_vaddr[i]; |
471 | u64 *paddr = &wq->block_vaddr[i]; |
472 | dma_addr_t dma_addr; |
473 | |
474 | *vaddr = dma_alloc_coherent(dev: &pdev->dev, size: wq->wq_page_size, |
475 | dma_handle: &dma_addr, GFP_KERNEL); |
476 | if (!*vaddr) { |
477 | dev_err(&pdev->dev, "Failed to allocate wq page\n" ); |
478 | goto err_alloc_wq_pages; |
479 | } |
480 | |
481 | /* HW uses Big Endian Format */ |
482 | *paddr = cpu_to_be64(dma_addr); |
483 | } |
484 | |
485 | return 0; |
486 | |
487 | err_alloc_wq_pages: |
488 | free_wq_pages(wq, hwif, num_q_pages: i); |
489 | return -ENOMEM; |
490 | } |
491 | |
492 | /** |
493 | * hinic_wq_allocate - Allocate the WQ resources from the WQS |
494 | * @wqs: WQ set from which to allocate the WQ resources |
495 | * @wq: WQ to allocate resources for it from the WQ set |
496 | * @wqebb_size: Work Queue Block Byte Size |
497 | * @wq_page_size: the page size in the Work Queue |
498 | * @q_depth: number of wqebbs in WQ |
499 | * @max_wqe_size: maximum WQE size that will be used in the WQ |
500 | * |
501 | * Return 0 - Success, negative - Failure |
502 | **/ |
503 | int hinic_wq_allocate(struct hinic_wqs *wqs, struct hinic_wq *wq, |
504 | u16 wqebb_size, u32 wq_page_size, u16 q_depth, |
505 | u16 max_wqe_size) |
506 | { |
507 | struct hinic_hwif *hwif = wqs->hwif; |
508 | struct pci_dev *pdev = hwif->pdev; |
509 | u16 num_wqebbs_per_page; |
510 | u16 wqebb_size_shift; |
511 | int err; |
512 | |
513 | if (!is_power_of_2(n: wqebb_size)) { |
514 | dev_err(&pdev->dev, "wqebb_size must be power of 2\n" ); |
515 | return -EINVAL; |
516 | } |
517 | |
518 | if (wq_page_size == 0) { |
519 | dev_err(&pdev->dev, "wq_page_size must be > 0\n" ); |
520 | return -EINVAL; |
521 | } |
522 | |
523 | if (q_depth & (q_depth - 1)) { |
524 | dev_err(&pdev->dev, "WQ q_depth must be power of 2\n" ); |
525 | return -EINVAL; |
526 | } |
527 | |
528 | wqebb_size_shift = ilog2(wqebb_size); |
529 | num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) |
530 | >> wqebb_size_shift; |
531 | |
532 | if (!is_power_of_2(n: num_wqebbs_per_page)) { |
533 | dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n" ); |
534 | return -EINVAL; |
535 | } |
536 | |
537 | wq->hwif = hwif; |
538 | |
539 | err = wqs_next_block(wqs, page_idx: &wq->page_idx, block_idx: &wq->block_idx); |
540 | if (err) { |
541 | dev_err(&pdev->dev, "Failed to get free wqs next block\n" ); |
542 | return err; |
543 | } |
544 | |
545 | wq->wqebb_size = wqebb_size; |
546 | wq->wq_page_size = wq_page_size; |
547 | wq->q_depth = q_depth; |
548 | wq->max_wqe_size = max_wqe_size; |
549 | wq->num_wqebbs_per_page = num_wqebbs_per_page; |
550 | wq->wqebbs_per_page_shift = ilog2(num_wqebbs_per_page); |
551 | wq->wqebb_size_shift = wqebb_size_shift; |
552 | wq->block_vaddr = WQ_BASE_VADDR(wqs, wq); |
553 | wq->shadow_block_vaddr = WQ_BASE_ADDR(wqs, wq); |
554 | wq->block_paddr = WQ_BASE_PADDR(wqs, wq); |
555 | |
556 | err = alloc_wq_pages(wq, hwif: wqs->hwif, WQ_MAX_PAGES); |
557 | if (err) { |
558 | dev_err(&pdev->dev, "Failed to allocate wq pages\n" ); |
559 | goto err_alloc_wq_pages; |
560 | } |
561 | |
562 | atomic_set(v: &wq->cons_idx, i: 0); |
563 | atomic_set(v: &wq->prod_idx, i: 0); |
564 | atomic_set(v: &wq->delta, i: q_depth); |
565 | wq->mask = q_depth - 1; |
566 | |
567 | return 0; |
568 | |
569 | err_alloc_wq_pages: |
570 | wqs_return_block(wqs, page_idx: wq->page_idx, block_idx: wq->block_idx); |
571 | return err; |
572 | } |
573 | |
574 | /** |
575 | * hinic_wq_free - Free the WQ resources to the WQS |
576 | * @wqs: WQ set to free the WQ resources to it |
577 | * @wq: WQ to free its resources to the WQ set resources |
578 | **/ |
579 | void hinic_wq_free(struct hinic_wqs *wqs, struct hinic_wq *wq) |
580 | { |
581 | free_wq_pages(wq, hwif: wqs->hwif, num_q_pages: wq->num_q_pages); |
582 | |
583 | wqs_return_block(wqs, page_idx: wq->page_idx, block_idx: wq->block_idx); |
584 | } |
585 | |
586 | /** |
587 | * hinic_wqs_cmdq_alloc - Allocate wqs for cmdqs |
588 | * @cmdq_pages: will hold the pages of the cmdq |
589 | * @wq: returned wqs |
590 | * @hwif: HW interface |
591 | * @cmdq_blocks: number of cmdq blocks/wq to allocate |
592 | * @wqebb_size: Work Queue Block Byte Size |
593 | * @wq_page_size: the page size in the Work Queue |
594 | * @q_depth: number of wqebbs in WQ |
595 | * @max_wqe_size: maximum WQE size that will be used in the WQ |
596 | * |
597 | * Return 0 - Success, negative - Failure |
598 | **/ |
599 | int hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages *cmdq_pages, |
600 | struct hinic_wq *wq, struct hinic_hwif *hwif, |
601 | int cmdq_blocks, u16 wqebb_size, u32 wq_page_size, |
602 | u16 q_depth, u16 max_wqe_size) |
603 | { |
604 | struct pci_dev *pdev = hwif->pdev; |
605 | u16 num_wqebbs_per_page_shift; |
606 | u16 num_wqebbs_per_page; |
607 | u16 wqebb_size_shift; |
608 | int i, j, err = -ENOMEM; |
609 | |
610 | if (!is_power_of_2(n: wqebb_size)) { |
611 | dev_err(&pdev->dev, "wqebb_size must be power of 2\n" ); |
612 | return -EINVAL; |
613 | } |
614 | |
615 | if (wq_page_size == 0) { |
616 | dev_err(&pdev->dev, "wq_page_size must be > 0\n" ); |
617 | return -EINVAL; |
618 | } |
619 | |
620 | if (q_depth & (q_depth - 1)) { |
621 | dev_err(&pdev->dev, "WQ q_depth must be power of 2\n" ); |
622 | return -EINVAL; |
623 | } |
624 | |
625 | wqebb_size_shift = ilog2(wqebb_size); |
626 | num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) |
627 | >> wqebb_size_shift; |
628 | |
629 | if (!is_power_of_2(n: num_wqebbs_per_page)) { |
630 | dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n" ); |
631 | return -EINVAL; |
632 | } |
633 | |
634 | cmdq_pages->hwif = hwif; |
635 | |
636 | err = cmdq_allocate_page(cmdq_pages); |
637 | if (err) { |
638 | dev_err(&pdev->dev, "Failed to allocate CMDQ page\n" ); |
639 | return err; |
640 | } |
641 | num_wqebbs_per_page_shift = ilog2(num_wqebbs_per_page); |
642 | |
643 | for (i = 0; i < cmdq_blocks; i++) { |
644 | wq[i].hwif = hwif; |
645 | wq[i].page_idx = 0; |
646 | wq[i].block_idx = i; |
647 | |
648 | wq[i].wqebb_size = wqebb_size; |
649 | wq[i].wq_page_size = wq_page_size; |
650 | wq[i].q_depth = q_depth; |
651 | wq[i].max_wqe_size = max_wqe_size; |
652 | wq[i].num_wqebbs_per_page = num_wqebbs_per_page; |
653 | wq[i].wqebbs_per_page_shift = num_wqebbs_per_page_shift; |
654 | wq[i].wqebb_size_shift = wqebb_size_shift; |
655 | wq[i].block_vaddr = CMDQ_BASE_VADDR(cmdq_pages, &wq[i]); |
656 | wq[i].shadow_block_vaddr = CMDQ_BASE_ADDR(cmdq_pages, &wq[i]); |
657 | wq[i].block_paddr = CMDQ_BASE_PADDR(cmdq_pages, &wq[i]); |
658 | |
659 | err = alloc_wq_pages(wq: &wq[i], hwif: cmdq_pages->hwif, |
660 | CMDQ_WQ_MAX_PAGES); |
661 | if (err) { |
662 | dev_err(&pdev->dev, "Failed to alloc CMDQ blocks\n" ); |
663 | goto err_cmdq_block; |
664 | } |
665 | |
666 | atomic_set(v: &wq[i].cons_idx, i: 0); |
667 | atomic_set(v: &wq[i].prod_idx, i: 0); |
668 | atomic_set(v: &wq[i].delta, i: q_depth); |
669 | wq[i].mask = q_depth - 1; |
670 | } |
671 | |
672 | return 0; |
673 | |
674 | err_cmdq_block: |
675 | for (j = 0; j < i; j++) |
676 | free_wq_pages(wq: &wq[j], hwif: cmdq_pages->hwif, num_q_pages: wq[j].num_q_pages); |
677 | |
678 | cmdq_free_page(cmdq_pages); |
679 | return err; |
680 | } |
681 | |
682 | /** |
683 | * hinic_wqs_cmdq_free - Free wqs from cmdqs |
684 | * @cmdq_pages: hold the pages of the cmdq |
685 | * @wq: wqs to free |
686 | * @cmdq_blocks: number of wqs to free |
687 | **/ |
688 | void hinic_wqs_cmdq_free(struct hinic_cmdq_pages *cmdq_pages, |
689 | struct hinic_wq *wq, int cmdq_blocks) |
690 | { |
691 | int i; |
692 | |
693 | for (i = 0; i < cmdq_blocks; i++) |
694 | free_wq_pages(wq: &wq[i], hwif: cmdq_pages->hwif, num_q_pages: wq[i].num_q_pages); |
695 | |
696 | cmdq_free_page(cmdq_pages); |
697 | } |
698 | |
699 | static void copy_wqe_to_shadow(struct hinic_wq *wq, void *shadow_addr, |
700 | int num_wqebbs, u16 idx) |
701 | { |
702 | void *wqebb_addr; |
703 | int i; |
704 | |
705 | for (i = 0; i < num_wqebbs; i++, idx++) { |
706 | idx = MASKED_WQE_IDX(wq, idx); |
707 | wqebb_addr = WQ_PAGE_ADDR(wq, idx) + |
708 | WQE_PAGE_OFF(wq, idx); |
709 | |
710 | memcpy(shadow_addr, wqebb_addr, wq->wqebb_size); |
711 | |
712 | shadow_addr += wq->wqebb_size; |
713 | } |
714 | } |
715 | |
716 | static void copy_wqe_from_shadow(struct hinic_wq *wq, void *shadow_addr, |
717 | int num_wqebbs, u16 idx) |
718 | { |
719 | void *wqebb_addr; |
720 | int i; |
721 | |
722 | for (i = 0; i < num_wqebbs; i++, idx++) { |
723 | idx = MASKED_WQE_IDX(wq, idx); |
724 | wqebb_addr = WQ_PAGE_ADDR(wq, idx) + |
725 | WQE_PAGE_OFF(wq, idx); |
726 | |
727 | memcpy(wqebb_addr, shadow_addr, wq->wqebb_size); |
728 | shadow_addr += wq->wqebb_size; |
729 | } |
730 | } |
731 | |
732 | /** |
733 | * hinic_get_wqe - get wqe ptr in the current pi and update the pi |
734 | * @wq: wq to get wqe from |
735 | * @wqe_size: wqe size |
736 | * @prod_idx: returned pi |
737 | * |
738 | * Return wqe pointer |
739 | **/ |
740 | struct hinic_hw_wqe *hinic_get_wqe(struct hinic_wq *wq, unsigned int wqe_size, |
741 | u16 *prod_idx) |
742 | { |
743 | int curr_pg, end_pg, num_wqebbs; |
744 | u16 curr_prod_idx, end_prod_idx; |
745 | |
746 | *prod_idx = MASKED_WQE_IDX(wq, atomic_read(&wq->prod_idx)); |
747 | |
748 | num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) >> wq->wqebb_size_shift; |
749 | |
750 | if (atomic_sub_return(i: num_wqebbs, v: &wq->delta) <= 0) { |
751 | atomic_add(i: num_wqebbs, v: &wq->delta); |
752 | return ERR_PTR(error: -EBUSY); |
753 | } |
754 | |
755 | end_prod_idx = atomic_add_return(i: num_wqebbs, v: &wq->prod_idx); |
756 | |
757 | end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx); |
758 | curr_prod_idx = end_prod_idx - num_wqebbs; |
759 | curr_prod_idx = MASKED_WQE_IDX(wq, curr_prod_idx); |
760 | |
761 | /* end prod index points to the next wqebb, therefore minus 1 */ |
762 | end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx - 1); |
763 | |
764 | curr_pg = WQE_PAGE_NUM(wq, idx: curr_prod_idx); |
765 | end_pg = WQE_PAGE_NUM(wq, idx: end_prod_idx); |
766 | |
767 | *prod_idx = curr_prod_idx; |
768 | |
769 | /* If we only have one page, still need to get shadown wqe when |
770 | * wqe rolling-over page |
771 | */ |
772 | if (curr_pg != end_pg || end_prod_idx < *prod_idx) { |
773 | void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; |
774 | |
775 | copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, idx: *prod_idx); |
776 | |
777 | wq->shadow_idx[curr_pg] = *prod_idx; |
778 | return shadow_addr; |
779 | } |
780 | |
781 | return WQ_PAGE_ADDR(wq, *prod_idx) + WQE_PAGE_OFF(wq, idx: *prod_idx); |
782 | } |
783 | |
784 | /** |
785 | * hinic_return_wqe - return the wqe when transmit failed |
786 | * @wq: wq to return wqe |
787 | * @wqe_size: wqe size |
788 | **/ |
789 | void hinic_return_wqe(struct hinic_wq *wq, unsigned int wqe_size) |
790 | { |
791 | int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; |
792 | |
793 | atomic_sub(i: num_wqebbs, v: &wq->prod_idx); |
794 | |
795 | atomic_add(i: num_wqebbs, v: &wq->delta); |
796 | } |
797 | |
798 | /** |
799 | * hinic_put_wqe - return the wqe place to use for a new wqe |
800 | * @wq: wq to return wqe |
801 | * @wqe_size: wqe size |
802 | **/ |
803 | void hinic_put_wqe(struct hinic_wq *wq, unsigned int wqe_size) |
804 | { |
805 | int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) |
806 | >> wq->wqebb_size_shift; |
807 | |
808 | atomic_add(i: num_wqebbs, v: &wq->cons_idx); |
809 | |
810 | atomic_add(i: num_wqebbs, v: &wq->delta); |
811 | } |
812 | |
813 | /** |
814 | * hinic_read_wqe - read wqe ptr in the current ci |
815 | * @wq: wq to get read from |
816 | * @wqe_size: wqe size |
817 | * @cons_idx: returned ci |
818 | * |
819 | * Return wqe pointer |
820 | **/ |
821 | struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size, |
822 | u16 *cons_idx) |
823 | { |
824 | int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) |
825 | >> wq->wqebb_size_shift; |
826 | u16 curr_cons_idx, end_cons_idx; |
827 | int curr_pg, end_pg; |
828 | |
829 | if ((atomic_read(v: &wq->delta) + num_wqebbs) > wq->q_depth) |
830 | return ERR_PTR(error: -EBUSY); |
831 | |
832 | curr_cons_idx = atomic_read(v: &wq->cons_idx); |
833 | |
834 | curr_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx); |
835 | end_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx + num_wqebbs - 1); |
836 | |
837 | curr_pg = WQE_PAGE_NUM(wq, idx: curr_cons_idx); |
838 | end_pg = WQE_PAGE_NUM(wq, idx: end_cons_idx); |
839 | |
840 | *cons_idx = curr_cons_idx; |
841 | |
842 | /* If we only have one page, still need to get shadown wqe when |
843 | * wqe rolling-over page |
844 | */ |
845 | if (curr_pg != end_pg || end_cons_idx < curr_cons_idx) { |
846 | void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; |
847 | |
848 | copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, idx: *cons_idx); |
849 | return shadow_addr; |
850 | } |
851 | |
852 | return WQ_PAGE_ADDR(wq, *cons_idx) + WQE_PAGE_OFF(wq, idx: *cons_idx); |
853 | } |
854 | |
855 | /** |
856 | * hinic_read_wqe_direct - read wqe directly from ci position |
857 | * @wq: wq |
858 | * @cons_idx: ci position |
859 | * |
860 | * Return wqe |
861 | **/ |
862 | struct hinic_hw_wqe *hinic_read_wqe_direct(struct hinic_wq *wq, u16 cons_idx) |
863 | { |
864 | return WQ_PAGE_ADDR(wq, cons_idx) + WQE_PAGE_OFF(wq, idx: cons_idx); |
865 | } |
866 | |
867 | /** |
868 | * wqe_shadow - check if a wqe is shadow |
869 | * @wq: wq of the wqe |
870 | * @wqe: the wqe for shadow checking |
871 | * |
872 | * Return true - shadow, false - Not shadow |
873 | **/ |
874 | static inline bool wqe_shadow(struct hinic_wq *wq, struct hinic_hw_wqe *wqe) |
875 | { |
876 | size_t wqe_shadow_size = wq->num_q_pages * wq->max_wqe_size; |
877 | |
878 | return WQE_IN_RANGE(wqe, wq->shadow_wqe, |
879 | &wq->shadow_wqe[wqe_shadow_size]); |
880 | } |
881 | |
882 | /** |
883 | * hinic_write_wqe - write the wqe to the wq |
884 | * @wq: wq to write wqe to |
885 | * @wqe: wqe to write |
886 | * @wqe_size: wqe size |
887 | **/ |
888 | void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe, |
889 | unsigned int wqe_size) |
890 | { |
891 | int curr_pg, num_wqebbs; |
892 | void *shadow_addr; |
893 | u16 prod_idx; |
894 | |
895 | if (wqe_shadow(wq, wqe)) { |
896 | curr_pg = WQE_SHADOW_PAGE(wq, wqe); |
897 | |
898 | prod_idx = wq->shadow_idx[curr_pg]; |
899 | num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; |
900 | shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; |
901 | |
902 | copy_wqe_from_shadow(wq, shadow_addr, num_wqebbs, idx: prod_idx); |
903 | } |
904 | } |
905 | |