1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/gfp.h> |
3 | #include <linux/workqueue.h> |
4 | #include <crypto/internal/skcipher.h> |
5 | |
6 | #include "nitrox_common.h" |
7 | #include "nitrox_dev.h" |
8 | #include "nitrox_req.h" |
9 | #include "nitrox_csr.h" |
10 | |
11 | /* SLC_STORE_INFO */ |
12 | #define MIN_UDD_LEN 16 |
13 | /* PKT_IN_HDR + SLC_STORE_INFO */ |
14 | #define FDATA_SIZE 32 |
15 | /* Base destination port for the solicited requests */ |
16 | #define SOLICIT_BASE_DPORT 256 |
17 | |
18 | #define REQ_NOT_POSTED 1 |
19 | #define REQ_BACKLOG 2 |
20 | #define REQ_POSTED 3 |
21 | |
22 | /* |
23 | * Response codes from SE microcode |
24 | * 0x00 - Success |
25 | * Completion with no error |
26 | * 0x43 - ERR_GC_DATA_LEN_INVALID |
27 | * Invalid Data length if Encryption Data length is |
28 | * less than 16 bytes for AES-XTS and AES-CTS. |
29 | * 0x45 - ERR_GC_CTX_LEN_INVALID |
30 | * Invalid context length: CTXL != 23 words. |
31 | * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID |
32 | * DOCSIS support is enabled with other than |
33 | * AES/DES-CBC mode encryption. |
34 | * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID |
35 | * Authentication offset is other than 0 with |
36 | * Encryption IV source = 0. |
37 | * Authentication offset is other than 8 (DES)/16 (AES) |
38 | * with Encryption IV source = 1 |
39 | * 0x51 - ERR_GC_CRC32_INVALID_SELECTION |
40 | * CRC32 is enabled for other than DOCSIS encryption. |
41 | * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID |
42 | * Invalid flag options in AES-CCM IV. |
43 | */ |
44 | |
45 | static inline int incr_index(int index, int count, int max) |
46 | { |
47 | if ((index + count) >= max) |
48 | index = index + count - max; |
49 | else |
50 | index += count; |
51 | |
52 | return index; |
53 | } |
54 | |
55 | static void softreq_unmap_sgbufs(struct nitrox_softreq *sr) |
56 | { |
57 | struct nitrox_device *ndev = sr->ndev; |
58 | struct device *dev = DEV(ndev); |
59 | |
60 | |
61 | dma_unmap_sg(dev, sr->in.sg, sg_nents(sr->in.sg), |
62 | DMA_BIDIRECTIONAL); |
63 | dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len, |
64 | DMA_TO_DEVICE); |
65 | kfree(objp: sr->in.sgcomp); |
66 | sr->in.sg = NULL; |
67 | sr->in.sgmap_cnt = 0; |
68 | |
69 | dma_unmap_sg(dev, sr->out.sg, sg_nents(sr->out.sg), |
70 | DMA_BIDIRECTIONAL); |
71 | dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len, |
72 | DMA_TO_DEVICE); |
73 | kfree(objp: sr->out.sgcomp); |
74 | sr->out.sg = NULL; |
75 | sr->out.sgmap_cnt = 0; |
76 | } |
77 | |
78 | static void softreq_destroy(struct nitrox_softreq *sr) |
79 | { |
80 | softreq_unmap_sgbufs(sr); |
81 | kfree(objp: sr); |
82 | } |
83 | |
84 | /** |
85 | * create_sg_component - create SG componets for N5 device. |
86 | * @sr: Request structure |
87 | * @sgtbl: SG table |
88 | * @map_nents: number of dma mapped entries |
89 | * |
90 | * Component structure |
91 | * |
92 | * 63 48 47 32 31 16 15 0 |
93 | * -------------------------------------- |
94 | * | LEN0 | LEN1 | LEN2 | LEN3 | |
95 | * |------------------------------------- |
96 | * | PTR0 | |
97 | * -------------------------------------- |
98 | * | PTR1 | |
99 | * -------------------------------------- |
100 | * | PTR2 | |
101 | * -------------------------------------- |
102 | * | PTR3 | |
103 | * -------------------------------------- |
104 | * |
105 | * Returns 0 if success or a negative errno code on error. |
106 | */ |
107 | static int create_sg_component(struct nitrox_softreq *sr, |
108 | struct nitrox_sgtable *sgtbl, int map_nents) |
109 | { |
110 | struct nitrox_device *ndev = sr->ndev; |
111 | struct nitrox_sgcomp *sgcomp; |
112 | struct scatterlist *sg; |
113 | dma_addr_t dma; |
114 | size_t sz_comp; |
115 | int i, j, nr_sgcomp; |
116 | |
117 | nr_sgcomp = roundup(map_nents, 4) / 4; |
118 | |
119 | /* each component holds 4 dma pointers */ |
120 | sz_comp = nr_sgcomp * sizeof(*sgcomp); |
121 | sgcomp = kzalloc(size: sz_comp, flags: sr->gfp); |
122 | if (!sgcomp) |
123 | return -ENOMEM; |
124 | |
125 | sgtbl->sgcomp = sgcomp; |
126 | |
127 | sg = sgtbl->sg; |
128 | /* populate device sg component */ |
129 | for (i = 0; i < nr_sgcomp; i++) { |
130 | for (j = 0; j < 4 && sg; j++) { |
131 | sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg)); |
132 | sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg)); |
133 | sg = sg_next(sg); |
134 | } |
135 | } |
136 | /* map the device sg component */ |
137 | dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE); |
138 | if (dma_mapping_error(DEV(ndev), dma_addr: dma)) { |
139 | kfree(objp: sgtbl->sgcomp); |
140 | sgtbl->sgcomp = NULL; |
141 | return -ENOMEM; |
142 | } |
143 | |
144 | sgtbl->sgcomp_dma = dma; |
145 | sgtbl->sgcomp_len = sz_comp; |
146 | |
147 | return 0; |
148 | } |
149 | |
150 | /** |
151 | * dma_map_inbufs - DMA map input sglist and creates sglist component |
152 | * for N5 device. |
153 | * @sr: Request structure |
154 | * @req: Crypto request structre |
155 | * |
156 | * Returns 0 if successful or a negative errno code on error. |
157 | */ |
158 | static int dma_map_inbufs(struct nitrox_softreq *sr, |
159 | struct se_crypto_request *req) |
160 | { |
161 | struct device *dev = DEV(sr->ndev); |
162 | struct scatterlist *sg; |
163 | int i, nents, ret = 0; |
164 | |
165 | nents = dma_map_sg(dev, req->src, sg_nents(req->src), |
166 | DMA_BIDIRECTIONAL); |
167 | if (!nents) |
168 | return -EINVAL; |
169 | |
170 | for_each_sg(req->src, sg, nents, i) |
171 | sr->in.total_bytes += sg_dma_len(sg); |
172 | |
173 | sr->in.sg = req->src; |
174 | sr->in.sgmap_cnt = nents; |
175 | ret = create_sg_component(sr, sgtbl: &sr->in, map_nents: sr->in.sgmap_cnt); |
176 | if (ret) |
177 | goto incomp_err; |
178 | |
179 | return 0; |
180 | |
181 | incomp_err: |
182 | dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL); |
183 | sr->in.sgmap_cnt = 0; |
184 | return ret; |
185 | } |
186 | |
187 | static int dma_map_outbufs(struct nitrox_softreq *sr, |
188 | struct se_crypto_request *req) |
189 | { |
190 | struct device *dev = DEV(sr->ndev); |
191 | int nents, ret = 0; |
192 | |
193 | nents = dma_map_sg(dev, req->dst, sg_nents(req->dst), |
194 | DMA_BIDIRECTIONAL); |
195 | if (!nents) |
196 | return -EINVAL; |
197 | |
198 | sr->out.sg = req->dst; |
199 | sr->out.sgmap_cnt = nents; |
200 | ret = create_sg_component(sr, sgtbl: &sr->out, map_nents: sr->out.sgmap_cnt); |
201 | if (ret) |
202 | goto outcomp_map_err; |
203 | |
204 | return 0; |
205 | |
206 | outcomp_map_err: |
207 | dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL); |
208 | sr->out.sgmap_cnt = 0; |
209 | sr->out.sg = NULL; |
210 | return ret; |
211 | } |
212 | |
213 | static inline int softreq_map_iobuf(struct nitrox_softreq *sr, |
214 | struct se_crypto_request *creq) |
215 | { |
216 | int ret; |
217 | |
218 | ret = dma_map_inbufs(sr, req: creq); |
219 | if (ret) |
220 | return ret; |
221 | |
222 | ret = dma_map_outbufs(sr, req: creq); |
223 | if (ret) |
224 | softreq_unmap_sgbufs(sr); |
225 | |
226 | return ret; |
227 | } |
228 | |
229 | static inline void backlog_list_add(struct nitrox_softreq *sr, |
230 | struct nitrox_cmdq *cmdq) |
231 | { |
232 | INIT_LIST_HEAD(list: &sr->backlog); |
233 | |
234 | spin_lock_bh(lock: &cmdq->backlog_qlock); |
235 | list_add_tail(new: &sr->backlog, head: &cmdq->backlog_head); |
236 | atomic_inc(v: &cmdq->backlog_count); |
237 | atomic_set(v: &sr->status, REQ_BACKLOG); |
238 | spin_unlock_bh(lock: &cmdq->backlog_qlock); |
239 | } |
240 | |
241 | static inline void response_list_add(struct nitrox_softreq *sr, |
242 | struct nitrox_cmdq *cmdq) |
243 | { |
244 | INIT_LIST_HEAD(list: &sr->response); |
245 | |
246 | spin_lock_bh(lock: &cmdq->resp_qlock); |
247 | list_add_tail(new: &sr->response, head: &cmdq->response_head); |
248 | spin_unlock_bh(lock: &cmdq->resp_qlock); |
249 | } |
250 | |
251 | static inline void response_list_del(struct nitrox_softreq *sr, |
252 | struct nitrox_cmdq *cmdq) |
253 | { |
254 | spin_lock_bh(lock: &cmdq->resp_qlock); |
255 | list_del(entry: &sr->response); |
256 | spin_unlock_bh(lock: &cmdq->resp_qlock); |
257 | } |
258 | |
259 | static struct nitrox_softreq * |
260 | get_first_response_entry(struct nitrox_cmdq *cmdq) |
261 | { |
262 | return list_first_entry_or_null(&cmdq->response_head, |
263 | struct nitrox_softreq, response); |
264 | } |
265 | |
266 | static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen) |
267 | { |
268 | if (atomic_inc_return(v: &cmdq->pending_count) > qlen) { |
269 | atomic_dec(v: &cmdq->pending_count); |
270 | /* sync with other cpus */ |
271 | smp_mb__after_atomic(); |
272 | return true; |
273 | } |
274 | /* sync with other cpus */ |
275 | smp_mb__after_atomic(); |
276 | return false; |
277 | } |
278 | |
279 | /** |
280 | * post_se_instr - Post SE instruction to Packet Input ring |
281 | * @sr: Request structure |
282 | * @cmdq: Command queue structure |
283 | * |
284 | * Returns 0 if successful or a negative error code, |
285 | * if no space in ring. |
286 | */ |
287 | static void post_se_instr(struct nitrox_softreq *sr, |
288 | struct nitrox_cmdq *cmdq) |
289 | { |
290 | struct nitrox_device *ndev = sr->ndev; |
291 | int idx; |
292 | u8 *ent; |
293 | |
294 | spin_lock_bh(lock: &cmdq->cmd_qlock); |
295 | |
296 | idx = cmdq->write_idx; |
297 | /* copy the instruction */ |
298 | ent = cmdq->base + (idx * cmdq->instr_size); |
299 | memcpy(ent, &sr->instr, cmdq->instr_size); |
300 | |
301 | atomic_set(v: &sr->status, REQ_POSTED); |
302 | response_list_add(sr, cmdq); |
303 | sr->tstamp = jiffies; |
304 | /* flush the command queue updates */ |
305 | dma_wmb(); |
306 | |
307 | /* Ring doorbell with count 1 */ |
308 | writeq(val: 1, addr: cmdq->dbell_csr_addr); |
309 | |
310 | cmdq->write_idx = incr_index(index: idx, count: 1, max: ndev->qlen); |
311 | |
312 | spin_unlock_bh(lock: &cmdq->cmd_qlock); |
313 | |
314 | /* increment the posted command count */ |
315 | atomic64_inc(v: &ndev->stats.posted); |
316 | } |
317 | |
318 | static int post_backlog_cmds(struct nitrox_cmdq *cmdq) |
319 | { |
320 | struct nitrox_device *ndev = cmdq->ndev; |
321 | struct nitrox_softreq *sr, *tmp; |
322 | int ret = 0; |
323 | |
324 | if (!atomic_read(v: &cmdq->backlog_count)) |
325 | return 0; |
326 | |
327 | spin_lock_bh(lock: &cmdq->backlog_qlock); |
328 | |
329 | list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) { |
330 | /* submit until space available */ |
331 | if (unlikely(cmdq_full(cmdq, ndev->qlen))) { |
332 | ret = -ENOSPC; |
333 | break; |
334 | } |
335 | /* delete from backlog list */ |
336 | list_del(entry: &sr->backlog); |
337 | atomic_dec(v: &cmdq->backlog_count); |
338 | /* sync with other cpus */ |
339 | smp_mb__after_atomic(); |
340 | |
341 | /* post the command */ |
342 | post_se_instr(sr, cmdq); |
343 | } |
344 | spin_unlock_bh(lock: &cmdq->backlog_qlock); |
345 | |
346 | return ret; |
347 | } |
348 | |
349 | static int nitrox_enqueue_request(struct nitrox_softreq *sr) |
350 | { |
351 | struct nitrox_cmdq *cmdq = sr->cmdq; |
352 | struct nitrox_device *ndev = sr->ndev; |
353 | |
354 | /* try to post backlog requests */ |
355 | post_backlog_cmds(cmdq); |
356 | |
357 | if (unlikely(cmdq_full(cmdq, ndev->qlen))) { |
358 | if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { |
359 | /* increment drop count */ |
360 | atomic64_inc(v: &ndev->stats.dropped); |
361 | return -ENOSPC; |
362 | } |
363 | /* add to backlog list */ |
364 | backlog_list_add(sr, cmdq); |
365 | return -EINPROGRESS; |
366 | } |
367 | post_se_instr(sr, cmdq); |
368 | |
369 | return -EINPROGRESS; |
370 | } |
371 | |
372 | /** |
373 | * nitrox_process_se_request - Send request to SE core |
374 | * @ndev: NITROX device |
375 | * @req: Crypto request |
376 | * @callback: Completion callback |
377 | * @cb_arg: Completion callback arguments |
378 | * |
379 | * Returns 0 on success, or a negative error code. |
380 | */ |
381 | int nitrox_process_se_request(struct nitrox_device *ndev, |
382 | struct se_crypto_request *req, |
383 | completion_t callback, |
384 | void *cb_arg) |
385 | { |
386 | struct nitrox_softreq *sr; |
387 | dma_addr_t ctx_handle = 0; |
388 | int qno, ret = 0; |
389 | |
390 | if (!nitrox_ready(ndev)) |
391 | return -ENODEV; |
392 | |
393 | sr = kzalloc(size: sizeof(*sr), flags: req->gfp); |
394 | if (!sr) |
395 | return -ENOMEM; |
396 | |
397 | sr->ndev = ndev; |
398 | sr->flags = req->flags; |
399 | sr->gfp = req->gfp; |
400 | sr->callback = callback; |
401 | sr->cb_arg = cb_arg; |
402 | |
403 | atomic_set(v: &sr->status, REQ_NOT_POSTED); |
404 | |
405 | sr->resp.orh = req->orh; |
406 | sr->resp.completion = req->comp; |
407 | |
408 | ret = softreq_map_iobuf(sr, creq: req); |
409 | if (ret) { |
410 | kfree(objp: sr); |
411 | return ret; |
412 | } |
413 | |
414 | /* get the context handle */ |
415 | if (req->ctx_handle) { |
416 | struct ctx_hdr *hdr; |
417 | u8 *ctx_ptr; |
418 | |
419 | ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle; |
420 | hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr)); |
421 | ctx_handle = hdr->ctx_dma; |
422 | } |
423 | |
424 | /* select the queue */ |
425 | qno = smp_processor_id() % ndev->nr_queues; |
426 | |
427 | sr->cmdq = &ndev->pkt_inq[qno]; |
428 | |
429 | /* |
430 | * 64-Byte Instruction Format |
431 | * |
432 | * ---------------------- |
433 | * | DPTR0 | 8 bytes |
434 | * ---------------------- |
435 | * | PKT_IN_INSTR_HDR | 8 bytes |
436 | * ---------------------- |
437 | * | PKT_IN_HDR | 16 bytes |
438 | * ---------------------- |
439 | * | SLC_INFO | 16 bytes |
440 | * ---------------------- |
441 | * | Front data | 16 bytes |
442 | * ---------------------- |
443 | */ |
444 | |
445 | /* fill the packet instruction */ |
446 | /* word 0 */ |
447 | sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma); |
448 | |
449 | /* word 1 */ |
450 | sr->instr.ih.value = 0; |
451 | sr->instr.ih.s.g = 1; |
452 | sr->instr.ih.s.gsz = sr->in.sgmap_cnt; |
453 | sr->instr.ih.s.ssz = sr->out.sgmap_cnt; |
454 | sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); |
455 | sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; |
456 | sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value); |
457 | |
458 | /* word 2 */ |
459 | sr->instr.irh.value[0] = 0; |
460 | sr->instr.irh.s.uddl = MIN_UDD_LEN; |
461 | /* context length in 64-bit words */ |
462 | sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8); |
463 | /* offset from solicit base port 256 */ |
464 | sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno; |
465 | sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; |
466 | sr->instr.irh.s.arg = req->ctrl.s.arg; |
467 | sr->instr.irh.s.opcode = req->opcode; |
468 | sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]); |
469 | |
470 | /* word 3 */ |
471 | sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); |
472 | |
473 | /* word 4 */ |
474 | sr->instr.slc.value[0] = 0; |
475 | sr->instr.slc.s.ssz = sr->out.sgmap_cnt; |
476 | sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]); |
477 | |
478 | /* word 5 */ |
479 | sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma); |
480 | |
481 | /* |
482 | * No conversion for front data, |
483 | * It goes into payload |
484 | * put GP Header in front data |
485 | */ |
486 | sr->instr.fdata[0] = *((u64 *)&req->gph); |
487 | sr->instr.fdata[1] = 0; |
488 | |
489 | ret = nitrox_enqueue_request(sr); |
490 | if (ret == -ENOSPC) |
491 | goto send_fail; |
492 | |
493 | return ret; |
494 | |
495 | send_fail: |
496 | softreq_destroy(sr); |
497 | return ret; |
498 | } |
499 | |
500 | static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout) |
501 | { |
502 | return time_after_eq(jiffies, (tstamp + timeout)); |
503 | } |
504 | |
505 | void backlog_qflush_work(struct work_struct *work) |
506 | { |
507 | struct nitrox_cmdq *cmdq; |
508 | |
509 | cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush); |
510 | post_backlog_cmds(cmdq); |
511 | } |
512 | |
513 | static bool sr_completed(struct nitrox_softreq *sr) |
514 | { |
515 | u64 orh = READ_ONCE(*sr->resp.orh); |
516 | unsigned long timeout = jiffies + msecs_to_jiffies(m: 1); |
517 | |
518 | if ((orh != PENDING_SIG) && (orh & 0xff)) |
519 | return true; |
520 | |
521 | while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) { |
522 | if (time_after(jiffies, timeout)) { |
523 | pr_err("comp not done\n" ); |
524 | return false; |
525 | } |
526 | } |
527 | |
528 | return true; |
529 | } |
530 | |
531 | /** |
532 | * process_response_list - process completed requests |
533 | * @cmdq: Command queue structure |
534 | * |
535 | * Returns the number of responses processed. |
536 | */ |
537 | static void process_response_list(struct nitrox_cmdq *cmdq) |
538 | { |
539 | struct nitrox_device *ndev = cmdq->ndev; |
540 | struct nitrox_softreq *sr; |
541 | int req_completed = 0, err = 0, budget; |
542 | completion_t callback; |
543 | void *cb_arg; |
544 | |
545 | /* check all pending requests */ |
546 | budget = atomic_read(v: &cmdq->pending_count); |
547 | |
548 | while (req_completed < budget) { |
549 | sr = get_first_response_entry(cmdq); |
550 | if (!sr) |
551 | break; |
552 | |
553 | if (atomic_read(v: &sr->status) != REQ_POSTED) |
554 | break; |
555 | |
556 | /* check orh and completion bytes updates */ |
557 | if (!sr_completed(sr)) { |
558 | /* request not completed, check for timeout */ |
559 | if (!cmd_timeout(tstamp: sr->tstamp, timeout: ndev->timeout)) |
560 | break; |
561 | dev_err_ratelimited(DEV(ndev), |
562 | "Request timeout, orh 0x%016llx\n" , |
563 | READ_ONCE(*sr->resp.orh)); |
564 | } |
565 | atomic_dec(v: &cmdq->pending_count); |
566 | atomic64_inc(v: &ndev->stats.completed); |
567 | /* sync with other cpus */ |
568 | smp_mb__after_atomic(); |
569 | /* remove from response list */ |
570 | response_list_del(sr, cmdq); |
571 | /* ORH error code */ |
572 | err = READ_ONCE(*sr->resp.orh) & 0xff; |
573 | callback = sr->callback; |
574 | cb_arg = sr->cb_arg; |
575 | softreq_destroy(sr); |
576 | if (callback) |
577 | callback(cb_arg, err); |
578 | |
579 | req_completed++; |
580 | } |
581 | } |
582 | |
583 | /* |
584 | * pkt_slc_resp_tasklet - post processing of SE responses |
585 | */ |
586 | void pkt_slc_resp_tasklet(unsigned long data) |
587 | { |
588 | struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data); |
589 | struct nitrox_cmdq *cmdq = qvec->cmdq; |
590 | union nps_pkt_slc_cnts slc_cnts; |
591 | |
592 | /* read completion count */ |
593 | slc_cnts.value = readq(addr: cmdq->compl_cnt_csr_addr); |
594 | /* resend the interrupt if more work to do */ |
595 | slc_cnts.s.resend = 1; |
596 | |
597 | process_response_list(cmdq); |
598 | |
599 | /* |
600 | * clear the interrupt with resend bit enabled, |
601 | * MSI-X interrupt generates if Completion count > Threshold |
602 | */ |
603 | writeq(val: slc_cnts.value, addr: cmdq->compl_cnt_csr_addr); |
604 | |
605 | if (atomic_read(v: &cmdq->backlog_count)) |
606 | schedule_work(work: &cmdq->backlog_qflush); |
607 | } |
608 | |