1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * CAAM/SEC 4.x QI transport/backend driver |
4 | * Queue Interface backend functionality |
5 | * |
6 | * Copyright 2013-2016 Freescale Semiconductor, Inc. |
7 | * Copyright 2016-2017, 2019-2020 NXP |
8 | */ |
9 | |
10 | #include <linux/cpumask.h> |
11 | #include <linux/device.h> |
12 | #include <linux/dma-mapping.h> |
13 | #include <linux/kernel.h> |
14 | #include <linux/kthread.h> |
15 | #include <linux/netdevice.h> |
16 | #include <linux/platform_device.h> |
17 | #include <linux/slab.h> |
18 | #include <linux/string.h> |
19 | #include <soc/fsl/qman.h> |
20 | |
21 | #include "debugfs.h" |
22 | #include "regs.h" |
23 | #include "qi.h" |
24 | #include "desc.h" |
25 | #include "intern.h" |
26 | #include "desc_constr.h" |
27 | |
28 | #define PREHDR_RSLS_SHIFT 31 |
29 | #define PREHDR_ABS BIT(25) |
30 | |
31 | /* |
32 | * Use a reasonable backlog of frames (per CPU) as congestion threshold, |
33 | * so that resources used by the in-flight buffers do not become a memory hog. |
34 | */ |
35 | #define MAX_RSP_FQ_BACKLOG_PER_CPU 256 |
36 | |
37 | #define CAAM_QI_ENQUEUE_RETRIES 10000 |
38 | |
39 | #define CAAM_NAPI_WEIGHT 63 |
40 | |
41 | /* |
42 | * caam_napi - struct holding CAAM NAPI-related params |
43 | * @irqtask: IRQ task for QI backend |
44 | * @p: QMan portal |
45 | */ |
46 | struct caam_napi { |
47 | struct napi_struct irqtask; |
48 | struct qman_portal *p; |
49 | }; |
50 | |
51 | /* |
52 | * caam_qi_pcpu_priv - percpu private data structure to main list of pending |
53 | * responses expected on each cpu. |
54 | * @caam_napi: CAAM NAPI params |
55 | * @net_dev: netdev used by NAPI |
56 | * @rsp_fq: response FQ from CAAM |
57 | */ |
58 | struct caam_qi_pcpu_priv { |
59 | struct caam_napi caam_napi; |
60 | struct net_device net_dev; |
61 | struct qman_fq *rsp_fq; |
62 | } ____cacheline_aligned; |
63 | |
64 | static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv); |
65 | static DEFINE_PER_CPU(int, last_cpu); |
66 | |
67 | /* |
68 | * caam_qi_priv - CAAM QI backend private params |
69 | * @cgr: QMan congestion group |
70 | */ |
71 | struct caam_qi_priv { |
72 | struct qman_cgr cgr; |
73 | }; |
74 | |
75 | static struct caam_qi_priv qipriv ____cacheline_aligned; |
76 | |
77 | /* |
78 | * This is written by only one core - the one that initialized the CGR - and |
79 | * read by multiple cores (all the others). |
80 | */ |
81 | bool caam_congested __read_mostly; |
82 | EXPORT_SYMBOL(caam_congested); |
83 | |
84 | /* |
85 | * This is a cache of buffers, from which the users of CAAM QI driver |
86 | * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than |
87 | * doing malloc on the hotpath. |
88 | * NOTE: A more elegant solution would be to have some headroom in the frames |
89 | * being processed. This could be added by the dpaa-ethernet driver. |
90 | * This would pose a problem for userspace application processing which |
91 | * cannot know of this limitation. So for now, this will work. |
92 | * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here |
93 | */ |
94 | static struct kmem_cache *qi_cache; |
95 | |
96 | static void *caam_iova_to_virt(struct iommu_domain *domain, |
97 | dma_addr_t iova_addr) |
98 | { |
99 | phys_addr_t phys_addr; |
100 | |
101 | phys_addr = domain ? iommu_iova_to_phys(domain, iova: iova_addr) : iova_addr; |
102 | |
103 | return phys_to_virt(address: phys_addr); |
104 | } |
105 | |
106 | int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req) |
107 | { |
108 | struct qm_fd fd; |
109 | dma_addr_t addr; |
110 | int ret; |
111 | int num_retries = 0; |
112 | |
113 | qm_fd_clear_fd(fd: &fd); |
114 | qm_fd_set_compound(&fd, qm_sg_entry_get_len(&req->fd_sgt[1])); |
115 | |
116 | addr = dma_map_single(qidev, req->fd_sgt, sizeof(req->fd_sgt), |
117 | DMA_BIDIRECTIONAL); |
118 | if (dma_mapping_error(dev: qidev, dma_addr: addr)) { |
119 | dev_err(qidev, "DMA mapping error for QI enqueue request\n" ); |
120 | return -EIO; |
121 | } |
122 | qm_fd_addr_set64(fd: &fd, addr); |
123 | |
124 | do { |
125 | ret = qman_enqueue(fq: req->drv_ctx->req_fq, fd: &fd); |
126 | if (likely(!ret)) { |
127 | refcount_inc(r: &req->drv_ctx->refcnt); |
128 | return 0; |
129 | } |
130 | |
131 | if (ret != -EBUSY) |
132 | break; |
133 | num_retries++; |
134 | } while (num_retries < CAAM_QI_ENQUEUE_RETRIES); |
135 | |
136 | dev_err(qidev, "qman_enqueue failed: %d\n" , ret); |
137 | |
138 | return ret; |
139 | } |
140 | EXPORT_SYMBOL(caam_qi_enqueue); |
141 | |
142 | static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq, |
143 | const union qm_mr_entry *msg) |
144 | { |
145 | const struct qm_fd *fd; |
146 | struct caam_drv_req *drv_req; |
147 | struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); |
148 | struct caam_drv_private *priv = dev_get_drvdata(dev: qidev); |
149 | |
150 | fd = &msg->ern.fd; |
151 | |
152 | drv_req = caam_iova_to_virt(domain: priv->domain, iova_addr: qm_fd_addr_get64(fd)); |
153 | if (!drv_req) { |
154 | dev_err(qidev, |
155 | "Can't find original request for CAAM response\n" ); |
156 | return; |
157 | } |
158 | |
159 | refcount_dec(r: &drv_req->drv_ctx->refcnt); |
160 | |
161 | if (qm_fd_get_format(fd) != qm_fd_compound) { |
162 | dev_err(qidev, "Non-compound FD from CAAM\n" ); |
163 | return; |
164 | } |
165 | |
166 | dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd), |
167 | sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL); |
168 | |
169 | if (fd->status) |
170 | drv_req->cbk(drv_req, be32_to_cpu(fd->status)); |
171 | else |
172 | drv_req->cbk(drv_req, JRSTA_SSRC_QI); |
173 | } |
174 | |
175 | static struct qman_fq *create_caam_req_fq(struct device *qidev, |
176 | struct qman_fq *rsp_fq, |
177 | dma_addr_t hwdesc, |
178 | int fq_sched_flag) |
179 | { |
180 | int ret; |
181 | struct qman_fq *req_fq; |
182 | struct qm_mcc_initfq opts; |
183 | |
184 | req_fq = kzalloc(size: sizeof(*req_fq), GFP_ATOMIC); |
185 | if (!req_fq) |
186 | return ERR_PTR(error: -ENOMEM); |
187 | |
188 | req_fq->cb.ern = caam_fq_ern_cb; |
189 | req_fq->cb.fqs = NULL; |
190 | |
191 | ret = qman_create_fq(fqid: 0, QMAN_FQ_FLAG_DYNAMIC_FQID | |
192 | QMAN_FQ_FLAG_TO_DCPORTAL, fq: req_fq); |
193 | if (ret) { |
194 | dev_err(qidev, "Failed to create session req FQ\n" ); |
195 | goto create_req_fq_fail; |
196 | } |
197 | |
198 | memset(&opts, 0, sizeof(opts)); |
199 | opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ | |
200 | QM_INITFQ_WE_CONTEXTB | |
201 | QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID); |
202 | opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE); |
203 | qm_fqd_set_destwq(fqd: &opts.fqd, ch: qm_channel_caam, wq: 2); |
204 | opts.fqd.context_b = cpu_to_be32(qman_fq_fqid(rsp_fq)); |
205 | qm_fqd_context_a_set64(fqd: &opts.fqd, addr: hwdesc); |
206 | opts.fqd.cgid = qipriv.cgr.cgrid; |
207 | |
208 | ret = qman_init_fq(fq: req_fq, flags: fq_sched_flag, opts: &opts); |
209 | if (ret) { |
210 | dev_err(qidev, "Failed to init session req FQ\n" ); |
211 | goto init_req_fq_fail; |
212 | } |
213 | |
214 | dev_dbg(qidev, "Allocated request FQ %u for CPU %u\n" , req_fq->fqid, |
215 | smp_processor_id()); |
216 | return req_fq; |
217 | |
218 | init_req_fq_fail: |
219 | qman_destroy_fq(fq: req_fq); |
220 | create_req_fq_fail: |
221 | kfree(objp: req_fq); |
222 | return ERR_PTR(error: ret); |
223 | } |
224 | |
225 | static int empty_retired_fq(struct device *qidev, struct qman_fq *fq) |
226 | { |
227 | int ret; |
228 | |
229 | ret = qman_volatile_dequeue(fq, QMAN_VOLATILE_FLAG_WAIT_INT | |
230 | QMAN_VOLATILE_FLAG_FINISH, |
231 | QM_VDQCR_PRECEDENCE_VDQCR | |
232 | QM_VDQCR_NUMFRAMES_TILLEMPTY); |
233 | if (ret) { |
234 | dev_err(qidev, "Volatile dequeue fail for FQ: %u\n" , fq->fqid); |
235 | return ret; |
236 | } |
237 | |
238 | do { |
239 | struct qman_portal *p; |
240 | |
241 | p = qman_get_affine_portal(smp_processor_id()); |
242 | qman_p_poll_dqrr(p, limit: 16); |
243 | } while (fq->flags & QMAN_FQ_STATE_NE); |
244 | |
245 | return 0; |
246 | } |
247 | |
248 | static int kill_fq(struct device *qidev, struct qman_fq *fq) |
249 | { |
250 | u32 flags; |
251 | int ret; |
252 | |
253 | ret = qman_retire_fq(fq, flags: &flags); |
254 | if (ret < 0) { |
255 | dev_err(qidev, "qman_retire_fq failed: %d\n" , ret); |
256 | return ret; |
257 | } |
258 | |
259 | if (!ret) |
260 | goto empty_fq; |
261 | |
262 | /* Async FQ retirement condition */ |
263 | if (ret == 1) { |
264 | /* Retry till FQ gets in retired state */ |
265 | do { |
266 | msleep(msecs: 20); |
267 | } while (fq->state != qman_fq_state_retired); |
268 | |
269 | WARN_ON(fq->flags & QMAN_FQ_STATE_BLOCKOOS); |
270 | WARN_ON(fq->flags & QMAN_FQ_STATE_ORL); |
271 | } |
272 | |
273 | empty_fq: |
274 | if (fq->flags & QMAN_FQ_STATE_NE) { |
275 | ret = empty_retired_fq(qidev, fq); |
276 | if (ret) { |
277 | dev_err(qidev, "empty_retired_fq fail for FQ: %u\n" , |
278 | fq->fqid); |
279 | return ret; |
280 | } |
281 | } |
282 | |
283 | ret = qman_oos_fq(fq); |
284 | if (ret) |
285 | dev_err(qidev, "OOS of FQID: %u failed\n" , fq->fqid); |
286 | |
287 | qman_destroy_fq(fq); |
288 | kfree(objp: fq); |
289 | |
290 | return ret; |
291 | } |
292 | |
293 | static int empty_caam_fq(struct qman_fq *fq, struct caam_drv_ctx *drv_ctx) |
294 | { |
295 | int ret; |
296 | int retries = 10; |
297 | struct qm_mcr_queryfq_np np; |
298 | |
299 | /* Wait till the older CAAM FQ get empty */ |
300 | do { |
301 | ret = qman_query_fq_np(fq, np: &np); |
302 | if (ret) |
303 | return ret; |
304 | |
305 | if (!qm_mcr_np_get(&np, frm_cnt)) |
306 | break; |
307 | |
308 | msleep(msecs: 20); |
309 | } while (1); |
310 | |
311 | /* Wait until pending jobs from this FQ are processed by CAAM */ |
312 | do { |
313 | if (refcount_read(r: &drv_ctx->refcnt) == 1) |
314 | break; |
315 | |
316 | msleep(msecs: 20); |
317 | } while (--retries); |
318 | |
319 | if (!retries) |
320 | dev_warn_once(drv_ctx->qidev, "%d frames from FQID %u still pending in CAAM\n" , |
321 | refcount_read(&drv_ctx->refcnt), fq->fqid); |
322 | |
323 | return 0; |
324 | } |
325 | |
326 | int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc) |
327 | { |
328 | int ret; |
329 | u32 num_words; |
330 | struct qman_fq *new_fq, *old_fq; |
331 | struct device *qidev = drv_ctx->qidev; |
332 | |
333 | num_words = desc_len(desc: sh_desc); |
334 | if (num_words > MAX_SDLEN) { |
335 | dev_err(qidev, "Invalid descriptor len: %d words\n" , num_words); |
336 | return -EINVAL; |
337 | } |
338 | |
339 | /* Note down older req FQ */ |
340 | old_fq = drv_ctx->req_fq; |
341 | |
342 | /* Create a new req FQ in parked state */ |
343 | new_fq = create_caam_req_fq(qidev: drv_ctx->qidev, rsp_fq: drv_ctx->rsp_fq, |
344 | hwdesc: drv_ctx->context_a, fq_sched_flag: 0); |
345 | if (IS_ERR(ptr: new_fq)) { |
346 | dev_err(qidev, "FQ allocation for shdesc update failed\n" ); |
347 | return PTR_ERR(ptr: new_fq); |
348 | } |
349 | |
350 | /* Hook up new FQ to context so that new requests keep queuing */ |
351 | drv_ctx->req_fq = new_fq; |
352 | |
353 | /* Empty and remove the older FQ */ |
354 | ret = empty_caam_fq(fq: old_fq, drv_ctx); |
355 | if (ret) { |
356 | dev_err(qidev, "Old CAAM FQ empty failed: %d\n" , ret); |
357 | |
358 | /* We can revert to older FQ */ |
359 | drv_ctx->req_fq = old_fq; |
360 | |
361 | if (kill_fq(qidev, fq: new_fq)) |
362 | dev_warn(qidev, "New CAAM FQ kill failed\n" ); |
363 | |
364 | return ret; |
365 | } |
366 | |
367 | /* |
368 | * Re-initialise pre-header. Set RSLS and SDLEN. |
369 | * Update the shared descriptor for driver context. |
370 | */ |
371 | drv_ctx->prehdr[0] = cpu_to_caam32(val: (1 << PREHDR_RSLS_SHIFT) | |
372 | num_words); |
373 | drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS); |
374 | memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); |
375 | dma_sync_single_for_device(dev: qidev, addr: drv_ctx->context_a, |
376 | size: sizeof(drv_ctx->sh_desc) + |
377 | sizeof(drv_ctx->prehdr), |
378 | dir: DMA_BIDIRECTIONAL); |
379 | |
380 | /* Put the new FQ in scheduled state */ |
381 | ret = qman_schedule_fq(fq: new_fq); |
382 | if (ret) { |
383 | dev_err(qidev, "Fail to sched new CAAM FQ, ecode = %d\n" , ret); |
384 | |
385 | /* |
386 | * We can kill new FQ and revert to old FQ. |
387 | * Since the desc is already modified, it is success case |
388 | */ |
389 | |
390 | drv_ctx->req_fq = old_fq; |
391 | |
392 | if (kill_fq(qidev, fq: new_fq)) |
393 | dev_warn(qidev, "New CAAM FQ kill failed\n" ); |
394 | } else if (kill_fq(qidev, fq: old_fq)) { |
395 | dev_warn(qidev, "Old CAAM FQ kill failed\n" ); |
396 | } |
397 | |
398 | return 0; |
399 | } |
400 | EXPORT_SYMBOL(caam_drv_ctx_update); |
401 | |
402 | struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, |
403 | int *cpu, |
404 | u32 *sh_desc) |
405 | { |
406 | size_t size; |
407 | u32 num_words; |
408 | dma_addr_t hwdesc; |
409 | struct caam_drv_ctx *drv_ctx; |
410 | const cpumask_t *cpus = qman_affine_cpus(); |
411 | |
412 | num_words = desc_len(desc: sh_desc); |
413 | if (num_words > MAX_SDLEN) { |
414 | dev_err(qidev, "Invalid descriptor len: %d words\n" , |
415 | num_words); |
416 | return ERR_PTR(error: -EINVAL); |
417 | } |
418 | |
419 | drv_ctx = kzalloc(size: sizeof(*drv_ctx), GFP_ATOMIC); |
420 | if (!drv_ctx) |
421 | return ERR_PTR(error: -ENOMEM); |
422 | |
423 | /* |
424 | * Initialise pre-header - set RSLS and SDLEN - and shared descriptor |
425 | * and dma-map them. |
426 | */ |
427 | drv_ctx->prehdr[0] = cpu_to_caam32(val: (1 << PREHDR_RSLS_SHIFT) | |
428 | num_words); |
429 | drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS); |
430 | memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); |
431 | size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc); |
432 | hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size, |
433 | DMA_BIDIRECTIONAL); |
434 | if (dma_mapping_error(dev: qidev, dma_addr: hwdesc)) { |
435 | dev_err(qidev, "DMA map error for preheader + shdesc\n" ); |
436 | kfree(objp: drv_ctx); |
437 | return ERR_PTR(error: -ENOMEM); |
438 | } |
439 | drv_ctx->context_a = hwdesc; |
440 | |
441 | /* If given CPU does not own the portal, choose another one that does */ |
442 | if (!cpumask_test_cpu(cpu: *cpu, cpumask: cpus)) { |
443 | int *pcpu = &get_cpu_var(last_cpu); |
444 | |
445 | *pcpu = cpumask_next(n: *pcpu, srcp: cpus); |
446 | if (*pcpu >= nr_cpu_ids) |
447 | *pcpu = cpumask_first(srcp: cpus); |
448 | *cpu = *pcpu; |
449 | |
450 | put_cpu_var(last_cpu); |
451 | } |
452 | drv_ctx->cpu = *cpu; |
453 | |
454 | /* Find response FQ hooked with this CPU */ |
455 | drv_ctx->rsp_fq = per_cpu(pcpu_qipriv.rsp_fq, drv_ctx->cpu); |
456 | |
457 | /* Attach request FQ */ |
458 | drv_ctx->req_fq = create_caam_req_fq(qidev, rsp_fq: drv_ctx->rsp_fq, hwdesc, |
459 | QMAN_INITFQ_FLAG_SCHED); |
460 | if (IS_ERR(ptr: drv_ctx->req_fq)) { |
461 | dev_err(qidev, "create_caam_req_fq failed\n" ); |
462 | dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL); |
463 | kfree(objp: drv_ctx); |
464 | return ERR_PTR(error: -ENOMEM); |
465 | } |
466 | |
467 | /* init reference counter used to track references to request FQ */ |
468 | refcount_set(r: &drv_ctx->refcnt, n: 1); |
469 | |
470 | drv_ctx->qidev = qidev; |
471 | return drv_ctx; |
472 | } |
473 | EXPORT_SYMBOL(caam_drv_ctx_init); |
474 | |
475 | void *qi_cache_alloc(gfp_t flags) |
476 | { |
477 | return kmem_cache_alloc(cachep: qi_cache, flags); |
478 | } |
479 | EXPORT_SYMBOL(qi_cache_alloc); |
480 | |
481 | void qi_cache_free(void *obj) |
482 | { |
483 | kmem_cache_free(s: qi_cache, objp: obj); |
484 | } |
485 | EXPORT_SYMBOL(qi_cache_free); |
486 | |
487 | static int caam_qi_poll(struct napi_struct *napi, int budget) |
488 | { |
489 | struct caam_napi *np = container_of(napi, struct caam_napi, irqtask); |
490 | |
491 | int cleaned = qman_p_poll_dqrr(p: np->p, limit: budget); |
492 | |
493 | if (cleaned < budget) { |
494 | napi_complete(n: napi); |
495 | qman_p_irqsource_add(p: np->p, QM_PIRQ_DQRI); |
496 | } |
497 | |
498 | return cleaned; |
499 | } |
500 | |
501 | void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx) |
502 | { |
503 | if (IS_ERR_OR_NULL(ptr: drv_ctx)) |
504 | return; |
505 | |
506 | /* Remove request FQ */ |
507 | if (kill_fq(qidev: drv_ctx->qidev, fq: drv_ctx->req_fq)) |
508 | dev_err(drv_ctx->qidev, "Crypto session req FQ kill failed\n" ); |
509 | |
510 | dma_unmap_single(drv_ctx->qidev, drv_ctx->context_a, |
511 | sizeof(drv_ctx->sh_desc) + sizeof(drv_ctx->prehdr), |
512 | DMA_BIDIRECTIONAL); |
513 | kfree(objp: drv_ctx); |
514 | } |
515 | EXPORT_SYMBOL(caam_drv_ctx_rel); |
516 | |
517 | static void caam_qi_shutdown(void *data) |
518 | { |
519 | int i; |
520 | struct device *qidev = data; |
521 | struct caam_qi_priv *priv = &qipriv; |
522 | const cpumask_t *cpus = qman_affine_cpus(); |
523 | |
524 | for_each_cpu(i, cpus) { |
525 | struct napi_struct *irqtask; |
526 | |
527 | irqtask = &per_cpu_ptr(&pcpu_qipriv.caam_napi, i)->irqtask; |
528 | napi_disable(n: irqtask); |
529 | netif_napi_del(napi: irqtask); |
530 | |
531 | if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i))) |
532 | dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n" , i); |
533 | } |
534 | |
535 | qman_delete_cgr_safe(cgr: &priv->cgr); |
536 | qman_release_cgrid(id: priv->cgr.cgrid); |
537 | |
538 | kmem_cache_destroy(s: qi_cache); |
539 | } |
540 | |
541 | static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested) |
542 | { |
543 | caam_congested = congested; |
544 | |
545 | if (congested) { |
546 | caam_debugfs_qi_congested(); |
547 | |
548 | pr_debug_ratelimited("CAAM entered congestion\n" ); |
549 | |
550 | } else { |
551 | pr_debug_ratelimited("CAAM exited congestion\n" ); |
552 | } |
553 | } |
554 | |
555 | static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np, |
556 | bool sched_napi) |
557 | { |
558 | if (sched_napi) { |
559 | /* Disable QMan IRQ source and invoke NAPI */ |
560 | qman_p_irqsource_remove(p, QM_PIRQ_DQRI); |
561 | np->p = p; |
562 | napi_schedule(n: &np->irqtask); |
563 | return 1; |
564 | } |
565 | return 0; |
566 | } |
567 | |
568 | static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, |
569 | struct qman_fq *rsp_fq, |
570 | const struct qm_dqrr_entry *dqrr, |
571 | bool sched_napi) |
572 | { |
573 | struct caam_napi *caam_napi = raw_cpu_ptr(&pcpu_qipriv.caam_napi); |
574 | struct caam_drv_req *drv_req; |
575 | const struct qm_fd *fd; |
576 | struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); |
577 | struct caam_drv_private *priv = dev_get_drvdata(dev: qidev); |
578 | u32 status; |
579 | |
580 | if (caam_qi_napi_schedule(p, np: caam_napi, sched_napi)) |
581 | return qman_cb_dqrr_stop; |
582 | |
583 | fd = &dqrr->fd; |
584 | |
585 | drv_req = caam_iova_to_virt(domain: priv->domain, iova_addr: qm_fd_addr_get64(fd)); |
586 | if (unlikely(!drv_req)) { |
587 | dev_err(qidev, |
588 | "Can't find original request for caam response\n" ); |
589 | return qman_cb_dqrr_consume; |
590 | } |
591 | |
592 | refcount_dec(r: &drv_req->drv_ctx->refcnt); |
593 | |
594 | status = be32_to_cpu(fd->status); |
595 | if (unlikely(status)) { |
596 | u32 ssrc = status & JRSTA_SSRC_MASK; |
597 | u8 err_id = status & JRSTA_CCBERR_ERRID_MASK; |
598 | |
599 | if (ssrc != JRSTA_SSRC_CCB_ERROR || |
600 | err_id != JRSTA_CCBERR_ERRID_ICVCHK) |
601 | dev_err_ratelimited(qidev, |
602 | "Error: %#x in CAAM response FD\n" , |
603 | status); |
604 | } |
605 | |
606 | if (unlikely(qm_fd_get_format(fd) != qm_fd_compound)) { |
607 | dev_err(qidev, "Non-compound FD from CAAM\n" ); |
608 | return qman_cb_dqrr_consume; |
609 | } |
610 | |
611 | dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd), |
612 | sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL); |
613 | |
614 | drv_req->cbk(drv_req, status); |
615 | return qman_cb_dqrr_consume; |
616 | } |
617 | |
618 | static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu) |
619 | { |
620 | struct qm_mcc_initfq opts; |
621 | struct qman_fq *fq; |
622 | int ret; |
623 | |
624 | fq = kzalloc(size: sizeof(*fq), GFP_KERNEL); |
625 | if (!fq) |
626 | return -ENOMEM; |
627 | |
628 | fq->cb.dqrr = caam_rsp_fq_dqrr_cb; |
629 | |
630 | ret = qman_create_fq(fqid: 0, QMAN_FQ_FLAG_NO_ENQUEUE | |
631 | QMAN_FQ_FLAG_DYNAMIC_FQID, fq); |
632 | if (ret) { |
633 | dev_err(qidev, "Rsp FQ create failed\n" ); |
634 | kfree(objp: fq); |
635 | return -ENODEV; |
636 | } |
637 | |
638 | memset(&opts, 0, sizeof(opts)); |
639 | opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ | |
640 | QM_INITFQ_WE_CONTEXTB | |
641 | QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID); |
642 | opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CTXASTASHING | |
643 | QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE); |
644 | qm_fqd_set_destwq(fqd: &opts.fqd, ch: qman_affine_channel(cpu), wq: 3); |
645 | opts.fqd.cgid = qipriv.cgr.cgrid; |
646 | opts.fqd.context_a.stashing.exclusive = QM_STASHING_EXCL_CTX | |
647 | QM_STASHING_EXCL_DATA; |
648 | qm_fqd_set_stashing(fqd: &opts.fqd, as: 0, ds: 1, cs: 1); |
649 | |
650 | ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, opts: &opts); |
651 | if (ret) { |
652 | dev_err(qidev, "Rsp FQ init failed\n" ); |
653 | kfree(objp: fq); |
654 | return -ENODEV; |
655 | } |
656 | |
657 | per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq; |
658 | |
659 | dev_dbg(qidev, "Allocated response FQ %u for CPU %u" , fq->fqid, cpu); |
660 | return 0; |
661 | } |
662 | |
663 | static int init_cgr(struct device *qidev) |
664 | { |
665 | int ret; |
666 | struct qm_mcc_initcgr opts; |
667 | const u64 val = (u64)cpumask_weight(srcp: qman_affine_cpus()) * |
668 | MAX_RSP_FQ_BACKLOG_PER_CPU; |
669 | |
670 | ret = qman_alloc_cgrid(&qipriv.cgr.cgrid); |
671 | if (ret) { |
672 | dev_err(qidev, "CGR alloc failed for rsp FQs: %d\n" , ret); |
673 | return ret; |
674 | } |
675 | |
676 | qipriv.cgr.cb = cgr_cb; |
677 | memset(&opts, 0, sizeof(opts)); |
678 | opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES | |
679 | QM_CGR_WE_MODE); |
680 | opts.cgr.cscn_en = QM_CGR_EN; |
681 | opts.cgr.mode = QMAN_CGR_MODE_FRAME; |
682 | qm_cgr_cs_thres_set64(th: &opts.cgr.cs_thres, val, roundup: 1); |
683 | |
684 | ret = qman_create_cgr(cgr: &qipriv.cgr, QMAN_CGR_FLAG_USE_INIT, opts: &opts); |
685 | if (ret) { |
686 | dev_err(qidev, "Error %d creating CAAM CGRID: %u\n" , ret, |
687 | qipriv.cgr.cgrid); |
688 | return ret; |
689 | } |
690 | |
691 | dev_dbg(qidev, "Congestion threshold set to %llu\n" , val); |
692 | return 0; |
693 | } |
694 | |
695 | static int alloc_rsp_fqs(struct device *qidev) |
696 | { |
697 | int ret, i; |
698 | const cpumask_t *cpus = qman_affine_cpus(); |
699 | |
700 | /*Now create response FQs*/ |
701 | for_each_cpu(i, cpus) { |
702 | ret = alloc_rsp_fq_cpu(qidev, cpu: i); |
703 | if (ret) { |
704 | dev_err(qidev, "CAAM rsp FQ alloc failed, cpu: %u" , i); |
705 | return ret; |
706 | } |
707 | } |
708 | |
709 | return 0; |
710 | } |
711 | |
712 | static void free_rsp_fqs(void) |
713 | { |
714 | int i; |
715 | const cpumask_t *cpus = qman_affine_cpus(); |
716 | |
717 | for_each_cpu(i, cpus) |
718 | kfree(per_cpu(pcpu_qipriv.rsp_fq, i)); |
719 | } |
720 | |
721 | int caam_qi_init(struct platform_device *caam_pdev) |
722 | { |
723 | int err, i; |
724 | struct device *ctrldev = &caam_pdev->dev, *qidev; |
725 | struct caam_drv_private *ctrlpriv; |
726 | const cpumask_t *cpus = qman_affine_cpus(); |
727 | |
728 | ctrlpriv = dev_get_drvdata(dev: ctrldev); |
729 | qidev = ctrldev; |
730 | |
731 | /* Initialize the congestion detection */ |
732 | err = init_cgr(qidev); |
733 | if (err) { |
734 | dev_err(qidev, "CGR initialization failed: %d\n" , err); |
735 | return err; |
736 | } |
737 | |
738 | /* Initialise response FQs */ |
739 | err = alloc_rsp_fqs(qidev); |
740 | if (err) { |
741 | dev_err(qidev, "Can't allocate CAAM response FQs: %d\n" , err); |
742 | free_rsp_fqs(); |
743 | return err; |
744 | } |
745 | |
746 | /* |
747 | * Enable the NAPI contexts on each of the core which has an affine |
748 | * portal. |
749 | */ |
750 | for_each_cpu(i, cpus) { |
751 | struct caam_qi_pcpu_priv *priv = per_cpu_ptr(&pcpu_qipriv, i); |
752 | struct caam_napi *caam_napi = &priv->caam_napi; |
753 | struct napi_struct *irqtask = &caam_napi->irqtask; |
754 | struct net_device *net_dev = &priv->net_dev; |
755 | |
756 | net_dev->dev = *qidev; |
757 | INIT_LIST_HEAD(list: &net_dev->napi_list); |
758 | |
759 | netif_napi_add_tx_weight(dev: net_dev, napi: irqtask, poll: caam_qi_poll, |
760 | CAAM_NAPI_WEIGHT); |
761 | |
762 | napi_enable(n: irqtask); |
763 | } |
764 | |
765 | qi_cache = kmem_cache_create(name: "caamqicache" , CAAM_QI_MEMCACHE_SIZE, |
766 | align: dma_get_cache_alignment(), flags: 0, NULL); |
767 | if (!qi_cache) { |
768 | dev_err(qidev, "Can't allocate CAAM cache\n" ); |
769 | free_rsp_fqs(); |
770 | return -ENOMEM; |
771 | } |
772 | |
773 | caam_debugfs_qi_init(ctrlpriv); |
774 | |
775 | err = devm_add_action_or_reset(qidev, caam_qi_shutdown, ctrlpriv); |
776 | if (err) |
777 | return err; |
778 | |
779 | dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n" ); |
780 | return 0; |
781 | } |
782 | |