1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * AMD Cryptographic Coprocessor (CCP) driver |
4 | * |
5 | * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. |
6 | * |
7 | * Author: Tom Lendacky <thomas.lendacky@amd.com> |
8 | * Author: Gary R Hook <gary.hook@amd.com> |
9 | */ |
10 | |
11 | #include <linux/module.h> |
12 | #include <linux/kernel.h> |
13 | #include <linux/kthread.h> |
14 | #include <linux/interrupt.h> |
15 | #include <linux/ccp.h> |
16 | |
17 | #include "ccp-dev.h" |
18 | |
19 | static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count) |
20 | { |
21 | int start; |
22 | struct ccp_device *ccp = cmd_q->ccp; |
23 | |
24 | for (;;) { |
25 | mutex_lock(&ccp->sb_mutex); |
26 | |
27 | start = (u32)bitmap_find_next_zero_area(map: ccp->sb, |
28 | size: ccp->sb_count, |
29 | start: ccp->sb_start, |
30 | nr: count, align_mask: 0); |
31 | if (start <= ccp->sb_count) { |
32 | bitmap_set(map: ccp->sb, start, nbits: count); |
33 | |
34 | mutex_unlock(lock: &ccp->sb_mutex); |
35 | break; |
36 | } |
37 | |
38 | ccp->sb_avail = 0; |
39 | |
40 | mutex_unlock(lock: &ccp->sb_mutex); |
41 | |
42 | /* Wait for KSB entries to become available */ |
43 | if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) |
44 | return 0; |
45 | } |
46 | |
47 | return KSB_START + start; |
48 | } |
49 | |
50 | static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start, |
51 | unsigned int count) |
52 | { |
53 | struct ccp_device *ccp = cmd_q->ccp; |
54 | |
55 | if (!start) |
56 | return; |
57 | |
58 | mutex_lock(&ccp->sb_mutex); |
59 | |
60 | bitmap_clear(map: ccp->sb, start: start - KSB_START, nbits: count); |
61 | |
62 | ccp->sb_avail = 1; |
63 | |
64 | mutex_unlock(lock: &ccp->sb_mutex); |
65 | |
66 | wake_up_interruptible_all(&ccp->sb_queue); |
67 | } |
68 | |
69 | static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q) |
70 | { |
71 | return CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); |
72 | } |
73 | |
74 | static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) |
75 | { |
76 | struct ccp_cmd_queue *cmd_q = op->cmd_q; |
77 | struct ccp_device *ccp = cmd_q->ccp; |
78 | void __iomem *cr_addr; |
79 | u32 cr0, cmd; |
80 | unsigned int i; |
81 | int ret = 0; |
82 | |
83 | /* We could read a status register to see how many free slots |
84 | * are actually available, but reading that register resets it |
85 | * and you could lose some error information. |
86 | */ |
87 | cmd_q->free_slots--; |
88 | |
89 | cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) |
90 | | (op->jobid << REQ0_JOBID_SHIFT) |
91 | | REQ0_WAIT_FOR_WRITE; |
92 | |
93 | if (op->soc) |
94 | cr0 |= REQ0_STOP_ON_COMPLETE |
95 | | REQ0_INT_ON_COMPLETE; |
96 | |
97 | if (op->ioc || !cmd_q->free_slots) |
98 | cr0 |= REQ0_INT_ON_COMPLETE; |
99 | |
100 | /* Start at CMD_REQ1 */ |
101 | cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; |
102 | |
103 | mutex_lock(&ccp->req_mutex); |
104 | |
105 | /* Write CMD_REQ1 through CMD_REQx first */ |
106 | for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) |
107 | iowrite32(*(cr + i), cr_addr); |
108 | |
109 | /* Tell the CCP to start */ |
110 | wmb(); |
111 | iowrite32(cr0, ccp->io_regs + CMD_REQ0); |
112 | |
113 | mutex_unlock(lock: &ccp->req_mutex); |
114 | |
115 | if (cr0 & REQ0_INT_ON_COMPLETE) { |
116 | /* Wait for the job to complete */ |
117 | ret = wait_event_interruptible(cmd_q->int_queue, |
118 | cmd_q->int_rcvd); |
119 | if (ret || cmd_q->cmd_error) { |
120 | /* On error delete all related jobs from the queue */ |
121 | cmd = (cmd_q->id << DEL_Q_ID_SHIFT) |
122 | | op->jobid; |
123 | if (cmd_q->cmd_error) |
124 | ccp_log_error(cmd_q->ccp, |
125 | cmd_q->cmd_error); |
126 | |
127 | iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); |
128 | |
129 | if (!ret) |
130 | ret = -EIO; |
131 | } else if (op->soc) { |
132 | /* Delete just head job from the queue on SoC */ |
133 | cmd = DEL_Q_ACTIVE |
134 | | (cmd_q->id << DEL_Q_ID_SHIFT) |
135 | | op->jobid; |
136 | |
137 | iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); |
138 | } |
139 | |
140 | cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); |
141 | |
142 | cmd_q->int_rcvd = 0; |
143 | } |
144 | |
145 | return ret; |
146 | } |
147 | |
148 | static int ccp_perform_aes(struct ccp_op *op) |
149 | { |
150 | u32 cr[6]; |
151 | |
152 | /* Fill out the register contents for REQ1 through REQ6 */ |
153 | cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) |
154 | | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) |
155 | | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) |
156 | | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) |
157 | | (op->sb_key << REQ1_KEY_KSB_SHIFT); |
158 | cr[1] = op->src.u.dma.length - 1; |
159 | cr[2] = ccp_addr_lo(info: &op->src.u.dma); |
160 | cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) |
161 | | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
162 | | ccp_addr_hi(info: &op->src.u.dma); |
163 | cr[4] = ccp_addr_lo(info: &op->dst.u.dma); |
164 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) |
165 | | ccp_addr_hi(info: &op->dst.u.dma); |
166 | |
167 | if (op->u.aes.mode == CCP_AES_MODE_CFB) |
168 | cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); |
169 | |
170 | if (op->eom) |
171 | cr[0] |= REQ1_EOM; |
172 | |
173 | if (op->init) |
174 | cr[0] |= REQ1_INIT; |
175 | |
176 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); |
177 | } |
178 | |
179 | static int ccp_perform_xts_aes(struct ccp_op *op) |
180 | { |
181 | u32 cr[6]; |
182 | |
183 | /* Fill out the register contents for REQ1 through REQ6 */ |
184 | cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) |
185 | | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) |
186 | | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) |
187 | | (op->sb_key << REQ1_KEY_KSB_SHIFT); |
188 | cr[1] = op->src.u.dma.length - 1; |
189 | cr[2] = ccp_addr_lo(info: &op->src.u.dma); |
190 | cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) |
191 | | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
192 | | ccp_addr_hi(info: &op->src.u.dma); |
193 | cr[4] = ccp_addr_lo(info: &op->dst.u.dma); |
194 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) |
195 | | ccp_addr_hi(info: &op->dst.u.dma); |
196 | |
197 | if (op->eom) |
198 | cr[0] |= REQ1_EOM; |
199 | |
200 | if (op->init) |
201 | cr[0] |= REQ1_INIT; |
202 | |
203 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); |
204 | } |
205 | |
206 | static int ccp_perform_sha(struct ccp_op *op) |
207 | { |
208 | u32 cr[6]; |
209 | |
210 | /* Fill out the register contents for REQ1 through REQ6 */ |
211 | cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) |
212 | | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) |
213 | | REQ1_INIT; |
214 | cr[1] = op->src.u.dma.length - 1; |
215 | cr[2] = ccp_addr_lo(info: &op->src.u.dma); |
216 | cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) |
217 | | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
218 | | ccp_addr_hi(info: &op->src.u.dma); |
219 | |
220 | if (op->eom) { |
221 | cr[0] |= REQ1_EOM; |
222 | cr[4] = lower_32_bits(op->u.sha.msg_bits); |
223 | cr[5] = upper_32_bits(op->u.sha.msg_bits); |
224 | } else { |
225 | cr[4] = 0; |
226 | cr[5] = 0; |
227 | } |
228 | |
229 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); |
230 | } |
231 | |
232 | static int ccp_perform_rsa(struct ccp_op *op) |
233 | { |
234 | u32 cr[6]; |
235 | |
236 | /* Fill out the register contents for REQ1 through REQ6 */ |
237 | cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) |
238 | | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) |
239 | | (op->sb_key << REQ1_KEY_KSB_SHIFT) |
240 | | REQ1_EOM; |
241 | cr[1] = op->u.rsa.input_len - 1; |
242 | cr[2] = ccp_addr_lo(info: &op->src.u.dma); |
243 | cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT) |
244 | | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
245 | | ccp_addr_hi(info: &op->src.u.dma); |
246 | cr[4] = ccp_addr_lo(info: &op->dst.u.dma); |
247 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) |
248 | | ccp_addr_hi(info: &op->dst.u.dma); |
249 | |
250 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); |
251 | } |
252 | |
253 | static int ccp_perform_passthru(struct ccp_op *op) |
254 | { |
255 | u32 cr[6]; |
256 | |
257 | /* Fill out the register contents for REQ1 through REQ6 */ |
258 | cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) |
259 | | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) |
260 | | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); |
261 | |
262 | if (op->src.type == CCP_MEMTYPE_SYSTEM) |
263 | cr[1] = op->src.u.dma.length - 1; |
264 | else |
265 | cr[1] = op->dst.u.dma.length - 1; |
266 | |
267 | if (op->src.type == CCP_MEMTYPE_SYSTEM) { |
268 | cr[2] = ccp_addr_lo(info: &op->src.u.dma); |
269 | cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
270 | | ccp_addr_hi(info: &op->src.u.dma); |
271 | |
272 | if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) |
273 | cr[3] |= (op->sb_key << REQ4_KSB_SHIFT); |
274 | } else { |
275 | cr[2] = op->src.u.sb * CCP_SB_BYTES; |
276 | cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT); |
277 | } |
278 | |
279 | if (op->dst.type == CCP_MEMTYPE_SYSTEM) { |
280 | cr[4] = ccp_addr_lo(info: &op->dst.u.dma); |
281 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) |
282 | | ccp_addr_hi(info: &op->dst.u.dma); |
283 | } else { |
284 | cr[4] = op->dst.u.sb * CCP_SB_BYTES; |
285 | cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT); |
286 | } |
287 | |
288 | if (op->eom) |
289 | cr[0] |= REQ1_EOM; |
290 | |
291 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); |
292 | } |
293 | |
294 | static int ccp_perform_ecc(struct ccp_op *op) |
295 | { |
296 | u32 cr[6]; |
297 | |
298 | /* Fill out the register contents for REQ1 through REQ6 */ |
299 | cr[0] = REQ1_ECC_AFFINE_CONVERT |
300 | | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) |
301 | | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) |
302 | | REQ1_EOM; |
303 | cr[1] = op->src.u.dma.length - 1; |
304 | cr[2] = ccp_addr_lo(info: &op->src.u.dma); |
305 | cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) |
306 | | ccp_addr_hi(info: &op->src.u.dma); |
307 | cr[4] = ccp_addr_lo(info: &op->dst.u.dma); |
308 | cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) |
309 | | ccp_addr_hi(info: &op->dst.u.dma); |
310 | |
311 | return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); |
312 | } |
313 | |
314 | static void ccp_disable_queue_interrupts(struct ccp_device *ccp) |
315 | { |
316 | iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); |
317 | } |
318 | |
319 | static void ccp_enable_queue_interrupts(struct ccp_device *ccp) |
320 | { |
321 | iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG); |
322 | } |
323 | |
324 | static void ccp_irq_bh(unsigned long data) |
325 | { |
326 | struct ccp_device *ccp = (struct ccp_device *)data; |
327 | struct ccp_cmd_queue *cmd_q; |
328 | u32 q_int, status; |
329 | unsigned int i; |
330 | |
331 | status = ioread32(ccp->io_regs + IRQ_STATUS_REG); |
332 | |
333 | for (i = 0; i < ccp->cmd_q_count; i++) { |
334 | cmd_q = &ccp->cmd_q[i]; |
335 | |
336 | q_int = status & (cmd_q->int_ok | cmd_q->int_err); |
337 | if (q_int) { |
338 | cmd_q->int_status = status; |
339 | cmd_q->q_status = ioread32(cmd_q->reg_status); |
340 | cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); |
341 | |
342 | /* On error, only save the first error value */ |
343 | if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) |
344 | cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); |
345 | |
346 | cmd_q->int_rcvd = 1; |
347 | |
348 | /* Acknowledge the interrupt and wake the kthread */ |
349 | iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); |
350 | wake_up_interruptible(&cmd_q->int_queue); |
351 | } |
352 | } |
353 | ccp_enable_queue_interrupts(ccp); |
354 | } |
355 | |
356 | static irqreturn_t ccp_irq_handler(int irq, void *data) |
357 | { |
358 | struct ccp_device *ccp = (struct ccp_device *)data; |
359 | |
360 | ccp_disable_queue_interrupts(ccp); |
361 | if (ccp->use_tasklet) |
362 | tasklet_schedule(t: &ccp->irq_tasklet); |
363 | else |
364 | ccp_irq_bh(data: (unsigned long)ccp); |
365 | |
366 | return IRQ_HANDLED; |
367 | } |
368 | |
369 | static int ccp_init(struct ccp_device *ccp) |
370 | { |
371 | struct device *dev = ccp->dev; |
372 | struct ccp_cmd_queue *cmd_q; |
373 | struct dma_pool *dma_pool; |
374 | char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; |
375 | unsigned int qmr, i; |
376 | int ret; |
377 | |
378 | /* Find available queues */ |
379 | ccp->qim = 0; |
380 | qmr = ioread32(ccp->io_regs + Q_MASK_REG); |
381 | for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) { |
382 | if (!(qmr & (1 << i))) |
383 | continue; |
384 | |
385 | /* Allocate a dma pool for this queue */ |
386 | snprintf(buf: dma_pool_name, size: sizeof(dma_pool_name), fmt: "%s_q%d" , |
387 | ccp->name, i); |
388 | dma_pool = dma_pool_create(name: dma_pool_name, dev, |
389 | CCP_DMAPOOL_MAX_SIZE, |
390 | CCP_DMAPOOL_ALIGN, allocation: 0); |
391 | if (!dma_pool) { |
392 | dev_err(dev, "unable to allocate dma pool\n" ); |
393 | ret = -ENOMEM; |
394 | goto e_pool; |
395 | } |
396 | |
397 | cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; |
398 | ccp->cmd_q_count++; |
399 | |
400 | cmd_q->ccp = ccp; |
401 | cmd_q->id = i; |
402 | cmd_q->dma_pool = dma_pool; |
403 | |
404 | /* Reserve 2 KSB regions for the queue */ |
405 | cmd_q->sb_key = KSB_START + ccp->sb_start++; |
406 | cmd_q->sb_ctx = KSB_START + ccp->sb_start++; |
407 | ccp->sb_count -= 2; |
408 | |
409 | /* Preset some register values and masks that are queue |
410 | * number dependent |
411 | */ |
412 | cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE + |
413 | (CMD_Q_STATUS_INCR * i); |
414 | cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE + |
415 | (CMD_Q_STATUS_INCR * i); |
416 | cmd_q->int_ok = 1 << (i * 2); |
417 | cmd_q->int_err = 1 << ((i * 2) + 1); |
418 | |
419 | cmd_q->free_slots = ccp_get_free_slots(cmd_q); |
420 | |
421 | init_waitqueue_head(&cmd_q->int_queue); |
422 | |
423 | /* Build queue interrupt mask (two interrupts per queue) */ |
424 | ccp->qim |= cmd_q->int_ok | cmd_q->int_err; |
425 | |
426 | #ifdef CONFIG_ARM64 |
427 | /* For arm64 set the recommended queue cache settings */ |
428 | iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE + |
429 | (CMD_Q_CACHE_INC * i)); |
430 | #endif |
431 | |
432 | dev_dbg(dev, "queue #%u available\n" , i); |
433 | } |
434 | if (ccp->cmd_q_count == 0) { |
435 | dev_notice(dev, "no command queues available\n" ); |
436 | ret = -EIO; |
437 | goto e_pool; |
438 | } |
439 | dev_notice(dev, "%u command queues available\n" , ccp->cmd_q_count); |
440 | |
441 | /* Disable and clear interrupts until ready */ |
442 | ccp_disable_queue_interrupts(ccp); |
443 | for (i = 0; i < ccp->cmd_q_count; i++) { |
444 | cmd_q = &ccp->cmd_q[i]; |
445 | |
446 | ioread32(cmd_q->reg_int_status); |
447 | ioread32(cmd_q->reg_status); |
448 | } |
449 | iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); |
450 | |
451 | /* Request an irq */ |
452 | ret = sp_request_ccp_irq(sp: ccp->sp, handler: ccp_irq_handler, name: ccp->name, data: ccp); |
453 | if (ret) { |
454 | dev_err(dev, "unable to allocate an IRQ\n" ); |
455 | goto e_pool; |
456 | } |
457 | |
458 | /* Initialize the ISR tasklet? */ |
459 | if (ccp->use_tasklet) |
460 | tasklet_init(t: &ccp->irq_tasklet, func: ccp_irq_bh, |
461 | data: (unsigned long)ccp); |
462 | |
463 | dev_dbg(dev, "Starting threads...\n" ); |
464 | /* Create a kthread for each queue */ |
465 | for (i = 0; i < ccp->cmd_q_count; i++) { |
466 | struct task_struct *kthread; |
467 | |
468 | cmd_q = &ccp->cmd_q[i]; |
469 | |
470 | kthread = kthread_run(ccp_cmd_queue_thread, cmd_q, |
471 | "%s-q%u" , ccp->name, cmd_q->id); |
472 | if (IS_ERR(ptr: kthread)) { |
473 | dev_err(dev, "error creating queue thread (%ld)\n" , |
474 | PTR_ERR(kthread)); |
475 | ret = PTR_ERR(ptr: kthread); |
476 | goto e_kthread; |
477 | } |
478 | |
479 | cmd_q->kthread = kthread; |
480 | } |
481 | |
482 | dev_dbg(dev, "Enabling interrupts...\n" ); |
483 | /* Enable interrupts */ |
484 | ccp_enable_queue_interrupts(ccp); |
485 | |
486 | dev_dbg(dev, "Registering device...\n" ); |
487 | ccp_add_device(ccp); |
488 | |
489 | ret = ccp_register_rng(ccp); |
490 | if (ret) |
491 | goto e_kthread; |
492 | |
493 | /* Register the DMA engine support */ |
494 | ret = ccp_dmaengine_register(ccp); |
495 | if (ret) |
496 | goto e_hwrng; |
497 | |
498 | return 0; |
499 | |
500 | e_hwrng: |
501 | ccp_unregister_rng(ccp); |
502 | |
503 | e_kthread: |
504 | for (i = 0; i < ccp->cmd_q_count; i++) |
505 | if (ccp->cmd_q[i].kthread) |
506 | kthread_stop(k: ccp->cmd_q[i].kthread); |
507 | |
508 | sp_free_ccp_irq(sp: ccp->sp, data: ccp); |
509 | |
510 | e_pool: |
511 | for (i = 0; i < ccp->cmd_q_count; i++) |
512 | dma_pool_destroy(pool: ccp->cmd_q[i].dma_pool); |
513 | |
514 | return ret; |
515 | } |
516 | |
517 | static void ccp_destroy(struct ccp_device *ccp) |
518 | { |
519 | struct ccp_cmd_queue *cmd_q; |
520 | struct ccp_cmd *cmd; |
521 | unsigned int i; |
522 | |
523 | /* Unregister the DMA engine */ |
524 | ccp_dmaengine_unregister(ccp); |
525 | |
526 | /* Unregister the RNG */ |
527 | ccp_unregister_rng(ccp); |
528 | |
529 | /* Remove this device from the list of available units */ |
530 | ccp_del_device(ccp); |
531 | |
532 | /* Disable and clear interrupts */ |
533 | ccp_disable_queue_interrupts(ccp); |
534 | for (i = 0; i < ccp->cmd_q_count; i++) { |
535 | cmd_q = &ccp->cmd_q[i]; |
536 | |
537 | ioread32(cmd_q->reg_int_status); |
538 | ioread32(cmd_q->reg_status); |
539 | } |
540 | iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); |
541 | |
542 | /* Stop the queue kthreads */ |
543 | for (i = 0; i < ccp->cmd_q_count; i++) |
544 | if (ccp->cmd_q[i].kthread) |
545 | kthread_stop(k: ccp->cmd_q[i].kthread); |
546 | |
547 | sp_free_ccp_irq(sp: ccp->sp, data: ccp); |
548 | |
549 | for (i = 0; i < ccp->cmd_q_count; i++) |
550 | dma_pool_destroy(pool: ccp->cmd_q[i].dma_pool); |
551 | |
552 | /* Flush the cmd and backlog queue */ |
553 | while (!list_empty(head: &ccp->cmd)) { |
554 | /* Invoke the callback directly with an error code */ |
555 | cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); |
556 | list_del(entry: &cmd->entry); |
557 | cmd->callback(cmd->data, -ENODEV); |
558 | } |
559 | while (!list_empty(head: &ccp->backlog)) { |
560 | /* Invoke the callback directly with an error code */ |
561 | cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); |
562 | list_del(entry: &cmd->entry); |
563 | cmd->callback(cmd->data, -ENODEV); |
564 | } |
565 | } |
566 | |
567 | static const struct ccp_actions ccp3_actions = { |
568 | .aes = ccp_perform_aes, |
569 | .xts_aes = ccp_perform_xts_aes, |
570 | .des3 = NULL, |
571 | .sha = ccp_perform_sha, |
572 | .rsa = ccp_perform_rsa, |
573 | .passthru = ccp_perform_passthru, |
574 | .ecc = ccp_perform_ecc, |
575 | .sballoc = ccp_alloc_ksb, |
576 | .sbfree = ccp_free_ksb, |
577 | .init = ccp_init, |
578 | .destroy = ccp_destroy, |
579 | .get_free_slots = ccp_get_free_slots, |
580 | .irqhandler = ccp_irq_handler, |
581 | }; |
582 | |
583 | const struct ccp_vdata ccpv3_platform = { |
584 | .version = CCP_VERSION(3, 0), |
585 | .setup = NULL, |
586 | .perform = &ccp3_actions, |
587 | .offset = 0, |
588 | .rsamax = CCP_RSA_MAX_WIDTH, |
589 | }; |
590 | |
591 | const struct ccp_vdata ccpv3 = { |
592 | .version = CCP_VERSION(3, 0), |
593 | .setup = NULL, |
594 | .perform = &ccp3_actions, |
595 | .offset = 0x20000, |
596 | .rsamax = CCP_RSA_MAX_WIDTH, |
597 | }; |
598 | |