1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * AMD Cryptographic Coprocessor (CCP) driver |
4 | * |
5 | * Copyright (C) 2016,2019 Advanced Micro Devices, Inc. |
6 | * |
7 | * Author: Gary R Hook <gary.hook@amd.com> |
8 | */ |
9 | |
10 | #include <linux/kernel.h> |
11 | #include <linux/kthread.h> |
12 | #include <linux/dma-mapping.h> |
13 | #include <linux/interrupt.h> |
14 | #include <linux/compiler.h> |
15 | #include <linux/ccp.h> |
16 | |
17 | #include "ccp-dev.h" |
18 | |
19 | /* Allocate the requested number of contiguous LSB slots |
20 | * from the LSB bitmap. Look in the private range for this |
21 | * queue first; failing that, check the public area. |
22 | * If no space is available, wait around. |
23 | * Return: first slot number |
24 | */ |
25 | static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count) |
26 | { |
27 | struct ccp_device *ccp; |
28 | int start; |
29 | |
30 | /* First look at the map for the queue */ |
31 | if (cmd_q->lsb >= 0) { |
32 | start = (u32)bitmap_find_next_zero_area(map: cmd_q->lsbmap, |
33 | LSB_SIZE, |
34 | start: 0, nr: count, align_mask: 0); |
35 | if (start < LSB_SIZE) { |
36 | bitmap_set(map: cmd_q->lsbmap, start, nbits: count); |
37 | return start + cmd_q->lsb * LSB_SIZE; |
38 | } |
39 | } |
40 | |
41 | /* No joy; try to get an entry from the shared blocks */ |
42 | ccp = cmd_q->ccp; |
43 | for (;;) { |
44 | mutex_lock(&ccp->sb_mutex); |
45 | |
46 | start = (u32)bitmap_find_next_zero_area(map: ccp->lsbmap, |
47 | MAX_LSB_CNT * LSB_SIZE, |
48 | start: 0, |
49 | nr: count, align_mask: 0); |
50 | if (start <= MAX_LSB_CNT * LSB_SIZE) { |
51 | bitmap_set(map: ccp->lsbmap, start, nbits: count); |
52 | |
53 | mutex_unlock(lock: &ccp->sb_mutex); |
54 | return start; |
55 | } |
56 | |
57 | ccp->sb_avail = 0; |
58 | |
59 | mutex_unlock(lock: &ccp->sb_mutex); |
60 | |
61 | /* Wait for KSB entries to become available */ |
62 | if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail)) |
63 | return 0; |
64 | } |
65 | } |
66 | |
67 | /* Free a number of LSB slots from the bitmap, starting at |
68 | * the indicated starting slot number. |
69 | */ |
70 | static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start, |
71 | unsigned int count) |
72 | { |
73 | if (!start) |
74 | return; |
75 | |
76 | if (cmd_q->lsb == start) { |
77 | /* An entry from the private LSB */ |
78 | bitmap_clear(map: cmd_q->lsbmap, start, nbits: count); |
79 | } else { |
80 | /* From the shared LSBs */ |
81 | struct ccp_device *ccp = cmd_q->ccp; |
82 | |
83 | mutex_lock(&ccp->sb_mutex); |
84 | bitmap_clear(map: ccp->lsbmap, start, nbits: count); |
85 | ccp->sb_avail = 1; |
86 | mutex_unlock(lock: &ccp->sb_mutex); |
87 | wake_up_interruptible_all(&ccp->sb_queue); |
88 | } |
89 | } |
90 | |
91 | /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */ |
92 | union ccp_function { |
93 | struct { |
94 | u16 size:7; |
95 | u16 encrypt:1; |
96 | u16 mode:5; |
97 | u16 type:2; |
98 | } aes; |
99 | struct { |
100 | u16 size:7; |
101 | u16 encrypt:1; |
102 | u16 rsvd:5; |
103 | u16 type:2; |
104 | } aes_xts; |
105 | struct { |
106 | u16 size:7; |
107 | u16 encrypt:1; |
108 | u16 mode:5; |
109 | u16 type:2; |
110 | } des3; |
111 | struct { |
112 | u16 rsvd1:10; |
113 | u16 type:4; |
114 | u16 rsvd2:1; |
115 | } sha; |
116 | struct { |
117 | u16 mode:3; |
118 | u16 size:12; |
119 | } rsa; |
120 | struct { |
121 | u16 byteswap:2; |
122 | u16 bitwise:3; |
123 | u16 reflect:2; |
124 | u16 rsvd:8; |
125 | } pt; |
126 | struct { |
127 | u16 rsvd:13; |
128 | } zlib; |
129 | struct { |
130 | u16 size:10; |
131 | u16 type:2; |
132 | u16 mode:3; |
133 | } ecc; |
134 | u16 raw; |
135 | }; |
136 | |
137 | #define CCP_AES_SIZE(p) ((p)->aes.size) |
138 | #define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt) |
139 | #define CCP_AES_MODE(p) ((p)->aes.mode) |
140 | #define CCP_AES_TYPE(p) ((p)->aes.type) |
141 | #define CCP_XTS_SIZE(p) ((p)->aes_xts.size) |
142 | #define CCP_XTS_TYPE(p) ((p)->aes_xts.type) |
143 | #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt) |
144 | #define CCP_DES3_SIZE(p) ((p)->des3.size) |
145 | #define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt) |
146 | #define CCP_DES3_MODE(p) ((p)->des3.mode) |
147 | #define CCP_DES3_TYPE(p) ((p)->des3.type) |
148 | #define CCP_SHA_TYPE(p) ((p)->sha.type) |
149 | #define CCP_RSA_SIZE(p) ((p)->rsa.size) |
150 | #define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap) |
151 | #define CCP_PT_BITWISE(p) ((p)->pt.bitwise) |
152 | #define CCP_ECC_MODE(p) ((p)->ecc.mode) |
153 | #define CCP_ECC_AFFINE(p) ((p)->ecc.one) |
154 | |
155 | /* Word 0 */ |
156 | #define CCP5_CMD_DW0(p) ((p)->dw0) |
157 | #define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc) |
158 | #define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc) |
159 | #define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init) |
160 | #define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom) |
161 | #define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function) |
162 | #define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine) |
163 | #define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot) |
164 | |
165 | /* Word 1 */ |
166 | #define CCP5_CMD_DW1(p) ((p)->length) |
167 | #define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p)) |
168 | |
169 | /* Word 2 */ |
170 | #define CCP5_CMD_DW2(p) ((p)->src_lo) |
171 | #define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p)) |
172 | |
173 | /* Word 3 */ |
174 | #define CCP5_CMD_DW3(p) ((p)->dw3) |
175 | #define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem) |
176 | #define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi) |
177 | #define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id) |
178 | #define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed) |
179 | |
180 | /* Words 4/5 */ |
181 | #define CCP5_CMD_DW4(p) ((p)->dw4) |
182 | #define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo) |
183 | #define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi) |
184 | #define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p)) |
185 | #define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem) |
186 | #define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed) |
187 | #define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo) |
188 | #define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi) |
189 | |
190 | /* Word 6/7 */ |
191 | #define CCP5_CMD_DW6(p) ((p)->key_lo) |
192 | #define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p)) |
193 | #define CCP5_CMD_DW7(p) ((p)->dw7) |
194 | #define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi) |
195 | #define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem) |
196 | |
197 | static inline u32 low_address(unsigned long addr) |
198 | { |
199 | return (u64)addr & 0x0ffffffff; |
200 | } |
201 | |
202 | static inline u32 high_address(unsigned long addr) |
203 | { |
204 | return ((u64)addr >> 32) & 0x00000ffff; |
205 | } |
206 | |
207 | static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q) |
208 | { |
209 | unsigned int head_idx, n; |
210 | u32 head_lo, queue_start; |
211 | |
212 | queue_start = low_address(addr: cmd_q->qdma_tail); |
213 | head_lo = ioread32(cmd_q->reg_head_lo); |
214 | head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc); |
215 | |
216 | n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1; |
217 | |
218 | return n % COMMANDS_PER_QUEUE; /* Always one unused spot */ |
219 | } |
220 | |
221 | static int ccp5_do_cmd(struct ccp5_desc *desc, |
222 | struct ccp_cmd_queue *cmd_q) |
223 | { |
224 | __le32 *mP; |
225 | u32 *dP; |
226 | u32 tail; |
227 | int i; |
228 | int ret = 0; |
229 | |
230 | cmd_q->total_ops++; |
231 | |
232 | if (CCP5_CMD_SOC(desc)) { |
233 | CCP5_CMD_IOC(desc) = 1; |
234 | CCP5_CMD_SOC(desc) = 0; |
235 | } |
236 | mutex_lock(&cmd_q->q_mutex); |
237 | |
238 | mP = (__le32 *)&cmd_q->qbase[cmd_q->qidx]; |
239 | dP = (u32 *)desc; |
240 | for (i = 0; i < 8; i++) |
241 | mP[i] = cpu_to_le32(dP[i]); /* handle endianness */ |
242 | |
243 | cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE; |
244 | |
245 | /* The data used by this command must be flushed to memory */ |
246 | wmb(); |
247 | |
248 | /* Write the new tail address back to the queue register */ |
249 | tail = low_address(addr: cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); |
250 | iowrite32(tail, cmd_q->reg_tail_lo); |
251 | |
252 | /* Turn the queue back on using our cached control register */ |
253 | iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control); |
254 | mutex_unlock(lock: &cmd_q->q_mutex); |
255 | |
256 | if (CCP5_CMD_IOC(desc)) { |
257 | /* Wait for the job to complete */ |
258 | ret = wait_event_interruptible(cmd_q->int_queue, |
259 | cmd_q->int_rcvd); |
260 | if (ret || cmd_q->cmd_error) { |
261 | /* Log the error and flush the queue by |
262 | * moving the head pointer |
263 | */ |
264 | if (cmd_q->cmd_error) |
265 | ccp_log_error(cmd_q->ccp, |
266 | cmd_q->cmd_error); |
267 | iowrite32(tail, cmd_q->reg_head_lo); |
268 | if (!ret) |
269 | ret = -EIO; |
270 | } |
271 | cmd_q->int_rcvd = 0; |
272 | } |
273 | |
274 | return ret; |
275 | } |
276 | |
277 | static int ccp5_perform_aes(struct ccp_op *op) |
278 | { |
279 | struct ccp5_desc desc; |
280 | union ccp_function function; |
281 | u32 key_addr = op->sb_key * LSB_ITEM_SIZE; |
282 | |
283 | op->cmd_q->total_aes_ops++; |
284 | |
285 | /* Zero out all the fields of the command desc */ |
286 | memset(&desc, 0, Q_DESC_SIZE); |
287 | |
288 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES; |
289 | |
290 | CCP5_CMD_SOC(&desc) = op->soc; |
291 | CCP5_CMD_IOC(&desc) = 1; |
292 | CCP5_CMD_INIT(&desc) = op->init; |
293 | CCP5_CMD_EOM(&desc) = op->eom; |
294 | CCP5_CMD_PROT(&desc) = 0; |
295 | |
296 | function.raw = 0; |
297 | CCP_AES_ENCRYPT(&function) = op->u.aes.action; |
298 | CCP_AES_MODE(&function) = op->u.aes.mode; |
299 | CCP_AES_TYPE(&function) = op->u.aes.type; |
300 | CCP_AES_SIZE(&function) = op->u.aes.size; |
301 | |
302 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
303 | |
304 | CCP5_CMD_LEN(&desc) = op->src.u.dma.length; |
305 | |
306 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(info: &op->src.u.dma); |
307 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(info: &op->src.u.dma); |
308 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
309 | |
310 | CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(info: &op->dst.u.dma); |
311 | CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(info: &op->dst.u.dma); |
312 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
313 | |
314 | CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); |
315 | CCP5_CMD_KEY_HI(&desc) = 0; |
316 | CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; |
317 | CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; |
318 | |
319 | return ccp5_do_cmd(desc: &desc, cmd_q: op->cmd_q); |
320 | } |
321 | |
322 | static int ccp5_perform_xts_aes(struct ccp_op *op) |
323 | { |
324 | struct ccp5_desc desc; |
325 | union ccp_function function; |
326 | u32 key_addr = op->sb_key * LSB_ITEM_SIZE; |
327 | |
328 | op->cmd_q->total_xts_aes_ops++; |
329 | |
330 | /* Zero out all the fields of the command desc */ |
331 | memset(&desc, 0, Q_DESC_SIZE); |
332 | |
333 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128; |
334 | |
335 | CCP5_CMD_SOC(&desc) = op->soc; |
336 | CCP5_CMD_IOC(&desc) = 1; |
337 | CCP5_CMD_INIT(&desc) = op->init; |
338 | CCP5_CMD_EOM(&desc) = op->eom; |
339 | CCP5_CMD_PROT(&desc) = 0; |
340 | |
341 | function.raw = 0; |
342 | CCP_XTS_TYPE(&function) = op->u.xts.type; |
343 | CCP_XTS_ENCRYPT(&function) = op->u.xts.action; |
344 | CCP_XTS_SIZE(&function) = op->u.xts.unit_size; |
345 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
346 | |
347 | CCP5_CMD_LEN(&desc) = op->src.u.dma.length; |
348 | |
349 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(info: &op->src.u.dma); |
350 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(info: &op->src.u.dma); |
351 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
352 | |
353 | CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(info: &op->dst.u.dma); |
354 | CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(info: &op->dst.u.dma); |
355 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
356 | |
357 | CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); |
358 | CCP5_CMD_KEY_HI(&desc) = 0; |
359 | CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; |
360 | CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; |
361 | |
362 | return ccp5_do_cmd(desc: &desc, cmd_q: op->cmd_q); |
363 | } |
364 | |
365 | static int ccp5_perform_sha(struct ccp_op *op) |
366 | { |
367 | struct ccp5_desc desc; |
368 | union ccp_function function; |
369 | |
370 | op->cmd_q->total_sha_ops++; |
371 | |
372 | /* Zero out all the fields of the command desc */ |
373 | memset(&desc, 0, Q_DESC_SIZE); |
374 | |
375 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA; |
376 | |
377 | CCP5_CMD_SOC(&desc) = op->soc; |
378 | CCP5_CMD_IOC(&desc) = 1; |
379 | CCP5_CMD_INIT(&desc) = 1; |
380 | CCP5_CMD_EOM(&desc) = op->eom; |
381 | CCP5_CMD_PROT(&desc) = 0; |
382 | |
383 | function.raw = 0; |
384 | CCP_SHA_TYPE(&function) = op->u.sha.type; |
385 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
386 | |
387 | CCP5_CMD_LEN(&desc) = op->src.u.dma.length; |
388 | |
389 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(info: &op->src.u.dma); |
390 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(info: &op->src.u.dma); |
391 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
392 | |
393 | CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; |
394 | |
395 | if (op->eom) { |
396 | CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits); |
397 | CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits); |
398 | } else { |
399 | CCP5_CMD_SHA_LO(&desc) = 0; |
400 | CCP5_CMD_SHA_HI(&desc) = 0; |
401 | } |
402 | |
403 | return ccp5_do_cmd(desc: &desc, cmd_q: op->cmd_q); |
404 | } |
405 | |
406 | static int ccp5_perform_des3(struct ccp_op *op) |
407 | { |
408 | struct ccp5_desc desc; |
409 | union ccp_function function; |
410 | u32 key_addr = op->sb_key * LSB_ITEM_SIZE; |
411 | |
412 | op->cmd_q->total_3des_ops++; |
413 | |
414 | /* Zero out all the fields of the command desc */ |
415 | memset(&desc, 0, sizeof(struct ccp5_desc)); |
416 | |
417 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3; |
418 | |
419 | CCP5_CMD_SOC(&desc) = op->soc; |
420 | CCP5_CMD_IOC(&desc) = 1; |
421 | CCP5_CMD_INIT(&desc) = op->init; |
422 | CCP5_CMD_EOM(&desc) = op->eom; |
423 | CCP5_CMD_PROT(&desc) = 0; |
424 | |
425 | function.raw = 0; |
426 | CCP_DES3_ENCRYPT(&function) = op->u.des3.action; |
427 | CCP_DES3_MODE(&function) = op->u.des3.mode; |
428 | CCP_DES3_TYPE(&function) = op->u.des3.type; |
429 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
430 | |
431 | CCP5_CMD_LEN(&desc) = op->src.u.dma.length; |
432 | |
433 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(info: &op->src.u.dma); |
434 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(info: &op->src.u.dma); |
435 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
436 | |
437 | CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(info: &op->dst.u.dma); |
438 | CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(info: &op->dst.u.dma); |
439 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
440 | |
441 | CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr); |
442 | CCP5_CMD_KEY_HI(&desc) = 0; |
443 | CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; |
444 | CCP5_CMD_LSB_ID(&desc) = op->sb_ctx; |
445 | |
446 | return ccp5_do_cmd(desc: &desc, cmd_q: op->cmd_q); |
447 | } |
448 | |
449 | static int ccp5_perform_rsa(struct ccp_op *op) |
450 | { |
451 | struct ccp5_desc desc; |
452 | union ccp_function function; |
453 | |
454 | op->cmd_q->total_rsa_ops++; |
455 | |
456 | /* Zero out all the fields of the command desc */ |
457 | memset(&desc, 0, Q_DESC_SIZE); |
458 | |
459 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA; |
460 | |
461 | CCP5_CMD_SOC(&desc) = op->soc; |
462 | CCP5_CMD_IOC(&desc) = 1; |
463 | CCP5_CMD_INIT(&desc) = 0; |
464 | CCP5_CMD_EOM(&desc) = 1; |
465 | CCP5_CMD_PROT(&desc) = 0; |
466 | |
467 | function.raw = 0; |
468 | CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3; |
469 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
470 | |
471 | CCP5_CMD_LEN(&desc) = op->u.rsa.input_len; |
472 | |
473 | /* Source is from external memory */ |
474 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(info: &op->src.u.dma); |
475 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(info: &op->src.u.dma); |
476 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
477 | |
478 | /* Destination is in external memory */ |
479 | CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(info: &op->dst.u.dma); |
480 | CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(info: &op->dst.u.dma); |
481 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
482 | |
483 | /* Key (Exponent) is in external memory */ |
484 | CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(info: &op->exp.u.dma); |
485 | CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(info: &op->exp.u.dma); |
486 | CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
487 | |
488 | return ccp5_do_cmd(desc: &desc, cmd_q: op->cmd_q); |
489 | } |
490 | |
491 | static int ccp5_perform_passthru(struct ccp_op *op) |
492 | { |
493 | struct ccp5_desc desc; |
494 | union ccp_function function; |
495 | struct ccp_dma_info *saddr = &op->src.u.dma; |
496 | struct ccp_dma_info *daddr = &op->dst.u.dma; |
497 | |
498 | |
499 | op->cmd_q->total_pt_ops++; |
500 | |
501 | memset(&desc, 0, Q_DESC_SIZE); |
502 | |
503 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU; |
504 | |
505 | CCP5_CMD_SOC(&desc) = 0; |
506 | CCP5_CMD_IOC(&desc) = 1; |
507 | CCP5_CMD_INIT(&desc) = 0; |
508 | CCP5_CMD_EOM(&desc) = op->eom; |
509 | CCP5_CMD_PROT(&desc) = 0; |
510 | |
511 | function.raw = 0; |
512 | CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap; |
513 | CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod; |
514 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
515 | |
516 | /* Length of source data is always 256 bytes */ |
517 | if (op->src.type == CCP_MEMTYPE_SYSTEM) |
518 | CCP5_CMD_LEN(&desc) = saddr->length; |
519 | else |
520 | CCP5_CMD_LEN(&desc) = daddr->length; |
521 | |
522 | if (op->src.type == CCP_MEMTYPE_SYSTEM) { |
523 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(info: &op->src.u.dma); |
524 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(info: &op->src.u.dma); |
525 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
526 | |
527 | if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) |
528 | CCP5_CMD_LSB_ID(&desc) = op->sb_key; |
529 | } else { |
530 | u32 key_addr = op->src.u.sb * CCP_SB_BYTES; |
531 | |
532 | CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr); |
533 | CCP5_CMD_SRC_HI(&desc) = 0; |
534 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB; |
535 | } |
536 | |
537 | if (op->dst.type == CCP_MEMTYPE_SYSTEM) { |
538 | CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(info: &op->dst.u.dma); |
539 | CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(info: &op->dst.u.dma); |
540 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
541 | } else { |
542 | u32 key_addr = op->dst.u.sb * CCP_SB_BYTES; |
543 | |
544 | CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr); |
545 | CCP5_CMD_DST_HI(&desc) = 0; |
546 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB; |
547 | } |
548 | |
549 | return ccp5_do_cmd(desc: &desc, cmd_q: op->cmd_q); |
550 | } |
551 | |
552 | static int ccp5_perform_ecc(struct ccp_op *op) |
553 | { |
554 | struct ccp5_desc desc; |
555 | union ccp_function function; |
556 | |
557 | op->cmd_q->total_ecc_ops++; |
558 | |
559 | /* Zero out all the fields of the command desc */ |
560 | memset(&desc, 0, Q_DESC_SIZE); |
561 | |
562 | CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC; |
563 | |
564 | CCP5_CMD_SOC(&desc) = 0; |
565 | CCP5_CMD_IOC(&desc) = 1; |
566 | CCP5_CMD_INIT(&desc) = 0; |
567 | CCP5_CMD_EOM(&desc) = 1; |
568 | CCP5_CMD_PROT(&desc) = 0; |
569 | |
570 | function.raw = 0; |
571 | function.ecc.mode = op->u.ecc.function; |
572 | CCP5_CMD_FUNCTION(&desc) = function.raw; |
573 | |
574 | CCP5_CMD_LEN(&desc) = op->src.u.dma.length; |
575 | |
576 | CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(info: &op->src.u.dma); |
577 | CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(info: &op->src.u.dma); |
578 | CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
579 | |
580 | CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(info: &op->dst.u.dma); |
581 | CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(info: &op->dst.u.dma); |
582 | CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; |
583 | |
584 | return ccp5_do_cmd(desc: &desc, cmd_q: op->cmd_q); |
585 | } |
586 | |
587 | static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status) |
588 | { |
589 | int q_mask = 1 << cmd_q->id; |
590 | int queues = 0; |
591 | int j; |
592 | |
593 | /* Build a bit mask to know which LSBs this queue has access to. |
594 | * Don't bother with segment 0 as it has special privileges. |
595 | */ |
596 | for (j = 1; j < MAX_LSB_CNT; j++) { |
597 | if (status & q_mask) |
598 | bitmap_set(map: cmd_q->lsbmask, start: j, nbits: 1); |
599 | status >>= LSB_REGION_WIDTH; |
600 | } |
601 | queues = bitmap_weight(src: cmd_q->lsbmask, MAX_LSB_CNT); |
602 | dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n" , |
603 | cmd_q->id, queues); |
604 | |
605 | return queues ? 0 : -EINVAL; |
606 | } |
607 | |
608 | static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp, |
609 | int lsb_cnt, int n_lsbs, |
610 | unsigned long *lsb_pub) |
611 | { |
612 | DECLARE_BITMAP(qlsb, MAX_LSB_CNT); |
613 | int bitno; |
614 | int qlsb_wgt; |
615 | int i; |
616 | |
617 | /* For each queue: |
618 | * If the count of potential LSBs available to a queue matches the |
619 | * ordinal given to us in lsb_cnt: |
620 | * Copy the mask of possible LSBs for this queue into "qlsb"; |
621 | * For each bit in qlsb, see if the corresponding bit in the |
622 | * aggregation mask is set; if so, we have a match. |
623 | * If we have a match, clear the bit in the aggregation to |
624 | * mark it as no longer available. |
625 | * If there is no match, clear the bit in qlsb and keep looking. |
626 | */ |
627 | for (i = 0; i < ccp->cmd_q_count; i++) { |
628 | struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; |
629 | |
630 | qlsb_wgt = bitmap_weight(src: cmd_q->lsbmask, MAX_LSB_CNT); |
631 | |
632 | if (qlsb_wgt == lsb_cnt) { |
633 | bitmap_copy(dst: qlsb, src: cmd_q->lsbmask, MAX_LSB_CNT); |
634 | |
635 | bitno = find_first_bit(addr: qlsb, MAX_LSB_CNT); |
636 | while (bitno < MAX_LSB_CNT) { |
637 | if (test_bit(bitno, lsb_pub)) { |
638 | /* We found an available LSB |
639 | * that this queue can access |
640 | */ |
641 | cmd_q->lsb = bitno; |
642 | bitmap_clear(map: lsb_pub, start: bitno, nbits: 1); |
643 | dev_dbg(ccp->dev, |
644 | "Queue %d gets LSB %d\n" , |
645 | i, bitno); |
646 | break; |
647 | } |
648 | bitmap_clear(map: qlsb, start: bitno, nbits: 1); |
649 | bitno = find_first_bit(addr: qlsb, MAX_LSB_CNT); |
650 | } |
651 | if (bitno >= MAX_LSB_CNT) |
652 | return -EINVAL; |
653 | n_lsbs--; |
654 | } |
655 | } |
656 | return n_lsbs; |
657 | } |
658 | |
659 | /* For each queue, from the most- to least-constrained: |
660 | * find an LSB that can be assigned to the queue. If there are N queues that |
661 | * can only use M LSBs, where N > M, fail; otherwise, every queue will get a |
662 | * dedicated LSB. Remaining LSB regions become a shared resource. |
663 | * If we have fewer LSBs than queues, all LSB regions become shared resources. |
664 | */ |
665 | static int ccp_assign_lsbs(struct ccp_device *ccp) |
666 | { |
667 | DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT); |
668 | DECLARE_BITMAP(qlsb, MAX_LSB_CNT); |
669 | int n_lsbs = 0; |
670 | int bitno; |
671 | int i, lsb_cnt; |
672 | int rc = 0; |
673 | |
674 | bitmap_zero(dst: lsb_pub, MAX_LSB_CNT); |
675 | |
676 | /* Create an aggregate bitmap to get a total count of available LSBs */ |
677 | for (i = 0; i < ccp->cmd_q_count; i++) |
678 | bitmap_or(dst: lsb_pub, |
679 | src1: lsb_pub, src2: ccp->cmd_q[i].lsbmask, |
680 | MAX_LSB_CNT); |
681 | |
682 | n_lsbs = bitmap_weight(src: lsb_pub, MAX_LSB_CNT); |
683 | |
684 | if (n_lsbs >= ccp->cmd_q_count) { |
685 | /* We have enough LSBS to give every queue a private LSB. |
686 | * Brute force search to start with the queues that are more |
687 | * constrained in LSB choice. When an LSB is privately |
688 | * assigned, it is removed from the public mask. |
689 | * This is an ugly N squared algorithm with some optimization. |
690 | */ |
691 | for (lsb_cnt = 1; |
692 | n_lsbs && (lsb_cnt <= MAX_LSB_CNT); |
693 | lsb_cnt++) { |
694 | rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs, |
695 | lsb_pub); |
696 | if (rc < 0) |
697 | return -EINVAL; |
698 | n_lsbs = rc; |
699 | } |
700 | } |
701 | |
702 | rc = 0; |
703 | /* What's left of the LSBs, according to the public mask, now become |
704 | * shared. Any zero bits in the lsb_pub mask represent an LSB region |
705 | * that can't be used as a shared resource, so mark the LSB slots for |
706 | * them as "in use". |
707 | */ |
708 | bitmap_copy(dst: qlsb, src: lsb_pub, MAX_LSB_CNT); |
709 | |
710 | bitno = find_first_zero_bit(addr: qlsb, MAX_LSB_CNT); |
711 | while (bitno < MAX_LSB_CNT) { |
712 | bitmap_set(map: ccp->lsbmap, start: bitno * LSB_SIZE, LSB_SIZE); |
713 | bitmap_set(map: qlsb, start: bitno, nbits: 1); |
714 | bitno = find_first_zero_bit(addr: qlsb, MAX_LSB_CNT); |
715 | } |
716 | |
717 | return rc; |
718 | } |
719 | |
720 | static void ccp5_disable_queue_interrupts(struct ccp_device *ccp) |
721 | { |
722 | unsigned int i; |
723 | |
724 | for (i = 0; i < ccp->cmd_q_count; i++) |
725 | iowrite32(0x0, ccp->cmd_q[i].reg_int_enable); |
726 | } |
727 | |
728 | static void ccp5_enable_queue_interrupts(struct ccp_device *ccp) |
729 | { |
730 | unsigned int i; |
731 | |
732 | for (i = 0; i < ccp->cmd_q_count; i++) |
733 | iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable); |
734 | } |
735 | |
736 | static void ccp5_irq_bh(unsigned long data) |
737 | { |
738 | struct ccp_device *ccp = (struct ccp_device *)data; |
739 | u32 status; |
740 | unsigned int i; |
741 | |
742 | for (i = 0; i < ccp->cmd_q_count; i++) { |
743 | struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i]; |
744 | |
745 | status = ioread32(cmd_q->reg_interrupt_status); |
746 | |
747 | if (status) { |
748 | cmd_q->int_status = status; |
749 | cmd_q->q_status = ioread32(cmd_q->reg_status); |
750 | cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); |
751 | |
752 | /* On error, only save the first error value */ |
753 | if ((status & INT_ERROR) && !cmd_q->cmd_error) |
754 | cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); |
755 | |
756 | cmd_q->int_rcvd = 1; |
757 | |
758 | /* Acknowledge the interrupt and wake the kthread */ |
759 | iowrite32(status, cmd_q->reg_interrupt_status); |
760 | wake_up_interruptible(&cmd_q->int_queue); |
761 | } |
762 | } |
763 | ccp5_enable_queue_interrupts(ccp); |
764 | } |
765 | |
766 | static irqreturn_t ccp5_irq_handler(int irq, void *data) |
767 | { |
768 | struct ccp_device *ccp = (struct ccp_device *)data; |
769 | |
770 | ccp5_disable_queue_interrupts(ccp); |
771 | ccp->total_interrupts++; |
772 | if (ccp->use_tasklet) |
773 | tasklet_schedule(t: &ccp->irq_tasklet); |
774 | else |
775 | ccp5_irq_bh(data: (unsigned long)ccp); |
776 | return IRQ_HANDLED; |
777 | } |
778 | |
779 | static int ccp5_init(struct ccp_device *ccp) |
780 | { |
781 | struct device *dev = ccp->dev; |
782 | struct ccp_cmd_queue *cmd_q; |
783 | struct dma_pool *dma_pool; |
784 | char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; |
785 | unsigned int qmr, i; |
786 | u64 status; |
787 | u32 status_lo, status_hi; |
788 | int ret; |
789 | |
790 | /* Find available queues */ |
791 | qmr = ioread32(ccp->io_regs + Q_MASK_REG); |
792 | /* |
793 | * Check for a access to the registers. If this read returns |
794 | * 0xffffffff, it's likely that the system is running a broken |
795 | * BIOS which disallows access to the device. Stop here and fail |
796 | * the initialization (but not the load, as the PSP could get |
797 | * properly initialized). |
798 | */ |
799 | if (qmr == 0xffffffff) { |
800 | dev_notice(dev, "ccp: unable to access the device: you might be running a broken BIOS.\n" ); |
801 | return 1; |
802 | } |
803 | |
804 | for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) { |
805 | if (!(qmr & (1 << i))) |
806 | continue; |
807 | |
808 | /* Allocate a dma pool for this queue */ |
809 | snprintf(buf: dma_pool_name, size: sizeof(dma_pool_name), fmt: "%s_q%d" , |
810 | ccp->name, i); |
811 | dma_pool = dma_pool_create(name: dma_pool_name, dev, |
812 | CCP_DMAPOOL_MAX_SIZE, |
813 | CCP_DMAPOOL_ALIGN, allocation: 0); |
814 | if (!dma_pool) { |
815 | dev_err(dev, "unable to allocate dma pool\n" ); |
816 | ret = -ENOMEM; |
817 | goto e_pool; |
818 | } |
819 | |
820 | cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; |
821 | ccp->cmd_q_count++; |
822 | |
823 | cmd_q->ccp = ccp; |
824 | cmd_q->id = i; |
825 | cmd_q->dma_pool = dma_pool; |
826 | mutex_init(&cmd_q->q_mutex); |
827 | |
828 | /* Page alignment satisfies our needs for N <= 128 */ |
829 | BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128); |
830 | cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); |
831 | cmd_q->qbase = dmam_alloc_coherent(dev, size: cmd_q->qsize, |
832 | dma_handle: &cmd_q->qbase_dma, |
833 | GFP_KERNEL); |
834 | if (!cmd_q->qbase) { |
835 | dev_err(dev, "unable to allocate command queue\n" ); |
836 | ret = -ENOMEM; |
837 | goto e_pool; |
838 | } |
839 | |
840 | cmd_q->qidx = 0; |
841 | /* Preset some register values and masks that are queue |
842 | * number dependent |
843 | */ |
844 | cmd_q->reg_control = ccp->io_regs + |
845 | CMD5_Q_STATUS_INCR * (i + 1); |
846 | cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE; |
847 | cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE; |
848 | cmd_q->reg_int_enable = cmd_q->reg_control + |
849 | CMD5_Q_INT_ENABLE_BASE; |
850 | cmd_q->reg_interrupt_status = cmd_q->reg_control + |
851 | CMD5_Q_INTERRUPT_STATUS_BASE; |
852 | cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE; |
853 | cmd_q->reg_int_status = cmd_q->reg_control + |
854 | CMD5_Q_INT_STATUS_BASE; |
855 | cmd_q->reg_dma_status = cmd_q->reg_control + |
856 | CMD5_Q_DMA_STATUS_BASE; |
857 | cmd_q->reg_dma_read_status = cmd_q->reg_control + |
858 | CMD5_Q_DMA_READ_STATUS_BASE; |
859 | cmd_q->reg_dma_write_status = cmd_q->reg_control + |
860 | CMD5_Q_DMA_WRITE_STATUS_BASE; |
861 | |
862 | init_waitqueue_head(&cmd_q->int_queue); |
863 | |
864 | dev_dbg(dev, "queue #%u available\n" , i); |
865 | } |
866 | |
867 | if (ccp->cmd_q_count == 0) { |
868 | dev_notice(dev, "no command queues available\n" ); |
869 | ret = 1; |
870 | goto e_pool; |
871 | } |
872 | |
873 | /* Turn off the queues and disable interrupts until ready */ |
874 | ccp5_disable_queue_interrupts(ccp); |
875 | for (i = 0; i < ccp->cmd_q_count; i++) { |
876 | cmd_q = &ccp->cmd_q[i]; |
877 | |
878 | cmd_q->qcontrol = 0; /* Start with nothing */ |
879 | iowrite32(cmd_q->qcontrol, cmd_q->reg_control); |
880 | |
881 | ioread32(cmd_q->reg_int_status); |
882 | ioread32(cmd_q->reg_status); |
883 | |
884 | /* Clear the interrupt status */ |
885 | iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); |
886 | } |
887 | |
888 | dev_dbg(dev, "Requesting an IRQ...\n" ); |
889 | /* Request an irq */ |
890 | ret = sp_request_ccp_irq(sp: ccp->sp, handler: ccp5_irq_handler, name: ccp->name, data: ccp); |
891 | if (ret) { |
892 | dev_err(dev, "unable to allocate an IRQ\n" ); |
893 | goto e_pool; |
894 | } |
895 | /* Initialize the ISR tasklet */ |
896 | if (ccp->use_tasklet) |
897 | tasklet_init(t: &ccp->irq_tasklet, func: ccp5_irq_bh, |
898 | data: (unsigned long)ccp); |
899 | |
900 | dev_dbg(dev, "Loading LSB map...\n" ); |
901 | /* Copy the private LSB mask to the public registers */ |
902 | status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); |
903 | status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); |
904 | iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET); |
905 | iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET); |
906 | status = ((u64)status_hi<<30) | (u64)status_lo; |
907 | |
908 | dev_dbg(dev, "Configuring virtual queues...\n" ); |
909 | /* Configure size of each virtual queue accessible to host */ |
910 | for (i = 0; i < ccp->cmd_q_count; i++) { |
911 | u32 dma_addr_lo; |
912 | u32 dma_addr_hi; |
913 | |
914 | cmd_q = &ccp->cmd_q[i]; |
915 | |
916 | cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT); |
917 | cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT; |
918 | |
919 | cmd_q->qdma_tail = cmd_q->qbase_dma; |
920 | dma_addr_lo = low_address(addr: cmd_q->qdma_tail); |
921 | iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo); |
922 | iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo); |
923 | |
924 | dma_addr_hi = high_address(addr: cmd_q->qdma_tail); |
925 | cmd_q->qcontrol |= (dma_addr_hi << 16); |
926 | iowrite32(cmd_q->qcontrol, cmd_q->reg_control); |
927 | |
928 | /* Find the LSB regions accessible to the queue */ |
929 | ccp_find_lsb_regions(cmd_q, status); |
930 | cmd_q->lsb = -1; /* Unassigned value */ |
931 | } |
932 | |
933 | dev_dbg(dev, "Assigning LSBs...\n" ); |
934 | ret = ccp_assign_lsbs(ccp); |
935 | if (ret) { |
936 | dev_err(dev, "Unable to assign LSBs (%d)\n" , ret); |
937 | goto e_irq; |
938 | } |
939 | |
940 | /* Optimization: pre-allocate LSB slots for each queue */ |
941 | for (i = 0; i < ccp->cmd_q_count; i++) { |
942 | ccp->cmd_q[i].sb_key = ccp_lsb_alloc(cmd_q: &ccp->cmd_q[i], count: 2); |
943 | ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(cmd_q: &ccp->cmd_q[i], count: 2); |
944 | } |
945 | |
946 | dev_dbg(dev, "Starting threads...\n" ); |
947 | /* Create a kthread for each queue */ |
948 | for (i = 0; i < ccp->cmd_q_count; i++) { |
949 | struct task_struct *kthread; |
950 | |
951 | cmd_q = &ccp->cmd_q[i]; |
952 | |
953 | kthread = kthread_run(ccp_cmd_queue_thread, cmd_q, |
954 | "%s-q%u" , ccp->name, cmd_q->id); |
955 | if (IS_ERR(ptr: kthread)) { |
956 | dev_err(dev, "error creating queue thread (%ld)\n" , |
957 | PTR_ERR(kthread)); |
958 | ret = PTR_ERR(ptr: kthread); |
959 | goto e_kthread; |
960 | } |
961 | |
962 | cmd_q->kthread = kthread; |
963 | } |
964 | |
965 | dev_dbg(dev, "Enabling interrupts...\n" ); |
966 | ccp5_enable_queue_interrupts(ccp); |
967 | |
968 | dev_dbg(dev, "Registering device...\n" ); |
969 | /* Put this on the unit list to make it available */ |
970 | ccp_add_device(ccp); |
971 | |
972 | ret = ccp_register_rng(ccp); |
973 | if (ret) |
974 | goto e_kthread; |
975 | |
976 | /* Register the DMA engine support */ |
977 | ret = ccp_dmaengine_register(ccp); |
978 | if (ret) |
979 | goto e_hwrng; |
980 | |
981 | #ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS |
982 | /* Set up debugfs entries */ |
983 | ccp5_debugfs_setup(ccp); |
984 | #endif |
985 | |
986 | return 0; |
987 | |
988 | e_hwrng: |
989 | ccp_unregister_rng(ccp); |
990 | |
991 | e_kthread: |
992 | for (i = 0; i < ccp->cmd_q_count; i++) |
993 | if (ccp->cmd_q[i].kthread) |
994 | kthread_stop(k: ccp->cmd_q[i].kthread); |
995 | |
996 | e_irq: |
997 | sp_free_ccp_irq(sp: ccp->sp, data: ccp); |
998 | |
999 | e_pool: |
1000 | for (i = 0; i < ccp->cmd_q_count; i++) |
1001 | dma_pool_destroy(pool: ccp->cmd_q[i].dma_pool); |
1002 | |
1003 | return ret; |
1004 | } |
1005 | |
1006 | static void ccp5_destroy(struct ccp_device *ccp) |
1007 | { |
1008 | struct ccp_cmd_queue *cmd_q; |
1009 | struct ccp_cmd *cmd; |
1010 | unsigned int i; |
1011 | |
1012 | /* Unregister the DMA engine */ |
1013 | ccp_dmaengine_unregister(ccp); |
1014 | |
1015 | /* Unregister the RNG */ |
1016 | ccp_unregister_rng(ccp); |
1017 | |
1018 | /* Remove this device from the list of available units first */ |
1019 | ccp_del_device(ccp); |
1020 | |
1021 | #ifdef CONFIG_CRYPTO_DEV_CCP_DEBUGFS |
1022 | /* We're in the process of tearing down the entire driver; |
1023 | * when all the devices are gone clean up debugfs |
1024 | */ |
1025 | if (ccp_present()) |
1026 | ccp5_debugfs_destroy(); |
1027 | #endif |
1028 | |
1029 | /* Disable and clear interrupts */ |
1030 | ccp5_disable_queue_interrupts(ccp); |
1031 | for (i = 0; i < ccp->cmd_q_count; i++) { |
1032 | cmd_q = &ccp->cmd_q[i]; |
1033 | |
1034 | /* Turn off the run bit */ |
1035 | iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control); |
1036 | |
1037 | /* Clear the interrupt status */ |
1038 | iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status); |
1039 | ioread32(cmd_q->reg_int_status); |
1040 | ioread32(cmd_q->reg_status); |
1041 | } |
1042 | |
1043 | /* Stop the queue kthreads */ |
1044 | for (i = 0; i < ccp->cmd_q_count; i++) |
1045 | if (ccp->cmd_q[i].kthread) |
1046 | kthread_stop(k: ccp->cmd_q[i].kthread); |
1047 | |
1048 | sp_free_ccp_irq(sp: ccp->sp, data: ccp); |
1049 | |
1050 | /* Flush the cmd and backlog queue */ |
1051 | while (!list_empty(head: &ccp->cmd)) { |
1052 | /* Invoke the callback directly with an error code */ |
1053 | cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); |
1054 | list_del(entry: &cmd->entry); |
1055 | cmd->callback(cmd->data, -ENODEV); |
1056 | } |
1057 | while (!list_empty(head: &ccp->backlog)) { |
1058 | /* Invoke the callback directly with an error code */ |
1059 | cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); |
1060 | list_del(entry: &cmd->entry); |
1061 | cmd->callback(cmd->data, -ENODEV); |
1062 | } |
1063 | } |
1064 | |
1065 | static void ccp5_config(struct ccp_device *ccp) |
1066 | { |
1067 | /* Public side */ |
1068 | iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET); |
1069 | } |
1070 | |
1071 | static void ccp5other_config(struct ccp_device *ccp) |
1072 | { |
1073 | int i; |
1074 | u32 rnd; |
1075 | |
1076 | /* We own all of the queues on the NTB CCP */ |
1077 | |
1078 | iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET); |
1079 | iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET); |
1080 | for (i = 0; i < 12; i++) { |
1081 | rnd = ioread32(ccp->io_regs + TRNG_OUT_REG); |
1082 | iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET); |
1083 | } |
1084 | |
1085 | iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET); |
1086 | iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET); |
1087 | iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET); |
1088 | |
1089 | iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET); |
1090 | iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET); |
1091 | |
1092 | iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET); |
1093 | |
1094 | ccp5_config(ccp); |
1095 | } |
1096 | |
1097 | /* Version 5 adds some function, but is essentially the same as v5 */ |
1098 | static const struct ccp_actions ccp5_actions = { |
1099 | .aes = ccp5_perform_aes, |
1100 | .xts_aes = ccp5_perform_xts_aes, |
1101 | .sha = ccp5_perform_sha, |
1102 | .des3 = ccp5_perform_des3, |
1103 | .rsa = ccp5_perform_rsa, |
1104 | .passthru = ccp5_perform_passthru, |
1105 | .ecc = ccp5_perform_ecc, |
1106 | .sballoc = ccp_lsb_alloc, |
1107 | .sbfree = ccp_lsb_free, |
1108 | .init = ccp5_init, |
1109 | .destroy = ccp5_destroy, |
1110 | .get_free_slots = ccp5_get_free_slots, |
1111 | }; |
1112 | |
1113 | const struct ccp_vdata ccpv5a = { |
1114 | .version = CCP_VERSION(5, 0), |
1115 | .setup = ccp5_config, |
1116 | .perform = &ccp5_actions, |
1117 | .offset = 0x0, |
1118 | .rsamax = CCP5_RSA_MAX_WIDTH, |
1119 | }; |
1120 | |
1121 | const struct ccp_vdata ccpv5b = { |
1122 | .version = CCP_VERSION(5, 0), |
1123 | .dma_chan_attr = DMA_PRIVATE, |
1124 | .setup = ccp5other_config, |
1125 | .perform = &ccp5_actions, |
1126 | .offset = 0x0, |
1127 | .rsamax = CCP5_RSA_MAX_WIDTH, |
1128 | }; |
1129 | |