1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Cryptographic API. |
4 | * |
5 | * Support for SAHARA cryptographic accelerator. |
6 | * |
7 | * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de> |
8 | * Copyright (c) 2013 Vista Silicon S.L. |
9 | * Author: Javier Martin <javier.martin@vista-silicon.com> |
10 | * |
11 | * Based on omap-aes.c and tegra-aes.c |
12 | */ |
13 | |
14 | #include <crypto/aes.h> |
15 | #include <crypto/internal/hash.h> |
16 | #include <crypto/internal/skcipher.h> |
17 | #include <crypto/scatterwalk.h> |
18 | #include <crypto/sha1.h> |
19 | #include <crypto/sha2.h> |
20 | |
21 | #include <linux/clk.h> |
22 | #include <linux/dma-mapping.h> |
23 | #include <linux/interrupt.h> |
24 | #include <linux/io.h> |
25 | #include <linux/irq.h> |
26 | #include <linux/kernel.h> |
27 | #include <linux/kthread.h> |
28 | #include <linux/module.h> |
29 | #include <linux/of.h> |
30 | #include <linux/platform_device.h> |
31 | #include <linux/spinlock.h> |
32 | |
33 | #define SHA_BUFFER_LEN PAGE_SIZE |
34 | #define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE |
35 | |
36 | #define SAHARA_NAME "sahara" |
37 | #define SAHARA_VERSION_3 3 |
38 | #define SAHARA_VERSION_4 4 |
39 | #define SAHARA_TIMEOUT_MS 1000 |
40 | #define SAHARA_MAX_HW_DESC 2 |
41 | #define SAHARA_MAX_HW_LINK 20 |
42 | |
43 | #define FLAGS_MODE_MASK 0x000f |
44 | #define FLAGS_ENCRYPT BIT(0) |
45 | #define FLAGS_CBC BIT(1) |
46 | #define FLAGS_NEW_KEY BIT(3) |
47 | |
48 | #define SAHARA_HDR_BASE 0x00800000 |
49 | #define SAHARA_HDR_SKHA_ALG_AES 0 |
50 | #define SAHARA_HDR_SKHA_OP_ENC (1 << 2) |
51 | #define SAHARA_HDR_SKHA_MODE_ECB (0 << 3) |
52 | #define SAHARA_HDR_SKHA_MODE_CBC (1 << 3) |
53 | #define SAHARA_HDR_FORM_DATA (5 << 16) |
54 | #define SAHARA_HDR_FORM_KEY (8 << 16) |
55 | #define SAHARA_HDR_LLO (1 << 24) |
56 | #define SAHARA_HDR_CHA_SKHA (1 << 28) |
57 | #define SAHARA_HDR_CHA_MDHA (2 << 28) |
58 | #define SAHARA_HDR_PARITY_BIT (1 << 31) |
59 | |
60 | #define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000 |
61 | #define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000 |
62 | #define SAHARA_HDR_MDHA_HASH 0xA0850000 |
63 | #define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000 |
64 | #define SAHARA_HDR_MDHA_ALG_SHA1 0 |
65 | #define SAHARA_HDR_MDHA_ALG_MD5 1 |
66 | #define SAHARA_HDR_MDHA_ALG_SHA256 2 |
67 | #define SAHARA_HDR_MDHA_ALG_SHA224 3 |
68 | #define SAHARA_HDR_MDHA_PDATA (1 << 2) |
69 | #define SAHARA_HDR_MDHA_HMAC (1 << 3) |
70 | #define SAHARA_HDR_MDHA_INIT (1 << 5) |
71 | #define SAHARA_HDR_MDHA_IPAD (1 << 6) |
72 | #define SAHARA_HDR_MDHA_OPAD (1 << 7) |
73 | #define SAHARA_HDR_MDHA_SWAP (1 << 8) |
74 | #define SAHARA_HDR_MDHA_MAC_FULL (1 << 9) |
75 | #define SAHARA_HDR_MDHA_SSL (1 << 10) |
76 | |
77 | /* SAHARA can only process one request at a time */ |
78 | #define SAHARA_QUEUE_LENGTH 1 |
79 | |
80 | #define SAHARA_REG_VERSION 0x00 |
81 | #define SAHARA_REG_DAR 0x04 |
82 | #define SAHARA_REG_CONTROL 0x08 |
83 | #define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24) |
84 | #define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16) |
85 | #define SAHARA_CONTROL_RNG_AUTORSD (1 << 7) |
86 | #define SAHARA_CONTROL_ENABLE_INT (1 << 4) |
87 | #define SAHARA_REG_CMD 0x0C |
88 | #define SAHARA_CMD_RESET (1 << 0) |
89 | #define SAHARA_CMD_CLEAR_INT (1 << 8) |
90 | #define SAHARA_CMD_CLEAR_ERR (1 << 9) |
91 | #define SAHARA_CMD_SINGLE_STEP (1 << 10) |
92 | #define SAHARA_CMD_MODE_BATCH (1 << 16) |
93 | #define SAHARA_CMD_MODE_DEBUG (1 << 18) |
94 | #define SAHARA_REG_STATUS 0x10 |
95 | #define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7) |
96 | #define SAHARA_STATE_IDLE 0 |
97 | #define SAHARA_STATE_BUSY 1 |
98 | #define SAHARA_STATE_ERR 2 |
99 | #define SAHARA_STATE_FAULT 3 |
100 | #define SAHARA_STATE_COMPLETE 4 |
101 | #define SAHARA_STATE_COMP_FLAG (1 << 2) |
102 | #define SAHARA_STATUS_DAR_FULL (1 << 3) |
103 | #define SAHARA_STATUS_ERROR (1 << 4) |
104 | #define SAHARA_STATUS_SECURE (1 << 5) |
105 | #define SAHARA_STATUS_FAIL (1 << 6) |
106 | #define SAHARA_STATUS_INIT (1 << 7) |
107 | #define SAHARA_STATUS_RNG_RESEED (1 << 8) |
108 | #define SAHARA_STATUS_ACTIVE_RNG (1 << 9) |
109 | #define SAHARA_STATUS_ACTIVE_MDHA (1 << 10) |
110 | #define SAHARA_STATUS_ACTIVE_SKHA (1 << 11) |
111 | #define SAHARA_STATUS_MODE_BATCH (1 << 16) |
112 | #define SAHARA_STATUS_MODE_DEDICATED (1 << 17) |
113 | #define SAHARA_STATUS_MODE_DEBUG (1 << 18) |
114 | #define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff) |
115 | #define SAHARA_REG_ERRSTATUS 0x14 |
116 | #define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf) |
117 | #define SAHARA_ERRSOURCE_CHA 14 |
118 | #define SAHARA_ERRSOURCE_DMA 15 |
119 | #define SAHARA_ERRSTATUS_DMA_DIR (1 << 8) |
120 | #define SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3) |
121 | #define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7) |
122 | #define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff) |
123 | #define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3) |
124 | #define SAHARA_REG_FADDR 0x18 |
125 | #define SAHARA_REG_CDAR 0x1C |
126 | #define SAHARA_REG_IDAR 0x20 |
127 | |
128 | struct sahara_hw_desc { |
129 | u32 hdr; |
130 | u32 len1; |
131 | u32 p1; |
132 | u32 len2; |
133 | u32 p2; |
134 | u32 next; |
135 | }; |
136 | |
137 | struct sahara_hw_link { |
138 | u32 len; |
139 | u32 p; |
140 | u32 next; |
141 | }; |
142 | |
143 | struct sahara_ctx { |
144 | unsigned long flags; |
145 | |
146 | /* AES-specific context */ |
147 | int keylen; |
148 | u8 key[AES_KEYSIZE_128]; |
149 | struct crypto_skcipher *fallback; |
150 | }; |
151 | |
152 | struct sahara_aes_reqctx { |
153 | unsigned long mode; |
154 | struct skcipher_request fallback_req; // keep at the end |
155 | }; |
156 | |
157 | /* |
158 | * struct sahara_sha_reqctx - private data per request |
159 | * @buf: holds data for requests smaller than block_size |
160 | * @rembuf: used to prepare one block_size-aligned request |
161 | * @context: hw-specific context for request. Digest is extracted from this |
162 | * @mode: specifies what type of hw-descriptor needs to be built |
163 | * @digest_size: length of digest for this request |
164 | * @context_size: length of hw-context for this request. |
165 | * Always digest_size + 4 |
166 | * @buf_cnt: number of bytes saved in buf |
167 | * @sg_in_idx: number of hw links |
168 | * @in_sg: scatterlist for input data |
169 | * @in_sg_chain: scatterlists for chained input data |
170 | * @total: total number of bytes for transfer |
171 | * @last: is this the last block |
172 | * @first: is this the first block |
173 | * @active: inside a transfer |
174 | */ |
175 | struct sahara_sha_reqctx { |
176 | u8 buf[SAHARA_MAX_SHA_BLOCK_SIZE]; |
177 | u8 rembuf[SAHARA_MAX_SHA_BLOCK_SIZE]; |
178 | u8 context[SHA256_DIGEST_SIZE + 4]; |
179 | unsigned int mode; |
180 | unsigned int digest_size; |
181 | unsigned int context_size; |
182 | unsigned int buf_cnt; |
183 | unsigned int sg_in_idx; |
184 | struct scatterlist *in_sg; |
185 | struct scatterlist in_sg_chain[2]; |
186 | size_t total; |
187 | unsigned int last; |
188 | unsigned int first; |
189 | unsigned int active; |
190 | }; |
191 | |
192 | struct sahara_dev { |
193 | struct device *device; |
194 | unsigned int version; |
195 | void __iomem *regs_base; |
196 | struct clk *clk_ipg; |
197 | struct clk *clk_ahb; |
198 | spinlock_t queue_spinlock; |
199 | struct task_struct *kthread; |
200 | struct completion dma_completion; |
201 | |
202 | struct sahara_ctx *ctx; |
203 | struct crypto_queue queue; |
204 | unsigned long flags; |
205 | |
206 | struct sahara_hw_desc *hw_desc[SAHARA_MAX_HW_DESC]; |
207 | dma_addr_t hw_phys_desc[SAHARA_MAX_HW_DESC]; |
208 | |
209 | u8 *key_base; |
210 | dma_addr_t key_phys_base; |
211 | |
212 | u8 *iv_base; |
213 | dma_addr_t iv_phys_base; |
214 | |
215 | u8 *context_base; |
216 | dma_addr_t context_phys_base; |
217 | |
218 | struct sahara_hw_link *hw_link[SAHARA_MAX_HW_LINK]; |
219 | dma_addr_t hw_phys_link[SAHARA_MAX_HW_LINK]; |
220 | |
221 | size_t total; |
222 | struct scatterlist *in_sg; |
223 | int nb_in_sg; |
224 | struct scatterlist *out_sg; |
225 | int nb_out_sg; |
226 | |
227 | u32 error; |
228 | }; |
229 | |
230 | static struct sahara_dev *dev_ptr; |
231 | |
232 | static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg) |
233 | { |
234 | writel(val: data, addr: dev->regs_base + reg); |
235 | } |
236 | |
237 | static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg) |
238 | { |
239 | return readl(addr: dev->regs_base + reg); |
240 | } |
241 | |
242 | static u32 sahara_aes_key_hdr(struct sahara_dev *dev) |
243 | { |
244 | u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES | |
245 | SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO | |
246 | SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT; |
247 | |
248 | if (dev->flags & FLAGS_CBC) { |
249 | hdr |= SAHARA_HDR_SKHA_MODE_CBC; |
250 | hdr ^= SAHARA_HDR_PARITY_BIT; |
251 | } |
252 | |
253 | if (dev->flags & FLAGS_ENCRYPT) { |
254 | hdr |= SAHARA_HDR_SKHA_OP_ENC; |
255 | hdr ^= SAHARA_HDR_PARITY_BIT; |
256 | } |
257 | |
258 | return hdr; |
259 | } |
260 | |
261 | static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev) |
262 | { |
263 | return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA | |
264 | SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT; |
265 | } |
266 | |
267 | static const char *sahara_err_src[16] = { |
268 | "No error" , |
269 | "Header error" , |
270 | "Descriptor length error" , |
271 | "Descriptor length or pointer error" , |
272 | "Link length error" , |
273 | "Link pointer error" , |
274 | "Input buffer error" , |
275 | "Output buffer error" , |
276 | "Output buffer starvation" , |
277 | "Internal state fault" , |
278 | "General descriptor problem" , |
279 | "Reserved" , |
280 | "Descriptor address error" , |
281 | "Link address error" , |
282 | "CHA error" , |
283 | "DMA error" |
284 | }; |
285 | |
286 | static const char *sahara_err_dmasize[4] = { |
287 | "Byte transfer" , |
288 | "Half-word transfer" , |
289 | "Word transfer" , |
290 | "Reserved" |
291 | }; |
292 | |
293 | static const char *sahara_err_dmasrc[8] = { |
294 | "No error" , |
295 | "AHB bus error" , |
296 | "Internal IP bus error" , |
297 | "Parity error" , |
298 | "DMA crosses 256 byte boundary" , |
299 | "DMA is busy" , |
300 | "Reserved" , |
301 | "DMA HW error" |
302 | }; |
303 | |
304 | static const char *sahara_cha_errsrc[12] = { |
305 | "Input buffer non-empty" , |
306 | "Illegal address" , |
307 | "Illegal mode" , |
308 | "Illegal data size" , |
309 | "Illegal key size" , |
310 | "Write during processing" , |
311 | "CTX read during processing" , |
312 | "HW error" , |
313 | "Input buffer disabled/underflow" , |
314 | "Output buffer disabled/overflow" , |
315 | "DES key parity error" , |
316 | "Reserved" |
317 | }; |
318 | |
319 | static const char *sahara_cha_err[4] = { "No error" , "SKHA" , "MDHA" , "RNG" }; |
320 | |
321 | static void sahara_decode_error(struct sahara_dev *dev, unsigned int error) |
322 | { |
323 | u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error); |
324 | u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error)); |
325 | |
326 | dev_err(dev->device, "%s: Error Register = 0x%08x\n" , __func__, error); |
327 | |
328 | dev_err(dev->device, " - %s.\n" , sahara_err_src[source]); |
329 | |
330 | if (source == SAHARA_ERRSOURCE_DMA) { |
331 | if (error & SAHARA_ERRSTATUS_DMA_DIR) |
332 | dev_err(dev->device, " * DMA read.\n" ); |
333 | else |
334 | dev_err(dev->device, " * DMA write.\n" ); |
335 | |
336 | dev_err(dev->device, " * %s.\n" , |
337 | sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]); |
338 | dev_err(dev->device, " * %s.\n" , |
339 | sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]); |
340 | } else if (source == SAHARA_ERRSOURCE_CHA) { |
341 | dev_err(dev->device, " * %s.\n" , |
342 | sahara_cha_errsrc[chasrc]); |
343 | dev_err(dev->device, " * %s.\n" , |
344 | sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]); |
345 | } |
346 | dev_err(dev->device, "\n" ); |
347 | } |
348 | |
349 | static const char *sahara_state[4] = { "Idle" , "Busy" , "Error" , "HW Fault" }; |
350 | |
351 | static void sahara_decode_status(struct sahara_dev *dev, unsigned int status) |
352 | { |
353 | u8 state; |
354 | |
355 | if (!__is_defined(DEBUG)) |
356 | return; |
357 | |
358 | state = SAHARA_STATUS_GET_STATE(status); |
359 | |
360 | dev_dbg(dev->device, "%s: Status Register = 0x%08x\n" , |
361 | __func__, status); |
362 | |
363 | dev_dbg(dev->device, " - State = %d:\n" , state); |
364 | if (state & SAHARA_STATE_COMP_FLAG) |
365 | dev_dbg(dev->device, " * Descriptor completed. IRQ pending.\n" ); |
366 | |
367 | dev_dbg(dev->device, " * %s.\n" , |
368 | sahara_state[state & ~SAHARA_STATE_COMP_FLAG]); |
369 | |
370 | if (status & SAHARA_STATUS_DAR_FULL) |
371 | dev_dbg(dev->device, " - DAR Full.\n" ); |
372 | if (status & SAHARA_STATUS_ERROR) |
373 | dev_dbg(dev->device, " - Error.\n" ); |
374 | if (status & SAHARA_STATUS_SECURE) |
375 | dev_dbg(dev->device, " - Secure.\n" ); |
376 | if (status & SAHARA_STATUS_FAIL) |
377 | dev_dbg(dev->device, " - Fail.\n" ); |
378 | if (status & SAHARA_STATUS_RNG_RESEED) |
379 | dev_dbg(dev->device, " - RNG Reseed Request.\n" ); |
380 | if (status & SAHARA_STATUS_ACTIVE_RNG) |
381 | dev_dbg(dev->device, " - RNG Active.\n" ); |
382 | if (status & SAHARA_STATUS_ACTIVE_MDHA) |
383 | dev_dbg(dev->device, " - MDHA Active.\n" ); |
384 | if (status & SAHARA_STATUS_ACTIVE_SKHA) |
385 | dev_dbg(dev->device, " - SKHA Active.\n" ); |
386 | |
387 | if (status & SAHARA_STATUS_MODE_BATCH) |
388 | dev_dbg(dev->device, " - Batch Mode.\n" ); |
389 | else if (status & SAHARA_STATUS_MODE_DEDICATED) |
390 | dev_dbg(dev->device, " - Dedicated Mode.\n" ); |
391 | else if (status & SAHARA_STATUS_MODE_DEBUG) |
392 | dev_dbg(dev->device, " - Debug Mode.\n" ); |
393 | |
394 | dev_dbg(dev->device, " - Internal state = 0x%02x\n" , |
395 | SAHARA_STATUS_GET_ISTATE(status)); |
396 | |
397 | dev_dbg(dev->device, "Current DAR: 0x%08x\n" , |
398 | sahara_read(dev, SAHARA_REG_CDAR)); |
399 | dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n" , |
400 | sahara_read(dev, SAHARA_REG_IDAR)); |
401 | } |
402 | |
403 | static void sahara_dump_descriptors(struct sahara_dev *dev) |
404 | { |
405 | int i; |
406 | |
407 | if (!__is_defined(DEBUG)) |
408 | return; |
409 | |
410 | for (i = 0; i < SAHARA_MAX_HW_DESC; i++) { |
411 | dev_dbg(dev->device, "Descriptor (%d) (%pad):\n" , |
412 | i, &dev->hw_phys_desc[i]); |
413 | dev_dbg(dev->device, "\thdr = 0x%08x\n" , dev->hw_desc[i]->hdr); |
414 | dev_dbg(dev->device, "\tlen1 = %u\n" , dev->hw_desc[i]->len1); |
415 | dev_dbg(dev->device, "\tp1 = 0x%08x\n" , dev->hw_desc[i]->p1); |
416 | dev_dbg(dev->device, "\tlen2 = %u\n" , dev->hw_desc[i]->len2); |
417 | dev_dbg(dev->device, "\tp2 = 0x%08x\n" , dev->hw_desc[i]->p2); |
418 | dev_dbg(dev->device, "\tnext = 0x%08x\n" , |
419 | dev->hw_desc[i]->next); |
420 | } |
421 | dev_dbg(dev->device, "\n" ); |
422 | } |
423 | |
424 | static void sahara_dump_links(struct sahara_dev *dev) |
425 | { |
426 | int i; |
427 | |
428 | if (!__is_defined(DEBUG)) |
429 | return; |
430 | |
431 | for (i = 0; i < SAHARA_MAX_HW_LINK; i++) { |
432 | dev_dbg(dev->device, "Link (%d) (%pad):\n" , |
433 | i, &dev->hw_phys_link[i]); |
434 | dev_dbg(dev->device, "\tlen = %u\n" , dev->hw_link[i]->len); |
435 | dev_dbg(dev->device, "\tp = 0x%08x\n" , dev->hw_link[i]->p); |
436 | dev_dbg(dev->device, "\tnext = 0x%08x\n" , |
437 | dev->hw_link[i]->next); |
438 | } |
439 | dev_dbg(dev->device, "\n" ); |
440 | } |
441 | |
442 | static int sahara_hw_descriptor_create(struct sahara_dev *dev) |
443 | { |
444 | struct sahara_ctx *ctx = dev->ctx; |
445 | struct scatterlist *sg; |
446 | int ret; |
447 | int i, j; |
448 | int idx = 0; |
449 | |
450 | /* Copy new key if necessary */ |
451 | if (ctx->flags & FLAGS_NEW_KEY) { |
452 | memcpy(dev->key_base, ctx->key, ctx->keylen); |
453 | ctx->flags &= ~FLAGS_NEW_KEY; |
454 | |
455 | if (dev->flags & FLAGS_CBC) { |
456 | dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE; |
457 | dev->hw_desc[idx]->p1 = dev->iv_phys_base; |
458 | } else { |
459 | dev->hw_desc[idx]->len1 = 0; |
460 | dev->hw_desc[idx]->p1 = 0; |
461 | } |
462 | dev->hw_desc[idx]->len2 = ctx->keylen; |
463 | dev->hw_desc[idx]->p2 = dev->key_phys_base; |
464 | dev->hw_desc[idx]->next = dev->hw_phys_desc[1]; |
465 | |
466 | dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev); |
467 | |
468 | idx++; |
469 | } |
470 | |
471 | dev->nb_in_sg = sg_nents_for_len(sg: dev->in_sg, len: dev->total); |
472 | if (dev->nb_in_sg < 0) { |
473 | dev_err(dev->device, "Invalid numbers of src SG.\n" ); |
474 | return dev->nb_in_sg; |
475 | } |
476 | dev->nb_out_sg = sg_nents_for_len(sg: dev->out_sg, len: dev->total); |
477 | if (dev->nb_out_sg < 0) { |
478 | dev_err(dev->device, "Invalid numbers of dst SG.\n" ); |
479 | return dev->nb_out_sg; |
480 | } |
481 | if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) { |
482 | dev_err(dev->device, "not enough hw links (%d)\n" , |
483 | dev->nb_in_sg + dev->nb_out_sg); |
484 | return -EINVAL; |
485 | } |
486 | |
487 | ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, |
488 | DMA_TO_DEVICE); |
489 | if (!ret) { |
490 | dev_err(dev->device, "couldn't map in sg\n" ); |
491 | goto unmap_in; |
492 | } |
493 | ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg, |
494 | DMA_FROM_DEVICE); |
495 | if (!ret) { |
496 | dev_err(dev->device, "couldn't map out sg\n" ); |
497 | goto unmap_out; |
498 | } |
499 | |
500 | /* Create input links */ |
501 | dev->hw_desc[idx]->p1 = dev->hw_phys_link[0]; |
502 | sg = dev->in_sg; |
503 | for (i = 0; i < dev->nb_in_sg; i++) { |
504 | dev->hw_link[i]->len = sg->length; |
505 | dev->hw_link[i]->p = sg->dma_address; |
506 | if (i == (dev->nb_in_sg - 1)) { |
507 | dev->hw_link[i]->next = 0; |
508 | } else { |
509 | dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; |
510 | sg = sg_next(sg); |
511 | } |
512 | } |
513 | |
514 | /* Create output links */ |
515 | dev->hw_desc[idx]->p2 = dev->hw_phys_link[i]; |
516 | sg = dev->out_sg; |
517 | for (j = i; j < dev->nb_out_sg + i; j++) { |
518 | dev->hw_link[j]->len = sg->length; |
519 | dev->hw_link[j]->p = sg->dma_address; |
520 | if (j == (dev->nb_out_sg + i - 1)) { |
521 | dev->hw_link[j]->next = 0; |
522 | } else { |
523 | dev->hw_link[j]->next = dev->hw_phys_link[j + 1]; |
524 | sg = sg_next(sg); |
525 | } |
526 | } |
527 | |
528 | /* Fill remaining fields of hw_desc[1] */ |
529 | dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev); |
530 | dev->hw_desc[idx]->len1 = dev->total; |
531 | dev->hw_desc[idx]->len2 = dev->total; |
532 | dev->hw_desc[idx]->next = 0; |
533 | |
534 | sahara_dump_descriptors(dev); |
535 | sahara_dump_links(dev); |
536 | |
537 | sahara_write(dev, data: dev->hw_phys_desc[0], SAHARA_REG_DAR); |
538 | |
539 | return 0; |
540 | |
541 | unmap_out: |
542 | dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, |
543 | DMA_FROM_DEVICE); |
544 | unmap_in: |
545 | dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, |
546 | DMA_TO_DEVICE); |
547 | |
548 | return -EINVAL; |
549 | } |
550 | |
551 | static int sahara_aes_process(struct skcipher_request *req) |
552 | { |
553 | struct sahara_dev *dev = dev_ptr; |
554 | struct sahara_ctx *ctx; |
555 | struct sahara_aes_reqctx *rctx; |
556 | int ret; |
557 | unsigned long timeout; |
558 | |
559 | /* Request is ready to be dispatched by the device */ |
560 | dev_dbg(dev->device, |
561 | "dispatch request (nbytes=%d, src=%p, dst=%p)\n" , |
562 | req->cryptlen, req->src, req->dst); |
563 | |
564 | /* assign new request to device */ |
565 | dev->total = req->cryptlen; |
566 | dev->in_sg = req->src; |
567 | dev->out_sg = req->dst; |
568 | |
569 | rctx = skcipher_request_ctx(req); |
570 | ctx = crypto_skcipher_ctx(tfm: crypto_skcipher_reqtfm(req)); |
571 | rctx->mode &= FLAGS_MODE_MASK; |
572 | dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode; |
573 | |
574 | if ((dev->flags & FLAGS_CBC) && req->iv) |
575 | memcpy(dev->iv_base, req->iv, AES_KEYSIZE_128); |
576 | |
577 | /* assign new context to device */ |
578 | dev->ctx = ctx; |
579 | |
580 | reinit_completion(x: &dev->dma_completion); |
581 | |
582 | ret = sahara_hw_descriptor_create(dev); |
583 | if (ret) |
584 | return -EINVAL; |
585 | |
586 | timeout = wait_for_completion_timeout(x: &dev->dma_completion, |
587 | timeout: msecs_to_jiffies(SAHARA_TIMEOUT_MS)); |
588 | if (!timeout) { |
589 | dev_err(dev->device, "AES timeout\n" ); |
590 | return -ETIMEDOUT; |
591 | } |
592 | |
593 | dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, |
594 | DMA_FROM_DEVICE); |
595 | dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, |
596 | DMA_TO_DEVICE); |
597 | |
598 | return 0; |
599 | } |
600 | |
601 | static int sahara_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, |
602 | unsigned int keylen) |
603 | { |
604 | struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm); |
605 | |
606 | ctx->keylen = keylen; |
607 | |
608 | /* SAHARA only supports 128bit keys */ |
609 | if (keylen == AES_KEYSIZE_128) { |
610 | memcpy(ctx->key, key, keylen); |
611 | ctx->flags |= FLAGS_NEW_KEY; |
612 | return 0; |
613 | } |
614 | |
615 | if (keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) |
616 | return -EINVAL; |
617 | |
618 | /* |
619 | * The requested key size is not supported by HW, do a fallback. |
620 | */ |
621 | crypto_skcipher_clear_flags(tfm: ctx->fallback, CRYPTO_TFM_REQ_MASK); |
622 | crypto_skcipher_set_flags(tfm: ctx->fallback, flags: tfm->base.crt_flags & |
623 | CRYPTO_TFM_REQ_MASK); |
624 | return crypto_skcipher_setkey(tfm: ctx->fallback, key, keylen); |
625 | } |
626 | |
627 | static int sahara_aes_crypt(struct skcipher_request *req, unsigned long mode) |
628 | { |
629 | struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); |
630 | struct sahara_dev *dev = dev_ptr; |
631 | int err = 0; |
632 | |
633 | dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n" , |
634 | req->cryptlen, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC)); |
635 | |
636 | if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) { |
637 | dev_err(dev->device, |
638 | "request size is not exact amount of AES blocks\n" ); |
639 | return -EINVAL; |
640 | } |
641 | |
642 | rctx->mode = mode; |
643 | |
644 | spin_lock_bh(lock: &dev->queue_spinlock); |
645 | err = crypto_enqueue_request(queue: &dev->queue, request: &req->base); |
646 | spin_unlock_bh(lock: &dev->queue_spinlock); |
647 | |
648 | wake_up_process(tsk: dev->kthread); |
649 | |
650 | return err; |
651 | } |
652 | |
653 | static int sahara_aes_ecb_encrypt(struct skcipher_request *req) |
654 | { |
655 | struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); |
656 | struct sahara_ctx *ctx = crypto_skcipher_ctx( |
657 | tfm: crypto_skcipher_reqtfm(req)); |
658 | |
659 | if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { |
660 | skcipher_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback); |
661 | skcipher_request_set_callback(req: &rctx->fallback_req, |
662 | flags: req->base.flags, |
663 | compl: req->base.complete, |
664 | data: req->base.data); |
665 | skcipher_request_set_crypt(req: &rctx->fallback_req, src: req->src, |
666 | dst: req->dst, cryptlen: req->cryptlen, iv: req->iv); |
667 | return crypto_skcipher_encrypt(req: &rctx->fallback_req); |
668 | } |
669 | |
670 | return sahara_aes_crypt(req, FLAGS_ENCRYPT); |
671 | } |
672 | |
673 | static int sahara_aes_ecb_decrypt(struct skcipher_request *req) |
674 | { |
675 | struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); |
676 | struct sahara_ctx *ctx = crypto_skcipher_ctx( |
677 | tfm: crypto_skcipher_reqtfm(req)); |
678 | |
679 | if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { |
680 | skcipher_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback); |
681 | skcipher_request_set_callback(req: &rctx->fallback_req, |
682 | flags: req->base.flags, |
683 | compl: req->base.complete, |
684 | data: req->base.data); |
685 | skcipher_request_set_crypt(req: &rctx->fallback_req, src: req->src, |
686 | dst: req->dst, cryptlen: req->cryptlen, iv: req->iv); |
687 | return crypto_skcipher_decrypt(req: &rctx->fallback_req); |
688 | } |
689 | |
690 | return sahara_aes_crypt(req, mode: 0); |
691 | } |
692 | |
693 | static int sahara_aes_cbc_encrypt(struct skcipher_request *req) |
694 | { |
695 | struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); |
696 | struct sahara_ctx *ctx = crypto_skcipher_ctx( |
697 | tfm: crypto_skcipher_reqtfm(req)); |
698 | |
699 | if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { |
700 | skcipher_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback); |
701 | skcipher_request_set_callback(req: &rctx->fallback_req, |
702 | flags: req->base.flags, |
703 | compl: req->base.complete, |
704 | data: req->base.data); |
705 | skcipher_request_set_crypt(req: &rctx->fallback_req, src: req->src, |
706 | dst: req->dst, cryptlen: req->cryptlen, iv: req->iv); |
707 | return crypto_skcipher_encrypt(req: &rctx->fallback_req); |
708 | } |
709 | |
710 | return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC); |
711 | } |
712 | |
713 | static int sahara_aes_cbc_decrypt(struct skcipher_request *req) |
714 | { |
715 | struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); |
716 | struct sahara_ctx *ctx = crypto_skcipher_ctx( |
717 | tfm: crypto_skcipher_reqtfm(req)); |
718 | |
719 | if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { |
720 | skcipher_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback); |
721 | skcipher_request_set_callback(req: &rctx->fallback_req, |
722 | flags: req->base.flags, |
723 | compl: req->base.complete, |
724 | data: req->base.data); |
725 | skcipher_request_set_crypt(req: &rctx->fallback_req, src: req->src, |
726 | dst: req->dst, cryptlen: req->cryptlen, iv: req->iv); |
727 | return crypto_skcipher_decrypt(req: &rctx->fallback_req); |
728 | } |
729 | |
730 | return sahara_aes_crypt(req, FLAGS_CBC); |
731 | } |
732 | |
733 | static int sahara_aes_init_tfm(struct crypto_skcipher *tfm) |
734 | { |
735 | const char *name = crypto_tfm_alg_name(tfm: &tfm->base); |
736 | struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm); |
737 | |
738 | ctx->fallback = crypto_alloc_skcipher(alg_name: name, type: 0, |
739 | CRYPTO_ALG_NEED_FALLBACK); |
740 | if (IS_ERR(ptr: ctx->fallback)) { |
741 | pr_err("Error allocating fallback algo %s\n" , name); |
742 | return PTR_ERR(ptr: ctx->fallback); |
743 | } |
744 | |
745 | crypto_skcipher_set_reqsize(skcipher: tfm, reqsize: sizeof(struct sahara_aes_reqctx) + |
746 | crypto_skcipher_reqsize(tfm: ctx->fallback)); |
747 | |
748 | return 0; |
749 | } |
750 | |
751 | static void sahara_aes_exit_tfm(struct crypto_skcipher *tfm) |
752 | { |
753 | struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm); |
754 | |
755 | crypto_free_skcipher(tfm: ctx->fallback); |
756 | } |
757 | |
758 | static u32 sahara_sha_init_hdr(struct sahara_dev *dev, |
759 | struct sahara_sha_reqctx *rctx) |
760 | { |
761 | u32 hdr = 0; |
762 | |
763 | hdr = rctx->mode; |
764 | |
765 | if (rctx->first) { |
766 | hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH; |
767 | hdr |= SAHARA_HDR_MDHA_INIT; |
768 | } else { |
769 | hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY; |
770 | } |
771 | |
772 | if (rctx->last) |
773 | hdr |= SAHARA_HDR_MDHA_PDATA; |
774 | |
775 | if (hweight_long(w: hdr) % 2 == 0) |
776 | hdr |= SAHARA_HDR_PARITY_BIT; |
777 | |
778 | return hdr; |
779 | } |
780 | |
781 | static int sahara_sha_hw_links_create(struct sahara_dev *dev, |
782 | struct sahara_sha_reqctx *rctx, |
783 | int start) |
784 | { |
785 | struct scatterlist *sg; |
786 | unsigned int i; |
787 | int ret; |
788 | |
789 | dev->in_sg = rctx->in_sg; |
790 | |
791 | dev->nb_in_sg = sg_nents_for_len(sg: dev->in_sg, len: rctx->total); |
792 | if (dev->nb_in_sg < 0) { |
793 | dev_err(dev->device, "Invalid numbers of src SG.\n" ); |
794 | return dev->nb_in_sg; |
795 | } |
796 | if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) { |
797 | dev_err(dev->device, "not enough hw links (%d)\n" , |
798 | dev->nb_in_sg + dev->nb_out_sg); |
799 | return -EINVAL; |
800 | } |
801 | |
802 | sg = dev->in_sg; |
803 | ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); |
804 | if (!ret) |
805 | return -EFAULT; |
806 | |
807 | for (i = start; i < dev->nb_in_sg + start; i++) { |
808 | dev->hw_link[i]->len = sg->length; |
809 | dev->hw_link[i]->p = sg->dma_address; |
810 | if (i == (dev->nb_in_sg + start - 1)) { |
811 | dev->hw_link[i]->next = 0; |
812 | } else { |
813 | dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; |
814 | sg = sg_next(sg); |
815 | } |
816 | } |
817 | |
818 | return i; |
819 | } |
820 | |
821 | static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev, |
822 | struct sahara_sha_reqctx *rctx, |
823 | struct ahash_request *req, |
824 | int index) |
825 | { |
826 | unsigned result_len; |
827 | int i = index; |
828 | |
829 | if (rctx->first) |
830 | /* Create initial descriptor: #8*/ |
831 | dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx); |
832 | else |
833 | /* Create hash descriptor: #10. Must follow #6. */ |
834 | dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH; |
835 | |
836 | dev->hw_desc[index]->len1 = rctx->total; |
837 | if (dev->hw_desc[index]->len1 == 0) { |
838 | /* if len1 is 0, p1 must be 0, too */ |
839 | dev->hw_desc[index]->p1 = 0; |
840 | rctx->sg_in_idx = 0; |
841 | } else { |
842 | /* Create input links */ |
843 | dev->hw_desc[index]->p1 = dev->hw_phys_link[index]; |
844 | i = sahara_sha_hw_links_create(dev, rctx, start: index); |
845 | |
846 | rctx->sg_in_idx = index; |
847 | if (i < 0) |
848 | return i; |
849 | } |
850 | |
851 | dev->hw_desc[index]->p2 = dev->hw_phys_link[i]; |
852 | |
853 | /* Save the context for the next operation */ |
854 | result_len = rctx->context_size; |
855 | dev->hw_link[i]->p = dev->context_phys_base; |
856 | |
857 | dev->hw_link[i]->len = result_len; |
858 | dev->hw_desc[index]->len2 = result_len; |
859 | |
860 | dev->hw_link[i]->next = 0; |
861 | |
862 | return 0; |
863 | } |
864 | |
865 | /* |
866 | * Load descriptor aka #6 |
867 | * |
868 | * To load a previously saved context back to the MDHA unit |
869 | * |
870 | * p1: Saved Context |
871 | * p2: NULL |
872 | * |
873 | */ |
874 | static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev, |
875 | struct sahara_sha_reqctx *rctx, |
876 | struct ahash_request *req, |
877 | int index) |
878 | { |
879 | dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx); |
880 | |
881 | dev->hw_desc[index]->len1 = rctx->context_size; |
882 | dev->hw_desc[index]->p1 = dev->hw_phys_link[index]; |
883 | dev->hw_desc[index]->len2 = 0; |
884 | dev->hw_desc[index]->p2 = 0; |
885 | |
886 | dev->hw_link[index]->len = rctx->context_size; |
887 | dev->hw_link[index]->p = dev->context_phys_base; |
888 | dev->hw_link[index]->next = 0; |
889 | |
890 | return 0; |
891 | } |
892 | |
893 | static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes) |
894 | { |
895 | if (!sg || !sg->length) |
896 | return nbytes; |
897 | |
898 | while (nbytes && sg) { |
899 | if (nbytes <= sg->length) { |
900 | sg->length = nbytes; |
901 | sg_mark_end(sg); |
902 | break; |
903 | } |
904 | nbytes -= sg->length; |
905 | sg = sg_next(sg); |
906 | } |
907 | |
908 | return nbytes; |
909 | } |
910 | |
911 | static int sahara_sha_prepare_request(struct ahash_request *req) |
912 | { |
913 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
914 | struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); |
915 | unsigned int hash_later; |
916 | unsigned int block_size; |
917 | unsigned int len; |
918 | |
919 | block_size = crypto_tfm_alg_blocksize(tfm: crypto_ahash_tfm(tfm)); |
920 | |
921 | /* append bytes from previous operation */ |
922 | len = rctx->buf_cnt + req->nbytes; |
923 | |
924 | /* only the last transfer can be padded in hardware */ |
925 | if (!rctx->last && (len < block_size)) { |
926 | /* to few data, save for next operation */ |
927 | scatterwalk_map_and_copy(buf: rctx->buf + rctx->buf_cnt, sg: req->src, |
928 | start: 0, nbytes: req->nbytes, out: 0); |
929 | rctx->buf_cnt += req->nbytes; |
930 | |
931 | return 0; |
932 | } |
933 | |
934 | /* add data from previous operation first */ |
935 | if (rctx->buf_cnt) |
936 | memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt); |
937 | |
938 | /* data must always be a multiple of block_size */ |
939 | hash_later = rctx->last ? 0 : len & (block_size - 1); |
940 | if (hash_later) { |
941 | unsigned int offset = req->nbytes - hash_later; |
942 | /* Save remaining bytes for later use */ |
943 | scatterwalk_map_and_copy(buf: rctx->buf, sg: req->src, start: offset, |
944 | nbytes: hash_later, out: 0); |
945 | } |
946 | |
947 | /* nbytes should now be multiple of blocksize */ |
948 | req->nbytes = req->nbytes - hash_later; |
949 | |
950 | sahara_walk_and_recalc(sg: req->src, nbytes: req->nbytes); |
951 | |
952 | /* have data from previous operation and current */ |
953 | if (rctx->buf_cnt && req->nbytes) { |
954 | sg_init_table(rctx->in_sg_chain, 2); |
955 | sg_set_buf(sg: rctx->in_sg_chain, buf: rctx->rembuf, buflen: rctx->buf_cnt); |
956 | |
957 | sg_chain(prv: rctx->in_sg_chain, prv_nents: 2, sgl: req->src); |
958 | |
959 | rctx->total = req->nbytes + rctx->buf_cnt; |
960 | rctx->in_sg = rctx->in_sg_chain; |
961 | |
962 | req->src = rctx->in_sg_chain; |
963 | /* only data from previous operation */ |
964 | } else if (rctx->buf_cnt) { |
965 | if (req->src) |
966 | rctx->in_sg = req->src; |
967 | else |
968 | rctx->in_sg = rctx->in_sg_chain; |
969 | /* buf was copied into rembuf above */ |
970 | sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt); |
971 | rctx->total = rctx->buf_cnt; |
972 | /* no data from previous operation */ |
973 | } else { |
974 | rctx->in_sg = req->src; |
975 | rctx->total = req->nbytes; |
976 | req->src = rctx->in_sg; |
977 | } |
978 | |
979 | /* on next call, we only have the remaining data in the buffer */ |
980 | rctx->buf_cnt = hash_later; |
981 | |
982 | return -EINPROGRESS; |
983 | } |
984 | |
985 | static int sahara_sha_process(struct ahash_request *req) |
986 | { |
987 | struct sahara_dev *dev = dev_ptr; |
988 | struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); |
989 | int ret; |
990 | unsigned long timeout; |
991 | |
992 | ret = sahara_sha_prepare_request(req); |
993 | if (!ret) |
994 | return ret; |
995 | |
996 | if (rctx->first) { |
997 | sahara_sha_hw_data_descriptor_create(dev, rctx, req, index: 0); |
998 | dev->hw_desc[0]->next = 0; |
999 | rctx->first = 0; |
1000 | } else { |
1001 | memcpy(dev->context_base, rctx->context, rctx->context_size); |
1002 | |
1003 | sahara_sha_hw_context_descriptor_create(dev, rctx, req, index: 0); |
1004 | dev->hw_desc[0]->next = dev->hw_phys_desc[1]; |
1005 | sahara_sha_hw_data_descriptor_create(dev, rctx, req, index: 1); |
1006 | dev->hw_desc[1]->next = 0; |
1007 | } |
1008 | |
1009 | sahara_dump_descriptors(dev); |
1010 | sahara_dump_links(dev); |
1011 | |
1012 | reinit_completion(x: &dev->dma_completion); |
1013 | |
1014 | sahara_write(dev, data: dev->hw_phys_desc[0], SAHARA_REG_DAR); |
1015 | |
1016 | timeout = wait_for_completion_timeout(x: &dev->dma_completion, |
1017 | timeout: msecs_to_jiffies(SAHARA_TIMEOUT_MS)); |
1018 | if (!timeout) { |
1019 | dev_err(dev->device, "SHA timeout\n" ); |
1020 | return -ETIMEDOUT; |
1021 | } |
1022 | |
1023 | if (rctx->sg_in_idx) |
1024 | dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, |
1025 | DMA_TO_DEVICE); |
1026 | |
1027 | memcpy(rctx->context, dev->context_base, rctx->context_size); |
1028 | |
1029 | if (req->result) |
1030 | memcpy(req->result, rctx->context, rctx->digest_size); |
1031 | |
1032 | return 0; |
1033 | } |
1034 | |
1035 | static int sahara_queue_manage(void *data) |
1036 | { |
1037 | struct sahara_dev *dev = data; |
1038 | struct crypto_async_request *async_req; |
1039 | struct crypto_async_request *backlog; |
1040 | int ret = 0; |
1041 | |
1042 | do { |
1043 | __set_current_state(TASK_INTERRUPTIBLE); |
1044 | |
1045 | spin_lock_bh(lock: &dev->queue_spinlock); |
1046 | backlog = crypto_get_backlog(queue: &dev->queue); |
1047 | async_req = crypto_dequeue_request(queue: &dev->queue); |
1048 | spin_unlock_bh(lock: &dev->queue_spinlock); |
1049 | |
1050 | if (backlog) |
1051 | crypto_request_complete(req: backlog, err: -EINPROGRESS); |
1052 | |
1053 | if (async_req) { |
1054 | if (crypto_tfm_alg_type(tfm: async_req->tfm) == |
1055 | CRYPTO_ALG_TYPE_AHASH) { |
1056 | struct ahash_request *req = |
1057 | ahash_request_cast(req: async_req); |
1058 | |
1059 | ret = sahara_sha_process(req); |
1060 | } else { |
1061 | struct skcipher_request *req = |
1062 | skcipher_request_cast(req: async_req); |
1063 | |
1064 | ret = sahara_aes_process(req); |
1065 | } |
1066 | |
1067 | crypto_request_complete(req: async_req, err: ret); |
1068 | |
1069 | continue; |
1070 | } |
1071 | |
1072 | schedule(); |
1073 | } while (!kthread_should_stop()); |
1074 | |
1075 | return 0; |
1076 | } |
1077 | |
1078 | static int sahara_sha_enqueue(struct ahash_request *req, int last) |
1079 | { |
1080 | struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); |
1081 | struct sahara_dev *dev = dev_ptr; |
1082 | int ret; |
1083 | |
1084 | if (!req->nbytes && !last) |
1085 | return 0; |
1086 | |
1087 | rctx->last = last; |
1088 | |
1089 | if (!rctx->active) { |
1090 | rctx->active = 1; |
1091 | rctx->first = 1; |
1092 | } |
1093 | |
1094 | spin_lock_bh(lock: &dev->queue_spinlock); |
1095 | ret = crypto_enqueue_request(queue: &dev->queue, request: &req->base); |
1096 | spin_unlock_bh(lock: &dev->queue_spinlock); |
1097 | |
1098 | wake_up_process(tsk: dev->kthread); |
1099 | |
1100 | return ret; |
1101 | } |
1102 | |
1103 | static int sahara_sha_init(struct ahash_request *req) |
1104 | { |
1105 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
1106 | struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); |
1107 | |
1108 | memset(rctx, 0, sizeof(*rctx)); |
1109 | |
1110 | switch (crypto_ahash_digestsize(tfm)) { |
1111 | case SHA1_DIGEST_SIZE: |
1112 | rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1; |
1113 | rctx->digest_size = SHA1_DIGEST_SIZE; |
1114 | break; |
1115 | case SHA256_DIGEST_SIZE: |
1116 | rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256; |
1117 | rctx->digest_size = SHA256_DIGEST_SIZE; |
1118 | break; |
1119 | default: |
1120 | return -EINVAL; |
1121 | } |
1122 | |
1123 | rctx->context_size = rctx->digest_size + 4; |
1124 | rctx->active = 0; |
1125 | |
1126 | return 0; |
1127 | } |
1128 | |
1129 | static int sahara_sha_update(struct ahash_request *req) |
1130 | { |
1131 | return sahara_sha_enqueue(req, last: 0); |
1132 | } |
1133 | |
1134 | static int sahara_sha_final(struct ahash_request *req) |
1135 | { |
1136 | req->nbytes = 0; |
1137 | return sahara_sha_enqueue(req, last: 1); |
1138 | } |
1139 | |
1140 | static int sahara_sha_finup(struct ahash_request *req) |
1141 | { |
1142 | return sahara_sha_enqueue(req, last: 1); |
1143 | } |
1144 | |
1145 | static int sahara_sha_digest(struct ahash_request *req) |
1146 | { |
1147 | sahara_sha_init(req); |
1148 | |
1149 | return sahara_sha_finup(req); |
1150 | } |
1151 | |
1152 | static int sahara_sha_export(struct ahash_request *req, void *out) |
1153 | { |
1154 | struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); |
1155 | |
1156 | memcpy(out, rctx, sizeof(struct sahara_sha_reqctx)); |
1157 | |
1158 | return 0; |
1159 | } |
1160 | |
1161 | static int sahara_sha_import(struct ahash_request *req, const void *in) |
1162 | { |
1163 | struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); |
1164 | |
1165 | memcpy(rctx, in, sizeof(struct sahara_sha_reqctx)); |
1166 | |
1167 | return 0; |
1168 | } |
1169 | |
1170 | static int sahara_sha_cra_init(struct crypto_tfm *tfm) |
1171 | { |
1172 | crypto_ahash_set_reqsize(tfm: __crypto_ahash_cast(tfm), |
1173 | reqsize: sizeof(struct sahara_sha_reqctx) + |
1174 | SHA_BUFFER_LEN + SHA256_BLOCK_SIZE); |
1175 | |
1176 | return 0; |
1177 | } |
1178 | |
1179 | static struct skcipher_alg aes_algs[] = { |
1180 | { |
1181 | .base.cra_name = "ecb(aes)" , |
1182 | .base.cra_driver_name = "sahara-ecb-aes" , |
1183 | .base.cra_priority = 300, |
1184 | .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, |
1185 | .base.cra_blocksize = AES_BLOCK_SIZE, |
1186 | .base.cra_ctxsize = sizeof(struct sahara_ctx), |
1187 | .base.cra_alignmask = 0x0, |
1188 | .base.cra_module = THIS_MODULE, |
1189 | |
1190 | .init = sahara_aes_init_tfm, |
1191 | .exit = sahara_aes_exit_tfm, |
1192 | .min_keysize = AES_MIN_KEY_SIZE , |
1193 | .max_keysize = AES_MAX_KEY_SIZE, |
1194 | .setkey = sahara_aes_setkey, |
1195 | .encrypt = sahara_aes_ecb_encrypt, |
1196 | .decrypt = sahara_aes_ecb_decrypt, |
1197 | }, { |
1198 | .base.cra_name = "cbc(aes)" , |
1199 | .base.cra_driver_name = "sahara-cbc-aes" , |
1200 | .base.cra_priority = 300, |
1201 | .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, |
1202 | .base.cra_blocksize = AES_BLOCK_SIZE, |
1203 | .base.cra_ctxsize = sizeof(struct sahara_ctx), |
1204 | .base.cra_alignmask = 0x0, |
1205 | .base.cra_module = THIS_MODULE, |
1206 | |
1207 | .init = sahara_aes_init_tfm, |
1208 | .exit = sahara_aes_exit_tfm, |
1209 | .min_keysize = AES_MIN_KEY_SIZE , |
1210 | .max_keysize = AES_MAX_KEY_SIZE, |
1211 | .ivsize = AES_BLOCK_SIZE, |
1212 | .setkey = sahara_aes_setkey, |
1213 | .encrypt = sahara_aes_cbc_encrypt, |
1214 | .decrypt = sahara_aes_cbc_decrypt, |
1215 | } |
1216 | }; |
1217 | |
1218 | static struct ahash_alg sha_v3_algs[] = { |
1219 | { |
1220 | .init = sahara_sha_init, |
1221 | .update = sahara_sha_update, |
1222 | .final = sahara_sha_final, |
1223 | .finup = sahara_sha_finup, |
1224 | .digest = sahara_sha_digest, |
1225 | .export = sahara_sha_export, |
1226 | .import = sahara_sha_import, |
1227 | .halg.digestsize = SHA1_DIGEST_SIZE, |
1228 | .halg.statesize = sizeof(struct sahara_sha_reqctx), |
1229 | .halg.base = { |
1230 | .cra_name = "sha1" , |
1231 | .cra_driver_name = "sahara-sha1" , |
1232 | .cra_priority = 300, |
1233 | .cra_flags = CRYPTO_ALG_ASYNC | |
1234 | CRYPTO_ALG_NEED_FALLBACK, |
1235 | .cra_blocksize = SHA1_BLOCK_SIZE, |
1236 | .cra_ctxsize = sizeof(struct sahara_ctx), |
1237 | .cra_alignmask = 0, |
1238 | .cra_module = THIS_MODULE, |
1239 | .cra_init = sahara_sha_cra_init, |
1240 | } |
1241 | }, |
1242 | }; |
1243 | |
1244 | static struct ahash_alg sha_v4_algs[] = { |
1245 | { |
1246 | .init = sahara_sha_init, |
1247 | .update = sahara_sha_update, |
1248 | .final = sahara_sha_final, |
1249 | .finup = sahara_sha_finup, |
1250 | .digest = sahara_sha_digest, |
1251 | .export = sahara_sha_export, |
1252 | .import = sahara_sha_import, |
1253 | .halg.digestsize = SHA256_DIGEST_SIZE, |
1254 | .halg.statesize = sizeof(struct sahara_sha_reqctx), |
1255 | .halg.base = { |
1256 | .cra_name = "sha256" , |
1257 | .cra_driver_name = "sahara-sha256" , |
1258 | .cra_priority = 300, |
1259 | .cra_flags = CRYPTO_ALG_ASYNC | |
1260 | CRYPTO_ALG_NEED_FALLBACK, |
1261 | .cra_blocksize = SHA256_BLOCK_SIZE, |
1262 | .cra_ctxsize = sizeof(struct sahara_ctx), |
1263 | .cra_alignmask = 0, |
1264 | .cra_module = THIS_MODULE, |
1265 | .cra_init = sahara_sha_cra_init, |
1266 | } |
1267 | }, |
1268 | }; |
1269 | |
1270 | static irqreturn_t sahara_irq_handler(int irq, void *data) |
1271 | { |
1272 | struct sahara_dev *dev = data; |
1273 | unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS); |
1274 | unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS); |
1275 | |
1276 | sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR, |
1277 | SAHARA_REG_CMD); |
1278 | |
1279 | sahara_decode_status(dev, status: stat); |
1280 | |
1281 | if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) { |
1282 | return IRQ_NONE; |
1283 | } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) { |
1284 | dev->error = 0; |
1285 | } else { |
1286 | sahara_decode_error(dev, error: err); |
1287 | dev->error = -EINVAL; |
1288 | } |
1289 | |
1290 | complete(&dev->dma_completion); |
1291 | |
1292 | return IRQ_HANDLED; |
1293 | } |
1294 | |
1295 | |
1296 | static int sahara_register_algs(struct sahara_dev *dev) |
1297 | { |
1298 | int err; |
1299 | unsigned int i, j, k, l; |
1300 | |
1301 | for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { |
1302 | err = crypto_register_skcipher(alg: &aes_algs[i]); |
1303 | if (err) |
1304 | goto err_aes_algs; |
1305 | } |
1306 | |
1307 | for (k = 0; k < ARRAY_SIZE(sha_v3_algs); k++) { |
1308 | err = crypto_register_ahash(alg: &sha_v3_algs[k]); |
1309 | if (err) |
1310 | goto err_sha_v3_algs; |
1311 | } |
1312 | |
1313 | if (dev->version > SAHARA_VERSION_3) |
1314 | for (l = 0; l < ARRAY_SIZE(sha_v4_algs); l++) { |
1315 | err = crypto_register_ahash(alg: &sha_v4_algs[l]); |
1316 | if (err) |
1317 | goto err_sha_v4_algs; |
1318 | } |
1319 | |
1320 | return 0; |
1321 | |
1322 | err_sha_v4_algs: |
1323 | for (j = 0; j < l; j++) |
1324 | crypto_unregister_ahash(alg: &sha_v4_algs[j]); |
1325 | |
1326 | err_sha_v3_algs: |
1327 | for (j = 0; j < k; j++) |
1328 | crypto_unregister_ahash(alg: &sha_v3_algs[j]); |
1329 | |
1330 | err_aes_algs: |
1331 | for (j = 0; j < i; j++) |
1332 | crypto_unregister_skcipher(alg: &aes_algs[j]); |
1333 | |
1334 | return err; |
1335 | } |
1336 | |
1337 | static void sahara_unregister_algs(struct sahara_dev *dev) |
1338 | { |
1339 | unsigned int i; |
1340 | |
1341 | for (i = 0; i < ARRAY_SIZE(aes_algs); i++) |
1342 | crypto_unregister_skcipher(alg: &aes_algs[i]); |
1343 | |
1344 | for (i = 0; i < ARRAY_SIZE(sha_v3_algs); i++) |
1345 | crypto_unregister_ahash(alg: &sha_v3_algs[i]); |
1346 | |
1347 | if (dev->version > SAHARA_VERSION_3) |
1348 | for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++) |
1349 | crypto_unregister_ahash(alg: &sha_v4_algs[i]); |
1350 | } |
1351 | |
1352 | static const struct of_device_id sahara_dt_ids[] = { |
1353 | { .compatible = "fsl,imx53-sahara" }, |
1354 | { .compatible = "fsl,imx27-sahara" }, |
1355 | { /* sentinel */ } |
1356 | }; |
1357 | MODULE_DEVICE_TABLE(of, sahara_dt_ids); |
1358 | |
1359 | static int sahara_probe(struct platform_device *pdev) |
1360 | { |
1361 | struct sahara_dev *dev; |
1362 | u32 version; |
1363 | int irq; |
1364 | int err; |
1365 | int i; |
1366 | |
1367 | dev = devm_kzalloc(dev: &pdev->dev, size: sizeof(*dev), GFP_KERNEL); |
1368 | if (!dev) |
1369 | return -ENOMEM; |
1370 | |
1371 | dev->device = &pdev->dev; |
1372 | platform_set_drvdata(pdev, data: dev); |
1373 | |
1374 | /* Get the base address */ |
1375 | dev->regs_base = devm_platform_ioremap_resource(pdev, index: 0); |
1376 | if (IS_ERR(ptr: dev->regs_base)) |
1377 | return PTR_ERR(ptr: dev->regs_base); |
1378 | |
1379 | /* Get the IRQ */ |
1380 | irq = platform_get_irq(pdev, 0); |
1381 | if (irq < 0) |
1382 | return irq; |
1383 | |
1384 | err = devm_request_irq(dev: &pdev->dev, irq, handler: sahara_irq_handler, |
1385 | irqflags: 0, devname: dev_name(dev: &pdev->dev), dev_id: dev); |
1386 | if (err) { |
1387 | dev_err(&pdev->dev, "failed to request irq\n" ); |
1388 | return err; |
1389 | } |
1390 | |
1391 | /* clocks */ |
1392 | dev->clk_ipg = devm_clk_get(dev: &pdev->dev, id: "ipg" ); |
1393 | if (IS_ERR(ptr: dev->clk_ipg)) { |
1394 | dev_err(&pdev->dev, "Could not get ipg clock\n" ); |
1395 | return PTR_ERR(ptr: dev->clk_ipg); |
1396 | } |
1397 | |
1398 | dev->clk_ahb = devm_clk_get(dev: &pdev->dev, id: "ahb" ); |
1399 | if (IS_ERR(ptr: dev->clk_ahb)) { |
1400 | dev_err(&pdev->dev, "Could not get ahb clock\n" ); |
1401 | return PTR_ERR(ptr: dev->clk_ahb); |
1402 | } |
1403 | |
1404 | /* Allocate HW descriptors */ |
1405 | dev->hw_desc[0] = dmam_alloc_coherent(dev: &pdev->dev, |
1406 | SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), |
1407 | dma_handle: &dev->hw_phys_desc[0], GFP_KERNEL); |
1408 | if (!dev->hw_desc[0]) { |
1409 | dev_err(&pdev->dev, "Could not allocate hw descriptors\n" ); |
1410 | return -ENOMEM; |
1411 | } |
1412 | dev->hw_desc[1] = dev->hw_desc[0] + 1; |
1413 | dev->hw_phys_desc[1] = dev->hw_phys_desc[0] + |
1414 | sizeof(struct sahara_hw_desc); |
1415 | |
1416 | /* Allocate space for iv and key */ |
1417 | dev->key_base = dmam_alloc_coherent(dev: &pdev->dev, size: 2 * AES_KEYSIZE_128, |
1418 | dma_handle: &dev->key_phys_base, GFP_KERNEL); |
1419 | if (!dev->key_base) { |
1420 | dev_err(&pdev->dev, "Could not allocate memory for key\n" ); |
1421 | return -ENOMEM; |
1422 | } |
1423 | dev->iv_base = dev->key_base + AES_KEYSIZE_128; |
1424 | dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128; |
1425 | |
1426 | /* Allocate space for context: largest digest + message length field */ |
1427 | dev->context_base = dmam_alloc_coherent(dev: &pdev->dev, |
1428 | SHA256_DIGEST_SIZE + 4, |
1429 | dma_handle: &dev->context_phys_base, GFP_KERNEL); |
1430 | if (!dev->context_base) { |
1431 | dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n" ); |
1432 | return -ENOMEM; |
1433 | } |
1434 | |
1435 | /* Allocate space for HW links */ |
1436 | dev->hw_link[0] = dmam_alloc_coherent(dev: &pdev->dev, |
1437 | SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), |
1438 | dma_handle: &dev->hw_phys_link[0], GFP_KERNEL); |
1439 | if (!dev->hw_link[0]) { |
1440 | dev_err(&pdev->dev, "Could not allocate hw links\n" ); |
1441 | return -ENOMEM; |
1442 | } |
1443 | for (i = 1; i < SAHARA_MAX_HW_LINK; i++) { |
1444 | dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] + |
1445 | sizeof(struct sahara_hw_link); |
1446 | dev->hw_link[i] = dev->hw_link[i - 1] + 1; |
1447 | } |
1448 | |
1449 | crypto_init_queue(queue: &dev->queue, SAHARA_QUEUE_LENGTH); |
1450 | |
1451 | spin_lock_init(&dev->queue_spinlock); |
1452 | |
1453 | dev_ptr = dev; |
1454 | |
1455 | dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto" ); |
1456 | if (IS_ERR(ptr: dev->kthread)) { |
1457 | return PTR_ERR(ptr: dev->kthread); |
1458 | } |
1459 | |
1460 | init_completion(x: &dev->dma_completion); |
1461 | |
1462 | err = clk_prepare_enable(clk: dev->clk_ipg); |
1463 | if (err) |
1464 | return err; |
1465 | err = clk_prepare_enable(clk: dev->clk_ahb); |
1466 | if (err) |
1467 | goto clk_ipg_disable; |
1468 | |
1469 | version = sahara_read(dev, SAHARA_REG_VERSION); |
1470 | if (of_device_is_compatible(device: pdev->dev.of_node, "fsl,imx27-sahara" )) { |
1471 | if (version != SAHARA_VERSION_3) |
1472 | err = -ENODEV; |
1473 | } else if (of_device_is_compatible(device: pdev->dev.of_node, |
1474 | "fsl,imx53-sahara" )) { |
1475 | if (((version >> 8) & 0xff) != SAHARA_VERSION_4) |
1476 | err = -ENODEV; |
1477 | version = (version >> 8) & 0xff; |
1478 | } |
1479 | if (err == -ENODEV) { |
1480 | dev_err(&pdev->dev, "SAHARA version %d not supported\n" , |
1481 | version); |
1482 | goto err_algs; |
1483 | } |
1484 | |
1485 | dev->version = version; |
1486 | |
1487 | sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH, |
1488 | SAHARA_REG_CMD); |
1489 | sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) | |
1490 | SAHARA_CONTROL_SET_MAXBURST(8) | |
1491 | SAHARA_CONTROL_RNG_AUTORSD | |
1492 | SAHARA_CONTROL_ENABLE_INT, |
1493 | SAHARA_REG_CONTROL); |
1494 | |
1495 | err = sahara_register_algs(dev); |
1496 | if (err) |
1497 | goto err_algs; |
1498 | |
1499 | dev_info(&pdev->dev, "SAHARA version %d initialized\n" , version); |
1500 | |
1501 | return 0; |
1502 | |
1503 | err_algs: |
1504 | kthread_stop(k: dev->kthread); |
1505 | dev_ptr = NULL; |
1506 | clk_disable_unprepare(clk: dev->clk_ahb); |
1507 | clk_ipg_disable: |
1508 | clk_disable_unprepare(clk: dev->clk_ipg); |
1509 | |
1510 | return err; |
1511 | } |
1512 | |
1513 | static void sahara_remove(struct platform_device *pdev) |
1514 | { |
1515 | struct sahara_dev *dev = platform_get_drvdata(pdev); |
1516 | |
1517 | kthread_stop(k: dev->kthread); |
1518 | |
1519 | sahara_unregister_algs(dev); |
1520 | |
1521 | clk_disable_unprepare(clk: dev->clk_ipg); |
1522 | clk_disable_unprepare(clk: dev->clk_ahb); |
1523 | |
1524 | dev_ptr = NULL; |
1525 | } |
1526 | |
1527 | static struct platform_driver sahara_driver = { |
1528 | .probe = sahara_probe, |
1529 | .remove_new = sahara_remove, |
1530 | .driver = { |
1531 | .name = SAHARA_NAME, |
1532 | .of_match_table = sahara_dt_ids, |
1533 | }, |
1534 | }; |
1535 | |
1536 | module_platform_driver(sahara_driver); |
1537 | |
1538 | MODULE_LICENSE("GPL" ); |
1539 | MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>" ); |
1540 | MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>" ); |
1541 | MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator" ); |
1542 | |