1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Cryptographic API. |
4 | * |
5 | * Support for SAHARA cryptographic accelerator. |
6 | * |
7 | * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de> |
8 | * Copyright (c) 2013 Vista Silicon S.L. |
9 | * Author: Javier Martin <javier.martin@vista-silicon.com> |
10 | * |
11 | * Based on omap-aes.c and tegra-aes.c |
12 | */ |
13 | |
14 | #include <crypto/aes.h> |
15 | #include <crypto/internal/hash.h> |
16 | #include <crypto/internal/skcipher.h> |
17 | #include <crypto/scatterwalk.h> |
18 | #include <crypto/engine.h> |
19 | #include <crypto/sha1.h> |
20 | #include <crypto/sha2.h> |
21 | |
22 | #include <linux/clk.h> |
23 | #include <linux/dma-mapping.h> |
24 | #include <linux/interrupt.h> |
25 | #include <linux/io.h> |
26 | #include <linux/irq.h> |
27 | #include <linux/kernel.h> |
28 | #include <linux/module.h> |
29 | #include <linux/of.h> |
30 | #include <linux/platform_device.h> |
31 | #include <linux/spinlock.h> |
32 | |
33 | #define SHA_BUFFER_LEN PAGE_SIZE |
34 | #define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE |
35 | |
36 | #define SAHARA_NAME "sahara" |
37 | #define SAHARA_VERSION_3 3 |
38 | #define SAHARA_VERSION_4 4 |
39 | #define SAHARA_TIMEOUT_MS 1000 |
40 | #define SAHARA_MAX_HW_DESC 2 |
41 | #define SAHARA_MAX_HW_LINK 20 |
42 | |
43 | #define FLAGS_MODE_MASK 0x000f |
44 | #define FLAGS_ENCRYPT BIT(0) |
45 | #define FLAGS_CBC BIT(1) |
46 | |
47 | #define SAHARA_HDR_BASE 0x00800000 |
48 | #define SAHARA_HDR_SKHA_ALG_AES 0 |
49 | #define SAHARA_HDR_SKHA_MODE_ECB 0 |
50 | #define SAHARA_HDR_SKHA_OP_ENC BIT(2) |
51 | #define SAHARA_HDR_SKHA_MODE_CBC BIT(3) |
52 | #define SAHARA_HDR_FORM_DATA (5 << 16) |
53 | #define SAHARA_HDR_FORM_KEY BIT(19) |
54 | #define SAHARA_HDR_LLO BIT(24) |
55 | #define SAHARA_HDR_CHA_SKHA BIT(28) |
56 | #define SAHARA_HDR_CHA_MDHA BIT(29) |
57 | #define SAHARA_HDR_PARITY_BIT BIT(31) |
58 | |
59 | #define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000 |
60 | #define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000 |
61 | #define SAHARA_HDR_MDHA_HASH 0xA0850000 |
62 | #define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000 |
63 | #define SAHARA_HDR_MDHA_ALG_SHA1 0 |
64 | #define SAHARA_HDR_MDHA_ALG_MD5 1 |
65 | #define SAHARA_HDR_MDHA_ALG_SHA256 2 |
66 | #define SAHARA_HDR_MDHA_ALG_SHA224 3 |
67 | #define SAHARA_HDR_MDHA_PDATA BIT(2) |
68 | #define SAHARA_HDR_MDHA_HMAC BIT(3) |
69 | #define SAHARA_HDR_MDHA_INIT BIT(5) |
70 | #define SAHARA_HDR_MDHA_IPAD BIT(6) |
71 | #define SAHARA_HDR_MDHA_OPAD BIT(7) |
72 | #define SAHARA_HDR_MDHA_SWAP BIT(8) |
73 | #define SAHARA_HDR_MDHA_MAC_FULL BIT(9) |
74 | #define SAHARA_HDR_MDHA_SSL BIT(10) |
75 | |
76 | #define SAHARA_REG_VERSION 0x00 |
77 | #define SAHARA_REG_DAR 0x04 |
78 | #define SAHARA_REG_CONTROL 0x08 |
79 | #define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24) |
80 | #define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16) |
81 | #define SAHARA_CONTROL_RNG_AUTORSD BIT(7) |
82 | #define SAHARA_CONTROL_ENABLE_INT BIT(4) |
83 | #define SAHARA_REG_CMD 0x0C |
84 | #define SAHARA_CMD_RESET BIT(0) |
85 | #define SAHARA_CMD_CLEAR_INT BIT(8) |
86 | #define SAHARA_CMD_CLEAR_ERR BIT(9) |
87 | #define SAHARA_CMD_SINGLE_STEP BIT(10) |
88 | #define SAHARA_CMD_MODE_BATCH BIT(16) |
89 | #define SAHARA_CMD_MODE_DEBUG BIT(18) |
90 | #define SAHARA_REG_STATUS 0x10 |
91 | #define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7) |
92 | #define SAHARA_STATE_IDLE 0 |
93 | #define SAHARA_STATE_BUSY 1 |
94 | #define SAHARA_STATE_ERR 2 |
95 | #define SAHARA_STATE_FAULT 3 |
96 | #define SAHARA_STATE_COMPLETE 4 |
97 | #define SAHARA_STATE_COMP_FLAG BIT(2) |
98 | #define SAHARA_STATUS_DAR_FULL BIT(3) |
99 | #define SAHARA_STATUS_ERROR BIT(4) |
100 | #define SAHARA_STATUS_SECURE BIT(5) |
101 | #define SAHARA_STATUS_FAIL BIT(6) |
102 | #define SAHARA_STATUS_INIT BIT(7) |
103 | #define SAHARA_STATUS_RNG_RESEED BIT(8) |
104 | #define SAHARA_STATUS_ACTIVE_RNG BIT(9) |
105 | #define SAHARA_STATUS_ACTIVE_MDHA BIT(10) |
106 | #define SAHARA_STATUS_ACTIVE_SKHA BIT(11) |
107 | #define SAHARA_STATUS_MODE_BATCH BIT(16) |
108 | #define SAHARA_STATUS_MODE_DEDICATED BIT(17) |
109 | #define SAHARA_STATUS_MODE_DEBUG BIT(18) |
110 | #define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff) |
111 | #define SAHARA_REG_ERRSTATUS 0x14 |
112 | #define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf) |
113 | #define SAHARA_ERRSOURCE_CHA 14 |
114 | #define SAHARA_ERRSOURCE_DMA 15 |
115 | #define SAHARA_ERRSTATUS_DMA_DIR BIT(8) |
116 | #define SAHARA_ERRSTATUS_GET_DMASZ(x) (((x) >> 9) & 0x3) |
117 | #define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7) |
118 | #define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff) |
119 | #define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3) |
120 | #define SAHARA_REG_FADDR 0x18 |
121 | #define SAHARA_REG_CDAR 0x1C |
122 | #define SAHARA_REG_IDAR 0x20 |
123 | |
124 | struct sahara_hw_desc { |
125 | u32 hdr; |
126 | u32 len1; |
127 | u32 p1; |
128 | u32 len2; |
129 | u32 p2; |
130 | u32 next; |
131 | }; |
132 | |
133 | struct sahara_hw_link { |
134 | u32 len; |
135 | u32 p; |
136 | u32 next; |
137 | }; |
138 | |
139 | struct sahara_ctx { |
140 | /* AES-specific context */ |
141 | int keylen; |
142 | u8 key[AES_KEYSIZE_128]; |
143 | struct crypto_skcipher *fallback; |
144 | }; |
145 | |
146 | struct sahara_aes_reqctx { |
147 | unsigned long mode; |
148 | u8 iv_out[AES_BLOCK_SIZE]; |
149 | struct skcipher_request fallback_req; // keep at the end |
150 | }; |
151 | |
152 | /* |
153 | * struct sahara_sha_reqctx - private data per request |
154 | * @buf: holds data for requests smaller than block_size |
155 | * @rembuf: used to prepare one block_size-aligned request |
156 | * @context: hw-specific context for request. Digest is extracted from this |
157 | * @mode: specifies what type of hw-descriptor needs to be built |
158 | * @digest_size: length of digest for this request |
159 | * @context_size: length of hw-context for this request. |
160 | * Always digest_size + 4 |
161 | * @buf_cnt: number of bytes saved in buf |
162 | * @sg_in_idx: number of hw links |
163 | * @in_sg: scatterlist for input data |
164 | * @in_sg_chain: scatterlists for chained input data |
165 | * @total: total number of bytes for transfer |
166 | * @last: is this the last block |
167 | * @first: is this the first block |
168 | */ |
169 | struct sahara_sha_reqctx { |
170 | u8 buf[SAHARA_MAX_SHA_BLOCK_SIZE]; |
171 | u8 rembuf[SAHARA_MAX_SHA_BLOCK_SIZE]; |
172 | u8 context[SHA256_DIGEST_SIZE + 4]; |
173 | unsigned int mode; |
174 | unsigned int digest_size; |
175 | unsigned int context_size; |
176 | unsigned int buf_cnt; |
177 | unsigned int sg_in_idx; |
178 | struct scatterlist *in_sg; |
179 | struct scatterlist in_sg_chain[2]; |
180 | size_t total; |
181 | unsigned int last; |
182 | unsigned int first; |
183 | }; |
184 | |
185 | struct sahara_dev { |
186 | struct device *device; |
187 | unsigned int version; |
188 | void __iomem *regs_base; |
189 | struct clk *clk_ipg; |
190 | struct clk *clk_ahb; |
191 | struct completion dma_completion; |
192 | |
193 | struct sahara_ctx *ctx; |
194 | unsigned long flags; |
195 | |
196 | struct sahara_hw_desc *hw_desc[SAHARA_MAX_HW_DESC]; |
197 | dma_addr_t hw_phys_desc[SAHARA_MAX_HW_DESC]; |
198 | |
199 | u8 *key_base; |
200 | dma_addr_t key_phys_base; |
201 | |
202 | u8 *iv_base; |
203 | dma_addr_t iv_phys_base; |
204 | |
205 | u8 *context_base; |
206 | dma_addr_t context_phys_base; |
207 | |
208 | struct sahara_hw_link *hw_link[SAHARA_MAX_HW_LINK]; |
209 | dma_addr_t hw_phys_link[SAHARA_MAX_HW_LINK]; |
210 | |
211 | size_t total; |
212 | struct scatterlist *in_sg; |
213 | int nb_in_sg; |
214 | struct scatterlist *out_sg; |
215 | int nb_out_sg; |
216 | |
217 | struct crypto_engine *engine; |
218 | }; |
219 | |
220 | static struct sahara_dev *dev_ptr; |
221 | |
222 | static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg) |
223 | { |
224 | writel(val: data, addr: dev->regs_base + reg); |
225 | } |
226 | |
227 | static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg) |
228 | { |
229 | return readl(addr: dev->regs_base + reg); |
230 | } |
231 | |
232 | static u32 sahara_aes_key_hdr(struct sahara_dev *dev) |
233 | { |
234 | u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES | |
235 | SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO | |
236 | SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT; |
237 | |
238 | if (dev->flags & FLAGS_CBC) { |
239 | hdr |= SAHARA_HDR_SKHA_MODE_CBC; |
240 | hdr ^= SAHARA_HDR_PARITY_BIT; |
241 | } |
242 | |
243 | if (dev->flags & FLAGS_ENCRYPT) { |
244 | hdr |= SAHARA_HDR_SKHA_OP_ENC; |
245 | hdr ^= SAHARA_HDR_PARITY_BIT; |
246 | } |
247 | |
248 | return hdr; |
249 | } |
250 | |
251 | static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev) |
252 | { |
253 | return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA | |
254 | SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT; |
255 | } |
256 | |
257 | static const char *sahara_err_src[16] = { |
258 | "No error" , |
259 | "Header error" , |
260 | "Descriptor length error" , |
261 | "Descriptor length or pointer error" , |
262 | "Link length error" , |
263 | "Link pointer error" , |
264 | "Input buffer error" , |
265 | "Output buffer error" , |
266 | "Output buffer starvation" , |
267 | "Internal state fault" , |
268 | "General descriptor problem" , |
269 | "Reserved" , |
270 | "Descriptor address error" , |
271 | "Link address error" , |
272 | "CHA error" , |
273 | "DMA error" |
274 | }; |
275 | |
276 | static const char *sahara_err_dmasize[4] = { |
277 | "Byte transfer" , |
278 | "Half-word transfer" , |
279 | "Word transfer" , |
280 | "Reserved" |
281 | }; |
282 | |
283 | static const char *sahara_err_dmasrc[8] = { |
284 | "No error" , |
285 | "AHB bus error" , |
286 | "Internal IP bus error" , |
287 | "Parity error" , |
288 | "DMA crosses 256 byte boundary" , |
289 | "DMA is busy" , |
290 | "Reserved" , |
291 | "DMA HW error" |
292 | }; |
293 | |
294 | static const char *sahara_cha_errsrc[12] = { |
295 | "Input buffer non-empty" , |
296 | "Illegal address" , |
297 | "Illegal mode" , |
298 | "Illegal data size" , |
299 | "Illegal key size" , |
300 | "Write during processing" , |
301 | "CTX read during processing" , |
302 | "HW error" , |
303 | "Input buffer disabled/underflow" , |
304 | "Output buffer disabled/overflow" , |
305 | "DES key parity error" , |
306 | "Reserved" |
307 | }; |
308 | |
309 | static const char *sahara_cha_err[4] = { "No error" , "SKHA" , "MDHA" , "RNG" }; |
310 | |
311 | static void sahara_decode_error(struct sahara_dev *dev, unsigned int error) |
312 | { |
313 | u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error); |
314 | u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error)); |
315 | |
316 | dev_err(dev->device, "%s: Error Register = 0x%08x\n" , __func__, error); |
317 | |
318 | dev_err(dev->device, " - %s.\n" , sahara_err_src[source]); |
319 | |
320 | if (source == SAHARA_ERRSOURCE_DMA) { |
321 | if (error & SAHARA_ERRSTATUS_DMA_DIR) |
322 | dev_err(dev->device, " * DMA read.\n" ); |
323 | else |
324 | dev_err(dev->device, " * DMA write.\n" ); |
325 | |
326 | dev_err(dev->device, " * %s.\n" , |
327 | sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]); |
328 | dev_err(dev->device, " * %s.\n" , |
329 | sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]); |
330 | } else if (source == SAHARA_ERRSOURCE_CHA) { |
331 | dev_err(dev->device, " * %s.\n" , |
332 | sahara_cha_errsrc[chasrc]); |
333 | dev_err(dev->device, " * %s.\n" , |
334 | sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]); |
335 | } |
336 | dev_err(dev->device, "\n" ); |
337 | } |
338 | |
339 | static const char *sahara_state[4] = { "Idle" , "Busy" , "Error" , "HW Fault" }; |
340 | |
341 | static void sahara_decode_status(struct sahara_dev *dev, unsigned int status) |
342 | { |
343 | u8 state; |
344 | |
345 | if (!__is_defined(DEBUG)) |
346 | return; |
347 | |
348 | state = SAHARA_STATUS_GET_STATE(status); |
349 | |
350 | dev_dbg(dev->device, "%s: Status Register = 0x%08x\n" , |
351 | __func__, status); |
352 | |
353 | dev_dbg(dev->device, " - State = %d:\n" , state); |
354 | if (state & SAHARA_STATE_COMP_FLAG) |
355 | dev_dbg(dev->device, " * Descriptor completed. IRQ pending.\n" ); |
356 | |
357 | dev_dbg(dev->device, " * %s.\n" , |
358 | sahara_state[state & ~SAHARA_STATE_COMP_FLAG]); |
359 | |
360 | if (status & SAHARA_STATUS_DAR_FULL) |
361 | dev_dbg(dev->device, " - DAR Full.\n" ); |
362 | if (status & SAHARA_STATUS_ERROR) |
363 | dev_dbg(dev->device, " - Error.\n" ); |
364 | if (status & SAHARA_STATUS_SECURE) |
365 | dev_dbg(dev->device, " - Secure.\n" ); |
366 | if (status & SAHARA_STATUS_FAIL) |
367 | dev_dbg(dev->device, " - Fail.\n" ); |
368 | if (status & SAHARA_STATUS_RNG_RESEED) |
369 | dev_dbg(dev->device, " - RNG Reseed Request.\n" ); |
370 | if (status & SAHARA_STATUS_ACTIVE_RNG) |
371 | dev_dbg(dev->device, " - RNG Active.\n" ); |
372 | if (status & SAHARA_STATUS_ACTIVE_MDHA) |
373 | dev_dbg(dev->device, " - MDHA Active.\n" ); |
374 | if (status & SAHARA_STATUS_ACTIVE_SKHA) |
375 | dev_dbg(dev->device, " - SKHA Active.\n" ); |
376 | |
377 | if (status & SAHARA_STATUS_MODE_BATCH) |
378 | dev_dbg(dev->device, " - Batch Mode.\n" ); |
379 | else if (status & SAHARA_STATUS_MODE_DEDICATED) |
380 | dev_dbg(dev->device, " - Dedicated Mode.\n" ); |
381 | else if (status & SAHARA_STATUS_MODE_DEBUG) |
382 | dev_dbg(dev->device, " - Debug Mode.\n" ); |
383 | |
384 | dev_dbg(dev->device, " - Internal state = 0x%02x\n" , |
385 | SAHARA_STATUS_GET_ISTATE(status)); |
386 | |
387 | dev_dbg(dev->device, "Current DAR: 0x%08x\n" , |
388 | sahara_read(dev, SAHARA_REG_CDAR)); |
389 | dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n" , |
390 | sahara_read(dev, SAHARA_REG_IDAR)); |
391 | } |
392 | |
393 | static void sahara_dump_descriptors(struct sahara_dev *dev) |
394 | { |
395 | int i; |
396 | |
397 | if (!__is_defined(DEBUG)) |
398 | return; |
399 | |
400 | for (i = 0; i < SAHARA_MAX_HW_DESC; i++) { |
401 | dev_dbg(dev->device, "Descriptor (%d) (%pad):\n" , |
402 | i, &dev->hw_phys_desc[i]); |
403 | dev_dbg(dev->device, "\thdr = 0x%08x\n" , dev->hw_desc[i]->hdr); |
404 | dev_dbg(dev->device, "\tlen1 = %u\n" , dev->hw_desc[i]->len1); |
405 | dev_dbg(dev->device, "\tp1 = 0x%08x\n" , dev->hw_desc[i]->p1); |
406 | dev_dbg(dev->device, "\tlen2 = %u\n" , dev->hw_desc[i]->len2); |
407 | dev_dbg(dev->device, "\tp2 = 0x%08x\n" , dev->hw_desc[i]->p2); |
408 | dev_dbg(dev->device, "\tnext = 0x%08x\n" , |
409 | dev->hw_desc[i]->next); |
410 | } |
411 | dev_dbg(dev->device, "\n" ); |
412 | } |
413 | |
414 | static void sahara_dump_links(struct sahara_dev *dev) |
415 | { |
416 | int i; |
417 | |
418 | if (!__is_defined(DEBUG)) |
419 | return; |
420 | |
421 | for (i = 0; i < SAHARA_MAX_HW_LINK; i++) { |
422 | dev_dbg(dev->device, "Link (%d) (%pad):\n" , |
423 | i, &dev->hw_phys_link[i]); |
424 | dev_dbg(dev->device, "\tlen = %u\n" , dev->hw_link[i]->len); |
425 | dev_dbg(dev->device, "\tp = 0x%08x\n" , dev->hw_link[i]->p); |
426 | dev_dbg(dev->device, "\tnext = 0x%08x\n" , |
427 | dev->hw_link[i]->next); |
428 | } |
429 | dev_dbg(dev->device, "\n" ); |
430 | } |
431 | |
432 | static int sahara_hw_descriptor_create(struct sahara_dev *dev) |
433 | { |
434 | struct sahara_ctx *ctx = dev->ctx; |
435 | struct scatterlist *sg; |
436 | int ret; |
437 | int i, j; |
438 | int idx = 0; |
439 | u32 len; |
440 | |
441 | memcpy(dev->key_base, ctx->key, ctx->keylen); |
442 | |
443 | if (dev->flags & FLAGS_CBC) { |
444 | dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE; |
445 | dev->hw_desc[idx]->p1 = dev->iv_phys_base; |
446 | } else { |
447 | dev->hw_desc[idx]->len1 = 0; |
448 | dev->hw_desc[idx]->p1 = 0; |
449 | } |
450 | dev->hw_desc[idx]->len2 = ctx->keylen; |
451 | dev->hw_desc[idx]->p2 = dev->key_phys_base; |
452 | dev->hw_desc[idx]->next = dev->hw_phys_desc[1]; |
453 | dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev); |
454 | |
455 | idx++; |
456 | |
457 | |
458 | dev->nb_in_sg = sg_nents_for_len(sg: dev->in_sg, len: dev->total); |
459 | if (dev->nb_in_sg < 0) { |
460 | dev_err(dev->device, "Invalid numbers of src SG.\n" ); |
461 | return dev->nb_in_sg; |
462 | } |
463 | dev->nb_out_sg = sg_nents_for_len(sg: dev->out_sg, len: dev->total); |
464 | if (dev->nb_out_sg < 0) { |
465 | dev_err(dev->device, "Invalid numbers of dst SG.\n" ); |
466 | return dev->nb_out_sg; |
467 | } |
468 | if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) { |
469 | dev_err(dev->device, "not enough hw links (%d)\n" , |
470 | dev->nb_in_sg + dev->nb_out_sg); |
471 | return -EINVAL; |
472 | } |
473 | |
474 | ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, |
475 | DMA_TO_DEVICE); |
476 | if (!ret) { |
477 | dev_err(dev->device, "couldn't map in sg\n" ); |
478 | return -EINVAL; |
479 | } |
480 | |
481 | ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg, |
482 | DMA_FROM_DEVICE); |
483 | if (!ret) { |
484 | dev_err(dev->device, "couldn't map out sg\n" ); |
485 | goto unmap_in; |
486 | } |
487 | |
488 | /* Create input links */ |
489 | dev->hw_desc[idx]->p1 = dev->hw_phys_link[0]; |
490 | sg = dev->in_sg; |
491 | len = dev->total; |
492 | for (i = 0; i < dev->nb_in_sg; i++) { |
493 | dev->hw_link[i]->len = min(len, sg->length); |
494 | dev->hw_link[i]->p = sg->dma_address; |
495 | if (i == (dev->nb_in_sg - 1)) { |
496 | dev->hw_link[i]->next = 0; |
497 | } else { |
498 | len -= min(len, sg->length); |
499 | dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; |
500 | sg = sg_next(sg); |
501 | } |
502 | } |
503 | |
504 | /* Create output links */ |
505 | dev->hw_desc[idx]->p2 = dev->hw_phys_link[i]; |
506 | sg = dev->out_sg; |
507 | len = dev->total; |
508 | for (j = i; j < dev->nb_out_sg + i; j++) { |
509 | dev->hw_link[j]->len = min(len, sg->length); |
510 | dev->hw_link[j]->p = sg->dma_address; |
511 | if (j == (dev->nb_out_sg + i - 1)) { |
512 | dev->hw_link[j]->next = 0; |
513 | } else { |
514 | len -= min(len, sg->length); |
515 | dev->hw_link[j]->next = dev->hw_phys_link[j + 1]; |
516 | sg = sg_next(sg); |
517 | } |
518 | } |
519 | |
520 | /* Fill remaining fields of hw_desc[1] */ |
521 | dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev); |
522 | dev->hw_desc[idx]->len1 = dev->total; |
523 | dev->hw_desc[idx]->len2 = dev->total; |
524 | dev->hw_desc[idx]->next = 0; |
525 | |
526 | sahara_dump_descriptors(dev); |
527 | sahara_dump_links(dev); |
528 | |
529 | sahara_write(dev, data: dev->hw_phys_desc[0], SAHARA_REG_DAR); |
530 | |
531 | return 0; |
532 | |
533 | unmap_in: |
534 | dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, |
535 | DMA_TO_DEVICE); |
536 | |
537 | return -EINVAL; |
538 | } |
539 | |
540 | static void sahara_aes_cbc_update_iv(struct skcipher_request *req) |
541 | { |
542 | struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); |
543 | struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); |
544 | unsigned int ivsize = crypto_skcipher_ivsize(tfm: skcipher); |
545 | |
546 | /* Update IV buffer to contain the last ciphertext block */ |
547 | if (rctx->mode & FLAGS_ENCRYPT) { |
548 | sg_pcopy_to_buffer(sgl: req->dst, nents: sg_nents(sg: req->dst), buf: req->iv, |
549 | buflen: ivsize, skip: req->cryptlen - ivsize); |
550 | } else { |
551 | memcpy(req->iv, rctx->iv_out, ivsize); |
552 | } |
553 | } |
554 | |
555 | static int sahara_aes_process(struct skcipher_request *req) |
556 | { |
557 | struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); |
558 | struct sahara_dev *dev = dev_ptr; |
559 | struct sahara_ctx *ctx; |
560 | struct sahara_aes_reqctx *rctx; |
561 | int ret; |
562 | unsigned long timeout; |
563 | |
564 | /* Request is ready to be dispatched by the device */ |
565 | dev_dbg(dev->device, |
566 | "dispatch request (nbytes=%d, src=%p, dst=%p)\n" , |
567 | req->cryptlen, req->src, req->dst); |
568 | |
569 | /* assign new request to device */ |
570 | dev->total = req->cryptlen; |
571 | dev->in_sg = req->src; |
572 | dev->out_sg = req->dst; |
573 | |
574 | rctx = skcipher_request_ctx(req); |
575 | ctx = crypto_skcipher_ctx(tfm: crypto_skcipher_reqtfm(req)); |
576 | rctx->mode &= FLAGS_MODE_MASK; |
577 | dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode; |
578 | |
579 | if ((dev->flags & FLAGS_CBC) && req->iv) { |
580 | unsigned int ivsize = crypto_skcipher_ivsize(tfm: skcipher); |
581 | |
582 | memcpy(dev->iv_base, req->iv, ivsize); |
583 | |
584 | if (!(dev->flags & FLAGS_ENCRYPT)) { |
585 | sg_pcopy_to_buffer(sgl: req->src, nents: sg_nents(sg: req->src), |
586 | buf: rctx->iv_out, buflen: ivsize, |
587 | skip: req->cryptlen - ivsize); |
588 | } |
589 | } |
590 | |
591 | /* assign new context to device */ |
592 | dev->ctx = ctx; |
593 | |
594 | reinit_completion(x: &dev->dma_completion); |
595 | |
596 | ret = sahara_hw_descriptor_create(dev); |
597 | if (ret) |
598 | return -EINVAL; |
599 | |
600 | timeout = wait_for_completion_timeout(x: &dev->dma_completion, |
601 | timeout: msecs_to_jiffies(SAHARA_TIMEOUT_MS)); |
602 | |
603 | dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, |
604 | DMA_FROM_DEVICE); |
605 | dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, |
606 | DMA_TO_DEVICE); |
607 | |
608 | if (!timeout) { |
609 | dev_err(dev->device, "AES timeout\n" ); |
610 | return -ETIMEDOUT; |
611 | } |
612 | |
613 | if ((dev->flags & FLAGS_CBC) && req->iv) |
614 | sahara_aes_cbc_update_iv(req); |
615 | |
616 | return 0; |
617 | } |
618 | |
619 | static int sahara_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, |
620 | unsigned int keylen) |
621 | { |
622 | struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm); |
623 | |
624 | ctx->keylen = keylen; |
625 | |
626 | /* SAHARA only supports 128bit keys */ |
627 | if (keylen == AES_KEYSIZE_128) { |
628 | memcpy(ctx->key, key, keylen); |
629 | return 0; |
630 | } |
631 | |
632 | if (keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) |
633 | return -EINVAL; |
634 | |
635 | /* |
636 | * The requested key size is not supported by HW, do a fallback. |
637 | */ |
638 | crypto_skcipher_clear_flags(tfm: ctx->fallback, CRYPTO_TFM_REQ_MASK); |
639 | crypto_skcipher_set_flags(tfm: ctx->fallback, flags: tfm->base.crt_flags & |
640 | CRYPTO_TFM_REQ_MASK); |
641 | return crypto_skcipher_setkey(tfm: ctx->fallback, key, keylen); |
642 | } |
643 | |
644 | static int sahara_aes_fallback(struct skcipher_request *req, unsigned long mode) |
645 | { |
646 | struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); |
647 | struct sahara_ctx *ctx = crypto_skcipher_ctx( |
648 | tfm: crypto_skcipher_reqtfm(req)); |
649 | |
650 | skcipher_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback); |
651 | skcipher_request_set_callback(req: &rctx->fallback_req, |
652 | flags: req->base.flags, |
653 | compl: req->base.complete, |
654 | data: req->base.data); |
655 | skcipher_request_set_crypt(req: &rctx->fallback_req, src: req->src, |
656 | dst: req->dst, cryptlen: req->cryptlen, iv: req->iv); |
657 | |
658 | if (mode & FLAGS_ENCRYPT) |
659 | return crypto_skcipher_encrypt(req: &rctx->fallback_req); |
660 | |
661 | return crypto_skcipher_decrypt(req: &rctx->fallback_req); |
662 | } |
663 | |
664 | static int sahara_aes_crypt(struct skcipher_request *req, unsigned long mode) |
665 | { |
666 | struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); |
667 | struct sahara_ctx *ctx = crypto_skcipher_ctx( |
668 | tfm: crypto_skcipher_reqtfm(req)); |
669 | struct sahara_dev *dev = dev_ptr; |
670 | |
671 | if (!req->cryptlen) |
672 | return 0; |
673 | |
674 | if (unlikely(ctx->keylen != AES_KEYSIZE_128)) |
675 | return sahara_aes_fallback(req, mode); |
676 | |
677 | dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n" , |
678 | req->cryptlen, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC)); |
679 | |
680 | if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) |
681 | return -EINVAL; |
682 | |
683 | rctx->mode = mode; |
684 | |
685 | return crypto_transfer_skcipher_request_to_engine(engine: dev->engine, req); |
686 | } |
687 | |
688 | static int sahara_aes_ecb_encrypt(struct skcipher_request *req) |
689 | { |
690 | return sahara_aes_crypt(req, FLAGS_ENCRYPT); |
691 | } |
692 | |
693 | static int sahara_aes_ecb_decrypt(struct skcipher_request *req) |
694 | { |
695 | return sahara_aes_crypt(req, mode: 0); |
696 | } |
697 | |
698 | static int sahara_aes_cbc_encrypt(struct skcipher_request *req) |
699 | { |
700 | return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC); |
701 | } |
702 | |
703 | static int sahara_aes_cbc_decrypt(struct skcipher_request *req) |
704 | { |
705 | return sahara_aes_crypt(req, FLAGS_CBC); |
706 | } |
707 | |
708 | static int sahara_aes_init_tfm(struct crypto_skcipher *tfm) |
709 | { |
710 | const char *name = crypto_tfm_alg_name(tfm: &tfm->base); |
711 | struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm); |
712 | |
713 | ctx->fallback = crypto_alloc_skcipher(alg_name: name, type: 0, |
714 | CRYPTO_ALG_NEED_FALLBACK); |
715 | if (IS_ERR(ptr: ctx->fallback)) { |
716 | pr_err("Error allocating fallback algo %s\n" , name); |
717 | return PTR_ERR(ptr: ctx->fallback); |
718 | } |
719 | |
720 | crypto_skcipher_set_reqsize(skcipher: tfm, reqsize: sizeof(struct sahara_aes_reqctx) + |
721 | crypto_skcipher_reqsize(tfm: ctx->fallback)); |
722 | |
723 | return 0; |
724 | } |
725 | |
726 | static void sahara_aes_exit_tfm(struct crypto_skcipher *tfm) |
727 | { |
728 | struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm); |
729 | |
730 | crypto_free_skcipher(tfm: ctx->fallback); |
731 | } |
732 | |
733 | static u32 sahara_sha_init_hdr(struct sahara_dev *dev, |
734 | struct sahara_sha_reqctx *rctx) |
735 | { |
736 | u32 hdr = 0; |
737 | |
738 | hdr = rctx->mode; |
739 | |
740 | if (rctx->first) { |
741 | hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH; |
742 | hdr |= SAHARA_HDR_MDHA_INIT; |
743 | } else { |
744 | hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY; |
745 | } |
746 | |
747 | if (rctx->last) |
748 | hdr |= SAHARA_HDR_MDHA_PDATA; |
749 | |
750 | if (hweight_long(w: hdr) % 2 == 0) |
751 | hdr |= SAHARA_HDR_PARITY_BIT; |
752 | |
753 | return hdr; |
754 | } |
755 | |
756 | static int sahara_sha_hw_links_create(struct sahara_dev *dev, |
757 | struct sahara_sha_reqctx *rctx, |
758 | int start) |
759 | { |
760 | struct scatterlist *sg; |
761 | unsigned int len; |
762 | unsigned int i; |
763 | int ret; |
764 | |
765 | dev->in_sg = rctx->in_sg; |
766 | |
767 | dev->nb_in_sg = sg_nents_for_len(sg: dev->in_sg, len: rctx->total); |
768 | if (dev->nb_in_sg < 0) { |
769 | dev_err(dev->device, "Invalid numbers of src SG.\n" ); |
770 | return dev->nb_in_sg; |
771 | } |
772 | if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) { |
773 | dev_err(dev->device, "not enough hw links (%d)\n" , |
774 | dev->nb_in_sg + dev->nb_out_sg); |
775 | return -EINVAL; |
776 | } |
777 | |
778 | sg = dev->in_sg; |
779 | ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); |
780 | if (!ret) |
781 | return -EFAULT; |
782 | |
783 | len = rctx->total; |
784 | for (i = start; i < dev->nb_in_sg + start; i++) { |
785 | dev->hw_link[i]->len = min(len, sg->length); |
786 | dev->hw_link[i]->p = sg->dma_address; |
787 | if (i == (dev->nb_in_sg + start - 1)) { |
788 | dev->hw_link[i]->next = 0; |
789 | } else { |
790 | len -= min(len, sg->length); |
791 | dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; |
792 | sg = sg_next(sg); |
793 | } |
794 | } |
795 | |
796 | return i; |
797 | } |
798 | |
799 | static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev, |
800 | struct sahara_sha_reqctx *rctx, |
801 | struct ahash_request *req, |
802 | int index) |
803 | { |
804 | unsigned result_len; |
805 | int i = index; |
806 | |
807 | if (rctx->first) |
808 | /* Create initial descriptor: #8*/ |
809 | dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx); |
810 | else |
811 | /* Create hash descriptor: #10. Must follow #6. */ |
812 | dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH; |
813 | |
814 | dev->hw_desc[index]->len1 = rctx->total; |
815 | if (dev->hw_desc[index]->len1 == 0) { |
816 | /* if len1 is 0, p1 must be 0, too */ |
817 | dev->hw_desc[index]->p1 = 0; |
818 | rctx->sg_in_idx = 0; |
819 | } else { |
820 | /* Create input links */ |
821 | dev->hw_desc[index]->p1 = dev->hw_phys_link[index]; |
822 | i = sahara_sha_hw_links_create(dev, rctx, start: index); |
823 | |
824 | rctx->sg_in_idx = index; |
825 | if (i < 0) |
826 | return i; |
827 | } |
828 | |
829 | dev->hw_desc[index]->p2 = dev->hw_phys_link[i]; |
830 | |
831 | /* Save the context for the next operation */ |
832 | result_len = rctx->context_size; |
833 | dev->hw_link[i]->p = dev->context_phys_base; |
834 | |
835 | dev->hw_link[i]->len = result_len; |
836 | dev->hw_desc[index]->len2 = result_len; |
837 | |
838 | dev->hw_link[i]->next = 0; |
839 | |
840 | return 0; |
841 | } |
842 | |
843 | /* |
844 | * Load descriptor aka #6 |
845 | * |
846 | * To load a previously saved context back to the MDHA unit |
847 | * |
848 | * p1: Saved Context |
849 | * p2: NULL |
850 | * |
851 | */ |
852 | static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev, |
853 | struct sahara_sha_reqctx *rctx, |
854 | struct ahash_request *req, |
855 | int index) |
856 | { |
857 | dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx); |
858 | |
859 | dev->hw_desc[index]->len1 = rctx->context_size; |
860 | dev->hw_desc[index]->p1 = dev->hw_phys_link[index]; |
861 | dev->hw_desc[index]->len2 = 0; |
862 | dev->hw_desc[index]->p2 = 0; |
863 | |
864 | dev->hw_link[index]->len = rctx->context_size; |
865 | dev->hw_link[index]->p = dev->context_phys_base; |
866 | dev->hw_link[index]->next = 0; |
867 | |
868 | return 0; |
869 | } |
870 | |
871 | static int sahara_sha_prepare_request(struct ahash_request *req) |
872 | { |
873 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
874 | struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); |
875 | unsigned int hash_later; |
876 | unsigned int block_size; |
877 | unsigned int len; |
878 | |
879 | block_size = crypto_tfm_alg_blocksize(tfm: crypto_ahash_tfm(tfm)); |
880 | |
881 | /* append bytes from previous operation */ |
882 | len = rctx->buf_cnt + req->nbytes; |
883 | |
884 | /* only the last transfer can be padded in hardware */ |
885 | if (!rctx->last && (len < block_size)) { |
886 | /* to few data, save for next operation */ |
887 | scatterwalk_map_and_copy(buf: rctx->buf + rctx->buf_cnt, sg: req->src, |
888 | start: 0, nbytes: req->nbytes, out: 0); |
889 | rctx->buf_cnt += req->nbytes; |
890 | |
891 | return 0; |
892 | } |
893 | |
894 | /* add data from previous operation first */ |
895 | if (rctx->buf_cnt) |
896 | memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt); |
897 | |
898 | /* data must always be a multiple of block_size */ |
899 | hash_later = rctx->last ? 0 : len & (block_size - 1); |
900 | if (hash_later) { |
901 | unsigned int offset = req->nbytes - hash_later; |
902 | /* Save remaining bytes for later use */ |
903 | scatterwalk_map_and_copy(buf: rctx->buf, sg: req->src, start: offset, |
904 | nbytes: hash_later, out: 0); |
905 | } |
906 | |
907 | rctx->total = len - hash_later; |
908 | /* have data from previous operation and current */ |
909 | if (rctx->buf_cnt && req->nbytes) { |
910 | sg_init_table(rctx->in_sg_chain, 2); |
911 | sg_set_buf(sg: rctx->in_sg_chain, buf: rctx->rembuf, buflen: rctx->buf_cnt); |
912 | sg_chain(prv: rctx->in_sg_chain, prv_nents: 2, sgl: req->src); |
913 | rctx->in_sg = rctx->in_sg_chain; |
914 | /* only data from previous operation */ |
915 | } else if (rctx->buf_cnt) { |
916 | rctx->in_sg = rctx->in_sg_chain; |
917 | sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt); |
918 | /* no data from previous operation */ |
919 | } else { |
920 | rctx->in_sg = req->src; |
921 | } |
922 | |
923 | /* on next call, we only have the remaining data in the buffer */ |
924 | rctx->buf_cnt = hash_later; |
925 | |
926 | return -EINPROGRESS; |
927 | } |
928 | |
929 | static int sahara_sha_process(struct ahash_request *req) |
930 | { |
931 | struct sahara_dev *dev = dev_ptr; |
932 | struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); |
933 | int ret; |
934 | unsigned long timeout; |
935 | |
936 | ret = sahara_sha_prepare_request(req); |
937 | if (!ret) |
938 | return ret; |
939 | |
940 | if (rctx->first) { |
941 | ret = sahara_sha_hw_data_descriptor_create(dev, rctx, req, index: 0); |
942 | if (ret) |
943 | return ret; |
944 | |
945 | dev->hw_desc[0]->next = 0; |
946 | rctx->first = 0; |
947 | } else { |
948 | memcpy(dev->context_base, rctx->context, rctx->context_size); |
949 | |
950 | sahara_sha_hw_context_descriptor_create(dev, rctx, req, index: 0); |
951 | dev->hw_desc[0]->next = dev->hw_phys_desc[1]; |
952 | ret = sahara_sha_hw_data_descriptor_create(dev, rctx, req, index: 1); |
953 | if (ret) |
954 | return ret; |
955 | |
956 | dev->hw_desc[1]->next = 0; |
957 | } |
958 | |
959 | sahara_dump_descriptors(dev); |
960 | sahara_dump_links(dev); |
961 | |
962 | reinit_completion(x: &dev->dma_completion); |
963 | |
964 | sahara_write(dev, data: dev->hw_phys_desc[0], SAHARA_REG_DAR); |
965 | |
966 | timeout = wait_for_completion_timeout(x: &dev->dma_completion, |
967 | timeout: msecs_to_jiffies(SAHARA_TIMEOUT_MS)); |
968 | |
969 | if (rctx->sg_in_idx) |
970 | dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, |
971 | DMA_TO_DEVICE); |
972 | |
973 | if (!timeout) { |
974 | dev_err(dev->device, "SHA timeout\n" ); |
975 | return -ETIMEDOUT; |
976 | } |
977 | |
978 | memcpy(rctx->context, dev->context_base, rctx->context_size); |
979 | |
980 | if (req->result && rctx->last) |
981 | memcpy(req->result, rctx->context, rctx->digest_size); |
982 | |
983 | return 0; |
984 | } |
985 | |
986 | static int sahara_do_one_request(struct crypto_engine *engine, void *areq) |
987 | { |
988 | struct crypto_async_request *async_req = areq; |
989 | int err; |
990 | |
991 | if (crypto_tfm_alg_type(tfm: async_req->tfm) == CRYPTO_ALG_TYPE_AHASH) { |
992 | struct ahash_request *req = ahash_request_cast(req: async_req); |
993 | |
994 | err = sahara_sha_process(req); |
995 | local_bh_disable(); |
996 | crypto_finalize_hash_request(engine, req, err); |
997 | local_bh_enable(); |
998 | } else { |
999 | struct skcipher_request *req = skcipher_request_cast(req: async_req); |
1000 | |
1001 | err = sahara_aes_process(req: skcipher_request_cast(req: async_req)); |
1002 | local_bh_disable(); |
1003 | crypto_finalize_skcipher_request(engine, req, err); |
1004 | local_bh_enable(); |
1005 | } |
1006 | |
1007 | return 0; |
1008 | } |
1009 | |
1010 | static int sahara_sha_enqueue(struct ahash_request *req, int last) |
1011 | { |
1012 | struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); |
1013 | struct sahara_dev *dev = dev_ptr; |
1014 | |
1015 | if (!req->nbytes && !last) |
1016 | return 0; |
1017 | |
1018 | rctx->last = last; |
1019 | |
1020 | return crypto_transfer_hash_request_to_engine(engine: dev->engine, req); |
1021 | } |
1022 | |
1023 | static int sahara_sha_init(struct ahash_request *req) |
1024 | { |
1025 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
1026 | struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); |
1027 | |
1028 | memset(rctx, 0, sizeof(*rctx)); |
1029 | |
1030 | switch (crypto_ahash_digestsize(tfm)) { |
1031 | case SHA1_DIGEST_SIZE: |
1032 | rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1; |
1033 | rctx->digest_size = SHA1_DIGEST_SIZE; |
1034 | break; |
1035 | case SHA256_DIGEST_SIZE: |
1036 | rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256; |
1037 | rctx->digest_size = SHA256_DIGEST_SIZE; |
1038 | break; |
1039 | default: |
1040 | return -EINVAL; |
1041 | } |
1042 | |
1043 | rctx->context_size = rctx->digest_size + 4; |
1044 | rctx->first = 1; |
1045 | |
1046 | return 0; |
1047 | } |
1048 | |
1049 | static int sahara_sha_update(struct ahash_request *req) |
1050 | { |
1051 | return sahara_sha_enqueue(req, last: 0); |
1052 | } |
1053 | |
1054 | static int sahara_sha_final(struct ahash_request *req) |
1055 | { |
1056 | req->nbytes = 0; |
1057 | return sahara_sha_enqueue(req, last: 1); |
1058 | } |
1059 | |
1060 | static int sahara_sha_finup(struct ahash_request *req) |
1061 | { |
1062 | return sahara_sha_enqueue(req, last: 1); |
1063 | } |
1064 | |
1065 | static int sahara_sha_digest(struct ahash_request *req) |
1066 | { |
1067 | sahara_sha_init(req); |
1068 | |
1069 | return sahara_sha_finup(req); |
1070 | } |
1071 | |
1072 | static int sahara_sha_export(struct ahash_request *req, void *out) |
1073 | { |
1074 | struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); |
1075 | |
1076 | memcpy(out, rctx, sizeof(struct sahara_sha_reqctx)); |
1077 | |
1078 | return 0; |
1079 | } |
1080 | |
1081 | static int sahara_sha_import(struct ahash_request *req, const void *in) |
1082 | { |
1083 | struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); |
1084 | |
1085 | memcpy(rctx, in, sizeof(struct sahara_sha_reqctx)); |
1086 | |
1087 | return 0; |
1088 | } |
1089 | |
1090 | static int sahara_sha_cra_init(struct crypto_tfm *tfm) |
1091 | { |
1092 | crypto_ahash_set_reqsize(tfm: __crypto_ahash_cast(tfm), |
1093 | reqsize: sizeof(struct sahara_sha_reqctx)); |
1094 | |
1095 | return 0; |
1096 | } |
1097 | |
1098 | static struct skcipher_engine_alg aes_algs[] = { |
1099 | { |
1100 | .base = { |
1101 | .base.cra_name = "ecb(aes)" , |
1102 | .base.cra_driver_name = "sahara-ecb-aes" , |
1103 | .base.cra_priority = 300, |
1104 | .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, |
1105 | .base.cra_blocksize = AES_BLOCK_SIZE, |
1106 | .base.cra_ctxsize = sizeof(struct sahara_ctx), |
1107 | .base.cra_alignmask = 0x0, |
1108 | .base.cra_module = THIS_MODULE, |
1109 | |
1110 | .init = sahara_aes_init_tfm, |
1111 | .exit = sahara_aes_exit_tfm, |
1112 | .min_keysize = AES_MIN_KEY_SIZE, |
1113 | .max_keysize = AES_MAX_KEY_SIZE, |
1114 | .setkey = sahara_aes_setkey, |
1115 | .encrypt = sahara_aes_ecb_encrypt, |
1116 | .decrypt = sahara_aes_ecb_decrypt, |
1117 | }, |
1118 | .op = { |
1119 | .do_one_request = sahara_do_one_request, |
1120 | }, |
1121 | }, { |
1122 | .base = { |
1123 | .base.cra_name = "cbc(aes)" , |
1124 | .base.cra_driver_name = "sahara-cbc-aes" , |
1125 | .base.cra_priority = 300, |
1126 | .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, |
1127 | .base.cra_blocksize = AES_BLOCK_SIZE, |
1128 | .base.cra_ctxsize = sizeof(struct sahara_ctx), |
1129 | .base.cra_alignmask = 0x0, |
1130 | .base.cra_module = THIS_MODULE, |
1131 | |
1132 | .init = sahara_aes_init_tfm, |
1133 | .exit = sahara_aes_exit_tfm, |
1134 | .min_keysize = AES_MIN_KEY_SIZE, |
1135 | .max_keysize = AES_MAX_KEY_SIZE, |
1136 | .ivsize = AES_BLOCK_SIZE, |
1137 | .setkey = sahara_aes_setkey, |
1138 | .encrypt = sahara_aes_cbc_encrypt, |
1139 | .decrypt = sahara_aes_cbc_decrypt, |
1140 | }, |
1141 | .op = { |
1142 | .do_one_request = sahara_do_one_request, |
1143 | }, |
1144 | } |
1145 | }; |
1146 | |
1147 | static struct ahash_engine_alg sha_v3_algs[] = { |
1148 | { |
1149 | .base = { |
1150 | .init = sahara_sha_init, |
1151 | .update = sahara_sha_update, |
1152 | .final = sahara_sha_final, |
1153 | .finup = sahara_sha_finup, |
1154 | .digest = sahara_sha_digest, |
1155 | .export = sahara_sha_export, |
1156 | .import = sahara_sha_import, |
1157 | .halg.digestsize = SHA1_DIGEST_SIZE, |
1158 | .halg.statesize = sizeof(struct sahara_sha_reqctx), |
1159 | .halg.base = { |
1160 | .cra_name = "sha1" , |
1161 | .cra_driver_name = "sahara-sha1" , |
1162 | .cra_priority = 300, |
1163 | .cra_flags = CRYPTO_ALG_ASYNC | |
1164 | CRYPTO_ALG_NEED_FALLBACK, |
1165 | .cra_blocksize = SHA1_BLOCK_SIZE, |
1166 | .cra_ctxsize = sizeof(struct sahara_ctx), |
1167 | .cra_alignmask = 0, |
1168 | .cra_module = THIS_MODULE, |
1169 | .cra_init = sahara_sha_cra_init, |
1170 | } |
1171 | }, |
1172 | .op = { |
1173 | .do_one_request = sahara_do_one_request, |
1174 | }, |
1175 | }, |
1176 | }; |
1177 | |
1178 | static struct ahash_engine_alg sha_v4_algs[] = { |
1179 | { |
1180 | .base = { |
1181 | .init = sahara_sha_init, |
1182 | .update = sahara_sha_update, |
1183 | .final = sahara_sha_final, |
1184 | .finup = sahara_sha_finup, |
1185 | .digest = sahara_sha_digest, |
1186 | .export = sahara_sha_export, |
1187 | .import = sahara_sha_import, |
1188 | .halg.digestsize = SHA256_DIGEST_SIZE, |
1189 | .halg.statesize = sizeof(struct sahara_sha_reqctx), |
1190 | .halg.base = { |
1191 | .cra_name = "sha256" , |
1192 | .cra_driver_name = "sahara-sha256" , |
1193 | .cra_priority = 300, |
1194 | .cra_flags = CRYPTO_ALG_ASYNC | |
1195 | CRYPTO_ALG_NEED_FALLBACK, |
1196 | .cra_blocksize = SHA256_BLOCK_SIZE, |
1197 | .cra_ctxsize = sizeof(struct sahara_ctx), |
1198 | .cra_alignmask = 0, |
1199 | .cra_module = THIS_MODULE, |
1200 | .cra_init = sahara_sha_cra_init, |
1201 | } |
1202 | }, |
1203 | .op = { |
1204 | .do_one_request = sahara_do_one_request, |
1205 | }, |
1206 | }, |
1207 | }; |
1208 | |
1209 | static irqreturn_t sahara_irq_handler(int irq, void *data) |
1210 | { |
1211 | struct sahara_dev *dev = data; |
1212 | unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS); |
1213 | unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS); |
1214 | |
1215 | sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR, |
1216 | SAHARA_REG_CMD); |
1217 | |
1218 | sahara_decode_status(dev, status: stat); |
1219 | |
1220 | if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) |
1221 | return IRQ_NONE; |
1222 | |
1223 | if (SAHARA_STATUS_GET_STATE(stat) != SAHARA_STATE_COMPLETE) |
1224 | sahara_decode_error(dev, error: err); |
1225 | |
1226 | complete(&dev->dma_completion); |
1227 | |
1228 | return IRQ_HANDLED; |
1229 | } |
1230 | |
1231 | |
1232 | static int sahara_register_algs(struct sahara_dev *dev) |
1233 | { |
1234 | int err; |
1235 | |
1236 | err = crypto_engine_register_skciphers(algs: aes_algs, ARRAY_SIZE(aes_algs)); |
1237 | if (err) |
1238 | return err; |
1239 | |
1240 | err = crypto_engine_register_ahashes(algs: sha_v3_algs, |
1241 | ARRAY_SIZE(sha_v3_algs)); |
1242 | if (err) |
1243 | goto err_aes_algs; |
1244 | |
1245 | if (dev->version > SAHARA_VERSION_3) { |
1246 | err = crypto_engine_register_ahashes(algs: sha_v4_algs, |
1247 | ARRAY_SIZE(sha_v4_algs)); |
1248 | if (err) |
1249 | goto err_sha_v3_algs; |
1250 | } |
1251 | |
1252 | return 0; |
1253 | |
1254 | err_sha_v3_algs: |
1255 | crypto_engine_unregister_ahashes(algs: sha_v3_algs, ARRAY_SIZE(sha_v3_algs)); |
1256 | |
1257 | err_aes_algs: |
1258 | crypto_engine_unregister_skciphers(algs: aes_algs, ARRAY_SIZE(aes_algs)); |
1259 | |
1260 | return err; |
1261 | } |
1262 | |
1263 | static void sahara_unregister_algs(struct sahara_dev *dev) |
1264 | { |
1265 | crypto_engine_unregister_skciphers(algs: aes_algs, ARRAY_SIZE(aes_algs)); |
1266 | crypto_engine_unregister_ahashes(algs: sha_v3_algs, ARRAY_SIZE(sha_v3_algs)); |
1267 | |
1268 | if (dev->version > SAHARA_VERSION_3) |
1269 | crypto_engine_unregister_ahashes(algs: sha_v4_algs, |
1270 | ARRAY_SIZE(sha_v4_algs)); |
1271 | } |
1272 | |
1273 | static const struct of_device_id sahara_dt_ids[] = { |
1274 | { .compatible = "fsl,imx53-sahara" }, |
1275 | { .compatible = "fsl,imx27-sahara" }, |
1276 | { /* sentinel */ } |
1277 | }; |
1278 | MODULE_DEVICE_TABLE(of, sahara_dt_ids); |
1279 | |
1280 | static int sahara_probe(struct platform_device *pdev) |
1281 | { |
1282 | struct sahara_dev *dev; |
1283 | u32 version; |
1284 | int irq; |
1285 | int err; |
1286 | int i; |
1287 | |
1288 | dev = devm_kzalloc(dev: &pdev->dev, size: sizeof(*dev), GFP_KERNEL); |
1289 | if (!dev) |
1290 | return -ENOMEM; |
1291 | |
1292 | dev->device = &pdev->dev; |
1293 | platform_set_drvdata(pdev, data: dev); |
1294 | |
1295 | /* Get the base address */ |
1296 | dev->regs_base = devm_platform_ioremap_resource(pdev, index: 0); |
1297 | if (IS_ERR(ptr: dev->regs_base)) |
1298 | return PTR_ERR(ptr: dev->regs_base); |
1299 | |
1300 | /* Get the IRQ */ |
1301 | irq = platform_get_irq(pdev, 0); |
1302 | if (irq < 0) |
1303 | return irq; |
1304 | |
1305 | err = devm_request_irq(dev: &pdev->dev, irq, handler: sahara_irq_handler, |
1306 | irqflags: 0, devname: dev_name(dev: &pdev->dev), dev_id: dev); |
1307 | if (err) |
1308 | return dev_err_probe(dev: &pdev->dev, err, |
1309 | fmt: "failed to request irq\n" ); |
1310 | |
1311 | /* clocks */ |
1312 | dev->clk_ipg = devm_clk_get_enabled(dev: &pdev->dev, id: "ipg" ); |
1313 | if (IS_ERR(ptr: dev->clk_ipg)) |
1314 | return dev_err_probe(dev: &pdev->dev, err: PTR_ERR(ptr: dev->clk_ipg), |
1315 | fmt: "Could not get ipg clock\n" ); |
1316 | |
1317 | dev->clk_ahb = devm_clk_get_enabled(dev: &pdev->dev, id: "ahb" ); |
1318 | if (IS_ERR(ptr: dev->clk_ahb)) |
1319 | return dev_err_probe(dev: &pdev->dev, err: PTR_ERR(ptr: dev->clk_ahb), |
1320 | fmt: "Could not get ahb clock\n" ); |
1321 | |
1322 | /* Allocate HW descriptors */ |
1323 | dev->hw_desc[0] = dmam_alloc_coherent(dev: &pdev->dev, |
1324 | SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), |
1325 | dma_handle: &dev->hw_phys_desc[0], GFP_KERNEL); |
1326 | if (!dev->hw_desc[0]) |
1327 | return -ENOMEM; |
1328 | dev->hw_desc[1] = dev->hw_desc[0] + 1; |
1329 | dev->hw_phys_desc[1] = dev->hw_phys_desc[0] + |
1330 | sizeof(struct sahara_hw_desc); |
1331 | |
1332 | /* Allocate space for iv and key */ |
1333 | dev->key_base = dmam_alloc_coherent(dev: &pdev->dev, size: 2 * AES_KEYSIZE_128, |
1334 | dma_handle: &dev->key_phys_base, GFP_KERNEL); |
1335 | if (!dev->key_base) |
1336 | return -ENOMEM; |
1337 | dev->iv_base = dev->key_base + AES_KEYSIZE_128; |
1338 | dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128; |
1339 | |
1340 | /* Allocate space for context: largest digest + message length field */ |
1341 | dev->context_base = dmam_alloc_coherent(dev: &pdev->dev, |
1342 | SHA256_DIGEST_SIZE + 4, |
1343 | dma_handle: &dev->context_phys_base, GFP_KERNEL); |
1344 | if (!dev->context_base) |
1345 | return -ENOMEM; |
1346 | |
1347 | /* Allocate space for HW links */ |
1348 | dev->hw_link[0] = dmam_alloc_coherent(dev: &pdev->dev, |
1349 | SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), |
1350 | dma_handle: &dev->hw_phys_link[0], GFP_KERNEL); |
1351 | if (!dev->hw_link[0]) |
1352 | return -ENOMEM; |
1353 | for (i = 1; i < SAHARA_MAX_HW_LINK; i++) { |
1354 | dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] + |
1355 | sizeof(struct sahara_hw_link); |
1356 | dev->hw_link[i] = dev->hw_link[i - 1] + 1; |
1357 | } |
1358 | |
1359 | dev_ptr = dev; |
1360 | |
1361 | dev->engine = crypto_engine_alloc_init(dev: &pdev->dev, rt: true); |
1362 | if (!dev->engine) |
1363 | return -ENOMEM; |
1364 | |
1365 | err = crypto_engine_start(engine: dev->engine); |
1366 | if (err) { |
1367 | crypto_engine_exit(engine: dev->engine); |
1368 | return dev_err_probe(dev: &pdev->dev, err, |
1369 | fmt: "Could not start crypto engine\n" ); |
1370 | } |
1371 | |
1372 | init_completion(x: &dev->dma_completion); |
1373 | |
1374 | version = sahara_read(dev, SAHARA_REG_VERSION); |
1375 | if (of_device_is_compatible(device: pdev->dev.of_node, "fsl,imx27-sahara" )) { |
1376 | if (version != SAHARA_VERSION_3) |
1377 | err = -ENODEV; |
1378 | } else if (of_device_is_compatible(device: pdev->dev.of_node, |
1379 | "fsl,imx53-sahara" )) { |
1380 | if (((version >> 8) & 0xff) != SAHARA_VERSION_4) |
1381 | err = -ENODEV; |
1382 | version = (version >> 8) & 0xff; |
1383 | } |
1384 | if (err == -ENODEV) { |
1385 | dev_err_probe(dev: &pdev->dev, err, |
1386 | fmt: "SAHARA version %d not supported\n" , version); |
1387 | goto err_algs; |
1388 | } |
1389 | |
1390 | dev->version = version; |
1391 | |
1392 | sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH, |
1393 | SAHARA_REG_CMD); |
1394 | sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) | |
1395 | SAHARA_CONTROL_SET_MAXBURST(8) | |
1396 | SAHARA_CONTROL_RNG_AUTORSD | |
1397 | SAHARA_CONTROL_ENABLE_INT, |
1398 | SAHARA_REG_CONTROL); |
1399 | |
1400 | err = sahara_register_algs(dev); |
1401 | if (err) |
1402 | goto err_algs; |
1403 | |
1404 | dev_info(&pdev->dev, "SAHARA version %d initialized\n" , version); |
1405 | |
1406 | return 0; |
1407 | |
1408 | err_algs: |
1409 | crypto_engine_exit(engine: dev->engine); |
1410 | |
1411 | return err; |
1412 | } |
1413 | |
1414 | static void sahara_remove(struct platform_device *pdev) |
1415 | { |
1416 | struct sahara_dev *dev = platform_get_drvdata(pdev); |
1417 | |
1418 | crypto_engine_exit(engine: dev->engine); |
1419 | sahara_unregister_algs(dev); |
1420 | } |
1421 | |
1422 | static struct platform_driver sahara_driver = { |
1423 | .probe = sahara_probe, |
1424 | .remove_new = sahara_remove, |
1425 | .driver = { |
1426 | .name = SAHARA_NAME, |
1427 | .of_match_table = sahara_dt_ids, |
1428 | }, |
1429 | }; |
1430 | |
1431 | module_platform_driver(sahara_driver); |
1432 | |
1433 | MODULE_LICENSE("GPL" ); |
1434 | MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>" ); |
1435 | MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>" ); |
1436 | MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator" ); |
1437 | |