1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * talitos - Freescale Integrated Security Engine (SEC) device driver |
4 | * |
5 | * Copyright (c) 2008-2011 Freescale Semiconductor, Inc. |
6 | * |
7 | * Scatterlist Crypto API glue code copied from files with the following: |
8 | * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au> |
9 | * |
10 | * Crypto algorithm registration code copied from hifn driver: |
11 | * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> |
12 | * All rights reserved. |
13 | */ |
14 | |
15 | #include <linux/kernel.h> |
16 | #include <linux/module.h> |
17 | #include <linux/mod_devicetable.h> |
18 | #include <linux/device.h> |
19 | #include <linux/interrupt.h> |
20 | #include <linux/crypto.h> |
21 | #include <linux/hw_random.h> |
22 | #include <linux/of.h> |
23 | #include <linux/of_irq.h> |
24 | #include <linux/platform_device.h> |
25 | #include <linux/dma-mapping.h> |
26 | #include <linux/io.h> |
27 | #include <linux/spinlock.h> |
28 | #include <linux/rtnetlink.h> |
29 | #include <linux/slab.h> |
30 | |
31 | #include <crypto/algapi.h> |
32 | #include <crypto/aes.h> |
33 | #include <crypto/internal/des.h> |
34 | #include <crypto/sha1.h> |
35 | #include <crypto/sha2.h> |
36 | #include <crypto/md5.h> |
37 | #include <crypto/internal/aead.h> |
38 | #include <crypto/authenc.h> |
39 | #include <crypto/internal/skcipher.h> |
40 | #include <crypto/hash.h> |
41 | #include <crypto/internal/hash.h> |
42 | #include <crypto/scatterwalk.h> |
43 | |
44 | #include "talitos.h" |
45 | |
46 | static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, |
47 | unsigned int len, bool is_sec1) |
48 | { |
49 | ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); |
50 | if (is_sec1) { |
51 | ptr->len1 = cpu_to_be16(len); |
52 | } else { |
53 | ptr->len = cpu_to_be16(len); |
54 | ptr->eptr = upper_32_bits(dma_addr); |
55 | } |
56 | } |
57 | |
58 | static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, |
59 | struct talitos_ptr *src_ptr, bool is_sec1) |
60 | { |
61 | dst_ptr->ptr = src_ptr->ptr; |
62 | if (is_sec1) { |
63 | dst_ptr->len1 = src_ptr->len1; |
64 | } else { |
65 | dst_ptr->len = src_ptr->len; |
66 | dst_ptr->eptr = src_ptr->eptr; |
67 | } |
68 | } |
69 | |
70 | static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, |
71 | bool is_sec1) |
72 | { |
73 | if (is_sec1) |
74 | return be16_to_cpu(ptr->len1); |
75 | else |
76 | return be16_to_cpu(ptr->len); |
77 | } |
78 | |
79 | static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val, |
80 | bool is_sec1) |
81 | { |
82 | if (!is_sec1) |
83 | ptr->j_extent = val; |
84 | } |
85 | |
86 | static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1) |
87 | { |
88 | if (!is_sec1) |
89 | ptr->j_extent |= val; |
90 | } |
91 | |
92 | /* |
93 | * map virtual single (contiguous) pointer to h/w descriptor pointer |
94 | */ |
95 | static void __map_single_talitos_ptr(struct device *dev, |
96 | struct talitos_ptr *ptr, |
97 | unsigned int len, void *data, |
98 | enum dma_data_direction dir, |
99 | unsigned long attrs) |
100 | { |
101 | dma_addr_t dma_addr = dma_map_single_attrs(dev, ptr: data, size: len, dir, attrs); |
102 | struct talitos_private *priv = dev_get_drvdata(dev); |
103 | bool is_sec1 = has_ftr_sec1(priv); |
104 | |
105 | to_talitos_ptr(ptr, dma_addr, len, is_sec1); |
106 | } |
107 | |
108 | static void map_single_talitos_ptr(struct device *dev, |
109 | struct talitos_ptr *ptr, |
110 | unsigned int len, void *data, |
111 | enum dma_data_direction dir) |
112 | { |
113 | __map_single_talitos_ptr(dev, ptr, len, data, dir, attrs: 0); |
114 | } |
115 | |
116 | static void map_single_talitos_ptr_nosync(struct device *dev, |
117 | struct talitos_ptr *ptr, |
118 | unsigned int len, void *data, |
119 | enum dma_data_direction dir) |
120 | { |
121 | __map_single_talitos_ptr(dev, ptr, len, data, dir, |
122 | DMA_ATTR_SKIP_CPU_SYNC); |
123 | } |
124 | |
125 | /* |
126 | * unmap bus single (contiguous) h/w descriptor pointer |
127 | */ |
128 | static void unmap_single_talitos_ptr(struct device *dev, |
129 | struct talitos_ptr *ptr, |
130 | enum dma_data_direction dir) |
131 | { |
132 | struct talitos_private *priv = dev_get_drvdata(dev); |
133 | bool is_sec1 = has_ftr_sec1(priv); |
134 | |
135 | dma_unmap_single(dev, be32_to_cpu(ptr->ptr), |
136 | from_talitos_ptr_len(ptr, is_sec1), dir); |
137 | } |
138 | |
139 | static int reset_channel(struct device *dev, int ch) |
140 | { |
141 | struct talitos_private *priv = dev_get_drvdata(dev); |
142 | unsigned int timeout = TALITOS_TIMEOUT; |
143 | bool is_sec1 = has_ftr_sec1(priv); |
144 | |
145 | if (is_sec1) { |
146 | setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, |
147 | TALITOS1_CCCR_LO_RESET); |
148 | |
149 | while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) & |
150 | TALITOS1_CCCR_LO_RESET) && --timeout) |
151 | cpu_relax(); |
152 | } else { |
153 | setbits32(priv->chan[ch].reg + TALITOS_CCCR, |
154 | TALITOS2_CCCR_RESET); |
155 | |
156 | while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & |
157 | TALITOS2_CCCR_RESET) && --timeout) |
158 | cpu_relax(); |
159 | } |
160 | |
161 | if (timeout == 0) { |
162 | dev_err(dev, "failed to reset channel %d\n" , ch); |
163 | return -EIO; |
164 | } |
165 | |
166 | /* set 36-bit addressing, done writeback enable and done IRQ enable */ |
167 | setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | |
168 | TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); |
169 | /* enable chaining descriptors */ |
170 | if (is_sec1) |
171 | setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, |
172 | TALITOS_CCCR_LO_NE); |
173 | |
174 | /* and ICCR writeback, if available */ |
175 | if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) |
176 | setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, |
177 | TALITOS_CCCR_LO_IWSE); |
178 | |
179 | return 0; |
180 | } |
181 | |
182 | static int reset_device(struct device *dev) |
183 | { |
184 | struct talitos_private *priv = dev_get_drvdata(dev); |
185 | unsigned int timeout = TALITOS_TIMEOUT; |
186 | bool is_sec1 = has_ftr_sec1(priv); |
187 | u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR; |
188 | |
189 | setbits32(priv->reg + TALITOS_MCR, mcr); |
190 | |
191 | while ((in_be32(priv->reg + TALITOS_MCR) & mcr) |
192 | && --timeout) |
193 | cpu_relax(); |
194 | |
195 | if (priv->irq[1]) { |
196 | mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3; |
197 | setbits32(priv->reg + TALITOS_MCR, mcr); |
198 | } |
199 | |
200 | if (timeout == 0) { |
201 | dev_err(dev, "failed to reset device\n" ); |
202 | return -EIO; |
203 | } |
204 | |
205 | return 0; |
206 | } |
207 | |
208 | /* |
209 | * Reset and initialize the device |
210 | */ |
211 | static int init_device(struct device *dev) |
212 | { |
213 | struct talitos_private *priv = dev_get_drvdata(dev); |
214 | int ch, err; |
215 | bool is_sec1 = has_ftr_sec1(priv); |
216 | |
217 | /* |
218 | * Master reset |
219 | * errata documentation: warning: certain SEC interrupts |
220 | * are not fully cleared by writing the MCR:SWR bit, |
221 | * set bit twice to completely reset |
222 | */ |
223 | err = reset_device(dev); |
224 | if (err) |
225 | return err; |
226 | |
227 | err = reset_device(dev); |
228 | if (err) |
229 | return err; |
230 | |
231 | /* reset channels */ |
232 | for (ch = 0; ch < priv->num_channels; ch++) { |
233 | err = reset_channel(dev, ch); |
234 | if (err) |
235 | return err; |
236 | } |
237 | |
238 | /* enable channel done and error interrupts */ |
239 | if (is_sec1) { |
240 | clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT); |
241 | clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); |
242 | /* disable parity error check in DEU (erroneous? test vect.) */ |
243 | setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE); |
244 | } else { |
245 | setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT); |
246 | setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); |
247 | } |
248 | |
249 | /* disable integrity check error interrupts (use writeback instead) */ |
250 | if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) |
251 | setbits32(priv->reg_mdeu + TALITOS_EUICR_LO, |
252 | TALITOS_MDEUICR_LO_ICE); |
253 | |
254 | return 0; |
255 | } |
256 | |
257 | /** |
258 | * talitos_submit - submits a descriptor to the device for processing |
259 | * @dev: the SEC device to be used |
260 | * @ch: the SEC device channel to be used |
261 | * @desc: the descriptor to be processed by the device |
262 | * @callback: whom to call when processing is complete |
263 | * @context: a handle for use by caller (optional) |
264 | * |
265 | * desc must contain valid dma-mapped (bus physical) address pointers. |
266 | * callback must check err and feedback in descriptor header |
267 | * for device processing status. |
268 | */ |
269 | static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, |
270 | void (*callback)(struct device *dev, |
271 | struct talitos_desc *desc, |
272 | void *context, int error), |
273 | void *context) |
274 | { |
275 | struct talitos_private *priv = dev_get_drvdata(dev); |
276 | struct talitos_request *request; |
277 | unsigned long flags; |
278 | int head; |
279 | bool is_sec1 = has_ftr_sec1(priv); |
280 | |
281 | spin_lock_irqsave(&priv->chan[ch].head_lock, flags); |
282 | |
283 | if (!atomic_inc_not_zero(v: &priv->chan[ch].submit_count)) { |
284 | /* h/w fifo is full */ |
285 | spin_unlock_irqrestore(lock: &priv->chan[ch].head_lock, flags); |
286 | return -EAGAIN; |
287 | } |
288 | |
289 | head = priv->chan[ch].head; |
290 | request = &priv->chan[ch].fifo[head]; |
291 | |
292 | /* map descriptor and save caller data */ |
293 | if (is_sec1) { |
294 | desc->hdr1 = desc->hdr; |
295 | request->dma_desc = dma_map_single(dev, &desc->hdr1, |
296 | TALITOS_DESC_SIZE, |
297 | DMA_BIDIRECTIONAL); |
298 | } else { |
299 | request->dma_desc = dma_map_single(dev, desc, |
300 | TALITOS_DESC_SIZE, |
301 | DMA_BIDIRECTIONAL); |
302 | } |
303 | request->callback = callback; |
304 | request->context = context; |
305 | |
306 | /* increment fifo head */ |
307 | priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1); |
308 | |
309 | smp_wmb(); |
310 | request->desc = desc; |
311 | |
312 | /* GO! */ |
313 | wmb(); |
314 | out_be32(priv->chan[ch].reg + TALITOS_FF, |
315 | upper_32_bits(request->dma_desc)); |
316 | out_be32(priv->chan[ch].reg + TALITOS_FF_LO, |
317 | lower_32_bits(request->dma_desc)); |
318 | |
319 | spin_unlock_irqrestore(lock: &priv->chan[ch].head_lock, flags); |
320 | |
321 | return -EINPROGRESS; |
322 | } |
323 | |
324 | static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1) |
325 | { |
326 | struct talitos_edesc *edesc; |
327 | |
328 | if (!is_sec1) |
329 | return request->desc->hdr; |
330 | |
331 | if (!request->desc->next_desc) |
332 | return request->desc->hdr1; |
333 | |
334 | edesc = container_of(request->desc, struct talitos_edesc, desc); |
335 | |
336 | return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1; |
337 | } |
338 | |
339 | /* |
340 | * process what was done, notify callback of error if not |
341 | */ |
342 | static void flush_channel(struct device *dev, int ch, int error, int reset_ch) |
343 | { |
344 | struct talitos_private *priv = dev_get_drvdata(dev); |
345 | struct talitos_request *request, saved_req; |
346 | unsigned long flags; |
347 | int tail, status; |
348 | bool is_sec1 = has_ftr_sec1(priv); |
349 | |
350 | spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); |
351 | |
352 | tail = priv->chan[ch].tail; |
353 | while (priv->chan[ch].fifo[tail].desc) { |
354 | __be32 hdr; |
355 | |
356 | request = &priv->chan[ch].fifo[tail]; |
357 | |
358 | /* descriptors with their done bits set don't get the error */ |
359 | rmb(); |
360 | hdr = get_request_hdr(request, is_sec1); |
361 | |
362 | if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) |
363 | status = 0; |
364 | else |
365 | if (!error) |
366 | break; |
367 | else |
368 | status = error; |
369 | |
370 | dma_unmap_single(dev, request->dma_desc, |
371 | TALITOS_DESC_SIZE, |
372 | DMA_BIDIRECTIONAL); |
373 | |
374 | /* copy entries so we can call callback outside lock */ |
375 | saved_req.desc = request->desc; |
376 | saved_req.callback = request->callback; |
377 | saved_req.context = request->context; |
378 | |
379 | /* release request entry in fifo */ |
380 | smp_wmb(); |
381 | request->desc = NULL; |
382 | |
383 | /* increment fifo tail */ |
384 | priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1); |
385 | |
386 | spin_unlock_irqrestore(lock: &priv->chan[ch].tail_lock, flags); |
387 | |
388 | atomic_dec(v: &priv->chan[ch].submit_count); |
389 | |
390 | saved_req.callback(dev, saved_req.desc, saved_req.context, |
391 | status); |
392 | /* channel may resume processing in single desc error case */ |
393 | if (error && !reset_ch && status == error) |
394 | return; |
395 | spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); |
396 | tail = priv->chan[ch].tail; |
397 | } |
398 | |
399 | spin_unlock_irqrestore(lock: &priv->chan[ch].tail_lock, flags); |
400 | } |
401 | |
402 | /* |
403 | * process completed requests for channels that have done status |
404 | */ |
405 | #define DEF_TALITOS1_DONE(name, ch_done_mask) \ |
406 | static void talitos1_done_##name(unsigned long data) \ |
407 | { \ |
408 | struct device *dev = (struct device *)data; \ |
409 | struct talitos_private *priv = dev_get_drvdata(dev); \ |
410 | unsigned long flags; \ |
411 | \ |
412 | if (ch_done_mask & 0x10000000) \ |
413 | flush_channel(dev, 0, 0, 0); \ |
414 | if (ch_done_mask & 0x40000000) \ |
415 | flush_channel(dev, 1, 0, 0); \ |
416 | if (ch_done_mask & 0x00010000) \ |
417 | flush_channel(dev, 2, 0, 0); \ |
418 | if (ch_done_mask & 0x00040000) \ |
419 | flush_channel(dev, 3, 0, 0); \ |
420 | \ |
421 | /* At this point, all completed channels have been processed */ \ |
422 | /* Unmask done interrupts for channels completed later on. */ \ |
423 | spin_lock_irqsave(&priv->reg_lock, flags); \ |
424 | clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ |
425 | clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \ |
426 | spin_unlock_irqrestore(&priv->reg_lock, flags); \ |
427 | } |
428 | |
429 | DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE) |
430 | DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE) |
431 | |
432 | #define DEF_TALITOS2_DONE(name, ch_done_mask) \ |
433 | static void talitos2_done_##name(unsigned long data) \ |
434 | { \ |
435 | struct device *dev = (struct device *)data; \ |
436 | struct talitos_private *priv = dev_get_drvdata(dev); \ |
437 | unsigned long flags; \ |
438 | \ |
439 | if (ch_done_mask & 1) \ |
440 | flush_channel(dev, 0, 0, 0); \ |
441 | if (ch_done_mask & (1 << 2)) \ |
442 | flush_channel(dev, 1, 0, 0); \ |
443 | if (ch_done_mask & (1 << 4)) \ |
444 | flush_channel(dev, 2, 0, 0); \ |
445 | if (ch_done_mask & (1 << 6)) \ |
446 | flush_channel(dev, 3, 0, 0); \ |
447 | \ |
448 | /* At this point, all completed channels have been processed */ \ |
449 | /* Unmask done interrupts for channels completed later on. */ \ |
450 | spin_lock_irqsave(&priv->reg_lock, flags); \ |
451 | setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ |
452 | setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \ |
453 | spin_unlock_irqrestore(&priv->reg_lock, flags); \ |
454 | } |
455 | |
456 | DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE) |
457 | DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE) |
458 | DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) |
459 | DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) |
460 | |
461 | /* |
462 | * locate current (offending) descriptor |
463 | */ |
464 | static __be32 current_desc_hdr(struct device *dev, int ch) |
465 | { |
466 | struct talitos_private *priv = dev_get_drvdata(dev); |
467 | int tail, iter; |
468 | dma_addr_t cur_desc; |
469 | |
470 | cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; |
471 | cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); |
472 | |
473 | if (!cur_desc) { |
474 | dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n" ); |
475 | return 0; |
476 | } |
477 | |
478 | tail = priv->chan[ch].tail; |
479 | |
480 | iter = tail; |
481 | while (priv->chan[ch].fifo[iter].dma_desc != cur_desc && |
482 | priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) { |
483 | iter = (iter + 1) & (priv->fifo_len - 1); |
484 | if (iter == tail) { |
485 | dev_err(dev, "couldn't locate current descriptor\n" ); |
486 | return 0; |
487 | } |
488 | } |
489 | |
490 | if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) { |
491 | struct talitos_edesc *edesc; |
492 | |
493 | edesc = container_of(priv->chan[ch].fifo[iter].desc, |
494 | struct talitos_edesc, desc); |
495 | return ((struct talitos_desc *) |
496 | (edesc->buf + edesc->dma_len))->hdr; |
497 | } |
498 | |
499 | return priv->chan[ch].fifo[iter].desc->hdr; |
500 | } |
501 | |
502 | /* |
503 | * user diagnostics; report root cause of error based on execution unit status |
504 | */ |
505 | static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr) |
506 | { |
507 | struct talitos_private *priv = dev_get_drvdata(dev); |
508 | int i; |
509 | |
510 | if (!desc_hdr) |
511 | desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF)); |
512 | |
513 | switch (desc_hdr & DESC_HDR_SEL0_MASK) { |
514 | case DESC_HDR_SEL0_AFEU: |
515 | dev_err(dev, "AFEUISR 0x%08x_%08x\n" , |
516 | in_be32(priv->reg_afeu + TALITOS_EUISR), |
517 | in_be32(priv->reg_afeu + TALITOS_EUISR_LO)); |
518 | break; |
519 | case DESC_HDR_SEL0_DEU: |
520 | dev_err(dev, "DEUISR 0x%08x_%08x\n" , |
521 | in_be32(priv->reg_deu + TALITOS_EUISR), |
522 | in_be32(priv->reg_deu + TALITOS_EUISR_LO)); |
523 | break; |
524 | case DESC_HDR_SEL0_MDEUA: |
525 | case DESC_HDR_SEL0_MDEUB: |
526 | dev_err(dev, "MDEUISR 0x%08x_%08x\n" , |
527 | in_be32(priv->reg_mdeu + TALITOS_EUISR), |
528 | in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); |
529 | break; |
530 | case DESC_HDR_SEL0_RNG: |
531 | dev_err(dev, "RNGUISR 0x%08x_%08x\n" , |
532 | in_be32(priv->reg_rngu + TALITOS_ISR), |
533 | in_be32(priv->reg_rngu + TALITOS_ISR_LO)); |
534 | break; |
535 | case DESC_HDR_SEL0_PKEU: |
536 | dev_err(dev, "PKEUISR 0x%08x_%08x\n" , |
537 | in_be32(priv->reg_pkeu + TALITOS_EUISR), |
538 | in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); |
539 | break; |
540 | case DESC_HDR_SEL0_AESU: |
541 | dev_err(dev, "AESUISR 0x%08x_%08x\n" , |
542 | in_be32(priv->reg_aesu + TALITOS_EUISR), |
543 | in_be32(priv->reg_aesu + TALITOS_EUISR_LO)); |
544 | break; |
545 | case DESC_HDR_SEL0_CRCU: |
546 | dev_err(dev, "CRCUISR 0x%08x_%08x\n" , |
547 | in_be32(priv->reg_crcu + TALITOS_EUISR), |
548 | in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); |
549 | break; |
550 | case DESC_HDR_SEL0_KEU: |
551 | dev_err(dev, "KEUISR 0x%08x_%08x\n" , |
552 | in_be32(priv->reg_pkeu + TALITOS_EUISR), |
553 | in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); |
554 | break; |
555 | } |
556 | |
557 | switch (desc_hdr & DESC_HDR_SEL1_MASK) { |
558 | case DESC_HDR_SEL1_MDEUA: |
559 | case DESC_HDR_SEL1_MDEUB: |
560 | dev_err(dev, "MDEUISR 0x%08x_%08x\n" , |
561 | in_be32(priv->reg_mdeu + TALITOS_EUISR), |
562 | in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); |
563 | break; |
564 | case DESC_HDR_SEL1_CRCU: |
565 | dev_err(dev, "CRCUISR 0x%08x_%08x\n" , |
566 | in_be32(priv->reg_crcu + TALITOS_EUISR), |
567 | in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); |
568 | break; |
569 | } |
570 | |
571 | for (i = 0; i < 8; i++) |
572 | dev_err(dev, "DESCBUF 0x%08x_%08x\n" , |
573 | in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i), |
574 | in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i)); |
575 | } |
576 | |
577 | /* |
578 | * recover from error interrupts |
579 | */ |
580 | static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) |
581 | { |
582 | struct talitos_private *priv = dev_get_drvdata(dev); |
583 | unsigned int timeout = TALITOS_TIMEOUT; |
584 | int ch, error, reset_dev = 0; |
585 | u32 v_lo; |
586 | bool is_sec1 = has_ftr_sec1(priv); |
587 | int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */ |
588 | |
589 | for (ch = 0; ch < priv->num_channels; ch++) { |
590 | /* skip channels without errors */ |
591 | if (is_sec1) { |
592 | /* bits 29, 31, 17, 19 */ |
593 | if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6)))) |
594 | continue; |
595 | } else { |
596 | if (!(isr & (1 << (ch * 2 + 1)))) |
597 | continue; |
598 | } |
599 | |
600 | error = -EINVAL; |
601 | |
602 | v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO); |
603 | |
604 | if (v_lo & TALITOS_CCPSR_LO_DOF) { |
605 | dev_err(dev, "double fetch fifo overflow error\n" ); |
606 | error = -EAGAIN; |
607 | reset_ch = 1; |
608 | } |
609 | if (v_lo & TALITOS_CCPSR_LO_SOF) { |
610 | /* h/w dropped descriptor */ |
611 | dev_err(dev, "single fetch fifo overflow error\n" ); |
612 | error = -EAGAIN; |
613 | } |
614 | if (v_lo & TALITOS_CCPSR_LO_MDTE) |
615 | dev_err(dev, "master data transfer error\n" ); |
616 | if (v_lo & TALITOS_CCPSR_LO_SGDLZ) |
617 | dev_err(dev, is_sec1 ? "pointer not complete error\n" |
618 | : "s/g data length zero error\n" ); |
619 | if (v_lo & TALITOS_CCPSR_LO_FPZ) |
620 | dev_err(dev, is_sec1 ? "parity error\n" |
621 | : "fetch pointer zero error\n" ); |
622 | if (v_lo & TALITOS_CCPSR_LO_IDH) |
623 | dev_err(dev, "illegal descriptor header error\n" ); |
624 | if (v_lo & TALITOS_CCPSR_LO_IEU) |
625 | dev_err(dev, is_sec1 ? "static assignment error\n" |
626 | : "invalid exec unit error\n" ); |
627 | if (v_lo & TALITOS_CCPSR_LO_EU) |
628 | report_eu_error(dev, ch, desc_hdr: current_desc_hdr(dev, ch)); |
629 | if (!is_sec1) { |
630 | if (v_lo & TALITOS_CCPSR_LO_GB) |
631 | dev_err(dev, "gather boundary error\n" ); |
632 | if (v_lo & TALITOS_CCPSR_LO_GRL) |
633 | dev_err(dev, "gather return/length error\n" ); |
634 | if (v_lo & TALITOS_CCPSR_LO_SB) |
635 | dev_err(dev, "scatter boundary error\n" ); |
636 | if (v_lo & TALITOS_CCPSR_LO_SRL) |
637 | dev_err(dev, "scatter return/length error\n" ); |
638 | } |
639 | |
640 | flush_channel(dev, ch, error, reset_ch); |
641 | |
642 | if (reset_ch) { |
643 | reset_channel(dev, ch); |
644 | } else { |
645 | setbits32(priv->chan[ch].reg + TALITOS_CCCR, |
646 | TALITOS2_CCCR_CONT); |
647 | setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0); |
648 | while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & |
649 | TALITOS2_CCCR_CONT) && --timeout) |
650 | cpu_relax(); |
651 | if (timeout == 0) { |
652 | dev_err(dev, "failed to restart channel %d\n" , |
653 | ch); |
654 | reset_dev = 1; |
655 | } |
656 | } |
657 | } |
658 | if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) || |
659 | (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) { |
660 | if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR)) |
661 | dev_err(dev, "TEA error: ISR 0x%08x_%08x\n" , |
662 | isr, isr_lo); |
663 | else |
664 | dev_err(dev, "done overflow, internal time out, or " |
665 | "rngu error: ISR 0x%08x_%08x\n" , isr, isr_lo); |
666 | |
667 | /* purge request queues */ |
668 | for (ch = 0; ch < priv->num_channels; ch++) |
669 | flush_channel(dev, ch, error: -EIO, reset_ch: 1); |
670 | |
671 | /* reset and reinitialize the device */ |
672 | init_device(dev); |
673 | } |
674 | } |
675 | |
676 | #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ |
677 | static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \ |
678 | { \ |
679 | struct device *dev = data; \ |
680 | struct talitos_private *priv = dev_get_drvdata(dev); \ |
681 | u32 isr, isr_lo; \ |
682 | unsigned long flags; \ |
683 | \ |
684 | spin_lock_irqsave(&priv->reg_lock, flags); \ |
685 | isr = in_be32(priv->reg + TALITOS_ISR); \ |
686 | isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ |
687 | /* Acknowledge interrupt */ \ |
688 | out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ |
689 | out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ |
690 | \ |
691 | if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \ |
692 | spin_unlock_irqrestore(&priv->reg_lock, flags); \ |
693 | talitos_error(dev, isr & ch_err_mask, isr_lo); \ |
694 | } \ |
695 | else { \ |
696 | if (likely(isr & ch_done_mask)) { \ |
697 | /* mask further done interrupts. */ \ |
698 | setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ |
699 | /* done_task will unmask done interrupts at exit */ \ |
700 | tasklet_schedule(&priv->done_task[tlet]); \ |
701 | } \ |
702 | spin_unlock_irqrestore(&priv->reg_lock, flags); \ |
703 | } \ |
704 | \ |
705 | return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ |
706 | IRQ_NONE; \ |
707 | } |
708 | |
709 | DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0) |
710 | |
711 | #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ |
712 | static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \ |
713 | { \ |
714 | struct device *dev = data; \ |
715 | struct talitos_private *priv = dev_get_drvdata(dev); \ |
716 | u32 isr, isr_lo; \ |
717 | unsigned long flags; \ |
718 | \ |
719 | spin_lock_irqsave(&priv->reg_lock, flags); \ |
720 | isr = in_be32(priv->reg + TALITOS_ISR); \ |
721 | isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ |
722 | /* Acknowledge interrupt */ \ |
723 | out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ |
724 | out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ |
725 | \ |
726 | if (unlikely(isr & ch_err_mask || isr_lo)) { \ |
727 | spin_unlock_irqrestore(&priv->reg_lock, flags); \ |
728 | talitos_error(dev, isr & ch_err_mask, isr_lo); \ |
729 | } \ |
730 | else { \ |
731 | if (likely(isr & ch_done_mask)) { \ |
732 | /* mask further done interrupts. */ \ |
733 | clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ |
734 | /* done_task will unmask done interrupts at exit */ \ |
735 | tasklet_schedule(&priv->done_task[tlet]); \ |
736 | } \ |
737 | spin_unlock_irqrestore(&priv->reg_lock, flags); \ |
738 | } \ |
739 | \ |
740 | return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ |
741 | IRQ_NONE; \ |
742 | } |
743 | |
744 | DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0) |
745 | DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR, |
746 | 0) |
747 | DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR, |
748 | 1) |
749 | |
750 | /* |
751 | * hwrng |
752 | */ |
753 | static int talitos_rng_data_present(struct hwrng *rng, int wait) |
754 | { |
755 | struct device *dev = (struct device *)rng->priv; |
756 | struct talitos_private *priv = dev_get_drvdata(dev); |
757 | u32 ofl; |
758 | int i; |
759 | |
760 | for (i = 0; i < 20; i++) { |
761 | ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) & |
762 | TALITOS_RNGUSR_LO_OFL; |
763 | if (ofl || !wait) |
764 | break; |
765 | udelay(10); |
766 | } |
767 | |
768 | return !!ofl; |
769 | } |
770 | |
771 | static int talitos_rng_data_read(struct hwrng *rng, u32 *data) |
772 | { |
773 | struct device *dev = (struct device *)rng->priv; |
774 | struct talitos_private *priv = dev_get_drvdata(dev); |
775 | |
776 | /* rng fifo requires 64-bit accesses */ |
777 | *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO); |
778 | *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO); |
779 | |
780 | return sizeof(u32); |
781 | } |
782 | |
783 | static int talitos_rng_init(struct hwrng *rng) |
784 | { |
785 | struct device *dev = (struct device *)rng->priv; |
786 | struct talitos_private *priv = dev_get_drvdata(dev); |
787 | unsigned int timeout = TALITOS_TIMEOUT; |
788 | |
789 | setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR); |
790 | while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO) |
791 | & TALITOS_RNGUSR_LO_RD) |
792 | && --timeout) |
793 | cpu_relax(); |
794 | if (timeout == 0) { |
795 | dev_err(dev, "failed to reset rng hw\n" ); |
796 | return -ENODEV; |
797 | } |
798 | |
799 | /* start generating */ |
800 | setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0); |
801 | |
802 | return 0; |
803 | } |
804 | |
805 | static int talitos_register_rng(struct device *dev) |
806 | { |
807 | struct talitos_private *priv = dev_get_drvdata(dev); |
808 | int err; |
809 | |
810 | priv->rng.name = dev_driver_string(dev); |
811 | priv->rng.init = talitos_rng_init; |
812 | priv->rng.data_present = talitos_rng_data_present; |
813 | priv->rng.data_read = talitos_rng_data_read; |
814 | priv->rng.priv = (unsigned long)dev; |
815 | |
816 | err = hwrng_register(rng: &priv->rng); |
817 | if (!err) |
818 | priv->rng_registered = true; |
819 | |
820 | return err; |
821 | } |
822 | |
823 | static void talitos_unregister_rng(struct device *dev) |
824 | { |
825 | struct talitos_private *priv = dev_get_drvdata(dev); |
826 | |
827 | if (!priv->rng_registered) |
828 | return; |
829 | |
830 | hwrng_unregister(rng: &priv->rng); |
831 | priv->rng_registered = false; |
832 | } |
833 | |
834 | /* |
835 | * crypto alg |
836 | */ |
837 | #define TALITOS_CRA_PRIORITY 3000 |
838 | /* |
839 | * Defines a priority for doing AEAD with descriptors type |
840 | * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP |
841 | */ |
842 | #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) |
843 | #ifdef CONFIG_CRYPTO_DEV_TALITOS2 |
844 | #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE) |
845 | #else |
846 | #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE) |
847 | #endif |
848 | #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ |
849 | |
850 | struct talitos_ctx { |
851 | struct device *dev; |
852 | int ch; |
853 | __be32 desc_hdr_template; |
854 | u8 key[TALITOS_MAX_KEY_SIZE]; |
855 | u8 iv[TALITOS_MAX_IV_LENGTH]; |
856 | dma_addr_t dma_key; |
857 | unsigned int keylen; |
858 | unsigned int enckeylen; |
859 | unsigned int authkeylen; |
860 | }; |
861 | |
862 | #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE |
863 | #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 |
864 | |
865 | struct talitos_ahash_req_ctx { |
866 | u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; |
867 | unsigned int hw_context_size; |
868 | u8 buf[2][HASH_MAX_BLOCK_SIZE]; |
869 | int buf_idx; |
870 | unsigned int swinit; |
871 | unsigned int first; |
872 | unsigned int last; |
873 | unsigned int to_hash_later; |
874 | unsigned int nbuf; |
875 | struct scatterlist bufsl[2]; |
876 | struct scatterlist *psrc; |
877 | }; |
878 | |
879 | struct talitos_export_state { |
880 | u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; |
881 | u8 buf[HASH_MAX_BLOCK_SIZE]; |
882 | unsigned int swinit; |
883 | unsigned int first; |
884 | unsigned int last; |
885 | unsigned int to_hash_later; |
886 | unsigned int nbuf; |
887 | }; |
888 | |
889 | static int aead_setkey(struct crypto_aead *authenc, |
890 | const u8 *key, unsigned int keylen) |
891 | { |
892 | struct talitos_ctx *ctx = crypto_aead_ctx(tfm: authenc); |
893 | struct device *dev = ctx->dev; |
894 | struct crypto_authenc_keys keys; |
895 | |
896 | if (crypto_authenc_extractkeys(keys: &keys, key, keylen) != 0) |
897 | goto badkey; |
898 | |
899 | if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) |
900 | goto badkey; |
901 | |
902 | if (ctx->keylen) |
903 | dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); |
904 | |
905 | memcpy(ctx->key, keys.authkey, keys.authkeylen); |
906 | memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); |
907 | |
908 | ctx->keylen = keys.authkeylen + keys.enckeylen; |
909 | ctx->enckeylen = keys.enckeylen; |
910 | ctx->authkeylen = keys.authkeylen; |
911 | ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, |
912 | DMA_TO_DEVICE); |
913 | |
914 | memzero_explicit(s: &keys, count: sizeof(keys)); |
915 | return 0; |
916 | |
917 | badkey: |
918 | memzero_explicit(s: &keys, count: sizeof(keys)); |
919 | return -EINVAL; |
920 | } |
921 | |
922 | static int aead_des3_setkey(struct crypto_aead *authenc, |
923 | const u8 *key, unsigned int keylen) |
924 | { |
925 | struct talitos_ctx *ctx = crypto_aead_ctx(tfm: authenc); |
926 | struct device *dev = ctx->dev; |
927 | struct crypto_authenc_keys keys; |
928 | int err; |
929 | |
930 | err = crypto_authenc_extractkeys(keys: &keys, key, keylen); |
931 | if (unlikely(err)) |
932 | goto out; |
933 | |
934 | err = -EINVAL; |
935 | if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) |
936 | goto out; |
937 | |
938 | err = verify_aead_des3_key(tfm: authenc, key: keys.enckey, keylen: keys.enckeylen); |
939 | if (err) |
940 | goto out; |
941 | |
942 | if (ctx->keylen) |
943 | dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); |
944 | |
945 | memcpy(ctx->key, keys.authkey, keys.authkeylen); |
946 | memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); |
947 | |
948 | ctx->keylen = keys.authkeylen + keys.enckeylen; |
949 | ctx->enckeylen = keys.enckeylen; |
950 | ctx->authkeylen = keys.authkeylen; |
951 | ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, |
952 | DMA_TO_DEVICE); |
953 | |
954 | out: |
955 | memzero_explicit(s: &keys, count: sizeof(keys)); |
956 | return err; |
957 | } |
958 | |
959 | static void talitos_sg_unmap(struct device *dev, |
960 | struct talitos_edesc *edesc, |
961 | struct scatterlist *src, |
962 | struct scatterlist *dst, |
963 | unsigned int len, unsigned int offset) |
964 | { |
965 | struct talitos_private *priv = dev_get_drvdata(dev); |
966 | bool is_sec1 = has_ftr_sec1(priv); |
967 | unsigned int src_nents = edesc->src_nents ? : 1; |
968 | unsigned int dst_nents = edesc->dst_nents ? : 1; |
969 | |
970 | if (is_sec1 && dst && dst_nents > 1) { |
971 | dma_sync_single_for_device(dev, addr: edesc->dma_link_tbl + offset, |
972 | size: len, dir: DMA_FROM_DEVICE); |
973 | sg_pcopy_from_buffer(sgl: dst, nents: dst_nents, buf: edesc->buf + offset, buflen: len, |
974 | skip: offset); |
975 | } |
976 | if (src != dst) { |
977 | if (src_nents == 1 || !is_sec1) |
978 | dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); |
979 | |
980 | if (dst && (dst_nents == 1 || !is_sec1)) |
981 | dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); |
982 | } else if (src_nents == 1 || !is_sec1) { |
983 | dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); |
984 | } |
985 | } |
986 | |
987 | static void ipsec_esp_unmap(struct device *dev, |
988 | struct talitos_edesc *edesc, |
989 | struct aead_request *areq, bool encrypt) |
990 | { |
991 | struct crypto_aead *aead = crypto_aead_reqtfm(req: areq); |
992 | struct talitos_ctx *ctx = crypto_aead_ctx(tfm: aead); |
993 | unsigned int ivsize = crypto_aead_ivsize(tfm: aead); |
994 | unsigned int authsize = crypto_aead_authsize(tfm: aead); |
995 | unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); |
996 | bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; |
997 | struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; |
998 | |
999 | if (is_ipsec_esp) |
1000 | unmap_single_talitos_ptr(dev, ptr: &edesc->desc.ptr[6], |
1001 | dir: DMA_FROM_DEVICE); |
1002 | unmap_single_talitos_ptr(dev, ptr: civ_ptr, dir: DMA_TO_DEVICE); |
1003 | |
1004 | talitos_sg_unmap(dev, edesc, src: areq->src, dst: areq->dst, |
1005 | len: cryptlen + authsize, offset: areq->assoclen); |
1006 | |
1007 | if (edesc->dma_len) |
1008 | dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, |
1009 | DMA_BIDIRECTIONAL); |
1010 | |
1011 | if (!is_ipsec_esp) { |
1012 | unsigned int dst_nents = edesc->dst_nents ? : 1; |
1013 | |
1014 | sg_pcopy_to_buffer(sgl: areq->dst, nents: dst_nents, buf: ctx->iv, buflen: ivsize, |
1015 | skip: areq->assoclen + cryptlen - ivsize); |
1016 | } |
1017 | } |
1018 | |
1019 | /* |
1020 | * ipsec_esp descriptor callbacks |
1021 | */ |
1022 | static void ipsec_esp_encrypt_done(struct device *dev, |
1023 | struct talitos_desc *desc, void *context, |
1024 | int err) |
1025 | { |
1026 | struct aead_request *areq = context; |
1027 | struct crypto_aead *authenc = crypto_aead_reqtfm(req: areq); |
1028 | unsigned int ivsize = crypto_aead_ivsize(tfm: authenc); |
1029 | struct talitos_edesc *edesc; |
1030 | |
1031 | edesc = container_of(desc, struct talitos_edesc, desc); |
1032 | |
1033 | ipsec_esp_unmap(dev, edesc, areq, encrypt: true); |
1034 | |
1035 | dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); |
1036 | |
1037 | kfree(objp: edesc); |
1038 | |
1039 | aead_request_complete(req: areq, err); |
1040 | } |
1041 | |
1042 | static void ipsec_esp_decrypt_swauth_done(struct device *dev, |
1043 | struct talitos_desc *desc, |
1044 | void *context, int err) |
1045 | { |
1046 | struct aead_request *req = context; |
1047 | struct crypto_aead *authenc = crypto_aead_reqtfm(req); |
1048 | unsigned int authsize = crypto_aead_authsize(tfm: authenc); |
1049 | struct talitos_edesc *edesc; |
1050 | char *oicv, *icv; |
1051 | |
1052 | edesc = container_of(desc, struct talitos_edesc, desc); |
1053 | |
1054 | ipsec_esp_unmap(dev, edesc, areq: req, encrypt: false); |
1055 | |
1056 | if (!err) { |
1057 | /* auth check */ |
1058 | oicv = edesc->buf + edesc->dma_len; |
1059 | icv = oicv - authsize; |
1060 | |
1061 | err = crypto_memneq(a: oicv, b: icv, size: authsize) ? -EBADMSG : 0; |
1062 | } |
1063 | |
1064 | kfree(objp: edesc); |
1065 | |
1066 | aead_request_complete(req, err); |
1067 | } |
1068 | |
1069 | static void ipsec_esp_decrypt_hwauth_done(struct device *dev, |
1070 | struct talitos_desc *desc, |
1071 | void *context, int err) |
1072 | { |
1073 | struct aead_request *req = context; |
1074 | struct talitos_edesc *edesc; |
1075 | |
1076 | edesc = container_of(desc, struct talitos_edesc, desc); |
1077 | |
1078 | ipsec_esp_unmap(dev, edesc, areq: req, encrypt: false); |
1079 | |
1080 | /* check ICV auth status */ |
1081 | if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != |
1082 | DESC_HDR_LO_ICCR1_PASS)) |
1083 | err = -EBADMSG; |
1084 | |
1085 | kfree(objp: edesc); |
1086 | |
1087 | aead_request_complete(req, err); |
1088 | } |
1089 | |
1090 | /* |
1091 | * convert scatterlist to SEC h/w link table format |
1092 | * stop at cryptlen bytes |
1093 | */ |
1094 | static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, |
1095 | unsigned int offset, int datalen, int elen, |
1096 | struct talitos_ptr *link_tbl_ptr, int align) |
1097 | { |
1098 | int n_sg = elen ? sg_count + 1 : sg_count; |
1099 | int count = 0; |
1100 | int cryptlen = datalen + elen; |
1101 | int padding = ALIGN(cryptlen, align) - cryptlen; |
1102 | |
1103 | while (cryptlen && sg && n_sg--) { |
1104 | unsigned int len = sg_dma_len(sg); |
1105 | |
1106 | if (offset >= len) { |
1107 | offset -= len; |
1108 | goto next; |
1109 | } |
1110 | |
1111 | len -= offset; |
1112 | |
1113 | if (len > cryptlen) |
1114 | len = cryptlen; |
1115 | |
1116 | if (datalen > 0 && len > datalen) { |
1117 | to_talitos_ptr(ptr: link_tbl_ptr + count, |
1118 | sg_dma_address(sg) + offset, len: datalen, is_sec1: 0); |
1119 | to_talitos_ptr_ext_set(ptr: link_tbl_ptr + count, val: 0, is_sec1: 0); |
1120 | count++; |
1121 | len -= datalen; |
1122 | offset += datalen; |
1123 | } |
1124 | to_talitos_ptr(ptr: link_tbl_ptr + count, |
1125 | sg_dma_address(sg) + offset, len: sg_next(sg) ? len : len + padding, is_sec1: 0); |
1126 | to_talitos_ptr_ext_set(ptr: link_tbl_ptr + count, val: 0, is_sec1: 0); |
1127 | count++; |
1128 | cryptlen -= len; |
1129 | datalen -= len; |
1130 | offset = 0; |
1131 | |
1132 | next: |
1133 | sg = sg_next(sg); |
1134 | } |
1135 | |
1136 | /* tag end of link table */ |
1137 | if (count > 0) |
1138 | to_talitos_ptr_ext_set(ptr: link_tbl_ptr + count - 1, |
1139 | DESC_PTR_LNKTBL_RET, is_sec1: 0); |
1140 | |
1141 | return count; |
1142 | } |
1143 | |
1144 | static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, |
1145 | unsigned int len, struct talitos_edesc *edesc, |
1146 | struct talitos_ptr *ptr, int sg_count, |
1147 | unsigned int offset, int tbl_off, int elen, |
1148 | bool force, int align) |
1149 | { |
1150 | struct talitos_private *priv = dev_get_drvdata(dev); |
1151 | bool is_sec1 = has_ftr_sec1(priv); |
1152 | int aligned_len = ALIGN(len, align); |
1153 | |
1154 | if (!src) { |
1155 | to_talitos_ptr(ptr, dma_addr: 0, len: 0, is_sec1); |
1156 | return 1; |
1157 | } |
1158 | to_talitos_ptr_ext_set(ptr, val: elen, is_sec1); |
1159 | if (sg_count == 1 && !force) { |
1160 | to_talitos_ptr(ptr, sg_dma_address(src) + offset, len: aligned_len, is_sec1); |
1161 | return sg_count; |
1162 | } |
1163 | if (is_sec1) { |
1164 | to_talitos_ptr(ptr, dma_addr: edesc->dma_link_tbl + offset, len: aligned_len, is_sec1); |
1165 | return sg_count; |
1166 | } |
1167 | sg_count = sg_to_link_tbl_offset(sg: src, sg_count, offset, datalen: len, elen, |
1168 | link_tbl_ptr: &edesc->link_tbl[tbl_off], align); |
1169 | if (sg_count == 1 && !force) { |
1170 | /* Only one segment now, so no link tbl needed*/ |
1171 | copy_talitos_ptr(dst_ptr: ptr, src_ptr: &edesc->link_tbl[tbl_off], is_sec1); |
1172 | return sg_count; |
1173 | } |
1174 | to_talitos_ptr(ptr, dma_addr: edesc->dma_link_tbl + |
1175 | tbl_off * sizeof(struct talitos_ptr), len: aligned_len, is_sec1); |
1176 | to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); |
1177 | |
1178 | return sg_count; |
1179 | } |
1180 | |
1181 | static int talitos_sg_map(struct device *dev, struct scatterlist *src, |
1182 | unsigned int len, struct talitos_edesc *edesc, |
1183 | struct talitos_ptr *ptr, int sg_count, |
1184 | unsigned int offset, int tbl_off) |
1185 | { |
1186 | return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset, |
1187 | tbl_off, elen: 0, force: false, align: 1); |
1188 | } |
1189 | |
1190 | /* |
1191 | * fill in and submit ipsec_esp descriptor |
1192 | */ |
1193 | static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, |
1194 | bool encrypt, |
1195 | void (*callback)(struct device *dev, |
1196 | struct talitos_desc *desc, |
1197 | void *context, int error)) |
1198 | { |
1199 | struct crypto_aead *aead = crypto_aead_reqtfm(req: areq); |
1200 | unsigned int authsize = crypto_aead_authsize(tfm: aead); |
1201 | struct talitos_ctx *ctx = crypto_aead_ctx(tfm: aead); |
1202 | struct device *dev = ctx->dev; |
1203 | struct talitos_desc *desc = &edesc->desc; |
1204 | unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); |
1205 | unsigned int ivsize = crypto_aead_ivsize(tfm: aead); |
1206 | int tbl_off = 0; |
1207 | int sg_count, ret; |
1208 | int elen = 0; |
1209 | bool sync_needed = false; |
1210 | struct talitos_private *priv = dev_get_drvdata(dev); |
1211 | bool is_sec1 = has_ftr_sec1(priv); |
1212 | bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP; |
1213 | struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3]; |
1214 | struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; |
1215 | dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize; |
1216 | |
1217 | /* hmac key */ |
1218 | to_talitos_ptr(ptr: &desc->ptr[0], dma_addr: ctx->dma_key, len: ctx->authkeylen, is_sec1); |
1219 | |
1220 | sg_count = edesc->src_nents ?: 1; |
1221 | if (is_sec1 && sg_count > 1) |
1222 | sg_copy_to_buffer(sgl: areq->src, nents: sg_count, buf: edesc->buf, |
1223 | buflen: areq->assoclen + cryptlen); |
1224 | else |
1225 | sg_count = dma_map_sg(dev, areq->src, sg_count, |
1226 | (areq->src == areq->dst) ? |
1227 | DMA_BIDIRECTIONAL : DMA_TO_DEVICE); |
1228 | |
1229 | /* hmac data */ |
1230 | ret = talitos_sg_map(dev, src: areq->src, len: areq->assoclen, edesc, |
1231 | ptr: &desc->ptr[1], sg_count, offset: 0, tbl_off); |
1232 | |
1233 | if (ret > 1) { |
1234 | tbl_off += ret; |
1235 | sync_needed = true; |
1236 | } |
1237 | |
1238 | /* cipher iv */ |
1239 | to_talitos_ptr(ptr: civ_ptr, dma_addr: edesc->iv_dma, len: ivsize, is_sec1); |
1240 | |
1241 | /* cipher key */ |
1242 | to_talitos_ptr(ptr: ckey_ptr, dma_addr: ctx->dma_key + ctx->authkeylen, |
1243 | len: ctx->enckeylen, is_sec1); |
1244 | |
1245 | /* |
1246 | * cipher in |
1247 | * map and adjust cipher len to aead request cryptlen. |
1248 | * extent is bytes of HMAC postpended to ciphertext, |
1249 | * typically 12 for ipsec |
1250 | */ |
1251 | if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV)) |
1252 | elen = authsize; |
1253 | |
1254 | ret = talitos_sg_map_ext(dev, src: areq->src, len: cryptlen, edesc, ptr: &desc->ptr[4], |
1255 | sg_count, offset: areq->assoclen, tbl_off, elen, |
1256 | force: false, align: 1); |
1257 | |
1258 | if (ret > 1) { |
1259 | tbl_off += ret; |
1260 | sync_needed = true; |
1261 | } |
1262 | |
1263 | /* cipher out */ |
1264 | if (areq->src != areq->dst) { |
1265 | sg_count = edesc->dst_nents ? : 1; |
1266 | if (!is_sec1 || sg_count == 1) |
1267 | dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); |
1268 | } |
1269 | |
1270 | if (is_ipsec_esp && encrypt) |
1271 | elen = authsize; |
1272 | else |
1273 | elen = 0; |
1274 | ret = talitos_sg_map_ext(dev, src: areq->dst, len: cryptlen, edesc, ptr: &desc->ptr[5], |
1275 | sg_count, offset: areq->assoclen, tbl_off, elen, |
1276 | force: is_ipsec_esp && !encrypt, align: 1); |
1277 | tbl_off += ret; |
1278 | |
1279 | if (!encrypt && is_ipsec_esp) { |
1280 | struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; |
1281 | |
1282 | /* Add an entry to the link table for ICV data */ |
1283 | to_talitos_ptr_ext_set(ptr: tbl_ptr - 1, val: 0, is_sec1); |
1284 | to_talitos_ptr_ext_set(ptr: tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1); |
1285 | |
1286 | /* icv data follows link tables */ |
1287 | to_talitos_ptr(ptr: tbl_ptr, dma_addr: dma_icv, len: authsize, is_sec1); |
1288 | to_talitos_ptr_ext_or(ptr: &desc->ptr[5], val: authsize, is_sec1); |
1289 | sync_needed = true; |
1290 | } else if (!encrypt) { |
1291 | to_talitos_ptr(ptr: &desc->ptr[6], dma_addr: dma_icv, len: authsize, is_sec1); |
1292 | sync_needed = true; |
1293 | } else if (!is_ipsec_esp) { |
1294 | talitos_sg_map(dev, src: areq->dst, len: authsize, edesc, ptr: &desc->ptr[6], |
1295 | sg_count, offset: areq->assoclen + cryptlen, tbl_off); |
1296 | } |
1297 | |
1298 | /* iv out */ |
1299 | if (is_ipsec_esp) |
1300 | map_single_talitos_ptr(dev, ptr: &desc->ptr[6], len: ivsize, data: ctx->iv, |
1301 | dir: DMA_FROM_DEVICE); |
1302 | |
1303 | if (sync_needed) |
1304 | dma_sync_single_for_device(dev, addr: edesc->dma_link_tbl, |
1305 | size: edesc->dma_len, |
1306 | dir: DMA_BIDIRECTIONAL); |
1307 | |
1308 | ret = talitos_submit(dev, ch: ctx->ch, desc, callback, context: areq); |
1309 | if (ret != -EINPROGRESS) { |
1310 | ipsec_esp_unmap(dev, edesc, areq, encrypt); |
1311 | kfree(objp: edesc); |
1312 | } |
1313 | return ret; |
1314 | } |
1315 | |
1316 | /* |
1317 | * allocate and map the extended descriptor |
1318 | */ |
1319 | static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, |
1320 | struct scatterlist *src, |
1321 | struct scatterlist *dst, |
1322 | u8 *iv, |
1323 | unsigned int assoclen, |
1324 | unsigned int cryptlen, |
1325 | unsigned int authsize, |
1326 | unsigned int ivsize, |
1327 | int icv_stashing, |
1328 | u32 cryptoflags, |
1329 | bool encrypt) |
1330 | { |
1331 | struct talitos_edesc *edesc; |
1332 | int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len; |
1333 | dma_addr_t iv_dma = 0; |
1334 | gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : |
1335 | GFP_ATOMIC; |
1336 | struct talitos_private *priv = dev_get_drvdata(dev); |
1337 | bool is_sec1 = has_ftr_sec1(priv); |
1338 | int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN; |
1339 | |
1340 | if (cryptlen + authsize > max_len) { |
1341 | dev_err(dev, "length exceeds h/w max limit\n" ); |
1342 | return ERR_PTR(error: -EINVAL); |
1343 | } |
1344 | |
1345 | if (!dst || dst == src) { |
1346 | src_len = assoclen + cryptlen + authsize; |
1347 | src_nents = sg_nents_for_len(sg: src, len: src_len); |
1348 | if (src_nents < 0) { |
1349 | dev_err(dev, "Invalid number of src SG.\n" ); |
1350 | return ERR_PTR(error: -EINVAL); |
1351 | } |
1352 | src_nents = (src_nents == 1) ? 0 : src_nents; |
1353 | dst_nents = dst ? src_nents : 0; |
1354 | dst_len = 0; |
1355 | } else { /* dst && dst != src*/ |
1356 | src_len = assoclen + cryptlen + (encrypt ? 0 : authsize); |
1357 | src_nents = sg_nents_for_len(sg: src, len: src_len); |
1358 | if (src_nents < 0) { |
1359 | dev_err(dev, "Invalid number of src SG.\n" ); |
1360 | return ERR_PTR(error: -EINVAL); |
1361 | } |
1362 | src_nents = (src_nents == 1) ? 0 : src_nents; |
1363 | dst_len = assoclen + cryptlen + (encrypt ? authsize : 0); |
1364 | dst_nents = sg_nents_for_len(sg: dst, len: dst_len); |
1365 | if (dst_nents < 0) { |
1366 | dev_err(dev, "Invalid number of dst SG.\n" ); |
1367 | return ERR_PTR(error: -EINVAL); |
1368 | } |
1369 | dst_nents = (dst_nents == 1) ? 0 : dst_nents; |
1370 | } |
1371 | |
1372 | /* |
1373 | * allocate space for base edesc plus the link tables, |
1374 | * allowing for two separate entries for AD and generated ICV (+ 2), |
1375 | * and space for two sets of ICVs (stashed and generated) |
1376 | */ |
1377 | alloc_len = sizeof(struct talitos_edesc); |
1378 | if (src_nents || dst_nents || !encrypt) { |
1379 | if (is_sec1) |
1380 | dma_len = (src_nents ? src_len : 0) + |
1381 | (dst_nents ? dst_len : 0) + authsize; |
1382 | else |
1383 | dma_len = (src_nents + dst_nents + 2) * |
1384 | sizeof(struct talitos_ptr) + authsize; |
1385 | alloc_len += dma_len; |
1386 | } else { |
1387 | dma_len = 0; |
1388 | } |
1389 | alloc_len += icv_stashing ? authsize : 0; |
1390 | |
1391 | /* if its a ahash, add space for a second desc next to the first one */ |
1392 | if (is_sec1 && !dst) |
1393 | alloc_len += sizeof(struct talitos_desc); |
1394 | alloc_len += ivsize; |
1395 | |
1396 | edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags); |
1397 | if (!edesc) |
1398 | return ERR_PTR(error: -ENOMEM); |
1399 | if (ivsize) { |
1400 | iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize); |
1401 | iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); |
1402 | } |
1403 | memset(&edesc->desc, 0, sizeof(edesc->desc)); |
1404 | |
1405 | edesc->src_nents = src_nents; |
1406 | edesc->dst_nents = dst_nents; |
1407 | edesc->iv_dma = iv_dma; |
1408 | edesc->dma_len = dma_len; |
1409 | if (dma_len) |
1410 | edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], |
1411 | edesc->dma_len, |
1412 | DMA_BIDIRECTIONAL); |
1413 | |
1414 | return edesc; |
1415 | } |
1416 | |
1417 | static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, |
1418 | int icv_stashing, bool encrypt) |
1419 | { |
1420 | struct crypto_aead *authenc = crypto_aead_reqtfm(req: areq); |
1421 | unsigned int authsize = crypto_aead_authsize(tfm: authenc); |
1422 | struct talitos_ctx *ctx = crypto_aead_ctx(tfm: authenc); |
1423 | unsigned int ivsize = crypto_aead_ivsize(tfm: authenc); |
1424 | unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); |
1425 | |
1426 | return talitos_edesc_alloc(dev: ctx->dev, src: areq->src, dst: areq->dst, |
1427 | iv, assoclen: areq->assoclen, cryptlen, |
1428 | authsize, ivsize, icv_stashing, |
1429 | cryptoflags: areq->base.flags, encrypt); |
1430 | } |
1431 | |
1432 | static int aead_encrypt(struct aead_request *req) |
1433 | { |
1434 | struct crypto_aead *authenc = crypto_aead_reqtfm(req); |
1435 | struct talitos_ctx *ctx = crypto_aead_ctx(tfm: authenc); |
1436 | struct talitos_edesc *edesc; |
1437 | |
1438 | /* allocate extended descriptor */ |
1439 | edesc = aead_edesc_alloc(areq: req, iv: req->iv, icv_stashing: 0, encrypt: true); |
1440 | if (IS_ERR(ptr: edesc)) |
1441 | return PTR_ERR(ptr: edesc); |
1442 | |
1443 | /* set encrypt */ |
1444 | edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; |
1445 | |
1446 | return ipsec_esp(edesc, areq: req, encrypt: true, callback: ipsec_esp_encrypt_done); |
1447 | } |
1448 | |
1449 | static int aead_decrypt(struct aead_request *req) |
1450 | { |
1451 | struct crypto_aead *authenc = crypto_aead_reqtfm(req); |
1452 | unsigned int authsize = crypto_aead_authsize(tfm: authenc); |
1453 | struct talitos_ctx *ctx = crypto_aead_ctx(tfm: authenc); |
1454 | struct talitos_private *priv = dev_get_drvdata(dev: ctx->dev); |
1455 | struct talitos_edesc *edesc; |
1456 | void *icvdata; |
1457 | |
1458 | /* allocate extended descriptor */ |
1459 | edesc = aead_edesc_alloc(areq: req, iv: req->iv, icv_stashing: 1, encrypt: false); |
1460 | if (IS_ERR(ptr: edesc)) |
1461 | return PTR_ERR(ptr: edesc); |
1462 | |
1463 | if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) && |
1464 | (priv->features & TALITOS_FTR_HW_AUTH_CHECK) && |
1465 | ((!edesc->src_nents && !edesc->dst_nents) || |
1466 | priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) { |
1467 | |
1468 | /* decrypt and check the ICV */ |
1469 | edesc->desc.hdr = ctx->desc_hdr_template | |
1470 | DESC_HDR_DIR_INBOUND | |
1471 | DESC_HDR_MODE1_MDEU_CICV; |
1472 | |
1473 | /* reset integrity check result bits */ |
1474 | |
1475 | return ipsec_esp(edesc, areq: req, encrypt: false, |
1476 | callback: ipsec_esp_decrypt_hwauth_done); |
1477 | } |
1478 | |
1479 | /* Have to check the ICV with software */ |
1480 | edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; |
1481 | |
1482 | /* stash incoming ICV for later cmp with ICV generated by the h/w */ |
1483 | icvdata = edesc->buf + edesc->dma_len; |
1484 | |
1485 | sg_pcopy_to_buffer(sgl: req->src, nents: edesc->src_nents ? : 1, buf: icvdata, buflen: authsize, |
1486 | skip: req->assoclen + req->cryptlen - authsize); |
1487 | |
1488 | return ipsec_esp(edesc, areq: req, encrypt: false, callback: ipsec_esp_decrypt_swauth_done); |
1489 | } |
1490 | |
1491 | static int skcipher_setkey(struct crypto_skcipher *cipher, |
1492 | const u8 *key, unsigned int keylen) |
1493 | { |
1494 | struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm: cipher); |
1495 | struct device *dev = ctx->dev; |
1496 | |
1497 | if (ctx->keylen) |
1498 | dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); |
1499 | |
1500 | memcpy(&ctx->key, key, keylen); |
1501 | ctx->keylen = keylen; |
1502 | |
1503 | ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE); |
1504 | |
1505 | return 0; |
1506 | } |
1507 | |
1508 | static int skcipher_des_setkey(struct crypto_skcipher *cipher, |
1509 | const u8 *key, unsigned int keylen) |
1510 | { |
1511 | return verify_skcipher_des_key(tfm: cipher, key) ?: |
1512 | skcipher_setkey(cipher, key, keylen); |
1513 | } |
1514 | |
1515 | static int skcipher_des3_setkey(struct crypto_skcipher *cipher, |
1516 | const u8 *key, unsigned int keylen) |
1517 | { |
1518 | return verify_skcipher_des3_key(tfm: cipher, key) ?: |
1519 | skcipher_setkey(cipher, key, keylen); |
1520 | } |
1521 | |
1522 | static int skcipher_aes_setkey(struct crypto_skcipher *cipher, |
1523 | const u8 *key, unsigned int keylen) |
1524 | { |
1525 | if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 || |
1526 | keylen == AES_KEYSIZE_256) |
1527 | return skcipher_setkey(cipher, key, keylen); |
1528 | |
1529 | return -EINVAL; |
1530 | } |
1531 | |
1532 | static void common_nonsnoop_unmap(struct device *dev, |
1533 | struct talitos_edesc *edesc, |
1534 | struct skcipher_request *areq) |
1535 | { |
1536 | unmap_single_talitos_ptr(dev, ptr: &edesc->desc.ptr[5], dir: DMA_FROM_DEVICE); |
1537 | |
1538 | talitos_sg_unmap(dev, edesc, src: areq->src, dst: areq->dst, len: areq->cryptlen, offset: 0); |
1539 | unmap_single_talitos_ptr(dev, ptr: &edesc->desc.ptr[1], dir: DMA_TO_DEVICE); |
1540 | |
1541 | if (edesc->dma_len) |
1542 | dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, |
1543 | DMA_BIDIRECTIONAL); |
1544 | } |
1545 | |
1546 | static void skcipher_done(struct device *dev, |
1547 | struct talitos_desc *desc, void *context, |
1548 | int err) |
1549 | { |
1550 | struct skcipher_request *areq = context; |
1551 | struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req: areq); |
1552 | struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm: cipher); |
1553 | unsigned int ivsize = crypto_skcipher_ivsize(tfm: cipher); |
1554 | struct talitos_edesc *edesc; |
1555 | |
1556 | edesc = container_of(desc, struct talitos_edesc, desc); |
1557 | |
1558 | common_nonsnoop_unmap(dev, edesc, areq); |
1559 | memcpy(areq->iv, ctx->iv, ivsize); |
1560 | |
1561 | kfree(objp: edesc); |
1562 | |
1563 | skcipher_request_complete(req: areq, err); |
1564 | } |
1565 | |
1566 | static int common_nonsnoop(struct talitos_edesc *edesc, |
1567 | struct skcipher_request *areq, |
1568 | void (*callback) (struct device *dev, |
1569 | struct talitos_desc *desc, |
1570 | void *context, int error)) |
1571 | { |
1572 | struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req: areq); |
1573 | struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm: cipher); |
1574 | struct device *dev = ctx->dev; |
1575 | struct talitos_desc *desc = &edesc->desc; |
1576 | unsigned int cryptlen = areq->cryptlen; |
1577 | unsigned int ivsize = crypto_skcipher_ivsize(tfm: cipher); |
1578 | int sg_count, ret; |
1579 | bool sync_needed = false; |
1580 | struct talitos_private *priv = dev_get_drvdata(dev); |
1581 | bool is_sec1 = has_ftr_sec1(priv); |
1582 | bool is_ctr = (desc->hdr & DESC_HDR_SEL0_MASK) == DESC_HDR_SEL0_AESU && |
1583 | (desc->hdr & DESC_HDR_MODE0_AESU_MASK) == DESC_HDR_MODE0_AESU_CTR; |
1584 | |
1585 | /* first DWORD empty */ |
1586 | |
1587 | /* cipher iv */ |
1588 | to_talitos_ptr(ptr: &desc->ptr[1], dma_addr: edesc->iv_dma, len: ivsize, is_sec1); |
1589 | |
1590 | /* cipher key */ |
1591 | to_talitos_ptr(ptr: &desc->ptr[2], dma_addr: ctx->dma_key, len: ctx->keylen, is_sec1); |
1592 | |
1593 | sg_count = edesc->src_nents ?: 1; |
1594 | if (is_sec1 && sg_count > 1) |
1595 | sg_copy_to_buffer(sgl: areq->src, nents: sg_count, buf: edesc->buf, |
1596 | buflen: cryptlen); |
1597 | else |
1598 | sg_count = dma_map_sg(dev, areq->src, sg_count, |
1599 | (areq->src == areq->dst) ? |
1600 | DMA_BIDIRECTIONAL : DMA_TO_DEVICE); |
1601 | /* |
1602 | * cipher in |
1603 | */ |
1604 | sg_count = talitos_sg_map_ext(dev, src: areq->src, len: cryptlen, edesc, ptr: &desc->ptr[3], |
1605 | sg_count, offset: 0, tbl_off: 0, elen: 0, force: false, align: is_ctr ? 16 : 1); |
1606 | if (sg_count > 1) |
1607 | sync_needed = true; |
1608 | |
1609 | /* cipher out */ |
1610 | if (areq->src != areq->dst) { |
1611 | sg_count = edesc->dst_nents ? : 1; |
1612 | if (!is_sec1 || sg_count == 1) |
1613 | dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); |
1614 | } |
1615 | |
1616 | ret = talitos_sg_map(dev, src: areq->dst, len: cryptlen, edesc, ptr: &desc->ptr[4], |
1617 | sg_count, offset: 0, tbl_off: (edesc->src_nents + 1)); |
1618 | if (ret > 1) |
1619 | sync_needed = true; |
1620 | |
1621 | /* iv out */ |
1622 | map_single_talitos_ptr(dev, ptr: &desc->ptr[5], len: ivsize, data: ctx->iv, |
1623 | dir: DMA_FROM_DEVICE); |
1624 | |
1625 | /* last DWORD empty */ |
1626 | |
1627 | if (sync_needed) |
1628 | dma_sync_single_for_device(dev, addr: edesc->dma_link_tbl, |
1629 | size: edesc->dma_len, dir: DMA_BIDIRECTIONAL); |
1630 | |
1631 | ret = talitos_submit(dev, ch: ctx->ch, desc, callback, context: areq); |
1632 | if (ret != -EINPROGRESS) { |
1633 | common_nonsnoop_unmap(dev, edesc, areq); |
1634 | kfree(objp: edesc); |
1635 | } |
1636 | return ret; |
1637 | } |
1638 | |
1639 | static struct talitos_edesc *skcipher_edesc_alloc(struct skcipher_request * |
1640 | areq, bool encrypt) |
1641 | { |
1642 | struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req: areq); |
1643 | struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm: cipher); |
1644 | unsigned int ivsize = crypto_skcipher_ivsize(tfm: cipher); |
1645 | |
1646 | return talitos_edesc_alloc(dev: ctx->dev, src: areq->src, dst: areq->dst, |
1647 | iv: areq->iv, assoclen: 0, cryptlen: areq->cryptlen, authsize: 0, ivsize, icv_stashing: 0, |
1648 | cryptoflags: areq->base.flags, encrypt); |
1649 | } |
1650 | |
1651 | static int skcipher_encrypt(struct skcipher_request *areq) |
1652 | { |
1653 | struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req: areq); |
1654 | struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm: cipher); |
1655 | struct talitos_edesc *edesc; |
1656 | unsigned int blocksize = |
1657 | crypto_tfm_alg_blocksize(tfm: crypto_skcipher_tfm(tfm: cipher)); |
1658 | |
1659 | if (!areq->cryptlen) |
1660 | return 0; |
1661 | |
1662 | if (areq->cryptlen % blocksize) |
1663 | return -EINVAL; |
1664 | |
1665 | /* allocate extended descriptor */ |
1666 | edesc = skcipher_edesc_alloc(areq, encrypt: true); |
1667 | if (IS_ERR(ptr: edesc)) |
1668 | return PTR_ERR(ptr: edesc); |
1669 | |
1670 | /* set encrypt */ |
1671 | edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; |
1672 | |
1673 | return common_nonsnoop(edesc, areq, callback: skcipher_done); |
1674 | } |
1675 | |
1676 | static int skcipher_decrypt(struct skcipher_request *areq) |
1677 | { |
1678 | struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req: areq); |
1679 | struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm: cipher); |
1680 | struct talitos_edesc *edesc; |
1681 | unsigned int blocksize = |
1682 | crypto_tfm_alg_blocksize(tfm: crypto_skcipher_tfm(tfm: cipher)); |
1683 | |
1684 | if (!areq->cryptlen) |
1685 | return 0; |
1686 | |
1687 | if (areq->cryptlen % blocksize) |
1688 | return -EINVAL; |
1689 | |
1690 | /* allocate extended descriptor */ |
1691 | edesc = skcipher_edesc_alloc(areq, encrypt: false); |
1692 | if (IS_ERR(ptr: edesc)) |
1693 | return PTR_ERR(ptr: edesc); |
1694 | |
1695 | edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; |
1696 | |
1697 | return common_nonsnoop(edesc, areq, callback: skcipher_done); |
1698 | } |
1699 | |
1700 | static void common_nonsnoop_hash_unmap(struct device *dev, |
1701 | struct talitos_edesc *edesc, |
1702 | struct ahash_request *areq) |
1703 | { |
1704 | struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq); |
1705 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq); |
1706 | struct talitos_private *priv = dev_get_drvdata(dev); |
1707 | bool is_sec1 = has_ftr_sec1(priv); |
1708 | struct talitos_desc *desc = &edesc->desc; |
1709 | struct talitos_desc *desc2 = (struct talitos_desc *) |
1710 | (edesc->buf + edesc->dma_len); |
1711 | |
1712 | unmap_single_talitos_ptr(dev, ptr: &desc->ptr[5], dir: DMA_FROM_DEVICE); |
1713 | if (desc->next_desc && |
1714 | desc->ptr[5].ptr != desc2->ptr[5].ptr) |
1715 | unmap_single_talitos_ptr(dev, ptr: &desc2->ptr[5], dir: DMA_FROM_DEVICE); |
1716 | if (req_ctx->last) |
1717 | memcpy(areq->result, req_ctx->hw_context, |
1718 | crypto_ahash_digestsize(tfm)); |
1719 | |
1720 | if (req_ctx->psrc) |
1721 | talitos_sg_unmap(dev, edesc, src: req_ctx->psrc, NULL, len: 0, offset: 0); |
1722 | |
1723 | /* When using hashctx-in, must unmap it. */ |
1724 | if (from_talitos_ptr_len(ptr: &desc->ptr[1], is_sec1)) |
1725 | unmap_single_talitos_ptr(dev, ptr: &desc->ptr[1], |
1726 | dir: DMA_TO_DEVICE); |
1727 | else if (desc->next_desc) |
1728 | unmap_single_talitos_ptr(dev, ptr: &desc2->ptr[1], |
1729 | dir: DMA_TO_DEVICE); |
1730 | |
1731 | if (is_sec1 && req_ctx->nbuf) |
1732 | unmap_single_talitos_ptr(dev, ptr: &desc->ptr[3], |
1733 | dir: DMA_TO_DEVICE); |
1734 | |
1735 | if (edesc->dma_len) |
1736 | dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, |
1737 | DMA_BIDIRECTIONAL); |
1738 | |
1739 | if (desc->next_desc) |
1740 | dma_unmap_single(dev, be32_to_cpu(desc->next_desc), |
1741 | TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); |
1742 | } |
1743 | |
1744 | static void ahash_done(struct device *dev, |
1745 | struct talitos_desc *desc, void *context, |
1746 | int err) |
1747 | { |
1748 | struct ahash_request *areq = context; |
1749 | struct talitos_edesc *edesc = |
1750 | container_of(desc, struct talitos_edesc, desc); |
1751 | struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq); |
1752 | |
1753 | if (!req_ctx->last && req_ctx->to_hash_later) { |
1754 | /* Position any partial block for next update/final/finup */ |
1755 | req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; |
1756 | req_ctx->nbuf = req_ctx->to_hash_later; |
1757 | } |
1758 | common_nonsnoop_hash_unmap(dev, edesc, areq); |
1759 | |
1760 | kfree(objp: edesc); |
1761 | |
1762 | ahash_request_complete(req: areq, err); |
1763 | } |
1764 | |
1765 | /* |
1766 | * SEC1 doesn't like hashing of 0 sized message, so we do the padding |
1767 | * ourself and submit a padded block |
1768 | */ |
1769 | static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, |
1770 | struct talitos_edesc *edesc, |
1771 | struct talitos_ptr *ptr) |
1772 | { |
1773 | static u8 padded_hash[64] = { |
1774 | 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
1775 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
1776 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
1777 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
1778 | }; |
1779 | |
1780 | pr_err_once("Bug in SEC1, padding ourself\n" ); |
1781 | edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD; |
1782 | map_single_talitos_ptr(dev: ctx->dev, ptr, len: sizeof(padded_hash), |
1783 | data: (char *)padded_hash, dir: DMA_TO_DEVICE); |
1784 | } |
1785 | |
1786 | static int common_nonsnoop_hash(struct talitos_edesc *edesc, |
1787 | struct ahash_request *areq, unsigned int length, |
1788 | void (*callback) (struct device *dev, |
1789 | struct talitos_desc *desc, |
1790 | void *context, int error)) |
1791 | { |
1792 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq); |
1793 | struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); |
1794 | struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq); |
1795 | struct device *dev = ctx->dev; |
1796 | struct talitos_desc *desc = &edesc->desc; |
1797 | int ret; |
1798 | bool sync_needed = false; |
1799 | struct talitos_private *priv = dev_get_drvdata(dev); |
1800 | bool is_sec1 = has_ftr_sec1(priv); |
1801 | int sg_count; |
1802 | |
1803 | /* first DWORD empty */ |
1804 | |
1805 | /* hash context in */ |
1806 | if (!req_ctx->first || req_ctx->swinit) { |
1807 | map_single_talitos_ptr_nosync(dev, ptr: &desc->ptr[1], |
1808 | len: req_ctx->hw_context_size, |
1809 | data: req_ctx->hw_context, |
1810 | dir: DMA_TO_DEVICE); |
1811 | req_ctx->swinit = 0; |
1812 | } |
1813 | /* Indicate next op is not the first. */ |
1814 | req_ctx->first = 0; |
1815 | |
1816 | /* HMAC key */ |
1817 | if (ctx->keylen) |
1818 | to_talitos_ptr(ptr: &desc->ptr[2], dma_addr: ctx->dma_key, len: ctx->keylen, |
1819 | is_sec1); |
1820 | |
1821 | if (is_sec1 && req_ctx->nbuf) |
1822 | length -= req_ctx->nbuf; |
1823 | |
1824 | sg_count = edesc->src_nents ?: 1; |
1825 | if (is_sec1 && sg_count > 1) |
1826 | sg_copy_to_buffer(sgl: req_ctx->psrc, nents: sg_count, buf: edesc->buf, buflen: length); |
1827 | else if (length) |
1828 | sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count, |
1829 | DMA_TO_DEVICE); |
1830 | /* |
1831 | * data in |
1832 | */ |
1833 | if (is_sec1 && req_ctx->nbuf) { |
1834 | map_single_talitos_ptr(dev, ptr: &desc->ptr[3], len: req_ctx->nbuf, |
1835 | data: req_ctx->buf[req_ctx->buf_idx], |
1836 | dir: DMA_TO_DEVICE); |
1837 | } else { |
1838 | sg_count = talitos_sg_map(dev, src: req_ctx->psrc, len: length, edesc, |
1839 | ptr: &desc->ptr[3], sg_count, offset: 0, tbl_off: 0); |
1840 | if (sg_count > 1) |
1841 | sync_needed = true; |
1842 | } |
1843 | |
1844 | /* fifth DWORD empty */ |
1845 | |
1846 | /* hash/HMAC out -or- hash context out */ |
1847 | if (req_ctx->last) |
1848 | map_single_talitos_ptr(dev, ptr: &desc->ptr[5], |
1849 | len: crypto_ahash_digestsize(tfm), |
1850 | data: req_ctx->hw_context, dir: DMA_FROM_DEVICE); |
1851 | else |
1852 | map_single_talitos_ptr_nosync(dev, ptr: &desc->ptr[5], |
1853 | len: req_ctx->hw_context_size, |
1854 | data: req_ctx->hw_context, |
1855 | dir: DMA_FROM_DEVICE); |
1856 | |
1857 | /* last DWORD empty */ |
1858 | |
1859 | if (is_sec1 && from_talitos_ptr_len(ptr: &desc->ptr[3], is_sec1: true) == 0) |
1860 | talitos_handle_buggy_hash(ctx, edesc, ptr: &desc->ptr[3]); |
1861 | |
1862 | if (is_sec1 && req_ctx->nbuf && length) { |
1863 | struct talitos_desc *desc2 = (struct talitos_desc *) |
1864 | (edesc->buf + edesc->dma_len); |
1865 | dma_addr_t next_desc; |
1866 | |
1867 | memset(desc2, 0, sizeof(*desc2)); |
1868 | desc2->hdr = desc->hdr; |
1869 | desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT; |
1870 | desc2->hdr1 = desc2->hdr; |
1871 | desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD; |
1872 | desc->hdr |= DESC_HDR_MODE0_MDEU_CONT; |
1873 | desc->hdr &= ~DESC_HDR_DONE_NOTIFY; |
1874 | |
1875 | if (desc->ptr[1].ptr) |
1876 | copy_talitos_ptr(dst_ptr: &desc2->ptr[1], src_ptr: &desc->ptr[1], |
1877 | is_sec1); |
1878 | else |
1879 | map_single_talitos_ptr_nosync(dev, ptr: &desc2->ptr[1], |
1880 | len: req_ctx->hw_context_size, |
1881 | data: req_ctx->hw_context, |
1882 | dir: DMA_TO_DEVICE); |
1883 | copy_talitos_ptr(dst_ptr: &desc2->ptr[2], src_ptr: &desc->ptr[2], is_sec1); |
1884 | sg_count = talitos_sg_map(dev, src: req_ctx->psrc, len: length, edesc, |
1885 | ptr: &desc2->ptr[3], sg_count, offset: 0, tbl_off: 0); |
1886 | if (sg_count > 1) |
1887 | sync_needed = true; |
1888 | copy_talitos_ptr(dst_ptr: &desc2->ptr[5], src_ptr: &desc->ptr[5], is_sec1); |
1889 | if (req_ctx->last) |
1890 | map_single_talitos_ptr_nosync(dev, ptr: &desc->ptr[5], |
1891 | len: req_ctx->hw_context_size, |
1892 | data: req_ctx->hw_context, |
1893 | dir: DMA_FROM_DEVICE); |
1894 | |
1895 | next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE, |
1896 | DMA_BIDIRECTIONAL); |
1897 | desc->next_desc = cpu_to_be32(next_desc); |
1898 | } |
1899 | |
1900 | if (sync_needed) |
1901 | dma_sync_single_for_device(dev, addr: edesc->dma_link_tbl, |
1902 | size: edesc->dma_len, dir: DMA_BIDIRECTIONAL); |
1903 | |
1904 | ret = talitos_submit(dev, ch: ctx->ch, desc, callback, context: areq); |
1905 | if (ret != -EINPROGRESS) { |
1906 | common_nonsnoop_hash_unmap(dev, edesc, areq); |
1907 | kfree(objp: edesc); |
1908 | } |
1909 | return ret; |
1910 | } |
1911 | |
1912 | static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, |
1913 | unsigned int nbytes) |
1914 | { |
1915 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq); |
1916 | struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); |
1917 | struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq); |
1918 | struct talitos_private *priv = dev_get_drvdata(dev: ctx->dev); |
1919 | bool is_sec1 = has_ftr_sec1(priv); |
1920 | |
1921 | if (is_sec1) |
1922 | nbytes -= req_ctx->nbuf; |
1923 | |
1924 | return talitos_edesc_alloc(dev: ctx->dev, src: req_ctx->psrc, NULL, NULL, assoclen: 0, |
1925 | cryptlen: nbytes, authsize: 0, ivsize: 0, icv_stashing: 0, cryptoflags: areq->base.flags, encrypt: false); |
1926 | } |
1927 | |
1928 | static int ahash_init(struct ahash_request *areq) |
1929 | { |
1930 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq); |
1931 | struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); |
1932 | struct device *dev = ctx->dev; |
1933 | struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq); |
1934 | unsigned int size; |
1935 | dma_addr_t dma; |
1936 | |
1937 | /* Initialize the context */ |
1938 | req_ctx->buf_idx = 0; |
1939 | req_ctx->nbuf = 0; |
1940 | req_ctx->first = 1; /* first indicates h/w must init its context */ |
1941 | req_ctx->swinit = 0; /* assume h/w init of context */ |
1942 | size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) |
1943 | ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 |
1944 | : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; |
1945 | req_ctx->hw_context_size = size; |
1946 | |
1947 | dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, |
1948 | DMA_TO_DEVICE); |
1949 | dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); |
1950 | |
1951 | return 0; |
1952 | } |
1953 | |
1954 | /* |
1955 | * on h/w without explicit sha224 support, we initialize h/w context |
1956 | * manually with sha224 constants, and tell it to run sha256. |
1957 | */ |
1958 | static int ahash_init_sha224_swinit(struct ahash_request *areq) |
1959 | { |
1960 | struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq); |
1961 | |
1962 | req_ctx->hw_context[0] = SHA224_H0; |
1963 | req_ctx->hw_context[1] = SHA224_H1; |
1964 | req_ctx->hw_context[2] = SHA224_H2; |
1965 | req_ctx->hw_context[3] = SHA224_H3; |
1966 | req_ctx->hw_context[4] = SHA224_H4; |
1967 | req_ctx->hw_context[5] = SHA224_H5; |
1968 | req_ctx->hw_context[6] = SHA224_H6; |
1969 | req_ctx->hw_context[7] = SHA224_H7; |
1970 | |
1971 | /* init 64-bit count */ |
1972 | req_ctx->hw_context[8] = 0; |
1973 | req_ctx->hw_context[9] = 0; |
1974 | |
1975 | ahash_init(areq); |
1976 | req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ |
1977 | |
1978 | return 0; |
1979 | } |
1980 | |
1981 | static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) |
1982 | { |
1983 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq); |
1984 | struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); |
1985 | struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq); |
1986 | struct talitos_edesc *edesc; |
1987 | unsigned int blocksize = |
1988 | crypto_tfm_alg_blocksize(tfm: crypto_ahash_tfm(tfm)); |
1989 | unsigned int nbytes_to_hash; |
1990 | unsigned int to_hash_later; |
1991 | unsigned int nsg; |
1992 | int nents; |
1993 | struct device *dev = ctx->dev; |
1994 | struct talitos_private *priv = dev_get_drvdata(dev); |
1995 | bool is_sec1 = has_ftr_sec1(priv); |
1996 | u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; |
1997 | |
1998 | if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { |
1999 | /* Buffer up to one whole block */ |
2000 | nents = sg_nents_for_len(sg: areq->src, len: nbytes); |
2001 | if (nents < 0) { |
2002 | dev_err(dev, "Invalid number of src SG.\n" ); |
2003 | return nents; |
2004 | } |
2005 | sg_copy_to_buffer(sgl: areq->src, nents, |
2006 | buf: ctx_buf + req_ctx->nbuf, buflen: nbytes); |
2007 | req_ctx->nbuf += nbytes; |
2008 | return 0; |
2009 | } |
2010 | |
2011 | /* At least (blocksize + 1) bytes are available to hash */ |
2012 | nbytes_to_hash = nbytes + req_ctx->nbuf; |
2013 | to_hash_later = nbytes_to_hash & (blocksize - 1); |
2014 | |
2015 | if (req_ctx->last) |
2016 | to_hash_later = 0; |
2017 | else if (to_hash_later) |
2018 | /* There is a partial block. Hash the full block(s) now */ |
2019 | nbytes_to_hash -= to_hash_later; |
2020 | else { |
2021 | /* Keep one block buffered */ |
2022 | nbytes_to_hash -= blocksize; |
2023 | to_hash_later = blocksize; |
2024 | } |
2025 | |
2026 | /* Chain in any previously buffered data */ |
2027 | if (!is_sec1 && req_ctx->nbuf) { |
2028 | nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; |
2029 | sg_init_table(req_ctx->bufsl, nsg); |
2030 | sg_set_buf(sg: req_ctx->bufsl, buf: ctx_buf, buflen: req_ctx->nbuf); |
2031 | if (nsg > 1) |
2032 | sg_chain(prv: req_ctx->bufsl, prv_nents: 2, sgl: areq->src); |
2033 | req_ctx->psrc = req_ctx->bufsl; |
2034 | } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) { |
2035 | int offset; |
2036 | |
2037 | if (nbytes_to_hash > blocksize) |
2038 | offset = blocksize - req_ctx->nbuf; |
2039 | else |
2040 | offset = nbytes_to_hash - req_ctx->nbuf; |
2041 | nents = sg_nents_for_len(sg: areq->src, len: offset); |
2042 | if (nents < 0) { |
2043 | dev_err(dev, "Invalid number of src SG.\n" ); |
2044 | return nents; |
2045 | } |
2046 | sg_copy_to_buffer(sgl: areq->src, nents, |
2047 | buf: ctx_buf + req_ctx->nbuf, buflen: offset); |
2048 | req_ctx->nbuf += offset; |
2049 | req_ctx->psrc = scatterwalk_ffwd(dst: req_ctx->bufsl, src: areq->src, |
2050 | len: offset); |
2051 | } else |
2052 | req_ctx->psrc = areq->src; |
2053 | |
2054 | if (to_hash_later) { |
2055 | nents = sg_nents_for_len(sg: areq->src, len: nbytes); |
2056 | if (nents < 0) { |
2057 | dev_err(dev, "Invalid number of src SG.\n" ); |
2058 | return nents; |
2059 | } |
2060 | sg_pcopy_to_buffer(sgl: areq->src, nents, |
2061 | buf: req_ctx->buf[(req_ctx->buf_idx + 1) & 1], |
2062 | buflen: to_hash_later, |
2063 | skip: nbytes - to_hash_later); |
2064 | } |
2065 | req_ctx->to_hash_later = to_hash_later; |
2066 | |
2067 | /* Allocate extended descriptor */ |
2068 | edesc = ahash_edesc_alloc(areq, nbytes: nbytes_to_hash); |
2069 | if (IS_ERR(ptr: edesc)) |
2070 | return PTR_ERR(ptr: edesc); |
2071 | |
2072 | edesc->desc.hdr = ctx->desc_hdr_template; |
2073 | |
2074 | /* On last one, request SEC to pad; otherwise continue */ |
2075 | if (req_ctx->last) |
2076 | edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; |
2077 | else |
2078 | edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; |
2079 | |
2080 | /* request SEC to INIT hash. */ |
2081 | if (req_ctx->first && !req_ctx->swinit) |
2082 | edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; |
2083 | |
2084 | /* When the tfm context has a keylen, it's an HMAC. |
2085 | * A first or last (ie. not middle) descriptor must request HMAC. |
2086 | */ |
2087 | if (ctx->keylen && (req_ctx->first || req_ctx->last)) |
2088 | edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; |
2089 | |
2090 | return common_nonsnoop_hash(edesc, areq, length: nbytes_to_hash, callback: ahash_done); |
2091 | } |
2092 | |
2093 | static int ahash_update(struct ahash_request *areq) |
2094 | { |
2095 | struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq); |
2096 | |
2097 | req_ctx->last = 0; |
2098 | |
2099 | return ahash_process_req(areq, nbytes: areq->nbytes); |
2100 | } |
2101 | |
2102 | static int ahash_final(struct ahash_request *areq) |
2103 | { |
2104 | struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq); |
2105 | |
2106 | req_ctx->last = 1; |
2107 | |
2108 | return ahash_process_req(areq, nbytes: 0); |
2109 | } |
2110 | |
2111 | static int ahash_finup(struct ahash_request *areq) |
2112 | { |
2113 | struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq); |
2114 | |
2115 | req_ctx->last = 1; |
2116 | |
2117 | return ahash_process_req(areq, nbytes: areq->nbytes); |
2118 | } |
2119 | |
2120 | static int ahash_digest(struct ahash_request *areq) |
2121 | { |
2122 | ahash_init(areq); |
2123 | return ahash_finup(areq); |
2124 | } |
2125 | |
2126 | static int ahash_digest_sha224_swinit(struct ahash_request *areq) |
2127 | { |
2128 | ahash_init_sha224_swinit(areq); |
2129 | return ahash_finup(areq); |
2130 | } |
2131 | |
2132 | static int ahash_export(struct ahash_request *areq, void *out) |
2133 | { |
2134 | struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq); |
2135 | struct talitos_export_state *export = out; |
2136 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq); |
2137 | struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); |
2138 | struct device *dev = ctx->dev; |
2139 | dma_addr_t dma; |
2140 | |
2141 | dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, |
2142 | DMA_FROM_DEVICE); |
2143 | dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE); |
2144 | |
2145 | memcpy(export->hw_context, req_ctx->hw_context, |
2146 | req_ctx->hw_context_size); |
2147 | memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf); |
2148 | export->swinit = req_ctx->swinit; |
2149 | export->first = req_ctx->first; |
2150 | export->last = req_ctx->last; |
2151 | export->to_hash_later = req_ctx->to_hash_later; |
2152 | export->nbuf = req_ctx->nbuf; |
2153 | |
2154 | return 0; |
2155 | } |
2156 | |
2157 | static int ahash_import(struct ahash_request *areq, const void *in) |
2158 | { |
2159 | struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(req: areq); |
2160 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req: areq); |
2161 | struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); |
2162 | struct device *dev = ctx->dev; |
2163 | const struct talitos_export_state *export = in; |
2164 | unsigned int size; |
2165 | dma_addr_t dma; |
2166 | |
2167 | memset(req_ctx, 0, sizeof(*req_ctx)); |
2168 | size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) |
2169 | ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 |
2170 | : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; |
2171 | req_ctx->hw_context_size = size; |
2172 | memcpy(req_ctx->hw_context, export->hw_context, size); |
2173 | memcpy(req_ctx->buf[0], export->buf, export->nbuf); |
2174 | req_ctx->swinit = export->swinit; |
2175 | req_ctx->first = export->first; |
2176 | req_ctx->last = export->last; |
2177 | req_ctx->to_hash_later = export->to_hash_later; |
2178 | req_ctx->nbuf = export->nbuf; |
2179 | |
2180 | dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, |
2181 | DMA_TO_DEVICE); |
2182 | dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); |
2183 | |
2184 | return 0; |
2185 | } |
2186 | |
2187 | static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, |
2188 | u8 *hash) |
2189 | { |
2190 | struct talitos_ctx *ctx = crypto_tfm_ctx(tfm: crypto_ahash_tfm(tfm)); |
2191 | |
2192 | struct scatterlist sg[1]; |
2193 | struct ahash_request *req; |
2194 | struct crypto_wait wait; |
2195 | int ret; |
2196 | |
2197 | crypto_init_wait(wait: &wait); |
2198 | |
2199 | req = ahash_request_alloc(tfm, GFP_KERNEL); |
2200 | if (!req) |
2201 | return -ENOMEM; |
2202 | |
2203 | /* Keep tfm keylen == 0 during hash of the long key */ |
2204 | ctx->keylen = 0; |
2205 | ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, |
2206 | compl: crypto_req_done, data: &wait); |
2207 | |
2208 | sg_init_one(&sg[0], key, keylen); |
2209 | |
2210 | ahash_request_set_crypt(req, src: sg, result: hash, nbytes: keylen); |
2211 | ret = crypto_wait_req(err: crypto_ahash_digest(req), wait: &wait); |
2212 | |
2213 | ahash_request_free(req); |
2214 | |
2215 | return ret; |
2216 | } |
2217 | |
2218 | static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, |
2219 | unsigned int keylen) |
2220 | { |
2221 | struct talitos_ctx *ctx = crypto_tfm_ctx(tfm: crypto_ahash_tfm(tfm)); |
2222 | struct device *dev = ctx->dev; |
2223 | unsigned int blocksize = |
2224 | crypto_tfm_alg_blocksize(tfm: crypto_ahash_tfm(tfm)); |
2225 | unsigned int digestsize = crypto_ahash_digestsize(tfm); |
2226 | unsigned int keysize = keylen; |
2227 | u8 hash[SHA512_DIGEST_SIZE]; |
2228 | int ret; |
2229 | |
2230 | if (keylen <= blocksize) |
2231 | memcpy(ctx->key, key, keysize); |
2232 | else { |
2233 | /* Must get the hash of the long key */ |
2234 | ret = keyhash(tfm, key, keylen, hash); |
2235 | |
2236 | if (ret) |
2237 | return -EINVAL; |
2238 | |
2239 | keysize = digestsize; |
2240 | memcpy(ctx->key, hash, digestsize); |
2241 | } |
2242 | |
2243 | if (ctx->keylen) |
2244 | dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); |
2245 | |
2246 | ctx->keylen = keysize; |
2247 | ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE); |
2248 | |
2249 | return 0; |
2250 | } |
2251 | |
2252 | |
2253 | struct talitos_alg_template { |
2254 | u32 type; |
2255 | u32 priority; |
2256 | union { |
2257 | struct skcipher_alg skcipher; |
2258 | struct ahash_alg hash; |
2259 | struct aead_alg aead; |
2260 | } alg; |
2261 | __be32 desc_hdr_template; |
2262 | }; |
2263 | |
2264 | static struct talitos_alg_template driver_algs[] = { |
2265 | /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */ |
2266 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2267 | .alg.aead = { |
2268 | .base = { |
2269 | .cra_name = "authenc(hmac(sha1),cbc(aes))" , |
2270 | .cra_driver_name = "authenc-hmac-sha1-" |
2271 | "cbc-aes-talitos" , |
2272 | .cra_blocksize = AES_BLOCK_SIZE, |
2273 | .cra_flags = CRYPTO_ALG_ASYNC | |
2274 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2275 | }, |
2276 | .ivsize = AES_BLOCK_SIZE, |
2277 | .maxauthsize = SHA1_DIGEST_SIZE, |
2278 | }, |
2279 | .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | |
2280 | DESC_HDR_SEL0_AESU | |
2281 | DESC_HDR_MODE0_AESU_CBC | |
2282 | DESC_HDR_SEL1_MDEUA | |
2283 | DESC_HDR_MODE1_MDEU_INIT | |
2284 | DESC_HDR_MODE1_MDEU_PAD | |
2285 | DESC_HDR_MODE1_MDEU_SHA1_HMAC, |
2286 | }, |
2287 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2288 | .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, |
2289 | .alg.aead = { |
2290 | .base = { |
2291 | .cra_name = "authenc(hmac(sha1),cbc(aes))" , |
2292 | .cra_driver_name = "authenc-hmac-sha1-" |
2293 | "cbc-aes-talitos-hsna" , |
2294 | .cra_blocksize = AES_BLOCK_SIZE, |
2295 | .cra_flags = CRYPTO_ALG_ASYNC | |
2296 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2297 | }, |
2298 | .ivsize = AES_BLOCK_SIZE, |
2299 | .maxauthsize = SHA1_DIGEST_SIZE, |
2300 | }, |
2301 | .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | |
2302 | DESC_HDR_SEL0_AESU | |
2303 | DESC_HDR_MODE0_AESU_CBC | |
2304 | DESC_HDR_SEL1_MDEUA | |
2305 | DESC_HDR_MODE1_MDEU_INIT | |
2306 | DESC_HDR_MODE1_MDEU_PAD | |
2307 | DESC_HDR_MODE1_MDEU_SHA1_HMAC, |
2308 | }, |
2309 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2310 | .alg.aead = { |
2311 | .base = { |
2312 | .cra_name = "authenc(hmac(sha1)," |
2313 | "cbc(des3_ede))" , |
2314 | .cra_driver_name = "authenc-hmac-sha1-" |
2315 | "cbc-3des-talitos" , |
2316 | .cra_blocksize = DES3_EDE_BLOCK_SIZE, |
2317 | .cra_flags = CRYPTO_ALG_ASYNC | |
2318 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2319 | }, |
2320 | .ivsize = DES3_EDE_BLOCK_SIZE, |
2321 | .maxauthsize = SHA1_DIGEST_SIZE, |
2322 | .setkey = aead_des3_setkey, |
2323 | }, |
2324 | .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | |
2325 | DESC_HDR_SEL0_DEU | |
2326 | DESC_HDR_MODE0_DEU_CBC | |
2327 | DESC_HDR_MODE0_DEU_3DES | |
2328 | DESC_HDR_SEL1_MDEUA | |
2329 | DESC_HDR_MODE1_MDEU_INIT | |
2330 | DESC_HDR_MODE1_MDEU_PAD | |
2331 | DESC_HDR_MODE1_MDEU_SHA1_HMAC, |
2332 | }, |
2333 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2334 | .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, |
2335 | .alg.aead = { |
2336 | .base = { |
2337 | .cra_name = "authenc(hmac(sha1)," |
2338 | "cbc(des3_ede))" , |
2339 | .cra_driver_name = "authenc-hmac-sha1-" |
2340 | "cbc-3des-talitos-hsna" , |
2341 | .cra_blocksize = DES3_EDE_BLOCK_SIZE, |
2342 | .cra_flags = CRYPTO_ALG_ASYNC | |
2343 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2344 | }, |
2345 | .ivsize = DES3_EDE_BLOCK_SIZE, |
2346 | .maxauthsize = SHA1_DIGEST_SIZE, |
2347 | .setkey = aead_des3_setkey, |
2348 | }, |
2349 | .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | |
2350 | DESC_HDR_SEL0_DEU | |
2351 | DESC_HDR_MODE0_DEU_CBC | |
2352 | DESC_HDR_MODE0_DEU_3DES | |
2353 | DESC_HDR_SEL1_MDEUA | |
2354 | DESC_HDR_MODE1_MDEU_INIT | |
2355 | DESC_HDR_MODE1_MDEU_PAD | |
2356 | DESC_HDR_MODE1_MDEU_SHA1_HMAC, |
2357 | }, |
2358 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2359 | .alg.aead = { |
2360 | .base = { |
2361 | .cra_name = "authenc(hmac(sha224),cbc(aes))" , |
2362 | .cra_driver_name = "authenc-hmac-sha224-" |
2363 | "cbc-aes-talitos" , |
2364 | .cra_blocksize = AES_BLOCK_SIZE, |
2365 | .cra_flags = CRYPTO_ALG_ASYNC | |
2366 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2367 | }, |
2368 | .ivsize = AES_BLOCK_SIZE, |
2369 | .maxauthsize = SHA224_DIGEST_SIZE, |
2370 | }, |
2371 | .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | |
2372 | DESC_HDR_SEL0_AESU | |
2373 | DESC_HDR_MODE0_AESU_CBC | |
2374 | DESC_HDR_SEL1_MDEUA | |
2375 | DESC_HDR_MODE1_MDEU_INIT | |
2376 | DESC_HDR_MODE1_MDEU_PAD | |
2377 | DESC_HDR_MODE1_MDEU_SHA224_HMAC, |
2378 | }, |
2379 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2380 | .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, |
2381 | .alg.aead = { |
2382 | .base = { |
2383 | .cra_name = "authenc(hmac(sha224),cbc(aes))" , |
2384 | .cra_driver_name = "authenc-hmac-sha224-" |
2385 | "cbc-aes-talitos-hsna" , |
2386 | .cra_blocksize = AES_BLOCK_SIZE, |
2387 | .cra_flags = CRYPTO_ALG_ASYNC | |
2388 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2389 | }, |
2390 | .ivsize = AES_BLOCK_SIZE, |
2391 | .maxauthsize = SHA224_DIGEST_SIZE, |
2392 | }, |
2393 | .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | |
2394 | DESC_HDR_SEL0_AESU | |
2395 | DESC_HDR_MODE0_AESU_CBC | |
2396 | DESC_HDR_SEL1_MDEUA | |
2397 | DESC_HDR_MODE1_MDEU_INIT | |
2398 | DESC_HDR_MODE1_MDEU_PAD | |
2399 | DESC_HDR_MODE1_MDEU_SHA224_HMAC, |
2400 | }, |
2401 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2402 | .alg.aead = { |
2403 | .base = { |
2404 | .cra_name = "authenc(hmac(sha224)," |
2405 | "cbc(des3_ede))" , |
2406 | .cra_driver_name = "authenc-hmac-sha224-" |
2407 | "cbc-3des-talitos" , |
2408 | .cra_blocksize = DES3_EDE_BLOCK_SIZE, |
2409 | .cra_flags = CRYPTO_ALG_ASYNC | |
2410 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2411 | }, |
2412 | .ivsize = DES3_EDE_BLOCK_SIZE, |
2413 | .maxauthsize = SHA224_DIGEST_SIZE, |
2414 | .setkey = aead_des3_setkey, |
2415 | }, |
2416 | .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | |
2417 | DESC_HDR_SEL0_DEU | |
2418 | DESC_HDR_MODE0_DEU_CBC | |
2419 | DESC_HDR_MODE0_DEU_3DES | |
2420 | DESC_HDR_SEL1_MDEUA | |
2421 | DESC_HDR_MODE1_MDEU_INIT | |
2422 | DESC_HDR_MODE1_MDEU_PAD | |
2423 | DESC_HDR_MODE1_MDEU_SHA224_HMAC, |
2424 | }, |
2425 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2426 | .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, |
2427 | .alg.aead = { |
2428 | .base = { |
2429 | .cra_name = "authenc(hmac(sha224)," |
2430 | "cbc(des3_ede))" , |
2431 | .cra_driver_name = "authenc-hmac-sha224-" |
2432 | "cbc-3des-talitos-hsna" , |
2433 | .cra_blocksize = DES3_EDE_BLOCK_SIZE, |
2434 | .cra_flags = CRYPTO_ALG_ASYNC | |
2435 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2436 | }, |
2437 | .ivsize = DES3_EDE_BLOCK_SIZE, |
2438 | .maxauthsize = SHA224_DIGEST_SIZE, |
2439 | .setkey = aead_des3_setkey, |
2440 | }, |
2441 | .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | |
2442 | DESC_HDR_SEL0_DEU | |
2443 | DESC_HDR_MODE0_DEU_CBC | |
2444 | DESC_HDR_MODE0_DEU_3DES | |
2445 | DESC_HDR_SEL1_MDEUA | |
2446 | DESC_HDR_MODE1_MDEU_INIT | |
2447 | DESC_HDR_MODE1_MDEU_PAD | |
2448 | DESC_HDR_MODE1_MDEU_SHA224_HMAC, |
2449 | }, |
2450 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2451 | .alg.aead = { |
2452 | .base = { |
2453 | .cra_name = "authenc(hmac(sha256),cbc(aes))" , |
2454 | .cra_driver_name = "authenc-hmac-sha256-" |
2455 | "cbc-aes-talitos" , |
2456 | .cra_blocksize = AES_BLOCK_SIZE, |
2457 | .cra_flags = CRYPTO_ALG_ASYNC | |
2458 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2459 | }, |
2460 | .ivsize = AES_BLOCK_SIZE, |
2461 | .maxauthsize = SHA256_DIGEST_SIZE, |
2462 | }, |
2463 | .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | |
2464 | DESC_HDR_SEL0_AESU | |
2465 | DESC_HDR_MODE0_AESU_CBC | |
2466 | DESC_HDR_SEL1_MDEUA | |
2467 | DESC_HDR_MODE1_MDEU_INIT | |
2468 | DESC_HDR_MODE1_MDEU_PAD | |
2469 | DESC_HDR_MODE1_MDEU_SHA256_HMAC, |
2470 | }, |
2471 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2472 | .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, |
2473 | .alg.aead = { |
2474 | .base = { |
2475 | .cra_name = "authenc(hmac(sha256),cbc(aes))" , |
2476 | .cra_driver_name = "authenc-hmac-sha256-" |
2477 | "cbc-aes-talitos-hsna" , |
2478 | .cra_blocksize = AES_BLOCK_SIZE, |
2479 | .cra_flags = CRYPTO_ALG_ASYNC | |
2480 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2481 | }, |
2482 | .ivsize = AES_BLOCK_SIZE, |
2483 | .maxauthsize = SHA256_DIGEST_SIZE, |
2484 | }, |
2485 | .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | |
2486 | DESC_HDR_SEL0_AESU | |
2487 | DESC_HDR_MODE0_AESU_CBC | |
2488 | DESC_HDR_SEL1_MDEUA | |
2489 | DESC_HDR_MODE1_MDEU_INIT | |
2490 | DESC_HDR_MODE1_MDEU_PAD | |
2491 | DESC_HDR_MODE1_MDEU_SHA256_HMAC, |
2492 | }, |
2493 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2494 | .alg.aead = { |
2495 | .base = { |
2496 | .cra_name = "authenc(hmac(sha256)," |
2497 | "cbc(des3_ede))" , |
2498 | .cra_driver_name = "authenc-hmac-sha256-" |
2499 | "cbc-3des-talitos" , |
2500 | .cra_blocksize = DES3_EDE_BLOCK_SIZE, |
2501 | .cra_flags = CRYPTO_ALG_ASYNC | |
2502 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2503 | }, |
2504 | .ivsize = DES3_EDE_BLOCK_SIZE, |
2505 | .maxauthsize = SHA256_DIGEST_SIZE, |
2506 | .setkey = aead_des3_setkey, |
2507 | }, |
2508 | .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | |
2509 | DESC_HDR_SEL0_DEU | |
2510 | DESC_HDR_MODE0_DEU_CBC | |
2511 | DESC_HDR_MODE0_DEU_3DES | |
2512 | DESC_HDR_SEL1_MDEUA | |
2513 | DESC_HDR_MODE1_MDEU_INIT | |
2514 | DESC_HDR_MODE1_MDEU_PAD | |
2515 | DESC_HDR_MODE1_MDEU_SHA256_HMAC, |
2516 | }, |
2517 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2518 | .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, |
2519 | .alg.aead = { |
2520 | .base = { |
2521 | .cra_name = "authenc(hmac(sha256)," |
2522 | "cbc(des3_ede))" , |
2523 | .cra_driver_name = "authenc-hmac-sha256-" |
2524 | "cbc-3des-talitos-hsna" , |
2525 | .cra_blocksize = DES3_EDE_BLOCK_SIZE, |
2526 | .cra_flags = CRYPTO_ALG_ASYNC | |
2527 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2528 | }, |
2529 | .ivsize = DES3_EDE_BLOCK_SIZE, |
2530 | .maxauthsize = SHA256_DIGEST_SIZE, |
2531 | .setkey = aead_des3_setkey, |
2532 | }, |
2533 | .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | |
2534 | DESC_HDR_SEL0_DEU | |
2535 | DESC_HDR_MODE0_DEU_CBC | |
2536 | DESC_HDR_MODE0_DEU_3DES | |
2537 | DESC_HDR_SEL1_MDEUA | |
2538 | DESC_HDR_MODE1_MDEU_INIT | |
2539 | DESC_HDR_MODE1_MDEU_PAD | |
2540 | DESC_HDR_MODE1_MDEU_SHA256_HMAC, |
2541 | }, |
2542 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2543 | .alg.aead = { |
2544 | .base = { |
2545 | .cra_name = "authenc(hmac(sha384),cbc(aes))" , |
2546 | .cra_driver_name = "authenc-hmac-sha384-" |
2547 | "cbc-aes-talitos" , |
2548 | .cra_blocksize = AES_BLOCK_SIZE, |
2549 | .cra_flags = CRYPTO_ALG_ASYNC | |
2550 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2551 | }, |
2552 | .ivsize = AES_BLOCK_SIZE, |
2553 | .maxauthsize = SHA384_DIGEST_SIZE, |
2554 | }, |
2555 | .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | |
2556 | DESC_HDR_SEL0_AESU | |
2557 | DESC_HDR_MODE0_AESU_CBC | |
2558 | DESC_HDR_SEL1_MDEUB | |
2559 | DESC_HDR_MODE1_MDEU_INIT | |
2560 | DESC_HDR_MODE1_MDEU_PAD | |
2561 | DESC_HDR_MODE1_MDEUB_SHA384_HMAC, |
2562 | }, |
2563 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2564 | .alg.aead = { |
2565 | .base = { |
2566 | .cra_name = "authenc(hmac(sha384)," |
2567 | "cbc(des3_ede))" , |
2568 | .cra_driver_name = "authenc-hmac-sha384-" |
2569 | "cbc-3des-talitos" , |
2570 | .cra_blocksize = DES3_EDE_BLOCK_SIZE, |
2571 | .cra_flags = CRYPTO_ALG_ASYNC | |
2572 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2573 | }, |
2574 | .ivsize = DES3_EDE_BLOCK_SIZE, |
2575 | .maxauthsize = SHA384_DIGEST_SIZE, |
2576 | .setkey = aead_des3_setkey, |
2577 | }, |
2578 | .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | |
2579 | DESC_HDR_SEL0_DEU | |
2580 | DESC_HDR_MODE0_DEU_CBC | |
2581 | DESC_HDR_MODE0_DEU_3DES | |
2582 | DESC_HDR_SEL1_MDEUB | |
2583 | DESC_HDR_MODE1_MDEU_INIT | |
2584 | DESC_HDR_MODE1_MDEU_PAD | |
2585 | DESC_HDR_MODE1_MDEUB_SHA384_HMAC, |
2586 | }, |
2587 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2588 | .alg.aead = { |
2589 | .base = { |
2590 | .cra_name = "authenc(hmac(sha512),cbc(aes))" , |
2591 | .cra_driver_name = "authenc-hmac-sha512-" |
2592 | "cbc-aes-talitos" , |
2593 | .cra_blocksize = AES_BLOCK_SIZE, |
2594 | .cra_flags = CRYPTO_ALG_ASYNC | |
2595 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2596 | }, |
2597 | .ivsize = AES_BLOCK_SIZE, |
2598 | .maxauthsize = SHA512_DIGEST_SIZE, |
2599 | }, |
2600 | .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | |
2601 | DESC_HDR_SEL0_AESU | |
2602 | DESC_HDR_MODE0_AESU_CBC | |
2603 | DESC_HDR_SEL1_MDEUB | |
2604 | DESC_HDR_MODE1_MDEU_INIT | |
2605 | DESC_HDR_MODE1_MDEU_PAD | |
2606 | DESC_HDR_MODE1_MDEUB_SHA512_HMAC, |
2607 | }, |
2608 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2609 | .alg.aead = { |
2610 | .base = { |
2611 | .cra_name = "authenc(hmac(sha512)," |
2612 | "cbc(des3_ede))" , |
2613 | .cra_driver_name = "authenc-hmac-sha512-" |
2614 | "cbc-3des-talitos" , |
2615 | .cra_blocksize = DES3_EDE_BLOCK_SIZE, |
2616 | .cra_flags = CRYPTO_ALG_ASYNC | |
2617 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2618 | }, |
2619 | .ivsize = DES3_EDE_BLOCK_SIZE, |
2620 | .maxauthsize = SHA512_DIGEST_SIZE, |
2621 | .setkey = aead_des3_setkey, |
2622 | }, |
2623 | .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | |
2624 | DESC_HDR_SEL0_DEU | |
2625 | DESC_HDR_MODE0_DEU_CBC | |
2626 | DESC_HDR_MODE0_DEU_3DES | |
2627 | DESC_HDR_SEL1_MDEUB | |
2628 | DESC_HDR_MODE1_MDEU_INIT | |
2629 | DESC_HDR_MODE1_MDEU_PAD | |
2630 | DESC_HDR_MODE1_MDEUB_SHA512_HMAC, |
2631 | }, |
2632 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2633 | .alg.aead = { |
2634 | .base = { |
2635 | .cra_name = "authenc(hmac(md5),cbc(aes))" , |
2636 | .cra_driver_name = "authenc-hmac-md5-" |
2637 | "cbc-aes-talitos" , |
2638 | .cra_blocksize = AES_BLOCK_SIZE, |
2639 | .cra_flags = CRYPTO_ALG_ASYNC | |
2640 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2641 | }, |
2642 | .ivsize = AES_BLOCK_SIZE, |
2643 | .maxauthsize = MD5_DIGEST_SIZE, |
2644 | }, |
2645 | .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | |
2646 | DESC_HDR_SEL0_AESU | |
2647 | DESC_HDR_MODE0_AESU_CBC | |
2648 | DESC_HDR_SEL1_MDEUA | |
2649 | DESC_HDR_MODE1_MDEU_INIT | |
2650 | DESC_HDR_MODE1_MDEU_PAD | |
2651 | DESC_HDR_MODE1_MDEU_MD5_HMAC, |
2652 | }, |
2653 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2654 | .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, |
2655 | .alg.aead = { |
2656 | .base = { |
2657 | .cra_name = "authenc(hmac(md5),cbc(aes))" , |
2658 | .cra_driver_name = "authenc-hmac-md5-" |
2659 | "cbc-aes-talitos-hsna" , |
2660 | .cra_blocksize = AES_BLOCK_SIZE, |
2661 | .cra_flags = CRYPTO_ALG_ASYNC | |
2662 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2663 | }, |
2664 | .ivsize = AES_BLOCK_SIZE, |
2665 | .maxauthsize = MD5_DIGEST_SIZE, |
2666 | }, |
2667 | .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | |
2668 | DESC_HDR_SEL0_AESU | |
2669 | DESC_HDR_MODE0_AESU_CBC | |
2670 | DESC_HDR_SEL1_MDEUA | |
2671 | DESC_HDR_MODE1_MDEU_INIT | |
2672 | DESC_HDR_MODE1_MDEU_PAD | |
2673 | DESC_HDR_MODE1_MDEU_MD5_HMAC, |
2674 | }, |
2675 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2676 | .alg.aead = { |
2677 | .base = { |
2678 | .cra_name = "authenc(hmac(md5),cbc(des3_ede))" , |
2679 | .cra_driver_name = "authenc-hmac-md5-" |
2680 | "cbc-3des-talitos" , |
2681 | .cra_blocksize = DES3_EDE_BLOCK_SIZE, |
2682 | .cra_flags = CRYPTO_ALG_ASYNC | |
2683 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2684 | }, |
2685 | .ivsize = DES3_EDE_BLOCK_SIZE, |
2686 | .maxauthsize = MD5_DIGEST_SIZE, |
2687 | .setkey = aead_des3_setkey, |
2688 | }, |
2689 | .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | |
2690 | DESC_HDR_SEL0_DEU | |
2691 | DESC_HDR_MODE0_DEU_CBC | |
2692 | DESC_HDR_MODE0_DEU_3DES | |
2693 | DESC_HDR_SEL1_MDEUA | |
2694 | DESC_HDR_MODE1_MDEU_INIT | |
2695 | DESC_HDR_MODE1_MDEU_PAD | |
2696 | DESC_HDR_MODE1_MDEU_MD5_HMAC, |
2697 | }, |
2698 | { .type = CRYPTO_ALG_TYPE_AEAD, |
2699 | .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA, |
2700 | .alg.aead = { |
2701 | .base = { |
2702 | .cra_name = "authenc(hmac(md5),cbc(des3_ede))" , |
2703 | .cra_driver_name = "authenc-hmac-md5-" |
2704 | "cbc-3des-talitos-hsna" , |
2705 | .cra_blocksize = DES3_EDE_BLOCK_SIZE, |
2706 | .cra_flags = CRYPTO_ALG_ASYNC | |
2707 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2708 | }, |
2709 | .ivsize = DES3_EDE_BLOCK_SIZE, |
2710 | .maxauthsize = MD5_DIGEST_SIZE, |
2711 | .setkey = aead_des3_setkey, |
2712 | }, |
2713 | .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU | |
2714 | DESC_HDR_SEL0_DEU | |
2715 | DESC_HDR_MODE0_DEU_CBC | |
2716 | DESC_HDR_MODE0_DEU_3DES | |
2717 | DESC_HDR_SEL1_MDEUA | |
2718 | DESC_HDR_MODE1_MDEU_INIT | |
2719 | DESC_HDR_MODE1_MDEU_PAD | |
2720 | DESC_HDR_MODE1_MDEU_MD5_HMAC, |
2721 | }, |
2722 | /* SKCIPHER algorithms. */ |
2723 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, |
2724 | .alg.skcipher = { |
2725 | .base.cra_name = "ecb(aes)" , |
2726 | .base.cra_driver_name = "ecb-aes-talitos" , |
2727 | .base.cra_blocksize = AES_BLOCK_SIZE, |
2728 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
2729 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2730 | .min_keysize = AES_MIN_KEY_SIZE, |
2731 | .max_keysize = AES_MAX_KEY_SIZE, |
2732 | .setkey = skcipher_aes_setkey, |
2733 | }, |
2734 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2735 | DESC_HDR_SEL0_AESU, |
2736 | }, |
2737 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, |
2738 | .alg.skcipher = { |
2739 | .base.cra_name = "cbc(aes)" , |
2740 | .base.cra_driver_name = "cbc-aes-talitos" , |
2741 | .base.cra_blocksize = AES_BLOCK_SIZE, |
2742 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
2743 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2744 | .min_keysize = AES_MIN_KEY_SIZE, |
2745 | .max_keysize = AES_MAX_KEY_SIZE, |
2746 | .ivsize = AES_BLOCK_SIZE, |
2747 | .setkey = skcipher_aes_setkey, |
2748 | }, |
2749 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2750 | DESC_HDR_SEL0_AESU | |
2751 | DESC_HDR_MODE0_AESU_CBC, |
2752 | }, |
2753 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, |
2754 | .alg.skcipher = { |
2755 | .base.cra_name = "ctr(aes)" , |
2756 | .base.cra_driver_name = "ctr-aes-talitos" , |
2757 | .base.cra_blocksize = 1, |
2758 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
2759 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2760 | .min_keysize = AES_MIN_KEY_SIZE, |
2761 | .max_keysize = AES_MAX_KEY_SIZE, |
2762 | .ivsize = AES_BLOCK_SIZE, |
2763 | .setkey = skcipher_aes_setkey, |
2764 | }, |
2765 | .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP | |
2766 | DESC_HDR_SEL0_AESU | |
2767 | DESC_HDR_MODE0_AESU_CTR, |
2768 | }, |
2769 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, |
2770 | .alg.skcipher = { |
2771 | .base.cra_name = "ctr(aes)" , |
2772 | .base.cra_driver_name = "ctr-aes-talitos" , |
2773 | .base.cra_blocksize = 1, |
2774 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
2775 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2776 | .min_keysize = AES_MIN_KEY_SIZE, |
2777 | .max_keysize = AES_MAX_KEY_SIZE, |
2778 | .ivsize = AES_BLOCK_SIZE, |
2779 | .setkey = skcipher_aes_setkey, |
2780 | }, |
2781 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2782 | DESC_HDR_SEL0_AESU | |
2783 | DESC_HDR_MODE0_AESU_CTR, |
2784 | }, |
2785 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, |
2786 | .alg.skcipher = { |
2787 | .base.cra_name = "ecb(des)" , |
2788 | .base.cra_driver_name = "ecb-des-talitos" , |
2789 | .base.cra_blocksize = DES_BLOCK_SIZE, |
2790 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
2791 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2792 | .min_keysize = DES_KEY_SIZE, |
2793 | .max_keysize = DES_KEY_SIZE, |
2794 | .setkey = skcipher_des_setkey, |
2795 | }, |
2796 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2797 | DESC_HDR_SEL0_DEU, |
2798 | }, |
2799 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, |
2800 | .alg.skcipher = { |
2801 | .base.cra_name = "cbc(des)" , |
2802 | .base.cra_driver_name = "cbc-des-talitos" , |
2803 | .base.cra_blocksize = DES_BLOCK_SIZE, |
2804 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
2805 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2806 | .min_keysize = DES_KEY_SIZE, |
2807 | .max_keysize = DES_KEY_SIZE, |
2808 | .ivsize = DES_BLOCK_SIZE, |
2809 | .setkey = skcipher_des_setkey, |
2810 | }, |
2811 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2812 | DESC_HDR_SEL0_DEU | |
2813 | DESC_HDR_MODE0_DEU_CBC, |
2814 | }, |
2815 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, |
2816 | .alg.skcipher = { |
2817 | .base.cra_name = "ecb(des3_ede)" , |
2818 | .base.cra_driver_name = "ecb-3des-talitos" , |
2819 | .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, |
2820 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
2821 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2822 | .min_keysize = DES3_EDE_KEY_SIZE, |
2823 | .max_keysize = DES3_EDE_KEY_SIZE, |
2824 | .setkey = skcipher_des3_setkey, |
2825 | }, |
2826 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2827 | DESC_HDR_SEL0_DEU | |
2828 | DESC_HDR_MODE0_DEU_3DES, |
2829 | }, |
2830 | { .type = CRYPTO_ALG_TYPE_SKCIPHER, |
2831 | .alg.skcipher = { |
2832 | .base.cra_name = "cbc(des3_ede)" , |
2833 | .base.cra_driver_name = "cbc-3des-talitos" , |
2834 | .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, |
2835 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
2836 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2837 | .min_keysize = DES3_EDE_KEY_SIZE, |
2838 | .max_keysize = DES3_EDE_KEY_SIZE, |
2839 | .ivsize = DES3_EDE_BLOCK_SIZE, |
2840 | .setkey = skcipher_des3_setkey, |
2841 | }, |
2842 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2843 | DESC_HDR_SEL0_DEU | |
2844 | DESC_HDR_MODE0_DEU_CBC | |
2845 | DESC_HDR_MODE0_DEU_3DES, |
2846 | }, |
2847 | /* AHASH algorithms. */ |
2848 | { .type = CRYPTO_ALG_TYPE_AHASH, |
2849 | .alg.hash = { |
2850 | .halg.digestsize = MD5_DIGEST_SIZE, |
2851 | .halg.statesize = sizeof(struct talitos_export_state), |
2852 | .halg.base = { |
2853 | .cra_name = "md5" , |
2854 | .cra_driver_name = "md5-talitos" , |
2855 | .cra_blocksize = MD5_HMAC_BLOCK_SIZE, |
2856 | .cra_flags = CRYPTO_ALG_ASYNC | |
2857 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2858 | } |
2859 | }, |
2860 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2861 | DESC_HDR_SEL0_MDEUA | |
2862 | DESC_HDR_MODE0_MDEU_MD5, |
2863 | }, |
2864 | { .type = CRYPTO_ALG_TYPE_AHASH, |
2865 | .alg.hash = { |
2866 | .halg.digestsize = SHA1_DIGEST_SIZE, |
2867 | .halg.statesize = sizeof(struct talitos_export_state), |
2868 | .halg.base = { |
2869 | .cra_name = "sha1" , |
2870 | .cra_driver_name = "sha1-talitos" , |
2871 | .cra_blocksize = SHA1_BLOCK_SIZE, |
2872 | .cra_flags = CRYPTO_ALG_ASYNC | |
2873 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2874 | } |
2875 | }, |
2876 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2877 | DESC_HDR_SEL0_MDEUA | |
2878 | DESC_HDR_MODE0_MDEU_SHA1, |
2879 | }, |
2880 | { .type = CRYPTO_ALG_TYPE_AHASH, |
2881 | .alg.hash = { |
2882 | .halg.digestsize = SHA224_DIGEST_SIZE, |
2883 | .halg.statesize = sizeof(struct talitos_export_state), |
2884 | .halg.base = { |
2885 | .cra_name = "sha224" , |
2886 | .cra_driver_name = "sha224-talitos" , |
2887 | .cra_blocksize = SHA224_BLOCK_SIZE, |
2888 | .cra_flags = CRYPTO_ALG_ASYNC | |
2889 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2890 | } |
2891 | }, |
2892 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2893 | DESC_HDR_SEL0_MDEUA | |
2894 | DESC_HDR_MODE0_MDEU_SHA224, |
2895 | }, |
2896 | { .type = CRYPTO_ALG_TYPE_AHASH, |
2897 | .alg.hash = { |
2898 | .halg.digestsize = SHA256_DIGEST_SIZE, |
2899 | .halg.statesize = sizeof(struct talitos_export_state), |
2900 | .halg.base = { |
2901 | .cra_name = "sha256" , |
2902 | .cra_driver_name = "sha256-talitos" , |
2903 | .cra_blocksize = SHA256_BLOCK_SIZE, |
2904 | .cra_flags = CRYPTO_ALG_ASYNC | |
2905 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2906 | } |
2907 | }, |
2908 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2909 | DESC_HDR_SEL0_MDEUA | |
2910 | DESC_HDR_MODE0_MDEU_SHA256, |
2911 | }, |
2912 | { .type = CRYPTO_ALG_TYPE_AHASH, |
2913 | .alg.hash = { |
2914 | .halg.digestsize = SHA384_DIGEST_SIZE, |
2915 | .halg.statesize = sizeof(struct talitos_export_state), |
2916 | .halg.base = { |
2917 | .cra_name = "sha384" , |
2918 | .cra_driver_name = "sha384-talitos" , |
2919 | .cra_blocksize = SHA384_BLOCK_SIZE, |
2920 | .cra_flags = CRYPTO_ALG_ASYNC | |
2921 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2922 | } |
2923 | }, |
2924 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2925 | DESC_HDR_SEL0_MDEUB | |
2926 | DESC_HDR_MODE0_MDEUB_SHA384, |
2927 | }, |
2928 | { .type = CRYPTO_ALG_TYPE_AHASH, |
2929 | .alg.hash = { |
2930 | .halg.digestsize = SHA512_DIGEST_SIZE, |
2931 | .halg.statesize = sizeof(struct talitos_export_state), |
2932 | .halg.base = { |
2933 | .cra_name = "sha512" , |
2934 | .cra_driver_name = "sha512-talitos" , |
2935 | .cra_blocksize = SHA512_BLOCK_SIZE, |
2936 | .cra_flags = CRYPTO_ALG_ASYNC | |
2937 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2938 | } |
2939 | }, |
2940 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2941 | DESC_HDR_SEL0_MDEUB | |
2942 | DESC_HDR_MODE0_MDEUB_SHA512, |
2943 | }, |
2944 | { .type = CRYPTO_ALG_TYPE_AHASH, |
2945 | .alg.hash = { |
2946 | .halg.digestsize = MD5_DIGEST_SIZE, |
2947 | .halg.statesize = sizeof(struct talitos_export_state), |
2948 | .halg.base = { |
2949 | .cra_name = "hmac(md5)" , |
2950 | .cra_driver_name = "hmac-md5-talitos" , |
2951 | .cra_blocksize = MD5_HMAC_BLOCK_SIZE, |
2952 | .cra_flags = CRYPTO_ALG_ASYNC | |
2953 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2954 | } |
2955 | }, |
2956 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2957 | DESC_HDR_SEL0_MDEUA | |
2958 | DESC_HDR_MODE0_MDEU_MD5, |
2959 | }, |
2960 | { .type = CRYPTO_ALG_TYPE_AHASH, |
2961 | .alg.hash = { |
2962 | .halg.digestsize = SHA1_DIGEST_SIZE, |
2963 | .halg.statesize = sizeof(struct talitos_export_state), |
2964 | .halg.base = { |
2965 | .cra_name = "hmac(sha1)" , |
2966 | .cra_driver_name = "hmac-sha1-talitos" , |
2967 | .cra_blocksize = SHA1_BLOCK_SIZE, |
2968 | .cra_flags = CRYPTO_ALG_ASYNC | |
2969 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2970 | } |
2971 | }, |
2972 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2973 | DESC_HDR_SEL0_MDEUA | |
2974 | DESC_HDR_MODE0_MDEU_SHA1, |
2975 | }, |
2976 | { .type = CRYPTO_ALG_TYPE_AHASH, |
2977 | .alg.hash = { |
2978 | .halg.digestsize = SHA224_DIGEST_SIZE, |
2979 | .halg.statesize = sizeof(struct talitos_export_state), |
2980 | .halg.base = { |
2981 | .cra_name = "hmac(sha224)" , |
2982 | .cra_driver_name = "hmac-sha224-talitos" , |
2983 | .cra_blocksize = SHA224_BLOCK_SIZE, |
2984 | .cra_flags = CRYPTO_ALG_ASYNC | |
2985 | CRYPTO_ALG_ALLOCATES_MEMORY, |
2986 | } |
2987 | }, |
2988 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
2989 | DESC_HDR_SEL0_MDEUA | |
2990 | DESC_HDR_MODE0_MDEU_SHA224, |
2991 | }, |
2992 | { .type = CRYPTO_ALG_TYPE_AHASH, |
2993 | .alg.hash = { |
2994 | .halg.digestsize = SHA256_DIGEST_SIZE, |
2995 | .halg.statesize = sizeof(struct talitos_export_state), |
2996 | .halg.base = { |
2997 | .cra_name = "hmac(sha256)" , |
2998 | .cra_driver_name = "hmac-sha256-talitos" , |
2999 | .cra_blocksize = SHA256_BLOCK_SIZE, |
3000 | .cra_flags = CRYPTO_ALG_ASYNC | |
3001 | CRYPTO_ALG_ALLOCATES_MEMORY, |
3002 | } |
3003 | }, |
3004 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
3005 | DESC_HDR_SEL0_MDEUA | |
3006 | DESC_HDR_MODE0_MDEU_SHA256, |
3007 | }, |
3008 | { .type = CRYPTO_ALG_TYPE_AHASH, |
3009 | .alg.hash = { |
3010 | .halg.digestsize = SHA384_DIGEST_SIZE, |
3011 | .halg.statesize = sizeof(struct talitos_export_state), |
3012 | .halg.base = { |
3013 | .cra_name = "hmac(sha384)" , |
3014 | .cra_driver_name = "hmac-sha384-talitos" , |
3015 | .cra_blocksize = SHA384_BLOCK_SIZE, |
3016 | .cra_flags = CRYPTO_ALG_ASYNC | |
3017 | CRYPTO_ALG_ALLOCATES_MEMORY, |
3018 | } |
3019 | }, |
3020 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
3021 | DESC_HDR_SEL0_MDEUB | |
3022 | DESC_HDR_MODE0_MDEUB_SHA384, |
3023 | }, |
3024 | { .type = CRYPTO_ALG_TYPE_AHASH, |
3025 | .alg.hash = { |
3026 | .halg.digestsize = SHA512_DIGEST_SIZE, |
3027 | .halg.statesize = sizeof(struct talitos_export_state), |
3028 | .halg.base = { |
3029 | .cra_name = "hmac(sha512)" , |
3030 | .cra_driver_name = "hmac-sha512-talitos" , |
3031 | .cra_blocksize = SHA512_BLOCK_SIZE, |
3032 | .cra_flags = CRYPTO_ALG_ASYNC | |
3033 | CRYPTO_ALG_ALLOCATES_MEMORY, |
3034 | } |
3035 | }, |
3036 | .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
3037 | DESC_HDR_SEL0_MDEUB | |
3038 | DESC_HDR_MODE0_MDEUB_SHA512, |
3039 | } |
3040 | }; |
3041 | |
3042 | struct talitos_crypto_alg { |
3043 | struct list_head entry; |
3044 | struct device *dev; |
3045 | struct talitos_alg_template algt; |
3046 | }; |
3047 | |
3048 | static int talitos_init_common(struct talitos_ctx *ctx, |
3049 | struct talitos_crypto_alg *talitos_alg) |
3050 | { |
3051 | struct talitos_private *priv; |
3052 | |
3053 | /* update context with ptr to dev */ |
3054 | ctx->dev = talitos_alg->dev; |
3055 | |
3056 | /* assign SEC channel to tfm in round-robin fashion */ |
3057 | priv = dev_get_drvdata(dev: ctx->dev); |
3058 | ctx->ch = atomic_inc_return(v: &priv->last_chan) & |
3059 | (priv->num_channels - 1); |
3060 | |
3061 | /* copy descriptor header template value */ |
3062 | ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template; |
3063 | |
3064 | /* select done notification */ |
3065 | ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY; |
3066 | |
3067 | return 0; |
3068 | } |
3069 | |
3070 | static int talitos_cra_init_aead(struct crypto_aead *tfm) |
3071 | { |
3072 | struct aead_alg *alg = crypto_aead_alg(tfm); |
3073 | struct talitos_crypto_alg *talitos_alg; |
3074 | struct talitos_ctx *ctx = crypto_aead_ctx(tfm); |
3075 | |
3076 | talitos_alg = container_of(alg, struct talitos_crypto_alg, |
3077 | algt.alg.aead); |
3078 | |
3079 | return talitos_init_common(ctx, talitos_alg); |
3080 | } |
3081 | |
3082 | static int talitos_cra_init_skcipher(struct crypto_skcipher *tfm) |
3083 | { |
3084 | struct skcipher_alg *alg = crypto_skcipher_alg(tfm); |
3085 | struct talitos_crypto_alg *talitos_alg; |
3086 | struct talitos_ctx *ctx = crypto_skcipher_ctx(tfm); |
3087 | |
3088 | talitos_alg = container_of(alg, struct talitos_crypto_alg, |
3089 | algt.alg.skcipher); |
3090 | |
3091 | return talitos_init_common(ctx, talitos_alg); |
3092 | } |
3093 | |
3094 | static int talitos_cra_init_ahash(struct crypto_tfm *tfm) |
3095 | { |
3096 | struct crypto_alg *alg = tfm->__crt_alg; |
3097 | struct talitos_crypto_alg *talitos_alg; |
3098 | struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); |
3099 | |
3100 | talitos_alg = container_of(__crypto_ahash_alg(alg), |
3101 | struct talitos_crypto_alg, |
3102 | algt.alg.hash); |
3103 | |
3104 | ctx->keylen = 0; |
3105 | crypto_ahash_set_reqsize(tfm: __crypto_ahash_cast(tfm), |
3106 | reqsize: sizeof(struct talitos_ahash_req_ctx)); |
3107 | |
3108 | return talitos_init_common(ctx, talitos_alg); |
3109 | } |
3110 | |
3111 | static void talitos_cra_exit(struct crypto_tfm *tfm) |
3112 | { |
3113 | struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); |
3114 | struct device *dev = ctx->dev; |
3115 | |
3116 | if (ctx->keylen) |
3117 | dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); |
3118 | } |
3119 | |
3120 | /* |
3121 | * given the alg's descriptor header template, determine whether descriptor |
3122 | * type and primary/secondary execution units required match the hw |
3123 | * capabilities description provided in the device tree node. |
3124 | */ |
3125 | static int hw_supports(struct device *dev, __be32 desc_hdr_template) |
3126 | { |
3127 | struct talitos_private *priv = dev_get_drvdata(dev); |
3128 | int ret; |
3129 | |
3130 | ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) && |
3131 | (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units); |
3132 | |
3133 | if (SECONDARY_EU(desc_hdr_template)) |
3134 | ret = ret && (1 << SECONDARY_EU(desc_hdr_template) |
3135 | & priv->exec_units); |
3136 | |
3137 | return ret; |
3138 | } |
3139 | |
3140 | static void talitos_remove(struct platform_device *ofdev) |
3141 | { |
3142 | struct device *dev = &ofdev->dev; |
3143 | struct talitos_private *priv = dev_get_drvdata(dev); |
3144 | struct talitos_crypto_alg *t_alg, *n; |
3145 | int i; |
3146 | |
3147 | list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { |
3148 | switch (t_alg->algt.type) { |
3149 | case CRYPTO_ALG_TYPE_SKCIPHER: |
3150 | crypto_unregister_skcipher(alg: &t_alg->algt.alg.skcipher); |
3151 | break; |
3152 | case CRYPTO_ALG_TYPE_AEAD: |
3153 | crypto_unregister_aead(alg: &t_alg->algt.alg.aead); |
3154 | break; |
3155 | case CRYPTO_ALG_TYPE_AHASH: |
3156 | crypto_unregister_ahash(alg: &t_alg->algt.alg.hash); |
3157 | break; |
3158 | } |
3159 | list_del(entry: &t_alg->entry); |
3160 | } |
3161 | |
3162 | if (hw_supports(dev, DESC_HDR_SEL0_RNG)) |
3163 | talitos_unregister_rng(dev); |
3164 | |
3165 | for (i = 0; i < 2; i++) |
3166 | if (priv->irq[i]) { |
3167 | free_irq(priv->irq[i], dev); |
3168 | irq_dispose_mapping(virq: priv->irq[i]); |
3169 | } |
3170 | |
3171 | tasklet_kill(t: &priv->done_task[0]); |
3172 | if (priv->irq[1]) |
3173 | tasklet_kill(t: &priv->done_task[1]); |
3174 | } |
3175 | |
3176 | static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, |
3177 | struct talitos_alg_template |
3178 | *template) |
3179 | { |
3180 | struct talitos_private *priv = dev_get_drvdata(dev); |
3181 | struct talitos_crypto_alg *t_alg; |
3182 | struct crypto_alg *alg; |
3183 | |
3184 | t_alg = devm_kzalloc(dev, size: sizeof(struct talitos_crypto_alg), |
3185 | GFP_KERNEL); |
3186 | if (!t_alg) |
3187 | return ERR_PTR(error: -ENOMEM); |
3188 | |
3189 | t_alg->algt = *template; |
3190 | |
3191 | switch (t_alg->algt.type) { |
3192 | case CRYPTO_ALG_TYPE_SKCIPHER: |
3193 | alg = &t_alg->algt.alg.skcipher.base; |
3194 | alg->cra_exit = talitos_cra_exit; |
3195 | t_alg->algt.alg.skcipher.init = talitos_cra_init_skcipher; |
3196 | t_alg->algt.alg.skcipher.setkey = |
3197 | t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey; |
3198 | t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt; |
3199 | t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt; |
3200 | if (!strcmp(alg->cra_name, "ctr(aes)" ) && !has_ftr_sec1(priv) && |
3201 | DESC_TYPE(t_alg->algt.desc_hdr_template) != |
3202 | DESC_TYPE(DESC_HDR_TYPE_AESU_CTR_NONSNOOP)) { |
3203 | devm_kfree(dev, p: t_alg); |
3204 | return ERR_PTR(error: -ENOTSUPP); |
3205 | } |
3206 | break; |
3207 | case CRYPTO_ALG_TYPE_AEAD: |
3208 | alg = &t_alg->algt.alg.aead.base; |
3209 | alg->cra_exit = talitos_cra_exit; |
3210 | t_alg->algt.alg.aead.init = talitos_cra_init_aead; |
3211 | t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?: |
3212 | aead_setkey; |
3213 | t_alg->algt.alg.aead.encrypt = aead_encrypt; |
3214 | t_alg->algt.alg.aead.decrypt = aead_decrypt; |
3215 | if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && |
3216 | !strncmp(alg->cra_name, "authenc(hmac(sha224)" , 20)) { |
3217 | devm_kfree(dev, p: t_alg); |
3218 | return ERR_PTR(error: -ENOTSUPP); |
3219 | } |
3220 | break; |
3221 | case CRYPTO_ALG_TYPE_AHASH: |
3222 | alg = &t_alg->algt.alg.hash.halg.base; |
3223 | alg->cra_init = talitos_cra_init_ahash; |
3224 | alg->cra_exit = talitos_cra_exit; |
3225 | t_alg->algt.alg.hash.init = ahash_init; |
3226 | t_alg->algt.alg.hash.update = ahash_update; |
3227 | t_alg->algt.alg.hash.final = ahash_final; |
3228 | t_alg->algt.alg.hash.finup = ahash_finup; |
3229 | t_alg->algt.alg.hash.digest = ahash_digest; |
3230 | if (!strncmp(alg->cra_name, "hmac" , 4)) |
3231 | t_alg->algt.alg.hash.setkey = ahash_setkey; |
3232 | t_alg->algt.alg.hash.import = ahash_import; |
3233 | t_alg->algt.alg.hash.export = ahash_export; |
3234 | |
3235 | if (!(priv->features & TALITOS_FTR_HMAC_OK) && |
3236 | !strncmp(alg->cra_name, "hmac" , 4)) { |
3237 | devm_kfree(dev, p: t_alg); |
3238 | return ERR_PTR(error: -ENOTSUPP); |
3239 | } |
3240 | if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && |
3241 | (!strcmp(alg->cra_name, "sha224" ) || |
3242 | !strcmp(alg->cra_name, "hmac(sha224)" ))) { |
3243 | t_alg->algt.alg.hash.init = ahash_init_sha224_swinit; |
3244 | t_alg->algt.alg.hash.digest = |
3245 | ahash_digest_sha224_swinit; |
3246 | t_alg->algt.desc_hdr_template = |
3247 | DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | |
3248 | DESC_HDR_SEL0_MDEUA | |
3249 | DESC_HDR_MODE0_MDEU_SHA256; |
3250 | } |
3251 | break; |
3252 | default: |
3253 | dev_err(dev, "unknown algorithm type %d\n" , t_alg->algt.type); |
3254 | devm_kfree(dev, p: t_alg); |
3255 | return ERR_PTR(error: -EINVAL); |
3256 | } |
3257 | |
3258 | alg->cra_module = THIS_MODULE; |
3259 | if (t_alg->algt.priority) |
3260 | alg->cra_priority = t_alg->algt.priority; |
3261 | else |
3262 | alg->cra_priority = TALITOS_CRA_PRIORITY; |
3263 | if (has_ftr_sec1(priv) && t_alg->algt.type != CRYPTO_ALG_TYPE_AHASH) |
3264 | alg->cra_alignmask = 3; |
3265 | else |
3266 | alg->cra_alignmask = 0; |
3267 | alg->cra_ctxsize = sizeof(struct talitos_ctx); |
3268 | alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; |
3269 | |
3270 | t_alg->dev = dev; |
3271 | |
3272 | return t_alg; |
3273 | } |
3274 | |
3275 | static int talitos_probe_irq(struct platform_device *ofdev) |
3276 | { |
3277 | struct device *dev = &ofdev->dev; |
3278 | struct device_node *np = ofdev->dev.of_node; |
3279 | struct talitos_private *priv = dev_get_drvdata(dev); |
3280 | int err; |
3281 | bool is_sec1 = has_ftr_sec1(priv); |
3282 | |
3283 | priv->irq[0] = irq_of_parse_and_map(node: np, index: 0); |
3284 | if (!priv->irq[0]) { |
3285 | dev_err(dev, "failed to map irq\n" ); |
3286 | return -EINVAL; |
3287 | } |
3288 | if (is_sec1) { |
3289 | err = request_irq(irq: priv->irq[0], handler: talitos1_interrupt_4ch, flags: 0, |
3290 | name: dev_driver_string(dev), dev); |
3291 | goto primary_out; |
3292 | } |
3293 | |
3294 | priv->irq[1] = irq_of_parse_and_map(node: np, index: 1); |
3295 | |
3296 | /* get the primary irq line */ |
3297 | if (!priv->irq[1]) { |
3298 | err = request_irq(irq: priv->irq[0], handler: talitos2_interrupt_4ch, flags: 0, |
3299 | name: dev_driver_string(dev), dev); |
3300 | goto primary_out; |
3301 | } |
3302 | |
3303 | err = request_irq(irq: priv->irq[0], handler: talitos2_interrupt_ch0_2, flags: 0, |
3304 | name: dev_driver_string(dev), dev); |
3305 | if (err) |
3306 | goto primary_out; |
3307 | |
3308 | /* get the secondary irq line */ |
3309 | err = request_irq(irq: priv->irq[1], handler: talitos2_interrupt_ch1_3, flags: 0, |
3310 | name: dev_driver_string(dev), dev); |
3311 | if (err) { |
3312 | dev_err(dev, "failed to request secondary irq\n" ); |
3313 | irq_dispose_mapping(virq: priv->irq[1]); |
3314 | priv->irq[1] = 0; |
3315 | } |
3316 | |
3317 | return err; |
3318 | |
3319 | primary_out: |
3320 | if (err) { |
3321 | dev_err(dev, "failed to request primary irq\n" ); |
3322 | irq_dispose_mapping(virq: priv->irq[0]); |
3323 | priv->irq[0] = 0; |
3324 | } |
3325 | |
3326 | return err; |
3327 | } |
3328 | |
3329 | static int talitos_probe(struct platform_device *ofdev) |
3330 | { |
3331 | struct device *dev = &ofdev->dev; |
3332 | struct device_node *np = ofdev->dev.of_node; |
3333 | struct talitos_private *priv; |
3334 | int i, err; |
3335 | int stride; |
3336 | struct resource *res; |
3337 | |
3338 | priv = devm_kzalloc(dev, size: sizeof(struct talitos_private), GFP_KERNEL); |
3339 | if (!priv) |
3340 | return -ENOMEM; |
3341 | |
3342 | INIT_LIST_HEAD(list: &priv->alg_list); |
3343 | |
3344 | dev_set_drvdata(dev, data: priv); |
3345 | |
3346 | priv->ofdev = ofdev; |
3347 | |
3348 | spin_lock_init(&priv->reg_lock); |
3349 | |
3350 | res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); |
3351 | if (!res) |
3352 | return -ENXIO; |
3353 | priv->reg = devm_ioremap(dev, offset: res->start, size: resource_size(res)); |
3354 | if (!priv->reg) { |
3355 | dev_err(dev, "failed to of_iomap\n" ); |
3356 | err = -ENOMEM; |
3357 | goto err_out; |
3358 | } |
3359 | |
3360 | /* get SEC version capabilities from device tree */ |
3361 | of_property_read_u32(np, propname: "fsl,num-channels" , out_value: &priv->num_channels); |
3362 | of_property_read_u32(np, propname: "fsl,channel-fifo-len" , out_value: &priv->chfifo_len); |
3363 | of_property_read_u32(np, propname: "fsl,exec-units-mask" , out_value: &priv->exec_units); |
3364 | of_property_read_u32(np, propname: "fsl,descriptor-types-mask" , |
3365 | out_value: &priv->desc_types); |
3366 | |
3367 | if (!is_power_of_2(n: priv->num_channels) || !priv->chfifo_len || |
3368 | !priv->exec_units || !priv->desc_types) { |
3369 | dev_err(dev, "invalid property data in device tree node\n" ); |
3370 | err = -EINVAL; |
3371 | goto err_out; |
3372 | } |
3373 | |
3374 | if (of_device_is_compatible(device: np, "fsl,sec3.0" )) |
3375 | priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT; |
3376 | |
3377 | if (of_device_is_compatible(device: np, "fsl,sec2.1" )) |
3378 | priv->features |= TALITOS_FTR_HW_AUTH_CHECK | |
3379 | TALITOS_FTR_SHA224_HWINIT | |
3380 | TALITOS_FTR_HMAC_OK; |
3381 | |
3382 | if (of_device_is_compatible(device: np, "fsl,sec1.0" )) |
3383 | priv->features |= TALITOS_FTR_SEC1; |
3384 | |
3385 | if (of_device_is_compatible(device: np, "fsl,sec1.2" )) { |
3386 | priv->reg_deu = priv->reg + TALITOS12_DEU; |
3387 | priv->reg_aesu = priv->reg + TALITOS12_AESU; |
3388 | priv->reg_mdeu = priv->reg + TALITOS12_MDEU; |
3389 | stride = TALITOS1_CH_STRIDE; |
3390 | } else if (of_device_is_compatible(device: np, "fsl,sec1.0" )) { |
3391 | priv->reg_deu = priv->reg + TALITOS10_DEU; |
3392 | priv->reg_aesu = priv->reg + TALITOS10_AESU; |
3393 | priv->reg_mdeu = priv->reg + TALITOS10_MDEU; |
3394 | priv->reg_afeu = priv->reg + TALITOS10_AFEU; |
3395 | priv->reg_rngu = priv->reg + TALITOS10_RNGU; |
3396 | priv->reg_pkeu = priv->reg + TALITOS10_PKEU; |
3397 | stride = TALITOS1_CH_STRIDE; |
3398 | } else { |
3399 | priv->reg_deu = priv->reg + TALITOS2_DEU; |
3400 | priv->reg_aesu = priv->reg + TALITOS2_AESU; |
3401 | priv->reg_mdeu = priv->reg + TALITOS2_MDEU; |
3402 | priv->reg_afeu = priv->reg + TALITOS2_AFEU; |
3403 | priv->reg_rngu = priv->reg + TALITOS2_RNGU; |
3404 | priv->reg_pkeu = priv->reg + TALITOS2_PKEU; |
3405 | priv->reg_keu = priv->reg + TALITOS2_KEU; |
3406 | priv->reg_crcu = priv->reg + TALITOS2_CRCU; |
3407 | stride = TALITOS2_CH_STRIDE; |
3408 | } |
3409 | |
3410 | err = talitos_probe_irq(ofdev); |
3411 | if (err) |
3412 | goto err_out; |
3413 | |
3414 | if (has_ftr_sec1(priv)) { |
3415 | if (priv->num_channels == 1) |
3416 | tasklet_init(t: &priv->done_task[0], func: talitos1_done_ch0, |
3417 | data: (unsigned long)dev); |
3418 | else |
3419 | tasklet_init(t: &priv->done_task[0], func: talitos1_done_4ch, |
3420 | data: (unsigned long)dev); |
3421 | } else { |
3422 | if (priv->irq[1]) { |
3423 | tasklet_init(t: &priv->done_task[0], func: talitos2_done_ch0_2, |
3424 | data: (unsigned long)dev); |
3425 | tasklet_init(t: &priv->done_task[1], func: talitos2_done_ch1_3, |
3426 | data: (unsigned long)dev); |
3427 | } else if (priv->num_channels == 1) { |
3428 | tasklet_init(t: &priv->done_task[0], func: talitos2_done_ch0, |
3429 | data: (unsigned long)dev); |
3430 | } else { |
3431 | tasklet_init(t: &priv->done_task[0], func: talitos2_done_4ch, |
3432 | data: (unsigned long)dev); |
3433 | } |
3434 | } |
3435 | |
3436 | priv->chan = devm_kcalloc(dev, |
3437 | n: priv->num_channels, |
3438 | size: sizeof(struct talitos_channel), |
3439 | GFP_KERNEL); |
3440 | if (!priv->chan) { |
3441 | dev_err(dev, "failed to allocate channel management space\n" ); |
3442 | err = -ENOMEM; |
3443 | goto err_out; |
3444 | } |
3445 | |
3446 | priv->fifo_len = roundup_pow_of_two(priv->chfifo_len); |
3447 | |
3448 | for (i = 0; i < priv->num_channels; i++) { |
3449 | priv->chan[i].reg = priv->reg + stride * (i + 1); |
3450 | if (!priv->irq[1] || !(i & 1)) |
3451 | priv->chan[i].reg += TALITOS_CH_BASE_OFFSET; |
3452 | |
3453 | spin_lock_init(&priv->chan[i].head_lock); |
3454 | spin_lock_init(&priv->chan[i].tail_lock); |
3455 | |
3456 | priv->chan[i].fifo = devm_kcalloc(dev, |
3457 | n: priv->fifo_len, |
3458 | size: sizeof(struct talitos_request), |
3459 | GFP_KERNEL); |
3460 | if (!priv->chan[i].fifo) { |
3461 | dev_err(dev, "failed to allocate request fifo %d\n" , i); |
3462 | err = -ENOMEM; |
3463 | goto err_out; |
3464 | } |
3465 | |
3466 | atomic_set(v: &priv->chan[i].submit_count, |
3467 | i: -(priv->chfifo_len - 1)); |
3468 | } |
3469 | |
3470 | dma_set_mask(dev, DMA_BIT_MASK(36)); |
3471 | |
3472 | /* reset and initialize the h/w */ |
3473 | err = init_device(dev); |
3474 | if (err) { |
3475 | dev_err(dev, "failed to initialize device\n" ); |
3476 | goto err_out; |
3477 | } |
3478 | |
3479 | /* register the RNG, if available */ |
3480 | if (hw_supports(dev, DESC_HDR_SEL0_RNG)) { |
3481 | err = talitos_register_rng(dev); |
3482 | if (err) { |
3483 | dev_err(dev, "failed to register hwrng: %d\n" , err); |
3484 | goto err_out; |
3485 | } else |
3486 | dev_info(dev, "hwrng\n" ); |
3487 | } |
3488 | |
3489 | /* register crypto algorithms the device supports */ |
3490 | for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { |
3491 | if (hw_supports(dev, desc_hdr_template: driver_algs[i].desc_hdr_template)) { |
3492 | struct talitos_crypto_alg *t_alg; |
3493 | struct crypto_alg *alg = NULL; |
3494 | |
3495 | t_alg = talitos_alg_alloc(dev, template: &driver_algs[i]); |
3496 | if (IS_ERR(ptr: t_alg)) { |
3497 | err = PTR_ERR(ptr: t_alg); |
3498 | if (err == -ENOTSUPP) |
3499 | continue; |
3500 | goto err_out; |
3501 | } |
3502 | |
3503 | switch (t_alg->algt.type) { |
3504 | case CRYPTO_ALG_TYPE_SKCIPHER: |
3505 | err = crypto_register_skcipher( |
3506 | alg: &t_alg->algt.alg.skcipher); |
3507 | alg = &t_alg->algt.alg.skcipher.base; |
3508 | break; |
3509 | |
3510 | case CRYPTO_ALG_TYPE_AEAD: |
3511 | err = crypto_register_aead( |
3512 | alg: &t_alg->algt.alg.aead); |
3513 | alg = &t_alg->algt.alg.aead.base; |
3514 | break; |
3515 | |
3516 | case CRYPTO_ALG_TYPE_AHASH: |
3517 | err = crypto_register_ahash( |
3518 | alg: &t_alg->algt.alg.hash); |
3519 | alg = &t_alg->algt.alg.hash.halg.base; |
3520 | break; |
3521 | } |
3522 | if (err) { |
3523 | dev_err(dev, "%s alg registration failed\n" , |
3524 | alg->cra_driver_name); |
3525 | devm_kfree(dev, p: t_alg); |
3526 | } else |
3527 | list_add_tail(new: &t_alg->entry, head: &priv->alg_list); |
3528 | } |
3529 | } |
3530 | if (!list_empty(head: &priv->alg_list)) |
3531 | dev_info(dev, "%s algorithms registered in /proc/crypto\n" , |
3532 | (char *)of_get_property(np, "compatible" , NULL)); |
3533 | |
3534 | return 0; |
3535 | |
3536 | err_out: |
3537 | talitos_remove(ofdev); |
3538 | |
3539 | return err; |
3540 | } |
3541 | |
3542 | static const struct of_device_id talitos_match[] = { |
3543 | #ifdef CONFIG_CRYPTO_DEV_TALITOS1 |
3544 | { |
3545 | .compatible = "fsl,sec1.0" , |
3546 | }, |
3547 | #endif |
3548 | #ifdef CONFIG_CRYPTO_DEV_TALITOS2 |
3549 | { |
3550 | .compatible = "fsl,sec2.0" , |
3551 | }, |
3552 | #endif |
3553 | {}, |
3554 | }; |
3555 | MODULE_DEVICE_TABLE(of, talitos_match); |
3556 | |
3557 | static struct platform_driver talitos_driver = { |
3558 | .driver = { |
3559 | .name = "talitos" , |
3560 | .of_match_table = talitos_match, |
3561 | }, |
3562 | .probe = talitos_probe, |
3563 | .remove_new = talitos_remove, |
3564 | }; |
3565 | |
3566 | module_platform_driver(talitos_driver); |
3567 | |
3568 | MODULE_LICENSE("GPL" ); |
3569 | MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>" ); |
3570 | MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver" ); |
3571 | |