1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | |
3 | /* |
4 | * Copyright (C) 2021, Linaro Limited. All rights reserved. |
5 | */ |
6 | #include <linux/dma-mapping.h> |
7 | #include <linux/interrupt.h> |
8 | #include <crypto/gcm.h> |
9 | #include <crypto/authenc.h> |
10 | #include <crypto/internal/aead.h> |
11 | #include <crypto/internal/des.h> |
12 | #include <crypto/sha1.h> |
13 | #include <crypto/sha2.h> |
14 | #include <crypto/scatterwalk.h> |
15 | #include "aead.h" |
16 | |
17 | #define CCM_NONCE_ADATA_SHIFT 6 |
18 | #define CCM_NONCE_AUTHSIZE_SHIFT 3 |
19 | #define 6 |
20 | |
21 | static LIST_HEAD(aead_algs); |
22 | |
23 | static void qce_aead_done(void *data) |
24 | { |
25 | struct crypto_async_request *async_req = data; |
26 | struct aead_request *req = aead_request_cast(req: async_req); |
27 | struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); |
28 | struct qce_aead_ctx *ctx = crypto_tfm_ctx(tfm: async_req->tfm); |
29 | struct qce_alg_template *tmpl = to_aead_tmpl(tfm: crypto_aead_reqtfm(req)); |
30 | struct qce_device *qce = tmpl->qce; |
31 | struct qce_result_dump *result_buf = qce->dma.result_buf; |
32 | enum dma_data_direction dir_src, dir_dst; |
33 | bool diff_dst; |
34 | int error; |
35 | u32 status; |
36 | unsigned int totallen; |
37 | unsigned char tag[SHA256_DIGEST_SIZE] = {0}; |
38 | int ret = 0; |
39 | |
40 | diff_dst = (req->src != req->dst) ? true : false; |
41 | dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; |
42 | dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; |
43 | |
44 | error = qce_dma_terminate_all(dma: &qce->dma); |
45 | if (error) |
46 | dev_dbg(qce->dev, "aead dma termination error (%d)\n" , |
47 | error); |
48 | if (diff_dst) |
49 | dma_unmap_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src); |
50 | |
51 | dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); |
52 | |
53 | if (IS_CCM(rctx->flags)) { |
54 | if (req->assoclen) { |
55 | sg_free_table(&rctx->src_tbl); |
56 | if (diff_dst) |
57 | sg_free_table(&rctx->dst_tbl); |
58 | } else { |
59 | if (!(IS_DECRYPT(rctx->flags) && !diff_dst)) |
60 | sg_free_table(&rctx->dst_tbl); |
61 | } |
62 | } else { |
63 | sg_free_table(&rctx->dst_tbl); |
64 | } |
65 | |
66 | error = qce_check_status(qce, status: &status); |
67 | if (error < 0 && (error != -EBADMSG)) |
68 | dev_err(qce->dev, "aead operation error (%x)\n" , status); |
69 | |
70 | if (IS_ENCRYPT(rctx->flags)) { |
71 | totallen = req->cryptlen + req->assoclen; |
72 | if (IS_CCM(rctx->flags)) |
73 | scatterwalk_map_and_copy(buf: rctx->ccmresult_buf, sg: req->dst, |
74 | start: totallen, nbytes: ctx->authsize, out: 1); |
75 | else |
76 | scatterwalk_map_and_copy(buf: result_buf->auth_iv, sg: req->dst, |
77 | start: totallen, nbytes: ctx->authsize, out: 1); |
78 | |
79 | } else if (!IS_CCM(rctx->flags)) { |
80 | totallen = req->cryptlen + req->assoclen - ctx->authsize; |
81 | scatterwalk_map_and_copy(buf: tag, sg: req->src, start: totallen, nbytes: ctx->authsize, out: 0); |
82 | ret = memcmp(p: result_buf->auth_iv, q: tag, size: ctx->authsize); |
83 | if (ret) { |
84 | pr_err("Bad message error\n" ); |
85 | error = -EBADMSG; |
86 | } |
87 | } |
88 | |
89 | qce->async_req_done(qce, error); |
90 | } |
91 | |
92 | static struct scatterlist * |
93 | qce_aead_prepare_result_buf(struct sg_table *tbl, struct aead_request *req) |
94 | { |
95 | struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); |
96 | struct qce_alg_template *tmpl = to_aead_tmpl(tfm: crypto_aead_reqtfm(req)); |
97 | struct qce_device *qce = tmpl->qce; |
98 | |
99 | sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); |
100 | return qce_sgtable_add(sgt: tbl, sg_add: &rctx->result_sg, QCE_RESULT_BUF_SZ); |
101 | } |
102 | |
103 | static struct scatterlist * |
104 | qce_aead_prepare_ccm_result_buf(struct sg_table *tbl, struct aead_request *req) |
105 | { |
106 | struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); |
107 | |
108 | sg_init_one(&rctx->result_sg, rctx->ccmresult_buf, QCE_BAM_BURST_SIZE); |
109 | return qce_sgtable_add(sgt: tbl, sg_add: &rctx->result_sg, QCE_BAM_BURST_SIZE); |
110 | } |
111 | |
112 | static struct scatterlist * |
113 | qce_aead_prepare_dst_buf(struct aead_request *req) |
114 | { |
115 | struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); |
116 | struct qce_alg_template *tmpl = to_aead_tmpl(tfm: crypto_aead_reqtfm(req)); |
117 | struct qce_device *qce = tmpl->qce; |
118 | struct scatterlist *sg, *msg_sg, __sg[2]; |
119 | gfp_t gfp; |
120 | unsigned int assoclen = req->assoclen; |
121 | unsigned int totallen; |
122 | int ret; |
123 | |
124 | totallen = rctx->cryptlen + assoclen; |
125 | rctx->dst_nents = sg_nents_for_len(sg: req->dst, len: totallen); |
126 | if (rctx->dst_nents < 0) { |
127 | dev_err(qce->dev, "Invalid numbers of dst SG.\n" ); |
128 | return ERR_PTR(error: -EINVAL); |
129 | } |
130 | if (IS_CCM(rctx->flags)) |
131 | rctx->dst_nents += 2; |
132 | else |
133 | rctx->dst_nents += 1; |
134 | |
135 | gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? |
136 | GFP_KERNEL : GFP_ATOMIC; |
137 | ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp); |
138 | if (ret) |
139 | return ERR_PTR(error: ret); |
140 | |
141 | if (IS_CCM(rctx->flags) && assoclen) { |
142 | /* Get the dst buffer */ |
143 | msg_sg = scatterwalk_ffwd(dst: __sg, src: req->dst, len: assoclen); |
144 | |
145 | sg = qce_sgtable_add(sgt: &rctx->dst_tbl, sg_add: &rctx->adata_sg, |
146 | max_len: rctx->assoclen); |
147 | if (IS_ERR(ptr: sg)) { |
148 | ret = PTR_ERR(ptr: sg); |
149 | goto dst_tbl_free; |
150 | } |
151 | /* dst buffer */ |
152 | sg = qce_sgtable_add(sgt: &rctx->dst_tbl, sg_add: msg_sg, max_len: rctx->cryptlen); |
153 | if (IS_ERR(ptr: sg)) { |
154 | ret = PTR_ERR(ptr: sg); |
155 | goto dst_tbl_free; |
156 | } |
157 | totallen = rctx->cryptlen + rctx->assoclen; |
158 | } else { |
159 | if (totallen) { |
160 | sg = qce_sgtable_add(sgt: &rctx->dst_tbl, sg_add: req->dst, max_len: totallen); |
161 | if (IS_ERR(ptr: sg)) |
162 | goto dst_tbl_free; |
163 | } |
164 | } |
165 | if (IS_CCM(rctx->flags)) |
166 | sg = qce_aead_prepare_ccm_result_buf(tbl: &rctx->dst_tbl, req); |
167 | else |
168 | sg = qce_aead_prepare_result_buf(tbl: &rctx->dst_tbl, req); |
169 | |
170 | if (IS_ERR(ptr: sg)) |
171 | goto dst_tbl_free; |
172 | |
173 | sg_mark_end(sg); |
174 | rctx->dst_sg = rctx->dst_tbl.sgl; |
175 | rctx->dst_nents = sg_nents_for_len(sg: rctx->dst_sg, len: totallen) + 1; |
176 | |
177 | return sg; |
178 | |
179 | dst_tbl_free: |
180 | sg_free_table(&rctx->dst_tbl); |
181 | return sg; |
182 | } |
183 | |
184 | static int |
185 | qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req) |
186 | { |
187 | struct scatterlist *sg, *msg_sg, __sg[2]; |
188 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); |
189 | struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); |
190 | struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); |
191 | unsigned int assoclen = rctx->assoclen; |
192 | unsigned int , cryptlen, totallen; |
193 | gfp_t gfp; |
194 | bool diff_dst; |
195 | int ret; |
196 | |
197 | if (IS_DECRYPT(rctx->flags)) |
198 | cryptlen = rctx->cryptlen + ctx->authsize; |
199 | else |
200 | cryptlen = rctx->cryptlen; |
201 | totallen = cryptlen + req->assoclen; |
202 | |
203 | /* Get the msg */ |
204 | msg_sg = scatterwalk_ffwd(dst: __sg, src: req->src, len: req->assoclen); |
205 | |
206 | rctx->adata = kzalloc(size: (ALIGN(assoclen, 16) + MAX_CCM_ADATA_HEADER_LEN) * |
207 | sizeof(unsigned char), GFP_ATOMIC); |
208 | if (!rctx->adata) |
209 | return -ENOMEM; |
210 | |
211 | /* |
212 | * Format associated data (RFC3610 and NIST 800-38C) |
213 | * Even though specification allows for AAD to be up to 2^64 - 1 bytes, |
214 | * the assoclen field in aead_request is unsigned int and thus limits |
215 | * the AAD to be up to 2^32 - 1 bytes. So we handle only two scenarios |
216 | * while forming the header for AAD. |
217 | */ |
218 | if (assoclen < 0xff00) { |
219 | adata_header_len = 2; |
220 | *(__be16 *)rctx->adata = cpu_to_be16(assoclen); |
221 | } else { |
222 | adata_header_len = 6; |
223 | *(__be16 *)rctx->adata = cpu_to_be16(0xfffe); |
224 | *(__be32 *)(rctx->adata + 2) = cpu_to_be32(assoclen); |
225 | } |
226 | |
227 | /* Copy the associated data */ |
228 | if (sg_copy_to_buffer(sgl: req->src, nents: sg_nents_for_len(sg: req->src, len: assoclen), |
229 | buf: rctx->adata + adata_header_len, |
230 | buflen: assoclen) != assoclen) |
231 | return -EINVAL; |
232 | |
233 | /* Pad associated data to block size */ |
234 | rctx->assoclen = ALIGN(assoclen + adata_header_len, 16); |
235 | |
236 | diff_dst = (req->src != req->dst) ? true : false; |
237 | |
238 | if (diff_dst) |
239 | rctx->src_nents = sg_nents_for_len(sg: req->src, len: totallen) + 1; |
240 | else |
241 | rctx->src_nents = sg_nents_for_len(sg: req->src, len: totallen) + 2; |
242 | |
243 | gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; |
244 | ret = sg_alloc_table(&rctx->src_tbl, rctx->src_nents, gfp); |
245 | if (ret) |
246 | return ret; |
247 | |
248 | /* Associated Data */ |
249 | sg_init_one(&rctx->adata_sg, rctx->adata, rctx->assoclen); |
250 | sg = qce_sgtable_add(sgt: &rctx->src_tbl, sg_add: &rctx->adata_sg, |
251 | max_len: rctx->assoclen); |
252 | if (IS_ERR(ptr: sg)) { |
253 | ret = PTR_ERR(ptr: sg); |
254 | goto err_free; |
255 | } |
256 | /* src msg */ |
257 | sg = qce_sgtable_add(sgt: &rctx->src_tbl, sg_add: msg_sg, max_len: cryptlen); |
258 | if (IS_ERR(ptr: sg)) { |
259 | ret = PTR_ERR(ptr: sg); |
260 | goto err_free; |
261 | } |
262 | if (!diff_dst) { |
263 | /* |
264 | * For decrypt, when src and dst buffers are same, there is already space |
265 | * in the buffer for padded 0's which is output in lieu of |
266 | * the MAC that is input. So skip the below. |
267 | */ |
268 | if (!IS_DECRYPT(rctx->flags)) { |
269 | sg = qce_aead_prepare_ccm_result_buf(tbl: &rctx->src_tbl, req); |
270 | if (IS_ERR(ptr: sg)) { |
271 | ret = PTR_ERR(ptr: sg); |
272 | goto err_free; |
273 | } |
274 | } |
275 | } |
276 | sg_mark_end(sg); |
277 | rctx->src_sg = rctx->src_tbl.sgl; |
278 | totallen = cryptlen + rctx->assoclen; |
279 | rctx->src_nents = sg_nents_for_len(sg: rctx->src_sg, len: totallen); |
280 | |
281 | if (diff_dst) { |
282 | sg = qce_aead_prepare_dst_buf(req); |
283 | if (IS_ERR(ptr: sg)) { |
284 | ret = PTR_ERR(ptr: sg); |
285 | goto err_free; |
286 | } |
287 | } else { |
288 | if (IS_ENCRYPT(rctx->flags)) |
289 | rctx->dst_nents = rctx->src_nents + 1; |
290 | else |
291 | rctx->dst_nents = rctx->src_nents; |
292 | rctx->dst_sg = rctx->src_sg; |
293 | } |
294 | |
295 | return 0; |
296 | err_free: |
297 | sg_free_table(&rctx->src_tbl); |
298 | return ret; |
299 | } |
300 | |
301 | static int qce_aead_prepare_buf(struct aead_request *req) |
302 | { |
303 | struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); |
304 | struct qce_alg_template *tmpl = to_aead_tmpl(tfm: crypto_aead_reqtfm(req)); |
305 | struct qce_device *qce = tmpl->qce; |
306 | struct scatterlist *sg; |
307 | bool diff_dst = (req->src != req->dst) ? true : false; |
308 | unsigned int totallen; |
309 | |
310 | totallen = rctx->cryptlen + rctx->assoclen; |
311 | |
312 | sg = qce_aead_prepare_dst_buf(req); |
313 | if (IS_ERR(ptr: sg)) |
314 | return PTR_ERR(ptr: sg); |
315 | if (diff_dst) { |
316 | rctx->src_nents = sg_nents_for_len(sg: req->src, len: totallen); |
317 | if (rctx->src_nents < 0) { |
318 | dev_err(qce->dev, "Invalid numbers of src SG.\n" ); |
319 | return -EINVAL; |
320 | } |
321 | rctx->src_sg = req->src; |
322 | } else { |
323 | rctx->src_nents = rctx->dst_nents - 1; |
324 | rctx->src_sg = rctx->dst_sg; |
325 | } |
326 | return 0; |
327 | } |
328 | |
329 | static int qce_aead_ccm_prepare_buf(struct aead_request *req) |
330 | { |
331 | struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); |
332 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); |
333 | struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); |
334 | struct scatterlist *sg; |
335 | bool diff_dst = (req->src != req->dst) ? true : false; |
336 | unsigned int cryptlen; |
337 | |
338 | if (rctx->assoclen) |
339 | return qce_aead_ccm_prepare_buf_assoclen(req); |
340 | |
341 | if (IS_ENCRYPT(rctx->flags)) |
342 | return qce_aead_prepare_buf(req); |
343 | |
344 | cryptlen = rctx->cryptlen + ctx->authsize; |
345 | if (diff_dst) { |
346 | rctx->src_nents = sg_nents_for_len(sg: req->src, len: cryptlen); |
347 | rctx->src_sg = req->src; |
348 | sg = qce_aead_prepare_dst_buf(req); |
349 | if (IS_ERR(ptr: sg)) |
350 | return PTR_ERR(ptr: sg); |
351 | } else { |
352 | rctx->src_nents = sg_nents_for_len(sg: req->src, len: cryptlen); |
353 | rctx->src_sg = req->src; |
354 | rctx->dst_nents = rctx->src_nents; |
355 | rctx->dst_sg = rctx->src_sg; |
356 | } |
357 | |
358 | return 0; |
359 | } |
360 | |
361 | static int qce_aead_create_ccm_nonce(struct qce_aead_reqctx *rctx, struct qce_aead_ctx *ctx) |
362 | { |
363 | unsigned int msglen_size, ivsize; |
364 | u8 msg_len[4]; |
365 | int i; |
366 | |
367 | if (!rctx || !rctx->iv) |
368 | return -EINVAL; |
369 | |
370 | msglen_size = rctx->iv[0] + 1; |
371 | |
372 | /* Verify that msg len size is valid */ |
373 | if (msglen_size < 2 || msglen_size > 8) |
374 | return -EINVAL; |
375 | |
376 | ivsize = rctx->ivsize; |
377 | |
378 | /* |
379 | * Clear the msglen bytes in IV. |
380 | * Else the h/w engine and nonce will use any stray value pending there. |
381 | */ |
382 | if (!IS_CCM_RFC4309(rctx->flags)) { |
383 | for (i = 0; i < msglen_size; i++) |
384 | rctx->iv[ivsize - i - 1] = 0; |
385 | } |
386 | |
387 | /* |
388 | * The crypto framework encodes cryptlen as unsigned int. Thus, even though |
389 | * spec allows for upto 8 bytes to encode msg_len only 4 bytes are needed. |
390 | */ |
391 | if (msglen_size > 4) |
392 | msglen_size = 4; |
393 | |
394 | memcpy(&msg_len[0], &rctx->cryptlen, 4); |
395 | |
396 | memcpy(&rctx->ccm_nonce[0], rctx->iv, rctx->ivsize); |
397 | if (rctx->assoclen) |
398 | rctx->ccm_nonce[0] |= 1 << CCM_NONCE_ADATA_SHIFT; |
399 | rctx->ccm_nonce[0] |= ((ctx->authsize - 2) / 2) << |
400 | CCM_NONCE_AUTHSIZE_SHIFT; |
401 | for (i = 0; i < msglen_size; i++) |
402 | rctx->ccm_nonce[QCE_MAX_NONCE - i - 1] = msg_len[i]; |
403 | |
404 | return 0; |
405 | } |
406 | |
407 | static int |
408 | qce_aead_async_req_handle(struct crypto_async_request *async_req) |
409 | { |
410 | struct aead_request *req = aead_request_cast(req: async_req); |
411 | struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); |
412 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); |
413 | struct qce_aead_ctx *ctx = crypto_tfm_ctx(tfm: async_req->tfm); |
414 | struct qce_alg_template *tmpl = to_aead_tmpl(tfm: crypto_aead_reqtfm(req)); |
415 | struct qce_device *qce = tmpl->qce; |
416 | enum dma_data_direction dir_src, dir_dst; |
417 | bool diff_dst; |
418 | int dst_nents, src_nents, ret; |
419 | |
420 | if (IS_CCM_RFC4309(rctx->flags)) { |
421 | memset(rctx->ccm_rfc4309_iv, 0, QCE_MAX_IV_SIZE); |
422 | rctx->ccm_rfc4309_iv[0] = 3; |
423 | memcpy(&rctx->ccm_rfc4309_iv[1], ctx->ccm4309_salt, QCE_CCM4309_SALT_SIZE); |
424 | memcpy(&rctx->ccm_rfc4309_iv[4], req->iv, 8); |
425 | rctx->iv = rctx->ccm_rfc4309_iv; |
426 | rctx->ivsize = AES_BLOCK_SIZE; |
427 | } else { |
428 | rctx->iv = req->iv; |
429 | rctx->ivsize = crypto_aead_ivsize(tfm); |
430 | } |
431 | if (IS_CCM_RFC4309(rctx->flags)) |
432 | rctx->assoclen = req->assoclen - 8; |
433 | else |
434 | rctx->assoclen = req->assoclen; |
435 | |
436 | diff_dst = (req->src != req->dst) ? true : false; |
437 | dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; |
438 | dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; |
439 | |
440 | if (IS_CCM(rctx->flags)) { |
441 | ret = qce_aead_create_ccm_nonce(rctx, ctx); |
442 | if (ret) |
443 | return ret; |
444 | } |
445 | if (IS_CCM(rctx->flags)) |
446 | ret = qce_aead_ccm_prepare_buf(req); |
447 | else |
448 | ret = qce_aead_prepare_buf(req); |
449 | |
450 | if (ret) |
451 | return ret; |
452 | dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); |
453 | if (!dst_nents) { |
454 | ret = -EIO; |
455 | goto error_free; |
456 | } |
457 | |
458 | if (diff_dst) { |
459 | src_nents = dma_map_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src); |
460 | if (src_nents < 0) { |
461 | ret = src_nents; |
462 | goto error_unmap_dst; |
463 | } |
464 | } else { |
465 | if (IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags)) |
466 | src_nents = dst_nents; |
467 | else |
468 | src_nents = dst_nents - 1; |
469 | } |
470 | |
471 | ret = qce_dma_prep_sgs(dma: &qce->dma, sg_in: rctx->src_sg, in_ents: src_nents, sg_out: rctx->dst_sg, out_ents: dst_nents, |
472 | cb: qce_aead_done, cb_param: async_req); |
473 | if (ret) |
474 | goto error_unmap_src; |
475 | |
476 | qce_dma_issue_pending(dma: &qce->dma); |
477 | |
478 | ret = qce_start(async_req, type: tmpl->crypto_alg_type); |
479 | if (ret) |
480 | goto error_terminate; |
481 | |
482 | return 0; |
483 | |
484 | error_terminate: |
485 | qce_dma_terminate_all(dma: &qce->dma); |
486 | error_unmap_src: |
487 | if (diff_dst) |
488 | dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src); |
489 | error_unmap_dst: |
490 | dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); |
491 | error_free: |
492 | if (IS_CCM(rctx->flags) && rctx->assoclen) { |
493 | sg_free_table(&rctx->src_tbl); |
494 | if (diff_dst) |
495 | sg_free_table(&rctx->dst_tbl); |
496 | } else { |
497 | sg_free_table(&rctx->dst_tbl); |
498 | } |
499 | return ret; |
500 | } |
501 | |
502 | static int qce_aead_crypt(struct aead_request *req, int encrypt) |
503 | { |
504 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); |
505 | struct qce_aead_reqctx *rctx = aead_request_ctx_dma(req); |
506 | struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); |
507 | struct qce_alg_template *tmpl = to_aead_tmpl(tfm); |
508 | unsigned int blocksize = crypto_aead_blocksize(tfm); |
509 | |
510 | rctx->flags = tmpl->alg_flags; |
511 | rctx->flags |= encrypt ? QCE_ENCRYPT : QCE_DECRYPT; |
512 | |
513 | if (encrypt) |
514 | rctx->cryptlen = req->cryptlen; |
515 | else |
516 | rctx->cryptlen = req->cryptlen - ctx->authsize; |
517 | |
518 | /* CE does not handle 0 length messages */ |
519 | if (!rctx->cryptlen) { |
520 | if (!(IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags))) |
521 | ctx->need_fallback = true; |
522 | } |
523 | |
524 | /* If fallback is needed, schedule and exit */ |
525 | if (ctx->need_fallback) { |
526 | /* Reset need_fallback in case the same ctx is used for another transaction */ |
527 | ctx->need_fallback = false; |
528 | |
529 | aead_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback); |
530 | aead_request_set_callback(req: &rctx->fallback_req, flags: req->base.flags, |
531 | compl: req->base.complete, data: req->base.data); |
532 | aead_request_set_crypt(req: &rctx->fallback_req, src: req->src, |
533 | dst: req->dst, cryptlen: req->cryptlen, iv: req->iv); |
534 | aead_request_set_ad(req: &rctx->fallback_req, assoclen: req->assoclen); |
535 | |
536 | return encrypt ? crypto_aead_encrypt(req: &rctx->fallback_req) : |
537 | crypto_aead_decrypt(req: &rctx->fallback_req); |
538 | } |
539 | |
540 | /* |
541 | * CBC algorithms require message lengths to be |
542 | * multiples of block size. |
543 | */ |
544 | if (IS_CBC(rctx->flags) && !IS_ALIGNED(rctx->cryptlen, blocksize)) |
545 | return -EINVAL; |
546 | |
547 | /* RFC4309 supported AAD size 16 bytes/20 bytes */ |
548 | if (IS_CCM_RFC4309(rctx->flags)) |
549 | if (crypto_ipsec_check_assoclen(assoclen: req->assoclen)) |
550 | return -EINVAL; |
551 | |
552 | return tmpl->qce->async_req_enqueue(tmpl->qce, &req->base); |
553 | } |
554 | |
555 | static int qce_aead_encrypt(struct aead_request *req) |
556 | { |
557 | return qce_aead_crypt(req, encrypt: 1); |
558 | } |
559 | |
560 | static int qce_aead_decrypt(struct aead_request *req) |
561 | { |
562 | return qce_aead_crypt(req, encrypt: 0); |
563 | } |
564 | |
565 | static int qce_aead_ccm_setkey(struct crypto_aead *tfm, const u8 *key, |
566 | unsigned int keylen) |
567 | { |
568 | struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); |
569 | unsigned long flags = to_aead_tmpl(tfm)->alg_flags; |
570 | |
571 | if (IS_CCM_RFC4309(flags)) { |
572 | if (keylen < QCE_CCM4309_SALT_SIZE) |
573 | return -EINVAL; |
574 | keylen -= QCE_CCM4309_SALT_SIZE; |
575 | memcpy(ctx->ccm4309_salt, key + keylen, QCE_CCM4309_SALT_SIZE); |
576 | } |
577 | |
578 | if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_256 && keylen != AES_KEYSIZE_192) |
579 | return -EINVAL; |
580 | |
581 | ctx->enc_keylen = keylen; |
582 | ctx->auth_keylen = keylen; |
583 | |
584 | memcpy(ctx->enc_key, key, keylen); |
585 | memcpy(ctx->auth_key, key, keylen); |
586 | |
587 | if (keylen == AES_KEYSIZE_192) |
588 | ctx->need_fallback = true; |
589 | |
590 | return IS_CCM_RFC4309(flags) ? |
591 | crypto_aead_setkey(tfm: ctx->fallback, key, keylen: keylen + QCE_CCM4309_SALT_SIZE) : |
592 | crypto_aead_setkey(tfm: ctx->fallback, key, keylen); |
593 | } |
594 | |
595 | static int qce_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) |
596 | { |
597 | struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); |
598 | struct crypto_authenc_keys authenc_keys; |
599 | unsigned long flags = to_aead_tmpl(tfm)->alg_flags; |
600 | u32 _key[6]; |
601 | int err; |
602 | |
603 | err = crypto_authenc_extractkeys(keys: &authenc_keys, key, keylen); |
604 | if (err) |
605 | return err; |
606 | |
607 | if (authenc_keys.enckeylen > QCE_MAX_KEY_SIZE || |
608 | authenc_keys.authkeylen > QCE_MAX_KEY_SIZE) |
609 | return -EINVAL; |
610 | |
611 | if (IS_DES(flags)) { |
612 | err = verify_aead_des_key(tfm, key: authenc_keys.enckey, keylen: authenc_keys.enckeylen); |
613 | if (err) |
614 | return err; |
615 | } else if (IS_3DES(flags)) { |
616 | err = verify_aead_des3_key(tfm, key: authenc_keys.enckey, keylen: authenc_keys.enckeylen); |
617 | if (err) |
618 | return err; |
619 | /* |
620 | * The crypto engine does not support any two keys |
621 | * being the same for triple des algorithms. The |
622 | * verify_skcipher_des3_key does not check for all the |
623 | * below conditions. Schedule fallback in this case. |
624 | */ |
625 | memcpy(_key, authenc_keys.enckey, DES3_EDE_KEY_SIZE); |
626 | if (!((_key[0] ^ _key[2]) | (_key[1] ^ _key[3])) || |
627 | !((_key[2] ^ _key[4]) | (_key[3] ^ _key[5])) || |
628 | !((_key[0] ^ _key[4]) | (_key[1] ^ _key[5]))) |
629 | ctx->need_fallback = true; |
630 | } else if (IS_AES(flags)) { |
631 | /* No random key sizes */ |
632 | if (authenc_keys.enckeylen != AES_KEYSIZE_128 && |
633 | authenc_keys.enckeylen != AES_KEYSIZE_192 && |
634 | authenc_keys.enckeylen != AES_KEYSIZE_256) |
635 | return -EINVAL; |
636 | if (authenc_keys.enckeylen == AES_KEYSIZE_192) |
637 | ctx->need_fallback = true; |
638 | } |
639 | |
640 | ctx->enc_keylen = authenc_keys.enckeylen; |
641 | ctx->auth_keylen = authenc_keys.authkeylen; |
642 | |
643 | memcpy(ctx->enc_key, authenc_keys.enckey, authenc_keys.enckeylen); |
644 | |
645 | memset(ctx->auth_key, 0, sizeof(ctx->auth_key)); |
646 | memcpy(ctx->auth_key, authenc_keys.authkey, authenc_keys.authkeylen); |
647 | |
648 | return crypto_aead_setkey(tfm: ctx->fallback, key, keylen); |
649 | } |
650 | |
651 | static int qce_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) |
652 | { |
653 | struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); |
654 | unsigned long flags = to_aead_tmpl(tfm)->alg_flags; |
655 | |
656 | if (IS_CCM(flags)) { |
657 | if (authsize < 4 || authsize > 16 || authsize % 2) |
658 | return -EINVAL; |
659 | if (IS_CCM_RFC4309(flags) && (authsize < 8 || authsize % 4)) |
660 | return -EINVAL; |
661 | } |
662 | ctx->authsize = authsize; |
663 | |
664 | return crypto_aead_setauthsize(tfm: ctx->fallback, authsize); |
665 | } |
666 | |
667 | static int qce_aead_init(struct crypto_aead *tfm) |
668 | { |
669 | struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); |
670 | |
671 | ctx->need_fallback = false; |
672 | ctx->fallback = crypto_alloc_aead(alg_name: crypto_tfm_alg_name(tfm: &tfm->base), |
673 | type: 0, CRYPTO_ALG_NEED_FALLBACK); |
674 | |
675 | if (IS_ERR(ptr: ctx->fallback)) |
676 | return PTR_ERR(ptr: ctx->fallback); |
677 | |
678 | crypto_aead_set_reqsize_dma(aead: tfm, reqsize: sizeof(struct qce_aead_reqctx) + |
679 | crypto_aead_reqsize(tfm: ctx->fallback)); |
680 | return 0; |
681 | } |
682 | |
683 | static void qce_aead_exit(struct crypto_aead *tfm) |
684 | { |
685 | struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); |
686 | |
687 | crypto_free_aead(tfm: ctx->fallback); |
688 | } |
689 | |
690 | struct qce_aead_def { |
691 | unsigned long flags; |
692 | const char *name; |
693 | const char *drv_name; |
694 | unsigned int blocksize; |
695 | unsigned int chunksize; |
696 | unsigned int ivsize; |
697 | unsigned int maxauthsize; |
698 | }; |
699 | |
700 | static const struct qce_aead_def aead_def[] = { |
701 | { |
702 | .flags = QCE_ALG_DES | QCE_MODE_CBC | QCE_HASH_SHA1_HMAC, |
703 | .name = "authenc(hmac(sha1),cbc(des))" , |
704 | .drv_name = "authenc-hmac-sha1-cbc-des-qce" , |
705 | .blocksize = DES_BLOCK_SIZE, |
706 | .ivsize = DES_BLOCK_SIZE, |
707 | .maxauthsize = SHA1_DIGEST_SIZE, |
708 | }, |
709 | { |
710 | .flags = QCE_ALG_3DES | QCE_MODE_CBC | QCE_HASH_SHA1_HMAC, |
711 | .name = "authenc(hmac(sha1),cbc(des3_ede))" , |
712 | .drv_name = "authenc-hmac-sha1-cbc-3des-qce" , |
713 | .blocksize = DES3_EDE_BLOCK_SIZE, |
714 | .ivsize = DES3_EDE_BLOCK_SIZE, |
715 | .maxauthsize = SHA1_DIGEST_SIZE, |
716 | }, |
717 | { |
718 | .flags = QCE_ALG_DES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, |
719 | .name = "authenc(hmac(sha256),cbc(des))" , |
720 | .drv_name = "authenc-hmac-sha256-cbc-des-qce" , |
721 | .blocksize = DES_BLOCK_SIZE, |
722 | .ivsize = DES_BLOCK_SIZE, |
723 | .maxauthsize = SHA256_DIGEST_SIZE, |
724 | }, |
725 | { |
726 | .flags = QCE_ALG_3DES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, |
727 | .name = "authenc(hmac(sha256),cbc(des3_ede))" , |
728 | .drv_name = "authenc-hmac-sha256-cbc-3des-qce" , |
729 | .blocksize = DES3_EDE_BLOCK_SIZE, |
730 | .ivsize = DES3_EDE_BLOCK_SIZE, |
731 | .maxauthsize = SHA256_DIGEST_SIZE, |
732 | }, |
733 | { |
734 | .flags = QCE_ALG_AES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, |
735 | .name = "authenc(hmac(sha256),cbc(aes))" , |
736 | .drv_name = "authenc-hmac-sha256-cbc-aes-qce" , |
737 | .blocksize = AES_BLOCK_SIZE, |
738 | .ivsize = AES_BLOCK_SIZE, |
739 | .maxauthsize = SHA256_DIGEST_SIZE, |
740 | }, |
741 | { |
742 | .flags = QCE_ALG_AES | QCE_MODE_CCM, |
743 | .name = "ccm(aes)" , |
744 | .drv_name = "ccm-aes-qce" , |
745 | .blocksize = 1, |
746 | .ivsize = AES_BLOCK_SIZE, |
747 | .maxauthsize = AES_BLOCK_SIZE, |
748 | }, |
749 | { |
750 | .flags = QCE_ALG_AES | QCE_MODE_CCM | QCE_MODE_CCM_RFC4309, |
751 | .name = "rfc4309(ccm(aes))" , |
752 | .drv_name = "rfc4309-ccm-aes-qce" , |
753 | .blocksize = 1, |
754 | .ivsize = 8, |
755 | .maxauthsize = AES_BLOCK_SIZE, |
756 | }, |
757 | }; |
758 | |
759 | static int qce_aead_register_one(const struct qce_aead_def *def, struct qce_device *qce) |
760 | { |
761 | struct qce_alg_template *tmpl; |
762 | struct aead_alg *alg; |
763 | int ret; |
764 | |
765 | tmpl = kzalloc(size: sizeof(*tmpl), GFP_KERNEL); |
766 | if (!tmpl) |
767 | return -ENOMEM; |
768 | |
769 | alg = &tmpl->alg.aead; |
770 | |
771 | snprintf(buf: alg->base.cra_name, CRYPTO_MAX_ALG_NAME, fmt: "%s" , def->name); |
772 | snprintf(buf: alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, fmt: "%s" , |
773 | def->drv_name); |
774 | |
775 | alg->base.cra_blocksize = def->blocksize; |
776 | alg->chunksize = def->chunksize; |
777 | alg->ivsize = def->ivsize; |
778 | alg->maxauthsize = def->maxauthsize; |
779 | if (IS_CCM(def->flags)) |
780 | alg->setkey = qce_aead_ccm_setkey; |
781 | else |
782 | alg->setkey = qce_aead_setkey; |
783 | alg->setauthsize = qce_aead_setauthsize; |
784 | alg->encrypt = qce_aead_encrypt; |
785 | alg->decrypt = qce_aead_decrypt; |
786 | alg->init = qce_aead_init; |
787 | alg->exit = qce_aead_exit; |
788 | |
789 | alg->base.cra_priority = 300; |
790 | alg->base.cra_flags = CRYPTO_ALG_ASYNC | |
791 | CRYPTO_ALG_ALLOCATES_MEMORY | |
792 | CRYPTO_ALG_KERN_DRIVER_ONLY | |
793 | CRYPTO_ALG_NEED_FALLBACK; |
794 | alg->base.cra_ctxsize = sizeof(struct qce_aead_ctx); |
795 | alg->base.cra_alignmask = 0; |
796 | alg->base.cra_module = THIS_MODULE; |
797 | |
798 | INIT_LIST_HEAD(list: &tmpl->entry); |
799 | tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_AEAD; |
800 | tmpl->alg_flags = def->flags; |
801 | tmpl->qce = qce; |
802 | |
803 | ret = crypto_register_aead(alg); |
804 | if (ret) { |
805 | dev_err(qce->dev, "%s registration failed\n" , alg->base.cra_name); |
806 | kfree(objp: tmpl); |
807 | return ret; |
808 | } |
809 | |
810 | list_add_tail(new: &tmpl->entry, head: &aead_algs); |
811 | dev_dbg(qce->dev, "%s is registered\n" , alg->base.cra_name); |
812 | return 0; |
813 | } |
814 | |
815 | static void qce_aead_unregister(struct qce_device *qce) |
816 | { |
817 | struct qce_alg_template *tmpl, *n; |
818 | |
819 | list_for_each_entry_safe(tmpl, n, &aead_algs, entry) { |
820 | crypto_unregister_aead(alg: &tmpl->alg.aead); |
821 | list_del(entry: &tmpl->entry); |
822 | kfree(objp: tmpl); |
823 | } |
824 | } |
825 | |
826 | static int qce_aead_register(struct qce_device *qce) |
827 | { |
828 | int ret, i; |
829 | |
830 | for (i = 0; i < ARRAY_SIZE(aead_def); i++) { |
831 | ret = qce_aead_register_one(def: &aead_def[i], qce); |
832 | if (ret) |
833 | goto err; |
834 | } |
835 | |
836 | return 0; |
837 | err: |
838 | qce_aead_unregister(qce); |
839 | return ret; |
840 | } |
841 | |
842 | const struct qce_algo_ops aead_ops = { |
843 | .type = CRYPTO_ALG_TYPE_AEAD, |
844 | .register_algs = qce_aead_register, |
845 | .unregister_algs = qce_aead_unregister, |
846 | .async_req_handle = qce_aead_async_req_handle, |
847 | }; |
848 | |