1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Cryptographic API. |
4 | * |
5 | * Support for OMAP AES GCM HW acceleration. |
6 | * |
7 | * Copyright (c) 2016 Texas Instruments Incorporated |
8 | */ |
9 | |
10 | #include <crypto/aes.h> |
11 | #include <crypto/engine.h> |
12 | #include <crypto/gcm.h> |
13 | #include <crypto/internal/aead.h> |
14 | #include <crypto/scatterwalk.h> |
15 | #include <crypto/skcipher.h> |
16 | #include <linux/errno.h> |
17 | #include <linux/dma-mapping.h> |
18 | #include <linux/dmaengine.h> |
19 | #include <linux/interrupt.h> |
20 | #include <linux/kernel.h> |
21 | #include <linux/omap-dma.h> |
22 | #include <linux/pm_runtime.h> |
23 | #include <linux/scatterlist.h> |
24 | #include <linux/string.h> |
25 | |
26 | #include "omap-crypto.h" |
27 | #include "omap-aes.h" |
28 | |
29 | static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, |
30 | struct aead_request *req); |
31 | |
32 | static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret) |
33 | { |
34 | struct aead_request *req = dd->aead_req; |
35 | |
36 | dd->in_sg = NULL; |
37 | dd->out_sg = NULL; |
38 | |
39 | crypto_finalize_aead_request(engine: dd->engine, req, err: ret); |
40 | |
41 | pm_runtime_mark_last_busy(dev: dd->dev); |
42 | pm_runtime_put_autosuspend(dev: dd->dev); |
43 | } |
44 | |
45 | static void omap_aes_gcm_done_task(struct omap_aes_dev *dd) |
46 | { |
47 | u8 *tag; |
48 | int alen, clen, i, ret = 0, nsg; |
49 | struct omap_aes_reqctx *rctx; |
50 | |
51 | alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE); |
52 | clen = ALIGN(dd->total, AES_BLOCK_SIZE); |
53 | rctx = aead_request_ctx(req: dd->aead_req); |
54 | |
55 | nsg = !!(dd->assoc_len && dd->total); |
56 | |
57 | dma_sync_sg_for_device(dev: dd->dev, sg: dd->out_sg, nelems: dd->out_sg_len, |
58 | dir: DMA_FROM_DEVICE); |
59 | dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); |
60 | dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); |
61 | omap_aes_crypt_dma_stop(dd); |
62 | |
63 | omap_crypto_cleanup(sg: dd->out_sg, orig: dd->orig_out, |
64 | offset: dd->aead_req->assoclen, len: dd->total, |
65 | FLAGS_OUT_DATA_ST_SHIFT, flags: dd->flags); |
66 | |
67 | if (dd->flags & FLAGS_ENCRYPT) |
68 | scatterwalk_map_and_copy(buf: rctx->auth_tag, |
69 | sg: dd->aead_req->dst, |
70 | start: dd->total + dd->aead_req->assoclen, |
71 | nbytes: dd->authsize, out: 1); |
72 | |
73 | omap_crypto_cleanup(sg: &dd->in_sgl[0], NULL, offset: 0, len: alen, |
74 | FLAGS_ASSOC_DATA_ST_SHIFT, flags: dd->flags); |
75 | |
76 | omap_crypto_cleanup(sg: &dd->in_sgl[nsg], NULL, offset: 0, len: clen, |
77 | FLAGS_IN_DATA_ST_SHIFT, flags: dd->flags); |
78 | |
79 | if (!(dd->flags & FLAGS_ENCRYPT)) { |
80 | tag = (u8 *)rctx->auth_tag; |
81 | for (i = 0; i < dd->authsize; i++) { |
82 | if (tag[i]) { |
83 | ret = -EBADMSG; |
84 | } |
85 | } |
86 | } |
87 | |
88 | omap_aes_gcm_finish_req(dd, ret); |
89 | } |
90 | |
91 | static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd, |
92 | struct aead_request *req) |
93 | { |
94 | int alen, clen, cryptlen, assoclen, ret; |
95 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
96 | unsigned int authlen = crypto_aead_authsize(tfm: aead); |
97 | struct scatterlist *tmp, sg_arr[2]; |
98 | int nsg; |
99 | u16 flags; |
100 | |
101 | assoclen = req->assoclen; |
102 | cryptlen = req->cryptlen; |
103 | |
104 | if (dd->flags & FLAGS_RFC4106_GCM) |
105 | assoclen -= 8; |
106 | |
107 | if (!(dd->flags & FLAGS_ENCRYPT)) |
108 | cryptlen -= authlen; |
109 | |
110 | alen = ALIGN(assoclen, AES_BLOCK_SIZE); |
111 | clen = ALIGN(cryptlen, AES_BLOCK_SIZE); |
112 | |
113 | nsg = !!(assoclen && cryptlen); |
114 | |
115 | omap_aes_clear_copy_flags(dd); |
116 | |
117 | sg_init_table(dd->in_sgl, nsg + 1); |
118 | if (assoclen) { |
119 | tmp = req->src; |
120 | ret = omap_crypto_align_sg(sg: &tmp, total: assoclen, |
121 | AES_BLOCK_SIZE, new_sg: dd->in_sgl, |
122 | OMAP_CRYPTO_COPY_DATA | |
123 | OMAP_CRYPTO_ZERO_BUF | |
124 | OMAP_CRYPTO_FORCE_SINGLE_ENTRY, |
125 | FLAGS_ASSOC_DATA_ST_SHIFT, |
126 | dd_flags: &dd->flags); |
127 | if (ret) |
128 | return ret; |
129 | } |
130 | |
131 | if (cryptlen) { |
132 | tmp = scatterwalk_ffwd(dst: sg_arr, src: req->src, len: req->assoclen); |
133 | |
134 | if (nsg) |
135 | sg_unmark_end(sg: dd->in_sgl); |
136 | |
137 | ret = omap_crypto_align_sg(sg: &tmp, total: cryptlen, |
138 | AES_BLOCK_SIZE, new_sg: &dd->in_sgl[nsg], |
139 | OMAP_CRYPTO_COPY_DATA | |
140 | OMAP_CRYPTO_ZERO_BUF | |
141 | OMAP_CRYPTO_FORCE_SINGLE_ENTRY, |
142 | FLAGS_IN_DATA_ST_SHIFT, |
143 | dd_flags: &dd->flags); |
144 | if (ret) |
145 | return ret; |
146 | } |
147 | |
148 | dd->in_sg = dd->in_sgl; |
149 | dd->total = cryptlen; |
150 | dd->assoc_len = assoclen; |
151 | dd->authsize = authlen; |
152 | |
153 | dd->out_sg = req->dst; |
154 | dd->orig_out = req->dst; |
155 | |
156 | dd->out_sg = scatterwalk_ffwd(dst: sg_arr, src: req->dst, len: req->assoclen); |
157 | |
158 | flags = 0; |
159 | if (req->src == req->dst || dd->out_sg == sg_arr) |
160 | flags |= OMAP_CRYPTO_FORCE_COPY; |
161 | |
162 | if (cryptlen) { |
163 | ret = omap_crypto_align_sg(sg: &dd->out_sg, total: cryptlen, |
164 | AES_BLOCK_SIZE, new_sg: &dd->out_sgl, |
165 | flags, |
166 | FLAGS_OUT_DATA_ST_SHIFT, dd_flags: &dd->flags); |
167 | if (ret) |
168 | return ret; |
169 | } |
170 | |
171 | dd->in_sg_len = sg_nents_for_len(sg: dd->in_sg, len: alen + clen); |
172 | dd->out_sg_len = sg_nents_for_len(sg: dd->out_sg, len: clen); |
173 | |
174 | return 0; |
175 | } |
176 | |
177 | static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) |
178 | { |
179 | struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm: crypto_aead_reqtfm(req)); |
180 | |
181 | aes_encrypt(ctx: &ctx->actx, out: (u8 *)tag, in: (u8 *)iv); |
182 | return 0; |
183 | } |
184 | |
185 | void omap_aes_gcm_dma_out_callback(void *data) |
186 | { |
187 | struct omap_aes_dev *dd = data; |
188 | struct omap_aes_reqctx *rctx; |
189 | int i, val; |
190 | u32 *auth_tag, tag[4]; |
191 | |
192 | if (!(dd->flags & FLAGS_ENCRYPT)) |
193 | scatterwalk_map_and_copy(buf: tag, sg: dd->aead_req->src, |
194 | start: dd->total + dd->aead_req->assoclen, |
195 | nbytes: dd->authsize, out: 0); |
196 | |
197 | rctx = aead_request_ctx(req: dd->aead_req); |
198 | auth_tag = (u32 *)rctx->auth_tag; |
199 | for (i = 0; i < 4; i++) { |
200 | val = omap_aes_read(dd, AES_REG_TAG_N(dd, i)); |
201 | auth_tag[i] = val ^ auth_tag[i]; |
202 | if (!(dd->flags & FLAGS_ENCRYPT)) |
203 | auth_tag[i] = auth_tag[i] ^ tag[i]; |
204 | } |
205 | |
206 | omap_aes_gcm_done_task(dd); |
207 | } |
208 | |
209 | static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd, |
210 | struct aead_request *req) |
211 | { |
212 | if (req) |
213 | return crypto_transfer_aead_request_to_engine(engine: dd->engine, req); |
214 | |
215 | return 0; |
216 | } |
217 | |
218 | static int omap_aes_gcm_prepare_req(struct aead_request *req, |
219 | struct omap_aes_dev *dd) |
220 | { |
221 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); |
222 | struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm: crypto_aead_reqtfm(req)); |
223 | int err; |
224 | |
225 | dd->aead_req = req; |
226 | |
227 | rctx->mode &= FLAGS_MODE_MASK; |
228 | dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; |
229 | |
230 | err = omap_aes_gcm_copy_buffers(dd, req); |
231 | if (err) |
232 | return err; |
233 | |
234 | dd->ctx = &ctx->octx; |
235 | |
236 | return omap_aes_write_ctrl(dd); |
237 | } |
238 | |
239 | static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode) |
240 | { |
241 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); |
242 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
243 | unsigned int authlen = crypto_aead_authsize(tfm: aead); |
244 | struct omap_aes_dev *dd; |
245 | __be32 counter = cpu_to_be32(1); |
246 | int err, assoclen; |
247 | |
248 | memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag)); |
249 | memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4); |
250 | |
251 | err = do_encrypt_iv(req, tag: (u32 *)rctx->auth_tag, iv: (u32 *)rctx->iv); |
252 | if (err) |
253 | return err; |
254 | |
255 | if (mode & FLAGS_RFC4106_GCM) |
256 | assoclen = req->assoclen - 8; |
257 | else |
258 | assoclen = req->assoclen; |
259 | if (assoclen + req->cryptlen == 0) { |
260 | scatterwalk_map_and_copy(buf: rctx->auth_tag, sg: req->dst, start: 0, nbytes: authlen, |
261 | out: 1); |
262 | return 0; |
263 | } |
264 | |
265 | dd = omap_aes_find_dev(rctx); |
266 | if (!dd) |
267 | return -ENODEV; |
268 | rctx->mode = mode; |
269 | |
270 | return omap_aes_gcm_handle_queue(dd, req); |
271 | } |
272 | |
273 | int omap_aes_gcm_encrypt(struct aead_request *req) |
274 | { |
275 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); |
276 | |
277 | memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); |
278 | return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM); |
279 | } |
280 | |
281 | int omap_aes_gcm_decrypt(struct aead_request *req) |
282 | { |
283 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); |
284 | |
285 | memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); |
286 | return omap_aes_gcm_crypt(req, FLAGS_GCM); |
287 | } |
288 | |
289 | int omap_aes_4106gcm_encrypt(struct aead_request *req) |
290 | { |
291 | struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm: crypto_aead_reqtfm(req)); |
292 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); |
293 | |
294 | memcpy(rctx->iv, ctx->octx.nonce, 4); |
295 | memcpy(rctx->iv + 4, req->iv, 8); |
296 | return crypto_ipsec_check_assoclen(assoclen: req->assoclen) ?: |
297 | omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM | |
298 | FLAGS_RFC4106_GCM); |
299 | } |
300 | |
301 | int omap_aes_4106gcm_decrypt(struct aead_request *req) |
302 | { |
303 | struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm: crypto_aead_reqtfm(req)); |
304 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); |
305 | |
306 | memcpy(rctx->iv, ctx->octx.nonce, 4); |
307 | memcpy(rctx->iv + 4, req->iv, 8); |
308 | return crypto_ipsec_check_assoclen(assoclen: req->assoclen) ?: |
309 | omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM); |
310 | } |
311 | |
312 | int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, |
313 | unsigned int keylen) |
314 | { |
315 | struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); |
316 | int ret; |
317 | |
318 | ret = aes_expandkey(ctx: &ctx->actx, in_key: key, key_len: keylen); |
319 | if (ret) |
320 | return ret; |
321 | |
322 | memcpy(ctx->octx.key, key, keylen); |
323 | ctx->octx.keylen = keylen; |
324 | |
325 | return 0; |
326 | } |
327 | |
328 | int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key, |
329 | unsigned int keylen) |
330 | { |
331 | struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); |
332 | int ret; |
333 | |
334 | if (keylen < 4) |
335 | return -EINVAL; |
336 | keylen -= 4; |
337 | |
338 | ret = aes_expandkey(ctx: &ctx->actx, in_key: key, key_len: keylen); |
339 | if (ret) |
340 | return ret; |
341 | |
342 | memcpy(ctx->octx.key, key, keylen); |
343 | memcpy(ctx->octx.nonce, key + keylen, 4); |
344 | ctx->octx.keylen = keylen; |
345 | |
346 | return 0; |
347 | } |
348 | |
349 | int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) |
350 | { |
351 | return crypto_gcm_check_authsize(authsize); |
352 | } |
353 | |
354 | int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent, |
355 | unsigned int authsize) |
356 | { |
357 | return crypto_rfc4106_check_authsize(authsize); |
358 | } |
359 | |
360 | int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq) |
361 | { |
362 | struct aead_request *req = container_of(areq, struct aead_request, |
363 | base); |
364 | struct omap_aes_reqctx *rctx = aead_request_ctx(req); |
365 | struct omap_aes_dev *dd = rctx->dd; |
366 | int ret; |
367 | |
368 | if (!dd) |
369 | return -ENODEV; |
370 | |
371 | ret = omap_aes_gcm_prepare_req(req, dd); |
372 | if (ret) |
373 | return ret; |
374 | |
375 | if (dd->in_sg_len) |
376 | ret = omap_aes_crypt_dma_start(dd); |
377 | else |
378 | omap_aes_gcm_dma_out_callback(data: dd); |
379 | |
380 | return ret; |
381 | } |
382 | |
383 | int omap_aes_gcm_cra_init(struct crypto_aead *tfm) |
384 | { |
385 | crypto_aead_set_reqsize(aead: tfm, reqsize: sizeof(struct omap_aes_reqctx)); |
386 | |
387 | return 0; |
388 | } |
389 | |