1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* caam - Freescale FSL CAAM support for crypto API
4	*
5	* Copyright 2008-2011 Freescale Semiconductor, Inc.
6	* Copyright 2016-2019, 2023 NXP
7	*
8	* Based on talitos crypto API driver.
9	*
10	* relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
11	*
12	* --------------- ---------------
13	* | JobDesc #1 |-------------------->| ShareDesc |
14	* | *(packet 1) | | (PDB) |
15	* --------------- |------------->| (hashKey) |
16	* . | | (cipherKey) |
17	* . | |-------->| (operation) |
18	* --------------- | | ---------------
19	* | JobDesc #2 |------| |
20	* | *(packet 2) | |
21	* --------------- |
22	* . |
23	* . |
24	* --------------- |
25	* | JobDesc #3 |------------
26	* | *(packet 3) |
27	* ---------------
28	*
29	* The SharedDesc never changes for a connection unless rekeyed, but
30	* each packet will likely be in a different place. So all we need
31	* to know to process the packet is where the input is, where the
32	* output goes, and what context we want to process with. Context is
33	* in the SharedDesc, packet references in the JobDesc.
34	*
35	* So, a job desc looks like:
36	*
37	* ---------------------
38	* | Header |
39	* | ShareDesc Pointer |
40	* | SEQ_OUT_PTR |
41	* | (output buffer) |
42	* | (output length) |
43	* | SEQ_IN_PTR |
44	* | (input buffer) |
45	* | (input length) |
46	* ---------------------
47	*/
48
49	#include "compat.h"
50
51	#include "regs.h"
52	#include "intern.h"
53	#include "desc_constr.h"
54	#include "jr.h"
55	#include "error.h"
56	#include "sg_sw_sec4.h"
57	#include "key_gen.h"
58	#include "caamalg_desc.h"
59	#include <asm/unaligned.h>
60	#include <crypto/internal/aead.h>
61	#include <crypto/internal/engine.h>
62	#include <crypto/internal/skcipher.h>
63	#include <crypto/xts.h>
64	#include <linux/dma-mapping.h>
65	#include <linux/device.h>
66	#include <linux/err.h>
67	#include <linux/module.h>
68	#include <linux/kernel.h>
69	#include <linux/slab.h>
70	#include <linux/string.h>
71
72	/*
73	* crypto alg
74	*/
75	#define CAAM_CRA_PRIORITY 3000
76	/* max key is sum of AES_MAX_KEY_SIZE, max split key size */
77	#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \
78	CTR_RFC3686_NONCE_SIZE + \
79	SHA512_DIGEST_SIZE * 2)
80
81	#define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
82	#define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
83	CAAM_CMD_SZ * 4)
84	#define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
85	CAAM_CMD_SZ * 5)
86
87	#define CHACHAPOLY_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6)
88
89	#define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN_MIN)
90	#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
91
92	struct caam_alg_entry {
93	int class1_alg_type;
94	int class2_alg_type;
95	bool rfc3686;
96	bool geniv;
97	bool nodkp;
98	};
99
100	struct caam_aead_alg {
101	struct aead_engine_alg aead;
102	struct caam_alg_entry caam;
103	bool registered;
104	};
105
106	struct caam_skcipher_alg {
107	struct skcipher_engine_alg skcipher;
108	struct caam_alg_entry caam;
109	bool registered;
110	};
111
112	/*
113	* per-session context
114	*/
115	struct caam_ctx {
116	u32 sh_desc_enc[DESC_MAX_USED_LEN];
117	u32 sh_desc_dec[DESC_MAX_USED_LEN];
118	u8 key[CAAM_MAX_KEY_SIZE];
119	dma_addr_t sh_desc_enc_dma;
120	dma_addr_t sh_desc_dec_dma;
121	dma_addr_t key_dma;
122	enum dma_data_direction dir;
123	struct device *jrdev;
124	struct alginfo adata;
125	struct alginfo cdata;
126	unsigned int authsize;
127	bool xts_key_fallback;
128	struct crypto_skcipher *fallback;
129	};
130
131	struct caam_skcipher_req_ctx {
132	struct skcipher_edesc *edesc;
133	struct skcipher_request fallback_req;
134	};
135
136	struct caam_aead_req_ctx {
137	struct aead_edesc *edesc;
138	};
139
140	static int aead_null_set_sh_desc(struct crypto_aead *aead)
141	{
142	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
143	struct device *jrdev = ctx->jrdev;
144	struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev: jrdev->parent);
145	u32 *desc;
146	int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN -
147	ctx->adata.keylen_pad;
148
149	/*
150	* Job Descriptor and Shared Descriptors
151	* must all fit into the 64-word Descriptor h/w Buffer
152	*/
153	if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) {
154	ctx->adata.key_inline = true;
155	ctx->adata.key_virt = ctx->key;
156	} else {
157	ctx->adata.key_inline = false;
158	ctx->adata.key_dma = ctx->key_dma;
159	}
160
161	/* aead_encrypt shared descriptor */
162	desc = ctx->sh_desc_enc;
163	cnstr_shdsc_aead_null_encap(desc, adata: &ctx->adata, icvsize: ctx->authsize,
164	era: ctrlpriv->era);
165	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_enc_dma,
166	size: desc_bytes(desc), dir: ctx->dir);
167
168	/*
169	* Job Descriptor and Shared Descriptors
170	* must all fit into the 64-word Descriptor h/w Buffer
171	*/
172	if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) {
173	ctx->adata.key_inline = true;
174	ctx->adata.key_virt = ctx->key;
175	} else {
176	ctx->adata.key_inline = false;
177	ctx->adata.key_dma = ctx->key_dma;
178	}
179
180	/* aead_decrypt shared descriptor */
181	desc = ctx->sh_desc_dec;
182	cnstr_shdsc_aead_null_decap(desc, adata: &ctx->adata, icvsize: ctx->authsize,
183	era: ctrlpriv->era);
184	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_dec_dma,
185	size: desc_bytes(desc), dir: ctx->dir);
186
187	return 0;
188	}
189
190	static int aead_set_sh_desc(struct crypto_aead *aead)
191	{
192	struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
193	struct caam_aead_alg,
194	aead.base);
195	unsigned int ivsize = crypto_aead_ivsize(tfm: aead);
196	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
197	struct device *jrdev = ctx->jrdev;
198	struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev: jrdev->parent);
199	u32 ctx1_iv_off = 0;
200	u32 *desc, *nonce = NULL;
201	u32 inl_mask;
202	unsigned int data_len[2];
203	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
204	OP_ALG_AAI_CTR_MOD128);
205	const bool is_rfc3686 = alg->caam.rfc3686;
206
207	if (!ctx->authsize)
208	return 0;
209
210	/* NULL encryption / decryption */
211	if (!ctx->cdata.keylen)
212	return aead_null_set_sh_desc(aead);
213
214	/*
215	* AES-CTR needs to load IV in CONTEXT1 reg
216	* at an offset of 128bits (16bytes)
217	* CONTEXT1[255:128] = IV
218	*/
219	if (ctr_mode)
220	ctx1_iv_off = 16;
221
222	/*
223	* RFC3686 specific:
224	* CONTEXT1[255:128] = {NONCE, IV, COUNTER}
225	*/
226	if (is_rfc3686) {
227	ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
228	nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
229	ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
230	}
231
232	/*
233	* In case |user key| > |derived key|, using DKP<imm,imm>
234	* would result in invalid opcodes (last bytes of user key) in
235	* the resulting descriptor. Use DKP<ptr,imm> instead => both
236	* virtual and dma key addresses are needed.
237	*/
238	ctx->adata.key_virt = ctx->key;
239	ctx->adata.key_dma = ctx->key_dma;
240
241	ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
242	ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
243
244	data_len[0] = ctx->adata.keylen_pad;
245	data_len[1] = ctx->cdata.keylen;
246
247	if (alg->caam.geniv)
248	goto skip_enc;
249
250	/*
251	* Job Descriptor and Shared Descriptors
252	* must all fit into the 64-word Descriptor h/w Buffer
253	*/
254	if (desc_inline_query(DESC_AEAD_ENC_LEN +
255	(is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
256	AUTHENC_DESC_JOB_IO_LEN, data_len, inl_mask: &inl_mask,
257	ARRAY_SIZE(data_len)) < 0)
258	return -EINVAL;
259
260	ctx->adata.key_inline = !!(inl_mask & 1);
261	ctx->cdata.key_inline = !!(inl_mask & 2);
262
263	/* aead_encrypt shared descriptor */
264	desc = ctx->sh_desc_enc;
265	cnstr_shdsc_aead_encap(desc, cdata: &ctx->cdata, adata: &ctx->adata, ivsize,
266	icvsize: ctx->authsize, is_rfc3686, nonce, ctx1_iv_off,
267	is_qi: false, era: ctrlpriv->era);
268	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_enc_dma,
269	size: desc_bytes(desc), dir: ctx->dir);
270
271	skip_enc:
272	/*
273	* Job Descriptor and Shared Descriptors
274	* must all fit into the 64-word Descriptor h/w Buffer
275	*/
276	if (desc_inline_query(DESC_AEAD_DEC_LEN +
277	(is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
278	AUTHENC_DESC_JOB_IO_LEN, data_len, inl_mask: &inl_mask,
279	ARRAY_SIZE(data_len)) < 0)
280	return -EINVAL;
281
282	ctx->adata.key_inline = !!(inl_mask & 1);
283	ctx->cdata.key_inline = !!(inl_mask & 2);
284
285	/* aead_decrypt shared descriptor */
286	desc = ctx->sh_desc_dec;
287	cnstr_shdsc_aead_decap(desc, cdata: &ctx->cdata, adata: &ctx->adata, ivsize,
288	icvsize: ctx->authsize, geniv: alg->caam.geniv, is_rfc3686,
289	nonce, ctx1_iv_off, is_qi: false, era: ctrlpriv->era);
290	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_dec_dma,
291	size: desc_bytes(desc), dir: ctx->dir);
292
293	if (!alg->caam.geniv)
294	goto skip_givenc;
295
296	/*
297	* Job Descriptor and Shared Descriptors
298	* must all fit into the 64-word Descriptor h/w Buffer
299	*/
300	if (desc_inline_query(DESC_AEAD_GIVENC_LEN +
301	(is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
302	AUTHENC_DESC_JOB_IO_LEN, data_len, inl_mask: &inl_mask,
303	ARRAY_SIZE(data_len)) < 0)
304	return -EINVAL;
305
306	ctx->adata.key_inline = !!(inl_mask & 1);
307	ctx->cdata.key_inline = !!(inl_mask & 2);
308
309	/* aead_givencrypt shared descriptor */
310	desc = ctx->sh_desc_enc;
311	cnstr_shdsc_aead_givencap(desc, cdata: &ctx->cdata, adata: &ctx->adata, ivsize,
312	icvsize: ctx->authsize, is_rfc3686, nonce,
313	ctx1_iv_off, is_qi: false, era: ctrlpriv->era);
314	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_enc_dma,
315	size: desc_bytes(desc), dir: ctx->dir);
316
317	skip_givenc:
318	return 0;
319	}
320
321	static int aead_setauthsize(struct crypto_aead *authenc,
322	unsigned int authsize)
323	{
324	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: authenc);
325
326	ctx->authsize = authsize;
327	aead_set_sh_desc(aead: authenc);
328
329	return 0;
330	}
331
332	static int gcm_set_sh_desc(struct crypto_aead *aead)
333	{
334	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
335	struct device *jrdev = ctx->jrdev;
336	unsigned int ivsize = crypto_aead_ivsize(tfm: aead);
337	u32 *desc;
338	int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
339	ctx->cdata.keylen;
340
341	if (!ctx->cdata.keylen || !ctx->authsize)
342	return 0;
343
344	/*
345	* AES GCM encrypt shared descriptor
346	* Job Descriptor and Shared Descriptor
347	* must fit into the 64-word Descriptor h/w Buffer
348	*/
349	if (rem_bytes >= DESC_GCM_ENC_LEN) {
350	ctx->cdata.key_inline = true;
351	ctx->cdata.key_virt = ctx->key;
352	} else {
353	ctx->cdata.key_inline = false;
354	ctx->cdata.key_dma = ctx->key_dma;
355	}
356
357	desc = ctx->sh_desc_enc;
358	cnstr_shdsc_gcm_encap(desc, cdata: &ctx->cdata, ivsize, icvsize: ctx->authsize, is_qi: false);
359	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_enc_dma,
360	size: desc_bytes(desc), dir: ctx->dir);
361
362	/*
363	* Job Descriptor and Shared Descriptors
364	* must all fit into the 64-word Descriptor h/w Buffer
365	*/
366	if (rem_bytes >= DESC_GCM_DEC_LEN) {
367	ctx->cdata.key_inline = true;
368	ctx->cdata.key_virt = ctx->key;
369	} else {
370	ctx->cdata.key_inline = false;
371	ctx->cdata.key_dma = ctx->key_dma;
372	}
373
374	desc = ctx->sh_desc_dec;
375	cnstr_shdsc_gcm_decap(desc, cdata: &ctx->cdata, ivsize, icvsize: ctx->authsize, is_qi: false);
376	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_dec_dma,
377	size: desc_bytes(desc), dir: ctx->dir);
378
379	return 0;
380	}
381
382	static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
383	{
384	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: authenc);
385	int err;
386
387	err = crypto_gcm_check_authsize(authsize);
388	if (err)
389	return err;
390
391	ctx->authsize = authsize;
392	gcm_set_sh_desc(aead: authenc);
393
394	return 0;
395	}
396
397	static int rfc4106_set_sh_desc(struct crypto_aead *aead)
398	{
399	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
400	struct device *jrdev = ctx->jrdev;
401	unsigned int ivsize = crypto_aead_ivsize(tfm: aead);
402	u32 *desc;
403	int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
404	ctx->cdata.keylen;
405
406	if (!ctx->cdata.keylen || !ctx->authsize)
407	return 0;
408
409	/*
410	* RFC4106 encrypt shared descriptor
411	* Job Descriptor and Shared Descriptor
412	* must fit into the 64-word Descriptor h/w Buffer
413	*/
414	if (rem_bytes >= DESC_RFC4106_ENC_LEN) {
415	ctx->cdata.key_inline = true;
416	ctx->cdata.key_virt = ctx->key;
417	} else {
418	ctx->cdata.key_inline = false;
419	ctx->cdata.key_dma = ctx->key_dma;
420	}
421
422	desc = ctx->sh_desc_enc;
423	cnstr_shdsc_rfc4106_encap(desc, cdata: &ctx->cdata, ivsize, icvsize: ctx->authsize,
424	is_qi: false);
425	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_enc_dma,
426	size: desc_bytes(desc), dir: ctx->dir);
427
428	/*
429	* Job Descriptor and Shared Descriptors
430	* must all fit into the 64-word Descriptor h/w Buffer
431	*/
432	if (rem_bytes >= DESC_RFC4106_DEC_LEN) {
433	ctx->cdata.key_inline = true;
434	ctx->cdata.key_virt = ctx->key;
435	} else {
436	ctx->cdata.key_inline = false;
437	ctx->cdata.key_dma = ctx->key_dma;
438	}
439
440	desc = ctx->sh_desc_dec;
441	cnstr_shdsc_rfc4106_decap(desc, cdata: &ctx->cdata, ivsize, icvsize: ctx->authsize,
442	is_qi: false);
443	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_dec_dma,
444	size: desc_bytes(desc), dir: ctx->dir);
445
446	return 0;
447	}
448
449	static int rfc4106_setauthsize(struct crypto_aead *authenc,
450	unsigned int authsize)
451	{
452	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: authenc);
453	int err;
454
455	err = crypto_rfc4106_check_authsize(authsize);
456	if (err)
457	return err;
458
459	ctx->authsize = authsize;
460	rfc4106_set_sh_desc(aead: authenc);
461
462	return 0;
463	}
464
465	static int rfc4543_set_sh_desc(struct crypto_aead *aead)
466	{
467	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
468	struct device *jrdev = ctx->jrdev;
469	unsigned int ivsize = crypto_aead_ivsize(tfm: aead);
470	u32 *desc;
471	int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
472	ctx->cdata.keylen;
473
474	if (!ctx->cdata.keylen || !ctx->authsize)
475	return 0;
476
477	/*
478	* RFC4543 encrypt shared descriptor
479	* Job Descriptor and Shared Descriptor
480	* must fit into the 64-word Descriptor h/w Buffer
481	*/
482	if (rem_bytes >= DESC_RFC4543_ENC_LEN) {
483	ctx->cdata.key_inline = true;
484	ctx->cdata.key_virt = ctx->key;
485	} else {
486	ctx->cdata.key_inline = false;
487	ctx->cdata.key_dma = ctx->key_dma;
488	}
489
490	desc = ctx->sh_desc_enc;
491	cnstr_shdsc_rfc4543_encap(desc, cdata: &ctx->cdata, ivsize, icvsize: ctx->authsize,
492	is_qi: false);
493	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_enc_dma,
494	size: desc_bytes(desc), dir: ctx->dir);
495
496	/*
497	* Job Descriptor and Shared Descriptors
498	* must all fit into the 64-word Descriptor h/w Buffer
499	*/
500	if (rem_bytes >= DESC_RFC4543_DEC_LEN) {
501	ctx->cdata.key_inline = true;
502	ctx->cdata.key_virt = ctx->key;
503	} else {
504	ctx->cdata.key_inline = false;
505	ctx->cdata.key_dma = ctx->key_dma;
506	}
507
508	desc = ctx->sh_desc_dec;
509	cnstr_shdsc_rfc4543_decap(desc, cdata: &ctx->cdata, ivsize, icvsize: ctx->authsize,
510	is_qi: false);
511	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_dec_dma,
512	size: desc_bytes(desc), dir: ctx->dir);
513
514	return 0;
515	}
516
517	static int rfc4543_setauthsize(struct crypto_aead *authenc,
518	unsigned int authsize)
519	{
520	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: authenc);
521
522	if (authsize != 16)
523	return -EINVAL;
524
525	ctx->authsize = authsize;
526	rfc4543_set_sh_desc(aead: authenc);
527
528	return 0;
529	}
530
531	static int chachapoly_set_sh_desc(struct crypto_aead *aead)
532	{
533	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
534	struct device *jrdev = ctx->jrdev;
535	unsigned int ivsize = crypto_aead_ivsize(tfm: aead);
536	u32 *desc;
537
538	if (!ctx->cdata.keylen || !ctx->authsize)
539	return 0;
540
541	desc = ctx->sh_desc_enc;
542	cnstr_shdsc_chachapoly(desc, cdata: &ctx->cdata, adata: &ctx->adata, ivsize,
543	icvsize: ctx->authsize, encap: true, is_qi: false);
544	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_enc_dma,
545	size: desc_bytes(desc), dir: ctx->dir);
546
547	desc = ctx->sh_desc_dec;
548	cnstr_shdsc_chachapoly(desc, cdata: &ctx->cdata, adata: &ctx->adata, ivsize,
549	icvsize: ctx->authsize, encap: false, is_qi: false);
550	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_dec_dma,
551	size: desc_bytes(desc), dir: ctx->dir);
552
553	return 0;
554	}
555
556	static int chachapoly_setauthsize(struct crypto_aead *aead,
557	unsigned int authsize)
558	{
559	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
560
561	if (authsize != POLY1305_DIGEST_SIZE)
562	return -EINVAL;
563
564	ctx->authsize = authsize;
565	return chachapoly_set_sh_desc(aead);
566	}
567
568	static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
569	unsigned int keylen)
570	{
571	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
572	unsigned int ivsize = crypto_aead_ivsize(tfm: aead);
573	unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
574
575	if (keylen != CHACHA_KEY_SIZE + saltlen)
576	return -EINVAL;
577
578	memcpy(ctx->key, key, keylen);
579	ctx->cdata.key_virt = ctx->key;
580	ctx->cdata.keylen = keylen - saltlen;
581
582	return chachapoly_set_sh_desc(aead);
583	}
584
585	static int aead_setkey(struct crypto_aead *aead,
586	const u8 *key, unsigned int keylen)
587	{
588	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
589	struct device *jrdev = ctx->jrdev;
590	struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev: jrdev->parent);
591	struct crypto_authenc_keys keys;
592	int ret = 0;
593
594	if (crypto_authenc_extractkeys(keys: &keys, key, keylen) != 0)
595	goto badkey;
596
597	dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
598	keys.authkeylen + keys.enckeylen, keys.enckeylen,
599	keys.authkeylen);
600	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
601	DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
602
603	/*
604	* If DKP is supported, use it in the shared descriptor to generate
605	* the split key.
606	*/
607	if (ctrlpriv->era >= 6) {
608	ctx->adata.keylen = keys.authkeylen;
609	ctx->adata.keylen_pad = split_key_len(hash: ctx->adata.algtype &
610	OP_ALG_ALGSEL_MASK);
611
612	if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
613	goto badkey;
614
615	memcpy(ctx->key, keys.authkey, keys.authkeylen);
616	memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
617	keys.enckeylen);
618	dma_sync_single_for_device(dev: jrdev, addr: ctx->key_dma,
619	size: ctx->adata.keylen_pad +
620	keys.enckeylen, dir: ctx->dir);
621	goto skip_split_key;
622	}
623
624	ret = gen_split_key(jrdev: ctx->jrdev, key_out: ctx->key, adata: &ctx->adata, key_in: keys.authkey,
625	keylen: keys.authkeylen, CAAM_MAX_KEY_SIZE -
626	keys.enckeylen);
627	if (ret) {
628	goto badkey;
629	}
630
631	/* postpend encryption key to auth split key */
632	memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
633	dma_sync_single_for_device(dev: jrdev, addr: ctx->key_dma, size: ctx->adata.keylen_pad +
634	keys.enckeylen, dir: ctx->dir);
635
636	print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
637	DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
638	ctx->adata.keylen_pad + keys.enckeylen, 1);
639
640	skip_split_key:
641	ctx->cdata.keylen = keys.enckeylen;
642	memzero_explicit(s: &keys, count: sizeof(keys));
643	return aead_set_sh_desc(aead);
644	badkey:
645	memzero_explicit(s: &keys, count: sizeof(keys));
646	return -EINVAL;
647	}
648
649	static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
650	unsigned int keylen)
651	{
652	struct crypto_authenc_keys keys;
653	int err;
654
655	err = crypto_authenc_extractkeys(keys: &keys, key, keylen);
656	if (unlikely(err))
657	return err;
658
659	err = verify_aead_des3_key(tfm: aead, key: keys.enckey, keylen: keys.enckeylen) ?:
660	aead_setkey(aead, key, keylen);
661
662	memzero_explicit(s: &keys, count: sizeof(keys));
663	return err;
664	}
665
666	static int gcm_setkey(struct crypto_aead *aead,
667	const u8 *key, unsigned int keylen)
668	{
669	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
670	struct device *jrdev = ctx->jrdev;
671	int err;
672
673	err = aes_check_keylen(keylen);
674	if (err)
675	return err;
676
677	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
678	DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
679
680	memcpy(ctx->key, key, keylen);
681	dma_sync_single_for_device(dev: jrdev, addr: ctx->key_dma, size: keylen, dir: ctx->dir);
682	ctx->cdata.keylen = keylen;
683
684	return gcm_set_sh_desc(aead);
685	}
686
687	static int rfc4106_setkey(struct crypto_aead *aead,
688	const u8 *key, unsigned int keylen)
689	{
690	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
691	struct device *jrdev = ctx->jrdev;
692	int err;
693
694	err = aes_check_keylen(keylen: keylen - 4);
695	if (err)
696	return err;
697
698	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
699	DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
700
701	memcpy(ctx->key, key, keylen);
702
703	/*
704	* The last four bytes of the key material are used as the salt value
705	* in the nonce. Update the AES key length.
706	*/
707	ctx->cdata.keylen = keylen - 4;
708	dma_sync_single_for_device(dev: jrdev, addr: ctx->key_dma, size: ctx->cdata.keylen,
709	dir: ctx->dir);
710	return rfc4106_set_sh_desc(aead);
711	}
712
713	static int rfc4543_setkey(struct crypto_aead *aead,
714	const u8 *key, unsigned int keylen)
715	{
716	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
717	struct device *jrdev = ctx->jrdev;
718	int err;
719
720	err = aes_check_keylen(keylen: keylen - 4);
721	if (err)
722	return err;
723
724	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
725	DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
726
727	memcpy(ctx->key, key, keylen);
728
729	/*
730	* The last four bytes of the key material are used as the salt value
731	* in the nonce. Update the AES key length.
732	*/
733	ctx->cdata.keylen = keylen - 4;
734	dma_sync_single_for_device(dev: jrdev, addr: ctx->key_dma, size: ctx->cdata.keylen,
735	dir: ctx->dir);
736	return rfc4543_set_sh_desc(aead);
737	}
738
739	static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
740	unsigned int keylen, const u32 ctx1_iv_off)
741	{
742	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm: skcipher);
743	struct caam_skcipher_alg *alg =
744	container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
745	skcipher.base);
746	struct device *jrdev = ctx->jrdev;
747	unsigned int ivsize = crypto_skcipher_ivsize(tfm: skcipher);
748	u32 *desc;
749	const bool is_rfc3686 = alg->caam.rfc3686;
750
751	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
752	DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
753
754	ctx->cdata.keylen = keylen;
755	ctx->cdata.key_virt = key;
756	ctx->cdata.key_inline = true;
757
758	/* skcipher_encrypt shared descriptor */
759	desc = ctx->sh_desc_enc;
760	cnstr_shdsc_skcipher_encap(desc, cdata: &ctx->cdata, ivsize, is_rfc3686,
761	ctx1_iv_off);
762	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_enc_dma,
763	size: desc_bytes(desc), dir: ctx->dir);
764
765	/* skcipher_decrypt shared descriptor */
766	desc = ctx->sh_desc_dec;
767	cnstr_shdsc_skcipher_decap(desc, cdata: &ctx->cdata, ivsize, is_rfc3686,
768	ctx1_iv_off);
769	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_dec_dma,
770	size: desc_bytes(desc), dir: ctx->dir);
771
772	return 0;
773	}
774
775	static int aes_skcipher_setkey(struct crypto_skcipher *skcipher,
776	const u8 *key, unsigned int keylen)
777	{
778	int err;
779
780	err = aes_check_keylen(keylen);
781	if (err)
782	return err;
783
784	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off: 0);
785	}
786
787	static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher,
788	const u8 *key, unsigned int keylen)
789	{
790	u32 ctx1_iv_off;
791	int err;
792
793	/*
794	* RFC3686 specific:
795	* | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
796	* | *key = {KEY, NONCE}
797	*/
798	ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
799	keylen -= CTR_RFC3686_NONCE_SIZE;
800
801	err = aes_check_keylen(keylen);
802	if (err)
803	return err;
804
805	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
806	}
807
808	static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher,
809	const u8 *key, unsigned int keylen)
810	{
811	u32 ctx1_iv_off;
812	int err;
813
814	/*
815	* AES-CTR needs to load IV in CONTEXT1 reg
816	* at an offset of 128bits (16bytes)
817	* CONTEXT1[255:128] = IV
818	*/
819	ctx1_iv_off = 16;
820
821	err = aes_check_keylen(keylen);
822	if (err)
823	return err;
824
825	return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off);
826	}
827
828	static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
829	const u8 *key, unsigned int keylen)
830	{
831	return verify_skcipher_des_key(tfm: skcipher, key) ?:
832	skcipher_setkey(skcipher, key, keylen, ctx1_iv_off: 0);
833	}
834
835	static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
836	const u8 *key, unsigned int keylen)
837	{
838	return verify_skcipher_des3_key(tfm: skcipher, key) ?:
839	skcipher_setkey(skcipher, key, keylen, ctx1_iv_off: 0);
840	}
841
842	static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
843	unsigned int keylen)
844	{
845	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm: skcipher);
846	struct device *jrdev = ctx->jrdev;
847	struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev: jrdev->parent);
848	u32 *desc;
849	int err;
850
851	err = xts_verify_key(tfm: skcipher, key, keylen);
852	if (err) {
853	dev_dbg(jrdev, "key size mismatch\n");
854	return err;
855	}
856
857	if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
858	ctx->xts_key_fallback = true;
859
860	if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) {
861	err = crypto_skcipher_setkey(tfm: ctx->fallback, key, keylen);
862	if (err)
863	return err;
864	}
865
866	ctx->cdata.keylen = keylen;
867	ctx->cdata.key_virt = key;
868	ctx->cdata.key_inline = true;
869
870	/* xts_skcipher_encrypt shared descriptor */
871	desc = ctx->sh_desc_enc;
872	cnstr_shdsc_xts_skcipher_encap(desc, cdata: &ctx->cdata);
873	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_enc_dma,
874	size: desc_bytes(desc), dir: ctx->dir);
875
876	/* xts_skcipher_decrypt shared descriptor */
877	desc = ctx->sh_desc_dec;
878	cnstr_shdsc_xts_skcipher_decap(desc, cdata: &ctx->cdata);
879	dma_sync_single_for_device(dev: jrdev, addr: ctx->sh_desc_dec_dma,
880	size: desc_bytes(desc), dir: ctx->dir);
881
882	return 0;
883	}
884
885	/*
886	* aead_edesc - s/w-extended aead descriptor
887	* @src_nents: number of segments in input s/w scatterlist
888	* @dst_nents: number of segments in output s/w scatterlist
889	* @mapped_src_nents: number of segments in input h/w link table
890	* @mapped_dst_nents: number of segments in output h/w link table
891	* @sec4_sg_bytes: length of dma mapped sec4_sg space
892	* @bklog: stored to determine if the request needs backlog
893	* @sec4_sg_dma: bus physical mapped address of h/w link table
894	* @sec4_sg: pointer to h/w link table
895	* @hw_desc: the h/w job descriptor followed by any referenced link tables
896	*/
897	struct aead_edesc {
898	int src_nents;
899	int dst_nents;
900	int mapped_src_nents;
901	int mapped_dst_nents;
902	int sec4_sg_bytes;
903	bool bklog;
904	dma_addr_t sec4_sg_dma;
905	struct sec4_sg_entry *sec4_sg;
906	u32 hw_desc[];
907	};
908
909	/*
910	* skcipher_edesc - s/w-extended skcipher descriptor
911	* @src_nents: number of segments in input s/w scatterlist
912	* @dst_nents: number of segments in output s/w scatterlist
913	* @mapped_src_nents: number of segments in input h/w link table
914	* @mapped_dst_nents: number of segments in output h/w link table
915	* @iv_dma: dma address of iv for checking continuity and link table
916	* @sec4_sg_bytes: length of dma mapped sec4_sg space
917	* @bklog: stored to determine if the request needs backlog
918	* @sec4_sg_dma: bus physical mapped address of h/w link table
919	* @sec4_sg: pointer to h/w link table
920	* @hw_desc: the h/w job descriptor followed by any referenced link tables
921	* and IV
922	*/
923	struct skcipher_edesc {
924	int src_nents;
925	int dst_nents;
926	int mapped_src_nents;
927	int mapped_dst_nents;
928	dma_addr_t iv_dma;
929	int sec4_sg_bytes;
930	bool bklog;
931	dma_addr_t sec4_sg_dma;
932	struct sec4_sg_entry *sec4_sg;
933	u32 hw_desc[];
934	};
935
936	static void caam_unmap(struct device *dev, struct scatterlist *src,
937	struct scatterlist *dst, int src_nents,
938	int dst_nents,
939	dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
940	int sec4_sg_bytes)
941	{
942	if (dst != src) {
943	if (src_nents)
944	dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
945	if (dst_nents)
946	dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
947	} else {
948	dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
949	}
950
951	if (iv_dma)
952	dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL);
953	if (sec4_sg_bytes)
954	dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
955	DMA_TO_DEVICE);
956	}
957
958	static void aead_unmap(struct device *dev,
959	struct aead_edesc *edesc,
960	struct aead_request *req)
961	{
962	caam_unmap(dev, src: req->src, dst: req->dst,
963	src_nents: edesc->src_nents, dst_nents: edesc->dst_nents, iv_dma: 0, ivsize: 0,
964	sec4_sg_dma: edesc->sec4_sg_dma, sec4_sg_bytes: edesc->sec4_sg_bytes);
965	}
966
967	static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
968	struct skcipher_request *req)
969	{
970	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
971	int ivsize = crypto_skcipher_ivsize(tfm: skcipher);
972
973	caam_unmap(dev, src: req->src, dst: req->dst,
974	src_nents: edesc->src_nents, dst_nents: edesc->dst_nents,
975	iv_dma: edesc->iv_dma, ivsize,
976	sec4_sg_dma: edesc->sec4_sg_dma, sec4_sg_bytes: edesc->sec4_sg_bytes);
977	}
978
979	static void aead_crypt_done(struct device *jrdev, u32 *desc, u32 err,
980	void *context)
981	{
982	struct aead_request *req = context;
983	struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
984	struct caam_drv_private_jr *jrp = dev_get_drvdata(dev: jrdev);
985	struct aead_edesc *edesc;
986	int ecode = 0;
987	bool has_bklog;
988
989	dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
990
991	edesc = rctx->edesc;
992	has_bklog = edesc->bklog;
993
994	if (err)
995	ecode = caam_jr_strstatus(jrdev, err);
996
997	aead_unmap(dev: jrdev, edesc, req);
998
999	kfree(objp: edesc);
1000
1001	/*
1002	* If no backlog flag, the completion of the request is done
1003	* by CAAM, not crypto engine.
1004	*/
1005	if (!has_bklog)
1006	aead_request_complete(req, err: ecode);
1007	else
1008	crypto_finalize_aead_request(engine: jrp->engine, req, err: ecode);
1009	}
1010
1011	static inline u8 *skcipher_edesc_iv(struct skcipher_edesc *edesc)
1012	{
1013
1014	return PTR_ALIGN((u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
1015	dma_get_cache_alignment());
1016	}
1017
1018	static void skcipher_crypt_done(struct device *jrdev, u32 *desc, u32 err,
1019	void *context)
1020	{
1021	struct skcipher_request *req = context;
1022	struct skcipher_edesc *edesc;
1023	struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1024	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1025	struct caam_drv_private_jr *jrp = dev_get_drvdata(dev: jrdev);
1026	int ivsize = crypto_skcipher_ivsize(tfm: skcipher);
1027	int ecode = 0;
1028	bool has_bklog;
1029
1030	dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1031
1032	edesc = rctx->edesc;
1033	has_bklog = edesc->bklog;
1034	if (err)
1035	ecode = caam_jr_strstatus(jrdev, err);
1036
1037	skcipher_unmap(dev: jrdev, edesc, req);
1038
1039	/*
1040	* The crypto API expects us to set the IV (req->iv) to the last
1041	* ciphertext block (CBC mode) or last counter (CTR mode).
1042	* This is used e.g. by the CTS mode.
1043	*/
1044	if (ivsize && !ecode) {
1045	memcpy(req->iv, skcipher_edesc_iv(edesc), ivsize);
1046
1047	print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
1048	DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1049	ivsize, 1);
1050	}
1051
1052	caam_dump_sg(prefix_str: "dst @" __stringify(__LINE__)": ",
1053	prefix_type: DUMP_PREFIX_ADDRESS, rowsize: 16, groupsize: 4, sg: req->dst,
1054	tlen: edesc->dst_nents > 1 ? 100 : req->cryptlen, ascii: 1);
1055
1056	kfree(objp: edesc);
1057
1058	/*
1059	* If no backlog flag, the completion of the request is done
1060	* by CAAM, not crypto engine.
1061	*/
1062	if (!has_bklog)
1063	skcipher_request_complete(req, err: ecode);
1064	else
1065	crypto_finalize_skcipher_request(engine: jrp->engine, req, err: ecode);
1066	}
1067
1068	/*
1069	* Fill in aead job descriptor
1070	*/
1071	static void init_aead_job(struct aead_request *req,
1072	struct aead_edesc *edesc,
1073	bool all_contig, bool encrypt)
1074	{
1075	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1076	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
1077	int authsize = ctx->authsize;
1078	u32 *desc = edesc->hw_desc;
1079	u32 out_options, in_options;
1080	dma_addr_t dst_dma, src_dma;
1081	int len, sec4_sg_index = 0;
1082	dma_addr_t ptr;
1083	u32 *sh_desc;
1084
1085	sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
1086	ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
1087
1088	len = desc_len(desc: sh_desc);
1089	init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1090
1091	if (all_contig) {
1092	src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) :
1093	0;
1094	in_options = 0;
1095	} else {
1096	src_dma = edesc->sec4_sg_dma;
1097	sec4_sg_index += edesc->mapped_src_nents;
1098	in_options = LDST_SGF;
1099	}
1100
1101	append_seq_in_ptr(desc, ptr: src_dma, len: req->assoclen + req->cryptlen,
1102	options: in_options);
1103
1104	dst_dma = src_dma;
1105	out_options = in_options;
1106
1107	if (unlikely(req->src != req->dst)) {
1108	if (!edesc->mapped_dst_nents) {
1109	dst_dma = 0;
1110	out_options = 0;
1111	} else if (edesc->mapped_dst_nents == 1) {
1112	dst_dma = sg_dma_address(req->dst);
1113	out_options = 0;
1114	} else {
1115	dst_dma = edesc->sec4_sg_dma +
1116	sec4_sg_index *
1117	sizeof(struct sec4_sg_entry);
1118	out_options = LDST_SGF;
1119	}
1120	}
1121
1122	if (encrypt)
1123	append_seq_out_ptr(desc, ptr: dst_dma,
1124	len: req->assoclen + req->cryptlen + authsize,
1125	options: out_options);
1126	else
1127	append_seq_out_ptr(desc, ptr: dst_dma,
1128	len: req->assoclen + req->cryptlen - authsize,
1129	options: out_options);
1130	}
1131
1132	static void init_gcm_job(struct aead_request *req,
1133	struct aead_edesc *edesc,
1134	bool all_contig, bool encrypt)
1135	{
1136	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1137	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
1138	unsigned int ivsize = crypto_aead_ivsize(tfm: aead);
1139	u32 *desc = edesc->hw_desc;
1140	bool generic_gcm = (ivsize == GCM_AES_IV_SIZE);
1141	unsigned int last;
1142
1143	init_aead_job(req, edesc, all_contig, encrypt);
1144	append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1145
1146	/* BUG This should not be specific to generic GCM. */
1147	last = 0;
1148	if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
1149	last = FIFOLD_TYPE_LAST1;
1150
1151	/* Read GCM IV */
1152	append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1153	FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last);
1154	/* Append Salt */
1155	if (!generic_gcm)
1156	append_data(desc, data: ctx->key + ctx->cdata.keylen, len: 4);
1157	/* Append IV */
1158	append_data(desc, data: req->iv, len: ivsize);
1159	/* End of blank commands */
1160	}
1161
1162	static void init_chachapoly_job(struct aead_request *req,
1163	struct aead_edesc *edesc, bool all_contig,
1164	bool encrypt)
1165	{
1166	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1167	unsigned int ivsize = crypto_aead_ivsize(tfm: aead);
1168	unsigned int assoclen = req->assoclen;
1169	u32 *desc = edesc->hw_desc;
1170	u32 ctx_iv_off = 4;
1171
1172	init_aead_job(req, edesc, all_contig, encrypt);
1173
1174	if (ivsize != CHACHAPOLY_IV_SIZE) {
1175	/* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */
1176	ctx_iv_off += 4;
1177
1178	/*
1179	* The associated data comes already with the IV but we need
1180	* to skip it when we authenticate or encrypt...
1181	*/
1182	assoclen -= ivsize;
1183	}
1184
1185	append_math_add_imm_u32(desc, REG3, ZERO, IMM, assoclen);
1186
1187	/*
1188	* For IPsec load the IV further in the same register.
1189	* For RFC7539 simply load the 12 bytes nonce in a single operation
1190	*/
1191	append_load_as_imm(desc, data: req->iv, len: ivsize, LDST_CLASS_1_CCB |
1192	LDST_SRCDST_BYTE_CONTEXT |
1193	ctx_iv_off << LDST_OFFSET_SHIFT);
1194	}
1195
1196	static void init_authenc_job(struct aead_request *req,
1197	struct aead_edesc *edesc,
1198	bool all_contig, bool encrypt)
1199	{
1200	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1201	struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
1202	struct caam_aead_alg,
1203	aead.base);
1204	unsigned int ivsize = crypto_aead_ivsize(tfm: aead);
1205	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
1206	struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev: ctx->jrdev->parent);
1207	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
1208	OP_ALG_AAI_CTR_MOD128);
1209	const bool is_rfc3686 = alg->caam.rfc3686;
1210	u32 *desc = edesc->hw_desc;
1211	u32 ivoffset = 0;
1212
1213	/*
1214	* AES-CTR needs to load IV in CONTEXT1 reg
1215	* at an offset of 128bits (16bytes)
1216	* CONTEXT1[255:128] = IV
1217	*/
1218	if (ctr_mode)
1219	ivoffset = 16;
1220
1221	/*
1222	* RFC3686 specific:
1223	* CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1224	*/
1225	if (is_rfc3686)
1226	ivoffset = 16 + CTR_RFC3686_NONCE_SIZE;
1227
1228	init_aead_job(req, edesc, all_contig, encrypt);
1229
1230	/*
1231	* {REG3, DPOVRD} = assoclen, depending on whether MATH command supports
1232	* having DPOVRD as destination.
1233	*/
1234	if (ctrlpriv->era < 3)
1235	append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1236	else
1237	append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen);
1238
1239	if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
1240	append_load_as_imm(desc, data: req->iv, len: ivsize,
1241	LDST_CLASS_1_CCB |
1242	LDST_SRCDST_BYTE_CONTEXT |
1243	(ivoffset << LDST_OFFSET_SHIFT));
1244	}
1245
1246	/*
1247	* Fill in skcipher job descriptor
1248	*/
1249	static void init_skcipher_job(struct skcipher_request *req,
1250	struct skcipher_edesc *edesc,
1251	const bool encrypt)
1252	{
1253	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1254	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm: skcipher);
1255	struct device *jrdev = ctx->jrdev;
1256	int ivsize = crypto_skcipher_ivsize(tfm: skcipher);
1257	u32 *desc = edesc->hw_desc;
1258	u32 *sh_desc;
1259	u32 in_options = 0, out_options = 0;
1260	dma_addr_t src_dma, dst_dma, ptr;
1261	int len, sec4_sg_index = 0;
1262
1263	print_hex_dump_debug("presciv@"__stringify(__LINE__)": ",
1264	DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
1265	dev_dbg(jrdev, "asked=%d, cryptlen%d\n",
1266	(int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen);
1267
1268	caam_dump_sg(prefix_str: "src @" __stringify(__LINE__)": ",
1269	prefix_type: DUMP_PREFIX_ADDRESS, rowsize: 16, groupsize: 4, sg: req->src,
1270	tlen: edesc->src_nents > 1 ? 100 : req->cryptlen, ascii: 1);
1271
1272	sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
1273	ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
1274
1275	len = desc_len(desc: sh_desc);
1276	init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1277
1278	if (ivsize || edesc->mapped_src_nents > 1) {
1279	src_dma = edesc->sec4_sg_dma;
1280	sec4_sg_index = edesc->mapped_src_nents + !!ivsize;
1281	in_options = LDST_SGF;
1282	} else {
1283	src_dma = sg_dma_address(req->src);
1284	}
1285
1286	append_seq_in_ptr(desc, ptr: src_dma, len: req->cryptlen + ivsize, options: in_options);
1287
1288	if (likely(req->src == req->dst)) {
1289	dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry);
1290	out_options = in_options;
1291	} else if (!ivsize && edesc->mapped_dst_nents == 1) {
1292	dst_dma = sg_dma_address(req->dst);
1293	} else {
1294	dst_dma = edesc->sec4_sg_dma + sec4_sg_index *
1295	sizeof(struct sec4_sg_entry);
1296	out_options = LDST_SGF;
1297	}
1298
1299	append_seq_out_ptr(desc, ptr: dst_dma, len: req->cryptlen + ivsize, options: out_options);
1300	}
1301
1302	/*
1303	* allocate and map the aead extended descriptor
1304	*/
1305	static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
1306	int desc_bytes, bool *all_contig_ptr,
1307	bool encrypt)
1308	{
1309	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1310	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
1311	struct device *jrdev = ctx->jrdev;
1312	struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
1313	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1314	GFP_KERNEL : GFP_ATOMIC;
1315	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1316	int src_len, dst_len = 0;
1317	struct aead_edesc *edesc;
1318	int sec4_sg_index, sec4_sg_len, sec4_sg_bytes;
1319	unsigned int authsize = ctx->authsize;
1320
1321	if (unlikely(req->dst != req->src)) {
1322	src_len = req->assoclen + req->cryptlen;
1323	dst_len = src_len + (encrypt ? authsize : (-authsize));
1324
1325	src_nents = sg_nents_for_len(sg: req->src, len: src_len);
1326	if (unlikely(src_nents < 0)) {
1327	dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1328	src_len);
1329	return ERR_PTR(error: src_nents);
1330	}
1331
1332	dst_nents = sg_nents_for_len(sg: req->dst, len: dst_len);
1333	if (unlikely(dst_nents < 0)) {
1334	dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1335	dst_len);
1336	return ERR_PTR(error: dst_nents);
1337	}
1338	} else {
1339	src_len = req->assoclen + req->cryptlen +
1340	(encrypt ? authsize : 0);
1341
1342	src_nents = sg_nents_for_len(sg: req->src, len: src_len);
1343	if (unlikely(src_nents < 0)) {
1344	dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1345	src_len);
1346	return ERR_PTR(error: src_nents);
1347	}
1348	}
1349
1350	if (likely(req->src == req->dst)) {
1351	mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1352	DMA_BIDIRECTIONAL);
1353	if (unlikely(!mapped_src_nents)) {
1354	dev_err(jrdev, "unable to map source\n");
1355	return ERR_PTR(error: -ENOMEM);
1356	}
1357	} else {
1358	/* Cover also the case of null (zero length) input data */
1359	if (src_nents) {
1360	mapped_src_nents = dma_map_sg(jrdev, req->src,
1361	src_nents, DMA_TO_DEVICE);
1362	if (unlikely(!mapped_src_nents)) {
1363	dev_err(jrdev, "unable to map source\n");
1364	return ERR_PTR(error: -ENOMEM);
1365	}
1366	} else {
1367	mapped_src_nents = 0;
1368	}
1369
1370	/* Cover also the case of null (zero length) output data */
1371	if (dst_nents) {
1372	mapped_dst_nents = dma_map_sg(jrdev, req->dst,
1373	dst_nents,
1374	DMA_FROM_DEVICE);
1375	if (unlikely(!mapped_dst_nents)) {
1376	dev_err(jrdev, "unable to map destination\n");
1377	dma_unmap_sg(jrdev, req->src, src_nents,
1378	DMA_TO_DEVICE);
1379	return ERR_PTR(error: -ENOMEM);
1380	}
1381	} else {
1382	mapped_dst_nents = 0;
1383	}
1384	}
1385
1386	/*
1387	* HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1388	* the end of the table by allocating more S/G entries.
1389	*/
1390	sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0;
1391	if (mapped_dst_nents > 1)
1392	sec4_sg_len += pad_sg_nents(sg_nents: mapped_dst_nents);
1393	else
1394	sec4_sg_len = pad_sg_nents(sg_nents: sec4_sg_len);
1395
1396	sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1397
1398	/* allocate space for base edesc and hw desc commands, link tables */
1399	edesc = kzalloc(size: sizeof(*edesc) + desc_bytes + sec4_sg_bytes, flags);
1400	if (!edesc) {
1401	caam_unmap(dev: jrdev, src: req->src, dst: req->dst, src_nents, dst_nents, iv_dma: 0,
1402	ivsize: 0, sec4_sg_dma: 0, sec4_sg_bytes: 0);
1403	return ERR_PTR(error: -ENOMEM);
1404	}
1405
1406	edesc->src_nents = src_nents;
1407	edesc->dst_nents = dst_nents;
1408	edesc->mapped_src_nents = mapped_src_nents;
1409	edesc->mapped_dst_nents = mapped_dst_nents;
1410	edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
1411	desc_bytes;
1412
1413	rctx->edesc = edesc;
1414
1415	*all_contig_ptr = !(mapped_src_nents > 1);
1416
1417	sec4_sg_index = 0;
1418	if (mapped_src_nents > 1) {
1419	sg_to_sec4_sg_last(sg: req->src, len: src_len,
1420	sec4_sg_ptr: edesc->sec4_sg + sec4_sg_index, offset: 0);
1421	sec4_sg_index += mapped_src_nents;
1422	}
1423	if (mapped_dst_nents > 1) {
1424	sg_to_sec4_sg_last(sg: req->dst, len: dst_len,
1425	sec4_sg_ptr: edesc->sec4_sg + sec4_sg_index, offset: 0);
1426	}
1427
1428	if (!sec4_sg_bytes)
1429	return edesc;
1430
1431	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1432	sec4_sg_bytes, DMA_TO_DEVICE);
1433	if (dma_mapping_error(dev: jrdev, dma_addr: edesc->sec4_sg_dma)) {
1434	dev_err(jrdev, "unable to map S/G table\n");
1435	aead_unmap(dev: jrdev, edesc, req);
1436	kfree(objp: edesc);
1437	return ERR_PTR(error: -ENOMEM);
1438	}
1439
1440	edesc->sec4_sg_bytes = sec4_sg_bytes;
1441
1442	return edesc;
1443	}
1444
1445	static int aead_enqueue_req(struct device *jrdev, struct aead_request *req)
1446	{
1447	struct caam_drv_private_jr *jrpriv = dev_get_drvdata(dev: jrdev);
1448	struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
1449	struct aead_edesc *edesc = rctx->edesc;
1450	u32 *desc = edesc->hw_desc;
1451	int ret;
1452
1453	/*
1454	* Only the backlog request are sent to crypto-engine since the others
1455	* can be handled by CAAM, if free, especially since JR has up to 1024
1456	* entries (more than the 10 entries from crypto-engine).
1457	*/
1458	if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
1459	ret = crypto_transfer_aead_request_to_engine(engine: jrpriv->engine,
1460	req);
1461	else
1462	ret = caam_jr_enqueue(dev: jrdev, desc, cbk: aead_crypt_done, areq: req);
1463
1464	if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
1465	aead_unmap(dev: jrdev, edesc, req);
1466	kfree(objp: rctx->edesc);
1467	}
1468
1469	return ret;
1470	}
1471
1472	static inline int chachapoly_crypt(struct aead_request *req, bool encrypt)
1473	{
1474	struct aead_edesc *edesc;
1475	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1476	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
1477	struct device *jrdev = ctx->jrdev;
1478	bool all_contig;
1479	u32 *desc;
1480
1481	edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, all_contig_ptr: &all_contig,
1482	encrypt);
1483	if (IS_ERR(ptr: edesc))
1484	return PTR_ERR(ptr: edesc);
1485
1486	desc = edesc->hw_desc;
1487
1488	init_chachapoly_job(req, edesc, all_contig, encrypt);
1489	print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ",
1490	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1491	1);
1492
1493	return aead_enqueue_req(jrdev, req);
1494	}
1495
1496	static int chachapoly_encrypt(struct aead_request *req)
1497	{
1498	return chachapoly_crypt(req, encrypt: true);
1499	}
1500
1501	static int chachapoly_decrypt(struct aead_request *req)
1502	{
1503	return chachapoly_crypt(req, encrypt: false);
1504	}
1505
1506	static inline int aead_crypt(struct aead_request *req, bool encrypt)
1507	{
1508	struct aead_edesc *edesc;
1509	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1510	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
1511	struct device *jrdev = ctx->jrdev;
1512	bool all_contig;
1513
1514	/* allocate extended descriptor */
1515	edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
1516	all_contig_ptr: &all_contig, encrypt);
1517	if (IS_ERR(ptr: edesc))
1518	return PTR_ERR(ptr: edesc);
1519
1520	/* Create and submit job descriptor */
1521	init_authenc_job(req, edesc, all_contig, encrypt);
1522
1523	print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
1524	DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1525	desc_bytes(edesc->hw_desc), 1);
1526
1527	return aead_enqueue_req(jrdev, req);
1528	}
1529
1530	static int aead_encrypt(struct aead_request *req)
1531	{
1532	return aead_crypt(req, encrypt: true);
1533	}
1534
1535	static int aead_decrypt(struct aead_request *req)
1536	{
1537	return aead_crypt(req, encrypt: false);
1538	}
1539
1540	static int aead_do_one_req(struct crypto_engine *engine, void *areq)
1541	{
1542	struct aead_request *req = aead_request_cast(req: areq);
1543	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: crypto_aead_reqtfm(req));
1544	struct caam_aead_req_ctx *rctx = aead_request_ctx(req);
1545	u32 *desc = rctx->edesc->hw_desc;
1546	int ret;
1547
1548	rctx->edesc->bklog = true;
1549
1550	ret = caam_jr_enqueue(dev: ctx->jrdev, desc, cbk: aead_crypt_done, areq: req);
1551
1552	if (ret == -ENOSPC && engine->retry_support)
1553	return ret;
1554
1555	if (ret != -EINPROGRESS) {
1556	aead_unmap(dev: ctx->jrdev, edesc: rctx->edesc, req);
1557	kfree(objp: rctx->edesc);
1558	} else {
1559	ret = 0;
1560	}
1561
1562	return ret;
1563	}
1564
1565	static inline int gcm_crypt(struct aead_request *req, bool encrypt)
1566	{
1567	struct aead_edesc *edesc;
1568	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1569	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
1570	struct device *jrdev = ctx->jrdev;
1571	bool all_contig;
1572
1573	/* allocate extended descriptor */
1574	edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, all_contig_ptr: &all_contig,
1575	encrypt);
1576	if (IS_ERR(ptr: edesc))
1577	return PTR_ERR(ptr: edesc);
1578
1579	/* Create and submit job descriptor */
1580	init_gcm_job(req, edesc, all_contig, encrypt);
1581
1582	print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
1583	DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1584	desc_bytes(edesc->hw_desc), 1);
1585
1586	return aead_enqueue_req(jrdev, req);
1587	}
1588
1589	static int gcm_encrypt(struct aead_request *req)
1590	{
1591	return gcm_crypt(req, encrypt: true);
1592	}
1593
1594	static int gcm_decrypt(struct aead_request *req)
1595	{
1596	return gcm_crypt(req, encrypt: false);
1597	}
1598
1599	static int ipsec_gcm_encrypt(struct aead_request *req)
1600	{
1601	return crypto_ipsec_check_assoclen(assoclen: req->assoclen) ? : gcm_encrypt(req);
1602	}
1603
1604	static int ipsec_gcm_decrypt(struct aead_request *req)
1605	{
1606	return crypto_ipsec_check_assoclen(assoclen: req->assoclen) ? : gcm_decrypt(req);
1607	}
1608
1609	/*
1610	* allocate and map the skcipher extended descriptor for skcipher
1611	*/
1612	static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
1613	int desc_bytes)
1614	{
1615	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1616	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm: skcipher);
1617	struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1618	struct device *jrdev = ctx->jrdev;
1619	gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1620	GFP_KERNEL : GFP_ATOMIC;
1621	int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1622	struct skcipher_edesc *edesc;
1623	dma_addr_t iv_dma = 0;
1624	u8 *iv;
1625	int ivsize = crypto_skcipher_ivsize(tfm: skcipher);
1626	int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
1627	unsigned int aligned_size;
1628
1629	src_nents = sg_nents_for_len(sg: req->src, len: req->cryptlen);
1630	if (unlikely(src_nents < 0)) {
1631	dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1632	req->cryptlen);
1633	return ERR_PTR(error: src_nents);
1634	}
1635
1636	if (req->dst != req->src) {
1637	dst_nents = sg_nents_for_len(sg: req->dst, len: req->cryptlen);
1638	if (unlikely(dst_nents < 0)) {
1639	dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1640	req->cryptlen);
1641	return ERR_PTR(error: dst_nents);
1642	}
1643	}
1644
1645	if (likely(req->src == req->dst)) {
1646	mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1647	DMA_BIDIRECTIONAL);
1648	if (unlikely(!mapped_src_nents)) {
1649	dev_err(jrdev, "unable to map source\n");
1650	return ERR_PTR(error: -ENOMEM);
1651	}
1652	} else {
1653	mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1654	DMA_TO_DEVICE);
1655	if (unlikely(!mapped_src_nents)) {
1656	dev_err(jrdev, "unable to map source\n");
1657	return ERR_PTR(error: -ENOMEM);
1658	}
1659	mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
1660	DMA_FROM_DEVICE);
1661	if (unlikely(!mapped_dst_nents)) {
1662	dev_err(jrdev, "unable to map destination\n");
1663	dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1664	return ERR_PTR(error: -ENOMEM);
1665	}
1666	}
1667
1668	if (!ivsize && mapped_src_nents == 1)
1669	sec4_sg_ents = 0; // no need for an input hw s/g table
1670	else
1671	sec4_sg_ents = mapped_src_nents + !!ivsize;
1672	dst_sg_idx = sec4_sg_ents;
1673
1674	/*
1675	* Input, output HW S/G tables: [IV, src][dst, IV]
1676	* IV entries point to the same buffer
1677	* If src == dst, S/G entries are reused (S/G tables overlap)
1678	*
1679	* HW reads 4 S/G entries at a time; make sure the reads don't go beyond
1680	* the end of the table by allocating more S/G entries. Logic:
1681	* if (output S/G)
1682	* pad output S/G, if needed
1683	* else if (input S/G) ...
1684	* pad input S/G, if needed
1685	*/
1686	if (ivsize || mapped_dst_nents > 1) {
1687	if (req->src == req->dst)
1688	sec4_sg_ents = !!ivsize + pad_sg_nents(sg_nents: sec4_sg_ents);
1689	else
1690	sec4_sg_ents += pad_sg_nents(sg_nents: mapped_dst_nents +
1691	!!ivsize);
1692	} else {
1693	sec4_sg_ents = pad_sg_nents(sg_nents: sec4_sg_ents);
1694	}
1695
1696	sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
1697
1698	/*
1699	* allocate space for base edesc and hw desc commands, link tables, IV
1700	*/
1701	aligned_size = sizeof(*edesc) + desc_bytes + sec4_sg_bytes;
1702	aligned_size = ALIGN(aligned_size, dma_get_cache_alignment());
1703	aligned_size += ~(ARCH_KMALLOC_MINALIGN - 1) &
1704	(dma_get_cache_alignment() - 1);
1705	aligned_size += ALIGN(ivsize, dma_get_cache_alignment());
1706	edesc = kzalloc(size: aligned_size, flags);
1707	if (!edesc) {
1708	dev_err(jrdev, "could not allocate extended descriptor\n");
1709	caam_unmap(dev: jrdev, src: req->src, dst: req->dst, src_nents, dst_nents, iv_dma: 0,
1710	ivsize: 0, sec4_sg_dma: 0, sec4_sg_bytes: 0);
1711	return ERR_PTR(error: -ENOMEM);
1712	}
1713
1714	edesc->src_nents = src_nents;
1715	edesc->dst_nents = dst_nents;
1716	edesc->mapped_src_nents = mapped_src_nents;
1717	edesc->mapped_dst_nents = mapped_dst_nents;
1718	edesc->sec4_sg_bytes = sec4_sg_bytes;
1719	edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc +
1720	desc_bytes);
1721	rctx->edesc = edesc;
1722
1723	/* Make sure IV is located in a DMAable area */
1724	if (ivsize) {
1725	iv = skcipher_edesc_iv(edesc);
1726	memcpy(iv, req->iv, ivsize);
1727
1728	iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);
1729	if (dma_mapping_error(dev: jrdev, dma_addr: iv_dma)) {
1730	dev_err(jrdev, "unable to map IV\n");
1731	caam_unmap(dev: jrdev, src: req->src, dst: req->dst, src_nents,
1732	dst_nents, iv_dma: 0, ivsize: 0, sec4_sg_dma: 0, sec4_sg_bytes: 0);
1733	kfree(objp: edesc);
1734	return ERR_PTR(error: -ENOMEM);
1735	}
1736
1737	dma_to_sec4_sg_one(sec4_sg_ptr: edesc->sec4_sg, dma: iv_dma, len: ivsize, offset: 0);
1738	}
1739	if (dst_sg_idx)
1740	sg_to_sec4_sg(sg: req->src, len: req->cryptlen, sec4_sg_ptr: edesc->sec4_sg +
1741	!!ivsize, offset: 0);
1742
1743	if (req->src != req->dst && (ivsize || mapped_dst_nents > 1))
1744	sg_to_sec4_sg(sg: req->dst, len: req->cryptlen, sec4_sg_ptr: edesc->sec4_sg +
1745	dst_sg_idx, offset: 0);
1746
1747	if (ivsize)
1748	dma_to_sec4_sg_one(sec4_sg_ptr: edesc->sec4_sg + dst_sg_idx +
1749	mapped_dst_nents, dma: iv_dma, len: ivsize, offset: 0);
1750
1751	if (ivsize || mapped_dst_nents > 1)
1752	sg_to_sec4_set_last(sec4_sg_ptr: edesc->sec4_sg + dst_sg_idx +
1753	mapped_dst_nents - 1 + !!ivsize);
1754
1755	if (sec4_sg_bytes) {
1756	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1757	sec4_sg_bytes,
1758	DMA_TO_DEVICE);
1759	if (dma_mapping_error(dev: jrdev, dma_addr: edesc->sec4_sg_dma)) {
1760	dev_err(jrdev, "unable to map S/G table\n");
1761	caam_unmap(dev: jrdev, src: req->src, dst: req->dst, src_nents,
1762	dst_nents, iv_dma, ivsize, sec4_sg_dma: 0, sec4_sg_bytes: 0);
1763	kfree(objp: edesc);
1764	return ERR_PTR(error: -ENOMEM);
1765	}
1766	}
1767
1768	edesc->iv_dma = iv_dma;
1769
1770	print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ",
1771	DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
1772	sec4_sg_bytes, 1);
1773
1774	return edesc;
1775	}
1776
1777	static int skcipher_do_one_req(struct crypto_engine *engine, void *areq)
1778	{
1779	struct skcipher_request *req = skcipher_request_cast(req: areq);
1780	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm: crypto_skcipher_reqtfm(req));
1781	struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1782	u32 *desc = rctx->edesc->hw_desc;
1783	int ret;
1784
1785	rctx->edesc->bklog = true;
1786
1787	ret = caam_jr_enqueue(dev: ctx->jrdev, desc, cbk: skcipher_crypt_done, areq: req);
1788
1789	if (ret == -ENOSPC && engine->retry_support)
1790	return ret;
1791
1792	if (ret != -EINPROGRESS) {
1793	skcipher_unmap(dev: ctx->jrdev, edesc: rctx->edesc, req);
1794	kfree(objp: rctx->edesc);
1795	} else {
1796	ret = 0;
1797	}
1798
1799	return ret;
1800	}
1801
1802	static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
1803	{
1804	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1805	unsigned int ivsize = crypto_skcipher_ivsize(tfm: skcipher);
1806
1807	return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
1808	}
1809
1810	static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
1811	{
1812	struct skcipher_edesc *edesc;
1813	struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1814	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm: skcipher);
1815	struct device *jrdev = ctx->jrdev;
1816	struct caam_drv_private_jr *jrpriv = dev_get_drvdata(dev: jrdev);
1817	struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev: jrdev->parent);
1818	u32 *desc;
1819	int ret = 0;
1820
1821	/*
1822	* XTS is expected to return an error even for input length = 0
1823	* Note that the case input length < block size will be caught during
1824	* HW offloading and return an error.
1825	*/
1826	if (!req->cryptlen && !ctx->fallback)
1827	return 0;
1828
1829	if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) ||
1830	ctx->xts_key_fallback)) {
1831	struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
1832
1833	skcipher_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback);
1834	skcipher_request_set_callback(req: &rctx->fallback_req,
1835	flags: req->base.flags,
1836	compl: req->base.complete,
1837	data: req->base.data);
1838	skcipher_request_set_crypt(req: &rctx->fallback_req, src: req->src,
1839	dst: req->dst, cryptlen: req->cryptlen, iv: req->iv);
1840
1841	return encrypt ? crypto_skcipher_encrypt(req: &rctx->fallback_req) :
1842	crypto_skcipher_decrypt(req: &rctx->fallback_req);
1843	}
1844
1845	/* allocate extended descriptor */
1846	edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
1847	if (IS_ERR(ptr: edesc))
1848	return PTR_ERR(ptr: edesc);
1849
1850	/* Create and submit job descriptor*/
1851	init_skcipher_job(req, edesc, encrypt);
1852
1853	print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ",
1854	DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1855	desc_bytes(edesc->hw_desc), 1);
1856
1857	desc = edesc->hw_desc;
1858	/*
1859	* Only the backlog request are sent to crypto-engine since the others
1860	* can be handled by CAAM, if free, especially since JR has up to 1024
1861	* entries (more than the 10 entries from crypto-engine).
1862	*/
1863	if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
1864	ret = crypto_transfer_skcipher_request_to_engine(engine: jrpriv->engine,
1865	req);
1866	else
1867	ret = caam_jr_enqueue(dev: jrdev, desc, cbk: skcipher_crypt_done, areq: req);
1868
1869	if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
1870	skcipher_unmap(dev: jrdev, edesc, req);
1871	kfree(objp: edesc);
1872	}
1873
1874	return ret;
1875	}
1876
1877	static int skcipher_encrypt(struct skcipher_request *req)
1878	{
1879	return skcipher_crypt(req, encrypt: true);
1880	}
1881
1882	static int skcipher_decrypt(struct skcipher_request *req)
1883	{
1884	return skcipher_crypt(req, encrypt: false);
1885	}
1886
1887	static struct caam_skcipher_alg driver_algs[] = {
1888	{
1889	.skcipher.base = {
1890	.base = {
1891	.cra_name = "cbc(aes)",
1892	.cra_driver_name = "cbc-aes-caam",
1893	.cra_blocksize = AES_BLOCK_SIZE,
1894	},
1895	.setkey = aes_skcipher_setkey,
1896	.encrypt = skcipher_encrypt,
1897	.decrypt = skcipher_decrypt,
1898	.min_keysize = AES_MIN_KEY_SIZE,
1899	.max_keysize = AES_MAX_KEY_SIZE,
1900	.ivsize = AES_BLOCK_SIZE,
1901	},
1902	.skcipher.op = {
1903	.do_one_request = skcipher_do_one_req,
1904	},
1905	.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1906	},
1907	{
1908	.skcipher.base = {
1909	.base = {
1910	.cra_name = "cbc(des3_ede)",
1911	.cra_driver_name = "cbc-3des-caam",
1912	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1913	},
1914	.setkey = des3_skcipher_setkey,
1915	.encrypt = skcipher_encrypt,
1916	.decrypt = skcipher_decrypt,
1917	.min_keysize = DES3_EDE_KEY_SIZE,
1918	.max_keysize = DES3_EDE_KEY_SIZE,
1919	.ivsize = DES3_EDE_BLOCK_SIZE,
1920	},
1921	.skcipher.op = {
1922	.do_one_request = skcipher_do_one_req,
1923	},
1924	.caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1925	},
1926	{
1927	.skcipher.base = {
1928	.base = {
1929	.cra_name = "cbc(des)",
1930	.cra_driver_name = "cbc-des-caam",
1931	.cra_blocksize = DES_BLOCK_SIZE,
1932	},
1933	.setkey = des_skcipher_setkey,
1934	.encrypt = skcipher_encrypt,
1935	.decrypt = skcipher_decrypt,
1936	.min_keysize = DES_KEY_SIZE,
1937	.max_keysize = DES_KEY_SIZE,
1938	.ivsize = DES_BLOCK_SIZE,
1939	},
1940	.skcipher.op = {
1941	.do_one_request = skcipher_do_one_req,
1942	},
1943	.caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1944	},
1945	{
1946	.skcipher.base = {
1947	.base = {
1948	.cra_name = "ctr(aes)",
1949	.cra_driver_name = "ctr-aes-caam",
1950	.cra_blocksize = 1,
1951	},
1952	.setkey = ctr_skcipher_setkey,
1953	.encrypt = skcipher_encrypt,
1954	.decrypt = skcipher_decrypt,
1955	.min_keysize = AES_MIN_KEY_SIZE,
1956	.max_keysize = AES_MAX_KEY_SIZE,
1957	.ivsize = AES_BLOCK_SIZE,
1958	.chunksize = AES_BLOCK_SIZE,
1959	},
1960	.skcipher.op = {
1961	.do_one_request = skcipher_do_one_req,
1962	},
1963	.caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1964	OP_ALG_AAI_CTR_MOD128,
1965	},
1966	{
1967	.skcipher.base = {
1968	.base = {
1969	.cra_name = "rfc3686(ctr(aes))",
1970	.cra_driver_name = "rfc3686-ctr-aes-caam",
1971	.cra_blocksize = 1,
1972	},
1973	.setkey = rfc3686_skcipher_setkey,
1974	.encrypt = skcipher_encrypt,
1975	.decrypt = skcipher_decrypt,
1976	.min_keysize = AES_MIN_KEY_SIZE +
1977	CTR_RFC3686_NONCE_SIZE,
1978	.max_keysize = AES_MAX_KEY_SIZE +
1979	CTR_RFC3686_NONCE_SIZE,
1980	.ivsize = CTR_RFC3686_IV_SIZE,
1981	.chunksize = AES_BLOCK_SIZE,
1982	},
1983	.skcipher.op = {
1984	.do_one_request = skcipher_do_one_req,
1985	},
1986	.caam = {
1987	.class1_alg_type = OP_ALG_ALGSEL_AES |
1988	OP_ALG_AAI_CTR_MOD128,
1989	.rfc3686 = true,
1990	},
1991	},
1992	{
1993	.skcipher.base = {
1994	.base = {
1995	.cra_name = "xts(aes)",
1996	.cra_driver_name = "xts-aes-caam",
1997	.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
1998	.cra_blocksize = AES_BLOCK_SIZE,
1999	},
2000	.setkey = xts_skcipher_setkey,
2001	.encrypt = skcipher_encrypt,
2002	.decrypt = skcipher_decrypt,
2003	.min_keysize = 2 * AES_MIN_KEY_SIZE,
2004	.max_keysize = 2 * AES_MAX_KEY_SIZE,
2005	.ivsize = AES_BLOCK_SIZE,
2006	},
2007	.skcipher.op = {
2008	.do_one_request = skcipher_do_one_req,
2009	},
2010	.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
2011	},
2012	{
2013	.skcipher.base = {
2014	.base = {
2015	.cra_name = "ecb(des)",
2016	.cra_driver_name = "ecb-des-caam",
2017	.cra_blocksize = DES_BLOCK_SIZE,
2018	},
2019	.setkey = des_skcipher_setkey,
2020	.encrypt = skcipher_encrypt,
2021	.decrypt = skcipher_decrypt,
2022	.min_keysize = DES_KEY_SIZE,
2023	.max_keysize = DES_KEY_SIZE,
2024	},
2025	.skcipher.op = {
2026	.do_one_request = skcipher_do_one_req,
2027	},
2028	.caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB,
2029	},
2030	{
2031	.skcipher.base = {
2032	.base = {
2033	.cra_name = "ecb(aes)",
2034	.cra_driver_name = "ecb-aes-caam",
2035	.cra_blocksize = AES_BLOCK_SIZE,
2036	},
2037	.setkey = aes_skcipher_setkey,
2038	.encrypt = skcipher_encrypt,
2039	.decrypt = skcipher_decrypt,
2040	.min_keysize = AES_MIN_KEY_SIZE,
2041	.max_keysize = AES_MAX_KEY_SIZE,
2042	},
2043	.skcipher.op = {
2044	.do_one_request = skcipher_do_one_req,
2045	},
2046	.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB,
2047	},
2048	{
2049	.skcipher.base = {
2050	.base = {
2051	.cra_name = "ecb(des3_ede)",
2052	.cra_driver_name = "ecb-des3-caam",
2053	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2054	},
2055	.setkey = des3_skcipher_setkey,
2056	.encrypt = skcipher_encrypt,
2057	.decrypt = skcipher_decrypt,
2058	.min_keysize = DES3_EDE_KEY_SIZE,
2059	.max_keysize = DES3_EDE_KEY_SIZE,
2060	},
2061	.skcipher.op = {
2062	.do_one_request = skcipher_do_one_req,
2063	},
2064	.caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB,
2065	},
2066	};
2067
2068	static struct caam_aead_alg driver_aeads[] = {
2069	{
2070	.aead.base = {
2071	.base = {
2072	.cra_name = "rfc4106(gcm(aes))",
2073	.cra_driver_name = "rfc4106-gcm-aes-caam",
2074	.cra_blocksize = 1,
2075	},
2076	.setkey = rfc4106_setkey,
2077	.setauthsize = rfc4106_setauthsize,
2078	.encrypt = ipsec_gcm_encrypt,
2079	.decrypt = ipsec_gcm_decrypt,
2080	.ivsize = GCM_RFC4106_IV_SIZE,
2081	.maxauthsize = AES_BLOCK_SIZE,
2082	},
2083	.aead.op = {
2084	.do_one_request = aead_do_one_req,
2085	},
2086	.caam = {
2087	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2088	.nodkp = true,
2089	},
2090	},
2091	{
2092	.aead.base = {
2093	.base = {
2094	.cra_name = "rfc4543(gcm(aes))",
2095	.cra_driver_name = "rfc4543-gcm-aes-caam",
2096	.cra_blocksize = 1,
2097	},
2098	.setkey = rfc4543_setkey,
2099	.setauthsize = rfc4543_setauthsize,
2100	.encrypt = ipsec_gcm_encrypt,
2101	.decrypt = ipsec_gcm_decrypt,
2102	.ivsize = GCM_RFC4543_IV_SIZE,
2103	.maxauthsize = AES_BLOCK_SIZE,
2104	},
2105	.aead.op = {
2106	.do_one_request = aead_do_one_req,
2107	},
2108	.caam = {
2109	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2110	.nodkp = true,
2111	},
2112	},
2113	/* Galois Counter Mode */
2114	{
2115	.aead.base = {
2116	.base = {
2117	.cra_name = "gcm(aes)",
2118	.cra_driver_name = "gcm-aes-caam",
2119	.cra_blocksize = 1,
2120	},
2121	.setkey = gcm_setkey,
2122	.setauthsize = gcm_setauthsize,
2123	.encrypt = gcm_encrypt,
2124	.decrypt = gcm_decrypt,
2125	.ivsize = GCM_AES_IV_SIZE,
2126	.maxauthsize = AES_BLOCK_SIZE,
2127	},
2128	.aead.op = {
2129	.do_one_request = aead_do_one_req,
2130	},
2131	.caam = {
2132	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2133	.nodkp = true,
2134	},
2135	},
2136	/* single-pass ipsec_esp descriptor */
2137	{
2138	.aead.base = {
2139	.base = {
2140	.cra_name = "authenc(hmac(md5),"
2141	"ecb(cipher_null))",
2142	.cra_driver_name = "authenc-hmac-md5-"
2143	"ecb-cipher_null-caam",
2144	.cra_blocksize = NULL_BLOCK_SIZE,
2145	},
2146	.setkey = aead_setkey,
2147	.setauthsize = aead_setauthsize,
2148	.encrypt = aead_encrypt,
2149	.decrypt = aead_decrypt,
2150	.ivsize = NULL_IV_SIZE,
2151	.maxauthsize = MD5_DIGEST_SIZE,
2152	},
2153	.aead.op = {
2154	.do_one_request = aead_do_one_req,
2155	},
2156	.caam = {
2157	.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2158	OP_ALG_AAI_HMAC_PRECOMP,
2159	},
2160	},
2161	{
2162	.aead.base = {
2163	.base = {
2164	.cra_name = "authenc(hmac(sha1),"
2165	"ecb(cipher_null))",
2166	.cra_driver_name = "authenc-hmac-sha1-"
2167	"ecb-cipher_null-caam",
2168	.cra_blocksize = NULL_BLOCK_SIZE,
2169	},
2170	.setkey = aead_setkey,
2171	.setauthsize = aead_setauthsize,
2172	.encrypt = aead_encrypt,
2173	.decrypt = aead_decrypt,
2174	.ivsize = NULL_IV_SIZE,
2175	.maxauthsize = SHA1_DIGEST_SIZE,
2176	},
2177	.aead.op = {
2178	.do_one_request = aead_do_one_req,
2179	},
2180	.caam = {
2181	.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2182	OP_ALG_AAI_HMAC_PRECOMP,
2183	},
2184	},
2185	{
2186	.aead.base = {
2187	.base = {
2188	.cra_name = "authenc(hmac(sha224),"
2189	"ecb(cipher_null))",
2190	.cra_driver_name = "authenc-hmac-sha224-"
2191	"ecb-cipher_null-caam",
2192	.cra_blocksize = NULL_BLOCK_SIZE,
2193	},
2194	.setkey = aead_setkey,
2195	.setauthsize = aead_setauthsize,
2196	.encrypt = aead_encrypt,
2197	.decrypt = aead_decrypt,
2198	.ivsize = NULL_IV_SIZE,
2199	.maxauthsize = SHA224_DIGEST_SIZE,
2200	},
2201	.aead.op = {
2202	.do_one_request = aead_do_one_req,
2203	},
2204	.caam = {
2205	.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2206	OP_ALG_AAI_HMAC_PRECOMP,
2207	},
2208	},
2209	{
2210	.aead.base = {
2211	.base = {
2212	.cra_name = "authenc(hmac(sha256),"
2213	"ecb(cipher_null))",
2214	.cra_driver_name = "authenc-hmac-sha256-"
2215	"ecb-cipher_null-caam",
2216	.cra_blocksize = NULL_BLOCK_SIZE,
2217	},
2218	.setkey = aead_setkey,
2219	.setauthsize = aead_setauthsize,
2220	.encrypt = aead_encrypt,
2221	.decrypt = aead_decrypt,
2222	.ivsize = NULL_IV_SIZE,
2223	.maxauthsize = SHA256_DIGEST_SIZE,
2224	},
2225	.aead.op = {
2226	.do_one_request = aead_do_one_req,
2227	},
2228	.caam = {
2229	.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2230	OP_ALG_AAI_HMAC_PRECOMP,
2231	},
2232	},
2233	{
2234	.aead.base = {
2235	.base = {
2236	.cra_name = "authenc(hmac(sha384),"
2237	"ecb(cipher_null))",
2238	.cra_driver_name = "authenc-hmac-sha384-"
2239	"ecb-cipher_null-caam",
2240	.cra_blocksize = NULL_BLOCK_SIZE,
2241	},
2242	.setkey = aead_setkey,
2243	.setauthsize = aead_setauthsize,
2244	.encrypt = aead_encrypt,
2245	.decrypt = aead_decrypt,
2246	.ivsize = NULL_IV_SIZE,
2247	.maxauthsize = SHA384_DIGEST_SIZE,
2248	},
2249	.aead.op = {
2250	.do_one_request = aead_do_one_req,
2251	},
2252	.caam = {
2253	.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2254	OP_ALG_AAI_HMAC_PRECOMP,
2255	},
2256	},
2257	{
2258	.aead.base = {
2259	.base = {
2260	.cra_name = "authenc(hmac(sha512),"
2261	"ecb(cipher_null))",
2262	.cra_driver_name = "authenc-hmac-sha512-"
2263	"ecb-cipher_null-caam",
2264	.cra_blocksize = NULL_BLOCK_SIZE,
2265	},
2266	.setkey = aead_setkey,
2267	.setauthsize = aead_setauthsize,
2268	.encrypt = aead_encrypt,
2269	.decrypt = aead_decrypt,
2270	.ivsize = NULL_IV_SIZE,
2271	.maxauthsize = SHA512_DIGEST_SIZE,
2272	},
2273	.aead.op = {
2274	.do_one_request = aead_do_one_req,
2275	},
2276	.caam = {
2277	.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2278	OP_ALG_AAI_HMAC_PRECOMP,
2279	},
2280	},
2281	{
2282	.aead.base = {
2283	.base = {
2284	.cra_name = "authenc(hmac(md5),cbc(aes))",
2285	.cra_driver_name = "authenc-hmac-md5-"
2286	"cbc-aes-caam",
2287	.cra_blocksize = AES_BLOCK_SIZE,
2288	},
2289	.setkey = aead_setkey,
2290	.setauthsize = aead_setauthsize,
2291	.encrypt = aead_encrypt,
2292	.decrypt = aead_decrypt,
2293	.ivsize = AES_BLOCK_SIZE,
2294	.maxauthsize = MD5_DIGEST_SIZE,
2295	},
2296	.aead.op = {
2297	.do_one_request = aead_do_one_req,
2298	},
2299	.caam = {
2300	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2301	.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2302	OP_ALG_AAI_HMAC_PRECOMP,
2303	},
2304	},
2305	{
2306	.aead.base = {
2307	.base = {
2308	.cra_name = "echainiv(authenc(hmac(md5),"
2309	"cbc(aes)))",
2310	.cra_driver_name = "echainiv-authenc-hmac-md5-"
2311	"cbc-aes-caam",
2312	.cra_blocksize = AES_BLOCK_SIZE,
2313	},
2314	.setkey = aead_setkey,
2315	.setauthsize = aead_setauthsize,
2316	.encrypt = aead_encrypt,
2317	.decrypt = aead_decrypt,
2318	.ivsize = AES_BLOCK_SIZE,
2319	.maxauthsize = MD5_DIGEST_SIZE,
2320	},
2321	.aead.op = {
2322	.do_one_request = aead_do_one_req,
2323	},
2324	.caam = {
2325	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2326	.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2327	OP_ALG_AAI_HMAC_PRECOMP,
2328	.geniv = true,
2329	},
2330	},
2331	{
2332	.aead.base = {
2333	.base = {
2334	.cra_name = "authenc(hmac(sha1),cbc(aes))",
2335	.cra_driver_name = "authenc-hmac-sha1-"
2336	"cbc-aes-caam",
2337	.cra_blocksize = AES_BLOCK_SIZE,
2338	},
2339	.setkey = aead_setkey,
2340	.setauthsize = aead_setauthsize,
2341	.encrypt = aead_encrypt,
2342	.decrypt = aead_decrypt,
2343	.ivsize = AES_BLOCK_SIZE,
2344	.maxauthsize = SHA1_DIGEST_SIZE,
2345	},
2346	.aead.op = {
2347	.do_one_request = aead_do_one_req,
2348	},
2349	.caam = {
2350	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2351	.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2352	OP_ALG_AAI_HMAC_PRECOMP,
2353	},
2354	},
2355	{
2356	.aead.base = {
2357	.base = {
2358	.cra_name = "echainiv(authenc(hmac(sha1),"
2359	"cbc(aes)))",
2360	.cra_driver_name = "echainiv-authenc-"
2361	"hmac-sha1-cbc-aes-caam",
2362	.cra_blocksize = AES_BLOCK_SIZE,
2363	},
2364	.setkey = aead_setkey,
2365	.setauthsize = aead_setauthsize,
2366	.encrypt = aead_encrypt,
2367	.decrypt = aead_decrypt,
2368	.ivsize = AES_BLOCK_SIZE,
2369	.maxauthsize = SHA1_DIGEST_SIZE,
2370	},
2371	.aead.op = {
2372	.do_one_request = aead_do_one_req,
2373	},
2374	.caam = {
2375	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2376	.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2377	OP_ALG_AAI_HMAC_PRECOMP,
2378	.geniv = true,
2379	},
2380	},
2381	{
2382	.aead.base = {
2383	.base = {
2384	.cra_name = "authenc(hmac(sha224),cbc(aes))",
2385	.cra_driver_name = "authenc-hmac-sha224-"
2386	"cbc-aes-caam",
2387	.cra_blocksize = AES_BLOCK_SIZE,
2388	},
2389	.setkey = aead_setkey,
2390	.setauthsize = aead_setauthsize,
2391	.encrypt = aead_encrypt,
2392	.decrypt = aead_decrypt,
2393	.ivsize = AES_BLOCK_SIZE,
2394	.maxauthsize = SHA224_DIGEST_SIZE,
2395	},
2396	.aead.op = {
2397	.do_one_request = aead_do_one_req,
2398	},
2399	.caam = {
2400	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2401	.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2402	OP_ALG_AAI_HMAC_PRECOMP,
2403	},
2404	},
2405	{
2406	.aead.base = {
2407	.base = {
2408	.cra_name = "echainiv(authenc(hmac(sha224),"
2409	"cbc(aes)))",
2410	.cra_driver_name = "echainiv-authenc-"
2411	"hmac-sha224-cbc-aes-caam",
2412	.cra_blocksize = AES_BLOCK_SIZE,
2413	},
2414	.setkey = aead_setkey,
2415	.setauthsize = aead_setauthsize,
2416	.encrypt = aead_encrypt,
2417	.decrypt = aead_decrypt,
2418	.ivsize = AES_BLOCK_SIZE,
2419	.maxauthsize = SHA224_DIGEST_SIZE,
2420	},
2421	.aead.op = {
2422	.do_one_request = aead_do_one_req,
2423	},
2424	.caam = {
2425	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2426	.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2427	OP_ALG_AAI_HMAC_PRECOMP,
2428	.geniv = true,
2429	},
2430	},
2431	{
2432	.aead.base = {
2433	.base = {
2434	.cra_name = "authenc(hmac(sha256),cbc(aes))",
2435	.cra_driver_name = "authenc-hmac-sha256-"
2436	"cbc-aes-caam",
2437	.cra_blocksize = AES_BLOCK_SIZE,
2438	},
2439	.setkey = aead_setkey,
2440	.setauthsize = aead_setauthsize,
2441	.encrypt = aead_encrypt,
2442	.decrypt = aead_decrypt,
2443	.ivsize = AES_BLOCK_SIZE,
2444	.maxauthsize = SHA256_DIGEST_SIZE,
2445	},
2446	.aead.op = {
2447	.do_one_request = aead_do_one_req,
2448	},
2449	.caam = {
2450	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2451	.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2452	OP_ALG_AAI_HMAC_PRECOMP,
2453	},
2454	},
2455	{
2456	.aead.base = {
2457	.base = {
2458	.cra_name = "echainiv(authenc(hmac(sha256),"
2459	"cbc(aes)))",
2460	.cra_driver_name = "echainiv-authenc-"
2461	"hmac-sha256-cbc-aes-caam",
2462	.cra_blocksize = AES_BLOCK_SIZE,
2463	},
2464	.setkey = aead_setkey,
2465	.setauthsize = aead_setauthsize,
2466	.encrypt = aead_encrypt,
2467	.decrypt = aead_decrypt,
2468	.ivsize = AES_BLOCK_SIZE,
2469	.maxauthsize = SHA256_DIGEST_SIZE,
2470	},
2471	.aead.op = {
2472	.do_one_request = aead_do_one_req,
2473	},
2474	.caam = {
2475	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2476	.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2477	OP_ALG_AAI_HMAC_PRECOMP,
2478	.geniv = true,
2479	},
2480	},
2481	{
2482	.aead.base = {
2483	.base = {
2484	.cra_name = "authenc(hmac(sha384),cbc(aes))",
2485	.cra_driver_name = "authenc-hmac-sha384-"
2486	"cbc-aes-caam",
2487	.cra_blocksize = AES_BLOCK_SIZE,
2488	},
2489	.setkey = aead_setkey,
2490	.setauthsize = aead_setauthsize,
2491	.encrypt = aead_encrypt,
2492	.decrypt = aead_decrypt,
2493	.ivsize = AES_BLOCK_SIZE,
2494	.maxauthsize = SHA384_DIGEST_SIZE,
2495	},
2496	.aead.op = {
2497	.do_one_request = aead_do_one_req,
2498	},
2499	.caam = {
2500	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2501	.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2502	OP_ALG_AAI_HMAC_PRECOMP,
2503	},
2504	},
2505	{
2506	.aead.base = {
2507	.base = {
2508	.cra_name = "echainiv(authenc(hmac(sha384),"
2509	"cbc(aes)))",
2510	.cra_driver_name = "echainiv-authenc-"
2511	"hmac-sha384-cbc-aes-caam",
2512	.cra_blocksize = AES_BLOCK_SIZE,
2513	},
2514	.setkey = aead_setkey,
2515	.setauthsize = aead_setauthsize,
2516	.encrypt = aead_encrypt,
2517	.decrypt = aead_decrypt,
2518	.ivsize = AES_BLOCK_SIZE,
2519	.maxauthsize = SHA384_DIGEST_SIZE,
2520	},
2521	.aead.op = {
2522	.do_one_request = aead_do_one_req,
2523	},
2524	.caam = {
2525	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2526	.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2527	OP_ALG_AAI_HMAC_PRECOMP,
2528	.geniv = true,
2529	},
2530	},
2531	{
2532	.aead.base = {
2533	.base = {
2534	.cra_name = "authenc(hmac(sha512),cbc(aes))",
2535	.cra_driver_name = "authenc-hmac-sha512-"
2536	"cbc-aes-caam",
2537	.cra_blocksize = AES_BLOCK_SIZE,
2538	},
2539	.setkey = aead_setkey,
2540	.setauthsize = aead_setauthsize,
2541	.encrypt = aead_encrypt,
2542	.decrypt = aead_decrypt,
2543	.ivsize = AES_BLOCK_SIZE,
2544	.maxauthsize = SHA512_DIGEST_SIZE,
2545	},
2546	.aead.op = {
2547	.do_one_request = aead_do_one_req,
2548	},
2549	.caam = {
2550	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2551	.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2552	OP_ALG_AAI_HMAC_PRECOMP,
2553	},
2554	},
2555	{
2556	.aead.base = {
2557	.base = {
2558	.cra_name = "echainiv(authenc(hmac(sha512),"
2559	"cbc(aes)))",
2560	.cra_driver_name = "echainiv-authenc-"
2561	"hmac-sha512-cbc-aes-caam",
2562	.cra_blocksize = AES_BLOCK_SIZE,
2563	},
2564	.setkey = aead_setkey,
2565	.setauthsize = aead_setauthsize,
2566	.encrypt = aead_encrypt,
2567	.decrypt = aead_decrypt,
2568	.ivsize = AES_BLOCK_SIZE,
2569	.maxauthsize = SHA512_DIGEST_SIZE,
2570	},
2571	.aead.op = {
2572	.do_one_request = aead_do_one_req,
2573	},
2574	.caam = {
2575	.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2576	.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2577	OP_ALG_AAI_HMAC_PRECOMP,
2578	.geniv = true,
2579	},
2580	},
2581	{
2582	.aead.base = {
2583	.base = {
2584	.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2585	.cra_driver_name = "authenc-hmac-md5-"
2586	"cbc-des3_ede-caam",
2587	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2588	},
2589	.setkey = des3_aead_setkey,
2590	.setauthsize = aead_setauthsize,
2591	.encrypt = aead_encrypt,
2592	.decrypt = aead_decrypt,
2593	.ivsize = DES3_EDE_BLOCK_SIZE,
2594	.maxauthsize = MD5_DIGEST_SIZE,
2595	},
2596	.aead.op = {
2597	.do_one_request = aead_do_one_req,
2598	},
2599	.caam = {
2600	.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2601	.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2602	OP_ALG_AAI_HMAC_PRECOMP,
2603	}
2604	},
2605	{
2606	.aead.base = {
2607	.base = {
2608	.cra_name = "echainiv(authenc(hmac(md5),"
2609	"cbc(des3_ede)))",
2610	.cra_driver_name = "echainiv-authenc-hmac-md5-"
2611	"cbc-des3_ede-caam",
2612	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2613	},
2614	.setkey = des3_aead_setkey,
2615	.setauthsize = aead_setauthsize,
2616	.encrypt = aead_encrypt,
2617	.decrypt = aead_decrypt,
2618	.ivsize = DES3_EDE_BLOCK_SIZE,
2619	.maxauthsize = MD5_DIGEST_SIZE,
2620	},
2621	.aead.op = {
2622	.do_one_request = aead_do_one_req,
2623	},
2624	.caam = {
2625	.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2626	.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2627	OP_ALG_AAI_HMAC_PRECOMP,
2628	.geniv = true,
2629	}
2630	},
2631	{
2632	.aead.base = {
2633	.base = {
2634	.cra_name = "authenc(hmac(sha1),"
2635	"cbc(des3_ede))",
2636	.cra_driver_name = "authenc-hmac-sha1-"
2637	"cbc-des3_ede-caam",
2638	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2639	},
2640	.setkey = des3_aead_setkey,
2641	.setauthsize = aead_setauthsize,
2642	.encrypt = aead_encrypt,
2643	.decrypt = aead_decrypt,
2644	.ivsize = DES3_EDE_BLOCK_SIZE,
2645	.maxauthsize = SHA1_DIGEST_SIZE,
2646	},
2647	.aead.op = {
2648	.do_one_request = aead_do_one_req,
2649	},
2650	.caam = {
2651	.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2652	.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2653	OP_ALG_AAI_HMAC_PRECOMP,
2654	},
2655	},
2656	{
2657	.aead.base = {
2658	.base = {
2659	.cra_name = "echainiv(authenc(hmac(sha1),"
2660	"cbc(des3_ede)))",
2661	.cra_driver_name = "echainiv-authenc-"
2662	"hmac-sha1-"
2663	"cbc-des3_ede-caam",
2664	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2665	},
2666	.setkey = des3_aead_setkey,
2667	.setauthsize = aead_setauthsize,
2668	.encrypt = aead_encrypt,
2669	.decrypt = aead_decrypt,
2670	.ivsize = DES3_EDE_BLOCK_SIZE,
2671	.maxauthsize = SHA1_DIGEST_SIZE,
2672	},
2673	.aead.op = {
2674	.do_one_request = aead_do_one_req,
2675	},
2676	.caam = {
2677	.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2678	.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2679	OP_ALG_AAI_HMAC_PRECOMP,
2680	.geniv = true,
2681	},
2682	},
2683	{
2684	.aead.base = {
2685	.base = {
2686	.cra_name = "authenc(hmac(sha224),"
2687	"cbc(des3_ede))",
2688	.cra_driver_name = "authenc-hmac-sha224-"
2689	"cbc-des3_ede-caam",
2690	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2691	},
2692	.setkey = des3_aead_setkey,
2693	.setauthsize = aead_setauthsize,
2694	.encrypt = aead_encrypt,
2695	.decrypt = aead_decrypt,
2696	.ivsize = DES3_EDE_BLOCK_SIZE,
2697	.maxauthsize = SHA224_DIGEST_SIZE,
2698	},
2699	.aead.op = {
2700	.do_one_request = aead_do_one_req,
2701	},
2702	.caam = {
2703	.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2704	.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2705	OP_ALG_AAI_HMAC_PRECOMP,
2706	},
2707	},
2708	{
2709	.aead.base = {
2710	.base = {
2711	.cra_name = "echainiv(authenc(hmac(sha224),"
2712	"cbc(des3_ede)))",
2713	.cra_driver_name = "echainiv-authenc-"
2714	"hmac-sha224-"
2715	"cbc-des3_ede-caam",
2716	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2717	},
2718	.setkey = des3_aead_setkey,
2719	.setauthsize = aead_setauthsize,
2720	.encrypt = aead_encrypt,
2721	.decrypt = aead_decrypt,
2722	.ivsize = DES3_EDE_BLOCK_SIZE,
2723	.maxauthsize = SHA224_DIGEST_SIZE,
2724	},
2725	.aead.op = {
2726	.do_one_request = aead_do_one_req,
2727	},
2728	.caam = {
2729	.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2730	.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2731	OP_ALG_AAI_HMAC_PRECOMP,
2732	.geniv = true,
2733	},
2734	},
2735	{
2736	.aead.base = {
2737	.base = {
2738	.cra_name = "authenc(hmac(sha256),"
2739	"cbc(des3_ede))",
2740	.cra_driver_name = "authenc-hmac-sha256-"
2741	"cbc-des3_ede-caam",
2742	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2743	},
2744	.setkey = des3_aead_setkey,
2745	.setauthsize = aead_setauthsize,
2746	.encrypt = aead_encrypt,
2747	.decrypt = aead_decrypt,
2748	.ivsize = DES3_EDE_BLOCK_SIZE,
2749	.maxauthsize = SHA256_DIGEST_SIZE,
2750	},
2751	.aead.op = {
2752	.do_one_request = aead_do_one_req,
2753	},
2754	.caam = {
2755	.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2756	.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2757	OP_ALG_AAI_HMAC_PRECOMP,
2758	},
2759	},
2760	{
2761	.aead.base = {
2762	.base = {
2763	.cra_name = "echainiv(authenc(hmac(sha256),"
2764	"cbc(des3_ede)))",
2765	.cra_driver_name = "echainiv-authenc-"
2766	"hmac-sha256-"
2767	"cbc-des3_ede-caam",
2768	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2769	},
2770	.setkey = des3_aead_setkey,
2771	.setauthsize = aead_setauthsize,
2772	.encrypt = aead_encrypt,
2773	.decrypt = aead_decrypt,
2774	.ivsize = DES3_EDE_BLOCK_SIZE,
2775	.maxauthsize = SHA256_DIGEST_SIZE,
2776	},
2777	.aead.op = {
2778	.do_one_request = aead_do_one_req,
2779	},
2780	.caam = {
2781	.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2782	.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2783	OP_ALG_AAI_HMAC_PRECOMP,
2784	.geniv = true,
2785	},
2786	},
2787	{
2788	.aead.base = {
2789	.base = {
2790	.cra_name = "authenc(hmac(sha384),"
2791	"cbc(des3_ede))",
2792	.cra_driver_name = "authenc-hmac-sha384-"
2793	"cbc-des3_ede-caam",
2794	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2795	},
2796	.setkey = des3_aead_setkey,
2797	.setauthsize = aead_setauthsize,
2798	.encrypt = aead_encrypt,
2799	.decrypt = aead_decrypt,
2800	.ivsize = DES3_EDE_BLOCK_SIZE,
2801	.maxauthsize = SHA384_DIGEST_SIZE,
2802	},
2803	.aead.op = {
2804	.do_one_request = aead_do_one_req,
2805	},
2806	.caam = {
2807	.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2808	.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2809	OP_ALG_AAI_HMAC_PRECOMP,
2810	},
2811	},
2812	{
2813	.aead.base = {
2814	.base = {
2815	.cra_name = "echainiv(authenc(hmac(sha384),"
2816	"cbc(des3_ede)))",
2817	.cra_driver_name = "echainiv-authenc-"
2818	"hmac-sha384-"
2819	"cbc-des3_ede-caam",
2820	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2821	},
2822	.setkey = des3_aead_setkey,
2823	.setauthsize = aead_setauthsize,
2824	.encrypt = aead_encrypt,
2825	.decrypt = aead_decrypt,
2826	.ivsize = DES3_EDE_BLOCK_SIZE,
2827	.maxauthsize = SHA384_DIGEST_SIZE,
2828	},
2829	.aead.op = {
2830	.do_one_request = aead_do_one_req,
2831	},
2832	.caam = {
2833	.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2834	.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2835	OP_ALG_AAI_HMAC_PRECOMP,
2836	.geniv = true,
2837	},
2838	},
2839	{
2840	.aead.base = {
2841	.base = {
2842	.cra_name = "authenc(hmac(sha512),"
2843	"cbc(des3_ede))",
2844	.cra_driver_name = "authenc-hmac-sha512-"
2845	"cbc-des3_ede-caam",
2846	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2847	},
2848	.setkey = des3_aead_setkey,
2849	.setauthsize = aead_setauthsize,
2850	.encrypt = aead_encrypt,
2851	.decrypt = aead_decrypt,
2852	.ivsize = DES3_EDE_BLOCK_SIZE,
2853	.maxauthsize = SHA512_DIGEST_SIZE,
2854	},
2855	.aead.op = {
2856	.do_one_request = aead_do_one_req,
2857	},
2858	.caam = {
2859	.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2860	.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2861	OP_ALG_AAI_HMAC_PRECOMP,
2862	},
2863	},
2864	{
2865	.aead.base = {
2866	.base = {
2867	.cra_name = "echainiv(authenc(hmac(sha512),"
2868	"cbc(des3_ede)))",
2869	.cra_driver_name = "echainiv-authenc-"
2870	"hmac-sha512-"
2871	"cbc-des3_ede-caam",
2872	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
2873	},
2874	.setkey = des3_aead_setkey,
2875	.setauthsize = aead_setauthsize,
2876	.encrypt = aead_encrypt,
2877	.decrypt = aead_decrypt,
2878	.ivsize = DES3_EDE_BLOCK_SIZE,
2879	.maxauthsize = SHA512_DIGEST_SIZE,
2880	},
2881	.aead.op = {
2882	.do_one_request = aead_do_one_req,
2883	},
2884	.caam = {
2885	.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2886	.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2887	OP_ALG_AAI_HMAC_PRECOMP,
2888	.geniv = true,
2889	},
2890	},
2891	{
2892	.aead.base = {
2893	.base = {
2894	.cra_name = "authenc(hmac(md5),cbc(des))",
2895	.cra_driver_name = "authenc-hmac-md5-"
2896	"cbc-des-caam",
2897	.cra_blocksize = DES_BLOCK_SIZE,
2898	},
2899	.setkey = aead_setkey,
2900	.setauthsize = aead_setauthsize,
2901	.encrypt = aead_encrypt,
2902	.decrypt = aead_decrypt,
2903	.ivsize = DES_BLOCK_SIZE,
2904	.maxauthsize = MD5_DIGEST_SIZE,
2905	},
2906	.aead.op = {
2907	.do_one_request = aead_do_one_req,
2908	},
2909	.caam = {
2910	.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2911	.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2912	OP_ALG_AAI_HMAC_PRECOMP,
2913	},
2914	},
2915	{
2916	.aead.base = {
2917	.base = {
2918	.cra_name = "echainiv(authenc(hmac(md5),"
2919	"cbc(des)))",
2920	.cra_driver_name = "echainiv-authenc-hmac-md5-"
2921	"cbc-des-caam",
2922	.cra_blocksize = DES_BLOCK_SIZE,
2923	},
2924	.setkey = aead_setkey,
2925	.setauthsize = aead_setauthsize,
2926	.encrypt = aead_encrypt,
2927	.decrypt = aead_decrypt,
2928	.ivsize = DES_BLOCK_SIZE,
2929	.maxauthsize = MD5_DIGEST_SIZE,
2930	},
2931	.aead.op = {
2932	.do_one_request = aead_do_one_req,
2933	},
2934	.caam = {
2935	.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2936	.class2_alg_type = OP_ALG_ALGSEL_MD5 |
2937	OP_ALG_AAI_HMAC_PRECOMP,
2938	.geniv = true,
2939	},
2940	},
2941	{
2942	.aead.base = {
2943	.base = {
2944	.cra_name = "authenc(hmac(sha1),cbc(des))",
2945	.cra_driver_name = "authenc-hmac-sha1-"
2946	"cbc-des-caam",
2947	.cra_blocksize = DES_BLOCK_SIZE,
2948	},
2949	.setkey = aead_setkey,
2950	.setauthsize = aead_setauthsize,
2951	.encrypt = aead_encrypt,
2952	.decrypt = aead_decrypt,
2953	.ivsize = DES_BLOCK_SIZE,
2954	.maxauthsize = SHA1_DIGEST_SIZE,
2955	},
2956	.aead.op = {
2957	.do_one_request = aead_do_one_req,
2958	},
2959	.caam = {
2960	.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2961	.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2962	OP_ALG_AAI_HMAC_PRECOMP,
2963	},
2964	},
2965	{
2966	.aead.base = {
2967	.base = {
2968	.cra_name = "echainiv(authenc(hmac(sha1),"
2969	"cbc(des)))",
2970	.cra_driver_name = "echainiv-authenc-"
2971	"hmac-sha1-cbc-des-caam",
2972	.cra_blocksize = DES_BLOCK_SIZE,
2973	},
2974	.setkey = aead_setkey,
2975	.setauthsize = aead_setauthsize,
2976	.encrypt = aead_encrypt,
2977	.decrypt = aead_decrypt,
2978	.ivsize = DES_BLOCK_SIZE,
2979	.maxauthsize = SHA1_DIGEST_SIZE,
2980	},
2981	.aead.op = {
2982	.do_one_request = aead_do_one_req,
2983	},
2984	.caam = {
2985	.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2986	.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2987	OP_ALG_AAI_HMAC_PRECOMP,
2988	.geniv = true,
2989	},
2990	},
2991	{
2992	.aead.base = {
2993	.base = {
2994	.cra_name = "authenc(hmac(sha224),cbc(des))",
2995	.cra_driver_name = "authenc-hmac-sha224-"
2996	"cbc-des-caam",
2997	.cra_blocksize = DES_BLOCK_SIZE,
2998	},
2999	.setkey = aead_setkey,
3000	.setauthsize = aead_setauthsize,
3001	.encrypt = aead_encrypt,
3002	.decrypt = aead_decrypt,
3003	.ivsize = DES_BLOCK_SIZE,
3004	.maxauthsize = SHA224_DIGEST_SIZE,
3005	},
3006	.aead.op = {
3007	.do_one_request = aead_do_one_req,
3008	},
3009	.caam = {
3010	.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3011	.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3012	OP_ALG_AAI_HMAC_PRECOMP,
3013	},
3014	},
3015	{
3016	.aead.base = {
3017	.base = {
3018	.cra_name = "echainiv(authenc(hmac(sha224),"
3019	"cbc(des)))",
3020	.cra_driver_name = "echainiv-authenc-"
3021	"hmac-sha224-cbc-des-caam",
3022	.cra_blocksize = DES_BLOCK_SIZE,
3023	},
3024	.setkey = aead_setkey,
3025	.setauthsize = aead_setauthsize,
3026	.encrypt = aead_encrypt,
3027	.decrypt = aead_decrypt,
3028	.ivsize = DES_BLOCK_SIZE,
3029	.maxauthsize = SHA224_DIGEST_SIZE,
3030	},
3031	.aead.op = {
3032	.do_one_request = aead_do_one_req,
3033	},
3034	.caam = {
3035	.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3036	.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3037	OP_ALG_AAI_HMAC_PRECOMP,
3038	.geniv = true,
3039	},
3040	},
3041	{
3042	.aead.base = {
3043	.base = {
3044	.cra_name = "authenc(hmac(sha256),cbc(des))",
3045	.cra_driver_name = "authenc-hmac-sha256-"
3046	"cbc-des-caam",
3047	.cra_blocksize = DES_BLOCK_SIZE,
3048	},
3049	.setkey = aead_setkey,
3050	.setauthsize = aead_setauthsize,
3051	.encrypt = aead_encrypt,
3052	.decrypt = aead_decrypt,
3053	.ivsize = DES_BLOCK_SIZE,
3054	.maxauthsize = SHA256_DIGEST_SIZE,
3055	},
3056	.aead.op = {
3057	.do_one_request = aead_do_one_req,
3058	},
3059	.caam = {
3060	.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3061	.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3062	OP_ALG_AAI_HMAC_PRECOMP,
3063	},
3064	},
3065	{
3066	.aead.base = {
3067	.base = {
3068	.cra_name = "echainiv(authenc(hmac(sha256),"
3069	"cbc(des)))",
3070	.cra_driver_name = "echainiv-authenc-"
3071	"hmac-sha256-cbc-des-caam",
3072	.cra_blocksize = DES_BLOCK_SIZE,
3073	},
3074	.setkey = aead_setkey,
3075	.setauthsize = aead_setauthsize,
3076	.encrypt = aead_encrypt,
3077	.decrypt = aead_decrypt,
3078	.ivsize = DES_BLOCK_SIZE,
3079	.maxauthsize = SHA256_DIGEST_SIZE,
3080	},
3081	.aead.op = {
3082	.do_one_request = aead_do_one_req,
3083	},
3084	.caam = {
3085	.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3086	.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3087	OP_ALG_AAI_HMAC_PRECOMP,
3088	.geniv = true,
3089	},
3090	},
3091	{
3092	.aead.base = {
3093	.base = {
3094	.cra_name = "authenc(hmac(sha384),cbc(des))",
3095	.cra_driver_name = "authenc-hmac-sha384-"
3096	"cbc-des-caam",
3097	.cra_blocksize = DES_BLOCK_SIZE,
3098	},
3099	.setkey = aead_setkey,
3100	.setauthsize = aead_setauthsize,
3101	.encrypt = aead_encrypt,
3102	.decrypt = aead_decrypt,
3103	.ivsize = DES_BLOCK_SIZE,
3104	.maxauthsize = SHA384_DIGEST_SIZE,
3105	},
3106	.aead.op = {
3107	.do_one_request = aead_do_one_req,
3108	},
3109	.caam = {
3110	.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3111	.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3112	OP_ALG_AAI_HMAC_PRECOMP,
3113	},
3114	},
3115	{
3116	.aead.base = {
3117	.base = {
3118	.cra_name = "echainiv(authenc(hmac(sha384),"
3119	"cbc(des)))",
3120	.cra_driver_name = "echainiv-authenc-"
3121	"hmac-sha384-cbc-des-caam",
3122	.cra_blocksize = DES_BLOCK_SIZE,
3123	},
3124	.setkey = aead_setkey,
3125	.setauthsize = aead_setauthsize,
3126	.encrypt = aead_encrypt,
3127	.decrypt = aead_decrypt,
3128	.ivsize = DES_BLOCK_SIZE,
3129	.maxauthsize = SHA384_DIGEST_SIZE,
3130	},
3131	.aead.op = {
3132	.do_one_request = aead_do_one_req,
3133	},
3134	.caam = {
3135	.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3136	.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3137	OP_ALG_AAI_HMAC_PRECOMP,
3138	.geniv = true,
3139	},
3140	},
3141	{
3142	.aead.base = {
3143	.base = {
3144	.cra_name = "authenc(hmac(sha512),cbc(des))",
3145	.cra_driver_name = "authenc-hmac-sha512-"
3146	"cbc-des-caam",
3147	.cra_blocksize = DES_BLOCK_SIZE,
3148	},
3149	.setkey = aead_setkey,
3150	.setauthsize = aead_setauthsize,
3151	.encrypt = aead_encrypt,
3152	.decrypt = aead_decrypt,
3153	.ivsize = DES_BLOCK_SIZE,
3154	.maxauthsize = SHA512_DIGEST_SIZE,
3155	},
3156	.aead.op = {
3157	.do_one_request = aead_do_one_req,
3158	},
3159	.caam = {
3160	.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3161	.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3162	OP_ALG_AAI_HMAC_PRECOMP,
3163	},
3164	},
3165	{
3166	.aead.base = {
3167	.base = {
3168	.cra_name = "echainiv(authenc(hmac(sha512),"
3169	"cbc(des)))",
3170	.cra_driver_name = "echainiv-authenc-"
3171	"hmac-sha512-cbc-des-caam",
3172	.cra_blocksize = DES_BLOCK_SIZE,
3173	},
3174	.setkey = aead_setkey,
3175	.setauthsize = aead_setauthsize,
3176	.encrypt = aead_encrypt,
3177	.decrypt = aead_decrypt,
3178	.ivsize = DES_BLOCK_SIZE,
3179	.maxauthsize = SHA512_DIGEST_SIZE,
3180	},
3181	.aead.op = {
3182	.do_one_request = aead_do_one_req,
3183	},
3184	.caam = {
3185	.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3186	.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3187	OP_ALG_AAI_HMAC_PRECOMP,
3188	.geniv = true,
3189	},
3190	},
3191	{
3192	.aead.base = {
3193	.base = {
3194	.cra_name = "authenc(hmac(md5),"
3195	"rfc3686(ctr(aes)))",
3196	.cra_driver_name = "authenc-hmac-md5-"
3197	"rfc3686-ctr-aes-caam",
3198	.cra_blocksize = 1,
3199	},
3200	.setkey = aead_setkey,
3201	.setauthsize = aead_setauthsize,
3202	.encrypt = aead_encrypt,
3203	.decrypt = aead_decrypt,
3204	.ivsize = CTR_RFC3686_IV_SIZE,
3205	.maxauthsize = MD5_DIGEST_SIZE,
3206	},
3207	.aead.op = {
3208	.do_one_request = aead_do_one_req,
3209	},
3210	.caam = {
3211	.class1_alg_type = OP_ALG_ALGSEL_AES |
3212	OP_ALG_AAI_CTR_MOD128,
3213	.class2_alg_type = OP_ALG_ALGSEL_MD5 |
3214	OP_ALG_AAI_HMAC_PRECOMP,
3215	.rfc3686 = true,
3216	},
3217	},
3218	{
3219	.aead.base = {
3220	.base = {
3221	.cra_name = "seqiv(authenc("
3222	"hmac(md5),rfc3686(ctr(aes))))",
3223	.cra_driver_name = "seqiv-authenc-hmac-md5-"
3224	"rfc3686-ctr-aes-caam",
3225	.cra_blocksize = 1,
3226	},
3227	.setkey = aead_setkey,
3228	.setauthsize = aead_setauthsize,
3229	.encrypt = aead_encrypt,
3230	.decrypt = aead_decrypt,
3231	.ivsize = CTR_RFC3686_IV_SIZE,
3232	.maxauthsize = MD5_DIGEST_SIZE,
3233	},
3234	.aead.op = {
3235	.do_one_request = aead_do_one_req,
3236	},
3237	.caam = {
3238	.class1_alg_type = OP_ALG_ALGSEL_AES |
3239	OP_ALG_AAI_CTR_MOD128,
3240	.class2_alg_type = OP_ALG_ALGSEL_MD5 |
3241	OP_ALG_AAI_HMAC_PRECOMP,
3242	.rfc3686 = true,
3243	.geniv = true,
3244	},
3245	},
3246	{
3247	.aead.base = {
3248	.base = {
3249	.cra_name = "authenc(hmac(sha1),"
3250	"rfc3686(ctr(aes)))",
3251	.cra_driver_name = "authenc-hmac-sha1-"
3252	"rfc3686-ctr-aes-caam",
3253	.cra_blocksize = 1,
3254	},
3255	.setkey = aead_setkey,
3256	.setauthsize = aead_setauthsize,
3257	.encrypt = aead_encrypt,
3258	.decrypt = aead_decrypt,
3259	.ivsize = CTR_RFC3686_IV_SIZE,
3260	.maxauthsize = SHA1_DIGEST_SIZE,
3261	},
3262	.aead.op = {
3263	.do_one_request = aead_do_one_req,
3264	},
3265	.caam = {
3266	.class1_alg_type = OP_ALG_ALGSEL_AES |
3267	OP_ALG_AAI_CTR_MOD128,
3268	.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3269	OP_ALG_AAI_HMAC_PRECOMP,
3270	.rfc3686 = true,
3271	},
3272	},
3273	{
3274	.aead.base = {
3275	.base = {
3276	.cra_name = "seqiv(authenc("
3277	"hmac(sha1),rfc3686(ctr(aes))))",
3278	.cra_driver_name = "seqiv-authenc-hmac-sha1-"
3279	"rfc3686-ctr-aes-caam",
3280	.cra_blocksize = 1,
3281	},
3282	.setkey = aead_setkey,
3283	.setauthsize = aead_setauthsize,
3284	.encrypt = aead_encrypt,
3285	.decrypt = aead_decrypt,
3286	.ivsize = CTR_RFC3686_IV_SIZE,
3287	.maxauthsize = SHA1_DIGEST_SIZE,
3288	},
3289	.aead.op = {
3290	.do_one_request = aead_do_one_req,
3291	},
3292	.caam = {
3293	.class1_alg_type = OP_ALG_ALGSEL_AES |
3294	OP_ALG_AAI_CTR_MOD128,
3295	.class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3296	OP_ALG_AAI_HMAC_PRECOMP,
3297	.rfc3686 = true,
3298	.geniv = true,
3299	},
3300	},
3301	{
3302	.aead.base = {
3303	.base = {
3304	.cra_name = "authenc(hmac(sha224),"
3305	"rfc3686(ctr(aes)))",
3306	.cra_driver_name = "authenc-hmac-sha224-"
3307	"rfc3686-ctr-aes-caam",
3308	.cra_blocksize = 1,
3309	},
3310	.setkey = aead_setkey,
3311	.setauthsize = aead_setauthsize,
3312	.encrypt = aead_encrypt,
3313	.decrypt = aead_decrypt,
3314	.ivsize = CTR_RFC3686_IV_SIZE,
3315	.maxauthsize = SHA224_DIGEST_SIZE,
3316	},
3317	.aead.op = {
3318	.do_one_request = aead_do_one_req,
3319	},
3320	.caam = {
3321	.class1_alg_type = OP_ALG_ALGSEL_AES |
3322	OP_ALG_AAI_CTR_MOD128,
3323	.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3324	OP_ALG_AAI_HMAC_PRECOMP,
3325	.rfc3686 = true,
3326	},
3327	},
3328	{
3329	.aead.base = {
3330	.base = {
3331	.cra_name = "seqiv(authenc("
3332	"hmac(sha224),rfc3686(ctr(aes))))",
3333	.cra_driver_name = "seqiv-authenc-hmac-sha224-"
3334	"rfc3686-ctr-aes-caam",
3335	.cra_blocksize = 1,
3336	},
3337	.setkey = aead_setkey,
3338	.setauthsize = aead_setauthsize,
3339	.encrypt = aead_encrypt,
3340	.decrypt = aead_decrypt,
3341	.ivsize = CTR_RFC3686_IV_SIZE,
3342	.maxauthsize = SHA224_DIGEST_SIZE,
3343	},
3344	.aead.op = {
3345	.do_one_request = aead_do_one_req,
3346	},
3347	.caam = {
3348	.class1_alg_type = OP_ALG_ALGSEL_AES |
3349	OP_ALG_AAI_CTR_MOD128,
3350	.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3351	OP_ALG_AAI_HMAC_PRECOMP,
3352	.rfc3686 = true,
3353	.geniv = true,
3354	},
3355	},
3356	{
3357	.aead.base = {
3358	.base = {
3359	.cra_name = "authenc(hmac(sha256),"
3360	"rfc3686(ctr(aes)))",
3361	.cra_driver_name = "authenc-hmac-sha256-"
3362	"rfc3686-ctr-aes-caam",
3363	.cra_blocksize = 1,
3364	},
3365	.setkey = aead_setkey,
3366	.setauthsize = aead_setauthsize,
3367	.encrypt = aead_encrypt,
3368	.decrypt = aead_decrypt,
3369	.ivsize = CTR_RFC3686_IV_SIZE,
3370	.maxauthsize = SHA256_DIGEST_SIZE,
3371	},
3372	.aead.op = {
3373	.do_one_request = aead_do_one_req,
3374	},
3375	.caam = {
3376	.class1_alg_type = OP_ALG_ALGSEL_AES |
3377	OP_ALG_AAI_CTR_MOD128,
3378	.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3379	OP_ALG_AAI_HMAC_PRECOMP,
3380	.rfc3686 = true,
3381	},
3382	},
3383	{
3384	.aead.base = {
3385	.base = {
3386	.cra_name = "seqiv(authenc(hmac(sha256),"
3387	"rfc3686(ctr(aes))))",
3388	.cra_driver_name = "seqiv-authenc-hmac-sha256-"
3389	"rfc3686-ctr-aes-caam",
3390	.cra_blocksize = 1,
3391	},
3392	.setkey = aead_setkey,
3393	.setauthsize = aead_setauthsize,
3394	.encrypt = aead_encrypt,
3395	.decrypt = aead_decrypt,
3396	.ivsize = CTR_RFC3686_IV_SIZE,
3397	.maxauthsize = SHA256_DIGEST_SIZE,
3398	},
3399	.aead.op = {
3400	.do_one_request = aead_do_one_req,
3401	},
3402	.caam = {
3403	.class1_alg_type = OP_ALG_ALGSEL_AES |
3404	OP_ALG_AAI_CTR_MOD128,
3405	.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3406	OP_ALG_AAI_HMAC_PRECOMP,
3407	.rfc3686 = true,
3408	.geniv = true,
3409	},
3410	},
3411	{
3412	.aead.base = {
3413	.base = {
3414	.cra_name = "authenc(hmac(sha384),"
3415	"rfc3686(ctr(aes)))",
3416	.cra_driver_name = "authenc-hmac-sha384-"
3417	"rfc3686-ctr-aes-caam",
3418	.cra_blocksize = 1,
3419	},
3420	.setkey = aead_setkey,
3421	.setauthsize = aead_setauthsize,
3422	.encrypt = aead_encrypt,
3423	.decrypt = aead_decrypt,
3424	.ivsize = CTR_RFC3686_IV_SIZE,
3425	.maxauthsize = SHA384_DIGEST_SIZE,
3426	},
3427	.aead.op = {
3428	.do_one_request = aead_do_one_req,
3429	},
3430	.caam = {
3431	.class1_alg_type = OP_ALG_ALGSEL_AES |
3432	OP_ALG_AAI_CTR_MOD128,
3433	.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3434	OP_ALG_AAI_HMAC_PRECOMP,
3435	.rfc3686 = true,
3436	},
3437	},
3438	{
3439	.aead.base = {
3440	.base = {
3441	.cra_name = "seqiv(authenc(hmac(sha384),"
3442	"rfc3686(ctr(aes))))",
3443	.cra_driver_name = "seqiv-authenc-hmac-sha384-"
3444	"rfc3686-ctr-aes-caam",
3445	.cra_blocksize = 1,
3446	},
3447	.setkey = aead_setkey,
3448	.setauthsize = aead_setauthsize,
3449	.encrypt = aead_encrypt,
3450	.decrypt = aead_decrypt,
3451	.ivsize = CTR_RFC3686_IV_SIZE,
3452	.maxauthsize = SHA384_DIGEST_SIZE,
3453	},
3454	.aead.op = {
3455	.do_one_request = aead_do_one_req,
3456	},
3457	.caam = {
3458	.class1_alg_type = OP_ALG_ALGSEL_AES |
3459	OP_ALG_AAI_CTR_MOD128,
3460	.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3461	OP_ALG_AAI_HMAC_PRECOMP,
3462	.rfc3686 = true,
3463	.geniv = true,
3464	},
3465	},
3466	{
3467	.aead.base = {
3468	.base = {
3469	.cra_name = "authenc(hmac(sha512),"
3470	"rfc3686(ctr(aes)))",
3471	.cra_driver_name = "authenc-hmac-sha512-"
3472	"rfc3686-ctr-aes-caam",
3473	.cra_blocksize = 1,
3474	},
3475	.setkey = aead_setkey,
3476	.setauthsize = aead_setauthsize,
3477	.encrypt = aead_encrypt,
3478	.decrypt = aead_decrypt,
3479	.ivsize = CTR_RFC3686_IV_SIZE,
3480	.maxauthsize = SHA512_DIGEST_SIZE,
3481	},
3482	.aead.op = {
3483	.do_one_request = aead_do_one_req,
3484	},
3485	.caam = {
3486	.class1_alg_type = OP_ALG_ALGSEL_AES |
3487	OP_ALG_AAI_CTR_MOD128,
3488	.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3489	OP_ALG_AAI_HMAC_PRECOMP,
3490	.rfc3686 = true,
3491	},
3492	},
3493	{
3494	.aead.base = {
3495	.base = {
3496	.cra_name = "seqiv(authenc(hmac(sha512),"
3497	"rfc3686(ctr(aes))))",
3498	.cra_driver_name = "seqiv-authenc-hmac-sha512-"
3499	"rfc3686-ctr-aes-caam",
3500	.cra_blocksize = 1,
3501	},
3502	.setkey = aead_setkey,
3503	.setauthsize = aead_setauthsize,
3504	.encrypt = aead_encrypt,
3505	.decrypt = aead_decrypt,
3506	.ivsize = CTR_RFC3686_IV_SIZE,
3507	.maxauthsize = SHA512_DIGEST_SIZE,
3508	},
3509	.aead.op = {
3510	.do_one_request = aead_do_one_req,
3511	},
3512	.caam = {
3513	.class1_alg_type = OP_ALG_ALGSEL_AES |
3514	OP_ALG_AAI_CTR_MOD128,
3515	.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3516	OP_ALG_AAI_HMAC_PRECOMP,
3517	.rfc3686 = true,
3518	.geniv = true,
3519	},
3520	},
3521	{
3522	.aead.base = {
3523	.base = {
3524	.cra_name = "rfc7539(chacha20,poly1305)",
3525	.cra_driver_name = "rfc7539-chacha20-poly1305-"
3526	"caam",
3527	.cra_blocksize = 1,
3528	},
3529	.setkey = chachapoly_setkey,
3530	.setauthsize = chachapoly_setauthsize,
3531	.encrypt = chachapoly_encrypt,
3532	.decrypt = chachapoly_decrypt,
3533	.ivsize = CHACHAPOLY_IV_SIZE,
3534	.maxauthsize = POLY1305_DIGEST_SIZE,
3535	},
3536	.aead.op = {
3537	.do_one_request = aead_do_one_req,
3538	},
3539	.caam = {
3540	.class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
3541	OP_ALG_AAI_AEAD,
3542	.class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
3543	OP_ALG_AAI_AEAD,
3544	.nodkp = true,
3545	},
3546	},
3547	{
3548	.aead.base = {
3549	.base = {
3550	.cra_name = "rfc7539esp(chacha20,poly1305)",
3551	.cra_driver_name = "rfc7539esp-chacha20-"
3552	"poly1305-caam",
3553	.cra_blocksize = 1,
3554	},
3555	.setkey = chachapoly_setkey,
3556	.setauthsize = chachapoly_setauthsize,
3557	.encrypt = chachapoly_encrypt,
3558	.decrypt = chachapoly_decrypt,
3559	.ivsize = 8,
3560	.maxauthsize = POLY1305_DIGEST_SIZE,
3561	},
3562	.aead.op = {
3563	.do_one_request = aead_do_one_req,
3564	},
3565	.caam = {
3566	.class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
3567	OP_ALG_AAI_AEAD,
3568	.class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
3569	OP_ALG_AAI_AEAD,
3570	.nodkp = true,
3571	},
3572	},
3573	};
3574
3575	static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
3576	bool uses_dkp)
3577	{
3578	dma_addr_t dma_addr;
3579	struct caam_drv_private *priv;
3580	const size_t sh_desc_enc_offset = offsetof(struct caam_ctx,
3581	sh_desc_enc);
3582
3583	ctx->jrdev = caam_jr_alloc();
3584	if (IS_ERR(ptr: ctx->jrdev)) {
3585	pr_err("Job Ring Device allocation for transform failed\n");
3586	return PTR_ERR(ptr: ctx->jrdev);
3587	}
3588
3589	priv = dev_get_drvdata(dev: ctx->jrdev->parent);
3590	if (priv->era >= 6 && uses_dkp)
3591	ctx->dir = DMA_BIDIRECTIONAL;
3592	else
3593	ctx->dir = DMA_TO_DEVICE;
3594
3595	dma_addr = dma_map_single_attrs(dev: ctx->jrdev, ptr: ctx->sh_desc_enc,
3596	offsetof(struct caam_ctx,
3597	sh_desc_enc_dma) -
3598	sh_desc_enc_offset,
3599	dir: ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
3600	if (dma_mapping_error(dev: ctx->jrdev, dma_addr)) {
3601	dev_err(ctx->jrdev, "unable to map key, shared descriptors\n");
3602	caam_jr_free(rdev: ctx->jrdev);
3603	return -ENOMEM;
3604	}
3605
3606	ctx->sh_desc_enc_dma = dma_addr;
3607	ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx,
3608	sh_desc_dec) -
3609	sh_desc_enc_offset;
3610	ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key) -
3611	sh_desc_enc_offset;
3612
3613	/* copy descriptor header template value */
3614	ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
3615	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
3616
3617	return 0;
3618	}
3619
3620	static int caam_cra_init(struct crypto_skcipher *tfm)
3621	{
3622	struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
3623	struct caam_skcipher_alg *caam_alg =
3624	container_of(alg, typeof(*caam_alg), skcipher.base);
3625	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
3626	u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
3627	int ret = 0;
3628
3629	if (alg_aai == OP_ALG_AAI_XTS) {
3630	const char *tfm_name = crypto_tfm_alg_name(tfm: &tfm->base);
3631	struct crypto_skcipher *fallback;
3632
3633	fallback = crypto_alloc_skcipher(alg_name: tfm_name, type: 0,
3634	CRYPTO_ALG_NEED_FALLBACK);
3635	if (IS_ERR(ptr: fallback)) {
3636	pr_err("Failed to allocate %s fallback: %ld\n",
3637	tfm_name, PTR_ERR(fallback));
3638	return PTR_ERR(ptr: fallback);
3639	}
3640
3641	ctx->fallback = fallback;
3642	crypto_skcipher_set_reqsize(skcipher: tfm, reqsize: sizeof(struct caam_skcipher_req_ctx) +
3643	crypto_skcipher_reqsize(tfm: fallback));
3644	} else {
3645	crypto_skcipher_set_reqsize(skcipher: tfm, reqsize: sizeof(struct caam_skcipher_req_ctx));
3646	}
3647
3648	ret = caam_init_common(ctx, caam: &caam_alg->caam, uses_dkp: false);
3649	if (ret && ctx->fallback)
3650	crypto_free_skcipher(tfm: ctx->fallback);
3651
3652	return ret;
3653	}
3654
3655	static int caam_aead_init(struct crypto_aead *tfm)
3656	{
3657	struct aead_alg *alg = crypto_aead_alg(tfm);
3658	struct caam_aead_alg *caam_alg =
3659	container_of(alg, struct caam_aead_alg, aead.base);
3660	struct caam_ctx *ctx = crypto_aead_ctx_dma(tfm);
3661
3662	crypto_aead_set_reqsize(aead: tfm, reqsize: sizeof(struct caam_aead_req_ctx));
3663
3664	return caam_init_common(ctx, caam: &caam_alg->caam, uses_dkp: !caam_alg->caam.nodkp);
3665	}
3666
3667	static void caam_exit_common(struct caam_ctx *ctx)
3668	{
3669	dma_unmap_single_attrs(dev: ctx->jrdev, addr: ctx->sh_desc_enc_dma,
3670	offsetof(struct caam_ctx, sh_desc_enc_dma) -
3671	offsetof(struct caam_ctx, sh_desc_enc),
3672	dir: ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
3673	caam_jr_free(rdev: ctx->jrdev);
3674	}
3675
3676	static void caam_cra_exit(struct crypto_skcipher *tfm)
3677	{
3678	struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm);
3679
3680	if (ctx->fallback)
3681	crypto_free_skcipher(tfm: ctx->fallback);
3682	caam_exit_common(ctx);
3683	}
3684
3685	static void caam_aead_exit(struct crypto_aead *tfm)
3686	{
3687	caam_exit_common(ctx: crypto_aead_ctx_dma(tfm));
3688	}
3689
3690	void caam_algapi_exit(void)
3691	{
3692	int i;
3693
3694	for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3695	struct caam_aead_alg *t_alg = driver_aeads + i;
3696
3697	if (t_alg->registered)
3698	crypto_engine_unregister_aead(alg: &t_alg->aead);
3699	}
3700
3701	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3702	struct caam_skcipher_alg *t_alg = driver_algs + i;
3703
3704	if (t_alg->registered)
3705	crypto_engine_unregister_skcipher(alg: &t_alg->skcipher);
3706	}
3707	}
3708
3709	static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
3710	{
3711	struct skcipher_alg *alg = &t_alg->skcipher.base;
3712
3713	alg->base.cra_module = THIS_MODULE;
3714	alg->base.cra_priority = CAAM_CRA_PRIORITY;
3715	alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
3716	alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3717	CRYPTO_ALG_KERN_DRIVER_ONLY);
3718
3719	alg->init = caam_cra_init;
3720	alg->exit = caam_cra_exit;
3721	}
3722
3723	static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3724	{
3725	struct aead_alg *alg = &t_alg->aead.base;
3726
3727	alg->base.cra_module = THIS_MODULE;
3728	alg->base.cra_priority = CAAM_CRA_PRIORITY;
3729	alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding();
3730	alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
3731	CRYPTO_ALG_KERN_DRIVER_ONLY;
3732
3733	alg->init = caam_aead_init;
3734	alg->exit = caam_aead_exit;
3735	}
3736
3737	int caam_algapi_init(struct device *ctrldev)
3738	{
3739	struct caam_drv_private *priv = dev_get_drvdata(dev: ctrldev);
3740	int i = 0, err = 0;
3741	u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst;
3742	unsigned int md_limit = SHA512_DIGEST_SIZE;
3743	bool registered = false, gcm_support;
3744
3745	/*
3746	* Register crypto algorithms the device supports.
3747	* First, detect presence and attributes of DES, AES, and MD blocks.
3748	*/
3749	if (priv->era < 10) {
3750	struct caam_perfmon __iomem *perfmon = &priv->jr[0]->perfmon;
3751	u32 cha_vid, cha_inst, aes_rn;
3752
3753	cha_vid = rd_reg32(reg: &perfmon->cha_id_ls);
3754	aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
3755	md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
3756
3757	cha_inst = rd_reg32(reg: &perfmon->cha_num_ls);
3758	des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
3759	CHA_ID_LS_DES_SHIFT;
3760	aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
3761	md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
3762	ccha_inst = 0;
3763	ptha_inst = 0;
3764
3765	aes_rn = rd_reg32(reg: &perfmon->cha_rev_ls) & CHA_ID_LS_AES_MASK;
3766	gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8);
3767	} else {
3768	struct version_regs __iomem *vreg = &priv->jr[0]->vreg;
3769	u32 aesa, mdha;
3770
3771	aesa = rd_reg32(reg: &vreg->aesa);
3772	mdha = rd_reg32(reg: &vreg->mdha);
3773
3774	aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
3775	md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
3776
3777	des_inst = rd_reg32(reg: &vreg->desa) & CHA_VER_NUM_MASK;
3778	aes_inst = aesa & CHA_VER_NUM_MASK;
3779	md_inst = mdha & CHA_VER_NUM_MASK;
3780	ccha_inst = rd_reg32(reg: &vreg->ccha) & CHA_VER_NUM_MASK;
3781	ptha_inst = rd_reg32(reg: &vreg->ptha) & CHA_VER_NUM_MASK;
3782
3783	gcm_support = aesa & CHA_VER_MISC_AES_GCM;
3784	}
3785
3786	/* If MD is present, limit digest size based on LP256 */
3787	if (md_inst && md_vid == CHA_VER_VID_MD_LP256)
3788	md_limit = SHA256_DIGEST_SIZE;
3789
3790	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3791	struct caam_skcipher_alg *t_alg = driver_algs + i;
3792	u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
3793
3794	/* Skip DES algorithms if not supported by device */
3795	if (!des_inst &&
3796	((alg_sel == OP_ALG_ALGSEL_3DES) ||
3797	(alg_sel == OP_ALG_ALGSEL_DES)))
3798	continue;
3799
3800	/* Skip AES algorithms if not supported by device */
3801	if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
3802	continue;
3803
3804	/*
3805	* Check support for AES modes not available
3806	* on LP devices.
3807	*/
3808	if (aes_vid == CHA_VER_VID_AES_LP &&
3809	(t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK) ==
3810	OP_ALG_AAI_XTS)
3811	continue;
3812
3813	caam_skcipher_alg_init(t_alg);
3814
3815	err = crypto_engine_register_skcipher(alg: &t_alg->skcipher);
3816	if (err) {
3817	pr_warn("%s alg registration failed\n",
3818	t_alg->skcipher.base.base.cra_driver_name);
3819	continue;
3820	}
3821
3822	t_alg->registered = true;
3823	registered = true;
3824	}
3825
3826	for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3827	struct caam_aead_alg *t_alg = driver_aeads + i;
3828	u32 c1_alg_sel = t_alg->caam.class1_alg_type &
3829	OP_ALG_ALGSEL_MASK;
3830	u32 c2_alg_sel = t_alg->caam.class2_alg_type &
3831	OP_ALG_ALGSEL_MASK;
3832	u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
3833
3834	/* Skip DES algorithms if not supported by device */
3835	if (!des_inst &&
3836	((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
3837	(c1_alg_sel == OP_ALG_ALGSEL_DES)))
3838	continue;
3839
3840	/* Skip AES algorithms if not supported by device */
3841	if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
3842	continue;
3843
3844	/* Skip CHACHA20 algorithms if not supported by device */
3845	if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && !ccha_inst)
3846	continue;
3847
3848	/* Skip POLY1305 algorithms if not supported by device */
3849	if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst)
3850	continue;
3851
3852	/* Skip GCM algorithms if not supported by device */
3853	if (c1_alg_sel == OP_ALG_ALGSEL_AES &&
3854	alg_aai == OP_ALG_AAI_GCM && !gcm_support)
3855	continue;
3856
3857	/*
3858	* Skip algorithms requiring message digests
3859	* if MD or MD size is not supported by device.
3860	*/
3861	if (is_mdha(algtype: c2_alg_sel) &&
3862	(!md_inst || t_alg->aead.base.maxauthsize > md_limit))
3863	continue;
3864
3865	caam_aead_alg_init(t_alg);
3866
3867	err = crypto_engine_register_aead(alg: &t_alg->aead);
3868	if (err) {
3869	pr_warn("%s alg registration failed\n",
3870	t_alg->aead.base.base.cra_driver_name);
3871	continue;
3872	}
3873
3874	t_alg->registered = true;
3875	registered = true;
3876	}
3877
3878	if (registered)
3879	pr_info("caam algorithms registered in /proc/crypto\n");
3880
3881	return err;
3882	}
3883