1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* Shared descriptors for aead, skcipher algorithms
4	*
5	* Copyright 2016-2019 NXP
6	*/
7
8	#include "compat.h"
9	#include "desc_constr.h"
10	#include "caamalg_desc.h"
11
12	/*
13	* For aead functions, read payload and write payload,
14	* both of which are specified in req->src and req->dst
15	*/
16	static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
17	{
18	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
19	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_BOTH |
20	KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
21	}
22
23	/* Set DK bit in class 1 operation if shared */
24	static inline void append_dec_op1(u32 *desc, u32 type)
25	{
26	u32 *jump_cmd, *uncond_jump_cmd;
27
28	/* DK bit is valid only for AES */
29	if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
30	append_operation(desc, options: type | OP_ALG_AS_INITFINAL |
31	OP_ALG_DECRYPT);
32	return;
33	}
34
35	jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
36	append_operation(desc, options: type | OP_ALG_AS_INIT | OP_ALG_DECRYPT);
37	uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
38	set_jump_tgt_here(desc, jump_cmd);
39	append_operation(desc, options: type | OP_ALG_AS_INIT | OP_ALG_DECRYPT |
40	OP_ALG_AAI_DK);
41	set_jump_tgt_here(desc, jump_cmd: uncond_jump_cmd);
42	}
43
44	/**
45	* cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
46	* (non-protocol) with no (null) encryption.
47	* @desc: pointer to buffer used for descriptor construction
48	* @adata: pointer to authentication transform definitions.
49	* A split key is required for SEC Era < 6; the size of the split key
50	* is specified in this case. Valid algorithm values - one of
51	* OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
52	* with OP_ALG_AAI_HMAC_PRECOMP.
53	* @icvsize: integrity check value (ICV) size (truncated or full)
54	* @era: SEC Era
55	*/
56	void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
57	unsigned int icvsize, int era)
58	{
59	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
60
61	init_sh_desc(desc, HDR_SHARE_SERIAL);
62
63	/* Skip if already shared */
64	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
65	JUMP_COND_SHRD);
66	if (era < 6) {
67	if (adata->key_inline)
68	append_key_as_imm(desc, data: adata->key_virt,
69	data_len: adata->keylen_pad, len: adata->keylen,
70	CLASS_2 | KEY_DEST_MDHA_SPLIT |
71	KEY_ENC);
72	else
73	append_key(desc, ptr: adata->key_dma, len: adata->keylen,
74	CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
75	} else {
76	append_proto_dkp(desc, adata);
77	}
78	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
79
80	/* assoclen + cryptlen = seqinlen */
81	append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
82
83	/* Prepare to read and write cryptlen + assoclen bytes */
84	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
85	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
86
87	/*
88	* MOVE_LEN opcode is not available in all SEC HW revisions,
89	* thus need to do some magic, i.e. self-patch the descriptor
90	* buffer.
91	*/
92	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
93	MOVE_DEST_MATH3 |
94	(0x6 << MOVE_LEN_SHIFT));
95	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
96	MOVE_DEST_DESCBUF |
97	MOVE_WAITCOMP |
98	(0x8 << MOVE_LEN_SHIFT));
99
100	/* Class 2 operation */
101	append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL |
102	OP_ALG_ENCRYPT);
103
104	/* Read and write cryptlen bytes */
105	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
106
107	set_move_tgt_here(desc, move_cmd: read_move_cmd);
108	set_move_tgt_here(desc, move_cmd: write_move_cmd);
109	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
110	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
111	MOVE_AUX_LS);
112
113	/* Write ICV */
114	append_seq_store(desc, len: icvsize, LDST_CLASS_2_CCB |
115	LDST_SRCDST_BYTE_CONTEXT);
116
117	print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ",
118	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
119	1);
120	}
121	EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
122
123	/**
124	* cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
125	* (non-protocol) with no (null) decryption.
126	* @desc: pointer to buffer used for descriptor construction
127	* @adata: pointer to authentication transform definitions.
128	* A split key is required for SEC Era < 6; the size of the split key
129	* is specified in this case. Valid algorithm values - one of
130	* OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
131	* with OP_ALG_AAI_HMAC_PRECOMP.
132	* @icvsize: integrity check value (ICV) size (truncated or full)
133	* @era: SEC Era
134	*/
135	void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
136	unsigned int icvsize, int era)
137	{
138	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
139
140	init_sh_desc(desc, HDR_SHARE_SERIAL);
141
142	/* Skip if already shared */
143	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
144	JUMP_COND_SHRD);
145	if (era < 6) {
146	if (adata->key_inline)
147	append_key_as_imm(desc, data: adata->key_virt,
148	data_len: adata->keylen_pad, len: adata->keylen,
149	CLASS_2 | KEY_DEST_MDHA_SPLIT |
150	KEY_ENC);
151	else
152	append_key(desc, ptr: adata->key_dma, len: adata->keylen,
153	CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
154	} else {
155	append_proto_dkp(desc, adata);
156	}
157	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
158
159	/* Class 2 operation */
160	append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL |
161	OP_ALG_DECRYPT | OP_ALG_ICV_ON);
162
163	/* assoclen + cryptlen = seqoutlen */
164	append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
165
166	/* Prepare to read and write cryptlen + assoclen bytes */
167	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
168	append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
169
170	/*
171	* MOVE_LEN opcode is not available in all SEC HW revisions,
172	* thus need to do some magic, i.e. self-patch the descriptor
173	* buffer.
174	*/
175	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
176	MOVE_DEST_MATH2 |
177	(0x6 << MOVE_LEN_SHIFT));
178	write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
179	MOVE_DEST_DESCBUF |
180	MOVE_WAITCOMP |
181	(0x8 << MOVE_LEN_SHIFT));
182
183	/* Read and write cryptlen bytes */
184	aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
185
186	/*
187	* Insert a NOP here, since we need at least 4 instructions between
188	* code patching the descriptor buffer and the location being patched.
189	*/
190	jump_cmd = append_jump(desc, JUMP_TEST_ALL);
191	set_jump_tgt_here(desc, jump_cmd);
192
193	set_move_tgt_here(desc, move_cmd: read_move_cmd);
194	set_move_tgt_here(desc, move_cmd: write_move_cmd);
195	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
196	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
197	MOVE_AUX_LS);
198	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
199
200	/* Load ICV */
201	append_seq_fifo_load(desc, len: icvsize, FIFOLD_CLASS_CLASS2 |
202	FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
203
204	print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ",
205	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
206	1);
207	}
208	EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
209
210	static void init_sh_desc_key_aead(u32 * const desc,
211	struct alginfo * const cdata,
212	struct alginfo * const adata,
213	const bool is_rfc3686, u32 *nonce, int era)
214	{
215	u32 *key_jump_cmd;
216	unsigned int enckeylen = cdata->keylen;
217
218	/* Note: Context registers are saved. */
219	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
220
221	/* Skip if already shared */
222	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
223	JUMP_COND_SHRD);
224
225	/*
226	* RFC3686 specific:
227	* | key = {AUTH_KEY, ENC_KEY, NONCE}
228	* | enckeylen = encryption key size + nonce size
229	*/
230	if (is_rfc3686)
231	enckeylen -= CTR_RFC3686_NONCE_SIZE;
232
233	if (era < 6) {
234	if (adata->key_inline)
235	append_key_as_imm(desc, data: adata->key_virt,
236	data_len: adata->keylen_pad, len: adata->keylen,
237	CLASS_2 | KEY_DEST_MDHA_SPLIT |
238	KEY_ENC);
239	else
240	append_key(desc, ptr: adata->key_dma, len: adata->keylen,
241	CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
242	} else {
243	append_proto_dkp(desc, adata);
244	}
245
246	if (cdata->key_inline)
247	append_key_as_imm(desc, data: cdata->key_virt, data_len: enckeylen,
248	len: enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
249	else
250	append_key(desc, ptr: cdata->key_dma, len: enckeylen, CLASS_1 |
251	KEY_DEST_CLASS_REG);
252
253	/* Load Counter into CONTEXT1 reg */
254	if (is_rfc3686) {
255	append_load_as_imm(desc, data: nonce, CTR_RFC3686_NONCE_SIZE,
256	LDST_CLASS_IND_CCB |
257	LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
258	append_move(desc,
259	MOVE_SRC_OUTFIFO |
260	MOVE_DEST_CLASS1CTX |
261	(16 << MOVE_OFFSET_SHIFT) |
262	(CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
263	}
264
265	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
266	}
267
268	/**
269	* cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
270	* (non-protocol).
271	* @desc: pointer to buffer used for descriptor construction
272	* @cdata: pointer to block cipher transform definitions
273	* Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
274	* with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
275	* @adata: pointer to authentication transform definitions.
276	* A split key is required for SEC Era < 6; the size of the split key
277	* is specified in this case. Valid algorithm values - one of
278	* OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
279	* with OP_ALG_AAI_HMAC_PRECOMP.
280	* @ivsize: initialization vector size
281	* @icvsize: integrity check value (ICV) size (truncated or full)
282	* @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
283	* @nonce: pointer to rfc3686 nonce
284	* @ctx1_iv_off: IV offset in CONTEXT1 register
285	* @is_qi: true when called from caam/qi
286	* @era: SEC Era
287	*/
288	void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
289	struct alginfo *adata, unsigned int ivsize,
290	unsigned int icvsize, const bool is_rfc3686,
291	u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
292	int era)
293	{
294	/* Note: Context registers are saved. */
295	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
296
297	/* Class 2 operation */
298	append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL |
299	OP_ALG_ENCRYPT);
300
301	if (is_qi) {
302	u32 *wait_load_cmd;
303
304	/* REG3 = assoclen */
305	append_seq_load(desc, len: 4, LDST_CLASS_DECO |
306	LDST_SRCDST_WORD_DECO_MATH3 |
307	(4 << LDST_OFFSET_SHIFT));
308
309	wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
310	JUMP_COND_CALM | JUMP_COND_NCP |
311	JUMP_COND_NOP | JUMP_COND_NIP |
312	JUMP_COND_NIFP);
313	set_jump_tgt_here(desc, jump_cmd: wait_load_cmd);
314
315	append_seq_load(desc, len: ivsize, LDST_CLASS_1_CCB |
316	LDST_SRCDST_BYTE_CONTEXT |
317	(ctx1_iv_off << LDST_OFFSET_SHIFT));
318	}
319
320	/* Read and write assoclen bytes */
321	if (is_qi || era < 3) {
322	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
323	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
324	} else {
325	append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
326	append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
327	}
328
329	/* Skip assoc data */
330	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
331
332	/* read assoc before reading payload */
333	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
334	FIFOLDST_VLF);
335
336	/* Load Counter into CONTEXT1 reg */
337	if (is_rfc3686)
338	append_load_imm_be32(desc, immediate: 1, LDST_IMM | LDST_CLASS_1_CCB |
339	LDST_SRCDST_BYTE_CONTEXT |
340	((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
341	LDST_OFFSET_SHIFT));
342
343	/* Class 1 operation */
344	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL |
345	OP_ALG_ENCRYPT);
346
347	/* Read and write cryptlen bytes */
348	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
349	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
350	aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
351
352	/* Write ICV */
353	append_seq_store(desc, len: icvsize, LDST_CLASS_2_CCB |
354	LDST_SRCDST_BYTE_CONTEXT);
355
356	print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ",
357	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
358	1);
359	}
360	EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
361
362	/**
363	* cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
364	* (non-protocol).
365	* @desc: pointer to buffer used for descriptor construction
366	* @cdata: pointer to block cipher transform definitions
367	* Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
368	* with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
369	* @adata: pointer to authentication transform definitions.
370	* A split key is required for SEC Era < 6; the size of the split key
371	* is specified in this case. Valid algorithm values - one of
372	* OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
373	* with OP_ALG_AAI_HMAC_PRECOMP.
374	* @ivsize: initialization vector size
375	* @icvsize: integrity check value (ICV) size (truncated or full)
376	* @geniv: whether to generate Encrypted Chain IV
377	* @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
378	* @nonce: pointer to rfc3686 nonce
379	* @ctx1_iv_off: IV offset in CONTEXT1 register
380	* @is_qi: true when called from caam/qi
381	* @era: SEC Era
382	*/
383	void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
384	struct alginfo *adata, unsigned int ivsize,
385	unsigned int icvsize, const bool geniv,
386	const bool is_rfc3686, u32 *nonce,
387	const u32 ctx1_iv_off, const bool is_qi, int era)
388	{
389	/* Note: Context registers are saved. */
390	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
391
392	/* Class 2 operation */
393	append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL |
394	OP_ALG_DECRYPT | OP_ALG_ICV_ON);
395
396	if (is_qi) {
397	u32 *wait_load_cmd;
398
399	/* REG3 = assoclen */
400	append_seq_load(desc, len: 4, LDST_CLASS_DECO |
401	LDST_SRCDST_WORD_DECO_MATH3 |
402	(4 << LDST_OFFSET_SHIFT));
403
404	wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
405	JUMP_COND_CALM | JUMP_COND_NCP |
406	JUMP_COND_NOP | JUMP_COND_NIP |
407	JUMP_COND_NIFP);
408	set_jump_tgt_here(desc, jump_cmd: wait_load_cmd);
409
410	if (!geniv)
411	append_seq_load(desc, len: ivsize, LDST_CLASS_1_CCB |
412	LDST_SRCDST_BYTE_CONTEXT |
413	(ctx1_iv_off << LDST_OFFSET_SHIFT));
414	}
415
416	/* Read and write assoclen bytes */
417	if (is_qi || era < 3) {
418	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
419	if (geniv)
420	append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
421	ivsize);
422	else
423	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
424	CAAM_CMD_SZ);
425	} else {
426	append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
427	if (geniv)
428	append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
429	ivsize);
430	else
431	append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
432	CAAM_CMD_SZ);
433	}
434
435	/* Skip assoc data */
436	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
437
438	/* read assoc before reading payload */
439	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
440	KEY_VLF);
441
442	if (geniv) {
443	append_seq_load(desc, len: ivsize, LDST_CLASS_1_CCB |
444	LDST_SRCDST_BYTE_CONTEXT |
445	(ctx1_iv_off << LDST_OFFSET_SHIFT));
446	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
447	(ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
448	}
449
450	/* Load Counter into CONTEXT1 reg */
451	if (is_rfc3686)
452	append_load_imm_be32(desc, immediate: 1, LDST_IMM | LDST_CLASS_1_CCB |
453	LDST_SRCDST_BYTE_CONTEXT |
454	((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
455	LDST_OFFSET_SHIFT));
456
457	/* Choose operation */
458	if (ctx1_iv_off)
459	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL |
460	OP_ALG_DECRYPT);
461	else
462	append_dec_op1(desc, type: cdata->algtype);
463
464	/* Read and write cryptlen bytes */
465	append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
466	append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
467	aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
468
469	/* Load ICV */
470	append_seq_fifo_load(desc, len: icvsize, FIFOLD_CLASS_CLASS2 |
471	FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
472
473	print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ",
474	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
475	1);
476	}
477	EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
478
479	/**
480	* cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
481	* (non-protocol) with HW-generated initialization
482	* vector.
483	* @desc: pointer to buffer used for descriptor construction
484	* @cdata: pointer to block cipher transform definitions
485	* Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
486	* with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
487	* @adata: pointer to authentication transform definitions.
488	* A split key is required for SEC Era < 6; the size of the split key
489	* is specified in this case. Valid algorithm values - one of
490	* OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
491	* with OP_ALG_AAI_HMAC_PRECOMP.
492	* @ivsize: initialization vector size
493	* @icvsize: integrity check value (ICV) size (truncated or full)
494	* @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
495	* @nonce: pointer to rfc3686 nonce
496	* @ctx1_iv_off: IV offset in CONTEXT1 register
497	* @is_qi: true when called from caam/qi
498	* @era: SEC Era
499	*/
500	void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
501	struct alginfo *adata, unsigned int ivsize,
502	unsigned int icvsize, const bool is_rfc3686,
503	u32 *nonce, const u32 ctx1_iv_off,
504	const bool is_qi, int era)
505	{
506	u32 geniv, moveiv;
507	u32 *wait_cmd;
508
509	/* Note: Context registers are saved. */
510	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
511
512	if (is_qi) {
513	u32 *wait_load_cmd;
514
515	/* REG3 = assoclen */
516	append_seq_load(desc, len: 4, LDST_CLASS_DECO |
517	LDST_SRCDST_WORD_DECO_MATH3 |
518	(4 << LDST_OFFSET_SHIFT));
519
520	wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
521	JUMP_COND_CALM | JUMP_COND_NCP |
522	JUMP_COND_NOP | JUMP_COND_NIP |
523	JUMP_COND_NIFP);
524	set_jump_tgt_here(desc, jump_cmd: wait_load_cmd);
525	}
526
527	if (is_rfc3686) {
528	if (is_qi)
529	append_seq_load(desc, len: ivsize, LDST_CLASS_1_CCB |
530	LDST_SRCDST_BYTE_CONTEXT |
531	(ctx1_iv_off << LDST_OFFSET_SHIFT));
532
533	goto copy_iv;
534	}
535
536	/* Generate IV */
537	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
538	NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
539	NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
540	append_load_imm_u32(desc, immediate: geniv, LDST_CLASS_IND_CCB |
541	LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
542	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
543	append_move(desc, MOVE_WAITCOMP |
544	MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
545	(ctx1_iv_off << MOVE_OFFSET_SHIFT) |
546	(ivsize << MOVE_LEN_SHIFT));
547	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
548
549	copy_iv:
550	/* Copy IV to class 1 context */
551	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
552	(ctx1_iv_off << MOVE_OFFSET_SHIFT) |
553	(ivsize << MOVE_LEN_SHIFT));
554
555	/* Return to encryption */
556	append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL |
557	OP_ALG_ENCRYPT);
558
559	/* Read and write assoclen bytes */
560	if (is_qi || era < 3) {
561	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
562	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
563	} else {
564	append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
565	append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
566	}
567
568	/* Skip assoc data */
569	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
570
571	/* read assoc before reading payload */
572	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
573	KEY_VLF);
574
575	/* Copy iv from outfifo to class 2 fifo */
576	moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
577	NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
578	append_load_imm_u32(desc, immediate: moveiv, LDST_CLASS_IND_CCB |
579	LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
580	append_load_imm_u32(desc, immediate: ivsize, LDST_CLASS_2_CCB |
581	LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
582
583	/* Load Counter into CONTEXT1 reg */
584	if (is_rfc3686)
585	append_load_imm_be32(desc, immediate: 1, LDST_IMM | LDST_CLASS_1_CCB |
586	LDST_SRCDST_BYTE_CONTEXT |
587	((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
588	LDST_OFFSET_SHIFT));
589
590	/* Class 1 operation */
591	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL |
592	OP_ALG_ENCRYPT);
593
594	/* Will write ivsize + cryptlen */
595	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
596
597	/* Not need to reload iv */
598	append_seq_fifo_load(desc, len: ivsize,
599	FIFOLD_CLASS_SKIP);
600
601	/* Will read cryptlen */
602	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
603
604	/*
605	* Wait for IV transfer (ofifo -> class2) to finish before starting
606	* ciphertext transfer (ofifo -> external memory).
607	*/
608	wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP);
609	set_jump_tgt_here(desc, jump_cmd: wait_cmd);
610
611	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_BOTH | KEY_VLF |
612	FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
613	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
614
615	/* Write ICV */
616	append_seq_store(desc, len: icvsize, LDST_CLASS_2_CCB |
617	LDST_SRCDST_BYTE_CONTEXT);
618
619	print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ",
620	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
621	1);
622	}
623	EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
624
625	/**
626	* cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
627	* @desc: pointer to buffer used for descriptor construction
628	* @cdata: pointer to block cipher transform definitions
629	* Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
630	* @ivsize: initialization vector size
631	* @icvsize: integrity check value (ICV) size (truncated or full)
632	* @is_qi: true when called from caam/qi
633	*/
634	void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
635	unsigned int ivsize, unsigned int icvsize,
636	const bool is_qi)
637	{
638	u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
639	*zero_assoc_jump_cmd2;
640
641	init_sh_desc(desc, HDR_SHARE_SERIAL);
642
643	/* skip key loading if they are loaded due to sharing */
644	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
645	JUMP_COND_SHRD);
646	if (cdata->key_inline)
647	append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen,
648	len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
649	else
650	append_key(desc, ptr: cdata->key_dma, len: cdata->keylen, CLASS_1 |
651	KEY_DEST_CLASS_REG);
652	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
653
654	/* class 1 operation */
655	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL |
656	OP_ALG_ENCRYPT);
657
658	if (is_qi) {
659	u32 *wait_load_cmd;
660
661	/* REG3 = assoclen */
662	append_seq_load(desc, len: 4, LDST_CLASS_DECO |
663	LDST_SRCDST_WORD_DECO_MATH3 |
664	(4 << LDST_OFFSET_SHIFT));
665
666	wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
667	JUMP_COND_CALM | JUMP_COND_NCP |
668	JUMP_COND_NOP | JUMP_COND_NIP |
669	JUMP_COND_NIFP);
670	set_jump_tgt_here(desc, jump_cmd: wait_load_cmd);
671
672	append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
673	ivsize);
674	} else {
675	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
676	CAAM_CMD_SZ);
677	}
678
679	/* if assoclen + cryptlen is ZERO, skip to ICV write */
680	zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
681	JUMP_COND_MATH_Z);
682
683	if (is_qi)
684	append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 |
685	FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
686
687	/* if assoclen is ZERO, skip reading the assoc data */
688	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
689	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
690	JUMP_COND_MATH_Z);
691
692	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
693
694	/* skip assoc data */
695	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
696
697	/* cryptlen = seqinlen - assoclen */
698	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
699
700	/* if cryptlen is ZERO jump to zero-payload commands */
701	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
702	JUMP_COND_MATH_Z);
703
704	/* read assoc data */
705	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
706	FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
707	set_jump_tgt_here(desc, jump_cmd: zero_assoc_jump_cmd1);
708
709	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
710
711	/* write encrypted data */
712	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
713
714	/* read payload data */
715	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
716	FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
717
718	/* jump to ICV writing */
719	if (is_qi)
720	append_jump(desc, JUMP_TEST_ALL | 4);
721	else
722	append_jump(desc, JUMP_TEST_ALL | 2);
723
724	/* zero-payload commands */
725	set_jump_tgt_here(desc, jump_cmd: zero_payload_jump_cmd);
726
727	/* read assoc data */
728	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
729	FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
730	if (is_qi)
731	/* jump to ICV writing */
732	append_jump(desc, JUMP_TEST_ALL | 2);
733
734	/* There is no input data */
735	set_jump_tgt_here(desc, jump_cmd: zero_assoc_jump_cmd2);
736
737	if (is_qi)
738	append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 |
739	FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
740	FIFOLD_TYPE_LAST1);
741
742	/* write ICV */
743	append_seq_store(desc, len: icvsize, LDST_CLASS_1_CCB |
744	LDST_SRCDST_BYTE_CONTEXT);
745
746	print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ",
747	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
748	1);
749	}
750	EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
751
752	/**
753	* cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
754	* @desc: pointer to buffer used for descriptor construction
755	* @cdata: pointer to block cipher transform definitions
756	* Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
757	* @ivsize: initialization vector size
758	* @icvsize: integrity check value (ICV) size (truncated or full)
759	* @is_qi: true when called from caam/qi
760	*/
761	void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
762	unsigned int ivsize, unsigned int icvsize,
763	const bool is_qi)
764	{
765	u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
766
767	init_sh_desc(desc, HDR_SHARE_SERIAL);
768
769	/* skip key loading if they are loaded due to sharing */
770	key_jump_cmd = append_jump(desc, JUMP_JSL |
771	JUMP_TEST_ALL | JUMP_COND_SHRD);
772	if (cdata->key_inline)
773	append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen,
774	len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
775	else
776	append_key(desc, ptr: cdata->key_dma, len: cdata->keylen, CLASS_1 |
777	KEY_DEST_CLASS_REG);
778	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
779
780	/* class 1 operation */
781	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL |
782	OP_ALG_DECRYPT | OP_ALG_ICV_ON);
783
784	if (is_qi) {
785	u32 *wait_load_cmd;
786
787	/* REG3 = assoclen */
788	append_seq_load(desc, len: 4, LDST_CLASS_DECO |
789	LDST_SRCDST_WORD_DECO_MATH3 |
790	(4 << LDST_OFFSET_SHIFT));
791
792	wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
793	JUMP_COND_CALM | JUMP_COND_NCP |
794	JUMP_COND_NOP | JUMP_COND_NIP |
795	JUMP_COND_NIFP);
796	set_jump_tgt_here(desc, jump_cmd: wait_load_cmd);
797
798	append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 |
799	FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
800	}
801
802	/* if assoclen is ZERO, skip reading the assoc data */
803	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
804	zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
805	JUMP_COND_MATH_Z);
806
807	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
808
809	/* skip assoc data */
810	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
811
812	/* read assoc data */
813	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
814	FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
815
816	set_jump_tgt_here(desc, jump_cmd: zero_assoc_jump_cmd1);
817
818	/* cryptlen = seqoutlen - assoclen */
819	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
820
821	/* jump to zero-payload command if cryptlen is zero */
822	zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
823	JUMP_COND_MATH_Z);
824
825	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
826
827	/* store encrypted data */
828	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
829
830	/* read payload data */
831	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
832	FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
833
834	/* zero-payload command */
835	set_jump_tgt_here(desc, jump_cmd: zero_payload_jump_cmd);
836
837	/* read ICV */
838	append_seq_fifo_load(desc, len: icvsize, FIFOLD_CLASS_CLASS1 |
839	FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
840
841	print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ",
842	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
843	1);
844	}
845	EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
846
847	/**
848	* cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
849	* (non-protocol).
850	* @desc: pointer to buffer used for descriptor construction
851	* @cdata: pointer to block cipher transform definitions
852	* Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
853	* @ivsize: initialization vector size
854	* @icvsize: integrity check value (ICV) size (truncated or full)
855	* @is_qi: true when called from caam/qi
856	*
857	* Input sequence: AAD | PTXT
858	* Output sequence: AAD | CTXT | ICV
859	* AAD length (assoclen), which includes the IV length, is available in Math3.
860	*/
861	void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
862	unsigned int ivsize, unsigned int icvsize,
863	const bool is_qi)
864	{
865	u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions;
866	init_sh_desc(desc, HDR_SHARE_SERIAL);
867
868	/* Skip key loading if it is loaded due to sharing */
869	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
870	JUMP_COND_SHRD);
871	if (cdata->key_inline)
872	append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen,
873	len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
874	else
875	append_key(desc, ptr: cdata->key_dma, len: cdata->keylen, CLASS_1 |
876	KEY_DEST_CLASS_REG);
877	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
878
879	/* Class 1 operation */
880	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL |
881	OP_ALG_ENCRYPT);
882
883	if (is_qi) {
884	u32 *wait_load_cmd;
885
886	/* REG3 = assoclen */
887	append_seq_load(desc, len: 4, LDST_CLASS_DECO |
888	LDST_SRCDST_WORD_DECO_MATH3 |
889	(4 << LDST_OFFSET_SHIFT));
890
891	wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
892	JUMP_COND_CALM | JUMP_COND_NCP |
893	JUMP_COND_NOP | JUMP_COND_NIP |
894	JUMP_COND_NIFP);
895	set_jump_tgt_here(desc, jump_cmd: wait_load_cmd);
896
897	/* Read salt and IV */
898	append_fifo_load_as_imm(desc, data: (void *)(cdata->key_virt +
899	cdata->keylen), len: 4, FIFOLD_CLASS_CLASS1 |
900	FIFOLD_TYPE_IV);
901	append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 |
902	FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
903	}
904
905	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
906	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
907
908	/* Skip AAD */
909	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
910
911	/* Read cryptlen and set this value into VARSEQOUTLEN */
912	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
913
914	/* If cryptlen is ZERO jump to AAD command */
915	zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
916	JUMP_COND_MATH_Z);
917
918	/* Read AAD data */
919	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
920	FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
921
922	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
923	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA);
924
925	/* Skip IV */
926	append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_SKIP);
927	append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ);
928
929	/* Write encrypted data */
930	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
931
932	/* Read payload data */
933	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
934	FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
935
936	/* Jump instructions to avoid double reading of AAD */
937	skip_instructions = append_jump(desc, JUMP_TEST_ALL);
938
939	/* There is no input data, cryptlen = 0 */
940	set_jump_tgt_here(desc, jump_cmd: zero_cryptlen_jump_cmd);
941
942	/* Read AAD */
943	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
944	FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
945
946	set_jump_tgt_here(desc, jump_cmd: skip_instructions);
947
948	/* Write ICV */
949	append_seq_store(desc, len: icvsize, LDST_CLASS_1_CCB |
950	LDST_SRCDST_BYTE_CONTEXT);
951
952	print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ",
953	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
954	1);
955	}
956	EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
957
958	/**
959	* cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
960	* (non-protocol).
961	* @desc: pointer to buffer used for descriptor construction
962	* @cdata: pointer to block cipher transform definitions
963	* Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
964	* @ivsize: initialization vector size
965	* @icvsize: integrity check value (ICV) size (truncated or full)
966	* @is_qi: true when called from caam/qi
967	*/
968	void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
969	unsigned int ivsize, unsigned int icvsize,
970	const bool is_qi)
971	{
972	u32 *key_jump_cmd;
973
974	init_sh_desc(desc, HDR_SHARE_SERIAL);
975
976	/* Skip key loading if it is loaded due to sharing */
977	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
978	JUMP_COND_SHRD);
979	if (cdata->key_inline)
980	append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen,
981	len: cdata->keylen, CLASS_1 |
982	KEY_DEST_CLASS_REG);
983	else
984	append_key(desc, ptr: cdata->key_dma, len: cdata->keylen, CLASS_1 |
985	KEY_DEST_CLASS_REG);
986	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
987
988	/* Class 1 operation */
989	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL |
990	OP_ALG_DECRYPT | OP_ALG_ICV_ON);
991
992	if (is_qi) {
993	u32 *wait_load_cmd;
994
995	/* REG3 = assoclen */
996	append_seq_load(desc, len: 4, LDST_CLASS_DECO |
997	LDST_SRCDST_WORD_DECO_MATH3 |
998	(4 << LDST_OFFSET_SHIFT));
999
1000	wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1001	JUMP_COND_CALM | JUMP_COND_NCP |
1002	JUMP_COND_NOP | JUMP_COND_NIP |
1003	JUMP_COND_NIFP);
1004	set_jump_tgt_here(desc, jump_cmd: wait_load_cmd);
1005
1006	/* Read salt and IV */
1007	append_fifo_load_as_imm(desc, data: (void *)(cdata->key_virt +
1008	cdata->keylen), len: 4, FIFOLD_CLASS_CLASS1 |
1009	FIFOLD_TYPE_IV);
1010	append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 |
1011	FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1012	}
1013
1014	append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
1015	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1016
1017	/* Read assoc data */
1018	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1019	FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1020
1021	/* Skip IV */
1022	append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_SKIP);
1023
1024	/* Will read cryptlen bytes */
1025	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1026
1027	/* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1028	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1029
1030	/* Skip assoc data */
1031	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1032
1033	/* Will write cryptlen bytes */
1034	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1035
1036	/* Store payload data */
1037	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1038
1039	/* Read encrypted data */
1040	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1041	FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1042
1043	/* Read ICV */
1044	append_seq_fifo_load(desc, len: icvsize, FIFOLD_CLASS_CLASS1 |
1045	FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1046
1047	print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1048	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1049	1);
1050	}
1051	EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1052
1053	/**
1054	* cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
1055	* (non-protocol).
1056	* @desc: pointer to buffer used for descriptor construction
1057	* @cdata: pointer to block cipher transform definitions
1058	* Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1059	* @ivsize: initialization vector size
1060	* @icvsize: integrity check value (ICV) size (truncated or full)
1061	* @is_qi: true when called from caam/qi
1062	*/
1063	void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1064	unsigned int ivsize, unsigned int icvsize,
1065	const bool is_qi)
1066	{
1067	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1068
1069	init_sh_desc(desc, HDR_SHARE_SERIAL);
1070
1071	/* Skip key loading if it is loaded due to sharing */
1072	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1073	JUMP_COND_SHRD);
1074	if (cdata->key_inline)
1075	append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen,
1076	len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1077	else
1078	append_key(desc, ptr: cdata->key_dma, len: cdata->keylen, CLASS_1 |
1079	KEY_DEST_CLASS_REG);
1080	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
1081
1082	/* Class 1 operation */
1083	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL |
1084	OP_ALG_ENCRYPT);
1085
1086	if (is_qi) {
1087	/* assoclen is not needed, skip it */
1088	append_seq_fifo_load(desc, len: 4, FIFOLD_CLASS_SKIP);
1089
1090	/* Read salt and IV */
1091	append_fifo_load_as_imm(desc, data: (void *)(cdata->key_virt +
1092	cdata->keylen), len: 4, FIFOLD_CLASS_CLASS1 |
1093	FIFOLD_TYPE_IV);
1094	append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 |
1095	FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1096	}
1097
1098	/* assoclen + cryptlen = seqinlen */
1099	append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1100
1101	/*
1102	* MOVE_LEN opcode is not available in all SEC HW revisions,
1103	* thus need to do some magic, i.e. self-patch the descriptor
1104	* buffer.
1105	*/
1106	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1107	(0x6 << MOVE_LEN_SHIFT));
1108	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1109	(0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1110
1111	/* Will read assoclen + cryptlen bytes */
1112	append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1113
1114	/* Will write assoclen + cryptlen bytes */
1115	append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1116
1117	/* Read and write assoclen + cryptlen bytes */
1118	aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1119
1120	set_move_tgt_here(desc, move_cmd: read_move_cmd);
1121	set_move_tgt_here(desc, move_cmd: write_move_cmd);
1122	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1123	/* Move payload data to OFIFO */
1124	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1125
1126	/* Write ICV */
1127	append_seq_store(desc, len: icvsize, LDST_CLASS_1_CCB |
1128	LDST_SRCDST_BYTE_CONTEXT);
1129
1130	print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1131	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1132	1);
1133	}
1134	EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1135
1136	/**
1137	* cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
1138	* (non-protocol).
1139	* @desc: pointer to buffer used for descriptor construction
1140	* @cdata: pointer to block cipher transform definitions
1141	* Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1142	* @ivsize: initialization vector size
1143	* @icvsize: integrity check value (ICV) size (truncated or full)
1144	* @is_qi: true when called from caam/qi
1145	*/
1146	void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1147	unsigned int ivsize, unsigned int icvsize,
1148	const bool is_qi)
1149	{
1150	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1151
1152	init_sh_desc(desc, HDR_SHARE_SERIAL);
1153
1154	/* Skip key loading if it is loaded due to sharing */
1155	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1156	JUMP_COND_SHRD);
1157	if (cdata->key_inline)
1158	append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen,
1159	len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1160	else
1161	append_key(desc, ptr: cdata->key_dma, len: cdata->keylen, CLASS_1 |
1162	KEY_DEST_CLASS_REG);
1163	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
1164
1165	/* Class 1 operation */
1166	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL |
1167	OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1168
1169	if (is_qi) {
1170	/* assoclen is not needed, skip it */
1171	append_seq_fifo_load(desc, len: 4, FIFOLD_CLASS_SKIP);
1172
1173	/* Read salt and IV */
1174	append_fifo_load_as_imm(desc, data: (void *)(cdata->key_virt +
1175	cdata->keylen), len: 4, FIFOLD_CLASS_CLASS1 |
1176	FIFOLD_TYPE_IV);
1177	append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 |
1178	FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1179	}
1180
1181	/* assoclen + cryptlen = seqoutlen */
1182	append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1183
1184	/*
1185	* MOVE_LEN opcode is not available in all SEC HW revisions,
1186	* thus need to do some magic, i.e. self-patch the descriptor
1187	* buffer.
1188	*/
1189	read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1190	(0x6 << MOVE_LEN_SHIFT));
1191	write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1192	(0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1193
1194	/* Will read assoclen + cryptlen bytes */
1195	append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1196
1197	/* Will write assoclen + cryptlen bytes */
1198	append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1199
1200	/* Store payload data */
1201	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1202
1203	/* In-snoop assoclen + cryptlen data */
1204	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1205	FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1206
1207	set_move_tgt_here(desc, move_cmd: read_move_cmd);
1208	set_move_tgt_here(desc, move_cmd: write_move_cmd);
1209	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1210	/* Move payload data to OFIFO */
1211	append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1212	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1213
1214	/* Read ICV */
1215	append_seq_fifo_load(desc, len: icvsize, FIFOLD_CLASS_CLASS1 |
1216	FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1217
1218	print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1219	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1220	1);
1221	}
1222	EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1223
1224	/**
1225	* cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and
1226	* IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared
1227	* descriptor (non-protocol).
1228	* @desc: pointer to buffer used for descriptor construction
1229	* @cdata: pointer to block cipher transform definitions
1230	* Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with
1231	* OP_ALG_AAI_AEAD.
1232	* @adata: pointer to authentication transform definitions
1233	* Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with
1234	* OP_ALG_AAI_AEAD.
1235	* @ivsize: initialization vector size
1236	* @icvsize: integrity check value (ICV) size (truncated or full)
1237	* @encap: true if encapsulation, false if decapsulation
1238	* @is_qi: true when called from caam/qi
1239	*/
1240	void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
1241	struct alginfo *adata, unsigned int ivsize,
1242	unsigned int icvsize, const bool encap,
1243	const bool is_qi)
1244	{
1245	u32 *key_jump_cmd, *wait_cmd;
1246	u32 nfifo;
1247	const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
1248
1249	/* Note: Context registers are saved. */
1250	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1251
1252	/* skip key loading if they are loaded due to sharing */
1253	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1254	JUMP_COND_SHRD);
1255
1256	append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen, len: cdata->keylen,
1257	CLASS_1 | KEY_DEST_CLASS_REG);
1258
1259	/* For IPsec load the salt from keymat in the context register */
1260	if (is_ipsec)
1261	append_load_as_imm(desc, data: cdata->key_virt + cdata->keylen, len: 4,
1262	LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
1263	4 << LDST_OFFSET_SHIFT);
1264
1265	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
1266
1267	/* Class 2 and 1 operations: Poly & ChaCha */
1268	if (encap) {
1269	append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL |
1270	OP_ALG_ENCRYPT);
1271	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL |
1272	OP_ALG_ENCRYPT);
1273	} else {
1274	append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL |
1275	OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1276	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL |
1277	OP_ALG_DECRYPT);
1278	}
1279
1280	if (is_qi) {
1281	u32 *wait_load_cmd;
1282	u32 ctx1_iv_off = is_ipsec ? 8 : 4;
1283
1284	/* REG3 = assoclen */
1285	append_seq_load(desc, len: 4, LDST_CLASS_DECO |
1286	LDST_SRCDST_WORD_DECO_MATH3 |
1287	4 << LDST_OFFSET_SHIFT);
1288
1289	wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1290	JUMP_COND_CALM | JUMP_COND_NCP |
1291	JUMP_COND_NOP | JUMP_COND_NIP |
1292	JUMP_COND_NIFP);
1293	set_jump_tgt_here(desc, jump_cmd: wait_load_cmd);
1294
1295	append_seq_load(desc, len: ivsize, LDST_CLASS_1_CCB |
1296	LDST_SRCDST_BYTE_CONTEXT |
1297	ctx1_iv_off << LDST_OFFSET_SHIFT);
1298	}
1299
1300	/*
1301	* MAGIC with NFIFO
1302	* Read associated data from the input and send them to class1 and
1303	* class2 alignment blocks. From class1 send data to output fifo and
1304	* then write it to memory since we don't need to encrypt AD.
1305	*/
1306	nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
1307	NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
1308	append_load_imm_u32(desc, immediate: nfifo, LDST_CLASS_IND_CCB |
1309	LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
1310
1311	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1312	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1313	append_seq_fifo_load(desc, len: 0, FIFOLD_TYPE_NOINFOFIFO |
1314	FIFOLD_CLASS_CLASS1 | LDST_VLF);
1315	append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
1316	MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
1317	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
1318
1319	/* IPsec - copy IV at the output */
1320	if (is_ipsec)
1321	append_seq_fifo_store(desc, len: ivsize, FIFOST_TYPE_METADATA |
1322	0x2 << 25);
1323
1324	wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1325	JUMP_COND_NOP | JUMP_TEST_ALL);
1326	set_jump_tgt_here(desc, jump_cmd: wait_cmd);
1327
1328	if (encap) {
1329	/* Read and write cryptlen bytes */
1330	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1331	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
1332	CAAM_CMD_SZ);
1333	aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
1334
1335	/* Write ICV */
1336	append_seq_store(desc, len: icvsize, LDST_CLASS_2_CCB |
1337	LDST_SRCDST_BYTE_CONTEXT);
1338	} else {
1339	/* Read and write cryptlen bytes */
1340	append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0,
1341	CAAM_CMD_SZ);
1342	append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
1343	CAAM_CMD_SZ);
1344	aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
1345
1346	/* Load ICV for verification */
1347	append_seq_fifo_load(desc, len: icvsize, FIFOLD_CLASS_CLASS2 |
1348	FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1349	}
1350
1351	print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
1352	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1353	1);
1354	}
1355	EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
1356
1357	/* For skcipher encrypt and decrypt, read from req->src and write to req->dst */
1358	static inline void skcipher_append_src_dst(u32 *desc)
1359	{
1360	append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1361	append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1362	append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 |
1363	KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1364	append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1365	}
1366
1367	/**
1368	* cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor
1369	* @desc: pointer to buffer used for descriptor construction
1370	* @cdata: pointer to block cipher transform definitions
1371	* Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1372	* with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1373	* - OP_ALG_ALGSEL_CHACHA20
1374	* @ivsize: initialization vector size
1375	* @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1376	* @ctx1_iv_off: IV offset in CONTEXT1 register
1377	*/
1378	void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
1379	unsigned int ivsize, const bool is_rfc3686,
1380	const u32 ctx1_iv_off)
1381	{
1382	u32 *key_jump_cmd;
1383	u32 options = cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT;
1384	bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1385	OP_ALG_ALGSEL_CHACHA20);
1386
1387	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1388	/* Skip if already shared */
1389	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1390	JUMP_COND_SHRD);
1391
1392	/* Load class1 key only */
1393	append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen,
1394	len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1395
1396	/* Load nonce into CONTEXT1 reg */
1397	if (is_rfc3686) {
1398	const u8 *nonce = cdata->key_virt + cdata->keylen;
1399
1400	append_load_as_imm(desc, data: nonce, CTR_RFC3686_NONCE_SIZE,
1401	LDST_CLASS_IND_CCB |
1402	LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1403	append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1404	MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1405	(CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1406	}
1407
1408	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
1409
1410	/* Load IV, if there is one */
1411	if (ivsize)
1412	append_seq_load(desc, len: ivsize, LDST_SRCDST_BYTE_CONTEXT |
1413	LDST_CLASS_1_CCB | (ctx1_iv_off <<
1414	LDST_OFFSET_SHIFT));
1415
1416	/* Load counter into CONTEXT1 reg */
1417	if (is_rfc3686)
1418	append_load_imm_be32(desc, immediate: 1, LDST_IMM | LDST_CLASS_1_CCB |
1419	LDST_SRCDST_BYTE_CONTEXT |
1420	((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1421	LDST_OFFSET_SHIFT));
1422
1423	/* Load operation */
1424	if (is_chacha20)
1425	options |= OP_ALG_AS_FINALIZE;
1426	append_operation(desc, options);
1427
1428	/* Perform operation */
1429	skcipher_append_src_dst(desc);
1430
1431	/* Store IV */
1432	if (!is_chacha20 && ivsize)
1433	append_seq_store(desc, len: ivsize, LDST_SRCDST_BYTE_CONTEXT |
1434	LDST_CLASS_1_CCB | (ctx1_iv_off <<
1435	LDST_OFFSET_SHIFT));
1436
1437	print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ",
1438	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1439	1);
1440	}
1441	EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
1442
1443	/**
1444	* cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor
1445	* @desc: pointer to buffer used for descriptor construction
1446	* @cdata: pointer to block cipher transform definitions
1447	* Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1448	* with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1449	* - OP_ALG_ALGSEL_CHACHA20
1450	* @ivsize: initialization vector size
1451	* @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1452	* @ctx1_iv_off: IV offset in CONTEXT1 register
1453	*/
1454	void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
1455	unsigned int ivsize, const bool is_rfc3686,
1456	const u32 ctx1_iv_off)
1457	{
1458	u32 *key_jump_cmd;
1459	bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1460	OP_ALG_ALGSEL_CHACHA20);
1461
1462	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1463	/* Skip if already shared */
1464	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1465	JUMP_COND_SHRD);
1466
1467	/* Load class1 key only */
1468	append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen,
1469	len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1470
1471	/* Load nonce into CONTEXT1 reg */
1472	if (is_rfc3686) {
1473	const u8 *nonce = cdata->key_virt + cdata->keylen;
1474
1475	append_load_as_imm(desc, data: nonce, CTR_RFC3686_NONCE_SIZE,
1476	LDST_CLASS_IND_CCB |
1477	LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1478	append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1479	MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1480	(CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1481	}
1482
1483	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
1484
1485	/* Load IV, if there is one */
1486	if (ivsize)
1487	append_seq_load(desc, len: ivsize, LDST_SRCDST_BYTE_CONTEXT |
1488	LDST_CLASS_1_CCB | (ctx1_iv_off <<
1489	LDST_OFFSET_SHIFT));
1490
1491	/* Load counter into CONTEXT1 reg */
1492	if (is_rfc3686)
1493	append_load_imm_be32(desc, immediate: 1, LDST_IMM | LDST_CLASS_1_CCB |
1494	LDST_SRCDST_BYTE_CONTEXT |
1495	((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1496	LDST_OFFSET_SHIFT));
1497
1498	/* Choose operation */
1499	if (ctx1_iv_off)
1500	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INIT |
1501	OP_ALG_DECRYPT);
1502	else
1503	append_dec_op1(desc, type: cdata->algtype);
1504
1505	/* Perform operation */
1506	skcipher_append_src_dst(desc);
1507
1508	/* Store IV */
1509	if (!is_chacha20 && ivsize)
1510	append_seq_store(desc, len: ivsize, LDST_SRCDST_BYTE_CONTEXT |
1511	LDST_CLASS_1_CCB | (ctx1_iv_off <<
1512	LDST_OFFSET_SHIFT));
1513
1514	print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ",
1515	DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1516	1);
1517	}
1518	EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
1519
1520	/**
1521	* cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor
1522	* @desc: pointer to buffer used for descriptor construction
1523	* @cdata: pointer to block cipher transform definitions
1524	* Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1525	*/
1526	void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
1527	{
1528	/*
1529	* Set sector size to a big value, practically disabling
1530	* sector size segmentation in xts implementation. We cannot
1531	* take full advantage of this HW feature with existing
1532	* crypto API / dm-crypt SW architecture.
1533	*/
1534	__be64 sector_size = cpu_to_be64(BIT(15));
1535	u32 *key_jump_cmd;
1536
1537	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1538	/* Skip if already shared */
1539	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1540	JUMP_COND_SHRD);
1541
1542	/* Load class1 keys only */
1543	append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen,
1544	len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1545
1546	/* Load sector size with index 40 bytes (0x28) */
1547	append_load_as_imm(desc, data: (void *)&sector_size, len: 8, LDST_CLASS_1_CCB |
1548	LDST_SRCDST_BYTE_CONTEXT |
1549	(0x28 << LDST_OFFSET_SHIFT));
1550
1551	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
1552
1553	/*
1554	* create sequence for loading the sector index / 16B tweak value
1555	* Lower 8B of IV - sector index / tweak lower half
1556	* Upper 8B of IV - upper half of 16B tweak
1557	*/
1558	append_seq_load(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1559	(0x20 << LDST_OFFSET_SHIFT));
1560	append_seq_load(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1561	(0x30 << LDST_OFFSET_SHIFT));
1562
1563	/* Load operation */
1564	append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL |
1565	OP_ALG_ENCRYPT);
1566
1567	/* Perform operation */
1568	skcipher_append_src_dst(desc);
1569
1570	/* Store lower 8B and upper 8B of IV */
1571	append_seq_store(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1572	(0x20 << LDST_OFFSET_SHIFT));
1573	append_seq_store(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1574	(0x30 << LDST_OFFSET_SHIFT));
1575
1576	print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
1577	": ", DUMP_PREFIX_ADDRESS, 16, 4,
1578	desc, desc_bytes(desc), 1);
1579	}
1580	EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
1581
1582	/**
1583	* cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor
1584	* @desc: pointer to buffer used for descriptor construction
1585	* @cdata: pointer to block cipher transform definitions
1586	* Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1587	*/
1588	void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
1589	{
1590	/*
1591	* Set sector size to a big value, practically disabling
1592	* sector size segmentation in xts implementation. We cannot
1593	* take full advantage of this HW feature with existing
1594	* crypto API / dm-crypt SW architecture.
1595	*/
1596	__be64 sector_size = cpu_to_be64(BIT(15));
1597	u32 *key_jump_cmd;
1598
1599	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1600	/* Skip if already shared */
1601	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1602	JUMP_COND_SHRD);
1603
1604	/* Load class1 key only */
1605	append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen,
1606	len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1607
1608	/* Load sector size with index 40 bytes (0x28) */
1609	append_load_as_imm(desc, data: (void *)&sector_size, len: 8, LDST_CLASS_1_CCB |
1610	LDST_SRCDST_BYTE_CONTEXT |
1611	(0x28 << LDST_OFFSET_SHIFT));
1612
1613	set_jump_tgt_here(desc, jump_cmd: key_jump_cmd);
1614
1615	/*
1616	* create sequence for loading the sector index / 16B tweak value
1617	* Lower 8B of IV - sector index / tweak lower half
1618	* Upper 8B of IV - upper half of 16B tweak
1619	*/
1620	append_seq_load(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1621	(0x20 << LDST_OFFSET_SHIFT));
1622	append_seq_load(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1623	(0x30 << LDST_OFFSET_SHIFT));
1624	/* Load operation */
1625	append_dec_op1(desc, type: cdata->algtype);
1626
1627	/* Perform operation */
1628	skcipher_append_src_dst(desc);
1629
1630	/* Store lower 8B and upper 8B of IV */
1631	append_seq_store(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1632	(0x20 << LDST_OFFSET_SHIFT));
1633	append_seq_store(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1634	(0x30 << LDST_OFFSET_SHIFT));
1635
1636	print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
1637	": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
1638	desc_bytes(desc), 1);
1639	}
1640	EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
1641
1642	MODULE_LICENSE("GPL");
1643	MODULE_DESCRIPTION("FSL CAAM descriptor support");
1644	MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
1645