1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* AMD Cryptographic Coprocessor (CCP) driver
4	*
5	* Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
6	*
7	* Author: Tom Lendacky <thomas.lendacky@amd.com>
8	* Author: Gary R Hook <gary.hook@amd.com>
9	*/
10
11	#include <linux/dma-mapping.h>
12	#include <linux/module.h>
13	#include <linux/kernel.h>
14	#include <linux/interrupt.h>
15	#include <crypto/scatterwalk.h>
16	#include <crypto/des.h>
17	#include <linux/ccp.h>
18
19	#include "ccp-dev.h"
20
21	/* SHA initial context values */
22	static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
23	cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
24	cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
25	cpu_to_be32(SHA1_H4),
26	};
27
28	static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
29	cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
30	cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
31	cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
32	cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
33	};
34
35	static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
36	cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
37	cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
38	cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
39	cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
40	};
41
42	static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
43	cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
44	cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
45	cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
46	cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
47	};
48
49	static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
50	cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
51	cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
52	cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
53	cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
54	};
55
56	#define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
57	ccp_gen_jobid(ccp) : 0)
58
59	static u32 ccp_gen_jobid(struct ccp_device *ccp)
60	{
61	return atomic_inc_return(v: &ccp->current_id) & CCP_JOBID_MASK;
62	}
63
64	static void ccp_sg_free(struct ccp_sg_workarea *wa)
65	{
66	if (wa->dma_count)
67	dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
68
69	wa->dma_count = 0;
70	}
71
72	static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
73	struct scatterlist *sg, u64 len,
74	enum dma_data_direction dma_dir)
75	{
76	memset(wa, 0, sizeof(*wa));
77
78	wa->sg = sg;
79	if (!sg)
80	return 0;
81
82	wa->nents = sg_nents_for_len(sg, len);
83	if (wa->nents < 0)
84	return wa->nents;
85
86	wa->bytes_left = len;
87	wa->sg_used = 0;
88
89	if (len == 0)
90	return 0;
91
92	if (dma_dir == DMA_NONE)
93	return 0;
94
95	wa->dma_sg = sg;
96	wa->dma_sg_head = sg;
97	wa->dma_dev = dev;
98	wa->dma_dir = dma_dir;
99	wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
100	if (!wa->dma_count)
101	return -ENOMEM;
102
103	return 0;
104	}
105
106	static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
107	{
108	unsigned int nbytes = min_t(u64, len, wa->bytes_left);
109	unsigned int sg_combined_len = 0;
110
111	if (!wa->sg)
112	return;
113
114	wa->sg_used += nbytes;
115	wa->bytes_left -= nbytes;
116	if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
117	/* Advance to the next DMA scatterlist entry */
118	wa->dma_sg = sg_next(wa->dma_sg);
119
120	/* In the case that the DMA mapped scatterlist has entries
121	* that have been merged, the non-DMA mapped scatterlist
122	* must be advanced multiple times for each merged entry.
123	* This ensures that the current non-DMA mapped entry
124	* corresponds to the current DMA mapped entry.
125	*/
126	do {
127	sg_combined_len += wa->sg->length;
128	wa->sg = sg_next(wa->sg);
129	} while (wa->sg_used > sg_combined_len);
130
131	wa->sg_used = 0;
132	}
133	}
134
135	static void ccp_dm_free(struct ccp_dm_workarea *wa)
136	{
137	if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
138	if (wa->address)
139	dma_pool_free(pool: wa->dma_pool, vaddr: wa->address,
140	addr: wa->dma.address);
141	} else {
142	if (wa->dma.address)
143	dma_unmap_single(wa->dev, wa->dma.address, wa->length,
144	wa->dma.dir);
145	kfree(objp: wa->address);
146	}
147
148	wa->address = NULL;
149	wa->dma.address = 0;
150	}
151
152	static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
153	struct ccp_cmd_queue *cmd_q,
154	unsigned int len,
155	enum dma_data_direction dir)
156	{
157	memset(wa, 0, sizeof(*wa));
158
159	if (!len)
160	return 0;
161
162	wa->dev = cmd_q->ccp->dev;
163	wa->length = len;
164
165	if (len <= CCP_DMAPOOL_MAX_SIZE) {
166	wa->dma_pool = cmd_q->dma_pool;
167
168	wa->address = dma_pool_zalloc(pool: wa->dma_pool, GFP_KERNEL,
169	handle: &wa->dma.address);
170	if (!wa->address)
171	return -ENOMEM;
172
173	wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
174
175	} else {
176	wa->address = kzalloc(size: len, GFP_KERNEL);
177	if (!wa->address)
178	return -ENOMEM;
179
180	wa->dma.address = dma_map_single(wa->dev, wa->address, len,
181	dir);
182	if (dma_mapping_error(dev: wa->dev, dma_addr: wa->dma.address))
183	return -ENOMEM;
184
185	wa->dma.length = len;
186	}
187	wa->dma.dir = dir;
188
189	return 0;
190	}
191
192	static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
193	struct scatterlist *sg, unsigned int sg_offset,
194	unsigned int len)
195	{
196	WARN_ON(!wa->address);
197
198	if (len > (wa->length - wa_offset))
199	return -EINVAL;
200
201	scatterwalk_map_and_copy(buf: wa->address + wa_offset, sg, start: sg_offset, nbytes: len,
202	out: 0);
203	return 0;
204	}
205
206	static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
207	struct scatterlist *sg, unsigned int sg_offset,
208	unsigned int len)
209	{
210	WARN_ON(!wa->address);
211
212	scatterwalk_map_and_copy(buf: wa->address + wa_offset, sg, start: sg_offset, nbytes: len,
213	out: 1);
214	}
215
216	static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
217	unsigned int wa_offset,
218	struct scatterlist *sg,
219	unsigned int sg_offset,
220	unsigned int len)
221	{
222	u8 *p, *q;
223	int rc;
224
225	rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
226	if (rc)
227	return rc;
228
229	p = wa->address + wa_offset;
230	q = p + len - 1;
231	while (p < q) {
232	*p = *p ^ *q;
233	*q = *p ^ *q;
234	*p = *p ^ *q;
235	p++;
236	q--;
237	}
238	return 0;
239	}
240
241	static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
242	unsigned int wa_offset,
243	struct scatterlist *sg,
244	unsigned int sg_offset,
245	unsigned int len)
246	{
247	u8 *p, *q;
248
249	p = wa->address + wa_offset;
250	q = p + len - 1;
251	while (p < q) {
252	*p = *p ^ *q;
253	*q = *p ^ *q;
254	*p = *p ^ *q;
255	p++;
256	q--;
257	}
258
259	ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
260	}
261
262	static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
263	{
264	ccp_dm_free(wa: &data->dm_wa);
265	ccp_sg_free(wa: &data->sg_wa);
266	}
267
268	static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
269	struct scatterlist *sg, u64 sg_len,
270	unsigned int dm_len,
271	enum dma_data_direction dir)
272	{
273	int ret;
274
275	memset(data, 0, sizeof(*data));
276
277	ret = ccp_init_sg_workarea(wa: &data->sg_wa, dev: cmd_q->ccp->dev, sg, len: sg_len,
278	dma_dir: dir);
279	if (ret)
280	goto e_err;
281
282	ret = ccp_init_dm_workarea(wa: &data->dm_wa, cmd_q, len: dm_len, dir);
283	if (ret)
284	goto e_err;
285
286	return 0;
287
288	e_err:
289	ccp_free_data(data, cmd_q);
290
291	return ret;
292	}
293
294	static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
295	{
296	struct ccp_sg_workarea *sg_wa = &data->sg_wa;
297	struct ccp_dm_workarea *dm_wa = &data->dm_wa;
298	unsigned int buf_count, nbytes;
299
300	/* Clear the buffer if setting it */
301	if (!from)
302	memset(dm_wa->address, 0, dm_wa->length);
303
304	if (!sg_wa->sg)
305	return 0;
306
307	/* Perform the copy operation
308	* nbytes will always be <= UINT_MAX because dm_wa->length is
309	* an unsigned int
310	*/
311	nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
312	scatterwalk_map_and_copy(buf: dm_wa->address, sg: sg_wa->sg, start: sg_wa->sg_used,
313	nbytes, out: from);
314
315	/* Update the structures and generate the count */
316	buf_count = 0;
317	while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
318	nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
319	dm_wa->length - buf_count);
320	nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
321
322	buf_count += nbytes;
323	ccp_update_sg_workarea(wa: sg_wa, len: nbytes);
324	}
325
326	return buf_count;
327	}
328
329	static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
330	{
331	return ccp_queue_buf(data, from: 0);
332	}
333
334	static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
335	{
336	return ccp_queue_buf(data, from: 1);
337	}
338
339	static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
340	struct ccp_op *op, unsigned int block_size,
341	bool blocksize_op)
342	{
343	unsigned int sg_src_len, sg_dst_len, op_len;
344
345	/* The CCP can only DMA from/to one address each per operation. This
346	* requires that we find the smallest DMA area between the source
347	* and destination. The resulting len values will always be <= UINT_MAX
348	* because the dma length is an unsigned int.
349	*/
350	sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
351	sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
352
353	if (dst) {
354	sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
355	sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
356	op_len = min(sg_src_len, sg_dst_len);
357	} else {
358	op_len = sg_src_len;
359	}
360
361	/* The data operation length will be at least block_size in length
362	* or the smaller of available sg room remaining for the source or
363	* the destination
364	*/
365	op_len = max(op_len, block_size);
366
367	/* Unless we have to buffer data, there's no reason to wait */
368	op->soc = 0;
369
370	if (sg_src_len < block_size) {
371	/* Not enough data in the sg element, so it
372	* needs to be buffered into a blocksize chunk
373	*/
374	int cp_len = ccp_fill_queue_buf(data: src);
375
376	op->soc = 1;
377	op->src.u.dma.address = src->dm_wa.dma.address;
378	op->src.u.dma.offset = 0;
379	op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
380	} else {
381	/* Enough data in the sg element, but we need to
382	* adjust for any previously copied data
383	*/
384	op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
385	op->src.u.dma.offset = src->sg_wa.sg_used;
386	op->src.u.dma.length = op_len & ~(block_size - 1);
387
388	ccp_update_sg_workarea(wa: &src->sg_wa, len: op->src.u.dma.length);
389	}
390
391	if (dst) {
392	if (sg_dst_len < block_size) {
393	/* Not enough room in the sg element or we're on the
394	* last piece of data (when using padding), so the
395	* output needs to be buffered into a blocksize chunk
396	*/
397	op->soc = 1;
398	op->dst.u.dma.address = dst->dm_wa.dma.address;
399	op->dst.u.dma.offset = 0;
400	op->dst.u.dma.length = op->src.u.dma.length;
401	} else {
402	/* Enough room in the sg element, but we need to
403	* adjust for any previously used area
404	*/
405	op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
406	op->dst.u.dma.offset = dst->sg_wa.sg_used;
407	op->dst.u.dma.length = op->src.u.dma.length;
408	}
409	}
410	}
411
412	static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
413	struct ccp_op *op)
414	{
415	op->init = 0;
416
417	if (dst) {
418	if (op->dst.u.dma.address == dst->dm_wa.dma.address)
419	ccp_empty_queue_buf(data: dst);
420	else
421	ccp_update_sg_workarea(wa: &dst->sg_wa,
422	len: op->dst.u.dma.length);
423	}
424	}
425
426	static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
427	struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
428	u32 byte_swap, bool from)
429	{
430	struct ccp_op op;
431
432	memset(&op, 0, sizeof(op));
433
434	op.cmd_q = cmd_q;
435	op.jobid = jobid;
436	op.eom = 1;
437
438	if (from) {
439	op.soc = 1;
440	op.src.type = CCP_MEMTYPE_SB;
441	op.src.u.sb = sb;
442	op.dst.type = CCP_MEMTYPE_SYSTEM;
443	op.dst.u.dma.address = wa->dma.address;
444	op.dst.u.dma.length = wa->length;
445	} else {
446	op.src.type = CCP_MEMTYPE_SYSTEM;
447	op.src.u.dma.address = wa->dma.address;
448	op.src.u.dma.length = wa->length;
449	op.dst.type = CCP_MEMTYPE_SB;
450	op.dst.u.sb = sb;
451	}
452
453	op.u.passthru.byte_swap = byte_swap;
454
455	return cmd_q->ccp->vdata->perform->passthru(&op);
456	}
457
458	static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
459	struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
460	u32 byte_swap)
461	{
462	return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, from: false);
463	}
464
465	static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
466	struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
467	u32 byte_swap)
468	{
469	return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, from: true);
470	}
471
472	static noinline_for_stack int
473	ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
474	{
475	struct ccp_aes_engine *aes = &cmd->u.aes;
476	struct ccp_dm_workarea key, ctx;
477	struct ccp_data src;
478	struct ccp_op op;
479	unsigned int dm_offset;
480	int ret;
481
482	if (!((aes->key_len == AES_KEYSIZE_128) ||
483	(aes->key_len == AES_KEYSIZE_192) ||
484	(aes->key_len == AES_KEYSIZE_256)))
485	return -EINVAL;
486
487	if (aes->src_len & (AES_BLOCK_SIZE - 1))
488	return -EINVAL;
489
490	if (aes->iv_len != AES_BLOCK_SIZE)
491	return -EINVAL;
492
493	if (!aes->key || !aes->iv || !aes->src)
494	return -EINVAL;
495
496	if (aes->cmac_final) {
497	if (aes->cmac_key_len != AES_BLOCK_SIZE)
498	return -EINVAL;
499
500	if (!aes->cmac_key)
501	return -EINVAL;
502	}
503
504	BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
505	BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
506
507	ret = -EIO;
508	memset(&op, 0, sizeof(op));
509	op.cmd_q = cmd_q;
510	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
511	op.sb_key = cmd_q->sb_key;
512	op.sb_ctx = cmd_q->sb_ctx;
513	op.init = 1;
514	op.u.aes.type = aes->type;
515	op.u.aes.mode = aes->mode;
516	op.u.aes.action = aes->action;
517
518	/* All supported key sizes fit in a single (32-byte) SB entry
519	* and must be in little endian format. Use the 256-bit byte
520	* swap passthru option to convert from big endian to little
521	* endian.
522	*/
523	ret = ccp_init_dm_workarea(wa: &key, cmd_q,
524	CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
525	dir: DMA_TO_DEVICE);
526	if (ret)
527	return ret;
528
529	dm_offset = CCP_SB_BYTES - aes->key_len;
530	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset, sg: aes->key, sg_offset: 0, len: aes->key_len);
531	if (ret)
532	goto e_key;
533	ret = ccp_copy_to_sb(cmd_q, wa: &key, jobid: op.jobid, sb: op.sb_key,
534	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
535	if (ret) {
536	cmd->engine_error = cmd_q->cmd_error;
537	goto e_key;
538	}
539
540	/* The AES context fits in a single (32-byte) SB entry and
541	* must be in little endian format. Use the 256-bit byte swap
542	* passthru option to convert from big endian to little endian.
543	*/
544	ret = ccp_init_dm_workarea(wa: &ctx, cmd_q,
545	CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
546	dir: DMA_BIDIRECTIONAL);
547	if (ret)
548	goto e_key;
549
550	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
551	ret = ccp_set_dm_area(wa: &ctx, wa_offset: dm_offset, sg: aes->iv, sg_offset: 0, len: aes->iv_len);
552	if (ret)
553	goto e_ctx;
554	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
555	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
556	if (ret) {
557	cmd->engine_error = cmd_q->cmd_error;
558	goto e_ctx;
559	}
560
561	/* Send data to the CCP AES engine */
562	ret = ccp_init_data(data: &src, cmd_q, sg: aes->src, sg_len: aes->src_len,
563	AES_BLOCK_SIZE, dir: DMA_TO_DEVICE);
564	if (ret)
565	goto e_ctx;
566
567	while (src.sg_wa.bytes_left) {
568	ccp_prepare_data(src: &src, NULL, op: &op, AES_BLOCK_SIZE, blocksize_op: true);
569	if (aes->cmac_final && !src.sg_wa.bytes_left) {
570	op.eom = 1;
571
572	/* Push the K1/K2 key to the CCP now */
573	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid,
574	sb: op.sb_ctx,
575	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
576	if (ret) {
577	cmd->engine_error = cmd_q->cmd_error;
578	goto e_src;
579	}
580
581	ret = ccp_set_dm_area(wa: &ctx, wa_offset: 0, sg: aes->cmac_key, sg_offset: 0,
582	len: aes->cmac_key_len);
583	if (ret)
584	goto e_src;
585	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
586	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
587	if (ret) {
588	cmd->engine_error = cmd_q->cmd_error;
589	goto e_src;
590	}
591	}
592
593	ret = cmd_q->ccp->vdata->perform->aes(&op);
594	if (ret) {
595	cmd->engine_error = cmd_q->cmd_error;
596	goto e_src;
597	}
598
599	ccp_process_data(src: &src, NULL, op: &op);
600	}
601
602	/* Retrieve the AES context - convert from LE to BE using
603	* 32-byte (256-bit) byteswapping
604	*/
605	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
606	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
607	if (ret) {
608	cmd->engine_error = cmd_q->cmd_error;
609	goto e_src;
610	}
611
612	/* ...but we only need AES_BLOCK_SIZE bytes */
613	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
614	ccp_get_dm_area(wa: &ctx, wa_offset: dm_offset, sg: aes->iv, sg_offset: 0, len: aes->iv_len);
615
616	e_src:
617	ccp_free_data(data: &src, cmd_q);
618
619	e_ctx:
620	ccp_dm_free(wa: &ctx);
621
622	e_key:
623	ccp_dm_free(wa: &key);
624
625	return ret;
626	}
627
628	static noinline_for_stack int
629	ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
630	{
631	struct ccp_aes_engine *aes = &cmd->u.aes;
632	struct ccp_dm_workarea key, ctx, final_wa, tag;
633	struct ccp_data src, dst;
634	struct ccp_data aad;
635	struct ccp_op op;
636	unsigned int dm_offset;
637	unsigned int authsize;
638	unsigned int jobid;
639	unsigned int ilen;
640	bool in_place = true; /* Default value */
641	__be64 *final;
642	int ret;
643
644	struct scatterlist *p_inp, sg_inp[2];
645	struct scatterlist *p_tag, sg_tag[2];
646	struct scatterlist *p_outp, sg_outp[2];
647	struct scatterlist *p_aad;
648
649	if (!aes->iv)
650	return -EINVAL;
651
652	if (!((aes->key_len == AES_KEYSIZE_128) ||
653	(aes->key_len == AES_KEYSIZE_192) ||
654	(aes->key_len == AES_KEYSIZE_256)))
655	return -EINVAL;
656
657	if (!aes->key) /* Gotta have a key SGL */
658	return -EINVAL;
659
660	/* Zero defaults to 16 bytes, the maximum size */
661	authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
662	switch (authsize) {
663	case 16:
664	case 15:
665	case 14:
666	case 13:
667	case 12:
668	case 8:
669	case 4:
670	break;
671	default:
672	return -EINVAL;
673	}
674
675	/* First, decompose the source buffer into AAD & PT,
676	* and the destination buffer into AAD, CT & tag, or
677	* the input into CT & tag.
678	* It is expected that the input and output SGs will
679	* be valid, even if the AAD and input lengths are 0.
680	*/
681	p_aad = aes->src;
682	p_inp = scatterwalk_ffwd(dst: sg_inp, src: aes->src, len: aes->aad_len);
683	p_outp = scatterwalk_ffwd(dst: sg_outp, src: aes->dst, len: aes->aad_len);
684	if (aes->action == CCP_AES_ACTION_ENCRYPT) {
685	ilen = aes->src_len;
686	p_tag = scatterwalk_ffwd(dst: sg_tag, src: p_outp, len: ilen);
687	} else {
688	/* Input length for decryption includes tag */
689	ilen = aes->src_len - authsize;
690	p_tag = scatterwalk_ffwd(dst: sg_tag, src: p_inp, len: ilen);
691	}
692
693	jobid = CCP_NEW_JOBID(cmd_q->ccp);
694
695	memset(&op, 0, sizeof(op));
696	op.cmd_q = cmd_q;
697	op.jobid = jobid;
698	op.sb_key = cmd_q->sb_key; /* Pre-allocated */
699	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
700	op.init = 1;
701	op.u.aes.type = aes->type;
702
703	/* Copy the key to the LSB */
704	ret = ccp_init_dm_workarea(wa: &key, cmd_q,
705	CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
706	dir: DMA_TO_DEVICE);
707	if (ret)
708	return ret;
709
710	dm_offset = CCP_SB_BYTES - aes->key_len;
711	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset, sg: aes->key, sg_offset: 0, len: aes->key_len);
712	if (ret)
713	goto e_key;
714	ret = ccp_copy_to_sb(cmd_q, wa: &key, jobid: op.jobid, sb: op.sb_key,
715	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
716	if (ret) {
717	cmd->engine_error = cmd_q->cmd_error;
718	goto e_key;
719	}
720
721	/* Copy the context (IV) to the LSB.
722	* There is an assumption here that the IV is 96 bits in length, plus
723	* a nonce of 32 bits. If no IV is present, use a zeroed buffer.
724	*/
725	ret = ccp_init_dm_workarea(wa: &ctx, cmd_q,
726	CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
727	dir: DMA_BIDIRECTIONAL);
728	if (ret)
729	goto e_key;
730
731	dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
732	ret = ccp_set_dm_area(wa: &ctx, wa_offset: dm_offset, sg: aes->iv, sg_offset: 0, len: aes->iv_len);
733	if (ret)
734	goto e_ctx;
735
736	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
737	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
738	if (ret) {
739	cmd->engine_error = cmd_q->cmd_error;
740	goto e_ctx;
741	}
742
743	op.init = 1;
744	if (aes->aad_len > 0) {
745	/* Step 1: Run a GHASH over the Additional Authenticated Data */
746	ret = ccp_init_data(data: &aad, cmd_q, sg: p_aad, sg_len: aes->aad_len,
747	AES_BLOCK_SIZE,
748	dir: DMA_TO_DEVICE);
749	if (ret)
750	goto e_ctx;
751
752	op.u.aes.mode = CCP_AES_MODE_GHASH;
753	op.u.aes.action = CCP_AES_GHASHAAD;
754
755	while (aad.sg_wa.bytes_left) {
756	ccp_prepare_data(src: &aad, NULL, op: &op, AES_BLOCK_SIZE, blocksize_op: true);
757
758	ret = cmd_q->ccp->vdata->perform->aes(&op);
759	if (ret) {
760	cmd->engine_error = cmd_q->cmd_error;
761	goto e_aad;
762	}
763
764	ccp_process_data(src: &aad, NULL, op: &op);
765	op.init = 0;
766	}
767	}
768
769	op.u.aes.mode = CCP_AES_MODE_GCTR;
770	op.u.aes.action = aes->action;
771
772	if (ilen > 0) {
773	/* Step 2: Run a GCTR over the plaintext */
774	in_place = (sg_virt(sg: p_inp) == sg_virt(sg: p_outp)) ? true : false;
775
776	ret = ccp_init_data(data: &src, cmd_q, sg: p_inp, sg_len: ilen,
777	AES_BLOCK_SIZE,
778	dir: in_place ? DMA_BIDIRECTIONAL
779	: DMA_TO_DEVICE);
780	if (ret)
781	goto e_aad;
782
783	if (in_place) {
784	dst = src;
785	} else {
786	ret = ccp_init_data(data: &dst, cmd_q, sg: p_outp, sg_len: ilen,
787	AES_BLOCK_SIZE, dir: DMA_FROM_DEVICE);
788	if (ret)
789	goto e_src;
790	}
791
792	op.soc = 0;
793	op.eom = 0;
794	op.init = 1;
795	while (src.sg_wa.bytes_left) {
796	ccp_prepare_data(src: &src, dst: &dst, op: &op, AES_BLOCK_SIZE, blocksize_op: true);
797	if (!src.sg_wa.bytes_left) {
798	unsigned int nbytes = ilen % AES_BLOCK_SIZE;
799
800	if (nbytes) {
801	op.eom = 1;
802	op.u.aes.size = (nbytes * 8) - 1;
803	}
804	}
805
806	ret = cmd_q->ccp->vdata->perform->aes(&op);
807	if (ret) {
808	cmd->engine_error = cmd_q->cmd_error;
809	goto e_dst;
810	}
811
812	ccp_process_data(src: &src, dst: &dst, op: &op);
813	op.init = 0;
814	}
815	}
816
817	/* Step 3: Update the IV portion of the context with the original IV */
818	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
819	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
820	if (ret) {
821	cmd->engine_error = cmd_q->cmd_error;
822	goto e_dst;
823	}
824
825	ret = ccp_set_dm_area(wa: &ctx, wa_offset: dm_offset, sg: aes->iv, sg_offset: 0, len: aes->iv_len);
826	if (ret)
827	goto e_dst;
828
829	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
830	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
831	if (ret) {
832	cmd->engine_error = cmd_q->cmd_error;
833	goto e_dst;
834	}
835
836	/* Step 4: Concatenate the lengths of the AAD and source, and
837	* hash that 16 byte buffer.
838	*/
839	ret = ccp_init_dm_workarea(wa: &final_wa, cmd_q, AES_BLOCK_SIZE,
840	dir: DMA_BIDIRECTIONAL);
841	if (ret)
842	goto e_dst;
843	final = (__be64 *)final_wa.address;
844	final[0] = cpu_to_be64(aes->aad_len * 8);
845	final[1] = cpu_to_be64(ilen * 8);
846
847	memset(&op, 0, sizeof(op));
848	op.cmd_q = cmd_q;
849	op.jobid = jobid;
850	op.sb_key = cmd_q->sb_key; /* Pre-allocated */
851	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
852	op.init = 1;
853	op.u.aes.type = aes->type;
854	op.u.aes.mode = CCP_AES_MODE_GHASH;
855	op.u.aes.action = CCP_AES_GHASHFINAL;
856	op.src.type = CCP_MEMTYPE_SYSTEM;
857	op.src.u.dma.address = final_wa.dma.address;
858	op.src.u.dma.length = AES_BLOCK_SIZE;
859	op.dst.type = CCP_MEMTYPE_SYSTEM;
860	op.dst.u.dma.address = final_wa.dma.address;
861	op.dst.u.dma.length = AES_BLOCK_SIZE;
862	op.eom = 1;
863	op.u.aes.size = 0;
864	ret = cmd_q->ccp->vdata->perform->aes(&op);
865	if (ret)
866	goto e_final_wa;
867
868	if (aes->action == CCP_AES_ACTION_ENCRYPT) {
869	/* Put the ciphered tag after the ciphertext. */
870	ccp_get_dm_area(wa: &final_wa, wa_offset: 0, sg: p_tag, sg_offset: 0, len: authsize);
871	} else {
872	/* Does this ciphered tag match the input? */
873	ret = ccp_init_dm_workarea(wa: &tag, cmd_q, len: authsize,
874	dir: DMA_BIDIRECTIONAL);
875	if (ret)
876	goto e_final_wa;
877	ret = ccp_set_dm_area(wa: &tag, wa_offset: 0, sg: p_tag, sg_offset: 0, len: authsize);
878	if (ret) {
879	ccp_dm_free(wa: &tag);
880	goto e_final_wa;
881	}
882
883	ret = crypto_memneq(a: tag.address, b: final_wa.address,
884	size: authsize) ? -EBADMSG : 0;
885	ccp_dm_free(wa: &tag);
886	}
887
888	e_final_wa:
889	ccp_dm_free(wa: &final_wa);
890
891	e_dst:
892	if (ilen > 0 && !in_place)
893	ccp_free_data(data: &dst, cmd_q);
894
895	e_src:
896	if (ilen > 0)
897	ccp_free_data(data: &src, cmd_q);
898
899	e_aad:
900	if (aes->aad_len)
901	ccp_free_data(data: &aad, cmd_q);
902
903	e_ctx:
904	ccp_dm_free(wa: &ctx);
905
906	e_key:
907	ccp_dm_free(wa: &key);
908
909	return ret;
910	}
911
912	static noinline_for_stack int
913	ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
914	{
915	struct ccp_aes_engine *aes = &cmd->u.aes;
916	struct ccp_dm_workarea key, ctx;
917	struct ccp_data src, dst;
918	struct ccp_op op;
919	unsigned int dm_offset;
920	bool in_place = false;
921	int ret;
922
923	if (!((aes->key_len == AES_KEYSIZE_128) ||
924	(aes->key_len == AES_KEYSIZE_192) ||
925	(aes->key_len == AES_KEYSIZE_256)))
926	return -EINVAL;
927
928	if (((aes->mode == CCP_AES_MODE_ECB) ||
929	(aes->mode == CCP_AES_MODE_CBC)) &&
930	(aes->src_len & (AES_BLOCK_SIZE - 1)))
931	return -EINVAL;
932
933	if (!aes->key || !aes->src || !aes->dst)
934	return -EINVAL;
935
936	if (aes->mode != CCP_AES_MODE_ECB) {
937	if (aes->iv_len != AES_BLOCK_SIZE)
938	return -EINVAL;
939
940	if (!aes->iv)
941	return -EINVAL;
942	}
943
944	BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
945	BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
946
947	ret = -EIO;
948	memset(&op, 0, sizeof(op));
949	op.cmd_q = cmd_q;
950	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
951	op.sb_key = cmd_q->sb_key;
952	op.sb_ctx = cmd_q->sb_ctx;
953	op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
954	op.u.aes.type = aes->type;
955	op.u.aes.mode = aes->mode;
956	op.u.aes.action = aes->action;
957
958	/* All supported key sizes fit in a single (32-byte) SB entry
959	* and must be in little endian format. Use the 256-bit byte
960	* swap passthru option to convert from big endian to little
961	* endian.
962	*/
963	ret = ccp_init_dm_workarea(wa: &key, cmd_q,
964	CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
965	dir: DMA_TO_DEVICE);
966	if (ret)
967	return ret;
968
969	dm_offset = CCP_SB_BYTES - aes->key_len;
970	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset, sg: aes->key, sg_offset: 0, len: aes->key_len);
971	if (ret)
972	goto e_key;
973	ret = ccp_copy_to_sb(cmd_q, wa: &key, jobid: op.jobid, sb: op.sb_key,
974	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
975	if (ret) {
976	cmd->engine_error = cmd_q->cmd_error;
977	goto e_key;
978	}
979
980	/* The AES context fits in a single (32-byte) SB entry and
981	* must be in little endian format. Use the 256-bit byte swap
982	* passthru option to convert from big endian to little endian.
983	*/
984	ret = ccp_init_dm_workarea(wa: &ctx, cmd_q,
985	CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
986	dir: DMA_BIDIRECTIONAL);
987	if (ret)
988	goto e_key;
989
990	if (aes->mode != CCP_AES_MODE_ECB) {
991	/* Load the AES context - convert to LE */
992	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
993	ret = ccp_set_dm_area(wa: &ctx, wa_offset: dm_offset, sg: aes->iv, sg_offset: 0, len: aes->iv_len);
994	if (ret)
995	goto e_ctx;
996	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
997	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
998	if (ret) {
999	cmd->engine_error = cmd_q->cmd_error;
1000	goto e_ctx;
1001	}
1002	}
1003	switch (aes->mode) {
1004	case CCP_AES_MODE_CFB: /* CFB128 only */
1005	case CCP_AES_MODE_CTR:
1006	op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
1007	break;
1008	default:
1009	op.u.aes.size = 0;
1010	}
1011
1012	/* Prepare the input and output data workareas. For in-place
1013	* operations we need to set the dma direction to BIDIRECTIONAL
1014	* and copy the src workarea to the dst workarea.
1015	*/
1016	if (sg_virt(sg: aes->src) == sg_virt(sg: aes->dst))
1017	in_place = true;
1018
1019	ret = ccp_init_data(data: &src, cmd_q, sg: aes->src, sg_len: aes->src_len,
1020	AES_BLOCK_SIZE,
1021	dir: in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1022	if (ret)
1023	goto e_ctx;
1024
1025	if (in_place) {
1026	dst = src;
1027	} else {
1028	ret = ccp_init_data(data: &dst, cmd_q, sg: aes->dst, sg_len: aes->src_len,
1029	AES_BLOCK_SIZE, dir: DMA_FROM_DEVICE);
1030	if (ret)
1031	goto e_src;
1032	}
1033
1034	/* Send data to the CCP AES engine */
1035	while (src.sg_wa.bytes_left) {
1036	ccp_prepare_data(src: &src, dst: &dst, op: &op, AES_BLOCK_SIZE, blocksize_op: true);
1037	if (!src.sg_wa.bytes_left) {
1038	op.eom = 1;
1039
1040	/* Since we don't retrieve the AES context in ECB
1041	* mode we have to wait for the operation to complete
1042	* on the last piece of data
1043	*/
1044	if (aes->mode == CCP_AES_MODE_ECB)
1045	op.soc = 1;
1046	}
1047
1048	ret = cmd_q->ccp->vdata->perform->aes(&op);
1049	if (ret) {
1050	cmd->engine_error = cmd_q->cmd_error;
1051	goto e_dst;
1052	}
1053
1054	ccp_process_data(src: &src, dst: &dst, op: &op);
1055	}
1056
1057	if (aes->mode != CCP_AES_MODE_ECB) {
1058	/* Retrieve the AES context - convert from LE to BE using
1059	* 32-byte (256-bit) byteswapping
1060	*/
1061	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1062	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1063	if (ret) {
1064	cmd->engine_error = cmd_q->cmd_error;
1065	goto e_dst;
1066	}
1067
1068	/* ...but we only need AES_BLOCK_SIZE bytes */
1069	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1070	ccp_get_dm_area(wa: &ctx, wa_offset: dm_offset, sg: aes->iv, sg_offset: 0, len: aes->iv_len);
1071	}
1072
1073	e_dst:
1074	if (!in_place)
1075	ccp_free_data(data: &dst, cmd_q);
1076
1077	e_src:
1078	ccp_free_data(data: &src, cmd_q);
1079
1080	e_ctx:
1081	ccp_dm_free(wa: &ctx);
1082
1083	e_key:
1084	ccp_dm_free(wa: &key);
1085
1086	return ret;
1087	}
1088
1089	static noinline_for_stack int
1090	ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1091	{
1092	struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1093	struct ccp_dm_workarea key, ctx;
1094	struct ccp_data src, dst;
1095	struct ccp_op op;
1096	unsigned int unit_size, dm_offset;
1097	bool in_place = false;
1098	unsigned int sb_count;
1099	enum ccp_aes_type aestype;
1100	int ret;
1101
1102	switch (xts->unit_size) {
1103	case CCP_XTS_AES_UNIT_SIZE_16:
1104	unit_size = 16;
1105	break;
1106	case CCP_XTS_AES_UNIT_SIZE_512:
1107	unit_size = 512;
1108	break;
1109	case CCP_XTS_AES_UNIT_SIZE_1024:
1110	unit_size = 1024;
1111	break;
1112	case CCP_XTS_AES_UNIT_SIZE_2048:
1113	unit_size = 2048;
1114	break;
1115	case CCP_XTS_AES_UNIT_SIZE_4096:
1116	unit_size = 4096;
1117	break;
1118
1119	default:
1120	return -EINVAL;
1121	}
1122
1123	if (xts->key_len == AES_KEYSIZE_128)
1124	aestype = CCP_AES_TYPE_128;
1125	else if (xts->key_len == AES_KEYSIZE_256)
1126	aestype = CCP_AES_TYPE_256;
1127	else
1128	return -EINVAL;
1129
1130	if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1131	return -EINVAL;
1132
1133	if (xts->iv_len != AES_BLOCK_SIZE)
1134	return -EINVAL;
1135
1136	if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1137	return -EINVAL;
1138
1139	BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1140	BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1141
1142	ret = -EIO;
1143	memset(&op, 0, sizeof(op));
1144	op.cmd_q = cmd_q;
1145	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1146	op.sb_key = cmd_q->sb_key;
1147	op.sb_ctx = cmd_q->sb_ctx;
1148	op.init = 1;
1149	op.u.xts.type = aestype;
1150	op.u.xts.action = xts->action;
1151	op.u.xts.unit_size = xts->unit_size;
1152
1153	/* A version 3 device only supports 128-bit keys, which fits into a
1154	* single SB entry. A version 5 device uses a 512-bit vector, so two
1155	* SB entries.
1156	*/
1157	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1158	sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1159	else
1160	sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1161	ret = ccp_init_dm_workarea(wa: &key, cmd_q,
1162	len: sb_count * CCP_SB_BYTES,
1163	dir: DMA_TO_DEVICE);
1164	if (ret)
1165	return ret;
1166
1167	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1168	/* All supported key sizes must be in little endian format.
1169	* Use the 256-bit byte swap passthru option to convert from
1170	* big endian to little endian.
1171	*/
1172	dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1173	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset, sg: xts->key, sg_offset: 0, len: xts->key_len);
1174	if (ret)
1175	goto e_key;
1176	ret = ccp_set_dm_area(wa: &key, wa_offset: 0, sg: xts->key, sg_offset: xts->key_len, len: xts->key_len);
1177	if (ret)
1178	goto e_key;
1179	} else {
1180	/* Version 5 CCPs use a 512-bit space for the key: each portion
1181	* occupies 256 bits, or one entire slot, and is zero-padded.
1182	*/
1183	unsigned int pad;
1184
1185	dm_offset = CCP_SB_BYTES;
1186	pad = dm_offset - xts->key_len;
1187	ret = ccp_set_dm_area(wa: &key, wa_offset: pad, sg: xts->key, sg_offset: 0, len: xts->key_len);
1188	if (ret)
1189	goto e_key;
1190	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset + pad, sg: xts->key,
1191	sg_offset: xts->key_len, len: xts->key_len);
1192	if (ret)
1193	goto e_key;
1194	}
1195	ret = ccp_copy_to_sb(cmd_q, wa: &key, jobid: op.jobid, sb: op.sb_key,
1196	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1197	if (ret) {
1198	cmd->engine_error = cmd_q->cmd_error;
1199	goto e_key;
1200	}
1201
1202	/* The AES context fits in a single (32-byte) SB entry and
1203	* for XTS is already in little endian format so no byte swapping
1204	* is needed.
1205	*/
1206	ret = ccp_init_dm_workarea(wa: &ctx, cmd_q,
1207	CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1208	dir: DMA_BIDIRECTIONAL);
1209	if (ret)
1210	goto e_key;
1211
1212	ret = ccp_set_dm_area(wa: &ctx, wa_offset: 0, sg: xts->iv, sg_offset: 0, len: xts->iv_len);
1213	if (ret)
1214	goto e_ctx;
1215	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1216	byte_swap: CCP_PASSTHRU_BYTESWAP_NOOP);
1217	if (ret) {
1218	cmd->engine_error = cmd_q->cmd_error;
1219	goto e_ctx;
1220	}
1221
1222	/* Prepare the input and output data workareas. For in-place
1223	* operations we need to set the dma direction to BIDIRECTIONAL
1224	* and copy the src workarea to the dst workarea.
1225	*/
1226	if (sg_virt(sg: xts->src) == sg_virt(sg: xts->dst))
1227	in_place = true;
1228
1229	ret = ccp_init_data(data: &src, cmd_q, sg: xts->src, sg_len: xts->src_len,
1230	dm_len: unit_size,
1231	dir: in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1232	if (ret)
1233	goto e_ctx;
1234
1235	if (in_place) {
1236	dst = src;
1237	} else {
1238	ret = ccp_init_data(data: &dst, cmd_q, sg: xts->dst, sg_len: xts->src_len,
1239	dm_len: unit_size, dir: DMA_FROM_DEVICE);
1240	if (ret)
1241	goto e_src;
1242	}
1243
1244	/* Send data to the CCP AES engine */
1245	while (src.sg_wa.bytes_left) {
1246	ccp_prepare_data(src: &src, dst: &dst, op: &op, block_size: unit_size, blocksize_op: true);
1247	if (!src.sg_wa.bytes_left)
1248	op.eom = 1;
1249
1250	ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1251	if (ret) {
1252	cmd->engine_error = cmd_q->cmd_error;
1253	goto e_dst;
1254	}
1255
1256	ccp_process_data(src: &src, dst: &dst, op: &op);
1257	}
1258
1259	/* Retrieve the AES context - convert from LE to BE using
1260	* 32-byte (256-bit) byteswapping
1261	*/
1262	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1263	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1264	if (ret) {
1265	cmd->engine_error = cmd_q->cmd_error;
1266	goto e_dst;
1267	}
1268
1269	/* ...but we only need AES_BLOCK_SIZE bytes */
1270	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1271	ccp_get_dm_area(wa: &ctx, wa_offset: dm_offset, sg: xts->iv, sg_offset: 0, len: xts->iv_len);
1272
1273	e_dst:
1274	if (!in_place)
1275	ccp_free_data(data: &dst, cmd_q);
1276
1277	e_src:
1278	ccp_free_data(data: &src, cmd_q);
1279
1280	e_ctx:
1281	ccp_dm_free(wa: &ctx);
1282
1283	e_key:
1284	ccp_dm_free(wa: &key);
1285
1286	return ret;
1287	}
1288
1289	static noinline_for_stack int
1290	ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1291	{
1292	struct ccp_des3_engine *des3 = &cmd->u.des3;
1293
1294	struct ccp_dm_workarea key, ctx;
1295	struct ccp_data src, dst;
1296	struct ccp_op op;
1297	unsigned int dm_offset;
1298	unsigned int len_singlekey;
1299	bool in_place = false;
1300	int ret;
1301
1302	/* Error checks */
1303	if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1304	return -EINVAL;
1305
1306	if (!cmd_q->ccp->vdata->perform->des3)
1307	return -EINVAL;
1308
1309	if (des3->key_len != DES3_EDE_KEY_SIZE)
1310	return -EINVAL;
1311
1312	if (((des3->mode == CCP_DES3_MODE_ECB) ||
1313	(des3->mode == CCP_DES3_MODE_CBC)) &&
1314	(des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1315	return -EINVAL;
1316
1317	if (!des3->key || !des3->src || !des3->dst)
1318	return -EINVAL;
1319
1320	if (des3->mode != CCP_DES3_MODE_ECB) {
1321	if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1322	return -EINVAL;
1323
1324	if (!des3->iv)
1325	return -EINVAL;
1326	}
1327
1328	/* Zero out all the fields of the command desc */
1329	memset(&op, 0, sizeof(op));
1330
1331	/* Set up the Function field */
1332	op.cmd_q = cmd_q;
1333	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1334	op.sb_key = cmd_q->sb_key;
1335
1336	op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1337	op.u.des3.type = des3->type;
1338	op.u.des3.mode = des3->mode;
1339	op.u.des3.action = des3->action;
1340
1341	/*
1342	* All supported key sizes fit in a single (32-byte) KSB entry and
1343	* (like AES) must be in little endian format. Use the 256-bit byte
1344	* swap passthru option to convert from big endian to little endian.
1345	*/
1346	ret = ccp_init_dm_workarea(wa: &key, cmd_q,
1347	CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1348	dir: DMA_TO_DEVICE);
1349	if (ret)
1350	return ret;
1351
1352	/*
1353	* The contents of the key triplet are in the reverse order of what
1354	* is required by the engine. Copy the 3 pieces individually to put
1355	* them where they belong.
1356	*/
1357	dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1358
1359	len_singlekey = des3->key_len / 3;
1360	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset + 2 * len_singlekey,
1361	sg: des3->key, sg_offset: 0, len: len_singlekey);
1362	if (ret)
1363	goto e_key;
1364	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset + len_singlekey,
1365	sg: des3->key, sg_offset: len_singlekey, len: len_singlekey);
1366	if (ret)
1367	goto e_key;
1368	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset,
1369	sg: des3->key, sg_offset: 2 * len_singlekey, len: len_singlekey);
1370	if (ret)
1371	goto e_key;
1372
1373	/* Copy the key to the SB */
1374	ret = ccp_copy_to_sb(cmd_q, wa: &key, jobid: op.jobid, sb: op.sb_key,
1375	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1376	if (ret) {
1377	cmd->engine_error = cmd_q->cmd_error;
1378	goto e_key;
1379	}
1380
1381	/*
1382	* The DES3 context fits in a single (32-byte) KSB entry and
1383	* must be in little endian format. Use the 256-bit byte swap
1384	* passthru option to convert from big endian to little endian.
1385	*/
1386	if (des3->mode != CCP_DES3_MODE_ECB) {
1387	op.sb_ctx = cmd_q->sb_ctx;
1388
1389	ret = ccp_init_dm_workarea(wa: &ctx, cmd_q,
1390	CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1391	dir: DMA_BIDIRECTIONAL);
1392	if (ret)
1393	goto e_key;
1394
1395	/* Load the context into the LSB */
1396	dm_offset = CCP_SB_BYTES - des3->iv_len;
1397	ret = ccp_set_dm_area(wa: &ctx, wa_offset: dm_offset, sg: des3->iv, sg_offset: 0,
1398	len: des3->iv_len);
1399	if (ret)
1400	goto e_ctx;
1401
1402	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1403	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1404	if (ret) {
1405	cmd->engine_error = cmd_q->cmd_error;
1406	goto e_ctx;
1407	}
1408	}
1409
1410	/*
1411	* Prepare the input and output data workareas. For in-place
1412	* operations we need to set the dma direction to BIDIRECTIONAL
1413	* and copy the src workarea to the dst workarea.
1414	*/
1415	if (sg_virt(sg: des3->src) == sg_virt(sg: des3->dst))
1416	in_place = true;
1417
1418	ret = ccp_init_data(data: &src, cmd_q, sg: des3->src, sg_len: des3->src_len,
1419	DES3_EDE_BLOCK_SIZE,
1420	dir: in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1421	if (ret)
1422	goto e_ctx;
1423
1424	if (in_place)
1425	dst = src;
1426	else {
1427	ret = ccp_init_data(data: &dst, cmd_q, sg: des3->dst, sg_len: des3->src_len,
1428	DES3_EDE_BLOCK_SIZE, dir: DMA_FROM_DEVICE);
1429	if (ret)
1430	goto e_src;
1431	}
1432
1433	/* Send data to the CCP DES3 engine */
1434	while (src.sg_wa.bytes_left) {
1435	ccp_prepare_data(src: &src, dst: &dst, op: &op, DES3_EDE_BLOCK_SIZE, blocksize_op: true);
1436	if (!src.sg_wa.bytes_left) {
1437	op.eom = 1;
1438
1439	/* Since we don't retrieve the context in ECB mode
1440	* we have to wait for the operation to complete
1441	* on the last piece of data
1442	*/
1443	op.soc = 0;
1444	}
1445
1446	ret = cmd_q->ccp->vdata->perform->des3(&op);
1447	if (ret) {
1448	cmd->engine_error = cmd_q->cmd_error;
1449	goto e_dst;
1450	}
1451
1452	ccp_process_data(src: &src, dst: &dst, op: &op);
1453	}
1454
1455	if (des3->mode != CCP_DES3_MODE_ECB) {
1456	/* Retrieve the context and make BE */
1457	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1458	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1459	if (ret) {
1460	cmd->engine_error = cmd_q->cmd_error;
1461	goto e_dst;
1462	}
1463
1464	/* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1465	ccp_get_dm_area(wa: &ctx, wa_offset: dm_offset, sg: des3->iv, sg_offset: 0,
1466	DES3_EDE_BLOCK_SIZE);
1467	}
1468	e_dst:
1469	if (!in_place)
1470	ccp_free_data(data: &dst, cmd_q);
1471
1472	e_src:
1473	ccp_free_data(data: &src, cmd_q);
1474
1475	e_ctx:
1476	if (des3->mode != CCP_DES3_MODE_ECB)
1477	ccp_dm_free(wa: &ctx);
1478
1479	e_key:
1480	ccp_dm_free(wa: &key);
1481
1482	return ret;
1483	}
1484
1485	static noinline_for_stack int
1486	ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1487	{
1488	struct ccp_sha_engine *sha = &cmd->u.sha;
1489	struct ccp_dm_workarea ctx;
1490	struct ccp_data src;
1491	struct ccp_op op;
1492	unsigned int ioffset, ooffset;
1493	unsigned int digest_size;
1494	int sb_count;
1495	const void *init;
1496	u64 block_size;
1497	int ctx_size;
1498	int ret;
1499
1500	switch (sha->type) {
1501	case CCP_SHA_TYPE_1:
1502	if (sha->ctx_len < SHA1_DIGEST_SIZE)
1503	return -EINVAL;
1504	block_size = SHA1_BLOCK_SIZE;
1505	break;
1506	case CCP_SHA_TYPE_224:
1507	if (sha->ctx_len < SHA224_DIGEST_SIZE)
1508	return -EINVAL;
1509	block_size = SHA224_BLOCK_SIZE;
1510	break;
1511	case CCP_SHA_TYPE_256:
1512	if (sha->ctx_len < SHA256_DIGEST_SIZE)
1513	return -EINVAL;
1514	block_size = SHA256_BLOCK_SIZE;
1515	break;
1516	case CCP_SHA_TYPE_384:
1517	if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1518	|| sha->ctx_len < SHA384_DIGEST_SIZE)
1519	return -EINVAL;
1520	block_size = SHA384_BLOCK_SIZE;
1521	break;
1522	case CCP_SHA_TYPE_512:
1523	if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1524	|| sha->ctx_len < SHA512_DIGEST_SIZE)
1525	return -EINVAL;
1526	block_size = SHA512_BLOCK_SIZE;
1527	break;
1528	default:
1529	return -EINVAL;
1530	}
1531
1532	if (!sha->ctx)
1533	return -EINVAL;
1534
1535	if (!sha->final && (sha->src_len & (block_size - 1)))
1536	return -EINVAL;
1537
1538	/* The version 3 device can't handle zero-length input */
1539	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1540
1541	if (!sha->src_len) {
1542	unsigned int digest_len;
1543	const u8 *sha_zero;
1544
1545	/* Not final, just return */
1546	if (!sha->final)
1547	return 0;
1548
1549	/* CCP can't do a zero length sha operation so the
1550	* caller must buffer the data.
1551	*/
1552	if (sha->msg_bits)
1553	return -EINVAL;
1554
1555	/* The CCP cannot perform zero-length sha operations
1556	* so the caller is required to buffer data for the
1557	* final operation. However, a sha operation for a
1558	* message with a total length of zero is valid so
1559	* known values are required to supply the result.
1560	*/
1561	switch (sha->type) {
1562	case CCP_SHA_TYPE_1:
1563	sha_zero = sha1_zero_message_hash;
1564	digest_len = SHA1_DIGEST_SIZE;
1565	break;
1566	case CCP_SHA_TYPE_224:
1567	sha_zero = sha224_zero_message_hash;
1568	digest_len = SHA224_DIGEST_SIZE;
1569	break;
1570	case CCP_SHA_TYPE_256:
1571	sha_zero = sha256_zero_message_hash;
1572	digest_len = SHA256_DIGEST_SIZE;
1573	break;
1574	default:
1575	return -EINVAL;
1576	}
1577
1578	scatterwalk_map_and_copy(buf: (void *)sha_zero, sg: sha->ctx, start: 0,
1579	nbytes: digest_len, out: 1);
1580
1581	return 0;
1582	}
1583	}
1584
1585	/* Set variables used throughout */
1586	switch (sha->type) {
1587	case CCP_SHA_TYPE_1:
1588	digest_size = SHA1_DIGEST_SIZE;
1589	init = (void *) ccp_sha1_init;
1590	ctx_size = SHA1_DIGEST_SIZE;
1591	sb_count = 1;
1592	if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1593	ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1594	else
1595	ooffset = ioffset = 0;
1596	break;
1597	case CCP_SHA_TYPE_224:
1598	digest_size = SHA224_DIGEST_SIZE;
1599	init = (void *) ccp_sha224_init;
1600	ctx_size = SHA256_DIGEST_SIZE;
1601	sb_count = 1;
1602	ioffset = 0;
1603	if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1604	ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1605	else
1606	ooffset = 0;
1607	break;
1608	case CCP_SHA_TYPE_256:
1609	digest_size = SHA256_DIGEST_SIZE;
1610	init = (void *) ccp_sha256_init;
1611	ctx_size = SHA256_DIGEST_SIZE;
1612	sb_count = 1;
1613	ooffset = ioffset = 0;
1614	break;
1615	case CCP_SHA_TYPE_384:
1616	digest_size = SHA384_DIGEST_SIZE;
1617	init = (void *) ccp_sha384_init;
1618	ctx_size = SHA512_DIGEST_SIZE;
1619	sb_count = 2;
1620	ioffset = 0;
1621	ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1622	break;
1623	case CCP_SHA_TYPE_512:
1624	digest_size = SHA512_DIGEST_SIZE;
1625	init = (void *) ccp_sha512_init;
1626	ctx_size = SHA512_DIGEST_SIZE;
1627	sb_count = 2;
1628	ooffset = ioffset = 0;
1629	break;
1630	default:
1631	ret = -EINVAL;
1632	goto e_data;
1633	}
1634
1635	/* For zero-length plaintext the src pointer is ignored;
1636	* otherwise both parts must be valid
1637	*/
1638	if (sha->src_len && !sha->src)
1639	return -EINVAL;
1640
1641	memset(&op, 0, sizeof(op));
1642	op.cmd_q = cmd_q;
1643	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1644	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1645	op.u.sha.type = sha->type;
1646	op.u.sha.msg_bits = sha->msg_bits;
1647
1648	/* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1649	* SHA384/512 require 2 adjacent SB slots, with the right half in the
1650	* first slot, and the left half in the second. Each portion must then
1651	* be in little endian format: use the 256-bit byte swap option.
1652	*/
1653	ret = ccp_init_dm_workarea(wa: &ctx, cmd_q, len: sb_count * CCP_SB_BYTES,
1654	dir: DMA_BIDIRECTIONAL);
1655	if (ret)
1656	return ret;
1657	if (sha->first) {
1658	switch (sha->type) {
1659	case CCP_SHA_TYPE_1:
1660	case CCP_SHA_TYPE_224:
1661	case CCP_SHA_TYPE_256:
1662	memcpy(ctx.address + ioffset, init, ctx_size);
1663	break;
1664	case CCP_SHA_TYPE_384:
1665	case CCP_SHA_TYPE_512:
1666	memcpy(ctx.address + ctx_size / 2, init,
1667	ctx_size / 2);
1668	memcpy(ctx.address, init + ctx_size / 2,
1669	ctx_size / 2);
1670	break;
1671	default:
1672	ret = -EINVAL;
1673	goto e_ctx;
1674	}
1675	} else {
1676	/* Restore the context */
1677	ret = ccp_set_dm_area(wa: &ctx, wa_offset: 0, sg: sha->ctx, sg_offset: 0,
1678	len: sb_count * CCP_SB_BYTES);
1679	if (ret)
1680	goto e_ctx;
1681	}
1682
1683	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1684	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1685	if (ret) {
1686	cmd->engine_error = cmd_q->cmd_error;
1687	goto e_ctx;
1688	}
1689
1690	if (sha->src) {
1691	/* Send data to the CCP SHA engine; block_size is set above */
1692	ret = ccp_init_data(data: &src, cmd_q, sg: sha->src, sg_len: sha->src_len,
1693	dm_len: block_size, dir: DMA_TO_DEVICE);
1694	if (ret)
1695	goto e_ctx;
1696
1697	while (src.sg_wa.bytes_left) {
1698	ccp_prepare_data(src: &src, NULL, op: &op, block_size, blocksize_op: false);
1699	if (sha->final && !src.sg_wa.bytes_left)
1700	op.eom = 1;
1701
1702	ret = cmd_q->ccp->vdata->perform->sha(&op);
1703	if (ret) {
1704	cmd->engine_error = cmd_q->cmd_error;
1705	goto e_data;
1706	}
1707
1708	ccp_process_data(src: &src, NULL, op: &op);
1709	}
1710	} else {
1711	op.eom = 1;
1712	ret = cmd_q->ccp->vdata->perform->sha(&op);
1713	if (ret) {
1714	cmd->engine_error = cmd_q->cmd_error;
1715	goto e_data;
1716	}
1717	}
1718
1719	/* Retrieve the SHA context - convert from LE to BE using
1720	* 32-byte (256-bit) byteswapping to BE
1721	*/
1722	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1723	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1724	if (ret) {
1725	cmd->engine_error = cmd_q->cmd_error;
1726	goto e_data;
1727	}
1728
1729	if (sha->final) {
1730	/* Finishing up, so get the digest */
1731	switch (sha->type) {
1732	case CCP_SHA_TYPE_1:
1733	case CCP_SHA_TYPE_224:
1734	case CCP_SHA_TYPE_256:
1735	ccp_get_dm_area(wa: &ctx, wa_offset: ooffset,
1736	sg: sha->ctx, sg_offset: 0,
1737	len: digest_size);
1738	break;
1739	case CCP_SHA_TYPE_384:
1740	case CCP_SHA_TYPE_512:
1741	ccp_get_dm_area(wa: &ctx, wa_offset: 0,
1742	sg: sha->ctx, LSB_ITEM_SIZE - ooffset,
1743	LSB_ITEM_SIZE);
1744	ccp_get_dm_area(wa: &ctx, LSB_ITEM_SIZE + ooffset,
1745	sg: sha->ctx, sg_offset: 0,
1746	LSB_ITEM_SIZE - ooffset);
1747	break;
1748	default:
1749	ret = -EINVAL;
1750	goto e_data;
1751	}
1752	} else {
1753	/* Stash the context */
1754	ccp_get_dm_area(wa: &ctx, wa_offset: 0, sg: sha->ctx, sg_offset: 0,
1755	len: sb_count * CCP_SB_BYTES);
1756	}
1757
1758	if (sha->final && sha->opad) {
1759	/* HMAC operation, recursively perform final SHA */
1760	struct ccp_cmd hmac_cmd;
1761	struct scatterlist sg;
1762	u8 *hmac_buf;
1763
1764	if (sha->opad_len != block_size) {
1765	ret = -EINVAL;
1766	goto e_data;
1767	}
1768
1769	hmac_buf = kmalloc(size: block_size + digest_size, GFP_KERNEL);
1770	if (!hmac_buf) {
1771	ret = -ENOMEM;
1772	goto e_data;
1773	}
1774	sg_init_one(&sg, hmac_buf, block_size + digest_size);
1775
1776	scatterwalk_map_and_copy(buf: hmac_buf, sg: sha->opad, start: 0, nbytes: block_size, out: 0);
1777	switch (sha->type) {
1778	case CCP_SHA_TYPE_1:
1779	case CCP_SHA_TYPE_224:
1780	case CCP_SHA_TYPE_256:
1781	memcpy(hmac_buf + block_size,
1782	ctx.address + ooffset,
1783	digest_size);
1784	break;
1785	case CCP_SHA_TYPE_384:
1786	case CCP_SHA_TYPE_512:
1787	memcpy(hmac_buf + block_size,
1788	ctx.address + LSB_ITEM_SIZE + ooffset,
1789	LSB_ITEM_SIZE);
1790	memcpy(hmac_buf + block_size +
1791	(LSB_ITEM_SIZE - ooffset),
1792	ctx.address,
1793	LSB_ITEM_SIZE);
1794	break;
1795	default:
1796	kfree(objp: hmac_buf);
1797	ret = -EINVAL;
1798	goto e_data;
1799	}
1800
1801	memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1802	hmac_cmd.engine = CCP_ENGINE_SHA;
1803	hmac_cmd.u.sha.type = sha->type;
1804	hmac_cmd.u.sha.ctx = sha->ctx;
1805	hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1806	hmac_cmd.u.sha.src = &sg;
1807	hmac_cmd.u.sha.src_len = block_size + digest_size;
1808	hmac_cmd.u.sha.opad = NULL;
1809	hmac_cmd.u.sha.opad_len = 0;
1810	hmac_cmd.u.sha.first = 1;
1811	hmac_cmd.u.sha.final = 1;
1812	hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1813
1814	ret = ccp_run_sha_cmd(cmd_q, cmd: &hmac_cmd);
1815	if (ret)
1816	cmd->engine_error = hmac_cmd.engine_error;
1817
1818	kfree(objp: hmac_buf);
1819	}
1820
1821	e_data:
1822	if (sha->src)
1823	ccp_free_data(data: &src, cmd_q);
1824
1825	e_ctx:
1826	ccp_dm_free(wa: &ctx);
1827
1828	return ret;
1829	}
1830
1831	static noinline_for_stack int
1832	ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1833	{
1834	struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1835	struct ccp_dm_workarea exp, src, dst;
1836	struct ccp_op op;
1837	unsigned int sb_count, i_len, o_len;
1838	int ret;
1839
1840	/* Check against the maximum allowable size, in bits */
1841	if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1842	return -EINVAL;
1843
1844	if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1845	return -EINVAL;
1846
1847	memset(&op, 0, sizeof(op));
1848	op.cmd_q = cmd_q;
1849	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1850
1851	/* The RSA modulus must precede the message being acted upon, so
1852	* it must be copied to a DMA area where the message and the
1853	* modulus can be concatenated. Therefore the input buffer
1854	* length required is twice the output buffer length (which
1855	* must be a multiple of 256-bits). Compute o_len, i_len in bytes.
1856	* Buffer sizes must be a multiple of 32 bytes; rounding up may be
1857	* required.
1858	*/
1859	o_len = 32 * ((rsa->key_size + 255) / 256);
1860	i_len = o_len * 2;
1861
1862	sb_count = 0;
1863	if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1864	/* sb_count is the number of storage block slots required
1865	* for the modulus.
1866	*/
1867	sb_count = o_len / CCP_SB_BYTES;
1868	op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1869	sb_count);
1870	if (!op.sb_key)
1871	return -EIO;
1872	} else {
1873	/* A version 5 device allows a modulus size that will not fit
1874	* in the LSB, so the command will transfer it from memory.
1875	* Set the sb key to the default, even though it's not used.
1876	*/
1877	op.sb_key = cmd_q->sb_key;
1878	}
1879
1880	/* The RSA exponent must be in little endian format. Reverse its
1881	* byte order.
1882	*/
1883	ret = ccp_init_dm_workarea(wa: &exp, cmd_q, len: o_len, dir: DMA_TO_DEVICE);
1884	if (ret)
1885	goto e_sb;
1886
1887	ret = ccp_reverse_set_dm_area(wa: &exp, wa_offset: 0, sg: rsa->exp, sg_offset: 0, len: rsa->exp_len);
1888	if (ret)
1889	goto e_exp;
1890
1891	if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1892	/* Copy the exponent to the local storage block, using
1893	* as many 32-byte blocks as were allocated above. It's
1894	* already little endian, so no further change is required.
1895	*/
1896	ret = ccp_copy_to_sb(cmd_q, wa: &exp, jobid: op.jobid, sb: op.sb_key,
1897	byte_swap: CCP_PASSTHRU_BYTESWAP_NOOP);
1898	if (ret) {
1899	cmd->engine_error = cmd_q->cmd_error;
1900	goto e_exp;
1901	}
1902	} else {
1903	/* The exponent can be retrieved from memory via DMA. */
1904	op.exp.u.dma.address = exp.dma.address;
1905	op.exp.u.dma.offset = 0;
1906	}
1907
1908	/* Concatenate the modulus and the message. Both the modulus and
1909	* the operands must be in little endian format. Since the input
1910	* is in big endian format it must be converted.
1911	*/
1912	ret = ccp_init_dm_workarea(wa: &src, cmd_q, len: i_len, dir: DMA_TO_DEVICE);
1913	if (ret)
1914	goto e_exp;
1915
1916	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: rsa->mod, sg_offset: 0, len: rsa->mod_len);
1917	if (ret)
1918	goto e_src;
1919	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: o_len, sg: rsa->src, sg_offset: 0, len: rsa->src_len);
1920	if (ret)
1921	goto e_src;
1922
1923	/* Prepare the output area for the operation */
1924	ret = ccp_init_dm_workarea(wa: &dst, cmd_q, len: o_len, dir: DMA_FROM_DEVICE);
1925	if (ret)
1926	goto e_src;
1927
1928	op.soc = 1;
1929	op.src.u.dma.address = src.dma.address;
1930	op.src.u.dma.offset = 0;
1931	op.src.u.dma.length = i_len;
1932	op.dst.u.dma.address = dst.dma.address;
1933	op.dst.u.dma.offset = 0;
1934	op.dst.u.dma.length = o_len;
1935
1936	op.u.rsa.mod_size = rsa->key_size;
1937	op.u.rsa.input_len = i_len;
1938
1939	ret = cmd_q->ccp->vdata->perform->rsa(&op);
1940	if (ret) {
1941	cmd->engine_error = cmd_q->cmd_error;
1942	goto e_dst;
1943	}
1944
1945	ccp_reverse_get_dm_area(wa: &dst, wa_offset: 0, sg: rsa->dst, sg_offset: 0, len: rsa->mod_len);
1946
1947	e_dst:
1948	ccp_dm_free(wa: &dst);
1949
1950	e_src:
1951	ccp_dm_free(wa: &src);
1952
1953	e_exp:
1954	ccp_dm_free(wa: &exp);
1955
1956	e_sb:
1957	if (sb_count)
1958	cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1959
1960	return ret;
1961	}
1962
1963	static noinline_for_stack int
1964	ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1965	{
1966	struct ccp_passthru_engine *pt = &cmd->u.passthru;
1967	struct ccp_dm_workarea mask;
1968	struct ccp_data src, dst;
1969	struct ccp_op op;
1970	bool in_place = false;
1971	unsigned int i;
1972	int ret = 0;
1973
1974	if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1975	return -EINVAL;
1976
1977	if (!pt->src || !pt->dst)
1978	return -EINVAL;
1979
1980	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1981	if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1982	return -EINVAL;
1983	if (!pt->mask)
1984	return -EINVAL;
1985	}
1986
1987	BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1988
1989	memset(&op, 0, sizeof(op));
1990	op.cmd_q = cmd_q;
1991	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1992
1993	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1994	/* Load the mask */
1995	op.sb_key = cmd_q->sb_key;
1996
1997	ret = ccp_init_dm_workarea(wa: &mask, cmd_q,
1998	CCP_PASSTHRU_SB_COUNT *
1999	CCP_SB_BYTES,
2000	dir: DMA_TO_DEVICE);
2001	if (ret)
2002	return ret;
2003
2004	ret = ccp_set_dm_area(wa: &mask, wa_offset: 0, sg: pt->mask, sg_offset: 0, len: pt->mask_len);
2005	if (ret)
2006	goto e_mask;
2007	ret = ccp_copy_to_sb(cmd_q, wa: &mask, jobid: op.jobid, sb: op.sb_key,
2008	byte_swap: CCP_PASSTHRU_BYTESWAP_NOOP);
2009	if (ret) {
2010	cmd->engine_error = cmd_q->cmd_error;
2011	goto e_mask;
2012	}
2013	}
2014
2015	/* Prepare the input and output data workareas. For in-place
2016	* operations we need to set the dma direction to BIDIRECTIONAL
2017	* and copy the src workarea to the dst workarea.
2018	*/
2019	if (sg_virt(sg: pt->src) == sg_virt(sg: pt->dst))
2020	in_place = true;
2021
2022	ret = ccp_init_data(data: &src, cmd_q, sg: pt->src, sg_len: pt->src_len,
2023	CCP_PASSTHRU_MASKSIZE,
2024	dir: in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2025	if (ret)
2026	goto e_mask;
2027
2028	if (in_place) {
2029	dst = src;
2030	} else {
2031	ret = ccp_init_data(data: &dst, cmd_q, sg: pt->dst, sg_len: pt->src_len,
2032	CCP_PASSTHRU_MASKSIZE, dir: DMA_FROM_DEVICE);
2033	if (ret)
2034	goto e_src;
2035	}
2036
2037	/* Send data to the CCP Passthru engine
2038	* Because the CCP engine works on a single source and destination
2039	* dma address at a time, each entry in the source scatterlist
2040	* (after the dma_map_sg call) must be less than or equal to the
2041	* (remaining) length in the destination scatterlist entry and the
2042	* length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2043	*/
2044	dst.sg_wa.sg_used = 0;
2045	for (i = 1; i <= src.sg_wa.dma_count; i++) {
2046	if (!dst.sg_wa.sg ||
2047	(sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
2048	ret = -EINVAL;
2049	goto e_dst;
2050	}
2051
2052	if (i == src.sg_wa.dma_count) {
2053	op.eom = 1;
2054	op.soc = 1;
2055	}
2056
2057	op.src.type = CCP_MEMTYPE_SYSTEM;
2058	op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2059	op.src.u.dma.offset = 0;
2060	op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2061
2062	op.dst.type = CCP_MEMTYPE_SYSTEM;
2063	op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2064	op.dst.u.dma.offset = dst.sg_wa.sg_used;
2065	op.dst.u.dma.length = op.src.u.dma.length;
2066
2067	ret = cmd_q->ccp->vdata->perform->passthru(&op);
2068	if (ret) {
2069	cmd->engine_error = cmd_q->cmd_error;
2070	goto e_dst;
2071	}
2072
2073	dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
2074	if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
2075	dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2076	dst.sg_wa.sg_used = 0;
2077	}
2078	src.sg_wa.sg = sg_next(src.sg_wa.sg);
2079	}
2080
2081	e_dst:
2082	if (!in_place)
2083	ccp_free_data(data: &dst, cmd_q);
2084
2085	e_src:
2086	ccp_free_data(data: &src, cmd_q);
2087
2088	e_mask:
2089	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2090	ccp_dm_free(wa: &mask);
2091
2092	return ret;
2093	}
2094
2095	static noinline_for_stack int
2096	ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2097	struct ccp_cmd *cmd)
2098	{
2099	struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2100	struct ccp_dm_workarea mask;
2101	struct ccp_op op;
2102	int ret;
2103
2104	if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2105	return -EINVAL;
2106
2107	if (!pt->src_dma || !pt->dst_dma)
2108	return -EINVAL;
2109
2110	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2111	if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2112	return -EINVAL;
2113	if (!pt->mask)
2114	return -EINVAL;
2115	}
2116
2117	BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2118
2119	memset(&op, 0, sizeof(op));
2120	op.cmd_q = cmd_q;
2121	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2122
2123	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2124	/* Load the mask */
2125	op.sb_key = cmd_q->sb_key;
2126
2127	mask.length = pt->mask_len;
2128	mask.dma.address = pt->mask;
2129	mask.dma.length = pt->mask_len;
2130
2131	ret = ccp_copy_to_sb(cmd_q, wa: &mask, jobid: op.jobid, sb: op.sb_key,
2132	byte_swap: CCP_PASSTHRU_BYTESWAP_NOOP);
2133	if (ret) {
2134	cmd->engine_error = cmd_q->cmd_error;
2135	return ret;
2136	}
2137	}
2138
2139	/* Send data to the CCP Passthru engine */
2140	op.eom = 1;
2141	op.soc = 1;
2142
2143	op.src.type = CCP_MEMTYPE_SYSTEM;
2144	op.src.u.dma.address = pt->src_dma;
2145	op.src.u.dma.offset = 0;
2146	op.src.u.dma.length = pt->src_len;
2147
2148	op.dst.type = CCP_MEMTYPE_SYSTEM;
2149	op.dst.u.dma.address = pt->dst_dma;
2150	op.dst.u.dma.offset = 0;
2151	op.dst.u.dma.length = pt->src_len;
2152
2153	ret = cmd_q->ccp->vdata->perform->passthru(&op);
2154	if (ret)
2155	cmd->engine_error = cmd_q->cmd_error;
2156
2157	return ret;
2158	}
2159
2160	static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2161	{
2162	struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2163	struct ccp_dm_workarea src, dst;
2164	struct ccp_op op;
2165	int ret;
2166	u8 *save;
2167
2168	if (!ecc->u.mm.operand_1 ||
2169	(ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2170	return -EINVAL;
2171
2172	if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2173	if (!ecc->u.mm.operand_2 ||
2174	(ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2175	return -EINVAL;
2176
2177	if (!ecc->u.mm.result ||
2178	(ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2179	return -EINVAL;
2180
2181	memset(&op, 0, sizeof(op));
2182	op.cmd_q = cmd_q;
2183	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2184
2185	/* Concatenate the modulus and the operands. Both the modulus and
2186	* the operands must be in little endian format. Since the input
2187	* is in big endian format it must be converted and placed in a
2188	* fixed length buffer.
2189	*/
2190	ret = ccp_init_dm_workarea(wa: &src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2191	dir: DMA_TO_DEVICE);
2192	if (ret)
2193	return ret;
2194
2195	/* Save the workarea address since it is updated in order to perform
2196	* the concatenation
2197	*/
2198	save = src.address;
2199
2200	/* Copy the ECC modulus */
2201	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->mod, sg_offset: 0, len: ecc->mod_len);
2202	if (ret)
2203	goto e_src;
2204	src.address += CCP_ECC_OPERAND_SIZE;
2205
2206	/* Copy the first operand */
2207	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.mm.operand_1, sg_offset: 0,
2208	len: ecc->u.mm.operand_1_len);
2209	if (ret)
2210	goto e_src;
2211	src.address += CCP_ECC_OPERAND_SIZE;
2212
2213	if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2214	/* Copy the second operand */
2215	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.mm.operand_2, sg_offset: 0,
2216	len: ecc->u.mm.operand_2_len);
2217	if (ret)
2218	goto e_src;
2219	src.address += CCP_ECC_OPERAND_SIZE;
2220	}
2221
2222	/* Restore the workarea address */
2223	src.address = save;
2224
2225	/* Prepare the output area for the operation */
2226	ret = ccp_init_dm_workarea(wa: &dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2227	dir: DMA_FROM_DEVICE);
2228	if (ret)
2229	goto e_src;
2230
2231	op.soc = 1;
2232	op.src.u.dma.address = src.dma.address;
2233	op.src.u.dma.offset = 0;
2234	op.src.u.dma.length = src.length;
2235	op.dst.u.dma.address = dst.dma.address;
2236	op.dst.u.dma.offset = 0;
2237	op.dst.u.dma.length = dst.length;
2238
2239	op.u.ecc.function = cmd->u.ecc.function;
2240
2241	ret = cmd_q->ccp->vdata->perform->ecc(&op);
2242	if (ret) {
2243	cmd->engine_error = cmd_q->cmd_error;
2244	goto e_dst;
2245	}
2246
2247	ecc->ecc_result = le16_to_cpup(
2248	p: (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2249	if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2250	ret = -EIO;
2251	goto e_dst;
2252	}
2253
2254	/* Save the ECC result */
2255	ccp_reverse_get_dm_area(wa: &dst, wa_offset: 0, sg: ecc->u.mm.result, sg_offset: 0,
2256	CCP_ECC_MODULUS_BYTES);
2257
2258	e_dst:
2259	ccp_dm_free(wa: &dst);
2260
2261	e_src:
2262	ccp_dm_free(wa: &src);
2263
2264	return ret;
2265	}
2266
2267	static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2268	{
2269	struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2270	struct ccp_dm_workarea src, dst;
2271	struct ccp_op op;
2272	int ret;
2273	u8 *save;
2274
2275	if (!ecc->u.pm.point_1.x ||
2276	(ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2277	!ecc->u.pm.point_1.y ||
2278	(ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2279	return -EINVAL;
2280
2281	if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2282	if (!ecc->u.pm.point_2.x ||
2283	(ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2284	!ecc->u.pm.point_2.y ||
2285	(ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2286	return -EINVAL;
2287	} else {
2288	if (!ecc->u.pm.domain_a ||
2289	(ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2290	return -EINVAL;
2291
2292	if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2293	if (!ecc->u.pm.scalar ||
2294	(ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2295	return -EINVAL;
2296	}
2297
2298	if (!ecc->u.pm.result.x ||
2299	(ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2300	!ecc->u.pm.result.y ||
2301	(ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2302	return -EINVAL;
2303
2304	memset(&op, 0, sizeof(op));
2305	op.cmd_q = cmd_q;
2306	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2307
2308	/* Concatenate the modulus and the operands. Both the modulus and
2309	* the operands must be in little endian format. Since the input
2310	* is in big endian format it must be converted and placed in a
2311	* fixed length buffer.
2312	*/
2313	ret = ccp_init_dm_workarea(wa: &src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2314	dir: DMA_TO_DEVICE);
2315	if (ret)
2316	return ret;
2317
2318	/* Save the workarea address since it is updated in order to perform
2319	* the concatenation
2320	*/
2321	save = src.address;
2322
2323	/* Copy the ECC modulus */
2324	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->mod, sg_offset: 0, len: ecc->mod_len);
2325	if (ret)
2326	goto e_src;
2327	src.address += CCP_ECC_OPERAND_SIZE;
2328
2329	/* Copy the first point X and Y coordinate */
2330	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.pm.point_1.x, sg_offset: 0,
2331	len: ecc->u.pm.point_1.x_len);
2332	if (ret)
2333	goto e_src;
2334	src.address += CCP_ECC_OPERAND_SIZE;
2335	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.pm.point_1.y, sg_offset: 0,
2336	len: ecc->u.pm.point_1.y_len);
2337	if (ret)
2338	goto e_src;
2339	src.address += CCP_ECC_OPERAND_SIZE;
2340
2341	/* Set the first point Z coordinate to 1 */
2342	*src.address = 0x01;
2343	src.address += CCP_ECC_OPERAND_SIZE;
2344
2345	if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2346	/* Copy the second point X and Y coordinate */
2347	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.pm.point_2.x, sg_offset: 0,
2348	len: ecc->u.pm.point_2.x_len);
2349	if (ret)
2350	goto e_src;
2351	src.address += CCP_ECC_OPERAND_SIZE;
2352	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.pm.point_2.y, sg_offset: 0,
2353	len: ecc->u.pm.point_2.y_len);
2354	if (ret)
2355	goto e_src;
2356	src.address += CCP_ECC_OPERAND_SIZE;
2357
2358	/* Set the second point Z coordinate to 1 */
2359	*src.address = 0x01;
2360	src.address += CCP_ECC_OPERAND_SIZE;
2361	} else {
2362	/* Copy the Domain "a" parameter */
2363	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.pm.domain_a, sg_offset: 0,
2364	len: ecc->u.pm.domain_a_len);
2365	if (ret)
2366	goto e_src;
2367	src.address += CCP_ECC_OPERAND_SIZE;
2368
2369	if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2370	/* Copy the scalar value */
2371	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0,
2372	sg: ecc->u.pm.scalar, sg_offset: 0,
2373	len: ecc->u.pm.scalar_len);
2374	if (ret)
2375	goto e_src;
2376	src.address += CCP_ECC_OPERAND_SIZE;
2377	}
2378	}
2379
2380	/* Restore the workarea address */
2381	src.address = save;
2382
2383	/* Prepare the output area for the operation */
2384	ret = ccp_init_dm_workarea(wa: &dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2385	dir: DMA_FROM_DEVICE);
2386	if (ret)
2387	goto e_src;
2388
2389	op.soc = 1;
2390	op.src.u.dma.address = src.dma.address;
2391	op.src.u.dma.offset = 0;
2392	op.src.u.dma.length = src.length;
2393	op.dst.u.dma.address = dst.dma.address;
2394	op.dst.u.dma.offset = 0;
2395	op.dst.u.dma.length = dst.length;
2396
2397	op.u.ecc.function = cmd->u.ecc.function;
2398
2399	ret = cmd_q->ccp->vdata->perform->ecc(&op);
2400	if (ret) {
2401	cmd->engine_error = cmd_q->cmd_error;
2402	goto e_dst;
2403	}
2404
2405	ecc->ecc_result = le16_to_cpup(
2406	p: (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2407	if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2408	ret = -EIO;
2409	goto e_dst;
2410	}
2411
2412	/* Save the workarea address since it is updated as we walk through
2413	* to copy the point math result
2414	*/
2415	save = dst.address;
2416
2417	/* Save the ECC result X and Y coordinates */
2418	ccp_reverse_get_dm_area(wa: &dst, wa_offset: 0, sg: ecc->u.pm.result.x, sg_offset: 0,
2419	CCP_ECC_MODULUS_BYTES);
2420	dst.address += CCP_ECC_OUTPUT_SIZE;
2421	ccp_reverse_get_dm_area(wa: &dst, wa_offset: 0, sg: ecc->u.pm.result.y, sg_offset: 0,
2422	CCP_ECC_MODULUS_BYTES);
2423
2424	/* Restore the workarea address */
2425	dst.address = save;
2426
2427	e_dst:
2428	ccp_dm_free(wa: &dst);
2429
2430	e_src:
2431	ccp_dm_free(wa: &src);
2432
2433	return ret;
2434	}
2435
2436	static noinline_for_stack int
2437	ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2438	{
2439	struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2440
2441	ecc->ecc_result = 0;
2442
2443	if (!ecc->mod ||
2444	(ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2445	return -EINVAL;
2446
2447	switch (ecc->function) {
2448	case CCP_ECC_FUNCTION_MMUL_384BIT:
2449	case CCP_ECC_FUNCTION_MADD_384BIT:
2450	case CCP_ECC_FUNCTION_MINV_384BIT:
2451	return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2452
2453	case CCP_ECC_FUNCTION_PADD_384BIT:
2454	case CCP_ECC_FUNCTION_PMUL_384BIT:
2455	case CCP_ECC_FUNCTION_PDBL_384BIT:
2456	return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2457
2458	default:
2459	return -EINVAL;
2460	}
2461	}
2462
2463	int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2464	{
2465	int ret;
2466
2467	cmd->engine_error = 0;
2468	cmd_q->cmd_error = 0;
2469	cmd_q->int_rcvd = 0;
2470	cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2471
2472	switch (cmd->engine) {
2473	case CCP_ENGINE_AES:
2474	switch (cmd->u.aes.mode) {
2475	case CCP_AES_MODE_CMAC:
2476	ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2477	break;
2478	case CCP_AES_MODE_GCM:
2479	ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2480	break;
2481	default:
2482	ret = ccp_run_aes_cmd(cmd_q, cmd);
2483	break;
2484	}
2485	break;
2486	case CCP_ENGINE_XTS_AES_128:
2487	ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2488	break;
2489	case CCP_ENGINE_DES3:
2490	ret = ccp_run_des3_cmd(cmd_q, cmd);
2491	break;
2492	case CCP_ENGINE_SHA:
2493	ret = ccp_run_sha_cmd(cmd_q, cmd);
2494	break;
2495	case CCP_ENGINE_RSA:
2496	ret = ccp_run_rsa_cmd(cmd_q, cmd);
2497	break;
2498	case CCP_ENGINE_PASSTHRU:
2499	if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2500	ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2501	else
2502	ret = ccp_run_passthru_cmd(cmd_q, cmd);
2503	break;
2504	case CCP_ENGINE_ECC:
2505	ret = ccp_run_ecc_cmd(cmd_q, cmd);
2506	break;
2507	default:
2508	ret = -EINVAL;
2509	}
2510
2511	return ret;
2512	}
2513