1	// SPDX-License-Identifier: GPL-2.0-only
2	/* Copyright (C) 2020 Marvell. */
3
4	#include <crypto/aes.h>
5	#include <crypto/authenc.h>
6	#include <crypto/cryptd.h>
7	#include <crypto/des.h>
8	#include <crypto/internal/aead.h>
9	#include <crypto/sha1.h>
10	#include <crypto/sha2.h>
11	#include <crypto/xts.h>
12	#include <crypto/gcm.h>
13	#include <crypto/scatterwalk.h>
14	#include <linux/rtnetlink.h>
15	#include <linux/sort.h>
16	#include <linux/module.h>
17	#include "otx2_cptvf.h"
18	#include "otx2_cptvf_algs.h"
19	#include "otx2_cpt_reqmgr.h"
20	#include "cn10k_cpt.h"
21
22	/* Size of salt in AES GCM mode */
23	#define AES_GCM_SALT_SIZE 4
24	/* Size of IV in AES GCM mode */
25	#define AES_GCM_IV_SIZE 8
26	/* Size of ICV (Integrity Check Value) in AES GCM mode */
27	#define AES_GCM_ICV_SIZE 16
28	/* Offset of IV in AES GCM mode */
29	#define AES_GCM_IV_OFFSET 8
30	#define CONTROL_WORD_LEN 8
31	#define KEY2_OFFSET 48
32	#define DMA_MODE_FLAG(dma_mode) \
33	(((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0)
34
35	/* Truncated SHA digest size */
36	#define SHA1_TRUNC_DIGEST_SIZE 12
37	#define SHA256_TRUNC_DIGEST_SIZE 16
38	#define SHA384_TRUNC_DIGEST_SIZE 24
39	#define SHA512_TRUNC_DIGEST_SIZE 32
40
41	static DEFINE_MUTEX(mutex);
42	static int is_crypto_registered;
43
44	struct cpt_device_desc {
45	struct pci_dev *dev;
46	int num_queues;
47	};
48
49	struct cpt_device_table {
50	atomic_t count;
51	struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM];
52	};
53
54	static struct cpt_device_table se_devices = {
55	.count = ATOMIC_INIT(0)
56	};
57
58	static inline int get_se_device(struct pci_dev **pdev, int *cpu_num)
59	{
60	int count;
61
62	count = atomic_read(v: &se_devices.count);
63	if (count < 1)
64	return -ENODEV;
65
66	*cpu_num = get_cpu();
67	/*
68	* On OcteonTX2 platform CPT instruction queue is bound to each
69	* local function LF, in turn LFs can be attached to PF
70	* or VF therefore we always use first device. We get maximum
71	* performance if one CPT queue is available for each cpu
72	* otherwise CPT queues need to be shared between cpus.
73	*/
74	if (*cpu_num >= se_devices.desc[0].num_queues)
75	*cpu_num %= se_devices.desc[0].num_queues;
76	*pdev = se_devices.desc[0].dev;
77
78	put_cpu();
79
80	return 0;
81	}
82
83	static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req)
84	{
85	struct otx2_cpt_req_ctx *rctx;
86	struct aead_request *req;
87	struct crypto_aead *tfm;
88
89	req = container_of(cpt_req->areq, struct aead_request, base);
90	tfm = crypto_aead_reqtfm(req);
91	rctx = aead_request_ctx_dma(req);
92	if (memcmp(p: rctx->fctx.hmac.s.hmac_calc,
93	q: rctx->fctx.hmac.s.hmac_recv,
94	size: crypto_aead_authsize(tfm)) != 0)
95	return -EBADMSG;
96
97	return 0;
98	}
99
100	static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2)
101	{
102	struct otx2_cpt_inst_info *inst_info = arg2;
103	struct crypto_async_request *areq = arg1;
104	struct otx2_cpt_req_info *cpt_req;
105	struct pci_dev *pdev;
106
107	if (inst_info) {
108	cpt_req = inst_info->req;
109	if (!status) {
110	/*
111	* When selected cipher is NULL we need to manually
112	* verify whether calculated hmac value matches
113	* received hmac value
114	*/
115	if (cpt_req->req_type ==
116	OTX2_CPT_AEAD_ENC_DEC_NULL_REQ &&
117	!cpt_req->is_enc)
118	status = validate_hmac_cipher_null(cpt_req);
119	}
120	pdev = inst_info->pdev;
121	otx2_cpt_info_destroy(pdev, info: inst_info);
122	}
123	if (areq)
124	crypto_request_complete(req: areq, err: status);
125	}
126
127	static void output_iv_copyback(struct crypto_async_request *areq)
128	{
129	struct otx2_cpt_req_info *req_info;
130	struct otx2_cpt_req_ctx *rctx;
131	struct skcipher_request *sreq;
132	struct crypto_skcipher *stfm;
133	struct otx2_cpt_enc_ctx *ctx;
134	u32 start, ivsize;
135
136	sreq = container_of(areq, struct skcipher_request, base);
137	stfm = crypto_skcipher_reqtfm(req: sreq);
138	ctx = crypto_skcipher_ctx(tfm: stfm);
139	if (ctx->cipher_type == OTX2_CPT_AES_CBC ||
140	ctx->cipher_type == OTX2_CPT_DES3_CBC) {
141	rctx = skcipher_request_ctx_dma(req: sreq);
142	req_info = &rctx->cpt_req;
143	ivsize = crypto_skcipher_ivsize(tfm: stfm);
144	start = sreq->cryptlen - ivsize;
145
146	if (req_info->is_enc) {
147	scatterwalk_map_and_copy(buf: sreq->iv, sg: sreq->dst, start,
148	nbytes: ivsize, out: 0);
149	} else {
150	if (sreq->src != sreq->dst) {
151	scatterwalk_map_and_copy(buf: sreq->iv, sg: sreq->src,
152	start, nbytes: ivsize, out: 0);
153	} else {
154	memcpy(sreq->iv, req_info->iv_out, ivsize);
155	kfree(objp: req_info->iv_out);
156	}
157	}
158	}
159	}
160
161	static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2)
162	{
163	struct otx2_cpt_inst_info *inst_info = arg2;
164	struct crypto_async_request *areq = arg1;
165	struct pci_dev *pdev;
166
167	if (areq) {
168	if (!status)
169	output_iv_copyback(areq);
170	if (inst_info) {
171	pdev = inst_info->pdev;
172	otx2_cpt_info_destroy(pdev, info: inst_info);
173	}
174	crypto_request_complete(req: areq, err: status);
175	}
176	}
177
178	static inline void update_input_data(struct otx2_cpt_req_info *req_info,
179	struct scatterlist *inp_sg,
180	u32 nbytes, u32 *argcnt)
181	{
182	req_info->req.dlen += nbytes;
183
184	while (nbytes) {
185	u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length;
186	u8 *ptr = sg_virt(sg: inp_sg);
187
188	req_info->in[*argcnt].vptr = (void *)ptr;
189	req_info->in[*argcnt].size = len;
190	nbytes -= len;
191	++(*argcnt);
192	inp_sg = sg_next(inp_sg);
193	}
194	}
195
196	static inline void update_output_data(struct otx2_cpt_req_info *req_info,
197	struct scatterlist *outp_sg,
198	u32 offset, u32 nbytes, u32 *argcnt)
199	{
200	u32 len, sg_len;
201	u8 *ptr;
202
203	req_info->rlen += nbytes;
204
205	while (nbytes) {
206	sg_len = outp_sg->length - offset;
207	len = (nbytes < sg_len) ? nbytes : sg_len;
208	ptr = sg_virt(sg: outp_sg);
209
210	req_info->out[*argcnt].vptr = (void *) (ptr + offset);
211	req_info->out[*argcnt].size = len;
212	nbytes -= len;
213	++(*argcnt);
214	offset = 0;
215	outp_sg = sg_next(outp_sg);
216	}
217	}
218
219	static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,
220	u32 *argcnt)
221	{
222	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
223	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
224	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm: stfm);
225	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
226	struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
227	int ivsize = crypto_skcipher_ivsize(tfm: stfm);
228	u32 start = req->cryptlen - ivsize;
229	gfp_t flags;
230
231	flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
232	GFP_KERNEL : GFP_ATOMIC;
233	req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
234	req_info->ctrl.s.se_req = 1;
235
236	req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
237	DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
238	if (enc) {
239	req_info->req.opcode.s.minor = 2;
240	} else {
241	req_info->req.opcode.s.minor = 3;
242	if ((ctx->cipher_type == OTX2_CPT_AES_CBC ||
243	ctx->cipher_type == OTX2_CPT_DES3_CBC) &&
244	req->src == req->dst) {
245	req_info->iv_out = kmalloc(size: ivsize, flags);
246	if (!req_info->iv_out)
247	return -ENOMEM;
248
249	scatterwalk_map_and_copy(buf: req_info->iv_out, sg: req->src,
250	start, nbytes: ivsize, out: 0);
251	}
252	}
253	/* Encryption data length */
254	req_info->req.param1 = req->cryptlen;
255	/* Authentication data length */
256	req_info->req.param2 = 0;
257
258	fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
259	fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
260	fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
261
262	if (ctx->cipher_type == OTX2_CPT_AES_XTS)
263	memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
264	else
265	memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
266
267	memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm));
268
269	cpu_to_be64s(&fctx->enc.enc_ctrl.u);
270
271	/*
272	* Storing Packet Data Information in offset
273	* Control Word First 8 bytes
274	*/
275	req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
276	req_info->in[*argcnt].size = CONTROL_WORD_LEN;
277	req_info->req.dlen += CONTROL_WORD_LEN;
278	++(*argcnt);
279
280	req_info->in[*argcnt].vptr = (u8 *)fctx;
281	req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
282	req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
283
284	++(*argcnt);
285
286	return 0;
287	}
288
289	static inline int create_input_list(struct skcipher_request *req, u32 enc,
290	u32 enc_iv_len)
291	{
292	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
293	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
294	u32 argcnt = 0;
295	int ret;
296
297	ret = create_ctx_hdr(req, enc, argcnt: &argcnt);
298	if (ret)
299	return ret;
300
301	update_input_data(req_info, inp_sg: req->src, nbytes: req->cryptlen, argcnt: &argcnt);
302	req_info->in_cnt = argcnt;
303
304	return 0;
305	}
306
307	static inline void create_output_list(struct skcipher_request *req,
308	u32 enc_iv_len)
309	{
310	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
311	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
312	u32 argcnt = 0;
313
314	/*
315	* OUTPUT Buffer Processing
316	* AES encryption/decryption output would be
317	* received in the following format
318	*
319	* ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
320	* [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ]
321	*/
322	update_output_data(req_info, outp_sg: req->dst, offset: 0, nbytes: req->cryptlen, argcnt: &argcnt);
323	req_info->out_cnt = argcnt;
324	}
325
326	static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)
327	{
328	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
329	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
330	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm: stfm);
331	int ret;
332
333	if (ctx->fbk_cipher) {
334	skcipher_request_set_tfm(req: &rctx->sk_fbk_req, tfm: ctx->fbk_cipher);
335	skcipher_request_set_callback(req: &rctx->sk_fbk_req,
336	flags: req->base.flags,
337	compl: req->base.complete,
338	data: req->base.data);
339	skcipher_request_set_crypt(req: &rctx->sk_fbk_req, src: req->src,
340	dst: req->dst, cryptlen: req->cryptlen, iv: req->iv);
341	ret = is_enc ? crypto_skcipher_encrypt(req: &rctx->sk_fbk_req) :
342	crypto_skcipher_decrypt(req: &rctx->sk_fbk_req);
343	} else {
344	ret = -EINVAL;
345	}
346	return ret;
347	}
348
349	static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
350	{
351	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
352	struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
353	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm: stfm);
354	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
355	u32 enc_iv_len = crypto_skcipher_ivsize(tfm: stfm);
356	struct pci_dev *pdev;
357	int status, cpu_num;
358
359	if (req->cryptlen == 0)
360	return 0;
361
362	if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len))
363	return -EINVAL;
364
365	if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE)
366	return skcipher_do_fallback(req, is_enc: enc);
367
368	/* Clear control words */
369	rctx->ctrl_word.flags = 0;
370	rctx->fctx.enc.enc_ctrl.u = 0;
371
372	status = create_input_list(req, enc, enc_iv_len);
373	if (status)
374	return status;
375	create_output_list(req, enc_iv_len);
376
377	status = get_se_device(pdev: &pdev, cpu_num: &cpu_num);
378	if (status)
379	return status;
380
381	req_info->callback = otx2_cpt_skcipher_callback;
382	req_info->areq = &req->base;
383	req_info->req_type = OTX2_CPT_ENC_DEC_REQ;
384	req_info->is_enc = enc;
385	req_info->is_trunc_hmac = false;
386	req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
387
388	req_info->req.cptr = ctx->er_ctx.hw_ctx;
389	req_info->req.cptr_dma = ctx->er_ctx.cptr_dma;
390
391	/*
392	* We perform an asynchronous send and once
393	* the request is completed the driver would
394	* intimate through registered call back functions
395	*/
396	status = otx2_cpt_do_request(pdev, req: req_info, cpu_num);
397
398	return status;
399	}
400
401	static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req)
402	{
403	return cpt_enc_dec(req, enc: true);
404	}
405
406	static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req)
407	{
408	return cpt_enc_dec(req, enc: false);
409	}
410
411	static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm,
412	const u8 *key, u32 keylen)
413	{
414	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
415	const u8 *key2 = key + (keylen / 2);
416	const u8 *key1 = key;
417	int ret;
418
419	ret = xts_verify_key(tfm, key, keylen);
420	if (ret)
421	return ret;
422	ctx->key_len = keylen;
423	ctx->enc_align_len = 1;
424	memcpy(ctx->enc_key, key1, keylen / 2);
425	memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
426	ctx->cipher_type = OTX2_CPT_AES_XTS;
427	switch (ctx->key_len) {
428	case 2 * AES_KEYSIZE_128:
429	ctx->key_type = OTX2_CPT_AES_128_BIT;
430	break;
431	case 2 * AES_KEYSIZE_192:
432	ctx->key_type = OTX2_CPT_AES_192_BIT;
433	break;
434	case 2 * AES_KEYSIZE_256:
435	ctx->key_type = OTX2_CPT_AES_256_BIT;
436	break;
437	default:
438	return -EINVAL;
439	}
440	return crypto_skcipher_setkey(tfm: ctx->fbk_cipher, key, keylen);
441	}
442
443	static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
444	u32 keylen, u8 cipher_type)
445	{
446	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
447
448	if (keylen != DES3_EDE_KEY_SIZE)
449	return -EINVAL;
450
451	ctx->key_len = keylen;
452	ctx->cipher_type = cipher_type;
453	ctx->enc_align_len = 8;
454
455	memcpy(ctx->enc_key, key, keylen);
456
457	return crypto_skcipher_setkey(tfm: ctx->fbk_cipher, key, keylen);
458	}
459
460	static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
461	u32 keylen, u8 cipher_type)
462	{
463	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
464
465	switch (keylen) {
466	case AES_KEYSIZE_128:
467	ctx->key_type = OTX2_CPT_AES_128_BIT;
468	break;
469	case AES_KEYSIZE_192:
470	ctx->key_type = OTX2_CPT_AES_192_BIT;
471	break;
472	case AES_KEYSIZE_256:
473	ctx->key_type = OTX2_CPT_AES_256_BIT;
474	break;
475	default:
476	return -EINVAL;
477	}
478	if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB)
479	ctx->enc_align_len = 16;
480	else
481	ctx->enc_align_len = 1;
482
483	ctx->key_len = keylen;
484	ctx->cipher_type = cipher_type;
485
486	memcpy(ctx->enc_key, key, keylen);
487
488	return crypto_skcipher_setkey(tfm: ctx->fbk_cipher, key, keylen);
489	}
490
491	static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm,
492	const u8 *key, u32 keylen)
493	{
494	return cpt_aes_setkey(tfm, key, keylen, cipher_type: OTX2_CPT_AES_CBC);
495	}
496
497	static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm,
498	const u8 *key, u32 keylen)
499	{
500	return cpt_aes_setkey(tfm, key, keylen, cipher_type: OTX2_CPT_AES_ECB);
501	}
502
503	static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm,
504	const u8 *key, u32 keylen)
505	{
506	return cpt_des_setkey(tfm, key, keylen, cipher_type: OTX2_CPT_DES3_CBC);
507	}
508
509	static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm,
510	const u8 *key, u32 keylen)
511	{
512	return cpt_des_setkey(tfm, key, keylen, cipher_type: OTX2_CPT_DES3_ECB);
513	}
514
515	static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx,
516	struct crypto_alg *alg)
517	{
518	if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
519	ctx->fbk_cipher =
520	crypto_alloc_skcipher(alg_name: alg->cra_name, type: 0,
521	CRYPTO_ALG_ASYNC |
522	CRYPTO_ALG_NEED_FALLBACK);
523	if (IS_ERR(ptr: ctx->fbk_cipher)) {
524	pr_err("%s() failed to allocate fallback for %s\n",
525	__func__, alg->cra_name);
526	return PTR_ERR(ptr: ctx->fbk_cipher);
527	}
528	}
529	return 0;
530	}
531
532	static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)
533	{
534	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm: stfm);
535	struct crypto_tfm *tfm = crypto_skcipher_tfm(tfm: stfm);
536	struct crypto_alg *alg = tfm->__crt_alg;
537	struct pci_dev *pdev;
538	int ret, cpu_num;
539
540	memset(ctx, 0, sizeof(*ctx));
541	/*
542	* Additional memory for skcipher_request is
543	* allocated since the cryptd daemon uses
544	* this memory for request_ctx information
545	*/
546	crypto_skcipher_set_reqsize_dma(
547	skcipher: stfm, reqsize: sizeof(struct otx2_cpt_req_ctx) +
548	sizeof(struct skcipher_request));
549
550	ret = get_se_device(pdev: &pdev, cpu_num: &cpu_num);
551	if (ret)
552	return ret;
553
554	ctx->pdev = pdev;
555	ret = cn10k_cpt_hw_ctx_init(pdev, er_ctx: &ctx->er_ctx);
556	if (ret)
557	return ret;
558
559	return cpt_skcipher_fallback_init(ctx, alg);
560	}
561
562	static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm)
563	{
564	struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
565
566	if (ctx->fbk_cipher) {
567	crypto_free_skcipher(tfm: ctx->fbk_cipher);
568	ctx->fbk_cipher = NULL;
569	}
570	cn10k_cpt_hw_ctx_clear(pdev: ctx->pdev, er_ctx: &ctx->er_ctx);
571	}
572
573	static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,
574	struct crypto_alg *alg)
575	{
576	if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
577	ctx->fbk_cipher =
578	crypto_alloc_aead(alg_name: alg->cra_name, type: 0,
579	CRYPTO_ALG_ASYNC |
580	CRYPTO_ALG_NEED_FALLBACK);
581	if (IS_ERR(ptr: ctx->fbk_cipher)) {
582	pr_err("%s() failed to allocate fallback for %s\n",
583	__func__, alg->cra_name);
584	return PTR_ERR(ptr: ctx->fbk_cipher);
585	}
586	}
587	return 0;
588	}
589
590	static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
591	{
592	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm: atfm);
593	struct crypto_tfm *tfm = crypto_aead_tfm(tfm: atfm);
594	struct crypto_alg *alg = tfm->__crt_alg;
595	struct pci_dev *pdev;
596	int ret, cpu_num;
597
598	ctx->cipher_type = cipher_type;
599	ctx->mac_type = mac_type;
600
601	/*
602	* When selected cipher is NULL we use HMAC opcode instead of
603	* FLEXICRYPTO opcode therefore we don't need to use HASH algorithms
604	* for calculating ipad and opad
605	*/
606	if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL) {
607	switch (ctx->mac_type) {
608	case OTX2_CPT_SHA1:
609	ctx->hashalg = crypto_alloc_shash(alg_name: "sha1", type: 0,
610	CRYPTO_ALG_ASYNC);
611	if (IS_ERR(ptr: ctx->hashalg))
612	return PTR_ERR(ptr: ctx->hashalg);
613	break;
614
615	case OTX2_CPT_SHA256:
616	ctx->hashalg = crypto_alloc_shash(alg_name: "sha256", type: 0,
617	CRYPTO_ALG_ASYNC);
618	if (IS_ERR(ptr: ctx->hashalg))
619	return PTR_ERR(ptr: ctx->hashalg);
620	break;
621
622	case OTX2_CPT_SHA384:
623	ctx->hashalg = crypto_alloc_shash(alg_name: "sha384", type: 0,
624	CRYPTO_ALG_ASYNC);
625	if (IS_ERR(ptr: ctx->hashalg))
626	return PTR_ERR(ptr: ctx->hashalg);
627	break;
628
629	case OTX2_CPT_SHA512:
630	ctx->hashalg = crypto_alloc_shash(alg_name: "sha512", type: 0,
631	CRYPTO_ALG_ASYNC);
632	if (IS_ERR(ptr: ctx->hashalg))
633	return PTR_ERR(ptr: ctx->hashalg);
634	break;
635	}
636	}
637	switch (ctx->cipher_type) {
638	case OTX2_CPT_AES_CBC:
639	case OTX2_CPT_AES_ECB:
640	ctx->enc_align_len = 16;
641	break;
642	case OTX2_CPT_DES3_CBC:
643	case OTX2_CPT_DES3_ECB:
644	ctx->enc_align_len = 8;
645	break;
646	case OTX2_CPT_AES_GCM:
647	case OTX2_CPT_CIPHER_NULL:
648	ctx->enc_align_len = 1;
649	break;
650	}
651	crypto_aead_set_reqsize_dma(aead: atfm, reqsize: sizeof(struct otx2_cpt_req_ctx));
652
653	ret = get_se_device(pdev: &pdev, cpu_num: &cpu_num);
654	if (ret)
655	return ret;
656
657	ctx->pdev = pdev;
658	ret = cn10k_cpt_hw_ctx_init(pdev, er_ctx: &ctx->er_ctx);
659	if (ret)
660	return ret;
661
662	return cpt_aead_fallback_init(ctx, alg);
663	}
664
665	static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm)
666	{
667	return cpt_aead_init(atfm: tfm, cipher_type: OTX2_CPT_AES_CBC, mac_type: OTX2_CPT_SHA1);
668	}
669
670	static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm)
671	{
672	return cpt_aead_init(atfm: tfm, cipher_type: OTX2_CPT_AES_CBC, mac_type: OTX2_CPT_SHA256);
673	}
674
675	static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm)
676	{
677	return cpt_aead_init(atfm: tfm, cipher_type: OTX2_CPT_AES_CBC, mac_type: OTX2_CPT_SHA384);
678	}
679
680	static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm)
681	{
682	return cpt_aead_init(atfm: tfm, cipher_type: OTX2_CPT_AES_CBC, mac_type: OTX2_CPT_SHA512);
683	}
684
685	static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm)
686	{
687	return cpt_aead_init(atfm: tfm, cipher_type: OTX2_CPT_CIPHER_NULL, mac_type: OTX2_CPT_SHA1);
688	}
689
690	static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm)
691	{
692	return cpt_aead_init(atfm: tfm, cipher_type: OTX2_CPT_CIPHER_NULL, mac_type: OTX2_CPT_SHA256);
693	}
694
695	static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm)
696	{
697	return cpt_aead_init(atfm: tfm, cipher_type: OTX2_CPT_CIPHER_NULL, mac_type: OTX2_CPT_SHA384);
698	}
699
700	static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm)
701	{
702	return cpt_aead_init(atfm: tfm, cipher_type: OTX2_CPT_CIPHER_NULL, mac_type: OTX2_CPT_SHA512);
703	}
704
705	static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)
706	{
707	return cpt_aead_init(atfm: tfm, cipher_type: OTX2_CPT_AES_GCM, mac_type: OTX2_CPT_MAC_NULL);
708	}
709
710	static void otx2_cpt_aead_exit(struct crypto_aead *tfm)
711	{
712	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
713
714	kfree(objp: ctx->ipad);
715	kfree(objp: ctx->opad);
716	if (ctx->hashalg)
717	crypto_free_shash(tfm: ctx->hashalg);
718	kfree(objp: ctx->sdesc);
719
720	if (ctx->fbk_cipher) {
721	crypto_free_aead(tfm: ctx->fbk_cipher);
722	ctx->fbk_cipher = NULL;
723	}
724	cn10k_cpt_hw_ctx_clear(pdev: ctx->pdev, er_ctx: &ctx->er_ctx);
725	}
726
727	static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm,
728	unsigned int authsize)
729	{
730	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
731
732	if (crypto_rfc4106_check_authsize(authsize))
733	return -EINVAL;
734
735	tfm->authsize = authsize;
736	/* Set authsize for fallback case */
737	if (ctx->fbk_cipher)
738	ctx->fbk_cipher->authsize = authsize;
739
740	return 0;
741	}
742
743	static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm,
744	unsigned int authsize)
745	{
746	tfm->authsize = authsize;
747
748	return 0;
749	}
750
751	static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm,
752	unsigned int authsize)
753	{
754	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
755
756	ctx->is_trunc_hmac = true;
757	tfm->authsize = authsize;
758
759	return 0;
760	}
761
762	static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg)
763	{
764	struct otx2_cpt_sdesc *sdesc;
765	int size;
766
767	size = sizeof(struct shash_desc) + crypto_shash_descsize(tfm: alg);
768	sdesc = kmalloc(size, GFP_KERNEL);
769	if (!sdesc)
770	return NULL;
771
772	sdesc->shash.tfm = alg;
773
774	return sdesc;
775	}
776
777	static inline void swap_data32(void *buf, u32 len)
778	{
779	cpu_to_be32_array(dst: buf, src: buf, len: len / 4);
780	}
781
782	static inline void swap_data64(void *buf, u32 len)
783	{
784	u64 *src = buf;
785	int i = 0;
786
787	for (i = 0 ; i < len / 8; i++, src++)
788	cpu_to_be64s(src);
789	}
790
791	static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad)
792	{
793	struct sha512_state *sha512;
794	struct sha256_state *sha256;
795	struct sha1_state *sha1;
796
797	switch (mac_type) {
798	case OTX2_CPT_SHA1:
799	sha1 = (struct sha1_state *) in_pad;
800	swap_data32(buf: sha1->state, SHA1_DIGEST_SIZE);
801	memcpy(out_pad, &sha1->state, SHA1_DIGEST_SIZE);
802	break;
803
804	case OTX2_CPT_SHA256:
805	sha256 = (struct sha256_state *) in_pad;
806	swap_data32(buf: sha256->state, SHA256_DIGEST_SIZE);
807	memcpy(out_pad, &sha256->state, SHA256_DIGEST_SIZE);
808	break;
809
810	case OTX2_CPT_SHA384:
811	case OTX2_CPT_SHA512:
812	sha512 = (struct sha512_state *) in_pad;
813	swap_data64(buf: sha512->state, SHA512_DIGEST_SIZE);
814	memcpy(out_pad, &sha512->state, SHA512_DIGEST_SIZE);
815	break;
816
817	default:
818	return -EINVAL;
819	}
820
821	return 0;
822	}
823
824	static int aead_hmac_init(struct crypto_aead *cipher)
825	{
826	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm: cipher);
827	int state_size = crypto_shash_statesize(tfm: ctx->hashalg);
828	int ds = crypto_shash_digestsize(tfm: ctx->hashalg);
829	int bs = crypto_shash_blocksize(tfm: ctx->hashalg);
830	int authkeylen = ctx->auth_key_len;
831	u8 *ipad = NULL, *opad = NULL;
832	int ret = 0, icount = 0;
833
834	ctx->sdesc = alloc_sdesc(alg: ctx->hashalg);
835	if (!ctx->sdesc)
836	return -ENOMEM;
837
838	ctx->ipad = kzalloc(size: bs, GFP_KERNEL);
839	if (!ctx->ipad) {
840	ret = -ENOMEM;
841	goto calc_fail;
842	}
843
844	ctx->opad = kzalloc(size: bs, GFP_KERNEL);
845	if (!ctx->opad) {
846	ret = -ENOMEM;
847	goto calc_fail;
848	}
849
850	ipad = kzalloc(size: state_size, GFP_KERNEL);
851	if (!ipad) {
852	ret = -ENOMEM;
853	goto calc_fail;
854	}
855
856	opad = kzalloc(size: state_size, GFP_KERNEL);
857	if (!opad) {
858	ret = -ENOMEM;
859	goto calc_fail;
860	}
861
862	if (authkeylen > bs) {
863	ret = crypto_shash_digest(desc: &ctx->sdesc->shash, data: ctx->key,
864	len: authkeylen, out: ipad);
865	if (ret)
866	goto calc_fail;
867
868	authkeylen = ds;
869	} else {
870	memcpy(ipad, ctx->key, authkeylen);
871	}
872
873	memset(ipad + authkeylen, 0, bs - authkeylen);
874	memcpy(opad, ipad, bs);
875
876	for (icount = 0; icount < bs; icount++) {
877	ipad[icount] ^= 0x36;
878	opad[icount] ^= 0x5c;
879	}
880
881	/*
882	* Partial Hash calculated from the software
883	* algorithm is retrieved for IPAD & OPAD
884	*/
885
886	/* IPAD Calculation */
887	crypto_shash_init(desc: &ctx->sdesc->shash);
888	crypto_shash_update(desc: &ctx->sdesc->shash, data: ipad, len: bs);
889	crypto_shash_export(desc: &ctx->sdesc->shash, out: ipad);
890	ret = copy_pad(mac_type: ctx->mac_type, out_pad: ctx->ipad, in_pad: ipad);
891	if (ret)
892	goto calc_fail;
893
894	/* OPAD Calculation */
895	crypto_shash_init(desc: &ctx->sdesc->shash);
896	crypto_shash_update(desc: &ctx->sdesc->shash, data: opad, len: bs);
897	crypto_shash_export(desc: &ctx->sdesc->shash, out: opad);
898	ret = copy_pad(mac_type: ctx->mac_type, out_pad: ctx->opad, in_pad: opad);
899	if (ret)
900	goto calc_fail;
901
902	kfree(objp: ipad);
903	kfree(objp: opad);
904
905	return 0;
906
907	calc_fail:
908	kfree(objp: ctx->ipad);
909	ctx->ipad = NULL;
910	kfree(objp: ctx->opad);
911	ctx->opad = NULL;
912	kfree(objp: ipad);
913	kfree(objp: opad);
914	kfree(objp: ctx->sdesc);
915	ctx->sdesc = NULL;
916
917	return ret;
918	}
919
920	static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,
921	const unsigned char *key,
922	unsigned int keylen)
923	{
924	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm: cipher);
925	struct crypto_authenc_key_param *param;
926	int enckeylen = 0, authkeylen = 0;
927	struct rtattr *rta = (void *)key;
928
929	if (!RTA_OK(rta, keylen))
930	return -EINVAL;
931
932	if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
933	return -EINVAL;
934
935	if (RTA_PAYLOAD(rta) < sizeof(*param))
936	return -EINVAL;
937
938	param = RTA_DATA(rta);
939	enckeylen = be32_to_cpu(param->enckeylen);
940	key += RTA_ALIGN(rta->rta_len);
941	keylen -= RTA_ALIGN(rta->rta_len);
942	if (keylen < enckeylen)
943	return -EINVAL;
944
945	if (keylen > OTX2_CPT_MAX_KEY_SIZE)
946	return -EINVAL;
947
948	authkeylen = keylen - enckeylen;
949	memcpy(ctx->key, key, keylen);
950
951	switch (enckeylen) {
952	case AES_KEYSIZE_128:
953	ctx->key_type = OTX2_CPT_AES_128_BIT;
954	break;
955	case AES_KEYSIZE_192:
956	ctx->key_type = OTX2_CPT_AES_192_BIT;
957	break;
958	case AES_KEYSIZE_256:
959	ctx->key_type = OTX2_CPT_AES_256_BIT;
960	break;
961	default:
962	/* Invalid key length */
963	return -EINVAL;
964	}
965
966	ctx->enc_key_len = enckeylen;
967	ctx->auth_key_len = authkeylen;
968
969	return aead_hmac_init(cipher);
970	}
971
972	static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,
973	const unsigned char *key,
974	unsigned int keylen)
975	{
976	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm: cipher);
977	struct crypto_authenc_key_param *param;
978	struct rtattr *rta = (void *)key;
979	int enckeylen = 0;
980
981	if (!RTA_OK(rta, keylen))
982	return -EINVAL;
983
984	if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
985	return -EINVAL;
986
987	if (RTA_PAYLOAD(rta) < sizeof(*param))
988	return -EINVAL;
989
990	param = RTA_DATA(rta);
991	enckeylen = be32_to_cpu(param->enckeylen);
992	key += RTA_ALIGN(rta->rta_len);
993	keylen -= RTA_ALIGN(rta->rta_len);
994	if (enckeylen != 0)
995	return -EINVAL;
996
997	if (keylen > OTX2_CPT_MAX_KEY_SIZE)
998	return -EINVAL;
999
1000	memcpy(ctx->key, key, keylen);
1001	ctx->enc_key_len = enckeylen;
1002	ctx->auth_key_len = keylen;
1003
1004	return 0;
1005	}
1006
1007	static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
1008	const unsigned char *key,
1009	unsigned int keylen)
1010	{
1011	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm: cipher);
1012
1013	/*
1014	* For aes gcm we expect to get encryption key (16, 24, 32 bytes)
1015	* and salt (4 bytes)
1016	*/
1017	switch (keylen) {
1018	case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE:
1019	ctx->key_type = OTX2_CPT_AES_128_BIT;
1020	ctx->enc_key_len = AES_KEYSIZE_128;
1021	break;
1022	case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE:
1023	ctx->key_type = OTX2_CPT_AES_192_BIT;
1024	ctx->enc_key_len = AES_KEYSIZE_192;
1025	break;
1026	case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE:
1027	ctx->key_type = OTX2_CPT_AES_256_BIT;
1028	ctx->enc_key_len = AES_KEYSIZE_256;
1029	break;
1030	default:
1031	/* Invalid key and salt length */
1032	return -EINVAL;
1033	}
1034
1035	/* Store encryption key and salt */
1036	memcpy(ctx->key, key, keylen);
1037
1038	return crypto_aead_setkey(tfm: ctx->fbk_cipher, key, keylen);
1039	}
1040
1041	static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,
1042	u32 *argcnt)
1043	{
1044	struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1045	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1046	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
1047	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1048	struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
1049	int mac_len = crypto_aead_authsize(tfm);
1050	int ds;
1051
1052	rctx->ctrl_word.e.enc_data_offset = req->assoclen;
1053
1054	switch (ctx->cipher_type) {
1055	case OTX2_CPT_AES_CBC:
1056	if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8))
1057	return -EINVAL;
1058
1059	fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
1060	/* Copy encryption key to context */
1061	memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len,
1062	ctx->enc_key_len);
1063	/* Copy IV to context */
1064	memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm));
1065
1066	ds = crypto_shash_digestsize(tfm: ctx->hashalg);
1067	if (ctx->mac_type == OTX2_CPT_SHA384)
1068	ds = SHA512_DIGEST_SIZE;
1069	if (ctx->ipad)
1070	memcpy(fctx->hmac.e.ipad, ctx->ipad, ds);
1071	if (ctx->opad)
1072	memcpy(fctx->hmac.e.opad, ctx->opad, ds);
1073	break;
1074
1075	case OTX2_CPT_AES_GCM:
1076	if (crypto_ipsec_check_assoclen(assoclen: req->assoclen))
1077	return -EINVAL;
1078
1079	fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR;
1080	/* Copy encryption key to context */
1081	memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len);
1082	/* Copy salt to context */
1083	memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len,
1084	AES_GCM_SALT_SIZE);
1085
1086	rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET;
1087	break;
1088
1089	default:
1090	/* Unknown cipher type */
1091	return -EINVAL;
1092	}
1093	cpu_to_be64s(&rctx->ctrl_word.flags);
1094
1095	req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1096	req_info->ctrl.s.se_req = 1;
1097	req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
1098	DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1099	if (enc) {
1100	req_info->req.opcode.s.minor = 2;
1101	req_info->req.param1 = req->cryptlen;
1102	req_info->req.param2 = req->cryptlen + req->assoclen;
1103	} else {
1104	req_info->req.opcode.s.minor = 3;
1105	req_info->req.param1 = req->cryptlen - mac_len;
1106	req_info->req.param2 = req->cryptlen + req->assoclen - mac_len;
1107	}
1108
1109	fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
1110	fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
1111	fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type;
1112	fctx->enc.enc_ctrl.e.mac_len = mac_len;
1113	cpu_to_be64s(&fctx->enc.enc_ctrl.u);
1114
1115	/*
1116	* Storing Packet Data Information in offset
1117	* Control Word First 8 bytes
1118	*/
1119	req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
1120	req_info->in[*argcnt].size = CONTROL_WORD_LEN;
1121	req_info->req.dlen += CONTROL_WORD_LEN;
1122	++(*argcnt);
1123
1124	req_info->in[*argcnt].vptr = (u8 *)fctx;
1125	req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
1126	req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
1127	++(*argcnt);
1128
1129	return 0;
1130	}
1131
1132	static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
1133	u32 enc)
1134	{
1135	struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1136	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1137	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
1138	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1139
1140	req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1141	req_info->ctrl.s.se_req = 1;
1142	req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC |
1143	DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1144	req_info->is_trunc_hmac = ctx->is_trunc_hmac;
1145
1146	req_info->req.opcode.s.minor = 0;
1147	req_info->req.param1 = ctx->auth_key_len;
1148	req_info->req.param2 = ctx->mac_type << 8;
1149
1150	/* Add authentication key */
1151	req_info->in[*argcnt].vptr = ctx->key;
1152	req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8);
1153	req_info->req.dlen += round_up(ctx->auth_key_len, 8);
1154	++(*argcnt);
1155	}
1156
1157	static inline int create_aead_input_list(struct aead_request *req, u32 enc)
1158	{
1159	struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1160	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1161	u32 inputlen = req->cryptlen + req->assoclen;
1162	u32 status, argcnt = 0;
1163
1164	status = create_aead_ctx_hdr(req, enc, argcnt: &argcnt);
1165	if (status)
1166	return status;
1167	update_input_data(req_info, inp_sg: req->src, nbytes: inputlen, argcnt: &argcnt);
1168	req_info->in_cnt = argcnt;
1169
1170	return 0;
1171	}
1172
1173	static inline void create_aead_output_list(struct aead_request *req, u32 enc,
1174	u32 mac_len)
1175	{
1176	struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1177	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1178	u32 argcnt = 0, outputlen = 0;
1179
1180	if (enc)
1181	outputlen = req->cryptlen + req->assoclen + mac_len;
1182	else
1183	outputlen = req->cryptlen + req->assoclen - mac_len;
1184
1185	update_output_data(req_info, outp_sg: req->dst, offset: 0, nbytes: outputlen, argcnt: &argcnt);
1186	req_info->out_cnt = argcnt;
1187	}
1188
1189	static inline void create_aead_null_input_list(struct aead_request *req,
1190	u32 enc, u32 mac_len)
1191	{
1192	struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1193	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1194	u32 inputlen, argcnt = 0;
1195
1196	if (enc)
1197	inputlen = req->cryptlen + req->assoclen;
1198	else
1199	inputlen = req->cryptlen + req->assoclen - mac_len;
1200
1201	create_hmac_ctx_hdr(req, argcnt: &argcnt, enc);
1202	update_input_data(req_info, inp_sg: req->src, nbytes: inputlen, argcnt: &argcnt);
1203	req_info->in_cnt = argcnt;
1204	}
1205
1206	static inline int create_aead_null_output_list(struct aead_request *req,
1207	u32 enc, u32 mac_len)
1208	{
1209	struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1210	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1211	struct scatterlist *dst;
1212	u8 *ptr = NULL;
1213	int argcnt = 0, status, offset;
1214	u32 inputlen;
1215
1216	if (enc)
1217	inputlen = req->cryptlen + req->assoclen;
1218	else
1219	inputlen = req->cryptlen + req->assoclen - mac_len;
1220
1221	/*
1222	* If source and destination are different
1223	* then copy payload to destination
1224	*/
1225	if (req->src != req->dst) {
1226
1227	ptr = kmalloc(size: inputlen, flags: (req_info->areq->flags &
1228	CRYPTO_TFM_REQ_MAY_SLEEP) ?
1229	GFP_KERNEL : GFP_ATOMIC);
1230	if (!ptr)
1231	return -ENOMEM;
1232
1233	status = sg_copy_to_buffer(sgl: req->src, nents: sg_nents(sg: req->src), buf: ptr,
1234	buflen: inputlen);
1235	if (status != inputlen) {
1236	status = -EINVAL;
1237	goto error_free;
1238	}
1239	status = sg_copy_from_buffer(sgl: req->dst, nents: sg_nents(sg: req->dst), buf: ptr,
1240	buflen: inputlen);
1241	if (status != inputlen) {
1242	status = -EINVAL;
1243	goto error_free;
1244	}
1245	kfree(objp: ptr);
1246	}
1247
1248	if (enc) {
1249	/*
1250	* In an encryption scenario hmac needs
1251	* to be appended after payload
1252	*/
1253	dst = req->dst;
1254	offset = inputlen;
1255	while (offset >= dst->length) {
1256	offset -= dst->length;
1257	dst = sg_next(dst);
1258	if (!dst)
1259	return -ENOENT;
1260	}
1261
1262	update_output_data(req_info, outp_sg: dst, offset, nbytes: mac_len, argcnt: &argcnt);
1263	} else {
1264	/*
1265	* In a decryption scenario calculated hmac for received
1266	* payload needs to be compare with hmac received
1267	*/
1268	status = sg_copy_buffer(sgl: req->src, nents: sg_nents(sg: req->src),
1269	buf: rctx->fctx.hmac.s.hmac_recv, buflen: mac_len,
1270	skip: inputlen, to_buffer: true);
1271	if (status != mac_len)
1272	return -EINVAL;
1273
1274	req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc;
1275	req_info->out[argcnt].size = mac_len;
1276	argcnt++;
1277	}
1278
1279	req_info->out_cnt = argcnt;
1280	return 0;
1281
1282	error_free:
1283	kfree(objp: ptr);
1284	return status;
1285	}
1286
1287	static int aead_do_fallback(struct aead_request *req, bool is_enc)
1288	{
1289	struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1290	struct crypto_aead *aead = crypto_aead_reqtfm(req);
1291	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm: aead);
1292	int ret;
1293
1294	if (ctx->fbk_cipher) {
1295	/* Store the cipher tfm and then use the fallback tfm */
1296	aead_request_set_tfm(req: &rctx->fbk_req, tfm: ctx->fbk_cipher);
1297	aead_request_set_callback(req: &rctx->fbk_req, flags: req->base.flags,
1298	compl: req->base.complete, data: req->base.data);
1299	aead_request_set_crypt(req: &rctx->fbk_req, src: req->src,
1300	dst: req->dst, cryptlen: req->cryptlen, iv: req->iv);
1301	aead_request_set_ad(req: &rctx->fbk_req, assoclen: req->assoclen);
1302	ret = is_enc ? crypto_aead_encrypt(req: &rctx->fbk_req) :
1303	crypto_aead_decrypt(req: &rctx->fbk_req);
1304	} else {
1305	ret = -EINVAL;
1306	}
1307
1308	return ret;
1309	}
1310
1311	static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
1312	{
1313	struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1314	struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1315	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1316	struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
1317	struct pci_dev *pdev;
1318	int status, cpu_num;
1319
1320	/* Clear control words */
1321	rctx->ctrl_word.flags = 0;
1322	rctx->fctx.enc.enc_ctrl.u = 0;
1323
1324	req_info->callback = otx2_cpt_aead_callback;
1325	req_info->areq = &req->base;
1326	req_info->req_type = reg_type;
1327	req_info->is_enc = enc;
1328	req_info->is_trunc_hmac = false;
1329
1330	req_info->req.cptr = ctx->er_ctx.hw_ctx;
1331	req_info->req.cptr_dma = ctx->er_ctx.cptr_dma;
1332
1333	switch (reg_type) {
1334	case OTX2_CPT_AEAD_ENC_DEC_REQ:
1335	status = create_aead_input_list(req, enc);
1336	if (status)
1337	return status;
1338	create_aead_output_list(req, enc, mac_len: crypto_aead_authsize(tfm));
1339	break;
1340
1341	case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ:
1342	create_aead_null_input_list(req, enc,
1343	mac_len: crypto_aead_authsize(tfm));
1344	status = create_aead_null_output_list(req, enc,
1345	mac_len: crypto_aead_authsize(tfm));
1346	if (status)
1347	return status;
1348	break;
1349
1350	default:
1351	return -EINVAL;
1352	}
1353	if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len))
1354	return -EINVAL;
1355
1356	if (!req_info->req.param2 ||
1357	(req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) ||
1358	(req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE))
1359	return aead_do_fallback(req, is_enc: enc);
1360
1361	status = get_se_device(pdev: &pdev, cpu_num: &cpu_num);
1362	if (status)
1363	return status;
1364
1365	req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
1366
1367	/*
1368	* We perform an asynchronous send and once
1369	* the request is completed the driver would
1370	* intimate through registered call back functions
1371	*/
1372	return otx2_cpt_do_request(pdev, req: req_info, cpu_num);
1373	}
1374
1375	static int otx2_cpt_aead_encrypt(struct aead_request *req)
1376	{
1377	return cpt_aead_enc_dec(req, reg_type: OTX2_CPT_AEAD_ENC_DEC_REQ, enc: true);
1378	}
1379
1380	static int otx2_cpt_aead_decrypt(struct aead_request *req)
1381	{
1382	return cpt_aead_enc_dec(req, reg_type: OTX2_CPT_AEAD_ENC_DEC_REQ, enc: false);
1383	}
1384
1385	static int otx2_cpt_aead_null_encrypt(struct aead_request *req)
1386	{
1387	return cpt_aead_enc_dec(req, reg_type: OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, enc: true);
1388	}
1389
1390	static int otx2_cpt_aead_null_decrypt(struct aead_request *req)
1391	{
1392	return cpt_aead_enc_dec(req, reg_type: OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, enc: false);
1393	}
1394
1395	static struct skcipher_alg otx2_cpt_skciphers[] = { {
1396	.base.cra_name = "xts(aes)",
1397	.base.cra_driver_name = "cpt_xts_aes",
1398	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1399	.base.cra_blocksize = AES_BLOCK_SIZE,
1400	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1401	.base.cra_alignmask = 7,
1402	.base.cra_priority = 4001,
1403	.base.cra_module = THIS_MODULE,
1404
1405	.init = otx2_cpt_enc_dec_init,
1406	.exit = otx2_cpt_skcipher_exit,
1407	.ivsize = AES_BLOCK_SIZE,
1408	.min_keysize = 2 * AES_MIN_KEY_SIZE,
1409	.max_keysize = 2 * AES_MAX_KEY_SIZE,
1410	.setkey = otx2_cpt_skcipher_xts_setkey,
1411	.encrypt = otx2_cpt_skcipher_encrypt,
1412	.decrypt = otx2_cpt_skcipher_decrypt,
1413	}, {
1414	.base.cra_name = "cbc(aes)",
1415	.base.cra_driver_name = "cpt_cbc_aes",
1416	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1417	.base.cra_blocksize = AES_BLOCK_SIZE,
1418	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1419	.base.cra_alignmask = 7,
1420	.base.cra_priority = 4001,
1421	.base.cra_module = THIS_MODULE,
1422
1423	.init = otx2_cpt_enc_dec_init,
1424	.exit = otx2_cpt_skcipher_exit,
1425	.ivsize = AES_BLOCK_SIZE,
1426	.min_keysize = AES_MIN_KEY_SIZE,
1427	.max_keysize = AES_MAX_KEY_SIZE,
1428	.setkey = otx2_cpt_skcipher_cbc_aes_setkey,
1429	.encrypt = otx2_cpt_skcipher_encrypt,
1430	.decrypt = otx2_cpt_skcipher_decrypt,
1431	}, {
1432	.base.cra_name = "ecb(aes)",
1433	.base.cra_driver_name = "cpt_ecb_aes",
1434	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1435	.base.cra_blocksize = AES_BLOCK_SIZE,
1436	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1437	.base.cra_alignmask = 7,
1438	.base.cra_priority = 4001,
1439	.base.cra_module = THIS_MODULE,
1440
1441	.init = otx2_cpt_enc_dec_init,
1442	.exit = otx2_cpt_skcipher_exit,
1443	.ivsize = 0,
1444	.min_keysize = AES_MIN_KEY_SIZE,
1445	.max_keysize = AES_MAX_KEY_SIZE,
1446	.setkey = otx2_cpt_skcipher_ecb_aes_setkey,
1447	.encrypt = otx2_cpt_skcipher_encrypt,
1448	.decrypt = otx2_cpt_skcipher_decrypt,
1449	}, {
1450	.base.cra_name = "cbc(des3_ede)",
1451	.base.cra_driver_name = "cpt_cbc_des3_ede",
1452	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1453	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1454	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1455	.base.cra_alignmask = 7,
1456	.base.cra_priority = 4001,
1457	.base.cra_module = THIS_MODULE,
1458
1459	.init = otx2_cpt_enc_dec_init,
1460	.exit = otx2_cpt_skcipher_exit,
1461	.min_keysize = DES3_EDE_KEY_SIZE,
1462	.max_keysize = DES3_EDE_KEY_SIZE,
1463	.ivsize = DES_BLOCK_SIZE,
1464	.setkey = otx2_cpt_skcipher_cbc_des3_setkey,
1465	.encrypt = otx2_cpt_skcipher_encrypt,
1466	.decrypt = otx2_cpt_skcipher_decrypt,
1467	}, {
1468	.base.cra_name = "ecb(des3_ede)",
1469	.base.cra_driver_name = "cpt_ecb_des3_ede",
1470	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1471	.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1472	.base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1473	.base.cra_alignmask = 7,
1474	.base.cra_priority = 4001,
1475	.base.cra_module = THIS_MODULE,
1476
1477	.init = otx2_cpt_enc_dec_init,
1478	.exit = otx2_cpt_skcipher_exit,
1479	.min_keysize = DES3_EDE_KEY_SIZE,
1480	.max_keysize = DES3_EDE_KEY_SIZE,
1481	.ivsize = 0,
1482	.setkey = otx2_cpt_skcipher_ecb_des3_setkey,
1483	.encrypt = otx2_cpt_skcipher_encrypt,
1484	.decrypt = otx2_cpt_skcipher_decrypt,
1485	} };
1486
1487	static struct aead_alg otx2_cpt_aeads[] = { {
1488	.base = {
1489	.cra_name = "authenc(hmac(sha1),cbc(aes))",
1490	.cra_driver_name = "cpt_hmac_sha1_cbc_aes",
1491	.cra_blocksize = AES_BLOCK_SIZE,
1492	.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1493	.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1494	.cra_priority = 4001,
1495	.cra_alignmask = 0,
1496	.cra_module = THIS_MODULE,
1497	},
1498	.init = otx2_cpt_aead_cbc_aes_sha1_init,
1499	.exit = otx2_cpt_aead_exit,
1500	.setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1501	.setauthsize = otx2_cpt_aead_set_authsize,
1502	.encrypt = otx2_cpt_aead_encrypt,
1503	.decrypt = otx2_cpt_aead_decrypt,
1504	.ivsize = AES_BLOCK_SIZE,
1505	.maxauthsize = SHA1_DIGEST_SIZE,
1506	}, {
1507	.base = {
1508	.cra_name = "authenc(hmac(sha256),cbc(aes))",
1509	.cra_driver_name = "cpt_hmac_sha256_cbc_aes",
1510	.cra_blocksize = AES_BLOCK_SIZE,
1511	.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1512	.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1513	.cra_priority = 4001,
1514	.cra_alignmask = 0,
1515	.cra_module = THIS_MODULE,
1516	},
1517	.init = otx2_cpt_aead_cbc_aes_sha256_init,
1518	.exit = otx2_cpt_aead_exit,
1519	.setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1520	.setauthsize = otx2_cpt_aead_set_authsize,
1521	.encrypt = otx2_cpt_aead_encrypt,
1522	.decrypt = otx2_cpt_aead_decrypt,
1523	.ivsize = AES_BLOCK_SIZE,
1524	.maxauthsize = SHA256_DIGEST_SIZE,
1525	}, {
1526	.base = {
1527	.cra_name = "authenc(hmac(sha384),cbc(aes))",
1528	.cra_driver_name = "cpt_hmac_sha384_cbc_aes",
1529	.cra_blocksize = AES_BLOCK_SIZE,
1530	.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1531	.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1532	.cra_priority = 4001,
1533	.cra_alignmask = 0,
1534	.cra_module = THIS_MODULE,
1535	},
1536	.init = otx2_cpt_aead_cbc_aes_sha384_init,
1537	.exit = otx2_cpt_aead_exit,
1538	.setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1539	.setauthsize = otx2_cpt_aead_set_authsize,
1540	.encrypt = otx2_cpt_aead_encrypt,
1541	.decrypt = otx2_cpt_aead_decrypt,
1542	.ivsize = AES_BLOCK_SIZE,
1543	.maxauthsize = SHA384_DIGEST_SIZE,
1544	}, {
1545	.base = {
1546	.cra_name = "authenc(hmac(sha512),cbc(aes))",
1547	.cra_driver_name = "cpt_hmac_sha512_cbc_aes",
1548	.cra_blocksize = AES_BLOCK_SIZE,
1549	.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1550	.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1551	.cra_priority = 4001,
1552	.cra_alignmask = 0,
1553	.cra_module = THIS_MODULE,
1554	},
1555	.init = otx2_cpt_aead_cbc_aes_sha512_init,
1556	.exit = otx2_cpt_aead_exit,
1557	.setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1558	.setauthsize = otx2_cpt_aead_set_authsize,
1559	.encrypt = otx2_cpt_aead_encrypt,
1560	.decrypt = otx2_cpt_aead_decrypt,
1561	.ivsize = AES_BLOCK_SIZE,
1562	.maxauthsize = SHA512_DIGEST_SIZE,
1563	}, {
1564	.base = {
1565	.cra_name = "authenc(hmac(sha1),ecb(cipher_null))",
1566	.cra_driver_name = "cpt_hmac_sha1_ecb_null",
1567	.cra_blocksize = 1,
1568	.cra_flags = CRYPTO_ALG_ASYNC,
1569	.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1570	.cra_priority = 4001,
1571	.cra_alignmask = 0,
1572	.cra_module = THIS_MODULE,
1573	},
1574	.init = otx2_cpt_aead_ecb_null_sha1_init,
1575	.exit = otx2_cpt_aead_exit,
1576	.setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1577	.setauthsize = otx2_cpt_aead_null_set_authsize,
1578	.encrypt = otx2_cpt_aead_null_encrypt,
1579	.decrypt = otx2_cpt_aead_null_decrypt,
1580	.ivsize = 0,
1581	.maxauthsize = SHA1_DIGEST_SIZE,
1582	}, {
1583	.base = {
1584	.cra_name = "authenc(hmac(sha256),ecb(cipher_null))",
1585	.cra_driver_name = "cpt_hmac_sha256_ecb_null",
1586	.cra_blocksize = 1,
1587	.cra_flags = CRYPTO_ALG_ASYNC,
1588	.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1589	.cra_priority = 4001,
1590	.cra_alignmask = 0,
1591	.cra_module = THIS_MODULE,
1592	},
1593	.init = otx2_cpt_aead_ecb_null_sha256_init,
1594	.exit = otx2_cpt_aead_exit,
1595	.setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1596	.setauthsize = otx2_cpt_aead_null_set_authsize,
1597	.encrypt = otx2_cpt_aead_null_encrypt,
1598	.decrypt = otx2_cpt_aead_null_decrypt,
1599	.ivsize = 0,
1600	.maxauthsize = SHA256_DIGEST_SIZE,
1601	}, {
1602	.base = {
1603	.cra_name = "authenc(hmac(sha384),ecb(cipher_null))",
1604	.cra_driver_name = "cpt_hmac_sha384_ecb_null",
1605	.cra_blocksize = 1,
1606	.cra_flags = CRYPTO_ALG_ASYNC,
1607	.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1608	.cra_priority = 4001,
1609	.cra_alignmask = 0,
1610	.cra_module = THIS_MODULE,
1611	},
1612	.init = otx2_cpt_aead_ecb_null_sha384_init,
1613	.exit = otx2_cpt_aead_exit,
1614	.setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1615	.setauthsize = otx2_cpt_aead_null_set_authsize,
1616	.encrypt = otx2_cpt_aead_null_encrypt,
1617	.decrypt = otx2_cpt_aead_null_decrypt,
1618	.ivsize = 0,
1619	.maxauthsize = SHA384_DIGEST_SIZE,
1620	}, {
1621	.base = {
1622	.cra_name = "authenc(hmac(sha512),ecb(cipher_null))",
1623	.cra_driver_name = "cpt_hmac_sha512_ecb_null",
1624	.cra_blocksize = 1,
1625	.cra_flags = CRYPTO_ALG_ASYNC,
1626	.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1627	.cra_priority = 4001,
1628	.cra_alignmask = 0,
1629	.cra_module = THIS_MODULE,
1630	},
1631	.init = otx2_cpt_aead_ecb_null_sha512_init,
1632	.exit = otx2_cpt_aead_exit,
1633	.setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1634	.setauthsize = otx2_cpt_aead_null_set_authsize,
1635	.encrypt = otx2_cpt_aead_null_encrypt,
1636	.decrypt = otx2_cpt_aead_null_decrypt,
1637	.ivsize = 0,
1638	.maxauthsize = SHA512_DIGEST_SIZE,
1639	}, {
1640	.base = {
1641	.cra_name = "rfc4106(gcm(aes))",
1642	.cra_driver_name = "cpt_rfc4106_gcm_aes",
1643	.cra_blocksize = 1,
1644	.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1645	.cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1646	.cra_priority = 4001,
1647	.cra_alignmask = 0,
1648	.cra_module = THIS_MODULE,
1649	},
1650	.init = otx2_cpt_aead_gcm_aes_init,
1651	.exit = otx2_cpt_aead_exit,
1652	.setkey = otx2_cpt_aead_gcm_aes_setkey,
1653	.setauthsize = otx2_cpt_aead_gcm_set_authsize,
1654	.encrypt = otx2_cpt_aead_encrypt,
1655	.decrypt = otx2_cpt_aead_decrypt,
1656	.ivsize = AES_GCM_IV_SIZE,
1657	.maxauthsize = AES_GCM_ICV_SIZE,
1658	} };
1659
1660	static inline int cpt_register_algs(void)
1661	{
1662	int i, err = 0;
1663
1664	for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++)
1665	otx2_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1666
1667	err = crypto_register_skciphers(algs: otx2_cpt_skciphers,
1668	ARRAY_SIZE(otx2_cpt_skciphers));
1669	if (err)
1670	return err;
1671
1672	for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++)
1673	otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1674
1675	err = crypto_register_aeads(algs: otx2_cpt_aeads,
1676	ARRAY_SIZE(otx2_cpt_aeads));
1677	if (err) {
1678	crypto_unregister_skciphers(algs: otx2_cpt_skciphers,
1679	ARRAY_SIZE(otx2_cpt_skciphers));
1680	return err;
1681	}
1682
1683	return 0;
1684	}
1685
1686	static inline void cpt_unregister_algs(void)
1687	{
1688	crypto_unregister_skciphers(algs: otx2_cpt_skciphers,
1689	ARRAY_SIZE(otx2_cpt_skciphers));
1690	crypto_unregister_aeads(algs: otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads));
1691	}
1692
1693	static int compare_func(const void *lptr, const void *rptr)
1694	{
1695	const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr;
1696	const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr;
1697
1698	if (ldesc->dev->devfn < rdesc->dev->devfn)
1699	return -1;
1700	if (ldesc->dev->devfn > rdesc->dev->devfn)
1701	return 1;
1702	return 0;
1703	}
1704
1705	static void swap_func(void *lptr, void *rptr, int size)
1706	{
1707	struct cpt_device_desc *ldesc = lptr;
1708	struct cpt_device_desc *rdesc = rptr;
1709
1710	swap(*ldesc, *rdesc);
1711	}
1712
1713	int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
1714	int num_queues, int num_devices)
1715	{
1716	int ret = 0;
1717	int count;
1718
1719	mutex_lock(&mutex);
1720	count = atomic_read(v: &se_devices.count);
1721	if (count >= OTX2_CPT_MAX_LFS_NUM) {
1722	dev_err(&pdev->dev, "No space to add a new device\n");
1723	ret = -ENOSPC;
1724	goto unlock;
1725	}
1726	se_devices.desc[count].num_queues = num_queues;
1727	se_devices.desc[count++].dev = pdev;
1728	atomic_inc(v: &se_devices.count);
1729
1730	if (atomic_read(v: &se_devices.count) == num_devices &&
1731	is_crypto_registered == false) {
1732	if (cpt_register_algs()) {
1733	dev_err(&pdev->dev,
1734	"Error in registering crypto algorithms\n");
1735	ret = -EINVAL;
1736	goto unlock;
1737	}
1738	try_module_get(module: mod);
1739	is_crypto_registered = true;
1740	}
1741	sort(base: se_devices.desc, num: count, size: sizeof(struct cpt_device_desc),
1742	cmp_func: compare_func, swap_func);
1743
1744	unlock:
1745	mutex_unlock(lock: &mutex);
1746	return ret;
1747	}
1748
1749	void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod)
1750	{
1751	struct cpt_device_table *dev_tbl;
1752	bool dev_found = false;
1753	int i, j, count;
1754
1755	mutex_lock(&mutex);
1756
1757	dev_tbl = &se_devices;
1758	count = atomic_read(v: &dev_tbl->count);
1759	for (i = 0; i < count; i++) {
1760	if (pdev == dev_tbl->desc[i].dev) {
1761	for (j = i; j < count-1; j++)
1762	dev_tbl->desc[j] = dev_tbl->desc[j+1];
1763	dev_found = true;
1764	break;
1765	}
1766	}
1767
1768	if (!dev_found) {
1769	dev_err(&pdev->dev, "%s device not found\n", __func__);
1770	goto unlock;
1771	}
1772	if (atomic_dec_and_test(v: &se_devices.count)) {
1773	cpt_unregister_algs();
1774	module_put(module: mod);
1775	is_crypto_registered = false;
1776	}
1777
1778	unlock:
1779	mutex_unlock(lock: &mutex);
1780	}
1781