1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | |
3 | /* |
4 | * Copyright (C) 2016 Cavium, Inc. |
5 | */ |
6 | |
7 | #include <crypto/aes.h> |
8 | #include <crypto/algapi.h> |
9 | #include <crypto/authenc.h> |
10 | #include <crypto/internal/des.h> |
11 | #include <crypto/xts.h> |
12 | #include <linux/crypto.h> |
13 | #include <linux/err.h> |
14 | #include <linux/list.h> |
15 | #include <linux/scatterlist.h> |
16 | |
17 | #include "cptvf.h" |
18 | #include "cptvf_algs.h" |
19 | |
20 | struct cpt_device_handle { |
21 | void *cdev[MAX_DEVICES]; |
22 | u32 dev_count; |
23 | }; |
24 | |
25 | static struct cpt_device_handle dev_handle; |
26 | |
27 | static void cvm_callback(u32 status, void *arg) |
28 | { |
29 | struct crypto_async_request *req = (struct crypto_async_request *)arg; |
30 | |
31 | crypto_request_complete(req, err: !status); |
32 | } |
33 | |
34 | static inline void update_input_iv(struct cpt_request_info *req_info, |
35 | u8 *iv, u32 enc_iv_len, |
36 | u32 *argcnt) |
37 | { |
38 | /* Setting the iv information */ |
39 | req_info->in[*argcnt].vptr = (void *)iv; |
40 | req_info->in[*argcnt].size = enc_iv_len; |
41 | req_info->req.dlen += enc_iv_len; |
42 | |
43 | ++(*argcnt); |
44 | } |
45 | |
46 | static inline void update_output_iv(struct cpt_request_info *req_info, |
47 | u8 *iv, u32 enc_iv_len, |
48 | u32 *argcnt) |
49 | { |
50 | /* Setting the iv information */ |
51 | req_info->out[*argcnt].vptr = (void *)iv; |
52 | req_info->out[*argcnt].size = enc_iv_len; |
53 | req_info->rlen += enc_iv_len; |
54 | |
55 | ++(*argcnt); |
56 | } |
57 | |
58 | static inline void update_input_data(struct cpt_request_info *req_info, |
59 | struct scatterlist *inp_sg, |
60 | u32 nbytes, u32 *argcnt) |
61 | { |
62 | req_info->req.dlen += nbytes; |
63 | |
64 | while (nbytes) { |
65 | u32 len = min(nbytes, inp_sg->length); |
66 | u8 *ptr = sg_virt(sg: inp_sg); |
67 | |
68 | req_info->in[*argcnt].vptr = (void *)ptr; |
69 | req_info->in[*argcnt].size = len; |
70 | nbytes -= len; |
71 | |
72 | ++(*argcnt); |
73 | ++inp_sg; |
74 | } |
75 | } |
76 | |
77 | static inline void update_output_data(struct cpt_request_info *req_info, |
78 | struct scatterlist *outp_sg, |
79 | u32 nbytes, u32 *argcnt) |
80 | { |
81 | req_info->rlen += nbytes; |
82 | |
83 | while (nbytes) { |
84 | u32 len = min(nbytes, outp_sg->length); |
85 | u8 *ptr = sg_virt(sg: outp_sg); |
86 | |
87 | req_info->out[*argcnt].vptr = (void *)ptr; |
88 | req_info->out[*argcnt].size = len; |
89 | nbytes -= len; |
90 | ++(*argcnt); |
91 | ++outp_sg; |
92 | } |
93 | } |
94 | |
95 | static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc, |
96 | u32 *argcnt) |
97 | { |
98 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
99 | struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm); |
100 | struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); |
101 | struct fc_context *fctx = &rctx->fctx; |
102 | u32 enc_iv_len = crypto_skcipher_ivsize(tfm); |
103 | struct cpt_request_info *req_info = &rctx->cpt_req; |
104 | __be64 *ctrl_flags = NULL; |
105 | __be64 *offset_control; |
106 | |
107 | req_info->ctrl.s.grp = 0; |
108 | req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER; |
109 | req_info->ctrl.s.se_req = SE_CORE_REQ; |
110 | |
111 | req_info->req.opcode.s.major = MAJOR_OP_FC | |
112 | DMA_MODE_FLAG(DMA_GATHER_SCATTER); |
113 | if (enc) |
114 | req_info->req.opcode.s.minor = 2; |
115 | else |
116 | req_info->req.opcode.s.minor = 3; |
117 | |
118 | req_info->req.param1 = req->cryptlen; /* Encryption Data length */ |
119 | req_info->req.param2 = 0; /*Auth data length */ |
120 | |
121 | fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; |
122 | fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; |
123 | fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR; |
124 | |
125 | if (ctx->cipher_type == AES_XTS) |
126 | memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); |
127 | else |
128 | memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); |
129 | ctrl_flags = (__be64 *)&fctx->enc.enc_ctrl.flags; |
130 | *ctrl_flags = cpu_to_be64(fctx->enc.enc_ctrl.flags); |
131 | |
132 | offset_control = (__be64 *)&rctx->control_word; |
133 | *offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16)); |
134 | /* Storing Packet Data Information in offset |
135 | * Control Word First 8 bytes |
136 | */ |
137 | req_info->in[*argcnt].vptr = (u8 *)offset_control; |
138 | req_info->in[*argcnt].size = CONTROL_WORD_LEN; |
139 | req_info->req.dlen += CONTROL_WORD_LEN; |
140 | ++(*argcnt); |
141 | |
142 | req_info->in[*argcnt].vptr = (u8 *)fctx; |
143 | req_info->in[*argcnt].size = sizeof(struct fc_context); |
144 | req_info->req.dlen += sizeof(struct fc_context); |
145 | |
146 | ++(*argcnt); |
147 | |
148 | return 0; |
149 | } |
150 | |
151 | static inline u32 create_input_list(struct skcipher_request *req, u32 enc, |
152 | u32 enc_iv_len) |
153 | { |
154 | struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); |
155 | struct cpt_request_info *req_info = &rctx->cpt_req; |
156 | u32 argcnt = 0; |
157 | |
158 | create_ctx_hdr(req, enc, argcnt: &argcnt); |
159 | update_input_iv(req_info, iv: req->iv, enc_iv_len, argcnt: &argcnt); |
160 | update_input_data(req_info, inp_sg: req->src, nbytes: req->cryptlen, argcnt: &argcnt); |
161 | req_info->incnt = argcnt; |
162 | |
163 | return 0; |
164 | } |
165 | |
166 | static inline void store_cb_info(struct skcipher_request *req, |
167 | struct cpt_request_info *req_info) |
168 | { |
169 | req_info->callback = (void *)cvm_callback; |
170 | req_info->callback_arg = (void *)&req->base; |
171 | } |
172 | |
173 | static inline void create_output_list(struct skcipher_request *req, |
174 | u32 enc_iv_len) |
175 | { |
176 | struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); |
177 | struct cpt_request_info *req_info = &rctx->cpt_req; |
178 | u32 argcnt = 0; |
179 | |
180 | /* OUTPUT Buffer Processing |
181 | * AES encryption/decryption output would be |
182 | * received in the following format |
183 | * |
184 | * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| |
185 | * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] |
186 | */ |
187 | /* Reading IV information */ |
188 | update_output_iv(req_info, iv: req->iv, enc_iv_len, argcnt: &argcnt); |
189 | update_output_data(req_info, outp_sg: req->dst, nbytes: req->cryptlen, argcnt: &argcnt); |
190 | req_info->outcnt = argcnt; |
191 | } |
192 | |
193 | static inline int cvm_enc_dec(struct skcipher_request *req, u32 enc) |
194 | { |
195 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
196 | struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req); |
197 | u32 enc_iv_len = crypto_skcipher_ivsize(tfm); |
198 | struct fc_context *fctx = &rctx->fctx; |
199 | struct cpt_request_info *req_info = &rctx->cpt_req; |
200 | void *cdev = NULL; |
201 | int status; |
202 | |
203 | memset(req_info, 0, sizeof(struct cpt_request_info)); |
204 | req_info->may_sleep = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) != 0; |
205 | memset(fctx, 0, sizeof(struct fc_context)); |
206 | create_input_list(req, enc, enc_iv_len); |
207 | create_output_list(req, enc_iv_len); |
208 | store_cb_info(req, req_info); |
209 | cdev = dev_handle.cdev[smp_processor_id()]; |
210 | status = cptvf_do_request(cptvf: cdev, req: req_info); |
211 | /* We perform an asynchronous send and once |
212 | * the request is completed the driver would |
213 | * intimate through registered call back functions |
214 | */ |
215 | |
216 | if (status) |
217 | return status; |
218 | else |
219 | return -EINPROGRESS; |
220 | } |
221 | |
222 | static int cvm_encrypt(struct skcipher_request *req) |
223 | { |
224 | return cvm_enc_dec(req, enc: true); |
225 | } |
226 | |
227 | static int cvm_decrypt(struct skcipher_request *req) |
228 | { |
229 | return cvm_enc_dec(req, enc: false); |
230 | } |
231 | |
232 | static int cvm_xts_setkey(struct crypto_skcipher *cipher, const u8 *key, |
233 | u32 keylen) |
234 | { |
235 | struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm: cipher); |
236 | int err; |
237 | const u8 *key1 = key; |
238 | const u8 *key2 = key + (keylen / 2); |
239 | |
240 | err = xts_verify_key(tfm: cipher, key, keylen); |
241 | if (err) |
242 | return err; |
243 | ctx->key_len = keylen; |
244 | memcpy(ctx->enc_key, key1, keylen / 2); |
245 | memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); |
246 | ctx->cipher_type = AES_XTS; |
247 | switch (ctx->key_len) { |
248 | case 32: |
249 | ctx->key_type = AES_128_BIT; |
250 | break; |
251 | case 64: |
252 | ctx->key_type = AES_256_BIT; |
253 | break; |
254 | default: |
255 | return -EINVAL; |
256 | } |
257 | |
258 | return 0; |
259 | } |
260 | |
261 | static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen) |
262 | { |
263 | if ((keylen == 16) || (keylen == 24) || (keylen == 32)) { |
264 | ctx->key_len = keylen; |
265 | switch (ctx->key_len) { |
266 | case 16: |
267 | ctx->key_type = AES_128_BIT; |
268 | break; |
269 | case 24: |
270 | ctx->key_type = AES_192_BIT; |
271 | break; |
272 | case 32: |
273 | ctx->key_type = AES_256_BIT; |
274 | break; |
275 | default: |
276 | return -EINVAL; |
277 | } |
278 | |
279 | if (ctx->cipher_type == DES3_CBC) |
280 | ctx->key_type = 0; |
281 | |
282 | return 0; |
283 | } |
284 | |
285 | return -EINVAL; |
286 | } |
287 | |
288 | static int cvm_setkey(struct crypto_skcipher *cipher, const u8 *key, |
289 | u32 keylen, u8 cipher_type) |
290 | { |
291 | struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm: cipher); |
292 | |
293 | ctx->cipher_type = cipher_type; |
294 | if (!cvm_validate_keylen(ctx, keylen)) { |
295 | memcpy(ctx->enc_key, key, keylen); |
296 | return 0; |
297 | } else { |
298 | return -EINVAL; |
299 | } |
300 | } |
301 | |
302 | static int cvm_cbc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, |
303 | u32 keylen) |
304 | { |
305 | return cvm_setkey(cipher, key, keylen, cipher_type: AES_CBC); |
306 | } |
307 | |
308 | static int cvm_ecb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, |
309 | u32 keylen) |
310 | { |
311 | return cvm_setkey(cipher, key, keylen, cipher_type: AES_ECB); |
312 | } |
313 | |
314 | static int cvm_cbc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, |
315 | u32 keylen) |
316 | { |
317 | return verify_skcipher_des3_key(tfm: cipher, key) ?: |
318 | cvm_setkey(cipher, key, keylen, cipher_type: DES3_CBC); |
319 | } |
320 | |
321 | static int cvm_ecb_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, |
322 | u32 keylen) |
323 | { |
324 | return verify_skcipher_des3_key(tfm: cipher, key) ?: |
325 | cvm_setkey(cipher, key, keylen, cipher_type: DES3_ECB); |
326 | } |
327 | |
328 | static int cvm_enc_dec_init(struct crypto_skcipher *tfm) |
329 | { |
330 | crypto_skcipher_set_reqsize_dma(skcipher: tfm, reqsize: sizeof(struct cvm_req_ctx)); |
331 | |
332 | return 0; |
333 | } |
334 | |
335 | static struct skcipher_alg algs[] = { { |
336 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
337 | CRYPTO_ALG_ALLOCATES_MEMORY, |
338 | .base.cra_blocksize = AES_BLOCK_SIZE, |
339 | .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), |
340 | .base.cra_alignmask = 7, |
341 | .base.cra_priority = 4001, |
342 | .base.cra_name = "xts(aes)" , |
343 | .base.cra_driver_name = "cavium-xts-aes" , |
344 | .base.cra_module = THIS_MODULE, |
345 | |
346 | .ivsize = AES_BLOCK_SIZE, |
347 | .min_keysize = 2 * AES_MIN_KEY_SIZE, |
348 | .max_keysize = 2 * AES_MAX_KEY_SIZE, |
349 | .setkey = cvm_xts_setkey, |
350 | .encrypt = cvm_encrypt, |
351 | .decrypt = cvm_decrypt, |
352 | .init = cvm_enc_dec_init, |
353 | }, { |
354 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
355 | CRYPTO_ALG_ALLOCATES_MEMORY, |
356 | .base.cra_blocksize = AES_BLOCK_SIZE, |
357 | .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), |
358 | .base.cra_alignmask = 7, |
359 | .base.cra_priority = 4001, |
360 | .base.cra_name = "cbc(aes)" , |
361 | .base.cra_driver_name = "cavium-cbc-aes" , |
362 | .base.cra_module = THIS_MODULE, |
363 | |
364 | .ivsize = AES_BLOCK_SIZE, |
365 | .min_keysize = AES_MIN_KEY_SIZE, |
366 | .max_keysize = AES_MAX_KEY_SIZE, |
367 | .setkey = cvm_cbc_aes_setkey, |
368 | .encrypt = cvm_encrypt, |
369 | .decrypt = cvm_decrypt, |
370 | .init = cvm_enc_dec_init, |
371 | }, { |
372 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
373 | CRYPTO_ALG_ALLOCATES_MEMORY, |
374 | .base.cra_blocksize = AES_BLOCK_SIZE, |
375 | .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), |
376 | .base.cra_alignmask = 7, |
377 | .base.cra_priority = 4001, |
378 | .base.cra_name = "ecb(aes)" , |
379 | .base.cra_driver_name = "cavium-ecb-aes" , |
380 | .base.cra_module = THIS_MODULE, |
381 | |
382 | .min_keysize = AES_MIN_KEY_SIZE, |
383 | .max_keysize = AES_MAX_KEY_SIZE, |
384 | .setkey = cvm_ecb_aes_setkey, |
385 | .encrypt = cvm_encrypt, |
386 | .decrypt = cvm_decrypt, |
387 | .init = cvm_enc_dec_init, |
388 | }, { |
389 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
390 | CRYPTO_ALG_ALLOCATES_MEMORY, |
391 | .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, |
392 | .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), |
393 | .base.cra_alignmask = 7, |
394 | .base.cra_priority = 4001, |
395 | .base.cra_name = "cbc(des3_ede)" , |
396 | .base.cra_driver_name = "cavium-cbc-des3_ede" , |
397 | .base.cra_module = THIS_MODULE, |
398 | |
399 | .min_keysize = DES3_EDE_KEY_SIZE, |
400 | .max_keysize = DES3_EDE_KEY_SIZE, |
401 | .ivsize = DES_BLOCK_SIZE, |
402 | .setkey = cvm_cbc_des3_setkey, |
403 | .encrypt = cvm_encrypt, |
404 | .decrypt = cvm_decrypt, |
405 | .init = cvm_enc_dec_init, |
406 | }, { |
407 | .base.cra_flags = CRYPTO_ALG_ASYNC | |
408 | CRYPTO_ALG_ALLOCATES_MEMORY, |
409 | .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, |
410 | .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), |
411 | .base.cra_alignmask = 7, |
412 | .base.cra_priority = 4001, |
413 | .base.cra_name = "ecb(des3_ede)" , |
414 | .base.cra_driver_name = "cavium-ecb-des3_ede" , |
415 | .base.cra_module = THIS_MODULE, |
416 | |
417 | .min_keysize = DES3_EDE_KEY_SIZE, |
418 | .max_keysize = DES3_EDE_KEY_SIZE, |
419 | .ivsize = DES_BLOCK_SIZE, |
420 | .setkey = cvm_ecb_des3_setkey, |
421 | .encrypt = cvm_encrypt, |
422 | .decrypt = cvm_decrypt, |
423 | .init = cvm_enc_dec_init, |
424 | } }; |
425 | |
426 | static inline int cav_register_algs(void) |
427 | { |
428 | return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); |
429 | } |
430 | |
431 | static inline void cav_unregister_algs(void) |
432 | { |
433 | crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); |
434 | } |
435 | |
436 | int cvm_crypto_init(struct cpt_vf *cptvf) |
437 | { |
438 | struct pci_dev *pdev = cptvf->pdev; |
439 | u32 dev_count; |
440 | |
441 | dev_count = dev_handle.dev_count; |
442 | dev_handle.cdev[dev_count] = cptvf; |
443 | dev_handle.dev_count++; |
444 | |
445 | if (dev_count == 3) { |
446 | if (cav_register_algs()) { |
447 | dev_err(&pdev->dev, "Error in registering crypto algorithms\n" ); |
448 | return -EINVAL; |
449 | } |
450 | } |
451 | |
452 | return 0; |
453 | } |
454 | |
455 | void cvm_crypto_exit(void) |
456 | { |
457 | u32 dev_count; |
458 | |
459 | dev_count = --dev_handle.dev_count; |
460 | if (!dev_count) |
461 | cav_unregister_algs(); |
462 | } |
463 | |