1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * SM4 Cipher Algorithm, using ARMv8 Crypto Extensions
4 * as specified in
5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6 *
7 * Copyright (C) 2022, Alibaba Group.
8 * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
9 */
10
11#include <linux/module.h>
12#include <linux/crypto.h>
13#include <linux/kernel.h>
14#include <linux/cpufeature.h>
15#include <asm/neon.h>
16#include <asm/simd.h>
17#include <crypto/b128ops.h>
18#include <crypto/internal/simd.h>
19#include <crypto/internal/skcipher.h>
20#include <crypto/internal/hash.h>
21#include <crypto/scatterwalk.h>
22#include <crypto/xts.h>
23#include <crypto/sm4.h>
24
25#define BYTES2BLKS(nbytes) ((nbytes) >> 4)
26
27asmlinkage void sm4_ce_expand_key(const u8 *key, u32 *rkey_enc, u32 *rkey_dec,
28 const u32 *fk, const u32 *ck);
29asmlinkage void sm4_ce_crypt_block(const u32 *rkey, u8 *dst, const u8 *src);
30asmlinkage void sm4_ce_crypt(const u32 *rkey, u8 *dst, const u8 *src,
31 unsigned int nblks);
32asmlinkage void sm4_ce_cbc_enc(const u32 *rkey, u8 *dst, const u8 *src,
33 u8 *iv, unsigned int nblocks);
34asmlinkage void sm4_ce_cbc_dec(const u32 *rkey, u8 *dst, const u8 *src,
35 u8 *iv, unsigned int nblocks);
36asmlinkage void sm4_ce_cbc_cts_enc(const u32 *rkey, u8 *dst, const u8 *src,
37 u8 *iv, unsigned int nbytes);
38asmlinkage void sm4_ce_cbc_cts_dec(const u32 *rkey, u8 *dst, const u8 *src,
39 u8 *iv, unsigned int nbytes);
40asmlinkage void sm4_ce_ctr_enc(const u32 *rkey, u8 *dst, const u8 *src,
41 u8 *iv, unsigned int nblks);
42asmlinkage void sm4_ce_xts_enc(const u32 *rkey1, u8 *dst, const u8 *src,
43 u8 *tweak, unsigned int nbytes,
44 const u32 *rkey2_enc);
45asmlinkage void sm4_ce_xts_dec(const u32 *rkey1, u8 *dst, const u8 *src,
46 u8 *tweak, unsigned int nbytes,
47 const u32 *rkey2_enc);
48asmlinkage void sm4_ce_mac_update(const u32 *rkey_enc, u8 *digest,
49 const u8 *src, unsigned int nblocks,
50 bool enc_before, bool enc_after);
51
52EXPORT_SYMBOL(sm4_ce_expand_key);
53EXPORT_SYMBOL(sm4_ce_crypt_block);
54EXPORT_SYMBOL(sm4_ce_cbc_enc);
55
56struct sm4_xts_ctx {
57 struct sm4_ctx key1;
58 struct sm4_ctx key2;
59};
60
61struct sm4_mac_tfm_ctx {
62 struct sm4_ctx key;
63 u8 __aligned(8) consts[];
64};
65
66struct sm4_mac_desc_ctx {
67 unsigned int len;
68 u8 digest[SM4_BLOCK_SIZE];
69};
70
71static int sm4_setkey(struct crypto_skcipher *tfm, const u8 *key,
72 unsigned int key_len)
73{
74 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
75
76 if (key_len != SM4_KEY_SIZE)
77 return -EINVAL;
78
79 kernel_neon_begin();
80 sm4_ce_expand_key(key, ctx->rkey_enc, ctx->rkey_dec,
81 crypto_sm4_fk, crypto_sm4_ck);
82 kernel_neon_end();
83 return 0;
84}
85
86static int sm4_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
87 unsigned int key_len)
88{
89 struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
90 int ret;
91
92 if (key_len != SM4_KEY_SIZE * 2)
93 return -EINVAL;
94
95 ret = xts_verify_key(tfm, key, keylen: key_len);
96 if (ret)
97 return ret;
98
99 kernel_neon_begin();
100 sm4_ce_expand_key(key, ctx->key1.rkey_enc,
101 ctx->key1.rkey_dec, crypto_sm4_fk, crypto_sm4_ck);
102 sm4_ce_expand_key(&key[SM4_KEY_SIZE], ctx->key2.rkey_enc,
103 ctx->key2.rkey_dec, crypto_sm4_fk, crypto_sm4_ck);
104 kernel_neon_end();
105
106 return 0;
107}
108
109static int sm4_ecb_do_crypt(struct skcipher_request *req, const u32 *rkey)
110{
111 struct skcipher_walk walk;
112 unsigned int nbytes;
113 int err;
114
115 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
116
117 while ((nbytes = walk.nbytes) > 0) {
118 const u8 *src = walk.src.virt.addr;
119 u8 *dst = walk.dst.virt.addr;
120 unsigned int nblks;
121
122 kernel_neon_begin();
123
124 nblks = BYTES2BLKS(nbytes);
125 if (nblks) {
126 sm4_ce_crypt(rkey, dst, src, nblks);
127 nbytes -= nblks * SM4_BLOCK_SIZE;
128 }
129
130 kernel_neon_end();
131
132 err = skcipher_walk_done(walk: &walk, err: nbytes);
133 }
134
135 return err;
136}
137
138static int sm4_ecb_encrypt(struct skcipher_request *req)
139{
140 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
141 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
142
143 return sm4_ecb_do_crypt(req, rkey: ctx->rkey_enc);
144}
145
146static int sm4_ecb_decrypt(struct skcipher_request *req)
147{
148 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
149 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
150
151 return sm4_ecb_do_crypt(req, rkey: ctx->rkey_dec);
152}
153
154static int sm4_cbc_crypt(struct skcipher_request *req,
155 struct sm4_ctx *ctx, bool encrypt)
156{
157 struct skcipher_walk walk;
158 unsigned int nbytes;
159 int err;
160
161 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
162 if (err)
163 return err;
164
165 while ((nbytes = walk.nbytes) > 0) {
166 const u8 *src = walk.src.virt.addr;
167 u8 *dst = walk.dst.virt.addr;
168 unsigned int nblocks;
169
170 nblocks = nbytes / SM4_BLOCK_SIZE;
171 if (nblocks) {
172 kernel_neon_begin();
173
174 if (encrypt)
175 sm4_ce_cbc_enc(ctx->rkey_enc, dst, src,
176 walk.iv, nblocks);
177 else
178 sm4_ce_cbc_dec(rkey: ctx->rkey_dec, dst, src,
179 iv: walk.iv, nblocks);
180
181 kernel_neon_end();
182 }
183
184 err = skcipher_walk_done(walk: &walk, err: nbytes % SM4_BLOCK_SIZE);
185 }
186
187 return err;
188}
189
190static int sm4_cbc_encrypt(struct skcipher_request *req)
191{
192 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
193 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
194
195 return sm4_cbc_crypt(req, ctx, encrypt: true);
196}
197
198static int sm4_cbc_decrypt(struct skcipher_request *req)
199{
200 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
201 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
202
203 return sm4_cbc_crypt(req, ctx, encrypt: false);
204}
205
206static int sm4_cbc_cts_crypt(struct skcipher_request *req, bool encrypt)
207{
208 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
209 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
210 struct scatterlist *src = req->src;
211 struct scatterlist *dst = req->dst;
212 struct scatterlist sg_src[2], sg_dst[2];
213 struct skcipher_request subreq;
214 struct skcipher_walk walk;
215 int cbc_blocks;
216 int err;
217
218 if (req->cryptlen < SM4_BLOCK_SIZE)
219 return -EINVAL;
220
221 if (req->cryptlen == SM4_BLOCK_SIZE)
222 return sm4_cbc_crypt(req, ctx, encrypt);
223
224 skcipher_request_set_tfm(req: &subreq, tfm);
225 skcipher_request_set_callback(req: &subreq, flags: skcipher_request_flags(req),
226 NULL, NULL);
227
228 /* handle the CBC cryption part */
229 cbc_blocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2;
230 if (cbc_blocks) {
231 skcipher_request_set_crypt(req: &subreq, src, dst,
232 cryptlen: cbc_blocks * SM4_BLOCK_SIZE,
233 iv: req->iv);
234
235 err = sm4_cbc_crypt(req: &subreq, ctx, encrypt);
236 if (err)
237 return err;
238
239 dst = src = scatterwalk_ffwd(dst: sg_src, src, len: subreq.cryptlen);
240 if (req->dst != req->src)
241 dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst,
242 len: subreq.cryptlen);
243 }
244
245 /* handle ciphertext stealing */
246 skcipher_request_set_crypt(req: &subreq, src, dst,
247 cryptlen: req->cryptlen - cbc_blocks * SM4_BLOCK_SIZE,
248 iv: req->iv);
249
250 err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false);
251 if (err)
252 return err;
253
254 kernel_neon_begin();
255
256 if (encrypt)
257 sm4_ce_cbc_cts_enc(rkey: ctx->rkey_enc, dst: walk.dst.virt.addr,
258 src: walk.src.virt.addr, iv: walk.iv, nbytes: walk.nbytes);
259 else
260 sm4_ce_cbc_cts_dec(rkey: ctx->rkey_dec, dst: walk.dst.virt.addr,
261 src: walk.src.virt.addr, iv: walk.iv, nbytes: walk.nbytes);
262
263 kernel_neon_end();
264
265 return skcipher_walk_done(walk: &walk, err: 0);
266}
267
268static int sm4_cbc_cts_encrypt(struct skcipher_request *req)
269{
270 return sm4_cbc_cts_crypt(req, encrypt: true);
271}
272
273static int sm4_cbc_cts_decrypt(struct skcipher_request *req)
274{
275 return sm4_cbc_cts_crypt(req, encrypt: false);
276}
277
278static int sm4_ctr_crypt(struct skcipher_request *req)
279{
280 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
281 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
282 struct skcipher_walk walk;
283 unsigned int nbytes;
284 int err;
285
286 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
287
288 while ((nbytes = walk.nbytes) > 0) {
289 const u8 *src = walk.src.virt.addr;
290 u8 *dst = walk.dst.virt.addr;
291 unsigned int nblks;
292
293 kernel_neon_begin();
294
295 nblks = BYTES2BLKS(nbytes);
296 if (nblks) {
297 sm4_ce_ctr_enc(rkey: ctx->rkey_enc, dst, src, iv: walk.iv, nblks);
298 dst += nblks * SM4_BLOCK_SIZE;
299 src += nblks * SM4_BLOCK_SIZE;
300 nbytes -= nblks * SM4_BLOCK_SIZE;
301 }
302
303 /* tail */
304 if (walk.nbytes == walk.total && nbytes > 0) {
305 u8 keystream[SM4_BLOCK_SIZE];
306
307 sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv);
308 crypto_inc(a: walk.iv, SM4_BLOCK_SIZE);
309 crypto_xor_cpy(dst, src1: src, src2: keystream, size: nbytes);
310 nbytes = 0;
311 }
312
313 kernel_neon_end();
314
315 err = skcipher_walk_done(walk: &walk, err: nbytes);
316 }
317
318 return err;
319}
320
321static int sm4_xts_crypt(struct skcipher_request *req, bool encrypt)
322{
323 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
324 struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
325 int tail = req->cryptlen % SM4_BLOCK_SIZE;
326 const u32 *rkey2_enc = ctx->key2.rkey_enc;
327 struct scatterlist sg_src[2], sg_dst[2];
328 struct skcipher_request subreq;
329 struct scatterlist *src, *dst;
330 struct skcipher_walk walk;
331 unsigned int nbytes;
332 int err;
333
334 if (req->cryptlen < SM4_BLOCK_SIZE)
335 return -EINVAL;
336
337 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
338 if (err)
339 return err;
340
341 if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
342 int nblocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2;
343
344 skcipher_walk_abort(walk: &walk);
345
346 skcipher_request_set_tfm(req: &subreq, tfm);
347 skcipher_request_set_callback(req: &subreq,
348 flags: skcipher_request_flags(req),
349 NULL, NULL);
350 skcipher_request_set_crypt(req: &subreq, src: req->src, dst: req->dst,
351 cryptlen: nblocks * SM4_BLOCK_SIZE, iv: req->iv);
352
353 err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false);
354 if (err)
355 return err;
356 } else {
357 tail = 0;
358 }
359
360 while ((nbytes = walk.nbytes) >= SM4_BLOCK_SIZE) {
361 if (nbytes < walk.total)
362 nbytes &= ~(SM4_BLOCK_SIZE - 1);
363
364 kernel_neon_begin();
365
366 if (encrypt)
367 sm4_ce_xts_enc(rkey1: ctx->key1.rkey_enc, dst: walk.dst.virt.addr,
368 src: walk.src.virt.addr, tweak: walk.iv, nbytes,
369 rkey2_enc);
370 else
371 sm4_ce_xts_dec(rkey1: ctx->key1.rkey_dec, dst: walk.dst.virt.addr,
372 src: walk.src.virt.addr, tweak: walk.iv, nbytes,
373 rkey2_enc);
374
375 kernel_neon_end();
376
377 rkey2_enc = NULL;
378
379 err = skcipher_walk_done(walk: &walk, err: walk.nbytes - nbytes);
380 if (err)
381 return err;
382 }
383
384 if (likely(tail == 0))
385 return 0;
386
387 /* handle ciphertext stealing */
388
389 dst = src = scatterwalk_ffwd(dst: sg_src, src: req->src, len: subreq.cryptlen);
390 if (req->dst != req->src)
391 dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst, len: subreq.cryptlen);
392
393 skcipher_request_set_crypt(req: &subreq, src, dst, SM4_BLOCK_SIZE + tail,
394 iv: req->iv);
395
396 err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false);
397 if (err)
398 return err;
399
400 kernel_neon_begin();
401
402 if (encrypt)
403 sm4_ce_xts_enc(rkey1: ctx->key1.rkey_enc, dst: walk.dst.virt.addr,
404 src: walk.src.virt.addr, tweak: walk.iv, nbytes: walk.nbytes,
405 rkey2_enc);
406 else
407 sm4_ce_xts_dec(rkey1: ctx->key1.rkey_dec, dst: walk.dst.virt.addr,
408 src: walk.src.virt.addr, tweak: walk.iv, nbytes: walk.nbytes,
409 rkey2_enc);
410
411 kernel_neon_end();
412
413 return skcipher_walk_done(walk: &walk, err: 0);
414}
415
416static int sm4_xts_encrypt(struct skcipher_request *req)
417{
418 return sm4_xts_crypt(req, encrypt: true);
419}
420
421static int sm4_xts_decrypt(struct skcipher_request *req)
422{
423 return sm4_xts_crypt(req, encrypt: false);
424}
425
426static struct skcipher_alg sm4_algs[] = {
427 {
428 .base = {
429 .cra_name = "ecb(sm4)",
430 .cra_driver_name = "ecb-sm4-ce",
431 .cra_priority = 400,
432 .cra_blocksize = SM4_BLOCK_SIZE,
433 .cra_ctxsize = sizeof(struct sm4_ctx),
434 .cra_module = THIS_MODULE,
435 },
436 .min_keysize = SM4_KEY_SIZE,
437 .max_keysize = SM4_KEY_SIZE,
438 .setkey = sm4_setkey,
439 .encrypt = sm4_ecb_encrypt,
440 .decrypt = sm4_ecb_decrypt,
441 }, {
442 .base = {
443 .cra_name = "cbc(sm4)",
444 .cra_driver_name = "cbc-sm4-ce",
445 .cra_priority = 400,
446 .cra_blocksize = SM4_BLOCK_SIZE,
447 .cra_ctxsize = sizeof(struct sm4_ctx),
448 .cra_module = THIS_MODULE,
449 },
450 .min_keysize = SM4_KEY_SIZE,
451 .max_keysize = SM4_KEY_SIZE,
452 .ivsize = SM4_BLOCK_SIZE,
453 .setkey = sm4_setkey,
454 .encrypt = sm4_cbc_encrypt,
455 .decrypt = sm4_cbc_decrypt,
456 }, {
457 .base = {
458 .cra_name = "ctr(sm4)",
459 .cra_driver_name = "ctr-sm4-ce",
460 .cra_priority = 400,
461 .cra_blocksize = 1,
462 .cra_ctxsize = sizeof(struct sm4_ctx),
463 .cra_module = THIS_MODULE,
464 },
465 .min_keysize = SM4_KEY_SIZE,
466 .max_keysize = SM4_KEY_SIZE,
467 .ivsize = SM4_BLOCK_SIZE,
468 .chunksize = SM4_BLOCK_SIZE,
469 .setkey = sm4_setkey,
470 .encrypt = sm4_ctr_crypt,
471 .decrypt = sm4_ctr_crypt,
472 }, {
473 .base = {
474 .cra_name = "cts(cbc(sm4))",
475 .cra_driver_name = "cts-cbc-sm4-ce",
476 .cra_priority = 400,
477 .cra_blocksize = SM4_BLOCK_SIZE,
478 .cra_ctxsize = sizeof(struct sm4_ctx),
479 .cra_module = THIS_MODULE,
480 },
481 .min_keysize = SM4_KEY_SIZE,
482 .max_keysize = SM4_KEY_SIZE,
483 .ivsize = SM4_BLOCK_SIZE,
484 .walksize = SM4_BLOCK_SIZE * 2,
485 .setkey = sm4_setkey,
486 .encrypt = sm4_cbc_cts_encrypt,
487 .decrypt = sm4_cbc_cts_decrypt,
488 }, {
489 .base = {
490 .cra_name = "xts(sm4)",
491 .cra_driver_name = "xts-sm4-ce",
492 .cra_priority = 400,
493 .cra_blocksize = SM4_BLOCK_SIZE,
494 .cra_ctxsize = sizeof(struct sm4_xts_ctx),
495 .cra_module = THIS_MODULE,
496 },
497 .min_keysize = SM4_KEY_SIZE * 2,
498 .max_keysize = SM4_KEY_SIZE * 2,
499 .ivsize = SM4_BLOCK_SIZE,
500 .walksize = SM4_BLOCK_SIZE * 2,
501 .setkey = sm4_xts_setkey,
502 .encrypt = sm4_xts_encrypt,
503 .decrypt = sm4_xts_decrypt,
504 }
505};
506
507static int sm4_cbcmac_setkey(struct crypto_shash *tfm, const u8 *key,
508 unsigned int key_len)
509{
510 struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
511
512 if (key_len != SM4_KEY_SIZE)
513 return -EINVAL;
514
515 kernel_neon_begin();
516 sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
517 crypto_sm4_fk, crypto_sm4_ck);
518 kernel_neon_end();
519
520 return 0;
521}
522
523static int sm4_cmac_setkey(struct crypto_shash *tfm, const u8 *key,
524 unsigned int key_len)
525{
526 struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
527 be128 *consts = (be128 *)ctx->consts;
528 u64 a, b;
529
530 if (key_len != SM4_KEY_SIZE)
531 return -EINVAL;
532
533 memset(consts, 0, SM4_BLOCK_SIZE);
534
535 kernel_neon_begin();
536
537 sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
538 crypto_sm4_fk, crypto_sm4_ck);
539
540 /* encrypt the zero block */
541 sm4_ce_crypt_block(ctx->key.rkey_enc, (u8 *)consts, (const u8 *)consts);
542
543 kernel_neon_end();
544
545 /* gf(2^128) multiply zero-ciphertext with u and u^2 */
546 a = be64_to_cpu(consts[0].a);
547 b = be64_to_cpu(consts[0].b);
548 consts[0].a = cpu_to_be64((a << 1) | (b >> 63));
549 consts[0].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
550
551 a = be64_to_cpu(consts[0].a);
552 b = be64_to_cpu(consts[0].b);
553 consts[1].a = cpu_to_be64((a << 1) | (b >> 63));
554 consts[1].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
555
556 return 0;
557}
558
559static int sm4_xcbc_setkey(struct crypto_shash *tfm, const u8 *key,
560 unsigned int key_len)
561{
562 struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
563 u8 __aligned(8) key2[SM4_BLOCK_SIZE];
564 static u8 const ks[3][SM4_BLOCK_SIZE] = {
565 { [0 ... SM4_BLOCK_SIZE - 1] = 0x1},
566 { [0 ... SM4_BLOCK_SIZE - 1] = 0x2},
567 { [0 ... SM4_BLOCK_SIZE - 1] = 0x3},
568 };
569
570 if (key_len != SM4_KEY_SIZE)
571 return -EINVAL;
572
573 kernel_neon_begin();
574
575 sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
576 crypto_sm4_fk, crypto_sm4_ck);
577
578 sm4_ce_crypt_block(ctx->key.rkey_enc, key2, ks[0]);
579 sm4_ce_crypt(rkey: ctx->key.rkey_enc, dst: ctx->consts, src: ks[1], nblks: 2);
580
581 sm4_ce_expand_key(key2, ctx->key.rkey_enc, ctx->key.rkey_dec,
582 crypto_sm4_fk, crypto_sm4_ck);
583
584 kernel_neon_end();
585
586 return 0;
587}
588
589static int sm4_mac_init(struct shash_desc *desc)
590{
591 struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
592
593 memset(ctx->digest, 0, SM4_BLOCK_SIZE);
594 ctx->len = 0;
595
596 return 0;
597}
598
599static int sm4_mac_update(struct shash_desc *desc, const u8 *p,
600 unsigned int len)
601{
602 struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(tfm: desc->tfm);
603 struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
604 unsigned int l, nblocks;
605
606 if (len == 0)
607 return 0;
608
609 if (ctx->len || ctx->len + len < SM4_BLOCK_SIZE) {
610 l = min(len, SM4_BLOCK_SIZE - ctx->len);
611
612 crypto_xor(dst: ctx->digest + ctx->len, src: p, size: l);
613 ctx->len += l;
614 len -= l;
615 p += l;
616 }
617
618 if (len && (ctx->len % SM4_BLOCK_SIZE) == 0) {
619 kernel_neon_begin();
620
621 if (len < SM4_BLOCK_SIZE && ctx->len == SM4_BLOCK_SIZE) {
622 sm4_ce_crypt_block(tctx->key.rkey_enc,
623 ctx->digest, ctx->digest);
624 ctx->len = 0;
625 } else {
626 nblocks = len / SM4_BLOCK_SIZE;
627 len %= SM4_BLOCK_SIZE;
628
629 sm4_ce_mac_update(rkey_enc: tctx->key.rkey_enc, digest: ctx->digest, src: p,
630 nblocks, enc_before: (ctx->len == SM4_BLOCK_SIZE),
631 enc_after: (len != 0));
632
633 p += nblocks * SM4_BLOCK_SIZE;
634
635 if (len == 0)
636 ctx->len = SM4_BLOCK_SIZE;
637 }
638
639 kernel_neon_end();
640
641 if (len) {
642 crypto_xor(dst: ctx->digest, src: p, size: len);
643 ctx->len = len;
644 }
645 }
646
647 return 0;
648}
649
650static int sm4_cmac_final(struct shash_desc *desc, u8 *out)
651{
652 struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(tfm: desc->tfm);
653 struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
654 const u8 *consts = tctx->consts;
655
656 if (ctx->len != SM4_BLOCK_SIZE) {
657 ctx->digest[ctx->len] ^= 0x80;
658 consts += SM4_BLOCK_SIZE;
659 }
660
661 kernel_neon_begin();
662 sm4_ce_mac_update(rkey_enc: tctx->key.rkey_enc, digest: ctx->digest, src: consts, nblocks: 1,
663 enc_before: false, enc_after: true);
664 kernel_neon_end();
665
666 memcpy(out, ctx->digest, SM4_BLOCK_SIZE);
667
668 return 0;
669}
670
671static int sm4_cbcmac_final(struct shash_desc *desc, u8 *out)
672{
673 struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(tfm: desc->tfm);
674 struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
675
676 if (ctx->len) {
677 kernel_neon_begin();
678 sm4_ce_crypt_block(tctx->key.rkey_enc, ctx->digest,
679 ctx->digest);
680 kernel_neon_end();
681 }
682
683 memcpy(out, ctx->digest, SM4_BLOCK_SIZE);
684
685 return 0;
686}
687
688static struct shash_alg sm4_mac_algs[] = {
689 {
690 .base = {
691 .cra_name = "cmac(sm4)",
692 .cra_driver_name = "cmac-sm4-ce",
693 .cra_priority = 400,
694 .cra_blocksize = SM4_BLOCK_SIZE,
695 .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx)
696 + SM4_BLOCK_SIZE * 2,
697 .cra_module = THIS_MODULE,
698 },
699 .digestsize = SM4_BLOCK_SIZE,
700 .init = sm4_mac_init,
701 .update = sm4_mac_update,
702 .final = sm4_cmac_final,
703 .setkey = sm4_cmac_setkey,
704 .descsize = sizeof(struct sm4_mac_desc_ctx),
705 }, {
706 .base = {
707 .cra_name = "xcbc(sm4)",
708 .cra_driver_name = "xcbc-sm4-ce",
709 .cra_priority = 400,
710 .cra_blocksize = SM4_BLOCK_SIZE,
711 .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx)
712 + SM4_BLOCK_SIZE * 2,
713 .cra_module = THIS_MODULE,
714 },
715 .digestsize = SM4_BLOCK_SIZE,
716 .init = sm4_mac_init,
717 .update = sm4_mac_update,
718 .final = sm4_cmac_final,
719 .setkey = sm4_xcbc_setkey,
720 .descsize = sizeof(struct sm4_mac_desc_ctx),
721 }, {
722 .base = {
723 .cra_name = "cbcmac(sm4)",
724 .cra_driver_name = "cbcmac-sm4-ce",
725 .cra_priority = 400,
726 .cra_blocksize = 1,
727 .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx),
728 .cra_module = THIS_MODULE,
729 },
730 .digestsize = SM4_BLOCK_SIZE,
731 .init = sm4_mac_init,
732 .update = sm4_mac_update,
733 .final = sm4_cbcmac_final,
734 .setkey = sm4_cbcmac_setkey,
735 .descsize = sizeof(struct sm4_mac_desc_ctx),
736 }
737};
738
739static int __init sm4_init(void)
740{
741 int err;
742
743 err = crypto_register_skciphers(algs: sm4_algs, ARRAY_SIZE(sm4_algs));
744 if (err)
745 return err;
746
747 err = crypto_register_shashes(algs: sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs));
748 if (err)
749 goto out_err;
750
751 return 0;
752
753out_err:
754 crypto_unregister_skciphers(algs: sm4_algs, ARRAY_SIZE(sm4_algs));
755 return err;
756}
757
758static void __exit sm4_exit(void)
759{
760 crypto_unregister_shashes(algs: sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs));
761 crypto_unregister_skciphers(algs: sm4_algs, ARRAY_SIZE(sm4_algs));
762}
763
764module_cpu_feature_match(SM4, sm4_init);
765module_exit(sm4_exit);
766
767MODULE_DESCRIPTION("SM4 ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
768MODULE_ALIAS_CRYPTO("sm4-ce");
769MODULE_ALIAS_CRYPTO("sm4");
770MODULE_ALIAS_CRYPTO("ecb(sm4)");
771MODULE_ALIAS_CRYPTO("cbc(sm4)");
772MODULE_ALIAS_CRYPTO("ctr(sm4)");
773MODULE_ALIAS_CRYPTO("cts(cbc(sm4))");
774MODULE_ALIAS_CRYPTO("xts(sm4)");
775MODULE_ALIAS_CRYPTO("cmac(sm4)");
776MODULE_ALIAS_CRYPTO("xcbc(sm4)");
777MODULE_ALIAS_CRYPTO("cbcmac(sm4)");
778MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
779MODULE_LICENSE("GPL v2");
780

source code of linux/arch/arm64/crypto/sm4-ce-glue.c