1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | /* |
3 | * SM4 Cipher Algorithm, AES-NI/AVX2 optimized. |
4 | * as specified in |
5 | * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html |
6 | * |
7 | * Copyright (c) 2021, Alibaba Group. |
8 | * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> |
9 | */ |
10 | |
11 | #include <linux/module.h> |
12 | #include <linux/crypto.h> |
13 | #include <linux/kernel.h> |
14 | #include <asm/simd.h> |
15 | #include <crypto/internal/simd.h> |
16 | #include <crypto/internal/skcipher.h> |
17 | #include <crypto/sm4.h> |
18 | #include "sm4-avx.h" |
19 | |
20 | #define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16) |
21 | |
22 | asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst, |
23 | const u8 *src, u8 *iv); |
24 | asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst, |
25 | const u8 *src, u8 *iv); |
26 | |
27 | static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, |
28 | unsigned int key_len) |
29 | { |
30 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
31 | |
32 | return sm4_expandkey(ctx, in_key: key, key_len); |
33 | } |
34 | |
35 | static int cbc_decrypt(struct skcipher_request *req) |
36 | { |
37 | return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE, |
38 | func: sm4_aesni_avx2_cbc_dec_blk16); |
39 | } |
40 | |
41 | |
42 | static int ctr_crypt(struct skcipher_request *req) |
43 | { |
44 | return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE, |
45 | func: sm4_aesni_avx2_ctr_enc_blk16); |
46 | } |
47 | |
48 | static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { |
49 | { |
50 | .base = { |
51 | .cra_name = "__ecb(sm4)" , |
52 | .cra_driver_name = "__ecb-sm4-aesni-avx2" , |
53 | .cra_priority = 500, |
54 | .cra_flags = CRYPTO_ALG_INTERNAL, |
55 | .cra_blocksize = SM4_BLOCK_SIZE, |
56 | .cra_ctxsize = sizeof(struct sm4_ctx), |
57 | .cra_module = THIS_MODULE, |
58 | }, |
59 | .min_keysize = SM4_KEY_SIZE, |
60 | .max_keysize = SM4_KEY_SIZE, |
61 | .walksize = 16 * SM4_BLOCK_SIZE, |
62 | .setkey = sm4_skcipher_setkey, |
63 | .encrypt = sm4_avx_ecb_encrypt, |
64 | .decrypt = sm4_avx_ecb_decrypt, |
65 | }, { |
66 | .base = { |
67 | .cra_name = "__cbc(sm4)" , |
68 | .cra_driver_name = "__cbc-sm4-aesni-avx2" , |
69 | .cra_priority = 500, |
70 | .cra_flags = CRYPTO_ALG_INTERNAL, |
71 | .cra_blocksize = SM4_BLOCK_SIZE, |
72 | .cra_ctxsize = sizeof(struct sm4_ctx), |
73 | .cra_module = THIS_MODULE, |
74 | }, |
75 | .min_keysize = SM4_KEY_SIZE, |
76 | .max_keysize = SM4_KEY_SIZE, |
77 | .ivsize = SM4_BLOCK_SIZE, |
78 | .walksize = 16 * SM4_BLOCK_SIZE, |
79 | .setkey = sm4_skcipher_setkey, |
80 | .encrypt = sm4_cbc_encrypt, |
81 | .decrypt = cbc_decrypt, |
82 | }, { |
83 | .base = { |
84 | .cra_name = "__ctr(sm4)" , |
85 | .cra_driver_name = "__ctr-sm4-aesni-avx2" , |
86 | .cra_priority = 500, |
87 | .cra_flags = CRYPTO_ALG_INTERNAL, |
88 | .cra_blocksize = 1, |
89 | .cra_ctxsize = sizeof(struct sm4_ctx), |
90 | .cra_module = THIS_MODULE, |
91 | }, |
92 | .min_keysize = SM4_KEY_SIZE, |
93 | .max_keysize = SM4_KEY_SIZE, |
94 | .ivsize = SM4_BLOCK_SIZE, |
95 | .chunksize = SM4_BLOCK_SIZE, |
96 | .walksize = 16 * SM4_BLOCK_SIZE, |
97 | .setkey = sm4_skcipher_setkey, |
98 | .encrypt = ctr_crypt, |
99 | .decrypt = ctr_crypt, |
100 | } |
101 | }; |
102 | |
103 | static struct simd_skcipher_alg * |
104 | simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)]; |
105 | |
106 | static int __init sm4_init(void) |
107 | { |
108 | const char *feature_name; |
109 | |
110 | if (!boot_cpu_has(X86_FEATURE_AVX) || |
111 | !boot_cpu_has(X86_FEATURE_AVX2) || |
112 | !boot_cpu_has(X86_FEATURE_AES) || |
113 | !boot_cpu_has(X86_FEATURE_OSXSAVE)) { |
114 | pr_info("AVX2 or AES-NI instructions are not detected.\n" ); |
115 | return -ENODEV; |
116 | } |
117 | |
118 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, |
119 | feature_name: &feature_name)) { |
120 | pr_info("CPU feature '%s' is not supported.\n" , feature_name); |
121 | return -ENODEV; |
122 | } |
123 | |
124 | return simd_register_skciphers_compat(algs: sm4_aesni_avx2_skciphers, |
125 | ARRAY_SIZE(sm4_aesni_avx2_skciphers), |
126 | simd_algs: simd_sm4_aesni_avx2_skciphers); |
127 | } |
128 | |
129 | static void __exit sm4_exit(void) |
130 | { |
131 | simd_unregister_skciphers(algs: sm4_aesni_avx2_skciphers, |
132 | ARRAY_SIZE(sm4_aesni_avx2_skciphers), |
133 | simd_algs: simd_sm4_aesni_avx2_skciphers); |
134 | } |
135 | |
136 | module_init(sm4_init); |
137 | module_exit(sm4_exit); |
138 | |
139 | MODULE_LICENSE("GPL v2" ); |
140 | MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>" ); |
141 | MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized" ); |
142 | MODULE_ALIAS_CRYPTO("sm4" ); |
143 | MODULE_ALIAS_CRYPTO("sm4-aesni-avx2" ); |
144 | |