1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | /* |
3 | * Glue Code for the AVX512/GFNI assembler implementation of the ARIA Cipher |
4 | * |
5 | * Copyright (c) 2022 Taehee Yoo <ap420073@gmail.com> |
6 | */ |
7 | |
8 | #include <crypto/algapi.h> |
9 | #include <crypto/internal/simd.h> |
10 | #include <crypto/aria.h> |
11 | #include <linux/crypto.h> |
12 | #include <linux/err.h> |
13 | #include <linux/module.h> |
14 | #include <linux/types.h> |
15 | |
16 | #include "ecb_cbc_helpers.h" |
17 | #include "aria-avx.h" |
18 | |
19 | asmlinkage void aria_gfni_avx512_encrypt_64way(const void *ctx, u8 *dst, |
20 | const u8 *src); |
21 | asmlinkage void aria_gfni_avx512_decrypt_64way(const void *ctx, u8 *dst, |
22 | const u8 *src); |
23 | asmlinkage void aria_gfni_avx512_ctr_crypt_64way(const void *ctx, u8 *dst, |
24 | const u8 *src, |
25 | u8 *keystream, u8 *iv); |
26 | |
27 | static struct aria_avx_ops aria_ops; |
28 | |
29 | struct aria_avx512_request_ctx { |
30 | u8 keystream[ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE]; |
31 | }; |
32 | |
33 | static int ecb_do_encrypt(struct skcipher_request *req, const u32 *rkey) |
34 | { |
35 | ECB_WALK_START(req, ARIA_BLOCK_SIZE, ARIA_AESNI_PARALLEL_BLOCKS); |
36 | ECB_BLOCK(ARIA_GFNI_AVX512_PARALLEL_BLOCKS, aria_ops.aria_encrypt_64way); |
37 | ECB_BLOCK(ARIA_AESNI_AVX2_PARALLEL_BLOCKS, aria_ops.aria_encrypt_32way); |
38 | ECB_BLOCK(ARIA_AESNI_PARALLEL_BLOCKS, aria_ops.aria_encrypt_16way); |
39 | ECB_BLOCK(1, aria_encrypt); |
40 | ECB_WALK_END(); |
41 | } |
42 | |
43 | static int ecb_do_decrypt(struct skcipher_request *req, const u32 *rkey) |
44 | { |
45 | ECB_WALK_START(req, ARIA_BLOCK_SIZE, ARIA_AESNI_PARALLEL_BLOCKS); |
46 | ECB_BLOCK(ARIA_GFNI_AVX512_PARALLEL_BLOCKS, aria_ops.aria_decrypt_64way); |
47 | ECB_BLOCK(ARIA_AESNI_AVX2_PARALLEL_BLOCKS, aria_ops.aria_decrypt_32way); |
48 | ECB_BLOCK(ARIA_AESNI_PARALLEL_BLOCKS, aria_ops.aria_decrypt_16way); |
49 | ECB_BLOCK(1, aria_decrypt); |
50 | ECB_WALK_END(); |
51 | } |
52 | |
53 | static int aria_avx512_ecb_encrypt(struct skcipher_request *req) |
54 | { |
55 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
56 | struct aria_ctx *ctx = crypto_skcipher_ctx(tfm); |
57 | |
58 | return ecb_do_encrypt(req, rkey: ctx->enc_key[0]); |
59 | } |
60 | |
61 | static int aria_avx512_ecb_decrypt(struct skcipher_request *req) |
62 | { |
63 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
64 | struct aria_ctx *ctx = crypto_skcipher_ctx(tfm); |
65 | |
66 | return ecb_do_decrypt(req, rkey: ctx->dec_key[0]); |
67 | } |
68 | |
69 | static int aria_avx512_set_key(struct crypto_skcipher *tfm, const u8 *key, |
70 | unsigned int keylen) |
71 | { |
72 | return aria_set_key(tfm: &tfm->base, in_key: key, key_len: keylen); |
73 | } |
74 | |
75 | static int aria_avx512_ctr_encrypt(struct skcipher_request *req) |
76 | { |
77 | struct aria_avx512_request_ctx *req_ctx = skcipher_request_ctx(req); |
78 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
79 | struct aria_ctx *ctx = crypto_skcipher_ctx(tfm); |
80 | struct skcipher_walk walk; |
81 | unsigned int nbytes; |
82 | int err; |
83 | |
84 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
85 | |
86 | while ((nbytes = walk.nbytes) > 0) { |
87 | const u8 *src = walk.src.virt.addr; |
88 | u8 *dst = walk.dst.virt.addr; |
89 | |
90 | while (nbytes >= ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE) { |
91 | kernel_fpu_begin(); |
92 | aria_ops.aria_ctr_crypt_64way(ctx, dst, src, |
93 | &req_ctx->keystream[0], |
94 | walk.iv); |
95 | kernel_fpu_end(); |
96 | dst += ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE; |
97 | src += ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE; |
98 | nbytes -= ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE; |
99 | } |
100 | |
101 | while (nbytes >= ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE) { |
102 | kernel_fpu_begin(); |
103 | aria_ops.aria_ctr_crypt_32way(ctx, dst, src, |
104 | &req_ctx->keystream[0], |
105 | walk.iv); |
106 | kernel_fpu_end(); |
107 | dst += ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE; |
108 | src += ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE; |
109 | nbytes -= ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE; |
110 | } |
111 | |
112 | while (nbytes >= ARIA_AESNI_PARALLEL_BLOCK_SIZE) { |
113 | kernel_fpu_begin(); |
114 | aria_ops.aria_ctr_crypt_16way(ctx, dst, src, |
115 | &req_ctx->keystream[0], |
116 | walk.iv); |
117 | kernel_fpu_end(); |
118 | dst += ARIA_AESNI_PARALLEL_BLOCK_SIZE; |
119 | src += ARIA_AESNI_PARALLEL_BLOCK_SIZE; |
120 | nbytes -= ARIA_AESNI_PARALLEL_BLOCK_SIZE; |
121 | } |
122 | |
123 | while (nbytes >= ARIA_BLOCK_SIZE) { |
124 | memcpy(&req_ctx->keystream[0], walk.iv, |
125 | ARIA_BLOCK_SIZE); |
126 | crypto_inc(a: walk.iv, ARIA_BLOCK_SIZE); |
127 | |
128 | aria_encrypt(ctx, out: &req_ctx->keystream[0], |
129 | in: &req_ctx->keystream[0]); |
130 | |
131 | crypto_xor_cpy(dst, src1: src, src2: &req_ctx->keystream[0], |
132 | ARIA_BLOCK_SIZE); |
133 | dst += ARIA_BLOCK_SIZE; |
134 | src += ARIA_BLOCK_SIZE; |
135 | nbytes -= ARIA_BLOCK_SIZE; |
136 | } |
137 | |
138 | if (walk.nbytes == walk.total && nbytes > 0) { |
139 | memcpy(&req_ctx->keystream[0], walk.iv, |
140 | ARIA_BLOCK_SIZE); |
141 | crypto_inc(a: walk.iv, ARIA_BLOCK_SIZE); |
142 | |
143 | aria_encrypt(ctx, out: &req_ctx->keystream[0], |
144 | in: &req_ctx->keystream[0]); |
145 | |
146 | crypto_xor_cpy(dst, src1: src, src2: &req_ctx->keystream[0], |
147 | size: nbytes); |
148 | dst += nbytes; |
149 | src += nbytes; |
150 | nbytes = 0; |
151 | } |
152 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
153 | } |
154 | |
155 | return err; |
156 | } |
157 | |
158 | static int aria_avx512_init_tfm(struct crypto_skcipher *tfm) |
159 | { |
160 | crypto_skcipher_set_reqsize(skcipher: tfm, |
161 | reqsize: sizeof(struct aria_avx512_request_ctx)); |
162 | |
163 | return 0; |
164 | } |
165 | |
166 | static struct skcipher_alg aria_algs[] = { |
167 | { |
168 | .base.cra_name = "__ecb(aria)" , |
169 | .base.cra_driver_name = "__ecb-aria-avx512" , |
170 | .base.cra_priority = 600, |
171 | .base.cra_flags = CRYPTO_ALG_INTERNAL, |
172 | .base.cra_blocksize = ARIA_BLOCK_SIZE, |
173 | .base.cra_ctxsize = sizeof(struct aria_ctx), |
174 | .base.cra_module = THIS_MODULE, |
175 | .min_keysize = ARIA_MIN_KEY_SIZE, |
176 | .max_keysize = ARIA_MAX_KEY_SIZE, |
177 | .setkey = aria_avx512_set_key, |
178 | .encrypt = aria_avx512_ecb_encrypt, |
179 | .decrypt = aria_avx512_ecb_decrypt, |
180 | }, { |
181 | .base.cra_name = "__ctr(aria)" , |
182 | .base.cra_driver_name = "__ctr-aria-avx512" , |
183 | .base.cra_priority = 600, |
184 | .base.cra_flags = CRYPTO_ALG_INTERNAL | |
185 | CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE, |
186 | .base.cra_blocksize = 1, |
187 | .base.cra_ctxsize = sizeof(struct aria_ctx), |
188 | .base.cra_module = THIS_MODULE, |
189 | .min_keysize = ARIA_MIN_KEY_SIZE, |
190 | .max_keysize = ARIA_MAX_KEY_SIZE, |
191 | .ivsize = ARIA_BLOCK_SIZE, |
192 | .chunksize = ARIA_BLOCK_SIZE, |
193 | .setkey = aria_avx512_set_key, |
194 | .encrypt = aria_avx512_ctr_encrypt, |
195 | .decrypt = aria_avx512_ctr_encrypt, |
196 | .init = aria_avx512_init_tfm, |
197 | } |
198 | }; |
199 | |
200 | static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)]; |
201 | |
202 | static int __init aria_avx512_init(void) |
203 | { |
204 | const char *feature_name; |
205 | |
206 | if (!boot_cpu_has(X86_FEATURE_AVX) || |
207 | !boot_cpu_has(X86_FEATURE_AVX2) || |
208 | !boot_cpu_has(X86_FEATURE_AVX512F) || |
209 | !boot_cpu_has(X86_FEATURE_AVX512VL) || |
210 | !boot_cpu_has(X86_FEATURE_GFNI) || |
211 | !boot_cpu_has(X86_FEATURE_OSXSAVE)) { |
212 | pr_info("AVX512/GFNI instructions are not detected.\n" ); |
213 | return -ENODEV; |
214 | } |
215 | |
216 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | |
217 | XFEATURE_MASK_AVX512, feature_name: &feature_name)) { |
218 | pr_info("CPU feature '%s' is not supported.\n" , feature_name); |
219 | return -ENODEV; |
220 | } |
221 | |
222 | aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way; |
223 | aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way; |
224 | aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_gfni_ctr_crypt_16way; |
225 | aria_ops.aria_encrypt_32way = aria_aesni_avx2_gfni_encrypt_32way; |
226 | aria_ops.aria_decrypt_32way = aria_aesni_avx2_gfni_decrypt_32way; |
227 | aria_ops.aria_ctr_crypt_32way = aria_aesni_avx2_gfni_ctr_crypt_32way; |
228 | aria_ops.aria_encrypt_64way = aria_gfni_avx512_encrypt_64way; |
229 | aria_ops.aria_decrypt_64way = aria_gfni_avx512_decrypt_64way; |
230 | aria_ops.aria_ctr_crypt_64way = aria_gfni_avx512_ctr_crypt_64way; |
231 | |
232 | return simd_register_skciphers_compat(algs: aria_algs, |
233 | ARRAY_SIZE(aria_algs), |
234 | simd_algs: aria_simd_algs); |
235 | } |
236 | |
237 | static void __exit aria_avx512_exit(void) |
238 | { |
239 | simd_unregister_skciphers(algs: aria_algs, ARRAY_SIZE(aria_algs), |
240 | simd_algs: aria_simd_algs); |
241 | } |
242 | |
243 | module_init(aria_avx512_init); |
244 | module_exit(aria_avx512_exit); |
245 | |
246 | MODULE_LICENSE("GPL" ); |
247 | MODULE_AUTHOR("Taehee Yoo <ap420073@gmail.com>" ); |
248 | MODULE_DESCRIPTION("ARIA Cipher Algorithm, AVX512/GFNI optimized" ); |
249 | MODULE_ALIAS_CRYPTO("aria" ); |
250 | MODULE_ALIAS_CRYPTO("aria-gfni-avx512" ); |
251 | |