1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Cryptographic API. |
4 | * |
5 | * Glue code for the SHA1 Secure Hash Algorithm assembler implementations |
6 | * using SSSE3, AVX, AVX2, and SHA-NI instructions. |
7 | * |
8 | * This file is based on sha1_generic.c |
9 | * |
10 | * Copyright (c) Alan Smithee. |
11 | * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> |
12 | * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> |
13 | * Copyright (c) Mathias Krause <minipli@googlemail.com> |
14 | * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com> |
15 | */ |
16 | |
17 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
18 | |
19 | #include <crypto/internal/hash.h> |
20 | #include <crypto/internal/simd.h> |
21 | #include <linux/init.h> |
22 | #include <linux/module.h> |
23 | #include <linux/mm.h> |
24 | #include <linux/types.h> |
25 | #include <crypto/sha1.h> |
26 | #include <crypto/sha1_base.h> |
27 | #include <asm/cpu_device_id.h> |
28 | #include <asm/simd.h> |
29 | |
30 | static const struct x86_cpu_id module_cpu_ids[] = { |
31 | #ifdef CONFIG_AS_SHA1_NI |
32 | X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL), |
33 | #endif |
34 | X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), |
35 | X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), |
36 | X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), |
37 | {} |
38 | }; |
39 | MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); |
40 | |
41 | static int sha1_update(struct shash_desc *desc, const u8 *data, |
42 | unsigned int len, sha1_block_fn *sha1_xform) |
43 | { |
44 | struct sha1_state *sctx = shash_desc_ctx(desc); |
45 | |
46 | if (!crypto_simd_usable() || |
47 | (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) |
48 | return crypto_sha1_update(desc, data, len); |
49 | |
50 | /* |
51 | * Make sure struct sha1_state begins directly with the SHA1 |
52 | * 160-bit internal state, as this is what the asm functions expect. |
53 | */ |
54 | BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); |
55 | |
56 | kernel_fpu_begin(); |
57 | sha1_base_do_update(desc, data, len, block_fn: sha1_xform); |
58 | kernel_fpu_end(); |
59 | |
60 | return 0; |
61 | } |
62 | |
63 | static int sha1_finup(struct shash_desc *desc, const u8 *data, |
64 | unsigned int len, u8 *out, sha1_block_fn *sha1_xform) |
65 | { |
66 | if (!crypto_simd_usable()) |
67 | return crypto_sha1_finup(desc, data, len, hash: out); |
68 | |
69 | kernel_fpu_begin(); |
70 | if (len) |
71 | sha1_base_do_update(desc, data, len, block_fn: sha1_xform); |
72 | sha1_base_do_finalize(desc, block_fn: sha1_xform); |
73 | kernel_fpu_end(); |
74 | |
75 | return sha1_base_finish(desc, out); |
76 | } |
77 | |
78 | asmlinkage void sha1_transform_ssse3(struct sha1_state *state, |
79 | const u8 *data, int blocks); |
80 | |
81 | static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data, |
82 | unsigned int len) |
83 | { |
84 | return sha1_update(desc, data, len, sha1_xform: sha1_transform_ssse3); |
85 | } |
86 | |
87 | static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data, |
88 | unsigned int len, u8 *out) |
89 | { |
90 | return sha1_finup(desc, data, len, out, sha1_xform: sha1_transform_ssse3); |
91 | } |
92 | |
93 | /* Add padding and return the message digest. */ |
94 | static int sha1_ssse3_final(struct shash_desc *desc, u8 *out) |
95 | { |
96 | return sha1_ssse3_finup(desc, NULL, len: 0, out); |
97 | } |
98 | |
99 | static struct shash_alg sha1_ssse3_alg = { |
100 | .digestsize = SHA1_DIGEST_SIZE, |
101 | .init = sha1_base_init, |
102 | .update = sha1_ssse3_update, |
103 | .final = sha1_ssse3_final, |
104 | .finup = sha1_ssse3_finup, |
105 | .descsize = sizeof(struct sha1_state), |
106 | .base = { |
107 | .cra_name = "sha1" , |
108 | .cra_driver_name = "sha1-ssse3" , |
109 | .cra_priority = 150, |
110 | .cra_blocksize = SHA1_BLOCK_SIZE, |
111 | .cra_module = THIS_MODULE, |
112 | } |
113 | }; |
114 | |
115 | static int register_sha1_ssse3(void) |
116 | { |
117 | if (boot_cpu_has(X86_FEATURE_SSSE3)) |
118 | return crypto_register_shash(alg: &sha1_ssse3_alg); |
119 | return 0; |
120 | } |
121 | |
122 | static void unregister_sha1_ssse3(void) |
123 | { |
124 | if (boot_cpu_has(X86_FEATURE_SSSE3)) |
125 | crypto_unregister_shash(alg: &sha1_ssse3_alg); |
126 | } |
127 | |
128 | asmlinkage void sha1_transform_avx(struct sha1_state *state, |
129 | const u8 *data, int blocks); |
130 | |
131 | static int sha1_avx_update(struct shash_desc *desc, const u8 *data, |
132 | unsigned int len) |
133 | { |
134 | return sha1_update(desc, data, len, sha1_xform: sha1_transform_avx); |
135 | } |
136 | |
137 | static int sha1_avx_finup(struct shash_desc *desc, const u8 *data, |
138 | unsigned int len, u8 *out) |
139 | { |
140 | return sha1_finup(desc, data, len, out, sha1_xform: sha1_transform_avx); |
141 | } |
142 | |
143 | static int sha1_avx_final(struct shash_desc *desc, u8 *out) |
144 | { |
145 | return sha1_avx_finup(desc, NULL, len: 0, out); |
146 | } |
147 | |
148 | static struct shash_alg sha1_avx_alg = { |
149 | .digestsize = SHA1_DIGEST_SIZE, |
150 | .init = sha1_base_init, |
151 | .update = sha1_avx_update, |
152 | .final = sha1_avx_final, |
153 | .finup = sha1_avx_finup, |
154 | .descsize = sizeof(struct sha1_state), |
155 | .base = { |
156 | .cra_name = "sha1" , |
157 | .cra_driver_name = "sha1-avx" , |
158 | .cra_priority = 160, |
159 | .cra_blocksize = SHA1_BLOCK_SIZE, |
160 | .cra_module = THIS_MODULE, |
161 | } |
162 | }; |
163 | |
164 | static bool avx_usable(void) |
165 | { |
166 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { |
167 | if (boot_cpu_has(X86_FEATURE_AVX)) |
168 | pr_info("AVX detected but unusable.\n" ); |
169 | return false; |
170 | } |
171 | |
172 | return true; |
173 | } |
174 | |
175 | static int register_sha1_avx(void) |
176 | { |
177 | if (avx_usable()) |
178 | return crypto_register_shash(alg: &sha1_avx_alg); |
179 | return 0; |
180 | } |
181 | |
182 | static void unregister_sha1_avx(void) |
183 | { |
184 | if (avx_usable()) |
185 | crypto_unregister_shash(alg: &sha1_avx_alg); |
186 | } |
187 | |
188 | #define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */ |
189 | |
190 | asmlinkage void sha1_transform_avx2(struct sha1_state *state, |
191 | const u8 *data, int blocks); |
192 | |
193 | static bool avx2_usable(void) |
194 | { |
195 | if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) |
196 | && boot_cpu_has(X86_FEATURE_BMI1) |
197 | && boot_cpu_has(X86_FEATURE_BMI2)) |
198 | return true; |
199 | |
200 | return false; |
201 | } |
202 | |
203 | static void sha1_apply_transform_avx2(struct sha1_state *state, |
204 | const u8 *data, int blocks) |
205 | { |
206 | /* Select the optimal transform based on data block size */ |
207 | if (blocks >= SHA1_AVX2_BLOCK_OPTSIZE) |
208 | sha1_transform_avx2(state, data, blocks); |
209 | else |
210 | sha1_transform_avx(state, data, blocks); |
211 | } |
212 | |
213 | static int sha1_avx2_update(struct shash_desc *desc, const u8 *data, |
214 | unsigned int len) |
215 | { |
216 | return sha1_update(desc, data, len, sha1_xform: sha1_apply_transform_avx2); |
217 | } |
218 | |
219 | static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data, |
220 | unsigned int len, u8 *out) |
221 | { |
222 | return sha1_finup(desc, data, len, out, sha1_xform: sha1_apply_transform_avx2); |
223 | } |
224 | |
225 | static int sha1_avx2_final(struct shash_desc *desc, u8 *out) |
226 | { |
227 | return sha1_avx2_finup(desc, NULL, len: 0, out); |
228 | } |
229 | |
230 | static struct shash_alg sha1_avx2_alg = { |
231 | .digestsize = SHA1_DIGEST_SIZE, |
232 | .init = sha1_base_init, |
233 | .update = sha1_avx2_update, |
234 | .final = sha1_avx2_final, |
235 | .finup = sha1_avx2_finup, |
236 | .descsize = sizeof(struct sha1_state), |
237 | .base = { |
238 | .cra_name = "sha1" , |
239 | .cra_driver_name = "sha1-avx2" , |
240 | .cra_priority = 170, |
241 | .cra_blocksize = SHA1_BLOCK_SIZE, |
242 | .cra_module = THIS_MODULE, |
243 | } |
244 | }; |
245 | |
246 | static int register_sha1_avx2(void) |
247 | { |
248 | if (avx2_usable()) |
249 | return crypto_register_shash(alg: &sha1_avx2_alg); |
250 | return 0; |
251 | } |
252 | |
253 | static void unregister_sha1_avx2(void) |
254 | { |
255 | if (avx2_usable()) |
256 | crypto_unregister_shash(alg: &sha1_avx2_alg); |
257 | } |
258 | |
259 | #ifdef CONFIG_AS_SHA1_NI |
260 | asmlinkage void sha1_ni_transform(struct sha1_state *digest, const u8 *data, |
261 | int rounds); |
262 | |
263 | static int sha1_ni_update(struct shash_desc *desc, const u8 *data, |
264 | unsigned int len) |
265 | { |
266 | return sha1_update(desc, data, len, sha1_xform: sha1_ni_transform); |
267 | } |
268 | |
269 | static int sha1_ni_finup(struct shash_desc *desc, const u8 *data, |
270 | unsigned int len, u8 *out) |
271 | { |
272 | return sha1_finup(desc, data, len, out, sha1_xform: sha1_ni_transform); |
273 | } |
274 | |
275 | static int sha1_ni_final(struct shash_desc *desc, u8 *out) |
276 | { |
277 | return sha1_ni_finup(desc, NULL, len: 0, out); |
278 | } |
279 | |
280 | static struct shash_alg sha1_ni_alg = { |
281 | .digestsize = SHA1_DIGEST_SIZE, |
282 | .init = sha1_base_init, |
283 | .update = sha1_ni_update, |
284 | .final = sha1_ni_final, |
285 | .finup = sha1_ni_finup, |
286 | .descsize = sizeof(struct sha1_state), |
287 | .base = { |
288 | .cra_name = "sha1" , |
289 | .cra_driver_name = "sha1-ni" , |
290 | .cra_priority = 250, |
291 | .cra_blocksize = SHA1_BLOCK_SIZE, |
292 | .cra_module = THIS_MODULE, |
293 | } |
294 | }; |
295 | |
296 | static int register_sha1_ni(void) |
297 | { |
298 | if (boot_cpu_has(X86_FEATURE_SHA_NI)) |
299 | return crypto_register_shash(alg: &sha1_ni_alg); |
300 | return 0; |
301 | } |
302 | |
303 | static void unregister_sha1_ni(void) |
304 | { |
305 | if (boot_cpu_has(X86_FEATURE_SHA_NI)) |
306 | crypto_unregister_shash(alg: &sha1_ni_alg); |
307 | } |
308 | |
309 | #else |
310 | static inline int register_sha1_ni(void) { return 0; } |
311 | static inline void unregister_sha1_ni(void) { } |
312 | #endif |
313 | |
314 | static int __init sha1_ssse3_mod_init(void) |
315 | { |
316 | if (!x86_match_cpu(match: module_cpu_ids)) |
317 | return -ENODEV; |
318 | |
319 | if (register_sha1_ssse3()) |
320 | goto fail; |
321 | |
322 | if (register_sha1_avx()) { |
323 | unregister_sha1_ssse3(); |
324 | goto fail; |
325 | } |
326 | |
327 | if (register_sha1_avx2()) { |
328 | unregister_sha1_avx(); |
329 | unregister_sha1_ssse3(); |
330 | goto fail; |
331 | } |
332 | |
333 | if (register_sha1_ni()) { |
334 | unregister_sha1_avx2(); |
335 | unregister_sha1_avx(); |
336 | unregister_sha1_ssse3(); |
337 | goto fail; |
338 | } |
339 | |
340 | return 0; |
341 | fail: |
342 | return -ENODEV; |
343 | } |
344 | |
345 | static void __exit sha1_ssse3_mod_fini(void) |
346 | { |
347 | unregister_sha1_ni(); |
348 | unregister_sha1_avx2(); |
349 | unregister_sha1_avx(); |
350 | unregister_sha1_ssse3(); |
351 | } |
352 | |
353 | module_init(sha1_ssse3_mod_init); |
354 | module_exit(sha1_ssse3_mod_fini); |
355 | |
356 | MODULE_LICENSE("GPL" ); |
357 | MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated" ); |
358 | |
359 | MODULE_ALIAS_CRYPTO("sha1" ); |
360 | MODULE_ALIAS_CRYPTO("sha1-ssse3" ); |
361 | MODULE_ALIAS_CRYPTO("sha1-avx" ); |
362 | MODULE_ALIAS_CRYPTO("sha1-avx2" ); |
363 | #ifdef CONFIG_AS_SHA1_NI |
364 | MODULE_ALIAS_CRYPTO("sha1-ni" ); |
365 | #endif |
366 | |