1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | /* |
3 | * SM3 Secure Hash Algorithm, AVX assembler accelerated. |
4 | * specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02 |
5 | * |
6 | * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> |
7 | */ |
8 | |
9 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
10 | |
11 | #include <crypto/internal/hash.h> |
12 | #include <crypto/internal/simd.h> |
13 | #include <linux/init.h> |
14 | #include <linux/module.h> |
15 | #include <linux/types.h> |
16 | #include <crypto/sm3.h> |
17 | #include <crypto/sm3_base.h> |
18 | #include <asm/simd.h> |
19 | |
20 | asmlinkage void sm3_transform_avx(struct sm3_state *state, |
21 | const u8 *data, int nblocks); |
22 | |
23 | static int sm3_avx_update(struct shash_desc *desc, const u8 *data, |
24 | unsigned int len) |
25 | { |
26 | struct sm3_state *sctx = shash_desc_ctx(desc); |
27 | |
28 | if (!crypto_simd_usable() || |
29 | (sctx->count % SM3_BLOCK_SIZE) + len < SM3_BLOCK_SIZE) { |
30 | sm3_update(sctx, data, len); |
31 | return 0; |
32 | } |
33 | |
34 | /* |
35 | * Make sure struct sm3_state begins directly with the SM3 |
36 | * 256-bit internal state, as this is what the asm functions expect. |
37 | */ |
38 | BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0); |
39 | |
40 | kernel_fpu_begin(); |
41 | sm3_base_do_update(desc, data, len, block_fn: sm3_transform_avx); |
42 | kernel_fpu_end(); |
43 | |
44 | return 0; |
45 | } |
46 | |
47 | static int sm3_avx_finup(struct shash_desc *desc, const u8 *data, |
48 | unsigned int len, u8 *out) |
49 | { |
50 | if (!crypto_simd_usable()) { |
51 | struct sm3_state *sctx = shash_desc_ctx(desc); |
52 | |
53 | if (len) |
54 | sm3_update(sctx, data, len); |
55 | |
56 | sm3_final(sctx, out); |
57 | return 0; |
58 | } |
59 | |
60 | kernel_fpu_begin(); |
61 | if (len) |
62 | sm3_base_do_update(desc, data, len, block_fn: sm3_transform_avx); |
63 | sm3_base_do_finalize(desc, block_fn: sm3_transform_avx); |
64 | kernel_fpu_end(); |
65 | |
66 | return sm3_base_finish(desc, out); |
67 | } |
68 | |
69 | static int sm3_avx_final(struct shash_desc *desc, u8 *out) |
70 | { |
71 | if (!crypto_simd_usable()) { |
72 | sm3_final(sctx: shash_desc_ctx(desc), out); |
73 | return 0; |
74 | } |
75 | |
76 | kernel_fpu_begin(); |
77 | sm3_base_do_finalize(desc, block_fn: sm3_transform_avx); |
78 | kernel_fpu_end(); |
79 | |
80 | return sm3_base_finish(desc, out); |
81 | } |
82 | |
83 | static struct shash_alg sm3_avx_alg = { |
84 | .digestsize = SM3_DIGEST_SIZE, |
85 | .init = sm3_base_init, |
86 | .update = sm3_avx_update, |
87 | .final = sm3_avx_final, |
88 | .finup = sm3_avx_finup, |
89 | .descsize = sizeof(struct sm3_state), |
90 | .base = { |
91 | .cra_name = "sm3" , |
92 | .cra_driver_name = "sm3-avx" , |
93 | .cra_priority = 300, |
94 | .cra_blocksize = SM3_BLOCK_SIZE, |
95 | .cra_module = THIS_MODULE, |
96 | } |
97 | }; |
98 | |
99 | static int __init sm3_avx_mod_init(void) |
100 | { |
101 | const char *feature_name; |
102 | |
103 | if (!boot_cpu_has(X86_FEATURE_AVX)) { |
104 | pr_info("AVX instruction are not detected.\n" ); |
105 | return -ENODEV; |
106 | } |
107 | |
108 | if (!boot_cpu_has(X86_FEATURE_BMI2)) { |
109 | pr_info("BMI2 instruction are not detected.\n" ); |
110 | return -ENODEV; |
111 | } |
112 | |
113 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, |
114 | feature_name: &feature_name)) { |
115 | pr_info("CPU feature '%s' is not supported.\n" , feature_name); |
116 | return -ENODEV; |
117 | } |
118 | |
119 | return crypto_register_shash(alg: &sm3_avx_alg); |
120 | } |
121 | |
122 | static void __exit sm3_avx_mod_exit(void) |
123 | { |
124 | crypto_unregister_shash(alg: &sm3_avx_alg); |
125 | } |
126 | |
127 | module_init(sm3_avx_mod_init); |
128 | module_exit(sm3_avx_mod_exit); |
129 | |
130 | MODULE_LICENSE("GPL v2" ); |
131 | MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>" ); |
132 | MODULE_DESCRIPTION("SM3 Secure Hash Algorithm, AVX assembler accelerated" ); |
133 | MODULE_ALIAS_CRYPTO("sm3" ); |
134 | MODULE_ALIAS_CRYPTO("sm3-avx" ); |
135 | |