1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Glue Code for x86_64/AVX2 assembler optimized version of Serpent |
4 | * |
5 | * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> |
6 | */ |
7 | |
8 | #include <linux/module.h> |
9 | #include <linux/types.h> |
10 | #include <linux/crypto.h> |
11 | #include <linux/err.h> |
12 | #include <crypto/algapi.h> |
13 | #include <crypto/internal/simd.h> |
14 | #include <crypto/serpent.h> |
15 | |
16 | #include "serpent-avx.h" |
17 | #include "ecb_cbc_helpers.h" |
18 | |
19 | #define SERPENT_AVX2_PARALLEL_BLOCKS 16 |
20 | |
21 | /* 16-way AVX2 parallel cipher functions */ |
22 | asmlinkage void serpent_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src); |
23 | asmlinkage void serpent_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src); |
24 | asmlinkage void serpent_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src); |
25 | |
26 | static int serpent_setkey_skcipher(struct crypto_skcipher *tfm, |
27 | const u8 *key, unsigned int keylen) |
28 | { |
29 | return __serpent_setkey(ctx: crypto_skcipher_ctx(tfm), key, keylen); |
30 | } |
31 | |
32 | static int ecb_encrypt(struct skcipher_request *req) |
33 | { |
34 | ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); |
35 | ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_enc_16way); |
36 | ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_enc_8way_avx); |
37 | ECB_BLOCK(1, __serpent_encrypt); |
38 | ECB_WALK_END(); |
39 | } |
40 | |
41 | static int ecb_decrypt(struct skcipher_request *req) |
42 | { |
43 | ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); |
44 | ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_dec_16way); |
45 | ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_dec_8way_avx); |
46 | ECB_BLOCK(1, __serpent_decrypt); |
47 | ECB_WALK_END(); |
48 | } |
49 | |
50 | static int cbc_encrypt(struct skcipher_request *req) |
51 | { |
52 | CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1); |
53 | CBC_ENC_BLOCK(__serpent_encrypt); |
54 | CBC_WALK_END(); |
55 | } |
56 | |
57 | static int cbc_decrypt(struct skcipher_request *req) |
58 | { |
59 | CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); |
60 | CBC_DEC_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_cbc_dec_16way); |
61 | CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_cbc_dec_8way_avx); |
62 | CBC_DEC_BLOCK(1, __serpent_decrypt); |
63 | CBC_WALK_END(); |
64 | } |
65 | |
66 | static struct skcipher_alg serpent_algs[] = { |
67 | { |
68 | .base.cra_name = "__ecb(serpent)" , |
69 | .base.cra_driver_name = "__ecb-serpent-avx2" , |
70 | .base.cra_priority = 600, |
71 | .base.cra_flags = CRYPTO_ALG_INTERNAL, |
72 | .base.cra_blocksize = SERPENT_BLOCK_SIZE, |
73 | .base.cra_ctxsize = sizeof(struct serpent_ctx), |
74 | .base.cra_module = THIS_MODULE, |
75 | .min_keysize = SERPENT_MIN_KEY_SIZE, |
76 | .max_keysize = SERPENT_MAX_KEY_SIZE, |
77 | .setkey = serpent_setkey_skcipher, |
78 | .encrypt = ecb_encrypt, |
79 | .decrypt = ecb_decrypt, |
80 | }, { |
81 | .base.cra_name = "__cbc(serpent)" , |
82 | .base.cra_driver_name = "__cbc-serpent-avx2" , |
83 | .base.cra_priority = 600, |
84 | .base.cra_flags = CRYPTO_ALG_INTERNAL, |
85 | .base.cra_blocksize = SERPENT_BLOCK_SIZE, |
86 | .base.cra_ctxsize = sizeof(struct serpent_ctx), |
87 | .base.cra_module = THIS_MODULE, |
88 | .min_keysize = SERPENT_MIN_KEY_SIZE, |
89 | .max_keysize = SERPENT_MAX_KEY_SIZE, |
90 | .ivsize = SERPENT_BLOCK_SIZE, |
91 | .setkey = serpent_setkey_skcipher, |
92 | .encrypt = cbc_encrypt, |
93 | .decrypt = cbc_decrypt, |
94 | }, |
95 | }; |
96 | |
97 | static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)]; |
98 | |
99 | static int __init serpent_avx2_init(void) |
100 | { |
101 | const char *feature_name; |
102 | |
103 | if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) { |
104 | pr_info("AVX2 instructions are not detected.\n" ); |
105 | return -ENODEV; |
106 | } |
107 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, |
108 | feature_name: &feature_name)) { |
109 | pr_info("CPU feature '%s' is not supported.\n" , feature_name); |
110 | return -ENODEV; |
111 | } |
112 | |
113 | return simd_register_skciphers_compat(algs: serpent_algs, |
114 | ARRAY_SIZE(serpent_algs), |
115 | simd_algs: serpent_simd_algs); |
116 | } |
117 | |
118 | static void __exit serpent_avx2_fini(void) |
119 | { |
120 | simd_unregister_skciphers(algs: serpent_algs, ARRAY_SIZE(serpent_algs), |
121 | simd_algs: serpent_simd_algs); |
122 | } |
123 | |
124 | module_init(serpent_avx2_init); |
125 | module_exit(serpent_avx2_fini); |
126 | |
127 | MODULE_LICENSE("GPL" ); |
128 | MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized" ); |
129 | MODULE_ALIAS_CRYPTO("serpent" ); |
130 | MODULE_ALIAS_CRYPTO("serpent-asm" ); |
131 | |