1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia |
4 | * |
5 | * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> |
6 | */ |
7 | |
8 | #include <crypto/algapi.h> |
9 | #include <crypto/internal/simd.h> |
10 | #include <linux/crypto.h> |
11 | #include <linux/err.h> |
12 | #include <linux/module.h> |
13 | #include <linux/types.h> |
14 | |
15 | #include "camellia.h" |
16 | #include "ecb_cbc_helpers.h" |
17 | |
18 | #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 |
19 | #define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32 |
20 | |
21 | /* 32-way AVX2/AES-NI parallel cipher functions */ |
22 | asmlinkage void camellia_ecb_enc_32way(const void *ctx, u8 *dst, const u8 *src); |
23 | asmlinkage void camellia_ecb_dec_32way(const void *ctx, u8 *dst, const u8 *src); |
24 | |
25 | asmlinkage void camellia_cbc_dec_32way(const void *ctx, u8 *dst, const u8 *src); |
26 | |
27 | static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, |
28 | unsigned int keylen) |
29 | { |
30 | return __camellia_setkey(cctx: crypto_skcipher_ctx(tfm), key, key_len: keylen); |
31 | } |
32 | |
33 | static int ecb_encrypt(struct skcipher_request *req) |
34 | { |
35 | ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); |
36 | ECB_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_ecb_enc_32way); |
37 | ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_enc_16way); |
38 | ECB_BLOCK(2, camellia_enc_blk_2way); |
39 | ECB_BLOCK(1, camellia_enc_blk); |
40 | ECB_WALK_END(); |
41 | } |
42 | |
43 | static int ecb_decrypt(struct skcipher_request *req) |
44 | { |
45 | ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); |
46 | ECB_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_ecb_dec_32way); |
47 | ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_dec_16way); |
48 | ECB_BLOCK(2, camellia_dec_blk_2way); |
49 | ECB_BLOCK(1, camellia_dec_blk); |
50 | ECB_WALK_END(); |
51 | } |
52 | |
53 | static int cbc_encrypt(struct skcipher_request *req) |
54 | { |
55 | CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); |
56 | CBC_ENC_BLOCK(camellia_enc_blk); |
57 | CBC_WALK_END(); |
58 | } |
59 | |
60 | static int cbc_decrypt(struct skcipher_request *req) |
61 | { |
62 | CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); |
63 | CBC_DEC_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_cbc_dec_32way); |
64 | CBC_DEC_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_cbc_dec_16way); |
65 | CBC_DEC_BLOCK(2, camellia_decrypt_cbc_2way); |
66 | CBC_DEC_BLOCK(1, camellia_dec_blk); |
67 | CBC_WALK_END(); |
68 | } |
69 | |
70 | static struct skcipher_alg camellia_algs[] = { |
71 | { |
72 | .base.cra_name = "__ecb(camellia)" , |
73 | .base.cra_driver_name = "__ecb-camellia-aesni-avx2" , |
74 | .base.cra_priority = 500, |
75 | .base.cra_flags = CRYPTO_ALG_INTERNAL, |
76 | .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, |
77 | .base.cra_ctxsize = sizeof(struct camellia_ctx), |
78 | .base.cra_module = THIS_MODULE, |
79 | .min_keysize = CAMELLIA_MIN_KEY_SIZE, |
80 | .max_keysize = CAMELLIA_MAX_KEY_SIZE, |
81 | .setkey = camellia_setkey, |
82 | .encrypt = ecb_encrypt, |
83 | .decrypt = ecb_decrypt, |
84 | }, { |
85 | .base.cra_name = "__cbc(camellia)" , |
86 | .base.cra_driver_name = "__cbc-camellia-aesni-avx2" , |
87 | .base.cra_priority = 500, |
88 | .base.cra_flags = CRYPTO_ALG_INTERNAL, |
89 | .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, |
90 | .base.cra_ctxsize = sizeof(struct camellia_ctx), |
91 | .base.cra_module = THIS_MODULE, |
92 | .min_keysize = CAMELLIA_MIN_KEY_SIZE, |
93 | .max_keysize = CAMELLIA_MAX_KEY_SIZE, |
94 | .ivsize = CAMELLIA_BLOCK_SIZE, |
95 | .setkey = camellia_setkey, |
96 | .encrypt = cbc_encrypt, |
97 | .decrypt = cbc_decrypt, |
98 | }, |
99 | }; |
100 | |
101 | static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)]; |
102 | |
103 | static int __init camellia_aesni_init(void) |
104 | { |
105 | const char *feature_name; |
106 | |
107 | if (!boot_cpu_has(X86_FEATURE_AVX) || |
108 | !boot_cpu_has(X86_FEATURE_AVX2) || |
109 | !boot_cpu_has(X86_FEATURE_AES) || |
110 | !boot_cpu_has(X86_FEATURE_OSXSAVE)) { |
111 | pr_info("AVX2 or AES-NI instructions are not detected.\n" ); |
112 | return -ENODEV; |
113 | } |
114 | |
115 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, |
116 | feature_name: &feature_name)) { |
117 | pr_info("CPU feature '%s' is not supported.\n" , feature_name); |
118 | return -ENODEV; |
119 | } |
120 | |
121 | return simd_register_skciphers_compat(algs: camellia_algs, |
122 | ARRAY_SIZE(camellia_algs), |
123 | simd_algs: camellia_simd_algs); |
124 | } |
125 | |
126 | static void __exit camellia_aesni_fini(void) |
127 | { |
128 | simd_unregister_skciphers(algs: camellia_algs, ARRAY_SIZE(camellia_algs), |
129 | simd_algs: camellia_simd_algs); |
130 | } |
131 | |
132 | module_init(camellia_aesni_init); |
133 | module_exit(camellia_aesni_fini); |
134 | |
135 | MODULE_LICENSE("GPL" ); |
136 | MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized" ); |
137 | MODULE_ALIAS_CRYPTO("camellia" ); |
138 | MODULE_ALIAS_CRYPTO("camellia-asm" ); |
139 | |