1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Glue Code for assembler optimized version of 3DES |
4 | * |
5 | * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> |
6 | * |
7 | * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: |
8 | * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> |
9 | */ |
10 | |
11 | #include <crypto/algapi.h> |
12 | #include <crypto/des.h> |
13 | #include <crypto/internal/skcipher.h> |
14 | #include <linux/crypto.h> |
15 | #include <linux/init.h> |
16 | #include <linux/module.h> |
17 | #include <linux/types.h> |
18 | |
19 | struct des3_ede_x86_ctx { |
20 | struct des3_ede_ctx enc; |
21 | struct des3_ede_ctx dec; |
22 | }; |
23 | |
24 | /* regular block cipher functions */ |
25 | asmlinkage void des3_ede_x86_64_crypt_blk(const u32 *expkey, u8 *dst, |
26 | const u8 *src); |
27 | |
28 | /* 3-way parallel cipher functions */ |
29 | asmlinkage void des3_ede_x86_64_crypt_blk_3way(const u32 *expkey, u8 *dst, |
30 | const u8 *src); |
31 | |
32 | static inline void des3_ede_enc_blk(struct des3_ede_x86_ctx *ctx, u8 *dst, |
33 | const u8 *src) |
34 | { |
35 | u32 *enc_ctx = ctx->enc.expkey; |
36 | |
37 | des3_ede_x86_64_crypt_blk(expkey: enc_ctx, dst, src); |
38 | } |
39 | |
40 | static inline void des3_ede_dec_blk(struct des3_ede_x86_ctx *ctx, u8 *dst, |
41 | const u8 *src) |
42 | { |
43 | u32 *dec_ctx = ctx->dec.expkey; |
44 | |
45 | des3_ede_x86_64_crypt_blk(expkey: dec_ctx, dst, src); |
46 | } |
47 | |
48 | static inline void des3_ede_dec_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst, |
49 | const u8 *src) |
50 | { |
51 | u32 *dec_ctx = ctx->dec.expkey; |
52 | |
53 | des3_ede_x86_64_crypt_blk_3way(expkey: dec_ctx, dst, src); |
54 | } |
55 | |
56 | static void des3_ede_x86_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) |
57 | { |
58 | des3_ede_enc_blk(ctx: crypto_tfm_ctx(tfm), dst, src); |
59 | } |
60 | |
61 | static void des3_ede_x86_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) |
62 | { |
63 | des3_ede_dec_blk(ctx: crypto_tfm_ctx(tfm), dst, src); |
64 | } |
65 | |
66 | static int ecb_crypt(struct skcipher_request *req, const u32 *expkey) |
67 | { |
68 | const unsigned int bsize = DES3_EDE_BLOCK_SIZE; |
69 | struct skcipher_walk walk; |
70 | unsigned int nbytes; |
71 | int err; |
72 | |
73 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
74 | |
75 | while ((nbytes = walk.nbytes)) { |
76 | u8 *wsrc = walk.src.virt.addr; |
77 | u8 *wdst = walk.dst.virt.addr; |
78 | |
79 | /* Process four block batch */ |
80 | if (nbytes >= bsize * 3) { |
81 | do { |
82 | des3_ede_x86_64_crypt_blk_3way(expkey, dst: wdst, |
83 | src: wsrc); |
84 | |
85 | wsrc += bsize * 3; |
86 | wdst += bsize * 3; |
87 | nbytes -= bsize * 3; |
88 | } while (nbytes >= bsize * 3); |
89 | |
90 | if (nbytes < bsize) |
91 | goto done; |
92 | } |
93 | |
94 | /* Handle leftovers */ |
95 | do { |
96 | des3_ede_x86_64_crypt_blk(expkey, dst: wdst, src: wsrc); |
97 | |
98 | wsrc += bsize; |
99 | wdst += bsize; |
100 | nbytes -= bsize; |
101 | } while (nbytes >= bsize); |
102 | |
103 | done: |
104 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
105 | } |
106 | |
107 | return err; |
108 | } |
109 | |
110 | static int ecb_encrypt(struct skcipher_request *req) |
111 | { |
112 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
113 | struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); |
114 | |
115 | return ecb_crypt(req, expkey: ctx->enc.expkey); |
116 | } |
117 | |
118 | static int ecb_decrypt(struct skcipher_request *req) |
119 | { |
120 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
121 | struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); |
122 | |
123 | return ecb_crypt(req, expkey: ctx->dec.expkey); |
124 | } |
125 | |
126 | static unsigned int __cbc_encrypt(struct des3_ede_x86_ctx *ctx, |
127 | struct skcipher_walk *walk) |
128 | { |
129 | unsigned int bsize = DES3_EDE_BLOCK_SIZE; |
130 | unsigned int nbytes = walk->nbytes; |
131 | u64 *src = (u64 *)walk->src.virt.addr; |
132 | u64 *dst = (u64 *)walk->dst.virt.addr; |
133 | u64 *iv = (u64 *)walk->iv; |
134 | |
135 | do { |
136 | *dst = *src ^ *iv; |
137 | des3_ede_enc_blk(ctx, dst: (u8 *)dst, src: (u8 *)dst); |
138 | iv = dst; |
139 | |
140 | src += 1; |
141 | dst += 1; |
142 | nbytes -= bsize; |
143 | } while (nbytes >= bsize); |
144 | |
145 | *(u64 *)walk->iv = *iv; |
146 | return nbytes; |
147 | } |
148 | |
149 | static int cbc_encrypt(struct skcipher_request *req) |
150 | { |
151 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
152 | struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); |
153 | struct skcipher_walk walk; |
154 | unsigned int nbytes; |
155 | int err; |
156 | |
157 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
158 | |
159 | while (walk.nbytes) { |
160 | nbytes = __cbc_encrypt(ctx, walk: &walk); |
161 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
162 | } |
163 | |
164 | return err; |
165 | } |
166 | |
167 | static unsigned int __cbc_decrypt(struct des3_ede_x86_ctx *ctx, |
168 | struct skcipher_walk *walk) |
169 | { |
170 | unsigned int bsize = DES3_EDE_BLOCK_SIZE; |
171 | unsigned int nbytes = walk->nbytes; |
172 | u64 *src = (u64 *)walk->src.virt.addr; |
173 | u64 *dst = (u64 *)walk->dst.virt.addr; |
174 | u64 ivs[3 - 1]; |
175 | u64 last_iv; |
176 | |
177 | /* Start of the last block. */ |
178 | src += nbytes / bsize - 1; |
179 | dst += nbytes / bsize - 1; |
180 | |
181 | last_iv = *src; |
182 | |
183 | /* Process four block batch */ |
184 | if (nbytes >= bsize * 3) { |
185 | do { |
186 | nbytes -= bsize * 3 - bsize; |
187 | src -= 3 - 1; |
188 | dst -= 3 - 1; |
189 | |
190 | ivs[0] = src[0]; |
191 | ivs[1] = src[1]; |
192 | |
193 | des3_ede_dec_blk_3way(ctx, dst: (u8 *)dst, src: (u8 *)src); |
194 | |
195 | dst[1] ^= ivs[0]; |
196 | dst[2] ^= ivs[1]; |
197 | |
198 | nbytes -= bsize; |
199 | if (nbytes < bsize) |
200 | goto done; |
201 | |
202 | *dst ^= *(src - 1); |
203 | src -= 1; |
204 | dst -= 1; |
205 | } while (nbytes >= bsize * 3); |
206 | } |
207 | |
208 | /* Handle leftovers */ |
209 | for (;;) { |
210 | des3_ede_dec_blk(ctx, dst: (u8 *)dst, src: (u8 *)src); |
211 | |
212 | nbytes -= bsize; |
213 | if (nbytes < bsize) |
214 | break; |
215 | |
216 | *dst ^= *(src - 1); |
217 | src -= 1; |
218 | dst -= 1; |
219 | } |
220 | |
221 | done: |
222 | *dst ^= *(u64 *)walk->iv; |
223 | *(u64 *)walk->iv = last_iv; |
224 | |
225 | return nbytes; |
226 | } |
227 | |
228 | static int cbc_decrypt(struct skcipher_request *req) |
229 | { |
230 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
231 | struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); |
232 | struct skcipher_walk walk; |
233 | unsigned int nbytes; |
234 | int err; |
235 | |
236 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
237 | |
238 | while (walk.nbytes) { |
239 | nbytes = __cbc_decrypt(ctx, walk: &walk); |
240 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
241 | } |
242 | |
243 | return err; |
244 | } |
245 | |
246 | static int des3_ede_x86_setkey(struct crypto_tfm *tfm, const u8 *key, |
247 | unsigned int keylen) |
248 | { |
249 | struct des3_ede_x86_ctx *ctx = crypto_tfm_ctx(tfm); |
250 | u32 i, j, tmp; |
251 | int err; |
252 | |
253 | err = des3_ede_expand_key(ctx: &ctx->enc, key, keylen); |
254 | if (err == -ENOKEY) { |
255 | if (crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) |
256 | err = -EINVAL; |
257 | else |
258 | err = 0; |
259 | } |
260 | |
261 | if (err) { |
262 | memset(ctx, 0, sizeof(*ctx)); |
263 | return err; |
264 | } |
265 | |
266 | /* Fix encryption context for this implementation and form decryption |
267 | * context. */ |
268 | j = DES3_EDE_EXPKEY_WORDS - 2; |
269 | for (i = 0; i < DES3_EDE_EXPKEY_WORDS; i += 2, j -= 2) { |
270 | tmp = ror32(word: ctx->enc.expkey[i + 1], shift: 4); |
271 | ctx->enc.expkey[i + 1] = tmp; |
272 | |
273 | ctx->dec.expkey[j + 0] = ctx->enc.expkey[i + 0]; |
274 | ctx->dec.expkey[j + 1] = tmp; |
275 | } |
276 | |
277 | return 0; |
278 | } |
279 | |
280 | static int des3_ede_x86_setkey_skcipher(struct crypto_skcipher *tfm, |
281 | const u8 *key, |
282 | unsigned int keylen) |
283 | { |
284 | return des3_ede_x86_setkey(tfm: &tfm->base, key, keylen); |
285 | } |
286 | |
287 | static struct crypto_alg des3_ede_cipher = { |
288 | .cra_name = "des3_ede" , |
289 | .cra_driver_name = "des3_ede-asm" , |
290 | .cra_priority = 200, |
291 | .cra_flags = CRYPTO_ALG_TYPE_CIPHER, |
292 | .cra_blocksize = DES3_EDE_BLOCK_SIZE, |
293 | .cra_ctxsize = sizeof(struct des3_ede_x86_ctx), |
294 | .cra_alignmask = 0, |
295 | .cra_module = THIS_MODULE, |
296 | .cra_u = { |
297 | .cipher = { |
298 | .cia_min_keysize = DES3_EDE_KEY_SIZE, |
299 | .cia_max_keysize = DES3_EDE_KEY_SIZE, |
300 | .cia_setkey = des3_ede_x86_setkey, |
301 | .cia_encrypt = des3_ede_x86_encrypt, |
302 | .cia_decrypt = des3_ede_x86_decrypt, |
303 | } |
304 | } |
305 | }; |
306 | |
307 | static struct skcipher_alg des3_ede_skciphers[] = { |
308 | { |
309 | .base.cra_name = "ecb(des3_ede)" , |
310 | .base.cra_driver_name = "ecb-des3_ede-asm" , |
311 | .base.cra_priority = 300, |
312 | .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, |
313 | .base.cra_ctxsize = sizeof(struct des3_ede_x86_ctx), |
314 | .base.cra_module = THIS_MODULE, |
315 | .min_keysize = DES3_EDE_KEY_SIZE, |
316 | .max_keysize = DES3_EDE_KEY_SIZE, |
317 | .setkey = des3_ede_x86_setkey_skcipher, |
318 | .encrypt = ecb_encrypt, |
319 | .decrypt = ecb_decrypt, |
320 | }, { |
321 | .base.cra_name = "cbc(des3_ede)" , |
322 | .base.cra_driver_name = "cbc-des3_ede-asm" , |
323 | .base.cra_priority = 300, |
324 | .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, |
325 | .base.cra_ctxsize = sizeof(struct des3_ede_x86_ctx), |
326 | .base.cra_module = THIS_MODULE, |
327 | .min_keysize = DES3_EDE_KEY_SIZE, |
328 | .max_keysize = DES3_EDE_KEY_SIZE, |
329 | .ivsize = DES3_EDE_BLOCK_SIZE, |
330 | .setkey = des3_ede_x86_setkey_skcipher, |
331 | .encrypt = cbc_encrypt, |
332 | .decrypt = cbc_decrypt, |
333 | } |
334 | }; |
335 | |
336 | static bool is_blacklisted_cpu(void) |
337 | { |
338 | if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) |
339 | return false; |
340 | |
341 | if (boot_cpu_data.x86 == 0x0f) { |
342 | /* |
343 | * On Pentium 4, des3_ede-x86_64 is slower than generic C |
344 | * implementation because use of 64bit rotates (which are really |
345 | * slow on P4). Therefore blacklist P4s. |
346 | */ |
347 | return true; |
348 | } |
349 | |
350 | return false; |
351 | } |
352 | |
353 | static int force; |
354 | module_param(force, int, 0); |
355 | MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist" ); |
356 | |
357 | static int __init des3_ede_x86_init(void) |
358 | { |
359 | int err; |
360 | |
361 | if (!force && is_blacklisted_cpu()) { |
362 | pr_info("des3_ede-x86_64: performance on this CPU would be suboptimal: disabling des3_ede-x86_64.\n" ); |
363 | return -ENODEV; |
364 | } |
365 | |
366 | err = crypto_register_alg(alg: &des3_ede_cipher); |
367 | if (err) |
368 | return err; |
369 | |
370 | err = crypto_register_skciphers(algs: des3_ede_skciphers, |
371 | ARRAY_SIZE(des3_ede_skciphers)); |
372 | if (err) |
373 | crypto_unregister_alg(alg: &des3_ede_cipher); |
374 | |
375 | return err; |
376 | } |
377 | |
378 | static void __exit des3_ede_x86_fini(void) |
379 | { |
380 | crypto_unregister_alg(alg: &des3_ede_cipher); |
381 | crypto_unregister_skciphers(algs: des3_ede_skciphers, |
382 | ARRAY_SIZE(des3_ede_skciphers)); |
383 | } |
384 | |
385 | module_init(des3_ede_x86_init); |
386 | module_exit(des3_ede_x86_fini); |
387 | |
388 | MODULE_LICENSE("GPL" ); |
389 | MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized" ); |
390 | MODULE_ALIAS_CRYPTO("des3_ede" ); |
391 | MODULE_ALIAS_CRYPTO("des3_ede-asm" ); |
392 | MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>" ); |
393 | |