1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * MIPS accelerated ChaCha and XChaCha stream ciphers, |
4 | * including ChaCha20 (RFC7539) |
5 | * |
6 | * Copyright (C) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org> |
7 | */ |
8 | |
9 | #include <asm/byteorder.h> |
10 | #include <crypto/algapi.h> |
11 | #include <crypto/internal/chacha.h> |
12 | #include <crypto/internal/skcipher.h> |
13 | #include <linux/kernel.h> |
14 | #include <linux/module.h> |
15 | |
16 | asmlinkage void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, |
17 | unsigned int bytes, int nrounds); |
18 | EXPORT_SYMBOL(chacha_crypt_arch); |
19 | |
20 | asmlinkage void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds); |
21 | EXPORT_SYMBOL(hchacha_block_arch); |
22 | |
23 | void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv) |
24 | { |
25 | chacha_init_generic(state, key, iv); |
26 | } |
27 | EXPORT_SYMBOL(chacha_init_arch); |
28 | |
29 | static int chacha_mips_stream_xor(struct skcipher_request *req, |
30 | const struct chacha_ctx *ctx, const u8 *iv) |
31 | { |
32 | struct skcipher_walk walk; |
33 | u32 state[16]; |
34 | int err; |
35 | |
36 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
37 | |
38 | chacha_init_generic(state, key: ctx->key, iv); |
39 | |
40 | while (walk.nbytes > 0) { |
41 | unsigned int nbytes = walk.nbytes; |
42 | |
43 | if (nbytes < walk.total) |
44 | nbytes = round_down(nbytes, walk.stride); |
45 | |
46 | chacha_crypt(state, dst: walk.dst.virt.addr, src: walk.src.virt.addr, |
47 | bytes: nbytes, nrounds: ctx->nrounds); |
48 | err = skcipher_walk_done(walk: &walk, err: walk.nbytes - nbytes); |
49 | } |
50 | |
51 | return err; |
52 | } |
53 | |
54 | static int chacha_mips(struct skcipher_request *req) |
55 | { |
56 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
57 | struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); |
58 | |
59 | return chacha_mips_stream_xor(req, ctx, iv: req->iv); |
60 | } |
61 | |
62 | static int xchacha_mips(struct skcipher_request *req) |
63 | { |
64 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
65 | struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); |
66 | struct chacha_ctx subctx; |
67 | u32 state[16]; |
68 | u8 real_iv[16]; |
69 | |
70 | chacha_init_generic(state, key: ctx->key, iv: req->iv); |
71 | |
72 | hchacha_block(state, out: subctx.key, nrounds: ctx->nrounds); |
73 | subctx.nrounds = ctx->nrounds; |
74 | |
75 | memcpy(&real_iv[0], req->iv + 24, 8); |
76 | memcpy(&real_iv[8], req->iv + 16, 8); |
77 | return chacha_mips_stream_xor(req, ctx: &subctx, iv: real_iv); |
78 | } |
79 | |
80 | static struct skcipher_alg algs[] = { |
81 | { |
82 | .base.cra_name = "chacha20" , |
83 | .base.cra_driver_name = "chacha20-mips" , |
84 | .base.cra_priority = 200, |
85 | .base.cra_blocksize = 1, |
86 | .base.cra_ctxsize = sizeof(struct chacha_ctx), |
87 | .base.cra_module = THIS_MODULE, |
88 | |
89 | .min_keysize = CHACHA_KEY_SIZE, |
90 | .max_keysize = CHACHA_KEY_SIZE, |
91 | .ivsize = CHACHA_IV_SIZE, |
92 | .chunksize = CHACHA_BLOCK_SIZE, |
93 | .setkey = chacha20_setkey, |
94 | .encrypt = chacha_mips, |
95 | .decrypt = chacha_mips, |
96 | }, { |
97 | .base.cra_name = "xchacha20" , |
98 | .base.cra_driver_name = "xchacha20-mips" , |
99 | .base.cra_priority = 200, |
100 | .base.cra_blocksize = 1, |
101 | .base.cra_ctxsize = sizeof(struct chacha_ctx), |
102 | .base.cra_module = THIS_MODULE, |
103 | |
104 | .min_keysize = CHACHA_KEY_SIZE, |
105 | .max_keysize = CHACHA_KEY_SIZE, |
106 | .ivsize = XCHACHA_IV_SIZE, |
107 | .chunksize = CHACHA_BLOCK_SIZE, |
108 | .setkey = chacha20_setkey, |
109 | .encrypt = xchacha_mips, |
110 | .decrypt = xchacha_mips, |
111 | }, { |
112 | .base.cra_name = "xchacha12" , |
113 | .base.cra_driver_name = "xchacha12-mips" , |
114 | .base.cra_priority = 200, |
115 | .base.cra_blocksize = 1, |
116 | .base.cra_ctxsize = sizeof(struct chacha_ctx), |
117 | .base.cra_module = THIS_MODULE, |
118 | |
119 | .min_keysize = CHACHA_KEY_SIZE, |
120 | .max_keysize = CHACHA_KEY_SIZE, |
121 | .ivsize = XCHACHA_IV_SIZE, |
122 | .chunksize = CHACHA_BLOCK_SIZE, |
123 | .setkey = chacha12_setkey, |
124 | .encrypt = xchacha_mips, |
125 | .decrypt = xchacha_mips, |
126 | } |
127 | }; |
128 | |
129 | static int __init chacha_simd_mod_init(void) |
130 | { |
131 | return IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER) ? |
132 | crypto_register_skciphers(algs, ARRAY_SIZE(algs)) : 0; |
133 | } |
134 | |
135 | static void __exit chacha_simd_mod_fini(void) |
136 | { |
137 | if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) |
138 | crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); |
139 | } |
140 | |
141 | module_init(chacha_simd_mod_init); |
142 | module_exit(chacha_simd_mod_fini); |
143 | |
144 | MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (MIPS accelerated)" ); |
145 | MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>" ); |
146 | MODULE_LICENSE("GPL v2" ); |
147 | MODULE_ALIAS_CRYPTO("chacha20" ); |
148 | MODULE_ALIAS_CRYPTO("chacha20-mips" ); |
149 | MODULE_ALIAS_CRYPTO("xchacha20" ); |
150 | MODULE_ALIAS_CRYPTO("xchacha20-mips" ); |
151 | MODULE_ALIAS_CRYPTO("xchacha12" ); |
152 | MODULE_ALIAS_CRYPTO("xchacha12-mips" ); |
153 | |