1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Support for Intel AES-NI instructions. This file contains glue |
4 | * code, the real AES implementation is in intel-aes_asm.S. |
5 | * |
6 | * Copyright (C) 2008, Intel Corp. |
7 | * Author: Huang Ying <ying.huang@intel.com> |
8 | * |
9 | * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD |
10 | * interface for 64-bit kernels. |
11 | * Authors: Adrian Hoban <adrian.hoban@intel.com> |
12 | * Gabriele Paoloni <gabriele.paoloni@intel.com> |
13 | * Tadeusz Struk (tadeusz.struk@intel.com) |
14 | * Aidan O'Mahony (aidan.o.mahony@intel.com) |
15 | * Copyright (c) 2010, Intel Corporation. |
16 | */ |
17 | |
18 | #include <linux/hardirq.h> |
19 | #include <linux/types.h> |
20 | #include <linux/module.h> |
21 | #include <linux/err.h> |
22 | #include <crypto/algapi.h> |
23 | #include <crypto/aes.h> |
24 | #include <crypto/ctr.h> |
25 | #include <crypto/b128ops.h> |
26 | #include <crypto/gcm.h> |
27 | #include <crypto/xts.h> |
28 | #include <asm/cpu_device_id.h> |
29 | #include <asm/simd.h> |
30 | #include <crypto/scatterwalk.h> |
31 | #include <crypto/internal/aead.h> |
32 | #include <crypto/internal/simd.h> |
33 | #include <crypto/internal/skcipher.h> |
34 | #include <linux/jump_label.h> |
35 | #include <linux/workqueue.h> |
36 | #include <linux/spinlock.h> |
37 | #include <linux/static_call.h> |
38 | |
39 | |
40 | #define AESNI_ALIGN 16 |
41 | #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN))) |
42 | #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1)) |
43 | #define RFC4106_HASH_SUBKEY_SIZE 16 |
44 | #define ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) |
45 | #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA) |
46 | #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA) |
47 | |
48 | /* This data is stored at the end of the crypto_tfm struct. |
49 | * It's a type of per "session" data storage location. |
50 | * This needs to be 16 byte aligned. |
51 | */ |
52 | struct aesni_rfc4106_gcm_ctx { |
53 | u8 hash_subkey[16] AESNI_ALIGN_ATTR; |
54 | struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; |
55 | u8 nonce[4]; |
56 | }; |
57 | |
58 | struct generic_gcmaes_ctx { |
59 | u8 hash_subkey[16] AESNI_ALIGN_ATTR; |
60 | struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR; |
61 | }; |
62 | |
63 | struct aesni_xts_ctx { |
64 | struct crypto_aes_ctx tweak_ctx AESNI_ALIGN_ATTR; |
65 | struct crypto_aes_ctx crypt_ctx AESNI_ALIGN_ATTR; |
66 | }; |
67 | |
68 | #define GCM_BLOCK_LEN 16 |
69 | |
70 | struct gcm_context_data { |
71 | /* init, update and finalize context data */ |
72 | u8 aad_hash[GCM_BLOCK_LEN]; |
73 | u64 aad_length; |
74 | u64 in_length; |
75 | u8 partial_block_enc_key[GCM_BLOCK_LEN]; |
76 | u8 orig_IV[GCM_BLOCK_LEN]; |
77 | u8 current_counter[GCM_BLOCK_LEN]; |
78 | u64 partial_block_len; |
79 | u64 unused; |
80 | u8 hash_keys[GCM_BLOCK_LEN * 16]; |
81 | }; |
82 | |
83 | static inline void *aes_align_addr(void *addr) |
84 | { |
85 | if (crypto_tfm_ctx_alignment() >= AESNI_ALIGN) |
86 | return addr; |
87 | return PTR_ALIGN(addr, AESNI_ALIGN); |
88 | } |
89 | |
90 | asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, |
91 | unsigned int key_len); |
92 | asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in); |
93 | asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in); |
94 | asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, |
95 | const u8 *in, unsigned int len); |
96 | asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, |
97 | const u8 *in, unsigned int len); |
98 | asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, |
99 | const u8 *in, unsigned int len, u8 *iv); |
100 | asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, |
101 | const u8 *in, unsigned int len, u8 *iv); |
102 | asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, |
103 | const u8 *in, unsigned int len, u8 *iv); |
104 | asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, |
105 | const u8 *in, unsigned int len, u8 *iv); |
106 | |
107 | #define AVX_GEN2_OPTSIZE 640 |
108 | #define AVX_GEN4_OPTSIZE 4096 |
109 | |
110 | asmlinkage void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, |
111 | const u8 *in, unsigned int len, u8 *iv); |
112 | |
113 | asmlinkage void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, |
114 | const u8 *in, unsigned int len, u8 *iv); |
115 | |
116 | #ifdef CONFIG_X86_64 |
117 | |
118 | asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, |
119 | const u8 *in, unsigned int len, u8 *iv); |
120 | DEFINE_STATIC_CALL(aesni_ctr_enc_tfm, aesni_ctr_enc); |
121 | |
122 | /* Scatter / Gather routines, with args similar to above */ |
123 | asmlinkage void aesni_gcm_init(void *ctx, |
124 | struct gcm_context_data *gdata, |
125 | u8 *iv, |
126 | u8 *hash_subkey, const u8 *aad, |
127 | unsigned long aad_len); |
128 | asmlinkage void aesni_gcm_enc_update(void *ctx, |
129 | struct gcm_context_data *gdata, u8 *out, |
130 | const u8 *in, unsigned long plaintext_len); |
131 | asmlinkage void aesni_gcm_dec_update(void *ctx, |
132 | struct gcm_context_data *gdata, u8 *out, |
133 | const u8 *in, |
134 | unsigned long ciphertext_len); |
135 | asmlinkage void aesni_gcm_finalize(void *ctx, |
136 | struct gcm_context_data *gdata, |
137 | u8 *auth_tag, unsigned long auth_tag_len); |
138 | |
139 | asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv, |
140 | void *keys, u8 *out, unsigned int num_bytes); |
141 | asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv, |
142 | void *keys, u8 *out, unsigned int num_bytes); |
143 | asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv, |
144 | void *keys, u8 *out, unsigned int num_bytes); |
145 | |
146 | |
147 | asmlinkage void aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv, |
148 | const void *keys, u8 *out, unsigned int num_bytes, |
149 | unsigned int byte_ctr); |
150 | |
151 | asmlinkage void aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv, |
152 | const void *keys, u8 *out, unsigned int num_bytes, |
153 | unsigned int byte_ctr); |
154 | |
155 | asmlinkage void aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv, |
156 | const void *keys, u8 *out, unsigned int num_bytes, |
157 | unsigned int byte_ctr); |
158 | |
159 | /* |
160 | * asmlinkage void aesni_gcm_init_avx_gen2() |
161 | * gcm_data *my_ctx_data, context data |
162 | * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. |
163 | */ |
164 | asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data, |
165 | struct gcm_context_data *gdata, |
166 | u8 *iv, |
167 | u8 *hash_subkey, |
168 | const u8 *aad, |
169 | unsigned long aad_len); |
170 | |
171 | asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx, |
172 | struct gcm_context_data *gdata, u8 *out, |
173 | const u8 *in, unsigned long plaintext_len); |
174 | asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx, |
175 | struct gcm_context_data *gdata, u8 *out, |
176 | const u8 *in, |
177 | unsigned long ciphertext_len); |
178 | asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx, |
179 | struct gcm_context_data *gdata, |
180 | u8 *auth_tag, unsigned long auth_tag_len); |
181 | |
182 | /* |
183 | * asmlinkage void aesni_gcm_init_avx_gen4() |
184 | * gcm_data *my_ctx_data, context data |
185 | * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. |
186 | */ |
187 | asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data, |
188 | struct gcm_context_data *gdata, |
189 | u8 *iv, |
190 | u8 *hash_subkey, |
191 | const u8 *aad, |
192 | unsigned long aad_len); |
193 | |
194 | asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx, |
195 | struct gcm_context_data *gdata, u8 *out, |
196 | const u8 *in, unsigned long plaintext_len); |
197 | asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx, |
198 | struct gcm_context_data *gdata, u8 *out, |
199 | const u8 *in, |
200 | unsigned long ciphertext_len); |
201 | asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx, |
202 | struct gcm_context_data *gdata, |
203 | u8 *auth_tag, unsigned long auth_tag_len); |
204 | |
205 | static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx); |
206 | static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx2); |
207 | |
208 | static inline struct |
209 | aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) |
210 | { |
211 | return aes_align_addr(addr: crypto_aead_ctx(tfm)); |
212 | } |
213 | |
214 | static inline struct |
215 | generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm) |
216 | { |
217 | return aes_align_addr(addr: crypto_aead_ctx(tfm)); |
218 | } |
219 | #endif |
220 | |
221 | static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) |
222 | { |
223 | return aes_align_addr(addr: raw_ctx); |
224 | } |
225 | |
226 | static inline struct aesni_xts_ctx *aes_xts_ctx(struct crypto_skcipher *tfm) |
227 | { |
228 | return aes_align_addr(addr: crypto_skcipher_ctx(tfm)); |
229 | } |
230 | |
231 | static int aes_set_key_common(struct crypto_aes_ctx *ctx, |
232 | const u8 *in_key, unsigned int key_len) |
233 | { |
234 | int err; |
235 | |
236 | if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && |
237 | key_len != AES_KEYSIZE_256) |
238 | return -EINVAL; |
239 | |
240 | if (!crypto_simd_usable()) |
241 | err = aes_expandkey(ctx, in_key, key_len); |
242 | else { |
243 | kernel_fpu_begin(); |
244 | err = aesni_set_key(ctx, in_key, key_len); |
245 | kernel_fpu_end(); |
246 | } |
247 | |
248 | return err; |
249 | } |
250 | |
251 | static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, |
252 | unsigned int key_len) |
253 | { |
254 | return aes_set_key_common(ctx: aes_ctx(raw_ctx: crypto_tfm_ctx(tfm)), in_key, |
255 | key_len); |
256 | } |
257 | |
258 | static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) |
259 | { |
260 | struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_tfm_ctx(tfm)); |
261 | |
262 | if (!crypto_simd_usable()) { |
263 | aes_encrypt(ctx, out: dst, in: src); |
264 | } else { |
265 | kernel_fpu_begin(); |
266 | aesni_enc(ctx, out: dst, in: src); |
267 | kernel_fpu_end(); |
268 | } |
269 | } |
270 | |
271 | static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) |
272 | { |
273 | struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_tfm_ctx(tfm)); |
274 | |
275 | if (!crypto_simd_usable()) { |
276 | aes_decrypt(ctx, out: dst, in: src); |
277 | } else { |
278 | kernel_fpu_begin(); |
279 | aesni_dec(ctx, out: dst, in: src); |
280 | kernel_fpu_end(); |
281 | } |
282 | } |
283 | |
284 | static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, |
285 | unsigned int len) |
286 | { |
287 | return aes_set_key_common(ctx: aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm)), in_key: key, key_len: len); |
288 | } |
289 | |
290 | static int ecb_encrypt(struct skcipher_request *req) |
291 | { |
292 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
293 | struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm)); |
294 | struct skcipher_walk walk; |
295 | unsigned int nbytes; |
296 | int err; |
297 | |
298 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
299 | |
300 | while ((nbytes = walk.nbytes)) { |
301 | kernel_fpu_begin(); |
302 | aesni_ecb_enc(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr, |
303 | len: nbytes & AES_BLOCK_MASK); |
304 | kernel_fpu_end(); |
305 | nbytes &= AES_BLOCK_SIZE - 1; |
306 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
307 | } |
308 | |
309 | return err; |
310 | } |
311 | |
312 | static int ecb_decrypt(struct skcipher_request *req) |
313 | { |
314 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
315 | struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm)); |
316 | struct skcipher_walk walk; |
317 | unsigned int nbytes; |
318 | int err; |
319 | |
320 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
321 | |
322 | while ((nbytes = walk.nbytes)) { |
323 | kernel_fpu_begin(); |
324 | aesni_ecb_dec(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr, |
325 | len: nbytes & AES_BLOCK_MASK); |
326 | kernel_fpu_end(); |
327 | nbytes &= AES_BLOCK_SIZE - 1; |
328 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
329 | } |
330 | |
331 | return err; |
332 | } |
333 | |
334 | static int cbc_encrypt(struct skcipher_request *req) |
335 | { |
336 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
337 | struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm)); |
338 | struct skcipher_walk walk; |
339 | unsigned int nbytes; |
340 | int err; |
341 | |
342 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
343 | |
344 | while ((nbytes = walk.nbytes)) { |
345 | kernel_fpu_begin(); |
346 | aesni_cbc_enc(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr, |
347 | len: nbytes & AES_BLOCK_MASK, iv: walk.iv); |
348 | kernel_fpu_end(); |
349 | nbytes &= AES_BLOCK_SIZE - 1; |
350 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
351 | } |
352 | |
353 | return err; |
354 | } |
355 | |
356 | static int cbc_decrypt(struct skcipher_request *req) |
357 | { |
358 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
359 | struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm)); |
360 | struct skcipher_walk walk; |
361 | unsigned int nbytes; |
362 | int err; |
363 | |
364 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
365 | |
366 | while ((nbytes = walk.nbytes)) { |
367 | kernel_fpu_begin(); |
368 | aesni_cbc_dec(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr, |
369 | len: nbytes & AES_BLOCK_MASK, iv: walk.iv); |
370 | kernel_fpu_end(); |
371 | nbytes &= AES_BLOCK_SIZE - 1; |
372 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
373 | } |
374 | |
375 | return err; |
376 | } |
377 | |
378 | static int cts_cbc_encrypt(struct skcipher_request *req) |
379 | { |
380 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
381 | struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm)); |
382 | int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; |
383 | struct scatterlist *src = req->src, *dst = req->dst; |
384 | struct scatterlist sg_src[2], sg_dst[2]; |
385 | struct skcipher_request subreq; |
386 | struct skcipher_walk walk; |
387 | int err; |
388 | |
389 | skcipher_request_set_tfm(req: &subreq, tfm); |
390 | skcipher_request_set_callback(req: &subreq, flags: skcipher_request_flags(req), |
391 | NULL, NULL); |
392 | |
393 | if (req->cryptlen <= AES_BLOCK_SIZE) { |
394 | if (req->cryptlen < AES_BLOCK_SIZE) |
395 | return -EINVAL; |
396 | cbc_blocks = 1; |
397 | } |
398 | |
399 | if (cbc_blocks > 0) { |
400 | skcipher_request_set_crypt(req: &subreq, src: req->src, dst: req->dst, |
401 | cryptlen: cbc_blocks * AES_BLOCK_SIZE, |
402 | iv: req->iv); |
403 | |
404 | err = cbc_encrypt(req: &subreq); |
405 | if (err) |
406 | return err; |
407 | |
408 | if (req->cryptlen == AES_BLOCK_SIZE) |
409 | return 0; |
410 | |
411 | dst = src = scatterwalk_ffwd(dst: sg_src, src: req->src, len: subreq.cryptlen); |
412 | if (req->dst != req->src) |
413 | dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst, |
414 | len: subreq.cryptlen); |
415 | } |
416 | |
417 | /* handle ciphertext stealing */ |
418 | skcipher_request_set_crypt(req: &subreq, src, dst, |
419 | cryptlen: req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, |
420 | iv: req->iv); |
421 | |
422 | err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false); |
423 | if (err) |
424 | return err; |
425 | |
426 | kernel_fpu_begin(); |
427 | aesni_cts_cbc_enc(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr, |
428 | len: walk.nbytes, iv: walk.iv); |
429 | kernel_fpu_end(); |
430 | |
431 | return skcipher_walk_done(walk: &walk, err: 0); |
432 | } |
433 | |
434 | static int cts_cbc_decrypt(struct skcipher_request *req) |
435 | { |
436 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
437 | struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm)); |
438 | int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; |
439 | struct scatterlist *src = req->src, *dst = req->dst; |
440 | struct scatterlist sg_src[2], sg_dst[2]; |
441 | struct skcipher_request subreq; |
442 | struct skcipher_walk walk; |
443 | int err; |
444 | |
445 | skcipher_request_set_tfm(req: &subreq, tfm); |
446 | skcipher_request_set_callback(req: &subreq, flags: skcipher_request_flags(req), |
447 | NULL, NULL); |
448 | |
449 | if (req->cryptlen <= AES_BLOCK_SIZE) { |
450 | if (req->cryptlen < AES_BLOCK_SIZE) |
451 | return -EINVAL; |
452 | cbc_blocks = 1; |
453 | } |
454 | |
455 | if (cbc_blocks > 0) { |
456 | skcipher_request_set_crypt(req: &subreq, src: req->src, dst: req->dst, |
457 | cryptlen: cbc_blocks * AES_BLOCK_SIZE, |
458 | iv: req->iv); |
459 | |
460 | err = cbc_decrypt(req: &subreq); |
461 | if (err) |
462 | return err; |
463 | |
464 | if (req->cryptlen == AES_BLOCK_SIZE) |
465 | return 0; |
466 | |
467 | dst = src = scatterwalk_ffwd(dst: sg_src, src: req->src, len: subreq.cryptlen); |
468 | if (req->dst != req->src) |
469 | dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst, |
470 | len: subreq.cryptlen); |
471 | } |
472 | |
473 | /* handle ciphertext stealing */ |
474 | skcipher_request_set_crypt(req: &subreq, src, dst, |
475 | cryptlen: req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, |
476 | iv: req->iv); |
477 | |
478 | err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false); |
479 | if (err) |
480 | return err; |
481 | |
482 | kernel_fpu_begin(); |
483 | aesni_cts_cbc_dec(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr, |
484 | len: walk.nbytes, iv: walk.iv); |
485 | kernel_fpu_end(); |
486 | |
487 | return skcipher_walk_done(walk: &walk, err: 0); |
488 | } |
489 | |
490 | #ifdef CONFIG_X86_64 |
491 | static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, |
492 | const u8 *in, unsigned int len, u8 *iv) |
493 | { |
494 | /* |
495 | * based on key length, override with the by8 version |
496 | * of ctr mode encryption/decryption for improved performance |
497 | * aes_set_key_common() ensures that key length is one of |
498 | * {128,192,256} |
499 | */ |
500 | if (ctx->key_length == AES_KEYSIZE_128) |
501 | aes_ctr_enc_128_avx_by8(in, iv, keys: (void *)ctx, out, num_bytes: len); |
502 | else if (ctx->key_length == AES_KEYSIZE_192) |
503 | aes_ctr_enc_192_avx_by8(in, iv, keys: (void *)ctx, out, num_bytes: len); |
504 | else |
505 | aes_ctr_enc_256_avx_by8(in, iv, keys: (void *)ctx, out, num_bytes: len); |
506 | } |
507 | |
508 | static int ctr_crypt(struct skcipher_request *req) |
509 | { |
510 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
511 | struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm)); |
512 | u8 keystream[AES_BLOCK_SIZE]; |
513 | struct skcipher_walk walk; |
514 | unsigned int nbytes; |
515 | int err; |
516 | |
517 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
518 | |
519 | while ((nbytes = walk.nbytes) > 0) { |
520 | kernel_fpu_begin(); |
521 | if (nbytes & AES_BLOCK_MASK) |
522 | static_call(aesni_ctr_enc_tfm)(ctx, walk.dst.virt.addr, |
523 | walk.src.virt.addr, |
524 | nbytes & AES_BLOCK_MASK, |
525 | walk.iv); |
526 | nbytes &= ~AES_BLOCK_MASK; |
527 | |
528 | if (walk.nbytes == walk.total && nbytes > 0) { |
529 | aesni_enc(ctx, out: keystream, in: walk.iv); |
530 | crypto_xor_cpy(dst: walk.dst.virt.addr + walk.nbytes - nbytes, |
531 | src1: walk.src.virt.addr + walk.nbytes - nbytes, |
532 | src2: keystream, size: nbytes); |
533 | crypto_inc(a: walk.iv, AES_BLOCK_SIZE); |
534 | nbytes = 0; |
535 | } |
536 | kernel_fpu_end(); |
537 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
538 | } |
539 | return err; |
540 | } |
541 | |
542 | static void aesni_xctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, |
543 | const u8 *in, unsigned int len, u8 *iv, |
544 | unsigned int byte_ctr) |
545 | { |
546 | if (ctx->key_length == AES_KEYSIZE_128) |
547 | aes_xctr_enc_128_avx_by8(in, iv, keys: (void *)ctx, out, num_bytes: len, |
548 | byte_ctr); |
549 | else if (ctx->key_length == AES_KEYSIZE_192) |
550 | aes_xctr_enc_192_avx_by8(in, iv, keys: (void *)ctx, out, num_bytes: len, |
551 | byte_ctr); |
552 | else |
553 | aes_xctr_enc_256_avx_by8(in, iv, keys: (void *)ctx, out, num_bytes: len, |
554 | byte_ctr); |
555 | } |
556 | |
557 | static int xctr_crypt(struct skcipher_request *req) |
558 | { |
559 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
560 | struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm)); |
561 | u8 keystream[AES_BLOCK_SIZE]; |
562 | struct skcipher_walk walk; |
563 | unsigned int nbytes; |
564 | unsigned int byte_ctr = 0; |
565 | int err; |
566 | __le32 block[AES_BLOCK_SIZE / sizeof(__le32)]; |
567 | |
568 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
569 | |
570 | while ((nbytes = walk.nbytes) > 0) { |
571 | kernel_fpu_begin(); |
572 | if (nbytes & AES_BLOCK_MASK) |
573 | aesni_xctr_enc_avx_tfm(ctx, out: walk.dst.virt.addr, |
574 | in: walk.src.virt.addr, len: nbytes & AES_BLOCK_MASK, |
575 | iv: walk.iv, byte_ctr); |
576 | nbytes &= ~AES_BLOCK_MASK; |
577 | byte_ctr += walk.nbytes - nbytes; |
578 | |
579 | if (walk.nbytes == walk.total && nbytes > 0) { |
580 | memcpy(block, walk.iv, AES_BLOCK_SIZE); |
581 | block[0] ^= cpu_to_le32(1 + byte_ctr / AES_BLOCK_SIZE); |
582 | aesni_enc(ctx, out: keystream, in: (u8 *)block); |
583 | crypto_xor_cpy(dst: walk.dst.virt.addr + walk.nbytes - |
584 | nbytes, src1: walk.src.virt.addr + walk.nbytes |
585 | - nbytes, src2: keystream, size: nbytes); |
586 | byte_ctr += nbytes; |
587 | nbytes = 0; |
588 | } |
589 | kernel_fpu_end(); |
590 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
591 | } |
592 | return err; |
593 | } |
594 | |
595 | static int |
596 | rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) |
597 | { |
598 | struct crypto_aes_ctx ctx; |
599 | int ret; |
600 | |
601 | ret = aes_expandkey(ctx: &ctx, in_key: key, key_len); |
602 | if (ret) |
603 | return ret; |
604 | |
605 | /* Clear the data in the hash sub key container to zero.*/ |
606 | /* We want to cipher all zeros to create the hash sub key. */ |
607 | memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); |
608 | |
609 | aes_encrypt(ctx: &ctx, out: hash_subkey, in: hash_subkey); |
610 | |
611 | memzero_explicit(s: &ctx, count: sizeof(ctx)); |
612 | return 0; |
613 | } |
614 | |
615 | static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key, |
616 | unsigned int key_len) |
617 | { |
618 | struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm: aead); |
619 | |
620 | if (key_len < 4) |
621 | return -EINVAL; |
622 | |
623 | /*Account for 4 byte nonce at the end.*/ |
624 | key_len -= 4; |
625 | |
626 | memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce)); |
627 | |
628 | return aes_set_key_common(ctx: &ctx->aes_key_expanded, in_key: key, key_len) ?: |
629 | rfc4106_set_hash_subkey(hash_subkey: ctx->hash_subkey, key, key_len); |
630 | } |
631 | |
632 | /* This is the Integrity Check Value (aka the authentication tag) length and can |
633 | * be 8, 12 or 16 bytes long. */ |
634 | static int common_rfc4106_set_authsize(struct crypto_aead *aead, |
635 | unsigned int authsize) |
636 | { |
637 | switch (authsize) { |
638 | case 8: |
639 | case 12: |
640 | case 16: |
641 | break; |
642 | default: |
643 | return -EINVAL; |
644 | } |
645 | |
646 | return 0; |
647 | } |
648 | |
649 | static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, |
650 | unsigned int authsize) |
651 | { |
652 | switch (authsize) { |
653 | case 4: |
654 | case 8: |
655 | case 12: |
656 | case 13: |
657 | case 14: |
658 | case 15: |
659 | case 16: |
660 | break; |
661 | default: |
662 | return -EINVAL; |
663 | } |
664 | |
665 | return 0; |
666 | } |
667 | |
668 | static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, |
669 | unsigned int assoclen, u8 *hash_subkey, |
670 | u8 *iv, void *aes_ctx, u8 *auth_tag, |
671 | unsigned long auth_tag_len) |
672 | { |
673 | u8 databuf[sizeof(struct gcm_context_data) + (AESNI_ALIGN - 8)] __aligned(8); |
674 | struct gcm_context_data *data = PTR_ALIGN((void *)databuf, AESNI_ALIGN); |
675 | unsigned long left = req->cryptlen; |
676 | struct scatter_walk assoc_sg_walk; |
677 | struct skcipher_walk walk; |
678 | bool do_avx, do_avx2; |
679 | u8 *assocmem = NULL; |
680 | u8 *assoc; |
681 | int err; |
682 | |
683 | if (!enc) |
684 | left -= auth_tag_len; |
685 | |
686 | do_avx = (left >= AVX_GEN2_OPTSIZE); |
687 | do_avx2 = (left >= AVX_GEN4_OPTSIZE); |
688 | |
689 | /* Linearize assoc, if not already linear */ |
690 | if (req->src->length >= assoclen && req->src->length) { |
691 | scatterwalk_start(walk: &assoc_sg_walk, sg: req->src); |
692 | assoc = scatterwalk_map(walk: &assoc_sg_walk); |
693 | } else { |
694 | gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? |
695 | GFP_KERNEL : GFP_ATOMIC; |
696 | |
697 | /* assoc can be any length, so must be on heap */ |
698 | assocmem = kmalloc(size: assoclen, flags); |
699 | if (unlikely(!assocmem)) |
700 | return -ENOMEM; |
701 | assoc = assocmem; |
702 | |
703 | scatterwalk_map_and_copy(buf: assoc, sg: req->src, start: 0, nbytes: assoclen, out: 0); |
704 | } |
705 | |
706 | kernel_fpu_begin(); |
707 | if (static_branch_likely(&gcm_use_avx2) && do_avx2) |
708 | aesni_gcm_init_avx_gen4(my_ctx_data: aes_ctx, gdata: data, iv, hash_subkey, aad: assoc, |
709 | aad_len: assoclen); |
710 | else if (static_branch_likely(&gcm_use_avx) && do_avx) |
711 | aesni_gcm_init_avx_gen2(my_ctx_data: aes_ctx, gdata: data, iv, hash_subkey, aad: assoc, |
712 | aad_len: assoclen); |
713 | else |
714 | aesni_gcm_init(ctx: aes_ctx, gdata: data, iv, hash_subkey, aad: assoc, aad_len: assoclen); |
715 | kernel_fpu_end(); |
716 | |
717 | if (!assocmem) |
718 | scatterwalk_unmap(vaddr: assoc); |
719 | else |
720 | kfree(objp: assocmem); |
721 | |
722 | err = enc ? skcipher_walk_aead_encrypt(walk: &walk, req, atomic: false) |
723 | : skcipher_walk_aead_decrypt(walk: &walk, req, atomic: false); |
724 | |
725 | while (walk.nbytes > 0) { |
726 | kernel_fpu_begin(); |
727 | if (static_branch_likely(&gcm_use_avx2) && do_avx2) { |
728 | if (enc) |
729 | aesni_gcm_enc_update_avx_gen4(ctx: aes_ctx, gdata: data, |
730 | out: walk.dst.virt.addr, |
731 | in: walk.src.virt.addr, |
732 | plaintext_len: walk.nbytes); |
733 | else |
734 | aesni_gcm_dec_update_avx_gen4(ctx: aes_ctx, gdata: data, |
735 | out: walk.dst.virt.addr, |
736 | in: walk.src.virt.addr, |
737 | ciphertext_len: walk.nbytes); |
738 | } else if (static_branch_likely(&gcm_use_avx) && do_avx) { |
739 | if (enc) |
740 | aesni_gcm_enc_update_avx_gen2(ctx: aes_ctx, gdata: data, |
741 | out: walk.dst.virt.addr, |
742 | in: walk.src.virt.addr, |
743 | plaintext_len: walk.nbytes); |
744 | else |
745 | aesni_gcm_dec_update_avx_gen2(ctx: aes_ctx, gdata: data, |
746 | out: walk.dst.virt.addr, |
747 | in: walk.src.virt.addr, |
748 | ciphertext_len: walk.nbytes); |
749 | } else if (enc) { |
750 | aesni_gcm_enc_update(ctx: aes_ctx, gdata: data, out: walk.dst.virt.addr, |
751 | in: walk.src.virt.addr, plaintext_len: walk.nbytes); |
752 | } else { |
753 | aesni_gcm_dec_update(ctx: aes_ctx, gdata: data, out: walk.dst.virt.addr, |
754 | in: walk.src.virt.addr, ciphertext_len: walk.nbytes); |
755 | } |
756 | kernel_fpu_end(); |
757 | |
758 | err = skcipher_walk_done(walk: &walk, err: 0); |
759 | } |
760 | |
761 | if (err) |
762 | return err; |
763 | |
764 | kernel_fpu_begin(); |
765 | if (static_branch_likely(&gcm_use_avx2) && do_avx2) |
766 | aesni_gcm_finalize_avx_gen4(ctx: aes_ctx, gdata: data, auth_tag, |
767 | auth_tag_len); |
768 | else if (static_branch_likely(&gcm_use_avx) && do_avx) |
769 | aesni_gcm_finalize_avx_gen2(ctx: aes_ctx, gdata: data, auth_tag, |
770 | auth_tag_len); |
771 | else |
772 | aesni_gcm_finalize(ctx: aes_ctx, gdata: data, auth_tag, auth_tag_len); |
773 | kernel_fpu_end(); |
774 | |
775 | return 0; |
776 | } |
777 | |
778 | static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen, |
779 | u8 *hash_subkey, u8 *iv, void *aes_ctx) |
780 | { |
781 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); |
782 | unsigned long auth_tag_len = crypto_aead_authsize(tfm); |
783 | u8 auth_tag[16]; |
784 | int err; |
785 | |
786 | err = gcmaes_crypt_by_sg(enc: true, req, assoclen, hash_subkey, iv, aes_ctx, |
787 | auth_tag, auth_tag_len); |
788 | if (err) |
789 | return err; |
790 | |
791 | scatterwalk_map_and_copy(buf: auth_tag, sg: req->dst, |
792 | start: req->assoclen + req->cryptlen, |
793 | nbytes: auth_tag_len, out: 1); |
794 | return 0; |
795 | } |
796 | |
797 | static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen, |
798 | u8 *hash_subkey, u8 *iv, void *aes_ctx) |
799 | { |
800 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); |
801 | unsigned long auth_tag_len = crypto_aead_authsize(tfm); |
802 | u8 auth_tag_msg[16]; |
803 | u8 auth_tag[16]; |
804 | int err; |
805 | |
806 | err = gcmaes_crypt_by_sg(enc: false, req, assoclen, hash_subkey, iv, aes_ctx, |
807 | auth_tag, auth_tag_len); |
808 | if (err) |
809 | return err; |
810 | |
811 | /* Copy out original auth_tag */ |
812 | scatterwalk_map_and_copy(buf: auth_tag_msg, sg: req->src, |
813 | start: req->assoclen + req->cryptlen - auth_tag_len, |
814 | nbytes: auth_tag_len, out: 0); |
815 | |
816 | /* Compare generated tag with passed in tag. */ |
817 | if (crypto_memneq(a: auth_tag_msg, b: auth_tag, size: auth_tag_len)) { |
818 | memzero_explicit(s: auth_tag, count: sizeof(auth_tag)); |
819 | return -EBADMSG; |
820 | } |
821 | return 0; |
822 | } |
823 | |
824 | static int helper_rfc4106_encrypt(struct aead_request *req) |
825 | { |
826 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); |
827 | struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); |
828 | void *aes_ctx = &(ctx->aes_key_expanded); |
829 | u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); |
830 | u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); |
831 | unsigned int i; |
832 | __be32 counter = cpu_to_be32(1); |
833 | |
834 | /* Assuming we are supporting rfc4106 64-bit extended */ |
835 | /* sequence numbers We need to have the AAD length equal */ |
836 | /* to 16 or 20 bytes */ |
837 | if (unlikely(req->assoclen != 16 && req->assoclen != 20)) |
838 | return -EINVAL; |
839 | |
840 | /* IV below built */ |
841 | for (i = 0; i < 4; i++) |
842 | *(iv+i) = ctx->nonce[i]; |
843 | for (i = 0; i < 8; i++) |
844 | *(iv+4+i) = req->iv[i]; |
845 | *((__be32 *)(iv+12)) = counter; |
846 | |
847 | return gcmaes_encrypt(req, assoclen: req->assoclen - 8, hash_subkey: ctx->hash_subkey, iv, |
848 | aes_ctx); |
849 | } |
850 | |
851 | static int helper_rfc4106_decrypt(struct aead_request *req) |
852 | { |
853 | __be32 counter = cpu_to_be32(1); |
854 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); |
855 | struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); |
856 | void *aes_ctx = &(ctx->aes_key_expanded); |
857 | u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); |
858 | u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); |
859 | unsigned int i; |
860 | |
861 | if (unlikely(req->assoclen != 16 && req->assoclen != 20)) |
862 | return -EINVAL; |
863 | |
864 | /* Assuming we are supporting rfc4106 64-bit extended */ |
865 | /* sequence numbers We need to have the AAD length */ |
866 | /* equal to 16 or 20 bytes */ |
867 | |
868 | /* IV below built */ |
869 | for (i = 0; i < 4; i++) |
870 | *(iv+i) = ctx->nonce[i]; |
871 | for (i = 0; i < 8; i++) |
872 | *(iv+4+i) = req->iv[i]; |
873 | *((__be32 *)(iv+12)) = counter; |
874 | |
875 | return gcmaes_decrypt(req, assoclen: req->assoclen - 8, hash_subkey: ctx->hash_subkey, iv, |
876 | aes_ctx); |
877 | } |
878 | #endif |
879 | |
880 | static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key, |
881 | unsigned int keylen) |
882 | { |
883 | struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm); |
884 | int err; |
885 | |
886 | err = xts_verify_key(tfm, key, keylen); |
887 | if (err) |
888 | return err; |
889 | |
890 | keylen /= 2; |
891 | |
892 | /* first half of xts-key is for crypt */ |
893 | err = aes_set_key_common(ctx: &ctx->crypt_ctx, in_key: key, key_len: keylen); |
894 | if (err) |
895 | return err; |
896 | |
897 | /* second half of xts-key is for tweak */ |
898 | return aes_set_key_common(ctx: &ctx->tweak_ctx, in_key: key + keylen, key_len: keylen); |
899 | } |
900 | |
901 | static int xts_crypt(struct skcipher_request *req, bool encrypt) |
902 | { |
903 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
904 | struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm); |
905 | int tail = req->cryptlen % AES_BLOCK_SIZE; |
906 | struct skcipher_request subreq; |
907 | struct skcipher_walk walk; |
908 | int err; |
909 | |
910 | if (req->cryptlen < AES_BLOCK_SIZE) |
911 | return -EINVAL; |
912 | |
913 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
914 | if (!walk.nbytes) |
915 | return err; |
916 | |
917 | if (unlikely(tail > 0 && walk.nbytes < walk.total)) { |
918 | int blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; |
919 | |
920 | skcipher_walk_abort(walk: &walk); |
921 | |
922 | skcipher_request_set_tfm(req: &subreq, tfm); |
923 | skcipher_request_set_callback(req: &subreq, |
924 | flags: skcipher_request_flags(req), |
925 | NULL, NULL); |
926 | skcipher_request_set_crypt(req: &subreq, src: req->src, dst: req->dst, |
927 | cryptlen: blocks * AES_BLOCK_SIZE, iv: req->iv); |
928 | req = &subreq; |
929 | |
930 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
931 | if (!walk.nbytes) |
932 | return err; |
933 | } else { |
934 | tail = 0; |
935 | } |
936 | |
937 | kernel_fpu_begin(); |
938 | |
939 | /* calculate first value of T */ |
940 | aesni_enc(ctx: &ctx->tweak_ctx, out: walk.iv, in: walk.iv); |
941 | |
942 | while (walk.nbytes > 0) { |
943 | int nbytes = walk.nbytes; |
944 | |
945 | if (nbytes < walk.total) |
946 | nbytes &= ~(AES_BLOCK_SIZE - 1); |
947 | |
948 | if (encrypt) |
949 | aesni_xts_encrypt(ctx: &ctx->crypt_ctx, |
950 | out: walk.dst.virt.addr, in: walk.src.virt.addr, |
951 | len: nbytes, iv: walk.iv); |
952 | else |
953 | aesni_xts_decrypt(ctx: &ctx->crypt_ctx, |
954 | out: walk.dst.virt.addr, in: walk.src.virt.addr, |
955 | len: nbytes, iv: walk.iv); |
956 | kernel_fpu_end(); |
957 | |
958 | err = skcipher_walk_done(walk: &walk, err: walk.nbytes - nbytes); |
959 | |
960 | if (walk.nbytes > 0) |
961 | kernel_fpu_begin(); |
962 | } |
963 | |
964 | if (unlikely(tail > 0 && !err)) { |
965 | struct scatterlist sg_src[2], sg_dst[2]; |
966 | struct scatterlist *src, *dst; |
967 | |
968 | dst = src = scatterwalk_ffwd(dst: sg_src, src: req->src, len: req->cryptlen); |
969 | if (req->dst != req->src) |
970 | dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst, len: req->cryptlen); |
971 | |
972 | skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, |
973 | iv: req->iv); |
974 | |
975 | err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false); |
976 | if (err) |
977 | return err; |
978 | |
979 | kernel_fpu_begin(); |
980 | if (encrypt) |
981 | aesni_xts_encrypt(ctx: &ctx->crypt_ctx, |
982 | out: walk.dst.virt.addr, in: walk.src.virt.addr, |
983 | len: walk.nbytes, iv: walk.iv); |
984 | else |
985 | aesni_xts_decrypt(ctx: &ctx->crypt_ctx, |
986 | out: walk.dst.virt.addr, in: walk.src.virt.addr, |
987 | len: walk.nbytes, iv: walk.iv); |
988 | kernel_fpu_end(); |
989 | |
990 | err = skcipher_walk_done(walk: &walk, err: 0); |
991 | } |
992 | return err; |
993 | } |
994 | |
995 | static int xts_encrypt(struct skcipher_request *req) |
996 | { |
997 | return xts_crypt(req, encrypt: true); |
998 | } |
999 | |
1000 | static int xts_decrypt(struct skcipher_request *req) |
1001 | { |
1002 | return xts_crypt(req, encrypt: false); |
1003 | } |
1004 | |
1005 | static struct crypto_alg aesni_cipher_alg = { |
1006 | .cra_name = "aes" , |
1007 | .cra_driver_name = "aes-aesni" , |
1008 | .cra_priority = 300, |
1009 | .cra_flags = CRYPTO_ALG_TYPE_CIPHER, |
1010 | .cra_blocksize = AES_BLOCK_SIZE, |
1011 | .cra_ctxsize = CRYPTO_AES_CTX_SIZE, |
1012 | .cra_module = THIS_MODULE, |
1013 | .cra_u = { |
1014 | .cipher = { |
1015 | .cia_min_keysize = AES_MIN_KEY_SIZE, |
1016 | .cia_max_keysize = AES_MAX_KEY_SIZE, |
1017 | .cia_setkey = aes_set_key, |
1018 | .cia_encrypt = aesni_encrypt, |
1019 | .cia_decrypt = aesni_decrypt |
1020 | } |
1021 | } |
1022 | }; |
1023 | |
1024 | static struct skcipher_alg aesni_skciphers[] = { |
1025 | { |
1026 | .base = { |
1027 | .cra_name = "__ecb(aes)" , |
1028 | .cra_driver_name = "__ecb-aes-aesni" , |
1029 | .cra_priority = 400, |
1030 | .cra_flags = CRYPTO_ALG_INTERNAL, |
1031 | .cra_blocksize = AES_BLOCK_SIZE, |
1032 | .cra_ctxsize = CRYPTO_AES_CTX_SIZE, |
1033 | .cra_module = THIS_MODULE, |
1034 | }, |
1035 | .min_keysize = AES_MIN_KEY_SIZE, |
1036 | .max_keysize = AES_MAX_KEY_SIZE, |
1037 | .setkey = aesni_skcipher_setkey, |
1038 | .encrypt = ecb_encrypt, |
1039 | .decrypt = ecb_decrypt, |
1040 | }, { |
1041 | .base = { |
1042 | .cra_name = "__cbc(aes)" , |
1043 | .cra_driver_name = "__cbc-aes-aesni" , |
1044 | .cra_priority = 400, |
1045 | .cra_flags = CRYPTO_ALG_INTERNAL, |
1046 | .cra_blocksize = AES_BLOCK_SIZE, |
1047 | .cra_ctxsize = CRYPTO_AES_CTX_SIZE, |
1048 | .cra_module = THIS_MODULE, |
1049 | }, |
1050 | .min_keysize = AES_MIN_KEY_SIZE, |
1051 | .max_keysize = AES_MAX_KEY_SIZE, |
1052 | .ivsize = AES_BLOCK_SIZE, |
1053 | .setkey = aesni_skcipher_setkey, |
1054 | .encrypt = cbc_encrypt, |
1055 | .decrypt = cbc_decrypt, |
1056 | }, { |
1057 | .base = { |
1058 | .cra_name = "__cts(cbc(aes))" , |
1059 | .cra_driver_name = "__cts-cbc-aes-aesni" , |
1060 | .cra_priority = 400, |
1061 | .cra_flags = CRYPTO_ALG_INTERNAL, |
1062 | .cra_blocksize = AES_BLOCK_SIZE, |
1063 | .cra_ctxsize = CRYPTO_AES_CTX_SIZE, |
1064 | .cra_module = THIS_MODULE, |
1065 | }, |
1066 | .min_keysize = AES_MIN_KEY_SIZE, |
1067 | .max_keysize = AES_MAX_KEY_SIZE, |
1068 | .ivsize = AES_BLOCK_SIZE, |
1069 | .walksize = 2 * AES_BLOCK_SIZE, |
1070 | .setkey = aesni_skcipher_setkey, |
1071 | .encrypt = cts_cbc_encrypt, |
1072 | .decrypt = cts_cbc_decrypt, |
1073 | #ifdef CONFIG_X86_64 |
1074 | }, { |
1075 | .base = { |
1076 | .cra_name = "__ctr(aes)" , |
1077 | .cra_driver_name = "__ctr-aes-aesni" , |
1078 | .cra_priority = 400, |
1079 | .cra_flags = CRYPTO_ALG_INTERNAL, |
1080 | .cra_blocksize = 1, |
1081 | .cra_ctxsize = CRYPTO_AES_CTX_SIZE, |
1082 | .cra_module = THIS_MODULE, |
1083 | }, |
1084 | .min_keysize = AES_MIN_KEY_SIZE, |
1085 | .max_keysize = AES_MAX_KEY_SIZE, |
1086 | .ivsize = AES_BLOCK_SIZE, |
1087 | .chunksize = AES_BLOCK_SIZE, |
1088 | .setkey = aesni_skcipher_setkey, |
1089 | .encrypt = ctr_crypt, |
1090 | .decrypt = ctr_crypt, |
1091 | #endif |
1092 | }, { |
1093 | .base = { |
1094 | .cra_name = "__xts(aes)" , |
1095 | .cra_driver_name = "__xts-aes-aesni" , |
1096 | .cra_priority = 401, |
1097 | .cra_flags = CRYPTO_ALG_INTERNAL, |
1098 | .cra_blocksize = AES_BLOCK_SIZE, |
1099 | .cra_ctxsize = XTS_AES_CTX_SIZE, |
1100 | .cra_module = THIS_MODULE, |
1101 | }, |
1102 | .min_keysize = 2 * AES_MIN_KEY_SIZE, |
1103 | .max_keysize = 2 * AES_MAX_KEY_SIZE, |
1104 | .ivsize = AES_BLOCK_SIZE, |
1105 | .walksize = 2 * AES_BLOCK_SIZE, |
1106 | .setkey = xts_aesni_setkey, |
1107 | .encrypt = xts_encrypt, |
1108 | .decrypt = xts_decrypt, |
1109 | } |
1110 | }; |
1111 | |
1112 | static |
1113 | struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)]; |
1114 | |
1115 | #ifdef CONFIG_X86_64 |
1116 | /* |
1117 | * XCTR does not have a non-AVX implementation, so it must be enabled |
1118 | * conditionally. |
1119 | */ |
1120 | static struct skcipher_alg aesni_xctr = { |
1121 | .base = { |
1122 | .cra_name = "__xctr(aes)" , |
1123 | .cra_driver_name = "__xctr-aes-aesni" , |
1124 | .cra_priority = 400, |
1125 | .cra_flags = CRYPTO_ALG_INTERNAL, |
1126 | .cra_blocksize = 1, |
1127 | .cra_ctxsize = CRYPTO_AES_CTX_SIZE, |
1128 | .cra_module = THIS_MODULE, |
1129 | }, |
1130 | .min_keysize = AES_MIN_KEY_SIZE, |
1131 | .max_keysize = AES_MAX_KEY_SIZE, |
1132 | .ivsize = AES_BLOCK_SIZE, |
1133 | .chunksize = AES_BLOCK_SIZE, |
1134 | .setkey = aesni_skcipher_setkey, |
1135 | .encrypt = xctr_crypt, |
1136 | .decrypt = xctr_crypt, |
1137 | }; |
1138 | |
1139 | static struct simd_skcipher_alg *aesni_simd_xctr; |
1140 | #endif /* CONFIG_X86_64 */ |
1141 | |
1142 | #ifdef CONFIG_X86_64 |
1143 | static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key, |
1144 | unsigned int key_len) |
1145 | { |
1146 | struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm: aead); |
1147 | |
1148 | return aes_set_key_common(ctx: &ctx->aes_key_expanded, in_key: key, key_len) ?: |
1149 | rfc4106_set_hash_subkey(hash_subkey: ctx->hash_subkey, key, key_len); |
1150 | } |
1151 | |
1152 | static int generic_gcmaes_encrypt(struct aead_request *req) |
1153 | { |
1154 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); |
1155 | struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); |
1156 | void *aes_ctx = &(ctx->aes_key_expanded); |
1157 | u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); |
1158 | u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); |
1159 | __be32 counter = cpu_to_be32(1); |
1160 | |
1161 | memcpy(iv, req->iv, 12); |
1162 | *((__be32 *)(iv+12)) = counter; |
1163 | |
1164 | return gcmaes_encrypt(req, assoclen: req->assoclen, hash_subkey: ctx->hash_subkey, iv, |
1165 | aes_ctx); |
1166 | } |
1167 | |
1168 | static int generic_gcmaes_decrypt(struct aead_request *req) |
1169 | { |
1170 | __be32 counter = cpu_to_be32(1); |
1171 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); |
1172 | struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); |
1173 | void *aes_ctx = &(ctx->aes_key_expanded); |
1174 | u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); |
1175 | u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); |
1176 | |
1177 | memcpy(iv, req->iv, 12); |
1178 | *((__be32 *)(iv+12)) = counter; |
1179 | |
1180 | return gcmaes_decrypt(req, assoclen: req->assoclen, hash_subkey: ctx->hash_subkey, iv, |
1181 | aes_ctx); |
1182 | } |
1183 | |
1184 | static struct aead_alg aesni_aeads[] = { { |
1185 | .setkey = common_rfc4106_set_key, |
1186 | .setauthsize = common_rfc4106_set_authsize, |
1187 | .encrypt = helper_rfc4106_encrypt, |
1188 | .decrypt = helper_rfc4106_decrypt, |
1189 | .ivsize = GCM_RFC4106_IV_SIZE, |
1190 | .maxauthsize = 16, |
1191 | .base = { |
1192 | .cra_name = "__rfc4106(gcm(aes))" , |
1193 | .cra_driver_name = "__rfc4106-gcm-aesni" , |
1194 | .cra_priority = 400, |
1195 | .cra_flags = CRYPTO_ALG_INTERNAL, |
1196 | .cra_blocksize = 1, |
1197 | .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx), |
1198 | .cra_alignmask = 0, |
1199 | .cra_module = THIS_MODULE, |
1200 | }, |
1201 | }, { |
1202 | .setkey = generic_gcmaes_set_key, |
1203 | .setauthsize = generic_gcmaes_set_authsize, |
1204 | .encrypt = generic_gcmaes_encrypt, |
1205 | .decrypt = generic_gcmaes_decrypt, |
1206 | .ivsize = GCM_AES_IV_SIZE, |
1207 | .maxauthsize = 16, |
1208 | .base = { |
1209 | .cra_name = "__gcm(aes)" , |
1210 | .cra_driver_name = "__generic-gcm-aesni" , |
1211 | .cra_priority = 400, |
1212 | .cra_flags = CRYPTO_ALG_INTERNAL, |
1213 | .cra_blocksize = 1, |
1214 | .cra_ctxsize = sizeof(struct generic_gcmaes_ctx), |
1215 | .cra_alignmask = 0, |
1216 | .cra_module = THIS_MODULE, |
1217 | }, |
1218 | } }; |
1219 | #else |
1220 | static struct aead_alg aesni_aeads[0]; |
1221 | #endif |
1222 | |
1223 | static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)]; |
1224 | |
1225 | static const struct x86_cpu_id aesni_cpu_id[] = { |
1226 | X86_MATCH_FEATURE(X86_FEATURE_AES, NULL), |
1227 | {} |
1228 | }; |
1229 | MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id); |
1230 | |
1231 | static int __init aesni_init(void) |
1232 | { |
1233 | int err; |
1234 | |
1235 | if (!x86_match_cpu(match: aesni_cpu_id)) |
1236 | return -ENODEV; |
1237 | #ifdef CONFIG_X86_64 |
1238 | if (boot_cpu_has(X86_FEATURE_AVX2)) { |
1239 | pr_info("AVX2 version of gcm_enc/dec engaged.\n" ); |
1240 | static_branch_enable(&gcm_use_avx); |
1241 | static_branch_enable(&gcm_use_avx2); |
1242 | } else |
1243 | if (boot_cpu_has(X86_FEATURE_AVX)) { |
1244 | pr_info("AVX version of gcm_enc/dec engaged.\n" ); |
1245 | static_branch_enable(&gcm_use_avx); |
1246 | } else { |
1247 | pr_info("SSE version of gcm_enc/dec engaged.\n" ); |
1248 | } |
1249 | if (boot_cpu_has(X86_FEATURE_AVX)) { |
1250 | /* optimize performance of ctr mode encryption transform */ |
1251 | static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm); |
1252 | pr_info("AES CTR mode by8 optimization enabled\n" ); |
1253 | } |
1254 | #endif /* CONFIG_X86_64 */ |
1255 | |
1256 | err = crypto_register_alg(alg: &aesni_cipher_alg); |
1257 | if (err) |
1258 | return err; |
1259 | |
1260 | err = simd_register_skciphers_compat(algs: aesni_skciphers, |
1261 | ARRAY_SIZE(aesni_skciphers), |
1262 | simd_algs: aesni_simd_skciphers); |
1263 | if (err) |
1264 | goto unregister_cipher; |
1265 | |
1266 | err = simd_register_aeads_compat(algs: aesni_aeads, ARRAY_SIZE(aesni_aeads), |
1267 | simd_algs: aesni_simd_aeads); |
1268 | if (err) |
1269 | goto unregister_skciphers; |
1270 | |
1271 | #ifdef CONFIG_X86_64 |
1272 | if (boot_cpu_has(X86_FEATURE_AVX)) |
1273 | err = simd_register_skciphers_compat(algs: &aesni_xctr, count: 1, |
1274 | simd_algs: &aesni_simd_xctr); |
1275 | if (err) |
1276 | goto unregister_aeads; |
1277 | #endif /* CONFIG_X86_64 */ |
1278 | |
1279 | return 0; |
1280 | |
1281 | #ifdef CONFIG_X86_64 |
1282 | unregister_aeads: |
1283 | simd_unregister_aeads(algs: aesni_aeads, ARRAY_SIZE(aesni_aeads), |
1284 | simd_algs: aesni_simd_aeads); |
1285 | #endif /* CONFIG_X86_64 */ |
1286 | |
1287 | unregister_skciphers: |
1288 | simd_unregister_skciphers(algs: aesni_skciphers, ARRAY_SIZE(aesni_skciphers), |
1289 | simd_algs: aesni_simd_skciphers); |
1290 | unregister_cipher: |
1291 | crypto_unregister_alg(alg: &aesni_cipher_alg); |
1292 | return err; |
1293 | } |
1294 | |
1295 | static void __exit aesni_exit(void) |
1296 | { |
1297 | simd_unregister_aeads(algs: aesni_aeads, ARRAY_SIZE(aesni_aeads), |
1298 | simd_algs: aesni_simd_aeads); |
1299 | simd_unregister_skciphers(algs: aesni_skciphers, ARRAY_SIZE(aesni_skciphers), |
1300 | simd_algs: aesni_simd_skciphers); |
1301 | crypto_unregister_alg(alg: &aesni_cipher_alg); |
1302 | #ifdef CONFIG_X86_64 |
1303 | if (boot_cpu_has(X86_FEATURE_AVX)) |
1304 | simd_unregister_skciphers(algs: &aesni_xctr, count: 1, simd_algs: &aesni_simd_xctr); |
1305 | #endif /* CONFIG_X86_64 */ |
1306 | } |
1307 | |
1308 | late_initcall(aesni_init); |
1309 | module_exit(aesni_exit); |
1310 | |
1311 | MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized" ); |
1312 | MODULE_LICENSE("GPL" ); |
1313 | MODULE_ALIAS_CRYPTO("aes" ); |
1314 | |