1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Support for Intel AES-NI instructions. This file contains glue
4 * code, the real AES implementation is in intel-aes_asm.S.
5 *
6 * Copyright (C) 2008, Intel Corp.
7 * Author: Huang Ying <ying.huang@intel.com>
8 *
9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
10 * interface for 64-bit kernels.
11 * Authors: Adrian Hoban <adrian.hoban@intel.com>
12 * Gabriele Paoloni <gabriele.paoloni@intel.com>
13 * Tadeusz Struk (tadeusz.struk@intel.com)
14 * Aidan O'Mahony (aidan.o.mahony@intel.com)
15 * Copyright (c) 2010, Intel Corporation.
16 */
17
18#include <linux/hardirq.h>
19#include <linux/types.h>
20#include <linux/module.h>
21#include <linux/err.h>
22#include <crypto/algapi.h>
23#include <crypto/aes.h>
24#include <crypto/ctr.h>
25#include <crypto/b128ops.h>
26#include <crypto/gcm.h>
27#include <crypto/xts.h>
28#include <asm/cpu_device_id.h>
29#include <asm/simd.h>
30#include <crypto/scatterwalk.h>
31#include <crypto/internal/aead.h>
32#include <crypto/internal/simd.h>
33#include <crypto/internal/skcipher.h>
34#include <linux/jump_label.h>
35#include <linux/workqueue.h>
36#include <linux/spinlock.h>
37#include <linux/static_call.h>
38
39
40#define AESNI_ALIGN 16
41#define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
42#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
43#define RFC4106_HASH_SUBKEY_SIZE 16
44#define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
45#define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
46#define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
47
48/* This data is stored at the end of the crypto_tfm struct.
49 * It's a type of per "session" data storage location.
50 * This needs to be 16 byte aligned.
51 */
52struct aesni_rfc4106_gcm_ctx {
53 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
54 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
55 u8 nonce[4];
56};
57
58struct generic_gcmaes_ctx {
59 u8 hash_subkey[16] AESNI_ALIGN_ATTR;
60 struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
61};
62
63struct aesni_xts_ctx {
64 struct crypto_aes_ctx tweak_ctx AESNI_ALIGN_ATTR;
65 struct crypto_aes_ctx crypt_ctx AESNI_ALIGN_ATTR;
66};
67
68#define GCM_BLOCK_LEN 16
69
70struct gcm_context_data {
71 /* init, update and finalize context data */
72 u8 aad_hash[GCM_BLOCK_LEN];
73 u64 aad_length;
74 u64 in_length;
75 u8 partial_block_enc_key[GCM_BLOCK_LEN];
76 u8 orig_IV[GCM_BLOCK_LEN];
77 u8 current_counter[GCM_BLOCK_LEN];
78 u64 partial_block_len;
79 u64 unused;
80 u8 hash_keys[GCM_BLOCK_LEN * 16];
81};
82
83static inline void *aes_align_addr(void *addr)
84{
85 if (crypto_tfm_ctx_alignment() >= AESNI_ALIGN)
86 return addr;
87 return PTR_ALIGN(addr, AESNI_ALIGN);
88}
89
90asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
91 unsigned int key_len);
92asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
93asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
94asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
95 const u8 *in, unsigned int len);
96asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
97 const u8 *in, unsigned int len);
98asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
99 const u8 *in, unsigned int len, u8 *iv);
100asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
101 const u8 *in, unsigned int len, u8 *iv);
102asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
103 const u8 *in, unsigned int len, u8 *iv);
104asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
105 const u8 *in, unsigned int len, u8 *iv);
106
107#define AVX_GEN2_OPTSIZE 640
108#define AVX_GEN4_OPTSIZE 4096
109
110asmlinkage void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *out,
111 const u8 *in, unsigned int len, u8 *iv);
112
113asmlinkage void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *out,
114 const u8 *in, unsigned int len, u8 *iv);
115
116#ifdef CONFIG_X86_64
117
118asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
119 const u8 *in, unsigned int len, u8 *iv);
120DEFINE_STATIC_CALL(aesni_ctr_enc_tfm, aesni_ctr_enc);
121
122/* Scatter / Gather routines, with args similar to above */
123asmlinkage void aesni_gcm_init(void *ctx,
124 struct gcm_context_data *gdata,
125 u8 *iv,
126 u8 *hash_subkey, const u8 *aad,
127 unsigned long aad_len);
128asmlinkage void aesni_gcm_enc_update(void *ctx,
129 struct gcm_context_data *gdata, u8 *out,
130 const u8 *in, unsigned long plaintext_len);
131asmlinkage void aesni_gcm_dec_update(void *ctx,
132 struct gcm_context_data *gdata, u8 *out,
133 const u8 *in,
134 unsigned long ciphertext_len);
135asmlinkage void aesni_gcm_finalize(void *ctx,
136 struct gcm_context_data *gdata,
137 u8 *auth_tag, unsigned long auth_tag_len);
138
139asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
140 void *keys, u8 *out, unsigned int num_bytes);
141asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
142 void *keys, u8 *out, unsigned int num_bytes);
143asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
144 void *keys, u8 *out, unsigned int num_bytes);
145
146
147asmlinkage void aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv,
148 const void *keys, u8 *out, unsigned int num_bytes,
149 unsigned int byte_ctr);
150
151asmlinkage void aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv,
152 const void *keys, u8 *out, unsigned int num_bytes,
153 unsigned int byte_ctr);
154
155asmlinkage void aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv,
156 const void *keys, u8 *out, unsigned int num_bytes,
157 unsigned int byte_ctr);
158
159/*
160 * asmlinkage void aesni_gcm_init_avx_gen2()
161 * gcm_data *my_ctx_data, context data
162 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
163 */
164asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
165 struct gcm_context_data *gdata,
166 u8 *iv,
167 u8 *hash_subkey,
168 const u8 *aad,
169 unsigned long aad_len);
170
171asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
172 struct gcm_context_data *gdata, u8 *out,
173 const u8 *in, unsigned long plaintext_len);
174asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
175 struct gcm_context_data *gdata, u8 *out,
176 const u8 *in,
177 unsigned long ciphertext_len);
178asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
179 struct gcm_context_data *gdata,
180 u8 *auth_tag, unsigned long auth_tag_len);
181
182/*
183 * asmlinkage void aesni_gcm_init_avx_gen4()
184 * gcm_data *my_ctx_data, context data
185 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
186 */
187asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
188 struct gcm_context_data *gdata,
189 u8 *iv,
190 u8 *hash_subkey,
191 const u8 *aad,
192 unsigned long aad_len);
193
194asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
195 struct gcm_context_data *gdata, u8 *out,
196 const u8 *in, unsigned long plaintext_len);
197asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
198 struct gcm_context_data *gdata, u8 *out,
199 const u8 *in,
200 unsigned long ciphertext_len);
201asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
202 struct gcm_context_data *gdata,
203 u8 *auth_tag, unsigned long auth_tag_len);
204
205static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx);
206static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx2);
207
208static inline struct
209aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
210{
211 return aes_align_addr(addr: crypto_aead_ctx(tfm));
212}
213
214static inline struct
215generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
216{
217 return aes_align_addr(addr: crypto_aead_ctx(tfm));
218}
219#endif
220
221static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
222{
223 return aes_align_addr(addr: raw_ctx);
224}
225
226static inline struct aesni_xts_ctx *aes_xts_ctx(struct crypto_skcipher *tfm)
227{
228 return aes_align_addr(addr: crypto_skcipher_ctx(tfm));
229}
230
231static int aes_set_key_common(struct crypto_aes_ctx *ctx,
232 const u8 *in_key, unsigned int key_len)
233{
234 int err;
235
236 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
237 key_len != AES_KEYSIZE_256)
238 return -EINVAL;
239
240 if (!crypto_simd_usable())
241 err = aes_expandkey(ctx, in_key, key_len);
242 else {
243 kernel_fpu_begin();
244 err = aesni_set_key(ctx, in_key, key_len);
245 kernel_fpu_end();
246 }
247
248 return err;
249}
250
251static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
252 unsigned int key_len)
253{
254 return aes_set_key_common(ctx: aes_ctx(raw_ctx: crypto_tfm_ctx(tfm)), in_key,
255 key_len);
256}
257
258static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
259{
260 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_tfm_ctx(tfm));
261
262 if (!crypto_simd_usable()) {
263 aes_encrypt(ctx, out: dst, in: src);
264 } else {
265 kernel_fpu_begin();
266 aesni_enc(ctx, out: dst, in: src);
267 kernel_fpu_end();
268 }
269}
270
271static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
272{
273 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_tfm_ctx(tfm));
274
275 if (!crypto_simd_usable()) {
276 aes_decrypt(ctx, out: dst, in: src);
277 } else {
278 kernel_fpu_begin();
279 aesni_dec(ctx, out: dst, in: src);
280 kernel_fpu_end();
281 }
282}
283
284static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
285 unsigned int len)
286{
287 return aes_set_key_common(ctx: aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm)), in_key: key, key_len: len);
288}
289
290static int ecb_encrypt(struct skcipher_request *req)
291{
292 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
293 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
294 struct skcipher_walk walk;
295 unsigned int nbytes;
296 int err;
297
298 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
299
300 while ((nbytes = walk.nbytes)) {
301 kernel_fpu_begin();
302 aesni_ecb_enc(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr,
303 len: nbytes & AES_BLOCK_MASK);
304 kernel_fpu_end();
305 nbytes &= AES_BLOCK_SIZE - 1;
306 err = skcipher_walk_done(walk: &walk, err: nbytes);
307 }
308
309 return err;
310}
311
312static int ecb_decrypt(struct skcipher_request *req)
313{
314 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
315 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
316 struct skcipher_walk walk;
317 unsigned int nbytes;
318 int err;
319
320 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
321
322 while ((nbytes = walk.nbytes)) {
323 kernel_fpu_begin();
324 aesni_ecb_dec(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr,
325 len: nbytes & AES_BLOCK_MASK);
326 kernel_fpu_end();
327 nbytes &= AES_BLOCK_SIZE - 1;
328 err = skcipher_walk_done(walk: &walk, err: nbytes);
329 }
330
331 return err;
332}
333
334static int cbc_encrypt(struct skcipher_request *req)
335{
336 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
337 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
338 struct skcipher_walk walk;
339 unsigned int nbytes;
340 int err;
341
342 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
343
344 while ((nbytes = walk.nbytes)) {
345 kernel_fpu_begin();
346 aesni_cbc_enc(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr,
347 len: nbytes & AES_BLOCK_MASK, iv: walk.iv);
348 kernel_fpu_end();
349 nbytes &= AES_BLOCK_SIZE - 1;
350 err = skcipher_walk_done(walk: &walk, err: nbytes);
351 }
352
353 return err;
354}
355
356static int cbc_decrypt(struct skcipher_request *req)
357{
358 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
359 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
360 struct skcipher_walk walk;
361 unsigned int nbytes;
362 int err;
363
364 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
365
366 while ((nbytes = walk.nbytes)) {
367 kernel_fpu_begin();
368 aesni_cbc_dec(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr,
369 len: nbytes & AES_BLOCK_MASK, iv: walk.iv);
370 kernel_fpu_end();
371 nbytes &= AES_BLOCK_SIZE - 1;
372 err = skcipher_walk_done(walk: &walk, err: nbytes);
373 }
374
375 return err;
376}
377
378static int cts_cbc_encrypt(struct skcipher_request *req)
379{
380 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
381 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
382 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
383 struct scatterlist *src = req->src, *dst = req->dst;
384 struct scatterlist sg_src[2], sg_dst[2];
385 struct skcipher_request subreq;
386 struct skcipher_walk walk;
387 int err;
388
389 skcipher_request_set_tfm(req: &subreq, tfm);
390 skcipher_request_set_callback(req: &subreq, flags: skcipher_request_flags(req),
391 NULL, NULL);
392
393 if (req->cryptlen <= AES_BLOCK_SIZE) {
394 if (req->cryptlen < AES_BLOCK_SIZE)
395 return -EINVAL;
396 cbc_blocks = 1;
397 }
398
399 if (cbc_blocks > 0) {
400 skcipher_request_set_crypt(req: &subreq, src: req->src, dst: req->dst,
401 cryptlen: cbc_blocks * AES_BLOCK_SIZE,
402 iv: req->iv);
403
404 err = cbc_encrypt(req: &subreq);
405 if (err)
406 return err;
407
408 if (req->cryptlen == AES_BLOCK_SIZE)
409 return 0;
410
411 dst = src = scatterwalk_ffwd(dst: sg_src, src: req->src, len: subreq.cryptlen);
412 if (req->dst != req->src)
413 dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst,
414 len: subreq.cryptlen);
415 }
416
417 /* handle ciphertext stealing */
418 skcipher_request_set_crypt(req: &subreq, src, dst,
419 cryptlen: req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
420 iv: req->iv);
421
422 err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false);
423 if (err)
424 return err;
425
426 kernel_fpu_begin();
427 aesni_cts_cbc_enc(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr,
428 len: walk.nbytes, iv: walk.iv);
429 kernel_fpu_end();
430
431 return skcipher_walk_done(walk: &walk, err: 0);
432}
433
434static int cts_cbc_decrypt(struct skcipher_request *req)
435{
436 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
437 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
438 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
439 struct scatterlist *src = req->src, *dst = req->dst;
440 struct scatterlist sg_src[2], sg_dst[2];
441 struct skcipher_request subreq;
442 struct skcipher_walk walk;
443 int err;
444
445 skcipher_request_set_tfm(req: &subreq, tfm);
446 skcipher_request_set_callback(req: &subreq, flags: skcipher_request_flags(req),
447 NULL, NULL);
448
449 if (req->cryptlen <= AES_BLOCK_SIZE) {
450 if (req->cryptlen < AES_BLOCK_SIZE)
451 return -EINVAL;
452 cbc_blocks = 1;
453 }
454
455 if (cbc_blocks > 0) {
456 skcipher_request_set_crypt(req: &subreq, src: req->src, dst: req->dst,
457 cryptlen: cbc_blocks * AES_BLOCK_SIZE,
458 iv: req->iv);
459
460 err = cbc_decrypt(req: &subreq);
461 if (err)
462 return err;
463
464 if (req->cryptlen == AES_BLOCK_SIZE)
465 return 0;
466
467 dst = src = scatterwalk_ffwd(dst: sg_src, src: req->src, len: subreq.cryptlen);
468 if (req->dst != req->src)
469 dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst,
470 len: subreq.cryptlen);
471 }
472
473 /* handle ciphertext stealing */
474 skcipher_request_set_crypt(req: &subreq, src, dst,
475 cryptlen: req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
476 iv: req->iv);
477
478 err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false);
479 if (err)
480 return err;
481
482 kernel_fpu_begin();
483 aesni_cts_cbc_dec(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr,
484 len: walk.nbytes, iv: walk.iv);
485 kernel_fpu_end();
486
487 return skcipher_walk_done(walk: &walk, err: 0);
488}
489
490#ifdef CONFIG_X86_64
491static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
492 const u8 *in, unsigned int len, u8 *iv)
493{
494 /*
495 * based on key length, override with the by8 version
496 * of ctr mode encryption/decryption for improved performance
497 * aes_set_key_common() ensures that key length is one of
498 * {128,192,256}
499 */
500 if (ctx->key_length == AES_KEYSIZE_128)
501 aes_ctr_enc_128_avx_by8(in, iv, keys: (void *)ctx, out, num_bytes: len);
502 else if (ctx->key_length == AES_KEYSIZE_192)
503 aes_ctr_enc_192_avx_by8(in, iv, keys: (void *)ctx, out, num_bytes: len);
504 else
505 aes_ctr_enc_256_avx_by8(in, iv, keys: (void *)ctx, out, num_bytes: len);
506}
507
508static int ctr_crypt(struct skcipher_request *req)
509{
510 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
511 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
512 u8 keystream[AES_BLOCK_SIZE];
513 struct skcipher_walk walk;
514 unsigned int nbytes;
515 int err;
516
517 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
518
519 while ((nbytes = walk.nbytes) > 0) {
520 kernel_fpu_begin();
521 if (nbytes & AES_BLOCK_MASK)
522 static_call(aesni_ctr_enc_tfm)(ctx, walk.dst.virt.addr,
523 walk.src.virt.addr,
524 nbytes & AES_BLOCK_MASK,
525 walk.iv);
526 nbytes &= ~AES_BLOCK_MASK;
527
528 if (walk.nbytes == walk.total && nbytes > 0) {
529 aesni_enc(ctx, out: keystream, in: walk.iv);
530 crypto_xor_cpy(dst: walk.dst.virt.addr + walk.nbytes - nbytes,
531 src1: walk.src.virt.addr + walk.nbytes - nbytes,
532 src2: keystream, size: nbytes);
533 crypto_inc(a: walk.iv, AES_BLOCK_SIZE);
534 nbytes = 0;
535 }
536 kernel_fpu_end();
537 err = skcipher_walk_done(walk: &walk, err: nbytes);
538 }
539 return err;
540}
541
542static void aesni_xctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
543 const u8 *in, unsigned int len, u8 *iv,
544 unsigned int byte_ctr)
545{
546 if (ctx->key_length == AES_KEYSIZE_128)
547 aes_xctr_enc_128_avx_by8(in, iv, keys: (void *)ctx, out, num_bytes: len,
548 byte_ctr);
549 else if (ctx->key_length == AES_KEYSIZE_192)
550 aes_xctr_enc_192_avx_by8(in, iv, keys: (void *)ctx, out, num_bytes: len,
551 byte_ctr);
552 else
553 aes_xctr_enc_256_avx_by8(in, iv, keys: (void *)ctx, out, num_bytes: len,
554 byte_ctr);
555}
556
557static int xctr_crypt(struct skcipher_request *req)
558{
559 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
560 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
561 u8 keystream[AES_BLOCK_SIZE];
562 struct skcipher_walk walk;
563 unsigned int nbytes;
564 unsigned int byte_ctr = 0;
565 int err;
566 __le32 block[AES_BLOCK_SIZE / sizeof(__le32)];
567
568 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
569
570 while ((nbytes = walk.nbytes) > 0) {
571 kernel_fpu_begin();
572 if (nbytes & AES_BLOCK_MASK)
573 aesni_xctr_enc_avx_tfm(ctx, out: walk.dst.virt.addr,
574 in: walk.src.virt.addr, len: nbytes & AES_BLOCK_MASK,
575 iv: walk.iv, byte_ctr);
576 nbytes &= ~AES_BLOCK_MASK;
577 byte_ctr += walk.nbytes - nbytes;
578
579 if (walk.nbytes == walk.total && nbytes > 0) {
580 memcpy(block, walk.iv, AES_BLOCK_SIZE);
581 block[0] ^= cpu_to_le32(1 + byte_ctr / AES_BLOCK_SIZE);
582 aesni_enc(ctx, out: keystream, in: (u8 *)block);
583 crypto_xor_cpy(dst: walk.dst.virt.addr + walk.nbytes -
584 nbytes, src1: walk.src.virt.addr + walk.nbytes
585 - nbytes, src2: keystream, size: nbytes);
586 byte_ctr += nbytes;
587 nbytes = 0;
588 }
589 kernel_fpu_end();
590 err = skcipher_walk_done(walk: &walk, err: nbytes);
591 }
592 return err;
593}
594
595static int
596rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
597{
598 struct crypto_aes_ctx ctx;
599 int ret;
600
601 ret = aes_expandkey(ctx: &ctx, in_key: key, key_len);
602 if (ret)
603 return ret;
604
605 /* Clear the data in the hash sub key container to zero.*/
606 /* We want to cipher all zeros to create the hash sub key. */
607 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
608
609 aes_encrypt(ctx: &ctx, out: hash_subkey, in: hash_subkey);
610
611 memzero_explicit(s: &ctx, count: sizeof(ctx));
612 return 0;
613}
614
615static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
616 unsigned int key_len)
617{
618 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm: aead);
619
620 if (key_len < 4)
621 return -EINVAL;
622
623 /*Account for 4 byte nonce at the end.*/
624 key_len -= 4;
625
626 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
627
628 return aes_set_key_common(ctx: &ctx->aes_key_expanded, in_key: key, key_len) ?:
629 rfc4106_set_hash_subkey(hash_subkey: ctx->hash_subkey, key, key_len);
630}
631
632/* This is the Integrity Check Value (aka the authentication tag) length and can
633 * be 8, 12 or 16 bytes long. */
634static int common_rfc4106_set_authsize(struct crypto_aead *aead,
635 unsigned int authsize)
636{
637 switch (authsize) {
638 case 8:
639 case 12:
640 case 16:
641 break;
642 default:
643 return -EINVAL;
644 }
645
646 return 0;
647}
648
649static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
650 unsigned int authsize)
651{
652 switch (authsize) {
653 case 4:
654 case 8:
655 case 12:
656 case 13:
657 case 14:
658 case 15:
659 case 16:
660 break;
661 default:
662 return -EINVAL;
663 }
664
665 return 0;
666}
667
668static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
669 unsigned int assoclen, u8 *hash_subkey,
670 u8 *iv, void *aes_ctx, u8 *auth_tag,
671 unsigned long auth_tag_len)
672{
673 u8 databuf[sizeof(struct gcm_context_data) + (AESNI_ALIGN - 8)] __aligned(8);
674 struct gcm_context_data *data = PTR_ALIGN((void *)databuf, AESNI_ALIGN);
675 unsigned long left = req->cryptlen;
676 struct scatter_walk assoc_sg_walk;
677 struct skcipher_walk walk;
678 bool do_avx, do_avx2;
679 u8 *assocmem = NULL;
680 u8 *assoc;
681 int err;
682
683 if (!enc)
684 left -= auth_tag_len;
685
686 do_avx = (left >= AVX_GEN2_OPTSIZE);
687 do_avx2 = (left >= AVX_GEN4_OPTSIZE);
688
689 /* Linearize assoc, if not already linear */
690 if (req->src->length >= assoclen && req->src->length) {
691 scatterwalk_start(walk: &assoc_sg_walk, sg: req->src);
692 assoc = scatterwalk_map(walk: &assoc_sg_walk);
693 } else {
694 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
695 GFP_KERNEL : GFP_ATOMIC;
696
697 /* assoc can be any length, so must be on heap */
698 assocmem = kmalloc(size: assoclen, flags);
699 if (unlikely(!assocmem))
700 return -ENOMEM;
701 assoc = assocmem;
702
703 scatterwalk_map_and_copy(buf: assoc, sg: req->src, start: 0, nbytes: assoclen, out: 0);
704 }
705
706 kernel_fpu_begin();
707 if (static_branch_likely(&gcm_use_avx2) && do_avx2)
708 aesni_gcm_init_avx_gen4(my_ctx_data: aes_ctx, gdata: data, iv, hash_subkey, aad: assoc,
709 aad_len: assoclen);
710 else if (static_branch_likely(&gcm_use_avx) && do_avx)
711 aesni_gcm_init_avx_gen2(my_ctx_data: aes_ctx, gdata: data, iv, hash_subkey, aad: assoc,
712 aad_len: assoclen);
713 else
714 aesni_gcm_init(ctx: aes_ctx, gdata: data, iv, hash_subkey, aad: assoc, aad_len: assoclen);
715 kernel_fpu_end();
716
717 if (!assocmem)
718 scatterwalk_unmap(vaddr: assoc);
719 else
720 kfree(objp: assocmem);
721
722 err = enc ? skcipher_walk_aead_encrypt(walk: &walk, req, atomic: false)
723 : skcipher_walk_aead_decrypt(walk: &walk, req, atomic: false);
724
725 while (walk.nbytes > 0) {
726 kernel_fpu_begin();
727 if (static_branch_likely(&gcm_use_avx2) && do_avx2) {
728 if (enc)
729 aesni_gcm_enc_update_avx_gen4(ctx: aes_ctx, gdata: data,
730 out: walk.dst.virt.addr,
731 in: walk.src.virt.addr,
732 plaintext_len: walk.nbytes);
733 else
734 aesni_gcm_dec_update_avx_gen4(ctx: aes_ctx, gdata: data,
735 out: walk.dst.virt.addr,
736 in: walk.src.virt.addr,
737 ciphertext_len: walk.nbytes);
738 } else if (static_branch_likely(&gcm_use_avx) && do_avx) {
739 if (enc)
740 aesni_gcm_enc_update_avx_gen2(ctx: aes_ctx, gdata: data,
741 out: walk.dst.virt.addr,
742 in: walk.src.virt.addr,
743 plaintext_len: walk.nbytes);
744 else
745 aesni_gcm_dec_update_avx_gen2(ctx: aes_ctx, gdata: data,
746 out: walk.dst.virt.addr,
747 in: walk.src.virt.addr,
748 ciphertext_len: walk.nbytes);
749 } else if (enc) {
750 aesni_gcm_enc_update(ctx: aes_ctx, gdata: data, out: walk.dst.virt.addr,
751 in: walk.src.virt.addr, plaintext_len: walk.nbytes);
752 } else {
753 aesni_gcm_dec_update(ctx: aes_ctx, gdata: data, out: walk.dst.virt.addr,
754 in: walk.src.virt.addr, ciphertext_len: walk.nbytes);
755 }
756 kernel_fpu_end();
757
758 err = skcipher_walk_done(walk: &walk, err: 0);
759 }
760
761 if (err)
762 return err;
763
764 kernel_fpu_begin();
765 if (static_branch_likely(&gcm_use_avx2) && do_avx2)
766 aesni_gcm_finalize_avx_gen4(ctx: aes_ctx, gdata: data, auth_tag,
767 auth_tag_len);
768 else if (static_branch_likely(&gcm_use_avx) && do_avx)
769 aesni_gcm_finalize_avx_gen2(ctx: aes_ctx, gdata: data, auth_tag,
770 auth_tag_len);
771 else
772 aesni_gcm_finalize(ctx: aes_ctx, gdata: data, auth_tag, auth_tag_len);
773 kernel_fpu_end();
774
775 return 0;
776}
777
778static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
779 u8 *hash_subkey, u8 *iv, void *aes_ctx)
780{
781 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
782 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
783 u8 auth_tag[16];
784 int err;
785
786 err = gcmaes_crypt_by_sg(enc: true, req, assoclen, hash_subkey, iv, aes_ctx,
787 auth_tag, auth_tag_len);
788 if (err)
789 return err;
790
791 scatterwalk_map_and_copy(buf: auth_tag, sg: req->dst,
792 start: req->assoclen + req->cryptlen,
793 nbytes: auth_tag_len, out: 1);
794 return 0;
795}
796
797static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
798 u8 *hash_subkey, u8 *iv, void *aes_ctx)
799{
800 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
801 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
802 u8 auth_tag_msg[16];
803 u8 auth_tag[16];
804 int err;
805
806 err = gcmaes_crypt_by_sg(enc: false, req, assoclen, hash_subkey, iv, aes_ctx,
807 auth_tag, auth_tag_len);
808 if (err)
809 return err;
810
811 /* Copy out original auth_tag */
812 scatterwalk_map_and_copy(buf: auth_tag_msg, sg: req->src,
813 start: req->assoclen + req->cryptlen - auth_tag_len,
814 nbytes: auth_tag_len, out: 0);
815
816 /* Compare generated tag with passed in tag. */
817 if (crypto_memneq(a: auth_tag_msg, b: auth_tag, size: auth_tag_len)) {
818 memzero_explicit(s: auth_tag, count: sizeof(auth_tag));
819 return -EBADMSG;
820 }
821 return 0;
822}
823
824static int helper_rfc4106_encrypt(struct aead_request *req)
825{
826 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
827 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
828 void *aes_ctx = &(ctx->aes_key_expanded);
829 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
830 u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
831 unsigned int i;
832 __be32 counter = cpu_to_be32(1);
833
834 /* Assuming we are supporting rfc4106 64-bit extended */
835 /* sequence numbers We need to have the AAD length equal */
836 /* to 16 or 20 bytes */
837 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
838 return -EINVAL;
839
840 /* IV below built */
841 for (i = 0; i < 4; i++)
842 *(iv+i) = ctx->nonce[i];
843 for (i = 0; i < 8; i++)
844 *(iv+4+i) = req->iv[i];
845 *((__be32 *)(iv+12)) = counter;
846
847 return gcmaes_encrypt(req, assoclen: req->assoclen - 8, hash_subkey: ctx->hash_subkey, iv,
848 aes_ctx);
849}
850
851static int helper_rfc4106_decrypt(struct aead_request *req)
852{
853 __be32 counter = cpu_to_be32(1);
854 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
855 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
856 void *aes_ctx = &(ctx->aes_key_expanded);
857 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
858 u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
859 unsigned int i;
860
861 if (unlikely(req->assoclen != 16 && req->assoclen != 20))
862 return -EINVAL;
863
864 /* Assuming we are supporting rfc4106 64-bit extended */
865 /* sequence numbers We need to have the AAD length */
866 /* equal to 16 or 20 bytes */
867
868 /* IV below built */
869 for (i = 0; i < 4; i++)
870 *(iv+i) = ctx->nonce[i];
871 for (i = 0; i < 8; i++)
872 *(iv+4+i) = req->iv[i];
873 *((__be32 *)(iv+12)) = counter;
874
875 return gcmaes_decrypt(req, assoclen: req->assoclen - 8, hash_subkey: ctx->hash_subkey, iv,
876 aes_ctx);
877}
878#endif
879
880static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
881 unsigned int keylen)
882{
883 struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
884 int err;
885
886 err = xts_verify_key(tfm, key, keylen);
887 if (err)
888 return err;
889
890 keylen /= 2;
891
892 /* first half of xts-key is for crypt */
893 err = aes_set_key_common(ctx: &ctx->crypt_ctx, in_key: key, key_len: keylen);
894 if (err)
895 return err;
896
897 /* second half of xts-key is for tweak */
898 return aes_set_key_common(ctx: &ctx->tweak_ctx, in_key: key + keylen, key_len: keylen);
899}
900
901static int xts_crypt(struct skcipher_request *req, bool encrypt)
902{
903 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
904 struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
905 int tail = req->cryptlen % AES_BLOCK_SIZE;
906 struct skcipher_request subreq;
907 struct skcipher_walk walk;
908 int err;
909
910 if (req->cryptlen < AES_BLOCK_SIZE)
911 return -EINVAL;
912
913 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
914 if (!walk.nbytes)
915 return err;
916
917 if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
918 int blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
919
920 skcipher_walk_abort(walk: &walk);
921
922 skcipher_request_set_tfm(req: &subreq, tfm);
923 skcipher_request_set_callback(req: &subreq,
924 flags: skcipher_request_flags(req),
925 NULL, NULL);
926 skcipher_request_set_crypt(req: &subreq, src: req->src, dst: req->dst,
927 cryptlen: blocks * AES_BLOCK_SIZE, iv: req->iv);
928 req = &subreq;
929
930 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
931 if (!walk.nbytes)
932 return err;
933 } else {
934 tail = 0;
935 }
936
937 kernel_fpu_begin();
938
939 /* calculate first value of T */
940 aesni_enc(ctx: &ctx->tweak_ctx, out: walk.iv, in: walk.iv);
941
942 while (walk.nbytes > 0) {
943 int nbytes = walk.nbytes;
944
945 if (nbytes < walk.total)
946 nbytes &= ~(AES_BLOCK_SIZE - 1);
947
948 if (encrypt)
949 aesni_xts_encrypt(ctx: &ctx->crypt_ctx,
950 out: walk.dst.virt.addr, in: walk.src.virt.addr,
951 len: nbytes, iv: walk.iv);
952 else
953 aesni_xts_decrypt(ctx: &ctx->crypt_ctx,
954 out: walk.dst.virt.addr, in: walk.src.virt.addr,
955 len: nbytes, iv: walk.iv);
956 kernel_fpu_end();
957
958 err = skcipher_walk_done(walk: &walk, err: walk.nbytes - nbytes);
959
960 if (walk.nbytes > 0)
961 kernel_fpu_begin();
962 }
963
964 if (unlikely(tail > 0 && !err)) {
965 struct scatterlist sg_src[2], sg_dst[2];
966 struct scatterlist *src, *dst;
967
968 dst = src = scatterwalk_ffwd(dst: sg_src, src: req->src, len: req->cryptlen);
969 if (req->dst != req->src)
970 dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst, len: req->cryptlen);
971
972 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
973 iv: req->iv);
974
975 err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false);
976 if (err)
977 return err;
978
979 kernel_fpu_begin();
980 if (encrypt)
981 aesni_xts_encrypt(ctx: &ctx->crypt_ctx,
982 out: walk.dst.virt.addr, in: walk.src.virt.addr,
983 len: walk.nbytes, iv: walk.iv);
984 else
985 aesni_xts_decrypt(ctx: &ctx->crypt_ctx,
986 out: walk.dst.virt.addr, in: walk.src.virt.addr,
987 len: walk.nbytes, iv: walk.iv);
988 kernel_fpu_end();
989
990 err = skcipher_walk_done(walk: &walk, err: 0);
991 }
992 return err;
993}
994
995static int xts_encrypt(struct skcipher_request *req)
996{
997 return xts_crypt(req, encrypt: true);
998}
999
1000static int xts_decrypt(struct skcipher_request *req)
1001{
1002 return xts_crypt(req, encrypt: false);
1003}
1004
1005static struct crypto_alg aesni_cipher_alg = {
1006 .cra_name = "aes",
1007 .cra_driver_name = "aes-aesni",
1008 .cra_priority = 300,
1009 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
1010 .cra_blocksize = AES_BLOCK_SIZE,
1011 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1012 .cra_module = THIS_MODULE,
1013 .cra_u = {
1014 .cipher = {
1015 .cia_min_keysize = AES_MIN_KEY_SIZE,
1016 .cia_max_keysize = AES_MAX_KEY_SIZE,
1017 .cia_setkey = aes_set_key,
1018 .cia_encrypt = aesni_encrypt,
1019 .cia_decrypt = aesni_decrypt
1020 }
1021 }
1022};
1023
1024static struct skcipher_alg aesni_skciphers[] = {
1025 {
1026 .base = {
1027 .cra_name = "__ecb(aes)",
1028 .cra_driver_name = "__ecb-aes-aesni",
1029 .cra_priority = 400,
1030 .cra_flags = CRYPTO_ALG_INTERNAL,
1031 .cra_blocksize = AES_BLOCK_SIZE,
1032 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1033 .cra_module = THIS_MODULE,
1034 },
1035 .min_keysize = AES_MIN_KEY_SIZE,
1036 .max_keysize = AES_MAX_KEY_SIZE,
1037 .setkey = aesni_skcipher_setkey,
1038 .encrypt = ecb_encrypt,
1039 .decrypt = ecb_decrypt,
1040 }, {
1041 .base = {
1042 .cra_name = "__cbc(aes)",
1043 .cra_driver_name = "__cbc-aes-aesni",
1044 .cra_priority = 400,
1045 .cra_flags = CRYPTO_ALG_INTERNAL,
1046 .cra_blocksize = AES_BLOCK_SIZE,
1047 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1048 .cra_module = THIS_MODULE,
1049 },
1050 .min_keysize = AES_MIN_KEY_SIZE,
1051 .max_keysize = AES_MAX_KEY_SIZE,
1052 .ivsize = AES_BLOCK_SIZE,
1053 .setkey = aesni_skcipher_setkey,
1054 .encrypt = cbc_encrypt,
1055 .decrypt = cbc_decrypt,
1056 }, {
1057 .base = {
1058 .cra_name = "__cts(cbc(aes))",
1059 .cra_driver_name = "__cts-cbc-aes-aesni",
1060 .cra_priority = 400,
1061 .cra_flags = CRYPTO_ALG_INTERNAL,
1062 .cra_blocksize = AES_BLOCK_SIZE,
1063 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1064 .cra_module = THIS_MODULE,
1065 },
1066 .min_keysize = AES_MIN_KEY_SIZE,
1067 .max_keysize = AES_MAX_KEY_SIZE,
1068 .ivsize = AES_BLOCK_SIZE,
1069 .walksize = 2 * AES_BLOCK_SIZE,
1070 .setkey = aesni_skcipher_setkey,
1071 .encrypt = cts_cbc_encrypt,
1072 .decrypt = cts_cbc_decrypt,
1073#ifdef CONFIG_X86_64
1074 }, {
1075 .base = {
1076 .cra_name = "__ctr(aes)",
1077 .cra_driver_name = "__ctr-aes-aesni",
1078 .cra_priority = 400,
1079 .cra_flags = CRYPTO_ALG_INTERNAL,
1080 .cra_blocksize = 1,
1081 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1082 .cra_module = THIS_MODULE,
1083 },
1084 .min_keysize = AES_MIN_KEY_SIZE,
1085 .max_keysize = AES_MAX_KEY_SIZE,
1086 .ivsize = AES_BLOCK_SIZE,
1087 .chunksize = AES_BLOCK_SIZE,
1088 .setkey = aesni_skcipher_setkey,
1089 .encrypt = ctr_crypt,
1090 .decrypt = ctr_crypt,
1091#endif
1092 }, {
1093 .base = {
1094 .cra_name = "__xts(aes)",
1095 .cra_driver_name = "__xts-aes-aesni",
1096 .cra_priority = 401,
1097 .cra_flags = CRYPTO_ALG_INTERNAL,
1098 .cra_blocksize = AES_BLOCK_SIZE,
1099 .cra_ctxsize = XTS_AES_CTX_SIZE,
1100 .cra_module = THIS_MODULE,
1101 },
1102 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1103 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1104 .ivsize = AES_BLOCK_SIZE,
1105 .walksize = 2 * AES_BLOCK_SIZE,
1106 .setkey = xts_aesni_setkey,
1107 .encrypt = xts_encrypt,
1108 .decrypt = xts_decrypt,
1109 }
1110};
1111
1112static
1113struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1114
1115#ifdef CONFIG_X86_64
1116/*
1117 * XCTR does not have a non-AVX implementation, so it must be enabled
1118 * conditionally.
1119 */
1120static struct skcipher_alg aesni_xctr = {
1121 .base = {
1122 .cra_name = "__xctr(aes)",
1123 .cra_driver_name = "__xctr-aes-aesni",
1124 .cra_priority = 400,
1125 .cra_flags = CRYPTO_ALG_INTERNAL,
1126 .cra_blocksize = 1,
1127 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
1128 .cra_module = THIS_MODULE,
1129 },
1130 .min_keysize = AES_MIN_KEY_SIZE,
1131 .max_keysize = AES_MAX_KEY_SIZE,
1132 .ivsize = AES_BLOCK_SIZE,
1133 .chunksize = AES_BLOCK_SIZE,
1134 .setkey = aesni_skcipher_setkey,
1135 .encrypt = xctr_crypt,
1136 .decrypt = xctr_crypt,
1137};
1138
1139static struct simd_skcipher_alg *aesni_simd_xctr;
1140#endif /* CONFIG_X86_64 */
1141
1142#ifdef CONFIG_X86_64
1143static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1144 unsigned int key_len)
1145{
1146 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm: aead);
1147
1148 return aes_set_key_common(ctx: &ctx->aes_key_expanded, in_key: key, key_len) ?:
1149 rfc4106_set_hash_subkey(hash_subkey: ctx->hash_subkey, key, key_len);
1150}
1151
1152static int generic_gcmaes_encrypt(struct aead_request *req)
1153{
1154 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1155 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1156 void *aes_ctx = &(ctx->aes_key_expanded);
1157 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
1158 u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
1159 __be32 counter = cpu_to_be32(1);
1160
1161 memcpy(iv, req->iv, 12);
1162 *((__be32 *)(iv+12)) = counter;
1163
1164 return gcmaes_encrypt(req, assoclen: req->assoclen, hash_subkey: ctx->hash_subkey, iv,
1165 aes_ctx);
1166}
1167
1168static int generic_gcmaes_decrypt(struct aead_request *req)
1169{
1170 __be32 counter = cpu_to_be32(1);
1171 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1172 struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1173 void *aes_ctx = &(ctx->aes_key_expanded);
1174 u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
1175 u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
1176
1177 memcpy(iv, req->iv, 12);
1178 *((__be32 *)(iv+12)) = counter;
1179
1180 return gcmaes_decrypt(req, assoclen: req->assoclen, hash_subkey: ctx->hash_subkey, iv,
1181 aes_ctx);
1182}
1183
1184static struct aead_alg aesni_aeads[] = { {
1185 .setkey = common_rfc4106_set_key,
1186 .setauthsize = common_rfc4106_set_authsize,
1187 .encrypt = helper_rfc4106_encrypt,
1188 .decrypt = helper_rfc4106_decrypt,
1189 .ivsize = GCM_RFC4106_IV_SIZE,
1190 .maxauthsize = 16,
1191 .base = {
1192 .cra_name = "__rfc4106(gcm(aes))",
1193 .cra_driver_name = "__rfc4106-gcm-aesni",
1194 .cra_priority = 400,
1195 .cra_flags = CRYPTO_ALG_INTERNAL,
1196 .cra_blocksize = 1,
1197 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
1198 .cra_alignmask = 0,
1199 .cra_module = THIS_MODULE,
1200 },
1201}, {
1202 .setkey = generic_gcmaes_set_key,
1203 .setauthsize = generic_gcmaes_set_authsize,
1204 .encrypt = generic_gcmaes_encrypt,
1205 .decrypt = generic_gcmaes_decrypt,
1206 .ivsize = GCM_AES_IV_SIZE,
1207 .maxauthsize = 16,
1208 .base = {
1209 .cra_name = "__gcm(aes)",
1210 .cra_driver_name = "__generic-gcm-aesni",
1211 .cra_priority = 400,
1212 .cra_flags = CRYPTO_ALG_INTERNAL,
1213 .cra_blocksize = 1,
1214 .cra_ctxsize = sizeof(struct generic_gcmaes_ctx),
1215 .cra_alignmask = 0,
1216 .cra_module = THIS_MODULE,
1217 },
1218} };
1219#else
1220static struct aead_alg aesni_aeads[0];
1221#endif
1222
1223static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)];
1224
1225static const struct x86_cpu_id aesni_cpu_id[] = {
1226 X86_MATCH_FEATURE(X86_FEATURE_AES, NULL),
1227 {}
1228};
1229MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1230
1231static int __init aesni_init(void)
1232{
1233 int err;
1234
1235 if (!x86_match_cpu(match: aesni_cpu_id))
1236 return -ENODEV;
1237#ifdef CONFIG_X86_64
1238 if (boot_cpu_has(X86_FEATURE_AVX2)) {
1239 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1240 static_branch_enable(&gcm_use_avx);
1241 static_branch_enable(&gcm_use_avx2);
1242 } else
1243 if (boot_cpu_has(X86_FEATURE_AVX)) {
1244 pr_info("AVX version of gcm_enc/dec engaged.\n");
1245 static_branch_enable(&gcm_use_avx);
1246 } else {
1247 pr_info("SSE version of gcm_enc/dec engaged.\n");
1248 }
1249 if (boot_cpu_has(X86_FEATURE_AVX)) {
1250 /* optimize performance of ctr mode encryption transform */
1251 static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm);
1252 pr_info("AES CTR mode by8 optimization enabled\n");
1253 }
1254#endif /* CONFIG_X86_64 */
1255
1256 err = crypto_register_alg(alg: &aesni_cipher_alg);
1257 if (err)
1258 return err;
1259
1260 err = simd_register_skciphers_compat(algs: aesni_skciphers,
1261 ARRAY_SIZE(aesni_skciphers),
1262 simd_algs: aesni_simd_skciphers);
1263 if (err)
1264 goto unregister_cipher;
1265
1266 err = simd_register_aeads_compat(algs: aesni_aeads, ARRAY_SIZE(aesni_aeads),
1267 simd_algs: aesni_simd_aeads);
1268 if (err)
1269 goto unregister_skciphers;
1270
1271#ifdef CONFIG_X86_64
1272 if (boot_cpu_has(X86_FEATURE_AVX))
1273 err = simd_register_skciphers_compat(algs: &aesni_xctr, count: 1,
1274 simd_algs: &aesni_simd_xctr);
1275 if (err)
1276 goto unregister_aeads;
1277#endif /* CONFIG_X86_64 */
1278
1279 return 0;
1280
1281#ifdef CONFIG_X86_64
1282unregister_aeads:
1283 simd_unregister_aeads(algs: aesni_aeads, ARRAY_SIZE(aesni_aeads),
1284 simd_algs: aesni_simd_aeads);
1285#endif /* CONFIG_X86_64 */
1286
1287unregister_skciphers:
1288 simd_unregister_skciphers(algs: aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1289 simd_algs: aesni_simd_skciphers);
1290unregister_cipher:
1291 crypto_unregister_alg(alg: &aesni_cipher_alg);
1292 return err;
1293}
1294
1295static void __exit aesni_exit(void)
1296{
1297 simd_unregister_aeads(algs: aesni_aeads, ARRAY_SIZE(aesni_aeads),
1298 simd_algs: aesni_simd_aeads);
1299 simd_unregister_skciphers(algs: aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1300 simd_algs: aesni_simd_skciphers);
1301 crypto_unregister_alg(alg: &aesni_cipher_alg);
1302#ifdef CONFIG_X86_64
1303 if (boot_cpu_has(X86_FEATURE_AVX))
1304 simd_unregister_skciphers(algs: &aesni_xctr, count: 1, simd_algs: &aesni_simd_xctr);
1305#endif /* CONFIG_X86_64 */
1306}
1307
1308late_initcall(aesni_init);
1309module_exit(aesni_exit);
1310
1311MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1312MODULE_LICENSE("GPL");
1313MODULE_ALIAS_CRYPTO("aes");
1314

source code of linux/arch/x86/crypto/aesni-intel_glue.c