1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum |
4 | * (AVX2 accelerated version) |
5 | * |
6 | * Copyright 2018 Google LLC |
7 | */ |
8 | |
9 | #include <crypto/internal/hash.h> |
10 | #include <crypto/internal/simd.h> |
11 | #include <crypto/nhpoly1305.h> |
12 | #include <linux/module.h> |
13 | #include <linux/sizes.h> |
14 | #include <asm/simd.h> |
15 | |
16 | asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len, |
17 | __le64 hash[NH_NUM_PASSES]); |
18 | |
19 | static int nhpoly1305_avx2_update(struct shash_desc *desc, |
20 | const u8 *src, unsigned int srclen) |
21 | { |
22 | if (srclen < 64 || !crypto_simd_usable()) |
23 | return crypto_nhpoly1305_update(desc, src, srclen); |
24 | |
25 | do { |
26 | unsigned int n = min_t(unsigned int, srclen, SZ_4K); |
27 | |
28 | kernel_fpu_begin(); |
29 | crypto_nhpoly1305_update_helper(desc, src, srclen: n, nh_fn: nh_avx2); |
30 | kernel_fpu_end(); |
31 | src += n; |
32 | srclen -= n; |
33 | } while (srclen); |
34 | return 0; |
35 | } |
36 | |
37 | static int nhpoly1305_avx2_digest(struct shash_desc *desc, |
38 | const u8 *src, unsigned int srclen, u8 *out) |
39 | { |
40 | return crypto_nhpoly1305_init(desc) ?: |
41 | nhpoly1305_avx2_update(desc, src, srclen) ?: |
42 | crypto_nhpoly1305_final(desc, dst: out); |
43 | } |
44 | |
45 | static struct shash_alg nhpoly1305_alg = { |
46 | .base.cra_name = "nhpoly1305" , |
47 | .base.cra_driver_name = "nhpoly1305-avx2" , |
48 | .base.cra_priority = 300, |
49 | .base.cra_ctxsize = sizeof(struct nhpoly1305_key), |
50 | .base.cra_module = THIS_MODULE, |
51 | .digestsize = POLY1305_DIGEST_SIZE, |
52 | .init = crypto_nhpoly1305_init, |
53 | .update = nhpoly1305_avx2_update, |
54 | .final = crypto_nhpoly1305_final, |
55 | .digest = nhpoly1305_avx2_digest, |
56 | .setkey = crypto_nhpoly1305_setkey, |
57 | .descsize = sizeof(struct nhpoly1305_state), |
58 | }; |
59 | |
60 | static int __init nhpoly1305_mod_init(void) |
61 | { |
62 | if (!boot_cpu_has(X86_FEATURE_AVX2) || |
63 | !boot_cpu_has(X86_FEATURE_OSXSAVE)) |
64 | return -ENODEV; |
65 | |
66 | return crypto_register_shash(alg: &nhpoly1305_alg); |
67 | } |
68 | |
69 | static void __exit nhpoly1305_mod_exit(void) |
70 | { |
71 | crypto_unregister_shash(alg: &nhpoly1305_alg); |
72 | } |
73 | |
74 | module_init(nhpoly1305_mod_init); |
75 | module_exit(nhpoly1305_mod_exit); |
76 | |
77 | MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)" ); |
78 | MODULE_LICENSE("GPL v2" ); |
79 | MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>" ); |
80 | MODULE_ALIAS_CRYPTO("nhpoly1305" ); |
81 | MODULE_ALIAS_CRYPTO("nhpoly1305-avx2" ); |
82 | |