1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Poly1305 authenticator algorithm, RFC7539. |
4 | * |
5 | * Copyright 2023- IBM Corp. All rights reserved. |
6 | */ |
7 | |
8 | #include <crypto/algapi.h> |
9 | #include <linux/crypto.h> |
10 | #include <linux/kernel.h> |
11 | #include <linux/module.h> |
12 | #include <linux/jump_label.h> |
13 | #include <crypto/internal/hash.h> |
14 | #include <crypto/internal/poly1305.h> |
15 | #include <crypto/internal/simd.h> |
16 | #include <linux/cpufeature.h> |
17 | #include <asm/unaligned.h> |
18 | #include <asm/simd.h> |
19 | #include <asm/switch_to.h> |
20 | |
21 | asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen); |
22 | asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit); |
23 | asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst); |
24 | |
25 | static void vsx_begin(void) |
26 | { |
27 | preempt_disable(); |
28 | enable_kernel_vsx(); |
29 | } |
30 | |
31 | static void vsx_end(void) |
32 | { |
33 | disable_kernel_vsx(); |
34 | preempt_enable(); |
35 | } |
36 | |
37 | static int crypto_poly1305_p10_init(struct shash_desc *desc) |
38 | { |
39 | struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); |
40 | |
41 | poly1305_core_init(state: &dctx->h); |
42 | dctx->buflen = 0; |
43 | dctx->rset = 0; |
44 | dctx->sset = false; |
45 | |
46 | return 0; |
47 | } |
48 | |
49 | static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx, |
50 | const u8 *inp, unsigned int len) |
51 | { |
52 | unsigned int acc = 0; |
53 | |
54 | if (unlikely(!dctx->sset)) { |
55 | if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) { |
56 | struct poly1305_core_key *key = &dctx->core_r; |
57 | |
58 | key->key.r64[0] = get_unaligned_le64(p: &inp[0]); |
59 | key->key.r64[1] = get_unaligned_le64(p: &inp[8]); |
60 | inp += POLY1305_BLOCK_SIZE; |
61 | len -= POLY1305_BLOCK_SIZE; |
62 | acc += POLY1305_BLOCK_SIZE; |
63 | dctx->rset = 1; |
64 | } |
65 | if (len >= POLY1305_BLOCK_SIZE) { |
66 | dctx->s[0] = get_unaligned_le32(p: &inp[0]); |
67 | dctx->s[1] = get_unaligned_le32(p: &inp[4]); |
68 | dctx->s[2] = get_unaligned_le32(p: &inp[8]); |
69 | dctx->s[3] = get_unaligned_le32(p: &inp[12]); |
70 | acc += POLY1305_BLOCK_SIZE; |
71 | dctx->sset = true; |
72 | } |
73 | } |
74 | return acc; |
75 | } |
76 | |
77 | static int crypto_poly1305_p10_update(struct shash_desc *desc, |
78 | const u8 *src, unsigned int srclen) |
79 | { |
80 | struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); |
81 | unsigned int bytes, used; |
82 | |
83 | if (unlikely(dctx->buflen)) { |
84 | bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); |
85 | memcpy(dctx->buf + dctx->buflen, src, bytes); |
86 | src += bytes; |
87 | srclen -= bytes; |
88 | dctx->buflen += bytes; |
89 | |
90 | if (dctx->buflen == POLY1305_BLOCK_SIZE) { |
91 | if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, |
92 | POLY1305_BLOCK_SIZE))) { |
93 | vsx_begin(); |
94 | poly1305_64s(h: &dctx->h, m: dctx->buf, |
95 | POLY1305_BLOCK_SIZE, highbit: 1); |
96 | vsx_end(); |
97 | } |
98 | dctx->buflen = 0; |
99 | } |
100 | } |
101 | |
102 | if (likely(srclen >= POLY1305_BLOCK_SIZE)) { |
103 | bytes = round_down(srclen, POLY1305_BLOCK_SIZE); |
104 | used = crypto_poly1305_setdctxkey(dctx, inp: src, len: bytes); |
105 | if (likely(used)) { |
106 | srclen -= used; |
107 | src += used; |
108 | } |
109 | if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) { |
110 | vsx_begin(); |
111 | poly1305_p10le_4blocks(h: &dctx->h, m: src, mlen: srclen); |
112 | vsx_end(); |
113 | src += srclen - (srclen % (POLY1305_BLOCK_SIZE * 4)); |
114 | srclen %= POLY1305_BLOCK_SIZE * 4; |
115 | } |
116 | while (srclen >= POLY1305_BLOCK_SIZE) { |
117 | vsx_begin(); |
118 | poly1305_64s(h: &dctx->h, m: src, POLY1305_BLOCK_SIZE, highbit: 1); |
119 | vsx_end(); |
120 | srclen -= POLY1305_BLOCK_SIZE; |
121 | src += POLY1305_BLOCK_SIZE; |
122 | } |
123 | } |
124 | |
125 | if (unlikely(srclen)) { |
126 | dctx->buflen = srclen; |
127 | memcpy(dctx->buf, src, srclen); |
128 | } |
129 | |
130 | return 0; |
131 | } |
132 | |
133 | static int crypto_poly1305_p10_final(struct shash_desc *desc, u8 *dst) |
134 | { |
135 | struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); |
136 | |
137 | if (unlikely(!dctx->sset)) |
138 | return -ENOKEY; |
139 | |
140 | if ((dctx->buflen)) { |
141 | dctx->buf[dctx->buflen++] = 1; |
142 | memset(dctx->buf + dctx->buflen, 0, |
143 | POLY1305_BLOCK_SIZE - dctx->buflen); |
144 | vsx_begin(); |
145 | poly1305_64s(h: &dctx->h, m: dctx->buf, POLY1305_BLOCK_SIZE, highbit: 0); |
146 | vsx_end(); |
147 | dctx->buflen = 0; |
148 | } |
149 | |
150 | poly1305_emit_64(h: &dctx->h, s: &dctx->s, dst); |
151 | return 0; |
152 | } |
153 | |
154 | static struct shash_alg poly1305_alg = { |
155 | .digestsize = POLY1305_DIGEST_SIZE, |
156 | .init = crypto_poly1305_p10_init, |
157 | .update = crypto_poly1305_p10_update, |
158 | .final = crypto_poly1305_p10_final, |
159 | .descsize = sizeof(struct poly1305_desc_ctx), |
160 | .base = { |
161 | .cra_name = "poly1305" , |
162 | .cra_driver_name = "poly1305-p10" , |
163 | .cra_priority = 300, |
164 | .cra_blocksize = POLY1305_BLOCK_SIZE, |
165 | .cra_module = THIS_MODULE, |
166 | }, |
167 | }; |
168 | |
169 | static int __init poly1305_p10_init(void) |
170 | { |
171 | return crypto_register_shash(alg: &poly1305_alg); |
172 | } |
173 | |
174 | static void __exit poly1305_p10_exit(void) |
175 | { |
176 | crypto_unregister_shash(alg: &poly1305_alg); |
177 | } |
178 | |
179 | module_cpu_feature_match(PPC_MODULE_FEATURE_P10, poly1305_p10_init); |
180 | module_exit(poly1305_p10_exit); |
181 | |
182 | MODULE_LICENSE("GPL" ); |
183 | MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>" ); |
184 | MODULE_DESCRIPTION("Optimized Poly1305 for P10" ); |
185 | MODULE_ALIAS_CRYPTO("poly1305" ); |
186 | MODULE_ALIAS_CRYPTO("poly1305-p10" ); |
187 | |