1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * GHASH routines supporting VMX instructions on the Power 8 |
4 | * |
5 | * Copyright (C) 2015, 2019 International Business Machines Inc. |
6 | * |
7 | * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com> |
8 | * |
9 | * Extended by Daniel Axtens <dja@axtens.net> to replace the fallback |
10 | * mechanism. The new approach is based on arm64 code, which is: |
11 | * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> |
12 | */ |
13 | |
14 | #include <linux/types.h> |
15 | #include <linux/err.h> |
16 | #include <linux/crypto.h> |
17 | #include <linux/delay.h> |
18 | #include <asm/simd.h> |
19 | #include <asm/switch_to.h> |
20 | #include <crypto/aes.h> |
21 | #include <crypto/ghash.h> |
22 | #include <crypto/scatterwalk.h> |
23 | #include <crypto/internal/hash.h> |
24 | #include <crypto/internal/simd.h> |
25 | #include <crypto/b128ops.h> |
26 | #include "aesp8-ppc.h" |
27 | |
28 | void gcm_init_p8(u128 htable[16], const u64 Xi[2]); |
29 | void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); |
30 | void gcm_ghash_p8(u64 Xi[2], const u128 htable[16], |
31 | const u8 *in, size_t len); |
32 | |
33 | struct p8_ghash_ctx { |
34 | /* key used by vector asm */ |
35 | u128 htable[16]; |
36 | /* key used by software fallback */ |
37 | be128 key; |
38 | }; |
39 | |
40 | struct p8_ghash_desc_ctx { |
41 | u64 shash[2]; |
42 | u8 buffer[GHASH_DIGEST_SIZE]; |
43 | int bytes; |
44 | }; |
45 | |
46 | static int p8_ghash_init(struct shash_desc *desc) |
47 | { |
48 | struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); |
49 | |
50 | dctx->bytes = 0; |
51 | memset(dctx->shash, 0, GHASH_DIGEST_SIZE); |
52 | return 0; |
53 | } |
54 | |
55 | static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, |
56 | unsigned int keylen) |
57 | { |
58 | struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm: crypto_shash_tfm(tfm)); |
59 | |
60 | if (keylen != GHASH_BLOCK_SIZE) |
61 | return -EINVAL; |
62 | |
63 | preempt_disable(); |
64 | pagefault_disable(); |
65 | enable_kernel_vsx(); |
66 | gcm_init_p8(htable: ctx->htable, Xi: (const u64 *) key); |
67 | disable_kernel_vsx(); |
68 | pagefault_enable(); |
69 | preempt_enable(); |
70 | |
71 | memcpy(&ctx->key, key, GHASH_BLOCK_SIZE); |
72 | |
73 | return 0; |
74 | } |
75 | |
76 | static inline void __ghash_block(struct p8_ghash_ctx *ctx, |
77 | struct p8_ghash_desc_ctx *dctx) |
78 | { |
79 | if (crypto_simd_usable()) { |
80 | preempt_disable(); |
81 | pagefault_disable(); |
82 | enable_kernel_vsx(); |
83 | gcm_ghash_p8(Xi: dctx->shash, htable: ctx->htable, |
84 | in: dctx->buffer, GHASH_DIGEST_SIZE); |
85 | disable_kernel_vsx(); |
86 | pagefault_enable(); |
87 | preempt_enable(); |
88 | } else { |
89 | crypto_xor(dst: (u8 *)dctx->shash, src: dctx->buffer, GHASH_BLOCK_SIZE); |
90 | gf128mul_lle(a: (be128 *)dctx->shash, b: &ctx->key); |
91 | } |
92 | } |
93 | |
94 | static inline void __ghash_blocks(struct p8_ghash_ctx *ctx, |
95 | struct p8_ghash_desc_ctx *dctx, |
96 | const u8 *src, unsigned int srclen) |
97 | { |
98 | if (crypto_simd_usable()) { |
99 | preempt_disable(); |
100 | pagefault_disable(); |
101 | enable_kernel_vsx(); |
102 | gcm_ghash_p8(Xi: dctx->shash, htable: ctx->htable, |
103 | in: src, len: srclen); |
104 | disable_kernel_vsx(); |
105 | pagefault_enable(); |
106 | preempt_enable(); |
107 | } else { |
108 | while (srclen >= GHASH_BLOCK_SIZE) { |
109 | crypto_xor(dst: (u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); |
110 | gf128mul_lle(a: (be128 *)dctx->shash, b: &ctx->key); |
111 | srclen -= GHASH_BLOCK_SIZE; |
112 | src += GHASH_BLOCK_SIZE; |
113 | } |
114 | } |
115 | } |
116 | |
117 | static int p8_ghash_update(struct shash_desc *desc, |
118 | const u8 *src, unsigned int srclen) |
119 | { |
120 | unsigned int len; |
121 | struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm: crypto_shash_tfm(tfm: desc->tfm)); |
122 | struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); |
123 | |
124 | if (dctx->bytes) { |
125 | if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) { |
126 | memcpy(dctx->buffer + dctx->bytes, src, |
127 | srclen); |
128 | dctx->bytes += srclen; |
129 | return 0; |
130 | } |
131 | memcpy(dctx->buffer + dctx->bytes, src, |
132 | GHASH_DIGEST_SIZE - dctx->bytes); |
133 | |
134 | __ghash_block(ctx, dctx); |
135 | |
136 | src += GHASH_DIGEST_SIZE - dctx->bytes; |
137 | srclen -= GHASH_DIGEST_SIZE - dctx->bytes; |
138 | dctx->bytes = 0; |
139 | } |
140 | len = srclen & ~(GHASH_DIGEST_SIZE - 1); |
141 | if (len) { |
142 | __ghash_blocks(ctx, dctx, src, srclen: len); |
143 | src += len; |
144 | srclen -= len; |
145 | } |
146 | if (srclen) { |
147 | memcpy(dctx->buffer, src, srclen); |
148 | dctx->bytes = srclen; |
149 | } |
150 | return 0; |
151 | } |
152 | |
153 | static int p8_ghash_final(struct shash_desc *desc, u8 *out) |
154 | { |
155 | int i; |
156 | struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm: crypto_shash_tfm(tfm: desc->tfm)); |
157 | struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); |
158 | |
159 | if (dctx->bytes) { |
160 | for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++) |
161 | dctx->buffer[i] = 0; |
162 | __ghash_block(ctx, dctx); |
163 | dctx->bytes = 0; |
164 | } |
165 | memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); |
166 | return 0; |
167 | } |
168 | |
169 | struct shash_alg p8_ghash_alg = { |
170 | .digestsize = GHASH_DIGEST_SIZE, |
171 | .init = p8_ghash_init, |
172 | .update = p8_ghash_update, |
173 | .final = p8_ghash_final, |
174 | .setkey = p8_ghash_setkey, |
175 | .descsize = sizeof(struct p8_ghash_desc_ctx) |
176 | + sizeof(struct ghash_desc_ctx), |
177 | .base = { |
178 | .cra_name = "ghash" , |
179 | .cra_driver_name = "p8_ghash" , |
180 | .cra_priority = 1000, |
181 | .cra_blocksize = GHASH_BLOCK_SIZE, |
182 | .cra_ctxsize = sizeof(struct p8_ghash_ctx), |
183 | .cra_module = THIS_MODULE, |
184 | }, |
185 | }; |
186 | |