1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * AES XTS routines supporting VMX In-core instructions on Power 8 |
4 | * |
5 | * Copyright (C) 2015 International Business Machines Inc. |
6 | * |
7 | * Author: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com> |
8 | */ |
9 | |
10 | #include <asm/simd.h> |
11 | #include <asm/switch_to.h> |
12 | #include <crypto/aes.h> |
13 | #include <crypto/internal/simd.h> |
14 | #include <crypto/internal/skcipher.h> |
15 | #include <crypto/xts.h> |
16 | |
17 | #include "aesp8-ppc.h" |
18 | |
19 | struct p8_aes_xts_ctx { |
20 | struct crypto_skcipher *fallback; |
21 | struct aes_key enc_key; |
22 | struct aes_key dec_key; |
23 | struct aes_key tweak_key; |
24 | }; |
25 | |
26 | static int p8_aes_xts_init(struct crypto_skcipher *tfm) |
27 | { |
28 | struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); |
29 | struct crypto_skcipher *fallback; |
30 | |
31 | fallback = crypto_alloc_skcipher(alg_name: "xts(aes)" , type: 0, |
32 | CRYPTO_ALG_NEED_FALLBACK | |
33 | CRYPTO_ALG_ASYNC); |
34 | if (IS_ERR(ptr: fallback)) { |
35 | pr_err("Failed to allocate xts(aes) fallback: %ld\n" , |
36 | PTR_ERR(fallback)); |
37 | return PTR_ERR(ptr: fallback); |
38 | } |
39 | |
40 | crypto_skcipher_set_reqsize(skcipher: tfm, reqsize: sizeof(struct skcipher_request) + |
41 | crypto_skcipher_reqsize(tfm: fallback)); |
42 | ctx->fallback = fallback; |
43 | return 0; |
44 | } |
45 | |
46 | static void p8_aes_xts_exit(struct crypto_skcipher *tfm) |
47 | { |
48 | struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); |
49 | |
50 | crypto_free_skcipher(tfm: ctx->fallback); |
51 | } |
52 | |
53 | static int p8_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, |
54 | unsigned int keylen) |
55 | { |
56 | struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); |
57 | int ret; |
58 | |
59 | ret = xts_verify_key(tfm, key, keylen); |
60 | if (ret) |
61 | return ret; |
62 | |
63 | preempt_disable(); |
64 | pagefault_disable(); |
65 | enable_kernel_vsx(); |
66 | ret = aes_p8_set_encrypt_key(userKey: key + keylen/2, bits: (keylen/2) * 8, key: &ctx->tweak_key); |
67 | ret |= aes_p8_set_encrypt_key(userKey: key, bits: (keylen/2) * 8, key: &ctx->enc_key); |
68 | ret |= aes_p8_set_decrypt_key(userKey: key, bits: (keylen/2) * 8, key: &ctx->dec_key); |
69 | disable_kernel_vsx(); |
70 | pagefault_enable(); |
71 | preempt_enable(); |
72 | |
73 | ret |= crypto_skcipher_setkey(tfm: ctx->fallback, key, keylen); |
74 | |
75 | return ret ? -EINVAL : 0; |
76 | } |
77 | |
78 | static int p8_aes_xts_crypt(struct skcipher_request *req, int enc) |
79 | { |
80 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
81 | const struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); |
82 | struct skcipher_walk walk; |
83 | unsigned int nbytes; |
84 | u8 tweak[AES_BLOCK_SIZE]; |
85 | int ret; |
86 | |
87 | if (req->cryptlen < AES_BLOCK_SIZE) |
88 | return -EINVAL; |
89 | |
90 | if (!crypto_simd_usable() || (req->cryptlen % XTS_BLOCK_SIZE) != 0) { |
91 | struct skcipher_request *subreq = skcipher_request_ctx(req); |
92 | |
93 | *subreq = *req; |
94 | skcipher_request_set_tfm(req: subreq, tfm: ctx->fallback); |
95 | return enc ? crypto_skcipher_encrypt(req: subreq) : |
96 | crypto_skcipher_decrypt(req: subreq); |
97 | } |
98 | |
99 | ret = skcipher_walk_virt(walk: &walk, req, atomic: false); |
100 | if (ret) |
101 | return ret; |
102 | |
103 | preempt_disable(); |
104 | pagefault_disable(); |
105 | enable_kernel_vsx(); |
106 | |
107 | aes_p8_encrypt(in: walk.iv, out: tweak, key: &ctx->tweak_key); |
108 | |
109 | disable_kernel_vsx(); |
110 | pagefault_enable(); |
111 | preempt_enable(); |
112 | |
113 | while ((nbytes = walk.nbytes) != 0) { |
114 | preempt_disable(); |
115 | pagefault_disable(); |
116 | enable_kernel_vsx(); |
117 | if (enc) |
118 | aes_p8_xts_encrypt(in: walk.src.virt.addr, |
119 | out: walk.dst.virt.addr, |
120 | round_down(nbytes, AES_BLOCK_SIZE), |
121 | key1: &ctx->enc_key, NULL, iv: tweak); |
122 | else |
123 | aes_p8_xts_decrypt(in: walk.src.virt.addr, |
124 | out: walk.dst.virt.addr, |
125 | round_down(nbytes, AES_BLOCK_SIZE), |
126 | key1: &ctx->dec_key, NULL, iv: tweak); |
127 | disable_kernel_vsx(); |
128 | pagefault_enable(); |
129 | preempt_enable(); |
130 | |
131 | ret = skcipher_walk_done(walk: &walk, err: nbytes % AES_BLOCK_SIZE); |
132 | } |
133 | return ret; |
134 | } |
135 | |
136 | static int p8_aes_xts_encrypt(struct skcipher_request *req) |
137 | { |
138 | return p8_aes_xts_crypt(req, enc: 1); |
139 | } |
140 | |
141 | static int p8_aes_xts_decrypt(struct skcipher_request *req) |
142 | { |
143 | return p8_aes_xts_crypt(req, enc: 0); |
144 | } |
145 | |
146 | struct skcipher_alg p8_aes_xts_alg = { |
147 | .base.cra_name = "xts(aes)" , |
148 | .base.cra_driver_name = "p8_aes_xts" , |
149 | .base.cra_module = THIS_MODULE, |
150 | .base.cra_priority = 2000, |
151 | .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, |
152 | .base.cra_blocksize = AES_BLOCK_SIZE, |
153 | .base.cra_ctxsize = sizeof(struct p8_aes_xts_ctx), |
154 | .setkey = p8_aes_xts_setkey, |
155 | .encrypt = p8_aes_xts_encrypt, |
156 | .decrypt = p8_aes_xts_decrypt, |
157 | .init = p8_aes_xts_init, |
158 | .exit = p8_aes_xts_exit, |
159 | .min_keysize = 2 * AES_MIN_KEY_SIZE, |
160 | .max_keysize = 2 * AES_MAX_KEY_SIZE, |
161 | .ivsize = AES_BLOCK_SIZE, |
162 | }; |
163 | |