1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * AES CTR routines supporting VMX instructions on the Power 8 |
4 | * |
5 | * Copyright (C) 2015 International Business Machines Inc. |
6 | * |
7 | * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com> |
8 | */ |
9 | |
10 | #include <asm/simd.h> |
11 | #include <asm/switch_to.h> |
12 | #include <crypto/aes.h> |
13 | #include <crypto/internal/simd.h> |
14 | #include <crypto/internal/skcipher.h> |
15 | |
16 | #include "aesp8-ppc.h" |
17 | |
18 | struct p8_aes_ctr_ctx { |
19 | struct crypto_skcipher *fallback; |
20 | struct aes_key enc_key; |
21 | }; |
22 | |
23 | static int p8_aes_ctr_init(struct crypto_skcipher *tfm) |
24 | { |
25 | struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); |
26 | struct crypto_skcipher *fallback; |
27 | |
28 | fallback = crypto_alloc_skcipher(alg_name: "ctr(aes)" , type: 0, |
29 | CRYPTO_ALG_NEED_FALLBACK | |
30 | CRYPTO_ALG_ASYNC); |
31 | if (IS_ERR(ptr: fallback)) { |
32 | pr_err("Failed to allocate ctr(aes) fallback: %ld\n" , |
33 | PTR_ERR(fallback)); |
34 | return PTR_ERR(ptr: fallback); |
35 | } |
36 | |
37 | crypto_skcipher_set_reqsize(skcipher: tfm, reqsize: sizeof(struct skcipher_request) + |
38 | crypto_skcipher_reqsize(tfm: fallback)); |
39 | ctx->fallback = fallback; |
40 | return 0; |
41 | } |
42 | |
43 | static void p8_aes_ctr_exit(struct crypto_skcipher *tfm) |
44 | { |
45 | struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); |
46 | |
47 | crypto_free_skcipher(tfm: ctx->fallback); |
48 | } |
49 | |
50 | static int p8_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key, |
51 | unsigned int keylen) |
52 | { |
53 | struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); |
54 | int ret; |
55 | |
56 | preempt_disable(); |
57 | pagefault_disable(); |
58 | enable_kernel_vsx(); |
59 | ret = aes_p8_set_encrypt_key(userKey: key, bits: keylen * 8, key: &ctx->enc_key); |
60 | disable_kernel_vsx(); |
61 | pagefault_enable(); |
62 | preempt_enable(); |
63 | |
64 | ret |= crypto_skcipher_setkey(tfm: ctx->fallback, key, keylen); |
65 | |
66 | return ret ? -EINVAL : 0; |
67 | } |
68 | |
69 | static void p8_aes_ctr_final(const struct p8_aes_ctr_ctx *ctx, |
70 | struct skcipher_walk *walk) |
71 | { |
72 | u8 *ctrblk = walk->iv; |
73 | u8 keystream[AES_BLOCK_SIZE]; |
74 | u8 *src = walk->src.virt.addr; |
75 | u8 *dst = walk->dst.virt.addr; |
76 | unsigned int nbytes = walk->nbytes; |
77 | |
78 | preempt_disable(); |
79 | pagefault_disable(); |
80 | enable_kernel_vsx(); |
81 | aes_p8_encrypt(in: ctrblk, out: keystream, key: &ctx->enc_key); |
82 | disable_kernel_vsx(); |
83 | pagefault_enable(); |
84 | preempt_enable(); |
85 | |
86 | crypto_xor_cpy(dst, src1: keystream, src2: src, size: nbytes); |
87 | crypto_inc(a: ctrblk, AES_BLOCK_SIZE); |
88 | } |
89 | |
90 | static int p8_aes_ctr_crypt(struct skcipher_request *req) |
91 | { |
92 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
93 | const struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); |
94 | struct skcipher_walk walk; |
95 | unsigned int nbytes; |
96 | int ret; |
97 | |
98 | if (!crypto_simd_usable()) { |
99 | struct skcipher_request *subreq = skcipher_request_ctx(req); |
100 | |
101 | *subreq = *req; |
102 | skcipher_request_set_tfm(req: subreq, tfm: ctx->fallback); |
103 | return crypto_skcipher_encrypt(req: subreq); |
104 | } |
105 | |
106 | ret = skcipher_walk_virt(walk: &walk, req, atomic: false); |
107 | while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { |
108 | preempt_disable(); |
109 | pagefault_disable(); |
110 | enable_kernel_vsx(); |
111 | aes_p8_ctr32_encrypt_blocks(in: walk.src.virt.addr, |
112 | out: walk.dst.virt.addr, |
113 | len: nbytes / AES_BLOCK_SIZE, |
114 | key: &ctx->enc_key, iv: walk.iv); |
115 | disable_kernel_vsx(); |
116 | pagefault_enable(); |
117 | preempt_enable(); |
118 | |
119 | do { |
120 | crypto_inc(a: walk.iv, AES_BLOCK_SIZE); |
121 | } while ((nbytes -= AES_BLOCK_SIZE) >= AES_BLOCK_SIZE); |
122 | |
123 | ret = skcipher_walk_done(walk: &walk, err: nbytes); |
124 | } |
125 | if (nbytes) { |
126 | p8_aes_ctr_final(ctx, walk: &walk); |
127 | ret = skcipher_walk_done(walk: &walk, err: 0); |
128 | } |
129 | return ret; |
130 | } |
131 | |
132 | struct skcipher_alg p8_aes_ctr_alg = { |
133 | .base.cra_name = "ctr(aes)" , |
134 | .base.cra_driver_name = "p8_aes_ctr" , |
135 | .base.cra_module = THIS_MODULE, |
136 | .base.cra_priority = 2000, |
137 | .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, |
138 | .base.cra_blocksize = 1, |
139 | .base.cra_ctxsize = sizeof(struct p8_aes_ctr_ctx), |
140 | .setkey = p8_aes_ctr_setkey, |
141 | .encrypt = p8_aes_ctr_crypt, |
142 | .decrypt = p8_aes_ctr_crypt, |
143 | .init = p8_aes_ctr_init, |
144 | .exit = p8_aes_ctr_exit, |
145 | .min_keysize = AES_MIN_KEY_SIZE, |
146 | .max_keysize = AES_MAX_KEY_SIZE, |
147 | .ivsize = AES_BLOCK_SIZE, |
148 | .chunksize = AES_BLOCK_SIZE, |
149 | }; |
150 | |