1/*
2 * AMD Cryptographic Coprocessor (CCP) crypto API support
3 *
4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/module.h>
14#include <linux/moduleparam.h>
15#include <linux/kernel.h>
16#include <linux/list.h>
17#include <linux/ccp.h>
18#include <linux/scatterlist.h>
19#include <crypto/internal/hash.h>
20#include <crypto/internal/akcipher.h>
21
22#include "ccp-crypto.h"
23
24MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>");
25MODULE_LICENSE("GPL");
26MODULE_VERSION("1.0.0");
27MODULE_DESCRIPTION("AMD Cryptographic Coprocessor crypto API support");
28
29static unsigned int aes_disable;
30module_param(aes_disable, uint, 0444);
31MODULE_PARM_DESC(aes_disable, "Disable use of AES - any non-zero value");
32
33static unsigned int sha_disable;
34module_param(sha_disable, uint, 0444);
35MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value");
36
37static unsigned int des3_disable;
38module_param(des3_disable, uint, 0444);
39MODULE_PARM_DESC(des3_disable, "Disable use of 3DES - any non-zero value");
40
41static unsigned int rsa_disable;
42module_param(rsa_disable, uint, 0444);
43MODULE_PARM_DESC(rsa_disable, "Disable use of RSA - any non-zero value");
44
45/* List heads for the supported algorithms */
46static LIST_HEAD(hash_algs);
47static LIST_HEAD(cipher_algs);
48static LIST_HEAD(aead_algs);
49static LIST_HEAD(akcipher_algs);
50
51/* For any tfm, requests for that tfm must be returned on the order
52 * received. With multiple queues available, the CCP can process more
53 * than one cmd at a time. Therefore we must maintain a cmd list to insure
54 * the proper ordering of requests on a given tfm.
55 */
56struct ccp_crypto_queue {
57 struct list_head cmds;
58 struct list_head *backlog;
59 unsigned int cmd_count;
60};
61
62#define CCP_CRYPTO_MAX_QLEN 100
63
64static struct ccp_crypto_queue req_queue;
65static spinlock_t req_queue_lock;
66
67struct ccp_crypto_cmd {
68 struct list_head entry;
69
70 struct ccp_cmd *cmd;
71
72 /* Save the crypto_tfm and crypto_async_request addresses
73 * separately to avoid any reference to a possibly invalid
74 * crypto_async_request structure after invoking the request
75 * callback
76 */
77 struct crypto_async_request *req;
78 struct crypto_tfm *tfm;
79
80 /* Used for held command processing to determine state */
81 int ret;
82};
83
84struct ccp_crypto_cpu {
85 struct work_struct work;
86 struct completion completion;
87 struct ccp_crypto_cmd *crypto_cmd;
88 int err;
89};
90
91static inline bool ccp_crypto_success(int err)
92{
93 if (err && (err != -EINPROGRESS) && (err != -EBUSY))
94 return false;
95
96 return true;
97}
98
99static struct ccp_crypto_cmd *ccp_crypto_cmd_complete(
100 struct ccp_crypto_cmd *crypto_cmd, struct ccp_crypto_cmd **backlog)
101{
102 struct ccp_crypto_cmd *held = NULL, *tmp;
103 unsigned long flags;
104
105 *backlog = NULL;
106
107 spin_lock_irqsave(&req_queue_lock, flags);
108
109 /* Held cmds will be after the current cmd in the queue so start
110 * searching for a cmd with a matching tfm for submission.
111 */
112 tmp = crypto_cmd;
113 list_for_each_entry_continue(tmp, &req_queue.cmds, entry) {
114 if (crypto_cmd->tfm != tmp->tfm)
115 continue;
116 held = tmp;
117 break;
118 }
119
120 /* Process the backlog:
121 * Because cmds can be executed from any point in the cmd list
122 * special precautions have to be taken when handling the backlog.
123 */
124 if (req_queue.backlog != &req_queue.cmds) {
125 /* Skip over this cmd if it is the next backlog cmd */
126 if (req_queue.backlog == &crypto_cmd->entry)
127 req_queue.backlog = crypto_cmd->entry.next;
128
129 *backlog = container_of(req_queue.backlog,
130 struct ccp_crypto_cmd, entry);
131 req_queue.backlog = req_queue.backlog->next;
132
133 /* Skip over this cmd if it is now the next backlog cmd */
134 if (req_queue.backlog == &crypto_cmd->entry)
135 req_queue.backlog = crypto_cmd->entry.next;
136 }
137
138 /* Remove the cmd entry from the list of cmds */
139 req_queue.cmd_count--;
140 list_del(&crypto_cmd->entry);
141
142 spin_unlock_irqrestore(&req_queue_lock, flags);
143
144 return held;
145}
146
147static void ccp_crypto_complete(void *data, int err)
148{
149 struct ccp_crypto_cmd *crypto_cmd = data;
150 struct ccp_crypto_cmd *held, *next, *backlog;
151 struct crypto_async_request *req = crypto_cmd->req;
152 struct ccp_ctx *ctx = crypto_tfm_ctx(req->tfm);
153 int ret;
154
155 if (err == -EINPROGRESS) {
156 /* Only propagate the -EINPROGRESS if necessary */
157 if (crypto_cmd->ret == -EBUSY) {
158 crypto_cmd->ret = -EINPROGRESS;
159 req->complete(req, -EINPROGRESS);
160 }
161
162 return;
163 }
164
165 /* Operation has completed - update the queue before invoking
166 * the completion callbacks and retrieve the next cmd (cmd with
167 * a matching tfm) that can be submitted to the CCP.
168 */
169 held = ccp_crypto_cmd_complete(crypto_cmd, &backlog);
170 if (backlog) {
171 backlog->ret = -EINPROGRESS;
172 backlog->req->complete(backlog->req, -EINPROGRESS);
173 }
174
175 /* Transition the state from -EBUSY to -EINPROGRESS first */
176 if (crypto_cmd->ret == -EBUSY)
177 req->complete(req, -EINPROGRESS);
178
179 /* Completion callbacks */
180 ret = err;
181 if (ctx->complete)
182 ret = ctx->complete(req, ret);
183 req->complete(req, ret);
184
185 /* Submit the next cmd */
186 while (held) {
187 /* Since we have already queued the cmd, we must indicate that
188 * we can backlog so as not to "lose" this request.
189 */
190 held->cmd->flags |= CCP_CMD_MAY_BACKLOG;
191 ret = ccp_enqueue_cmd(held->cmd);
192 if (ccp_crypto_success(ret))
193 break;
194
195 /* Error occurred, report it and get the next entry */
196 ctx = crypto_tfm_ctx(held->req->tfm);
197 if (ctx->complete)
198 ret = ctx->complete(held->req, ret);
199 held->req->complete(held->req, ret);
200
201 next = ccp_crypto_cmd_complete(held, &backlog);
202 if (backlog) {
203 backlog->ret = -EINPROGRESS;
204 backlog->req->complete(backlog->req, -EINPROGRESS);
205 }
206
207 kfree(held);
208 held = next;
209 }
210
211 kfree(crypto_cmd);
212}
213
214static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd)
215{
216 struct ccp_crypto_cmd *active = NULL, *tmp;
217 unsigned long flags;
218 bool free_cmd = true;
219 int ret;
220
221 spin_lock_irqsave(&req_queue_lock, flags);
222
223 /* Check if the cmd can/should be queued */
224 if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {
225 if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) {
226 ret = -ENOSPC;
227 goto e_lock;
228 }
229 }
230
231 /* Look for an entry with the same tfm. If there is a cmd
232 * with the same tfm in the list then the current cmd cannot
233 * be submitted to the CCP yet.
234 */
235 list_for_each_entry(tmp, &req_queue.cmds, entry) {
236 if (crypto_cmd->tfm != tmp->tfm)
237 continue;
238 active = tmp;
239 break;
240 }
241
242 ret = -EINPROGRESS;
243 if (!active) {
244 ret = ccp_enqueue_cmd(crypto_cmd->cmd);
245 if (!ccp_crypto_success(ret))
246 goto e_lock; /* Error, don't queue it */
247 }
248
249 if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {
250 ret = -EBUSY;
251 if (req_queue.backlog == &req_queue.cmds)
252 req_queue.backlog = &crypto_cmd->entry;
253 }
254 crypto_cmd->ret = ret;
255
256 req_queue.cmd_count++;
257 list_add_tail(&crypto_cmd->entry, &req_queue.cmds);
258
259 free_cmd = false;
260
261e_lock:
262 spin_unlock_irqrestore(&req_queue_lock, flags);
263
264 if (free_cmd)
265 kfree(crypto_cmd);
266
267 return ret;
268}
269
270/**
271 * ccp_crypto_enqueue_request - queue an crypto async request for processing
272 * by the CCP
273 *
274 * @req: crypto_async_request struct to be processed
275 * @cmd: ccp_cmd struct to be sent to the CCP
276 */
277int ccp_crypto_enqueue_request(struct crypto_async_request *req,
278 struct ccp_cmd *cmd)
279{
280 struct ccp_crypto_cmd *crypto_cmd;
281 gfp_t gfp;
282
283 gfp = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC;
284
285 crypto_cmd = kzalloc(sizeof(*crypto_cmd), gfp);
286 if (!crypto_cmd)
287 return -ENOMEM;
288
289 /* The tfm pointer must be saved and not referenced from the
290 * crypto_async_request (req) pointer because it is used after
291 * completion callback for the request and the req pointer
292 * might not be valid anymore.
293 */
294 crypto_cmd->cmd = cmd;
295 crypto_cmd->req = req;
296 crypto_cmd->tfm = req->tfm;
297
298 cmd->callback = ccp_crypto_complete;
299 cmd->data = crypto_cmd;
300
301 if (req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
302 cmd->flags |= CCP_CMD_MAY_BACKLOG;
303 else
304 cmd->flags &= ~CCP_CMD_MAY_BACKLOG;
305
306 return ccp_crypto_enqueue_cmd(crypto_cmd);
307}
308
309struct scatterlist *ccp_crypto_sg_table_add(struct sg_table *table,
310 struct scatterlist *sg_add)
311{
312 struct scatterlist *sg, *sg_last = NULL;
313
314 for (sg = table->sgl; sg; sg = sg_next(sg))
315 if (!sg_page(sg))
316 break;
317 if (WARN_ON(!sg))
318 return NULL;
319
320 for (; sg && sg_add; sg = sg_next(sg), sg_add = sg_next(sg_add)) {
321 sg_set_page(sg, sg_page(sg_add), sg_add->length,
322 sg_add->offset);
323 sg_last = sg;
324 }
325 if (WARN_ON(sg_add))
326 return NULL;
327
328 return sg_last;
329}
330
331static int ccp_register_algs(void)
332{
333 int ret;
334
335 if (!aes_disable) {
336 ret = ccp_register_aes_algs(&cipher_algs);
337 if (ret)
338 return ret;
339
340 ret = ccp_register_aes_cmac_algs(&hash_algs);
341 if (ret)
342 return ret;
343
344 ret = ccp_register_aes_xts_algs(&cipher_algs);
345 if (ret)
346 return ret;
347
348 ret = ccp_register_aes_aeads(&aead_algs);
349 if (ret)
350 return ret;
351 }
352
353 if (!des3_disable) {
354 ret = ccp_register_des3_algs(&cipher_algs);
355 if (ret)
356 return ret;
357 }
358
359 if (!sha_disable) {
360 ret = ccp_register_sha_algs(&hash_algs);
361 if (ret)
362 return ret;
363 }
364
365 if (!rsa_disable) {
366 ret = ccp_register_rsa_algs(&akcipher_algs);
367 if (ret)
368 return ret;
369 }
370
371 return 0;
372}
373
374static void ccp_unregister_algs(void)
375{
376 struct ccp_crypto_ahash_alg *ahash_alg, *ahash_tmp;
377 struct ccp_crypto_ablkcipher_alg *ablk_alg, *ablk_tmp;
378 struct ccp_crypto_aead *aead_alg, *aead_tmp;
379 struct ccp_crypto_akcipher_alg *akc_alg, *akc_tmp;
380
381 list_for_each_entry_safe(ahash_alg, ahash_tmp, &hash_algs, entry) {
382 crypto_unregister_ahash(&ahash_alg->alg);
383 list_del(&ahash_alg->entry);
384 kfree(ahash_alg);
385 }
386
387 list_for_each_entry_safe(ablk_alg, ablk_tmp, &cipher_algs, entry) {
388 crypto_unregister_alg(&ablk_alg->alg);
389 list_del(&ablk_alg->entry);
390 kfree(ablk_alg);
391 }
392
393 list_for_each_entry_safe(aead_alg, aead_tmp, &aead_algs, entry) {
394 crypto_unregister_aead(&aead_alg->alg);
395 list_del(&aead_alg->entry);
396 kfree(aead_alg);
397 }
398
399 list_for_each_entry_safe(akc_alg, akc_tmp, &akcipher_algs, entry) {
400 crypto_unregister_akcipher(&akc_alg->alg);
401 list_del(&akc_alg->entry);
402 kfree(akc_alg);
403 }
404}
405
406static int ccp_crypto_init(void)
407{
408 int ret;
409
410 ret = ccp_present();
411 if (ret)
412 return ret;
413
414 spin_lock_init(&req_queue_lock);
415 INIT_LIST_HEAD(&req_queue.cmds);
416 req_queue.backlog = &req_queue.cmds;
417 req_queue.cmd_count = 0;
418
419 ret = ccp_register_algs();
420 if (ret)
421 ccp_unregister_algs();
422
423 return ret;
424}
425
426static void ccp_crypto_exit(void)
427{
428 ccp_unregister_algs();
429}
430
431module_init(ccp_crypto_init);
432module_exit(ccp_crypto_exit);
433