1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support. |
3 | * |
4 | * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net> |
5 | */ |
6 | |
7 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
8 | |
9 | #include <linux/kernel.h> |
10 | #include <linux/module.h> |
11 | #include <linux/of.h> |
12 | #include <linux/of_address.h> |
13 | #include <linux/platform_device.h> |
14 | #include <linux/cpumask.h> |
15 | #include <linux/slab.h> |
16 | #include <linux/interrupt.h> |
17 | #include <linux/crypto.h> |
18 | #include <crypto/md5.h> |
19 | #include <crypto/sha1.h> |
20 | #include <crypto/sha2.h> |
21 | #include <crypto/aes.h> |
22 | #include <crypto/internal/des.h> |
23 | #include <linux/mutex.h> |
24 | #include <linux/delay.h> |
25 | #include <linux/sched.h> |
26 | |
27 | #include <crypto/internal/hash.h> |
28 | #include <crypto/internal/skcipher.h> |
29 | #include <crypto/scatterwalk.h> |
30 | #include <crypto/algapi.h> |
31 | |
32 | #include <asm/hypervisor.h> |
33 | #include <asm/mdesc.h> |
34 | |
35 | #include "n2_core.h" |
36 | |
37 | #define DRV_MODULE_NAME "n2_crypto" |
38 | #define DRV_MODULE_VERSION "0.2" |
39 | #define DRV_MODULE_RELDATE "July 28, 2011" |
40 | |
41 | static const char version[] = |
42 | DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n" ; |
43 | |
44 | MODULE_AUTHOR("David S. Miller (davem@davemloft.net)" ); |
45 | MODULE_DESCRIPTION("Niagara2 Crypto driver" ); |
46 | MODULE_LICENSE("GPL" ); |
47 | MODULE_VERSION(DRV_MODULE_VERSION); |
48 | |
49 | #define N2_CRA_PRIORITY 200 |
50 | |
51 | static DEFINE_MUTEX(spu_lock); |
52 | |
53 | struct spu_queue { |
54 | cpumask_t sharing; |
55 | unsigned long qhandle; |
56 | |
57 | spinlock_t lock; |
58 | u8 q_type; |
59 | void *q; |
60 | unsigned long head; |
61 | unsigned long tail; |
62 | struct list_head jobs; |
63 | |
64 | unsigned long devino; |
65 | |
66 | char irq_name[32]; |
67 | unsigned int irq; |
68 | |
69 | struct list_head list; |
70 | }; |
71 | |
72 | struct spu_qreg { |
73 | struct spu_queue *queue; |
74 | unsigned long type; |
75 | }; |
76 | |
77 | static struct spu_queue **cpu_to_cwq; |
78 | static struct spu_queue **cpu_to_mau; |
79 | |
80 | static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off) |
81 | { |
82 | if (q->q_type == HV_NCS_QTYPE_MAU) { |
83 | off += MAU_ENTRY_SIZE; |
84 | if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES)) |
85 | off = 0; |
86 | } else { |
87 | off += CWQ_ENTRY_SIZE; |
88 | if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES)) |
89 | off = 0; |
90 | } |
91 | return off; |
92 | } |
93 | |
94 | struct n2_request_common { |
95 | struct list_head entry; |
96 | unsigned int offset; |
97 | }; |
98 | #define OFFSET_NOT_RUNNING (~(unsigned int)0) |
99 | |
100 | /* An async job request records the final tail value it used in |
101 | * n2_request_common->offset, test to see if that offset is in |
102 | * the range old_head, new_head, inclusive. |
103 | */ |
104 | static inline bool job_finished(struct spu_queue *q, unsigned int offset, |
105 | unsigned long old_head, unsigned long new_head) |
106 | { |
107 | if (old_head <= new_head) { |
108 | if (offset > old_head && offset <= new_head) |
109 | return true; |
110 | } else { |
111 | if (offset > old_head || offset <= new_head) |
112 | return true; |
113 | } |
114 | return false; |
115 | } |
116 | |
117 | /* When the HEAD marker is unequal to the actual HEAD, we get |
118 | * a virtual device INO interrupt. We should process the |
119 | * completed CWQ entries and adjust the HEAD marker to clear |
120 | * the IRQ. |
121 | */ |
122 | static irqreturn_t cwq_intr(int irq, void *dev_id) |
123 | { |
124 | unsigned long off, new_head, hv_ret; |
125 | struct spu_queue *q = dev_id; |
126 | |
127 | pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n" , |
128 | smp_processor_id(), q->qhandle); |
129 | |
130 | spin_lock(lock: &q->lock); |
131 | |
132 | hv_ret = sun4v_ncs_gethead(qhandle: q->qhandle, head: &new_head); |
133 | |
134 | pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n" , |
135 | smp_processor_id(), new_head, hv_ret); |
136 | |
137 | for (off = q->head; off != new_head; off = spu_next_offset(q, off)) { |
138 | /* XXX ... XXX */ |
139 | } |
140 | |
141 | hv_ret = sun4v_ncs_sethead_marker(qhandle: q->qhandle, head: new_head); |
142 | if (hv_ret == HV_EOK) |
143 | q->head = new_head; |
144 | |
145 | spin_unlock(lock: &q->lock); |
146 | |
147 | return IRQ_HANDLED; |
148 | } |
149 | |
150 | static irqreturn_t mau_intr(int irq, void *dev_id) |
151 | { |
152 | struct spu_queue *q = dev_id; |
153 | unsigned long head, hv_ret; |
154 | |
155 | spin_lock(lock: &q->lock); |
156 | |
157 | pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n" , |
158 | smp_processor_id(), q->qhandle); |
159 | |
160 | hv_ret = sun4v_ncs_gethead(qhandle: q->qhandle, head: &head); |
161 | |
162 | pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n" , |
163 | smp_processor_id(), head, hv_ret); |
164 | |
165 | sun4v_ncs_sethead_marker(qhandle: q->qhandle, head); |
166 | |
167 | spin_unlock(lock: &q->lock); |
168 | |
169 | return IRQ_HANDLED; |
170 | } |
171 | |
172 | static void *spu_queue_next(struct spu_queue *q, void *cur) |
173 | { |
174 | return q->q + spu_next_offset(q, off: cur - q->q); |
175 | } |
176 | |
177 | static int spu_queue_num_free(struct spu_queue *q) |
178 | { |
179 | unsigned long head = q->head; |
180 | unsigned long tail = q->tail; |
181 | unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES); |
182 | unsigned long diff; |
183 | |
184 | if (head > tail) |
185 | diff = head - tail; |
186 | else |
187 | diff = (end - tail) + head; |
188 | |
189 | return (diff / CWQ_ENTRY_SIZE) - 1; |
190 | } |
191 | |
192 | static void *spu_queue_alloc(struct spu_queue *q, int num_entries) |
193 | { |
194 | int avail = spu_queue_num_free(q); |
195 | |
196 | if (avail >= num_entries) |
197 | return q->q + q->tail; |
198 | |
199 | return NULL; |
200 | } |
201 | |
202 | static unsigned long spu_queue_submit(struct spu_queue *q, void *last) |
203 | { |
204 | unsigned long hv_ret, new_tail; |
205 | |
206 | new_tail = spu_next_offset(q, off: last - q->q); |
207 | |
208 | hv_ret = sun4v_ncs_settail(qhandle: q->qhandle, tail: new_tail); |
209 | if (hv_ret == HV_EOK) |
210 | q->tail = new_tail; |
211 | return hv_ret; |
212 | } |
213 | |
214 | static u64 control_word_base(unsigned int len, unsigned int hmac_key_len, |
215 | int enc_type, int auth_type, |
216 | unsigned int hash_len, |
217 | bool sfas, bool sob, bool eob, bool encrypt, |
218 | int opcode) |
219 | { |
220 | u64 word = (len - 1) & CONTROL_LEN; |
221 | |
222 | word |= ((u64) opcode << CONTROL_OPCODE_SHIFT); |
223 | word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT); |
224 | word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT); |
225 | if (sfas) |
226 | word |= CONTROL_STORE_FINAL_AUTH_STATE; |
227 | if (sob) |
228 | word |= CONTROL_START_OF_BLOCK; |
229 | if (eob) |
230 | word |= CONTROL_END_OF_BLOCK; |
231 | if (encrypt) |
232 | word |= CONTROL_ENCRYPT; |
233 | if (hmac_key_len) |
234 | word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT; |
235 | if (hash_len) |
236 | word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT; |
237 | |
238 | return word; |
239 | } |
240 | |
241 | #if 0 |
242 | static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) |
243 | { |
244 | if (this_len >= 64 || |
245 | qp->head != qp->tail) |
246 | return true; |
247 | return false; |
248 | } |
249 | #endif |
250 | |
251 | struct n2_ahash_alg { |
252 | struct list_head entry; |
253 | const u8 *hash_zero; |
254 | const u8 *hash_init; |
255 | u8 hw_op_hashsz; |
256 | u8 digest_size; |
257 | u8 auth_type; |
258 | u8 hmac_type; |
259 | struct ahash_alg alg; |
260 | }; |
261 | |
262 | static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm) |
263 | { |
264 | struct crypto_alg *alg = tfm->__crt_alg; |
265 | struct ahash_alg *ahash_alg; |
266 | |
267 | ahash_alg = container_of(alg, struct ahash_alg, halg.base); |
268 | |
269 | return container_of(ahash_alg, struct n2_ahash_alg, alg); |
270 | } |
271 | |
272 | struct n2_hmac_alg { |
273 | const char *child_alg; |
274 | struct n2_ahash_alg derived; |
275 | }; |
276 | |
277 | static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm) |
278 | { |
279 | struct crypto_alg *alg = tfm->__crt_alg; |
280 | struct ahash_alg *ahash_alg; |
281 | |
282 | ahash_alg = container_of(alg, struct ahash_alg, halg.base); |
283 | |
284 | return container_of(ahash_alg, struct n2_hmac_alg, derived.alg); |
285 | } |
286 | |
287 | struct n2_hash_ctx { |
288 | struct crypto_ahash *fallback_tfm; |
289 | }; |
290 | |
291 | #define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */ |
292 | |
293 | struct n2_hmac_ctx { |
294 | struct n2_hash_ctx base; |
295 | |
296 | struct crypto_shash *child_shash; |
297 | |
298 | int hash_key_len; |
299 | unsigned char hash_key[N2_HASH_KEY_MAX]; |
300 | }; |
301 | |
302 | struct n2_hash_req_ctx { |
303 | union { |
304 | struct md5_state md5; |
305 | struct sha1_state sha1; |
306 | struct sha256_state sha256; |
307 | } u; |
308 | |
309 | struct ahash_request fallback_req; |
310 | }; |
311 | |
312 | static int n2_hash_async_init(struct ahash_request *req) |
313 | { |
314 | struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); |
315 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
316 | struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); |
317 | |
318 | ahash_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback_tfm); |
319 | rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; |
320 | |
321 | return crypto_ahash_init(req: &rctx->fallback_req); |
322 | } |
323 | |
324 | static int n2_hash_async_update(struct ahash_request *req) |
325 | { |
326 | struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); |
327 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
328 | struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); |
329 | |
330 | ahash_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback_tfm); |
331 | rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; |
332 | rctx->fallback_req.nbytes = req->nbytes; |
333 | rctx->fallback_req.src = req->src; |
334 | |
335 | return crypto_ahash_update(req: &rctx->fallback_req); |
336 | } |
337 | |
338 | static int n2_hash_async_final(struct ahash_request *req) |
339 | { |
340 | struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); |
341 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
342 | struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); |
343 | |
344 | ahash_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback_tfm); |
345 | rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; |
346 | rctx->fallback_req.result = req->result; |
347 | |
348 | return crypto_ahash_final(req: &rctx->fallback_req); |
349 | } |
350 | |
351 | static int n2_hash_async_finup(struct ahash_request *req) |
352 | { |
353 | struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); |
354 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
355 | struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); |
356 | |
357 | ahash_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback_tfm); |
358 | rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; |
359 | rctx->fallback_req.nbytes = req->nbytes; |
360 | rctx->fallback_req.src = req->src; |
361 | rctx->fallback_req.result = req->result; |
362 | |
363 | return crypto_ahash_finup(req: &rctx->fallback_req); |
364 | } |
365 | |
366 | static int n2_hash_async_noimport(struct ahash_request *req, const void *in) |
367 | { |
368 | return -ENOSYS; |
369 | } |
370 | |
371 | static int n2_hash_async_noexport(struct ahash_request *req, void *out) |
372 | { |
373 | return -ENOSYS; |
374 | } |
375 | |
376 | static int n2_hash_cra_init(struct crypto_tfm *tfm) |
377 | { |
378 | const char *fallback_driver_name = crypto_tfm_alg_name(tfm); |
379 | struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); |
380 | struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm: ahash); |
381 | struct crypto_ahash *fallback_tfm; |
382 | int err; |
383 | |
384 | fallback_tfm = crypto_alloc_ahash(alg_name: fallback_driver_name, type: 0, |
385 | CRYPTO_ALG_NEED_FALLBACK); |
386 | if (IS_ERR(ptr: fallback_tfm)) { |
387 | pr_warn("Fallback driver '%s' could not be loaded!\n" , |
388 | fallback_driver_name); |
389 | err = PTR_ERR(ptr: fallback_tfm); |
390 | goto out; |
391 | } |
392 | |
393 | crypto_ahash_set_reqsize(tfm: ahash, reqsize: (sizeof(struct n2_hash_req_ctx) + |
394 | crypto_ahash_reqsize(tfm: fallback_tfm))); |
395 | |
396 | ctx->fallback_tfm = fallback_tfm; |
397 | return 0; |
398 | |
399 | out: |
400 | return err; |
401 | } |
402 | |
403 | static void n2_hash_cra_exit(struct crypto_tfm *tfm) |
404 | { |
405 | struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); |
406 | struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm: ahash); |
407 | |
408 | crypto_free_ahash(tfm: ctx->fallback_tfm); |
409 | } |
410 | |
411 | static int n2_hmac_cra_init(struct crypto_tfm *tfm) |
412 | { |
413 | const char *fallback_driver_name = crypto_tfm_alg_name(tfm); |
414 | struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); |
415 | struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm: ahash); |
416 | struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm); |
417 | struct crypto_ahash *fallback_tfm; |
418 | struct crypto_shash *child_shash; |
419 | int err; |
420 | |
421 | fallback_tfm = crypto_alloc_ahash(alg_name: fallback_driver_name, type: 0, |
422 | CRYPTO_ALG_NEED_FALLBACK); |
423 | if (IS_ERR(ptr: fallback_tfm)) { |
424 | pr_warn("Fallback driver '%s' could not be loaded!\n" , |
425 | fallback_driver_name); |
426 | err = PTR_ERR(ptr: fallback_tfm); |
427 | goto out; |
428 | } |
429 | |
430 | child_shash = crypto_alloc_shash(alg_name: n2alg->child_alg, type: 0, mask: 0); |
431 | if (IS_ERR(ptr: child_shash)) { |
432 | pr_warn("Child shash '%s' could not be loaded!\n" , |
433 | n2alg->child_alg); |
434 | err = PTR_ERR(ptr: child_shash); |
435 | goto out_free_fallback; |
436 | } |
437 | |
438 | crypto_ahash_set_reqsize(tfm: ahash, reqsize: (sizeof(struct n2_hash_req_ctx) + |
439 | crypto_ahash_reqsize(tfm: fallback_tfm))); |
440 | |
441 | ctx->child_shash = child_shash; |
442 | ctx->base.fallback_tfm = fallback_tfm; |
443 | return 0; |
444 | |
445 | out_free_fallback: |
446 | crypto_free_ahash(tfm: fallback_tfm); |
447 | |
448 | out: |
449 | return err; |
450 | } |
451 | |
452 | static void n2_hmac_cra_exit(struct crypto_tfm *tfm) |
453 | { |
454 | struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); |
455 | struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm: ahash); |
456 | |
457 | crypto_free_ahash(tfm: ctx->base.fallback_tfm); |
458 | crypto_free_shash(tfm: ctx->child_shash); |
459 | } |
460 | |
461 | static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key, |
462 | unsigned int keylen) |
463 | { |
464 | struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); |
465 | struct crypto_shash *child_shash = ctx->child_shash; |
466 | struct crypto_ahash *fallback_tfm; |
467 | int err, bs, ds; |
468 | |
469 | fallback_tfm = ctx->base.fallback_tfm; |
470 | err = crypto_ahash_setkey(tfm: fallback_tfm, key, keylen); |
471 | if (err) |
472 | return err; |
473 | |
474 | bs = crypto_shash_blocksize(tfm: child_shash); |
475 | ds = crypto_shash_digestsize(tfm: child_shash); |
476 | BUG_ON(ds > N2_HASH_KEY_MAX); |
477 | if (keylen > bs) { |
478 | err = crypto_shash_tfm_digest(tfm: child_shash, data: key, len: keylen, |
479 | out: ctx->hash_key); |
480 | if (err) |
481 | return err; |
482 | keylen = ds; |
483 | } else if (keylen <= N2_HASH_KEY_MAX) |
484 | memcpy(ctx->hash_key, key, keylen); |
485 | |
486 | ctx->hash_key_len = keylen; |
487 | |
488 | return err; |
489 | } |
490 | |
491 | static unsigned long wait_for_tail(struct spu_queue *qp) |
492 | { |
493 | unsigned long head, hv_ret; |
494 | |
495 | do { |
496 | hv_ret = sun4v_ncs_gethead(qhandle: qp->qhandle, head: &head); |
497 | if (hv_ret != HV_EOK) { |
498 | pr_err("Hypervisor error on gethead\n" ); |
499 | break; |
500 | } |
501 | if (head == qp->tail) { |
502 | qp->head = head; |
503 | break; |
504 | } |
505 | } while (1); |
506 | return hv_ret; |
507 | } |
508 | |
509 | static unsigned long submit_and_wait_for_tail(struct spu_queue *qp, |
510 | struct cwq_initial_entry *ent) |
511 | { |
512 | unsigned long hv_ret = spu_queue_submit(q: qp, last: ent); |
513 | |
514 | if (hv_ret == HV_EOK) |
515 | hv_ret = wait_for_tail(qp); |
516 | |
517 | return hv_ret; |
518 | } |
519 | |
520 | static int n2_do_async_digest(struct ahash_request *req, |
521 | unsigned int auth_type, unsigned int digest_size, |
522 | unsigned int result_size, void *hash_loc, |
523 | unsigned long auth_key, unsigned int auth_key_len) |
524 | { |
525 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
526 | struct cwq_initial_entry *ent; |
527 | struct crypto_hash_walk walk; |
528 | struct spu_queue *qp; |
529 | unsigned long flags; |
530 | int err = -ENODEV; |
531 | int nbytes, cpu; |
532 | |
533 | /* The total effective length of the operation may not |
534 | * exceed 2^16. |
535 | */ |
536 | if (unlikely(req->nbytes > (1 << 16))) { |
537 | struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); |
538 | struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); |
539 | |
540 | ahash_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback_tfm); |
541 | rctx->fallback_req.base.flags = |
542 | req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; |
543 | rctx->fallback_req.nbytes = req->nbytes; |
544 | rctx->fallback_req.src = req->src; |
545 | rctx->fallback_req.result = req->result; |
546 | |
547 | return crypto_ahash_digest(req: &rctx->fallback_req); |
548 | } |
549 | |
550 | nbytes = crypto_hash_walk_first(req, walk: &walk); |
551 | |
552 | cpu = get_cpu(); |
553 | qp = cpu_to_cwq[cpu]; |
554 | if (!qp) |
555 | goto out; |
556 | |
557 | spin_lock_irqsave(&qp->lock, flags); |
558 | |
559 | /* XXX can do better, improve this later by doing a by-hand scatterlist |
560 | * XXX walk, etc. |
561 | */ |
562 | ent = qp->q + qp->tail; |
563 | |
564 | ent->control = control_word_base(len: nbytes, hmac_key_len: auth_key_len, enc_type: 0, |
565 | auth_type, hash_len: digest_size, |
566 | sfas: false, sob: true, eob: false, encrypt: false, |
567 | OPCODE_INPLACE_BIT | |
568 | OPCODE_AUTH_MAC); |
569 | ent->src_addr = __pa(walk.data); |
570 | ent->auth_key_addr = auth_key; |
571 | ent->auth_iv_addr = __pa(hash_loc); |
572 | ent->final_auth_state_addr = 0UL; |
573 | ent->enc_key_addr = 0UL; |
574 | ent->enc_iv_addr = 0UL; |
575 | ent->dest_addr = __pa(hash_loc); |
576 | |
577 | nbytes = crypto_hash_walk_done(walk: &walk, err: 0); |
578 | while (nbytes > 0) { |
579 | ent = spu_queue_next(q: qp, cur: ent); |
580 | |
581 | ent->control = (nbytes - 1); |
582 | ent->src_addr = __pa(walk.data); |
583 | ent->auth_key_addr = 0UL; |
584 | ent->auth_iv_addr = 0UL; |
585 | ent->final_auth_state_addr = 0UL; |
586 | ent->enc_key_addr = 0UL; |
587 | ent->enc_iv_addr = 0UL; |
588 | ent->dest_addr = 0UL; |
589 | |
590 | nbytes = crypto_hash_walk_done(walk: &walk, err: 0); |
591 | } |
592 | ent->control |= CONTROL_END_OF_BLOCK; |
593 | |
594 | if (submit_and_wait_for_tail(qp, ent) != HV_EOK) |
595 | err = -EINVAL; |
596 | else |
597 | err = 0; |
598 | |
599 | spin_unlock_irqrestore(lock: &qp->lock, flags); |
600 | |
601 | if (!err) |
602 | memcpy(req->result, hash_loc, result_size); |
603 | out: |
604 | put_cpu(); |
605 | |
606 | return err; |
607 | } |
608 | |
609 | static int n2_hash_async_digest(struct ahash_request *req) |
610 | { |
611 | struct n2_ahash_alg *n2alg = n2_ahash_alg(tfm: req->base.tfm); |
612 | struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); |
613 | int ds; |
614 | |
615 | ds = n2alg->digest_size; |
616 | if (unlikely(req->nbytes == 0)) { |
617 | memcpy(req->result, n2alg->hash_zero, ds); |
618 | return 0; |
619 | } |
620 | memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz); |
621 | |
622 | return n2_do_async_digest(req, auth_type: n2alg->auth_type, |
623 | digest_size: n2alg->hw_op_hashsz, result_size: ds, |
624 | hash_loc: &rctx->u, auth_key: 0UL, auth_key_len: 0); |
625 | } |
626 | |
627 | static int n2_hmac_async_digest(struct ahash_request *req) |
628 | { |
629 | struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm: req->base.tfm); |
630 | struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); |
631 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
632 | struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); |
633 | int ds; |
634 | |
635 | ds = n2alg->derived.digest_size; |
636 | if (unlikely(req->nbytes == 0) || |
637 | unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) { |
638 | struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); |
639 | struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); |
640 | |
641 | ahash_request_set_tfm(req: &rctx->fallback_req, tfm: ctx->fallback_tfm); |
642 | rctx->fallback_req.base.flags = |
643 | req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; |
644 | rctx->fallback_req.nbytes = req->nbytes; |
645 | rctx->fallback_req.src = req->src; |
646 | rctx->fallback_req.result = req->result; |
647 | |
648 | return crypto_ahash_digest(req: &rctx->fallback_req); |
649 | } |
650 | memcpy(&rctx->u, n2alg->derived.hash_init, |
651 | n2alg->derived.hw_op_hashsz); |
652 | |
653 | return n2_do_async_digest(req, auth_type: n2alg->derived.hmac_type, |
654 | digest_size: n2alg->derived.hw_op_hashsz, result_size: ds, |
655 | hash_loc: &rctx->u, |
656 | __pa(&ctx->hash_key), |
657 | auth_key_len: ctx->hash_key_len); |
658 | } |
659 | |
660 | struct n2_skcipher_context { |
661 | int key_len; |
662 | int enc_type; |
663 | union { |
664 | u8 aes[AES_MAX_KEY_SIZE]; |
665 | u8 des[DES_KEY_SIZE]; |
666 | u8 des3[3 * DES_KEY_SIZE]; |
667 | } key; |
668 | }; |
669 | |
670 | #define N2_CHUNK_ARR_LEN 16 |
671 | |
672 | struct n2_crypto_chunk { |
673 | struct list_head entry; |
674 | unsigned long iv_paddr : 44; |
675 | unsigned long arr_len : 20; |
676 | unsigned long dest_paddr; |
677 | unsigned long dest_final; |
678 | struct { |
679 | unsigned long src_paddr : 44; |
680 | unsigned long src_len : 20; |
681 | } arr[N2_CHUNK_ARR_LEN]; |
682 | }; |
683 | |
684 | struct n2_request_context { |
685 | struct skcipher_walk walk; |
686 | struct list_head chunk_list; |
687 | struct n2_crypto_chunk chunk; |
688 | u8 temp_iv[16]; |
689 | }; |
690 | |
691 | /* The SPU allows some level of flexibility for partial cipher blocks |
692 | * being specified in a descriptor. |
693 | * |
694 | * It merely requires that every descriptor's length field is at least |
695 | * as large as the cipher block size. This means that a cipher block |
696 | * can span at most 2 descriptors. However, this does not allow a |
697 | * partial block to span into the final descriptor as that would |
698 | * violate the rule (since every descriptor's length must be at lest |
699 | * the block size). So, for example, assuming an 8 byte block size: |
700 | * |
701 | * 0xe --> 0xa --> 0x8 |
702 | * |
703 | * is a valid length sequence, whereas: |
704 | * |
705 | * 0xe --> 0xb --> 0x7 |
706 | * |
707 | * is not a valid sequence. |
708 | */ |
709 | |
710 | struct n2_skcipher_alg { |
711 | struct list_head entry; |
712 | u8 enc_type; |
713 | struct skcipher_alg skcipher; |
714 | }; |
715 | |
716 | static inline struct n2_skcipher_alg *n2_skcipher_alg(struct crypto_skcipher *tfm) |
717 | { |
718 | struct skcipher_alg *alg = crypto_skcipher_alg(tfm); |
719 | |
720 | return container_of(alg, struct n2_skcipher_alg, skcipher); |
721 | } |
722 | |
723 | struct n2_skcipher_request_context { |
724 | struct skcipher_walk walk; |
725 | }; |
726 | |
727 | static int n2_aes_setkey(struct crypto_skcipher *skcipher, const u8 *key, |
728 | unsigned int keylen) |
729 | { |
730 | struct crypto_tfm *tfm = crypto_skcipher_tfm(tfm: skcipher); |
731 | struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); |
732 | struct n2_skcipher_alg *n2alg = n2_skcipher_alg(tfm: skcipher); |
733 | |
734 | ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK); |
735 | |
736 | switch (keylen) { |
737 | case AES_KEYSIZE_128: |
738 | ctx->enc_type |= ENC_TYPE_ALG_AES128; |
739 | break; |
740 | case AES_KEYSIZE_192: |
741 | ctx->enc_type |= ENC_TYPE_ALG_AES192; |
742 | break; |
743 | case AES_KEYSIZE_256: |
744 | ctx->enc_type |= ENC_TYPE_ALG_AES256; |
745 | break; |
746 | default: |
747 | return -EINVAL; |
748 | } |
749 | |
750 | ctx->key_len = keylen; |
751 | memcpy(ctx->key.aes, key, keylen); |
752 | return 0; |
753 | } |
754 | |
755 | static int n2_des_setkey(struct crypto_skcipher *skcipher, const u8 *key, |
756 | unsigned int keylen) |
757 | { |
758 | struct crypto_tfm *tfm = crypto_skcipher_tfm(tfm: skcipher); |
759 | struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); |
760 | struct n2_skcipher_alg *n2alg = n2_skcipher_alg(tfm: skcipher); |
761 | int err; |
762 | |
763 | err = verify_skcipher_des_key(tfm: skcipher, key); |
764 | if (err) |
765 | return err; |
766 | |
767 | ctx->enc_type = n2alg->enc_type; |
768 | |
769 | ctx->key_len = keylen; |
770 | memcpy(ctx->key.des, key, keylen); |
771 | return 0; |
772 | } |
773 | |
774 | static int n2_3des_setkey(struct crypto_skcipher *skcipher, const u8 *key, |
775 | unsigned int keylen) |
776 | { |
777 | struct crypto_tfm *tfm = crypto_skcipher_tfm(tfm: skcipher); |
778 | struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); |
779 | struct n2_skcipher_alg *n2alg = n2_skcipher_alg(tfm: skcipher); |
780 | int err; |
781 | |
782 | err = verify_skcipher_des3_key(tfm: skcipher, key); |
783 | if (err) |
784 | return err; |
785 | |
786 | ctx->enc_type = n2alg->enc_type; |
787 | |
788 | ctx->key_len = keylen; |
789 | memcpy(ctx->key.des3, key, keylen); |
790 | return 0; |
791 | } |
792 | |
793 | static inline int skcipher_descriptor_len(int nbytes, unsigned int block_size) |
794 | { |
795 | int this_len = nbytes; |
796 | |
797 | this_len -= (nbytes & (block_size - 1)); |
798 | return this_len > (1 << 16) ? (1 << 16) : this_len; |
799 | } |
800 | |
801 | static int __n2_crypt_chunk(struct crypto_skcipher *skcipher, |
802 | struct n2_crypto_chunk *cp, |
803 | struct spu_queue *qp, bool encrypt) |
804 | { |
805 | struct n2_skcipher_context *ctx = crypto_skcipher_ctx(tfm: skcipher); |
806 | struct cwq_initial_entry *ent; |
807 | bool in_place; |
808 | int i; |
809 | |
810 | ent = spu_queue_alloc(q: qp, num_entries: cp->arr_len); |
811 | if (!ent) { |
812 | pr_info("queue_alloc() of %d fails\n" , |
813 | cp->arr_len); |
814 | return -EBUSY; |
815 | } |
816 | |
817 | in_place = (cp->dest_paddr == cp->arr[0].src_paddr); |
818 | |
819 | ent->control = control_word_base(len: cp->arr[0].src_len, |
820 | hmac_key_len: 0, enc_type: ctx->enc_type, auth_type: 0, hash_len: 0, |
821 | sfas: false, sob: true, eob: false, encrypt, |
822 | OPCODE_ENCRYPT | |
823 | (in_place ? OPCODE_INPLACE_BIT : 0)); |
824 | ent->src_addr = cp->arr[0].src_paddr; |
825 | ent->auth_key_addr = 0UL; |
826 | ent->auth_iv_addr = 0UL; |
827 | ent->final_auth_state_addr = 0UL; |
828 | ent->enc_key_addr = __pa(&ctx->key); |
829 | ent->enc_iv_addr = cp->iv_paddr; |
830 | ent->dest_addr = (in_place ? 0UL : cp->dest_paddr); |
831 | |
832 | for (i = 1; i < cp->arr_len; i++) { |
833 | ent = spu_queue_next(q: qp, cur: ent); |
834 | |
835 | ent->control = cp->arr[i].src_len - 1; |
836 | ent->src_addr = cp->arr[i].src_paddr; |
837 | ent->auth_key_addr = 0UL; |
838 | ent->auth_iv_addr = 0UL; |
839 | ent->final_auth_state_addr = 0UL; |
840 | ent->enc_key_addr = 0UL; |
841 | ent->enc_iv_addr = 0UL; |
842 | ent->dest_addr = 0UL; |
843 | } |
844 | ent->control |= CONTROL_END_OF_BLOCK; |
845 | |
846 | return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0; |
847 | } |
848 | |
849 | static int n2_compute_chunks(struct skcipher_request *req) |
850 | { |
851 | struct n2_request_context *rctx = skcipher_request_ctx(req); |
852 | struct skcipher_walk *walk = &rctx->walk; |
853 | struct n2_crypto_chunk *chunk; |
854 | unsigned long dest_prev; |
855 | unsigned int tot_len; |
856 | bool prev_in_place; |
857 | int err, nbytes; |
858 | |
859 | err = skcipher_walk_async(walk, req); |
860 | if (err) |
861 | return err; |
862 | |
863 | INIT_LIST_HEAD(list: &rctx->chunk_list); |
864 | |
865 | chunk = &rctx->chunk; |
866 | INIT_LIST_HEAD(list: &chunk->entry); |
867 | |
868 | chunk->iv_paddr = 0UL; |
869 | chunk->arr_len = 0; |
870 | chunk->dest_paddr = 0UL; |
871 | |
872 | prev_in_place = false; |
873 | dest_prev = ~0UL; |
874 | tot_len = 0; |
875 | |
876 | while ((nbytes = walk->nbytes) != 0) { |
877 | unsigned long dest_paddr, src_paddr; |
878 | bool in_place; |
879 | int this_len; |
880 | |
881 | src_paddr = (page_to_phys(walk->src.phys.page) + |
882 | walk->src.phys.offset); |
883 | dest_paddr = (page_to_phys(walk->dst.phys.page) + |
884 | walk->dst.phys.offset); |
885 | in_place = (src_paddr == dest_paddr); |
886 | this_len = skcipher_descriptor_len(nbytes, block_size: walk->blocksize); |
887 | |
888 | if (chunk->arr_len != 0) { |
889 | if (in_place != prev_in_place || |
890 | (!prev_in_place && |
891 | dest_paddr != dest_prev) || |
892 | chunk->arr_len == N2_CHUNK_ARR_LEN || |
893 | tot_len + this_len > (1 << 16)) { |
894 | chunk->dest_final = dest_prev; |
895 | list_add_tail(new: &chunk->entry, |
896 | head: &rctx->chunk_list); |
897 | chunk = kzalloc(size: sizeof(*chunk), GFP_ATOMIC); |
898 | if (!chunk) { |
899 | err = -ENOMEM; |
900 | break; |
901 | } |
902 | INIT_LIST_HEAD(list: &chunk->entry); |
903 | } |
904 | } |
905 | if (chunk->arr_len == 0) { |
906 | chunk->dest_paddr = dest_paddr; |
907 | tot_len = 0; |
908 | } |
909 | chunk->arr[chunk->arr_len].src_paddr = src_paddr; |
910 | chunk->arr[chunk->arr_len].src_len = this_len; |
911 | chunk->arr_len++; |
912 | |
913 | dest_prev = dest_paddr + this_len; |
914 | prev_in_place = in_place; |
915 | tot_len += this_len; |
916 | |
917 | err = skcipher_walk_done(walk, err: nbytes - this_len); |
918 | if (err) |
919 | break; |
920 | } |
921 | if (!err && chunk->arr_len != 0) { |
922 | chunk->dest_final = dest_prev; |
923 | list_add_tail(new: &chunk->entry, head: &rctx->chunk_list); |
924 | } |
925 | |
926 | return err; |
927 | } |
928 | |
929 | static void n2_chunk_complete(struct skcipher_request *req, void *final_iv) |
930 | { |
931 | struct n2_request_context *rctx = skcipher_request_ctx(req); |
932 | struct n2_crypto_chunk *c, *tmp; |
933 | |
934 | if (final_iv) |
935 | memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize); |
936 | |
937 | list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { |
938 | list_del(entry: &c->entry); |
939 | if (unlikely(c != &rctx->chunk)) |
940 | kfree(objp: c); |
941 | } |
942 | |
943 | } |
944 | |
945 | static int n2_do_ecb(struct skcipher_request *req, bool encrypt) |
946 | { |
947 | struct n2_request_context *rctx = skcipher_request_ctx(req); |
948 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
949 | int err = n2_compute_chunks(req); |
950 | struct n2_crypto_chunk *c, *tmp; |
951 | unsigned long flags, hv_ret; |
952 | struct spu_queue *qp; |
953 | |
954 | if (err) |
955 | return err; |
956 | |
957 | qp = cpu_to_cwq[get_cpu()]; |
958 | err = -ENODEV; |
959 | if (!qp) |
960 | goto out; |
961 | |
962 | spin_lock_irqsave(&qp->lock, flags); |
963 | |
964 | list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { |
965 | err = __n2_crypt_chunk(skcipher: tfm, cp: c, qp, encrypt); |
966 | if (err) |
967 | break; |
968 | list_del(entry: &c->entry); |
969 | if (unlikely(c != &rctx->chunk)) |
970 | kfree(objp: c); |
971 | } |
972 | if (!err) { |
973 | hv_ret = wait_for_tail(qp); |
974 | if (hv_ret != HV_EOK) |
975 | err = -EINVAL; |
976 | } |
977 | |
978 | spin_unlock_irqrestore(lock: &qp->lock, flags); |
979 | |
980 | out: |
981 | put_cpu(); |
982 | |
983 | n2_chunk_complete(req, NULL); |
984 | return err; |
985 | } |
986 | |
987 | static int n2_encrypt_ecb(struct skcipher_request *req) |
988 | { |
989 | return n2_do_ecb(req, encrypt: true); |
990 | } |
991 | |
992 | static int n2_decrypt_ecb(struct skcipher_request *req) |
993 | { |
994 | return n2_do_ecb(req, encrypt: false); |
995 | } |
996 | |
997 | static int n2_do_chaining(struct skcipher_request *req, bool encrypt) |
998 | { |
999 | struct n2_request_context *rctx = skcipher_request_ctx(req); |
1000 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
1001 | unsigned long flags, hv_ret, iv_paddr; |
1002 | int err = n2_compute_chunks(req); |
1003 | struct n2_crypto_chunk *c, *tmp; |
1004 | struct spu_queue *qp; |
1005 | void *final_iv_addr; |
1006 | |
1007 | final_iv_addr = NULL; |
1008 | |
1009 | if (err) |
1010 | return err; |
1011 | |
1012 | qp = cpu_to_cwq[get_cpu()]; |
1013 | err = -ENODEV; |
1014 | if (!qp) |
1015 | goto out; |
1016 | |
1017 | spin_lock_irqsave(&qp->lock, flags); |
1018 | |
1019 | if (encrypt) { |
1020 | iv_paddr = __pa(rctx->walk.iv); |
1021 | list_for_each_entry_safe(c, tmp, &rctx->chunk_list, |
1022 | entry) { |
1023 | c->iv_paddr = iv_paddr; |
1024 | err = __n2_crypt_chunk(skcipher: tfm, cp: c, qp, encrypt: true); |
1025 | if (err) |
1026 | break; |
1027 | iv_paddr = c->dest_final - rctx->walk.blocksize; |
1028 | list_del(entry: &c->entry); |
1029 | if (unlikely(c != &rctx->chunk)) |
1030 | kfree(objp: c); |
1031 | } |
1032 | final_iv_addr = __va(iv_paddr); |
1033 | } else { |
1034 | list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list, |
1035 | entry) { |
1036 | if (c == &rctx->chunk) { |
1037 | iv_paddr = __pa(rctx->walk.iv); |
1038 | } else { |
1039 | iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr + |
1040 | tmp->arr[tmp->arr_len-1].src_len - |
1041 | rctx->walk.blocksize); |
1042 | } |
1043 | if (!final_iv_addr) { |
1044 | unsigned long pa; |
1045 | |
1046 | pa = (c->arr[c->arr_len-1].src_paddr + |
1047 | c->arr[c->arr_len-1].src_len - |
1048 | rctx->walk.blocksize); |
1049 | final_iv_addr = rctx->temp_iv; |
1050 | memcpy(rctx->temp_iv, __va(pa), |
1051 | rctx->walk.blocksize); |
1052 | } |
1053 | c->iv_paddr = iv_paddr; |
1054 | err = __n2_crypt_chunk(skcipher: tfm, cp: c, qp, encrypt: false); |
1055 | if (err) |
1056 | break; |
1057 | list_del(entry: &c->entry); |
1058 | if (unlikely(c != &rctx->chunk)) |
1059 | kfree(objp: c); |
1060 | } |
1061 | } |
1062 | if (!err) { |
1063 | hv_ret = wait_for_tail(qp); |
1064 | if (hv_ret != HV_EOK) |
1065 | err = -EINVAL; |
1066 | } |
1067 | |
1068 | spin_unlock_irqrestore(lock: &qp->lock, flags); |
1069 | |
1070 | out: |
1071 | put_cpu(); |
1072 | |
1073 | n2_chunk_complete(req, final_iv: err ? NULL : final_iv_addr); |
1074 | return err; |
1075 | } |
1076 | |
1077 | static int n2_encrypt_chaining(struct skcipher_request *req) |
1078 | { |
1079 | return n2_do_chaining(req, encrypt: true); |
1080 | } |
1081 | |
1082 | static int n2_decrypt_chaining(struct skcipher_request *req) |
1083 | { |
1084 | return n2_do_chaining(req, encrypt: false); |
1085 | } |
1086 | |
1087 | struct n2_skcipher_tmpl { |
1088 | const char *name; |
1089 | const char *drv_name; |
1090 | u8 block_size; |
1091 | u8 enc_type; |
1092 | struct skcipher_alg skcipher; |
1093 | }; |
1094 | |
1095 | static const struct n2_skcipher_tmpl skcipher_tmpls[] = { |
1096 | /* DES: ECB CBC and CFB are supported */ |
1097 | { .name = "ecb(des)" , |
1098 | .drv_name = "ecb-des" , |
1099 | .block_size = DES_BLOCK_SIZE, |
1100 | .enc_type = (ENC_TYPE_ALG_DES | |
1101 | ENC_TYPE_CHAINING_ECB), |
1102 | .skcipher = { |
1103 | .min_keysize = DES_KEY_SIZE, |
1104 | .max_keysize = DES_KEY_SIZE, |
1105 | .setkey = n2_des_setkey, |
1106 | .encrypt = n2_encrypt_ecb, |
1107 | .decrypt = n2_decrypt_ecb, |
1108 | }, |
1109 | }, |
1110 | { .name = "cbc(des)" , |
1111 | .drv_name = "cbc-des" , |
1112 | .block_size = DES_BLOCK_SIZE, |
1113 | .enc_type = (ENC_TYPE_ALG_DES | |
1114 | ENC_TYPE_CHAINING_CBC), |
1115 | .skcipher = { |
1116 | .ivsize = DES_BLOCK_SIZE, |
1117 | .min_keysize = DES_KEY_SIZE, |
1118 | .max_keysize = DES_KEY_SIZE, |
1119 | .setkey = n2_des_setkey, |
1120 | .encrypt = n2_encrypt_chaining, |
1121 | .decrypt = n2_decrypt_chaining, |
1122 | }, |
1123 | }, |
1124 | { .name = "cfb(des)" , |
1125 | .drv_name = "cfb-des" , |
1126 | .block_size = DES_BLOCK_SIZE, |
1127 | .enc_type = (ENC_TYPE_ALG_DES | |
1128 | ENC_TYPE_CHAINING_CFB), |
1129 | .skcipher = { |
1130 | .min_keysize = DES_KEY_SIZE, |
1131 | .max_keysize = DES_KEY_SIZE, |
1132 | .setkey = n2_des_setkey, |
1133 | .encrypt = n2_encrypt_chaining, |
1134 | .decrypt = n2_decrypt_chaining, |
1135 | }, |
1136 | }, |
1137 | |
1138 | /* 3DES: ECB CBC and CFB are supported */ |
1139 | { .name = "ecb(des3_ede)" , |
1140 | .drv_name = "ecb-3des" , |
1141 | .block_size = DES_BLOCK_SIZE, |
1142 | .enc_type = (ENC_TYPE_ALG_3DES | |
1143 | ENC_TYPE_CHAINING_ECB), |
1144 | .skcipher = { |
1145 | .min_keysize = 3 * DES_KEY_SIZE, |
1146 | .max_keysize = 3 * DES_KEY_SIZE, |
1147 | .setkey = n2_3des_setkey, |
1148 | .encrypt = n2_encrypt_ecb, |
1149 | .decrypt = n2_decrypt_ecb, |
1150 | }, |
1151 | }, |
1152 | { .name = "cbc(des3_ede)" , |
1153 | .drv_name = "cbc-3des" , |
1154 | .block_size = DES_BLOCK_SIZE, |
1155 | .enc_type = (ENC_TYPE_ALG_3DES | |
1156 | ENC_TYPE_CHAINING_CBC), |
1157 | .skcipher = { |
1158 | .ivsize = DES_BLOCK_SIZE, |
1159 | .min_keysize = 3 * DES_KEY_SIZE, |
1160 | .max_keysize = 3 * DES_KEY_SIZE, |
1161 | .setkey = n2_3des_setkey, |
1162 | .encrypt = n2_encrypt_chaining, |
1163 | .decrypt = n2_decrypt_chaining, |
1164 | }, |
1165 | }, |
1166 | { .name = "cfb(des3_ede)" , |
1167 | .drv_name = "cfb-3des" , |
1168 | .block_size = DES_BLOCK_SIZE, |
1169 | .enc_type = (ENC_TYPE_ALG_3DES | |
1170 | ENC_TYPE_CHAINING_CFB), |
1171 | .skcipher = { |
1172 | .min_keysize = 3 * DES_KEY_SIZE, |
1173 | .max_keysize = 3 * DES_KEY_SIZE, |
1174 | .setkey = n2_3des_setkey, |
1175 | .encrypt = n2_encrypt_chaining, |
1176 | .decrypt = n2_decrypt_chaining, |
1177 | }, |
1178 | }, |
1179 | /* AES: ECB CBC and CTR are supported */ |
1180 | { .name = "ecb(aes)" , |
1181 | .drv_name = "ecb-aes" , |
1182 | .block_size = AES_BLOCK_SIZE, |
1183 | .enc_type = (ENC_TYPE_ALG_AES128 | |
1184 | ENC_TYPE_CHAINING_ECB), |
1185 | .skcipher = { |
1186 | .min_keysize = AES_MIN_KEY_SIZE, |
1187 | .max_keysize = AES_MAX_KEY_SIZE, |
1188 | .setkey = n2_aes_setkey, |
1189 | .encrypt = n2_encrypt_ecb, |
1190 | .decrypt = n2_decrypt_ecb, |
1191 | }, |
1192 | }, |
1193 | { .name = "cbc(aes)" , |
1194 | .drv_name = "cbc-aes" , |
1195 | .block_size = AES_BLOCK_SIZE, |
1196 | .enc_type = (ENC_TYPE_ALG_AES128 | |
1197 | ENC_TYPE_CHAINING_CBC), |
1198 | .skcipher = { |
1199 | .ivsize = AES_BLOCK_SIZE, |
1200 | .min_keysize = AES_MIN_KEY_SIZE, |
1201 | .max_keysize = AES_MAX_KEY_SIZE, |
1202 | .setkey = n2_aes_setkey, |
1203 | .encrypt = n2_encrypt_chaining, |
1204 | .decrypt = n2_decrypt_chaining, |
1205 | }, |
1206 | }, |
1207 | { .name = "ctr(aes)" , |
1208 | .drv_name = "ctr-aes" , |
1209 | .block_size = AES_BLOCK_SIZE, |
1210 | .enc_type = (ENC_TYPE_ALG_AES128 | |
1211 | ENC_TYPE_CHAINING_COUNTER), |
1212 | .skcipher = { |
1213 | .ivsize = AES_BLOCK_SIZE, |
1214 | .min_keysize = AES_MIN_KEY_SIZE, |
1215 | .max_keysize = AES_MAX_KEY_SIZE, |
1216 | .setkey = n2_aes_setkey, |
1217 | .encrypt = n2_encrypt_chaining, |
1218 | .decrypt = n2_encrypt_chaining, |
1219 | }, |
1220 | }, |
1221 | |
1222 | }; |
1223 | #define NUM_CIPHER_TMPLS ARRAY_SIZE(skcipher_tmpls) |
1224 | |
1225 | static LIST_HEAD(skcipher_algs); |
1226 | |
1227 | struct n2_hash_tmpl { |
1228 | const char *name; |
1229 | const u8 *hash_zero; |
1230 | const u8 *hash_init; |
1231 | u8 hw_op_hashsz; |
1232 | u8 digest_size; |
1233 | u8 statesize; |
1234 | u8 block_size; |
1235 | u8 auth_type; |
1236 | u8 hmac_type; |
1237 | }; |
1238 | |
1239 | static const __le32 n2_md5_init[MD5_HASH_WORDS] = { |
1240 | cpu_to_le32(MD5_H0), |
1241 | cpu_to_le32(MD5_H1), |
1242 | cpu_to_le32(MD5_H2), |
1243 | cpu_to_le32(MD5_H3), |
1244 | }; |
1245 | static const u32 n2_sha1_init[SHA1_DIGEST_SIZE / 4] = { |
1246 | SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, |
1247 | }; |
1248 | static const u32 n2_sha256_init[SHA256_DIGEST_SIZE / 4] = { |
1249 | SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, |
1250 | SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, |
1251 | }; |
1252 | static const u32 n2_sha224_init[SHA256_DIGEST_SIZE / 4] = { |
1253 | SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, |
1254 | SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, |
1255 | }; |
1256 | |
1257 | static const struct n2_hash_tmpl hash_tmpls[] = { |
1258 | { .name = "md5" , |
1259 | .hash_zero = md5_zero_message_hash, |
1260 | .hash_init = (u8 *)n2_md5_init, |
1261 | .auth_type = AUTH_TYPE_MD5, |
1262 | .hmac_type = AUTH_TYPE_HMAC_MD5, |
1263 | .hw_op_hashsz = MD5_DIGEST_SIZE, |
1264 | .digest_size = MD5_DIGEST_SIZE, |
1265 | .statesize = sizeof(struct md5_state), |
1266 | .block_size = MD5_HMAC_BLOCK_SIZE }, |
1267 | { .name = "sha1" , |
1268 | .hash_zero = sha1_zero_message_hash, |
1269 | .hash_init = (u8 *)n2_sha1_init, |
1270 | .auth_type = AUTH_TYPE_SHA1, |
1271 | .hmac_type = AUTH_TYPE_HMAC_SHA1, |
1272 | .hw_op_hashsz = SHA1_DIGEST_SIZE, |
1273 | .digest_size = SHA1_DIGEST_SIZE, |
1274 | .statesize = sizeof(struct sha1_state), |
1275 | .block_size = SHA1_BLOCK_SIZE }, |
1276 | { .name = "sha256" , |
1277 | .hash_zero = sha256_zero_message_hash, |
1278 | .hash_init = (u8 *)n2_sha256_init, |
1279 | .auth_type = AUTH_TYPE_SHA256, |
1280 | .hmac_type = AUTH_TYPE_HMAC_SHA256, |
1281 | .hw_op_hashsz = SHA256_DIGEST_SIZE, |
1282 | .digest_size = SHA256_DIGEST_SIZE, |
1283 | .statesize = sizeof(struct sha256_state), |
1284 | .block_size = SHA256_BLOCK_SIZE }, |
1285 | { .name = "sha224" , |
1286 | .hash_zero = sha224_zero_message_hash, |
1287 | .hash_init = (u8 *)n2_sha224_init, |
1288 | .auth_type = AUTH_TYPE_SHA256, |
1289 | .hmac_type = AUTH_TYPE_RESERVED, |
1290 | .hw_op_hashsz = SHA256_DIGEST_SIZE, |
1291 | .digest_size = SHA224_DIGEST_SIZE, |
1292 | .statesize = sizeof(struct sha256_state), |
1293 | .block_size = SHA224_BLOCK_SIZE }, |
1294 | }; |
1295 | #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls) |
1296 | |
1297 | static LIST_HEAD(ahash_algs); |
1298 | static LIST_HEAD(hmac_algs); |
1299 | |
1300 | static int algs_registered; |
1301 | |
1302 | static void __n2_unregister_algs(void) |
1303 | { |
1304 | struct n2_skcipher_alg *skcipher, *skcipher_tmp; |
1305 | struct n2_ahash_alg *alg, *alg_tmp; |
1306 | struct n2_hmac_alg *hmac, *hmac_tmp; |
1307 | |
1308 | list_for_each_entry_safe(skcipher, skcipher_tmp, &skcipher_algs, entry) { |
1309 | crypto_unregister_skcipher(alg: &skcipher->skcipher); |
1310 | list_del(entry: &skcipher->entry); |
1311 | kfree(objp: skcipher); |
1312 | } |
1313 | list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) { |
1314 | crypto_unregister_ahash(alg: &hmac->derived.alg); |
1315 | list_del(entry: &hmac->derived.entry); |
1316 | kfree(objp: hmac); |
1317 | } |
1318 | list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) { |
1319 | crypto_unregister_ahash(alg: &alg->alg); |
1320 | list_del(entry: &alg->entry); |
1321 | kfree(objp: alg); |
1322 | } |
1323 | } |
1324 | |
1325 | static int n2_skcipher_init_tfm(struct crypto_skcipher *tfm) |
1326 | { |
1327 | crypto_skcipher_set_reqsize(skcipher: tfm, reqsize: sizeof(struct n2_request_context)); |
1328 | return 0; |
1329 | } |
1330 | |
1331 | static int __n2_register_one_skcipher(const struct n2_skcipher_tmpl *tmpl) |
1332 | { |
1333 | struct n2_skcipher_alg *p = kzalloc(size: sizeof(*p), GFP_KERNEL); |
1334 | struct skcipher_alg *alg; |
1335 | int err; |
1336 | |
1337 | if (!p) |
1338 | return -ENOMEM; |
1339 | |
1340 | alg = &p->skcipher; |
1341 | *alg = tmpl->skcipher; |
1342 | |
1343 | snprintf(buf: alg->base.cra_name, CRYPTO_MAX_ALG_NAME, fmt: "%s" , tmpl->name); |
1344 | snprintf(buf: alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, fmt: "%s-n2" , tmpl->drv_name); |
1345 | alg->base.cra_priority = N2_CRA_PRIORITY; |
1346 | alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | |
1347 | CRYPTO_ALG_ALLOCATES_MEMORY; |
1348 | alg->base.cra_blocksize = tmpl->block_size; |
1349 | p->enc_type = tmpl->enc_type; |
1350 | alg->base.cra_ctxsize = sizeof(struct n2_skcipher_context); |
1351 | alg->base.cra_module = THIS_MODULE; |
1352 | alg->init = n2_skcipher_init_tfm; |
1353 | |
1354 | list_add(new: &p->entry, head: &skcipher_algs); |
1355 | err = crypto_register_skcipher(alg); |
1356 | if (err) { |
1357 | pr_err("%s alg registration failed\n" , alg->base.cra_name); |
1358 | list_del(entry: &p->entry); |
1359 | kfree(objp: p); |
1360 | } else { |
1361 | pr_info("%s alg registered\n" , alg->base.cra_name); |
1362 | } |
1363 | return err; |
1364 | } |
1365 | |
1366 | static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash) |
1367 | { |
1368 | struct n2_hmac_alg *p = kzalloc(size: sizeof(*p), GFP_KERNEL); |
1369 | struct ahash_alg *ahash; |
1370 | struct crypto_alg *base; |
1371 | int err; |
1372 | |
1373 | if (!p) |
1374 | return -ENOMEM; |
1375 | |
1376 | p->child_alg = n2ahash->alg.halg.base.cra_name; |
1377 | memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg)); |
1378 | INIT_LIST_HEAD(list: &p->derived.entry); |
1379 | |
1380 | ahash = &p->derived.alg; |
1381 | ahash->digest = n2_hmac_async_digest; |
1382 | ahash->setkey = n2_hmac_async_setkey; |
1383 | |
1384 | base = &ahash->halg.base; |
1385 | snprintf(buf: base->cra_name, CRYPTO_MAX_ALG_NAME, fmt: "hmac(%s)" , p->child_alg); |
1386 | snprintf(buf: base->cra_driver_name, CRYPTO_MAX_ALG_NAME, fmt: "hmac-%s-n2" , p->child_alg); |
1387 | |
1388 | base->cra_ctxsize = sizeof(struct n2_hmac_ctx); |
1389 | base->cra_init = n2_hmac_cra_init; |
1390 | base->cra_exit = n2_hmac_cra_exit; |
1391 | |
1392 | list_add(new: &p->derived.entry, head: &hmac_algs); |
1393 | err = crypto_register_ahash(alg: ahash); |
1394 | if (err) { |
1395 | pr_err("%s alg registration failed\n" , base->cra_name); |
1396 | list_del(entry: &p->derived.entry); |
1397 | kfree(objp: p); |
1398 | } else { |
1399 | pr_info("%s alg registered\n" , base->cra_name); |
1400 | } |
1401 | return err; |
1402 | } |
1403 | |
1404 | static int __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl) |
1405 | { |
1406 | struct n2_ahash_alg *p = kzalloc(size: sizeof(*p), GFP_KERNEL); |
1407 | struct hash_alg_common *halg; |
1408 | struct crypto_alg *base; |
1409 | struct ahash_alg *ahash; |
1410 | int err; |
1411 | |
1412 | if (!p) |
1413 | return -ENOMEM; |
1414 | |
1415 | p->hash_zero = tmpl->hash_zero; |
1416 | p->hash_init = tmpl->hash_init; |
1417 | p->auth_type = tmpl->auth_type; |
1418 | p->hmac_type = tmpl->hmac_type; |
1419 | p->hw_op_hashsz = tmpl->hw_op_hashsz; |
1420 | p->digest_size = tmpl->digest_size; |
1421 | |
1422 | ahash = &p->alg; |
1423 | ahash->init = n2_hash_async_init; |
1424 | ahash->update = n2_hash_async_update; |
1425 | ahash->final = n2_hash_async_final; |
1426 | ahash->finup = n2_hash_async_finup; |
1427 | ahash->digest = n2_hash_async_digest; |
1428 | ahash->export = n2_hash_async_noexport; |
1429 | ahash->import = n2_hash_async_noimport; |
1430 | |
1431 | halg = &ahash->halg; |
1432 | halg->digestsize = tmpl->digest_size; |
1433 | halg->statesize = tmpl->statesize; |
1434 | |
1435 | base = &halg->base; |
1436 | snprintf(buf: base->cra_name, CRYPTO_MAX_ALG_NAME, fmt: "%s" , tmpl->name); |
1437 | snprintf(buf: base->cra_driver_name, CRYPTO_MAX_ALG_NAME, fmt: "%s-n2" , tmpl->name); |
1438 | base->cra_priority = N2_CRA_PRIORITY; |
1439 | base->cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | |
1440 | CRYPTO_ALG_NEED_FALLBACK; |
1441 | base->cra_blocksize = tmpl->block_size; |
1442 | base->cra_ctxsize = sizeof(struct n2_hash_ctx); |
1443 | base->cra_module = THIS_MODULE; |
1444 | base->cra_init = n2_hash_cra_init; |
1445 | base->cra_exit = n2_hash_cra_exit; |
1446 | |
1447 | list_add(new: &p->entry, head: &ahash_algs); |
1448 | err = crypto_register_ahash(alg: ahash); |
1449 | if (err) { |
1450 | pr_err("%s alg registration failed\n" , base->cra_name); |
1451 | list_del(entry: &p->entry); |
1452 | kfree(objp: p); |
1453 | } else { |
1454 | pr_info("%s alg registered\n" , base->cra_name); |
1455 | } |
1456 | if (!err && p->hmac_type != AUTH_TYPE_RESERVED) |
1457 | err = __n2_register_one_hmac(n2ahash: p); |
1458 | return err; |
1459 | } |
1460 | |
1461 | static int n2_register_algs(void) |
1462 | { |
1463 | int i, err = 0; |
1464 | |
1465 | mutex_lock(&spu_lock); |
1466 | if (algs_registered++) |
1467 | goto out; |
1468 | |
1469 | for (i = 0; i < NUM_HASH_TMPLS; i++) { |
1470 | err = __n2_register_one_ahash(tmpl: &hash_tmpls[i]); |
1471 | if (err) { |
1472 | __n2_unregister_algs(); |
1473 | goto out; |
1474 | } |
1475 | } |
1476 | for (i = 0; i < NUM_CIPHER_TMPLS; i++) { |
1477 | err = __n2_register_one_skcipher(tmpl: &skcipher_tmpls[i]); |
1478 | if (err) { |
1479 | __n2_unregister_algs(); |
1480 | goto out; |
1481 | } |
1482 | } |
1483 | |
1484 | out: |
1485 | mutex_unlock(lock: &spu_lock); |
1486 | return err; |
1487 | } |
1488 | |
1489 | static void n2_unregister_algs(void) |
1490 | { |
1491 | mutex_lock(&spu_lock); |
1492 | if (!--algs_registered) |
1493 | __n2_unregister_algs(); |
1494 | mutex_unlock(lock: &spu_lock); |
1495 | } |
1496 | |
1497 | /* To map CWQ queues to interrupt sources, the hypervisor API provides |
1498 | * a devino. This isn't very useful to us because all of the |
1499 | * interrupts listed in the device_node have been translated to |
1500 | * Linux virtual IRQ cookie numbers. |
1501 | * |
1502 | * So we have to back-translate, going through the 'intr' and 'ino' |
1503 | * property tables of the n2cp MDESC node, matching it with the OF |
1504 | * 'interrupts' property entries, in order to figure out which |
1505 | * devino goes to which already-translated IRQ. |
1506 | */ |
1507 | static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip, |
1508 | unsigned long dev_ino) |
1509 | { |
1510 | const unsigned int *dev_intrs; |
1511 | unsigned int intr; |
1512 | int i; |
1513 | |
1514 | for (i = 0; i < ip->num_intrs; i++) { |
1515 | if (ip->ino_table[i].ino == dev_ino) |
1516 | break; |
1517 | } |
1518 | if (i == ip->num_intrs) |
1519 | return -ENODEV; |
1520 | |
1521 | intr = ip->ino_table[i].intr; |
1522 | |
1523 | dev_intrs = of_get_property(node: dev->dev.of_node, name: "interrupts" , NULL); |
1524 | if (!dev_intrs) |
1525 | return -ENODEV; |
1526 | |
1527 | for (i = 0; i < dev->archdata.num_irqs; i++) { |
1528 | if (dev_intrs[i] == intr) |
1529 | return i; |
1530 | } |
1531 | |
1532 | return -ENODEV; |
1533 | } |
1534 | |
1535 | static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip, |
1536 | const char *irq_name, struct spu_queue *p, |
1537 | irq_handler_t handler) |
1538 | { |
1539 | unsigned long herr; |
1540 | int index; |
1541 | |
1542 | herr = sun4v_ncs_qhandle_to_devino(qhandle: p->qhandle, devino: &p->devino); |
1543 | if (herr) |
1544 | return -EINVAL; |
1545 | |
1546 | index = find_devino_index(dev, ip, dev_ino: p->devino); |
1547 | if (index < 0) |
1548 | return index; |
1549 | |
1550 | p->irq = dev->archdata.irqs[index]; |
1551 | |
1552 | sprintf(buf: p->irq_name, fmt: "%s-%d" , irq_name, index); |
1553 | |
1554 | return request_irq(irq: p->irq, handler, flags: 0, name: p->irq_name, dev: p); |
1555 | } |
1556 | |
1557 | static struct kmem_cache *queue_cache[2]; |
1558 | |
1559 | static void *new_queue(unsigned long q_type) |
1560 | { |
1561 | return kmem_cache_zalloc(k: queue_cache[q_type - 1], GFP_KERNEL); |
1562 | } |
1563 | |
1564 | static void free_queue(void *p, unsigned long q_type) |
1565 | { |
1566 | kmem_cache_free(s: queue_cache[q_type - 1], objp: p); |
1567 | } |
1568 | |
1569 | static int queue_cache_init(void) |
1570 | { |
1571 | if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) |
1572 | queue_cache[HV_NCS_QTYPE_MAU - 1] = |
1573 | kmem_cache_create(name: "mau_queue" , |
1574 | size: (MAU_NUM_ENTRIES * |
1575 | MAU_ENTRY_SIZE), |
1576 | MAU_ENTRY_SIZE, flags: 0, NULL); |
1577 | if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) |
1578 | return -ENOMEM; |
1579 | |
1580 | if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) |
1581 | queue_cache[HV_NCS_QTYPE_CWQ - 1] = |
1582 | kmem_cache_create(name: "cwq_queue" , |
1583 | size: (CWQ_NUM_ENTRIES * |
1584 | CWQ_ENTRY_SIZE), |
1585 | CWQ_ENTRY_SIZE, flags: 0, NULL); |
1586 | if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) { |
1587 | kmem_cache_destroy(s: queue_cache[HV_NCS_QTYPE_MAU - 1]); |
1588 | queue_cache[HV_NCS_QTYPE_MAU - 1] = NULL; |
1589 | return -ENOMEM; |
1590 | } |
1591 | return 0; |
1592 | } |
1593 | |
1594 | static void queue_cache_destroy(void) |
1595 | { |
1596 | kmem_cache_destroy(s: queue_cache[HV_NCS_QTYPE_MAU - 1]); |
1597 | kmem_cache_destroy(s: queue_cache[HV_NCS_QTYPE_CWQ - 1]); |
1598 | queue_cache[HV_NCS_QTYPE_MAU - 1] = NULL; |
1599 | queue_cache[HV_NCS_QTYPE_CWQ - 1] = NULL; |
1600 | } |
1601 | |
1602 | static long spu_queue_register_workfn(void *arg) |
1603 | { |
1604 | struct spu_qreg *qr = arg; |
1605 | struct spu_queue *p = qr->queue; |
1606 | unsigned long q_type = qr->type; |
1607 | unsigned long hv_ret; |
1608 | |
1609 | hv_ret = sun4v_ncs_qconf(queue_type: q_type, __pa(p->q), |
1610 | CWQ_NUM_ENTRIES, qhandle: &p->qhandle); |
1611 | if (!hv_ret) |
1612 | sun4v_ncs_sethead_marker(qhandle: p->qhandle, head: 0); |
1613 | |
1614 | return hv_ret ? -EINVAL : 0; |
1615 | } |
1616 | |
1617 | static int spu_queue_register(struct spu_queue *p, unsigned long q_type) |
1618 | { |
1619 | int cpu = cpumask_any_and(&p->sharing, cpu_online_mask); |
1620 | struct spu_qreg qr = { .queue = p, .type = q_type }; |
1621 | |
1622 | return work_on_cpu_safe(cpu, spu_queue_register_workfn, &qr); |
1623 | } |
1624 | |
1625 | static int spu_queue_setup(struct spu_queue *p) |
1626 | { |
1627 | int err; |
1628 | |
1629 | p->q = new_queue(q_type: p->q_type); |
1630 | if (!p->q) |
1631 | return -ENOMEM; |
1632 | |
1633 | err = spu_queue_register(p, q_type: p->q_type); |
1634 | if (err) { |
1635 | free_queue(p: p->q, q_type: p->q_type); |
1636 | p->q = NULL; |
1637 | } |
1638 | |
1639 | return err; |
1640 | } |
1641 | |
1642 | static void spu_queue_destroy(struct spu_queue *p) |
1643 | { |
1644 | unsigned long hv_ret; |
1645 | |
1646 | if (!p->q) |
1647 | return; |
1648 | |
1649 | hv_ret = sun4v_ncs_qconf(queue_type: p->q_type, queue_ra: p->qhandle, num_entries: 0, qhandle: &p->qhandle); |
1650 | |
1651 | if (!hv_ret) |
1652 | free_queue(p: p->q, q_type: p->q_type); |
1653 | } |
1654 | |
1655 | static void spu_list_destroy(struct list_head *list) |
1656 | { |
1657 | struct spu_queue *p, *n; |
1658 | |
1659 | list_for_each_entry_safe(p, n, list, list) { |
1660 | int i; |
1661 | |
1662 | for (i = 0; i < NR_CPUS; i++) { |
1663 | if (cpu_to_cwq[i] == p) |
1664 | cpu_to_cwq[i] = NULL; |
1665 | } |
1666 | |
1667 | if (p->irq) { |
1668 | free_irq(p->irq, p); |
1669 | p->irq = 0; |
1670 | } |
1671 | spu_queue_destroy(p); |
1672 | list_del(entry: &p->list); |
1673 | kfree(objp: p); |
1674 | } |
1675 | } |
1676 | |
1677 | /* Walk the backward arcs of a CWQ 'exec-unit' node, |
1678 | * gathering cpu membership information. |
1679 | */ |
1680 | static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc, |
1681 | struct platform_device *dev, |
1682 | u64 node, struct spu_queue *p, |
1683 | struct spu_queue **table) |
1684 | { |
1685 | u64 arc; |
1686 | |
1687 | mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) { |
1688 | u64 tgt = mdesc_arc_target(mdesc, arc); |
1689 | const char *name = mdesc_node_name(mdesc, tgt); |
1690 | const u64 *id; |
1691 | |
1692 | if (strcmp(name, "cpu" )) |
1693 | continue; |
1694 | id = mdesc_get_property(mdesc, tgt, "id" , NULL); |
1695 | if (table[*id] != NULL) { |
1696 | dev_err(&dev->dev, "%pOF: SPU cpu slot already set.\n" , |
1697 | dev->dev.of_node); |
1698 | return -EINVAL; |
1699 | } |
1700 | cpumask_set_cpu(cpu: *id, dstp: &p->sharing); |
1701 | table[*id] = p; |
1702 | } |
1703 | return 0; |
1704 | } |
1705 | |
1706 | /* Process an 'exec-unit' MDESC node of type 'cwq'. */ |
1707 | static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list, |
1708 | struct platform_device *dev, struct mdesc_handle *mdesc, |
1709 | u64 node, const char *iname, unsigned long q_type, |
1710 | irq_handler_t handler, struct spu_queue **table) |
1711 | { |
1712 | struct spu_queue *p; |
1713 | int err; |
1714 | |
1715 | p = kzalloc(size: sizeof(struct spu_queue), GFP_KERNEL); |
1716 | if (!p) { |
1717 | dev_err(&dev->dev, "%pOF: Could not allocate SPU queue.\n" , |
1718 | dev->dev.of_node); |
1719 | return -ENOMEM; |
1720 | } |
1721 | |
1722 | cpumask_clear(dstp: &p->sharing); |
1723 | spin_lock_init(&p->lock); |
1724 | p->q_type = q_type; |
1725 | INIT_LIST_HEAD(list: &p->jobs); |
1726 | list_add(new: &p->list, head: list); |
1727 | |
1728 | err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table); |
1729 | if (err) |
1730 | return err; |
1731 | |
1732 | err = spu_queue_setup(p); |
1733 | if (err) |
1734 | return err; |
1735 | |
1736 | return spu_map_ino(dev, ip, irq_name: iname, p, handler); |
1737 | } |
1738 | |
1739 | static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev, |
1740 | struct spu_mdesc_info *ip, struct list_head *list, |
1741 | const char *exec_name, unsigned long q_type, |
1742 | irq_handler_t handler, struct spu_queue **table) |
1743 | { |
1744 | int err = 0; |
1745 | u64 node; |
1746 | |
1747 | mdesc_for_each_node_by_name(mdesc, node, "exec-unit" ) { |
1748 | const char *type; |
1749 | |
1750 | type = mdesc_get_property(mdesc, node, "type" , NULL); |
1751 | if (!type || strcmp(type, exec_name)) |
1752 | continue; |
1753 | |
1754 | err = handle_exec_unit(ip, list, dev, mdesc, node, |
1755 | iname: exec_name, q_type, handler, table); |
1756 | if (err) { |
1757 | spu_list_destroy(list); |
1758 | break; |
1759 | } |
1760 | } |
1761 | |
1762 | return err; |
1763 | } |
1764 | |
1765 | static int get_irq_props(struct mdesc_handle *mdesc, u64 node, |
1766 | struct spu_mdesc_info *ip) |
1767 | { |
1768 | const u64 *ino; |
1769 | int ino_len; |
1770 | int i; |
1771 | |
1772 | ino = mdesc_get_property(mdesc, node, "ino" , &ino_len); |
1773 | if (!ino) { |
1774 | printk("NO 'ino'\n" ); |
1775 | return -ENODEV; |
1776 | } |
1777 | |
1778 | ip->num_intrs = ino_len / sizeof(u64); |
1779 | ip->ino_table = kzalloc(size: (sizeof(struct ino_blob) * |
1780 | ip->num_intrs), |
1781 | GFP_KERNEL); |
1782 | if (!ip->ino_table) |
1783 | return -ENOMEM; |
1784 | |
1785 | for (i = 0; i < ip->num_intrs; i++) { |
1786 | struct ino_blob *b = &ip->ino_table[i]; |
1787 | b->intr = i + 1; |
1788 | b->ino = ino[i]; |
1789 | } |
1790 | |
1791 | return 0; |
1792 | } |
1793 | |
1794 | static int grab_mdesc_irq_props(struct mdesc_handle *mdesc, |
1795 | struct platform_device *dev, |
1796 | struct spu_mdesc_info *ip, |
1797 | const char *node_name) |
1798 | { |
1799 | u64 node, reg; |
1800 | |
1801 | if (of_property_read_reg(np: dev->dev.of_node, idx: 0, addr: ®, NULL) < 0) |
1802 | return -ENODEV; |
1803 | |
1804 | mdesc_for_each_node_by_name(mdesc, node, "virtual-device" ) { |
1805 | const char *name; |
1806 | const u64 *chdl; |
1807 | |
1808 | name = mdesc_get_property(mdesc, node, "name" , NULL); |
1809 | if (!name || strcmp(name, node_name)) |
1810 | continue; |
1811 | chdl = mdesc_get_property(mdesc, node, "cfg-handle" , NULL); |
1812 | if (!chdl || (*chdl != reg)) |
1813 | continue; |
1814 | ip->cfg_handle = *chdl; |
1815 | return get_irq_props(mdesc, node, ip); |
1816 | } |
1817 | |
1818 | return -ENODEV; |
1819 | } |
1820 | |
1821 | static unsigned long n2_spu_hvapi_major; |
1822 | static unsigned long n2_spu_hvapi_minor; |
1823 | |
1824 | static int n2_spu_hvapi_register(void) |
1825 | { |
1826 | int err; |
1827 | |
1828 | n2_spu_hvapi_major = 2; |
1829 | n2_spu_hvapi_minor = 0; |
1830 | |
1831 | err = sun4v_hvapi_register(HV_GRP_NCS, |
1832 | n2_spu_hvapi_major, |
1833 | &n2_spu_hvapi_minor); |
1834 | |
1835 | if (!err) |
1836 | pr_info("Registered NCS HVAPI version %lu.%lu\n" , |
1837 | n2_spu_hvapi_major, |
1838 | n2_spu_hvapi_minor); |
1839 | |
1840 | return err; |
1841 | } |
1842 | |
1843 | static void n2_spu_hvapi_unregister(void) |
1844 | { |
1845 | sun4v_hvapi_unregister(HV_GRP_NCS); |
1846 | } |
1847 | |
1848 | static int global_ref; |
1849 | |
1850 | static int grab_global_resources(void) |
1851 | { |
1852 | int err = 0; |
1853 | |
1854 | mutex_lock(&spu_lock); |
1855 | |
1856 | if (global_ref++) |
1857 | goto out; |
1858 | |
1859 | err = n2_spu_hvapi_register(); |
1860 | if (err) |
1861 | goto out; |
1862 | |
1863 | err = queue_cache_init(); |
1864 | if (err) |
1865 | goto out_hvapi_release; |
1866 | |
1867 | err = -ENOMEM; |
1868 | cpu_to_cwq = kcalloc(NR_CPUS, size: sizeof(struct spu_queue *), |
1869 | GFP_KERNEL); |
1870 | if (!cpu_to_cwq) |
1871 | goto out_queue_cache_destroy; |
1872 | |
1873 | cpu_to_mau = kcalloc(NR_CPUS, size: sizeof(struct spu_queue *), |
1874 | GFP_KERNEL); |
1875 | if (!cpu_to_mau) |
1876 | goto out_free_cwq_table; |
1877 | |
1878 | err = 0; |
1879 | |
1880 | out: |
1881 | if (err) |
1882 | global_ref--; |
1883 | mutex_unlock(lock: &spu_lock); |
1884 | return err; |
1885 | |
1886 | out_free_cwq_table: |
1887 | kfree(objp: cpu_to_cwq); |
1888 | cpu_to_cwq = NULL; |
1889 | |
1890 | out_queue_cache_destroy: |
1891 | queue_cache_destroy(); |
1892 | |
1893 | out_hvapi_release: |
1894 | n2_spu_hvapi_unregister(); |
1895 | goto out; |
1896 | } |
1897 | |
1898 | static void release_global_resources(void) |
1899 | { |
1900 | mutex_lock(&spu_lock); |
1901 | if (!--global_ref) { |
1902 | kfree(objp: cpu_to_cwq); |
1903 | cpu_to_cwq = NULL; |
1904 | |
1905 | kfree(objp: cpu_to_mau); |
1906 | cpu_to_mau = NULL; |
1907 | |
1908 | queue_cache_destroy(); |
1909 | n2_spu_hvapi_unregister(); |
1910 | } |
1911 | mutex_unlock(lock: &spu_lock); |
1912 | } |
1913 | |
1914 | static struct n2_crypto *alloc_n2cp(void) |
1915 | { |
1916 | struct n2_crypto *np = kzalloc(size: sizeof(struct n2_crypto), GFP_KERNEL); |
1917 | |
1918 | if (np) |
1919 | INIT_LIST_HEAD(list: &np->cwq_list); |
1920 | |
1921 | return np; |
1922 | } |
1923 | |
1924 | static void free_n2cp(struct n2_crypto *np) |
1925 | { |
1926 | kfree(objp: np->cwq_info.ino_table); |
1927 | np->cwq_info.ino_table = NULL; |
1928 | |
1929 | kfree(objp: np); |
1930 | } |
1931 | |
1932 | static void n2_spu_driver_version(void) |
1933 | { |
1934 | static int n2_spu_version_printed; |
1935 | |
1936 | if (n2_spu_version_printed++ == 0) |
1937 | pr_info("%s" , version); |
1938 | } |
1939 | |
1940 | static int n2_crypto_probe(struct platform_device *dev) |
1941 | { |
1942 | struct mdesc_handle *mdesc; |
1943 | struct n2_crypto *np; |
1944 | int err; |
1945 | |
1946 | n2_spu_driver_version(); |
1947 | |
1948 | pr_info("Found N2CP at %pOF\n" , dev->dev.of_node); |
1949 | |
1950 | np = alloc_n2cp(); |
1951 | if (!np) { |
1952 | dev_err(&dev->dev, "%pOF: Unable to allocate n2cp.\n" , |
1953 | dev->dev.of_node); |
1954 | return -ENOMEM; |
1955 | } |
1956 | |
1957 | err = grab_global_resources(); |
1958 | if (err) { |
1959 | dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n" , |
1960 | dev->dev.of_node); |
1961 | goto out_free_n2cp; |
1962 | } |
1963 | |
1964 | mdesc = mdesc_grab(); |
1965 | |
1966 | if (!mdesc) { |
1967 | dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n" , |
1968 | dev->dev.of_node); |
1969 | err = -ENODEV; |
1970 | goto out_free_global; |
1971 | } |
1972 | err = grab_mdesc_irq_props(mdesc, dev, ip: &np->cwq_info, node_name: "n2cp" ); |
1973 | if (err) { |
1974 | dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n" , |
1975 | dev->dev.of_node); |
1976 | mdesc_release(mdesc); |
1977 | goto out_free_global; |
1978 | } |
1979 | |
1980 | err = spu_mdesc_scan(mdesc, dev, ip: &np->cwq_info, list: &np->cwq_list, |
1981 | exec_name: "cwq" , HV_NCS_QTYPE_CWQ, handler: cwq_intr, |
1982 | table: cpu_to_cwq); |
1983 | mdesc_release(mdesc); |
1984 | |
1985 | if (err) { |
1986 | dev_err(&dev->dev, "%pOF: CWQ MDESC scan failed.\n" , |
1987 | dev->dev.of_node); |
1988 | goto out_free_global; |
1989 | } |
1990 | |
1991 | err = n2_register_algs(); |
1992 | if (err) { |
1993 | dev_err(&dev->dev, "%pOF: Unable to register algorithms.\n" , |
1994 | dev->dev.of_node); |
1995 | goto out_free_spu_list; |
1996 | } |
1997 | |
1998 | dev_set_drvdata(dev: &dev->dev, data: np); |
1999 | |
2000 | return 0; |
2001 | |
2002 | out_free_spu_list: |
2003 | spu_list_destroy(list: &np->cwq_list); |
2004 | |
2005 | out_free_global: |
2006 | release_global_resources(); |
2007 | |
2008 | out_free_n2cp: |
2009 | free_n2cp(np); |
2010 | |
2011 | return err; |
2012 | } |
2013 | |
2014 | static void n2_crypto_remove(struct platform_device *dev) |
2015 | { |
2016 | struct n2_crypto *np = dev_get_drvdata(dev: &dev->dev); |
2017 | |
2018 | n2_unregister_algs(); |
2019 | |
2020 | spu_list_destroy(list: &np->cwq_list); |
2021 | |
2022 | release_global_resources(); |
2023 | |
2024 | free_n2cp(np); |
2025 | } |
2026 | |
2027 | static struct n2_mau *alloc_ncp(void) |
2028 | { |
2029 | struct n2_mau *mp = kzalloc(size: sizeof(struct n2_mau), GFP_KERNEL); |
2030 | |
2031 | if (mp) |
2032 | INIT_LIST_HEAD(list: &mp->mau_list); |
2033 | |
2034 | return mp; |
2035 | } |
2036 | |
2037 | static void free_ncp(struct n2_mau *mp) |
2038 | { |
2039 | kfree(objp: mp->mau_info.ino_table); |
2040 | mp->mau_info.ino_table = NULL; |
2041 | |
2042 | kfree(objp: mp); |
2043 | } |
2044 | |
2045 | static int n2_mau_probe(struct platform_device *dev) |
2046 | { |
2047 | struct mdesc_handle *mdesc; |
2048 | struct n2_mau *mp; |
2049 | int err; |
2050 | |
2051 | n2_spu_driver_version(); |
2052 | |
2053 | pr_info("Found NCP at %pOF\n" , dev->dev.of_node); |
2054 | |
2055 | mp = alloc_ncp(); |
2056 | if (!mp) { |
2057 | dev_err(&dev->dev, "%pOF: Unable to allocate ncp.\n" , |
2058 | dev->dev.of_node); |
2059 | return -ENOMEM; |
2060 | } |
2061 | |
2062 | err = grab_global_resources(); |
2063 | if (err) { |
2064 | dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n" , |
2065 | dev->dev.of_node); |
2066 | goto out_free_ncp; |
2067 | } |
2068 | |
2069 | mdesc = mdesc_grab(); |
2070 | |
2071 | if (!mdesc) { |
2072 | dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n" , |
2073 | dev->dev.of_node); |
2074 | err = -ENODEV; |
2075 | goto out_free_global; |
2076 | } |
2077 | |
2078 | err = grab_mdesc_irq_props(mdesc, dev, ip: &mp->mau_info, node_name: "ncp" ); |
2079 | if (err) { |
2080 | dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n" , |
2081 | dev->dev.of_node); |
2082 | mdesc_release(mdesc); |
2083 | goto out_free_global; |
2084 | } |
2085 | |
2086 | err = spu_mdesc_scan(mdesc, dev, ip: &mp->mau_info, list: &mp->mau_list, |
2087 | exec_name: "mau" , HV_NCS_QTYPE_MAU, handler: mau_intr, |
2088 | table: cpu_to_mau); |
2089 | mdesc_release(mdesc); |
2090 | |
2091 | if (err) { |
2092 | dev_err(&dev->dev, "%pOF: MAU MDESC scan failed.\n" , |
2093 | dev->dev.of_node); |
2094 | goto out_free_global; |
2095 | } |
2096 | |
2097 | dev_set_drvdata(dev: &dev->dev, data: mp); |
2098 | |
2099 | return 0; |
2100 | |
2101 | out_free_global: |
2102 | release_global_resources(); |
2103 | |
2104 | out_free_ncp: |
2105 | free_ncp(mp); |
2106 | |
2107 | return err; |
2108 | } |
2109 | |
2110 | static void n2_mau_remove(struct platform_device *dev) |
2111 | { |
2112 | struct n2_mau *mp = dev_get_drvdata(dev: &dev->dev); |
2113 | |
2114 | spu_list_destroy(list: &mp->mau_list); |
2115 | |
2116 | release_global_resources(); |
2117 | |
2118 | free_ncp(mp); |
2119 | } |
2120 | |
2121 | static const struct of_device_id n2_crypto_match[] = { |
2122 | { |
2123 | .name = "n2cp" , |
2124 | .compatible = "SUNW,n2-cwq" , |
2125 | }, |
2126 | { |
2127 | .name = "n2cp" , |
2128 | .compatible = "SUNW,vf-cwq" , |
2129 | }, |
2130 | { |
2131 | .name = "n2cp" , |
2132 | .compatible = "SUNW,kt-cwq" , |
2133 | }, |
2134 | {}, |
2135 | }; |
2136 | |
2137 | MODULE_DEVICE_TABLE(of, n2_crypto_match); |
2138 | |
2139 | static struct platform_driver n2_crypto_driver = { |
2140 | .driver = { |
2141 | .name = "n2cp" , |
2142 | .of_match_table = n2_crypto_match, |
2143 | }, |
2144 | .probe = n2_crypto_probe, |
2145 | .remove_new = n2_crypto_remove, |
2146 | }; |
2147 | |
2148 | static const struct of_device_id n2_mau_match[] = { |
2149 | { |
2150 | .name = "ncp" , |
2151 | .compatible = "SUNW,n2-mau" , |
2152 | }, |
2153 | { |
2154 | .name = "ncp" , |
2155 | .compatible = "SUNW,vf-mau" , |
2156 | }, |
2157 | { |
2158 | .name = "ncp" , |
2159 | .compatible = "SUNW,kt-mau" , |
2160 | }, |
2161 | {}, |
2162 | }; |
2163 | |
2164 | MODULE_DEVICE_TABLE(of, n2_mau_match); |
2165 | |
2166 | static struct platform_driver n2_mau_driver = { |
2167 | .driver = { |
2168 | .name = "ncp" , |
2169 | .of_match_table = n2_mau_match, |
2170 | }, |
2171 | .probe = n2_mau_probe, |
2172 | .remove_new = n2_mau_remove, |
2173 | }; |
2174 | |
2175 | static struct platform_driver * const drivers[] = { |
2176 | &n2_crypto_driver, |
2177 | &n2_mau_driver, |
2178 | }; |
2179 | |
2180 | static int __init n2_init(void) |
2181 | { |
2182 | return platform_register_drivers(drivers, ARRAY_SIZE(drivers)); |
2183 | } |
2184 | |
2185 | static void __exit n2_exit(void) |
2186 | { |
2187 | platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); |
2188 | } |
2189 | |
2190 | module_init(n2_init); |
2191 | module_exit(n2_exit); |
2192 | |