1/*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
5 *
6 * Author: Gary R Hook <gary.hook@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/module.h>
14#include <linux/kernel.h>
15#include <linux/dmaengine.h>
16#include <linux/spinlock.h>
17#include <linux/mutex.h>
18#include <linux/ccp.h>
19
20#include "ccp-dev.h"
21#include "../../dma/dmaengine.h"
22
23#define CCP_DMA_WIDTH(_mask) \
24({ \
25 u64 mask = _mask + 1; \
26 (mask == 0) ? 64 : fls64(mask); \
27})
28
29/* The CCP as a DMA provider can be configured for public or private
30 * channels. Default is specified in the vdata for the device (PCI ID).
31 * This module parameter will override for all channels on all devices:
32 * dma_chan_attr = 0x2 to force all channels public
33 * = 0x1 to force all channels private
34 * = 0x0 to defer to the vdata setting
35 * = any other value: warning, revert to 0x0
36 */
37static unsigned int dma_chan_attr = CCP_DMA_DFLT;
38module_param(dma_chan_attr, uint, 0444);
39MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public");
40
41static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp)
42{
43 switch (dma_chan_attr) {
44 case CCP_DMA_DFLT:
45 return ccp->vdata->dma_chan_attr;
46
47 case CCP_DMA_PRIV:
48 return DMA_PRIVATE;
49
50 case CCP_DMA_PUB:
51 return 0;
52
53 default:
54 dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n",
55 dma_chan_attr);
56 return ccp->vdata->dma_chan_attr;
57 }
58}
59
60static void ccp_free_cmd_resources(struct ccp_device *ccp,
61 struct list_head *list)
62{
63 struct ccp_dma_cmd *cmd, *ctmp;
64
65 list_for_each_entry_safe(cmd, ctmp, list, entry) {
66 list_del(&cmd->entry);
67 kmem_cache_free(ccp->dma_cmd_cache, cmd);
68 }
69}
70
71static void ccp_free_desc_resources(struct ccp_device *ccp,
72 struct list_head *list)
73{
74 struct ccp_dma_desc *desc, *dtmp;
75
76 list_for_each_entry_safe(desc, dtmp, list, entry) {
77 ccp_free_cmd_resources(ccp, &desc->active);
78 ccp_free_cmd_resources(ccp, &desc->pending);
79
80 list_del(&desc->entry);
81 kmem_cache_free(ccp->dma_desc_cache, desc);
82 }
83}
84
85static void ccp_free_chan_resources(struct dma_chan *dma_chan)
86{
87 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
88 dma_chan);
89 unsigned long flags;
90
91 dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan);
92
93 spin_lock_irqsave(&chan->lock, flags);
94
95 ccp_free_desc_resources(chan->ccp, &chan->complete);
96 ccp_free_desc_resources(chan->ccp, &chan->active);
97 ccp_free_desc_resources(chan->ccp, &chan->pending);
98 ccp_free_desc_resources(chan->ccp, &chan->created);
99
100 spin_unlock_irqrestore(&chan->lock, flags);
101}
102
103static void ccp_cleanup_desc_resources(struct ccp_device *ccp,
104 struct list_head *list)
105{
106 struct ccp_dma_desc *desc, *dtmp;
107
108 list_for_each_entry_safe_reverse(desc, dtmp, list, entry) {
109 if (!async_tx_test_ack(&desc->tx_desc))
110 continue;
111
112 dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);
113
114 ccp_free_cmd_resources(ccp, &desc->active);
115 ccp_free_cmd_resources(ccp, &desc->pending);
116
117 list_del(&desc->entry);
118 kmem_cache_free(ccp->dma_desc_cache, desc);
119 }
120}
121
122static void ccp_do_cleanup(unsigned long data)
123{
124 struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data;
125 unsigned long flags;
126
127 dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__,
128 dma_chan_name(&chan->dma_chan));
129
130 spin_lock_irqsave(&chan->lock, flags);
131
132 ccp_cleanup_desc_resources(chan->ccp, &chan->complete);
133
134 spin_unlock_irqrestore(&chan->lock, flags);
135}
136
137static int ccp_issue_next_cmd(struct ccp_dma_desc *desc)
138{
139 struct ccp_dma_cmd *cmd;
140 int ret;
141
142 cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry);
143 list_move(&cmd->entry, &desc->active);
144
145 dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__,
146 desc->tx_desc.cookie, cmd);
147
148 ret = ccp_enqueue_cmd(&cmd->ccp_cmd);
149 if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY))
150 return 0;
151
152 dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__,
153 ret, desc->tx_desc.cookie, cmd);
154
155 return ret;
156}
157
158static void ccp_free_active_cmd(struct ccp_dma_desc *desc)
159{
160 struct ccp_dma_cmd *cmd;
161
162 cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd,
163 entry);
164 if (!cmd)
165 return;
166
167 dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n",
168 __func__, desc->tx_desc.cookie, cmd);
169
170 list_del(&cmd->entry);
171 kmem_cache_free(desc->ccp->dma_cmd_cache, cmd);
172}
173
174static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan,
175 struct ccp_dma_desc *desc)
176{
177 /* Move current DMA descriptor to the complete list */
178 if (desc)
179 list_move(&desc->entry, &chan->complete);
180
181 /* Get the next DMA descriptor on the active list */
182 desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,
183 entry);
184
185 return desc;
186}
187
188static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan,
189 struct ccp_dma_desc *desc)
190{
191 struct dma_async_tx_descriptor *tx_desc;
192 unsigned long flags;
193
194 /* Loop over descriptors until one is found with commands */
195 do {
196 if (desc) {
197 /* Remove the DMA command from the list and free it */
198 ccp_free_active_cmd(desc);
199
200 if (!list_empty(&desc->pending)) {
201 /* No errors, keep going */
202 if (desc->status != DMA_ERROR)
203 return desc;
204
205 /* Error, free remaining commands and move on */
206 ccp_free_cmd_resources(desc->ccp,
207 &desc->pending);
208 }
209
210 tx_desc = &desc->tx_desc;
211 } else {
212 tx_desc = NULL;
213 }
214
215 spin_lock_irqsave(&chan->lock, flags);
216
217 if (desc) {
218 if (desc->status != DMA_ERROR)
219 desc->status = DMA_COMPLETE;
220
221 dev_dbg(desc->ccp->dev,
222 "%s - tx %d complete, status=%u\n", __func__,
223 desc->tx_desc.cookie, desc->status);
224
225 dma_cookie_complete(tx_desc);
226 dma_descriptor_unmap(tx_desc);
227 }
228
229 desc = __ccp_next_dma_desc(chan, desc);
230
231 spin_unlock_irqrestore(&chan->lock, flags);
232
233 if (tx_desc) {
234 dmaengine_desc_get_callback_invoke(tx_desc, NULL);
235
236 dma_run_dependencies(tx_desc);
237 }
238 } while (desc);
239
240 return NULL;
241}
242
243static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan)
244{
245 struct ccp_dma_desc *desc;
246
247 if (list_empty(&chan->pending))
248 return NULL;
249
250 desc = list_empty(&chan->active)
251 ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry)
252 : NULL;
253
254 list_splice_tail_init(&chan->pending, &chan->active);
255
256 return desc;
257}
258
259static void ccp_cmd_callback(void *data, int err)
260{
261 struct ccp_dma_desc *desc = data;
262 struct ccp_dma_chan *chan;
263 int ret;
264
265 if (err == -EINPROGRESS)
266 return;
267
268 chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan,
269 dma_chan);
270
271 dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n",
272 __func__, desc->tx_desc.cookie, err);
273
274 if (err)
275 desc->status = DMA_ERROR;
276
277 while (true) {
278 /* Check for DMA descriptor completion */
279 desc = ccp_handle_active_desc(chan, desc);
280
281 /* Don't submit cmd if no descriptor or DMA is paused */
282 if (!desc || (chan->status == DMA_PAUSED))
283 break;
284
285 ret = ccp_issue_next_cmd(desc);
286 if (!ret)
287 break;
288
289 desc->status = DMA_ERROR;
290 }
291
292 tasklet_schedule(&chan->cleanup_tasklet);
293}
294
295static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc)
296{
297 struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc,
298 tx_desc);
299 struct ccp_dma_chan *chan;
300 dma_cookie_t cookie;
301 unsigned long flags;
302
303 chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan);
304
305 spin_lock_irqsave(&chan->lock, flags);
306
307 cookie = dma_cookie_assign(tx_desc);
308 list_del(&desc->entry);
309 list_add_tail(&desc->entry, &chan->pending);
310
311 spin_unlock_irqrestore(&chan->lock, flags);
312
313 dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n",
314 __func__, cookie);
315
316 return cookie;
317}
318
319static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan)
320{
321 struct ccp_dma_cmd *cmd;
322
323 cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT);
324 if (cmd)
325 memset(cmd, 0, sizeof(*cmd));
326
327 return cmd;
328}
329
330static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan,
331 unsigned long flags)
332{
333 struct ccp_dma_desc *desc;
334
335 desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT);
336 if (!desc)
337 return NULL;
338
339 dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan);
340 desc->tx_desc.flags = flags;
341 desc->tx_desc.tx_submit = ccp_tx_submit;
342 desc->ccp = chan->ccp;
343 INIT_LIST_HEAD(&desc->pending);
344 INIT_LIST_HEAD(&desc->active);
345 desc->status = DMA_IN_PROGRESS;
346
347 return desc;
348}
349
350static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan,
351 struct scatterlist *dst_sg,
352 unsigned int dst_nents,
353 struct scatterlist *src_sg,
354 unsigned int src_nents,
355 unsigned long flags)
356{
357 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
358 dma_chan);
359 struct ccp_device *ccp = chan->ccp;
360 struct ccp_dma_desc *desc;
361 struct ccp_dma_cmd *cmd;
362 struct ccp_cmd *ccp_cmd;
363 struct ccp_passthru_nomap_engine *ccp_pt;
364 unsigned int src_offset, src_len;
365 unsigned int dst_offset, dst_len;
366 unsigned int len;
367 unsigned long sflags;
368 size_t total_len;
369
370 if (!dst_sg || !src_sg)
371 return NULL;
372
373 if (!dst_nents || !src_nents)
374 return NULL;
375
376 desc = ccp_alloc_dma_desc(chan, flags);
377 if (!desc)
378 return NULL;
379
380 total_len = 0;
381
382 src_len = sg_dma_len(src_sg);
383 src_offset = 0;
384
385 dst_len = sg_dma_len(dst_sg);
386 dst_offset = 0;
387
388 while (true) {
389 if (!src_len) {
390 src_nents--;
391 if (!src_nents)
392 break;
393
394 src_sg = sg_next(src_sg);
395 if (!src_sg)
396 break;
397
398 src_len = sg_dma_len(src_sg);
399 src_offset = 0;
400 continue;
401 }
402
403 if (!dst_len) {
404 dst_nents--;
405 if (!dst_nents)
406 break;
407
408 dst_sg = sg_next(dst_sg);
409 if (!dst_sg)
410 break;
411
412 dst_len = sg_dma_len(dst_sg);
413 dst_offset = 0;
414 continue;
415 }
416
417 len = min(dst_len, src_len);
418
419 cmd = ccp_alloc_dma_cmd(chan);
420 if (!cmd)
421 goto err;
422
423 ccp_cmd = &cmd->ccp_cmd;
424 ccp_cmd->ccp = chan->ccp;
425 ccp_pt = &ccp_cmd->u.passthru_nomap;
426 ccp_cmd->flags = CCP_CMD_MAY_BACKLOG;
427 ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP;
428 ccp_cmd->engine = CCP_ENGINE_PASSTHRU;
429 ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
430 ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
431 ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset;
432 ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset;
433 ccp_pt->src_len = len;
434 ccp_pt->final = 1;
435 ccp_cmd->callback = ccp_cmd_callback;
436 ccp_cmd->data = desc;
437
438 list_add_tail(&cmd->entry, &desc->pending);
439
440 dev_dbg(ccp->dev,
441 "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__,
442 cmd, &ccp_pt->src_dma,
443 &ccp_pt->dst_dma, ccp_pt->src_len);
444
445 total_len += len;
446
447 src_len -= len;
448 src_offset += len;
449
450 dst_len -= len;
451 dst_offset += len;
452 }
453
454 desc->len = total_len;
455
456 if (list_empty(&desc->pending))
457 goto err;
458
459 dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);
460
461 spin_lock_irqsave(&chan->lock, sflags);
462
463 list_add_tail(&desc->entry, &chan->created);
464
465 spin_unlock_irqrestore(&chan->lock, sflags);
466
467 return desc;
468
469err:
470 ccp_free_cmd_resources(ccp, &desc->pending);
471 kmem_cache_free(ccp->dma_desc_cache, desc);
472
473 return NULL;
474}
475
476static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy(
477 struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len,
478 unsigned long flags)
479{
480 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
481 dma_chan);
482 struct ccp_dma_desc *desc;
483 struct scatterlist dst_sg, src_sg;
484
485 dev_dbg(chan->ccp->dev,
486 "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n",
487 __func__, &src, &dst, len, flags);
488
489 sg_init_table(&dst_sg, 1);
490 sg_dma_address(&dst_sg) = dst;
491 sg_dma_len(&dst_sg) = len;
492
493 sg_init_table(&src_sg, 1);
494 sg_dma_address(&src_sg) = src;
495 sg_dma_len(&src_sg) = len;
496
497 desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags);
498 if (!desc)
499 return NULL;
500
501 return &desc->tx_desc;
502}
503
504static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt(
505 struct dma_chan *dma_chan, unsigned long flags)
506{
507 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
508 dma_chan);
509 struct ccp_dma_desc *desc;
510
511 desc = ccp_alloc_dma_desc(chan, flags);
512 if (!desc)
513 return NULL;
514
515 return &desc->tx_desc;
516}
517
518static void ccp_issue_pending(struct dma_chan *dma_chan)
519{
520 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
521 dma_chan);
522 struct ccp_dma_desc *desc;
523 unsigned long flags;
524
525 dev_dbg(chan->ccp->dev, "%s\n", __func__);
526
527 spin_lock_irqsave(&chan->lock, flags);
528
529 desc = __ccp_pending_to_active(chan);
530
531 spin_unlock_irqrestore(&chan->lock, flags);
532
533 /* If there was nothing active, start processing */
534 if (desc)
535 ccp_cmd_callback(desc, 0);
536}
537
538static enum dma_status ccp_tx_status(struct dma_chan *dma_chan,
539 dma_cookie_t cookie,
540 struct dma_tx_state *state)
541{
542 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
543 dma_chan);
544 struct ccp_dma_desc *desc;
545 enum dma_status ret;
546 unsigned long flags;
547
548 if (chan->status == DMA_PAUSED) {
549 ret = DMA_PAUSED;
550 goto out;
551 }
552
553 ret = dma_cookie_status(dma_chan, cookie, state);
554 if (ret == DMA_COMPLETE) {
555 spin_lock_irqsave(&chan->lock, flags);
556
557 /* Get status from complete chain, if still there */
558 list_for_each_entry(desc, &chan->complete, entry) {
559 if (desc->tx_desc.cookie != cookie)
560 continue;
561
562 ret = desc->status;
563 break;
564 }
565
566 spin_unlock_irqrestore(&chan->lock, flags);
567 }
568
569out:
570 dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret);
571
572 return ret;
573}
574
575static int ccp_pause(struct dma_chan *dma_chan)
576{
577 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
578 dma_chan);
579
580 chan->status = DMA_PAUSED;
581
582 /*TODO: Wait for active DMA to complete before returning? */
583
584 return 0;
585}
586
587static int ccp_resume(struct dma_chan *dma_chan)
588{
589 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
590 dma_chan);
591 struct ccp_dma_desc *desc;
592 unsigned long flags;
593
594 spin_lock_irqsave(&chan->lock, flags);
595
596 desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,
597 entry);
598
599 spin_unlock_irqrestore(&chan->lock, flags);
600
601 /* Indicate the channel is running again */
602 chan->status = DMA_IN_PROGRESS;
603
604 /* If there was something active, re-start */
605 if (desc)
606 ccp_cmd_callback(desc, 0);
607
608 return 0;
609}
610
611static int ccp_terminate_all(struct dma_chan *dma_chan)
612{
613 struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
614 dma_chan);
615 unsigned long flags;
616
617 dev_dbg(chan->ccp->dev, "%s\n", __func__);
618
619 /*TODO: Wait for active DMA to complete before continuing */
620
621 spin_lock_irqsave(&chan->lock, flags);
622
623 /*TODO: Purge the complete list? */
624 ccp_free_desc_resources(chan->ccp, &chan->active);
625 ccp_free_desc_resources(chan->ccp, &chan->pending);
626 ccp_free_desc_resources(chan->ccp, &chan->created);
627
628 spin_unlock_irqrestore(&chan->lock, flags);
629
630 return 0;
631}
632
633int ccp_dmaengine_register(struct ccp_device *ccp)
634{
635 struct ccp_dma_chan *chan;
636 struct dma_device *dma_dev = &ccp->dma_dev;
637 struct dma_chan *dma_chan;
638 char *dma_cmd_cache_name;
639 char *dma_desc_cache_name;
640 unsigned int i;
641 int ret;
642
643 ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count,
644 sizeof(*(ccp->ccp_dma_chan)),
645 GFP_KERNEL);
646 if (!ccp->ccp_dma_chan)
647 return -ENOMEM;
648
649 dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
650 "%s-dmaengine-cmd-cache",
651 ccp->name);
652 if (!dma_cmd_cache_name)
653 return -ENOMEM;
654
655 ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name,
656 sizeof(struct ccp_dma_cmd),
657 sizeof(void *),
658 SLAB_HWCACHE_ALIGN, NULL);
659 if (!ccp->dma_cmd_cache)
660 return -ENOMEM;
661
662 dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
663 "%s-dmaengine-desc-cache",
664 ccp->name);
665 if (!dma_desc_cache_name) {
666 ret = -ENOMEM;
667 goto err_cache;
668 }
669
670 ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name,
671 sizeof(struct ccp_dma_desc),
672 sizeof(void *),
673 SLAB_HWCACHE_ALIGN, NULL);
674 if (!ccp->dma_desc_cache) {
675 ret = -ENOMEM;
676 goto err_cache;
677 }
678
679 dma_dev->dev = ccp->dev;
680 dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev));
681 dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev));
682 dma_dev->directions = DMA_MEM_TO_MEM;
683 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
684 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
685 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
686
687 /* The DMA channels for this device can be set to public or private,
688 * and overridden by the module parameter dma_chan_attr.
689 * Default: according to the value in vdata (dma_chan_attr=0)
690 * dma_chan_attr=0x1: all channels private (override vdata)
691 * dma_chan_attr=0x2: all channels public (override vdata)
692 */
693 if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE)
694 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
695
696 INIT_LIST_HEAD(&dma_dev->channels);
697 for (i = 0; i < ccp->cmd_q_count; i++) {
698 chan = ccp->ccp_dma_chan + i;
699 dma_chan = &chan->dma_chan;
700
701 chan->ccp = ccp;
702
703 spin_lock_init(&chan->lock);
704 INIT_LIST_HEAD(&chan->created);
705 INIT_LIST_HEAD(&chan->pending);
706 INIT_LIST_HEAD(&chan->active);
707 INIT_LIST_HEAD(&chan->complete);
708
709 tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup,
710 (unsigned long)chan);
711
712 dma_chan->device = dma_dev;
713 dma_cookie_init(dma_chan);
714
715 list_add_tail(&dma_chan->device_node, &dma_dev->channels);
716 }
717
718 dma_dev->device_free_chan_resources = ccp_free_chan_resources;
719 dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy;
720 dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt;
721 dma_dev->device_issue_pending = ccp_issue_pending;
722 dma_dev->device_tx_status = ccp_tx_status;
723 dma_dev->device_pause = ccp_pause;
724 dma_dev->device_resume = ccp_resume;
725 dma_dev->device_terminate_all = ccp_terminate_all;
726
727 ret = dma_async_device_register(dma_dev);
728 if (ret)
729 goto err_reg;
730
731 return 0;
732
733err_reg:
734 kmem_cache_destroy(ccp->dma_desc_cache);
735
736err_cache:
737 kmem_cache_destroy(ccp->dma_cmd_cache);
738
739 return ret;
740}
741
742void ccp_dmaengine_unregister(struct ccp_device *ccp)
743{
744 struct dma_device *dma_dev = &ccp->dma_dev;
745
746 dma_async_device_unregister(dma_dev);
747
748 kmem_cache_destroy(ccp->dma_desc_cache);
749 kmem_cache_destroy(ccp->dma_cmd_cache);
750}
751