1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2018-2019 MediaTek Inc.
3
4/*
5 * Driver for MediaTek Command-Queue DMA Controller
6 *
7 * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com>
8 *
9 */
10
11#include <linux/bitops.h>
12#include <linux/clk.h>
13#include <linux/dmaengine.h>
14#include <linux/dma-mapping.h>
15#include <linux/err.h>
16#include <linux/iopoll.h>
17#include <linux/interrupt.h>
18#include <linux/list.h>
19#include <linux/module.h>
20#include <linux/of.h>
21#include <linux/of_dma.h>
22#include <linux/platform_device.h>
23#include <linux/pm_runtime.h>
24#include <linux/refcount.h>
25#include <linux/slab.h>
26
27#include "../virt-dma.h"
28
29#define MTK_CQDMA_USEC_POLL 10
30#define MTK_CQDMA_TIMEOUT_POLL 1000
31#define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
32#define MTK_CQDMA_ALIGN_SIZE 1
33
34/* The default number of virtual channel */
35#define MTK_CQDMA_NR_VCHANS 32
36
37/* The default number of physical channel */
38#define MTK_CQDMA_NR_PCHANS 3
39
40/* Registers for underlying dma manipulation */
41#define MTK_CQDMA_INT_FLAG 0x0
42#define MTK_CQDMA_INT_EN 0x4
43#define MTK_CQDMA_EN 0x8
44#define MTK_CQDMA_RESET 0xc
45#define MTK_CQDMA_FLUSH 0x14
46#define MTK_CQDMA_SRC 0x1c
47#define MTK_CQDMA_DST 0x20
48#define MTK_CQDMA_LEN1 0x24
49#define MTK_CQDMA_LEN2 0x28
50#define MTK_CQDMA_SRC2 0x60
51#define MTK_CQDMA_DST2 0x64
52
53/* Registers setting */
54#define MTK_CQDMA_EN_BIT BIT(0)
55#define MTK_CQDMA_INT_FLAG_BIT BIT(0)
56#define MTK_CQDMA_INT_EN_BIT BIT(0)
57#define MTK_CQDMA_FLUSH_BIT BIT(0)
58
59#define MTK_CQDMA_WARM_RST_BIT BIT(0)
60#define MTK_CQDMA_HARD_RST_BIT BIT(1)
61
62#define MTK_CQDMA_MAX_LEN GENMASK(27, 0)
63#define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0)
64#define MTK_CQDMA_ADDR2_SHFIT (32)
65
66/**
67 * struct mtk_cqdma_vdesc - The struct holding info describing virtual
68 * descriptor (CVD)
69 * @vd: An instance for struct virt_dma_desc
70 * @len: The total data size device wants to move
71 * @residue: The remaining data size device will move
72 * @dest: The destination address device wants to move to
73 * @src: The source address device wants to move from
74 * @ch: The pointer to the corresponding dma channel
75 * @node: The lise_head struct to build link-list for VDs
76 * @parent: The pointer to the parent CVD
77 */
78struct mtk_cqdma_vdesc {
79 struct virt_dma_desc vd;
80 size_t len;
81 size_t residue;
82 dma_addr_t dest;
83 dma_addr_t src;
84 struct dma_chan *ch;
85
86 struct list_head node;
87 struct mtk_cqdma_vdesc *parent;
88};
89
90/**
91 * struct mtk_cqdma_pchan - The struct holding info describing physical
92 * channel (PC)
93 * @queue: Queue for the PDs issued to this PC
94 * @base: The mapped register I/O base of this PC
95 * @irq: The IRQ that this PC are using
96 * @refcnt: Track how many VCs are using this PC
97 * @tasklet: Tasklet for this PC
98 * @lock: Lock protect agaisting multiple VCs access PC
99 */
100struct mtk_cqdma_pchan {
101 struct list_head queue;
102 void __iomem *base;
103 u32 irq;
104
105 refcount_t refcnt;
106
107 struct tasklet_struct tasklet;
108
109 /* lock to protect PC */
110 spinlock_t lock;
111};
112
113/**
114 * struct mtk_cqdma_vchan - The struct holding info describing virtual
115 * channel (VC)
116 * @vc: An instance for struct virt_dma_chan
117 * @pc: The pointer to the underlying PC
118 * @issue_completion: The wait for all issued descriptors completited
119 * @issue_synchronize: Bool indicating channel synchronization starts
120 */
121struct mtk_cqdma_vchan {
122 struct virt_dma_chan vc;
123 struct mtk_cqdma_pchan *pc;
124 struct completion issue_completion;
125 bool issue_synchronize;
126};
127
128/**
129 * struct mtk_cqdma_device - The struct holding info describing CQDMA
130 * device
131 * @ddev: An instance for struct dma_device
132 * @clk: The clock that device internal is using
133 * @dma_requests: The number of VCs the device supports to
134 * @dma_channels: The number of PCs the device supports to
135 * @vc: The pointer to all available VCs
136 * @pc: The pointer to all the underlying PCs
137 */
138struct mtk_cqdma_device {
139 struct dma_device ddev;
140 struct clk *clk;
141
142 u32 dma_requests;
143 u32 dma_channels;
144 struct mtk_cqdma_vchan *vc;
145 struct mtk_cqdma_pchan **pc;
146};
147
148static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan)
149{
150 return container_of(chan->device, struct mtk_cqdma_device, ddev);
151}
152
153static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan)
154{
155 return container_of(chan, struct mtk_cqdma_vchan, vc.chan);
156}
157
158static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd)
159{
160 return container_of(vd, struct mtk_cqdma_vdesc, vd);
161}
162
163static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma)
164{
165 return cqdma->ddev.dev;
166}
167
168static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg)
169{
170 return readl(addr: pc->base + reg);
171}
172
173static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
174{
175 writel_relaxed(val, pc->base + reg);
176}
177
178static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg,
179 u32 mask, u32 set)
180{
181 u32 val;
182
183 val = mtk_dma_read(pc, reg);
184 val &= ~mask;
185 val |= set;
186 mtk_dma_write(pc, reg, val);
187}
188
189static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
190{
191 mtk_dma_rmw(pc, reg, mask: 0, set: val);
192}
193
194static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
195{
196 mtk_dma_rmw(pc, reg, mask: val, set: 0);
197}
198
199static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd)
200{
201 kfree(objp: to_cqdma_vdesc(vd));
202}
203
204static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic)
205{
206 u32 status = 0;
207
208 if (!atomic)
209 return readl_poll_timeout(pc->base + MTK_CQDMA_EN,
210 status,
211 !(status & MTK_CQDMA_EN_BIT),
212 MTK_CQDMA_USEC_POLL,
213 MTK_CQDMA_TIMEOUT_POLL);
214
215 return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN,
216 status,
217 !(status & MTK_CQDMA_EN_BIT),
218 MTK_CQDMA_USEC_POLL,
219 MTK_CQDMA_TIMEOUT_POLL);
220}
221
222static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc)
223{
224 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
225 mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
226
227 return mtk_cqdma_poll_engine_done(pc, atomic: true);
228}
229
230static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc,
231 struct mtk_cqdma_vdesc *cvd)
232{
233 /* wait for the previous transaction done */
234 if (mtk_cqdma_poll_engine_done(pc, atomic: true) < 0)
235 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n");
236
237 /* warm reset the dma engine for the new transaction */
238 mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT);
239 if (mtk_cqdma_poll_engine_done(pc, atomic: true) < 0)
240 dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n");
241
242 /* setup the source */
243 mtk_dma_set(pc, MTK_CQDMA_SRC, val: cvd->src & MTK_CQDMA_ADDR_LIMIT);
244#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
245 mtk_dma_set(pc, MTK_CQDMA_SRC2, val: cvd->src >> MTK_CQDMA_ADDR2_SHFIT);
246#else
247 mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);
248#endif
249
250 /* setup the destination */
251 mtk_dma_set(pc, MTK_CQDMA_DST, val: cvd->dest & MTK_CQDMA_ADDR_LIMIT);
252#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
253 mtk_dma_set(pc, MTK_CQDMA_DST2, val: cvd->dest >> MTK_CQDMA_ADDR2_SHFIT);
254#else
255 mtk_dma_set(pc, MTK_CQDMA_DST2, 0);
256#endif
257
258 /* setup the length */
259 mtk_dma_set(pc, MTK_CQDMA_LEN1, val: cvd->len);
260
261 /* start dma engine */
262 mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT);
263}
264
265static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc)
266{
267 struct virt_dma_desc *vd, *vd2;
268 struct mtk_cqdma_pchan *pc = cvc->pc;
269 struct mtk_cqdma_vdesc *cvd;
270 bool trigger_engine = false;
271
272 lockdep_assert_held(&cvc->vc.lock);
273 lockdep_assert_held(&pc->lock);
274
275 list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) {
276 /* need to trigger dma engine if PC's queue is empty */
277 if (list_empty(head: &pc->queue))
278 trigger_engine = true;
279
280 cvd = to_cqdma_vdesc(vd);
281
282 /* add VD into PC's queue */
283 list_add_tail(new: &cvd->node, head: &pc->queue);
284
285 /* start the dma engine */
286 if (trigger_engine)
287 mtk_cqdma_start(pc, cvd);
288
289 /* remove VD from list desc_issued */
290 list_del(entry: &vd->node);
291 }
292}
293
294/*
295 * return true if this VC is active,
296 * meaning that there are VDs under processing by the PC
297 */
298static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc)
299{
300 struct mtk_cqdma_vdesc *cvd;
301
302 list_for_each_entry(cvd, &cvc->pc->queue, node)
303 if (cvc == to_cqdma_vchan(chan: cvd->ch))
304 return true;
305
306 return false;
307}
308
309/*
310 * return the pointer of the CVD that is just consumed by the PC
311 */
312static struct mtk_cqdma_vdesc
313*mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc)
314{
315 struct mtk_cqdma_vchan *cvc;
316 struct mtk_cqdma_vdesc *cvd, *ret = NULL;
317
318 /* consume a CVD from PC's queue */
319 cvd = list_first_entry_or_null(&pc->queue,
320 struct mtk_cqdma_vdesc, node);
321 if (unlikely(!cvd || !cvd->parent))
322 return NULL;
323
324 cvc = to_cqdma_vchan(chan: cvd->ch);
325 ret = cvd;
326
327 /* update residue of the parent CVD */
328 cvd->parent->residue -= cvd->len;
329
330 /* delete CVD from PC's queue */
331 list_del(entry: &cvd->node);
332
333 spin_lock(lock: &cvc->vc.lock);
334
335 /* check whether all the child CVDs completed */
336 if (!cvd->parent->residue) {
337 /* add the parent VD into list desc_completed */
338 vchan_cookie_complete(vd: &cvd->parent->vd);
339
340 /* setup completion if this VC is under synchronization */
341 if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) {
342 complete(&cvc->issue_completion);
343 cvc->issue_synchronize = false;
344 }
345 }
346
347 spin_unlock(lock: &cvc->vc.lock);
348
349 /* start transaction for next CVD in the queue */
350 cvd = list_first_entry_or_null(&pc->queue,
351 struct mtk_cqdma_vdesc, node);
352 if (cvd)
353 mtk_cqdma_start(pc, cvd);
354
355 return ret;
356}
357
358static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t)
359{
360 struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet);
361 struct mtk_cqdma_vdesc *cvd = NULL;
362 unsigned long flags;
363
364 spin_lock_irqsave(&pc->lock, flags);
365 /* consume the queue */
366 cvd = mtk_cqdma_consume_work_queue(pc);
367 spin_unlock_irqrestore(lock: &pc->lock, flags);
368
369 /* submit the next CVD */
370 if (cvd) {
371 dma_run_dependencies(tx: &cvd->vd.tx);
372
373 /*
374 * free child CVD after completion.
375 * the parent CVD would be freed with desc_free by user.
376 */
377 if (cvd->parent != cvd)
378 kfree(objp: cvd);
379 }
380
381 /* re-enable interrupt before leaving tasklet */
382 enable_irq(irq: pc->irq);
383}
384
385static irqreturn_t mtk_cqdma_irq(int irq, void *devid)
386{
387 struct mtk_cqdma_device *cqdma = devid;
388 irqreturn_t ret = IRQ_NONE;
389 bool schedule_tasklet = false;
390 u32 i;
391
392 /* clear interrupt flags for each PC */
393 for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) {
394 spin_lock(lock: &cqdma->pc[i]->lock);
395 if (mtk_dma_read(pc: cqdma->pc[i],
396 MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) {
397 /* clear interrupt */
398 mtk_dma_clr(pc: cqdma->pc[i], MTK_CQDMA_INT_FLAG,
399 MTK_CQDMA_INT_FLAG_BIT);
400
401 schedule_tasklet = true;
402 ret = IRQ_HANDLED;
403 }
404 spin_unlock(lock: &cqdma->pc[i]->lock);
405
406 if (schedule_tasklet) {
407 /* disable interrupt */
408 disable_irq_nosync(irq: cqdma->pc[i]->irq);
409
410 /* schedule the tasklet to handle the transactions */
411 tasklet_schedule(t: &cqdma->pc[i]->tasklet);
412 }
413 }
414
415 return ret;
416}
417
418static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c,
419 dma_cookie_t cookie)
420{
421 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(chan: c);
422 struct virt_dma_desc *vd;
423 unsigned long flags;
424
425 spin_lock_irqsave(&cvc->pc->lock, flags);
426 list_for_each_entry(vd, &cvc->pc->queue, node)
427 if (vd->tx.cookie == cookie) {
428 spin_unlock_irqrestore(lock: &cvc->pc->lock, flags);
429 return vd;
430 }
431 spin_unlock_irqrestore(lock: &cvc->pc->lock, flags);
432
433 list_for_each_entry(vd, &cvc->vc.desc_issued, node)
434 if (vd->tx.cookie == cookie)
435 return vd;
436
437 return NULL;
438}
439
440static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c,
441 dma_cookie_t cookie,
442 struct dma_tx_state *txstate)
443{
444 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(chan: c);
445 struct mtk_cqdma_vdesc *cvd;
446 struct virt_dma_desc *vd;
447 enum dma_status ret;
448 unsigned long flags;
449 size_t bytes = 0;
450
451 ret = dma_cookie_status(chan: c, cookie, state: txstate);
452 if (ret == DMA_COMPLETE || !txstate)
453 return ret;
454
455 spin_lock_irqsave(&cvc->vc.lock, flags);
456 vd = mtk_cqdma_find_active_desc(c, cookie);
457 spin_unlock_irqrestore(lock: &cvc->vc.lock, flags);
458
459 if (vd) {
460 cvd = to_cqdma_vdesc(vd);
461 bytes = cvd->residue;
462 }
463
464 dma_set_residue(state: txstate, residue: bytes);
465
466 return ret;
467}
468
469static void mtk_cqdma_issue_pending(struct dma_chan *c)
470{
471 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(chan: c);
472 unsigned long pc_flags;
473 unsigned long vc_flags;
474
475 /* acquire PC's lock before VS's lock for lock dependency in tasklet */
476 spin_lock_irqsave(&cvc->pc->lock, pc_flags);
477 spin_lock_irqsave(&cvc->vc.lock, vc_flags);
478
479 if (vchan_issue_pending(vc: &cvc->vc))
480 mtk_cqdma_issue_vchan_pending(cvc);
481
482 spin_unlock_irqrestore(lock: &cvc->vc.lock, flags: vc_flags);
483 spin_unlock_irqrestore(lock: &cvc->pc->lock, flags: pc_flags);
484}
485
486static struct dma_async_tx_descriptor *
487mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
488 dma_addr_t src, size_t len, unsigned long flags)
489{
490 struct mtk_cqdma_vdesc **cvd;
491 struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL;
492 size_t i, tlen, nr_vd;
493
494 /*
495 * In the case that trsanction length is larger than the
496 * DMA engine supports, a single memcpy transaction needs
497 * to be separated into several DMA transactions.
498 * Each DMA transaction would be described by a CVD,
499 * and the first one is referred as the parent CVD,
500 * while the others are child CVDs.
501 * The parent CVD's tx descriptor is the only tx descriptor
502 * returned to the DMA user, and it should not be completed
503 * until all the child CVDs completed.
504 */
505 nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN);
506 cvd = kcalloc(n: nr_vd, size: sizeof(*cvd), GFP_NOWAIT);
507 if (!cvd)
508 return NULL;
509
510 for (i = 0; i < nr_vd; ++i) {
511 cvd[i] = kzalloc(size: sizeof(*cvd[i]), GFP_NOWAIT);
512 if (!cvd[i]) {
513 for (; i > 0; --i)
514 kfree(objp: cvd[i - 1]);
515 return NULL;
516 }
517
518 /* setup dma channel */
519 cvd[i]->ch = c;
520
521 /* setup sourece, destination, and length */
522 tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len;
523 cvd[i]->len = tlen;
524 cvd[i]->src = src;
525 cvd[i]->dest = dest;
526
527 /* setup tx descriptor */
528 tx = vchan_tx_prep(vc: to_virt_chan(chan: c), vd: &cvd[i]->vd, tx_flags: flags);
529 tx->next = NULL;
530
531 if (!i) {
532 cvd[0]->residue = len;
533 } else {
534 prev_tx->next = tx;
535 cvd[i]->residue = tlen;
536 }
537
538 cvd[i]->parent = cvd[0];
539
540 /* update the src, dest, len, prev_tx for the next CVD */
541 src += tlen;
542 dest += tlen;
543 len -= tlen;
544 prev_tx = tx;
545 }
546
547 return &cvd[0]->vd.tx;
548}
549
550static void mtk_cqdma_free_inactive_desc(struct dma_chan *c)
551{
552 struct virt_dma_chan *vc = to_virt_chan(chan: c);
553 unsigned long flags;
554 LIST_HEAD(head);
555
556 /*
557 * set desc_allocated, desc_submitted,
558 * and desc_issued as the candicates to be freed
559 */
560 spin_lock_irqsave(&vc->lock, flags);
561 list_splice_tail_init(list: &vc->desc_allocated, head: &head);
562 list_splice_tail_init(list: &vc->desc_submitted, head: &head);
563 list_splice_tail_init(list: &vc->desc_issued, head: &head);
564 spin_unlock_irqrestore(lock: &vc->lock, flags);
565
566 /* free descriptor lists */
567 vchan_dma_desc_free_list(vc, head: &head);
568}
569
570static void mtk_cqdma_free_active_desc(struct dma_chan *c)
571{
572 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(chan: c);
573 bool sync_needed = false;
574 unsigned long pc_flags;
575 unsigned long vc_flags;
576
577 /* acquire PC's lock first due to lock dependency in dma ISR */
578 spin_lock_irqsave(&cvc->pc->lock, pc_flags);
579 spin_lock_irqsave(&cvc->vc.lock, vc_flags);
580
581 /* synchronization is required if this VC is active */
582 if (mtk_cqdma_is_vchan_active(cvc)) {
583 cvc->issue_synchronize = true;
584 sync_needed = true;
585 }
586
587 spin_unlock_irqrestore(lock: &cvc->vc.lock, flags: vc_flags);
588 spin_unlock_irqrestore(lock: &cvc->pc->lock, flags: pc_flags);
589
590 /* waiting for the completion of this VC */
591 if (sync_needed)
592 wait_for_completion(&cvc->issue_completion);
593
594 /* free all descriptors in list desc_completed */
595 vchan_synchronize(vc: &cvc->vc);
596
597 WARN_ONCE(!list_empty(&cvc->vc.desc_completed),
598 "Desc pending still in list desc_completed\n");
599}
600
601static int mtk_cqdma_terminate_all(struct dma_chan *c)
602{
603 /* free descriptors not processed yet by hardware */
604 mtk_cqdma_free_inactive_desc(c);
605
606 /* free descriptors being processed by hardware */
607 mtk_cqdma_free_active_desc(c);
608
609 return 0;
610}
611
612static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c)
613{
614 struct mtk_cqdma_device *cqdma = to_cqdma_dev(chan: c);
615 struct mtk_cqdma_vchan *vc = to_cqdma_vchan(chan: c);
616 struct mtk_cqdma_pchan *pc = NULL;
617 u32 i, min_refcnt = U32_MAX, refcnt;
618 unsigned long flags;
619
620 /* allocate PC with the minimun refcount */
621 for (i = 0; i < cqdma->dma_channels; ++i) {
622 refcnt = refcount_read(r: &cqdma->pc[i]->refcnt);
623 if (refcnt < min_refcnt) {
624 pc = cqdma->pc[i];
625 min_refcnt = refcnt;
626 }
627 }
628
629 if (!pc)
630 return -ENOSPC;
631
632 spin_lock_irqsave(&pc->lock, flags);
633
634 if (!refcount_read(r: &pc->refcnt)) {
635 /* allocate PC when the refcount is zero */
636 mtk_cqdma_hard_reset(pc);
637
638 /* enable interrupt for this PC */
639 mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
640
641 /*
642 * refcount_inc would complain increment on 0; use-after-free.
643 * Thus, we need to explicitly set it as 1 initially.
644 */
645 refcount_set(r: &pc->refcnt, n: 1);
646 } else {
647 refcount_inc(r: &pc->refcnt);
648 }
649
650 spin_unlock_irqrestore(lock: &pc->lock, flags);
651
652 vc->pc = pc;
653
654 return 0;
655}
656
657static void mtk_cqdma_free_chan_resources(struct dma_chan *c)
658{
659 struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(chan: c);
660 unsigned long flags;
661
662 /* free all descriptors in all lists on the VC */
663 mtk_cqdma_terminate_all(c);
664
665 spin_lock_irqsave(&cvc->pc->lock, flags);
666
667 /* PC is not freed until there is no VC mapped to it */
668 if (refcount_dec_and_test(r: &cvc->pc->refcnt)) {
669 /* start the flush operation and stop the engine */
670 mtk_dma_set(pc: cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
671
672 /* wait for the completion of flush operation */
673 if (mtk_cqdma_poll_engine_done(pc: cvc->pc, atomic: true) < 0)
674 dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n");
675
676 /* clear the flush bit and interrupt flag */
677 mtk_dma_clr(pc: cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
678 mtk_dma_clr(pc: cvc->pc, MTK_CQDMA_INT_FLAG,
679 MTK_CQDMA_INT_FLAG_BIT);
680
681 /* disable interrupt for this PC */
682 mtk_dma_clr(pc: cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
683 }
684
685 spin_unlock_irqrestore(lock: &cvc->pc->lock, flags);
686}
687
688static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma)
689{
690 unsigned long flags;
691 int err;
692 u32 i;
693
694 pm_runtime_enable(dev: cqdma2dev(cqdma));
695 pm_runtime_get_sync(dev: cqdma2dev(cqdma));
696
697 err = clk_prepare_enable(clk: cqdma->clk);
698
699 if (err) {
700 pm_runtime_put_sync(dev: cqdma2dev(cqdma));
701 pm_runtime_disable(dev: cqdma2dev(cqdma));
702 return err;
703 }
704
705 /* reset all PCs */
706 for (i = 0; i < cqdma->dma_channels; ++i) {
707 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
708 if (mtk_cqdma_hard_reset(pc: cqdma->pc[i]) < 0) {
709 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
710 spin_unlock_irqrestore(lock: &cqdma->pc[i]->lock, flags);
711
712 clk_disable_unprepare(clk: cqdma->clk);
713 pm_runtime_put_sync(dev: cqdma2dev(cqdma));
714 pm_runtime_disable(dev: cqdma2dev(cqdma));
715 return -EINVAL;
716 }
717 spin_unlock_irqrestore(lock: &cqdma->pc[i]->lock, flags);
718 }
719
720 return 0;
721}
722
723static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma)
724{
725 unsigned long flags;
726 u32 i;
727
728 /* reset all PCs */
729 for (i = 0; i < cqdma->dma_channels; ++i) {
730 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
731 if (mtk_cqdma_hard_reset(pc: cqdma->pc[i]) < 0)
732 dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
733 spin_unlock_irqrestore(lock: &cqdma->pc[i]->lock, flags);
734 }
735
736 clk_disable_unprepare(clk: cqdma->clk);
737
738 pm_runtime_put_sync(dev: cqdma2dev(cqdma));
739 pm_runtime_disable(dev: cqdma2dev(cqdma));
740}
741
742static const struct of_device_id mtk_cqdma_match[] = {
743 { .compatible = "mediatek,mt6765-cqdma" },
744 { /* sentinel */ }
745};
746MODULE_DEVICE_TABLE(of, mtk_cqdma_match);
747
748static int mtk_cqdma_probe(struct platform_device *pdev)
749{
750 struct mtk_cqdma_device *cqdma;
751 struct mtk_cqdma_vchan *vc;
752 struct dma_device *dd;
753 int err;
754 u32 i;
755
756 cqdma = devm_kzalloc(dev: &pdev->dev, size: sizeof(*cqdma), GFP_KERNEL);
757 if (!cqdma)
758 return -ENOMEM;
759
760 dd = &cqdma->ddev;
761
762 cqdma->clk = devm_clk_get(dev: &pdev->dev, id: "cqdma");
763 if (IS_ERR(ptr: cqdma->clk)) {
764 dev_err(&pdev->dev, "No clock for %s\n",
765 dev_name(&pdev->dev));
766 return PTR_ERR(ptr: cqdma->clk);
767 }
768
769 dma_cap_set(DMA_MEMCPY, dd->cap_mask);
770
771 dd->copy_align = MTK_CQDMA_ALIGN_SIZE;
772 dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources;
773 dd->device_free_chan_resources = mtk_cqdma_free_chan_resources;
774 dd->device_tx_status = mtk_cqdma_tx_status;
775 dd->device_issue_pending = mtk_cqdma_issue_pending;
776 dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy;
777 dd->device_terminate_all = mtk_cqdma_terminate_all;
778 dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
779 dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
780 dd->directions = BIT(DMA_MEM_TO_MEM);
781 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
782 dd->dev = &pdev->dev;
783 INIT_LIST_HEAD(list: &dd->channels);
784
785 if (pdev->dev.of_node && of_property_read_u32(np: pdev->dev.of_node,
786 propname: "dma-requests",
787 out_value: &cqdma->dma_requests)) {
788 dev_info(&pdev->dev,
789 "Using %u as missing dma-requests property\n",
790 MTK_CQDMA_NR_VCHANS);
791
792 cqdma->dma_requests = MTK_CQDMA_NR_VCHANS;
793 }
794
795 if (pdev->dev.of_node && of_property_read_u32(np: pdev->dev.of_node,
796 propname: "dma-channels",
797 out_value: &cqdma->dma_channels)) {
798 dev_info(&pdev->dev,
799 "Using %u as missing dma-channels property\n",
800 MTK_CQDMA_NR_PCHANS);
801
802 cqdma->dma_channels = MTK_CQDMA_NR_PCHANS;
803 }
804
805 cqdma->pc = devm_kcalloc(dev: &pdev->dev, n: cqdma->dma_channels,
806 size: sizeof(*cqdma->pc), GFP_KERNEL);
807 if (!cqdma->pc)
808 return -ENOMEM;
809
810 /* initialization for PCs */
811 for (i = 0; i < cqdma->dma_channels; ++i) {
812 cqdma->pc[i] = devm_kcalloc(dev: &pdev->dev, n: 1,
813 size: sizeof(**cqdma->pc), GFP_KERNEL);
814 if (!cqdma->pc[i])
815 return -ENOMEM;
816
817 INIT_LIST_HEAD(list: &cqdma->pc[i]->queue);
818 spin_lock_init(&cqdma->pc[i]->lock);
819 refcount_set(r: &cqdma->pc[i]->refcnt, n: 0);
820 cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, index: i);
821 if (IS_ERR(ptr: cqdma->pc[i]->base))
822 return PTR_ERR(ptr: cqdma->pc[i]->base);
823
824 /* allocate IRQ resource */
825 err = platform_get_irq(pdev, i);
826 if (err < 0)
827 return err;
828 cqdma->pc[i]->irq = err;
829
830 err = devm_request_irq(dev: &pdev->dev, irq: cqdma->pc[i]->irq,
831 handler: mtk_cqdma_irq, irqflags: 0, devname: dev_name(dev: &pdev->dev),
832 dev_id: cqdma);
833 if (err) {
834 dev_err(&pdev->dev,
835 "request_irq failed with err %d\n", err);
836 return -EINVAL;
837 }
838 }
839
840 /* allocate resource for VCs */
841 cqdma->vc = devm_kcalloc(dev: &pdev->dev, n: cqdma->dma_requests,
842 size: sizeof(*cqdma->vc), GFP_KERNEL);
843 if (!cqdma->vc)
844 return -ENOMEM;
845
846 for (i = 0; i < cqdma->dma_requests; i++) {
847 vc = &cqdma->vc[i];
848 vc->vc.desc_free = mtk_cqdma_vdesc_free;
849 vchan_init(vc: &vc->vc, dmadev: dd);
850 init_completion(x: &vc->issue_completion);
851 }
852
853 err = dma_async_device_register(device: dd);
854 if (err)
855 return err;
856
857 err = of_dma_controller_register(np: pdev->dev.of_node,
858 of_dma_xlate: of_dma_xlate_by_chan_id, data: cqdma);
859 if (err) {
860 dev_err(&pdev->dev,
861 "MediaTek CQDMA OF registration failed %d\n", err);
862 goto err_unregister;
863 }
864
865 err = mtk_cqdma_hw_init(cqdma);
866 if (err) {
867 dev_err(&pdev->dev,
868 "MediaTek CQDMA HW initialization failed %d\n", err);
869 goto err_unregister;
870 }
871
872 platform_set_drvdata(pdev, data: cqdma);
873
874 /* initialize tasklet for each PC */
875 for (i = 0; i < cqdma->dma_channels; ++i)
876 tasklet_setup(t: &cqdma->pc[i]->tasklet, callback: mtk_cqdma_tasklet_cb);
877
878 dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n");
879
880 return 0;
881
882err_unregister:
883 dma_async_device_unregister(device: dd);
884
885 return err;
886}
887
888static void mtk_cqdma_remove(struct platform_device *pdev)
889{
890 struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev);
891 struct mtk_cqdma_vchan *vc;
892 unsigned long flags;
893 int i;
894
895 /* kill VC task */
896 for (i = 0; i < cqdma->dma_requests; i++) {
897 vc = &cqdma->vc[i];
898
899 list_del(entry: &vc->vc.chan.device_node);
900 tasklet_kill(t: &vc->vc.task);
901 }
902
903 /* disable interrupt */
904 for (i = 0; i < cqdma->dma_channels; i++) {
905 spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
906 mtk_dma_clr(pc: cqdma->pc[i], MTK_CQDMA_INT_EN,
907 MTK_CQDMA_INT_EN_BIT);
908 spin_unlock_irqrestore(lock: &cqdma->pc[i]->lock, flags);
909
910 /* Waits for any pending IRQ handlers to complete */
911 synchronize_irq(irq: cqdma->pc[i]->irq);
912
913 tasklet_kill(t: &cqdma->pc[i]->tasklet);
914 }
915
916 /* disable hardware */
917 mtk_cqdma_hw_deinit(cqdma);
918
919 dma_async_device_unregister(device: &cqdma->ddev);
920 of_dma_controller_free(np: pdev->dev.of_node);
921}
922
923static struct platform_driver mtk_cqdma_driver = {
924 .probe = mtk_cqdma_probe,
925 .remove_new = mtk_cqdma_remove,
926 .driver = {
927 .name = KBUILD_MODNAME,
928 .of_match_table = mtk_cqdma_match,
929 },
930};
931module_platform_driver(mtk_cqdma_driver);
932
933MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver");
934MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>");
935MODULE_LICENSE("GPL v2");
936

source code of linux/drivers/dma/mediatek/mtk-cqdma.c