1 | // SPDX-License-Identifier: GPL-2.0 |
2 | // Copyright (c) 2018-2019 MediaTek Inc. |
3 | |
4 | /* |
5 | * Driver for MediaTek Command-Queue DMA Controller |
6 | * |
7 | * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com> |
8 | * |
9 | */ |
10 | |
11 | #include <linux/bitops.h> |
12 | #include <linux/clk.h> |
13 | #include <linux/dmaengine.h> |
14 | #include <linux/dma-mapping.h> |
15 | #include <linux/err.h> |
16 | #include <linux/iopoll.h> |
17 | #include <linux/interrupt.h> |
18 | #include <linux/list.h> |
19 | #include <linux/module.h> |
20 | #include <linux/of.h> |
21 | #include <linux/of_dma.h> |
22 | #include <linux/platform_device.h> |
23 | #include <linux/pm_runtime.h> |
24 | #include <linux/refcount.h> |
25 | #include <linux/slab.h> |
26 | |
27 | #include "../virt-dma.h" |
28 | |
29 | #define MTK_CQDMA_USEC_POLL 10 |
30 | #define MTK_CQDMA_TIMEOUT_POLL 1000 |
31 | #define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
32 | #define MTK_CQDMA_ALIGN_SIZE 1 |
33 | |
34 | /* The default number of virtual channel */ |
35 | #define MTK_CQDMA_NR_VCHANS 32 |
36 | |
37 | /* The default number of physical channel */ |
38 | #define MTK_CQDMA_NR_PCHANS 3 |
39 | |
40 | /* Registers for underlying dma manipulation */ |
41 | #define MTK_CQDMA_INT_FLAG 0x0 |
42 | #define MTK_CQDMA_INT_EN 0x4 |
43 | #define MTK_CQDMA_EN 0x8 |
44 | #define MTK_CQDMA_RESET 0xc |
45 | #define MTK_CQDMA_FLUSH 0x14 |
46 | #define MTK_CQDMA_SRC 0x1c |
47 | #define MTK_CQDMA_DST 0x20 |
48 | #define MTK_CQDMA_LEN1 0x24 |
49 | #define MTK_CQDMA_LEN2 0x28 |
50 | #define MTK_CQDMA_SRC2 0x60 |
51 | #define MTK_CQDMA_DST2 0x64 |
52 | |
53 | /* Registers setting */ |
54 | #define MTK_CQDMA_EN_BIT BIT(0) |
55 | #define MTK_CQDMA_INT_FLAG_BIT BIT(0) |
56 | #define MTK_CQDMA_INT_EN_BIT BIT(0) |
57 | #define MTK_CQDMA_FLUSH_BIT BIT(0) |
58 | |
59 | #define MTK_CQDMA_WARM_RST_BIT BIT(0) |
60 | #define MTK_CQDMA_HARD_RST_BIT BIT(1) |
61 | |
62 | #define MTK_CQDMA_MAX_LEN GENMASK(27, 0) |
63 | #define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0) |
64 | #define MTK_CQDMA_ADDR2_SHFIT (32) |
65 | |
66 | /** |
67 | * struct mtk_cqdma_vdesc - The struct holding info describing virtual |
68 | * descriptor (CVD) |
69 | * @vd: An instance for struct virt_dma_desc |
70 | * @len: The total data size device wants to move |
71 | * @residue: The remaining data size device will move |
72 | * @dest: The destination address device wants to move to |
73 | * @src: The source address device wants to move from |
74 | * @ch: The pointer to the corresponding dma channel |
75 | * @node: The lise_head struct to build link-list for VDs |
76 | * @parent: The pointer to the parent CVD |
77 | */ |
78 | struct mtk_cqdma_vdesc { |
79 | struct virt_dma_desc vd; |
80 | size_t len; |
81 | size_t residue; |
82 | dma_addr_t dest; |
83 | dma_addr_t src; |
84 | struct dma_chan *ch; |
85 | |
86 | struct list_head node; |
87 | struct mtk_cqdma_vdesc *parent; |
88 | }; |
89 | |
90 | /** |
91 | * struct mtk_cqdma_pchan - The struct holding info describing physical |
92 | * channel (PC) |
93 | * @queue: Queue for the PDs issued to this PC |
94 | * @base: The mapped register I/O base of this PC |
95 | * @irq: The IRQ that this PC are using |
96 | * @refcnt: Track how many VCs are using this PC |
97 | * @tasklet: Tasklet for this PC |
98 | * @lock: Lock protect agaisting multiple VCs access PC |
99 | */ |
100 | struct mtk_cqdma_pchan { |
101 | struct list_head queue; |
102 | void __iomem *base; |
103 | u32 irq; |
104 | |
105 | refcount_t refcnt; |
106 | |
107 | struct tasklet_struct tasklet; |
108 | |
109 | /* lock to protect PC */ |
110 | spinlock_t lock; |
111 | }; |
112 | |
113 | /** |
114 | * struct mtk_cqdma_vchan - The struct holding info describing virtual |
115 | * channel (VC) |
116 | * @vc: An instance for struct virt_dma_chan |
117 | * @pc: The pointer to the underlying PC |
118 | * @issue_completion: The wait for all issued descriptors completited |
119 | * @issue_synchronize: Bool indicating channel synchronization starts |
120 | */ |
121 | struct mtk_cqdma_vchan { |
122 | struct virt_dma_chan vc; |
123 | struct mtk_cqdma_pchan *pc; |
124 | struct completion issue_completion; |
125 | bool issue_synchronize; |
126 | }; |
127 | |
128 | /** |
129 | * struct mtk_cqdma_device - The struct holding info describing CQDMA |
130 | * device |
131 | * @ddev: An instance for struct dma_device |
132 | * @clk: The clock that device internal is using |
133 | * @dma_requests: The number of VCs the device supports to |
134 | * @dma_channels: The number of PCs the device supports to |
135 | * @vc: The pointer to all available VCs |
136 | * @pc: The pointer to all the underlying PCs |
137 | */ |
138 | struct mtk_cqdma_device { |
139 | struct dma_device ddev; |
140 | struct clk *clk; |
141 | |
142 | u32 dma_requests; |
143 | u32 dma_channels; |
144 | struct mtk_cqdma_vchan *vc; |
145 | struct mtk_cqdma_pchan **pc; |
146 | }; |
147 | |
148 | static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan) |
149 | { |
150 | return container_of(chan->device, struct mtk_cqdma_device, ddev); |
151 | } |
152 | |
153 | static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan) |
154 | { |
155 | return container_of(chan, struct mtk_cqdma_vchan, vc.chan); |
156 | } |
157 | |
158 | static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd) |
159 | { |
160 | return container_of(vd, struct mtk_cqdma_vdesc, vd); |
161 | } |
162 | |
163 | static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma) |
164 | { |
165 | return cqdma->ddev.dev; |
166 | } |
167 | |
168 | static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg) |
169 | { |
170 | return readl(addr: pc->base + reg); |
171 | } |
172 | |
173 | static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) |
174 | { |
175 | writel_relaxed(val, pc->base + reg); |
176 | } |
177 | |
178 | static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg, |
179 | u32 mask, u32 set) |
180 | { |
181 | u32 val; |
182 | |
183 | val = mtk_dma_read(pc, reg); |
184 | val &= ~mask; |
185 | val |= set; |
186 | mtk_dma_write(pc, reg, val); |
187 | } |
188 | |
189 | static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) |
190 | { |
191 | mtk_dma_rmw(pc, reg, mask: 0, set: val); |
192 | } |
193 | |
194 | static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val) |
195 | { |
196 | mtk_dma_rmw(pc, reg, mask: val, set: 0); |
197 | } |
198 | |
199 | static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd) |
200 | { |
201 | kfree(objp: to_cqdma_vdesc(vd)); |
202 | } |
203 | |
204 | static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic) |
205 | { |
206 | u32 status = 0; |
207 | |
208 | if (!atomic) |
209 | return readl_poll_timeout(pc->base + MTK_CQDMA_EN, |
210 | status, |
211 | !(status & MTK_CQDMA_EN_BIT), |
212 | MTK_CQDMA_USEC_POLL, |
213 | MTK_CQDMA_TIMEOUT_POLL); |
214 | |
215 | return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN, |
216 | status, |
217 | !(status & MTK_CQDMA_EN_BIT), |
218 | MTK_CQDMA_USEC_POLL, |
219 | MTK_CQDMA_TIMEOUT_POLL); |
220 | } |
221 | |
222 | static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc) |
223 | { |
224 | mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT); |
225 | mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT); |
226 | |
227 | return mtk_cqdma_poll_engine_done(pc, atomic: true); |
228 | } |
229 | |
230 | static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc, |
231 | struct mtk_cqdma_vdesc *cvd) |
232 | { |
233 | /* wait for the previous transaction done */ |
234 | if (mtk_cqdma_poll_engine_done(pc, atomic: true) < 0) |
235 | dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n" ); |
236 | |
237 | /* warm reset the dma engine for the new transaction */ |
238 | mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT); |
239 | if (mtk_cqdma_poll_engine_done(pc, atomic: true) < 0) |
240 | dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n" ); |
241 | |
242 | /* setup the source */ |
243 | mtk_dma_set(pc, MTK_CQDMA_SRC, val: cvd->src & MTK_CQDMA_ADDR_LIMIT); |
244 | #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT |
245 | mtk_dma_set(pc, MTK_CQDMA_SRC2, val: cvd->src >> MTK_CQDMA_ADDR2_SHFIT); |
246 | #else |
247 | mtk_dma_set(pc, MTK_CQDMA_SRC2, 0); |
248 | #endif |
249 | |
250 | /* setup the destination */ |
251 | mtk_dma_set(pc, MTK_CQDMA_DST, val: cvd->dest & MTK_CQDMA_ADDR_LIMIT); |
252 | #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT |
253 | mtk_dma_set(pc, MTK_CQDMA_DST2, val: cvd->dest >> MTK_CQDMA_ADDR2_SHFIT); |
254 | #else |
255 | mtk_dma_set(pc, MTK_CQDMA_DST2, 0); |
256 | #endif |
257 | |
258 | /* setup the length */ |
259 | mtk_dma_set(pc, MTK_CQDMA_LEN1, val: cvd->len); |
260 | |
261 | /* start dma engine */ |
262 | mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT); |
263 | } |
264 | |
265 | static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc) |
266 | { |
267 | struct virt_dma_desc *vd, *vd2; |
268 | struct mtk_cqdma_pchan *pc = cvc->pc; |
269 | struct mtk_cqdma_vdesc *cvd; |
270 | bool trigger_engine = false; |
271 | |
272 | lockdep_assert_held(&cvc->vc.lock); |
273 | lockdep_assert_held(&pc->lock); |
274 | |
275 | list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) { |
276 | /* need to trigger dma engine if PC's queue is empty */ |
277 | if (list_empty(head: &pc->queue)) |
278 | trigger_engine = true; |
279 | |
280 | cvd = to_cqdma_vdesc(vd); |
281 | |
282 | /* add VD into PC's queue */ |
283 | list_add_tail(new: &cvd->node, head: &pc->queue); |
284 | |
285 | /* start the dma engine */ |
286 | if (trigger_engine) |
287 | mtk_cqdma_start(pc, cvd); |
288 | |
289 | /* remove VD from list desc_issued */ |
290 | list_del(entry: &vd->node); |
291 | } |
292 | } |
293 | |
294 | /* |
295 | * return true if this VC is active, |
296 | * meaning that there are VDs under processing by the PC |
297 | */ |
298 | static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc) |
299 | { |
300 | struct mtk_cqdma_vdesc *cvd; |
301 | |
302 | list_for_each_entry(cvd, &cvc->pc->queue, node) |
303 | if (cvc == to_cqdma_vchan(chan: cvd->ch)) |
304 | return true; |
305 | |
306 | return false; |
307 | } |
308 | |
309 | /* |
310 | * return the pointer of the CVD that is just consumed by the PC |
311 | */ |
312 | static struct mtk_cqdma_vdesc |
313 | *mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc) |
314 | { |
315 | struct mtk_cqdma_vchan *cvc; |
316 | struct mtk_cqdma_vdesc *cvd, *ret = NULL; |
317 | |
318 | /* consume a CVD from PC's queue */ |
319 | cvd = list_first_entry_or_null(&pc->queue, |
320 | struct mtk_cqdma_vdesc, node); |
321 | if (unlikely(!cvd || !cvd->parent)) |
322 | return NULL; |
323 | |
324 | cvc = to_cqdma_vchan(chan: cvd->ch); |
325 | ret = cvd; |
326 | |
327 | /* update residue of the parent CVD */ |
328 | cvd->parent->residue -= cvd->len; |
329 | |
330 | /* delete CVD from PC's queue */ |
331 | list_del(entry: &cvd->node); |
332 | |
333 | spin_lock(lock: &cvc->vc.lock); |
334 | |
335 | /* check whether all the child CVDs completed */ |
336 | if (!cvd->parent->residue) { |
337 | /* add the parent VD into list desc_completed */ |
338 | vchan_cookie_complete(vd: &cvd->parent->vd); |
339 | |
340 | /* setup completion if this VC is under synchronization */ |
341 | if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) { |
342 | complete(&cvc->issue_completion); |
343 | cvc->issue_synchronize = false; |
344 | } |
345 | } |
346 | |
347 | spin_unlock(lock: &cvc->vc.lock); |
348 | |
349 | /* start transaction for next CVD in the queue */ |
350 | cvd = list_first_entry_or_null(&pc->queue, |
351 | struct mtk_cqdma_vdesc, node); |
352 | if (cvd) |
353 | mtk_cqdma_start(pc, cvd); |
354 | |
355 | return ret; |
356 | } |
357 | |
358 | static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t) |
359 | { |
360 | struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet); |
361 | struct mtk_cqdma_vdesc *cvd = NULL; |
362 | unsigned long flags; |
363 | |
364 | spin_lock_irqsave(&pc->lock, flags); |
365 | /* consume the queue */ |
366 | cvd = mtk_cqdma_consume_work_queue(pc); |
367 | spin_unlock_irqrestore(lock: &pc->lock, flags); |
368 | |
369 | /* submit the next CVD */ |
370 | if (cvd) { |
371 | dma_run_dependencies(tx: &cvd->vd.tx); |
372 | |
373 | /* |
374 | * free child CVD after completion. |
375 | * the parent CVD would be freed with desc_free by user. |
376 | */ |
377 | if (cvd->parent != cvd) |
378 | kfree(objp: cvd); |
379 | } |
380 | |
381 | /* re-enable interrupt before leaving tasklet */ |
382 | enable_irq(irq: pc->irq); |
383 | } |
384 | |
385 | static irqreturn_t mtk_cqdma_irq(int irq, void *devid) |
386 | { |
387 | struct mtk_cqdma_device *cqdma = devid; |
388 | irqreturn_t ret = IRQ_NONE; |
389 | bool schedule_tasklet = false; |
390 | u32 i; |
391 | |
392 | /* clear interrupt flags for each PC */ |
393 | for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) { |
394 | spin_lock(lock: &cqdma->pc[i]->lock); |
395 | if (mtk_dma_read(pc: cqdma->pc[i], |
396 | MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) { |
397 | /* clear interrupt */ |
398 | mtk_dma_clr(pc: cqdma->pc[i], MTK_CQDMA_INT_FLAG, |
399 | MTK_CQDMA_INT_FLAG_BIT); |
400 | |
401 | schedule_tasklet = true; |
402 | ret = IRQ_HANDLED; |
403 | } |
404 | spin_unlock(lock: &cqdma->pc[i]->lock); |
405 | |
406 | if (schedule_tasklet) { |
407 | /* disable interrupt */ |
408 | disable_irq_nosync(irq: cqdma->pc[i]->irq); |
409 | |
410 | /* schedule the tasklet to handle the transactions */ |
411 | tasklet_schedule(t: &cqdma->pc[i]->tasklet); |
412 | } |
413 | } |
414 | |
415 | return ret; |
416 | } |
417 | |
418 | static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c, |
419 | dma_cookie_t cookie) |
420 | { |
421 | struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(chan: c); |
422 | struct virt_dma_desc *vd; |
423 | unsigned long flags; |
424 | |
425 | spin_lock_irqsave(&cvc->pc->lock, flags); |
426 | list_for_each_entry(vd, &cvc->pc->queue, node) |
427 | if (vd->tx.cookie == cookie) { |
428 | spin_unlock_irqrestore(lock: &cvc->pc->lock, flags); |
429 | return vd; |
430 | } |
431 | spin_unlock_irqrestore(lock: &cvc->pc->lock, flags); |
432 | |
433 | list_for_each_entry(vd, &cvc->vc.desc_issued, node) |
434 | if (vd->tx.cookie == cookie) |
435 | return vd; |
436 | |
437 | return NULL; |
438 | } |
439 | |
440 | static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c, |
441 | dma_cookie_t cookie, |
442 | struct dma_tx_state *txstate) |
443 | { |
444 | struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(chan: c); |
445 | struct mtk_cqdma_vdesc *cvd; |
446 | struct virt_dma_desc *vd; |
447 | enum dma_status ret; |
448 | unsigned long flags; |
449 | size_t bytes = 0; |
450 | |
451 | ret = dma_cookie_status(chan: c, cookie, state: txstate); |
452 | if (ret == DMA_COMPLETE || !txstate) |
453 | return ret; |
454 | |
455 | spin_lock_irqsave(&cvc->vc.lock, flags); |
456 | vd = mtk_cqdma_find_active_desc(c, cookie); |
457 | spin_unlock_irqrestore(lock: &cvc->vc.lock, flags); |
458 | |
459 | if (vd) { |
460 | cvd = to_cqdma_vdesc(vd); |
461 | bytes = cvd->residue; |
462 | } |
463 | |
464 | dma_set_residue(state: txstate, residue: bytes); |
465 | |
466 | return ret; |
467 | } |
468 | |
469 | static void mtk_cqdma_issue_pending(struct dma_chan *c) |
470 | { |
471 | struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(chan: c); |
472 | unsigned long pc_flags; |
473 | unsigned long vc_flags; |
474 | |
475 | /* acquire PC's lock before VS's lock for lock dependency in tasklet */ |
476 | spin_lock_irqsave(&cvc->pc->lock, pc_flags); |
477 | spin_lock_irqsave(&cvc->vc.lock, vc_flags); |
478 | |
479 | if (vchan_issue_pending(vc: &cvc->vc)) |
480 | mtk_cqdma_issue_vchan_pending(cvc); |
481 | |
482 | spin_unlock_irqrestore(lock: &cvc->vc.lock, flags: vc_flags); |
483 | spin_unlock_irqrestore(lock: &cvc->pc->lock, flags: pc_flags); |
484 | } |
485 | |
486 | static struct dma_async_tx_descriptor * |
487 | mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, |
488 | dma_addr_t src, size_t len, unsigned long flags) |
489 | { |
490 | struct mtk_cqdma_vdesc **cvd; |
491 | struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL; |
492 | size_t i, tlen, nr_vd; |
493 | |
494 | /* |
495 | * In the case that trsanction length is larger than the |
496 | * DMA engine supports, a single memcpy transaction needs |
497 | * to be separated into several DMA transactions. |
498 | * Each DMA transaction would be described by a CVD, |
499 | * and the first one is referred as the parent CVD, |
500 | * while the others are child CVDs. |
501 | * The parent CVD's tx descriptor is the only tx descriptor |
502 | * returned to the DMA user, and it should not be completed |
503 | * until all the child CVDs completed. |
504 | */ |
505 | nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN); |
506 | cvd = kcalloc(n: nr_vd, size: sizeof(*cvd), GFP_NOWAIT); |
507 | if (!cvd) |
508 | return NULL; |
509 | |
510 | for (i = 0; i < nr_vd; ++i) { |
511 | cvd[i] = kzalloc(size: sizeof(*cvd[i]), GFP_NOWAIT); |
512 | if (!cvd[i]) { |
513 | for (; i > 0; --i) |
514 | kfree(objp: cvd[i - 1]); |
515 | return NULL; |
516 | } |
517 | |
518 | /* setup dma channel */ |
519 | cvd[i]->ch = c; |
520 | |
521 | /* setup sourece, destination, and length */ |
522 | tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len; |
523 | cvd[i]->len = tlen; |
524 | cvd[i]->src = src; |
525 | cvd[i]->dest = dest; |
526 | |
527 | /* setup tx descriptor */ |
528 | tx = vchan_tx_prep(vc: to_virt_chan(chan: c), vd: &cvd[i]->vd, tx_flags: flags); |
529 | tx->next = NULL; |
530 | |
531 | if (!i) { |
532 | cvd[0]->residue = len; |
533 | } else { |
534 | prev_tx->next = tx; |
535 | cvd[i]->residue = tlen; |
536 | } |
537 | |
538 | cvd[i]->parent = cvd[0]; |
539 | |
540 | /* update the src, dest, len, prev_tx for the next CVD */ |
541 | src += tlen; |
542 | dest += tlen; |
543 | len -= tlen; |
544 | prev_tx = tx; |
545 | } |
546 | |
547 | return &cvd[0]->vd.tx; |
548 | } |
549 | |
550 | static void mtk_cqdma_free_inactive_desc(struct dma_chan *c) |
551 | { |
552 | struct virt_dma_chan *vc = to_virt_chan(chan: c); |
553 | unsigned long flags; |
554 | LIST_HEAD(head); |
555 | |
556 | /* |
557 | * set desc_allocated, desc_submitted, |
558 | * and desc_issued as the candicates to be freed |
559 | */ |
560 | spin_lock_irqsave(&vc->lock, flags); |
561 | list_splice_tail_init(list: &vc->desc_allocated, head: &head); |
562 | list_splice_tail_init(list: &vc->desc_submitted, head: &head); |
563 | list_splice_tail_init(list: &vc->desc_issued, head: &head); |
564 | spin_unlock_irqrestore(lock: &vc->lock, flags); |
565 | |
566 | /* free descriptor lists */ |
567 | vchan_dma_desc_free_list(vc, head: &head); |
568 | } |
569 | |
570 | static void mtk_cqdma_free_active_desc(struct dma_chan *c) |
571 | { |
572 | struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(chan: c); |
573 | bool sync_needed = false; |
574 | unsigned long pc_flags; |
575 | unsigned long vc_flags; |
576 | |
577 | /* acquire PC's lock first due to lock dependency in dma ISR */ |
578 | spin_lock_irqsave(&cvc->pc->lock, pc_flags); |
579 | spin_lock_irqsave(&cvc->vc.lock, vc_flags); |
580 | |
581 | /* synchronization is required if this VC is active */ |
582 | if (mtk_cqdma_is_vchan_active(cvc)) { |
583 | cvc->issue_synchronize = true; |
584 | sync_needed = true; |
585 | } |
586 | |
587 | spin_unlock_irqrestore(lock: &cvc->vc.lock, flags: vc_flags); |
588 | spin_unlock_irqrestore(lock: &cvc->pc->lock, flags: pc_flags); |
589 | |
590 | /* waiting for the completion of this VC */ |
591 | if (sync_needed) |
592 | wait_for_completion(&cvc->issue_completion); |
593 | |
594 | /* free all descriptors in list desc_completed */ |
595 | vchan_synchronize(vc: &cvc->vc); |
596 | |
597 | WARN_ONCE(!list_empty(&cvc->vc.desc_completed), |
598 | "Desc pending still in list desc_completed\n" ); |
599 | } |
600 | |
601 | static int mtk_cqdma_terminate_all(struct dma_chan *c) |
602 | { |
603 | /* free descriptors not processed yet by hardware */ |
604 | mtk_cqdma_free_inactive_desc(c); |
605 | |
606 | /* free descriptors being processed by hardware */ |
607 | mtk_cqdma_free_active_desc(c); |
608 | |
609 | return 0; |
610 | } |
611 | |
612 | static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c) |
613 | { |
614 | struct mtk_cqdma_device *cqdma = to_cqdma_dev(chan: c); |
615 | struct mtk_cqdma_vchan *vc = to_cqdma_vchan(chan: c); |
616 | struct mtk_cqdma_pchan *pc = NULL; |
617 | u32 i, min_refcnt = U32_MAX, refcnt; |
618 | unsigned long flags; |
619 | |
620 | /* allocate PC with the minimun refcount */ |
621 | for (i = 0; i < cqdma->dma_channels; ++i) { |
622 | refcnt = refcount_read(r: &cqdma->pc[i]->refcnt); |
623 | if (refcnt < min_refcnt) { |
624 | pc = cqdma->pc[i]; |
625 | min_refcnt = refcnt; |
626 | } |
627 | } |
628 | |
629 | if (!pc) |
630 | return -ENOSPC; |
631 | |
632 | spin_lock_irqsave(&pc->lock, flags); |
633 | |
634 | if (!refcount_read(r: &pc->refcnt)) { |
635 | /* allocate PC when the refcount is zero */ |
636 | mtk_cqdma_hard_reset(pc); |
637 | |
638 | /* enable interrupt for this PC */ |
639 | mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT); |
640 | |
641 | /* |
642 | * refcount_inc would complain increment on 0; use-after-free. |
643 | * Thus, we need to explicitly set it as 1 initially. |
644 | */ |
645 | refcount_set(r: &pc->refcnt, n: 1); |
646 | } else { |
647 | refcount_inc(r: &pc->refcnt); |
648 | } |
649 | |
650 | spin_unlock_irqrestore(lock: &pc->lock, flags); |
651 | |
652 | vc->pc = pc; |
653 | |
654 | return 0; |
655 | } |
656 | |
657 | static void mtk_cqdma_free_chan_resources(struct dma_chan *c) |
658 | { |
659 | struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(chan: c); |
660 | unsigned long flags; |
661 | |
662 | /* free all descriptors in all lists on the VC */ |
663 | mtk_cqdma_terminate_all(c); |
664 | |
665 | spin_lock_irqsave(&cvc->pc->lock, flags); |
666 | |
667 | /* PC is not freed until there is no VC mapped to it */ |
668 | if (refcount_dec_and_test(r: &cvc->pc->refcnt)) { |
669 | /* start the flush operation and stop the engine */ |
670 | mtk_dma_set(pc: cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT); |
671 | |
672 | /* wait for the completion of flush operation */ |
673 | if (mtk_cqdma_poll_engine_done(pc: cvc->pc, atomic: true) < 0) |
674 | dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n" ); |
675 | |
676 | /* clear the flush bit and interrupt flag */ |
677 | mtk_dma_clr(pc: cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT); |
678 | mtk_dma_clr(pc: cvc->pc, MTK_CQDMA_INT_FLAG, |
679 | MTK_CQDMA_INT_FLAG_BIT); |
680 | |
681 | /* disable interrupt for this PC */ |
682 | mtk_dma_clr(pc: cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT); |
683 | } |
684 | |
685 | spin_unlock_irqrestore(lock: &cvc->pc->lock, flags); |
686 | } |
687 | |
688 | static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma) |
689 | { |
690 | unsigned long flags; |
691 | int err; |
692 | u32 i; |
693 | |
694 | pm_runtime_enable(dev: cqdma2dev(cqdma)); |
695 | pm_runtime_get_sync(dev: cqdma2dev(cqdma)); |
696 | |
697 | err = clk_prepare_enable(clk: cqdma->clk); |
698 | |
699 | if (err) { |
700 | pm_runtime_put_sync(dev: cqdma2dev(cqdma)); |
701 | pm_runtime_disable(dev: cqdma2dev(cqdma)); |
702 | return err; |
703 | } |
704 | |
705 | /* reset all PCs */ |
706 | for (i = 0; i < cqdma->dma_channels; ++i) { |
707 | spin_lock_irqsave(&cqdma->pc[i]->lock, flags); |
708 | if (mtk_cqdma_hard_reset(pc: cqdma->pc[i]) < 0) { |
709 | dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n" ); |
710 | spin_unlock_irqrestore(lock: &cqdma->pc[i]->lock, flags); |
711 | |
712 | clk_disable_unprepare(clk: cqdma->clk); |
713 | pm_runtime_put_sync(dev: cqdma2dev(cqdma)); |
714 | pm_runtime_disable(dev: cqdma2dev(cqdma)); |
715 | return -EINVAL; |
716 | } |
717 | spin_unlock_irqrestore(lock: &cqdma->pc[i]->lock, flags); |
718 | } |
719 | |
720 | return 0; |
721 | } |
722 | |
723 | static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma) |
724 | { |
725 | unsigned long flags; |
726 | u32 i; |
727 | |
728 | /* reset all PCs */ |
729 | for (i = 0; i < cqdma->dma_channels; ++i) { |
730 | spin_lock_irqsave(&cqdma->pc[i]->lock, flags); |
731 | if (mtk_cqdma_hard_reset(pc: cqdma->pc[i]) < 0) |
732 | dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n" ); |
733 | spin_unlock_irqrestore(lock: &cqdma->pc[i]->lock, flags); |
734 | } |
735 | |
736 | clk_disable_unprepare(clk: cqdma->clk); |
737 | |
738 | pm_runtime_put_sync(dev: cqdma2dev(cqdma)); |
739 | pm_runtime_disable(dev: cqdma2dev(cqdma)); |
740 | } |
741 | |
742 | static const struct of_device_id mtk_cqdma_match[] = { |
743 | { .compatible = "mediatek,mt6765-cqdma" }, |
744 | { /* sentinel */ } |
745 | }; |
746 | MODULE_DEVICE_TABLE(of, mtk_cqdma_match); |
747 | |
748 | static int mtk_cqdma_probe(struct platform_device *pdev) |
749 | { |
750 | struct mtk_cqdma_device *cqdma; |
751 | struct mtk_cqdma_vchan *vc; |
752 | struct dma_device *dd; |
753 | int err; |
754 | u32 i; |
755 | |
756 | cqdma = devm_kzalloc(dev: &pdev->dev, size: sizeof(*cqdma), GFP_KERNEL); |
757 | if (!cqdma) |
758 | return -ENOMEM; |
759 | |
760 | dd = &cqdma->ddev; |
761 | |
762 | cqdma->clk = devm_clk_get(dev: &pdev->dev, id: "cqdma" ); |
763 | if (IS_ERR(ptr: cqdma->clk)) { |
764 | dev_err(&pdev->dev, "No clock for %s\n" , |
765 | dev_name(&pdev->dev)); |
766 | return PTR_ERR(ptr: cqdma->clk); |
767 | } |
768 | |
769 | dma_cap_set(DMA_MEMCPY, dd->cap_mask); |
770 | |
771 | dd->copy_align = MTK_CQDMA_ALIGN_SIZE; |
772 | dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources; |
773 | dd->device_free_chan_resources = mtk_cqdma_free_chan_resources; |
774 | dd->device_tx_status = mtk_cqdma_tx_status; |
775 | dd->device_issue_pending = mtk_cqdma_issue_pending; |
776 | dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy; |
777 | dd->device_terminate_all = mtk_cqdma_terminate_all; |
778 | dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS; |
779 | dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS; |
780 | dd->directions = BIT(DMA_MEM_TO_MEM); |
781 | dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; |
782 | dd->dev = &pdev->dev; |
783 | INIT_LIST_HEAD(list: &dd->channels); |
784 | |
785 | if (pdev->dev.of_node && of_property_read_u32(np: pdev->dev.of_node, |
786 | propname: "dma-requests" , |
787 | out_value: &cqdma->dma_requests)) { |
788 | dev_info(&pdev->dev, |
789 | "Using %u as missing dma-requests property\n" , |
790 | MTK_CQDMA_NR_VCHANS); |
791 | |
792 | cqdma->dma_requests = MTK_CQDMA_NR_VCHANS; |
793 | } |
794 | |
795 | if (pdev->dev.of_node && of_property_read_u32(np: pdev->dev.of_node, |
796 | propname: "dma-channels" , |
797 | out_value: &cqdma->dma_channels)) { |
798 | dev_info(&pdev->dev, |
799 | "Using %u as missing dma-channels property\n" , |
800 | MTK_CQDMA_NR_PCHANS); |
801 | |
802 | cqdma->dma_channels = MTK_CQDMA_NR_PCHANS; |
803 | } |
804 | |
805 | cqdma->pc = devm_kcalloc(dev: &pdev->dev, n: cqdma->dma_channels, |
806 | size: sizeof(*cqdma->pc), GFP_KERNEL); |
807 | if (!cqdma->pc) |
808 | return -ENOMEM; |
809 | |
810 | /* initialization for PCs */ |
811 | for (i = 0; i < cqdma->dma_channels; ++i) { |
812 | cqdma->pc[i] = devm_kcalloc(dev: &pdev->dev, n: 1, |
813 | size: sizeof(**cqdma->pc), GFP_KERNEL); |
814 | if (!cqdma->pc[i]) |
815 | return -ENOMEM; |
816 | |
817 | INIT_LIST_HEAD(list: &cqdma->pc[i]->queue); |
818 | spin_lock_init(&cqdma->pc[i]->lock); |
819 | refcount_set(r: &cqdma->pc[i]->refcnt, n: 0); |
820 | cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, index: i); |
821 | if (IS_ERR(ptr: cqdma->pc[i]->base)) |
822 | return PTR_ERR(ptr: cqdma->pc[i]->base); |
823 | |
824 | /* allocate IRQ resource */ |
825 | err = platform_get_irq(pdev, i); |
826 | if (err < 0) |
827 | return err; |
828 | cqdma->pc[i]->irq = err; |
829 | |
830 | err = devm_request_irq(dev: &pdev->dev, irq: cqdma->pc[i]->irq, |
831 | handler: mtk_cqdma_irq, irqflags: 0, devname: dev_name(dev: &pdev->dev), |
832 | dev_id: cqdma); |
833 | if (err) { |
834 | dev_err(&pdev->dev, |
835 | "request_irq failed with err %d\n" , err); |
836 | return -EINVAL; |
837 | } |
838 | } |
839 | |
840 | /* allocate resource for VCs */ |
841 | cqdma->vc = devm_kcalloc(dev: &pdev->dev, n: cqdma->dma_requests, |
842 | size: sizeof(*cqdma->vc), GFP_KERNEL); |
843 | if (!cqdma->vc) |
844 | return -ENOMEM; |
845 | |
846 | for (i = 0; i < cqdma->dma_requests; i++) { |
847 | vc = &cqdma->vc[i]; |
848 | vc->vc.desc_free = mtk_cqdma_vdesc_free; |
849 | vchan_init(vc: &vc->vc, dmadev: dd); |
850 | init_completion(x: &vc->issue_completion); |
851 | } |
852 | |
853 | err = dma_async_device_register(device: dd); |
854 | if (err) |
855 | return err; |
856 | |
857 | err = of_dma_controller_register(np: pdev->dev.of_node, |
858 | of_dma_xlate: of_dma_xlate_by_chan_id, data: cqdma); |
859 | if (err) { |
860 | dev_err(&pdev->dev, |
861 | "MediaTek CQDMA OF registration failed %d\n" , err); |
862 | goto err_unregister; |
863 | } |
864 | |
865 | err = mtk_cqdma_hw_init(cqdma); |
866 | if (err) { |
867 | dev_err(&pdev->dev, |
868 | "MediaTek CQDMA HW initialization failed %d\n" , err); |
869 | goto err_unregister; |
870 | } |
871 | |
872 | platform_set_drvdata(pdev, data: cqdma); |
873 | |
874 | /* initialize tasklet for each PC */ |
875 | for (i = 0; i < cqdma->dma_channels; ++i) |
876 | tasklet_setup(t: &cqdma->pc[i]->tasklet, callback: mtk_cqdma_tasklet_cb); |
877 | |
878 | dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n" ); |
879 | |
880 | return 0; |
881 | |
882 | err_unregister: |
883 | dma_async_device_unregister(device: dd); |
884 | |
885 | return err; |
886 | } |
887 | |
888 | static void mtk_cqdma_remove(struct platform_device *pdev) |
889 | { |
890 | struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev); |
891 | struct mtk_cqdma_vchan *vc; |
892 | unsigned long flags; |
893 | int i; |
894 | |
895 | /* kill VC task */ |
896 | for (i = 0; i < cqdma->dma_requests; i++) { |
897 | vc = &cqdma->vc[i]; |
898 | |
899 | list_del(entry: &vc->vc.chan.device_node); |
900 | tasklet_kill(t: &vc->vc.task); |
901 | } |
902 | |
903 | /* disable interrupt */ |
904 | for (i = 0; i < cqdma->dma_channels; i++) { |
905 | spin_lock_irqsave(&cqdma->pc[i]->lock, flags); |
906 | mtk_dma_clr(pc: cqdma->pc[i], MTK_CQDMA_INT_EN, |
907 | MTK_CQDMA_INT_EN_BIT); |
908 | spin_unlock_irqrestore(lock: &cqdma->pc[i]->lock, flags); |
909 | |
910 | /* Waits for any pending IRQ handlers to complete */ |
911 | synchronize_irq(irq: cqdma->pc[i]->irq); |
912 | |
913 | tasklet_kill(t: &cqdma->pc[i]->tasklet); |
914 | } |
915 | |
916 | /* disable hardware */ |
917 | mtk_cqdma_hw_deinit(cqdma); |
918 | |
919 | dma_async_device_unregister(device: &cqdma->ddev); |
920 | of_dma_controller_free(np: pdev->dev.of_node); |
921 | } |
922 | |
923 | static struct platform_driver mtk_cqdma_driver = { |
924 | .probe = mtk_cqdma_probe, |
925 | .remove_new = mtk_cqdma_remove, |
926 | .driver = { |
927 | .name = KBUILD_MODNAME, |
928 | .of_match_table = mtk_cqdma_match, |
929 | }, |
930 | }; |
931 | module_platform_driver(mtk_cqdma_driver); |
932 | |
933 | MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver" ); |
934 | MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>" ); |
935 | MODULE_LICENSE("GPL v2" ); |
936 | |