1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2017-2018 MediaTek Inc.
3
4/*
5 * Driver for MediaTek High-Speed DMA Controller
6 *
7 * Author: Sean Wang <sean.wang@mediatek.com>
8 *
9 */
10
11#include <linux/bitops.h>
12#include <linux/clk.h>
13#include <linux/dmaengine.h>
14#include <linux/dma-mapping.h>
15#include <linux/err.h>
16#include <linux/iopoll.h>
17#include <linux/list.h>
18#include <linux/module.h>
19#include <linux/of.h>
20#include <linux/of_dma.h>
21#include <linux/platform_device.h>
22#include <linux/pm_runtime.h>
23#include <linux/refcount.h>
24#include <linux/slab.h>
25
26#include "../virt-dma.h"
27
28#define MTK_HSDMA_USEC_POLL 20
29#define MTK_HSDMA_TIMEOUT_POLL 200000
30#define MTK_HSDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
31
32/* The default number of virtual channel */
33#define MTK_HSDMA_NR_VCHANS 3
34
35/* Only one physical channel supported */
36#define MTK_HSDMA_NR_MAX_PCHANS 1
37
38/* Macro for physical descriptor (PD) manipulation */
39/* The number of PD which must be 2 of power */
40#define MTK_DMA_SIZE 64
41#define MTK_HSDMA_NEXT_DESP_IDX(x, y) (((x) + 1) & ((y) - 1))
42#define MTK_HSDMA_LAST_DESP_IDX(x, y) (((x) - 1) & ((y) - 1))
43#define MTK_HSDMA_MAX_LEN 0x3f80
44#define MTK_HSDMA_ALIGN_SIZE 4
45#define MTK_HSDMA_PLEN_MASK 0x3fff
46#define MTK_HSDMA_DESC_PLEN(x) (((x) & MTK_HSDMA_PLEN_MASK) << 16)
47#define MTK_HSDMA_DESC_PLEN_GET(x) (((x) >> 16) & MTK_HSDMA_PLEN_MASK)
48
49/* Registers for underlying ring manipulation */
50#define MTK_HSDMA_TX_BASE 0x0
51#define MTK_HSDMA_TX_CNT 0x4
52#define MTK_HSDMA_TX_CPU 0x8
53#define MTK_HSDMA_TX_DMA 0xc
54#define MTK_HSDMA_RX_BASE 0x100
55#define MTK_HSDMA_RX_CNT 0x104
56#define MTK_HSDMA_RX_CPU 0x108
57#define MTK_HSDMA_RX_DMA 0x10c
58
59/* Registers for global setup */
60#define MTK_HSDMA_GLO 0x204
61#define MTK_HSDMA_GLO_MULTI_DMA BIT(10)
62#define MTK_HSDMA_TX_WB_DDONE BIT(6)
63#define MTK_HSDMA_BURST_64BYTES (0x2 << 4)
64#define MTK_HSDMA_GLO_RX_BUSY BIT(3)
65#define MTK_HSDMA_GLO_RX_DMA BIT(2)
66#define MTK_HSDMA_GLO_TX_BUSY BIT(1)
67#define MTK_HSDMA_GLO_TX_DMA BIT(0)
68#define MTK_HSDMA_GLO_DMA (MTK_HSDMA_GLO_TX_DMA | \
69 MTK_HSDMA_GLO_RX_DMA)
70#define MTK_HSDMA_GLO_BUSY (MTK_HSDMA_GLO_RX_BUSY | \
71 MTK_HSDMA_GLO_TX_BUSY)
72#define MTK_HSDMA_GLO_DEFAULT (MTK_HSDMA_GLO_TX_DMA | \
73 MTK_HSDMA_GLO_RX_DMA | \
74 MTK_HSDMA_TX_WB_DDONE | \
75 MTK_HSDMA_BURST_64BYTES | \
76 MTK_HSDMA_GLO_MULTI_DMA)
77
78/* Registers for reset */
79#define MTK_HSDMA_RESET 0x208
80#define MTK_HSDMA_RST_TX BIT(0)
81#define MTK_HSDMA_RST_RX BIT(16)
82
83/* Registers for interrupt control */
84#define MTK_HSDMA_DLYINT 0x20c
85#define MTK_HSDMA_RXDLY_INT_EN BIT(15)
86
87/* Interrupt fires when the pending number's more than the specified */
88#define MTK_HSDMA_RXMAX_PINT(x) (((x) & 0x7f) << 8)
89
90/* Interrupt fires when the pending time's more than the specified in 20 us */
91#define MTK_HSDMA_RXMAX_PTIME(x) ((x) & 0x7f)
92#define MTK_HSDMA_DLYINT_DEFAULT (MTK_HSDMA_RXDLY_INT_EN | \
93 MTK_HSDMA_RXMAX_PINT(20) | \
94 MTK_HSDMA_RXMAX_PTIME(20))
95#define MTK_HSDMA_INT_STATUS 0x220
96#define MTK_HSDMA_INT_ENABLE 0x228
97#define MTK_HSDMA_INT_RXDONE BIT(16)
98
99enum mtk_hsdma_vdesc_flag {
100 MTK_HSDMA_VDESC_FINISHED = 0x01,
101};
102
103#define IS_MTK_HSDMA_VDESC_FINISHED(x) ((x) == MTK_HSDMA_VDESC_FINISHED)
104
105/**
106 * struct mtk_hsdma_pdesc - This is the struct holding info describing physical
107 * descriptor (PD) and its placement must be kept at
108 * 4-bytes alignment in little endian order.
109 * @desc1: | The control pad used to indicate hardware how to
110 * @desc2: | deal with the descriptor such as source and
111 * @desc3: | destination address and data length. The maximum
112 * @desc4: | data length each pdesc can handle is 0x3f80 bytes
113 */
114struct mtk_hsdma_pdesc {
115 __le32 desc1;
116 __le32 desc2;
117 __le32 desc3;
118 __le32 desc4;
119} __packed __aligned(4);
120
121/**
122 * struct mtk_hsdma_vdesc - This is the struct holding info describing virtual
123 * descriptor (VD)
124 * @vd: An instance for struct virt_dma_desc
125 * @len: The total data size device wants to move
126 * @residue: The remaining data size device will move
127 * @dest: The destination address device wants to move to
128 * @src: The source address device wants to move from
129 */
130struct mtk_hsdma_vdesc {
131 struct virt_dma_desc vd;
132 size_t len;
133 size_t residue;
134 dma_addr_t dest;
135 dma_addr_t src;
136};
137
138/**
139 * struct mtk_hsdma_cb - This is the struct holding extra info required for RX
140 * ring to know what relevant VD the PD is being
141 * mapped to.
142 * @vd: Pointer to the relevant VD.
143 * @flag: Flag indicating what action should be taken when VD
144 * is completed.
145 */
146struct mtk_hsdma_cb {
147 struct virt_dma_desc *vd;
148 enum mtk_hsdma_vdesc_flag flag;
149};
150
151/**
152 * struct mtk_hsdma_ring - This struct holds info describing underlying ring
153 * space
154 * @txd: The descriptor TX ring which describes DMA source
155 * information
156 * @rxd: The descriptor RX ring which describes DMA
157 * destination information
158 * @cb: The extra information pointed at by RX ring
159 * @tphys: The physical addr of TX ring
160 * @rphys: The physical addr of RX ring
161 * @cur_tptr: Pointer to the next free descriptor used by the host
162 * @cur_rptr: Pointer to the last done descriptor by the device
163 */
164struct mtk_hsdma_ring {
165 struct mtk_hsdma_pdesc *txd;
166 struct mtk_hsdma_pdesc *rxd;
167 struct mtk_hsdma_cb *cb;
168 dma_addr_t tphys;
169 dma_addr_t rphys;
170 u16 cur_tptr;
171 u16 cur_rptr;
172};
173
174/**
175 * struct mtk_hsdma_pchan - This is the struct holding info describing physical
176 * channel (PC)
177 * @ring: An instance for the underlying ring
178 * @sz_ring: Total size allocated for the ring
179 * @nr_free: Total number of free rooms in the ring. It would
180 * be accessed and updated frequently between IRQ
181 * context and user context to reflect whether ring
182 * can accept requests from VD.
183 */
184struct mtk_hsdma_pchan {
185 struct mtk_hsdma_ring ring;
186 size_t sz_ring;
187 atomic_t nr_free;
188};
189
190/**
191 * struct mtk_hsdma_vchan - This is the struct holding info describing virtual
192 * channel (VC)
193 * @vc: An instance for struct virt_dma_chan
194 * @issue_completion: The wait for all issued descriptors completited
195 * @issue_synchronize: Bool indicating channel synchronization starts
196 * @desc_hw_processing: List those descriptors the hardware is processing,
197 * which is protected by vc.lock
198 */
199struct mtk_hsdma_vchan {
200 struct virt_dma_chan vc;
201 struct completion issue_completion;
202 bool issue_synchronize;
203 struct list_head desc_hw_processing;
204};
205
206/**
207 * struct mtk_hsdma_soc - This is the struct holding differences among SoCs
208 * @ddone: Bit mask for DDONE
209 * @ls0: Bit mask for LS0
210 */
211struct mtk_hsdma_soc {
212 __le32 ddone;
213 __le32 ls0;
214};
215
216/**
217 * struct mtk_hsdma_device - This is the struct holding info describing HSDMA
218 * device
219 * @ddev: An instance for struct dma_device
220 * @base: The mapped register I/O base
221 * @clk: The clock that device internal is using
222 * @irq: The IRQ that device are using
223 * @dma_requests: The number of VCs the device supports to
224 * @vc: The pointer to all available VCs
225 * @pc: The pointer to the underlying PC
226 * @pc_refcnt: Track how many VCs are using the PC
227 * @lock: Lock protect agaisting multiple VCs access PC
228 * @soc: The pointer to area holding differences among
229 * vaious platform
230 */
231struct mtk_hsdma_device {
232 struct dma_device ddev;
233 void __iomem *base;
234 struct clk *clk;
235 u32 irq;
236
237 u32 dma_requests;
238 struct mtk_hsdma_vchan *vc;
239 struct mtk_hsdma_pchan *pc;
240 refcount_t pc_refcnt;
241
242 /* Lock used to protect against multiple VCs access PC */
243 spinlock_t lock;
244
245 const struct mtk_hsdma_soc *soc;
246};
247
248static struct mtk_hsdma_device *to_hsdma_dev(struct dma_chan *chan)
249{
250 return container_of(chan->device, struct mtk_hsdma_device, ddev);
251}
252
253static inline struct mtk_hsdma_vchan *to_hsdma_vchan(struct dma_chan *chan)
254{
255 return container_of(chan, struct mtk_hsdma_vchan, vc.chan);
256}
257
258static struct mtk_hsdma_vdesc *to_hsdma_vdesc(struct virt_dma_desc *vd)
259{
260 return container_of(vd, struct mtk_hsdma_vdesc, vd);
261}
262
263static struct device *hsdma2dev(struct mtk_hsdma_device *hsdma)
264{
265 return hsdma->ddev.dev;
266}
267
268static u32 mtk_dma_read(struct mtk_hsdma_device *hsdma, u32 reg)
269{
270 return readl(addr: hsdma->base + reg);
271}
272
273static void mtk_dma_write(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)
274{
275 writel(val, addr: hsdma->base + reg);
276}
277
278static void mtk_dma_rmw(struct mtk_hsdma_device *hsdma, u32 reg,
279 u32 mask, u32 set)
280{
281 u32 val;
282
283 val = mtk_dma_read(hsdma, reg);
284 val &= ~mask;
285 val |= set;
286 mtk_dma_write(hsdma, reg, val);
287}
288
289static void mtk_dma_set(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)
290{
291 mtk_dma_rmw(hsdma, reg, mask: 0, set: val);
292}
293
294static void mtk_dma_clr(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)
295{
296 mtk_dma_rmw(hsdma, reg, mask: val, set: 0);
297}
298
299static void mtk_hsdma_vdesc_free(struct virt_dma_desc *vd)
300{
301 kfree(container_of(vd, struct mtk_hsdma_vdesc, vd));
302}
303
304static int mtk_hsdma_busy_wait(struct mtk_hsdma_device *hsdma)
305{
306 u32 status = 0;
307
308 return readl_poll_timeout(hsdma->base + MTK_HSDMA_GLO, status,
309 !(status & MTK_HSDMA_GLO_BUSY),
310 MTK_HSDMA_USEC_POLL,
311 MTK_HSDMA_TIMEOUT_POLL);
312}
313
314static int mtk_hsdma_alloc_pchan(struct mtk_hsdma_device *hsdma,
315 struct mtk_hsdma_pchan *pc)
316{
317 struct mtk_hsdma_ring *ring = &pc->ring;
318 int err;
319
320 memset(pc, 0, sizeof(*pc));
321
322 /*
323 * Allocate ring space where [0 ... MTK_DMA_SIZE - 1] is for TX ring
324 * and [MTK_DMA_SIZE ... 2 * MTK_DMA_SIZE - 1] is for RX ring.
325 */
326 pc->sz_ring = 2 * MTK_DMA_SIZE * sizeof(*ring->txd);
327 ring->txd = dma_alloc_coherent(dev: hsdma2dev(hsdma), size: pc->sz_ring,
328 dma_handle: &ring->tphys, GFP_NOWAIT);
329 if (!ring->txd)
330 return -ENOMEM;
331
332 ring->rxd = &ring->txd[MTK_DMA_SIZE];
333 ring->rphys = ring->tphys + MTK_DMA_SIZE * sizeof(*ring->txd);
334 ring->cur_tptr = 0;
335 ring->cur_rptr = MTK_DMA_SIZE - 1;
336
337 ring->cb = kcalloc(MTK_DMA_SIZE, size: sizeof(*ring->cb), GFP_NOWAIT);
338 if (!ring->cb) {
339 err = -ENOMEM;
340 goto err_free_dma;
341 }
342
343 atomic_set(v: &pc->nr_free, MTK_DMA_SIZE - 1);
344
345 /* Disable HSDMA and wait for the completion */
346 mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);
347 err = mtk_hsdma_busy_wait(hsdma);
348 if (err)
349 goto err_free_cb;
350
351 /* Reset */
352 mtk_dma_set(hsdma, MTK_HSDMA_RESET,
353 MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX);
354 mtk_dma_clr(hsdma, MTK_HSDMA_RESET,
355 MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX);
356
357 /* Setup HSDMA initial pointer in the ring */
358 mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, val: ring->tphys);
359 mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, MTK_DMA_SIZE);
360 mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, val: ring->cur_tptr);
361 mtk_dma_write(hsdma, MTK_HSDMA_TX_DMA, val: 0);
362 mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, val: ring->rphys);
363 mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, MTK_DMA_SIZE);
364 mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, val: ring->cur_rptr);
365 mtk_dma_write(hsdma, MTK_HSDMA_RX_DMA, val: 0);
366
367 /* Enable HSDMA */
368 mtk_dma_set(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);
369
370 /* Setup delayed interrupt */
371 mtk_dma_write(hsdma, MTK_HSDMA_DLYINT, MTK_HSDMA_DLYINT_DEFAULT);
372
373 /* Enable interrupt */
374 mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
375
376 return 0;
377
378err_free_cb:
379 kfree(objp: ring->cb);
380
381err_free_dma:
382 dma_free_coherent(dev: hsdma2dev(hsdma),
383 size: pc->sz_ring, cpu_addr: ring->txd, dma_handle: ring->tphys);
384 return err;
385}
386
387static void mtk_hsdma_free_pchan(struct mtk_hsdma_device *hsdma,
388 struct mtk_hsdma_pchan *pc)
389{
390 struct mtk_hsdma_ring *ring = &pc->ring;
391
392 /* Disable HSDMA and then wait for the completion */
393 mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);
394 mtk_hsdma_busy_wait(hsdma);
395
396 /* Reset pointer in the ring */
397 mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
398 mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, val: 0);
399 mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, val: 0);
400 mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, val: 0);
401 mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, val: 0);
402 mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, val: 0);
403 mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, MTK_DMA_SIZE - 1);
404
405 kfree(objp: ring->cb);
406
407 dma_free_coherent(dev: hsdma2dev(hsdma),
408 size: pc->sz_ring, cpu_addr: ring->txd, dma_handle: ring->tphys);
409}
410
411static int mtk_hsdma_issue_pending_vdesc(struct mtk_hsdma_device *hsdma,
412 struct mtk_hsdma_pchan *pc,
413 struct mtk_hsdma_vdesc *hvd)
414{
415 struct mtk_hsdma_ring *ring = &pc->ring;
416 struct mtk_hsdma_pdesc *txd, *rxd;
417 u16 reserved, prev, tlen, num_sgs;
418 unsigned long flags;
419
420 /* Protect against PC is accessed by multiple VCs simultaneously */
421 spin_lock_irqsave(&hsdma->lock, flags);
422
423 /*
424 * Reserve rooms, where pc->nr_free is used to track how many free
425 * rooms in the ring being updated in user and IRQ context.
426 */
427 num_sgs = DIV_ROUND_UP(hvd->len, MTK_HSDMA_MAX_LEN);
428 reserved = min_t(u16, num_sgs, atomic_read(&pc->nr_free));
429
430 if (!reserved) {
431 spin_unlock_irqrestore(lock: &hsdma->lock, flags);
432 return -ENOSPC;
433 }
434
435 atomic_sub(i: reserved, v: &pc->nr_free);
436
437 while (reserved--) {
438 /* Limit size by PD capability for valid data moving */
439 tlen = (hvd->len > MTK_HSDMA_MAX_LEN) ?
440 MTK_HSDMA_MAX_LEN : hvd->len;
441
442 /*
443 * Setup PDs using the remaining VD info mapped on those
444 * reserved rooms. And since RXD is shared memory between the
445 * host and the device allocated by dma_alloc_coherent call,
446 * the helper macro WRITE_ONCE can ensure the data written to
447 * RAM would really happens.
448 */
449 txd = &ring->txd[ring->cur_tptr];
450 WRITE_ONCE(txd->desc1, hvd->src);
451 WRITE_ONCE(txd->desc2,
452 hsdma->soc->ls0 | MTK_HSDMA_DESC_PLEN(tlen));
453
454 rxd = &ring->rxd[ring->cur_tptr];
455 WRITE_ONCE(rxd->desc1, hvd->dest);
456 WRITE_ONCE(rxd->desc2, MTK_HSDMA_DESC_PLEN(tlen));
457
458 /* Associate VD, the PD belonged to */
459 ring->cb[ring->cur_tptr].vd = &hvd->vd;
460
461 /* Move forward the pointer of TX ring */
462 ring->cur_tptr = MTK_HSDMA_NEXT_DESP_IDX(ring->cur_tptr,
463 MTK_DMA_SIZE);
464
465 /* Update VD with remaining data */
466 hvd->src += tlen;
467 hvd->dest += tlen;
468 hvd->len -= tlen;
469 }
470
471 /*
472 * Tagging flag for the last PD for VD will be responsible for
473 * completing VD.
474 */
475 if (!hvd->len) {
476 prev = MTK_HSDMA_LAST_DESP_IDX(ring->cur_tptr, MTK_DMA_SIZE);
477 ring->cb[prev].flag = MTK_HSDMA_VDESC_FINISHED;
478 }
479
480 /* Ensure all changes indeed done before we're going on */
481 wmb();
482
483 /*
484 * Updating into hardware the pointer of TX ring lets HSDMA to take
485 * action for those pending PDs.
486 */
487 mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, val: ring->cur_tptr);
488
489 spin_unlock_irqrestore(lock: &hsdma->lock, flags);
490
491 return 0;
492}
493
494static void mtk_hsdma_issue_vchan_pending(struct mtk_hsdma_device *hsdma,
495 struct mtk_hsdma_vchan *hvc)
496{
497 struct virt_dma_desc *vd, *vd2;
498 int err;
499
500 lockdep_assert_held(&hvc->vc.lock);
501
502 list_for_each_entry_safe(vd, vd2, &hvc->vc.desc_issued, node) {
503 struct mtk_hsdma_vdesc *hvd;
504
505 hvd = to_hsdma_vdesc(vd);
506
507 /* Map VD into PC and all VCs shares a single PC */
508 err = mtk_hsdma_issue_pending_vdesc(hsdma, pc: hsdma->pc, hvd);
509
510 /*
511 * Move VD from desc_issued to desc_hw_processing when entire
512 * VD is fit into available PDs. Otherwise, the uncompleted
513 * VDs would stay in list desc_issued and then restart the
514 * processing as soon as possible once underlying ring space
515 * got freed.
516 */
517 if (err == -ENOSPC || hvd->len > 0)
518 break;
519
520 /*
521 * The extra list desc_hw_processing is used because
522 * hardware can't provide sufficient information allowing us
523 * to know what VDs are still working on the underlying ring.
524 * Through the additional list, it can help us to implement
525 * terminate_all, residue calculation and such thing needed
526 * to know detail descriptor status on the hardware.
527 */
528 list_move_tail(list: &vd->node, head: &hvc->desc_hw_processing);
529 }
530}
531
532static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma)
533{
534 struct mtk_hsdma_vchan *hvc;
535 struct mtk_hsdma_pdesc *rxd;
536 struct mtk_hsdma_vdesc *hvd;
537 struct mtk_hsdma_pchan *pc;
538 struct mtk_hsdma_cb *cb;
539 int i = MTK_DMA_SIZE;
540 __le32 desc2;
541 u32 status;
542 u16 next;
543
544 /* Read IRQ status */
545 status = mtk_dma_read(hsdma, MTK_HSDMA_INT_STATUS);
546 if (unlikely(!(status & MTK_HSDMA_INT_RXDONE)))
547 goto rx_done;
548
549 pc = hsdma->pc;
550
551 /*
552 * Using a fail-safe loop with iterations of up to MTK_DMA_SIZE to
553 * reclaim these finished descriptors: The most number of PDs the ISR
554 * can handle at one time shouldn't be more than MTK_DMA_SIZE so we
555 * take it as limited count instead of just using a dangerous infinite
556 * poll.
557 */
558 while (i--) {
559 next = MTK_HSDMA_NEXT_DESP_IDX(pc->ring.cur_rptr,
560 MTK_DMA_SIZE);
561 rxd = &pc->ring.rxd[next];
562
563 /*
564 * If MTK_HSDMA_DESC_DDONE is no specified, that means data
565 * moving for the PD is still under going.
566 */
567 desc2 = READ_ONCE(rxd->desc2);
568 if (!(desc2 & hsdma->soc->ddone))
569 break;
570
571 cb = &pc->ring.cb[next];
572 if (unlikely(!cb->vd)) {
573 dev_err(hsdma2dev(hsdma), "cb->vd cannot be null\n");
574 break;
575 }
576
577 /* Update residue of VD the associated PD belonged to */
578 hvd = to_hsdma_vdesc(vd: cb->vd);
579 hvd->residue -= MTK_HSDMA_DESC_PLEN_GET(rxd->desc2);
580
581 /* Complete VD until the relevant last PD is finished */
582 if (IS_MTK_HSDMA_VDESC_FINISHED(cb->flag)) {
583 hvc = to_hsdma_vchan(chan: cb->vd->tx.chan);
584
585 spin_lock(lock: &hvc->vc.lock);
586
587 /* Remove VD from list desc_hw_processing */
588 list_del(entry: &cb->vd->node);
589
590 /* Add VD into list desc_completed */
591 vchan_cookie_complete(vd: cb->vd);
592
593 if (hvc->issue_synchronize &&
594 list_empty(head: &hvc->desc_hw_processing)) {
595 complete(&hvc->issue_completion);
596 hvc->issue_synchronize = false;
597 }
598 spin_unlock(lock: &hvc->vc.lock);
599
600 cb->flag = 0;
601 }
602
603 cb->vd = NULL;
604
605 /*
606 * Recycle the RXD with the helper WRITE_ONCE that can ensure
607 * data written into RAM would really happens.
608 */
609 WRITE_ONCE(rxd->desc1, 0);
610 WRITE_ONCE(rxd->desc2, 0);
611 pc->ring.cur_rptr = next;
612
613 /* Release rooms */
614 atomic_inc(v: &pc->nr_free);
615 }
616
617 /* Ensure all changes indeed done before we're going on */
618 wmb();
619
620 /* Update CPU pointer for those completed PDs */
621 mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, val: pc->ring.cur_rptr);
622
623 /*
624 * Acking the pending IRQ allows hardware no longer to keep the used
625 * IRQ line in certain trigger state when software has completed all
626 * the finished physical descriptors.
627 */
628 if (atomic_read(v: &pc->nr_free) >= MTK_DMA_SIZE - 1)
629 mtk_dma_write(hsdma, MTK_HSDMA_INT_STATUS, val: status);
630
631 /* ASAP handles pending VDs in all VCs after freeing some rooms */
632 for (i = 0; i < hsdma->dma_requests; i++) {
633 hvc = &hsdma->vc[i];
634 spin_lock(lock: &hvc->vc.lock);
635 mtk_hsdma_issue_vchan_pending(hsdma, hvc);
636 spin_unlock(lock: &hvc->vc.lock);
637 }
638
639rx_done:
640 /* All completed PDs are cleaned up, so enable interrupt again */
641 mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
642}
643
644static irqreturn_t mtk_hsdma_irq(int irq, void *devid)
645{
646 struct mtk_hsdma_device *hsdma = devid;
647
648 /*
649 * Disable interrupt until all completed PDs are cleaned up in
650 * mtk_hsdma_free_rooms call.
651 */
652 mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
653
654 mtk_hsdma_free_rooms_in_ring(hsdma);
655
656 return IRQ_HANDLED;
657}
658
659static struct virt_dma_desc *mtk_hsdma_find_active_desc(struct dma_chan *c,
660 dma_cookie_t cookie)
661{
662 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(chan: c);
663 struct virt_dma_desc *vd;
664
665 list_for_each_entry(vd, &hvc->desc_hw_processing, node)
666 if (vd->tx.cookie == cookie)
667 return vd;
668
669 list_for_each_entry(vd, &hvc->vc.desc_issued, node)
670 if (vd->tx.cookie == cookie)
671 return vd;
672
673 return NULL;
674}
675
676static enum dma_status mtk_hsdma_tx_status(struct dma_chan *c,
677 dma_cookie_t cookie,
678 struct dma_tx_state *txstate)
679{
680 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(chan: c);
681 struct mtk_hsdma_vdesc *hvd;
682 struct virt_dma_desc *vd;
683 enum dma_status ret;
684 unsigned long flags;
685 size_t bytes = 0;
686
687 ret = dma_cookie_status(chan: c, cookie, state: txstate);
688 if (ret == DMA_COMPLETE || !txstate)
689 return ret;
690
691 spin_lock_irqsave(&hvc->vc.lock, flags);
692 vd = mtk_hsdma_find_active_desc(c, cookie);
693 spin_unlock_irqrestore(lock: &hvc->vc.lock, flags);
694
695 if (vd) {
696 hvd = to_hsdma_vdesc(vd);
697 bytes = hvd->residue;
698 }
699
700 dma_set_residue(state: txstate, residue: bytes);
701
702 return ret;
703}
704
705static void mtk_hsdma_issue_pending(struct dma_chan *c)
706{
707 struct mtk_hsdma_device *hsdma = to_hsdma_dev(chan: c);
708 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(chan: c);
709 unsigned long flags;
710
711 spin_lock_irqsave(&hvc->vc.lock, flags);
712
713 if (vchan_issue_pending(vc: &hvc->vc))
714 mtk_hsdma_issue_vchan_pending(hsdma, hvc);
715
716 spin_unlock_irqrestore(lock: &hvc->vc.lock, flags);
717}
718
719static struct dma_async_tx_descriptor *
720mtk_hsdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
721 dma_addr_t src, size_t len, unsigned long flags)
722{
723 struct mtk_hsdma_vdesc *hvd;
724
725 hvd = kzalloc(size: sizeof(*hvd), GFP_NOWAIT);
726 if (!hvd)
727 return NULL;
728
729 hvd->len = len;
730 hvd->residue = len;
731 hvd->src = src;
732 hvd->dest = dest;
733
734 return vchan_tx_prep(vc: to_virt_chan(chan: c), vd: &hvd->vd, tx_flags: flags);
735}
736
737static int mtk_hsdma_free_inactive_desc(struct dma_chan *c)
738{
739 struct virt_dma_chan *vc = to_virt_chan(chan: c);
740 unsigned long flags;
741 LIST_HEAD(head);
742
743 spin_lock_irqsave(&vc->lock, flags);
744 list_splice_tail_init(list: &vc->desc_allocated, head: &head);
745 list_splice_tail_init(list: &vc->desc_submitted, head: &head);
746 list_splice_tail_init(list: &vc->desc_issued, head: &head);
747 spin_unlock_irqrestore(lock: &vc->lock, flags);
748
749 /* At the point, we don't expect users put descriptor into VC again */
750 vchan_dma_desc_free_list(vc, head: &head);
751
752 return 0;
753}
754
755static void mtk_hsdma_free_active_desc(struct dma_chan *c)
756{
757 struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(chan: c);
758 bool sync_needed = false;
759
760 /*
761 * Once issue_synchronize is being set, which means once the hardware
762 * consumes all descriptors for the channel in the ring, the
763 * synchronization must be notified immediately it is completed.
764 */
765 spin_lock(lock: &hvc->vc.lock);
766 if (!list_empty(head: &hvc->desc_hw_processing)) {
767 hvc->issue_synchronize = true;
768 sync_needed = true;
769 }
770 spin_unlock(lock: &hvc->vc.lock);
771
772 if (sync_needed)
773 wait_for_completion(&hvc->issue_completion);
774 /*
775 * At the point, we expect that all remaining descriptors in the ring
776 * for the channel should be all processing done.
777 */
778 WARN_ONCE(!list_empty(&hvc->desc_hw_processing),
779 "Desc pending still in list desc_hw_processing\n");
780
781 /* Free all descriptors in list desc_completed */
782 vchan_synchronize(vc: &hvc->vc);
783
784 WARN_ONCE(!list_empty(&hvc->vc.desc_completed),
785 "Desc pending still in list desc_completed\n");
786}
787
788static int mtk_hsdma_terminate_all(struct dma_chan *c)
789{
790 /*
791 * Free pending descriptors not processed yet by hardware that have
792 * previously been submitted to the channel.
793 */
794 mtk_hsdma_free_inactive_desc(c);
795
796 /*
797 * However, the DMA engine doesn't provide any way to stop these
798 * descriptors being processed currently by hardware. The only way is
799 * to just waiting until these descriptors are all processed completely
800 * through mtk_hsdma_free_active_desc call.
801 */
802 mtk_hsdma_free_active_desc(c);
803
804 return 0;
805}
806
807static int mtk_hsdma_alloc_chan_resources(struct dma_chan *c)
808{
809 struct mtk_hsdma_device *hsdma = to_hsdma_dev(chan: c);
810 int err;
811
812 /*
813 * Since HSDMA has only one PC, the resource for PC is being allocated
814 * when the first VC is being created and the other VCs would run on
815 * the same PC.
816 */
817 if (!refcount_read(r: &hsdma->pc_refcnt)) {
818 err = mtk_hsdma_alloc_pchan(hsdma, pc: hsdma->pc);
819 if (err)
820 return err;
821 /*
822 * refcount_inc would complain increment on 0; use-after-free.
823 * Thus, we need to explicitly set it as 1 initially.
824 */
825 refcount_set(r: &hsdma->pc_refcnt, n: 1);
826 } else {
827 refcount_inc(r: &hsdma->pc_refcnt);
828 }
829
830 return 0;
831}
832
833static void mtk_hsdma_free_chan_resources(struct dma_chan *c)
834{
835 struct mtk_hsdma_device *hsdma = to_hsdma_dev(chan: c);
836
837 /* Free all descriptors in all lists on the VC */
838 mtk_hsdma_terminate_all(c);
839
840 /* The resource for PC is not freed until all the VCs are destroyed */
841 if (!refcount_dec_and_test(r: &hsdma->pc_refcnt))
842 return;
843
844 mtk_hsdma_free_pchan(hsdma, pc: hsdma->pc);
845}
846
847static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma)
848{
849 int err;
850
851 pm_runtime_enable(dev: hsdma2dev(hsdma));
852 pm_runtime_get_sync(dev: hsdma2dev(hsdma));
853
854 err = clk_prepare_enable(clk: hsdma->clk);
855 if (err)
856 return err;
857
858 mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, val: 0);
859 mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT);
860
861 return 0;
862}
863
864static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma)
865{
866 mtk_dma_write(hsdma, MTK_HSDMA_GLO, val: 0);
867
868 clk_disable_unprepare(clk: hsdma->clk);
869
870 pm_runtime_put_sync(dev: hsdma2dev(hsdma));
871 pm_runtime_disable(dev: hsdma2dev(hsdma));
872
873 return 0;
874}
875
876static const struct mtk_hsdma_soc mt7623_soc = {
877 .ddone = BIT(31),
878 .ls0 = BIT(30),
879};
880
881static const struct mtk_hsdma_soc mt7622_soc = {
882 .ddone = BIT(15),
883 .ls0 = BIT(14),
884};
885
886static const struct of_device_id mtk_hsdma_match[] = {
887 { .compatible = "mediatek,mt7623-hsdma", .data = &mt7623_soc},
888 { .compatible = "mediatek,mt7622-hsdma", .data = &mt7622_soc},
889 { /* sentinel */ }
890};
891MODULE_DEVICE_TABLE(of, mtk_hsdma_match);
892
893static int mtk_hsdma_probe(struct platform_device *pdev)
894{
895 struct mtk_hsdma_device *hsdma;
896 struct mtk_hsdma_vchan *vc;
897 struct dma_device *dd;
898 int i, err;
899
900 hsdma = devm_kzalloc(dev: &pdev->dev, size: sizeof(*hsdma), GFP_KERNEL);
901 if (!hsdma)
902 return -ENOMEM;
903
904 dd = &hsdma->ddev;
905
906 hsdma->base = devm_platform_ioremap_resource(pdev, index: 0);
907 if (IS_ERR(ptr: hsdma->base))
908 return PTR_ERR(ptr: hsdma->base);
909
910 hsdma->soc = of_device_get_match_data(dev: &pdev->dev);
911 if (!hsdma->soc) {
912 dev_err(&pdev->dev, "No device match found\n");
913 return -ENODEV;
914 }
915
916 hsdma->clk = devm_clk_get(dev: &pdev->dev, id: "hsdma");
917 if (IS_ERR(ptr: hsdma->clk)) {
918 dev_err(&pdev->dev, "No clock for %s\n",
919 dev_name(&pdev->dev));
920 return PTR_ERR(ptr: hsdma->clk);
921 }
922
923 err = platform_get_irq(pdev, 0);
924 if (err < 0)
925 return err;
926 hsdma->irq = err;
927
928 refcount_set(r: &hsdma->pc_refcnt, n: 0);
929 spin_lock_init(&hsdma->lock);
930
931 dma_cap_set(DMA_MEMCPY, dd->cap_mask);
932
933 dd->copy_align = MTK_HSDMA_ALIGN_SIZE;
934 dd->device_alloc_chan_resources = mtk_hsdma_alloc_chan_resources;
935 dd->device_free_chan_resources = mtk_hsdma_free_chan_resources;
936 dd->device_tx_status = mtk_hsdma_tx_status;
937 dd->device_issue_pending = mtk_hsdma_issue_pending;
938 dd->device_prep_dma_memcpy = mtk_hsdma_prep_dma_memcpy;
939 dd->device_terminate_all = mtk_hsdma_terminate_all;
940 dd->src_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS;
941 dd->dst_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS;
942 dd->directions = BIT(DMA_MEM_TO_MEM);
943 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
944 dd->dev = &pdev->dev;
945 INIT_LIST_HEAD(list: &dd->channels);
946
947 hsdma->dma_requests = MTK_HSDMA_NR_VCHANS;
948 if (pdev->dev.of_node && of_property_read_u32(np: pdev->dev.of_node,
949 propname: "dma-requests",
950 out_value: &hsdma->dma_requests)) {
951 dev_info(&pdev->dev,
952 "Using %u as missing dma-requests property\n",
953 MTK_HSDMA_NR_VCHANS);
954 }
955
956 hsdma->pc = devm_kcalloc(dev: &pdev->dev, MTK_HSDMA_NR_MAX_PCHANS,
957 size: sizeof(*hsdma->pc), GFP_KERNEL);
958 if (!hsdma->pc)
959 return -ENOMEM;
960
961 hsdma->vc = devm_kcalloc(dev: &pdev->dev, n: hsdma->dma_requests,
962 size: sizeof(*hsdma->vc), GFP_KERNEL);
963 if (!hsdma->vc)
964 return -ENOMEM;
965
966 for (i = 0; i < hsdma->dma_requests; i++) {
967 vc = &hsdma->vc[i];
968 vc->vc.desc_free = mtk_hsdma_vdesc_free;
969 vchan_init(vc: &vc->vc, dmadev: dd);
970 init_completion(x: &vc->issue_completion);
971 INIT_LIST_HEAD(list: &vc->desc_hw_processing);
972 }
973
974 err = dma_async_device_register(device: dd);
975 if (err)
976 return err;
977
978 err = of_dma_controller_register(np: pdev->dev.of_node,
979 of_dma_xlate: of_dma_xlate_by_chan_id, data: hsdma);
980 if (err) {
981 dev_err(&pdev->dev,
982 "MediaTek HSDMA OF registration failed %d\n", err);
983 goto err_unregister;
984 }
985
986 mtk_hsdma_hw_init(hsdma);
987
988 err = devm_request_irq(dev: &pdev->dev, irq: hsdma->irq,
989 handler: mtk_hsdma_irq, irqflags: 0,
990 devname: dev_name(dev: &pdev->dev), dev_id: hsdma);
991 if (err) {
992 dev_err(&pdev->dev,
993 "request_irq failed with err %d\n", err);
994 goto err_free;
995 }
996
997 platform_set_drvdata(pdev, data: hsdma);
998
999 dev_info(&pdev->dev, "MediaTek HSDMA driver registered\n");
1000
1001 return 0;
1002
1003err_free:
1004 mtk_hsdma_hw_deinit(hsdma);
1005 of_dma_controller_free(np: pdev->dev.of_node);
1006err_unregister:
1007 dma_async_device_unregister(device: dd);
1008
1009 return err;
1010}
1011
1012static void mtk_hsdma_remove(struct platform_device *pdev)
1013{
1014 struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev);
1015 struct mtk_hsdma_vchan *vc;
1016 int i;
1017
1018 /* Kill VC task */
1019 for (i = 0; i < hsdma->dma_requests; i++) {
1020 vc = &hsdma->vc[i];
1021
1022 list_del(entry: &vc->vc.chan.device_node);
1023 tasklet_kill(t: &vc->vc.task);
1024 }
1025
1026 /* Disable DMA interrupt */
1027 mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, val: 0);
1028
1029 /* Waits for any pending IRQ handlers to complete */
1030 synchronize_irq(irq: hsdma->irq);
1031
1032 /* Disable hardware */
1033 mtk_hsdma_hw_deinit(hsdma);
1034
1035 dma_async_device_unregister(device: &hsdma->ddev);
1036 of_dma_controller_free(np: pdev->dev.of_node);
1037}
1038
1039static struct platform_driver mtk_hsdma_driver = {
1040 .probe = mtk_hsdma_probe,
1041 .remove_new = mtk_hsdma_remove,
1042 .driver = {
1043 .name = KBUILD_MODNAME,
1044 .of_match_table = mtk_hsdma_match,
1045 },
1046};
1047module_platform_driver(mtk_hsdma_driver);
1048
1049MODULE_DESCRIPTION("MediaTek High-Speed DMA Controller Driver");
1050MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");
1051MODULE_LICENSE("GPL v2");
1052

source code of linux/drivers/dma/mediatek/mtk-hsdma.c