1 | // SPDX-License-Identifier: GPL-2.0 |
2 | // Copyright (c) 2017-2018 MediaTek Inc. |
3 | |
4 | /* |
5 | * Driver for MediaTek High-Speed DMA Controller |
6 | * |
7 | * Author: Sean Wang <sean.wang@mediatek.com> |
8 | * |
9 | */ |
10 | |
11 | #include <linux/bitops.h> |
12 | #include <linux/clk.h> |
13 | #include <linux/dmaengine.h> |
14 | #include <linux/dma-mapping.h> |
15 | #include <linux/err.h> |
16 | #include <linux/iopoll.h> |
17 | #include <linux/list.h> |
18 | #include <linux/module.h> |
19 | #include <linux/of.h> |
20 | #include <linux/of_dma.h> |
21 | #include <linux/platform_device.h> |
22 | #include <linux/pm_runtime.h> |
23 | #include <linux/refcount.h> |
24 | #include <linux/slab.h> |
25 | |
26 | #include "../virt-dma.h" |
27 | |
28 | #define MTK_HSDMA_USEC_POLL 20 |
29 | #define MTK_HSDMA_TIMEOUT_POLL 200000 |
30 | #define MTK_HSDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
31 | |
32 | /* The default number of virtual channel */ |
33 | #define MTK_HSDMA_NR_VCHANS 3 |
34 | |
35 | /* Only one physical channel supported */ |
36 | #define MTK_HSDMA_NR_MAX_PCHANS 1 |
37 | |
38 | /* Macro for physical descriptor (PD) manipulation */ |
39 | /* The number of PD which must be 2 of power */ |
40 | #define MTK_DMA_SIZE 64 |
41 | #define MTK_HSDMA_NEXT_DESP_IDX(x, y) (((x) + 1) & ((y) - 1)) |
42 | #define MTK_HSDMA_LAST_DESP_IDX(x, y) (((x) - 1) & ((y) - 1)) |
43 | #define MTK_HSDMA_MAX_LEN 0x3f80 |
44 | #define MTK_HSDMA_ALIGN_SIZE 4 |
45 | #define MTK_HSDMA_PLEN_MASK 0x3fff |
46 | #define MTK_HSDMA_DESC_PLEN(x) (((x) & MTK_HSDMA_PLEN_MASK) << 16) |
47 | #define MTK_HSDMA_DESC_PLEN_GET(x) (((x) >> 16) & MTK_HSDMA_PLEN_MASK) |
48 | |
49 | /* Registers for underlying ring manipulation */ |
50 | #define MTK_HSDMA_TX_BASE 0x0 |
51 | #define MTK_HSDMA_TX_CNT 0x4 |
52 | #define MTK_HSDMA_TX_CPU 0x8 |
53 | #define MTK_HSDMA_TX_DMA 0xc |
54 | #define MTK_HSDMA_RX_BASE 0x100 |
55 | #define MTK_HSDMA_RX_CNT 0x104 |
56 | #define MTK_HSDMA_RX_CPU 0x108 |
57 | #define MTK_HSDMA_RX_DMA 0x10c |
58 | |
59 | /* Registers for global setup */ |
60 | #define MTK_HSDMA_GLO 0x204 |
61 | #define MTK_HSDMA_GLO_MULTI_DMA BIT(10) |
62 | #define MTK_HSDMA_TX_WB_DDONE BIT(6) |
63 | #define MTK_HSDMA_BURST_64BYTES (0x2 << 4) |
64 | #define MTK_HSDMA_GLO_RX_BUSY BIT(3) |
65 | #define MTK_HSDMA_GLO_RX_DMA BIT(2) |
66 | #define MTK_HSDMA_GLO_TX_BUSY BIT(1) |
67 | #define MTK_HSDMA_GLO_TX_DMA BIT(0) |
68 | #define MTK_HSDMA_GLO_DMA (MTK_HSDMA_GLO_TX_DMA | \ |
69 | MTK_HSDMA_GLO_RX_DMA) |
70 | #define MTK_HSDMA_GLO_BUSY (MTK_HSDMA_GLO_RX_BUSY | \ |
71 | MTK_HSDMA_GLO_TX_BUSY) |
72 | #define MTK_HSDMA_GLO_DEFAULT (MTK_HSDMA_GLO_TX_DMA | \ |
73 | MTK_HSDMA_GLO_RX_DMA | \ |
74 | MTK_HSDMA_TX_WB_DDONE | \ |
75 | MTK_HSDMA_BURST_64BYTES | \ |
76 | MTK_HSDMA_GLO_MULTI_DMA) |
77 | |
78 | /* Registers for reset */ |
79 | #define MTK_HSDMA_RESET 0x208 |
80 | #define MTK_HSDMA_RST_TX BIT(0) |
81 | #define MTK_HSDMA_RST_RX BIT(16) |
82 | |
83 | /* Registers for interrupt control */ |
84 | #define MTK_HSDMA_DLYINT 0x20c |
85 | #define MTK_HSDMA_RXDLY_INT_EN BIT(15) |
86 | |
87 | /* Interrupt fires when the pending number's more than the specified */ |
88 | #define MTK_HSDMA_RXMAX_PINT(x) (((x) & 0x7f) << 8) |
89 | |
90 | /* Interrupt fires when the pending time's more than the specified in 20 us */ |
91 | #define MTK_HSDMA_RXMAX_PTIME(x) ((x) & 0x7f) |
92 | #define MTK_HSDMA_DLYINT_DEFAULT (MTK_HSDMA_RXDLY_INT_EN | \ |
93 | MTK_HSDMA_RXMAX_PINT(20) | \ |
94 | MTK_HSDMA_RXMAX_PTIME(20)) |
95 | #define MTK_HSDMA_INT_STATUS 0x220 |
96 | #define MTK_HSDMA_INT_ENABLE 0x228 |
97 | #define MTK_HSDMA_INT_RXDONE BIT(16) |
98 | |
99 | enum mtk_hsdma_vdesc_flag { |
100 | MTK_HSDMA_VDESC_FINISHED = 0x01, |
101 | }; |
102 | |
103 | #define IS_MTK_HSDMA_VDESC_FINISHED(x) ((x) == MTK_HSDMA_VDESC_FINISHED) |
104 | |
105 | /** |
106 | * struct mtk_hsdma_pdesc - This is the struct holding info describing physical |
107 | * descriptor (PD) and its placement must be kept at |
108 | * 4-bytes alignment in little endian order. |
109 | * @desc1: | The control pad used to indicate hardware how to |
110 | * @desc2: | deal with the descriptor such as source and |
111 | * @desc3: | destination address and data length. The maximum |
112 | * @desc4: | data length each pdesc can handle is 0x3f80 bytes |
113 | */ |
114 | struct mtk_hsdma_pdesc { |
115 | __le32 desc1; |
116 | __le32 desc2; |
117 | __le32 desc3; |
118 | __le32 desc4; |
119 | } __packed __aligned(4); |
120 | |
121 | /** |
122 | * struct mtk_hsdma_vdesc - This is the struct holding info describing virtual |
123 | * descriptor (VD) |
124 | * @vd: An instance for struct virt_dma_desc |
125 | * @len: The total data size device wants to move |
126 | * @residue: The remaining data size device will move |
127 | * @dest: The destination address device wants to move to |
128 | * @src: The source address device wants to move from |
129 | */ |
130 | struct mtk_hsdma_vdesc { |
131 | struct virt_dma_desc vd; |
132 | size_t len; |
133 | size_t residue; |
134 | dma_addr_t dest; |
135 | dma_addr_t src; |
136 | }; |
137 | |
138 | /** |
139 | * struct mtk_hsdma_cb - This is the struct holding extra info required for RX |
140 | * ring to know what relevant VD the PD is being |
141 | * mapped to. |
142 | * @vd: Pointer to the relevant VD. |
143 | * @flag: Flag indicating what action should be taken when VD |
144 | * is completed. |
145 | */ |
146 | struct mtk_hsdma_cb { |
147 | struct virt_dma_desc *vd; |
148 | enum mtk_hsdma_vdesc_flag flag; |
149 | }; |
150 | |
151 | /** |
152 | * struct mtk_hsdma_ring - This struct holds info describing underlying ring |
153 | * space |
154 | * @txd: The descriptor TX ring which describes DMA source |
155 | * information |
156 | * @rxd: The descriptor RX ring which describes DMA |
157 | * destination information |
158 | * @cb: The extra information pointed at by RX ring |
159 | * @tphys: The physical addr of TX ring |
160 | * @rphys: The physical addr of RX ring |
161 | * @cur_tptr: Pointer to the next free descriptor used by the host |
162 | * @cur_rptr: Pointer to the last done descriptor by the device |
163 | */ |
164 | struct mtk_hsdma_ring { |
165 | struct mtk_hsdma_pdesc *txd; |
166 | struct mtk_hsdma_pdesc *rxd; |
167 | struct mtk_hsdma_cb *cb; |
168 | dma_addr_t tphys; |
169 | dma_addr_t rphys; |
170 | u16 cur_tptr; |
171 | u16 cur_rptr; |
172 | }; |
173 | |
174 | /** |
175 | * struct mtk_hsdma_pchan - This is the struct holding info describing physical |
176 | * channel (PC) |
177 | * @ring: An instance for the underlying ring |
178 | * @sz_ring: Total size allocated for the ring |
179 | * @nr_free: Total number of free rooms in the ring. It would |
180 | * be accessed and updated frequently between IRQ |
181 | * context and user context to reflect whether ring |
182 | * can accept requests from VD. |
183 | */ |
184 | struct mtk_hsdma_pchan { |
185 | struct mtk_hsdma_ring ring; |
186 | size_t sz_ring; |
187 | atomic_t nr_free; |
188 | }; |
189 | |
190 | /** |
191 | * struct mtk_hsdma_vchan - This is the struct holding info describing virtual |
192 | * channel (VC) |
193 | * @vc: An instance for struct virt_dma_chan |
194 | * @issue_completion: The wait for all issued descriptors completited |
195 | * @issue_synchronize: Bool indicating channel synchronization starts |
196 | * @desc_hw_processing: List those descriptors the hardware is processing, |
197 | * which is protected by vc.lock |
198 | */ |
199 | struct mtk_hsdma_vchan { |
200 | struct virt_dma_chan vc; |
201 | struct completion issue_completion; |
202 | bool issue_synchronize; |
203 | struct list_head desc_hw_processing; |
204 | }; |
205 | |
206 | /** |
207 | * struct mtk_hsdma_soc - This is the struct holding differences among SoCs |
208 | * @ddone: Bit mask for DDONE |
209 | * @ls0: Bit mask for LS0 |
210 | */ |
211 | struct mtk_hsdma_soc { |
212 | __le32 ddone; |
213 | __le32 ls0; |
214 | }; |
215 | |
216 | /** |
217 | * struct mtk_hsdma_device - This is the struct holding info describing HSDMA |
218 | * device |
219 | * @ddev: An instance for struct dma_device |
220 | * @base: The mapped register I/O base |
221 | * @clk: The clock that device internal is using |
222 | * @irq: The IRQ that device are using |
223 | * @dma_requests: The number of VCs the device supports to |
224 | * @vc: The pointer to all available VCs |
225 | * @pc: The pointer to the underlying PC |
226 | * @pc_refcnt: Track how many VCs are using the PC |
227 | * @lock: Lock protect agaisting multiple VCs access PC |
228 | * @soc: The pointer to area holding differences among |
229 | * vaious platform |
230 | */ |
231 | struct mtk_hsdma_device { |
232 | struct dma_device ddev; |
233 | void __iomem *base; |
234 | struct clk *clk; |
235 | u32 irq; |
236 | |
237 | u32 dma_requests; |
238 | struct mtk_hsdma_vchan *vc; |
239 | struct mtk_hsdma_pchan *pc; |
240 | refcount_t pc_refcnt; |
241 | |
242 | /* Lock used to protect against multiple VCs access PC */ |
243 | spinlock_t lock; |
244 | |
245 | const struct mtk_hsdma_soc *soc; |
246 | }; |
247 | |
248 | static struct mtk_hsdma_device *to_hsdma_dev(struct dma_chan *chan) |
249 | { |
250 | return container_of(chan->device, struct mtk_hsdma_device, ddev); |
251 | } |
252 | |
253 | static inline struct mtk_hsdma_vchan *to_hsdma_vchan(struct dma_chan *chan) |
254 | { |
255 | return container_of(chan, struct mtk_hsdma_vchan, vc.chan); |
256 | } |
257 | |
258 | static struct mtk_hsdma_vdesc *to_hsdma_vdesc(struct virt_dma_desc *vd) |
259 | { |
260 | return container_of(vd, struct mtk_hsdma_vdesc, vd); |
261 | } |
262 | |
263 | static struct device *hsdma2dev(struct mtk_hsdma_device *hsdma) |
264 | { |
265 | return hsdma->ddev.dev; |
266 | } |
267 | |
268 | static u32 mtk_dma_read(struct mtk_hsdma_device *hsdma, u32 reg) |
269 | { |
270 | return readl(addr: hsdma->base + reg); |
271 | } |
272 | |
273 | static void mtk_dma_write(struct mtk_hsdma_device *hsdma, u32 reg, u32 val) |
274 | { |
275 | writel(val, addr: hsdma->base + reg); |
276 | } |
277 | |
278 | static void mtk_dma_rmw(struct mtk_hsdma_device *hsdma, u32 reg, |
279 | u32 mask, u32 set) |
280 | { |
281 | u32 val; |
282 | |
283 | val = mtk_dma_read(hsdma, reg); |
284 | val &= ~mask; |
285 | val |= set; |
286 | mtk_dma_write(hsdma, reg, val); |
287 | } |
288 | |
289 | static void mtk_dma_set(struct mtk_hsdma_device *hsdma, u32 reg, u32 val) |
290 | { |
291 | mtk_dma_rmw(hsdma, reg, mask: 0, set: val); |
292 | } |
293 | |
294 | static void mtk_dma_clr(struct mtk_hsdma_device *hsdma, u32 reg, u32 val) |
295 | { |
296 | mtk_dma_rmw(hsdma, reg, mask: val, set: 0); |
297 | } |
298 | |
299 | static void mtk_hsdma_vdesc_free(struct virt_dma_desc *vd) |
300 | { |
301 | kfree(container_of(vd, struct mtk_hsdma_vdesc, vd)); |
302 | } |
303 | |
304 | static int mtk_hsdma_busy_wait(struct mtk_hsdma_device *hsdma) |
305 | { |
306 | u32 status = 0; |
307 | |
308 | return readl_poll_timeout(hsdma->base + MTK_HSDMA_GLO, status, |
309 | !(status & MTK_HSDMA_GLO_BUSY), |
310 | MTK_HSDMA_USEC_POLL, |
311 | MTK_HSDMA_TIMEOUT_POLL); |
312 | } |
313 | |
314 | static int mtk_hsdma_alloc_pchan(struct mtk_hsdma_device *hsdma, |
315 | struct mtk_hsdma_pchan *pc) |
316 | { |
317 | struct mtk_hsdma_ring *ring = &pc->ring; |
318 | int err; |
319 | |
320 | memset(pc, 0, sizeof(*pc)); |
321 | |
322 | /* |
323 | * Allocate ring space where [0 ... MTK_DMA_SIZE - 1] is for TX ring |
324 | * and [MTK_DMA_SIZE ... 2 * MTK_DMA_SIZE - 1] is for RX ring. |
325 | */ |
326 | pc->sz_ring = 2 * MTK_DMA_SIZE * sizeof(*ring->txd); |
327 | ring->txd = dma_alloc_coherent(dev: hsdma2dev(hsdma), size: pc->sz_ring, |
328 | dma_handle: &ring->tphys, GFP_NOWAIT); |
329 | if (!ring->txd) |
330 | return -ENOMEM; |
331 | |
332 | ring->rxd = &ring->txd[MTK_DMA_SIZE]; |
333 | ring->rphys = ring->tphys + MTK_DMA_SIZE * sizeof(*ring->txd); |
334 | ring->cur_tptr = 0; |
335 | ring->cur_rptr = MTK_DMA_SIZE - 1; |
336 | |
337 | ring->cb = kcalloc(MTK_DMA_SIZE, size: sizeof(*ring->cb), GFP_NOWAIT); |
338 | if (!ring->cb) { |
339 | err = -ENOMEM; |
340 | goto err_free_dma; |
341 | } |
342 | |
343 | atomic_set(v: &pc->nr_free, MTK_DMA_SIZE - 1); |
344 | |
345 | /* Disable HSDMA and wait for the completion */ |
346 | mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA); |
347 | err = mtk_hsdma_busy_wait(hsdma); |
348 | if (err) |
349 | goto err_free_cb; |
350 | |
351 | /* Reset */ |
352 | mtk_dma_set(hsdma, MTK_HSDMA_RESET, |
353 | MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX); |
354 | mtk_dma_clr(hsdma, MTK_HSDMA_RESET, |
355 | MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX); |
356 | |
357 | /* Setup HSDMA initial pointer in the ring */ |
358 | mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, val: ring->tphys); |
359 | mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, MTK_DMA_SIZE); |
360 | mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, val: ring->cur_tptr); |
361 | mtk_dma_write(hsdma, MTK_HSDMA_TX_DMA, val: 0); |
362 | mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, val: ring->rphys); |
363 | mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, MTK_DMA_SIZE); |
364 | mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, val: ring->cur_rptr); |
365 | mtk_dma_write(hsdma, MTK_HSDMA_RX_DMA, val: 0); |
366 | |
367 | /* Enable HSDMA */ |
368 | mtk_dma_set(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA); |
369 | |
370 | /* Setup delayed interrupt */ |
371 | mtk_dma_write(hsdma, MTK_HSDMA_DLYINT, MTK_HSDMA_DLYINT_DEFAULT); |
372 | |
373 | /* Enable interrupt */ |
374 | mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); |
375 | |
376 | return 0; |
377 | |
378 | err_free_cb: |
379 | kfree(objp: ring->cb); |
380 | |
381 | err_free_dma: |
382 | dma_free_coherent(dev: hsdma2dev(hsdma), |
383 | size: pc->sz_ring, cpu_addr: ring->txd, dma_handle: ring->tphys); |
384 | return err; |
385 | } |
386 | |
387 | static void mtk_hsdma_free_pchan(struct mtk_hsdma_device *hsdma, |
388 | struct mtk_hsdma_pchan *pc) |
389 | { |
390 | struct mtk_hsdma_ring *ring = &pc->ring; |
391 | |
392 | /* Disable HSDMA and then wait for the completion */ |
393 | mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA); |
394 | mtk_hsdma_busy_wait(hsdma); |
395 | |
396 | /* Reset pointer in the ring */ |
397 | mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); |
398 | mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, val: 0); |
399 | mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, val: 0); |
400 | mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, val: 0); |
401 | mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, val: 0); |
402 | mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, val: 0); |
403 | mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, MTK_DMA_SIZE - 1); |
404 | |
405 | kfree(objp: ring->cb); |
406 | |
407 | dma_free_coherent(dev: hsdma2dev(hsdma), |
408 | size: pc->sz_ring, cpu_addr: ring->txd, dma_handle: ring->tphys); |
409 | } |
410 | |
411 | static int mtk_hsdma_issue_pending_vdesc(struct mtk_hsdma_device *hsdma, |
412 | struct mtk_hsdma_pchan *pc, |
413 | struct mtk_hsdma_vdesc *hvd) |
414 | { |
415 | struct mtk_hsdma_ring *ring = &pc->ring; |
416 | struct mtk_hsdma_pdesc *txd, *rxd; |
417 | u16 reserved, prev, tlen, num_sgs; |
418 | unsigned long flags; |
419 | |
420 | /* Protect against PC is accessed by multiple VCs simultaneously */ |
421 | spin_lock_irqsave(&hsdma->lock, flags); |
422 | |
423 | /* |
424 | * Reserve rooms, where pc->nr_free is used to track how many free |
425 | * rooms in the ring being updated in user and IRQ context. |
426 | */ |
427 | num_sgs = DIV_ROUND_UP(hvd->len, MTK_HSDMA_MAX_LEN); |
428 | reserved = min_t(u16, num_sgs, atomic_read(&pc->nr_free)); |
429 | |
430 | if (!reserved) { |
431 | spin_unlock_irqrestore(lock: &hsdma->lock, flags); |
432 | return -ENOSPC; |
433 | } |
434 | |
435 | atomic_sub(i: reserved, v: &pc->nr_free); |
436 | |
437 | while (reserved--) { |
438 | /* Limit size by PD capability for valid data moving */ |
439 | tlen = (hvd->len > MTK_HSDMA_MAX_LEN) ? |
440 | MTK_HSDMA_MAX_LEN : hvd->len; |
441 | |
442 | /* |
443 | * Setup PDs using the remaining VD info mapped on those |
444 | * reserved rooms. And since RXD is shared memory between the |
445 | * host and the device allocated by dma_alloc_coherent call, |
446 | * the helper macro WRITE_ONCE can ensure the data written to |
447 | * RAM would really happens. |
448 | */ |
449 | txd = &ring->txd[ring->cur_tptr]; |
450 | WRITE_ONCE(txd->desc1, hvd->src); |
451 | WRITE_ONCE(txd->desc2, |
452 | hsdma->soc->ls0 | MTK_HSDMA_DESC_PLEN(tlen)); |
453 | |
454 | rxd = &ring->rxd[ring->cur_tptr]; |
455 | WRITE_ONCE(rxd->desc1, hvd->dest); |
456 | WRITE_ONCE(rxd->desc2, MTK_HSDMA_DESC_PLEN(tlen)); |
457 | |
458 | /* Associate VD, the PD belonged to */ |
459 | ring->cb[ring->cur_tptr].vd = &hvd->vd; |
460 | |
461 | /* Move forward the pointer of TX ring */ |
462 | ring->cur_tptr = MTK_HSDMA_NEXT_DESP_IDX(ring->cur_tptr, |
463 | MTK_DMA_SIZE); |
464 | |
465 | /* Update VD with remaining data */ |
466 | hvd->src += tlen; |
467 | hvd->dest += tlen; |
468 | hvd->len -= tlen; |
469 | } |
470 | |
471 | /* |
472 | * Tagging flag for the last PD for VD will be responsible for |
473 | * completing VD. |
474 | */ |
475 | if (!hvd->len) { |
476 | prev = MTK_HSDMA_LAST_DESP_IDX(ring->cur_tptr, MTK_DMA_SIZE); |
477 | ring->cb[prev].flag = MTK_HSDMA_VDESC_FINISHED; |
478 | } |
479 | |
480 | /* Ensure all changes indeed done before we're going on */ |
481 | wmb(); |
482 | |
483 | /* |
484 | * Updating into hardware the pointer of TX ring lets HSDMA to take |
485 | * action for those pending PDs. |
486 | */ |
487 | mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, val: ring->cur_tptr); |
488 | |
489 | spin_unlock_irqrestore(lock: &hsdma->lock, flags); |
490 | |
491 | return 0; |
492 | } |
493 | |
494 | static void mtk_hsdma_issue_vchan_pending(struct mtk_hsdma_device *hsdma, |
495 | struct mtk_hsdma_vchan *hvc) |
496 | { |
497 | struct virt_dma_desc *vd, *vd2; |
498 | int err; |
499 | |
500 | lockdep_assert_held(&hvc->vc.lock); |
501 | |
502 | list_for_each_entry_safe(vd, vd2, &hvc->vc.desc_issued, node) { |
503 | struct mtk_hsdma_vdesc *hvd; |
504 | |
505 | hvd = to_hsdma_vdesc(vd); |
506 | |
507 | /* Map VD into PC and all VCs shares a single PC */ |
508 | err = mtk_hsdma_issue_pending_vdesc(hsdma, pc: hsdma->pc, hvd); |
509 | |
510 | /* |
511 | * Move VD from desc_issued to desc_hw_processing when entire |
512 | * VD is fit into available PDs. Otherwise, the uncompleted |
513 | * VDs would stay in list desc_issued and then restart the |
514 | * processing as soon as possible once underlying ring space |
515 | * got freed. |
516 | */ |
517 | if (err == -ENOSPC || hvd->len > 0) |
518 | break; |
519 | |
520 | /* |
521 | * The extra list desc_hw_processing is used because |
522 | * hardware can't provide sufficient information allowing us |
523 | * to know what VDs are still working on the underlying ring. |
524 | * Through the additional list, it can help us to implement |
525 | * terminate_all, residue calculation and such thing needed |
526 | * to know detail descriptor status on the hardware. |
527 | */ |
528 | list_move_tail(list: &vd->node, head: &hvc->desc_hw_processing); |
529 | } |
530 | } |
531 | |
532 | static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma) |
533 | { |
534 | struct mtk_hsdma_vchan *hvc; |
535 | struct mtk_hsdma_pdesc *rxd; |
536 | struct mtk_hsdma_vdesc *hvd; |
537 | struct mtk_hsdma_pchan *pc; |
538 | struct mtk_hsdma_cb *cb; |
539 | int i = MTK_DMA_SIZE; |
540 | __le32 desc2; |
541 | u32 status; |
542 | u16 next; |
543 | |
544 | /* Read IRQ status */ |
545 | status = mtk_dma_read(hsdma, MTK_HSDMA_INT_STATUS); |
546 | if (unlikely(!(status & MTK_HSDMA_INT_RXDONE))) |
547 | goto rx_done; |
548 | |
549 | pc = hsdma->pc; |
550 | |
551 | /* |
552 | * Using a fail-safe loop with iterations of up to MTK_DMA_SIZE to |
553 | * reclaim these finished descriptors: The most number of PDs the ISR |
554 | * can handle at one time shouldn't be more than MTK_DMA_SIZE so we |
555 | * take it as limited count instead of just using a dangerous infinite |
556 | * poll. |
557 | */ |
558 | while (i--) { |
559 | next = MTK_HSDMA_NEXT_DESP_IDX(pc->ring.cur_rptr, |
560 | MTK_DMA_SIZE); |
561 | rxd = &pc->ring.rxd[next]; |
562 | |
563 | /* |
564 | * If MTK_HSDMA_DESC_DDONE is no specified, that means data |
565 | * moving for the PD is still under going. |
566 | */ |
567 | desc2 = READ_ONCE(rxd->desc2); |
568 | if (!(desc2 & hsdma->soc->ddone)) |
569 | break; |
570 | |
571 | cb = &pc->ring.cb[next]; |
572 | if (unlikely(!cb->vd)) { |
573 | dev_err(hsdma2dev(hsdma), "cb->vd cannot be null\n" ); |
574 | break; |
575 | } |
576 | |
577 | /* Update residue of VD the associated PD belonged to */ |
578 | hvd = to_hsdma_vdesc(vd: cb->vd); |
579 | hvd->residue -= MTK_HSDMA_DESC_PLEN_GET(rxd->desc2); |
580 | |
581 | /* Complete VD until the relevant last PD is finished */ |
582 | if (IS_MTK_HSDMA_VDESC_FINISHED(cb->flag)) { |
583 | hvc = to_hsdma_vchan(chan: cb->vd->tx.chan); |
584 | |
585 | spin_lock(lock: &hvc->vc.lock); |
586 | |
587 | /* Remove VD from list desc_hw_processing */ |
588 | list_del(entry: &cb->vd->node); |
589 | |
590 | /* Add VD into list desc_completed */ |
591 | vchan_cookie_complete(vd: cb->vd); |
592 | |
593 | if (hvc->issue_synchronize && |
594 | list_empty(head: &hvc->desc_hw_processing)) { |
595 | complete(&hvc->issue_completion); |
596 | hvc->issue_synchronize = false; |
597 | } |
598 | spin_unlock(lock: &hvc->vc.lock); |
599 | |
600 | cb->flag = 0; |
601 | } |
602 | |
603 | cb->vd = NULL; |
604 | |
605 | /* |
606 | * Recycle the RXD with the helper WRITE_ONCE that can ensure |
607 | * data written into RAM would really happens. |
608 | */ |
609 | WRITE_ONCE(rxd->desc1, 0); |
610 | WRITE_ONCE(rxd->desc2, 0); |
611 | pc->ring.cur_rptr = next; |
612 | |
613 | /* Release rooms */ |
614 | atomic_inc(v: &pc->nr_free); |
615 | } |
616 | |
617 | /* Ensure all changes indeed done before we're going on */ |
618 | wmb(); |
619 | |
620 | /* Update CPU pointer for those completed PDs */ |
621 | mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, val: pc->ring.cur_rptr); |
622 | |
623 | /* |
624 | * Acking the pending IRQ allows hardware no longer to keep the used |
625 | * IRQ line in certain trigger state when software has completed all |
626 | * the finished physical descriptors. |
627 | */ |
628 | if (atomic_read(v: &pc->nr_free) >= MTK_DMA_SIZE - 1) |
629 | mtk_dma_write(hsdma, MTK_HSDMA_INT_STATUS, val: status); |
630 | |
631 | /* ASAP handles pending VDs in all VCs after freeing some rooms */ |
632 | for (i = 0; i < hsdma->dma_requests; i++) { |
633 | hvc = &hsdma->vc[i]; |
634 | spin_lock(lock: &hvc->vc.lock); |
635 | mtk_hsdma_issue_vchan_pending(hsdma, hvc); |
636 | spin_unlock(lock: &hvc->vc.lock); |
637 | } |
638 | |
639 | rx_done: |
640 | /* All completed PDs are cleaned up, so enable interrupt again */ |
641 | mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); |
642 | } |
643 | |
644 | static irqreturn_t mtk_hsdma_irq(int irq, void *devid) |
645 | { |
646 | struct mtk_hsdma_device *hsdma = devid; |
647 | |
648 | /* |
649 | * Disable interrupt until all completed PDs are cleaned up in |
650 | * mtk_hsdma_free_rooms call. |
651 | */ |
652 | mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE); |
653 | |
654 | mtk_hsdma_free_rooms_in_ring(hsdma); |
655 | |
656 | return IRQ_HANDLED; |
657 | } |
658 | |
659 | static struct virt_dma_desc *mtk_hsdma_find_active_desc(struct dma_chan *c, |
660 | dma_cookie_t cookie) |
661 | { |
662 | struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(chan: c); |
663 | struct virt_dma_desc *vd; |
664 | |
665 | list_for_each_entry(vd, &hvc->desc_hw_processing, node) |
666 | if (vd->tx.cookie == cookie) |
667 | return vd; |
668 | |
669 | list_for_each_entry(vd, &hvc->vc.desc_issued, node) |
670 | if (vd->tx.cookie == cookie) |
671 | return vd; |
672 | |
673 | return NULL; |
674 | } |
675 | |
676 | static enum dma_status mtk_hsdma_tx_status(struct dma_chan *c, |
677 | dma_cookie_t cookie, |
678 | struct dma_tx_state *txstate) |
679 | { |
680 | struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(chan: c); |
681 | struct mtk_hsdma_vdesc *hvd; |
682 | struct virt_dma_desc *vd; |
683 | enum dma_status ret; |
684 | unsigned long flags; |
685 | size_t bytes = 0; |
686 | |
687 | ret = dma_cookie_status(chan: c, cookie, state: txstate); |
688 | if (ret == DMA_COMPLETE || !txstate) |
689 | return ret; |
690 | |
691 | spin_lock_irqsave(&hvc->vc.lock, flags); |
692 | vd = mtk_hsdma_find_active_desc(c, cookie); |
693 | spin_unlock_irqrestore(lock: &hvc->vc.lock, flags); |
694 | |
695 | if (vd) { |
696 | hvd = to_hsdma_vdesc(vd); |
697 | bytes = hvd->residue; |
698 | } |
699 | |
700 | dma_set_residue(state: txstate, residue: bytes); |
701 | |
702 | return ret; |
703 | } |
704 | |
705 | static void mtk_hsdma_issue_pending(struct dma_chan *c) |
706 | { |
707 | struct mtk_hsdma_device *hsdma = to_hsdma_dev(chan: c); |
708 | struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(chan: c); |
709 | unsigned long flags; |
710 | |
711 | spin_lock_irqsave(&hvc->vc.lock, flags); |
712 | |
713 | if (vchan_issue_pending(vc: &hvc->vc)) |
714 | mtk_hsdma_issue_vchan_pending(hsdma, hvc); |
715 | |
716 | spin_unlock_irqrestore(lock: &hvc->vc.lock, flags); |
717 | } |
718 | |
719 | static struct dma_async_tx_descriptor * |
720 | mtk_hsdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, |
721 | dma_addr_t src, size_t len, unsigned long flags) |
722 | { |
723 | struct mtk_hsdma_vdesc *hvd; |
724 | |
725 | hvd = kzalloc(size: sizeof(*hvd), GFP_NOWAIT); |
726 | if (!hvd) |
727 | return NULL; |
728 | |
729 | hvd->len = len; |
730 | hvd->residue = len; |
731 | hvd->src = src; |
732 | hvd->dest = dest; |
733 | |
734 | return vchan_tx_prep(vc: to_virt_chan(chan: c), vd: &hvd->vd, tx_flags: flags); |
735 | } |
736 | |
737 | static int mtk_hsdma_free_inactive_desc(struct dma_chan *c) |
738 | { |
739 | struct virt_dma_chan *vc = to_virt_chan(chan: c); |
740 | unsigned long flags; |
741 | LIST_HEAD(head); |
742 | |
743 | spin_lock_irqsave(&vc->lock, flags); |
744 | list_splice_tail_init(list: &vc->desc_allocated, head: &head); |
745 | list_splice_tail_init(list: &vc->desc_submitted, head: &head); |
746 | list_splice_tail_init(list: &vc->desc_issued, head: &head); |
747 | spin_unlock_irqrestore(lock: &vc->lock, flags); |
748 | |
749 | /* At the point, we don't expect users put descriptor into VC again */ |
750 | vchan_dma_desc_free_list(vc, head: &head); |
751 | |
752 | return 0; |
753 | } |
754 | |
755 | static void mtk_hsdma_free_active_desc(struct dma_chan *c) |
756 | { |
757 | struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(chan: c); |
758 | bool sync_needed = false; |
759 | |
760 | /* |
761 | * Once issue_synchronize is being set, which means once the hardware |
762 | * consumes all descriptors for the channel in the ring, the |
763 | * synchronization must be notified immediately it is completed. |
764 | */ |
765 | spin_lock(lock: &hvc->vc.lock); |
766 | if (!list_empty(head: &hvc->desc_hw_processing)) { |
767 | hvc->issue_synchronize = true; |
768 | sync_needed = true; |
769 | } |
770 | spin_unlock(lock: &hvc->vc.lock); |
771 | |
772 | if (sync_needed) |
773 | wait_for_completion(&hvc->issue_completion); |
774 | /* |
775 | * At the point, we expect that all remaining descriptors in the ring |
776 | * for the channel should be all processing done. |
777 | */ |
778 | WARN_ONCE(!list_empty(&hvc->desc_hw_processing), |
779 | "Desc pending still in list desc_hw_processing\n" ); |
780 | |
781 | /* Free all descriptors in list desc_completed */ |
782 | vchan_synchronize(vc: &hvc->vc); |
783 | |
784 | WARN_ONCE(!list_empty(&hvc->vc.desc_completed), |
785 | "Desc pending still in list desc_completed\n" ); |
786 | } |
787 | |
788 | static int mtk_hsdma_terminate_all(struct dma_chan *c) |
789 | { |
790 | /* |
791 | * Free pending descriptors not processed yet by hardware that have |
792 | * previously been submitted to the channel. |
793 | */ |
794 | mtk_hsdma_free_inactive_desc(c); |
795 | |
796 | /* |
797 | * However, the DMA engine doesn't provide any way to stop these |
798 | * descriptors being processed currently by hardware. The only way is |
799 | * to just waiting until these descriptors are all processed completely |
800 | * through mtk_hsdma_free_active_desc call. |
801 | */ |
802 | mtk_hsdma_free_active_desc(c); |
803 | |
804 | return 0; |
805 | } |
806 | |
807 | static int mtk_hsdma_alloc_chan_resources(struct dma_chan *c) |
808 | { |
809 | struct mtk_hsdma_device *hsdma = to_hsdma_dev(chan: c); |
810 | int err; |
811 | |
812 | /* |
813 | * Since HSDMA has only one PC, the resource for PC is being allocated |
814 | * when the first VC is being created and the other VCs would run on |
815 | * the same PC. |
816 | */ |
817 | if (!refcount_read(r: &hsdma->pc_refcnt)) { |
818 | err = mtk_hsdma_alloc_pchan(hsdma, pc: hsdma->pc); |
819 | if (err) |
820 | return err; |
821 | /* |
822 | * refcount_inc would complain increment on 0; use-after-free. |
823 | * Thus, we need to explicitly set it as 1 initially. |
824 | */ |
825 | refcount_set(r: &hsdma->pc_refcnt, n: 1); |
826 | } else { |
827 | refcount_inc(r: &hsdma->pc_refcnt); |
828 | } |
829 | |
830 | return 0; |
831 | } |
832 | |
833 | static void mtk_hsdma_free_chan_resources(struct dma_chan *c) |
834 | { |
835 | struct mtk_hsdma_device *hsdma = to_hsdma_dev(chan: c); |
836 | |
837 | /* Free all descriptors in all lists on the VC */ |
838 | mtk_hsdma_terminate_all(c); |
839 | |
840 | /* The resource for PC is not freed until all the VCs are destroyed */ |
841 | if (!refcount_dec_and_test(r: &hsdma->pc_refcnt)) |
842 | return; |
843 | |
844 | mtk_hsdma_free_pchan(hsdma, pc: hsdma->pc); |
845 | } |
846 | |
847 | static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma) |
848 | { |
849 | int err; |
850 | |
851 | pm_runtime_enable(dev: hsdma2dev(hsdma)); |
852 | pm_runtime_get_sync(dev: hsdma2dev(hsdma)); |
853 | |
854 | err = clk_prepare_enable(clk: hsdma->clk); |
855 | if (err) |
856 | return err; |
857 | |
858 | mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, val: 0); |
859 | mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT); |
860 | |
861 | return 0; |
862 | } |
863 | |
864 | static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma) |
865 | { |
866 | mtk_dma_write(hsdma, MTK_HSDMA_GLO, val: 0); |
867 | |
868 | clk_disable_unprepare(clk: hsdma->clk); |
869 | |
870 | pm_runtime_put_sync(dev: hsdma2dev(hsdma)); |
871 | pm_runtime_disable(dev: hsdma2dev(hsdma)); |
872 | |
873 | return 0; |
874 | } |
875 | |
876 | static const struct mtk_hsdma_soc mt7623_soc = { |
877 | .ddone = BIT(31), |
878 | .ls0 = BIT(30), |
879 | }; |
880 | |
881 | static const struct mtk_hsdma_soc mt7622_soc = { |
882 | .ddone = BIT(15), |
883 | .ls0 = BIT(14), |
884 | }; |
885 | |
886 | static const struct of_device_id mtk_hsdma_match[] = { |
887 | { .compatible = "mediatek,mt7623-hsdma" , .data = &mt7623_soc}, |
888 | { .compatible = "mediatek,mt7622-hsdma" , .data = &mt7622_soc}, |
889 | { /* sentinel */ } |
890 | }; |
891 | MODULE_DEVICE_TABLE(of, mtk_hsdma_match); |
892 | |
893 | static int mtk_hsdma_probe(struct platform_device *pdev) |
894 | { |
895 | struct mtk_hsdma_device *hsdma; |
896 | struct mtk_hsdma_vchan *vc; |
897 | struct dma_device *dd; |
898 | int i, err; |
899 | |
900 | hsdma = devm_kzalloc(dev: &pdev->dev, size: sizeof(*hsdma), GFP_KERNEL); |
901 | if (!hsdma) |
902 | return -ENOMEM; |
903 | |
904 | dd = &hsdma->ddev; |
905 | |
906 | hsdma->base = devm_platform_ioremap_resource(pdev, index: 0); |
907 | if (IS_ERR(ptr: hsdma->base)) |
908 | return PTR_ERR(ptr: hsdma->base); |
909 | |
910 | hsdma->soc = of_device_get_match_data(dev: &pdev->dev); |
911 | if (!hsdma->soc) { |
912 | dev_err(&pdev->dev, "No device match found\n" ); |
913 | return -ENODEV; |
914 | } |
915 | |
916 | hsdma->clk = devm_clk_get(dev: &pdev->dev, id: "hsdma" ); |
917 | if (IS_ERR(ptr: hsdma->clk)) { |
918 | dev_err(&pdev->dev, "No clock for %s\n" , |
919 | dev_name(&pdev->dev)); |
920 | return PTR_ERR(ptr: hsdma->clk); |
921 | } |
922 | |
923 | err = platform_get_irq(pdev, 0); |
924 | if (err < 0) |
925 | return err; |
926 | hsdma->irq = err; |
927 | |
928 | refcount_set(r: &hsdma->pc_refcnt, n: 0); |
929 | spin_lock_init(&hsdma->lock); |
930 | |
931 | dma_cap_set(DMA_MEMCPY, dd->cap_mask); |
932 | |
933 | dd->copy_align = MTK_HSDMA_ALIGN_SIZE; |
934 | dd->device_alloc_chan_resources = mtk_hsdma_alloc_chan_resources; |
935 | dd->device_free_chan_resources = mtk_hsdma_free_chan_resources; |
936 | dd->device_tx_status = mtk_hsdma_tx_status; |
937 | dd->device_issue_pending = mtk_hsdma_issue_pending; |
938 | dd->device_prep_dma_memcpy = mtk_hsdma_prep_dma_memcpy; |
939 | dd->device_terminate_all = mtk_hsdma_terminate_all; |
940 | dd->src_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS; |
941 | dd->dst_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS; |
942 | dd->directions = BIT(DMA_MEM_TO_MEM); |
943 | dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; |
944 | dd->dev = &pdev->dev; |
945 | INIT_LIST_HEAD(list: &dd->channels); |
946 | |
947 | hsdma->dma_requests = MTK_HSDMA_NR_VCHANS; |
948 | if (pdev->dev.of_node && of_property_read_u32(np: pdev->dev.of_node, |
949 | propname: "dma-requests" , |
950 | out_value: &hsdma->dma_requests)) { |
951 | dev_info(&pdev->dev, |
952 | "Using %u as missing dma-requests property\n" , |
953 | MTK_HSDMA_NR_VCHANS); |
954 | } |
955 | |
956 | hsdma->pc = devm_kcalloc(dev: &pdev->dev, MTK_HSDMA_NR_MAX_PCHANS, |
957 | size: sizeof(*hsdma->pc), GFP_KERNEL); |
958 | if (!hsdma->pc) |
959 | return -ENOMEM; |
960 | |
961 | hsdma->vc = devm_kcalloc(dev: &pdev->dev, n: hsdma->dma_requests, |
962 | size: sizeof(*hsdma->vc), GFP_KERNEL); |
963 | if (!hsdma->vc) |
964 | return -ENOMEM; |
965 | |
966 | for (i = 0; i < hsdma->dma_requests; i++) { |
967 | vc = &hsdma->vc[i]; |
968 | vc->vc.desc_free = mtk_hsdma_vdesc_free; |
969 | vchan_init(vc: &vc->vc, dmadev: dd); |
970 | init_completion(x: &vc->issue_completion); |
971 | INIT_LIST_HEAD(list: &vc->desc_hw_processing); |
972 | } |
973 | |
974 | err = dma_async_device_register(device: dd); |
975 | if (err) |
976 | return err; |
977 | |
978 | err = of_dma_controller_register(np: pdev->dev.of_node, |
979 | of_dma_xlate: of_dma_xlate_by_chan_id, data: hsdma); |
980 | if (err) { |
981 | dev_err(&pdev->dev, |
982 | "MediaTek HSDMA OF registration failed %d\n" , err); |
983 | goto err_unregister; |
984 | } |
985 | |
986 | mtk_hsdma_hw_init(hsdma); |
987 | |
988 | err = devm_request_irq(dev: &pdev->dev, irq: hsdma->irq, |
989 | handler: mtk_hsdma_irq, irqflags: 0, |
990 | devname: dev_name(dev: &pdev->dev), dev_id: hsdma); |
991 | if (err) { |
992 | dev_err(&pdev->dev, |
993 | "request_irq failed with err %d\n" , err); |
994 | goto err_free; |
995 | } |
996 | |
997 | platform_set_drvdata(pdev, data: hsdma); |
998 | |
999 | dev_info(&pdev->dev, "MediaTek HSDMA driver registered\n" ); |
1000 | |
1001 | return 0; |
1002 | |
1003 | err_free: |
1004 | mtk_hsdma_hw_deinit(hsdma); |
1005 | of_dma_controller_free(np: pdev->dev.of_node); |
1006 | err_unregister: |
1007 | dma_async_device_unregister(device: dd); |
1008 | |
1009 | return err; |
1010 | } |
1011 | |
1012 | static void mtk_hsdma_remove(struct platform_device *pdev) |
1013 | { |
1014 | struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev); |
1015 | struct mtk_hsdma_vchan *vc; |
1016 | int i; |
1017 | |
1018 | /* Kill VC task */ |
1019 | for (i = 0; i < hsdma->dma_requests; i++) { |
1020 | vc = &hsdma->vc[i]; |
1021 | |
1022 | list_del(entry: &vc->vc.chan.device_node); |
1023 | tasklet_kill(t: &vc->vc.task); |
1024 | } |
1025 | |
1026 | /* Disable DMA interrupt */ |
1027 | mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, val: 0); |
1028 | |
1029 | /* Waits for any pending IRQ handlers to complete */ |
1030 | synchronize_irq(irq: hsdma->irq); |
1031 | |
1032 | /* Disable hardware */ |
1033 | mtk_hsdma_hw_deinit(hsdma); |
1034 | |
1035 | dma_async_device_unregister(device: &hsdma->ddev); |
1036 | of_dma_controller_free(np: pdev->dev.of_node); |
1037 | } |
1038 | |
1039 | static struct platform_driver mtk_hsdma_driver = { |
1040 | .probe = mtk_hsdma_probe, |
1041 | .remove_new = mtk_hsdma_remove, |
1042 | .driver = { |
1043 | .name = KBUILD_MODNAME, |
1044 | .of_match_table = mtk_hsdma_match, |
1045 | }, |
1046 | }; |
1047 | module_platform_driver(mtk_hsdma_driver); |
1048 | |
1049 | MODULE_DESCRIPTION("MediaTek High-Speed DMA Controller Driver" ); |
1050 | MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>" ); |
1051 | MODULE_LICENSE("GPL v2" ); |
1052 | |