1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * drivers/dma/fsl-edma.c |
4 | * |
5 | * Copyright 2013-2014 Freescale Semiconductor, Inc. |
6 | * |
7 | * Driver for the Freescale eDMA engine with flexible channel multiplexing |
8 | * capability for DMA request sources. The eDMA block can be found on some |
9 | * Vybrid and Layerscape SoCs. |
10 | */ |
11 | |
12 | #include <linux/module.h> |
13 | #include <linux/interrupt.h> |
14 | #include <linux/clk.h> |
15 | #include <linux/of.h> |
16 | #include <linux/of_dma.h> |
17 | #include <linux/dma-mapping.h> |
18 | #include <linux/pm_runtime.h> |
19 | #include <linux/pm_domain.h> |
20 | #include <linux/property.h> |
21 | |
22 | #include "fsl-edma-common.h" |
23 | |
24 | #define ARGS_RX BIT(0) |
25 | #define ARGS_REMOTE BIT(1) |
26 | #define ARGS_MULTI_FIFO BIT(2) |
27 | |
28 | static void fsl_edma_synchronize(struct dma_chan *chan) |
29 | { |
30 | struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); |
31 | |
32 | vchan_synchronize(vc: &fsl_chan->vchan); |
33 | } |
34 | |
35 | static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) |
36 | { |
37 | struct fsl_edma_engine *fsl_edma = dev_id; |
38 | unsigned int intr, ch; |
39 | struct edma_regs *regs = &fsl_edma->regs; |
40 | |
41 | intr = edma_readl(edma: fsl_edma, addr: regs->intl); |
42 | if (!intr) |
43 | return IRQ_NONE; |
44 | |
45 | for (ch = 0; ch < fsl_edma->n_chans; ch++) { |
46 | if (intr & (0x1 << ch)) { |
47 | edma_writeb(edma: fsl_edma, EDMA_CINT_CINT(ch), addr: regs->cint); |
48 | fsl_edma_tx_chan_handler(fsl_chan: &fsl_edma->chans[ch]); |
49 | } |
50 | } |
51 | return IRQ_HANDLED; |
52 | } |
53 | |
54 | static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id) |
55 | { |
56 | struct fsl_edma_chan *fsl_chan = dev_id; |
57 | unsigned int intr; |
58 | |
59 | intr = edma_readl_chreg(fsl_chan, ch_int); |
60 | if (!intr) |
61 | return IRQ_HANDLED; |
62 | |
63 | edma_writel_chreg(fsl_chan, 1, ch_int); |
64 | |
65 | fsl_edma_tx_chan_handler(fsl_chan); |
66 | |
67 | return IRQ_HANDLED; |
68 | } |
69 | |
70 | static irqreturn_t fsl_edma_err_handler(int irq, void *dev_id) |
71 | { |
72 | struct fsl_edma_engine *fsl_edma = dev_id; |
73 | unsigned int err, ch; |
74 | struct edma_regs *regs = &fsl_edma->regs; |
75 | |
76 | err = edma_readl(edma: fsl_edma, addr: regs->errl); |
77 | if (!err) |
78 | return IRQ_NONE; |
79 | |
80 | for (ch = 0; ch < fsl_edma->n_chans; ch++) { |
81 | if (err & (0x1 << ch)) { |
82 | fsl_edma_disable_request(fsl_chan: &fsl_edma->chans[ch]); |
83 | edma_writeb(edma: fsl_edma, EDMA_CERR_CERR(ch), addr: regs->cerr); |
84 | fsl_edma_err_chan_handler(fsl_chan: &fsl_edma->chans[ch]); |
85 | } |
86 | } |
87 | return IRQ_HANDLED; |
88 | } |
89 | |
90 | static irqreturn_t fsl_edma_irq_handler(int irq, void *dev_id) |
91 | { |
92 | if (fsl_edma_tx_handler(irq, dev_id) == IRQ_HANDLED) |
93 | return IRQ_HANDLED; |
94 | |
95 | return fsl_edma_err_handler(irq, dev_id); |
96 | } |
97 | |
98 | static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, |
99 | struct of_dma *ofdma) |
100 | { |
101 | struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data; |
102 | struct dma_chan *chan, *_chan; |
103 | struct fsl_edma_chan *fsl_chan; |
104 | u32 dmamux_nr = fsl_edma->drvdata->dmamuxs; |
105 | unsigned long chans_per_mux = fsl_edma->n_chans / dmamux_nr; |
106 | |
107 | if (dma_spec->args_count != 2) |
108 | return NULL; |
109 | |
110 | mutex_lock(&fsl_edma->fsl_edma_mutex); |
111 | list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels, device_node) { |
112 | if (chan->client_count) |
113 | continue; |
114 | if ((chan->chan_id / chans_per_mux) == dma_spec->args[0]) { |
115 | chan = dma_get_slave_channel(chan); |
116 | if (chan) { |
117 | chan->device->privatecnt++; |
118 | fsl_chan = to_fsl_edma_chan(chan); |
119 | fsl_chan->slave_id = dma_spec->args[1]; |
120 | fsl_edma_chan_mux(fsl_chan, slot: fsl_chan->slave_id, |
121 | enable: true); |
122 | mutex_unlock(lock: &fsl_edma->fsl_edma_mutex); |
123 | return chan; |
124 | } |
125 | } |
126 | } |
127 | mutex_unlock(lock: &fsl_edma->fsl_edma_mutex); |
128 | return NULL; |
129 | } |
130 | |
131 | static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, |
132 | struct of_dma *ofdma) |
133 | { |
134 | struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data; |
135 | struct dma_chan *chan, *_chan; |
136 | struct fsl_edma_chan *fsl_chan; |
137 | bool b_chmux; |
138 | int i; |
139 | |
140 | if (dma_spec->args_count != 3) |
141 | return NULL; |
142 | |
143 | b_chmux = !!(fsl_edma->drvdata->flags & FSL_EDMA_DRV_HAS_CHMUX); |
144 | |
145 | mutex_lock(&fsl_edma->fsl_edma_mutex); |
146 | list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels, |
147 | device_node) { |
148 | |
149 | if (chan->client_count) |
150 | continue; |
151 | |
152 | fsl_chan = to_fsl_edma_chan(chan); |
153 | i = fsl_chan - fsl_edma->chans; |
154 | |
155 | fsl_chan->priority = dma_spec->args[1]; |
156 | fsl_chan->is_rxchan = dma_spec->args[2] & ARGS_RX; |
157 | fsl_chan->is_remote = dma_spec->args[2] & ARGS_REMOTE; |
158 | fsl_chan->is_multi_fifo = dma_spec->args[2] & ARGS_MULTI_FIFO; |
159 | |
160 | if (!b_chmux && i == dma_spec->args[0]) { |
161 | chan = dma_get_slave_channel(chan); |
162 | chan->device->privatecnt++; |
163 | mutex_unlock(lock: &fsl_edma->fsl_edma_mutex); |
164 | return chan; |
165 | } else if (b_chmux && !fsl_chan->srcid) { |
166 | /* if controller support channel mux, choose a free channel */ |
167 | chan = dma_get_slave_channel(chan); |
168 | chan->device->privatecnt++; |
169 | fsl_chan->srcid = dma_spec->args[0]; |
170 | mutex_unlock(lock: &fsl_edma->fsl_edma_mutex); |
171 | return chan; |
172 | } |
173 | } |
174 | mutex_unlock(lock: &fsl_edma->fsl_edma_mutex); |
175 | return NULL; |
176 | } |
177 | |
178 | static int |
179 | fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) |
180 | { |
181 | int ret; |
182 | |
183 | edma_writel(edma: fsl_edma, val: ~0, addr: fsl_edma->regs.intl); |
184 | |
185 | fsl_edma->txirq = platform_get_irq_byname(pdev, "edma-tx" ); |
186 | if (fsl_edma->txirq < 0) |
187 | return fsl_edma->txirq; |
188 | |
189 | fsl_edma->errirq = platform_get_irq_byname(pdev, "edma-err" ); |
190 | if (fsl_edma->errirq < 0) |
191 | return fsl_edma->errirq; |
192 | |
193 | if (fsl_edma->txirq == fsl_edma->errirq) { |
194 | ret = devm_request_irq(dev: &pdev->dev, irq: fsl_edma->txirq, |
195 | handler: fsl_edma_irq_handler, irqflags: 0, devname: "eDMA" , dev_id: fsl_edma); |
196 | if (ret) { |
197 | dev_err(&pdev->dev, "Can't register eDMA IRQ.\n" ); |
198 | return ret; |
199 | } |
200 | } else { |
201 | ret = devm_request_irq(dev: &pdev->dev, irq: fsl_edma->txirq, |
202 | handler: fsl_edma_tx_handler, irqflags: 0, devname: "eDMA tx" , dev_id: fsl_edma); |
203 | if (ret) { |
204 | dev_err(&pdev->dev, "Can't register eDMA tx IRQ.\n" ); |
205 | return ret; |
206 | } |
207 | |
208 | ret = devm_request_irq(dev: &pdev->dev, irq: fsl_edma->errirq, |
209 | handler: fsl_edma_err_handler, irqflags: 0, devname: "eDMA err" , dev_id: fsl_edma); |
210 | if (ret) { |
211 | dev_err(&pdev->dev, "Can't register eDMA err IRQ.\n" ); |
212 | return ret; |
213 | } |
214 | } |
215 | |
216 | return 0; |
217 | } |
218 | |
219 | static int fsl_edma3_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) |
220 | { |
221 | int ret; |
222 | int i; |
223 | |
224 | for (i = 0; i < fsl_edma->n_chans; i++) { |
225 | |
226 | struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i]; |
227 | |
228 | if (fsl_edma->chan_masked & BIT(i)) |
229 | continue; |
230 | |
231 | /* request channel irq */ |
232 | fsl_chan->txirq = platform_get_irq(pdev, i); |
233 | if (fsl_chan->txirq < 0) |
234 | return -EINVAL; |
235 | |
236 | ret = devm_request_irq(dev: &pdev->dev, irq: fsl_chan->txirq, |
237 | handler: fsl_edma3_tx_handler, IRQF_SHARED, |
238 | devname: fsl_chan->chan_name, dev_id: fsl_chan); |
239 | if (ret) { |
240 | dev_err(&pdev->dev, "Can't register chan%d's IRQ.\n" , i); |
241 | return -EINVAL; |
242 | } |
243 | } |
244 | |
245 | return 0; |
246 | } |
247 | |
248 | static int |
249 | fsl_edma2_irq_init(struct platform_device *pdev, |
250 | struct fsl_edma_engine *fsl_edma) |
251 | { |
252 | int i, ret, irq; |
253 | int count; |
254 | |
255 | edma_writel(edma: fsl_edma, val: ~0, addr: fsl_edma->regs.intl); |
256 | |
257 | count = platform_irq_count(pdev); |
258 | dev_dbg(&pdev->dev, "%s Found %d interrupts\r\n" , __func__, count); |
259 | if (count <= 2) { |
260 | dev_err(&pdev->dev, "Interrupts in DTS not correct.\n" ); |
261 | return -EINVAL; |
262 | } |
263 | /* |
264 | * 16 channel independent interrupts + 1 error interrupt on i.mx7ulp. |
265 | * 2 channel share one interrupt, for example, ch0/ch16, ch1/ch17... |
266 | * For now, just simply request irq without IRQF_SHARED flag, since 16 |
267 | * channels are enough on i.mx7ulp whose M4 domain own some peripherals. |
268 | */ |
269 | for (i = 0; i < count; i++) { |
270 | irq = platform_get_irq(pdev, i); |
271 | if (irq < 0) |
272 | return -ENXIO; |
273 | |
274 | /* The last IRQ is for eDMA err */ |
275 | if (i == count - 1) |
276 | ret = devm_request_irq(dev: &pdev->dev, irq, |
277 | handler: fsl_edma_err_handler, |
278 | irqflags: 0, devname: "eDMA2-ERR" , dev_id: fsl_edma); |
279 | else |
280 | ret = devm_request_irq(dev: &pdev->dev, irq, |
281 | handler: fsl_edma_tx_handler, irqflags: 0, |
282 | devname: fsl_edma->chans[i].chan_name, |
283 | dev_id: fsl_edma); |
284 | if (ret) |
285 | return ret; |
286 | } |
287 | |
288 | return 0; |
289 | } |
290 | |
291 | static void fsl_edma_irq_exit( |
292 | struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) |
293 | { |
294 | if (fsl_edma->txirq == fsl_edma->errirq) { |
295 | devm_free_irq(dev: &pdev->dev, irq: fsl_edma->txirq, dev_id: fsl_edma); |
296 | } else { |
297 | devm_free_irq(dev: &pdev->dev, irq: fsl_edma->txirq, dev_id: fsl_edma); |
298 | devm_free_irq(dev: &pdev->dev, irq: fsl_edma->errirq, dev_id: fsl_edma); |
299 | } |
300 | } |
301 | |
302 | static void fsl_disable_clocks(struct fsl_edma_engine *fsl_edma, int nr_clocks) |
303 | { |
304 | int i; |
305 | |
306 | for (i = 0; i < nr_clocks; i++) |
307 | clk_disable_unprepare(clk: fsl_edma->muxclk[i]); |
308 | } |
309 | |
310 | static struct fsl_edma_drvdata vf610_data = { |
311 | .dmamuxs = DMAMUX_NR, |
312 | .flags = FSL_EDMA_DRV_WRAP_IO, |
313 | .chreg_off = EDMA_TCD, |
314 | .chreg_space_sz = sizeof(struct fsl_edma_hw_tcd), |
315 | .setup_irq = fsl_edma_irq_init, |
316 | }; |
317 | |
318 | static struct fsl_edma_drvdata ls1028a_data = { |
319 | .dmamuxs = DMAMUX_NR, |
320 | .flags = FSL_EDMA_DRV_MUX_SWAP | FSL_EDMA_DRV_WRAP_IO, |
321 | .chreg_off = EDMA_TCD, |
322 | .chreg_space_sz = sizeof(struct fsl_edma_hw_tcd), |
323 | .setup_irq = fsl_edma_irq_init, |
324 | }; |
325 | |
326 | static struct fsl_edma_drvdata imx7ulp_data = { |
327 | .dmamuxs = 1, |
328 | .chreg_off = EDMA_TCD, |
329 | .chreg_space_sz = sizeof(struct fsl_edma_hw_tcd), |
330 | .flags = FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_CONFIG32, |
331 | .setup_irq = fsl_edma2_irq_init, |
332 | }; |
333 | |
334 | static struct fsl_edma_drvdata imx8qm_data = { |
335 | .flags = FSL_EDMA_DRV_HAS_PD | FSL_EDMA_DRV_EDMA3, |
336 | .chreg_space_sz = 0x10000, |
337 | .chreg_off = 0x10000, |
338 | .setup_irq = fsl_edma3_irq_init, |
339 | }; |
340 | |
341 | static struct fsl_edma_drvdata imx8qm_audio_data = { |
342 | .flags = FSL_EDMA_DRV_QUIRK_SWAPPED | FSL_EDMA_DRV_HAS_PD | FSL_EDMA_DRV_EDMA3, |
343 | .chreg_space_sz = 0x10000, |
344 | .chreg_off = 0x10000, |
345 | .setup_irq = fsl_edma3_irq_init, |
346 | }; |
347 | |
348 | static struct fsl_edma_drvdata imx93_data3 = { |
349 | .flags = FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_EDMA3, |
350 | .chreg_space_sz = 0x10000, |
351 | .chreg_off = 0x10000, |
352 | .setup_irq = fsl_edma3_irq_init, |
353 | }; |
354 | |
355 | static struct fsl_edma_drvdata imx93_data4 = { |
356 | .flags = FSL_EDMA_DRV_HAS_CHMUX | FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_EDMA4, |
357 | .chreg_space_sz = 0x8000, |
358 | .chreg_off = 0x10000, |
359 | .setup_irq = fsl_edma3_irq_init, |
360 | }; |
361 | |
362 | static const struct of_device_id fsl_edma_dt_ids[] = { |
363 | { .compatible = "fsl,vf610-edma" , .data = &vf610_data}, |
364 | { .compatible = "fsl,ls1028a-edma" , .data = &ls1028a_data}, |
365 | { .compatible = "fsl,imx7ulp-edma" , .data = &imx7ulp_data}, |
366 | { .compatible = "fsl,imx8qm-edma" , .data = &imx8qm_data}, |
367 | { .compatible = "fsl,imx8qm-adma" , .data = &imx8qm_audio_data}, |
368 | { .compatible = "fsl,imx93-edma3" , .data = &imx93_data3}, |
369 | { .compatible = "fsl,imx93-edma4" , .data = &imx93_data4}, |
370 | { /* sentinel */ } |
371 | }; |
372 | MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids); |
373 | |
374 | static int fsl_edma3_attach_pd(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) |
375 | { |
376 | struct fsl_edma_chan *fsl_chan; |
377 | struct device_link *link; |
378 | struct device *pd_chan; |
379 | struct device *dev; |
380 | int i; |
381 | |
382 | dev = &pdev->dev; |
383 | |
384 | for (i = 0; i < fsl_edma->n_chans; i++) { |
385 | if (fsl_edma->chan_masked & BIT(i)) |
386 | continue; |
387 | |
388 | fsl_chan = &fsl_edma->chans[i]; |
389 | |
390 | pd_chan = dev_pm_domain_attach_by_id(dev, index: i); |
391 | if (IS_ERR_OR_NULL(ptr: pd_chan)) { |
392 | dev_err(dev, "Failed attach pd %d\n" , i); |
393 | return -EINVAL; |
394 | } |
395 | |
396 | link = device_link_add(consumer: dev, supplier: pd_chan, DL_FLAG_STATELESS | |
397 | DL_FLAG_PM_RUNTIME | |
398 | DL_FLAG_RPM_ACTIVE); |
399 | if (IS_ERR(ptr: link)) { |
400 | dev_err(dev, "Failed to add device_link to %d: %ld\n" , i, |
401 | PTR_ERR(link)); |
402 | return -EINVAL; |
403 | } |
404 | |
405 | fsl_chan->pd_dev = pd_chan; |
406 | |
407 | pm_runtime_use_autosuspend(dev: fsl_chan->pd_dev); |
408 | pm_runtime_set_autosuspend_delay(dev: fsl_chan->pd_dev, delay: 200); |
409 | pm_runtime_set_active(dev: fsl_chan->pd_dev); |
410 | } |
411 | |
412 | return 0; |
413 | } |
414 | |
415 | static int fsl_edma_probe(struct platform_device *pdev) |
416 | { |
417 | struct device_node *np = pdev->dev.of_node; |
418 | struct fsl_edma_engine *fsl_edma; |
419 | const struct fsl_edma_drvdata *drvdata = NULL; |
420 | u32 chan_mask[2] = {0, 0}; |
421 | struct edma_regs *regs; |
422 | int chans; |
423 | int ret, i; |
424 | |
425 | drvdata = device_get_match_data(dev: &pdev->dev); |
426 | if (!drvdata) { |
427 | dev_err(&pdev->dev, "unable to find driver data\n" ); |
428 | return -EINVAL; |
429 | } |
430 | |
431 | ret = of_property_read_u32(np, propname: "dma-channels" , out_value: &chans); |
432 | if (ret) { |
433 | dev_err(&pdev->dev, "Can't get dma-channels.\n" ); |
434 | return ret; |
435 | } |
436 | |
437 | fsl_edma = devm_kzalloc(dev: &pdev->dev, struct_size(fsl_edma, chans, chans), |
438 | GFP_KERNEL); |
439 | if (!fsl_edma) |
440 | return -ENOMEM; |
441 | |
442 | fsl_edma->drvdata = drvdata; |
443 | fsl_edma->n_chans = chans; |
444 | mutex_init(&fsl_edma->fsl_edma_mutex); |
445 | |
446 | fsl_edma->membase = devm_platform_ioremap_resource(pdev, index: 0); |
447 | if (IS_ERR(ptr: fsl_edma->membase)) |
448 | return PTR_ERR(ptr: fsl_edma->membase); |
449 | |
450 | if (!(drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)) { |
451 | fsl_edma_setup_regs(edma: fsl_edma); |
452 | regs = &fsl_edma->regs; |
453 | } |
454 | |
455 | if (drvdata->flags & FSL_EDMA_DRV_HAS_DMACLK) { |
456 | fsl_edma->dmaclk = devm_clk_get_enabled(dev: &pdev->dev, id: "dma" ); |
457 | if (IS_ERR(ptr: fsl_edma->dmaclk)) { |
458 | dev_err(&pdev->dev, "Missing DMA block clock.\n" ); |
459 | return PTR_ERR(ptr: fsl_edma->dmaclk); |
460 | } |
461 | } |
462 | |
463 | if (drvdata->flags & FSL_EDMA_DRV_HAS_CHCLK) { |
464 | fsl_edma->chclk = devm_clk_get_enabled(dev: &pdev->dev, id: "mp" ); |
465 | if (IS_ERR(ptr: fsl_edma->chclk)) { |
466 | dev_err(&pdev->dev, "Missing MP block clock.\n" ); |
467 | return PTR_ERR(ptr: fsl_edma->chclk); |
468 | } |
469 | } |
470 | |
471 | ret = of_property_read_variable_u32_array(np, propname: "dma-channel-mask" , out_values: chan_mask, sz_min: 1, sz_max: 2); |
472 | |
473 | if (ret > 0) { |
474 | fsl_edma->chan_masked = chan_mask[1]; |
475 | fsl_edma->chan_masked <<= 32; |
476 | fsl_edma->chan_masked |= chan_mask[0]; |
477 | } |
478 | |
479 | for (i = 0; i < fsl_edma->drvdata->dmamuxs; i++) { |
480 | char clkname[32]; |
481 | |
482 | /* eDMAv3 mux register move to TCD area if ch_mux exist */ |
483 | if (drvdata->flags & FSL_EDMA_DRV_SPLIT_REG) |
484 | break; |
485 | |
486 | fsl_edma->muxbase[i] = devm_platform_ioremap_resource(pdev, |
487 | index: 1 + i); |
488 | if (IS_ERR(ptr: fsl_edma->muxbase[i])) { |
489 | /* on error: disable all previously enabled clks */ |
490 | fsl_disable_clocks(fsl_edma, nr_clocks: i); |
491 | return PTR_ERR(ptr: fsl_edma->muxbase[i]); |
492 | } |
493 | |
494 | sprintf(buf: clkname, fmt: "dmamux%d" , i); |
495 | fsl_edma->muxclk[i] = devm_clk_get_enabled(dev: &pdev->dev, id: clkname); |
496 | if (IS_ERR(ptr: fsl_edma->muxclk[i])) { |
497 | dev_err(&pdev->dev, "Missing DMAMUX block clock.\n" ); |
498 | /* on error: disable all previously enabled clks */ |
499 | return PTR_ERR(ptr: fsl_edma->muxclk[i]); |
500 | } |
501 | } |
502 | |
503 | fsl_edma->big_endian = of_property_read_bool(np, propname: "big-endian" ); |
504 | |
505 | if (drvdata->flags & FSL_EDMA_DRV_HAS_PD) { |
506 | ret = fsl_edma3_attach_pd(pdev, fsl_edma); |
507 | if (ret) |
508 | return ret; |
509 | } |
510 | |
511 | INIT_LIST_HEAD(list: &fsl_edma->dma_dev.channels); |
512 | for (i = 0; i < fsl_edma->n_chans; i++) { |
513 | struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i]; |
514 | int len; |
515 | |
516 | if (fsl_edma->chan_masked & BIT(i)) |
517 | continue; |
518 | |
519 | snprintf(buf: fsl_chan->chan_name, size: sizeof(fsl_chan->chan_name), fmt: "%s-CH%02d" , |
520 | dev_name(dev: &pdev->dev), i); |
521 | |
522 | fsl_chan->edma = fsl_edma; |
523 | fsl_chan->pm_state = RUNNING; |
524 | fsl_chan->slave_id = 0; |
525 | fsl_chan->idle = true; |
526 | fsl_chan->dma_dir = DMA_NONE; |
527 | fsl_chan->vchan.desc_free = fsl_edma_free_desc; |
528 | |
529 | len = (drvdata->flags & FSL_EDMA_DRV_SPLIT_REG) ? |
530 | offsetof(struct fsl_edma3_ch_reg, tcd) : 0; |
531 | fsl_chan->tcd = fsl_edma->membase |
532 | + i * drvdata->chreg_space_sz + drvdata->chreg_off + len; |
533 | |
534 | fsl_chan->pdev = pdev; |
535 | vchan_init(vc: &fsl_chan->vchan, dmadev: &fsl_edma->dma_dev); |
536 | |
537 | edma_write_tcdreg(fsl_chan, 0, csr); |
538 | fsl_edma_chan_mux(fsl_chan, slot: 0, enable: false); |
539 | } |
540 | |
541 | ret = fsl_edma->drvdata->setup_irq(pdev, fsl_edma); |
542 | if (ret) |
543 | return ret; |
544 | |
545 | dma_cap_set(DMA_PRIVATE, fsl_edma->dma_dev.cap_mask); |
546 | dma_cap_set(DMA_SLAVE, fsl_edma->dma_dev.cap_mask); |
547 | dma_cap_set(DMA_CYCLIC, fsl_edma->dma_dev.cap_mask); |
548 | dma_cap_set(DMA_MEMCPY, fsl_edma->dma_dev.cap_mask); |
549 | |
550 | fsl_edma->dma_dev.dev = &pdev->dev; |
551 | fsl_edma->dma_dev.device_alloc_chan_resources |
552 | = fsl_edma_alloc_chan_resources; |
553 | fsl_edma->dma_dev.device_free_chan_resources |
554 | = fsl_edma_free_chan_resources; |
555 | fsl_edma->dma_dev.device_tx_status = fsl_edma_tx_status; |
556 | fsl_edma->dma_dev.device_prep_slave_sg = fsl_edma_prep_slave_sg; |
557 | fsl_edma->dma_dev.device_prep_dma_cyclic = fsl_edma_prep_dma_cyclic; |
558 | fsl_edma->dma_dev.device_prep_dma_memcpy = fsl_edma_prep_memcpy; |
559 | fsl_edma->dma_dev.device_config = fsl_edma_slave_config; |
560 | fsl_edma->dma_dev.device_pause = fsl_edma_pause; |
561 | fsl_edma->dma_dev.device_resume = fsl_edma_resume; |
562 | fsl_edma->dma_dev.device_terminate_all = fsl_edma_terminate_all; |
563 | fsl_edma->dma_dev.device_synchronize = fsl_edma_synchronize; |
564 | fsl_edma->dma_dev.device_issue_pending = fsl_edma_issue_pending; |
565 | |
566 | fsl_edma->dma_dev.src_addr_widths = FSL_EDMA_BUSWIDTHS; |
567 | fsl_edma->dma_dev.dst_addr_widths = FSL_EDMA_BUSWIDTHS; |
568 | |
569 | if (drvdata->flags & FSL_EDMA_DRV_BUS_8BYTE) { |
570 | fsl_edma->dma_dev.src_addr_widths |= BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); |
571 | fsl_edma->dma_dev.dst_addr_widths |= BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); |
572 | } |
573 | |
574 | fsl_edma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); |
575 | if (drvdata->flags & FSL_EDMA_DRV_DEV_TO_DEV) |
576 | fsl_edma->dma_dev.directions |= BIT(DMA_DEV_TO_DEV); |
577 | |
578 | fsl_edma->dma_dev.copy_align = drvdata->flags & FSL_EDMA_DRV_ALIGN_64BYTE ? |
579 | DMAENGINE_ALIGN_64_BYTES : |
580 | DMAENGINE_ALIGN_32_BYTES; |
581 | |
582 | /* Per worst case 'nbytes = 1' take CITER as the max_seg_size */ |
583 | dma_set_max_seg_size(dev: fsl_edma->dma_dev.dev, size: 0x3fff); |
584 | |
585 | fsl_edma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; |
586 | |
587 | platform_set_drvdata(pdev, data: fsl_edma); |
588 | |
589 | ret = dma_async_device_register(device: &fsl_edma->dma_dev); |
590 | if (ret) { |
591 | dev_err(&pdev->dev, |
592 | "Can't register Freescale eDMA engine. (%d)\n" , ret); |
593 | return ret; |
594 | } |
595 | |
596 | ret = of_dma_controller_register(np, |
597 | of_dma_xlate: drvdata->flags & FSL_EDMA_DRV_SPLIT_REG ? fsl_edma3_xlate : fsl_edma_xlate, |
598 | data: fsl_edma); |
599 | if (ret) { |
600 | dev_err(&pdev->dev, |
601 | "Can't register Freescale eDMA of_dma. (%d)\n" , ret); |
602 | dma_async_device_unregister(device: &fsl_edma->dma_dev); |
603 | return ret; |
604 | } |
605 | |
606 | /* enable round robin arbitration */ |
607 | if (!(drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)) |
608 | edma_writel(edma: fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, addr: regs->cr); |
609 | |
610 | return 0; |
611 | } |
612 | |
613 | static void fsl_edma_remove(struct platform_device *pdev) |
614 | { |
615 | struct device_node *np = pdev->dev.of_node; |
616 | struct fsl_edma_engine *fsl_edma = platform_get_drvdata(pdev); |
617 | |
618 | fsl_edma_irq_exit(pdev, fsl_edma); |
619 | fsl_edma_cleanup_vchan(dmadev: &fsl_edma->dma_dev); |
620 | of_dma_controller_free(np); |
621 | dma_async_device_unregister(device: &fsl_edma->dma_dev); |
622 | fsl_disable_clocks(fsl_edma, nr_clocks: fsl_edma->drvdata->dmamuxs); |
623 | } |
624 | |
625 | static int fsl_edma_suspend_late(struct device *dev) |
626 | { |
627 | struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev); |
628 | struct fsl_edma_chan *fsl_chan; |
629 | unsigned long flags; |
630 | int i; |
631 | |
632 | for (i = 0; i < fsl_edma->n_chans; i++) { |
633 | fsl_chan = &fsl_edma->chans[i]; |
634 | spin_lock_irqsave(&fsl_chan->vchan.lock, flags); |
635 | /* Make sure chan is idle or will force disable. */ |
636 | if (unlikely(!fsl_chan->idle)) { |
637 | dev_warn(dev, "WARN: There is non-idle channel." ); |
638 | fsl_edma_disable_request(fsl_chan); |
639 | fsl_edma_chan_mux(fsl_chan, slot: 0, enable: false); |
640 | } |
641 | |
642 | fsl_chan->pm_state = SUSPENDED; |
643 | spin_unlock_irqrestore(lock: &fsl_chan->vchan.lock, flags); |
644 | } |
645 | |
646 | return 0; |
647 | } |
648 | |
649 | static int fsl_edma_resume_early(struct device *dev) |
650 | { |
651 | struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev); |
652 | struct fsl_edma_chan *fsl_chan; |
653 | struct edma_regs *regs = &fsl_edma->regs; |
654 | int i; |
655 | |
656 | for (i = 0; i < fsl_edma->n_chans; i++) { |
657 | fsl_chan = &fsl_edma->chans[i]; |
658 | fsl_chan->pm_state = RUNNING; |
659 | edma_write_tcdreg(fsl_chan, 0, csr); |
660 | if (fsl_chan->slave_id != 0) |
661 | fsl_edma_chan_mux(fsl_chan, slot: fsl_chan->slave_id, enable: true); |
662 | } |
663 | |
664 | edma_writel(edma: fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, addr: regs->cr); |
665 | |
666 | return 0; |
667 | } |
668 | |
669 | /* |
670 | * eDMA provides the service to others, so it should be suspend late |
671 | * and resume early. When eDMA suspend, all of the clients should stop |
672 | * the DMA data transmission and let the channel idle. |
673 | */ |
674 | static const struct dev_pm_ops fsl_edma_pm_ops = { |
675 | .suspend_late = fsl_edma_suspend_late, |
676 | .resume_early = fsl_edma_resume_early, |
677 | }; |
678 | |
679 | static struct platform_driver fsl_edma_driver = { |
680 | .driver = { |
681 | .name = "fsl-edma" , |
682 | .of_match_table = fsl_edma_dt_ids, |
683 | .pm = &fsl_edma_pm_ops, |
684 | }, |
685 | .probe = fsl_edma_probe, |
686 | .remove_new = fsl_edma_remove, |
687 | }; |
688 | |
689 | static int __init fsl_edma_init(void) |
690 | { |
691 | return platform_driver_register(&fsl_edma_driver); |
692 | } |
693 | subsys_initcall(fsl_edma_init); |
694 | |
695 | static void __exit fsl_edma_exit(void) |
696 | { |
697 | platform_driver_unregister(&fsl_edma_driver); |
698 | } |
699 | module_exit(fsl_edma_exit); |
700 | |
701 | MODULE_ALIAS("platform:fsl-edma" ); |
702 | MODULE_DESCRIPTION("Freescale eDMA engine driver" ); |
703 | MODULE_LICENSE("GPL v2" ); |
704 | |