1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Freescale MPC85xx, MPC83xx DMA Engine support |
4 | * |
5 | * Copyright (C) 2007-2010 Freescale Semiconductor, Inc. All rights reserved. |
6 | * |
7 | * Author: |
8 | * Zhang Wei <wei.zhang@freescale.com>, Jul 2007 |
9 | * Ebony Zhu <ebony.zhu@freescale.com>, May 2007 |
10 | * |
11 | * Description: |
12 | * DMA engine driver for Freescale MPC8540 DMA controller, which is |
13 | * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc. |
14 | * The support for MPC8349 DMA controller is also added. |
15 | * |
16 | * This driver instructs the DMA controller to issue the PCI Read Multiple |
17 | * command for PCI read operations, instead of using the default PCI Read Line |
18 | * command. Please be aware that this setting may result in read pre-fetching |
19 | * on some platforms. |
20 | */ |
21 | |
22 | #include <linux/init.h> |
23 | #include <linux/module.h> |
24 | #include <linux/pci.h> |
25 | #include <linux/slab.h> |
26 | #include <linux/interrupt.h> |
27 | #include <linux/dmaengine.h> |
28 | #include <linux/delay.h> |
29 | #include <linux/dma-mapping.h> |
30 | #include <linux/dmapool.h> |
31 | #include <linux/of.h> |
32 | #include <linux/of_address.h> |
33 | #include <linux/of_irq.h> |
34 | #include <linux/platform_device.h> |
35 | #include <linux/fsldma.h> |
36 | #include "dmaengine.h" |
37 | #include "fsldma.h" |
38 | |
39 | #define chan_dbg(chan, fmt, arg...) \ |
40 | dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg) |
41 | #define chan_err(chan, fmt, arg...) \ |
42 | dev_err(chan->dev, "%s: " fmt, chan->name, ##arg) |
43 | |
44 | static const char msg_ld_oom[] = "No free memory for link descriptor" ; |
45 | |
46 | /* |
47 | * Register Helpers |
48 | */ |
49 | |
50 | static void set_sr(struct fsldma_chan *chan, u32 val) |
51 | { |
52 | FSL_DMA_OUT(chan, &chan->regs->sr, val, 32); |
53 | } |
54 | |
55 | static u32 get_sr(struct fsldma_chan *chan) |
56 | { |
57 | return FSL_DMA_IN(chan, &chan->regs->sr, 32); |
58 | } |
59 | |
60 | static void set_mr(struct fsldma_chan *chan, u32 val) |
61 | { |
62 | FSL_DMA_OUT(chan, &chan->regs->mr, val, 32); |
63 | } |
64 | |
65 | static u32 get_mr(struct fsldma_chan *chan) |
66 | { |
67 | return FSL_DMA_IN(chan, &chan->regs->mr, 32); |
68 | } |
69 | |
70 | static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr) |
71 | { |
72 | FSL_DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64); |
73 | } |
74 | |
75 | static dma_addr_t get_cdar(struct fsldma_chan *chan) |
76 | { |
77 | return FSL_DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN; |
78 | } |
79 | |
80 | static void set_bcr(struct fsldma_chan *chan, u32 val) |
81 | { |
82 | FSL_DMA_OUT(chan, &chan->regs->bcr, val, 32); |
83 | } |
84 | |
85 | static u32 get_bcr(struct fsldma_chan *chan) |
86 | { |
87 | return FSL_DMA_IN(chan, &chan->regs->bcr, 32); |
88 | } |
89 | |
90 | /* |
91 | * Descriptor Helpers |
92 | */ |
93 | |
94 | static void set_desc_cnt(struct fsldma_chan *chan, |
95 | struct fsl_dma_ld_hw *hw, u32 count) |
96 | { |
97 | hw->count = CPU_TO_DMA(chan, count, 32); |
98 | } |
99 | |
100 | static void set_desc_src(struct fsldma_chan *chan, |
101 | struct fsl_dma_ld_hw *hw, dma_addr_t src) |
102 | { |
103 | u64 snoop_bits; |
104 | |
105 | snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) |
106 | ? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0; |
107 | hw->src_addr = CPU_TO_DMA(chan, snoop_bits | src, 64); |
108 | } |
109 | |
110 | static void set_desc_dst(struct fsldma_chan *chan, |
111 | struct fsl_dma_ld_hw *hw, dma_addr_t dst) |
112 | { |
113 | u64 snoop_bits; |
114 | |
115 | snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) |
116 | ? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0; |
117 | hw->dst_addr = CPU_TO_DMA(chan, snoop_bits | dst, 64); |
118 | } |
119 | |
120 | static void set_desc_next(struct fsldma_chan *chan, |
121 | struct fsl_dma_ld_hw *hw, dma_addr_t next) |
122 | { |
123 | u64 snoop_bits; |
124 | |
125 | snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) |
126 | ? FSL_DMA_SNEN : 0; |
127 | hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits | next, 64); |
128 | } |
129 | |
130 | static void set_ld_eol(struct fsldma_chan *chan, struct fsl_desc_sw *desc) |
131 | { |
132 | u64 snoop_bits; |
133 | |
134 | snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) |
135 | ? FSL_DMA_SNEN : 0; |
136 | |
137 | desc->hw.next_ln_addr = CPU_TO_DMA(chan, |
138 | DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL |
139 | | snoop_bits, 64); |
140 | } |
141 | |
142 | /* |
143 | * DMA Engine Hardware Control Helpers |
144 | */ |
145 | |
146 | static void dma_init(struct fsldma_chan *chan) |
147 | { |
148 | /* Reset the channel */ |
149 | set_mr(chan, val: 0); |
150 | |
151 | switch (chan->feature & FSL_DMA_IP_MASK) { |
152 | case FSL_DMA_IP_85XX: |
153 | /* Set the channel to below modes: |
154 | * EIE - Error interrupt enable |
155 | * EOLNIE - End of links interrupt enable |
156 | * BWC - Bandwidth sharing among channels |
157 | */ |
158 | set_mr(chan, FSL_DMA_MR_BWC | FSL_DMA_MR_EIE |
159 | | FSL_DMA_MR_EOLNIE); |
160 | break; |
161 | case FSL_DMA_IP_83XX: |
162 | /* Set the channel to below modes: |
163 | * EOTIE - End-of-transfer interrupt enable |
164 | * PRC_RM - PCI read multiple |
165 | */ |
166 | set_mr(chan, FSL_DMA_MR_EOTIE | FSL_DMA_MR_PRC_RM); |
167 | break; |
168 | } |
169 | } |
170 | |
171 | static int dma_is_idle(struct fsldma_chan *chan) |
172 | { |
173 | u32 sr = get_sr(chan); |
174 | return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH); |
175 | } |
176 | |
177 | /* |
178 | * Start the DMA controller |
179 | * |
180 | * Preconditions: |
181 | * - the CDAR register must point to the start descriptor |
182 | * - the MRn[CS] bit must be cleared |
183 | */ |
184 | static void dma_start(struct fsldma_chan *chan) |
185 | { |
186 | u32 mode; |
187 | |
188 | mode = get_mr(chan); |
189 | |
190 | if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) { |
191 | set_bcr(chan, val: 0); |
192 | mode |= FSL_DMA_MR_EMP_EN; |
193 | } else { |
194 | mode &= ~FSL_DMA_MR_EMP_EN; |
195 | } |
196 | |
197 | if (chan->feature & FSL_DMA_CHAN_START_EXT) { |
198 | mode |= FSL_DMA_MR_EMS_EN; |
199 | } else { |
200 | mode &= ~FSL_DMA_MR_EMS_EN; |
201 | mode |= FSL_DMA_MR_CS; |
202 | } |
203 | |
204 | set_mr(chan, val: mode); |
205 | } |
206 | |
207 | static void dma_halt(struct fsldma_chan *chan) |
208 | { |
209 | u32 mode; |
210 | int i; |
211 | |
212 | /* read the mode register */ |
213 | mode = get_mr(chan); |
214 | |
215 | /* |
216 | * The 85xx controller supports channel abort, which will stop |
217 | * the current transfer. On 83xx, this bit is the transfer error |
218 | * mask bit, which should not be changed. |
219 | */ |
220 | if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { |
221 | mode |= FSL_DMA_MR_CA; |
222 | set_mr(chan, val: mode); |
223 | |
224 | mode &= ~FSL_DMA_MR_CA; |
225 | } |
226 | |
227 | /* stop the DMA controller */ |
228 | mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN); |
229 | set_mr(chan, val: mode); |
230 | |
231 | /* wait for the DMA controller to become idle */ |
232 | for (i = 0; i < 100; i++) { |
233 | if (dma_is_idle(chan)) |
234 | return; |
235 | |
236 | udelay(10); |
237 | } |
238 | |
239 | if (!dma_is_idle(chan)) |
240 | chan_err(chan, "DMA halt timeout!\n" ); |
241 | } |
242 | |
243 | /** |
244 | * fsl_chan_set_src_loop_size - Set source address hold transfer size |
245 | * @chan : Freescale DMA channel |
246 | * @size : Address loop size, 0 for disable loop |
247 | * |
248 | * The set source address hold transfer size. The source |
249 | * address hold or loop transfer size is when the DMA transfer |
250 | * data from source address (SA), if the loop size is 4, the DMA will |
251 | * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA, |
252 | * SA + 1 ... and so on. |
253 | */ |
254 | static void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size) |
255 | { |
256 | u32 mode; |
257 | |
258 | mode = get_mr(chan); |
259 | |
260 | switch (size) { |
261 | case 0: |
262 | mode &= ~FSL_DMA_MR_SAHE; |
263 | break; |
264 | case 1: |
265 | case 2: |
266 | case 4: |
267 | case 8: |
268 | mode &= ~FSL_DMA_MR_SAHTS_MASK; |
269 | mode |= FSL_DMA_MR_SAHE | (__ilog2(size) << 14); |
270 | break; |
271 | } |
272 | |
273 | set_mr(chan, val: mode); |
274 | } |
275 | |
276 | /** |
277 | * fsl_chan_set_dst_loop_size - Set destination address hold transfer size |
278 | * @chan : Freescale DMA channel |
279 | * @size : Address loop size, 0 for disable loop |
280 | * |
281 | * The set destination address hold transfer size. The destination |
282 | * address hold or loop transfer size is when the DMA transfer |
283 | * data to destination address (TA), if the loop size is 4, the DMA will |
284 | * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA, |
285 | * TA + 1 ... and so on. |
286 | */ |
287 | static void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size) |
288 | { |
289 | u32 mode; |
290 | |
291 | mode = get_mr(chan); |
292 | |
293 | switch (size) { |
294 | case 0: |
295 | mode &= ~FSL_DMA_MR_DAHE; |
296 | break; |
297 | case 1: |
298 | case 2: |
299 | case 4: |
300 | case 8: |
301 | mode &= ~FSL_DMA_MR_DAHTS_MASK; |
302 | mode |= FSL_DMA_MR_DAHE | (__ilog2(size) << 16); |
303 | break; |
304 | } |
305 | |
306 | set_mr(chan, val: mode); |
307 | } |
308 | |
309 | /** |
310 | * fsl_chan_set_request_count - Set DMA Request Count for external control |
311 | * @chan : Freescale DMA channel |
312 | * @size : Number of bytes to transfer in a single request |
313 | * |
314 | * The Freescale DMA channel can be controlled by the external signal DREQ#. |
315 | * The DMA request count is how many bytes are allowed to transfer before |
316 | * pausing the channel, after which a new assertion of DREQ# resumes channel |
317 | * operation. |
318 | * |
319 | * A size of 0 disables external pause control. The maximum size is 1024. |
320 | */ |
321 | static void fsl_chan_set_request_count(struct fsldma_chan *chan, int size) |
322 | { |
323 | u32 mode; |
324 | |
325 | BUG_ON(size > 1024); |
326 | |
327 | mode = get_mr(chan); |
328 | mode &= ~FSL_DMA_MR_BWC_MASK; |
329 | mode |= (__ilog2(size) << 24) & FSL_DMA_MR_BWC_MASK; |
330 | |
331 | set_mr(chan, val: mode); |
332 | } |
333 | |
334 | /** |
335 | * fsl_chan_toggle_ext_pause - Toggle channel external pause status |
336 | * @chan : Freescale DMA channel |
337 | * @enable : 0 is disabled, 1 is enabled. |
338 | * |
339 | * The Freescale DMA channel can be controlled by the external signal DREQ#. |
340 | * The DMA Request Count feature should be used in addition to this feature |
341 | * to set the number of bytes to transfer before pausing the channel. |
342 | */ |
343 | static void fsl_chan_toggle_ext_pause(struct fsldma_chan *chan, int enable) |
344 | { |
345 | if (enable) |
346 | chan->feature |= FSL_DMA_CHAN_PAUSE_EXT; |
347 | else |
348 | chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT; |
349 | } |
350 | |
351 | /** |
352 | * fsl_chan_toggle_ext_start - Toggle channel external start status |
353 | * @chan : Freescale DMA channel |
354 | * @enable : 0 is disabled, 1 is enabled. |
355 | * |
356 | * If enable the external start, the channel can be started by an |
357 | * external DMA start pin. So the dma_start() does not start the |
358 | * transfer immediately. The DMA channel will wait for the |
359 | * control pin asserted. |
360 | */ |
361 | static void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable) |
362 | { |
363 | if (enable) |
364 | chan->feature |= FSL_DMA_CHAN_START_EXT; |
365 | else |
366 | chan->feature &= ~FSL_DMA_CHAN_START_EXT; |
367 | } |
368 | |
369 | int fsl_dma_external_start(struct dma_chan *dchan, int enable) |
370 | { |
371 | struct fsldma_chan *chan; |
372 | |
373 | if (!dchan) |
374 | return -EINVAL; |
375 | |
376 | chan = to_fsl_chan(dchan); |
377 | |
378 | fsl_chan_toggle_ext_start(chan, enable); |
379 | return 0; |
380 | } |
381 | EXPORT_SYMBOL_GPL(fsl_dma_external_start); |
382 | |
383 | static void append_ld_queue(struct fsldma_chan *chan, struct fsl_desc_sw *desc) |
384 | { |
385 | struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev); |
386 | |
387 | if (list_empty(head: &chan->ld_pending)) |
388 | goto out_splice; |
389 | |
390 | /* |
391 | * Add the hardware descriptor to the chain of hardware descriptors |
392 | * that already exists in memory. |
393 | * |
394 | * This will un-set the EOL bit of the existing transaction, and the |
395 | * last link in this transaction will become the EOL descriptor. |
396 | */ |
397 | set_desc_next(chan, hw: &tail->hw, next: desc->async_tx.phys); |
398 | |
399 | /* |
400 | * Add the software descriptor and all children to the list |
401 | * of pending transactions |
402 | */ |
403 | out_splice: |
404 | list_splice_tail_init(list: &desc->tx_list, head: &chan->ld_pending); |
405 | } |
406 | |
407 | static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx) |
408 | { |
409 | struct fsldma_chan *chan = to_fsl_chan(tx->chan); |
410 | struct fsl_desc_sw *desc = tx_to_fsl_desc(tx); |
411 | struct fsl_desc_sw *child; |
412 | dma_cookie_t cookie = -EINVAL; |
413 | |
414 | spin_lock_bh(lock: &chan->desc_lock); |
415 | |
416 | #ifdef CONFIG_PM |
417 | if (unlikely(chan->pm_state != RUNNING)) { |
418 | chan_dbg(chan, "cannot submit due to suspend\n" ); |
419 | spin_unlock_bh(lock: &chan->desc_lock); |
420 | return -1; |
421 | } |
422 | #endif |
423 | |
424 | /* |
425 | * assign cookies to all of the software descriptors |
426 | * that make up this transaction |
427 | */ |
428 | list_for_each_entry(child, &desc->tx_list, node) { |
429 | cookie = dma_cookie_assign(tx: &child->async_tx); |
430 | } |
431 | |
432 | /* put this transaction onto the tail of the pending queue */ |
433 | append_ld_queue(chan, desc); |
434 | |
435 | spin_unlock_bh(lock: &chan->desc_lock); |
436 | |
437 | return cookie; |
438 | } |
439 | |
440 | /** |
441 | * fsl_dma_free_descriptor - Free descriptor from channel's DMA pool. |
442 | * @chan : Freescale DMA channel |
443 | * @desc: descriptor to be freed |
444 | */ |
445 | static void fsl_dma_free_descriptor(struct fsldma_chan *chan, |
446 | struct fsl_desc_sw *desc) |
447 | { |
448 | list_del(entry: &desc->node); |
449 | chan_dbg(chan, "LD %p free\n" , desc); |
450 | dma_pool_free(pool: chan->desc_pool, vaddr: desc, addr: desc->async_tx.phys); |
451 | } |
452 | |
453 | /** |
454 | * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool. |
455 | * @chan : Freescale DMA channel |
456 | * |
457 | * Return - The descriptor allocated. NULL for failed. |
458 | */ |
459 | static struct fsl_desc_sw *fsl_dma_alloc_descriptor(struct fsldma_chan *chan) |
460 | { |
461 | struct fsl_desc_sw *desc; |
462 | dma_addr_t pdesc; |
463 | |
464 | desc = dma_pool_zalloc(pool: chan->desc_pool, GFP_ATOMIC, handle: &pdesc); |
465 | if (!desc) { |
466 | chan_dbg(chan, "out of memory for link descriptor\n" ); |
467 | return NULL; |
468 | } |
469 | |
470 | INIT_LIST_HEAD(list: &desc->tx_list); |
471 | dma_async_tx_descriptor_init(tx: &desc->async_tx, chan: &chan->common); |
472 | desc->async_tx.tx_submit = fsl_dma_tx_submit; |
473 | desc->async_tx.phys = pdesc; |
474 | |
475 | chan_dbg(chan, "LD %p allocated\n" , desc); |
476 | |
477 | return desc; |
478 | } |
479 | |
480 | /** |
481 | * fsldma_clean_completed_descriptor - free all descriptors which |
482 | * has been completed and acked |
483 | * @chan: Freescale DMA channel |
484 | * |
485 | * This function is used on all completed and acked descriptors. |
486 | * All descriptors should only be freed in this function. |
487 | */ |
488 | static void fsldma_clean_completed_descriptor(struct fsldma_chan *chan) |
489 | { |
490 | struct fsl_desc_sw *desc, *_desc; |
491 | |
492 | /* Run the callback for each descriptor, in order */ |
493 | list_for_each_entry_safe(desc, _desc, &chan->ld_completed, node) |
494 | if (async_tx_test_ack(tx: &desc->async_tx)) |
495 | fsl_dma_free_descriptor(chan, desc); |
496 | } |
497 | |
498 | /** |
499 | * fsldma_run_tx_complete_actions - cleanup a single link descriptor |
500 | * @chan: Freescale DMA channel |
501 | * @desc: descriptor to cleanup and free |
502 | * @cookie: Freescale DMA transaction identifier |
503 | * |
504 | * This function is used on a descriptor which has been executed by the DMA |
505 | * controller. It will run any callbacks, submit any dependencies. |
506 | */ |
507 | static dma_cookie_t fsldma_run_tx_complete_actions(struct fsldma_chan *chan, |
508 | struct fsl_desc_sw *desc, dma_cookie_t cookie) |
509 | { |
510 | struct dma_async_tx_descriptor *txd = &desc->async_tx; |
511 | dma_cookie_t ret = cookie; |
512 | |
513 | BUG_ON(txd->cookie < 0); |
514 | |
515 | if (txd->cookie > 0) { |
516 | ret = txd->cookie; |
517 | |
518 | dma_descriptor_unmap(tx: txd); |
519 | /* Run the link descriptor callback function */ |
520 | dmaengine_desc_get_callback_invoke(tx: txd, NULL); |
521 | } |
522 | |
523 | /* Run any dependencies */ |
524 | dma_run_dependencies(tx: txd); |
525 | |
526 | return ret; |
527 | } |
528 | |
529 | /** |
530 | * fsldma_clean_running_descriptor - move the completed descriptor from |
531 | * ld_running to ld_completed |
532 | * @chan: Freescale DMA channel |
533 | * @desc: the descriptor which is completed |
534 | * |
535 | * Free the descriptor directly if acked by async_tx api, or move it to |
536 | * queue ld_completed. |
537 | */ |
538 | static void fsldma_clean_running_descriptor(struct fsldma_chan *chan, |
539 | struct fsl_desc_sw *desc) |
540 | { |
541 | /* Remove from the list of transactions */ |
542 | list_del(entry: &desc->node); |
543 | |
544 | /* |
545 | * the client is allowed to attach dependent operations |
546 | * until 'ack' is set |
547 | */ |
548 | if (!async_tx_test_ack(tx: &desc->async_tx)) { |
549 | /* |
550 | * Move this descriptor to the list of descriptors which is |
551 | * completed, but still awaiting the 'ack' bit to be set. |
552 | */ |
553 | list_add_tail(new: &desc->node, head: &chan->ld_completed); |
554 | return; |
555 | } |
556 | |
557 | dma_pool_free(pool: chan->desc_pool, vaddr: desc, addr: desc->async_tx.phys); |
558 | } |
559 | |
560 | /** |
561 | * fsl_chan_xfer_ld_queue - transfer any pending transactions |
562 | * @chan : Freescale DMA channel |
563 | * |
564 | * HARDWARE STATE: idle |
565 | * LOCKING: must hold chan->desc_lock |
566 | */ |
567 | static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan) |
568 | { |
569 | struct fsl_desc_sw *desc; |
570 | |
571 | /* |
572 | * If the list of pending descriptors is empty, then we |
573 | * don't need to do any work at all |
574 | */ |
575 | if (list_empty(head: &chan->ld_pending)) { |
576 | chan_dbg(chan, "no pending LDs\n" ); |
577 | return; |
578 | } |
579 | |
580 | /* |
581 | * The DMA controller is not idle, which means that the interrupt |
582 | * handler will start any queued transactions when it runs after |
583 | * this transaction finishes |
584 | */ |
585 | if (!chan->idle) { |
586 | chan_dbg(chan, "DMA controller still busy\n" ); |
587 | return; |
588 | } |
589 | |
590 | /* |
591 | * If there are some link descriptors which have not been |
592 | * transferred, we need to start the controller |
593 | */ |
594 | |
595 | /* |
596 | * Move all elements from the queue of pending transactions |
597 | * onto the list of running transactions |
598 | */ |
599 | chan_dbg(chan, "idle, starting controller\n" ); |
600 | desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node); |
601 | list_splice_tail_init(list: &chan->ld_pending, head: &chan->ld_running); |
602 | |
603 | /* |
604 | * The 85xx DMA controller doesn't clear the channel start bit |
605 | * automatically at the end of a transfer. Therefore we must clear |
606 | * it in software before starting the transfer. |
607 | */ |
608 | if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { |
609 | u32 mode; |
610 | |
611 | mode = get_mr(chan); |
612 | mode &= ~FSL_DMA_MR_CS; |
613 | set_mr(chan, val: mode); |
614 | } |
615 | |
616 | /* |
617 | * Program the descriptor's address into the DMA controller, |
618 | * then start the DMA transaction |
619 | */ |
620 | set_cdar(chan, addr: desc->async_tx.phys); |
621 | get_cdar(chan); |
622 | |
623 | dma_start(chan); |
624 | chan->idle = false; |
625 | } |
626 | |
627 | /** |
628 | * fsldma_cleanup_descriptors - cleanup link descriptors which are completed |
629 | * and move them to ld_completed to free until flag 'ack' is set |
630 | * @chan: Freescale DMA channel |
631 | * |
632 | * This function is used on descriptors which have been executed by the DMA |
633 | * controller. It will run any callbacks, submit any dependencies, then |
634 | * free these descriptors if flag 'ack' is set. |
635 | */ |
636 | static void fsldma_cleanup_descriptors(struct fsldma_chan *chan) |
637 | { |
638 | struct fsl_desc_sw *desc, *_desc; |
639 | dma_cookie_t cookie = 0; |
640 | dma_addr_t curr_phys = get_cdar(chan); |
641 | int seen_current = 0; |
642 | |
643 | fsldma_clean_completed_descriptor(chan); |
644 | |
645 | /* Run the callback for each descriptor, in order */ |
646 | list_for_each_entry_safe(desc, _desc, &chan->ld_running, node) { |
647 | /* |
648 | * do not advance past the current descriptor loaded into the |
649 | * hardware channel, subsequent descriptors are either in |
650 | * process or have not been submitted |
651 | */ |
652 | if (seen_current) |
653 | break; |
654 | |
655 | /* |
656 | * stop the search if we reach the current descriptor and the |
657 | * channel is busy |
658 | */ |
659 | if (desc->async_tx.phys == curr_phys) { |
660 | seen_current = 1; |
661 | if (!dma_is_idle(chan)) |
662 | break; |
663 | } |
664 | |
665 | cookie = fsldma_run_tx_complete_actions(chan, desc, cookie); |
666 | |
667 | fsldma_clean_running_descriptor(chan, desc); |
668 | } |
669 | |
670 | /* |
671 | * Start any pending transactions automatically |
672 | * |
673 | * In the ideal case, we keep the DMA controller busy while we go |
674 | * ahead and free the descriptors below. |
675 | */ |
676 | fsl_chan_xfer_ld_queue(chan); |
677 | |
678 | if (cookie > 0) |
679 | chan->common.completed_cookie = cookie; |
680 | } |
681 | |
682 | /** |
683 | * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel. |
684 | * @chan : Freescale DMA channel |
685 | * |
686 | * This function will create a dma pool for descriptor allocation. |
687 | * |
688 | * Return - The number of descriptors allocated. |
689 | */ |
690 | static int fsl_dma_alloc_chan_resources(struct dma_chan *dchan) |
691 | { |
692 | struct fsldma_chan *chan = to_fsl_chan(dchan); |
693 | |
694 | /* Has this channel already been allocated? */ |
695 | if (chan->desc_pool) |
696 | return 1; |
697 | |
698 | /* |
699 | * We need the descriptor to be aligned to 32bytes |
700 | * for meeting FSL DMA specification requirement. |
701 | */ |
702 | chan->desc_pool = dma_pool_create(name: chan->name, dev: chan->dev, |
703 | size: sizeof(struct fsl_desc_sw), |
704 | align: __alignof__(struct fsl_desc_sw), allocation: 0); |
705 | if (!chan->desc_pool) { |
706 | chan_err(chan, "unable to allocate descriptor pool\n" ); |
707 | return -ENOMEM; |
708 | } |
709 | |
710 | /* there is at least one descriptor free to be allocated */ |
711 | return 1; |
712 | } |
713 | |
714 | /** |
715 | * fsldma_free_desc_list - Free all descriptors in a queue |
716 | * @chan: Freescae DMA channel |
717 | * @list: the list to free |
718 | * |
719 | * LOCKING: must hold chan->desc_lock |
720 | */ |
721 | static void fsldma_free_desc_list(struct fsldma_chan *chan, |
722 | struct list_head *list) |
723 | { |
724 | struct fsl_desc_sw *desc, *_desc; |
725 | |
726 | list_for_each_entry_safe(desc, _desc, list, node) |
727 | fsl_dma_free_descriptor(chan, desc); |
728 | } |
729 | |
730 | static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan, |
731 | struct list_head *list) |
732 | { |
733 | struct fsl_desc_sw *desc, *_desc; |
734 | |
735 | list_for_each_entry_safe_reverse(desc, _desc, list, node) |
736 | fsl_dma_free_descriptor(chan, desc); |
737 | } |
738 | |
739 | /** |
740 | * fsl_dma_free_chan_resources - Free all resources of the channel. |
741 | * @chan : Freescale DMA channel |
742 | */ |
743 | static void fsl_dma_free_chan_resources(struct dma_chan *dchan) |
744 | { |
745 | struct fsldma_chan *chan = to_fsl_chan(dchan); |
746 | |
747 | chan_dbg(chan, "free all channel resources\n" ); |
748 | spin_lock_bh(lock: &chan->desc_lock); |
749 | fsldma_cleanup_descriptors(chan); |
750 | fsldma_free_desc_list(chan, list: &chan->ld_pending); |
751 | fsldma_free_desc_list(chan, list: &chan->ld_running); |
752 | fsldma_free_desc_list(chan, list: &chan->ld_completed); |
753 | spin_unlock_bh(lock: &chan->desc_lock); |
754 | |
755 | dma_pool_destroy(pool: chan->desc_pool); |
756 | chan->desc_pool = NULL; |
757 | } |
758 | |
759 | static struct dma_async_tx_descriptor * |
760 | fsl_dma_prep_memcpy(struct dma_chan *dchan, |
761 | dma_addr_t dma_dst, dma_addr_t dma_src, |
762 | size_t len, unsigned long flags) |
763 | { |
764 | struct fsldma_chan *chan; |
765 | struct fsl_desc_sw *first = NULL, *prev = NULL, *new; |
766 | size_t copy; |
767 | |
768 | if (!dchan) |
769 | return NULL; |
770 | |
771 | if (!len) |
772 | return NULL; |
773 | |
774 | chan = to_fsl_chan(dchan); |
775 | |
776 | do { |
777 | |
778 | /* Allocate the link descriptor from DMA pool */ |
779 | new = fsl_dma_alloc_descriptor(chan); |
780 | if (!new) { |
781 | chan_err(chan, "%s\n" , msg_ld_oom); |
782 | goto fail; |
783 | } |
784 | |
785 | copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT); |
786 | |
787 | set_desc_cnt(chan, hw: &new->hw, count: copy); |
788 | set_desc_src(chan, hw: &new->hw, src: dma_src); |
789 | set_desc_dst(chan, hw: &new->hw, dst: dma_dst); |
790 | |
791 | if (!first) |
792 | first = new; |
793 | else |
794 | set_desc_next(chan, hw: &prev->hw, next: new->async_tx.phys); |
795 | |
796 | new->async_tx.cookie = 0; |
797 | async_tx_ack(tx: &new->async_tx); |
798 | |
799 | prev = new; |
800 | len -= copy; |
801 | dma_src += copy; |
802 | dma_dst += copy; |
803 | |
804 | /* Insert the link descriptor to the LD ring */ |
805 | list_add_tail(new: &new->node, head: &first->tx_list); |
806 | } while (len); |
807 | |
808 | new->async_tx.flags = flags; /* client is in control of this ack */ |
809 | new->async_tx.cookie = -EBUSY; |
810 | |
811 | /* Set End-of-link to the last link descriptor of new list */ |
812 | set_ld_eol(chan, desc: new); |
813 | |
814 | return &first->async_tx; |
815 | |
816 | fail: |
817 | if (!first) |
818 | return NULL; |
819 | |
820 | fsldma_free_desc_list_reverse(chan, list: &first->tx_list); |
821 | return NULL; |
822 | } |
823 | |
824 | static int fsl_dma_device_terminate_all(struct dma_chan *dchan) |
825 | { |
826 | struct fsldma_chan *chan; |
827 | |
828 | if (!dchan) |
829 | return -EINVAL; |
830 | |
831 | chan = to_fsl_chan(dchan); |
832 | |
833 | spin_lock_bh(lock: &chan->desc_lock); |
834 | |
835 | /* Halt the DMA engine */ |
836 | dma_halt(chan); |
837 | |
838 | /* Remove and free all of the descriptors in the LD queue */ |
839 | fsldma_free_desc_list(chan, list: &chan->ld_pending); |
840 | fsldma_free_desc_list(chan, list: &chan->ld_running); |
841 | fsldma_free_desc_list(chan, list: &chan->ld_completed); |
842 | chan->idle = true; |
843 | |
844 | spin_unlock_bh(lock: &chan->desc_lock); |
845 | return 0; |
846 | } |
847 | |
848 | static int fsl_dma_device_config(struct dma_chan *dchan, |
849 | struct dma_slave_config *config) |
850 | { |
851 | struct fsldma_chan *chan; |
852 | int size; |
853 | |
854 | if (!dchan) |
855 | return -EINVAL; |
856 | |
857 | chan = to_fsl_chan(dchan); |
858 | |
859 | /* make sure the channel supports setting burst size */ |
860 | if (!chan->set_request_count) |
861 | return -ENXIO; |
862 | |
863 | /* we set the controller burst size depending on direction */ |
864 | if (config->direction == DMA_MEM_TO_DEV) |
865 | size = config->dst_addr_width * config->dst_maxburst; |
866 | else |
867 | size = config->src_addr_width * config->src_maxburst; |
868 | |
869 | chan->set_request_count(chan, size); |
870 | return 0; |
871 | } |
872 | |
873 | |
874 | /** |
875 | * fsl_dma_memcpy_issue_pending - Issue the DMA start command |
876 | * @chan : Freescale DMA channel |
877 | */ |
878 | static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan) |
879 | { |
880 | struct fsldma_chan *chan = to_fsl_chan(dchan); |
881 | |
882 | spin_lock_bh(lock: &chan->desc_lock); |
883 | fsl_chan_xfer_ld_queue(chan); |
884 | spin_unlock_bh(lock: &chan->desc_lock); |
885 | } |
886 | |
887 | /** |
888 | * fsl_tx_status - Determine the DMA status |
889 | * @chan : Freescale DMA channel |
890 | */ |
891 | static enum dma_status fsl_tx_status(struct dma_chan *dchan, |
892 | dma_cookie_t cookie, |
893 | struct dma_tx_state *txstate) |
894 | { |
895 | struct fsldma_chan *chan = to_fsl_chan(dchan); |
896 | enum dma_status ret; |
897 | |
898 | ret = dma_cookie_status(chan: dchan, cookie, state: txstate); |
899 | if (ret == DMA_COMPLETE) |
900 | return ret; |
901 | |
902 | spin_lock_bh(lock: &chan->desc_lock); |
903 | fsldma_cleanup_descriptors(chan); |
904 | spin_unlock_bh(lock: &chan->desc_lock); |
905 | |
906 | return dma_cookie_status(chan: dchan, cookie, state: txstate); |
907 | } |
908 | |
909 | /*----------------------------------------------------------------------------*/ |
910 | /* Interrupt Handling */ |
911 | /*----------------------------------------------------------------------------*/ |
912 | |
913 | static irqreturn_t fsldma_chan_irq(int irq, void *data) |
914 | { |
915 | struct fsldma_chan *chan = data; |
916 | u32 stat; |
917 | |
918 | /* save and clear the status register */ |
919 | stat = get_sr(chan); |
920 | set_sr(chan, val: stat); |
921 | chan_dbg(chan, "irq: stat = 0x%x\n" , stat); |
922 | |
923 | /* check that this was really our device */ |
924 | stat &= ~(FSL_DMA_SR_CB | FSL_DMA_SR_CH); |
925 | if (!stat) |
926 | return IRQ_NONE; |
927 | |
928 | if (stat & FSL_DMA_SR_TE) |
929 | chan_err(chan, "Transfer Error!\n" ); |
930 | |
931 | /* |
932 | * Programming Error |
933 | * The DMA_INTERRUPT async_tx is a NULL transfer, which will |
934 | * trigger a PE interrupt. |
935 | */ |
936 | if (stat & FSL_DMA_SR_PE) { |
937 | chan_dbg(chan, "irq: Programming Error INT\n" ); |
938 | stat &= ~FSL_DMA_SR_PE; |
939 | if (get_bcr(chan) != 0) |
940 | chan_err(chan, "Programming Error!\n" ); |
941 | } |
942 | |
943 | /* |
944 | * For MPC8349, EOCDI event need to update cookie |
945 | * and start the next transfer if it exist. |
946 | */ |
947 | if (stat & FSL_DMA_SR_EOCDI) { |
948 | chan_dbg(chan, "irq: End-of-Chain link INT\n" ); |
949 | stat &= ~FSL_DMA_SR_EOCDI; |
950 | } |
951 | |
952 | /* |
953 | * If it current transfer is the end-of-transfer, |
954 | * we should clear the Channel Start bit for |
955 | * prepare next transfer. |
956 | */ |
957 | if (stat & FSL_DMA_SR_EOLNI) { |
958 | chan_dbg(chan, "irq: End-of-link INT\n" ); |
959 | stat &= ~FSL_DMA_SR_EOLNI; |
960 | } |
961 | |
962 | /* check that the DMA controller is really idle */ |
963 | if (!dma_is_idle(chan)) |
964 | chan_err(chan, "irq: controller not idle!\n" ); |
965 | |
966 | /* check that we handled all of the bits */ |
967 | if (stat) |
968 | chan_err(chan, "irq: unhandled sr 0x%08x\n" , stat); |
969 | |
970 | /* |
971 | * Schedule the tasklet to handle all cleanup of the current |
972 | * transaction. It will start a new transaction if there is |
973 | * one pending. |
974 | */ |
975 | tasklet_schedule(t: &chan->tasklet); |
976 | chan_dbg(chan, "irq: Exit\n" ); |
977 | return IRQ_HANDLED; |
978 | } |
979 | |
980 | static void dma_do_tasklet(struct tasklet_struct *t) |
981 | { |
982 | struct fsldma_chan *chan = from_tasklet(chan, t, tasklet); |
983 | |
984 | chan_dbg(chan, "tasklet entry\n" ); |
985 | |
986 | spin_lock(lock: &chan->desc_lock); |
987 | |
988 | /* the hardware is now idle and ready for more */ |
989 | chan->idle = true; |
990 | |
991 | /* Run all cleanup for descriptors which have been completed */ |
992 | fsldma_cleanup_descriptors(chan); |
993 | |
994 | spin_unlock(lock: &chan->desc_lock); |
995 | |
996 | chan_dbg(chan, "tasklet exit\n" ); |
997 | } |
998 | |
999 | static irqreturn_t fsldma_ctrl_irq(int irq, void *data) |
1000 | { |
1001 | struct fsldma_device *fdev = data; |
1002 | struct fsldma_chan *chan; |
1003 | unsigned int handled = 0; |
1004 | u32 gsr, mask; |
1005 | int i; |
1006 | |
1007 | gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->regs) |
1008 | : in_le32(fdev->regs); |
1009 | mask = 0xff000000; |
1010 | dev_dbg(fdev->dev, "IRQ: gsr 0x%.8x\n" , gsr); |
1011 | |
1012 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { |
1013 | chan = fdev->chan[i]; |
1014 | if (!chan) |
1015 | continue; |
1016 | |
1017 | if (gsr & mask) { |
1018 | dev_dbg(fdev->dev, "IRQ: chan %d\n" , chan->id); |
1019 | fsldma_chan_irq(irq, data: chan); |
1020 | handled++; |
1021 | } |
1022 | |
1023 | gsr &= ~mask; |
1024 | mask >>= 8; |
1025 | } |
1026 | |
1027 | return IRQ_RETVAL(handled); |
1028 | } |
1029 | |
1030 | static void fsldma_free_irqs(struct fsldma_device *fdev) |
1031 | { |
1032 | struct fsldma_chan *chan; |
1033 | int i; |
1034 | |
1035 | if (fdev->irq) { |
1036 | dev_dbg(fdev->dev, "free per-controller IRQ\n" ); |
1037 | free_irq(fdev->irq, fdev); |
1038 | return; |
1039 | } |
1040 | |
1041 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { |
1042 | chan = fdev->chan[i]; |
1043 | if (chan && chan->irq) { |
1044 | chan_dbg(chan, "free per-channel IRQ\n" ); |
1045 | free_irq(chan->irq, chan); |
1046 | } |
1047 | } |
1048 | } |
1049 | |
1050 | static int fsldma_request_irqs(struct fsldma_device *fdev) |
1051 | { |
1052 | struct fsldma_chan *chan; |
1053 | int ret; |
1054 | int i; |
1055 | |
1056 | /* if we have a per-controller IRQ, use that */ |
1057 | if (fdev->irq) { |
1058 | dev_dbg(fdev->dev, "request per-controller IRQ\n" ); |
1059 | ret = request_irq(irq: fdev->irq, handler: fsldma_ctrl_irq, IRQF_SHARED, |
1060 | name: "fsldma-controller" , dev: fdev); |
1061 | return ret; |
1062 | } |
1063 | |
1064 | /* no per-controller IRQ, use the per-channel IRQs */ |
1065 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { |
1066 | chan = fdev->chan[i]; |
1067 | if (!chan) |
1068 | continue; |
1069 | |
1070 | if (!chan->irq) { |
1071 | chan_err(chan, "interrupts property missing in device tree\n" ); |
1072 | ret = -ENODEV; |
1073 | goto out_unwind; |
1074 | } |
1075 | |
1076 | chan_dbg(chan, "request per-channel IRQ\n" ); |
1077 | ret = request_irq(irq: chan->irq, handler: fsldma_chan_irq, IRQF_SHARED, |
1078 | name: "fsldma-chan" , dev: chan); |
1079 | if (ret) { |
1080 | chan_err(chan, "unable to request per-channel IRQ\n" ); |
1081 | goto out_unwind; |
1082 | } |
1083 | } |
1084 | |
1085 | return 0; |
1086 | |
1087 | out_unwind: |
1088 | for (/* none */; i >= 0; i--) { |
1089 | chan = fdev->chan[i]; |
1090 | if (!chan) |
1091 | continue; |
1092 | |
1093 | if (!chan->irq) |
1094 | continue; |
1095 | |
1096 | free_irq(chan->irq, chan); |
1097 | } |
1098 | |
1099 | return ret; |
1100 | } |
1101 | |
1102 | /*----------------------------------------------------------------------------*/ |
1103 | /* OpenFirmware Subsystem */ |
1104 | /*----------------------------------------------------------------------------*/ |
1105 | |
1106 | static int fsl_dma_chan_probe(struct fsldma_device *fdev, |
1107 | struct device_node *node, u32 feature, const char *compatible) |
1108 | { |
1109 | struct fsldma_chan *chan; |
1110 | struct resource res; |
1111 | int err; |
1112 | |
1113 | /* alloc channel */ |
1114 | chan = kzalloc(size: sizeof(*chan), GFP_KERNEL); |
1115 | if (!chan) { |
1116 | err = -ENOMEM; |
1117 | goto out_return; |
1118 | } |
1119 | |
1120 | /* ioremap registers for use */ |
1121 | chan->regs = of_iomap(node, index: 0); |
1122 | if (!chan->regs) { |
1123 | dev_err(fdev->dev, "unable to ioremap registers\n" ); |
1124 | err = -ENOMEM; |
1125 | goto out_free_chan; |
1126 | } |
1127 | |
1128 | err = of_address_to_resource(dev: node, index: 0, r: &res); |
1129 | if (err) { |
1130 | dev_err(fdev->dev, "unable to find 'reg' property\n" ); |
1131 | goto out_iounmap_regs; |
1132 | } |
1133 | |
1134 | chan->feature = feature; |
1135 | if (!fdev->feature) |
1136 | fdev->feature = chan->feature; |
1137 | |
1138 | /* |
1139 | * If the DMA device's feature is different than the feature |
1140 | * of its channels, report the bug |
1141 | */ |
1142 | WARN_ON(fdev->feature != chan->feature); |
1143 | |
1144 | chan->dev = fdev->dev; |
1145 | chan->id = (res.start & 0xfff) < 0x300 ? |
1146 | ((res.start - 0x100) & 0xfff) >> 7 : |
1147 | ((res.start - 0x200) & 0xfff) >> 7; |
1148 | if (chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) { |
1149 | dev_err(fdev->dev, "too many channels for device\n" ); |
1150 | err = -EINVAL; |
1151 | goto out_iounmap_regs; |
1152 | } |
1153 | |
1154 | fdev->chan[chan->id] = chan; |
1155 | tasklet_setup(t: &chan->tasklet, callback: dma_do_tasklet); |
1156 | snprintf(buf: chan->name, size: sizeof(chan->name), fmt: "chan%d" , chan->id); |
1157 | |
1158 | /* Initialize the channel */ |
1159 | dma_init(chan); |
1160 | |
1161 | /* Clear cdar registers */ |
1162 | set_cdar(chan, addr: 0); |
1163 | |
1164 | switch (chan->feature & FSL_DMA_IP_MASK) { |
1165 | case FSL_DMA_IP_85XX: |
1166 | chan->toggle_ext_pause = fsl_chan_toggle_ext_pause; |
1167 | fallthrough; |
1168 | case FSL_DMA_IP_83XX: |
1169 | chan->toggle_ext_start = fsl_chan_toggle_ext_start; |
1170 | chan->set_src_loop_size = fsl_chan_set_src_loop_size; |
1171 | chan->set_dst_loop_size = fsl_chan_set_dst_loop_size; |
1172 | chan->set_request_count = fsl_chan_set_request_count; |
1173 | } |
1174 | |
1175 | spin_lock_init(&chan->desc_lock); |
1176 | INIT_LIST_HEAD(list: &chan->ld_pending); |
1177 | INIT_LIST_HEAD(list: &chan->ld_running); |
1178 | INIT_LIST_HEAD(list: &chan->ld_completed); |
1179 | chan->idle = true; |
1180 | #ifdef CONFIG_PM |
1181 | chan->pm_state = RUNNING; |
1182 | #endif |
1183 | |
1184 | chan->common.device = &fdev->common; |
1185 | dma_cookie_init(chan: &chan->common); |
1186 | |
1187 | /* find the IRQ line, if it exists in the device tree */ |
1188 | chan->irq = irq_of_parse_and_map(node, index: 0); |
1189 | |
1190 | /* Add the channel to DMA device channel list */ |
1191 | list_add_tail(new: &chan->common.device_node, head: &fdev->common.channels); |
1192 | |
1193 | dev_info(fdev->dev, "#%d (%s), irq %d\n" , chan->id, compatible, |
1194 | chan->irq ? chan->irq : fdev->irq); |
1195 | |
1196 | return 0; |
1197 | |
1198 | out_iounmap_regs: |
1199 | iounmap(addr: chan->regs); |
1200 | out_free_chan: |
1201 | kfree(objp: chan); |
1202 | out_return: |
1203 | return err; |
1204 | } |
1205 | |
1206 | static void fsl_dma_chan_remove(struct fsldma_chan *chan) |
1207 | { |
1208 | irq_dispose_mapping(virq: chan->irq); |
1209 | list_del(entry: &chan->common.device_node); |
1210 | iounmap(addr: chan->regs); |
1211 | kfree(objp: chan); |
1212 | } |
1213 | |
1214 | static int fsldma_of_probe(struct platform_device *op) |
1215 | { |
1216 | struct fsldma_device *fdev; |
1217 | struct device_node *child; |
1218 | unsigned int i; |
1219 | int err; |
1220 | |
1221 | fdev = kzalloc(size: sizeof(*fdev), GFP_KERNEL); |
1222 | if (!fdev) { |
1223 | err = -ENOMEM; |
1224 | goto out_return; |
1225 | } |
1226 | |
1227 | fdev->dev = &op->dev; |
1228 | INIT_LIST_HEAD(list: &fdev->common.channels); |
1229 | |
1230 | /* ioremap the registers for use */ |
1231 | fdev->regs = of_iomap(node: op->dev.of_node, index: 0); |
1232 | if (!fdev->regs) { |
1233 | dev_err(&op->dev, "unable to ioremap registers\n" ); |
1234 | err = -ENOMEM; |
1235 | goto out_free; |
1236 | } |
1237 | |
1238 | /* map the channel IRQ if it exists, but don't hookup the handler yet */ |
1239 | fdev->irq = irq_of_parse_and_map(node: op->dev.of_node, index: 0); |
1240 | |
1241 | dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask); |
1242 | dma_cap_set(DMA_SLAVE, fdev->common.cap_mask); |
1243 | fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources; |
1244 | fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources; |
1245 | fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy; |
1246 | fdev->common.device_tx_status = fsl_tx_status; |
1247 | fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending; |
1248 | fdev->common.device_config = fsl_dma_device_config; |
1249 | fdev->common.device_terminate_all = fsl_dma_device_terminate_all; |
1250 | fdev->common.dev = &op->dev; |
1251 | |
1252 | fdev->common.src_addr_widths = FSL_DMA_BUSWIDTHS; |
1253 | fdev->common.dst_addr_widths = FSL_DMA_BUSWIDTHS; |
1254 | fdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); |
1255 | fdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; |
1256 | |
1257 | dma_set_mask(dev: &(op->dev), DMA_BIT_MASK(36)); |
1258 | |
1259 | platform_set_drvdata(pdev: op, data: fdev); |
1260 | |
1261 | /* |
1262 | * We cannot use of_platform_bus_probe() because there is no |
1263 | * of_platform_bus_remove(). Instead, we manually instantiate every DMA |
1264 | * channel object. |
1265 | */ |
1266 | for_each_child_of_node(op->dev.of_node, child) { |
1267 | if (of_device_is_compatible(device: child, "fsl,eloplus-dma-channel" )) { |
1268 | fsl_dma_chan_probe(fdev, node: child, |
1269 | FSL_DMA_IP_85XX | FSL_DMA_BIG_ENDIAN, |
1270 | compatible: "fsl,eloplus-dma-channel" ); |
1271 | } |
1272 | |
1273 | if (of_device_is_compatible(device: child, "fsl,elo-dma-channel" )) { |
1274 | fsl_dma_chan_probe(fdev, node: child, |
1275 | FSL_DMA_IP_83XX | FSL_DMA_LITTLE_ENDIAN, |
1276 | compatible: "fsl,elo-dma-channel" ); |
1277 | } |
1278 | } |
1279 | |
1280 | /* |
1281 | * Hookup the IRQ handler(s) |
1282 | * |
1283 | * If we have a per-controller interrupt, we prefer that to the |
1284 | * per-channel interrupts to reduce the number of shared interrupt |
1285 | * handlers on the same IRQ line |
1286 | */ |
1287 | err = fsldma_request_irqs(fdev); |
1288 | if (err) { |
1289 | dev_err(fdev->dev, "unable to request IRQs\n" ); |
1290 | goto out_free_fdev; |
1291 | } |
1292 | |
1293 | dma_async_device_register(device: &fdev->common); |
1294 | return 0; |
1295 | |
1296 | out_free_fdev: |
1297 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { |
1298 | if (fdev->chan[i]) |
1299 | fsl_dma_chan_remove(chan: fdev->chan[i]); |
1300 | } |
1301 | irq_dispose_mapping(virq: fdev->irq); |
1302 | iounmap(addr: fdev->regs); |
1303 | out_free: |
1304 | kfree(objp: fdev); |
1305 | out_return: |
1306 | return err; |
1307 | } |
1308 | |
1309 | static void fsldma_of_remove(struct platform_device *op) |
1310 | { |
1311 | struct fsldma_device *fdev; |
1312 | unsigned int i; |
1313 | |
1314 | fdev = platform_get_drvdata(pdev: op); |
1315 | dma_async_device_unregister(device: &fdev->common); |
1316 | |
1317 | fsldma_free_irqs(fdev); |
1318 | |
1319 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { |
1320 | if (fdev->chan[i]) |
1321 | fsl_dma_chan_remove(chan: fdev->chan[i]); |
1322 | } |
1323 | irq_dispose_mapping(virq: fdev->irq); |
1324 | |
1325 | iounmap(addr: fdev->regs); |
1326 | kfree(objp: fdev); |
1327 | } |
1328 | |
1329 | #ifdef CONFIG_PM |
1330 | static int fsldma_suspend_late(struct device *dev) |
1331 | { |
1332 | struct fsldma_device *fdev = dev_get_drvdata(dev); |
1333 | struct fsldma_chan *chan; |
1334 | int i; |
1335 | |
1336 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { |
1337 | chan = fdev->chan[i]; |
1338 | if (!chan) |
1339 | continue; |
1340 | |
1341 | spin_lock_bh(lock: &chan->desc_lock); |
1342 | if (unlikely(!chan->idle)) |
1343 | goto out; |
1344 | chan->regs_save.mr = get_mr(chan); |
1345 | chan->pm_state = SUSPENDED; |
1346 | spin_unlock_bh(lock: &chan->desc_lock); |
1347 | } |
1348 | return 0; |
1349 | |
1350 | out: |
1351 | for (; i >= 0; i--) { |
1352 | chan = fdev->chan[i]; |
1353 | if (!chan) |
1354 | continue; |
1355 | chan->pm_state = RUNNING; |
1356 | spin_unlock_bh(lock: &chan->desc_lock); |
1357 | } |
1358 | return -EBUSY; |
1359 | } |
1360 | |
1361 | static int fsldma_resume_early(struct device *dev) |
1362 | { |
1363 | struct fsldma_device *fdev = dev_get_drvdata(dev); |
1364 | struct fsldma_chan *chan; |
1365 | u32 mode; |
1366 | int i; |
1367 | |
1368 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { |
1369 | chan = fdev->chan[i]; |
1370 | if (!chan) |
1371 | continue; |
1372 | |
1373 | spin_lock_bh(lock: &chan->desc_lock); |
1374 | mode = chan->regs_save.mr |
1375 | & ~FSL_DMA_MR_CS & ~FSL_DMA_MR_CC & ~FSL_DMA_MR_CA; |
1376 | set_mr(chan, val: mode); |
1377 | chan->pm_state = RUNNING; |
1378 | spin_unlock_bh(lock: &chan->desc_lock); |
1379 | } |
1380 | |
1381 | return 0; |
1382 | } |
1383 | |
1384 | static const struct dev_pm_ops fsldma_pm_ops = { |
1385 | .suspend_late = fsldma_suspend_late, |
1386 | .resume_early = fsldma_resume_early, |
1387 | }; |
1388 | #endif |
1389 | |
1390 | static const struct of_device_id fsldma_of_ids[] = { |
1391 | { .compatible = "fsl,elo3-dma" , }, |
1392 | { .compatible = "fsl,eloplus-dma" , }, |
1393 | { .compatible = "fsl,elo-dma" , }, |
1394 | {} |
1395 | }; |
1396 | MODULE_DEVICE_TABLE(of, fsldma_of_ids); |
1397 | |
1398 | static struct platform_driver fsldma_of_driver = { |
1399 | .driver = { |
1400 | .name = "fsl-elo-dma" , |
1401 | .of_match_table = fsldma_of_ids, |
1402 | #ifdef CONFIG_PM |
1403 | .pm = &fsldma_pm_ops, |
1404 | #endif |
1405 | }, |
1406 | .probe = fsldma_of_probe, |
1407 | .remove_new = fsldma_of_remove, |
1408 | }; |
1409 | |
1410 | /*----------------------------------------------------------------------------*/ |
1411 | /* Module Init / Exit */ |
1412 | /*----------------------------------------------------------------------------*/ |
1413 | |
1414 | static __init int fsldma_init(void) |
1415 | { |
1416 | pr_info("Freescale Elo series DMA driver\n" ); |
1417 | return platform_driver_register(&fsldma_of_driver); |
1418 | } |
1419 | |
1420 | static void __exit fsldma_exit(void) |
1421 | { |
1422 | platform_driver_unregister(&fsldma_of_driver); |
1423 | } |
1424 | |
1425 | subsys_initcall(fsldma_init); |
1426 | module_exit(fsldma_exit); |
1427 | |
1428 | MODULE_DESCRIPTION("Freescale Elo series DMA driver" ); |
1429 | MODULE_LICENSE("GPL" ); |
1430 | |