1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. |
4 | * Synopsys DesignWare eDMA core driver |
5 | * |
6 | * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> |
7 | */ |
8 | |
9 | #include <linux/module.h> |
10 | #include <linux/device.h> |
11 | #include <linux/kernel.h> |
12 | #include <linux/dmaengine.h> |
13 | #include <linux/err.h> |
14 | #include <linux/interrupt.h> |
15 | #include <linux/irq.h> |
16 | #include <linux/dma/edma.h> |
17 | #include <linux/dma-mapping.h> |
18 | |
19 | #include "dw-edma-core.h" |
20 | #include "dw-edma-v0-core.h" |
21 | #include "dw-hdma-v0-core.h" |
22 | #include "../dmaengine.h" |
23 | #include "../virt-dma.h" |
24 | |
25 | static inline |
26 | struct device *dchan2dev(struct dma_chan *dchan) |
27 | { |
28 | return &dchan->dev->device; |
29 | } |
30 | |
31 | static inline |
32 | struct device *chan2dev(struct dw_edma_chan *chan) |
33 | { |
34 | return &chan->vc.chan.dev->device; |
35 | } |
36 | |
37 | static inline |
38 | struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) |
39 | { |
40 | return container_of(vd, struct dw_edma_desc, vd); |
41 | } |
42 | |
43 | static inline |
44 | u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr) |
45 | { |
46 | struct dw_edma_chip *chip = chan->dw->chip; |
47 | |
48 | if (chip->ops->pci_address) |
49 | return chip->ops->pci_address(chip->dev, cpu_addr); |
50 | |
51 | return cpu_addr; |
52 | } |
53 | |
54 | static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) |
55 | { |
56 | struct dw_edma_burst *burst; |
57 | |
58 | burst = kzalloc(size: sizeof(*burst), GFP_NOWAIT); |
59 | if (unlikely(!burst)) |
60 | return NULL; |
61 | |
62 | INIT_LIST_HEAD(list: &burst->list); |
63 | if (chunk->burst) { |
64 | /* Create and add new element into the linked list */ |
65 | chunk->bursts_alloc++; |
66 | list_add_tail(new: &burst->list, head: &chunk->burst->list); |
67 | } else { |
68 | /* List head */ |
69 | chunk->bursts_alloc = 0; |
70 | chunk->burst = burst; |
71 | } |
72 | |
73 | return burst; |
74 | } |
75 | |
76 | static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) |
77 | { |
78 | struct dw_edma_chip *chip = desc->chan->dw->chip; |
79 | struct dw_edma_chan *chan = desc->chan; |
80 | struct dw_edma_chunk *chunk; |
81 | |
82 | chunk = kzalloc(size: sizeof(*chunk), GFP_NOWAIT); |
83 | if (unlikely(!chunk)) |
84 | return NULL; |
85 | |
86 | INIT_LIST_HEAD(list: &chunk->list); |
87 | chunk->chan = chan; |
88 | /* Toggling change bit (CB) in each chunk, this is a mechanism to |
89 | * inform the eDMA HW block that this is a new linked list ready |
90 | * to be consumed. |
91 | * - Odd chunks originate CB equal to 0 |
92 | * - Even chunks originate CB equal to 1 |
93 | */ |
94 | chunk->cb = !(desc->chunks_alloc % 2); |
95 | if (chan->dir == EDMA_DIR_WRITE) { |
96 | chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr; |
97 | chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr; |
98 | } else { |
99 | chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr; |
100 | chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr; |
101 | } |
102 | |
103 | if (desc->chunk) { |
104 | /* Create and add new element into the linked list */ |
105 | if (!dw_edma_alloc_burst(chunk)) { |
106 | kfree(objp: chunk); |
107 | return NULL; |
108 | } |
109 | desc->chunks_alloc++; |
110 | list_add_tail(new: &chunk->list, head: &desc->chunk->list); |
111 | } else { |
112 | /* List head */ |
113 | chunk->burst = NULL; |
114 | desc->chunks_alloc = 0; |
115 | desc->chunk = chunk; |
116 | } |
117 | |
118 | return chunk; |
119 | } |
120 | |
121 | static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) |
122 | { |
123 | struct dw_edma_desc *desc; |
124 | |
125 | desc = kzalloc(size: sizeof(*desc), GFP_NOWAIT); |
126 | if (unlikely(!desc)) |
127 | return NULL; |
128 | |
129 | desc->chan = chan; |
130 | if (!dw_edma_alloc_chunk(desc)) { |
131 | kfree(objp: desc); |
132 | return NULL; |
133 | } |
134 | |
135 | return desc; |
136 | } |
137 | |
138 | static void dw_edma_free_burst(struct dw_edma_chunk *chunk) |
139 | { |
140 | struct dw_edma_burst *child, *_next; |
141 | |
142 | /* Remove all the list elements */ |
143 | list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { |
144 | list_del(entry: &child->list); |
145 | kfree(objp: child); |
146 | chunk->bursts_alloc--; |
147 | } |
148 | |
149 | /* Remove the list head */ |
150 | kfree(objp: child); |
151 | chunk->burst = NULL; |
152 | } |
153 | |
154 | static void dw_edma_free_chunk(struct dw_edma_desc *desc) |
155 | { |
156 | struct dw_edma_chunk *child, *_next; |
157 | |
158 | if (!desc->chunk) |
159 | return; |
160 | |
161 | /* Remove all the list elements */ |
162 | list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { |
163 | dw_edma_free_burst(chunk: child); |
164 | list_del(entry: &child->list); |
165 | kfree(objp: child); |
166 | desc->chunks_alloc--; |
167 | } |
168 | |
169 | /* Remove the list head */ |
170 | kfree(objp: child); |
171 | desc->chunk = NULL; |
172 | } |
173 | |
174 | static void dw_edma_free_desc(struct dw_edma_desc *desc) |
175 | { |
176 | dw_edma_free_chunk(desc); |
177 | kfree(objp: desc); |
178 | } |
179 | |
180 | static void vchan_free_desc(struct virt_dma_desc *vdesc) |
181 | { |
182 | dw_edma_free_desc(desc: vd2dw_edma_desc(vd: vdesc)); |
183 | } |
184 | |
185 | static int dw_edma_start_transfer(struct dw_edma_chan *chan) |
186 | { |
187 | struct dw_edma *dw = chan->dw; |
188 | struct dw_edma_chunk *child; |
189 | struct dw_edma_desc *desc; |
190 | struct virt_dma_desc *vd; |
191 | |
192 | vd = vchan_next_desc(vc: &chan->vc); |
193 | if (!vd) |
194 | return 0; |
195 | |
196 | desc = vd2dw_edma_desc(vd); |
197 | if (!desc) |
198 | return 0; |
199 | |
200 | child = list_first_entry_or_null(&desc->chunk->list, |
201 | struct dw_edma_chunk, list); |
202 | if (!child) |
203 | return 0; |
204 | |
205 | dw_edma_core_start(dw, chunk: child, first: !desc->xfer_sz); |
206 | desc->xfer_sz += child->ll_region.sz; |
207 | dw_edma_free_burst(chunk: child); |
208 | list_del(entry: &child->list); |
209 | kfree(objp: child); |
210 | desc->chunks_alloc--; |
211 | |
212 | return 1; |
213 | } |
214 | |
215 | static void dw_edma_device_caps(struct dma_chan *dchan, |
216 | struct dma_slave_caps *caps) |
217 | { |
218 | struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); |
219 | |
220 | if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { |
221 | if (chan->dir == EDMA_DIR_READ) |
222 | caps->directions = BIT(DMA_DEV_TO_MEM); |
223 | else |
224 | caps->directions = BIT(DMA_MEM_TO_DEV); |
225 | } else { |
226 | if (chan->dir == EDMA_DIR_WRITE) |
227 | caps->directions = BIT(DMA_DEV_TO_MEM); |
228 | else |
229 | caps->directions = BIT(DMA_MEM_TO_DEV); |
230 | } |
231 | } |
232 | |
233 | static int dw_edma_device_config(struct dma_chan *dchan, |
234 | struct dma_slave_config *config) |
235 | { |
236 | struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); |
237 | |
238 | memcpy(&chan->config, config, sizeof(*config)); |
239 | chan->configured = true; |
240 | |
241 | return 0; |
242 | } |
243 | |
244 | static int dw_edma_device_pause(struct dma_chan *dchan) |
245 | { |
246 | struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); |
247 | int err = 0; |
248 | |
249 | if (!chan->configured) |
250 | err = -EPERM; |
251 | else if (chan->status != EDMA_ST_BUSY) |
252 | err = -EPERM; |
253 | else if (chan->request != EDMA_REQ_NONE) |
254 | err = -EPERM; |
255 | else |
256 | chan->request = EDMA_REQ_PAUSE; |
257 | |
258 | return err; |
259 | } |
260 | |
261 | static int dw_edma_device_resume(struct dma_chan *dchan) |
262 | { |
263 | struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); |
264 | int err = 0; |
265 | |
266 | if (!chan->configured) { |
267 | err = -EPERM; |
268 | } else if (chan->status != EDMA_ST_PAUSE) { |
269 | err = -EPERM; |
270 | } else if (chan->request != EDMA_REQ_NONE) { |
271 | err = -EPERM; |
272 | } else { |
273 | chan->status = EDMA_ST_BUSY; |
274 | dw_edma_start_transfer(chan); |
275 | } |
276 | |
277 | return err; |
278 | } |
279 | |
280 | static int dw_edma_device_terminate_all(struct dma_chan *dchan) |
281 | { |
282 | struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); |
283 | int err = 0; |
284 | |
285 | if (!chan->configured) { |
286 | /* Do nothing */ |
287 | } else if (chan->status == EDMA_ST_PAUSE) { |
288 | chan->status = EDMA_ST_IDLE; |
289 | chan->configured = false; |
290 | } else if (chan->status == EDMA_ST_IDLE) { |
291 | chan->configured = false; |
292 | } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) { |
293 | /* |
294 | * The channel is in a false BUSY state, probably didn't |
295 | * receive or lost an interrupt |
296 | */ |
297 | chan->status = EDMA_ST_IDLE; |
298 | chan->configured = false; |
299 | } else if (chan->request > EDMA_REQ_PAUSE) { |
300 | err = -EPERM; |
301 | } else { |
302 | chan->request = EDMA_REQ_STOP; |
303 | } |
304 | |
305 | return err; |
306 | } |
307 | |
308 | static void dw_edma_device_issue_pending(struct dma_chan *dchan) |
309 | { |
310 | struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); |
311 | unsigned long flags; |
312 | |
313 | if (!chan->configured) |
314 | return; |
315 | |
316 | spin_lock_irqsave(&chan->vc.lock, flags); |
317 | if (vchan_issue_pending(vc: &chan->vc) && chan->request == EDMA_REQ_NONE && |
318 | chan->status == EDMA_ST_IDLE) { |
319 | chan->status = EDMA_ST_BUSY; |
320 | dw_edma_start_transfer(chan); |
321 | } |
322 | spin_unlock_irqrestore(lock: &chan->vc.lock, flags); |
323 | } |
324 | |
325 | static enum dma_status |
326 | dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, |
327 | struct dma_tx_state *txstate) |
328 | { |
329 | struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); |
330 | struct dw_edma_desc *desc; |
331 | struct virt_dma_desc *vd; |
332 | unsigned long flags; |
333 | enum dma_status ret; |
334 | u32 residue = 0; |
335 | |
336 | ret = dma_cookie_status(chan: dchan, cookie, state: txstate); |
337 | if (ret == DMA_COMPLETE) |
338 | return ret; |
339 | |
340 | if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE) |
341 | ret = DMA_PAUSED; |
342 | |
343 | if (!txstate) |
344 | goto ret_residue; |
345 | |
346 | spin_lock_irqsave(&chan->vc.lock, flags); |
347 | vd = vchan_find_desc(&chan->vc, cookie); |
348 | if (vd) { |
349 | desc = vd2dw_edma_desc(vd); |
350 | if (desc) |
351 | residue = desc->alloc_sz - desc->xfer_sz; |
352 | } |
353 | spin_unlock_irqrestore(lock: &chan->vc.lock, flags); |
354 | |
355 | ret_residue: |
356 | dma_set_residue(state: txstate, residue); |
357 | |
358 | return ret; |
359 | } |
360 | |
361 | static struct dma_async_tx_descriptor * |
362 | dw_edma_device_transfer(struct dw_edma_transfer *xfer) |
363 | { |
364 | struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan: xfer->dchan); |
365 | enum dma_transfer_direction dir = xfer->direction; |
366 | struct scatterlist *sg = NULL; |
367 | struct dw_edma_chunk *chunk; |
368 | struct dw_edma_burst *burst; |
369 | struct dw_edma_desc *desc; |
370 | u64 src_addr, dst_addr; |
371 | size_t fsz = 0; |
372 | u32 cnt = 0; |
373 | int i; |
374 | |
375 | if (!chan->configured) |
376 | return NULL; |
377 | |
378 | /* |
379 | * Local Root Port/End-point Remote End-point |
380 | * +-----------------------+ PCIe bus +----------------------+ |
381 | * | | +-+ | | |
382 | * | DEV_TO_MEM Rx Ch <----+ +---+ Tx Ch DEV_TO_MEM | |
383 | * | | | | | | |
384 | * | MEM_TO_DEV Tx Ch +----+ +---> Rx Ch MEM_TO_DEV | |
385 | * | | +-+ | | |
386 | * +-----------------------+ +----------------------+ |
387 | * |
388 | * 1. Normal logic: |
389 | * If eDMA is embedded into the DW PCIe RP/EP and controlled from the |
390 | * CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used |
391 | * for the device read operations (DEV_TO_MEM) and the Tx channel |
392 | * (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV). |
393 | * |
394 | * 2. Inverted logic: |
395 | * If eDMA is embedded into a Remote PCIe EP and is controlled by the |
396 | * MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx |
397 | * channel (EDMA_DIR_WRITE) will be used for the device read operations |
398 | * (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write |
399 | * operations (MEM_TO_DEV). |
400 | * |
401 | * It is the client driver responsibility to choose a proper channel |
402 | * for the DMA transfers. |
403 | */ |
404 | if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { |
405 | if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) || |
406 | (chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV)) |
407 | return NULL; |
408 | } else { |
409 | if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) || |
410 | (chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV)) |
411 | return NULL; |
412 | } |
413 | |
414 | if (xfer->type == EDMA_XFER_CYCLIC) { |
415 | if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt) |
416 | return NULL; |
417 | } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { |
418 | if (xfer->xfer.sg.len < 1) |
419 | return NULL; |
420 | } else if (xfer->type == EDMA_XFER_INTERLEAVED) { |
421 | if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1) |
422 | return NULL; |
423 | if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc) |
424 | return NULL; |
425 | } else { |
426 | return NULL; |
427 | } |
428 | |
429 | desc = dw_edma_alloc_desc(chan); |
430 | if (unlikely(!desc)) |
431 | goto err_alloc; |
432 | |
433 | chunk = dw_edma_alloc_chunk(desc); |
434 | if (unlikely(!chunk)) |
435 | goto err_alloc; |
436 | |
437 | if (xfer->type == EDMA_XFER_INTERLEAVED) { |
438 | src_addr = xfer->xfer.il->src_start; |
439 | dst_addr = xfer->xfer.il->dst_start; |
440 | } else { |
441 | src_addr = chan->config.src_addr; |
442 | dst_addr = chan->config.dst_addr; |
443 | } |
444 | |
445 | if (dir == DMA_DEV_TO_MEM) |
446 | src_addr = dw_edma_get_pci_address(chan, cpu_addr: (phys_addr_t)src_addr); |
447 | else |
448 | dst_addr = dw_edma_get_pci_address(chan, cpu_addr: (phys_addr_t)dst_addr); |
449 | |
450 | if (xfer->type == EDMA_XFER_CYCLIC) { |
451 | cnt = xfer->xfer.cyclic.cnt; |
452 | } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { |
453 | cnt = xfer->xfer.sg.len; |
454 | sg = xfer->xfer.sg.sgl; |
455 | } else if (xfer->type == EDMA_XFER_INTERLEAVED) { |
456 | cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size; |
457 | fsz = xfer->xfer.il->frame_size; |
458 | } |
459 | |
460 | for (i = 0; i < cnt; i++) { |
461 | if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg) |
462 | break; |
463 | |
464 | if (chunk->bursts_alloc == chan->ll_max) { |
465 | chunk = dw_edma_alloc_chunk(desc); |
466 | if (unlikely(!chunk)) |
467 | goto err_alloc; |
468 | } |
469 | |
470 | burst = dw_edma_alloc_burst(chunk); |
471 | if (unlikely(!burst)) |
472 | goto err_alloc; |
473 | |
474 | if (xfer->type == EDMA_XFER_CYCLIC) |
475 | burst->sz = xfer->xfer.cyclic.len; |
476 | else if (xfer->type == EDMA_XFER_SCATTER_GATHER) |
477 | burst->sz = sg_dma_len(sg); |
478 | else if (xfer->type == EDMA_XFER_INTERLEAVED) |
479 | burst->sz = xfer->xfer.il->sgl[i % fsz].size; |
480 | |
481 | chunk->ll_region.sz += burst->sz; |
482 | desc->alloc_sz += burst->sz; |
483 | |
484 | if (dir == DMA_DEV_TO_MEM) { |
485 | burst->sar = src_addr; |
486 | if (xfer->type == EDMA_XFER_CYCLIC) { |
487 | burst->dar = xfer->xfer.cyclic.paddr; |
488 | } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { |
489 | src_addr += sg_dma_len(sg); |
490 | burst->dar = sg_dma_address(sg); |
491 | /* Unlike the typical assumption by other |
492 | * drivers/IPs the peripheral memory isn't |
493 | * a FIFO memory, in this case, it's a |
494 | * linear memory and that why the source |
495 | * and destination addresses are increased |
496 | * by the same portion (data length) |
497 | */ |
498 | } else if (xfer->type == EDMA_XFER_INTERLEAVED) { |
499 | burst->dar = dst_addr; |
500 | } |
501 | } else { |
502 | burst->dar = dst_addr; |
503 | if (xfer->type == EDMA_XFER_CYCLIC) { |
504 | burst->sar = xfer->xfer.cyclic.paddr; |
505 | } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { |
506 | dst_addr += sg_dma_len(sg); |
507 | burst->sar = sg_dma_address(sg); |
508 | /* Unlike the typical assumption by other |
509 | * drivers/IPs the peripheral memory isn't |
510 | * a FIFO memory, in this case, it's a |
511 | * linear memory and that why the source |
512 | * and destination addresses are increased |
513 | * by the same portion (data length) |
514 | */ |
515 | } else if (xfer->type == EDMA_XFER_INTERLEAVED) { |
516 | burst->sar = src_addr; |
517 | } |
518 | } |
519 | |
520 | if (xfer->type == EDMA_XFER_SCATTER_GATHER) { |
521 | sg = sg_next(sg); |
522 | } else if (xfer->type == EDMA_XFER_INTERLEAVED) { |
523 | struct dma_interleaved_template *il = xfer->xfer.il; |
524 | struct data_chunk *dc = &il->sgl[i % fsz]; |
525 | |
526 | src_addr += burst->sz; |
527 | if (il->src_sgl) |
528 | src_addr += dmaengine_get_src_icg(xt: il, chunk: dc); |
529 | |
530 | dst_addr += burst->sz; |
531 | if (il->dst_sgl) |
532 | dst_addr += dmaengine_get_dst_icg(xt: il, chunk: dc); |
533 | } |
534 | } |
535 | |
536 | return vchan_tx_prep(vc: &chan->vc, vd: &desc->vd, tx_flags: xfer->flags); |
537 | |
538 | err_alloc: |
539 | if (desc) |
540 | dw_edma_free_desc(desc); |
541 | |
542 | return NULL; |
543 | } |
544 | |
545 | static struct dma_async_tx_descriptor * |
546 | dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, |
547 | unsigned int len, |
548 | enum dma_transfer_direction direction, |
549 | unsigned long flags, void *context) |
550 | { |
551 | struct dw_edma_transfer xfer; |
552 | |
553 | xfer.dchan = dchan; |
554 | xfer.direction = direction; |
555 | xfer.xfer.sg.sgl = sgl; |
556 | xfer.xfer.sg.len = len; |
557 | xfer.flags = flags; |
558 | xfer.type = EDMA_XFER_SCATTER_GATHER; |
559 | |
560 | return dw_edma_device_transfer(xfer: &xfer); |
561 | } |
562 | |
563 | static struct dma_async_tx_descriptor * |
564 | dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, |
565 | size_t len, size_t count, |
566 | enum dma_transfer_direction direction, |
567 | unsigned long flags) |
568 | { |
569 | struct dw_edma_transfer xfer; |
570 | |
571 | xfer.dchan = dchan; |
572 | xfer.direction = direction; |
573 | xfer.xfer.cyclic.paddr = paddr; |
574 | xfer.xfer.cyclic.len = len; |
575 | xfer.xfer.cyclic.cnt = count; |
576 | xfer.flags = flags; |
577 | xfer.type = EDMA_XFER_CYCLIC; |
578 | |
579 | return dw_edma_device_transfer(xfer: &xfer); |
580 | } |
581 | |
582 | static struct dma_async_tx_descriptor * |
583 | dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, |
584 | struct dma_interleaved_template *ilt, |
585 | unsigned long flags) |
586 | { |
587 | struct dw_edma_transfer xfer; |
588 | |
589 | xfer.dchan = dchan; |
590 | xfer.direction = ilt->dir; |
591 | xfer.xfer.il = ilt; |
592 | xfer.flags = flags; |
593 | xfer.type = EDMA_XFER_INTERLEAVED; |
594 | |
595 | return dw_edma_device_transfer(xfer: &xfer); |
596 | } |
597 | |
598 | static void dw_edma_done_interrupt(struct dw_edma_chan *chan) |
599 | { |
600 | struct dw_edma_desc *desc; |
601 | struct virt_dma_desc *vd; |
602 | unsigned long flags; |
603 | |
604 | spin_lock_irqsave(&chan->vc.lock, flags); |
605 | vd = vchan_next_desc(vc: &chan->vc); |
606 | if (vd) { |
607 | switch (chan->request) { |
608 | case EDMA_REQ_NONE: |
609 | desc = vd2dw_edma_desc(vd); |
610 | if (!desc->chunks_alloc) { |
611 | list_del(entry: &vd->node); |
612 | vchan_cookie_complete(vd); |
613 | } |
614 | |
615 | /* Continue transferring if there are remaining chunks or issued requests. |
616 | */ |
617 | chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; |
618 | break; |
619 | |
620 | case EDMA_REQ_STOP: |
621 | list_del(entry: &vd->node); |
622 | vchan_cookie_complete(vd); |
623 | chan->request = EDMA_REQ_NONE; |
624 | chan->status = EDMA_ST_IDLE; |
625 | break; |
626 | |
627 | case EDMA_REQ_PAUSE: |
628 | chan->request = EDMA_REQ_NONE; |
629 | chan->status = EDMA_ST_PAUSE; |
630 | break; |
631 | |
632 | default: |
633 | break; |
634 | } |
635 | } |
636 | spin_unlock_irqrestore(lock: &chan->vc.lock, flags); |
637 | } |
638 | |
639 | static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) |
640 | { |
641 | struct virt_dma_desc *vd; |
642 | unsigned long flags; |
643 | |
644 | spin_lock_irqsave(&chan->vc.lock, flags); |
645 | vd = vchan_next_desc(vc: &chan->vc); |
646 | if (vd) { |
647 | list_del(entry: &vd->node); |
648 | vchan_cookie_complete(vd); |
649 | } |
650 | spin_unlock_irqrestore(lock: &chan->vc.lock, flags); |
651 | chan->request = EDMA_REQ_NONE; |
652 | chan->status = EDMA_ST_IDLE; |
653 | } |
654 | |
655 | static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) |
656 | { |
657 | struct dw_edma_irq *dw_irq = data; |
658 | |
659 | return dw_edma_core_handle_int(dw_irq, dir: EDMA_DIR_WRITE, |
660 | done: dw_edma_done_interrupt, |
661 | abort: dw_edma_abort_interrupt); |
662 | } |
663 | |
664 | static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) |
665 | { |
666 | struct dw_edma_irq *dw_irq = data; |
667 | |
668 | return dw_edma_core_handle_int(dw_irq, dir: EDMA_DIR_READ, |
669 | done: dw_edma_done_interrupt, |
670 | abort: dw_edma_abort_interrupt); |
671 | } |
672 | |
673 | static irqreturn_t dw_edma_interrupt_common(int irq, void *data) |
674 | { |
675 | irqreturn_t ret = IRQ_NONE; |
676 | |
677 | ret |= dw_edma_interrupt_write(irq, data); |
678 | ret |= dw_edma_interrupt_read(irq, data); |
679 | |
680 | return ret; |
681 | } |
682 | |
683 | static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) |
684 | { |
685 | struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); |
686 | |
687 | if (chan->status != EDMA_ST_IDLE) |
688 | return -EBUSY; |
689 | |
690 | return 0; |
691 | } |
692 | |
693 | static void dw_edma_free_chan_resources(struct dma_chan *dchan) |
694 | { |
695 | unsigned long timeout = jiffies + msecs_to_jiffies(m: 5000); |
696 | int ret; |
697 | |
698 | while (time_before(jiffies, timeout)) { |
699 | ret = dw_edma_device_terminate_all(dchan); |
700 | if (!ret) |
701 | break; |
702 | |
703 | if (time_after_eq(jiffies, timeout)) |
704 | return; |
705 | |
706 | cpu_relax(); |
707 | } |
708 | } |
709 | |
710 | static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) |
711 | { |
712 | struct dw_edma_chip *chip = dw->chip; |
713 | struct device *dev = chip->dev; |
714 | struct dw_edma_chan *chan; |
715 | struct dw_edma_irq *irq; |
716 | struct dma_device *dma; |
717 | u32 i, ch_cnt; |
718 | u32 pos; |
719 | |
720 | ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; |
721 | dma = &dw->dma; |
722 | |
723 | INIT_LIST_HEAD(list: &dma->channels); |
724 | |
725 | for (i = 0; i < ch_cnt; i++) { |
726 | chan = &dw->chan[i]; |
727 | |
728 | chan->dw = dw; |
729 | |
730 | if (i < dw->wr_ch_cnt) { |
731 | chan->id = i; |
732 | chan->dir = EDMA_DIR_WRITE; |
733 | } else { |
734 | chan->id = i - dw->wr_ch_cnt; |
735 | chan->dir = EDMA_DIR_READ; |
736 | } |
737 | |
738 | chan->configured = false; |
739 | chan->request = EDMA_REQ_NONE; |
740 | chan->status = EDMA_ST_IDLE; |
741 | |
742 | if (chan->dir == EDMA_DIR_WRITE) |
743 | chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ); |
744 | else |
745 | chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ); |
746 | chan->ll_max -= 1; |
747 | |
748 | dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n" , |
749 | chan->dir == EDMA_DIR_WRITE ? "write" : "read" , |
750 | chan->id, chan->ll_max); |
751 | |
752 | if (dw->nr_irqs == 1) |
753 | pos = 0; |
754 | else if (chan->dir == EDMA_DIR_WRITE) |
755 | pos = chan->id % wr_alloc; |
756 | else |
757 | pos = wr_alloc + chan->id % rd_alloc; |
758 | |
759 | irq = &dw->irq[pos]; |
760 | |
761 | if (chan->dir == EDMA_DIR_WRITE) |
762 | irq->wr_mask |= BIT(chan->id); |
763 | else |
764 | irq->rd_mask |= BIT(chan->id); |
765 | |
766 | irq->dw = dw; |
767 | memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); |
768 | |
769 | dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n" , |
770 | chan->dir == EDMA_DIR_WRITE ? "write" : "read" , chan->id, |
771 | chan->msi.address_hi, chan->msi.address_lo, |
772 | chan->msi.data); |
773 | |
774 | chan->vc.desc_free = vchan_free_desc; |
775 | chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ? |
776 | &dw->chip->dt_region_wr[chan->id] : |
777 | &dw->chip->dt_region_rd[chan->id]; |
778 | |
779 | vchan_init(vc: &chan->vc, dmadev: dma); |
780 | |
781 | dw_edma_core_ch_config(chan); |
782 | } |
783 | |
784 | /* Set DMA channel capabilities */ |
785 | dma_cap_zero(dma->cap_mask); |
786 | dma_cap_set(DMA_SLAVE, dma->cap_mask); |
787 | dma_cap_set(DMA_CYCLIC, dma->cap_mask); |
788 | dma_cap_set(DMA_PRIVATE, dma->cap_mask); |
789 | dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); |
790 | dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); |
791 | dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
792 | dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
793 | dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; |
794 | |
795 | /* Set DMA channel callbacks */ |
796 | dma->dev = chip->dev; |
797 | dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources; |
798 | dma->device_free_chan_resources = dw_edma_free_chan_resources; |
799 | dma->device_caps = dw_edma_device_caps; |
800 | dma->device_config = dw_edma_device_config; |
801 | dma->device_pause = dw_edma_device_pause; |
802 | dma->device_resume = dw_edma_device_resume; |
803 | dma->device_terminate_all = dw_edma_device_terminate_all; |
804 | dma->device_issue_pending = dw_edma_device_issue_pending; |
805 | dma->device_tx_status = dw_edma_device_tx_status; |
806 | dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; |
807 | dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; |
808 | dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma; |
809 | |
810 | dma_set_max_seg_size(dev: dma->dev, U32_MAX); |
811 | |
812 | /* Register DMA device */ |
813 | return dma_async_device_register(device: dma); |
814 | } |
815 | |
816 | static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) |
817 | { |
818 | if (*nr_irqs && *alloc < cnt) { |
819 | (*alloc)++; |
820 | (*nr_irqs)--; |
821 | } |
822 | } |
823 | |
824 | static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) |
825 | { |
826 | while (*mask * alloc < cnt) |
827 | (*mask)++; |
828 | } |
829 | |
830 | static int dw_edma_irq_request(struct dw_edma *dw, |
831 | u32 *wr_alloc, u32 *rd_alloc) |
832 | { |
833 | struct dw_edma_chip *chip = dw->chip; |
834 | struct device *dev = dw->chip->dev; |
835 | u32 wr_mask = 1; |
836 | u32 rd_mask = 1; |
837 | int i, err = 0; |
838 | u32 ch_cnt; |
839 | int irq; |
840 | |
841 | ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; |
842 | |
843 | if (chip->nr_irqs < 1 || !chip->ops->irq_vector) |
844 | return -EINVAL; |
845 | |
846 | dw->irq = devm_kcalloc(dev, n: chip->nr_irqs, size: sizeof(*dw->irq), GFP_KERNEL); |
847 | if (!dw->irq) |
848 | return -ENOMEM; |
849 | |
850 | if (chip->nr_irqs == 1) { |
851 | /* Common IRQ shared among all channels */ |
852 | irq = chip->ops->irq_vector(dev, 0); |
853 | err = request_irq(irq, handler: dw_edma_interrupt_common, |
854 | IRQF_SHARED, name: dw->name, dev: &dw->irq[0]); |
855 | if (err) { |
856 | dw->nr_irqs = 0; |
857 | return err; |
858 | } |
859 | |
860 | if (irq_get_msi_desc(irq)) |
861 | get_cached_msi_msg(irq, msg: &dw->irq[0].msi); |
862 | |
863 | dw->nr_irqs = 1; |
864 | } else { |
865 | /* Distribute IRQs equally among all channels */ |
866 | int tmp = chip->nr_irqs; |
867 | |
868 | while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { |
869 | dw_edma_dec_irq_alloc(nr_irqs: &tmp, alloc: wr_alloc, cnt: dw->wr_ch_cnt); |
870 | dw_edma_dec_irq_alloc(nr_irqs: &tmp, alloc: rd_alloc, cnt: dw->rd_ch_cnt); |
871 | } |
872 | |
873 | dw_edma_add_irq_mask(mask: &wr_mask, alloc: *wr_alloc, cnt: dw->wr_ch_cnt); |
874 | dw_edma_add_irq_mask(mask: &rd_mask, alloc: *rd_alloc, cnt: dw->rd_ch_cnt); |
875 | |
876 | for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { |
877 | irq = chip->ops->irq_vector(dev, i); |
878 | err = request_irq(irq, |
879 | handler: i < *wr_alloc ? |
880 | dw_edma_interrupt_write : |
881 | dw_edma_interrupt_read, |
882 | IRQF_SHARED, name: dw->name, |
883 | dev: &dw->irq[i]); |
884 | if (err) |
885 | goto err_irq_free; |
886 | |
887 | if (irq_get_msi_desc(irq)) |
888 | get_cached_msi_msg(irq, msg: &dw->irq[i].msi); |
889 | } |
890 | |
891 | dw->nr_irqs = i; |
892 | } |
893 | |
894 | return 0; |
895 | |
896 | err_irq_free: |
897 | for (i--; i >= 0; i--) { |
898 | irq = chip->ops->irq_vector(dev, i); |
899 | free_irq(irq, &dw->irq[i]); |
900 | } |
901 | |
902 | return err; |
903 | } |
904 | |
905 | int dw_edma_probe(struct dw_edma_chip *chip) |
906 | { |
907 | struct device *dev; |
908 | struct dw_edma *dw; |
909 | u32 wr_alloc = 0; |
910 | u32 rd_alloc = 0; |
911 | int i, err; |
912 | |
913 | if (!chip) |
914 | return -EINVAL; |
915 | |
916 | dev = chip->dev; |
917 | if (!dev || !chip->ops) |
918 | return -EINVAL; |
919 | |
920 | dw = devm_kzalloc(dev, size: sizeof(*dw), GFP_KERNEL); |
921 | if (!dw) |
922 | return -ENOMEM; |
923 | |
924 | dw->chip = chip; |
925 | |
926 | if (dw->chip->mf == EDMA_MF_HDMA_NATIVE) |
927 | dw_hdma_v0_core_register(dw); |
928 | else |
929 | dw_edma_v0_core_register(dw); |
930 | |
931 | raw_spin_lock_init(&dw->lock); |
932 | |
933 | dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt, |
934 | dw_edma_core_ch_count(dw, EDMA_DIR_WRITE)); |
935 | dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); |
936 | |
937 | dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt, |
938 | dw_edma_core_ch_count(dw, EDMA_DIR_READ)); |
939 | dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); |
940 | |
941 | if (!dw->wr_ch_cnt && !dw->rd_ch_cnt) |
942 | return -EINVAL; |
943 | |
944 | dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n" , |
945 | dw->wr_ch_cnt, dw->rd_ch_cnt); |
946 | |
947 | /* Allocate channels */ |
948 | dw->chan = devm_kcalloc(dev, n: dw->wr_ch_cnt + dw->rd_ch_cnt, |
949 | size: sizeof(*dw->chan), GFP_KERNEL); |
950 | if (!dw->chan) |
951 | return -ENOMEM; |
952 | |
953 | snprintf(buf: dw->name, size: sizeof(dw->name), fmt: "dw-edma-core:%s" , |
954 | dev_name(dev: chip->dev)); |
955 | |
956 | /* Disable eDMA, only to establish the ideal initial conditions */ |
957 | dw_edma_core_off(dw); |
958 | |
959 | /* Request IRQs */ |
960 | err = dw_edma_irq_request(dw, wr_alloc: &wr_alloc, rd_alloc: &rd_alloc); |
961 | if (err) |
962 | return err; |
963 | |
964 | /* Setup write/read channels */ |
965 | err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc); |
966 | if (err) |
967 | goto err_irq_free; |
968 | |
969 | /* Turn debugfs on */ |
970 | dw_edma_core_debugfs_on(dw); |
971 | |
972 | chip->dw = dw; |
973 | |
974 | return 0; |
975 | |
976 | err_irq_free: |
977 | for (i = (dw->nr_irqs - 1); i >= 0; i--) |
978 | free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); |
979 | |
980 | return err; |
981 | } |
982 | EXPORT_SYMBOL_GPL(dw_edma_probe); |
983 | |
984 | int dw_edma_remove(struct dw_edma_chip *chip) |
985 | { |
986 | struct dw_edma_chan *chan, *_chan; |
987 | struct device *dev = chip->dev; |
988 | struct dw_edma *dw = chip->dw; |
989 | int i; |
990 | |
991 | /* Skip removal if no private data found */ |
992 | if (!dw) |
993 | return -ENODEV; |
994 | |
995 | /* Disable eDMA */ |
996 | dw_edma_core_off(dw); |
997 | |
998 | /* Free irqs */ |
999 | for (i = (dw->nr_irqs - 1); i >= 0; i--) |
1000 | free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); |
1001 | |
1002 | /* Deregister eDMA device */ |
1003 | dma_async_device_unregister(device: &dw->dma); |
1004 | list_for_each_entry_safe(chan, _chan, &dw->dma.channels, |
1005 | vc.chan.device_node) { |
1006 | tasklet_kill(t: &chan->vc.task); |
1007 | list_del(entry: &chan->vc.chan.device_node); |
1008 | } |
1009 | |
1010 | return 0; |
1011 | } |
1012 | EXPORT_SYMBOL_GPL(dw_edma_remove); |
1013 | |
1014 | MODULE_LICENSE("GPL v2" ); |
1015 | MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver" ); |
1016 | MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>" ); |
1017 | |