1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | /* |
3 | * BCM2835 DMA engine support |
4 | * |
5 | * Author: Florian Meier <florian.meier@koalo.de> |
6 | * Copyright 2013 |
7 | * |
8 | * Based on |
9 | * OMAP DMAengine support by Russell King |
10 | * |
11 | * BCM2708 DMA Driver |
12 | * Copyright (C) 2010 Broadcom |
13 | * |
14 | * Raspberry Pi PCM I2S ALSA Driver |
15 | * Copyright (c) by Phil Poole 2013 |
16 | * |
17 | * MARVELL MMP Peripheral DMA Driver |
18 | * Copyright 2012 Marvell International Ltd. |
19 | */ |
20 | #include <linux/dmaengine.h> |
21 | #include <linux/dma-mapping.h> |
22 | #include <linux/dmapool.h> |
23 | #include <linux/err.h> |
24 | #include <linux/init.h> |
25 | #include <linux/interrupt.h> |
26 | #include <linux/list.h> |
27 | #include <linux/module.h> |
28 | #include <linux/platform_device.h> |
29 | #include <linux/slab.h> |
30 | #include <linux/io.h> |
31 | #include <linux/spinlock.h> |
32 | #include <linux/of.h> |
33 | #include <linux/of_dma.h> |
34 | |
35 | #include "virt-dma.h" |
36 | |
37 | #define BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED 14 |
38 | #define BCM2835_DMA_CHAN_NAME_SIZE 8 |
39 | |
40 | /** |
41 | * struct bcm2835_dmadev - BCM2835 DMA controller |
42 | * @ddev: DMA device |
43 | * @base: base address of register map |
44 | * @zero_page: bus address of zero page (to detect transactions copying from |
45 | * zero page and avoid accessing memory if so) |
46 | */ |
47 | struct bcm2835_dmadev { |
48 | struct dma_device ddev; |
49 | void __iomem *base; |
50 | dma_addr_t zero_page; |
51 | }; |
52 | |
53 | struct bcm2835_dma_cb { |
54 | uint32_t info; |
55 | uint32_t src; |
56 | uint32_t dst; |
57 | uint32_t length; |
58 | uint32_t stride; |
59 | uint32_t next; |
60 | uint32_t pad[2]; |
61 | }; |
62 | |
63 | struct bcm2835_cb_entry { |
64 | struct bcm2835_dma_cb *cb; |
65 | dma_addr_t paddr; |
66 | }; |
67 | |
68 | struct bcm2835_chan { |
69 | struct virt_dma_chan vc; |
70 | |
71 | struct dma_slave_config cfg; |
72 | unsigned int dreq; |
73 | |
74 | int ch; |
75 | struct bcm2835_desc *desc; |
76 | struct dma_pool *cb_pool; |
77 | |
78 | void __iomem *chan_base; |
79 | int irq_number; |
80 | unsigned int irq_flags; |
81 | |
82 | bool is_lite_channel; |
83 | }; |
84 | |
85 | struct bcm2835_desc { |
86 | struct bcm2835_chan *c; |
87 | struct virt_dma_desc vd; |
88 | enum dma_transfer_direction dir; |
89 | |
90 | unsigned int frames; |
91 | size_t size; |
92 | |
93 | bool cyclic; |
94 | |
95 | struct bcm2835_cb_entry cb_list[]; |
96 | }; |
97 | |
98 | #define BCM2835_DMA_CS 0x00 |
99 | #define BCM2835_DMA_ADDR 0x04 |
100 | #define BCM2835_DMA_TI 0x08 |
101 | #define BCM2835_DMA_SOURCE_AD 0x0c |
102 | #define BCM2835_DMA_DEST_AD 0x10 |
103 | #define BCM2835_DMA_LEN 0x14 |
104 | #define BCM2835_DMA_STRIDE 0x18 |
105 | #define BCM2835_DMA_NEXTCB 0x1c |
106 | #define BCM2835_DMA_DEBUG 0x20 |
107 | |
108 | /* DMA CS Control and Status bits */ |
109 | #define BCM2835_DMA_ACTIVE BIT(0) /* activate the DMA */ |
110 | #define BCM2835_DMA_END BIT(1) /* current CB has ended */ |
111 | #define BCM2835_DMA_INT BIT(2) /* interrupt status */ |
112 | #define BCM2835_DMA_DREQ BIT(3) /* DREQ state */ |
113 | #define BCM2835_DMA_ISPAUSED BIT(4) /* Pause requested or not active */ |
114 | #define BCM2835_DMA_ISHELD BIT(5) /* Is held by DREQ flow control */ |
115 | #define BCM2835_DMA_WAITING_FOR_WRITES BIT(6) /* waiting for last |
116 | * AXI-write to ack |
117 | */ |
118 | #define BCM2835_DMA_ERR BIT(8) |
119 | #define BCM2835_DMA_PRIORITY(x) ((x & 15) << 16) /* AXI priority */ |
120 | #define BCM2835_DMA_PANIC_PRIORITY(x) ((x & 15) << 20) /* panic priority */ |
121 | /* current value of TI.BCM2835_DMA_WAIT_RESP */ |
122 | #define BCM2835_DMA_WAIT_FOR_WRITES BIT(28) |
123 | #define BCM2835_DMA_DIS_DEBUG BIT(29) /* disable debug pause signal */ |
124 | #define BCM2835_DMA_ABORT BIT(30) /* Stop current CB, go to next, WO */ |
125 | #define BCM2835_DMA_RESET BIT(31) /* WO, self clearing */ |
126 | |
127 | /* Transfer information bits - also bcm2835_cb.info field */ |
128 | #define BCM2835_DMA_INT_EN BIT(0) |
129 | #define BCM2835_DMA_TDMODE BIT(1) /* 2D-Mode */ |
130 | #define BCM2835_DMA_WAIT_RESP BIT(3) /* wait for AXI-write to be acked */ |
131 | #define BCM2835_DMA_D_INC BIT(4) |
132 | #define BCM2835_DMA_D_WIDTH BIT(5) /* 128bit writes if set */ |
133 | #define BCM2835_DMA_D_DREQ BIT(6) /* enable DREQ for destination */ |
134 | #define BCM2835_DMA_D_IGNORE BIT(7) /* ignore destination writes */ |
135 | #define BCM2835_DMA_S_INC BIT(8) |
136 | #define BCM2835_DMA_S_WIDTH BIT(9) /* 128bit writes if set */ |
137 | #define BCM2835_DMA_S_DREQ BIT(10) /* enable SREQ for source */ |
138 | #define BCM2835_DMA_S_IGNORE BIT(11) /* ignore source reads - read 0 */ |
139 | #define BCM2835_DMA_BURST_LENGTH(x) ((x & 15) << 12) |
140 | #define BCM2835_DMA_PER_MAP(x) ((x & 31) << 16) /* REQ source */ |
141 | #define BCM2835_DMA_WAIT(x) ((x & 31) << 21) /* add DMA-wait cycles */ |
142 | #define BCM2835_DMA_NO_WIDE_BURSTS BIT(26) /* no 2 beat write bursts */ |
143 | |
144 | /* debug register bits */ |
145 | #define BCM2835_DMA_DEBUG_LAST_NOT_SET_ERR BIT(0) |
146 | #define BCM2835_DMA_DEBUG_FIFO_ERR BIT(1) |
147 | #define BCM2835_DMA_DEBUG_READ_ERR BIT(2) |
148 | #define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_SHIFT 4 |
149 | #define BCM2835_DMA_DEBUG_OUTSTANDING_WRITES_BITS 4 |
150 | #define BCM2835_DMA_DEBUG_ID_SHIFT 16 |
151 | #define BCM2835_DMA_DEBUG_ID_BITS 9 |
152 | #define BCM2835_DMA_DEBUG_STATE_SHIFT 16 |
153 | #define BCM2835_DMA_DEBUG_STATE_BITS 9 |
154 | #define BCM2835_DMA_DEBUG_VERSION_SHIFT 25 |
155 | #define BCM2835_DMA_DEBUG_VERSION_BITS 3 |
156 | #define BCM2835_DMA_DEBUG_LITE BIT(28) |
157 | |
158 | /* shared registers for all dma channels */ |
159 | #define BCM2835_DMA_INT_STATUS 0xfe0 |
160 | #define BCM2835_DMA_ENABLE 0xff0 |
161 | |
162 | #define BCM2835_DMA_DATA_TYPE_S8 1 |
163 | #define BCM2835_DMA_DATA_TYPE_S16 2 |
164 | #define BCM2835_DMA_DATA_TYPE_S32 4 |
165 | #define BCM2835_DMA_DATA_TYPE_S128 16 |
166 | |
167 | /* Valid only for channels 0 - 14, 15 has its own base address */ |
168 | #define BCM2835_DMA_CHAN(n) ((n) << 8) /* Base address */ |
169 | #define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n)) |
170 | |
171 | /* the max dma length for different channels */ |
172 | #define MAX_DMA_LEN SZ_1G |
173 | #define MAX_LITE_DMA_LEN (SZ_64K - 4) |
174 | |
175 | static inline size_t bcm2835_dma_max_frame_length(struct bcm2835_chan *c) |
176 | { |
177 | /* lite and normal channels have different max frame length */ |
178 | return c->is_lite_channel ? MAX_LITE_DMA_LEN : MAX_DMA_LEN; |
179 | } |
180 | |
181 | /* how many frames of max_len size do we need to transfer len bytes */ |
182 | static inline size_t bcm2835_dma_frames_for_length(size_t len, |
183 | size_t max_len) |
184 | { |
185 | return DIV_ROUND_UP(len, max_len); |
186 | } |
187 | |
188 | static inline struct bcm2835_dmadev *to_bcm2835_dma_dev(struct dma_device *d) |
189 | { |
190 | return container_of(d, struct bcm2835_dmadev, ddev); |
191 | } |
192 | |
193 | static inline struct bcm2835_chan *to_bcm2835_dma_chan(struct dma_chan *c) |
194 | { |
195 | return container_of(c, struct bcm2835_chan, vc.chan); |
196 | } |
197 | |
198 | static inline struct bcm2835_desc *to_bcm2835_dma_desc( |
199 | struct dma_async_tx_descriptor *t) |
200 | { |
201 | return container_of(t, struct bcm2835_desc, vd.tx); |
202 | } |
203 | |
204 | static void bcm2835_dma_free_cb_chain(struct bcm2835_desc *desc) |
205 | { |
206 | size_t i; |
207 | |
208 | for (i = 0; i < desc->frames; i++) |
209 | dma_pool_free(pool: desc->c->cb_pool, vaddr: desc->cb_list[i].cb, |
210 | addr: desc->cb_list[i].paddr); |
211 | |
212 | kfree(objp: desc); |
213 | } |
214 | |
215 | static void bcm2835_dma_desc_free(struct virt_dma_desc *vd) |
216 | { |
217 | bcm2835_dma_free_cb_chain( |
218 | container_of(vd, struct bcm2835_desc, vd)); |
219 | } |
220 | |
221 | static void bcm2835_dma_create_cb_set_length( |
222 | struct bcm2835_chan *chan, |
223 | struct bcm2835_dma_cb *control_block, |
224 | size_t len, |
225 | size_t period_len, |
226 | size_t *total_len, |
227 | u32 ) |
228 | { |
229 | size_t max_len = bcm2835_dma_max_frame_length(c: chan); |
230 | |
231 | /* set the length taking lite-channel limitations into account */ |
232 | control_block->length = min_t(u32, len, max_len); |
233 | |
234 | /* finished if we have no period_length */ |
235 | if (!period_len) |
236 | return; |
237 | |
238 | /* |
239 | * period_len means: that we need to generate |
240 | * transfers that are terminating at every |
241 | * multiple of period_len - this is typically |
242 | * used to set the interrupt flag in info |
243 | * which is required during cyclic transfers |
244 | */ |
245 | |
246 | /* have we filled in period_length yet? */ |
247 | if (*total_len + control_block->length < period_len) { |
248 | /* update number of bytes in this period so far */ |
249 | *total_len += control_block->length; |
250 | return; |
251 | } |
252 | |
253 | /* calculate the length that remains to reach period_length */ |
254 | control_block->length = period_len - *total_len; |
255 | |
256 | /* reset total_length for next period */ |
257 | *total_len = 0; |
258 | |
259 | /* add extrainfo bits in info */ |
260 | control_block->info |= finalextrainfo; |
261 | } |
262 | |
263 | static inline size_t bcm2835_dma_count_frames_for_sg( |
264 | struct bcm2835_chan *c, |
265 | struct scatterlist *sgl, |
266 | unsigned int sg_len) |
267 | { |
268 | size_t frames = 0; |
269 | struct scatterlist *sgent; |
270 | unsigned int i; |
271 | size_t plength = bcm2835_dma_max_frame_length(c); |
272 | |
273 | for_each_sg(sgl, sgent, sg_len, i) |
274 | frames += bcm2835_dma_frames_for_length( |
275 | sg_dma_len(sgent), max_len: plength); |
276 | |
277 | return frames; |
278 | } |
279 | |
280 | /** |
281 | * bcm2835_dma_create_cb_chain - create a control block and fills data in |
282 | * |
283 | * @chan: the @dma_chan for which we run this |
284 | * @direction: the direction in which we transfer |
285 | * @cyclic: it is a cyclic transfer |
286 | * @info: the default info bits to apply per controlblock |
287 | * @frames: number of controlblocks to allocate |
288 | * @src: the src address to assign (if the S_INC bit is set |
289 | * in @info, then it gets incremented) |
290 | * @dst: the dst address to assign (if the D_INC bit is set |
291 | * in @info, then it gets incremented) |
292 | * @buf_len: the full buffer length (may also be 0) |
293 | * @period_len: the period length when to apply @finalextrainfo |
294 | * in addition to the last transfer |
295 | * this will also break some control-blocks early |
296 | * @finalextrainfo: additional bits in last controlblock |
297 | * (or when period_len is reached in case of cyclic) |
298 | * @gfp: the GFP flag to use for allocation |
299 | */ |
300 | static struct bcm2835_desc *bcm2835_dma_create_cb_chain( |
301 | struct dma_chan *chan, enum dma_transfer_direction direction, |
302 | bool cyclic, u32 info, u32 , size_t frames, |
303 | dma_addr_t src, dma_addr_t dst, size_t buf_len, |
304 | size_t period_len, gfp_t gfp) |
305 | { |
306 | struct bcm2835_chan *c = to_bcm2835_dma_chan(c: chan); |
307 | size_t len = buf_len, total_len; |
308 | size_t frame; |
309 | struct bcm2835_desc *d; |
310 | struct bcm2835_cb_entry *cb_entry; |
311 | struct bcm2835_dma_cb *control_block; |
312 | |
313 | if (!frames) |
314 | return NULL; |
315 | |
316 | /* allocate and setup the descriptor. */ |
317 | d = kzalloc(struct_size(d, cb_list, frames), flags: gfp); |
318 | if (!d) |
319 | return NULL; |
320 | |
321 | d->c = c; |
322 | d->dir = direction; |
323 | d->cyclic = cyclic; |
324 | |
325 | /* |
326 | * Iterate over all frames, create a control block |
327 | * for each frame and link them together. |
328 | */ |
329 | for (frame = 0, total_len = 0; frame < frames; d->frames++, frame++) { |
330 | cb_entry = &d->cb_list[frame]; |
331 | cb_entry->cb = dma_pool_alloc(pool: c->cb_pool, mem_flags: gfp, |
332 | handle: &cb_entry->paddr); |
333 | if (!cb_entry->cb) |
334 | goto error_cb; |
335 | |
336 | /* fill in the control block */ |
337 | control_block = cb_entry->cb; |
338 | control_block->info = info; |
339 | control_block->src = src; |
340 | control_block->dst = dst; |
341 | control_block->stride = 0; |
342 | control_block->next = 0; |
343 | /* set up length in control_block if requested */ |
344 | if (buf_len) { |
345 | /* calculate length honoring period_length */ |
346 | bcm2835_dma_create_cb_set_length( |
347 | chan: c, control_block, |
348 | len, period_len, total_len: &total_len, |
349 | finalextrainfo: cyclic ? finalextrainfo : 0); |
350 | |
351 | /* calculate new remaining length */ |
352 | len -= control_block->length; |
353 | } |
354 | |
355 | /* link this the last controlblock */ |
356 | if (frame) |
357 | d->cb_list[frame - 1].cb->next = cb_entry->paddr; |
358 | |
359 | /* update src and dst and length */ |
360 | if (src && (info & BCM2835_DMA_S_INC)) |
361 | src += control_block->length; |
362 | if (dst && (info & BCM2835_DMA_D_INC)) |
363 | dst += control_block->length; |
364 | |
365 | /* Length of total transfer */ |
366 | d->size += control_block->length; |
367 | } |
368 | |
369 | /* the last frame requires extra flags */ |
370 | d->cb_list[d->frames - 1].cb->info |= finalextrainfo; |
371 | |
372 | /* detect a size missmatch */ |
373 | if (buf_len && (d->size != buf_len)) |
374 | goto error_cb; |
375 | |
376 | return d; |
377 | error_cb: |
378 | bcm2835_dma_free_cb_chain(desc: d); |
379 | |
380 | return NULL; |
381 | } |
382 | |
383 | static void bcm2835_dma_fill_cb_chain_with_sg( |
384 | struct dma_chan *chan, |
385 | enum dma_transfer_direction direction, |
386 | struct bcm2835_cb_entry *cb, |
387 | struct scatterlist *sgl, |
388 | unsigned int sg_len) |
389 | { |
390 | struct bcm2835_chan *c = to_bcm2835_dma_chan(c: chan); |
391 | size_t len, max_len; |
392 | unsigned int i; |
393 | dma_addr_t addr; |
394 | struct scatterlist *sgent; |
395 | |
396 | max_len = bcm2835_dma_max_frame_length(c); |
397 | for_each_sg(sgl, sgent, sg_len, i) { |
398 | for (addr = sg_dma_address(sgent), len = sg_dma_len(sgent); |
399 | len > 0; |
400 | addr += cb->cb->length, len -= cb->cb->length, cb++) { |
401 | if (direction == DMA_DEV_TO_MEM) |
402 | cb->cb->dst = addr; |
403 | else |
404 | cb->cb->src = addr; |
405 | cb->cb->length = min(len, max_len); |
406 | } |
407 | } |
408 | } |
409 | |
410 | static void bcm2835_dma_abort(struct bcm2835_chan *c) |
411 | { |
412 | void __iomem *chan_base = c->chan_base; |
413 | long int timeout = 10000; |
414 | |
415 | /* |
416 | * A zero control block address means the channel is idle. |
417 | * (The ACTIVE flag in the CS register is not a reliable indicator.) |
418 | */ |
419 | if (!readl(addr: chan_base + BCM2835_DMA_ADDR)) |
420 | return; |
421 | |
422 | /* Write 0 to the active bit - Pause the DMA */ |
423 | writel(val: 0, addr: chan_base + BCM2835_DMA_CS); |
424 | |
425 | /* Wait for any current AXI transfer to complete */ |
426 | while ((readl(addr: chan_base + BCM2835_DMA_CS) & |
427 | BCM2835_DMA_WAITING_FOR_WRITES) && --timeout) |
428 | cpu_relax(); |
429 | |
430 | /* Peripheral might be stuck and fail to signal AXI write responses */ |
431 | if (!timeout) |
432 | dev_err(c->vc.chan.device->dev, |
433 | "failed to complete outstanding writes\n" ); |
434 | |
435 | writel(BCM2835_DMA_RESET, addr: chan_base + BCM2835_DMA_CS); |
436 | } |
437 | |
438 | static void bcm2835_dma_start_desc(struct bcm2835_chan *c) |
439 | { |
440 | struct virt_dma_desc *vd = vchan_next_desc(vc: &c->vc); |
441 | struct bcm2835_desc *d; |
442 | |
443 | if (!vd) { |
444 | c->desc = NULL; |
445 | return; |
446 | } |
447 | |
448 | list_del(entry: &vd->node); |
449 | |
450 | c->desc = d = to_bcm2835_dma_desc(t: &vd->tx); |
451 | |
452 | writel(val: d->cb_list[0].paddr, addr: c->chan_base + BCM2835_DMA_ADDR); |
453 | writel(BCM2835_DMA_ACTIVE, addr: c->chan_base + BCM2835_DMA_CS); |
454 | } |
455 | |
456 | static irqreturn_t bcm2835_dma_callback(int irq, void *data) |
457 | { |
458 | struct bcm2835_chan *c = data; |
459 | struct bcm2835_desc *d; |
460 | unsigned long flags; |
461 | |
462 | /* check the shared interrupt */ |
463 | if (c->irq_flags & IRQF_SHARED) { |
464 | /* check if the interrupt is enabled */ |
465 | flags = readl(addr: c->chan_base + BCM2835_DMA_CS); |
466 | /* if not set then we are not the reason for the irq */ |
467 | if (!(flags & BCM2835_DMA_INT)) |
468 | return IRQ_NONE; |
469 | } |
470 | |
471 | spin_lock_irqsave(&c->vc.lock, flags); |
472 | |
473 | /* |
474 | * Clear the INT flag to receive further interrupts. Keep the channel |
475 | * active in case the descriptor is cyclic or in case the client has |
476 | * already terminated the descriptor and issued a new one. (May happen |
477 | * if this IRQ handler is threaded.) If the channel is finished, it |
478 | * will remain idle despite the ACTIVE flag being set. |
479 | */ |
480 | writel(BCM2835_DMA_INT | BCM2835_DMA_ACTIVE, |
481 | addr: c->chan_base + BCM2835_DMA_CS); |
482 | |
483 | d = c->desc; |
484 | |
485 | if (d) { |
486 | if (d->cyclic) { |
487 | /* call the cyclic callback */ |
488 | vchan_cyclic_callback(vd: &d->vd); |
489 | } else if (!readl(addr: c->chan_base + BCM2835_DMA_ADDR)) { |
490 | vchan_cookie_complete(vd: &c->desc->vd); |
491 | bcm2835_dma_start_desc(c); |
492 | } |
493 | } |
494 | |
495 | spin_unlock_irqrestore(lock: &c->vc.lock, flags); |
496 | |
497 | return IRQ_HANDLED; |
498 | } |
499 | |
500 | static int bcm2835_dma_alloc_chan_resources(struct dma_chan *chan) |
501 | { |
502 | struct bcm2835_chan *c = to_bcm2835_dma_chan(c: chan); |
503 | struct device *dev = c->vc.chan.device->dev; |
504 | |
505 | dev_dbg(dev, "Allocating DMA channel %d\n" , c->ch); |
506 | |
507 | /* |
508 | * Control blocks are 256 bit in length and must start at a 256 bit |
509 | * (32 byte) aligned address (BCM2835 ARM Peripherals, sec. 4.2.1.1). |
510 | */ |
511 | c->cb_pool = dma_pool_create(name: dev_name(dev), dev, |
512 | size: sizeof(struct bcm2835_dma_cb), align: 32, allocation: 0); |
513 | if (!c->cb_pool) { |
514 | dev_err(dev, "unable to allocate descriptor pool\n" ); |
515 | return -ENOMEM; |
516 | } |
517 | |
518 | return request_irq(irq: c->irq_number, handler: bcm2835_dma_callback, |
519 | flags: c->irq_flags, name: "DMA IRQ" , dev: c); |
520 | } |
521 | |
522 | static void bcm2835_dma_free_chan_resources(struct dma_chan *chan) |
523 | { |
524 | struct bcm2835_chan *c = to_bcm2835_dma_chan(c: chan); |
525 | |
526 | vchan_free_chan_resources(vc: &c->vc); |
527 | free_irq(c->irq_number, c); |
528 | dma_pool_destroy(pool: c->cb_pool); |
529 | |
530 | dev_dbg(c->vc.chan.device->dev, "Freeing DMA channel %u\n" , c->ch); |
531 | } |
532 | |
533 | static size_t bcm2835_dma_desc_size(struct bcm2835_desc *d) |
534 | { |
535 | return d->size; |
536 | } |
537 | |
538 | static size_t bcm2835_dma_desc_size_pos(struct bcm2835_desc *d, dma_addr_t addr) |
539 | { |
540 | unsigned int i; |
541 | size_t size; |
542 | |
543 | for (size = i = 0; i < d->frames; i++) { |
544 | struct bcm2835_dma_cb *control_block = d->cb_list[i].cb; |
545 | size_t this_size = control_block->length; |
546 | dma_addr_t dma; |
547 | |
548 | if (d->dir == DMA_DEV_TO_MEM) |
549 | dma = control_block->dst; |
550 | else |
551 | dma = control_block->src; |
552 | |
553 | if (size) |
554 | size += this_size; |
555 | else if (addr >= dma && addr < dma + this_size) |
556 | size += dma + this_size - addr; |
557 | } |
558 | |
559 | return size; |
560 | } |
561 | |
562 | static enum dma_status bcm2835_dma_tx_status(struct dma_chan *chan, |
563 | dma_cookie_t cookie, struct dma_tx_state *txstate) |
564 | { |
565 | struct bcm2835_chan *c = to_bcm2835_dma_chan(c: chan); |
566 | struct virt_dma_desc *vd; |
567 | enum dma_status ret; |
568 | unsigned long flags; |
569 | |
570 | ret = dma_cookie_status(chan, cookie, state: txstate); |
571 | if (ret == DMA_COMPLETE || !txstate) |
572 | return ret; |
573 | |
574 | spin_lock_irqsave(&c->vc.lock, flags); |
575 | vd = vchan_find_desc(&c->vc, cookie); |
576 | if (vd) { |
577 | txstate->residue = |
578 | bcm2835_dma_desc_size(d: to_bcm2835_dma_desc(t: &vd->tx)); |
579 | } else if (c->desc && c->desc->vd.tx.cookie == cookie) { |
580 | struct bcm2835_desc *d = c->desc; |
581 | dma_addr_t pos; |
582 | |
583 | if (d->dir == DMA_MEM_TO_DEV) |
584 | pos = readl(addr: c->chan_base + BCM2835_DMA_SOURCE_AD); |
585 | else if (d->dir == DMA_DEV_TO_MEM) |
586 | pos = readl(addr: c->chan_base + BCM2835_DMA_DEST_AD); |
587 | else |
588 | pos = 0; |
589 | |
590 | txstate->residue = bcm2835_dma_desc_size_pos(d, addr: pos); |
591 | } else { |
592 | txstate->residue = 0; |
593 | } |
594 | |
595 | spin_unlock_irqrestore(lock: &c->vc.lock, flags); |
596 | |
597 | return ret; |
598 | } |
599 | |
600 | static void bcm2835_dma_issue_pending(struct dma_chan *chan) |
601 | { |
602 | struct bcm2835_chan *c = to_bcm2835_dma_chan(c: chan); |
603 | unsigned long flags; |
604 | |
605 | spin_lock_irqsave(&c->vc.lock, flags); |
606 | if (vchan_issue_pending(vc: &c->vc) && !c->desc) |
607 | bcm2835_dma_start_desc(c); |
608 | |
609 | spin_unlock_irqrestore(lock: &c->vc.lock, flags); |
610 | } |
611 | |
612 | static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_memcpy( |
613 | struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, |
614 | size_t len, unsigned long flags) |
615 | { |
616 | struct bcm2835_chan *c = to_bcm2835_dma_chan(c: chan); |
617 | struct bcm2835_desc *d; |
618 | u32 info = BCM2835_DMA_D_INC | BCM2835_DMA_S_INC; |
619 | u32 = BCM2835_DMA_INT_EN | BCM2835_DMA_WAIT_RESP; |
620 | size_t max_len = bcm2835_dma_max_frame_length(c); |
621 | size_t frames; |
622 | |
623 | /* if src, dst or len is not given return with an error */ |
624 | if (!src || !dst || !len) |
625 | return NULL; |
626 | |
627 | /* calculate number of frames */ |
628 | frames = bcm2835_dma_frames_for_length(len, max_len); |
629 | |
630 | /* allocate the CB chain - this also fills in the pointers */ |
631 | d = bcm2835_dma_create_cb_chain(chan, direction: DMA_MEM_TO_MEM, cyclic: false, |
632 | info, finalextrainfo: extra, frames, |
633 | src, dst, buf_len: len, period_len: 0, GFP_KERNEL); |
634 | if (!d) |
635 | return NULL; |
636 | |
637 | return vchan_tx_prep(vc: &c->vc, vd: &d->vd, tx_flags: flags); |
638 | } |
639 | |
640 | static struct dma_async_tx_descriptor *bcm2835_dma_prep_slave_sg( |
641 | struct dma_chan *chan, |
642 | struct scatterlist *sgl, unsigned int sg_len, |
643 | enum dma_transfer_direction direction, |
644 | unsigned long flags, void *context) |
645 | { |
646 | struct bcm2835_chan *c = to_bcm2835_dma_chan(c: chan); |
647 | struct bcm2835_desc *d; |
648 | dma_addr_t src = 0, dst = 0; |
649 | u32 info = BCM2835_DMA_WAIT_RESP; |
650 | u32 = BCM2835_DMA_INT_EN; |
651 | size_t frames; |
652 | |
653 | if (!is_slave_direction(direction)) { |
654 | dev_err(chan->device->dev, |
655 | "%s: bad direction?\n" , __func__); |
656 | return NULL; |
657 | } |
658 | |
659 | if (c->dreq != 0) |
660 | info |= BCM2835_DMA_PER_MAP(c->dreq); |
661 | |
662 | if (direction == DMA_DEV_TO_MEM) { |
663 | if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) |
664 | return NULL; |
665 | src = c->cfg.src_addr; |
666 | info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC; |
667 | } else { |
668 | if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) |
669 | return NULL; |
670 | dst = c->cfg.dst_addr; |
671 | info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC; |
672 | } |
673 | |
674 | /* count frames in sg list */ |
675 | frames = bcm2835_dma_count_frames_for_sg(c, sgl, sg_len); |
676 | |
677 | /* allocate the CB chain */ |
678 | d = bcm2835_dma_create_cb_chain(chan, direction, cyclic: false, |
679 | info, finalextrainfo: extra, |
680 | frames, src, dst, buf_len: 0, period_len: 0, |
681 | GFP_NOWAIT); |
682 | if (!d) |
683 | return NULL; |
684 | |
685 | /* fill in frames with scatterlist pointers */ |
686 | bcm2835_dma_fill_cb_chain_with_sg(chan, direction, cb: d->cb_list, |
687 | sgl, sg_len); |
688 | |
689 | return vchan_tx_prep(vc: &c->vc, vd: &d->vd, tx_flags: flags); |
690 | } |
691 | |
692 | static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic( |
693 | struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, |
694 | size_t period_len, enum dma_transfer_direction direction, |
695 | unsigned long flags) |
696 | { |
697 | struct bcm2835_dmadev *od = to_bcm2835_dma_dev(d: chan->device); |
698 | struct bcm2835_chan *c = to_bcm2835_dma_chan(c: chan); |
699 | struct bcm2835_desc *d; |
700 | dma_addr_t src, dst; |
701 | u32 info = BCM2835_DMA_WAIT_RESP; |
702 | u32 = 0; |
703 | size_t max_len = bcm2835_dma_max_frame_length(c); |
704 | size_t frames; |
705 | |
706 | /* Grab configuration */ |
707 | if (!is_slave_direction(direction)) { |
708 | dev_err(chan->device->dev, "%s: bad direction?\n" , __func__); |
709 | return NULL; |
710 | } |
711 | |
712 | if (!buf_len) { |
713 | dev_err(chan->device->dev, |
714 | "%s: bad buffer length (= 0)\n" , __func__); |
715 | return NULL; |
716 | } |
717 | |
718 | if (flags & DMA_PREP_INTERRUPT) |
719 | extra |= BCM2835_DMA_INT_EN; |
720 | else |
721 | period_len = buf_len; |
722 | |
723 | /* |
724 | * warn if buf_len is not a multiple of period_len - this may leed |
725 | * to unexpected latencies for interrupts and thus audiable clicks |
726 | */ |
727 | if (buf_len % period_len) |
728 | dev_warn_once(chan->device->dev, |
729 | "%s: buffer_length (%zd) is not a multiple of period_len (%zd)\n" , |
730 | __func__, buf_len, period_len); |
731 | |
732 | /* Setup DREQ channel */ |
733 | if (c->dreq != 0) |
734 | info |= BCM2835_DMA_PER_MAP(c->dreq); |
735 | |
736 | if (direction == DMA_DEV_TO_MEM) { |
737 | if (c->cfg.src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) |
738 | return NULL; |
739 | src = c->cfg.src_addr; |
740 | dst = buf_addr; |
741 | info |= BCM2835_DMA_S_DREQ | BCM2835_DMA_D_INC; |
742 | } else { |
743 | if (c->cfg.dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) |
744 | return NULL; |
745 | dst = c->cfg.dst_addr; |
746 | src = buf_addr; |
747 | info |= BCM2835_DMA_D_DREQ | BCM2835_DMA_S_INC; |
748 | |
749 | /* non-lite channels can write zeroes w/o accessing memory */ |
750 | if (buf_addr == od->zero_page && !c->is_lite_channel) |
751 | info |= BCM2835_DMA_S_IGNORE; |
752 | } |
753 | |
754 | /* calculate number of frames */ |
755 | frames = /* number of periods */ |
756 | DIV_ROUND_UP(buf_len, period_len) * |
757 | /* number of frames per period */ |
758 | bcm2835_dma_frames_for_length(len: period_len, max_len); |
759 | |
760 | /* |
761 | * allocate the CB chain |
762 | * note that we need to use GFP_NOWAIT, as the ALSA i2s dmaengine |
763 | * implementation calls prep_dma_cyclic with interrupts disabled. |
764 | */ |
765 | d = bcm2835_dma_create_cb_chain(chan, direction, cyclic: true, |
766 | info, finalextrainfo: extra, |
767 | frames, src, dst, buf_len, |
768 | period_len, GFP_NOWAIT); |
769 | if (!d) |
770 | return NULL; |
771 | |
772 | /* wrap around into a loop */ |
773 | d->cb_list[d->frames - 1].cb->next = d->cb_list[0].paddr; |
774 | |
775 | return vchan_tx_prep(vc: &c->vc, vd: &d->vd, tx_flags: flags); |
776 | } |
777 | |
778 | static int bcm2835_dma_slave_config(struct dma_chan *chan, |
779 | struct dma_slave_config *cfg) |
780 | { |
781 | struct bcm2835_chan *c = to_bcm2835_dma_chan(c: chan); |
782 | |
783 | c->cfg = *cfg; |
784 | |
785 | return 0; |
786 | } |
787 | |
788 | static int bcm2835_dma_terminate_all(struct dma_chan *chan) |
789 | { |
790 | struct bcm2835_chan *c = to_bcm2835_dma_chan(c: chan); |
791 | unsigned long flags; |
792 | LIST_HEAD(head); |
793 | |
794 | spin_lock_irqsave(&c->vc.lock, flags); |
795 | |
796 | /* stop DMA activity */ |
797 | if (c->desc) { |
798 | vchan_terminate_vdesc(vd: &c->desc->vd); |
799 | c->desc = NULL; |
800 | bcm2835_dma_abort(c); |
801 | } |
802 | |
803 | vchan_get_all_descriptors(vc: &c->vc, head: &head); |
804 | spin_unlock_irqrestore(lock: &c->vc.lock, flags); |
805 | vchan_dma_desc_free_list(vc: &c->vc, head: &head); |
806 | |
807 | return 0; |
808 | } |
809 | |
810 | static void bcm2835_dma_synchronize(struct dma_chan *chan) |
811 | { |
812 | struct bcm2835_chan *c = to_bcm2835_dma_chan(c: chan); |
813 | |
814 | vchan_synchronize(vc: &c->vc); |
815 | } |
816 | |
817 | static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id, |
818 | int irq, unsigned int irq_flags) |
819 | { |
820 | struct bcm2835_chan *c; |
821 | |
822 | c = devm_kzalloc(dev: d->ddev.dev, size: sizeof(*c), GFP_KERNEL); |
823 | if (!c) |
824 | return -ENOMEM; |
825 | |
826 | c->vc.desc_free = bcm2835_dma_desc_free; |
827 | vchan_init(vc: &c->vc, dmadev: &d->ddev); |
828 | |
829 | c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id); |
830 | c->ch = chan_id; |
831 | c->irq_number = irq; |
832 | c->irq_flags = irq_flags; |
833 | |
834 | /* check in DEBUG register if this is a LITE channel */ |
835 | if (readl(addr: c->chan_base + BCM2835_DMA_DEBUG) & |
836 | BCM2835_DMA_DEBUG_LITE) |
837 | c->is_lite_channel = true; |
838 | |
839 | return 0; |
840 | } |
841 | |
842 | static void bcm2835_dma_free(struct bcm2835_dmadev *od) |
843 | { |
844 | struct bcm2835_chan *c, *next; |
845 | |
846 | list_for_each_entry_safe(c, next, &od->ddev.channels, |
847 | vc.chan.device_node) { |
848 | list_del(entry: &c->vc.chan.device_node); |
849 | tasklet_kill(t: &c->vc.task); |
850 | } |
851 | |
852 | dma_unmap_page_attrs(dev: od->ddev.dev, addr: od->zero_page, PAGE_SIZE, |
853 | dir: DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); |
854 | } |
855 | |
856 | static const struct of_device_id bcm2835_dma_of_match[] = { |
857 | { .compatible = "brcm,bcm2835-dma" , }, |
858 | {}, |
859 | }; |
860 | MODULE_DEVICE_TABLE(of, bcm2835_dma_of_match); |
861 | |
862 | static struct dma_chan *bcm2835_dma_xlate(struct of_phandle_args *spec, |
863 | struct of_dma *ofdma) |
864 | { |
865 | struct bcm2835_dmadev *d = ofdma->of_dma_data; |
866 | struct dma_chan *chan; |
867 | |
868 | chan = dma_get_any_slave_channel(device: &d->ddev); |
869 | if (!chan) |
870 | return NULL; |
871 | |
872 | /* Set DREQ from param */ |
873 | to_bcm2835_dma_chan(c: chan)->dreq = spec->args[0]; |
874 | |
875 | return chan; |
876 | } |
877 | |
878 | static int bcm2835_dma_probe(struct platform_device *pdev) |
879 | { |
880 | struct bcm2835_dmadev *od; |
881 | void __iomem *base; |
882 | int rc; |
883 | int i, j; |
884 | int irq[BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED + 1]; |
885 | int irq_flags; |
886 | uint32_t chans_available; |
887 | char chan_name[BCM2835_DMA_CHAN_NAME_SIZE]; |
888 | |
889 | if (!pdev->dev.dma_mask) |
890 | pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask; |
891 | |
892 | rc = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(32)); |
893 | if (rc) { |
894 | dev_err(&pdev->dev, "Unable to set DMA mask\n" ); |
895 | return rc; |
896 | } |
897 | |
898 | od = devm_kzalloc(dev: &pdev->dev, size: sizeof(*od), GFP_KERNEL); |
899 | if (!od) |
900 | return -ENOMEM; |
901 | |
902 | dma_set_max_seg_size(dev: &pdev->dev, size: 0x3FFFFFFF); |
903 | |
904 | base = devm_platform_ioremap_resource(pdev, index: 0); |
905 | if (IS_ERR(ptr: base)) |
906 | return PTR_ERR(ptr: base); |
907 | |
908 | od->base = base; |
909 | |
910 | dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); |
911 | dma_cap_set(DMA_PRIVATE, od->ddev.cap_mask); |
912 | dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); |
913 | dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask); |
914 | od->ddev.device_alloc_chan_resources = bcm2835_dma_alloc_chan_resources; |
915 | od->ddev.device_free_chan_resources = bcm2835_dma_free_chan_resources; |
916 | od->ddev.device_tx_status = bcm2835_dma_tx_status; |
917 | od->ddev.device_issue_pending = bcm2835_dma_issue_pending; |
918 | od->ddev.device_prep_dma_cyclic = bcm2835_dma_prep_dma_cyclic; |
919 | od->ddev.device_prep_slave_sg = bcm2835_dma_prep_slave_sg; |
920 | od->ddev.device_prep_dma_memcpy = bcm2835_dma_prep_dma_memcpy; |
921 | od->ddev.device_config = bcm2835_dma_slave_config; |
922 | od->ddev.device_terminate_all = bcm2835_dma_terminate_all; |
923 | od->ddev.device_synchronize = bcm2835_dma_synchronize; |
924 | od->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
925 | od->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
926 | od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | |
927 | BIT(DMA_MEM_TO_MEM); |
928 | od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; |
929 | od->ddev.descriptor_reuse = true; |
930 | od->ddev.dev = &pdev->dev; |
931 | INIT_LIST_HEAD(list: &od->ddev.channels); |
932 | |
933 | platform_set_drvdata(pdev, data: od); |
934 | |
935 | od->zero_page = dma_map_page_attrs(dev: od->ddev.dev, ZERO_PAGE(0), offset: 0, |
936 | PAGE_SIZE, dir: DMA_TO_DEVICE, |
937 | DMA_ATTR_SKIP_CPU_SYNC); |
938 | if (dma_mapping_error(dev: od->ddev.dev, dma_addr: od->zero_page)) { |
939 | dev_err(&pdev->dev, "Failed to map zero page\n" ); |
940 | return -ENOMEM; |
941 | } |
942 | |
943 | /* Request DMA channel mask from device tree */ |
944 | if (of_property_read_u32(np: pdev->dev.of_node, |
945 | propname: "brcm,dma-channel-mask" , |
946 | out_value: &chans_available)) { |
947 | dev_err(&pdev->dev, "Failed to get channel mask\n" ); |
948 | rc = -EINVAL; |
949 | goto err_no_dma; |
950 | } |
951 | |
952 | /* get irqs for each channel that we support */ |
953 | for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) { |
954 | /* skip masked out channels */ |
955 | if (!(chans_available & (1 << i))) { |
956 | irq[i] = -1; |
957 | continue; |
958 | } |
959 | |
960 | /* get the named irq */ |
961 | snprintf(buf: chan_name, size: sizeof(chan_name), fmt: "dma%i" , i); |
962 | irq[i] = platform_get_irq_byname(pdev, chan_name); |
963 | if (irq[i] >= 0) |
964 | continue; |
965 | |
966 | /* legacy device tree case handling */ |
967 | dev_warn_once(&pdev->dev, |
968 | "missing interrupt-names property in device tree - legacy interpretation is used\n" ); |
969 | /* |
970 | * in case of channel >= 11 |
971 | * use the 11th interrupt and that is shared |
972 | */ |
973 | irq[i] = platform_get_irq(pdev, i < 11 ? i : 11); |
974 | } |
975 | |
976 | /* get irqs for each channel */ |
977 | for (i = 0; i <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; i++) { |
978 | /* skip channels without irq */ |
979 | if (irq[i] < 0) |
980 | continue; |
981 | |
982 | /* check if there are other channels that also use this irq */ |
983 | irq_flags = 0; |
984 | for (j = 0; j <= BCM2835_DMA_MAX_DMA_CHAN_SUPPORTED; j++) |
985 | if ((i != j) && (irq[j] == irq[i])) { |
986 | irq_flags = IRQF_SHARED; |
987 | break; |
988 | } |
989 | |
990 | /* initialize the channel */ |
991 | rc = bcm2835_dma_chan_init(d: od, chan_id: i, irq: irq[i], irq_flags); |
992 | if (rc) |
993 | goto err_no_dma; |
994 | } |
995 | |
996 | dev_dbg(&pdev->dev, "Initialized %i DMA channels\n" , i); |
997 | |
998 | /* Device-tree DMA controller registration */ |
999 | rc = of_dma_controller_register(np: pdev->dev.of_node, |
1000 | of_dma_xlate: bcm2835_dma_xlate, data: od); |
1001 | if (rc) { |
1002 | dev_err(&pdev->dev, "Failed to register DMA controller\n" ); |
1003 | goto err_no_dma; |
1004 | } |
1005 | |
1006 | rc = dma_async_device_register(device: &od->ddev); |
1007 | if (rc) { |
1008 | dev_err(&pdev->dev, |
1009 | "Failed to register slave DMA engine device: %d\n" , rc); |
1010 | goto err_no_dma; |
1011 | } |
1012 | |
1013 | dev_dbg(&pdev->dev, "Load BCM2835 DMA engine driver\n" ); |
1014 | |
1015 | return 0; |
1016 | |
1017 | err_no_dma: |
1018 | bcm2835_dma_free(od); |
1019 | return rc; |
1020 | } |
1021 | |
1022 | static void bcm2835_dma_remove(struct platform_device *pdev) |
1023 | { |
1024 | struct bcm2835_dmadev *od = platform_get_drvdata(pdev); |
1025 | |
1026 | dma_async_device_unregister(device: &od->ddev); |
1027 | bcm2835_dma_free(od); |
1028 | } |
1029 | |
1030 | static struct platform_driver bcm2835_dma_driver = { |
1031 | .probe = bcm2835_dma_probe, |
1032 | .remove_new = bcm2835_dma_remove, |
1033 | .driver = { |
1034 | .name = "bcm2835-dma" , |
1035 | .of_match_table = of_match_ptr(bcm2835_dma_of_match), |
1036 | }, |
1037 | }; |
1038 | |
1039 | module_platform_driver(bcm2835_dma_driver); |
1040 | |
1041 | MODULE_ALIAS("platform:bcm2835-dma" ); |
1042 | MODULE_DESCRIPTION("BCM2835 DMA engine driver" ); |
1043 | MODULE_AUTHOR("Florian Meier <florian.meier@koalo.de>" ); |
1044 | MODULE_LICENSE("GPL" ); |
1045 | |