1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Driver for Audio DMA Controller (ADMAC) on t8103 (M1) and other Apple chips |
4 | * |
5 | * Copyright (C) The Asahi Linux Contributors |
6 | */ |
7 | |
8 | #include <linux/bits.h> |
9 | #include <linux/bitfield.h> |
10 | #include <linux/device.h> |
11 | #include <linux/init.h> |
12 | #include <linux/module.h> |
13 | #include <linux/of.h> |
14 | #include <linux/of_dma.h> |
15 | #include <linux/platform_device.h> |
16 | #include <linux/reset.h> |
17 | #include <linux/spinlock.h> |
18 | #include <linux/interrupt.h> |
19 | |
20 | #include "dmaengine.h" |
21 | |
22 | #define NCHANNELS_MAX 64 |
23 | #define IRQ_NOUTPUTS 4 |
24 | |
25 | /* |
26 | * For allocation purposes we split the cache |
27 | * memory into blocks of fixed size (given in bytes). |
28 | */ |
29 | #define SRAM_BLOCK 2048 |
30 | |
31 | #define RING_WRITE_SLOT GENMASK(1, 0) |
32 | #define RING_READ_SLOT GENMASK(5, 4) |
33 | #define RING_FULL BIT(9) |
34 | #define RING_EMPTY BIT(8) |
35 | #define RING_ERR BIT(10) |
36 | |
37 | #define STATUS_DESC_DONE BIT(0) |
38 | #define STATUS_ERR BIT(6) |
39 | |
40 | #define FLAG_DESC_NOTIFY BIT(16) |
41 | |
42 | #define REG_TX_START 0x0000 |
43 | #define REG_TX_STOP 0x0004 |
44 | #define REG_RX_START 0x0008 |
45 | #define REG_RX_STOP 0x000c |
46 | #define REG_IMPRINT 0x0090 |
47 | #define REG_TX_SRAM_SIZE 0x0094 |
48 | #define REG_RX_SRAM_SIZE 0x0098 |
49 | |
50 | #define REG_CHAN_CTL(ch) (0x8000 + (ch) * 0x200) |
51 | #define REG_CHAN_CTL_RST_RINGS BIT(0) |
52 | |
53 | #define REG_DESC_RING(ch) (0x8070 + (ch) * 0x200) |
54 | #define REG_REPORT_RING(ch) (0x8074 + (ch) * 0x200) |
55 | |
56 | #define REG_RESIDUE(ch) (0x8064 + (ch) * 0x200) |
57 | |
58 | #define REG_BUS_WIDTH(ch) (0x8040 + (ch) * 0x200) |
59 | |
60 | #define BUS_WIDTH_8BIT 0x00 |
61 | #define BUS_WIDTH_16BIT 0x01 |
62 | #define BUS_WIDTH_32BIT 0x02 |
63 | #define BUS_WIDTH_FRAME_2_WORDS 0x10 |
64 | #define BUS_WIDTH_FRAME_4_WORDS 0x20 |
65 | |
66 | #define REG_CHAN_SRAM_CARVEOUT(ch) (0x8050 + (ch) * 0x200) |
67 | #define CHAN_SRAM_CARVEOUT_SIZE GENMASK(31, 16) |
68 | #define CHAN_SRAM_CARVEOUT_BASE GENMASK(15, 0) |
69 | |
70 | #define REG_CHAN_FIFOCTL(ch) (0x8054 + (ch) * 0x200) |
71 | #define CHAN_FIFOCTL_LIMIT GENMASK(31, 16) |
72 | #define CHAN_FIFOCTL_THRESHOLD GENMASK(15, 0) |
73 | |
74 | #define REG_DESC_WRITE(ch) (0x10000 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000) |
75 | #define REG_REPORT_READ(ch) (0x10100 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000) |
76 | |
77 | #define REG_TX_INTSTATE(idx) (0x0030 + (idx) * 4) |
78 | #define REG_RX_INTSTATE(idx) (0x0040 + (idx) * 4) |
79 | #define REG_GLOBAL_INTSTATE(idx) (0x0050 + (idx) * 4) |
80 | #define REG_CHAN_INTSTATUS(ch, idx) (0x8010 + (ch) * 0x200 + (idx) * 4) |
81 | #define REG_CHAN_INTMASK(ch, idx) (0x8020 + (ch) * 0x200 + (idx) * 4) |
82 | |
83 | struct admac_data; |
84 | struct admac_tx; |
85 | |
86 | struct admac_chan { |
87 | unsigned int no; |
88 | struct admac_data *host; |
89 | struct dma_chan chan; |
90 | struct tasklet_struct tasklet; |
91 | |
92 | u32 carveout; |
93 | |
94 | spinlock_t lock; |
95 | struct admac_tx *current_tx; |
96 | int nperiod_acks; |
97 | |
98 | /* |
99 | * We maintain a 'submitted' and 'issued' list mainly for interface |
100 | * correctness. Typical use of the driver (per channel) will be |
101 | * prepping, submitting and issuing a single cyclic transaction which |
102 | * will stay current until terminate_all is called. |
103 | */ |
104 | struct list_head submitted; |
105 | struct list_head issued; |
106 | |
107 | struct list_head to_free; |
108 | }; |
109 | |
110 | struct admac_sram { |
111 | u32 size; |
112 | /* |
113 | * SRAM_CARVEOUT has 16-bit fields, so the SRAM cannot be larger than |
114 | * 64K and a 32-bit bitfield over 2K blocks covers it. |
115 | */ |
116 | u32 allocated; |
117 | }; |
118 | |
119 | struct admac_data { |
120 | struct dma_device dma; |
121 | struct device *dev; |
122 | __iomem void *base; |
123 | struct reset_control *rstc; |
124 | |
125 | struct mutex cache_alloc_lock; |
126 | struct admac_sram txcache, rxcache; |
127 | |
128 | int irq; |
129 | int irq_index; |
130 | int nchannels; |
131 | struct admac_chan channels[] __counted_by(nchannels); |
132 | }; |
133 | |
134 | struct admac_tx { |
135 | struct dma_async_tx_descriptor tx; |
136 | bool cyclic; |
137 | dma_addr_t buf_addr; |
138 | dma_addr_t buf_end; |
139 | size_t buf_len; |
140 | size_t period_len; |
141 | |
142 | size_t submitted_pos; |
143 | size_t reclaimed_pos; |
144 | |
145 | struct list_head node; |
146 | }; |
147 | |
148 | static int admac_alloc_sram_carveout(struct admac_data *ad, |
149 | enum dma_transfer_direction dir, |
150 | u32 *out) |
151 | { |
152 | struct admac_sram *sram; |
153 | int i, ret = 0, nblocks; |
154 | |
155 | if (dir == DMA_MEM_TO_DEV) |
156 | sram = &ad->txcache; |
157 | else |
158 | sram = &ad->rxcache; |
159 | |
160 | mutex_lock(&ad->cache_alloc_lock); |
161 | |
162 | nblocks = sram->size / SRAM_BLOCK; |
163 | for (i = 0; i < nblocks; i++) |
164 | if (!(sram->allocated & BIT(i))) |
165 | break; |
166 | |
167 | if (i < nblocks) { |
168 | *out = FIELD_PREP(CHAN_SRAM_CARVEOUT_BASE, i * SRAM_BLOCK) | |
169 | FIELD_PREP(CHAN_SRAM_CARVEOUT_SIZE, SRAM_BLOCK); |
170 | sram->allocated |= BIT(i); |
171 | } else { |
172 | ret = -EBUSY; |
173 | } |
174 | |
175 | mutex_unlock(lock: &ad->cache_alloc_lock); |
176 | |
177 | return ret; |
178 | } |
179 | |
180 | static void admac_free_sram_carveout(struct admac_data *ad, |
181 | enum dma_transfer_direction dir, |
182 | u32 carveout) |
183 | { |
184 | struct admac_sram *sram; |
185 | u32 base = FIELD_GET(CHAN_SRAM_CARVEOUT_BASE, carveout); |
186 | int i; |
187 | |
188 | if (dir == DMA_MEM_TO_DEV) |
189 | sram = &ad->txcache; |
190 | else |
191 | sram = &ad->rxcache; |
192 | |
193 | if (WARN_ON(base >= sram->size)) |
194 | return; |
195 | |
196 | mutex_lock(&ad->cache_alloc_lock); |
197 | i = base / SRAM_BLOCK; |
198 | sram->allocated &= ~BIT(i); |
199 | mutex_unlock(lock: &ad->cache_alloc_lock); |
200 | } |
201 | |
202 | static void admac_modify(struct admac_data *ad, int reg, u32 mask, u32 val) |
203 | { |
204 | void __iomem *addr = ad->base + reg; |
205 | u32 curr = readl_relaxed(addr); |
206 | |
207 | writel_relaxed((curr & ~mask) | (val & mask), addr); |
208 | } |
209 | |
210 | static struct admac_chan *to_admac_chan(struct dma_chan *chan) |
211 | { |
212 | return container_of(chan, struct admac_chan, chan); |
213 | } |
214 | |
215 | static struct admac_tx *to_admac_tx(struct dma_async_tx_descriptor *tx) |
216 | { |
217 | return container_of(tx, struct admac_tx, tx); |
218 | } |
219 | |
220 | static enum dma_transfer_direction admac_chan_direction(int channo) |
221 | { |
222 | /* Channel directions are hardwired */ |
223 | return (channo & 1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; |
224 | } |
225 | |
226 | static dma_cookie_t admac_tx_submit(struct dma_async_tx_descriptor *tx) |
227 | { |
228 | struct admac_tx *adtx = to_admac_tx(tx); |
229 | struct admac_chan *adchan = to_admac_chan(chan: tx->chan); |
230 | unsigned long flags; |
231 | dma_cookie_t cookie; |
232 | |
233 | spin_lock_irqsave(&adchan->lock, flags); |
234 | cookie = dma_cookie_assign(tx); |
235 | list_add_tail(new: &adtx->node, head: &adchan->submitted); |
236 | spin_unlock_irqrestore(lock: &adchan->lock, flags); |
237 | |
238 | return cookie; |
239 | } |
240 | |
241 | static int admac_desc_free(struct dma_async_tx_descriptor *tx) |
242 | { |
243 | kfree(objp: to_admac_tx(tx)); |
244 | |
245 | return 0; |
246 | } |
247 | |
248 | static struct dma_async_tx_descriptor *admac_prep_dma_cyclic( |
249 | struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, |
250 | size_t period_len, enum dma_transfer_direction direction, |
251 | unsigned long flags) |
252 | { |
253 | struct admac_chan *adchan = container_of(chan, struct admac_chan, chan); |
254 | struct admac_tx *adtx; |
255 | |
256 | if (direction != admac_chan_direction(channo: adchan->no)) |
257 | return NULL; |
258 | |
259 | adtx = kzalloc(size: sizeof(*adtx), GFP_NOWAIT); |
260 | if (!adtx) |
261 | return NULL; |
262 | |
263 | adtx->cyclic = true; |
264 | |
265 | adtx->buf_addr = buf_addr; |
266 | adtx->buf_len = buf_len; |
267 | adtx->buf_end = buf_addr + buf_len; |
268 | adtx->period_len = period_len; |
269 | |
270 | adtx->submitted_pos = 0; |
271 | adtx->reclaimed_pos = 0; |
272 | |
273 | dma_async_tx_descriptor_init(tx: &adtx->tx, chan); |
274 | adtx->tx.tx_submit = admac_tx_submit; |
275 | adtx->tx.desc_free = admac_desc_free; |
276 | |
277 | return &adtx->tx; |
278 | } |
279 | |
280 | /* |
281 | * Write one hardware descriptor for a dmaengine cyclic transaction. |
282 | */ |
283 | static void admac_cyclic_write_one_desc(struct admac_data *ad, int channo, |
284 | struct admac_tx *tx) |
285 | { |
286 | dma_addr_t addr; |
287 | |
288 | addr = tx->buf_addr + (tx->submitted_pos % tx->buf_len); |
289 | |
290 | /* If happens means we have buggy code */ |
291 | WARN_ON_ONCE(addr + tx->period_len > tx->buf_end); |
292 | |
293 | dev_dbg(ad->dev, "ch%d descriptor: addr=0x%pad len=0x%zx flags=0x%lx\n" , |
294 | channo, &addr, tx->period_len, FLAG_DESC_NOTIFY); |
295 | |
296 | writel_relaxed(lower_32_bits(addr), ad->base + REG_DESC_WRITE(channo)); |
297 | writel_relaxed(upper_32_bits(addr), ad->base + REG_DESC_WRITE(channo)); |
298 | writel_relaxed(tx->period_len, ad->base + REG_DESC_WRITE(channo)); |
299 | writel_relaxed(FLAG_DESC_NOTIFY, ad->base + REG_DESC_WRITE(channo)); |
300 | |
301 | tx->submitted_pos += tx->period_len; |
302 | tx->submitted_pos %= 2 * tx->buf_len; |
303 | } |
304 | |
305 | /* |
306 | * Write all the hardware descriptors for a dmaengine cyclic |
307 | * transaction there is space for. |
308 | */ |
309 | static void admac_cyclic_write_desc(struct admac_data *ad, int channo, |
310 | struct admac_tx *tx) |
311 | { |
312 | int i; |
313 | |
314 | for (i = 0; i < 4; i++) { |
315 | if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_FULL) |
316 | break; |
317 | admac_cyclic_write_one_desc(ad, channo, tx); |
318 | } |
319 | } |
320 | |
321 | static int admac_ring_noccupied_slots(int ringval) |
322 | { |
323 | int wrslot = FIELD_GET(RING_WRITE_SLOT, ringval); |
324 | int rdslot = FIELD_GET(RING_READ_SLOT, ringval); |
325 | |
326 | if (wrslot != rdslot) { |
327 | return (wrslot + 4 - rdslot) % 4; |
328 | } else { |
329 | WARN_ON((ringval & (RING_FULL | RING_EMPTY)) == 0); |
330 | |
331 | if (ringval & RING_FULL) |
332 | return 4; |
333 | else |
334 | return 0; |
335 | } |
336 | } |
337 | |
338 | /* |
339 | * Read from hardware the residue of a cyclic dmaengine transaction. |
340 | */ |
341 | static u32 admac_cyclic_read_residue(struct admac_data *ad, int channo, |
342 | struct admac_tx *adtx) |
343 | { |
344 | u32 ring1, ring2; |
345 | u32 residue1, residue2; |
346 | int nreports; |
347 | size_t pos; |
348 | |
349 | ring1 = readl_relaxed(ad->base + REG_REPORT_RING(channo)); |
350 | residue1 = readl_relaxed(ad->base + REG_RESIDUE(channo)); |
351 | ring2 = readl_relaxed(ad->base + REG_REPORT_RING(channo)); |
352 | residue2 = readl_relaxed(ad->base + REG_RESIDUE(channo)); |
353 | |
354 | if (residue2 > residue1) { |
355 | /* |
356 | * Controller must have loaded next descriptor between |
357 | * the two residue reads |
358 | */ |
359 | nreports = admac_ring_noccupied_slots(ringval: ring1) + 1; |
360 | } else { |
361 | /* No descriptor load between the two reads, ring2 is safe to use */ |
362 | nreports = admac_ring_noccupied_slots(ringval: ring2); |
363 | } |
364 | |
365 | pos = adtx->reclaimed_pos + adtx->period_len * (nreports + 1) - residue2; |
366 | |
367 | return adtx->buf_len - pos % adtx->buf_len; |
368 | } |
369 | |
370 | static enum dma_status admac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, |
371 | struct dma_tx_state *txstate) |
372 | { |
373 | struct admac_chan *adchan = to_admac_chan(chan); |
374 | struct admac_data *ad = adchan->host; |
375 | struct admac_tx *adtx; |
376 | |
377 | enum dma_status ret; |
378 | size_t residue; |
379 | unsigned long flags; |
380 | |
381 | ret = dma_cookie_status(chan, cookie, state: txstate); |
382 | if (ret == DMA_COMPLETE || !txstate) |
383 | return ret; |
384 | |
385 | spin_lock_irqsave(&adchan->lock, flags); |
386 | adtx = adchan->current_tx; |
387 | |
388 | if (adtx && adtx->tx.cookie == cookie) { |
389 | ret = DMA_IN_PROGRESS; |
390 | residue = admac_cyclic_read_residue(ad, channo: adchan->no, adtx); |
391 | } else { |
392 | ret = DMA_IN_PROGRESS; |
393 | residue = 0; |
394 | list_for_each_entry(adtx, &adchan->issued, node) { |
395 | if (adtx->tx.cookie == cookie) { |
396 | residue = adtx->buf_len; |
397 | break; |
398 | } |
399 | } |
400 | } |
401 | spin_unlock_irqrestore(lock: &adchan->lock, flags); |
402 | |
403 | dma_set_residue(state: txstate, residue); |
404 | return ret; |
405 | } |
406 | |
407 | static void admac_start_chan(struct admac_chan *adchan) |
408 | { |
409 | struct admac_data *ad = adchan->host; |
410 | u32 startbit = 1 << (adchan->no / 2); |
411 | |
412 | writel_relaxed(STATUS_DESC_DONE | STATUS_ERR, |
413 | ad->base + REG_CHAN_INTSTATUS(adchan->no, ad->irq_index)); |
414 | writel_relaxed(STATUS_DESC_DONE | STATUS_ERR, |
415 | ad->base + REG_CHAN_INTMASK(adchan->no, ad->irq_index)); |
416 | |
417 | switch (admac_chan_direction(channo: adchan->no)) { |
418 | case DMA_MEM_TO_DEV: |
419 | writel_relaxed(startbit, ad->base + REG_TX_START); |
420 | break; |
421 | case DMA_DEV_TO_MEM: |
422 | writel_relaxed(startbit, ad->base + REG_RX_START); |
423 | break; |
424 | default: |
425 | break; |
426 | } |
427 | dev_dbg(adchan->host->dev, "ch%d start\n" , adchan->no); |
428 | } |
429 | |
430 | static void admac_stop_chan(struct admac_chan *adchan) |
431 | { |
432 | struct admac_data *ad = adchan->host; |
433 | u32 stopbit = 1 << (adchan->no / 2); |
434 | |
435 | switch (admac_chan_direction(channo: adchan->no)) { |
436 | case DMA_MEM_TO_DEV: |
437 | writel_relaxed(stopbit, ad->base + REG_TX_STOP); |
438 | break; |
439 | case DMA_DEV_TO_MEM: |
440 | writel_relaxed(stopbit, ad->base + REG_RX_STOP); |
441 | break; |
442 | default: |
443 | break; |
444 | } |
445 | dev_dbg(adchan->host->dev, "ch%d stop\n" , adchan->no); |
446 | } |
447 | |
448 | static void admac_reset_rings(struct admac_chan *adchan) |
449 | { |
450 | struct admac_data *ad = adchan->host; |
451 | |
452 | writel_relaxed(REG_CHAN_CTL_RST_RINGS, |
453 | ad->base + REG_CHAN_CTL(adchan->no)); |
454 | writel_relaxed(0, ad->base + REG_CHAN_CTL(adchan->no)); |
455 | } |
456 | |
457 | static void admac_start_current_tx(struct admac_chan *adchan) |
458 | { |
459 | struct admac_data *ad = adchan->host; |
460 | int ch = adchan->no; |
461 | |
462 | admac_reset_rings(adchan); |
463 | writel_relaxed(0, ad->base + REG_CHAN_CTL(ch)); |
464 | |
465 | admac_cyclic_write_one_desc(ad, channo: ch, tx: adchan->current_tx); |
466 | admac_start_chan(adchan); |
467 | admac_cyclic_write_desc(ad, channo: ch, tx: adchan->current_tx); |
468 | } |
469 | |
470 | static void admac_issue_pending(struct dma_chan *chan) |
471 | { |
472 | struct admac_chan *adchan = to_admac_chan(chan); |
473 | struct admac_tx *tx; |
474 | unsigned long flags; |
475 | |
476 | spin_lock_irqsave(&adchan->lock, flags); |
477 | list_splice_tail_init(list: &adchan->submitted, head: &adchan->issued); |
478 | if (!list_empty(head: &adchan->issued) && !adchan->current_tx) { |
479 | tx = list_first_entry(&adchan->issued, struct admac_tx, node); |
480 | list_del(entry: &tx->node); |
481 | |
482 | adchan->current_tx = tx; |
483 | adchan->nperiod_acks = 0; |
484 | admac_start_current_tx(adchan); |
485 | } |
486 | spin_unlock_irqrestore(lock: &adchan->lock, flags); |
487 | } |
488 | |
489 | static int admac_pause(struct dma_chan *chan) |
490 | { |
491 | struct admac_chan *adchan = to_admac_chan(chan); |
492 | |
493 | admac_stop_chan(adchan); |
494 | |
495 | return 0; |
496 | } |
497 | |
498 | static int admac_resume(struct dma_chan *chan) |
499 | { |
500 | struct admac_chan *adchan = to_admac_chan(chan); |
501 | |
502 | admac_start_chan(adchan); |
503 | |
504 | return 0; |
505 | } |
506 | |
507 | static int admac_terminate_all(struct dma_chan *chan) |
508 | { |
509 | struct admac_chan *adchan = to_admac_chan(chan); |
510 | unsigned long flags; |
511 | |
512 | spin_lock_irqsave(&adchan->lock, flags); |
513 | admac_stop_chan(adchan); |
514 | admac_reset_rings(adchan); |
515 | |
516 | if (adchan->current_tx) { |
517 | list_add_tail(new: &adchan->current_tx->node, head: &adchan->to_free); |
518 | adchan->current_tx = NULL; |
519 | } |
520 | /* |
521 | * Descriptors can only be freed after the tasklet |
522 | * has been killed (in admac_synchronize). |
523 | */ |
524 | list_splice_tail_init(list: &adchan->submitted, head: &adchan->to_free); |
525 | list_splice_tail_init(list: &adchan->issued, head: &adchan->to_free); |
526 | spin_unlock_irqrestore(lock: &adchan->lock, flags); |
527 | |
528 | return 0; |
529 | } |
530 | |
531 | static void admac_synchronize(struct dma_chan *chan) |
532 | { |
533 | struct admac_chan *adchan = to_admac_chan(chan); |
534 | struct admac_tx *adtx, *_adtx; |
535 | unsigned long flags; |
536 | LIST_HEAD(head); |
537 | |
538 | spin_lock_irqsave(&adchan->lock, flags); |
539 | list_splice_tail_init(list: &adchan->to_free, head: &head); |
540 | spin_unlock_irqrestore(lock: &adchan->lock, flags); |
541 | |
542 | tasklet_kill(t: &adchan->tasklet); |
543 | |
544 | list_for_each_entry_safe(adtx, _adtx, &head, node) { |
545 | list_del(entry: &adtx->node); |
546 | admac_desc_free(tx: &adtx->tx); |
547 | } |
548 | } |
549 | |
550 | static int admac_alloc_chan_resources(struct dma_chan *chan) |
551 | { |
552 | struct admac_chan *adchan = to_admac_chan(chan); |
553 | struct admac_data *ad = adchan->host; |
554 | int ret; |
555 | |
556 | dma_cookie_init(chan: &adchan->chan); |
557 | ret = admac_alloc_sram_carveout(ad, dir: admac_chan_direction(channo: adchan->no), |
558 | out: &adchan->carveout); |
559 | if (ret < 0) |
560 | return ret; |
561 | |
562 | writel_relaxed(adchan->carveout, |
563 | ad->base + REG_CHAN_SRAM_CARVEOUT(adchan->no)); |
564 | return 0; |
565 | } |
566 | |
567 | static void admac_free_chan_resources(struct dma_chan *chan) |
568 | { |
569 | struct admac_chan *adchan = to_admac_chan(chan); |
570 | |
571 | admac_terminate_all(chan); |
572 | admac_synchronize(chan); |
573 | admac_free_sram_carveout(ad: adchan->host, dir: admac_chan_direction(channo: adchan->no), |
574 | carveout: adchan->carveout); |
575 | } |
576 | |
577 | static struct dma_chan *admac_dma_of_xlate(struct of_phandle_args *dma_spec, |
578 | struct of_dma *ofdma) |
579 | { |
580 | struct admac_data *ad = (struct admac_data *) ofdma->of_dma_data; |
581 | unsigned int index; |
582 | |
583 | if (dma_spec->args_count != 1) |
584 | return NULL; |
585 | |
586 | index = dma_spec->args[0]; |
587 | |
588 | if (index >= ad->nchannels) { |
589 | dev_err(ad->dev, "channel index %u out of bounds\n" , index); |
590 | return NULL; |
591 | } |
592 | |
593 | return dma_get_slave_channel(chan: &ad->channels[index].chan); |
594 | } |
595 | |
596 | static int admac_drain_reports(struct admac_data *ad, int channo) |
597 | { |
598 | int count; |
599 | |
600 | for (count = 0; count < 4; count++) { |
601 | u32 countval_hi, countval_lo, unk1, flags; |
602 | |
603 | if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_EMPTY) |
604 | break; |
605 | |
606 | countval_lo = readl_relaxed(ad->base + REG_REPORT_READ(channo)); |
607 | countval_hi = readl_relaxed(ad->base + REG_REPORT_READ(channo)); |
608 | unk1 = readl_relaxed(ad->base + REG_REPORT_READ(channo)); |
609 | flags = readl_relaxed(ad->base + REG_REPORT_READ(channo)); |
610 | |
611 | dev_dbg(ad->dev, "ch%d report: countval=0x%llx unk1=0x%x flags=0x%x\n" , |
612 | channo, ((u64) countval_hi) << 32 | countval_lo, unk1, flags); |
613 | } |
614 | |
615 | return count; |
616 | } |
617 | |
618 | static void admac_handle_status_err(struct admac_data *ad, int channo) |
619 | { |
620 | bool handled = false; |
621 | |
622 | if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_ERR) { |
623 | writel_relaxed(RING_ERR, ad->base + REG_DESC_RING(channo)); |
624 | dev_err_ratelimited(ad->dev, "ch%d descriptor ring error\n" , channo); |
625 | handled = true; |
626 | } |
627 | |
628 | if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_ERR) { |
629 | writel_relaxed(RING_ERR, ad->base + REG_REPORT_RING(channo)); |
630 | dev_err_ratelimited(ad->dev, "ch%d report ring error\n" , channo); |
631 | handled = true; |
632 | } |
633 | |
634 | if (unlikely(!handled)) { |
635 | dev_err(ad->dev, "ch%d unknown error, masking errors as cause of IRQs\n" , channo); |
636 | admac_modify(ad, REG_CHAN_INTMASK(channo, ad->irq_index), |
637 | STATUS_ERR, val: 0); |
638 | } |
639 | } |
640 | |
641 | static void admac_handle_status_desc_done(struct admac_data *ad, int channo) |
642 | { |
643 | struct admac_chan *adchan = &ad->channels[channo]; |
644 | unsigned long flags; |
645 | int nreports; |
646 | |
647 | writel_relaxed(STATUS_DESC_DONE, |
648 | ad->base + REG_CHAN_INTSTATUS(channo, ad->irq_index)); |
649 | |
650 | spin_lock_irqsave(&adchan->lock, flags); |
651 | nreports = admac_drain_reports(ad, channo); |
652 | |
653 | if (adchan->current_tx) { |
654 | struct admac_tx *tx = adchan->current_tx; |
655 | |
656 | adchan->nperiod_acks += nreports; |
657 | tx->reclaimed_pos += nreports * tx->period_len; |
658 | tx->reclaimed_pos %= 2 * tx->buf_len; |
659 | |
660 | admac_cyclic_write_desc(ad, channo, tx); |
661 | tasklet_schedule(t: &adchan->tasklet); |
662 | } |
663 | spin_unlock_irqrestore(lock: &adchan->lock, flags); |
664 | } |
665 | |
666 | static void admac_handle_chan_int(struct admac_data *ad, int no) |
667 | { |
668 | u32 cause = readl_relaxed(ad->base + REG_CHAN_INTSTATUS(no, ad->irq_index)); |
669 | |
670 | if (cause & STATUS_ERR) |
671 | admac_handle_status_err(ad, channo: no); |
672 | |
673 | if (cause & STATUS_DESC_DONE) |
674 | admac_handle_status_desc_done(ad, channo: no); |
675 | } |
676 | |
677 | static irqreturn_t admac_interrupt(int irq, void *devid) |
678 | { |
679 | struct admac_data *ad = devid; |
680 | u32 rx_intstate, tx_intstate, global_intstate; |
681 | int i; |
682 | |
683 | rx_intstate = readl_relaxed(ad->base + REG_RX_INTSTATE(ad->irq_index)); |
684 | tx_intstate = readl_relaxed(ad->base + REG_TX_INTSTATE(ad->irq_index)); |
685 | global_intstate = readl_relaxed(ad->base + REG_GLOBAL_INTSTATE(ad->irq_index)); |
686 | |
687 | if (!tx_intstate && !rx_intstate && !global_intstate) |
688 | return IRQ_NONE; |
689 | |
690 | for (i = 0; i < ad->nchannels; i += 2) { |
691 | if (tx_intstate & 1) |
692 | admac_handle_chan_int(ad, no: i); |
693 | tx_intstate >>= 1; |
694 | } |
695 | |
696 | for (i = 1; i < ad->nchannels; i += 2) { |
697 | if (rx_intstate & 1) |
698 | admac_handle_chan_int(ad, no: i); |
699 | rx_intstate >>= 1; |
700 | } |
701 | |
702 | if (global_intstate) { |
703 | dev_warn(ad->dev, "clearing unknown global interrupt flag: %x\n" , |
704 | global_intstate); |
705 | writel_relaxed(~(u32) 0, ad->base + REG_GLOBAL_INTSTATE(ad->irq_index)); |
706 | } |
707 | |
708 | return IRQ_HANDLED; |
709 | } |
710 | |
711 | static void admac_chan_tasklet(struct tasklet_struct *t) |
712 | { |
713 | struct admac_chan *adchan = from_tasklet(adchan, t, tasklet); |
714 | struct admac_tx *adtx; |
715 | struct dmaengine_desc_callback cb; |
716 | struct dmaengine_result tx_result; |
717 | int nacks; |
718 | |
719 | spin_lock_irq(lock: &adchan->lock); |
720 | adtx = adchan->current_tx; |
721 | nacks = adchan->nperiod_acks; |
722 | adchan->nperiod_acks = 0; |
723 | spin_unlock_irq(lock: &adchan->lock); |
724 | |
725 | if (!adtx || !nacks) |
726 | return; |
727 | |
728 | tx_result.result = DMA_TRANS_NOERROR; |
729 | tx_result.residue = 0; |
730 | |
731 | dmaengine_desc_get_callback(tx: &adtx->tx, cb: &cb); |
732 | while (nacks--) |
733 | dmaengine_desc_callback_invoke(cb: &cb, result: &tx_result); |
734 | } |
735 | |
736 | static int admac_device_config(struct dma_chan *chan, |
737 | struct dma_slave_config *config) |
738 | { |
739 | struct admac_chan *adchan = to_admac_chan(chan); |
740 | struct admac_data *ad = adchan->host; |
741 | bool is_tx = admac_chan_direction(channo: adchan->no) == DMA_MEM_TO_DEV; |
742 | int wordsize = 0; |
743 | u32 bus_width = 0; |
744 | |
745 | switch (is_tx ? config->dst_addr_width : config->src_addr_width) { |
746 | case DMA_SLAVE_BUSWIDTH_1_BYTE: |
747 | wordsize = 1; |
748 | bus_width |= BUS_WIDTH_8BIT; |
749 | break; |
750 | case DMA_SLAVE_BUSWIDTH_2_BYTES: |
751 | wordsize = 2; |
752 | bus_width |= BUS_WIDTH_16BIT; |
753 | break; |
754 | case DMA_SLAVE_BUSWIDTH_4_BYTES: |
755 | wordsize = 4; |
756 | bus_width |= BUS_WIDTH_32BIT; |
757 | break; |
758 | default: |
759 | return -EINVAL; |
760 | } |
761 | |
762 | /* |
763 | * We take port_window_size to be the number of words in a frame. |
764 | * |
765 | * The controller has some means of out-of-band signalling, to the peripheral, |
766 | * of words position in a frame. That's where the importance of this control |
767 | * comes from. |
768 | */ |
769 | switch (is_tx ? config->dst_port_window_size : config->src_port_window_size) { |
770 | case 0 ... 1: |
771 | break; |
772 | case 2: |
773 | bus_width |= BUS_WIDTH_FRAME_2_WORDS; |
774 | break; |
775 | case 4: |
776 | bus_width |= BUS_WIDTH_FRAME_4_WORDS; |
777 | break; |
778 | default: |
779 | return -EINVAL; |
780 | } |
781 | |
782 | writel_relaxed(bus_width, ad->base + REG_BUS_WIDTH(adchan->no)); |
783 | |
784 | /* |
785 | * By FIFOCTL_LIMIT we seem to set the maximal number of bytes allowed to be |
786 | * held in controller's per-channel FIFO. Transfers seem to be triggered |
787 | * around the time FIFO occupancy touches FIFOCTL_THRESHOLD. |
788 | * |
789 | * The numbers we set are more or less arbitrary. |
790 | */ |
791 | writel_relaxed(FIELD_PREP(CHAN_FIFOCTL_LIMIT, 0x30 * wordsize) |
792 | | FIELD_PREP(CHAN_FIFOCTL_THRESHOLD, 0x18 * wordsize), |
793 | ad->base + REG_CHAN_FIFOCTL(adchan->no)); |
794 | |
795 | return 0; |
796 | } |
797 | |
798 | static int admac_probe(struct platform_device *pdev) |
799 | { |
800 | struct device_node *np = pdev->dev.of_node; |
801 | struct admac_data *ad; |
802 | struct dma_device *dma; |
803 | int nchannels; |
804 | int err, irq, i; |
805 | |
806 | err = of_property_read_u32(np, propname: "dma-channels" , out_value: &nchannels); |
807 | if (err || nchannels > NCHANNELS_MAX) { |
808 | dev_err(&pdev->dev, "missing or invalid dma-channels property\n" ); |
809 | return -EINVAL; |
810 | } |
811 | |
812 | ad = devm_kzalloc(dev: &pdev->dev, struct_size(ad, channels, nchannels), GFP_KERNEL); |
813 | if (!ad) |
814 | return -ENOMEM; |
815 | |
816 | platform_set_drvdata(pdev, data: ad); |
817 | ad->dev = &pdev->dev; |
818 | ad->nchannels = nchannels; |
819 | mutex_init(&ad->cache_alloc_lock); |
820 | |
821 | /* |
822 | * The controller has 4 IRQ outputs. Try them all until |
823 | * we find one we can use. |
824 | */ |
825 | for (i = 0; i < IRQ_NOUTPUTS; i++) { |
826 | irq = platform_get_irq_optional(pdev, i); |
827 | if (irq >= 0) { |
828 | ad->irq_index = i; |
829 | break; |
830 | } |
831 | } |
832 | |
833 | if (irq < 0) |
834 | return dev_err_probe(dev: &pdev->dev, err: irq, fmt: "no usable interrupt\n" ); |
835 | ad->irq = irq; |
836 | |
837 | ad->base = devm_platform_ioremap_resource(pdev, index: 0); |
838 | if (IS_ERR(ptr: ad->base)) |
839 | return dev_err_probe(dev: &pdev->dev, err: PTR_ERR(ptr: ad->base), |
840 | fmt: "unable to obtain MMIO resource\n" ); |
841 | |
842 | ad->rstc = devm_reset_control_get_optional_shared(dev: &pdev->dev, NULL); |
843 | if (IS_ERR(ptr: ad->rstc)) |
844 | return PTR_ERR(ptr: ad->rstc); |
845 | |
846 | dma = &ad->dma; |
847 | |
848 | dma_cap_set(DMA_PRIVATE, dma->cap_mask); |
849 | dma_cap_set(DMA_CYCLIC, dma->cap_mask); |
850 | |
851 | dma->dev = &pdev->dev; |
852 | dma->device_alloc_chan_resources = admac_alloc_chan_resources; |
853 | dma->device_free_chan_resources = admac_free_chan_resources; |
854 | dma->device_tx_status = admac_tx_status; |
855 | dma->device_issue_pending = admac_issue_pending; |
856 | dma->device_terminate_all = admac_terminate_all; |
857 | dma->device_synchronize = admac_synchronize; |
858 | dma->device_prep_dma_cyclic = admac_prep_dma_cyclic; |
859 | dma->device_config = admac_device_config; |
860 | dma->device_pause = admac_pause; |
861 | dma->device_resume = admac_resume; |
862 | |
863 | dma->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); |
864 | dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; |
865 | dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | |
866 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | |
867 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
868 | dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | |
869 | BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | |
870 | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
871 | |
872 | INIT_LIST_HEAD(list: &dma->channels); |
873 | for (i = 0; i < nchannels; i++) { |
874 | struct admac_chan *adchan = &ad->channels[i]; |
875 | |
876 | adchan->host = ad; |
877 | adchan->no = i; |
878 | adchan->chan.device = &ad->dma; |
879 | spin_lock_init(&adchan->lock); |
880 | INIT_LIST_HEAD(list: &adchan->submitted); |
881 | INIT_LIST_HEAD(list: &adchan->issued); |
882 | INIT_LIST_HEAD(list: &adchan->to_free); |
883 | list_add_tail(new: &adchan->chan.device_node, head: &dma->channels); |
884 | tasklet_setup(t: &adchan->tasklet, callback: admac_chan_tasklet); |
885 | } |
886 | |
887 | err = reset_control_reset(rstc: ad->rstc); |
888 | if (err) |
889 | return dev_err_probe(dev: &pdev->dev, err, |
890 | fmt: "unable to trigger reset\n" ); |
891 | |
892 | err = request_irq(irq, handler: admac_interrupt, flags: 0, name: dev_name(dev: &pdev->dev), dev: ad); |
893 | if (err) { |
894 | dev_err_probe(dev: &pdev->dev, err, |
895 | fmt: "unable to register interrupt\n" ); |
896 | goto free_reset; |
897 | } |
898 | |
899 | err = dma_async_device_register(device: &ad->dma); |
900 | if (err) { |
901 | dev_err_probe(dev: &pdev->dev, err, fmt: "failed to register DMA device\n" ); |
902 | goto free_irq; |
903 | } |
904 | |
905 | err = of_dma_controller_register(np: pdev->dev.of_node, of_dma_xlate: admac_dma_of_xlate, data: ad); |
906 | if (err) { |
907 | dma_async_device_unregister(device: &ad->dma); |
908 | dev_err_probe(dev: &pdev->dev, err, fmt: "failed to register with OF\n" ); |
909 | goto free_irq; |
910 | } |
911 | |
912 | ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE); |
913 | ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE); |
914 | |
915 | dev_info(&pdev->dev, "Audio DMA Controller\n" ); |
916 | dev_info(&pdev->dev, "imprint %x TX cache %u RX cache %u\n" , |
917 | readl_relaxed(ad->base + REG_IMPRINT), ad->txcache.size, ad->rxcache.size); |
918 | |
919 | return 0; |
920 | |
921 | free_irq: |
922 | free_irq(ad->irq, ad); |
923 | free_reset: |
924 | reset_control_rearm(rstc: ad->rstc); |
925 | return err; |
926 | } |
927 | |
928 | static void admac_remove(struct platform_device *pdev) |
929 | { |
930 | struct admac_data *ad = platform_get_drvdata(pdev); |
931 | |
932 | of_dma_controller_free(np: pdev->dev.of_node); |
933 | dma_async_device_unregister(device: &ad->dma); |
934 | free_irq(ad->irq, ad); |
935 | reset_control_rearm(rstc: ad->rstc); |
936 | } |
937 | |
938 | static const struct of_device_id admac_of_match[] = { |
939 | { .compatible = "apple,admac" , }, |
940 | { } |
941 | }; |
942 | MODULE_DEVICE_TABLE(of, admac_of_match); |
943 | |
944 | static struct platform_driver apple_admac_driver = { |
945 | .driver = { |
946 | .name = "apple-admac" , |
947 | .of_match_table = admac_of_match, |
948 | }, |
949 | .probe = admac_probe, |
950 | .remove_new = admac_remove, |
951 | }; |
952 | module_platform_driver(apple_admac_driver); |
953 | |
954 | MODULE_AUTHOR("Martin Povišer <povik+lin@cutebit.org>" ); |
955 | MODULE_DESCRIPTION("Driver for Audio DMA Controller (ADMAC) on Apple SoCs" ); |
956 | MODULE_LICENSE("GPL" ); |
957 | |