1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * ADMA driver for Nvidia's Tegra210 ADMA controller. |
4 | * |
5 | * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. |
6 | */ |
7 | |
8 | #include <linux/clk.h> |
9 | #include <linux/iopoll.h> |
10 | #include <linux/module.h> |
11 | #include <linux/of.h> |
12 | #include <linux/of_dma.h> |
13 | #include <linux/of_irq.h> |
14 | #include <linux/platform_device.h> |
15 | #include <linux/pm_runtime.h> |
16 | #include <linux/slab.h> |
17 | |
18 | #include "virt-dma.h" |
19 | |
20 | #define ADMA_CH_CMD 0x00 |
21 | #define ADMA_CH_STATUS 0x0c |
22 | #define ADMA_CH_STATUS_XFER_EN BIT(0) |
23 | #define ADMA_CH_STATUS_XFER_PAUSED BIT(1) |
24 | |
25 | #define ADMA_CH_INT_STATUS 0x10 |
26 | #define ADMA_CH_INT_STATUS_XFER_DONE BIT(0) |
27 | |
28 | #define ADMA_CH_INT_CLEAR 0x1c |
29 | #define ADMA_CH_CTRL 0x24 |
30 | #define ADMA_CH_CTRL_DIR(val) (((val) & 0xf) << 12) |
31 | #define ADMA_CH_CTRL_DIR_AHUB2MEM 2 |
32 | #define ADMA_CH_CTRL_DIR_MEM2AHUB 4 |
33 | #define ADMA_CH_CTRL_MODE_CONTINUOUS (2 << 8) |
34 | #define ADMA_CH_CTRL_FLOWCTRL_EN BIT(1) |
35 | #define ADMA_CH_CTRL_XFER_PAUSE_SHIFT 0 |
36 | |
37 | #define ADMA_CH_CONFIG 0x28 |
38 | #define ADMA_CH_CONFIG_SRC_BUF(val) (((val) & 0x7) << 28) |
39 | #define ADMA_CH_CONFIG_TRG_BUF(val) (((val) & 0x7) << 24) |
40 | #define ADMA_CH_CONFIG_BURST_SIZE_SHIFT 20 |
41 | #define ADMA_CH_CONFIG_MAX_BURST_SIZE 16 |
42 | #define ADMA_CH_CONFIG_WEIGHT_FOR_WRR(val) ((val) & 0xf) |
43 | #define ADMA_CH_CONFIG_MAX_BUFS 8 |
44 | #define TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(reqs) (reqs << 4) |
45 | |
46 | #define ADMA_CH_FIFO_CTRL 0x2c |
47 | #define ADMA_CH_TX_FIFO_SIZE_SHIFT 8 |
48 | #define ADMA_CH_RX_FIFO_SIZE_SHIFT 0 |
49 | |
50 | #define ADMA_CH_LOWER_SRC_ADDR 0x34 |
51 | #define ADMA_CH_LOWER_TRG_ADDR 0x3c |
52 | #define ADMA_CH_TC 0x44 |
53 | #define ADMA_CH_TC_COUNT_MASK 0x3ffffffc |
54 | |
55 | #define ADMA_CH_XFER_STATUS 0x54 |
56 | #define ADMA_CH_XFER_STATUS_COUNT_MASK 0xffff |
57 | |
58 | #define ADMA_GLOBAL_CMD 0x00 |
59 | #define ADMA_GLOBAL_SOFT_RESET 0x04 |
60 | |
61 | #define TEGRA_ADMA_BURST_COMPLETE_TIME 20 |
62 | |
63 | #define ADMA_CH_REG_FIELD_VAL(val, mask, shift) (((val) & mask) << shift) |
64 | |
65 | struct tegra_adma; |
66 | |
67 | /* |
68 | * struct tegra_adma_chip_data - Tegra chip specific data |
69 | * @adma_get_burst_config: Function callback used to set DMA burst size. |
70 | * @global_reg_offset: Register offset of DMA global register. |
71 | * @global_int_clear: Register offset of DMA global interrupt clear. |
72 | * @ch_req_tx_shift: Register offset for AHUB transmit channel select. |
73 | * @ch_req_rx_shift: Register offset for AHUB receive channel select. |
74 | * @ch_base_offset: Register offset of DMA channel registers. |
75 | * @ch_fifo_ctrl: Default value for channel FIFO CTRL register. |
76 | * @ch_req_mask: Mask for Tx or Rx channel select. |
77 | * @ch_req_max: Maximum number of Tx or Rx channels available. |
78 | * @ch_reg_size: Size of DMA channel register space. |
79 | * @nr_channels: Number of DMA channels available. |
80 | * @ch_fifo_size_mask: Mask for FIFO size field. |
81 | * @sreq_index_offset: Slave channel index offset. |
82 | * @has_outstanding_reqs: If DMA channel can have outstanding requests. |
83 | */ |
84 | struct tegra_adma_chip_data { |
85 | unsigned int (*adma_get_burst_config)(unsigned int burst_size); |
86 | unsigned int global_reg_offset; |
87 | unsigned int global_int_clear; |
88 | unsigned int ch_req_tx_shift; |
89 | unsigned int ch_req_rx_shift; |
90 | unsigned int ch_base_offset; |
91 | unsigned int ch_fifo_ctrl; |
92 | unsigned int ch_req_mask; |
93 | unsigned int ch_req_max; |
94 | unsigned int ch_reg_size; |
95 | unsigned int nr_channels; |
96 | unsigned int ch_fifo_size_mask; |
97 | unsigned int sreq_index_offset; |
98 | bool has_outstanding_reqs; |
99 | }; |
100 | |
101 | /* |
102 | * struct tegra_adma_chan_regs - Tegra ADMA channel registers |
103 | */ |
104 | struct tegra_adma_chan_regs { |
105 | unsigned int ctrl; |
106 | unsigned int config; |
107 | unsigned int src_addr; |
108 | unsigned int trg_addr; |
109 | unsigned int fifo_ctrl; |
110 | unsigned int cmd; |
111 | unsigned int tc; |
112 | }; |
113 | |
114 | /* |
115 | * struct tegra_adma_desc - Tegra ADMA descriptor to manage transfer requests. |
116 | */ |
117 | struct tegra_adma_desc { |
118 | struct virt_dma_desc vd; |
119 | struct tegra_adma_chan_regs ch_regs; |
120 | size_t buf_len; |
121 | size_t period_len; |
122 | size_t num_periods; |
123 | }; |
124 | |
125 | /* |
126 | * struct tegra_adma_chan - Tegra ADMA channel information |
127 | */ |
128 | struct tegra_adma_chan { |
129 | struct virt_dma_chan vc; |
130 | struct tegra_adma_desc *desc; |
131 | struct tegra_adma *tdma; |
132 | int irq; |
133 | void __iomem *chan_addr; |
134 | |
135 | /* Slave channel configuration info */ |
136 | struct dma_slave_config sconfig; |
137 | enum dma_transfer_direction sreq_dir; |
138 | unsigned int sreq_index; |
139 | bool sreq_reserved; |
140 | struct tegra_adma_chan_regs ch_regs; |
141 | |
142 | /* Transfer count and position info */ |
143 | unsigned int tx_buf_count; |
144 | unsigned int tx_buf_pos; |
145 | }; |
146 | |
147 | /* |
148 | * struct tegra_adma - Tegra ADMA controller information |
149 | */ |
150 | struct tegra_adma { |
151 | struct dma_device dma_dev; |
152 | struct device *dev; |
153 | void __iomem *base_addr; |
154 | struct clk *ahub_clk; |
155 | unsigned int nr_channels; |
156 | unsigned long rx_requests_reserved; |
157 | unsigned long tx_requests_reserved; |
158 | |
159 | /* Used to store global command register state when suspending */ |
160 | unsigned int global_cmd; |
161 | |
162 | const struct tegra_adma_chip_data *cdata; |
163 | |
164 | /* Last member of the structure */ |
165 | struct tegra_adma_chan channels[] __counted_by(nr_channels); |
166 | }; |
167 | |
168 | static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val) |
169 | { |
170 | writel(val, addr: tdma->base_addr + tdma->cdata->global_reg_offset + reg); |
171 | } |
172 | |
173 | static inline u32 tdma_read(struct tegra_adma *tdma, u32 reg) |
174 | { |
175 | return readl(addr: tdma->base_addr + tdma->cdata->global_reg_offset + reg); |
176 | } |
177 | |
178 | static inline void tdma_ch_write(struct tegra_adma_chan *tdc, u32 reg, u32 val) |
179 | { |
180 | writel(val, addr: tdc->chan_addr + reg); |
181 | } |
182 | |
183 | static inline u32 tdma_ch_read(struct tegra_adma_chan *tdc, u32 reg) |
184 | { |
185 | return readl(addr: tdc->chan_addr + reg); |
186 | } |
187 | |
188 | static inline struct tegra_adma_chan *to_tegra_adma_chan(struct dma_chan *dc) |
189 | { |
190 | return container_of(dc, struct tegra_adma_chan, vc.chan); |
191 | } |
192 | |
193 | static inline struct tegra_adma_desc *to_tegra_adma_desc( |
194 | struct dma_async_tx_descriptor *td) |
195 | { |
196 | return container_of(td, struct tegra_adma_desc, vd.tx); |
197 | } |
198 | |
199 | static inline struct device *tdc2dev(struct tegra_adma_chan *tdc) |
200 | { |
201 | return tdc->tdma->dev; |
202 | } |
203 | |
204 | static void tegra_adma_desc_free(struct virt_dma_desc *vd) |
205 | { |
206 | kfree(container_of(vd, struct tegra_adma_desc, vd)); |
207 | } |
208 | |
209 | static int tegra_adma_slave_config(struct dma_chan *dc, |
210 | struct dma_slave_config *sconfig) |
211 | { |
212 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
213 | |
214 | memcpy(&tdc->sconfig, sconfig, sizeof(*sconfig)); |
215 | |
216 | return 0; |
217 | } |
218 | |
219 | static int tegra_adma_init(struct tegra_adma *tdma) |
220 | { |
221 | u32 status; |
222 | int ret; |
223 | |
224 | /* Clear any interrupts */ |
225 | tdma_write(tdma, reg: tdma->cdata->ch_base_offset + tdma->cdata->global_int_clear, val: 0x1); |
226 | |
227 | /* Assert soft reset */ |
228 | tdma_write(tdma, ADMA_GLOBAL_SOFT_RESET, val: 0x1); |
229 | |
230 | /* Wait for reset to clear */ |
231 | ret = readx_poll_timeout(readl, |
232 | tdma->base_addr + |
233 | tdma->cdata->global_reg_offset + |
234 | ADMA_GLOBAL_SOFT_RESET, |
235 | status, status == 0, 20, 10000); |
236 | if (ret) |
237 | return ret; |
238 | |
239 | /* Enable global ADMA registers */ |
240 | tdma_write(tdma, ADMA_GLOBAL_CMD, val: 1); |
241 | |
242 | return 0; |
243 | } |
244 | |
245 | static int tegra_adma_request_alloc(struct tegra_adma_chan *tdc, |
246 | enum dma_transfer_direction direction) |
247 | { |
248 | struct tegra_adma *tdma = tdc->tdma; |
249 | unsigned int sreq_index = tdc->sreq_index; |
250 | |
251 | if (tdc->sreq_reserved) |
252 | return tdc->sreq_dir == direction ? 0 : -EINVAL; |
253 | |
254 | if (sreq_index > tdma->cdata->ch_req_max) { |
255 | dev_err(tdma->dev, "invalid DMA request\n" ); |
256 | return -EINVAL; |
257 | } |
258 | |
259 | switch (direction) { |
260 | case DMA_MEM_TO_DEV: |
261 | if (test_and_set_bit(nr: sreq_index, addr: &tdma->tx_requests_reserved)) { |
262 | dev_err(tdma->dev, "DMA request reserved\n" ); |
263 | return -EINVAL; |
264 | } |
265 | break; |
266 | |
267 | case DMA_DEV_TO_MEM: |
268 | if (test_and_set_bit(nr: sreq_index, addr: &tdma->rx_requests_reserved)) { |
269 | dev_err(tdma->dev, "DMA request reserved\n" ); |
270 | return -EINVAL; |
271 | } |
272 | break; |
273 | |
274 | default: |
275 | dev_WARN(tdma->dev, "channel %s has invalid transfer type\n" , |
276 | dma_chan_name(&tdc->vc.chan)); |
277 | return -EINVAL; |
278 | } |
279 | |
280 | tdc->sreq_dir = direction; |
281 | tdc->sreq_reserved = true; |
282 | |
283 | return 0; |
284 | } |
285 | |
286 | static void tegra_adma_request_free(struct tegra_adma_chan *tdc) |
287 | { |
288 | struct tegra_adma *tdma = tdc->tdma; |
289 | |
290 | if (!tdc->sreq_reserved) |
291 | return; |
292 | |
293 | switch (tdc->sreq_dir) { |
294 | case DMA_MEM_TO_DEV: |
295 | clear_bit(nr: tdc->sreq_index, addr: &tdma->tx_requests_reserved); |
296 | break; |
297 | |
298 | case DMA_DEV_TO_MEM: |
299 | clear_bit(nr: tdc->sreq_index, addr: &tdma->rx_requests_reserved); |
300 | break; |
301 | |
302 | default: |
303 | dev_WARN(tdma->dev, "channel %s has invalid transfer type\n" , |
304 | dma_chan_name(&tdc->vc.chan)); |
305 | return; |
306 | } |
307 | |
308 | tdc->sreq_reserved = false; |
309 | } |
310 | |
311 | static u32 tegra_adma_irq_status(struct tegra_adma_chan *tdc) |
312 | { |
313 | u32 status = tdma_ch_read(tdc, ADMA_CH_INT_STATUS); |
314 | |
315 | return status & ADMA_CH_INT_STATUS_XFER_DONE; |
316 | } |
317 | |
318 | static u32 tegra_adma_irq_clear(struct tegra_adma_chan *tdc) |
319 | { |
320 | u32 status = tegra_adma_irq_status(tdc); |
321 | |
322 | if (status) |
323 | tdma_ch_write(tdc, ADMA_CH_INT_CLEAR, val: status); |
324 | |
325 | return status; |
326 | } |
327 | |
328 | static void tegra_adma_stop(struct tegra_adma_chan *tdc) |
329 | { |
330 | unsigned int status; |
331 | |
332 | /* Disable ADMA */ |
333 | tdma_ch_write(tdc, ADMA_CH_CMD, val: 0); |
334 | |
335 | /* Clear interrupt status */ |
336 | tegra_adma_irq_clear(tdc); |
337 | |
338 | if (readx_poll_timeout_atomic(readl, tdc->chan_addr + ADMA_CH_STATUS, |
339 | status, !(status & ADMA_CH_STATUS_XFER_EN), |
340 | 20, 10000)) { |
341 | dev_err(tdc2dev(tdc), "unable to stop DMA channel\n" ); |
342 | return; |
343 | } |
344 | |
345 | kfree(objp: tdc->desc); |
346 | tdc->desc = NULL; |
347 | } |
348 | |
349 | static void tegra_adma_start(struct tegra_adma_chan *tdc) |
350 | { |
351 | struct virt_dma_desc *vd = vchan_next_desc(vc: &tdc->vc); |
352 | struct tegra_adma_chan_regs *ch_regs; |
353 | struct tegra_adma_desc *desc; |
354 | |
355 | if (!vd) |
356 | return; |
357 | |
358 | list_del(entry: &vd->node); |
359 | |
360 | desc = to_tegra_adma_desc(td: &vd->tx); |
361 | |
362 | if (!desc) { |
363 | dev_warn(tdc2dev(tdc), "unable to start DMA, no descriptor\n" ); |
364 | return; |
365 | } |
366 | |
367 | ch_regs = &desc->ch_regs; |
368 | |
369 | tdc->tx_buf_pos = 0; |
370 | tdc->tx_buf_count = 0; |
371 | tdma_ch_write(tdc, ADMA_CH_TC, val: ch_regs->tc); |
372 | tdma_ch_write(tdc, ADMA_CH_CTRL, val: ch_regs->ctrl); |
373 | tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, val: ch_regs->src_addr); |
374 | tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, val: ch_regs->trg_addr); |
375 | tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, val: ch_regs->fifo_ctrl); |
376 | tdma_ch_write(tdc, ADMA_CH_CONFIG, val: ch_regs->config); |
377 | |
378 | /* Start ADMA */ |
379 | tdma_ch_write(tdc, ADMA_CH_CMD, val: 1); |
380 | |
381 | tdc->desc = desc; |
382 | } |
383 | |
384 | static unsigned int tegra_adma_get_residue(struct tegra_adma_chan *tdc) |
385 | { |
386 | struct tegra_adma_desc *desc = tdc->desc; |
387 | unsigned int max = ADMA_CH_XFER_STATUS_COUNT_MASK + 1; |
388 | unsigned int pos = tdma_ch_read(tdc, ADMA_CH_XFER_STATUS); |
389 | unsigned int periods_remaining; |
390 | |
391 | /* |
392 | * Handle wrap around of buffer count register |
393 | */ |
394 | if (pos < tdc->tx_buf_pos) |
395 | tdc->tx_buf_count += pos + (max - tdc->tx_buf_pos); |
396 | else |
397 | tdc->tx_buf_count += pos - tdc->tx_buf_pos; |
398 | |
399 | periods_remaining = tdc->tx_buf_count % desc->num_periods; |
400 | tdc->tx_buf_pos = pos; |
401 | |
402 | return desc->buf_len - (periods_remaining * desc->period_len); |
403 | } |
404 | |
405 | static irqreturn_t tegra_adma_isr(int irq, void *dev_id) |
406 | { |
407 | struct tegra_adma_chan *tdc = dev_id; |
408 | unsigned long status; |
409 | |
410 | spin_lock(lock: &tdc->vc.lock); |
411 | |
412 | status = tegra_adma_irq_clear(tdc); |
413 | if (status == 0 || !tdc->desc) { |
414 | spin_unlock(lock: &tdc->vc.lock); |
415 | return IRQ_NONE; |
416 | } |
417 | |
418 | vchan_cyclic_callback(vd: &tdc->desc->vd); |
419 | |
420 | spin_unlock(lock: &tdc->vc.lock); |
421 | |
422 | return IRQ_HANDLED; |
423 | } |
424 | |
425 | static void tegra_adma_issue_pending(struct dma_chan *dc) |
426 | { |
427 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
428 | unsigned long flags; |
429 | |
430 | spin_lock_irqsave(&tdc->vc.lock, flags); |
431 | |
432 | if (vchan_issue_pending(vc: &tdc->vc)) { |
433 | if (!tdc->desc) |
434 | tegra_adma_start(tdc); |
435 | } |
436 | |
437 | spin_unlock_irqrestore(lock: &tdc->vc.lock, flags); |
438 | } |
439 | |
440 | static bool tegra_adma_is_paused(struct tegra_adma_chan *tdc) |
441 | { |
442 | u32 csts; |
443 | |
444 | csts = tdma_ch_read(tdc, ADMA_CH_STATUS); |
445 | csts &= ADMA_CH_STATUS_XFER_PAUSED; |
446 | |
447 | return csts ? true : false; |
448 | } |
449 | |
450 | static int tegra_adma_pause(struct dma_chan *dc) |
451 | { |
452 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
453 | struct tegra_adma_desc *desc = tdc->desc; |
454 | struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; |
455 | int dcnt = 10; |
456 | |
457 | ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); |
458 | ch_regs->ctrl |= (1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT); |
459 | tdma_ch_write(tdc, ADMA_CH_CTRL, val: ch_regs->ctrl); |
460 | |
461 | while (dcnt-- && !tegra_adma_is_paused(tdc)) |
462 | udelay(TEGRA_ADMA_BURST_COMPLETE_TIME); |
463 | |
464 | if (dcnt < 0) { |
465 | dev_err(tdc2dev(tdc), "unable to pause DMA channel\n" ); |
466 | return -EBUSY; |
467 | } |
468 | |
469 | return 0; |
470 | } |
471 | |
472 | static int tegra_adma_resume(struct dma_chan *dc) |
473 | { |
474 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
475 | struct tegra_adma_desc *desc = tdc->desc; |
476 | struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; |
477 | |
478 | ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); |
479 | ch_regs->ctrl &= ~(1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT); |
480 | tdma_ch_write(tdc, ADMA_CH_CTRL, val: ch_regs->ctrl); |
481 | |
482 | return 0; |
483 | } |
484 | |
485 | static int tegra_adma_terminate_all(struct dma_chan *dc) |
486 | { |
487 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
488 | unsigned long flags; |
489 | LIST_HEAD(head); |
490 | |
491 | spin_lock_irqsave(&tdc->vc.lock, flags); |
492 | |
493 | if (tdc->desc) |
494 | tegra_adma_stop(tdc); |
495 | |
496 | tegra_adma_request_free(tdc); |
497 | vchan_get_all_descriptors(vc: &tdc->vc, head: &head); |
498 | spin_unlock_irqrestore(lock: &tdc->vc.lock, flags); |
499 | vchan_dma_desc_free_list(vc: &tdc->vc, head: &head); |
500 | |
501 | return 0; |
502 | } |
503 | |
504 | static enum dma_status tegra_adma_tx_status(struct dma_chan *dc, |
505 | dma_cookie_t cookie, |
506 | struct dma_tx_state *txstate) |
507 | { |
508 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
509 | struct tegra_adma_desc *desc; |
510 | struct virt_dma_desc *vd; |
511 | enum dma_status ret; |
512 | unsigned long flags; |
513 | unsigned int residual; |
514 | |
515 | ret = dma_cookie_status(chan: dc, cookie, state: txstate); |
516 | if (ret == DMA_COMPLETE || !txstate) |
517 | return ret; |
518 | |
519 | spin_lock_irqsave(&tdc->vc.lock, flags); |
520 | |
521 | vd = vchan_find_desc(&tdc->vc, cookie); |
522 | if (vd) { |
523 | desc = to_tegra_adma_desc(td: &vd->tx); |
524 | residual = desc->ch_regs.tc; |
525 | } else if (tdc->desc && tdc->desc->vd.tx.cookie == cookie) { |
526 | residual = tegra_adma_get_residue(tdc); |
527 | } else { |
528 | residual = 0; |
529 | } |
530 | |
531 | spin_unlock_irqrestore(lock: &tdc->vc.lock, flags); |
532 | |
533 | dma_set_residue(state: txstate, residue: residual); |
534 | |
535 | return ret; |
536 | } |
537 | |
538 | static unsigned int tegra210_adma_get_burst_config(unsigned int burst_size) |
539 | { |
540 | if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE) |
541 | burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE; |
542 | |
543 | return fls(x: burst_size) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT; |
544 | } |
545 | |
546 | static unsigned int tegra186_adma_get_burst_config(unsigned int burst_size) |
547 | { |
548 | if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE) |
549 | burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE; |
550 | |
551 | return (burst_size - 1) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT; |
552 | } |
553 | |
554 | static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc, |
555 | struct tegra_adma_desc *desc, |
556 | dma_addr_t buf_addr, |
557 | enum dma_transfer_direction direction) |
558 | { |
559 | struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; |
560 | const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata; |
561 | unsigned int burst_size, adma_dir, fifo_size_shift; |
562 | |
563 | if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS) |
564 | return -EINVAL; |
565 | |
566 | switch (direction) { |
567 | case DMA_MEM_TO_DEV: |
568 | fifo_size_shift = ADMA_CH_TX_FIFO_SIZE_SHIFT; |
569 | adma_dir = ADMA_CH_CTRL_DIR_MEM2AHUB; |
570 | burst_size = tdc->sconfig.dst_maxburst; |
571 | ch_regs->config = ADMA_CH_CONFIG_SRC_BUF(desc->num_periods - 1); |
572 | ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index, |
573 | cdata->ch_req_mask, |
574 | cdata->ch_req_tx_shift); |
575 | ch_regs->src_addr = buf_addr; |
576 | break; |
577 | |
578 | case DMA_DEV_TO_MEM: |
579 | fifo_size_shift = ADMA_CH_RX_FIFO_SIZE_SHIFT; |
580 | adma_dir = ADMA_CH_CTRL_DIR_AHUB2MEM; |
581 | burst_size = tdc->sconfig.src_maxburst; |
582 | ch_regs->config = ADMA_CH_CONFIG_TRG_BUF(desc->num_periods - 1); |
583 | ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index, |
584 | cdata->ch_req_mask, |
585 | cdata->ch_req_rx_shift); |
586 | ch_regs->trg_addr = buf_addr; |
587 | break; |
588 | |
589 | default: |
590 | dev_err(tdc2dev(tdc), "DMA direction is not supported\n" ); |
591 | return -EINVAL; |
592 | } |
593 | |
594 | ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir) | |
595 | ADMA_CH_CTRL_MODE_CONTINUOUS | |
596 | ADMA_CH_CTRL_FLOWCTRL_EN; |
597 | ch_regs->config |= cdata->adma_get_burst_config(burst_size); |
598 | ch_regs->config |= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1); |
599 | if (cdata->has_outstanding_reqs) |
600 | ch_regs->config |= TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(8); |
601 | |
602 | /* |
603 | * 'sreq_index' represents the current ADMAIF channel number and as per |
604 | * HW recommendation its FIFO size should match with the corresponding |
605 | * ADMA channel. |
606 | * |
607 | * ADMA FIFO size is set as per below (based on default ADMAIF channel |
608 | * FIFO sizes): |
609 | * fifo_size = 0x2 (sreq_index > sreq_index_offset) |
610 | * fifo_size = 0x3 (sreq_index <= sreq_index_offset) |
611 | * |
612 | */ |
613 | if (tdc->sreq_index > cdata->sreq_index_offset) |
614 | ch_regs->fifo_ctrl = |
615 | ADMA_CH_REG_FIELD_VAL(2, cdata->ch_fifo_size_mask, |
616 | fifo_size_shift); |
617 | else |
618 | ch_regs->fifo_ctrl = |
619 | ADMA_CH_REG_FIELD_VAL(3, cdata->ch_fifo_size_mask, |
620 | fifo_size_shift); |
621 | |
622 | ch_regs->tc = desc->period_len & ADMA_CH_TC_COUNT_MASK; |
623 | |
624 | return tegra_adma_request_alloc(tdc, direction); |
625 | } |
626 | |
627 | static struct dma_async_tx_descriptor *tegra_adma_prep_dma_cyclic( |
628 | struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len, |
629 | size_t period_len, enum dma_transfer_direction direction, |
630 | unsigned long flags) |
631 | { |
632 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
633 | struct tegra_adma_desc *desc = NULL; |
634 | |
635 | if (!buf_len || !period_len || period_len > ADMA_CH_TC_COUNT_MASK) { |
636 | dev_err(tdc2dev(tdc), "invalid buffer/period len\n" ); |
637 | return NULL; |
638 | } |
639 | |
640 | if (buf_len % period_len) { |
641 | dev_err(tdc2dev(tdc), "buf_len not a multiple of period_len\n" ); |
642 | return NULL; |
643 | } |
644 | |
645 | if (!IS_ALIGNED(buf_addr, 4)) { |
646 | dev_err(tdc2dev(tdc), "invalid buffer alignment\n" ); |
647 | return NULL; |
648 | } |
649 | |
650 | desc = kzalloc(size: sizeof(*desc), GFP_NOWAIT); |
651 | if (!desc) |
652 | return NULL; |
653 | |
654 | desc->buf_len = buf_len; |
655 | desc->period_len = period_len; |
656 | desc->num_periods = buf_len / period_len; |
657 | |
658 | if (tegra_adma_set_xfer_params(tdc, desc, buf_addr, direction)) { |
659 | kfree(objp: desc); |
660 | return NULL; |
661 | } |
662 | |
663 | return vchan_tx_prep(vc: &tdc->vc, vd: &desc->vd, tx_flags: flags); |
664 | } |
665 | |
666 | static int tegra_adma_alloc_chan_resources(struct dma_chan *dc) |
667 | { |
668 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
669 | int ret; |
670 | |
671 | ret = request_irq(irq: tdc->irq, handler: tegra_adma_isr, flags: 0, name: dma_chan_name(chan: dc), dev: tdc); |
672 | if (ret) { |
673 | dev_err(tdc2dev(tdc), "failed to get interrupt for %s\n" , |
674 | dma_chan_name(dc)); |
675 | return ret; |
676 | } |
677 | |
678 | ret = pm_runtime_resume_and_get(dev: tdc2dev(tdc)); |
679 | if (ret < 0) { |
680 | free_irq(tdc->irq, tdc); |
681 | return ret; |
682 | } |
683 | |
684 | dma_cookie_init(chan: &tdc->vc.chan); |
685 | |
686 | return 0; |
687 | } |
688 | |
689 | static void tegra_adma_free_chan_resources(struct dma_chan *dc) |
690 | { |
691 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
692 | |
693 | tegra_adma_terminate_all(dc); |
694 | vchan_free_chan_resources(vc: &tdc->vc); |
695 | tasklet_kill(t: &tdc->vc.task); |
696 | free_irq(tdc->irq, tdc); |
697 | pm_runtime_put(dev: tdc2dev(tdc)); |
698 | |
699 | tdc->sreq_index = 0; |
700 | tdc->sreq_dir = DMA_TRANS_NONE; |
701 | } |
702 | |
703 | static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, |
704 | struct of_dma *ofdma) |
705 | { |
706 | struct tegra_adma *tdma = ofdma->of_dma_data; |
707 | struct tegra_adma_chan *tdc; |
708 | struct dma_chan *chan; |
709 | unsigned int sreq_index; |
710 | |
711 | if (dma_spec->args_count != 1) |
712 | return NULL; |
713 | |
714 | sreq_index = dma_spec->args[0]; |
715 | |
716 | if (sreq_index == 0) { |
717 | dev_err(tdma->dev, "DMA request must not be 0\n" ); |
718 | return NULL; |
719 | } |
720 | |
721 | chan = dma_get_any_slave_channel(device: &tdma->dma_dev); |
722 | if (!chan) |
723 | return NULL; |
724 | |
725 | tdc = to_tegra_adma_chan(dc: chan); |
726 | tdc->sreq_index = sreq_index; |
727 | |
728 | return chan; |
729 | } |
730 | |
731 | static int __maybe_unused tegra_adma_runtime_suspend(struct device *dev) |
732 | { |
733 | struct tegra_adma *tdma = dev_get_drvdata(dev); |
734 | struct tegra_adma_chan_regs *ch_reg; |
735 | struct tegra_adma_chan *tdc; |
736 | int i; |
737 | |
738 | tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD); |
739 | if (!tdma->global_cmd) |
740 | goto clk_disable; |
741 | |
742 | for (i = 0; i < tdma->nr_channels; i++) { |
743 | tdc = &tdma->channels[i]; |
744 | ch_reg = &tdc->ch_regs; |
745 | ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD); |
746 | /* skip if channel is not active */ |
747 | if (!ch_reg->cmd) |
748 | continue; |
749 | ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC); |
750 | ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR); |
751 | ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR); |
752 | ch_reg->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); |
753 | ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL); |
754 | ch_reg->config = tdma_ch_read(tdc, ADMA_CH_CONFIG); |
755 | } |
756 | |
757 | clk_disable: |
758 | clk_disable_unprepare(clk: tdma->ahub_clk); |
759 | |
760 | return 0; |
761 | } |
762 | |
763 | static int __maybe_unused tegra_adma_runtime_resume(struct device *dev) |
764 | { |
765 | struct tegra_adma *tdma = dev_get_drvdata(dev); |
766 | struct tegra_adma_chan_regs *ch_reg; |
767 | struct tegra_adma_chan *tdc; |
768 | int ret, i; |
769 | |
770 | ret = clk_prepare_enable(clk: tdma->ahub_clk); |
771 | if (ret) { |
772 | dev_err(dev, "ahub clk_enable failed: %d\n" , ret); |
773 | return ret; |
774 | } |
775 | tdma_write(tdma, ADMA_GLOBAL_CMD, val: tdma->global_cmd); |
776 | |
777 | if (!tdma->global_cmd) |
778 | return 0; |
779 | |
780 | for (i = 0; i < tdma->nr_channels; i++) { |
781 | tdc = &tdma->channels[i]; |
782 | ch_reg = &tdc->ch_regs; |
783 | /* skip if channel was not active earlier */ |
784 | if (!ch_reg->cmd) |
785 | continue; |
786 | tdma_ch_write(tdc, ADMA_CH_TC, val: ch_reg->tc); |
787 | tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, val: ch_reg->src_addr); |
788 | tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, val: ch_reg->trg_addr); |
789 | tdma_ch_write(tdc, ADMA_CH_CTRL, val: ch_reg->ctrl); |
790 | tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, val: ch_reg->fifo_ctrl); |
791 | tdma_ch_write(tdc, ADMA_CH_CONFIG, val: ch_reg->config); |
792 | tdma_ch_write(tdc, ADMA_CH_CMD, val: ch_reg->cmd); |
793 | } |
794 | |
795 | return 0; |
796 | } |
797 | |
798 | static const struct tegra_adma_chip_data tegra210_chip_data = { |
799 | .adma_get_burst_config = tegra210_adma_get_burst_config, |
800 | .global_reg_offset = 0xc00, |
801 | .global_int_clear = 0x20, |
802 | .ch_req_tx_shift = 28, |
803 | .ch_req_rx_shift = 24, |
804 | .ch_base_offset = 0, |
805 | .ch_req_mask = 0xf, |
806 | .ch_req_max = 10, |
807 | .ch_reg_size = 0x80, |
808 | .nr_channels = 22, |
809 | .ch_fifo_size_mask = 0xf, |
810 | .sreq_index_offset = 2, |
811 | .has_outstanding_reqs = false, |
812 | }; |
813 | |
814 | static const struct tegra_adma_chip_data tegra186_chip_data = { |
815 | .adma_get_burst_config = tegra186_adma_get_burst_config, |
816 | .global_reg_offset = 0, |
817 | .global_int_clear = 0x402c, |
818 | .ch_req_tx_shift = 27, |
819 | .ch_req_rx_shift = 22, |
820 | .ch_base_offset = 0x10000, |
821 | .ch_req_mask = 0x1f, |
822 | .ch_req_max = 20, |
823 | .ch_reg_size = 0x100, |
824 | .nr_channels = 32, |
825 | .ch_fifo_size_mask = 0x1f, |
826 | .sreq_index_offset = 4, |
827 | .has_outstanding_reqs = true, |
828 | }; |
829 | |
830 | static const struct of_device_id tegra_adma_of_match[] = { |
831 | { .compatible = "nvidia,tegra210-adma" , .data = &tegra210_chip_data }, |
832 | { .compatible = "nvidia,tegra186-adma" , .data = &tegra186_chip_data }, |
833 | { }, |
834 | }; |
835 | MODULE_DEVICE_TABLE(of, tegra_adma_of_match); |
836 | |
837 | static int tegra_adma_probe(struct platform_device *pdev) |
838 | { |
839 | const struct tegra_adma_chip_data *cdata; |
840 | struct tegra_adma *tdma; |
841 | int ret, i; |
842 | |
843 | cdata = of_device_get_match_data(dev: &pdev->dev); |
844 | if (!cdata) { |
845 | dev_err(&pdev->dev, "device match data not found\n" ); |
846 | return -ENODEV; |
847 | } |
848 | |
849 | tdma = devm_kzalloc(dev: &pdev->dev, |
850 | struct_size(tdma, channels, cdata->nr_channels), |
851 | GFP_KERNEL); |
852 | if (!tdma) |
853 | return -ENOMEM; |
854 | |
855 | tdma->dev = &pdev->dev; |
856 | tdma->cdata = cdata; |
857 | tdma->nr_channels = cdata->nr_channels; |
858 | platform_set_drvdata(pdev, data: tdma); |
859 | |
860 | tdma->base_addr = devm_platform_ioremap_resource(pdev, index: 0); |
861 | if (IS_ERR(ptr: tdma->base_addr)) |
862 | return PTR_ERR(ptr: tdma->base_addr); |
863 | |
864 | tdma->ahub_clk = devm_clk_get(dev: &pdev->dev, id: "d_audio" ); |
865 | if (IS_ERR(ptr: tdma->ahub_clk)) { |
866 | dev_err(&pdev->dev, "Error: Missing ahub controller clock\n" ); |
867 | return PTR_ERR(ptr: tdma->ahub_clk); |
868 | } |
869 | |
870 | INIT_LIST_HEAD(list: &tdma->dma_dev.channels); |
871 | for (i = 0; i < tdma->nr_channels; i++) { |
872 | struct tegra_adma_chan *tdc = &tdma->channels[i]; |
873 | |
874 | tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset |
875 | + (cdata->ch_reg_size * i); |
876 | |
877 | tdc->irq = of_irq_get(dev: pdev->dev.of_node, index: i); |
878 | if (tdc->irq <= 0) { |
879 | ret = tdc->irq ?: -ENXIO; |
880 | goto irq_dispose; |
881 | } |
882 | |
883 | vchan_init(vc: &tdc->vc, dmadev: &tdma->dma_dev); |
884 | tdc->vc.desc_free = tegra_adma_desc_free; |
885 | tdc->tdma = tdma; |
886 | } |
887 | |
888 | pm_runtime_enable(dev: &pdev->dev); |
889 | |
890 | ret = pm_runtime_resume_and_get(dev: &pdev->dev); |
891 | if (ret < 0) |
892 | goto rpm_disable; |
893 | |
894 | ret = tegra_adma_init(tdma); |
895 | if (ret) |
896 | goto rpm_put; |
897 | |
898 | dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask); |
899 | dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); |
900 | dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask); |
901 | |
902 | tdma->dma_dev.dev = &pdev->dev; |
903 | tdma->dma_dev.device_alloc_chan_resources = |
904 | tegra_adma_alloc_chan_resources; |
905 | tdma->dma_dev.device_free_chan_resources = |
906 | tegra_adma_free_chan_resources; |
907 | tdma->dma_dev.device_issue_pending = tegra_adma_issue_pending; |
908 | tdma->dma_dev.device_prep_dma_cyclic = tegra_adma_prep_dma_cyclic; |
909 | tdma->dma_dev.device_config = tegra_adma_slave_config; |
910 | tdma->dma_dev.device_tx_status = tegra_adma_tx_status; |
911 | tdma->dma_dev.device_terminate_all = tegra_adma_terminate_all; |
912 | tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
913 | tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
914 | tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); |
915 | tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; |
916 | tdma->dma_dev.device_pause = tegra_adma_pause; |
917 | tdma->dma_dev.device_resume = tegra_adma_resume; |
918 | |
919 | ret = dma_async_device_register(device: &tdma->dma_dev); |
920 | if (ret < 0) { |
921 | dev_err(&pdev->dev, "ADMA registration failed: %d\n" , ret); |
922 | goto rpm_put; |
923 | } |
924 | |
925 | ret = of_dma_controller_register(np: pdev->dev.of_node, |
926 | of_dma_xlate: tegra_dma_of_xlate, data: tdma); |
927 | if (ret < 0) { |
928 | dev_err(&pdev->dev, "ADMA OF registration failed %d\n" , ret); |
929 | goto dma_remove; |
930 | } |
931 | |
932 | pm_runtime_put(dev: &pdev->dev); |
933 | |
934 | dev_info(&pdev->dev, "Tegra210 ADMA driver registered %d channels\n" , |
935 | tdma->nr_channels); |
936 | |
937 | return 0; |
938 | |
939 | dma_remove: |
940 | dma_async_device_unregister(device: &tdma->dma_dev); |
941 | rpm_put: |
942 | pm_runtime_put_sync(dev: &pdev->dev); |
943 | rpm_disable: |
944 | pm_runtime_disable(dev: &pdev->dev); |
945 | irq_dispose: |
946 | while (--i >= 0) |
947 | irq_dispose_mapping(virq: tdma->channels[i].irq); |
948 | |
949 | return ret; |
950 | } |
951 | |
952 | static void tegra_adma_remove(struct platform_device *pdev) |
953 | { |
954 | struct tegra_adma *tdma = platform_get_drvdata(pdev); |
955 | int i; |
956 | |
957 | of_dma_controller_free(np: pdev->dev.of_node); |
958 | dma_async_device_unregister(device: &tdma->dma_dev); |
959 | |
960 | for (i = 0; i < tdma->nr_channels; ++i) |
961 | irq_dispose_mapping(virq: tdma->channels[i].irq); |
962 | |
963 | pm_runtime_disable(dev: &pdev->dev); |
964 | } |
965 | |
966 | static const struct dev_pm_ops tegra_adma_dev_pm_ops = { |
967 | SET_RUNTIME_PM_OPS(tegra_adma_runtime_suspend, |
968 | tegra_adma_runtime_resume, NULL) |
969 | SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, |
970 | pm_runtime_force_resume) |
971 | }; |
972 | |
973 | static struct platform_driver tegra_admac_driver = { |
974 | .driver = { |
975 | .name = "tegra-adma" , |
976 | .pm = &tegra_adma_dev_pm_ops, |
977 | .of_match_table = tegra_adma_of_match, |
978 | }, |
979 | .probe = tegra_adma_probe, |
980 | .remove_new = tegra_adma_remove, |
981 | }; |
982 | |
983 | module_platform_driver(tegra_admac_driver); |
984 | |
985 | MODULE_ALIAS("platform:tegra210-adma" ); |
986 | MODULE_DESCRIPTION("NVIDIA Tegra ADMA driver" ); |
987 | MODULE_AUTHOR("Dara Ramesh <dramesh@nvidia.com>" ); |
988 | MODULE_AUTHOR("Jon Hunter <jonathanh@nvidia.com>" ); |
989 | MODULE_LICENSE("GPL v2" ); |
990 | |