1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * DMA driver for Xilinx ZynqMP DMA Engine |
4 | * |
5 | * Copyright (C) 2016 Xilinx, Inc. All rights reserved. |
6 | */ |
7 | |
8 | #include <linux/bitops.h> |
9 | #include <linux/dma-mapping.h> |
10 | #include <linux/init.h> |
11 | #include <linux/interrupt.h> |
12 | #include <linux/io.h> |
13 | #include <linux/module.h> |
14 | #include <linux/of.h> |
15 | #include <linux/of_dma.h> |
16 | #include <linux/platform_device.h> |
17 | #include <linux/slab.h> |
18 | #include <linux/clk.h> |
19 | #include <linux/io-64-nonatomic-lo-hi.h> |
20 | #include <linux/pm_runtime.h> |
21 | |
22 | #include "../dmaengine.h" |
23 | |
24 | /* Register Offsets */ |
25 | #define ZYNQMP_DMA_ISR 0x100 |
26 | #define ZYNQMP_DMA_IMR 0x104 |
27 | #define ZYNQMP_DMA_IER 0x108 |
28 | #define ZYNQMP_DMA_IDS 0x10C |
29 | #define ZYNQMP_DMA_CTRL0 0x110 |
30 | #define ZYNQMP_DMA_CTRL1 0x114 |
31 | #define ZYNQMP_DMA_DATA_ATTR 0x120 |
32 | #define ZYNQMP_DMA_DSCR_ATTR 0x124 |
33 | #define ZYNQMP_DMA_SRC_DSCR_WRD0 0x128 |
34 | #define ZYNQMP_DMA_SRC_DSCR_WRD1 0x12C |
35 | #define ZYNQMP_DMA_SRC_DSCR_WRD2 0x130 |
36 | #define ZYNQMP_DMA_SRC_DSCR_WRD3 0x134 |
37 | #define ZYNQMP_DMA_DST_DSCR_WRD0 0x138 |
38 | #define ZYNQMP_DMA_DST_DSCR_WRD1 0x13C |
39 | #define ZYNQMP_DMA_DST_DSCR_WRD2 0x140 |
40 | #define ZYNQMP_DMA_DST_DSCR_WRD3 0x144 |
41 | #define ZYNQMP_DMA_SRC_START_LSB 0x158 |
42 | #define ZYNQMP_DMA_SRC_START_MSB 0x15C |
43 | #define ZYNQMP_DMA_DST_START_LSB 0x160 |
44 | #define ZYNQMP_DMA_DST_START_MSB 0x164 |
45 | #define ZYNQMP_DMA_TOTAL_BYTE 0x188 |
46 | #define ZYNQMP_DMA_RATE_CTRL 0x18C |
47 | #define ZYNQMP_DMA_IRQ_SRC_ACCT 0x190 |
48 | #define ZYNQMP_DMA_IRQ_DST_ACCT 0x194 |
49 | #define ZYNQMP_DMA_CTRL2 0x200 |
50 | |
51 | /* Interrupt registers bit field definitions */ |
52 | #define ZYNQMP_DMA_DONE BIT(10) |
53 | #define ZYNQMP_DMA_AXI_WR_DATA BIT(9) |
54 | #define ZYNQMP_DMA_AXI_RD_DATA BIT(8) |
55 | #define ZYNQMP_DMA_AXI_RD_DST_DSCR BIT(7) |
56 | #define ZYNQMP_DMA_AXI_RD_SRC_DSCR BIT(6) |
57 | #define ZYNQMP_DMA_IRQ_DST_ACCT_ERR BIT(5) |
58 | #define ZYNQMP_DMA_IRQ_SRC_ACCT_ERR BIT(4) |
59 | #define ZYNQMP_DMA_BYTE_CNT_OVRFL BIT(3) |
60 | #define ZYNQMP_DMA_DST_DSCR_DONE BIT(2) |
61 | #define ZYNQMP_DMA_INV_APB BIT(0) |
62 | |
63 | /* Control 0 register bit field definitions */ |
64 | #define ZYNQMP_DMA_OVR_FETCH BIT(7) |
65 | #define ZYNQMP_DMA_POINT_TYPE_SG BIT(6) |
66 | #define ZYNQMP_DMA_RATE_CTRL_EN BIT(3) |
67 | |
68 | /* Control 1 register bit field definitions */ |
69 | #define ZYNQMP_DMA_SRC_ISSUE GENMASK(4, 0) |
70 | |
71 | /* Data Attribute register bit field definitions */ |
72 | #define ZYNQMP_DMA_ARBURST GENMASK(27, 26) |
73 | #define ZYNQMP_DMA_ARCACHE GENMASK(25, 22) |
74 | #define ZYNQMP_DMA_ARCACHE_OFST 22 |
75 | #define ZYNQMP_DMA_ARQOS GENMASK(21, 18) |
76 | #define ZYNQMP_DMA_ARQOS_OFST 18 |
77 | #define ZYNQMP_DMA_ARLEN GENMASK(17, 14) |
78 | #define ZYNQMP_DMA_ARLEN_OFST 14 |
79 | #define ZYNQMP_DMA_AWBURST GENMASK(13, 12) |
80 | #define ZYNQMP_DMA_AWCACHE GENMASK(11, 8) |
81 | #define ZYNQMP_DMA_AWCACHE_OFST 8 |
82 | #define ZYNQMP_DMA_AWQOS GENMASK(7, 4) |
83 | #define ZYNQMP_DMA_AWQOS_OFST 4 |
84 | #define ZYNQMP_DMA_AWLEN GENMASK(3, 0) |
85 | #define ZYNQMP_DMA_AWLEN_OFST 0 |
86 | |
87 | /* Descriptor Attribute register bit field definitions */ |
88 | #define ZYNQMP_DMA_AXCOHRNT BIT(8) |
89 | #define ZYNQMP_DMA_AXCACHE GENMASK(7, 4) |
90 | #define ZYNQMP_DMA_AXCACHE_OFST 4 |
91 | #define ZYNQMP_DMA_AXQOS GENMASK(3, 0) |
92 | #define ZYNQMP_DMA_AXQOS_OFST 0 |
93 | |
94 | /* Control register 2 bit field definitions */ |
95 | #define ZYNQMP_DMA_ENABLE BIT(0) |
96 | |
97 | /* Buffer Descriptor definitions */ |
98 | #define ZYNQMP_DMA_DESC_CTRL_STOP 0x10 |
99 | #define ZYNQMP_DMA_DESC_CTRL_COMP_INT 0x4 |
100 | #define ZYNQMP_DMA_DESC_CTRL_SIZE_256 0x2 |
101 | #define ZYNQMP_DMA_DESC_CTRL_COHRNT 0x1 |
102 | |
103 | /* Interrupt Mask specific definitions */ |
104 | #define ZYNQMP_DMA_INT_ERR (ZYNQMP_DMA_AXI_RD_DATA | \ |
105 | ZYNQMP_DMA_AXI_WR_DATA | \ |
106 | ZYNQMP_DMA_AXI_RD_DST_DSCR | \ |
107 | ZYNQMP_DMA_AXI_RD_SRC_DSCR | \ |
108 | ZYNQMP_DMA_INV_APB) |
109 | #define ZYNQMP_DMA_INT_OVRFL (ZYNQMP_DMA_BYTE_CNT_OVRFL | \ |
110 | ZYNQMP_DMA_IRQ_SRC_ACCT_ERR | \ |
111 | ZYNQMP_DMA_IRQ_DST_ACCT_ERR) |
112 | #define ZYNQMP_DMA_INT_DONE (ZYNQMP_DMA_DONE | ZYNQMP_DMA_DST_DSCR_DONE) |
113 | #define ZYNQMP_DMA_INT_EN_DEFAULT_MASK (ZYNQMP_DMA_INT_DONE | \ |
114 | ZYNQMP_DMA_INT_ERR | \ |
115 | ZYNQMP_DMA_INT_OVRFL | \ |
116 | ZYNQMP_DMA_DST_DSCR_DONE) |
117 | |
118 | /* Max number of descriptors per channel */ |
119 | #define ZYNQMP_DMA_NUM_DESCS 32 |
120 | |
121 | /* Max transfer size per descriptor */ |
122 | #define ZYNQMP_DMA_MAX_TRANS_LEN 0x40000000 |
123 | |
124 | /* Max burst lengths */ |
125 | #define ZYNQMP_DMA_MAX_DST_BURST_LEN 32768U |
126 | #define ZYNQMP_DMA_MAX_SRC_BURST_LEN 32768U |
127 | |
128 | /* Reset values for data attributes */ |
129 | #define ZYNQMP_DMA_AXCACHE_VAL 0xF |
130 | |
131 | #define ZYNQMP_DMA_SRC_ISSUE_RST_VAL 0x1F |
132 | |
133 | #define ZYNQMP_DMA_IDS_DEFAULT_MASK 0xFFF |
134 | |
135 | /* Bus width in bits */ |
136 | #define ZYNQMP_DMA_BUS_WIDTH_64 64 |
137 | #define ZYNQMP_DMA_BUS_WIDTH_128 128 |
138 | |
139 | #define ZDMA_PM_TIMEOUT 100 |
140 | |
141 | #define ZYNQMP_DMA_DESC_SIZE(chan) (chan->desc_size) |
142 | |
143 | #define to_chan(chan) container_of(chan, struct zynqmp_dma_chan, \ |
144 | common) |
145 | #define tx_to_desc(tx) container_of(tx, struct zynqmp_dma_desc_sw, \ |
146 | async_tx) |
147 | |
148 | /** |
149 | * struct zynqmp_dma_desc_ll - Hw linked list descriptor |
150 | * @addr: Buffer address |
151 | * @size: Size of the buffer |
152 | * @ctrl: Control word |
153 | * @nxtdscraddr: Next descriptor base address |
154 | * @rsvd: Reserved field and for Hw internal use. |
155 | */ |
156 | struct zynqmp_dma_desc_ll { |
157 | u64 addr; |
158 | u32 size; |
159 | u32 ctrl; |
160 | u64 nxtdscraddr; |
161 | u64 rsvd; |
162 | }; |
163 | |
164 | /** |
165 | * struct zynqmp_dma_desc_sw - Per Transaction structure |
166 | * @src: Source address for simple mode dma |
167 | * @dst: Destination address for simple mode dma |
168 | * @len: Transfer length for simple mode dma |
169 | * @node: Node in the channel descriptor list |
170 | * @tx_list: List head for the current transfer |
171 | * @async_tx: Async transaction descriptor |
172 | * @src_v: Virtual address of the src descriptor |
173 | * @src_p: Physical address of the src descriptor |
174 | * @dst_v: Virtual address of the dst descriptor |
175 | * @dst_p: Physical address of the dst descriptor |
176 | */ |
177 | struct zynqmp_dma_desc_sw { |
178 | u64 src; |
179 | u64 dst; |
180 | u32 len; |
181 | struct list_head node; |
182 | struct list_head tx_list; |
183 | struct dma_async_tx_descriptor async_tx; |
184 | struct zynqmp_dma_desc_ll *src_v; |
185 | dma_addr_t src_p; |
186 | struct zynqmp_dma_desc_ll *dst_v; |
187 | dma_addr_t dst_p; |
188 | }; |
189 | |
190 | /** |
191 | * struct zynqmp_dma_chan - Driver specific DMA channel structure |
192 | * @zdev: Driver specific device structure |
193 | * @regs: Control registers offset |
194 | * @lock: Descriptor operation lock |
195 | * @pending_list: Descriptors waiting |
196 | * @free_list: Descriptors free |
197 | * @active_list: Descriptors active |
198 | * @sw_desc_pool: SW descriptor pool |
199 | * @done_list: Complete descriptors |
200 | * @common: DMA common channel |
201 | * @desc_pool_v: Statically allocated descriptor base |
202 | * @desc_pool_p: Physical allocated descriptor base |
203 | * @desc_free_cnt: Descriptor available count |
204 | * @dev: The dma device |
205 | * @irq: Channel IRQ |
206 | * @is_dmacoherent: Tells whether dma operations are coherent or not |
207 | * @tasklet: Cleanup work after irq |
208 | * @idle : Channel status; |
209 | * @desc_size: Size of the low level descriptor |
210 | * @err: Channel has errors |
211 | * @bus_width: Bus width |
212 | * @src_burst_len: Source burst length |
213 | * @dst_burst_len: Dest burst length |
214 | */ |
215 | struct zynqmp_dma_chan { |
216 | struct zynqmp_dma_device *zdev; |
217 | void __iomem *regs; |
218 | spinlock_t lock; |
219 | struct list_head pending_list; |
220 | struct list_head free_list; |
221 | struct list_head active_list; |
222 | struct zynqmp_dma_desc_sw *sw_desc_pool; |
223 | struct list_head done_list; |
224 | struct dma_chan common; |
225 | void *desc_pool_v; |
226 | dma_addr_t desc_pool_p; |
227 | u32 desc_free_cnt; |
228 | struct device *dev; |
229 | int irq; |
230 | bool is_dmacoherent; |
231 | struct tasklet_struct tasklet; |
232 | bool idle; |
233 | size_t desc_size; |
234 | bool err; |
235 | u32 bus_width; |
236 | u32 src_burst_len; |
237 | u32 dst_burst_len; |
238 | }; |
239 | |
240 | /** |
241 | * struct zynqmp_dma_device - DMA device structure |
242 | * @dev: Device Structure |
243 | * @common: DMA device structure |
244 | * @chan: Driver specific DMA channel |
245 | * @clk_main: Pointer to main clock |
246 | * @clk_apb: Pointer to apb clock |
247 | */ |
248 | struct zynqmp_dma_device { |
249 | struct device *dev; |
250 | struct dma_device common; |
251 | struct zynqmp_dma_chan *chan; |
252 | struct clk *clk_main; |
253 | struct clk *clk_apb; |
254 | }; |
255 | |
256 | static inline void zynqmp_dma_writeq(struct zynqmp_dma_chan *chan, u32 reg, |
257 | u64 value) |
258 | { |
259 | lo_hi_writeq(val: value, addr: chan->regs + reg); |
260 | } |
261 | |
262 | /** |
263 | * zynqmp_dma_update_desc_to_ctrlr - Updates descriptor to the controller |
264 | * @chan: ZynqMP DMA DMA channel pointer |
265 | * @desc: Transaction descriptor pointer |
266 | */ |
267 | static void zynqmp_dma_update_desc_to_ctrlr(struct zynqmp_dma_chan *chan, |
268 | struct zynqmp_dma_desc_sw *desc) |
269 | { |
270 | dma_addr_t addr; |
271 | |
272 | addr = desc->src_p; |
273 | zynqmp_dma_writeq(chan, ZYNQMP_DMA_SRC_START_LSB, value: addr); |
274 | addr = desc->dst_p; |
275 | zynqmp_dma_writeq(chan, ZYNQMP_DMA_DST_START_LSB, value: addr); |
276 | } |
277 | |
278 | /** |
279 | * zynqmp_dma_desc_config_eod - Mark the descriptor as end descriptor |
280 | * @chan: ZynqMP DMA channel pointer |
281 | * @desc: Hw descriptor pointer |
282 | */ |
283 | static void zynqmp_dma_desc_config_eod(struct zynqmp_dma_chan *chan, |
284 | void *desc) |
285 | { |
286 | struct zynqmp_dma_desc_ll *hw = (struct zynqmp_dma_desc_ll *)desc; |
287 | |
288 | hw->ctrl |= ZYNQMP_DMA_DESC_CTRL_STOP; |
289 | hw++; |
290 | hw->ctrl |= ZYNQMP_DMA_DESC_CTRL_COMP_INT | ZYNQMP_DMA_DESC_CTRL_STOP; |
291 | } |
292 | |
293 | /** |
294 | * zynqmp_dma_config_sg_ll_desc - Configure the linked list descriptor |
295 | * @chan: ZynqMP DMA channel pointer |
296 | * @sdesc: Hw descriptor pointer |
297 | * @src: Source buffer address |
298 | * @dst: Destination buffer address |
299 | * @len: Transfer length |
300 | * @prev: Previous hw descriptor pointer |
301 | */ |
302 | static void zynqmp_dma_config_sg_ll_desc(struct zynqmp_dma_chan *chan, |
303 | struct zynqmp_dma_desc_ll *sdesc, |
304 | dma_addr_t src, dma_addr_t dst, size_t len, |
305 | struct zynqmp_dma_desc_ll *prev) |
306 | { |
307 | struct zynqmp_dma_desc_ll *ddesc = sdesc + 1; |
308 | |
309 | sdesc->size = ddesc->size = len; |
310 | sdesc->addr = src; |
311 | ddesc->addr = dst; |
312 | |
313 | sdesc->ctrl = ddesc->ctrl = ZYNQMP_DMA_DESC_CTRL_SIZE_256; |
314 | if (chan->is_dmacoherent) { |
315 | sdesc->ctrl |= ZYNQMP_DMA_DESC_CTRL_COHRNT; |
316 | ddesc->ctrl |= ZYNQMP_DMA_DESC_CTRL_COHRNT; |
317 | } |
318 | |
319 | if (prev) { |
320 | dma_addr_t addr = chan->desc_pool_p + |
321 | ((uintptr_t)sdesc - (uintptr_t)chan->desc_pool_v); |
322 | ddesc = prev + 1; |
323 | prev->nxtdscraddr = addr; |
324 | ddesc->nxtdscraddr = addr + ZYNQMP_DMA_DESC_SIZE(chan); |
325 | } |
326 | } |
327 | |
328 | /** |
329 | * zynqmp_dma_init - Initialize the channel |
330 | * @chan: ZynqMP DMA channel pointer |
331 | */ |
332 | static void zynqmp_dma_init(struct zynqmp_dma_chan *chan) |
333 | { |
334 | u32 val; |
335 | |
336 | writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, addr: chan->regs + ZYNQMP_DMA_IDS); |
337 | val = readl(addr: chan->regs + ZYNQMP_DMA_ISR); |
338 | writel(val, addr: chan->regs + ZYNQMP_DMA_ISR); |
339 | |
340 | if (chan->is_dmacoherent) { |
341 | val = ZYNQMP_DMA_AXCOHRNT; |
342 | val = (val & ~ZYNQMP_DMA_AXCACHE) | |
343 | (ZYNQMP_DMA_AXCACHE_VAL << ZYNQMP_DMA_AXCACHE_OFST); |
344 | writel(val, addr: chan->regs + ZYNQMP_DMA_DSCR_ATTR); |
345 | } |
346 | |
347 | val = readl(addr: chan->regs + ZYNQMP_DMA_DATA_ATTR); |
348 | if (chan->is_dmacoherent) { |
349 | val = (val & ~ZYNQMP_DMA_ARCACHE) | |
350 | (ZYNQMP_DMA_AXCACHE_VAL << ZYNQMP_DMA_ARCACHE_OFST); |
351 | val = (val & ~ZYNQMP_DMA_AWCACHE) | |
352 | (ZYNQMP_DMA_AXCACHE_VAL << ZYNQMP_DMA_AWCACHE_OFST); |
353 | } |
354 | writel(val, addr: chan->regs + ZYNQMP_DMA_DATA_ATTR); |
355 | |
356 | /* Clearing the interrupt account rgisters */ |
357 | val = readl(addr: chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT); |
358 | val = readl(addr: chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT); |
359 | |
360 | chan->idle = true; |
361 | } |
362 | |
363 | /** |
364 | * zynqmp_dma_tx_submit - Submit DMA transaction |
365 | * @tx: Async transaction descriptor pointer |
366 | * |
367 | * Return: cookie value |
368 | */ |
369 | static dma_cookie_t zynqmp_dma_tx_submit(struct dma_async_tx_descriptor *tx) |
370 | { |
371 | struct zynqmp_dma_chan *chan = to_chan(tx->chan); |
372 | struct zynqmp_dma_desc_sw *desc, *new; |
373 | dma_cookie_t cookie; |
374 | unsigned long irqflags; |
375 | |
376 | new = tx_to_desc(tx); |
377 | spin_lock_irqsave(&chan->lock, irqflags); |
378 | cookie = dma_cookie_assign(tx); |
379 | |
380 | if (!list_empty(head: &chan->pending_list)) { |
381 | desc = list_last_entry(&chan->pending_list, |
382 | struct zynqmp_dma_desc_sw, node); |
383 | if (!list_empty(head: &desc->tx_list)) |
384 | desc = list_last_entry(&desc->tx_list, |
385 | struct zynqmp_dma_desc_sw, node); |
386 | desc->src_v->nxtdscraddr = new->src_p; |
387 | desc->src_v->ctrl &= ~ZYNQMP_DMA_DESC_CTRL_STOP; |
388 | desc->dst_v->nxtdscraddr = new->dst_p; |
389 | desc->dst_v->ctrl &= ~ZYNQMP_DMA_DESC_CTRL_STOP; |
390 | } |
391 | |
392 | list_add_tail(new: &new->node, head: &chan->pending_list); |
393 | spin_unlock_irqrestore(lock: &chan->lock, flags: irqflags); |
394 | |
395 | return cookie; |
396 | } |
397 | |
398 | /** |
399 | * zynqmp_dma_get_descriptor - Get the sw descriptor from the pool |
400 | * @chan: ZynqMP DMA channel pointer |
401 | * |
402 | * Return: The sw descriptor |
403 | */ |
404 | static struct zynqmp_dma_desc_sw * |
405 | zynqmp_dma_get_descriptor(struct zynqmp_dma_chan *chan) |
406 | { |
407 | struct zynqmp_dma_desc_sw *desc; |
408 | unsigned long irqflags; |
409 | |
410 | spin_lock_irqsave(&chan->lock, irqflags); |
411 | desc = list_first_entry(&chan->free_list, |
412 | struct zynqmp_dma_desc_sw, node); |
413 | list_del(entry: &desc->node); |
414 | spin_unlock_irqrestore(lock: &chan->lock, flags: irqflags); |
415 | |
416 | INIT_LIST_HEAD(list: &desc->tx_list); |
417 | /* Clear the src and dst descriptor memory */ |
418 | memset((void *)desc->src_v, 0, ZYNQMP_DMA_DESC_SIZE(chan)); |
419 | memset((void *)desc->dst_v, 0, ZYNQMP_DMA_DESC_SIZE(chan)); |
420 | |
421 | return desc; |
422 | } |
423 | |
424 | /** |
425 | * zynqmp_dma_free_descriptor - Issue pending transactions |
426 | * @chan: ZynqMP DMA channel pointer |
427 | * @sdesc: Transaction descriptor pointer |
428 | */ |
429 | static void zynqmp_dma_free_descriptor(struct zynqmp_dma_chan *chan, |
430 | struct zynqmp_dma_desc_sw *sdesc) |
431 | { |
432 | struct zynqmp_dma_desc_sw *child, *next; |
433 | |
434 | chan->desc_free_cnt++; |
435 | list_move_tail(list: &sdesc->node, head: &chan->free_list); |
436 | list_for_each_entry_safe(child, next, &sdesc->tx_list, node) { |
437 | chan->desc_free_cnt++; |
438 | list_move_tail(list: &child->node, head: &chan->free_list); |
439 | } |
440 | } |
441 | |
442 | /** |
443 | * zynqmp_dma_free_desc_list - Free descriptors list |
444 | * @chan: ZynqMP DMA channel pointer |
445 | * @list: List to parse and delete the descriptor |
446 | */ |
447 | static void zynqmp_dma_free_desc_list(struct zynqmp_dma_chan *chan, |
448 | struct list_head *list) |
449 | { |
450 | struct zynqmp_dma_desc_sw *desc, *next; |
451 | |
452 | list_for_each_entry_safe(desc, next, list, node) |
453 | zynqmp_dma_free_descriptor(chan, sdesc: desc); |
454 | } |
455 | |
456 | /** |
457 | * zynqmp_dma_alloc_chan_resources - Allocate channel resources |
458 | * @dchan: DMA channel |
459 | * |
460 | * Return: Number of descriptors on success and failure value on error |
461 | */ |
462 | static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan) |
463 | { |
464 | struct zynqmp_dma_chan *chan = to_chan(dchan); |
465 | struct zynqmp_dma_desc_sw *desc; |
466 | int i, ret; |
467 | |
468 | ret = pm_runtime_resume_and_get(dev: chan->dev); |
469 | if (ret < 0) |
470 | return ret; |
471 | |
472 | chan->sw_desc_pool = kcalloc(ZYNQMP_DMA_NUM_DESCS, size: sizeof(*desc), |
473 | GFP_KERNEL); |
474 | if (!chan->sw_desc_pool) |
475 | return -ENOMEM; |
476 | |
477 | chan->idle = true; |
478 | chan->desc_free_cnt = ZYNQMP_DMA_NUM_DESCS; |
479 | |
480 | INIT_LIST_HEAD(list: &chan->free_list); |
481 | |
482 | for (i = 0; i < ZYNQMP_DMA_NUM_DESCS; i++) { |
483 | desc = chan->sw_desc_pool + i; |
484 | dma_async_tx_descriptor_init(tx: &desc->async_tx, chan: &chan->common); |
485 | desc->async_tx.tx_submit = zynqmp_dma_tx_submit; |
486 | list_add_tail(new: &desc->node, head: &chan->free_list); |
487 | } |
488 | |
489 | chan->desc_pool_v = dma_alloc_coherent(dev: chan->dev, |
490 | size: (2 * ZYNQMP_DMA_DESC_SIZE(chan) * |
491 | ZYNQMP_DMA_NUM_DESCS), |
492 | dma_handle: &chan->desc_pool_p, GFP_KERNEL); |
493 | if (!chan->desc_pool_v) |
494 | return -ENOMEM; |
495 | |
496 | for (i = 0; i < ZYNQMP_DMA_NUM_DESCS; i++) { |
497 | desc = chan->sw_desc_pool + i; |
498 | desc->src_v = (struct zynqmp_dma_desc_ll *) (chan->desc_pool_v + |
499 | (i * ZYNQMP_DMA_DESC_SIZE(chan) * 2)); |
500 | desc->dst_v = (struct zynqmp_dma_desc_ll *) (desc->src_v + 1); |
501 | desc->src_p = chan->desc_pool_p + |
502 | (i * ZYNQMP_DMA_DESC_SIZE(chan) * 2); |
503 | desc->dst_p = desc->src_p + ZYNQMP_DMA_DESC_SIZE(chan); |
504 | } |
505 | |
506 | return ZYNQMP_DMA_NUM_DESCS; |
507 | } |
508 | |
509 | /** |
510 | * zynqmp_dma_start - Start DMA channel |
511 | * @chan: ZynqMP DMA channel pointer |
512 | */ |
513 | static void zynqmp_dma_start(struct zynqmp_dma_chan *chan) |
514 | { |
515 | writel(ZYNQMP_DMA_INT_EN_DEFAULT_MASK, addr: chan->regs + ZYNQMP_DMA_IER); |
516 | writel(val: 0, addr: chan->regs + ZYNQMP_DMA_TOTAL_BYTE); |
517 | chan->idle = false; |
518 | writel(ZYNQMP_DMA_ENABLE, addr: chan->regs + ZYNQMP_DMA_CTRL2); |
519 | } |
520 | |
521 | /** |
522 | * zynqmp_dma_handle_ovfl_int - Process the overflow interrupt |
523 | * @chan: ZynqMP DMA channel pointer |
524 | * @status: Interrupt status value |
525 | */ |
526 | static void zynqmp_dma_handle_ovfl_int(struct zynqmp_dma_chan *chan, u32 status) |
527 | { |
528 | if (status & ZYNQMP_DMA_BYTE_CNT_OVRFL) |
529 | writel(val: 0, addr: chan->regs + ZYNQMP_DMA_TOTAL_BYTE); |
530 | if (status & ZYNQMP_DMA_IRQ_DST_ACCT_ERR) |
531 | readl(addr: chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT); |
532 | if (status & ZYNQMP_DMA_IRQ_SRC_ACCT_ERR) |
533 | readl(addr: chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT); |
534 | } |
535 | |
536 | static void zynqmp_dma_config(struct zynqmp_dma_chan *chan) |
537 | { |
538 | u32 val, burst_val; |
539 | |
540 | val = readl(addr: chan->regs + ZYNQMP_DMA_CTRL0); |
541 | val |= ZYNQMP_DMA_POINT_TYPE_SG; |
542 | writel(val, addr: chan->regs + ZYNQMP_DMA_CTRL0); |
543 | |
544 | val = readl(addr: chan->regs + ZYNQMP_DMA_DATA_ATTR); |
545 | burst_val = __ilog2_u32(n: chan->src_burst_len); |
546 | val = (val & ~ZYNQMP_DMA_ARLEN) | |
547 | ((burst_val << ZYNQMP_DMA_ARLEN_OFST) & ZYNQMP_DMA_ARLEN); |
548 | burst_val = __ilog2_u32(n: chan->dst_burst_len); |
549 | val = (val & ~ZYNQMP_DMA_AWLEN) | |
550 | ((burst_val << ZYNQMP_DMA_AWLEN_OFST) & ZYNQMP_DMA_AWLEN); |
551 | writel(val, addr: chan->regs + ZYNQMP_DMA_DATA_ATTR); |
552 | } |
553 | |
554 | /** |
555 | * zynqmp_dma_device_config - Zynqmp dma device configuration |
556 | * @dchan: DMA channel |
557 | * @config: DMA device config |
558 | * |
559 | * Return: 0 always |
560 | */ |
561 | static int zynqmp_dma_device_config(struct dma_chan *dchan, |
562 | struct dma_slave_config *config) |
563 | { |
564 | struct zynqmp_dma_chan *chan = to_chan(dchan); |
565 | |
566 | chan->src_burst_len = clamp(config->src_maxburst, 1U, |
567 | ZYNQMP_DMA_MAX_SRC_BURST_LEN); |
568 | chan->dst_burst_len = clamp(config->dst_maxburst, 1U, |
569 | ZYNQMP_DMA_MAX_DST_BURST_LEN); |
570 | |
571 | return 0; |
572 | } |
573 | |
574 | /** |
575 | * zynqmp_dma_start_transfer - Initiate the new transfer |
576 | * @chan: ZynqMP DMA channel pointer |
577 | */ |
578 | static void zynqmp_dma_start_transfer(struct zynqmp_dma_chan *chan) |
579 | { |
580 | struct zynqmp_dma_desc_sw *desc; |
581 | |
582 | if (!chan->idle) |
583 | return; |
584 | |
585 | zynqmp_dma_config(chan); |
586 | |
587 | desc = list_first_entry_or_null(&chan->pending_list, |
588 | struct zynqmp_dma_desc_sw, node); |
589 | if (!desc) |
590 | return; |
591 | |
592 | list_splice_tail_init(list: &chan->pending_list, head: &chan->active_list); |
593 | zynqmp_dma_update_desc_to_ctrlr(chan, desc); |
594 | zynqmp_dma_start(chan); |
595 | } |
596 | |
597 | |
598 | /** |
599 | * zynqmp_dma_chan_desc_cleanup - Cleanup the completed descriptors |
600 | * @chan: ZynqMP DMA channel |
601 | */ |
602 | static void zynqmp_dma_chan_desc_cleanup(struct zynqmp_dma_chan *chan) |
603 | { |
604 | struct zynqmp_dma_desc_sw *desc, *next; |
605 | unsigned long irqflags; |
606 | |
607 | spin_lock_irqsave(&chan->lock, irqflags); |
608 | |
609 | list_for_each_entry_safe(desc, next, &chan->done_list, node) { |
610 | struct dmaengine_desc_callback cb; |
611 | |
612 | dmaengine_desc_get_callback(tx: &desc->async_tx, cb: &cb); |
613 | if (dmaengine_desc_callback_valid(cb: &cb)) { |
614 | spin_unlock_irqrestore(lock: &chan->lock, flags: irqflags); |
615 | dmaengine_desc_callback_invoke(cb: &cb, NULL); |
616 | spin_lock_irqsave(&chan->lock, irqflags); |
617 | } |
618 | |
619 | /* Run any dependencies, then free the descriptor */ |
620 | zynqmp_dma_free_descriptor(chan, sdesc: desc); |
621 | } |
622 | |
623 | spin_unlock_irqrestore(lock: &chan->lock, flags: irqflags); |
624 | } |
625 | |
626 | /** |
627 | * zynqmp_dma_complete_descriptor - Mark the active descriptor as complete |
628 | * @chan: ZynqMP DMA channel pointer |
629 | */ |
630 | static void zynqmp_dma_complete_descriptor(struct zynqmp_dma_chan *chan) |
631 | { |
632 | struct zynqmp_dma_desc_sw *desc; |
633 | |
634 | desc = list_first_entry_or_null(&chan->active_list, |
635 | struct zynqmp_dma_desc_sw, node); |
636 | if (!desc) |
637 | return; |
638 | list_del(entry: &desc->node); |
639 | dma_cookie_complete(tx: &desc->async_tx); |
640 | list_add_tail(new: &desc->node, head: &chan->done_list); |
641 | } |
642 | |
643 | /** |
644 | * zynqmp_dma_issue_pending - Issue pending transactions |
645 | * @dchan: DMA channel pointer |
646 | */ |
647 | static void zynqmp_dma_issue_pending(struct dma_chan *dchan) |
648 | { |
649 | struct zynqmp_dma_chan *chan = to_chan(dchan); |
650 | unsigned long irqflags; |
651 | |
652 | spin_lock_irqsave(&chan->lock, irqflags); |
653 | zynqmp_dma_start_transfer(chan); |
654 | spin_unlock_irqrestore(lock: &chan->lock, flags: irqflags); |
655 | } |
656 | |
657 | /** |
658 | * zynqmp_dma_free_descriptors - Free channel descriptors |
659 | * @chan: ZynqMP DMA channel pointer |
660 | */ |
661 | static void zynqmp_dma_free_descriptors(struct zynqmp_dma_chan *chan) |
662 | { |
663 | unsigned long irqflags; |
664 | |
665 | spin_lock_irqsave(&chan->lock, irqflags); |
666 | zynqmp_dma_free_desc_list(chan, list: &chan->active_list); |
667 | zynqmp_dma_free_desc_list(chan, list: &chan->pending_list); |
668 | zynqmp_dma_free_desc_list(chan, list: &chan->done_list); |
669 | spin_unlock_irqrestore(lock: &chan->lock, flags: irqflags); |
670 | } |
671 | |
672 | /** |
673 | * zynqmp_dma_free_chan_resources - Free channel resources |
674 | * @dchan: DMA channel pointer |
675 | */ |
676 | static void zynqmp_dma_free_chan_resources(struct dma_chan *dchan) |
677 | { |
678 | struct zynqmp_dma_chan *chan = to_chan(dchan); |
679 | |
680 | zynqmp_dma_free_descriptors(chan); |
681 | dma_free_coherent(dev: chan->dev, |
682 | size: (2 * ZYNQMP_DMA_DESC_SIZE(chan) * ZYNQMP_DMA_NUM_DESCS), |
683 | cpu_addr: chan->desc_pool_v, dma_handle: chan->desc_pool_p); |
684 | kfree(objp: chan->sw_desc_pool); |
685 | pm_runtime_mark_last_busy(dev: chan->dev); |
686 | pm_runtime_put_autosuspend(dev: chan->dev); |
687 | } |
688 | |
689 | /** |
690 | * zynqmp_dma_reset - Reset the channel |
691 | * @chan: ZynqMP DMA channel pointer |
692 | */ |
693 | static void zynqmp_dma_reset(struct zynqmp_dma_chan *chan) |
694 | { |
695 | unsigned long irqflags; |
696 | |
697 | writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, addr: chan->regs + ZYNQMP_DMA_IDS); |
698 | |
699 | spin_lock_irqsave(&chan->lock, irqflags); |
700 | zynqmp_dma_complete_descriptor(chan); |
701 | spin_unlock_irqrestore(lock: &chan->lock, flags: irqflags); |
702 | zynqmp_dma_chan_desc_cleanup(chan); |
703 | zynqmp_dma_free_descriptors(chan); |
704 | |
705 | zynqmp_dma_init(chan); |
706 | } |
707 | |
708 | /** |
709 | * zynqmp_dma_irq_handler - ZynqMP DMA Interrupt handler |
710 | * @irq: IRQ number |
711 | * @data: Pointer to the ZynqMP DMA channel structure |
712 | * |
713 | * Return: IRQ_HANDLED/IRQ_NONE |
714 | */ |
715 | static irqreturn_t zynqmp_dma_irq_handler(int irq, void *data) |
716 | { |
717 | struct zynqmp_dma_chan *chan = (struct zynqmp_dma_chan *)data; |
718 | u32 isr, imr, status; |
719 | irqreturn_t ret = IRQ_NONE; |
720 | |
721 | isr = readl(addr: chan->regs + ZYNQMP_DMA_ISR); |
722 | imr = readl(addr: chan->regs + ZYNQMP_DMA_IMR); |
723 | status = isr & ~imr; |
724 | |
725 | writel(val: isr, addr: chan->regs + ZYNQMP_DMA_ISR); |
726 | if (status & ZYNQMP_DMA_INT_DONE) { |
727 | tasklet_schedule(t: &chan->tasklet); |
728 | ret = IRQ_HANDLED; |
729 | } |
730 | |
731 | if (status & ZYNQMP_DMA_DONE) |
732 | chan->idle = true; |
733 | |
734 | if (status & ZYNQMP_DMA_INT_ERR) { |
735 | chan->err = true; |
736 | tasklet_schedule(t: &chan->tasklet); |
737 | dev_err(chan->dev, "Channel %p has errors\n" , chan); |
738 | ret = IRQ_HANDLED; |
739 | } |
740 | |
741 | if (status & ZYNQMP_DMA_INT_OVRFL) { |
742 | zynqmp_dma_handle_ovfl_int(chan, status); |
743 | dev_dbg(chan->dev, "Channel %p overflow interrupt\n" , chan); |
744 | ret = IRQ_HANDLED; |
745 | } |
746 | |
747 | return ret; |
748 | } |
749 | |
750 | /** |
751 | * zynqmp_dma_do_tasklet - Schedule completion tasklet |
752 | * @t: Pointer to the ZynqMP DMA channel structure |
753 | */ |
754 | static void zynqmp_dma_do_tasklet(struct tasklet_struct *t) |
755 | { |
756 | struct zynqmp_dma_chan *chan = from_tasklet(chan, t, tasklet); |
757 | u32 count; |
758 | unsigned long irqflags; |
759 | |
760 | if (chan->err) { |
761 | zynqmp_dma_reset(chan); |
762 | chan->err = false; |
763 | return; |
764 | } |
765 | |
766 | spin_lock_irqsave(&chan->lock, irqflags); |
767 | count = readl(addr: chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT); |
768 | while (count) { |
769 | zynqmp_dma_complete_descriptor(chan); |
770 | count--; |
771 | } |
772 | spin_unlock_irqrestore(lock: &chan->lock, flags: irqflags); |
773 | |
774 | zynqmp_dma_chan_desc_cleanup(chan); |
775 | |
776 | if (chan->idle) { |
777 | spin_lock_irqsave(&chan->lock, irqflags); |
778 | zynqmp_dma_start_transfer(chan); |
779 | spin_unlock_irqrestore(lock: &chan->lock, flags: irqflags); |
780 | } |
781 | } |
782 | |
783 | /** |
784 | * zynqmp_dma_device_terminate_all - Aborts all transfers on a channel |
785 | * @dchan: DMA channel pointer |
786 | * |
787 | * Return: Always '0' |
788 | */ |
789 | static int zynqmp_dma_device_terminate_all(struct dma_chan *dchan) |
790 | { |
791 | struct zynqmp_dma_chan *chan = to_chan(dchan); |
792 | |
793 | writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, addr: chan->regs + ZYNQMP_DMA_IDS); |
794 | zynqmp_dma_free_descriptors(chan); |
795 | |
796 | return 0; |
797 | } |
798 | |
799 | /** |
800 | * zynqmp_dma_synchronize - Synchronizes the termination of a transfers to the current context. |
801 | * @dchan: DMA channel pointer |
802 | */ |
803 | static void zynqmp_dma_synchronize(struct dma_chan *dchan) |
804 | { |
805 | struct zynqmp_dma_chan *chan = to_chan(dchan); |
806 | |
807 | tasklet_kill(t: &chan->tasklet); |
808 | } |
809 | |
810 | /** |
811 | * zynqmp_dma_prep_memcpy - prepare descriptors for memcpy transaction |
812 | * @dchan: DMA channel |
813 | * @dma_dst: Destination buffer address |
814 | * @dma_src: Source buffer address |
815 | * @len: Transfer length |
816 | * @flags: transfer ack flags |
817 | * |
818 | * Return: Async transaction descriptor on success and NULL on failure |
819 | */ |
820 | static struct dma_async_tx_descriptor *zynqmp_dma_prep_memcpy( |
821 | struct dma_chan *dchan, dma_addr_t dma_dst, |
822 | dma_addr_t dma_src, size_t len, ulong flags) |
823 | { |
824 | struct zynqmp_dma_chan *chan; |
825 | struct zynqmp_dma_desc_sw *new, *first = NULL; |
826 | void *desc = NULL, *prev = NULL; |
827 | size_t copy; |
828 | u32 desc_cnt; |
829 | unsigned long irqflags; |
830 | |
831 | chan = to_chan(dchan); |
832 | |
833 | desc_cnt = DIV_ROUND_UP(len, ZYNQMP_DMA_MAX_TRANS_LEN); |
834 | |
835 | spin_lock_irqsave(&chan->lock, irqflags); |
836 | if (desc_cnt > chan->desc_free_cnt) { |
837 | spin_unlock_irqrestore(lock: &chan->lock, flags: irqflags); |
838 | dev_dbg(chan->dev, "chan %p descs are not available\n" , chan); |
839 | return NULL; |
840 | } |
841 | chan->desc_free_cnt = chan->desc_free_cnt - desc_cnt; |
842 | spin_unlock_irqrestore(lock: &chan->lock, flags: irqflags); |
843 | |
844 | do { |
845 | /* Allocate and populate the descriptor */ |
846 | new = zynqmp_dma_get_descriptor(chan); |
847 | |
848 | copy = min_t(size_t, len, ZYNQMP_DMA_MAX_TRANS_LEN); |
849 | desc = (struct zynqmp_dma_desc_ll *)new->src_v; |
850 | zynqmp_dma_config_sg_ll_desc(chan, sdesc: desc, src: dma_src, |
851 | dst: dma_dst, len: copy, prev); |
852 | prev = desc; |
853 | len -= copy; |
854 | dma_src += copy; |
855 | dma_dst += copy; |
856 | if (!first) |
857 | first = new; |
858 | else |
859 | list_add_tail(new: &new->node, head: &first->tx_list); |
860 | } while (len); |
861 | |
862 | zynqmp_dma_desc_config_eod(chan, desc); |
863 | async_tx_ack(tx: &first->async_tx); |
864 | first->async_tx.flags = (enum dma_ctrl_flags)flags; |
865 | return &first->async_tx; |
866 | } |
867 | |
868 | /** |
869 | * zynqmp_dma_chan_remove - Channel remove function |
870 | * @chan: ZynqMP DMA channel pointer |
871 | */ |
872 | static void zynqmp_dma_chan_remove(struct zynqmp_dma_chan *chan) |
873 | { |
874 | if (!chan) |
875 | return; |
876 | |
877 | if (chan->irq) |
878 | devm_free_irq(dev: chan->zdev->dev, irq: chan->irq, dev_id: chan); |
879 | tasklet_kill(t: &chan->tasklet); |
880 | list_del(entry: &chan->common.device_node); |
881 | } |
882 | |
883 | /** |
884 | * zynqmp_dma_chan_probe - Per Channel Probing |
885 | * @zdev: Driver specific device structure |
886 | * @pdev: Pointer to the platform_device structure |
887 | * |
888 | * Return: '0' on success and failure value on error |
889 | */ |
890 | static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev, |
891 | struct platform_device *pdev) |
892 | { |
893 | struct zynqmp_dma_chan *chan; |
894 | struct device_node *node = pdev->dev.of_node; |
895 | int err; |
896 | |
897 | chan = devm_kzalloc(dev: zdev->dev, size: sizeof(*chan), GFP_KERNEL); |
898 | if (!chan) |
899 | return -ENOMEM; |
900 | chan->dev = zdev->dev; |
901 | chan->zdev = zdev; |
902 | |
903 | chan->regs = devm_platform_ioremap_resource(pdev, index: 0); |
904 | if (IS_ERR(ptr: chan->regs)) |
905 | return PTR_ERR(ptr: chan->regs); |
906 | |
907 | chan->bus_width = ZYNQMP_DMA_BUS_WIDTH_64; |
908 | chan->dst_burst_len = ZYNQMP_DMA_MAX_DST_BURST_LEN; |
909 | chan->src_burst_len = ZYNQMP_DMA_MAX_SRC_BURST_LEN; |
910 | err = of_property_read_u32(np: node, propname: "xlnx,bus-width" , out_value: &chan->bus_width); |
911 | if (err < 0) { |
912 | dev_err(&pdev->dev, "missing xlnx,bus-width property\n" ); |
913 | return err; |
914 | } |
915 | |
916 | if (chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_64 && |
917 | chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_128) { |
918 | dev_err(zdev->dev, "invalid bus-width value" ); |
919 | return -EINVAL; |
920 | } |
921 | |
922 | chan->is_dmacoherent = of_property_read_bool(np: node, propname: "dma-coherent" ); |
923 | zdev->chan = chan; |
924 | tasklet_setup(t: &chan->tasklet, callback: zynqmp_dma_do_tasklet); |
925 | spin_lock_init(&chan->lock); |
926 | INIT_LIST_HEAD(list: &chan->active_list); |
927 | INIT_LIST_HEAD(list: &chan->pending_list); |
928 | INIT_LIST_HEAD(list: &chan->done_list); |
929 | INIT_LIST_HEAD(list: &chan->free_list); |
930 | |
931 | dma_cookie_init(chan: &chan->common); |
932 | chan->common.device = &zdev->common; |
933 | list_add_tail(new: &chan->common.device_node, head: &zdev->common.channels); |
934 | |
935 | zynqmp_dma_init(chan); |
936 | chan->irq = platform_get_irq(pdev, 0); |
937 | if (chan->irq < 0) |
938 | return -ENXIO; |
939 | err = devm_request_irq(dev: &pdev->dev, irq: chan->irq, handler: zynqmp_dma_irq_handler, irqflags: 0, |
940 | devname: "zynqmp-dma" , dev_id: chan); |
941 | if (err) |
942 | return err; |
943 | |
944 | chan->desc_size = sizeof(struct zynqmp_dma_desc_ll); |
945 | chan->idle = true; |
946 | return 0; |
947 | } |
948 | |
949 | /** |
950 | * of_zynqmp_dma_xlate - Translation function |
951 | * @dma_spec: Pointer to DMA specifier as found in the device tree |
952 | * @ofdma: Pointer to DMA controller data |
953 | * |
954 | * Return: DMA channel pointer on success and NULL on error |
955 | */ |
956 | static struct dma_chan *of_zynqmp_dma_xlate(struct of_phandle_args *dma_spec, |
957 | struct of_dma *ofdma) |
958 | { |
959 | struct zynqmp_dma_device *zdev = ofdma->of_dma_data; |
960 | |
961 | return dma_get_slave_channel(chan: &zdev->chan->common); |
962 | } |
963 | |
964 | /** |
965 | * zynqmp_dma_suspend - Suspend method for the driver |
966 | * @dev: Address of the device structure |
967 | * |
968 | * Put the driver into low power mode. |
969 | * Return: 0 on success and failure value on error |
970 | */ |
971 | static int __maybe_unused zynqmp_dma_suspend(struct device *dev) |
972 | { |
973 | if (!device_may_wakeup(dev)) |
974 | return pm_runtime_force_suspend(dev); |
975 | |
976 | return 0; |
977 | } |
978 | |
979 | /** |
980 | * zynqmp_dma_resume - Resume from suspend |
981 | * @dev: Address of the device structure |
982 | * |
983 | * Resume operation after suspend. |
984 | * Return: 0 on success and failure value on error |
985 | */ |
986 | static int __maybe_unused zynqmp_dma_resume(struct device *dev) |
987 | { |
988 | if (!device_may_wakeup(dev)) |
989 | return pm_runtime_force_resume(dev); |
990 | |
991 | return 0; |
992 | } |
993 | |
994 | /** |
995 | * zynqmp_dma_runtime_suspend - Runtime suspend method for the driver |
996 | * @dev: Address of the device structure |
997 | * |
998 | * Put the driver into low power mode. |
999 | * Return: 0 always |
1000 | */ |
1001 | static int __maybe_unused zynqmp_dma_runtime_suspend(struct device *dev) |
1002 | { |
1003 | struct zynqmp_dma_device *zdev = dev_get_drvdata(dev); |
1004 | |
1005 | clk_disable_unprepare(clk: zdev->clk_main); |
1006 | clk_disable_unprepare(clk: zdev->clk_apb); |
1007 | |
1008 | return 0; |
1009 | } |
1010 | |
1011 | /** |
1012 | * zynqmp_dma_runtime_resume - Runtime suspend method for the driver |
1013 | * @dev: Address of the device structure |
1014 | * |
1015 | * Put the driver into low power mode. |
1016 | * Return: 0 always |
1017 | */ |
1018 | static int __maybe_unused zynqmp_dma_runtime_resume(struct device *dev) |
1019 | { |
1020 | struct zynqmp_dma_device *zdev = dev_get_drvdata(dev); |
1021 | int err; |
1022 | |
1023 | err = clk_prepare_enable(clk: zdev->clk_main); |
1024 | if (err) { |
1025 | dev_err(dev, "Unable to enable main clock.\n" ); |
1026 | return err; |
1027 | } |
1028 | |
1029 | err = clk_prepare_enable(clk: zdev->clk_apb); |
1030 | if (err) { |
1031 | dev_err(dev, "Unable to enable apb clock.\n" ); |
1032 | clk_disable_unprepare(clk: zdev->clk_main); |
1033 | return err; |
1034 | } |
1035 | |
1036 | return 0; |
1037 | } |
1038 | |
1039 | static const struct dev_pm_ops zynqmp_dma_dev_pm_ops = { |
1040 | SET_SYSTEM_SLEEP_PM_OPS(zynqmp_dma_suspend, zynqmp_dma_resume) |
1041 | SET_RUNTIME_PM_OPS(zynqmp_dma_runtime_suspend, |
1042 | zynqmp_dma_runtime_resume, NULL) |
1043 | }; |
1044 | |
1045 | /** |
1046 | * zynqmp_dma_probe - Driver probe function |
1047 | * @pdev: Pointer to the platform_device structure |
1048 | * |
1049 | * Return: '0' on success and failure value on error |
1050 | */ |
1051 | static int zynqmp_dma_probe(struct platform_device *pdev) |
1052 | { |
1053 | struct zynqmp_dma_device *zdev; |
1054 | struct dma_device *p; |
1055 | int ret; |
1056 | |
1057 | zdev = devm_kzalloc(dev: &pdev->dev, size: sizeof(*zdev), GFP_KERNEL); |
1058 | if (!zdev) |
1059 | return -ENOMEM; |
1060 | |
1061 | zdev->dev = &pdev->dev; |
1062 | INIT_LIST_HEAD(list: &zdev->common.channels); |
1063 | |
1064 | ret = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(44)); |
1065 | if (ret) { |
1066 | dev_err(&pdev->dev, "DMA not available for address range\n" ); |
1067 | return ret; |
1068 | } |
1069 | dma_cap_set(DMA_MEMCPY, zdev->common.cap_mask); |
1070 | |
1071 | p = &zdev->common; |
1072 | p->device_prep_dma_memcpy = zynqmp_dma_prep_memcpy; |
1073 | p->device_terminate_all = zynqmp_dma_device_terminate_all; |
1074 | p->device_synchronize = zynqmp_dma_synchronize; |
1075 | p->device_issue_pending = zynqmp_dma_issue_pending; |
1076 | p->device_alloc_chan_resources = zynqmp_dma_alloc_chan_resources; |
1077 | p->device_free_chan_resources = zynqmp_dma_free_chan_resources; |
1078 | p->device_tx_status = dma_cookie_status; |
1079 | p->device_config = zynqmp_dma_device_config; |
1080 | p->dev = &pdev->dev; |
1081 | |
1082 | zdev->clk_main = devm_clk_get(dev: &pdev->dev, id: "clk_main" ); |
1083 | if (IS_ERR(ptr: zdev->clk_main)) |
1084 | return dev_err_probe(dev: &pdev->dev, err: PTR_ERR(ptr: zdev->clk_main), |
1085 | fmt: "main clock not found.\n" ); |
1086 | |
1087 | zdev->clk_apb = devm_clk_get(dev: &pdev->dev, id: "clk_apb" ); |
1088 | if (IS_ERR(ptr: zdev->clk_apb)) |
1089 | return dev_err_probe(dev: &pdev->dev, err: PTR_ERR(ptr: zdev->clk_apb), |
1090 | fmt: "apb clock not found.\n" ); |
1091 | |
1092 | platform_set_drvdata(pdev, data: zdev); |
1093 | pm_runtime_set_autosuspend_delay(dev: zdev->dev, ZDMA_PM_TIMEOUT); |
1094 | pm_runtime_use_autosuspend(dev: zdev->dev); |
1095 | pm_runtime_enable(dev: zdev->dev); |
1096 | ret = pm_runtime_resume_and_get(dev: zdev->dev); |
1097 | if (ret < 0) { |
1098 | dev_err(&pdev->dev, "device wakeup failed.\n" ); |
1099 | pm_runtime_disable(dev: zdev->dev); |
1100 | } |
1101 | if (!pm_runtime_enabled(dev: zdev->dev)) { |
1102 | ret = zynqmp_dma_runtime_resume(dev: zdev->dev); |
1103 | if (ret) |
1104 | return ret; |
1105 | } |
1106 | |
1107 | ret = zynqmp_dma_chan_probe(zdev, pdev); |
1108 | if (ret) { |
1109 | dev_err_probe(dev: &pdev->dev, err: ret, fmt: "Probing channel failed\n" ); |
1110 | goto err_disable_pm; |
1111 | } |
1112 | |
1113 | p->dst_addr_widths = BIT(zdev->chan->bus_width / 8); |
1114 | p->src_addr_widths = BIT(zdev->chan->bus_width / 8); |
1115 | |
1116 | ret = dma_async_device_register(device: &zdev->common); |
1117 | if (ret) { |
1118 | dev_err(zdev->dev, "failed to register the dma device\n" ); |
1119 | goto free_chan_resources; |
1120 | } |
1121 | |
1122 | ret = of_dma_controller_register(np: pdev->dev.of_node, |
1123 | of_dma_xlate: of_zynqmp_dma_xlate, data: zdev); |
1124 | if (ret) { |
1125 | dev_err_probe(dev: &pdev->dev, err: ret, fmt: "Unable to register DMA to DT\n" ); |
1126 | dma_async_device_unregister(device: &zdev->common); |
1127 | goto free_chan_resources; |
1128 | } |
1129 | |
1130 | pm_runtime_mark_last_busy(dev: zdev->dev); |
1131 | pm_runtime_put_sync_autosuspend(dev: zdev->dev); |
1132 | |
1133 | return 0; |
1134 | |
1135 | free_chan_resources: |
1136 | zynqmp_dma_chan_remove(chan: zdev->chan); |
1137 | err_disable_pm: |
1138 | if (!pm_runtime_enabled(dev: zdev->dev)) |
1139 | zynqmp_dma_runtime_suspend(dev: zdev->dev); |
1140 | pm_runtime_disable(dev: zdev->dev); |
1141 | return ret; |
1142 | } |
1143 | |
1144 | /** |
1145 | * zynqmp_dma_remove - Driver remove function |
1146 | * @pdev: Pointer to the platform_device structure |
1147 | * |
1148 | * Return: Always '0' |
1149 | */ |
1150 | static void zynqmp_dma_remove(struct platform_device *pdev) |
1151 | { |
1152 | struct zynqmp_dma_device *zdev = platform_get_drvdata(pdev); |
1153 | |
1154 | of_dma_controller_free(np: pdev->dev.of_node); |
1155 | dma_async_device_unregister(device: &zdev->common); |
1156 | |
1157 | zynqmp_dma_chan_remove(chan: zdev->chan); |
1158 | pm_runtime_disable(dev: zdev->dev); |
1159 | if (!pm_runtime_enabled(dev: zdev->dev)) |
1160 | zynqmp_dma_runtime_suspend(dev: zdev->dev); |
1161 | } |
1162 | |
1163 | static const struct of_device_id zynqmp_dma_of_match[] = { |
1164 | { .compatible = "xlnx,zynqmp-dma-1.0" , }, |
1165 | {} |
1166 | }; |
1167 | MODULE_DEVICE_TABLE(of, zynqmp_dma_of_match); |
1168 | |
1169 | static struct platform_driver zynqmp_dma_driver = { |
1170 | .driver = { |
1171 | .name = "xilinx-zynqmp-dma" , |
1172 | .of_match_table = zynqmp_dma_of_match, |
1173 | .pm = &zynqmp_dma_dev_pm_ops, |
1174 | }, |
1175 | .probe = zynqmp_dma_probe, |
1176 | .remove_new = zynqmp_dma_remove, |
1177 | }; |
1178 | |
1179 | module_platform_driver(zynqmp_dma_driver); |
1180 | |
1181 | MODULE_LICENSE("GPL" ); |
1182 | MODULE_AUTHOR("Xilinx, Inc." ); |
1183 | MODULE_DESCRIPTION("Xilinx ZynqMP DMA driver" ); |
1184 | |