1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Renesas R-Car Gen2/Gen3 DMA Controller Driver
4	*
5	* Copyright (C) 2014-2019 Renesas Electronics Inc.
6	*
7	* Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
8	*/
9
10	#include <linux/delay.h>
11	#include <linux/dma-mapping.h>
12	#include <linux/dmaengine.h>
13	#include <linux/interrupt.h>
14	#include <linux/list.h>
15	#include <linux/module.h>
16	#include <linux/mutex.h>
17	#include <linux/of.h>
18	#include <linux/of_dma.h>
19	#include <linux/of_platform.h>
20	#include <linux/platform_device.h>
21	#include <linux/pm_runtime.h>
22	#include <linux/slab.h>
23	#include <linux/spinlock.h>
24
25	#include "../dmaengine.h"
26
27	/*
28	* struct rcar_dmac_xfer_chunk - Descriptor for a hardware transfer
29	* @node: entry in the parent's chunks list
30	* @src_addr: device source address
31	* @dst_addr: device destination address
32	* @size: transfer size in bytes
33	*/
34	struct rcar_dmac_xfer_chunk {
35	struct list_head node;
36
37	dma_addr_t src_addr;
38	dma_addr_t dst_addr;
39	u32 size;
40	};
41
42	/*
43	* struct rcar_dmac_hw_desc - Hardware descriptor for a transfer chunk
44	* @sar: value of the SAR register (source address)
45	* @dar: value of the DAR register (destination address)
46	* @tcr: value of the TCR register (transfer count)
47	*/
48	struct rcar_dmac_hw_desc {
49	u32 sar;
50	u32 dar;
51	u32 tcr;
52	u32 reserved;
53	} __attribute__((__packed__));
54
55	/*
56	* struct rcar_dmac_desc - R-Car Gen2 DMA Transfer Descriptor
57	* @async_tx: base DMA asynchronous transaction descriptor
58	* @direction: direction of the DMA transfer
59	* @xfer_shift: log2 of the transfer size
60	* @chcr: value of the channel configuration register for this transfer
61	* @node: entry in the channel's descriptors lists
62	* @chunks: list of transfer chunks for this transfer
63	* @running: the transfer chunk being currently processed
64	* @nchunks: number of transfer chunks for this transfer
65	* @hwdescs.use: whether the transfer descriptor uses hardware descriptors
66	* @hwdescs.mem: hardware descriptors memory for the transfer
67	* @hwdescs.dma: device address of the hardware descriptors memory
68	* @hwdescs.size: size of the hardware descriptors in bytes
69	* @size: transfer size in bytes
70	* @cyclic: when set indicates that the DMA transfer is cyclic
71	*/
72	struct rcar_dmac_desc {
73	struct dma_async_tx_descriptor async_tx;
74	enum dma_transfer_direction direction;
75	unsigned int xfer_shift;
76	u32 chcr;
77
78	struct list_head node;
79	struct list_head chunks;
80	struct rcar_dmac_xfer_chunk *running;
81	unsigned int nchunks;
82
83	struct {
84	bool use;
85	struct rcar_dmac_hw_desc *mem;
86	dma_addr_t dma;
87	size_t size;
88	} hwdescs;
89
90	unsigned int size;
91	bool cyclic;
92	};
93
94	#define to_rcar_dmac_desc(d) container_of(d, struct rcar_dmac_desc, async_tx)
95
96	/*
97	* struct rcar_dmac_desc_page - One page worth of descriptors
98	* @node: entry in the channel's pages list
99	* @descs: array of DMA descriptors
100	* @chunks: array of transfer chunk descriptors
101	*/
102	struct rcar_dmac_desc_page {
103	struct list_head node;
104
105	union {
106	DECLARE_FLEX_ARRAY(struct rcar_dmac_desc, descs);
107	DECLARE_FLEX_ARRAY(struct rcar_dmac_xfer_chunk, chunks);
108	};
109	};
110
111	#define RCAR_DMAC_DESCS_PER_PAGE \
112	((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, descs)) / \
113	sizeof(struct rcar_dmac_desc))
114	#define RCAR_DMAC_XFER_CHUNKS_PER_PAGE \
115	((PAGE_SIZE - offsetof(struct rcar_dmac_desc_page, chunks)) / \
116	sizeof(struct rcar_dmac_xfer_chunk))
117
118	/*
119	* struct rcar_dmac_chan_slave - Slave configuration
120	* @slave_addr: slave memory address
121	* @xfer_size: size (in bytes) of hardware transfers
122	*/
123	struct rcar_dmac_chan_slave {
124	phys_addr_t slave_addr;
125	unsigned int xfer_size;
126	};
127
128	/*
129	* struct rcar_dmac_chan_map - Map of slave device phys to dma address
130	* @addr: slave dma address
131	* @dir: direction of mapping
132	* @slave: slave configuration that is mapped
133	*/
134	struct rcar_dmac_chan_map {
135	dma_addr_t addr;
136	enum dma_data_direction dir;
137	struct rcar_dmac_chan_slave slave;
138	};
139
140	/*
141	* struct rcar_dmac_chan - R-Car Gen2 DMA Controller Channel
142	* @chan: base DMA channel object
143	* @iomem: channel I/O memory base
144	* @index: index of this channel in the controller
145	* @irq: channel IRQ
146	* @src: slave memory address and size on the source side
147	* @dst: slave memory address and size on the destination side
148	* @mid_rid: hardware MID/RID for the DMA client using this channel
149	* @lock: protects the channel CHCR register and the desc members
150	* @desc.free: list of free descriptors
151	* @desc.pending: list of pending descriptors (submitted with tx_submit)
152	* @desc.active: list of active descriptors (activated with issue_pending)
153	* @desc.done: list of completed descriptors
154	* @desc.wait: list of descriptors waiting for an ack
155	* @desc.running: the descriptor being processed (a member of the active list)
156	* @desc.chunks_free: list of free transfer chunk descriptors
157	* @desc.pages: list of pages used by allocated descriptors
158	*/
159	struct rcar_dmac_chan {
160	struct dma_chan chan;
161	void __iomem *iomem;
162	unsigned int index;
163	int irq;
164
165	struct rcar_dmac_chan_slave src;
166	struct rcar_dmac_chan_slave dst;
167	struct rcar_dmac_chan_map map;
168	int mid_rid;
169
170	spinlock_t lock;
171
172	struct {
173	struct list_head free;
174	struct list_head pending;
175	struct list_head active;
176	struct list_head done;
177	struct list_head wait;
178	struct rcar_dmac_desc *running;
179
180	struct list_head chunks_free;
181
182	struct list_head pages;
183	} desc;
184	};
185
186	#define to_rcar_dmac_chan(c) container_of(c, struct rcar_dmac_chan, chan)
187
188	/*
189	* struct rcar_dmac - R-Car Gen2 DMA Controller
190	* @engine: base DMA engine object
191	* @dev: the hardware device
192	* @dmac_base: remapped base register block
193	* @chan_base: remapped channel register block (optional)
194	* @n_channels: number of available channels
195	* @channels: array of DMAC channels
196	* @channels_mask: bitfield of which DMA channels are managed by this driver
197	* @modules: bitmask of client modules in use
198	*/
199	struct rcar_dmac {
200	struct dma_device engine;
201	struct device *dev;
202	void __iomem *dmac_base;
203	void __iomem *chan_base;
204
205	unsigned int n_channels;
206	struct rcar_dmac_chan *channels;
207	u32 channels_mask;
208
209	DECLARE_BITMAP(modules, 256);
210	};
211
212	#define to_rcar_dmac(d) container_of(d, struct rcar_dmac, engine)
213
214	#define for_each_rcar_dmac_chan(i, dmac, chan) \
215	for (i = 0, chan = &(dmac)->channels[0]; i < (dmac)->n_channels; i++, chan++) \
216	if (!((dmac)->channels_mask & BIT(i))) continue; else
217
218	/*
219	* struct rcar_dmac_of_data - This driver's OF data
220	* @chan_offset_base: DMAC channels base offset
221	* @chan_offset_stride: DMAC channels offset stride
222	*/
223	struct rcar_dmac_of_data {
224	u32 chan_offset_base;
225	u32 chan_offset_stride;
226	};
227
228	/* -----------------------------------------------------------------------------
229	* Registers
230	*/
231
232	#define RCAR_DMAISTA 0x0020
233	#define RCAR_DMASEC 0x0030
234	#define RCAR_DMAOR 0x0060
235	#define RCAR_DMAOR_PRI_FIXED (0 << 8)
236	#define RCAR_DMAOR_PRI_ROUND_ROBIN (3 << 8)
237	#define RCAR_DMAOR_AE (1 << 2)
238	#define RCAR_DMAOR_DME (1 << 0)
239	#define RCAR_DMACHCLR 0x0080 /* Not on R-Car Gen4 */
240	#define RCAR_DMADPSEC 0x00a0
241
242	#define RCAR_DMASAR 0x0000
243	#define RCAR_DMADAR 0x0004
244	#define RCAR_DMATCR 0x0008
245	#define RCAR_DMATCR_MASK 0x00ffffff
246	#define RCAR_DMATSR 0x0028
247	#define RCAR_DMACHCR 0x000c
248	#define RCAR_DMACHCR_CAE (1 << 31)
249	#define RCAR_DMACHCR_CAIE (1 << 30)
250	#define RCAR_DMACHCR_DPM_DISABLED (0 << 28)
251	#define RCAR_DMACHCR_DPM_ENABLED (1 << 28)
252	#define RCAR_DMACHCR_DPM_REPEAT (2 << 28)
253	#define RCAR_DMACHCR_DPM_INFINITE (3 << 28)
254	#define RCAR_DMACHCR_RPT_SAR (1 << 27)
255	#define RCAR_DMACHCR_RPT_DAR (1 << 26)
256	#define RCAR_DMACHCR_RPT_TCR (1 << 25)
257	#define RCAR_DMACHCR_DPB (1 << 22)
258	#define RCAR_DMACHCR_DSE (1 << 19)
259	#define RCAR_DMACHCR_DSIE (1 << 18)
260	#define RCAR_DMACHCR_TS_1B ((0 << 20) | (0 << 3))
261	#define RCAR_DMACHCR_TS_2B ((0 << 20) | (1 << 3))
262	#define RCAR_DMACHCR_TS_4B ((0 << 20) | (2 << 3))
263	#define RCAR_DMACHCR_TS_16B ((0 << 20) | (3 << 3))
264	#define RCAR_DMACHCR_TS_32B ((1 << 20) | (0 << 3))
265	#define RCAR_DMACHCR_TS_64B ((1 << 20) | (1 << 3))
266	#define RCAR_DMACHCR_TS_8B ((1 << 20) | (3 << 3))
267	#define RCAR_DMACHCR_DM_FIXED (0 << 14)
268	#define RCAR_DMACHCR_DM_INC (1 << 14)
269	#define RCAR_DMACHCR_DM_DEC (2 << 14)
270	#define RCAR_DMACHCR_SM_FIXED (0 << 12)
271	#define RCAR_DMACHCR_SM_INC (1 << 12)
272	#define RCAR_DMACHCR_SM_DEC (2 << 12)
273	#define RCAR_DMACHCR_RS_AUTO (4 << 8)
274	#define RCAR_DMACHCR_RS_DMARS (8 << 8)
275	#define RCAR_DMACHCR_IE (1 << 2)
276	#define RCAR_DMACHCR_TE (1 << 1)
277	#define RCAR_DMACHCR_DE (1 << 0)
278	#define RCAR_DMATCRB 0x0018
279	#define RCAR_DMATSRB 0x0038
280	#define RCAR_DMACHCRB 0x001c
281	#define RCAR_DMACHCRB_DCNT(n) ((n) << 24)
282	#define RCAR_DMACHCRB_DPTR_MASK (0xff << 16)
283	#define RCAR_DMACHCRB_DPTR_SHIFT 16
284	#define RCAR_DMACHCRB_DRST (1 << 15)
285	#define RCAR_DMACHCRB_DTS (1 << 8)
286	#define RCAR_DMACHCRB_SLM_NORMAL (0 << 4)
287	#define RCAR_DMACHCRB_SLM_CLK(n) ((8 | (n)) << 4)
288	#define RCAR_DMACHCRB_PRI(n) ((n) << 0)
289	#define RCAR_DMARS 0x0040
290	#define RCAR_DMABUFCR 0x0048
291	#define RCAR_DMABUFCR_MBU(n) ((n) << 16)
292	#define RCAR_DMABUFCR_ULB(n) ((n) << 0)
293	#define RCAR_DMADPBASE 0x0050
294	#define RCAR_DMADPBASE_MASK 0xfffffff0
295	#define RCAR_DMADPBASE_SEL (1 << 0)
296	#define RCAR_DMADPCR 0x0054
297	#define RCAR_DMADPCR_DIPT(n) ((n) << 24)
298	#define RCAR_DMAFIXSAR 0x0010
299	#define RCAR_DMAFIXDAR 0x0014
300	#define RCAR_DMAFIXDPBASE 0x0060
301
302	/* For R-Car Gen4 */
303	#define RCAR_GEN4_DMACHCLR 0x0100
304
305	/* Hardcode the MEMCPY transfer size to 4 bytes. */
306	#define RCAR_DMAC_MEMCPY_XFER_SIZE 4
307
308	/* -----------------------------------------------------------------------------
309	* Device access
310	*/
311
312	static void rcar_dmac_write(struct rcar_dmac *dmac, u32 reg, u32 data)
313	{
314	if (reg == RCAR_DMAOR)
315	writew(val: data, addr: dmac->dmac_base + reg);
316	else
317	writel(val: data, addr: dmac->dmac_base + reg);
318	}
319
320	static u32 rcar_dmac_read(struct rcar_dmac *dmac, u32 reg)
321	{
322	if (reg == RCAR_DMAOR)
323	return readw(addr: dmac->dmac_base + reg);
324	else
325	return readl(addr: dmac->dmac_base + reg);
326	}
327
328	static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg)
329	{
330	if (reg == RCAR_DMARS)
331	return readw(addr: chan->iomem + reg);
332	else
333	return readl(addr: chan->iomem + reg);
334	}
335
336	static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data)
337	{
338	if (reg == RCAR_DMARS)
339	writew(val: data, addr: chan->iomem + reg);
340	else
341	writel(val: data, addr: chan->iomem + reg);
342	}
343
344	static void rcar_dmac_chan_clear(struct rcar_dmac *dmac,
345	struct rcar_dmac_chan *chan)
346	{
347	if (dmac->chan_base)
348	rcar_dmac_chan_write(chan, RCAR_GEN4_DMACHCLR, data: 1);
349	else
350	rcar_dmac_write(dmac, RCAR_DMACHCLR, BIT(chan->index));
351	}
352
353	static void rcar_dmac_chan_clear_all(struct rcar_dmac *dmac)
354	{
355	struct rcar_dmac_chan *chan;
356	unsigned int i;
357
358	if (dmac->chan_base) {
359	for_each_rcar_dmac_chan(i, dmac, chan)
360	rcar_dmac_chan_write(chan, RCAR_GEN4_DMACHCLR, data: 1);
361	} else {
362	rcar_dmac_write(dmac, RCAR_DMACHCLR, data: dmac->channels_mask);
363	}
364	}
365
366	/* -----------------------------------------------------------------------------
367	* Initialization and configuration
368	*/
369
370	static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan)
371	{
372	u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
373
374	return !!(chcr & (RCAR_DMACHCR_DE | RCAR_DMACHCR_TE));
375	}
376
377	static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan)
378	{
379	struct rcar_dmac_desc *desc = chan->desc.running;
380	u32 chcr = desc->chcr;
381
382	WARN_ON_ONCE(rcar_dmac_chan_is_busy(chan));
383
384	if (chan->mid_rid >= 0)
385	rcar_dmac_chan_write(chan, RCAR_DMARS, data: chan->mid_rid);
386
387	if (desc->hwdescs.use) {
388	struct rcar_dmac_xfer_chunk *chunk =
389	list_first_entry(&desc->chunks,
390	struct rcar_dmac_xfer_chunk, node);
391
392	dev_dbg(chan->chan.device->dev,
393	"chan%u: queue desc %p: %u@%pad\n",
394	chan->index, desc, desc->nchunks, &desc->hwdescs.dma);
395
396	#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
397	rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR,
398	data: chunk->src_addr >> 32);
399	rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR,
400	data: chunk->dst_addr >> 32);
401	rcar_dmac_chan_write(chan, RCAR_DMAFIXDPBASE,
402	data: desc->hwdescs.dma >> 32);
403	#endif
404	rcar_dmac_chan_write(chan, RCAR_DMADPBASE,
405	data: (desc->hwdescs.dma & 0xfffffff0) |
406	RCAR_DMADPBASE_SEL);
407	rcar_dmac_chan_write(chan, RCAR_DMACHCRB,
408	RCAR_DMACHCRB_DCNT(desc->nchunks - 1) |
409	RCAR_DMACHCRB_DRST);
410
411	/*
412	* Errata: When descriptor memory is accessed through an IOMMU
413	* the DMADAR register isn't initialized automatically from the
414	* first descriptor at beginning of transfer by the DMAC like it
415	* should. Initialize it manually with the destination address
416	* of the first chunk.
417	*/
418	rcar_dmac_chan_write(chan, RCAR_DMADAR,
419	data: chunk->dst_addr & 0xffffffff);
420
421	/*
422	* Program the descriptor stage interrupt to occur after the end
423	* of the first stage.
424	*/
425	rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(1));
426
427	chcr |= RCAR_DMACHCR_RPT_SAR | RCAR_DMACHCR_RPT_DAR
428	| RCAR_DMACHCR_RPT_TCR | RCAR_DMACHCR_DPB;
429
430	/*
431	* If the descriptor isn't cyclic enable normal descriptor mode
432	* and the transfer completion interrupt.
433	*/
434	if (!desc->cyclic)
435	chcr |= RCAR_DMACHCR_DPM_ENABLED | RCAR_DMACHCR_IE;
436	/*
437	* If the descriptor is cyclic and has a callback enable the
438	* descriptor stage interrupt in infinite repeat mode.
439	*/
440	else if (desc->async_tx.callback)
441	chcr |= RCAR_DMACHCR_DPM_INFINITE | RCAR_DMACHCR_DSIE;
442	/*
443	* Otherwise just select infinite repeat mode without any
444	* interrupt.
445	*/
446	else
447	chcr |= RCAR_DMACHCR_DPM_INFINITE;
448	} else {
449	struct rcar_dmac_xfer_chunk *chunk = desc->running;
450
451	dev_dbg(chan->chan.device->dev,
452	"chan%u: queue chunk %p: %u@%pad -> %pad\n",
453	chan->index, chunk, chunk->size, &chunk->src_addr,
454	&chunk->dst_addr);
455
456	#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
457	rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR,
458	data: chunk->src_addr >> 32);
459	rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR,
460	data: chunk->dst_addr >> 32);
461	#endif
462	rcar_dmac_chan_write(chan, RCAR_DMASAR,
463	data: chunk->src_addr & 0xffffffff);
464	rcar_dmac_chan_write(chan, RCAR_DMADAR,
465	data: chunk->dst_addr & 0xffffffff);
466	rcar_dmac_chan_write(chan, RCAR_DMATCR,
467	data: chunk->size >> desc->xfer_shift);
468
469	chcr |= RCAR_DMACHCR_DPM_DISABLED | RCAR_DMACHCR_IE;
470	}
471
472	rcar_dmac_chan_write(chan, RCAR_DMACHCR,
473	data: chcr | RCAR_DMACHCR_DE | RCAR_DMACHCR_CAIE);
474	}
475
476	static int rcar_dmac_init(struct rcar_dmac *dmac)
477	{
478	u16 dmaor;
479
480	/* Clear all channels and enable the DMAC globally. */
481	rcar_dmac_chan_clear_all(dmac);
482	rcar_dmac_write(dmac, RCAR_DMAOR,
483	RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME);
484
485	dmaor = rcar_dmac_read(dmac, RCAR_DMAOR);
486	if ((dmaor & (RCAR_DMAOR_AE | RCAR_DMAOR_DME)) != RCAR_DMAOR_DME) {
487	dev_warn(dmac->dev, "DMAOR initialization failed.\n");
488	return -EIO;
489	}
490
491	return 0;
492	}
493
494	/* -----------------------------------------------------------------------------
495	* Descriptors submission
496	*/
497
498	static dma_cookie_t rcar_dmac_tx_submit(struct dma_async_tx_descriptor *tx)
499	{
500	struct rcar_dmac_chan *chan = to_rcar_dmac_chan(tx->chan);
501	struct rcar_dmac_desc *desc = to_rcar_dmac_desc(tx);
502	unsigned long flags;
503	dma_cookie_t cookie;
504
505	spin_lock_irqsave(&chan->lock, flags);
506
507	cookie = dma_cookie_assign(tx);
508
509	dev_dbg(chan->chan.device->dev, "chan%u: submit #%d@%p\n",
510	chan->index, tx->cookie, desc);
511
512	list_add_tail(new: &desc->node, head: &chan->desc.pending);
513	desc->running = list_first_entry(&desc->chunks,
514	struct rcar_dmac_xfer_chunk, node);
515
516	spin_unlock_irqrestore(lock: &chan->lock, flags);
517
518	return cookie;
519	}
520
521	/* -----------------------------------------------------------------------------
522	* Descriptors allocation and free
523	*/
524
525	/*
526	* rcar_dmac_desc_alloc - Allocate a page worth of DMA descriptors
527	* @chan: the DMA channel
528	* @gfp: allocation flags
529	*/
530	static int rcar_dmac_desc_alloc(struct rcar_dmac_chan *chan, gfp_t gfp)
531	{
532	struct rcar_dmac_desc_page *page;
533	unsigned long flags;
534	LIST_HEAD(list);
535	unsigned int i;
536
537	page = (void *)get_zeroed_page(gfp_mask: gfp);
538	if (!page)
539	return -ENOMEM;
540
541	for (i = 0; i < RCAR_DMAC_DESCS_PER_PAGE; ++i) {
542	struct rcar_dmac_desc *desc = &page->descs[i];
543
544	dma_async_tx_descriptor_init(tx: &desc->async_tx, chan: &chan->chan);
545	desc->async_tx.tx_submit = rcar_dmac_tx_submit;
546	INIT_LIST_HEAD(list: &desc->chunks);
547
548	list_add_tail(new: &desc->node, head: &list);
549	}
550
551	spin_lock_irqsave(&chan->lock, flags);
552	list_splice_tail(list: &list, head: &chan->desc.free);
553	list_add_tail(new: &page->node, head: &chan->desc.pages);
554	spin_unlock_irqrestore(lock: &chan->lock, flags);
555
556	return 0;
557	}
558
559	/*
560	* rcar_dmac_desc_put - Release a DMA transfer descriptor
561	* @chan: the DMA channel
562	* @desc: the descriptor
563	*
564	* Put the descriptor and its transfer chunk descriptors back in the channel's
565	* free descriptors lists. The descriptor's chunks list will be reinitialized to
566	* an empty list as a result.
567	*
568	* The descriptor must have been removed from the channel's lists before calling
569	* this function.
570	*/
571	static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan,
572	struct rcar_dmac_desc *desc)
573	{
574	unsigned long flags;
575
576	spin_lock_irqsave(&chan->lock, flags);
577	list_splice_tail_init(list: &desc->chunks, head: &chan->desc.chunks_free);
578	list_add(new: &desc->node, head: &chan->desc.free);
579	spin_unlock_irqrestore(lock: &chan->lock, flags);
580	}
581
582	static void rcar_dmac_desc_recycle_acked(struct rcar_dmac_chan *chan)
583	{
584	struct rcar_dmac_desc *desc, *_desc;
585	unsigned long flags;
586	LIST_HEAD(list);
587
588	/*
589	* We have to temporarily move all descriptors from the wait list to a
590	* local list as iterating over the wait list, even with
591	* list_for_each_entry_safe, isn't safe if we release the channel lock
592	* around the rcar_dmac_desc_put() call.
593	*/
594	spin_lock_irqsave(&chan->lock, flags);
595	list_splice_init(list: &chan->desc.wait, head: &list);
596	spin_unlock_irqrestore(lock: &chan->lock, flags);
597
598	list_for_each_entry_safe(desc, _desc, &list, node) {
599	if (async_tx_test_ack(tx: &desc->async_tx)) {
600	list_del(entry: &desc->node);
601	rcar_dmac_desc_put(chan, desc);
602	}
603	}
604
605	if (list_empty(head: &list))
606	return;
607
608	/* Put the remaining descriptors back in the wait list. */
609	spin_lock_irqsave(&chan->lock, flags);
610	list_splice(list: &list, head: &chan->desc.wait);
611	spin_unlock_irqrestore(lock: &chan->lock, flags);
612	}
613
614	/*
615	* rcar_dmac_desc_get - Allocate a descriptor for a DMA transfer
616	* @chan: the DMA channel
617	*
618	* Locking: This function must be called in a non-atomic context.
619	*
620	* Return: A pointer to the allocated descriptor or NULL if no descriptor can
621	* be allocated.
622	*/
623	static struct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan)
624	{
625	struct rcar_dmac_desc *desc;
626	unsigned long flags;
627	int ret;
628
629	/* Recycle acked descriptors before attempting allocation. */
630	rcar_dmac_desc_recycle_acked(chan);
631
632	spin_lock_irqsave(&chan->lock, flags);
633
634	while (list_empty(head: &chan->desc.free)) {
635	/*
636	* No free descriptors, allocate a page worth of them and try
637	* again, as someone else could race us to get the newly
638	* allocated descriptors. If the allocation fails return an
639	* error.
640	*/
641	spin_unlock_irqrestore(lock: &chan->lock, flags);
642	ret = rcar_dmac_desc_alloc(chan, GFP_NOWAIT);
643	if (ret < 0)
644	return NULL;
645	spin_lock_irqsave(&chan->lock, flags);
646	}
647
648	desc = list_first_entry(&chan->desc.free, struct rcar_dmac_desc, node);
649	list_del(entry: &desc->node);
650
651	spin_unlock_irqrestore(lock: &chan->lock, flags);
652
653	return desc;
654	}
655
656	/*
657	* rcar_dmac_xfer_chunk_alloc - Allocate a page worth of transfer chunks
658	* @chan: the DMA channel
659	* @gfp: allocation flags
660	*/
661	static int rcar_dmac_xfer_chunk_alloc(struct rcar_dmac_chan *chan, gfp_t gfp)
662	{
663	struct rcar_dmac_desc_page *page;
664	unsigned long flags;
665	LIST_HEAD(list);
666	unsigned int i;
667
668	page = (void *)get_zeroed_page(gfp_mask: gfp);
669	if (!page)
670	return -ENOMEM;
671
672	for (i = 0; i < RCAR_DMAC_XFER_CHUNKS_PER_PAGE; ++i) {
673	struct rcar_dmac_xfer_chunk *chunk = &page->chunks[i];
674
675	list_add_tail(new: &chunk->node, head: &list);
676	}
677
678	spin_lock_irqsave(&chan->lock, flags);
679	list_splice_tail(list: &list, head: &chan->desc.chunks_free);
680	list_add_tail(new: &page->node, head: &chan->desc.pages);
681	spin_unlock_irqrestore(lock: &chan->lock, flags);
682
683	return 0;
684	}
685
686	/*
687	* rcar_dmac_xfer_chunk_get - Allocate a transfer chunk for a DMA transfer
688	* @chan: the DMA channel
689	*
690	* Locking: This function must be called in a non-atomic context.
691	*
692	* Return: A pointer to the allocated transfer chunk descriptor or NULL if no
693	* descriptor can be allocated.
694	*/
695	static struct rcar_dmac_xfer_chunk *
696	rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan)
697	{
698	struct rcar_dmac_xfer_chunk *chunk;
699	unsigned long flags;
700	int ret;
701
702	spin_lock_irqsave(&chan->lock, flags);
703
704	while (list_empty(head: &chan->desc.chunks_free)) {
705	/*
706	* No free descriptors, allocate a page worth of them and try
707	* again, as someone else could race us to get the newly
708	* allocated descriptors. If the allocation fails return an
709	* error.
710	*/
711	spin_unlock_irqrestore(lock: &chan->lock, flags);
712	ret = rcar_dmac_xfer_chunk_alloc(chan, GFP_NOWAIT);
713	if (ret < 0)
714	return NULL;
715	spin_lock_irqsave(&chan->lock, flags);
716	}
717
718	chunk = list_first_entry(&chan->desc.chunks_free,
719	struct rcar_dmac_xfer_chunk, node);
720	list_del(entry: &chunk->node);
721
722	spin_unlock_irqrestore(lock: &chan->lock, flags);
723
724	return chunk;
725	}
726
727	static void rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan,
728	struct rcar_dmac_desc *desc, size_t size)
729	{
730	/*
731	* dma_alloc_coherent() allocates memory in page size increments. To
732	* avoid reallocating the hardware descriptors when the allocated size
733	* wouldn't change align the requested size to a multiple of the page
734	* size.
735	*/
736	size = PAGE_ALIGN(size);
737
738	if (desc->hwdescs.size == size)
739	return;
740
741	if (desc->hwdescs.mem) {
742	dma_free_coherent(dev: chan->chan.device->dev, size: desc->hwdescs.size,
743	cpu_addr: desc->hwdescs.mem, dma_handle: desc->hwdescs.dma);
744	desc->hwdescs.mem = NULL;
745	desc->hwdescs.size = 0;
746	}
747
748	if (!size)
749	return;
750
751	desc->hwdescs.mem = dma_alloc_coherent(dev: chan->chan.device->dev, size,
752	dma_handle: &desc->hwdescs.dma, GFP_NOWAIT);
753	if (!desc->hwdescs.mem)
754	return;
755
756	desc->hwdescs.size = size;
757	}
758
759	static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan,
760	struct rcar_dmac_desc *desc)
761	{
762	struct rcar_dmac_xfer_chunk *chunk;
763	struct rcar_dmac_hw_desc *hwdesc;
764
765	rcar_dmac_realloc_hwdesc(chan, desc, size: desc->nchunks * sizeof(*hwdesc));
766
767	hwdesc = desc->hwdescs.mem;
768	if (!hwdesc)
769	return -ENOMEM;
770
771	list_for_each_entry(chunk, &desc->chunks, node) {
772	hwdesc->sar = chunk->src_addr;
773	hwdesc->dar = chunk->dst_addr;
774	hwdesc->tcr = chunk->size >> desc->xfer_shift;
775	hwdesc++;
776	}
777
778	return 0;
779	}
780
781	/* -----------------------------------------------------------------------------
782	* Stop and reset
783	*/
784	static void rcar_dmac_chcr_de_barrier(struct rcar_dmac_chan *chan)
785	{
786	u32 chcr;
787	unsigned int i;
788
789	/*
790	* Ensure that the setting of the DE bit is actually 0 after
791	* clearing it.
792	*/
793	for (i = 0; i < 1024; i++) {
794	chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
795	if (!(chcr & RCAR_DMACHCR_DE))
796	return;
797	udelay(1);
798	}
799
800	dev_err(chan->chan.device->dev, "CHCR DE check error\n");
801	}
802
803	static void rcar_dmac_clear_chcr_de(struct rcar_dmac_chan *chan)
804	{
805	u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
806
807	/* set DE=0 and flush remaining data */
808	rcar_dmac_chan_write(chan, RCAR_DMACHCR, data: (chcr & ~RCAR_DMACHCR_DE));
809
810	/* make sure all remaining data was flushed */
811	rcar_dmac_chcr_de_barrier(chan);
812	}
813
814	static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan)
815	{
816	u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
817
818	chcr &= ~(RCAR_DMACHCR_DSE | RCAR_DMACHCR_DSIE | RCAR_DMACHCR_IE |
819	RCAR_DMACHCR_TE | RCAR_DMACHCR_DE |
820	RCAR_DMACHCR_CAE | RCAR_DMACHCR_CAIE);
821	rcar_dmac_chan_write(chan, RCAR_DMACHCR, data: chcr);
822	rcar_dmac_chcr_de_barrier(chan);
823	}
824
825	static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan)
826	{
827	struct rcar_dmac_desc *desc, *_desc;
828	unsigned long flags;
829	LIST_HEAD(descs);
830
831	spin_lock_irqsave(&chan->lock, flags);
832
833	/* Move all non-free descriptors to the local lists. */
834	list_splice_init(list: &chan->desc.pending, head: &descs);
835	list_splice_init(list: &chan->desc.active, head: &descs);
836	list_splice_init(list: &chan->desc.done, head: &descs);
837	list_splice_init(list: &chan->desc.wait, head: &descs);
838
839	chan->desc.running = NULL;
840
841	spin_unlock_irqrestore(lock: &chan->lock, flags);
842
843	list_for_each_entry_safe(desc, _desc, &descs, node) {
844	list_del(entry: &desc->node);
845	rcar_dmac_desc_put(chan, desc);
846	}
847	}
848
849	static void rcar_dmac_stop_all_chan(struct rcar_dmac *dmac)
850	{
851	struct rcar_dmac_chan *chan;
852	unsigned int i;
853
854	/* Stop all channels. */
855	for_each_rcar_dmac_chan(i, dmac, chan) {
856	/* Stop and reinitialize the channel. */
857	spin_lock_irq(lock: &chan->lock);
858	rcar_dmac_chan_halt(chan);
859	spin_unlock_irq(lock: &chan->lock);
860	}
861	}
862
863	static int rcar_dmac_chan_pause(struct dma_chan *chan)
864	{
865	unsigned long flags;
866	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
867
868	spin_lock_irqsave(&rchan->lock, flags);
869	rcar_dmac_clear_chcr_de(chan: rchan);
870	spin_unlock_irqrestore(lock: &rchan->lock, flags);
871
872	return 0;
873	}
874
875	/* -----------------------------------------------------------------------------
876	* Descriptors preparation
877	*/
878
879	static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan,
880	struct rcar_dmac_desc *desc)
881	{
882	static const u32 chcr_ts[] = {
883	RCAR_DMACHCR_TS_1B, RCAR_DMACHCR_TS_2B,
884	RCAR_DMACHCR_TS_4B, RCAR_DMACHCR_TS_8B,
885	RCAR_DMACHCR_TS_16B, RCAR_DMACHCR_TS_32B,
886	RCAR_DMACHCR_TS_64B,
887	};
888
889	unsigned int xfer_size;
890	u32 chcr;
891
892	switch (desc->direction) {
893	case DMA_DEV_TO_MEM:
894	chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_FIXED
895	| RCAR_DMACHCR_RS_DMARS;
896	xfer_size = chan->src.xfer_size;
897	break;
898
899	case DMA_MEM_TO_DEV:
900	chcr = RCAR_DMACHCR_DM_FIXED | RCAR_DMACHCR_SM_INC
901	| RCAR_DMACHCR_RS_DMARS;
902	xfer_size = chan->dst.xfer_size;
903	break;
904
905	case DMA_MEM_TO_MEM:
906	default:
907	chcr = RCAR_DMACHCR_DM_INC | RCAR_DMACHCR_SM_INC
908	| RCAR_DMACHCR_RS_AUTO;
909	xfer_size = RCAR_DMAC_MEMCPY_XFER_SIZE;
910	break;
911	}
912
913	desc->xfer_shift = ilog2(xfer_size);
914	desc->chcr = chcr | chcr_ts[desc->xfer_shift];
915	}
916
917	/*
918	* rcar_dmac_chan_prep_sg - prepare transfer descriptors from an SG list
919	*
920	* Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also
921	* converted to scatter-gather to guarantee consistent locking and a correct
922	* list manipulation. For slave DMA direction carries the usual meaning, and,
923	* logically, the SG list is RAM and the addr variable contains slave address,
924	* e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM
925	* and the SG list contains only one element and points at the source buffer.
926	*/
927	static struct dma_async_tx_descriptor *
928	rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl,
929	unsigned int sg_len, dma_addr_t dev_addr,
930	enum dma_transfer_direction dir, unsigned long dma_flags,
931	bool cyclic)
932	{
933	struct rcar_dmac_xfer_chunk *chunk;
934	struct rcar_dmac_desc *desc;
935	struct scatterlist *sg;
936	unsigned int nchunks = 0;
937	unsigned int max_chunk_size;
938	unsigned int full_size = 0;
939	bool cross_boundary = false;
940	unsigned int i;
941	#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
942	u32 high_dev_addr;
943	u32 high_mem_addr;
944	#endif
945
946	desc = rcar_dmac_desc_get(chan);
947	if (!desc)
948	return NULL;
949
950	desc->async_tx.flags = dma_flags;
951	desc->async_tx.cookie = -EBUSY;
952
953	desc->cyclic = cyclic;
954	desc->direction = dir;
955
956	rcar_dmac_chan_configure_desc(chan, desc);
957
958	max_chunk_size = RCAR_DMATCR_MASK << desc->xfer_shift;
959
960	/*
961	* Allocate and fill the transfer chunk descriptors. We own the only
962	* reference to the DMA descriptor, there's no need for locking.
963	*/
964	for_each_sg(sgl, sg, sg_len, i) {
965	dma_addr_t mem_addr = sg_dma_address(sg);
966	unsigned int len = sg_dma_len(sg);
967
968	full_size += len;
969
970	#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
971	if (i == 0) {
972	high_dev_addr = dev_addr >> 32;
973	high_mem_addr = mem_addr >> 32;
974	}
975
976	if ((dev_addr >> 32 != high_dev_addr) ||
977	(mem_addr >> 32 != high_mem_addr))
978	cross_boundary = true;
979	#endif
980	while (len) {
981	unsigned int size = min(len, max_chunk_size);
982
983	#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
984	/*
985	* Prevent individual transfers from crossing 4GB
986	* boundaries.
987	*/
988	if (dev_addr >> 32 != (dev_addr + size - 1) >> 32) {
989	size = ALIGN(dev_addr, 1ULL << 32) - dev_addr;
990	cross_boundary = true;
991	}
992	if (mem_addr >> 32 != (mem_addr + size - 1) >> 32) {
993	size = ALIGN(mem_addr, 1ULL << 32) - mem_addr;
994	cross_boundary = true;
995	}
996	#endif
997
998	chunk = rcar_dmac_xfer_chunk_get(chan);
999	if (!chunk) {
1000	rcar_dmac_desc_put(chan, desc);
1001	return NULL;
1002	}
1003
1004	if (dir == DMA_DEV_TO_MEM) {
1005	chunk->src_addr = dev_addr;
1006	chunk->dst_addr = mem_addr;
1007	} else {
1008	chunk->src_addr = mem_addr;
1009	chunk->dst_addr = dev_addr;
1010	}
1011
1012	chunk->size = size;
1013
1014	dev_dbg(chan->chan.device->dev,
1015	"chan%u: chunk %p/%p sgl %u@%p, %u/%u %pad -> %pad\n",
1016	chan->index, chunk, desc, i, sg, size, len,
1017	&chunk->src_addr, &chunk->dst_addr);
1018
1019	mem_addr += size;
1020	if (dir == DMA_MEM_TO_MEM)
1021	dev_addr += size;
1022
1023	len -= size;
1024
1025	list_add_tail(new: &chunk->node, head: &desc->chunks);
1026	nchunks++;
1027	}
1028	}
1029
1030	desc->nchunks = nchunks;
1031	desc->size = full_size;
1032
1033	/*
1034	* Use hardware descriptor lists if possible when more than one chunk
1035	* needs to be transferred (otherwise they don't make much sense).
1036	*
1037	* Source/Destination address should be located in same 4GiB region
1038	* in the 40bit address space when it uses Hardware descriptor,
1039	* and cross_boundary is checking it.
1040	*/
1041	desc->hwdescs.use = !cross_boundary && nchunks > 1;
1042	if (desc->hwdescs.use) {
1043	if (rcar_dmac_fill_hwdesc(chan, desc) < 0)
1044	desc->hwdescs.use = false;
1045	}
1046
1047	return &desc->async_tx;
1048	}
1049
1050	/* -----------------------------------------------------------------------------
1051	* DMA engine operations
1052	*/
1053
1054	static int rcar_dmac_alloc_chan_resources(struct dma_chan *chan)
1055	{
1056	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1057	int ret;
1058
1059	INIT_LIST_HEAD(list: &rchan->desc.chunks_free);
1060	INIT_LIST_HEAD(list: &rchan->desc.pages);
1061
1062	/* Preallocate descriptors. */
1063	ret = rcar_dmac_xfer_chunk_alloc(chan: rchan, GFP_KERNEL);
1064	if (ret < 0)
1065	return -ENOMEM;
1066
1067	ret = rcar_dmac_desc_alloc(chan: rchan, GFP_KERNEL);
1068	if (ret < 0)
1069	return -ENOMEM;
1070
1071	return pm_runtime_get_sync(dev: chan->device->dev);
1072	}
1073
1074	static void rcar_dmac_free_chan_resources(struct dma_chan *chan)
1075	{
1076	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1077	struct rcar_dmac *dmac = to_rcar_dmac(chan->device);
1078	struct rcar_dmac_chan_map *map = &rchan->map;
1079	struct rcar_dmac_desc_page *page, *_page;
1080	struct rcar_dmac_desc *desc;
1081	LIST_HEAD(list);
1082
1083	/* Protect against ISR */
1084	spin_lock_irq(lock: &rchan->lock);
1085	rcar_dmac_chan_halt(chan: rchan);
1086	spin_unlock_irq(lock: &rchan->lock);
1087
1088	/*
1089	* Now no new interrupts will occur, but one might already be
1090	* running. Wait for it to finish before freeing resources.
1091	*/
1092	synchronize_irq(irq: rchan->irq);
1093
1094	if (rchan->mid_rid >= 0) {
1095	/* The caller is holding dma_list_mutex */
1096	clear_bit(nr: rchan->mid_rid, addr: dmac->modules);
1097	rchan->mid_rid = -EINVAL;
1098	}
1099
1100	list_splice_init(list: &rchan->desc.free, head: &list);
1101	list_splice_init(list: &rchan->desc.pending, head: &list);
1102	list_splice_init(list: &rchan->desc.active, head: &list);
1103	list_splice_init(list: &rchan->desc.done, head: &list);
1104	list_splice_init(list: &rchan->desc.wait, head: &list);
1105
1106	rchan->desc.running = NULL;
1107
1108	list_for_each_entry(desc, &list, node)
1109	rcar_dmac_realloc_hwdesc(chan: rchan, desc, size: 0);
1110
1111	list_for_each_entry_safe(page, _page, &rchan->desc.pages, node) {
1112	list_del(entry: &page->node);
1113	free_page((unsigned long)page);
1114	}
1115
1116	/* Remove slave mapping if present. */
1117	if (map->slave.xfer_size) {
1118	dma_unmap_resource(dev: chan->device->dev, addr: map->addr,
1119	size: map->slave.xfer_size, dir: map->dir, attrs: 0);
1120	map->slave.xfer_size = 0;
1121	}
1122
1123	pm_runtime_put(dev: chan->device->dev);
1124	}
1125
1126	static struct dma_async_tx_descriptor *
1127	rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
1128	dma_addr_t dma_src, size_t len, unsigned long flags)
1129	{
1130	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1131	struct scatterlist sgl;
1132
1133	if (!len)
1134	return NULL;
1135
1136	sg_init_table(&sgl, 1);
1137	sg_set_page(sg: &sgl, pfn_to_page(PFN_DOWN(dma_src)), len,
1138	offset_in_page(dma_src));
1139	sg_dma_address(&sgl) = dma_src;
1140	sg_dma_len(&sgl) = len;
1141
1142	return rcar_dmac_chan_prep_sg(chan: rchan, sgl: &sgl, sg_len: 1, dev_addr: dma_dest,
1143	dir: DMA_MEM_TO_MEM, dma_flags: flags, cyclic: false);
1144	}
1145
1146	static int rcar_dmac_map_slave_addr(struct dma_chan *chan,
1147	enum dma_transfer_direction dir)
1148	{
1149	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1150	struct rcar_dmac_chan_map *map = &rchan->map;
1151	phys_addr_t dev_addr;
1152	size_t dev_size;
1153	enum dma_data_direction dev_dir;
1154
1155	if (dir == DMA_DEV_TO_MEM) {
1156	dev_addr = rchan->src.slave_addr;
1157	dev_size = rchan->src.xfer_size;
1158	dev_dir = DMA_TO_DEVICE;
1159	} else {
1160	dev_addr = rchan->dst.slave_addr;
1161	dev_size = rchan->dst.xfer_size;
1162	dev_dir = DMA_FROM_DEVICE;
1163	}
1164
1165	/* Reuse current map if possible. */
1166	if (dev_addr == map->slave.slave_addr &&
1167	dev_size == map->slave.xfer_size &&
1168	dev_dir == map->dir)
1169	return 0;
1170
1171	/* Remove old mapping if present. */
1172	if (map->slave.xfer_size)
1173	dma_unmap_resource(dev: chan->device->dev, addr: map->addr,
1174	size: map->slave.xfer_size, dir: map->dir, attrs: 0);
1175	map->slave.xfer_size = 0;
1176
1177	/* Create new slave address map. */
1178	map->addr = dma_map_resource(dev: chan->device->dev, phys_addr: dev_addr, size: dev_size,
1179	dir: dev_dir, attrs: 0);
1180
1181	if (dma_mapping_error(dev: chan->device->dev, dma_addr: map->addr)) {
1182	dev_err(chan->device->dev,
1183	"chan%u: failed to map %zx@%pap", rchan->index,
1184	dev_size, &dev_addr);
1185	return -EIO;
1186	}
1187
1188	dev_dbg(chan->device->dev, "chan%u: map %zx@%pap to %pad dir: %s\n",
1189	rchan->index, dev_size, &dev_addr, &map->addr,
1190	dev_dir == DMA_TO_DEVICE ? "DMA_TO_DEVICE" : "DMA_FROM_DEVICE");
1191
1192	map->slave.slave_addr = dev_addr;
1193	map->slave.xfer_size = dev_size;
1194	map->dir = dev_dir;
1195
1196	return 0;
1197	}
1198
1199	static struct dma_async_tx_descriptor *
1200	rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
1201	unsigned int sg_len, enum dma_transfer_direction dir,
1202	unsigned long flags, void *context)
1203	{
1204	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1205
1206	/* Someone calling slave DMA on a generic channel? */
1207	if (rchan->mid_rid < 0 || !sg_len || !sg_dma_len(sgl)) {
1208	dev_warn(chan->device->dev,
1209	"%s: bad parameter: len=%d, id=%d\n",
1210	__func__, sg_len, rchan->mid_rid);
1211	return NULL;
1212	}
1213
1214	if (rcar_dmac_map_slave_addr(chan, dir))
1215	return NULL;
1216
1217	return rcar_dmac_chan_prep_sg(chan: rchan, sgl, sg_len, dev_addr: rchan->map.addr,
1218	dir, dma_flags: flags, cyclic: false);
1219	}
1220
1221	#define RCAR_DMAC_MAX_SG_LEN 32
1222
1223	static struct dma_async_tx_descriptor *
1224	rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
1225	size_t buf_len, size_t period_len,
1226	enum dma_transfer_direction dir, unsigned long flags)
1227	{
1228	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1229	struct dma_async_tx_descriptor *desc;
1230	struct scatterlist *sgl;
1231	unsigned int sg_len;
1232	unsigned int i;
1233
1234	/* Someone calling slave DMA on a generic channel? */
1235	if (rchan->mid_rid < 0 || buf_len < period_len) {
1236	dev_warn(chan->device->dev,
1237	"%s: bad parameter: buf_len=%zu, period_len=%zu, id=%d\n",
1238	__func__, buf_len, period_len, rchan->mid_rid);
1239	return NULL;
1240	}
1241
1242	if (rcar_dmac_map_slave_addr(chan, dir))
1243	return NULL;
1244
1245	sg_len = buf_len / period_len;
1246	if (sg_len > RCAR_DMAC_MAX_SG_LEN) {
1247	dev_err(chan->device->dev,
1248	"chan%u: sg length %d exceeds limit %d",
1249	rchan->index, sg_len, RCAR_DMAC_MAX_SG_LEN);
1250	return NULL;
1251	}
1252
1253	/*
1254	* Allocate the sg list dynamically as it would consume too much stack
1255	* space.
1256	*/
1257	sgl = kmalloc_array(n: sg_len, size: sizeof(*sgl), GFP_NOWAIT);
1258	if (!sgl)
1259	return NULL;
1260
1261	sg_init_table(sgl, sg_len);
1262
1263	for (i = 0; i < sg_len; ++i) {
1264	dma_addr_t src = buf_addr + (period_len * i);
1265
1266	sg_set_page(sg: &sgl[i], pfn_to_page(PFN_DOWN(src)), len: period_len,
1267	offset_in_page(src));
1268	sg_dma_address(&sgl[i]) = src;
1269	sg_dma_len(&sgl[i]) = period_len;
1270	}
1271
1272	desc = rcar_dmac_chan_prep_sg(chan: rchan, sgl, sg_len, dev_addr: rchan->map.addr,
1273	dir, dma_flags: flags, cyclic: true);
1274
1275	kfree(objp: sgl);
1276	return desc;
1277	}
1278
1279	static int rcar_dmac_device_config(struct dma_chan *chan,
1280	struct dma_slave_config *cfg)
1281	{
1282	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1283
1284	/*
1285	* We could lock this, but you shouldn't be configuring the
1286	* channel, while using it...
1287	*/
1288	rchan->src.slave_addr = cfg->src_addr;
1289	rchan->dst.slave_addr = cfg->dst_addr;
1290	rchan->src.xfer_size = cfg->src_addr_width;
1291	rchan->dst.xfer_size = cfg->dst_addr_width;
1292
1293	return 0;
1294	}
1295
1296	static int rcar_dmac_chan_terminate_all(struct dma_chan *chan)
1297	{
1298	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1299	unsigned long flags;
1300
1301	spin_lock_irqsave(&rchan->lock, flags);
1302	rcar_dmac_chan_halt(chan: rchan);
1303	spin_unlock_irqrestore(lock: &rchan->lock, flags);
1304
1305	/*
1306	* FIXME: No new interrupt can occur now, but the IRQ thread might still
1307	* be running.
1308	*/
1309
1310	rcar_dmac_chan_reinit(chan: rchan);
1311
1312	return 0;
1313	}
1314
1315	static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan,
1316	dma_cookie_t cookie)
1317	{
1318	struct rcar_dmac_desc *desc = chan->desc.running;
1319	struct rcar_dmac_xfer_chunk *running = NULL;
1320	struct rcar_dmac_xfer_chunk *chunk;
1321	enum dma_status status;
1322	unsigned int residue = 0;
1323	unsigned int dptr = 0;
1324	unsigned int chcrb;
1325	unsigned int tcrb;
1326	unsigned int i;
1327
1328	if (!desc)
1329	return 0;
1330
1331	/*
1332	* If the cookie corresponds to a descriptor that has been completed
1333	* there is no residue. The same check has already been performed by the
1334	* caller but without holding the channel lock, so the descriptor could
1335	* now be complete.
1336	*/
1337	status = dma_cookie_status(chan: &chan->chan, cookie, NULL);
1338	if (status == DMA_COMPLETE)
1339	return 0;
1340
1341	/*
1342	* If the cookie doesn't correspond to the currently running transfer
1343	* then the descriptor hasn't been processed yet, and the residue is
1344	* equal to the full descriptor size.
1345	* Also, a client driver is possible to call this function before
1346	* rcar_dmac_isr_channel_thread() runs. In this case, the "desc.running"
1347	* will be the next descriptor, and the done list will appear. So, if
1348	* the argument cookie matches the done list's cookie, we can assume
1349	* the residue is zero.
1350	*/
1351	if (cookie != desc->async_tx.cookie) {
1352	list_for_each_entry(desc, &chan->desc.done, node) {
1353	if (cookie == desc->async_tx.cookie)
1354	return 0;
1355	}
1356	list_for_each_entry(desc, &chan->desc.pending, node) {
1357	if (cookie == desc->async_tx.cookie)
1358	return desc->size;
1359	}
1360	list_for_each_entry(desc, &chan->desc.active, node) {
1361	if (cookie == desc->async_tx.cookie)
1362	return desc->size;
1363	}
1364
1365	/*
1366	* No descriptor found for the cookie, there's thus no residue.
1367	* This shouldn't happen if the calling driver passes a correct
1368	* cookie value.
1369	*/
1370	WARN(1, "No descriptor for cookie!");
1371	return 0;
1372	}
1373
1374	/*
1375	* We need to read two registers.
1376	* Make sure the control register does not skip to next chunk
1377	* while reading the counter.
1378	* Trying it 3 times should be enough: Initial read, retry, retry
1379	* for the paranoid.
1380	*/
1381	for (i = 0; i < 3; i++) {
1382	chcrb = rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
1383	RCAR_DMACHCRB_DPTR_MASK;
1384	tcrb = rcar_dmac_chan_read(chan, RCAR_DMATCRB);
1385	/* Still the same? */
1386	if (chcrb == (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
1387	RCAR_DMACHCRB_DPTR_MASK))
1388	break;
1389	}
1390	WARN_ONCE(i >= 3, "residue might be not continuous!");
1391
1392	/*
1393	* In descriptor mode the descriptor running pointer is not maintained
1394	* by the interrupt handler, find the running descriptor from the
1395	* descriptor pointer field in the CHCRB register. In non-descriptor
1396	* mode just use the running descriptor pointer.
1397	*/
1398	if (desc->hwdescs.use) {
1399	dptr = chcrb >> RCAR_DMACHCRB_DPTR_SHIFT;
1400	if (dptr == 0)
1401	dptr = desc->nchunks;
1402	dptr--;
1403	WARN_ON(dptr >= desc->nchunks);
1404	} else {
1405	running = desc->running;
1406	}
1407
1408	/* Compute the size of all chunks still to be transferred. */
1409	list_for_each_entry_reverse(chunk, &desc->chunks, node) {
1410	if (chunk == running || ++dptr == desc->nchunks)
1411	break;
1412
1413	residue += chunk->size;
1414	}
1415
1416	/* Add the residue for the current chunk. */
1417	residue += tcrb << desc->xfer_shift;
1418
1419	return residue;
1420	}
1421
1422	static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan,
1423	dma_cookie_t cookie,
1424	struct dma_tx_state *txstate)
1425	{
1426	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1427	enum dma_status status;
1428	unsigned long flags;
1429	unsigned int residue;
1430	bool cyclic;
1431
1432	status = dma_cookie_status(chan, cookie, state: txstate);
1433	if (status == DMA_COMPLETE || !txstate)
1434	return status;
1435
1436	spin_lock_irqsave(&rchan->lock, flags);
1437	residue = rcar_dmac_chan_get_residue(chan: rchan, cookie);
1438	cyclic = rchan->desc.running ? rchan->desc.running->cyclic : false;
1439	spin_unlock_irqrestore(lock: &rchan->lock, flags);
1440
1441	/* if there's no residue, the cookie is complete */
1442	if (!residue && !cyclic)
1443	return DMA_COMPLETE;
1444
1445	dma_set_residue(state: txstate, residue);
1446
1447	return status;
1448	}
1449
1450	static void rcar_dmac_issue_pending(struct dma_chan *chan)
1451	{
1452	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1453	unsigned long flags;
1454
1455	spin_lock_irqsave(&rchan->lock, flags);
1456
1457	if (list_empty(head: &rchan->desc.pending))
1458	goto done;
1459
1460	/* Append the pending list to the active list. */
1461	list_splice_tail_init(list: &rchan->desc.pending, head: &rchan->desc.active);
1462
1463	/*
1464	* If no transfer is running pick the first descriptor from the active
1465	* list and start the transfer.
1466	*/
1467	if (!rchan->desc.running) {
1468	struct rcar_dmac_desc *desc;
1469
1470	desc = list_first_entry(&rchan->desc.active,
1471	struct rcar_dmac_desc, node);
1472	rchan->desc.running = desc;
1473
1474	rcar_dmac_chan_start_xfer(chan: rchan);
1475	}
1476
1477	done:
1478	spin_unlock_irqrestore(lock: &rchan->lock, flags);
1479	}
1480
1481	static void rcar_dmac_device_synchronize(struct dma_chan *chan)
1482	{
1483	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);
1484
1485	synchronize_irq(irq: rchan->irq);
1486	}
1487
1488	/* -----------------------------------------------------------------------------
1489	* IRQ handling
1490	*/
1491
1492	static irqreturn_t rcar_dmac_isr_desc_stage_end(struct rcar_dmac_chan *chan)
1493	{
1494	struct rcar_dmac_desc *desc = chan->desc.running;
1495	unsigned int stage;
1496
1497	if (WARN_ON(!desc || !desc->cyclic)) {
1498	/*
1499	* This should never happen, there should always be a running
1500	* cyclic descriptor when a descriptor stage end interrupt is
1501	* triggered. Warn and return.
1502	*/
1503	return IRQ_NONE;
1504	}
1505
1506	/* Program the interrupt pointer to the next stage. */
1507	stage = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &
1508	RCAR_DMACHCRB_DPTR_MASK) >> RCAR_DMACHCRB_DPTR_SHIFT;
1509	rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(stage));
1510
1511	return IRQ_WAKE_THREAD;
1512	}
1513
1514	static irqreturn_t rcar_dmac_isr_transfer_end(struct rcar_dmac_chan *chan)
1515	{
1516	struct rcar_dmac_desc *desc = chan->desc.running;
1517	irqreturn_t ret = IRQ_WAKE_THREAD;
1518
1519	if (WARN_ON_ONCE(!desc)) {
1520	/*
1521	* This should never happen, there should always be a running
1522	* descriptor when a transfer end interrupt is triggered. Warn
1523	* and return.
1524	*/
1525	return IRQ_NONE;
1526	}
1527
1528	/*
1529	* The transfer end interrupt isn't generated for each chunk when using
1530	* descriptor mode. Only update the running chunk pointer in
1531	* non-descriptor mode.
1532	*/
1533	if (!desc->hwdescs.use) {
1534	/*
1535	* If we haven't completed the last transfer chunk simply move
1536	* to the next one. Only wake the IRQ thread if the transfer is
1537	* cyclic.
1538	*/
1539	if (!list_is_last(list: &desc->running->node, head: &desc->chunks)) {
1540	desc->running = list_next_entry(desc->running, node);
1541	if (!desc->cyclic)
1542	ret = IRQ_HANDLED;
1543	goto done;
1544	}
1545
1546	/*
1547	* We've completed the last transfer chunk. If the transfer is
1548	* cyclic, move back to the first one.
1549	*/
1550	if (desc->cyclic) {
1551	desc->running =
1552	list_first_entry(&desc->chunks,
1553	struct rcar_dmac_xfer_chunk,
1554	node);
1555	goto done;
1556	}
1557	}
1558
1559	/* The descriptor is complete, move it to the done list. */
1560	list_move_tail(list: &desc->node, head: &chan->desc.done);
1561
1562	/* Queue the next descriptor, if any. */
1563	if (!list_empty(head: &chan->desc.active))
1564	chan->desc.running = list_first_entry(&chan->desc.active,
1565	struct rcar_dmac_desc,
1566	node);
1567	else
1568	chan->desc.running = NULL;
1569
1570	done:
1571	if (chan->desc.running)
1572	rcar_dmac_chan_start_xfer(chan);
1573
1574	return ret;
1575	}
1576
1577	static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev)
1578	{
1579	u32 mask = RCAR_DMACHCR_DSE | RCAR_DMACHCR_TE;
1580	struct rcar_dmac_chan *chan = dev;
1581	irqreturn_t ret = IRQ_NONE;
1582	bool reinit = false;
1583	u32 chcr;
1584
1585	spin_lock(lock: &chan->lock);
1586
1587	chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
1588	if (chcr & RCAR_DMACHCR_CAE) {
1589	struct rcar_dmac *dmac = to_rcar_dmac(chan->chan.device);
1590
1591	/*
1592	* We don't need to call rcar_dmac_chan_halt()
1593	* because channel is already stopped in error case.
1594	* We need to clear register and check DE bit as recovery.
1595	*/
1596	rcar_dmac_chan_clear(dmac, chan);
1597	rcar_dmac_chcr_de_barrier(chan);
1598	reinit = true;
1599	goto spin_lock_end;
1600	}
1601
1602	if (chcr & RCAR_DMACHCR_TE)
1603	mask |= RCAR_DMACHCR_DE;
1604	rcar_dmac_chan_write(chan, RCAR_DMACHCR, data: chcr & ~mask);
1605	if (mask & RCAR_DMACHCR_DE)
1606	rcar_dmac_chcr_de_barrier(chan);
1607
1608	if (chcr & RCAR_DMACHCR_DSE)
1609	ret |= rcar_dmac_isr_desc_stage_end(chan);
1610
1611	if (chcr & RCAR_DMACHCR_TE)
1612	ret |= rcar_dmac_isr_transfer_end(chan);
1613
1614	spin_lock_end:
1615	spin_unlock(lock: &chan->lock);
1616
1617	if (reinit) {
1618	dev_err(chan->chan.device->dev, "Channel Address Error\n");
1619
1620	rcar_dmac_chan_reinit(chan);
1621	ret = IRQ_HANDLED;
1622	}
1623
1624	return ret;
1625	}
1626
1627	static irqreturn_t rcar_dmac_isr_channel_thread(int irq, void *dev)
1628	{
1629	struct rcar_dmac_chan *chan = dev;
1630	struct rcar_dmac_desc *desc;
1631	struct dmaengine_desc_callback cb;
1632
1633	spin_lock_irq(lock: &chan->lock);
1634
1635	/* For cyclic transfers notify the user after every chunk. */
1636	if (chan->desc.running && chan->desc.running->cyclic) {
1637	desc = chan->desc.running;
1638	dmaengine_desc_get_callback(tx: &desc->async_tx, cb: &cb);
1639
1640	if (dmaengine_desc_callback_valid(cb: &cb)) {
1641	spin_unlock_irq(lock: &chan->lock);
1642	dmaengine_desc_callback_invoke(cb: &cb, NULL);
1643	spin_lock_irq(lock: &chan->lock);
1644	}
1645	}
1646
1647	/*
1648	* Call the callback function for all descriptors on the done list and
1649	* move them to the ack wait list.
1650	*/
1651	while (!list_empty(head: &chan->desc.done)) {
1652	desc = list_first_entry(&chan->desc.done, struct rcar_dmac_desc,
1653	node);
1654	dma_cookie_complete(tx: &desc->async_tx);
1655	list_del(entry: &desc->node);
1656
1657	dmaengine_desc_get_callback(tx: &desc->async_tx, cb: &cb);
1658	if (dmaengine_desc_callback_valid(cb: &cb)) {
1659	spin_unlock_irq(lock: &chan->lock);
1660	/*
1661	* We own the only reference to this descriptor, we can
1662	* safely dereference it without holding the channel
1663	* lock.
1664	*/
1665	dmaengine_desc_callback_invoke(cb: &cb, NULL);
1666	spin_lock_irq(lock: &chan->lock);
1667	}
1668
1669	list_add_tail(new: &desc->node, head: &chan->desc.wait);
1670	}
1671
1672	spin_unlock_irq(lock: &chan->lock);
1673
1674	/* Recycle all acked descriptors. */
1675	rcar_dmac_desc_recycle_acked(chan);
1676
1677	return IRQ_HANDLED;
1678	}
1679
1680	/* -----------------------------------------------------------------------------
1681	* OF xlate and channel filter
1682	*/
1683
1684	static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg)
1685	{
1686	struct rcar_dmac *dmac = to_rcar_dmac(chan->device);
1687	struct of_phandle_args *dma_spec = arg;
1688
1689	/*
1690	* FIXME: Using a filter on OF platforms is a nonsense. The OF xlate
1691	* function knows from which device it wants to allocate a channel from,
1692	* and would be perfectly capable of selecting the channel it wants.
1693	* Forcing it to call dma_request_channel() and iterate through all
1694	* channels from all controllers is just pointless.
1695	*/
1696	if (chan->device->device_config != rcar_dmac_device_config)
1697	return false;
1698
1699	return !test_and_set_bit(nr: dma_spec->args[0], addr: dmac->modules);
1700	}
1701
1702	static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec,
1703	struct of_dma *ofdma)
1704	{
1705	struct rcar_dmac_chan *rchan;
1706	struct dma_chan *chan;
1707	dma_cap_mask_t mask;
1708
1709	if (dma_spec->args_count != 1)
1710	return NULL;
1711
1712	/* Only slave DMA channels can be allocated via DT */
1713	dma_cap_zero(mask);
1714	dma_cap_set(DMA_SLAVE, mask);
1715
1716	chan = __dma_request_channel(mask: &mask, fn: rcar_dmac_chan_filter, fn_param: dma_spec,
1717	np: ofdma->of_node);
1718	if (!chan)
1719	return NULL;
1720
1721	rchan = to_rcar_dmac_chan(chan);
1722	rchan->mid_rid = dma_spec->args[0];
1723
1724	return chan;
1725	}
1726
1727	/* -----------------------------------------------------------------------------
1728	* Power management
1729	*/
1730
1731	#ifdef CONFIG_PM
1732	static int rcar_dmac_runtime_suspend(struct device *dev)
1733	{
1734	return 0;
1735	}
1736
1737	static int rcar_dmac_runtime_resume(struct device *dev)
1738	{
1739	struct rcar_dmac *dmac = dev_get_drvdata(dev);
1740
1741	return rcar_dmac_init(dmac);
1742	}
1743	#endif
1744
1745	static const struct dev_pm_ops rcar_dmac_pm = {
1746	/*
1747	* TODO for system sleep/resume:
1748	* - Wait for the current transfer to complete and stop the device,
1749	* - Resume transfers, if any.
1750	*/
1751	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
1752	pm_runtime_force_resume)
1753	SET_RUNTIME_PM_OPS(rcar_dmac_runtime_suspend, rcar_dmac_runtime_resume,
1754	NULL)
1755	};
1756
1757	/* -----------------------------------------------------------------------------
1758	* Probe and remove
1759	*/
1760
1761	static int rcar_dmac_chan_probe(struct rcar_dmac *dmac,
1762	struct rcar_dmac_chan *rchan)
1763	{
1764	struct platform_device *pdev = to_platform_device(dmac->dev);
1765	struct dma_chan *chan = &rchan->chan;
1766	char pdev_irqname[5];
1767	char *irqname;
1768	int ret;
1769
1770	rchan->mid_rid = -EINVAL;
1771
1772	spin_lock_init(&rchan->lock);
1773
1774	INIT_LIST_HEAD(list: &rchan->desc.free);
1775	INIT_LIST_HEAD(list: &rchan->desc.pending);
1776	INIT_LIST_HEAD(list: &rchan->desc.active);
1777	INIT_LIST_HEAD(list: &rchan->desc.done);
1778	INIT_LIST_HEAD(list: &rchan->desc.wait);
1779
1780	/* Request the channel interrupt. */
1781	sprintf(buf: pdev_irqname, fmt: "ch%u", rchan->index);
1782	rchan->irq = platform_get_irq_byname(pdev, pdev_irqname);
1783	if (rchan->irq < 0)
1784	return -ENODEV;
1785
1786	irqname = devm_kasprintf(dev: dmac->dev, GFP_KERNEL, fmt: "%s:%u",
1787	dev_name(dev: dmac->dev), rchan->index);
1788	if (!irqname)
1789	return -ENOMEM;
1790
1791	/*
1792	* Initialize the DMA engine channel and add it to the DMA engine
1793	* channels list.
1794	*/
1795	chan->device = &dmac->engine;
1796	dma_cookie_init(chan);
1797
1798	list_add_tail(new: &chan->device_node, head: &dmac->engine.channels);
1799
1800	ret = devm_request_threaded_irq(dev: dmac->dev, irq: rchan->irq,
1801	handler: rcar_dmac_isr_channel,
1802	thread_fn: rcar_dmac_isr_channel_thread, irqflags: 0,
1803	devname: irqname, dev_id: rchan);
1804	if (ret) {
1805	dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",
1806	rchan->irq, ret);
1807	return ret;
1808	}
1809
1810	return 0;
1811	}
1812
1813	#define RCAR_DMAC_MAX_CHANNELS 32
1814
1815	static int rcar_dmac_parse_of(struct device *dev, struct rcar_dmac *dmac)
1816	{
1817	struct device_node *np = dev->of_node;
1818	int ret;
1819
1820	ret = of_property_read_u32(np, propname: "dma-channels", out_value: &dmac->n_channels);
1821	if (ret < 0) {
1822	dev_err(dev, "unable to read dma-channels property\n");
1823	return ret;
1824	}
1825
1826	/* The hardware and driver don't support more than 32 bits in CHCLR */
1827	if (dmac->n_channels <= 0 ||
1828	dmac->n_channels >= RCAR_DMAC_MAX_CHANNELS) {
1829	dev_err(dev, "invalid number of channels %u\n",
1830	dmac->n_channels);
1831	return -EINVAL;
1832	}
1833
1834	/*
1835	* If the driver is unable to read dma-channel-mask property,
1836	* the driver assumes that it can use all channels.
1837	*/
1838	dmac->channels_mask = GENMASK(dmac->n_channels - 1, 0);
1839	of_property_read_u32(np, propname: "dma-channel-mask", out_value: &dmac->channels_mask);
1840
1841	/* If the property has out-of-channel mask, this driver clears it */
1842	dmac->channels_mask &= GENMASK(dmac->n_channels - 1, 0);
1843
1844	return 0;
1845	}
1846
1847	static int rcar_dmac_probe(struct platform_device *pdev)
1848	{
1849	const enum dma_slave_buswidth widths = DMA_SLAVE_BUSWIDTH_1_BYTE |
1850	DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES |
1851	DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES |
1852	DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES;
1853	const struct rcar_dmac_of_data *data;
1854	struct rcar_dmac_chan *chan;
1855	struct dma_device *engine;
1856	void __iomem *chan_base;
1857	struct rcar_dmac *dmac;
1858	unsigned int i;
1859	int ret;
1860
1861	data = of_device_get_match_data(dev: &pdev->dev);
1862	if (!data)
1863	return -EINVAL;
1864
1865	dmac = devm_kzalloc(dev: &pdev->dev, size: sizeof(*dmac), GFP_KERNEL);
1866	if (!dmac)
1867	return -ENOMEM;
1868
1869	dmac->dev = &pdev->dev;
1870	platform_set_drvdata(pdev, data: dmac);
1871	ret = dma_set_max_seg_size(dev: dmac->dev, RCAR_DMATCR_MASK);
1872	if (ret)
1873	return ret;
1874
1875	ret = dma_set_mask_and_coherent(dev: dmac->dev, DMA_BIT_MASK(40));
1876	if (ret)
1877	return ret;
1878
1879	ret = rcar_dmac_parse_of(dev: &pdev->dev, dmac);
1880	if (ret < 0)
1881	return ret;
1882
1883	/*
1884	* A still unconfirmed hardware bug prevents the IPMMU microTLB 0 to be
1885	* flushed correctly, resulting in memory corruption. DMAC 0 channel 0
1886	* is connected to microTLB 0 on currently supported platforms, so we
1887	* can't use it with the IPMMU. As the IOMMU API operates at the device
1888	* level we can't disable it selectively, so ignore channel 0 for now if
1889	* the device is part of an IOMMU group.
1890	*/
1891	if (device_iommu_mapped(dev: &pdev->dev))
1892	dmac->channels_mask &= ~BIT(0);
1893
1894	dmac->channels = devm_kcalloc(dev: &pdev->dev, n: dmac->n_channels,
1895	size: sizeof(*dmac->channels), GFP_KERNEL);
1896	if (!dmac->channels)
1897	return -ENOMEM;
1898
1899	/* Request resources. */
1900	dmac->dmac_base = devm_platform_ioremap_resource(pdev, index: 0);
1901	if (IS_ERR(ptr: dmac->dmac_base))
1902	return PTR_ERR(ptr: dmac->dmac_base);
1903
1904	if (!data->chan_offset_base) {
1905	dmac->chan_base = devm_platform_ioremap_resource(pdev, index: 1);
1906	if (IS_ERR(ptr: dmac->chan_base))
1907	return PTR_ERR(ptr: dmac->chan_base);
1908
1909	chan_base = dmac->chan_base;
1910	} else {
1911	chan_base = dmac->dmac_base + data->chan_offset_base;
1912	}
1913
1914	for_each_rcar_dmac_chan(i, dmac, chan) {
1915	chan->index = i;
1916	chan->iomem = chan_base + i * data->chan_offset_stride;
1917	}
1918
1919	/* Enable runtime PM and initialize the device. */
1920	pm_runtime_enable(dev: &pdev->dev);
1921	ret = pm_runtime_resume_and_get(dev: &pdev->dev);
1922	if (ret < 0) {
1923	dev_err(&pdev->dev, "runtime PM get sync failed (%d)\n", ret);
1924	goto err_pm_disable;
1925	}
1926
1927	ret = rcar_dmac_init(dmac);
1928	pm_runtime_put(dev: &pdev->dev);
1929
1930	if (ret) {
1931	dev_err(&pdev->dev, "failed to reset device\n");
1932	goto err_pm_disable;
1933	}
1934
1935	/* Initialize engine */
1936	engine = &dmac->engine;
1937
1938	dma_cap_set(DMA_MEMCPY, engine->cap_mask);
1939	dma_cap_set(DMA_SLAVE, engine->cap_mask);
1940
1941	engine->dev = &pdev->dev;
1942	engine->copy_align = ilog2(RCAR_DMAC_MEMCPY_XFER_SIZE);
1943
1944	engine->src_addr_widths = widths;
1945	engine->dst_addr_widths = widths;
1946	engine->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
1947	engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1948
1949	engine->device_alloc_chan_resources = rcar_dmac_alloc_chan_resources;
1950	engine->device_free_chan_resources = rcar_dmac_free_chan_resources;
1951	engine->device_prep_dma_memcpy = rcar_dmac_prep_dma_memcpy;
1952	engine->device_prep_slave_sg = rcar_dmac_prep_slave_sg;
1953	engine->device_prep_dma_cyclic = rcar_dmac_prep_dma_cyclic;
1954	engine->device_config = rcar_dmac_device_config;
1955	engine->device_pause = rcar_dmac_chan_pause;
1956	engine->device_terminate_all = rcar_dmac_chan_terminate_all;
1957	engine->device_tx_status = rcar_dmac_tx_status;
1958	engine->device_issue_pending = rcar_dmac_issue_pending;
1959	engine->device_synchronize = rcar_dmac_device_synchronize;
1960
1961	INIT_LIST_HEAD(list: &engine->channels);
1962
1963	for_each_rcar_dmac_chan(i, dmac, chan) {
1964	ret = rcar_dmac_chan_probe(dmac, rchan: chan);
1965	if (ret < 0)
1966	goto err_pm_disable;
1967	}
1968
1969	/* Register the DMAC as a DMA provider for DT. */
1970	ret = of_dma_controller_register(np: pdev->dev.of_node, of_dma_xlate: rcar_dmac_of_xlate,
1971	NULL);
1972	if (ret < 0)
1973	goto err_pm_disable;
1974
1975	/*
1976	* Register the DMA engine device.
1977	*
1978	* Default transfer size of 32 bytes requires 32-byte alignment.
1979	*/
1980	ret = dma_async_device_register(device: engine);
1981	if (ret < 0)
1982	goto err_dma_free;
1983
1984	return 0;
1985
1986	err_dma_free:
1987	of_dma_controller_free(np: pdev->dev.of_node);
1988	err_pm_disable:
1989	pm_runtime_disable(dev: &pdev->dev);
1990	return ret;
1991	}
1992
1993	static void rcar_dmac_remove(struct platform_device *pdev)
1994	{
1995	struct rcar_dmac *dmac = platform_get_drvdata(pdev);
1996
1997	of_dma_controller_free(np: pdev->dev.of_node);
1998	dma_async_device_unregister(device: &dmac->engine);
1999
2000	pm_runtime_disable(dev: &pdev->dev);
2001	}
2002
2003	static void rcar_dmac_shutdown(struct platform_device *pdev)
2004	{
2005	struct rcar_dmac *dmac = platform_get_drvdata(pdev);
2006
2007	rcar_dmac_stop_all_chan(dmac);
2008	}
2009
2010	static const struct rcar_dmac_of_data rcar_dmac_data = {
2011	.chan_offset_base = 0x8000,
2012	.chan_offset_stride = 0x80,
2013	};
2014
2015	static const struct rcar_dmac_of_data rcar_gen4_dmac_data = {
2016	.chan_offset_base = 0x0,
2017	.chan_offset_stride = 0x1000,
2018	};
2019
2020	static const struct of_device_id rcar_dmac_of_ids[] = {
2021	{
2022	.compatible = "renesas,rcar-dmac",
2023	.data = &rcar_dmac_data,
2024	}, {
2025	.compatible = "renesas,rcar-gen4-dmac",
2026	.data = &rcar_gen4_dmac_data,
2027	}, {
2028	.compatible = "renesas,dmac-r8a779a0",
2029	.data = &rcar_gen4_dmac_data,
2030	},
2031	{ /* Sentinel */ }
2032	};
2033	MODULE_DEVICE_TABLE(of, rcar_dmac_of_ids);
2034
2035	static struct platform_driver rcar_dmac_driver = {
2036	.driver = {
2037	.pm = &rcar_dmac_pm,
2038	.name = "rcar-dmac",
2039	.of_match_table = rcar_dmac_of_ids,
2040	},
2041	.probe = rcar_dmac_probe,
2042	.remove_new = rcar_dmac_remove,
2043	.shutdown = rcar_dmac_shutdown,
2044	};
2045
2046	module_platform_driver(rcar_dmac_driver);
2047
2048	MODULE_DESCRIPTION("R-Car Gen2 DMA Controller Driver");
2049	MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
2050	MODULE_LICENSE("GPL v2");
2051