1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	*
4	* Copyright (C) STMicroelectronics SA 2017
5	* Author(s): M'boumba Cedric Madianga <cedric.madianga@gmail.com>
6	* Pierre-Yves Mordret <pierre-yves.mordret@st.com>
7	*
8	* Driver for STM32 MDMA controller
9	*
10	* Inspired by stm32-dma.c and dma-jz4780.c
11	*/
12
13	#include <linux/bitfield.h>
14	#include <linux/clk.h>
15	#include <linux/delay.h>
16	#include <linux/dmaengine.h>
17	#include <linux/dma-mapping.h>
18	#include <linux/dmapool.h>
19	#include <linux/err.h>
20	#include <linux/init.h>
21	#include <linux/iopoll.h>
22	#include <linux/jiffies.h>
23	#include <linux/list.h>
24	#include <linux/log2.h>
25	#include <linux/module.h>
26	#include <linux/of.h>
27	#include <linux/of_dma.h>
28	#include <linux/platform_device.h>
29	#include <linux/pm_runtime.h>
30	#include <linux/reset.h>
31	#include <linux/slab.h>
32
33	#include "virt-dma.h"
34
35	#define STM32_MDMA_GISR0 0x0000 /* MDMA Int Status Reg 1 */
36
37	/* MDMA Channel x interrupt/status register */
38	#define STM32_MDMA_CISR(x) (0x40 + 0x40 * (x)) /* x = 0..62 */
39	#define STM32_MDMA_CISR_CRQA BIT(16)
40	#define STM32_MDMA_CISR_TCIF BIT(4)
41	#define STM32_MDMA_CISR_BTIF BIT(3)
42	#define STM32_MDMA_CISR_BRTIF BIT(2)
43	#define STM32_MDMA_CISR_CTCIF BIT(1)
44	#define STM32_MDMA_CISR_TEIF BIT(0)
45
46	/* MDMA Channel x interrupt flag clear register */
47	#define STM32_MDMA_CIFCR(x) (0x44 + 0x40 * (x))
48	#define STM32_MDMA_CIFCR_CLTCIF BIT(4)
49	#define STM32_MDMA_CIFCR_CBTIF BIT(3)
50	#define STM32_MDMA_CIFCR_CBRTIF BIT(2)
51	#define STM32_MDMA_CIFCR_CCTCIF BIT(1)
52	#define STM32_MDMA_CIFCR_CTEIF BIT(0)
53	#define STM32_MDMA_CIFCR_CLEAR_ALL (STM32_MDMA_CIFCR_CLTCIF \
54	| STM32_MDMA_CIFCR_CBTIF \
55	| STM32_MDMA_CIFCR_CBRTIF \
56	| STM32_MDMA_CIFCR_CCTCIF \
57	| STM32_MDMA_CIFCR_CTEIF)
58
59	/* MDMA Channel x error status register */
60	#define STM32_MDMA_CESR(x) (0x48 + 0x40 * (x))
61	#define STM32_MDMA_CESR_BSE BIT(11)
62	#define STM32_MDMA_CESR_ASR BIT(10)
63	#define STM32_MDMA_CESR_TEMD BIT(9)
64	#define STM32_MDMA_CESR_TELD BIT(8)
65	#define STM32_MDMA_CESR_TED BIT(7)
66	#define STM32_MDMA_CESR_TEA_MASK GENMASK(6, 0)
67
68	/* MDMA Channel x control register */
69	#define STM32_MDMA_CCR(x) (0x4C + 0x40 * (x))
70	#define STM32_MDMA_CCR_SWRQ BIT(16)
71	#define STM32_MDMA_CCR_WEX BIT(14)
72	#define STM32_MDMA_CCR_HEX BIT(13)
73	#define STM32_MDMA_CCR_BEX BIT(12)
74	#define STM32_MDMA_CCR_SM BIT(8)
75	#define STM32_MDMA_CCR_PL_MASK GENMASK(7, 6)
76	#define STM32_MDMA_CCR_PL(n) FIELD_PREP(STM32_MDMA_CCR_PL_MASK, (n))
77	#define STM32_MDMA_CCR_TCIE BIT(5)
78	#define STM32_MDMA_CCR_BTIE BIT(4)
79	#define STM32_MDMA_CCR_BRTIE BIT(3)
80	#define STM32_MDMA_CCR_CTCIE BIT(2)
81	#define STM32_MDMA_CCR_TEIE BIT(1)
82	#define STM32_MDMA_CCR_EN BIT(0)
83	#define STM32_MDMA_CCR_IRQ_MASK (STM32_MDMA_CCR_TCIE \
84	| STM32_MDMA_CCR_BTIE \
85	| STM32_MDMA_CCR_BRTIE \
86	| STM32_MDMA_CCR_CTCIE \
87	| STM32_MDMA_CCR_TEIE)
88
89	/* MDMA Channel x transfer configuration register */
90	#define STM32_MDMA_CTCR(x) (0x50 + 0x40 * (x))
91	#define STM32_MDMA_CTCR_BWM BIT(31)
92	#define STM32_MDMA_CTCR_SWRM BIT(30)
93	#define STM32_MDMA_CTCR_TRGM_MSK GENMASK(29, 28)
94	#define STM32_MDMA_CTCR_TRGM(n) FIELD_PREP(STM32_MDMA_CTCR_TRGM_MSK, (n))
95	#define STM32_MDMA_CTCR_TRGM_GET(n) FIELD_GET(STM32_MDMA_CTCR_TRGM_MSK, (n))
96	#define STM32_MDMA_CTCR_PAM_MASK GENMASK(27, 26)
97	#define STM32_MDMA_CTCR_PAM(n) FIELD_PREP(STM32_MDMA_CTCR_PAM_MASK, (n))
98	#define STM32_MDMA_CTCR_PKE BIT(25)
99	#define STM32_MDMA_CTCR_TLEN_MSK GENMASK(24, 18)
100	#define STM32_MDMA_CTCR_TLEN(n) FIELD_PREP(STM32_MDMA_CTCR_TLEN_MSK, (n))
101	#define STM32_MDMA_CTCR_TLEN_GET(n) FIELD_GET(STM32_MDMA_CTCR_TLEN_MSK, (n))
102	#define STM32_MDMA_CTCR_LEN2_MSK GENMASK(25, 18)
103	#define STM32_MDMA_CTCR_LEN2(n) FIELD_PREP(STM32_MDMA_CTCR_LEN2_MSK, (n))
104	#define STM32_MDMA_CTCR_LEN2_GET(n) FIELD_GET(STM32_MDMA_CTCR_LEN2_MSK, (n))
105	#define STM32_MDMA_CTCR_DBURST_MASK GENMASK(17, 15)
106	#define STM32_MDMA_CTCR_DBURST(n) FIELD_PREP(STM32_MDMA_CTCR_DBURST_MASK, (n))
107	#define STM32_MDMA_CTCR_SBURST_MASK GENMASK(14, 12)
108	#define STM32_MDMA_CTCR_SBURST(n) FIELD_PREP(STM32_MDMA_CTCR_SBURST_MASK, (n))
109	#define STM32_MDMA_CTCR_DINCOS_MASK GENMASK(11, 10)
110	#define STM32_MDMA_CTCR_DINCOS(n) FIELD_PREP(STM32_MDMA_CTCR_DINCOS_MASK, (n))
111	#define STM32_MDMA_CTCR_SINCOS_MASK GENMASK(9, 8)
112	#define STM32_MDMA_CTCR_SINCOS(n) FIELD_PREP(STM32_MDMA_CTCR_SINCOS_MASK, (n))
113	#define STM32_MDMA_CTCR_DSIZE_MASK GENMASK(7, 6)
114	#define STM32_MDMA_CTCR_DSIZE(n) FIELD_PREP(STM32_MDMA_CTCR_DSIZE_MASK, (n))
115	#define STM32_MDMA_CTCR_SSIZE_MASK GENMASK(5, 4)
116	#define STM32_MDMA_CTCR_SSIZE(n) FIELD_PREP(STM32_MDMA_CTCR_SSIZE_MASK, (n))
117	#define STM32_MDMA_CTCR_DINC_MASK GENMASK(3, 2)
118	#define STM32_MDMA_CTCR_DINC(n) FIELD_PREP(STM32_MDMA_CTCR_DINC_MASK, (n))
119	#define STM32_MDMA_CTCR_SINC_MASK GENMASK(1, 0)
120	#define STM32_MDMA_CTCR_SINC(n) FIELD_PREP(STM32_MDMA_CTCR_SINC_MASK, (n))
121	#define STM32_MDMA_CTCR_CFG_MASK (STM32_MDMA_CTCR_SINC_MASK \
122	| STM32_MDMA_CTCR_DINC_MASK \
123	| STM32_MDMA_CTCR_SINCOS_MASK \
124	| STM32_MDMA_CTCR_DINCOS_MASK \
125	| STM32_MDMA_CTCR_LEN2_MSK \
126	| STM32_MDMA_CTCR_TRGM_MSK)
127
128	/* MDMA Channel x block number of data register */
129	#define STM32_MDMA_CBNDTR(x) (0x54 + 0x40 * (x))
130	#define STM32_MDMA_CBNDTR_BRC_MK GENMASK(31, 20)
131	#define STM32_MDMA_CBNDTR_BRC(n) FIELD_PREP(STM32_MDMA_CBNDTR_BRC_MK, (n))
132	#define STM32_MDMA_CBNDTR_BRC_GET(n) FIELD_GET(STM32_MDMA_CBNDTR_BRC_MK, (n))
133
134	#define STM32_MDMA_CBNDTR_BRDUM BIT(19)
135	#define STM32_MDMA_CBNDTR_BRSUM BIT(18)
136	#define STM32_MDMA_CBNDTR_BNDT_MASK GENMASK(16, 0)
137	#define STM32_MDMA_CBNDTR_BNDT(n) FIELD_PREP(STM32_MDMA_CBNDTR_BNDT_MASK, (n))
138
139	/* MDMA Channel x source address register */
140	#define STM32_MDMA_CSAR(x) (0x58 + 0x40 * (x))
141
142	/* MDMA Channel x destination address register */
143	#define STM32_MDMA_CDAR(x) (0x5C + 0x40 * (x))
144
145	/* MDMA Channel x block repeat address update register */
146	#define STM32_MDMA_CBRUR(x) (0x60 + 0x40 * (x))
147	#define STM32_MDMA_CBRUR_DUV_MASK GENMASK(31, 16)
148	#define STM32_MDMA_CBRUR_DUV(n) FIELD_PREP(STM32_MDMA_CBRUR_DUV_MASK, (n))
149	#define STM32_MDMA_CBRUR_SUV_MASK GENMASK(15, 0)
150	#define STM32_MDMA_CBRUR_SUV(n) FIELD_PREP(STM32_MDMA_CBRUR_SUV_MASK, (n))
151
152	/* MDMA Channel x link address register */
153	#define STM32_MDMA_CLAR(x) (0x64 + 0x40 * (x))
154
155	/* MDMA Channel x trigger and bus selection register */
156	#define STM32_MDMA_CTBR(x) (0x68 + 0x40 * (x))
157	#define STM32_MDMA_CTBR_DBUS BIT(17)
158	#define STM32_MDMA_CTBR_SBUS BIT(16)
159	#define STM32_MDMA_CTBR_TSEL_MASK GENMASK(5, 0)
160	#define STM32_MDMA_CTBR_TSEL(n) FIELD_PREP(STM32_MDMA_CTBR_TSEL_MASK, (n))
161
162	/* MDMA Channel x mask address register */
163	#define STM32_MDMA_CMAR(x) (0x70 + 0x40 * (x))
164
165	/* MDMA Channel x mask data register */
166	#define STM32_MDMA_CMDR(x) (0x74 + 0x40 * (x))
167
168	#define STM32_MDMA_MAX_BUF_LEN 128
169	#define STM32_MDMA_MAX_BLOCK_LEN 65536
170	#define STM32_MDMA_MAX_CHANNELS 32
171	#define STM32_MDMA_MAX_REQUESTS 256
172	#define STM32_MDMA_MAX_BURST 128
173	#define STM32_MDMA_VERY_HIGH_PRIORITY 0x3
174
175	enum stm32_mdma_trigger_mode {
176	STM32_MDMA_BUFFER,
177	STM32_MDMA_BLOCK,
178	STM32_MDMA_BLOCK_REP,
179	STM32_MDMA_LINKED_LIST,
180	};
181
182	enum stm32_mdma_width {
183	STM32_MDMA_BYTE,
184	STM32_MDMA_HALF_WORD,
185	STM32_MDMA_WORD,
186	STM32_MDMA_DOUBLE_WORD,
187	};
188
189	enum stm32_mdma_inc_mode {
190	STM32_MDMA_FIXED = 0,
191	STM32_MDMA_INC = 2,
192	STM32_MDMA_DEC = 3,
193	};
194
195	struct stm32_mdma_chan_config {
196	u32 request;
197	u32 priority_level;
198	u32 transfer_config;
199	u32 mask_addr;
200	u32 mask_data;
201	bool m2m_hw; /* True when MDMA is triggered by STM32 DMA */
202	};
203
204	struct stm32_mdma_hwdesc {
205	u32 ctcr;
206	u32 cbndtr;
207	u32 csar;
208	u32 cdar;
209	u32 cbrur;
210	u32 clar;
211	u32 ctbr;
212	u32 dummy;
213	u32 cmar;
214	u32 cmdr;
215	} __aligned(64);
216
217	struct stm32_mdma_desc_node {
218	struct stm32_mdma_hwdesc *hwdesc;
219	dma_addr_t hwdesc_phys;
220	};
221
222	struct stm32_mdma_desc {
223	struct virt_dma_desc vdesc;
224	u32 ccr;
225	bool cyclic;
226	u32 count;
227	struct stm32_mdma_desc_node node[] __counted_by(count);
228	};
229
230	struct stm32_mdma_dma_config {
231	u32 request; /* STM32 DMA channel stream id, triggering MDMA */
232	u32 cmar; /* STM32 DMA interrupt flag clear register address */
233	u32 cmdr; /* STM32 DMA Transfer Complete flag */
234	};
235
236	struct stm32_mdma_chan {
237	struct virt_dma_chan vchan;
238	struct dma_pool *desc_pool;
239	u32 id;
240	struct stm32_mdma_desc *desc;
241	u32 curr_hwdesc;
242	struct dma_slave_config dma_config;
243	struct stm32_mdma_chan_config chan_config;
244	bool busy;
245	u32 mem_burst;
246	u32 mem_width;
247	};
248
249	struct stm32_mdma_device {
250	struct dma_device ddev;
251	void __iomem *base;
252	struct clk *clk;
253	int irq;
254	u32 nr_channels;
255	u32 nr_requests;
256	u32 nr_ahb_addr_masks;
257	u32 chan_reserved;
258	struct stm32_mdma_chan chan[STM32_MDMA_MAX_CHANNELS];
259	u32 ahb_addr_masks[] __counted_by(nr_ahb_addr_masks);
260	};
261
262	static struct stm32_mdma_device *stm32_mdma_get_dev(
263	struct stm32_mdma_chan *chan)
264	{
265	return container_of(chan->vchan.chan.device, struct stm32_mdma_device,
266	ddev);
267	}
268
269	static struct stm32_mdma_chan *to_stm32_mdma_chan(struct dma_chan *c)
270	{
271	return container_of(c, struct stm32_mdma_chan, vchan.chan);
272	}
273
274	static struct stm32_mdma_desc *to_stm32_mdma_desc(struct virt_dma_desc *vdesc)
275	{
276	return container_of(vdesc, struct stm32_mdma_desc, vdesc);
277	}
278
279	static struct device *chan2dev(struct stm32_mdma_chan *chan)
280	{
281	return &chan->vchan.chan.dev->device;
282	}
283
284	static struct device *mdma2dev(struct stm32_mdma_device *mdma_dev)
285	{
286	return mdma_dev->ddev.dev;
287	}
288
289	static u32 stm32_mdma_read(struct stm32_mdma_device *dmadev, u32 reg)
290	{
291	return readl_relaxed(dmadev->base + reg);
292	}
293
294	static void stm32_mdma_write(struct stm32_mdma_device *dmadev, u32 reg, u32 val)
295	{
296	writel_relaxed(val, dmadev->base + reg);
297	}
298
299	static void stm32_mdma_set_bits(struct stm32_mdma_device *dmadev, u32 reg,
300	u32 mask)
301	{
302	void __iomem *addr = dmadev->base + reg;
303
304	writel_relaxed(readl_relaxed(addr) | mask, addr);
305	}
306
307	static void stm32_mdma_clr_bits(struct stm32_mdma_device *dmadev, u32 reg,
308	u32 mask)
309	{
310	void __iomem *addr = dmadev->base + reg;
311
312	writel_relaxed(readl_relaxed(addr) & ~mask, addr);
313	}
314
315	static struct stm32_mdma_desc *stm32_mdma_alloc_desc(
316	struct stm32_mdma_chan *chan, u32 count)
317	{
318	struct stm32_mdma_desc *desc;
319	int i;
320
321	desc = kzalloc(struct_size(desc, node, count), GFP_NOWAIT);
322	if (!desc)
323	return NULL;
324	desc->count = count;
325
326	for (i = 0; i < count; i++) {
327	desc->node[i].hwdesc =
328	dma_pool_alloc(pool: chan->desc_pool, GFP_NOWAIT,
329	handle: &desc->node[i].hwdesc_phys);
330	if (!desc->node[i].hwdesc)
331	goto err;
332	}
333
334	return desc;
335
336	err:
337	dev_err(chan2dev(chan), "Failed to allocate descriptor\n");
338	while (--i >= 0)
339	dma_pool_free(pool: chan->desc_pool, vaddr: desc->node[i].hwdesc,
340	addr: desc->node[i].hwdesc_phys);
341	kfree(objp: desc);
342	return NULL;
343	}
344
345	static void stm32_mdma_desc_free(struct virt_dma_desc *vdesc)
346	{
347	struct stm32_mdma_desc *desc = to_stm32_mdma_desc(vdesc);
348	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c: vdesc->tx.chan);
349	int i;
350
351	for (i = 0; i < desc->count; i++)
352	dma_pool_free(pool: chan->desc_pool, vaddr: desc->node[i].hwdesc,
353	addr: desc->node[i].hwdesc_phys);
354	kfree(objp: desc);
355	}
356
357	static int stm32_mdma_get_width(struct stm32_mdma_chan *chan,
358	enum dma_slave_buswidth width)
359	{
360	switch (width) {
361	case DMA_SLAVE_BUSWIDTH_1_BYTE:
362	case DMA_SLAVE_BUSWIDTH_2_BYTES:
363	case DMA_SLAVE_BUSWIDTH_4_BYTES:
364	case DMA_SLAVE_BUSWIDTH_8_BYTES:
365	return ffs(width) - 1;
366	default:
367	dev_err(chan2dev(chan), "Dma bus width %i not supported\n",
368	width);
369	return -EINVAL;
370	}
371	}
372
373	static enum dma_slave_buswidth stm32_mdma_get_max_width(dma_addr_t addr,
374	u32 buf_len, u32 tlen)
375	{
376	enum dma_slave_buswidth max_width = DMA_SLAVE_BUSWIDTH_8_BYTES;
377
378	for (max_width = DMA_SLAVE_BUSWIDTH_8_BYTES;
379	max_width > DMA_SLAVE_BUSWIDTH_1_BYTE;
380	max_width >>= 1) {
381	/*
382	* Address and buffer length both have to be aligned on
383	* bus width
384	*/
385	if ((((buf_len | addr) & (max_width - 1)) == 0) &&
386	tlen >= max_width)
387	break;
388	}
389
390	return max_width;
391	}
392
393	static u32 stm32_mdma_get_best_burst(u32 buf_len, u32 tlen, u32 max_burst,
394	enum dma_slave_buswidth width)
395	{
396	u32 best_burst;
397
398	best_burst = min((u32)1 << __ffs(tlen | buf_len),
399	max_burst * width) / width;
400
401	return (best_burst > 0) ? best_burst : 1;
402	}
403
404	static int stm32_mdma_disable_chan(struct stm32_mdma_chan *chan)
405	{
406	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
407	u32 ccr, cisr, id, reg;
408	int ret;
409
410	id = chan->id;
411	reg = STM32_MDMA_CCR(id);
412
413	/* Disable interrupts */
414	stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_IRQ_MASK);
415
416	ccr = stm32_mdma_read(dmadev, reg);
417	if (ccr & STM32_MDMA_CCR_EN) {
418	stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_EN);
419
420	/* Ensure that any ongoing transfer has been completed */
421	ret = readl_relaxed_poll_timeout_atomic(
422	dmadev->base + STM32_MDMA_CISR(id), cisr,
423	(cisr & STM32_MDMA_CISR_CTCIF), 10, 1000);
424	if (ret) {
425	dev_err(chan2dev(chan), "%s: timeout!\n", __func__);
426	return -EBUSY;
427	}
428	}
429
430	return 0;
431	}
432
433	static void stm32_mdma_stop(struct stm32_mdma_chan *chan)
434	{
435	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
436	u32 status;
437	int ret;
438
439	/* Disable DMA */
440	ret = stm32_mdma_disable_chan(chan);
441	if (ret < 0)
442	return;
443
444	/* Clear interrupt status if it is there */
445	status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
446	if (status) {
447	dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n",
448	__func__, status);
449	stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), mask: status);
450	}
451
452	chan->busy = false;
453	}
454
455	static void stm32_mdma_set_bus(struct stm32_mdma_device *dmadev, u32 *ctbr,
456	u32 ctbr_mask, u32 src_addr)
457	{
458	u32 mask;
459	int i;
460
461	/* Check if memory device is on AHB or AXI */
462	*ctbr &= ~ctbr_mask;
463	mask = src_addr & 0xF0000000;
464	for (i = 0; i < dmadev->nr_ahb_addr_masks; i++) {
465	if (mask == dmadev->ahb_addr_masks[i]) {
466	*ctbr |= ctbr_mask;
467	break;
468	}
469	}
470	}
471
472	static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan,
473	enum dma_transfer_direction direction,
474	u32 *mdma_ccr, u32 *mdma_ctcr,
475	u32 *mdma_ctbr, dma_addr_t addr,
476	u32 buf_len)
477	{
478	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
479	struct stm32_mdma_chan_config *chan_config = &chan->chan_config;
480	enum dma_slave_buswidth src_addr_width, dst_addr_width;
481	phys_addr_t src_addr, dst_addr;
482	int src_bus_width, dst_bus_width;
483	u32 src_maxburst, dst_maxburst, src_best_burst, dst_best_burst;
484	u32 ccr, ctcr, ctbr, tlen;
485
486	src_addr_width = chan->dma_config.src_addr_width;
487	dst_addr_width = chan->dma_config.dst_addr_width;
488	src_maxburst = chan->dma_config.src_maxburst;
489	dst_maxburst = chan->dma_config.dst_maxburst;
490
491	ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & ~STM32_MDMA_CCR_EN;
492	ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id));
493	ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id));
494
495	/* Enable HW request mode */
496	ctcr &= ~STM32_MDMA_CTCR_SWRM;
497
498	/* Set DINC, SINC, DINCOS, SINCOS, TRGM and TLEN retrieve from DT */
499	ctcr &= ~STM32_MDMA_CTCR_CFG_MASK;
500	ctcr |= chan_config->transfer_config & STM32_MDMA_CTCR_CFG_MASK;
501
502	/*
503	* For buffer transfer length (TLEN) we have to set
504	* the number of bytes - 1 in CTCR register
505	*/
506	tlen = STM32_MDMA_CTCR_LEN2_GET(ctcr);
507	ctcr &= ~STM32_MDMA_CTCR_LEN2_MSK;
508	ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1));
509
510	/* Disable Pack Enable */
511	ctcr &= ~STM32_MDMA_CTCR_PKE;
512
513	/* Check burst size constraints */
514	if (src_maxburst * src_addr_width > STM32_MDMA_MAX_BURST ||
515	dst_maxburst * dst_addr_width > STM32_MDMA_MAX_BURST) {
516	dev_err(chan2dev(chan),
517	"burst size * bus width higher than %d bytes\n",
518	STM32_MDMA_MAX_BURST);
519	return -EINVAL;
520	}
521
522	if ((!is_power_of_2(n: src_maxburst) && src_maxburst > 0) ||
523	(!is_power_of_2(n: dst_maxburst) && dst_maxburst > 0)) {
524	dev_err(chan2dev(chan), "burst size must be a power of 2\n");
525	return -EINVAL;
526	}
527
528	/*
529	* Configure channel control:
530	* - Clear SW request as in this case this is a HW one
531	* - Clear WEX, HEX and BEX bits
532	* - Set priority level
533	*/
534	ccr &= ~(STM32_MDMA_CCR_SWRQ | STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX |
535	STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK);
536	ccr |= STM32_MDMA_CCR_PL(chan_config->priority_level);
537
538	/* Configure Trigger selection */
539	ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK;
540	ctbr |= STM32_MDMA_CTBR_TSEL(chan_config->request);
541
542	switch (direction) {
543	case DMA_MEM_TO_DEV:
544	dst_addr = chan->dma_config.dst_addr;
545
546	/* Set device data size */
547	if (chan_config->m2m_hw)
548	dst_addr_width = stm32_mdma_get_max_width(addr: dst_addr, buf_len,
549	STM32_MDMA_MAX_BUF_LEN);
550	dst_bus_width = stm32_mdma_get_width(chan, width: dst_addr_width);
551	if (dst_bus_width < 0)
552	return dst_bus_width;
553	ctcr &= ~STM32_MDMA_CTCR_DSIZE_MASK;
554	ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width);
555	if (chan_config->m2m_hw) {
556	ctcr &= ~STM32_MDMA_CTCR_DINCOS_MASK;
557	ctcr |= STM32_MDMA_CTCR_DINCOS(dst_bus_width);
558	}
559
560	/* Set device burst value */
561	if (chan_config->m2m_hw)
562	dst_maxburst = STM32_MDMA_MAX_BUF_LEN / dst_addr_width;
563
564	dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
565	max_burst: dst_maxburst,
566	width: dst_addr_width);
567	chan->mem_burst = dst_best_burst;
568	ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK;
569	ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst)));
570
571	/* Set memory data size */
572	src_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen);
573	chan->mem_width = src_addr_width;
574	src_bus_width = stm32_mdma_get_width(chan, width: src_addr_width);
575	if (src_bus_width < 0)
576	return src_bus_width;
577	ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK |
578	STM32_MDMA_CTCR_SINCOS_MASK;
579	ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width) |
580	STM32_MDMA_CTCR_SINCOS(src_bus_width);
581
582	/* Set memory burst value */
583	src_maxburst = STM32_MDMA_MAX_BUF_LEN / src_addr_width;
584	src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
585	max_burst: src_maxburst,
586	width: src_addr_width);
587	chan->mem_burst = src_best_burst;
588	ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK;
589	ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst)));
590
591	/* Select bus */
592	stm32_mdma_set_bus(dmadev, ctbr: &ctbr, STM32_MDMA_CTBR_DBUS,
593	src_addr: dst_addr);
594
595	if (dst_bus_width != src_bus_width)
596	ctcr |= STM32_MDMA_CTCR_PKE;
597
598	/* Set destination address */
599	stm32_mdma_write(dmadev, STM32_MDMA_CDAR(chan->id), val: dst_addr);
600	break;
601
602	case DMA_DEV_TO_MEM:
603	src_addr = chan->dma_config.src_addr;
604
605	/* Set device data size */
606	if (chan_config->m2m_hw)
607	src_addr_width = stm32_mdma_get_max_width(addr: src_addr, buf_len,
608	STM32_MDMA_MAX_BUF_LEN);
609
610	src_bus_width = stm32_mdma_get_width(chan, width: src_addr_width);
611	if (src_bus_width < 0)
612	return src_bus_width;
613	ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK;
614	ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width);
615	if (chan_config->m2m_hw) {
616	ctcr &= ~STM32_MDMA_CTCR_SINCOS_MASK;
617	ctcr |= STM32_MDMA_CTCR_SINCOS(src_bus_width);
618	}
619
620	/* Set device burst value */
621	if (chan_config->m2m_hw)
622	src_maxburst = STM32_MDMA_MAX_BUF_LEN / src_addr_width;
623
624	src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
625	max_burst: src_maxburst,
626	width: src_addr_width);
627	ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK;
628	ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst)));
629
630	/* Set memory data size */
631	dst_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen);
632	chan->mem_width = dst_addr_width;
633	dst_bus_width = stm32_mdma_get_width(chan, width: dst_addr_width);
634	if (dst_bus_width < 0)
635	return dst_bus_width;
636	ctcr &= ~(STM32_MDMA_CTCR_DSIZE_MASK |
637	STM32_MDMA_CTCR_DINCOS_MASK);
638	ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width) |
639	STM32_MDMA_CTCR_DINCOS(dst_bus_width);
640
641	/* Set memory burst value */
642	dst_maxburst = STM32_MDMA_MAX_BUF_LEN / dst_addr_width;
643	dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
644	max_burst: dst_maxburst,
645	width: dst_addr_width);
646	ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK;
647	ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst)));
648
649	/* Select bus */
650	stm32_mdma_set_bus(dmadev, ctbr: &ctbr, STM32_MDMA_CTBR_SBUS,
651	src_addr);
652
653	if (dst_bus_width != src_bus_width)
654	ctcr |= STM32_MDMA_CTCR_PKE;
655
656	/* Set source address */
657	stm32_mdma_write(dmadev, STM32_MDMA_CSAR(chan->id), val: src_addr);
658	break;
659
660	default:
661	dev_err(chan2dev(chan), "Dma direction is not supported\n");
662	return -EINVAL;
663	}
664
665	*mdma_ccr = ccr;
666	*mdma_ctcr = ctcr;
667	*mdma_ctbr = ctbr;
668
669	return 0;
670	}
671
672	static void stm32_mdma_dump_hwdesc(struct stm32_mdma_chan *chan,
673	struct stm32_mdma_desc_node *node)
674	{
675	dev_dbg(chan2dev(chan), "hwdesc: %pad\n", &node->hwdesc_phys);
676	dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n", node->hwdesc->ctcr);
677	dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n", node->hwdesc->cbndtr);
678	dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n", node->hwdesc->csar);
679	dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n", node->hwdesc->cdar);
680	dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n", node->hwdesc->cbrur);
681	dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n", node->hwdesc->clar);
682	dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n", node->hwdesc->ctbr);
683	dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n", node->hwdesc->cmar);
684	dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n\n", node->hwdesc->cmdr);
685	}
686
687	static void stm32_mdma_setup_hwdesc(struct stm32_mdma_chan *chan,
688	struct stm32_mdma_desc *desc,
689	enum dma_transfer_direction dir, u32 count,
690	dma_addr_t src_addr, dma_addr_t dst_addr,
691	u32 len, u32 ctcr, u32 ctbr, bool is_last,
692	bool is_first, bool is_cyclic)
693	{
694	struct stm32_mdma_chan_config *config = &chan->chan_config;
695	struct stm32_mdma_hwdesc *hwdesc;
696	u32 next = count + 1;
697
698	hwdesc = desc->node[count].hwdesc;
699	hwdesc->ctcr = ctcr;
700	hwdesc->cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK |
701	STM32_MDMA_CBNDTR_BRDUM |
702	STM32_MDMA_CBNDTR_BRSUM |
703	STM32_MDMA_CBNDTR_BNDT_MASK);
704	hwdesc->cbndtr |= STM32_MDMA_CBNDTR_BNDT(len);
705	hwdesc->csar = src_addr;
706	hwdesc->cdar = dst_addr;
707	hwdesc->cbrur = 0;
708	hwdesc->ctbr = ctbr;
709	hwdesc->cmar = config->mask_addr;
710	hwdesc->cmdr = config->mask_data;
711
712	if (is_last) {
713	if (is_cyclic)
714	hwdesc->clar = desc->node[0].hwdesc_phys;
715	else
716	hwdesc->clar = 0;
717	} else {
718	hwdesc->clar = desc->node[next].hwdesc_phys;
719	}
720
721	stm32_mdma_dump_hwdesc(chan, node: &desc->node[count]);
722	}
723
724	static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan,
725	struct stm32_mdma_desc *desc,
726	struct scatterlist *sgl, u32 sg_len,
727	enum dma_transfer_direction direction)
728	{
729	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
730	struct dma_slave_config *dma_config = &chan->dma_config;
731	struct stm32_mdma_chan_config *chan_config = &chan->chan_config;
732	struct scatterlist *sg;
733	dma_addr_t src_addr, dst_addr;
734	u32 m2m_hw_period, ccr, ctcr, ctbr;
735	int i, ret = 0;
736
737	if (chan_config->m2m_hw)
738	m2m_hw_period = sg_dma_len(sgl);
739
740	for_each_sg(sgl, sg, sg_len, i) {
741	if (sg_dma_len(sg) > STM32_MDMA_MAX_BLOCK_LEN) {
742	dev_err(chan2dev(chan), "Invalid block len\n");
743	return -EINVAL;
744	}
745
746	if (direction == DMA_MEM_TO_DEV) {
747	src_addr = sg_dma_address(sg);
748	dst_addr = dma_config->dst_addr;
749	if (chan_config->m2m_hw && (i & 1))
750	dst_addr += m2m_hw_period;
751	ret = stm32_mdma_set_xfer_param(chan, direction, mdma_ccr: &ccr,
752	mdma_ctcr: &ctcr, mdma_ctbr: &ctbr, addr: src_addr,
753	sg_dma_len(sg));
754	stm32_mdma_set_bus(dmadev, ctbr: &ctbr, STM32_MDMA_CTBR_SBUS,
755	src_addr);
756	} else {
757	src_addr = dma_config->src_addr;
758	if (chan_config->m2m_hw && (i & 1))
759	src_addr += m2m_hw_period;
760	dst_addr = sg_dma_address(sg);
761	ret = stm32_mdma_set_xfer_param(chan, direction, mdma_ccr: &ccr,
762	mdma_ctcr: &ctcr, mdma_ctbr: &ctbr, addr: dst_addr,
763	sg_dma_len(sg));
764	stm32_mdma_set_bus(dmadev, ctbr: &ctbr, STM32_MDMA_CTBR_DBUS,
765	src_addr: dst_addr);
766	}
767
768	if (ret < 0)
769	return ret;
770
771	stm32_mdma_setup_hwdesc(chan, desc, dir: direction, count: i, src_addr,
772	dst_addr, sg_dma_len(sg), ctcr, ctbr,
773	is_last: i == sg_len - 1, is_first: i == 0, is_cyclic: false);
774	}
775
776	/* Enable interrupts */
777	ccr &= ~STM32_MDMA_CCR_IRQ_MASK;
778	ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE;
779	desc->ccr = ccr;
780
781	return 0;
782	}
783
784	static struct dma_async_tx_descriptor *
785	stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl,
786	u32 sg_len, enum dma_transfer_direction direction,
787	unsigned long flags, void *context)
788	{
789	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
790	struct stm32_mdma_chan_config *chan_config = &chan->chan_config;
791	struct stm32_mdma_desc *desc;
792	int i, ret;
793
794	/*
795	* Once DMA is in setup cyclic mode the channel we cannot assign this
796	* channel anymore. The DMA channel needs to be aborted or terminated
797	* for allowing another request.
798	*/
799	if (chan->desc && chan->desc->cyclic) {
800	dev_err(chan2dev(chan),
801	"Request not allowed when dma in cyclic mode\n");
802	return NULL;
803	}
804
805	desc = stm32_mdma_alloc_desc(chan, count: sg_len);
806	if (!desc)
807	return NULL;
808
809	ret = stm32_mdma_setup_xfer(chan, desc, sgl, sg_len, direction);
810	if (ret < 0)
811	goto xfer_setup_err;
812
813	/*
814	* In case of M2M HW transfer triggered by STM32 DMA, we do not have to clear the
815	* transfer complete flag by hardware in order to let the CPU rearm the STM32 DMA
816	* with the next sg element and update some data in dmaengine framework.
817	*/
818	if (chan_config->m2m_hw && direction == DMA_MEM_TO_DEV) {
819	struct stm32_mdma_hwdesc *hwdesc;
820
821	for (i = 0; i < sg_len; i++) {
822	hwdesc = desc->node[i].hwdesc;
823	hwdesc->cmar = 0;
824	hwdesc->cmdr = 0;
825	}
826	}
827
828	desc->cyclic = false;
829
830	return vchan_tx_prep(vc: &chan->vchan, vd: &desc->vdesc, tx_flags: flags);
831
832	xfer_setup_err:
833	for (i = 0; i < desc->count; i++)
834	dma_pool_free(pool: chan->desc_pool, vaddr: desc->node[i].hwdesc,
835	addr: desc->node[i].hwdesc_phys);
836	kfree(objp: desc);
837	return NULL;
838	}
839
840	static struct dma_async_tx_descriptor *
841	stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr,
842	size_t buf_len, size_t period_len,
843	enum dma_transfer_direction direction,
844	unsigned long flags)
845	{
846	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
847	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
848	struct dma_slave_config *dma_config = &chan->dma_config;
849	struct stm32_mdma_chan_config *chan_config = &chan->chan_config;
850	struct stm32_mdma_desc *desc;
851	dma_addr_t src_addr, dst_addr;
852	u32 ccr, ctcr, ctbr, count;
853	int i, ret;
854
855	/*
856	* Once DMA is in setup cyclic mode the channel we cannot assign this
857	* channel anymore. The DMA channel needs to be aborted or terminated
858	* for allowing another request.
859	*/
860	if (chan->desc && chan->desc->cyclic) {
861	dev_err(chan2dev(chan),
862	"Request not allowed when dma in cyclic mode\n");
863	return NULL;
864	}
865
866	if (!buf_len || !period_len || period_len > STM32_MDMA_MAX_BLOCK_LEN) {
867	dev_err(chan2dev(chan), "Invalid buffer/period len\n");
868	return NULL;
869	}
870
871	if (buf_len % period_len) {
872	dev_err(chan2dev(chan), "buf_len not multiple of period_len\n");
873	return NULL;
874	}
875
876	count = buf_len / period_len;
877
878	desc = stm32_mdma_alloc_desc(chan, count);
879	if (!desc)
880	return NULL;
881
882	/* Select bus */
883	if (direction == DMA_MEM_TO_DEV) {
884	src_addr = buf_addr;
885	ret = stm32_mdma_set_xfer_param(chan, direction, mdma_ccr: &ccr, mdma_ctcr: &ctcr,
886	mdma_ctbr: &ctbr, addr: src_addr, buf_len: period_len);
887	stm32_mdma_set_bus(dmadev, ctbr: &ctbr, STM32_MDMA_CTBR_SBUS,
888	src_addr);
889	} else {
890	dst_addr = buf_addr;
891	ret = stm32_mdma_set_xfer_param(chan, direction, mdma_ccr: &ccr, mdma_ctcr: &ctcr,
892	mdma_ctbr: &ctbr, addr: dst_addr, buf_len: period_len);
893	stm32_mdma_set_bus(dmadev, ctbr: &ctbr, STM32_MDMA_CTBR_DBUS,
894	src_addr: dst_addr);
895	}
896
897	if (ret < 0)
898	goto xfer_setup_err;
899
900	/* Enable interrupts */
901	ccr &= ~STM32_MDMA_CCR_IRQ_MASK;
902	ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE | STM32_MDMA_CCR_BTIE;
903	desc->ccr = ccr;
904
905	/* Configure hwdesc list */
906	for (i = 0; i < count; i++) {
907	if (direction == DMA_MEM_TO_DEV) {
908	src_addr = buf_addr + i * period_len;
909	dst_addr = dma_config->dst_addr;
910	if (chan_config->m2m_hw && (i & 1))
911	dst_addr += period_len;
912	} else {
913	src_addr = dma_config->src_addr;
914	if (chan_config->m2m_hw && (i & 1))
915	src_addr += period_len;
916	dst_addr = buf_addr + i * period_len;
917	}
918
919	stm32_mdma_setup_hwdesc(chan, desc, dir: direction, count: i, src_addr,
920	dst_addr, len: period_len, ctcr, ctbr,
921	is_last: i == count - 1, is_first: i == 0, is_cyclic: true);
922	}
923
924	desc->cyclic = true;
925
926	return vchan_tx_prep(vc: &chan->vchan, vd: &desc->vdesc, tx_flags: flags);
927
928	xfer_setup_err:
929	for (i = 0; i < desc->count; i++)
930	dma_pool_free(pool: chan->desc_pool, vaddr: desc->node[i].hwdesc,
931	addr: desc->node[i].hwdesc_phys);
932	kfree(objp: desc);
933	return NULL;
934	}
935
936	static struct dma_async_tx_descriptor *
937	stm32_mdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, dma_addr_t src,
938	size_t len, unsigned long flags)
939	{
940	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
941	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
942	enum dma_slave_buswidth max_width;
943	struct stm32_mdma_desc *desc;
944	struct stm32_mdma_hwdesc *hwdesc;
945	u32 ccr, ctcr, ctbr, cbndtr, count, max_burst, mdma_burst;
946	u32 best_burst, tlen;
947	size_t xfer_count, offset;
948	int src_bus_width, dst_bus_width;
949	int i;
950
951	/*
952	* Once DMA is in setup cyclic mode the channel we cannot assign this
953	* channel anymore. The DMA channel needs to be aborted or terminated
954	* to allow another request
955	*/
956	if (chan->desc && chan->desc->cyclic) {
957	dev_err(chan2dev(chan),
958	"Request not allowed when dma in cyclic mode\n");
959	return NULL;
960	}
961
962	count = DIV_ROUND_UP(len, STM32_MDMA_MAX_BLOCK_LEN);
963	desc = stm32_mdma_alloc_desc(chan, count);
964	if (!desc)
965	return NULL;
966
967	ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & ~STM32_MDMA_CCR_EN;
968	ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id));
969	ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id));
970	cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id));
971
972	/* Enable sw req, some interrupts and clear other bits */
973	ccr &= ~(STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX |
974	STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK |
975	STM32_MDMA_CCR_IRQ_MASK);
976	ccr |= STM32_MDMA_CCR_TEIE;
977
978	/* Enable SW request mode, dest/src inc and clear other bits */
979	ctcr &= ~(STM32_MDMA_CTCR_BWM | STM32_MDMA_CTCR_TRGM_MSK |
980	STM32_MDMA_CTCR_PAM_MASK | STM32_MDMA_CTCR_PKE |
981	STM32_MDMA_CTCR_TLEN_MSK | STM32_MDMA_CTCR_DBURST_MASK |
982	STM32_MDMA_CTCR_SBURST_MASK | STM32_MDMA_CTCR_DINCOS_MASK |
983	STM32_MDMA_CTCR_SINCOS_MASK | STM32_MDMA_CTCR_DSIZE_MASK |
984	STM32_MDMA_CTCR_SSIZE_MASK | STM32_MDMA_CTCR_DINC_MASK |
985	STM32_MDMA_CTCR_SINC_MASK);
986	ctcr |= STM32_MDMA_CTCR_SWRM | STM32_MDMA_CTCR_SINC(STM32_MDMA_INC) |
987	STM32_MDMA_CTCR_DINC(STM32_MDMA_INC);
988
989	/* Reset HW request */
990	ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK;
991
992	/* Select bus */
993	stm32_mdma_set_bus(dmadev, ctbr: &ctbr, STM32_MDMA_CTBR_SBUS, src_addr: src);
994	stm32_mdma_set_bus(dmadev, ctbr: &ctbr, STM32_MDMA_CTBR_DBUS, src_addr: dest);
995
996	/* Clear CBNDTR registers */
997	cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK | STM32_MDMA_CBNDTR_BRDUM |
998	STM32_MDMA_CBNDTR_BRSUM | STM32_MDMA_CBNDTR_BNDT_MASK);
999
1000	if (len <= STM32_MDMA_MAX_BLOCK_LEN) {
1001	cbndtr |= STM32_MDMA_CBNDTR_BNDT(len);
1002	if (len <= STM32_MDMA_MAX_BUF_LEN) {
1003	/* Setup a buffer transfer */
1004	ccr |= STM32_MDMA_CCR_TCIE | STM32_MDMA_CCR_CTCIE;
1005	ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_BUFFER);
1006	} else {
1007	/* Setup a block transfer */
1008	ccr |= STM32_MDMA_CCR_BTIE | STM32_MDMA_CCR_CTCIE;
1009	ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_BLOCK);
1010	}
1011
1012	tlen = STM32_MDMA_MAX_BUF_LEN;
1013	ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1));
1014
1015	/* Set source best burst size */
1016	max_width = stm32_mdma_get_max_width(addr: src, buf_len: len, tlen);
1017	src_bus_width = stm32_mdma_get_width(chan, width: max_width);
1018
1019	max_burst = tlen / max_width;
1020	best_burst = stm32_mdma_get_best_burst(buf_len: len, tlen, max_burst,
1021	width: max_width);
1022	mdma_burst = ilog2(best_burst);
1023
1024	ctcr |= STM32_MDMA_CTCR_SBURST(mdma_burst) |
1025	STM32_MDMA_CTCR_SSIZE(src_bus_width) |
1026	STM32_MDMA_CTCR_SINCOS(src_bus_width);
1027
1028	/* Set destination best burst size */
1029	max_width = stm32_mdma_get_max_width(addr: dest, buf_len: len, tlen);
1030	dst_bus_width = stm32_mdma_get_width(chan, width: max_width);
1031
1032	max_burst = tlen / max_width;
1033	best_burst = stm32_mdma_get_best_burst(buf_len: len, tlen, max_burst,
1034	width: max_width);
1035	mdma_burst = ilog2(best_burst);
1036
1037	ctcr |= STM32_MDMA_CTCR_DBURST(mdma_burst) |
1038	STM32_MDMA_CTCR_DSIZE(dst_bus_width) |
1039	STM32_MDMA_CTCR_DINCOS(dst_bus_width);
1040
1041	if (dst_bus_width != src_bus_width)
1042	ctcr |= STM32_MDMA_CTCR_PKE;
1043
1044	/* Prepare hardware descriptor */
1045	hwdesc = desc->node[0].hwdesc;
1046	hwdesc->ctcr = ctcr;
1047	hwdesc->cbndtr = cbndtr;
1048	hwdesc->csar = src;
1049	hwdesc->cdar = dest;
1050	hwdesc->cbrur = 0;
1051	hwdesc->clar = 0;
1052	hwdesc->ctbr = ctbr;
1053	hwdesc->cmar = 0;
1054	hwdesc->cmdr = 0;
1055
1056	stm32_mdma_dump_hwdesc(chan, node: &desc->node[0]);
1057	} else {
1058	/* Setup a LLI transfer */
1059	ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_LINKED_LIST) |
1060	STM32_MDMA_CTCR_TLEN((STM32_MDMA_MAX_BUF_LEN - 1));
1061	ccr |= STM32_MDMA_CCR_BTIE | STM32_MDMA_CCR_CTCIE;
1062	tlen = STM32_MDMA_MAX_BUF_LEN;
1063
1064	for (i = 0, offset = 0; offset < len;
1065	i++, offset += xfer_count) {
1066	xfer_count = min_t(size_t, len - offset,
1067	STM32_MDMA_MAX_BLOCK_LEN);
1068
1069	/* Set source best burst size */
1070	max_width = stm32_mdma_get_max_width(addr: src, buf_len: len, tlen);
1071	src_bus_width = stm32_mdma_get_width(chan, width: max_width);
1072
1073	max_burst = tlen / max_width;
1074	best_burst = stm32_mdma_get_best_burst(buf_len: len, tlen,
1075	max_burst,
1076	width: max_width);
1077	mdma_burst = ilog2(best_burst);
1078
1079	ctcr |= STM32_MDMA_CTCR_SBURST(mdma_burst) |
1080	STM32_MDMA_CTCR_SSIZE(src_bus_width) |
1081	STM32_MDMA_CTCR_SINCOS(src_bus_width);
1082
1083	/* Set destination best burst size */
1084	max_width = stm32_mdma_get_max_width(addr: dest, buf_len: len, tlen);
1085	dst_bus_width = stm32_mdma_get_width(chan, width: max_width);
1086
1087	max_burst = tlen / max_width;
1088	best_burst = stm32_mdma_get_best_burst(buf_len: len, tlen,
1089	max_burst,
1090	width: max_width);
1091	mdma_burst = ilog2(best_burst);
1092
1093	ctcr |= STM32_MDMA_CTCR_DBURST(mdma_burst) |
1094	STM32_MDMA_CTCR_DSIZE(dst_bus_width) |
1095	STM32_MDMA_CTCR_DINCOS(dst_bus_width);
1096
1097	if (dst_bus_width != src_bus_width)
1098	ctcr |= STM32_MDMA_CTCR_PKE;
1099
1100	/* Prepare hardware descriptor */
1101	stm32_mdma_setup_hwdesc(chan, desc, dir: DMA_MEM_TO_MEM, count: i,
1102	src_addr: src + offset, dst_addr: dest + offset,
1103	len: xfer_count, ctcr, ctbr,
1104	is_last: i == count - 1, is_first: i == 0, is_cyclic: false);
1105	}
1106	}
1107
1108	desc->ccr = ccr;
1109
1110	desc->cyclic = false;
1111
1112	return vchan_tx_prep(vc: &chan->vchan, vd: &desc->vdesc, tx_flags: flags);
1113	}
1114
1115	static void stm32_mdma_dump_reg(struct stm32_mdma_chan *chan)
1116	{
1117	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1118
1119	dev_dbg(chan2dev(chan), "CCR: 0x%08x\n",
1120	stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)));
1121	dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n",
1122	stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)));
1123	dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n",
1124	stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id)));
1125	dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n",
1126	stm32_mdma_read(dmadev, STM32_MDMA_CSAR(chan->id)));
1127	dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n",
1128	stm32_mdma_read(dmadev, STM32_MDMA_CDAR(chan->id)));
1129	dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n",
1130	stm32_mdma_read(dmadev, STM32_MDMA_CBRUR(chan->id)));
1131	dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n",
1132	stm32_mdma_read(dmadev, STM32_MDMA_CLAR(chan->id)));
1133	dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n",
1134	stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)));
1135	dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n",
1136	stm32_mdma_read(dmadev, STM32_MDMA_CMAR(chan->id)));
1137	dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n",
1138	stm32_mdma_read(dmadev, STM32_MDMA_CMDR(chan->id)));
1139	}
1140
1141	static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan)
1142	{
1143	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1144	struct virt_dma_desc *vdesc;
1145	struct stm32_mdma_hwdesc *hwdesc;
1146	u32 id = chan->id;
1147	u32 status, reg;
1148
1149	vdesc = vchan_next_desc(vc: &chan->vchan);
1150	if (!vdesc) {
1151	chan->desc = NULL;
1152	return;
1153	}
1154
1155	list_del(entry: &vdesc->node);
1156
1157	chan->desc = to_stm32_mdma_desc(vdesc);
1158	hwdesc = chan->desc->node[0].hwdesc;
1159	chan->curr_hwdesc = 0;
1160
1161	stm32_mdma_write(dmadev, STM32_MDMA_CCR(id), val: chan->desc->ccr);
1162	stm32_mdma_write(dmadev, STM32_MDMA_CTCR(id), val: hwdesc->ctcr);
1163	stm32_mdma_write(dmadev, STM32_MDMA_CBNDTR(id), val: hwdesc->cbndtr);
1164	stm32_mdma_write(dmadev, STM32_MDMA_CSAR(id), val: hwdesc->csar);
1165	stm32_mdma_write(dmadev, STM32_MDMA_CDAR(id), val: hwdesc->cdar);
1166	stm32_mdma_write(dmadev, STM32_MDMA_CBRUR(id), val: hwdesc->cbrur);
1167	stm32_mdma_write(dmadev, STM32_MDMA_CLAR(id), val: hwdesc->clar);
1168	stm32_mdma_write(dmadev, STM32_MDMA_CTBR(id), val: hwdesc->ctbr);
1169	stm32_mdma_write(dmadev, STM32_MDMA_CMAR(id), val: hwdesc->cmar);
1170	stm32_mdma_write(dmadev, STM32_MDMA_CMDR(id), val: hwdesc->cmdr);
1171
1172	/* Clear interrupt status if it is there */
1173	status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id));
1174	if (status)
1175	stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(id), mask: status);
1176
1177	stm32_mdma_dump_reg(chan);
1178
1179	/* Start DMA */
1180	stm32_mdma_set_bits(dmadev, STM32_MDMA_CCR(id), STM32_MDMA_CCR_EN);
1181
1182	/* Set SW request in case of MEM2MEM transfer */
1183	if (hwdesc->ctcr & STM32_MDMA_CTCR_SWRM) {
1184	reg = STM32_MDMA_CCR(id);
1185	stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_SWRQ);
1186	}
1187
1188	chan->busy = true;
1189
1190	dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);
1191	}
1192
1193	static void stm32_mdma_issue_pending(struct dma_chan *c)
1194	{
1195	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1196	unsigned long flags;
1197
1198	spin_lock_irqsave(&chan->vchan.lock, flags);
1199
1200	if (!vchan_issue_pending(vc: &chan->vchan))
1201	goto end;
1202
1203	dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan);
1204
1205	if (!chan->desc && !chan->busy)
1206	stm32_mdma_start_transfer(chan);
1207
1208	end:
1209	spin_unlock_irqrestore(lock: &chan->vchan.lock, flags);
1210	}
1211
1212	static int stm32_mdma_pause(struct dma_chan *c)
1213	{
1214	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1215	unsigned long flags;
1216	int ret;
1217
1218	spin_lock_irqsave(&chan->vchan.lock, flags);
1219	ret = stm32_mdma_disable_chan(chan);
1220	spin_unlock_irqrestore(lock: &chan->vchan.lock, flags);
1221
1222	if (!ret)
1223	dev_dbg(chan2dev(chan), "vchan %pK: pause\n", &chan->vchan);
1224
1225	return ret;
1226	}
1227
1228	static int stm32_mdma_resume(struct dma_chan *c)
1229	{
1230	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1231	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1232	struct stm32_mdma_hwdesc *hwdesc;
1233	unsigned long flags;
1234	u32 status, reg;
1235
1236	/* Transfer can be terminated */
1237	if (!chan->desc || (stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & STM32_MDMA_CCR_EN))
1238	return -EPERM;
1239
1240	hwdesc = chan->desc->node[chan->curr_hwdesc].hwdesc;
1241
1242	spin_lock_irqsave(&chan->vchan.lock, flags);
1243
1244	/* Re-configure control register */
1245	stm32_mdma_write(dmadev, STM32_MDMA_CCR(chan->id), val: chan->desc->ccr);
1246
1247	/* Clear interrupt status if it is there */
1248	status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
1249	if (status)
1250	stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), mask: status);
1251
1252	stm32_mdma_dump_reg(chan);
1253
1254	/* Re-start DMA */
1255	reg = STM32_MDMA_CCR(chan->id);
1256	stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_EN);
1257
1258	/* Set SW request in case of MEM2MEM transfer */
1259	if (hwdesc->ctcr & STM32_MDMA_CTCR_SWRM)
1260	stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_SWRQ);
1261
1262	spin_unlock_irqrestore(lock: &chan->vchan.lock, flags);
1263
1264	dev_dbg(chan2dev(chan), "vchan %pK: resume\n", &chan->vchan);
1265
1266	return 0;
1267	}
1268
1269	static int stm32_mdma_terminate_all(struct dma_chan *c)
1270	{
1271	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1272	unsigned long flags;
1273	LIST_HEAD(head);
1274
1275	spin_lock_irqsave(&chan->vchan.lock, flags);
1276	if (chan->desc) {
1277	vchan_terminate_vdesc(vd: &chan->desc->vdesc);
1278	if (chan->busy)
1279	stm32_mdma_stop(chan);
1280	chan->desc = NULL;
1281	}
1282	vchan_get_all_descriptors(vc: &chan->vchan, head: &head);
1283	spin_unlock_irqrestore(lock: &chan->vchan.lock, flags);
1284
1285	vchan_dma_desc_free_list(vc: &chan->vchan, head: &head);
1286
1287	return 0;
1288	}
1289
1290	static void stm32_mdma_synchronize(struct dma_chan *c)
1291	{
1292	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1293
1294	vchan_synchronize(vc: &chan->vchan);
1295	}
1296
1297	static int stm32_mdma_slave_config(struct dma_chan *c,
1298	struct dma_slave_config *config)
1299	{
1300	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1301
1302	memcpy(&chan->dma_config, config, sizeof(*config));
1303
1304	/* Check if user is requesting STM32 DMA to trigger MDMA */
1305	if (config->peripheral_size) {
1306	struct stm32_mdma_dma_config *mdma_config;
1307
1308	mdma_config = (struct stm32_mdma_dma_config *)chan->dma_config.peripheral_config;
1309	chan->chan_config.request = mdma_config->request;
1310	chan->chan_config.mask_addr = mdma_config->cmar;
1311	chan->chan_config.mask_data = mdma_config->cmdr;
1312	chan->chan_config.m2m_hw = true;
1313	}
1314
1315	return 0;
1316	}
1317
1318	static size_t stm32_mdma_desc_residue(struct stm32_mdma_chan *chan,
1319	struct stm32_mdma_desc *desc,
1320	u32 curr_hwdesc,
1321	struct dma_tx_state *state)
1322	{
1323	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1324	struct stm32_mdma_hwdesc *hwdesc;
1325	u32 cisr, clar, cbndtr, residue, modulo, burst_size;
1326	int i;
1327
1328	cisr = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
1329
1330	residue = 0;
1331	/* Get the next hw descriptor to process from current transfer */
1332	clar = stm32_mdma_read(dmadev, STM32_MDMA_CLAR(chan->id));
1333	for (i = desc->count - 1; i >= 0; i--) {
1334	hwdesc = desc->node[i].hwdesc;
1335
1336	if (hwdesc->clar == clar)
1337	break;/* Current transfer found, stop cumulating */
1338
1339	/* Cumulate residue of unprocessed hw descriptors */
1340	residue += STM32_MDMA_CBNDTR_BNDT(hwdesc->cbndtr);
1341	}
1342	cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id));
1343	residue += cbndtr & STM32_MDMA_CBNDTR_BNDT_MASK;
1344
1345	state->in_flight_bytes = 0;
1346	if (chan->chan_config.m2m_hw && (cisr & STM32_MDMA_CISR_CRQA))
1347	state->in_flight_bytes = cbndtr & STM32_MDMA_CBNDTR_BNDT_MASK;
1348
1349	if (!chan->mem_burst)
1350	return residue;
1351
1352	burst_size = chan->mem_burst * chan->mem_width;
1353	modulo = residue % burst_size;
1354	if (modulo)
1355	residue = residue - modulo + burst_size;
1356
1357	return residue;
1358	}
1359
1360	static enum dma_status stm32_mdma_tx_status(struct dma_chan *c,
1361	dma_cookie_t cookie,
1362	struct dma_tx_state *state)
1363	{
1364	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1365	struct virt_dma_desc *vdesc;
1366	enum dma_status status;
1367	unsigned long flags;
1368	u32 residue = 0;
1369
1370	status = dma_cookie_status(chan: c, cookie, state);
1371	if ((status == DMA_COMPLETE) || (!state))
1372	return status;
1373
1374	spin_lock_irqsave(&chan->vchan.lock, flags);
1375
1376	vdesc = vchan_find_desc(&chan->vchan, cookie);
1377	if (chan->desc && cookie == chan->desc->vdesc.tx.cookie)
1378	residue = stm32_mdma_desc_residue(chan, desc: chan->desc, curr_hwdesc: chan->curr_hwdesc, state);
1379	else if (vdesc)
1380	residue = stm32_mdma_desc_residue(chan, desc: to_stm32_mdma_desc(vdesc), curr_hwdesc: 0, state);
1381
1382	dma_set_residue(state, residue);
1383
1384	spin_unlock_irqrestore(lock: &chan->vchan.lock, flags);
1385
1386	return status;
1387	}
1388
1389	static void stm32_mdma_xfer_end(struct stm32_mdma_chan *chan)
1390	{
1391	vchan_cookie_complete(vd: &chan->desc->vdesc);
1392	chan->desc = NULL;
1393	chan->busy = false;
1394
1395	/* Start the next transfer if this driver has a next desc */
1396	stm32_mdma_start_transfer(chan);
1397	}
1398
1399	static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
1400	{
1401	struct stm32_mdma_device *dmadev = devid;
1402	struct stm32_mdma_chan *chan;
1403	u32 reg, id, ccr, ien, status;
1404
1405	/* Find out which channel generates the interrupt */
1406	status = readl_relaxed(dmadev->base + STM32_MDMA_GISR0);
1407	if (!status) {
1408	dev_dbg(mdma2dev(dmadev), "spurious it\n");
1409	return IRQ_NONE;
1410	}
1411	id = __ffs(status);
1412	chan = &dmadev->chan[id];
1413
1414	/* Handle interrupt for the channel */
1415	spin_lock(lock: &chan->vchan.lock);
1416	status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id));
1417	/* Mask Channel ReQuest Active bit which can be set in case of MEM2MEM */
1418	status &= ~STM32_MDMA_CISR_CRQA;
1419	ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id));
1420	ien = (ccr & STM32_MDMA_CCR_IRQ_MASK) >> 1;
1421
1422	if (!(status & ien)) {
1423	spin_unlock(lock: &chan->vchan.lock);
1424	if (chan->busy)
1425	dev_warn(chan2dev(chan),
1426	"spurious it (status=0x%04x, ien=0x%04x)\n", status, ien);
1427	else
1428	dev_dbg(chan2dev(chan),
1429	"spurious it (status=0x%04x, ien=0x%04x)\n", status, ien);
1430	return IRQ_NONE;
1431	}
1432
1433	reg = STM32_MDMA_CIFCR(id);
1434
1435	if (status & STM32_MDMA_CISR_TEIF) {
1436	dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n",
1437	readl_relaxed(dmadev->base + STM32_MDMA_CESR(id)));
1438	stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CTEIF);
1439	status &= ~STM32_MDMA_CISR_TEIF;
1440	}
1441
1442	if (status & STM32_MDMA_CISR_CTCIF) {
1443	stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CCTCIF);
1444	status &= ~STM32_MDMA_CISR_CTCIF;
1445	stm32_mdma_xfer_end(chan);
1446	}
1447
1448	if (status & STM32_MDMA_CISR_BRTIF) {
1449	stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBRTIF);
1450	status &= ~STM32_MDMA_CISR_BRTIF;
1451	}
1452
1453	if (status & STM32_MDMA_CISR_BTIF) {
1454	stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBTIF);
1455	status &= ~STM32_MDMA_CISR_BTIF;
1456	chan->curr_hwdesc++;
1457	if (chan->desc && chan->desc->cyclic) {
1458	if (chan->curr_hwdesc == chan->desc->count)
1459	chan->curr_hwdesc = 0;
1460	vchan_cyclic_callback(vd: &chan->desc->vdesc);
1461	}
1462	}
1463
1464	if (status & STM32_MDMA_CISR_TCIF) {
1465	stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CLTCIF);
1466	status &= ~STM32_MDMA_CISR_TCIF;
1467	}
1468
1469	if (status) {
1470	stm32_mdma_set_bits(dmadev, reg, mask: status);
1471	dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status);
1472	if (!(ccr & STM32_MDMA_CCR_EN))
1473	dev_err(chan2dev(chan), "chan disabled by HW\n");
1474	}
1475
1476	spin_unlock(lock: &chan->vchan.lock);
1477
1478	return IRQ_HANDLED;
1479	}
1480
1481	static int stm32_mdma_alloc_chan_resources(struct dma_chan *c)
1482	{
1483	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1484	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1485	int ret;
1486
1487	chan->desc_pool = dmam_pool_create(name: dev_name(dev: &c->dev->device),
1488	dev: c->device->dev,
1489	size: sizeof(struct stm32_mdma_hwdesc),
1490	align: __alignof__(struct stm32_mdma_hwdesc),
1491	allocation: 0);
1492	if (!chan->desc_pool) {
1493	dev_err(chan2dev(chan), "failed to allocate descriptor pool\n");
1494	return -ENOMEM;
1495	}
1496
1497	ret = pm_runtime_resume_and_get(dev: dmadev->ddev.dev);
1498	if (ret < 0)
1499	return ret;
1500
1501	ret = stm32_mdma_disable_chan(chan);
1502	if (ret < 0)
1503	pm_runtime_put(dev: dmadev->ddev.dev);
1504
1505	return ret;
1506	}
1507
1508	static void stm32_mdma_free_chan_resources(struct dma_chan *c)
1509	{
1510	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1511	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1512	unsigned long flags;
1513
1514	dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id);
1515
1516	if (chan->busy) {
1517	spin_lock_irqsave(&chan->vchan.lock, flags);
1518	stm32_mdma_stop(chan);
1519	chan->desc = NULL;
1520	spin_unlock_irqrestore(lock: &chan->vchan.lock, flags);
1521	}
1522
1523	pm_runtime_put(dev: dmadev->ddev.dev);
1524	vchan_free_chan_resources(vc: to_virt_chan(chan: c));
1525	dmam_pool_destroy(pool: chan->desc_pool);
1526	chan->desc_pool = NULL;
1527	}
1528
1529	static bool stm32_mdma_filter_fn(struct dma_chan *c, void *fn_param)
1530	{
1531	struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1532	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1533
1534	/* Check if chan is marked Secure */
1535	if (dmadev->chan_reserved & BIT(chan->id))
1536	return false;
1537
1538	return true;
1539	}
1540
1541	static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec,
1542	struct of_dma *ofdma)
1543	{
1544	struct stm32_mdma_device *dmadev = ofdma->of_dma_data;
1545	dma_cap_mask_t mask = dmadev->ddev.cap_mask;
1546	struct stm32_mdma_chan *chan;
1547	struct dma_chan *c;
1548	struct stm32_mdma_chan_config config;
1549
1550	if (dma_spec->args_count < 5) {
1551	dev_err(mdma2dev(dmadev), "Bad number of args\n");
1552	return NULL;
1553	}
1554
1555	memset(&config, 0, sizeof(config));
1556	config.request = dma_spec->args[0];
1557	config.priority_level = dma_spec->args[1];
1558	config.transfer_config = dma_spec->args[2];
1559	config.mask_addr = dma_spec->args[3];
1560	config.mask_data = dma_spec->args[4];
1561
1562	if (config.request >= dmadev->nr_requests) {
1563	dev_err(mdma2dev(dmadev), "Bad request line\n");
1564	return NULL;
1565	}
1566
1567	if (config.priority_level > STM32_MDMA_VERY_HIGH_PRIORITY) {
1568	dev_err(mdma2dev(dmadev), "Priority level not supported\n");
1569	return NULL;
1570	}
1571
1572	c = __dma_request_channel(mask: &mask, fn: stm32_mdma_filter_fn, fn_param: &config, np: ofdma->of_node);
1573	if (!c) {
1574	dev_err(mdma2dev(dmadev), "No more channels available\n");
1575	return NULL;
1576	}
1577
1578	chan = to_stm32_mdma_chan(c);
1579	chan->chan_config = config;
1580
1581	return c;
1582	}
1583
1584	static const struct of_device_id stm32_mdma_of_match[] = {
1585	{ .compatible = "st,stm32h7-mdma", },
1586	{ /* sentinel */ },
1587	};
1588	MODULE_DEVICE_TABLE(of, stm32_mdma_of_match);
1589
1590	static int stm32_mdma_probe(struct platform_device *pdev)
1591	{
1592	struct stm32_mdma_chan *chan;
1593	struct stm32_mdma_device *dmadev;
1594	struct dma_device *dd;
1595	struct device_node *of_node;
1596	struct reset_control *rst;
1597	u32 nr_channels, nr_requests;
1598	int i, count, ret;
1599
1600	of_node = pdev->dev.of_node;
1601	if (!of_node)
1602	return -ENODEV;
1603
1604	ret = device_property_read_u32(dev: &pdev->dev, propname: "dma-channels",
1605	val: &nr_channels);
1606	if (ret) {
1607	nr_channels = STM32_MDMA_MAX_CHANNELS;
1608	dev_warn(&pdev->dev, "MDMA defaulting on %i channels\n",
1609	nr_channels);
1610	}
1611
1612	ret = device_property_read_u32(dev: &pdev->dev, propname: "dma-requests",
1613	val: &nr_requests);
1614	if (ret) {
1615	nr_requests = STM32_MDMA_MAX_REQUESTS;
1616	dev_warn(&pdev->dev, "MDMA defaulting on %i request lines\n",
1617	nr_requests);
1618	}
1619
1620	count = device_property_count_u32(dev: &pdev->dev, propname: "st,ahb-addr-masks");
1621	if (count < 0)
1622	count = 0;
1623
1624	dmadev = devm_kzalloc(dev: &pdev->dev,
1625	struct_size(dmadev, ahb_addr_masks, count),
1626	GFP_KERNEL);
1627	if (!dmadev)
1628	return -ENOMEM;
1629	dmadev->nr_ahb_addr_masks = count;
1630
1631	dmadev->nr_channels = nr_channels;
1632	dmadev->nr_requests = nr_requests;
1633	device_property_read_u32_array(dev: &pdev->dev, propname: "st,ahb-addr-masks",
1634	val: dmadev->ahb_addr_masks,
1635	nval: count);
1636
1637	dmadev->base = devm_platform_ioremap_resource(pdev, index: 0);
1638	if (IS_ERR(ptr: dmadev->base))
1639	return PTR_ERR(ptr: dmadev->base);
1640
1641	dmadev->clk = devm_clk_get(dev: &pdev->dev, NULL);
1642	if (IS_ERR(ptr: dmadev->clk))
1643	return dev_err_probe(dev: &pdev->dev, err: PTR_ERR(ptr: dmadev->clk),
1644	fmt: "Missing clock controller\n");
1645
1646	ret = clk_prepare_enable(clk: dmadev->clk);
1647	if (ret < 0) {
1648	dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret);
1649	return ret;
1650	}
1651
1652	rst = devm_reset_control_get(dev: &pdev->dev, NULL);
1653	if (IS_ERR(ptr: rst)) {
1654	ret = PTR_ERR(ptr: rst);
1655	if (ret == -EPROBE_DEFER)
1656	goto err_clk;
1657	} else {
1658	reset_control_assert(rstc: rst);
1659	udelay(2);
1660	reset_control_deassert(rstc: rst);
1661	}
1662
1663	dd = &dmadev->ddev;
1664	dma_cap_set(DMA_SLAVE, dd->cap_mask);
1665	dma_cap_set(DMA_PRIVATE, dd->cap_mask);
1666	dma_cap_set(DMA_CYCLIC, dd->cap_mask);
1667	dma_cap_set(DMA_MEMCPY, dd->cap_mask);
1668	dd->device_alloc_chan_resources = stm32_mdma_alloc_chan_resources;
1669	dd->device_free_chan_resources = stm32_mdma_free_chan_resources;
1670	dd->device_tx_status = stm32_mdma_tx_status;
1671	dd->device_issue_pending = stm32_mdma_issue_pending;
1672	dd->device_prep_slave_sg = stm32_mdma_prep_slave_sg;
1673	dd->device_prep_dma_cyclic = stm32_mdma_prep_dma_cyclic;
1674	dd->device_prep_dma_memcpy = stm32_mdma_prep_dma_memcpy;
1675	dd->device_config = stm32_mdma_slave_config;
1676	dd->device_pause = stm32_mdma_pause;
1677	dd->device_resume = stm32_mdma_resume;
1678	dd->device_terminate_all = stm32_mdma_terminate_all;
1679	dd->device_synchronize = stm32_mdma_synchronize;
1680	dd->descriptor_reuse = true;
1681
1682	dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1683	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1684	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1685	BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1686	dd->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1687	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1688	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1689	BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1690	dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
1691	BIT(DMA_MEM_TO_MEM);
1692	dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1693	dd->max_burst = STM32_MDMA_MAX_BURST;
1694	dd->dev = &pdev->dev;
1695	INIT_LIST_HEAD(list: &dd->channels);
1696
1697	for (i = 0; i < dmadev->nr_channels; i++) {
1698	chan = &dmadev->chan[i];
1699	chan->id = i;
1700
1701	if (stm32_mdma_read(dmadev, STM32_MDMA_CCR(i)) & STM32_MDMA_CCR_SM)
1702	dmadev->chan_reserved |= BIT(i);
1703
1704	chan->vchan.desc_free = stm32_mdma_desc_free;
1705	vchan_init(vc: &chan->vchan, dmadev: dd);
1706	}
1707
1708	dmadev->irq = platform_get_irq(pdev, 0);
1709	if (dmadev->irq < 0) {
1710	ret = dmadev->irq;
1711	goto err_clk;
1712	}
1713
1714	ret = devm_request_irq(dev: &pdev->dev, irq: dmadev->irq, handler: stm32_mdma_irq_handler,
1715	irqflags: 0, devname: dev_name(dev: &pdev->dev), dev_id: dmadev);
1716	if (ret) {
1717	dev_err(&pdev->dev, "failed to request IRQ\n");
1718	goto err_clk;
1719	}
1720
1721	ret = dmaenginem_async_device_register(device: dd);
1722	if (ret)
1723	goto err_clk;
1724
1725	ret = of_dma_controller_register(np: of_node, of_dma_xlate: stm32_mdma_of_xlate, data: dmadev);
1726	if (ret < 0) {
1727	dev_err(&pdev->dev,
1728	"STM32 MDMA DMA OF registration failed %d\n", ret);
1729	goto err_clk;
1730	}
1731
1732	platform_set_drvdata(pdev, data: dmadev);
1733	pm_runtime_set_active(dev: &pdev->dev);
1734	pm_runtime_enable(dev: &pdev->dev);
1735	pm_runtime_get_noresume(dev: &pdev->dev);
1736	pm_runtime_put(dev: &pdev->dev);
1737
1738	dev_info(&pdev->dev, "STM32 MDMA driver registered\n");
1739
1740	return 0;
1741
1742	err_clk:
1743	clk_disable_unprepare(clk: dmadev->clk);
1744
1745	return ret;
1746	}
1747
1748	#ifdef CONFIG_PM
1749	static int stm32_mdma_runtime_suspend(struct device *dev)
1750	{
1751	struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);
1752
1753	clk_disable_unprepare(clk: dmadev->clk);
1754
1755	return 0;
1756	}
1757
1758	static int stm32_mdma_runtime_resume(struct device *dev)
1759	{
1760	struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);
1761	int ret;
1762
1763	ret = clk_prepare_enable(clk: dmadev->clk);
1764	if (ret) {
1765	dev_err(dev, "failed to prepare_enable clock\n");
1766	return ret;
1767	}
1768
1769	return 0;
1770	}
1771	#endif
1772
1773	#ifdef CONFIG_PM_SLEEP
1774	static int stm32_mdma_pm_suspend(struct device *dev)
1775	{
1776	struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);
1777	u32 ccr, id;
1778	int ret;
1779
1780	ret = pm_runtime_resume_and_get(dev);
1781	if (ret < 0)
1782	return ret;
1783
1784	for (id = 0; id < dmadev->nr_channels; id++) {
1785	ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id));
1786	if (ccr & STM32_MDMA_CCR_EN) {
1787	dev_warn(dev, "Suspend is prevented by Chan %i\n", id);
1788	return -EBUSY;
1789	}
1790	}
1791
1792	pm_runtime_put_sync(dev);
1793
1794	pm_runtime_force_suspend(dev);
1795
1796	return 0;
1797	}
1798
1799	static int stm32_mdma_pm_resume(struct device *dev)
1800	{
1801	return pm_runtime_force_resume(dev);
1802	}
1803	#endif
1804
1805	static const struct dev_pm_ops stm32_mdma_pm_ops = {
1806	SET_SYSTEM_SLEEP_PM_OPS(stm32_mdma_pm_suspend, stm32_mdma_pm_resume)
1807	SET_RUNTIME_PM_OPS(stm32_mdma_runtime_suspend,
1808	stm32_mdma_runtime_resume, NULL)
1809	};
1810
1811	static struct platform_driver stm32_mdma_driver = {
1812	.probe = stm32_mdma_probe,
1813	.driver = {
1814	.name = "stm32-mdma",
1815	.of_match_table = stm32_mdma_of_match,
1816	.pm = &stm32_mdma_pm_ops,
1817	},
1818	};
1819
1820	static int __init stm32_mdma_init(void)
1821	{
1822	return platform_driver_register(&stm32_mdma_driver);
1823	}
1824
1825	subsys_initcall(stm32_mdma_init);
1826
1827	MODULE_DESCRIPTION("Driver for STM32 MDMA controller");
1828	MODULE_AUTHOR("M'boumba Cedric Madianga <cedric.madianga@gmail.com>");
1829	MODULE_AUTHOR("Pierre-Yves Mordret <pierre-yves.mordret@st.com>");
1830