1	// SPDX-License-Identifier: GPL-2.0+
2	//
3	// drivers/dma/imx-sdma.c
4	//
5	// This file contains a driver for the Freescale Smart DMA engine
6	//
7	// Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
8	//
9	// Based on code from Freescale:
10	//
11	// Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved.
12
13	#include <linux/init.h>
14	#include <linux/iopoll.h>
15	#include <linux/module.h>
16	#include <linux/types.h>
17	#include <linux/bitfield.h>
18	#include <linux/bitops.h>
19	#include <linux/mm.h>
20	#include <linux/interrupt.h>
21	#include <linux/clk.h>
22	#include <linux/delay.h>
23	#include <linux/sched.h>
24	#include <linux/semaphore.h>
25	#include <linux/spinlock.h>
26	#include <linux/device.h>
27	#include <linux/dma-mapping.h>
28	#include <linux/firmware.h>
29	#include <linux/slab.h>
30	#include <linux/platform_device.h>
31	#include <linux/dmaengine.h>
32	#include <linux/of.h>
33	#include <linux/of_address.h>
34	#include <linux/of_dma.h>
35	#include <linux/workqueue.h>
36
37	#include <asm/irq.h>
38	#include <linux/dma/imx-dma.h>
39	#include <linux/regmap.h>
40	#include <linux/mfd/syscon.h>
41	#include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
42
43	#include "dmaengine.h"
44	#include "virt-dma.h"
45
46	/* SDMA registers */
47	#define SDMA_H_C0PTR 0x000
48	#define SDMA_H_INTR 0x004
49	#define SDMA_H_STATSTOP 0x008
50	#define SDMA_H_START 0x00c
51	#define SDMA_H_EVTOVR 0x010
52	#define SDMA_H_DSPOVR 0x014
53	#define SDMA_H_HOSTOVR 0x018
54	#define SDMA_H_EVTPEND 0x01c
55	#define SDMA_H_DSPENBL 0x020
56	#define SDMA_H_RESET 0x024
57	#define SDMA_H_EVTERR 0x028
58	#define SDMA_H_INTRMSK 0x02c
59	#define SDMA_H_PSW 0x030
60	#define SDMA_H_EVTERRDBG 0x034
61	#define SDMA_H_CONFIG 0x038
62	#define SDMA_ONCE_ENB 0x040
63	#define SDMA_ONCE_DATA 0x044
64	#define SDMA_ONCE_INSTR 0x048
65	#define SDMA_ONCE_STAT 0x04c
66	#define SDMA_ONCE_CMD 0x050
67	#define SDMA_EVT_MIRROR 0x054
68	#define SDMA_ILLINSTADDR 0x058
69	#define SDMA_CHN0ADDR 0x05c
70	#define SDMA_ONCE_RTB 0x060
71	#define SDMA_XTRIG_CONF1 0x070
72	#define SDMA_XTRIG_CONF2 0x074
73	#define SDMA_CHNENBL0_IMX35 0x200
74	#define SDMA_CHNENBL0_IMX31 0x080
75	#define SDMA_CHNPRI_0 0x100
76	#define SDMA_DONE0_CONFIG 0x1000
77
78	/*
79	* Buffer descriptor status values.
80	*/
81	#define BD_DONE 0x01
82	#define BD_WRAP 0x02
83	#define BD_CONT 0x04
84	#define BD_INTR 0x08
85	#define BD_RROR 0x10
86	#define BD_LAST 0x20
87	#define BD_EXTD 0x80
88
89	/*
90	* Data Node descriptor status values.
91	*/
92	#define DND_END_OF_FRAME 0x80
93	#define DND_END_OF_XFER 0x40
94	#define DND_DONE 0x20
95	#define DND_UNUSED 0x01
96
97	/*
98	* IPCV2 descriptor status values.
99	*/
100	#define BD_IPCV2_END_OF_FRAME 0x40
101
102	#define IPCV2_MAX_NODES 50
103	/*
104	* Error bit set in the CCB status field by the SDMA,
105	* in setbd routine, in case of a transfer error
106	*/
107	#define DATA_ERROR 0x10000000
108
109	/*
110	* Buffer descriptor commands.
111	*/
112	#define C0_ADDR 0x01
113	#define C0_LOAD 0x02
114	#define C0_DUMP 0x03
115	#define C0_SETCTX 0x07
116	#define C0_GETCTX 0x03
117	#define C0_SETDM 0x01
118	#define C0_SETPM 0x04
119	#define C0_GETDM 0x02
120	#define C0_GETPM 0x08
121	/*
122	* Change endianness indicator in the BD command field
123	*/
124	#define CHANGE_ENDIANNESS 0x80
125
126	/*
127	* p_2_p watermark_level description
128	* Bits Name Description
129	* 0-7 Lower WML Lower watermark level
130	* 8 PS 1: Pad Swallowing
131	* 0: No Pad Swallowing
132	* 9 PA 1: Pad Adding
133	* 0: No Pad Adding
134	* 10 SPDIF If this bit is set both source
135	* and destination are on SPBA
136	* 11 Source Bit(SP) 1: Source on SPBA
137	* 0: Source on AIPS
138	* 12 Destination Bit(DP) 1: Destination on SPBA
139	* 0: Destination on AIPS
140	* 13-15 --------- MUST BE 0
141	* 16-23 Higher WML HWML
142	* 24-27 N Total number of samples after
143	* which Pad adding/Swallowing
144	* must be done. It must be odd.
145	* 28 Lower WML Event(LWE) SDMA events reg to check for
146	* LWML event mask
147	* 0: LWE in EVENTS register
148	* 1: LWE in EVENTS2 register
149	* 29 Higher WML Event(HWE) SDMA events reg to check for
150	* HWML event mask
151	* 0: HWE in EVENTS register
152	* 1: HWE in EVENTS2 register
153	* 30 --------- MUST BE 0
154	* 31 CONT 1: Amount of samples to be
155	* transferred is unknown and
156	* script will keep on
157	* transferring samples as long as
158	* both events are detected and
159	* script must be manually stopped
160	* by the application
161	* 0: The amount of samples to be
162	* transferred is equal to the
163	* count field of mode word
164	*/
165	#define SDMA_WATERMARK_LEVEL_LWML 0xFF
166	#define SDMA_WATERMARK_LEVEL_PS BIT(8)
167	#define SDMA_WATERMARK_LEVEL_PA BIT(9)
168	#define SDMA_WATERMARK_LEVEL_SPDIF BIT(10)
169	#define SDMA_WATERMARK_LEVEL_SP BIT(11)
170	#define SDMA_WATERMARK_LEVEL_DP BIT(12)
171	#define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16)
172	#define SDMA_WATERMARK_LEVEL_LWE BIT(28)
173	#define SDMA_WATERMARK_LEVEL_HWE BIT(29)
174	#define SDMA_WATERMARK_LEVEL_CONT BIT(31)
175
176	#define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
177	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
178	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
179
180	#define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \
181	BIT(DMA_MEM_TO_DEV) | \
182	BIT(DMA_DEV_TO_DEV))
183
184	#define SDMA_WATERMARK_LEVEL_N_FIFOS GENMASK(15, 12)
185	#define SDMA_WATERMARK_LEVEL_OFF_FIFOS GENMASK(19, 16)
186	#define SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO GENMASK(31, 28)
187	#define SDMA_WATERMARK_LEVEL_SW_DONE BIT(23)
188
189	#define SDMA_DONE0_CONFIG_DONE_SEL BIT(7)
190	#define SDMA_DONE0_CONFIG_DONE_DIS BIT(6)
191
192	/*
193	* struct sdma_script_start_addrs - SDMA script start pointers
194	*
195	* start addresses of the different functions in the physical
196	* address space of the SDMA engine.
197	*/
198	struct sdma_script_start_addrs {
199	s32 ap_2_ap_addr;
200	s32 ap_2_bp_addr;
201	s32 ap_2_ap_fixed_addr;
202	s32 bp_2_ap_addr;
203	s32 loopback_on_dsp_side_addr;
204	s32 mcu_interrupt_only_addr;
205	s32 firi_2_per_addr;
206	s32 firi_2_mcu_addr;
207	s32 per_2_firi_addr;
208	s32 mcu_2_firi_addr;
209	s32 uart_2_per_addr;
210	s32 uart_2_mcu_addr;
211	s32 per_2_app_addr;
212	s32 mcu_2_app_addr;
213	s32 per_2_per_addr;
214	s32 uartsh_2_per_addr;
215	s32 uartsh_2_mcu_addr;
216	s32 per_2_shp_addr;
217	s32 mcu_2_shp_addr;
218	s32 ata_2_mcu_addr;
219	s32 mcu_2_ata_addr;
220	s32 app_2_per_addr;
221	s32 app_2_mcu_addr;
222	s32 shp_2_per_addr;
223	s32 shp_2_mcu_addr;
224	s32 mshc_2_mcu_addr;
225	s32 mcu_2_mshc_addr;
226	s32 spdif_2_mcu_addr;
227	s32 mcu_2_spdif_addr;
228	s32 asrc_2_mcu_addr;
229	s32 ext_mem_2_ipu_addr;
230	s32 descrambler_addr;
231	s32 dptc_dvfs_addr;
232	s32 utra_addr;
233	s32 ram_code_start_addr;
234	/* End of v1 array */
235	s32 mcu_2_ssish_addr;
236	s32 ssish_2_mcu_addr;
237	s32 hdmi_dma_addr;
238	/* End of v2 array */
239	s32 zcanfd_2_mcu_addr;
240	s32 zqspi_2_mcu_addr;
241	s32 mcu_2_ecspi_addr;
242	s32 mcu_2_sai_addr;
243	s32 sai_2_mcu_addr;
244	s32 uart_2_mcu_rom_addr;
245	s32 uartsh_2_mcu_rom_addr;
246	/* End of v3 array */
247	s32 mcu_2_zqspi_addr;
248	/* End of v4 array */
249	};
250
251	/*
252	* Mode/Count of data node descriptors - IPCv2
253	*/
254	struct sdma_mode_count {
255	#define SDMA_BD_MAX_CNT 0xffff
256	u32 count : 16; /* size of the buffer pointed by this BD */
257	u32 status : 8; /* E,R,I,C,W,D status bits stored here */
258	u32 command : 8; /* command mostly used for channel 0 */
259	};
260
261	/*
262	* Buffer descriptor
263	*/
264	struct sdma_buffer_descriptor {
265	struct sdma_mode_count mode;
266	u32 buffer_addr; /* address of the buffer described */
267	u32 ext_buffer_addr; /* extended buffer address */
268	} __attribute__ ((packed));
269
270	/**
271	* struct sdma_channel_control - Channel control Block
272	*
273	* @current_bd_ptr: current buffer descriptor processed
274	* @base_bd_ptr: first element of buffer descriptor array
275	* @unused: padding. The SDMA engine expects an array of 128 byte
276	* control blocks
277	*/
278	struct sdma_channel_control {
279	u32 current_bd_ptr;
280	u32 base_bd_ptr;
281	u32 unused[2];
282	} __attribute__ ((packed));
283
284	/**
285	* struct sdma_state_registers - SDMA context for a channel
286	*
287	* @pc: program counter
288	* @unused1: unused
289	* @t: test bit: status of arithmetic & test instruction
290	* @rpc: return program counter
291	* @unused0: unused
292	* @sf: source fault while loading data
293	* @spc: loop start program counter
294	* @unused2: unused
295	* @df: destination fault while storing data
296	* @epc: loop end program counter
297	* @lm: loop mode
298	*/
299	struct sdma_state_registers {
300	u32 pc :14;
301	u32 unused1: 1;
302	u32 t : 1;
303	u32 rpc :14;
304	u32 unused0: 1;
305	u32 sf : 1;
306	u32 spc :14;
307	u32 unused2: 1;
308	u32 df : 1;
309	u32 epc :14;
310	u32 lm : 2;
311	} __attribute__ ((packed));
312
313	/**
314	* struct sdma_context_data - sdma context specific to a channel
315	*
316	* @channel_state: channel state bits
317	* @gReg: general registers
318	* @mda: burst dma destination address register
319	* @msa: burst dma source address register
320	* @ms: burst dma status register
321	* @md: burst dma data register
322	* @pda: peripheral dma destination address register
323	* @psa: peripheral dma source address register
324	* @ps: peripheral dma status register
325	* @pd: peripheral dma data register
326	* @ca: CRC polynomial register
327	* @cs: CRC accumulator register
328	* @dda: dedicated core destination address register
329	* @dsa: dedicated core source address register
330	* @ds: dedicated core status register
331	* @dd: dedicated core data register
332	* @scratch0: 1st word of dedicated ram for context switch
333	* @scratch1: 2nd word of dedicated ram for context switch
334	* @scratch2: 3rd word of dedicated ram for context switch
335	* @scratch3: 4th word of dedicated ram for context switch
336	* @scratch4: 5th word of dedicated ram for context switch
337	* @scratch5: 6th word of dedicated ram for context switch
338	* @scratch6: 7th word of dedicated ram for context switch
339	* @scratch7: 8th word of dedicated ram for context switch
340	*/
341	struct sdma_context_data {
342	struct sdma_state_registers channel_state;
343	u32 gReg[8];
344	u32 mda;
345	u32 msa;
346	u32 ms;
347	u32 md;
348	u32 pda;
349	u32 psa;
350	u32 ps;
351	u32 pd;
352	u32 ca;
353	u32 cs;
354	u32 dda;
355	u32 dsa;
356	u32 ds;
357	u32 dd;
358	u32 scratch0;
359	u32 scratch1;
360	u32 scratch2;
361	u32 scratch3;
362	u32 scratch4;
363	u32 scratch5;
364	u32 scratch6;
365	u32 scratch7;
366	} __attribute__ ((packed));
367
368
369	struct sdma_engine;
370
371	/**
372	* struct sdma_desc - descriptor structor for one transfer
373	* @vd: descriptor for virt dma
374	* @num_bd: number of descriptors currently handling
375	* @bd_phys: physical address of bd
376	* @buf_tail: ID of the buffer that was processed
377	* @buf_ptail: ID of the previous buffer that was processed
378	* @period_len: period length, used in cyclic.
379	* @chn_real_count: the real count updated from bd->mode.count
380	* @chn_count: the transfer count set
381	* @sdmac: sdma_channel pointer
382	* @bd: pointer of allocate bd
383	*/
384	struct sdma_desc {
385	struct virt_dma_desc vd;
386	unsigned int num_bd;
387	dma_addr_t bd_phys;
388	unsigned int buf_tail;
389	unsigned int buf_ptail;
390	unsigned int period_len;
391	unsigned int chn_real_count;
392	unsigned int chn_count;
393	struct sdma_channel *sdmac;
394	struct sdma_buffer_descriptor *bd;
395	};
396
397	/**
398	* struct sdma_channel - housekeeping for a SDMA channel
399	*
400	* @vc: virt_dma base structure
401	* @desc: sdma description including vd and other special member
402	* @sdma: pointer to the SDMA engine for this channel
403	* @channel: the channel number, matches dmaengine chan_id + 1
404	* @direction: transfer type. Needed for setting SDMA script
405	* @slave_config: Slave configuration
406	* @peripheral_type: Peripheral type. Needed for setting SDMA script
407	* @event_id0: aka dma request line
408	* @event_id1: for channels that use 2 events
409	* @word_size: peripheral access size
410	* @pc_from_device: script address for those device_2_memory
411	* @pc_to_device: script address for those memory_2_device
412	* @device_to_device: script address for those device_2_device
413	* @pc_to_pc: script address for those memory_2_memory
414	* @flags: loop mode or not
415	* @per_address: peripheral source or destination address in common case
416	* destination address in p_2_p case
417	* @per_address2: peripheral source address in p_2_p case
418	* @event_mask: event mask used in p_2_p script
419	* @watermark_level: value for gReg[7], some script will extend it from
420	* basic watermark such as p_2_p
421	* @shp_addr: value for gReg[6]
422	* @per_addr: value for gReg[2]
423	* @status: status of dma channel
424	* @context_loaded: ensure context is only loaded once
425	* @data: specific sdma interface structure
426	* @bd_pool: dma_pool for bd
427	* @terminate_worker: used to call back into terminate work function
428	* @terminated: terminated list
429	* @is_ram_script: flag for script in ram
430	* @n_fifos_src: number of source device fifos
431	* @n_fifos_dst: number of destination device fifos
432	* @sw_done: software done flag
433	* @stride_fifos_src: stride for source device FIFOs
434	* @stride_fifos_dst: stride for destination device FIFOs
435	* @words_per_fifo: copy number of words one time for one FIFO
436	*/
437	struct sdma_channel {
438	struct virt_dma_chan vc;
439	struct sdma_desc *desc;
440	struct sdma_engine *sdma;
441	unsigned int channel;
442	enum dma_transfer_direction direction;
443	struct dma_slave_config slave_config;
444	enum sdma_peripheral_type peripheral_type;
445	unsigned int event_id0;
446	unsigned int event_id1;
447	enum dma_slave_buswidth word_size;
448	unsigned int pc_from_device, pc_to_device;
449	unsigned int device_to_device;
450	unsigned int pc_to_pc;
451	unsigned long flags;
452	dma_addr_t per_address, per_address2;
453	unsigned long event_mask[2];
454	unsigned long watermark_level;
455	u32 shp_addr, per_addr;
456	enum dma_status status;
457	struct imx_dma_data data;
458	struct work_struct terminate_worker;
459	struct list_head terminated;
460	bool is_ram_script;
461	unsigned int n_fifos_src;
462	unsigned int n_fifos_dst;
463	unsigned int stride_fifos_src;
464	unsigned int stride_fifos_dst;
465	unsigned int words_per_fifo;
466	bool sw_done;
467	};
468
469	#define IMX_DMA_SG_LOOP BIT(0)
470
471	#define MAX_DMA_CHANNELS 32
472	#define MXC_SDMA_DEFAULT_PRIORITY 1
473	#define MXC_SDMA_MIN_PRIORITY 1
474	#define MXC_SDMA_MAX_PRIORITY 7
475
476	#define SDMA_FIRMWARE_MAGIC 0x414d4453
477
478	/**
479	* struct sdma_firmware_header - Layout of the firmware image
480	*
481	* @magic: "SDMA"
482	* @version_major: increased whenever layout of struct
483	* sdma_script_start_addrs changes.
484	* @version_minor: firmware minor version (for binary compatible changes)
485	* @script_addrs_start: offset of struct sdma_script_start_addrs in this image
486	* @num_script_addrs: Number of script addresses in this image
487	* @ram_code_start: offset of SDMA ram image in this firmware image
488	* @ram_code_size: size of SDMA ram image
489	* @script_addrs: Stores the start address of the SDMA scripts
490	* (in SDMA memory space)
491	*/
492	struct sdma_firmware_header {
493	u32 magic;
494	u32 version_major;
495	u32 version_minor;
496	u32 script_addrs_start;
497	u32 num_script_addrs;
498	u32 ram_code_start;
499	u32 ram_code_size;
500	};
501
502	struct sdma_driver_data {
503	int chnenbl0;
504	int num_events;
505	struct sdma_script_start_addrs *script_addrs;
506	bool check_ratio;
507	/*
508	* ecspi ERR009165 fixed should be done in sdma script
509	* and it has been fixed in soc from i.mx6ul.
510	* please get more information from the below link:
511	* https://www.nxp.com/docs/en/errata/IMX6DQCE.pdf
512	*/
513	bool ecspi_fixed;
514	};
515
516	struct sdma_engine {
517	struct device *dev;
518	struct sdma_channel channel[MAX_DMA_CHANNELS];
519	struct sdma_channel_control *channel_control;
520	void __iomem *regs;
521	struct sdma_context_data *context;
522	dma_addr_t context_phys;
523	struct dma_device dma_device;
524	struct clk *clk_ipg;
525	struct clk *clk_ahb;
526	spinlock_t channel_0_lock;
527	u32 script_number;
528	struct sdma_script_start_addrs *script_addrs;
529	const struct sdma_driver_data *drvdata;
530	u32 spba_start_addr;
531	u32 spba_end_addr;
532	unsigned int irq;
533	dma_addr_t bd0_phys;
534	struct sdma_buffer_descriptor *bd0;
535	/* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/
536	bool clk_ratio;
537	bool fw_loaded;
538	};
539
540	static int sdma_config_write(struct dma_chan *chan,
541	struct dma_slave_config *dmaengine_cfg,
542	enum dma_transfer_direction direction);
543
544	static struct sdma_driver_data sdma_imx31 = {
545	.chnenbl0 = SDMA_CHNENBL0_IMX31,
546	.num_events = 32,
547	};
548
549	static struct sdma_script_start_addrs sdma_script_imx25 = {
550	.ap_2_ap_addr = 729,
551	.uart_2_mcu_addr = 904,
552	.per_2_app_addr = 1255,
553	.mcu_2_app_addr = 834,
554	.uartsh_2_mcu_addr = 1120,
555	.per_2_shp_addr = 1329,
556	.mcu_2_shp_addr = 1048,
557	.ata_2_mcu_addr = 1560,
558	.mcu_2_ata_addr = 1479,
559	.app_2_per_addr = 1189,
560	.app_2_mcu_addr = 770,
561	.shp_2_per_addr = 1407,
562	.shp_2_mcu_addr = 979,
563	};
564
565	static struct sdma_driver_data sdma_imx25 = {
566	.chnenbl0 = SDMA_CHNENBL0_IMX35,
567	.num_events = 48,
568	.script_addrs = &sdma_script_imx25,
569	};
570
571	static struct sdma_driver_data sdma_imx35 = {
572	.chnenbl0 = SDMA_CHNENBL0_IMX35,
573	.num_events = 48,
574	};
575
576	static struct sdma_script_start_addrs sdma_script_imx51 = {
577	.ap_2_ap_addr = 642,
578	.uart_2_mcu_addr = 817,
579	.mcu_2_app_addr = 747,
580	.mcu_2_shp_addr = 961,
581	.ata_2_mcu_addr = 1473,
582	.mcu_2_ata_addr = 1392,
583	.app_2_per_addr = 1033,
584	.app_2_mcu_addr = 683,
585	.shp_2_per_addr = 1251,
586	.shp_2_mcu_addr = 892,
587	};
588
589	static struct sdma_driver_data sdma_imx51 = {
590	.chnenbl0 = SDMA_CHNENBL0_IMX35,
591	.num_events = 48,
592	.script_addrs = &sdma_script_imx51,
593	};
594
595	static struct sdma_script_start_addrs sdma_script_imx53 = {
596	.ap_2_ap_addr = 642,
597	.app_2_mcu_addr = 683,
598	.mcu_2_app_addr = 747,
599	.uart_2_mcu_addr = 817,
600	.shp_2_mcu_addr = 891,
601	.mcu_2_shp_addr = 960,
602	.uartsh_2_mcu_addr = 1032,
603	.spdif_2_mcu_addr = 1100,
604	.mcu_2_spdif_addr = 1134,
605	.firi_2_mcu_addr = 1193,
606	.mcu_2_firi_addr = 1290,
607	};
608
609	static struct sdma_driver_data sdma_imx53 = {
610	.chnenbl0 = SDMA_CHNENBL0_IMX35,
611	.num_events = 48,
612	.script_addrs = &sdma_script_imx53,
613	};
614
615	static struct sdma_script_start_addrs sdma_script_imx6q = {
616	.ap_2_ap_addr = 642,
617	.uart_2_mcu_addr = 817,
618	.mcu_2_app_addr = 747,
619	.per_2_per_addr = 6331,
620	.uartsh_2_mcu_addr = 1032,
621	.mcu_2_shp_addr = 960,
622	.app_2_mcu_addr = 683,
623	.shp_2_mcu_addr = 891,
624	.spdif_2_mcu_addr = 1100,
625	.mcu_2_spdif_addr = 1134,
626	};
627
628	static struct sdma_driver_data sdma_imx6q = {
629	.chnenbl0 = SDMA_CHNENBL0_IMX35,
630	.num_events = 48,
631	.script_addrs = &sdma_script_imx6q,
632	};
633
634	static struct sdma_driver_data sdma_imx6ul = {
635	.chnenbl0 = SDMA_CHNENBL0_IMX35,
636	.num_events = 48,
637	.script_addrs = &sdma_script_imx6q,
638	.ecspi_fixed = true,
639	};
640
641	static struct sdma_script_start_addrs sdma_script_imx7d = {
642	.ap_2_ap_addr = 644,
643	.uart_2_mcu_addr = 819,
644	.mcu_2_app_addr = 749,
645	.uartsh_2_mcu_addr = 1034,
646	.mcu_2_shp_addr = 962,
647	.app_2_mcu_addr = 685,
648	.shp_2_mcu_addr = 893,
649	.spdif_2_mcu_addr = 1102,
650	.mcu_2_spdif_addr = 1136,
651	};
652
653	static struct sdma_driver_data sdma_imx7d = {
654	.chnenbl0 = SDMA_CHNENBL0_IMX35,
655	.num_events = 48,
656	.script_addrs = &sdma_script_imx7d,
657	};
658
659	static struct sdma_driver_data sdma_imx8mq = {
660	.chnenbl0 = SDMA_CHNENBL0_IMX35,
661	.num_events = 48,
662	.script_addrs = &sdma_script_imx7d,
663	.check_ratio = 1,
664	};
665
666	static const struct of_device_id sdma_dt_ids[] = {
667	{ .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, },
668	{ .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, },
669	{ .compatible = "fsl,imx51-sdma", .data = &sdma_imx51, },
670	{ .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, },
671	{ .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, },
672	{ .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, },
673	{ .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, },
674	{ .compatible = "fsl,imx6ul-sdma", .data = &sdma_imx6ul, },
675	{ .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, },
676	{ /* sentinel */ }
677	};
678	MODULE_DEVICE_TABLE(of, sdma_dt_ids);
679
680	#define SDMA_H_CONFIG_DSPDMA BIT(12) /* indicates if the DSPDMA is used */
681	#define SDMA_H_CONFIG_RTD_PINS BIT(11) /* indicates if Real-Time Debug pins are enabled */
682	#define SDMA_H_CONFIG_ACR BIT(4) /* indicates if AHB freq /core freq = 2 or 1 */
683	#define SDMA_H_CONFIG_CSM (3) /* indicates which context switch mode is selected*/
684
685	static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event)
686	{
687	u32 chnenbl0 = sdma->drvdata->chnenbl0;
688	return chnenbl0 + event * 4;
689	}
690
691	static int sdma_config_ownership(struct sdma_channel *sdmac,
692	bool event_override, bool mcu_override, bool dsp_override)
693	{
694	struct sdma_engine *sdma = sdmac->sdma;
695	int channel = sdmac->channel;
696	unsigned long evt, mcu, dsp;
697
698	if (event_override && mcu_override && dsp_override)
699	return -EINVAL;
700
701	evt = readl_relaxed(sdma->regs + SDMA_H_EVTOVR);
702	mcu = readl_relaxed(sdma->regs + SDMA_H_HOSTOVR);
703	dsp = readl_relaxed(sdma->regs + SDMA_H_DSPOVR);
704
705	if (dsp_override)
706	__clear_bit(channel, &dsp);
707	else
708	__set_bit(channel, &dsp);
709
710	if (event_override)
711	__clear_bit(channel, &evt);
712	else
713	__set_bit(channel, &evt);
714
715	if (mcu_override)
716	__clear_bit(channel, &mcu);
717	else
718	__set_bit(channel, &mcu);
719
720	writel_relaxed(evt, sdma->regs + SDMA_H_EVTOVR);
721	writel_relaxed(mcu, sdma->regs + SDMA_H_HOSTOVR);
722	writel_relaxed(dsp, sdma->regs + SDMA_H_DSPOVR);
723
724	return 0;
725	}
726
727	static int is_sdma_channel_enabled(struct sdma_engine *sdma, int channel)
728	{
729	return !!(readl(addr: sdma->regs + SDMA_H_STATSTOP) & BIT(channel));
730	}
731
732	static void sdma_enable_channel(struct sdma_engine *sdma, int channel)
733	{
734	writel(BIT(channel), addr: sdma->regs + SDMA_H_START);
735	}
736
737	/*
738	* sdma_run_channel0 - run a channel and wait till it's done
739	*/
740	static int sdma_run_channel0(struct sdma_engine *sdma)
741	{
742	int ret;
743	u32 reg;
744
745	sdma_enable_channel(sdma, channel: 0);
746
747	ret = readl_relaxed_poll_timeout_atomic(sdma->regs + SDMA_H_STATSTOP,
748	reg, !(reg & 1), 1, 500);
749	if (ret)
750	dev_err(sdma->dev, "Timeout waiting for CH0 ready\n");
751
752	/* Set bits of CONFIG register with dynamic context switching */
753	reg = readl(addr: sdma->regs + SDMA_H_CONFIG);
754	if ((reg & SDMA_H_CONFIG_CSM) == 0) {
755	reg |= SDMA_H_CONFIG_CSM;
756	writel_relaxed(reg, sdma->regs + SDMA_H_CONFIG);
757	}
758
759	return ret;
760	}
761
762	static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size,
763	u32 address)
764	{
765	struct sdma_buffer_descriptor *bd0 = sdma->bd0;
766	void *buf_virt;
767	dma_addr_t buf_phys;
768	int ret;
769	unsigned long flags;
770
771	buf_virt = dma_alloc_coherent(dev: sdma->dev, size, dma_handle: &buf_phys, GFP_KERNEL);
772	if (!buf_virt)
773	return -ENOMEM;
774
775	spin_lock_irqsave(&sdma->channel_0_lock, flags);
776
777	bd0->mode.command = C0_SETPM;
778	bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
779	bd0->mode.count = size / 2;
780	bd0->buffer_addr = buf_phys;
781	bd0->ext_buffer_addr = address;
782
783	memcpy(buf_virt, buf, size);
784
785	ret = sdma_run_channel0(sdma);
786
787	spin_unlock_irqrestore(lock: &sdma->channel_0_lock, flags);
788
789	dma_free_coherent(dev: sdma->dev, size, cpu_addr: buf_virt, dma_handle: buf_phys);
790
791	return ret;
792	}
793
794	static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)
795	{
796	struct sdma_engine *sdma = sdmac->sdma;
797	int channel = sdmac->channel;
798	unsigned long val;
799	u32 chnenbl = chnenbl_ofs(sdma, event);
800
801	val = readl_relaxed(sdma->regs + chnenbl);
802	__set_bit(channel, &val);
803	writel_relaxed(val, sdma->regs + chnenbl);
804
805	/* Set SDMA_DONEx_CONFIG is sw_done enabled */
806	if (sdmac->sw_done) {
807	val = readl_relaxed(sdma->regs + SDMA_DONE0_CONFIG);
808	val |= SDMA_DONE0_CONFIG_DONE_SEL;
809	val &= ~SDMA_DONE0_CONFIG_DONE_DIS;
810	writel_relaxed(val, sdma->regs + SDMA_DONE0_CONFIG);
811	}
812	}
813
814	static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)
815	{
816	struct sdma_engine *sdma = sdmac->sdma;
817	int channel = sdmac->channel;
818	u32 chnenbl = chnenbl_ofs(sdma, event);
819	unsigned long val;
820
821	val = readl_relaxed(sdma->regs + chnenbl);
822	__clear_bit(channel, &val);
823	writel_relaxed(val, sdma->regs + chnenbl);
824	}
825
826	static struct sdma_desc *to_sdma_desc(struct dma_async_tx_descriptor *t)
827	{
828	return container_of(t, struct sdma_desc, vd.tx);
829	}
830
831	static void sdma_start_desc(struct sdma_channel *sdmac)
832	{
833	struct virt_dma_desc *vd = vchan_next_desc(vc: &sdmac->vc);
834	struct sdma_desc *desc;
835	struct sdma_engine *sdma = sdmac->sdma;
836	int channel = sdmac->channel;
837
838	if (!vd) {
839	sdmac->desc = NULL;
840	return;
841	}
842	sdmac->desc = desc = to_sdma_desc(t: &vd->tx);
843
844	list_del(entry: &vd->node);
845
846	sdma->channel_control[channel].base_bd_ptr = desc->bd_phys;
847	sdma->channel_control[channel].current_bd_ptr = desc->bd_phys;
848	sdma_enable_channel(sdma, channel: sdmac->channel);
849	}
850
851	static void sdma_update_channel_loop(struct sdma_channel *sdmac)
852	{
853	struct sdma_buffer_descriptor *bd;
854	int error = 0;
855	enum dma_status old_status = sdmac->status;
856
857	/*
858	* loop mode. Iterate over descriptors, re-setup them and
859	* call callback function.
860	*/
861	while (sdmac->desc) {
862	struct sdma_desc *desc = sdmac->desc;
863
864	bd = &desc->bd[desc->buf_tail];
865
866	if (bd->mode.status & BD_DONE)
867	break;
868
869	if (bd->mode.status & BD_RROR) {
870	bd->mode.status &= ~BD_RROR;
871	sdmac->status = DMA_ERROR;
872	error = -EIO;
873	}
874
875	/*
876	* We use bd->mode.count to calculate the residue, since contains
877	* the number of bytes present in the current buffer descriptor.
878	*/
879
880	desc->chn_real_count = bd->mode.count;
881	bd->mode.count = desc->period_len;
882	desc->buf_ptail = desc->buf_tail;
883	desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd;
884
885	/*
886	* The callback is called from the interrupt context in order
887	* to reduce latency and to avoid the risk of altering the
888	* SDMA transaction status by the time the client tasklet is
889	* executed.
890	*/
891	spin_unlock(lock: &sdmac->vc.lock);
892	dmaengine_desc_get_callback_invoke(tx: &desc->vd.tx, NULL);
893	spin_lock(lock: &sdmac->vc.lock);
894
895	/* Assign buffer ownership to SDMA */
896	bd->mode.status |= BD_DONE;
897
898	if (error)
899	sdmac->status = old_status;
900	}
901
902	/*
903	* SDMA stops cyclic channel when DMA request triggers a channel and no SDMA
904	* owned buffer is available (i.e. BD_DONE was set too late).
905	*/
906	if (sdmac->desc && !is_sdma_channel_enabled(sdma: sdmac->sdma, channel: sdmac->channel)) {
907	dev_warn(sdmac->sdma->dev, "restart cyclic channel %d\n", sdmac->channel);
908	sdma_enable_channel(sdma: sdmac->sdma, channel: sdmac->channel);
909	}
910	}
911
912	static void mxc_sdma_handle_channel_normal(struct sdma_channel *data)
913	{
914	struct sdma_channel *sdmac = (struct sdma_channel *) data;
915	struct sdma_buffer_descriptor *bd;
916	int i, error = 0;
917
918	sdmac->desc->chn_real_count = 0;
919	/*
920	* non loop mode. Iterate over all descriptors, collect
921	* errors and call callback function
922	*/
923	for (i = 0; i < sdmac->desc->num_bd; i++) {
924	bd = &sdmac->desc->bd[i];
925
926	if (bd->mode.status & (BD_DONE | BD_RROR))
927	error = -EIO;
928	sdmac->desc->chn_real_count += bd->mode.count;
929	}
930
931	if (error)
932	sdmac->status = DMA_ERROR;
933	else
934	sdmac->status = DMA_COMPLETE;
935	}
936
937	static irqreturn_t sdma_int_handler(int irq, void *dev_id)
938	{
939	struct sdma_engine *sdma = dev_id;
940	unsigned long stat;
941
942	stat = readl_relaxed(sdma->regs + SDMA_H_INTR);
943	writel_relaxed(stat, sdma->regs + SDMA_H_INTR);
944	/* channel 0 is special and not handled here, see run_channel0() */
945	stat &= ~1;
946
947	while (stat) {
948	int channel = fls(x: stat) - 1;
949	struct sdma_channel *sdmac = &sdma->channel[channel];
950	struct sdma_desc *desc;
951
952	spin_lock(lock: &sdmac->vc.lock);
953	desc = sdmac->desc;
954	if (desc) {
955	if (sdmac->flags & IMX_DMA_SG_LOOP) {
956	if (sdmac->peripheral_type != IMX_DMATYPE_HDMI)
957	sdma_update_channel_loop(sdmac);
958	else
959	vchan_cyclic_callback(vd: &desc->vd);
960	} else {
961	mxc_sdma_handle_channel_normal(data: sdmac);
962	vchan_cookie_complete(vd: &desc->vd);
963	sdma_start_desc(sdmac);
964	}
965	}
966
967	spin_unlock(lock: &sdmac->vc.lock);
968	__clear_bit(channel, &stat);
969	}
970
971	return IRQ_HANDLED;
972	}
973
974	/*
975	* sets the pc of SDMA script according to the peripheral type
976	*/
977	static int sdma_get_pc(struct sdma_channel *sdmac,
978	enum sdma_peripheral_type peripheral_type)
979	{
980	struct sdma_engine *sdma = sdmac->sdma;
981	int per_2_emi = 0, emi_2_per = 0;
982	/*
983	* These are needed once we start to support transfers between
984	* two peripherals or memory-to-memory transfers
985	*/
986	int per_2_per = 0, emi_2_emi = 0;
987
988	sdmac->pc_from_device = 0;
989	sdmac->pc_to_device = 0;
990	sdmac->device_to_device = 0;
991	sdmac->pc_to_pc = 0;
992	sdmac->is_ram_script = false;
993
994	switch (peripheral_type) {
995	case IMX_DMATYPE_MEMORY:
996	emi_2_emi = sdma->script_addrs->ap_2_ap_addr;
997	break;
998	case IMX_DMATYPE_DSP:
999	emi_2_per = sdma->script_addrs->bp_2_ap_addr;
1000	per_2_emi = sdma->script_addrs->ap_2_bp_addr;
1001	break;
1002	case IMX_DMATYPE_FIRI:
1003	per_2_emi = sdma->script_addrs->firi_2_mcu_addr;
1004	emi_2_per = sdma->script_addrs->mcu_2_firi_addr;
1005	break;
1006	case IMX_DMATYPE_UART:
1007	per_2_emi = sdma->script_addrs->uart_2_mcu_addr;
1008	emi_2_per = sdma->script_addrs->mcu_2_app_addr;
1009	break;
1010	case IMX_DMATYPE_UART_SP:
1011	per_2_emi = sdma->script_addrs->uartsh_2_mcu_addr;
1012	emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
1013	break;
1014	case IMX_DMATYPE_ATA:
1015	per_2_emi = sdma->script_addrs->ata_2_mcu_addr;
1016	emi_2_per = sdma->script_addrs->mcu_2_ata_addr;
1017	break;
1018	case IMX_DMATYPE_CSPI:
1019	per_2_emi = sdma->script_addrs->app_2_mcu_addr;
1020
1021	/* Use rom script mcu_2_app if ERR009165 fixed */
1022	if (sdmac->sdma->drvdata->ecspi_fixed) {
1023	emi_2_per = sdma->script_addrs->mcu_2_app_addr;
1024	} else {
1025	emi_2_per = sdma->script_addrs->mcu_2_ecspi_addr;
1026	sdmac->is_ram_script = true;
1027	}
1028
1029	break;
1030	case IMX_DMATYPE_EXT:
1031	case IMX_DMATYPE_SSI:
1032	case IMX_DMATYPE_SAI:
1033	per_2_emi = sdma->script_addrs->app_2_mcu_addr;
1034	emi_2_per = sdma->script_addrs->mcu_2_app_addr;
1035	break;
1036	case IMX_DMATYPE_SSI_DUAL:
1037	per_2_emi = sdma->script_addrs->ssish_2_mcu_addr;
1038	emi_2_per = sdma->script_addrs->mcu_2_ssish_addr;
1039	sdmac->is_ram_script = true;
1040	break;
1041	case IMX_DMATYPE_SSI_SP:
1042	case IMX_DMATYPE_MMC:
1043	case IMX_DMATYPE_SDHC:
1044	case IMX_DMATYPE_CSPI_SP:
1045	case IMX_DMATYPE_ESAI:
1046	case IMX_DMATYPE_MSHC_SP:
1047	per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
1048	emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
1049	break;
1050	case IMX_DMATYPE_ASRC:
1051	per_2_emi = sdma->script_addrs->asrc_2_mcu_addr;
1052	emi_2_per = sdma->script_addrs->asrc_2_mcu_addr;
1053	per_2_per = sdma->script_addrs->per_2_per_addr;
1054	sdmac->is_ram_script = true;
1055	break;
1056	case IMX_DMATYPE_ASRC_SP:
1057	per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
1058	emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
1059	per_2_per = sdma->script_addrs->per_2_per_addr;
1060	break;
1061	case IMX_DMATYPE_MSHC:
1062	per_2_emi = sdma->script_addrs->mshc_2_mcu_addr;
1063	emi_2_per = sdma->script_addrs->mcu_2_mshc_addr;
1064	break;
1065	case IMX_DMATYPE_CCM:
1066	per_2_emi = sdma->script_addrs->dptc_dvfs_addr;
1067	break;
1068	case IMX_DMATYPE_SPDIF:
1069	per_2_emi = sdma->script_addrs->spdif_2_mcu_addr;
1070	emi_2_per = sdma->script_addrs->mcu_2_spdif_addr;
1071	break;
1072	case IMX_DMATYPE_IPU_MEMORY:
1073	emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr;
1074	break;
1075	case IMX_DMATYPE_MULTI_SAI:
1076	per_2_emi = sdma->script_addrs->sai_2_mcu_addr;
1077	emi_2_per = sdma->script_addrs->mcu_2_sai_addr;
1078	break;
1079	case IMX_DMATYPE_HDMI:
1080	emi_2_per = sdma->script_addrs->hdmi_dma_addr;
1081	sdmac->is_ram_script = true;
1082	break;
1083	default:
1084	dev_err(sdma->dev, "Unsupported transfer type %d\n",
1085	peripheral_type);
1086	return -EINVAL;
1087	}
1088
1089	sdmac->pc_from_device = per_2_emi;
1090	sdmac->pc_to_device = emi_2_per;
1091	sdmac->device_to_device = per_2_per;
1092	sdmac->pc_to_pc = emi_2_emi;
1093
1094	return 0;
1095	}
1096
1097	static int sdma_load_context(struct sdma_channel *sdmac)
1098	{
1099	struct sdma_engine *sdma = sdmac->sdma;
1100	int channel = sdmac->channel;
1101	int load_address;
1102	struct sdma_context_data *context = sdma->context;
1103	struct sdma_buffer_descriptor *bd0 = sdma->bd0;
1104	int ret;
1105	unsigned long flags;
1106
1107	if (sdmac->direction == DMA_DEV_TO_MEM)
1108	load_address = sdmac->pc_from_device;
1109	else if (sdmac->direction == DMA_DEV_TO_DEV)
1110	load_address = sdmac->device_to_device;
1111	else if (sdmac->direction == DMA_MEM_TO_MEM)
1112	load_address = sdmac->pc_to_pc;
1113	else
1114	load_address = sdmac->pc_to_device;
1115
1116	if (load_address < 0)
1117	return load_address;
1118
1119	dev_dbg(sdma->dev, "load_address = %d\n", load_address);
1120	dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);
1121	dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);
1122	dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);
1123	dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);
1124	dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);
1125
1126	spin_lock_irqsave(&sdma->channel_0_lock, flags);
1127
1128	memset(context, 0, sizeof(*context));
1129	context->channel_state.pc = load_address;
1130
1131	/* Send by context the event mask,base address for peripheral
1132	* and watermark level
1133	*/
1134	if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) {
1135	context->gReg[4] = sdmac->per_addr;
1136	context->gReg[6] = sdmac->shp_addr;
1137	} else {
1138	context->gReg[0] = sdmac->event_mask[1];
1139	context->gReg[1] = sdmac->event_mask[0];
1140	context->gReg[2] = sdmac->per_addr;
1141	context->gReg[6] = sdmac->shp_addr;
1142	context->gReg[7] = sdmac->watermark_level;
1143	}
1144
1145	bd0->mode.command = C0_SETDM;
1146	bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
1147	bd0->mode.count = sizeof(*context) / 4;
1148	bd0->buffer_addr = sdma->context_phys;
1149	bd0->ext_buffer_addr = 2048 + (sizeof(*context) / 4) * channel;
1150	ret = sdma_run_channel0(sdma);
1151
1152	spin_unlock_irqrestore(lock: &sdma->channel_0_lock, flags);
1153
1154	return ret;
1155	}
1156
1157	static struct sdma_channel *to_sdma_chan(struct dma_chan *chan)
1158	{
1159	return container_of(chan, struct sdma_channel, vc.chan);
1160	}
1161
1162	static int sdma_disable_channel(struct dma_chan *chan)
1163	{
1164	struct sdma_channel *sdmac = to_sdma_chan(chan);
1165	struct sdma_engine *sdma = sdmac->sdma;
1166	int channel = sdmac->channel;
1167
1168	writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP);
1169	sdmac->status = DMA_ERROR;
1170
1171	return 0;
1172	}
1173	static void sdma_channel_terminate_work(struct work_struct *work)
1174	{
1175	struct sdma_channel *sdmac = container_of(work, struct sdma_channel,
1176	terminate_worker);
1177	/*
1178	* According to NXP R&D team a delay of one BD SDMA cost time
1179	* (maximum is 1ms) should be added after disable of the channel
1180	* bit, to ensure SDMA core has really been stopped after SDMA
1181	* clients call .device_terminate_all.
1182	*/
1183	usleep_range(min: 1000, max: 2000);
1184
1185	vchan_dma_desc_free_list(vc: &sdmac->vc, head: &sdmac->terminated);
1186	}
1187
1188	static int sdma_terminate_all(struct dma_chan *chan)
1189	{
1190	struct sdma_channel *sdmac = to_sdma_chan(chan);
1191	unsigned long flags;
1192
1193	spin_lock_irqsave(&sdmac->vc.lock, flags);
1194
1195	sdma_disable_channel(chan);
1196
1197	if (sdmac->desc) {
1198	vchan_terminate_vdesc(vd: &sdmac->desc->vd);
1199	/*
1200	* move out current descriptor into terminated list so that
1201	* it could be free in sdma_channel_terminate_work alone
1202	* later without potential involving next descriptor raised
1203	* up before the last descriptor terminated.
1204	*/
1205	vchan_get_all_descriptors(vc: &sdmac->vc, head: &sdmac->terminated);
1206	sdmac->desc = NULL;
1207	schedule_work(work: &sdmac->terminate_worker);
1208	}
1209
1210	spin_unlock_irqrestore(lock: &sdmac->vc.lock, flags);
1211
1212	return 0;
1213	}
1214
1215	static void sdma_channel_synchronize(struct dma_chan *chan)
1216	{
1217	struct sdma_channel *sdmac = to_sdma_chan(chan);
1218
1219	vchan_synchronize(vc: &sdmac->vc);
1220
1221	flush_work(work: &sdmac->terminate_worker);
1222	}
1223
1224	static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac)
1225	{
1226	struct sdma_engine *sdma = sdmac->sdma;
1227
1228	int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML;
1229	int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16;
1230
1231	set_bit(nr: sdmac->event_id0 % 32, addr: &sdmac->event_mask[1]);
1232	set_bit(nr: sdmac->event_id1 % 32, addr: &sdmac->event_mask[0]);
1233
1234	if (sdmac->event_id0 > 31)
1235	sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE;
1236
1237	if (sdmac->event_id1 > 31)
1238	sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE;
1239
1240	/*
1241	* If LWML(src_maxburst) > HWML(dst_maxburst), we need
1242	* swap LWML and HWML of INFO(A.3.2.5.1), also need swap
1243	* r0(event_mask[1]) and r1(event_mask[0]).
1244	*/
1245	if (lwml > hwml) {
1246	sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML |
1247	SDMA_WATERMARK_LEVEL_HWML);
1248	sdmac->watermark_level |= hwml;
1249	sdmac->watermark_level |= lwml << 16;
1250	swap(sdmac->event_mask[0], sdmac->event_mask[1]);
1251	}
1252
1253	if (sdmac->per_address2 >= sdma->spba_start_addr &&
1254	sdmac->per_address2 <= sdma->spba_end_addr)
1255	sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP;
1256
1257	if (sdmac->per_address >= sdma->spba_start_addr &&
1258	sdmac->per_address <= sdma->spba_end_addr)
1259	sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP;
1260
1261	sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT;
1262	}
1263
1264	static void sdma_set_watermarklevel_for_sais(struct sdma_channel *sdmac)
1265	{
1266	unsigned int n_fifos;
1267	unsigned int stride_fifos;
1268	unsigned int words_per_fifo;
1269
1270	if (sdmac->sw_done)
1271	sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SW_DONE;
1272
1273	if (sdmac->direction == DMA_DEV_TO_MEM) {
1274	n_fifos = sdmac->n_fifos_src;
1275	stride_fifos = sdmac->stride_fifos_src;
1276	} else {
1277	n_fifos = sdmac->n_fifos_dst;
1278	stride_fifos = sdmac->stride_fifos_dst;
1279	}
1280
1281	words_per_fifo = sdmac->words_per_fifo;
1282
1283	sdmac->watermark_level |=
1284	FIELD_PREP(SDMA_WATERMARK_LEVEL_N_FIFOS, n_fifos);
1285	sdmac->watermark_level |=
1286	FIELD_PREP(SDMA_WATERMARK_LEVEL_OFF_FIFOS, stride_fifos);
1287	if (words_per_fifo)
1288	sdmac->watermark_level |=
1289	FIELD_PREP(SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO, (words_per_fifo - 1));
1290	}
1291
1292	static int sdma_config_channel(struct dma_chan *chan)
1293	{
1294	struct sdma_channel *sdmac = to_sdma_chan(chan);
1295	int ret;
1296
1297	sdma_disable_channel(chan);
1298
1299	sdmac->event_mask[0] = 0;
1300	sdmac->event_mask[1] = 0;
1301	sdmac->shp_addr = 0;
1302	sdmac->per_addr = 0;
1303
1304	switch (sdmac->peripheral_type) {
1305	case IMX_DMATYPE_DSP:
1306	sdma_config_ownership(sdmac, event_override: false, mcu_override: true, dsp_override: true);
1307	break;
1308	case IMX_DMATYPE_MEMORY:
1309	sdma_config_ownership(sdmac, event_override: false, mcu_override: true, dsp_override: false);
1310	break;
1311	default:
1312	sdma_config_ownership(sdmac, event_override: true, mcu_override: true, dsp_override: false);
1313	break;
1314	}
1315
1316	ret = sdma_get_pc(sdmac, peripheral_type: sdmac->peripheral_type);
1317	if (ret)
1318	return ret;
1319
1320	if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&
1321	(sdmac->peripheral_type != IMX_DMATYPE_DSP)) {
1322	/* Handle multiple event channels differently */
1323	if (sdmac->event_id1) {
1324	if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||
1325	sdmac->peripheral_type == IMX_DMATYPE_ASRC)
1326	sdma_set_watermarklevel_for_p2p(sdmac);
1327	} else {
1328	if (sdmac->peripheral_type ==
1329	IMX_DMATYPE_MULTI_SAI)
1330	sdma_set_watermarklevel_for_sais(sdmac);
1331
1332	__set_bit(sdmac->event_id0, sdmac->event_mask);
1333	}
1334
1335	/* Address */
1336	sdmac->shp_addr = sdmac->per_address;
1337	sdmac->per_addr = sdmac->per_address2;
1338	} else {
1339	sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */
1340	}
1341
1342	return 0;
1343	}
1344
1345	static int sdma_set_channel_priority(struct sdma_channel *sdmac,
1346	unsigned int priority)
1347	{
1348	struct sdma_engine *sdma = sdmac->sdma;
1349	int channel = sdmac->channel;
1350
1351	if (priority < MXC_SDMA_MIN_PRIORITY
1352	|| priority > MXC_SDMA_MAX_PRIORITY) {
1353	return -EINVAL;
1354	}
1355
1356	writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel);
1357
1358	return 0;
1359	}
1360
1361	static int sdma_request_channel0(struct sdma_engine *sdma)
1362	{
1363	int ret = -EBUSY;
1364
1365	sdma->bd0 = dma_alloc_coherent(dev: sdma->dev, PAGE_SIZE, dma_handle: &sdma->bd0_phys,
1366	GFP_NOWAIT);
1367	if (!sdma->bd0) {
1368	ret = -ENOMEM;
1369	goto out;
1370	}
1371
1372	sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys;
1373	sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys;
1374
1375	sdma_set_channel_priority(sdmac: &sdma->channel[0], MXC_SDMA_DEFAULT_PRIORITY);
1376	return 0;
1377	out:
1378
1379	return ret;
1380	}
1381
1382
1383	static int sdma_alloc_bd(struct sdma_desc *desc)
1384	{
1385	u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);
1386	int ret = 0;
1387
1388	desc->bd = dma_alloc_coherent(dev: desc->sdmac->sdma->dev, size: bd_size,
1389	dma_handle: &desc->bd_phys, GFP_NOWAIT);
1390	if (!desc->bd) {
1391	ret = -ENOMEM;
1392	goto out;
1393	}
1394	out:
1395	return ret;
1396	}
1397
1398	static void sdma_free_bd(struct sdma_desc *desc)
1399	{
1400	u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);
1401
1402	dma_free_coherent(dev: desc->sdmac->sdma->dev, size: bd_size, cpu_addr: desc->bd,
1403	dma_handle: desc->bd_phys);
1404	}
1405
1406	static void sdma_desc_free(struct virt_dma_desc *vd)
1407	{
1408	struct sdma_desc *desc = container_of(vd, struct sdma_desc, vd);
1409
1410	sdma_free_bd(desc);
1411	kfree(objp: desc);
1412	}
1413
1414	static int sdma_alloc_chan_resources(struct dma_chan *chan)
1415	{
1416	struct sdma_channel *sdmac = to_sdma_chan(chan);
1417	struct imx_dma_data *data = chan->private;
1418	struct imx_dma_data mem_data;
1419	int prio, ret;
1420
1421	/*
1422	* MEMCPY may never setup chan->private by filter function such as
1423	* dmatest, thus create 'struct imx_dma_data mem_data' for this case.
1424	* Please note in any other slave case, you have to setup chan->private
1425	* with 'struct imx_dma_data' in your own filter function if you want to
1426	* request dma channel by dma_request_channel() rather than
1427	* dma_request_slave_channel(). Othwise, 'MEMCPY in case?' will appear
1428	* to warn you to correct your filter function.
1429	*/
1430	if (!data) {
1431	dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n");
1432	mem_data.priority = 2;
1433	mem_data.peripheral_type = IMX_DMATYPE_MEMORY;
1434	mem_data.dma_request = 0;
1435	mem_data.dma_request2 = 0;
1436	data = &mem_data;
1437
1438	ret = sdma_get_pc(sdmac, peripheral_type: IMX_DMATYPE_MEMORY);
1439	if (ret)
1440	return ret;
1441	}
1442
1443	switch (data->priority) {
1444	case DMA_PRIO_HIGH:
1445	prio = 3;
1446	break;
1447	case DMA_PRIO_MEDIUM:
1448	prio = 2;
1449	break;
1450	case DMA_PRIO_LOW:
1451	default:
1452	prio = 1;
1453	break;
1454	}
1455
1456	sdmac->peripheral_type = data->peripheral_type;
1457	sdmac->event_id0 = data->dma_request;
1458	sdmac->event_id1 = data->dma_request2;
1459
1460	ret = clk_enable(clk: sdmac->sdma->clk_ipg);
1461	if (ret)
1462	return ret;
1463	ret = clk_enable(clk: sdmac->sdma->clk_ahb);
1464	if (ret)
1465	goto disable_clk_ipg;
1466
1467	ret = sdma_set_channel_priority(sdmac, priority: prio);
1468	if (ret)
1469	goto disable_clk_ahb;
1470
1471	return 0;
1472
1473	disable_clk_ahb:
1474	clk_disable(clk: sdmac->sdma->clk_ahb);
1475	disable_clk_ipg:
1476	clk_disable(clk: sdmac->sdma->clk_ipg);
1477	return ret;
1478	}
1479
1480	static void sdma_free_chan_resources(struct dma_chan *chan)
1481	{
1482	struct sdma_channel *sdmac = to_sdma_chan(chan);
1483	struct sdma_engine *sdma = sdmac->sdma;
1484
1485	sdma_terminate_all(chan);
1486
1487	sdma_channel_synchronize(chan);
1488
1489	sdma_event_disable(sdmac, event: sdmac->event_id0);
1490	if (sdmac->event_id1)
1491	sdma_event_disable(sdmac, event: sdmac->event_id1);
1492
1493	sdmac->event_id0 = 0;
1494	sdmac->event_id1 = 0;
1495
1496	sdma_set_channel_priority(sdmac, priority: 0);
1497
1498	clk_disable(clk: sdma->clk_ipg);
1499	clk_disable(clk: sdma->clk_ahb);
1500	}
1501
1502	static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac,
1503	enum dma_transfer_direction direction, u32 bds)
1504	{
1505	struct sdma_desc *desc;
1506
1507	if (!sdmac->sdma->fw_loaded && sdmac->is_ram_script) {
1508	dev_warn_once(sdmac->sdma->dev, "sdma firmware not ready!\n");
1509	goto err_out;
1510	}
1511
1512	desc = kzalloc(size: (sizeof(*desc)), GFP_NOWAIT);
1513	if (!desc)
1514	goto err_out;
1515
1516	sdmac->status = DMA_IN_PROGRESS;
1517	sdmac->direction = direction;
1518	sdmac->flags = 0;
1519
1520	desc->chn_count = 0;
1521	desc->chn_real_count = 0;
1522	desc->buf_tail = 0;
1523	desc->buf_ptail = 0;
1524	desc->sdmac = sdmac;
1525	desc->num_bd = bds;
1526
1527	if (bds && sdma_alloc_bd(desc))
1528	goto err_desc_out;
1529
1530	/* No slave_config called in MEMCPY case, so do here */
1531	if (direction == DMA_MEM_TO_MEM)
1532	sdma_config_ownership(sdmac, event_override: false, mcu_override: true, dsp_override: false);
1533
1534	if (sdma_load_context(sdmac))
1535	goto err_bd_out;
1536
1537	return desc;
1538
1539	err_bd_out:
1540	sdma_free_bd(desc);
1541	err_desc_out:
1542	kfree(objp: desc);
1543	err_out:
1544	return NULL;
1545	}
1546
1547	static struct dma_async_tx_descriptor *sdma_prep_memcpy(
1548	struct dma_chan *chan, dma_addr_t dma_dst,
1549	dma_addr_t dma_src, size_t len, unsigned long flags)
1550	{
1551	struct sdma_channel *sdmac = to_sdma_chan(chan);
1552	struct sdma_engine *sdma = sdmac->sdma;
1553	int channel = sdmac->channel;
1554	size_t count;
1555	int i = 0, param;
1556	struct sdma_buffer_descriptor *bd;
1557	struct sdma_desc *desc;
1558
1559	if (!chan || !len)
1560	return NULL;
1561
1562	dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.\n",
1563	&dma_src, &dma_dst, len, channel);
1564
1565	desc = sdma_transfer_init(sdmac, direction: DMA_MEM_TO_MEM,
1566	bds: len / SDMA_BD_MAX_CNT + 1);
1567	if (!desc)
1568	return NULL;
1569
1570	do {
1571	count = min_t(size_t, len, SDMA_BD_MAX_CNT);
1572	bd = &desc->bd[i];
1573	bd->buffer_addr = dma_src;
1574	bd->ext_buffer_addr = dma_dst;
1575	bd->mode.count = count;
1576	desc->chn_count += count;
1577	bd->mode.command = 0;
1578
1579	dma_src += count;
1580	dma_dst += count;
1581	len -= count;
1582	i++;
1583
1584	param = BD_DONE | BD_EXTD | BD_CONT;
1585	/* last bd */
1586	if (!len) {
1587	param |= BD_INTR;
1588	param |= BD_LAST;
1589	param &= ~BD_CONT;
1590	}
1591
1592	dev_dbg(sdma->dev, "entry %d: count: %zd dma: 0x%x %s%s\n",
1593	i, count, bd->buffer_addr,
1594	param & BD_WRAP ? "wrap" : "",
1595	param & BD_INTR ? " intr" : "");
1596
1597	bd->mode.status = param;
1598	} while (len);
1599
1600	return vchan_tx_prep(vc: &sdmac->vc, vd: &desc->vd, tx_flags: flags);
1601	}
1602
1603	static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
1604	struct dma_chan *chan, struct scatterlist *sgl,
1605	unsigned int sg_len, enum dma_transfer_direction direction,
1606	unsigned long flags, void *context)
1607	{
1608	struct sdma_channel *sdmac = to_sdma_chan(chan);
1609	struct sdma_engine *sdma = sdmac->sdma;
1610	int i, count;
1611	int channel = sdmac->channel;
1612	struct scatterlist *sg;
1613	struct sdma_desc *desc;
1614
1615	sdma_config_write(chan, dmaengine_cfg: &sdmac->slave_config, direction);
1616
1617	desc = sdma_transfer_init(sdmac, direction, bds: sg_len);
1618	if (!desc)
1619	goto err_out;
1620
1621	dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n",
1622	sg_len, channel);
1623
1624	for_each_sg(sgl, sg, sg_len, i) {
1625	struct sdma_buffer_descriptor *bd = &desc->bd[i];
1626	int param;
1627
1628	bd->buffer_addr = sg->dma_address;
1629
1630	count = sg_dma_len(sg);
1631
1632	if (count > SDMA_BD_MAX_CNT) {
1633	dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n",
1634	channel, count, SDMA_BD_MAX_CNT);
1635	goto err_bd_out;
1636	}
1637
1638	bd->mode.count = count;
1639	desc->chn_count += count;
1640
1641	if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
1642	goto err_bd_out;
1643
1644	switch (sdmac->word_size) {
1645	case DMA_SLAVE_BUSWIDTH_4_BYTES:
1646	bd->mode.command = 0;
1647	if (count & 3 || sg->dma_address & 3)
1648	goto err_bd_out;
1649	break;
1650	case DMA_SLAVE_BUSWIDTH_2_BYTES:
1651	bd->mode.command = 2;
1652	if (count & 1 || sg->dma_address & 1)
1653	goto err_bd_out;
1654	break;
1655	case DMA_SLAVE_BUSWIDTH_1_BYTE:
1656	bd->mode.command = 1;
1657	break;
1658	default:
1659	goto err_bd_out;
1660	}
1661
1662	param = BD_DONE | BD_EXTD | BD_CONT;
1663
1664	if (i + 1 == sg_len) {
1665	param |= BD_INTR;
1666	param |= BD_LAST;
1667	param &= ~BD_CONT;
1668	}
1669
1670	dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n",
1671	i, count, (u64)sg->dma_address,
1672	param & BD_WRAP ? "wrap" : "",
1673	param & BD_INTR ? " intr" : "");
1674
1675	bd->mode.status = param;
1676	}
1677
1678	return vchan_tx_prep(vc: &sdmac->vc, vd: &desc->vd, tx_flags: flags);
1679	err_bd_out:
1680	sdma_free_bd(desc);
1681	kfree(objp: desc);
1682	err_out:
1683	sdmac->status = DMA_ERROR;
1684	return NULL;
1685	}
1686
1687	static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
1688	struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
1689	size_t period_len, enum dma_transfer_direction direction,
1690	unsigned long flags)
1691	{
1692	struct sdma_channel *sdmac = to_sdma_chan(chan);
1693	struct sdma_engine *sdma = sdmac->sdma;
1694	int num_periods = 0;
1695	int channel = sdmac->channel;
1696	int i = 0, buf = 0;
1697	struct sdma_desc *desc;
1698
1699	dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel);
1700
1701	if (sdmac->peripheral_type != IMX_DMATYPE_HDMI)
1702	num_periods = buf_len / period_len;
1703
1704	sdma_config_write(chan, dmaengine_cfg: &sdmac->slave_config, direction);
1705
1706	desc = sdma_transfer_init(sdmac, direction, bds: num_periods);
1707	if (!desc)
1708	goto err_out;
1709
1710	desc->period_len = period_len;
1711
1712	sdmac->flags |= IMX_DMA_SG_LOOP;
1713
1714	if (period_len > SDMA_BD_MAX_CNT) {
1715	dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n",
1716	channel, period_len, SDMA_BD_MAX_CNT);
1717	goto err_bd_out;
1718	}
1719
1720	if (sdmac->peripheral_type == IMX_DMATYPE_HDMI)
1721	return vchan_tx_prep(vc: &sdmac->vc, vd: &desc->vd, tx_flags: flags);
1722
1723	while (buf < buf_len) {
1724	struct sdma_buffer_descriptor *bd = &desc->bd[i];
1725	int param;
1726
1727	bd->buffer_addr = dma_addr;
1728
1729	bd->mode.count = period_len;
1730
1731	if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
1732	goto err_bd_out;
1733	if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)
1734	bd->mode.command = 0;
1735	else
1736	bd->mode.command = sdmac->word_size;
1737
1738	param = BD_DONE | BD_EXTD | BD_CONT | BD_INTR;
1739	if (i + 1 == num_periods)
1740	param |= BD_WRAP;
1741
1742	dev_dbg(sdma->dev, "entry %d: count: %zu dma: %#llx %s%s\n",
1743	i, period_len, (u64)dma_addr,
1744	param & BD_WRAP ? "wrap" : "",
1745	param & BD_INTR ? " intr" : "");
1746
1747	bd->mode.status = param;
1748
1749	dma_addr += period_len;
1750	buf += period_len;
1751
1752	i++;
1753	}
1754
1755	return vchan_tx_prep(vc: &sdmac->vc, vd: &desc->vd, tx_flags: flags);
1756	err_bd_out:
1757	sdma_free_bd(desc);
1758	kfree(objp: desc);
1759	err_out:
1760	sdmac->status = DMA_ERROR;
1761	return NULL;
1762	}
1763
1764	static int sdma_config_write(struct dma_chan *chan,
1765	struct dma_slave_config *dmaengine_cfg,
1766	enum dma_transfer_direction direction)
1767	{
1768	struct sdma_channel *sdmac = to_sdma_chan(chan);
1769
1770	if (direction == DMA_DEV_TO_MEM) {
1771	sdmac->per_address = dmaengine_cfg->src_addr;
1772	sdmac->watermark_level = dmaengine_cfg->src_maxburst *
1773	dmaengine_cfg->src_addr_width;
1774	sdmac->word_size = dmaengine_cfg->src_addr_width;
1775	} else if (direction == DMA_DEV_TO_DEV) {
1776	sdmac->per_address2 = dmaengine_cfg->src_addr;
1777	sdmac->per_address = dmaengine_cfg->dst_addr;
1778	sdmac->watermark_level = dmaengine_cfg->src_maxburst &
1779	SDMA_WATERMARK_LEVEL_LWML;
1780	sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) &
1781	SDMA_WATERMARK_LEVEL_HWML;
1782	sdmac->word_size = dmaengine_cfg->dst_addr_width;
1783	} else if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) {
1784	sdmac->per_address = dmaengine_cfg->dst_addr;
1785	sdmac->per_address2 = dmaengine_cfg->src_addr;
1786	sdmac->watermark_level = 0;
1787	} else {
1788	sdmac->per_address = dmaengine_cfg->dst_addr;
1789	sdmac->watermark_level = dmaengine_cfg->dst_maxburst *
1790	dmaengine_cfg->dst_addr_width;
1791	sdmac->word_size = dmaengine_cfg->dst_addr_width;
1792	}
1793	sdmac->direction = direction;
1794	return sdma_config_channel(chan);
1795	}
1796
1797	static int sdma_config(struct dma_chan *chan,
1798	struct dma_slave_config *dmaengine_cfg)
1799	{
1800	struct sdma_channel *sdmac = to_sdma_chan(chan);
1801	struct sdma_engine *sdma = sdmac->sdma;
1802
1803	memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg));
1804
1805	if (dmaengine_cfg->peripheral_config) {
1806	struct sdma_peripheral_config *sdmacfg = dmaengine_cfg->peripheral_config;
1807	if (dmaengine_cfg->peripheral_size != sizeof(struct sdma_peripheral_config)) {
1808	dev_err(sdma->dev, "Invalid peripheral size %zu, expected %zu\n",
1809	dmaengine_cfg->peripheral_size,
1810	sizeof(struct sdma_peripheral_config));
1811	return -EINVAL;
1812	}
1813	sdmac->n_fifos_src = sdmacfg->n_fifos_src;
1814	sdmac->n_fifos_dst = sdmacfg->n_fifos_dst;
1815	sdmac->stride_fifos_src = sdmacfg->stride_fifos_src;
1816	sdmac->stride_fifos_dst = sdmacfg->stride_fifos_dst;
1817	sdmac->words_per_fifo = sdmacfg->words_per_fifo;
1818	sdmac->sw_done = sdmacfg->sw_done;
1819	}
1820
1821	/* Set ENBLn earlier to make sure dma request triggered after that */
1822	if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)
1823	return -EINVAL;
1824	sdma_event_enable(sdmac, event: sdmac->event_id0);
1825
1826	if (sdmac->event_id1) {
1827	if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)
1828	return -EINVAL;
1829	sdma_event_enable(sdmac, event: sdmac->event_id1);
1830	}
1831
1832	return 0;
1833	}
1834
1835	static enum dma_status sdma_tx_status(struct dma_chan *chan,
1836	dma_cookie_t cookie,
1837	struct dma_tx_state *txstate)
1838	{
1839	struct sdma_channel *sdmac = to_sdma_chan(chan);
1840	struct sdma_desc *desc = NULL;
1841	u32 residue;
1842	struct virt_dma_desc *vd;
1843	enum dma_status ret;
1844	unsigned long flags;
1845
1846	ret = dma_cookie_status(chan, cookie, state: txstate);
1847	if (ret == DMA_COMPLETE || !txstate)
1848	return ret;
1849
1850	spin_lock_irqsave(&sdmac->vc.lock, flags);
1851
1852	vd = vchan_find_desc(&sdmac->vc, cookie);
1853	if (vd)
1854	desc = to_sdma_desc(t: &vd->tx);
1855	else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie)
1856	desc = sdmac->desc;
1857
1858	if (desc) {
1859	if (sdmac->flags & IMX_DMA_SG_LOOP)
1860	residue = (desc->num_bd - desc->buf_ptail) *
1861	desc->period_len - desc->chn_real_count;
1862	else
1863	residue = desc->chn_count - desc->chn_real_count;
1864	} else {
1865	residue = 0;
1866	}
1867
1868	spin_unlock_irqrestore(lock: &sdmac->vc.lock, flags);
1869
1870	dma_set_tx_state(st: txstate, last: chan->completed_cookie, used: chan->cookie,
1871	residue);
1872
1873	return sdmac->status;
1874	}
1875
1876	static void sdma_issue_pending(struct dma_chan *chan)
1877	{
1878	struct sdma_channel *sdmac = to_sdma_chan(chan);
1879	unsigned long flags;
1880
1881	spin_lock_irqsave(&sdmac->vc.lock, flags);
1882	if (vchan_issue_pending(vc: &sdmac->vc) && !sdmac->desc)
1883	sdma_start_desc(sdmac);
1884	spin_unlock_irqrestore(lock: &sdmac->vc.lock, flags);
1885	}
1886
1887	#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34
1888	#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38
1889	#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 45
1890	#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 46
1891
1892	static void sdma_add_scripts(struct sdma_engine *sdma,
1893	const struct sdma_script_start_addrs *addr)
1894	{
1895	s32 *addr_arr = (u32 *)addr;
1896	s32 *saddr_arr = (u32 *)sdma->script_addrs;
1897	int i;
1898
1899	/* use the default firmware in ROM if missing external firmware */
1900	if (!sdma->script_number)
1901	sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
1902
1903	if (sdma->script_number > sizeof(struct sdma_script_start_addrs)
1904	/ sizeof(s32)) {
1905	dev_err(sdma->dev,
1906	"SDMA script number %d not match with firmware.\n",
1907	sdma->script_number);
1908	return;
1909	}
1910
1911	for (i = 0; i < sdma->script_number; i++)
1912	if (addr_arr[i] > 0)
1913	saddr_arr[i] = addr_arr[i];
1914
1915	/*
1916	* For compatibility with NXP internal legacy kernel before 4.19 which
1917	* is based on uart ram script and mainline kernel based on uart rom
1918	* script, both uart ram/rom scripts are present in newer sdma
1919	* firmware. Use the rom versions if they are present (V3 or newer).
1920	*/
1921	if (sdma->script_number >= SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3) {
1922	if (addr->uart_2_mcu_rom_addr)
1923	sdma->script_addrs->uart_2_mcu_addr = addr->uart_2_mcu_rom_addr;
1924	if (addr->uartsh_2_mcu_rom_addr)
1925	sdma->script_addrs->uartsh_2_mcu_addr = addr->uartsh_2_mcu_rom_addr;
1926	}
1927	}
1928
1929	static void sdma_load_firmware(const struct firmware *fw, void *context)
1930	{
1931	struct sdma_engine *sdma = context;
1932	const struct sdma_firmware_header *header;
1933	const struct sdma_script_start_addrs *addr;
1934	unsigned short *ram_code;
1935
1936	if (!fw) {
1937	dev_info(sdma->dev, "external firmware not found, using ROM firmware\n");
1938	/* In this case we just use the ROM firmware. */
1939	return;
1940	}
1941
1942	if (fw->size < sizeof(*header))
1943	goto err_firmware;
1944
1945	header = (struct sdma_firmware_header *)fw->data;
1946
1947	if (header->magic != SDMA_FIRMWARE_MAGIC)
1948	goto err_firmware;
1949	if (header->ram_code_start + header->ram_code_size > fw->size)
1950	goto err_firmware;
1951	switch (header->version_major) {
1952	case 1:
1953	sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
1954	break;
1955	case 2:
1956	sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2;
1957	break;
1958	case 3:
1959	sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3;
1960	break;
1961	case 4:
1962	sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4;
1963	break;
1964	default:
1965	dev_err(sdma->dev, "unknown firmware version\n");
1966	goto err_firmware;
1967	}
1968
1969	addr = (void *)header + header->script_addrs_start;
1970	ram_code = (void *)header + header->ram_code_start;
1971
1972	clk_enable(clk: sdma->clk_ipg);
1973	clk_enable(clk: sdma->clk_ahb);
1974	/* download the RAM image for SDMA */
1975	sdma_load_script(sdma, buf: ram_code,
1976	size: header->ram_code_size,
1977	address: addr->ram_code_start_addr);
1978	clk_disable(clk: sdma->clk_ipg);
1979	clk_disable(clk: sdma->clk_ahb);
1980
1981	sdma_add_scripts(sdma, addr);
1982
1983	sdma->fw_loaded = true;
1984
1985	dev_info(sdma->dev, "loaded firmware %d.%d\n",
1986	header->version_major,
1987	header->version_minor);
1988
1989	err_firmware:
1990	release_firmware(fw);
1991	}
1992
1993	#define EVENT_REMAP_CELLS 3
1994
1995	static int sdma_event_remap(struct sdma_engine *sdma)
1996	{
1997	struct device_node *np = sdma->dev->of_node;
1998	struct device_node *gpr_np = of_parse_phandle(np, phandle_name: "gpr", index: 0);
1999	struct property *event_remap;
2000	struct regmap *gpr;
2001	char propname[] = "fsl,sdma-event-remap";
2002	u32 reg, val, shift, num_map, i;
2003	int ret = 0;
2004
2005	if (IS_ERR(ptr: np) || !gpr_np)
2006	goto out;
2007
2008	event_remap = of_find_property(np, name: propname, NULL);
2009	num_map = event_remap ? (event_remap->length / sizeof(u32)) : 0;
2010	if (!num_map) {
2011	dev_dbg(sdma->dev, "no event needs to be remapped\n");
2012	goto out;
2013	} else if (num_map % EVENT_REMAP_CELLS) {
2014	dev_err(sdma->dev, "the property %s must modulo %d\n",
2015	propname, EVENT_REMAP_CELLS);
2016	ret = -EINVAL;
2017	goto out;
2018	}
2019
2020	gpr = syscon_node_to_regmap(np: gpr_np);
2021	if (IS_ERR(ptr: gpr)) {
2022	dev_err(sdma->dev, "failed to get gpr regmap\n");
2023	ret = PTR_ERR(ptr: gpr);
2024	goto out;
2025	}
2026
2027	for (i = 0; i < num_map; i += EVENT_REMAP_CELLS) {
2028	ret = of_property_read_u32_index(np, propname, index: i, out_value: &reg);
2029	if (ret) {
2030	dev_err(sdma->dev, "failed to read property %s index %d\n",
2031	propname, i);
2032	goto out;
2033	}
2034
2035	ret = of_property_read_u32_index(np, propname, index: i + 1, out_value: &shift);
2036	if (ret) {
2037	dev_err(sdma->dev, "failed to read property %s index %d\n",
2038	propname, i + 1);
2039	goto out;
2040	}
2041
2042	ret = of_property_read_u32_index(np, propname, index: i + 2, out_value: &val);
2043	if (ret) {
2044	dev_err(sdma->dev, "failed to read property %s index %d\n",
2045	propname, i + 2);
2046	goto out;
2047	}
2048
2049	regmap_update_bits(map: gpr, reg, BIT(shift), val: val << shift);
2050	}
2051
2052	out:
2053	if (gpr_np)
2054	of_node_put(node: gpr_np);
2055
2056	return ret;
2057	}
2058
2059	static int sdma_get_firmware(struct sdma_engine *sdma,
2060	const char *fw_name)
2061	{
2062	int ret;
2063
2064	ret = request_firmware_nowait(THIS_MODULE,
2065	FW_ACTION_UEVENT, name: fw_name, device: sdma->dev,
2066	GFP_KERNEL, context: sdma, cont: sdma_load_firmware);
2067
2068	return ret;
2069	}
2070
2071	static int sdma_init(struct sdma_engine *sdma)
2072	{
2073	int i, ret;
2074	dma_addr_t ccb_phys;
2075
2076	ret = clk_enable(clk: sdma->clk_ipg);
2077	if (ret)
2078	return ret;
2079	ret = clk_enable(clk: sdma->clk_ahb);
2080	if (ret)
2081	goto disable_clk_ipg;
2082
2083	if (sdma->drvdata->check_ratio &&
2084	(clk_get_rate(clk: sdma->clk_ahb) == clk_get_rate(clk: sdma->clk_ipg)))
2085	sdma->clk_ratio = 1;
2086
2087	/* Be sure SDMA has not started yet */
2088	writel_relaxed(0, sdma->regs + SDMA_H_C0PTR);
2089
2090	sdma->channel_control = dma_alloc_coherent(dev: sdma->dev,
2091	MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control) +
2092	sizeof(struct sdma_context_data),
2093	dma_handle: &ccb_phys, GFP_KERNEL);
2094
2095	if (!sdma->channel_control) {
2096	ret = -ENOMEM;
2097	goto err_dma_alloc;
2098	}
2099
2100	sdma->context = (void *)sdma->channel_control +
2101	MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control);
2102	sdma->context_phys = ccb_phys +
2103	MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control);
2104
2105	/* disable all channels */
2106	for (i = 0; i < sdma->drvdata->num_events; i++)
2107	writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i));
2108
2109	/* All channels have priority 0 */
2110	for (i = 0; i < MAX_DMA_CHANNELS; i++)
2111	writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4);
2112
2113	ret = sdma_request_channel0(sdma);
2114	if (ret)
2115	goto err_dma_alloc;
2116
2117	sdma_config_ownership(sdmac: &sdma->channel[0], event_override: false, mcu_override: true, dsp_override: false);
2118
2119	/* Set Command Channel (Channel Zero) */
2120	writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR);
2121
2122	/* Set bits of CONFIG register but with static context switching */
2123	if (sdma->clk_ratio)
2124	writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG);
2125	else
2126	writel_relaxed(0, sdma->regs + SDMA_H_CONFIG);
2127
2128	writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR);
2129
2130	/* Initializes channel's priorities */
2131	sdma_set_channel_priority(sdmac: &sdma->channel[0], priority: 7);
2132
2133	clk_disable(clk: sdma->clk_ipg);
2134	clk_disable(clk: sdma->clk_ahb);
2135
2136	return 0;
2137
2138	err_dma_alloc:
2139	clk_disable(clk: sdma->clk_ahb);
2140	disable_clk_ipg:
2141	clk_disable(clk: sdma->clk_ipg);
2142	dev_err(sdma->dev, "initialisation failed with %d\n", ret);
2143	return ret;
2144	}
2145
2146	static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param)
2147	{
2148	struct sdma_channel *sdmac = to_sdma_chan(chan);
2149	struct imx_dma_data *data = fn_param;
2150
2151	if (!imx_dma_is_general_purpose(chan))
2152	return false;
2153
2154	sdmac->data = *data;
2155	chan->private = &sdmac->data;
2156
2157	return true;
2158	}
2159
2160	static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec,
2161	struct of_dma *ofdma)
2162	{
2163	struct sdma_engine *sdma = ofdma->of_dma_data;
2164	dma_cap_mask_t mask = sdma->dma_device.cap_mask;
2165	struct imx_dma_data data;
2166
2167	if (dma_spec->args_count != 3)
2168	return NULL;
2169
2170	data.dma_request = dma_spec->args[0];
2171	data.peripheral_type = dma_spec->args[1];
2172	data.priority = dma_spec->args[2];
2173	/*
2174	* init dma_request2 to zero, which is not used by the dts.
2175	* For P2P, dma_request2 is init from dma_request_channel(),
2176	* chan->private will point to the imx_dma_data, and in
2177	* device_alloc_chan_resources(), imx_dma_data.dma_request2 will
2178	* be set to sdmac->event_id1.
2179	*/
2180	data.dma_request2 = 0;
2181
2182	return __dma_request_channel(mask: &mask, fn: sdma_filter_fn, fn_param: &data,
2183	np: ofdma->of_node);
2184	}
2185
2186	static int sdma_probe(struct platform_device *pdev)
2187	{
2188	struct device_node *np = pdev->dev.of_node;
2189	struct device_node *spba_bus;
2190	const char *fw_name;
2191	int ret;
2192	int irq;
2193	struct resource spba_res;
2194	int i;
2195	struct sdma_engine *sdma;
2196	s32 *saddr_arr;
2197
2198	ret = dma_coerce_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(32));
2199	if (ret)
2200	return ret;
2201
2202	sdma = devm_kzalloc(dev: &pdev->dev, size: sizeof(*sdma), GFP_KERNEL);
2203	if (!sdma)
2204	return -ENOMEM;
2205
2206	spin_lock_init(&sdma->channel_0_lock);
2207
2208	sdma->dev = &pdev->dev;
2209	sdma->drvdata = of_device_get_match_data(dev: sdma->dev);
2210
2211	irq = platform_get_irq(pdev, 0);
2212	if (irq < 0)
2213	return irq;
2214
2215	sdma->regs = devm_platform_ioremap_resource(pdev, index: 0);
2216	if (IS_ERR(ptr: sdma->regs))
2217	return PTR_ERR(ptr: sdma->regs);
2218
2219	sdma->clk_ipg = devm_clk_get(dev: &pdev->dev, id: "ipg");
2220	if (IS_ERR(ptr: sdma->clk_ipg))
2221	return PTR_ERR(ptr: sdma->clk_ipg);
2222
2223	sdma->clk_ahb = devm_clk_get(dev: &pdev->dev, id: "ahb");
2224	if (IS_ERR(ptr: sdma->clk_ahb))
2225	return PTR_ERR(ptr: sdma->clk_ahb);
2226
2227	ret = clk_prepare(clk: sdma->clk_ipg);
2228	if (ret)
2229	return ret;
2230
2231	ret = clk_prepare(clk: sdma->clk_ahb);
2232	if (ret)
2233	goto err_clk;
2234
2235	ret = devm_request_irq(dev: &pdev->dev, irq, handler: sdma_int_handler, irqflags: 0,
2236	devname: dev_name(dev: &pdev->dev), dev_id: sdma);
2237	if (ret)
2238	goto err_irq;
2239
2240	sdma->irq = irq;
2241
2242	sdma->script_addrs = kzalloc(size: sizeof(*sdma->script_addrs), GFP_KERNEL);
2243	if (!sdma->script_addrs) {
2244	ret = -ENOMEM;
2245	goto err_irq;
2246	}
2247
2248	/* initially no scripts available */
2249	saddr_arr = (s32 *)sdma->script_addrs;
2250	for (i = 0; i < sizeof(*sdma->script_addrs) / sizeof(s32); i++)
2251	saddr_arr[i] = -EINVAL;
2252
2253	dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask);
2254	dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask);
2255	dma_cap_set(DMA_MEMCPY, sdma->dma_device.cap_mask);
2256	dma_cap_set(DMA_PRIVATE, sdma->dma_device.cap_mask);
2257
2258	INIT_LIST_HEAD(list: &sdma->dma_device.channels);
2259	/* Initialize channel parameters */
2260	for (i = 0; i < MAX_DMA_CHANNELS; i++) {
2261	struct sdma_channel *sdmac = &sdma->channel[i];
2262
2263	sdmac->sdma = sdma;
2264
2265	sdmac->channel = i;
2266	sdmac->vc.desc_free = sdma_desc_free;
2267	INIT_LIST_HEAD(list: &sdmac->terminated);
2268	INIT_WORK(&sdmac->terminate_worker,
2269	sdma_channel_terminate_work);
2270	/*
2271	* Add the channel to the DMAC list. Do not add channel 0 though
2272	* because we need it internally in the SDMA driver. This also means
2273	* that channel 0 in dmaengine counting matches sdma channel 1.
2274	*/
2275	if (i)
2276	vchan_init(vc: &sdmac->vc, dmadev: &sdma->dma_device);
2277	}
2278
2279	ret = sdma_init(sdma);
2280	if (ret)
2281	goto err_init;
2282
2283	ret = sdma_event_remap(sdma);
2284	if (ret)
2285	goto err_init;
2286
2287	if (sdma->drvdata->script_addrs)
2288	sdma_add_scripts(sdma, addr: sdma->drvdata->script_addrs);
2289
2290	sdma->dma_device.dev = &pdev->dev;
2291
2292	sdma->dma_device.device_alloc_chan_resources = sdma_alloc_chan_resources;
2293	sdma->dma_device.device_free_chan_resources = sdma_free_chan_resources;
2294	sdma->dma_device.device_tx_status = sdma_tx_status;
2295	sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg;
2296	sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic;
2297	sdma->dma_device.device_config = sdma_config;
2298	sdma->dma_device.device_terminate_all = sdma_terminate_all;
2299	sdma->dma_device.device_synchronize = sdma_channel_synchronize;
2300	sdma->dma_device.src_addr_widths = SDMA_DMA_BUSWIDTHS;
2301	sdma->dma_device.dst_addr_widths = SDMA_DMA_BUSWIDTHS;
2302	sdma->dma_device.directions = SDMA_DMA_DIRECTIONS;
2303	sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
2304	sdma->dma_device.device_prep_dma_memcpy = sdma_prep_memcpy;
2305	sdma->dma_device.device_issue_pending = sdma_issue_pending;
2306	sdma->dma_device.copy_align = 2;
2307	dma_set_max_seg_size(dev: sdma->dma_device.dev, SDMA_BD_MAX_CNT);
2308
2309	platform_set_drvdata(pdev, data: sdma);
2310
2311	ret = dma_async_device_register(device: &sdma->dma_device);
2312	if (ret) {
2313	dev_err(&pdev->dev, "unable to register\n");
2314	goto err_init;
2315	}
2316
2317	if (np) {
2318	ret = of_dma_controller_register(np, of_dma_xlate: sdma_xlate, data: sdma);
2319	if (ret) {
2320	dev_err(&pdev->dev, "failed to register controller\n");
2321	goto err_register;
2322	}
2323
2324	spba_bus = of_find_compatible_node(NULL, NULL, compat: "fsl,spba-bus");
2325	ret = of_address_to_resource(dev: spba_bus, index: 0, r: &spba_res);
2326	if (!ret) {
2327	sdma->spba_start_addr = spba_res.start;
2328	sdma->spba_end_addr = spba_res.end;
2329	}
2330	of_node_put(node: spba_bus);
2331	}
2332
2333	/*
2334	* Because that device tree does not encode ROM script address,
2335	* the RAM script in firmware is mandatory for device tree
2336	* probe, otherwise it fails.
2337	*/
2338	ret = of_property_read_string(np, propname: "fsl,sdma-ram-script-name",
2339	out_string: &fw_name);
2340	if (ret) {
2341	dev_warn(&pdev->dev, "failed to get firmware name\n");
2342	} else {
2343	ret = sdma_get_firmware(sdma, fw_name);
2344	if (ret)
2345	dev_warn(&pdev->dev, "failed to get firmware from device tree\n");
2346	}
2347
2348	return 0;
2349
2350	err_register:
2351	dma_async_device_unregister(device: &sdma->dma_device);
2352	err_init:
2353	kfree(objp: sdma->script_addrs);
2354	err_irq:
2355	clk_unprepare(clk: sdma->clk_ahb);
2356	err_clk:
2357	clk_unprepare(clk: sdma->clk_ipg);
2358	return ret;
2359	}
2360
2361	static void sdma_remove(struct platform_device *pdev)
2362	{
2363	struct sdma_engine *sdma = platform_get_drvdata(pdev);
2364	int i;
2365
2366	devm_free_irq(dev: &pdev->dev, irq: sdma->irq, dev_id: sdma);
2367	dma_async_device_unregister(device: &sdma->dma_device);
2368	kfree(objp: sdma->script_addrs);
2369	clk_unprepare(clk: sdma->clk_ahb);
2370	clk_unprepare(clk: sdma->clk_ipg);
2371	/* Kill the tasklet */
2372	for (i = 0; i < MAX_DMA_CHANNELS; i++) {
2373	struct sdma_channel *sdmac = &sdma->channel[i];
2374
2375	tasklet_kill(t: &sdmac->vc.task);
2376	sdma_free_chan_resources(chan: &sdmac->vc.chan);
2377	}
2378
2379	platform_set_drvdata(pdev, NULL);
2380	}
2381
2382	static struct platform_driver sdma_driver = {
2383	.driver = {
2384	.name = "imx-sdma",
2385	.of_match_table = sdma_dt_ids,
2386	},
2387	.remove_new = sdma_remove,
2388	.probe = sdma_probe,
2389	};
2390
2391	module_platform_driver(sdma_driver);
2392
2393	MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>");
2394	MODULE_DESCRIPTION("i.MX SDMA driver");
2395	#if IS_ENABLED(CONFIG_SOC_IMX6Q)
2396	MODULE_FIRMWARE("imx/sdma/sdma-imx6q.bin");
2397	#endif
2398	#if IS_ENABLED(CONFIG_SOC_IMX7D) || IS_ENABLED(CONFIG_SOC_IMX8M)
2399	MODULE_FIRMWARE("imx/sdma/sdma-imx7d.bin");
2400	#endif
2401	MODULE_LICENSE("GPL");
2402