pl330.c source code [linux/drivers/dma/pl330.c]

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright (c) 2012 Samsung Electronics Co., Ltd.
4	* http://www.samsung.com
5	*
6	* Copyright (C) 2010 Samsung Electronics Co. Ltd.
7	* Jaswinder Singh <jassi.brar@samsung.com>
8	*/
9
10	#include <linux/debugfs.h>
11	#include <linux/kernel.h>
12	#include <linux/io.h>
13	#include <linux/init.h>
14	#include <linux/slab.h>
15	#include <linux/module.h>
16	#include <linux/string.h>
17	#include <linux/delay.h>
18	#include <linux/interrupt.h>
19	#include <linux/dma-mapping.h>
20	#include <linux/dmaengine.h>
21	#include <linux/amba/bus.h>
22	#include <linux/scatterlist.h>
23	#include <linux/of.h>
24	#include <linux/of_dma.h>
25	#include <linux/err.h>
26	#include <linux/pm_runtime.h>
27	#include <linux/bug.h>
28	#include <linux/reset.h>
29
30	#include "dmaengine.h"
31	#define PL330_MAX_CHAN 8
32	#define PL330_MAX_IRQS 32
33	#define PL330_MAX_PERI 32
34	#define PL330_MAX_BURST 16
35
36	#define PL330_QUIRK_BROKEN_NO_FLUSHP BIT(0)
37	#define PL330_QUIRK_PERIPH_BURST BIT(1)
38
39	enum pl330_cachectrl {
40	CCTRL0, / Noncacheable and nonbufferable /
41	CCTRL1, / Bufferable only /
42	CCTRL2, / Cacheable, but do not allocate /
43	CCTRL3, / Cacheable and bufferable, but do not allocate /
44	INVALID1, / AWCACHE = 0x1000 /
45	INVALID2,
46	CCTRL6, / Cacheable write-through, allocate on writes only /
47	CCTRL7, / Cacheable write-back, allocate on writes only /
48	};
49
50	enum pl330_byteswap {
51	SWAP_NO,
52	SWAP_2,
53	SWAP_4,
54	SWAP_8,
55	SWAP_16,
56	};
57
58	/ Register and Bit field Definitions /
59	#define DS 0x0
60	#define DS_ST_STOP 0x0
61	#define DS_ST_EXEC 0x1
62	#define DS_ST_CMISS 0x2
63	#define DS_ST_UPDTPC 0x3
64	#define DS_ST_WFE 0x4
65	#define DS_ST_ATBRR 0x5
66	#define DS_ST_QBUSY 0x6
67	#define DS_ST_WFP 0x7
68	#define DS_ST_KILL 0x8
69	#define DS_ST_CMPLT 0x9
70	#define DS_ST_FLTCMP 0xe
71	#define DS_ST_FAULT 0xf
72
73	#define DPC 0x4
74	#define INTEN 0x20
75	#define ES 0x24
76	#define INTSTATUS 0x28
77	#define INTCLR 0x2c
78	#define FSM 0x30
79	#define FSC 0x34
80	#define FTM 0x38
81
82	#define _FTC 0x40
83	#define FTC(n) (_FTC + (n)*0x4)
84
85	#define _CS 0x100
86	#define CS(n) (_CS + (n)*0x8)
87	#define CS_CNS (1 << 21)
88
89	#define _CPC 0x104
90	#define CPC(n) (_CPC + (n)*0x8)
91
92	#define _SA 0x400
93	#define SA(n) (_SA + (n)*0x20)
94
95	#define _DA 0x404
96	#define DA(n) (_DA + (n)*0x20)
97
98	#define _CC 0x408
99	#define CC(n) (_CC + (n)*0x20)
100
101	#define CC_SRCINC (1 << 0)
102	#define CC_DSTINC (1 << 14)
103	#define CC_SRCPRI (1 << 8)
104	#define CC_DSTPRI (1 << 22)
105	#define CC_SRCNS (1 << 9)
106	#define CC_DSTNS (1 << 23)
107	#define CC_SRCIA (1 << 10)
108	#define CC_DSTIA (1 << 24)
109	#define CC_SRCBRSTLEN_SHFT 4
110	#define CC_DSTBRSTLEN_SHFT 18
111	#define CC_SRCBRSTSIZE_SHFT 1
112	#define CC_DSTBRSTSIZE_SHFT 15
113	#define CC_SRCCCTRL_SHFT 11
114	#define CC_SRCCCTRL_MASK 0x7
115	#define CC_DSTCCTRL_SHFT 25
116	#define CC_DRCCCTRL_MASK 0x7
117	#define CC_SWAP_SHFT 28
118
119	#define _LC0 0x40c
120	#define LC0(n) (_LC0 + (n)*0x20)
121
122	#define _LC1 0x410
123	#define LC1(n) (_LC1 + (n)*0x20)
124
125	#define DBGSTATUS 0xd00
126	#define DBG_BUSY (1 << 0)
127
128	#define DBGCMD 0xd04
129	#define DBGINST0 0xd08
130	#define DBGINST1 0xd0c
131
132	#define CR0 0xe00
133	#define CR1 0xe04
134	#define CR2 0xe08
135	#define CR3 0xe0c
136	#define CR4 0xe10
137	#define CRD 0xe14
138
139	#define PERIPH_ID 0xfe0
140	#define PERIPH_REV_SHIFT 20
141	#define PERIPH_REV_MASK 0xf
142	#define PERIPH_REV_R0P0 0
143	#define PERIPH_REV_R1P0 1
144	#define PERIPH_REV_R1P1 2
145
146	#define CR0_PERIPH_REQ_SET (1 << 0)
147	#define CR0_BOOT_EN_SET (1 << 1)
148	#define CR0_BOOT_MAN_NS (1 << 2)
149	#define CR0_NUM_CHANS_SHIFT 4
150	#define CR0_NUM_CHANS_MASK 0x7
151	#define CR0_NUM_PERIPH_SHIFT 12
152	#define CR0_NUM_PERIPH_MASK 0x1f
153	#define CR0_NUM_EVENTS_SHIFT 17
154	#define CR0_NUM_EVENTS_MASK 0x1f
155
156	#define CR1_ICACHE_LEN_SHIFT 0
157	#define CR1_ICACHE_LEN_MASK 0x7
158	#define CR1_NUM_ICACHELINES_SHIFT 4
159	#define CR1_NUM_ICACHELINES_MASK 0xf
160
161	#define CRD_DATA_WIDTH_SHIFT 0
162	#define CRD_DATA_WIDTH_MASK 0x7
163	#define CRD_WR_CAP_SHIFT 4
164	#define CRD_WR_CAP_MASK 0x7
165	#define CRD_WR_Q_DEP_SHIFT 8
166	#define CRD_WR_Q_DEP_MASK 0xf
167	#define CRD_RD_CAP_SHIFT 12
168	#define CRD_RD_CAP_MASK 0x7
169	#define CRD_RD_Q_DEP_SHIFT 16
170	#define CRD_RD_Q_DEP_MASK 0xf
171	#define CRD_DATA_BUFF_SHIFT 20
172	#define CRD_DATA_BUFF_MASK 0x3ff
173
174	#define PART 0x330
175	#define DESIGNER 0x41
176	#define REVISION 0x0
177	#define INTEG_CFG 0x0
178	#define PERIPH_ID_VAL ((PART << 0) \| (DESIGNER << 12))
179
180	#define PL330_STATE_STOPPED (1 << 0)
181	#define PL330_STATE_EXECUTING (1 << 1)
182	#define PL330_STATE_WFE (1 << 2)
183	#define PL330_STATE_FAULTING (1 << 3)
184	#define PL330_STATE_COMPLETING (1 << 4)
185	#define PL330_STATE_WFP (1 << 5)
186	#define PL330_STATE_KILLING (1 << 6)
187	#define PL330_STATE_FAULT_COMPLETING (1 << 7)
188	#define PL330_STATE_CACHEMISS (1 << 8)
189	#define PL330_STATE_UPDTPC (1 << 9)
190	#define PL330_STATE_ATBARRIER (1 << 10)
191	#define PL330_STATE_QUEUEBUSY (1 << 11)
192	#define PL330_STATE_INVALID (1 << 15)
193
194	#define PL330_STABLE_STATES (PL330_STATE_STOPPED \| PL330_STATE_EXECUTING \
195	\| PL330_STATE_WFE \| PL330_STATE_FAULTING)
196
197	#define CMD_DMAADDH 0x54
198	#define CMD_DMAEND 0x00
199	#define CMD_DMAFLUSHP 0x35
200	#define CMD_DMAGO 0xa0
201	#define CMD_DMALD 0x04
202	#define CMD_DMALDP 0x25
203	#define CMD_DMALP 0x20
204	#define CMD_DMALPEND 0x28
205	#define CMD_DMAKILL 0x01
206	#define CMD_DMAMOV 0xbc
207	#define CMD_DMANOP 0x18
208	#define CMD_DMARMB 0x12
209	#define CMD_DMASEV 0x34
210	#define CMD_DMAST 0x08
211	#define CMD_DMASTP 0x29
212	#define CMD_DMASTZ 0x0c
213	#define CMD_DMAWFE 0x36
214	#define CMD_DMAWFP 0x30
215	#define CMD_DMAWMB 0x13
216
217	#define SZ_DMAADDH 3
218	#define SZ_DMAEND 1
219	#define SZ_DMAFLUSHP 2
220	#define SZ_DMALD 1
221	#define SZ_DMALDP 2
222	#define SZ_DMALP 2
223	#define SZ_DMALPEND 2
224	#define SZ_DMAKILL 1
225	#define SZ_DMAMOV 6
226	#define SZ_DMANOP 1
227	#define SZ_DMARMB 1
228	#define SZ_DMASEV 2
229	#define SZ_DMAST 1
230	#define SZ_DMASTP 2
231	#define SZ_DMASTZ 1
232	#define SZ_DMAWFE 2
233	#define SZ_DMAWFP 2
234	#define SZ_DMAWMB 1
235	#define SZ_DMAGO 6
236
237	#define BRST_LEN(ccr) ((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1)
238	#define BRST_SIZE(ccr) (1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7))
239
240	#define BYTE_TO_BURST(b, ccr) ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr))
241	#define BURST_TO_BYTE(c, ccr) ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr))
242
243	/*
244	* With 256 bytes, we can do more than 2.5MB and 5MB xfers per req
245	* at 1byte/burst for P<->M and M<->M respectively.
246	* For typical scenario, at 1word/burst, 10MB and 20MB xfers per req
247	* should be enough for P<->M and M<->M respectively.
248	*/
249	#define MCODE_BUFF_PER_REQ 256
250
251	/ Use this _only_ to wait on transient states /
252	#define UNTIL(t, s) while (!(_state(t) & (s))) cpu_relax();
253
254	#ifdef PL330_DEBUG_MCGEN
255	static unsigned cmd_line;
256	#define PL330_DBGCMD_DUMP(off, x...) do { \
257	printk("%x:", cmd_line); \
258	printk(KERN_CONT x); \
259	cmd_line += off; \
260	} while (0)
261	#define PL330_DBGMC_START(addr) (cmd_line = addr)
262	#else
263	#define PL330_DBGCMD_DUMP(off, x...) do {} while (0)
264	#define PL330_DBGMC_START(addr) do {} while (0)
265	#endif
266
267	/ The number of default descriptors /
268
269	#define NR_DEFAULT_DESC 16
270
271	/ Delay for runtime PM autosuspend, ms /
272	#define PL330_AUTOSUSPEND_DELAY 20
273
274	/ Populated by the PL330 core driver for DMA API driver's info /
275	struct pl330_config {
276	u32 periph_id;
277	#define DMAC_MODE_NS (1 << 0)
278	unsigned int mode;
279	unsigned int data_bus_width:`10`; / In number of bits /
280	unsigned int data_buf_dep:`11`;
281	unsigned int num_chan:`4`;
282	unsigned int num_peri:`6`;
283	u32 peri_ns;
284	unsigned int num_events:`6`;
285	u32 irq_ns;
286	};
287
288	/*
289	* Request Configuration.
290	* The PL330 core does not modify this and uses the last
291	* working configuration if the request doesn't provide any.
292	*
293	* The Client may want to provide this info only for the
294	* first request and a request with new settings.
295	*/
296	struct pl330_reqcfg {
297	/ Address Incrementing /
298	unsigned dst_inc:`1`;
299	unsigned src_inc:`1`;
300
301	/*
302	* For now, the SRC & DST protection levels
303	* and burst size/length are assumed same.
304	*/
305	bool nonsecure;
306	bool privileged;
307	bool insnaccess;
308	unsigned brst_len:`5`;
309	unsigned brst_size:`3`; / in power of 2 /
310
311	enum pl330_cachectrl dcctl;
312	enum pl330_cachectrl scctl;
313	enum pl330_byteswap swap;
314	struct pl330_config *pcfg;
315	};
316
317	/*
318	* One cycle of DMAC operation.
319	* There may be more than one xfer in a request.
320	*/
321	struct pl330_xfer {
322	u32 src_addr;
323	u32 dst_addr;
324	/ Size to xfer /
325	u32 bytes;
326	};
327
328	/ The xfer callbacks are made with one of these arguments. /
329	enum pl330_op_err {
330	/ The all xfers in the request were success. /
331	PL330_ERR_NONE,
332	/ If req aborted due to global error. /
333	PL330_ERR_ABORT,
334	/ If req failed due to problem with Channel. /
335	PL330_ERR_FAIL,
336	};
337
338	enum dmamov_dst {
339	SAR = `0`,
340	CCR,
341	DAR,
342	};
343
344	enum pl330_dst {
345	SRC = `0`,
346	DST,
347	};
348
349	enum pl330_cond {
350	SINGLE,
351	BURST,
352	ALWAYS,
353	};
354
355	struct dma_pl330_desc;
356
357	struct _pl330_req {
358	u32 mc_bus;
359	void *mc_cpu;
360	struct dma_pl330_desc *desc;
361	};
362
363	/ ToBeDone for tasklet /
364	struct _pl330_tbd {
365	bool reset_dmac;
366	bool reset_mngr;
367	u8 reset_chan;
368	};
369
370	/ A DMAC Thread /
371	struct pl330_thread {
372	u8 id;
373	int ev;
374	/ If the channel is not yet acquired by any client /
375	bool free;
376	/ Parent DMAC /
377	struct pl330_dmac *dmac;
378	/ Only two at a time /
379	struct _pl330_req req[`2`];
380	/ Index of the last enqueued request /
381	unsigned lstenq;
382	/ Index of the last submitted request or -1 if the DMA is stopped /
383	int req_running;
384	};
385
386	enum pl330_dmac_state {
387	UNINIT,
388	INIT,
389	DYING,
390	};
391
392	enum desc_status {
393	/ In the DMAC pool /
394	FREE,
395	/*
396	* Allocated to some channel during prep_xxx
397	* Also may be sitting on the work_list.
398	*/
399	PREP,
400	/*
401	* Sitting on the work_list and already submitted
402	* to the PL330 core. Not more than two descriptors
403	* of a channel can be BUSY at any time.
404	*/
405	BUSY,
406	/*
407	* Pause was called while descriptor was BUSY. Due to hardware
408	* limitations, only termination is possible for descriptors
409	* that have been paused.
410	*/
411	PAUSED,
412	/*
413	* Sitting on the channel work_list but xfer done
414	* by PL330 core
415	*/
416	DONE,
417	};
418
419	struct dma_pl330_chan {
420	/ Schedule desc completion /
421	struct tasklet_struct task;
422
423	/ DMA-Engine Channel /
424	struct dma_chan chan;
425
426	/ List of submitted descriptors /
427	struct list_head submitted_list;
428	/ List of issued descriptors /
429	struct list_head work_list;
430	/ List of completed descriptors /
431	struct list_head completed_list;
432
433	/ Pointer to the DMAC that manages this channel,*
434	* NULL if the channel is available to be acquired.
435	* As the parent, this DMAC also provides descriptors
436	* to the channel.
437	*/
438	struct pl330_dmac *dmac;
439
440	/ To protect channel manipulation /
441	spinlock_t lock;
442
443	/*
444	* Hardware channel thread of PL330 DMAC. NULL if the channel is
445	* available.
446	*/
447	struct pl330_thread *thread;
448
449	/ For D-to-M and M-to-D channels /
450	int burst_sz; / the peripheral fifo width /
451	int burst_len; / the number of burst /
452	phys_addr_t fifo_addr;
453	/ DMA-mapped view of the FIFO; may differ if an IOMMU is present /
454	dma_addr_t fifo_dma;
455	enum dma_data_direction dir;
456	struct dma_slave_config slave_config;
457
458	/ for cyclic capability /
459	bool cyclic;
460
461	/ for runtime pm tracking /
462	bool active;
463	};
464
465	struct pl330_dmac {
466	/ DMA-Engine Device /
467	struct dma_device ddma;
468
469	/ Pool of descriptors available for the DMAC's channels /
470	struct list_head desc_pool;
471	/ To protect desc_pool manipulation /
472	spinlock_t pool_lock;
473
474	/ Size of MicroCode buffers for each channel. /
475	unsigned mcbufsz;
476	/ ioremap'ed address of PL330 registers. /
477	void __iomem *base;
478	/ Populated by the PL330 core driver during pl330_add /
479	struct pl330_config pcfg;
480
481	spinlock_t lock;
482	/ Maximum possible events/irqs /
483	int events[`32`];
484	/ BUS address of MicroCode buffer /
485	dma_addr_t mcode_bus;
486	/ CPU address of MicroCode buffer /
487	void *mcode_cpu;
488	/ List of all Channel threads /
489	struct pl330_thread *channels;
490	/ Pointer to the MANAGER thread /
491	struct pl330_thread *manager;
492	/ To handle bad news in interrupt /
493	struct tasklet_struct tasks;
494	struct _pl330_tbd dmac_tbd;
495	/ State of DMAC operation /
496	enum pl330_dmac_state state;
497	/ Holds list of reqs with due callbacks /
498	struct list_head req_done;
499
500	/ Peripheral channels connected to this DMAC /
501	unsigned int num_peripherals;
502	struct dma_pl330_chan peripherals; /* keep at end /
503	int quirks;
504
505	struct reset_control *rstc;
506	struct reset_control *rstc_ocp;
507	};
508
509	static struct pl330_of_quirks {
510	char *quirk;
511	int id;
512	} of_quirks[] = {
513	{
514	.quirk = "arm,pl330-broken-no-flushp",
515	.id = PL330_QUIRK_BROKEN_NO_FLUSHP,
516	},
517	{
518	.quirk = "arm,pl330-periph-burst",
519	.id = PL330_QUIRK_PERIPH_BURST,
520	}
521	};
522
523	struct dma_pl330_desc {
524	/ To attach to a queue as child /
525	struct list_head node;
526
527	/ Descriptor for the DMA Engine API /
528	struct dma_async_tx_descriptor txd;
529
530	/ Xfer for PL330 core /
531	struct pl330_xfer px;
532
533	struct pl330_reqcfg rqcfg;
534
535	enum desc_status status;
536
537	int bytes_requested;
538	bool last;
539
540	/ The channel which currently holds this desc /
541	struct dma_pl330_chan *pchan;
542
543	enum dma_transfer_direction rqtype;
544	/ Index of peripheral for the xfer. /
545	unsigned peri:`5`;
546	/ Hook to attach to DMAC's list of reqs with due callback /
547	struct list_head rqd;
548	};
549
550	struct _xfer_spec {
551	u32 ccr;
552	struct dma_pl330_desc *desc;
553	};
554
555	static int pl330_config_write(struct dma_chan *chan,
556	struct dma_slave_config *slave_config,
557	enum dma_transfer_direction direction);
558
559	static inline bool _queue_full(struct pl330_thread *thrd)
560	{
561	return thrd->req[`0`].desc != NULL && thrd->req[`1`].desc != NULL;
562	}
563
564	static inline bool is_manager(struct pl330_thread *thrd)
565	{
566	return thrd->dmac->manager == thrd;
567	}
568
569	/ If manager of the thread is in Non-Secure mode /
570	static inline bool _manager_ns(struct pl330_thread *thrd)
571	{
572	return (thrd->dmac->pcfg.mode & DMAC_MODE_NS) ? true : false;
573	}
574
575	static inline u32 get_revision(u32 periph_id)
576	{
577	return (periph_id >> PERIPH_REV_SHIFT) & PERIPH_REV_MASK;
578	}
579
580	static inline u32 _emit_END(unsigned dry_run, u8 buf[])
581	{
582	if (dry_run)
583	return SZ_DMAEND;
584
585	buf[`0`] = CMD_DMAEND;
586
587	PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n");
588
589	return SZ_DMAEND;
590	}
591
592	static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri)
593	{
594	if (dry_run)
595	return SZ_DMAFLUSHP;
596
597	buf[`0`] = CMD_DMAFLUSHP;
598
599	peri &= `0x1f`;
600	peri <<= `3`;
601	buf[`1`] = peri;
602
603	PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> `3`);
604
605	return SZ_DMAFLUSHP;
606	}
607
608	static inline u32 _emit_LD(unsigned dry_run, u8 buf[], enum pl330_cond cond)
609	{
610	if (dry_run)
611	return SZ_DMALD;
612
613	buf[`0`] = CMD_DMALD;
614
615	if (cond == SINGLE)
616	buf[`0`] \|= (`0` << `1`) \| (`1` << `0`);
617	else if (cond == BURST)
618	buf[`0`] \|= (`1` << `1`) \| (`1` << `0`);
619
620	PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n",
621	cond == SINGLE ? `'S'` : (cond == BURST ? `'B'` : `'A'`));
622
623	return SZ_DMALD;
624	}
625
626	static inline u32 _emit_LDP(unsigned dry_run, u8 buf[],
627	enum pl330_cond cond, u8 peri)
628	{
629	if (dry_run)
630	return SZ_DMALDP;
631
632	buf[`0`] = CMD_DMALDP;
633
634	if (cond == BURST)
635	buf[`0`] \|= (`1` << `1`);
636
637	peri &= `0x1f`;
638	peri <<= `3`;
639	buf[`1`] = peri;
640
641	PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n",
642	cond == SINGLE ? `'S'` : `'B'`, peri >> `3`);
643
644	return SZ_DMALDP;
645	}
646
647	static inline u32 _emit_LP(unsigned dry_run, u8 buf[],
648	unsigned loop, u8 cnt)
649	{
650	if (dry_run)
651	return SZ_DMALP;
652
653	buf[`0`] = CMD_DMALP;
654
655	if (loop)
656	buf[`0`] \|= (`1` << `1`);
657
658	cnt--; / DMAC increments by 1 internally /
659	buf[`1`] = cnt;
660
661	PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? `'1'` : `'0'`, cnt);
662
663	return SZ_DMALP;
664	}
665
666	struct _arg_LPEND {
667	enum pl330_cond cond;
668	bool forever;
669	unsigned loop;
670	u8 bjump;
671	};
672
673	static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[],
674	const struct _arg_LPEND *arg)
675	{
676	enum pl330_cond cond = arg->cond;
677	bool forever = arg->forever;
678	unsigned loop = arg->loop;
679	u8 bjump = arg->bjump;
680
681	if (dry_run)
682	return SZ_DMALPEND;
683
684	buf[`0`] = CMD_DMALPEND;
685
686	if (loop)
687	buf[`0`] \|= (`1` << `2`);
688
689	if (!forever)
690	buf[`0`] \|= (`1` << `4`);
691
692	if (cond == SINGLE)
693	buf[`0`] \|= (`0` << `1`) \| (`1` << `0`);
694	else if (cond == BURST)
695	buf[`0`] \|= (`1` << `1`) \| (`1` << `0`);
696
697	buf[`1`] = bjump;
698
699	PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n",
700	forever ? "FE" : "END",
701	cond == SINGLE ? `'S'` : (cond == BURST ? `'B'` : `'A'`),
702	loop ? `'1'` : `'0'`,
703	bjump);
704
705	return SZ_DMALPEND;
706	}
707
708	static inline u32 _emit_KILL(unsigned dry_run, u8 buf[])
709	{
710	if (dry_run)
711	return SZ_DMAKILL;
712
713	buf[`0`] = CMD_DMAKILL;
714
715	return SZ_DMAKILL;
716	}
717
718	static inline u32 _emit_MOV(unsigned dry_run, u8 buf[],
719	enum dmamov_dst dst, u32 val)
720	{
721	if (dry_run)
722	return SZ_DMAMOV;
723
724	buf[`0`] = CMD_DMAMOV;
725	buf[`1`] = dst;
726	buf[`2`] = val;
727	buf[`3`] = val >> `8`;
728	buf[`4`] = val >> `16`;
729	buf[`5`] = val >> `24`;
730
731	PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n",
732	dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val);
733
734	return SZ_DMAMOV;
735	}
736
737	static inline u32 _emit_RMB(unsigned dry_run, u8 buf[])
738	{
739	if (dry_run)
740	return SZ_DMARMB;
741
742	buf[`0`] = CMD_DMARMB;
743
744	PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n");
745
746	return SZ_DMARMB;
747	}
748
749	static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev)
750	{
751	if (dry_run)
752	return SZ_DMASEV;
753
754	buf[`0`] = CMD_DMASEV;
755
756	ev &= `0x1f`;
757	ev <<= `3`;
758	buf[`1`] = ev;
759
760	PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> `3`);
761
762	return SZ_DMASEV;
763	}
764
765	static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond)
766	{
767	if (dry_run)
768	return SZ_DMAST;
769
770	buf[`0`] = CMD_DMAST;
771
772	if (cond == SINGLE)
773	buf[`0`] \|= (`0` << `1`) \| (`1` << `0`);
774	else if (cond == BURST)
775	buf[`0`] \|= (`1` << `1`) \| (`1` << `0`);
776
777	PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n",
778	cond == SINGLE ? `'S'` : (cond == BURST ? `'B'` : `'A'`));
779
780	return SZ_DMAST;
781	}
782
783	static inline u32 _emit_STP(unsigned dry_run, u8 buf[],
784	enum pl330_cond cond, u8 peri)
785	{
786	if (dry_run)
787	return SZ_DMASTP;
788
789	buf[`0`] = CMD_DMASTP;
790
791	if (cond == BURST)
792	buf[`0`] \|= (`1` << `1`);
793
794	peri &= `0x1f`;
795	peri <<= `3`;
796	buf[`1`] = peri;
797
798	PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n",
799	cond == SINGLE ? `'S'` : `'B'`, peri >> `3`);
800
801	return SZ_DMASTP;
802	}
803
804	static inline u32 _emit_WFP(unsigned dry_run, u8 buf[],
805	enum pl330_cond cond, u8 peri)
806	{
807	if (dry_run)
808	return SZ_DMAWFP;
809
810	buf[`0`] = CMD_DMAWFP;
811
812	if (cond == SINGLE)
813	buf[`0`] \|= (`0` << `1`) \| (`0` << `0`);
814	else if (cond == BURST)
815	buf[`0`] \|= (`1` << `1`) \| (`0` << `0`);
816	else
817	buf[`0`] \|= (`0` << `1`) \| (`1` << `0`);
818
819	peri &= `0x1f`;
820	peri <<= `3`;
821	buf[`1`] = peri;
822
823	PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n",
824	cond == SINGLE ? `'S'` : (cond == BURST ? `'B'` : `'P'`), peri >> `3`);
825
826	return SZ_DMAWFP;
827	}
828
829	static inline u32 _emit_WMB(unsigned dry_run, u8 buf[])
830	{
831	if (dry_run)
832	return SZ_DMAWMB;
833
834	buf[`0`] = CMD_DMAWMB;
835
836	PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n");
837
838	return SZ_DMAWMB;
839	}
840
841	struct _arg_GO {
842	u8 chan;
843	u32 addr;
844	unsigned ns;
845	};
846
847	static inline u32 _emit_GO(unsigned dry_run, u8 buf[],
848	const struct _arg_GO *arg)
849	{
850	u8 chan = arg->chan;
851	u32 addr = arg->addr;
852	unsigned ns = arg->ns;
853
854	if (dry_run)
855	return SZ_DMAGO;
856
857	buf[`0`] = CMD_DMAGO;
858	buf[`0`] \|= (ns << `1`);
859	buf[`1`] = chan & `0x7`;
860	buf[`2`] = addr;
861	buf[`3`] = addr >> `8`;
862	buf[`4`] = addr >> `16`;
863	buf[`5`] = addr >> `24`;
864
865	return SZ_DMAGO;
866	}
867
868	#define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t)
869
870	/ Returns Time-Out /
871	static bool _until_dmac_idle(struct pl330_thread *thrd)
872	{
873	void __iomem *regs = thrd->dmac->base;
874	unsigned long loops = msecs_to_loops(`5`);
875
876	do {
877	/ Until Manager is Idle /
878	if (!(readl(addr: regs + DBGSTATUS) & DBG_BUSY))
879	break;
880
881	cpu_relax();
882	} while (--loops);
883
884	if (!loops)
885	return true;
886
887	return false;
888	}
889
890	static inline void _execute_DBGINSN(struct pl330_thread *thrd,
891	u8 insn[], bool as_manager)
892	{
893	void __iomem *regs = thrd->dmac->base;
894	u32 val;
895
896	/ If timed out due to halted state-machine /
897	if (_until_dmac_idle(thrd)) {
898	dev_err(thrd->dmac->ddma.dev, "DMAC halted!\n");
899	return;
900	}
901
902	val = (insn[`0`] << `16`) \| (insn[`1`] << `24`);
903	if (!as_manager) {
904	val \|= (`1` << `0`);
905	val \|= (thrd->id << `8`); / Channel Number /
906	}
907	writel(val, addr: regs + DBGINST0);
908
909	val = le32_to_cpu(((__le32 )&insn[`2`]));
910	writel(val, addr: regs + DBGINST1);
911
912	/ Get going /
913	writel(val: `0`, addr: regs + DBGCMD);
914	}
915
916	static inline u32 _state(struct pl330_thread *thrd)
917	{
918	void __iomem *regs = thrd->dmac->base;
919	u32 val;
920
921	if (is_manager(thrd))
922	val = readl(addr: regs + DS) & `0xf`;
923	else
924	val = readl(addr: regs + CS(thrd->id)) & `0xf`;
925
926	switch (val) {
927	case DS_ST_STOP:
928	return PL330_STATE_STOPPED;
929	case DS_ST_EXEC:
930	return PL330_STATE_EXECUTING;
931	case DS_ST_CMISS:
932	return PL330_STATE_CACHEMISS;
933	case DS_ST_UPDTPC:
934	return PL330_STATE_UPDTPC;
935	case DS_ST_WFE:
936	return PL330_STATE_WFE;
937	case DS_ST_FAULT:
938	return PL330_STATE_FAULTING;
939	case DS_ST_ATBRR:
940	if (is_manager(thrd))
941	return PL330_STATE_INVALID;
942	else
943	return PL330_STATE_ATBARRIER;
944	case DS_ST_QBUSY:
945	if (is_manager(thrd))
946	return PL330_STATE_INVALID;
947	else
948	return PL330_STATE_QUEUEBUSY;
949	case DS_ST_WFP:
950	if (is_manager(thrd))
951	return PL330_STATE_INVALID;
952	else
953	return PL330_STATE_WFP;
954	case DS_ST_KILL:
955	if (is_manager(thrd))
956	return PL330_STATE_INVALID;
957	else
958	return PL330_STATE_KILLING;
959	case DS_ST_CMPLT:
960	if (is_manager(thrd))
961	return PL330_STATE_INVALID;
962	else
963	return PL330_STATE_COMPLETING;
964	case DS_ST_FLTCMP:
965	if (is_manager(thrd))
966	return PL330_STATE_INVALID;
967	else
968	return PL330_STATE_FAULT_COMPLETING;
969	default:
970	return PL330_STATE_INVALID;
971	}
972	}
973
974	static void _stop(struct pl330_thread *thrd)
975	{
976	void __iomem *regs = thrd->dmac->base;
977	u8 insn[`6`] = {`0`, `0`, `0`, `0`, `0`, `0`};
978	u32 inten = readl(addr: regs + INTEN);
979
980	if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)
981	UNTIL(thrd, PL330_STATE_FAULTING \| PL330_STATE_KILLING);
982
983	/ Return if nothing needs to be done /
984	if (_state(thrd) == PL330_STATE_COMPLETING
985	\|\| _state(thrd) == PL330_STATE_KILLING
986	\|\| _state(thrd) == PL330_STATE_STOPPED)
987	return;
988
989	_emit_KILL(dry_run: `0`, buf: insn);
990
991	_execute_DBGINSN(thrd, insn, as_manager: is_manager(thrd));
992
993	/ clear the event /
994	if (inten & (`1` << thrd->ev))
995	writel(val: `1` << thrd->ev, addr: regs + INTCLR);
996	/ Stop generating interrupts for SEV /
997	writel(val: inten & ~(`1` << thrd->ev), addr: regs + INTEN);
998	}
999
1000	/ Start doing req 'idx' of thread 'thrd' /
1001	static bool _trigger(struct pl330_thread *thrd)
1002	{
1003	void __iomem *regs = thrd->dmac->base;
1004	struct _pl330_req *req;
1005	struct dma_pl330_desc *desc;
1006	struct _arg_GO go;
1007	unsigned ns;
1008	u8 insn[`6`] = {`0`, `0`, `0`, `0`, `0`, `0`};
1009	int idx;
1010
1011	/ Return if already ACTIVE /
1012	if (_state(thrd) != PL330_STATE_STOPPED)
1013	return true;
1014
1015	idx = `1` - thrd->lstenq;
1016	if (thrd->req[idx].desc != NULL) {
1017	req = &thrd->req[idx];
1018	} else {
1019	idx = thrd->lstenq;
1020	if (thrd->req[idx].desc != NULL)
1021	req = &thrd->req[idx];
1022	else
1023	req = NULL;
1024	}
1025
1026	/ Return if no request /
1027	if (!req)
1028	return true;
1029
1030	/ Return if req is running /
1031	if (idx == thrd->req_running)
1032	return true;
1033
1034	desc = req->desc;
1035
1036	ns = desc->rqcfg.nonsecure ? `1` : `0`;
1037
1038	/ See 'Abort Sources' point-4 at Page 2-25 /
1039	if (_manager_ns(thrd) && !ns)
1040	dev_info(thrd->dmac->ddma.dev, "%s:%d Recipe for ABORT!\n",
1041	__func__, __LINE__);
1042
1043	go.chan = thrd->id;
1044	go.addr = req->mc_bus;
1045	go.ns = ns;
1046	_emit_GO(dry_run: `0`, buf: insn, arg: &go);
1047
1048	/ Set to generate interrupts for SEV /
1049	writel(readl(addr: regs + INTEN) \| (`1` << thrd->ev), addr: regs + INTEN);
1050
1051	/ Only manager can execute GO /
1052	_execute_DBGINSN(thrd, insn, as_manager: true);
1053
1054	thrd->req_running = idx;
1055
1056	return true;
1057	}
1058
1059	static bool pl330_start_thread(struct pl330_thread *thrd)
1060	{
1061	switch (_state(thrd)) {
1062	case PL330_STATE_FAULT_COMPLETING:
1063	UNTIL(thrd, PL330_STATE_FAULTING \| PL330_STATE_KILLING);
1064
1065	if (_state(thrd) == PL330_STATE_KILLING)
1066	UNTIL(thrd, PL330_STATE_STOPPED)
1067	fallthrough;
1068
1069	case PL330_STATE_FAULTING:
1070	_stop(thrd);
1071	fallthrough;
1072
1073	case PL330_STATE_KILLING:
1074	case PL330_STATE_COMPLETING:
1075	UNTIL(thrd, PL330_STATE_STOPPED)
1076	fallthrough;
1077
1078	case PL330_STATE_STOPPED:
1079	return _trigger(thrd);
1080
1081	case PL330_STATE_WFP:
1082	case PL330_STATE_QUEUEBUSY:
1083	case PL330_STATE_ATBARRIER:
1084	case PL330_STATE_UPDTPC:
1085	case PL330_STATE_CACHEMISS:
1086	case PL330_STATE_EXECUTING:
1087	return true;
1088
1089	case PL330_STATE_WFE: / For RESUME, nothing yet /
1090	default:
1091	return false;
1092	}
1093	}
1094
1095	static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
1096	const struct _xfer_spec pxs, int* cyc)
1097	{
1098	int off = `0`;
1099	struct pl330_config *pcfg = pxs->desc->rqcfg.pcfg;
1100
1101	/ check lock-up free version /
1102	if (get_revision(periph_id: pcfg->periph_id) >= PERIPH_REV_R1P0) {
1103	while (cyc--) {
1104	off += _emit_LD(dry_run, buf: &buf[off], cond: ALWAYS);
1105	off += _emit_ST(dry_run, buf: &buf[off], cond: ALWAYS);
1106	}
1107	} else {
1108	while (cyc--) {
1109	off += _emit_LD(dry_run, buf: &buf[off], cond: ALWAYS);
1110	off += _emit_RMB(dry_run, buf: &buf[off]);
1111	off += _emit_ST(dry_run, buf: &buf[off], cond: ALWAYS);
1112	off += _emit_WMB(dry_run, buf: &buf[off]);
1113	}
1114	}
1115
1116	return off;
1117	}
1118
1119	static u32 _emit_load(unsigned int dry_run, u8 buf[],
1120	enum pl330_cond cond, enum dma_transfer_direction direction,
1121	u8 peri)
1122	{
1123	int off = `0`;
1124
1125	switch (direction) {
1126	case DMA_MEM_TO_MEM:
1127	case DMA_MEM_TO_DEV:
1128	off += _emit_LD(dry_run, buf: &buf[off], cond);
1129	break;
1130
1131	case DMA_DEV_TO_MEM:
1132	if (cond == ALWAYS) {
1133	off += _emit_LDP(dry_run, buf: &buf[off], cond: SINGLE,
1134	peri);
1135	off += _emit_LDP(dry_run, buf: &buf[off], cond: BURST,
1136	peri);
1137	} else {
1138	off += _emit_LDP(dry_run, buf: &buf[off], cond,
1139	peri);
1140	}
1141	break;
1142
1143	default:
1144	/ this code should be unreachable /
1145	WARN_ON(`1`);
1146	break;
1147	}
1148
1149	return off;
1150	}
1151
1152	static inline u32 _emit_store(unsigned int dry_run, u8 buf[],
1153	enum pl330_cond cond, enum dma_transfer_direction direction,
1154	u8 peri)
1155	{
1156	int off = `0`;
1157
1158	switch (direction) {
1159	case DMA_MEM_TO_MEM:
1160	case DMA_DEV_TO_MEM:
1161	off += _emit_ST(dry_run, buf: &buf[off], cond);
1162	break;
1163
1164	case DMA_MEM_TO_DEV:
1165	if (cond == ALWAYS) {
1166	off += _emit_STP(dry_run, buf: &buf[off], cond: SINGLE,
1167	peri);
1168	off += _emit_STP(dry_run, buf: &buf[off], cond: BURST,
1169	peri);
1170	} else {
1171	off += _emit_STP(dry_run, buf: &buf[off], cond,
1172	peri);
1173	}
1174	break;
1175
1176	default:
1177	/ this code should be unreachable /
1178	WARN_ON(`1`);
1179	break;
1180	}
1181
1182	return off;
1183	}
1184
1185	static inline int _ldst_peripheral(struct pl330_dmac *pl330,
1186	unsigned dry_run, u8 buf[],
1187	const struct _xfer_spec pxs, int* cyc,
1188	enum pl330_cond cond)
1189	{
1190	int off = `0`;
1191
1192	/*
1193	* do FLUSHP at beginning to clear any stale dma requests before the
1194	* first WFP.
1195	*/
1196	if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
1197	off += _emit_FLUSHP(dry_run, buf: &buf[off], peri: pxs->desc->peri);
1198	while (cyc--) {
1199	off += _emit_WFP(dry_run, buf: &buf[off], cond, peri: pxs->desc->peri);
1200	off += _emit_load(dry_run, buf: &buf[off], cond, direction: pxs->desc->rqtype,
1201	peri: pxs->desc->peri);
1202	off += _emit_store(dry_run, buf: &buf[off], cond, direction: pxs->desc->rqtype,
1203	peri: pxs->desc->peri);
1204	}
1205
1206	return off;
1207	}
1208
1209	static int _bursts(struct pl330_dmac pl330, unsigned* dry_run, u8 buf[],
1210	const struct _xfer_spec pxs, int* cyc)
1211	{
1212	int off = `0`;
1213	enum pl330_cond cond = BRST_LEN(pxs->ccr) > `1` ? BURST : SINGLE;
1214
1215	if (pl330->quirks & PL330_QUIRK_PERIPH_BURST)
1216	cond = BURST;
1217
1218	switch (pxs->desc->rqtype) {
1219	case DMA_MEM_TO_DEV:
1220	case DMA_DEV_TO_MEM:
1221	off += _ldst_peripheral(pl330, dry_run, buf: &buf[off], pxs, cyc,
1222	cond);
1223	break;
1224
1225	case DMA_MEM_TO_MEM:
1226	off += _ldst_memtomem(dry_run, buf: &buf[off], pxs, cyc);
1227	break;
1228
1229	default:
1230	/ this code should be unreachable /
1231	WARN_ON(`1`);
1232	break;
1233	}
1234
1235	return off;
1236	}
1237
1238	/*
1239	* only the unaligned burst transfers have the dregs.
1240	* so, still transfer dregs with a reduced size burst
1241	* for mem-to-mem, mem-to-dev or dev-to-mem.
1242	*/
1243	static int _dregs(struct pl330_dmac pl330, unsigned* int dry_run, u8 buf[],
1244	const struct _xfer_spec pxs, int* transfer_length)
1245	{
1246	int off = `0`;
1247	int dregs_ccr;
1248
1249	if (transfer_length == `0`)
1250	return off;
1251
1252	/*
1253	* dregs_len = (total bytes - BURST_TO_BYTE(bursts, ccr)) /
1254	* BRST_SIZE(ccr)
1255	* the dregs len must be smaller than burst len,
1256	* so, for higher efficiency, we can modify CCR
1257	* to use a reduced size burst len for the dregs.
1258	*/
1259	dregs_ccr = pxs->ccr;
1260	dregs_ccr &= ~((`0xf` << CC_SRCBRSTLEN_SHFT) \|
1261	(`0xf` << CC_DSTBRSTLEN_SHFT));
1262	dregs_ccr \|= (((transfer_length - `1`) & `0xf`) <<
1263	CC_SRCBRSTLEN_SHFT);
1264	dregs_ccr \|= (((transfer_length - `1`) & `0xf`) <<
1265	CC_DSTBRSTLEN_SHFT);
1266
1267	switch (pxs->desc->rqtype) {
1268	case DMA_MEM_TO_DEV:
1269	case DMA_DEV_TO_MEM:
1270	off += _emit_MOV(dry_run, buf: &buf[off], dst: CCR, val: dregs_ccr);
1271	off += _ldst_peripheral(pl330, dry_run, buf: &buf[off], pxs, cyc: `1`,
1272	cond: BURST);
1273	break;
1274
1275	case DMA_MEM_TO_MEM:
1276	off += _emit_MOV(dry_run, buf: &buf[off], dst: CCR, val: dregs_ccr);
1277	off += _ldst_memtomem(dry_run, buf: &buf[off], pxs, cyc: `1`);
1278	break;
1279
1280	default:
1281	/ this code should be unreachable /
1282	WARN_ON(`1`);
1283	break;
1284	}
1285
1286	return off;
1287	}
1288
1289	/ Returns bytes consumed and updates bursts /
1290	static inline int _loop(struct pl330_dmac pl330, unsigned* dry_run, u8 buf[],
1291	unsigned long bursts, const* struct _xfer_spec *pxs)
1292	{
1293	int cyc, cycmax, szlp, szlpend, szbrst, off;
1294	unsigned lcnt0, lcnt1, ljmp0, ljmp1;
1295	struct _arg_LPEND lpend;
1296
1297	if (*bursts == `1`)
1298	return _bursts(pl330, dry_run, buf, pxs, cyc: `1`);
1299
1300	/ Max iterations possible in DMALP is 256 /
1301	if (bursts >= `256``256`) {
1302	lcnt1 = `256`;
1303	lcnt0 = `256`;
1304	cyc = *bursts / lcnt1 / lcnt0;
1305	} else if (*bursts > `256`) {
1306	lcnt1 = `256`;
1307	lcnt0 = *bursts / lcnt1;
1308	cyc = `1`;
1309	} else {
1310	lcnt1 = *bursts;
1311	lcnt0 = `0`;
1312	cyc = `1`;
1313	}
1314
1315	szlp = _emit_LP(dry_run: `1`, buf, loop: `0`, cnt: `0`);
1316	szbrst = _bursts(pl330, dry_run: `1`, buf, pxs, cyc: `1`);
1317
1318	lpend.cond = ALWAYS;
1319	lpend.forever = false;
1320	lpend.loop = `0`;
1321	lpend.bjump = `0`;
1322	szlpend = _emit_LPEND(dry_run: `1`, buf, arg: &lpend);
1323
1324	if (lcnt0) {
1325	szlp *= `2`;
1326	szlpend *= `2`;
1327	}
1328
1329	/*
1330	* Max bursts that we can unroll due to limit on the
1331	* size of backward jump that can be encoded in DMALPEND
1332	* which is 8-bits and hence 255
1333	*/
1334	cycmax = (`255` - (szlp + szlpend)) / szbrst;
1335
1336	cyc = (cycmax < cyc) ? cycmax : cyc;
1337
1338	off = `0`;
1339
1340	if (lcnt0) {
1341	off += _emit_LP(dry_run, buf: &buf[off], loop: `0`, cnt: lcnt0);
1342	ljmp0 = off;
1343	}
1344
1345	off += _emit_LP(dry_run, buf: &buf[off], loop: `1`, cnt: lcnt1);
1346	ljmp1 = off;
1347
1348	off += _bursts(pl330, dry_run, buf: &buf[off], pxs, cyc);
1349
1350	lpend.cond = ALWAYS;
1351	lpend.forever = false;
1352	lpend.loop = `1`;
1353	lpend.bjump = off - ljmp1;
1354	off += _emit_LPEND(dry_run, buf: &buf[off], arg: &lpend);
1355
1356	if (lcnt0) {
1357	lpend.cond = ALWAYS;
1358	lpend.forever = false;
1359	lpend.loop = `0`;
1360	lpend.bjump = off - ljmp0;
1361	off += _emit_LPEND(dry_run, buf: &buf[off], arg: &lpend);
1362	}
1363
1364	bursts = lcnt1 cyc;
1365	if (lcnt0)
1366	bursts = lcnt0;
1367
1368	return off;
1369	}
1370
1371	static inline int _setup_loops(struct pl330_dmac *pl330,
1372	unsigned dry_run, u8 buf[],
1373	const struct _xfer_spec *pxs)
1374	{
1375	struct pl330_xfer *x = &pxs->desc->px;
1376	u32 ccr = pxs->ccr;
1377	unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
1378	int num_dregs = (x->bytes - BURST_TO_BYTE(bursts, ccr)) /
1379	BRST_SIZE(ccr);
1380	int off = `0`;
1381
1382	while (bursts) {
1383	c = bursts;
1384	off += _loop(pl330, dry_run, buf: &buf[off], bursts: &c, pxs);
1385	bursts -= c;
1386	}
1387	off += _dregs(pl330, dry_run, buf: &buf[off], pxs, transfer_length: num_dregs);
1388
1389	return off;
1390	}
1391
1392	static inline int _setup_xfer(struct pl330_dmac *pl330,
1393	unsigned dry_run, u8 buf[],
1394	const struct _xfer_spec *pxs)
1395	{
1396	struct pl330_xfer *x = &pxs->desc->px;
1397	int off = `0`;
1398
1399	/ DMAMOV SAR, x->src_addr /
1400	off += _emit_MOV(dry_run, buf: &buf[off], dst: SAR, val: x->src_addr);
1401	/ DMAMOV DAR, x->dst_addr /
1402	off += _emit_MOV(dry_run, buf: &buf[off], dst: DAR, val: x->dst_addr);
1403
1404	/ Setup Loop(s) /
1405	off += _setup_loops(pl330, dry_run, buf: &buf[off], pxs);
1406
1407	return off;
1408	}
1409
1410	/*
1411	* A req is a sequence of one or more xfer units.
1412	* Returns the number of bytes taken to setup the MC for the req.
1413	*/
1414	static int _setup_req(struct pl330_dmac pl330, unsigned* dry_run,
1415	struct pl330_thread thrd, unsigned* index,
1416	struct _xfer_spec *pxs)
1417	{
1418	struct _pl330_req *req = &thrd->req[index];
1419	u8 *buf = req->mc_cpu;
1420	int off = `0`;
1421
1422	PL330_DBGMC_START(req->mc_bus);
1423
1424	/ DMAMOV CCR, ccr /
1425	off += _emit_MOV(dry_run, buf: &buf[off], dst: CCR, val: pxs->ccr);
1426
1427	off += _setup_xfer(pl330, dry_run, buf: &buf[off], pxs);
1428
1429	/ DMASEV peripheral/event /
1430	off += _emit_SEV(dry_run, buf: &buf[off], ev: thrd->ev);
1431	/ DMAEND /
1432	off += _emit_END(dry_run, buf: &buf[off]);
1433
1434	return off;
1435	}
1436
1437	static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc)
1438	{
1439	u32 ccr = `0`;
1440
1441	if (rqc->src_inc)
1442	ccr \|= CC_SRCINC;
1443
1444	if (rqc->dst_inc)
1445	ccr \|= CC_DSTINC;
1446
1447	/ We set same protection levels for Src and DST for now /
1448	if (rqc->privileged)
1449	ccr \|= CC_SRCPRI \| CC_DSTPRI;
1450	if (rqc->nonsecure)
1451	ccr \|= CC_SRCNS \| CC_DSTNS;
1452	if (rqc->insnaccess)
1453	ccr \|= CC_SRCIA \| CC_DSTIA;
1454
1455	ccr \|= (((rqc->brst_len - `1`) & `0xf`) << CC_SRCBRSTLEN_SHFT);
1456	ccr \|= (((rqc->brst_len - `1`) & `0xf`) << CC_DSTBRSTLEN_SHFT);
1457
1458	ccr \|= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT);
1459	ccr \|= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT);
1460
1461	ccr \|= (rqc->scctl << CC_SRCCCTRL_SHFT);
1462	ccr \|= (rqc->dcctl << CC_DSTCCTRL_SHFT);
1463
1464	ccr \|= (rqc->swap << CC_SWAP_SHFT);
1465
1466	return ccr;
1467	}
1468
1469	/*
1470	* Submit a list of xfers after which the client wants notification.
1471	* Client is not notified after each xfer unit, just once after all
1472	* xfer units are done or some error occurs.
1473	*/
1474	static int pl330_submit_req(struct pl330_thread *thrd,
1475	struct dma_pl330_desc *desc)
1476	{
1477	struct pl330_dmac *pl330 = thrd->dmac;
1478	struct _xfer_spec xs;
1479	unsigned long flags;
1480	unsigned idx;
1481	u32 ccr;
1482	int ret = `0`;
1483
1484	switch (desc->rqtype) {
1485	case DMA_MEM_TO_DEV:
1486	break;
1487
1488	case DMA_DEV_TO_MEM:
1489	break;
1490
1491	case DMA_MEM_TO_MEM:
1492	break;
1493
1494	default:
1495	return -ENOTSUPP;
1496	}
1497
1498	if (pl330->state == DYING
1499	\|\| pl330->dmac_tbd.reset_chan & (`1` << thrd->id)) {
1500	dev_info(thrd->dmac->ddma.dev, "%s:%d\n",
1501	__func__, __LINE__);
1502	return -EAGAIN;
1503	}
1504
1505	/ If request for non-existing peripheral /
1506	if (desc->rqtype != DMA_MEM_TO_MEM &&
1507	desc->peri >= pl330->pcfg.num_peri) {
1508	dev_info(thrd->dmac->ddma.dev,
1509	"%s:%d Invalid peripheral(%u)!\n",
1510	__func__, __LINE__, desc->peri);
1511	return -EINVAL;
1512	}
1513
1514	spin_lock_irqsave(&pl330->lock, flags);
1515
1516	if (_queue_full(thrd)) {
1517	ret = -EAGAIN;
1518	goto xfer_exit;
1519	}
1520
1521	/ Prefer Secure Channel /
1522	if (!_manager_ns(thrd))
1523	desc->rqcfg.nonsecure = `0`;
1524	else
1525	desc->rqcfg.nonsecure = `1`;
1526
1527	ccr = _prepare_ccr(rqc: &desc->rqcfg);
1528
1529	idx = thrd->req[`0`].desc == NULL ? `0` : `1`;
1530
1531	xs.ccr = ccr;
1532	xs.desc = desc;
1533
1534	/ First dry run to check if req is acceptable /
1535	ret = _setup_req(pl330, dry_run: `1`, thrd, index: idx, pxs: &xs);
1536
1537	if (ret > pl330->mcbufsz / `2`) {
1538	dev_info(pl330->ddma.dev, "%s:%d Try increasing mcbufsz (%i/%i)\n",
1539	__func__, __LINE__, ret, pl330->mcbufsz / `2`);
1540	ret = -ENOMEM;
1541	goto xfer_exit;
1542	}
1543
1544	/ Hook the request /
1545	thrd->lstenq = idx;
1546	thrd->req[idx].desc = desc;
1547	_setup_req(pl330, dry_run: `0`, thrd, index: idx, pxs: &xs);
1548
1549	ret = `0`;
1550
1551	xfer_exit:
1552	spin_unlock_irqrestore(lock: &pl330->lock, flags);
1553
1554	return ret;
1555	}
1556
1557	static void dma_pl330_rqcb(struct dma_pl330_desc desc, enum* pl330_op_err err)
1558	{
1559	struct dma_pl330_chan *pch;
1560	unsigned long flags;
1561
1562	if (!desc)
1563	return;
1564
1565	pch = desc->pchan;
1566
1567	/ If desc aborted /
1568	if (!pch)
1569	return;
1570
1571	spin_lock_irqsave(&pch->lock, flags);
1572
1573	desc->status = DONE;
1574
1575	spin_unlock_irqrestore(lock: &pch->lock, flags);
1576
1577	tasklet_schedule(t: &pch->task);
1578	}
1579
1580	static void pl330_dotask(struct tasklet_struct *t)
1581	{
1582	struct pl330_dmac *pl330 = from_tasklet(pl330, t, tasks);
1583	unsigned long flags;
1584	int i;
1585
1586	spin_lock_irqsave(&pl330->lock, flags);
1587
1588	/ The DMAC itself gone nuts /
1589	if (pl330->dmac_tbd.reset_dmac) {
1590	pl330->state = DYING;
1591	/ Reset the manager too /
1592	pl330->dmac_tbd.reset_mngr = true;
1593	/ Clear the reset flag /
1594	pl330->dmac_tbd.reset_dmac = false;
1595	}
1596
1597	if (pl330->dmac_tbd.reset_mngr) {
1598	_stop(thrd: pl330->manager);
1599	/ Reset all channels /
1600	pl330->dmac_tbd.reset_chan = (`1` << pl330->pcfg.num_chan) - `1`;
1601	/ Clear the reset flag /
1602	pl330->dmac_tbd.reset_mngr = false;
1603	}
1604
1605	for (i = `0`; i < pl330->pcfg.num_chan; i++) {
1606
1607	if (pl330->dmac_tbd.reset_chan & (`1` << i)) {
1608	struct pl330_thread *thrd = &pl330->channels[i];
1609	void __iomem *regs = pl330->base;
1610	enum pl330_op_err err;
1611
1612	_stop(thrd);
1613
1614	if (readl(addr: regs + FSC) & (`1` << thrd->id))
1615	err = PL330_ERR_FAIL;
1616	else
1617	err = PL330_ERR_ABORT;
1618
1619	spin_unlock_irqrestore(lock: &pl330->lock, flags);
1620	dma_pl330_rqcb(desc: thrd->req[`1` - thrd->lstenq].desc, err);
1621	dma_pl330_rqcb(desc: thrd->req[thrd->lstenq].desc, err);
1622	spin_lock_irqsave(&pl330->lock, flags);
1623
1624	thrd->req[`0`].desc = NULL;
1625	thrd->req[`1`].desc = NULL;
1626	thrd->req_running = -`1`;
1627
1628	/ Clear the reset flag /
1629	pl330->dmac_tbd.reset_chan &= ~(`1` << i);
1630	}
1631	}
1632
1633	spin_unlock_irqrestore(lock: &pl330->lock, flags);
1634
1635	return;
1636	}
1637
1638	/ Returns 1 if state was updated, 0 otherwise /
1639	static int pl330_update(struct pl330_dmac *pl330)
1640	{
1641	struct dma_pl330_desc *descdone;
1642	unsigned long flags;
1643	void __iomem *regs;
1644	u32 val;
1645	int id, ev, ret = `0`;
1646
1647	regs = pl330->base;
1648
1649	spin_lock_irqsave(&pl330->lock, flags);
1650
1651	val = readl(addr: regs + FSM) & `0x1`;
1652	if (val)
1653	pl330->dmac_tbd.reset_mngr = true;
1654	else
1655	pl330->dmac_tbd.reset_mngr = false;
1656
1657	val = readl(addr: regs + FSC) & ((`1` << pl330->pcfg.num_chan) - `1`);
1658	pl330->dmac_tbd.reset_chan \|= val;
1659	if (val) {
1660	int i = `0`;
1661	while (i < pl330->pcfg.num_chan) {
1662	if (val & (`1` << i)) {
1663	dev_info(pl330->ddma.dev,
1664	"Reset Channel-%d\t CS-%x FTC-%x\n",
1665	i, readl(regs + CS(i)),
1666	readl(regs + FTC(i)));
1667	_stop(thrd: &pl330->channels[i]);
1668	}
1669	i++;
1670	}
1671	}
1672
1673	/ Check which event happened i.e, thread notified /
1674	val = readl(addr: regs + ES);
1675	if (pl330->pcfg.num_events < `32`
1676	&& val & ~((`1` << pl330->pcfg.num_events) - `1`)) {
1677	pl330->dmac_tbd.reset_dmac = true;
1678	dev_err(pl330->ddma.dev, "%s:%d Unexpected!\n", __func__,
1679	__LINE__);
1680	ret = `1`;
1681	goto updt_exit;
1682	}
1683
1684	for (ev = `0`; ev < pl330->pcfg.num_events; ev++) {
1685	if (val & (`1` << ev)) { / Event occurred /
1686	struct pl330_thread *thrd;
1687	u32 inten = readl(addr: regs + INTEN);
1688	int active;
1689
1690	/ Clear the event /
1691	if (inten & (`1` << ev))
1692	writel(val: `1` << ev, addr: regs + INTCLR);
1693
1694	ret = `1`;
1695
1696	id = pl330->events[ev];
1697
1698	thrd = &pl330->channels[id];
1699
1700	active = thrd->req_running;
1701	if (active == -`1`) / Aborted /
1702	continue;
1703
1704	/ Detach the req /
1705	descdone = thrd->req[active].desc;
1706	thrd->req[active].desc = NULL;
1707
1708	thrd->req_running = -`1`;
1709
1710	/ Get going again ASAP /
1711	pl330_start_thread(thrd);
1712
1713	/ For now, just make a list of callbacks to be done /
1714	list_add_tail(new: &descdone->rqd, head: &pl330->req_done);
1715	}
1716	}
1717
1718	/ Now that we are in no hurry, do the callbacks /
1719	while (!list_empty(head: &pl330->req_done)) {
1720	descdone = list_first_entry(&pl330->req_done,
1721	struct dma_pl330_desc, rqd);
1722	list_del(entry: &descdone->rqd);
1723	spin_unlock_irqrestore(lock: &pl330->lock, flags);
1724	dma_pl330_rqcb(desc: descdone, err: PL330_ERR_NONE);
1725	spin_lock_irqsave(&pl330->lock, flags);
1726	}
1727
1728	updt_exit:
1729	spin_unlock_irqrestore(lock: &pl330->lock, flags);
1730
1731	if (pl330->dmac_tbd.reset_dmac
1732	\|\| pl330->dmac_tbd.reset_mngr
1733	\|\| pl330->dmac_tbd.reset_chan) {
1734	ret = `1`;
1735	tasklet_schedule(t: &pl330->tasks);
1736	}
1737
1738	return ret;
1739	}
1740
1741	/ Reserve an event /
1742	static inline int _alloc_event(struct pl330_thread *thrd)
1743	{
1744	struct pl330_dmac *pl330 = thrd->dmac;
1745	int ev;
1746
1747	for (ev = `0`; ev < pl330->pcfg.num_events; ev++)
1748	if (pl330->events[ev] == -`1`) {
1749	pl330->events[ev] = thrd->id;
1750	return ev;
1751	}
1752
1753	return -`1`;
1754	}
1755
1756	static bool _chan_ns(const struct pl330_dmac pl330, int* i)
1757	{
1758	return pl330->pcfg.irq_ns & (`1` << i);
1759	}
1760
1761	/ Upon success, returns IdentityToken for the*
1762	* allocated channel, NULL otherwise.
1763	*/
1764	static struct pl330_thread pl330_request_channel(struct* pl330_dmac *pl330)
1765	{
1766	struct pl330_thread *thrd = NULL;
1767	int chans, i;
1768
1769	if (pl330->state == DYING)
1770	return NULL;
1771
1772	chans = pl330->pcfg.num_chan;
1773
1774	for (i = `0`; i < chans; i++) {
1775	thrd = &pl330->channels[i];
1776	if ((thrd->free) && (!_manager_ns(thrd) \|\|
1777	_chan_ns(pl330, i))) {
1778	thrd->ev = _alloc_event(thrd);
1779	if (thrd->ev >= `0`) {
1780	thrd->free = false;
1781	thrd->lstenq = `1`;
1782	thrd->req[`0`].desc = NULL;
1783	thrd->req[`1`].desc = NULL;
1784	thrd->req_running = -`1`;
1785	break;
1786	}
1787	}
1788	thrd = NULL;
1789	}
1790
1791	return thrd;
1792	}
1793
1794	/ Release an event /
1795	static inline void _free_event(struct pl330_thread thrd, int* ev)
1796	{
1797	struct pl330_dmac *pl330 = thrd->dmac;
1798
1799	/ If the event is valid and was held by the thread /
1800	if (ev >= `0` && ev < pl330->pcfg.num_events
1801	&& pl330->events[ev] == thrd->id)
1802	pl330->events[ev] = -`1`;
1803	}
1804
1805	static void pl330_release_channel(struct pl330_thread *thrd)
1806	{
1807	if (!thrd \|\| thrd->free)
1808	return;
1809
1810	_stop(thrd);
1811
1812	dma_pl330_rqcb(desc: thrd->req[`1` - thrd->lstenq].desc, err: PL330_ERR_ABORT);
1813	dma_pl330_rqcb(desc: thrd->req[thrd->lstenq].desc, err: PL330_ERR_ABORT);
1814
1815	_free_event(thrd, ev: thrd->ev);
1816	thrd->free = true;
1817	}
1818
1819	/ Initialize the structure for PL330 configuration, that can be used*
1820	* by the client driver the make best use of the DMAC
1821	*/
1822	static void read_dmac_config(struct pl330_dmac *pl330)
1823	{
1824	void __iomem *regs = pl330->base;
1825	u32 val;
1826
1827	val = readl(addr: regs + CRD) >> CRD_DATA_WIDTH_SHIFT;
1828	val &= CRD_DATA_WIDTH_MASK;
1829	pl330->pcfg.data_bus_width = `8` * (`1` << val);
1830
1831	val = readl(addr: regs + CRD) >> CRD_DATA_BUFF_SHIFT;
1832	val &= CRD_DATA_BUFF_MASK;
1833	pl330->pcfg.data_buf_dep = val + `1`;
1834
1835	val = readl(addr: regs + CR0) >> CR0_NUM_CHANS_SHIFT;
1836	val &= CR0_NUM_CHANS_MASK;
1837	val += `1`;
1838	pl330->pcfg.num_chan = val;
1839
1840	val = readl(addr: regs + CR0);
1841	if (val & CR0_PERIPH_REQ_SET) {
1842	val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK;
1843	val += `1`;
1844	pl330->pcfg.num_peri = val;
1845	pl330->pcfg.peri_ns = readl(addr: regs + CR4);
1846	} else {
1847	pl330->pcfg.num_peri = `0`;
1848	}
1849
1850	val = readl(addr: regs + CR0);
1851	if (val & CR0_BOOT_MAN_NS)
1852	pl330->pcfg.mode \|= DMAC_MODE_NS;
1853	else
1854	pl330->pcfg.mode &= ~DMAC_MODE_NS;
1855
1856	val = readl(addr: regs + CR0) >> CR0_NUM_EVENTS_SHIFT;
1857	val &= CR0_NUM_EVENTS_MASK;
1858	val += `1`;
1859	pl330->pcfg.num_events = val;
1860
1861	pl330->pcfg.irq_ns = readl(addr: regs + CR3);
1862	}
1863
1864	static inline void _reset_thread(struct pl330_thread *thrd)
1865	{
1866	struct pl330_dmac *pl330 = thrd->dmac;
1867
1868	thrd->req[`0`].mc_cpu = pl330->mcode_cpu
1869	+ (thrd->id * pl330->mcbufsz);
1870	thrd->req[`0`].mc_bus = pl330->mcode_bus
1871	+ (thrd->id * pl330->mcbufsz);
1872	thrd->req[`0`].desc = NULL;
1873
1874	thrd->req[`1`].mc_cpu = thrd->req[`0`].mc_cpu
1875	+ pl330->mcbufsz / `2`;
1876	thrd->req[`1`].mc_bus = thrd->req[`0`].mc_bus
1877	+ pl330->mcbufsz / `2`;
1878	thrd->req[`1`].desc = NULL;
1879
1880	thrd->req_running = -`1`;
1881	}
1882
1883	static int dmac_alloc_threads(struct pl330_dmac *pl330)
1884	{
1885	int chans = pl330->pcfg.num_chan;
1886	struct pl330_thread *thrd;
1887	int i;
1888
1889	/ Allocate 1 Manager and 'chans' Channel threads /
1890	pl330->channels = kcalloc(n: `1` + chans, size: sizeof(*thrd),
1891	GFP_KERNEL);
1892	if (!pl330->channels)
1893	return -ENOMEM;
1894
1895	/ Init Channel threads /
1896	for (i = `0`; i < chans; i++) {
1897	thrd = &pl330->channels[i];
1898	thrd->id = i;
1899	thrd->dmac = pl330;
1900	_reset_thread(thrd);
1901	thrd->free = true;
1902	}
1903
1904	/ MANAGER is indexed at the end /
1905	thrd = &pl330->channels[chans];
1906	thrd->id = chans;
1907	thrd->dmac = pl330;
1908	thrd->free = false;
1909	pl330->manager = thrd;
1910
1911	return `0`;
1912	}
1913
1914	static int dmac_alloc_resources(struct pl330_dmac *pl330)
1915	{
1916	int chans = pl330->pcfg.num_chan;
1917	int ret;
1918
1919	/*
1920	* Alloc MicroCode buffer for 'chans' Channel threads.
1921	* A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN)
1922	*/
1923	pl330->mcode_cpu = dma_alloc_attrs(dev: pl330->ddma.dev,
1924	size: chans * pl330->mcbufsz,
1925	dma_handle: &pl330->mcode_bus, GFP_KERNEL,
1926	DMA_ATTR_PRIVILEGED);
1927	if (!pl330->mcode_cpu) {
1928	dev_err(pl330->ddma.dev, "%s:%d Can't allocate memory!\n",
1929	__func__, __LINE__);
1930	return -ENOMEM;
1931	}
1932
1933	ret = dmac_alloc_threads(pl330);
1934	if (ret) {
1935	dev_err(pl330->ddma.dev, "%s:%d Can't to create channels for DMAC!\n",
1936	__func__, __LINE__);
1937	dma_free_attrs(dev: pl330->ddma.dev,
1938	size: chans * pl330->mcbufsz,
1939	cpu_addr: pl330->mcode_cpu, dma_handle: pl330->mcode_bus,
1940	DMA_ATTR_PRIVILEGED);
1941	return ret;
1942	}
1943
1944	return `0`;
1945	}
1946
1947	static int pl330_add(struct pl330_dmac *pl330)
1948	{
1949	int i, ret;
1950
1951	/ Check if we can handle this DMAC /
1952	if ((pl330->pcfg.periph_id & `0xfffff`) != PERIPH_ID_VAL) {
1953	dev_err(pl330->ddma.dev, "PERIPH_ID 0x%x !\n",
1954	pl330->pcfg.periph_id);
1955	return -EINVAL;
1956	}
1957
1958	/ Read the configuration of the DMAC /
1959	read_dmac_config(pl330);
1960
1961	if (pl330->pcfg.num_events == `0`) {
1962	dev_err(pl330->ddma.dev, "%s:%d Can't work without events!\n",
1963	__func__, __LINE__);
1964	return -EINVAL;
1965	}
1966
1967	spin_lock_init(&pl330->lock);
1968
1969	INIT_LIST_HEAD(list: &pl330->req_done);
1970
1971	/ Use default MC buffer size if not provided /
1972	if (!pl330->mcbufsz)
1973	pl330->mcbufsz = MCODE_BUFF_PER_REQ * `2`;
1974
1975	/ Mark all events as free /
1976	for (i = `0`; i < pl330->pcfg.num_events; i++)
1977	pl330->events[i] = -`1`;
1978
1979	/ Allocate resources needed by the DMAC /
1980	ret = dmac_alloc_resources(pl330);
1981	if (ret) {
1982	dev_err(pl330->ddma.dev, "Unable to create channels for DMAC\n");
1983	return ret;
1984	}
1985
1986	tasklet_setup(t: &pl330->tasks, callback: pl330_dotask);
1987
1988	pl330->state = INIT;
1989
1990	return `0`;
1991	}
1992
1993	static int dmac_free_threads(struct pl330_dmac *pl330)
1994	{
1995	struct pl330_thread *thrd;
1996	int i;
1997
1998	/ Release Channel threads /
1999	for (i = `0`; i < pl330->pcfg.num_chan; i++) {
2000	thrd = &pl330->channels[i];
2001	pl330_release_channel(thrd);
2002	}
2003
2004	/ Free memory /
2005	kfree(objp: pl330->channels);
2006
2007	return `0`;
2008	}
2009
2010	static void pl330_del(struct pl330_dmac *pl330)
2011	{
2012	pl330->state = UNINIT;
2013
2014	tasklet_kill(t: &pl330->tasks);
2015
2016	/ Free DMAC resources /
2017	dmac_free_threads(pl330);
2018
2019	dma_free_attrs(dev: pl330->ddma.dev,
2020	size: pl330->pcfg.num_chan * pl330->mcbufsz, cpu_addr: pl330->mcode_cpu,
2021	dma_handle: pl330->mcode_bus, DMA_ATTR_PRIVILEGED);
2022	}
2023
2024	/ forward declaration /
2025	static struct amba_driver pl330_driver;
2026
2027	static inline struct dma_pl330_chan *
2028	to_pchan(struct dma_chan *ch)
2029	{
2030	if (!ch)
2031	return NULL;
2032
2033	return container_of(ch, struct dma_pl330_chan, chan);
2034	}
2035
2036	static inline struct dma_pl330_desc *
2037	to_desc(struct dma_async_tx_descriptor *tx)
2038	{
2039	return container_of(tx, struct dma_pl330_desc, txd);
2040	}
2041
2042	static inline void fill_queue(struct dma_pl330_chan *pch)
2043	{
2044	struct dma_pl330_desc *desc;
2045	int ret;
2046
2047	list_for_each_entry(desc, &pch->work_list, node) {
2048
2049	/ If already submitted /
2050	if (desc->status == BUSY \|\| desc->status == PAUSED)
2051	continue;
2052
2053	ret = pl330_submit_req(thrd: pch->thread, desc);
2054	if (!ret) {
2055	desc->status = BUSY;
2056	} else if (ret == -EAGAIN) {
2057	/ QFull or DMAC Dying /
2058	break;
2059	} else {
2060	/ Unacceptable request /
2061	desc->status = DONE;
2062	dev_err(pch->dmac->ddma.dev, "%s:%d Bad Desc(%d)\n",
2063	__func__, __LINE__, desc->txd.cookie);
2064	tasklet_schedule(t: &pch->task);
2065	}
2066	}
2067	}
2068
2069	static void pl330_tasklet(struct tasklet_struct *t)
2070	{
2071	struct dma_pl330_chan *pch = from_tasklet(pch, t, task);
2072	struct dma_pl330_desc desc, _dt;
2073	unsigned long flags;
2074	bool power_down = false;
2075
2076	spin_lock_irqsave(&pch->lock, flags);
2077
2078	/ Pick up ripe tomatoes /
2079	list_for_each_entry_safe(desc, _dt, &pch->work_list, node)
2080	if (desc->status == DONE) {
2081	if (!pch->cyclic)
2082	dma_cookie_complete(tx: &desc->txd);
2083	list_move_tail(list: &desc->node, head: &pch->completed_list);
2084	}
2085
2086	/ Try to submit a req imm. next to the last completed cookie /
2087	fill_queue(pch);
2088
2089	if (list_empty(head: &pch->work_list)) {
2090	spin_lock(lock: &pch->thread->dmac->lock);
2091	_stop(thrd: pch->thread);
2092	spin_unlock(lock: &pch->thread->dmac->lock);
2093	power_down = true;
2094	pch->active = false;
2095	} else {
2096	/ Make sure the PL330 Channel thread is active /
2097	spin_lock(lock: &pch->thread->dmac->lock);
2098	pl330_start_thread(thrd: pch->thread);
2099	spin_unlock(lock: &pch->thread->dmac->lock);
2100	}
2101
2102	while (!list_empty(head: &pch->completed_list)) {
2103	struct dmaengine_desc_callback cb;
2104
2105	desc = list_first_entry(&pch->completed_list,
2106	struct dma_pl330_desc, node);
2107
2108	dmaengine_desc_get_callback(tx: &desc->txd, cb: &cb);
2109
2110	if (pch->cyclic) {
2111	desc->status = PREP;
2112	list_move_tail(list: &desc->node, head: &pch->work_list);
2113	if (power_down) {
2114	pch->active = true;
2115	spin_lock(lock: &pch->thread->dmac->lock);
2116	pl330_start_thread(thrd: pch->thread);
2117	spin_unlock(lock: &pch->thread->dmac->lock);
2118	power_down = false;
2119	}
2120	} else {
2121	desc->status = FREE;
2122	list_move_tail(list: &desc->node, head: &pch->dmac->desc_pool);
2123	}
2124
2125	dma_descriptor_unmap(tx: &desc->txd);
2126
2127	if (dmaengine_desc_callback_valid(cb: &cb)) {
2128	spin_unlock_irqrestore(lock: &pch->lock, flags);
2129	dmaengine_desc_callback_invoke(cb: &cb, NULL);
2130	spin_lock_irqsave(&pch->lock, flags);
2131	}
2132	}
2133	spin_unlock_irqrestore(lock: &pch->lock, flags);
2134
2135	/ If work list empty, power down /
2136	if (power_down) {
2137	pm_runtime_mark_last_busy(dev: pch->dmac->ddma.dev);
2138	pm_runtime_put_autosuspend(dev: pch->dmac->ddma.dev);
2139	}
2140	}
2141
2142	static struct dma_chan of_dma_pl330_xlate(struct* of_phandle_args *dma_spec,
2143	struct of_dma *ofdma)
2144	{
2145	int count = dma_spec->args_count;
2146	struct pl330_dmac *pl330 = ofdma->of_dma_data;
2147	unsigned int chan_id;
2148
2149	if (!pl330)
2150	return NULL;
2151
2152	if (count != `1`)
2153	return NULL;
2154
2155	chan_id = dma_spec->args[`0`];
2156	if (chan_id >= pl330->num_peripherals)
2157	return NULL;
2158
2159	return dma_get_slave_channel(chan: &pl330->peripherals[chan_id].chan);
2160	}
2161
2162	static int pl330_alloc_chan_resources(struct dma_chan *chan)
2163	{
2164	struct dma_pl330_chan *pch = to_pchan(ch: chan);
2165	struct pl330_dmac *pl330 = pch->dmac;
2166	unsigned long flags;
2167
2168	spin_lock_irqsave(&pl330->lock, flags);
2169
2170	dma_cookie_init(chan);
2171	pch->cyclic = false;
2172
2173	pch->thread = pl330_request_channel(pl330);
2174	if (!pch->thread) {
2175	spin_unlock_irqrestore(lock: &pl330->lock, flags);
2176	return -ENOMEM;
2177	}
2178
2179	tasklet_setup(t: &pch->task, callback: pl330_tasklet);
2180
2181	spin_unlock_irqrestore(lock: &pl330->lock, flags);
2182
2183	return `1`;
2184	}
2185
2186	/*
2187	* We need the data direction between the DMAC (the dma-mapping "device") and
2188	* the FIFO (the dmaengine "dev"), from the FIFO's point of view. Confusing!
2189	*/
2190	static enum dma_data_direction
2191	pl330_dma_slave_map_dir(enum dma_transfer_direction dir)
2192	{
2193	switch (dir) {
2194	case DMA_MEM_TO_DEV:
2195	return DMA_FROM_DEVICE;
2196	case DMA_DEV_TO_MEM:
2197	return DMA_TO_DEVICE;
2198	case DMA_DEV_TO_DEV:
2199	return DMA_BIDIRECTIONAL;
2200	default:
2201	return DMA_NONE;
2202	}
2203	}
2204
2205	static void pl330_unprep_slave_fifo(struct dma_pl330_chan *pch)
2206	{
2207	if (pch->dir != DMA_NONE)
2208	dma_unmap_resource(dev: pch->chan.device->dev, addr: pch->fifo_dma,
2209	size: `1` << pch->burst_sz, dir: pch->dir, attrs: `0`);
2210	pch->dir = DMA_NONE;
2211	}
2212
2213
2214	static bool pl330_prep_slave_fifo(struct dma_pl330_chan *pch,
2215	enum dma_transfer_direction dir)
2216	{
2217	struct device *dev = pch->chan.device->dev;
2218	enum dma_data_direction dma_dir = pl330_dma_slave_map_dir(dir);
2219
2220	/ Already mapped for this config? /
2221	if (pch->dir == dma_dir)
2222	return true;
2223
2224	pl330_unprep_slave_fifo(pch);
2225	pch->fifo_dma = dma_map_resource(dev, phys_addr: pch->fifo_addr,
2226	size: `1` << pch->burst_sz, dir: dma_dir, attrs: `0`);
2227	if (dma_mapping_error(dev, dma_addr: pch->fifo_dma))
2228	return false;
2229
2230	pch->dir = dma_dir;
2231	return true;
2232	}
2233
2234	static int fixup_burst_len(int max_burst_len, int quirks)
2235	{
2236	if (max_burst_len > PL330_MAX_BURST)
2237	return PL330_MAX_BURST;
2238	else if (max_burst_len < `1`)
2239	return `1`;
2240	else
2241	return max_burst_len;
2242	}
2243
2244	static int pl330_config_write(struct dma_chan *chan,
2245	struct dma_slave_config *slave_config,
2246	enum dma_transfer_direction direction)
2247	{
2248	struct dma_pl330_chan *pch = to_pchan(ch: chan);
2249
2250	pl330_unprep_slave_fifo(pch);
2251	if (direction == DMA_MEM_TO_DEV) {
2252	if (slave_config->dst_addr)
2253	pch->fifo_addr = slave_config->dst_addr;
2254	if (slave_config->dst_addr_width)
2255	pch->burst_sz = __ffs(slave_config->dst_addr_width);
2256	pch->burst_len = fixup_burst_len(max_burst_len: slave_config->dst_maxburst,
2257	quirks: pch->dmac->quirks);
2258	} else if (direction == DMA_DEV_TO_MEM) {
2259	if (slave_config->src_addr)
2260	pch->fifo_addr = slave_config->src_addr;
2261	if (slave_config->src_addr_width)
2262	pch->burst_sz = __ffs(slave_config->src_addr_width);
2263	pch->burst_len = fixup_burst_len(max_burst_len: slave_config->src_maxburst,
2264	quirks: pch->dmac->quirks);
2265	}
2266
2267	return `0`;
2268	}
2269
2270	static int pl330_config(struct dma_chan *chan,
2271	struct dma_slave_config *slave_config)
2272	{
2273	struct dma_pl330_chan *pch = to_pchan(ch: chan);
2274
2275	memcpy(&pch->slave_config, slave_config, sizeof(*slave_config));
2276
2277	return `0`;
2278	}
2279
2280	static int pl330_terminate_all(struct dma_chan *chan)
2281	{
2282	struct dma_pl330_chan *pch = to_pchan(ch: chan);
2283	struct dma_pl330_desc *desc;
2284	unsigned long flags;
2285	struct pl330_dmac *pl330 = pch->dmac;
2286	bool power_down = false;
2287
2288	pm_runtime_get_sync(dev: pl330->ddma.dev);
2289	spin_lock_irqsave(&pch->lock, flags);
2290
2291	spin_lock(lock: &pl330->lock);
2292	_stop(thrd: pch->thread);
2293	pch->thread->req[`0`].desc = NULL;
2294	pch->thread->req[`1`].desc = NULL;
2295	pch->thread->req_running = -`1`;
2296	spin_unlock(lock: &pl330->lock);
2297
2298	power_down = pch->active;
2299	pch->active = false;
2300
2301	/ Mark all desc done /
2302	list_for_each_entry(desc, &pch->submitted_list, node) {
2303	desc->status = FREE;
2304	dma_cookie_complete(tx: &desc->txd);
2305	}
2306
2307	list_for_each_entry(desc, &pch->work_list , node) {
2308	desc->status = FREE;
2309	dma_cookie_complete(tx: &desc->txd);
2310	}
2311
2312	list_splice_tail_init(list: &pch->submitted_list, head: &pl330->desc_pool);
2313	list_splice_tail_init(list: &pch->work_list, head: &pl330->desc_pool);
2314	list_splice_tail_init(list: &pch->completed_list, head: &pl330->desc_pool);
2315	spin_unlock_irqrestore(lock: &pch->lock, flags);
2316	pm_runtime_mark_last_busy(dev: pl330->ddma.dev);
2317	if (power_down)
2318	pm_runtime_put_autosuspend(dev: pl330->ddma.dev);
2319	pm_runtime_put_autosuspend(dev: pl330->ddma.dev);
2320
2321	return `0`;
2322	}
2323
2324	/*
2325	* We don't support DMA_RESUME command because of hardware
2326	* limitations, so after pausing the channel we cannot restore
2327	* it to active state. We have to terminate channel and setup
2328	* DMA transfer again. This pause feature was implemented to
2329	* allow safely read residue before channel termination.
2330	*/
2331	static int pl330_pause(struct dma_chan *chan)
2332	{
2333	struct dma_pl330_chan *pch = to_pchan(ch: chan);
2334	struct pl330_dmac *pl330 = pch->dmac;
2335	struct dma_pl330_desc *desc;
2336	unsigned long flags;
2337
2338	pm_runtime_get_sync(dev: pl330->ddma.dev);
2339	spin_lock_irqsave(&pch->lock, flags);
2340
2341	spin_lock(lock: &pl330->lock);
2342	_stop(thrd: pch->thread);
2343	spin_unlock(lock: &pl330->lock);
2344
2345	list_for_each_entry(desc, &pch->work_list, node) {
2346	if (desc->status == BUSY)
2347	desc->status = PAUSED;
2348	}
2349	spin_unlock_irqrestore(lock: &pch->lock, flags);
2350	pm_runtime_mark_last_busy(dev: pl330->ddma.dev);
2351	pm_runtime_put_autosuspend(dev: pl330->ddma.dev);
2352
2353	return `0`;
2354	}
2355
2356	static void pl330_free_chan_resources(struct dma_chan *chan)
2357	{
2358	struct dma_pl330_chan *pch = to_pchan(ch: chan);
2359	struct pl330_dmac *pl330 = pch->dmac;
2360	unsigned long flags;
2361
2362	tasklet_kill(t: &pch->task);
2363
2364	pm_runtime_get_sync(dev: pch->dmac->ddma.dev);
2365	spin_lock_irqsave(&pl330->lock, flags);
2366
2367	pl330_release_channel(thrd: pch->thread);
2368	pch->thread = NULL;
2369
2370	if (pch->cyclic)
2371	list_splice_tail_init(list: &pch->work_list, head: &pch->dmac->desc_pool);
2372
2373	spin_unlock_irqrestore(lock: &pl330->lock, flags);
2374	pm_runtime_mark_last_busy(dev: pch->dmac->ddma.dev);
2375	pm_runtime_put_autosuspend(dev: pch->dmac->ddma.dev);
2376	pl330_unprep_slave_fifo(pch);
2377	}
2378
2379	static int pl330_get_current_xferred_count(struct dma_pl330_chan *pch,
2380	struct dma_pl330_desc *desc)
2381	{
2382	struct pl330_thread *thrd = pch->thread;
2383	struct pl330_dmac *pl330 = pch->dmac;
2384	void __iomem *regs = thrd->dmac->base;
2385	u32 val, addr;
2386
2387	pm_runtime_get_sync(dev: pl330->ddma.dev);
2388	val = addr = `0`;
2389	if (desc->rqcfg.src_inc) {
2390	val = readl(addr: regs + SA(thrd->id));
2391	addr = desc->px.src_addr;
2392	} else {
2393	val = readl(addr: regs + DA(thrd->id));
2394	addr = desc->px.dst_addr;
2395	}
2396	pm_runtime_mark_last_busy(dev: pch->dmac->ddma.dev);
2397	pm_runtime_put_autosuspend(dev: pl330->ddma.dev);
2398
2399	/ If DMAMOV hasn't finished yet, SAR/DAR can be zero /
2400	if (!val)
2401	return `0`;
2402
2403	return val - addr;
2404	}
2405
2406	static enum dma_status
2407	pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
2408	struct dma_tx_state *txstate)
2409	{
2410	enum dma_status ret;
2411	unsigned long flags;
2412	struct dma_pl330_desc desc, running = NULL, *last_enq = NULL;
2413	struct dma_pl330_chan *pch = to_pchan(ch: chan);
2414	unsigned int transferred, residual = `0`;
2415
2416	ret = dma_cookie_status(chan, cookie, state: txstate);
2417
2418	if (!txstate)
2419	return ret;
2420
2421	if (ret == DMA_COMPLETE)
2422	goto out;
2423
2424	spin_lock_irqsave(&pch->lock, flags);
2425	spin_lock(lock: &pch->thread->dmac->lock);
2426
2427	if (pch->thread->req_running != -`1`)
2428	running = pch->thread->req[pch->thread->req_running].desc;
2429
2430	last_enq = pch->thread->req[pch->thread->lstenq].desc;
2431
2432	/ Check in pending list /
2433	list_for_each_entry(desc, &pch->work_list, node) {
2434	if (desc->status == DONE)
2435	transferred = desc->bytes_requested;
2436	else if (running && desc == running)
2437	transferred =
2438	pl330_get_current_xferred_count(pch, desc);
2439	else if (desc->status == BUSY \|\| desc->status == PAUSED)
2440	/*
2441	* Busy but not running means either just enqueued,
2442	* or finished and not yet marked done
2443	*/
2444	if (desc == last_enq)
2445	transferred = `0`;
2446	else
2447	transferred = desc->bytes_requested;
2448	else
2449	transferred = `0`;
2450	residual += desc->bytes_requested - transferred;
2451	if (desc->txd.cookie == cookie) {
2452	switch (desc->status) {
2453	case DONE:
2454	ret = DMA_COMPLETE;
2455	break;
2456	case PAUSED:
2457	ret = DMA_PAUSED;
2458	break;
2459	case PREP:
2460	case BUSY:
2461	ret = DMA_IN_PROGRESS;
2462	break;
2463	default:
2464	WARN_ON(`1`);
2465	}
2466	break;
2467	}
2468	if (desc->last)
2469	residual = `0`;
2470	}
2471	spin_unlock(lock: &pch->thread->dmac->lock);
2472	spin_unlock_irqrestore(lock: &pch->lock, flags);
2473
2474	out:
2475	dma_set_residue(state: txstate, residue: residual);
2476
2477	return ret;
2478	}
2479
2480	static void pl330_issue_pending(struct dma_chan *chan)
2481	{
2482	struct dma_pl330_chan *pch = to_pchan(ch: chan);
2483	unsigned long flags;
2484
2485	spin_lock_irqsave(&pch->lock, flags);
2486	if (list_empty(head: &pch->work_list)) {
2487	/*
2488	* Warn on nothing pending. Empty submitted_list may
2489	* break our pm_runtime usage counter as it is
2490	* updated on work_list emptiness status.
2491	*/
2492	WARN_ON(list_empty(&pch->submitted_list));
2493	pch->active = true;
2494	pm_runtime_get_sync(dev: pch->dmac->ddma.dev);
2495	}
2496	list_splice_tail_init(list: &pch->submitted_list, head: &pch->work_list);
2497	spin_unlock_irqrestore(lock: &pch->lock, flags);
2498
2499	pl330_tasklet(t: &pch->task);
2500	}
2501
2502	/*
2503	* We returned the last one of the circular list of descriptor(s)
2504	* from prep_xxx, so the argument to submit corresponds to the last
2505	* descriptor of the list.
2506	*/
2507	static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx)
2508	{
2509	struct dma_pl330_desc desc, last = to_desc(tx);
2510	struct dma_pl330_chan *pch = to_pchan(ch: tx->chan);
2511	dma_cookie_t cookie;
2512	unsigned long flags;
2513
2514	spin_lock_irqsave(&pch->lock, flags);
2515
2516	/ Assign cookies to all nodes /
2517	while (!list_empty(head: &last->node)) {
2518	desc = list_entry(last->node.next, struct dma_pl330_desc, node);
2519	if (pch->cyclic) {
2520	desc->txd.callback = last->txd.callback;
2521	desc->txd.callback_param = last->txd.callback_param;
2522	}
2523	desc->last = false;
2524
2525	dma_cookie_assign(tx: &desc->txd);
2526
2527	list_move_tail(list: &desc->node, head: &pch->submitted_list);
2528	}
2529
2530	last->last = true;
2531	cookie = dma_cookie_assign(tx: &last->txd);
2532	list_add_tail(new: &last->node, head: &pch->submitted_list);
2533	spin_unlock_irqrestore(lock: &pch->lock, flags);
2534
2535	return cookie;
2536	}
2537
2538	static inline void _init_desc(struct dma_pl330_desc *desc)
2539	{
2540	desc->rqcfg.swap = SWAP_NO;
2541	desc->rqcfg.scctl = CCTRL0;
2542	desc->rqcfg.dcctl = CCTRL0;
2543	desc->txd.tx_submit = pl330_tx_submit;
2544
2545	INIT_LIST_HEAD(list: &desc->node);
2546	}
2547
2548	/ Returns the number of descriptors added to the DMAC pool /
2549	static int add_desc(struct list_head pool, spinlock_t lock,
2550	gfp_t flg, int count)
2551	{
2552	struct dma_pl330_desc *desc;
2553	unsigned long flags;
2554	int i;
2555
2556	desc = kcalloc(n: count, size: sizeof(*desc), flags: flg);
2557	if (!desc)
2558	return `0`;
2559
2560	spin_lock_irqsave(lock, flags);
2561
2562	for (i = `0`; i < count; i++) {
2563	_init_desc(desc: &desc[i]);
2564	list_add_tail(new: &desc[i].node, head: pool);
2565	}
2566
2567	spin_unlock_irqrestore(lock, flags);
2568
2569	return count;
2570	}
2571
2572	static struct dma_pl330_desc pluck_desc(struct* list_head *pool,
2573	spinlock_t *lock)
2574	{
2575	struct dma_pl330_desc *desc = NULL;
2576	unsigned long flags;
2577
2578	spin_lock_irqsave(lock, flags);
2579
2580	if (!list_empty(head: pool)) {
2581	desc = list_entry(pool->next,
2582	struct dma_pl330_desc, node);
2583
2584	list_del_init(entry: &desc->node);
2585
2586	desc->status = PREP;
2587	desc->txd.callback = NULL;
2588	}
2589
2590	spin_unlock_irqrestore(lock, flags);
2591
2592	return desc;
2593	}
2594
2595	static struct dma_pl330_desc pl330_get_desc(struct* dma_pl330_chan *pch)
2596	{
2597	struct pl330_dmac *pl330 = pch->dmac;
2598	u8 *peri_id = pch->chan.private;
2599	struct dma_pl330_desc *desc;
2600
2601	/ Pluck one desc from the pool of DMAC /
2602	desc = pluck_desc(pool: &pl330->desc_pool, lock: &pl330->pool_lock);
2603
2604	/ If the DMAC pool is empty, alloc new /
2605	if (!desc) {
2606	static DEFINE_SPINLOCK(lock);
2607	LIST_HEAD(pool);
2608
2609	if (!add_desc(pool: &pool, lock: &lock, GFP_ATOMIC, count: `1`))
2610	return NULL;
2611
2612	desc = pluck_desc(pool: &pool, lock: &lock);
2613	WARN_ON(!desc \|\| !list_empty(&pool));
2614	}
2615
2616	/ Initialize the descriptor /
2617	desc->pchan = pch;
2618	desc->txd.cookie = `0`;
2619	async_tx_ack(tx: &desc->txd);
2620
2621	desc->peri = peri_id ? pch->chan.chan_id : `0`;
2622	desc->rqcfg.pcfg = &pch->dmac->pcfg;
2623
2624	dma_async_tx_descriptor_init(tx: &desc->txd, chan: &pch->chan);
2625
2626	return desc;
2627	}
2628
2629	static inline void fill_px(struct pl330_xfer *px,
2630	dma_addr_t dst, dma_addr_t src, size_t len)
2631	{
2632	px->bytes = len;
2633	px->dst_addr = dst;
2634	px->src_addr = src;
2635	}
2636
2637	static struct dma_pl330_desc *
2638	__pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,
2639	dma_addr_t src, size_t len)
2640	{
2641	struct dma_pl330_desc *desc = pl330_get_desc(pch);
2642
2643	if (!desc) {
2644	dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",
2645	__func__, __LINE__);
2646	return NULL;
2647	}
2648
2649	/*
2650	* Ideally we should lookout for reqs bigger than
2651	* those that can be programmed with 256 bytes of
2652	* MC buffer, but considering a req size is seldom
2653	* going to be word-unaligned and more than 200MB,
2654	* we take it easy.
2655	* Also, should the limit is reached we'd rather
2656	* have the platform increase MC buffer size than
2657	* complicating this API driver.
2658	*/
2659	fill_px(px: &desc->px, dst, src, len);
2660
2661	return desc;
2662	}
2663
2664	/ Call after fixing burst size /
2665	static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
2666	{
2667	struct dma_pl330_chan *pch = desc->pchan;
2668	struct pl330_dmac *pl330 = pch->dmac;
2669	int burst_len;
2670
2671	burst_len = pl330->pcfg.data_bus_width / `8`;
2672	burst_len *= pl330->pcfg.data_buf_dep / pl330->pcfg.num_chan;
2673	burst_len >>= desc->rqcfg.brst_size;
2674
2675	/ src/dst_burst_len can't be more than 16 /
2676	if (burst_len > PL330_MAX_BURST)
2677	burst_len = PL330_MAX_BURST;
2678
2679	return burst_len;
2680	}
2681
2682	static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
2683	struct dma_chan *chan, dma_addr_t dma_addr, size_t len,
2684	size_t period_len, enum dma_transfer_direction direction,
2685	unsigned long flags)
2686	{
2687	struct dma_pl330_desc desc = NULL, first = NULL;
2688	struct dma_pl330_chan *pch = to_pchan(ch: chan);
2689	struct pl330_dmac *pl330 = pch->dmac;
2690	unsigned int i;
2691	dma_addr_t dst;
2692	dma_addr_t src;
2693
2694	if (len % period_len != `0`)
2695	return NULL;
2696
2697	if (!is_slave_direction(direction)) {
2698	dev_err(pch->dmac->ddma.dev, "%s:%d Invalid dma direction\n",
2699	__func__, __LINE__);
2700	return NULL;
2701	}
2702
2703	pl330_config_write(chan, slave_config: &pch->slave_config, direction);
2704
2705	if (!pl330_prep_slave_fifo(pch, dir: direction))
2706	return NULL;
2707
2708	for (i = `0`; i < len / period_len; i++) {
2709	desc = pl330_get_desc(pch);
2710	if (!desc) {
2711	unsigned long iflags;
2712
2713	dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",
2714	__func__, __LINE__);
2715
2716	if (!first)
2717	return NULL;
2718
2719	spin_lock_irqsave(&pl330->pool_lock, iflags);
2720
2721	while (!list_empty(head: &first->node)) {
2722	desc = list_entry(first->node.next,
2723	struct dma_pl330_desc, node);
2724	list_move_tail(list: &desc->node, head: &pl330->desc_pool);
2725	}
2726
2727	list_move_tail(list: &first->node, head: &pl330->desc_pool);
2728
2729	spin_unlock_irqrestore(lock: &pl330->pool_lock, flags: iflags);
2730
2731	return NULL;
2732	}
2733
2734	switch (direction) {
2735	case DMA_MEM_TO_DEV:
2736	desc->rqcfg.src_inc = `1`;
2737	desc->rqcfg.dst_inc = `0`;
2738	src = dma_addr;
2739	dst = pch->fifo_dma;
2740	break;
2741	case DMA_DEV_TO_MEM:
2742	desc->rqcfg.src_inc = `0`;
2743	desc->rqcfg.dst_inc = `1`;
2744	src = pch->fifo_dma;
2745	dst = dma_addr;
2746	break;
2747	default:
2748	break;
2749	}
2750
2751	desc->rqtype = direction;
2752	desc->rqcfg.brst_size = pch->burst_sz;
2753	desc->rqcfg.brst_len = pch->burst_len;
2754	desc->bytes_requested = period_len;
2755	fill_px(px: &desc->px, dst, src, len: period_len);
2756
2757	if (!first)
2758	first = desc;
2759	else
2760	list_add_tail(new: &desc->node, head: &first->node);
2761
2762	dma_addr += period_len;
2763	}
2764
2765	if (!desc)
2766	return NULL;
2767
2768	pch->cyclic = true;
2769
2770	return &desc->txd;
2771	}
2772
2773	static struct dma_async_tx_descriptor *
2774	pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
2775	dma_addr_t src, size_t len, unsigned long flags)
2776	{
2777	struct dma_pl330_desc *desc;
2778	struct dma_pl330_chan *pch = to_pchan(ch: chan);
2779	struct pl330_dmac *pl330;
2780	int burst;
2781
2782	if (unlikely(!pch \|\| !len))
2783	return NULL;
2784
2785	pl330 = pch->dmac;
2786
2787	desc = __pl330_prep_dma_memcpy(pch, dst, src, len);
2788	if (!desc)
2789	return NULL;
2790
2791	desc->rqcfg.src_inc = `1`;
2792	desc->rqcfg.dst_inc = `1`;
2793	desc->rqtype = DMA_MEM_TO_MEM;
2794
2795	/ Select max possible burst size /
2796	burst = pl330->pcfg.data_bus_width / `8`;
2797
2798	/*
2799	* Make sure we use a burst size that aligns with all the memcpy
2800	* parameters because our DMA programming algorithm doesn't cope with
2801	* transfers which straddle an entry in the DMA device's MFIFO.
2802	*/
2803	while ((src \| dst \| len) & (burst - `1`))
2804	burst /= `2`;
2805
2806	desc->rqcfg.brst_size = `0`;
2807	while (burst != (`1` << desc->rqcfg.brst_size))
2808	desc->rqcfg.brst_size++;
2809
2810	desc->rqcfg.brst_len = get_burst_len(desc, len);
2811	/*
2812	* If burst size is smaller than bus width then make sure we only
2813	* transfer one at a time to avoid a burst stradling an MFIFO entry.
2814	*/
2815	if (burst * `8` < pl330->pcfg.data_bus_width)
2816	desc->rqcfg.brst_len = `1`;
2817
2818	desc->bytes_requested = len;
2819
2820	return &desc->txd;
2821	}
2822
2823	static void __pl330_giveback_desc(struct pl330_dmac *pl330,
2824	struct dma_pl330_desc *first)
2825	{
2826	unsigned long flags;
2827	struct dma_pl330_desc *desc;
2828
2829	if (!first)
2830	return;
2831
2832	spin_lock_irqsave(&pl330->pool_lock, flags);
2833
2834	while (!list_empty(head: &first->node)) {
2835	desc = list_entry(first->node.next,
2836	struct dma_pl330_desc, node);
2837	list_move_tail(list: &desc->node, head: &pl330->desc_pool);
2838	}
2839
2840	list_move_tail(list: &first->node, head: &pl330->desc_pool);
2841
2842	spin_unlock_irqrestore(lock: &pl330->pool_lock, flags);
2843	}
2844
2845	static struct dma_async_tx_descriptor *
2846	pl330_prep_slave_sg(struct dma_chan chan, struct* scatterlist *sgl,
2847	unsigned int sg_len, enum dma_transfer_direction direction,
2848	unsigned long flg, void *context)
2849	{
2850	struct dma_pl330_desc first, desc = NULL;
2851	struct dma_pl330_chan *pch = to_pchan(ch: chan);
2852	struct scatterlist *sg;
2853	int i;
2854
2855	if (unlikely(!pch \|\| !sgl \|\| !sg_len))
2856	return NULL;
2857
2858	pl330_config_write(chan, slave_config: &pch->slave_config, direction);
2859
2860	if (!pl330_prep_slave_fifo(pch, dir: direction))
2861	return NULL;
2862
2863	first = NULL;
2864
2865	for_each_sg(sgl, sg, sg_len, i) {
2866
2867	desc = pl330_get_desc(pch);
2868	if (!desc) {
2869	struct pl330_dmac *pl330 = pch->dmac;
2870
2871	dev_err(pch->dmac->ddma.dev,
2872	"%s:%d Unable to fetch desc\n",
2873	__func__, __LINE__);
2874	__pl330_giveback_desc(pl330, first);
2875
2876	return NULL;
2877	}
2878
2879	if (!first)
2880	first = desc;
2881	else
2882	list_add_tail(new: &desc->node, head: &first->node);
2883
2884	if (direction == DMA_MEM_TO_DEV) {
2885	desc->rqcfg.src_inc = `1`;
2886	desc->rqcfg.dst_inc = `0`;
2887	fill_px(px: &desc->px, dst: pch->fifo_dma, sg_dma_address(sg),
2888	sg_dma_len(sg));
2889	} else {
2890	desc->rqcfg.src_inc = `0`;
2891	desc->rqcfg.dst_inc = `1`;
2892	fill_px(px: &desc->px, sg_dma_address(sg), src: pch->fifo_dma,
2893	sg_dma_len(sg));
2894	}
2895
2896	desc->rqcfg.brst_size = pch->burst_sz;
2897	desc->rqcfg.brst_len = pch->burst_len;
2898	desc->rqtype = direction;
2899	desc->bytes_requested = sg_dma_len(sg);
2900	}
2901
2902	/ Return the last desc in the chain /
2903	return &desc->txd;
2904	}
2905
2906	static irqreturn_t pl330_irq_handler(int irq, void *data)
2907	{
2908	if (pl330_update(pl330: data))
2909	return IRQ_HANDLED;
2910	else
2911	return IRQ_NONE;
2912	}
2913
2914	#define PL330_DMA_BUSWIDTHS \
2915	BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) \| \
2916	BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) \| \
2917	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) \| \
2918	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) \| \
2919	BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)
2920
2921	#ifdef CONFIG_DEBUG_FS
2922	static int pl330_debugfs_show(struct seq_file s, void* *data)
2923	{
2924	struct pl330_dmac *pl330 = s->private;
2925	int chans, pchs, ch, pr;
2926
2927	chans = pl330->pcfg.num_chan;
2928	pchs = pl330->num_peripherals;
2929
2930	seq_puts(m: s, s: "PL330 physical channels:\n");
2931	seq_puts(m: s, s: "THREAD:\t\tCHANNEL:\n");
2932	seq_puts(m: s, s: "--------\t-----\n");
2933	for (ch = `0`; ch < chans; ch++) {
2934	struct pl330_thread *thrd = &pl330->channels[ch];
2935	int found = -`1`;
2936
2937	for (pr = `0`; pr < pchs; pr++) {
2938	struct dma_pl330_chan *pch = &pl330->peripherals[pr];
2939
2940	if (!pch->thread \|\| thrd->id != pch->thread->id)
2941	continue;
2942
2943	found = pr;
2944	}
2945
2946	seq_printf(m: s, fmt: "%d\t\t", thrd->id);
2947	if (found == -`1`)
2948	seq_puts(m: s, s: "--\n");
2949	else
2950	seq_printf(m: s, fmt: "%d\n", found);
2951	}
2952
2953	return `0`;
2954	}
2955
2956	DEFINE_SHOW_ATTRIBUTE(pl330_debugfs);
2957
2958	static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
2959	{
2960	debugfs_create_file(name: dev_name(dev: pl330->ddma.dev),
2961	S_IFREG \| `0444`, NULL, data: pl330,
2962	fops: &pl330_debugfs_fops);
2963	}
2964	#else
2965	static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
2966	{
2967	}
2968	#endif
2969
2970	/*
2971	* Runtime PM callbacks are provided by amba/bus.c driver.
2972	*
2973	* It is assumed here that IRQ safe runtime PM is chosen in probe and amba
2974	* bus driver will only disable/enable the clock in runtime PM callbacks.
2975	*/
2976	static int __maybe_unused pl330_suspend(struct device *dev)
2977	{
2978	struct amba_device *pcdev = to_amba_device(dev);
2979
2980	pm_runtime_force_suspend(dev);
2981	clk_unprepare(clk: pcdev->pclk);
2982
2983	return `0`;
2984	}
2985
2986	static int __maybe_unused pl330_resume(struct device *dev)
2987	{
2988	struct amba_device *pcdev = to_amba_device(dev);
2989	int ret;
2990
2991	ret = clk_prepare(clk: pcdev->pclk);
2992	if (ret)
2993	return ret;
2994
2995	pm_runtime_force_resume(dev);
2996
2997	return ret;
2998	}
2999
3000	static const struct dev_pm_ops pl330_pm = {
3001	SET_LATE_SYSTEM_SLEEP_PM_OPS(pl330_suspend, pl330_resume)
3002	};
3003
3004	static int
3005	pl330_probe(struct amba_device adev, const* struct amba_id *id)
3006	{
3007	struct pl330_config *pcfg;
3008	struct pl330_dmac *pl330;
3009	struct dma_pl330_chan pch, _p;
3010	struct dma_device *pd;
3011	struct resource *res;
3012	int i, ret, irq;
3013	int num_chan;
3014	struct device_node *np = adev->dev.of_node;
3015
3016	ret = dma_set_mask_and_coherent(dev: &adev->dev, DMA_BIT_MASK(`32`));
3017	if (ret)
3018	return ret;
3019
3020	/ Allocate a new DMAC and its Channels /
3021	pl330 = devm_kzalloc(dev: &adev->dev, size: sizeof(*pl330), GFP_KERNEL);
3022	if (!pl330)
3023	return -ENOMEM;
3024
3025	pd = &pl330->ddma;
3026	pd->dev = &adev->dev;
3027
3028	pl330->mcbufsz = `0`;
3029
3030	/ get quirk /
3031	for (i = `0`; i < ARRAY_SIZE(of_quirks); i++)
3032	if (of_property_read_bool(np, propname: of_quirks[i].quirk))
3033	pl330->quirks \|= of_quirks[i].id;
3034
3035	res = &adev->res;
3036	pl330->base = devm_ioremap_resource(dev: &adev->dev, res);
3037	if (IS_ERR(ptr: pl330->base))
3038	return PTR_ERR(ptr: pl330->base);
3039
3040	amba_set_drvdata(adev, pl330);
3041
3042	pl330->rstc = devm_reset_control_get_optional(dev: &adev->dev, id: "dma");
3043	if (IS_ERR(ptr: pl330->rstc)) {
3044	return dev_err_probe(dev: &adev->dev, err: PTR_ERR(ptr: pl330->rstc), fmt: "Failed to get reset!\n");
3045	} else {
3046	ret = reset_control_deassert(rstc: pl330->rstc);
3047	if (ret) {
3048	dev_err(&adev->dev, "Couldn't deassert the device from reset!\n");
3049	return ret;
3050	}
3051	}
3052
3053	pl330->rstc_ocp = devm_reset_control_get_optional(dev: &adev->dev, id: "dma-ocp");
3054	if (IS_ERR(ptr: pl330->rstc_ocp)) {
3055	return dev_err_probe(dev: &adev->dev, err: PTR_ERR(ptr: pl330->rstc_ocp),
3056	fmt: "Failed to get OCP reset!\n");
3057	} else {
3058	ret = reset_control_deassert(rstc: pl330->rstc_ocp);
3059	if (ret) {
3060	dev_err(&adev->dev, "Couldn't deassert the device from OCP reset!\n");
3061	return ret;
3062	}
3063	}
3064
3065	for (i = `0`; i < AMBA_NR_IRQS; i++) {
3066	irq = adev->irq[i];
3067	if (irq) {
3068	ret = devm_request_irq(dev: &adev->dev, irq,
3069	handler: pl330_irq_handler, irqflags: `0`,
3070	devname: dev_name(dev: &adev->dev), dev_id: pl330);
3071	if (ret)
3072	return ret;
3073	} else {
3074	break;
3075	}
3076	}
3077
3078	pcfg = &pl330->pcfg;
3079
3080	pcfg->periph_id = adev->periphid;
3081	ret = pl330_add(pl330);
3082	if (ret)
3083	return ret;
3084
3085	INIT_LIST_HEAD(list: &pl330->desc_pool);
3086	spin_lock_init(&pl330->pool_lock);
3087
3088	/ Create a descriptor pool of default size /
3089	if (!add_desc(pool: &pl330->desc_pool, lock: &pl330->pool_lock,
3090	GFP_KERNEL, NR_DEFAULT_DESC))
3091	dev_warn(&adev->dev, "unable to allocate desc\n");
3092
3093	INIT_LIST_HEAD(list: &pd->channels);
3094
3095	/ Initialize channel parameters /
3096	num_chan = max_t(int, pcfg->num_peri, pcfg->num_chan);
3097
3098	pl330->num_peripherals = num_chan;
3099
3100	pl330->peripherals = kcalloc(n: num_chan, size: sizeof(*pch), GFP_KERNEL);
3101	if (!pl330->peripherals) {
3102	ret = -ENOMEM;
3103	goto probe_err2;
3104	}
3105
3106	for (i = `0`; i < num_chan; i++) {
3107	pch = &pl330->peripherals[i];
3108
3109	pch->chan.private = adev->dev.of_node;
3110	INIT_LIST_HEAD(list: &pch->submitted_list);
3111	INIT_LIST_HEAD(list: &pch->work_list);
3112	INIT_LIST_HEAD(list: &pch->completed_list);
3113	spin_lock_init(&pch->lock);
3114	pch->thread = NULL;
3115	pch->chan.device = pd;
3116	pch->dmac = pl330;
3117	pch->dir = DMA_NONE;
3118
3119	/ Add the channel to the DMAC list /
3120	list_add_tail(new: &pch->chan.device_node, head: &pd->channels);
3121	}
3122
3123	dma_cap_set(DMA_MEMCPY, pd->cap_mask);
3124	if (pcfg->num_peri) {
3125	dma_cap_set(DMA_SLAVE, pd->cap_mask);
3126	dma_cap_set(DMA_CYCLIC, pd->cap_mask);
3127	dma_cap_set(DMA_PRIVATE, pd->cap_mask);
3128	}
3129
3130	pd->device_alloc_chan_resources = pl330_alloc_chan_resources;
3131	pd->device_free_chan_resources = pl330_free_chan_resources;
3132	pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;
3133	pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic;
3134	pd->device_tx_status = pl330_tx_status;
3135	pd->device_prep_slave_sg = pl330_prep_slave_sg;
3136	pd->device_config = pl330_config;
3137	pd->device_pause = pl330_pause;
3138	pd->device_terminate_all = pl330_terminate_all;
3139	pd->device_issue_pending = pl330_issue_pending;
3140	pd->src_addr_widths = PL330_DMA_BUSWIDTHS;
3141	pd->dst_addr_widths = PL330_DMA_BUSWIDTHS;
3142	pd->directions = BIT(DMA_DEV_TO_MEM) \| BIT(DMA_MEM_TO_DEV);
3143	pd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
3144	pd->max_burst = PL330_MAX_BURST;
3145
3146	ret = dma_async_device_register(device: pd);
3147	if (ret) {
3148	dev_err(&adev->dev, "unable to register DMAC\n");
3149	goto probe_err3;
3150	}
3151
3152	if (adev->dev.of_node) {
3153	ret = of_dma_controller_register(np: adev->dev.of_node,
3154	of_dma_xlate: of_dma_pl330_xlate, data: pl330);
3155	if (ret) {
3156	dev_err(&adev->dev,
3157	"unable to register DMA to the generic DT DMA helpers\n");
3158	}
3159	}
3160
3161	/*
3162	* This is the limit for transfers with a buswidth of 1, larger
3163	* buswidths will have larger limits.
3164	*/
3165	ret = dma_set_max_seg_size(dev: &adev->dev, size: `1900800`);
3166	if (ret)
3167	dev_err(&adev->dev, "unable to set the seg size\n");
3168
3169
3170	init_pl330_debugfs(pl330);
3171	dev_info(&adev->dev,
3172	"Loaded driver for PL330 DMAC-%x\n", adev->periphid);
3173	dev_info(&adev->dev,
3174	"\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n",
3175	pcfg->data_buf_dep, pcfg->data_bus_width / `8`, pcfg->num_chan,
3176	pcfg->num_peri, pcfg->num_events);
3177
3178	pm_runtime_irq_safe(dev: &adev->dev);
3179	pm_runtime_use_autosuspend(dev: &adev->dev);
3180	pm_runtime_set_autosuspend_delay(dev: &adev->dev, PL330_AUTOSUSPEND_DELAY);
3181	pm_runtime_mark_last_busy(dev: &adev->dev);
3182	pm_runtime_put_autosuspend(dev: &adev->dev);
3183
3184	return `0`;
3185	probe_err3:
3186	/ Idle the DMAC /
3187	list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,
3188	chan.device_node) {
3189
3190	/ Remove the channel /
3191	list_del(entry: &pch->chan.device_node);
3192
3193	/ Flush the channel /
3194	if (pch->thread) {
3195	pl330_terminate_all(chan: &pch->chan);
3196	pl330_free_chan_resources(chan: &pch->chan);
3197	}
3198	}
3199	probe_err2:
3200	pl330_del(pl330);
3201
3202	if (pl330->rstc_ocp)
3203	reset_control_assert(rstc: pl330->rstc_ocp);
3204
3205	if (pl330->rstc)
3206	reset_control_assert(rstc: pl330->rstc);
3207	return ret;
3208	}
3209
3210	static void pl330_remove(struct amba_device *adev)
3211	{
3212	struct pl330_dmac *pl330 = amba_get_drvdata(adev);
3213	struct dma_pl330_chan pch, _p;
3214	int i, irq;
3215
3216	pm_runtime_get_noresume(dev: pl330->ddma.dev);
3217
3218	if (adev->dev.of_node)
3219	of_dma_controller_free(np: adev->dev.of_node);
3220
3221	for (i = `0`; i < AMBA_NR_IRQS; i++) {
3222	irq = adev->irq[i];
3223	if (irq)
3224	devm_free_irq(dev: &adev->dev, irq, dev_id: pl330);
3225	}
3226
3227	dma_async_device_unregister(device: &pl330->ddma);
3228
3229	/ Idle the DMAC /
3230	list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,
3231	chan.device_node) {
3232
3233	/ Remove the channel /
3234	list_del(entry: &pch->chan.device_node);
3235
3236	/ Flush the channel /
3237	if (pch->thread) {
3238	pl330_terminate_all(chan: &pch->chan);
3239	pl330_free_chan_resources(chan: &pch->chan);
3240	}
3241	}
3242
3243	pl330_del(pl330);
3244
3245	if (pl330->rstc_ocp)
3246	reset_control_assert(rstc: pl330->rstc_ocp);
3247
3248	if (pl330->rstc)
3249	reset_control_assert(rstc: pl330->rstc);
3250	}
3251
3252	static const struct amba_id pl330_ids[] = {
3253	{
3254	.id = `0x00041330`,
3255	.mask = `0x000fffff`,
3256	},
3257	{ `0`, `0` },
3258	};
3259
3260	MODULE_DEVICE_TABLE(amba, pl330_ids);
3261
3262	static struct amba_driver pl330_driver = {
3263	.drv = {
3264	.owner = THIS_MODULE,
3265	.name = "dma-pl330",
3266	.pm = &pl330_pm,
3267	},
3268	.id_table = pl330_ids,
3269	.probe = pl330_probe,
3270	.remove = pl330_remove,
3271	};
3272
3273	module_amba_driver(pl330_driver);
3274
3275	MODULE_AUTHOR("Jaswinder Singh <jassisinghbrar@gmail.com>");
3276	MODULE_DESCRIPTION("API Driver for PL330 DMAC");
3277	MODULE_LICENSE("GPL");
3278

source code of linux/drivers/dma/pl330.c