1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright (C) 2006-2009 DENX Software Engineering.
4	*
5	* Author: Yuri Tikhonov <yur@emcraft.com>
6	*
7	* Further porting to arch/powerpc by
8	* Anatolij Gustschin <agust@denx.de>
9	*/
10
11	/*
12	* This driver supports the asynchrounous DMA copy and RAID engines available
13	* on the AMCC PPC440SPe Processors.
14	* Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
15	* ADMA driver written by D.Williams.
16	*/
17
18	#include <linux/init.h>
19	#include <linux/module.h>
20	#include <linux/async_tx.h>
21	#include <linux/delay.h>
22	#include <linux/dma-mapping.h>
23	#include <linux/spinlock.h>
24	#include <linux/interrupt.h>
25	#include <linux/slab.h>
26	#include <linux/uaccess.h>
27	#include <linux/proc_fs.h>
28	#include <linux/of.h>
29	#include <linux/of_address.h>
30	#include <linux/of_irq.h>
31	#include <linux/platform_device.h>
32	#include <asm/dcr.h>
33	#include <asm/dcr-regs.h>
34	#include "adma.h"
35	#include "../dmaengine.h"
36
37	enum ppc_adma_init_code {
38	PPC_ADMA_INIT_OK = 0,
39	PPC_ADMA_INIT_MEMRES,
40	PPC_ADMA_INIT_MEMREG,
41	PPC_ADMA_INIT_ALLOC,
42	PPC_ADMA_INIT_COHERENT,
43	PPC_ADMA_INIT_CHANNEL,
44	PPC_ADMA_INIT_IRQ1,
45	PPC_ADMA_INIT_IRQ2,
46	PPC_ADMA_INIT_REGISTER
47	};
48
49	static char *ppc_adma_errors[] = {
50	[PPC_ADMA_INIT_OK] = "ok",
51	[PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
52	[PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
53	[PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
54	"structure",
55	[PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
56	"hardware descriptors",
57	[PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
58	[PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
59	[PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
60	[PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
61	};
62
63	static enum ppc_adma_init_code
64	ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
65
66	struct ppc_dma_chan_ref {
67	struct dma_chan *chan;
68	struct list_head node;
69	};
70
71	/* The list of channels exported by ppc440spe ADMA */
72	static struct list_head
73	ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
74
75	/* This flag is set when want to refetch the xor chain in the interrupt
76	* handler
77	*/
78	static u32 do_xor_refetch;
79
80	/* Pointer to DMA0, DMA1 CP/CS FIFO */
81	static void *ppc440spe_dma_fifo_buf;
82
83	/* Pointers to last submitted to DMA0, DMA1 CDBs */
84	static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
85	static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
86
87	/* Pointer to last linked and submitted xor CB */
88	static struct ppc440spe_adma_desc_slot *xor_last_linked;
89	static struct ppc440spe_adma_desc_slot *xor_last_submit;
90
91	/* This array is used in data-check operations for storing a pattern */
92	static char ppc440spe_qword[16];
93
94	static atomic_t ppc440spe_adma_err_irq_ref;
95	static dcr_host_t ppc440spe_mq_dcr_host;
96	static unsigned int ppc440spe_mq_dcr_len;
97
98	/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
99	* the block size in transactions, then we do not allow to activate more than
100	* only one RXOR transactions simultaneously. So use this var to store
101	* the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
102	* set) or not (PPC440SPE_RXOR_RUN is clear).
103	*/
104	static unsigned long ppc440spe_rxor_state;
105
106	/* These are used in enable & check routines
107	*/
108	static u32 ppc440spe_r6_enabled;
109	static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
110	static struct completion ppc440spe_r6_test_comp;
111
112	static int ppc440spe_adma_dma2rxor_prep_src(
113	struct ppc440spe_adma_desc_slot *desc,
114	struct ppc440spe_rxor *cursor, int index,
115	int src_cnt, u32 addr);
116	static void ppc440spe_adma_dma2rxor_set_src(
117	struct ppc440spe_adma_desc_slot *desc,
118	int index, dma_addr_t addr);
119	static void ppc440spe_adma_dma2rxor_set_mult(
120	struct ppc440spe_adma_desc_slot *desc,
121	int index, u8 mult);
122
123	#ifdef ADMA_LL_DEBUG
124	#define ADMA_LL_DBG(x) ({ if (1) x; 0; })
125	#else
126	#define ADMA_LL_DBG(x) ({ if (0) x; 0; })
127	#endif
128
129	static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
130	{
131	struct dma_cdb *cdb;
132	struct xor_cb *cb;
133	int i;
134
135	switch (chan->device->id) {
136	case 0:
137	case 1:
138	cdb = block;
139
140	pr_debug("CDB at %p [%d]:\n"
141	"\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
142	"\t sg1u 0x%08x sg1l 0x%08x\n"
143	"\t sg2u 0x%08x sg2l 0x%08x\n"
144	"\t sg3u 0x%08x sg3l 0x%08x\n",
145	cdb, chan->device->id,
146	cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
147	le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
148	le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
149	le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
150	);
151	break;
152	case 2:
153	cb = block;
154
155	pr_debug("CB at %p [%d]:\n"
156	"\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
157	"\t cbtah 0x%08x cbtal 0x%08x\n"
158	"\t cblah 0x%08x cblal 0x%08x\n",
159	cb, chan->device->id,
160	cb->cbc, cb->cbbc, cb->cbs,
161	cb->cbtah, cb->cbtal,
162	cb->cblah, cb->cblal);
163	for (i = 0; i < 16; i++) {
164	if (i && !cb->ops[i].h && !cb->ops[i].l)
165	continue;
166	pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
167	i, cb->ops[i].h, cb->ops[i].l);
168	}
169	break;
170	}
171	}
172
173	static void print_cb_list(struct ppc440spe_adma_chan *chan,
174	struct ppc440spe_adma_desc_slot *iter)
175	{
176	for (; iter; iter = iter->hw_next)
177	print_cb(chan, block: iter->hw_desc);
178	}
179
180	static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
181	unsigned int src_cnt)
182	{
183	int i;
184
185	pr_debug("\n%s(%d):\nsrc: ", __func__, id);
186	for (i = 0; i < src_cnt; i++)
187	pr_debug("\t0x%016llx ", src[i]);
188	pr_debug("dst:\n\t0x%016llx\n", dst);
189	}
190
191	static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
192	unsigned int src_cnt)
193	{
194	int i;
195
196	pr_debug("\n%s(%d):\nsrc: ", __func__, id);
197	for (i = 0; i < src_cnt; i++)
198	pr_debug("\t0x%016llx ", src[i]);
199	pr_debug("dst: ");
200	for (i = 0; i < 2; i++)
201	pr_debug("\t0x%016llx ", dst[i]);
202	}
203
204	static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
205	unsigned int src_cnt,
206	const unsigned char *scf)
207	{
208	int i;
209
210	pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
211	if (scf) {
212	for (i = 0; i < src_cnt; i++)
213	pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
214	} else {
215	for (i = 0; i < src_cnt; i++)
216	pr_debug("\t0x%016llx(no) ", src[i]);
217	}
218
219	pr_debug("dst: ");
220	for (i = 0; i < 2; i++)
221	pr_debug("\t0x%016llx ", src[src_cnt + i]);
222	}
223
224	/******************************************************************************
225	* Command (Descriptor) Blocks low-level routines
226	******************************************************************************/
227	/**
228	* ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
229	* pseudo operation
230	*/
231	static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
232	struct ppc440spe_adma_chan *chan)
233	{
234	struct xor_cb *p;
235
236	switch (chan->device->id) {
237	case PPC440SPE_XOR_ID:
238	p = desc->hw_desc;
239	memset(desc->hw_desc, 0, sizeof(struct xor_cb));
240	/* NOP with Command Block Complete Enable */
241	p->cbc = XOR_CBCR_CBCE_BIT;
242	break;
243	case PPC440SPE_DMA0_ID:
244	case PPC440SPE_DMA1_ID:
245	memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
246	/* NOP with interrupt */
247	set_bit(PPC440SPE_DESC_INT, addr: &desc->flags);
248	break;
249	default:
250	printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
251	__func__);
252	break;
253	}
254	}
255
256	/**
257	* ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
258	* pseudo operation
259	*/
260	static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
261	{
262	memset(desc->hw_desc, 0, sizeof(struct xor_cb));
263	desc->hw_next = NULL;
264	desc->src_cnt = 0;
265	desc->dst_cnt = 1;
266	}
267
268	/**
269	* ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
270	*/
271	static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
272	int src_cnt, unsigned long flags)
273	{
274	struct xor_cb *hw_desc = desc->hw_desc;
275
276	memset(desc->hw_desc, 0, sizeof(struct xor_cb));
277	desc->hw_next = NULL;
278	desc->src_cnt = src_cnt;
279	desc->dst_cnt = 1;
280
281	hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
282	if (flags & DMA_PREP_INTERRUPT)
283	/* Enable interrupt on completion */
284	hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
285	}
286
287	/**
288	* ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
289	* operation in DMA2 controller
290	*/
291	static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
292	int dst_cnt, int src_cnt, unsigned long flags)
293	{
294	struct xor_cb *hw_desc = desc->hw_desc;
295
296	memset(desc->hw_desc, 0, sizeof(struct xor_cb));
297	desc->hw_next = NULL;
298	desc->src_cnt = src_cnt;
299	desc->dst_cnt = dst_cnt;
300	memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
301	desc->descs_per_op = 0;
302
303	hw_desc->cbc = XOR_CBCR_TGT_BIT;
304	if (flags & DMA_PREP_INTERRUPT)
305	/* Enable interrupt on completion */
306	hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
307	}
308
309	#define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE
310	#define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1)
311	#define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1)
312
313	/**
314	* ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
315	* with DMA0/1
316	*/
317	static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
318	int dst_cnt, int src_cnt, unsigned long flags,
319	unsigned long op)
320	{
321	struct dma_cdb *hw_desc;
322	struct ppc440spe_adma_desc_slot *iter;
323	u8 dopc;
324
325	/* Common initialization of a PQ descriptors chain */
326	set_bits(op, &desc->flags);
327	desc->src_cnt = src_cnt;
328	desc->dst_cnt = dst_cnt;
329
330	/* WXOR MULTICAST if both P and Q are being computed
331	* MV_SG1_SG2 if Q only
332	*/
333	dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
334	DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
335
336	list_for_each_entry(iter, &desc->group_list, chain_node) {
337	hw_desc = iter->hw_desc;
338	memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
339
340	if (likely(!list_is_last(&iter->chain_node,
341	&desc->group_list))) {
342	/* set 'next' pointer */
343	iter->hw_next = list_entry(iter->chain_node.next,
344	struct ppc440spe_adma_desc_slot, chain_node);
345	clear_bit(PPC440SPE_DESC_INT, addr: &iter->flags);
346	} else {
347	/* this is the last descriptor.
348	* this slot will be pasted from ADMA level
349	* each time it wants to configure parameters
350	* of the transaction (src, dst, ...)
351	*/
352	iter->hw_next = NULL;
353	if (flags & DMA_PREP_INTERRUPT)
354	set_bit(PPC440SPE_DESC_INT, addr: &iter->flags);
355	else
356	clear_bit(PPC440SPE_DESC_INT, addr: &iter->flags);
357	}
358	}
359
360	/* Set OPS depending on WXOR/RXOR type of operation */
361	if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
362	/* This is a WXOR only chain:
363	* - first descriptors are for zeroing destinations
364	* if PPC440SPE_ZERO_P/Q set;
365	* - descriptors remained are for GF-XOR operations.
366	*/
367	iter = list_first_entry(&desc->group_list,
368	struct ppc440spe_adma_desc_slot,
369	chain_node);
370
371	if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
372	hw_desc = iter->hw_desc;
373	hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
374	iter = list_first_entry(&iter->chain_node,
375	struct ppc440spe_adma_desc_slot,
376	chain_node);
377	}
378
379	if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
380	hw_desc = iter->hw_desc;
381	hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
382	iter = list_first_entry(&iter->chain_node,
383	struct ppc440spe_adma_desc_slot,
384	chain_node);
385	}
386
387	list_for_each_entry_from(iter, &desc->group_list, chain_node) {
388	hw_desc = iter->hw_desc;
389	hw_desc->opc = dopc;
390	}
391	} else {
392	/* This is either RXOR-only or mixed RXOR/WXOR */
393
394	/* The first 1 or 2 slots in chain are always RXOR,
395	* if need to calculate P & Q, then there are two
396	* RXOR slots; if only P or only Q, then there is one
397	*/
398	iter = list_first_entry(&desc->group_list,
399	struct ppc440spe_adma_desc_slot,
400	chain_node);
401	hw_desc = iter->hw_desc;
402	hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
403
404	if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
405	iter = list_first_entry(&iter->chain_node,
406	struct ppc440spe_adma_desc_slot,
407	chain_node);
408	hw_desc = iter->hw_desc;
409	hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
410	}
411
412	/* The remaining descs (if any) are WXORs */
413	if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
414	iter = list_first_entry(&iter->chain_node,
415	struct ppc440spe_adma_desc_slot,
416	chain_node);
417	list_for_each_entry_from(iter, &desc->group_list,
418	chain_node) {
419	hw_desc = iter->hw_desc;
420	hw_desc->opc = dopc;
421	}
422	}
423	}
424	}
425
426	/**
427	* ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
428	* for PQ_ZERO_SUM operation
429	*/
430	static void ppc440spe_desc_init_dma01pqzero_sum(
431	struct ppc440spe_adma_desc_slot *desc,
432	int dst_cnt, int src_cnt)
433	{
434	struct dma_cdb *hw_desc;
435	struct ppc440spe_adma_desc_slot *iter;
436	int i = 0;
437	u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
438	DMA_CDB_OPC_MV_SG1_SG2;
439	/*
440	* Initialize starting from 2nd or 3rd descriptor dependent
441	* on dst_cnt. First one or two slots are for cloning P
442	* and/or Q to chan->pdest and/or chan->qdest as we have
443	* to preserve original P/Q.
444	*/
445	iter = list_first_entry(&desc->group_list,
446	struct ppc440spe_adma_desc_slot, chain_node);
447	iter = list_entry(iter->chain_node.next,
448	struct ppc440spe_adma_desc_slot, chain_node);
449
450	if (dst_cnt > 1) {
451	iter = list_entry(iter->chain_node.next,
452	struct ppc440spe_adma_desc_slot, chain_node);
453	}
454	/* initialize each source descriptor in chain */
455	list_for_each_entry_from(iter, &desc->group_list, chain_node) {
456	hw_desc = iter->hw_desc;
457	memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
458	iter->src_cnt = 0;
459	iter->dst_cnt = 0;
460
461	/* This is a ZERO_SUM operation:
462	* - <src_cnt> descriptors starting from 2nd or 3rd
463	* descriptor are for GF-XOR operations;
464	* - remaining <dst_cnt> descriptors are for checking the result
465	*/
466	if (i++ < src_cnt)
467	/* MV_SG1_SG2 if only Q is being verified
468	* MULTICAST if both P and Q are being verified
469	*/
470	hw_desc->opc = dopc;
471	else
472	/* DMA_CDB_OPC_DCHECK128 operation */
473	hw_desc->opc = DMA_CDB_OPC_DCHECK128;
474
475	if (likely(!list_is_last(&iter->chain_node,
476	&desc->group_list))) {
477	/* set 'next' pointer */
478	iter->hw_next = list_entry(iter->chain_node.next,
479	struct ppc440spe_adma_desc_slot,
480	chain_node);
481	} else {
482	/* this is the last descriptor.
483	* this slot will be pasted from ADMA level
484	* each time it wants to configure parameters
485	* of the transaction (src, dst, ...)
486	*/
487	iter->hw_next = NULL;
488	/* always enable interrupt generation since we get
489	* the status of pqzero from the handler
490	*/
491	set_bit(PPC440SPE_DESC_INT, addr: &iter->flags);
492	}
493	}
494	desc->src_cnt = src_cnt;
495	desc->dst_cnt = dst_cnt;
496	}
497
498	/**
499	* ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
500	*/
501	static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
502	unsigned long flags)
503	{
504	struct dma_cdb *hw_desc = desc->hw_desc;
505
506	memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
507	desc->hw_next = NULL;
508	desc->src_cnt = 1;
509	desc->dst_cnt = 1;
510
511	if (flags & DMA_PREP_INTERRUPT)
512	set_bit(PPC440SPE_DESC_INT, addr: &desc->flags);
513	else
514	clear_bit(PPC440SPE_DESC_INT, addr: &desc->flags);
515
516	hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
517	}
518
519	/**
520	* ppc440spe_desc_set_src_addr - set source address into the descriptor
521	*/
522	static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
523	struct ppc440spe_adma_chan *chan,
524	int src_idx, dma_addr_t addrh,
525	dma_addr_t addrl)
526	{
527	struct dma_cdb *dma_hw_desc;
528	struct xor_cb *xor_hw_desc;
529	phys_addr_t addr64, tmplow, tmphi;
530
531	switch (chan->device->id) {
532	case PPC440SPE_DMA0_ID:
533	case PPC440SPE_DMA1_ID:
534	if (!addrh) {
535	addr64 = addrl;
536	tmphi = (addr64 >> 32);
537	tmplow = (addr64 & 0xFFFFFFFF);
538	} else {
539	tmphi = addrh;
540	tmplow = addrl;
541	}
542	dma_hw_desc = desc->hw_desc;
543	dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
544	dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
545	break;
546	case PPC440SPE_XOR_ID:
547	xor_hw_desc = desc->hw_desc;
548	xor_hw_desc->ops[src_idx].l = addrl;
549	xor_hw_desc->ops[src_idx].h |= addrh;
550	break;
551	}
552	}
553
554	/**
555	* ppc440spe_desc_set_src_mult - set source address mult into the descriptor
556	*/
557	static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
558	struct ppc440spe_adma_chan *chan, u32 mult_index,
559	int sg_index, unsigned char mult_value)
560	{
561	struct dma_cdb *dma_hw_desc;
562	u32 *psgu;
563
564	switch (chan->device->id) {
565	case PPC440SPE_DMA0_ID:
566	case PPC440SPE_DMA1_ID:
567	dma_hw_desc = desc->hw_desc;
568
569	switch (sg_index) {
570	/* for RXOR operations set multiplier
571	* into source cued address
572	*/
573	case DMA_CDB_SG_SRC:
574	psgu = &dma_hw_desc->sg1u;
575	break;
576	/* for WXOR operations set multiplier
577	* into destination cued address(es)
578	*/
579	case DMA_CDB_SG_DST1:
580	psgu = &dma_hw_desc->sg2u;
581	break;
582	case DMA_CDB_SG_DST2:
583	psgu = &dma_hw_desc->sg3u;
584	break;
585	default:
586	BUG();
587	}
588
589	*psgu |= cpu_to_le32(mult_value << mult_index);
590	break;
591	case PPC440SPE_XOR_ID:
592	break;
593	default:
594	BUG();
595	}
596	}
597
598	/**
599	* ppc440spe_desc_set_dest_addr - set destination address into the descriptor
600	*/
601	static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
602	struct ppc440spe_adma_chan *chan,
603	dma_addr_t addrh, dma_addr_t addrl,
604	u32 dst_idx)
605	{
606	struct dma_cdb *dma_hw_desc;
607	struct xor_cb *xor_hw_desc;
608	phys_addr_t addr64, tmphi, tmplow;
609	u32 *psgu, *psgl;
610
611	switch (chan->device->id) {
612	case PPC440SPE_DMA0_ID:
613	case PPC440SPE_DMA1_ID:
614	if (!addrh) {
615	addr64 = addrl;
616	tmphi = (addr64 >> 32);
617	tmplow = (addr64 & 0xFFFFFFFF);
618	} else {
619	tmphi = addrh;
620	tmplow = addrl;
621	}
622	dma_hw_desc = desc->hw_desc;
623
624	psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
625	psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
626
627	*psgl = cpu_to_le32((u32)tmplow);
628	*psgu |= cpu_to_le32((u32)tmphi);
629	break;
630	case PPC440SPE_XOR_ID:
631	xor_hw_desc = desc->hw_desc;
632	xor_hw_desc->cbtal = addrl;
633	xor_hw_desc->cbtah |= addrh;
634	break;
635	}
636	}
637
638	/**
639	* ppc440spe_desc_set_byte_count - set number of data bytes involved
640	* into the operation
641	*/
642	static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
643	struct ppc440spe_adma_chan *chan,
644	u32 byte_count)
645	{
646	struct dma_cdb *dma_hw_desc;
647	struct xor_cb *xor_hw_desc;
648
649	switch (chan->device->id) {
650	case PPC440SPE_DMA0_ID:
651	case PPC440SPE_DMA1_ID:
652	dma_hw_desc = desc->hw_desc;
653	dma_hw_desc->cnt = cpu_to_le32(byte_count);
654	break;
655	case PPC440SPE_XOR_ID:
656	xor_hw_desc = desc->hw_desc;
657	xor_hw_desc->cbbc = byte_count;
658	break;
659	}
660	}
661
662	/**
663	* ppc440spe_desc_set_rxor_block_size - set RXOR block size
664	*/
665	static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
666	{
667	/* assume that byte_count is aligned on the 512-boundary;
668	* thus write it directly to the register (bits 23:31 are
669	* reserved there).
670	*/
671	dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
672	}
673
674	/**
675	* ppc440spe_desc_set_dcheck - set CHECK pattern
676	*/
677	static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
678	struct ppc440spe_adma_chan *chan, u8 *qword)
679	{
680	struct dma_cdb *dma_hw_desc;
681
682	switch (chan->device->id) {
683	case PPC440SPE_DMA0_ID:
684	case PPC440SPE_DMA1_ID:
685	dma_hw_desc = desc->hw_desc;
686	iowrite32(qword[0], &dma_hw_desc->sg3l);
687	iowrite32(qword[4], &dma_hw_desc->sg3u);
688	iowrite32(qword[8], &dma_hw_desc->sg2l);
689	iowrite32(qword[12], &dma_hw_desc->sg2u);
690	break;
691	default:
692	BUG();
693	}
694	}
695
696	/**
697	* ppc440spe_xor_set_link - set link address in xor CB
698	*/
699	static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
700	struct ppc440spe_adma_desc_slot *next_desc)
701	{
702	struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
703
704	if (unlikely(!next_desc || !(next_desc->phys))) {
705	printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
706	__func__, next_desc,
707	next_desc ? next_desc->phys : 0);
708	BUG();
709	}
710
711	xor_hw_desc->cbs = 0;
712	xor_hw_desc->cblal = next_desc->phys;
713	xor_hw_desc->cblah = 0;
714	xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
715	}
716
717	/**
718	* ppc440spe_desc_set_link - set the address of descriptor following this
719	* descriptor in chain
720	*/
721	static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
722	struct ppc440spe_adma_desc_slot *prev_desc,
723	struct ppc440spe_adma_desc_slot *next_desc)
724	{
725	unsigned long flags;
726	struct ppc440spe_adma_desc_slot *tail = next_desc;
727
728	if (unlikely(!prev_desc || !next_desc ||
729	(prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
730	/* If previous next is overwritten something is wrong.
731	* though we may refetch from append to initiate list
732	* processing; in this case - it's ok.
733	*/
734	printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
735	"prev->hw_next=0x%p\n", __func__, prev_desc,
736	next_desc, prev_desc ? prev_desc->hw_next : 0);
737	BUG();
738	}
739
740	local_irq_save(flags);
741
742	/* do s/w chaining both for DMA and XOR descriptors */
743	prev_desc->hw_next = next_desc;
744
745	switch (chan->device->id) {
746	case PPC440SPE_DMA0_ID:
747	case PPC440SPE_DMA1_ID:
748	break;
749	case PPC440SPE_XOR_ID:
750	/* bind descriptor to the chain */
751	while (tail->hw_next)
752	tail = tail->hw_next;
753	xor_last_linked = tail;
754
755	if (prev_desc == xor_last_submit)
756	/* do not link to the last submitted CB */
757	break;
758	ppc440spe_xor_set_link(prev_desc, next_desc);
759	break;
760	}
761
762	local_irq_restore(flags);
763	}
764
765	/**
766	* ppc440spe_desc_get_link - get the address of the descriptor that
767	* follows this one
768	*/
769	static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
770	struct ppc440spe_adma_chan *chan)
771	{
772	if (!desc->hw_next)
773	return 0;
774
775	return desc->hw_next->phys;
776	}
777
778	/**
779	* ppc440spe_desc_is_aligned - check alignment
780	*/
781	static inline int ppc440spe_desc_is_aligned(
782	struct ppc440spe_adma_desc_slot *desc, int num_slots)
783	{
784	return (desc->idx & (num_slots - 1)) ? 0 : 1;
785	}
786
787	/**
788	* ppc440spe_chan_xor_slot_count - get the number of slots necessary for
789	* XOR operation
790	*/
791	static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
792	int *slots_per_op)
793	{
794	int slot_cnt;
795
796	/* each XOR descriptor provides up to 16 source operands */
797	slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
798
799	if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
800	return slot_cnt;
801
802	printk(KERN_ERR "%s: len %d > max %d !!\n",
803	__func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
804	BUG();
805	return slot_cnt;
806	}
807
808	/**
809	* ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
810	* DMA2 PQ operation
811	*/
812	static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
813	int src_cnt, size_t len)
814	{
815	signed long long order = 0;
816	int state = 0;
817	int addr_count = 0;
818	int i;
819	for (i = 1; i < src_cnt; i++) {
820	dma_addr_t cur_addr = srcs[i];
821	dma_addr_t old_addr = srcs[i-1];
822	switch (state) {
823	case 0:
824	if (cur_addr == old_addr + len) {
825	/* direct RXOR */
826	order = 1;
827	state = 1;
828	if (i == src_cnt-1)
829	addr_count++;
830	} else if (old_addr == cur_addr + len) {
831	/* reverse RXOR */
832	order = -1;
833	state = 1;
834	if (i == src_cnt-1)
835	addr_count++;
836	} else {
837	state = 3;
838	}
839	break;
840	case 1:
841	if (i == src_cnt-2 || (order == -1
842	&& cur_addr != old_addr - len)) {
843	order = 0;
844	state = 0;
845	addr_count++;
846	} else if (cur_addr == old_addr + len*order) {
847	state = 2;
848	if (i == src_cnt-1)
849	addr_count++;
850	} else if (cur_addr == old_addr + 2*len) {
851	state = 2;
852	if (i == src_cnt-1)
853	addr_count++;
854	} else if (cur_addr == old_addr + 3*len) {
855	state = 2;
856	if (i == src_cnt-1)
857	addr_count++;
858	} else {
859	order = 0;
860	state = 0;
861	addr_count++;
862	}
863	break;
864	case 2:
865	order = 0;
866	state = 0;
867	addr_count++;
868	break;
869	}
870	if (state == 3)
871	break;
872	}
873	if (src_cnt <= 1 || (state != 1 && state != 2)) {
874	pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
875	__func__, src_cnt, state, addr_count, order);
876	for (i = 0; i < src_cnt; i++)
877	pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
878	BUG();
879	}
880
881	return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
882	}
883
884
885	/******************************************************************************
886	* ADMA channel low-level routines
887	******************************************************************************/
888
889	static u32
890	ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
891	static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
892
893	/**
894	* ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
895	*/
896	static void ppc440spe_adma_device_clear_eot_status(
897	struct ppc440spe_adma_chan *chan)
898	{
899	struct dma_regs *dma_reg;
900	struct xor_regs *xor_reg;
901	u8 *p = chan->device->dma_desc_pool_virt;
902	struct dma_cdb *cdb;
903	u32 rv, i;
904
905	switch (chan->device->id) {
906	case PPC440SPE_DMA0_ID:
907	case PPC440SPE_DMA1_ID:
908	/* read FIFO to ack */
909	dma_reg = chan->device->dma_reg;
910	while ((rv = ioread32(&dma_reg->csfpl))) {
911	i = rv & DMA_CDB_ADDR_MSK;
912	cdb = (struct dma_cdb *)&p[i -
913	(u32)chan->device->dma_desc_pool];
914
915	/* Clear opcode to ack. This is necessary for
916	* ZeroSum operations only
917	*/
918	cdb->opc = 0;
919
920	if (test_bit(PPC440SPE_RXOR_RUN,
921	&ppc440spe_rxor_state)) {
922	/* probably this is a completed RXOR op,
923	* get pointer to CDB using the fact that
924	* physical and virtual addresses of CDB
925	* in pools have the same offsets
926	*/
927	if (le32_to_cpu(cdb->sg1u) &
928	DMA_CUED_XOR_BASE) {
929	/* this is a RXOR */
930	clear_bit(PPC440SPE_RXOR_RUN,
931	addr: &ppc440spe_rxor_state);
932	}
933	}
934
935	if (rv & DMA_CDB_STATUS_MSK) {
936	/* ZeroSum check failed
937	*/
938	struct ppc440spe_adma_desc_slot *iter;
939	dma_addr_t phys = rv & ~DMA_CDB_MSK;
940
941	/*
942	* Update the status of corresponding
943	* descriptor.
944	*/
945	list_for_each_entry(iter, &chan->chain,
946	chain_node) {
947	if (iter->phys == phys)
948	break;
949	}
950	/*
951	* if cannot find the corresponding
952	* slot it's a bug
953	*/
954	BUG_ON(&iter->chain_node == &chan->chain);
955
956	if (iter->xor_check_result) {
957	if (test_bit(PPC440SPE_DESC_PCHECK,
958	&iter->flags)) {
959	*iter->xor_check_result |=
960	SUM_CHECK_P_RESULT;
961	} else
962	if (test_bit(PPC440SPE_DESC_QCHECK,
963	&iter->flags)) {
964	*iter->xor_check_result |=
965	SUM_CHECK_Q_RESULT;
966	} else
967	BUG();
968	}
969	}
970	}
971
972	rv = ioread32(&dma_reg->dsts);
973	if (rv) {
974	pr_err("DMA%d err status: 0x%x\n",
975	chan->device->id, rv);
976	/* write back to clear */
977	iowrite32(rv, &dma_reg->dsts);
978	}
979	break;
980	case PPC440SPE_XOR_ID:
981	/* reset status bits to ack */
982	xor_reg = chan->device->xor_reg;
983	rv = ioread32be(&xor_reg->sr);
984	iowrite32be(rv, &xor_reg->sr);
985
986	if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
987	if (rv & XOR_IE_RPTIE_BIT) {
988	/* Read PLB Timeout Error.
989	* Try to resubmit the CB
990	*/
991	u32 val = ioread32be(&xor_reg->ccbalr);
992
993	iowrite32be(val, &xor_reg->cblalr);
994
995	val = ioread32be(&xor_reg->crsr);
996	iowrite32be(val | XOR_CRSR_XAE_BIT,
997	&xor_reg->crsr);
998	} else
999	pr_err("XOR ERR 0x%x status\n", rv);
1000	break;
1001	}
1002
1003	/* if the XORcore is idle, but there are unprocessed CBs
1004	* then refetch the s/w chain here
1005	*/
1006	if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1007	do_xor_refetch)
1008	ppc440spe_chan_append(chan);
1009	break;
1010	}
1011	}
1012
1013	/**
1014	* ppc440spe_chan_is_busy - get the channel status
1015	*/
1016	static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1017	{
1018	struct dma_regs *dma_reg;
1019	struct xor_regs *xor_reg;
1020	int busy = 0;
1021
1022	switch (chan->device->id) {
1023	case PPC440SPE_DMA0_ID:
1024	case PPC440SPE_DMA1_ID:
1025	dma_reg = chan->device->dma_reg;
1026	/* if command FIFO's head and tail pointers are equal and
1027	* status tail is the same as command, then channel is free
1028	*/
1029	if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1030	ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1031	busy = 1;
1032	break;
1033	case PPC440SPE_XOR_ID:
1034	/* use the special status bit for the XORcore
1035	*/
1036	xor_reg = chan->device->xor_reg;
1037	busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1038	break;
1039	}
1040
1041	return busy;
1042	}
1043
1044	/**
1045	* ppc440spe_chan_set_first_xor_descriptor - init XORcore chain
1046	*/
1047	static void ppc440spe_chan_set_first_xor_descriptor(
1048	struct ppc440spe_adma_chan *chan,
1049	struct ppc440spe_adma_desc_slot *next_desc)
1050	{
1051	struct xor_regs *xor_reg = chan->device->xor_reg;
1052
1053	if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1054	printk(KERN_INFO "%s: Warn: XORcore is running "
1055	"when try to set the first CDB!\n",
1056	__func__);
1057
1058	xor_last_submit = xor_last_linked = next_desc;
1059
1060	iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1061
1062	iowrite32be(next_desc->phys, &xor_reg->cblalr);
1063	iowrite32be(0, &xor_reg->cblahr);
1064	iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1065	&xor_reg->cbcr);
1066
1067	chan->hw_chain_inited = 1;
1068	}
1069
1070	/**
1071	* ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1072	* called with irqs disabled
1073	*/
1074	static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1075	struct ppc440spe_adma_desc_slot *desc)
1076	{
1077	u32 pcdb;
1078	struct dma_regs *dma_reg = chan->device->dma_reg;
1079
1080	pcdb = desc->phys;
1081	if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1082	pcdb |= DMA_CDB_NO_INT;
1083
1084	chan_last_sub[chan->device->id] = desc;
1085
1086	ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1087
1088	iowrite32(pcdb, &dma_reg->cpfpl);
1089	}
1090
1091	/**
1092	* ppc440spe_chan_append - update the h/w chain in the channel
1093	*/
1094	static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1095	{
1096	struct xor_regs *xor_reg;
1097	struct ppc440spe_adma_desc_slot *iter;
1098	struct xor_cb *xcb;
1099	u32 cur_desc;
1100	unsigned long flags;
1101
1102	local_irq_save(flags);
1103
1104	switch (chan->device->id) {
1105	case PPC440SPE_DMA0_ID:
1106	case PPC440SPE_DMA1_ID:
1107	cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1108
1109	if (likely(cur_desc)) {
1110	iter = chan_last_sub[chan->device->id];
1111	BUG_ON(!iter);
1112	} else {
1113	/* first peer */
1114	iter = chan_first_cdb[chan->device->id];
1115	BUG_ON(!iter);
1116	ppc440spe_dma_put_desc(chan, desc: iter);
1117	chan->hw_chain_inited = 1;
1118	}
1119
1120	/* is there something new to append */
1121	if (!iter->hw_next)
1122	break;
1123
1124	/* flush descriptors from the s/w queue to fifo */
1125	list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1126	ppc440spe_dma_put_desc(chan, desc: iter);
1127	if (!iter->hw_next)
1128	break;
1129	}
1130	break;
1131	case PPC440SPE_XOR_ID:
1132	/* update h/w links and refetch */
1133	if (!xor_last_submit->hw_next)
1134	break;
1135
1136	xor_reg = chan->device->xor_reg;
1137	/* the last linked CDB has to generate an interrupt
1138	* that we'd be able to append the next lists to h/w
1139	* regardless of the XOR engine state at the moment of
1140	* appending of these next lists
1141	*/
1142	xcb = xor_last_linked->hw_desc;
1143	xcb->cbc |= XOR_CBCR_CBCE_BIT;
1144
1145	if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1146	/* XORcore is idle. Refetch now */
1147	do_xor_refetch = 0;
1148	ppc440spe_xor_set_link(prev_desc: xor_last_submit,
1149	next_desc: xor_last_submit->hw_next);
1150
1151	ADMA_LL_DBG(print_cb_list(chan,
1152	xor_last_submit->hw_next));
1153
1154	xor_last_submit = xor_last_linked;
1155	iowrite32be(ioread32be(&xor_reg->crsr) |
1156	XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1157	&xor_reg->crsr);
1158	} else {
1159	/* XORcore is running. Refetch later in the handler */
1160	do_xor_refetch = 1;
1161	}
1162
1163	break;
1164	}
1165
1166	local_irq_restore(flags);
1167	}
1168
1169	/**
1170	* ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1171	*/
1172	static u32
1173	ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1174	{
1175	struct dma_regs *dma_reg;
1176	struct xor_regs *xor_reg;
1177
1178	if (unlikely(!chan->hw_chain_inited))
1179	/* h/w descriptor chain is not initialized yet */
1180	return 0;
1181
1182	switch (chan->device->id) {
1183	case PPC440SPE_DMA0_ID:
1184	case PPC440SPE_DMA1_ID:
1185	dma_reg = chan->device->dma_reg;
1186	return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1187	case PPC440SPE_XOR_ID:
1188	xor_reg = chan->device->xor_reg;
1189	return ioread32be(&xor_reg->ccbalr);
1190	}
1191	return 0;
1192	}
1193
1194	/**
1195	* ppc440spe_chan_run - enable the channel
1196	*/
1197	static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1198	{
1199	struct xor_regs *xor_reg;
1200
1201	switch (chan->device->id) {
1202	case PPC440SPE_DMA0_ID:
1203	case PPC440SPE_DMA1_ID:
1204	/* DMAs are always enabled, do nothing */
1205	break;
1206	case PPC440SPE_XOR_ID:
1207	/* drain write buffer */
1208	xor_reg = chan->device->xor_reg;
1209
1210	/* fetch descriptor pointed to in <link> */
1211	iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1212	&xor_reg->crsr);
1213	break;
1214	}
1215	}
1216
1217	/******************************************************************************
1218	* ADMA device level
1219	******************************************************************************/
1220
1221	static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1222	static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1223
1224	static dma_cookie_t
1225	ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1226
1227	static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1228	dma_addr_t addr, int index);
1229	static void
1230	ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1231	dma_addr_t addr, int index);
1232
1233	static void
1234	ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1235	dma_addr_t *paddr, unsigned long flags);
1236	static void
1237	ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1238	dma_addr_t addr, int index);
1239	static void
1240	ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1241	unsigned char mult, int index, int dst_pos);
1242	static void
1243	ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1244	dma_addr_t paddr, dma_addr_t qaddr);
1245
1246	static struct page *ppc440spe_rxor_srcs[32];
1247
1248	/**
1249	* ppc440spe_can_rxor - check if the operands may be processed with RXOR
1250	*/
1251	static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1252	{
1253	int i, order = 0, state = 0;
1254	int idx = 0;
1255
1256	if (unlikely(!(src_cnt > 1)))
1257	return 0;
1258
1259	BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1260
1261	/* Skip holes in the source list before checking */
1262	for (i = 0; i < src_cnt; i++) {
1263	if (!srcs[i])
1264	continue;
1265	ppc440spe_rxor_srcs[idx++] = srcs[i];
1266	}
1267	src_cnt = idx;
1268
1269	for (i = 1; i < src_cnt; i++) {
1270	char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1271	char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1272
1273	switch (state) {
1274	case 0:
1275	if (cur_addr == old_addr + len) {
1276	/* direct RXOR */
1277	order = 1;
1278	state = 1;
1279	} else if (old_addr == cur_addr + len) {
1280	/* reverse RXOR */
1281	order = -1;
1282	state = 1;
1283	} else
1284	goto out;
1285	break;
1286	case 1:
1287	if ((i == src_cnt - 2) ||
1288	(order == -1 && cur_addr != old_addr - len)) {
1289	order = 0;
1290	state = 0;
1291	} else if ((cur_addr == old_addr + len * order) ||
1292	(cur_addr == old_addr + 2 * len) ||
1293	(cur_addr == old_addr + 3 * len)) {
1294	state = 2;
1295	} else {
1296	order = 0;
1297	state = 0;
1298	}
1299	break;
1300	case 2:
1301	order = 0;
1302	state = 0;
1303	break;
1304	}
1305	}
1306
1307	out:
1308	if (state == 1 || state == 2)
1309	return 1;
1310
1311	return 0;
1312	}
1313
1314	/**
1315	* ppc440spe_adma_device_estimate - estimate the efficiency of processing
1316	* the operation given on this channel. It's assumed that 'chan' is
1317	* capable to process 'cap' type of operation.
1318	* @chan: channel to use
1319	* @cap: type of transaction
1320	* @dst_lst: array of destination pointers
1321	* @dst_cnt: number of destination operands
1322	* @src_lst: array of source pointers
1323	* @src_cnt: number of source operands
1324	* @src_sz: size of each source operand
1325	*/
1326	static int ppc440spe_adma_estimate(struct dma_chan *chan,
1327	enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1328	struct page **src_lst, int src_cnt, size_t src_sz)
1329	{
1330	int ef = 1;
1331
1332	if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1333	/* If RAID-6 capabilities were not activated don't try
1334	* to use them
1335	*/
1336	if (unlikely(!ppc440spe_r6_enabled))
1337	return -1;
1338	}
1339	/* In the current implementation of ppc440spe ADMA driver it
1340	* makes sense to pick out only pq case, because it may be
1341	* processed:
1342	* (1) either using Biskup method on DMA2;
1343	* (2) or on DMA0/1.
1344	* Thus we give a favour to (1) if the sources are suitable;
1345	* else let it be processed on one of the DMA0/1 engines.
1346	* In the sum_product case where destination is also the
1347	* source process it on DMA0/1 only.
1348	*/
1349	if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1350
1351	if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1352	ef = 0; /* sum_product case, process on DMA0/1 */
1353	else if (ppc440spe_can_rxor(srcs: src_lst, src_cnt, len: src_sz))
1354	ef = 3; /* override (DMA0/1 + idle) */
1355	else
1356	ef = 0; /* can't process on DMA2 if !rxor */
1357	}
1358
1359	/* channel idleness increases the priority */
1360	if (likely(ef) &&
1361	!ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1362	ef++;
1363
1364	return ef;
1365	}
1366
1367	struct dma_chan *
1368	ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1369	struct page **dst_lst, int dst_cnt, struct page **src_lst,
1370	int src_cnt, size_t src_sz)
1371	{
1372	struct dma_chan *best_chan = NULL;
1373	struct ppc_dma_chan_ref *ref;
1374	int best_rank = -1;
1375
1376	if (unlikely(!src_sz))
1377	return NULL;
1378	if (src_sz > PAGE_SIZE) {
1379	/*
1380	* should a user of the api ever pass > PAGE_SIZE requests
1381	* we sort out cases where temporary page-sized buffers
1382	* are used.
1383	*/
1384	switch (cap) {
1385	case DMA_PQ:
1386	if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1387	return NULL;
1388	if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1389	return NULL;
1390	break;
1391	case DMA_PQ_VAL:
1392	case DMA_XOR_VAL:
1393	return NULL;
1394	default:
1395	break;
1396	}
1397	}
1398
1399	list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1400	if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1401	int rank;
1402
1403	rank = ppc440spe_adma_estimate(chan: ref->chan, cap, dst_lst,
1404	dst_cnt, src_lst, src_cnt, src_sz);
1405	if (rank > best_rank) {
1406	best_rank = rank;
1407	best_chan = ref->chan;
1408	}
1409	}
1410	}
1411
1412	return best_chan;
1413	}
1414	EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1415
1416	/**
1417	* ppc440spe_get_group_entry - get group entry with index idx
1418	* @tdesc: is the last allocated slot in the group.
1419	*/
1420	static struct ppc440spe_adma_desc_slot *
1421	ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1422	{
1423	struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1424	int i = 0;
1425
1426	if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1427	printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1428	__func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1429	BUG();
1430	}
1431
1432	list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1433	if (i++ == entry_idx)
1434	break;
1435	}
1436	return iter;
1437	}
1438
1439	/**
1440	* ppc440spe_adma_free_slots - flags descriptor slots for reuse
1441	* @slot: Slot to free
1442	* Caller must hold &ppc440spe_chan->lock while calling this function
1443	*/
1444	static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1445	struct ppc440spe_adma_chan *chan)
1446	{
1447	int stride = slot->slots_per_op;
1448
1449	while (stride--) {
1450	slot->slots_per_op = 0;
1451	slot = list_entry(slot->slot_node.next,
1452	struct ppc440spe_adma_desc_slot,
1453	slot_node);
1454	}
1455	}
1456
1457	/**
1458	* ppc440spe_adma_run_tx_complete_actions - call functions to be called
1459	* upon completion
1460	*/
1461	static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1462	struct ppc440spe_adma_desc_slot *desc,
1463	struct ppc440spe_adma_chan *chan,
1464	dma_cookie_t cookie)
1465	{
1466	BUG_ON(desc->async_tx.cookie < 0);
1467	if (desc->async_tx.cookie > 0) {
1468	cookie = desc->async_tx.cookie;
1469	desc->async_tx.cookie = 0;
1470
1471	dma_descriptor_unmap(tx: &desc->async_tx);
1472	/* call the callback (must not sleep or submit new
1473	* operations to this channel)
1474	*/
1475	dmaengine_desc_get_callback_invoke(tx: &desc->async_tx, NULL);
1476	}
1477
1478	/* run dependent operations */
1479	dma_run_dependencies(tx: &desc->async_tx);
1480
1481	return cookie;
1482	}
1483
1484	/**
1485	* ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1486	*/
1487	static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1488	struct ppc440spe_adma_chan *chan)
1489	{
1490	/* the client is allowed to attach dependent operations
1491	* until 'ack' is set
1492	*/
1493	if (!async_tx_test_ack(tx: &desc->async_tx))
1494	return 0;
1495
1496	/* leave the last descriptor in the chain
1497	* so we can append to it
1498	*/
1499	if (list_is_last(list: &desc->chain_node, head: &chan->chain) ||
1500	desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1501	return 1;
1502
1503	if (chan->device->id != PPC440SPE_XOR_ID) {
1504	/* our DMA interrupt handler clears opc field of
1505	* each processed descriptor. For all types of
1506	* operations except for ZeroSum we do not actually
1507	* need ack from the interrupt handler. ZeroSum is a
1508	* special case since the result of this operation
1509	* is available from the handler only, so if we see
1510	* such type of descriptor (which is unprocessed yet)
1511	* then leave it in chain.
1512	*/
1513	struct dma_cdb *cdb = desc->hw_desc;
1514	if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1515	return 1;
1516	}
1517
1518	dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1519	desc->phys, desc->idx, desc->slots_per_op);
1520
1521	list_del(entry: &desc->chain_node);
1522	ppc440spe_adma_free_slots(slot: desc, chan);
1523	return 0;
1524	}
1525
1526	/**
1527	* __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1528	* which runs through the channel CDBs list until reach the descriptor
1529	* currently processed. When routine determines that all CDBs of group
1530	* are completed then corresponding callbacks (if any) are called and slots
1531	* are freed.
1532	*/
1533	static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1534	{
1535	struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1536	dma_cookie_t cookie = 0;
1537	u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1538	int busy = ppc440spe_chan_is_busy(chan);
1539	int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1540
1541	dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1542	chan->device->id, __func__);
1543
1544	if (!current_desc) {
1545	/* There were no transactions yet, so
1546	* nothing to clean
1547	*/
1548	return;
1549	}
1550
1551	/* free completed slots from the chain starting with
1552	* the oldest descriptor
1553	*/
1554	list_for_each_entry_safe(iter, _iter, &chan->chain,
1555	chain_node) {
1556	dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1557	"busy: %d this_desc: %#llx next_desc: %#x "
1558	"cur: %#x ack: %d\n",
1559	iter->async_tx.cookie, iter->idx, busy, iter->phys,
1560	ppc440spe_desc_get_link(iter, chan), current_desc,
1561	async_tx_test_ack(&iter->async_tx));
1562	prefetch(x: _iter);
1563	prefetch(x: &_iter->async_tx);
1564
1565	/* do not advance past the current descriptor loaded into the
1566	* hardware channel,subsequent descriptors are either in process
1567	* or have not been submitted
1568	*/
1569	if (seen_current)
1570	break;
1571
1572	/* stop the search if we reach the current descriptor and the
1573	* channel is busy, or if it appears that the current descriptor
1574	* needs to be re-read (i.e. has been appended to)
1575	*/
1576	if (iter->phys == current_desc) {
1577	BUG_ON(seen_current++);
1578	if (busy || ppc440spe_desc_get_link(desc: iter, chan)) {
1579	/* not all descriptors of the group have
1580	* been completed; exit.
1581	*/
1582	break;
1583	}
1584	}
1585
1586	/* detect the start of a group transaction */
1587	if (!slot_cnt && !slots_per_op) {
1588	slot_cnt = iter->slot_cnt;
1589	slots_per_op = iter->slots_per_op;
1590	if (slot_cnt <= slots_per_op) {
1591	slot_cnt = 0;
1592	slots_per_op = 0;
1593	}
1594	}
1595
1596	if (slot_cnt) {
1597	if (!group_start)
1598	group_start = iter;
1599	slot_cnt -= slots_per_op;
1600	}
1601
1602	/* all the members of a group are complete */
1603	if (slots_per_op != 0 && slot_cnt == 0) {
1604	struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1605	int end_of_chain = 0;
1606
1607	/* clean up the group */
1608	slot_cnt = group_start->slot_cnt;
1609	grp_iter = group_start;
1610	list_for_each_entry_safe_from(grp_iter, _grp_iter,
1611	&chan->chain, chain_node) {
1612
1613	cookie = ppc440spe_adma_run_tx_complete_actions(
1614	desc: grp_iter, chan, cookie);
1615
1616	slot_cnt -= slots_per_op;
1617	end_of_chain = ppc440spe_adma_clean_slot(
1618	desc: grp_iter, chan);
1619	if (end_of_chain && slot_cnt) {
1620	/* Should wait for ZeroSum completion */
1621	if (cookie > 0)
1622	chan->common.completed_cookie = cookie;
1623	return;
1624	}
1625
1626	if (slot_cnt == 0 || end_of_chain)
1627	break;
1628	}
1629
1630	/* the group should be complete at this point */
1631	BUG_ON(slot_cnt);
1632
1633	slots_per_op = 0;
1634	group_start = NULL;
1635	if (end_of_chain)
1636	break;
1637	else
1638	continue;
1639	} else if (slots_per_op) /* wait for group completion */
1640	continue;
1641
1642	cookie = ppc440spe_adma_run_tx_complete_actions(desc: iter, chan,
1643	cookie);
1644
1645	if (ppc440spe_adma_clean_slot(desc: iter, chan))
1646	break;
1647	}
1648
1649	BUG_ON(!seen_current);
1650
1651	if (cookie > 0) {
1652	chan->common.completed_cookie = cookie;
1653	pr_debug("\tcompleted cookie %d\n", cookie);
1654	}
1655
1656	}
1657
1658	/**
1659	* ppc440spe_adma_tasklet - clean up watch-dog initiator
1660	*/
1661	static void ppc440spe_adma_tasklet(struct tasklet_struct *t)
1662	{
1663	struct ppc440spe_adma_chan *chan = from_tasklet(chan, t, irq_tasklet);
1664
1665	spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1666	__ppc440spe_adma_slot_cleanup(chan);
1667	spin_unlock(lock: &chan->lock);
1668	}
1669
1670	/**
1671	* ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1672	*/
1673	static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1674	{
1675	spin_lock_bh(lock: &chan->lock);
1676	__ppc440spe_adma_slot_cleanup(chan);
1677	spin_unlock_bh(lock: &chan->lock);
1678	}
1679
1680	/**
1681	* ppc440spe_adma_alloc_slots - allocate free slots (if any)
1682	*/
1683	static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1684	struct ppc440spe_adma_chan *chan, int num_slots,
1685	int slots_per_op)
1686	{
1687	struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1688	struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1689	int slots_found, retry = 0;
1690	LIST_HEAD(chain);
1691
1692
1693	BUG_ON(!num_slots || !slots_per_op);
1694	/* start search from the last allocated descrtiptor
1695	* if a contiguous allocation can not be found start searching
1696	* from the beginning of the list
1697	*/
1698	retry:
1699	slots_found = 0;
1700	if (retry == 0)
1701	iter = chan->last_used;
1702	else
1703	iter = list_entry(&chan->all_slots,
1704	struct ppc440spe_adma_desc_slot,
1705	slot_node);
1706	list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
1707	slot_node) {
1708	prefetch(x: _iter);
1709	prefetch(x: &_iter->async_tx);
1710	if (iter->slots_per_op) {
1711	slots_found = 0;
1712	continue;
1713	}
1714
1715	/* start the allocation if the slot is correctly aligned */
1716	if (!slots_found++)
1717	alloc_start = iter;
1718
1719	if (slots_found == num_slots) {
1720	struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
1721	struct ppc440spe_adma_desc_slot *last_used = NULL;
1722
1723	iter = alloc_start;
1724	while (num_slots) {
1725	int i;
1726	/* pre-ack all but the last descriptor */
1727	if (num_slots != slots_per_op)
1728	async_tx_ack(tx: &iter->async_tx);
1729
1730	list_add_tail(new: &iter->chain_node, head: &chain);
1731	alloc_tail = iter;
1732	iter->async_tx.cookie = 0;
1733	iter->hw_next = NULL;
1734	iter->flags = 0;
1735	iter->slot_cnt = num_slots;
1736	iter->xor_check_result = NULL;
1737	for (i = 0; i < slots_per_op; i++) {
1738	iter->slots_per_op = slots_per_op - i;
1739	last_used = iter;
1740	iter = list_entry(iter->slot_node.next,
1741	struct ppc440spe_adma_desc_slot,
1742	slot_node);
1743	}
1744	num_slots -= slots_per_op;
1745	}
1746	alloc_tail->group_head = alloc_start;
1747	alloc_tail->async_tx.cookie = -EBUSY;
1748	list_splice(list: &chain, head: &alloc_tail->group_list);
1749	chan->last_used = last_used;
1750	return alloc_tail;
1751	}
1752	}
1753	if (!retry++)
1754	goto retry;
1755
1756	/* try to free some slots if the allocation fails */
1757	tasklet_schedule(t: &chan->irq_tasklet);
1758	return NULL;
1759	}
1760
1761	/**
1762	* ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots
1763	*/
1764	static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
1765	{
1766	struct ppc440spe_adma_chan *ppc440spe_chan;
1767	struct ppc440spe_adma_desc_slot *slot = NULL;
1768	char *hw_desc;
1769	int i, db_sz;
1770	int init;
1771
1772	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1773	init = ppc440spe_chan->slots_allocated ? 0 : 1;
1774	chan->chan_id = ppc440spe_chan->device->id;
1775
1776	/* Allocate descriptor slots */
1777	i = ppc440spe_chan->slots_allocated;
1778	if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
1779	db_sz = sizeof(struct dma_cdb);
1780	else
1781	db_sz = sizeof(struct xor_cb);
1782
1783	for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
1784	slot = kzalloc(size: sizeof(struct ppc440spe_adma_desc_slot),
1785	GFP_KERNEL);
1786	if (!slot) {
1787	printk(KERN_INFO "SPE ADMA Channel only initialized"
1788	" %d descriptor slots", i--);
1789	break;
1790	}
1791
1792	hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
1793	slot->hw_desc = (void *) &hw_desc[i * db_sz];
1794	dma_async_tx_descriptor_init(tx: &slot->async_tx, chan);
1795	slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
1796	INIT_LIST_HEAD(list: &slot->chain_node);
1797	INIT_LIST_HEAD(list: &slot->slot_node);
1798	INIT_LIST_HEAD(list: &slot->group_list);
1799	slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
1800	slot->idx = i;
1801
1802	spin_lock_bh(lock: &ppc440spe_chan->lock);
1803	ppc440spe_chan->slots_allocated++;
1804	list_add_tail(new: &slot->slot_node, head: &ppc440spe_chan->all_slots);
1805	spin_unlock_bh(lock: &ppc440spe_chan->lock);
1806	}
1807
1808	if (i && !ppc440spe_chan->last_used) {
1809	ppc440spe_chan->last_used =
1810	list_entry(ppc440spe_chan->all_slots.next,
1811	struct ppc440spe_adma_desc_slot,
1812	slot_node);
1813	}
1814
1815	dev_dbg(ppc440spe_chan->device->common.dev,
1816	"ppc440spe adma%d: allocated %d descriptor slots\n",
1817	ppc440spe_chan->device->id, i);
1818
1819	/* initialize the channel and the chain with a null operation */
1820	if (init) {
1821	switch (ppc440spe_chan->device->id) {
1822	case PPC440SPE_DMA0_ID:
1823	case PPC440SPE_DMA1_ID:
1824	ppc440spe_chan->hw_chain_inited = 0;
1825	/* Use WXOR for self-testing */
1826	if (!ppc440spe_r6_tchan)
1827	ppc440spe_r6_tchan = ppc440spe_chan;
1828	break;
1829	case PPC440SPE_XOR_ID:
1830	ppc440spe_chan_start_null_xor(chan: ppc440spe_chan);
1831	break;
1832	default:
1833	BUG();
1834	}
1835	ppc440spe_chan->needs_unmap = 1;
1836	}
1837
1838	return (i > 0) ? i : -ENOMEM;
1839	}
1840
1841	/**
1842	* ppc440spe_rxor_set_region_data -
1843	*/
1844	static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
1845	u8 xor_arg_no, u32 mask)
1846	{
1847	struct xor_cb *xcb = desc->hw_desc;
1848
1849	xcb->ops[xor_arg_no].h |= mask;
1850	}
1851
1852	/**
1853	* ppc440spe_rxor_set_src -
1854	*/
1855	static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
1856	u8 xor_arg_no, dma_addr_t addr)
1857	{
1858	struct xor_cb *xcb = desc->hw_desc;
1859
1860	xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
1861	xcb->ops[xor_arg_no].l = addr;
1862	}
1863
1864	/**
1865	* ppc440spe_rxor_set_mult -
1866	*/
1867	static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
1868	u8 xor_arg_no, u8 idx, u8 mult)
1869	{
1870	struct xor_cb *xcb = desc->hw_desc;
1871
1872	xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
1873	}
1874
1875	/**
1876	* ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
1877	* has been achieved
1878	*/
1879	static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
1880	{
1881	dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
1882	chan->device->id, chan->pending);
1883
1884	if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
1885	chan->pending = 0;
1886	ppc440spe_chan_append(chan);
1887	}
1888	}
1889
1890	/**
1891	* ppc440spe_adma_tx_submit - submit new descriptor group to the channel
1892	* (it's not necessary that descriptors will be submitted to the h/w
1893	* chains too right now)
1894	*/
1895	static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
1896	{
1897	struct ppc440spe_adma_desc_slot *sw_desc;
1898	struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
1899	struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
1900	int slot_cnt;
1901	int slots_per_op;
1902	dma_cookie_t cookie;
1903
1904	sw_desc = tx_to_ppc440spe_adma_slot(tx);
1905
1906	group_start = sw_desc->group_head;
1907	slot_cnt = group_start->slot_cnt;
1908	slots_per_op = group_start->slots_per_op;
1909
1910	spin_lock_bh(lock: &chan->lock);
1911	cookie = dma_cookie_assign(tx);
1912
1913	if (unlikely(list_empty(&chan->chain))) {
1914	/* first peer */
1915	list_splice_init(list: &sw_desc->group_list, head: &chan->chain);
1916	chan_first_cdb[chan->device->id] = group_start;
1917	} else {
1918	/* isn't first peer, bind CDBs to chain */
1919	old_chain_tail = list_entry(chan->chain.prev,
1920	struct ppc440spe_adma_desc_slot,
1921	chain_node);
1922	list_splice_init(list: &sw_desc->group_list,
1923	head: &old_chain_tail->chain_node);
1924	/* fix up the hardware chain */
1925	ppc440spe_desc_set_link(chan, prev_desc: old_chain_tail, next_desc: group_start);
1926	}
1927
1928	/* increment the pending count by the number of operations */
1929	chan->pending += slot_cnt / slots_per_op;
1930	ppc440spe_adma_check_threshold(chan);
1931	spin_unlock_bh(lock: &chan->lock);
1932
1933	dev_dbg(chan->device->common.dev,
1934	"ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
1935	chan->device->id, __func__,
1936	sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
1937
1938	return cookie;
1939	}
1940
1941	/**
1942	* ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
1943	*/
1944	static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
1945	struct dma_chan *chan, unsigned long flags)
1946	{
1947	struct ppc440spe_adma_chan *ppc440spe_chan;
1948	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1949	int slot_cnt, slots_per_op;
1950
1951	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1952
1953	dev_dbg(ppc440spe_chan->device->common.dev,
1954	"ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
1955	__func__);
1956
1957	spin_lock_bh(lock: &ppc440spe_chan->lock);
1958	slot_cnt = slots_per_op = 1;
1959	sw_desc = ppc440spe_adma_alloc_slots(chan: ppc440spe_chan, num_slots: slot_cnt,
1960	slots_per_op);
1961	if (sw_desc) {
1962	group_start = sw_desc->group_head;
1963	ppc440spe_desc_init_interrupt(desc: group_start, chan: ppc440spe_chan);
1964	group_start->unmap_len = 0;
1965	sw_desc->async_tx.flags = flags;
1966	}
1967	spin_unlock_bh(lock: &ppc440spe_chan->lock);
1968
1969	return sw_desc ? &sw_desc->async_tx : NULL;
1970	}
1971
1972	/**
1973	* ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
1974	*/
1975	static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
1976	struct dma_chan *chan, dma_addr_t dma_dest,
1977	dma_addr_t dma_src, size_t len, unsigned long flags)
1978	{
1979	struct ppc440spe_adma_chan *ppc440spe_chan;
1980	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1981	int slot_cnt, slots_per_op;
1982
1983	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1984
1985	if (unlikely(!len))
1986	return NULL;
1987
1988	BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT);
1989
1990	spin_lock_bh(lock: &ppc440spe_chan->lock);
1991
1992	dev_dbg(ppc440spe_chan->device->common.dev,
1993	"ppc440spe adma%d: %s len: %u int_en %d\n",
1994	ppc440spe_chan->device->id, __func__, len,
1995	flags & DMA_PREP_INTERRUPT ? 1 : 0);
1996	slot_cnt = slots_per_op = 1;
1997	sw_desc = ppc440spe_adma_alloc_slots(chan: ppc440spe_chan, num_slots: slot_cnt,
1998	slots_per_op);
1999	if (sw_desc) {
2000	group_start = sw_desc->group_head;
2001	ppc440spe_desc_init_memcpy(desc: group_start, flags);
2002	ppc440spe_adma_set_dest(tx: group_start, addr: dma_dest, index: 0);
2003	ppc440spe_adma_memcpy_xor_set_src(tx: group_start, addr: dma_src, index: 0);
2004	ppc440spe_desc_set_byte_count(desc: group_start, chan: ppc440spe_chan, byte_count: len);
2005	sw_desc->unmap_len = len;
2006	sw_desc->async_tx.flags = flags;
2007	}
2008	spin_unlock_bh(lock: &ppc440spe_chan->lock);
2009
2010	return sw_desc ? &sw_desc->async_tx : NULL;
2011	}
2012
2013	/**
2014	* ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2015	*/
2016	static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2017	struct dma_chan *chan, dma_addr_t dma_dest,
2018	dma_addr_t *dma_src, u32 src_cnt, size_t len,
2019	unsigned long flags)
2020	{
2021	struct ppc440spe_adma_chan *ppc440spe_chan;
2022	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2023	int slot_cnt, slots_per_op;
2024
2025	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2026
2027	ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2028	dma_dest, dma_src, src_cnt));
2029	if (unlikely(!len))
2030	return NULL;
2031	BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2032
2033	dev_dbg(ppc440spe_chan->device->common.dev,
2034	"ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2035	ppc440spe_chan->device->id, __func__, src_cnt, len,
2036	flags & DMA_PREP_INTERRUPT ? 1 : 0);
2037
2038	spin_lock_bh(lock: &ppc440spe_chan->lock);
2039	slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, slots_per_op: &slots_per_op);
2040	sw_desc = ppc440spe_adma_alloc_slots(chan: ppc440spe_chan, num_slots: slot_cnt,
2041	slots_per_op);
2042	if (sw_desc) {
2043	group_start = sw_desc->group_head;
2044	ppc440spe_desc_init_xor(desc: group_start, src_cnt, flags);
2045	ppc440spe_adma_set_dest(tx: group_start, addr: dma_dest, index: 0);
2046	while (src_cnt--)
2047	ppc440spe_adma_memcpy_xor_set_src(tx: group_start,
2048	addr: dma_src[src_cnt], index: src_cnt);
2049	ppc440spe_desc_set_byte_count(desc: group_start, chan: ppc440spe_chan, byte_count: len);
2050	sw_desc->unmap_len = len;
2051	sw_desc->async_tx.flags = flags;
2052	}
2053	spin_unlock_bh(lock: &ppc440spe_chan->lock);
2054
2055	return sw_desc ? &sw_desc->async_tx : NULL;
2056	}
2057
2058	static inline void
2059	ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2060	int src_cnt);
2061	static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2062
2063	/**
2064	* ppc440spe_adma_init_dma2rxor_slot -
2065	*/
2066	static void ppc440spe_adma_init_dma2rxor_slot(
2067	struct ppc440spe_adma_desc_slot *desc,
2068	dma_addr_t *src, int src_cnt)
2069	{
2070	int i;
2071
2072	/* initialize CDB */
2073	for (i = 0; i < src_cnt; i++) {
2074	ppc440spe_adma_dma2rxor_prep_src(desc, cursor: &desc->rxor_cursor, index: i,
2075	src_cnt: desc->src_cnt, addr: (u32)src[i]);
2076	}
2077	}
2078
2079	/**
2080	* ppc440spe_dma01_prep_mult -
2081	* for Q operation where destination is also the source
2082	*/
2083	static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2084	struct ppc440spe_adma_chan *ppc440spe_chan,
2085	dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2086	const unsigned char *scf, size_t len, unsigned long flags)
2087	{
2088	struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2089	unsigned long op = 0;
2090	int slot_cnt;
2091
2092	set_bit(PPC440SPE_DESC_WXOR, addr: &op);
2093	slot_cnt = 2;
2094
2095	spin_lock_bh(lock: &ppc440spe_chan->lock);
2096
2097	/* use WXOR, each descriptor occupies one slot */
2098	sw_desc = ppc440spe_adma_alloc_slots(chan: ppc440spe_chan, num_slots: slot_cnt, slots_per_op: 1);
2099	if (sw_desc) {
2100	struct ppc440spe_adma_chan *chan;
2101	struct ppc440spe_adma_desc_slot *iter;
2102	struct dma_cdb *hw_desc;
2103
2104	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2105	set_bits(op, &sw_desc->flags);
2106	sw_desc->src_cnt = src_cnt;
2107	sw_desc->dst_cnt = dst_cnt;
2108	/* First descriptor, zero data in the destination and copy it
2109	* to q page using MULTICAST transfer.
2110	*/
2111	iter = list_first_entry(&sw_desc->group_list,
2112	struct ppc440spe_adma_desc_slot,
2113	chain_node);
2114	memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2115	/* set 'next' pointer */
2116	iter->hw_next = list_entry(iter->chain_node.next,
2117	struct ppc440spe_adma_desc_slot,
2118	chain_node);
2119	clear_bit(PPC440SPE_DESC_INT, addr: &iter->flags);
2120	hw_desc = iter->hw_desc;
2121	hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2122
2123	ppc440spe_desc_set_dest_addr(desc: iter, chan,
2124	DMA_CUED_XOR_BASE, addrl: dst[0], dst_idx: 0);
2125	ppc440spe_desc_set_dest_addr(desc: iter, chan, addrh: 0, addrl: dst[1], dst_idx: 1);
2126	ppc440spe_desc_set_src_addr(desc: iter, chan, src_idx: 0, DMA_CUED_XOR_HB,
2127	addrl: src[0]);
2128	ppc440spe_desc_set_byte_count(desc: iter, chan: ppc440spe_chan, byte_count: len);
2129	iter->unmap_len = len;
2130
2131	/*
2132	* Second descriptor, multiply data from the q page
2133	* and store the result in real destination.
2134	*/
2135	iter = list_first_entry(&iter->chain_node,
2136	struct ppc440spe_adma_desc_slot,
2137	chain_node);
2138	memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2139	iter->hw_next = NULL;
2140	if (flags & DMA_PREP_INTERRUPT)
2141	set_bit(PPC440SPE_DESC_INT, addr: &iter->flags);
2142	else
2143	clear_bit(PPC440SPE_DESC_INT, addr: &iter->flags);
2144
2145	hw_desc = iter->hw_desc;
2146	hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2147	ppc440spe_desc_set_src_addr(desc: iter, chan, src_idx: 0,
2148	DMA_CUED_XOR_HB, addrl: dst[1]);
2149	ppc440spe_desc_set_dest_addr(desc: iter, chan,
2150	DMA_CUED_XOR_BASE, addrl: dst[0], dst_idx: 0);
2151
2152	ppc440spe_desc_set_src_mult(desc: iter, chan, DMA_CUED_MULT1_OFF,
2153	DMA_CDB_SG_DST1, mult_value: scf[0]);
2154	ppc440spe_desc_set_byte_count(desc: iter, chan: ppc440spe_chan, byte_count: len);
2155	iter->unmap_len = len;
2156	sw_desc->async_tx.flags = flags;
2157	}
2158
2159	spin_unlock_bh(lock: &ppc440spe_chan->lock);
2160
2161	return sw_desc;
2162	}
2163
2164	/**
2165	* ppc440spe_dma01_prep_sum_product -
2166	* Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2167	* the source.
2168	*/
2169	static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2170	struct ppc440spe_adma_chan *ppc440spe_chan,
2171	dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2172	const unsigned char *scf, size_t len, unsigned long flags)
2173	{
2174	struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2175	unsigned long op = 0;
2176	int slot_cnt;
2177
2178	set_bit(PPC440SPE_DESC_WXOR, addr: &op);
2179	slot_cnt = 3;
2180
2181	spin_lock_bh(lock: &ppc440spe_chan->lock);
2182
2183	/* WXOR, each descriptor occupies one slot */
2184	sw_desc = ppc440spe_adma_alloc_slots(chan: ppc440spe_chan, num_slots: slot_cnt, slots_per_op: 1);
2185	if (sw_desc) {
2186	struct ppc440spe_adma_chan *chan;
2187	struct ppc440spe_adma_desc_slot *iter;
2188	struct dma_cdb *hw_desc;
2189
2190	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2191	set_bits(op, &sw_desc->flags);
2192	sw_desc->src_cnt = src_cnt;
2193	sw_desc->dst_cnt = 1;
2194	/* 1st descriptor, src[1] data to q page and zero destination */
2195	iter = list_first_entry(&sw_desc->group_list,
2196	struct ppc440spe_adma_desc_slot,
2197	chain_node);
2198	memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2199	iter->hw_next = list_entry(iter->chain_node.next,
2200	struct ppc440spe_adma_desc_slot,
2201	chain_node);
2202	clear_bit(PPC440SPE_DESC_INT, addr: &iter->flags);
2203	hw_desc = iter->hw_desc;
2204	hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2205
2206	ppc440spe_desc_set_dest_addr(desc: iter, chan, DMA_CUED_XOR_BASE,
2207	addrl: *dst, dst_idx: 0);
2208	ppc440spe_desc_set_dest_addr(desc: iter, chan, addrh: 0,
2209	addrl: ppc440spe_chan->qdest, dst_idx: 1);
2210	ppc440spe_desc_set_src_addr(desc: iter, chan, src_idx: 0, DMA_CUED_XOR_HB,
2211	addrl: src[1]);
2212	ppc440spe_desc_set_byte_count(desc: iter, chan: ppc440spe_chan, byte_count: len);
2213	iter->unmap_len = len;
2214
2215	/* 2nd descriptor, multiply src[1] data and store the
2216	* result in destination */
2217	iter = list_first_entry(&iter->chain_node,
2218	struct ppc440spe_adma_desc_slot,
2219	chain_node);
2220	memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2221	/* set 'next' pointer */
2222	iter->hw_next = list_entry(iter->chain_node.next,
2223	struct ppc440spe_adma_desc_slot,
2224	chain_node);
2225	if (flags & DMA_PREP_INTERRUPT)
2226	set_bit(PPC440SPE_DESC_INT, addr: &iter->flags);
2227	else
2228	clear_bit(PPC440SPE_DESC_INT, addr: &iter->flags);
2229
2230	hw_desc = iter->hw_desc;
2231	hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2232	ppc440spe_desc_set_src_addr(desc: iter, chan, src_idx: 0, DMA_CUED_XOR_HB,
2233	addrl: ppc440spe_chan->qdest);
2234	ppc440spe_desc_set_dest_addr(desc: iter, chan, DMA_CUED_XOR_BASE,
2235	addrl: *dst, dst_idx: 0);
2236	ppc440spe_desc_set_src_mult(desc: iter, chan, DMA_CUED_MULT1_OFF,
2237	DMA_CDB_SG_DST1, mult_value: scf[1]);
2238	ppc440spe_desc_set_byte_count(desc: iter, chan: ppc440spe_chan, byte_count: len);
2239	iter->unmap_len = len;
2240
2241	/*
2242	* 3rd descriptor, multiply src[0] data and xor it
2243	* with destination
2244	*/
2245	iter = list_first_entry(&iter->chain_node,
2246	struct ppc440spe_adma_desc_slot,
2247	chain_node);
2248	memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2249	iter->hw_next = NULL;
2250	if (flags & DMA_PREP_INTERRUPT)
2251	set_bit(PPC440SPE_DESC_INT, addr: &iter->flags);
2252	else
2253	clear_bit(PPC440SPE_DESC_INT, addr: &iter->flags);
2254
2255	hw_desc = iter->hw_desc;
2256	hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2257	ppc440spe_desc_set_src_addr(desc: iter, chan, src_idx: 0, DMA_CUED_XOR_HB,
2258	addrl: src[0]);
2259	ppc440spe_desc_set_dest_addr(desc: iter, chan, DMA_CUED_XOR_BASE,
2260	addrl: *dst, dst_idx: 0);
2261	ppc440spe_desc_set_src_mult(desc: iter, chan, DMA_CUED_MULT1_OFF,
2262	DMA_CDB_SG_DST1, mult_value: scf[0]);
2263	ppc440spe_desc_set_byte_count(desc: iter, chan: ppc440spe_chan, byte_count: len);
2264	iter->unmap_len = len;
2265	sw_desc->async_tx.flags = flags;
2266	}
2267
2268	spin_unlock_bh(lock: &ppc440spe_chan->lock);
2269
2270	return sw_desc;
2271	}
2272
2273	static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2274	struct ppc440spe_adma_chan *ppc440spe_chan,
2275	dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2276	const unsigned char *scf, size_t len, unsigned long flags)
2277	{
2278	int slot_cnt;
2279	struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2280	unsigned long op = 0;
2281	unsigned char mult = 1;
2282
2283	pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2284	__func__, dst_cnt, src_cnt, len);
2285	/* select operations WXOR/RXOR depending on the
2286	* source addresses of operators and the number
2287	* of destinations (RXOR support only Q-parity calculations)
2288	*/
2289	set_bit(PPC440SPE_DESC_WXOR, addr: &op);
2290	if (!test_and_set_bit(PPC440SPE_RXOR_RUN, addr: &ppc440spe_rxor_state)) {
2291	/* no active RXOR;
2292	* do RXOR if:
2293	* - there are more than 1 source,
2294	* - len is aligned on 512-byte boundary,
2295	* - source addresses fit to one of 4 possible regions.
2296	*/
2297	if (src_cnt > 1 &&
2298	!(len & MQ0_CF2H_RXOR_BS_MASK) &&
2299	(src[0] + len) == src[1]) {
2300	/* may do RXOR R1 R2 */
2301	set_bit(PPC440SPE_DESC_RXOR, addr: &op);
2302	if (src_cnt != 2) {
2303	/* may try to enhance region of RXOR */
2304	if ((src[1] + len) == src[2]) {
2305	/* do RXOR R1 R2 R3 */
2306	set_bit(PPC440SPE_DESC_RXOR123,
2307	addr: &op);
2308	} else if ((src[1] + len * 2) == src[2]) {
2309	/* do RXOR R1 R2 R4 */
2310	set_bit(PPC440SPE_DESC_RXOR124, addr: &op);
2311	} else if ((src[1] + len * 3) == src[2]) {
2312	/* do RXOR R1 R2 R5 */
2313	set_bit(PPC440SPE_DESC_RXOR125,
2314	addr: &op);
2315	} else {
2316	/* do RXOR R1 R2 */
2317	set_bit(PPC440SPE_DESC_RXOR12,
2318	addr: &op);
2319	}
2320	} else {
2321	/* do RXOR R1 R2 */
2322	set_bit(PPC440SPE_DESC_RXOR12, addr: &op);
2323	}
2324	}
2325
2326	if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2327	/* can not do this operation with RXOR */
2328	clear_bit(PPC440SPE_RXOR_RUN,
2329	addr: &ppc440spe_rxor_state);
2330	} else {
2331	/* can do; set block size right now */
2332	ppc440spe_desc_set_rxor_block_size(byte_count: len);
2333	}
2334	}
2335
2336	/* Number of necessary slots depends on operation type selected */
2337	if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2338	/* This is a WXOR only chain. Need descriptors for each
2339	* source to GF-XOR them with WXOR, and need descriptors
2340	* for each destination to zero them with WXOR
2341	*/
2342	slot_cnt = src_cnt;
2343
2344	if (flags & DMA_PREP_ZERO_P) {
2345	slot_cnt++;
2346	set_bit(PPC440SPE_ZERO_P, addr: &op);
2347	}
2348	if (flags & DMA_PREP_ZERO_Q) {
2349	slot_cnt++;
2350	set_bit(PPC440SPE_ZERO_Q, addr: &op);
2351	}
2352	} else {
2353	/* Need 1/2 descriptor for RXOR operation, and
2354	* need (src_cnt - (2 or 3)) for WXOR of sources
2355	* remained (if any)
2356	*/
2357	slot_cnt = dst_cnt;
2358
2359	if (flags & DMA_PREP_ZERO_P)
2360	set_bit(PPC440SPE_ZERO_P, addr: &op);
2361	if (flags & DMA_PREP_ZERO_Q)
2362	set_bit(PPC440SPE_ZERO_Q, addr: &op);
2363
2364	if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2365	slot_cnt += src_cnt - 2;
2366	else
2367	slot_cnt += src_cnt - 3;
2368
2369	/* Thus we have either RXOR only chain or
2370	* mixed RXOR/WXOR
2371	*/
2372	if (slot_cnt == dst_cnt)
2373	/* RXOR only chain */
2374	clear_bit(PPC440SPE_DESC_WXOR, addr: &op);
2375	}
2376
2377	spin_lock_bh(lock: &ppc440spe_chan->lock);
2378	/* for both RXOR/WXOR each descriptor occupies one slot */
2379	sw_desc = ppc440spe_adma_alloc_slots(chan: ppc440spe_chan, num_slots: slot_cnt, slots_per_op: 1);
2380	if (sw_desc) {
2381	ppc440spe_desc_init_dma01pq(desc: sw_desc, dst_cnt, src_cnt,
2382	flags, op);
2383
2384	/* setup dst/src/mult */
2385	pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2386	__func__, dst[0], dst[1]);
2387	ppc440spe_adma_pq_set_dest(tx: sw_desc, paddr: dst, flags);
2388	while (src_cnt--) {
2389	ppc440spe_adma_pq_set_src(tx: sw_desc, addr: src[src_cnt],
2390	index: src_cnt);
2391
2392	/* NOTE: "Multi = 0 is equivalent to = 1" as it
2393	* stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2394	* doesn't work for RXOR with DMA0/1! Instead, multi=0
2395	* leads to zeroing source data after RXOR.
2396	* So, for P case set-up mult=1 explicitly.
2397	*/
2398	if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2399	mult = scf[src_cnt];
2400	ppc440spe_adma_pq_set_src_mult(tx: sw_desc,
2401	mult, index: src_cnt, dst_pos: dst_cnt - 1);
2402	}
2403
2404	/* Setup byte count foreach slot just allocated */
2405	sw_desc->async_tx.flags = flags;
2406	list_for_each_entry(iter, &sw_desc->group_list,
2407	chain_node) {
2408	ppc440spe_desc_set_byte_count(desc: iter,
2409	chan: ppc440spe_chan, byte_count: len);
2410	iter->unmap_len = len;
2411	}
2412	}
2413	spin_unlock_bh(lock: &ppc440spe_chan->lock);
2414
2415	return sw_desc;
2416	}
2417
2418	static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2419	struct ppc440spe_adma_chan *ppc440spe_chan,
2420	dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2421	const unsigned char *scf, size_t len, unsigned long flags)
2422	{
2423	int slot_cnt, descs_per_op;
2424	struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2425	unsigned long op = 0;
2426	unsigned char mult = 1;
2427
2428	BUG_ON(!dst_cnt);
2429	/*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2430	__func__, dst_cnt, src_cnt, len);*/
2431
2432	spin_lock_bh(lock: &ppc440spe_chan->lock);
2433	descs_per_op = ppc440spe_dma2_pq_slot_count(srcs: src, src_cnt, len);
2434	if (descs_per_op < 0) {
2435	spin_unlock_bh(lock: &ppc440spe_chan->lock);
2436	return NULL;
2437	}
2438
2439	/* depending on number of sources we have 1 or 2 RXOR chains */
2440	slot_cnt = descs_per_op * dst_cnt;
2441
2442	sw_desc = ppc440spe_adma_alloc_slots(chan: ppc440spe_chan, num_slots: slot_cnt, slots_per_op: 1);
2443	if (sw_desc) {
2444	op = slot_cnt;
2445	sw_desc->async_tx.flags = flags;
2446	list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2447	ppc440spe_desc_init_dma2pq(desc: iter, dst_cnt, src_cnt,
2448	flags: --op ? 0 : flags);
2449	ppc440spe_desc_set_byte_count(desc: iter, chan: ppc440spe_chan,
2450	byte_count: len);
2451	iter->unmap_len = len;
2452
2453	ppc440spe_init_rxor_cursor(cursor: &(iter->rxor_cursor));
2454	iter->rxor_cursor.len = len;
2455	iter->descs_per_op = descs_per_op;
2456	}
2457	op = 0;
2458	list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2459	op++;
2460	if (op % descs_per_op == 0)
2461	ppc440spe_adma_init_dma2rxor_slot(desc: iter, src,
2462	src_cnt);
2463	if (likely(!list_is_last(&iter->chain_node,
2464	&sw_desc->group_list))) {
2465	/* set 'next' pointer */
2466	iter->hw_next =
2467	list_entry(iter->chain_node.next,
2468	struct ppc440spe_adma_desc_slot,
2469	chain_node);
2470	ppc440spe_xor_set_link(prev_desc: iter, next_desc: iter->hw_next);
2471	} else {
2472	/* this is the last descriptor. */
2473	iter->hw_next = NULL;
2474	}
2475	}
2476
2477	/* fixup head descriptor */
2478	sw_desc->dst_cnt = dst_cnt;
2479	if (flags & DMA_PREP_ZERO_P)
2480	set_bit(PPC440SPE_ZERO_P, addr: &sw_desc->flags);
2481	if (flags & DMA_PREP_ZERO_Q)
2482	set_bit(PPC440SPE_ZERO_Q, addr: &sw_desc->flags);
2483
2484	/* setup dst/src/mult */
2485	ppc440spe_adma_pq_set_dest(tx: sw_desc, paddr: dst, flags);
2486
2487	while (src_cnt--) {
2488	/* handle descriptors (if dst_cnt == 2) inside
2489	* the ppc440spe_adma_pq_set_srcxxx() functions
2490	*/
2491	ppc440spe_adma_pq_set_src(tx: sw_desc, addr: src[src_cnt],
2492	index: src_cnt);
2493	if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2494	mult = scf[src_cnt];
2495	ppc440spe_adma_pq_set_src_mult(tx: sw_desc,
2496	mult, index: src_cnt, dst_pos: dst_cnt - 1);
2497	}
2498	}
2499	spin_unlock_bh(lock: &ppc440spe_chan->lock);
2500	ppc440spe_desc_set_rxor_block_size(byte_count: len);
2501	return sw_desc;
2502	}
2503
2504	/**
2505	* ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2506	*/
2507	static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2508	struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2509	unsigned int src_cnt, const unsigned char *scf,
2510	size_t len, unsigned long flags)
2511	{
2512	struct ppc440spe_adma_chan *ppc440spe_chan;
2513	struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2514	int dst_cnt = 0;
2515
2516	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2517
2518	ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2519	dst, src, src_cnt));
2520	BUG_ON(!len);
2521	BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2522	BUG_ON(!src_cnt);
2523
2524	if (src_cnt == 1 && dst[1] == src[0]) {
2525	dma_addr_t dest[2];
2526
2527	/* dst[1] is real destination (Q) */
2528	dest[0] = dst[1];
2529	/* this is the page to multicast source data to */
2530	dest[1] = ppc440spe_chan->qdest;
2531	sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2532	dst: dest, dst_cnt: 2, src, src_cnt, scf, len, flags);
2533	return sw_desc ? &sw_desc->async_tx : NULL;
2534	}
2535
2536	if (src_cnt == 2 && dst[1] == src[1]) {
2537	sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2538	dst: &dst[1], src, src_cnt: 2, scf, len, flags);
2539	return sw_desc ? &sw_desc->async_tx : NULL;
2540	}
2541
2542	if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2543	BUG_ON(!dst[0]);
2544	dst_cnt++;
2545	flags |= DMA_PREP_ZERO_P;
2546	}
2547
2548	if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2549	BUG_ON(!dst[1]);
2550	dst_cnt++;
2551	flags |= DMA_PREP_ZERO_Q;
2552	}
2553
2554	BUG_ON(!dst_cnt);
2555
2556	dev_dbg(ppc440spe_chan->device->common.dev,
2557	"ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2558	ppc440spe_chan->device->id, __func__, src_cnt, len,
2559	flags & DMA_PREP_INTERRUPT ? 1 : 0);
2560
2561	switch (ppc440spe_chan->device->id) {
2562	case PPC440SPE_DMA0_ID:
2563	case PPC440SPE_DMA1_ID:
2564	sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2565	dst, dst_cnt, src, src_cnt, scf,
2566	len, flags);
2567	break;
2568
2569	case PPC440SPE_XOR_ID:
2570	sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2571	dst, dst_cnt, src, src_cnt, scf,
2572	len, flags);
2573	break;
2574	}
2575
2576	return sw_desc ? &sw_desc->async_tx : NULL;
2577	}
2578
2579	/**
2580	* ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2581	* a PQ_ZERO_SUM operation
2582	*/
2583	static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2584	struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2585	unsigned int src_cnt, const unsigned char *scf, size_t len,
2586	enum sum_check_flags *pqres, unsigned long flags)
2587	{
2588	struct ppc440spe_adma_chan *ppc440spe_chan;
2589	struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2590	dma_addr_t pdest, qdest;
2591	int slot_cnt, slots_per_op, idst, dst_cnt;
2592
2593	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2594
2595	if (flags & DMA_PREP_PQ_DISABLE_P)
2596	pdest = 0;
2597	else
2598	pdest = pq[0];
2599
2600	if (flags & DMA_PREP_PQ_DISABLE_Q)
2601	qdest = 0;
2602	else
2603	qdest = pq[1];
2604
2605	ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2606	src, src_cnt, scf));
2607
2608	/* Always use WXOR for P/Q calculations (two destinations).
2609	* Need 1 or 2 extra slots to verify results are zero.
2610	*/
2611	idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2612
2613	/* One additional slot per destination to clone P/Q
2614	* before calculation (we have to preserve destinations).
2615	*/
2616	slot_cnt = src_cnt + dst_cnt * 2;
2617	slots_per_op = 1;
2618
2619	spin_lock_bh(lock: &ppc440spe_chan->lock);
2620	sw_desc = ppc440spe_adma_alloc_slots(chan: ppc440spe_chan, num_slots: slot_cnt,
2621	slots_per_op);
2622	if (sw_desc) {
2623	ppc440spe_desc_init_dma01pqzero_sum(desc: sw_desc, dst_cnt, src_cnt);
2624
2625	/* Setup byte count for each slot just allocated */
2626	sw_desc->async_tx.flags = flags;
2627	list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2628	ppc440spe_desc_set_byte_count(desc: iter, chan: ppc440spe_chan,
2629	byte_count: len);
2630	iter->unmap_len = len;
2631	}
2632
2633	if (pdest) {
2634	struct dma_cdb *hw_desc;
2635	struct ppc440spe_adma_chan *chan;
2636
2637	iter = sw_desc->group_head;
2638	chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2639	memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2640	iter->hw_next = list_entry(iter->chain_node.next,
2641	struct ppc440spe_adma_desc_slot,
2642	chain_node);
2643	hw_desc = iter->hw_desc;
2644	hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2645	iter->src_cnt = 0;
2646	iter->dst_cnt = 0;
2647	ppc440spe_desc_set_dest_addr(desc: iter, chan, addrh: 0,
2648	addrl: ppc440spe_chan->pdest, dst_idx: 0);
2649	ppc440spe_desc_set_src_addr(desc: iter, chan, src_idx: 0, addrh: 0, addrl: pdest);
2650	ppc440spe_desc_set_byte_count(desc: iter, chan: ppc440spe_chan,
2651	byte_count: len);
2652	iter->unmap_len = 0;
2653	/* override pdest to preserve original P */
2654	pdest = ppc440spe_chan->pdest;
2655	}
2656	if (qdest) {
2657	struct dma_cdb *hw_desc;
2658	struct ppc440spe_adma_chan *chan;
2659
2660	iter = list_first_entry(&sw_desc->group_list,
2661	struct ppc440spe_adma_desc_slot,
2662	chain_node);
2663	chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2664
2665	if (pdest) {
2666	iter = list_entry(iter->chain_node.next,
2667	struct ppc440spe_adma_desc_slot,
2668	chain_node);
2669	}
2670
2671	memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2672	iter->hw_next = list_entry(iter->chain_node.next,
2673	struct ppc440spe_adma_desc_slot,
2674	chain_node);
2675	hw_desc = iter->hw_desc;
2676	hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2677	iter->src_cnt = 0;
2678	iter->dst_cnt = 0;
2679	ppc440spe_desc_set_dest_addr(desc: iter, chan, addrh: 0,
2680	addrl: ppc440spe_chan->qdest, dst_idx: 0);
2681	ppc440spe_desc_set_src_addr(desc: iter, chan, src_idx: 0, addrh: 0, addrl: qdest);
2682	ppc440spe_desc_set_byte_count(desc: iter, chan: ppc440spe_chan,
2683	byte_count: len);
2684	iter->unmap_len = 0;
2685	/* override qdest to preserve original Q */
2686	qdest = ppc440spe_chan->qdest;
2687	}
2688
2689	/* Setup destinations for P/Q ops */
2690	ppc440spe_adma_pqzero_sum_set_dest(tx: sw_desc, paddr: pdest, qaddr: qdest);
2691
2692	/* Setup zero QWORDs into DCHECK CDBs */
2693	idst = dst_cnt;
2694	list_for_each_entry_reverse(iter, &sw_desc->group_list,
2695	chain_node) {
2696	/*
2697	* The last CDB corresponds to Q-parity check,
2698	* the one before last CDB corresponds
2699	* P-parity check
2700	*/
2701	if (idst == DMA_DEST_MAX_NUM) {
2702	if (idst == dst_cnt) {
2703	set_bit(PPC440SPE_DESC_QCHECK,
2704	addr: &iter->flags);
2705	} else {
2706	set_bit(PPC440SPE_DESC_PCHECK,
2707	addr: &iter->flags);
2708	}
2709	} else {
2710	if (qdest) {
2711	set_bit(PPC440SPE_DESC_QCHECK,
2712	addr: &iter->flags);
2713	} else {
2714	set_bit(PPC440SPE_DESC_PCHECK,
2715	addr: &iter->flags);
2716	}
2717	}
2718	iter->xor_check_result = pqres;
2719
2720	/*
2721	* set it to zero, if check fail then result will
2722	* be updated
2723	*/
2724	*iter->xor_check_result = 0;
2725	ppc440spe_desc_set_dcheck(desc: iter, chan: ppc440spe_chan,
2726	qword: ppc440spe_qword);
2727
2728	if (!(--dst_cnt))
2729	break;
2730	}
2731
2732	/* Setup sources and mults for P/Q ops */
2733	list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
2734	chain_node) {
2735	struct ppc440spe_adma_chan *chan;
2736	u32 mult_dst;
2737
2738	chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2739	ppc440spe_desc_set_src_addr(desc: iter, chan, src_idx: 0,
2740	DMA_CUED_XOR_HB,
2741	addrl: src[src_cnt - 1]);
2742	if (qdest) {
2743	mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
2744	DMA_CDB_SG_DST1;
2745	ppc440spe_desc_set_src_mult(desc: iter, chan,
2746	DMA_CUED_MULT1_OFF,
2747	sg_index: mult_dst,
2748	mult_value: scf[src_cnt - 1]);
2749	}
2750	if (!(--src_cnt))
2751	break;
2752	}
2753	}
2754	spin_unlock_bh(lock: &ppc440spe_chan->lock);
2755	return sw_desc ? &sw_desc->async_tx : NULL;
2756	}
2757
2758	/**
2759	* ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
2760	* XOR ZERO_SUM operation
2761	*/
2762	static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
2763	struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
2764	size_t len, enum sum_check_flags *result, unsigned long flags)
2765	{
2766	struct dma_async_tx_descriptor *tx;
2767	dma_addr_t pq[2];
2768
2769	/* validate P, disable Q */
2770	pq[0] = src[0];
2771	pq[1] = 0;
2772	flags |= DMA_PREP_PQ_DISABLE_Q;
2773
2774	tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, src: &src[1],
2775	src_cnt: src_cnt - 1, scf: 0, len,
2776	pqres: result, flags);
2777	return tx;
2778	}
2779
2780	/**
2781	* ppc440spe_adma_set_dest - set destination address into descriptor
2782	*/
2783	static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2784	dma_addr_t addr, int index)
2785	{
2786	struct ppc440spe_adma_chan *chan;
2787
2788	BUG_ON(index >= sw_desc->dst_cnt);
2789
2790	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2791
2792	switch (chan->device->id) {
2793	case PPC440SPE_DMA0_ID:
2794	case PPC440SPE_DMA1_ID:
2795	/* to do: support transfers lengths >
2796	* PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
2797	*/
2798	ppc440spe_desc_set_dest_addr(desc: sw_desc->group_head,
2799	chan, addrh: 0, addrl: addr, dst_idx: index);
2800	break;
2801	case PPC440SPE_XOR_ID:
2802	sw_desc = ppc440spe_get_group_entry(tdesc: sw_desc, entry_idx: index);
2803	ppc440spe_desc_set_dest_addr(desc: sw_desc,
2804	chan, addrh: 0, addrl: addr, dst_idx: index);
2805	break;
2806	}
2807	}
2808
2809	static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
2810	struct ppc440spe_adma_chan *chan, dma_addr_t addr)
2811	{
2812	/* To clear destinations update the descriptor
2813	* (P or Q depending on index) as follows:
2814	* addr is destination (0 corresponds to SG2):
2815	*/
2816	ppc440spe_desc_set_dest_addr(desc: iter, chan, DMA_CUED_XOR_BASE, addrl: addr, dst_idx: 0);
2817
2818	/* ... and the addr is source: */
2819	ppc440spe_desc_set_src_addr(desc: iter, chan, src_idx: 0, DMA_CUED_XOR_HB, addrl: addr);
2820
2821	/* addr is always SG2 then the mult is always DST1 */
2822	ppc440spe_desc_set_src_mult(desc: iter, chan, DMA_CUED_MULT1_OFF,
2823	DMA_CDB_SG_DST1, mult_value: 1);
2824	}
2825
2826	/**
2827	* ppc440spe_adma_pq_set_dest - set destination address into descriptor
2828	* for the PQXOR operation
2829	*/
2830	static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2831	dma_addr_t *addrs, unsigned long flags)
2832	{
2833	struct ppc440spe_adma_desc_slot *iter;
2834	struct ppc440spe_adma_chan *chan;
2835	dma_addr_t paddr, qaddr;
2836	dma_addr_t addr = 0, ppath, qpath;
2837	int index = 0, i;
2838
2839	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2840
2841	if (flags & DMA_PREP_PQ_DISABLE_P)
2842	paddr = 0;
2843	else
2844	paddr = addrs[0];
2845
2846	if (flags & DMA_PREP_PQ_DISABLE_Q)
2847	qaddr = 0;
2848	else
2849	qaddr = addrs[1];
2850
2851	if (!paddr || !qaddr)
2852	addr = paddr ? paddr : qaddr;
2853
2854	switch (chan->device->id) {
2855	case PPC440SPE_DMA0_ID:
2856	case PPC440SPE_DMA1_ID:
2857	/* walk through the WXOR source list and set P/Q-destinations
2858	* for each slot:
2859	*/
2860	if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
2861	/* This is WXOR-only chain; may have 1/2 zero descs */
2862	if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
2863	index++;
2864	if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
2865	index++;
2866
2867	iter = ppc440spe_get_group_entry(tdesc: sw_desc, entry_idx: index);
2868	if (addr) {
2869	/* one destination */
2870	list_for_each_entry_from(iter,
2871	&sw_desc->group_list, chain_node)
2872	ppc440spe_desc_set_dest_addr(desc: iter, chan,
2873	DMA_CUED_XOR_BASE, addrl: addr, dst_idx: 0);
2874	} else {
2875	/* two destinations */
2876	list_for_each_entry_from(iter,
2877	&sw_desc->group_list, chain_node) {
2878	ppc440spe_desc_set_dest_addr(desc: iter, chan,
2879	DMA_CUED_XOR_BASE, addrl: paddr, dst_idx: 0);
2880	ppc440spe_desc_set_dest_addr(desc: iter, chan,
2881	DMA_CUED_XOR_BASE, addrl: qaddr, dst_idx: 1);
2882	}
2883	}
2884
2885	if (index) {
2886	/* To clear destinations update the descriptor
2887	* (1st,2nd, or both depending on flags)
2888	*/
2889	index = 0;
2890	if (test_bit(PPC440SPE_ZERO_P,
2891	&sw_desc->flags)) {
2892	iter = ppc440spe_get_group_entry(
2893	tdesc: sw_desc, entry_idx: index++);
2894	ppc440spe_adma_pq_zero_op(iter, chan,
2895	addr: paddr);
2896	}
2897
2898	if (test_bit(PPC440SPE_ZERO_Q,
2899	&sw_desc->flags)) {
2900	iter = ppc440spe_get_group_entry(
2901	tdesc: sw_desc, entry_idx: index++);
2902	ppc440spe_adma_pq_zero_op(iter, chan,
2903	addr: qaddr);
2904	}
2905
2906	return;
2907	}
2908	} else {
2909	/* This is RXOR-only or RXOR/WXOR mixed chain */
2910
2911	/* If we want to include destination into calculations,
2912	* then make dest addresses cued with mult=1 (XOR).
2913	*/
2914	ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2915	DMA_CUED_XOR_HB :
2916	DMA_CUED_XOR_BASE |
2917	(1 << DMA_CUED_MULT1_OFF);
2918	qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2919	DMA_CUED_XOR_HB :
2920	DMA_CUED_XOR_BASE |
2921	(1 << DMA_CUED_MULT1_OFF);
2922
2923	/* Setup destination(s) in RXOR slot(s) */
2924	iter = ppc440spe_get_group_entry(tdesc: sw_desc, entry_idx: index++);
2925	ppc440spe_desc_set_dest_addr(desc: iter, chan,
2926	addrh: paddr ? ppath : qpath,
2927	addrl: paddr ? paddr : qaddr, dst_idx: 0);
2928	if (!addr) {
2929	/* two destinations */
2930	iter = ppc440spe_get_group_entry(tdesc: sw_desc,
2931	entry_idx: index++);
2932	ppc440spe_desc_set_dest_addr(desc: iter, chan,
2933	addrh: qpath, addrl: qaddr, dst_idx: 0);
2934	}
2935
2936	if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
2937	/* Setup destination(s) in remaining WXOR
2938	* slots
2939	*/
2940	iter = ppc440spe_get_group_entry(tdesc: sw_desc,
2941	entry_idx: index);
2942	if (addr) {
2943	/* one destination */
2944	list_for_each_entry_from(iter,
2945	&sw_desc->group_list,
2946	chain_node)
2947	ppc440spe_desc_set_dest_addr(
2948	desc: iter, chan,
2949	DMA_CUED_XOR_BASE,
2950	addrl: addr, dst_idx: 0);
2951
2952	} else {
2953	/* two destinations */
2954	list_for_each_entry_from(iter,
2955	&sw_desc->group_list,
2956	chain_node) {
2957	ppc440spe_desc_set_dest_addr(
2958	desc: iter, chan,
2959	DMA_CUED_XOR_BASE,
2960	addrl: paddr, dst_idx: 0);
2961	ppc440spe_desc_set_dest_addr(
2962	desc: iter, chan,
2963	DMA_CUED_XOR_BASE,
2964	addrl: qaddr, dst_idx: 1);
2965	}
2966	}
2967	}
2968
2969	}
2970	break;
2971
2972	case PPC440SPE_XOR_ID:
2973	/* DMA2 descriptors have only 1 destination, so there are
2974	* two chains - one for each dest.
2975	* If we want to include destination into calculations,
2976	* then make dest addresses cued with mult=1 (XOR).
2977	*/
2978	ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2979	DMA_CUED_XOR_HB :
2980	DMA_CUED_XOR_BASE |
2981	(1 << DMA_CUED_MULT1_OFF);
2982
2983	qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2984	DMA_CUED_XOR_HB :
2985	DMA_CUED_XOR_BASE |
2986	(1 << DMA_CUED_MULT1_OFF);
2987
2988	iter = ppc440spe_get_group_entry(tdesc: sw_desc, entry_idx: 0);
2989	for (i = 0; i < sw_desc->descs_per_op; i++) {
2990	ppc440spe_desc_set_dest_addr(desc: iter, chan,
2991	addrh: paddr ? ppath : qpath,
2992	addrl: paddr ? paddr : qaddr, dst_idx: 0);
2993	iter = list_entry(iter->chain_node.next,
2994	struct ppc440spe_adma_desc_slot,
2995	chain_node);
2996	}
2997
2998	if (!addr) {
2999	/* Two destinations; setup Q here */
3000	iter = ppc440spe_get_group_entry(tdesc: sw_desc,
3001	entry_idx: sw_desc->descs_per_op);
3002	for (i = 0; i < sw_desc->descs_per_op; i++) {
3003	ppc440spe_desc_set_dest_addr(desc: iter,
3004	chan, addrh: qpath, addrl: qaddr, dst_idx: 0);
3005	iter = list_entry(iter->chain_node.next,
3006	struct ppc440spe_adma_desc_slot,
3007	chain_node);
3008	}
3009	}
3010
3011	break;
3012	}
3013	}
3014
3015	/**
3016	* ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3017	* for the PQ_ZERO_SUM operation
3018	*/
3019	static void ppc440spe_adma_pqzero_sum_set_dest(
3020	struct ppc440spe_adma_desc_slot *sw_desc,
3021	dma_addr_t paddr, dma_addr_t qaddr)
3022	{
3023	struct ppc440spe_adma_desc_slot *iter, *end;
3024	struct ppc440spe_adma_chan *chan;
3025	dma_addr_t addr = 0;
3026	int idx;
3027
3028	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3029
3030	/* walk through the WXOR source list and set P/Q-destinations
3031	* for each slot
3032	*/
3033	idx = (paddr && qaddr) ? 2 : 1;
3034	/* set end */
3035	list_for_each_entry_reverse(end, &sw_desc->group_list,
3036	chain_node) {
3037	if (!(--idx))
3038	break;
3039	}
3040	/* set start */
3041	idx = (paddr && qaddr) ? 2 : 1;
3042	iter = ppc440spe_get_group_entry(tdesc: sw_desc, entry_idx: idx);
3043
3044	if (paddr && qaddr) {
3045	/* two destinations */
3046	list_for_each_entry_from(iter, &sw_desc->group_list,
3047	chain_node) {
3048	if (unlikely(iter == end))
3049	break;
3050	ppc440spe_desc_set_dest_addr(desc: iter, chan,
3051	DMA_CUED_XOR_BASE, addrl: paddr, dst_idx: 0);
3052	ppc440spe_desc_set_dest_addr(desc: iter, chan,
3053	DMA_CUED_XOR_BASE, addrl: qaddr, dst_idx: 1);
3054	}
3055	} else {
3056	/* one destination */
3057	addr = paddr ? paddr : qaddr;
3058	list_for_each_entry_from(iter, &sw_desc->group_list,
3059	chain_node) {
3060	if (unlikely(iter == end))
3061	break;
3062	ppc440spe_desc_set_dest_addr(desc: iter, chan,
3063	DMA_CUED_XOR_BASE, addrl: addr, dst_idx: 0);
3064	}
3065	}
3066
3067	/* The remaining descriptors are DATACHECK. These have no need in
3068	* destination. Actually, these destinations are used there
3069	* as sources for check operation. So, set addr as source.
3070	*/
3071	ppc440spe_desc_set_src_addr(desc: end, chan, src_idx: 0, addrh: 0, addrl: addr ? addr : paddr);
3072
3073	if (!addr) {
3074	end = list_entry(end->chain_node.next,
3075	struct ppc440spe_adma_desc_slot, chain_node);
3076	ppc440spe_desc_set_src_addr(desc: end, chan, src_idx: 0, addrh: 0, addrl: qaddr);
3077	}
3078	}
3079
3080	/**
3081	* ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3082	*/
3083	static inline void ppc440spe_desc_set_xor_src_cnt(
3084	struct ppc440spe_adma_desc_slot *desc,
3085	int src_cnt)
3086	{
3087	struct xor_cb *hw_desc = desc->hw_desc;
3088
3089	hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3090	hw_desc->cbc |= src_cnt;
3091	}
3092
3093	/**
3094	* ppc440spe_adma_pq_set_src - set source address into descriptor
3095	*/
3096	static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3097	dma_addr_t addr, int index)
3098	{
3099	struct ppc440spe_adma_chan *chan;
3100	dma_addr_t haddr = 0;
3101	struct ppc440spe_adma_desc_slot *iter = NULL;
3102
3103	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3104
3105	switch (chan->device->id) {
3106	case PPC440SPE_DMA0_ID:
3107	case PPC440SPE_DMA1_ID:
3108	/* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3109	*/
3110	if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3111	/* RXOR-only or RXOR/WXOR operation */
3112	int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3113	&sw_desc->flags) ? 2 : 3;
3114
3115	if (index == 0) {
3116	/* 1st slot (RXOR) */
3117	/* setup sources region (R1-2-3, R1-2-4,
3118	* or R1-2-5)
3119	*/
3120	if (test_bit(PPC440SPE_DESC_RXOR12,
3121	&sw_desc->flags))
3122	haddr = DMA_RXOR12 <<
3123	DMA_CUED_REGION_OFF;
3124	else if (test_bit(PPC440SPE_DESC_RXOR123,
3125	&sw_desc->flags))
3126	haddr = DMA_RXOR123 <<
3127	DMA_CUED_REGION_OFF;
3128	else if (test_bit(PPC440SPE_DESC_RXOR124,
3129	&sw_desc->flags))
3130	haddr = DMA_RXOR124 <<
3131	DMA_CUED_REGION_OFF;
3132	else if (test_bit(PPC440SPE_DESC_RXOR125,
3133	&sw_desc->flags))
3134	haddr = DMA_RXOR125 <<
3135	DMA_CUED_REGION_OFF;
3136	else
3137	BUG();
3138	haddr |= DMA_CUED_XOR_BASE;
3139	iter = ppc440spe_get_group_entry(tdesc: sw_desc, entry_idx: 0);
3140	} else if (index < iskip) {
3141	/* 1st slot (RXOR)
3142	* shall actually set source address only once
3143	* instead of first <iskip>
3144	*/
3145	iter = NULL;
3146	} else {
3147	/* 2nd/3d and next slots (WXOR);
3148	* skip first slot with RXOR
3149	*/
3150	haddr = DMA_CUED_XOR_HB;
3151	iter = ppc440spe_get_group_entry(tdesc: sw_desc,
3152	entry_idx: index - iskip + sw_desc->dst_cnt);
3153	}
3154	} else {
3155	int znum = 0;
3156
3157	/* WXOR-only operation; skip first slots with
3158	* zeroing destinations
3159	*/
3160	if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3161	znum++;
3162	if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3163	znum++;
3164
3165	haddr = DMA_CUED_XOR_HB;
3166	iter = ppc440spe_get_group_entry(tdesc: sw_desc,
3167	entry_idx: index + znum);
3168	}
3169
3170	if (likely(iter)) {
3171	ppc440spe_desc_set_src_addr(desc: iter, chan, src_idx: 0, addrh: haddr, addrl: addr);
3172
3173	if (!index &&
3174	test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3175	sw_desc->dst_cnt == 2) {
3176	/* if we have two destinations for RXOR, then
3177	* setup source in the second descr too
3178	*/
3179	iter = ppc440spe_get_group_entry(tdesc: sw_desc, entry_idx: 1);
3180	ppc440spe_desc_set_src_addr(desc: iter, chan, src_idx: 0,
3181	addrh: haddr, addrl: addr);
3182	}
3183	}
3184	break;
3185
3186	case PPC440SPE_XOR_ID:
3187	/* DMA2 may do Biskup */
3188	iter = sw_desc->group_head;
3189	if (iter->dst_cnt == 2) {
3190	/* both P & Q calculations required; set P src here */
3191	ppc440spe_adma_dma2rxor_set_src(desc: iter, index, addr);
3192
3193	/* this is for Q */
3194	iter = ppc440spe_get_group_entry(tdesc: sw_desc,
3195	entry_idx: sw_desc->descs_per_op);
3196	}
3197	ppc440spe_adma_dma2rxor_set_src(desc: iter, index, addr);
3198	break;
3199	}
3200	}
3201
3202	/**
3203	* ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3204	*/
3205	static void ppc440spe_adma_memcpy_xor_set_src(
3206	struct ppc440spe_adma_desc_slot *sw_desc,
3207	dma_addr_t addr, int index)
3208	{
3209	struct ppc440spe_adma_chan *chan;
3210
3211	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3212	sw_desc = sw_desc->group_head;
3213
3214	if (likely(sw_desc))
3215	ppc440spe_desc_set_src_addr(desc: sw_desc, chan, src_idx: index, addrh: 0, addrl: addr);
3216	}
3217
3218	/**
3219	* ppc440spe_adma_dma2rxor_inc_addr -
3220	*/
3221	static void ppc440spe_adma_dma2rxor_inc_addr(
3222	struct ppc440spe_adma_desc_slot *desc,
3223	struct ppc440spe_rxor *cursor, int index, int src_cnt)
3224	{
3225	cursor->addr_count++;
3226	if (index == src_cnt - 1) {
3227	ppc440spe_desc_set_xor_src_cnt(desc, src_cnt: cursor->addr_count);
3228	} else if (cursor->addr_count == XOR_MAX_OPS) {
3229	ppc440spe_desc_set_xor_src_cnt(desc, src_cnt: cursor->addr_count);
3230	cursor->addr_count = 0;
3231	cursor->desc_count++;
3232	}
3233	}
3234
3235	/**
3236	* ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3237	*/
3238	static int ppc440spe_adma_dma2rxor_prep_src(
3239	struct ppc440spe_adma_desc_slot *hdesc,
3240	struct ppc440spe_rxor *cursor, int index,
3241	int src_cnt, u32 addr)
3242	{
3243	u32 sign;
3244	struct ppc440spe_adma_desc_slot *desc = hdesc;
3245	int i;
3246
3247	for (i = 0; i < cursor->desc_count; i++) {
3248	desc = list_entry(hdesc->chain_node.next,
3249	struct ppc440spe_adma_desc_slot,
3250	chain_node);
3251	}
3252
3253	switch (cursor->state) {
3254	case 0:
3255	if (addr == cursor->addrl + cursor->len) {
3256	/* direct RXOR */
3257	cursor->state = 1;
3258	cursor->xor_count++;
3259	if (index == src_cnt-1) {
3260	ppc440spe_rxor_set_region(desc,
3261	xor_arg_no: cursor->addr_count,
3262	DMA_RXOR12 << DMA_CUED_REGION_OFF);
3263	ppc440spe_adma_dma2rxor_inc_addr(
3264	desc, cursor, index, src_cnt);
3265	}
3266	} else if (cursor->addrl == addr + cursor->len) {
3267	/* reverse RXOR */
3268	cursor->state = 1;
3269	cursor->xor_count++;
3270	set_bit(nr: cursor->addr_count, addr: &desc->reverse_flags[0]);
3271	if (index == src_cnt-1) {
3272	ppc440spe_rxor_set_region(desc,
3273	xor_arg_no: cursor->addr_count,
3274	DMA_RXOR12 << DMA_CUED_REGION_OFF);
3275	ppc440spe_adma_dma2rxor_inc_addr(
3276	desc, cursor, index, src_cnt);
3277	}
3278	} else {
3279	printk(KERN_ERR "Cannot build "
3280	"DMA2 RXOR command block.\n");
3281	BUG();
3282	}
3283	break;
3284	case 1:
3285	sign = test_bit(cursor->addr_count,
3286	desc->reverse_flags)
3287	? -1 : 1;
3288	if (index == src_cnt-2 || (sign == -1
3289	&& addr != cursor->addrl - 2*cursor->len)) {
3290	cursor->state = 0;
3291	cursor->xor_count = 1;
3292	cursor->addrl = addr;
3293	ppc440spe_rxor_set_region(desc,
3294	xor_arg_no: cursor->addr_count,
3295	DMA_RXOR12 << DMA_CUED_REGION_OFF);
3296	ppc440spe_adma_dma2rxor_inc_addr(
3297	desc, cursor, index, src_cnt);
3298	} else if (addr == cursor->addrl + 2*sign*cursor->len) {
3299	cursor->state = 2;
3300	cursor->xor_count = 0;
3301	ppc440spe_rxor_set_region(desc,
3302	xor_arg_no: cursor->addr_count,
3303	DMA_RXOR123 << DMA_CUED_REGION_OFF);
3304	if (index == src_cnt-1) {
3305	ppc440spe_adma_dma2rxor_inc_addr(
3306	desc, cursor, index, src_cnt);
3307	}
3308	} else if (addr == cursor->addrl + 3*cursor->len) {
3309	cursor->state = 2;
3310	cursor->xor_count = 0;
3311	ppc440spe_rxor_set_region(desc,
3312	xor_arg_no: cursor->addr_count,
3313	DMA_RXOR124 << DMA_CUED_REGION_OFF);
3314	if (index == src_cnt-1) {
3315	ppc440spe_adma_dma2rxor_inc_addr(
3316	desc, cursor, index, src_cnt);
3317	}
3318	} else if (addr == cursor->addrl + 4*cursor->len) {
3319	cursor->state = 2;
3320	cursor->xor_count = 0;
3321	ppc440spe_rxor_set_region(desc,
3322	xor_arg_no: cursor->addr_count,
3323	DMA_RXOR125 << DMA_CUED_REGION_OFF);
3324	if (index == src_cnt-1) {
3325	ppc440spe_adma_dma2rxor_inc_addr(
3326	desc, cursor, index, src_cnt);
3327	}
3328	} else {
3329	cursor->state = 0;
3330	cursor->xor_count = 1;
3331	cursor->addrl = addr;
3332	ppc440spe_rxor_set_region(desc,
3333	xor_arg_no: cursor->addr_count,
3334	DMA_RXOR12 << DMA_CUED_REGION_OFF);
3335	ppc440spe_adma_dma2rxor_inc_addr(
3336	desc, cursor, index, src_cnt);
3337	}
3338	break;
3339	case 2:
3340	cursor->state = 0;
3341	cursor->addrl = addr;
3342	cursor->xor_count++;
3343	if (index) {
3344	ppc440spe_adma_dma2rxor_inc_addr(
3345	desc, cursor, index, src_cnt);
3346	}
3347	break;
3348	}
3349
3350	return 0;
3351	}
3352
3353	/**
3354	* ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3355	* ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3356	*/
3357	static void ppc440spe_adma_dma2rxor_set_src(
3358	struct ppc440spe_adma_desc_slot *desc,
3359	int index, dma_addr_t addr)
3360	{
3361	struct xor_cb *xcb = desc->hw_desc;
3362	int k = 0, op = 0, lop = 0;
3363
3364	/* get the RXOR operand which corresponds to index addr */
3365	while (op <= index) {
3366	lop = op;
3367	if (k == XOR_MAX_OPS) {
3368	k = 0;
3369	desc = list_entry(desc->chain_node.next,
3370	struct ppc440spe_adma_desc_slot, chain_node);
3371	xcb = desc->hw_desc;
3372
3373	}
3374	if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3375	(DMA_RXOR12 << DMA_CUED_REGION_OFF))
3376	op += 2;
3377	else
3378	op += 3;
3379	}
3380
3381	BUG_ON(k < 1);
3382
3383	if (test_bit(k-1, desc->reverse_flags)) {
3384	/* reverse operand order; put last op in RXOR group */
3385	if (index == op - 1)
3386	ppc440spe_rxor_set_src(desc, xor_arg_no: k - 1, addr);
3387	} else {
3388	/* direct operand order; put first op in RXOR group */
3389	if (index == lop)
3390	ppc440spe_rxor_set_src(desc, xor_arg_no: k - 1, addr);
3391	}
3392	}
3393
3394	/**
3395	* ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3396	* ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3397	*/
3398	static void ppc440spe_adma_dma2rxor_set_mult(
3399	struct ppc440spe_adma_desc_slot *desc,
3400	int index, u8 mult)
3401	{
3402	struct xor_cb *xcb = desc->hw_desc;
3403	int k = 0, op = 0, lop = 0;
3404
3405	/* get the RXOR operand which corresponds to index mult */
3406	while (op <= index) {
3407	lop = op;
3408	if (k == XOR_MAX_OPS) {
3409	k = 0;
3410	desc = list_entry(desc->chain_node.next,
3411	struct ppc440spe_adma_desc_slot,
3412	chain_node);
3413	xcb = desc->hw_desc;
3414
3415	}
3416	if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3417	(DMA_RXOR12 << DMA_CUED_REGION_OFF))
3418	op += 2;
3419	else
3420	op += 3;
3421	}
3422
3423	BUG_ON(k < 1);
3424	if (test_bit(k-1, desc->reverse_flags)) {
3425	/* reverse order */
3426	ppc440spe_rxor_set_mult(desc, xor_arg_no: k - 1, idx: op - index - 1, mult);
3427	} else {
3428	/* direct order */
3429	ppc440spe_rxor_set_mult(desc, xor_arg_no: k - 1, idx: index - lop, mult);
3430	}
3431	}
3432
3433	/**
3434	* ppc440spe_init_rxor_cursor -
3435	*/
3436	static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3437	{
3438	memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3439	cursor->state = 2;
3440	}
3441
3442	/**
3443	* ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3444	* descriptor for the PQXOR operation
3445	*/
3446	static void ppc440spe_adma_pq_set_src_mult(
3447	struct ppc440spe_adma_desc_slot *sw_desc,
3448	unsigned char mult, int index, int dst_pos)
3449	{
3450	struct ppc440spe_adma_chan *chan;
3451	u32 mult_idx, mult_dst;
3452	struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3453
3454	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3455
3456	switch (chan->device->id) {
3457	case PPC440SPE_DMA0_ID:
3458	case PPC440SPE_DMA1_ID:
3459	if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3460	int region = test_bit(PPC440SPE_DESC_RXOR12,
3461	&sw_desc->flags) ? 2 : 3;
3462
3463	if (index < region) {
3464	/* RXOR multipliers */
3465	iter = ppc440spe_get_group_entry(tdesc: sw_desc,
3466	entry_idx: sw_desc->dst_cnt - 1);
3467	if (sw_desc->dst_cnt == 2)
3468	iter1 = ppc440spe_get_group_entry(
3469	tdesc: sw_desc, entry_idx: 0);
3470
3471	mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3472	mult_dst = DMA_CDB_SG_SRC;
3473	} else {
3474	/* WXOR multiplier */
3475	iter = ppc440spe_get_group_entry(tdesc: sw_desc,
3476	entry_idx: index - region +
3477	sw_desc->dst_cnt);
3478	mult_idx = DMA_CUED_MULT1_OFF;
3479	mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3480	DMA_CDB_SG_DST1;
3481	}
3482	} else {
3483	int znum = 0;
3484
3485	/* WXOR-only;
3486	* skip first slots with destinations (if ZERO_DST has
3487	* place)
3488	*/
3489	if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3490	znum++;
3491	if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3492	znum++;
3493
3494	iter = ppc440spe_get_group_entry(tdesc: sw_desc, entry_idx: index + znum);
3495	mult_idx = DMA_CUED_MULT1_OFF;
3496	mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3497	}
3498
3499	if (likely(iter)) {
3500	ppc440spe_desc_set_src_mult(desc: iter, chan,
3501	mult_index: mult_idx, sg_index: mult_dst, mult_value: mult);
3502
3503	if (unlikely(iter1)) {
3504	/* if we have two destinations for RXOR, then
3505	* we've just set Q mult. Set-up P now.
3506	*/
3507	ppc440spe_desc_set_src_mult(desc: iter1, chan,
3508	mult_index: mult_idx, sg_index: mult_dst, mult_value: 1);
3509	}
3510
3511	}
3512	break;
3513
3514	case PPC440SPE_XOR_ID:
3515	iter = sw_desc->group_head;
3516	if (sw_desc->dst_cnt == 2) {
3517	/* both P & Q calculations required; set P mult here */
3518	ppc440spe_adma_dma2rxor_set_mult(desc: iter, index, mult: 1);
3519
3520	/* and then set Q mult */
3521	iter = ppc440spe_get_group_entry(tdesc: sw_desc,
3522	entry_idx: sw_desc->descs_per_op);
3523	}
3524	ppc440spe_adma_dma2rxor_set_mult(desc: iter, index, mult);
3525	break;
3526	}
3527	}
3528
3529	/**
3530	* ppc440spe_adma_free_chan_resources - free the resources allocated
3531	*/
3532	static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3533	{
3534	struct ppc440spe_adma_chan *ppc440spe_chan;
3535	struct ppc440spe_adma_desc_slot *iter, *_iter;
3536	int in_use_descs = 0;
3537
3538	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3539	ppc440spe_adma_slot_cleanup(chan: ppc440spe_chan);
3540
3541	spin_lock_bh(lock: &ppc440spe_chan->lock);
3542	list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3543	chain_node) {
3544	in_use_descs++;
3545	list_del(entry: &iter->chain_node);
3546	}
3547	list_for_each_entry_safe_reverse(iter, _iter,
3548	&ppc440spe_chan->all_slots, slot_node) {
3549	list_del(entry: &iter->slot_node);
3550	kfree(objp: iter);
3551	ppc440spe_chan->slots_allocated--;
3552	}
3553	ppc440spe_chan->last_used = NULL;
3554
3555	dev_dbg(ppc440spe_chan->device->common.dev,
3556	"ppc440spe adma%d %s slots_allocated %d\n",
3557	ppc440spe_chan->device->id,
3558	__func__, ppc440spe_chan->slots_allocated);
3559	spin_unlock_bh(lock: &ppc440spe_chan->lock);
3560
3561	/* one is ok since we left it on there on purpose */
3562	if (in_use_descs > 1)
3563	printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3564	in_use_descs - 1);
3565	}
3566
3567	/**
3568	* ppc440spe_adma_tx_status - poll the status of an ADMA transaction
3569	* @chan: ADMA channel handle
3570	* @cookie: ADMA transaction identifier
3571	* @txstate: a holder for the current state of the channel
3572	*/
3573	static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan,
3574	dma_cookie_t cookie, struct dma_tx_state *txstate)
3575	{
3576	struct ppc440spe_adma_chan *ppc440spe_chan;
3577	enum dma_status ret;
3578
3579	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3580	ret = dma_cookie_status(chan, cookie, state: txstate);
3581	if (ret == DMA_COMPLETE)
3582	return ret;
3583
3584	ppc440spe_adma_slot_cleanup(chan: ppc440spe_chan);
3585
3586	return dma_cookie_status(chan, cookie, state: txstate);
3587	}
3588
3589	/**
3590	* ppc440spe_adma_eot_handler - end of transfer interrupt handler
3591	*/
3592	static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3593	{
3594	struct ppc440spe_adma_chan *chan = data;
3595
3596	dev_dbg(chan->device->common.dev,
3597	"ppc440spe adma%d: %s\n", chan->device->id, __func__);
3598
3599	tasklet_schedule(t: &chan->irq_tasklet);
3600	ppc440spe_adma_device_clear_eot_status(chan);
3601
3602	return IRQ_HANDLED;
3603	}
3604
3605	/**
3606	* ppc440spe_adma_err_handler - DMA error interrupt handler;
3607	* do the same things as a eot handler
3608	*/
3609	static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3610	{
3611	struct ppc440spe_adma_chan *chan = data;
3612
3613	dev_dbg(chan->device->common.dev,
3614	"ppc440spe adma%d: %s\n", chan->device->id, __func__);
3615
3616	tasklet_schedule(t: &chan->irq_tasklet);
3617	ppc440spe_adma_device_clear_eot_status(chan);
3618
3619	return IRQ_HANDLED;
3620	}
3621
3622	/**
3623	* ppc440spe_test_callback - called when test operation has been done
3624	*/
3625	static void ppc440spe_test_callback(void *unused)
3626	{
3627	complete(&ppc440spe_r6_test_comp);
3628	}
3629
3630	/**
3631	* ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
3632	*/
3633	static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
3634	{
3635	struct ppc440spe_adma_chan *ppc440spe_chan;
3636
3637	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3638	dev_dbg(ppc440spe_chan->device->common.dev,
3639	"ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
3640	__func__, ppc440spe_chan->pending);
3641
3642	if (ppc440spe_chan->pending) {
3643	ppc440spe_chan->pending = 0;
3644	ppc440spe_chan_append(chan: ppc440spe_chan);
3645	}
3646	}
3647
3648	/**
3649	* ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
3650	* use FIFOs (as opposite to chains used in XOR) so this is a XOR
3651	* specific operation)
3652	*/
3653	static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
3654	{
3655	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
3656	dma_cookie_t cookie;
3657	int slot_cnt, slots_per_op;
3658
3659	dev_dbg(chan->device->common.dev,
3660	"ppc440spe adma%d: %s\n", chan->device->id, __func__);
3661
3662	spin_lock_bh(lock: &chan->lock);
3663	slot_cnt = ppc440spe_chan_xor_slot_count(len: 0, src_cnt: 2, slots_per_op: &slots_per_op);
3664	sw_desc = ppc440spe_adma_alloc_slots(chan, num_slots: slot_cnt, slots_per_op);
3665	if (sw_desc) {
3666	group_start = sw_desc->group_head;
3667	list_splice_init(list: &sw_desc->group_list, head: &chan->chain);
3668	async_tx_ack(tx: &sw_desc->async_tx);
3669	ppc440spe_desc_init_null_xor(desc: group_start);
3670
3671	cookie = dma_cookie_assign(tx: &sw_desc->async_tx);
3672
3673	/* initialize the completed cookie to be less than
3674	* the most recently used cookie
3675	*/
3676	chan->common.completed_cookie = cookie - 1;
3677
3678	/* channel should not be busy */
3679	BUG_ON(ppc440spe_chan_is_busy(chan));
3680
3681	/* set the descriptor address */
3682	ppc440spe_chan_set_first_xor_descriptor(chan, next_desc: sw_desc);
3683
3684	/* run the descriptor */
3685	ppc440spe_chan_run(chan);
3686	} else
3687	printk(KERN_ERR "ppc440spe adma%d"
3688	" failed to allocate null descriptor\n",
3689	chan->device->id);
3690	spin_unlock_bh(lock: &chan->lock);
3691	}
3692
3693	/**
3694	* ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
3695	* For this we just perform one WXOR operation with the same source
3696	* and destination addresses, the GF-multiplier is 1; so if RAID-6
3697	* capabilities are enabled then we'll get src/dst filled with zero.
3698	*/
3699	static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
3700	{
3701	struct ppc440spe_adma_desc_slot *sw_desc, *iter;
3702	struct page *pg;
3703	char *a;
3704	dma_addr_t dma_addr, addrs[2];
3705	unsigned long op = 0;
3706	int rval = 0;
3707
3708	set_bit(PPC440SPE_DESC_WXOR, addr: &op);
3709
3710	pg = alloc_page(GFP_KERNEL);
3711	if (!pg)
3712	return -ENOMEM;
3713
3714	spin_lock_bh(lock: &chan->lock);
3715	sw_desc = ppc440spe_adma_alloc_slots(chan, num_slots: 1, slots_per_op: 1);
3716	if (sw_desc) {
3717	/* 1 src, 1 dsr, int_ena, WXOR */
3718	ppc440spe_desc_init_dma01pq(desc: sw_desc, dst_cnt: 1, src_cnt: 1, flags: 1, op);
3719	list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
3720	ppc440spe_desc_set_byte_count(desc: iter, chan, PAGE_SIZE);
3721	iter->unmap_len = PAGE_SIZE;
3722	}
3723	} else {
3724	rval = -EFAULT;
3725	spin_unlock_bh(lock: &chan->lock);
3726	goto exit;
3727	}
3728	spin_unlock_bh(lock: &chan->lock);
3729
3730	/* Fill the test page with ones */
3731	memset(page_address(pg), 0xFF, PAGE_SIZE);
3732	dma_addr = dma_map_page(chan->device->dev, pg, 0,
3733	PAGE_SIZE, DMA_BIDIRECTIONAL);
3734
3735	/* Setup addresses */
3736	ppc440spe_adma_pq_set_src(sw_desc, addr: dma_addr, index: 0);
3737	ppc440spe_adma_pq_set_src_mult(sw_desc, mult: 1, index: 0, dst_pos: 0);
3738	addrs[0] = dma_addr;
3739	addrs[1] = 0;
3740	ppc440spe_adma_pq_set_dest(sw_desc, addrs, flags: DMA_PREP_PQ_DISABLE_Q);
3741
3742	async_tx_ack(tx: &sw_desc->async_tx);
3743	sw_desc->async_tx.callback = ppc440spe_test_callback;
3744	sw_desc->async_tx.callback_param = NULL;
3745
3746	init_completion(x: &ppc440spe_r6_test_comp);
3747
3748	ppc440spe_adma_tx_submit(tx: &sw_desc->async_tx);
3749	ppc440spe_adma_issue_pending(chan: &chan->common);
3750
3751	wait_for_completion(&ppc440spe_r6_test_comp);
3752
3753	/* Now check if the test page is zeroed */
3754	a = page_address(pg);
3755	if ((*(u32 *)a) == 0 && memcmp(p: a, q: a+4, PAGE_SIZE-4) == 0) {
3756	/* page is zero - RAID-6 enabled */
3757	rval = 0;
3758	} else {
3759	/* RAID-6 was not enabled */
3760	rval = -EINVAL;
3761	}
3762	exit:
3763	__free_page(pg);
3764	return rval;
3765	}
3766
3767	static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
3768	{
3769	switch (adev->id) {
3770	case PPC440SPE_DMA0_ID:
3771	case PPC440SPE_DMA1_ID:
3772	dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
3773	dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3774	dma_cap_set(DMA_PQ, adev->common.cap_mask);
3775	dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
3776	dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
3777	break;
3778	case PPC440SPE_XOR_ID:
3779	dma_cap_set(DMA_XOR, adev->common.cap_mask);
3780	dma_cap_set(DMA_PQ, adev->common.cap_mask);
3781	dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3782	adev->common.cap_mask = adev->common.cap_mask;
3783	break;
3784	}
3785
3786	/* Set base routines */
3787	adev->common.device_alloc_chan_resources =
3788	ppc440spe_adma_alloc_chan_resources;
3789	adev->common.device_free_chan_resources =
3790	ppc440spe_adma_free_chan_resources;
3791	adev->common.device_tx_status = ppc440spe_adma_tx_status;
3792	adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
3793
3794	/* Set prep routines based on capability */
3795	if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
3796	adev->common.device_prep_dma_memcpy =
3797	ppc440spe_adma_prep_dma_memcpy;
3798	}
3799	if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
3800	adev->common.max_xor = XOR_MAX_OPS;
3801	adev->common.device_prep_dma_xor =
3802	ppc440spe_adma_prep_dma_xor;
3803	}
3804	if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
3805	switch (adev->id) {
3806	case PPC440SPE_DMA0_ID:
3807	dma_set_maxpq(dma: &adev->common,
3808	DMA0_FIFO_SIZE / sizeof(struct dma_cdb), has_pq_continue: 0);
3809	break;
3810	case PPC440SPE_DMA1_ID:
3811	dma_set_maxpq(dma: &adev->common,
3812	DMA1_FIFO_SIZE / sizeof(struct dma_cdb), has_pq_continue: 0);
3813	break;
3814	case PPC440SPE_XOR_ID:
3815	adev->common.max_pq = XOR_MAX_OPS * 3;
3816	break;
3817	}
3818	adev->common.device_prep_dma_pq =
3819	ppc440spe_adma_prep_dma_pq;
3820	}
3821	if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
3822	switch (adev->id) {
3823	case PPC440SPE_DMA0_ID:
3824	adev->common.max_pq = DMA0_FIFO_SIZE /
3825	sizeof(struct dma_cdb);
3826	break;
3827	case PPC440SPE_DMA1_ID:
3828	adev->common.max_pq = DMA1_FIFO_SIZE /
3829	sizeof(struct dma_cdb);
3830	break;
3831	}
3832	adev->common.device_prep_dma_pq_val =
3833	ppc440spe_adma_prep_dma_pqzero_sum;
3834	}
3835	if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
3836	switch (adev->id) {
3837	case PPC440SPE_DMA0_ID:
3838	adev->common.max_xor = DMA0_FIFO_SIZE /
3839	sizeof(struct dma_cdb);
3840	break;
3841	case PPC440SPE_DMA1_ID:
3842	adev->common.max_xor = DMA1_FIFO_SIZE /
3843	sizeof(struct dma_cdb);
3844	break;
3845	}
3846	adev->common.device_prep_dma_xor_val =
3847	ppc440spe_adma_prep_dma_xor_zero_sum;
3848	}
3849	if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
3850	adev->common.device_prep_dma_interrupt =
3851	ppc440spe_adma_prep_dma_interrupt;
3852	}
3853	pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
3854	"( %s%s%s%s%s%s)\n",
3855	dev_name(adev->dev),
3856	dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
3857	dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
3858	dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
3859	dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
3860	dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
3861	dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
3862	}
3863
3864	static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
3865	struct ppc440spe_adma_chan *chan,
3866	int *initcode)
3867	{
3868	struct platform_device *ofdev;
3869	struct device_node *np;
3870	int ret;
3871
3872	ofdev = container_of(adev->dev, struct platform_device, dev);
3873	np = ofdev->dev.of_node;
3874	if (adev->id != PPC440SPE_XOR_ID) {
3875	adev->err_irq = irq_of_parse_and_map(node: np, index: 1);
3876	if (!adev->err_irq) {
3877	dev_warn(adev->dev, "no err irq resource?\n");
3878	*initcode = PPC_ADMA_INIT_IRQ2;
3879	adev->err_irq = -ENXIO;
3880	} else
3881	atomic_inc(v: &ppc440spe_adma_err_irq_ref);
3882	} else {
3883	adev->err_irq = -ENXIO;
3884	}
3885
3886	adev->irq = irq_of_parse_and_map(node: np, index: 0);
3887	if (!adev->irq) {
3888	dev_err(adev->dev, "no irq resource\n");
3889	*initcode = PPC_ADMA_INIT_IRQ1;
3890	ret = -ENXIO;
3891	goto err_irq_map;
3892	}
3893	dev_dbg(adev->dev, "irq %d, err irq %d\n",
3894	adev->irq, adev->err_irq);
3895
3896	ret = request_irq(irq: adev->irq, handler: ppc440spe_adma_eot_handler,
3897	flags: 0, name: dev_driver_string(dev: adev->dev), dev: chan);
3898	if (ret) {
3899	dev_err(adev->dev, "can't request irq %d\n",
3900	adev->irq);
3901	*initcode = PPC_ADMA_INIT_IRQ1;
3902	ret = -EIO;
3903	goto err_req1;
3904	}
3905
3906	/* only DMA engines have a separate error IRQ
3907	* so it's Ok if err_irq < 0 in XOR engine case.
3908	*/
3909	if (adev->err_irq > 0) {
3910	/* both DMA engines share common error IRQ */
3911	ret = request_irq(irq: adev->err_irq,
3912	handler: ppc440spe_adma_err_handler,
3913	IRQF_SHARED,
3914	name: dev_driver_string(dev: adev->dev),
3915	dev: chan);
3916	if (ret) {
3917	dev_err(adev->dev, "can't request irq %d\n",
3918	adev->err_irq);
3919	*initcode = PPC_ADMA_INIT_IRQ2;
3920	ret = -EIO;
3921	goto err_req2;
3922	}
3923	}
3924
3925	if (adev->id == PPC440SPE_XOR_ID) {
3926	/* enable XOR engine interrupts */
3927	iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3928	XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
3929	&adev->xor_reg->ier);
3930	} else {
3931	u32 mask, enable;
3932
3933	np = of_find_compatible_node(NULL, NULL, compat: "ibm,i2o-440spe");
3934	if (!np) {
3935	pr_err("%s: can't find I2O device tree node\n",
3936	__func__);
3937	ret = -ENODEV;
3938	goto err_req2;
3939	}
3940	adev->i2o_reg = of_iomap(node: np, index: 0);
3941	if (!adev->i2o_reg) {
3942	pr_err("%s: failed to map I2O registers\n", __func__);
3943	of_node_put(node: np);
3944	ret = -EINVAL;
3945	goto err_req2;
3946	}
3947	of_node_put(node: np);
3948	/* Unmask 'CS FIFO Attention' interrupts and
3949	* enable generating interrupts on errors
3950	*/
3951	enable = (adev->id == PPC440SPE_DMA0_ID) ?
3952	~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3953	~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3954	mask = ioread32(&adev->i2o_reg->iopim) & enable;
3955	iowrite32(mask, &adev->i2o_reg->iopim);
3956	}
3957	return 0;
3958
3959	err_req2:
3960	free_irq(adev->irq, chan);
3961	err_req1:
3962	irq_dispose_mapping(virq: adev->irq);
3963	err_irq_map:
3964	if (adev->err_irq > 0) {
3965	if (atomic_dec_and_test(v: &ppc440spe_adma_err_irq_ref))
3966	irq_dispose_mapping(virq: adev->err_irq);
3967	}
3968	return ret;
3969	}
3970
3971	static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
3972	struct ppc440spe_adma_chan *chan)
3973	{
3974	u32 mask, disable;
3975
3976	if (adev->id == PPC440SPE_XOR_ID) {
3977	/* disable XOR engine interrupts */
3978	mask = ioread32be(&adev->xor_reg->ier);
3979	mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3980	XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
3981	iowrite32be(mask, &adev->xor_reg->ier);
3982	} else {
3983	/* disable DMAx engine interrupts */
3984	disable = (adev->id == PPC440SPE_DMA0_ID) ?
3985	(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3986	(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3987	mask = ioread32(&adev->i2o_reg->iopim) | disable;
3988	iowrite32(mask, &adev->i2o_reg->iopim);
3989	}
3990	free_irq(adev->irq, chan);
3991	irq_dispose_mapping(virq: adev->irq);
3992	if (adev->err_irq > 0) {
3993	free_irq(adev->err_irq, chan);
3994	if (atomic_dec_and_test(v: &ppc440spe_adma_err_irq_ref)) {
3995	irq_dispose_mapping(virq: adev->err_irq);
3996	iounmap(addr: adev->i2o_reg);
3997	}
3998	}
3999	}
4000
4001	/**
4002	* ppc440spe_adma_probe - probe the asynch device
4003	*/
4004	static int ppc440spe_adma_probe(struct platform_device *ofdev)
4005	{
4006	struct device_node *np = ofdev->dev.of_node;
4007	struct resource res;
4008	struct ppc440spe_adma_device *adev;
4009	struct ppc440spe_adma_chan *chan;
4010	struct ppc_dma_chan_ref *ref, *_ref;
4011	int ret = 0, initcode = PPC_ADMA_INIT_OK;
4012	const u32 *idx;
4013	int len;
4014	void *regs;
4015	u32 id, pool_size;
4016
4017	if (of_device_is_compatible(device: np, "amcc,xor-accelerator")) {
4018	id = PPC440SPE_XOR_ID;
4019	/* As far as the XOR engine is concerned, it does not
4020	* use FIFOs but uses linked list. So there is no dependency
4021	* between pool size to allocate and the engine configuration.
4022	*/
4023	pool_size = PAGE_SIZE << 1;
4024	} else {
4025	/* it is DMA0 or DMA1 */
4026	idx = of_get_property(node: np, name: "cell-index", lenp: &len);
4027	if (!idx || (len != sizeof(u32))) {
4028	dev_err(&ofdev->dev, "Device node %pOF has missing "
4029	"or invalid cell-index property\n",
4030	np);
4031	return -EINVAL;
4032	}
4033	id = *idx;
4034	/* DMA0,1 engines use FIFO to maintain CDBs, so we
4035	* should allocate the pool accordingly to size of this
4036	* FIFO. Thus, the pool size depends on the FIFO depth:
4037	* how much CDBs pointers the FIFO may contain then so
4038	* much CDBs we should provide in the pool.
4039	* That is
4040	* CDB size = 32B;
4041	* CDBs number = (DMA0_FIFO_SIZE >> 3);
4042	* Pool size = CDBs number * CDB size =
4043	* = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4044	*/
4045	pool_size = (id == PPC440SPE_DMA0_ID) ?
4046	DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4047	pool_size <<= 2;
4048	}
4049
4050	if (of_address_to_resource(dev: np, index: 0, r: &res)) {
4051	dev_err(&ofdev->dev, "failed to get memory resource\n");
4052	initcode = PPC_ADMA_INIT_MEMRES;
4053	ret = -ENODEV;
4054	goto out;
4055	}
4056
4057	if (!request_mem_region(res.start, resource_size(&res),
4058	dev_driver_string(&ofdev->dev))) {
4059	dev_err(&ofdev->dev, "failed to request memory region %pR\n",
4060	&res);
4061	initcode = PPC_ADMA_INIT_MEMREG;
4062	ret = -EBUSY;
4063	goto out;
4064	}
4065
4066	/* create a device */
4067	adev = kzalloc(size: sizeof(*adev), GFP_KERNEL);
4068	if (!adev) {
4069	initcode = PPC_ADMA_INIT_ALLOC;
4070	ret = -ENOMEM;
4071	goto err_adev_alloc;
4072	}
4073
4074	adev->id = id;
4075	adev->pool_size = pool_size;
4076	/* allocate coherent memory for hardware descriptors */
4077	adev->dma_desc_pool_virt = dma_alloc_coherent(dev: &ofdev->dev,
4078	size: adev->pool_size, dma_handle: &adev->dma_desc_pool,
4079	GFP_KERNEL);
4080	if (adev->dma_desc_pool_virt == NULL) {
4081	dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4082	"memory for hardware descriptors\n",
4083	adev->pool_size);
4084	initcode = PPC_ADMA_INIT_COHERENT;
4085	ret = -ENOMEM;
4086	goto err_dma_alloc;
4087	}
4088	dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
4089	adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4090
4091	regs = ioremap(offset: res.start, size: resource_size(res: &res));
4092	if (!regs) {
4093	dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4094	ret = -ENOMEM;
4095	goto err_regs_alloc;
4096	}
4097
4098	if (adev->id == PPC440SPE_XOR_ID) {
4099	adev->xor_reg = regs;
4100	/* Reset XOR */
4101	iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4102	iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4103	} else {
4104	size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4105	DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4106	adev->dma_reg = regs;
4107	/* DMAx_FIFO_SIZE is defined in bytes,
4108	* <fsiz> - is defined in number of CDB pointers (8byte).
4109	* DMA FIFO Length = CSlength + CPlength, where
4110	* CSlength = CPlength = (fsiz + 1) * 8.
4111	*/
4112	iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4113	&adev->dma_reg->fsiz);
4114	/* Configure DMA engine */
4115	iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4116	&adev->dma_reg->cfg);
4117	/* Clear Status */
4118	iowrite32(~0, &adev->dma_reg->dsts);
4119	}
4120
4121	adev->dev = &ofdev->dev;
4122	adev->common.dev = &ofdev->dev;
4123	INIT_LIST_HEAD(list: &adev->common.channels);
4124	platform_set_drvdata(pdev: ofdev, data: adev);
4125
4126	/* create a channel */
4127	chan = kzalloc(size: sizeof(*chan), GFP_KERNEL);
4128	if (!chan) {
4129	initcode = PPC_ADMA_INIT_CHANNEL;
4130	ret = -ENOMEM;
4131	goto err_chan_alloc;
4132	}
4133
4134	spin_lock_init(&chan->lock);
4135	INIT_LIST_HEAD(list: &chan->chain);
4136	INIT_LIST_HEAD(list: &chan->all_slots);
4137	chan->device = adev;
4138	chan->common.device = &adev->common;
4139	dma_cookie_init(chan: &chan->common);
4140	list_add_tail(new: &chan->common.device_node, head: &adev->common.channels);
4141	tasklet_setup(t: &chan->irq_tasklet, callback: ppc440spe_adma_tasklet);
4142
4143	/* allocate and map helper pages for async validation or
4144	* async_mult/async_sum_product operations on DMA0/1.
4145	*/
4146	if (adev->id != PPC440SPE_XOR_ID) {
4147	chan->pdest_page = alloc_page(GFP_KERNEL);
4148	chan->qdest_page = alloc_page(GFP_KERNEL);
4149	if (!chan->pdest_page ||
4150	!chan->qdest_page) {
4151	if (chan->pdest_page)
4152	__free_page(chan->pdest_page);
4153	if (chan->qdest_page)
4154	__free_page(chan->qdest_page);
4155	ret = -ENOMEM;
4156	goto err_page_alloc;
4157	}
4158	chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4159	PAGE_SIZE, DMA_BIDIRECTIONAL);
4160	chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4161	PAGE_SIZE, DMA_BIDIRECTIONAL);
4162	}
4163
4164	ref = kmalloc(size: sizeof(*ref), GFP_KERNEL);
4165	if (ref) {
4166	ref->chan = &chan->common;
4167	INIT_LIST_HEAD(list: &ref->node);
4168	list_add_tail(new: &ref->node, head: &ppc440spe_adma_chan_list);
4169	} else {
4170	dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4171	ret = -ENOMEM;
4172	goto err_ref_alloc;
4173	}
4174
4175	ret = ppc440spe_adma_setup_irqs(adev, chan, initcode: &initcode);
4176	if (ret)
4177	goto err_irq;
4178
4179	ppc440spe_adma_init_capabilities(adev);
4180
4181	ret = dma_async_device_register(device: &adev->common);
4182	if (ret) {
4183	initcode = PPC_ADMA_INIT_REGISTER;
4184	dev_err(&ofdev->dev, "failed to register dma device\n");
4185	goto err_dev_reg;
4186	}
4187
4188	goto out;
4189
4190	err_dev_reg:
4191	ppc440spe_adma_release_irqs(adev, chan);
4192	err_irq:
4193	list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4194	if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4195	list_del(entry: &ref->node);
4196	kfree(objp: ref);
4197	}
4198	}
4199	err_ref_alloc:
4200	if (adev->id != PPC440SPE_XOR_ID) {
4201	dma_unmap_page(&ofdev->dev, chan->pdest,
4202	PAGE_SIZE, DMA_BIDIRECTIONAL);
4203	dma_unmap_page(&ofdev->dev, chan->qdest,
4204	PAGE_SIZE, DMA_BIDIRECTIONAL);
4205	__free_page(chan->pdest_page);
4206	__free_page(chan->qdest_page);
4207	}
4208	err_page_alloc:
4209	kfree(objp: chan);
4210	err_chan_alloc:
4211	if (adev->id == PPC440SPE_XOR_ID)
4212	iounmap(addr: adev->xor_reg);
4213	else
4214	iounmap(addr: adev->dma_reg);
4215	err_regs_alloc:
4216	dma_free_coherent(dev: adev->dev, size: adev->pool_size,
4217	cpu_addr: adev->dma_desc_pool_virt,
4218	dma_handle: adev->dma_desc_pool);
4219	err_dma_alloc:
4220	kfree(objp: adev);
4221	err_adev_alloc:
4222	release_mem_region(res.start, resource_size(&res));
4223	out:
4224	if (id < PPC440SPE_ADMA_ENGINES_NUM)
4225	ppc440spe_adma_devices[id] = initcode;
4226
4227	return ret;
4228	}
4229
4230	/**
4231	* ppc440spe_adma_remove - remove the asynch device
4232	*/
4233	static void ppc440spe_adma_remove(struct platform_device *ofdev)
4234	{
4235	struct ppc440spe_adma_device *adev = platform_get_drvdata(pdev: ofdev);
4236	struct device_node *np = ofdev->dev.of_node;
4237	struct resource res;
4238	struct dma_chan *chan, *_chan;
4239	struct ppc_dma_chan_ref *ref, *_ref;
4240	struct ppc440spe_adma_chan *ppc440spe_chan;
4241
4242	if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4243	ppc440spe_adma_devices[adev->id] = -1;
4244
4245	dma_async_device_unregister(device: &adev->common);
4246
4247	list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4248	device_node) {
4249	ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4250	ppc440spe_adma_release_irqs(adev, chan: ppc440spe_chan);
4251	tasklet_kill(t: &ppc440spe_chan->irq_tasklet);
4252	if (adev->id != PPC440SPE_XOR_ID) {
4253	dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4254	PAGE_SIZE, DMA_BIDIRECTIONAL);
4255	dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4256	PAGE_SIZE, DMA_BIDIRECTIONAL);
4257	__free_page(ppc440spe_chan->pdest_page);
4258	__free_page(ppc440spe_chan->qdest_page);
4259	}
4260	list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4261	node) {
4262	if (ppc440spe_chan ==
4263	to_ppc440spe_adma_chan(ref->chan)) {
4264	list_del(entry: &ref->node);
4265	kfree(objp: ref);
4266	}
4267	}
4268	list_del(entry: &chan->device_node);
4269	kfree(objp: ppc440spe_chan);
4270	}
4271
4272	dma_free_coherent(dev: adev->dev, size: adev->pool_size,
4273	cpu_addr: adev->dma_desc_pool_virt, dma_handle: adev->dma_desc_pool);
4274	if (adev->id == PPC440SPE_XOR_ID)
4275	iounmap(addr: adev->xor_reg);
4276	else
4277	iounmap(addr: adev->dma_reg);
4278	of_address_to_resource(dev: np, index: 0, r: &res);
4279	release_mem_region(res.start, resource_size(&res));
4280	kfree(objp: adev);
4281	}
4282
4283	/*
4284	* /sys driver interface to enable h/w RAID-6 capabilities
4285	* Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4286	* directory are "devices", "enable" and "poly".
4287	* "devices" shows available engines.
4288	* "enable" is used to enable RAID-6 capabilities or to check
4289	* whether these has been activated.
4290	* "poly" allows setting/checking used polynomial (for PPC440SPe only).
4291	*/
4292
4293	static ssize_t devices_show(struct device_driver *dev, char *buf)
4294	{
4295	ssize_t size = 0;
4296	int i;
4297
4298	for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4299	if (ppc440spe_adma_devices[i] == -1)
4300	continue;
4301	size += sysfs_emit_at(buf, at: size, fmt: "PPC440SP(E)-ADMA.%d: %s\n",
4302	i, ppc_adma_errors[ppc440spe_adma_devices[i]]);
4303	}
4304	return size;
4305	}
4306	static DRIVER_ATTR_RO(devices);
4307
4308	static ssize_t enable_show(struct device_driver *dev, char *buf)
4309	{
4310	return sysfs_emit(buf, fmt: "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4311	ppc440spe_r6_enabled ? "EN" : "DIS");
4312	}
4313
4314	static ssize_t enable_store(struct device_driver *dev, const char *buf,
4315	size_t count)
4316	{
4317	unsigned long val;
4318	int err;
4319
4320	if (!count || count > 11)
4321	return -EINVAL;
4322
4323	if (!ppc440spe_r6_tchan)
4324	return -EFAULT;
4325
4326	/* Write a key */
4327	err = kstrtoul(s: buf, base: 16, res: &val);
4328	if (err)
4329	return err;
4330
4331	dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4332	isync();
4333
4334	/* Verify whether it really works now */
4335	if (ppc440spe_test_raid6(chan: ppc440spe_r6_tchan) == 0) {
4336	pr_info("PPC440SP(e) RAID-6 has been activated "
4337	"successfully\n");
4338	ppc440spe_r6_enabled = 1;
4339	} else {
4340	pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4341	" Error key ?\n");
4342	ppc440spe_r6_enabled = 0;
4343	}
4344	return count;
4345	}
4346	static DRIVER_ATTR_RW(enable);
4347
4348	static ssize_t poly_show(struct device_driver *dev, char *buf)
4349	{
4350	ssize_t size = 0;
4351	u32 reg;
4352
4353	#ifdef CONFIG_440SP
4354	/* 440SP has fixed polynomial */
4355	reg = 0x4d;
4356	#else
4357	reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4358	reg >>= MQ0_CFBHL_POLY;
4359	reg &= 0xFF;
4360	#endif
4361
4362	size = sysfs_emit(buf, fmt: "PPC440SP(e) RAID-6 driver "
4363	"uses 0x1%02x polynomial.\n", reg);
4364	return size;
4365	}
4366
4367	static ssize_t poly_store(struct device_driver *dev, const char *buf,
4368	size_t count)
4369	{
4370	unsigned long reg, val;
4371	int err;
4372	#ifdef CONFIG_440SP
4373	/* 440SP uses default 0x14D polynomial only */
4374	return -EINVAL;
4375	#endif
4376
4377	if (!count || count > 6)
4378	return -EINVAL;
4379
4380	/* e.g., 0x14D or 0x11D */
4381	err = kstrtoul(s: buf, base: 16, res: &val);
4382	if (err)
4383	return err;
4384
4385	if (val & ~0x1FF)
4386	return -EINVAL;
4387
4388	val &= 0xFF;
4389	reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4390	reg &= ~(0xFF << MQ0_CFBHL_POLY);
4391	reg |= val << MQ0_CFBHL_POLY;
4392	dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4393
4394	return count;
4395	}
4396	static DRIVER_ATTR_RW(poly);
4397
4398	/*
4399	* Common initialisation for RAID engines; allocate memory for
4400	* DMAx FIFOs, perform configuration common for all DMA engines.
4401	* Further DMA engine specific configuration is done at probe time.
4402	*/
4403	static int ppc440spe_configure_raid_devices(void)
4404	{
4405	struct device_node *np;
4406	struct resource i2o_res;
4407	struct i2o_regs __iomem *i2o_reg;
4408	dcr_host_t i2o_dcr_host;
4409	unsigned int dcr_base, dcr_len;
4410	int i, ret;
4411
4412	np = of_find_compatible_node(NULL, NULL, compat: "ibm,i2o-440spe");
4413	if (!np) {
4414	pr_err("%s: can't find I2O device tree node\n",
4415	__func__);
4416	return -ENODEV;
4417	}
4418
4419	if (of_address_to_resource(dev: np, index: 0, r: &i2o_res)) {
4420	of_node_put(node: np);
4421	return -EINVAL;
4422	}
4423
4424	i2o_reg = of_iomap(node: np, index: 0);
4425	if (!i2o_reg) {
4426	pr_err("%s: failed to map I2O registers\n", __func__);
4427	of_node_put(node: np);
4428	return -EINVAL;
4429	}
4430
4431	/* Get I2O DCRs base */
4432	dcr_base = dcr_resource_start(np, 0);
4433	dcr_len = dcr_resource_len(np, 0);
4434	if (!dcr_base && !dcr_len) {
4435	pr_err("%pOF: can't get DCR registers base/len!\n", np);
4436	of_node_put(node: np);
4437	iounmap(addr: i2o_reg);
4438	return -ENODEV;
4439	}
4440
4441	i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4442	if (!DCR_MAP_OK(i2o_dcr_host)) {
4443	pr_err("%pOF: failed to map DCRs!\n", np);
4444	of_node_put(node: np);
4445	iounmap(addr: i2o_reg);
4446	return -ENODEV;
4447	}
4448	of_node_put(node: np);
4449
4450	/* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4451	* the base address of FIFO memory space.
4452	* Actually we need twice more physical memory than programmed in the
4453	* <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4454	*/
4455	ppc440spe_dma_fifo_buf = kmalloc(size: (DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4456	GFP_KERNEL);
4457	if (!ppc440spe_dma_fifo_buf) {
4458	pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4459	iounmap(addr: i2o_reg);
4460	dcr_unmap(i2o_dcr_host, dcr_len);
4461	return -ENOMEM;
4462	}
4463
4464	/*
4465	* Configure h/w
4466	*/
4467	/* Reset I2O/DMA */
4468	mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4469	mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4470
4471	/* Setup the base address of mmaped registers */
4472	dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4473	dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4474	I2O_REG_ENABLE);
4475	dcr_unmap(i2o_dcr_host, dcr_len);
4476
4477	/* Setup FIFO memory space base address */
4478	iowrite32(0, &i2o_reg->ifbah);
4479	iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4480
4481	/* set zero FIFO size for I2O, so the whole
4482	* ppc440spe_dma_fifo_buf is used by DMAs.
4483	* DMAx_FIFOs will be configured while probe.
4484	*/
4485	iowrite32(0, &i2o_reg->ifsiz);
4486	iounmap(addr: i2o_reg);
4487
4488	/* To prepare WXOR/RXOR functionality we need access to
4489	* Memory Queue Module DCRs (finally it will be enabled
4490	* via /sys interface of the ppc440spe ADMA driver).
4491	*/
4492	np = of_find_compatible_node(NULL, NULL, compat: "ibm,mq-440spe");
4493	if (!np) {
4494	pr_err("%s: can't find MQ device tree node\n",
4495	__func__);
4496	ret = -ENODEV;
4497	goto out_free;
4498	}
4499
4500	/* Get MQ DCRs base */
4501	dcr_base = dcr_resource_start(np, 0);
4502	dcr_len = dcr_resource_len(np, 0);
4503	if (!dcr_base && !dcr_len) {
4504	pr_err("%pOF: can't get DCR registers base/len!\n", np);
4505	ret = -ENODEV;
4506	goto out_mq;
4507	}
4508
4509	ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4510	if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4511	pr_err("%pOF: failed to map DCRs!\n", np);
4512	ret = -ENODEV;
4513	goto out_mq;
4514	}
4515	of_node_put(node: np);
4516	ppc440spe_mq_dcr_len = dcr_len;
4517
4518	/* Set HB alias */
4519	dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4520
4521	/* Set:
4522	* - LL transaction passing limit to 1;
4523	* - Memory controller cycle limit to 1;
4524	* - Galois Polynomial to 0x14d (default)
4525	*/
4526	dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4527	(1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4528	(PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4529
4530	atomic_set(v: &ppc440spe_adma_err_irq_ref, i: 0);
4531	for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4532	ppc440spe_adma_devices[i] = -1;
4533
4534	return 0;
4535
4536	out_mq:
4537	of_node_put(node: np);
4538	out_free:
4539	kfree(objp: ppc440spe_dma_fifo_buf);
4540	return ret;
4541	}
4542
4543	static const struct of_device_id ppc440spe_adma_of_match[] = {
4544	{ .compatible = "ibm,dma-440spe", },
4545	{ .compatible = "amcc,xor-accelerator", },
4546	{},
4547	};
4548	MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4549
4550	static struct platform_driver ppc440spe_adma_driver = {
4551	.probe = ppc440spe_adma_probe,
4552	.remove_new = ppc440spe_adma_remove,
4553	.driver = {
4554	.name = "PPC440SP(E)-ADMA",
4555	.of_match_table = ppc440spe_adma_of_match,
4556	},
4557	};
4558
4559	static __init int ppc440spe_adma_init(void)
4560	{
4561	int ret;
4562
4563	ret = ppc440spe_configure_raid_devices();
4564	if (ret)
4565	return ret;
4566
4567	ret = platform_driver_register(&ppc440spe_adma_driver);
4568	if (ret) {
4569	pr_err("%s: failed to register platform driver\n",
4570	__func__);
4571	goto out_reg;
4572	}
4573
4574	/* Initialization status */
4575	ret = driver_create_file(driver: &ppc440spe_adma_driver.driver,
4576	attr: &driver_attr_devices);
4577	if (ret)
4578	goto out_dev;
4579
4580	/* RAID-6 h/w enable entry */
4581	ret = driver_create_file(driver: &ppc440spe_adma_driver.driver,
4582	attr: &driver_attr_enable);
4583	if (ret)
4584	goto out_en;
4585
4586	/* GF polynomial to use */
4587	ret = driver_create_file(driver: &ppc440spe_adma_driver.driver,
4588	attr: &driver_attr_poly);
4589	if (!ret)
4590	return ret;
4591
4592	driver_remove_file(driver: &ppc440spe_adma_driver.driver,
4593	attr: &driver_attr_enable);
4594	out_en:
4595	driver_remove_file(driver: &ppc440spe_adma_driver.driver,
4596	attr: &driver_attr_devices);
4597	out_dev:
4598	/* User will not be able to enable h/w RAID-6 */
4599	pr_err("%s: failed to create RAID-6 driver interface\n",
4600	__func__);
4601	platform_driver_unregister(&ppc440spe_adma_driver);
4602	out_reg:
4603	dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4604	kfree(objp: ppc440spe_dma_fifo_buf);
4605	return ret;
4606	}
4607
4608	static void __exit ppc440spe_adma_exit(void)
4609	{
4610	driver_remove_file(driver: &ppc440spe_adma_driver.driver,
4611	attr: &driver_attr_poly);
4612	driver_remove_file(driver: &ppc440spe_adma_driver.driver,
4613	attr: &driver_attr_enable);
4614	driver_remove_file(driver: &ppc440spe_adma_driver.driver,
4615	attr: &driver_attr_devices);
4616	platform_driver_unregister(&ppc440spe_adma_driver);
4617	dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4618	kfree(objp: ppc440spe_dma_fifo_buf);
4619	}
4620
4621	arch_initcall(ppc440spe_adma_init);
4622	module_exit(ppc440spe_adma_exit);
4623
4624	MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
4625	MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
4626	MODULE_LICENSE("GPL");
4627