1 | /* |
2 | * drivers/dma/fsl_raid.c |
3 | * |
4 | * Freescale RAID Engine device driver |
5 | * |
6 | * Author: |
7 | * Harninder Rai <harninder.rai@freescale.com> |
8 | * Naveen Burmi <naveenburmi@freescale.com> |
9 | * |
10 | * Rewrite: |
11 | * Xuelin Shi <xuelin.shi@freescale.com> |
12 | * |
13 | * Copyright (c) 2010-2014 Freescale Semiconductor, Inc. |
14 | * |
15 | * Redistribution and use in source and binary forms, with or without |
16 | * modification, are permitted provided that the following conditions are met: |
17 | * * Redistributions of source code must retain the above copyright |
18 | * notice, this list of conditions and the following disclaimer. |
19 | * * Redistributions in binary form must reproduce the above copyright |
20 | * notice, this list of conditions and the following disclaimer in the |
21 | * documentation and/or other materials provided with the distribution. |
22 | * * Neither the name of Freescale Semiconductor nor the |
23 | * names of its contributors may be used to endorse or promote products |
24 | * derived from this software without specific prior written permission. |
25 | * |
26 | * ALTERNATIVELY, this software may be distributed under the terms of the |
27 | * GNU General Public License ("GPL") as published by the Free Software |
28 | * Foundation, either version 2 of that License or (at your option) any |
29 | * later version. |
30 | * |
31 | * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY |
32 | * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
33 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
34 | * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY |
35 | * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
36 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
37 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
38 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
39 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
40 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
41 | * |
42 | * Theory of operation: |
43 | * |
44 | * General capabilities: |
45 | * RAID Engine (RE) block is capable of offloading XOR, memcpy and P/Q |
46 | * calculations required in RAID5 and RAID6 operations. RE driver |
47 | * registers with Linux's ASYNC layer as dma driver. RE hardware |
48 | * maintains strict ordering of the requests through chained |
49 | * command queueing. |
50 | * |
51 | * Data flow: |
52 | * Software RAID layer of Linux (MD layer) maintains RAID partitions, |
53 | * strips, stripes etc. It sends requests to the underlying ASYNC layer |
54 | * which further passes it to RE driver. ASYNC layer decides which request |
55 | * goes to which job ring of RE hardware. For every request processed by |
56 | * RAID Engine, driver gets an interrupt unless coalescing is set. The |
57 | * per job ring interrupt handler checks the status register for errors, |
58 | * clears the interrupt and leave the post interrupt processing to the irq |
59 | * thread. |
60 | */ |
61 | #include <linux/interrupt.h> |
62 | #include <linux/module.h> |
63 | #include <linux/of.h> |
64 | #include <linux/of_irq.h> |
65 | #include <linux/of_platform.h> |
66 | #include <linux/platform_device.h> |
67 | #include <linux/dma-mapping.h> |
68 | #include <linux/dmapool.h> |
69 | #include <linux/dmaengine.h> |
70 | #include <linux/io.h> |
71 | #include <linux/spinlock.h> |
72 | #include <linux/slab.h> |
73 | |
74 | #include "dmaengine.h" |
75 | #include "fsl_raid.h" |
76 | |
77 | #define FSL_RE_MAX_XOR_SRCS 16 |
78 | #define FSL_RE_MAX_PQ_SRCS 16 |
79 | #define FSL_RE_MIN_DESCS 256 |
80 | #define FSL_RE_MAX_DESCS (4 * FSL_RE_MIN_DESCS) |
81 | #define FSL_RE_FRAME_FORMAT 0x1 |
82 | #define FSL_RE_MAX_DATA_LEN (1024*1024) |
83 | |
84 | #define to_fsl_re_dma_desc(tx) container_of(tx, struct fsl_re_desc, async_tx) |
85 | |
86 | /* Add descriptors into per chan software queue - submit_q */ |
87 | static dma_cookie_t fsl_re_tx_submit(struct dma_async_tx_descriptor *tx) |
88 | { |
89 | struct fsl_re_desc *desc; |
90 | struct fsl_re_chan *re_chan; |
91 | dma_cookie_t cookie; |
92 | unsigned long flags; |
93 | |
94 | desc = to_fsl_re_dma_desc(tx); |
95 | re_chan = container_of(tx->chan, struct fsl_re_chan, chan); |
96 | |
97 | spin_lock_irqsave(&re_chan->desc_lock, flags); |
98 | cookie = dma_cookie_assign(tx); |
99 | list_add_tail(new: &desc->node, head: &re_chan->submit_q); |
100 | spin_unlock_irqrestore(lock: &re_chan->desc_lock, flags); |
101 | |
102 | return cookie; |
103 | } |
104 | |
105 | /* Copy descriptor from per chan software queue into hardware job ring */ |
106 | static void fsl_re_issue_pending(struct dma_chan *chan) |
107 | { |
108 | struct fsl_re_chan *re_chan; |
109 | int avail; |
110 | struct fsl_re_desc *desc, *_desc; |
111 | unsigned long flags; |
112 | |
113 | re_chan = container_of(chan, struct fsl_re_chan, chan); |
114 | |
115 | spin_lock_irqsave(&re_chan->desc_lock, flags); |
116 | avail = FSL_RE_SLOT_AVAIL( |
117 | in_be32(&re_chan->jrregs->inbring_slot_avail)); |
118 | |
119 | list_for_each_entry_safe(desc, _desc, &re_chan->submit_q, node) { |
120 | if (!avail) |
121 | break; |
122 | |
123 | list_move_tail(list: &desc->node, head: &re_chan->active_q); |
124 | |
125 | memcpy(&re_chan->inb_ring_virt_addr[re_chan->inb_count], |
126 | &desc->hwdesc, sizeof(struct fsl_re_hw_desc)); |
127 | |
128 | re_chan->inb_count = (re_chan->inb_count + 1) & |
129 | FSL_RE_RING_SIZE_MASK; |
130 | out_be32(&re_chan->jrregs->inbring_add_job, FSL_RE_ADD_JOB(1)); |
131 | avail--; |
132 | } |
133 | spin_unlock_irqrestore(lock: &re_chan->desc_lock, flags); |
134 | } |
135 | |
136 | static void fsl_re_desc_done(struct fsl_re_desc *desc) |
137 | { |
138 | dma_cookie_complete(tx: &desc->async_tx); |
139 | dma_descriptor_unmap(tx: &desc->async_tx); |
140 | dmaengine_desc_get_callback_invoke(tx: &desc->async_tx, NULL); |
141 | } |
142 | |
143 | static void fsl_re_cleanup_descs(struct fsl_re_chan *re_chan) |
144 | { |
145 | struct fsl_re_desc *desc, *_desc; |
146 | unsigned long flags; |
147 | |
148 | spin_lock_irqsave(&re_chan->desc_lock, flags); |
149 | list_for_each_entry_safe(desc, _desc, &re_chan->ack_q, node) { |
150 | if (async_tx_test_ack(tx: &desc->async_tx)) |
151 | list_move_tail(list: &desc->node, head: &re_chan->free_q); |
152 | } |
153 | spin_unlock_irqrestore(lock: &re_chan->desc_lock, flags); |
154 | |
155 | fsl_re_issue_pending(chan: &re_chan->chan); |
156 | } |
157 | |
158 | static void fsl_re_dequeue(struct tasklet_struct *t) |
159 | { |
160 | struct fsl_re_chan *re_chan = from_tasklet(re_chan, t, irqtask); |
161 | struct fsl_re_desc *desc, *_desc; |
162 | struct fsl_re_hw_desc *hwdesc; |
163 | unsigned long flags; |
164 | unsigned int count, oub_count; |
165 | int found; |
166 | |
167 | fsl_re_cleanup_descs(re_chan); |
168 | |
169 | spin_lock_irqsave(&re_chan->desc_lock, flags); |
170 | count = FSL_RE_SLOT_FULL(in_be32(&re_chan->jrregs->oubring_slot_full)); |
171 | while (count--) { |
172 | found = 0; |
173 | hwdesc = &re_chan->oub_ring_virt_addr[re_chan->oub_count]; |
174 | list_for_each_entry_safe(desc, _desc, &re_chan->active_q, |
175 | node) { |
176 | /* compare the hw dma addr to find the completed */ |
177 | if (desc->hwdesc.lbea32 == hwdesc->lbea32 && |
178 | desc->hwdesc.addr_low == hwdesc->addr_low) { |
179 | found = 1; |
180 | break; |
181 | } |
182 | } |
183 | |
184 | if (found) { |
185 | fsl_re_desc_done(desc); |
186 | list_move_tail(list: &desc->node, head: &re_chan->ack_q); |
187 | } else { |
188 | dev_err(re_chan->dev, |
189 | "found hwdesc not in sw queue, discard it\n" ); |
190 | } |
191 | |
192 | oub_count = (re_chan->oub_count + 1) & FSL_RE_RING_SIZE_MASK; |
193 | re_chan->oub_count = oub_count; |
194 | |
195 | out_be32(&re_chan->jrregs->oubring_job_rmvd, |
196 | FSL_RE_RMVD_JOB(1)); |
197 | } |
198 | spin_unlock_irqrestore(lock: &re_chan->desc_lock, flags); |
199 | } |
200 | |
201 | /* Per Job Ring interrupt handler */ |
202 | static irqreturn_t fsl_re_isr(int irq, void *data) |
203 | { |
204 | struct fsl_re_chan *re_chan; |
205 | u32 irqstate, status; |
206 | |
207 | re_chan = dev_get_drvdata(dev: (struct device *)data); |
208 | |
209 | irqstate = in_be32(&re_chan->jrregs->jr_interrupt_status); |
210 | if (!irqstate) |
211 | return IRQ_NONE; |
212 | |
213 | /* |
214 | * There's no way in upper layer (read MD layer) to recover from |
215 | * error conditions except restart everything. In long term we |
216 | * need to do something more than just crashing |
217 | */ |
218 | if (irqstate & FSL_RE_ERROR) { |
219 | status = in_be32(&re_chan->jrregs->jr_status); |
220 | dev_err(re_chan->dev, "chan error irqstate: %x, status: %x\n" , |
221 | irqstate, status); |
222 | } |
223 | |
224 | /* Clear interrupt */ |
225 | out_be32(&re_chan->jrregs->jr_interrupt_status, FSL_RE_CLR_INTR); |
226 | |
227 | tasklet_schedule(t: &re_chan->irqtask); |
228 | |
229 | return IRQ_HANDLED; |
230 | } |
231 | |
232 | static enum dma_status fsl_re_tx_status(struct dma_chan *chan, |
233 | dma_cookie_t cookie, |
234 | struct dma_tx_state *txstate) |
235 | { |
236 | return dma_cookie_status(chan, cookie, state: txstate); |
237 | } |
238 | |
239 | static void fill_cfd_frame(struct fsl_re_cmpnd_frame *cf, u8 index, |
240 | size_t length, dma_addr_t addr, bool final) |
241 | { |
242 | u32 efrl = length & FSL_RE_CF_LENGTH_MASK; |
243 | |
244 | efrl |= final << FSL_RE_CF_FINAL_SHIFT; |
245 | cf[index].efrl32 = efrl; |
246 | cf[index].addr_high = upper_32_bits(addr); |
247 | cf[index].addr_low = lower_32_bits(addr); |
248 | } |
249 | |
250 | static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan, |
251 | struct fsl_re_desc *desc, |
252 | void *cf, dma_addr_t paddr) |
253 | { |
254 | desc->re_chan = re_chan; |
255 | desc->async_tx.tx_submit = fsl_re_tx_submit; |
256 | dma_async_tx_descriptor_init(tx: &desc->async_tx, chan: &re_chan->chan); |
257 | INIT_LIST_HEAD(list: &desc->node); |
258 | |
259 | desc->hwdesc.fmt32 = FSL_RE_FRAME_FORMAT << FSL_RE_HWDESC_FMT_SHIFT; |
260 | desc->hwdesc.lbea32 = upper_32_bits(paddr); |
261 | desc->hwdesc.addr_low = lower_32_bits(paddr); |
262 | desc->cf_addr = cf; |
263 | desc->cf_paddr = paddr; |
264 | |
265 | desc->cdb_addr = (void *)(cf + FSL_RE_CF_DESC_SIZE); |
266 | desc->cdb_paddr = paddr + FSL_RE_CF_DESC_SIZE; |
267 | |
268 | return desc; |
269 | } |
270 | |
271 | static struct fsl_re_desc *fsl_re_chan_alloc_desc(struct fsl_re_chan *re_chan, |
272 | unsigned long flags) |
273 | { |
274 | struct fsl_re_desc *desc = NULL; |
275 | void *cf; |
276 | dma_addr_t paddr; |
277 | unsigned long lock_flag; |
278 | |
279 | fsl_re_cleanup_descs(re_chan); |
280 | |
281 | spin_lock_irqsave(&re_chan->desc_lock, lock_flag); |
282 | if (!list_empty(head: &re_chan->free_q)) { |
283 | /* take one desc from free_q */ |
284 | desc = list_first_entry(&re_chan->free_q, |
285 | struct fsl_re_desc, node); |
286 | list_del(entry: &desc->node); |
287 | |
288 | desc->async_tx.flags = flags; |
289 | } |
290 | spin_unlock_irqrestore(lock: &re_chan->desc_lock, flags: lock_flag); |
291 | |
292 | if (!desc) { |
293 | desc = kzalloc(size: sizeof(*desc), GFP_NOWAIT); |
294 | if (!desc) |
295 | return NULL; |
296 | |
297 | cf = dma_pool_alloc(pool: re_chan->re_dev->cf_desc_pool, GFP_NOWAIT, |
298 | handle: &paddr); |
299 | if (!cf) { |
300 | kfree(objp: desc); |
301 | return NULL; |
302 | } |
303 | |
304 | desc = fsl_re_init_desc(re_chan, desc, cf, paddr); |
305 | desc->async_tx.flags = flags; |
306 | |
307 | spin_lock_irqsave(&re_chan->desc_lock, lock_flag); |
308 | re_chan->alloc_count++; |
309 | spin_unlock_irqrestore(lock: &re_chan->desc_lock, flags: lock_flag); |
310 | } |
311 | |
312 | return desc; |
313 | } |
314 | |
315 | static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq( |
316 | struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, |
317 | unsigned int src_cnt, const unsigned char *scf, size_t len, |
318 | unsigned long flags) |
319 | { |
320 | struct fsl_re_chan *re_chan; |
321 | struct fsl_re_desc *desc; |
322 | struct fsl_re_xor_cdb *xor; |
323 | struct fsl_re_cmpnd_frame *cf; |
324 | u32 cdb; |
325 | unsigned int i, j; |
326 | unsigned int save_src_cnt = src_cnt; |
327 | int cont_q = 0; |
328 | |
329 | re_chan = container_of(chan, struct fsl_re_chan, chan); |
330 | if (len > FSL_RE_MAX_DATA_LEN) { |
331 | dev_err(re_chan->dev, "genq tx length %zu, max length %d\n" , |
332 | len, FSL_RE_MAX_DATA_LEN); |
333 | return NULL; |
334 | } |
335 | |
336 | desc = fsl_re_chan_alloc_desc(re_chan, flags); |
337 | if (desc <= 0) |
338 | return NULL; |
339 | |
340 | if (scf && (flags & DMA_PREP_CONTINUE)) { |
341 | cont_q = 1; |
342 | src_cnt += 1; |
343 | } |
344 | |
345 | /* Filling xor CDB */ |
346 | cdb = FSL_RE_XOR_OPCODE << FSL_RE_CDB_OPCODE_SHIFT; |
347 | cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT; |
348 | cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT; |
349 | cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT; |
350 | cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; |
351 | xor = desc->cdb_addr; |
352 | xor->cdb32 = cdb; |
353 | |
354 | if (scf) { |
355 | /* compute q = src0*coef0^src1*coef1^..., * is GF(8) mult */ |
356 | for (i = 0; i < save_src_cnt; i++) |
357 | xor->gfm[i] = scf[i]; |
358 | if (cont_q) |
359 | xor->gfm[i++] = 1; |
360 | } else { |
361 | /* compute P, that is XOR all srcs */ |
362 | for (i = 0; i < src_cnt; i++) |
363 | xor->gfm[i] = 1; |
364 | } |
365 | |
366 | /* Filling frame 0 of compound frame descriptor with CDB */ |
367 | cf = desc->cf_addr; |
368 | fill_cfd_frame(cf, index: 0, length: sizeof(*xor), addr: desc->cdb_paddr, final: 0); |
369 | |
370 | /* Fill CFD's 1st frame with dest buffer */ |
371 | fill_cfd_frame(cf, index: 1, length: len, addr: dest, final: 0); |
372 | |
373 | /* Fill CFD's rest of the frames with source buffers */ |
374 | for (i = 2, j = 0; j < save_src_cnt; i++, j++) |
375 | fill_cfd_frame(cf, index: i, length: len, addr: src[j], final: 0); |
376 | |
377 | if (cont_q) |
378 | fill_cfd_frame(cf, index: i++, length: len, addr: dest, final: 0); |
379 | |
380 | /* Setting the final bit in the last source buffer frame in CFD */ |
381 | cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT; |
382 | |
383 | return &desc->async_tx; |
384 | } |
385 | |
386 | /* |
387 | * Prep function for P parity calculation.In RAID Engine terminology, |
388 | * XOR calculation is called GenQ calculation done through GenQ command |
389 | */ |
390 | static struct dma_async_tx_descriptor *fsl_re_prep_dma_xor( |
391 | struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, |
392 | unsigned int src_cnt, size_t len, unsigned long flags) |
393 | { |
394 | /* NULL let genq take all coef as 1 */ |
395 | return fsl_re_prep_dma_genq(chan, dest, src, src_cnt, NULL, len, flags); |
396 | } |
397 | |
398 | /* |
399 | * Prep function for P/Q parity calculation.In RAID Engine terminology, |
400 | * P/Q calculation is called GenQQ done through GenQQ command |
401 | */ |
402 | static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq( |
403 | struct dma_chan *chan, dma_addr_t *dest, dma_addr_t *src, |
404 | unsigned int src_cnt, const unsigned char *scf, size_t len, |
405 | unsigned long flags) |
406 | { |
407 | struct fsl_re_chan *re_chan; |
408 | struct fsl_re_desc *desc; |
409 | struct fsl_re_pq_cdb *pq; |
410 | struct fsl_re_cmpnd_frame *cf; |
411 | u32 cdb; |
412 | u8 *p; |
413 | int gfmq_len, i, j; |
414 | unsigned int save_src_cnt = src_cnt; |
415 | |
416 | re_chan = container_of(chan, struct fsl_re_chan, chan); |
417 | if (len > FSL_RE_MAX_DATA_LEN) { |
418 | dev_err(re_chan->dev, "pq tx length is %zu, max length is %d\n" , |
419 | len, FSL_RE_MAX_DATA_LEN); |
420 | return NULL; |
421 | } |
422 | |
423 | /* |
424 | * RE requires at least 2 sources, if given only one source, we pass the |
425 | * second source same as the first one. |
426 | * With only one source, generating P is meaningless, only generate Q. |
427 | */ |
428 | if (src_cnt == 1) { |
429 | struct dma_async_tx_descriptor *tx; |
430 | dma_addr_t dma_src[2]; |
431 | unsigned char coef[2]; |
432 | |
433 | dma_src[0] = *src; |
434 | coef[0] = *scf; |
435 | dma_src[1] = *src; |
436 | coef[1] = 0; |
437 | tx = fsl_re_prep_dma_genq(chan, dest: dest[1], src: dma_src, src_cnt: 2, scf: coef, len, |
438 | flags); |
439 | if (tx) |
440 | desc = to_fsl_re_dma_desc(tx); |
441 | |
442 | return tx; |
443 | } |
444 | |
445 | /* |
446 | * During RAID6 array creation, Linux's MD layer gets P and Q |
447 | * calculated separately in two steps. But our RAID Engine has |
448 | * the capability to calculate both P and Q with a single command |
449 | * Hence to merge well with MD layer, we need to provide a hook |
450 | * here and call re_jq_prep_dma_genq() function |
451 | */ |
452 | |
453 | if (flags & DMA_PREP_PQ_DISABLE_P) |
454 | return fsl_re_prep_dma_genq(chan, dest: dest[1], src, src_cnt, |
455 | scf, len, flags); |
456 | |
457 | if (flags & DMA_PREP_CONTINUE) |
458 | src_cnt += 3; |
459 | |
460 | desc = fsl_re_chan_alloc_desc(re_chan, flags); |
461 | if (desc <= 0) |
462 | return NULL; |
463 | |
464 | /* Filling GenQQ CDB */ |
465 | cdb = FSL_RE_PQ_OPCODE << FSL_RE_CDB_OPCODE_SHIFT; |
466 | cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT; |
467 | cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT; |
468 | cdb |= FSL_RE_BUFFER_OUTPUT << FSL_RE_CDB_BUFFER_SHIFT; |
469 | cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; |
470 | |
471 | pq = desc->cdb_addr; |
472 | pq->cdb32 = cdb; |
473 | |
474 | p = pq->gfm_q1; |
475 | /* Init gfm_q1[] */ |
476 | for (i = 0; i < src_cnt; i++) |
477 | p[i] = 1; |
478 | |
479 | /* Align gfm[] to 32bit */ |
480 | gfmq_len = ALIGN(src_cnt, 4); |
481 | |
482 | /* Init gfm_q2[] */ |
483 | p += gfmq_len; |
484 | for (i = 0; i < src_cnt; i++) |
485 | p[i] = scf[i]; |
486 | |
487 | /* Filling frame 0 of compound frame descriptor with CDB */ |
488 | cf = desc->cf_addr; |
489 | fill_cfd_frame(cf, index: 0, length: sizeof(struct fsl_re_pq_cdb), addr: desc->cdb_paddr, final: 0); |
490 | |
491 | /* Fill CFD's 1st & 2nd frame with dest buffers */ |
492 | for (i = 1, j = 0; i < 3; i++, j++) |
493 | fill_cfd_frame(cf, index: i, length: len, addr: dest[j], final: 0); |
494 | |
495 | /* Fill CFD's rest of the frames with source buffers */ |
496 | for (i = 3, j = 0; j < save_src_cnt; i++, j++) |
497 | fill_cfd_frame(cf, index: i, length: len, addr: src[j], final: 0); |
498 | |
499 | /* PQ computation continuation */ |
500 | if (flags & DMA_PREP_CONTINUE) { |
501 | if (src_cnt - save_src_cnt == 3) { |
502 | p[save_src_cnt] = 0; |
503 | p[save_src_cnt + 1] = 0; |
504 | p[save_src_cnt + 2] = 1; |
505 | fill_cfd_frame(cf, index: i++, length: len, addr: dest[0], final: 0); |
506 | fill_cfd_frame(cf, index: i++, length: len, addr: dest[1], final: 0); |
507 | fill_cfd_frame(cf, index: i++, length: len, addr: dest[1], final: 0); |
508 | } else { |
509 | dev_err(re_chan->dev, "PQ tx continuation error!\n" ); |
510 | return NULL; |
511 | } |
512 | } |
513 | |
514 | /* Setting the final bit in the last source buffer frame in CFD */ |
515 | cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT; |
516 | |
517 | return &desc->async_tx; |
518 | } |
519 | |
520 | /* |
521 | * Prep function for memcpy. In RAID Engine, memcpy is done through MOVE |
522 | * command. Logic of this function will need to be modified once multipage |
523 | * support is added in Linux's MD/ASYNC Layer |
524 | */ |
525 | static struct dma_async_tx_descriptor *fsl_re_prep_dma_memcpy( |
526 | struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, |
527 | size_t len, unsigned long flags) |
528 | { |
529 | struct fsl_re_chan *re_chan; |
530 | struct fsl_re_desc *desc; |
531 | size_t length; |
532 | struct fsl_re_cmpnd_frame *cf; |
533 | struct fsl_re_move_cdb *move; |
534 | u32 cdb; |
535 | |
536 | re_chan = container_of(chan, struct fsl_re_chan, chan); |
537 | |
538 | if (len > FSL_RE_MAX_DATA_LEN) { |
539 | dev_err(re_chan->dev, "cp tx length is %zu, max length is %d\n" , |
540 | len, FSL_RE_MAX_DATA_LEN); |
541 | return NULL; |
542 | } |
543 | |
544 | desc = fsl_re_chan_alloc_desc(re_chan, flags); |
545 | if (desc <= 0) |
546 | return NULL; |
547 | |
548 | /* Filling move CDB */ |
549 | cdb = FSL_RE_MOVE_OPCODE << FSL_RE_CDB_OPCODE_SHIFT; |
550 | cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT; |
551 | cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT; |
552 | cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; |
553 | |
554 | move = desc->cdb_addr; |
555 | move->cdb32 = cdb; |
556 | |
557 | /* Filling frame 0 of CFD with move CDB */ |
558 | cf = desc->cf_addr; |
559 | fill_cfd_frame(cf, index: 0, length: sizeof(*move), addr: desc->cdb_paddr, final: 0); |
560 | |
561 | length = min_t(size_t, len, FSL_RE_MAX_DATA_LEN); |
562 | |
563 | /* Fill CFD's 1st frame with dest buffer */ |
564 | fill_cfd_frame(cf, index: 1, length, addr: dest, final: 0); |
565 | |
566 | /* Fill CFD's 2nd frame with src buffer */ |
567 | fill_cfd_frame(cf, index: 2, length, addr: src, final: 1); |
568 | |
569 | return &desc->async_tx; |
570 | } |
571 | |
572 | static int fsl_re_alloc_chan_resources(struct dma_chan *chan) |
573 | { |
574 | struct fsl_re_chan *re_chan; |
575 | struct fsl_re_desc *desc; |
576 | void *cf; |
577 | dma_addr_t paddr; |
578 | int i; |
579 | |
580 | re_chan = container_of(chan, struct fsl_re_chan, chan); |
581 | for (i = 0; i < FSL_RE_MIN_DESCS; i++) { |
582 | desc = kzalloc(size: sizeof(*desc), GFP_KERNEL); |
583 | if (!desc) |
584 | break; |
585 | |
586 | cf = dma_pool_alloc(pool: re_chan->re_dev->cf_desc_pool, GFP_KERNEL, |
587 | handle: &paddr); |
588 | if (!cf) { |
589 | kfree(objp: desc); |
590 | break; |
591 | } |
592 | |
593 | INIT_LIST_HEAD(list: &desc->node); |
594 | fsl_re_init_desc(re_chan, desc, cf, paddr); |
595 | |
596 | list_add_tail(new: &desc->node, head: &re_chan->free_q); |
597 | re_chan->alloc_count++; |
598 | } |
599 | return re_chan->alloc_count; |
600 | } |
601 | |
602 | static void fsl_re_free_chan_resources(struct dma_chan *chan) |
603 | { |
604 | struct fsl_re_chan *re_chan; |
605 | struct fsl_re_desc *desc; |
606 | |
607 | re_chan = container_of(chan, struct fsl_re_chan, chan); |
608 | while (re_chan->alloc_count--) { |
609 | desc = list_first_entry(&re_chan->free_q, |
610 | struct fsl_re_desc, |
611 | node); |
612 | |
613 | list_del(entry: &desc->node); |
614 | dma_pool_free(pool: re_chan->re_dev->cf_desc_pool, vaddr: desc->cf_addr, |
615 | addr: desc->cf_paddr); |
616 | kfree(objp: desc); |
617 | } |
618 | |
619 | if (!list_empty(head: &re_chan->free_q)) |
620 | dev_err(re_chan->dev, "chan resource cannot be cleaned!\n" ); |
621 | } |
622 | |
623 | static int fsl_re_chan_probe(struct platform_device *ofdev, |
624 | struct device_node *np, u8 q, u32 off) |
625 | { |
626 | struct device *dev, *chandev; |
627 | struct fsl_re_drv_private *re_priv; |
628 | struct fsl_re_chan *chan; |
629 | struct dma_device *dma_dev; |
630 | u32 ptr; |
631 | u32 status; |
632 | int ret = 0, rc; |
633 | struct platform_device *chan_ofdev; |
634 | |
635 | dev = &ofdev->dev; |
636 | re_priv = dev_get_drvdata(dev); |
637 | dma_dev = &re_priv->dma_dev; |
638 | |
639 | chan = devm_kzalloc(dev, size: sizeof(*chan), GFP_KERNEL); |
640 | if (!chan) |
641 | return -ENOMEM; |
642 | |
643 | /* create platform device for chan node */ |
644 | chan_ofdev = of_platform_device_create(np, NULL, parent: dev); |
645 | if (!chan_ofdev) { |
646 | dev_err(dev, "Not able to create ofdev for jr %d\n" , q); |
647 | ret = -EINVAL; |
648 | goto err_free; |
649 | } |
650 | |
651 | /* read reg property from dts */ |
652 | rc = of_property_read_u32(np, propname: "reg" , out_value: &ptr); |
653 | if (rc) { |
654 | dev_err(dev, "Reg property not found in jr %d\n" , q); |
655 | ret = -ENODEV; |
656 | goto err_free; |
657 | } |
658 | |
659 | chan->jrregs = (struct fsl_re_chan_cfg *)((u8 *)re_priv->re_regs + |
660 | off + ptr); |
661 | |
662 | /* read irq property from dts */ |
663 | chan->irq = irq_of_parse_and_map(node: np, index: 0); |
664 | if (!chan->irq) { |
665 | dev_err(dev, "No IRQ defined for JR %d\n" , q); |
666 | ret = -ENODEV; |
667 | goto err_free; |
668 | } |
669 | |
670 | snprintf(buf: chan->name, size: sizeof(chan->name), fmt: "re_jr%02d" , q); |
671 | |
672 | chandev = &chan_ofdev->dev; |
673 | tasklet_setup(t: &chan->irqtask, callback: fsl_re_dequeue); |
674 | |
675 | ret = request_irq(irq: chan->irq, handler: fsl_re_isr, flags: 0, name: chan->name, dev: chandev); |
676 | if (ret) { |
677 | dev_err(dev, "Unable to register interrupt for JR %d\n" , q); |
678 | ret = -EINVAL; |
679 | goto err_free; |
680 | } |
681 | |
682 | re_priv->re_jrs[q] = chan; |
683 | chan->chan.device = dma_dev; |
684 | chan->chan.private = chan; |
685 | chan->dev = chandev; |
686 | chan->re_dev = re_priv; |
687 | |
688 | spin_lock_init(&chan->desc_lock); |
689 | INIT_LIST_HEAD(list: &chan->ack_q); |
690 | INIT_LIST_HEAD(list: &chan->active_q); |
691 | INIT_LIST_HEAD(list: &chan->submit_q); |
692 | INIT_LIST_HEAD(list: &chan->free_q); |
693 | |
694 | chan->inb_ring_virt_addr = dma_pool_alloc(pool: chan->re_dev->hw_desc_pool, |
695 | GFP_KERNEL, handle: &chan->inb_phys_addr); |
696 | if (!chan->inb_ring_virt_addr) { |
697 | dev_err(dev, "No dma memory for inb_ring_virt_addr\n" ); |
698 | ret = -ENOMEM; |
699 | goto err_free; |
700 | } |
701 | |
702 | chan->oub_ring_virt_addr = dma_pool_alloc(pool: chan->re_dev->hw_desc_pool, |
703 | GFP_KERNEL, handle: &chan->oub_phys_addr); |
704 | if (!chan->oub_ring_virt_addr) { |
705 | dev_err(dev, "No dma memory for oub_ring_virt_addr\n" ); |
706 | ret = -ENOMEM; |
707 | goto err_free_1; |
708 | } |
709 | |
710 | /* Program the Inbound/Outbound ring base addresses and size */ |
711 | out_be32(&chan->jrregs->inbring_base_h, |
712 | chan->inb_phys_addr & FSL_RE_ADDR_BIT_MASK); |
713 | out_be32(&chan->jrregs->oubring_base_h, |
714 | chan->oub_phys_addr & FSL_RE_ADDR_BIT_MASK); |
715 | out_be32(&chan->jrregs->inbring_base_l, |
716 | chan->inb_phys_addr >> FSL_RE_ADDR_BIT_SHIFT); |
717 | out_be32(&chan->jrregs->oubring_base_l, |
718 | chan->oub_phys_addr >> FSL_RE_ADDR_BIT_SHIFT); |
719 | out_be32(&chan->jrregs->inbring_size, |
720 | FSL_RE_RING_SIZE << FSL_RE_RING_SIZE_SHIFT); |
721 | out_be32(&chan->jrregs->oubring_size, |
722 | FSL_RE_RING_SIZE << FSL_RE_RING_SIZE_SHIFT); |
723 | |
724 | /* Read LIODN value from u-boot */ |
725 | status = in_be32(&chan->jrregs->jr_config_1) & FSL_RE_REG_LIODN_MASK; |
726 | |
727 | /* Program the CFG reg */ |
728 | out_be32(&chan->jrregs->jr_config_1, |
729 | FSL_RE_CFG1_CBSI | FSL_RE_CFG1_CBS0 | status); |
730 | |
731 | dev_set_drvdata(dev: chandev, data: chan); |
732 | |
733 | /* Enable RE/CHAN */ |
734 | out_be32(&chan->jrregs->jr_command, FSL_RE_ENABLE); |
735 | |
736 | return 0; |
737 | |
738 | err_free_1: |
739 | dma_pool_free(pool: chan->re_dev->hw_desc_pool, vaddr: chan->inb_ring_virt_addr, |
740 | addr: chan->inb_phys_addr); |
741 | err_free: |
742 | return ret; |
743 | } |
744 | |
745 | /* Probe function for RAID Engine */ |
746 | static int fsl_re_probe(struct platform_device *ofdev) |
747 | { |
748 | struct fsl_re_drv_private *re_priv; |
749 | struct device_node *np; |
750 | struct device_node *child; |
751 | u32 off; |
752 | u8 ridx = 0; |
753 | struct dma_device *dma_dev; |
754 | struct resource *res; |
755 | int rc; |
756 | struct device *dev = &ofdev->dev; |
757 | |
758 | re_priv = devm_kzalloc(dev, size: sizeof(*re_priv), GFP_KERNEL); |
759 | if (!re_priv) |
760 | return -ENOMEM; |
761 | |
762 | res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); |
763 | if (!res) |
764 | return -ENODEV; |
765 | |
766 | /* IOMAP the entire RAID Engine region */ |
767 | re_priv->re_regs = devm_ioremap(dev, offset: res->start, size: resource_size(res)); |
768 | if (!re_priv->re_regs) |
769 | return -EBUSY; |
770 | |
771 | /* Program the RE mode */ |
772 | out_be32(&re_priv->re_regs->global_config, FSL_RE_NON_DPAA_MODE); |
773 | |
774 | /* Program Galois Field polynomial */ |
775 | out_be32(&re_priv->re_regs->galois_field_config, FSL_RE_GFM_POLY); |
776 | |
777 | dev_info(dev, "version %x, mode %x, gfp %x\n" , |
778 | in_be32(&re_priv->re_regs->re_version_id), |
779 | in_be32(&re_priv->re_regs->global_config), |
780 | in_be32(&re_priv->re_regs->galois_field_config)); |
781 | |
782 | dma_dev = &re_priv->dma_dev; |
783 | dma_dev->dev = dev; |
784 | INIT_LIST_HEAD(list: &dma_dev->channels); |
785 | dma_set_mask(dev, DMA_BIT_MASK(40)); |
786 | |
787 | dma_dev->device_alloc_chan_resources = fsl_re_alloc_chan_resources; |
788 | dma_dev->device_tx_status = fsl_re_tx_status; |
789 | dma_dev->device_issue_pending = fsl_re_issue_pending; |
790 | |
791 | dma_dev->max_xor = FSL_RE_MAX_XOR_SRCS; |
792 | dma_dev->device_prep_dma_xor = fsl_re_prep_dma_xor; |
793 | dma_cap_set(DMA_XOR, dma_dev->cap_mask); |
794 | |
795 | dma_dev->max_pq = FSL_RE_MAX_PQ_SRCS; |
796 | dma_dev->device_prep_dma_pq = fsl_re_prep_dma_pq; |
797 | dma_cap_set(DMA_PQ, dma_dev->cap_mask); |
798 | |
799 | dma_dev->device_prep_dma_memcpy = fsl_re_prep_dma_memcpy; |
800 | dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); |
801 | |
802 | dma_dev->device_free_chan_resources = fsl_re_free_chan_resources; |
803 | |
804 | re_priv->total_chans = 0; |
805 | |
806 | re_priv->cf_desc_pool = dmam_pool_create(name: "fsl_re_cf_desc_pool" , dev, |
807 | FSL_RE_CF_CDB_SIZE, |
808 | FSL_RE_CF_CDB_ALIGN, allocation: 0); |
809 | |
810 | if (!re_priv->cf_desc_pool) { |
811 | dev_err(dev, "No memory for fsl re_cf desc pool\n" ); |
812 | return -ENOMEM; |
813 | } |
814 | |
815 | re_priv->hw_desc_pool = dmam_pool_create(name: "fsl_re_hw_desc_pool" , dev, |
816 | size: sizeof(struct fsl_re_hw_desc) * FSL_RE_RING_SIZE, |
817 | FSL_RE_FRAME_ALIGN, allocation: 0); |
818 | if (!re_priv->hw_desc_pool) { |
819 | dev_err(dev, "No memory for fsl re_hw desc pool\n" ); |
820 | return -ENOMEM; |
821 | } |
822 | |
823 | dev_set_drvdata(dev, data: re_priv); |
824 | |
825 | /* Parse Device tree to find out the total number of JQs present */ |
826 | for_each_compatible_node(np, NULL, "fsl,raideng-v1.0-job-queue" ) { |
827 | rc = of_property_read_u32(np, propname: "reg" , out_value: &off); |
828 | if (rc) { |
829 | dev_err(dev, "Reg property not found in JQ node\n" ); |
830 | of_node_put(node: np); |
831 | return -ENODEV; |
832 | } |
833 | /* Find out the Job Rings present under each JQ */ |
834 | for_each_child_of_node(np, child) { |
835 | rc = of_device_is_compatible(device: child, |
836 | "fsl,raideng-v1.0-job-ring" ); |
837 | if (rc) { |
838 | fsl_re_chan_probe(ofdev, np: child, q: ridx++, off); |
839 | re_priv->total_chans++; |
840 | } |
841 | } |
842 | } |
843 | |
844 | dma_async_device_register(device: dma_dev); |
845 | |
846 | return 0; |
847 | } |
848 | |
849 | static void fsl_re_remove_chan(struct fsl_re_chan *chan) |
850 | { |
851 | tasklet_kill(t: &chan->irqtask); |
852 | |
853 | dma_pool_free(pool: chan->re_dev->hw_desc_pool, vaddr: chan->inb_ring_virt_addr, |
854 | addr: chan->inb_phys_addr); |
855 | |
856 | dma_pool_free(pool: chan->re_dev->hw_desc_pool, vaddr: chan->oub_ring_virt_addr, |
857 | addr: chan->oub_phys_addr); |
858 | } |
859 | |
860 | static void fsl_re_remove(struct platform_device *ofdev) |
861 | { |
862 | struct fsl_re_drv_private *re_priv; |
863 | struct device *dev; |
864 | int i; |
865 | |
866 | dev = &ofdev->dev; |
867 | re_priv = dev_get_drvdata(dev); |
868 | |
869 | /* Cleanup chan related memory areas */ |
870 | for (i = 0; i < re_priv->total_chans; i++) |
871 | fsl_re_remove_chan(chan: re_priv->re_jrs[i]); |
872 | |
873 | /* Unregister the driver */ |
874 | dma_async_device_unregister(device: &re_priv->dma_dev); |
875 | } |
876 | |
877 | static const struct of_device_id fsl_re_ids[] = { |
878 | { .compatible = "fsl,raideng-v1.0" , }, |
879 | {} |
880 | }; |
881 | MODULE_DEVICE_TABLE(of, fsl_re_ids); |
882 | |
883 | static struct platform_driver fsl_re_driver = { |
884 | .driver = { |
885 | .name = "fsl-raideng" , |
886 | .of_match_table = fsl_re_ids, |
887 | }, |
888 | .probe = fsl_re_probe, |
889 | .remove_new = fsl_re_remove, |
890 | }; |
891 | |
892 | module_platform_driver(fsl_re_driver); |
893 | |
894 | MODULE_AUTHOR("Harninder Rai <harninder.rai@freescale.com>" ); |
895 | MODULE_LICENSE("GPL v2" ); |
896 | MODULE_DESCRIPTION("Freescale RAID Engine Device Driver" ); |
897 | |