1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | // Copyright (C) 2017 Broadcom |
3 | |
4 | /* |
5 | * Broadcom SBA RAID Driver |
6 | * |
7 | * The Broadcom stream buffer accelerator (SBA) provides offloading |
8 | * capabilities for RAID operations. The SBA offload engine is accessible |
9 | * via Broadcom SoC specific ring manager. Two or more offload engines |
10 | * can share same Broadcom SoC specific ring manager due to this Broadcom |
11 | * SoC specific ring manager driver is implemented as a mailbox controller |
12 | * driver and offload engine drivers are implemented as mallbox clients. |
13 | * |
14 | * Typically, Broadcom SoC specific ring manager will implement larger |
15 | * number of hardware rings over one or more SBA hardware devices. By |
16 | * design, the internal buffer size of SBA hardware device is limited |
17 | * but all offload operations supported by SBA can be broken down into |
18 | * multiple small size requests and executed parallely on multiple SBA |
19 | * hardware devices for achieving high through-put. |
20 | * |
21 | * The Broadcom SBA RAID driver does not require any register programming |
22 | * except submitting request to SBA hardware device via mailbox channels. |
23 | * This driver implements a DMA device with one DMA channel using a single |
24 | * mailbox channel provided by Broadcom SoC specific ring manager driver. |
25 | * For having more SBA DMA channels, we can create more SBA device nodes |
26 | * in Broadcom SoC specific DTS based on number of hardware rings supported |
27 | * by Broadcom SoC ring manager. |
28 | */ |
29 | |
30 | #include <linux/bitops.h> |
31 | #include <linux/debugfs.h> |
32 | #include <linux/dma-mapping.h> |
33 | #include <linux/dmaengine.h> |
34 | #include <linux/list.h> |
35 | #include <linux/mailbox_client.h> |
36 | #include <linux/mailbox/brcm-message.h> |
37 | #include <linux/module.h> |
38 | #include <linux/of.h> |
39 | #include <linux/of_platform.h> |
40 | #include <linux/platform_device.h> |
41 | #include <linux/slab.h> |
42 | #include <linux/raid/pq.h> |
43 | |
44 | #include "dmaengine.h" |
45 | |
46 | /* ====== Driver macros and defines ===== */ |
47 | |
48 | #define SBA_TYPE_SHIFT 48 |
49 | #define SBA_TYPE_MASK GENMASK(1, 0) |
50 | #define SBA_TYPE_A 0x0 |
51 | #define SBA_TYPE_B 0x2 |
52 | #define SBA_TYPE_C 0x3 |
53 | #define SBA_USER_DEF_SHIFT 32 |
54 | #define SBA_USER_DEF_MASK GENMASK(15, 0) |
55 | #define SBA_R_MDATA_SHIFT 24 |
56 | #define SBA_R_MDATA_MASK GENMASK(7, 0) |
57 | #define SBA_C_MDATA_MS_SHIFT 18 |
58 | #define SBA_C_MDATA_MS_MASK GENMASK(1, 0) |
59 | #define SBA_INT_SHIFT 17 |
60 | #define SBA_INT_MASK BIT(0) |
61 | #define SBA_RESP_SHIFT 16 |
62 | #define SBA_RESP_MASK BIT(0) |
63 | #define SBA_C_MDATA_SHIFT 8 |
64 | #define SBA_C_MDATA_MASK GENMASK(7, 0) |
65 | #define SBA_C_MDATA_BNUMx_SHIFT(__bnum) (2 * (__bnum)) |
66 | #define SBA_C_MDATA_BNUMx_MASK GENMASK(1, 0) |
67 | #define SBA_C_MDATA_DNUM_SHIFT 5 |
68 | #define SBA_C_MDATA_DNUM_MASK GENMASK(4, 0) |
69 | #define SBA_C_MDATA_LS(__v) ((__v) & 0xff) |
70 | #define SBA_C_MDATA_MS(__v) (((__v) >> 8) & 0x3) |
71 | #define SBA_CMD_SHIFT 0 |
72 | #define SBA_CMD_MASK GENMASK(3, 0) |
73 | #define SBA_CMD_ZERO_BUFFER 0x4 |
74 | #define SBA_CMD_ZERO_ALL_BUFFERS 0x8 |
75 | #define SBA_CMD_LOAD_BUFFER 0x9 |
76 | #define SBA_CMD_XOR 0xa |
77 | #define SBA_CMD_GALOIS_XOR 0xb |
78 | #define SBA_CMD_WRITE_BUFFER 0xc |
79 | #define SBA_CMD_GALOIS 0xe |
80 | |
81 | #define SBA_MAX_REQ_PER_MBOX_CHANNEL 8192 |
82 | #define SBA_MAX_MSG_SEND_PER_MBOX_CHANNEL 8 |
83 | |
84 | /* Driver helper macros */ |
85 | #define to_sba_request(tx) \ |
86 | container_of(tx, struct sba_request, tx) |
87 | #define to_sba_device(dchan) \ |
88 | container_of(dchan, struct sba_device, dma_chan) |
89 | |
90 | /* ===== Driver data structures ===== */ |
91 | |
92 | enum sba_request_flags { |
93 | SBA_REQUEST_STATE_FREE = 0x001, |
94 | SBA_REQUEST_STATE_ALLOCED = 0x002, |
95 | SBA_REQUEST_STATE_PENDING = 0x004, |
96 | SBA_REQUEST_STATE_ACTIVE = 0x008, |
97 | SBA_REQUEST_STATE_ABORTED = 0x010, |
98 | SBA_REQUEST_STATE_MASK = 0x0ff, |
99 | SBA_REQUEST_FENCE = 0x100, |
100 | }; |
101 | |
102 | struct sba_request { |
103 | /* Global state */ |
104 | struct list_head node; |
105 | struct sba_device *sba; |
106 | u32 flags; |
107 | /* Chained requests management */ |
108 | struct sba_request *first; |
109 | struct list_head next; |
110 | atomic_t next_pending_count; |
111 | /* BRCM message data */ |
112 | struct brcm_message msg; |
113 | struct dma_async_tx_descriptor tx; |
114 | /* SBA commands */ |
115 | struct brcm_sba_command cmds[]; |
116 | }; |
117 | |
118 | enum sba_version { |
119 | SBA_VER_1 = 0, |
120 | SBA_VER_2 |
121 | }; |
122 | |
123 | struct sba_device { |
124 | /* Underlying device */ |
125 | struct device *dev; |
126 | /* DT configuration parameters */ |
127 | enum sba_version ver; |
128 | /* Derived configuration parameters */ |
129 | u32 max_req; |
130 | u32 hw_buf_size; |
131 | u32 hw_resp_size; |
132 | u32 max_pq_coefs; |
133 | u32 max_pq_srcs; |
134 | u32 max_cmd_per_req; |
135 | u32 max_xor_srcs; |
136 | u32 max_resp_pool_size; |
137 | u32 max_cmds_pool_size; |
138 | /* Maibox client and Mailbox channels */ |
139 | struct mbox_client client; |
140 | struct mbox_chan *mchan; |
141 | struct device *mbox_dev; |
142 | /* DMA device and DMA channel */ |
143 | struct dma_device dma_dev; |
144 | struct dma_chan dma_chan; |
145 | /* DMA channel resources */ |
146 | void *resp_base; |
147 | dma_addr_t resp_dma_base; |
148 | void *cmds_base; |
149 | dma_addr_t cmds_dma_base; |
150 | spinlock_t reqs_lock; |
151 | bool reqs_fence; |
152 | struct list_head reqs_alloc_list; |
153 | struct list_head reqs_pending_list; |
154 | struct list_head reqs_active_list; |
155 | struct list_head reqs_aborted_list; |
156 | struct list_head reqs_free_list; |
157 | /* DebugFS directory entries */ |
158 | struct dentry *root; |
159 | }; |
160 | |
161 | /* ====== Command helper routines ===== */ |
162 | |
163 | static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask) |
164 | { |
165 | cmd &= ~((u64)mask << shift); |
166 | cmd |= ((u64)(val & mask) << shift); |
167 | return cmd; |
168 | } |
169 | |
170 | static inline u32 __pure sba_cmd_load_c_mdata(u32 b0) |
171 | { |
172 | return b0 & SBA_C_MDATA_BNUMx_MASK; |
173 | } |
174 | |
175 | static inline u32 __pure sba_cmd_write_c_mdata(u32 b0) |
176 | { |
177 | return b0 & SBA_C_MDATA_BNUMx_MASK; |
178 | } |
179 | |
180 | static inline u32 __pure sba_cmd_xor_c_mdata(u32 b1, u32 b0) |
181 | { |
182 | return (b0 & SBA_C_MDATA_BNUMx_MASK) | |
183 | ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)); |
184 | } |
185 | |
186 | static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0) |
187 | { |
188 | return (b0 & SBA_C_MDATA_BNUMx_MASK) | |
189 | ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)) | |
190 | ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT); |
191 | } |
192 | |
193 | /* ====== General helper routines ===== */ |
194 | |
195 | static struct sba_request *sba_alloc_request(struct sba_device *sba) |
196 | { |
197 | bool found = false; |
198 | unsigned long flags; |
199 | struct sba_request *req = NULL; |
200 | |
201 | spin_lock_irqsave(&sba->reqs_lock, flags); |
202 | list_for_each_entry(req, &sba->reqs_free_list, node) { |
203 | if (async_tx_test_ack(tx: &req->tx)) { |
204 | list_move_tail(list: &req->node, head: &sba->reqs_alloc_list); |
205 | found = true; |
206 | break; |
207 | } |
208 | } |
209 | spin_unlock_irqrestore(lock: &sba->reqs_lock, flags); |
210 | |
211 | if (!found) { |
212 | /* |
213 | * We have no more free requests so, we peek |
214 | * mailbox channels hoping few active requests |
215 | * would have completed which will create more |
216 | * room for new requests. |
217 | */ |
218 | mbox_client_peek_data(chan: sba->mchan); |
219 | return NULL; |
220 | } |
221 | |
222 | req->flags = SBA_REQUEST_STATE_ALLOCED; |
223 | req->first = req; |
224 | INIT_LIST_HEAD(list: &req->next); |
225 | atomic_set(v: &req->next_pending_count, i: 1); |
226 | |
227 | dma_async_tx_descriptor_init(tx: &req->tx, chan: &sba->dma_chan); |
228 | async_tx_ack(tx: &req->tx); |
229 | |
230 | return req; |
231 | } |
232 | |
233 | /* Note: Must be called with sba->reqs_lock held */ |
234 | static void _sba_pending_request(struct sba_device *sba, |
235 | struct sba_request *req) |
236 | { |
237 | lockdep_assert_held(&sba->reqs_lock); |
238 | req->flags &= ~SBA_REQUEST_STATE_MASK; |
239 | req->flags |= SBA_REQUEST_STATE_PENDING; |
240 | list_move_tail(list: &req->node, head: &sba->reqs_pending_list); |
241 | if (list_empty(head: &sba->reqs_active_list)) |
242 | sba->reqs_fence = false; |
243 | } |
244 | |
245 | /* Note: Must be called with sba->reqs_lock held */ |
246 | static bool _sba_active_request(struct sba_device *sba, |
247 | struct sba_request *req) |
248 | { |
249 | lockdep_assert_held(&sba->reqs_lock); |
250 | if (list_empty(head: &sba->reqs_active_list)) |
251 | sba->reqs_fence = false; |
252 | if (sba->reqs_fence) |
253 | return false; |
254 | req->flags &= ~SBA_REQUEST_STATE_MASK; |
255 | req->flags |= SBA_REQUEST_STATE_ACTIVE; |
256 | list_move_tail(list: &req->node, head: &sba->reqs_active_list); |
257 | if (req->flags & SBA_REQUEST_FENCE) |
258 | sba->reqs_fence = true; |
259 | return true; |
260 | } |
261 | |
262 | /* Note: Must be called with sba->reqs_lock held */ |
263 | static void _sba_abort_request(struct sba_device *sba, |
264 | struct sba_request *req) |
265 | { |
266 | lockdep_assert_held(&sba->reqs_lock); |
267 | req->flags &= ~SBA_REQUEST_STATE_MASK; |
268 | req->flags |= SBA_REQUEST_STATE_ABORTED; |
269 | list_move_tail(list: &req->node, head: &sba->reqs_aborted_list); |
270 | if (list_empty(head: &sba->reqs_active_list)) |
271 | sba->reqs_fence = false; |
272 | } |
273 | |
274 | /* Note: Must be called with sba->reqs_lock held */ |
275 | static void _sba_free_request(struct sba_device *sba, |
276 | struct sba_request *req) |
277 | { |
278 | lockdep_assert_held(&sba->reqs_lock); |
279 | req->flags &= ~SBA_REQUEST_STATE_MASK; |
280 | req->flags |= SBA_REQUEST_STATE_FREE; |
281 | list_move_tail(list: &req->node, head: &sba->reqs_free_list); |
282 | if (list_empty(head: &sba->reqs_active_list)) |
283 | sba->reqs_fence = false; |
284 | } |
285 | |
286 | static void sba_free_chained_requests(struct sba_request *req) |
287 | { |
288 | unsigned long flags; |
289 | struct sba_request *nreq; |
290 | struct sba_device *sba = req->sba; |
291 | |
292 | spin_lock_irqsave(&sba->reqs_lock, flags); |
293 | |
294 | _sba_free_request(sba, req); |
295 | list_for_each_entry(nreq, &req->next, next) |
296 | _sba_free_request(sba, req: nreq); |
297 | |
298 | spin_unlock_irqrestore(lock: &sba->reqs_lock, flags); |
299 | } |
300 | |
301 | static void sba_chain_request(struct sba_request *first, |
302 | struct sba_request *req) |
303 | { |
304 | unsigned long flags; |
305 | struct sba_device *sba = req->sba; |
306 | |
307 | spin_lock_irqsave(&sba->reqs_lock, flags); |
308 | |
309 | list_add_tail(new: &req->next, head: &first->next); |
310 | req->first = first; |
311 | atomic_inc(v: &first->next_pending_count); |
312 | |
313 | spin_unlock_irqrestore(lock: &sba->reqs_lock, flags); |
314 | } |
315 | |
316 | static void sba_cleanup_nonpending_requests(struct sba_device *sba) |
317 | { |
318 | unsigned long flags; |
319 | struct sba_request *req, *req1; |
320 | |
321 | spin_lock_irqsave(&sba->reqs_lock, flags); |
322 | |
323 | /* Freeup all alloced request */ |
324 | list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node) |
325 | _sba_free_request(sba, req); |
326 | |
327 | /* Set all active requests as aborted */ |
328 | list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node) |
329 | _sba_abort_request(sba, req); |
330 | |
331 | /* |
332 | * Note: We expect that aborted request will be eventually |
333 | * freed by sba_receive_message() |
334 | */ |
335 | |
336 | spin_unlock_irqrestore(lock: &sba->reqs_lock, flags); |
337 | } |
338 | |
339 | static void sba_cleanup_pending_requests(struct sba_device *sba) |
340 | { |
341 | unsigned long flags; |
342 | struct sba_request *req, *req1; |
343 | |
344 | spin_lock_irqsave(&sba->reqs_lock, flags); |
345 | |
346 | /* Freeup all pending request */ |
347 | list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node) |
348 | _sba_free_request(sba, req); |
349 | |
350 | spin_unlock_irqrestore(lock: &sba->reqs_lock, flags); |
351 | } |
352 | |
353 | static int sba_send_mbox_request(struct sba_device *sba, |
354 | struct sba_request *req) |
355 | { |
356 | int ret = 0; |
357 | |
358 | /* Send message for the request */ |
359 | req->msg.error = 0; |
360 | ret = mbox_send_message(chan: sba->mchan, mssg: &req->msg); |
361 | if (ret < 0) { |
362 | dev_err(sba->dev, "send message failed with error %d" , ret); |
363 | return ret; |
364 | } |
365 | |
366 | /* Check error returned by mailbox controller */ |
367 | ret = req->msg.error; |
368 | if (ret < 0) { |
369 | dev_err(sba->dev, "message error %d" , ret); |
370 | } |
371 | |
372 | /* Signal txdone for mailbox channel */ |
373 | mbox_client_txdone(chan: sba->mchan, r: ret); |
374 | |
375 | return ret; |
376 | } |
377 | |
378 | /* Note: Must be called with sba->reqs_lock held */ |
379 | static void _sba_process_pending_requests(struct sba_device *sba) |
380 | { |
381 | int ret; |
382 | u32 count; |
383 | struct sba_request *req; |
384 | |
385 | /* Process few pending requests */ |
386 | count = SBA_MAX_MSG_SEND_PER_MBOX_CHANNEL; |
387 | while (!list_empty(head: &sba->reqs_pending_list) && count) { |
388 | /* Get the first pending request */ |
389 | req = list_first_entry(&sba->reqs_pending_list, |
390 | struct sba_request, node); |
391 | |
392 | /* Try to make request active */ |
393 | if (!_sba_active_request(sba, req)) |
394 | break; |
395 | |
396 | /* Send request to mailbox channel */ |
397 | ret = sba_send_mbox_request(sba, req); |
398 | if (ret < 0) { |
399 | _sba_pending_request(sba, req); |
400 | break; |
401 | } |
402 | |
403 | count--; |
404 | } |
405 | } |
406 | |
407 | static void sba_process_received_request(struct sba_device *sba, |
408 | struct sba_request *req) |
409 | { |
410 | unsigned long flags; |
411 | struct dma_async_tx_descriptor *tx; |
412 | struct sba_request *nreq, *first = req->first; |
413 | |
414 | /* Process only after all chained requests are received */ |
415 | if (!atomic_dec_return(v: &first->next_pending_count)) { |
416 | tx = &first->tx; |
417 | |
418 | WARN_ON(tx->cookie < 0); |
419 | if (tx->cookie > 0) { |
420 | spin_lock_irqsave(&sba->reqs_lock, flags); |
421 | dma_cookie_complete(tx); |
422 | spin_unlock_irqrestore(lock: &sba->reqs_lock, flags); |
423 | dmaengine_desc_get_callback_invoke(tx, NULL); |
424 | dma_descriptor_unmap(tx); |
425 | tx->callback = NULL; |
426 | tx->callback_result = NULL; |
427 | } |
428 | |
429 | dma_run_dependencies(tx); |
430 | |
431 | spin_lock_irqsave(&sba->reqs_lock, flags); |
432 | |
433 | /* Free all requests chained to first request */ |
434 | list_for_each_entry(nreq, &first->next, next) |
435 | _sba_free_request(sba, req: nreq); |
436 | INIT_LIST_HEAD(list: &first->next); |
437 | |
438 | /* Free the first request */ |
439 | _sba_free_request(sba, req: first); |
440 | |
441 | /* Process pending requests */ |
442 | _sba_process_pending_requests(sba); |
443 | |
444 | spin_unlock_irqrestore(lock: &sba->reqs_lock, flags); |
445 | } |
446 | } |
447 | |
448 | static void sba_write_stats_in_seqfile(struct sba_device *sba, |
449 | struct seq_file *file) |
450 | { |
451 | unsigned long flags; |
452 | struct sba_request *req; |
453 | u32 free_count = 0, alloced_count = 0; |
454 | u32 pending_count = 0, active_count = 0, aborted_count = 0; |
455 | |
456 | spin_lock_irqsave(&sba->reqs_lock, flags); |
457 | |
458 | list_for_each_entry(req, &sba->reqs_free_list, node) |
459 | if (async_tx_test_ack(tx: &req->tx)) |
460 | free_count++; |
461 | |
462 | list_for_each_entry(req, &sba->reqs_alloc_list, node) |
463 | alloced_count++; |
464 | |
465 | list_for_each_entry(req, &sba->reqs_pending_list, node) |
466 | pending_count++; |
467 | |
468 | list_for_each_entry(req, &sba->reqs_active_list, node) |
469 | active_count++; |
470 | |
471 | list_for_each_entry(req, &sba->reqs_aborted_list, node) |
472 | aborted_count++; |
473 | |
474 | spin_unlock_irqrestore(lock: &sba->reqs_lock, flags); |
475 | |
476 | seq_printf(m: file, fmt: "maximum requests = %d\n" , sba->max_req); |
477 | seq_printf(m: file, fmt: "free requests = %d\n" , free_count); |
478 | seq_printf(m: file, fmt: "alloced requests = %d\n" , alloced_count); |
479 | seq_printf(m: file, fmt: "pending requests = %d\n" , pending_count); |
480 | seq_printf(m: file, fmt: "active requests = %d\n" , active_count); |
481 | seq_printf(m: file, fmt: "aborted requests = %d\n" , aborted_count); |
482 | } |
483 | |
484 | /* ====== DMAENGINE callbacks ===== */ |
485 | |
486 | static void sba_free_chan_resources(struct dma_chan *dchan) |
487 | { |
488 | /* |
489 | * Channel resources are pre-alloced so we just free-up |
490 | * whatever we can so that we can re-use pre-alloced |
491 | * channel resources next time. |
492 | */ |
493 | sba_cleanup_nonpending_requests(to_sba_device(dchan)); |
494 | } |
495 | |
496 | static int sba_device_terminate_all(struct dma_chan *dchan) |
497 | { |
498 | /* Cleanup all pending requests */ |
499 | sba_cleanup_pending_requests(to_sba_device(dchan)); |
500 | |
501 | return 0; |
502 | } |
503 | |
504 | static void sba_issue_pending(struct dma_chan *dchan) |
505 | { |
506 | unsigned long flags; |
507 | struct sba_device *sba = to_sba_device(dchan); |
508 | |
509 | /* Process pending requests */ |
510 | spin_lock_irqsave(&sba->reqs_lock, flags); |
511 | _sba_process_pending_requests(sba); |
512 | spin_unlock_irqrestore(lock: &sba->reqs_lock, flags); |
513 | } |
514 | |
515 | static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx) |
516 | { |
517 | unsigned long flags; |
518 | dma_cookie_t cookie; |
519 | struct sba_device *sba; |
520 | struct sba_request *req, *nreq; |
521 | |
522 | if (unlikely(!tx)) |
523 | return -EINVAL; |
524 | |
525 | sba = to_sba_device(tx->chan); |
526 | req = to_sba_request(tx); |
527 | |
528 | /* Assign cookie and mark all chained requests pending */ |
529 | spin_lock_irqsave(&sba->reqs_lock, flags); |
530 | cookie = dma_cookie_assign(tx); |
531 | _sba_pending_request(sba, req); |
532 | list_for_each_entry(nreq, &req->next, next) |
533 | _sba_pending_request(sba, req: nreq); |
534 | spin_unlock_irqrestore(lock: &sba->reqs_lock, flags); |
535 | |
536 | return cookie; |
537 | } |
538 | |
539 | static enum dma_status sba_tx_status(struct dma_chan *dchan, |
540 | dma_cookie_t cookie, |
541 | struct dma_tx_state *txstate) |
542 | { |
543 | enum dma_status ret; |
544 | struct sba_device *sba = to_sba_device(dchan); |
545 | |
546 | ret = dma_cookie_status(chan: dchan, cookie, state: txstate); |
547 | if (ret == DMA_COMPLETE) |
548 | return ret; |
549 | |
550 | mbox_client_peek_data(chan: sba->mchan); |
551 | |
552 | return dma_cookie_status(chan: dchan, cookie, state: txstate); |
553 | } |
554 | |
555 | static void sba_fillup_interrupt_msg(struct sba_request *req, |
556 | struct brcm_sba_command *cmds, |
557 | struct brcm_message *msg) |
558 | { |
559 | u64 cmd; |
560 | u32 c_mdata; |
561 | dma_addr_t resp_dma = req->tx.phys; |
562 | struct brcm_sba_command *cmdsp = cmds; |
563 | |
564 | /* Type-B command to load dummy data into buf0 */ |
565 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_B, |
566 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
567 | cmd = sba_cmd_enc(cmd, val: req->sba->hw_resp_size, |
568 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
569 | c_mdata = sba_cmd_load_c_mdata(b0: 0); |
570 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
571 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
572 | cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, |
573 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
574 | cmdsp->cmd = cmd; |
575 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
576 | cmdsp->flags = BRCM_SBA_CMD_TYPE_B; |
577 | cmdsp->data = resp_dma; |
578 | cmdsp->data_len = req->sba->hw_resp_size; |
579 | cmdsp++; |
580 | |
581 | /* Type-A command to write buf0 to dummy location */ |
582 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_A, |
583 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
584 | cmd = sba_cmd_enc(cmd, val: req->sba->hw_resp_size, |
585 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
586 | cmd = sba_cmd_enc(cmd, val: 0x1, |
587 | SBA_RESP_SHIFT, SBA_RESP_MASK); |
588 | c_mdata = sba_cmd_write_c_mdata(b0: 0); |
589 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
590 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
591 | cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, |
592 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
593 | cmdsp->cmd = cmd; |
594 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
595 | cmdsp->flags = BRCM_SBA_CMD_TYPE_A; |
596 | if (req->sba->hw_resp_size) { |
597 | cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; |
598 | cmdsp->resp = resp_dma; |
599 | cmdsp->resp_len = req->sba->hw_resp_size; |
600 | } |
601 | cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; |
602 | cmdsp->data = resp_dma; |
603 | cmdsp->data_len = req->sba->hw_resp_size; |
604 | cmdsp++; |
605 | |
606 | /* Fillup brcm_message */ |
607 | msg->type = BRCM_MESSAGE_SBA; |
608 | msg->sba.cmds = cmds; |
609 | msg->sba.cmds_count = cmdsp - cmds; |
610 | msg->ctx = req; |
611 | msg->error = 0; |
612 | } |
613 | |
614 | static struct dma_async_tx_descriptor * |
615 | sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags) |
616 | { |
617 | struct sba_request *req = NULL; |
618 | struct sba_device *sba = to_sba_device(dchan); |
619 | |
620 | /* Alloc new request */ |
621 | req = sba_alloc_request(sba); |
622 | if (!req) |
623 | return NULL; |
624 | |
625 | /* |
626 | * Force fence so that no requests are submitted |
627 | * until DMA callback for this request is invoked. |
628 | */ |
629 | req->flags |= SBA_REQUEST_FENCE; |
630 | |
631 | /* Fillup request message */ |
632 | sba_fillup_interrupt_msg(req, cmds: req->cmds, msg: &req->msg); |
633 | |
634 | /* Init async_tx descriptor */ |
635 | req->tx.flags = flags; |
636 | req->tx.cookie = -EBUSY; |
637 | |
638 | return &req->tx; |
639 | } |
640 | |
641 | static void sba_fillup_memcpy_msg(struct sba_request *req, |
642 | struct brcm_sba_command *cmds, |
643 | struct brcm_message *msg, |
644 | dma_addr_t msg_offset, size_t msg_len, |
645 | dma_addr_t dst, dma_addr_t src) |
646 | { |
647 | u64 cmd; |
648 | u32 c_mdata; |
649 | dma_addr_t resp_dma = req->tx.phys; |
650 | struct brcm_sba_command *cmdsp = cmds; |
651 | |
652 | /* Type-B command to load data into buf0 */ |
653 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_B, |
654 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
655 | cmd = sba_cmd_enc(cmd, val: msg_len, |
656 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
657 | c_mdata = sba_cmd_load_c_mdata(b0: 0); |
658 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
659 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
660 | cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, |
661 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
662 | cmdsp->cmd = cmd; |
663 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
664 | cmdsp->flags = BRCM_SBA_CMD_TYPE_B; |
665 | cmdsp->data = src + msg_offset; |
666 | cmdsp->data_len = msg_len; |
667 | cmdsp++; |
668 | |
669 | /* Type-A command to write buf0 */ |
670 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_A, |
671 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
672 | cmd = sba_cmd_enc(cmd, val: msg_len, |
673 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
674 | cmd = sba_cmd_enc(cmd, val: 0x1, |
675 | SBA_RESP_SHIFT, SBA_RESP_MASK); |
676 | c_mdata = sba_cmd_write_c_mdata(b0: 0); |
677 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
678 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
679 | cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, |
680 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
681 | cmdsp->cmd = cmd; |
682 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
683 | cmdsp->flags = BRCM_SBA_CMD_TYPE_A; |
684 | if (req->sba->hw_resp_size) { |
685 | cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; |
686 | cmdsp->resp = resp_dma; |
687 | cmdsp->resp_len = req->sba->hw_resp_size; |
688 | } |
689 | cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; |
690 | cmdsp->data = dst + msg_offset; |
691 | cmdsp->data_len = msg_len; |
692 | cmdsp++; |
693 | |
694 | /* Fillup brcm_message */ |
695 | msg->type = BRCM_MESSAGE_SBA; |
696 | msg->sba.cmds = cmds; |
697 | msg->sba.cmds_count = cmdsp - cmds; |
698 | msg->ctx = req; |
699 | msg->error = 0; |
700 | } |
701 | |
702 | static struct sba_request * |
703 | sba_prep_dma_memcpy_req(struct sba_device *sba, |
704 | dma_addr_t off, dma_addr_t dst, dma_addr_t src, |
705 | size_t len, unsigned long flags) |
706 | { |
707 | struct sba_request *req = NULL; |
708 | |
709 | /* Alloc new request */ |
710 | req = sba_alloc_request(sba); |
711 | if (!req) |
712 | return NULL; |
713 | if (flags & DMA_PREP_FENCE) |
714 | req->flags |= SBA_REQUEST_FENCE; |
715 | |
716 | /* Fillup request message */ |
717 | sba_fillup_memcpy_msg(req, cmds: req->cmds, msg: &req->msg, |
718 | msg_offset: off, msg_len: len, dst, src); |
719 | |
720 | /* Init async_tx descriptor */ |
721 | req->tx.flags = flags; |
722 | req->tx.cookie = -EBUSY; |
723 | |
724 | return req; |
725 | } |
726 | |
727 | static struct dma_async_tx_descriptor * |
728 | sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src, |
729 | size_t len, unsigned long flags) |
730 | { |
731 | size_t req_len; |
732 | dma_addr_t off = 0; |
733 | struct sba_device *sba = to_sba_device(dchan); |
734 | struct sba_request *first = NULL, *req; |
735 | |
736 | /* Create chained requests where each request is upto hw_buf_size */ |
737 | while (len) { |
738 | req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; |
739 | |
740 | req = sba_prep_dma_memcpy_req(sba, off, dst, src, |
741 | len: req_len, flags); |
742 | if (!req) { |
743 | if (first) |
744 | sba_free_chained_requests(req: first); |
745 | return NULL; |
746 | } |
747 | |
748 | if (first) |
749 | sba_chain_request(first, req); |
750 | else |
751 | first = req; |
752 | |
753 | off += req_len; |
754 | len -= req_len; |
755 | } |
756 | |
757 | return (first) ? &first->tx : NULL; |
758 | } |
759 | |
760 | static void sba_fillup_xor_msg(struct sba_request *req, |
761 | struct brcm_sba_command *cmds, |
762 | struct brcm_message *msg, |
763 | dma_addr_t msg_offset, size_t msg_len, |
764 | dma_addr_t dst, dma_addr_t *src, u32 src_cnt) |
765 | { |
766 | u64 cmd; |
767 | u32 c_mdata; |
768 | unsigned int i; |
769 | dma_addr_t resp_dma = req->tx.phys; |
770 | struct brcm_sba_command *cmdsp = cmds; |
771 | |
772 | /* Type-B command to load data into buf0 */ |
773 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_B, |
774 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
775 | cmd = sba_cmd_enc(cmd, val: msg_len, |
776 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
777 | c_mdata = sba_cmd_load_c_mdata(b0: 0); |
778 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
779 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
780 | cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, |
781 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
782 | cmdsp->cmd = cmd; |
783 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
784 | cmdsp->flags = BRCM_SBA_CMD_TYPE_B; |
785 | cmdsp->data = src[0] + msg_offset; |
786 | cmdsp->data_len = msg_len; |
787 | cmdsp++; |
788 | |
789 | /* Type-B commands to xor data with buf0 and put it back in buf0 */ |
790 | for (i = 1; i < src_cnt; i++) { |
791 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_B, |
792 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
793 | cmd = sba_cmd_enc(cmd, val: msg_len, |
794 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
795 | c_mdata = sba_cmd_xor_c_mdata(b1: 0, b0: 0); |
796 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
797 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
798 | cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, |
799 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
800 | cmdsp->cmd = cmd; |
801 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
802 | cmdsp->flags = BRCM_SBA_CMD_TYPE_B; |
803 | cmdsp->data = src[i] + msg_offset; |
804 | cmdsp->data_len = msg_len; |
805 | cmdsp++; |
806 | } |
807 | |
808 | /* Type-A command to write buf0 */ |
809 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_A, |
810 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
811 | cmd = sba_cmd_enc(cmd, val: msg_len, |
812 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
813 | cmd = sba_cmd_enc(cmd, val: 0x1, |
814 | SBA_RESP_SHIFT, SBA_RESP_MASK); |
815 | c_mdata = sba_cmd_write_c_mdata(b0: 0); |
816 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
817 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
818 | cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, |
819 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
820 | cmdsp->cmd = cmd; |
821 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
822 | cmdsp->flags = BRCM_SBA_CMD_TYPE_A; |
823 | if (req->sba->hw_resp_size) { |
824 | cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; |
825 | cmdsp->resp = resp_dma; |
826 | cmdsp->resp_len = req->sba->hw_resp_size; |
827 | } |
828 | cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; |
829 | cmdsp->data = dst + msg_offset; |
830 | cmdsp->data_len = msg_len; |
831 | cmdsp++; |
832 | |
833 | /* Fillup brcm_message */ |
834 | msg->type = BRCM_MESSAGE_SBA; |
835 | msg->sba.cmds = cmds; |
836 | msg->sba.cmds_count = cmdsp - cmds; |
837 | msg->ctx = req; |
838 | msg->error = 0; |
839 | } |
840 | |
841 | static struct sba_request * |
842 | sba_prep_dma_xor_req(struct sba_device *sba, |
843 | dma_addr_t off, dma_addr_t dst, dma_addr_t *src, |
844 | u32 src_cnt, size_t len, unsigned long flags) |
845 | { |
846 | struct sba_request *req = NULL; |
847 | |
848 | /* Alloc new request */ |
849 | req = sba_alloc_request(sba); |
850 | if (!req) |
851 | return NULL; |
852 | if (flags & DMA_PREP_FENCE) |
853 | req->flags |= SBA_REQUEST_FENCE; |
854 | |
855 | /* Fillup request message */ |
856 | sba_fillup_xor_msg(req, cmds: req->cmds, msg: &req->msg, |
857 | msg_offset: off, msg_len: len, dst, src, src_cnt); |
858 | |
859 | /* Init async_tx descriptor */ |
860 | req->tx.flags = flags; |
861 | req->tx.cookie = -EBUSY; |
862 | |
863 | return req; |
864 | } |
865 | |
866 | static struct dma_async_tx_descriptor * |
867 | sba_prep_dma_xor(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src, |
868 | u32 src_cnt, size_t len, unsigned long flags) |
869 | { |
870 | size_t req_len; |
871 | dma_addr_t off = 0; |
872 | struct sba_device *sba = to_sba_device(dchan); |
873 | struct sba_request *first = NULL, *req; |
874 | |
875 | /* Sanity checks */ |
876 | if (unlikely(src_cnt > sba->max_xor_srcs)) |
877 | return NULL; |
878 | |
879 | /* Create chained requests where each request is upto hw_buf_size */ |
880 | while (len) { |
881 | req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; |
882 | |
883 | req = sba_prep_dma_xor_req(sba, off, dst, src, src_cnt, |
884 | len: req_len, flags); |
885 | if (!req) { |
886 | if (first) |
887 | sba_free_chained_requests(req: first); |
888 | return NULL; |
889 | } |
890 | |
891 | if (first) |
892 | sba_chain_request(first, req); |
893 | else |
894 | first = req; |
895 | |
896 | off += req_len; |
897 | len -= req_len; |
898 | } |
899 | |
900 | return (first) ? &first->tx : NULL; |
901 | } |
902 | |
903 | static void sba_fillup_pq_msg(struct sba_request *req, |
904 | bool pq_continue, |
905 | struct brcm_sba_command *cmds, |
906 | struct brcm_message *msg, |
907 | dma_addr_t msg_offset, size_t msg_len, |
908 | dma_addr_t *dst_p, dma_addr_t *dst_q, |
909 | const u8 *scf, dma_addr_t *src, u32 src_cnt) |
910 | { |
911 | u64 cmd; |
912 | u32 c_mdata; |
913 | unsigned int i; |
914 | dma_addr_t resp_dma = req->tx.phys; |
915 | struct brcm_sba_command *cmdsp = cmds; |
916 | |
917 | if (pq_continue) { |
918 | /* Type-B command to load old P into buf0 */ |
919 | if (dst_p) { |
920 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_B, |
921 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
922 | cmd = sba_cmd_enc(cmd, val: msg_len, |
923 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
924 | c_mdata = sba_cmd_load_c_mdata(b0: 0); |
925 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
926 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
927 | cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, |
928 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
929 | cmdsp->cmd = cmd; |
930 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
931 | cmdsp->flags = BRCM_SBA_CMD_TYPE_B; |
932 | cmdsp->data = *dst_p + msg_offset; |
933 | cmdsp->data_len = msg_len; |
934 | cmdsp++; |
935 | } |
936 | |
937 | /* Type-B command to load old Q into buf1 */ |
938 | if (dst_q) { |
939 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_B, |
940 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
941 | cmd = sba_cmd_enc(cmd, val: msg_len, |
942 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
943 | c_mdata = sba_cmd_load_c_mdata(b0: 1); |
944 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
945 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
946 | cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, |
947 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
948 | cmdsp->cmd = cmd; |
949 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
950 | cmdsp->flags = BRCM_SBA_CMD_TYPE_B; |
951 | cmdsp->data = *dst_q + msg_offset; |
952 | cmdsp->data_len = msg_len; |
953 | cmdsp++; |
954 | } |
955 | } else { |
956 | /* Type-A command to zero all buffers */ |
957 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_A, |
958 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
959 | cmd = sba_cmd_enc(cmd, val: msg_len, |
960 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
961 | cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS, |
962 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
963 | cmdsp->cmd = cmd; |
964 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
965 | cmdsp->flags = BRCM_SBA_CMD_TYPE_A; |
966 | cmdsp++; |
967 | } |
968 | |
969 | /* Type-B commands for generate P onto buf0 and Q onto buf1 */ |
970 | for (i = 0; i < src_cnt; i++) { |
971 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_B, |
972 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
973 | cmd = sba_cmd_enc(cmd, val: msg_len, |
974 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
975 | c_mdata = sba_cmd_pq_c_mdata(d: raid6_gflog[scf[i]], b1: 1, b0: 0); |
976 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
977 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
978 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), |
979 | SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); |
980 | cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS_XOR, |
981 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
982 | cmdsp->cmd = cmd; |
983 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
984 | cmdsp->flags = BRCM_SBA_CMD_TYPE_B; |
985 | cmdsp->data = src[i] + msg_offset; |
986 | cmdsp->data_len = msg_len; |
987 | cmdsp++; |
988 | } |
989 | |
990 | /* Type-A command to write buf0 */ |
991 | if (dst_p) { |
992 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_A, |
993 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
994 | cmd = sba_cmd_enc(cmd, val: msg_len, |
995 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
996 | cmd = sba_cmd_enc(cmd, val: 0x1, |
997 | SBA_RESP_SHIFT, SBA_RESP_MASK); |
998 | c_mdata = sba_cmd_write_c_mdata(b0: 0); |
999 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
1000 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
1001 | cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, |
1002 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
1003 | cmdsp->cmd = cmd; |
1004 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
1005 | cmdsp->flags = BRCM_SBA_CMD_TYPE_A; |
1006 | if (req->sba->hw_resp_size) { |
1007 | cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; |
1008 | cmdsp->resp = resp_dma; |
1009 | cmdsp->resp_len = req->sba->hw_resp_size; |
1010 | } |
1011 | cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; |
1012 | cmdsp->data = *dst_p + msg_offset; |
1013 | cmdsp->data_len = msg_len; |
1014 | cmdsp++; |
1015 | } |
1016 | |
1017 | /* Type-A command to write buf1 */ |
1018 | if (dst_q) { |
1019 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_A, |
1020 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
1021 | cmd = sba_cmd_enc(cmd, val: msg_len, |
1022 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
1023 | cmd = sba_cmd_enc(cmd, val: 0x1, |
1024 | SBA_RESP_SHIFT, SBA_RESP_MASK); |
1025 | c_mdata = sba_cmd_write_c_mdata(b0: 1); |
1026 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
1027 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
1028 | cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, |
1029 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
1030 | cmdsp->cmd = cmd; |
1031 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
1032 | cmdsp->flags = BRCM_SBA_CMD_TYPE_A; |
1033 | if (req->sba->hw_resp_size) { |
1034 | cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; |
1035 | cmdsp->resp = resp_dma; |
1036 | cmdsp->resp_len = req->sba->hw_resp_size; |
1037 | } |
1038 | cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; |
1039 | cmdsp->data = *dst_q + msg_offset; |
1040 | cmdsp->data_len = msg_len; |
1041 | cmdsp++; |
1042 | } |
1043 | |
1044 | /* Fillup brcm_message */ |
1045 | msg->type = BRCM_MESSAGE_SBA; |
1046 | msg->sba.cmds = cmds; |
1047 | msg->sba.cmds_count = cmdsp - cmds; |
1048 | msg->ctx = req; |
1049 | msg->error = 0; |
1050 | } |
1051 | |
1052 | static struct sba_request * |
1053 | sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off, |
1054 | dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src, |
1055 | u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) |
1056 | { |
1057 | struct sba_request *req = NULL; |
1058 | |
1059 | /* Alloc new request */ |
1060 | req = sba_alloc_request(sba); |
1061 | if (!req) |
1062 | return NULL; |
1063 | if (flags & DMA_PREP_FENCE) |
1064 | req->flags |= SBA_REQUEST_FENCE; |
1065 | |
1066 | /* Fillup request messages */ |
1067 | sba_fillup_pq_msg(req, pq_continue: dmaf_continue(flags), |
1068 | cmds: req->cmds, msg: &req->msg, |
1069 | msg_offset: off, msg_len: len, dst_p, dst_q, scf, src, src_cnt); |
1070 | |
1071 | /* Init async_tx descriptor */ |
1072 | req->tx.flags = flags; |
1073 | req->tx.cookie = -EBUSY; |
1074 | |
1075 | return req; |
1076 | } |
1077 | |
1078 | static void sba_fillup_pq_single_msg(struct sba_request *req, |
1079 | bool pq_continue, |
1080 | struct brcm_sba_command *cmds, |
1081 | struct brcm_message *msg, |
1082 | dma_addr_t msg_offset, size_t msg_len, |
1083 | dma_addr_t *dst_p, dma_addr_t *dst_q, |
1084 | dma_addr_t src, u8 scf) |
1085 | { |
1086 | u64 cmd; |
1087 | u32 c_mdata; |
1088 | u8 pos, dpos = raid6_gflog[scf]; |
1089 | dma_addr_t resp_dma = req->tx.phys; |
1090 | struct brcm_sba_command *cmdsp = cmds; |
1091 | |
1092 | if (!dst_p) |
1093 | goto skip_p; |
1094 | |
1095 | if (pq_continue) { |
1096 | /* Type-B command to load old P into buf0 */ |
1097 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_B, |
1098 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
1099 | cmd = sba_cmd_enc(cmd, val: msg_len, |
1100 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
1101 | c_mdata = sba_cmd_load_c_mdata(b0: 0); |
1102 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
1103 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
1104 | cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, |
1105 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
1106 | cmdsp->cmd = cmd; |
1107 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
1108 | cmdsp->flags = BRCM_SBA_CMD_TYPE_B; |
1109 | cmdsp->data = *dst_p + msg_offset; |
1110 | cmdsp->data_len = msg_len; |
1111 | cmdsp++; |
1112 | |
1113 | /* |
1114 | * Type-B commands to xor data with buf0 and put it |
1115 | * back in buf0 |
1116 | */ |
1117 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_B, |
1118 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
1119 | cmd = sba_cmd_enc(cmd, val: msg_len, |
1120 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
1121 | c_mdata = sba_cmd_xor_c_mdata(b1: 0, b0: 0); |
1122 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
1123 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
1124 | cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, |
1125 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
1126 | cmdsp->cmd = cmd; |
1127 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
1128 | cmdsp->flags = BRCM_SBA_CMD_TYPE_B; |
1129 | cmdsp->data = src + msg_offset; |
1130 | cmdsp->data_len = msg_len; |
1131 | cmdsp++; |
1132 | } else { |
1133 | /* Type-B command to load old P into buf0 */ |
1134 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_B, |
1135 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
1136 | cmd = sba_cmd_enc(cmd, val: msg_len, |
1137 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
1138 | c_mdata = sba_cmd_load_c_mdata(b0: 0); |
1139 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
1140 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
1141 | cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, |
1142 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
1143 | cmdsp->cmd = cmd; |
1144 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
1145 | cmdsp->flags = BRCM_SBA_CMD_TYPE_B; |
1146 | cmdsp->data = src + msg_offset; |
1147 | cmdsp->data_len = msg_len; |
1148 | cmdsp++; |
1149 | } |
1150 | |
1151 | /* Type-A command to write buf0 */ |
1152 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_A, |
1153 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
1154 | cmd = sba_cmd_enc(cmd, val: msg_len, |
1155 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
1156 | cmd = sba_cmd_enc(cmd, val: 0x1, |
1157 | SBA_RESP_SHIFT, SBA_RESP_MASK); |
1158 | c_mdata = sba_cmd_write_c_mdata(b0: 0); |
1159 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
1160 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
1161 | cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, |
1162 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
1163 | cmdsp->cmd = cmd; |
1164 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
1165 | cmdsp->flags = BRCM_SBA_CMD_TYPE_A; |
1166 | if (req->sba->hw_resp_size) { |
1167 | cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; |
1168 | cmdsp->resp = resp_dma; |
1169 | cmdsp->resp_len = req->sba->hw_resp_size; |
1170 | } |
1171 | cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; |
1172 | cmdsp->data = *dst_p + msg_offset; |
1173 | cmdsp->data_len = msg_len; |
1174 | cmdsp++; |
1175 | |
1176 | skip_p: |
1177 | if (!dst_q) |
1178 | goto skip_q; |
1179 | |
1180 | /* Type-A command to zero all buffers */ |
1181 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_A, |
1182 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
1183 | cmd = sba_cmd_enc(cmd, val: msg_len, |
1184 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
1185 | cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS, |
1186 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
1187 | cmdsp->cmd = cmd; |
1188 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
1189 | cmdsp->flags = BRCM_SBA_CMD_TYPE_A; |
1190 | cmdsp++; |
1191 | |
1192 | if (dpos == 255) |
1193 | goto skip_q_computation; |
1194 | pos = (dpos < req->sba->max_pq_coefs) ? |
1195 | dpos : (req->sba->max_pq_coefs - 1); |
1196 | |
1197 | /* |
1198 | * Type-B command to generate initial Q from data |
1199 | * and store output into buf0 |
1200 | */ |
1201 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_B, |
1202 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
1203 | cmd = sba_cmd_enc(cmd, val: msg_len, |
1204 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
1205 | c_mdata = sba_cmd_pq_c_mdata(d: pos, b1: 0, b0: 0); |
1206 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
1207 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
1208 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), |
1209 | SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); |
1210 | cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS, |
1211 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
1212 | cmdsp->cmd = cmd; |
1213 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
1214 | cmdsp->flags = BRCM_SBA_CMD_TYPE_B; |
1215 | cmdsp->data = src + msg_offset; |
1216 | cmdsp->data_len = msg_len; |
1217 | cmdsp++; |
1218 | |
1219 | dpos -= pos; |
1220 | |
1221 | /* Multiple Type-A command to generate final Q */ |
1222 | while (dpos) { |
1223 | pos = (dpos < req->sba->max_pq_coefs) ? |
1224 | dpos : (req->sba->max_pq_coefs - 1); |
1225 | |
1226 | /* |
1227 | * Type-A command to generate Q with buf0 and |
1228 | * buf1 store result in buf0 |
1229 | */ |
1230 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_A, |
1231 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
1232 | cmd = sba_cmd_enc(cmd, val: msg_len, |
1233 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
1234 | c_mdata = sba_cmd_pq_c_mdata(d: pos, b1: 0, b0: 1); |
1235 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
1236 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
1237 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), |
1238 | SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); |
1239 | cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS, |
1240 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
1241 | cmdsp->cmd = cmd; |
1242 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
1243 | cmdsp->flags = BRCM_SBA_CMD_TYPE_A; |
1244 | cmdsp++; |
1245 | |
1246 | dpos -= pos; |
1247 | } |
1248 | |
1249 | skip_q_computation: |
1250 | if (pq_continue) { |
1251 | /* |
1252 | * Type-B command to XOR previous output with |
1253 | * buf0 and write it into buf0 |
1254 | */ |
1255 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_B, |
1256 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
1257 | cmd = sba_cmd_enc(cmd, val: msg_len, |
1258 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
1259 | c_mdata = sba_cmd_xor_c_mdata(b1: 0, b0: 0); |
1260 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
1261 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
1262 | cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, |
1263 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
1264 | cmdsp->cmd = cmd; |
1265 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
1266 | cmdsp->flags = BRCM_SBA_CMD_TYPE_B; |
1267 | cmdsp->data = *dst_q + msg_offset; |
1268 | cmdsp->data_len = msg_len; |
1269 | cmdsp++; |
1270 | } |
1271 | |
1272 | /* Type-A command to write buf0 */ |
1273 | cmd = sba_cmd_enc(cmd: 0x0, SBA_TYPE_A, |
1274 | SBA_TYPE_SHIFT, SBA_TYPE_MASK); |
1275 | cmd = sba_cmd_enc(cmd, val: msg_len, |
1276 | SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); |
1277 | cmd = sba_cmd_enc(cmd, val: 0x1, |
1278 | SBA_RESP_SHIFT, SBA_RESP_MASK); |
1279 | c_mdata = sba_cmd_write_c_mdata(b0: 0); |
1280 | cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), |
1281 | SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); |
1282 | cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, |
1283 | SBA_CMD_SHIFT, SBA_CMD_MASK); |
1284 | cmdsp->cmd = cmd; |
1285 | *cmdsp->cmd_dma = cpu_to_le64(cmd); |
1286 | cmdsp->flags = BRCM_SBA_CMD_TYPE_A; |
1287 | if (req->sba->hw_resp_size) { |
1288 | cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; |
1289 | cmdsp->resp = resp_dma; |
1290 | cmdsp->resp_len = req->sba->hw_resp_size; |
1291 | } |
1292 | cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; |
1293 | cmdsp->data = *dst_q + msg_offset; |
1294 | cmdsp->data_len = msg_len; |
1295 | cmdsp++; |
1296 | |
1297 | skip_q: |
1298 | /* Fillup brcm_message */ |
1299 | msg->type = BRCM_MESSAGE_SBA; |
1300 | msg->sba.cmds = cmds; |
1301 | msg->sba.cmds_count = cmdsp - cmds; |
1302 | msg->ctx = req; |
1303 | msg->error = 0; |
1304 | } |
1305 | |
1306 | static struct sba_request * |
1307 | sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off, |
1308 | dma_addr_t *dst_p, dma_addr_t *dst_q, |
1309 | dma_addr_t src, u8 scf, size_t len, |
1310 | unsigned long flags) |
1311 | { |
1312 | struct sba_request *req = NULL; |
1313 | |
1314 | /* Alloc new request */ |
1315 | req = sba_alloc_request(sba); |
1316 | if (!req) |
1317 | return NULL; |
1318 | if (flags & DMA_PREP_FENCE) |
1319 | req->flags |= SBA_REQUEST_FENCE; |
1320 | |
1321 | /* Fillup request messages */ |
1322 | sba_fillup_pq_single_msg(req, pq_continue: dmaf_continue(flags), |
1323 | cmds: req->cmds, msg: &req->msg, msg_offset: off, msg_len: len, |
1324 | dst_p, dst_q, src, scf); |
1325 | |
1326 | /* Init async_tx descriptor */ |
1327 | req->tx.flags = flags; |
1328 | req->tx.cookie = -EBUSY; |
1329 | |
1330 | return req; |
1331 | } |
1332 | |
1333 | static struct dma_async_tx_descriptor * |
1334 | sba_prep_dma_pq(struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src, |
1335 | u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) |
1336 | { |
1337 | u32 i, dst_q_index; |
1338 | size_t req_len; |
1339 | bool slow = false; |
1340 | dma_addr_t off = 0; |
1341 | dma_addr_t *dst_p = NULL, *dst_q = NULL; |
1342 | struct sba_device *sba = to_sba_device(dchan); |
1343 | struct sba_request *first = NULL, *req; |
1344 | |
1345 | /* Sanity checks */ |
1346 | if (unlikely(src_cnt > sba->max_pq_srcs)) |
1347 | return NULL; |
1348 | for (i = 0; i < src_cnt; i++) |
1349 | if (sba->max_pq_coefs <= raid6_gflog[scf[i]]) |
1350 | slow = true; |
1351 | |
1352 | /* Figure-out P and Q destination addresses */ |
1353 | if (!(flags & DMA_PREP_PQ_DISABLE_P)) |
1354 | dst_p = &dst[0]; |
1355 | if (!(flags & DMA_PREP_PQ_DISABLE_Q)) |
1356 | dst_q = &dst[1]; |
1357 | |
1358 | /* Create chained requests where each request is upto hw_buf_size */ |
1359 | while (len) { |
1360 | req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; |
1361 | |
1362 | if (slow) { |
1363 | dst_q_index = src_cnt; |
1364 | |
1365 | if (dst_q) { |
1366 | for (i = 0; i < src_cnt; i++) { |
1367 | if (*dst_q == src[i]) { |
1368 | dst_q_index = i; |
1369 | break; |
1370 | } |
1371 | } |
1372 | } |
1373 | |
1374 | if (dst_q_index < src_cnt) { |
1375 | i = dst_q_index; |
1376 | req = sba_prep_dma_pq_single_req(sba, |
1377 | off, dst_p, dst_q, src: src[i], scf: scf[i], |
1378 | len: req_len, flags: flags | DMA_PREP_FENCE); |
1379 | if (!req) |
1380 | goto fail; |
1381 | |
1382 | if (first) |
1383 | sba_chain_request(first, req); |
1384 | else |
1385 | first = req; |
1386 | |
1387 | flags |= DMA_PREP_CONTINUE; |
1388 | } |
1389 | |
1390 | for (i = 0; i < src_cnt; i++) { |
1391 | if (dst_q_index == i) |
1392 | continue; |
1393 | |
1394 | req = sba_prep_dma_pq_single_req(sba, |
1395 | off, dst_p, dst_q, src: src[i], scf: scf[i], |
1396 | len: req_len, flags: flags | DMA_PREP_FENCE); |
1397 | if (!req) |
1398 | goto fail; |
1399 | |
1400 | if (first) |
1401 | sba_chain_request(first, req); |
1402 | else |
1403 | first = req; |
1404 | |
1405 | flags |= DMA_PREP_CONTINUE; |
1406 | } |
1407 | } else { |
1408 | req = sba_prep_dma_pq_req(sba, off, |
1409 | dst_p, dst_q, src, src_cnt, |
1410 | scf, len: req_len, flags); |
1411 | if (!req) |
1412 | goto fail; |
1413 | |
1414 | if (first) |
1415 | sba_chain_request(first, req); |
1416 | else |
1417 | first = req; |
1418 | } |
1419 | |
1420 | off += req_len; |
1421 | len -= req_len; |
1422 | } |
1423 | |
1424 | return (first) ? &first->tx : NULL; |
1425 | |
1426 | fail: |
1427 | if (first) |
1428 | sba_free_chained_requests(req: first); |
1429 | return NULL; |
1430 | } |
1431 | |
1432 | /* ====== Mailbox callbacks ===== */ |
1433 | |
1434 | static void sba_receive_message(struct mbox_client *cl, void *msg) |
1435 | { |
1436 | struct brcm_message *m = msg; |
1437 | struct sba_request *req = m->ctx; |
1438 | struct sba_device *sba = req->sba; |
1439 | |
1440 | /* Error count if message has error */ |
1441 | if (m->error < 0) |
1442 | dev_err(sba->dev, "%s got message with error %d" , |
1443 | dma_chan_name(&sba->dma_chan), m->error); |
1444 | |
1445 | /* Process received request */ |
1446 | sba_process_received_request(sba, req); |
1447 | } |
1448 | |
1449 | /* ====== Debugfs callbacks ====== */ |
1450 | |
1451 | static int sba_debugfs_stats_show(struct seq_file *file, void *offset) |
1452 | { |
1453 | struct sba_device *sba = dev_get_drvdata(dev: file->private); |
1454 | |
1455 | /* Write stats in file */ |
1456 | sba_write_stats_in_seqfile(sba, file); |
1457 | |
1458 | return 0; |
1459 | } |
1460 | |
1461 | /* ====== Platform driver routines ===== */ |
1462 | |
1463 | static int sba_prealloc_channel_resources(struct sba_device *sba) |
1464 | { |
1465 | int i, j, ret = 0; |
1466 | struct sba_request *req = NULL; |
1467 | |
1468 | sba->resp_base = dma_alloc_coherent(dev: sba->mbox_dev, |
1469 | size: sba->max_resp_pool_size, |
1470 | dma_handle: &sba->resp_dma_base, GFP_KERNEL); |
1471 | if (!sba->resp_base) |
1472 | return -ENOMEM; |
1473 | |
1474 | sba->cmds_base = dma_alloc_coherent(dev: sba->mbox_dev, |
1475 | size: sba->max_cmds_pool_size, |
1476 | dma_handle: &sba->cmds_dma_base, GFP_KERNEL); |
1477 | if (!sba->cmds_base) { |
1478 | ret = -ENOMEM; |
1479 | goto fail_free_resp_pool; |
1480 | } |
1481 | |
1482 | spin_lock_init(&sba->reqs_lock); |
1483 | sba->reqs_fence = false; |
1484 | INIT_LIST_HEAD(list: &sba->reqs_alloc_list); |
1485 | INIT_LIST_HEAD(list: &sba->reqs_pending_list); |
1486 | INIT_LIST_HEAD(list: &sba->reqs_active_list); |
1487 | INIT_LIST_HEAD(list: &sba->reqs_aborted_list); |
1488 | INIT_LIST_HEAD(list: &sba->reqs_free_list); |
1489 | |
1490 | for (i = 0; i < sba->max_req; i++) { |
1491 | req = devm_kzalloc(dev: sba->dev, |
1492 | struct_size(req, cmds, sba->max_cmd_per_req), |
1493 | GFP_KERNEL); |
1494 | if (!req) { |
1495 | ret = -ENOMEM; |
1496 | goto fail_free_cmds_pool; |
1497 | } |
1498 | INIT_LIST_HEAD(list: &req->node); |
1499 | req->sba = sba; |
1500 | req->flags = SBA_REQUEST_STATE_FREE; |
1501 | INIT_LIST_HEAD(list: &req->next); |
1502 | atomic_set(v: &req->next_pending_count, i: 0); |
1503 | for (j = 0; j < sba->max_cmd_per_req; j++) { |
1504 | req->cmds[j].cmd = 0; |
1505 | req->cmds[j].cmd_dma = sba->cmds_base + |
1506 | (i * sba->max_cmd_per_req + j) * sizeof(u64); |
1507 | req->cmds[j].cmd_dma_addr = sba->cmds_dma_base + |
1508 | (i * sba->max_cmd_per_req + j) * sizeof(u64); |
1509 | req->cmds[j].flags = 0; |
1510 | } |
1511 | memset(&req->msg, 0, sizeof(req->msg)); |
1512 | dma_async_tx_descriptor_init(tx: &req->tx, chan: &sba->dma_chan); |
1513 | async_tx_ack(tx: &req->tx); |
1514 | req->tx.tx_submit = sba_tx_submit; |
1515 | req->tx.phys = sba->resp_dma_base + i * sba->hw_resp_size; |
1516 | list_add_tail(new: &req->node, head: &sba->reqs_free_list); |
1517 | } |
1518 | |
1519 | return 0; |
1520 | |
1521 | fail_free_cmds_pool: |
1522 | dma_free_coherent(dev: sba->mbox_dev, |
1523 | size: sba->max_cmds_pool_size, |
1524 | cpu_addr: sba->cmds_base, dma_handle: sba->cmds_dma_base); |
1525 | fail_free_resp_pool: |
1526 | dma_free_coherent(dev: sba->mbox_dev, |
1527 | size: sba->max_resp_pool_size, |
1528 | cpu_addr: sba->resp_base, dma_handle: sba->resp_dma_base); |
1529 | return ret; |
1530 | } |
1531 | |
1532 | static void sba_freeup_channel_resources(struct sba_device *sba) |
1533 | { |
1534 | dmaengine_terminate_all(chan: &sba->dma_chan); |
1535 | dma_free_coherent(dev: sba->mbox_dev, size: sba->max_cmds_pool_size, |
1536 | cpu_addr: sba->cmds_base, dma_handle: sba->cmds_dma_base); |
1537 | dma_free_coherent(dev: sba->mbox_dev, size: sba->max_resp_pool_size, |
1538 | cpu_addr: sba->resp_base, dma_handle: sba->resp_dma_base); |
1539 | sba->resp_base = NULL; |
1540 | sba->resp_dma_base = 0; |
1541 | } |
1542 | |
1543 | static int sba_async_register(struct sba_device *sba) |
1544 | { |
1545 | int ret; |
1546 | struct dma_device *dma_dev = &sba->dma_dev; |
1547 | |
1548 | /* Initialize DMA channel cookie */ |
1549 | sba->dma_chan.device = dma_dev; |
1550 | dma_cookie_init(chan: &sba->dma_chan); |
1551 | |
1552 | /* Initialize DMA device capability mask */ |
1553 | dma_cap_zero(dma_dev->cap_mask); |
1554 | dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); |
1555 | dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); |
1556 | dma_cap_set(DMA_XOR, dma_dev->cap_mask); |
1557 | dma_cap_set(DMA_PQ, dma_dev->cap_mask); |
1558 | |
1559 | /* |
1560 | * Set mailbox channel device as the base device of |
1561 | * our dma_device because the actual memory accesses |
1562 | * will be done by mailbox controller |
1563 | */ |
1564 | dma_dev->dev = sba->mbox_dev; |
1565 | |
1566 | /* Set base prep routines */ |
1567 | dma_dev->device_free_chan_resources = sba_free_chan_resources; |
1568 | dma_dev->device_terminate_all = sba_device_terminate_all; |
1569 | dma_dev->device_issue_pending = sba_issue_pending; |
1570 | dma_dev->device_tx_status = sba_tx_status; |
1571 | |
1572 | /* Set interrupt routine */ |
1573 | if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) |
1574 | dma_dev->device_prep_dma_interrupt = sba_prep_dma_interrupt; |
1575 | |
1576 | /* Set memcpy routine */ |
1577 | if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) |
1578 | dma_dev->device_prep_dma_memcpy = sba_prep_dma_memcpy; |
1579 | |
1580 | /* Set xor routine and capability */ |
1581 | if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { |
1582 | dma_dev->device_prep_dma_xor = sba_prep_dma_xor; |
1583 | dma_dev->max_xor = sba->max_xor_srcs; |
1584 | } |
1585 | |
1586 | /* Set pq routine and capability */ |
1587 | if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { |
1588 | dma_dev->device_prep_dma_pq = sba_prep_dma_pq; |
1589 | dma_set_maxpq(dma: dma_dev, maxpq: sba->max_pq_srcs, has_pq_continue: 0); |
1590 | } |
1591 | |
1592 | /* Initialize DMA device channel list */ |
1593 | INIT_LIST_HEAD(list: &dma_dev->channels); |
1594 | list_add_tail(new: &sba->dma_chan.device_node, head: &dma_dev->channels); |
1595 | |
1596 | /* Register with Linux async DMA framework*/ |
1597 | ret = dma_async_device_register(device: dma_dev); |
1598 | if (ret) { |
1599 | dev_err(sba->dev, "async device register error %d" , ret); |
1600 | return ret; |
1601 | } |
1602 | |
1603 | dev_info(sba->dev, "%s capabilities: %s%s%s%s\n" , |
1604 | dma_chan_name(&sba->dma_chan), |
1605 | dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "interrupt " : "" , |
1606 | dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "memcpy " : "" , |
1607 | dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "" , |
1608 | dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "" ); |
1609 | |
1610 | return 0; |
1611 | } |
1612 | |
1613 | static int sba_probe(struct platform_device *pdev) |
1614 | { |
1615 | int ret = 0; |
1616 | struct sba_device *sba; |
1617 | struct platform_device *mbox_pdev; |
1618 | struct of_phandle_args args; |
1619 | |
1620 | /* Allocate main SBA struct */ |
1621 | sba = devm_kzalloc(dev: &pdev->dev, size: sizeof(*sba), GFP_KERNEL); |
1622 | if (!sba) |
1623 | return -ENOMEM; |
1624 | |
1625 | sba->dev = &pdev->dev; |
1626 | platform_set_drvdata(pdev, data: sba); |
1627 | |
1628 | /* Number of mailbox channels should be atleast 1 */ |
1629 | ret = of_count_phandle_with_args(np: pdev->dev.of_node, |
1630 | list_name: "mboxes" , cells_name: "#mbox-cells" ); |
1631 | if (ret <= 0) |
1632 | return -ENODEV; |
1633 | |
1634 | /* Determine SBA version from DT compatible string */ |
1635 | if (of_device_is_compatible(device: sba->dev->of_node, "brcm,iproc-sba" )) |
1636 | sba->ver = SBA_VER_1; |
1637 | else if (of_device_is_compatible(device: sba->dev->of_node, |
1638 | "brcm,iproc-sba-v2" )) |
1639 | sba->ver = SBA_VER_2; |
1640 | else |
1641 | return -ENODEV; |
1642 | |
1643 | /* Derived Configuration parameters */ |
1644 | switch (sba->ver) { |
1645 | case SBA_VER_1: |
1646 | sba->hw_buf_size = 4096; |
1647 | sba->hw_resp_size = 8; |
1648 | sba->max_pq_coefs = 6; |
1649 | sba->max_pq_srcs = 6; |
1650 | break; |
1651 | case SBA_VER_2: |
1652 | sba->hw_buf_size = 4096; |
1653 | sba->hw_resp_size = 8; |
1654 | sba->max_pq_coefs = 30; |
1655 | /* |
1656 | * We can support max_pq_srcs == max_pq_coefs because |
1657 | * we are limited by number of SBA commands that we can |
1658 | * fit in one message for underlying ring manager HW. |
1659 | */ |
1660 | sba->max_pq_srcs = 12; |
1661 | break; |
1662 | default: |
1663 | return -EINVAL; |
1664 | } |
1665 | sba->max_req = SBA_MAX_REQ_PER_MBOX_CHANNEL; |
1666 | sba->max_cmd_per_req = sba->max_pq_srcs + 3; |
1667 | sba->max_xor_srcs = sba->max_cmd_per_req - 1; |
1668 | sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size; |
1669 | sba->max_cmds_pool_size = sba->max_req * |
1670 | sba->max_cmd_per_req * sizeof(u64); |
1671 | |
1672 | /* Setup mailbox client */ |
1673 | sba->client.dev = &pdev->dev; |
1674 | sba->client.rx_callback = sba_receive_message; |
1675 | sba->client.tx_block = false; |
1676 | sba->client.knows_txdone = true; |
1677 | sba->client.tx_tout = 0; |
1678 | |
1679 | /* Request mailbox channel */ |
1680 | sba->mchan = mbox_request_channel(cl: &sba->client, index: 0); |
1681 | if (IS_ERR(ptr: sba->mchan)) { |
1682 | ret = PTR_ERR(ptr: sba->mchan); |
1683 | goto fail_free_mchan; |
1684 | } |
1685 | |
1686 | /* Find-out underlying mailbox device */ |
1687 | ret = of_parse_phandle_with_args(np: pdev->dev.of_node, |
1688 | list_name: "mboxes" , cells_name: "#mbox-cells" , index: 0, out_args: &args); |
1689 | if (ret) |
1690 | goto fail_free_mchan; |
1691 | mbox_pdev = of_find_device_by_node(np: args.np); |
1692 | of_node_put(node: args.np); |
1693 | if (!mbox_pdev) { |
1694 | ret = -ENODEV; |
1695 | goto fail_free_mchan; |
1696 | } |
1697 | sba->mbox_dev = &mbox_pdev->dev; |
1698 | |
1699 | /* Prealloc channel resource */ |
1700 | ret = sba_prealloc_channel_resources(sba); |
1701 | if (ret) |
1702 | goto fail_free_mchan; |
1703 | |
1704 | /* Check availability of debugfs */ |
1705 | if (!debugfs_initialized()) |
1706 | goto skip_debugfs; |
1707 | |
1708 | /* Create debugfs root entry */ |
1709 | sba->root = debugfs_create_dir(name: dev_name(dev: sba->dev), NULL); |
1710 | |
1711 | /* Create debugfs stats entry */ |
1712 | debugfs_create_devm_seqfile(dev: sba->dev, name: "stats" , parent: sba->root, |
1713 | read_fn: sba_debugfs_stats_show); |
1714 | |
1715 | skip_debugfs: |
1716 | |
1717 | /* Register DMA device with Linux async framework */ |
1718 | ret = sba_async_register(sba); |
1719 | if (ret) |
1720 | goto fail_free_resources; |
1721 | |
1722 | /* Print device info */ |
1723 | dev_info(sba->dev, "%s using SBAv%d mailbox channel from %s" , |
1724 | dma_chan_name(&sba->dma_chan), sba->ver+1, |
1725 | dev_name(sba->mbox_dev)); |
1726 | |
1727 | return 0; |
1728 | |
1729 | fail_free_resources: |
1730 | debugfs_remove_recursive(dentry: sba->root); |
1731 | sba_freeup_channel_resources(sba); |
1732 | fail_free_mchan: |
1733 | mbox_free_channel(chan: sba->mchan); |
1734 | return ret; |
1735 | } |
1736 | |
1737 | static void sba_remove(struct platform_device *pdev) |
1738 | { |
1739 | struct sba_device *sba = platform_get_drvdata(pdev); |
1740 | |
1741 | dma_async_device_unregister(device: &sba->dma_dev); |
1742 | |
1743 | debugfs_remove_recursive(dentry: sba->root); |
1744 | |
1745 | sba_freeup_channel_resources(sba); |
1746 | |
1747 | mbox_free_channel(chan: sba->mchan); |
1748 | } |
1749 | |
1750 | static const struct of_device_id sba_of_match[] = { |
1751 | { .compatible = "brcm,iproc-sba" , }, |
1752 | { .compatible = "brcm,iproc-sba-v2" , }, |
1753 | {}, |
1754 | }; |
1755 | MODULE_DEVICE_TABLE(of, sba_of_match); |
1756 | |
1757 | static struct platform_driver sba_driver = { |
1758 | .probe = sba_probe, |
1759 | .remove_new = sba_remove, |
1760 | .driver = { |
1761 | .name = "bcm-sba-raid" , |
1762 | .of_match_table = sba_of_match, |
1763 | }, |
1764 | }; |
1765 | module_platform_driver(sba_driver); |
1766 | |
1767 | MODULE_DESCRIPTION("Broadcom SBA RAID driver" ); |
1768 | MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>" ); |
1769 | MODULE_LICENSE("GPL v2" ); |
1770 | |