1 | // SPDX-License-Identifier: GPL-2.0 |
2 | // Copyright 2019 NXP |
3 | |
4 | #include <linux/init.h> |
5 | #include <linux/module.h> |
6 | #include <linux/dmapool.h> |
7 | #include <linux/of_irq.h> |
8 | #include <linux/iommu.h> |
9 | #include <linux/sys_soc.h> |
10 | #include <linux/fsl/mc.h> |
11 | #include <soc/fsl/dpaa2-io.h> |
12 | |
13 | #include "../virt-dma.h" |
14 | #include "dpdmai.h" |
15 | #include "dpaa2-qdma.h" |
16 | |
17 | static bool smmu_disable = true; |
18 | |
19 | static struct dpaa2_qdma_chan *to_dpaa2_qdma_chan(struct dma_chan *chan) |
20 | { |
21 | return container_of(chan, struct dpaa2_qdma_chan, vchan.chan); |
22 | } |
23 | |
24 | static struct dpaa2_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd) |
25 | { |
26 | return container_of(vd, struct dpaa2_qdma_comp, vdesc); |
27 | } |
28 | |
29 | static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan) |
30 | { |
31 | struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); |
32 | struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma; |
33 | struct device *dev = &dpaa2_qdma->priv->dpdmai_dev->dev; |
34 | |
35 | dpaa2_chan->fd_pool = dma_pool_create(name: "fd_pool" , dev, |
36 | size: sizeof(struct dpaa2_fd), |
37 | align: sizeof(struct dpaa2_fd), allocation: 0); |
38 | if (!dpaa2_chan->fd_pool) |
39 | goto err; |
40 | |
41 | dpaa2_chan->fl_pool = |
42 | dma_pool_create(name: "fl_pool" , dev, |
43 | size: sizeof(struct dpaa2_fl_entry) * 3, |
44 | align: sizeof(struct dpaa2_fl_entry), allocation: 0); |
45 | |
46 | if (!dpaa2_chan->fl_pool) |
47 | goto err_fd; |
48 | |
49 | dpaa2_chan->sdd_pool = |
50 | dma_pool_create(name: "sdd_pool" , dev, |
51 | size: sizeof(struct dpaa2_qdma_sd_d) * 2, |
52 | align: sizeof(struct dpaa2_qdma_sd_d), allocation: 0); |
53 | if (!dpaa2_chan->sdd_pool) |
54 | goto err_fl; |
55 | |
56 | return dpaa2_qdma->desc_allocated++; |
57 | err_fl: |
58 | dma_pool_destroy(pool: dpaa2_chan->fl_pool); |
59 | err_fd: |
60 | dma_pool_destroy(pool: dpaa2_chan->fd_pool); |
61 | err: |
62 | return -ENOMEM; |
63 | } |
64 | |
65 | static void dpaa2_qdma_free_chan_resources(struct dma_chan *chan) |
66 | { |
67 | struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); |
68 | struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma; |
69 | unsigned long flags; |
70 | |
71 | LIST_HEAD(head); |
72 | |
73 | spin_lock_irqsave(&dpaa2_chan->vchan.lock, flags); |
74 | vchan_get_all_descriptors(vc: &dpaa2_chan->vchan, head: &head); |
75 | spin_unlock_irqrestore(lock: &dpaa2_chan->vchan.lock, flags); |
76 | |
77 | vchan_dma_desc_free_list(vc: &dpaa2_chan->vchan, head: &head); |
78 | |
79 | dpaa2_dpdmai_free_comp(qchan: dpaa2_chan, head: &dpaa2_chan->comp_used); |
80 | dpaa2_dpdmai_free_comp(qchan: dpaa2_chan, head: &dpaa2_chan->comp_free); |
81 | |
82 | dma_pool_destroy(pool: dpaa2_chan->fd_pool); |
83 | dma_pool_destroy(pool: dpaa2_chan->fl_pool); |
84 | dma_pool_destroy(pool: dpaa2_chan->sdd_pool); |
85 | dpaa2_qdma->desc_allocated--; |
86 | } |
87 | |
88 | /* |
89 | * Request a command descriptor for enqueue. |
90 | */ |
91 | static struct dpaa2_qdma_comp * |
92 | dpaa2_qdma_request_desc(struct dpaa2_qdma_chan *dpaa2_chan) |
93 | { |
94 | struct dpaa2_qdma_priv *qdma_priv = dpaa2_chan->qdma->priv; |
95 | struct device *dev = &qdma_priv->dpdmai_dev->dev; |
96 | struct dpaa2_qdma_comp *comp_temp = NULL; |
97 | unsigned long flags; |
98 | |
99 | spin_lock_irqsave(&dpaa2_chan->queue_lock, flags); |
100 | if (list_empty(head: &dpaa2_chan->comp_free)) { |
101 | spin_unlock_irqrestore(lock: &dpaa2_chan->queue_lock, flags); |
102 | comp_temp = kzalloc(size: sizeof(*comp_temp), GFP_NOWAIT); |
103 | if (!comp_temp) |
104 | goto err; |
105 | comp_temp->fd_virt_addr = |
106 | dma_pool_alloc(pool: dpaa2_chan->fd_pool, GFP_NOWAIT, |
107 | handle: &comp_temp->fd_bus_addr); |
108 | if (!comp_temp->fd_virt_addr) |
109 | goto err_comp; |
110 | |
111 | comp_temp->fl_virt_addr = |
112 | dma_pool_alloc(pool: dpaa2_chan->fl_pool, GFP_NOWAIT, |
113 | handle: &comp_temp->fl_bus_addr); |
114 | if (!comp_temp->fl_virt_addr) |
115 | goto err_fd_virt; |
116 | |
117 | comp_temp->desc_virt_addr = |
118 | dma_pool_alloc(pool: dpaa2_chan->sdd_pool, GFP_NOWAIT, |
119 | handle: &comp_temp->desc_bus_addr); |
120 | if (!comp_temp->desc_virt_addr) |
121 | goto err_fl_virt; |
122 | |
123 | comp_temp->qchan = dpaa2_chan; |
124 | return comp_temp; |
125 | } |
126 | |
127 | comp_temp = list_first_entry(&dpaa2_chan->comp_free, |
128 | struct dpaa2_qdma_comp, list); |
129 | list_del(entry: &comp_temp->list); |
130 | spin_unlock_irqrestore(lock: &dpaa2_chan->queue_lock, flags); |
131 | |
132 | comp_temp->qchan = dpaa2_chan; |
133 | |
134 | return comp_temp; |
135 | |
136 | err_fl_virt: |
137 | dma_pool_free(pool: dpaa2_chan->fl_pool, |
138 | vaddr: comp_temp->fl_virt_addr, |
139 | addr: comp_temp->fl_bus_addr); |
140 | err_fd_virt: |
141 | dma_pool_free(pool: dpaa2_chan->fd_pool, |
142 | vaddr: comp_temp->fd_virt_addr, |
143 | addr: comp_temp->fd_bus_addr); |
144 | err_comp: |
145 | kfree(objp: comp_temp); |
146 | err: |
147 | dev_err(dev, "Failed to request descriptor\n" ); |
148 | return NULL; |
149 | } |
150 | |
151 | static void |
152 | dpaa2_qdma_populate_fd(u32 format, struct dpaa2_qdma_comp *dpaa2_comp) |
153 | { |
154 | struct dpaa2_fd *fd; |
155 | |
156 | fd = dpaa2_comp->fd_virt_addr; |
157 | memset(fd, 0, sizeof(struct dpaa2_fd)); |
158 | |
159 | /* fd populated */ |
160 | dpaa2_fd_set_addr(fd, addr: dpaa2_comp->fl_bus_addr); |
161 | |
162 | /* |
163 | * Bypass memory translation, Frame list format, short length disable |
164 | * we need to disable BMT if fsl-mc use iova addr |
165 | */ |
166 | if (smmu_disable) |
167 | dpaa2_fd_set_bpid(fd, QMAN_FD_BMT_ENABLE); |
168 | dpaa2_fd_set_format(fd, QMAN_FD_FMT_ENABLE | QMAN_FD_SL_DISABLE); |
169 | |
170 | dpaa2_fd_set_frc(fd, frc: format | QDMA_SER_CTX); |
171 | } |
172 | |
173 | /* first frame list for descriptor buffer */ |
174 | static void |
175 | dpaa2_qdma_populate_first_framel(struct dpaa2_fl_entry *f_list, |
176 | struct dpaa2_qdma_comp *dpaa2_comp, |
177 | bool wrt_changed) |
178 | { |
179 | struct dpaa2_qdma_sd_d *sdd; |
180 | |
181 | sdd = dpaa2_comp->desc_virt_addr; |
182 | memset(sdd, 0, 2 * (sizeof(*sdd))); |
183 | |
184 | /* source descriptor CMD */ |
185 | sdd->cmd = cpu_to_le32(QDMA_SD_CMD_RDTTYPE_COHERENT); |
186 | sdd++; |
187 | |
188 | /* dest descriptor CMD */ |
189 | if (wrt_changed) |
190 | sdd->cmd = cpu_to_le32(LX2160_QDMA_DD_CMD_WRTTYPE_COHERENT); |
191 | else |
192 | sdd->cmd = cpu_to_le32(QDMA_DD_CMD_WRTTYPE_COHERENT); |
193 | |
194 | memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); |
195 | |
196 | /* first frame list to source descriptor */ |
197 | dpaa2_fl_set_addr(fle: f_list, addr: dpaa2_comp->desc_bus_addr); |
198 | dpaa2_fl_set_len(fle: f_list, len: 0x20); |
199 | dpaa2_fl_set_format(fle: f_list, QDMA_FL_FMT_SBF | QDMA_FL_SL_LONG); |
200 | |
201 | /* bypass memory translation */ |
202 | if (smmu_disable) |
203 | f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); |
204 | } |
205 | |
206 | /* source and destination frame list */ |
207 | static void |
208 | dpaa2_qdma_populate_frames(struct dpaa2_fl_entry *f_list, |
209 | dma_addr_t dst, dma_addr_t src, |
210 | size_t len, uint8_t fmt) |
211 | { |
212 | /* source frame list to source buffer */ |
213 | memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); |
214 | |
215 | dpaa2_fl_set_addr(fle: f_list, addr: src); |
216 | dpaa2_fl_set_len(fle: f_list, len); |
217 | |
218 | /* single buffer frame or scatter gather frame */ |
219 | dpaa2_fl_set_format(fle: f_list, format: (fmt | QDMA_FL_SL_LONG)); |
220 | |
221 | /* bypass memory translation */ |
222 | if (smmu_disable) |
223 | f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); |
224 | |
225 | f_list++; |
226 | |
227 | /* destination frame list to destination buffer */ |
228 | memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); |
229 | |
230 | dpaa2_fl_set_addr(fle: f_list, addr: dst); |
231 | dpaa2_fl_set_len(fle: f_list, len); |
232 | dpaa2_fl_set_format(fle: f_list, format: (fmt | QDMA_FL_SL_LONG)); |
233 | /* single buffer frame or scatter gather frame */ |
234 | dpaa2_fl_set_final(fle: f_list, QDMA_FL_F); |
235 | /* bypass memory translation */ |
236 | if (smmu_disable) |
237 | f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); |
238 | } |
239 | |
240 | static struct dma_async_tx_descriptor |
241 | *dpaa2_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, |
242 | dma_addr_t src, size_t len, ulong flags) |
243 | { |
244 | struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); |
245 | struct dpaa2_qdma_engine *dpaa2_qdma; |
246 | struct dpaa2_qdma_comp *dpaa2_comp; |
247 | struct dpaa2_fl_entry *f_list; |
248 | bool wrt_changed; |
249 | |
250 | dpaa2_qdma = dpaa2_chan->qdma; |
251 | dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan); |
252 | if (!dpaa2_comp) |
253 | return NULL; |
254 | |
255 | wrt_changed = (bool)dpaa2_qdma->qdma_wrtype_fixup; |
256 | |
257 | /* populate Frame descriptor */ |
258 | dpaa2_qdma_populate_fd(QDMA_FD_LONG_FORMAT, dpaa2_comp); |
259 | |
260 | f_list = dpaa2_comp->fl_virt_addr; |
261 | |
262 | /* first frame list for descriptor buffer (logn format) */ |
263 | dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp, wrt_changed); |
264 | |
265 | f_list++; |
266 | |
267 | dpaa2_qdma_populate_frames(f_list, dst, src, len, QDMA_FL_FMT_SBF); |
268 | |
269 | return vchan_tx_prep(vc: &dpaa2_chan->vchan, vd: &dpaa2_comp->vdesc, tx_flags: flags); |
270 | } |
271 | |
272 | static void dpaa2_qdma_issue_pending(struct dma_chan *chan) |
273 | { |
274 | struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); |
275 | struct dpaa2_qdma_comp *dpaa2_comp; |
276 | struct virt_dma_desc *vdesc; |
277 | struct dpaa2_fd *fd; |
278 | unsigned long flags; |
279 | int err; |
280 | |
281 | spin_lock_irqsave(&dpaa2_chan->queue_lock, flags); |
282 | spin_lock(lock: &dpaa2_chan->vchan.lock); |
283 | if (vchan_issue_pending(vc: &dpaa2_chan->vchan)) { |
284 | vdesc = vchan_next_desc(vc: &dpaa2_chan->vchan); |
285 | if (!vdesc) |
286 | goto err_enqueue; |
287 | dpaa2_comp = to_fsl_qdma_comp(vd: vdesc); |
288 | |
289 | fd = dpaa2_comp->fd_virt_addr; |
290 | |
291 | list_del(entry: &vdesc->node); |
292 | list_add_tail(new: &dpaa2_comp->list, head: &dpaa2_chan->comp_used); |
293 | |
294 | err = dpaa2_io_service_enqueue_fq(NULL, fqid: dpaa2_chan->fqid, fd); |
295 | if (err) { |
296 | list_move_tail(list: &dpaa2_comp->list, |
297 | head: &dpaa2_chan->comp_free); |
298 | } |
299 | } |
300 | err_enqueue: |
301 | spin_unlock(lock: &dpaa2_chan->vchan.lock); |
302 | spin_unlock_irqrestore(lock: &dpaa2_chan->queue_lock, flags); |
303 | } |
304 | |
305 | static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev) |
306 | { |
307 | struct dpaa2_qdma_priv_per_prio *ppriv; |
308 | struct device *dev = &ls_dev->dev; |
309 | struct dpaa2_qdma_priv *priv; |
310 | u8 prio_def = DPDMAI_PRIO_NUM; |
311 | int err = -EINVAL; |
312 | int i; |
313 | |
314 | priv = dev_get_drvdata(dev); |
315 | |
316 | priv->dev = dev; |
317 | priv->dpqdma_id = ls_dev->obj_desc.id; |
318 | |
319 | /* Get the handle for the DPDMAI this interface is associate with */ |
320 | err = dpdmai_open(mc_io: priv->mc_io, cmd_flags: 0, dpdmai_id: priv->dpqdma_id, token: &ls_dev->mc_handle); |
321 | if (err) { |
322 | dev_err(dev, "dpdmai_open() failed\n" ); |
323 | return err; |
324 | } |
325 | |
326 | dev_dbg(dev, "Opened dpdmai object successfully\n" ); |
327 | |
328 | err = dpdmai_get_attributes(mc_io: priv->mc_io, cmd_flags: 0, token: ls_dev->mc_handle, |
329 | attr: &priv->dpdmai_attr); |
330 | if (err) { |
331 | dev_err(dev, "dpdmai_get_attributes() failed\n" ); |
332 | goto exit; |
333 | } |
334 | |
335 | if (priv->dpdmai_attr.version.major > DPDMAI_VER_MAJOR) { |
336 | err = -EINVAL; |
337 | dev_err(dev, "DPDMAI major version mismatch\n" |
338 | "Found %u.%u, supported version is %u.%u\n" , |
339 | priv->dpdmai_attr.version.major, |
340 | priv->dpdmai_attr.version.minor, |
341 | DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR); |
342 | goto exit; |
343 | } |
344 | |
345 | if (priv->dpdmai_attr.version.minor > DPDMAI_VER_MINOR) { |
346 | err = -EINVAL; |
347 | dev_err(dev, "DPDMAI minor version mismatch\n" |
348 | "Found %u.%u, supported version is %u.%u\n" , |
349 | priv->dpdmai_attr.version.major, |
350 | priv->dpdmai_attr.version.minor, |
351 | DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR); |
352 | goto exit; |
353 | } |
354 | |
355 | priv->num_pairs = min(priv->dpdmai_attr.num_of_priorities, prio_def); |
356 | ppriv = kcalloc(n: priv->num_pairs, size: sizeof(*ppriv), GFP_KERNEL); |
357 | if (!ppriv) { |
358 | err = -ENOMEM; |
359 | goto exit; |
360 | } |
361 | priv->ppriv = ppriv; |
362 | |
363 | for (i = 0; i < priv->num_pairs; i++) { |
364 | err = dpdmai_get_rx_queue(mc_io: priv->mc_io, cmd_flags: 0, token: ls_dev->mc_handle, |
365 | priority: i, attr: &priv->rx_queue_attr[i]); |
366 | if (err) { |
367 | dev_err(dev, "dpdmai_get_rx_queue() failed\n" ); |
368 | goto exit; |
369 | } |
370 | ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid; |
371 | |
372 | err = dpdmai_get_tx_queue(mc_io: priv->mc_io, cmd_flags: 0, token: ls_dev->mc_handle, |
373 | priority: i, fqid: &priv->tx_fqid[i]); |
374 | if (err) { |
375 | dev_err(dev, "dpdmai_get_tx_queue() failed\n" ); |
376 | goto exit; |
377 | } |
378 | ppriv->req_fqid = priv->tx_fqid[i]; |
379 | ppriv->prio = i; |
380 | ppriv->priv = priv; |
381 | ppriv++; |
382 | } |
383 | |
384 | return 0; |
385 | exit: |
386 | dpdmai_close(mc_io: priv->mc_io, cmd_flags: 0, token: ls_dev->mc_handle); |
387 | return err; |
388 | } |
389 | |
390 | static void dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx *ctx) |
391 | { |
392 | struct dpaa2_qdma_priv_per_prio *ppriv = container_of(ctx, |
393 | struct dpaa2_qdma_priv_per_prio, nctx); |
394 | struct dpaa2_qdma_comp *dpaa2_comp, *_comp_tmp; |
395 | struct dpaa2_qdma_priv *priv = ppriv->priv; |
396 | u32 n_chans = priv->dpaa2_qdma->n_chans; |
397 | struct dpaa2_qdma_chan *qchan; |
398 | const struct dpaa2_fd *fd_eq; |
399 | const struct dpaa2_fd *fd; |
400 | struct dpaa2_dq *dq; |
401 | int is_last = 0; |
402 | int found; |
403 | u8 status; |
404 | int err; |
405 | int i; |
406 | |
407 | do { |
408 | err = dpaa2_io_service_pull_fq(NULL, fqid: ppriv->rsp_fqid, |
409 | s: ppriv->store); |
410 | } while (err); |
411 | |
412 | while (!is_last) { |
413 | do { |
414 | dq = dpaa2_io_store_next(s: ppriv->store, is_last: &is_last); |
415 | } while (!is_last && !dq); |
416 | if (!dq) { |
417 | dev_err(priv->dev, "FQID returned no valid frames!\n" ); |
418 | continue; |
419 | } |
420 | |
421 | /* obtain FD and process the error */ |
422 | fd = dpaa2_dq_fd(dq); |
423 | |
424 | status = dpaa2_fd_get_ctrl(fd) & 0xff; |
425 | if (status) |
426 | dev_err(priv->dev, "FD error occurred\n" ); |
427 | found = 0; |
428 | for (i = 0; i < n_chans; i++) { |
429 | qchan = &priv->dpaa2_qdma->chans[i]; |
430 | spin_lock(lock: &qchan->queue_lock); |
431 | if (list_empty(head: &qchan->comp_used)) { |
432 | spin_unlock(lock: &qchan->queue_lock); |
433 | continue; |
434 | } |
435 | list_for_each_entry_safe(dpaa2_comp, _comp_tmp, |
436 | &qchan->comp_used, list) { |
437 | fd_eq = dpaa2_comp->fd_virt_addr; |
438 | |
439 | if (le64_to_cpu(fd_eq->simple.addr) == |
440 | le64_to_cpu(fd->simple.addr)) { |
441 | spin_lock(lock: &qchan->vchan.lock); |
442 | vchan_cookie_complete(vd: & |
443 | dpaa2_comp->vdesc); |
444 | spin_unlock(lock: &qchan->vchan.lock); |
445 | found = 1; |
446 | break; |
447 | } |
448 | } |
449 | spin_unlock(lock: &qchan->queue_lock); |
450 | if (found) |
451 | break; |
452 | } |
453 | } |
454 | |
455 | dpaa2_io_service_rearm(NULL, ctx); |
456 | } |
457 | |
458 | static int __cold dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv *priv) |
459 | { |
460 | struct dpaa2_qdma_priv_per_prio *ppriv; |
461 | struct device *dev = priv->dev; |
462 | int err = -EINVAL; |
463 | int i, num; |
464 | |
465 | num = priv->num_pairs; |
466 | ppriv = priv->ppriv; |
467 | for (i = 0; i < num; i++) { |
468 | ppriv->nctx.is_cdan = 0; |
469 | ppriv->nctx.desired_cpu = DPAA2_IO_ANY_CPU; |
470 | ppriv->nctx.id = ppriv->rsp_fqid; |
471 | ppriv->nctx.cb = dpaa2_qdma_fqdan_cb; |
472 | err = dpaa2_io_service_register(NULL, ctx: &ppriv->nctx, dev); |
473 | if (err) { |
474 | dev_err(dev, "Notification register failed\n" ); |
475 | goto err_service; |
476 | } |
477 | |
478 | ppriv->store = |
479 | dpaa2_io_store_create(DPAA2_QDMA_STORE_SIZE, dev); |
480 | if (!ppriv->store) { |
481 | err = -ENOMEM; |
482 | dev_err(dev, "dpaa2_io_store_create() failed\n" ); |
483 | goto err_store; |
484 | } |
485 | |
486 | ppriv++; |
487 | } |
488 | return 0; |
489 | |
490 | err_store: |
491 | dpaa2_io_service_deregister(NULL, ctx: &ppriv->nctx, dev); |
492 | err_service: |
493 | ppriv--; |
494 | while (ppriv >= priv->ppriv) { |
495 | dpaa2_io_service_deregister(NULL, ctx: &ppriv->nctx, dev); |
496 | dpaa2_io_store_destroy(s: ppriv->store); |
497 | ppriv--; |
498 | } |
499 | return err; |
500 | } |
501 | |
502 | static void dpaa2_dpmai_store_free(struct dpaa2_qdma_priv *priv) |
503 | { |
504 | struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; |
505 | int i; |
506 | |
507 | for (i = 0; i < priv->num_pairs; i++) { |
508 | dpaa2_io_store_destroy(s: ppriv->store); |
509 | ppriv++; |
510 | } |
511 | } |
512 | |
513 | static void dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv *priv) |
514 | { |
515 | struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; |
516 | struct device *dev = priv->dev; |
517 | int i; |
518 | |
519 | for (i = 0; i < priv->num_pairs; i++) { |
520 | dpaa2_io_service_deregister(NULL, ctx: &ppriv->nctx, dev); |
521 | ppriv++; |
522 | } |
523 | } |
524 | |
525 | static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv) |
526 | { |
527 | struct dpdmai_rx_queue_cfg rx_queue_cfg; |
528 | struct dpaa2_qdma_priv_per_prio *ppriv; |
529 | struct device *dev = priv->dev; |
530 | struct fsl_mc_device *ls_dev; |
531 | int i, num; |
532 | int err; |
533 | |
534 | ls_dev = to_fsl_mc_device(dev); |
535 | num = priv->num_pairs; |
536 | ppriv = priv->ppriv; |
537 | for (i = 0; i < num; i++) { |
538 | rx_queue_cfg.options = DPDMAI_QUEUE_OPT_USER_CTX | |
539 | DPDMAI_QUEUE_OPT_DEST; |
540 | rx_queue_cfg.user_ctx = ppriv->nctx.qman64; |
541 | rx_queue_cfg.dest_cfg.dest_type = DPDMAI_DEST_DPIO; |
542 | rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; |
543 | rx_queue_cfg.dest_cfg.priority = ppriv->prio; |
544 | err = dpdmai_set_rx_queue(mc_io: priv->mc_io, cmd_flags: 0, token: ls_dev->mc_handle, |
545 | priority: rx_queue_cfg.dest_cfg.priority, |
546 | cfg: &rx_queue_cfg); |
547 | if (err) { |
548 | dev_err(dev, "dpdmai_set_rx_queue() failed\n" ); |
549 | return err; |
550 | } |
551 | |
552 | ppriv++; |
553 | } |
554 | |
555 | return 0; |
556 | } |
557 | |
558 | static int __cold dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv *priv) |
559 | { |
560 | struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; |
561 | struct device *dev = priv->dev; |
562 | struct fsl_mc_device *ls_dev; |
563 | int err = 0; |
564 | int i; |
565 | |
566 | ls_dev = to_fsl_mc_device(dev); |
567 | |
568 | for (i = 0; i < priv->num_pairs; i++) { |
569 | ppriv->nctx.qman64 = 0; |
570 | ppriv->nctx.dpio_id = 0; |
571 | ppriv++; |
572 | } |
573 | |
574 | err = dpdmai_reset(mc_io: priv->mc_io, cmd_flags: 0, token: ls_dev->mc_handle); |
575 | if (err) |
576 | dev_err(dev, "dpdmai_reset() failed\n" ); |
577 | |
578 | return err; |
579 | } |
580 | |
581 | static void dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan, |
582 | struct list_head *head) |
583 | { |
584 | struct dpaa2_qdma_comp *comp_tmp, *_comp_tmp; |
585 | unsigned long flags; |
586 | |
587 | list_for_each_entry_safe(comp_tmp, _comp_tmp, |
588 | head, list) { |
589 | spin_lock_irqsave(&qchan->queue_lock, flags); |
590 | list_del(entry: &comp_tmp->list); |
591 | spin_unlock_irqrestore(lock: &qchan->queue_lock, flags); |
592 | dma_pool_free(pool: qchan->fd_pool, |
593 | vaddr: comp_tmp->fd_virt_addr, |
594 | addr: comp_tmp->fd_bus_addr); |
595 | dma_pool_free(pool: qchan->fl_pool, |
596 | vaddr: comp_tmp->fl_virt_addr, |
597 | addr: comp_tmp->fl_bus_addr); |
598 | dma_pool_free(pool: qchan->sdd_pool, |
599 | vaddr: comp_tmp->desc_virt_addr, |
600 | addr: comp_tmp->desc_bus_addr); |
601 | kfree(objp: comp_tmp); |
602 | } |
603 | } |
604 | |
605 | static void dpaa2_dpdmai_free_channels(struct dpaa2_qdma_engine *dpaa2_qdma) |
606 | { |
607 | struct dpaa2_qdma_chan *qchan; |
608 | int num, i; |
609 | |
610 | num = dpaa2_qdma->n_chans; |
611 | for (i = 0; i < num; i++) { |
612 | qchan = &dpaa2_qdma->chans[i]; |
613 | dpaa2_dpdmai_free_comp(qchan, head: &qchan->comp_used); |
614 | dpaa2_dpdmai_free_comp(qchan, head: &qchan->comp_free); |
615 | dma_pool_destroy(pool: qchan->fd_pool); |
616 | dma_pool_destroy(pool: qchan->fl_pool); |
617 | dma_pool_destroy(pool: qchan->sdd_pool); |
618 | } |
619 | } |
620 | |
621 | static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc) |
622 | { |
623 | struct dpaa2_qdma_comp *dpaa2_comp; |
624 | struct dpaa2_qdma_chan *qchan; |
625 | unsigned long flags; |
626 | |
627 | dpaa2_comp = to_fsl_qdma_comp(vd: vdesc); |
628 | qchan = dpaa2_comp->qchan; |
629 | spin_lock_irqsave(&qchan->queue_lock, flags); |
630 | list_move_tail(list: &dpaa2_comp->list, head: &qchan->comp_free); |
631 | spin_unlock_irqrestore(lock: &qchan->queue_lock, flags); |
632 | } |
633 | |
634 | static int dpaa2_dpdmai_init_channels(struct dpaa2_qdma_engine *dpaa2_qdma) |
635 | { |
636 | struct dpaa2_qdma_priv *priv = dpaa2_qdma->priv; |
637 | struct dpaa2_qdma_chan *dpaa2_chan; |
638 | int num = priv->num_pairs; |
639 | int i; |
640 | |
641 | INIT_LIST_HEAD(list: &dpaa2_qdma->dma_dev.channels); |
642 | for (i = 0; i < dpaa2_qdma->n_chans; i++) { |
643 | dpaa2_chan = &dpaa2_qdma->chans[i]; |
644 | dpaa2_chan->qdma = dpaa2_qdma; |
645 | dpaa2_chan->fqid = priv->tx_fqid[i % num]; |
646 | dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc; |
647 | vchan_init(vc: &dpaa2_chan->vchan, dmadev: &dpaa2_qdma->dma_dev); |
648 | spin_lock_init(&dpaa2_chan->queue_lock); |
649 | INIT_LIST_HEAD(list: &dpaa2_chan->comp_used); |
650 | INIT_LIST_HEAD(list: &dpaa2_chan->comp_free); |
651 | } |
652 | return 0; |
653 | } |
654 | |
655 | static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev) |
656 | { |
657 | struct device *dev = &dpdmai_dev->dev; |
658 | struct dpaa2_qdma_engine *dpaa2_qdma; |
659 | struct dpaa2_qdma_priv *priv; |
660 | int err; |
661 | |
662 | priv = kzalloc(size: sizeof(*priv), GFP_KERNEL); |
663 | if (!priv) |
664 | return -ENOMEM; |
665 | dev_set_drvdata(dev, data: priv); |
666 | priv->dpdmai_dev = dpdmai_dev; |
667 | |
668 | priv->iommu_domain = iommu_get_domain_for_dev(dev); |
669 | if (priv->iommu_domain) |
670 | smmu_disable = false; |
671 | |
672 | /* obtain a MC portal */ |
673 | err = fsl_mc_portal_allocate(mc_dev: dpdmai_dev, mc_io_flags: 0, new_mc_io: &priv->mc_io); |
674 | if (err) { |
675 | if (err == -ENXIO) |
676 | err = -EPROBE_DEFER; |
677 | else |
678 | dev_err(dev, "MC portal allocation failed\n" ); |
679 | goto err_mcportal; |
680 | } |
681 | |
682 | /* DPDMAI initialization */ |
683 | err = dpaa2_qdma_setup(ls_dev: dpdmai_dev); |
684 | if (err) { |
685 | dev_err(dev, "dpaa2_dpdmai_setup() failed\n" ); |
686 | goto err_dpdmai_setup; |
687 | } |
688 | |
689 | /* DPIO */ |
690 | err = dpaa2_qdma_dpio_setup(priv); |
691 | if (err) { |
692 | dev_err(dev, "dpaa2_dpdmai_dpio_setup() failed\n" ); |
693 | goto err_dpio_setup; |
694 | } |
695 | |
696 | /* DPDMAI binding to DPIO */ |
697 | err = dpaa2_dpdmai_bind(priv); |
698 | if (err) { |
699 | dev_err(dev, "dpaa2_dpdmai_bind() failed\n" ); |
700 | goto err_bind; |
701 | } |
702 | |
703 | /* DPDMAI enable */ |
704 | err = dpdmai_enable(mc_io: priv->mc_io, cmd_flags: 0, token: dpdmai_dev->mc_handle); |
705 | if (err) { |
706 | dev_err(dev, "dpdmai_enable() failed\n" ); |
707 | goto err_enable; |
708 | } |
709 | |
710 | dpaa2_qdma = kzalloc(size: sizeof(*dpaa2_qdma), GFP_KERNEL); |
711 | if (!dpaa2_qdma) { |
712 | err = -ENOMEM; |
713 | goto err_eng; |
714 | } |
715 | |
716 | priv->dpaa2_qdma = dpaa2_qdma; |
717 | dpaa2_qdma->priv = priv; |
718 | |
719 | dpaa2_qdma->desc_allocated = 0; |
720 | dpaa2_qdma->n_chans = NUM_CH; |
721 | |
722 | dpaa2_dpdmai_init_channels(dpaa2_qdma); |
723 | |
724 | if (soc_device_match(matches: soc_fixup_tuning)) |
725 | dpaa2_qdma->qdma_wrtype_fixup = true; |
726 | else |
727 | dpaa2_qdma->qdma_wrtype_fixup = false; |
728 | |
729 | dma_cap_set(DMA_PRIVATE, dpaa2_qdma->dma_dev.cap_mask); |
730 | dma_cap_set(DMA_SLAVE, dpaa2_qdma->dma_dev.cap_mask); |
731 | dma_cap_set(DMA_MEMCPY, dpaa2_qdma->dma_dev.cap_mask); |
732 | |
733 | dpaa2_qdma->dma_dev.dev = dev; |
734 | dpaa2_qdma->dma_dev.device_alloc_chan_resources = |
735 | dpaa2_qdma_alloc_chan_resources; |
736 | dpaa2_qdma->dma_dev.device_free_chan_resources = |
737 | dpaa2_qdma_free_chan_resources; |
738 | dpaa2_qdma->dma_dev.device_tx_status = dma_cookie_status; |
739 | dpaa2_qdma->dma_dev.device_prep_dma_memcpy = dpaa2_qdma_prep_memcpy; |
740 | dpaa2_qdma->dma_dev.device_issue_pending = dpaa2_qdma_issue_pending; |
741 | |
742 | err = dma_async_device_register(device: &dpaa2_qdma->dma_dev); |
743 | if (err) { |
744 | dev_err(dev, "Can't register NXP QDMA engine.\n" ); |
745 | goto err_dpaa2_qdma; |
746 | } |
747 | |
748 | return 0; |
749 | |
750 | err_dpaa2_qdma: |
751 | kfree(objp: dpaa2_qdma); |
752 | err_eng: |
753 | dpdmai_disable(mc_io: priv->mc_io, cmd_flags: 0, token: dpdmai_dev->mc_handle); |
754 | err_enable: |
755 | dpaa2_dpdmai_dpio_unbind(priv); |
756 | err_bind: |
757 | dpaa2_dpmai_store_free(priv); |
758 | dpaa2_dpdmai_dpio_free(priv); |
759 | err_dpio_setup: |
760 | kfree(objp: priv->ppriv); |
761 | dpdmai_close(mc_io: priv->mc_io, cmd_flags: 0, token: dpdmai_dev->mc_handle); |
762 | err_dpdmai_setup: |
763 | fsl_mc_portal_free(mc_io: priv->mc_io); |
764 | err_mcportal: |
765 | kfree(objp: priv); |
766 | dev_set_drvdata(dev, NULL); |
767 | return err; |
768 | } |
769 | |
770 | static void dpaa2_qdma_remove(struct fsl_mc_device *ls_dev) |
771 | { |
772 | struct dpaa2_qdma_engine *dpaa2_qdma; |
773 | struct dpaa2_qdma_priv *priv; |
774 | struct device *dev; |
775 | |
776 | dev = &ls_dev->dev; |
777 | priv = dev_get_drvdata(dev); |
778 | dpaa2_qdma = priv->dpaa2_qdma; |
779 | |
780 | dpdmai_disable(mc_io: priv->mc_io, cmd_flags: 0, token: ls_dev->mc_handle); |
781 | dpaa2_dpdmai_dpio_unbind(priv); |
782 | dpaa2_dpmai_store_free(priv); |
783 | dpaa2_dpdmai_dpio_free(priv); |
784 | dpdmai_close(mc_io: priv->mc_io, cmd_flags: 0, token: ls_dev->mc_handle); |
785 | fsl_mc_portal_free(mc_io: priv->mc_io); |
786 | dev_set_drvdata(dev, NULL); |
787 | dpaa2_dpdmai_free_channels(dpaa2_qdma); |
788 | |
789 | dma_async_device_unregister(device: &dpaa2_qdma->dma_dev); |
790 | kfree(objp: priv); |
791 | kfree(objp: dpaa2_qdma); |
792 | } |
793 | |
794 | static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev) |
795 | { |
796 | struct dpaa2_qdma_priv *priv; |
797 | struct device *dev; |
798 | |
799 | dev = &ls_dev->dev; |
800 | priv = dev_get_drvdata(dev); |
801 | |
802 | dpdmai_disable(mc_io: priv->mc_io, cmd_flags: 0, token: ls_dev->mc_handle); |
803 | dpaa2_dpdmai_dpio_unbind(priv); |
804 | dpdmai_close(mc_io: priv->mc_io, cmd_flags: 0, token: ls_dev->mc_handle); |
805 | dpdmai_destroy(mc_io: priv->mc_io, cmd_flags: 0, token: ls_dev->mc_handle); |
806 | } |
807 | |
808 | static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { |
809 | { |
810 | .vendor = FSL_MC_VENDOR_FREESCALE, |
811 | .obj_type = "dpdmai" , |
812 | }, |
813 | { .vendor = 0x0 } |
814 | }; |
815 | |
816 | static struct fsl_mc_driver dpaa2_qdma_driver = { |
817 | .driver = { |
818 | .name = "dpaa2-qdma" , |
819 | }, |
820 | .probe = dpaa2_qdma_probe, |
821 | .remove = dpaa2_qdma_remove, |
822 | .shutdown = dpaa2_qdma_shutdown, |
823 | .match_id_table = dpaa2_qdma_id_table |
824 | }; |
825 | |
826 | static int __init dpaa2_qdma_driver_init(void) |
827 | { |
828 | return fsl_mc_driver_register(&(dpaa2_qdma_driver)); |
829 | } |
830 | late_initcall(dpaa2_qdma_driver_init); |
831 | |
832 | static void __exit fsl_qdma_exit(void) |
833 | { |
834 | fsl_mc_driver_unregister(driver: &(dpaa2_qdma_driver)); |
835 | } |
836 | module_exit(fsl_qdma_exit); |
837 | |
838 | MODULE_ALIAS("platform:fsl-dpaa2-qdma" ); |
839 | MODULE_LICENSE("GPL v2" ); |
840 | MODULE_DESCRIPTION("NXP Layerscape DPAA2 qDMA engine driver" ); |
841 | |