1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * K3 NAVSS DMA glue interface |
4 | * |
5 | * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com |
6 | * |
7 | */ |
8 | |
9 | #include <linux/module.h> |
10 | #include <linux/atomic.h> |
11 | #include <linux/delay.h> |
12 | #include <linux/dma-mapping.h> |
13 | #include <linux/io.h> |
14 | #include <linux/init.h> |
15 | #include <linux/of.h> |
16 | #include <linux/platform_device.h> |
17 | #include <linux/soc/ti/k3-ringacc.h> |
18 | #include <linux/dma/ti-cppi5.h> |
19 | #include <linux/dma/k3-udma-glue.h> |
20 | |
21 | #include "k3-udma.h" |
22 | #include "k3-psil-priv.h" |
23 | |
24 | struct k3_udma_glue_common { |
25 | struct device *dev; |
26 | struct device chan_dev; |
27 | struct udma_dev *udmax; |
28 | const struct udma_tisci_rm *tisci_rm; |
29 | struct k3_ringacc *ringacc; |
30 | u32 src_thread; |
31 | u32 dst_thread; |
32 | |
33 | u32 hdesc_size; |
34 | bool epib; |
35 | u32 psdata_size; |
36 | u32 swdata_size; |
37 | u32 atype_asel; |
38 | struct psil_endpoint_config *ep_config; |
39 | }; |
40 | |
41 | struct k3_udma_glue_tx_channel { |
42 | struct k3_udma_glue_common common; |
43 | |
44 | struct udma_tchan *udma_tchanx; |
45 | int udma_tchan_id; |
46 | |
47 | struct k3_ring *ringtx; |
48 | struct k3_ring *ringtxcq; |
49 | |
50 | bool psil_paired; |
51 | |
52 | int virq; |
53 | |
54 | atomic_t free_pkts; |
55 | bool tx_pause_on_err; |
56 | bool tx_filt_einfo; |
57 | bool tx_filt_pswords; |
58 | bool tx_supr_tdpkt; |
59 | |
60 | int udma_tflow_id; |
61 | }; |
62 | |
63 | struct k3_udma_glue_rx_flow { |
64 | struct udma_rflow *udma_rflow; |
65 | int udma_rflow_id; |
66 | struct k3_ring *ringrx; |
67 | struct k3_ring *ringrxfdq; |
68 | |
69 | int virq; |
70 | }; |
71 | |
72 | struct k3_udma_glue_rx_channel { |
73 | struct k3_udma_glue_common common; |
74 | |
75 | struct udma_rchan *udma_rchanx; |
76 | int udma_rchan_id; |
77 | bool remote; |
78 | |
79 | bool psil_paired; |
80 | |
81 | u32 swdata_size; |
82 | int flow_id_base; |
83 | |
84 | struct k3_udma_glue_rx_flow *flows; |
85 | u32 flow_num; |
86 | u32 flows_ready; |
87 | }; |
88 | |
89 | static void k3_udma_chan_dev_release(struct device *dev) |
90 | { |
91 | /* The struct containing the device is devm managed */ |
92 | } |
93 | |
94 | static struct class k3_udma_glue_devclass = { |
95 | .name = "k3_udma_glue_chan" , |
96 | .dev_release = k3_udma_chan_dev_release, |
97 | }; |
98 | |
99 | #define K3_UDMAX_TDOWN_TIMEOUT_US 1000 |
100 | |
101 | static int of_k3_udma_glue_parse(struct device_node *udmax_np, |
102 | struct k3_udma_glue_common *common) |
103 | { |
104 | common->udmax = of_xudma_dev_get(np: udmax_np, NULL); |
105 | if (IS_ERR(ptr: common->udmax)) |
106 | return PTR_ERR(ptr: common->udmax); |
107 | |
108 | common->ringacc = xudma_get_ringacc(ud: common->udmax); |
109 | common->tisci_rm = xudma_dev_get_tisci_rm(ud: common->udmax); |
110 | |
111 | return 0; |
112 | } |
113 | |
114 | static int of_k3_udma_glue_parse_chn_common(struct k3_udma_glue_common *common, u32 thread_id, |
115 | bool tx_chn) |
116 | { |
117 | if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) |
118 | return -EINVAL; |
119 | |
120 | if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) |
121 | return -EINVAL; |
122 | |
123 | /* get psil endpoint config */ |
124 | common->ep_config = psil_get_ep_config(thread_id); |
125 | if (IS_ERR(ptr: common->ep_config)) { |
126 | dev_err(common->dev, |
127 | "No configuration for psi-l thread 0x%04x\n" , |
128 | thread_id); |
129 | return PTR_ERR(ptr: common->ep_config); |
130 | } |
131 | |
132 | common->epib = common->ep_config->needs_epib; |
133 | common->psdata_size = common->ep_config->psd_size; |
134 | |
135 | if (tx_chn) |
136 | common->dst_thread = thread_id; |
137 | else |
138 | common->src_thread = thread_id; |
139 | |
140 | return 0; |
141 | } |
142 | |
143 | static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, |
144 | const char *name, struct k3_udma_glue_common *common, |
145 | bool tx_chn) |
146 | { |
147 | struct of_phandle_args dma_spec; |
148 | u32 thread_id; |
149 | int ret = 0; |
150 | int index; |
151 | |
152 | if (unlikely(!name)) |
153 | return -EINVAL; |
154 | |
155 | index = of_property_match_string(np: chn_np, propname: "dma-names" , string: name); |
156 | if (index < 0) |
157 | return index; |
158 | |
159 | if (of_parse_phandle_with_args(np: chn_np, list_name: "dmas" , cells_name: "#dma-cells" , index, |
160 | out_args: &dma_spec)) |
161 | return -ENOENT; |
162 | |
163 | ret = of_k3_udma_glue_parse(udmax_np: dma_spec.np, common); |
164 | if (ret) |
165 | goto out_put_spec; |
166 | |
167 | thread_id = dma_spec.args[0]; |
168 | if (dma_spec.args_count == 2) { |
169 | if (dma_spec.args[1] > 2 && !xudma_is_pktdma(ud: common->udmax)) { |
170 | dev_err(common->dev, "Invalid channel atype: %u\n" , |
171 | dma_spec.args[1]); |
172 | ret = -EINVAL; |
173 | goto out_put_spec; |
174 | } |
175 | if (dma_spec.args[1] > 15 && xudma_is_pktdma(ud: common->udmax)) { |
176 | dev_err(common->dev, "Invalid channel asel: %u\n" , |
177 | dma_spec.args[1]); |
178 | ret = -EINVAL; |
179 | goto out_put_spec; |
180 | } |
181 | |
182 | common->atype_asel = dma_spec.args[1]; |
183 | } |
184 | |
185 | ret = of_k3_udma_glue_parse_chn_common(common, thread_id, tx_chn); |
186 | |
187 | out_put_spec: |
188 | of_node_put(node: dma_spec.np); |
189 | return ret; |
190 | } |
191 | |
192 | static int |
193 | of_k3_udma_glue_parse_chn_by_id(struct device_node *udmax_np, struct k3_udma_glue_common *common, |
194 | bool tx_chn, u32 thread_id) |
195 | { |
196 | int ret = 0; |
197 | |
198 | if (unlikely(!udmax_np)) |
199 | return -EINVAL; |
200 | |
201 | ret = of_k3_udma_glue_parse(udmax_np, common); |
202 | if (ret) |
203 | goto out_put_spec; |
204 | |
205 | ret = of_k3_udma_glue_parse_chn_common(common, thread_id, tx_chn); |
206 | |
207 | out_put_spec: |
208 | of_node_put(node: udmax_np); |
209 | return ret; |
210 | } |
211 | |
212 | static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) |
213 | { |
214 | struct device *dev = tx_chn->common.dev; |
215 | |
216 | dev_dbg(dev, "dump_tx_chn:\n" |
217 | "udma_tchan_id: %d\n" |
218 | "src_thread: %08x\n" |
219 | "dst_thread: %08x\n" , |
220 | tx_chn->udma_tchan_id, |
221 | tx_chn->common.src_thread, |
222 | tx_chn->common.dst_thread); |
223 | } |
224 | |
225 | static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn, |
226 | char *mark) |
227 | { |
228 | struct device *dev = chn->common.dev; |
229 | |
230 | dev_dbg(dev, "=== dump ===> %s\n" , mark); |
231 | dev_dbg(dev, "0x%08X: %08X\n" , UDMA_CHAN_RT_CTL_REG, |
232 | xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG)); |
233 | dev_dbg(dev, "0x%08X: %08X\n" , UDMA_CHAN_RT_PEER_RT_EN_REG, |
234 | xudma_tchanrt_read(chn->udma_tchanx, |
235 | UDMA_CHAN_RT_PEER_RT_EN_REG)); |
236 | dev_dbg(dev, "0x%08X: %08X\n" , UDMA_CHAN_RT_PCNT_REG, |
237 | xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_PCNT_REG)); |
238 | dev_dbg(dev, "0x%08X: %08X\n" , UDMA_CHAN_RT_BCNT_REG, |
239 | xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_BCNT_REG)); |
240 | dev_dbg(dev, "0x%08X: %08X\n" , UDMA_CHAN_RT_SBCNT_REG, |
241 | xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_SBCNT_REG)); |
242 | } |
243 | |
244 | static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) |
245 | { |
246 | const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm; |
247 | struct ti_sci_msg_rm_udmap_tx_ch_cfg req; |
248 | |
249 | memset(&req, 0, sizeof(req)); |
250 | |
251 | req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | |
252 | TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID | |
253 | TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID | |
254 | TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | |
255 | TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | |
256 | TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | |
257 | TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | |
258 | TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; |
259 | req.nav_id = tisci_rm->tisci_dev_id; |
260 | req.index = tx_chn->udma_tchan_id; |
261 | if (tx_chn->tx_pause_on_err) |
262 | req.tx_pause_on_err = 1; |
263 | if (tx_chn->tx_filt_einfo) |
264 | req.tx_filt_einfo = 1; |
265 | if (tx_chn->tx_filt_pswords) |
266 | req.tx_filt_pswords = 1; |
267 | req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; |
268 | if (tx_chn->tx_supr_tdpkt) |
269 | req.tx_supr_tdpkt = 1; |
270 | req.tx_fetch_size = tx_chn->common.hdesc_size >> 2; |
271 | req.txcq_qnum = k3_ringacc_get_ring_id(ring: tx_chn->ringtxcq); |
272 | req.tx_atype = tx_chn->common.atype_asel; |
273 | |
274 | return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); |
275 | } |
276 | |
277 | static int |
278 | k3_udma_glue_request_tx_chn_common(struct device *dev, |
279 | struct k3_udma_glue_tx_channel *tx_chn, |
280 | struct k3_udma_glue_tx_channel_cfg *cfg) |
281 | { |
282 | int ret; |
283 | |
284 | tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(epib: tx_chn->common.epib, |
285 | psdata_size: tx_chn->common.psdata_size, |
286 | sw_data_size: tx_chn->common.swdata_size); |
287 | |
288 | if (xudma_is_pktdma(ud: tx_chn->common.udmax)) |
289 | tx_chn->udma_tchan_id = tx_chn->common.ep_config->mapped_channel_id; |
290 | else |
291 | tx_chn->udma_tchan_id = -1; |
292 | |
293 | /* request and cfg UDMAP TX channel */ |
294 | tx_chn->udma_tchanx = xudma_tchan_get(ud: tx_chn->common.udmax, |
295 | id: tx_chn->udma_tchan_id); |
296 | if (IS_ERR(ptr: tx_chn->udma_tchanx)) { |
297 | ret = PTR_ERR(ptr: tx_chn->udma_tchanx); |
298 | dev_err(dev, "UDMAX tchanx get err %d\n" , ret); |
299 | return ret; |
300 | } |
301 | tx_chn->udma_tchan_id = xudma_tchan_get_id(p: tx_chn->udma_tchanx); |
302 | |
303 | tx_chn->common.chan_dev.class = &k3_udma_glue_devclass; |
304 | tx_chn->common.chan_dev.parent = xudma_get_device(ud: tx_chn->common.udmax); |
305 | dev_set_name(dev: &tx_chn->common.chan_dev, name: "tchan%d-0x%04x" , |
306 | tx_chn->udma_tchan_id, tx_chn->common.dst_thread); |
307 | ret = device_register(dev: &tx_chn->common.chan_dev); |
308 | if (ret) { |
309 | dev_err(dev, "Channel Device registration failed %d\n" , ret); |
310 | put_device(dev: &tx_chn->common.chan_dev); |
311 | tx_chn->common.chan_dev.parent = NULL; |
312 | return ret; |
313 | } |
314 | |
315 | if (xudma_is_pktdma(ud: tx_chn->common.udmax)) { |
316 | /* prepare the channel device as coherent */ |
317 | tx_chn->common.chan_dev.dma_coherent = true; |
318 | dma_coerce_mask_and_coherent(dev: &tx_chn->common.chan_dev, |
319 | DMA_BIT_MASK(48)); |
320 | } |
321 | |
322 | atomic_set(v: &tx_chn->free_pkts, i: cfg->txcq_cfg.size); |
323 | |
324 | if (xudma_is_pktdma(ud: tx_chn->common.udmax)) |
325 | tx_chn->udma_tflow_id = tx_chn->common.ep_config->default_flow_id; |
326 | else |
327 | tx_chn->udma_tflow_id = tx_chn->udma_tchan_id; |
328 | |
329 | /* request and cfg rings */ |
330 | ret = k3_ringacc_request_rings_pair(ringacc: tx_chn->common.ringacc, |
331 | fwd_id: tx_chn->udma_tflow_id, compl_id: -1, |
332 | fwd_ring: &tx_chn->ringtx, |
333 | compl_ring: &tx_chn->ringtxcq); |
334 | if (ret) { |
335 | dev_err(dev, "Failed to get TX/TXCQ rings %d\n" , ret); |
336 | return ret; |
337 | } |
338 | |
339 | /* Set the dma_dev for the rings to be configured */ |
340 | cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn); |
341 | cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev; |
342 | |
343 | /* Set the ASEL value for DMA rings of PKTDMA */ |
344 | if (xudma_is_pktdma(ud: tx_chn->common.udmax)) { |
345 | cfg->tx_cfg.asel = tx_chn->common.atype_asel; |
346 | cfg->txcq_cfg.asel = tx_chn->common.atype_asel; |
347 | } |
348 | |
349 | ret = k3_ringacc_ring_cfg(ring: tx_chn->ringtx, cfg: &cfg->tx_cfg); |
350 | if (ret) { |
351 | dev_err(dev, "Failed to cfg ringtx %d\n" , ret); |
352 | return ret; |
353 | } |
354 | |
355 | ret = k3_ringacc_ring_cfg(ring: tx_chn->ringtxcq, cfg: &cfg->txcq_cfg); |
356 | if (ret) { |
357 | dev_err(dev, "Failed to cfg ringtx %d\n" , ret); |
358 | return ret; |
359 | } |
360 | |
361 | /* request and cfg psi-l */ |
362 | tx_chn->common.src_thread = |
363 | xudma_dev_get_psil_base(ud: tx_chn->common.udmax) + |
364 | tx_chn->udma_tchan_id; |
365 | |
366 | ret = k3_udma_glue_cfg_tx_chn(tx_chn); |
367 | if (ret) { |
368 | dev_err(dev, "Failed to cfg tchan %d\n" , ret); |
369 | return ret; |
370 | } |
371 | |
372 | k3_udma_glue_dump_tx_chn(tx_chn); |
373 | |
374 | return 0; |
375 | } |
376 | |
377 | struct k3_udma_glue_tx_channel * |
378 | k3_udma_glue_request_tx_chn(struct device *dev, const char *name, |
379 | struct k3_udma_glue_tx_channel_cfg *cfg) |
380 | { |
381 | struct k3_udma_glue_tx_channel *tx_chn; |
382 | int ret; |
383 | |
384 | tx_chn = devm_kzalloc(dev, size: sizeof(*tx_chn), GFP_KERNEL); |
385 | if (!tx_chn) |
386 | return ERR_PTR(error: -ENOMEM); |
387 | |
388 | tx_chn->common.dev = dev; |
389 | tx_chn->common.swdata_size = cfg->swdata_size; |
390 | tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; |
391 | tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; |
392 | tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; |
393 | tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; |
394 | |
395 | /* parse of udmap channel */ |
396 | ret = of_k3_udma_glue_parse_chn(chn_np: dev->of_node, name, |
397 | common: &tx_chn->common, tx_chn: true); |
398 | if (ret) |
399 | goto err; |
400 | |
401 | ret = k3_udma_glue_request_tx_chn_common(dev, tx_chn, cfg); |
402 | if (ret) |
403 | goto err; |
404 | |
405 | return tx_chn; |
406 | |
407 | err: |
408 | k3_udma_glue_release_tx_chn(tx_chn); |
409 | return ERR_PTR(error: ret); |
410 | } |
411 | EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn); |
412 | |
413 | struct k3_udma_glue_tx_channel * |
414 | k3_udma_glue_request_tx_chn_for_thread_id(struct device *dev, |
415 | struct k3_udma_glue_tx_channel_cfg *cfg, |
416 | struct device_node *udmax_np, u32 thread_id) |
417 | { |
418 | struct k3_udma_glue_tx_channel *tx_chn; |
419 | int ret; |
420 | |
421 | tx_chn = devm_kzalloc(dev, size: sizeof(*tx_chn), GFP_KERNEL); |
422 | if (!tx_chn) |
423 | return ERR_PTR(error: -ENOMEM); |
424 | |
425 | tx_chn->common.dev = dev; |
426 | tx_chn->common.swdata_size = cfg->swdata_size; |
427 | tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; |
428 | tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; |
429 | tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; |
430 | tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; |
431 | |
432 | ret = of_k3_udma_glue_parse_chn_by_id(udmax_np, common: &tx_chn->common, tx_chn: true, thread_id); |
433 | if (ret) |
434 | goto err; |
435 | |
436 | ret = k3_udma_glue_request_tx_chn_common(dev, tx_chn, cfg); |
437 | if (ret) |
438 | goto err; |
439 | |
440 | return tx_chn; |
441 | |
442 | err: |
443 | k3_udma_glue_release_tx_chn(tx_chn); |
444 | return ERR_PTR(error: ret); |
445 | } |
446 | EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn_for_thread_id); |
447 | |
448 | void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) |
449 | { |
450 | if (tx_chn->psil_paired) { |
451 | xudma_navss_psil_unpair(ud: tx_chn->common.udmax, |
452 | src_thread: tx_chn->common.src_thread, |
453 | dst_thread: tx_chn->common.dst_thread); |
454 | tx_chn->psil_paired = false; |
455 | } |
456 | |
457 | if (!IS_ERR_OR_NULL(ptr: tx_chn->udma_tchanx)) |
458 | xudma_tchan_put(ud: tx_chn->common.udmax, |
459 | p: tx_chn->udma_tchanx); |
460 | |
461 | if (tx_chn->ringtxcq) |
462 | k3_ringacc_ring_free(ring: tx_chn->ringtxcq); |
463 | |
464 | if (tx_chn->ringtx) |
465 | k3_ringacc_ring_free(ring: tx_chn->ringtx); |
466 | |
467 | if (tx_chn->common.chan_dev.parent) { |
468 | device_unregister(dev: &tx_chn->common.chan_dev); |
469 | tx_chn->common.chan_dev.parent = NULL; |
470 | } |
471 | } |
472 | EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn); |
473 | |
474 | int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, |
475 | struct cppi5_host_desc_t *desc_tx, |
476 | dma_addr_t desc_dma) |
477 | { |
478 | u32 ringtxcq_id; |
479 | |
480 | if (!atomic_add_unless(v: &tx_chn->free_pkts, a: -1, u: 0)) |
481 | return -ENOMEM; |
482 | |
483 | ringtxcq_id = k3_ringacc_get_ring_id(ring: tx_chn->ringtxcq); |
484 | cppi5_desc_set_retpolicy(desc_hdr: &desc_tx->hdr, flags: 0, return_ring_id: ringtxcq_id); |
485 | |
486 | return k3_ringacc_ring_push(ring: tx_chn->ringtx, elem: &desc_dma); |
487 | } |
488 | EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn); |
489 | |
490 | int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, |
491 | dma_addr_t *desc_dma) |
492 | { |
493 | int ret; |
494 | |
495 | ret = k3_ringacc_ring_pop(ring: tx_chn->ringtxcq, elem: desc_dma); |
496 | if (!ret) |
497 | atomic_inc(v: &tx_chn->free_pkts); |
498 | |
499 | return ret; |
500 | } |
501 | EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn); |
502 | |
503 | int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) |
504 | { |
505 | int ret; |
506 | |
507 | ret = xudma_navss_psil_pair(ud: tx_chn->common.udmax, |
508 | src_thread: tx_chn->common.src_thread, |
509 | dst_thread: tx_chn->common.dst_thread); |
510 | if (ret) { |
511 | dev_err(tx_chn->common.dev, "PSI-L request err %d\n" , ret); |
512 | return ret; |
513 | } |
514 | |
515 | tx_chn->psil_paired = true; |
516 | |
517 | xudma_tchanrt_write(tchan: tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG, |
518 | UDMA_PEER_RT_EN_ENABLE); |
519 | |
520 | xudma_tchanrt_write(tchan: tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, |
521 | UDMA_CHAN_RT_CTL_EN); |
522 | |
523 | k3_udma_glue_dump_tx_rt_chn(chn: tx_chn, mark: "txchn en" ); |
524 | return 0; |
525 | } |
526 | EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn); |
527 | |
528 | void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) |
529 | { |
530 | k3_udma_glue_dump_tx_rt_chn(chn: tx_chn, mark: "txchn dis1" ); |
531 | |
532 | xudma_tchanrt_write(tchan: tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, val: 0); |
533 | |
534 | xudma_tchanrt_write(tchan: tx_chn->udma_tchanx, |
535 | UDMA_CHAN_RT_PEER_RT_EN_REG, val: 0); |
536 | k3_udma_glue_dump_tx_rt_chn(chn: tx_chn, mark: "txchn dis2" ); |
537 | |
538 | if (tx_chn->psil_paired) { |
539 | xudma_navss_psil_unpair(ud: tx_chn->common.udmax, |
540 | src_thread: tx_chn->common.src_thread, |
541 | dst_thread: tx_chn->common.dst_thread); |
542 | tx_chn->psil_paired = false; |
543 | } |
544 | } |
545 | EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn); |
546 | |
547 | void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, |
548 | bool sync) |
549 | { |
550 | int i = 0; |
551 | u32 val; |
552 | |
553 | k3_udma_glue_dump_tx_rt_chn(chn: tx_chn, mark: "txchn tdown1" ); |
554 | |
555 | xudma_tchanrt_write(tchan: tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, |
556 | UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN); |
557 | |
558 | val = xudma_tchanrt_read(tchan: tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG); |
559 | |
560 | while (sync && (val & UDMA_CHAN_RT_CTL_EN)) { |
561 | val = xudma_tchanrt_read(tchan: tx_chn->udma_tchanx, |
562 | UDMA_CHAN_RT_CTL_REG); |
563 | udelay(1); |
564 | if (i > K3_UDMAX_TDOWN_TIMEOUT_US) { |
565 | dev_err(tx_chn->common.dev, "TX tdown timeout\n" ); |
566 | break; |
567 | } |
568 | i++; |
569 | } |
570 | |
571 | val = xudma_tchanrt_read(tchan: tx_chn->udma_tchanx, |
572 | UDMA_CHAN_RT_PEER_RT_EN_REG); |
573 | if (sync && (val & UDMA_PEER_RT_EN_ENABLE)) |
574 | dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n" ); |
575 | k3_udma_glue_dump_tx_rt_chn(chn: tx_chn, mark: "txchn tdown2" ); |
576 | } |
577 | EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn); |
578 | |
579 | void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn, |
580 | void *data, |
581 | void (*cleanup)(void *data, dma_addr_t desc_dma)) |
582 | { |
583 | struct device *dev = tx_chn->common.dev; |
584 | dma_addr_t desc_dma; |
585 | int occ_tx, i, ret; |
586 | |
587 | /* |
588 | * TXQ reset need to be special way as it is input for udma and its |
589 | * state cached by udma, so: |
590 | * 1) save TXQ occ |
591 | * 2) clean up TXQ and call callback .cleanup() for each desc |
592 | * 3) reset TXQ in a special way |
593 | */ |
594 | occ_tx = k3_ringacc_ring_get_occ(ring: tx_chn->ringtx); |
595 | dev_dbg(dev, "TX reset occ_tx %u\n" , occ_tx); |
596 | |
597 | for (i = 0; i < occ_tx; i++) { |
598 | ret = k3_ringacc_ring_pop(ring: tx_chn->ringtx, elem: &desc_dma); |
599 | if (ret) { |
600 | if (ret != -ENODATA) |
601 | dev_err(dev, "TX reset pop %d\n" , ret); |
602 | break; |
603 | } |
604 | cleanup(data, desc_dma); |
605 | } |
606 | |
607 | /* reset TXCQ as it is not input for udma - expected to be empty */ |
608 | k3_ringacc_ring_reset(ring: tx_chn->ringtxcq); |
609 | k3_ringacc_ring_reset_dma(ring: tx_chn->ringtx, occ: occ_tx); |
610 | } |
611 | EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn); |
612 | |
613 | u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn) |
614 | { |
615 | return tx_chn->common.hdesc_size; |
616 | } |
617 | EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size); |
618 | |
619 | u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn) |
620 | { |
621 | return k3_ringacc_get_ring_id(ring: tx_chn->ringtxcq); |
622 | } |
623 | EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id); |
624 | |
625 | int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn) |
626 | { |
627 | if (xudma_is_pktdma(ud: tx_chn->common.udmax)) { |
628 | tx_chn->virq = xudma_pktdma_tflow_get_irq(ud: tx_chn->common.udmax, |
629 | udma_tflow_id: tx_chn->udma_tflow_id); |
630 | } else { |
631 | tx_chn->virq = k3_ringacc_get_ring_irq_num(ring: tx_chn->ringtxcq); |
632 | } |
633 | |
634 | if (!tx_chn->virq) |
635 | return -ENXIO; |
636 | |
637 | return tx_chn->virq; |
638 | } |
639 | EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq); |
640 | |
641 | struct device * |
642 | k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn) |
643 | { |
644 | if (xudma_is_pktdma(ud: tx_chn->common.udmax) && |
645 | (tx_chn->common.atype_asel == 14 || tx_chn->common.atype_asel == 15)) |
646 | return &tx_chn->common.chan_dev; |
647 | |
648 | return xudma_get_device(ud: tx_chn->common.udmax); |
649 | } |
650 | EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device); |
651 | |
652 | void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn, |
653 | dma_addr_t *addr) |
654 | { |
655 | if (!xudma_is_pktdma(ud: tx_chn->common.udmax) || |
656 | !tx_chn->common.atype_asel) |
657 | return; |
658 | |
659 | *addr |= (u64)tx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT; |
660 | } |
661 | EXPORT_SYMBOL_GPL(k3_udma_glue_tx_dma_to_cppi5_addr); |
662 | |
663 | void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn, |
664 | dma_addr_t *addr) |
665 | { |
666 | if (!xudma_is_pktdma(ud: tx_chn->common.udmax) || |
667 | !tx_chn->common.atype_asel) |
668 | return; |
669 | |
670 | *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0); |
671 | } |
672 | EXPORT_SYMBOL_GPL(k3_udma_glue_tx_cppi5_to_dma_addr); |
673 | |
674 | static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) |
675 | { |
676 | const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; |
677 | struct ti_sci_msg_rm_udmap_rx_ch_cfg req; |
678 | int ret; |
679 | |
680 | memset(&req, 0, sizeof(req)); |
681 | |
682 | req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | |
683 | TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | |
684 | TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | |
685 | TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; |
686 | |
687 | req.nav_id = tisci_rm->tisci_dev_id; |
688 | req.index = rx_chn->udma_rchan_id; |
689 | req.rx_fetch_size = rx_chn->common.hdesc_size >> 2; |
690 | /* |
691 | * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw |
692 | * and udmax impl, so just configure it to invalid value. |
693 | * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx); |
694 | */ |
695 | req.rxcq_qnum = 0xFFFF; |
696 | if (!xudma_is_pktdma(ud: rx_chn->common.udmax) && rx_chn->flow_num && |
697 | rx_chn->flow_id_base != rx_chn->udma_rchan_id) { |
698 | /* Default flow + extra ones */ |
699 | req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | |
700 | TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID; |
701 | req.flowid_start = rx_chn->flow_id_base; |
702 | req.flowid_cnt = rx_chn->flow_num; |
703 | } |
704 | req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; |
705 | req.rx_atype = rx_chn->common.atype_asel; |
706 | |
707 | ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req); |
708 | if (ret) |
709 | dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n" , |
710 | rx_chn->udma_rchan_id, ret); |
711 | |
712 | return ret; |
713 | } |
714 | |
715 | static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, |
716 | u32 flow_num) |
717 | { |
718 | struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; |
719 | |
720 | if (IS_ERR_OR_NULL(ptr: flow->udma_rflow)) |
721 | return; |
722 | |
723 | if (flow->ringrxfdq) |
724 | k3_ringacc_ring_free(ring: flow->ringrxfdq); |
725 | |
726 | if (flow->ringrx) |
727 | k3_ringacc_ring_free(ring: flow->ringrx); |
728 | |
729 | xudma_rflow_put(ud: rx_chn->common.udmax, p: flow->udma_rflow); |
730 | flow->udma_rflow = NULL; |
731 | rx_chn->flows_ready--; |
732 | } |
733 | |
734 | static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn, |
735 | u32 flow_idx, |
736 | struct k3_udma_glue_rx_flow_cfg *flow_cfg) |
737 | { |
738 | struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; |
739 | const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; |
740 | struct device *dev = rx_chn->common.dev; |
741 | struct ti_sci_msg_rm_udmap_flow_cfg req; |
742 | int rx_ring_id; |
743 | int rx_ringfdq_id; |
744 | int ret = 0; |
745 | |
746 | flow->udma_rflow = xudma_rflow_get(ud: rx_chn->common.udmax, |
747 | id: flow->udma_rflow_id); |
748 | if (IS_ERR(ptr: flow->udma_rflow)) { |
749 | ret = PTR_ERR(ptr: flow->udma_rflow); |
750 | dev_err(dev, "UDMAX rflow get err %d\n" , ret); |
751 | return ret; |
752 | } |
753 | |
754 | if (flow->udma_rflow_id != xudma_rflow_get_id(p: flow->udma_rflow)) { |
755 | ret = -ENODEV; |
756 | goto err_rflow_put; |
757 | } |
758 | |
759 | if (xudma_is_pktdma(ud: rx_chn->common.udmax)) { |
760 | rx_ringfdq_id = flow->udma_rflow_id + |
761 | xudma_get_rflow_ring_offset(ud: rx_chn->common.udmax); |
762 | rx_ring_id = 0; |
763 | } else { |
764 | rx_ring_id = flow_cfg->ring_rxq_id; |
765 | rx_ringfdq_id = flow_cfg->ring_rxfdq0_id; |
766 | } |
767 | |
768 | /* request and cfg rings */ |
769 | ret = k3_ringacc_request_rings_pair(ringacc: rx_chn->common.ringacc, |
770 | fwd_id: rx_ringfdq_id, compl_id: rx_ring_id, |
771 | fwd_ring: &flow->ringrxfdq, |
772 | compl_ring: &flow->ringrx); |
773 | if (ret) { |
774 | dev_err(dev, "Failed to get RX/RXFDQ rings %d\n" , ret); |
775 | goto err_rflow_put; |
776 | } |
777 | |
778 | /* Set the dma_dev for the rings to be configured */ |
779 | flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn); |
780 | flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev; |
781 | |
782 | /* Set the ASEL value for DMA rings of PKTDMA */ |
783 | if (xudma_is_pktdma(ud: rx_chn->common.udmax)) { |
784 | flow_cfg->rx_cfg.asel = rx_chn->common.atype_asel; |
785 | flow_cfg->rxfdq_cfg.asel = rx_chn->common.atype_asel; |
786 | } |
787 | |
788 | ret = k3_ringacc_ring_cfg(ring: flow->ringrx, cfg: &flow_cfg->rx_cfg); |
789 | if (ret) { |
790 | dev_err(dev, "Failed to cfg ringrx %d\n" , ret); |
791 | goto err_ringrxfdq_free; |
792 | } |
793 | |
794 | ret = k3_ringacc_ring_cfg(ring: flow->ringrxfdq, cfg: &flow_cfg->rxfdq_cfg); |
795 | if (ret) { |
796 | dev_err(dev, "Failed to cfg ringrxfdq %d\n" , ret); |
797 | goto err_ringrxfdq_free; |
798 | } |
799 | |
800 | if (rx_chn->remote) { |
801 | rx_ring_id = TI_SCI_RESOURCE_NULL; |
802 | rx_ringfdq_id = TI_SCI_RESOURCE_NULL; |
803 | } else { |
804 | rx_ring_id = k3_ringacc_get_ring_id(ring: flow->ringrx); |
805 | rx_ringfdq_id = k3_ringacc_get_ring_id(ring: flow->ringrxfdq); |
806 | } |
807 | |
808 | memset(&req, 0, sizeof(req)); |
809 | |
810 | req.valid_params = |
811 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID | |
812 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID | |
813 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID | |
814 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID | |
815 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | |
816 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID | |
817 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID | |
818 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID | |
819 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID | |
820 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | |
821 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | |
822 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | |
823 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; |
824 | req.nav_id = tisci_rm->tisci_dev_id; |
825 | req.flow_index = flow->udma_rflow_id; |
826 | if (rx_chn->common.epib) |
827 | req.rx_einfo_present = 1; |
828 | if (rx_chn->common.psdata_size) |
829 | req.rx_psinfo_present = 1; |
830 | if (flow_cfg->rx_error_handling) |
831 | req.rx_error_handling = 1; |
832 | req.rx_desc_type = 0; |
833 | req.rx_dest_qnum = rx_ring_id; |
834 | req.rx_src_tag_hi_sel = 0; |
835 | req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel; |
836 | req.rx_dest_tag_hi_sel = 0; |
837 | req.rx_dest_tag_lo_sel = 0; |
838 | req.rx_fdq0_sz0_qnum = rx_ringfdq_id; |
839 | req.rx_fdq1_qnum = rx_ringfdq_id; |
840 | req.rx_fdq2_qnum = rx_ringfdq_id; |
841 | req.rx_fdq3_qnum = rx_ringfdq_id; |
842 | |
843 | ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); |
844 | if (ret) { |
845 | dev_err(dev, "flow%d config failed: %d\n" , flow->udma_rflow_id, |
846 | ret); |
847 | goto err_ringrxfdq_free; |
848 | } |
849 | |
850 | rx_chn->flows_ready++; |
851 | dev_dbg(dev, "flow%d config done. ready:%d\n" , |
852 | flow->udma_rflow_id, rx_chn->flows_ready); |
853 | |
854 | return 0; |
855 | |
856 | err_ringrxfdq_free: |
857 | k3_ringacc_ring_free(ring: flow->ringrxfdq); |
858 | k3_ringacc_ring_free(ring: flow->ringrx); |
859 | |
860 | err_rflow_put: |
861 | xudma_rflow_put(ud: rx_chn->common.udmax, p: flow->udma_rflow); |
862 | flow->udma_rflow = NULL; |
863 | |
864 | return ret; |
865 | } |
866 | |
867 | static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn) |
868 | { |
869 | struct device *dev = chn->common.dev; |
870 | |
871 | dev_dbg(dev, "dump_rx_chn:\n" |
872 | "udma_rchan_id: %d\n" |
873 | "src_thread: %08x\n" |
874 | "dst_thread: %08x\n" |
875 | "epib: %d\n" |
876 | "hdesc_size: %u\n" |
877 | "psdata_size: %u\n" |
878 | "swdata_size: %u\n" |
879 | "flow_id_base: %d\n" |
880 | "flow_num: %d\n" , |
881 | chn->udma_rchan_id, |
882 | chn->common.src_thread, |
883 | chn->common.dst_thread, |
884 | chn->common.epib, |
885 | chn->common.hdesc_size, |
886 | chn->common.psdata_size, |
887 | chn->common.swdata_size, |
888 | chn->flow_id_base, |
889 | chn->flow_num); |
890 | } |
891 | |
892 | static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn, |
893 | char *mark) |
894 | { |
895 | struct device *dev = chn->common.dev; |
896 | |
897 | dev_dbg(dev, "=== dump ===> %s\n" , mark); |
898 | |
899 | dev_dbg(dev, "0x%08X: %08X\n" , UDMA_CHAN_RT_CTL_REG, |
900 | xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG)); |
901 | dev_dbg(dev, "0x%08X: %08X\n" , UDMA_CHAN_RT_PEER_RT_EN_REG, |
902 | xudma_rchanrt_read(chn->udma_rchanx, |
903 | UDMA_CHAN_RT_PEER_RT_EN_REG)); |
904 | dev_dbg(dev, "0x%08X: %08X\n" , UDMA_CHAN_RT_PCNT_REG, |
905 | xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_PCNT_REG)); |
906 | dev_dbg(dev, "0x%08X: %08X\n" , UDMA_CHAN_RT_BCNT_REG, |
907 | xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_BCNT_REG)); |
908 | dev_dbg(dev, "0x%08X: %08X\n" , UDMA_CHAN_RT_SBCNT_REG, |
909 | xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_SBCNT_REG)); |
910 | } |
911 | |
912 | static int |
913 | k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn, |
914 | struct k3_udma_glue_rx_channel_cfg *cfg) |
915 | { |
916 | int ret; |
917 | |
918 | /* default rflow */ |
919 | if (cfg->flow_id_use_rxchan_id) |
920 | return 0; |
921 | |
922 | /* not a GP rflows */ |
923 | if (rx_chn->flow_id_base != -1 && |
924 | !xudma_rflow_is_gp(ud: rx_chn->common.udmax, id: rx_chn->flow_id_base)) |
925 | return 0; |
926 | |
927 | /* Allocate range of GP rflows */ |
928 | ret = xudma_alloc_gp_rflow_range(ud: rx_chn->common.udmax, |
929 | from: rx_chn->flow_id_base, |
930 | cnt: rx_chn->flow_num); |
931 | if (ret < 0) { |
932 | dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n" , |
933 | rx_chn->flow_id_base, rx_chn->flow_num, ret); |
934 | return ret; |
935 | } |
936 | rx_chn->flow_id_base = ret; |
937 | |
938 | return 0; |
939 | } |
940 | |
941 | static struct k3_udma_glue_rx_channel * |
942 | k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, |
943 | struct k3_udma_glue_rx_channel_cfg *cfg) |
944 | { |
945 | struct k3_udma_glue_rx_channel *rx_chn; |
946 | struct psil_endpoint_config *ep_cfg; |
947 | int ret, i; |
948 | |
949 | if (cfg->flow_id_num <= 0) |
950 | return ERR_PTR(error: -EINVAL); |
951 | |
952 | if (cfg->flow_id_num != 1 && |
953 | (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id)) |
954 | return ERR_PTR(error: -EINVAL); |
955 | |
956 | rx_chn = devm_kzalloc(dev, size: sizeof(*rx_chn), GFP_KERNEL); |
957 | if (!rx_chn) |
958 | return ERR_PTR(error: -ENOMEM); |
959 | |
960 | rx_chn->common.dev = dev; |
961 | rx_chn->common.swdata_size = cfg->swdata_size; |
962 | rx_chn->remote = false; |
963 | |
964 | /* parse of udmap channel */ |
965 | ret = of_k3_udma_glue_parse_chn(chn_np: dev->of_node, name, |
966 | common: &rx_chn->common, tx_chn: false); |
967 | if (ret) |
968 | goto err; |
969 | |
970 | rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(epib: rx_chn->common.epib, |
971 | psdata_size: rx_chn->common.psdata_size, |
972 | sw_data_size: rx_chn->common.swdata_size); |
973 | |
974 | ep_cfg = rx_chn->common.ep_config; |
975 | |
976 | if (xudma_is_pktdma(ud: rx_chn->common.udmax)) |
977 | rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id; |
978 | else |
979 | rx_chn->udma_rchan_id = -1; |
980 | |
981 | /* request and cfg UDMAP RX channel */ |
982 | rx_chn->udma_rchanx = xudma_rchan_get(ud: rx_chn->common.udmax, |
983 | id: rx_chn->udma_rchan_id); |
984 | if (IS_ERR(ptr: rx_chn->udma_rchanx)) { |
985 | ret = PTR_ERR(ptr: rx_chn->udma_rchanx); |
986 | dev_err(dev, "UDMAX rchanx get err %d\n" , ret); |
987 | goto err; |
988 | } |
989 | rx_chn->udma_rchan_id = xudma_rchan_get_id(p: rx_chn->udma_rchanx); |
990 | |
991 | rx_chn->common.chan_dev.class = &k3_udma_glue_devclass; |
992 | rx_chn->common.chan_dev.parent = xudma_get_device(ud: rx_chn->common.udmax); |
993 | dev_set_name(dev: &rx_chn->common.chan_dev, name: "rchan%d-0x%04x" , |
994 | rx_chn->udma_rchan_id, rx_chn->common.src_thread); |
995 | ret = device_register(dev: &rx_chn->common.chan_dev); |
996 | if (ret) { |
997 | dev_err(dev, "Channel Device registration failed %d\n" , ret); |
998 | put_device(dev: &rx_chn->common.chan_dev); |
999 | rx_chn->common.chan_dev.parent = NULL; |
1000 | goto err; |
1001 | } |
1002 | |
1003 | if (xudma_is_pktdma(ud: rx_chn->common.udmax)) { |
1004 | /* prepare the channel device as coherent */ |
1005 | rx_chn->common.chan_dev.dma_coherent = true; |
1006 | dma_coerce_mask_and_coherent(dev: &rx_chn->common.chan_dev, |
1007 | DMA_BIT_MASK(48)); |
1008 | } |
1009 | |
1010 | if (xudma_is_pktdma(ud: rx_chn->common.udmax)) { |
1011 | int flow_start = cfg->flow_id_base; |
1012 | int flow_end; |
1013 | |
1014 | if (flow_start == -1) |
1015 | flow_start = ep_cfg->flow_start; |
1016 | |
1017 | flow_end = flow_start + cfg->flow_id_num - 1; |
1018 | if (flow_start < ep_cfg->flow_start || |
1019 | flow_end > (ep_cfg->flow_start + ep_cfg->flow_num - 1)) { |
1020 | dev_err(dev, "Invalid flow range requested\n" ); |
1021 | ret = -EINVAL; |
1022 | goto err; |
1023 | } |
1024 | rx_chn->flow_id_base = flow_start; |
1025 | } else { |
1026 | rx_chn->flow_id_base = cfg->flow_id_base; |
1027 | |
1028 | /* Use RX channel id as flow id: target dev can't generate flow_id */ |
1029 | if (cfg->flow_id_use_rxchan_id) |
1030 | rx_chn->flow_id_base = rx_chn->udma_rchan_id; |
1031 | } |
1032 | |
1033 | rx_chn->flow_num = cfg->flow_id_num; |
1034 | |
1035 | rx_chn->flows = devm_kcalloc(dev, n: rx_chn->flow_num, |
1036 | size: sizeof(*rx_chn->flows), GFP_KERNEL); |
1037 | if (!rx_chn->flows) { |
1038 | ret = -ENOMEM; |
1039 | goto err; |
1040 | } |
1041 | |
1042 | ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); |
1043 | if (ret) |
1044 | goto err; |
1045 | |
1046 | for (i = 0; i < rx_chn->flow_num; i++) |
1047 | rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; |
1048 | |
1049 | /* request and cfg psi-l */ |
1050 | rx_chn->common.dst_thread = |
1051 | xudma_dev_get_psil_base(ud: rx_chn->common.udmax) + |
1052 | rx_chn->udma_rchan_id; |
1053 | |
1054 | ret = k3_udma_glue_cfg_rx_chn(rx_chn); |
1055 | if (ret) { |
1056 | dev_err(dev, "Failed to cfg rchan %d\n" , ret); |
1057 | goto err; |
1058 | } |
1059 | |
1060 | /* init default RX flow only if flow_num = 1 */ |
1061 | if (cfg->def_flow_cfg) { |
1062 | ret = k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx: 0, flow_cfg: cfg->def_flow_cfg); |
1063 | if (ret) |
1064 | goto err; |
1065 | } |
1066 | |
1067 | k3_udma_glue_dump_rx_chn(chn: rx_chn); |
1068 | |
1069 | return rx_chn; |
1070 | |
1071 | err: |
1072 | k3_udma_glue_release_rx_chn(rx_chn); |
1073 | return ERR_PTR(error: ret); |
1074 | } |
1075 | |
1076 | static int |
1077 | k3_udma_glue_request_remote_rx_chn_common(struct k3_udma_glue_rx_channel *rx_chn, |
1078 | struct k3_udma_glue_rx_channel_cfg *cfg, |
1079 | struct device *dev) |
1080 | { |
1081 | int ret, i; |
1082 | |
1083 | rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(epib: rx_chn->common.epib, |
1084 | psdata_size: rx_chn->common.psdata_size, |
1085 | sw_data_size: rx_chn->common.swdata_size); |
1086 | |
1087 | rx_chn->flows = devm_kcalloc(dev, n: rx_chn->flow_num, |
1088 | size: sizeof(*rx_chn->flows), GFP_KERNEL); |
1089 | if (!rx_chn->flows) |
1090 | return -ENOMEM; |
1091 | |
1092 | rx_chn->common.chan_dev.class = &k3_udma_glue_devclass; |
1093 | rx_chn->common.chan_dev.parent = xudma_get_device(ud: rx_chn->common.udmax); |
1094 | dev_set_name(dev: &rx_chn->common.chan_dev, name: "rchan_remote-0x%04x-0x%02x" , |
1095 | rx_chn->common.src_thread, rx_chn->flow_id_base); |
1096 | ret = device_register(dev: &rx_chn->common.chan_dev); |
1097 | if (ret) { |
1098 | dev_err(dev, "Channel Device registration failed %d\n" , ret); |
1099 | put_device(dev: &rx_chn->common.chan_dev); |
1100 | rx_chn->common.chan_dev.parent = NULL; |
1101 | return ret; |
1102 | } |
1103 | |
1104 | if (xudma_is_pktdma(ud: rx_chn->common.udmax)) { |
1105 | /* prepare the channel device as coherent */ |
1106 | rx_chn->common.chan_dev.dma_coherent = true; |
1107 | dma_coerce_mask_and_coherent(dev: &rx_chn->common.chan_dev, |
1108 | DMA_BIT_MASK(48)); |
1109 | } |
1110 | |
1111 | ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); |
1112 | if (ret) |
1113 | return ret; |
1114 | |
1115 | for (i = 0; i < rx_chn->flow_num; i++) |
1116 | rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; |
1117 | |
1118 | k3_udma_glue_dump_rx_chn(chn: rx_chn); |
1119 | |
1120 | return 0; |
1121 | } |
1122 | |
1123 | static struct k3_udma_glue_rx_channel * |
1124 | k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, |
1125 | struct k3_udma_glue_rx_channel_cfg *cfg) |
1126 | { |
1127 | struct k3_udma_glue_rx_channel *rx_chn; |
1128 | int ret; |
1129 | |
1130 | if (cfg->flow_id_num <= 0 || |
1131 | cfg->flow_id_use_rxchan_id || |
1132 | cfg->def_flow_cfg || |
1133 | cfg->flow_id_base < 0) |
1134 | return ERR_PTR(error: -EINVAL); |
1135 | |
1136 | /* |
1137 | * Remote RX channel is under control of Remote CPU core, so |
1138 | * Linux can only request and manipulate by dedicated RX flows |
1139 | */ |
1140 | |
1141 | rx_chn = devm_kzalloc(dev, size: sizeof(*rx_chn), GFP_KERNEL); |
1142 | if (!rx_chn) |
1143 | return ERR_PTR(error: -ENOMEM); |
1144 | |
1145 | rx_chn->common.dev = dev; |
1146 | rx_chn->common.swdata_size = cfg->swdata_size; |
1147 | rx_chn->remote = true; |
1148 | rx_chn->udma_rchan_id = -1; |
1149 | rx_chn->flow_num = cfg->flow_id_num; |
1150 | rx_chn->flow_id_base = cfg->flow_id_base; |
1151 | rx_chn->psil_paired = false; |
1152 | |
1153 | /* parse of udmap channel */ |
1154 | ret = of_k3_udma_glue_parse_chn(chn_np: dev->of_node, name, |
1155 | common: &rx_chn->common, tx_chn: false); |
1156 | if (ret) |
1157 | goto err; |
1158 | |
1159 | ret = k3_udma_glue_request_remote_rx_chn_common(rx_chn, cfg, dev); |
1160 | if (ret) |
1161 | goto err; |
1162 | |
1163 | return rx_chn; |
1164 | |
1165 | err: |
1166 | k3_udma_glue_release_rx_chn(rx_chn); |
1167 | return ERR_PTR(error: ret); |
1168 | } |
1169 | |
1170 | struct k3_udma_glue_rx_channel * |
1171 | k3_udma_glue_request_remote_rx_chn_for_thread_id(struct device *dev, |
1172 | struct k3_udma_glue_rx_channel_cfg *cfg, |
1173 | struct device_node *udmax_np, u32 thread_id) |
1174 | { |
1175 | struct k3_udma_glue_rx_channel *rx_chn; |
1176 | int ret; |
1177 | |
1178 | if (cfg->flow_id_num <= 0 || |
1179 | cfg->flow_id_use_rxchan_id || |
1180 | cfg->def_flow_cfg || |
1181 | cfg->flow_id_base < 0) |
1182 | return ERR_PTR(error: -EINVAL); |
1183 | |
1184 | /* |
1185 | * Remote RX channel is under control of Remote CPU core, so |
1186 | * Linux can only request and manipulate by dedicated RX flows |
1187 | */ |
1188 | |
1189 | rx_chn = devm_kzalloc(dev, size: sizeof(*rx_chn), GFP_KERNEL); |
1190 | if (!rx_chn) |
1191 | return ERR_PTR(error: -ENOMEM); |
1192 | |
1193 | rx_chn->common.dev = dev; |
1194 | rx_chn->common.swdata_size = cfg->swdata_size; |
1195 | rx_chn->remote = true; |
1196 | rx_chn->udma_rchan_id = -1; |
1197 | rx_chn->flow_num = cfg->flow_id_num; |
1198 | rx_chn->flow_id_base = cfg->flow_id_base; |
1199 | rx_chn->psil_paired = false; |
1200 | |
1201 | ret = of_k3_udma_glue_parse_chn_by_id(udmax_np, common: &rx_chn->common, tx_chn: false, thread_id); |
1202 | if (ret) |
1203 | goto err; |
1204 | |
1205 | ret = k3_udma_glue_request_remote_rx_chn_common(rx_chn, cfg, dev); |
1206 | if (ret) |
1207 | goto err; |
1208 | |
1209 | return rx_chn; |
1210 | |
1211 | err: |
1212 | k3_udma_glue_release_rx_chn(rx_chn); |
1213 | return ERR_PTR(error: ret); |
1214 | } |
1215 | EXPORT_SYMBOL_GPL(k3_udma_glue_request_remote_rx_chn_for_thread_id); |
1216 | |
1217 | struct k3_udma_glue_rx_channel * |
1218 | k3_udma_glue_request_rx_chn(struct device *dev, const char *name, |
1219 | struct k3_udma_glue_rx_channel_cfg *cfg) |
1220 | { |
1221 | if (cfg->remote) |
1222 | return k3_udma_glue_request_remote_rx_chn(dev, name, cfg); |
1223 | else |
1224 | return k3_udma_glue_request_rx_chn_priv(dev, name, cfg); |
1225 | } |
1226 | EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn); |
1227 | |
1228 | void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) |
1229 | { |
1230 | int i; |
1231 | |
1232 | if (IS_ERR_OR_NULL(ptr: rx_chn->common.udmax)) |
1233 | return; |
1234 | |
1235 | if (rx_chn->psil_paired) { |
1236 | xudma_navss_psil_unpair(ud: rx_chn->common.udmax, |
1237 | src_thread: rx_chn->common.src_thread, |
1238 | dst_thread: rx_chn->common.dst_thread); |
1239 | rx_chn->psil_paired = false; |
1240 | } |
1241 | |
1242 | for (i = 0; i < rx_chn->flow_num; i++) |
1243 | k3_udma_glue_release_rx_flow(rx_chn, flow_num: i); |
1244 | |
1245 | if (xudma_rflow_is_gp(ud: rx_chn->common.udmax, id: rx_chn->flow_id_base)) |
1246 | xudma_free_gp_rflow_range(ud: rx_chn->common.udmax, |
1247 | from: rx_chn->flow_id_base, |
1248 | cnt: rx_chn->flow_num); |
1249 | |
1250 | if (!IS_ERR_OR_NULL(ptr: rx_chn->udma_rchanx)) |
1251 | xudma_rchan_put(ud: rx_chn->common.udmax, |
1252 | p: rx_chn->udma_rchanx); |
1253 | |
1254 | if (rx_chn->common.chan_dev.parent) { |
1255 | device_unregister(dev: &rx_chn->common.chan_dev); |
1256 | rx_chn->common.chan_dev.parent = NULL; |
1257 | } |
1258 | } |
1259 | EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn); |
1260 | |
1261 | int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn, |
1262 | u32 flow_idx, |
1263 | struct k3_udma_glue_rx_flow_cfg *flow_cfg) |
1264 | { |
1265 | if (flow_idx >= rx_chn->flow_num) |
1266 | return -EINVAL; |
1267 | |
1268 | return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg); |
1269 | } |
1270 | EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init); |
1271 | |
1272 | u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn, |
1273 | u32 flow_idx) |
1274 | { |
1275 | struct k3_udma_glue_rx_flow *flow; |
1276 | |
1277 | if (flow_idx >= rx_chn->flow_num) |
1278 | return -EINVAL; |
1279 | |
1280 | flow = &rx_chn->flows[flow_idx]; |
1281 | |
1282 | return k3_ringacc_get_ring_id(ring: flow->ringrxfdq); |
1283 | } |
1284 | EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id); |
1285 | |
1286 | u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn) |
1287 | { |
1288 | return rx_chn->flow_id_base; |
1289 | } |
1290 | EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base); |
1291 | |
1292 | int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn, |
1293 | u32 flow_idx) |
1294 | { |
1295 | struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; |
1296 | const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; |
1297 | struct device *dev = rx_chn->common.dev; |
1298 | struct ti_sci_msg_rm_udmap_flow_cfg req; |
1299 | int rx_ring_id; |
1300 | int rx_ringfdq_id; |
1301 | int ret = 0; |
1302 | |
1303 | if (!rx_chn->remote) |
1304 | return -EINVAL; |
1305 | |
1306 | rx_ring_id = k3_ringacc_get_ring_id(ring: flow->ringrx); |
1307 | rx_ringfdq_id = k3_ringacc_get_ring_id(ring: flow->ringrxfdq); |
1308 | |
1309 | memset(&req, 0, sizeof(req)); |
1310 | |
1311 | req.valid_params = |
1312 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | |
1313 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | |
1314 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | |
1315 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | |
1316 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; |
1317 | req.nav_id = tisci_rm->tisci_dev_id; |
1318 | req.flow_index = flow->udma_rflow_id; |
1319 | req.rx_dest_qnum = rx_ring_id; |
1320 | req.rx_fdq0_sz0_qnum = rx_ringfdq_id; |
1321 | req.rx_fdq1_qnum = rx_ringfdq_id; |
1322 | req.rx_fdq2_qnum = rx_ringfdq_id; |
1323 | req.rx_fdq3_qnum = rx_ringfdq_id; |
1324 | |
1325 | ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); |
1326 | if (ret) { |
1327 | dev_err(dev, "flow%d enable failed: %d\n" , flow->udma_rflow_id, |
1328 | ret); |
1329 | } |
1330 | |
1331 | return ret; |
1332 | } |
1333 | EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable); |
1334 | |
1335 | int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn, |
1336 | u32 flow_idx) |
1337 | { |
1338 | struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx]; |
1339 | const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm; |
1340 | struct device *dev = rx_chn->common.dev; |
1341 | struct ti_sci_msg_rm_udmap_flow_cfg req; |
1342 | int ret = 0; |
1343 | |
1344 | if (!rx_chn->remote) |
1345 | return -EINVAL; |
1346 | |
1347 | memset(&req, 0, sizeof(req)); |
1348 | req.valid_params = |
1349 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID | |
1350 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID | |
1351 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID | |
1352 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID | |
1353 | TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID; |
1354 | req.nav_id = tisci_rm->tisci_dev_id; |
1355 | req.flow_index = flow->udma_rflow_id; |
1356 | req.rx_dest_qnum = TI_SCI_RESOURCE_NULL; |
1357 | req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL; |
1358 | req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL; |
1359 | req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL; |
1360 | req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL; |
1361 | |
1362 | ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req); |
1363 | if (ret) { |
1364 | dev_err(dev, "flow%d disable failed: %d\n" , flow->udma_rflow_id, |
1365 | ret); |
1366 | } |
1367 | |
1368 | return ret; |
1369 | } |
1370 | EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable); |
1371 | |
1372 | int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) |
1373 | { |
1374 | int ret; |
1375 | |
1376 | if (rx_chn->remote) |
1377 | return -EINVAL; |
1378 | |
1379 | if (rx_chn->flows_ready < rx_chn->flow_num) |
1380 | return -EINVAL; |
1381 | |
1382 | ret = xudma_navss_psil_pair(ud: rx_chn->common.udmax, |
1383 | src_thread: rx_chn->common.src_thread, |
1384 | dst_thread: rx_chn->common.dst_thread); |
1385 | if (ret) { |
1386 | dev_err(rx_chn->common.dev, "PSI-L request err %d\n" , ret); |
1387 | return ret; |
1388 | } |
1389 | |
1390 | rx_chn->psil_paired = true; |
1391 | |
1392 | xudma_rchanrt_write(rchan: rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, |
1393 | UDMA_CHAN_RT_CTL_EN); |
1394 | |
1395 | xudma_rchanrt_write(rchan: rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG, |
1396 | UDMA_PEER_RT_EN_ENABLE); |
1397 | |
1398 | k3_udma_glue_dump_rx_rt_chn(chn: rx_chn, mark: "rxrt en" ); |
1399 | return 0; |
1400 | } |
1401 | EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn); |
1402 | |
1403 | void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) |
1404 | { |
1405 | k3_udma_glue_dump_rx_rt_chn(chn: rx_chn, mark: "rxrt dis1" ); |
1406 | |
1407 | xudma_rchanrt_write(rchan: rx_chn->udma_rchanx, |
1408 | UDMA_CHAN_RT_PEER_RT_EN_REG, val: 0); |
1409 | xudma_rchanrt_write(rchan: rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, val: 0); |
1410 | |
1411 | k3_udma_glue_dump_rx_rt_chn(chn: rx_chn, mark: "rxrt dis2" ); |
1412 | |
1413 | if (rx_chn->psil_paired) { |
1414 | xudma_navss_psil_unpair(ud: rx_chn->common.udmax, |
1415 | src_thread: rx_chn->common.src_thread, |
1416 | dst_thread: rx_chn->common.dst_thread); |
1417 | rx_chn->psil_paired = false; |
1418 | } |
1419 | } |
1420 | EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn); |
1421 | |
1422 | void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, |
1423 | bool sync) |
1424 | { |
1425 | int i = 0; |
1426 | u32 val; |
1427 | |
1428 | if (rx_chn->remote) |
1429 | return; |
1430 | |
1431 | k3_udma_glue_dump_rx_rt_chn(chn: rx_chn, mark: "rxrt tdown1" ); |
1432 | |
1433 | xudma_rchanrt_write(rchan: rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG, |
1434 | UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN); |
1435 | |
1436 | val = xudma_rchanrt_read(rchan: rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG); |
1437 | |
1438 | while (sync && (val & UDMA_CHAN_RT_CTL_EN)) { |
1439 | val = xudma_rchanrt_read(rchan: rx_chn->udma_rchanx, |
1440 | UDMA_CHAN_RT_CTL_REG); |
1441 | udelay(1); |
1442 | if (i > K3_UDMAX_TDOWN_TIMEOUT_US) { |
1443 | dev_err(rx_chn->common.dev, "RX tdown timeout\n" ); |
1444 | break; |
1445 | } |
1446 | i++; |
1447 | } |
1448 | |
1449 | val = xudma_rchanrt_read(rchan: rx_chn->udma_rchanx, |
1450 | UDMA_CHAN_RT_PEER_RT_EN_REG); |
1451 | if (sync && (val & UDMA_PEER_RT_EN_ENABLE)) |
1452 | dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n" ); |
1453 | k3_udma_glue_dump_rx_rt_chn(chn: rx_chn, mark: "rxrt tdown2" ); |
1454 | } |
1455 | EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn); |
1456 | |
1457 | void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, |
1458 | u32 flow_num, void *data, |
1459 | void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq) |
1460 | { |
1461 | struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; |
1462 | struct device *dev = rx_chn->common.dev; |
1463 | dma_addr_t desc_dma; |
1464 | int occ_rx, i, ret; |
1465 | |
1466 | /* reset RXCQ as it is not input for udma - expected to be empty */ |
1467 | occ_rx = k3_ringacc_ring_get_occ(ring: flow->ringrx); |
1468 | dev_dbg(dev, "RX reset flow %u occ_rx %u\n" , flow_num, occ_rx); |
1469 | |
1470 | /* Skip RX FDQ in case one FDQ is used for the set of flows */ |
1471 | if (skip_fdq) |
1472 | goto do_reset; |
1473 | |
1474 | /* |
1475 | * RX FDQ reset need to be special way as it is input for udma and its |
1476 | * state cached by udma, so: |
1477 | * 1) save RX FDQ occ |
1478 | * 2) clean up RX FDQ and call callback .cleanup() for each desc |
1479 | * 3) reset RX FDQ in a special way |
1480 | */ |
1481 | occ_rx = k3_ringacc_ring_get_occ(ring: flow->ringrxfdq); |
1482 | dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n" , flow_num, occ_rx); |
1483 | |
1484 | for (i = 0; i < occ_rx; i++) { |
1485 | ret = k3_ringacc_ring_pop(ring: flow->ringrxfdq, elem: &desc_dma); |
1486 | if (ret) { |
1487 | if (ret != -ENODATA) |
1488 | dev_err(dev, "RX reset pop %d\n" , ret); |
1489 | break; |
1490 | } |
1491 | cleanup(data, desc_dma); |
1492 | } |
1493 | |
1494 | k3_ringacc_ring_reset_dma(ring: flow->ringrxfdq, occ: occ_rx); |
1495 | |
1496 | do_reset: |
1497 | k3_ringacc_ring_reset(ring: flow->ringrx); |
1498 | } |
1499 | EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn); |
1500 | |
1501 | int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, |
1502 | u32 flow_num, struct cppi5_host_desc_t *desc_rx, |
1503 | dma_addr_t desc_dma) |
1504 | { |
1505 | struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; |
1506 | |
1507 | return k3_ringacc_ring_push(ring: flow->ringrxfdq, elem: &desc_dma); |
1508 | } |
1509 | EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn); |
1510 | |
1511 | int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, |
1512 | u32 flow_num, dma_addr_t *desc_dma) |
1513 | { |
1514 | struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; |
1515 | |
1516 | return k3_ringacc_ring_pop(ring: flow->ringrx, elem: desc_dma); |
1517 | } |
1518 | EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn); |
1519 | |
1520 | int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn, |
1521 | u32 flow_num) |
1522 | { |
1523 | struct k3_udma_glue_rx_flow *flow; |
1524 | |
1525 | flow = &rx_chn->flows[flow_num]; |
1526 | |
1527 | if (xudma_is_pktdma(ud: rx_chn->common.udmax)) { |
1528 | flow->virq = xudma_pktdma_rflow_get_irq(ud: rx_chn->common.udmax, |
1529 | udma_rflow_id: flow->udma_rflow_id); |
1530 | } else { |
1531 | flow->virq = k3_ringacc_get_ring_irq_num(ring: flow->ringrx); |
1532 | } |
1533 | |
1534 | return flow->virq; |
1535 | } |
1536 | EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq); |
1537 | |
1538 | struct device * |
1539 | k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn) |
1540 | { |
1541 | if (xudma_is_pktdma(ud: rx_chn->common.udmax) && |
1542 | (rx_chn->common.atype_asel == 14 || rx_chn->common.atype_asel == 15)) |
1543 | return &rx_chn->common.chan_dev; |
1544 | |
1545 | return xudma_get_device(ud: rx_chn->common.udmax); |
1546 | } |
1547 | EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device); |
1548 | |
1549 | void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn, |
1550 | dma_addr_t *addr) |
1551 | { |
1552 | if (!xudma_is_pktdma(ud: rx_chn->common.udmax) || |
1553 | !rx_chn->common.atype_asel) |
1554 | return; |
1555 | |
1556 | *addr |= (u64)rx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT; |
1557 | } |
1558 | EXPORT_SYMBOL_GPL(k3_udma_glue_rx_dma_to_cppi5_addr); |
1559 | |
1560 | void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn, |
1561 | dma_addr_t *addr) |
1562 | { |
1563 | if (!xudma_is_pktdma(ud: rx_chn->common.udmax) || |
1564 | !rx_chn->common.atype_asel) |
1565 | return; |
1566 | |
1567 | *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0); |
1568 | } |
1569 | EXPORT_SYMBOL_GPL(k3_udma_glue_rx_cppi5_to_dma_addr); |
1570 | |
1571 | static int __init k3_udma_glue_class_init(void) |
1572 | { |
1573 | return class_register(class: &k3_udma_glue_devclass); |
1574 | } |
1575 | |
1576 | module_init(k3_udma_glue_class_init); |
1577 | MODULE_LICENSE("GPL v2" ); |
1578 | |