1// SPDX-License-Identifier: ISC
2/*
3 * Copyright (c) 2005-2011 Atheros Communications Inc.
4 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
5 * Copyright (c) 2018 The Linux Foundation. All rights reserved.
6 * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
7 */
8
9#include "hif.h"
10#include "ce.h"
11#include "debug.h"
12
13/*
14 * Support for Copy Engine hardware, which is mainly used for
15 * communication between Host and Target over a PCIe interconnect.
16 */
17
18/*
19 * A single CopyEngine (CE) comprises two "rings":
20 * a source ring
21 * a destination ring
22 *
23 * Each ring consists of a number of descriptors which specify
24 * an address, length, and meta-data.
25 *
26 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
27 * controls one ring and the other side controls the other ring.
28 * The source side chooses when to initiate a transfer and it
29 * chooses what to send (buffer address, length). The destination
30 * side keeps a supply of "anonymous receive buffers" available and
31 * it handles incoming data as it arrives (when the destination
32 * receives an interrupt).
33 *
34 * The sender may send a simple buffer (address/length) or it may
35 * send a small list of buffers. When a small list is sent, hardware
36 * "gathers" these and they end up in a single destination buffer
37 * with a single interrupt.
38 *
39 * There are several "contexts" managed by this layer -- more, it
40 * may seem -- than should be needed. These are provided mainly for
41 * maximum flexibility and especially to facilitate a simpler HIF
42 * implementation. There are per-CopyEngine recv, send, and watermark
43 * contexts. These are supplied by the caller when a recv, send,
44 * or watermark handler is established and they are echoed back to
45 * the caller when the respective callbacks are invoked. There is
46 * also a per-transfer context supplied by the caller when a buffer
47 * (or sendlist) is sent and when a buffer is enqueued for recv.
48 * These per-transfer contexts are echoed back to the caller when
49 * the buffer is sent/received.
50 */
51
52static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
53 struct ath10k_ce_pipe *ce_state)
54{
55 u32 ce_id = ce_state->id;
56 u32 addr = 0;
57
58 switch (ce_id) {
59 case 0:
60 addr = 0x00032000;
61 break;
62 case 3:
63 addr = 0x0003200C;
64 break;
65 case 4:
66 addr = 0x00032010;
67 break;
68 case 5:
69 addr = 0x00032014;
70 break;
71 case 7:
72 addr = 0x0003201C;
73 break;
74 default:
75 ath10k_warn(ar, fmt: "invalid CE id: %d", ce_id);
76 break;
77 }
78 return addr;
79}
80
81static inline unsigned int
82ath10k_set_ring_byte(unsigned int offset,
83 struct ath10k_hw_ce_regs_addr_map *addr_map)
84{
85 return ((offset << addr_map->lsb) & addr_map->mask);
86}
87
88static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
89{
90 struct ath10k_ce *ce = ath10k_ce_priv(ar);
91
92 return ce->bus_ops->read32(ar, offset);
93}
94
95static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
96{
97 struct ath10k_ce *ce = ath10k_ce_priv(ar);
98
99 ce->bus_ops->write32(ar, offset, value);
100}
101
102static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
103 u32 ce_ctrl_addr,
104 unsigned int n)
105{
106 ath10k_ce_write32(ar, offset: ce_ctrl_addr +
107 ar->hw_ce_regs->dst_wr_index_addr, value: n);
108}
109
110static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
111 u32 ce_ctrl_addr)
112{
113 return ath10k_ce_read32(ar, offset: ce_ctrl_addr +
114 ar->hw_ce_regs->dst_wr_index_addr);
115}
116
117static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
118 u32 ce_ctrl_addr,
119 unsigned int n)
120{
121 ath10k_ce_write32(ar, offset: ce_ctrl_addr +
122 ar->hw_ce_regs->sr_wr_index_addr, value: n);
123}
124
125static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
126 u32 ce_ctrl_addr)
127{
128 return ath10k_ce_read32(ar, offset: ce_ctrl_addr +
129 ar->hw_ce_regs->sr_wr_index_addr);
130}
131
132static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
133 u32 ce_id)
134{
135 struct ath10k_ce *ce = ath10k_ce_priv(ar);
136
137 return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
138}
139
140static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
141 u32 ce_ctrl_addr)
142{
143 struct ath10k_ce *ce = ath10k_ce_priv(ar);
144 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
145 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
146 u32 index;
147
148 if (ar->hw_params.rri_on_ddr &&
149 (ce_state->attr_flags & CE_ATTR_DIS_INTR))
150 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
151 else
152 index = ath10k_ce_read32(ar, offset: ce_ctrl_addr +
153 ar->hw_ce_regs->current_srri_addr);
154
155 return index;
156}
157
158static inline void
159ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
160 struct ath10k_ce_pipe *ce_state,
161 unsigned int value)
162{
163 ath10k_ce_write32(ar, offset: shadow_sr_wr_ind_addr(ar, ce_state), value);
164}
165
166static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
167 u32 ce_id,
168 u64 addr)
169{
170 struct ath10k_ce *ce = ath10k_ce_priv(ar);
171 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
172 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
173 u32 addr_lo = lower_32_bits(addr);
174
175 ath10k_ce_write32(ar, offset: ce_ctrl_addr +
176 ar->hw_ce_regs->sr_base_addr_lo, value: addr_lo);
177
178 if (ce_state->ops->ce_set_src_ring_base_addr_hi) {
179 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,
180 addr);
181 }
182}
183
184static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar,
185 u32 ce_ctrl_addr,
186 u64 addr)
187{
188 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
189
190 ath10k_ce_write32(ar, offset: ce_ctrl_addr +
191 ar->hw_ce_regs->sr_base_addr_hi, value: addr_hi);
192}
193
194static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
195 u32 ce_ctrl_addr,
196 unsigned int n)
197{
198 ath10k_ce_write32(ar, offset: ce_ctrl_addr +
199 ar->hw_ce_regs->sr_size_addr, value: n);
200}
201
202static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
203 u32 ce_ctrl_addr,
204 unsigned int n)
205{
206 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
207
208 u32 ctrl1_addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr +
209 ctrl_regs->addr);
210
211 ath10k_ce_write32(ar, offset: ce_ctrl_addr + ctrl_regs->addr,
212 value: (ctrl1_addr & ~(ctrl_regs->dmax->mask)) |
213 ath10k_set_ring_byte(offset: n, addr_map: ctrl_regs->dmax));
214}
215
216static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
217 u32 ce_ctrl_addr,
218 unsigned int n)
219{
220 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
221
222 u32 ctrl1_addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr +
223 ctrl_regs->addr);
224
225 ath10k_ce_write32(ar, offset: ce_ctrl_addr + ctrl_regs->addr,
226 value: (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
227 ath10k_set_ring_byte(offset: n, addr_map: ctrl_regs->src_ring));
228}
229
230static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
231 u32 ce_ctrl_addr,
232 unsigned int n)
233{
234 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
235
236 u32 ctrl1_addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr +
237 ctrl_regs->addr);
238
239 ath10k_ce_write32(ar, offset: ce_ctrl_addr + ctrl_regs->addr,
240 value: (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
241 ath10k_set_ring_byte(offset: n, addr_map: ctrl_regs->dst_ring));
242}
243
244static inline
245 u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
246{
247 struct ath10k_ce *ce = ath10k_ce_priv(ar);
248
249 return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
250 CE_DDR_RRI_MASK;
251}
252
253static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
254 u32 ce_ctrl_addr)
255{
256 struct ath10k_ce *ce = ath10k_ce_priv(ar);
257 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
258 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
259 u32 index;
260
261 if (ar->hw_params.rri_on_ddr &&
262 (ce_state->attr_flags & CE_ATTR_DIS_INTR))
263 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
264 else
265 index = ath10k_ce_read32(ar, offset: ce_ctrl_addr +
266 ar->hw_ce_regs->current_drri_addr);
267
268 return index;
269}
270
271static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
272 u32 ce_id,
273 u64 addr)
274{
275 struct ath10k_ce *ce = ath10k_ce_priv(ar);
276 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
277 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
278 u32 addr_lo = lower_32_bits(addr);
279
280 ath10k_ce_write32(ar, offset: ce_ctrl_addr +
281 ar->hw_ce_regs->dr_base_addr_lo, value: addr_lo);
282
283 if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {
284 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,
285 addr);
286 }
287}
288
289static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar,
290 u32 ce_ctrl_addr,
291 u64 addr)
292{
293 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
294 u32 reg_value;
295
296 reg_value = ath10k_ce_read32(ar, offset: ce_ctrl_addr +
297 ar->hw_ce_regs->dr_base_addr_hi);
298 reg_value &= ~CE_DESC_ADDR_HI_MASK;
299 reg_value |= addr_hi;
300 ath10k_ce_write32(ar, offset: ce_ctrl_addr +
301 ar->hw_ce_regs->dr_base_addr_hi, value: reg_value);
302}
303
304static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
305 u32 ce_ctrl_addr,
306 unsigned int n)
307{
308 ath10k_ce_write32(ar, offset: ce_ctrl_addr +
309 ar->hw_ce_regs->dr_size_addr, value: n);
310}
311
312static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
313 u32 ce_ctrl_addr,
314 unsigned int n)
315{
316 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
317 u32 addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + srcr_wm->addr);
318
319 ath10k_ce_write32(ar, offset: ce_ctrl_addr + srcr_wm->addr,
320 value: (addr & ~(srcr_wm->wm_high->mask)) |
321 (ath10k_set_ring_byte(offset: n, addr_map: srcr_wm->wm_high)));
322}
323
324static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
325 u32 ce_ctrl_addr,
326 unsigned int n)
327{
328 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
329 u32 addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + srcr_wm->addr);
330
331 ath10k_ce_write32(ar, offset: ce_ctrl_addr + srcr_wm->addr,
332 value: (addr & ~(srcr_wm->wm_low->mask)) |
333 (ath10k_set_ring_byte(offset: n, addr_map: srcr_wm->wm_low)));
334}
335
336static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
337 u32 ce_ctrl_addr,
338 unsigned int n)
339{
340 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
341 u32 addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + dstr_wm->addr);
342
343 ath10k_ce_write32(ar, offset: ce_ctrl_addr + dstr_wm->addr,
344 value: (addr & ~(dstr_wm->wm_high->mask)) |
345 (ath10k_set_ring_byte(offset: n, addr_map: dstr_wm->wm_high)));
346}
347
348static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
349 u32 ce_ctrl_addr,
350 unsigned int n)
351{
352 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
353 u32 addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + dstr_wm->addr);
354
355 ath10k_ce_write32(ar, offset: ce_ctrl_addr + dstr_wm->addr,
356 value: (addr & ~(dstr_wm->wm_low->mask)) |
357 (ath10k_set_ring_byte(offset: n, addr_map: dstr_wm->wm_low)));
358}
359
360static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
361 u32 ce_ctrl_addr)
362{
363 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
364
365 u32 host_ie_addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr +
366 ar->hw_ce_regs->host_ie_addr);
367
368 ath10k_ce_write32(ar, offset: ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
369 value: host_ie_addr | host_ie->copy_complete->mask);
370}
371
372static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
373 u32 ce_ctrl_addr)
374{
375 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
376
377 u32 host_ie_addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr +
378 ar->hw_ce_regs->host_ie_addr);
379
380 ath10k_ce_write32(ar, offset: ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
381 value: host_ie_addr & ~(host_ie->copy_complete->mask));
382}
383
384static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
385 u32 ce_ctrl_addr)
386{
387 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
388
389 u32 host_ie_addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr +
390 ar->hw_ce_regs->host_ie_addr);
391
392 ath10k_ce_write32(ar, offset: ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
393 value: host_ie_addr & ~(wm_regs->wm_mask));
394}
395
396static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
397 u32 ce_ctrl_addr)
398{
399 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
400
401 u32 misc_ie_addr = ath10k_ce_read32(ar,
402 offset: ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
403
404 ath10k_ce_write32(ar,
405 offset: ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
406 value: misc_ie_addr & ~(misc_regs->err_mask));
407}
408
409static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
410 u32 ce_ctrl_addr,
411 unsigned int mask)
412{
413 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
414
415 ath10k_ce_write32(ar, offset: ce_ctrl_addr + wm_regs->addr, value: mask);
416}
417
418/*
419 * Guts of ath10k_ce_send.
420 * The caller takes responsibility for any needed locking.
421 */
422static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
423 void *per_transfer_context,
424 dma_addr_t buffer,
425 unsigned int nbytes,
426 unsigned int transfer_id,
427 unsigned int flags)
428{
429 struct ath10k *ar = ce_state->ar;
430 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
431 struct ce_desc *desc, sdesc;
432 unsigned int nentries_mask = src_ring->nentries_mask;
433 unsigned int sw_index = src_ring->sw_index;
434 unsigned int write_index = src_ring->write_index;
435 u32 ctrl_addr = ce_state->ctrl_addr;
436 u32 desc_flags = 0;
437 int ret = 0;
438
439 if (nbytes > ce_state->src_sz_max)
440 ath10k_warn(ar, fmt: "%s: send more we can (nbytes: %d, max: %d)\n",
441 __func__, nbytes, ce_state->src_sz_max);
442
443 if (unlikely(CE_RING_DELTA(nentries_mask,
444 write_index, sw_index - 1) <= 0)) {
445 ret = -ENOSR;
446 goto exit;
447 }
448
449 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
450 write_index);
451
452 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
453
454 if (flags & CE_SEND_FLAG_GATHER)
455 desc_flags |= CE_DESC_FLAGS_GATHER;
456 if (flags & CE_SEND_FLAG_BYTE_SWAP)
457 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
458
459 sdesc.addr = __cpu_to_le32(buffer);
460 sdesc.nbytes = __cpu_to_le16(nbytes);
461 sdesc.flags = __cpu_to_le16(desc_flags);
462
463 *desc = sdesc;
464
465 src_ring->per_transfer_context[write_index] = per_transfer_context;
466
467 /* Update Source Ring Write Index */
468 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
469
470 /* WORKAROUND */
471 if (!(flags & CE_SEND_FLAG_GATHER))
472 ath10k_ce_src_ring_write_index_set(ar, ce_ctrl_addr: ctrl_addr, n: write_index);
473
474 src_ring->write_index = write_index;
475exit:
476 return ret;
477}
478
479static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
480 void *per_transfer_context,
481 dma_addr_t buffer,
482 unsigned int nbytes,
483 unsigned int transfer_id,
484 unsigned int flags)
485{
486 struct ath10k *ar = ce_state->ar;
487 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
488 struct ce_desc_64 *desc, sdesc;
489 unsigned int nentries_mask = src_ring->nentries_mask;
490 unsigned int sw_index;
491 unsigned int write_index = src_ring->write_index;
492 u32 ctrl_addr = ce_state->ctrl_addr;
493 __le32 *addr;
494 u32 desc_flags = 0;
495 int ret = 0;
496
497 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
498 return -ESHUTDOWN;
499
500 if (nbytes > ce_state->src_sz_max)
501 ath10k_warn(ar, fmt: "%s: send more we can (nbytes: %d, max: %d)\n",
502 __func__, nbytes, ce_state->src_sz_max);
503
504 if (ar->hw_params.rri_on_ddr)
505 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id: ce_state->id);
506 else
507 sw_index = src_ring->sw_index;
508
509 if (unlikely(CE_RING_DELTA(nentries_mask,
510 write_index, sw_index - 1) <= 0)) {
511 ret = -ENOSR;
512 goto exit;
513 }
514
515 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
516 write_index);
517
518 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
519
520 if (flags & CE_SEND_FLAG_GATHER)
521 desc_flags |= CE_DESC_FLAGS_GATHER;
522
523 if (flags & CE_SEND_FLAG_BYTE_SWAP)
524 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
525
526 addr = (__le32 *)&sdesc.addr;
527
528 flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK;
529 addr[0] = __cpu_to_le32(buffer);
530 addr[1] = __cpu_to_le32(flags);
531 if (flags & CE_SEND_FLAG_GATHER)
532 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
533 else
534 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
535
536 sdesc.nbytes = __cpu_to_le16(nbytes);
537 sdesc.flags = __cpu_to_le16(desc_flags);
538
539 *desc = sdesc;
540
541 src_ring->per_transfer_context[write_index] = per_transfer_context;
542
543 /* Update Source Ring Write Index */
544 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
545
546 if (!(flags & CE_SEND_FLAG_GATHER)) {
547 if (ar->hw_params.shadow_reg_support)
548 ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
549 value: write_index);
550 else
551 ath10k_ce_src_ring_write_index_set(ar, ce_ctrl_addr: ctrl_addr,
552 n: write_index);
553 }
554
555 src_ring->write_index = write_index;
556exit:
557 return ret;
558}
559
560int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
561 void *per_transfer_context,
562 dma_addr_t buffer,
563 unsigned int nbytes,
564 unsigned int transfer_id,
565 unsigned int flags)
566{
567 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
568 buffer, nbytes, transfer_id, flags);
569}
570EXPORT_SYMBOL(ath10k_ce_send_nolock);
571
572void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
573{
574 struct ath10k *ar = pipe->ar;
575 struct ath10k_ce *ce = ath10k_ce_priv(ar);
576 struct ath10k_ce_ring *src_ring = pipe->src_ring;
577 u32 ctrl_addr = pipe->ctrl_addr;
578
579 lockdep_assert_held(&ce->ce_lock);
580
581 /*
582 * This function must be called only if there is an incomplete
583 * scatter-gather transfer (before index register is updated)
584 * that needs to be cleaned up.
585 */
586 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
587 return;
588
589 if (WARN_ON_ONCE(src_ring->write_index ==
590 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
591 return;
592
593 src_ring->write_index--;
594 src_ring->write_index &= src_ring->nentries_mask;
595
596 src_ring->per_transfer_context[src_ring->write_index] = NULL;
597}
598EXPORT_SYMBOL(__ath10k_ce_send_revert);
599
600int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
601 void *per_transfer_context,
602 dma_addr_t buffer,
603 unsigned int nbytes,
604 unsigned int transfer_id,
605 unsigned int flags)
606{
607 struct ath10k *ar = ce_state->ar;
608 struct ath10k_ce *ce = ath10k_ce_priv(ar);
609 int ret;
610
611 spin_lock_bh(lock: &ce->ce_lock);
612 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
613 buffer, nbytes, transfer_id, flags);
614 spin_unlock_bh(lock: &ce->ce_lock);
615
616 return ret;
617}
618EXPORT_SYMBOL(ath10k_ce_send);
619
620int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
621{
622 struct ath10k *ar = pipe->ar;
623 struct ath10k_ce *ce = ath10k_ce_priv(ar);
624 int delta;
625
626 spin_lock_bh(lock: &ce->ce_lock);
627 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
628 pipe->src_ring->write_index,
629 pipe->src_ring->sw_index - 1);
630 spin_unlock_bh(lock: &ce->ce_lock);
631
632 return delta;
633}
634EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
635
636int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
637{
638 struct ath10k *ar = pipe->ar;
639 struct ath10k_ce *ce = ath10k_ce_priv(ar);
640 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
641 unsigned int nentries_mask = dest_ring->nentries_mask;
642 unsigned int write_index = dest_ring->write_index;
643 unsigned int sw_index = dest_ring->sw_index;
644
645 lockdep_assert_held(&ce->ce_lock);
646
647 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
648}
649EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
650
651static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
652 dma_addr_t paddr)
653{
654 struct ath10k *ar = pipe->ar;
655 struct ath10k_ce *ce = ath10k_ce_priv(ar);
656 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
657 unsigned int nentries_mask = dest_ring->nentries_mask;
658 unsigned int write_index = dest_ring->write_index;
659 unsigned int sw_index = dest_ring->sw_index;
660 struct ce_desc *base = dest_ring->base_addr_owner_space;
661 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
662 u32 ctrl_addr = pipe->ctrl_addr;
663
664 lockdep_assert_held(&ce->ce_lock);
665
666 if ((pipe->id != 5) &&
667 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
668 return -ENOSPC;
669
670 desc->addr = __cpu_to_le32(paddr);
671 desc->nbytes = 0;
672
673 dest_ring->per_transfer_context[write_index] = ctx;
674 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
675 ath10k_ce_dest_ring_write_index_set(ar, ce_ctrl_addr: ctrl_addr, n: write_index);
676 dest_ring->write_index = write_index;
677
678 return 0;
679}
680
681static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
682 void *ctx,
683 dma_addr_t paddr)
684{
685 struct ath10k *ar = pipe->ar;
686 struct ath10k_ce *ce = ath10k_ce_priv(ar);
687 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
688 unsigned int nentries_mask = dest_ring->nentries_mask;
689 unsigned int write_index = dest_ring->write_index;
690 unsigned int sw_index = dest_ring->sw_index;
691 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
692 struct ce_desc_64 *desc =
693 CE_DEST_RING_TO_DESC_64(base, write_index);
694 u32 ctrl_addr = pipe->ctrl_addr;
695
696 lockdep_assert_held(&ce->ce_lock);
697
698 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
699 return -ENOSPC;
700
701 desc->addr = __cpu_to_le64(paddr);
702 desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK);
703
704 desc->nbytes = 0;
705
706 dest_ring->per_transfer_context[write_index] = ctx;
707 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
708 ath10k_ce_dest_ring_write_index_set(ar, ce_ctrl_addr: ctrl_addr, n: write_index);
709 dest_ring->write_index = write_index;
710
711 return 0;
712}
713
714void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
715{
716 struct ath10k *ar = pipe->ar;
717 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
718 unsigned int nentries_mask = dest_ring->nentries_mask;
719 unsigned int write_index = dest_ring->write_index;
720 u32 ctrl_addr = pipe->ctrl_addr;
721 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ce_ctrl_addr: ctrl_addr);
722
723 /* Prevent CE ring stuck issue that will occur when ring is full.
724 * Make sure that write index is 1 less than read index.
725 */
726 if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
727 nentries -= 1;
728
729 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
730 ath10k_ce_dest_ring_write_index_set(ar, ce_ctrl_addr: ctrl_addr, n: write_index);
731 dest_ring->write_index = write_index;
732}
733EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
734
735int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
736 dma_addr_t paddr)
737{
738 struct ath10k *ar = pipe->ar;
739 struct ath10k_ce *ce = ath10k_ce_priv(ar);
740 int ret;
741
742 spin_lock_bh(lock: &ce->ce_lock);
743 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
744 spin_unlock_bh(lock: &ce->ce_lock);
745
746 return ret;
747}
748EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
749
750/*
751 * Guts of ath10k_ce_completed_recv_next.
752 * The caller takes responsibility for any necessary locking.
753 */
754static int
755 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
756 void **per_transfer_contextp,
757 unsigned int *nbytesp)
758{
759 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
760 unsigned int nentries_mask = dest_ring->nentries_mask;
761 unsigned int sw_index = dest_ring->sw_index;
762
763 struct ce_desc *base = dest_ring->base_addr_owner_space;
764 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
765 struct ce_desc sdesc;
766 u16 nbytes;
767
768 /* Copy in one go for performance reasons */
769 sdesc = *desc;
770
771 nbytes = __le16_to_cpu(sdesc.nbytes);
772 if (nbytes == 0) {
773 /*
774 * This closes a relatively unusual race where the Host
775 * sees the updated DRRI before the update to the
776 * corresponding descriptor has completed. We treat this
777 * as a descriptor that is not yet done.
778 */
779 return -EIO;
780 }
781
782 desc->nbytes = 0;
783
784 /* Return data from completed destination descriptor */
785 *nbytesp = nbytes;
786
787 if (per_transfer_contextp)
788 *per_transfer_contextp =
789 dest_ring->per_transfer_context[sw_index];
790
791 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
792 * So update transfer context all CEs except CE5.
793 */
794 if (ce_state->id != 5)
795 dest_ring->per_transfer_context[sw_index] = NULL;
796
797 /* Update sw_index */
798 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
799 dest_ring->sw_index = sw_index;
800
801 return 0;
802}
803
804static int
805_ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
806 void **per_transfer_contextp,
807 unsigned int *nbytesp)
808{
809 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
810 unsigned int nentries_mask = dest_ring->nentries_mask;
811 unsigned int sw_index = dest_ring->sw_index;
812 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
813 struct ce_desc_64 *desc =
814 CE_DEST_RING_TO_DESC_64(base, sw_index);
815 struct ce_desc_64 sdesc;
816 u16 nbytes;
817
818 /* Copy in one go for performance reasons */
819 sdesc = *desc;
820
821 nbytes = __le16_to_cpu(sdesc.nbytes);
822 if (nbytes == 0) {
823 /* This closes a relatively unusual race where the Host
824 * sees the updated DRRI before the update to the
825 * corresponding descriptor has completed. We treat this
826 * as a descriptor that is not yet done.
827 */
828 return -EIO;
829 }
830
831 desc->nbytes = 0;
832
833 /* Return data from completed destination descriptor */
834 *nbytesp = nbytes;
835
836 if (per_transfer_contextp)
837 *per_transfer_contextp =
838 dest_ring->per_transfer_context[sw_index];
839
840 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
841 * So update transfer context all CEs except CE5.
842 */
843 if (ce_state->id != 5)
844 dest_ring->per_transfer_context[sw_index] = NULL;
845
846 /* Update sw_index */
847 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
848 dest_ring->sw_index = sw_index;
849
850 return 0;
851}
852
853int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
854 void **per_transfer_ctx,
855 unsigned int *nbytesp)
856{
857 return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
858 per_transfer_ctx,
859 nbytesp);
860}
861EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
862
863int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
864 void **per_transfer_contextp,
865 unsigned int *nbytesp)
866{
867 struct ath10k *ar = ce_state->ar;
868 struct ath10k_ce *ce = ath10k_ce_priv(ar);
869 int ret;
870
871 spin_lock_bh(lock: &ce->ce_lock);
872 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
873 per_transfer_contextp,
874 nbytesp);
875
876 spin_unlock_bh(lock: &ce->ce_lock);
877
878 return ret;
879}
880EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
881
882static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
883 void **per_transfer_contextp,
884 dma_addr_t *bufferp)
885{
886 struct ath10k_ce_ring *dest_ring;
887 unsigned int nentries_mask;
888 unsigned int sw_index;
889 unsigned int write_index;
890 int ret;
891 struct ath10k *ar;
892 struct ath10k_ce *ce;
893
894 dest_ring = ce_state->dest_ring;
895
896 if (!dest_ring)
897 return -EIO;
898
899 ar = ce_state->ar;
900 ce = ath10k_ce_priv(ar);
901
902 spin_lock_bh(lock: &ce->ce_lock);
903
904 nentries_mask = dest_ring->nentries_mask;
905 sw_index = dest_ring->sw_index;
906 write_index = dest_ring->write_index;
907 if (write_index != sw_index) {
908 struct ce_desc *base = dest_ring->base_addr_owner_space;
909 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
910
911 /* Return data from completed destination descriptor */
912 *bufferp = __le32_to_cpu(desc->addr);
913
914 if (per_transfer_contextp)
915 *per_transfer_contextp =
916 dest_ring->per_transfer_context[sw_index];
917
918 /* sanity */
919 dest_ring->per_transfer_context[sw_index] = NULL;
920 desc->nbytes = 0;
921
922 /* Update sw_index */
923 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
924 dest_ring->sw_index = sw_index;
925 ret = 0;
926 } else {
927 ret = -EIO;
928 }
929
930 spin_unlock_bh(lock: &ce->ce_lock);
931
932 return ret;
933}
934
935static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
936 void **per_transfer_contextp,
937 dma_addr_t *bufferp)
938{
939 struct ath10k_ce_ring *dest_ring;
940 unsigned int nentries_mask;
941 unsigned int sw_index;
942 unsigned int write_index;
943 int ret;
944 struct ath10k *ar;
945 struct ath10k_ce *ce;
946
947 dest_ring = ce_state->dest_ring;
948
949 if (!dest_ring)
950 return -EIO;
951
952 ar = ce_state->ar;
953 ce = ath10k_ce_priv(ar);
954
955 spin_lock_bh(lock: &ce->ce_lock);
956
957 nentries_mask = dest_ring->nentries_mask;
958 sw_index = dest_ring->sw_index;
959 write_index = dest_ring->write_index;
960 if (write_index != sw_index) {
961 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
962 struct ce_desc_64 *desc =
963 CE_DEST_RING_TO_DESC_64(base, sw_index);
964
965 /* Return data from completed destination descriptor */
966 *bufferp = __le64_to_cpu(desc->addr);
967
968 if (per_transfer_contextp)
969 *per_transfer_contextp =
970 dest_ring->per_transfer_context[sw_index];
971
972 /* sanity */
973 dest_ring->per_transfer_context[sw_index] = NULL;
974 desc->nbytes = 0;
975
976 /* Update sw_index */
977 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
978 dest_ring->sw_index = sw_index;
979 ret = 0;
980 } else {
981 ret = -EIO;
982 }
983
984 spin_unlock_bh(lock: &ce->ce_lock);
985
986 return ret;
987}
988
989int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
990 void **per_transfer_contextp,
991 dma_addr_t *bufferp)
992{
993 return ce_state->ops->ce_revoke_recv_next(ce_state,
994 per_transfer_contextp,
995 bufferp);
996}
997EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
998
999/*
1000 * Guts of ath10k_ce_completed_send_next.
1001 * The caller takes responsibility for any necessary locking.
1002 */
1003static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1004 void **per_transfer_contextp)
1005{
1006 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1007 u32 ctrl_addr = ce_state->ctrl_addr;
1008 struct ath10k *ar = ce_state->ar;
1009 unsigned int nentries_mask = src_ring->nentries_mask;
1010 unsigned int sw_index = src_ring->sw_index;
1011 unsigned int read_index;
1012 struct ce_desc *desc;
1013
1014 if (src_ring->hw_index == sw_index) {
1015 /*
1016 * The SW completion index has caught up with the cached
1017 * version of the HW completion index.
1018 * Update the cached HW completion index to see whether
1019 * the SW has really caught up to the HW, or if the cached
1020 * value of the HW index has become stale.
1021 */
1022
1023 read_index = ath10k_ce_src_ring_read_index_get(ar, ce_ctrl_addr: ctrl_addr);
1024 if (read_index == 0xffffffff)
1025 return -ENODEV;
1026
1027 read_index &= nentries_mask;
1028 src_ring->hw_index = read_index;
1029 }
1030
1031 if (ar->hw_params.rri_on_ddr)
1032 read_index = ath10k_ce_src_ring_read_index_get(ar, ce_ctrl_addr: ctrl_addr);
1033 else
1034 read_index = src_ring->hw_index;
1035
1036 if (read_index == sw_index)
1037 return -EIO;
1038
1039 if (per_transfer_contextp)
1040 *per_transfer_contextp =
1041 src_ring->per_transfer_context[sw_index];
1042
1043 /* sanity */
1044 src_ring->per_transfer_context[sw_index] = NULL;
1045 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1046 sw_index);
1047 desc->nbytes = 0;
1048
1049 /* Update sw_index */
1050 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1051 src_ring->sw_index = sw_index;
1052
1053 return 0;
1054}
1055
1056static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state,
1057 void **per_transfer_contextp)
1058{
1059 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1060 u32 ctrl_addr = ce_state->ctrl_addr;
1061 struct ath10k *ar = ce_state->ar;
1062 unsigned int nentries_mask = src_ring->nentries_mask;
1063 unsigned int sw_index = src_ring->sw_index;
1064 unsigned int read_index;
1065 struct ce_desc_64 *desc;
1066
1067 if (src_ring->hw_index == sw_index) {
1068 /*
1069 * The SW completion index has caught up with the cached
1070 * version of the HW completion index.
1071 * Update the cached HW completion index to see whether
1072 * the SW has really caught up to the HW, or if the cached
1073 * value of the HW index has become stale.
1074 */
1075
1076 read_index = ath10k_ce_src_ring_read_index_get(ar, ce_ctrl_addr: ctrl_addr);
1077 if (read_index == 0xffffffff)
1078 return -ENODEV;
1079
1080 read_index &= nentries_mask;
1081 src_ring->hw_index = read_index;
1082 }
1083
1084 if (ar->hw_params.rri_on_ddr)
1085 read_index = ath10k_ce_src_ring_read_index_get(ar, ce_ctrl_addr: ctrl_addr);
1086 else
1087 read_index = src_ring->hw_index;
1088
1089 if (read_index == sw_index)
1090 return -EIO;
1091
1092 if (per_transfer_contextp)
1093 *per_transfer_contextp =
1094 src_ring->per_transfer_context[sw_index];
1095
1096 /* sanity */
1097 src_ring->per_transfer_context[sw_index] = NULL;
1098 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
1099 sw_index);
1100 desc->nbytes = 0;
1101
1102 /* Update sw_index */
1103 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1104 src_ring->sw_index = sw_index;
1105
1106 return 0;
1107}
1108
1109int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1110 void **per_transfer_contextp)
1111{
1112 return ce_state->ops->ce_completed_send_next_nolock(ce_state,
1113 per_transfer_contextp);
1114}
1115EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1116
1117static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1118 struct ath10k_ce_ring *src_ring,
1119 u32 sw_index,
1120 dma_addr_t *bufferp,
1121 u32 *nbytesp,
1122 u32 *transfer_idp)
1123{
1124 struct ce_desc *base = src_ring->base_addr_owner_space;
1125 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1126
1127 /* Return data from completed source descriptor */
1128 *bufferp = __le32_to_cpu(desc->addr);
1129 *nbytesp = __le16_to_cpu(desc->nbytes);
1130 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1131 CE_DESC_FLAGS_META_DATA);
1132}
1133
1134static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1135 struct ath10k_ce_ring *src_ring,
1136 u32 sw_index,
1137 dma_addr_t *bufferp,
1138 u32 *nbytesp,
1139 u32 *transfer_idp)
1140{
1141 struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1142 struct ce_desc_64 *desc =
1143 CE_SRC_RING_TO_DESC_64(base, sw_index);
1144
1145 /* Return data from completed source descriptor */
1146 *bufferp = __le64_to_cpu(desc->addr);
1147 *nbytesp = __le16_to_cpu(desc->nbytes);
1148 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1149 CE_DESC_FLAGS_META_DATA);
1150}
1151
1152/* NB: Modeled after ath10k_ce_completed_send_next */
1153int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1154 void **per_transfer_contextp,
1155 dma_addr_t *bufferp,
1156 unsigned int *nbytesp,
1157 unsigned int *transfer_idp)
1158{
1159 struct ath10k_ce_ring *src_ring;
1160 unsigned int nentries_mask;
1161 unsigned int sw_index;
1162 unsigned int write_index;
1163 int ret;
1164 struct ath10k *ar;
1165 struct ath10k_ce *ce;
1166
1167 src_ring = ce_state->src_ring;
1168
1169 if (!src_ring)
1170 return -EIO;
1171
1172 ar = ce_state->ar;
1173 ce = ath10k_ce_priv(ar);
1174
1175 spin_lock_bh(lock: &ce->ce_lock);
1176
1177 nentries_mask = src_ring->nentries_mask;
1178 sw_index = src_ring->sw_index;
1179 write_index = src_ring->write_index;
1180
1181 if (write_index != sw_index) {
1182 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1183 bufferp, nbytesp,
1184 transfer_idp);
1185
1186 if (per_transfer_contextp)
1187 *per_transfer_contextp =
1188 src_ring->per_transfer_context[sw_index];
1189
1190 /* sanity */
1191 src_ring->per_transfer_context[sw_index] = NULL;
1192
1193 /* Update sw_index */
1194 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1195 src_ring->sw_index = sw_index;
1196 ret = 0;
1197 } else {
1198 ret = -EIO;
1199 }
1200
1201 spin_unlock_bh(lock: &ce->ce_lock);
1202
1203 return ret;
1204}
1205EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1206
1207int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1208 void **per_transfer_contextp)
1209{
1210 struct ath10k *ar = ce_state->ar;
1211 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1212 int ret;
1213
1214 spin_lock_bh(lock: &ce->ce_lock);
1215 ret = ath10k_ce_completed_send_next_nolock(ce_state,
1216 per_transfer_contextp);
1217 spin_unlock_bh(lock: &ce->ce_lock);
1218
1219 return ret;
1220}
1221EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1222
1223/*
1224 * Guts of interrupt handler for per-engine interrupts on a particular CE.
1225 *
1226 * Invokes registered callbacks for recv_complete,
1227 * send_complete, and watermarks.
1228 */
1229void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1230{
1231 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1232 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1233 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1234 u32 ctrl_addr = ce_state->ctrl_addr;
1235
1236 /*
1237 * Clear before handling
1238 *
1239 * Misc CE interrupts are not being handled, but still need
1240 * to be cleared.
1241 *
1242 * NOTE: When the last copy engine interrupt is cleared the
1243 * hardware will go to sleep. Once this happens any access to
1244 * the CE registers can cause a hardware fault.
1245 */
1246 ath10k_ce_engine_int_status_clear(ar, ce_ctrl_addr: ctrl_addr,
1247 mask: wm_regs->cc_mask | wm_regs->wm_mask);
1248
1249 if (ce_state->recv_cb)
1250 ce_state->recv_cb(ce_state);
1251
1252 if (ce_state->send_cb)
1253 ce_state->send_cb(ce_state);
1254}
1255EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1256
1257/*
1258 * Handler for per-engine interrupts on ALL active CEs.
1259 * This is used in cases where the system is sharing a
1260 * single interrupt for all CEs
1261 */
1262
1263void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1264{
1265 int ce_id;
1266 u32 intr_summary;
1267
1268 intr_summary = ath10k_ce_interrupt_summary(ar);
1269
1270 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1271 if (intr_summary & (1 << ce_id))
1272 intr_summary &= ~(1 << ce_id);
1273 else
1274 /* no intr pending on this CE */
1275 continue;
1276
1277 ath10k_ce_per_engine_service(ar, ce_id);
1278 }
1279}
1280EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1281
1282/*
1283 * Adjust interrupts for the copy complete handler.
1284 * If it's needed for either send or recv, then unmask
1285 * this interrupt; otherwise, mask it.
1286 *
1287 * Called with ce_lock held.
1288 */
1289static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1290{
1291 u32 ctrl_addr = ce_state->ctrl_addr;
1292 struct ath10k *ar = ce_state->ar;
1293 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1294
1295 if ((!disable_copy_compl_intr) &&
1296 (ce_state->send_cb || ce_state->recv_cb))
1297 ath10k_ce_copy_complete_inter_enable(ar, ce_ctrl_addr: ctrl_addr);
1298 else
1299 ath10k_ce_copy_complete_intr_disable(ar, ce_ctrl_addr: ctrl_addr);
1300
1301 ath10k_ce_watermark_intr_disable(ar, ce_ctrl_addr: ctrl_addr);
1302}
1303
1304void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id)
1305{
1306 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1307 struct ath10k_ce_pipe *ce_state;
1308 u32 ctrl_addr;
1309
1310 ce_state = &ce->ce_states[ce_id];
1311 if (ce_state->attr_flags & CE_ATTR_POLL)
1312 return;
1313
1314 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1315
1316 ath10k_ce_copy_complete_intr_disable(ar, ce_ctrl_addr: ctrl_addr);
1317 ath10k_ce_error_intr_disable(ar, ce_ctrl_addr: ctrl_addr);
1318 ath10k_ce_watermark_intr_disable(ar, ce_ctrl_addr: ctrl_addr);
1319}
1320EXPORT_SYMBOL(ath10k_ce_disable_interrupt);
1321
1322void ath10k_ce_disable_interrupts(struct ath10k *ar)
1323{
1324 int ce_id;
1325
1326 for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1327 ath10k_ce_disable_interrupt(ar, ce_id);
1328}
1329EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1330
1331void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id)
1332{
1333 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1334 struct ath10k_ce_pipe *ce_state;
1335
1336 ce_state = &ce->ce_states[ce_id];
1337 if (ce_state->attr_flags & CE_ATTR_POLL)
1338 return;
1339
1340 ath10k_ce_per_engine_handler_adjust(ce_state);
1341}
1342EXPORT_SYMBOL(ath10k_ce_enable_interrupt);
1343
1344void ath10k_ce_enable_interrupts(struct ath10k *ar)
1345{
1346 int ce_id;
1347
1348 /* Enable interrupts for copy engine that
1349 * are not using polling mode.
1350 */
1351 for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1352 ath10k_ce_enable_interrupt(ar, ce_id);
1353}
1354EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1355
1356static int ath10k_ce_init_src_ring(struct ath10k *ar,
1357 unsigned int ce_id,
1358 const struct ce_attr *attr)
1359{
1360 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1361 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1362 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1363 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1364
1365 nentries = roundup_pow_of_two(attr->src_nentries);
1366
1367 if (ar->hw_params.target_64bit)
1368 memset(src_ring->base_addr_owner_space, 0,
1369 nentries * sizeof(struct ce_desc_64));
1370 else
1371 memset(src_ring->base_addr_owner_space, 0,
1372 nentries * sizeof(struct ce_desc));
1373
1374 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ce_ctrl_addr: ctrl_addr);
1375 src_ring->sw_index &= src_ring->nentries_mask;
1376 src_ring->hw_index = src_ring->sw_index;
1377
1378 src_ring->write_index =
1379 ath10k_ce_src_ring_write_index_get(ar, ce_ctrl_addr: ctrl_addr);
1380 src_ring->write_index &= src_ring->nentries_mask;
1381
1382 ath10k_ce_src_ring_base_addr_set(ar, ce_id,
1383 addr: src_ring->base_addr_ce_space);
1384 ath10k_ce_src_ring_size_set(ar, ce_ctrl_addr: ctrl_addr, n: nentries);
1385 ath10k_ce_src_ring_dmax_set(ar, ce_ctrl_addr: ctrl_addr, n: attr->src_sz_max);
1386 ath10k_ce_src_ring_byte_swap_set(ar, ce_ctrl_addr: ctrl_addr, n: 0);
1387 ath10k_ce_src_ring_lowmark_set(ar, ce_ctrl_addr: ctrl_addr, n: 0);
1388 ath10k_ce_src_ring_highmark_set(ar, ce_ctrl_addr: ctrl_addr, n: nentries);
1389
1390 ath10k_dbg(ar, ATH10K_DBG_BOOT,
1391 "boot init ce src ring id %d entries %d base_addr %pK\n",
1392 ce_id, nentries, src_ring->base_addr_owner_space);
1393
1394 return 0;
1395}
1396
1397static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1398 unsigned int ce_id,
1399 const struct ce_attr *attr)
1400{
1401 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1402 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1403 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1404 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1405
1406 nentries = roundup_pow_of_two(attr->dest_nentries);
1407
1408 if (ar->hw_params.target_64bit)
1409 memset(dest_ring->base_addr_owner_space, 0,
1410 nentries * sizeof(struct ce_desc_64));
1411 else
1412 memset(dest_ring->base_addr_owner_space, 0,
1413 nentries * sizeof(struct ce_desc));
1414
1415 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ce_ctrl_addr: ctrl_addr);
1416 dest_ring->sw_index &= dest_ring->nentries_mask;
1417 dest_ring->write_index =
1418 ath10k_ce_dest_ring_write_index_get(ar, ce_ctrl_addr: ctrl_addr);
1419 dest_ring->write_index &= dest_ring->nentries_mask;
1420
1421 ath10k_ce_dest_ring_base_addr_set(ar, ce_id,
1422 addr: dest_ring->base_addr_ce_space);
1423 ath10k_ce_dest_ring_size_set(ar, ce_ctrl_addr: ctrl_addr, n: nentries);
1424 ath10k_ce_dest_ring_byte_swap_set(ar, ce_ctrl_addr: ctrl_addr, n: 0);
1425 ath10k_ce_dest_ring_lowmark_set(ar, ce_ctrl_addr: ctrl_addr, n: 0);
1426 ath10k_ce_dest_ring_highmark_set(ar, ce_ctrl_addr: ctrl_addr, n: nentries);
1427
1428 ath10k_dbg(ar, ATH10K_DBG_BOOT,
1429 "boot ce dest ring id %d entries %d base_addr %pK\n",
1430 ce_id, nentries, dest_ring->base_addr_owner_space);
1431
1432 return 0;
1433}
1434
1435static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1436 struct ath10k_ce_ring *src_ring,
1437 u32 nentries)
1438{
1439 src_ring->shadow_base_unaligned = kcalloc(n: nentries,
1440 size: sizeof(struct ce_desc_64),
1441 GFP_KERNEL);
1442 if (!src_ring->shadow_base_unaligned)
1443 return -ENOMEM;
1444
1445 src_ring->shadow_base = (struct ce_desc_64 *)
1446 PTR_ALIGN(src_ring->shadow_base_unaligned,
1447 CE_DESC_RING_ALIGN);
1448 return 0;
1449}
1450
1451static struct ath10k_ce_ring *
1452ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1453 const struct ce_attr *attr)
1454{
1455 struct ath10k_ce_ring *src_ring;
1456 u32 nentries = attr->src_nentries;
1457 dma_addr_t base_addr;
1458 int ret;
1459
1460 nentries = roundup_pow_of_two(nentries);
1461
1462 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1463 nentries), GFP_KERNEL);
1464 if (src_ring == NULL)
1465 return ERR_PTR(error: -ENOMEM);
1466
1467 src_ring->nentries = nentries;
1468 src_ring->nentries_mask = nentries - 1;
1469
1470 /*
1471 * Legacy platforms that do not support cache
1472 * coherent DMA are unsupported
1473 */
1474 src_ring->base_addr_owner_space_unaligned =
1475 dma_alloc_coherent(dev: ar->dev,
1476 size: (nentries * sizeof(struct ce_desc) +
1477 CE_DESC_RING_ALIGN),
1478 dma_handle: &base_addr, GFP_KERNEL);
1479 if (!src_ring->base_addr_owner_space_unaligned) {
1480 kfree(objp: src_ring);
1481 return ERR_PTR(error: -ENOMEM);
1482 }
1483
1484 src_ring->base_addr_ce_space_unaligned = base_addr;
1485
1486 src_ring->base_addr_owner_space =
1487 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1488 CE_DESC_RING_ALIGN);
1489 src_ring->base_addr_ce_space =
1490 ALIGN(src_ring->base_addr_ce_space_unaligned,
1491 CE_DESC_RING_ALIGN);
1492
1493 if (ar->hw_params.shadow_reg_support) {
1494 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1495 if (ret) {
1496 dma_free_coherent(dev: ar->dev,
1497 size: (nentries * sizeof(struct ce_desc) +
1498 CE_DESC_RING_ALIGN),
1499 cpu_addr: src_ring->base_addr_owner_space_unaligned,
1500 dma_handle: base_addr);
1501 kfree(objp: src_ring);
1502 return ERR_PTR(error: ret);
1503 }
1504 }
1505
1506 return src_ring;
1507}
1508
1509static struct ath10k_ce_ring *
1510ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1511 const struct ce_attr *attr)
1512{
1513 struct ath10k_ce_ring *src_ring;
1514 u32 nentries = attr->src_nentries;
1515 dma_addr_t base_addr;
1516 int ret;
1517
1518 nentries = roundup_pow_of_two(nentries);
1519
1520 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1521 nentries), GFP_KERNEL);
1522 if (!src_ring)
1523 return ERR_PTR(error: -ENOMEM);
1524
1525 src_ring->nentries = nentries;
1526 src_ring->nentries_mask = nentries - 1;
1527
1528 /* Legacy platforms that do not support cache
1529 * coherent DMA are unsupported
1530 */
1531 src_ring->base_addr_owner_space_unaligned =
1532 dma_alloc_coherent(dev: ar->dev,
1533 size: (nentries * sizeof(struct ce_desc_64) +
1534 CE_DESC_RING_ALIGN),
1535 dma_handle: &base_addr, GFP_KERNEL);
1536 if (!src_ring->base_addr_owner_space_unaligned) {
1537 kfree(objp: src_ring);
1538 return ERR_PTR(error: -ENOMEM);
1539 }
1540
1541 src_ring->base_addr_ce_space_unaligned = base_addr;
1542
1543 src_ring->base_addr_owner_space =
1544 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1545 CE_DESC_RING_ALIGN);
1546 src_ring->base_addr_ce_space =
1547 ALIGN(src_ring->base_addr_ce_space_unaligned,
1548 CE_DESC_RING_ALIGN);
1549
1550 if (ar->hw_params.shadow_reg_support) {
1551 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1552 if (ret) {
1553 dma_free_coherent(dev: ar->dev,
1554 size: (nentries * sizeof(struct ce_desc_64) +
1555 CE_DESC_RING_ALIGN),
1556 cpu_addr: src_ring->base_addr_owner_space_unaligned,
1557 dma_handle: base_addr);
1558 kfree(objp: src_ring);
1559 return ERR_PTR(error: ret);
1560 }
1561 }
1562
1563 return src_ring;
1564}
1565
1566static struct ath10k_ce_ring *
1567ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1568 const struct ce_attr *attr)
1569{
1570 struct ath10k_ce_ring *dest_ring;
1571 u32 nentries;
1572 dma_addr_t base_addr;
1573
1574 nentries = roundup_pow_of_two(attr->dest_nentries);
1575
1576 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1577 nentries), GFP_KERNEL);
1578 if (dest_ring == NULL)
1579 return ERR_PTR(error: -ENOMEM);
1580
1581 dest_ring->nentries = nentries;
1582 dest_ring->nentries_mask = nentries - 1;
1583
1584 /*
1585 * Legacy platforms that do not support cache
1586 * coherent DMA are unsupported
1587 */
1588 dest_ring->base_addr_owner_space_unaligned =
1589 dma_alloc_coherent(dev: ar->dev,
1590 size: (nentries * sizeof(struct ce_desc) +
1591 CE_DESC_RING_ALIGN),
1592 dma_handle: &base_addr, GFP_KERNEL);
1593 if (!dest_ring->base_addr_owner_space_unaligned) {
1594 kfree(objp: dest_ring);
1595 return ERR_PTR(error: -ENOMEM);
1596 }
1597
1598 dest_ring->base_addr_ce_space_unaligned = base_addr;
1599
1600 dest_ring->base_addr_owner_space =
1601 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1602 CE_DESC_RING_ALIGN);
1603 dest_ring->base_addr_ce_space =
1604 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1605 CE_DESC_RING_ALIGN);
1606
1607 return dest_ring;
1608}
1609
1610static struct ath10k_ce_ring *
1611ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1612 const struct ce_attr *attr)
1613{
1614 struct ath10k_ce_ring *dest_ring;
1615 u32 nentries;
1616 dma_addr_t base_addr;
1617
1618 nentries = roundup_pow_of_two(attr->dest_nentries);
1619
1620 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1621 nentries), GFP_KERNEL);
1622 if (!dest_ring)
1623 return ERR_PTR(error: -ENOMEM);
1624
1625 dest_ring->nentries = nentries;
1626 dest_ring->nentries_mask = nentries - 1;
1627
1628 /* Legacy platforms that do not support cache
1629 * coherent DMA are unsupported
1630 */
1631 dest_ring->base_addr_owner_space_unaligned =
1632 dma_alloc_coherent(dev: ar->dev,
1633 size: (nentries * sizeof(struct ce_desc_64) +
1634 CE_DESC_RING_ALIGN),
1635 dma_handle: &base_addr, GFP_KERNEL);
1636 if (!dest_ring->base_addr_owner_space_unaligned) {
1637 kfree(objp: dest_ring);
1638 return ERR_PTR(error: -ENOMEM);
1639 }
1640
1641 dest_ring->base_addr_ce_space_unaligned = base_addr;
1642
1643 /* Correctly initialize memory to 0 to prevent garbage
1644 * data crashing system when download firmware
1645 */
1646 dest_ring->base_addr_owner_space =
1647 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1648 CE_DESC_RING_ALIGN);
1649 dest_ring->base_addr_ce_space =
1650 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1651 CE_DESC_RING_ALIGN);
1652
1653 return dest_ring;
1654}
1655
1656/*
1657 * Initialize a Copy Engine based on caller-supplied attributes.
1658 * This may be called once to initialize both source and destination
1659 * rings or it may be called twice for separate source and destination
1660 * initialization. It may be that only one side or the other is
1661 * initialized by software/firmware.
1662 */
1663int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1664 const struct ce_attr *attr)
1665{
1666 int ret;
1667
1668 if (attr->src_nentries) {
1669 ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1670 if (ret) {
1671 ath10k_err(ar, fmt: "Failed to initialize CE src ring for ID: %d (%d)\n",
1672 ce_id, ret);
1673 return ret;
1674 }
1675 }
1676
1677 if (attr->dest_nentries) {
1678 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1679 if (ret) {
1680 ath10k_err(ar, fmt: "Failed to initialize CE dest ring for ID: %d (%d)\n",
1681 ce_id, ret);
1682 return ret;
1683 }
1684 }
1685
1686 return 0;
1687}
1688EXPORT_SYMBOL(ath10k_ce_init_pipe);
1689
1690static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1691{
1692 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1693
1694 ath10k_ce_src_ring_base_addr_set(ar, ce_id, addr: 0);
1695 ath10k_ce_src_ring_size_set(ar, ce_ctrl_addr: ctrl_addr, n: 0);
1696 ath10k_ce_src_ring_dmax_set(ar, ce_ctrl_addr: ctrl_addr, n: 0);
1697 ath10k_ce_src_ring_highmark_set(ar, ce_ctrl_addr: ctrl_addr, n: 0);
1698}
1699
1700static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1701{
1702 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1703
1704 ath10k_ce_dest_ring_base_addr_set(ar, ce_id, addr: 0);
1705 ath10k_ce_dest_ring_size_set(ar, ce_ctrl_addr: ctrl_addr, n: 0);
1706 ath10k_ce_dest_ring_highmark_set(ar, ce_ctrl_addr: ctrl_addr, n: 0);
1707}
1708
1709void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1710{
1711 ath10k_ce_deinit_src_ring(ar, ce_id);
1712 ath10k_ce_deinit_dest_ring(ar, ce_id);
1713}
1714EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1715
1716static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1717{
1718 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1719 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1720
1721 if (ce_state->src_ring) {
1722 if (ar->hw_params.shadow_reg_support)
1723 kfree(objp: ce_state->src_ring->shadow_base_unaligned);
1724 dma_free_coherent(dev: ar->dev,
1725 size: (ce_state->src_ring->nentries *
1726 sizeof(struct ce_desc) +
1727 CE_DESC_RING_ALIGN),
1728 cpu_addr: ce_state->src_ring->base_addr_owner_space,
1729 dma_handle: ce_state->src_ring->base_addr_ce_space);
1730 kfree(objp: ce_state->src_ring);
1731 }
1732
1733 if (ce_state->dest_ring) {
1734 dma_free_coherent(dev: ar->dev,
1735 size: (ce_state->dest_ring->nentries *
1736 sizeof(struct ce_desc) +
1737 CE_DESC_RING_ALIGN),
1738 cpu_addr: ce_state->dest_ring->base_addr_owner_space,
1739 dma_handle: ce_state->dest_ring->base_addr_ce_space);
1740 kfree(objp: ce_state->dest_ring);
1741 }
1742
1743 ce_state->src_ring = NULL;
1744 ce_state->dest_ring = NULL;
1745}
1746
1747static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1748{
1749 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1750 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1751
1752 if (ce_state->src_ring) {
1753 if (ar->hw_params.shadow_reg_support)
1754 kfree(objp: ce_state->src_ring->shadow_base_unaligned);
1755 dma_free_coherent(dev: ar->dev,
1756 size: (ce_state->src_ring->nentries *
1757 sizeof(struct ce_desc_64) +
1758 CE_DESC_RING_ALIGN),
1759 cpu_addr: ce_state->src_ring->base_addr_owner_space,
1760 dma_handle: ce_state->src_ring->base_addr_ce_space);
1761 kfree(objp: ce_state->src_ring);
1762 }
1763
1764 if (ce_state->dest_ring) {
1765 dma_free_coherent(dev: ar->dev,
1766 size: (ce_state->dest_ring->nentries *
1767 sizeof(struct ce_desc_64) +
1768 CE_DESC_RING_ALIGN),
1769 cpu_addr: ce_state->dest_ring->base_addr_owner_space,
1770 dma_handle: ce_state->dest_ring->base_addr_ce_space);
1771 kfree(objp: ce_state->dest_ring);
1772 }
1773
1774 ce_state->src_ring = NULL;
1775 ce_state->dest_ring = NULL;
1776}
1777
1778void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1779{
1780 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1781 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1782
1783 ce_state->ops->ce_free_pipe(ar, ce_id);
1784}
1785EXPORT_SYMBOL(ath10k_ce_free_pipe);
1786
1787void ath10k_ce_dump_registers(struct ath10k *ar,
1788 struct ath10k_fw_crash_data *crash_data)
1789{
1790 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1791 struct ath10k_ce_crash_data ce_data;
1792 u32 addr, id;
1793
1794 lockdep_assert_held(&ar->dump_mutex);
1795
1796 ath10k_err(ar, fmt: "Copy Engine register dump:\n");
1797
1798 spin_lock_bh(lock: &ce->ce_lock);
1799 for (id = 0; id < CE_COUNT; id++) {
1800 addr = ath10k_ce_base_address(ar, ce_id: id);
1801 ce_data.base_addr = cpu_to_le32(addr);
1802
1803 ce_data.src_wr_idx =
1804 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1805 ce_data.src_r_idx =
1806 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1807 ce_data.dst_wr_idx =
1808 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1809 ce_data.dst_r_idx =
1810 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1811
1812 if (crash_data)
1813 crash_data->ce_crash_data[id] = ce_data;
1814
1815 ath10k_err(ar, fmt: "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1816 le32_to_cpu(ce_data.base_addr),
1817 le32_to_cpu(ce_data.src_wr_idx),
1818 le32_to_cpu(ce_data.src_r_idx),
1819 le32_to_cpu(ce_data.dst_wr_idx),
1820 le32_to_cpu(ce_data.dst_r_idx));
1821 }
1822
1823 spin_unlock_bh(lock: &ce->ce_lock);
1824}
1825EXPORT_SYMBOL(ath10k_ce_dump_registers);
1826
1827static const struct ath10k_ce_ops ce_ops = {
1828 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1829 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1830 .ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1831 .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1832 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1833 .ce_extract_desc_data = ath10k_ce_extract_desc_data,
1834 .ce_free_pipe = _ath10k_ce_free_pipe,
1835 .ce_send_nolock = _ath10k_ce_send_nolock,
1836 .ce_set_src_ring_base_addr_hi = NULL,
1837 .ce_set_dest_ring_base_addr_hi = NULL,
1838 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock,
1839};
1840
1841static const struct ath10k_ce_ops ce_64_ops = {
1842 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1843 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1844 .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1845 .ce_completed_recv_next_nolock =
1846 _ath10k_ce_completed_recv_next_nolock_64,
1847 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1848 .ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1849 .ce_free_pipe = _ath10k_ce_free_pipe_64,
1850 .ce_send_nolock = _ath10k_ce_send_nolock_64,
1851 .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi,
1852 .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi,
1853 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64,
1854};
1855
1856static void ath10k_ce_set_ops(struct ath10k *ar,
1857 struct ath10k_ce_pipe *ce_state)
1858{
1859 switch (ar->hw_rev) {
1860 case ATH10K_HW_WCN3990:
1861 ce_state->ops = &ce_64_ops;
1862 break;
1863 default:
1864 ce_state->ops = &ce_ops;
1865 break;
1866 }
1867}
1868
1869int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1870 const struct ce_attr *attr)
1871{
1872 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1873 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1874 int ret;
1875
1876 ath10k_ce_set_ops(ar, ce_state);
1877 /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1878 * additional TX locking checks.
1879 *
1880 * For the lack of a better place do the check here.
1881 */
1882 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1883 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1884 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1885 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1886 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1887 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1888
1889 ce_state->ar = ar;
1890 ce_state->id = ce_id;
1891 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1892 ce_state->attr_flags = attr->flags;
1893 ce_state->src_sz_max = attr->src_sz_max;
1894
1895 if (attr->src_nentries)
1896 ce_state->send_cb = attr->send_cb;
1897
1898 if (attr->dest_nentries)
1899 ce_state->recv_cb = attr->recv_cb;
1900
1901 if (attr->src_nentries) {
1902 ce_state->src_ring =
1903 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1904 if (IS_ERR(ptr: ce_state->src_ring)) {
1905 ret = PTR_ERR(ptr: ce_state->src_ring);
1906 ath10k_err(ar, fmt: "failed to alloc CE src ring %d: %d\n",
1907 ce_id, ret);
1908 ce_state->src_ring = NULL;
1909 return ret;
1910 }
1911 }
1912
1913 if (attr->dest_nentries) {
1914 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1915 ce_id,
1916 attr);
1917 if (IS_ERR(ptr: ce_state->dest_ring)) {
1918 ret = PTR_ERR(ptr: ce_state->dest_ring);
1919 ath10k_err(ar, fmt: "failed to alloc CE dest ring %d: %d\n",
1920 ce_id, ret);
1921 ce_state->dest_ring = NULL;
1922 return ret;
1923 }
1924 }
1925
1926 return 0;
1927}
1928EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1929
1930void ath10k_ce_alloc_rri(struct ath10k *ar)
1931{
1932 int i;
1933 u32 value;
1934 u32 ctrl1_regs;
1935 u32 ce_base_addr;
1936 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1937
1938 ce->vaddr_rri = dma_alloc_coherent(dev: ar->dev,
1939 size: (CE_COUNT * sizeof(u32)),
1940 dma_handle: &ce->paddr_rri, GFP_KERNEL);
1941
1942 if (!ce->vaddr_rri)
1943 return;
1944
1945 ath10k_ce_write32(ar, offset: ar->hw_ce_regs->ce_rri_low,
1946 lower_32_bits(ce->paddr_rri));
1947 ath10k_ce_write32(ar, offset: ar->hw_ce_regs->ce_rri_high,
1948 value: (upper_32_bits(ce->paddr_rri) &
1949 CE_DESC_ADDR_HI_MASK));
1950
1951 for (i = 0; i < CE_COUNT; i++) {
1952 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
1953 ce_base_addr = ath10k_ce_base_address(ar, ce_id: i);
1954 value = ath10k_ce_read32(ar, offset: ce_base_addr + ctrl1_regs);
1955 value |= ar->hw_ce_regs->upd->mask;
1956 ath10k_ce_write32(ar, offset: ce_base_addr + ctrl1_regs, value);
1957 }
1958}
1959EXPORT_SYMBOL(ath10k_ce_alloc_rri);
1960
1961void ath10k_ce_free_rri(struct ath10k *ar)
1962{
1963 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1964
1965 dma_free_coherent(dev: ar->dev, size: (CE_COUNT * sizeof(u32)),
1966 cpu_addr: ce->vaddr_rri,
1967 dma_handle: ce->paddr_rri);
1968}
1969EXPORT_SYMBOL(ath10k_ce_free_rri);
1970

source code of linux/drivers/net/wireless/ath/ath10k/ce.c