1 | // SPDX-License-Identifier: ISC |
2 | /* |
3 | * Copyright (c) 2005-2011 Atheros Communications Inc. |
4 | * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. |
5 | * Copyright (c) 2018 The Linux Foundation. All rights reserved. |
6 | * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. |
7 | */ |
8 | |
9 | #include "hif.h" |
10 | #include "ce.h" |
11 | #include "debug.h" |
12 | |
13 | /* |
14 | * Support for Copy Engine hardware, which is mainly used for |
15 | * communication between Host and Target over a PCIe interconnect. |
16 | */ |
17 | |
18 | /* |
19 | * A single CopyEngine (CE) comprises two "rings": |
20 | * a source ring |
21 | * a destination ring |
22 | * |
23 | * Each ring consists of a number of descriptors which specify |
24 | * an address, length, and meta-data. |
25 | * |
26 | * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target) |
27 | * controls one ring and the other side controls the other ring. |
28 | * The source side chooses when to initiate a transfer and it |
29 | * chooses what to send (buffer address, length). The destination |
30 | * side keeps a supply of "anonymous receive buffers" available and |
31 | * it handles incoming data as it arrives (when the destination |
32 | * receives an interrupt). |
33 | * |
34 | * The sender may send a simple buffer (address/length) or it may |
35 | * send a small list of buffers. When a small list is sent, hardware |
36 | * "gathers" these and they end up in a single destination buffer |
37 | * with a single interrupt. |
38 | * |
39 | * There are several "contexts" managed by this layer -- more, it |
40 | * may seem -- than should be needed. These are provided mainly for |
41 | * maximum flexibility and especially to facilitate a simpler HIF |
42 | * implementation. There are per-CopyEngine recv, send, and watermark |
43 | * contexts. These are supplied by the caller when a recv, send, |
44 | * or watermark handler is established and they are echoed back to |
45 | * the caller when the respective callbacks are invoked. There is |
46 | * also a per-transfer context supplied by the caller when a buffer |
47 | * (or sendlist) is sent and when a buffer is enqueued for recv. |
48 | * These per-transfer contexts are echoed back to the caller when |
49 | * the buffer is sent/received. |
50 | */ |
51 | |
52 | static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar, |
53 | struct ath10k_ce_pipe *ce_state) |
54 | { |
55 | u32 ce_id = ce_state->id; |
56 | u32 addr = 0; |
57 | |
58 | switch (ce_id) { |
59 | case 0: |
60 | addr = 0x00032000; |
61 | break; |
62 | case 3: |
63 | addr = 0x0003200C; |
64 | break; |
65 | case 4: |
66 | addr = 0x00032010; |
67 | break; |
68 | case 5: |
69 | addr = 0x00032014; |
70 | break; |
71 | case 7: |
72 | addr = 0x0003201C; |
73 | break; |
74 | default: |
75 | ath10k_warn(ar, fmt: "invalid CE id: %d" , ce_id); |
76 | break; |
77 | } |
78 | return addr; |
79 | } |
80 | |
81 | static inline unsigned int |
82 | ath10k_set_ring_byte(unsigned int offset, |
83 | struct ath10k_hw_ce_regs_addr_map *addr_map) |
84 | { |
85 | return ((offset << addr_map->lsb) & addr_map->mask); |
86 | } |
87 | |
88 | static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset) |
89 | { |
90 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
91 | |
92 | return ce->bus_ops->read32(ar, offset); |
93 | } |
94 | |
95 | static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value) |
96 | { |
97 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
98 | |
99 | ce->bus_ops->write32(ar, offset, value); |
100 | } |
101 | |
102 | static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, |
103 | u32 ce_ctrl_addr, |
104 | unsigned int n) |
105 | { |
106 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + |
107 | ar->hw_ce_regs->dst_wr_index_addr, value: n); |
108 | } |
109 | |
110 | static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, |
111 | u32 ce_ctrl_addr) |
112 | { |
113 | return ath10k_ce_read32(ar, offset: ce_ctrl_addr + |
114 | ar->hw_ce_regs->dst_wr_index_addr); |
115 | } |
116 | |
117 | static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, |
118 | u32 ce_ctrl_addr, |
119 | unsigned int n) |
120 | { |
121 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + |
122 | ar->hw_ce_regs->sr_wr_index_addr, value: n); |
123 | } |
124 | |
125 | static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, |
126 | u32 ce_ctrl_addr) |
127 | { |
128 | return ath10k_ce_read32(ar, offset: ce_ctrl_addr + |
129 | ar->hw_ce_regs->sr_wr_index_addr); |
130 | } |
131 | |
132 | static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar, |
133 | u32 ce_id) |
134 | { |
135 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
136 | |
137 | return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK; |
138 | } |
139 | |
140 | static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, |
141 | u32 ce_ctrl_addr) |
142 | { |
143 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
144 | u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr); |
145 | struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; |
146 | u32 index; |
147 | |
148 | if (ar->hw_params.rri_on_ddr && |
149 | (ce_state->attr_flags & CE_ATTR_DIS_INTR)) |
150 | index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id); |
151 | else |
152 | index = ath10k_ce_read32(ar, offset: ce_ctrl_addr + |
153 | ar->hw_ce_regs->current_srri_addr); |
154 | |
155 | return index; |
156 | } |
157 | |
158 | static inline void |
159 | ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar, |
160 | struct ath10k_ce_pipe *ce_state, |
161 | unsigned int value) |
162 | { |
163 | ath10k_ce_write32(ar, offset: shadow_sr_wr_ind_addr(ar, ce_state), value); |
164 | } |
165 | |
166 | static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, |
167 | u32 ce_id, |
168 | u64 addr) |
169 | { |
170 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
171 | struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; |
172 | u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id); |
173 | u32 addr_lo = lower_32_bits(addr); |
174 | |
175 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + |
176 | ar->hw_ce_regs->sr_base_addr_lo, value: addr_lo); |
177 | |
178 | if (ce_state->ops->ce_set_src_ring_base_addr_hi) { |
179 | ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr, |
180 | addr); |
181 | } |
182 | } |
183 | |
184 | static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar, |
185 | u32 ce_ctrl_addr, |
186 | u64 addr) |
187 | { |
188 | u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK; |
189 | |
190 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + |
191 | ar->hw_ce_regs->sr_base_addr_hi, value: addr_hi); |
192 | } |
193 | |
194 | static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, |
195 | u32 ce_ctrl_addr, |
196 | unsigned int n) |
197 | { |
198 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + |
199 | ar->hw_ce_regs->sr_size_addr, value: n); |
200 | } |
201 | |
202 | static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, |
203 | u32 ce_ctrl_addr, |
204 | unsigned int n) |
205 | { |
206 | struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; |
207 | |
208 | u32 ctrl1_addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + |
209 | ctrl_regs->addr); |
210 | |
211 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + ctrl_regs->addr, |
212 | value: (ctrl1_addr & ~(ctrl_regs->dmax->mask)) | |
213 | ath10k_set_ring_byte(offset: n, addr_map: ctrl_regs->dmax)); |
214 | } |
215 | |
216 | static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, |
217 | u32 ce_ctrl_addr, |
218 | unsigned int n) |
219 | { |
220 | struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; |
221 | |
222 | u32 ctrl1_addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + |
223 | ctrl_regs->addr); |
224 | |
225 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + ctrl_regs->addr, |
226 | value: (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) | |
227 | ath10k_set_ring_byte(offset: n, addr_map: ctrl_regs->src_ring)); |
228 | } |
229 | |
230 | static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, |
231 | u32 ce_ctrl_addr, |
232 | unsigned int n) |
233 | { |
234 | struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; |
235 | |
236 | u32 ctrl1_addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + |
237 | ctrl_regs->addr); |
238 | |
239 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + ctrl_regs->addr, |
240 | value: (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) | |
241 | ath10k_set_ring_byte(offset: n, addr_map: ctrl_regs->dst_ring)); |
242 | } |
243 | |
244 | static inline |
245 | u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id) |
246 | { |
247 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
248 | |
249 | return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) & |
250 | CE_DDR_RRI_MASK; |
251 | } |
252 | |
253 | static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, |
254 | u32 ce_ctrl_addr) |
255 | { |
256 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
257 | u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr); |
258 | struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; |
259 | u32 index; |
260 | |
261 | if (ar->hw_params.rri_on_ddr && |
262 | (ce_state->attr_flags & CE_ATTR_DIS_INTR)) |
263 | index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id); |
264 | else |
265 | index = ath10k_ce_read32(ar, offset: ce_ctrl_addr + |
266 | ar->hw_ce_regs->current_drri_addr); |
267 | |
268 | return index; |
269 | } |
270 | |
271 | static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, |
272 | u32 ce_id, |
273 | u64 addr) |
274 | { |
275 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
276 | struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; |
277 | u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id); |
278 | u32 addr_lo = lower_32_bits(addr); |
279 | |
280 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + |
281 | ar->hw_ce_regs->dr_base_addr_lo, value: addr_lo); |
282 | |
283 | if (ce_state->ops->ce_set_dest_ring_base_addr_hi) { |
284 | ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr, |
285 | addr); |
286 | } |
287 | } |
288 | |
289 | static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar, |
290 | u32 ce_ctrl_addr, |
291 | u64 addr) |
292 | { |
293 | u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK; |
294 | u32 reg_value; |
295 | |
296 | reg_value = ath10k_ce_read32(ar, offset: ce_ctrl_addr + |
297 | ar->hw_ce_regs->dr_base_addr_hi); |
298 | reg_value &= ~CE_DESC_ADDR_HI_MASK; |
299 | reg_value |= addr_hi; |
300 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + |
301 | ar->hw_ce_regs->dr_base_addr_hi, value: reg_value); |
302 | } |
303 | |
304 | static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, |
305 | u32 ce_ctrl_addr, |
306 | unsigned int n) |
307 | { |
308 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + |
309 | ar->hw_ce_regs->dr_size_addr, value: n); |
310 | } |
311 | |
312 | static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, |
313 | u32 ce_ctrl_addr, |
314 | unsigned int n) |
315 | { |
316 | struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; |
317 | u32 addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + srcr_wm->addr); |
318 | |
319 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + srcr_wm->addr, |
320 | value: (addr & ~(srcr_wm->wm_high->mask)) | |
321 | (ath10k_set_ring_byte(offset: n, addr_map: srcr_wm->wm_high))); |
322 | } |
323 | |
324 | static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, |
325 | u32 ce_ctrl_addr, |
326 | unsigned int n) |
327 | { |
328 | struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; |
329 | u32 addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + srcr_wm->addr); |
330 | |
331 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + srcr_wm->addr, |
332 | value: (addr & ~(srcr_wm->wm_low->mask)) | |
333 | (ath10k_set_ring_byte(offset: n, addr_map: srcr_wm->wm_low))); |
334 | } |
335 | |
336 | static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, |
337 | u32 ce_ctrl_addr, |
338 | unsigned int n) |
339 | { |
340 | struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; |
341 | u32 addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + dstr_wm->addr); |
342 | |
343 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + dstr_wm->addr, |
344 | value: (addr & ~(dstr_wm->wm_high->mask)) | |
345 | (ath10k_set_ring_byte(offset: n, addr_map: dstr_wm->wm_high))); |
346 | } |
347 | |
348 | static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, |
349 | u32 ce_ctrl_addr, |
350 | unsigned int n) |
351 | { |
352 | struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; |
353 | u32 addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + dstr_wm->addr); |
354 | |
355 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + dstr_wm->addr, |
356 | value: (addr & ~(dstr_wm->wm_low->mask)) | |
357 | (ath10k_set_ring_byte(offset: n, addr_map: dstr_wm->wm_low))); |
358 | } |
359 | |
360 | static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, |
361 | u32 ce_ctrl_addr) |
362 | { |
363 | struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; |
364 | |
365 | u32 host_ie_addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + |
366 | ar->hw_ce_regs->host_ie_addr); |
367 | |
368 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, |
369 | value: host_ie_addr | host_ie->copy_complete->mask); |
370 | } |
371 | |
372 | static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, |
373 | u32 ce_ctrl_addr) |
374 | { |
375 | struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; |
376 | |
377 | u32 host_ie_addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + |
378 | ar->hw_ce_regs->host_ie_addr); |
379 | |
380 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, |
381 | value: host_ie_addr & ~(host_ie->copy_complete->mask)); |
382 | } |
383 | |
384 | static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, |
385 | u32 ce_ctrl_addr) |
386 | { |
387 | struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; |
388 | |
389 | u32 host_ie_addr = ath10k_ce_read32(ar, offset: ce_ctrl_addr + |
390 | ar->hw_ce_regs->host_ie_addr); |
391 | |
392 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, |
393 | value: host_ie_addr & ~(wm_regs->wm_mask)); |
394 | } |
395 | |
396 | static inline void ath10k_ce_error_intr_disable(struct ath10k *ar, |
397 | u32 ce_ctrl_addr) |
398 | { |
399 | struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs; |
400 | |
401 | u32 misc_ie_addr = ath10k_ce_read32(ar, |
402 | offset: ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr); |
403 | |
404 | ath10k_ce_write32(ar, |
405 | offset: ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr, |
406 | value: misc_ie_addr & ~(misc_regs->err_mask)); |
407 | } |
408 | |
409 | static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, |
410 | u32 ce_ctrl_addr, |
411 | unsigned int mask) |
412 | { |
413 | struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; |
414 | |
415 | ath10k_ce_write32(ar, offset: ce_ctrl_addr + wm_regs->addr, value: mask); |
416 | } |
417 | |
418 | /* |
419 | * Guts of ath10k_ce_send. |
420 | * The caller takes responsibility for any needed locking. |
421 | */ |
422 | static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, |
423 | void *per_transfer_context, |
424 | dma_addr_t buffer, |
425 | unsigned int nbytes, |
426 | unsigned int transfer_id, |
427 | unsigned int flags) |
428 | { |
429 | struct ath10k *ar = ce_state->ar; |
430 | struct ath10k_ce_ring *src_ring = ce_state->src_ring; |
431 | struct ce_desc *desc, sdesc; |
432 | unsigned int nentries_mask = src_ring->nentries_mask; |
433 | unsigned int sw_index = src_ring->sw_index; |
434 | unsigned int write_index = src_ring->write_index; |
435 | u32 ctrl_addr = ce_state->ctrl_addr; |
436 | u32 desc_flags = 0; |
437 | int ret = 0; |
438 | |
439 | if (nbytes > ce_state->src_sz_max) |
440 | ath10k_warn(ar, fmt: "%s: send more we can (nbytes: %d, max: %d)\n" , |
441 | __func__, nbytes, ce_state->src_sz_max); |
442 | |
443 | if (unlikely(CE_RING_DELTA(nentries_mask, |
444 | write_index, sw_index - 1) <= 0)) { |
445 | ret = -ENOSR; |
446 | goto exit; |
447 | } |
448 | |
449 | desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, |
450 | write_index); |
451 | |
452 | desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); |
453 | |
454 | if (flags & CE_SEND_FLAG_GATHER) |
455 | desc_flags |= CE_DESC_FLAGS_GATHER; |
456 | if (flags & CE_SEND_FLAG_BYTE_SWAP) |
457 | desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; |
458 | |
459 | sdesc.addr = __cpu_to_le32(buffer); |
460 | sdesc.nbytes = __cpu_to_le16(nbytes); |
461 | sdesc.flags = __cpu_to_le16(desc_flags); |
462 | |
463 | *desc = sdesc; |
464 | |
465 | src_ring->per_transfer_context[write_index] = per_transfer_context; |
466 | |
467 | /* Update Source Ring Write Index */ |
468 | write_index = CE_RING_IDX_INCR(nentries_mask, write_index); |
469 | |
470 | /* WORKAROUND */ |
471 | if (!(flags & CE_SEND_FLAG_GATHER)) |
472 | ath10k_ce_src_ring_write_index_set(ar, ce_ctrl_addr: ctrl_addr, n: write_index); |
473 | |
474 | src_ring->write_index = write_index; |
475 | exit: |
476 | return ret; |
477 | } |
478 | |
479 | static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state, |
480 | void *per_transfer_context, |
481 | dma_addr_t buffer, |
482 | unsigned int nbytes, |
483 | unsigned int transfer_id, |
484 | unsigned int flags) |
485 | { |
486 | struct ath10k *ar = ce_state->ar; |
487 | struct ath10k_ce_ring *src_ring = ce_state->src_ring; |
488 | struct ce_desc_64 *desc, sdesc; |
489 | unsigned int nentries_mask = src_ring->nentries_mask; |
490 | unsigned int sw_index; |
491 | unsigned int write_index = src_ring->write_index; |
492 | u32 ctrl_addr = ce_state->ctrl_addr; |
493 | __le32 *addr; |
494 | u32 desc_flags = 0; |
495 | int ret = 0; |
496 | |
497 | if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags)) |
498 | return -ESHUTDOWN; |
499 | |
500 | if (nbytes > ce_state->src_sz_max) |
501 | ath10k_warn(ar, fmt: "%s: send more we can (nbytes: %d, max: %d)\n" , |
502 | __func__, nbytes, ce_state->src_sz_max); |
503 | |
504 | if (ar->hw_params.rri_on_ddr) |
505 | sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id: ce_state->id); |
506 | else |
507 | sw_index = src_ring->sw_index; |
508 | |
509 | if (unlikely(CE_RING_DELTA(nentries_mask, |
510 | write_index, sw_index - 1) <= 0)) { |
511 | ret = -ENOSR; |
512 | goto exit; |
513 | } |
514 | |
515 | desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space, |
516 | write_index); |
517 | |
518 | desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); |
519 | |
520 | if (flags & CE_SEND_FLAG_GATHER) |
521 | desc_flags |= CE_DESC_FLAGS_GATHER; |
522 | |
523 | if (flags & CE_SEND_FLAG_BYTE_SWAP) |
524 | desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; |
525 | |
526 | addr = (__le32 *)&sdesc.addr; |
527 | |
528 | flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK; |
529 | addr[0] = __cpu_to_le32(buffer); |
530 | addr[1] = __cpu_to_le32(flags); |
531 | if (flags & CE_SEND_FLAG_GATHER) |
532 | addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER); |
533 | else |
534 | addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER)); |
535 | |
536 | sdesc.nbytes = __cpu_to_le16(nbytes); |
537 | sdesc.flags = __cpu_to_le16(desc_flags); |
538 | |
539 | *desc = sdesc; |
540 | |
541 | src_ring->per_transfer_context[write_index] = per_transfer_context; |
542 | |
543 | /* Update Source Ring Write Index */ |
544 | write_index = CE_RING_IDX_INCR(nentries_mask, write_index); |
545 | |
546 | if (!(flags & CE_SEND_FLAG_GATHER)) { |
547 | if (ar->hw_params.shadow_reg_support) |
548 | ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state, |
549 | value: write_index); |
550 | else |
551 | ath10k_ce_src_ring_write_index_set(ar, ce_ctrl_addr: ctrl_addr, |
552 | n: write_index); |
553 | } |
554 | |
555 | src_ring->write_index = write_index; |
556 | exit: |
557 | return ret; |
558 | } |
559 | |
560 | int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, |
561 | void *per_transfer_context, |
562 | dma_addr_t buffer, |
563 | unsigned int nbytes, |
564 | unsigned int transfer_id, |
565 | unsigned int flags) |
566 | { |
567 | return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context, |
568 | buffer, nbytes, transfer_id, flags); |
569 | } |
570 | EXPORT_SYMBOL(ath10k_ce_send_nolock); |
571 | |
572 | void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe) |
573 | { |
574 | struct ath10k *ar = pipe->ar; |
575 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
576 | struct ath10k_ce_ring *src_ring = pipe->src_ring; |
577 | u32 ctrl_addr = pipe->ctrl_addr; |
578 | |
579 | lockdep_assert_held(&ce->ce_lock); |
580 | |
581 | /* |
582 | * This function must be called only if there is an incomplete |
583 | * scatter-gather transfer (before index register is updated) |
584 | * that needs to be cleaned up. |
585 | */ |
586 | if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index)) |
587 | return; |
588 | |
589 | if (WARN_ON_ONCE(src_ring->write_index == |
590 | ath10k_ce_src_ring_write_index_get(ar, ctrl_addr))) |
591 | return; |
592 | |
593 | src_ring->write_index--; |
594 | src_ring->write_index &= src_ring->nentries_mask; |
595 | |
596 | src_ring->per_transfer_context[src_ring->write_index] = NULL; |
597 | } |
598 | EXPORT_SYMBOL(__ath10k_ce_send_revert); |
599 | |
600 | int ath10k_ce_send(struct ath10k_ce_pipe *ce_state, |
601 | void *per_transfer_context, |
602 | dma_addr_t buffer, |
603 | unsigned int nbytes, |
604 | unsigned int transfer_id, |
605 | unsigned int flags) |
606 | { |
607 | struct ath10k *ar = ce_state->ar; |
608 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
609 | int ret; |
610 | |
611 | spin_lock_bh(lock: &ce->ce_lock); |
612 | ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, |
613 | buffer, nbytes, transfer_id, flags); |
614 | spin_unlock_bh(lock: &ce->ce_lock); |
615 | |
616 | return ret; |
617 | } |
618 | EXPORT_SYMBOL(ath10k_ce_send); |
619 | |
620 | int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe) |
621 | { |
622 | struct ath10k *ar = pipe->ar; |
623 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
624 | int delta; |
625 | |
626 | spin_lock_bh(lock: &ce->ce_lock); |
627 | delta = CE_RING_DELTA(pipe->src_ring->nentries_mask, |
628 | pipe->src_ring->write_index, |
629 | pipe->src_ring->sw_index - 1); |
630 | spin_unlock_bh(lock: &ce->ce_lock); |
631 | |
632 | return delta; |
633 | } |
634 | EXPORT_SYMBOL(ath10k_ce_num_free_src_entries); |
635 | |
636 | int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe) |
637 | { |
638 | struct ath10k *ar = pipe->ar; |
639 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
640 | struct ath10k_ce_ring *dest_ring = pipe->dest_ring; |
641 | unsigned int nentries_mask = dest_ring->nentries_mask; |
642 | unsigned int write_index = dest_ring->write_index; |
643 | unsigned int sw_index = dest_ring->sw_index; |
644 | |
645 | lockdep_assert_held(&ce->ce_lock); |
646 | |
647 | return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); |
648 | } |
649 | EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs); |
650 | |
651 | static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, |
652 | dma_addr_t paddr) |
653 | { |
654 | struct ath10k *ar = pipe->ar; |
655 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
656 | struct ath10k_ce_ring *dest_ring = pipe->dest_ring; |
657 | unsigned int nentries_mask = dest_ring->nentries_mask; |
658 | unsigned int write_index = dest_ring->write_index; |
659 | unsigned int sw_index = dest_ring->sw_index; |
660 | struct ce_desc *base = dest_ring->base_addr_owner_space; |
661 | struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); |
662 | u32 ctrl_addr = pipe->ctrl_addr; |
663 | |
664 | lockdep_assert_held(&ce->ce_lock); |
665 | |
666 | if ((pipe->id != 5) && |
667 | CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) |
668 | return -ENOSPC; |
669 | |
670 | desc->addr = __cpu_to_le32(paddr); |
671 | desc->nbytes = 0; |
672 | |
673 | dest_ring->per_transfer_context[write_index] = ctx; |
674 | write_index = CE_RING_IDX_INCR(nentries_mask, write_index); |
675 | ath10k_ce_dest_ring_write_index_set(ar, ce_ctrl_addr: ctrl_addr, n: write_index); |
676 | dest_ring->write_index = write_index; |
677 | |
678 | return 0; |
679 | } |
680 | |
681 | static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe, |
682 | void *ctx, |
683 | dma_addr_t paddr) |
684 | { |
685 | struct ath10k *ar = pipe->ar; |
686 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
687 | struct ath10k_ce_ring *dest_ring = pipe->dest_ring; |
688 | unsigned int nentries_mask = dest_ring->nentries_mask; |
689 | unsigned int write_index = dest_ring->write_index; |
690 | unsigned int sw_index = dest_ring->sw_index; |
691 | struct ce_desc_64 *base = dest_ring->base_addr_owner_space; |
692 | struct ce_desc_64 *desc = |
693 | CE_DEST_RING_TO_DESC_64(base, write_index); |
694 | u32 ctrl_addr = pipe->ctrl_addr; |
695 | |
696 | lockdep_assert_held(&ce->ce_lock); |
697 | |
698 | if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) |
699 | return -ENOSPC; |
700 | |
701 | desc->addr = __cpu_to_le64(paddr); |
702 | desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK); |
703 | |
704 | desc->nbytes = 0; |
705 | |
706 | dest_ring->per_transfer_context[write_index] = ctx; |
707 | write_index = CE_RING_IDX_INCR(nentries_mask, write_index); |
708 | ath10k_ce_dest_ring_write_index_set(ar, ce_ctrl_addr: ctrl_addr, n: write_index); |
709 | dest_ring->write_index = write_index; |
710 | |
711 | return 0; |
712 | } |
713 | |
714 | void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries) |
715 | { |
716 | struct ath10k *ar = pipe->ar; |
717 | struct ath10k_ce_ring *dest_ring = pipe->dest_ring; |
718 | unsigned int nentries_mask = dest_ring->nentries_mask; |
719 | unsigned int write_index = dest_ring->write_index; |
720 | u32 ctrl_addr = pipe->ctrl_addr; |
721 | u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ce_ctrl_addr: ctrl_addr); |
722 | |
723 | /* Prevent CE ring stuck issue that will occur when ring is full. |
724 | * Make sure that write index is 1 less than read index. |
725 | */ |
726 | if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index) |
727 | nentries -= 1; |
728 | |
729 | write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries); |
730 | ath10k_ce_dest_ring_write_index_set(ar, ce_ctrl_addr: ctrl_addr, n: write_index); |
731 | dest_ring->write_index = write_index; |
732 | } |
733 | EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx); |
734 | |
735 | int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, |
736 | dma_addr_t paddr) |
737 | { |
738 | struct ath10k *ar = pipe->ar; |
739 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
740 | int ret; |
741 | |
742 | spin_lock_bh(lock: &ce->ce_lock); |
743 | ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr); |
744 | spin_unlock_bh(lock: &ce->ce_lock); |
745 | |
746 | return ret; |
747 | } |
748 | EXPORT_SYMBOL(ath10k_ce_rx_post_buf); |
749 | |
750 | /* |
751 | * Guts of ath10k_ce_completed_recv_next. |
752 | * The caller takes responsibility for any necessary locking. |
753 | */ |
754 | static int |
755 | _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, |
756 | void **per_transfer_contextp, |
757 | unsigned int *nbytesp) |
758 | { |
759 | struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; |
760 | unsigned int nentries_mask = dest_ring->nentries_mask; |
761 | unsigned int sw_index = dest_ring->sw_index; |
762 | |
763 | struct ce_desc *base = dest_ring->base_addr_owner_space; |
764 | struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); |
765 | struct ce_desc sdesc; |
766 | u16 nbytes; |
767 | |
768 | /* Copy in one go for performance reasons */ |
769 | sdesc = *desc; |
770 | |
771 | nbytes = __le16_to_cpu(sdesc.nbytes); |
772 | if (nbytes == 0) { |
773 | /* |
774 | * This closes a relatively unusual race where the Host |
775 | * sees the updated DRRI before the update to the |
776 | * corresponding descriptor has completed. We treat this |
777 | * as a descriptor that is not yet done. |
778 | */ |
779 | return -EIO; |
780 | } |
781 | |
782 | desc->nbytes = 0; |
783 | |
784 | /* Return data from completed destination descriptor */ |
785 | *nbytesp = nbytes; |
786 | |
787 | if (per_transfer_contextp) |
788 | *per_transfer_contextp = |
789 | dest_ring->per_transfer_context[sw_index]; |
790 | |
791 | /* Copy engine 5 (HTT Rx) will reuse the same transfer context. |
792 | * So update transfer context all CEs except CE5. |
793 | */ |
794 | if (ce_state->id != 5) |
795 | dest_ring->per_transfer_context[sw_index] = NULL; |
796 | |
797 | /* Update sw_index */ |
798 | sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); |
799 | dest_ring->sw_index = sw_index; |
800 | |
801 | return 0; |
802 | } |
803 | |
804 | static int |
805 | _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state, |
806 | void **per_transfer_contextp, |
807 | unsigned int *nbytesp) |
808 | { |
809 | struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; |
810 | unsigned int nentries_mask = dest_ring->nentries_mask; |
811 | unsigned int sw_index = dest_ring->sw_index; |
812 | struct ce_desc_64 *base = dest_ring->base_addr_owner_space; |
813 | struct ce_desc_64 *desc = |
814 | CE_DEST_RING_TO_DESC_64(base, sw_index); |
815 | struct ce_desc_64 sdesc; |
816 | u16 nbytes; |
817 | |
818 | /* Copy in one go for performance reasons */ |
819 | sdesc = *desc; |
820 | |
821 | nbytes = __le16_to_cpu(sdesc.nbytes); |
822 | if (nbytes == 0) { |
823 | /* This closes a relatively unusual race where the Host |
824 | * sees the updated DRRI before the update to the |
825 | * corresponding descriptor has completed. We treat this |
826 | * as a descriptor that is not yet done. |
827 | */ |
828 | return -EIO; |
829 | } |
830 | |
831 | desc->nbytes = 0; |
832 | |
833 | /* Return data from completed destination descriptor */ |
834 | *nbytesp = nbytes; |
835 | |
836 | if (per_transfer_contextp) |
837 | *per_transfer_contextp = |
838 | dest_ring->per_transfer_context[sw_index]; |
839 | |
840 | /* Copy engine 5 (HTT Rx) will reuse the same transfer context. |
841 | * So update transfer context all CEs except CE5. |
842 | */ |
843 | if (ce_state->id != 5) |
844 | dest_ring->per_transfer_context[sw_index] = NULL; |
845 | |
846 | /* Update sw_index */ |
847 | sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); |
848 | dest_ring->sw_index = sw_index; |
849 | |
850 | return 0; |
851 | } |
852 | |
853 | int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, |
854 | void **per_transfer_ctx, |
855 | unsigned int *nbytesp) |
856 | { |
857 | return ce_state->ops->ce_completed_recv_next_nolock(ce_state, |
858 | per_transfer_ctx, |
859 | nbytesp); |
860 | } |
861 | EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock); |
862 | |
863 | int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state, |
864 | void **per_transfer_contextp, |
865 | unsigned int *nbytesp) |
866 | { |
867 | struct ath10k *ar = ce_state->ar; |
868 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
869 | int ret; |
870 | |
871 | spin_lock_bh(lock: &ce->ce_lock); |
872 | ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state, |
873 | per_transfer_contextp, |
874 | nbytesp); |
875 | |
876 | spin_unlock_bh(lock: &ce->ce_lock); |
877 | |
878 | return ret; |
879 | } |
880 | EXPORT_SYMBOL(ath10k_ce_completed_recv_next); |
881 | |
882 | static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, |
883 | void **per_transfer_contextp, |
884 | dma_addr_t *bufferp) |
885 | { |
886 | struct ath10k_ce_ring *dest_ring; |
887 | unsigned int nentries_mask; |
888 | unsigned int sw_index; |
889 | unsigned int write_index; |
890 | int ret; |
891 | struct ath10k *ar; |
892 | struct ath10k_ce *ce; |
893 | |
894 | dest_ring = ce_state->dest_ring; |
895 | |
896 | if (!dest_ring) |
897 | return -EIO; |
898 | |
899 | ar = ce_state->ar; |
900 | ce = ath10k_ce_priv(ar); |
901 | |
902 | spin_lock_bh(lock: &ce->ce_lock); |
903 | |
904 | nentries_mask = dest_ring->nentries_mask; |
905 | sw_index = dest_ring->sw_index; |
906 | write_index = dest_ring->write_index; |
907 | if (write_index != sw_index) { |
908 | struct ce_desc *base = dest_ring->base_addr_owner_space; |
909 | struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); |
910 | |
911 | /* Return data from completed destination descriptor */ |
912 | *bufferp = __le32_to_cpu(desc->addr); |
913 | |
914 | if (per_transfer_contextp) |
915 | *per_transfer_contextp = |
916 | dest_ring->per_transfer_context[sw_index]; |
917 | |
918 | /* sanity */ |
919 | dest_ring->per_transfer_context[sw_index] = NULL; |
920 | desc->nbytes = 0; |
921 | |
922 | /* Update sw_index */ |
923 | sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); |
924 | dest_ring->sw_index = sw_index; |
925 | ret = 0; |
926 | } else { |
927 | ret = -EIO; |
928 | } |
929 | |
930 | spin_unlock_bh(lock: &ce->ce_lock); |
931 | |
932 | return ret; |
933 | } |
934 | |
935 | static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state, |
936 | void **per_transfer_contextp, |
937 | dma_addr_t *bufferp) |
938 | { |
939 | struct ath10k_ce_ring *dest_ring; |
940 | unsigned int nentries_mask; |
941 | unsigned int sw_index; |
942 | unsigned int write_index; |
943 | int ret; |
944 | struct ath10k *ar; |
945 | struct ath10k_ce *ce; |
946 | |
947 | dest_ring = ce_state->dest_ring; |
948 | |
949 | if (!dest_ring) |
950 | return -EIO; |
951 | |
952 | ar = ce_state->ar; |
953 | ce = ath10k_ce_priv(ar); |
954 | |
955 | spin_lock_bh(lock: &ce->ce_lock); |
956 | |
957 | nentries_mask = dest_ring->nentries_mask; |
958 | sw_index = dest_ring->sw_index; |
959 | write_index = dest_ring->write_index; |
960 | if (write_index != sw_index) { |
961 | struct ce_desc_64 *base = dest_ring->base_addr_owner_space; |
962 | struct ce_desc_64 *desc = |
963 | CE_DEST_RING_TO_DESC_64(base, sw_index); |
964 | |
965 | /* Return data from completed destination descriptor */ |
966 | *bufferp = __le64_to_cpu(desc->addr); |
967 | |
968 | if (per_transfer_contextp) |
969 | *per_transfer_contextp = |
970 | dest_ring->per_transfer_context[sw_index]; |
971 | |
972 | /* sanity */ |
973 | dest_ring->per_transfer_context[sw_index] = NULL; |
974 | desc->nbytes = 0; |
975 | |
976 | /* Update sw_index */ |
977 | sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); |
978 | dest_ring->sw_index = sw_index; |
979 | ret = 0; |
980 | } else { |
981 | ret = -EIO; |
982 | } |
983 | |
984 | spin_unlock_bh(lock: &ce->ce_lock); |
985 | |
986 | return ret; |
987 | } |
988 | |
989 | int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, |
990 | void **per_transfer_contextp, |
991 | dma_addr_t *bufferp) |
992 | { |
993 | return ce_state->ops->ce_revoke_recv_next(ce_state, |
994 | per_transfer_contextp, |
995 | bufferp); |
996 | } |
997 | EXPORT_SYMBOL(ath10k_ce_revoke_recv_next); |
998 | |
999 | /* |
1000 | * Guts of ath10k_ce_completed_send_next. |
1001 | * The caller takes responsibility for any necessary locking. |
1002 | */ |
1003 | static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, |
1004 | void **per_transfer_contextp) |
1005 | { |
1006 | struct ath10k_ce_ring *src_ring = ce_state->src_ring; |
1007 | u32 ctrl_addr = ce_state->ctrl_addr; |
1008 | struct ath10k *ar = ce_state->ar; |
1009 | unsigned int nentries_mask = src_ring->nentries_mask; |
1010 | unsigned int sw_index = src_ring->sw_index; |
1011 | unsigned int read_index; |
1012 | struct ce_desc *desc; |
1013 | |
1014 | if (src_ring->hw_index == sw_index) { |
1015 | /* |
1016 | * The SW completion index has caught up with the cached |
1017 | * version of the HW completion index. |
1018 | * Update the cached HW completion index to see whether |
1019 | * the SW has really caught up to the HW, or if the cached |
1020 | * value of the HW index has become stale. |
1021 | */ |
1022 | |
1023 | read_index = ath10k_ce_src_ring_read_index_get(ar, ce_ctrl_addr: ctrl_addr); |
1024 | if (read_index == 0xffffffff) |
1025 | return -ENODEV; |
1026 | |
1027 | read_index &= nentries_mask; |
1028 | src_ring->hw_index = read_index; |
1029 | } |
1030 | |
1031 | if (ar->hw_params.rri_on_ddr) |
1032 | read_index = ath10k_ce_src_ring_read_index_get(ar, ce_ctrl_addr: ctrl_addr); |
1033 | else |
1034 | read_index = src_ring->hw_index; |
1035 | |
1036 | if (read_index == sw_index) |
1037 | return -EIO; |
1038 | |
1039 | if (per_transfer_contextp) |
1040 | *per_transfer_contextp = |
1041 | src_ring->per_transfer_context[sw_index]; |
1042 | |
1043 | /* sanity */ |
1044 | src_ring->per_transfer_context[sw_index] = NULL; |
1045 | desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, |
1046 | sw_index); |
1047 | desc->nbytes = 0; |
1048 | |
1049 | /* Update sw_index */ |
1050 | sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); |
1051 | src_ring->sw_index = sw_index; |
1052 | |
1053 | return 0; |
1054 | } |
1055 | |
1056 | static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state, |
1057 | void **per_transfer_contextp) |
1058 | { |
1059 | struct ath10k_ce_ring *src_ring = ce_state->src_ring; |
1060 | u32 ctrl_addr = ce_state->ctrl_addr; |
1061 | struct ath10k *ar = ce_state->ar; |
1062 | unsigned int nentries_mask = src_ring->nentries_mask; |
1063 | unsigned int sw_index = src_ring->sw_index; |
1064 | unsigned int read_index; |
1065 | struct ce_desc_64 *desc; |
1066 | |
1067 | if (src_ring->hw_index == sw_index) { |
1068 | /* |
1069 | * The SW completion index has caught up with the cached |
1070 | * version of the HW completion index. |
1071 | * Update the cached HW completion index to see whether |
1072 | * the SW has really caught up to the HW, or if the cached |
1073 | * value of the HW index has become stale. |
1074 | */ |
1075 | |
1076 | read_index = ath10k_ce_src_ring_read_index_get(ar, ce_ctrl_addr: ctrl_addr); |
1077 | if (read_index == 0xffffffff) |
1078 | return -ENODEV; |
1079 | |
1080 | read_index &= nentries_mask; |
1081 | src_ring->hw_index = read_index; |
1082 | } |
1083 | |
1084 | if (ar->hw_params.rri_on_ddr) |
1085 | read_index = ath10k_ce_src_ring_read_index_get(ar, ce_ctrl_addr: ctrl_addr); |
1086 | else |
1087 | read_index = src_ring->hw_index; |
1088 | |
1089 | if (read_index == sw_index) |
1090 | return -EIO; |
1091 | |
1092 | if (per_transfer_contextp) |
1093 | *per_transfer_contextp = |
1094 | src_ring->per_transfer_context[sw_index]; |
1095 | |
1096 | /* sanity */ |
1097 | src_ring->per_transfer_context[sw_index] = NULL; |
1098 | desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space, |
1099 | sw_index); |
1100 | desc->nbytes = 0; |
1101 | |
1102 | /* Update sw_index */ |
1103 | sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); |
1104 | src_ring->sw_index = sw_index; |
1105 | |
1106 | return 0; |
1107 | } |
1108 | |
1109 | int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, |
1110 | void **per_transfer_contextp) |
1111 | { |
1112 | return ce_state->ops->ce_completed_send_next_nolock(ce_state, |
1113 | per_transfer_contextp); |
1114 | } |
1115 | EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock); |
1116 | |
1117 | static void (struct ath10k *ar, |
1118 | struct ath10k_ce_ring *src_ring, |
1119 | u32 sw_index, |
1120 | dma_addr_t *bufferp, |
1121 | u32 *nbytesp, |
1122 | u32 *transfer_idp) |
1123 | { |
1124 | struct ce_desc *base = src_ring->base_addr_owner_space; |
1125 | struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); |
1126 | |
1127 | /* Return data from completed source descriptor */ |
1128 | *bufferp = __le32_to_cpu(desc->addr); |
1129 | *nbytesp = __le16_to_cpu(desc->nbytes); |
1130 | *transfer_idp = MS(__le16_to_cpu(desc->flags), |
1131 | CE_DESC_FLAGS_META_DATA); |
1132 | } |
1133 | |
1134 | static void (struct ath10k *ar, |
1135 | struct ath10k_ce_ring *src_ring, |
1136 | u32 sw_index, |
1137 | dma_addr_t *bufferp, |
1138 | u32 *nbytesp, |
1139 | u32 *transfer_idp) |
1140 | { |
1141 | struct ce_desc_64 *base = src_ring->base_addr_owner_space; |
1142 | struct ce_desc_64 *desc = |
1143 | CE_SRC_RING_TO_DESC_64(base, sw_index); |
1144 | |
1145 | /* Return data from completed source descriptor */ |
1146 | *bufferp = __le64_to_cpu(desc->addr); |
1147 | *nbytesp = __le16_to_cpu(desc->nbytes); |
1148 | *transfer_idp = MS(__le16_to_cpu(desc->flags), |
1149 | CE_DESC_FLAGS_META_DATA); |
1150 | } |
1151 | |
1152 | /* NB: Modeled after ath10k_ce_completed_send_next */ |
1153 | int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state, |
1154 | void **per_transfer_contextp, |
1155 | dma_addr_t *bufferp, |
1156 | unsigned int *nbytesp, |
1157 | unsigned int *transfer_idp) |
1158 | { |
1159 | struct ath10k_ce_ring *src_ring; |
1160 | unsigned int nentries_mask; |
1161 | unsigned int sw_index; |
1162 | unsigned int write_index; |
1163 | int ret; |
1164 | struct ath10k *ar; |
1165 | struct ath10k_ce *ce; |
1166 | |
1167 | src_ring = ce_state->src_ring; |
1168 | |
1169 | if (!src_ring) |
1170 | return -EIO; |
1171 | |
1172 | ar = ce_state->ar; |
1173 | ce = ath10k_ce_priv(ar); |
1174 | |
1175 | spin_lock_bh(lock: &ce->ce_lock); |
1176 | |
1177 | nentries_mask = src_ring->nentries_mask; |
1178 | sw_index = src_ring->sw_index; |
1179 | write_index = src_ring->write_index; |
1180 | |
1181 | if (write_index != sw_index) { |
1182 | ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index, |
1183 | bufferp, nbytesp, |
1184 | transfer_idp); |
1185 | |
1186 | if (per_transfer_contextp) |
1187 | *per_transfer_contextp = |
1188 | src_ring->per_transfer_context[sw_index]; |
1189 | |
1190 | /* sanity */ |
1191 | src_ring->per_transfer_context[sw_index] = NULL; |
1192 | |
1193 | /* Update sw_index */ |
1194 | sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); |
1195 | src_ring->sw_index = sw_index; |
1196 | ret = 0; |
1197 | } else { |
1198 | ret = -EIO; |
1199 | } |
1200 | |
1201 | spin_unlock_bh(lock: &ce->ce_lock); |
1202 | |
1203 | return ret; |
1204 | } |
1205 | EXPORT_SYMBOL(ath10k_ce_cancel_send_next); |
1206 | |
1207 | int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state, |
1208 | void **per_transfer_contextp) |
1209 | { |
1210 | struct ath10k *ar = ce_state->ar; |
1211 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1212 | int ret; |
1213 | |
1214 | spin_lock_bh(lock: &ce->ce_lock); |
1215 | ret = ath10k_ce_completed_send_next_nolock(ce_state, |
1216 | per_transfer_contextp); |
1217 | spin_unlock_bh(lock: &ce->ce_lock); |
1218 | |
1219 | return ret; |
1220 | } |
1221 | EXPORT_SYMBOL(ath10k_ce_completed_send_next); |
1222 | |
1223 | /* |
1224 | * Guts of interrupt handler for per-engine interrupts on a particular CE. |
1225 | * |
1226 | * Invokes registered callbacks for recv_complete, |
1227 | * send_complete, and watermarks. |
1228 | */ |
1229 | void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) |
1230 | { |
1231 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1232 | struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; |
1233 | struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; |
1234 | u32 ctrl_addr = ce_state->ctrl_addr; |
1235 | |
1236 | /* |
1237 | * Clear before handling |
1238 | * |
1239 | * Misc CE interrupts are not being handled, but still need |
1240 | * to be cleared. |
1241 | * |
1242 | * NOTE: When the last copy engine interrupt is cleared the |
1243 | * hardware will go to sleep. Once this happens any access to |
1244 | * the CE registers can cause a hardware fault. |
1245 | */ |
1246 | ath10k_ce_engine_int_status_clear(ar, ce_ctrl_addr: ctrl_addr, |
1247 | mask: wm_regs->cc_mask | wm_regs->wm_mask); |
1248 | |
1249 | if (ce_state->recv_cb) |
1250 | ce_state->recv_cb(ce_state); |
1251 | |
1252 | if (ce_state->send_cb) |
1253 | ce_state->send_cb(ce_state); |
1254 | } |
1255 | EXPORT_SYMBOL(ath10k_ce_per_engine_service); |
1256 | |
1257 | /* |
1258 | * Handler for per-engine interrupts on ALL active CEs. |
1259 | * This is used in cases where the system is sharing a |
1260 | * single interrupt for all CEs |
1261 | */ |
1262 | |
1263 | void ath10k_ce_per_engine_service_any(struct ath10k *ar) |
1264 | { |
1265 | int ce_id; |
1266 | u32 intr_summary; |
1267 | |
1268 | intr_summary = ath10k_ce_interrupt_summary(ar); |
1269 | |
1270 | for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) { |
1271 | if (intr_summary & (1 << ce_id)) |
1272 | intr_summary &= ~(1 << ce_id); |
1273 | else |
1274 | /* no intr pending on this CE */ |
1275 | continue; |
1276 | |
1277 | ath10k_ce_per_engine_service(ar, ce_id); |
1278 | } |
1279 | } |
1280 | EXPORT_SYMBOL(ath10k_ce_per_engine_service_any); |
1281 | |
1282 | /* |
1283 | * Adjust interrupts for the copy complete handler. |
1284 | * If it's needed for either send or recv, then unmask |
1285 | * this interrupt; otherwise, mask it. |
1286 | * |
1287 | * Called with ce_lock held. |
1288 | */ |
1289 | static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state) |
1290 | { |
1291 | u32 ctrl_addr = ce_state->ctrl_addr; |
1292 | struct ath10k *ar = ce_state->ar; |
1293 | bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR; |
1294 | |
1295 | if ((!disable_copy_compl_intr) && |
1296 | (ce_state->send_cb || ce_state->recv_cb)) |
1297 | ath10k_ce_copy_complete_inter_enable(ar, ce_ctrl_addr: ctrl_addr); |
1298 | else |
1299 | ath10k_ce_copy_complete_intr_disable(ar, ce_ctrl_addr: ctrl_addr); |
1300 | |
1301 | ath10k_ce_watermark_intr_disable(ar, ce_ctrl_addr: ctrl_addr); |
1302 | } |
1303 | |
1304 | void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id) |
1305 | { |
1306 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1307 | struct ath10k_ce_pipe *ce_state; |
1308 | u32 ctrl_addr; |
1309 | |
1310 | ce_state = &ce->ce_states[ce_id]; |
1311 | if (ce_state->attr_flags & CE_ATTR_POLL) |
1312 | return; |
1313 | |
1314 | ctrl_addr = ath10k_ce_base_address(ar, ce_id); |
1315 | |
1316 | ath10k_ce_copy_complete_intr_disable(ar, ce_ctrl_addr: ctrl_addr); |
1317 | ath10k_ce_error_intr_disable(ar, ce_ctrl_addr: ctrl_addr); |
1318 | ath10k_ce_watermark_intr_disable(ar, ce_ctrl_addr: ctrl_addr); |
1319 | } |
1320 | EXPORT_SYMBOL(ath10k_ce_disable_interrupt); |
1321 | |
1322 | void ath10k_ce_disable_interrupts(struct ath10k *ar) |
1323 | { |
1324 | int ce_id; |
1325 | |
1326 | for (ce_id = 0; ce_id < CE_COUNT; ce_id++) |
1327 | ath10k_ce_disable_interrupt(ar, ce_id); |
1328 | } |
1329 | EXPORT_SYMBOL(ath10k_ce_disable_interrupts); |
1330 | |
1331 | void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id) |
1332 | { |
1333 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1334 | struct ath10k_ce_pipe *ce_state; |
1335 | |
1336 | ce_state = &ce->ce_states[ce_id]; |
1337 | if (ce_state->attr_flags & CE_ATTR_POLL) |
1338 | return; |
1339 | |
1340 | ath10k_ce_per_engine_handler_adjust(ce_state); |
1341 | } |
1342 | EXPORT_SYMBOL(ath10k_ce_enable_interrupt); |
1343 | |
1344 | void ath10k_ce_enable_interrupts(struct ath10k *ar) |
1345 | { |
1346 | int ce_id; |
1347 | |
1348 | /* Enable interrupts for copy engine that |
1349 | * are not using polling mode. |
1350 | */ |
1351 | for (ce_id = 0; ce_id < CE_COUNT; ce_id++) |
1352 | ath10k_ce_enable_interrupt(ar, ce_id); |
1353 | } |
1354 | EXPORT_SYMBOL(ath10k_ce_enable_interrupts); |
1355 | |
1356 | static int ath10k_ce_init_src_ring(struct ath10k *ar, |
1357 | unsigned int ce_id, |
1358 | const struct ce_attr *attr) |
1359 | { |
1360 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1361 | struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; |
1362 | struct ath10k_ce_ring *src_ring = ce_state->src_ring; |
1363 | u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); |
1364 | |
1365 | nentries = roundup_pow_of_two(attr->src_nentries); |
1366 | |
1367 | if (ar->hw_params.target_64bit) |
1368 | memset(src_ring->base_addr_owner_space, 0, |
1369 | nentries * sizeof(struct ce_desc_64)); |
1370 | else |
1371 | memset(src_ring->base_addr_owner_space, 0, |
1372 | nentries * sizeof(struct ce_desc)); |
1373 | |
1374 | src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ce_ctrl_addr: ctrl_addr); |
1375 | src_ring->sw_index &= src_ring->nentries_mask; |
1376 | src_ring->hw_index = src_ring->sw_index; |
1377 | |
1378 | src_ring->write_index = |
1379 | ath10k_ce_src_ring_write_index_get(ar, ce_ctrl_addr: ctrl_addr); |
1380 | src_ring->write_index &= src_ring->nentries_mask; |
1381 | |
1382 | ath10k_ce_src_ring_base_addr_set(ar, ce_id, |
1383 | addr: src_ring->base_addr_ce_space); |
1384 | ath10k_ce_src_ring_size_set(ar, ce_ctrl_addr: ctrl_addr, n: nentries); |
1385 | ath10k_ce_src_ring_dmax_set(ar, ce_ctrl_addr: ctrl_addr, n: attr->src_sz_max); |
1386 | ath10k_ce_src_ring_byte_swap_set(ar, ce_ctrl_addr: ctrl_addr, n: 0); |
1387 | ath10k_ce_src_ring_lowmark_set(ar, ce_ctrl_addr: ctrl_addr, n: 0); |
1388 | ath10k_ce_src_ring_highmark_set(ar, ce_ctrl_addr: ctrl_addr, n: nentries); |
1389 | |
1390 | ath10k_dbg(ar, ATH10K_DBG_BOOT, |
1391 | "boot init ce src ring id %d entries %d base_addr %pK\n" , |
1392 | ce_id, nentries, src_ring->base_addr_owner_space); |
1393 | |
1394 | return 0; |
1395 | } |
1396 | |
1397 | static int ath10k_ce_init_dest_ring(struct ath10k *ar, |
1398 | unsigned int ce_id, |
1399 | const struct ce_attr *attr) |
1400 | { |
1401 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1402 | struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; |
1403 | struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; |
1404 | u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); |
1405 | |
1406 | nentries = roundup_pow_of_two(attr->dest_nentries); |
1407 | |
1408 | if (ar->hw_params.target_64bit) |
1409 | memset(dest_ring->base_addr_owner_space, 0, |
1410 | nentries * sizeof(struct ce_desc_64)); |
1411 | else |
1412 | memset(dest_ring->base_addr_owner_space, 0, |
1413 | nentries * sizeof(struct ce_desc)); |
1414 | |
1415 | dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ce_ctrl_addr: ctrl_addr); |
1416 | dest_ring->sw_index &= dest_ring->nentries_mask; |
1417 | dest_ring->write_index = |
1418 | ath10k_ce_dest_ring_write_index_get(ar, ce_ctrl_addr: ctrl_addr); |
1419 | dest_ring->write_index &= dest_ring->nentries_mask; |
1420 | |
1421 | ath10k_ce_dest_ring_base_addr_set(ar, ce_id, |
1422 | addr: dest_ring->base_addr_ce_space); |
1423 | ath10k_ce_dest_ring_size_set(ar, ce_ctrl_addr: ctrl_addr, n: nentries); |
1424 | ath10k_ce_dest_ring_byte_swap_set(ar, ce_ctrl_addr: ctrl_addr, n: 0); |
1425 | ath10k_ce_dest_ring_lowmark_set(ar, ce_ctrl_addr: ctrl_addr, n: 0); |
1426 | ath10k_ce_dest_ring_highmark_set(ar, ce_ctrl_addr: ctrl_addr, n: nentries); |
1427 | |
1428 | ath10k_dbg(ar, ATH10K_DBG_BOOT, |
1429 | "boot ce dest ring id %d entries %d base_addr %pK\n" , |
1430 | ce_id, nentries, dest_ring->base_addr_owner_space); |
1431 | |
1432 | return 0; |
1433 | } |
1434 | |
1435 | static int ath10k_ce_alloc_shadow_base(struct ath10k *ar, |
1436 | struct ath10k_ce_ring *src_ring, |
1437 | u32 nentries) |
1438 | { |
1439 | src_ring->shadow_base_unaligned = kcalloc(n: nentries, |
1440 | size: sizeof(struct ce_desc_64), |
1441 | GFP_KERNEL); |
1442 | if (!src_ring->shadow_base_unaligned) |
1443 | return -ENOMEM; |
1444 | |
1445 | src_ring->shadow_base = (struct ce_desc_64 *) |
1446 | PTR_ALIGN(src_ring->shadow_base_unaligned, |
1447 | CE_DESC_RING_ALIGN); |
1448 | return 0; |
1449 | } |
1450 | |
1451 | static struct ath10k_ce_ring * |
1452 | ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id, |
1453 | const struct ce_attr *attr) |
1454 | { |
1455 | struct ath10k_ce_ring *src_ring; |
1456 | u32 nentries = attr->src_nentries; |
1457 | dma_addr_t base_addr; |
1458 | int ret; |
1459 | |
1460 | nentries = roundup_pow_of_two(nentries); |
1461 | |
1462 | src_ring = kzalloc(struct_size(src_ring, per_transfer_context, |
1463 | nentries), GFP_KERNEL); |
1464 | if (src_ring == NULL) |
1465 | return ERR_PTR(error: -ENOMEM); |
1466 | |
1467 | src_ring->nentries = nentries; |
1468 | src_ring->nentries_mask = nentries - 1; |
1469 | |
1470 | /* |
1471 | * Legacy platforms that do not support cache |
1472 | * coherent DMA are unsupported |
1473 | */ |
1474 | src_ring->base_addr_owner_space_unaligned = |
1475 | dma_alloc_coherent(dev: ar->dev, |
1476 | size: (nentries * sizeof(struct ce_desc) + |
1477 | CE_DESC_RING_ALIGN), |
1478 | dma_handle: &base_addr, GFP_KERNEL); |
1479 | if (!src_ring->base_addr_owner_space_unaligned) { |
1480 | kfree(objp: src_ring); |
1481 | return ERR_PTR(error: -ENOMEM); |
1482 | } |
1483 | |
1484 | src_ring->base_addr_ce_space_unaligned = base_addr; |
1485 | |
1486 | src_ring->base_addr_owner_space = |
1487 | PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, |
1488 | CE_DESC_RING_ALIGN); |
1489 | src_ring->base_addr_ce_space = |
1490 | ALIGN(src_ring->base_addr_ce_space_unaligned, |
1491 | CE_DESC_RING_ALIGN); |
1492 | |
1493 | if (ar->hw_params.shadow_reg_support) { |
1494 | ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries); |
1495 | if (ret) { |
1496 | dma_free_coherent(dev: ar->dev, |
1497 | size: (nentries * sizeof(struct ce_desc) + |
1498 | CE_DESC_RING_ALIGN), |
1499 | cpu_addr: src_ring->base_addr_owner_space_unaligned, |
1500 | dma_handle: base_addr); |
1501 | kfree(objp: src_ring); |
1502 | return ERR_PTR(error: ret); |
1503 | } |
1504 | } |
1505 | |
1506 | return src_ring; |
1507 | } |
1508 | |
1509 | static struct ath10k_ce_ring * |
1510 | ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id, |
1511 | const struct ce_attr *attr) |
1512 | { |
1513 | struct ath10k_ce_ring *src_ring; |
1514 | u32 nentries = attr->src_nentries; |
1515 | dma_addr_t base_addr; |
1516 | int ret; |
1517 | |
1518 | nentries = roundup_pow_of_two(nentries); |
1519 | |
1520 | src_ring = kzalloc(struct_size(src_ring, per_transfer_context, |
1521 | nentries), GFP_KERNEL); |
1522 | if (!src_ring) |
1523 | return ERR_PTR(error: -ENOMEM); |
1524 | |
1525 | src_ring->nentries = nentries; |
1526 | src_ring->nentries_mask = nentries - 1; |
1527 | |
1528 | /* Legacy platforms that do not support cache |
1529 | * coherent DMA are unsupported |
1530 | */ |
1531 | src_ring->base_addr_owner_space_unaligned = |
1532 | dma_alloc_coherent(dev: ar->dev, |
1533 | size: (nentries * sizeof(struct ce_desc_64) + |
1534 | CE_DESC_RING_ALIGN), |
1535 | dma_handle: &base_addr, GFP_KERNEL); |
1536 | if (!src_ring->base_addr_owner_space_unaligned) { |
1537 | kfree(objp: src_ring); |
1538 | return ERR_PTR(error: -ENOMEM); |
1539 | } |
1540 | |
1541 | src_ring->base_addr_ce_space_unaligned = base_addr; |
1542 | |
1543 | src_ring->base_addr_owner_space = |
1544 | PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, |
1545 | CE_DESC_RING_ALIGN); |
1546 | src_ring->base_addr_ce_space = |
1547 | ALIGN(src_ring->base_addr_ce_space_unaligned, |
1548 | CE_DESC_RING_ALIGN); |
1549 | |
1550 | if (ar->hw_params.shadow_reg_support) { |
1551 | ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries); |
1552 | if (ret) { |
1553 | dma_free_coherent(dev: ar->dev, |
1554 | size: (nentries * sizeof(struct ce_desc_64) + |
1555 | CE_DESC_RING_ALIGN), |
1556 | cpu_addr: src_ring->base_addr_owner_space_unaligned, |
1557 | dma_handle: base_addr); |
1558 | kfree(objp: src_ring); |
1559 | return ERR_PTR(error: ret); |
1560 | } |
1561 | } |
1562 | |
1563 | return src_ring; |
1564 | } |
1565 | |
1566 | static struct ath10k_ce_ring * |
1567 | ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id, |
1568 | const struct ce_attr *attr) |
1569 | { |
1570 | struct ath10k_ce_ring *dest_ring; |
1571 | u32 nentries; |
1572 | dma_addr_t base_addr; |
1573 | |
1574 | nentries = roundup_pow_of_two(attr->dest_nentries); |
1575 | |
1576 | dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context, |
1577 | nentries), GFP_KERNEL); |
1578 | if (dest_ring == NULL) |
1579 | return ERR_PTR(error: -ENOMEM); |
1580 | |
1581 | dest_ring->nentries = nentries; |
1582 | dest_ring->nentries_mask = nentries - 1; |
1583 | |
1584 | /* |
1585 | * Legacy platforms that do not support cache |
1586 | * coherent DMA are unsupported |
1587 | */ |
1588 | dest_ring->base_addr_owner_space_unaligned = |
1589 | dma_alloc_coherent(dev: ar->dev, |
1590 | size: (nentries * sizeof(struct ce_desc) + |
1591 | CE_DESC_RING_ALIGN), |
1592 | dma_handle: &base_addr, GFP_KERNEL); |
1593 | if (!dest_ring->base_addr_owner_space_unaligned) { |
1594 | kfree(objp: dest_ring); |
1595 | return ERR_PTR(error: -ENOMEM); |
1596 | } |
1597 | |
1598 | dest_ring->base_addr_ce_space_unaligned = base_addr; |
1599 | |
1600 | dest_ring->base_addr_owner_space = |
1601 | PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, |
1602 | CE_DESC_RING_ALIGN); |
1603 | dest_ring->base_addr_ce_space = |
1604 | ALIGN(dest_ring->base_addr_ce_space_unaligned, |
1605 | CE_DESC_RING_ALIGN); |
1606 | |
1607 | return dest_ring; |
1608 | } |
1609 | |
1610 | static struct ath10k_ce_ring * |
1611 | ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id, |
1612 | const struct ce_attr *attr) |
1613 | { |
1614 | struct ath10k_ce_ring *dest_ring; |
1615 | u32 nentries; |
1616 | dma_addr_t base_addr; |
1617 | |
1618 | nentries = roundup_pow_of_two(attr->dest_nentries); |
1619 | |
1620 | dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context, |
1621 | nentries), GFP_KERNEL); |
1622 | if (!dest_ring) |
1623 | return ERR_PTR(error: -ENOMEM); |
1624 | |
1625 | dest_ring->nentries = nentries; |
1626 | dest_ring->nentries_mask = nentries - 1; |
1627 | |
1628 | /* Legacy platforms that do not support cache |
1629 | * coherent DMA are unsupported |
1630 | */ |
1631 | dest_ring->base_addr_owner_space_unaligned = |
1632 | dma_alloc_coherent(dev: ar->dev, |
1633 | size: (nentries * sizeof(struct ce_desc_64) + |
1634 | CE_DESC_RING_ALIGN), |
1635 | dma_handle: &base_addr, GFP_KERNEL); |
1636 | if (!dest_ring->base_addr_owner_space_unaligned) { |
1637 | kfree(objp: dest_ring); |
1638 | return ERR_PTR(error: -ENOMEM); |
1639 | } |
1640 | |
1641 | dest_ring->base_addr_ce_space_unaligned = base_addr; |
1642 | |
1643 | /* Correctly initialize memory to 0 to prevent garbage |
1644 | * data crashing system when download firmware |
1645 | */ |
1646 | dest_ring->base_addr_owner_space = |
1647 | PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, |
1648 | CE_DESC_RING_ALIGN); |
1649 | dest_ring->base_addr_ce_space = |
1650 | ALIGN(dest_ring->base_addr_ce_space_unaligned, |
1651 | CE_DESC_RING_ALIGN); |
1652 | |
1653 | return dest_ring; |
1654 | } |
1655 | |
1656 | /* |
1657 | * Initialize a Copy Engine based on caller-supplied attributes. |
1658 | * This may be called once to initialize both source and destination |
1659 | * rings or it may be called twice for separate source and destination |
1660 | * initialization. It may be that only one side or the other is |
1661 | * initialized by software/firmware. |
1662 | */ |
1663 | int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id, |
1664 | const struct ce_attr *attr) |
1665 | { |
1666 | int ret; |
1667 | |
1668 | if (attr->src_nentries) { |
1669 | ret = ath10k_ce_init_src_ring(ar, ce_id, attr); |
1670 | if (ret) { |
1671 | ath10k_err(ar, fmt: "Failed to initialize CE src ring for ID: %d (%d)\n" , |
1672 | ce_id, ret); |
1673 | return ret; |
1674 | } |
1675 | } |
1676 | |
1677 | if (attr->dest_nentries) { |
1678 | ret = ath10k_ce_init_dest_ring(ar, ce_id, attr); |
1679 | if (ret) { |
1680 | ath10k_err(ar, fmt: "Failed to initialize CE dest ring for ID: %d (%d)\n" , |
1681 | ce_id, ret); |
1682 | return ret; |
1683 | } |
1684 | } |
1685 | |
1686 | return 0; |
1687 | } |
1688 | EXPORT_SYMBOL(ath10k_ce_init_pipe); |
1689 | |
1690 | static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id) |
1691 | { |
1692 | u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); |
1693 | |
1694 | ath10k_ce_src_ring_base_addr_set(ar, ce_id, addr: 0); |
1695 | ath10k_ce_src_ring_size_set(ar, ce_ctrl_addr: ctrl_addr, n: 0); |
1696 | ath10k_ce_src_ring_dmax_set(ar, ce_ctrl_addr: ctrl_addr, n: 0); |
1697 | ath10k_ce_src_ring_highmark_set(ar, ce_ctrl_addr: ctrl_addr, n: 0); |
1698 | } |
1699 | |
1700 | static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id) |
1701 | { |
1702 | u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); |
1703 | |
1704 | ath10k_ce_dest_ring_base_addr_set(ar, ce_id, addr: 0); |
1705 | ath10k_ce_dest_ring_size_set(ar, ce_ctrl_addr: ctrl_addr, n: 0); |
1706 | ath10k_ce_dest_ring_highmark_set(ar, ce_ctrl_addr: ctrl_addr, n: 0); |
1707 | } |
1708 | |
1709 | void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id) |
1710 | { |
1711 | ath10k_ce_deinit_src_ring(ar, ce_id); |
1712 | ath10k_ce_deinit_dest_ring(ar, ce_id); |
1713 | } |
1714 | EXPORT_SYMBOL(ath10k_ce_deinit_pipe); |
1715 | |
1716 | static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) |
1717 | { |
1718 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1719 | struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; |
1720 | |
1721 | if (ce_state->src_ring) { |
1722 | if (ar->hw_params.shadow_reg_support) |
1723 | kfree(objp: ce_state->src_ring->shadow_base_unaligned); |
1724 | dma_free_coherent(dev: ar->dev, |
1725 | size: (ce_state->src_ring->nentries * |
1726 | sizeof(struct ce_desc) + |
1727 | CE_DESC_RING_ALIGN), |
1728 | cpu_addr: ce_state->src_ring->base_addr_owner_space, |
1729 | dma_handle: ce_state->src_ring->base_addr_ce_space); |
1730 | kfree(objp: ce_state->src_ring); |
1731 | } |
1732 | |
1733 | if (ce_state->dest_ring) { |
1734 | dma_free_coherent(dev: ar->dev, |
1735 | size: (ce_state->dest_ring->nentries * |
1736 | sizeof(struct ce_desc) + |
1737 | CE_DESC_RING_ALIGN), |
1738 | cpu_addr: ce_state->dest_ring->base_addr_owner_space, |
1739 | dma_handle: ce_state->dest_ring->base_addr_ce_space); |
1740 | kfree(objp: ce_state->dest_ring); |
1741 | } |
1742 | |
1743 | ce_state->src_ring = NULL; |
1744 | ce_state->dest_ring = NULL; |
1745 | } |
1746 | |
1747 | static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id) |
1748 | { |
1749 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1750 | struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; |
1751 | |
1752 | if (ce_state->src_ring) { |
1753 | if (ar->hw_params.shadow_reg_support) |
1754 | kfree(objp: ce_state->src_ring->shadow_base_unaligned); |
1755 | dma_free_coherent(dev: ar->dev, |
1756 | size: (ce_state->src_ring->nentries * |
1757 | sizeof(struct ce_desc_64) + |
1758 | CE_DESC_RING_ALIGN), |
1759 | cpu_addr: ce_state->src_ring->base_addr_owner_space, |
1760 | dma_handle: ce_state->src_ring->base_addr_ce_space); |
1761 | kfree(objp: ce_state->src_ring); |
1762 | } |
1763 | |
1764 | if (ce_state->dest_ring) { |
1765 | dma_free_coherent(dev: ar->dev, |
1766 | size: (ce_state->dest_ring->nentries * |
1767 | sizeof(struct ce_desc_64) + |
1768 | CE_DESC_RING_ALIGN), |
1769 | cpu_addr: ce_state->dest_ring->base_addr_owner_space, |
1770 | dma_handle: ce_state->dest_ring->base_addr_ce_space); |
1771 | kfree(objp: ce_state->dest_ring); |
1772 | } |
1773 | |
1774 | ce_state->src_ring = NULL; |
1775 | ce_state->dest_ring = NULL; |
1776 | } |
1777 | |
1778 | void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) |
1779 | { |
1780 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1781 | struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; |
1782 | |
1783 | ce_state->ops->ce_free_pipe(ar, ce_id); |
1784 | } |
1785 | EXPORT_SYMBOL(ath10k_ce_free_pipe); |
1786 | |
1787 | void ath10k_ce_dump_registers(struct ath10k *ar, |
1788 | struct ath10k_fw_crash_data *crash_data) |
1789 | { |
1790 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1791 | struct ath10k_ce_crash_data ce_data; |
1792 | u32 addr, id; |
1793 | |
1794 | lockdep_assert_held(&ar->dump_mutex); |
1795 | |
1796 | ath10k_err(ar, fmt: "Copy Engine register dump:\n" ); |
1797 | |
1798 | spin_lock_bh(lock: &ce->ce_lock); |
1799 | for (id = 0; id < CE_COUNT; id++) { |
1800 | addr = ath10k_ce_base_address(ar, ce_id: id); |
1801 | ce_data.base_addr = cpu_to_le32(addr); |
1802 | |
1803 | ce_data.src_wr_idx = |
1804 | cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr)); |
1805 | ce_data.src_r_idx = |
1806 | cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr)); |
1807 | ce_data.dst_wr_idx = |
1808 | cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr)); |
1809 | ce_data.dst_r_idx = |
1810 | cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr)); |
1811 | |
1812 | if (crash_data) |
1813 | crash_data->ce_crash_data[id] = ce_data; |
1814 | |
1815 | ath10k_err(ar, fmt: "[%02d]: 0x%08x %3u %3u %3u %3u" , id, |
1816 | le32_to_cpu(ce_data.base_addr), |
1817 | le32_to_cpu(ce_data.src_wr_idx), |
1818 | le32_to_cpu(ce_data.src_r_idx), |
1819 | le32_to_cpu(ce_data.dst_wr_idx), |
1820 | le32_to_cpu(ce_data.dst_r_idx)); |
1821 | } |
1822 | |
1823 | spin_unlock_bh(lock: &ce->ce_lock); |
1824 | } |
1825 | EXPORT_SYMBOL(ath10k_ce_dump_registers); |
1826 | |
1827 | static const struct ath10k_ce_ops ce_ops = { |
1828 | .ce_alloc_src_ring = ath10k_ce_alloc_src_ring, |
1829 | .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring, |
1830 | .ce_rx_post_buf = __ath10k_ce_rx_post_buf, |
1831 | .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock, |
1832 | .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next, |
1833 | .ce_extract_desc_data = ath10k_ce_extract_desc_data, |
1834 | .ce_free_pipe = _ath10k_ce_free_pipe, |
1835 | .ce_send_nolock = _ath10k_ce_send_nolock, |
1836 | .ce_set_src_ring_base_addr_hi = NULL, |
1837 | .ce_set_dest_ring_base_addr_hi = NULL, |
1838 | .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock, |
1839 | }; |
1840 | |
1841 | static const struct ath10k_ce_ops ce_64_ops = { |
1842 | .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64, |
1843 | .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64, |
1844 | .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64, |
1845 | .ce_completed_recv_next_nolock = |
1846 | _ath10k_ce_completed_recv_next_nolock_64, |
1847 | .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64, |
1848 | .ce_extract_desc_data = ath10k_ce_extract_desc_data_64, |
1849 | .ce_free_pipe = _ath10k_ce_free_pipe_64, |
1850 | .ce_send_nolock = _ath10k_ce_send_nolock_64, |
1851 | .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi, |
1852 | .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi, |
1853 | .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64, |
1854 | }; |
1855 | |
1856 | static void ath10k_ce_set_ops(struct ath10k *ar, |
1857 | struct ath10k_ce_pipe *ce_state) |
1858 | { |
1859 | switch (ar->hw_rev) { |
1860 | case ATH10K_HW_WCN3990: |
1861 | ce_state->ops = &ce_64_ops; |
1862 | break; |
1863 | default: |
1864 | ce_state->ops = &ce_ops; |
1865 | break; |
1866 | } |
1867 | } |
1868 | |
1869 | int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id, |
1870 | const struct ce_attr *attr) |
1871 | { |
1872 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1873 | struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; |
1874 | int ret; |
1875 | |
1876 | ath10k_ce_set_ops(ar, ce_state); |
1877 | /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid |
1878 | * additional TX locking checks. |
1879 | * |
1880 | * For the lack of a better place do the check here. |
1881 | */ |
1882 | BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC > |
1883 | (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); |
1884 | BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC > |
1885 | (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); |
1886 | BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC > |
1887 | (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); |
1888 | |
1889 | ce_state->ar = ar; |
1890 | ce_state->id = ce_id; |
1891 | ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id); |
1892 | ce_state->attr_flags = attr->flags; |
1893 | ce_state->src_sz_max = attr->src_sz_max; |
1894 | |
1895 | if (attr->src_nentries) |
1896 | ce_state->send_cb = attr->send_cb; |
1897 | |
1898 | if (attr->dest_nentries) |
1899 | ce_state->recv_cb = attr->recv_cb; |
1900 | |
1901 | if (attr->src_nentries) { |
1902 | ce_state->src_ring = |
1903 | ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr); |
1904 | if (IS_ERR(ptr: ce_state->src_ring)) { |
1905 | ret = PTR_ERR(ptr: ce_state->src_ring); |
1906 | ath10k_err(ar, fmt: "failed to alloc CE src ring %d: %d\n" , |
1907 | ce_id, ret); |
1908 | ce_state->src_ring = NULL; |
1909 | return ret; |
1910 | } |
1911 | } |
1912 | |
1913 | if (attr->dest_nentries) { |
1914 | ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar, |
1915 | ce_id, |
1916 | attr); |
1917 | if (IS_ERR(ptr: ce_state->dest_ring)) { |
1918 | ret = PTR_ERR(ptr: ce_state->dest_ring); |
1919 | ath10k_err(ar, fmt: "failed to alloc CE dest ring %d: %d\n" , |
1920 | ce_id, ret); |
1921 | ce_state->dest_ring = NULL; |
1922 | return ret; |
1923 | } |
1924 | } |
1925 | |
1926 | return 0; |
1927 | } |
1928 | EXPORT_SYMBOL(ath10k_ce_alloc_pipe); |
1929 | |
1930 | void ath10k_ce_alloc_rri(struct ath10k *ar) |
1931 | { |
1932 | int i; |
1933 | u32 value; |
1934 | u32 ctrl1_regs; |
1935 | u32 ce_base_addr; |
1936 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1937 | |
1938 | ce->vaddr_rri = dma_alloc_coherent(dev: ar->dev, |
1939 | size: (CE_COUNT * sizeof(u32)), |
1940 | dma_handle: &ce->paddr_rri, GFP_KERNEL); |
1941 | |
1942 | if (!ce->vaddr_rri) |
1943 | return; |
1944 | |
1945 | ath10k_ce_write32(ar, offset: ar->hw_ce_regs->ce_rri_low, |
1946 | lower_32_bits(ce->paddr_rri)); |
1947 | ath10k_ce_write32(ar, offset: ar->hw_ce_regs->ce_rri_high, |
1948 | value: (upper_32_bits(ce->paddr_rri) & |
1949 | CE_DESC_ADDR_HI_MASK)); |
1950 | |
1951 | for (i = 0; i < CE_COUNT; i++) { |
1952 | ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr; |
1953 | ce_base_addr = ath10k_ce_base_address(ar, ce_id: i); |
1954 | value = ath10k_ce_read32(ar, offset: ce_base_addr + ctrl1_regs); |
1955 | value |= ar->hw_ce_regs->upd->mask; |
1956 | ath10k_ce_write32(ar, offset: ce_base_addr + ctrl1_regs, value); |
1957 | } |
1958 | } |
1959 | EXPORT_SYMBOL(ath10k_ce_alloc_rri); |
1960 | |
1961 | void ath10k_ce_free_rri(struct ath10k *ar) |
1962 | { |
1963 | struct ath10k_ce *ce = ath10k_ce_priv(ar); |
1964 | |
1965 | dma_free_coherent(dev: ar->dev, size: (CE_COUNT * sizeof(u32)), |
1966 | cpu_addr: ce->vaddr_rri, |
1967 | dma_handle: ce->paddr_rri); |
1968 | } |
1969 | EXPORT_SYMBOL(ath10k_ce_free_rri); |
1970 | |