1	// SPDX-License-Identifier: BSD-3-Clause-Clear
2	/*
3	* Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
4	* Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved.
5	*/
6
7	#include <crypto/hash.h>
8	#include "core.h"
9	#include "dp_tx.h"
10	#include "hal_tx.h"
11	#include "hif.h"
12	#include "debug.h"
13	#include "dp_rx.h"
14	#include "peer.h"
15	#include "dp_mon.h"
16
17	static void ath12k_dp_htt_htc_tx_complete(struct ath12k_base *ab,
18	struct sk_buff *skb)
19	{
20	dev_kfree_skb_any(skb);
21	}
22
23	void ath12k_dp_peer_cleanup(struct ath12k *ar, int vdev_id, const u8 *addr)
24	{
25	struct ath12k_base *ab = ar->ab;
26	struct ath12k_peer *peer;
27
28	/* TODO: Any other peer specific DP cleanup */
29
30	spin_lock_bh(lock: &ab->base_lock);
31	peer = ath12k_peer_find(ab, vdev_id, addr);
32	if (!peer) {
33	ath12k_warn(ab, fmt: "failed to lookup peer %pM on vdev %d\n",
34	addr, vdev_id);
35	spin_unlock_bh(lock: &ab->base_lock);
36	return;
37	}
38
39	ath12k_dp_rx_peer_tid_cleanup(ar, peer);
40	crypto_free_shash(tfm: peer->tfm_mmic);
41	peer->dp_setup_done = false;
42	spin_unlock_bh(lock: &ab->base_lock);
43	}
44
45	int ath12k_dp_peer_setup(struct ath12k *ar, int vdev_id, const u8 *addr)
46	{
47	struct ath12k_base *ab = ar->ab;
48	struct ath12k_peer *peer;
49	u32 reo_dest;
50	int ret = 0, tid;
51
52	/* NOTE: reo_dest ring id starts from 1 unlike mac_id which starts from 0 */
53	reo_dest = ar->dp.mac_id + 1;
54	ret = ath12k_wmi_set_peer_param(ar, peer_addr: addr, vdev_id,
55	param_id: WMI_PEER_SET_DEFAULT_ROUTING,
56	DP_RX_HASH_ENABLE | (reo_dest << 1));
57
58	if (ret) {
59	ath12k_warn(ab, fmt: "failed to set default routing %d peer :%pM vdev_id :%d\n",
60	ret, addr, vdev_id);
61	return ret;
62	}
63
64	for (tid = 0; tid <= IEEE80211_NUM_TIDS; tid++) {
65	ret = ath12k_dp_rx_peer_tid_setup(ar, peer_mac: addr, vdev_id, tid, ba_win_sz: 1, ssn: 0,
66	pn_type: HAL_PN_TYPE_NONE);
67	if (ret) {
68	ath12k_warn(ab, fmt: "failed to setup rxd tid queue for tid %d: %d\n",
69	tid, ret);
70	goto peer_clean;
71	}
72	}
73
74	ret = ath12k_dp_rx_peer_frag_setup(ar, peer_mac: addr, vdev_id);
75	if (ret) {
76	ath12k_warn(ab, fmt: "failed to setup rx defrag context\n");
77	goto peer_clean;
78	}
79
80	/* TODO: Setup other peer specific resource used in data path */
81
82	return 0;
83
84	peer_clean:
85	spin_lock_bh(lock: &ab->base_lock);
86
87	peer = ath12k_peer_find(ab, vdev_id, addr);
88	if (!peer) {
89	ath12k_warn(ab, fmt: "failed to find the peer to del rx tid\n");
90	spin_unlock_bh(lock: &ab->base_lock);
91	return -ENOENT;
92	}
93
94	for (; tid >= 0; tid--)
95	ath12k_dp_rx_peer_tid_delete(ar, peer, tid);
96
97	spin_unlock_bh(lock: &ab->base_lock);
98
99	return ret;
100	}
101
102	void ath12k_dp_srng_cleanup(struct ath12k_base *ab, struct dp_srng *ring)
103	{
104	if (!ring->vaddr_unaligned)
105	return;
106
107	dma_free_coherent(dev: ab->dev, size: ring->size, cpu_addr: ring->vaddr_unaligned,
108	dma_handle: ring->paddr_unaligned);
109
110	ring->vaddr_unaligned = NULL;
111	}
112
113	static int ath12k_dp_srng_find_ring_in_mask(int ring_num, const u8 *grp_mask)
114	{
115	int ext_group_num;
116	u8 mask = 1 << ring_num;
117
118	for (ext_group_num = 0; ext_group_num < ATH12K_EXT_IRQ_GRP_NUM_MAX;
119	ext_group_num++) {
120	if (mask & grp_mask[ext_group_num])
121	return ext_group_num;
122	}
123
124	return -ENOENT;
125	}
126
127	static int ath12k_dp_srng_calculate_msi_group(struct ath12k_base *ab,
128	enum hal_ring_type type, int ring_num)
129	{
130	const u8 *grp_mask;
131
132	switch (type) {
133	case HAL_WBM2SW_RELEASE:
134	if (ring_num == HAL_WBM2SW_REL_ERR_RING_NUM) {
135	grp_mask = &ab->hw_params->ring_mask->rx_wbm_rel[0];
136	ring_num = 0;
137	} else {
138	grp_mask = &ab->hw_params->ring_mask->tx[0];
139	}
140	break;
141	case HAL_REO_EXCEPTION:
142	grp_mask = &ab->hw_params->ring_mask->rx_err[0];
143	break;
144	case HAL_REO_DST:
145	grp_mask = &ab->hw_params->ring_mask->rx[0];
146	break;
147	case HAL_REO_STATUS:
148	grp_mask = &ab->hw_params->ring_mask->reo_status[0];
149	break;
150	case HAL_RXDMA_MONITOR_STATUS:
151	case HAL_RXDMA_MONITOR_DST:
152	grp_mask = &ab->hw_params->ring_mask->rx_mon_dest[0];
153	break;
154	case HAL_TX_MONITOR_DST:
155	grp_mask = &ab->hw_params->ring_mask->tx_mon_dest[0];
156	break;
157	case HAL_RXDMA_BUF:
158	grp_mask = &ab->hw_params->ring_mask->host2rxdma[0];
159	break;
160	case HAL_RXDMA_MONITOR_BUF:
161	case HAL_TCL_DATA:
162	case HAL_TCL_CMD:
163	case HAL_REO_CMD:
164	case HAL_SW2WBM_RELEASE:
165	case HAL_WBM_IDLE_LINK:
166	case HAL_TCL_STATUS:
167	case HAL_REO_REINJECT:
168	case HAL_CE_SRC:
169	case HAL_CE_DST:
170	case HAL_CE_DST_STATUS:
171	default:
172	return -ENOENT;
173	}
174
175	return ath12k_dp_srng_find_ring_in_mask(ring_num, grp_mask);
176	}
177
178	static void ath12k_dp_srng_msi_setup(struct ath12k_base *ab,
179	struct hal_srng_params *ring_params,
180	enum hal_ring_type type, int ring_num)
181	{
182	int msi_group_number, msi_data_count;
183	u32 msi_data_start, msi_irq_start, addr_lo, addr_hi;
184	int ret;
185
186	ret = ath12k_hif_get_user_msi_vector(ab, user_name: "DP",
187	num_vectors: &msi_data_count, user_base_data: &msi_data_start,
188	base_vector: &msi_irq_start);
189	if (ret)
190	return;
191
192	msi_group_number = ath12k_dp_srng_calculate_msi_group(ab, type,
193	ring_num);
194	if (msi_group_number < 0) {
195	ath12k_dbg(ab, ATH12K_DBG_PCI,
196	"ring not part of an ext_group; ring_type: %d,ring_num %d",
197	type, ring_num);
198	ring_params->msi_addr = 0;
199	ring_params->msi_data = 0;
200	return;
201	}
202
203	if (msi_group_number > msi_data_count) {
204	ath12k_dbg(ab, ATH12K_DBG_PCI,
205	"multiple msi_groups share one msi, msi_group_num %d",
206	msi_group_number);
207	}
208
209	ath12k_hif_get_msi_address(ab, msi_addr_lo: &addr_lo, msi_addr_hi: &addr_hi);
210
211	ring_params->msi_addr = addr_lo;
212	ring_params->msi_addr |= (dma_addr_t)(((uint64_t)addr_hi) << 32);
213	ring_params->msi_data = (msi_group_number % msi_data_count)
214	+ msi_data_start;
215	ring_params->flags |= HAL_SRNG_FLAGS_MSI_INTR;
216	}
217
218	int ath12k_dp_srng_setup(struct ath12k_base *ab, struct dp_srng *ring,
219	enum hal_ring_type type, int ring_num,
220	int mac_id, int num_entries)
221	{
222	struct hal_srng_params params = { 0 };
223	int entry_sz = ath12k_hal_srng_get_entrysize(ab, ring_type: type);
224	int max_entries = ath12k_hal_srng_get_max_entries(ab, ring_type: type);
225	int ret;
226
227	if (max_entries < 0 || entry_sz < 0)
228	return -EINVAL;
229
230	if (num_entries > max_entries)
231	num_entries = max_entries;
232
233	ring->size = (num_entries * entry_sz) + HAL_RING_BASE_ALIGN - 1;
234	ring->vaddr_unaligned = dma_alloc_coherent(dev: ab->dev, size: ring->size,
235	dma_handle: &ring->paddr_unaligned,
236	GFP_KERNEL);
237	if (!ring->vaddr_unaligned)
238	return -ENOMEM;
239
240	ring->vaddr = PTR_ALIGN(ring->vaddr_unaligned, HAL_RING_BASE_ALIGN);
241	ring->paddr = ring->paddr_unaligned + ((unsigned long)ring->vaddr -
242	(unsigned long)ring->vaddr_unaligned);
243
244	params.ring_base_vaddr = ring->vaddr;
245	params.ring_base_paddr = ring->paddr;
246	params.num_entries = num_entries;
247	ath12k_dp_srng_msi_setup(ab, ring_params: &params, type, ring_num: ring_num + mac_id);
248
249	switch (type) {
250	case HAL_REO_DST:
251	params.intr_batch_cntr_thres_entries =
252	HAL_SRNG_INT_BATCH_THRESHOLD_RX;
253	params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX;
254	break;
255	case HAL_RXDMA_BUF:
256	case HAL_RXDMA_MONITOR_BUF:
257	case HAL_RXDMA_MONITOR_STATUS:
258	params.low_threshold = num_entries >> 3;
259	params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN;
260	params.intr_batch_cntr_thres_entries = 0;
261	params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX;
262	break;
263	case HAL_TX_MONITOR_DST:
264	params.low_threshold = DP_TX_MONITOR_BUF_SIZE_MAX >> 3;
265	params.flags |= HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN;
266	params.intr_batch_cntr_thres_entries = 0;
267	params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_RX;
268	break;
269	case HAL_WBM2SW_RELEASE:
270	if (ab->hw_params->hw_ops->dp_srng_is_tx_comp_ring(ring_num)) {
271	params.intr_batch_cntr_thres_entries =
272	HAL_SRNG_INT_BATCH_THRESHOLD_TX;
273	params.intr_timer_thres_us =
274	HAL_SRNG_INT_TIMER_THRESHOLD_TX;
275	break;
276	}
277	/* follow through when ring_num != HAL_WBM2SW_REL_ERR_RING_NUM */
278	fallthrough;
279	case HAL_REO_EXCEPTION:
280	case HAL_REO_REINJECT:
281	case HAL_REO_CMD:
282	case HAL_REO_STATUS:
283	case HAL_TCL_DATA:
284	case HAL_TCL_CMD:
285	case HAL_TCL_STATUS:
286	case HAL_WBM_IDLE_LINK:
287	case HAL_SW2WBM_RELEASE:
288	case HAL_RXDMA_DST:
289	case HAL_RXDMA_MONITOR_DST:
290	case HAL_RXDMA_MONITOR_DESC:
291	params.intr_batch_cntr_thres_entries =
292	HAL_SRNG_INT_BATCH_THRESHOLD_OTHER;
293	params.intr_timer_thres_us = HAL_SRNG_INT_TIMER_THRESHOLD_OTHER;
294	break;
295	case HAL_RXDMA_DIR_BUF:
296	break;
297	default:
298	ath12k_warn(ab, fmt: "Not a valid ring type in dp :%d\n", type);
299	return -EINVAL;
300	}
301
302	ret = ath12k_hal_srng_setup(ab, type, ring_num, mac_id, params: &params);
303	if (ret < 0) {
304	ath12k_warn(ab, fmt: "failed to setup srng: %d ring_id %d\n",
305	ret, ring_num);
306	return ret;
307	}
308
309	ring->ring_id = ret;
310
311	return 0;
312	}
313
314	static
315	u32 ath12k_dp_tx_get_vdev_bank_config(struct ath12k_base *ab, struct ath12k_vif *arvif)
316	{
317	u32 bank_config = 0;
318
319	/* Only valid for raw frames with HW crypto enabled.
320	* With SW crypto, mac80211 sets key per packet
321	*/
322	if (arvif->tx_encap_type == HAL_TCL_ENCAP_TYPE_RAW &&
323	test_bit(ATH12K_FLAG_HW_CRYPTO_DISABLED, &ab->dev_flags))
324	bank_config |=
325	u32_encode_bits(v: ath12k_dp_tx_get_encrypt_type(cipher: arvif->key_cipher),
326	HAL_TX_BANK_CONFIG_ENCRYPT_TYPE);
327
328	bank_config |= u32_encode_bits(v: arvif->tx_encap_type,
329	HAL_TX_BANK_CONFIG_ENCAP_TYPE);
330	bank_config |= u32_encode_bits(v: 0, HAL_TX_BANK_CONFIG_SRC_BUFFER_SWAP) |
331	u32_encode_bits(v: 0, HAL_TX_BANK_CONFIG_LINK_META_SWAP) |
332	u32_encode_bits(v: 0, HAL_TX_BANK_CONFIG_EPD);
333
334	/* only valid if idx_lookup_override is not set in tcl_data_cmd */
335	bank_config |= u32_encode_bits(v: 0, HAL_TX_BANK_CONFIG_INDEX_LOOKUP_EN);
336
337	bank_config |= u32_encode_bits(v: arvif->hal_addr_search_flags & HAL_TX_ADDRX_EN,
338	HAL_TX_BANK_CONFIG_ADDRX_EN) |
339	u32_encode_bits(v: !!(arvif->hal_addr_search_flags &
340	HAL_TX_ADDRY_EN),
341	HAL_TX_BANK_CONFIG_ADDRY_EN);
342
343	bank_config |= u32_encode_bits(v: ieee80211_vif_is_mesh(vif: arvif->vif) ? 3 : 0,
344	HAL_TX_BANK_CONFIG_MESH_EN) |
345	u32_encode_bits(v: arvif->vdev_id_check_en,
346	HAL_TX_BANK_CONFIG_VDEV_ID_CHECK_EN);
347
348	bank_config |= u32_encode_bits(v: 0, HAL_TX_BANK_CONFIG_DSCP_TIP_MAP_ID);
349
350	return bank_config;
351	}
352
353	static int ath12k_dp_tx_get_bank_profile(struct ath12k_base *ab, struct ath12k_vif *arvif,
354	struct ath12k_dp *dp)
355	{
356	int bank_id = DP_INVALID_BANK_ID;
357	int i;
358	u32 bank_config;
359	bool configure_register = false;
360
361	/* convert vdev params into hal_tx_bank_config */
362	bank_config = ath12k_dp_tx_get_vdev_bank_config(ab, arvif);
363
364	spin_lock_bh(lock: &dp->tx_bank_lock);
365	/* TODO: implement using idr kernel framework*/
366	for (i = 0; i < dp->num_bank_profiles; i++) {
367	if (dp->bank_profiles[i].is_configured &&
368	(dp->bank_profiles[i].bank_config ^ bank_config) == 0) {
369	bank_id = i;
370	goto inc_ref_and_return;
371	}
372	if (!dp->bank_profiles[i].is_configured ||
373	!dp->bank_profiles[i].num_users) {
374	bank_id = i;
375	goto configure_and_return;
376	}
377	}
378
379	if (bank_id == DP_INVALID_BANK_ID) {
380	spin_unlock_bh(lock: &dp->tx_bank_lock);
381	ath12k_err(ab, fmt: "unable to find TX bank!");
382	return bank_id;
383	}
384
385	configure_and_return:
386	dp->bank_profiles[bank_id].is_configured = true;
387	dp->bank_profiles[bank_id].bank_config = bank_config;
388	configure_register = true;
389	inc_ref_and_return:
390	dp->bank_profiles[bank_id].num_users++;
391	spin_unlock_bh(lock: &dp->tx_bank_lock);
392
393	if (configure_register)
394	ath12k_hal_tx_configure_bank_register(ab, bank_config, bank_id);
395
396	ath12k_dbg(ab, ATH12K_DBG_DP_HTT, "dp_htt tcl bank_id %d input 0x%x match 0x%x num_users %u",
397	bank_id, bank_config, dp->bank_profiles[bank_id].bank_config,
398	dp->bank_profiles[bank_id].num_users);
399
400	return bank_id;
401	}
402
403	void ath12k_dp_tx_put_bank_profile(struct ath12k_dp *dp, u8 bank_id)
404	{
405	spin_lock_bh(lock: &dp->tx_bank_lock);
406	dp->bank_profiles[bank_id].num_users--;
407	spin_unlock_bh(lock: &dp->tx_bank_lock);
408	}
409
410	static void ath12k_dp_deinit_bank_profiles(struct ath12k_base *ab)
411	{
412	struct ath12k_dp *dp = &ab->dp;
413
414	kfree(objp: dp->bank_profiles);
415	dp->bank_profiles = NULL;
416	}
417
418	static int ath12k_dp_init_bank_profiles(struct ath12k_base *ab)
419	{
420	struct ath12k_dp *dp = &ab->dp;
421	u32 num_tcl_banks = ab->hw_params->num_tcl_banks;
422	int i;
423
424	dp->num_bank_profiles = num_tcl_banks;
425	dp->bank_profiles = kmalloc_array(n: num_tcl_banks,
426	size: sizeof(struct ath12k_dp_tx_bank_profile),
427	GFP_KERNEL);
428	if (!dp->bank_profiles)
429	return -ENOMEM;
430
431	spin_lock_init(&dp->tx_bank_lock);
432
433	for (i = 0; i < num_tcl_banks; i++) {
434	dp->bank_profiles[i].is_configured = false;
435	dp->bank_profiles[i].num_users = 0;
436	}
437
438	return 0;
439	}
440
441	static void ath12k_dp_srng_common_cleanup(struct ath12k_base *ab)
442	{
443	struct ath12k_dp *dp = &ab->dp;
444	int i;
445
446	ath12k_dp_srng_cleanup(ab, ring: &dp->reo_status_ring);
447	ath12k_dp_srng_cleanup(ab, ring: &dp->reo_cmd_ring);
448	ath12k_dp_srng_cleanup(ab, ring: &dp->reo_except_ring);
449	ath12k_dp_srng_cleanup(ab, ring: &dp->rx_rel_ring);
450	ath12k_dp_srng_cleanup(ab, ring: &dp->reo_reinject_ring);
451	for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
452	ath12k_dp_srng_cleanup(ab, ring: &dp->tx_ring[i].tcl_comp_ring);
453	ath12k_dp_srng_cleanup(ab, ring: &dp->tx_ring[i].tcl_data_ring);
454	}
455	ath12k_dp_srng_cleanup(ab, ring: &dp->tcl_status_ring);
456	ath12k_dp_srng_cleanup(ab, ring: &dp->tcl_cmd_ring);
457	ath12k_dp_srng_cleanup(ab, ring: &dp->wbm_desc_rel_ring);
458	}
459
460	static int ath12k_dp_srng_common_setup(struct ath12k_base *ab)
461	{
462	struct ath12k_dp *dp = &ab->dp;
463	const struct ath12k_hal_tcl_to_wbm_rbm_map *map;
464	struct hal_srng *srng;
465	int i, ret, tx_comp_ring_num;
466	u32 ring_hash_map;
467
468	ret = ath12k_dp_srng_setup(ab, ring: &dp->wbm_desc_rel_ring,
469	type: HAL_SW2WBM_RELEASE, ring_num: 0, mac_id: 0,
470	DP_WBM_RELEASE_RING_SIZE);
471	if (ret) {
472	ath12k_warn(ab, fmt: "failed to set up wbm2sw_release ring :%d\n",
473	ret);
474	goto err;
475	}
476
477	ret = ath12k_dp_srng_setup(ab, ring: &dp->tcl_cmd_ring, type: HAL_TCL_CMD, ring_num: 0, mac_id: 0,
478	DP_TCL_CMD_RING_SIZE);
479	if (ret) {
480	ath12k_warn(ab, fmt: "failed to set up tcl_cmd ring :%d\n", ret);
481	goto err;
482	}
483
484	ret = ath12k_dp_srng_setup(ab, ring: &dp->tcl_status_ring, type: HAL_TCL_STATUS,
485	ring_num: 0, mac_id: 0, DP_TCL_STATUS_RING_SIZE);
486	if (ret) {
487	ath12k_warn(ab, fmt: "failed to set up tcl_status ring :%d\n", ret);
488	goto err;
489	}
490
491	for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
492	map = ab->hw_params->hal_ops->tcl_to_wbm_rbm_map;
493	tx_comp_ring_num = map[i].wbm_ring_num;
494
495	ret = ath12k_dp_srng_setup(ab, ring: &dp->tx_ring[i].tcl_data_ring,
496	type: HAL_TCL_DATA, ring_num: i, mac_id: 0,
497	DP_TCL_DATA_RING_SIZE);
498	if (ret) {
499	ath12k_warn(ab, fmt: "failed to set up tcl_data ring (%d) :%d\n",
500	i, ret);
501	goto err;
502	}
503
504	ret = ath12k_dp_srng_setup(ab, ring: &dp->tx_ring[i].tcl_comp_ring,
505	type: HAL_WBM2SW_RELEASE, ring_num: tx_comp_ring_num, mac_id: 0,
506	DP_TX_COMP_RING_SIZE);
507	if (ret) {
508	ath12k_warn(ab, fmt: "failed to set up tcl_comp ring (%d) :%d\n",
509	tx_comp_ring_num, ret);
510	goto err;
511	}
512	}
513
514	ret = ath12k_dp_srng_setup(ab, ring: &dp->reo_reinject_ring, type: HAL_REO_REINJECT,
515	ring_num: 0, mac_id: 0, DP_REO_REINJECT_RING_SIZE);
516	if (ret) {
517	ath12k_warn(ab, fmt: "failed to set up reo_reinject ring :%d\n",
518	ret);
519	goto err;
520	}
521
522	ret = ath12k_dp_srng_setup(ab, ring: &dp->rx_rel_ring, type: HAL_WBM2SW_RELEASE,
523	HAL_WBM2SW_REL_ERR_RING_NUM, mac_id: 0,
524	DP_RX_RELEASE_RING_SIZE);
525	if (ret) {
526	ath12k_warn(ab, fmt: "failed to set up rx_rel ring :%d\n", ret);
527	goto err;
528	}
529
530	ret = ath12k_dp_srng_setup(ab, ring: &dp->reo_except_ring, type: HAL_REO_EXCEPTION,
531	ring_num: 0, mac_id: 0, DP_REO_EXCEPTION_RING_SIZE);
532	if (ret) {
533	ath12k_warn(ab, fmt: "failed to set up reo_exception ring :%d\n",
534	ret);
535	goto err;
536	}
537
538	ret = ath12k_dp_srng_setup(ab, ring: &dp->reo_cmd_ring, type: HAL_REO_CMD,
539	ring_num: 0, mac_id: 0, DP_REO_CMD_RING_SIZE);
540	if (ret) {
541	ath12k_warn(ab, fmt: "failed to set up reo_cmd ring :%d\n", ret);
542	goto err;
543	}
544
545	srng = &ab->hal.srng_list[dp->reo_cmd_ring.ring_id];
546	ath12k_hal_reo_init_cmd_ring(ab, srng);
547
548	ret = ath12k_dp_srng_setup(ab, ring: &dp->reo_status_ring, type: HAL_REO_STATUS,
549	ring_num: 0, mac_id: 0, DP_REO_STATUS_RING_SIZE);
550	if (ret) {
551	ath12k_warn(ab, fmt: "failed to set up reo_status ring :%d\n", ret);
552	goto err;
553	}
554
555	/* When hash based routing of rx packet is enabled, 32 entries to map
556	* the hash values to the ring will be configured. Each hash entry uses
557	* four bits to map to a particular ring. The ring mapping will be
558	* 0:TCL, 1:SW1, 2:SW2, 3:SW3, 4:SW4, 5:Release, 6:FW and 7:SW5
559	* 8:SW6, 9:SW7, 10:SW8, 11:Not used.
560	*/
561	ring_hash_map = HAL_HASH_ROUTING_RING_SW1 |
562	HAL_HASH_ROUTING_RING_SW2 << 4 |
563	HAL_HASH_ROUTING_RING_SW3 << 8 |
564	HAL_HASH_ROUTING_RING_SW4 << 12 |
565	HAL_HASH_ROUTING_RING_SW1 << 16 |
566	HAL_HASH_ROUTING_RING_SW2 << 20 |
567	HAL_HASH_ROUTING_RING_SW3 << 24 |
568	HAL_HASH_ROUTING_RING_SW4 << 28;
569
570	ath12k_hal_reo_hw_setup(ab, ring_hash_map);
571
572	return 0;
573
574	err:
575	ath12k_dp_srng_common_cleanup(ab);
576
577	return ret;
578	}
579
580	static void ath12k_dp_scatter_idle_link_desc_cleanup(struct ath12k_base *ab)
581	{
582	struct ath12k_dp *dp = &ab->dp;
583	struct hal_wbm_idle_scatter_list *slist = dp->scatter_list;
584	int i;
585
586	for (i = 0; i < DP_IDLE_SCATTER_BUFS_MAX; i++) {
587	if (!slist[i].vaddr)
588	continue;
589
590	dma_free_coherent(dev: ab->dev, HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX,
591	cpu_addr: slist[i].vaddr, dma_handle: slist[i].paddr);
592	slist[i].vaddr = NULL;
593	}
594	}
595
596	static int ath12k_dp_scatter_idle_link_desc_setup(struct ath12k_base *ab,
597	int size,
598	u32 n_link_desc_bank,
599	u32 n_link_desc,
600	u32 last_bank_sz)
601	{
602	struct ath12k_dp *dp = &ab->dp;
603	struct dp_link_desc_bank *link_desc_banks = dp->link_desc_banks;
604	struct hal_wbm_idle_scatter_list *slist = dp->scatter_list;
605	u32 n_entries_per_buf;
606	int num_scatter_buf, scatter_idx;
607	struct hal_wbm_link_desc *scatter_buf;
608	int align_bytes, n_entries;
609	dma_addr_t paddr;
610	int rem_entries;
611	int i;
612	int ret = 0;
613	u32 end_offset, cookie;
614
615	n_entries_per_buf = HAL_WBM_IDLE_SCATTER_BUF_SIZE /
616	ath12k_hal_srng_get_entrysize(ab, ring_type: HAL_WBM_IDLE_LINK);
617	num_scatter_buf = DIV_ROUND_UP(size, HAL_WBM_IDLE_SCATTER_BUF_SIZE);
618
619	if (num_scatter_buf > DP_IDLE_SCATTER_BUFS_MAX)
620	return -EINVAL;
621
622	for (i = 0; i < num_scatter_buf; i++) {
623	slist[i].vaddr = dma_alloc_coherent(dev: ab->dev,
624	HAL_WBM_IDLE_SCATTER_BUF_SIZE_MAX,
625	dma_handle: &slist[i].paddr, GFP_KERNEL);
626	if (!slist[i].vaddr) {
627	ret = -ENOMEM;
628	goto err;
629	}
630	}
631
632	scatter_idx = 0;
633	scatter_buf = slist[scatter_idx].vaddr;
634	rem_entries = n_entries_per_buf;
635
636	for (i = 0; i < n_link_desc_bank; i++) {
637	align_bytes = link_desc_banks[i].vaddr -
638	link_desc_banks[i].vaddr_unaligned;
639	n_entries = (DP_LINK_DESC_ALLOC_SIZE_THRESH - align_bytes) /
640	HAL_LINK_DESC_SIZE;
641	paddr = link_desc_banks[i].paddr;
642	while (n_entries) {
643	cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i);
644	ath12k_hal_set_link_desc_addr(desc: scatter_buf, cookie, paddr);
645	n_entries--;
646	paddr += HAL_LINK_DESC_SIZE;
647	if (rem_entries) {
648	rem_entries--;
649	scatter_buf++;
650	continue;
651	}
652
653	rem_entries = n_entries_per_buf;
654	scatter_idx++;
655	scatter_buf = slist[scatter_idx].vaddr;
656	}
657	}
658
659	end_offset = (scatter_buf - slist[scatter_idx].vaddr) *
660	sizeof(struct hal_wbm_link_desc);
661	ath12k_hal_setup_link_idle_list(ab, sbuf: slist, nsbufs: num_scatter_buf,
662	tot_link_desc: n_link_desc, end_offset);
663
664	return 0;
665
666	err:
667	ath12k_dp_scatter_idle_link_desc_cleanup(ab);
668
669	return ret;
670	}
671
672	static void
673	ath12k_dp_link_desc_bank_free(struct ath12k_base *ab,
674	struct dp_link_desc_bank *link_desc_banks)
675	{
676	int i;
677
678	for (i = 0; i < DP_LINK_DESC_BANKS_MAX; i++) {
679	if (link_desc_banks[i].vaddr_unaligned) {
680	dma_free_coherent(dev: ab->dev,
681	size: link_desc_banks[i].size,
682	cpu_addr: link_desc_banks[i].vaddr_unaligned,
683	dma_handle: link_desc_banks[i].paddr_unaligned);
684	link_desc_banks[i].vaddr_unaligned = NULL;
685	}
686	}
687	}
688
689	static int ath12k_dp_link_desc_bank_alloc(struct ath12k_base *ab,
690	struct dp_link_desc_bank *desc_bank,
691	int n_link_desc_bank,
692	int last_bank_sz)
693	{
694	struct ath12k_dp *dp = &ab->dp;
695	int i;
696	int ret = 0;
697	int desc_sz = DP_LINK_DESC_ALLOC_SIZE_THRESH;
698
699	for (i = 0; i < n_link_desc_bank; i++) {
700	if (i == (n_link_desc_bank - 1) && last_bank_sz)
701	desc_sz = last_bank_sz;
702
703	desc_bank[i].vaddr_unaligned =
704	dma_alloc_coherent(dev: ab->dev, size: desc_sz,
705	dma_handle: &desc_bank[i].paddr_unaligned,
706	GFP_KERNEL);
707	if (!desc_bank[i].vaddr_unaligned) {
708	ret = -ENOMEM;
709	goto err;
710	}
711
712	desc_bank[i].vaddr = PTR_ALIGN(desc_bank[i].vaddr_unaligned,
713	HAL_LINK_DESC_ALIGN);
714	desc_bank[i].paddr = desc_bank[i].paddr_unaligned +
715	((unsigned long)desc_bank[i].vaddr -
716	(unsigned long)desc_bank[i].vaddr_unaligned);
717	desc_bank[i].size = desc_sz;
718	}
719
720	return 0;
721
722	err:
723	ath12k_dp_link_desc_bank_free(ab, link_desc_banks: dp->link_desc_banks);
724
725	return ret;
726	}
727
728	void ath12k_dp_link_desc_cleanup(struct ath12k_base *ab,
729	struct dp_link_desc_bank *desc_bank,
730	u32 ring_type, struct dp_srng *ring)
731	{
732	ath12k_dp_link_desc_bank_free(ab, link_desc_banks: desc_bank);
733
734	if (ring_type != HAL_RXDMA_MONITOR_DESC) {
735	ath12k_dp_srng_cleanup(ab, ring);
736	ath12k_dp_scatter_idle_link_desc_cleanup(ab);
737	}
738	}
739
740	static int ath12k_wbm_idle_ring_setup(struct ath12k_base *ab, u32 *n_link_desc)
741	{
742	struct ath12k_dp *dp = &ab->dp;
743	u32 n_mpdu_link_desc, n_mpdu_queue_desc;
744	u32 n_tx_msdu_link_desc, n_rx_msdu_link_desc;
745	int ret = 0;
746
747	n_mpdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX) /
748	HAL_NUM_MPDUS_PER_LINK_DESC;
749
750	n_mpdu_queue_desc = n_mpdu_link_desc /
751	HAL_NUM_MPDU_LINKS_PER_QUEUE_DESC;
752
753	n_tx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_FLOWS_PER_TID *
754	DP_AVG_MSDUS_PER_FLOW) /
755	HAL_NUM_TX_MSDUS_PER_LINK_DESC;
756
757	n_rx_msdu_link_desc = (DP_NUM_TIDS_MAX * DP_AVG_MPDUS_PER_TID_MAX *
758	DP_AVG_MSDUS_PER_MPDU) /
759	HAL_NUM_RX_MSDUS_PER_LINK_DESC;
760
761	*n_link_desc = n_mpdu_link_desc + n_mpdu_queue_desc +
762	n_tx_msdu_link_desc + n_rx_msdu_link_desc;
763
764	if (*n_link_desc & (*n_link_desc - 1))
765	*n_link_desc = 1 << fls(x: *n_link_desc);
766
767	ret = ath12k_dp_srng_setup(ab, ring: &dp->wbm_idle_ring,
768	type: HAL_WBM_IDLE_LINK, ring_num: 0, mac_id: 0, num_entries: *n_link_desc);
769	if (ret) {
770	ath12k_warn(ab, fmt: "failed to setup wbm_idle_ring: %d\n", ret);
771	return ret;
772	}
773	return ret;
774	}
775
776	int ath12k_dp_link_desc_setup(struct ath12k_base *ab,
777	struct dp_link_desc_bank *link_desc_banks,
778	u32 ring_type, struct hal_srng *srng,
779	u32 n_link_desc)
780	{
781	u32 tot_mem_sz;
782	u32 n_link_desc_bank, last_bank_sz;
783	u32 entry_sz, align_bytes, n_entries;
784	struct hal_wbm_link_desc *desc;
785	u32 paddr;
786	int i, ret;
787	u32 cookie;
788
789	tot_mem_sz = n_link_desc * HAL_LINK_DESC_SIZE;
790	tot_mem_sz += HAL_LINK_DESC_ALIGN;
791
792	if (tot_mem_sz <= DP_LINK_DESC_ALLOC_SIZE_THRESH) {
793	n_link_desc_bank = 1;
794	last_bank_sz = tot_mem_sz;
795	} else {
796	n_link_desc_bank = tot_mem_sz /
797	(DP_LINK_DESC_ALLOC_SIZE_THRESH -
798	HAL_LINK_DESC_ALIGN);
799	last_bank_sz = tot_mem_sz %
800	(DP_LINK_DESC_ALLOC_SIZE_THRESH -
801	HAL_LINK_DESC_ALIGN);
802
803	if (last_bank_sz)
804	n_link_desc_bank += 1;
805	}
806
807	if (n_link_desc_bank > DP_LINK_DESC_BANKS_MAX)
808	return -EINVAL;
809
810	ret = ath12k_dp_link_desc_bank_alloc(ab, desc_bank: link_desc_banks,
811	n_link_desc_bank, last_bank_sz);
812	if (ret)
813	return ret;
814
815	/* Setup link desc idle list for HW internal usage */
816	entry_sz = ath12k_hal_srng_get_entrysize(ab, ring_type);
817	tot_mem_sz = entry_sz * n_link_desc;
818
819	/* Setup scatter desc list when the total memory requirement is more */
820	if (tot_mem_sz > DP_LINK_DESC_ALLOC_SIZE_THRESH &&
821	ring_type != HAL_RXDMA_MONITOR_DESC) {
822	ret = ath12k_dp_scatter_idle_link_desc_setup(ab, size: tot_mem_sz,
823	n_link_desc_bank,
824	n_link_desc,
825	last_bank_sz);
826	if (ret) {
827	ath12k_warn(ab, fmt: "failed to setup scatting idle list descriptor :%d\n",
828	ret);
829	goto fail_desc_bank_free;
830	}
831
832	return 0;
833	}
834
835	spin_lock_bh(lock: &srng->lock);
836
837	ath12k_hal_srng_access_begin(ab, srng);
838
839	for (i = 0; i < n_link_desc_bank; i++) {
840	align_bytes = link_desc_banks[i].vaddr -
841	link_desc_banks[i].vaddr_unaligned;
842	n_entries = (link_desc_banks[i].size - align_bytes) /
843	HAL_LINK_DESC_SIZE;
844	paddr = link_desc_banks[i].paddr;
845	while (n_entries &&
846	(desc = ath12k_hal_srng_src_get_next_entry(ab, srng))) {
847	cookie = DP_LINK_DESC_COOKIE_SET(n_entries, i);
848	ath12k_hal_set_link_desc_addr(desc,
849	cookie, paddr);
850	n_entries--;
851	paddr += HAL_LINK_DESC_SIZE;
852	}
853	}
854
855	ath12k_hal_srng_access_end(ab, srng);
856
857	spin_unlock_bh(lock: &srng->lock);
858
859	return 0;
860
861	fail_desc_bank_free:
862	ath12k_dp_link_desc_bank_free(ab, link_desc_banks);
863
864	return ret;
865	}
866
867	int ath12k_dp_service_srng(struct ath12k_base *ab,
868	struct ath12k_ext_irq_grp *irq_grp,
869	int budget)
870	{
871	struct napi_struct *napi = &irq_grp->napi;
872	int grp_id = irq_grp->grp_id;
873	int work_done = 0;
874	int i = 0, j;
875	int tot_work_done = 0;
876	enum dp_monitor_mode monitor_mode;
877	u8 ring_mask;
878
879	while (i < ab->hw_params->max_tx_ring) {
880	if (ab->hw_params->ring_mask->tx[grp_id] &
881	BIT(ab->hw_params->hal_ops->tcl_to_wbm_rbm_map[i].wbm_ring_num))
882	ath12k_dp_tx_completion_handler(ab, ring_id: i);
883	i++;
884	}
885
886	if (ab->hw_params->ring_mask->rx_err[grp_id]) {
887	work_done = ath12k_dp_rx_process_err(ab, napi, budget);
888	budget -= work_done;
889	tot_work_done += work_done;
890	if (budget <= 0)
891	goto done;
892	}
893
894	if (ab->hw_params->ring_mask->rx_wbm_rel[grp_id]) {
895	work_done = ath12k_dp_rx_process_wbm_err(ab,
896	napi,
897	budget);
898	budget -= work_done;
899	tot_work_done += work_done;
900
901	if (budget <= 0)
902	goto done;
903	}
904
905	if (ab->hw_params->ring_mask->rx[grp_id]) {
906	i = fls(x: ab->hw_params->ring_mask->rx[grp_id]) - 1;
907	work_done = ath12k_dp_rx_process(ab, mac_id: i, napi,
908	budget);
909	budget -= work_done;
910	tot_work_done += work_done;
911	if (budget <= 0)
912	goto done;
913	}
914
915	if (ab->hw_params->ring_mask->rx_mon_dest[grp_id]) {
916	monitor_mode = ATH12K_DP_RX_MONITOR_MODE;
917	ring_mask = ab->hw_params->ring_mask->rx_mon_dest[grp_id];
918	for (i = 0; i < ab->num_radios; i++) {
919	for (j = 0; j < ab->hw_params->num_rxmda_per_pdev; j++) {
920	int id = i * ab->hw_params->num_rxmda_per_pdev + j;
921
922	if (ring_mask & BIT(id)) {
923	work_done =
924	ath12k_dp_mon_process_ring(ab, mac_id: id, napi, budget,
925	monitor_mode);
926	budget -= work_done;
927	tot_work_done += work_done;
928
929	if (budget <= 0)
930	goto done;
931	}
932	}
933	}
934	}
935
936	if (ab->hw_params->ring_mask->tx_mon_dest[grp_id]) {
937	monitor_mode = ATH12K_DP_TX_MONITOR_MODE;
938	ring_mask = ab->hw_params->ring_mask->tx_mon_dest[grp_id];
939	for (i = 0; i < ab->num_radios; i++) {
940	for (j = 0; j < ab->hw_params->num_rxmda_per_pdev; j++) {
941	int id = i * ab->hw_params->num_rxmda_per_pdev + j;
942
943	if (ring_mask & BIT(id)) {
944	work_done =
945	ath12k_dp_mon_process_ring(ab, mac_id: id, napi, budget,
946	monitor_mode);
947	budget -= work_done;
948	tot_work_done += work_done;
949
950	if (budget <= 0)
951	goto done;
952	}
953	}
954	}
955	}
956
957	if (ab->hw_params->ring_mask->reo_status[grp_id])
958	ath12k_dp_rx_process_reo_status(ab);
959
960	if (ab->hw_params->ring_mask->host2rxdma[grp_id]) {
961	struct ath12k_dp *dp = &ab->dp;
962	struct dp_rxdma_ring *rx_ring = &dp->rx_refill_buf_ring;
963
964	ath12k_dp_rx_bufs_replenish(ab, rx_ring, req_entries: 0);
965	}
966
967	/* TODO: Implement handler for other interrupts */
968
969	done:
970	return tot_work_done;
971	}
972
973	void ath12k_dp_pdev_free(struct ath12k_base *ab)
974	{
975	int i;
976
977	del_timer_sync(timer: &ab->mon_reap_timer);
978
979	for (i = 0; i < ab->num_radios; i++)
980	ath12k_dp_rx_pdev_free(ab, pdev_idx: i);
981	}
982
983	void ath12k_dp_pdev_pre_alloc(struct ath12k_base *ab)
984	{
985	struct ath12k *ar;
986	struct ath12k_pdev_dp *dp;
987	int i;
988
989	for (i = 0; i < ab->num_radios; i++) {
990	ar = ab->pdevs[i].ar;
991	dp = &ar->dp;
992	dp->mac_id = i;
993	atomic_set(v: &dp->num_tx_pending, i: 0);
994	init_waitqueue_head(&dp->tx_empty_waitq);
995
996	/* TODO: Add any RXDMA setup required per pdev */
997	}
998	}
999
1000	bool ath12k_dp_wmask_compaction_rx_tlv_supported(struct ath12k_base *ab)
1001	{
1002	if (test_bit(WMI_TLV_SERVICE_WMSK_COMPACTION_RX_TLVS, ab->wmi_ab.svc_map) &&
1003	ab->hw_params->hal_ops->rxdma_ring_wmask_rx_mpdu_start &&
1004	ab->hw_params->hal_ops->rxdma_ring_wmask_rx_msdu_end &&
1005	ab->hw_params->hal_ops->get_hal_rx_compact_ops) {
1006	return true;
1007	}
1008	return false;
1009	}
1010
1011	void ath12k_dp_hal_rx_desc_init(struct ath12k_base *ab)
1012	{
1013	if (ath12k_dp_wmask_compaction_rx_tlv_supported(ab)) {
1014	/* RX TLVS compaction is supported, hence change the hal_rx_ops
1015	* to compact hal_rx_ops.
1016	*/
1017	ab->hal_rx_ops = ab->hw_params->hal_ops->get_hal_rx_compact_ops();
1018	}
1019	ab->hal.hal_desc_sz =
1020	ab->hal_rx_ops->rx_desc_get_desc_size();
1021	}
1022
1023	static void ath12k_dp_service_mon_ring(struct timer_list *t)
1024	{
1025	struct ath12k_base *ab = from_timer(ab, t, mon_reap_timer);
1026	int i;
1027
1028	for (i = 0; i < ab->hw_params->num_rxmda_per_pdev; i++)
1029	ath12k_dp_mon_process_ring(ab, mac_id: i, NULL, DP_MON_SERVICE_BUDGET,
1030	monitor_mode: ATH12K_DP_RX_MONITOR_MODE);
1031
1032	mod_timer(timer: &ab->mon_reap_timer, expires: jiffies +
1033	msecs_to_jiffies(ATH12K_MON_TIMER_INTERVAL));
1034	}
1035
1036	static void ath12k_dp_mon_reap_timer_init(struct ath12k_base *ab)
1037	{
1038	if (ab->hw_params->rxdma1_enable)
1039	return;
1040
1041	timer_setup(&ab->mon_reap_timer, ath12k_dp_service_mon_ring, 0);
1042	}
1043
1044	int ath12k_dp_pdev_alloc(struct ath12k_base *ab)
1045	{
1046	struct ath12k *ar;
1047	int ret;
1048	int i;
1049
1050	ret = ath12k_dp_rx_htt_setup(ab);
1051	if (ret)
1052	goto out;
1053
1054	ath12k_dp_mon_reap_timer_init(ab);
1055
1056	/* TODO: Per-pdev rx ring unlike tx ring which is mapped to different AC's */
1057	for (i = 0; i < ab->num_radios; i++) {
1058	ar = ab->pdevs[i].ar;
1059	ret = ath12k_dp_rx_pdev_alloc(ab, pdev_idx: i);
1060	if (ret) {
1061	ath12k_warn(ab, fmt: "failed to allocate pdev rx for pdev_id :%d\n",
1062	i);
1063	goto err;
1064	}
1065	ret = ath12k_dp_rx_pdev_mon_attach(ar);
1066	if (ret) {
1067	ath12k_warn(ab, fmt: "failed to initialize mon pdev %d\n", i);
1068	goto err;
1069	}
1070	}
1071
1072	return 0;
1073	err:
1074	ath12k_dp_pdev_free(ab);
1075	out:
1076	return ret;
1077	}
1078
1079	int ath12k_dp_htt_connect(struct ath12k_dp *dp)
1080	{
1081	struct ath12k_htc_svc_conn_req conn_req = {0};
1082	struct ath12k_htc_svc_conn_resp conn_resp = {0};
1083	int status;
1084
1085	conn_req.ep_ops.ep_tx_complete = ath12k_dp_htt_htc_tx_complete;
1086	conn_req.ep_ops.ep_rx_complete = ath12k_dp_htt_htc_t2h_msg_handler;
1087
1088	/* connect to control service */
1089	conn_req.service_id = ATH12K_HTC_SVC_ID_HTT_DATA_MSG;
1090
1091	status = ath12k_htc_connect_service(htc: &dp->ab->htc, conn_req: &conn_req,
1092	conn_resp: &conn_resp);
1093
1094	if (status)
1095	return status;
1096
1097	dp->eid = conn_resp.eid;
1098
1099	return 0;
1100	}
1101
1102	static void ath12k_dp_update_vdev_search(struct ath12k_vif *arvif)
1103	{
1104	switch (arvif->vdev_type) {
1105	case WMI_VDEV_TYPE_STA:
1106	/* TODO: Verify the search type and flags since ast hash
1107	* is not part of peer mapv3
1108	*/
1109	arvif->hal_addr_search_flags = HAL_TX_ADDRY_EN;
1110	arvif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT;
1111	break;
1112	case WMI_VDEV_TYPE_AP:
1113	case WMI_VDEV_TYPE_IBSS:
1114	arvif->hal_addr_search_flags = HAL_TX_ADDRX_EN;
1115	arvif->search_type = HAL_TX_ADDR_SEARCH_DEFAULT;
1116	break;
1117	case WMI_VDEV_TYPE_MONITOR:
1118	default:
1119	return;
1120	}
1121	}
1122
1123	void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_vif *arvif)
1124	{
1125	struct ath12k_base *ab = ar->ab;
1126
1127	arvif->tcl_metadata |= u32_encode_bits(v: 1, HTT_TCL_META_DATA_TYPE) |
1128	u32_encode_bits(v: arvif->vdev_id,
1129	HTT_TCL_META_DATA_VDEV_ID) |
1130	u32_encode_bits(v: ar->pdev->pdev_id,
1131	HTT_TCL_META_DATA_PDEV_ID);
1132
1133	/* set HTT extension valid bit to 0 by default */
1134	arvif->tcl_metadata &= ~HTT_TCL_META_DATA_VALID_HTT;
1135
1136	ath12k_dp_update_vdev_search(arvif);
1137	arvif->vdev_id_check_en = true;
1138	arvif->bank_id = ath12k_dp_tx_get_bank_profile(ab, arvif, dp: &ab->dp);
1139
1140	/* TODO: error path for bank id failure */
1141	if (arvif->bank_id == DP_INVALID_BANK_ID) {
1142	ath12k_err(ab: ar->ab, fmt: "Failed to initialize DP TX Banks");
1143	return;
1144	}
1145	}
1146
1147	static void ath12k_dp_cc_cleanup(struct ath12k_base *ab)
1148	{
1149	struct ath12k_rx_desc_info *desc_info, *tmp;
1150	struct ath12k_tx_desc_info *tx_desc_info, *tmp1;
1151	struct ath12k_dp *dp = &ab->dp;
1152	struct sk_buff *skb;
1153	int i;
1154	u32 pool_id, tx_spt_page;
1155
1156	if (!dp->spt_info)
1157	return;
1158
1159	/* RX Descriptor cleanup */
1160	spin_lock_bh(lock: &dp->rx_desc_lock);
1161
1162	list_for_each_entry_safe(desc_info, tmp, &dp->rx_desc_used_list, list) {
1163	list_del(entry: &desc_info->list);
1164	skb = desc_info->skb;
1165
1166	if (!skb)
1167	continue;
1168
1169	dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(skb)->paddr,
1170	skb->len + skb_tailroom(skb), DMA_FROM_DEVICE);
1171	dev_kfree_skb_any(skb);
1172	}
1173
1174	for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES; i++) {
1175	if (!dp->spt_info->rxbaddr[i])
1176	continue;
1177
1178	kfree(objp: dp->spt_info->rxbaddr[i]);
1179	dp->spt_info->rxbaddr[i] = NULL;
1180	}
1181
1182	spin_unlock_bh(lock: &dp->rx_desc_lock);
1183
1184	/* TX Descriptor cleanup */
1185	for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) {
1186	spin_lock_bh(lock: &dp->tx_desc_lock[i]);
1187
1188	list_for_each_entry_safe(tx_desc_info, tmp1, &dp->tx_desc_used_list[i],
1189	list) {
1190	list_del(entry: &tx_desc_info->list);
1191	skb = tx_desc_info->skb;
1192
1193	if (!skb)
1194	continue;
1195
1196	dma_unmap_single(ab->dev, ATH12K_SKB_CB(skb)->paddr,
1197	skb->len, DMA_TO_DEVICE);
1198	dev_kfree_skb_any(skb);
1199	}
1200
1201	spin_unlock_bh(lock: &dp->tx_desc_lock[i]);
1202	}
1203
1204	for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) {
1205	spin_lock_bh(lock: &dp->tx_desc_lock[pool_id]);
1206
1207	for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL; i++) {
1208	tx_spt_page = i + pool_id * ATH12K_TX_SPT_PAGES_PER_POOL;
1209	if (!dp->spt_info->txbaddr[tx_spt_page])
1210	continue;
1211
1212	kfree(objp: dp->spt_info->txbaddr[tx_spt_page]);
1213	dp->spt_info->txbaddr[tx_spt_page] = NULL;
1214	}
1215
1216	spin_unlock_bh(lock: &dp->tx_desc_lock[pool_id]);
1217	}
1218
1219	/* unmap SPT pages */
1220	for (i = 0; i < dp->num_spt_pages; i++) {
1221	if (!dp->spt_info[i].vaddr)
1222	continue;
1223
1224	dma_free_coherent(dev: ab->dev, ATH12K_PAGE_SIZE,
1225	cpu_addr: dp->spt_info[i].vaddr, dma_handle: dp->spt_info[i].paddr);
1226	dp->spt_info[i].vaddr = NULL;
1227	}
1228
1229	kfree(objp: dp->spt_info);
1230	}
1231
1232	static void ath12k_dp_reoq_lut_cleanup(struct ath12k_base *ab)
1233	{
1234	struct ath12k_dp *dp = &ab->dp;
1235
1236	if (!ab->hw_params->reoq_lut_support)
1237	return;
1238
1239	if (!dp->reoq_lut.vaddr)
1240	return;
1241
1242	dma_free_coherent(dev: ab->dev, DP_REOQ_LUT_SIZE,
1243	cpu_addr: dp->reoq_lut.vaddr, dma_handle: dp->reoq_lut.paddr);
1244	dp->reoq_lut.vaddr = NULL;
1245
1246	ath12k_hif_write32(ab,
1247	HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE0(ab), data: 0);
1248	}
1249
1250	void ath12k_dp_free(struct ath12k_base *ab)
1251	{
1252	struct ath12k_dp *dp = &ab->dp;
1253	int i;
1254
1255	ath12k_dp_link_desc_cleanup(ab, desc_bank: dp->link_desc_banks,
1256	ring_type: HAL_WBM_IDLE_LINK, ring: &dp->wbm_idle_ring);
1257
1258	ath12k_dp_cc_cleanup(ab);
1259	ath12k_dp_reoq_lut_cleanup(ab);
1260	ath12k_dp_deinit_bank_profiles(ab);
1261	ath12k_dp_srng_common_cleanup(ab);
1262
1263	ath12k_dp_rx_reo_cmd_list_cleanup(ab);
1264
1265	for (i = 0; i < ab->hw_params->max_tx_ring; i++)
1266	kfree(objp: dp->tx_ring[i].tx_status);
1267
1268	ath12k_dp_rx_free(ab);
1269	/* Deinit any SOC level resource */
1270	}
1271
1272	void ath12k_dp_cc_config(struct ath12k_base *ab)
1273	{
1274	u32 cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start;
1275	u32 reo_base = HAL_SEQ_WCSS_UMAC_REO_REG;
1276	u32 wbm_base = HAL_SEQ_WCSS_UMAC_WBM_REG;
1277	u32 val = 0;
1278
1279	ath12k_hif_write32(ab, address: reo_base + HAL_REO1_SW_COOKIE_CFG0(ab), data: cmem_base);
1280
1281	val |= u32_encode_bits(ATH12K_CMEM_ADDR_MSB,
1282	HAL_REO1_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) |
1283	u32_encode_bits(ATH12K_CC_PPT_MSB,
1284	HAL_REO1_SW_COOKIE_CFG_COOKIE_PPT_MSB) |
1285	u32_encode_bits(ATH12K_CC_SPT_MSB,
1286	HAL_REO1_SW_COOKIE_CFG_COOKIE_SPT_MSB) |
1287	u32_encode_bits(v: 1, HAL_REO1_SW_COOKIE_CFG_ALIGN) |
1288	u32_encode_bits(v: 1, HAL_REO1_SW_COOKIE_CFG_ENABLE) |
1289	u32_encode_bits(v: 1, HAL_REO1_SW_COOKIE_CFG_GLOBAL_ENABLE);
1290
1291	ath12k_hif_write32(ab, address: reo_base + HAL_REO1_SW_COOKIE_CFG1(ab), data: val);
1292
1293	/* Enable HW CC for WBM */
1294	ath12k_hif_write32(ab, address: wbm_base + HAL_WBM_SW_COOKIE_CFG0, data: cmem_base);
1295
1296	val = u32_encode_bits(ATH12K_CMEM_ADDR_MSB,
1297	HAL_WBM_SW_COOKIE_CFG_CMEM_BASE_ADDR_MSB) |
1298	u32_encode_bits(ATH12K_CC_PPT_MSB,
1299	HAL_WBM_SW_COOKIE_CFG_COOKIE_PPT_MSB) |
1300	u32_encode_bits(ATH12K_CC_SPT_MSB,
1301	HAL_WBM_SW_COOKIE_CFG_COOKIE_SPT_MSB) |
1302	u32_encode_bits(v: 1, HAL_WBM_SW_COOKIE_CFG_ALIGN);
1303
1304	ath12k_hif_write32(ab, address: wbm_base + HAL_WBM_SW_COOKIE_CFG1, data: val);
1305
1306	/* Enable conversion complete indication */
1307	val = ath12k_hif_read32(ab, address: wbm_base + HAL_WBM_SW_COOKIE_CFG2);
1308	val |= u32_encode_bits(v: 1, HAL_WBM_SW_COOKIE_CFG_RELEASE_PATH_EN) |
1309	u32_encode_bits(v: 1, HAL_WBM_SW_COOKIE_CFG_ERR_PATH_EN) |
1310	u32_encode_bits(v: 1, HAL_WBM_SW_COOKIE_CFG_CONV_IND_EN);
1311
1312	ath12k_hif_write32(ab, address: wbm_base + HAL_WBM_SW_COOKIE_CFG2, data: val);
1313
1314	/* Enable Cookie conversion for WBM2SW Rings */
1315	val = ath12k_hif_read32(ab, address: wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG);
1316	val |= u32_encode_bits(v: 1, HAL_WBM_SW_COOKIE_CONV_CFG_GLOBAL_EN) |
1317	ab->hw_params->hal_params->wbm2sw_cc_enable;
1318
1319	ath12k_hif_write32(ab, address: wbm_base + HAL_WBM_SW_COOKIE_CONVERT_CFG, data: val);
1320	}
1321
1322	static u32 ath12k_dp_cc_cookie_gen(u16 ppt_idx, u16 spt_idx)
1323	{
1324	return (u32)ppt_idx << ATH12K_CC_PPT_SHIFT | spt_idx;
1325	}
1326
1327	static inline void *ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_base *ab,
1328	u16 ppt_idx, u16 spt_idx)
1329	{
1330	struct ath12k_dp *dp = &ab->dp;
1331
1332	return dp->spt_info[ppt_idx].vaddr + spt_idx;
1333	}
1334
1335	struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_base *ab,
1336	u32 cookie)
1337	{
1338	struct ath12k_rx_desc_info **desc_addr_ptr;
1339	u16 ppt_idx, spt_idx;
1340
1341	ppt_idx = u32_get_bits(v: cookie, ATH12K_DP_CC_COOKIE_PPT);
1342	spt_idx = u32_get_bits(v: cookie, ATH12k_DP_CC_COOKIE_SPT);
1343
1344	if (ppt_idx > ATH12K_NUM_RX_SPT_PAGES ||
1345	spt_idx > ATH12K_MAX_SPT_ENTRIES)
1346	return NULL;
1347
1348	desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx);
1349
1350	return *desc_addr_ptr;
1351	}
1352
1353	struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_base *ab,
1354	u32 cookie)
1355	{
1356	struct ath12k_tx_desc_info **desc_addr_ptr;
1357	u16 ppt_idx, spt_idx;
1358
1359	ppt_idx = u32_get_bits(v: cookie, ATH12K_DP_CC_COOKIE_PPT);
1360	spt_idx = u32_get_bits(v: cookie, ATH12k_DP_CC_COOKIE_SPT);
1361
1362	if (ppt_idx < ATH12K_NUM_RX_SPT_PAGES ||
1363	ppt_idx > ab->dp.num_spt_pages ||
1364	spt_idx > ATH12K_MAX_SPT_ENTRIES)
1365	return NULL;
1366
1367	desc_addr_ptr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx);
1368
1369	return *desc_addr_ptr;
1370	}
1371
1372	static int ath12k_dp_cc_desc_init(struct ath12k_base *ab)
1373	{
1374	struct ath12k_dp *dp = &ab->dp;
1375	struct ath12k_rx_desc_info *rx_descs, **rx_desc_addr;
1376	struct ath12k_tx_desc_info *tx_descs, **tx_desc_addr;
1377	u32 i, j, pool_id, tx_spt_page;
1378	u32 ppt_idx;
1379
1380	spin_lock_bh(lock: &dp->rx_desc_lock);
1381
1382	/* First ATH12K_NUM_RX_SPT_PAGES of allocated SPT pages are used for RX */
1383	for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES; i++) {
1384	rx_descs = kcalloc(ATH12K_MAX_SPT_ENTRIES, size: sizeof(*rx_descs),
1385	GFP_ATOMIC);
1386
1387	if (!rx_descs) {
1388	spin_unlock_bh(lock: &dp->rx_desc_lock);
1389	return -ENOMEM;
1390	}
1391
1392	dp->spt_info->rxbaddr[i] = &rx_descs[0];
1393
1394	for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) {
1395	rx_descs[j].cookie = ath12k_dp_cc_cookie_gen(ppt_idx: i, spt_idx: j);
1396	rx_descs[j].magic = ATH12K_DP_RX_DESC_MAGIC;
1397	list_add_tail(new: &rx_descs[j].list, head: &dp->rx_desc_free_list);
1398
1399	/* Update descriptor VA in SPT */
1400	rx_desc_addr = ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx: i, spt_idx: j);
1401	*rx_desc_addr = &rx_descs[j];
1402	}
1403	}
1404
1405	spin_unlock_bh(lock: &dp->rx_desc_lock);
1406
1407	for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) {
1408	spin_lock_bh(lock: &dp->tx_desc_lock[pool_id]);
1409	for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL; i++) {
1410	tx_descs = kcalloc(ATH12K_MAX_SPT_ENTRIES, size: sizeof(*tx_descs),
1411	GFP_ATOMIC);
1412
1413	if (!tx_descs) {
1414	spin_unlock_bh(lock: &dp->tx_desc_lock[pool_id]);
1415	/* Caller takes care of TX pending and RX desc cleanup */
1416	return -ENOMEM;
1417	}
1418
1419	tx_spt_page = i + pool_id * ATH12K_TX_SPT_PAGES_PER_POOL;
1420	dp->spt_info->txbaddr[tx_spt_page] = &tx_descs[0];
1421
1422	for (j = 0; j < ATH12K_MAX_SPT_ENTRIES; j++) {
1423	ppt_idx = ATH12K_NUM_RX_SPT_PAGES + tx_spt_page;
1424	tx_descs[j].desc_id = ath12k_dp_cc_cookie_gen(ppt_idx, spt_idx: j);
1425	tx_descs[j].pool_id = pool_id;
1426	list_add_tail(new: &tx_descs[j].list,
1427	head: &dp->tx_desc_free_list[pool_id]);
1428
1429	/* Update descriptor VA in SPT */
1430	tx_desc_addr =
1431	ath12k_dp_cc_get_desc_addr_ptr(ab, ppt_idx, spt_idx: j);
1432	*tx_desc_addr = &tx_descs[j];
1433	}
1434	}
1435	spin_unlock_bh(lock: &dp->tx_desc_lock[pool_id]);
1436	}
1437	return 0;
1438	}
1439
1440	static int ath12k_dp_cc_init(struct ath12k_base *ab)
1441	{
1442	struct ath12k_dp *dp = &ab->dp;
1443	int i, ret = 0;
1444	u32 cmem_base;
1445
1446	INIT_LIST_HEAD(list: &dp->rx_desc_free_list);
1447	INIT_LIST_HEAD(list: &dp->rx_desc_used_list);
1448	spin_lock_init(&dp->rx_desc_lock);
1449
1450	for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++) {
1451	INIT_LIST_HEAD(list: &dp->tx_desc_free_list[i]);
1452	INIT_LIST_HEAD(list: &dp->tx_desc_used_list[i]);
1453	spin_lock_init(&dp->tx_desc_lock[i]);
1454	}
1455
1456	dp->num_spt_pages = ATH12K_NUM_SPT_PAGES;
1457	if (dp->num_spt_pages > ATH12K_MAX_PPT_ENTRIES)
1458	dp->num_spt_pages = ATH12K_MAX_PPT_ENTRIES;
1459
1460	dp->spt_info = kcalloc(n: dp->num_spt_pages, size: sizeof(struct ath12k_spt_info),
1461	GFP_KERNEL);
1462
1463	if (!dp->spt_info) {
1464	ath12k_warn(ab, fmt: "SPT page allocation failure");
1465	return -ENOMEM;
1466	}
1467
1468	cmem_base = ab->qmi.dev_mem[ATH12K_QMI_DEVMEM_CMEM_INDEX].start;
1469
1470	for (i = 0; i < dp->num_spt_pages; i++) {
1471	dp->spt_info[i].vaddr = dma_alloc_coherent(dev: ab->dev,
1472	ATH12K_PAGE_SIZE,
1473	dma_handle: &dp->spt_info[i].paddr,
1474	GFP_KERNEL);
1475
1476	if (!dp->spt_info[i].vaddr) {
1477	ret = -ENOMEM;
1478	goto free;
1479	}
1480
1481	if (dp->spt_info[i].paddr & ATH12K_SPT_4K_ALIGN_CHECK) {
1482	ath12k_warn(ab, fmt: "SPT allocated memory is not 4K aligned");
1483	ret = -EINVAL;
1484	goto free;
1485	}
1486
1487	/* Write to PPT in CMEM */
1488	ath12k_hif_write32(ab, address: cmem_base + ATH12K_PPT_ADDR_OFFSET(i),
1489	data: dp->spt_info[i].paddr >> ATH12K_SPT_4K_ALIGN_OFFSET);
1490	}
1491
1492	ret = ath12k_dp_cc_desc_init(ab);
1493	if (ret) {
1494	ath12k_warn(ab, fmt: "HW CC desc init failed %d", ret);
1495	goto free;
1496	}
1497
1498	return 0;
1499	free:
1500	ath12k_dp_cc_cleanup(ab);
1501	return ret;
1502	}
1503
1504	static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab)
1505	{
1506	struct ath12k_dp *dp = &ab->dp;
1507
1508	if (!ab->hw_params->reoq_lut_support)
1509	return 0;
1510
1511	dp->reoq_lut.vaddr = dma_alloc_coherent(dev: ab->dev,
1512	DP_REOQ_LUT_SIZE,
1513	dma_handle: &dp->reoq_lut.paddr,
1514	GFP_KERNEL | __GFP_ZERO);
1515	if (!dp->reoq_lut.vaddr) {
1516	ath12k_warn(ab, fmt: "failed to allocate memory for reoq table");
1517	return -ENOMEM;
1518	}
1519
1520	ath12k_hif_write32(ab, HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_QDESC_LUT_BASE0(ab),
1521	data: dp->reoq_lut.paddr);
1522	return 0;
1523	}
1524
1525	int ath12k_dp_alloc(struct ath12k_base *ab)
1526	{
1527	struct ath12k_dp *dp = &ab->dp;
1528	struct hal_srng *srng = NULL;
1529	size_t size = 0;
1530	u32 n_link_desc = 0;
1531	int ret;
1532	int i;
1533
1534	dp->ab = ab;
1535
1536	INIT_LIST_HEAD(list: &dp->reo_cmd_list);
1537	INIT_LIST_HEAD(list: &dp->reo_cmd_cache_flush_list);
1538	spin_lock_init(&dp->reo_cmd_lock);
1539
1540	dp->reo_cmd_cache_flush_count = 0;
1541
1542	ret = ath12k_wbm_idle_ring_setup(ab, n_link_desc: &n_link_desc);
1543	if (ret) {
1544	ath12k_warn(ab, fmt: "failed to setup wbm_idle_ring: %d\n", ret);
1545	return ret;
1546	}
1547
1548	srng = &ab->hal.srng_list[dp->wbm_idle_ring.ring_id];
1549
1550	ret = ath12k_dp_link_desc_setup(ab, link_desc_banks: dp->link_desc_banks,
1551	ring_type: HAL_WBM_IDLE_LINK, srng, n_link_desc);
1552	if (ret) {
1553	ath12k_warn(ab, fmt: "failed to setup link desc: %d\n", ret);
1554	return ret;
1555	}
1556
1557	ret = ath12k_dp_cc_init(ab);
1558
1559	if (ret) {
1560	ath12k_warn(ab, fmt: "failed to setup cookie converter %d\n", ret);
1561	goto fail_link_desc_cleanup;
1562	}
1563	ret = ath12k_dp_init_bank_profiles(ab);
1564	if (ret) {
1565	ath12k_warn(ab, fmt: "failed to setup bank profiles %d\n", ret);
1566	goto fail_hw_cc_cleanup;
1567	}
1568
1569	ret = ath12k_dp_srng_common_setup(ab);
1570	if (ret)
1571	goto fail_dp_bank_profiles_cleanup;
1572
1573	size = sizeof(struct hal_wbm_release_ring_tx) * DP_TX_COMP_RING_SIZE;
1574
1575	ret = ath12k_dp_reoq_lut_setup(ab);
1576	if (ret) {
1577	ath12k_warn(ab, fmt: "failed to setup reoq table %d\n", ret);
1578	goto fail_cmn_srng_cleanup;
1579	}
1580
1581	for (i = 0; i < ab->hw_params->max_tx_ring; i++) {
1582	dp->tx_ring[i].tcl_data_ring_id = i;
1583
1584	dp->tx_ring[i].tx_status_head = 0;
1585	dp->tx_ring[i].tx_status_tail = DP_TX_COMP_RING_SIZE - 1;
1586	dp->tx_ring[i].tx_status = kmalloc(size, GFP_KERNEL);
1587	if (!dp->tx_ring[i].tx_status) {
1588	ret = -ENOMEM;
1589	/* FIXME: The allocated tx status is not freed
1590	* properly here
1591	*/
1592	goto fail_cmn_reoq_cleanup;
1593	}
1594	}
1595
1596	for (i = 0; i < HAL_DSCP_TID_MAP_TBL_NUM_ENTRIES_MAX; i++)
1597	ath12k_hal_tx_set_dscp_tid_map(ab, id: i);
1598
1599	ret = ath12k_dp_rx_alloc(ab);
1600	if (ret)
1601	goto fail_dp_rx_free;
1602
1603	/* Init any SOC level resource for DP */
1604
1605	return 0;
1606
1607	fail_dp_rx_free:
1608	ath12k_dp_rx_free(ab);
1609
1610	fail_cmn_reoq_cleanup:
1611	ath12k_dp_reoq_lut_cleanup(ab);
1612
1613	fail_cmn_srng_cleanup:
1614	ath12k_dp_srng_common_cleanup(ab);
1615
1616	fail_dp_bank_profiles_cleanup:
1617	ath12k_dp_deinit_bank_profiles(ab);
1618
1619	fail_hw_cc_cleanup:
1620	ath12k_dp_cc_cleanup(ab);
1621
1622	fail_link_desc_cleanup:
1623	ath12k_dp_link_desc_cleanup(ab, desc_bank: dp->link_desc_banks,
1624	ring_type: HAL_WBM_IDLE_LINK, ring: &dp->wbm_idle_ring);
1625
1626	return ret;
1627	}
1628