1	// SPDX-License-Identifier: ISC
2	/*
3	* Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
4	* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
5	*/
6
7	#include <linux/etherdevice.h>
8	#include <net/ieee80211_radiotap.h>
9	#include <linux/if_arp.h>
10	#include <linux/moduleparam.h>
11	#include <linux/ip.h>
12	#include <linux/ipv6.h>
13	#include <linux/if_vlan.h>
14	#include <net/ipv6.h>
15	#include <linux/prefetch.h>
16
17	#include "wil6210.h"
18	#include "wmi.h"
19	#include "txrx.h"
20	#include "trace.h"
21	#include "txrx_edma.h"
22
23	bool rx_align_2;
24	module_param(rx_align_2, bool, 0444);
25	MODULE_PARM_DESC(rx_align_2, " align Rx buffers on 4*n+2, default - no");
26
27	bool rx_large_buf;
28	module_param(rx_large_buf, bool, 0444);
29	MODULE_PARM_DESC(rx_large_buf, " allocate 8KB RX buffers, default - no");
30
31	/* Drop Tx packets in case Tx ring is full */
32	bool drop_if_ring_full;
33
34	static inline uint wil_rx_snaplen(void)
35	{
36	return rx_align_2 ? 6 : 0;
37	}
38
39	/* wil_ring_wmark_low - low watermark for available descriptor space */
40	static inline int wil_ring_wmark_low(struct wil_ring *ring)
41	{
42	return ring->size / 8;
43	}
44
45	/* wil_ring_wmark_high - high watermark for available descriptor space */
46	static inline int wil_ring_wmark_high(struct wil_ring *ring)
47	{
48	return ring->size / 4;
49	}
50
51	/* returns true if num avail descriptors is lower than wmark_low */
52	static inline int wil_ring_avail_low(struct wil_ring *ring)
53	{
54	return wil_ring_avail_tx(ring) < wil_ring_wmark_low(ring);
55	}
56
57	/* returns true if num avail descriptors is higher than wmark_high */
58	static inline int wil_ring_avail_high(struct wil_ring *ring)
59	{
60	return wil_ring_avail_tx(ring) > wil_ring_wmark_high(ring);
61	}
62
63	/* returns true when all tx vrings are empty */
64	bool wil_is_tx_idle(struct wil6210_priv *wil)
65	{
66	int i;
67	unsigned long data_comp_to;
68	int min_ring_id = wil_get_min_tx_ring_id(wil);
69
70	for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
71	struct wil_ring *vring = &wil->ring_tx[i];
72	int vring_index = vring - wil->ring_tx;
73	struct wil_ring_tx_data *txdata =
74	&wil->ring_tx_data[vring_index];
75
76	spin_lock(lock: &txdata->lock);
77
78	if (!vring->va || !txdata->enabled) {
79	spin_unlock(lock: &txdata->lock);
80	continue;
81	}
82
83	data_comp_to = jiffies + msecs_to_jiffies(
84	WIL_DATA_COMPLETION_TO_MS);
85	if (test_bit(wil_status_napi_en, wil->status)) {
86	while (!wil_ring_is_empty(ring: vring)) {
87	if (time_after(jiffies, data_comp_to)) {
88	wil_dbg_pm(wil,
89	"TO waiting for idle tx\n");
90	spin_unlock(lock: &txdata->lock);
91	return false;
92	}
93	wil_dbg_ratelimited(wil,
94	fmt: "tx vring is not empty -> NAPI\n");
95	spin_unlock(lock: &txdata->lock);
96	napi_synchronize(n: &wil->napi_tx);
97	msleep(msecs: 20);
98	spin_lock(lock: &txdata->lock);
99	if (!vring->va || !txdata->enabled)
100	break;
101	}
102	}
103
104	spin_unlock(lock: &txdata->lock);
105	}
106
107	return true;
108	}
109
110	static int wil_vring_alloc(struct wil6210_priv *wil, struct wil_ring *vring)
111	{
112	struct device *dev = wil_to_dev(wil);
113	size_t sz = vring->size * sizeof(vring->va[0]);
114	uint i;
115
116	wil_dbg_misc(wil, "vring_alloc:\n");
117
118	BUILD_BUG_ON(sizeof(vring->va[0]) != 32);
119
120	vring->swhead = 0;
121	vring->swtail = 0;
122	vring->ctx = kcalloc(n: vring->size, size: sizeof(vring->ctx[0]), GFP_KERNEL);
123	if (!vring->ctx) {
124	vring->va = NULL;
125	return -ENOMEM;
126	}
127
128	/* vring->va should be aligned on its size rounded up to power of 2
129	* This is granted by the dma_alloc_coherent.
130	*
131	* HW has limitation that all vrings addresses must share the same
132	* upper 16 msb bits part of 48 bits address. To workaround that,
133	* if we are using more than 32 bit addresses switch to 32 bit
134	* allocation before allocating vring memory.
135	*
136	* There's no check for the return value of dma_set_mask_and_coherent,
137	* since we assume if we were able to set the mask during
138	* initialization in this system it will not fail if we set it again
139	*/
140	if (wil->dma_addr_size > 32)
141	dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
142
143	vring->va = dma_alloc_coherent(dev, size: sz, dma_handle: &vring->pa, GFP_KERNEL);
144	if (!vring->va) {
145	kfree(objp: vring->ctx);
146	vring->ctx = NULL;
147	return -ENOMEM;
148	}
149
150	if (wil->dma_addr_size > 32)
151	dma_set_mask_and_coherent(dev,
152	DMA_BIT_MASK(wil->dma_addr_size));
153
154	/* initially, all descriptors are SW owned
155	* For Tx and Rx, ownership bit is at the same location, thus
156	* we can use any
157	*/
158	for (i = 0; i < vring->size; i++) {
159	volatile struct vring_tx_desc *_d =
160	&vring->va[i].tx.legacy;
161
162	_d->dma.status = TX_DMA_STATUS_DU;
163	}
164
165	wil_dbg_misc(wil, "vring[%d] 0x%p:%pad 0x%p\n", vring->size,
166	vring->va, &vring->pa, vring->ctx);
167
168	return 0;
169	}
170
171	static void wil_txdesc_unmap(struct device *dev, union wil_tx_desc *desc,
172	struct wil_ctx *ctx)
173	{
174	struct vring_tx_desc *d = &desc->legacy;
175	dma_addr_t pa = wil_desc_addr(addr: &d->dma.addr);
176	u16 dmalen = le16_to_cpu(d->dma.length);
177
178	switch (ctx->mapped_as) {
179	case wil_mapped_as_single:
180	dma_unmap_single(dev, pa, dmalen, DMA_TO_DEVICE);
181	break;
182	case wil_mapped_as_page:
183	dma_unmap_page(dev, pa, dmalen, DMA_TO_DEVICE);
184	break;
185	default:
186	break;
187	}
188	}
189
190	static void wil_vring_free(struct wil6210_priv *wil, struct wil_ring *vring)
191	{
192	struct device *dev = wil_to_dev(wil);
193	size_t sz = vring->size * sizeof(vring->va[0]);
194
195	lockdep_assert_held(&wil->mutex);
196	if (!vring->is_rx) {
197	int vring_index = vring - wil->ring_tx;
198
199	wil_dbg_misc(wil, "free Tx vring %d [%d] 0x%p:%pad 0x%p\n",
200	vring_index, vring->size, vring->va,
201	&vring->pa, vring->ctx);
202	} else {
203	wil_dbg_misc(wil, "free Rx vring [%d] 0x%p:%pad 0x%p\n",
204	vring->size, vring->va,
205	&vring->pa, vring->ctx);
206	}
207
208	while (!wil_ring_is_empty(ring: vring)) {
209	dma_addr_t pa;
210	u16 dmalen;
211	struct wil_ctx *ctx;
212
213	if (!vring->is_rx) {
214	struct vring_tx_desc dd, *d = &dd;
215	volatile struct vring_tx_desc *_d =
216	&vring->va[vring->swtail].tx.legacy;
217
218	ctx = &vring->ctx[vring->swtail];
219	if (!ctx) {
220	wil_dbg_txrx(wil,
221	"ctx(%d) was already completed\n",
222	vring->swtail);
223	vring->swtail = wil_ring_next_tail(ring: vring);
224	continue;
225	}
226	*d = *_d;
227	wil_txdesc_unmap(dev, desc: (union wil_tx_desc *)d, ctx);
228	if (ctx->skb)
229	dev_kfree_skb_any(skb: ctx->skb);
230	vring->swtail = wil_ring_next_tail(ring: vring);
231	} else { /* rx */
232	struct vring_rx_desc dd, *d = &dd;
233	volatile struct vring_rx_desc *_d =
234	&vring->va[vring->swhead].rx.legacy;
235
236	ctx = &vring->ctx[vring->swhead];
237	*d = *_d;
238	pa = wil_desc_addr(addr: &d->dma.addr);
239	dmalen = le16_to_cpu(d->dma.length);
240	dma_unmap_single(dev, pa, dmalen, DMA_FROM_DEVICE);
241	kfree_skb(skb: ctx->skb);
242	wil_ring_advance_head(ring: vring, n: 1);
243	}
244	}
245	dma_free_coherent(dev, size: sz, cpu_addr: (void *)vring->va, dma_handle: vring->pa);
246	kfree(objp: vring->ctx);
247	vring->pa = 0;
248	vring->va = NULL;
249	vring->ctx = NULL;
250	}
251
252	/* Allocate one skb for Rx VRING
253	*
254	* Safe to call from IRQ
255	*/
256	static int wil_vring_alloc_skb(struct wil6210_priv *wil, struct wil_ring *vring,
257	u32 i, int headroom)
258	{
259	struct device *dev = wil_to_dev(wil);
260	unsigned int sz = wil->rx_buf_len + ETH_HLEN + wil_rx_snaplen();
261	struct vring_rx_desc dd, *d = &dd;
262	volatile struct vring_rx_desc *_d = &vring->va[i].rx.legacy;
263	dma_addr_t pa;
264	struct sk_buff *skb = dev_alloc_skb(length: sz + headroom);
265
266	if (unlikely(!skb))
267	return -ENOMEM;
268
269	skb_reserve(skb, len: headroom);
270	skb_put(skb, len: sz);
271
272	/**
273	* Make sure that the network stack calculates checksum for packets
274	* which failed the HW checksum calculation
275	*/
276	skb->ip_summed = CHECKSUM_NONE;
277
278	pa = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE);
279	if (unlikely(dma_mapping_error(dev, pa))) {
280	kfree_skb(skb);
281	return -ENOMEM;
282	}
283
284	d->dma.d0 = RX_DMA_D0_CMD_DMA_RT | RX_DMA_D0_CMD_DMA_IT;
285	wil_desc_addr_set(addr: &d->dma.addr, pa);
286	/* ip_length don't care */
287	/* b11 don't care */
288	/* error don't care */
289	d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */
290	d->dma.length = cpu_to_le16(sz);
291	*_d = *d;
292	vring->ctx[i].skb = skb;
293
294	return 0;
295	}
296
297	/* Adds radiotap header
298	*
299	* Any error indicated as "Bad FCS"
300	*
301	* Vendor data for 04:ce:14-1 (Wilocity-1) consists of:
302	* - Rx descriptor: 32 bytes
303	* - Phy info
304	*/
305	static void wil_rx_add_radiotap_header(struct wil6210_priv *wil,
306	struct sk_buff *skb)
307	{
308	struct wil6210_rtap {
309	struct ieee80211_radiotap_header rthdr;
310	/* fields should be in the order of bits in rthdr.it_present */
311	/* flags */
312	u8 flags;
313	/* channel */
314	__le16 chnl_freq __aligned(2);
315	__le16 chnl_flags;
316	/* MCS */
317	u8 mcs_present;
318	u8 mcs_flags;
319	u8 mcs_index;
320	} __packed;
321	struct vring_rx_desc *d = wil_skb_rxdesc(skb);
322	struct wil6210_rtap *rtap;
323	int rtap_len = sizeof(struct wil6210_rtap);
324	struct ieee80211_channel *ch = wil->monitor_chandef.chan;
325
326	if (skb_headroom(skb) < rtap_len &&
327	pskb_expand_head(skb, nhead: rtap_len, ntail: 0, GFP_ATOMIC)) {
328	wil_err(wil, "Unable to expand headroom to %d\n", rtap_len);
329	return;
330	}
331
332	rtap = skb_push(skb, len: rtap_len);
333	memset(rtap, 0, rtap_len);
334
335	rtap->rthdr.it_version = PKTHDR_RADIOTAP_VERSION;
336	rtap->rthdr.it_len = cpu_to_le16(rtap_len);
337	rtap->rthdr.it_present = cpu_to_le32((1 << IEEE80211_RADIOTAP_FLAGS) |
338	(1 << IEEE80211_RADIOTAP_CHANNEL) |
339	(1 << IEEE80211_RADIOTAP_MCS));
340	if (d->dma.status & RX_DMA_STATUS_ERROR)
341	rtap->flags |= IEEE80211_RADIOTAP_F_BADFCS;
342
343	rtap->chnl_freq = cpu_to_le16(ch ? ch->center_freq : 58320);
344	rtap->chnl_flags = cpu_to_le16(0);
345
346	rtap->mcs_present = IEEE80211_RADIOTAP_MCS_HAVE_MCS;
347	rtap->mcs_flags = 0;
348	rtap->mcs_index = wil_rxdesc_mcs(d);
349	}
350
351	static bool wil_is_rx_idle(struct wil6210_priv *wil)
352	{
353	struct vring_rx_desc *_d;
354	struct wil_ring *ring = &wil->ring_rx;
355
356	_d = (struct vring_rx_desc *)&ring->va[ring->swhead].rx.legacy;
357	if (_d->dma.status & RX_DMA_STATUS_DU)
358	return false;
359
360	return true;
361	}
362
363	static int wil_rx_get_cid_by_skb(struct wil6210_priv *wil, struct sk_buff *skb)
364	{
365	struct vring_rx_desc *d = wil_skb_rxdesc(skb);
366	int mid = wil_rxdesc_mid(d);
367	struct wil6210_vif *vif = wil->vifs[mid];
368	/* cid from DMA descriptor is limited to 3 bits.
369	* In case of cid>=8, the value would be cid modulo 8 and we need to
370	* find real cid by locating the transmitter (ta) inside sta array
371	*/
372	int cid = wil_rxdesc_cid(d);
373	unsigned int snaplen = wil_rx_snaplen();
374	struct ieee80211_hdr_3addr *hdr;
375	int i;
376	unsigned char *ta;
377	u8 ftype;
378
379	/* in monitor mode there are no connections */
380	if (vif->wdev.iftype == NL80211_IFTYPE_MONITOR)
381	return cid;
382
383	ftype = wil_rxdesc_ftype(d) << 2;
384	if (likely(ftype == IEEE80211_FTYPE_DATA)) {
385	if (unlikely(skb->len < ETH_HLEN + snaplen)) {
386	wil_err_ratelimited(wil,
387	"Short data frame, len = %d\n",
388	skb->len);
389	return -ENOENT;
390	}
391	ta = wil_skb_get_sa(skb);
392	} else {
393	if (unlikely(skb->len < sizeof(struct ieee80211_hdr_3addr))) {
394	wil_err_ratelimited(wil, "Short frame, len = %d\n",
395	skb->len);
396	return -ENOENT;
397	}
398	hdr = (void *)skb->data;
399	ta = hdr->addr2;
400	}
401
402	if (wil->max_assoc_sta <= WIL6210_RX_DESC_MAX_CID)
403	return cid;
404
405	/* assuming no concurrency between AP interfaces and STA interfaces.
406	* multista is used only in P2P_GO or AP mode. In other modes return
407	* cid from the rx descriptor
408	*/
409	if (vif->wdev.iftype != NL80211_IFTYPE_P2P_GO &&
410	vif->wdev.iftype != NL80211_IFTYPE_AP)
411	return cid;
412
413	/* For Rx packets cid from rx descriptor is limited to 3 bits (0..7),
414	* to find the real cid, compare transmitter address with the stored
415	* stations mac address in the driver sta array
416	*/
417	for (i = cid; i < wil->max_assoc_sta; i += WIL6210_RX_DESC_MAX_CID) {
418	if (wil->sta[i].status != wil_sta_unused &&
419	ether_addr_equal(addr1: wil->sta[i].addr, addr2: ta)) {
420	cid = i;
421	break;
422	}
423	}
424	if (i >= wil->max_assoc_sta) {
425	wil_err_ratelimited(wil, "Could not find cid for frame with transmit addr = %pM, iftype = %d, frametype = %d, len = %d\n",
426	ta, vif->wdev.iftype, ftype, skb->len);
427	cid = -ENOENT;
428	}
429
430	return cid;
431	}
432
433	/* reap 1 frame from @swhead
434	*
435	* Rx descriptor copied to skb->cb
436	*
437	* Safe to call from IRQ
438	*/
439	static struct sk_buff *wil_vring_reap_rx(struct wil6210_priv *wil,
440	struct wil_ring *vring)
441	{
442	struct device *dev = wil_to_dev(wil);
443	struct wil6210_vif *vif;
444	struct net_device *ndev;
445	volatile struct vring_rx_desc *_d;
446	struct vring_rx_desc *d;
447	struct sk_buff *skb;
448	dma_addr_t pa;
449	unsigned int snaplen = wil_rx_snaplen();
450	unsigned int sz = wil->rx_buf_len + ETH_HLEN + snaplen;
451	u16 dmalen;
452	u8 ftype;
453	int cid, mid;
454	int i;
455	struct wil_net_stats *stats;
456
457	BUILD_BUG_ON(sizeof(struct skb_rx_info) > sizeof(skb->cb));
458
459	again:
460	if (unlikely(wil_ring_is_empty(vring)))
461	return NULL;
462
463	i = (int)vring->swhead;
464	_d = &vring->va[i].rx.legacy;
465	if (unlikely(!(_d->dma.status & RX_DMA_STATUS_DU))) {
466	/* it is not error, we just reached end of Rx done area */
467	return NULL;
468	}
469
470	skb = vring->ctx[i].skb;
471	vring->ctx[i].skb = NULL;
472	wil_ring_advance_head(ring: vring, n: 1);
473	if (!skb) {
474	wil_err(wil, "No Rx skb at [%d]\n", i);
475	goto again;
476	}
477	d = wil_skb_rxdesc(skb);
478	*d = *_d;
479	pa = wil_desc_addr(addr: &d->dma.addr);
480
481	dma_unmap_single(dev, pa, sz, DMA_FROM_DEVICE);
482	dmalen = le16_to_cpu(d->dma.length);
483
484	trace_wil6210_rx(index: i, d);
485	wil_dbg_txrx(wil, "Rx[%3d] : %d bytes\n", i, dmalen);
486	wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4,
487	(const void *)d, sizeof(*d), false);
488
489	mid = wil_rxdesc_mid(d);
490	vif = wil->vifs[mid];
491
492	if (unlikely(!vif)) {
493	wil_dbg_txrx(wil, "skipped RX descriptor with invalid mid %d",
494	mid);
495	kfree_skb(skb);
496	goto again;
497	}
498	ndev = vif_to_ndev(vif);
499	if (unlikely(dmalen > sz)) {
500	wil_err_ratelimited(wil, "Rx size too large: %d bytes!\n",
501	dmalen);
502	kfree_skb(skb);
503	goto again;
504	}
505	skb_trim(skb, len: dmalen);
506
507	prefetch(skb->data);
508
509	wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1,
510	skb->data, skb_headlen(skb), false);
511
512	cid = wil_rx_get_cid_by_skb(wil, skb);
513	if (cid == -ENOENT) {
514	kfree_skb(skb);
515	goto again;
516	}
517	wil_skb_set_cid(skb, cid: (u8)cid);
518	stats = &wil->sta[cid].stats;
519
520	stats->last_mcs_rx = wil_rxdesc_mcs(d);
521	if (stats->last_mcs_rx < ARRAY_SIZE(stats->rx_per_mcs))
522	stats->rx_per_mcs[stats->last_mcs_rx]++;
523
524	/* use radiotap header only if required */
525	if (ndev->type == ARPHRD_IEEE80211_RADIOTAP)
526	wil_rx_add_radiotap_header(wil, skb);
527
528	/* no extra checks if in sniffer mode */
529	if (ndev->type != ARPHRD_ETHER)
530	return skb;
531	/* Non-data frames may be delivered through Rx DMA channel (ex: BAR)
532	* Driver should recognize it by frame type, that is found
533	* in Rx descriptor. If type is not data, it is 802.11 frame as is
534	*/
535	ftype = wil_rxdesc_ftype(d) << 2;
536	if (unlikely(ftype != IEEE80211_FTYPE_DATA)) {
537	u8 fc1 = wil_rxdesc_fc1(d);
538	int tid = wil_rxdesc_tid(d);
539	u16 seq = wil_rxdesc_seq(d);
540
541	wil_dbg_txrx(wil,
542	"Non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
543	fc1, mid, cid, tid, seq);
544	stats->rx_non_data_frame++;
545	if (wil_is_back_req(fc: fc1)) {
546	wil_dbg_txrx(wil,
547	"BAR: MID %d CID %d TID %d Seq 0x%03x\n",
548	mid, cid, tid, seq);
549	wil_rx_bar(wil, vif, cid, tid, seq);
550	} else {
551	/* print again all info. One can enable only this
552	* without overhead for printing every Rx frame
553	*/
554	wil_dbg_txrx(wil,
555	"Unhandled non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
556	fc1, mid, cid, tid, seq);
557	wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4,
558	(const void *)d, sizeof(*d), false);
559	wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1,
560	skb->data, skb_headlen(skb), false);
561	}
562	kfree_skb(skb);
563	goto again;
564	}
565
566	/* L4 IDENT is on when HW calculated checksum, check status
567	* and in case of error drop the packet
568	* higher stack layers will handle retransmission (if required)
569	*/
570	if (likely(d->dma.status & RX_DMA_STATUS_L4I)) {
571	/* L4 protocol identified, csum calculated */
572	if (likely((d->dma.error & RX_DMA_ERROR_L4_ERR) == 0))
573	skb->ip_summed = CHECKSUM_UNNECESSARY;
574	/* If HW reports bad checksum, let IP stack re-check it
575	* For example, HW don't understand Microsoft IP stack that
576	* mis-calculates TCP checksum - if it should be 0x0,
577	* it writes 0xffff in violation of RFC 1624
578	*/
579	else
580	stats->rx_csum_err++;
581	}
582
583	if (snaplen) {
584	/* Packet layout
585	* +-------+-------+---------+------------+------+
586	* | SA(6) | DA(6) | SNAP(6) | ETHTYPE(2) | DATA |
587	* +-------+-------+---------+------------+------+
588	* Need to remove SNAP, shifting SA and DA forward
589	*/
590	memmove(skb->data + snaplen, skb->data, 2 * ETH_ALEN);
591	skb_pull(skb, len: snaplen);
592	}
593
594	return skb;
595	}
596
597	/* allocate and fill up to @count buffers in rx ring
598	* buffers posted at @swtail
599	* Note: we have a single RX queue for servicing all VIFs, but we
600	* allocate skbs with headroom according to main interface only. This
601	* means it will not work with monitor interface together with other VIFs.
602	* Currently we only support monitor interface on its own without other VIFs,
603	* and we will need to fix this code once we add support.
604	*/
605	static int wil_rx_refill(struct wil6210_priv *wil, int count)
606	{
607	struct net_device *ndev = wil->main_ndev;
608	struct wil_ring *v = &wil->ring_rx;
609	u32 next_tail;
610	int rc = 0;
611	int headroom = ndev->type == ARPHRD_IEEE80211_RADIOTAP ?
612	WIL6210_RTAP_SIZE : 0;
613
614	for (; next_tail = wil_ring_next_tail(ring: v),
615	(next_tail != v->swhead) && (count-- > 0);
616	v->swtail = next_tail) {
617	rc = wil_vring_alloc_skb(wil, vring: v, i: v->swtail, headroom);
618	if (unlikely(rc)) {
619	wil_err_ratelimited(wil, "Error %d in rx refill[%d]\n",
620	rc, v->swtail);
621	break;
622	}
623	}
624
625	/* make sure all writes to descriptors (shared memory) are done before
626	* committing them to HW
627	*/
628	wmb();
629
630	wil_w(wil, reg: v->hwtail, val: v->swtail);
631
632	return rc;
633	}
634
635	/**
636	* reverse_memcmp - Compare two areas of memory, in reverse order
637	* @cs: One area of memory
638	* @ct: Another area of memory
639	* @count: The size of the area.
640	*
641	* Cut'n'paste from original memcmp (see lib/string.c)
642	* with minimal modifications
643	*/
644	int reverse_memcmp(const void *cs, const void *ct, size_t count)
645	{
646	const unsigned char *su1, *su2;
647	int res = 0;
648
649	for (su1 = cs + count - 1, su2 = ct + count - 1; count > 0;
650	--su1, --su2, count--) {
651	res = *su1 - *su2;
652	if (res)
653	break;
654	}
655	return res;
656	}
657
658	static int wil_rx_crypto_check(struct wil6210_priv *wil, struct sk_buff *skb)
659	{
660	struct vring_rx_desc *d = wil_skb_rxdesc(skb);
661	int cid = wil_skb_get_cid(skb);
662	int tid = wil_rxdesc_tid(d);
663	int key_id = wil_rxdesc_key_id(d);
664	int mc = wil_rxdesc_mcast(d);
665	struct wil_sta_info *s = &wil->sta[cid];
666	struct wil_tid_crypto_rx *c = mc ? &s->group_crypto_rx :
667	&s->tid_crypto_rx[tid];
668	struct wil_tid_crypto_rx_single *cc = &c->key_id[key_id];
669	const u8 *pn = (u8 *)&d->mac.pn;
670
671	if (!cc->key_set) {
672	wil_err_ratelimited(wil,
673	"Key missing. CID %d TID %d MCast %d KEY_ID %d\n",
674	cid, tid, mc, key_id);
675	return -EINVAL;
676	}
677
678	if (reverse_memcmp(cs: pn, ct: cc->pn, IEEE80211_GCMP_PN_LEN) <= 0) {
679	wil_err_ratelimited(wil,
680	"Replay attack. CID %d TID %d MCast %d KEY_ID %d PN %6phN last %6phN\n",
681	cid, tid, mc, key_id, pn, cc->pn);
682	return -EINVAL;
683	}
684	memcpy(cc->pn, pn, IEEE80211_GCMP_PN_LEN);
685
686	return 0;
687	}
688
689	static int wil_rx_error_check(struct wil6210_priv *wil, struct sk_buff *skb,
690	struct wil_net_stats *stats)
691	{
692	struct vring_rx_desc *d = wil_skb_rxdesc(skb);
693
694	if ((d->dma.status & RX_DMA_STATUS_ERROR) &&
695	(d->dma.error & RX_DMA_ERROR_MIC)) {
696	stats->rx_mic_error++;
697	wil_dbg_txrx(wil, "MIC error, dropping packet\n");
698	return -EFAULT;
699	}
700
701	return 0;
702	}
703
704	static void wil_get_netif_rx_params(struct sk_buff *skb, int *cid,
705	int *security)
706	{
707	struct vring_rx_desc *d = wil_skb_rxdesc(skb);
708
709	*cid = wil_skb_get_cid(skb);
710	*security = wil_rxdesc_security(d);
711	}
712
713	/*
714	* Check if skb is ptk eapol key message
715	*
716	* returns a pointer to the start of the eapol key structure, NULL
717	* if frame is not PTK eapol key
718	*/
719	static struct wil_eapol_key *wil_is_ptk_eapol_key(struct wil6210_priv *wil,
720	struct sk_buff *skb)
721	{
722	u8 *buf;
723	const struct wil_1x_hdr *hdr;
724	struct wil_eapol_key *key;
725	u16 key_info;
726	int len = skb->len;
727
728	if (!skb_mac_header_was_set(skb)) {
729	wil_err(wil, "mac header was not set\n");
730	return NULL;
731	}
732
733	len -= skb_mac_offset(skb);
734
735	if (len < sizeof(struct ethhdr) + sizeof(struct wil_1x_hdr) +
736	sizeof(struct wil_eapol_key))
737	return NULL;
738
739	buf = skb_mac_header(skb) + sizeof(struct ethhdr);
740
741	hdr = (const struct wil_1x_hdr *)buf;
742	if (hdr->type != WIL_1X_TYPE_EAPOL_KEY)
743	return NULL;
744
745	key = (struct wil_eapol_key *)(buf + sizeof(struct wil_1x_hdr));
746	if (key->type != WIL_EAPOL_KEY_TYPE_WPA &&
747	key->type != WIL_EAPOL_KEY_TYPE_RSN)
748	return NULL;
749
750	key_info = be16_to_cpu(key->key_info);
751	if (!(key_info & WIL_KEY_INFO_KEY_TYPE)) /* check if pairwise */
752	return NULL;
753
754	return key;
755	}
756
757	static bool wil_skb_is_eap_3(struct wil6210_priv *wil, struct sk_buff *skb)
758	{
759	struct wil_eapol_key *key;
760	u16 key_info;
761
762	key = wil_is_ptk_eapol_key(wil, skb);
763	if (!key)
764	return false;
765
766	key_info = be16_to_cpu(key->key_info);
767	if (key_info & (WIL_KEY_INFO_MIC |
768	WIL_KEY_INFO_ENCR_KEY_DATA)) {
769	/* 3/4 of 4-Way Handshake */
770	wil_dbg_misc(wil, "EAPOL key message 3\n");
771	return true;
772	}
773	/* 1/4 of 4-Way Handshake */
774	wil_dbg_misc(wil, "EAPOL key message 1\n");
775
776	return false;
777	}
778
779	static bool wil_skb_is_eap_4(struct wil6210_priv *wil, struct sk_buff *skb)
780	{
781	struct wil_eapol_key *key;
782	u32 *nonce, i;
783
784	key = wil_is_ptk_eapol_key(wil, skb);
785	if (!key)
786	return false;
787
788	nonce = (u32 *)key->key_nonce;
789	for (i = 0; i < WIL_EAP_NONCE_LEN / sizeof(u32); i++, nonce++) {
790	if (*nonce != 0) {
791	/* message 2/4 */
792	wil_dbg_misc(wil, "EAPOL key message 2\n");
793	return false;
794	}
795	}
796	wil_dbg_misc(wil, "EAPOL key message 4\n");
797
798	return true;
799	}
800
801	void wil_enable_tx_key_worker(struct work_struct *work)
802	{
803	struct wil6210_vif *vif = container_of(work,
804	struct wil6210_vif, enable_tx_key_worker);
805	struct wil6210_priv *wil = vif_to_wil(vif);
806	int rc, cid;
807
808	rtnl_lock();
809	if (vif->ptk_rekey_state != WIL_REKEY_WAIT_M4_SENT) {
810	wil_dbg_misc(wil, "Invalid rekey state = %d\n",
811	vif->ptk_rekey_state);
812	rtnl_unlock();
813	return;
814	}
815
816	cid = wil_find_cid_by_idx(wil, mid: vif->mid, idx: 0);
817	if (!wil_cid_valid(wil, cid)) {
818	wil_err(wil, "Invalid cid = %d\n", cid);
819	rtnl_unlock();
820	return;
821	}
822
823	wil_dbg_misc(wil, "Apply PTK key after eapol was sent out\n");
824	rc = wmi_add_cipher_key(vif, key_index: 0, mac_addr: wil->sta[cid].addr, key_len: 0, NULL,
825	key_usage: WMI_KEY_USE_APPLY_PTK);
826
827	vif->ptk_rekey_state = WIL_REKEY_IDLE;
828	rtnl_unlock();
829
830	if (rc)
831	wil_err(wil, "Apply PTK key failed %d\n", rc);
832	}
833
834	void wil_tx_complete_handle_eapol(struct wil6210_vif *vif, struct sk_buff *skb)
835	{
836	struct wil6210_priv *wil = vif_to_wil(vif);
837	struct wireless_dev *wdev = vif_to_wdev(vif);
838	bool q = false;
839
840	if (wdev->iftype != NL80211_IFTYPE_STATION ||
841	!test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY, wil->fw_capabilities))
842	return;
843
844	/* check if skb is an EAP message 4/4 */
845	if (!wil_skb_is_eap_4(wil, skb))
846	return;
847
848	spin_lock_bh(lock: &wil->eap_lock);
849	switch (vif->ptk_rekey_state) {
850	case WIL_REKEY_IDLE:
851	/* ignore idle state, can happen due to M4 retransmission */
852	break;
853	case WIL_REKEY_M3_RECEIVED:
854	vif->ptk_rekey_state = WIL_REKEY_IDLE;
855	break;
856	case WIL_REKEY_WAIT_M4_SENT:
857	q = true;
858	break;
859	default:
860	wil_err(wil, "Unknown rekey state = %d",
861	vif->ptk_rekey_state);
862	}
863	spin_unlock_bh(lock: &wil->eap_lock);
864
865	if (q) {
866	q = queue_work(wq: wil->wmi_wq, work: &vif->enable_tx_key_worker);
867	wil_dbg_misc(wil, "queue_work of enable_tx_key_worker -> %d\n",
868	q);
869	}
870	}
871
872	static void wil_rx_handle_eapol(struct wil6210_vif *vif, struct sk_buff *skb)
873	{
874	struct wil6210_priv *wil = vif_to_wil(vif);
875	struct wireless_dev *wdev = vif_to_wdev(vif);
876
877	if (wdev->iftype != NL80211_IFTYPE_STATION ||
878	!test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY, wil->fw_capabilities))
879	return;
880
881	/* check if skb is a EAP message 3/4 */
882	if (!wil_skb_is_eap_3(wil, skb))
883	return;
884
885	if (vif->ptk_rekey_state == WIL_REKEY_IDLE)
886	vif->ptk_rekey_state = WIL_REKEY_M3_RECEIVED;
887	}
888
889	/*
890	* Pass Rx packet to the netif. Update statistics.
891	* Called in softirq context (NAPI poll).
892	*/
893	void wil_netif_rx(struct sk_buff *skb, struct net_device *ndev, int cid,
894	struct wil_net_stats *stats, bool gro)
895	{
896	struct wil6210_vif *vif = ndev_to_vif(ndev);
897	struct wil6210_priv *wil = ndev_to_wil(ndev);
898	struct wireless_dev *wdev = vif_to_wdev(vif);
899	unsigned int len = skb->len;
900	u8 *sa, *da = wil_skb_get_da(skb);
901	/* here looking for DA, not A1, thus Rxdesc's 'mcast' indication
902	* is not suitable, need to look at data
903	*/
904	int mcast = is_multicast_ether_addr(addr: da);
905	struct sk_buff *xmit_skb = NULL;
906
907	if (wdev->iftype == NL80211_IFTYPE_STATION) {
908	sa = wil_skb_get_sa(skb);
909	if (mcast && ether_addr_equal(addr1: sa, addr2: ndev->dev_addr)) {
910	/* mcast packet looped back to us */
911	dev_kfree_skb(skb);
912	ndev->stats.rx_dropped++;
913	stats->rx_dropped++;
914	wil_dbg_txrx(wil, "Rx drop %d bytes\n", len);
915	return;
916	}
917	} else if (wdev->iftype == NL80211_IFTYPE_AP && !vif->ap_isolate) {
918	if (mcast) {
919	/* send multicast frames both to higher layers in
920	* local net stack and back to the wireless medium
921	*/
922	xmit_skb = skb_copy(skb, GFP_ATOMIC);
923	} else {
924	int xmit_cid = wil_find_cid(wil, mid: vif->mid, mac: da);
925
926	if (xmit_cid >= 0) {
927	/* The destination station is associated to
928	* this AP (in this VLAN), so send the frame
929	* directly to it and do not pass it to local
930	* net stack.
931	*/
932	xmit_skb = skb;
933	skb = NULL;
934	}
935	}
936	}
937	if (xmit_skb) {
938	/* Send to wireless media and increase priority by 256 to
939	* keep the received priority instead of reclassifying
940	* the frame (see cfg80211_classify8021d).
941	*/
942	xmit_skb->dev = ndev;
943	xmit_skb->priority += 256;
944	xmit_skb->protocol = htons(ETH_P_802_3);
945	skb_reset_network_header(skb: xmit_skb);
946	skb_reset_mac_header(skb: xmit_skb);
947	wil_dbg_txrx(wil, "Rx -> Tx %d bytes\n", len);
948	dev_queue_xmit(skb: xmit_skb);
949	}
950
951	if (skb) { /* deliver to local stack */
952	skb->protocol = eth_type_trans(skb, dev: ndev);
953	skb->dev = ndev;
954
955	if (skb->protocol == cpu_to_be16(ETH_P_PAE))
956	wil_rx_handle_eapol(vif, skb);
957
958	if (gro)
959	napi_gro_receive(napi: &wil->napi_rx, skb);
960	else
961	netif_rx(skb);
962	}
963	ndev->stats.rx_packets++;
964	stats->rx_packets++;
965	ndev->stats.rx_bytes += len;
966	stats->rx_bytes += len;
967	if (mcast)
968	ndev->stats.multicast++;
969	}
970
971	void wil_netif_rx_any(struct sk_buff *skb, struct net_device *ndev)
972	{
973	int cid, security;
974	struct wil6210_priv *wil = ndev_to_wil(ndev);
975	struct wil_net_stats *stats;
976
977	wil->txrx_ops.get_netif_rx_params(skb, &cid, &security);
978
979	stats = &wil->sta[cid].stats;
980
981	skb_orphan(skb);
982
983	if (security && (wil->txrx_ops.rx_crypto_check(wil, skb) != 0)) {
984	wil_dbg_txrx(wil, "Rx drop %d bytes\n", skb->len);
985	dev_kfree_skb(skb);
986	ndev->stats.rx_dropped++;
987	stats->rx_replay++;
988	stats->rx_dropped++;
989	return;
990	}
991
992	/* check errors reported by HW and update statistics */
993	if (unlikely(wil->txrx_ops.rx_error_check(wil, skb, stats))) {
994	dev_kfree_skb(skb);
995	return;
996	}
997
998	wil_netif_rx(skb, ndev, cid, stats, gro: true);
999	}
1000
1001	/* Proceed all completed skb's from Rx VRING
1002	*
1003	* Safe to call from NAPI poll, i.e. softirq with interrupts enabled
1004	*/
1005	void wil_rx_handle(struct wil6210_priv *wil, int *quota)
1006	{
1007	struct net_device *ndev = wil->main_ndev;
1008	struct wireless_dev *wdev = ndev->ieee80211_ptr;
1009	struct wil_ring *v = &wil->ring_rx;
1010	struct sk_buff *skb;
1011
1012	if (unlikely(!v->va)) {
1013	wil_err(wil, "Rx IRQ while Rx not yet initialized\n");
1014	return;
1015	}
1016	wil_dbg_txrx(wil, "rx_handle\n");
1017	while ((*quota > 0) && (NULL != (skb = wil_vring_reap_rx(wil, vring: v)))) {
1018	(*quota)--;
1019
1020	/* monitor is currently supported on main interface only */
1021	if (wdev->iftype == NL80211_IFTYPE_MONITOR) {
1022	skb->dev = ndev;
1023	skb_reset_mac_header(skb);
1024	skb->ip_summed = CHECKSUM_UNNECESSARY;
1025	skb->pkt_type = PACKET_OTHERHOST;
1026	skb->protocol = htons(ETH_P_802_2);
1027	wil_netif_rx_any(skb, ndev);
1028	} else {
1029	wil_rx_reorder(wil, skb);
1030	}
1031	}
1032	wil_rx_refill(wil, count: v->size);
1033	}
1034
1035	static void wil_rx_buf_len_init(struct wil6210_priv *wil)
1036	{
1037	wil->rx_buf_len = rx_large_buf ?
1038	WIL_MAX_ETH_MTU : TXRX_BUF_LEN_DEFAULT - WIL_MAX_MPDU_OVERHEAD;
1039	if (mtu_max > wil->rx_buf_len) {
1040	/* do not allow RX buffers to be smaller than mtu_max, for
1041	* backward compatibility (mtu_max parameter was also used
1042	* to support receiving large packets)
1043	*/
1044	wil_info(wil, "Override RX buffer to mtu_max(%d)\n", mtu_max);
1045	wil->rx_buf_len = mtu_max;
1046	}
1047	}
1048
1049	static int wil_rx_init(struct wil6210_priv *wil, uint order)
1050	{
1051	struct wil_ring *vring = &wil->ring_rx;
1052	int rc;
1053
1054	wil_dbg_misc(wil, "rx_init\n");
1055
1056	if (vring->va) {
1057	wil_err(wil, "Rx ring already allocated\n");
1058	return -EINVAL;
1059	}
1060
1061	wil_rx_buf_len_init(wil);
1062
1063	vring->size = 1 << order;
1064	vring->is_rx = true;
1065	rc = wil_vring_alloc(wil, vring);
1066	if (rc)
1067	return rc;
1068
1069	rc = wmi_rx_chain_add(wil, vring);
1070	if (rc)
1071	goto err_free;
1072
1073	rc = wil_rx_refill(wil, count: vring->size);
1074	if (rc)
1075	goto err_free;
1076
1077	return 0;
1078	err_free:
1079	wil_vring_free(wil, vring);
1080
1081	return rc;
1082	}
1083
1084	static void wil_rx_fini(struct wil6210_priv *wil)
1085	{
1086	struct wil_ring *vring = &wil->ring_rx;
1087
1088	wil_dbg_misc(wil, "rx_fini\n");
1089
1090	if (vring->va)
1091	wil_vring_free(wil, vring);
1092	}
1093
1094	static int wil_tx_desc_map(union wil_tx_desc *desc, dma_addr_t pa,
1095	u32 len, int vring_index)
1096	{
1097	struct vring_tx_desc *d = &desc->legacy;
1098
1099	wil_desc_addr_set(addr: &d->dma.addr, pa);
1100	d->dma.ip_length = 0;
1101	/* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/
1102	d->dma.b11 = 0/*14 | BIT(7)*/;
1103	d->dma.error = 0;
1104	d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */
1105	d->dma.length = cpu_to_le16((u16)len);
1106	d->dma.d0 = (vring_index << DMA_CFG_DESC_TX_0_QID_POS);
1107	d->mac.d[0] = 0;
1108	d->mac.d[1] = 0;
1109	d->mac.d[2] = 0;
1110	d->mac.ucode_cmd = 0;
1111	/* translation type: 0 - bypass; 1 - 802.3; 2 - native wifi */
1112	d->mac.d[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS) |
1113	(1 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS);
1114
1115	return 0;
1116	}
1117
1118	void wil_tx_data_init(struct wil_ring_tx_data *txdata)
1119	{
1120	spin_lock_bh(lock: &txdata->lock);
1121	txdata->dot1x_open = false;
1122	txdata->enabled = 0;
1123	txdata->idle = 0;
1124	txdata->last_idle = 0;
1125	txdata->begin = 0;
1126	txdata->agg_wsize = 0;
1127	txdata->agg_timeout = 0;
1128	txdata->agg_amsdu = 0;
1129	txdata->addba_in_progress = false;
1130	txdata->mid = U8_MAX;
1131	spin_unlock_bh(lock: &txdata->lock);
1132	}
1133
1134	static int wil_vring_init_tx(struct wil6210_vif *vif, int id, int size,
1135	int cid, int tid)
1136	{
1137	struct wil6210_priv *wil = vif_to_wil(vif);
1138	int rc;
1139	struct wmi_vring_cfg_cmd cmd = {
1140	.action = cpu_to_le32(WMI_VRING_CMD_ADD),
1141	.vring_cfg = {
1142	.tx_sw_ring = {
1143	.max_mpdu_size =
1144	cpu_to_le16(wil_mtu2macbuf(mtu_max)),
1145	.ring_size = cpu_to_le16(size),
1146	},
1147	.ringid = id,
1148	.encap_trans_type = WMI_VRING_ENC_TYPE_802_3,
1149	.mac_ctrl = 0,
1150	.to_resolution = 0,
1151	.agg_max_wsize = 0,
1152	.schd_params = {
1153	.priority = cpu_to_le16(0),
1154	.timeslot_us = cpu_to_le16(0xfff),
1155	},
1156	},
1157	};
1158	struct {
1159	struct wmi_cmd_hdr wmi;
1160	struct wmi_vring_cfg_done_event cmd;
1161	} __packed reply = {
1162	.cmd = {.status = WMI_FW_STATUS_FAILURE},
1163	};
1164	struct wil_ring *vring = &wil->ring_tx[id];
1165	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id];
1166
1167	if (cid >= WIL6210_RX_DESC_MAX_CID) {
1168	cmd.vring_cfg.cidxtid = CIDXTID_EXTENDED_CID_TID;
1169	cmd.vring_cfg.cid = cid;
1170	cmd.vring_cfg.tid = tid;
1171	} else {
1172	cmd.vring_cfg.cidxtid = mk_cidxtid(cid, tid);
1173	}
1174
1175	wil_dbg_misc(wil, "vring_init_tx: max_mpdu_size %d\n",
1176	cmd.vring_cfg.tx_sw_ring.max_mpdu_size);
1177	lockdep_assert_held(&wil->mutex);
1178
1179	if (vring->va) {
1180	wil_err(wil, "Tx ring [%d] already allocated\n", id);
1181	rc = -EINVAL;
1182	goto out;
1183	}
1184
1185	wil_tx_data_init(txdata);
1186	vring->is_rx = false;
1187	vring->size = size;
1188	rc = wil_vring_alloc(wil, vring);
1189	if (rc)
1190	goto out;
1191
1192	wil->ring2cid_tid[id][0] = cid;
1193	wil->ring2cid_tid[id][1] = tid;
1194
1195	cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa);
1196
1197	if (!vif->privacy)
1198	txdata->dot1x_open = true;
1199	rc = wmi_call(wil, cmdid: WMI_VRING_CFG_CMDID, mid: vif->mid, buf: &cmd, len: sizeof(cmd),
1200	reply_id: WMI_VRING_CFG_DONE_EVENTID, reply: &reply, reply_size: sizeof(reply),
1201	WIL_WMI_CALL_GENERAL_TO_MS);
1202	if (rc)
1203	goto out_free;
1204
1205	if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) {
1206	wil_err(wil, "Tx config failed, status 0x%02x\n",
1207	reply.cmd.status);
1208	rc = -EINVAL;
1209	goto out_free;
1210	}
1211
1212	spin_lock_bh(lock: &txdata->lock);
1213	vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr);
1214	txdata->mid = vif->mid;
1215	txdata->enabled = 1;
1216	spin_unlock_bh(lock: &txdata->lock);
1217
1218	if (txdata->dot1x_open && (agg_wsize >= 0))
1219	wil_addba_tx_request(wil, ringid: id, wsize: agg_wsize);
1220
1221	return 0;
1222	out_free:
1223	spin_lock_bh(lock: &txdata->lock);
1224	txdata->dot1x_open = false;
1225	txdata->enabled = 0;
1226	spin_unlock_bh(lock: &txdata->lock);
1227	wil_vring_free(wil, vring);
1228	wil->ring2cid_tid[id][0] = wil->max_assoc_sta;
1229	wil->ring2cid_tid[id][1] = 0;
1230
1231	out:
1232
1233	return rc;
1234	}
1235
1236	static int wil_tx_vring_modify(struct wil6210_vif *vif, int ring_id, int cid,
1237	int tid)
1238	{
1239	struct wil6210_priv *wil = vif_to_wil(vif);
1240	int rc;
1241	struct wmi_vring_cfg_cmd cmd = {
1242	.action = cpu_to_le32(WMI_VRING_CMD_MODIFY),
1243	.vring_cfg = {
1244	.tx_sw_ring = {
1245	.max_mpdu_size =
1246	cpu_to_le16(wil_mtu2macbuf(mtu_max)),
1247	.ring_size = 0,
1248	},
1249	.ringid = ring_id,
1250	.cidxtid = mk_cidxtid(cid, tid),
1251	.encap_trans_type = WMI_VRING_ENC_TYPE_802_3,
1252	.mac_ctrl = 0,
1253	.to_resolution = 0,
1254	.agg_max_wsize = 0,
1255	.schd_params = {
1256	.priority = cpu_to_le16(0),
1257	.timeslot_us = cpu_to_le16(0xfff),
1258	},
1259	},
1260	};
1261	struct {
1262	struct wmi_cmd_hdr wmi;
1263	struct wmi_vring_cfg_done_event cmd;
1264	} __packed reply = {
1265	.cmd = {.status = WMI_FW_STATUS_FAILURE},
1266	};
1267	struct wil_ring *vring = &wil->ring_tx[ring_id];
1268	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id];
1269
1270	wil_dbg_misc(wil, "vring_modify: ring %d cid %d tid %d\n", ring_id,
1271	cid, tid);
1272	lockdep_assert_held(&wil->mutex);
1273
1274	if (!vring->va) {
1275	wil_err(wil, "Tx ring [%d] not allocated\n", ring_id);
1276	return -EINVAL;
1277	}
1278
1279	if (wil->ring2cid_tid[ring_id][0] != cid ||
1280	wil->ring2cid_tid[ring_id][1] != tid) {
1281	wil_err(wil, "ring info does not match cid=%u tid=%u\n",
1282	wil->ring2cid_tid[ring_id][0],
1283	wil->ring2cid_tid[ring_id][1]);
1284	}
1285
1286	cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa);
1287
1288	rc = wmi_call(wil, cmdid: WMI_VRING_CFG_CMDID, mid: vif->mid, buf: &cmd, len: sizeof(cmd),
1289	reply_id: WMI_VRING_CFG_DONE_EVENTID, reply: &reply, reply_size: sizeof(reply),
1290	WIL_WMI_CALL_GENERAL_TO_MS);
1291	if (rc)
1292	goto fail;
1293
1294	if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) {
1295	wil_err(wil, "Tx modify failed, status 0x%02x\n",
1296	reply.cmd.status);
1297	rc = -EINVAL;
1298	goto fail;
1299	}
1300
1301	/* set BA aggregation window size to 0 to force a new BA with the
1302	* new AP
1303	*/
1304	txdata->agg_wsize = 0;
1305	if (txdata->dot1x_open && agg_wsize >= 0)
1306	wil_addba_tx_request(wil, ringid: ring_id, wsize: agg_wsize);
1307
1308	return 0;
1309	fail:
1310	spin_lock_bh(lock: &txdata->lock);
1311	txdata->dot1x_open = false;
1312	txdata->enabled = 0;
1313	spin_unlock_bh(lock: &txdata->lock);
1314	wil->ring2cid_tid[ring_id][0] = wil->max_assoc_sta;
1315	wil->ring2cid_tid[ring_id][1] = 0;
1316	return rc;
1317	}
1318
1319	int wil_vring_init_bcast(struct wil6210_vif *vif, int id, int size)
1320	{
1321	struct wil6210_priv *wil = vif_to_wil(vif);
1322	int rc;
1323	struct wmi_bcast_vring_cfg_cmd cmd = {
1324	.action = cpu_to_le32(WMI_VRING_CMD_ADD),
1325	.vring_cfg = {
1326	.tx_sw_ring = {
1327	.max_mpdu_size =
1328	cpu_to_le16(wil_mtu2macbuf(mtu_max)),
1329	.ring_size = cpu_to_le16(size),
1330	},
1331	.ringid = id,
1332	.encap_trans_type = WMI_VRING_ENC_TYPE_802_3,
1333	},
1334	};
1335	struct {
1336	struct wmi_cmd_hdr wmi;
1337	struct wmi_vring_cfg_done_event cmd;
1338	} __packed reply = {
1339	.cmd = {.status = WMI_FW_STATUS_FAILURE},
1340	};
1341	struct wil_ring *vring = &wil->ring_tx[id];
1342	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id];
1343
1344	wil_dbg_misc(wil, "vring_init_bcast: max_mpdu_size %d\n",
1345	cmd.vring_cfg.tx_sw_ring.max_mpdu_size);
1346	lockdep_assert_held(&wil->mutex);
1347
1348	if (vring->va) {
1349	wil_err(wil, "Tx ring [%d] already allocated\n", id);
1350	rc = -EINVAL;
1351	goto out;
1352	}
1353
1354	wil_tx_data_init(txdata);
1355	vring->is_rx = false;
1356	vring->size = size;
1357	rc = wil_vring_alloc(wil, vring);
1358	if (rc)
1359	goto out;
1360
1361	wil->ring2cid_tid[id][0] = wil->max_assoc_sta; /* CID */
1362	wil->ring2cid_tid[id][1] = 0; /* TID */
1363
1364	cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa);
1365
1366	if (!vif->privacy)
1367	txdata->dot1x_open = true;
1368	rc = wmi_call(wil, cmdid: WMI_BCAST_VRING_CFG_CMDID, mid: vif->mid,
1369	buf: &cmd, len: sizeof(cmd),
1370	reply_id: WMI_VRING_CFG_DONE_EVENTID, reply: &reply, reply_size: sizeof(reply),
1371	WIL_WMI_CALL_GENERAL_TO_MS);
1372	if (rc)
1373	goto out_free;
1374
1375	if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) {
1376	wil_err(wil, "Tx config failed, status 0x%02x\n",
1377	reply.cmd.status);
1378	rc = -EINVAL;
1379	goto out_free;
1380	}
1381
1382	spin_lock_bh(lock: &txdata->lock);
1383	vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr);
1384	txdata->mid = vif->mid;
1385	txdata->enabled = 1;
1386	spin_unlock_bh(lock: &txdata->lock);
1387
1388	return 0;
1389	out_free:
1390	spin_lock_bh(lock: &txdata->lock);
1391	txdata->enabled = 0;
1392	txdata->dot1x_open = false;
1393	spin_unlock_bh(lock: &txdata->lock);
1394	wil_vring_free(wil, vring);
1395	out:
1396
1397	return rc;
1398	}
1399
1400	static struct wil_ring *wil_find_tx_ucast(struct wil6210_priv *wil,
1401	struct wil6210_vif *vif,
1402	struct sk_buff *skb)
1403	{
1404	int i, cid;
1405	const u8 *da = wil_skb_get_da(skb);
1406	int min_ring_id = wil_get_min_tx_ring_id(wil);
1407
1408	cid = wil_find_cid(wil, mid: vif->mid, mac: da);
1409
1410	if (cid < 0 || cid >= wil->max_assoc_sta)
1411	return NULL;
1412
1413	/* TODO: fix for multiple TID */
1414	for (i = min_ring_id; i < ARRAY_SIZE(wil->ring2cid_tid); i++) {
1415	if (!wil->ring_tx_data[i].dot1x_open &&
1416	skb->protocol != cpu_to_be16(ETH_P_PAE))
1417	continue;
1418	if (wil->ring2cid_tid[i][0] == cid) {
1419	struct wil_ring *v = &wil->ring_tx[i];
1420	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[i];
1421
1422	wil_dbg_txrx(wil, "find_tx_ucast: (%pM) -> [%d]\n",
1423	da, i);
1424	if (v->va && txdata->enabled) {
1425	return v;
1426	} else {
1427	wil_dbg_txrx(wil,
1428	"find_tx_ucast: vring[%d] not valid\n",
1429	i);
1430	return NULL;
1431	}
1432	}
1433	}
1434
1435	return NULL;
1436	}
1437
1438	static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
1439	struct wil_ring *ring, struct sk_buff *skb);
1440
1441	static struct wil_ring *wil_find_tx_ring_sta(struct wil6210_priv *wil,
1442	struct wil6210_vif *vif,
1443	struct sk_buff *skb)
1444	{
1445	struct wil_ring *ring;
1446	int i;
1447	u8 cid;
1448	struct wil_ring_tx_data *txdata;
1449	int min_ring_id = wil_get_min_tx_ring_id(wil);
1450
1451	/* In the STA mode, it is expected to have only 1 VRING
1452	* for the AP we connected to.
1453	* find 1-st vring eligible for this skb and use it.
1454	*/
1455	for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
1456	ring = &wil->ring_tx[i];
1457	txdata = &wil->ring_tx_data[i];
1458	if (!ring->va || !txdata->enabled || txdata->mid != vif->mid)
1459	continue;
1460
1461	cid = wil->ring2cid_tid[i][0];
1462	if (cid >= wil->max_assoc_sta) /* skip BCAST */
1463	continue;
1464
1465	if (!wil->ring_tx_data[i].dot1x_open &&
1466	skb->protocol != cpu_to_be16(ETH_P_PAE))
1467	continue;
1468
1469	wil_dbg_txrx(wil, "Tx -> ring %d\n", i);
1470
1471	return ring;
1472	}
1473
1474	wil_dbg_txrx(wil, "Tx while no rings active?\n");
1475
1476	return NULL;
1477	}
1478
1479	/* Use one of 2 strategies:
1480	*
1481	* 1. New (real broadcast):
1482	* use dedicated broadcast vring
1483	* 2. Old (pseudo-DMS):
1484	* Find 1-st vring and return it;
1485	* duplicate skb and send it to other active vrings;
1486	* in all cases override dest address to unicast peer's address
1487	* Use old strategy when new is not supported yet:
1488	* - for PBSS
1489	*/
1490	static struct wil_ring *wil_find_tx_bcast_1(struct wil6210_priv *wil,
1491	struct wil6210_vif *vif,
1492	struct sk_buff *skb)
1493	{
1494	struct wil_ring *v;
1495	struct wil_ring_tx_data *txdata;
1496	int i = vif->bcast_ring;
1497
1498	if (i < 0)
1499	return NULL;
1500	v = &wil->ring_tx[i];
1501	txdata = &wil->ring_tx_data[i];
1502	if (!v->va || !txdata->enabled)
1503	return NULL;
1504	if (!wil->ring_tx_data[i].dot1x_open &&
1505	skb->protocol != cpu_to_be16(ETH_P_PAE))
1506	return NULL;
1507
1508	return v;
1509	}
1510
1511	/* apply multicast to unicast only for ARP and IP packets
1512	* (see NL80211_CMD_SET_MULTICAST_TO_UNICAST for more info)
1513	*/
1514	static bool wil_check_multicast_to_unicast(struct wil6210_priv *wil,
1515	struct sk_buff *skb)
1516	{
1517	const struct ethhdr *eth = (void *)skb->data;
1518	const struct vlan_ethhdr *ethvlan = (void *)skb->data;
1519	__be16 ethertype;
1520
1521	if (!wil->multicast_to_unicast)
1522	return false;
1523
1524	/* multicast to unicast conversion only for some payload */
1525	ethertype = eth->h_proto;
1526	if (ethertype == htons(ETH_P_8021Q) && skb->len >= VLAN_ETH_HLEN)
1527	ethertype = ethvlan->h_vlan_encapsulated_proto;
1528	switch (ethertype) {
1529	case htons(ETH_P_ARP):
1530	case htons(ETH_P_IP):
1531	case htons(ETH_P_IPV6):
1532	break;
1533	default:
1534	return false;
1535	}
1536
1537	return true;
1538	}
1539
1540	static void wil_set_da_for_vring(struct wil6210_priv *wil,
1541	struct sk_buff *skb, int vring_index)
1542	{
1543	u8 *da = wil_skb_get_da(skb);
1544	int cid = wil->ring2cid_tid[vring_index][0];
1545
1546	ether_addr_copy(dst: da, src: wil->sta[cid].addr);
1547	}
1548
1549	static struct wil_ring *wil_find_tx_bcast_2(struct wil6210_priv *wil,
1550	struct wil6210_vif *vif,
1551	struct sk_buff *skb)
1552	{
1553	struct wil_ring *v, *v2;
1554	struct sk_buff *skb2;
1555	int i;
1556	u8 cid;
1557	const u8 *src = wil_skb_get_sa(skb);
1558	struct wil_ring_tx_data *txdata, *txdata2;
1559	int min_ring_id = wil_get_min_tx_ring_id(wil);
1560
1561	/* find 1-st vring eligible for data */
1562	for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
1563	v = &wil->ring_tx[i];
1564	txdata = &wil->ring_tx_data[i];
1565	if (!v->va || !txdata->enabled || txdata->mid != vif->mid)
1566	continue;
1567
1568	cid = wil->ring2cid_tid[i][0];
1569	if (cid >= wil->max_assoc_sta) /* skip BCAST */
1570	continue;
1571	if (!wil->ring_tx_data[i].dot1x_open &&
1572	skb->protocol != cpu_to_be16(ETH_P_PAE))
1573	continue;
1574
1575	/* don't Tx back to source when re-routing Rx->Tx at the AP */
1576	if (0 == memcmp(p: wil->sta[cid].addr, q: src, ETH_ALEN))
1577	continue;
1578
1579	goto found;
1580	}
1581
1582	wil_dbg_txrx(wil, "Tx while no vrings active?\n");
1583
1584	return NULL;
1585
1586	found:
1587	wil_dbg_txrx(wil, "BCAST -> ring %d\n", i);
1588	wil_set_da_for_vring(wil, skb, vring_index: i);
1589
1590	/* find other active vrings and duplicate skb for each */
1591	for (i++; i < WIL6210_MAX_TX_RINGS; i++) {
1592	v2 = &wil->ring_tx[i];
1593	txdata2 = &wil->ring_tx_data[i];
1594	if (!v2->va || txdata2->mid != vif->mid)
1595	continue;
1596	cid = wil->ring2cid_tid[i][0];
1597	if (cid >= wil->max_assoc_sta) /* skip BCAST */
1598	continue;
1599	if (!wil->ring_tx_data[i].dot1x_open &&
1600	skb->protocol != cpu_to_be16(ETH_P_PAE))
1601	continue;
1602
1603	if (0 == memcmp(p: wil->sta[cid].addr, q: src, ETH_ALEN))
1604	continue;
1605
1606	skb2 = skb_copy(skb, GFP_ATOMIC);
1607	if (skb2) {
1608	wil_dbg_txrx(wil, "BCAST DUP -> ring %d\n", i);
1609	wil_set_da_for_vring(wil, skb: skb2, vring_index: i);
1610	wil_tx_ring(wil, vif, ring: v2, skb: skb2);
1611	/* successful call to wil_tx_ring takes skb2 ref */
1612	dev_kfree_skb_any(skb: skb2);
1613	} else {
1614	wil_err(wil, "skb_copy failed\n");
1615	}
1616	}
1617
1618	return v;
1619	}
1620
1621	static inline
1622	void wil_tx_desc_set_nr_frags(struct vring_tx_desc *d, int nr_frags)
1623	{
1624	d->mac.d[2] |= (nr_frags << MAC_CFG_DESC_TX_2_NUM_OF_DESCRIPTORS_POS);
1625	}
1626
1627	/* Sets the descriptor @d up for csum and/or TSO offloading. The corresponding
1628	* @skb is used to obtain the protocol and headers length.
1629	* @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data,
1630	* 2 - middle, 3 - last descriptor.
1631	*/
1632
1633	static void wil_tx_desc_offload_setup_tso(struct vring_tx_desc *d,
1634	struct sk_buff *skb,
1635	int tso_desc_type, bool is_ipv4,
1636	int tcp_hdr_len, int skb_net_hdr_len)
1637	{
1638	d->dma.b11 = ETH_HLEN; /* MAC header length */
1639	d->dma.b11 |= is_ipv4 << DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS;
1640
1641	d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS);
1642	/* L4 header len: TCP header length */
1643	d->dma.d0 |= (tcp_hdr_len & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK);
1644
1645	/* Setup TSO: bit and desc type */
1646	d->dma.d0 |= (BIT(DMA_CFG_DESC_TX_0_TCP_SEG_EN_POS)) |
1647	(tso_desc_type << DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS);
1648	d->dma.d0 |= (is_ipv4 << DMA_CFG_DESC_TX_0_IPV4_CHECKSUM_EN_POS);
1649
1650	d->dma.ip_length = skb_net_hdr_len;
1651	/* Enable TCP/UDP checksum */
1652	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS);
1653	/* Calculate pseudo-header */
1654	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS);
1655	}
1656
1657	/* Sets the descriptor @d up for csum. The corresponding
1658	* @skb is used to obtain the protocol and headers length.
1659	* Returns the protocol: 0 - not TCP, 1 - TCPv4, 2 - TCPv6.
1660	* Note, if d==NULL, the function only returns the protocol result.
1661	*
1662	* It is very similar to previous wil_tx_desc_offload_setup_tso. This
1663	* is "if unrolling" to optimize the critical path.
1664	*/
1665
1666	static int wil_tx_desc_offload_setup(struct vring_tx_desc *d,
1667	struct sk_buff *skb){
1668	int protocol;
1669
1670	if (skb->ip_summed != CHECKSUM_PARTIAL)
1671	return 0;
1672
1673	d->dma.b11 = ETH_HLEN; /* MAC header length */
1674
1675	switch (skb->protocol) {
1676	case cpu_to_be16(ETH_P_IP):
1677	protocol = ip_hdr(skb)->protocol;
1678	d->dma.b11 |= BIT(DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS);
1679	break;
1680	case cpu_to_be16(ETH_P_IPV6):
1681	protocol = ipv6_hdr(skb)->nexthdr;
1682	break;
1683	default:
1684	return -EINVAL;
1685	}
1686
1687	switch (protocol) {
1688	case IPPROTO_TCP:
1689	d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS);
1690	/* L4 header len: TCP header length */
1691	d->dma.d0 |=
1692	(tcp_hdrlen(skb) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK);
1693	break;
1694	case IPPROTO_UDP:
1695	/* L4 header len: UDP header length */
1696	d->dma.d0 |=
1697	(sizeof(struct udphdr) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK);
1698	break;
1699	default:
1700	return -EINVAL;
1701	}
1702
1703	d->dma.ip_length = skb_network_header_len(skb);
1704	/* Enable TCP/UDP checksum */
1705	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS);
1706	/* Calculate pseudo-header */
1707	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS);
1708
1709	return 0;
1710	}
1711
1712	static inline void wil_tx_last_desc(struct vring_tx_desc *d)
1713	{
1714	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS) |
1715	BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS) |
1716	BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS);
1717	}
1718
1719	static inline void wil_set_tx_desc_last_tso(volatile struct vring_tx_desc *d)
1720	{
1721	d->dma.d0 |= wil_tso_type_lst <<
1722	DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS;
1723	}
1724
1725	static int __wil_tx_vring_tso(struct wil6210_priv *wil, struct wil6210_vif *vif,
1726	struct wil_ring *vring, struct sk_buff *skb)
1727	{
1728	struct device *dev = wil_to_dev(wil);
1729
1730	/* point to descriptors in shared memory */
1731	volatile struct vring_tx_desc *_desc = NULL, *_hdr_desc,
1732	*_first_desc = NULL;
1733
1734	/* pointers to shadow descriptors */
1735	struct vring_tx_desc desc_mem, hdr_desc_mem, first_desc_mem,
1736	*d = &hdr_desc_mem, *hdr_desc = &hdr_desc_mem,
1737	*first_desc = &first_desc_mem;
1738
1739	/* pointer to shadow descriptors' context */
1740	struct wil_ctx *hdr_ctx, *first_ctx = NULL;
1741
1742	int descs_used = 0; /* total number of used descriptors */
1743	int sg_desc_cnt = 0; /* number of descriptors for current mss*/
1744
1745	u32 swhead = vring->swhead;
1746	int used, avail = wil_ring_avail_tx(ring: vring);
1747	int nr_frags = skb_shinfo(skb)->nr_frags;
1748	int min_desc_required = nr_frags + 1;
1749	int mss = skb_shinfo(skb)->gso_size; /* payload size w/o headers */
1750	int f, len, hdrlen, headlen;
1751	int vring_index = vring - wil->ring_tx;
1752	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[vring_index];
1753	uint i = swhead;
1754	dma_addr_t pa;
1755	const skb_frag_t *frag = NULL;
1756	int rem_data = mss;
1757	int lenmss;
1758	int hdr_compensation_need = true;
1759	int desc_tso_type = wil_tso_type_first;
1760	bool is_ipv4;
1761	int tcp_hdr_len;
1762	int skb_net_hdr_len;
1763	int gso_type;
1764	int rc = -EINVAL;
1765
1766	wil_dbg_txrx(wil, "tx_vring_tso: %d bytes to vring %d\n", skb->len,
1767	vring_index);
1768
1769	if (unlikely(!txdata->enabled))
1770	return -EINVAL;
1771
1772	/* A typical page 4K is 3-4 payloads, we assume each fragment
1773	* is a full payload, that's how min_desc_required has been
1774	* calculated. In real we might need more or less descriptors,
1775	* this is the initial check only.
1776	*/
1777	if (unlikely(avail < min_desc_required)) {
1778	wil_err_ratelimited(wil,
1779	"TSO: Tx ring[%2d] full. No space for %d fragments\n",
1780	vring_index, min_desc_required);
1781	return -ENOMEM;
1782	}
1783
1784	/* Header Length = MAC header len + IP header len + TCP header len*/
1785	hdrlen = skb_tcp_all_headers(skb);
1786
1787	gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV6 | SKB_GSO_TCPV4);
1788	switch (gso_type) {
1789	case SKB_GSO_TCPV4:
1790	/* TCP v4, zero out the IP length and IPv4 checksum fields
1791	* as required by the offloading doc
1792	*/
1793	ip_hdr(skb)->tot_len = 0;
1794	ip_hdr(skb)->check = 0;
1795	is_ipv4 = true;
1796	break;
1797	case SKB_GSO_TCPV6:
1798	/* TCP v6, zero out the payload length */
1799	ipv6_hdr(skb)->payload_len = 0;
1800	is_ipv4 = false;
1801	break;
1802	default:
1803	/* other than TCPv4 or TCPv6 types are not supported for TSO.
1804	* It is also illegal for both to be set simultaneously
1805	*/
1806	return -EINVAL;
1807	}
1808
1809	if (skb->ip_summed != CHECKSUM_PARTIAL)
1810	return -EINVAL;
1811
1812	/* tcp header length and skb network header length are fixed for all
1813	* packet's descriptors - read then once here
1814	*/
1815	tcp_hdr_len = tcp_hdrlen(skb);
1816	skb_net_hdr_len = skb_network_header_len(skb);
1817
1818	_hdr_desc = &vring->va[i].tx.legacy;
1819
1820	pa = dma_map_single(dev, skb->data, hdrlen, DMA_TO_DEVICE);
1821	if (unlikely(dma_mapping_error(dev, pa))) {
1822	wil_err(wil, "TSO: Skb head DMA map error\n");
1823	goto err_exit;
1824	}
1825
1826	wil->txrx_ops.tx_desc_map((union wil_tx_desc *)hdr_desc, pa,
1827	hdrlen, vring_index);
1828	wil_tx_desc_offload_setup_tso(d: hdr_desc, skb, tso_desc_type: wil_tso_type_hdr, is_ipv4,
1829	tcp_hdr_len, skb_net_hdr_len);
1830	wil_tx_last_desc(d: hdr_desc);
1831
1832	vring->ctx[i].mapped_as = wil_mapped_as_single;
1833	hdr_ctx = &vring->ctx[i];
1834
1835	descs_used++;
1836	headlen = skb_headlen(skb) - hdrlen;
1837
1838	for (f = headlen ? -1 : 0; f < nr_frags; f++) {
1839	if (headlen) {
1840	len = headlen;
1841	wil_dbg_txrx(wil, "TSO: process skb head, len %u\n",
1842	len);
1843	} else {
1844	frag = &skb_shinfo(skb)->frags[f];
1845	len = skb_frag_size(frag);
1846	wil_dbg_txrx(wil, "TSO: frag[%d]: len %u\n", f, len);
1847	}
1848
1849	while (len) {
1850	wil_dbg_txrx(wil,
1851	"TSO: len %d, rem_data %d, descs_used %d\n",
1852	len, rem_data, descs_used);
1853
1854	if (descs_used == avail) {
1855	wil_err_ratelimited(wil, "TSO: ring overflow\n");
1856	rc = -ENOMEM;
1857	goto mem_error;
1858	}
1859
1860	lenmss = min_t(int, rem_data, len);
1861	i = (swhead + descs_used) % vring->size;
1862	wil_dbg_txrx(wil, "TSO: lenmss %d, i %d\n", lenmss, i);
1863
1864	if (!headlen) {
1865	pa = skb_frag_dma_map(dev, frag,
1866	offset: skb_frag_size(frag) - len,
1867	size: lenmss, dir: DMA_TO_DEVICE);
1868	vring->ctx[i].mapped_as = wil_mapped_as_page;
1869	} else {
1870	pa = dma_map_single(dev,
1871	skb->data +
1872	skb_headlen(skb) - headlen,
1873	lenmss,
1874	DMA_TO_DEVICE);
1875	vring->ctx[i].mapped_as = wil_mapped_as_single;
1876	headlen -= lenmss;
1877	}
1878
1879	if (unlikely(dma_mapping_error(dev, pa))) {
1880	wil_err(wil, "TSO: DMA map page error\n");
1881	goto mem_error;
1882	}
1883
1884	_desc = &vring->va[i].tx.legacy;
1885
1886	if (!_first_desc) {
1887	_first_desc = _desc;
1888	first_ctx = &vring->ctx[i];
1889	d = first_desc;
1890	} else {
1891	d = &desc_mem;
1892	}
1893
1894	wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d,
1895	pa, lenmss, vring_index);
1896	wil_tx_desc_offload_setup_tso(d, skb, tso_desc_type: desc_tso_type,
1897	is_ipv4, tcp_hdr_len,
1898	skb_net_hdr_len);
1899
1900	/* use tso_type_first only once */
1901	desc_tso_type = wil_tso_type_mid;
1902
1903	descs_used++; /* desc used so far */
1904	sg_desc_cnt++; /* desc used for this segment */
1905	len -= lenmss;
1906	rem_data -= lenmss;
1907
1908	wil_dbg_txrx(wil,
1909	"TSO: len %d, rem_data %d, descs_used %d, sg_desc_cnt %d,\n",
1910	len, rem_data, descs_used, sg_desc_cnt);
1911
1912	/* Close the segment if reached mss size or last frag*/
1913	if (rem_data == 0 || (f == nr_frags - 1 && len == 0)) {
1914	if (hdr_compensation_need) {
1915	/* first segment include hdr desc for
1916	* release
1917	*/
1918	hdr_ctx->nr_frags = sg_desc_cnt;
1919	wil_tx_desc_set_nr_frags(d: first_desc,
1920	nr_frags: sg_desc_cnt +
1921	1);
1922	hdr_compensation_need = false;
1923	} else {
1924	wil_tx_desc_set_nr_frags(d: first_desc,
1925	nr_frags: sg_desc_cnt);
1926	}
1927	first_ctx->nr_frags = sg_desc_cnt - 1;
1928
1929	wil_tx_last_desc(d);
1930
1931	/* first descriptor may also be the last
1932	* for this mss - make sure not to copy
1933	* it twice
1934	*/
1935	if (first_desc != d)
1936	*_first_desc = *first_desc;
1937
1938	/*last descriptor will be copied at the end
1939	* of this TS processing
1940	*/
1941	if (f < nr_frags - 1 || len > 0)
1942	*_desc = *d;
1943
1944	rem_data = mss;
1945	_first_desc = NULL;
1946	sg_desc_cnt = 0;
1947	} else if (first_desc != d) /* update mid descriptor */
1948	*_desc = *d;
1949	}
1950	}
1951
1952	if (!_desc)
1953	goto mem_error;
1954
1955	/* first descriptor may also be the last.
1956	* in this case d pointer is invalid
1957	*/
1958	if (_first_desc == _desc)
1959	d = first_desc;
1960
1961	/* Last data descriptor */
1962	wil_set_tx_desc_last_tso(d);
1963	*_desc = *d;
1964
1965	/* Fill the total number of descriptors in first desc (hdr)*/
1966	wil_tx_desc_set_nr_frags(d: hdr_desc, nr_frags: descs_used);
1967	*_hdr_desc = *hdr_desc;
1968
1969	/* hold reference to skb
1970	* to prevent skb release before accounting
1971	* in case of immediate "tx done"
1972	*/
1973	vring->ctx[i].skb = skb_get(skb);
1974
1975	/* performance monitoring */
1976	used = wil_ring_used_tx(ring: vring);
1977	if (wil_val_in_range(val: wil->ring_idle_trsh,
1978	min: used, max: used + descs_used)) {
1979	txdata->idle += get_cycles() - txdata->last_idle;
1980	wil_dbg_txrx(wil, "Ring[%2d] not idle %d -> %d\n",
1981	vring_index, used, used + descs_used);
1982	}
1983
1984	/* Make sure to advance the head only after descriptor update is done.
1985	* This will prevent a race condition where the completion thread
1986	* will see the DU bit set from previous run and will handle the
1987	* skb before it was completed.
1988	*/
1989	wmb();
1990
1991	/* advance swhead */
1992	wil_ring_advance_head(ring: vring, n: descs_used);
1993	wil_dbg_txrx(wil, "TSO: Tx swhead %d -> %d\n", swhead, vring->swhead);
1994
1995	/* make sure all writes to descriptors (shared memory) are done before
1996	* committing them to HW
1997	*/
1998	wmb();
1999
2000	if (wil->tx_latency)
2001	*(ktime_t *)&skb->cb = ktime_get();
2002	else
2003	memset(skb->cb, 0, sizeof(ktime_t));
2004
2005	wil_w(wil, reg: vring->hwtail, val: vring->swhead);
2006	return 0;
2007
2008	mem_error:
2009	while (descs_used > 0) {
2010	struct wil_ctx *ctx;
2011
2012	i = (swhead + descs_used - 1) % vring->size;
2013	d = (struct vring_tx_desc *)&vring->va[i].tx.legacy;
2014	_desc = &vring->va[i].tx.legacy;
2015	*d = *_desc;
2016	_desc->dma.status = TX_DMA_STATUS_DU;
2017	ctx = &vring->ctx[i];
2018	wil_txdesc_unmap(dev, desc: (union wil_tx_desc *)d, ctx);
2019	memset(ctx, 0, sizeof(*ctx));
2020	descs_used--;
2021	}
2022	err_exit:
2023	return rc;
2024	}
2025
2026	static int __wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
2027	struct wil_ring *ring, struct sk_buff *skb)
2028	{
2029	struct device *dev = wil_to_dev(wil);
2030	struct vring_tx_desc dd, *d = &dd;
2031	volatile struct vring_tx_desc *_d;
2032	u32 swhead = ring->swhead;
2033	int avail = wil_ring_avail_tx(ring);
2034	int nr_frags = skb_shinfo(skb)->nr_frags;
2035	uint f = 0;
2036	int ring_index = ring - wil->ring_tx;
2037	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_index];
2038	uint i = swhead;
2039	dma_addr_t pa;
2040	int used;
2041	bool mcast = (ring_index == vif->bcast_ring);
2042	uint len = skb_headlen(skb);
2043
2044	wil_dbg_txrx(wil, "tx_ring: %d bytes to ring %d, nr_frags %d\n",
2045	skb->len, ring_index, nr_frags);
2046
2047	if (unlikely(!txdata->enabled))
2048	return -EINVAL;
2049
2050	if (unlikely(avail < 1 + nr_frags)) {
2051	wil_err_ratelimited(wil,
2052	"Tx ring[%2d] full. No space for %d fragments\n",
2053	ring_index, 1 + nr_frags);
2054	return -ENOMEM;
2055	}
2056	_d = &ring->va[i].tx.legacy;
2057
2058	pa = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
2059
2060	wil_dbg_txrx(wil, "Tx[%2d] skb %d bytes 0x%p -> %pad\n", ring_index,
2061	skb_headlen(skb), skb->data, &pa);
2062	wil_hex_dump_txrx("Tx ", DUMP_PREFIX_OFFSET, 16, 1,
2063	skb->data, skb_headlen(skb), false);
2064
2065	if (unlikely(dma_mapping_error(dev, pa)))
2066	return -EINVAL;
2067	ring->ctx[i].mapped_as = wil_mapped_as_single;
2068	/* 1-st segment */
2069	wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, pa, len,
2070	ring_index);
2071	if (unlikely(mcast)) {
2072	d->mac.d[0] |= BIT(MAC_CFG_DESC_TX_0_MCS_EN_POS); /* MCS 0 */
2073	if (unlikely(len > WIL_BCAST_MCS0_LIMIT)) /* set MCS 1 */
2074	d->mac.d[0] |= (1 << MAC_CFG_DESC_TX_0_MCS_INDEX_POS);
2075	}
2076	/* Process TCP/UDP checksum offloading */
2077	if (unlikely(wil_tx_desc_offload_setup(d, skb))) {
2078	wil_err(wil, "Tx[%2d] Failed to set cksum, drop packet\n",
2079	ring_index);
2080	goto dma_error;
2081	}
2082
2083	ring->ctx[i].nr_frags = nr_frags;
2084	wil_tx_desc_set_nr_frags(d, nr_frags: nr_frags + 1);
2085
2086	/* middle segments */
2087	for (; f < nr_frags; f++) {
2088	const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
2089	int len = skb_frag_size(frag);
2090
2091	*_d = *d;
2092	wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i);
2093	wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4,
2094	(const void *)d, sizeof(*d), false);
2095	i = (swhead + f + 1) % ring->size;
2096	_d = &ring->va[i].tx.legacy;
2097	pa = skb_frag_dma_map(dev, frag, offset: 0, size: skb_frag_size(frag),
2098	dir: DMA_TO_DEVICE);
2099	if (unlikely(dma_mapping_error(dev, pa))) {
2100	wil_err(wil, "Tx[%2d] failed to map fragment\n",
2101	ring_index);
2102	goto dma_error;
2103	}
2104	ring->ctx[i].mapped_as = wil_mapped_as_page;
2105	wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d,
2106	pa, len, ring_index);
2107	/* no need to check return code -
2108	* if it succeeded for 1-st descriptor,
2109	* it will succeed here too
2110	*/
2111	wil_tx_desc_offload_setup(d, skb);
2112	}
2113	/* for the last seg only */
2114	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS);
2115	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS);
2116	d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS);
2117	*_d = *d;
2118	wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i);
2119	wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4,
2120	(const void *)d, sizeof(*d), false);
2121
2122	/* hold reference to skb
2123	* to prevent skb release before accounting
2124	* in case of immediate "tx done"
2125	*/
2126	ring->ctx[i].skb = skb_get(skb);
2127
2128	/* performance monitoring */
2129	used = wil_ring_used_tx(ring);
2130	if (wil_val_in_range(val: wil->ring_idle_trsh,
2131	min: used, max: used + nr_frags + 1)) {
2132	txdata->idle += get_cycles() - txdata->last_idle;
2133	wil_dbg_txrx(wil, "Ring[%2d] not idle %d -> %d\n",
2134	ring_index, used, used + nr_frags + 1);
2135	}
2136
2137	/* Make sure to advance the head only after descriptor update is done.
2138	* This will prevent a race condition where the completion thread
2139	* will see the DU bit set from previous run and will handle the
2140	* skb before it was completed.
2141	*/
2142	wmb();
2143
2144	/* advance swhead */
2145	wil_ring_advance_head(ring, n: nr_frags + 1);
2146	wil_dbg_txrx(wil, "Tx[%2d] swhead %d -> %d\n", ring_index, swhead,
2147	ring->swhead);
2148	trace_wil6210_tx(vring: ring_index, index: swhead, len: skb->len, frags: nr_frags);
2149
2150	/* make sure all writes to descriptors (shared memory) are done before
2151	* committing them to HW
2152	*/
2153	wmb();
2154
2155	if (wil->tx_latency)
2156	*(ktime_t *)&skb->cb = ktime_get();
2157	else
2158	memset(skb->cb, 0, sizeof(ktime_t));
2159
2160	wil_w(wil, reg: ring->hwtail, val: ring->swhead);
2161
2162	return 0;
2163	dma_error:
2164	/* unmap what we have mapped */
2165	nr_frags = f + 1; /* frags mapped + one for skb head */
2166	for (f = 0; f < nr_frags; f++) {
2167	struct wil_ctx *ctx;
2168
2169	i = (swhead + f) % ring->size;
2170	ctx = &ring->ctx[i];
2171	_d = &ring->va[i].tx.legacy;
2172	*d = *_d;
2173	_d->dma.status = TX_DMA_STATUS_DU;
2174	wil->txrx_ops.tx_desc_unmap(dev,
2175	(union wil_tx_desc *)d,
2176	ctx);
2177
2178	memset(ctx, 0, sizeof(*ctx));
2179	}
2180
2181	return -EINVAL;
2182	}
2183
2184	static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
2185	struct wil_ring *ring, struct sk_buff *skb)
2186	{
2187	int ring_index = ring - wil->ring_tx;
2188	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_index];
2189	int rc;
2190
2191	spin_lock(lock: &txdata->lock);
2192
2193	if (test_bit(wil_status_suspending, wil->status) ||
2194	test_bit(wil_status_suspended, wil->status) ||
2195	test_bit(wil_status_resuming, wil->status)) {
2196	wil_dbg_txrx(wil,
2197	"suspend/resume in progress. drop packet\n");
2198	spin_unlock(lock: &txdata->lock);
2199	return -EINVAL;
2200	}
2201
2202	rc = (skb_is_gso(skb) ? wil->txrx_ops.tx_ring_tso : __wil_tx_ring)
2203	(wil, vif, ring, skb);
2204
2205	spin_unlock(lock: &txdata->lock);
2206
2207	return rc;
2208	}
2209
2210	/* Check status of tx vrings and stop/wake net queues if needed
2211	* It will start/stop net queues of a specific VIF net_device.
2212	*
2213	* This function does one of two checks:
2214	* In case check_stop is true, will check if net queues need to be stopped. If
2215	* the conditions for stopping are met, netif_tx_stop_all_queues() is called.
2216	* In case check_stop is false, will check if net queues need to be waked. If
2217	* the conditions for waking are met, netif_tx_wake_all_queues() is called.
2218	* vring is the vring which is currently being modified by either adding
2219	* descriptors (tx) into it or removing descriptors (tx complete) from it. Can
2220	* be null when irrelevant (e.g. connect/disconnect events).
2221	*
2222	* The implementation is to stop net queues if modified vring has low
2223	* descriptor availability. Wake if all vrings are not in low descriptor
2224	* availability and modified vring has high descriptor availability.
2225	*/
2226	static inline void __wil_update_net_queues(struct wil6210_priv *wil,
2227	struct wil6210_vif *vif,
2228	struct wil_ring *ring,
2229	bool check_stop)
2230	{
2231	int i;
2232	int min_ring_id = wil_get_min_tx_ring_id(wil);
2233
2234	if (unlikely(!vif))
2235	return;
2236
2237	if (ring)
2238	wil_dbg_txrx(wil, "vring %d, mid %d, check_stop=%d, stopped=%d",
2239	(int)(ring - wil->ring_tx), vif->mid, check_stop,
2240	vif->net_queue_stopped);
2241	else
2242	wil_dbg_txrx(wil, "check_stop=%d, mid=%d, stopped=%d",
2243	check_stop, vif->mid, vif->net_queue_stopped);
2244
2245	if (ring && drop_if_ring_full)
2246	/* no need to stop/wake net queues */
2247	return;
2248
2249	if (check_stop == vif->net_queue_stopped)
2250	/* net queues already in desired state */
2251	return;
2252
2253	if (check_stop) {
2254	if (!ring || unlikely(wil_ring_avail_low(ring))) {
2255	/* not enough room in the vring */
2256	netif_tx_stop_all_queues(vif_to_ndev(vif));
2257	vif->net_queue_stopped = true;
2258	wil_dbg_txrx(wil, "netif_tx_stop called\n");
2259	}
2260	return;
2261	}
2262
2263	/* Do not wake the queues in suspend flow */
2264	if (test_bit(wil_status_suspending, wil->status) ||
2265	test_bit(wil_status_suspended, wil->status))
2266	return;
2267
2268	/* check wake */
2269	for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) {
2270	struct wil_ring *cur_ring = &wil->ring_tx[i];
2271	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[i];
2272
2273	if (txdata->mid != vif->mid || !cur_ring->va ||
2274	!txdata->enabled || cur_ring == ring)
2275	continue;
2276
2277	if (wil_ring_avail_low(ring: cur_ring)) {
2278	wil_dbg_txrx(wil, "ring %d full, can't wake\n",
2279	(int)(cur_ring - wil->ring_tx));
2280	return;
2281	}
2282	}
2283
2284	if (!ring || wil_ring_avail_high(ring)) {
2285	/* enough room in the ring */
2286	wil_dbg_txrx(wil, "calling netif_tx_wake\n");
2287	netif_tx_wake_all_queues(vif_to_ndev(vif));
2288	vif->net_queue_stopped = false;
2289	}
2290	}
2291
2292	void wil_update_net_queues(struct wil6210_priv *wil, struct wil6210_vif *vif,
2293	struct wil_ring *ring, bool check_stop)
2294	{
2295	spin_lock(lock: &wil->net_queue_lock);
2296	__wil_update_net_queues(wil, vif, ring, check_stop);
2297	spin_unlock(lock: &wil->net_queue_lock);
2298	}
2299
2300	void wil_update_net_queues_bh(struct wil6210_priv *wil, struct wil6210_vif *vif,
2301	struct wil_ring *ring, bool check_stop)
2302	{
2303	spin_lock_bh(lock: &wil->net_queue_lock);
2304	__wil_update_net_queues(wil, vif, ring, check_stop);
2305	spin_unlock_bh(lock: &wil->net_queue_lock);
2306	}
2307
2308	netdev_tx_t wil_start_xmit(struct sk_buff *skb, struct net_device *ndev)
2309	{
2310	struct wil6210_vif *vif = ndev_to_vif(ndev);
2311	struct wil6210_priv *wil = vif_to_wil(vif);
2312	const u8 *da = wil_skb_get_da(skb);
2313	bool bcast = is_multicast_ether_addr(addr: da);
2314	struct wil_ring *ring;
2315	static bool pr_once_fw;
2316	int rc;
2317
2318	wil_dbg_txrx(wil, "start_xmit\n");
2319	if (unlikely(!test_bit(wil_status_fwready, wil->status))) {
2320	if (!pr_once_fw) {
2321	wil_err(wil, "FW not ready\n");
2322	pr_once_fw = true;
2323	}
2324	goto drop;
2325	}
2326	if (unlikely(!test_bit(wil_vif_fwconnected, vif->status))) {
2327	wil_dbg_ratelimited(wil,
2328	fmt: "VIF not connected, packet dropped\n");
2329	goto drop;
2330	}
2331	if (unlikely(vif->wdev.iftype == NL80211_IFTYPE_MONITOR)) {
2332	wil_err(wil, "Xmit in monitor mode not supported\n");
2333	goto drop;
2334	}
2335	pr_once_fw = false;
2336
2337	/* find vring */
2338	if (vif->wdev.iftype == NL80211_IFTYPE_STATION && !vif->pbss) {
2339	/* in STA mode (ESS), all to same VRING (to AP) */
2340	ring = wil_find_tx_ring_sta(wil, vif, skb);
2341	} else if (bcast) {
2342	if (vif->pbss || wil_check_multicast_to_unicast(wil, skb))
2343	/* in pbss, no bcast VRING - duplicate skb in
2344	* all stations VRINGs
2345	*/
2346	ring = wil_find_tx_bcast_2(wil, vif, skb);
2347	else if (vif->wdev.iftype == NL80211_IFTYPE_AP)
2348	/* AP has a dedicated bcast VRING */
2349	ring = wil_find_tx_bcast_1(wil, vif, skb);
2350	else
2351	/* unexpected combination, fallback to duplicating
2352	* the skb in all stations VRINGs
2353	*/
2354	ring = wil_find_tx_bcast_2(wil, vif, skb);
2355	} else {
2356	/* unicast, find specific VRING by dest. address */
2357	ring = wil_find_tx_ucast(wil, vif, skb);
2358	}
2359	if (unlikely(!ring)) {
2360	wil_dbg_txrx(wil, "No Tx RING found for %pM\n", da);
2361	goto drop;
2362	}
2363	/* set up vring entry */
2364	rc = wil_tx_ring(wil, vif, ring, skb);
2365
2366	switch (rc) {
2367	case 0:
2368	/* shall we stop net queues? */
2369	wil_update_net_queues_bh(wil, vif, ring, check_stop: true);
2370	/* statistics will be updated on the tx_complete */
2371	dev_kfree_skb_any(skb);
2372	return NETDEV_TX_OK;
2373	case -ENOMEM:
2374	if (drop_if_ring_full)
2375	goto drop;
2376	return NETDEV_TX_BUSY;
2377	default:
2378	break; /* goto drop; */
2379	}
2380	drop:
2381	ndev->stats.tx_dropped++;
2382	dev_kfree_skb_any(skb);
2383
2384	return NET_XMIT_DROP;
2385	}
2386
2387	void wil_tx_latency_calc(struct wil6210_priv *wil, struct sk_buff *skb,
2388	struct wil_sta_info *sta)
2389	{
2390	int skb_time_us;
2391	int bin;
2392
2393	if (!wil->tx_latency)
2394	return;
2395
2396	if (ktime_to_ms(kt: *(ktime_t *)&skb->cb) == 0)
2397	return;
2398
2399	skb_time_us = ktime_us_delta(later: ktime_get(), earlier: *(ktime_t *)&skb->cb);
2400	bin = skb_time_us / wil->tx_latency_res;
2401	bin = min_t(int, bin, WIL_NUM_LATENCY_BINS - 1);
2402
2403	wil_dbg_txrx(wil, "skb time %dus => bin %d\n", skb_time_us, bin);
2404	sta->tx_latency_bins[bin]++;
2405	sta->stats.tx_latency_total_us += skb_time_us;
2406	if (skb_time_us < sta->stats.tx_latency_min_us)
2407	sta->stats.tx_latency_min_us = skb_time_us;
2408	if (skb_time_us > sta->stats.tx_latency_max_us)
2409	sta->stats.tx_latency_max_us = skb_time_us;
2410	}
2411
2412	/* Clean up transmitted skb's from the Tx VRING
2413	*
2414	* Return number of descriptors cleared
2415	*
2416	* Safe to call from IRQ
2417	*/
2418	int wil_tx_complete(struct wil6210_vif *vif, int ringid)
2419	{
2420	struct wil6210_priv *wil = vif_to_wil(vif);
2421	struct net_device *ndev = vif_to_ndev(vif);
2422	struct device *dev = wil_to_dev(wil);
2423	struct wil_ring *vring = &wil->ring_tx[ringid];
2424	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ringid];
2425	int done = 0;
2426	int cid = wil->ring2cid_tid[ringid][0];
2427	struct wil_net_stats *stats = NULL;
2428	volatile struct vring_tx_desc *_d;
2429	int used_before_complete;
2430	int used_new;
2431
2432	if (unlikely(!vring->va)) {
2433	wil_err(wil, "Tx irq[%d]: vring not initialized\n", ringid);
2434	return 0;
2435	}
2436
2437	if (unlikely(!txdata->enabled)) {
2438	wil_info(wil, "Tx irq[%d]: vring disabled\n", ringid);
2439	return 0;
2440	}
2441
2442	wil_dbg_txrx(wil, "tx_complete: (%d)\n", ringid);
2443
2444	used_before_complete = wil_ring_used_tx(ring: vring);
2445
2446	if (cid < wil->max_assoc_sta)
2447	stats = &wil->sta[cid].stats;
2448
2449	while (!wil_ring_is_empty(ring: vring)) {
2450	int new_swtail;
2451	struct wil_ctx *ctx = &vring->ctx[vring->swtail];
2452	/* For the fragmented skb, HW will set DU bit only for the
2453	* last fragment. look for it.
2454	* In TSO the first DU will include hdr desc
2455	*/
2456	int lf = (vring->swtail + ctx->nr_frags) % vring->size;
2457	/* TODO: check we are not past head */
2458
2459	_d = &vring->va[lf].tx.legacy;
2460	if (unlikely(!(_d->dma.status & TX_DMA_STATUS_DU)))
2461	break;
2462
2463	new_swtail = (lf + 1) % vring->size;
2464	while (vring->swtail != new_swtail) {
2465	struct vring_tx_desc dd, *d = &dd;
2466	u16 dmalen;
2467	struct sk_buff *skb;
2468
2469	ctx = &vring->ctx[vring->swtail];
2470	skb = ctx->skb;
2471	_d = &vring->va[vring->swtail].tx.legacy;
2472
2473	*d = *_d;
2474
2475	dmalen = le16_to_cpu(d->dma.length);
2476	trace_wil6210_tx_done(vring: ringid, index: vring->swtail, len: dmalen,
2477	err: d->dma.error);
2478	wil_dbg_txrx(wil,
2479	"TxC[%2d][%3d] : %d bytes, status 0x%02x err 0x%02x\n",
2480	ringid, vring->swtail, dmalen,
2481	d->dma.status, d->dma.error);
2482	wil_hex_dump_txrx("TxCD ", DUMP_PREFIX_NONE, 32, 4,
2483	(const void *)d, sizeof(*d), false);
2484
2485	wil->txrx_ops.tx_desc_unmap(dev,
2486	(union wil_tx_desc *)d,
2487	ctx);
2488
2489	if (skb) {
2490	if (likely(d->dma.error == 0)) {
2491	ndev->stats.tx_packets++;
2492	ndev->stats.tx_bytes += skb->len;
2493	if (stats) {
2494	stats->tx_packets++;
2495	stats->tx_bytes += skb->len;
2496
2497	wil_tx_latency_calc(wil, skb,
2498	sta: &wil->sta[cid]);
2499	}
2500	} else {
2501	ndev->stats.tx_errors++;
2502	if (stats)
2503	stats->tx_errors++;
2504	}
2505
2506	if (skb->protocol == cpu_to_be16(ETH_P_PAE))
2507	wil_tx_complete_handle_eapol(vif, skb);
2508
2509	wil_consume_skb(skb, acked: d->dma.error == 0);
2510	}
2511	memset(ctx, 0, sizeof(*ctx));
2512	/* Make sure the ctx is zeroed before updating the tail
2513	* to prevent a case where wil_tx_ring will see
2514	* this descriptor as used and handle it before ctx zero
2515	* is completed.
2516	*/
2517	wmb();
2518	/* There is no need to touch HW descriptor:
2519	* - ststus bit TX_DMA_STATUS_DU is set by design,
2520	* so hardware will not try to process this desc.,
2521	* - rest of descriptor will be initialized on Tx.
2522	*/
2523	vring->swtail = wil_ring_next_tail(ring: vring);
2524	done++;
2525	}
2526	}
2527
2528	/* performance monitoring */
2529	used_new = wil_ring_used_tx(ring: vring);
2530	if (wil_val_in_range(val: wil->ring_idle_trsh,
2531	min: used_new, max: used_before_complete)) {
2532	wil_dbg_txrx(wil, "Ring[%2d] idle %d -> %d\n",
2533	ringid, used_before_complete, used_new);
2534	txdata->last_idle = get_cycles();
2535	}
2536
2537	/* shall we wake net queues? */
2538	if (done)
2539	wil_update_net_queues(wil, vif, ring: vring, check_stop: false);
2540
2541	return done;
2542	}
2543
2544	static inline int wil_tx_init(struct wil6210_priv *wil)
2545	{
2546	return 0;
2547	}
2548
2549	static inline void wil_tx_fini(struct wil6210_priv *wil) {}
2550
2551	static void wil_get_reorder_params(struct wil6210_priv *wil,
2552	struct sk_buff *skb, int *tid, int *cid,
2553	int *mid, u16 *seq, int *mcast, int *retry)
2554	{
2555	struct vring_rx_desc *d = wil_skb_rxdesc(skb);
2556
2557	*tid = wil_rxdesc_tid(d);
2558	*cid = wil_skb_get_cid(skb);
2559	*mid = wil_rxdesc_mid(d);
2560	*seq = wil_rxdesc_seq(d);
2561	*mcast = wil_rxdesc_mcast(d);
2562	*retry = wil_rxdesc_retry(d);
2563	}
2564
2565	void wil_init_txrx_ops_legacy_dma(struct wil6210_priv *wil)
2566	{
2567	wil->txrx_ops.configure_interrupt_moderation =
2568	wil_configure_interrupt_moderation;
2569	/* TX ops */
2570	wil->txrx_ops.tx_desc_map = wil_tx_desc_map;
2571	wil->txrx_ops.tx_desc_unmap = wil_txdesc_unmap;
2572	wil->txrx_ops.tx_ring_tso = __wil_tx_vring_tso;
2573	wil->txrx_ops.ring_init_tx = wil_vring_init_tx;
2574	wil->txrx_ops.ring_fini_tx = wil_vring_free;
2575	wil->txrx_ops.ring_init_bcast = wil_vring_init_bcast;
2576	wil->txrx_ops.tx_init = wil_tx_init;
2577	wil->txrx_ops.tx_fini = wil_tx_fini;
2578	wil->txrx_ops.tx_ring_modify = wil_tx_vring_modify;
2579	/* RX ops */
2580	wil->txrx_ops.rx_init = wil_rx_init;
2581	wil->txrx_ops.wmi_addba_rx_resp = wmi_addba_rx_resp;
2582	wil->txrx_ops.get_reorder_params = wil_get_reorder_params;
2583	wil->txrx_ops.get_netif_rx_params =
2584	wil_get_netif_rx_params;
2585	wil->txrx_ops.rx_crypto_check = wil_rx_crypto_check;
2586	wil->txrx_ops.rx_error_check = wil_rx_error_check;
2587	wil->txrx_ops.is_rx_idle = wil_is_rx_idle;
2588	wil->txrx_ops.rx_fini = wil_rx_fini;
2589	}
2590