1	// SPDX-License-Identifier: GPL-2.0-only
2	/* Copyright (C) 2023 Intel Corporation */
3
4	#include "idpf.h"
5	#include "idpf_virtchnl.h"
6
7	/**
8	* idpf_buf_lifo_push - push a buffer pointer onto stack
9	* @stack: pointer to stack struct
10	* @buf: pointer to buf to push
11	*
12	* Returns 0 on success, negative on failure
13	**/
14	static int idpf_buf_lifo_push(struct idpf_buf_lifo *stack,
15	struct idpf_tx_stash *buf)
16	{
17	if (unlikely(stack->top == stack->size))
18	return -ENOSPC;
19
20	stack->bufs[stack->top++] = buf;
21
22	return 0;
23	}
24
25	/**
26	* idpf_buf_lifo_pop - pop a buffer pointer from stack
27	* @stack: pointer to stack struct
28	**/
29	static struct idpf_tx_stash *idpf_buf_lifo_pop(struct idpf_buf_lifo *stack)
30	{
31	if (unlikely(!stack->top))
32	return NULL;
33
34	return stack->bufs[--stack->top];
35	}
36
37	/**
38	* idpf_tx_timeout - Respond to a Tx Hang
39	* @netdev: network interface device structure
40	* @txqueue: TX queue
41	*/
42	void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
43	{
44	struct idpf_adapter *adapter = idpf_netdev_to_adapter(netdev);
45
46	adapter->tx_timeout_count++;
47
48	netdev_err(dev: netdev, format: "Detected Tx timeout: Count %d, Queue %d\n",
49	adapter->tx_timeout_count, txqueue);
50	if (!idpf_is_reset_in_prog(adapter)) {
51	set_bit(nr: IDPF_HR_FUNC_RESET, addr: adapter->flags);
52	queue_delayed_work(wq: adapter->vc_event_wq,
53	dwork: &adapter->vc_event_task,
54	delay: msecs_to_jiffies(m: 10));
55	}
56	}
57
58	/**
59	* idpf_tx_buf_rel - Release a Tx buffer
60	* @tx_q: the queue that owns the buffer
61	* @tx_buf: the buffer to free
62	*/
63	static void idpf_tx_buf_rel(struct idpf_queue *tx_q, struct idpf_tx_buf *tx_buf)
64	{
65	if (tx_buf->skb) {
66	if (dma_unmap_len(tx_buf, len))
67	dma_unmap_single(tx_q->dev,
68	dma_unmap_addr(tx_buf, dma),
69	dma_unmap_len(tx_buf, len),
70	DMA_TO_DEVICE);
71	dev_kfree_skb_any(skb: tx_buf->skb);
72	} else if (dma_unmap_len(tx_buf, len)) {
73	dma_unmap_page(tx_q->dev,
74	dma_unmap_addr(tx_buf, dma),
75	dma_unmap_len(tx_buf, len),
76	DMA_TO_DEVICE);
77	}
78
79	tx_buf->next_to_watch = NULL;
80	tx_buf->skb = NULL;
81	tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG;
82	dma_unmap_len_set(tx_buf, len, 0);
83	}
84
85	/**
86	* idpf_tx_buf_rel_all - Free any empty Tx buffers
87	* @txq: queue to be cleaned
88	*/
89	static void idpf_tx_buf_rel_all(struct idpf_queue *txq)
90	{
91	u16 i;
92
93	/* Buffers already cleared, nothing to do */
94	if (!txq->tx_buf)
95	return;
96
97	/* Free all the Tx buffer sk_buffs */
98	for (i = 0; i < txq->desc_count; i++)
99	idpf_tx_buf_rel(tx_q: txq, tx_buf: &txq->tx_buf[i]);
100
101	kfree(objp: txq->tx_buf);
102	txq->tx_buf = NULL;
103
104	if (!txq->buf_stack.bufs)
105	return;
106
107	for (i = 0; i < txq->buf_stack.size; i++)
108	kfree(objp: txq->buf_stack.bufs[i]);
109
110	kfree(objp: txq->buf_stack.bufs);
111	txq->buf_stack.bufs = NULL;
112	}
113
114	/**
115	* idpf_tx_desc_rel - Free Tx resources per queue
116	* @txq: Tx descriptor ring for a specific queue
117	* @bufq: buffer q or completion q
118	*
119	* Free all transmit software resources
120	*/
121	static void idpf_tx_desc_rel(struct idpf_queue *txq, bool bufq)
122	{
123	if (bufq)
124	idpf_tx_buf_rel_all(txq);
125
126	if (!txq->desc_ring)
127	return;
128
129	dmam_free_coherent(dev: txq->dev, size: txq->size, vaddr: txq->desc_ring, dma_handle: txq->dma);
130	txq->desc_ring = NULL;
131	txq->next_to_alloc = 0;
132	txq->next_to_use = 0;
133	txq->next_to_clean = 0;
134	}
135
136	/**
137	* idpf_tx_desc_rel_all - Free Tx Resources for All Queues
138	* @vport: virtual port structure
139	*
140	* Free all transmit software resources
141	*/
142	static void idpf_tx_desc_rel_all(struct idpf_vport *vport)
143	{
144	int i, j;
145
146	if (!vport->txq_grps)
147	return;
148
149	for (i = 0; i < vport->num_txq_grp; i++) {
150	struct idpf_txq_group *txq_grp = &vport->txq_grps[i];
151
152	for (j = 0; j < txq_grp->num_txq; j++)
153	idpf_tx_desc_rel(txq: txq_grp->txqs[j], bufq: true);
154
155	if (idpf_is_queue_model_split(q_model: vport->txq_model))
156	idpf_tx_desc_rel(txq: txq_grp->complq, bufq: false);
157	}
158	}
159
160	/**
161	* idpf_tx_buf_alloc_all - Allocate memory for all buffer resources
162	* @tx_q: queue for which the buffers are allocated
163	*
164	* Returns 0 on success, negative on failure
165	*/
166	static int idpf_tx_buf_alloc_all(struct idpf_queue *tx_q)
167	{
168	int buf_size;
169	int i;
170
171	/* Allocate book keeping buffers only. Buffers to be supplied to HW
172	* are allocated by kernel network stack and received as part of skb
173	*/
174	buf_size = sizeof(struct idpf_tx_buf) * tx_q->desc_count;
175	tx_q->tx_buf = kzalloc(size: buf_size, GFP_KERNEL);
176	if (!tx_q->tx_buf)
177	return -ENOMEM;
178
179	/* Initialize tx_bufs with invalid completion tags */
180	for (i = 0; i < tx_q->desc_count; i++)
181	tx_q->tx_buf[i].compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG;
182
183	/* Initialize tx buf stack for out-of-order completions if
184	* flow scheduling offload is enabled
185	*/
186	tx_q->buf_stack.bufs =
187	kcalloc(n: tx_q->desc_count, size: sizeof(struct idpf_tx_stash *),
188	GFP_KERNEL);
189	if (!tx_q->buf_stack.bufs)
190	return -ENOMEM;
191
192	tx_q->buf_stack.size = tx_q->desc_count;
193	tx_q->buf_stack.top = tx_q->desc_count;
194
195	for (i = 0; i < tx_q->desc_count; i++) {
196	tx_q->buf_stack.bufs[i] = kzalloc(size: sizeof(*tx_q->buf_stack.bufs[i]),
197	GFP_KERNEL);
198	if (!tx_q->buf_stack.bufs[i])
199	return -ENOMEM;
200	}
201
202	return 0;
203	}
204
205	/**
206	* idpf_tx_desc_alloc - Allocate the Tx descriptors
207	* @tx_q: the tx ring to set up
208	* @bufq: buffer or completion queue
209	*
210	* Returns 0 on success, negative on failure
211	*/
212	static int idpf_tx_desc_alloc(struct idpf_queue *tx_q, bool bufq)
213	{
214	struct device *dev = tx_q->dev;
215	u32 desc_sz;
216	int err;
217
218	if (bufq) {
219	err = idpf_tx_buf_alloc_all(tx_q);
220	if (err)
221	goto err_alloc;
222
223	desc_sz = sizeof(struct idpf_base_tx_desc);
224	} else {
225	desc_sz = sizeof(struct idpf_splitq_tx_compl_desc);
226	}
227
228	tx_q->size = tx_q->desc_count * desc_sz;
229
230	/* Allocate descriptors also round up to nearest 4K */
231	tx_q->size = ALIGN(tx_q->size, 4096);
232	tx_q->desc_ring = dmam_alloc_coherent(dev, size: tx_q->size, dma_handle: &tx_q->dma,
233	GFP_KERNEL);
234	if (!tx_q->desc_ring) {
235	dev_err(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
236	tx_q->size);
237	err = -ENOMEM;
238	goto err_alloc;
239	}
240
241	tx_q->next_to_alloc = 0;
242	tx_q->next_to_use = 0;
243	tx_q->next_to_clean = 0;
244	set_bit(nr: __IDPF_Q_GEN_CHK, addr: tx_q->flags);
245
246	return 0;
247
248	err_alloc:
249	idpf_tx_desc_rel(txq: tx_q, bufq);
250
251	return err;
252	}
253
254	/**
255	* idpf_tx_desc_alloc_all - allocate all queues Tx resources
256	* @vport: virtual port private structure
257	*
258	* Returns 0 on success, negative on failure
259	*/
260	static int idpf_tx_desc_alloc_all(struct idpf_vport *vport)
261	{
262	struct device *dev = &vport->adapter->pdev->dev;
263	int err = 0;
264	int i, j;
265
266	/* Setup buffer queues. In single queue model buffer queues and
267	* completion queues will be same
268	*/
269	for (i = 0; i < vport->num_txq_grp; i++) {
270	for (j = 0; j < vport->txq_grps[i].num_txq; j++) {
271	struct idpf_queue *txq = vport->txq_grps[i].txqs[j];
272	u8 gen_bits = 0;
273	u16 bufidx_mask;
274
275	err = idpf_tx_desc_alloc(tx_q: txq, bufq: true);
276	if (err) {
277	dev_err(dev, "Allocation for Tx Queue %u failed\n",
278	i);
279	goto err_out;
280	}
281
282	if (!idpf_is_queue_model_split(q_model: vport->txq_model))
283	continue;
284
285	txq->compl_tag_cur_gen = 0;
286
287	/* Determine the number of bits in the bufid
288	* mask and add one to get the start of the
289	* generation bits
290	*/
291	bufidx_mask = txq->desc_count - 1;
292	while (bufidx_mask >> 1) {
293	txq->compl_tag_gen_s++;
294	bufidx_mask = bufidx_mask >> 1;
295	}
296	txq->compl_tag_gen_s++;
297
298	gen_bits = IDPF_TX_SPLITQ_COMPL_TAG_WIDTH -
299	txq->compl_tag_gen_s;
300	txq->compl_tag_gen_max = GETMAXVAL(gen_bits);
301
302	/* Set bufid mask based on location of first
303	* gen bit; it cannot simply be the descriptor
304	* ring size-1 since we can have size values
305	* where not all of those bits are set.
306	*/
307	txq->compl_tag_bufid_m =
308	GETMAXVAL(txq->compl_tag_gen_s);
309	}
310
311	if (!idpf_is_queue_model_split(q_model: vport->txq_model))
312	continue;
313
314	/* Setup completion queues */
315	err = idpf_tx_desc_alloc(tx_q: vport->txq_grps[i].complq, bufq: false);
316	if (err) {
317	dev_err(dev, "Allocation for Tx Completion Queue %u failed\n",
318	i);
319	goto err_out;
320	}
321	}
322
323	err_out:
324	if (err)
325	idpf_tx_desc_rel_all(vport);
326
327	return err;
328	}
329
330	/**
331	* idpf_rx_page_rel - Release an rx buffer page
332	* @rxq: the queue that owns the buffer
333	* @rx_buf: the buffer to free
334	*/
335	static void idpf_rx_page_rel(struct idpf_queue *rxq, struct idpf_rx_buf *rx_buf)
336	{
337	if (unlikely(!rx_buf->page))
338	return;
339
340	page_pool_put_full_page(pool: rxq->pp, page: rx_buf->page, allow_direct: false);
341
342	rx_buf->page = NULL;
343	rx_buf->page_offset = 0;
344	}
345
346	/**
347	* idpf_rx_hdr_buf_rel_all - Release header buffer memory
348	* @rxq: queue to use
349	*/
350	static void idpf_rx_hdr_buf_rel_all(struct idpf_queue *rxq)
351	{
352	struct idpf_adapter *adapter = rxq->vport->adapter;
353
354	dma_free_coherent(dev: &adapter->pdev->dev,
355	size: rxq->desc_count * IDPF_HDR_BUF_SIZE,
356	cpu_addr: rxq->rx_buf.hdr_buf_va,
357	dma_handle: rxq->rx_buf.hdr_buf_pa);
358	rxq->rx_buf.hdr_buf_va = NULL;
359	}
360
361	/**
362	* idpf_rx_buf_rel_all - Free all Rx buffer resources for a queue
363	* @rxq: queue to be cleaned
364	*/
365	static void idpf_rx_buf_rel_all(struct idpf_queue *rxq)
366	{
367	u16 i;
368
369	/* queue already cleared, nothing to do */
370	if (!rxq->rx_buf.buf)
371	return;
372
373	/* Free all the bufs allocated and given to hw on Rx queue */
374	for (i = 0; i < rxq->desc_count; i++)
375	idpf_rx_page_rel(rxq, rx_buf: &rxq->rx_buf.buf[i]);
376
377	if (rxq->rx_hsplit_en)
378	idpf_rx_hdr_buf_rel_all(rxq);
379
380	page_pool_destroy(pool: rxq->pp);
381	rxq->pp = NULL;
382
383	kfree(objp: rxq->rx_buf.buf);
384	rxq->rx_buf.buf = NULL;
385	}
386
387	/**
388	* idpf_rx_desc_rel - Free a specific Rx q resources
389	* @rxq: queue to clean the resources from
390	* @bufq: buffer q or completion q
391	* @q_model: single or split q model
392	*
393	* Free a specific rx queue resources
394	*/
395	static void idpf_rx_desc_rel(struct idpf_queue *rxq, bool bufq, s32 q_model)
396	{
397	if (!rxq)
398	return;
399
400	if (rxq->skb) {
401	dev_kfree_skb_any(skb: rxq->skb);
402	rxq->skb = NULL;
403	}
404
405	if (bufq || !idpf_is_queue_model_split(q_model))
406	idpf_rx_buf_rel_all(rxq);
407
408	rxq->next_to_alloc = 0;
409	rxq->next_to_clean = 0;
410	rxq->next_to_use = 0;
411	if (!rxq->desc_ring)
412	return;
413
414	dmam_free_coherent(dev: rxq->dev, size: rxq->size, vaddr: rxq->desc_ring, dma_handle: rxq->dma);
415	rxq->desc_ring = NULL;
416	}
417
418	/**
419	* idpf_rx_desc_rel_all - Free Rx Resources for All Queues
420	* @vport: virtual port structure
421	*
422	* Free all rx queues resources
423	*/
424	static void idpf_rx_desc_rel_all(struct idpf_vport *vport)
425	{
426	struct idpf_rxq_group *rx_qgrp;
427	u16 num_rxq;
428	int i, j;
429
430	if (!vport->rxq_grps)
431	return;
432
433	for (i = 0; i < vport->num_rxq_grp; i++) {
434	rx_qgrp = &vport->rxq_grps[i];
435
436	if (!idpf_is_queue_model_split(q_model: vport->rxq_model)) {
437	for (j = 0; j < rx_qgrp->singleq.num_rxq; j++)
438	idpf_rx_desc_rel(rxq: rx_qgrp->singleq.rxqs[j],
439	bufq: false, q_model: vport->rxq_model);
440	continue;
441	}
442
443	num_rxq = rx_qgrp->splitq.num_rxq_sets;
444	for (j = 0; j < num_rxq; j++)
445	idpf_rx_desc_rel(rxq: &rx_qgrp->splitq.rxq_sets[j]->rxq,
446	bufq: false, q_model: vport->rxq_model);
447
448	if (!rx_qgrp->splitq.bufq_sets)
449	continue;
450
451	for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
452	struct idpf_bufq_set *bufq_set =
453	&rx_qgrp->splitq.bufq_sets[j];
454
455	idpf_rx_desc_rel(rxq: &bufq_set->bufq, bufq: true,
456	q_model: vport->rxq_model);
457	}
458	}
459	}
460
461	/**
462	* idpf_rx_buf_hw_update - Store the new tail and head values
463	* @rxq: queue to bump
464	* @val: new head index
465	*/
466	void idpf_rx_buf_hw_update(struct idpf_queue *rxq, u32 val)
467	{
468	rxq->next_to_use = val;
469
470	if (unlikely(!rxq->tail))
471	return;
472
473	/* writel has an implicit memory barrier */
474	writel(val, addr: rxq->tail);
475	}
476
477	/**
478	* idpf_rx_hdr_buf_alloc_all - Allocate memory for header buffers
479	* @rxq: ring to use
480	*
481	* Returns 0 on success, negative on failure.
482	*/
483	static int idpf_rx_hdr_buf_alloc_all(struct idpf_queue *rxq)
484	{
485	struct idpf_adapter *adapter = rxq->vport->adapter;
486
487	rxq->rx_buf.hdr_buf_va =
488	dma_alloc_coherent(dev: &adapter->pdev->dev,
489	IDPF_HDR_BUF_SIZE * rxq->desc_count,
490	dma_handle: &rxq->rx_buf.hdr_buf_pa,
491	GFP_KERNEL);
492	if (!rxq->rx_buf.hdr_buf_va)
493	return -ENOMEM;
494
495	return 0;
496	}
497
498	/**
499	* idpf_rx_post_buf_refill - Post buffer id to refill queue
500	* @refillq: refill queue to post to
501	* @buf_id: buffer id to post
502	*/
503	static void idpf_rx_post_buf_refill(struct idpf_sw_queue *refillq, u16 buf_id)
504	{
505	u16 nta = refillq->next_to_alloc;
506
507	/* store the buffer ID and the SW maintained GEN bit to the refillq */
508	refillq->ring[nta] =
509	FIELD_PREP(IDPF_RX_BI_BUFID_M, buf_id) |
510	FIELD_PREP(IDPF_RX_BI_GEN_M,
511	test_bit(__IDPF_Q_GEN_CHK, refillq->flags));
512
513	if (unlikely(++nta == refillq->desc_count)) {
514	nta = 0;
515	change_bit(nr: __IDPF_Q_GEN_CHK, addr: refillq->flags);
516	}
517	refillq->next_to_alloc = nta;
518	}
519
520	/**
521	* idpf_rx_post_buf_desc - Post buffer to bufq descriptor ring
522	* @bufq: buffer queue to post to
523	* @buf_id: buffer id to post
524	*
525	* Returns false if buffer could not be allocated, true otherwise.
526	*/
527	static bool idpf_rx_post_buf_desc(struct idpf_queue *bufq, u16 buf_id)
528	{
529	struct virtchnl2_splitq_rx_buf_desc *splitq_rx_desc = NULL;
530	u16 nta = bufq->next_to_alloc;
531	struct idpf_rx_buf *buf;
532	dma_addr_t addr;
533
534	splitq_rx_desc = IDPF_SPLITQ_RX_BUF_DESC(bufq, nta);
535	buf = &bufq->rx_buf.buf[buf_id];
536
537	if (bufq->rx_hsplit_en) {
538	splitq_rx_desc->hdr_addr =
539	cpu_to_le64(bufq->rx_buf.hdr_buf_pa +
540	(u32)buf_id * IDPF_HDR_BUF_SIZE);
541	}
542
543	addr = idpf_alloc_page(pool: bufq->pp, buf, buf_size: bufq->rx_buf_size);
544	if (unlikely(addr == DMA_MAPPING_ERROR))
545	return false;
546
547	splitq_rx_desc->pkt_addr = cpu_to_le64(addr);
548	splitq_rx_desc->qword0.buf_id = cpu_to_le16(buf_id);
549
550	nta++;
551	if (unlikely(nta == bufq->desc_count))
552	nta = 0;
553	bufq->next_to_alloc = nta;
554
555	return true;
556	}
557
558	/**
559	* idpf_rx_post_init_bufs - Post initial buffers to bufq
560	* @bufq: buffer queue to post working set to
561	* @working_set: number of buffers to put in working set
562	*
563	* Returns true if @working_set bufs were posted successfully, false otherwise.
564	*/
565	static bool idpf_rx_post_init_bufs(struct idpf_queue *bufq, u16 working_set)
566	{
567	int i;
568
569	for (i = 0; i < working_set; i++) {
570	if (!idpf_rx_post_buf_desc(bufq, buf_id: i))
571	return false;
572	}
573
574	idpf_rx_buf_hw_update(rxq: bufq,
575	val: bufq->next_to_alloc & ~(bufq->rx_buf_stride - 1));
576
577	return true;
578	}
579
580	/**
581	* idpf_rx_create_page_pool - Create a page pool
582	* @rxbufq: RX queue to create page pool for
583	*
584	* Returns &page_pool on success, casted -errno on failure
585	*/
586	static struct page_pool *idpf_rx_create_page_pool(struct idpf_queue *rxbufq)
587	{
588	struct page_pool_params pp = {
589	.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
590	.order = 0,
591	.pool_size = rxbufq->desc_count,
592	.nid = NUMA_NO_NODE,
593	.dev = rxbufq->vport->netdev->dev.parent,
594	.max_len = PAGE_SIZE,
595	.dma_dir = DMA_FROM_DEVICE,
596	.offset = 0,
597	};
598
599	return page_pool_create(params: &pp);
600	}
601
602	/**
603	* idpf_rx_buf_alloc_all - Allocate memory for all buffer resources
604	* @rxbufq: queue for which the buffers are allocated; equivalent to
605	* rxq when operating in singleq mode
606	*
607	* Returns 0 on success, negative on failure
608	*/
609	static int idpf_rx_buf_alloc_all(struct idpf_queue *rxbufq)
610	{
611	int err = 0;
612
613	/* Allocate book keeping buffers */
614	rxbufq->rx_buf.buf = kcalloc(n: rxbufq->desc_count,
615	size: sizeof(struct idpf_rx_buf), GFP_KERNEL);
616	if (!rxbufq->rx_buf.buf) {
617	err = -ENOMEM;
618	goto rx_buf_alloc_all_out;
619	}
620
621	if (rxbufq->rx_hsplit_en) {
622	err = idpf_rx_hdr_buf_alloc_all(rxq: rxbufq);
623	if (err)
624	goto rx_buf_alloc_all_out;
625	}
626
627	/* Allocate buffers to be given to HW. */
628	if (idpf_is_queue_model_split(q_model: rxbufq->vport->rxq_model)) {
629	int working_set = IDPF_RX_BUFQ_WORKING_SET(rxbufq);
630
631	if (!idpf_rx_post_init_bufs(bufq: rxbufq, working_set))
632	err = -ENOMEM;
633	} else {
634	if (idpf_rx_singleq_buf_hw_alloc_all(rxq: rxbufq,
635	cleaned_count: rxbufq->desc_count - 1))
636	err = -ENOMEM;
637	}
638
639	rx_buf_alloc_all_out:
640	if (err)
641	idpf_rx_buf_rel_all(rxq: rxbufq);
642
643	return err;
644	}
645
646	/**
647	* idpf_rx_bufs_init - Initialize page pool, allocate rx bufs, and post to HW
648	* @rxbufq: RX queue to create page pool for
649	*
650	* Returns 0 on success, negative on failure
651	*/
652	static int idpf_rx_bufs_init(struct idpf_queue *rxbufq)
653	{
654	struct page_pool *pool;
655
656	pool = idpf_rx_create_page_pool(rxbufq);
657	if (IS_ERR(ptr: pool))
658	return PTR_ERR(ptr: pool);
659
660	rxbufq->pp = pool;
661
662	return idpf_rx_buf_alloc_all(rxbufq);
663	}
664
665	/**
666	* idpf_rx_bufs_init_all - Initialize all RX bufs
667	* @vport: virtual port struct
668	*
669	* Returns 0 on success, negative on failure
670	*/
671	int idpf_rx_bufs_init_all(struct idpf_vport *vport)
672	{
673	struct idpf_rxq_group *rx_qgrp;
674	struct idpf_queue *q;
675	int i, j, err;
676
677	for (i = 0; i < vport->num_rxq_grp; i++) {
678	rx_qgrp = &vport->rxq_grps[i];
679
680	/* Allocate bufs for the rxq itself in singleq */
681	if (!idpf_is_queue_model_split(q_model: vport->rxq_model)) {
682	int num_rxq = rx_qgrp->singleq.num_rxq;
683
684	for (j = 0; j < num_rxq; j++) {
685	q = rx_qgrp->singleq.rxqs[j];
686	err = idpf_rx_bufs_init(rxbufq: q);
687	if (err)
688	return err;
689	}
690
691	continue;
692	}
693
694	/* Otherwise, allocate bufs for the buffer queues */
695	for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
696	q = &rx_qgrp->splitq.bufq_sets[j].bufq;
697	err = idpf_rx_bufs_init(rxbufq: q);
698	if (err)
699	return err;
700	}
701	}
702
703	return 0;
704	}
705
706	/**
707	* idpf_rx_desc_alloc - Allocate queue Rx resources
708	* @rxq: Rx queue for which the resources are setup
709	* @bufq: buffer or completion queue
710	* @q_model: single or split queue model
711	*
712	* Returns 0 on success, negative on failure
713	*/
714	static int idpf_rx_desc_alloc(struct idpf_queue *rxq, bool bufq, s32 q_model)
715	{
716	struct device *dev = rxq->dev;
717
718	if (bufq)
719	rxq->size = rxq->desc_count *
720	sizeof(struct virtchnl2_splitq_rx_buf_desc);
721	else
722	rxq->size = rxq->desc_count *
723	sizeof(union virtchnl2_rx_desc);
724
725	/* Allocate descriptors and also round up to nearest 4K */
726	rxq->size = ALIGN(rxq->size, 4096);
727	rxq->desc_ring = dmam_alloc_coherent(dev, size: rxq->size,
728	dma_handle: &rxq->dma, GFP_KERNEL);
729	if (!rxq->desc_ring) {
730	dev_err(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
731	rxq->size);
732	return -ENOMEM;
733	}
734
735	rxq->next_to_alloc = 0;
736	rxq->next_to_clean = 0;
737	rxq->next_to_use = 0;
738	set_bit(nr: __IDPF_Q_GEN_CHK, addr: rxq->flags);
739
740	return 0;
741	}
742
743	/**
744	* idpf_rx_desc_alloc_all - allocate all RX queues resources
745	* @vport: virtual port structure
746	*
747	* Returns 0 on success, negative on failure
748	*/
749	static int idpf_rx_desc_alloc_all(struct idpf_vport *vport)
750	{
751	struct device *dev = &vport->adapter->pdev->dev;
752	struct idpf_rxq_group *rx_qgrp;
753	struct idpf_queue *q;
754	int i, j, err;
755	u16 num_rxq;
756
757	for (i = 0; i < vport->num_rxq_grp; i++) {
758	rx_qgrp = &vport->rxq_grps[i];
759	if (idpf_is_queue_model_split(q_model: vport->rxq_model))
760	num_rxq = rx_qgrp->splitq.num_rxq_sets;
761	else
762	num_rxq = rx_qgrp->singleq.num_rxq;
763
764	for (j = 0; j < num_rxq; j++) {
765	if (idpf_is_queue_model_split(q_model: vport->rxq_model))
766	q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
767	else
768	q = rx_qgrp->singleq.rxqs[j];
769	err = idpf_rx_desc_alloc(rxq: q, bufq: false, q_model: vport->rxq_model);
770	if (err) {
771	dev_err(dev, "Memory allocation for Rx Queue %u failed\n",
772	i);
773	goto err_out;
774	}
775	}
776
777	if (!idpf_is_queue_model_split(q_model: vport->rxq_model))
778	continue;
779
780	for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
781	q = &rx_qgrp->splitq.bufq_sets[j].bufq;
782	err = idpf_rx_desc_alloc(rxq: q, bufq: true, q_model: vport->rxq_model);
783	if (err) {
784	dev_err(dev, "Memory allocation for Rx Buffer Queue %u failed\n",
785	i);
786	goto err_out;
787	}
788	}
789	}
790
791	return 0;
792
793	err_out:
794	idpf_rx_desc_rel_all(vport);
795
796	return err;
797	}
798
799	/**
800	* idpf_txq_group_rel - Release all resources for txq groups
801	* @vport: vport to release txq groups on
802	*/
803	static void idpf_txq_group_rel(struct idpf_vport *vport)
804	{
805	int i, j;
806
807	if (!vport->txq_grps)
808	return;
809
810	for (i = 0; i < vport->num_txq_grp; i++) {
811	struct idpf_txq_group *txq_grp = &vport->txq_grps[i];
812
813	for (j = 0; j < txq_grp->num_txq; j++) {
814	kfree(objp: txq_grp->txqs[j]);
815	txq_grp->txqs[j] = NULL;
816	}
817	kfree(objp: txq_grp->complq);
818	txq_grp->complq = NULL;
819	}
820	kfree(objp: vport->txq_grps);
821	vport->txq_grps = NULL;
822	}
823
824	/**
825	* idpf_rxq_sw_queue_rel - Release software queue resources
826	* @rx_qgrp: rx queue group with software queues
827	*/
828	static void idpf_rxq_sw_queue_rel(struct idpf_rxq_group *rx_qgrp)
829	{
830	int i, j;
831
832	for (i = 0; i < rx_qgrp->vport->num_bufqs_per_qgrp; i++) {
833	struct idpf_bufq_set *bufq_set = &rx_qgrp->splitq.bufq_sets[i];
834
835	for (j = 0; j < bufq_set->num_refillqs; j++) {
836	kfree(objp: bufq_set->refillqs[j].ring);
837	bufq_set->refillqs[j].ring = NULL;
838	}
839	kfree(objp: bufq_set->refillqs);
840	bufq_set->refillqs = NULL;
841	}
842	}
843
844	/**
845	* idpf_rxq_group_rel - Release all resources for rxq groups
846	* @vport: vport to release rxq groups on
847	*/
848	static void idpf_rxq_group_rel(struct idpf_vport *vport)
849	{
850	int i;
851
852	if (!vport->rxq_grps)
853	return;
854
855	for (i = 0; i < vport->num_rxq_grp; i++) {
856	struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i];
857	u16 num_rxq;
858	int j;
859
860	if (idpf_is_queue_model_split(q_model: vport->rxq_model)) {
861	num_rxq = rx_qgrp->splitq.num_rxq_sets;
862	for (j = 0; j < num_rxq; j++) {
863	kfree(objp: rx_qgrp->splitq.rxq_sets[j]);
864	rx_qgrp->splitq.rxq_sets[j] = NULL;
865	}
866
867	idpf_rxq_sw_queue_rel(rx_qgrp);
868	kfree(objp: rx_qgrp->splitq.bufq_sets);
869	rx_qgrp->splitq.bufq_sets = NULL;
870	} else {
871	num_rxq = rx_qgrp->singleq.num_rxq;
872	for (j = 0; j < num_rxq; j++) {
873	kfree(objp: rx_qgrp->singleq.rxqs[j]);
874	rx_qgrp->singleq.rxqs[j] = NULL;
875	}
876	}
877	}
878	kfree(objp: vport->rxq_grps);
879	vport->rxq_grps = NULL;
880	}
881
882	/**
883	* idpf_vport_queue_grp_rel_all - Release all queue groups
884	* @vport: vport to release queue groups for
885	*/
886	static void idpf_vport_queue_grp_rel_all(struct idpf_vport *vport)
887	{
888	idpf_txq_group_rel(vport);
889	idpf_rxq_group_rel(vport);
890	}
891
892	/**
893	* idpf_vport_queues_rel - Free memory for all queues
894	* @vport: virtual port
895	*
896	* Free the memory allocated for queues associated to a vport
897	*/
898	void idpf_vport_queues_rel(struct idpf_vport *vport)
899	{
900	idpf_tx_desc_rel_all(vport);
901	idpf_rx_desc_rel_all(vport);
902	idpf_vport_queue_grp_rel_all(vport);
903
904	kfree(objp: vport->txqs);
905	vport->txqs = NULL;
906	}
907
908	/**
909	* idpf_vport_init_fast_path_txqs - Initialize fast path txq array
910	* @vport: vport to init txqs on
911	*
912	* We get a queue index from skb->queue_mapping and we need a fast way to
913	* dereference the queue from queue groups. This allows us to quickly pull a
914	* txq based on a queue index.
915	*
916	* Returns 0 on success, negative on failure
917	*/
918	static int idpf_vport_init_fast_path_txqs(struct idpf_vport *vport)
919	{
920	int i, j, k = 0;
921
922	vport->txqs = kcalloc(n: vport->num_txq, size: sizeof(struct idpf_queue *),
923	GFP_KERNEL);
924
925	if (!vport->txqs)
926	return -ENOMEM;
927
928	for (i = 0; i < vport->num_txq_grp; i++) {
929	struct idpf_txq_group *tx_grp = &vport->txq_grps[i];
930
931	for (j = 0; j < tx_grp->num_txq; j++, k++) {
932	vport->txqs[k] = tx_grp->txqs[j];
933	vport->txqs[k]->idx = k;
934	}
935	}
936
937	return 0;
938	}
939
940	/**
941	* idpf_vport_init_num_qs - Initialize number of queues
942	* @vport: vport to initialize queues
943	* @vport_msg: data to be filled into vport
944	*/
945	void idpf_vport_init_num_qs(struct idpf_vport *vport,
946	struct virtchnl2_create_vport *vport_msg)
947	{
948	struct idpf_vport_user_config_data *config_data;
949	u16 idx = vport->idx;
950
951	config_data = &vport->adapter->vport_config[idx]->user_config;
952	vport->num_txq = le16_to_cpu(vport_msg->num_tx_q);
953	vport->num_rxq = le16_to_cpu(vport_msg->num_rx_q);
954	/* number of txqs and rxqs in config data will be zeros only in the
955	* driver load path and we dont update them there after
956	*/
957	if (!config_data->num_req_tx_qs && !config_data->num_req_rx_qs) {
958	config_data->num_req_tx_qs = le16_to_cpu(vport_msg->num_tx_q);
959	config_data->num_req_rx_qs = le16_to_cpu(vport_msg->num_rx_q);
960	}
961
962	if (idpf_is_queue_model_split(q_model: vport->txq_model))
963	vport->num_complq = le16_to_cpu(vport_msg->num_tx_complq);
964	if (idpf_is_queue_model_split(q_model: vport->rxq_model))
965	vport->num_bufq = le16_to_cpu(vport_msg->num_rx_bufq);
966
967	/* Adjust number of buffer queues per Rx queue group. */
968	if (!idpf_is_queue_model_split(q_model: vport->rxq_model)) {
969	vport->num_bufqs_per_qgrp = 0;
970	vport->bufq_size[0] = IDPF_RX_BUF_2048;
971
972	return;
973	}
974
975	vport->num_bufqs_per_qgrp = IDPF_MAX_BUFQS_PER_RXQ_GRP;
976	/* Bufq[0] default buffer size is 4K
977	* Bufq[1] default buffer size is 2K
978	*/
979	vport->bufq_size[0] = IDPF_RX_BUF_4096;
980	vport->bufq_size[1] = IDPF_RX_BUF_2048;
981	}
982
983	/**
984	* idpf_vport_calc_num_q_desc - Calculate number of queue groups
985	* @vport: vport to calculate q groups for
986	*/
987	void idpf_vport_calc_num_q_desc(struct idpf_vport *vport)
988	{
989	struct idpf_vport_user_config_data *config_data;
990	int num_bufqs = vport->num_bufqs_per_qgrp;
991	u32 num_req_txq_desc, num_req_rxq_desc;
992	u16 idx = vport->idx;
993	int i;
994
995	config_data = &vport->adapter->vport_config[idx]->user_config;
996	num_req_txq_desc = config_data->num_req_txq_desc;
997	num_req_rxq_desc = config_data->num_req_rxq_desc;
998
999	vport->complq_desc_count = 0;
1000	if (num_req_txq_desc) {
1001	vport->txq_desc_count = num_req_txq_desc;
1002	if (idpf_is_queue_model_split(q_model: vport->txq_model)) {
1003	vport->complq_desc_count = num_req_txq_desc;
1004	if (vport->complq_desc_count < IDPF_MIN_TXQ_COMPLQ_DESC)
1005	vport->complq_desc_count =
1006	IDPF_MIN_TXQ_COMPLQ_DESC;
1007	}
1008	} else {
1009	vport->txq_desc_count = IDPF_DFLT_TX_Q_DESC_COUNT;
1010	if (idpf_is_queue_model_split(q_model: vport->txq_model))
1011	vport->complq_desc_count =
1012	IDPF_DFLT_TX_COMPLQ_DESC_COUNT;
1013	}
1014
1015	if (num_req_rxq_desc)
1016	vport->rxq_desc_count = num_req_rxq_desc;
1017	else
1018	vport->rxq_desc_count = IDPF_DFLT_RX_Q_DESC_COUNT;
1019
1020	for (i = 0; i < num_bufqs; i++) {
1021	if (!vport->bufq_desc_count[i])
1022	vport->bufq_desc_count[i] =
1023	IDPF_RX_BUFQ_DESC_COUNT(vport->rxq_desc_count,
1024	num_bufqs);
1025	}
1026	}
1027
1028	/**
1029	* idpf_vport_calc_total_qs - Calculate total number of queues
1030	* @adapter: private data struct
1031	* @vport_idx: vport idx to retrieve vport pointer
1032	* @vport_msg: message to fill with data
1033	* @max_q: vport max queue info
1034	*
1035	* Return 0 on success, error value on failure.
1036	*/
1037	int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_idx,
1038	struct virtchnl2_create_vport *vport_msg,
1039	struct idpf_vport_max_q *max_q)
1040	{
1041	int dflt_splitq_txq_grps = 0, dflt_singleq_txqs = 0;
1042	int dflt_splitq_rxq_grps = 0, dflt_singleq_rxqs = 0;
1043	u16 num_req_tx_qs = 0, num_req_rx_qs = 0;
1044	struct idpf_vport_config *vport_config;
1045	u16 num_txq_grps, num_rxq_grps;
1046	u32 num_qs;
1047
1048	vport_config = adapter->vport_config[vport_idx];
1049	if (vport_config) {
1050	num_req_tx_qs = vport_config->user_config.num_req_tx_qs;
1051	num_req_rx_qs = vport_config->user_config.num_req_rx_qs;
1052	} else {
1053	int num_cpus;
1054
1055	/* Restrict num of queues to cpus online as a default
1056	* configuration to give best performance. User can always
1057	* override to a max number of queues via ethtool.
1058	*/
1059	num_cpus = num_online_cpus();
1060
1061	dflt_splitq_txq_grps = min_t(int, max_q->max_txq, num_cpus);
1062	dflt_singleq_txqs = min_t(int, max_q->max_txq, num_cpus);
1063	dflt_splitq_rxq_grps = min_t(int, max_q->max_rxq, num_cpus);
1064	dflt_singleq_rxqs = min_t(int, max_q->max_rxq, num_cpus);
1065	}
1066
1067	if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->txq_model))) {
1068	num_txq_grps = num_req_tx_qs ? num_req_tx_qs : dflt_splitq_txq_grps;
1069	vport_msg->num_tx_complq = cpu_to_le16(num_txq_grps *
1070	IDPF_COMPLQ_PER_GROUP);
1071	vport_msg->num_tx_q = cpu_to_le16(num_txq_grps *
1072	IDPF_DFLT_SPLITQ_TXQ_PER_GROUP);
1073	} else {
1074	num_txq_grps = IDPF_DFLT_SINGLEQ_TX_Q_GROUPS;
1075	num_qs = num_txq_grps * (num_req_tx_qs ? num_req_tx_qs :
1076	dflt_singleq_txqs);
1077	vport_msg->num_tx_q = cpu_to_le16(num_qs);
1078	vport_msg->num_tx_complq = 0;
1079	}
1080	if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->rxq_model))) {
1081	num_rxq_grps = num_req_rx_qs ? num_req_rx_qs : dflt_splitq_rxq_grps;
1082	vport_msg->num_rx_bufq = cpu_to_le16(num_rxq_grps *
1083	IDPF_MAX_BUFQS_PER_RXQ_GRP);
1084	vport_msg->num_rx_q = cpu_to_le16(num_rxq_grps *
1085	IDPF_DFLT_SPLITQ_RXQ_PER_GROUP);
1086	} else {
1087	num_rxq_grps = IDPF_DFLT_SINGLEQ_RX_Q_GROUPS;
1088	num_qs = num_rxq_grps * (num_req_rx_qs ? num_req_rx_qs :
1089	dflt_singleq_rxqs);
1090	vport_msg->num_rx_q = cpu_to_le16(num_qs);
1091	vport_msg->num_rx_bufq = 0;
1092	}
1093
1094	return 0;
1095	}
1096
1097	/**
1098	* idpf_vport_calc_num_q_groups - Calculate number of queue groups
1099	* @vport: vport to calculate q groups for
1100	*/
1101	void idpf_vport_calc_num_q_groups(struct idpf_vport *vport)
1102	{
1103	if (idpf_is_queue_model_split(q_model: vport->txq_model))
1104	vport->num_txq_grp = vport->num_txq;
1105	else
1106	vport->num_txq_grp = IDPF_DFLT_SINGLEQ_TX_Q_GROUPS;
1107
1108	if (idpf_is_queue_model_split(q_model: vport->rxq_model))
1109	vport->num_rxq_grp = vport->num_rxq;
1110	else
1111	vport->num_rxq_grp = IDPF_DFLT_SINGLEQ_RX_Q_GROUPS;
1112	}
1113
1114	/**
1115	* idpf_vport_calc_numq_per_grp - Calculate number of queues per group
1116	* @vport: vport to calculate queues for
1117	* @num_txq: return parameter for number of TX queues
1118	* @num_rxq: return parameter for number of RX queues
1119	*/
1120	static void idpf_vport_calc_numq_per_grp(struct idpf_vport *vport,
1121	u16 *num_txq, u16 *num_rxq)
1122	{
1123	if (idpf_is_queue_model_split(q_model: vport->txq_model))
1124	*num_txq = IDPF_DFLT_SPLITQ_TXQ_PER_GROUP;
1125	else
1126	*num_txq = vport->num_txq;
1127
1128	if (idpf_is_queue_model_split(q_model: vport->rxq_model))
1129	*num_rxq = IDPF_DFLT_SPLITQ_RXQ_PER_GROUP;
1130	else
1131	*num_rxq = vport->num_rxq;
1132	}
1133
1134	/**
1135	* idpf_rxq_set_descids - set the descids supported by this queue
1136	* @vport: virtual port data structure
1137	* @q: rx queue for which descids are set
1138	*
1139	*/
1140	static void idpf_rxq_set_descids(struct idpf_vport *vport, struct idpf_queue *q)
1141	{
1142	if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) {
1143	q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M;
1144	} else {
1145	if (vport->base_rxd)
1146	q->rxdids = VIRTCHNL2_RXDID_1_32B_BASE_M;
1147	else
1148	q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M;
1149	}
1150	}
1151
1152	/**
1153	* idpf_txq_group_alloc - Allocate all txq group resources
1154	* @vport: vport to allocate txq groups for
1155	* @num_txq: number of txqs to allocate for each group
1156	*
1157	* Returns 0 on success, negative on failure
1158	*/
1159	static int idpf_txq_group_alloc(struct idpf_vport *vport, u16 num_txq)
1160	{
1161	bool flow_sch_en;
1162	int err, i;
1163
1164	vport->txq_grps = kcalloc(n: vport->num_txq_grp,
1165	size: sizeof(*vport->txq_grps), GFP_KERNEL);
1166	if (!vport->txq_grps)
1167	return -ENOMEM;
1168
1169	flow_sch_en = !idpf_is_cap_ena(vport->adapter, IDPF_OTHER_CAPS,
1170	VIRTCHNL2_CAP_SPLITQ_QSCHED);
1171
1172	for (i = 0; i < vport->num_txq_grp; i++) {
1173	struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i];
1174	struct idpf_adapter *adapter = vport->adapter;
1175	int j;
1176
1177	tx_qgrp->vport = vport;
1178	tx_qgrp->num_txq = num_txq;
1179
1180	for (j = 0; j < tx_qgrp->num_txq; j++) {
1181	tx_qgrp->txqs[j] = kzalloc(size: sizeof(*tx_qgrp->txqs[j]),
1182	GFP_KERNEL);
1183	if (!tx_qgrp->txqs[j]) {
1184	err = -ENOMEM;
1185	goto err_alloc;
1186	}
1187	}
1188
1189	for (j = 0; j < tx_qgrp->num_txq; j++) {
1190	struct idpf_queue *q = tx_qgrp->txqs[j];
1191
1192	q->dev = &adapter->pdev->dev;
1193	q->desc_count = vport->txq_desc_count;
1194	q->tx_max_bufs = idpf_get_max_tx_bufs(adapter);
1195	q->tx_min_pkt_len = idpf_get_min_tx_pkt_len(adapter);
1196	q->vport = vport;
1197	q->txq_grp = tx_qgrp;
1198	hash_init(q->sched_buf_hash);
1199
1200	if (flow_sch_en)
1201	set_bit(nr: __IDPF_Q_FLOW_SCH_EN, addr: q->flags);
1202	}
1203
1204	if (!idpf_is_queue_model_split(q_model: vport->txq_model))
1205	continue;
1206
1207	tx_qgrp->complq = kcalloc(IDPF_COMPLQ_PER_GROUP,
1208	size: sizeof(*tx_qgrp->complq),
1209	GFP_KERNEL);
1210	if (!tx_qgrp->complq) {
1211	err = -ENOMEM;
1212	goto err_alloc;
1213	}
1214
1215	tx_qgrp->complq->dev = &adapter->pdev->dev;
1216	tx_qgrp->complq->desc_count = vport->complq_desc_count;
1217	tx_qgrp->complq->vport = vport;
1218	tx_qgrp->complq->txq_grp = tx_qgrp;
1219
1220	if (flow_sch_en)
1221	__set_bit(__IDPF_Q_FLOW_SCH_EN, tx_qgrp->complq->flags);
1222	}
1223
1224	return 0;
1225
1226	err_alloc:
1227	idpf_txq_group_rel(vport);
1228
1229	return err;
1230	}
1231
1232	/**
1233	* idpf_rxq_group_alloc - Allocate all rxq group resources
1234	* @vport: vport to allocate rxq groups for
1235	* @num_rxq: number of rxqs to allocate for each group
1236	*
1237	* Returns 0 on success, negative on failure
1238	*/
1239	static int idpf_rxq_group_alloc(struct idpf_vport *vport, u16 num_rxq)
1240	{
1241	struct idpf_adapter *adapter = vport->adapter;
1242	struct idpf_queue *q;
1243	int i, k, err = 0;
1244	bool hs;
1245
1246	vport->rxq_grps = kcalloc(n: vport->num_rxq_grp,
1247	size: sizeof(struct idpf_rxq_group), GFP_KERNEL);
1248	if (!vport->rxq_grps)
1249	return -ENOMEM;
1250
1251	hs = idpf_vport_get_hsplit(vport) == ETHTOOL_TCP_DATA_SPLIT_ENABLED;
1252
1253	for (i = 0; i < vport->num_rxq_grp; i++) {
1254	struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i];
1255	int j;
1256
1257	rx_qgrp->vport = vport;
1258	if (!idpf_is_queue_model_split(q_model: vport->rxq_model)) {
1259	rx_qgrp->singleq.num_rxq = num_rxq;
1260	for (j = 0; j < num_rxq; j++) {
1261	rx_qgrp->singleq.rxqs[j] =
1262	kzalloc(size: sizeof(*rx_qgrp->singleq.rxqs[j]),
1263	GFP_KERNEL);
1264	if (!rx_qgrp->singleq.rxqs[j]) {
1265	err = -ENOMEM;
1266	goto err_alloc;
1267	}
1268	}
1269	goto skip_splitq_rx_init;
1270	}
1271	rx_qgrp->splitq.num_rxq_sets = num_rxq;
1272
1273	for (j = 0; j < num_rxq; j++) {
1274	rx_qgrp->splitq.rxq_sets[j] =
1275	kzalloc(size: sizeof(struct idpf_rxq_set),
1276	GFP_KERNEL);
1277	if (!rx_qgrp->splitq.rxq_sets[j]) {
1278	err = -ENOMEM;
1279	goto err_alloc;
1280	}
1281	}
1282
1283	rx_qgrp->splitq.bufq_sets = kcalloc(n: vport->num_bufqs_per_qgrp,
1284	size: sizeof(struct idpf_bufq_set),
1285	GFP_KERNEL);
1286	if (!rx_qgrp->splitq.bufq_sets) {
1287	err = -ENOMEM;
1288	goto err_alloc;
1289	}
1290
1291	for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
1292	struct idpf_bufq_set *bufq_set =
1293	&rx_qgrp->splitq.bufq_sets[j];
1294	int swq_size = sizeof(struct idpf_sw_queue);
1295
1296	q = &rx_qgrp->splitq.bufq_sets[j].bufq;
1297	q->dev = &adapter->pdev->dev;
1298	q->desc_count = vport->bufq_desc_count[j];
1299	q->vport = vport;
1300	q->rxq_grp = rx_qgrp;
1301	q->idx = j;
1302	q->rx_buf_size = vport->bufq_size[j];
1303	q->rx_buffer_low_watermark = IDPF_LOW_WATERMARK;
1304	q->rx_buf_stride = IDPF_RX_BUF_STRIDE;
1305
1306	if (hs) {
1307	q->rx_hsplit_en = true;
1308	q->rx_hbuf_size = IDPF_HDR_BUF_SIZE;
1309	}
1310
1311	bufq_set->num_refillqs = num_rxq;
1312	bufq_set->refillqs = kcalloc(n: num_rxq, size: swq_size,
1313	GFP_KERNEL);
1314	if (!bufq_set->refillqs) {
1315	err = -ENOMEM;
1316	goto err_alloc;
1317	}
1318	for (k = 0; k < bufq_set->num_refillqs; k++) {
1319	struct idpf_sw_queue *refillq =
1320	&bufq_set->refillqs[k];
1321
1322	refillq->dev = &vport->adapter->pdev->dev;
1323	refillq->desc_count =
1324	vport->bufq_desc_count[j];
1325	set_bit(nr: __IDPF_Q_GEN_CHK, addr: refillq->flags);
1326	set_bit(nr: __IDPF_RFLQ_GEN_CHK, addr: refillq->flags);
1327	refillq->ring = kcalloc(n: refillq->desc_count,
1328	size: sizeof(u16),
1329	GFP_KERNEL);
1330	if (!refillq->ring) {
1331	err = -ENOMEM;
1332	goto err_alloc;
1333	}
1334	}
1335	}
1336
1337	skip_splitq_rx_init:
1338	for (j = 0; j < num_rxq; j++) {
1339	if (!idpf_is_queue_model_split(q_model: vport->rxq_model)) {
1340	q = rx_qgrp->singleq.rxqs[j];
1341	goto setup_rxq;
1342	}
1343	q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
1344	rx_qgrp->splitq.rxq_sets[j]->refillq0 =
1345	&rx_qgrp->splitq.bufq_sets[0].refillqs[j];
1346	if (vport->num_bufqs_per_qgrp > IDPF_SINGLE_BUFQ_PER_RXQ_GRP)
1347	rx_qgrp->splitq.rxq_sets[j]->refillq1 =
1348	&rx_qgrp->splitq.bufq_sets[1].refillqs[j];
1349
1350	if (hs) {
1351	q->rx_hsplit_en = true;
1352	q->rx_hbuf_size = IDPF_HDR_BUF_SIZE;
1353	}
1354
1355	setup_rxq:
1356	q->dev = &adapter->pdev->dev;
1357	q->desc_count = vport->rxq_desc_count;
1358	q->vport = vport;
1359	q->rxq_grp = rx_qgrp;
1360	q->idx = (i * num_rxq) + j;
1361	/* In splitq mode, RXQ buffer size should be
1362	* set to that of the first buffer queue
1363	* associated with this RXQ
1364	*/
1365	q->rx_buf_size = vport->bufq_size[0];
1366	q->rx_buffer_low_watermark = IDPF_LOW_WATERMARK;
1367	q->rx_max_pkt_size = vport->netdev->mtu +
1368	IDPF_PACKET_HDR_PAD;
1369	idpf_rxq_set_descids(vport, q);
1370	}
1371	}
1372
1373	err_alloc:
1374	if (err)
1375	idpf_rxq_group_rel(vport);
1376
1377	return err;
1378	}
1379
1380	/**
1381	* idpf_vport_queue_grp_alloc_all - Allocate all queue groups/resources
1382	* @vport: vport with qgrps to allocate
1383	*
1384	* Returns 0 on success, negative on failure
1385	*/
1386	static int idpf_vport_queue_grp_alloc_all(struct idpf_vport *vport)
1387	{
1388	u16 num_txq, num_rxq;
1389	int err;
1390
1391	idpf_vport_calc_numq_per_grp(vport, num_txq: &num_txq, num_rxq: &num_rxq);
1392
1393	err = idpf_txq_group_alloc(vport, num_txq);
1394	if (err)
1395	goto err_out;
1396
1397	err = idpf_rxq_group_alloc(vport, num_rxq);
1398	if (err)
1399	goto err_out;
1400
1401	return 0;
1402
1403	err_out:
1404	idpf_vport_queue_grp_rel_all(vport);
1405
1406	return err;
1407	}
1408
1409	/**
1410	* idpf_vport_queues_alloc - Allocate memory for all queues
1411	* @vport: virtual port
1412	*
1413	* Allocate memory for queues associated with a vport. Returns 0 on success,
1414	* negative on failure.
1415	*/
1416	int idpf_vport_queues_alloc(struct idpf_vport *vport)
1417	{
1418	int err;
1419
1420	err = idpf_vport_queue_grp_alloc_all(vport);
1421	if (err)
1422	goto err_out;
1423
1424	err = idpf_tx_desc_alloc_all(vport);
1425	if (err)
1426	goto err_out;
1427
1428	err = idpf_rx_desc_alloc_all(vport);
1429	if (err)
1430	goto err_out;
1431
1432	err = idpf_vport_init_fast_path_txqs(vport);
1433	if (err)
1434	goto err_out;
1435
1436	return 0;
1437
1438	err_out:
1439	idpf_vport_queues_rel(vport);
1440
1441	return err;
1442	}
1443
1444	/**
1445	* idpf_tx_handle_sw_marker - Handle queue marker packet
1446	* @tx_q: tx queue to handle software marker
1447	*/
1448	static void idpf_tx_handle_sw_marker(struct idpf_queue *tx_q)
1449	{
1450	struct idpf_vport *vport = tx_q->vport;
1451	int i;
1452
1453	clear_bit(nr: __IDPF_Q_SW_MARKER, addr: tx_q->flags);
1454	/* Hardware must write marker packets to all queues associated with
1455	* completion queues. So check if all queues received marker packets
1456	*/
1457	for (i = 0; i < vport->num_txq; i++)
1458	/* If we're still waiting on any other TXQ marker completions,
1459	* just return now since we cannot wake up the marker_wq yet.
1460	*/
1461	if (test_bit(__IDPF_Q_SW_MARKER, vport->txqs[i]->flags))
1462	return;
1463
1464	/* Drain complete */
1465	set_bit(nr: IDPF_VPORT_SW_MARKER, addr: vport->flags);
1466	wake_up(&vport->sw_marker_wq);
1467	}
1468
1469	/**
1470	* idpf_tx_splitq_clean_hdr - Clean TX buffer resources for header portion of
1471	* packet
1472	* @tx_q: tx queue to clean buffer from
1473	* @tx_buf: buffer to be cleaned
1474	* @cleaned: pointer to stats struct to track cleaned packets/bytes
1475	* @napi_budget: Used to determine if we are in netpoll
1476	*/
1477	static void idpf_tx_splitq_clean_hdr(struct idpf_queue *tx_q,
1478	struct idpf_tx_buf *tx_buf,
1479	struct idpf_cleaned_stats *cleaned,
1480	int napi_budget)
1481	{
1482	napi_consume_skb(skb: tx_buf->skb, budget: napi_budget);
1483
1484	if (dma_unmap_len(tx_buf, len)) {
1485	dma_unmap_single(tx_q->dev,
1486	dma_unmap_addr(tx_buf, dma),
1487	dma_unmap_len(tx_buf, len),
1488	DMA_TO_DEVICE);
1489
1490	dma_unmap_len_set(tx_buf, len, 0);
1491	}
1492
1493	/* clear tx_buf data */
1494	tx_buf->skb = NULL;
1495
1496	cleaned->bytes += tx_buf->bytecount;
1497	cleaned->packets += tx_buf->gso_segs;
1498	}
1499
1500	/**
1501	* idpf_tx_clean_stashed_bufs - clean bufs that were stored for
1502	* out of order completions
1503	* @txq: queue to clean
1504	* @compl_tag: completion tag of packet to clean (from completion descriptor)
1505	* @cleaned: pointer to stats struct to track cleaned packets/bytes
1506	* @budget: Used to determine if we are in netpoll
1507	*/
1508	static void idpf_tx_clean_stashed_bufs(struct idpf_queue *txq, u16 compl_tag,
1509	struct idpf_cleaned_stats *cleaned,
1510	int budget)
1511	{
1512	struct idpf_tx_stash *stash;
1513	struct hlist_node *tmp_buf;
1514
1515	/* Buffer completion */
1516	hash_for_each_possible_safe(txq->sched_buf_hash, stash, tmp_buf,
1517	hlist, compl_tag) {
1518	if (unlikely(stash->buf.compl_tag != (int)compl_tag))
1519	continue;
1520
1521	if (stash->buf.skb) {
1522	idpf_tx_splitq_clean_hdr(tx_q: txq, tx_buf: &stash->buf, cleaned,
1523	napi_budget: budget);
1524	} else if (dma_unmap_len(&stash->buf, len)) {
1525	dma_unmap_page(txq->dev,
1526	dma_unmap_addr(&stash->buf, dma),
1527	dma_unmap_len(&stash->buf, len),
1528	DMA_TO_DEVICE);
1529	dma_unmap_len_set(&stash->buf, len, 0);
1530	}
1531
1532	/* Push shadow buf back onto stack */
1533	idpf_buf_lifo_push(stack: &txq->buf_stack, buf: stash);
1534
1535	hash_del(node: &stash->hlist);
1536	}
1537	}
1538
1539	/**
1540	* idpf_stash_flow_sch_buffers - store buffer parameters info to be freed at a
1541	* later time (only relevant for flow scheduling mode)
1542	* @txq: Tx queue to clean
1543	* @tx_buf: buffer to store
1544	*/
1545	static int idpf_stash_flow_sch_buffers(struct idpf_queue *txq,
1546	struct idpf_tx_buf *tx_buf)
1547	{
1548	struct idpf_tx_stash *stash;
1549
1550	if (unlikely(!dma_unmap_addr(tx_buf, dma) &&
1551	!dma_unmap_len(tx_buf, len)))
1552	return 0;
1553
1554	stash = idpf_buf_lifo_pop(stack: &txq->buf_stack);
1555	if (unlikely(!stash)) {
1556	net_err_ratelimited("%s: No out-of-order TX buffers left!\n",
1557	txq->vport->netdev->name);
1558
1559	return -ENOMEM;
1560	}
1561
1562	/* Store buffer params in shadow buffer */
1563	stash->buf.skb = tx_buf->skb;
1564	stash->buf.bytecount = tx_buf->bytecount;
1565	stash->buf.gso_segs = tx_buf->gso_segs;
1566	dma_unmap_addr_set(&stash->buf, dma, dma_unmap_addr(tx_buf, dma));
1567	dma_unmap_len_set(&stash->buf, len, dma_unmap_len(tx_buf, len));
1568	stash->buf.compl_tag = tx_buf->compl_tag;
1569
1570	/* Add buffer to buf_hash table to be freed later */
1571	hash_add(txq->sched_buf_hash, &stash->hlist, stash->buf.compl_tag);
1572
1573	memset(tx_buf, 0, sizeof(struct idpf_tx_buf));
1574
1575	/* Reinitialize buf_id portion of tag */
1576	tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG;
1577
1578	return 0;
1579	}
1580
1581	#define idpf_tx_splitq_clean_bump_ntc(txq, ntc, desc, buf) \
1582	do { \
1583	(ntc)++; \
1584	if (unlikely(!(ntc))) { \
1585	ntc -= (txq)->desc_count; \
1586	buf = (txq)->tx_buf; \
1587	desc = IDPF_FLEX_TX_DESC(txq, 0); \
1588	} else { \
1589	(buf)++; \
1590	(desc)++; \
1591	} \
1592	} while (0)
1593
1594	/**
1595	* idpf_tx_splitq_clean - Reclaim resources from buffer queue
1596	* @tx_q: Tx queue to clean
1597	* @end: queue index until which it should be cleaned
1598	* @napi_budget: Used to determine if we are in netpoll
1599	* @cleaned: pointer to stats struct to track cleaned packets/bytes
1600	* @descs_only: true if queue is using flow-based scheduling and should
1601	* not clean buffers at this time
1602	*
1603	* Cleans the queue descriptor ring. If the queue is using queue-based
1604	* scheduling, the buffers will be cleaned as well. If the queue is using
1605	* flow-based scheduling, only the descriptors are cleaned at this time.
1606	* Separate packet completion events will be reported on the completion queue,
1607	* and the buffers will be cleaned separately. The stats are not updated from
1608	* this function when using flow-based scheduling.
1609	*/
1610	static void idpf_tx_splitq_clean(struct idpf_queue *tx_q, u16 end,
1611	int napi_budget,
1612	struct idpf_cleaned_stats *cleaned,
1613	bool descs_only)
1614	{
1615	union idpf_tx_flex_desc *next_pending_desc = NULL;
1616	union idpf_tx_flex_desc *tx_desc;
1617	s16 ntc = tx_q->next_to_clean;
1618	struct idpf_tx_buf *tx_buf;
1619
1620	tx_desc = IDPF_FLEX_TX_DESC(tx_q, ntc);
1621	next_pending_desc = IDPF_FLEX_TX_DESC(tx_q, end);
1622	tx_buf = &tx_q->tx_buf[ntc];
1623	ntc -= tx_q->desc_count;
1624
1625	while (tx_desc != next_pending_desc) {
1626	union idpf_tx_flex_desc *eop_desc;
1627
1628	/* If this entry in the ring was used as a context descriptor,
1629	* it's corresponding entry in the buffer ring will have an
1630	* invalid completion tag since no buffer was used. We can
1631	* skip this descriptor since there is no buffer to clean.
1632	*/
1633	if (unlikely(tx_buf->compl_tag == IDPF_SPLITQ_TX_INVAL_COMPL_TAG))
1634	goto fetch_next_txq_desc;
1635
1636	eop_desc = (union idpf_tx_flex_desc *)tx_buf->next_to_watch;
1637
1638	/* clear next_to_watch to prevent false hangs */
1639	tx_buf->next_to_watch = NULL;
1640
1641	if (descs_only) {
1642	if (idpf_stash_flow_sch_buffers(txq: tx_q, tx_buf))
1643	goto tx_splitq_clean_out;
1644
1645	while (tx_desc != eop_desc) {
1646	idpf_tx_splitq_clean_bump_ntc(tx_q, ntc,
1647	tx_desc, tx_buf);
1648
1649	if (dma_unmap_len(tx_buf, len)) {
1650	if (idpf_stash_flow_sch_buffers(txq: tx_q,
1651	tx_buf))
1652	goto tx_splitq_clean_out;
1653	}
1654	}
1655	} else {
1656	idpf_tx_splitq_clean_hdr(tx_q, tx_buf, cleaned,
1657	napi_budget);
1658
1659	/* unmap remaining buffers */
1660	while (tx_desc != eop_desc) {
1661	idpf_tx_splitq_clean_bump_ntc(tx_q, ntc,
1662	tx_desc, tx_buf);
1663
1664	/* unmap any remaining paged data */
1665	if (dma_unmap_len(tx_buf, len)) {
1666	dma_unmap_page(tx_q->dev,
1667	dma_unmap_addr(tx_buf, dma),
1668	dma_unmap_len(tx_buf, len),
1669	DMA_TO_DEVICE);
1670	dma_unmap_len_set(tx_buf, len, 0);
1671	}
1672	}
1673	}
1674
1675	fetch_next_txq_desc:
1676	idpf_tx_splitq_clean_bump_ntc(tx_q, ntc, tx_desc, tx_buf);
1677	}
1678
1679	tx_splitq_clean_out:
1680	ntc += tx_q->desc_count;
1681	tx_q->next_to_clean = ntc;
1682	}
1683
1684	#define idpf_tx_clean_buf_ring_bump_ntc(txq, ntc, buf) \
1685	do { \
1686	(buf)++; \
1687	(ntc)++; \
1688	if (unlikely((ntc) == (txq)->desc_count)) { \
1689	buf = (txq)->tx_buf; \
1690	ntc = 0; \
1691	} \
1692	} while (0)
1693
1694	/**
1695	* idpf_tx_clean_buf_ring - clean flow scheduling TX queue buffers
1696	* @txq: queue to clean
1697	* @compl_tag: completion tag of packet to clean (from completion descriptor)
1698	* @cleaned: pointer to stats struct to track cleaned packets/bytes
1699	* @budget: Used to determine if we are in netpoll
1700	*
1701	* Cleans all buffers associated with the input completion tag either from the
1702	* TX buffer ring or from the hash table if the buffers were previously
1703	* stashed. Returns the byte/segment count for the cleaned packet associated
1704	* this completion tag.
1705	*/
1706	static bool idpf_tx_clean_buf_ring(struct idpf_queue *txq, u16 compl_tag,
1707	struct idpf_cleaned_stats *cleaned,
1708	int budget)
1709	{
1710	u16 idx = compl_tag & txq->compl_tag_bufid_m;
1711	struct idpf_tx_buf *tx_buf = NULL;
1712	u16 ntc = txq->next_to_clean;
1713	u16 num_descs_cleaned = 0;
1714	u16 orig_idx = idx;
1715
1716	tx_buf = &txq->tx_buf[idx];
1717
1718	while (tx_buf->compl_tag == (int)compl_tag) {
1719	if (tx_buf->skb) {
1720	idpf_tx_splitq_clean_hdr(tx_q: txq, tx_buf, cleaned, napi_budget: budget);
1721	} else if (dma_unmap_len(tx_buf, len)) {
1722	dma_unmap_page(txq->dev,
1723	dma_unmap_addr(tx_buf, dma),
1724	dma_unmap_len(tx_buf, len),
1725	DMA_TO_DEVICE);
1726	dma_unmap_len_set(tx_buf, len, 0);
1727	}
1728
1729	memset(tx_buf, 0, sizeof(struct idpf_tx_buf));
1730	tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG;
1731
1732	num_descs_cleaned++;
1733	idpf_tx_clean_buf_ring_bump_ntc(txq, idx, tx_buf);
1734	}
1735
1736	/* If we didn't clean anything on the ring for this completion, there's
1737	* nothing more to do.
1738	*/
1739	if (unlikely(!num_descs_cleaned))
1740	return false;
1741
1742	/* Otherwise, if we did clean a packet on the ring directly, it's safe
1743	* to assume that the descriptors starting from the original
1744	* next_to_clean up until the previously cleaned packet can be reused.
1745	* Therefore, we will go back in the ring and stash any buffers still
1746	* in the ring into the hash table to be cleaned later.
1747	*/
1748	tx_buf = &txq->tx_buf[ntc];
1749	while (tx_buf != &txq->tx_buf[orig_idx]) {
1750	idpf_stash_flow_sch_buffers(txq, tx_buf);
1751	idpf_tx_clean_buf_ring_bump_ntc(txq, ntc, tx_buf);
1752	}
1753
1754	/* Finally, update next_to_clean to reflect the work that was just done
1755	* on the ring, if any. If the packet was only cleaned from the hash
1756	* table, the ring will not be impacted, therefore we should not touch
1757	* next_to_clean. The updated idx is used here
1758	*/
1759	txq->next_to_clean = idx;
1760
1761	return true;
1762	}
1763
1764	/**
1765	* idpf_tx_handle_rs_completion - clean a single packet and all of its buffers
1766	* whether on the buffer ring or in the hash table
1767	* @txq: Tx ring to clean
1768	* @desc: pointer to completion queue descriptor to extract completion
1769	* information from
1770	* @cleaned: pointer to stats struct to track cleaned packets/bytes
1771	* @budget: Used to determine if we are in netpoll
1772	*
1773	* Returns bytes/packets cleaned
1774	*/
1775	static void idpf_tx_handle_rs_completion(struct idpf_queue *txq,
1776	struct idpf_splitq_tx_compl_desc *desc,
1777	struct idpf_cleaned_stats *cleaned,
1778	int budget)
1779	{
1780	u16 compl_tag;
1781
1782	if (!test_bit(__IDPF_Q_FLOW_SCH_EN, txq->flags)) {
1783	u16 head = le16_to_cpu(desc->q_head_compl_tag.q_head);
1784
1785	return idpf_tx_splitq_clean(tx_q: txq, end: head, napi_budget: budget, cleaned, descs_only: false);
1786	}
1787
1788	compl_tag = le16_to_cpu(desc->q_head_compl_tag.compl_tag);
1789
1790	/* If we didn't clean anything on the ring, this packet must be
1791	* in the hash table. Go clean it there.
1792	*/
1793	if (!idpf_tx_clean_buf_ring(txq, compl_tag, cleaned, budget))
1794	idpf_tx_clean_stashed_bufs(txq, compl_tag, cleaned, budget);
1795	}
1796
1797	/**
1798	* idpf_tx_clean_complq - Reclaim resources on completion queue
1799	* @complq: Tx ring to clean
1800	* @budget: Used to determine if we are in netpoll
1801	* @cleaned: returns number of packets cleaned
1802	*
1803	* Returns true if there's any budget left (e.g. the clean is finished)
1804	*/
1805	static bool idpf_tx_clean_complq(struct idpf_queue *complq, int budget,
1806	int *cleaned)
1807	{
1808	struct idpf_splitq_tx_compl_desc *tx_desc;
1809	struct idpf_vport *vport = complq->vport;
1810	s16 ntc = complq->next_to_clean;
1811	struct idpf_netdev_priv *np;
1812	unsigned int complq_budget;
1813	bool complq_ok = true;
1814	int i;
1815
1816	complq_budget = vport->compln_clean_budget;
1817	tx_desc = IDPF_SPLITQ_TX_COMPLQ_DESC(complq, ntc);
1818	ntc -= complq->desc_count;
1819
1820	do {
1821	struct idpf_cleaned_stats cleaned_stats = { };
1822	struct idpf_queue *tx_q;
1823	int rel_tx_qid;
1824	u16 hw_head;
1825	u8 ctype; /* completion type */
1826	u16 gen;
1827
1828	/* if the descriptor isn't done, no work yet to do */
1829	gen = le16_get_bits(v: tx_desc->qid_comptype_gen,
1830	IDPF_TXD_COMPLQ_GEN_M);
1831	if (test_bit(__IDPF_Q_GEN_CHK, complq->flags) != gen)
1832	break;
1833
1834	/* Find necessary info of TX queue to clean buffers */
1835	rel_tx_qid = le16_get_bits(v: tx_desc->qid_comptype_gen,
1836	IDPF_TXD_COMPLQ_QID_M);
1837	if (rel_tx_qid >= complq->txq_grp->num_txq ||
1838	!complq->txq_grp->txqs[rel_tx_qid]) {
1839	dev_err(&complq->vport->adapter->pdev->dev,
1840	"TxQ not found\n");
1841	goto fetch_next_desc;
1842	}
1843	tx_q = complq->txq_grp->txqs[rel_tx_qid];
1844
1845	/* Determine completion type */
1846	ctype = le16_get_bits(v: tx_desc->qid_comptype_gen,
1847	IDPF_TXD_COMPLQ_COMPL_TYPE_M);
1848	switch (ctype) {
1849	case IDPF_TXD_COMPLT_RE:
1850	hw_head = le16_to_cpu(tx_desc->q_head_compl_tag.q_head);
1851
1852	idpf_tx_splitq_clean(tx_q, end: hw_head, napi_budget: budget,
1853	cleaned: &cleaned_stats, descs_only: true);
1854	break;
1855	case IDPF_TXD_COMPLT_RS:
1856	idpf_tx_handle_rs_completion(txq: tx_q, desc: tx_desc,
1857	cleaned: &cleaned_stats, budget);
1858	break;
1859	case IDPF_TXD_COMPLT_SW_MARKER:
1860	idpf_tx_handle_sw_marker(tx_q);
1861	break;
1862	default:
1863	dev_err(&tx_q->vport->adapter->pdev->dev,
1864	"Unknown TX completion type: %d\n",
1865	ctype);
1866	goto fetch_next_desc;
1867	}
1868
1869	u64_stats_update_begin(syncp: &tx_q->stats_sync);
1870	u64_stats_add(p: &tx_q->q_stats.tx.packets, val: cleaned_stats.packets);
1871	u64_stats_add(p: &tx_q->q_stats.tx.bytes, val: cleaned_stats.bytes);
1872	tx_q->cleaned_pkts += cleaned_stats.packets;
1873	tx_q->cleaned_bytes += cleaned_stats.bytes;
1874	complq->num_completions++;
1875	u64_stats_update_end(syncp: &tx_q->stats_sync);
1876
1877	fetch_next_desc:
1878	tx_desc++;
1879	ntc++;
1880	if (unlikely(!ntc)) {
1881	ntc -= complq->desc_count;
1882	tx_desc = IDPF_SPLITQ_TX_COMPLQ_DESC(complq, 0);
1883	change_bit(nr: __IDPF_Q_GEN_CHK, addr: complq->flags);
1884	}
1885
1886	prefetch(tx_desc);
1887
1888	/* update budget accounting */
1889	complq_budget--;
1890	} while (likely(complq_budget));
1891
1892	/* Store the state of the complq to be used later in deciding if a
1893	* TXQ can be started again
1894	*/
1895	if (unlikely(IDPF_TX_COMPLQ_PENDING(complq->txq_grp) >
1896	IDPF_TX_COMPLQ_OVERFLOW_THRESH(complq)))
1897	complq_ok = false;
1898
1899	np = netdev_priv(dev: complq->vport->netdev);
1900	for (i = 0; i < complq->txq_grp->num_txq; ++i) {
1901	struct idpf_queue *tx_q = complq->txq_grp->txqs[i];
1902	struct netdev_queue *nq;
1903	bool dont_wake;
1904
1905	/* We didn't clean anything on this queue, move along */
1906	if (!tx_q->cleaned_bytes)
1907	continue;
1908
1909	*cleaned += tx_q->cleaned_pkts;
1910
1911	/* Update BQL */
1912	nq = netdev_get_tx_queue(dev: tx_q->vport->netdev, index: tx_q->idx);
1913
1914	dont_wake = !complq_ok || IDPF_TX_BUF_RSV_LOW(tx_q) ||
1915	np->state != __IDPF_VPORT_UP ||
1916	!netif_carrier_ok(dev: tx_q->vport->netdev);
1917	/* Check if the TXQ needs to and can be restarted */
1918	__netif_txq_completed_wake(nq, tx_q->cleaned_pkts, tx_q->cleaned_bytes,
1919	IDPF_DESC_UNUSED(tx_q), IDPF_TX_WAKE_THRESH,
1920	dont_wake);
1921
1922	/* Reset cleaned stats for the next time this queue is
1923	* cleaned
1924	*/
1925	tx_q->cleaned_bytes = 0;
1926	tx_q->cleaned_pkts = 0;
1927	}
1928
1929	ntc += complq->desc_count;
1930	complq->next_to_clean = ntc;
1931
1932	return !!complq_budget;
1933	}
1934
1935	/**
1936	* idpf_tx_splitq_build_ctb - populate command tag and size for queue
1937	* based scheduling descriptors
1938	* @desc: descriptor to populate
1939	* @params: pointer to tx params struct
1940	* @td_cmd: command to be filled in desc
1941	* @size: size of buffer
1942	*/
1943	void idpf_tx_splitq_build_ctb(union idpf_tx_flex_desc *desc,
1944	struct idpf_tx_splitq_params *params,
1945	u16 td_cmd, u16 size)
1946	{
1947	desc->q.qw1.cmd_dtype =
1948	le16_encode_bits(v: params->dtype, IDPF_FLEX_TXD_QW1_DTYPE_M);
1949	desc->q.qw1.cmd_dtype |=
1950	le16_encode_bits(v: td_cmd, IDPF_FLEX_TXD_QW1_CMD_M);
1951	desc->q.qw1.buf_size = cpu_to_le16(size);
1952	desc->q.qw1.l2tags.l2tag1 = cpu_to_le16(params->td_tag);
1953	}
1954
1955	/**
1956	* idpf_tx_splitq_build_flow_desc - populate command tag and size for flow
1957	* scheduling descriptors
1958	* @desc: descriptor to populate
1959	* @params: pointer to tx params struct
1960	* @td_cmd: command to be filled in desc
1961	* @size: size of buffer
1962	*/
1963	void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc,
1964	struct idpf_tx_splitq_params *params,
1965	u16 td_cmd, u16 size)
1966	{
1967	desc->flow.qw1.cmd_dtype = (u16)params->dtype | td_cmd;
1968	desc->flow.qw1.rxr_bufsize = cpu_to_le16((u16)size);
1969	desc->flow.qw1.compl_tag = cpu_to_le16(params->compl_tag);
1970	}
1971
1972	/**
1973	* idpf_tx_maybe_stop_common - 1st level check for common Tx stop conditions
1974	* @tx_q: the queue to be checked
1975	* @size: number of descriptors we want to assure is available
1976	*
1977	* Returns 0 if stop is not needed
1978	*/
1979	int idpf_tx_maybe_stop_common(struct idpf_queue *tx_q, unsigned int size)
1980	{
1981	struct netdev_queue *nq;
1982
1983	if (likely(IDPF_DESC_UNUSED(tx_q) >= size))
1984	return 0;
1985
1986	u64_stats_update_begin(syncp: &tx_q->stats_sync);
1987	u64_stats_inc(p: &tx_q->q_stats.tx.q_busy);
1988	u64_stats_update_end(syncp: &tx_q->stats_sync);
1989
1990	nq = netdev_get_tx_queue(dev: tx_q->vport->netdev, index: tx_q->idx);
1991
1992	return netif_txq_maybe_stop(nq, IDPF_DESC_UNUSED(tx_q), size, size);
1993	}
1994
1995	/**
1996	* idpf_tx_maybe_stop_splitq - 1st level check for Tx splitq stop conditions
1997	* @tx_q: the queue to be checked
1998	* @descs_needed: number of descriptors required for this packet
1999	*
2000	* Returns 0 if stop is not needed
2001	*/
2002	static int idpf_tx_maybe_stop_splitq(struct idpf_queue *tx_q,
2003	unsigned int descs_needed)
2004	{
2005	if (idpf_tx_maybe_stop_common(tx_q, size: descs_needed))
2006	goto splitq_stop;
2007
2008	/* If there are too many outstanding completions expected on the
2009	* completion queue, stop the TX queue to give the device some time to
2010	* catch up
2011	*/
2012	if (unlikely(IDPF_TX_COMPLQ_PENDING(tx_q->txq_grp) >
2013	IDPF_TX_COMPLQ_OVERFLOW_THRESH(tx_q->txq_grp->complq)))
2014	goto splitq_stop;
2015
2016	/* Also check for available book keeping buffers; if we are low, stop
2017	* the queue to wait for more completions
2018	*/
2019	if (unlikely(IDPF_TX_BUF_RSV_LOW(tx_q)))
2020	goto splitq_stop;
2021
2022	return 0;
2023
2024	splitq_stop:
2025	u64_stats_update_begin(syncp: &tx_q->stats_sync);
2026	u64_stats_inc(p: &tx_q->q_stats.tx.q_busy);
2027	u64_stats_update_end(syncp: &tx_q->stats_sync);
2028	netif_stop_subqueue(dev: tx_q->vport->netdev, queue_index: tx_q->idx);
2029
2030	return -EBUSY;
2031	}
2032
2033	/**
2034	* idpf_tx_buf_hw_update - Store the new tail value
2035	* @tx_q: queue to bump
2036	* @val: new tail index
2037	* @xmit_more: more skb's pending
2038	*
2039	* The naming here is special in that 'hw' signals that this function is about
2040	* to do a register write to update our queue status. We know this can only
2041	* mean tail here as HW should be owning head for TX.
2042	*/
2043	void idpf_tx_buf_hw_update(struct idpf_queue *tx_q, u32 val,
2044	bool xmit_more)
2045	{
2046	struct netdev_queue *nq;
2047
2048	nq = netdev_get_tx_queue(dev: tx_q->vport->netdev, index: tx_q->idx);
2049	tx_q->next_to_use = val;
2050
2051	idpf_tx_maybe_stop_common(tx_q, IDPF_TX_DESC_NEEDED);
2052
2053	/* Force memory writes to complete before letting h/w
2054	* know there are new descriptors to fetch. (Only
2055	* applicable for weak-ordered memory model archs,
2056	* such as IA-64).
2057	*/
2058	wmb();
2059
2060	/* notify HW of packet */
2061	if (netif_xmit_stopped(dev_queue: nq) || !xmit_more)
2062	writel(val, addr: tx_q->tail);
2063	}
2064
2065	/**
2066	* idpf_tx_desc_count_required - calculate number of Tx descriptors needed
2067	* @txq: queue to send buffer on
2068	* @skb: send buffer
2069	*
2070	* Returns number of data descriptors needed for this skb.
2071	*/
2072	unsigned int idpf_tx_desc_count_required(struct idpf_queue *txq,
2073	struct sk_buff *skb)
2074	{
2075	const struct skb_shared_info *shinfo;
2076	unsigned int count = 0, i;
2077
2078	count += !!skb_headlen(skb);
2079
2080	if (!skb_is_nonlinear(skb))
2081	return count;
2082
2083	shinfo = skb_shinfo(skb);
2084	for (i = 0; i < shinfo->nr_frags; i++) {
2085	unsigned int size;
2086
2087	size = skb_frag_size(frag: &shinfo->frags[i]);
2088
2089	/* We only need to use the idpf_size_to_txd_count check if the
2090	* fragment is going to span multiple descriptors,
2091	* i.e. size >= 16K.
2092	*/
2093	if (size >= SZ_16K)
2094	count += idpf_size_to_txd_count(size);
2095	else
2096	count++;
2097	}
2098
2099	if (idpf_chk_linearize(skb, max_bufs: txq->tx_max_bufs, count)) {
2100	if (__skb_linearize(skb))
2101	return 0;
2102
2103	count = idpf_size_to_txd_count(size: skb->len);
2104	u64_stats_update_begin(syncp: &txq->stats_sync);
2105	u64_stats_inc(p: &txq->q_stats.tx.linearize);
2106	u64_stats_update_end(syncp: &txq->stats_sync);
2107	}
2108
2109	return count;
2110	}
2111
2112	/**
2113	* idpf_tx_dma_map_error - handle TX DMA map errors
2114	* @txq: queue to send buffer on
2115	* @skb: send buffer
2116	* @first: original first buffer info buffer for packet
2117	* @idx: starting point on ring to unwind
2118	*/
2119	void idpf_tx_dma_map_error(struct idpf_queue *txq, struct sk_buff *skb,
2120	struct idpf_tx_buf *first, u16 idx)
2121	{
2122	u64_stats_update_begin(syncp: &txq->stats_sync);
2123	u64_stats_inc(p: &txq->q_stats.tx.dma_map_errs);
2124	u64_stats_update_end(syncp: &txq->stats_sync);
2125
2126	/* clear dma mappings for failed tx_buf map */
2127	for (;;) {
2128	struct idpf_tx_buf *tx_buf;
2129
2130	tx_buf = &txq->tx_buf[idx];
2131	idpf_tx_buf_rel(tx_q: txq, tx_buf);
2132	if (tx_buf == first)
2133	break;
2134	if (idx == 0)
2135	idx = txq->desc_count;
2136	idx--;
2137	}
2138
2139	if (skb_is_gso(skb)) {
2140	union idpf_tx_flex_desc *tx_desc;
2141
2142	/* If we failed a DMA mapping for a TSO packet, we will have
2143	* used one additional descriptor for a context
2144	* descriptor. Reset that here.
2145	*/
2146	tx_desc = IDPF_FLEX_TX_DESC(txq, idx);
2147	memset(tx_desc, 0, sizeof(struct idpf_flex_tx_ctx_desc));
2148	if (idx == 0)
2149	idx = txq->desc_count;
2150	idx--;
2151	}
2152
2153	/* Update tail in case netdev_xmit_more was previously true */
2154	idpf_tx_buf_hw_update(tx_q: txq, val: idx, xmit_more: false);
2155	}
2156
2157	/**
2158	* idpf_tx_splitq_bump_ntu - adjust NTU and generation
2159	* @txq: the tx ring to wrap
2160	* @ntu: ring index to bump
2161	*/
2162	static unsigned int idpf_tx_splitq_bump_ntu(struct idpf_queue *txq, u16 ntu)
2163	{
2164	ntu++;
2165
2166	if (ntu == txq->desc_count) {
2167	ntu = 0;
2168	txq->compl_tag_cur_gen = IDPF_TX_ADJ_COMPL_TAG_GEN(txq);
2169	}
2170
2171	return ntu;
2172	}
2173
2174	/**
2175	* idpf_tx_splitq_map - Build the Tx flex descriptor
2176	* @tx_q: queue to send buffer on
2177	* @params: pointer to splitq params struct
2178	* @first: first buffer info buffer to use
2179	*
2180	* This function loops over the skb data pointed to by *first
2181	* and gets a physical address for each memory location and programs
2182	* it and the length into the transmit flex descriptor.
2183	*/
2184	static void idpf_tx_splitq_map(struct idpf_queue *tx_q,
2185	struct idpf_tx_splitq_params *params,
2186	struct idpf_tx_buf *first)
2187	{
2188	union idpf_tx_flex_desc *tx_desc;
2189	unsigned int data_len, size;
2190	struct idpf_tx_buf *tx_buf;
2191	u16 i = tx_q->next_to_use;
2192	struct netdev_queue *nq;
2193	struct sk_buff *skb;
2194	skb_frag_t *frag;
2195	u16 td_cmd = 0;
2196	dma_addr_t dma;
2197
2198	skb = first->skb;
2199
2200	td_cmd = params->offload.td_cmd;
2201
2202	data_len = skb->data_len;
2203	size = skb_headlen(skb);
2204
2205	tx_desc = IDPF_FLEX_TX_DESC(tx_q, i);
2206
2207	dma = dma_map_single(tx_q->dev, skb->data, size, DMA_TO_DEVICE);
2208
2209	tx_buf = first;
2210
2211	params->compl_tag =
2212	(tx_q->compl_tag_cur_gen << tx_q->compl_tag_gen_s) | i;
2213
2214	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2215	unsigned int max_data = IDPF_TX_MAX_DESC_DATA_ALIGNED;
2216
2217	if (dma_mapping_error(dev: tx_q->dev, dma_addr: dma))
2218	return idpf_tx_dma_map_error(txq: tx_q, skb, first, idx: i);
2219
2220	tx_buf->compl_tag = params->compl_tag;
2221
2222	/* record length, and DMA address */
2223	dma_unmap_len_set(tx_buf, len, size);
2224	dma_unmap_addr_set(tx_buf, dma, dma);
2225
2226	/* buf_addr is in same location for both desc types */
2227	tx_desc->q.buf_addr = cpu_to_le64(dma);
2228
2229	/* The stack can send us fragments that are too large for a
2230	* single descriptor i.e. frag size > 16K-1. We will need to
2231	* split the fragment across multiple descriptors in this case.
2232	* To adhere to HW alignment restrictions, the fragment needs
2233	* to be split such that the first chunk ends on a 4K boundary
2234	* and all subsequent chunks start on a 4K boundary. We still
2235	* want to send as much data as possible though, so our
2236	* intermediate descriptor chunk size will be 12K.
2237	*
2238	* For example, consider a 32K fragment mapped to DMA addr 2600.
2239	* ------------------------------------------------------------
2240	* | frag_size = 32K |
2241	* ------------------------------------------------------------
2242	* |2600 |16384 |28672
2243	*
2244	* 3 descriptors will be used for this fragment. The HW expects
2245	* the descriptors to contain the following:
2246	* ------------------------------------------------------------
2247	* | size = 13784 | size = 12K | size = 6696 |
2248	* | dma = 2600 | dma = 16384 | dma = 28672 |
2249	* ------------------------------------------------------------
2250	*
2251	* We need to first adjust the max_data for the first chunk so
2252	* that it ends on a 4K boundary. By negating the value of the
2253	* DMA address and taking only the low order bits, we're
2254	* effectively calculating
2255	* 4K - (DMA addr lower order bits) =
2256	* bytes to next boundary.
2257	*
2258	* Add that to our base aligned max_data (12K) and we have
2259	* our first chunk size. In the example above,
2260	* 13784 = 12K + (4096-2600)
2261	*
2262	* After guaranteeing the first chunk ends on a 4K boundary, we
2263	* will give the intermediate descriptors 12K chunks and
2264	* whatever is left to the final descriptor. This ensures that
2265	* all descriptors used for the remaining chunks of the
2266	* fragment start on a 4K boundary and we use as few
2267	* descriptors as possible.
2268	*/
2269	max_data += -dma & (IDPF_TX_MAX_READ_REQ_SIZE - 1);
2270	while (unlikely(size > IDPF_TX_MAX_DESC_DATA)) {
2271	idpf_tx_splitq_build_desc(desc: tx_desc, params, td_cmd,
2272	size: max_data);
2273
2274	tx_desc++;
2275	i++;
2276
2277	if (i == tx_q->desc_count) {
2278	tx_desc = IDPF_FLEX_TX_DESC(tx_q, 0);
2279	i = 0;
2280	tx_q->compl_tag_cur_gen =
2281	IDPF_TX_ADJ_COMPL_TAG_GEN(tx_q);
2282	}
2283
2284	/* Since this packet has a buffer that is going to span
2285	* multiple descriptors, it's going to leave holes in
2286	* to the TX buffer ring. To ensure these holes do not
2287	* cause issues in the cleaning routines, we will clear
2288	* them of any stale data and assign them the same
2289	* completion tag as the current packet. Then when the
2290	* packet is being cleaned, the cleaning routines will
2291	* simply pass over these holes and finish cleaning the
2292	* rest of the packet.
2293	*/
2294	memset(&tx_q->tx_buf[i], 0, sizeof(struct idpf_tx_buf));
2295	tx_q->tx_buf[i].compl_tag = params->compl_tag;
2296
2297	/* Adjust the DMA offset and the remaining size of the
2298	* fragment. On the first iteration of this loop,
2299	* max_data will be >= 12K and <= 16K-1. On any
2300	* subsequent iteration of this loop, max_data will
2301	* always be 12K.
2302	*/
2303	dma += max_data;
2304	size -= max_data;
2305
2306	/* Reset max_data since remaining chunks will be 12K
2307	* at most
2308	*/
2309	max_data = IDPF_TX_MAX_DESC_DATA_ALIGNED;
2310
2311	/* buf_addr is in same location for both desc types */
2312	tx_desc->q.buf_addr = cpu_to_le64(dma);
2313	}
2314
2315	if (!data_len)
2316	break;
2317
2318	idpf_tx_splitq_build_desc(desc: tx_desc, params, td_cmd, size);
2319	tx_desc++;
2320	i++;
2321
2322	if (i == tx_q->desc_count) {
2323	tx_desc = IDPF_FLEX_TX_DESC(tx_q, 0);
2324	i = 0;
2325	tx_q->compl_tag_cur_gen = IDPF_TX_ADJ_COMPL_TAG_GEN(tx_q);
2326	}
2327
2328	size = skb_frag_size(frag);
2329	data_len -= size;
2330
2331	dma = skb_frag_dma_map(dev: tx_q->dev, frag, offset: 0, size,
2332	dir: DMA_TO_DEVICE);
2333
2334	tx_buf = &tx_q->tx_buf[i];
2335	}
2336
2337	/* record SW timestamp if HW timestamp is not available */
2338	skb_tx_timestamp(skb);
2339
2340	/* write last descriptor with RS and EOP bits */
2341	td_cmd |= params->eop_cmd;
2342	idpf_tx_splitq_build_desc(desc: tx_desc, params, td_cmd, size);
2343	i = idpf_tx_splitq_bump_ntu(txq: tx_q, ntu: i);
2344
2345	/* set next_to_watch value indicating a packet is present */
2346	first->next_to_watch = tx_desc;
2347
2348	tx_q->txq_grp->num_completions_pending++;
2349
2350	/* record bytecount for BQL */
2351	nq = netdev_get_tx_queue(dev: tx_q->vport->netdev, index: tx_q->idx);
2352	netdev_tx_sent_queue(dev_queue: nq, bytes: first->bytecount);
2353
2354	idpf_tx_buf_hw_update(tx_q, val: i, xmit_more: netdev_xmit_more());
2355	}
2356
2357	/**
2358	* idpf_tso - computes mss and TSO length to prepare for TSO
2359	* @skb: pointer to skb
2360	* @off: pointer to struct that holds offload parameters
2361	*
2362	* Returns error (negative) if TSO was requested but cannot be applied to the
2363	* given skb, 0 if TSO does not apply to the given skb, or 1 otherwise.
2364	*/
2365	int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off)
2366	{
2367	const struct skb_shared_info *shinfo;
2368	union {
2369	struct iphdr *v4;
2370	struct ipv6hdr *v6;
2371	unsigned char *hdr;
2372	} ip;
2373	union {
2374	struct tcphdr *tcp;
2375	struct udphdr *udp;
2376	unsigned char *hdr;
2377	} l4;
2378	u32 paylen, l4_start;
2379	int err;
2380
2381	if (!skb_is_gso(skb))
2382	return 0;
2383
2384	err = skb_cow_head(skb, headroom: 0);
2385	if (err < 0)
2386	return err;
2387
2388	shinfo = skb_shinfo(skb);
2389
2390	ip.hdr = skb_network_header(skb);
2391	l4.hdr = skb_transport_header(skb);
2392
2393	/* initialize outer IP header fields */
2394	if (ip.v4->version == 4) {
2395	ip.v4->tot_len = 0;
2396	ip.v4->check = 0;
2397	} else if (ip.v6->version == 6) {
2398	ip.v6->payload_len = 0;
2399	}
2400
2401	l4_start = skb_transport_offset(skb);
2402
2403	/* remove payload length from checksum */
2404	paylen = skb->len - l4_start;
2405
2406	switch (shinfo->gso_type & ~SKB_GSO_DODGY) {
2407	case SKB_GSO_TCPV4:
2408	case SKB_GSO_TCPV6:
2409	csum_replace_by_diff(sum: &l4.tcp->check,
2410	diff: (__force __wsum)htonl(paylen));
2411	off->tso_hdr_len = __tcp_hdrlen(th: l4.tcp) + l4_start;
2412	break;
2413	case SKB_GSO_UDP_L4:
2414	csum_replace_by_diff(sum: &l4.udp->check,
2415	diff: (__force __wsum)htonl(paylen));
2416	/* compute length of segmentation header */
2417	off->tso_hdr_len = sizeof(struct udphdr) + l4_start;
2418	l4.udp->len = htons(shinfo->gso_size + sizeof(struct udphdr));
2419	break;
2420	default:
2421	return -EINVAL;
2422	}
2423
2424	off->tso_len = skb->len - off->tso_hdr_len;
2425	off->mss = shinfo->gso_size;
2426	off->tso_segs = shinfo->gso_segs;
2427
2428	off->tx_flags |= IDPF_TX_FLAGS_TSO;
2429
2430	return 1;
2431	}
2432
2433	/**
2434	* __idpf_chk_linearize - Check skb is not using too many buffers
2435	* @skb: send buffer
2436	* @max_bufs: maximum number of buffers
2437	*
2438	* For TSO we need to count the TSO header and segment payload separately. As
2439	* such we need to check cases where we have max_bufs-1 fragments or more as we
2440	* can potentially require max_bufs+1 DMA transactions, 1 for the TSO header, 1
2441	* for the segment payload in the first descriptor, and another max_buf-1 for
2442	* the fragments.
2443	*/
2444	static bool __idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs)
2445	{
2446	const struct skb_shared_info *shinfo = skb_shinfo(skb);
2447	const skb_frag_t *frag, *stale;
2448	int nr_frags, sum;
2449
2450	/* no need to check if number of frags is less than max_bufs - 1 */
2451	nr_frags = shinfo->nr_frags;
2452	if (nr_frags < (max_bufs - 1))
2453	return false;
2454
2455	/* We need to walk through the list and validate that each group
2456	* of max_bufs-2 fragments totals at least gso_size.
2457	*/
2458	nr_frags -= max_bufs - 2;
2459	frag = &shinfo->frags[0];
2460
2461	/* Initialize size to the negative value of gso_size minus 1. We use
2462	* this as the worst case scenario in which the frag ahead of us only
2463	* provides one byte which is why we are limited to max_bufs-2
2464	* descriptors for a single transmit as the header and previous
2465	* fragment are already consuming 2 descriptors.
2466	*/
2467	sum = 1 - shinfo->gso_size;
2468
2469	/* Add size of frags 0 through 4 to create our initial sum */
2470	sum += skb_frag_size(frag: frag++);
2471	sum += skb_frag_size(frag: frag++);
2472	sum += skb_frag_size(frag: frag++);
2473	sum += skb_frag_size(frag: frag++);
2474	sum += skb_frag_size(frag: frag++);
2475
2476	/* Walk through fragments adding latest fragment, testing it, and
2477	* then removing stale fragments from the sum.
2478	*/
2479	for (stale = &shinfo->frags[0];; stale++) {
2480	int stale_size = skb_frag_size(frag: stale);
2481
2482	sum += skb_frag_size(frag: frag++);
2483
2484	/* The stale fragment may present us with a smaller
2485	* descriptor than the actual fragment size. To account
2486	* for that we need to remove all the data on the front and
2487	* figure out what the remainder would be in the last
2488	* descriptor associated with the fragment.
2489	*/
2490	if (stale_size > IDPF_TX_MAX_DESC_DATA) {
2491	int align_pad = -(skb_frag_off(frag: stale)) &
2492	(IDPF_TX_MAX_READ_REQ_SIZE - 1);
2493
2494	sum -= align_pad;
2495	stale_size -= align_pad;
2496
2497	do {
2498	sum -= IDPF_TX_MAX_DESC_DATA_ALIGNED;
2499	stale_size -= IDPF_TX_MAX_DESC_DATA_ALIGNED;
2500	} while (stale_size > IDPF_TX_MAX_DESC_DATA);
2501	}
2502
2503	/* if sum is negative we failed to make sufficient progress */
2504	if (sum < 0)
2505	return true;
2506
2507	if (!nr_frags--)
2508	break;
2509
2510	sum -= stale_size;
2511	}
2512
2513	return false;
2514	}
2515
2516	/**
2517	* idpf_chk_linearize - Check if skb exceeds max descriptors per packet
2518	* @skb: send buffer
2519	* @max_bufs: maximum scatter gather buffers for single packet
2520	* @count: number of buffers this packet needs
2521	*
2522	* Make sure we don't exceed maximum scatter gather buffers for a single
2523	* packet. We have to do some special checking around the boundary (max_bufs-1)
2524	* if TSO is on since we need count the TSO header and payload separately.
2525	* E.g.: a packet with 7 fragments can require 9 DMA transactions; 1 for TSO
2526	* header, 1 for segment payload, and then 7 for the fragments.
2527	*/
2528	bool idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs,
2529	unsigned int count)
2530	{
2531	if (likely(count < max_bufs))
2532	return false;
2533	if (skb_is_gso(skb))
2534	return __idpf_chk_linearize(skb, max_bufs);
2535
2536	return count > max_bufs;
2537	}
2538
2539	/**
2540	* idpf_tx_splitq_get_ctx_desc - grab next desc and update buffer ring
2541	* @txq: queue to put context descriptor on
2542	*
2543	* Since the TX buffer rings mimics the descriptor ring, update the tx buffer
2544	* ring entry to reflect that this index is a context descriptor
2545	*/
2546	static struct idpf_flex_tx_ctx_desc *
2547	idpf_tx_splitq_get_ctx_desc(struct idpf_queue *txq)
2548	{
2549	struct idpf_flex_tx_ctx_desc *desc;
2550	int i = txq->next_to_use;
2551
2552	memset(&txq->tx_buf[i], 0, sizeof(struct idpf_tx_buf));
2553	txq->tx_buf[i].compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG;
2554
2555	/* grab the next descriptor */
2556	desc = IDPF_FLEX_TX_CTX_DESC(txq, i);
2557	txq->next_to_use = idpf_tx_splitq_bump_ntu(txq, ntu: i);
2558
2559	return desc;
2560	}
2561
2562	/**
2563	* idpf_tx_drop_skb - free the SKB and bump tail if necessary
2564	* @tx_q: queue to send buffer on
2565	* @skb: pointer to skb
2566	*/
2567	netdev_tx_t idpf_tx_drop_skb(struct idpf_queue *tx_q, struct sk_buff *skb)
2568	{
2569	u64_stats_update_begin(syncp: &tx_q->stats_sync);
2570	u64_stats_inc(p: &tx_q->q_stats.tx.skb_drops);
2571	u64_stats_update_end(syncp: &tx_q->stats_sync);
2572
2573	idpf_tx_buf_hw_update(tx_q, val: tx_q->next_to_use, xmit_more: false);
2574
2575	dev_kfree_skb(skb);
2576
2577	return NETDEV_TX_OK;
2578	}
2579
2580	/**
2581	* idpf_tx_splitq_frame - Sends buffer on Tx ring using flex descriptors
2582	* @skb: send buffer
2583	* @tx_q: queue to send buffer on
2584	*
2585	* Returns NETDEV_TX_OK if sent, else an error code
2586	*/
2587	static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb,
2588	struct idpf_queue *tx_q)
2589	{
2590	struct idpf_tx_splitq_params tx_params = { };
2591	struct idpf_tx_buf *first;
2592	unsigned int count;
2593	int tso;
2594
2595	count = idpf_tx_desc_count_required(txq: tx_q, skb);
2596	if (unlikely(!count))
2597	return idpf_tx_drop_skb(tx_q, skb);
2598
2599	tso = idpf_tso(skb, off: &tx_params.offload);
2600	if (unlikely(tso < 0))
2601	return idpf_tx_drop_skb(tx_q, skb);
2602
2603	/* Check for splitq specific TX resources */
2604	count += (IDPF_TX_DESCS_PER_CACHE_LINE + tso);
2605	if (idpf_tx_maybe_stop_splitq(tx_q, descs_needed: count)) {
2606	idpf_tx_buf_hw_update(tx_q, val: tx_q->next_to_use, xmit_more: false);
2607
2608	return NETDEV_TX_BUSY;
2609	}
2610
2611	if (tso) {
2612	/* If tso is needed, set up context desc */
2613	struct idpf_flex_tx_ctx_desc *ctx_desc =
2614	idpf_tx_splitq_get_ctx_desc(txq: tx_q);
2615
2616	ctx_desc->tso.qw1.cmd_dtype =
2617	cpu_to_le16(IDPF_TX_DESC_DTYPE_FLEX_TSO_CTX |
2618	IDPF_TX_FLEX_CTX_DESC_CMD_TSO);
2619	ctx_desc->tso.qw0.flex_tlen =
2620	cpu_to_le32(tx_params.offload.tso_len &
2621	IDPF_TXD_FLEX_CTX_TLEN_M);
2622	ctx_desc->tso.qw0.mss_rt =
2623	cpu_to_le16(tx_params.offload.mss &
2624	IDPF_TXD_FLEX_CTX_MSS_RT_M);
2625	ctx_desc->tso.qw0.hdr_len = tx_params.offload.tso_hdr_len;
2626
2627	u64_stats_update_begin(syncp: &tx_q->stats_sync);
2628	u64_stats_inc(p: &tx_q->q_stats.tx.lso_pkts);
2629	u64_stats_update_end(syncp: &tx_q->stats_sync);
2630	}
2631
2632	/* record the location of the first descriptor for this packet */
2633	first = &tx_q->tx_buf[tx_q->next_to_use];
2634	first->skb = skb;
2635
2636	if (tso) {
2637	first->gso_segs = tx_params.offload.tso_segs;
2638	first->bytecount = skb->len +
2639	((first->gso_segs - 1) * tx_params.offload.tso_hdr_len);
2640	} else {
2641	first->gso_segs = 1;
2642	first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN);
2643	}
2644
2645	if (test_bit(__IDPF_Q_FLOW_SCH_EN, tx_q->flags)) {
2646	tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_FLOW_SCHE;
2647	tx_params.eop_cmd = IDPF_TXD_FLEX_FLOW_CMD_EOP;
2648	/* Set the RE bit to catch any packets that may have not been
2649	* stashed during RS completion cleaning. MIN_GAP is set to
2650	* MIN_RING size to ensure it will be set at least once each
2651	* time around the ring.
2652	*/
2653	if (!(tx_q->next_to_use % IDPF_TX_SPLITQ_RE_MIN_GAP)) {
2654	tx_params.eop_cmd |= IDPF_TXD_FLEX_FLOW_CMD_RE;
2655	tx_q->txq_grp->num_completions_pending++;
2656	}
2657
2658	if (skb->ip_summed == CHECKSUM_PARTIAL)
2659	tx_params.offload.td_cmd |= IDPF_TXD_FLEX_FLOW_CMD_CS_EN;
2660
2661	} else {
2662	tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2;
2663	tx_params.eop_cmd = IDPF_TXD_LAST_DESC_CMD;
2664
2665	if (skb->ip_summed == CHECKSUM_PARTIAL)
2666	tx_params.offload.td_cmd |= IDPF_TX_FLEX_DESC_CMD_CS_EN;
2667	}
2668
2669	idpf_tx_splitq_map(tx_q, params: &tx_params, first);
2670
2671	return NETDEV_TX_OK;
2672	}
2673
2674	/**
2675	* idpf_tx_splitq_start - Selects the right Tx queue to send buffer
2676	* @skb: send buffer
2677	* @netdev: network interface device structure
2678	*
2679	* Returns NETDEV_TX_OK if sent, else an error code
2680	*/
2681	netdev_tx_t idpf_tx_splitq_start(struct sk_buff *skb,
2682	struct net_device *netdev)
2683	{
2684	struct idpf_vport *vport = idpf_netdev_to_vport(netdev);
2685	struct idpf_queue *tx_q;
2686
2687	if (unlikely(skb_get_queue_mapping(skb) >= vport->num_txq)) {
2688	dev_kfree_skb_any(skb);
2689
2690	return NETDEV_TX_OK;
2691	}
2692
2693	tx_q = vport->txqs[skb_get_queue_mapping(skb)];
2694
2695	/* hardware can't handle really short frames, hardware padding works
2696	* beyond this point
2697	*/
2698	if (skb_put_padto(skb, len: tx_q->tx_min_pkt_len)) {
2699	idpf_tx_buf_hw_update(tx_q, val: tx_q->next_to_use, xmit_more: false);
2700
2701	return NETDEV_TX_OK;
2702	}
2703
2704	return idpf_tx_splitq_frame(skb, tx_q);
2705	}
2706
2707	/**
2708	* idpf_ptype_to_htype - get a hash type
2709	* @decoded: Decoded Rx packet type related fields
2710	*
2711	* Returns appropriate hash type (such as PKT_HASH_TYPE_L2/L3/L4) to be used by
2712	* skb_set_hash based on PTYPE as parsed by HW Rx pipeline and is part of
2713	* Rx desc.
2714	*/
2715	enum pkt_hash_types idpf_ptype_to_htype(const struct idpf_rx_ptype_decoded *decoded)
2716	{
2717	if (!decoded->known)
2718	return PKT_HASH_TYPE_NONE;
2719	if (decoded->payload_layer == IDPF_RX_PTYPE_PAYLOAD_LAYER_PAY2 &&
2720	decoded->inner_prot)
2721	return PKT_HASH_TYPE_L4;
2722	if (decoded->payload_layer == IDPF_RX_PTYPE_PAYLOAD_LAYER_PAY2 &&
2723	decoded->outer_ip)
2724	return PKT_HASH_TYPE_L3;
2725	if (decoded->outer_ip == IDPF_RX_PTYPE_OUTER_L2)
2726	return PKT_HASH_TYPE_L2;
2727
2728	return PKT_HASH_TYPE_NONE;
2729	}
2730
2731	/**
2732	* idpf_rx_hash - set the hash value in the skb
2733	* @rxq: Rx descriptor ring packet is being transacted on
2734	* @skb: pointer to current skb being populated
2735	* @rx_desc: Receive descriptor
2736	* @decoded: Decoded Rx packet type related fields
2737	*/
2738	static void idpf_rx_hash(struct idpf_queue *rxq, struct sk_buff *skb,
2739	struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc,
2740	struct idpf_rx_ptype_decoded *decoded)
2741	{
2742	u32 hash;
2743
2744	if (unlikely(!idpf_is_feature_ena(rxq->vport, NETIF_F_RXHASH)))
2745	return;
2746
2747	hash = le16_to_cpu(rx_desc->hash1) |
2748	(rx_desc->ff2_mirrid_hash2.hash2 << 16) |
2749	(rx_desc->hash3 << 24);
2750
2751	skb_set_hash(skb, hash, type: idpf_ptype_to_htype(decoded));
2752	}
2753
2754	/**
2755	* idpf_rx_csum - Indicate in skb if checksum is good
2756	* @rxq: Rx descriptor ring packet is being transacted on
2757	* @skb: pointer to current skb being populated
2758	* @csum_bits: checksum fields extracted from the descriptor
2759	* @decoded: Decoded Rx packet type related fields
2760	*
2761	* skb->protocol must be set before this function is called
2762	*/
2763	static void idpf_rx_csum(struct idpf_queue *rxq, struct sk_buff *skb,
2764	struct idpf_rx_csum_decoded *csum_bits,
2765	struct idpf_rx_ptype_decoded *decoded)
2766	{
2767	bool ipv4, ipv6;
2768
2769	/* check if Rx checksum is enabled */
2770	if (unlikely(!idpf_is_feature_ena(rxq->vport, NETIF_F_RXCSUM)))
2771	return;
2772
2773	/* check if HW has decoded the packet and checksum */
2774	if (!(csum_bits->l3l4p))
2775	return;
2776
2777	ipv4 = IDPF_RX_PTYPE_TO_IPV(decoded, IDPF_RX_PTYPE_OUTER_IPV4);
2778	ipv6 = IDPF_RX_PTYPE_TO_IPV(decoded, IDPF_RX_PTYPE_OUTER_IPV6);
2779
2780	if (ipv4 && (csum_bits->ipe || csum_bits->eipe))
2781	goto checksum_fail;
2782
2783	if (ipv6 && csum_bits->ipv6exadd)
2784	return;
2785
2786	/* check for L4 errors and handle packets that were not able to be
2787	* checksummed
2788	*/
2789	if (csum_bits->l4e)
2790	goto checksum_fail;
2791
2792	/* Only report checksum unnecessary for ICMP, TCP, UDP, or SCTP */
2793	switch (decoded->inner_prot) {
2794	case IDPF_RX_PTYPE_INNER_PROT_ICMP:
2795	case IDPF_RX_PTYPE_INNER_PROT_TCP:
2796	case IDPF_RX_PTYPE_INNER_PROT_UDP:
2797	if (!csum_bits->raw_csum_inv) {
2798	u16 csum = csum_bits->raw_csum;
2799
2800	skb->csum = csum_unfold(n: (__force __sum16)~swab16(csum));
2801	skb->ip_summed = CHECKSUM_COMPLETE;
2802	} else {
2803	skb->ip_summed = CHECKSUM_UNNECESSARY;
2804	}
2805	break;
2806	case IDPF_RX_PTYPE_INNER_PROT_SCTP:
2807	skb->ip_summed = CHECKSUM_UNNECESSARY;
2808	break;
2809	default:
2810	break;
2811	}
2812
2813	return;
2814
2815	checksum_fail:
2816	u64_stats_update_begin(syncp: &rxq->stats_sync);
2817	u64_stats_inc(p: &rxq->q_stats.rx.hw_csum_err);
2818	u64_stats_update_end(syncp: &rxq->stats_sync);
2819	}
2820
2821	/**
2822	* idpf_rx_splitq_extract_csum_bits - Extract checksum bits from descriptor
2823	* @rx_desc: receive descriptor
2824	* @csum: structure to extract checksum fields
2825	*
2826	**/
2827	static void idpf_rx_splitq_extract_csum_bits(struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc,
2828	struct idpf_rx_csum_decoded *csum)
2829	{
2830	u8 qword0, qword1;
2831
2832	qword0 = rx_desc->status_err0_qw0;
2833	qword1 = rx_desc->status_err0_qw1;
2834
2835	csum->ipe = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_IPE_M,
2836	qword1);
2837	csum->eipe = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_EIPE_M,
2838	qword1);
2839	csum->l4e = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_L4E_M,
2840	qword1);
2841	csum->l3l4p = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_L3L4P_M,
2842	qword1);
2843	csum->ipv6exadd = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_IPV6EXADD_M,
2844	qword0);
2845	csum->raw_csum_inv =
2846	le16_get_bits(v: rx_desc->ptype_err_fflags0,
2847	VIRTCHNL2_RX_FLEX_DESC_ADV_RAW_CSUM_INV_M);
2848	csum->raw_csum = le16_to_cpu(rx_desc->misc.raw_cs);
2849	}
2850
2851	/**
2852	* idpf_rx_rsc - Set the RSC fields in the skb
2853	* @rxq : Rx descriptor ring packet is being transacted on
2854	* @skb : pointer to current skb being populated
2855	* @rx_desc: Receive descriptor
2856	* @decoded: Decoded Rx packet type related fields
2857	*
2858	* Return 0 on success and error code on failure
2859	*
2860	* Populate the skb fields with the total number of RSC segments, RSC payload
2861	* length and packet type.
2862	*/
2863	static int idpf_rx_rsc(struct idpf_queue *rxq, struct sk_buff *skb,
2864	struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc,
2865	struct idpf_rx_ptype_decoded *decoded)
2866	{
2867	u16 rsc_segments, rsc_seg_len;
2868	bool ipv4, ipv6;
2869	int len;
2870
2871	if (unlikely(!decoded->outer_ip))
2872	return -EINVAL;
2873
2874	rsc_seg_len = le16_to_cpu(rx_desc->misc.rscseglen);
2875	if (unlikely(!rsc_seg_len))
2876	return -EINVAL;
2877
2878	ipv4 = IDPF_RX_PTYPE_TO_IPV(decoded, IDPF_RX_PTYPE_OUTER_IPV4);
2879	ipv6 = IDPF_RX_PTYPE_TO_IPV(decoded, IDPF_RX_PTYPE_OUTER_IPV6);
2880
2881	if (unlikely(!(ipv4 ^ ipv6)))
2882	return -EINVAL;
2883
2884	rsc_segments = DIV_ROUND_UP(skb->data_len, rsc_seg_len);
2885	if (unlikely(rsc_segments == 1))
2886	return 0;
2887
2888	NAPI_GRO_CB(skb)->count = rsc_segments;
2889	skb_shinfo(skb)->gso_size = rsc_seg_len;
2890
2891	skb_reset_network_header(skb);
2892	len = skb->len - skb_transport_offset(skb);
2893
2894	if (ipv4) {
2895	struct iphdr *ipv4h = ip_hdr(skb);
2896
2897	skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
2898
2899	/* Reset and set transport header offset in skb */
2900	skb_set_transport_header(skb, offset: sizeof(struct iphdr));
2901
2902	/* Compute the TCP pseudo header checksum*/
2903	tcp_hdr(skb)->check =
2904	~tcp_v4_check(len, saddr: ipv4h->saddr, daddr: ipv4h->daddr, base: 0);
2905	} else {
2906	struct ipv6hdr *ipv6h = ipv6_hdr(skb);
2907
2908	skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
2909	skb_set_transport_header(skb, offset: sizeof(struct ipv6hdr));
2910	tcp_hdr(skb)->check =
2911	~tcp_v6_check(len, saddr: &ipv6h->saddr, daddr: &ipv6h->daddr, base: 0);
2912	}
2913
2914	tcp_gro_complete(skb);
2915
2916	u64_stats_update_begin(syncp: &rxq->stats_sync);
2917	u64_stats_inc(p: &rxq->q_stats.rx.rsc_pkts);
2918	u64_stats_update_end(syncp: &rxq->stats_sync);
2919
2920	return 0;
2921	}
2922
2923	/**
2924	* idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor
2925	* @rxq: Rx descriptor ring packet is being transacted on
2926	* @skb: pointer to current skb being populated
2927	* @rx_desc: Receive descriptor
2928	*
2929	* This function checks the ring, descriptor, and packet information in
2930	* order to populate the hash, checksum, protocol, and
2931	* other fields within the skb.
2932	*/
2933	static int idpf_rx_process_skb_fields(struct idpf_queue *rxq,
2934	struct sk_buff *skb,
2935	struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc)
2936	{
2937	struct idpf_rx_csum_decoded csum_bits = { };
2938	struct idpf_rx_ptype_decoded decoded;
2939	u16 rx_ptype;
2940
2941	rx_ptype = le16_get_bits(v: rx_desc->ptype_err_fflags0,
2942	VIRTCHNL2_RX_FLEX_DESC_ADV_PTYPE_M);
2943
2944	skb->protocol = eth_type_trans(skb, dev: rxq->vport->netdev);
2945
2946	decoded = rxq->vport->rx_ptype_lkup[rx_ptype];
2947	/* If we don't know the ptype we can't do anything else with it. Just
2948	* pass it up the stack as-is.
2949	*/
2950	if (!decoded.known)
2951	return 0;
2952
2953	/* process RSS/hash */
2954	idpf_rx_hash(rxq, skb, rx_desc, decoded: &decoded);
2955
2956	if (le16_get_bits(v: rx_desc->hdrlen_flags,
2957	VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_M))
2958	return idpf_rx_rsc(rxq, skb, rx_desc, decoded: &decoded);
2959
2960	idpf_rx_splitq_extract_csum_bits(rx_desc, csum: &csum_bits);
2961	idpf_rx_csum(rxq, skb, csum_bits: &csum_bits, decoded: &decoded);
2962
2963	return 0;
2964	}
2965
2966	/**
2967	* idpf_rx_add_frag - Add contents of Rx buffer to sk_buff as a frag
2968	* @rx_buf: buffer containing page to add
2969	* @skb: sk_buff to place the data into
2970	* @size: packet length from rx_desc
2971	*
2972	* This function will add the data contained in rx_buf->page to the skb.
2973	* It will just attach the page as a frag to the skb.
2974	* The function will then update the page offset.
2975	*/
2976	void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb,
2977	unsigned int size)
2978	{
2979	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page: rx_buf->page,
2980	off: rx_buf->page_offset, size, truesize: rx_buf->truesize);
2981
2982	rx_buf->page = NULL;
2983	}
2984
2985	/**
2986	* idpf_rx_construct_skb - Allocate skb and populate it
2987	* @rxq: Rx descriptor queue
2988	* @rx_buf: Rx buffer to pull data from
2989	* @size: the length of the packet
2990	*
2991	* This function allocates an skb. It then populates it with the page
2992	* data from the current receive descriptor, taking care to set up the
2993	* skb correctly.
2994	*/
2995	struct sk_buff *idpf_rx_construct_skb(struct idpf_queue *rxq,
2996	struct idpf_rx_buf *rx_buf,
2997	unsigned int size)
2998	{
2999	unsigned int headlen;
3000	struct sk_buff *skb;
3001	void *va;
3002
3003	va = page_address(rx_buf->page) + rx_buf->page_offset;
3004
3005	/* prefetch first cache line of first page */
3006	net_prefetch(p: va);
3007	/* allocate a skb to store the frags */
3008	skb = __napi_alloc_skb(napi: &rxq->q_vector->napi, IDPF_RX_HDR_SIZE,
3009	GFP_ATOMIC);
3010	if (unlikely(!skb)) {
3011	idpf_rx_put_page(rx_buf);
3012
3013	return NULL;
3014	}
3015
3016	skb_record_rx_queue(skb, rx_queue: rxq->idx);
3017	skb_mark_for_recycle(skb);
3018
3019	/* Determine available headroom for copy */
3020	headlen = size;
3021	if (headlen > IDPF_RX_HDR_SIZE)
3022	headlen = eth_get_headlen(dev: skb->dev, data: va, IDPF_RX_HDR_SIZE);
3023
3024	/* align pull length to size of long to optimize memcpy performance */
3025	memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
3026
3027	/* if we exhaust the linear part then add what is left as a frag */
3028	size -= headlen;
3029	if (!size) {
3030	idpf_rx_put_page(rx_buf);
3031
3032	return skb;
3033	}
3034
3035	skb_add_rx_frag(skb, i: 0, page: rx_buf->page, off: rx_buf->page_offset + headlen,
3036	size, truesize: rx_buf->truesize);
3037
3038	/* Since we're giving the page to the stack, clear our reference to it.
3039	* We'll get a new one during buffer posting.
3040	*/
3041	rx_buf->page = NULL;
3042
3043	return skb;
3044	}
3045
3046	/**
3047	* idpf_rx_hdr_construct_skb - Allocate skb and populate it from header buffer
3048	* @rxq: Rx descriptor queue
3049	* @va: Rx buffer to pull data from
3050	* @size: the length of the packet
3051	*
3052	* This function allocates an skb. It then populates it with the page data from
3053	* the current receive descriptor, taking care to set up the skb correctly.
3054	* This specifically uses a header buffer to start building the skb.
3055	*/
3056	static struct sk_buff *idpf_rx_hdr_construct_skb(struct idpf_queue *rxq,
3057	const void *va,
3058	unsigned int size)
3059	{
3060	struct sk_buff *skb;
3061
3062	/* allocate a skb to store the frags */
3063	skb = __napi_alloc_skb(napi: &rxq->q_vector->napi, length: size, GFP_ATOMIC);
3064	if (unlikely(!skb))
3065	return NULL;
3066
3067	skb_record_rx_queue(skb, rx_queue: rxq->idx);
3068
3069	memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
3070
3071	/* More than likely, a payload fragment, which will use a page from
3072	* page_pool will be added to the SKB so mark it for recycle
3073	* preemptively. And if not, it's inconsequential.
3074	*/
3075	skb_mark_for_recycle(skb);
3076
3077	return skb;
3078	}
3079
3080	/**
3081	* idpf_rx_splitq_test_staterr - tests bits in Rx descriptor
3082	* status and error fields
3083	* @stat_err_field: field from descriptor to test bits in
3084	* @stat_err_bits: value to mask
3085	*
3086	*/
3087	static bool idpf_rx_splitq_test_staterr(const u8 stat_err_field,
3088	const u8 stat_err_bits)
3089	{
3090	return !!(stat_err_field & stat_err_bits);
3091	}
3092
3093	/**
3094	* idpf_rx_splitq_is_eop - process handling of EOP buffers
3095	* @rx_desc: Rx descriptor for current buffer
3096	*
3097	* If the buffer is an EOP buffer, this function exits returning true,
3098	* otherwise return false indicating that this is in fact a non-EOP buffer.
3099	*/
3100	static bool idpf_rx_splitq_is_eop(struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc)
3101	{
3102	/* if we are the last buffer then there is nothing else to do */
3103	return likely(idpf_rx_splitq_test_staterr(rx_desc->status_err0_qw1,
3104	IDPF_RXD_EOF_SPLITQ));
3105	}
3106
3107	/**
3108	* idpf_rx_splitq_clean - Clean completed descriptors from Rx queue
3109	* @rxq: Rx descriptor queue to retrieve receive buffer queue
3110	* @budget: Total limit on number of packets to process
3111	*
3112	* This function provides a "bounce buffer" approach to Rx interrupt
3113	* processing. The advantage to this is that on systems that have
3114	* expensive overhead for IOMMU access this provides a means of avoiding
3115	* it by maintaining the mapping of the page to the system.
3116	*
3117	* Returns amount of work completed
3118	*/
3119	static int idpf_rx_splitq_clean(struct idpf_queue *rxq, int budget)
3120	{
3121	int total_rx_bytes = 0, total_rx_pkts = 0;
3122	struct idpf_queue *rx_bufq = NULL;
3123	struct sk_buff *skb = rxq->skb;
3124	u16 ntc = rxq->next_to_clean;
3125
3126	/* Process Rx packets bounded by budget */
3127	while (likely(total_rx_pkts < budget)) {
3128	struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc;
3129	struct idpf_sw_queue *refillq = NULL;
3130	struct idpf_rxq_set *rxq_set = NULL;
3131	struct idpf_rx_buf *rx_buf = NULL;
3132	union virtchnl2_rx_desc *desc;
3133	unsigned int pkt_len = 0;
3134	unsigned int hdr_len = 0;
3135	u16 gen_id, buf_id = 0;
3136	/* Header buffer overflow only valid for header split */
3137	bool hbo = false;
3138	int bufq_id;
3139	u8 rxdid;
3140
3141	/* get the Rx desc from Rx queue based on 'next_to_clean' */
3142	desc = IDPF_RX_DESC(rxq, ntc);
3143	rx_desc = (struct virtchnl2_rx_flex_desc_adv_nic_3 *)desc;
3144
3145	/* This memory barrier is needed to keep us from reading
3146	* any other fields out of the rx_desc
3147	*/
3148	dma_rmb();
3149
3150	/* if the descriptor isn't done, no work yet to do */
3151	gen_id = le16_get_bits(v: rx_desc->pktlen_gen_bufq_id,
3152	VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_M);
3153
3154	if (test_bit(__IDPF_Q_GEN_CHK, rxq->flags) != gen_id)
3155	break;
3156
3157	rxdid = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_RXDID_M,
3158	rx_desc->rxdid_ucast);
3159	if (rxdid != VIRTCHNL2_RXDID_2_FLEX_SPLITQ) {
3160	IDPF_RX_BUMP_NTC(rxq, ntc);
3161	u64_stats_update_begin(syncp: &rxq->stats_sync);
3162	u64_stats_inc(p: &rxq->q_stats.rx.bad_descs);
3163	u64_stats_update_end(syncp: &rxq->stats_sync);
3164	continue;
3165	}
3166
3167	pkt_len = le16_get_bits(v: rx_desc->pktlen_gen_bufq_id,
3168	VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_PBUF_M);
3169
3170	hbo = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_HBO_M,
3171	rx_desc->status_err0_qw1);
3172
3173	if (unlikely(hbo)) {
3174	/* If a header buffer overflow, occurs, i.e. header is
3175	* too large to fit in the header split buffer, HW will
3176	* put the entire packet, including headers, in the
3177	* data/payload buffer.
3178	*/
3179	u64_stats_update_begin(syncp: &rxq->stats_sync);
3180	u64_stats_inc(p: &rxq->q_stats.rx.hsplit_buf_ovf);
3181	u64_stats_update_end(syncp: &rxq->stats_sync);
3182	goto bypass_hsplit;
3183	}
3184
3185	hdr_len = le16_get_bits(v: rx_desc->hdrlen_flags,
3186	VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_HDR_M);
3187
3188	bypass_hsplit:
3189	bufq_id = le16_get_bits(v: rx_desc->pktlen_gen_bufq_id,
3190	VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_M);
3191
3192	rxq_set = container_of(rxq, struct idpf_rxq_set, rxq);
3193	if (!bufq_id)
3194	refillq = rxq_set->refillq0;
3195	else
3196	refillq = rxq_set->refillq1;
3197
3198	/* retrieve buffer from the rxq */
3199	rx_bufq = &rxq->rxq_grp->splitq.bufq_sets[bufq_id].bufq;
3200
3201	buf_id = le16_to_cpu(rx_desc->buf_id);
3202
3203	rx_buf = &rx_bufq->rx_buf.buf[buf_id];
3204
3205	if (hdr_len) {
3206	const void *va = (u8 *)rx_bufq->rx_buf.hdr_buf_va +
3207	(u32)buf_id * IDPF_HDR_BUF_SIZE;
3208
3209	skb = idpf_rx_hdr_construct_skb(rxq, va, size: hdr_len);
3210	u64_stats_update_begin(syncp: &rxq->stats_sync);
3211	u64_stats_inc(p: &rxq->q_stats.rx.hsplit_pkts);
3212	u64_stats_update_end(syncp: &rxq->stats_sync);
3213	}
3214
3215	if (pkt_len) {
3216	idpf_rx_sync_for_cpu(rx_buf, len: pkt_len);
3217	if (skb)
3218	idpf_rx_add_frag(rx_buf, skb, size: pkt_len);
3219	else
3220	skb = idpf_rx_construct_skb(rxq, rx_buf,
3221	size: pkt_len);
3222	} else {
3223	idpf_rx_put_page(rx_buf);
3224	}
3225
3226	/* exit if we failed to retrieve a buffer */
3227	if (!skb)
3228	break;
3229
3230	idpf_rx_post_buf_refill(refillq, buf_id);
3231
3232	IDPF_RX_BUMP_NTC(rxq, ntc);
3233	/* skip if it is non EOP desc */
3234	if (!idpf_rx_splitq_is_eop(rx_desc))
3235	continue;
3236
3237	/* pad skb if needed (to make valid ethernet frame) */
3238	if (eth_skb_pad(skb)) {
3239	skb = NULL;
3240	continue;
3241	}
3242
3243	/* probably a little skewed due to removing CRC */
3244	total_rx_bytes += skb->len;
3245
3246	/* protocol */
3247	if (unlikely(idpf_rx_process_skb_fields(rxq, skb, rx_desc))) {
3248	dev_kfree_skb_any(skb);
3249	skb = NULL;
3250	continue;
3251	}
3252
3253	/* send completed skb up the stack */
3254	napi_gro_receive(napi: &rxq->q_vector->napi, skb);
3255	skb = NULL;
3256
3257	/* update budget accounting */
3258	total_rx_pkts++;
3259	}
3260
3261	rxq->next_to_clean = ntc;
3262
3263	rxq->skb = skb;
3264	u64_stats_update_begin(syncp: &rxq->stats_sync);
3265	u64_stats_add(p: &rxq->q_stats.rx.packets, val: total_rx_pkts);
3266	u64_stats_add(p: &rxq->q_stats.rx.bytes, val: total_rx_bytes);
3267	u64_stats_update_end(syncp: &rxq->stats_sync);
3268
3269	/* guarantee a trip back through this routine if there was a failure */
3270	return total_rx_pkts;
3271	}
3272
3273	/**
3274	* idpf_rx_update_bufq_desc - Update buffer queue descriptor
3275	* @bufq: Pointer to the buffer queue
3276	* @refill_desc: SW Refill queue descriptor containing buffer ID
3277	* @buf_desc: Buffer queue descriptor
3278	*
3279	* Return 0 on success and negative on failure.
3280	*/
3281	static int idpf_rx_update_bufq_desc(struct idpf_queue *bufq, u16 refill_desc,
3282	struct virtchnl2_splitq_rx_buf_desc *buf_desc)
3283	{
3284	struct idpf_rx_buf *buf;
3285	dma_addr_t addr;
3286	u16 buf_id;
3287
3288	buf_id = FIELD_GET(IDPF_RX_BI_BUFID_M, refill_desc);
3289
3290	buf = &bufq->rx_buf.buf[buf_id];
3291
3292	addr = idpf_alloc_page(pool: bufq->pp, buf, buf_size: bufq->rx_buf_size);
3293	if (unlikely(addr == DMA_MAPPING_ERROR))
3294	return -ENOMEM;
3295
3296	buf_desc->pkt_addr = cpu_to_le64(addr);
3297	buf_desc->qword0.buf_id = cpu_to_le16(buf_id);
3298
3299	if (!bufq->rx_hsplit_en)
3300	return 0;
3301
3302	buf_desc->hdr_addr = cpu_to_le64(bufq->rx_buf.hdr_buf_pa +
3303	(u32)buf_id * IDPF_HDR_BUF_SIZE);
3304
3305	return 0;
3306	}
3307
3308	/**
3309	* idpf_rx_clean_refillq - Clean refill queue buffers
3310	* @bufq: buffer queue to post buffers back to
3311	* @refillq: refill queue to clean
3312	*
3313	* This function takes care of the buffer refill management
3314	*/
3315	static void idpf_rx_clean_refillq(struct idpf_queue *bufq,
3316	struct idpf_sw_queue *refillq)
3317	{
3318	struct virtchnl2_splitq_rx_buf_desc *buf_desc;
3319	u16 bufq_nta = bufq->next_to_alloc;
3320	u16 ntc = refillq->next_to_clean;
3321	int cleaned = 0;
3322	u16 gen;
3323
3324	buf_desc = IDPF_SPLITQ_RX_BUF_DESC(bufq, bufq_nta);
3325
3326	/* make sure we stop at ring wrap in the unlikely case ring is full */
3327	while (likely(cleaned < refillq->desc_count)) {
3328	u16 refill_desc = IDPF_SPLITQ_RX_BI_DESC(refillq, ntc);
3329	bool failure;
3330
3331	gen = FIELD_GET(IDPF_RX_BI_GEN_M, refill_desc);
3332	if (test_bit(__IDPF_RFLQ_GEN_CHK, refillq->flags) != gen)
3333	break;
3334
3335	failure = idpf_rx_update_bufq_desc(bufq, refill_desc,
3336	buf_desc);
3337	if (failure)
3338	break;
3339
3340	if (unlikely(++ntc == refillq->desc_count)) {
3341	change_bit(nr: __IDPF_RFLQ_GEN_CHK, addr: refillq->flags);
3342	ntc = 0;
3343	}
3344
3345	if (unlikely(++bufq_nta == bufq->desc_count)) {
3346	buf_desc = IDPF_SPLITQ_RX_BUF_DESC(bufq, 0);
3347	bufq_nta = 0;
3348	} else {
3349	buf_desc++;
3350	}
3351
3352	cleaned++;
3353	}
3354
3355	if (!cleaned)
3356	return;
3357
3358	/* We want to limit how many transactions on the bus we trigger with
3359	* tail writes so we only do it in strides. It's also important we
3360	* align the write to a multiple of 8 as required by HW.
3361	*/
3362	if (((bufq->next_to_use <= bufq_nta ? 0 : bufq->desc_count) +
3363	bufq_nta - bufq->next_to_use) >= IDPF_RX_BUF_POST_STRIDE)
3364	idpf_rx_buf_hw_update(rxq: bufq, ALIGN_DOWN(bufq_nta,
3365	IDPF_RX_BUF_POST_STRIDE));
3366
3367	/* update next to alloc since we have filled the ring */
3368	refillq->next_to_clean = ntc;
3369	bufq->next_to_alloc = bufq_nta;
3370	}
3371
3372	/**
3373	* idpf_rx_clean_refillq_all - Clean all refill queues
3374	* @bufq: buffer queue with refill queues
3375	*
3376	* Iterates through all refill queues assigned to the buffer queue assigned to
3377	* this vector. Returns true if clean is complete within budget, false
3378	* otherwise.
3379	*/
3380	static void idpf_rx_clean_refillq_all(struct idpf_queue *bufq)
3381	{
3382	struct idpf_bufq_set *bufq_set;
3383	int i;
3384
3385	bufq_set = container_of(bufq, struct idpf_bufq_set, bufq);
3386	for (i = 0; i < bufq_set->num_refillqs; i++)
3387	idpf_rx_clean_refillq(bufq, refillq: &bufq_set->refillqs[i]);
3388	}
3389
3390	/**
3391	* idpf_vport_intr_clean_queues - MSIX mode Interrupt Handler
3392	* @irq: interrupt number
3393	* @data: pointer to a q_vector
3394	*
3395	*/
3396	static irqreturn_t idpf_vport_intr_clean_queues(int __always_unused irq,
3397	void *data)
3398	{
3399	struct idpf_q_vector *q_vector = (struct idpf_q_vector *)data;
3400
3401	q_vector->total_events++;
3402	napi_schedule(n: &q_vector->napi);
3403
3404	return IRQ_HANDLED;
3405	}
3406
3407	/**
3408	* idpf_vport_intr_napi_del_all - Unregister napi for all q_vectors in vport
3409	* @vport: virtual port structure
3410	*
3411	*/
3412	static void idpf_vport_intr_napi_del_all(struct idpf_vport *vport)
3413	{
3414	u16 v_idx;
3415
3416	for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++)
3417	netif_napi_del(napi: &vport->q_vectors[v_idx].napi);
3418	}
3419
3420	/**
3421	* idpf_vport_intr_napi_dis_all - Disable NAPI for all q_vectors in the vport
3422	* @vport: main vport structure
3423	*/
3424	static void idpf_vport_intr_napi_dis_all(struct idpf_vport *vport)
3425	{
3426	int v_idx;
3427
3428	for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++)
3429	napi_disable(n: &vport->q_vectors[v_idx].napi);
3430	}
3431
3432	/**
3433	* idpf_vport_intr_rel - Free memory allocated for interrupt vectors
3434	* @vport: virtual port
3435	*
3436	* Free the memory allocated for interrupt vectors associated to a vport
3437	*/
3438	void idpf_vport_intr_rel(struct idpf_vport *vport)
3439	{
3440	int i, j, v_idx;
3441
3442	for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) {
3443	struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx];
3444
3445	kfree(objp: q_vector->bufq);
3446	q_vector->bufq = NULL;
3447	kfree(objp: q_vector->tx);
3448	q_vector->tx = NULL;
3449	kfree(objp: q_vector->rx);
3450	q_vector->rx = NULL;
3451	}
3452
3453	/* Clean up the mapping of queues to vectors */
3454	for (i = 0; i < vport->num_rxq_grp; i++) {
3455	struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i];
3456
3457	if (idpf_is_queue_model_split(q_model: vport->rxq_model))
3458	for (j = 0; j < rx_qgrp->splitq.num_rxq_sets; j++)
3459	rx_qgrp->splitq.rxq_sets[j]->rxq.q_vector = NULL;
3460	else
3461	for (j = 0; j < rx_qgrp->singleq.num_rxq; j++)
3462	rx_qgrp->singleq.rxqs[j]->q_vector = NULL;
3463	}
3464
3465	if (idpf_is_queue_model_split(q_model: vport->txq_model))
3466	for (i = 0; i < vport->num_txq_grp; i++)
3467	vport->txq_grps[i].complq->q_vector = NULL;
3468	else
3469	for (i = 0; i < vport->num_txq_grp; i++)
3470	for (j = 0; j < vport->txq_grps[i].num_txq; j++)
3471	vport->txq_grps[i].txqs[j]->q_vector = NULL;
3472
3473	kfree(objp: vport->q_vectors);
3474	vport->q_vectors = NULL;
3475	}
3476
3477	/**
3478	* idpf_vport_intr_rel_irq - Free the IRQ association with the OS
3479	* @vport: main vport structure
3480	*/
3481	static void idpf_vport_intr_rel_irq(struct idpf_vport *vport)
3482	{
3483	struct idpf_adapter *adapter = vport->adapter;
3484	int vector;
3485
3486	for (vector = 0; vector < vport->num_q_vectors; vector++) {
3487	struct idpf_q_vector *q_vector = &vport->q_vectors[vector];
3488	int irq_num, vidx;
3489
3490	/* free only the irqs that were actually requested */
3491	if (!q_vector)
3492	continue;
3493
3494	vidx = vport->q_vector_idxs[vector];
3495	irq_num = adapter->msix_entries[vidx].vector;
3496
3497	/* clear the affinity_mask in the IRQ descriptor */
3498	irq_set_affinity_hint(irq: irq_num, NULL);
3499	free_irq(irq_num, q_vector);
3500	}
3501	}
3502
3503	/**
3504	* idpf_vport_intr_dis_irq_all - Disable all interrupt
3505	* @vport: main vport structure
3506	*/
3507	static void idpf_vport_intr_dis_irq_all(struct idpf_vport *vport)
3508	{
3509	struct idpf_q_vector *q_vector = vport->q_vectors;
3510	int q_idx;
3511
3512	for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++)
3513	writel(val: 0, addr: q_vector[q_idx].intr_reg.dyn_ctl);
3514	}
3515
3516	/**
3517	* idpf_vport_intr_buildreg_itr - Enable default interrupt generation settings
3518	* @q_vector: pointer to q_vector
3519	* @type: itr index
3520	* @itr: itr value
3521	*/
3522	static u32 idpf_vport_intr_buildreg_itr(struct idpf_q_vector *q_vector,
3523	const int type, u16 itr)
3524	{
3525	u32 itr_val;
3526
3527	itr &= IDPF_ITR_MASK;
3528	/* Don't clear PBA because that can cause lost interrupts that
3529	* came in while we were cleaning/polling
3530	*/
3531	itr_val = q_vector->intr_reg.dyn_ctl_intena_m |
3532	(type << q_vector->intr_reg.dyn_ctl_itridx_s) |
3533	(itr << (q_vector->intr_reg.dyn_ctl_intrvl_s - 1));
3534
3535	return itr_val;
3536	}
3537
3538	/**
3539	* idpf_update_dim_sample - Update dim sample with packets and bytes
3540	* @q_vector: the vector associated with the interrupt
3541	* @dim_sample: dim sample to update
3542	* @dim: dim instance structure
3543	* @packets: total packets
3544	* @bytes: total bytes
3545	*
3546	* Update the dim sample with the packets and bytes which are passed to this
3547	* function. Set the dim state appropriately if the dim settings gets stale.
3548	*/
3549	static void idpf_update_dim_sample(struct idpf_q_vector *q_vector,
3550	struct dim_sample *dim_sample,
3551	struct dim *dim, u64 packets, u64 bytes)
3552	{
3553	dim_update_sample(event_ctr: q_vector->total_events, packets, bytes, s: dim_sample);
3554	dim_sample->comp_ctr = 0;
3555
3556	/* if dim settings get stale, like when not updated for 1 second or
3557	* longer, force it to start again. This addresses the frequent case
3558	* of an idle queue being switched to by the scheduler.
3559	*/
3560	if (ktime_ms_delta(later: dim_sample->time, earlier: dim->start_sample.time) >= HZ)
3561	dim->state = DIM_START_MEASURE;
3562	}
3563
3564	/**
3565	* idpf_net_dim - Update net DIM algorithm
3566	* @q_vector: the vector associated with the interrupt
3567	*
3568	* Create a DIM sample and notify net_dim() so that it can possibly decide
3569	* a new ITR value based on incoming packets, bytes, and interrupts.
3570	*
3571	* This function is a no-op if the queue is not configured to dynamic ITR.
3572	*/
3573	static void idpf_net_dim(struct idpf_q_vector *q_vector)
3574	{
3575	struct dim_sample dim_sample = { };
3576	u64 packets, bytes;
3577	u32 i;
3578
3579	if (!IDPF_ITR_IS_DYNAMIC(q_vector->tx_intr_mode))
3580	goto check_rx_itr;
3581
3582	for (i = 0, packets = 0, bytes = 0; i < q_vector->num_txq; i++) {
3583	struct idpf_queue *txq = q_vector->tx[i];
3584	unsigned int start;
3585
3586	do {
3587	start = u64_stats_fetch_begin(syncp: &txq->stats_sync);
3588	packets += u64_stats_read(p: &txq->q_stats.tx.packets);
3589	bytes += u64_stats_read(p: &txq->q_stats.tx.bytes);
3590	} while (u64_stats_fetch_retry(syncp: &txq->stats_sync, start));
3591	}
3592
3593	idpf_update_dim_sample(q_vector, dim_sample: &dim_sample, dim: &q_vector->tx_dim,
3594	packets, bytes);
3595	net_dim(dim: &q_vector->tx_dim, end_sample: dim_sample);
3596
3597	check_rx_itr:
3598	if (!IDPF_ITR_IS_DYNAMIC(q_vector->rx_intr_mode))
3599	return;
3600
3601	for (i = 0, packets = 0, bytes = 0; i < q_vector->num_rxq; i++) {
3602	struct idpf_queue *rxq = q_vector->rx[i];
3603	unsigned int start;
3604
3605	do {
3606	start = u64_stats_fetch_begin(syncp: &rxq->stats_sync);
3607	packets += u64_stats_read(p: &rxq->q_stats.rx.packets);
3608	bytes += u64_stats_read(p: &rxq->q_stats.rx.bytes);
3609	} while (u64_stats_fetch_retry(syncp: &rxq->stats_sync, start));
3610	}
3611
3612	idpf_update_dim_sample(q_vector, dim_sample: &dim_sample, dim: &q_vector->rx_dim,
3613	packets, bytes);
3614	net_dim(dim: &q_vector->rx_dim, end_sample: dim_sample);
3615	}
3616
3617	/**
3618	* idpf_vport_intr_update_itr_ena_irq - Update itr and re-enable MSIX interrupt
3619	* @q_vector: q_vector for which itr is being updated and interrupt enabled
3620	*
3621	* Update the net_dim() algorithm and re-enable the interrupt associated with
3622	* this vector.
3623	*/
3624	void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector)
3625	{
3626	u32 intval;
3627
3628	/* net_dim() updates ITR out-of-band using a work item */
3629	idpf_net_dim(q_vector);
3630
3631	intval = idpf_vport_intr_buildreg_itr(q_vector,
3632	IDPF_NO_ITR_UPDATE_IDX, itr: 0);
3633
3634	writel(val: intval, addr: q_vector->intr_reg.dyn_ctl);
3635	}
3636
3637	/**
3638	* idpf_vport_intr_req_irq - get MSI-X vectors from the OS for the vport
3639	* @vport: main vport structure
3640	* @basename: name for the vector
3641	*/
3642	static int idpf_vport_intr_req_irq(struct idpf_vport *vport, char *basename)
3643	{
3644	struct idpf_adapter *adapter = vport->adapter;
3645	int vector, err, irq_num, vidx;
3646	const char *vec_name;
3647
3648	for (vector = 0; vector < vport->num_q_vectors; vector++) {
3649	struct idpf_q_vector *q_vector = &vport->q_vectors[vector];
3650
3651	vidx = vport->q_vector_idxs[vector];
3652	irq_num = adapter->msix_entries[vidx].vector;
3653
3654	if (q_vector->num_rxq && q_vector->num_txq)
3655	vec_name = "TxRx";
3656	else if (q_vector->num_rxq)
3657	vec_name = "Rx";
3658	else if (q_vector->num_txq)
3659	vec_name = "Tx";
3660	else
3661	continue;
3662
3663	q_vector->name = kasprintf(GFP_KERNEL, fmt: "%s-%s-%d",
3664	basename, vec_name, vidx);
3665
3666	err = request_irq(irq: irq_num, handler: idpf_vport_intr_clean_queues, flags: 0,
3667	name: q_vector->name, dev: q_vector);
3668	if (err) {
3669	netdev_err(dev: vport->netdev,
3670	format: "Request_irq failed, error: %d\n", err);
3671	goto free_q_irqs;
3672	}
3673	/* assign the mask for this irq */
3674	irq_set_affinity_hint(irq: irq_num, m: &q_vector->affinity_mask);
3675	}
3676
3677	return 0;
3678
3679	free_q_irqs:
3680	while (--vector >= 0) {
3681	vidx = vport->q_vector_idxs[vector];
3682	irq_num = adapter->msix_entries[vidx].vector;
3683	free_irq(irq_num, &vport->q_vectors[vector]);
3684	}
3685
3686	return err;
3687	}
3688
3689	/**
3690	* idpf_vport_intr_write_itr - Write ITR value to the ITR register
3691	* @q_vector: q_vector structure
3692	* @itr: Interrupt throttling rate
3693	* @tx: Tx or Rx ITR
3694	*/
3695	void idpf_vport_intr_write_itr(struct idpf_q_vector *q_vector, u16 itr, bool tx)
3696	{
3697	struct idpf_intr_reg *intr_reg;
3698
3699	if (tx && !q_vector->tx)
3700	return;
3701	else if (!tx && !q_vector->rx)
3702	return;
3703
3704	intr_reg = &q_vector->intr_reg;
3705	writel(ITR_REG_ALIGN(itr) >> IDPF_ITR_GRAN_S,
3706	addr: tx ? intr_reg->tx_itr : intr_reg->rx_itr);
3707	}
3708
3709	/**
3710	* idpf_vport_intr_ena_irq_all - Enable IRQ for the given vport
3711	* @vport: main vport structure
3712	*/
3713	static void idpf_vport_intr_ena_irq_all(struct idpf_vport *vport)
3714	{
3715	bool dynamic;
3716	int q_idx;
3717	u16 itr;
3718
3719	for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) {
3720	struct idpf_q_vector *qv = &vport->q_vectors[q_idx];
3721
3722	/* Set the initial ITR values */
3723	if (qv->num_txq) {
3724	dynamic = IDPF_ITR_IS_DYNAMIC(qv->tx_intr_mode);
3725	itr = vport->tx_itr_profile[qv->tx_dim.profile_ix];
3726	idpf_vport_intr_write_itr(q_vector: qv, itr: dynamic ?
3727	itr : qv->tx_itr_value,
3728	tx: true);
3729	}
3730
3731	if (qv->num_rxq) {
3732	dynamic = IDPF_ITR_IS_DYNAMIC(qv->rx_intr_mode);
3733	itr = vport->rx_itr_profile[qv->rx_dim.profile_ix];
3734	idpf_vport_intr_write_itr(q_vector: qv, itr: dynamic ?
3735	itr : qv->rx_itr_value,
3736	tx: false);
3737	}
3738
3739	if (qv->num_txq || qv->num_rxq)
3740	idpf_vport_intr_update_itr_ena_irq(q_vector: qv);
3741	}
3742	}
3743
3744	/**
3745	* idpf_vport_intr_deinit - Release all vector associations for the vport
3746	* @vport: main vport structure
3747	*/
3748	void idpf_vport_intr_deinit(struct idpf_vport *vport)
3749	{
3750	idpf_vport_intr_napi_dis_all(vport);
3751	idpf_vport_intr_napi_del_all(vport);
3752	idpf_vport_intr_dis_irq_all(vport);
3753	idpf_vport_intr_rel_irq(vport);
3754	}
3755
3756	/**
3757	* idpf_tx_dim_work - Call back from the stack
3758	* @work: work queue structure
3759	*/
3760	static void idpf_tx_dim_work(struct work_struct *work)
3761	{
3762	struct idpf_q_vector *q_vector;
3763	struct idpf_vport *vport;
3764	struct dim *dim;
3765	u16 itr;
3766
3767	dim = container_of(work, struct dim, work);
3768	q_vector = container_of(dim, struct idpf_q_vector, tx_dim);
3769	vport = q_vector->vport;
3770
3771	if (dim->profile_ix >= ARRAY_SIZE(vport->tx_itr_profile))
3772	dim->profile_ix = ARRAY_SIZE(vport->tx_itr_profile) - 1;
3773
3774	/* look up the values in our local table */
3775	itr = vport->tx_itr_profile[dim->profile_ix];
3776
3777	idpf_vport_intr_write_itr(q_vector, itr, tx: true);
3778
3779	dim->state = DIM_START_MEASURE;
3780	}
3781
3782	/**
3783	* idpf_rx_dim_work - Call back from the stack
3784	* @work: work queue structure
3785	*/
3786	static void idpf_rx_dim_work(struct work_struct *work)
3787	{
3788	struct idpf_q_vector *q_vector;
3789	struct idpf_vport *vport;
3790	struct dim *dim;
3791	u16 itr;
3792
3793	dim = container_of(work, struct dim, work);
3794	q_vector = container_of(dim, struct idpf_q_vector, rx_dim);
3795	vport = q_vector->vport;
3796
3797	if (dim->profile_ix >= ARRAY_SIZE(vport->rx_itr_profile))
3798	dim->profile_ix = ARRAY_SIZE(vport->rx_itr_profile) - 1;
3799
3800	/* look up the values in our local table */
3801	itr = vport->rx_itr_profile[dim->profile_ix];
3802
3803	idpf_vport_intr_write_itr(q_vector, itr, tx: false);
3804
3805	dim->state = DIM_START_MEASURE;
3806	}
3807
3808	/**
3809	* idpf_init_dim - Set up dynamic interrupt moderation
3810	* @qv: q_vector structure
3811	*/
3812	static void idpf_init_dim(struct idpf_q_vector *qv)
3813	{
3814	INIT_WORK(&qv->tx_dim.work, idpf_tx_dim_work);
3815	qv->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
3816	qv->tx_dim.profile_ix = IDPF_DIM_DEFAULT_PROFILE_IX;
3817
3818	INIT_WORK(&qv->rx_dim.work, idpf_rx_dim_work);
3819	qv->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
3820	qv->rx_dim.profile_ix = IDPF_DIM_DEFAULT_PROFILE_IX;
3821	}
3822
3823	/**
3824	* idpf_vport_intr_napi_ena_all - Enable NAPI for all q_vectors in the vport
3825	* @vport: main vport structure
3826	*/
3827	static void idpf_vport_intr_napi_ena_all(struct idpf_vport *vport)
3828	{
3829	int q_idx;
3830
3831	for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) {
3832	struct idpf_q_vector *q_vector = &vport->q_vectors[q_idx];
3833
3834	idpf_init_dim(qv: q_vector);
3835	napi_enable(n: &q_vector->napi);
3836	}
3837	}
3838
3839	/**
3840	* idpf_tx_splitq_clean_all- Clean completion queues
3841	* @q_vec: queue vector
3842	* @budget: Used to determine if we are in netpoll
3843	* @cleaned: returns number of packets cleaned
3844	*
3845	* Returns false if clean is not complete else returns true
3846	*/
3847	static bool idpf_tx_splitq_clean_all(struct idpf_q_vector *q_vec,
3848	int budget, int *cleaned)
3849	{
3850	u16 num_txq = q_vec->num_txq;
3851	bool clean_complete = true;
3852	int i, budget_per_q;
3853
3854	if (unlikely(!num_txq))
3855	return true;
3856
3857	budget_per_q = DIV_ROUND_UP(budget, num_txq);
3858	for (i = 0; i < num_txq; i++)
3859	clean_complete &= idpf_tx_clean_complq(complq: q_vec->tx[i],
3860	budget: budget_per_q, cleaned);
3861
3862	return clean_complete;
3863	}
3864
3865	/**
3866	* idpf_rx_splitq_clean_all- Clean completion queues
3867	* @q_vec: queue vector
3868	* @budget: Used to determine if we are in netpoll
3869	* @cleaned: returns number of packets cleaned
3870	*
3871	* Returns false if clean is not complete else returns true
3872	*/
3873	static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget,
3874	int *cleaned)
3875	{
3876	u16 num_rxq = q_vec->num_rxq;
3877	bool clean_complete = true;
3878	int pkts_cleaned = 0;
3879	int i, budget_per_q;
3880
3881	/* We attempt to distribute budget to each Rx queue fairly, but don't
3882	* allow the budget to go below 1 because that would exit polling early.
3883	*/
3884	budget_per_q = num_rxq ? max(budget / num_rxq, 1) : 0;
3885	for (i = 0; i < num_rxq; i++) {
3886	struct idpf_queue *rxq = q_vec->rx[i];
3887	int pkts_cleaned_per_q;
3888
3889	pkts_cleaned_per_q = idpf_rx_splitq_clean(rxq, budget: budget_per_q);
3890	/* if we clean as many as budgeted, we must not be done */
3891	if (pkts_cleaned_per_q >= budget_per_q)
3892	clean_complete = false;
3893	pkts_cleaned += pkts_cleaned_per_q;
3894	}
3895	*cleaned = pkts_cleaned;
3896
3897	for (i = 0; i < q_vec->num_bufq; i++)
3898	idpf_rx_clean_refillq_all(bufq: q_vec->bufq[i]);
3899
3900	return clean_complete;
3901	}
3902
3903	/**
3904	* idpf_vport_splitq_napi_poll - NAPI handler
3905	* @napi: struct from which you get q_vector
3906	* @budget: budget provided by stack
3907	*/
3908	static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget)
3909	{
3910	struct idpf_q_vector *q_vector =
3911	container_of(napi, struct idpf_q_vector, napi);
3912	bool clean_complete;
3913	int work_done = 0;
3914
3915	/* Handle case where we are called by netpoll with a budget of 0 */
3916	if (unlikely(!budget)) {
3917	idpf_tx_splitq_clean_all(q_vec: q_vector, budget, cleaned: &work_done);
3918
3919	return 0;
3920	}
3921
3922	clean_complete = idpf_rx_splitq_clean_all(q_vec: q_vector, budget, cleaned: &work_done);
3923	clean_complete &= idpf_tx_splitq_clean_all(q_vec: q_vector, budget, cleaned: &work_done);
3924
3925	/* If work not completed, return budget and polling will return */
3926	if (!clean_complete)
3927	return budget;
3928
3929	work_done = min_t(int, work_done, budget - 1);
3930
3931	/* Exit the polling mode, but don't re-enable interrupts if stack might
3932	* poll us due to busy-polling
3933	*/
3934	if (likely(napi_complete_done(napi, work_done)))
3935	idpf_vport_intr_update_itr_ena_irq(q_vector);
3936
3937	/* Switch to poll mode in the tear-down path after sending disable
3938	* queues virtchnl message, as the interrupts will be disabled after
3939	* that
3940	*/
3941	if (unlikely(q_vector->num_txq && test_bit(__IDPF_Q_POLL_MODE,
3942	q_vector->tx[0]->flags)))
3943	return budget;
3944	else
3945	return work_done;
3946	}
3947
3948	/**
3949	* idpf_vport_intr_map_vector_to_qs - Map vectors to queues
3950	* @vport: virtual port
3951	*
3952	* Mapping for vectors to queues
3953	*/
3954	static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport)
3955	{
3956	u16 num_txq_grp = vport->num_txq_grp;
3957	int i, j, qv_idx, bufq_vidx = 0;
3958	struct idpf_rxq_group *rx_qgrp;
3959	struct idpf_txq_group *tx_qgrp;
3960	struct idpf_queue *q, *bufq;
3961	u16 q_index;
3962
3963	for (i = 0, qv_idx = 0; i < vport->num_rxq_grp; i++) {
3964	u16 num_rxq;
3965
3966	rx_qgrp = &vport->rxq_grps[i];
3967	if (idpf_is_queue_model_split(q_model: vport->rxq_model))
3968	num_rxq = rx_qgrp->splitq.num_rxq_sets;
3969	else
3970	num_rxq = rx_qgrp->singleq.num_rxq;
3971
3972	for (j = 0; j < num_rxq; j++) {
3973	if (qv_idx >= vport->num_q_vectors)
3974	qv_idx = 0;
3975
3976	if (idpf_is_queue_model_split(q_model: vport->rxq_model))
3977	q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
3978	else
3979	q = rx_qgrp->singleq.rxqs[j];
3980	q->q_vector = &vport->q_vectors[qv_idx];
3981	q_index = q->q_vector->num_rxq;
3982	q->q_vector->rx[q_index] = q;
3983	q->q_vector->num_rxq++;
3984	qv_idx++;
3985	}
3986
3987	if (idpf_is_queue_model_split(q_model: vport->rxq_model)) {
3988	for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
3989	bufq = &rx_qgrp->splitq.bufq_sets[j].bufq;
3990	bufq->q_vector = &vport->q_vectors[bufq_vidx];
3991	q_index = bufq->q_vector->num_bufq;
3992	bufq->q_vector->bufq[q_index] = bufq;
3993	bufq->q_vector->num_bufq++;
3994	}
3995	if (++bufq_vidx >= vport->num_q_vectors)
3996	bufq_vidx = 0;
3997	}
3998	}
3999
4000	for (i = 0, qv_idx = 0; i < num_txq_grp; i++) {
4001	u16 num_txq;
4002
4003	tx_qgrp = &vport->txq_grps[i];
4004	num_txq = tx_qgrp->num_txq;
4005
4006	if (idpf_is_queue_model_split(q_model: vport->txq_model)) {
4007	if (qv_idx >= vport->num_q_vectors)
4008	qv_idx = 0;
4009
4010	q = tx_qgrp->complq;
4011	q->q_vector = &vport->q_vectors[qv_idx];
4012	q_index = q->q_vector->num_txq;
4013	q->q_vector->tx[q_index] = q;
4014	q->q_vector->num_txq++;
4015	qv_idx++;
4016	} else {
4017	for (j = 0; j < num_txq; j++) {
4018	if (qv_idx >= vport->num_q_vectors)
4019	qv_idx = 0;
4020
4021	q = tx_qgrp->txqs[j];
4022	q->q_vector = &vport->q_vectors[qv_idx];
4023	q_index = q->q_vector->num_txq;
4024	q->q_vector->tx[q_index] = q;
4025	q->q_vector->num_txq++;
4026
4027	qv_idx++;
4028	}
4029	}
4030	}
4031	}
4032
4033	/**
4034	* idpf_vport_intr_init_vec_idx - Initialize the vector indexes
4035	* @vport: virtual port
4036	*
4037	* Initialize vector indexes with values returened over mailbox
4038	*/
4039	static int idpf_vport_intr_init_vec_idx(struct idpf_vport *vport)
4040	{
4041	struct idpf_adapter *adapter = vport->adapter;
4042	struct virtchnl2_alloc_vectors *ac;
4043	u16 *vecids, total_vecs;
4044	int i;
4045
4046	ac = adapter->req_vec_chunks;
4047	if (!ac) {
4048	for (i = 0; i < vport->num_q_vectors; i++)
4049	vport->q_vectors[i].v_idx = vport->q_vector_idxs[i];
4050
4051	return 0;
4052	}
4053
4054	total_vecs = idpf_get_reserved_vecs(adapter);
4055	vecids = kcalloc(n: total_vecs, size: sizeof(u16), GFP_KERNEL);
4056	if (!vecids)
4057	return -ENOMEM;
4058
4059	idpf_get_vec_ids(adapter, vecids, num_vecids: total_vecs, chunks: &ac->vchunks);
4060
4061	for (i = 0; i < vport->num_q_vectors; i++)
4062	vport->q_vectors[i].v_idx = vecids[vport->q_vector_idxs[i]];
4063
4064	kfree(objp: vecids);
4065
4066	return 0;
4067	}
4068
4069	/**
4070	* idpf_vport_intr_napi_add_all- Register napi handler for all qvectors
4071	* @vport: virtual port structure
4072	*/
4073	static void idpf_vport_intr_napi_add_all(struct idpf_vport *vport)
4074	{
4075	int (*napi_poll)(struct napi_struct *napi, int budget);
4076	u16 v_idx;
4077
4078	if (idpf_is_queue_model_split(q_model: vport->txq_model))
4079	napi_poll = idpf_vport_splitq_napi_poll;
4080	else
4081	napi_poll = idpf_vport_singleq_napi_poll;
4082
4083	for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) {
4084	struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx];
4085
4086	netif_napi_add(dev: vport->netdev, napi: &q_vector->napi, poll: napi_poll);
4087
4088	/* only set affinity_mask if the CPU is online */
4089	if (cpu_online(cpu: v_idx))
4090	cpumask_set_cpu(cpu: v_idx, dstp: &q_vector->affinity_mask);
4091	}
4092	}
4093
4094	/**
4095	* idpf_vport_intr_alloc - Allocate memory for interrupt vectors
4096	* @vport: virtual port
4097	*
4098	* We allocate one q_vector per queue interrupt. If allocation fails we
4099	* return -ENOMEM.
4100	*/
4101	int idpf_vport_intr_alloc(struct idpf_vport *vport)
4102	{
4103	u16 txqs_per_vector, rxqs_per_vector, bufqs_per_vector;
4104	struct idpf_q_vector *q_vector;
4105	int v_idx, err;
4106
4107	vport->q_vectors = kcalloc(n: vport->num_q_vectors,
4108	size: sizeof(struct idpf_q_vector), GFP_KERNEL);
4109	if (!vport->q_vectors)
4110	return -ENOMEM;
4111
4112	txqs_per_vector = DIV_ROUND_UP(vport->num_txq, vport->num_q_vectors);
4113	rxqs_per_vector = DIV_ROUND_UP(vport->num_rxq, vport->num_q_vectors);
4114	bufqs_per_vector = vport->num_bufqs_per_qgrp *
4115	DIV_ROUND_UP(vport->num_rxq_grp,
4116	vport->num_q_vectors);
4117
4118	for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) {
4119	q_vector = &vport->q_vectors[v_idx];
4120	q_vector->vport = vport;
4121
4122	q_vector->tx_itr_value = IDPF_ITR_TX_DEF;
4123	q_vector->tx_intr_mode = IDPF_ITR_DYNAMIC;
4124	q_vector->tx_itr_idx = VIRTCHNL2_ITR_IDX_1;
4125
4126	q_vector->rx_itr_value = IDPF_ITR_RX_DEF;
4127	q_vector->rx_intr_mode = IDPF_ITR_DYNAMIC;
4128	q_vector->rx_itr_idx = VIRTCHNL2_ITR_IDX_0;
4129
4130	q_vector->tx = kcalloc(n: txqs_per_vector,
4131	size: sizeof(struct idpf_queue *),
4132	GFP_KERNEL);
4133	if (!q_vector->tx) {
4134	err = -ENOMEM;
4135	goto error;
4136	}
4137
4138	q_vector->rx = kcalloc(n: rxqs_per_vector,
4139	size: sizeof(struct idpf_queue *),
4140	GFP_KERNEL);
4141	if (!q_vector->rx) {
4142	err = -ENOMEM;
4143	goto error;
4144	}
4145
4146	if (!idpf_is_queue_model_split(q_model: vport->rxq_model))
4147	continue;
4148
4149	q_vector->bufq = kcalloc(n: bufqs_per_vector,
4150	size: sizeof(struct idpf_queue *),
4151	GFP_KERNEL);
4152	if (!q_vector->bufq) {
4153	err = -ENOMEM;
4154	goto error;
4155	}
4156	}
4157
4158	return 0;
4159
4160	error:
4161	idpf_vport_intr_rel(vport);
4162
4163	return err;
4164	}
4165
4166	/**
4167	* idpf_vport_intr_init - Setup all vectors for the given vport
4168	* @vport: virtual port
4169	*
4170	* Returns 0 on success or negative on failure
4171	*/
4172	int idpf_vport_intr_init(struct idpf_vport *vport)
4173	{
4174	char *int_name;
4175	int err;
4176
4177	err = idpf_vport_intr_init_vec_idx(vport);
4178	if (err)
4179	return err;
4180
4181	idpf_vport_intr_map_vector_to_qs(vport);
4182	idpf_vport_intr_napi_add_all(vport);
4183	idpf_vport_intr_napi_ena_all(vport);
4184
4185	err = vport->adapter->dev_ops.reg_ops.intr_reg_init(vport);
4186	if (err)
4187	goto unroll_vectors_alloc;
4188
4189	int_name = kasprintf(GFP_KERNEL, fmt: "%s-%s",
4190	dev_driver_string(dev: &vport->adapter->pdev->dev),
4191	vport->netdev->name);
4192
4193	err = idpf_vport_intr_req_irq(vport, basename: int_name);
4194	if (err)
4195	goto unroll_vectors_alloc;
4196
4197	idpf_vport_intr_ena_irq_all(vport);
4198
4199	return 0;
4200
4201	unroll_vectors_alloc:
4202	idpf_vport_intr_napi_dis_all(vport);
4203	idpf_vport_intr_napi_del_all(vport);
4204
4205	return err;
4206	}
4207
4208	/**
4209	* idpf_config_rss - Send virtchnl messages to configure RSS
4210	* @vport: virtual port
4211	*
4212	* Return 0 on success, negative on failure
4213	*/
4214	int idpf_config_rss(struct idpf_vport *vport)
4215	{
4216	int err;
4217
4218	err = idpf_send_get_set_rss_key_msg(vport, get: false);
4219	if (err)
4220	return err;
4221
4222	return idpf_send_get_set_rss_lut_msg(vport, get: false);
4223	}
4224
4225	/**
4226	* idpf_fill_dflt_rss_lut - Fill the indirection table with the default values
4227	* @vport: virtual port structure
4228	*/
4229	static void idpf_fill_dflt_rss_lut(struct idpf_vport *vport)
4230	{
4231	struct idpf_adapter *adapter = vport->adapter;
4232	u16 num_active_rxq = vport->num_rxq;
4233	struct idpf_rss_data *rss_data;
4234	int i;
4235
4236	rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data;
4237
4238	for (i = 0; i < rss_data->rss_lut_size; i++) {
4239	rss_data->rss_lut[i] = i % num_active_rxq;
4240	rss_data->cached_lut[i] = rss_data->rss_lut[i];
4241	}
4242	}
4243
4244	/**
4245	* idpf_init_rss - Allocate and initialize RSS resources
4246	* @vport: virtual port
4247	*
4248	* Return 0 on success, negative on failure
4249	*/
4250	int idpf_init_rss(struct idpf_vport *vport)
4251	{
4252	struct idpf_adapter *adapter = vport->adapter;
4253	struct idpf_rss_data *rss_data;
4254	u32 lut_size;
4255
4256	rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data;
4257
4258	lut_size = rss_data->rss_lut_size * sizeof(u32);
4259	rss_data->rss_lut = kzalloc(size: lut_size, GFP_KERNEL);
4260	if (!rss_data->rss_lut)
4261	return -ENOMEM;
4262
4263	rss_data->cached_lut = kzalloc(size: lut_size, GFP_KERNEL);
4264	if (!rss_data->cached_lut) {
4265	kfree(objp: rss_data->rss_lut);
4266	rss_data->rss_lut = NULL;
4267
4268	return -ENOMEM;
4269	}
4270
4271	/* Fill the default RSS lut values */
4272	idpf_fill_dflt_rss_lut(vport);
4273
4274	return idpf_config_rss(vport);
4275	}
4276
4277	/**
4278	* idpf_deinit_rss - Release RSS resources
4279	* @vport: virtual port
4280	*/
4281	void idpf_deinit_rss(struct idpf_vport *vport)
4282	{
4283	struct idpf_adapter *adapter = vport->adapter;
4284	struct idpf_rss_data *rss_data;
4285
4286	rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data;
4287	kfree(objp: rss_data->cached_lut);
4288	rss_data->cached_lut = NULL;
4289	kfree(objp: rss_data->rss_lut);
4290	rss_data->rss_lut = NULL;
4291	}
4292