1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright(c) 2013 - 2021 Intel Corporation. */
3
4	#include <generated/utsrelease.h>
5	#include <linux/crash_dump.h>
6	#include <linux/if_bridge.h>
7	#include <linux/if_macvlan.h>
8	#include <linux/module.h>
9	#include <net/pkt_cls.h>
10	#include <net/xdp_sock_drv.h>
11
12	/* Local includes */
13	#include "i40e.h"
14	#include "i40e_devids.h"
15	#include "i40e_diag.h"
16	#include "i40e_lan_hmc.h"
17	#include "i40e_virtchnl_pf.h"
18	#include "i40e_xsk.h"
19
20	/* All i40e tracepoints are defined by the include below, which
21	* must be included exactly once across the whole kernel with
22	* CREATE_TRACE_POINTS defined
23	*/
24	#define CREATE_TRACE_POINTS
25	#include "i40e_trace.h"
26
27	const char i40e_driver_name[] = "i40e";
28	static const char i40e_driver_string[] =
29	"Intel(R) Ethernet Connection XL710 Network Driver";
30
31	static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
32
33	/* a bit of forward declarations */
34	static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
35	static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
36	static int i40e_add_vsi(struct i40e_vsi *vsi);
37	static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
38	static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired);
39	static int i40e_setup_misc_vector(struct i40e_pf *pf);
40	static void i40e_determine_queue_usage(struct i40e_pf *pf);
41	static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
42	static void i40e_prep_for_reset(struct i40e_pf *pf);
43	static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
44	bool lock_acquired);
45	static int i40e_reset(struct i40e_pf *pf);
46	static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
47	static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
48	static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
49	static bool i40e_check_recovery_mode(struct i40e_pf *pf);
50	static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
51	static void i40e_fdir_sb_setup(struct i40e_pf *pf);
52	static int i40e_veb_get_bw_info(struct i40e_veb *veb);
53	static int i40e_get_capabilities(struct i40e_pf *pf,
54	enum i40e_admin_queue_opc list_type);
55	static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);
56
57	/* i40e_pci_tbl - PCI Device ID Table
58	*
59	* Last entry must be all 0s
60	*
61	* { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
62	* Class, Class Mask, private data (not used) }
63	*/
64	static const struct pci_device_id i40e_pci_tbl[] = {
65	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
66	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
67	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
68	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
69	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
70	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
71	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
72	{PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0},
73	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
74	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
75	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
76	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
77	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
78	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
79	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
80	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
81	{PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
82	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
83	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
84	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A), 0},
85	{PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
86	{PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
87	{PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
88	{PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
89	{PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
90	{PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
91	/* required last entry */
92	{0, }
93	};
94	MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
95
96	#define I40E_MAX_VF_COUNT 128
97	static int debug = -1;
98	module_param(debug, uint, 0);
99	MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
100
101	MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
102	MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
103	MODULE_LICENSE("GPL v2");
104
105	static struct workqueue_struct *i40e_wq;
106
107	static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f,
108	struct net_device *netdev, int delta)
109	{
110	struct netdev_hw_addr_list *ha_list;
111	struct netdev_hw_addr *ha;
112
113	if (!f || !netdev)
114	return;
115
116	if (is_unicast_ether_addr(addr: f->macaddr) || is_link_local_ether_addr(addr: f->macaddr))
117	ha_list = &netdev->uc;
118	else
119	ha_list = &netdev->mc;
120
121	netdev_hw_addr_list_for_each(ha, ha_list) {
122	if (ether_addr_equal(addr1: ha->addr, addr2: f->macaddr)) {
123	ha->refcount += delta;
124	if (ha->refcount <= 0)
125	ha->refcount = 1;
126	break;
127	}
128	}
129	}
130
131	/**
132	* i40e_hw_to_dev - get device pointer from the hardware structure
133	* @hw: pointer to the device HW structure
134	**/
135	struct device *i40e_hw_to_dev(struct i40e_hw *hw)
136	{
137	struct i40e_pf *pf = i40e_hw_to_pf(hw);
138
139	return &pf->pdev->dev;
140	}
141
142	/**
143	* i40e_allocate_dma_mem - OS specific memory alloc for shared code
144	* @hw: pointer to the HW structure
145	* @mem: ptr to mem struct to fill out
146	* @size: size of memory requested
147	* @alignment: what to align the allocation to
148	**/
149	int i40e_allocate_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem,
150	u64 size, u32 alignment)
151	{
152	struct i40e_pf *pf = i40e_hw_to_pf(hw);
153
154	mem->size = ALIGN(size, alignment);
155	mem->va = dma_alloc_coherent(dev: &pf->pdev->dev, size: mem->size, dma_handle: &mem->pa,
156	GFP_KERNEL);
157	if (!mem->va)
158	return -ENOMEM;
159
160	return 0;
161	}
162
163	/**
164	* i40e_free_dma_mem - OS specific memory free for shared code
165	* @hw: pointer to the HW structure
166	* @mem: ptr to mem struct to free
167	**/
168	int i40e_free_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem)
169	{
170	struct i40e_pf *pf = i40e_hw_to_pf(hw);
171
172	dma_free_coherent(dev: &pf->pdev->dev, size: mem->size, cpu_addr: mem->va, dma_handle: mem->pa);
173	mem->va = NULL;
174	mem->pa = 0;
175	mem->size = 0;
176
177	return 0;
178	}
179
180	/**
181	* i40e_allocate_virt_mem - OS specific memory alloc for shared code
182	* @hw: pointer to the HW structure
183	* @mem: ptr to mem struct to fill out
184	* @size: size of memory requested
185	**/
186	int i40e_allocate_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem,
187	u32 size)
188	{
189	mem->size = size;
190	mem->va = kzalloc(size, GFP_KERNEL);
191
192	if (!mem->va)
193	return -ENOMEM;
194
195	return 0;
196	}
197
198	/**
199	* i40e_free_virt_mem - OS specific memory free for shared code
200	* @hw: pointer to the HW structure
201	* @mem: ptr to mem struct to free
202	**/
203	int i40e_free_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem)
204	{
205	/* it's ok to kfree a NULL pointer */
206	kfree(objp: mem->va);
207	mem->va = NULL;
208	mem->size = 0;
209
210	return 0;
211	}
212
213	/**
214	* i40e_get_lump - find a lump of free generic resource
215	* @pf: board private structure
216	* @pile: the pile of resource to search
217	* @needed: the number of items needed
218	* @id: an owner id to stick on the items assigned
219	*
220	* Returns the base item index of the lump, or negative for error
221	**/
222	static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
223	u16 needed, u16 id)
224	{
225	int ret = -ENOMEM;
226	int i, j;
227
228	if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
229	dev_info(&pf->pdev->dev,
230	"param err: pile=%s needed=%d id=0x%04x\n",
231	pile ? "<valid>" : "<null>", needed, id);
232	return -EINVAL;
233	}
234
235	/* Allocate last queue in the pile for FDIR VSI queue
236	* so it doesn't fragment the qp_pile
237	*/
238	if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) {
239	if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) {
240	dev_err(&pf->pdev->dev,
241	"Cannot allocate queue %d for I40E_VSI_FDIR\n",
242	pile->num_entries - 1);
243	return -ENOMEM;
244	}
245	pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT;
246	return pile->num_entries - 1;
247	}
248
249	i = 0;
250	while (i < pile->num_entries) {
251	/* skip already allocated entries */
252	if (pile->list[i] & I40E_PILE_VALID_BIT) {
253	i++;
254	continue;
255	}
256
257	/* do we have enough in this lump? */
258	for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
259	if (pile->list[i+j] & I40E_PILE_VALID_BIT)
260	break;
261	}
262
263	if (j == needed) {
264	/* there was enough, so assign it to the requestor */
265	for (j = 0; j < needed; j++)
266	pile->list[i+j] = id | I40E_PILE_VALID_BIT;
267	ret = i;
268	break;
269	}
270
271	/* not enough, so skip over it and continue looking */
272	i += j;
273	}
274
275	return ret;
276	}
277
278	/**
279	* i40e_put_lump - return a lump of generic resource
280	* @pile: the pile of resource to search
281	* @index: the base item index
282	* @id: the owner id of the items assigned
283	*
284	* Returns the count of items in the lump
285	**/
286	static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
287	{
288	int valid_id = (id | I40E_PILE_VALID_BIT);
289	int count = 0;
290	u16 i;
291
292	if (!pile || index >= pile->num_entries)
293	return -EINVAL;
294
295	for (i = index;
296	i < pile->num_entries && pile->list[i] == valid_id;
297	i++) {
298	pile->list[i] = 0;
299	count++;
300	}
301
302
303	return count;
304	}
305
306	/**
307	* i40e_find_vsi_from_id - searches for the vsi with the given id
308	* @pf: the pf structure to search for the vsi
309	* @id: id of the vsi it is searching for
310	**/
311	struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
312	{
313	struct i40e_vsi *vsi;
314	int i;
315
316	i40e_pf_for_each_vsi(pf, i, vsi)
317	if (vsi->id == id)
318	return vsi;
319
320	return NULL;
321	}
322
323	/**
324	* i40e_service_event_schedule - Schedule the service task to wake up
325	* @pf: board private structure
326	*
327	* If not already scheduled, this puts the task into the work queue
328	**/
329	void i40e_service_event_schedule(struct i40e_pf *pf)
330	{
331	if ((!test_bit(__I40E_DOWN, pf->state) &&
332	!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
333	test_bit(__I40E_RECOVERY_MODE, pf->state))
334	queue_work(wq: i40e_wq, work: &pf->service_task);
335	}
336
337	/**
338	* i40e_tx_timeout - Respond to a Tx Hang
339	* @netdev: network interface device structure
340	* @txqueue: queue number timing out
341	*
342	* If any port has noticed a Tx timeout, it is likely that the whole
343	* device is munged, not just the one netdev port, so go for the full
344	* reset.
345	**/
346	static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)
347	{
348	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
349	struct i40e_vsi *vsi = np->vsi;
350	struct i40e_pf *pf = vsi->back;
351	struct i40e_ring *tx_ring = NULL;
352	unsigned int i;
353	u32 head, val;
354
355	pf->tx_timeout_count++;
356
357	/* with txqueue index, find the tx_ring struct */
358	for (i = 0; i < vsi->num_queue_pairs; i++) {
359	if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
360	if (txqueue ==
361	vsi->tx_rings[i]->queue_index) {
362	tx_ring = vsi->tx_rings[i];
363	break;
364	}
365	}
366	}
367
368	if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
369	pf->tx_timeout_recovery_level = 1; /* reset after some time */
370	else if (time_before(jiffies,
371	(pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
372	return; /* don't do any new action before the next timeout */
373
374	/* don't kick off another recovery if one is already pending */
375	if (test_and_set_bit(nr: __I40E_TIMEOUT_RECOVERY_PENDING, addr: pf->state))
376	return;
377
378	if (tx_ring) {
379	head = i40e_get_head(tx_ring);
380	/* Read interrupt register */
381	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
382	val = rd32(&pf->hw,
383	I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
384	tx_ring->vsi->base_vector - 1));
385	else
386	val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
387
388	netdev_info(dev: netdev, format: "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
389	vsi->seid, txqueue, tx_ring->next_to_clean,
390	head, tx_ring->next_to_use,
391	readl(addr: tx_ring->tail), val);
392	}
393
394	pf->tx_timeout_last_recovery = jiffies;
395	netdev_info(dev: netdev, format: "tx_timeout recovery level %d, txqueue %d\n",
396	pf->tx_timeout_recovery_level, txqueue);
397
398	switch (pf->tx_timeout_recovery_level) {
399	case 1:
400	set_bit(nr: __I40E_PF_RESET_REQUESTED, addr: pf->state);
401	break;
402	case 2:
403	set_bit(nr: __I40E_CORE_RESET_REQUESTED, addr: pf->state);
404	break;
405	case 3:
406	set_bit(nr: __I40E_GLOBAL_RESET_REQUESTED, addr: pf->state);
407	break;
408	default:
409	netdev_err(dev: netdev, format: "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n");
410	set_bit(nr: __I40E_DOWN_REQUESTED, addr: pf->state);
411	set_bit(nr: __I40E_VSI_DOWN_REQUESTED, addr: vsi->state);
412	break;
413	}
414
415	i40e_service_event_schedule(pf);
416	pf->tx_timeout_recovery_level++;
417	}
418
419	/**
420	* i40e_get_vsi_stats_struct - Get System Network Statistics
421	* @vsi: the VSI we care about
422	*
423	* Returns the address of the device statistics structure.
424	* The statistics are actually updated from the service task.
425	**/
426	struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
427	{
428	return &vsi->net_stats;
429	}
430
431	/**
432	* i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
433	* @ring: Tx ring to get statistics from
434	* @stats: statistics entry to be updated
435	**/
436	static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
437	struct rtnl_link_stats64 *stats)
438	{
439	u64 bytes, packets;
440	unsigned int start;
441
442	do {
443	start = u64_stats_fetch_begin(syncp: &ring->syncp);
444	packets = ring->stats.packets;
445	bytes = ring->stats.bytes;
446	} while (u64_stats_fetch_retry(syncp: &ring->syncp, start));
447
448	stats->tx_packets += packets;
449	stats->tx_bytes += bytes;
450	}
451
452	/**
453	* i40e_get_netdev_stats_struct - Get statistics for netdev interface
454	* @netdev: network interface device structure
455	* @stats: data structure to store statistics
456	*
457	* Returns the address of the device statistics structure.
458	* The statistics are actually updated from the service task.
459	**/
460	static void i40e_get_netdev_stats_struct(struct net_device *netdev,
461	struct rtnl_link_stats64 *stats)
462	{
463	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
464	struct i40e_vsi *vsi = np->vsi;
465	struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
466	struct i40e_ring *ring;
467	int i;
468
469	if (test_bit(__I40E_VSI_DOWN, vsi->state))
470	return;
471
472	if (!vsi->tx_rings)
473	return;
474
475	rcu_read_lock();
476	for (i = 0; i < vsi->num_queue_pairs; i++) {
477	u64 bytes, packets;
478	unsigned int start;
479
480	ring = READ_ONCE(vsi->tx_rings[i]);
481	if (!ring)
482	continue;
483	i40e_get_netdev_stats_struct_tx(ring, stats);
484
485	if (i40e_enabled_xdp_vsi(vsi)) {
486	ring = READ_ONCE(vsi->xdp_rings[i]);
487	if (!ring)
488	continue;
489	i40e_get_netdev_stats_struct_tx(ring, stats);
490	}
491
492	ring = READ_ONCE(vsi->rx_rings[i]);
493	if (!ring)
494	continue;
495	do {
496	start = u64_stats_fetch_begin(syncp: &ring->syncp);
497	packets = ring->stats.packets;
498	bytes = ring->stats.bytes;
499	} while (u64_stats_fetch_retry(syncp: &ring->syncp, start));
500
501	stats->rx_packets += packets;
502	stats->rx_bytes += bytes;
503
504	}
505	rcu_read_unlock();
506
507	/* following stats updated by i40e_watchdog_subtask() */
508	stats->multicast = vsi_stats->multicast;
509	stats->tx_errors = vsi_stats->tx_errors;
510	stats->tx_dropped = vsi_stats->tx_dropped;
511	stats->rx_errors = vsi_stats->rx_errors;
512	stats->rx_dropped = vsi_stats->rx_dropped;
513	stats->rx_missed_errors = vsi_stats->rx_missed_errors;
514	stats->rx_crc_errors = vsi_stats->rx_crc_errors;
515	stats->rx_length_errors = vsi_stats->rx_length_errors;
516	}
517
518	/**
519	* i40e_vsi_reset_stats - Resets all stats of the given vsi
520	* @vsi: the VSI to have its stats reset
521	**/
522	void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
523	{
524	struct rtnl_link_stats64 *ns;
525	int i;
526
527	if (!vsi)
528	return;
529
530	ns = i40e_get_vsi_stats_struct(vsi);
531	memset(ns, 0, sizeof(*ns));
532	memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
533	memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
534	memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
535	if (vsi->rx_rings && vsi->rx_rings[0]) {
536	for (i = 0; i < vsi->num_queue_pairs; i++) {
537	memset(&vsi->rx_rings[i]->stats, 0,
538	sizeof(vsi->rx_rings[i]->stats));
539	memset(&vsi->rx_rings[i]->rx_stats, 0,
540	sizeof(vsi->rx_rings[i]->rx_stats));
541	memset(&vsi->tx_rings[i]->stats, 0,
542	sizeof(vsi->tx_rings[i]->stats));
543	memset(&vsi->tx_rings[i]->tx_stats, 0,
544	sizeof(vsi->tx_rings[i]->tx_stats));
545	}
546	}
547	vsi->stat_offsets_loaded = false;
548	}
549
550	/**
551	* i40e_pf_reset_stats - Reset all of the stats for the given PF
552	* @pf: the PF to be reset
553	**/
554	void i40e_pf_reset_stats(struct i40e_pf *pf)
555	{
556	struct i40e_veb *veb;
557	int i;
558
559	memset(&pf->stats, 0, sizeof(pf->stats));
560	memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
561	pf->stat_offsets_loaded = false;
562
563	i40e_pf_for_each_veb(pf, i, veb) {
564	memset(&veb->stats, 0, sizeof(veb->stats));
565	memset(&veb->stats_offsets, 0, sizeof(veb->stats_offsets));
566	memset(&veb->tc_stats, 0, sizeof(veb->tc_stats));
567	memset(&veb->tc_stats_offsets, 0, sizeof(veb->tc_stats_offsets));
568	veb->stat_offsets_loaded = false;
569	}
570	pf->hw_csum_rx_error = 0;
571	}
572
573	/**
574	* i40e_compute_pci_to_hw_id - compute index form PCI function.
575	* @vsi: ptr to the VSI to read from.
576	* @hw: ptr to the hardware info.
577	**/
578	static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw)
579	{
580	int pf_count = i40e_get_pf_count(hw);
581
582	if (vsi->type == I40E_VSI_SRIOV)
583	return (hw->port * BIT(7)) / pf_count + vsi->vf_id;
584
585	return hw->port + BIT(7);
586	}
587
588	/**
589	* i40e_stat_update64 - read and update a 64 bit stat from the chip.
590	* @hw: ptr to the hardware info.
591	* @hireg: the high 32 bit reg to read.
592	* @loreg: the low 32 bit reg to read.
593	* @offset_loaded: has the initial offset been loaded yet.
594	* @offset: ptr to current offset value.
595	* @stat: ptr to the stat.
596	*
597	* Since the device stats are not reset at PFReset, they will not
598	* be zeroed when the driver starts. We'll save the first values read
599	* and use them as offsets to be subtracted from the raw values in order
600	* to report stats that count from zero.
601	**/
602	static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg,
603	bool offset_loaded, u64 *offset, u64 *stat)
604	{
605	u64 new_data;
606
607	new_data = rd64(hw, loreg);
608
609	if (!offset_loaded || new_data < *offset)
610	*offset = new_data;
611	*stat = new_data - *offset;
612	}
613
614	/**
615	* i40e_stat_update48 - read and update a 48 bit stat from the chip
616	* @hw: ptr to the hardware info
617	* @hireg: the high 32 bit reg to read
618	* @loreg: the low 32 bit reg to read
619	* @offset_loaded: has the initial offset been loaded yet
620	* @offset: ptr to current offset value
621	* @stat: ptr to the stat
622	*
623	* Since the device stats are not reset at PFReset, they likely will not
624	* be zeroed when the driver starts. We'll save the first values read
625	* and use them as offsets to be subtracted from the raw values in order
626	* to report stats that count from zero. In the process, we also manage
627	* the potential roll-over.
628	**/
629	static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
630	bool offset_loaded, u64 *offset, u64 *stat)
631	{
632	u64 new_data;
633
634	if (hw->device_id == I40E_DEV_ID_QEMU) {
635	new_data = rd32(hw, loreg);
636	new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
637	} else {
638	new_data = rd64(hw, loreg);
639	}
640	if (!offset_loaded)
641	*offset = new_data;
642	if (likely(new_data >= *offset))
643	*stat = new_data - *offset;
644	else
645	*stat = (new_data + BIT_ULL(48)) - *offset;
646	*stat &= 0xFFFFFFFFFFFFULL;
647	}
648
649	/**
650	* i40e_stat_update32 - read and update a 32 bit stat from the chip
651	* @hw: ptr to the hardware info
652	* @reg: the hw reg to read
653	* @offset_loaded: has the initial offset been loaded yet
654	* @offset: ptr to current offset value
655	* @stat: ptr to the stat
656	**/
657	static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
658	bool offset_loaded, u64 *offset, u64 *stat)
659	{
660	u32 new_data;
661
662	new_data = rd32(hw, reg);
663	if (!offset_loaded)
664	*offset = new_data;
665	if (likely(new_data >= *offset))
666	*stat = (u32)(new_data - *offset);
667	else
668	*stat = (u32)((new_data + BIT_ULL(32)) - *offset);
669	}
670
671	/**
672	* i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
673	* @hw: ptr to the hardware info
674	* @reg: the hw reg to read and clear
675	* @stat: ptr to the stat
676	**/
677	static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
678	{
679	u32 new_data = rd32(hw, reg);
680
681	wr32(hw, reg, 1); /* must write a nonzero value to clear register */
682	*stat += new_data;
683	}
684
685	/**
686	* i40e_stats_update_rx_discards - update rx_discards.
687	* @vsi: ptr to the VSI to be updated.
688	* @hw: ptr to the hardware info.
689	* @stat_idx: VSI's stat_counter_idx.
690	* @offset_loaded: ptr to the VSI's stat_offsets_loaded.
691	* @stat_offset: ptr to stat_offset to store first read of specific register.
692	* @stat: ptr to VSI's stat to be updated.
693	**/
694	static void
695	i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw,
696	int stat_idx, bool offset_loaded,
697	struct i40e_eth_stats *stat_offset,
698	struct i40e_eth_stats *stat)
699	{
700	i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded,
701	offset: &stat_offset->rx_discards, stat: &stat->rx_discards);
702	i40e_stat_update64(hw,
703	I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)),
704	I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)),
705	offset_loaded, offset: &stat_offset->rx_discards_other,
706	stat: &stat->rx_discards_other);
707	}
708
709	/**
710	* i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
711	* @vsi: the VSI to be updated
712	**/
713	void i40e_update_eth_stats(struct i40e_vsi *vsi)
714	{
715	int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
716	struct i40e_pf *pf = vsi->back;
717	struct i40e_hw *hw = &pf->hw;
718	struct i40e_eth_stats *oes;
719	struct i40e_eth_stats *es; /* device's eth stats */
720
721	es = &vsi->eth_stats;
722	oes = &vsi->eth_stats_offsets;
723
724	/* Gather up the stats that the hw collects */
725	i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
726	offset_loaded: vsi->stat_offsets_loaded,
727	offset: &oes->tx_errors, stat: &es->tx_errors);
728	i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
729	offset_loaded: vsi->stat_offsets_loaded,
730	offset: &oes->rx_unknown_protocol, stat: &es->rx_unknown_protocol);
731
732	i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
733	I40E_GLV_GORCL(stat_idx),
734	offset_loaded: vsi->stat_offsets_loaded,
735	offset: &oes->rx_bytes, stat: &es->rx_bytes);
736	i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
737	I40E_GLV_UPRCL(stat_idx),
738	offset_loaded: vsi->stat_offsets_loaded,
739	offset: &oes->rx_unicast, stat: &es->rx_unicast);
740	i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
741	I40E_GLV_MPRCL(stat_idx),
742	offset_loaded: vsi->stat_offsets_loaded,
743	offset: &oes->rx_multicast, stat: &es->rx_multicast);
744	i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
745	I40E_GLV_BPRCL(stat_idx),
746	offset_loaded: vsi->stat_offsets_loaded,
747	offset: &oes->rx_broadcast, stat: &es->rx_broadcast);
748
749	i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
750	I40E_GLV_GOTCL(stat_idx),
751	offset_loaded: vsi->stat_offsets_loaded,
752	offset: &oes->tx_bytes, stat: &es->tx_bytes);
753	i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
754	I40E_GLV_UPTCL(stat_idx),
755	offset_loaded: vsi->stat_offsets_loaded,
756	offset: &oes->tx_unicast, stat: &es->tx_unicast);
757	i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
758	I40E_GLV_MPTCL(stat_idx),
759	offset_loaded: vsi->stat_offsets_loaded,
760	offset: &oes->tx_multicast, stat: &es->tx_multicast);
761	i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
762	I40E_GLV_BPTCL(stat_idx),
763	offset_loaded: vsi->stat_offsets_loaded,
764	offset: &oes->tx_broadcast, stat: &es->tx_broadcast);
765
766	i40e_stats_update_rx_discards(vsi, hw, stat_idx,
767	offset_loaded: vsi->stat_offsets_loaded, stat_offset: oes, stat: es);
768
769	vsi->stat_offsets_loaded = true;
770	}
771
772	/**
773	* i40e_update_veb_stats - Update Switch component statistics
774	* @veb: the VEB being updated
775	**/
776	void i40e_update_veb_stats(struct i40e_veb *veb)
777	{
778	struct i40e_pf *pf = veb->pf;
779	struct i40e_hw *hw = &pf->hw;
780	struct i40e_eth_stats *oes;
781	struct i40e_eth_stats *es; /* device's eth stats */
782	struct i40e_veb_tc_stats *veb_oes;
783	struct i40e_veb_tc_stats *veb_es;
784	int i, idx = 0;
785
786	idx = veb->stats_idx;
787	es = &veb->stats;
788	oes = &veb->stats_offsets;
789	veb_es = &veb->tc_stats;
790	veb_oes = &veb->tc_stats_offsets;
791
792	/* Gather up the stats that the hw collects */
793	i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
794	offset_loaded: veb->stat_offsets_loaded,
795	offset: &oes->tx_discards, stat: &es->tx_discards);
796	if (hw->revision_id > 0)
797	i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
798	offset_loaded: veb->stat_offsets_loaded,
799	offset: &oes->rx_unknown_protocol,
800	stat: &es->rx_unknown_protocol);
801	i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
802	offset_loaded: veb->stat_offsets_loaded,
803	offset: &oes->rx_bytes, stat: &es->rx_bytes);
804	i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
805	offset_loaded: veb->stat_offsets_loaded,
806	offset: &oes->rx_unicast, stat: &es->rx_unicast);
807	i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
808	offset_loaded: veb->stat_offsets_loaded,
809	offset: &oes->rx_multicast, stat: &es->rx_multicast);
810	i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
811	offset_loaded: veb->stat_offsets_loaded,
812	offset: &oes->rx_broadcast, stat: &es->rx_broadcast);
813
814	i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
815	offset_loaded: veb->stat_offsets_loaded,
816	offset: &oes->tx_bytes, stat: &es->tx_bytes);
817	i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
818	offset_loaded: veb->stat_offsets_loaded,
819	offset: &oes->tx_unicast, stat: &es->tx_unicast);
820	i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
821	offset_loaded: veb->stat_offsets_loaded,
822	offset: &oes->tx_multicast, stat: &es->tx_multicast);
823	i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
824	offset_loaded: veb->stat_offsets_loaded,
825	offset: &oes->tx_broadcast, stat: &es->tx_broadcast);
826	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
827	i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
828	I40E_GLVEBTC_RPCL(i, idx),
829	offset_loaded: veb->stat_offsets_loaded,
830	offset: &veb_oes->tc_rx_packets[i],
831	stat: &veb_es->tc_rx_packets[i]);
832	i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
833	I40E_GLVEBTC_RBCL(i, idx),
834	offset_loaded: veb->stat_offsets_loaded,
835	offset: &veb_oes->tc_rx_bytes[i],
836	stat: &veb_es->tc_rx_bytes[i]);
837	i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
838	I40E_GLVEBTC_TPCL(i, idx),
839	offset_loaded: veb->stat_offsets_loaded,
840	offset: &veb_oes->tc_tx_packets[i],
841	stat: &veb_es->tc_tx_packets[i]);
842	i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
843	I40E_GLVEBTC_TBCL(i, idx),
844	offset_loaded: veb->stat_offsets_loaded,
845	offset: &veb_oes->tc_tx_bytes[i],
846	stat: &veb_es->tc_tx_bytes[i]);
847	}
848	veb->stat_offsets_loaded = true;
849	}
850
851	/**
852	* i40e_update_vsi_stats - Update the vsi statistics counters.
853	* @vsi: the VSI to be updated
854	*
855	* There are a few instances where we store the same stat in a
856	* couple of different structs. This is partly because we have
857	* the netdev stats that need to be filled out, which is slightly
858	* different from the "eth_stats" defined by the chip and used in
859	* VF communications. We sort it out here.
860	**/
861	static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
862	{
863	u64 rx_page, rx_buf, rx_reuse, rx_alloc, rx_waive, rx_busy;
864	struct i40e_pf *pf = vsi->back;
865	struct rtnl_link_stats64 *ons;
866	struct rtnl_link_stats64 *ns; /* netdev stats */
867	struct i40e_eth_stats *oes;
868	struct i40e_eth_stats *es; /* device's eth stats */
869	u64 tx_restart, tx_busy;
870	struct i40e_ring *p;
871	u64 bytes, packets;
872	unsigned int start;
873	u64 tx_linearize;
874	u64 tx_force_wb;
875	u64 tx_stopped;
876	u64 rx_p, rx_b;
877	u64 tx_p, tx_b;
878	u16 q;
879
880	if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
881	test_bit(__I40E_CONFIG_BUSY, pf->state))
882	return;
883
884	ns = i40e_get_vsi_stats_struct(vsi);
885	ons = &vsi->net_stats_offsets;
886	es = &vsi->eth_stats;
887	oes = &vsi->eth_stats_offsets;
888
889	/* Gather up the netdev and vsi stats that the driver collects
890	* on the fly during packet processing
891	*/
892	rx_b = rx_p = 0;
893	tx_b = tx_p = 0;
894	tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
895	tx_stopped = 0;
896	rx_page = 0;
897	rx_buf = 0;
898	rx_reuse = 0;
899	rx_alloc = 0;
900	rx_waive = 0;
901	rx_busy = 0;
902	rcu_read_lock();
903	for (q = 0; q < vsi->num_queue_pairs; q++) {
904	/* locate Tx ring */
905	p = READ_ONCE(vsi->tx_rings[q]);
906	if (!p)
907	continue;
908
909	do {
910	start = u64_stats_fetch_begin(syncp: &p->syncp);
911	packets = p->stats.packets;
912	bytes = p->stats.bytes;
913	} while (u64_stats_fetch_retry(syncp: &p->syncp, start));
914	tx_b += bytes;
915	tx_p += packets;
916	tx_restart += p->tx_stats.restart_queue;
917	tx_busy += p->tx_stats.tx_busy;
918	tx_linearize += p->tx_stats.tx_linearize;
919	tx_force_wb += p->tx_stats.tx_force_wb;
920	tx_stopped += p->tx_stats.tx_stopped;
921
922	/* locate Rx ring */
923	p = READ_ONCE(vsi->rx_rings[q]);
924	if (!p)
925	continue;
926
927	do {
928	start = u64_stats_fetch_begin(syncp: &p->syncp);
929	packets = p->stats.packets;
930	bytes = p->stats.bytes;
931	} while (u64_stats_fetch_retry(syncp: &p->syncp, start));
932	rx_b += bytes;
933	rx_p += packets;
934	rx_buf += p->rx_stats.alloc_buff_failed;
935	rx_page += p->rx_stats.alloc_page_failed;
936	rx_reuse += p->rx_stats.page_reuse_count;
937	rx_alloc += p->rx_stats.page_alloc_count;
938	rx_waive += p->rx_stats.page_waive_count;
939	rx_busy += p->rx_stats.page_busy_count;
940
941	if (i40e_enabled_xdp_vsi(vsi)) {
942	/* locate XDP ring */
943	p = READ_ONCE(vsi->xdp_rings[q]);
944	if (!p)
945	continue;
946
947	do {
948	start = u64_stats_fetch_begin(syncp: &p->syncp);
949	packets = p->stats.packets;
950	bytes = p->stats.bytes;
951	} while (u64_stats_fetch_retry(syncp: &p->syncp, start));
952	tx_b += bytes;
953	tx_p += packets;
954	tx_restart += p->tx_stats.restart_queue;
955	tx_busy += p->tx_stats.tx_busy;
956	tx_linearize += p->tx_stats.tx_linearize;
957	tx_force_wb += p->tx_stats.tx_force_wb;
958	}
959	}
960	rcu_read_unlock();
961	vsi->tx_restart = tx_restart;
962	vsi->tx_busy = tx_busy;
963	vsi->tx_linearize = tx_linearize;
964	vsi->tx_force_wb = tx_force_wb;
965	vsi->tx_stopped = tx_stopped;
966	vsi->rx_page_failed = rx_page;
967	vsi->rx_buf_failed = rx_buf;
968	vsi->rx_page_reuse = rx_reuse;
969	vsi->rx_page_alloc = rx_alloc;
970	vsi->rx_page_waive = rx_waive;
971	vsi->rx_page_busy = rx_busy;
972
973	ns->rx_packets = rx_p;
974	ns->rx_bytes = rx_b;
975	ns->tx_packets = tx_p;
976	ns->tx_bytes = tx_b;
977
978	/* update netdev stats from eth stats */
979	i40e_update_eth_stats(vsi);
980	ons->tx_errors = oes->tx_errors;
981	ns->tx_errors = es->tx_errors;
982	ons->multicast = oes->rx_multicast;
983	ns->multicast = es->rx_multicast;
984	ons->rx_dropped = oes->rx_discards_other;
985	ns->rx_dropped = es->rx_discards_other;
986	ons->rx_missed_errors = oes->rx_discards;
987	ns->rx_missed_errors = es->rx_discards;
988	ons->tx_dropped = oes->tx_discards;
989	ns->tx_dropped = es->tx_discards;
990
991	/* pull in a couple PF stats if this is the main vsi */
992	if (vsi == pf->vsi[pf->lan_vsi]) {
993	ns->rx_crc_errors = pf->stats.crc_errors;
994	ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
995	ns->rx_length_errors = pf->stats.rx_length_errors;
996	}
997	}
998
999	/**
1000	* i40e_update_pf_stats - Update the PF statistics counters.
1001	* @pf: the PF to be updated
1002	**/
1003	static void i40e_update_pf_stats(struct i40e_pf *pf)
1004	{
1005	struct i40e_hw_port_stats *osd = &pf->stats_offsets;
1006	struct i40e_hw_port_stats *nsd = &pf->stats;
1007	struct i40e_hw *hw = &pf->hw;
1008	u32 val;
1009	int i;
1010
1011	i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
1012	I40E_GLPRT_GORCL(hw->port),
1013	offset_loaded: pf->stat_offsets_loaded,
1014	offset: &osd->eth.rx_bytes, stat: &nsd->eth.rx_bytes);
1015	i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
1016	I40E_GLPRT_GOTCL(hw->port),
1017	offset_loaded: pf->stat_offsets_loaded,
1018	offset: &osd->eth.tx_bytes, stat: &nsd->eth.tx_bytes);
1019	i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
1020	offset_loaded: pf->stat_offsets_loaded,
1021	offset: &osd->eth.rx_discards,
1022	stat: &nsd->eth.rx_discards);
1023	i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
1024	I40E_GLPRT_UPRCL(hw->port),
1025	offset_loaded: pf->stat_offsets_loaded,
1026	offset: &osd->eth.rx_unicast,
1027	stat: &nsd->eth.rx_unicast);
1028	i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
1029	I40E_GLPRT_MPRCL(hw->port),
1030	offset_loaded: pf->stat_offsets_loaded,
1031	offset: &osd->eth.rx_multicast,
1032	stat: &nsd->eth.rx_multicast);
1033	i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
1034	I40E_GLPRT_BPRCL(hw->port),
1035	offset_loaded: pf->stat_offsets_loaded,
1036	offset: &osd->eth.rx_broadcast,
1037	stat: &nsd->eth.rx_broadcast);
1038	i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
1039	I40E_GLPRT_UPTCL(hw->port),
1040	offset_loaded: pf->stat_offsets_loaded,
1041	offset: &osd->eth.tx_unicast,
1042	stat: &nsd->eth.tx_unicast);
1043	i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
1044	I40E_GLPRT_MPTCL(hw->port),
1045	offset_loaded: pf->stat_offsets_loaded,
1046	offset: &osd->eth.tx_multicast,
1047	stat: &nsd->eth.tx_multicast);
1048	i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
1049	I40E_GLPRT_BPTCL(hw->port),
1050	offset_loaded: pf->stat_offsets_loaded,
1051	offset: &osd->eth.tx_broadcast,
1052	stat: &nsd->eth.tx_broadcast);
1053
1054	i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
1055	offset_loaded: pf->stat_offsets_loaded,
1056	offset: &osd->tx_dropped_link_down,
1057	stat: &nsd->tx_dropped_link_down);
1058
1059	i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
1060	offset_loaded: pf->stat_offsets_loaded,
1061	offset: &osd->crc_errors, stat: &nsd->crc_errors);
1062
1063	i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
1064	offset_loaded: pf->stat_offsets_loaded,
1065	offset: &osd->illegal_bytes, stat: &nsd->illegal_bytes);
1066
1067	i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
1068	offset_loaded: pf->stat_offsets_loaded,
1069	offset: &osd->mac_local_faults,
1070	stat: &nsd->mac_local_faults);
1071	i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
1072	offset_loaded: pf->stat_offsets_loaded,
1073	offset: &osd->mac_remote_faults,
1074	stat: &nsd->mac_remote_faults);
1075
1076	i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
1077	offset_loaded: pf->stat_offsets_loaded,
1078	offset: &osd->rx_length_errors,
1079	stat: &nsd->rx_length_errors);
1080
1081	i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
1082	offset_loaded: pf->stat_offsets_loaded,
1083	offset: &osd->link_xon_rx, stat: &nsd->link_xon_rx);
1084	i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
1085	offset_loaded: pf->stat_offsets_loaded,
1086	offset: &osd->link_xon_tx, stat: &nsd->link_xon_tx);
1087	i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1088	offset_loaded: pf->stat_offsets_loaded,
1089	offset: &osd->link_xoff_rx, stat: &nsd->link_xoff_rx);
1090	i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1091	offset_loaded: pf->stat_offsets_loaded,
1092	offset: &osd->link_xoff_tx, stat: &nsd->link_xoff_tx);
1093
1094	for (i = 0; i < 8; i++) {
1095	i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1096	offset_loaded: pf->stat_offsets_loaded,
1097	offset: &osd->priority_xoff_rx[i],
1098	stat: &nsd->priority_xoff_rx[i]);
1099	i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1100	offset_loaded: pf->stat_offsets_loaded,
1101	offset: &osd->priority_xon_rx[i],
1102	stat: &nsd->priority_xon_rx[i]);
1103	i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1104	offset_loaded: pf->stat_offsets_loaded,
1105	offset: &osd->priority_xon_tx[i],
1106	stat: &nsd->priority_xon_tx[i]);
1107	i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1108	offset_loaded: pf->stat_offsets_loaded,
1109	offset: &osd->priority_xoff_tx[i],
1110	stat: &nsd->priority_xoff_tx[i]);
1111	i40e_stat_update32(hw,
1112	I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1113	offset_loaded: pf->stat_offsets_loaded,
1114	offset: &osd->priority_xon_2_xoff[i],
1115	stat: &nsd->priority_xon_2_xoff[i]);
1116	}
1117
1118	i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1119	I40E_GLPRT_PRC64L(hw->port),
1120	offset_loaded: pf->stat_offsets_loaded,
1121	offset: &osd->rx_size_64, stat: &nsd->rx_size_64);
1122	i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1123	I40E_GLPRT_PRC127L(hw->port),
1124	offset_loaded: pf->stat_offsets_loaded,
1125	offset: &osd->rx_size_127, stat: &nsd->rx_size_127);
1126	i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1127	I40E_GLPRT_PRC255L(hw->port),
1128	offset_loaded: pf->stat_offsets_loaded,
1129	offset: &osd->rx_size_255, stat: &nsd->rx_size_255);
1130	i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1131	I40E_GLPRT_PRC511L(hw->port),
1132	offset_loaded: pf->stat_offsets_loaded,
1133	offset: &osd->rx_size_511, stat: &nsd->rx_size_511);
1134	i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1135	I40E_GLPRT_PRC1023L(hw->port),
1136	offset_loaded: pf->stat_offsets_loaded,
1137	offset: &osd->rx_size_1023, stat: &nsd->rx_size_1023);
1138	i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1139	I40E_GLPRT_PRC1522L(hw->port),
1140	offset_loaded: pf->stat_offsets_loaded,
1141	offset: &osd->rx_size_1522, stat: &nsd->rx_size_1522);
1142	i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1143	I40E_GLPRT_PRC9522L(hw->port),
1144	offset_loaded: pf->stat_offsets_loaded,
1145	offset: &osd->rx_size_big, stat: &nsd->rx_size_big);
1146
1147	i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1148	I40E_GLPRT_PTC64L(hw->port),
1149	offset_loaded: pf->stat_offsets_loaded,
1150	offset: &osd->tx_size_64, stat: &nsd->tx_size_64);
1151	i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1152	I40E_GLPRT_PTC127L(hw->port),
1153	offset_loaded: pf->stat_offsets_loaded,
1154	offset: &osd->tx_size_127, stat: &nsd->tx_size_127);
1155	i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1156	I40E_GLPRT_PTC255L(hw->port),
1157	offset_loaded: pf->stat_offsets_loaded,
1158	offset: &osd->tx_size_255, stat: &nsd->tx_size_255);
1159	i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1160	I40E_GLPRT_PTC511L(hw->port),
1161	offset_loaded: pf->stat_offsets_loaded,
1162	offset: &osd->tx_size_511, stat: &nsd->tx_size_511);
1163	i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1164	I40E_GLPRT_PTC1023L(hw->port),
1165	offset_loaded: pf->stat_offsets_loaded,
1166	offset: &osd->tx_size_1023, stat: &nsd->tx_size_1023);
1167	i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1168	I40E_GLPRT_PTC1522L(hw->port),
1169	offset_loaded: pf->stat_offsets_loaded,
1170	offset: &osd->tx_size_1522, stat: &nsd->tx_size_1522);
1171	i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1172	I40E_GLPRT_PTC9522L(hw->port),
1173	offset_loaded: pf->stat_offsets_loaded,
1174	offset: &osd->tx_size_big, stat: &nsd->tx_size_big);
1175
1176	i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1177	offset_loaded: pf->stat_offsets_loaded,
1178	offset: &osd->rx_undersize, stat: &nsd->rx_undersize);
1179	i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1180	offset_loaded: pf->stat_offsets_loaded,
1181	offset: &osd->rx_fragments, stat: &nsd->rx_fragments);
1182	i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1183	offset_loaded: pf->stat_offsets_loaded,
1184	offset: &osd->rx_oversize, stat: &nsd->rx_oversize);
1185	i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1186	offset_loaded: pf->stat_offsets_loaded,
1187	offset: &osd->rx_jabber, stat: &nsd->rx_jabber);
1188
1189	/* FDIR stats */
1190	i40e_stat_update_and_clear32(hw,
1191	I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1192	stat: &nsd->fd_atr_match);
1193	i40e_stat_update_and_clear32(hw,
1194	I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1195	stat: &nsd->fd_sb_match);
1196	i40e_stat_update_and_clear32(hw,
1197	I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1198	stat: &nsd->fd_atr_tunnel_match);
1199
1200	val = rd32(hw, I40E_PRTPM_EEE_STAT);
1201	nsd->tx_lpi_status =
1202	FIELD_GET(I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK, val);
1203	nsd->rx_lpi_status =
1204	FIELD_GET(I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK, val);
1205	i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1206	offset_loaded: pf->stat_offsets_loaded,
1207	offset: &osd->tx_lpi_count, stat: &nsd->tx_lpi_count);
1208	i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1209	offset_loaded: pf->stat_offsets_loaded,
1210	offset: &osd->rx_lpi_count, stat: &nsd->rx_lpi_count);
1211
1212	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
1213	!test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1214	nsd->fd_sb_status = true;
1215	else
1216	nsd->fd_sb_status = false;
1217
1218	if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
1219	!test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1220	nsd->fd_atr_status = true;
1221	else
1222	nsd->fd_atr_status = false;
1223
1224	pf->stat_offsets_loaded = true;
1225	}
1226
1227	/**
1228	* i40e_update_stats - Update the various statistics counters.
1229	* @vsi: the VSI to be updated
1230	*
1231	* Update the various stats for this VSI and its related entities.
1232	**/
1233	void i40e_update_stats(struct i40e_vsi *vsi)
1234	{
1235	struct i40e_pf *pf = vsi->back;
1236
1237	if (vsi == pf->vsi[pf->lan_vsi])
1238	i40e_update_pf_stats(pf);
1239
1240	i40e_update_vsi_stats(vsi);
1241	}
1242
1243	/**
1244	* i40e_count_filters - counts VSI mac filters
1245	* @vsi: the VSI to be searched
1246	*
1247	* Returns count of mac filters
1248	**/
1249	int i40e_count_filters(struct i40e_vsi *vsi)
1250	{
1251	struct i40e_mac_filter *f;
1252	struct hlist_node *h;
1253	int bkt;
1254	int cnt = 0;
1255
1256	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1257	if (f->state == I40E_FILTER_NEW ||
1258	f->state == I40E_FILTER_ACTIVE)
1259	++cnt;
1260	}
1261
1262	return cnt;
1263	}
1264
1265	/**
1266	* i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1267	* @vsi: the VSI to be searched
1268	* @macaddr: the MAC address
1269	* @vlan: the vlan
1270	*
1271	* Returns ptr to the filter object or NULL
1272	**/
1273	static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1274	const u8 *macaddr, s16 vlan)
1275	{
1276	struct i40e_mac_filter *f;
1277	u64 key;
1278
1279	if (!vsi || !macaddr)
1280	return NULL;
1281
1282	key = i40e_addr_to_hkey(macaddr);
1283	hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1284	if ((ether_addr_equal(addr1: macaddr, addr2: f->macaddr)) &&
1285	(vlan == f->vlan))
1286	return f;
1287	}
1288	return NULL;
1289	}
1290
1291	/**
1292	* i40e_find_mac - Find a mac addr in the macvlan filters list
1293	* @vsi: the VSI to be searched
1294	* @macaddr: the MAC address we are searching for
1295	*
1296	* Returns the first filter with the provided MAC address or NULL if
1297	* MAC address was not found
1298	**/
1299	struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1300	{
1301	struct i40e_mac_filter *f;
1302	u64 key;
1303
1304	if (!vsi || !macaddr)
1305	return NULL;
1306
1307	key = i40e_addr_to_hkey(macaddr);
1308	hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1309	if ((ether_addr_equal(addr1: macaddr, addr2: f->macaddr)))
1310	return f;
1311	}
1312	return NULL;
1313	}
1314
1315	/**
1316	* i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1317	* @vsi: the VSI to be searched
1318	*
1319	* Returns true if VSI is in vlan mode or false otherwise
1320	**/
1321	bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1322	{
1323	/* If we have a PVID, always operate in VLAN mode */
1324	if (vsi->info.pvid)
1325	return true;
1326
1327	/* We need to operate in VLAN mode whenever we have any filters with
1328	* a VLAN other than I40E_VLAN_ALL. We could check the table each
1329	* time, incurring search cost repeatedly. However, we can notice two
1330	* things:
1331	*
1332	* 1) the only place where we can gain a VLAN filter is in
1333	* i40e_add_filter.
1334	*
1335	* 2) the only place where filters are actually removed is in
1336	* i40e_sync_filters_subtask.
1337	*
1338	* Thus, we can simply use a boolean value, has_vlan_filters which we
1339	* will set to true when we add a VLAN filter in i40e_add_filter. Then
1340	* we have to perform the full search after deleting filters in
1341	* i40e_sync_filters_subtask, but we already have to search
1342	* filters here and can perform the check at the same time. This
1343	* results in avoiding embedding a loop for VLAN mode inside another
1344	* loop over all the filters, and should maintain correctness as noted
1345	* above.
1346	*/
1347	return vsi->has_vlan_filter;
1348	}
1349
1350	/**
1351	* i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1352	* @vsi: the VSI to configure
1353	* @tmp_add_list: list of filters ready to be added
1354	* @tmp_del_list: list of filters ready to be deleted
1355	* @vlan_filters: the number of active VLAN filters
1356	*
1357	* Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1358	* behave as expected. If we have any active VLAN filters remaining or about
1359	* to be added then we need to update non-VLAN filters to be marked as VLAN=0
1360	* so that they only match against untagged traffic. If we no longer have any
1361	* active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1362	* so that they match against both tagged and untagged traffic. In this way,
1363	* we ensure that we correctly receive the desired traffic. This ensures that
1364	* when we have an active VLAN we will receive only untagged traffic and
1365	* traffic matching active VLANs. If we have no active VLANs then we will
1366	* operate in non-VLAN mode and receive all traffic, tagged or untagged.
1367	*
1368	* Finally, in a similar fashion, this function also corrects filters when
1369	* there is an active PVID assigned to this VSI.
1370	*
1371	* In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1372	*
1373	* This function is only expected to be called from within
1374	* i40e_sync_vsi_filters.
1375	*
1376	* NOTE: This function expects to be called while under the
1377	* mac_filter_hash_lock
1378	*/
1379	static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1380	struct hlist_head *tmp_add_list,
1381	struct hlist_head *tmp_del_list,
1382	int vlan_filters)
1383	{
1384	s16 pvid = le16_to_cpu(vsi->info.pvid);
1385	struct i40e_mac_filter *f, *add_head;
1386	struct i40e_new_mac_filter *new;
1387	struct hlist_node *h;
1388	int bkt, new_vlan;
1389
1390	/* To determine if a particular filter needs to be replaced we
1391	* have the three following conditions:
1392	*
1393	* a) if we have a PVID assigned, then all filters which are
1394	* not marked as VLAN=PVID must be replaced with filters that
1395	* are.
1396	* b) otherwise, if we have any active VLANS, all filters
1397	* which are marked as VLAN=-1 must be replaced with
1398	* filters marked as VLAN=0
1399	* c) finally, if we do not have any active VLANS, all filters
1400	* which are marked as VLAN=0 must be replaced with filters
1401	* marked as VLAN=-1
1402	*/
1403
1404	/* Update the filters about to be added in place */
1405	hlist_for_each_entry(new, tmp_add_list, hlist) {
1406	if (pvid && new->f->vlan != pvid)
1407	new->f->vlan = pvid;
1408	else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1409	new->f->vlan = 0;
1410	else if (!vlan_filters && new->f->vlan == 0)
1411	new->f->vlan = I40E_VLAN_ANY;
1412	}
1413
1414	/* Update the remaining active filters */
1415	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1416	/* Combine the checks for whether a filter needs to be changed
1417	* and then determine the new VLAN inside the if block, in
1418	* order to avoid duplicating code for adding the new filter
1419	* then deleting the old filter.
1420	*/
1421	if ((pvid && f->vlan != pvid) ||
1422	(vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1423	(!vlan_filters && f->vlan == 0)) {
1424	/* Determine the new vlan we will be adding */
1425	if (pvid)
1426	new_vlan = pvid;
1427	else if (vlan_filters)
1428	new_vlan = 0;
1429	else
1430	new_vlan = I40E_VLAN_ANY;
1431
1432	/* Create the new filter */
1433	add_head = i40e_add_filter(vsi, macaddr: f->macaddr, vlan: new_vlan);
1434	if (!add_head)
1435	return -ENOMEM;
1436
1437	/* Create a temporary i40e_new_mac_filter */
1438	new = kzalloc(size: sizeof(*new), GFP_ATOMIC);
1439	if (!new)
1440	return -ENOMEM;
1441
1442	new->f = add_head;
1443	new->state = add_head->state;
1444
1445	/* Add the new filter to the tmp list */
1446	hlist_add_head(n: &new->hlist, h: tmp_add_list);
1447
1448	/* Put the original filter into the delete list */
1449	f->state = I40E_FILTER_REMOVE;
1450	hash_del(node: &f->hlist);
1451	hlist_add_head(n: &f->hlist, h: tmp_del_list);
1452	}
1453	}
1454
1455	vsi->has_vlan_filter = !!vlan_filters;
1456
1457	return 0;
1458	}
1459
1460	/**
1461	* i40e_get_vf_new_vlan - Get new vlan id on a vf
1462	* @vsi: the vsi to configure
1463	* @new_mac: new mac filter to be added
1464	* @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL
1465	* @vlan_filters: the number of active VLAN filters
1466	* @trusted: flag if the VF is trusted
1467	*
1468	* Get new VLAN id based on current VLAN filters, trust, PVID
1469	* and vf-vlan-prune-disable flag.
1470	*
1471	* Returns the value of the new vlan filter or
1472	* the old value if no new filter is needed.
1473	*/
1474	static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi,
1475	struct i40e_new_mac_filter *new_mac,
1476	struct i40e_mac_filter *f,
1477	int vlan_filters,
1478	bool trusted)
1479	{
1480	s16 pvid = le16_to_cpu(vsi->info.pvid);
1481	struct i40e_pf *pf = vsi->back;
1482	bool is_any;
1483
1484	if (new_mac)
1485	f = new_mac->f;
1486
1487	if (pvid && f->vlan != pvid)
1488	return pvid;
1489
1490	is_any = (trusted ||
1491	!test_bit(I40E_FLAG_VF_VLAN_PRUNING_ENA, pf->flags));
1492
1493	if ((vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1494	(!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1495	(is_any && !vlan_filters && f->vlan == 0)) {
1496	if (is_any)
1497	return I40E_VLAN_ANY;
1498	else
1499	return 0;
1500	}
1501
1502	return f->vlan;
1503	}
1504
1505	/**
1506	* i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary
1507	* @vsi: the vsi to configure
1508	* @tmp_add_list: list of filters ready to be added
1509	* @tmp_del_list: list of filters ready to be deleted
1510	* @vlan_filters: the number of active VLAN filters
1511	* @trusted: flag if the VF is trusted
1512	*
1513	* Correct VF VLAN filters based on current VLAN filters, trust, PVID
1514	* and vf-vlan-prune-disable flag.
1515	*
1516	* In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1517	*
1518	* This function is only expected to be called from within
1519	* i40e_sync_vsi_filters.
1520	*
1521	* NOTE: This function expects to be called while under the
1522	* mac_filter_hash_lock
1523	*/
1524	static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi,
1525	struct hlist_head *tmp_add_list,
1526	struct hlist_head *tmp_del_list,
1527	int vlan_filters,
1528	bool trusted)
1529	{
1530	struct i40e_mac_filter *f, *add_head;
1531	struct i40e_new_mac_filter *new_mac;
1532	struct hlist_node *h;
1533	int bkt, new_vlan;
1534
1535	hlist_for_each_entry(new_mac, tmp_add_list, hlist) {
1536	new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL,
1537	vlan_filters, trusted);
1538	}
1539
1540	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1541	new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters,
1542	trusted);
1543	if (new_vlan != f->vlan) {
1544	add_head = i40e_add_filter(vsi, macaddr: f->macaddr, vlan: new_vlan);
1545	if (!add_head)
1546	return -ENOMEM;
1547	/* Create a temporary i40e_new_mac_filter */
1548	new_mac = kzalloc(size: sizeof(*new_mac), GFP_ATOMIC);
1549	if (!new_mac)
1550	return -ENOMEM;
1551	new_mac->f = add_head;
1552	new_mac->state = add_head->state;
1553
1554	/* Add the new filter to the tmp list */
1555	hlist_add_head(n: &new_mac->hlist, h: tmp_add_list);
1556
1557	/* Put the original filter into the delete list */
1558	f->state = I40E_FILTER_REMOVE;
1559	hash_del(node: &f->hlist);
1560	hlist_add_head(n: &f->hlist, h: tmp_del_list);
1561	}
1562	}
1563
1564	vsi->has_vlan_filter = !!vlan_filters;
1565	return 0;
1566	}
1567
1568	/**
1569	* i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1570	* @vsi: the PF Main VSI - inappropriate for any other VSI
1571	* @macaddr: the MAC address
1572	*
1573	* Remove whatever filter the firmware set up so the driver can manage
1574	* its own filtering intelligently.
1575	**/
1576	static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1577	{
1578	struct i40e_aqc_remove_macvlan_element_data element;
1579	struct i40e_pf *pf = vsi->back;
1580
1581	/* Only appropriate for the PF main VSI */
1582	if (vsi->type != I40E_VSI_MAIN)
1583	return;
1584
1585	memset(&element, 0, sizeof(element));
1586	ether_addr_copy(dst: element.mac_addr, src: macaddr);
1587	element.vlan_tag = 0;
1588	/* Ignore error returns, some firmware does it this way... */
1589	element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1590	i40e_aq_remove_macvlan(hw: &pf->hw, vsi_id: vsi->seid, mv_list: &element, count: 1, NULL);
1591
1592	memset(&element, 0, sizeof(element));
1593	ether_addr_copy(dst: element.mac_addr, src: macaddr);
1594	element.vlan_tag = 0;
1595	/* ...and some firmware does it this way. */
1596	element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1597	I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1598	i40e_aq_remove_macvlan(hw: &pf->hw, vsi_id: vsi->seid, mv_list: &element, count: 1, NULL);
1599	}
1600
1601	/**
1602	* i40e_add_filter - Add a mac/vlan filter to the VSI
1603	* @vsi: the VSI to be searched
1604	* @macaddr: the MAC address
1605	* @vlan: the vlan
1606	*
1607	* Returns ptr to the filter object or NULL when no memory available.
1608	*
1609	* NOTE: This function is expected to be called with mac_filter_hash_lock
1610	* being held.
1611	**/
1612	struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1613	const u8 *macaddr, s16 vlan)
1614	{
1615	struct i40e_mac_filter *f;
1616	u64 key;
1617
1618	if (!vsi || !macaddr)
1619	return NULL;
1620
1621	f = i40e_find_filter(vsi, macaddr, vlan);
1622	if (!f) {
1623	f = kzalloc(size: sizeof(*f), GFP_ATOMIC);
1624	if (!f)
1625	return NULL;
1626
1627	/* Update the boolean indicating if we need to function in
1628	* VLAN mode.
1629	*/
1630	if (vlan >= 0)
1631	vsi->has_vlan_filter = true;
1632
1633	ether_addr_copy(dst: f->macaddr, src: macaddr);
1634	f->vlan = vlan;
1635	f->state = I40E_FILTER_NEW;
1636	INIT_HLIST_NODE(h: &f->hlist);
1637
1638	key = i40e_addr_to_hkey(macaddr);
1639	hash_add(vsi->mac_filter_hash, &f->hlist, key);
1640
1641	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1642	set_bit(nr: __I40E_MACVLAN_SYNC_PENDING, addr: vsi->back->state);
1643	}
1644
1645	/* If we're asked to add a filter that has been marked for removal, it
1646	* is safe to simply restore it to active state. __i40e_del_filter
1647	* will have simply deleted any filters which were previously marked
1648	* NEW or FAILED, so if it is currently marked REMOVE it must have
1649	* previously been ACTIVE. Since we haven't yet run the sync filters
1650	* task, just restore this filter to the ACTIVE state so that the
1651	* sync task leaves it in place
1652	*/
1653	if (f->state == I40E_FILTER_REMOVE)
1654	f->state = I40E_FILTER_ACTIVE;
1655
1656	return f;
1657	}
1658
1659	/**
1660	* __i40e_del_filter - Remove a specific filter from the VSI
1661	* @vsi: VSI to remove from
1662	* @f: the filter to remove from the list
1663	*
1664	* This function should be called instead of i40e_del_filter only if you know
1665	* the exact filter you will remove already, such as via i40e_find_filter or
1666	* i40e_find_mac.
1667	*
1668	* NOTE: This function is expected to be called with mac_filter_hash_lock
1669	* being held.
1670	* ANOTHER NOTE: This function MUST be called from within the context of
1671	* the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1672	* instead of list_for_each_entry().
1673	**/
1674	void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1675	{
1676	if (!f)
1677	return;
1678
1679	/* If the filter was never added to firmware then we can just delete it
1680	* directly and we don't want to set the status to remove or else an
1681	* admin queue command will unnecessarily fire.
1682	*/
1683	if ((f->state == I40E_FILTER_FAILED) ||
1684	(f->state == I40E_FILTER_NEW)) {
1685	hash_del(node: &f->hlist);
1686	kfree(objp: f);
1687	} else {
1688	f->state = I40E_FILTER_REMOVE;
1689	}
1690
1691	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1692	set_bit(nr: __I40E_MACVLAN_SYNC_PENDING, addr: vsi->back->state);
1693	}
1694
1695	/**
1696	* i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1697	* @vsi: the VSI to be searched
1698	* @macaddr: the MAC address
1699	* @vlan: the VLAN
1700	*
1701	* NOTE: This function is expected to be called with mac_filter_hash_lock
1702	* being held.
1703	* ANOTHER NOTE: This function MUST be called from within the context of
1704	* the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1705	* instead of list_for_each_entry().
1706	**/
1707	void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1708	{
1709	struct i40e_mac_filter *f;
1710
1711	if (!vsi || !macaddr)
1712	return;
1713
1714	f = i40e_find_filter(vsi, macaddr, vlan);
1715	__i40e_del_filter(vsi, f);
1716	}
1717
1718	/**
1719	* i40e_add_mac_filter - Add a MAC filter for all active VLANs
1720	* @vsi: the VSI to be searched
1721	* @macaddr: the mac address to be filtered
1722	*
1723	* If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1724	* go through all the macvlan filters and add a macvlan filter for each
1725	* unique vlan that already exists. If a PVID has been assigned, instead only
1726	* add the macaddr to that VLAN.
1727	*
1728	* Returns last filter added on success, else NULL
1729	**/
1730	struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1731	const u8 *macaddr)
1732	{
1733	struct i40e_mac_filter *f, *add = NULL;
1734	struct hlist_node *h;
1735	int bkt;
1736
1737	if (vsi->info.pvid)
1738	return i40e_add_filter(vsi, macaddr,
1739	le16_to_cpu(vsi->info.pvid));
1740
1741	if (!i40e_is_vsi_in_vlan(vsi))
1742	return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1743
1744	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1745	if (f->state == I40E_FILTER_REMOVE)
1746	continue;
1747	add = i40e_add_filter(vsi, macaddr, vlan: f->vlan);
1748	if (!add)
1749	return NULL;
1750	}
1751
1752	return add;
1753	}
1754
1755	/**
1756	* i40e_del_mac_filter - Remove a MAC filter from all VLANs
1757	* @vsi: the VSI to be searched
1758	* @macaddr: the mac address to be removed
1759	*
1760	* Removes a given MAC address from a VSI regardless of what VLAN it has been
1761	* associated with.
1762	*
1763	* Returns 0 for success, or error
1764	**/
1765	int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1766	{
1767	struct i40e_mac_filter *f;
1768	struct hlist_node *h;
1769	bool found = false;
1770	int bkt;
1771
1772	lockdep_assert_held(&vsi->mac_filter_hash_lock);
1773	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1774	if (ether_addr_equal(addr1: macaddr, addr2: f->macaddr)) {
1775	__i40e_del_filter(vsi, f);
1776	found = true;
1777	}
1778	}
1779
1780	if (found)
1781	return 0;
1782	else
1783	return -ENOENT;
1784	}
1785
1786	/**
1787	* i40e_set_mac - NDO callback to set mac address
1788	* @netdev: network interface device structure
1789	* @p: pointer to an address structure
1790	*
1791	* Returns 0 on success, negative on failure
1792	**/
1793	static int i40e_set_mac(struct net_device *netdev, void *p)
1794	{
1795	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
1796	struct i40e_vsi *vsi = np->vsi;
1797	struct i40e_pf *pf = vsi->back;
1798	struct i40e_hw *hw = &pf->hw;
1799	struct sockaddr *addr = p;
1800
1801	if (!is_valid_ether_addr(addr: addr->sa_data))
1802	return -EADDRNOTAVAIL;
1803
1804	if (test_bit(__I40E_DOWN, pf->state) ||
1805	test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1806	return -EADDRNOTAVAIL;
1807
1808	if (ether_addr_equal(addr1: hw->mac.addr, addr2: addr->sa_data))
1809	netdev_info(dev: netdev, format: "returning to hw mac address %pM\n",
1810	hw->mac.addr);
1811	else
1812	netdev_info(dev: netdev, format: "set new mac address %pM\n", addr->sa_data);
1813
1814	/* Copy the address first, so that we avoid a possible race with
1815	* .set_rx_mode().
1816	* - Remove old address from MAC filter
1817	* - Copy new address
1818	* - Add new address to MAC filter
1819	*/
1820	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
1821	i40e_del_mac_filter(vsi, macaddr: netdev->dev_addr);
1822	eth_hw_addr_set(dev: netdev, addr: addr->sa_data);
1823	i40e_add_mac_filter(vsi, macaddr: netdev->dev_addr);
1824	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
1825
1826	if (vsi->type == I40E_VSI_MAIN) {
1827	int ret;
1828
1829	ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1830	mac_addr: addr->sa_data, NULL);
1831	if (ret)
1832	netdev_info(dev: netdev, format: "Ignoring error from firmware on LAA update, status %pe, AQ ret %s\n",
1833	ERR_PTR(error: ret),
1834	i40e_aq_str(hw, aq_err: hw->aq.asq_last_status));
1835	}
1836
1837	/* schedule our worker thread which will take care of
1838	* applying the new filter changes
1839	*/
1840	i40e_service_event_schedule(pf);
1841	return 0;
1842	}
1843
1844	/**
1845	* i40e_config_rss_aq - Prepare for RSS using AQ commands
1846	* @vsi: vsi structure
1847	* @seed: RSS hash seed
1848	* @lut: pointer to lookup table of lut_size
1849	* @lut_size: size of the lookup table
1850	**/
1851	static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1852	u8 *lut, u16 lut_size)
1853	{
1854	struct i40e_pf *pf = vsi->back;
1855	struct i40e_hw *hw = &pf->hw;
1856	int ret = 0;
1857
1858	if (seed) {
1859	struct i40e_aqc_get_set_rss_key_data *seed_dw =
1860	(struct i40e_aqc_get_set_rss_key_data *)seed;
1861	ret = i40e_aq_set_rss_key(hw, seid: vsi->id, key: seed_dw);
1862	if (ret) {
1863	dev_info(&pf->pdev->dev,
1864	"Cannot set RSS key, err %pe aq_err %s\n",
1865	ERR_PTR(ret),
1866	i40e_aq_str(hw, hw->aq.asq_last_status));
1867	return ret;
1868	}
1869	}
1870	if (lut) {
1871	bool pf_lut = vsi->type == I40E_VSI_MAIN;
1872
1873	ret = i40e_aq_set_rss_lut(hw, seid: vsi->id, pf_lut, lut, lut_size);
1874	if (ret) {
1875	dev_info(&pf->pdev->dev,
1876	"Cannot set RSS lut, err %pe aq_err %s\n",
1877	ERR_PTR(ret),
1878	i40e_aq_str(hw, hw->aq.asq_last_status));
1879	return ret;
1880	}
1881	}
1882	return ret;
1883	}
1884
1885	/**
1886	* i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1887	* @vsi: VSI structure
1888	**/
1889	static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1890	{
1891	struct i40e_pf *pf = vsi->back;
1892	u8 seed[I40E_HKEY_ARRAY_SIZE];
1893	u8 *lut;
1894	int ret;
1895
1896	if (!test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
1897	return 0;
1898	if (!vsi->rss_size)
1899	vsi->rss_size = min_t(int, pf->alloc_rss_size,
1900	vsi->num_queue_pairs);
1901	if (!vsi->rss_size)
1902	return -EINVAL;
1903	lut = kzalloc(size: vsi->rss_table_size, GFP_KERNEL);
1904	if (!lut)
1905	return -ENOMEM;
1906
1907	/* Use the user configured hash keys and lookup table if there is one,
1908	* otherwise use default
1909	*/
1910	if (vsi->rss_lut_user)
1911	memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1912	else
1913	i40e_fill_rss_lut(pf, lut, rss_table_size: vsi->rss_table_size, rss_size: vsi->rss_size);
1914	if (vsi->rss_hkey_user)
1915	memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1916	else
1917	netdev_rss_key_fill(buffer: (void *)seed, I40E_HKEY_ARRAY_SIZE);
1918	ret = i40e_config_rss_aq(vsi, seed, lut, lut_size: vsi->rss_table_size);
1919	kfree(objp: lut);
1920	return ret;
1921	}
1922
1923	/**
1924	* i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1925	* @vsi: the VSI being configured,
1926	* @ctxt: VSI context structure
1927	* @enabled_tc: number of traffic classes to enable
1928	*
1929	* Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1930	**/
1931	static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1932	struct i40e_vsi_context *ctxt,
1933	u8 enabled_tc)
1934	{
1935	u16 qcount = 0, max_qcount, qmap, sections = 0;
1936	int i, override_q, pow, num_qps, ret;
1937	u8 netdev_tc = 0, offset = 0;
1938
1939	if (vsi->type != I40E_VSI_MAIN)
1940	return -EINVAL;
1941	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1942	sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1943	vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1944	vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1945	num_qps = vsi->mqprio_qopt.qopt.count[0];
1946
1947	/* find the next higher power-of-2 of num queue pairs */
1948	pow = ilog2(num_qps);
1949	if (!is_power_of_2(n: num_qps))
1950	pow++;
1951	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1952	(pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1953
1954	/* Setup queue offset/count for all TCs for given VSI */
1955	max_qcount = vsi->mqprio_qopt.qopt.count[0];
1956	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1957	/* See if the given TC is enabled for the given VSI */
1958	if (vsi->tc_config.enabled_tc & BIT(i)) {
1959	offset = vsi->mqprio_qopt.qopt.offset[i];
1960	qcount = vsi->mqprio_qopt.qopt.count[i];
1961	if (qcount > max_qcount)
1962	max_qcount = qcount;
1963	vsi->tc_config.tc_info[i].qoffset = offset;
1964	vsi->tc_config.tc_info[i].qcount = qcount;
1965	vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1966	} else {
1967	/* TC is not enabled so set the offset to
1968	* default queue and allocate one queue
1969	* for the given TC.
1970	*/
1971	vsi->tc_config.tc_info[i].qoffset = 0;
1972	vsi->tc_config.tc_info[i].qcount = 1;
1973	vsi->tc_config.tc_info[i].netdev_tc = 0;
1974	}
1975	}
1976
1977	/* Set actual Tx/Rx queue pairs */
1978	vsi->num_queue_pairs = offset + qcount;
1979
1980	/* Setup queue TC[0].qmap for given VSI context */
1981	ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1982	ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1983	ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1984	ctxt->info.valid_sections |= cpu_to_le16(sections);
1985
1986	/* Reconfigure RSS for main VSI with max queue count */
1987	vsi->rss_size = max_qcount;
1988	ret = i40e_vsi_config_rss(vsi);
1989	if (ret) {
1990	dev_info(&vsi->back->pdev->dev,
1991	"Failed to reconfig rss for num_queues (%u)\n",
1992	max_qcount);
1993	return ret;
1994	}
1995	vsi->reconfig_rss = true;
1996	dev_dbg(&vsi->back->pdev->dev,
1997	"Reconfigured rss with num_queues (%u)\n", max_qcount);
1998
1999	/* Find queue count available for channel VSIs and starting offset
2000	* for channel VSIs
2001	*/
2002	override_q = vsi->mqprio_qopt.qopt.count[0];
2003	if (override_q && override_q < vsi->num_queue_pairs) {
2004	vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
2005	vsi->next_base_queue = override_q;
2006	}
2007	return 0;
2008	}
2009
2010	/**
2011	* i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
2012	* @vsi: the VSI being setup
2013	* @ctxt: VSI context structure
2014	* @enabled_tc: Enabled TCs bitmap
2015	* @is_add: True if called before Add VSI
2016	*
2017	* Setup VSI queue mapping for enabled traffic classes.
2018	**/
2019	static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
2020	struct i40e_vsi_context *ctxt,
2021	u8 enabled_tc,
2022	bool is_add)
2023	{
2024	struct i40e_pf *pf = vsi->back;
2025	u16 num_tc_qps = 0;
2026	u16 sections = 0;
2027	u8 netdev_tc = 0;
2028	u16 numtc = 1;
2029	u16 qcount;
2030	u8 offset;
2031	u16 qmap;
2032	int i;
2033
2034	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2035	offset = 0;
2036	/* zero out queue mapping, it will get updated on the end of the function */
2037	memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));
2038
2039	if (vsi->type == I40E_VSI_MAIN) {
2040	/* This code helps add more queue to the VSI if we have
2041	* more cores than RSS can support, the higher cores will
2042	* be served by ATR or other filters. Furthermore, the
2043	* non-zero req_queue_pairs says that user requested a new
2044	* queue count via ethtool's set_channels, so use this
2045	* value for queues distribution across traffic classes
2046	* We need at least one queue pair for the interface
2047	* to be usable as we see in else statement.
2048	*/
2049	if (vsi->req_queue_pairs > 0)
2050	vsi->num_queue_pairs = vsi->req_queue_pairs;
2051	else if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
2052	vsi->num_queue_pairs = pf->num_lan_msix;
2053	else
2054	vsi->num_queue_pairs = 1;
2055	}
2056
2057	/* Number of queues per enabled TC */
2058	if (vsi->type == I40E_VSI_MAIN ||
2059	(vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))
2060	num_tc_qps = vsi->num_queue_pairs;
2061	else
2062	num_tc_qps = vsi->alloc_queue_pairs;
2063
2064	if (enabled_tc && test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {
2065	/* Find numtc from enabled TC bitmap */
2066	for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2067	if (enabled_tc & BIT(i)) /* TC is enabled */
2068	numtc++;
2069	}
2070	if (!numtc) {
2071	dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
2072	numtc = 1;
2073	}
2074	num_tc_qps = num_tc_qps / numtc;
2075	num_tc_qps = min_t(int, num_tc_qps,
2076	i40e_pf_get_max_q_per_tc(pf));
2077	}
2078
2079	vsi->tc_config.numtc = numtc;
2080	vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
2081
2082	/* Do not allow use more TC queue pairs than MSI-X vectors exist */
2083	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
2084	num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
2085
2086	/* Setup queue offset/count for all TCs for given VSI */
2087	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2088	/* See if the given TC is enabled for the given VSI */
2089	if (vsi->tc_config.enabled_tc & BIT(i)) {
2090	/* TC is enabled */
2091	int pow, num_qps;
2092
2093	switch (vsi->type) {
2094	case I40E_VSI_MAIN:
2095	if ((!test_bit(I40E_FLAG_FD_SB_ENA,
2096	pf->flags) &&
2097	!test_bit(I40E_FLAG_FD_ATR_ENA,
2098	pf->flags)) ||
2099	vsi->tc_config.enabled_tc != 1) {
2100	qcount = min_t(int, pf->alloc_rss_size,
2101	num_tc_qps);
2102	break;
2103	}
2104	fallthrough;
2105	case I40E_VSI_FDIR:
2106	case I40E_VSI_SRIOV:
2107	case I40E_VSI_VMDQ2:
2108	default:
2109	qcount = num_tc_qps;
2110	WARN_ON(i != 0);
2111	break;
2112	}
2113	vsi->tc_config.tc_info[i].qoffset = offset;
2114	vsi->tc_config.tc_info[i].qcount = qcount;
2115
2116	/* find the next higher power-of-2 of num queue pairs */
2117	num_qps = qcount;
2118	pow = 0;
2119	while (num_qps && (BIT_ULL(pow) < qcount)) {
2120	pow++;
2121	num_qps >>= 1;
2122	}
2123
2124	vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
2125	qmap =
2126	(offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
2127	(pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
2128
2129	offset += qcount;
2130	} else {
2131	/* TC is not enabled so set the offset to
2132	* default queue and allocate one queue
2133	* for the given TC.
2134	*/
2135	vsi->tc_config.tc_info[i].qoffset = 0;
2136	vsi->tc_config.tc_info[i].qcount = 1;
2137	vsi->tc_config.tc_info[i].netdev_tc = 0;
2138
2139	qmap = 0;
2140	}
2141	ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
2142	}
2143	/* Do not change previously set num_queue_pairs for PFs and VFs*/
2144	if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||
2145	(vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||
2146	(vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))
2147	vsi->num_queue_pairs = offset;
2148
2149	/* Scheduler section valid can only be set for ADD VSI */
2150	if (is_add) {
2151	sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
2152
2153	ctxt->info.up_enable_bits = enabled_tc;
2154	}
2155	if (vsi->type == I40E_VSI_SRIOV) {
2156	ctxt->info.mapping_flags |=
2157	cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
2158	for (i = 0; i < vsi->num_queue_pairs; i++)
2159	ctxt->info.queue_mapping[i] =
2160	cpu_to_le16(vsi->base_queue + i);
2161	} else {
2162	ctxt->info.mapping_flags |=
2163	cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
2164	ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
2165	}
2166	ctxt->info.valid_sections |= cpu_to_le16(sections);
2167	}
2168
2169	/**
2170	* i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
2171	* @netdev: the netdevice
2172	* @addr: address to add
2173	*
2174	* Called by __dev_(mc|uc)_sync when an address needs to be added. We call
2175	* __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2176	*/
2177	static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
2178	{
2179	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
2180	struct i40e_vsi *vsi = np->vsi;
2181
2182	if (i40e_add_mac_filter(vsi, macaddr: addr))
2183	return 0;
2184	else
2185	return -ENOMEM;
2186	}
2187
2188	/**
2189	* i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
2190	* @netdev: the netdevice
2191	* @addr: address to add
2192	*
2193	* Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
2194	* __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2195	*/
2196	static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
2197	{
2198	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
2199	struct i40e_vsi *vsi = np->vsi;
2200
2201	/* Under some circumstances, we might receive a request to delete
2202	* our own device address from our uc list. Because we store the
2203	* device address in the VSI's MAC/VLAN filter list, we need to ignore
2204	* such requests and not delete our device address from this list.
2205	*/
2206	if (ether_addr_equal(addr1: addr, addr2: netdev->dev_addr))
2207	return 0;
2208
2209	i40e_del_mac_filter(vsi, macaddr: addr);
2210
2211	return 0;
2212	}
2213
2214	/**
2215	* i40e_set_rx_mode - NDO callback to set the netdev filters
2216	* @netdev: network interface device structure
2217	**/
2218	static void i40e_set_rx_mode(struct net_device *netdev)
2219	{
2220	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
2221	struct i40e_vsi *vsi = np->vsi;
2222
2223	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
2224
2225	__dev_uc_sync(dev: netdev, sync: i40e_addr_sync, unsync: i40e_addr_unsync);
2226	__dev_mc_sync(dev: netdev, sync: i40e_addr_sync, unsync: i40e_addr_unsync);
2227
2228	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
2229
2230	/* check for other flag changes */
2231	if (vsi->current_netdev_flags != vsi->netdev->flags) {
2232	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2233	set_bit(nr: __I40E_MACVLAN_SYNC_PENDING, addr: vsi->back->state);
2234	}
2235	}
2236
2237	/**
2238	* i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
2239	* @vsi: Pointer to VSI struct
2240	* @from: Pointer to list which contains MAC filter entries - changes to
2241	* those entries needs to be undone.
2242	*
2243	* MAC filter entries from this list were slated for deletion.
2244	**/
2245	static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
2246	struct hlist_head *from)
2247	{
2248	struct i40e_mac_filter *f;
2249	struct hlist_node *h;
2250
2251	hlist_for_each_entry_safe(f, h, from, hlist) {
2252	u64 key = i40e_addr_to_hkey(macaddr: f->macaddr);
2253
2254	/* Move the element back into MAC filter list*/
2255	hlist_del(n: &f->hlist);
2256	hash_add(vsi->mac_filter_hash, &f->hlist, key);
2257	}
2258	}
2259
2260	/**
2261	* i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2262	* @vsi: Pointer to vsi struct
2263	* @from: Pointer to list which contains MAC filter entries - changes to
2264	* those entries needs to be undone.
2265	*
2266	* MAC filter entries from this list were slated for addition.
2267	**/
2268	static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2269	struct hlist_head *from)
2270	{
2271	struct i40e_new_mac_filter *new;
2272	struct hlist_node *h;
2273
2274	hlist_for_each_entry_safe(new, h, from, hlist) {
2275	/* We can simply free the wrapper structure */
2276	hlist_del(n: &new->hlist);
2277	netdev_hw_addr_refcnt(f: new->f, netdev: vsi->netdev, delta: -1);
2278	kfree(objp: new);
2279	}
2280	}
2281
2282	/**
2283	* i40e_next_filter - Get the next non-broadcast filter from a list
2284	* @next: pointer to filter in list
2285	*
2286	* Returns the next non-broadcast filter in the list. Required so that we
2287	* ignore broadcast filters within the list, since these are not handled via
2288	* the normal firmware update path.
2289	*/
2290	static
2291	struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2292	{
2293	hlist_for_each_entry_continue(next, hlist) {
2294	if (!is_broadcast_ether_addr(addr: next->f->macaddr))
2295	return next;
2296	}
2297
2298	return NULL;
2299	}
2300
2301	/**
2302	* i40e_update_filter_state - Update filter state based on return data
2303	* from firmware
2304	* @count: Number of filters added
2305	* @add_list: return data from fw
2306	* @add_head: pointer to first filter in current batch
2307	*
2308	* MAC filter entries from list were slated to be added to device. Returns
2309	* number of successful filters. Note that 0 does NOT mean success!
2310	**/
2311	static int
2312	i40e_update_filter_state(int count,
2313	struct i40e_aqc_add_macvlan_element_data *add_list,
2314	struct i40e_new_mac_filter *add_head)
2315	{
2316	int retval = 0;
2317	int i;
2318
2319	for (i = 0; i < count; i++) {
2320	/* Always check status of each filter. We don't need to check
2321	* the firmware return status because we pre-set the filter
2322	* status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2323	* request to the adminq. Thus, if it no longer matches then
2324	* we know the filter is active.
2325	*/
2326	if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2327	add_head->state = I40E_FILTER_FAILED;
2328	} else {
2329	add_head->state = I40E_FILTER_ACTIVE;
2330	retval++;
2331	}
2332
2333	add_head = i40e_next_filter(next: add_head);
2334	if (!add_head)
2335	break;
2336	}
2337
2338	return retval;
2339	}
2340
2341	/**
2342	* i40e_aqc_del_filters - Request firmware to delete a set of filters
2343	* @vsi: ptr to the VSI
2344	* @vsi_name: name to display in messages
2345	* @list: the list of filters to send to firmware
2346	* @num_del: the number of filters to delete
2347	* @retval: Set to -EIO on failure to delete
2348	*
2349	* Send a request to firmware via AdminQ to delete a set of filters. Uses
2350	* *retval instead of a return value so that success does not force ret_val to
2351	* be set to 0. This ensures that a sequence of calls to this function
2352	* preserve the previous value of *retval on successful delete.
2353	*/
2354	static
2355	void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2356	struct i40e_aqc_remove_macvlan_element_data *list,
2357	int num_del, int *retval)
2358	{
2359	struct i40e_hw *hw = &vsi->back->hw;
2360	enum i40e_admin_queue_err aq_status;
2361	int aq_ret;
2362
2363	aq_ret = i40e_aq_remove_macvlan_v2(hw, seid: vsi->seid, mv_list: list, count: num_del, NULL,
2364	aq_status: &aq_status);
2365
2366	/* Explicitly ignore and do not report when firmware returns ENOENT */
2367	if (aq_ret && !(aq_status == I40E_AQ_RC_ENOENT)) {
2368	*retval = -EIO;
2369	dev_info(&vsi->back->pdev->dev,
2370	"ignoring delete macvlan error on %s, err %pe, aq_err %s\n",
2371	vsi_name, ERR_PTR(aq_ret),
2372	i40e_aq_str(hw, aq_status));
2373	}
2374	}
2375
2376	/**
2377	* i40e_aqc_add_filters - Request firmware to add a set of filters
2378	* @vsi: ptr to the VSI
2379	* @vsi_name: name to display in messages
2380	* @list: the list of filters to send to firmware
2381	* @add_head: Position in the add hlist
2382	* @num_add: the number of filters to add
2383	*
2384	* Send a request to firmware via AdminQ to add a chunk of filters. Will set
2385	* __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2386	* space for more filters.
2387	*/
2388	static
2389	void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2390	struct i40e_aqc_add_macvlan_element_data *list,
2391	struct i40e_new_mac_filter *add_head,
2392	int num_add)
2393	{
2394	struct i40e_hw *hw = &vsi->back->hw;
2395	enum i40e_admin_queue_err aq_status;
2396	int fcnt;
2397
2398	i40e_aq_add_macvlan_v2(hw, seid: vsi->seid, mv_list: list, count: num_add, NULL, aq_status: &aq_status);
2399	fcnt = i40e_update_filter_state(count: num_add, add_list: list, add_head);
2400
2401	if (fcnt != num_add) {
2402	if (vsi->type == I40E_VSI_MAIN) {
2403	set_bit(nr: __I40E_VSI_OVERFLOW_PROMISC, addr: vsi->state);
2404	dev_warn(&vsi->back->pdev->dev,
2405	"Error %s adding RX filters on %s, promiscuous mode forced on\n",
2406	i40e_aq_str(hw, aq_status), vsi_name);
2407	} else if (vsi->type == I40E_VSI_SRIOV ||
2408	vsi->type == I40E_VSI_VMDQ1 ||
2409	vsi->type == I40E_VSI_VMDQ2) {
2410	dev_warn(&vsi->back->pdev->dev,
2411	"Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2412	i40e_aq_str(hw, aq_status), vsi_name,
2413	vsi_name);
2414	} else {
2415	dev_warn(&vsi->back->pdev->dev,
2416	"Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2417	i40e_aq_str(hw, aq_status), vsi_name,
2418	vsi->type);
2419	}
2420	}
2421	}
2422
2423	/**
2424	* i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2425	* @vsi: pointer to the VSI
2426	* @vsi_name: the VSI name
2427	* @f: filter data
2428	*
2429	* This function sets or clears the promiscuous broadcast flags for VLAN
2430	* filters in order to properly receive broadcast frames. Assumes that only
2431	* broadcast filters are passed.
2432	*
2433	* Returns status indicating success or failure;
2434	**/
2435	static int
2436	i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2437	struct i40e_mac_filter *f)
2438	{
2439	bool enable = f->state == I40E_FILTER_NEW;
2440	struct i40e_hw *hw = &vsi->back->hw;
2441	int aq_ret;
2442
2443	if (f->vlan == I40E_VLAN_ANY) {
2444	aq_ret = i40e_aq_set_vsi_broadcast(hw,
2445	vsi_id: vsi->seid,
2446	set_filter: enable,
2447	NULL);
2448	} else {
2449	aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2450	seid: vsi->seid,
2451	enable,
2452	vid: f->vlan,
2453	NULL);
2454	}
2455
2456	if (aq_ret) {
2457	set_bit(nr: __I40E_VSI_OVERFLOW_PROMISC, addr: vsi->state);
2458	dev_warn(&vsi->back->pdev->dev,
2459	"Error %s, forcing overflow promiscuous on %s\n",
2460	i40e_aq_str(hw, hw->aq.asq_last_status),
2461	vsi_name);
2462	}
2463
2464	return aq_ret;
2465	}
2466
2467	/**
2468	* i40e_set_promiscuous - set promiscuous mode
2469	* @pf: board private structure
2470	* @promisc: promisc on or off
2471	*
2472	* There are different ways of setting promiscuous mode on a PF depending on
2473	* what state/environment we're in. This identifies and sets it appropriately.
2474	* Returns 0 on success.
2475	**/
2476	static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2477	{
2478	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
2479	struct i40e_hw *hw = &pf->hw;
2480	int aq_ret;
2481
2482	if (vsi->type == I40E_VSI_MAIN &&
2483	pf->lan_veb != I40E_NO_VEB &&
2484	!test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
2485	/* set defport ON for Main VSI instead of true promisc
2486	* this way we will get all unicast/multicast and VLAN
2487	* promisc behavior but will not get VF or VMDq traffic
2488	* replicated on the Main VSI.
2489	*/
2490	if (promisc)
2491	aq_ret = i40e_aq_set_default_vsi(hw,
2492	vsi_id: vsi->seid,
2493	NULL);
2494	else
2495	aq_ret = i40e_aq_clear_default_vsi(hw,
2496	vsi_id: vsi->seid,
2497	NULL);
2498	if (aq_ret) {
2499	dev_info(&pf->pdev->dev,
2500	"Set default VSI failed, err %pe, aq_err %s\n",
2501	ERR_PTR(aq_ret),
2502	i40e_aq_str(hw, hw->aq.asq_last_status));
2503	}
2504	} else {
2505	aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2506	hw,
2507	vsi_id: vsi->seid,
2508	set: promisc, NULL,
2509	rx_only_promisc: true);
2510	if (aq_ret) {
2511	dev_info(&pf->pdev->dev,
2512	"set unicast promisc failed, err %pe, aq_err %s\n",
2513	ERR_PTR(aq_ret),
2514	i40e_aq_str(hw, hw->aq.asq_last_status));
2515	}
2516	aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2517	hw,
2518	vsi_id: vsi->seid,
2519	set: promisc, NULL);
2520	if (aq_ret) {
2521	dev_info(&pf->pdev->dev,
2522	"set multicast promisc failed, err %pe, aq_err %s\n",
2523	ERR_PTR(aq_ret),
2524	i40e_aq_str(hw, hw->aq.asq_last_status));
2525	}
2526	}
2527
2528	if (!aq_ret)
2529	pf->cur_promisc = promisc;
2530
2531	return aq_ret;
2532	}
2533
2534	/**
2535	* i40e_sync_vsi_filters - Update the VSI filter list to the HW
2536	* @vsi: ptr to the VSI
2537	*
2538	* Push any outstanding VSI filter changes through the AdminQ.
2539	*
2540	* Returns 0 or error value
2541	**/
2542	int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2543	{
2544	struct hlist_head tmp_add_list, tmp_del_list;
2545	struct i40e_mac_filter *f;
2546	struct i40e_new_mac_filter *new, *add_head = NULL;
2547	struct i40e_hw *hw = &vsi->back->hw;
2548	bool old_overflow, new_overflow;
2549	unsigned int failed_filters = 0;
2550	unsigned int vlan_filters = 0;
2551	char vsi_name[16] = "PF";
2552	int filter_list_len = 0;
2553	u32 changed_flags = 0;
2554	struct hlist_node *h;
2555	struct i40e_pf *pf;
2556	int num_add = 0;
2557	int num_del = 0;
2558	int aq_ret = 0;
2559	int retval = 0;
2560	u16 cmd_flags;
2561	int list_size;
2562	int bkt;
2563
2564	/* empty array typed pointers, kcalloc later */
2565	struct i40e_aqc_add_macvlan_element_data *add_list;
2566	struct i40e_aqc_remove_macvlan_element_data *del_list;
2567
2568	while (test_and_set_bit(nr: __I40E_VSI_SYNCING_FILTERS, addr: vsi->state))
2569	usleep_range(min: 1000, max: 2000);
2570	pf = vsi->back;
2571
2572	old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2573
2574	if (vsi->netdev) {
2575	changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2576	vsi->current_netdev_flags = vsi->netdev->flags;
2577	}
2578
2579	INIT_HLIST_HEAD(&tmp_add_list);
2580	INIT_HLIST_HEAD(&tmp_del_list);
2581
2582	if (vsi->type == I40E_VSI_SRIOV)
2583	snprintf(buf: vsi_name, size: sizeof(vsi_name) - 1, fmt: "VF %d", vsi->vf_id);
2584	else if (vsi->type != I40E_VSI_MAIN)
2585	snprintf(buf: vsi_name, size: sizeof(vsi_name) - 1, fmt: "vsi %d", vsi->seid);
2586
2587	if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2588	vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2589
2590	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
2591	/* Create a list of filters to delete. */
2592	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2593	if (f->state == I40E_FILTER_REMOVE) {
2594	/* Move the element into temporary del_list */
2595	hash_del(node: &f->hlist);
2596	hlist_add_head(n: &f->hlist, h: &tmp_del_list);
2597
2598	/* Avoid counting removed filters */
2599	continue;
2600	}
2601	if (f->state == I40E_FILTER_NEW) {
2602	/* Create a temporary i40e_new_mac_filter */
2603	new = kzalloc(size: sizeof(*new), GFP_ATOMIC);
2604	if (!new)
2605	goto err_no_memory_locked;
2606
2607	/* Store pointer to the real filter */
2608	new->f = f;
2609	new->state = f->state;
2610
2611	/* Add it to the hash list */
2612	hlist_add_head(n: &new->hlist, h: &tmp_add_list);
2613	}
2614
2615	/* Count the number of active (current and new) VLAN
2616	* filters we have now. Does not count filters which
2617	* are marked for deletion.
2618	*/
2619	if (f->vlan > 0)
2620	vlan_filters++;
2621	}
2622
2623	if (vsi->type != I40E_VSI_SRIOV)
2624	retval = i40e_correct_mac_vlan_filters
2625	(vsi, tmp_add_list: &tmp_add_list, tmp_del_list: &tmp_del_list,
2626	vlan_filters);
2627	else if (pf->vf)
2628	retval = i40e_correct_vf_mac_vlan_filters
2629	(vsi, tmp_add_list: &tmp_add_list, tmp_del_list: &tmp_del_list,
2630	vlan_filters, trusted: pf->vf[vsi->vf_id].trusted);
2631
2632	hlist_for_each_entry(new, &tmp_add_list, hlist)
2633	netdev_hw_addr_refcnt(f: new->f, netdev: vsi->netdev, delta: 1);
2634
2635	if (retval)
2636	goto err_no_memory_locked;
2637
2638	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
2639	}
2640
2641	/* Now process 'del_list' outside the lock */
2642	if (!hlist_empty(h: &tmp_del_list)) {
2643	filter_list_len = hw->aq.asq_buf_size /
2644	sizeof(struct i40e_aqc_remove_macvlan_element_data);
2645	list_size = filter_list_len *
2646	sizeof(struct i40e_aqc_remove_macvlan_element_data);
2647	del_list = kzalloc(size: list_size, GFP_ATOMIC);
2648	if (!del_list)
2649	goto err_no_memory;
2650
2651	hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2652	cmd_flags = 0;
2653
2654	/* handle broadcast filters by updating the broadcast
2655	* promiscuous flag and release filter list.
2656	*/
2657	if (is_broadcast_ether_addr(addr: f->macaddr)) {
2658	i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2659
2660	hlist_del(n: &f->hlist);
2661	kfree(objp: f);
2662	continue;
2663	}
2664
2665	/* add to delete list */
2666	ether_addr_copy(dst: del_list[num_del].mac_addr, src: f->macaddr);
2667	if (f->vlan == I40E_VLAN_ANY) {
2668	del_list[num_del].vlan_tag = 0;
2669	cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2670	} else {
2671	del_list[num_del].vlan_tag =
2672	cpu_to_le16((u16)(f->vlan));
2673	}
2674
2675	cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2676	del_list[num_del].flags = cmd_flags;
2677	num_del++;
2678
2679	/* flush a full buffer */
2680	if (num_del == filter_list_len) {
2681	i40e_aqc_del_filters(vsi, vsi_name, list: del_list,
2682	num_del, retval: &retval);
2683	memset(del_list, 0, list_size);
2684	num_del = 0;
2685	}
2686	/* Release memory for MAC filter entries which were
2687	* synced up with HW.
2688	*/
2689	hlist_del(n: &f->hlist);
2690	kfree(objp: f);
2691	}
2692
2693	if (num_del) {
2694	i40e_aqc_del_filters(vsi, vsi_name, list: del_list,
2695	num_del, retval: &retval);
2696	}
2697
2698	kfree(objp: del_list);
2699	del_list = NULL;
2700	}
2701
2702	if (!hlist_empty(h: &tmp_add_list)) {
2703	/* Do all the adds now. */
2704	filter_list_len = hw->aq.asq_buf_size /
2705	sizeof(struct i40e_aqc_add_macvlan_element_data);
2706	list_size = filter_list_len *
2707	sizeof(struct i40e_aqc_add_macvlan_element_data);
2708	add_list = kzalloc(size: list_size, GFP_ATOMIC);
2709	if (!add_list)
2710	goto err_no_memory;
2711
2712	num_add = 0;
2713	hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2714	/* handle broadcast filters by updating the broadcast
2715	* promiscuous flag instead of adding a MAC filter.
2716	*/
2717	if (is_broadcast_ether_addr(addr: new->f->macaddr)) {
2718	if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2719	f: new->f))
2720	new->state = I40E_FILTER_FAILED;
2721	else
2722	new->state = I40E_FILTER_ACTIVE;
2723	continue;
2724	}
2725
2726	/* add to add array */
2727	if (num_add == 0)
2728	add_head = new;
2729	cmd_flags = 0;
2730	ether_addr_copy(dst: add_list[num_add].mac_addr,
2731	src: new->f->macaddr);
2732	if (new->f->vlan == I40E_VLAN_ANY) {
2733	add_list[num_add].vlan_tag = 0;
2734	cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2735	} else {
2736	add_list[num_add].vlan_tag =
2737	cpu_to_le16((u16)(new->f->vlan));
2738	}
2739	add_list[num_add].queue_number = 0;
2740	/* set invalid match method for later detection */
2741	add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2742	cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2743	add_list[num_add].flags = cpu_to_le16(cmd_flags);
2744	num_add++;
2745
2746	/* flush a full buffer */
2747	if (num_add == filter_list_len) {
2748	i40e_aqc_add_filters(vsi, vsi_name, list: add_list,
2749	add_head, num_add);
2750	memset(add_list, 0, list_size);
2751	num_add = 0;
2752	}
2753	}
2754	if (num_add) {
2755	i40e_aqc_add_filters(vsi, vsi_name, list: add_list, add_head,
2756	num_add);
2757	}
2758	/* Now move all of the filters from the temp add list back to
2759	* the VSI's list.
2760	*/
2761	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
2762	hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2763	/* Only update the state if we're still NEW */
2764	if (new->f->state == I40E_FILTER_NEW)
2765	new->f->state = new->state;
2766	hlist_del(n: &new->hlist);
2767	netdev_hw_addr_refcnt(f: new->f, netdev: vsi->netdev, delta: -1);
2768	kfree(objp: new);
2769	}
2770	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
2771	kfree(objp: add_list);
2772	add_list = NULL;
2773	}
2774
2775	/* Determine the number of active and failed filters. */
2776	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
2777	vsi->active_filters = 0;
2778	hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2779	if (f->state == I40E_FILTER_ACTIVE)
2780	vsi->active_filters++;
2781	else if (f->state == I40E_FILTER_FAILED)
2782	failed_filters++;
2783	}
2784	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
2785
2786	/* Check if we are able to exit overflow promiscuous mode. We can
2787	* safely exit if we didn't just enter, we no longer have any failed
2788	* filters, and we have reduced filters below the threshold value.
2789	*/
2790	if (old_overflow && !failed_filters &&
2791	vsi->active_filters < vsi->promisc_threshold) {
2792	dev_info(&pf->pdev->dev,
2793	"filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2794	vsi_name);
2795	clear_bit(nr: __I40E_VSI_OVERFLOW_PROMISC, addr: vsi->state);
2796	vsi->promisc_threshold = 0;
2797	}
2798
2799	/* if the VF is not trusted do not do promisc */
2800	if (vsi->type == I40E_VSI_SRIOV && pf->vf &&
2801	!pf->vf[vsi->vf_id].trusted) {
2802	clear_bit(nr: __I40E_VSI_OVERFLOW_PROMISC, addr: vsi->state);
2803	goto out;
2804	}
2805
2806	new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2807
2808	/* If we are entering overflow promiscuous, we need to calculate a new
2809	* threshold for when we are safe to exit
2810	*/
2811	if (!old_overflow && new_overflow)
2812	vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2813
2814	/* check for changes in promiscuous modes */
2815	if (changed_flags & IFF_ALLMULTI) {
2816	bool cur_multipromisc;
2817
2818	cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2819	aq_ret = i40e_aq_set_vsi_multicast_promiscuous(hw: &vsi->back->hw,
2820	vsi_id: vsi->seid,
2821	set: cur_multipromisc,
2822	NULL);
2823	if (aq_ret) {
2824	retval = i40e_aq_rc_to_posix(aq_ret,
2825	aq_rc: hw->aq.asq_last_status);
2826	dev_info(&pf->pdev->dev,
2827	"set multi promisc failed on %s, err %pe aq_err %s\n",
2828	vsi_name,
2829	ERR_PTR(aq_ret),
2830	i40e_aq_str(hw, hw->aq.asq_last_status));
2831	} else {
2832	dev_info(&pf->pdev->dev, "%s allmulti mode.\n",
2833	cur_multipromisc ? "entering" : "leaving");
2834	}
2835	}
2836
2837	if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2838	bool cur_promisc;
2839
2840	cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2841	new_overflow);
2842	aq_ret = i40e_set_promiscuous(pf, promisc: cur_promisc);
2843	if (aq_ret) {
2844	retval = i40e_aq_rc_to_posix(aq_ret,
2845	aq_rc: hw->aq.asq_last_status);
2846	dev_info(&pf->pdev->dev,
2847	"Setting promiscuous %s failed on %s, err %pe aq_err %s\n",
2848	cur_promisc ? "on" : "off",
2849	vsi_name,
2850	ERR_PTR(aq_ret),
2851	i40e_aq_str(hw, hw->aq.asq_last_status));
2852	}
2853	}
2854	out:
2855	/* if something went wrong then set the changed flag so we try again */
2856	if (retval)
2857	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2858
2859	clear_bit(nr: __I40E_VSI_SYNCING_FILTERS, addr: vsi->state);
2860	return retval;
2861
2862	err_no_memory:
2863	/* Restore elements on the temporary add and delete lists */
2864	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
2865	err_no_memory_locked:
2866	i40e_undo_del_filter_entries(vsi, from: &tmp_del_list);
2867	i40e_undo_add_filter_entries(vsi, from: &tmp_add_list);
2868	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
2869
2870	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2871	clear_bit(nr: __I40E_VSI_SYNCING_FILTERS, addr: vsi->state);
2872	return -ENOMEM;
2873	}
2874
2875	/**
2876	* i40e_sync_filters_subtask - Sync the VSI filter list with HW
2877	* @pf: board private structure
2878	**/
2879	static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2880	{
2881	struct i40e_vsi *vsi;
2882	int v;
2883
2884	if (!pf)
2885	return;
2886	if (!test_and_clear_bit(nr: __I40E_MACVLAN_SYNC_PENDING, addr: pf->state))
2887	return;
2888	if (test_bit(__I40E_VF_DISABLE, pf->state)) {
2889	set_bit(nr: __I40E_MACVLAN_SYNC_PENDING, addr: pf->state);
2890	return;
2891	}
2892
2893	i40e_pf_for_each_vsi(pf, v, vsi) {
2894	if ((vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&
2895	!test_bit(__I40E_VSI_RELEASING, vsi->state)) {
2896	int ret = i40e_sync_vsi_filters(vsi);
2897
2898	if (ret) {
2899	/* come back and try again later */
2900	set_bit(nr: __I40E_MACVLAN_SYNC_PENDING,
2901	addr: pf->state);
2902	break;
2903	}
2904	}
2905	}
2906	}
2907
2908	/**
2909	* i40e_calculate_vsi_rx_buf_len - Calculates buffer length
2910	*
2911	* @vsi: VSI to calculate rx_buf_len from
2912	*/
2913	static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi)
2914	{
2915	if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags))
2916	return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048);
2917
2918	return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048;
2919	}
2920
2921	/**
2922	* i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI
2923	* @vsi: the vsi
2924	* @xdp_prog: XDP program
2925	**/
2926	static int i40e_max_vsi_frame_size(struct i40e_vsi *vsi,
2927	struct bpf_prog *xdp_prog)
2928	{
2929	u16 rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
2930	u16 chain_len;
2931
2932	if (xdp_prog && !xdp_prog->aux->xdp_has_frags)
2933	chain_len = 1;
2934	else
2935	chain_len = I40E_MAX_CHAINED_RX_BUFFERS;
2936
2937	return min_t(u16, rx_buf_len * chain_len, I40E_MAX_RXBUFFER);
2938	}
2939
2940	/**
2941	* i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2942	* @netdev: network interface device structure
2943	* @new_mtu: new value for maximum frame size
2944	*
2945	* Returns 0 on success, negative on failure
2946	**/
2947	static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2948	{
2949	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
2950	struct i40e_vsi *vsi = np->vsi;
2951	struct i40e_pf *pf = vsi->back;
2952	int frame_size;
2953
2954	frame_size = i40e_max_vsi_frame_size(vsi, xdp_prog: vsi->xdp_prog);
2955	if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) {
2956	netdev_err(dev: netdev, format: "Error changing mtu to %d, Max is %d\n",
2957	new_mtu, frame_size - I40E_PACKET_HDR_PAD);
2958	return -EINVAL;
2959	}
2960
2961	netdev_dbg(netdev, "changing MTU from %d to %d\n",
2962	netdev->mtu, new_mtu);
2963	netdev->mtu = new_mtu;
2964	if (netif_running(dev: netdev))
2965	i40e_vsi_reinit_locked(vsi);
2966	set_bit(nr: __I40E_CLIENT_SERVICE_REQUESTED, addr: pf->state);
2967	set_bit(nr: __I40E_CLIENT_L2_CHANGE, addr: pf->state);
2968	return 0;
2969	}
2970
2971	/**
2972	* i40e_ioctl - Access the hwtstamp interface
2973	* @netdev: network interface device structure
2974	* @ifr: interface request data
2975	* @cmd: ioctl command
2976	**/
2977	int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2978	{
2979	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
2980	struct i40e_pf *pf = np->vsi->back;
2981
2982	switch (cmd) {
2983	case SIOCGHWTSTAMP:
2984	return i40e_ptp_get_ts_config(pf, ifr);
2985	case SIOCSHWTSTAMP:
2986	return i40e_ptp_set_ts_config(pf, ifr);
2987	default:
2988	return -EOPNOTSUPP;
2989	}
2990	}
2991
2992	/**
2993	* i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2994	* @vsi: the vsi being adjusted
2995	**/
2996	void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2997	{
2998	struct i40e_vsi_context ctxt;
2999	int ret;
3000
3001	/* Don't modify stripping options if a port VLAN is active */
3002	if (vsi->info.pvid)
3003	return;
3004
3005	if ((vsi->info.valid_sections &
3006	cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
3007	((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
3008	return; /* already enabled */
3009
3010	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3011	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
3012	I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
3013
3014	ctxt.seid = vsi->seid;
3015	ctxt.info = vsi->info;
3016	ret = i40e_aq_update_vsi_params(hw: &vsi->back->hw, vsi_ctx: &ctxt, NULL);
3017	if (ret) {
3018	dev_info(&vsi->back->pdev->dev,
3019	"update vlan stripping failed, err %pe aq_err %s\n",
3020	ERR_PTR(ret),
3021	i40e_aq_str(&vsi->back->hw,
3022	vsi->back->hw.aq.asq_last_status));
3023	}
3024	}
3025
3026	/**
3027	* i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
3028	* @vsi: the vsi being adjusted
3029	**/
3030	void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
3031	{
3032	struct i40e_vsi_context ctxt;
3033	int ret;
3034
3035	/* Don't modify stripping options if a port VLAN is active */
3036	if (vsi->info.pvid)
3037	return;
3038
3039	if ((vsi->info.valid_sections &
3040	cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
3041	((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
3042	I40E_AQ_VSI_PVLAN_EMOD_MASK))
3043	return; /* already disabled */
3044
3045	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3046	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
3047	I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3048
3049	ctxt.seid = vsi->seid;
3050	ctxt.info = vsi->info;
3051	ret = i40e_aq_update_vsi_params(hw: &vsi->back->hw, vsi_ctx: &ctxt, NULL);
3052	if (ret) {
3053	dev_info(&vsi->back->pdev->dev,
3054	"update vlan stripping failed, err %pe aq_err %s\n",
3055	ERR_PTR(ret),
3056	i40e_aq_str(&vsi->back->hw,
3057	vsi->back->hw.aq.asq_last_status));
3058	}
3059	}
3060
3061	/**
3062	* i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
3063	* @vsi: the vsi being configured
3064	* @vid: vlan id to be added (0 = untagged only , -1 = any)
3065	*
3066	* This is a helper function for adding a new MAC/VLAN filter with the
3067	* specified VLAN for each existing MAC address already in the hash table.
3068	* This function does *not* perform any accounting to update filters based on
3069	* VLAN mode.
3070	*
3071	* NOTE: this function expects to be called while under the
3072	* mac_filter_hash_lock
3073	**/
3074	int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3075	{
3076	struct i40e_mac_filter *f, *add_f;
3077	struct hlist_node *h;
3078	int bkt;
3079
3080	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3081	/* If we're asked to add a filter that has been marked for
3082	* removal, it is safe to simply restore it to active state.
3083	* __i40e_del_filter will have simply deleted any filters which
3084	* were previously marked NEW or FAILED, so if it is currently
3085	* marked REMOVE it must have previously been ACTIVE. Since we
3086	* haven't yet run the sync filters task, just restore this
3087	* filter to the ACTIVE state so that the sync task leaves it
3088	* in place.
3089	*/
3090	if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) {
3091	f->state = I40E_FILTER_ACTIVE;
3092	continue;
3093	} else if (f->state == I40E_FILTER_REMOVE) {
3094	continue;
3095	}
3096	add_f = i40e_add_filter(vsi, macaddr: f->macaddr, vlan: vid);
3097	if (!add_f) {
3098	dev_info(&vsi->back->pdev->dev,
3099	"Could not add vlan filter %d for %pM\n",
3100	vid, f->macaddr);
3101	return -ENOMEM;
3102	}
3103	}
3104
3105	return 0;
3106	}
3107
3108	/**
3109	* i40e_vsi_add_vlan - Add VSI membership for given VLAN
3110	* @vsi: the VSI being configured
3111	* @vid: VLAN id to be added
3112	**/
3113	int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
3114	{
3115	int err;
3116
3117	if (vsi->info.pvid)
3118	return -EINVAL;
3119
3120	/* The network stack will attempt to add VID=0, with the intention to
3121	* receive priority tagged packets with a VLAN of 0. Our HW receives
3122	* these packets by default when configured to receive untagged
3123	* packets, so we don't need to add a filter for this case.
3124	* Additionally, HW interprets adding a VID=0 filter as meaning to
3125	* receive *only* tagged traffic and stops receiving untagged traffic.
3126	* Thus, we do not want to actually add a filter for VID=0
3127	*/
3128	if (!vid)
3129	return 0;
3130
3131	/* Locked once because all functions invoked below iterates list*/
3132	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
3133	err = i40e_add_vlan_all_mac(vsi, vid);
3134	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
3135	if (err)
3136	return err;
3137
3138	/* schedule our worker thread which will take care of
3139	* applying the new filter changes
3140	*/
3141	i40e_service_event_schedule(pf: vsi->back);
3142	return 0;
3143	}
3144
3145	/**
3146	* i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
3147	* @vsi: the vsi being configured
3148	* @vid: vlan id to be removed (0 = untagged only , -1 = any)
3149	*
3150	* This function should be used to remove all VLAN filters which match the
3151	* given VID. It does not schedule the service event and does not take the
3152	* mac_filter_hash_lock so it may be combined with other operations under
3153	* a single invocation of the mac_filter_hash_lock.
3154	*
3155	* NOTE: this function expects to be called while under the
3156	* mac_filter_hash_lock
3157	*/
3158	void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3159	{
3160	struct i40e_mac_filter *f;
3161	struct hlist_node *h;
3162	int bkt;
3163
3164	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3165	if (f->vlan == vid)
3166	__i40e_del_filter(vsi, f);
3167	}
3168	}
3169
3170	/**
3171	* i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
3172	* @vsi: the VSI being configured
3173	* @vid: VLAN id to be removed
3174	**/
3175	void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
3176	{
3177	if (!vid || vsi->info.pvid)
3178	return;
3179
3180	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
3181	i40e_rm_vlan_all_mac(vsi, vid);
3182	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
3183
3184	/* schedule our worker thread which will take care of
3185	* applying the new filter changes
3186	*/
3187	i40e_service_event_schedule(pf: vsi->back);
3188	}
3189
3190	/**
3191	* i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
3192	* @netdev: network interface to be adjusted
3193	* @proto: unused protocol value
3194	* @vid: vlan id to be added
3195	*
3196	* net_device_ops implementation for adding vlan ids
3197	**/
3198	static int i40e_vlan_rx_add_vid(struct net_device *netdev,
3199	__always_unused __be16 proto, u16 vid)
3200	{
3201	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
3202	struct i40e_vsi *vsi = np->vsi;
3203	int ret = 0;
3204
3205	if (vid >= VLAN_N_VID)
3206	return -EINVAL;
3207
3208	ret = i40e_vsi_add_vlan(vsi, vid);
3209	if (!ret)
3210	set_bit(nr: vid, addr: vsi->active_vlans);
3211
3212	return ret;
3213	}
3214
3215	/**
3216	* i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
3217	* @netdev: network interface to be adjusted
3218	* @proto: unused protocol value
3219	* @vid: vlan id to be added
3220	**/
3221	static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
3222	__always_unused __be16 proto, u16 vid)
3223	{
3224	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
3225	struct i40e_vsi *vsi = np->vsi;
3226
3227	if (vid >= VLAN_N_VID)
3228	return;
3229	set_bit(nr: vid, addr: vsi->active_vlans);
3230	}
3231
3232	/**
3233	* i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
3234	* @netdev: network interface to be adjusted
3235	* @proto: unused protocol value
3236	* @vid: vlan id to be removed
3237	*
3238	* net_device_ops implementation for removing vlan ids
3239	**/
3240	static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
3241	__always_unused __be16 proto, u16 vid)
3242	{
3243	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
3244	struct i40e_vsi *vsi = np->vsi;
3245
3246	/* return code is ignored as there is nothing a user
3247	* can do about failure to remove and a log message was
3248	* already printed from the other function
3249	*/
3250	i40e_vsi_kill_vlan(vsi, vid);
3251
3252	clear_bit(nr: vid, addr: vsi->active_vlans);
3253
3254	return 0;
3255	}
3256
3257	/**
3258	* i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
3259	* @vsi: the vsi being brought back up
3260	**/
3261	static void i40e_restore_vlan(struct i40e_vsi *vsi)
3262	{
3263	u16 vid;
3264
3265	if (!vsi->netdev)
3266	return;
3267
3268	if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
3269	i40e_vlan_stripping_enable(vsi);
3270	else
3271	i40e_vlan_stripping_disable(vsi);
3272
3273	for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
3274	i40e_vlan_rx_add_vid_up(netdev: vsi->netdev, htons(ETH_P_8021Q),
3275	vid);
3276	}
3277
3278	/**
3279	* i40e_vsi_add_pvid - Add pvid for the VSI
3280	* @vsi: the vsi being adjusted
3281	* @vid: the vlan id to set as a PVID
3282	**/
3283	int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
3284	{
3285	struct i40e_vsi_context ctxt;
3286	int ret;
3287
3288	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3289	vsi->info.pvid = cpu_to_le16(vid);
3290	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
3291	I40E_AQ_VSI_PVLAN_INSERT_PVID |
3292	I40E_AQ_VSI_PVLAN_EMOD_STR;
3293
3294	ctxt.seid = vsi->seid;
3295	ctxt.info = vsi->info;
3296	ret = i40e_aq_update_vsi_params(hw: &vsi->back->hw, vsi_ctx: &ctxt, NULL);
3297	if (ret) {
3298	dev_info(&vsi->back->pdev->dev,
3299	"add pvid failed, err %pe aq_err %s\n",
3300	ERR_PTR(ret),
3301	i40e_aq_str(&vsi->back->hw,
3302	vsi->back->hw.aq.asq_last_status));
3303	return -ENOENT;
3304	}
3305
3306	return 0;
3307	}
3308
3309	/**
3310	* i40e_vsi_remove_pvid - Remove the pvid from the VSI
3311	* @vsi: the vsi being adjusted
3312	*
3313	* Just use the vlan_rx_register() service to put it back to normal
3314	**/
3315	void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3316	{
3317	vsi->info.pvid = 0;
3318
3319	i40e_vlan_stripping_disable(vsi);
3320	}
3321
3322	/**
3323	* i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3324	* @vsi: ptr to the VSI
3325	*
3326	* If this function returns with an error, then it's possible one or
3327	* more of the rings is populated (while the rest are not). It is the
3328	* callers duty to clean those orphaned rings.
3329	*
3330	* Return 0 on success, negative on failure
3331	**/
3332	static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3333	{
3334	int i, err = 0;
3335
3336	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3337	err = i40e_setup_tx_descriptors(tx_ring: vsi->tx_rings[i]);
3338
3339	if (!i40e_enabled_xdp_vsi(vsi))
3340	return err;
3341
3342	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3343	err = i40e_setup_tx_descriptors(tx_ring: vsi->xdp_rings[i]);
3344
3345	return err;
3346	}
3347
3348	/**
3349	* i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3350	* @vsi: ptr to the VSI
3351	*
3352	* Free VSI's transmit software resources
3353	**/
3354	static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3355	{
3356	int i;
3357
3358	if (vsi->tx_rings) {
3359	for (i = 0; i < vsi->num_queue_pairs; i++)
3360	if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3361	i40e_free_tx_resources(tx_ring: vsi->tx_rings[i]);
3362	}
3363
3364	if (vsi->xdp_rings) {
3365	for (i = 0; i < vsi->num_queue_pairs; i++)
3366	if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3367	i40e_free_tx_resources(tx_ring: vsi->xdp_rings[i]);
3368	}
3369	}
3370
3371	/**
3372	* i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3373	* @vsi: ptr to the VSI
3374	*
3375	* If this function returns with an error, then it's possible one or
3376	* more of the rings is populated (while the rest are not). It is the
3377	* callers duty to clean those orphaned rings.
3378	*
3379	* Return 0 on success, negative on failure
3380	**/
3381	static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3382	{
3383	int i, err = 0;
3384
3385	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3386	err = i40e_setup_rx_descriptors(rx_ring: vsi->rx_rings[i]);
3387	return err;
3388	}
3389
3390	/**
3391	* i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3392	* @vsi: ptr to the VSI
3393	*
3394	* Free all receive software resources
3395	**/
3396	static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3397	{
3398	int i;
3399
3400	if (!vsi->rx_rings)
3401	return;
3402
3403	for (i = 0; i < vsi->num_queue_pairs; i++)
3404	if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3405	i40e_free_rx_resources(rx_ring: vsi->rx_rings[i]);
3406	}
3407
3408	/**
3409	* i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3410	* @ring: The Tx ring to configure
3411	*
3412	* This enables/disables XPS for a given Tx descriptor ring
3413	* based on the TCs enabled for the VSI that ring belongs to.
3414	**/
3415	static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3416	{
3417	int cpu;
3418
3419	if (!ring->q_vector || !ring->netdev || ring->ch)
3420	return;
3421
3422	/* We only initialize XPS once, so as not to overwrite user settings */
3423	if (test_and_set_bit(nr: __I40E_TX_XPS_INIT_DONE, addr: ring->state))
3424	return;
3425
3426	cpu = cpumask_local_spread(i: ring->q_vector->v_idx, node: -1);
3427	netif_set_xps_queue(dev: ring->netdev, mask: get_cpu_mask(cpu),
3428	index: ring->queue_index);
3429	}
3430
3431	/**
3432	* i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled
3433	* @ring: The Tx or Rx ring
3434	*
3435	* Returns the AF_XDP buffer pool or NULL.
3436	**/
3437	static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring)
3438	{
3439	bool xdp_on = i40e_enabled_xdp_vsi(vsi: ring->vsi);
3440	int qid = ring->queue_index;
3441
3442	if (ring_is_xdp(ring))
3443	qid -= ring->vsi->alloc_queue_pairs;
3444
3445	if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3446	return NULL;
3447
3448	return xsk_get_pool_from_qid(dev: ring->vsi->netdev, queue_id: qid);
3449	}
3450
3451	/**
3452	* i40e_configure_tx_ring - Configure a transmit ring context and rest
3453	* @ring: The Tx ring to configure
3454	*
3455	* Configure the Tx descriptor ring in the HMC context.
3456	**/
3457	static int i40e_configure_tx_ring(struct i40e_ring *ring)
3458	{
3459	struct i40e_vsi *vsi = ring->vsi;
3460	u16 pf_q = vsi->base_queue + ring->queue_index;
3461	struct i40e_hw *hw = &vsi->back->hw;
3462	struct i40e_hmc_obj_txq tx_ctx;
3463	u32 qtx_ctl = 0;
3464	int err = 0;
3465
3466	if (ring_is_xdp(ring))
3467	ring->xsk_pool = i40e_xsk_pool(ring);
3468
3469	/* some ATR related tx ring init */
3470	if (test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) {
3471	ring->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
3472	ring->atr_count = 0;
3473	} else {
3474	ring->atr_sample_rate = 0;
3475	}
3476
3477	/* configure XPS */
3478	i40e_config_xps_tx_ring(ring);
3479
3480	/* clear the context structure first */
3481	memset(&tx_ctx, 0, sizeof(tx_ctx));
3482
3483	tx_ctx.new_context = 1;
3484	tx_ctx.base = (ring->dma / 128);
3485	tx_ctx.qlen = ring->count;
3486	if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags) ||
3487	test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags))
3488	tx_ctx.fd_ena = 1;
3489	if (test_bit(I40E_FLAG_PTP_ENA, vsi->back->flags))
3490	tx_ctx.timesync_ena = 1;
3491	/* FDIR VSI tx ring can still use RS bit and writebacks */
3492	if (vsi->type != I40E_VSI_FDIR)
3493	tx_ctx.head_wb_ena = 1;
3494	tx_ctx.head_wb_addr = ring->dma +
3495	(ring->count * sizeof(struct i40e_tx_desc));
3496
3497	/* As part of VSI creation/update, FW allocates certain
3498	* Tx arbitration queue sets for each TC enabled for
3499	* the VSI. The FW returns the handles to these queue
3500	* sets as part of the response buffer to Add VSI,
3501	* Update VSI, etc. AQ commands. It is expected that
3502	* these queue set handles be associated with the Tx
3503	* queues by the driver as part of the TX queue context
3504	* initialization. This has to be done regardless of
3505	* DCB as by default everything is mapped to TC0.
3506	*/
3507
3508	if (ring->ch)
3509	tx_ctx.rdylist =
3510	le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3511
3512	else
3513	tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3514
3515	tx_ctx.rdylist_act = 0;
3516
3517	/* clear the context in the HMC */
3518	err = i40e_clear_lan_tx_queue_context(hw, queue: pf_q);
3519	if (err) {
3520	dev_info(&vsi->back->pdev->dev,
3521	"Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3522	ring->queue_index, pf_q, err);
3523	return -ENOMEM;
3524	}
3525
3526	/* set the context in the HMC */
3527	err = i40e_set_lan_tx_queue_context(hw, queue: pf_q, s: &tx_ctx);
3528	if (err) {
3529	dev_info(&vsi->back->pdev->dev,
3530	"Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3531	ring->queue_index, pf_q, err);
3532	return -ENOMEM;
3533	}
3534
3535	/* Now associate this queue with this PCI function */
3536	if (ring->ch) {
3537	if (ring->ch->type == I40E_VSI_VMDQ2)
3538	qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3539	else
3540	return -EINVAL;
3541
3542	qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK,
3543	ring->ch->vsi_number);
3544	} else {
3545	if (vsi->type == I40E_VSI_VMDQ2) {
3546	qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3547	qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK,
3548	vsi->id);
3549	} else {
3550	qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3551	}
3552	}
3553
3554	qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_PF_INDX_MASK, hw->pf_id);
3555	wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3556	i40e_flush(hw);
3557
3558	/* cache tail off for easier writes later */
3559	ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3560
3561	return 0;
3562	}
3563
3564	/**
3565	* i40e_rx_offset - Return expected offset into page to access data
3566	* @rx_ring: Ring we are requesting offset of
3567	*
3568	* Returns the offset value for ring into the data buffer.
3569	*/
3570	static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
3571	{
3572	return ring_uses_build_skb(ring: rx_ring) ? I40E_SKB_PAD : 0;
3573	}
3574
3575	/**
3576	* i40e_configure_rx_ring - Configure a receive ring context
3577	* @ring: The Rx ring to configure
3578	*
3579	* Configure the Rx descriptor ring in the HMC context.
3580	**/
3581	static int i40e_configure_rx_ring(struct i40e_ring *ring)
3582	{
3583	struct i40e_vsi *vsi = ring->vsi;
3584	u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3585	u16 pf_q = vsi->base_queue + ring->queue_index;
3586	struct i40e_hw *hw = &vsi->back->hw;
3587	struct i40e_hmc_obj_rxq rx_ctx;
3588	int err = 0;
3589	bool ok;
3590
3591	bitmap_zero(dst: ring->state, nbits: __I40E_RING_STATE_NBITS);
3592
3593	/* clear the context structure first */
3594	memset(&rx_ctx, 0, sizeof(rx_ctx));
3595
3596	ring->rx_buf_len = vsi->rx_buf_len;
3597
3598	/* XDP RX-queue info only needed for RX rings exposed to XDP */
3599	if (ring->vsi->type != I40E_VSI_MAIN)
3600	goto skip;
3601
3602	if (!xdp_rxq_info_is_reg(xdp_rxq: &ring->xdp_rxq)) {
3603	err = __xdp_rxq_info_reg(xdp_rxq: &ring->xdp_rxq, dev: ring->netdev,
3604	queue_index: ring->queue_index,
3605	napi_id: ring->q_vector->napi.napi_id,
3606	frag_size: ring->rx_buf_len);
3607	if (err)
3608	return err;
3609	}
3610
3611	ring->xsk_pool = i40e_xsk_pool(ring);
3612	if (ring->xsk_pool) {
3613	xdp_rxq_info_unreg(xdp_rxq: &ring->xdp_rxq);
3614	ring->rx_buf_len = xsk_pool_get_rx_frame_size(pool: ring->xsk_pool);
3615	err = __xdp_rxq_info_reg(xdp_rxq: &ring->xdp_rxq, dev: ring->netdev,
3616	queue_index: ring->queue_index,
3617	napi_id: ring->q_vector->napi.napi_id,
3618	frag_size: ring->rx_buf_len);
3619	if (err)
3620	return err;
3621	err = xdp_rxq_info_reg_mem_model(xdp_rxq: &ring->xdp_rxq,
3622	type: MEM_TYPE_XSK_BUFF_POOL,
3623	NULL);
3624	if (err)
3625	return err;
3626	dev_info(&vsi->back->pdev->dev,
3627	"Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
3628	ring->queue_index);
3629
3630	} else {
3631	err = xdp_rxq_info_reg_mem_model(xdp_rxq: &ring->xdp_rxq,
3632	type: MEM_TYPE_PAGE_SHARED,
3633	NULL);
3634	if (err)
3635	return err;
3636	}
3637
3638	skip:
3639	xdp_init_buff(xdp: &ring->xdp, i40e_rx_pg_size(ring) / 2, rxq: &ring->xdp_rxq);
3640
3641	rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3642	BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3643
3644	rx_ctx.base = (ring->dma / 128);
3645	rx_ctx.qlen = ring->count;
3646
3647	/* use 16 byte descriptors */
3648	rx_ctx.dsize = 0;
3649
3650	/* descriptor type is always zero
3651	* rx_ctx.dtype = 0;
3652	*/
3653	rx_ctx.hsplit_0 = 0;
3654
3655	rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3656	if (hw->revision_id == 0)
3657	rx_ctx.lrxqthresh = 0;
3658	else
3659	rx_ctx.lrxqthresh = 1;
3660	rx_ctx.crcstrip = 1;
3661	rx_ctx.l2tsel = 1;
3662	/* this controls whether VLAN is stripped from inner headers */
3663	rx_ctx.showiv = 0;
3664	/* set the prefena field to 1 because the manual says to */
3665	rx_ctx.prefena = 1;
3666
3667	/* clear the context in the HMC */
3668	err = i40e_clear_lan_rx_queue_context(hw, queue: pf_q);
3669	if (err) {
3670	dev_info(&vsi->back->pdev->dev,
3671	"Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3672	ring->queue_index, pf_q, err);
3673	return -ENOMEM;
3674	}
3675
3676	/* set the context in the HMC */
3677	err = i40e_set_lan_rx_queue_context(hw, queue: pf_q, s: &rx_ctx);
3678	if (err) {
3679	dev_info(&vsi->back->pdev->dev,
3680	"Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3681	ring->queue_index, pf_q, err);
3682	return -ENOMEM;
3683	}
3684
3685	/* configure Rx buffer alignment */
3686	if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) {
3687	if (I40E_2K_TOO_SMALL_WITH_PADDING) {
3688	dev_info(&vsi->back->pdev->dev,
3689	"2k Rx buffer is too small to fit standard MTU and skb_shared_info\n");
3690	return -EOPNOTSUPP;
3691	}
3692	clear_ring_build_skb_enabled(ring);
3693	} else {
3694	set_ring_build_skb_enabled(ring);
3695	}
3696
3697	ring->rx_offset = i40e_rx_offset(rx_ring: ring);
3698
3699	/* cache tail for quicker writes, and clear the reg before use */
3700	ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3701	writel(val: 0, addr: ring->tail);
3702
3703	if (ring->xsk_pool) {
3704	xsk_pool_set_rxq_info(pool: ring->xsk_pool, rxq: &ring->xdp_rxq);
3705	ok = i40e_alloc_rx_buffers_zc(rx_ring: ring, I40E_DESC_UNUSED(ring));
3706	} else {
3707	ok = !i40e_alloc_rx_buffers(rxr: ring, I40E_DESC_UNUSED(ring));
3708	}
3709	if (!ok) {
3710	/* Log this in case the user has forgotten to give the kernel
3711	* any buffers, even later in the application.
3712	*/
3713	dev_info(&vsi->back->pdev->dev,
3714	"Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3715	ring->xsk_pool ? "AF_XDP ZC enabled " : "",
3716	ring->queue_index, pf_q);
3717	}
3718
3719	return 0;
3720	}
3721
3722	/**
3723	* i40e_vsi_configure_tx - Configure the VSI for Tx
3724	* @vsi: VSI structure describing this set of rings and resources
3725	*
3726	* Configure the Tx VSI for operation.
3727	**/
3728	static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3729	{
3730	int err = 0;
3731	u16 i;
3732
3733	for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3734	err = i40e_configure_tx_ring(ring: vsi->tx_rings[i]);
3735
3736	if (err || !i40e_enabled_xdp_vsi(vsi))
3737	return err;
3738
3739	for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3740	err = i40e_configure_tx_ring(ring: vsi->xdp_rings[i]);
3741
3742	return err;
3743	}
3744
3745	/**
3746	* i40e_vsi_configure_rx - Configure the VSI for Rx
3747	* @vsi: the VSI being configured
3748	*
3749	* Configure the Rx VSI for operation.
3750	**/
3751	static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3752	{
3753	int err = 0;
3754	u16 i;
3755
3756	vsi->max_frame = i40e_max_vsi_frame_size(vsi, xdp_prog: vsi->xdp_prog);
3757	vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
3758
3759	#if (PAGE_SIZE < 8192)
3760	if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING &&
3761	vsi->netdev->mtu <= ETH_DATA_LEN) {
3762	vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3763	vsi->max_frame = vsi->rx_buf_len;
3764	}
3765	#endif
3766
3767	/* set up individual rings */
3768	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3769	err = i40e_configure_rx_ring(ring: vsi->rx_rings[i]);
3770
3771	return err;
3772	}
3773
3774	/**
3775	* i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3776	* @vsi: ptr to the VSI
3777	**/
3778	static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3779	{
3780	struct i40e_ring *tx_ring, *rx_ring;
3781	u16 qoffset, qcount;
3782	int i, n;
3783
3784	if (!test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {
3785	/* Reset the TC information */
3786	for (i = 0; i < vsi->num_queue_pairs; i++) {
3787	rx_ring = vsi->rx_rings[i];
3788	tx_ring = vsi->tx_rings[i];
3789	rx_ring->dcb_tc = 0;
3790	tx_ring->dcb_tc = 0;
3791	}
3792	return;
3793	}
3794
3795	for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3796	if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3797	continue;
3798
3799	qoffset = vsi->tc_config.tc_info[n].qoffset;
3800	qcount = vsi->tc_config.tc_info[n].qcount;
3801	for (i = qoffset; i < (qoffset + qcount); i++) {
3802	rx_ring = vsi->rx_rings[i];
3803	tx_ring = vsi->tx_rings[i];
3804	rx_ring->dcb_tc = n;
3805	tx_ring->dcb_tc = n;
3806	}
3807	}
3808	}
3809
3810	/**
3811	* i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3812	* @vsi: ptr to the VSI
3813	**/
3814	static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3815	{
3816	if (vsi->netdev)
3817	i40e_set_rx_mode(netdev: vsi->netdev);
3818	}
3819
3820	/**
3821	* i40e_reset_fdir_filter_cnt - Reset flow director filter counters
3822	* @pf: Pointer to the targeted PF
3823	*
3824	* Set all flow director counters to 0.
3825	*/
3826	static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf)
3827	{
3828	pf->fd_tcp4_filter_cnt = 0;
3829	pf->fd_udp4_filter_cnt = 0;
3830	pf->fd_sctp4_filter_cnt = 0;
3831	pf->fd_ip4_filter_cnt = 0;
3832	pf->fd_tcp6_filter_cnt = 0;
3833	pf->fd_udp6_filter_cnt = 0;
3834	pf->fd_sctp6_filter_cnt = 0;
3835	pf->fd_ip6_filter_cnt = 0;
3836	}
3837
3838	/**
3839	* i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3840	* @vsi: Pointer to the targeted VSI
3841	*
3842	* This function replays the hlist on the hw where all the SB Flow Director
3843	* filters were saved.
3844	**/
3845	static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3846	{
3847	struct i40e_fdir_filter *filter;
3848	struct i40e_pf *pf = vsi->back;
3849	struct hlist_node *node;
3850
3851	if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
3852	return;
3853
3854	/* Reset FDir counters as we're replaying all existing filters */
3855	i40e_reset_fdir_filter_cnt(pf);
3856
3857	hlist_for_each_entry_safe(filter, node,
3858	&pf->fdir_filter_list, fdir_node) {
3859	i40e_add_del_fdir(vsi, input: filter, add: true);
3860	}
3861	}
3862
3863	/**
3864	* i40e_vsi_configure - Set up the VSI for action
3865	* @vsi: the VSI being configured
3866	**/
3867	static int i40e_vsi_configure(struct i40e_vsi *vsi)
3868	{
3869	int err;
3870
3871	i40e_set_vsi_rx_mode(vsi);
3872	i40e_restore_vlan(vsi);
3873	i40e_vsi_config_dcb_rings(vsi);
3874	err = i40e_vsi_configure_tx(vsi);
3875	if (!err)
3876	err = i40e_vsi_configure_rx(vsi);
3877
3878	return err;
3879	}
3880
3881	/**
3882	* i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3883	* @vsi: the VSI being configured
3884	**/
3885	static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3886	{
3887	bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3888	struct i40e_pf *pf = vsi->back;
3889	struct i40e_hw *hw = &pf->hw;
3890	u16 vector;
3891	int i, q;
3892	u32 qp;
3893
3894	/* The interrupt indexing is offset by 1 in the PFINT_ITRn
3895	* and PFINT_LNKLSTn registers, e.g.:
3896	* PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts)
3897	*/
3898	qp = vsi->base_queue;
3899	vector = vsi->base_vector;
3900	for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3901	struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3902
3903	q_vector->rx.next_update = jiffies + 1;
3904	q_vector->rx.target_itr =
3905	ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3906	wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3907	q_vector->rx.target_itr >> 1);
3908	q_vector->rx.current_itr = q_vector->rx.target_itr;
3909
3910	q_vector->tx.next_update = jiffies + 1;
3911	q_vector->tx.target_itr =
3912	ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3913	wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3914	q_vector->tx.target_itr >> 1);
3915	q_vector->tx.current_itr = q_vector->tx.target_itr;
3916
3917	/* Set ITR for software interrupts triggered after exiting
3918	* busy-loop polling.
3919	*/
3920	wr32(hw, I40E_PFINT_ITRN(I40E_SW_ITR, vector - 1),
3921	I40E_ITR_20K);
3922
3923	wr32(hw, I40E_PFINT_RATEN(vector - 1),
3924	i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3925
3926	/* begin of linked list for RX queue assigned to this vector */
3927	wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3928	for (q = 0; q < q_vector->num_ringpairs; q++) {
3929	u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3930	u32 val;
3931
3932	val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3933	(I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3934	(vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3935	(nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3936	(I40E_QUEUE_TYPE_TX <<
3937	I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3938
3939	wr32(hw, I40E_QINT_RQCTL(qp), val);
3940
3941	if (has_xdp) {
3942	/* TX queue with next queue set to TX */
3943	val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3944	(I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3945	(vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3946	(qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3947	(I40E_QUEUE_TYPE_TX <<
3948	I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3949
3950	wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3951	}
3952	/* TX queue with next RX or end of linked list */
3953	val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3954	(I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3955	(vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3956	((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3957	(I40E_QUEUE_TYPE_RX <<
3958	I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3959
3960	/* Terminate the linked list */
3961	if (q == (q_vector->num_ringpairs - 1))
3962	val |= (I40E_QUEUE_END_OF_LIST <<
3963	I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3964
3965	wr32(hw, I40E_QINT_TQCTL(qp), val);
3966	qp++;
3967	}
3968	}
3969
3970	i40e_flush(hw);
3971	}
3972
3973	/**
3974	* i40e_enable_misc_int_causes - enable the non-queue interrupts
3975	* @pf: pointer to private device data structure
3976	**/
3977	static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3978	{
3979	struct i40e_hw *hw = &pf->hw;
3980	u32 val;
3981
3982	/* clear things first */
3983	wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */
3984	rd32(hw, I40E_PFINT_ICR0); /* read to clear */
3985
3986	val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
3987	I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
3988	I40E_PFINT_ICR0_ENA_GRST_MASK |
3989	I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3990	I40E_PFINT_ICR0_ENA_GPIO_MASK |
3991	I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
3992	I40E_PFINT_ICR0_ENA_VFLR_MASK |
3993	I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3994
3995	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))
3996	val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3997
3998	if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
3999	val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4000
4001	wr32(hw, I40E_PFINT_ICR0_ENA, val);
4002
4003	/* SW_ITR_IDX = 0, but don't change INTENA */
4004	wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
4005	I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
4006
4007	/* OTHER_ITR_IDX = 0 */
4008	wr32(hw, I40E_PFINT_STAT_CTL0, 0);
4009	}
4010
4011	/**
4012	* i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
4013	* @vsi: the VSI being configured
4014	**/
4015	static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
4016	{
4017	u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
4018	struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4019	struct i40e_pf *pf = vsi->back;
4020	struct i40e_hw *hw = &pf->hw;
4021
4022	/* set the ITR configuration */
4023	q_vector->rx.next_update = jiffies + 1;
4024	q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
4025	wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
4026	q_vector->rx.current_itr = q_vector->rx.target_itr;
4027	q_vector->tx.next_update = jiffies + 1;
4028	q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
4029	wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
4030	q_vector->tx.current_itr = q_vector->tx.target_itr;
4031
4032	i40e_enable_misc_int_causes(pf);
4033
4034	/* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
4035	wr32(hw, I40E_PFINT_LNKLST0, 0);
4036
4037	/* Associate the queue pair to the vector and enable the queue
4038	* interrupt RX queue in linked list with next queue set to TX
4039	*/
4040	wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX));
4041
4042	if (i40e_enabled_xdp_vsi(vsi)) {
4043	/* TX queue in linked list with next queue set to TX */
4044	wr32(hw, I40E_QINT_TQCTL(nextqp),
4045	I40E_QINT_TQCTL_VAL(nextqp, 0, TX));
4046	}
4047
4048	/* last TX queue so the next RX queue doesn't matter */
4049	wr32(hw, I40E_QINT_TQCTL(0),
4050	I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX));
4051	i40e_flush(hw);
4052	}
4053
4054	/**
4055	* i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
4056	* @pf: board private structure
4057	**/
4058	void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
4059	{
4060	struct i40e_hw *hw = &pf->hw;
4061
4062	wr32(hw, I40E_PFINT_DYN_CTL0,
4063	I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4064	i40e_flush(hw);
4065	}
4066
4067	/**
4068	* i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
4069	* @pf: board private structure
4070	**/
4071	void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
4072	{
4073	struct i40e_hw *hw = &pf->hw;
4074	u32 val;
4075
4076	val = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4077	I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4078	(I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4079
4080	wr32(hw, I40E_PFINT_DYN_CTL0, val);
4081	i40e_flush(hw);
4082	}
4083
4084	/**
4085	* i40e_msix_clean_rings - MSIX mode Interrupt Handler
4086	* @irq: interrupt number
4087	* @data: pointer to a q_vector
4088	**/
4089	static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
4090	{
4091	struct i40e_q_vector *q_vector = data;
4092
4093	if (!q_vector->tx.ring && !q_vector->rx.ring)
4094	return IRQ_HANDLED;
4095
4096	napi_schedule_irqoff(n: &q_vector->napi);
4097
4098	return IRQ_HANDLED;
4099	}
4100
4101	/**
4102	* i40e_irq_affinity_notify - Callback for affinity changes
4103	* @notify: context as to what irq was changed
4104	* @mask: the new affinity mask
4105	*
4106	* This is a callback function used by the irq_set_affinity_notifier function
4107	* so that we may register to receive changes to the irq affinity masks.
4108	**/
4109	static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
4110	const cpumask_t *mask)
4111	{
4112	struct i40e_q_vector *q_vector =
4113	container_of(notify, struct i40e_q_vector, affinity_notify);
4114
4115	cpumask_copy(dstp: &q_vector->affinity_mask, srcp: mask);
4116	}
4117
4118	/**
4119	* i40e_irq_affinity_release - Callback for affinity notifier release
4120	* @ref: internal core kernel usage
4121	*
4122	* This is a callback function used by the irq_set_affinity_notifier function
4123	* to inform the current notification subscriber that they will no longer
4124	* receive notifications.
4125	**/
4126	static void i40e_irq_affinity_release(struct kref *ref) {}
4127
4128	/**
4129	* i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
4130	* @vsi: the VSI being configured
4131	* @basename: name for the vector
4132	*
4133	* Allocates MSI-X vectors and requests interrupts from the kernel.
4134	**/
4135	static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
4136	{
4137	int q_vectors = vsi->num_q_vectors;
4138	struct i40e_pf *pf = vsi->back;
4139	int base = vsi->base_vector;
4140	int rx_int_idx = 0;
4141	int tx_int_idx = 0;
4142	int vector, err;
4143	int irq_num;
4144	int cpu;
4145
4146	for (vector = 0; vector < q_vectors; vector++) {
4147	struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
4148
4149	irq_num = pf->msix_entries[base + vector].vector;
4150
4151	if (q_vector->tx.ring && q_vector->rx.ring) {
4152	snprintf(buf: q_vector->name, size: sizeof(q_vector->name) - 1,
4153	fmt: "%s-%s-%d", basename, "TxRx", rx_int_idx++);
4154	tx_int_idx++;
4155	} else if (q_vector->rx.ring) {
4156	snprintf(buf: q_vector->name, size: sizeof(q_vector->name) - 1,
4157	fmt: "%s-%s-%d", basename, "rx", rx_int_idx++);
4158	} else if (q_vector->tx.ring) {
4159	snprintf(buf: q_vector->name, size: sizeof(q_vector->name) - 1,
4160	fmt: "%s-%s-%d", basename, "tx", tx_int_idx++);
4161	} else {
4162	/* skip this unused q_vector */
4163	continue;
4164	}
4165	err = request_irq(irq: irq_num,
4166	handler: vsi->irq_handler,
4167	flags: 0,
4168	name: q_vector->name,
4169	dev: q_vector);
4170	if (err) {
4171	dev_info(&pf->pdev->dev,
4172	"MSIX request_irq failed, error: %d\n", err);
4173	goto free_queue_irqs;
4174	}
4175
4176	/* register for affinity change notifications */
4177	q_vector->irq_num = irq_num;
4178	q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
4179	q_vector->affinity_notify.release = i40e_irq_affinity_release;
4180	irq_set_affinity_notifier(irq: irq_num, notify: &q_vector->affinity_notify);
4181	/* Spread affinity hints out across online CPUs.
4182	*
4183	* get_cpu_mask returns a static constant mask with
4184	* a permanent lifetime so it's ok to pass to
4185	* irq_update_affinity_hint without making a copy.
4186	*/
4187	cpu = cpumask_local_spread(i: q_vector->v_idx, node: -1);
4188	irq_update_affinity_hint(irq: irq_num, m: get_cpu_mask(cpu));
4189	}
4190
4191	vsi->irqs_ready = true;
4192	return 0;
4193
4194	free_queue_irqs:
4195	while (vector) {
4196	vector--;
4197	irq_num = pf->msix_entries[base + vector].vector;
4198	irq_set_affinity_notifier(irq: irq_num, NULL);
4199	irq_update_affinity_hint(irq: irq_num, NULL);
4200	free_irq(irq_num, &vsi->q_vectors[vector]);
4201	}
4202	return err;
4203	}
4204
4205	/**
4206	* i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
4207	* @vsi: the VSI being un-configured
4208	**/
4209	static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
4210	{
4211	struct i40e_pf *pf = vsi->back;
4212	struct i40e_hw *hw = &pf->hw;
4213	int base = vsi->base_vector;
4214	int i;
4215
4216	/* disable interrupt causation from each queue */
4217	for (i = 0; i < vsi->num_queue_pairs; i++) {
4218	u32 val;
4219
4220	val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
4221	val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
4222	wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
4223
4224	val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
4225	val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
4226	wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
4227
4228	if (!i40e_enabled_xdp_vsi(vsi))
4229	continue;
4230	wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
4231	}
4232
4233	/* disable each interrupt */
4234	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4235	for (i = vsi->base_vector;
4236	i < (vsi->num_q_vectors + vsi->base_vector); i++)
4237	wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
4238
4239	i40e_flush(hw);
4240	for (i = 0; i < vsi->num_q_vectors; i++)
4241	synchronize_irq(irq: pf->msix_entries[i + base].vector);
4242	} else {
4243	/* Legacy and MSI mode - this stops all interrupt handling */
4244	wr32(hw, I40E_PFINT_ICR0_ENA, 0);
4245	wr32(hw, I40E_PFINT_DYN_CTL0, 0);
4246	i40e_flush(hw);
4247	synchronize_irq(irq: pf->pdev->irq);
4248	}
4249	}
4250
4251	/**
4252	* i40e_vsi_enable_irq - Enable IRQ for the given VSI
4253	* @vsi: the VSI being configured
4254	**/
4255	static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
4256	{
4257	struct i40e_pf *pf = vsi->back;
4258	int i;
4259
4260	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4261	for (i = 0; i < vsi->num_q_vectors; i++)
4262	i40e_irq_dynamic_enable(vsi, vector: i);
4263	} else {
4264	i40e_irq_dynamic_enable_icr0(pf);
4265	}
4266
4267	i40e_flush(&pf->hw);
4268	return 0;
4269	}
4270
4271	/**
4272	* i40e_free_misc_vector - Free the vector that handles non-queue events
4273	* @pf: board private structure
4274	**/
4275	static void i40e_free_misc_vector(struct i40e_pf *pf)
4276	{
4277	/* Disable ICR 0 */
4278	wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
4279	i40e_flush(&pf->hw);
4280
4281	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) {
4282	free_irq(pf->msix_entries[0].vector, pf);
4283	clear_bit(nr: __I40E_MISC_IRQ_REQUESTED, addr: pf->state);
4284	}
4285	}
4286
4287	/**
4288	* i40e_intr - MSI/Legacy and non-queue interrupt handler
4289	* @irq: interrupt number
4290	* @data: pointer to a q_vector
4291	*
4292	* This is the handler used for all MSI/Legacy interrupts, and deals
4293	* with both queue and non-queue interrupts. This is also used in
4294	* MSIX mode to handle the non-queue interrupts.
4295	**/
4296	static irqreturn_t i40e_intr(int irq, void *data)
4297	{
4298	struct i40e_pf *pf = (struct i40e_pf *)data;
4299	struct i40e_hw *hw = &pf->hw;
4300	irqreturn_t ret = IRQ_NONE;
4301	u32 icr0, icr0_remaining;
4302	u32 val, ena_mask;
4303
4304	icr0 = rd32(hw, I40E_PFINT_ICR0);
4305	ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
4306
4307	/* if sharing a legacy IRQ, we might get called w/o an intr pending */
4308	if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
4309	goto enable_intr;
4310
4311	/* if interrupt but no bits showing, must be SWINT */
4312	if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
4313	(icr0 & I40E_PFINT_ICR0_SWINT_MASK))
4314	pf->sw_int_count++;
4315
4316	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&
4317	(icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
4318	ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
4319	dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
4320	set_bit(nr: __I40E_CORE_RESET_REQUESTED, addr: pf->state);
4321	}
4322
4323	/* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
4324	if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
4325	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
4326	struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4327
4328	/* We do not have a way to disarm Queue causes while leaving
4329	* interrupt enabled for all other causes, ideally
4330	* interrupt should be disabled while we are in NAPI but
4331	* this is not a performance path and napi_schedule()
4332	* can deal with rescheduling.
4333	*/
4334	if (!test_bit(__I40E_DOWN, pf->state))
4335	napi_schedule_irqoff(n: &q_vector->napi);
4336	}
4337
4338	if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
4339	ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4340	set_bit(nr: __I40E_ADMINQ_EVENT_PENDING, addr: pf->state);
4341	i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
4342	}
4343
4344	if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
4345	ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4346	set_bit(nr: __I40E_MDD_EVENT_PENDING, addr: pf->state);
4347	}
4348
4349	if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
4350	/* disable any further VFLR event notifications */
4351	if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {
4352	u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4353
4354	reg &= ~I40E_PFINT_ICR0_VFLR_MASK;
4355	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4356	} else {
4357	ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
4358	set_bit(nr: __I40E_VFLR_EVENT_PENDING, addr: pf->state);
4359	}
4360	}
4361
4362	if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
4363	if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4364	set_bit(nr: __I40E_RESET_INTR_RECEIVED, addr: pf->state);
4365	ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
4366	val = rd32(hw, I40E_GLGEN_RSTAT);
4367	val = FIELD_GET(I40E_GLGEN_RSTAT_RESET_TYPE_MASK, val);
4368	if (val == I40E_RESET_CORER) {
4369	pf->corer_count++;
4370	} else if (val == I40E_RESET_GLOBR) {
4371	pf->globr_count++;
4372	} else if (val == I40E_RESET_EMPR) {
4373	pf->empr_count++;
4374	set_bit(nr: __I40E_EMP_RESET_INTR_RECEIVED, addr: pf->state);
4375	}
4376	}
4377
4378	if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
4379	icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
4380	dev_info(&pf->pdev->dev, "HMC error interrupt\n");
4381	dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
4382	rd32(hw, I40E_PFHMC_ERRORINFO),
4383	rd32(hw, I40E_PFHMC_ERRORDATA));
4384	}
4385
4386	if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
4387	u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
4388
4389	if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK)
4390	schedule_work(work: &pf->ptp_extts0_work);
4391
4392	if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK)
4393	i40e_ptp_tx_hwtstamp(pf);
4394
4395	icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4396	}
4397
4398	/* If a critical error is pending we have no choice but to reset the
4399	* device.
4400	* Report and mask out any remaining unexpected interrupts.
4401	*/
4402	icr0_remaining = icr0 & ena_mask;
4403	if (icr0_remaining) {
4404	dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4405	icr0_remaining);
4406	if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4407	(icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4408	(icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4409	dev_info(&pf->pdev->dev, "device will be reset\n");
4410	set_bit(nr: __I40E_PF_RESET_REQUESTED, addr: pf->state);
4411	i40e_service_event_schedule(pf);
4412	}
4413	ena_mask &= ~icr0_remaining;
4414	}
4415	ret = IRQ_HANDLED;
4416
4417	enable_intr:
4418	/* re-enable interrupt causes */
4419	wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4420	if (!test_bit(__I40E_DOWN, pf->state) ||
4421	test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4422	i40e_service_event_schedule(pf);
4423	i40e_irq_dynamic_enable_icr0(pf);
4424	}
4425
4426	return ret;
4427	}
4428
4429	/**
4430	* i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4431	* @tx_ring: tx ring to clean
4432	* @budget: how many cleans we're allowed
4433	*
4434	* Returns true if there's any budget left (e.g. the clean is finished)
4435	**/
4436	static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4437	{
4438	struct i40e_vsi *vsi = tx_ring->vsi;
4439	u16 i = tx_ring->next_to_clean;
4440	struct i40e_tx_buffer *tx_buf;
4441	struct i40e_tx_desc *tx_desc;
4442
4443	tx_buf = &tx_ring->tx_bi[i];
4444	tx_desc = I40E_TX_DESC(tx_ring, i);
4445	i -= tx_ring->count;
4446
4447	do {
4448	struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4449
4450	/* if next_to_watch is not set then there is no work pending */
4451	if (!eop_desc)
4452	break;
4453
4454	/* prevent any other reads prior to eop_desc */
4455	smp_rmb();
4456
4457	/* if the descriptor isn't done, no work yet to do */
4458	if (!(eop_desc->cmd_type_offset_bsz &
4459	cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4460	break;
4461
4462	/* clear next_to_watch to prevent false hangs */
4463	tx_buf->next_to_watch = NULL;
4464
4465	tx_desc->buffer_addr = 0;
4466	tx_desc->cmd_type_offset_bsz = 0;
4467	/* move past filter desc */
4468	tx_buf++;
4469	tx_desc++;
4470	i++;
4471	if (unlikely(!i)) {
4472	i -= tx_ring->count;
4473	tx_buf = tx_ring->tx_bi;
4474	tx_desc = I40E_TX_DESC(tx_ring, 0);
4475	}
4476	/* unmap skb header data */
4477	dma_unmap_single(tx_ring->dev,
4478	dma_unmap_addr(tx_buf, dma),
4479	dma_unmap_len(tx_buf, len),
4480	DMA_TO_DEVICE);
4481	if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4482	kfree(objp: tx_buf->raw_buf);
4483
4484	tx_buf->raw_buf = NULL;
4485	tx_buf->tx_flags = 0;
4486	tx_buf->next_to_watch = NULL;
4487	dma_unmap_len_set(tx_buf, len, 0);
4488	tx_desc->buffer_addr = 0;
4489	tx_desc->cmd_type_offset_bsz = 0;
4490
4491	/* move us past the eop_desc for start of next FD desc */
4492	tx_buf++;
4493	tx_desc++;
4494	i++;
4495	if (unlikely(!i)) {
4496	i -= tx_ring->count;
4497	tx_buf = tx_ring->tx_bi;
4498	tx_desc = I40E_TX_DESC(tx_ring, 0);
4499	}
4500
4501	/* update budget accounting */
4502	budget--;
4503	} while (likely(budget));
4504
4505	i += tx_ring->count;
4506	tx_ring->next_to_clean = i;
4507
4508	if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags))
4509	i40e_irq_dynamic_enable(vsi, vector: tx_ring->q_vector->v_idx);
4510
4511	return budget > 0;
4512	}
4513
4514	/**
4515	* i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4516	* @irq: interrupt number
4517	* @data: pointer to a q_vector
4518	**/
4519	static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4520	{
4521	struct i40e_q_vector *q_vector = data;
4522	struct i40e_vsi *vsi;
4523
4524	if (!q_vector->tx.ring)
4525	return IRQ_HANDLED;
4526
4527	vsi = q_vector->tx.ring->vsi;
4528	i40e_clean_fdir_tx_irq(tx_ring: q_vector->tx.ring, budget: vsi->work_limit);
4529
4530	return IRQ_HANDLED;
4531	}
4532
4533	/**
4534	* i40e_map_vector_to_qp - Assigns the queue pair to the vector
4535	* @vsi: the VSI being configured
4536	* @v_idx: vector index
4537	* @qp_idx: queue pair index
4538	**/
4539	static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4540	{
4541	struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4542	struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4543	struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4544
4545	tx_ring->q_vector = q_vector;
4546	tx_ring->next = q_vector->tx.ring;
4547	q_vector->tx.ring = tx_ring;
4548	q_vector->tx.count++;
4549
4550	/* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4551	if (i40e_enabled_xdp_vsi(vsi)) {
4552	struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4553
4554	xdp_ring->q_vector = q_vector;
4555	xdp_ring->next = q_vector->tx.ring;
4556	q_vector->tx.ring = xdp_ring;
4557	q_vector->tx.count++;
4558	}
4559
4560	rx_ring->q_vector = q_vector;
4561	rx_ring->next = q_vector->rx.ring;
4562	q_vector->rx.ring = rx_ring;
4563	q_vector->rx.count++;
4564	}
4565
4566	/**
4567	* i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4568	* @vsi: the VSI being configured
4569	*
4570	* This function maps descriptor rings to the queue-specific vectors
4571	* we were allotted through the MSI-X enabling code. Ideally, we'd have
4572	* one vector per queue pair, but on a constrained vector budget, we
4573	* group the queue pairs as "efficiently" as possible.
4574	**/
4575	static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4576	{
4577	int qp_remaining = vsi->num_queue_pairs;
4578	int q_vectors = vsi->num_q_vectors;
4579	int num_ringpairs;
4580	int v_start = 0;
4581	int qp_idx = 0;
4582
4583	/* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4584	* group them so there are multiple queues per vector.
4585	* It is also important to go through all the vectors available to be
4586	* sure that if we don't use all the vectors, that the remaining vectors
4587	* are cleared. This is especially important when decreasing the
4588	* number of queues in use.
4589	*/
4590	for (; v_start < q_vectors; v_start++) {
4591	struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4592
4593	num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4594
4595	q_vector->num_ringpairs = num_ringpairs;
4596	q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4597
4598	q_vector->rx.count = 0;
4599	q_vector->tx.count = 0;
4600	q_vector->rx.ring = NULL;
4601	q_vector->tx.ring = NULL;
4602
4603	while (num_ringpairs--) {
4604	i40e_map_vector_to_qp(vsi, v_idx: v_start, qp_idx);
4605	qp_idx++;
4606	qp_remaining--;
4607	}
4608	}
4609	}
4610
4611	/**
4612	* i40e_vsi_request_irq - Request IRQ from the OS
4613	* @vsi: the VSI being configured
4614	* @basename: name for the vector
4615	**/
4616	static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4617	{
4618	struct i40e_pf *pf = vsi->back;
4619	int err;
4620
4621	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
4622	err = i40e_vsi_request_irq_msix(vsi, basename);
4623	else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags))
4624	err = request_irq(irq: pf->pdev->irq, handler: i40e_intr, flags: 0,
4625	name: pf->int_name, dev: pf);
4626	else
4627	err = request_irq(irq: pf->pdev->irq, handler: i40e_intr, IRQF_SHARED,
4628	name: pf->int_name, dev: pf);
4629
4630	if (err)
4631	dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4632
4633	return err;
4634	}
4635
4636	#ifdef CONFIG_NET_POLL_CONTROLLER
4637	/**
4638	* i40e_netpoll - A Polling 'interrupt' handler
4639	* @netdev: network interface device structure
4640	*
4641	* This is used by netconsole to send skbs without having to re-enable
4642	* interrupts. It's not called while the normal interrupt routine is executing.
4643	**/
4644	static void i40e_netpoll(struct net_device *netdev)
4645	{
4646	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
4647	struct i40e_vsi *vsi = np->vsi;
4648	struct i40e_pf *pf = vsi->back;
4649	int i;
4650
4651	/* if interface is down do nothing */
4652	if (test_bit(__I40E_VSI_DOWN, vsi->state))
4653	return;
4654
4655	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4656	for (i = 0; i < vsi->num_q_vectors; i++)
4657	i40e_msix_clean_rings(irq: 0, data: vsi->q_vectors[i]);
4658	} else {
4659	i40e_intr(irq: pf->pdev->irq, data: netdev);
4660	}
4661	}
4662	#endif
4663
4664	#define I40E_QTX_ENA_WAIT_COUNT 50
4665
4666	/**
4667	* i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4668	* @pf: the PF being configured
4669	* @pf_q: the PF queue
4670	* @enable: enable or disable state of the queue
4671	*
4672	* This routine will wait for the given Tx queue of the PF to reach the
4673	* enabled or disabled state.
4674	* Returns -ETIMEDOUT in case of failing to reach the requested state after
4675	* multiple retries; else will return 0 in case of success.
4676	**/
4677	static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4678	{
4679	int i;
4680	u32 tx_reg;
4681
4682	for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4683	tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4684	if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4685	break;
4686
4687	usleep_range(min: 10, max: 20);
4688	}
4689	if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4690	return -ETIMEDOUT;
4691
4692	return 0;
4693	}
4694
4695	/**
4696	* i40e_control_tx_q - Start or stop a particular Tx queue
4697	* @pf: the PF structure
4698	* @pf_q: the PF queue to configure
4699	* @enable: start or stop the queue
4700	*
4701	* This function enables or disables a single queue. Note that any delay
4702	* required after the operation is expected to be handled by the caller of
4703	* this function.
4704	**/
4705	static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4706	{
4707	struct i40e_hw *hw = &pf->hw;
4708	u32 tx_reg;
4709	int i;
4710
4711	/* warn the TX unit of coming changes */
4712	i40e_pre_tx_queue_cfg(hw: &pf->hw, queue: pf_q, enable);
4713	if (!enable)
4714	usleep_range(min: 10, max: 20);
4715
4716	for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4717	tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4718	if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4719	((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4720	break;
4721	usleep_range(min: 1000, max: 2000);
4722	}
4723
4724	/* Skip if the queue is already in the requested state */
4725	if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4726	return;
4727
4728	/* turn on/off the queue */
4729	if (enable) {
4730	wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4731	tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4732	} else {
4733	tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4734	}
4735
4736	wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4737	}
4738
4739	/**
4740	* i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4741	* @seid: VSI SEID
4742	* @pf: the PF structure
4743	* @pf_q: the PF queue to configure
4744	* @is_xdp: true if the queue is used for XDP
4745	* @enable: start or stop the queue
4746	**/
4747	int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4748	bool is_xdp, bool enable)
4749	{
4750	int ret;
4751
4752	i40e_control_tx_q(pf, pf_q, enable);
4753
4754	/* wait for the change to finish */
4755	ret = i40e_pf_txq_wait(pf, pf_q, enable);
4756	if (ret) {
4757	dev_info(&pf->pdev->dev,
4758	"VSI seid %d %sTx ring %d %sable timeout\n",
4759	seid, (is_xdp ? "XDP " : ""), pf_q,
4760	(enable ? "en" : "dis"));
4761	}
4762
4763	return ret;
4764	}
4765
4766	/**
4767	* i40e_vsi_enable_tx - Start a VSI's rings
4768	* @vsi: the VSI being configured
4769	**/
4770	static int i40e_vsi_enable_tx(struct i40e_vsi *vsi)
4771	{
4772	struct i40e_pf *pf = vsi->back;
4773	int i, pf_q, ret = 0;
4774
4775	pf_q = vsi->base_queue;
4776	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4777	ret = i40e_control_wait_tx_q(seid: vsi->seid, pf,
4778	pf_q,
4779	is_xdp: false /*is xdp*/, enable: true);
4780	if (ret)
4781	break;
4782
4783	if (!i40e_enabled_xdp_vsi(vsi))
4784	continue;
4785
4786	ret = i40e_control_wait_tx_q(seid: vsi->seid, pf,
4787	pf_q: pf_q + vsi->alloc_queue_pairs,
4788	is_xdp: true /*is xdp*/, enable: true);
4789	if (ret)
4790	break;
4791	}
4792	return ret;
4793	}
4794
4795	/**
4796	* i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4797	* @pf: the PF being configured
4798	* @pf_q: the PF queue
4799	* @enable: enable or disable state of the queue
4800	*
4801	* This routine will wait for the given Rx queue of the PF to reach the
4802	* enabled or disabled state.
4803	* Returns -ETIMEDOUT in case of failing to reach the requested state after
4804	* multiple retries; else will return 0 in case of success.
4805	**/
4806	static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4807	{
4808	int i;
4809	u32 rx_reg;
4810
4811	for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4812	rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4813	if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4814	break;
4815
4816	usleep_range(min: 10, max: 20);
4817	}
4818	if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4819	return -ETIMEDOUT;
4820
4821	return 0;
4822	}
4823
4824	/**
4825	* i40e_control_rx_q - Start or stop a particular Rx queue
4826	* @pf: the PF structure
4827	* @pf_q: the PF queue to configure
4828	* @enable: start or stop the queue
4829	*
4830	* This function enables or disables a single queue. Note that
4831	* any delay required after the operation is expected to be
4832	* handled by the caller of this function.
4833	**/
4834	static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4835	{
4836	struct i40e_hw *hw = &pf->hw;
4837	u32 rx_reg;
4838	int i;
4839
4840	for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4841	rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4842	if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4843	((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4844	break;
4845	usleep_range(min: 1000, max: 2000);
4846	}
4847
4848	/* Skip if the queue is already in the requested state */
4849	if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4850	return;
4851
4852	/* turn on/off the queue */
4853	if (enable)
4854	rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4855	else
4856	rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4857
4858	wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4859	}
4860
4861	/**
4862	* i40e_control_wait_rx_q
4863	* @pf: the PF structure
4864	* @pf_q: queue being configured
4865	* @enable: start or stop the rings
4866	*
4867	* This function enables or disables a single queue along with waiting
4868	* for the change to finish. The caller of this function should handle
4869	* the delays needed in the case of disabling queues.
4870	**/
4871	int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4872	{
4873	int ret = 0;
4874
4875	i40e_control_rx_q(pf, pf_q, enable);
4876
4877	/* wait for the change to finish */
4878	ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4879	if (ret)
4880	return ret;
4881
4882	return ret;
4883	}
4884
4885	/**
4886	* i40e_vsi_enable_rx - Start a VSI's rings
4887	* @vsi: the VSI being configured
4888	**/
4889	static int i40e_vsi_enable_rx(struct i40e_vsi *vsi)
4890	{
4891	struct i40e_pf *pf = vsi->back;
4892	int i, pf_q, ret = 0;
4893
4894	pf_q = vsi->base_queue;
4895	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4896	ret = i40e_control_wait_rx_q(pf, pf_q, enable: true);
4897	if (ret) {
4898	dev_info(&pf->pdev->dev,
4899	"VSI seid %d Rx ring %d enable timeout\n",
4900	vsi->seid, pf_q);
4901	break;
4902	}
4903	}
4904
4905	return ret;
4906	}
4907
4908	/**
4909	* i40e_vsi_start_rings - Start a VSI's rings
4910	* @vsi: the VSI being configured
4911	**/
4912	int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4913	{
4914	int ret = 0;
4915
4916	/* do rx first for enable and last for disable */
4917	ret = i40e_vsi_enable_rx(vsi);
4918	if (ret)
4919	return ret;
4920	ret = i40e_vsi_enable_tx(vsi);
4921
4922	return ret;
4923	}
4924
4925	#define I40E_DISABLE_TX_GAP_MSEC 50
4926
4927	/**
4928	* i40e_vsi_stop_rings - Stop a VSI's rings
4929	* @vsi: the VSI being configured
4930	**/
4931	void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4932	{
4933	struct i40e_pf *pf = vsi->back;
4934	u32 pf_q, tx_q_end, rx_q_end;
4935
4936	/* When port TX is suspended, don't wait */
4937	if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4938	return i40e_vsi_stop_rings_no_wait(vsi);
4939
4940	tx_q_end = vsi->base_queue +
4941	vsi->alloc_queue_pairs * (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
4942	for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)
4943	i40e_pre_tx_queue_cfg(hw: &pf->hw, queue: pf_q, enable: false);
4944
4945	rx_q_end = vsi->base_queue + vsi->num_queue_pairs;
4946	for (pf_q = vsi->base_queue; pf_q < rx_q_end; pf_q++)
4947	i40e_control_rx_q(pf, pf_q, enable: false);
4948
4949	msleep(I40E_DISABLE_TX_GAP_MSEC);
4950	for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)
4951	wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);
4952
4953	i40e_vsi_wait_queues_disabled(vsi);
4954	}
4955
4956	/**
4957	* i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4958	* @vsi: the VSI being shutdown
4959	*
4960	* This function stops all the rings for a VSI but does not delay to verify
4961	* that rings have been disabled. It is expected that the caller is shutting
4962	* down multiple VSIs at once and will delay together for all the VSIs after
4963	* initiating the shutdown. This is particularly useful for shutting down lots
4964	* of VFs together. Otherwise, a large delay can be incurred while configuring
4965	* each VSI in serial.
4966	**/
4967	void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4968	{
4969	struct i40e_pf *pf = vsi->back;
4970	int i, pf_q;
4971
4972	pf_q = vsi->base_queue;
4973	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4974	i40e_control_tx_q(pf, pf_q, enable: false);
4975	i40e_control_rx_q(pf, pf_q, enable: false);
4976	}
4977	}
4978
4979	/**
4980	* i40e_vsi_free_irq - Free the irq association with the OS
4981	* @vsi: the VSI being configured
4982	**/
4983	static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4984	{
4985	struct i40e_pf *pf = vsi->back;
4986	struct i40e_hw *hw = &pf->hw;
4987	int base = vsi->base_vector;
4988	u32 val, qp;
4989	int i;
4990
4991	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4992	if (!vsi->q_vectors)
4993	return;
4994
4995	if (!vsi->irqs_ready)
4996	return;
4997
4998	vsi->irqs_ready = false;
4999	for (i = 0; i < vsi->num_q_vectors; i++) {
5000	int irq_num;
5001	u16 vector;
5002
5003	vector = i + base;
5004	irq_num = pf->msix_entries[vector].vector;
5005
5006	/* free only the irqs that were actually requested */
5007	if (!vsi->q_vectors[i] ||
5008	!vsi->q_vectors[i]->num_ringpairs)
5009	continue;
5010
5011	/* clear the affinity notifier in the IRQ descriptor */
5012	irq_set_affinity_notifier(irq: irq_num, NULL);
5013	/* remove our suggested affinity mask for this IRQ */
5014	irq_update_affinity_hint(irq: irq_num, NULL);
5015	free_irq(irq_num, vsi->q_vectors[i]);
5016
5017	/* Tear down the interrupt queue link list
5018	*
5019	* We know that they come in pairs and always
5020	* the Rx first, then the Tx. To clear the
5021	* link list, stick the EOL value into the
5022	* next_q field of the registers.
5023	*/
5024	val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
5025	qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK,
5026	val);
5027	val |= I40E_QUEUE_END_OF_LIST
5028	<< I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
5029	wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
5030
5031	while (qp != I40E_QUEUE_END_OF_LIST) {
5032	u32 next;
5033
5034	val = rd32(hw, I40E_QINT_RQCTL(qp));
5035
5036	val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
5037	I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5038	I40E_QINT_RQCTL_CAUSE_ENA_MASK |
5039	I40E_QINT_RQCTL_INTEVENT_MASK);
5040
5041	val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5042	I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5043
5044	wr32(hw, I40E_QINT_RQCTL(qp), val);
5045
5046	val = rd32(hw, I40E_QINT_TQCTL(qp));
5047
5048	next = FIELD_GET(I40E_QINT_TQCTL_NEXTQ_INDX_MASK,
5049	val);
5050
5051	val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
5052	I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5053	I40E_QINT_TQCTL_CAUSE_ENA_MASK |
5054	I40E_QINT_TQCTL_INTEVENT_MASK);
5055
5056	val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5057	I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5058
5059	wr32(hw, I40E_QINT_TQCTL(qp), val);
5060	qp = next;
5061	}
5062	}
5063	} else {
5064	free_irq(pf->pdev->irq, pf);
5065
5066	val = rd32(hw, I40E_PFINT_LNKLST0);
5067	qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK, val);
5068	val |= I40E_QUEUE_END_OF_LIST
5069	<< I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
5070	wr32(hw, I40E_PFINT_LNKLST0, val);
5071
5072	val = rd32(hw, I40E_QINT_RQCTL(qp));
5073	val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
5074	I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5075	I40E_QINT_RQCTL_CAUSE_ENA_MASK |
5076	I40E_QINT_RQCTL_INTEVENT_MASK);
5077
5078	val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5079	I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5080
5081	wr32(hw, I40E_QINT_RQCTL(qp), val);
5082
5083	val = rd32(hw, I40E_QINT_TQCTL(qp));
5084
5085	val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
5086	I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5087	I40E_QINT_TQCTL_CAUSE_ENA_MASK |
5088	I40E_QINT_TQCTL_INTEVENT_MASK);
5089
5090	val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5091	I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5092
5093	wr32(hw, I40E_QINT_TQCTL(qp), val);
5094	}
5095	}
5096
5097	/**
5098	* i40e_free_q_vector - Free memory allocated for specific interrupt vector
5099	* @vsi: the VSI being configured
5100	* @v_idx: Index of vector to be freed
5101	*
5102	* This function frees the memory allocated to the q_vector. In addition if
5103	* NAPI is enabled it will delete any references to the NAPI struct prior
5104	* to freeing the q_vector.
5105	**/
5106	static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
5107	{
5108	struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
5109	struct i40e_ring *ring;
5110
5111	if (!q_vector)
5112	return;
5113
5114	/* disassociate q_vector from rings */
5115	i40e_for_each_ring(ring, q_vector->tx)
5116	ring->q_vector = NULL;
5117
5118	i40e_for_each_ring(ring, q_vector->rx)
5119	ring->q_vector = NULL;
5120
5121	/* only VSI w/ an associated netdev is set up w/ NAPI */
5122	if (vsi->netdev)
5123	netif_napi_del(napi: &q_vector->napi);
5124
5125	vsi->q_vectors[v_idx] = NULL;
5126
5127	kfree_rcu(q_vector, rcu);
5128	}
5129
5130	/**
5131	* i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
5132	* @vsi: the VSI being un-configured
5133	*
5134	* This frees the memory allocated to the q_vectors and
5135	* deletes references to the NAPI struct.
5136	**/
5137	static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
5138	{
5139	int v_idx;
5140
5141	for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
5142	i40e_free_q_vector(vsi, v_idx);
5143	}
5144
5145	/**
5146	* i40e_reset_interrupt_capability - Disable interrupt setup in OS
5147	* @pf: board private structure
5148	**/
5149	static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
5150	{
5151	/* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
5152	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
5153	pci_disable_msix(dev: pf->pdev);
5154	kfree(objp: pf->msix_entries);
5155	pf->msix_entries = NULL;
5156	kfree(objp: pf->irq_pile);
5157	pf->irq_pile = NULL;
5158	} else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {
5159	pci_disable_msi(dev: pf->pdev);
5160	}
5161	clear_bit(nr: I40E_FLAG_MSI_ENA, addr: pf->flags);
5162	clear_bit(nr: I40E_FLAG_MSIX_ENA, addr: pf->flags);
5163	}
5164
5165	/**
5166	* i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
5167	* @pf: board private structure
5168	*
5169	* We go through and clear interrupt specific resources and reset the structure
5170	* to pre-load conditions
5171	**/
5172	static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
5173	{
5174	struct i40e_vsi *vsi;
5175	int i;
5176
5177	if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))
5178	i40e_free_misc_vector(pf);
5179
5180	i40e_put_lump(pile: pf->irq_pile, index: pf->iwarp_base_vector,
5181	I40E_IWARP_IRQ_PILE_ID);
5182
5183	i40e_put_lump(pile: pf->irq_pile, index: 0, I40E_PILE_VALID_BIT-1);
5184
5185	i40e_pf_for_each_vsi(pf, i, vsi)
5186	i40e_vsi_free_q_vectors(vsi);
5187
5188	i40e_reset_interrupt_capability(pf);
5189	}
5190
5191	/**
5192	* i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
5193	* @vsi: the VSI being configured
5194	**/
5195	static void i40e_napi_enable_all(struct i40e_vsi *vsi)
5196	{
5197	int q_idx;
5198
5199	if (!vsi->netdev)
5200	return;
5201
5202	for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5203	struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5204
5205	if (q_vector->rx.ring || q_vector->tx.ring)
5206	napi_enable(n: &q_vector->napi);
5207	}
5208	}
5209
5210	/**
5211	* i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
5212	* @vsi: the VSI being configured
5213	**/
5214	static void i40e_napi_disable_all(struct i40e_vsi *vsi)
5215	{
5216	int q_idx;
5217
5218	if (!vsi->netdev)
5219	return;
5220
5221	for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5222	struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5223
5224	if (q_vector->rx.ring || q_vector->tx.ring)
5225	napi_disable(n: &q_vector->napi);
5226	}
5227	}
5228
5229	/**
5230	* i40e_vsi_close - Shut down a VSI
5231	* @vsi: the vsi to be quelled
5232	**/
5233	static void i40e_vsi_close(struct i40e_vsi *vsi)
5234	{
5235	struct i40e_pf *pf = vsi->back;
5236	if (!test_and_set_bit(nr: __I40E_VSI_DOWN, addr: vsi->state))
5237	i40e_down(vsi);
5238	i40e_vsi_free_irq(vsi);
5239	i40e_vsi_free_tx_resources(vsi);
5240	i40e_vsi_free_rx_resources(vsi);
5241	vsi->current_netdev_flags = 0;
5242	set_bit(nr: __I40E_CLIENT_SERVICE_REQUESTED, addr: pf->state);
5243	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
5244	set_bit(nr: __I40E_CLIENT_RESET, addr: pf->state);
5245	}
5246
5247	/**
5248	* i40e_quiesce_vsi - Pause a given VSI
5249	* @vsi: the VSI being paused
5250	**/
5251	static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
5252	{
5253	if (test_bit(__I40E_VSI_DOWN, vsi->state))
5254	return;
5255
5256	set_bit(nr: __I40E_VSI_NEEDS_RESTART, addr: vsi->state);
5257	if (vsi->netdev && netif_running(dev: vsi->netdev))
5258	vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
5259	else
5260	i40e_vsi_close(vsi);
5261	}
5262
5263	/**
5264	* i40e_unquiesce_vsi - Resume a given VSI
5265	* @vsi: the VSI being resumed
5266	**/
5267	static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
5268	{
5269	if (!test_and_clear_bit(nr: __I40E_VSI_NEEDS_RESTART, addr: vsi->state))
5270	return;
5271
5272	if (vsi->netdev && netif_running(dev: vsi->netdev))
5273	vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
5274	else
5275	i40e_vsi_open(vsi); /* this clears the DOWN bit */
5276	}
5277
5278	/**
5279	* i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
5280	* @pf: the PF
5281	**/
5282	static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
5283	{
5284	struct i40e_vsi *vsi;
5285	int v;
5286
5287	i40e_pf_for_each_vsi(pf, v, vsi)
5288	i40e_quiesce_vsi(vsi);
5289	}
5290
5291	/**
5292	* i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
5293	* @pf: the PF
5294	**/
5295	static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
5296	{
5297	struct i40e_vsi *vsi;
5298	int v;
5299
5300	i40e_pf_for_each_vsi(pf, v, vsi)
5301	i40e_unquiesce_vsi(vsi);
5302	}
5303
5304	/**
5305	* i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
5306	* @vsi: the VSI being configured
5307	*
5308	* Wait until all queues on a given VSI have been disabled.
5309	**/
5310	int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
5311	{
5312	struct i40e_pf *pf = vsi->back;
5313	int i, pf_q, ret;
5314
5315	pf_q = vsi->base_queue;
5316	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
5317	/* Check and wait for the Tx queue */
5318	ret = i40e_pf_txq_wait(pf, pf_q, enable: false);
5319	if (ret) {
5320	dev_info(&pf->pdev->dev,
5321	"VSI seid %d Tx ring %d disable timeout\n",
5322	vsi->seid, pf_q);
5323	return ret;
5324	}
5325
5326	if (!i40e_enabled_xdp_vsi(vsi))
5327	goto wait_rx;
5328
5329	/* Check and wait for the XDP Tx queue */
5330	ret = i40e_pf_txq_wait(pf, pf_q: pf_q + vsi->alloc_queue_pairs,
5331	enable: false);
5332	if (ret) {
5333	dev_info(&pf->pdev->dev,
5334	"VSI seid %d XDP Tx ring %d disable timeout\n",
5335	vsi->seid, pf_q);
5336	return ret;
5337	}
5338	wait_rx:
5339	/* Check and wait for the Rx queue */
5340	ret = i40e_pf_rxq_wait(pf, pf_q, enable: false);
5341	if (ret) {
5342	dev_info(&pf->pdev->dev,
5343	"VSI seid %d Rx ring %d disable timeout\n",
5344	vsi->seid, pf_q);
5345	return ret;
5346	}
5347	}
5348
5349	return 0;
5350	}
5351
5352	#ifdef CONFIG_I40E_DCB
5353	/**
5354	* i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
5355	* @pf: the PF
5356	*
5357	* This function waits for the queues to be in disabled state for all the
5358	* VSIs that are managed by this PF.
5359	**/
5360	static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
5361	{
5362	struct i40e_vsi *vsi;
5363	int v, ret = 0;
5364
5365	i40e_pf_for_each_vsi(pf, v, vsi) {
5366	ret = i40e_vsi_wait_queues_disabled(vsi);
5367	if (ret)
5368	break;
5369	}
5370
5371	return ret;
5372	}
5373
5374	#endif
5375
5376	/**
5377	* i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5378	* @pf: pointer to PF
5379	*
5380	* Get TC map for ISCSI PF type that will include iSCSI TC
5381	* and LAN TC.
5382	**/
5383	static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
5384	{
5385	struct i40e_dcb_app_priority_table app;
5386	struct i40e_hw *hw = &pf->hw;
5387	u8 enabled_tc = 1; /* TC0 is always enabled */
5388	u8 tc, i;
5389	/* Get the iSCSI APP TLV */
5390	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5391
5392	for (i = 0; i < dcbcfg->numapps; i++) {
5393	app = dcbcfg->app[i];
5394	if (app.selector == I40E_APP_SEL_TCPIP &&
5395	app.protocolid == I40E_APP_PROTOID_ISCSI) {
5396	tc = dcbcfg->etscfg.prioritytable[app.priority];
5397	enabled_tc |= BIT(tc);
5398	break;
5399	}
5400	}
5401
5402	return enabled_tc;
5403	}
5404
5405	/**
5406	* i40e_dcb_get_num_tc - Get the number of TCs from DCBx config
5407	* @dcbcfg: the corresponding DCBx configuration structure
5408	*
5409	* Return the number of TCs from given DCBx configuration
5410	**/
5411	static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5412	{
5413	int i, tc_unused = 0;
5414	u8 num_tc = 0;
5415	u8 ret = 0;
5416
5417	/* Scan the ETS Config Priority Table to find
5418	* traffic class enabled for a given priority
5419	* and create a bitmask of enabled TCs
5420	*/
5421	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5422	num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5423
5424	/* Now scan the bitmask to check for
5425	* contiguous TCs starting with TC0
5426	*/
5427	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5428	if (num_tc & BIT(i)) {
5429	if (!tc_unused) {
5430	ret++;
5431	} else {
5432	pr_err("Non-contiguous TC - Disabling DCB\n");
5433	return 1;
5434	}
5435	} else {
5436	tc_unused = 1;
5437	}
5438	}
5439
5440	/* There is always at least TC0 */
5441	if (!ret)
5442	ret = 1;
5443
5444	return ret;
5445	}
5446
5447	/**
5448	* i40e_dcb_get_enabled_tc - Get enabled traffic classes
5449	* @dcbcfg: the corresponding DCBx configuration structure
5450	*
5451	* Query the current DCB configuration and return the number of
5452	* traffic classes enabled from the given DCBX config
5453	**/
5454	static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5455	{
5456	u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5457	u8 enabled_tc = 1;
5458	u8 i;
5459
5460	for (i = 0; i < num_tc; i++)
5461	enabled_tc |= BIT(i);
5462
5463	return enabled_tc;
5464	}
5465
5466	/**
5467	* i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5468	* @pf: PF being queried
5469	*
5470	* Query the current MQPRIO configuration and return the number of
5471	* traffic classes enabled.
5472	**/
5473	static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5474	{
5475	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
5476	u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5477	u8 enabled_tc = 1, i;
5478
5479	for (i = 1; i < num_tc; i++)
5480	enabled_tc |= BIT(i);
5481	return enabled_tc;
5482	}
5483
5484	/**
5485	* i40e_pf_get_num_tc - Get enabled traffic classes for PF
5486	* @pf: PF being queried
5487	*
5488	* Return number of traffic classes enabled for the given PF
5489	**/
5490	static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5491	{
5492	struct i40e_hw *hw = &pf->hw;
5493	u8 i, enabled_tc = 1;
5494	u8 num_tc = 0;
5495	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5496
5497	if (i40e_is_tc_mqprio_enabled(pf))
5498	return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
5499
5500	/* If neither MQPRIO nor DCB is enabled, then always use single TC */
5501	if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
5502	return 1;
5503
5504	/* SFP mode will be enabled for all TCs on port */
5505	if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
5506	return i40e_dcb_get_num_tc(dcbcfg);
5507
5508	/* MFP mode return count of enabled TCs for this PF */
5509	if (pf->hw.func_caps.iscsi)
5510	enabled_tc = i40e_get_iscsi_tc_map(pf);
5511	else
5512	return 1; /* Only TC0 */
5513
5514	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5515	if (enabled_tc & BIT(i))
5516	num_tc++;
5517	}
5518	return num_tc;
5519	}
5520
5521	/**
5522	* i40e_pf_get_tc_map - Get bitmap for enabled traffic classes
5523	* @pf: PF being queried
5524	*
5525	* Return a bitmap for enabled traffic classes for this PF.
5526	**/
5527	static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5528	{
5529	if (i40e_is_tc_mqprio_enabled(pf))
5530	return i40e_mqprio_get_enabled_tc(pf);
5531
5532	/* If neither MQPRIO nor DCB is enabled for this PF then just return
5533	* default TC
5534	*/
5535	if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
5536	return I40E_DEFAULT_TRAFFIC_CLASS;
5537
5538	/* SFP mode we want PF to be enabled for all TCs */
5539	if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
5540	return i40e_dcb_get_enabled_tc(dcbcfg: &pf->hw.local_dcbx_config);
5541
5542	/* MFP enabled and iSCSI PF type */
5543	if (pf->hw.func_caps.iscsi)
5544	return i40e_get_iscsi_tc_map(pf);
5545	else
5546	return I40E_DEFAULT_TRAFFIC_CLASS;
5547	}
5548
5549	/**
5550	* i40e_vsi_get_bw_info - Query VSI BW Information
5551	* @vsi: the VSI being queried
5552	*
5553	* Returns 0 on success, negative value on failure
5554	**/
5555	static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5556	{
5557	struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5558	struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5559	struct i40e_pf *pf = vsi->back;
5560	struct i40e_hw *hw = &pf->hw;
5561	u32 tc_bw_max;
5562	int ret;
5563	int i;
5564
5565	/* Get the VSI level BW configuration */
5566	ret = i40e_aq_query_vsi_bw_config(hw, seid: vsi->seid, bw_data: &bw_config, NULL);
5567	if (ret) {
5568	dev_info(&pf->pdev->dev,
5569	"couldn't get PF vsi bw config, err %pe aq_err %s\n",
5570	ERR_PTR(ret),
5571	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5572	return -EINVAL;
5573	}
5574
5575	/* Get the VSI level BW configuration per TC */
5576	ret = i40e_aq_query_vsi_ets_sla_config(hw, seid: vsi->seid, bw_data: &bw_ets_config,
5577	NULL);
5578	if (ret) {
5579	dev_info(&pf->pdev->dev,
5580	"couldn't get PF vsi ets bw config, err %pe aq_err %s\n",
5581	ERR_PTR(ret),
5582	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5583	return -EINVAL;
5584	}
5585
5586	if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5587	dev_info(&pf->pdev->dev,
5588	"Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5589	bw_config.tc_valid_bits,
5590	bw_ets_config.tc_valid_bits);
5591	/* Still continuing */
5592	}
5593
5594	vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5595	vsi->bw_max_quanta = bw_config.max_bw;
5596	tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5597	(le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5598	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5599	vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5600	vsi->bw_ets_limit_credits[i] =
5601	le16_to_cpu(bw_ets_config.credits[i]);
5602	/* 3 bits out of 4 for each TC */
5603	vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5604	}
5605
5606	return 0;
5607	}
5608
5609	/**
5610	* i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5611	* @vsi: the VSI being configured
5612	* @enabled_tc: TC bitmap
5613	* @bw_share: BW shared credits per TC
5614	*
5615	* Returns 0 on success, negative value on failure
5616	**/
5617	static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5618	u8 *bw_share)
5619	{
5620	struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5621	struct i40e_pf *pf = vsi->back;
5622	int ret;
5623	int i;
5624
5625	/* There is no need to reset BW when mqprio mode is on. */
5626	if (i40e_is_tc_mqprio_enabled(pf))
5627	return 0;
5628	if (!vsi->mqprio_qopt.qopt.hw && !test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
5629	ret = i40e_set_bw_limit(vsi, seid: vsi->seid, max_tx_rate: 0);
5630	if (ret)
5631	dev_info(&pf->pdev->dev,
5632	"Failed to reset tx rate for vsi->seid %u\n",
5633	vsi->seid);
5634	return ret;
5635	}
5636	memset(&bw_data, 0, sizeof(bw_data));
5637	bw_data.tc_valid_bits = enabled_tc;
5638	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5639	bw_data.tc_bw_credits[i] = bw_share[i];
5640
5641	ret = i40e_aq_config_vsi_tc_bw(hw: &pf->hw, seid: vsi->seid, bw_data: &bw_data, NULL);
5642	if (ret) {
5643	dev_info(&pf->pdev->dev,
5644	"AQ command Config VSI BW allocation per TC failed = %d\n",
5645	pf->hw.aq.asq_last_status);
5646	return -EINVAL;
5647	}
5648
5649	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5650	vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5651
5652	return 0;
5653	}
5654
5655	/**
5656	* i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5657	* @vsi: the VSI being configured
5658	* @enabled_tc: TC map to be enabled
5659	*
5660	**/
5661	static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5662	{
5663	struct net_device *netdev = vsi->netdev;
5664	struct i40e_pf *pf = vsi->back;
5665	struct i40e_hw *hw = &pf->hw;
5666	u8 netdev_tc = 0;
5667	int i;
5668	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5669
5670	if (!netdev)
5671	return;
5672
5673	if (!enabled_tc) {
5674	netdev_reset_tc(dev: netdev);
5675	return;
5676	}
5677
5678	/* Set up actual enabled TCs on the VSI */
5679	if (netdev_set_num_tc(dev: netdev, num_tc: vsi->tc_config.numtc))
5680	return;
5681
5682	/* set per TC queues for the VSI */
5683	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5684	/* Only set TC queues for enabled tcs
5685	*
5686	* e.g. For a VSI that has TC0 and TC3 enabled the
5687	* enabled_tc bitmap would be 0x00001001; the driver
5688	* will set the numtc for netdev as 2 that will be
5689	* referenced by the netdev layer as TC 0 and 1.
5690	*/
5691	if (vsi->tc_config.enabled_tc & BIT(i))
5692	netdev_set_tc_queue(dev: netdev,
5693	tc: vsi->tc_config.tc_info[i].netdev_tc,
5694	count: vsi->tc_config.tc_info[i].qcount,
5695	offset: vsi->tc_config.tc_info[i].qoffset);
5696	}
5697
5698	if (i40e_is_tc_mqprio_enabled(pf))
5699	return;
5700
5701	/* Assign UP2TC map for the VSI */
5702	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5703	/* Get the actual TC# for the UP */
5704	u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5705	/* Get the mapped netdev TC# for the UP */
5706	netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc;
5707	netdev_set_prio_tc_map(dev: netdev, prio: i, tc: netdev_tc);
5708	}
5709	}
5710
5711	/**
5712	* i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5713	* @vsi: the VSI being configured
5714	* @ctxt: the ctxt buffer returned from AQ VSI update param command
5715	**/
5716	static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5717	struct i40e_vsi_context *ctxt)
5718	{
5719	/* copy just the sections touched not the entire info
5720	* since not all sections are valid as returned by
5721	* update vsi params
5722	*/
5723	vsi->info.mapping_flags = ctxt->info.mapping_flags;
5724	memcpy(&vsi->info.queue_mapping,
5725	&ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5726	memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5727	sizeof(vsi->info.tc_mapping));
5728	}
5729
5730	/**
5731	* i40e_update_adq_vsi_queues - update queue mapping for ADq VSI
5732	* @vsi: the VSI being reconfigured
5733	* @vsi_offset: offset from main VF VSI
5734	*/
5735	int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset)
5736	{
5737	struct i40e_vsi_context ctxt = {};
5738	struct i40e_pf *pf;
5739	struct i40e_hw *hw;
5740	int ret;
5741
5742	if (!vsi)
5743	return -EINVAL;
5744	pf = vsi->back;
5745	hw = &pf->hw;
5746
5747	ctxt.seid = vsi->seid;
5748	ctxt.pf_num = hw->pf_id;
5749	ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;
5750	ctxt.uplink_seid = vsi->uplink_seid;
5751	ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5752	ctxt.flags = I40E_AQ_VSI_TYPE_VF;
5753	ctxt.info = vsi->info;
5754
5755	i40e_vsi_setup_queue_map(vsi, ctxt: &ctxt, enabled_tc: vsi->tc_config.enabled_tc,
5756	is_add: false);
5757	if (vsi->reconfig_rss) {
5758	vsi->rss_size = min_t(int, pf->alloc_rss_size,
5759	vsi->num_queue_pairs);
5760	ret = i40e_vsi_config_rss(vsi);
5761	if (ret) {
5762	dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");
5763	return ret;
5764	}
5765	vsi->reconfig_rss = false;
5766	}
5767
5768	ret = i40e_aq_update_vsi_params(hw, vsi_ctx: &ctxt, NULL);
5769	if (ret) {
5770	dev_info(&pf->pdev->dev, "Update vsi config failed, err %pe aq_err %s\n",
5771	ERR_PTR(ret),
5772	i40e_aq_str(hw, hw->aq.asq_last_status));
5773	return ret;
5774	}
5775	/* update the local VSI info with updated queue map */
5776	i40e_vsi_update_queue_map(vsi, ctxt: &ctxt);
5777	vsi->info.valid_sections = 0;
5778
5779	return ret;
5780	}
5781
5782	/**
5783	* i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5784	* @vsi: VSI to be configured
5785	* @enabled_tc: TC bitmap
5786	*
5787	* This configures a particular VSI for TCs that are mapped to the
5788	* given TC bitmap. It uses default bandwidth share for TCs across
5789	* VSIs to configure TC for a particular VSI.
5790	*
5791	* NOTE:
5792	* It is expected that the VSI queues have been quisced before calling
5793	* this function.
5794	**/
5795	static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5796	{
5797	u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5798	struct i40e_pf *pf = vsi->back;
5799	struct i40e_hw *hw = &pf->hw;
5800	struct i40e_vsi_context ctxt;
5801	int ret = 0;
5802	int i;
5803
5804	/* Check if enabled_tc is same as existing or new TCs */
5805	if (vsi->tc_config.enabled_tc == enabled_tc &&
5806	vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5807	return ret;
5808
5809	/* Enable ETS TCs with equal BW Share for now across all VSIs */
5810	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5811	if (enabled_tc & BIT(i))
5812	bw_share[i] = 1;
5813	}
5814
5815	ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5816	if (ret) {
5817	struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5818
5819	dev_info(&pf->pdev->dev,
5820	"Failed configuring TC map %d for VSI %d\n",
5821	enabled_tc, vsi->seid);
5822	ret = i40e_aq_query_vsi_bw_config(hw, seid: vsi->seid,
5823	bw_data: &bw_config, NULL);
5824	if (ret) {
5825	dev_info(&pf->pdev->dev,
5826	"Failed querying vsi bw info, err %pe aq_err %s\n",
5827	ERR_PTR(ret),
5828	i40e_aq_str(hw, hw->aq.asq_last_status));
5829	goto out;
5830	}
5831	if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5832	u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5833
5834	if (!valid_tc)
5835	valid_tc = bw_config.tc_valid_bits;
5836	/* Always enable TC0, no matter what */
5837	valid_tc |= 1;
5838	dev_info(&pf->pdev->dev,
5839	"Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5840	enabled_tc, bw_config.tc_valid_bits, valid_tc);
5841	enabled_tc = valid_tc;
5842	}
5843
5844	ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5845	if (ret) {
5846	dev_err(&pf->pdev->dev,
5847	"Unable to configure TC map %d for VSI %d\n",
5848	enabled_tc, vsi->seid);
5849	goto out;
5850	}
5851	}
5852
5853	/* Update Queue Pairs Mapping for currently enabled UPs */
5854	ctxt.seid = vsi->seid;
5855	ctxt.pf_num = vsi->back->hw.pf_id;
5856	ctxt.vf_num = 0;
5857	ctxt.uplink_seid = vsi->uplink_seid;
5858	ctxt.info = vsi->info;
5859	if (i40e_is_tc_mqprio_enabled(pf)) {
5860	ret = i40e_vsi_setup_queue_map_mqprio(vsi, ctxt: &ctxt, enabled_tc);
5861	if (ret)
5862	goto out;
5863	} else {
5864	i40e_vsi_setup_queue_map(vsi, ctxt: &ctxt, enabled_tc, is_add: false);
5865	}
5866
5867	/* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5868	* queues changed.
5869	*/
5870	if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5871	vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5872	vsi->num_queue_pairs);
5873	ret = i40e_vsi_config_rss(vsi);
5874	if (ret) {
5875	dev_info(&vsi->back->pdev->dev,
5876	"Failed to reconfig rss for num_queues\n");
5877	return ret;
5878	}
5879	vsi->reconfig_rss = false;
5880	}
5881	if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {
5882	ctxt.info.valid_sections |=
5883	cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5884	ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5885	}
5886
5887	/* Update the VSI after updating the VSI queue-mapping
5888	* information
5889	*/
5890	ret = i40e_aq_update_vsi_params(hw, vsi_ctx: &ctxt, NULL);
5891	if (ret) {
5892	dev_info(&pf->pdev->dev,
5893	"Update vsi tc config failed, err %pe aq_err %s\n",
5894	ERR_PTR(ret),
5895	i40e_aq_str(hw, hw->aq.asq_last_status));
5896	goto out;
5897	}
5898	/* update the local VSI info with updated queue map */
5899	i40e_vsi_update_queue_map(vsi, ctxt: &ctxt);
5900	vsi->info.valid_sections = 0;
5901
5902	/* Update current VSI BW information */
5903	ret = i40e_vsi_get_bw_info(vsi);
5904	if (ret) {
5905	dev_info(&pf->pdev->dev,
5906	"Failed updating vsi bw info, err %pe aq_err %s\n",
5907	ERR_PTR(ret),
5908	i40e_aq_str(hw, hw->aq.asq_last_status));
5909	goto out;
5910	}
5911
5912	/* Update the netdev TC setup */
5913	i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5914	out:
5915	return ret;
5916	}
5917
5918	/**
5919	* i40e_get_link_speed - Returns link speed for the interface
5920	* @vsi: VSI to be configured
5921	*
5922	**/
5923	static int i40e_get_link_speed(struct i40e_vsi *vsi)
5924	{
5925	struct i40e_pf *pf = vsi->back;
5926
5927	switch (pf->hw.phy.link_info.link_speed) {
5928	case I40E_LINK_SPEED_40GB:
5929	return 40000;
5930	case I40E_LINK_SPEED_25GB:
5931	return 25000;
5932	case I40E_LINK_SPEED_20GB:
5933	return 20000;
5934	case I40E_LINK_SPEED_10GB:
5935	return 10000;
5936	case I40E_LINK_SPEED_1GB:
5937	return 1000;
5938	default:
5939	return -EINVAL;
5940	}
5941	}
5942
5943	/**
5944	* i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits
5945	* @vsi: Pointer to vsi structure
5946	* @max_tx_rate: max TX rate in bytes to be converted into Mbits
5947	*
5948	* Helper function to convert units before send to set BW limit
5949	**/
5950	static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate)
5951	{
5952	if (max_tx_rate < I40E_BW_MBPS_DIVISOR) {
5953	dev_warn(&vsi->back->pdev->dev,
5954	"Setting max tx rate to minimum usable value of 50Mbps.\n");
5955	max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5956	} else {
5957	do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
5958	}
5959
5960	return max_tx_rate;
5961	}
5962
5963	/**
5964	* i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5965	* @vsi: VSI to be configured
5966	* @seid: seid of the channel/VSI
5967	* @max_tx_rate: max TX rate to be configured as BW limit
5968	*
5969	* Helper function to set BW limit for a given VSI
5970	**/
5971	int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5972	{
5973	struct i40e_pf *pf = vsi->back;
5974	u64 credits = 0;
5975	int speed = 0;
5976	int ret = 0;
5977
5978	speed = i40e_get_link_speed(vsi);
5979	if (max_tx_rate > speed) {
5980	dev_err(&pf->pdev->dev,
5981	"Invalid max tx rate %llu specified for VSI seid %d.",
5982	max_tx_rate, seid);
5983	return -EINVAL;
5984	}
5985	if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) {
5986	dev_warn(&pf->pdev->dev,
5987	"Setting max tx rate to minimum usable value of 50Mbps.\n");
5988	max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5989	}
5990
5991	/* Tx rate credits are in values of 50Mbps, 0 is disabled */
5992	credits = max_tx_rate;
5993	do_div(credits, I40E_BW_CREDIT_DIVISOR);
5994	ret = i40e_aq_config_vsi_bw_limit(hw: &pf->hw, seid, credit: credits,
5995	I40E_MAX_BW_INACTIVE_ACCUM, NULL);
5996	if (ret)
5997	dev_err(&pf->pdev->dev,
5998	"Failed set tx rate (%llu Mbps) for vsi->seid %u, err %pe aq_err %s\n",
5999	max_tx_rate, seid, ERR_PTR(ret),
6000	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6001	return ret;
6002	}
6003
6004	/**
6005	* i40e_remove_queue_channels - Remove queue channels for the TCs
6006	* @vsi: VSI to be configured
6007	*
6008	* Remove queue channels for the TCs
6009	**/
6010	static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
6011	{
6012	enum i40e_admin_queue_err last_aq_status;
6013	struct i40e_cloud_filter *cfilter;
6014	struct i40e_channel *ch, *ch_tmp;
6015	struct i40e_pf *pf = vsi->back;
6016	struct hlist_node *node;
6017	int ret, i;
6018
6019	/* Reset rss size that was stored when reconfiguring rss for
6020	* channel VSIs with non-power-of-2 queue count.
6021	*/
6022	vsi->current_rss_size = 0;
6023
6024	/* perform cleanup for channels if they exist */
6025	if (list_empty(head: &vsi->ch_list))
6026	return;
6027
6028	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
6029	struct i40e_vsi *p_vsi;
6030
6031	list_del(entry: &ch->list);
6032	p_vsi = ch->parent_vsi;
6033	if (!p_vsi || !ch->initialized) {
6034	kfree(objp: ch);
6035	continue;
6036	}
6037	/* Reset queue contexts */
6038	for (i = 0; i < ch->num_queue_pairs; i++) {
6039	struct i40e_ring *tx_ring, *rx_ring;
6040	u16 pf_q;
6041
6042	pf_q = ch->base_queue + i;
6043	tx_ring = vsi->tx_rings[pf_q];
6044	tx_ring->ch = NULL;
6045
6046	rx_ring = vsi->rx_rings[pf_q];
6047	rx_ring->ch = NULL;
6048	}
6049
6050	/* Reset BW configured for this VSI via mqprio */
6051	ret = i40e_set_bw_limit(vsi, seid: ch->seid, max_tx_rate: 0);
6052	if (ret)
6053	dev_info(&vsi->back->pdev->dev,
6054	"Failed to reset tx rate for ch->seid %u\n",
6055	ch->seid);
6056
6057	/* delete cloud filters associated with this channel */
6058	hlist_for_each_entry_safe(cfilter, node,
6059	&pf->cloud_filter_list, cloud_node) {
6060	if (cfilter->seid != ch->seid)
6061	continue;
6062
6063	hash_del(node: &cfilter->cloud_node);
6064	if (cfilter->dst_port)
6065	ret = i40e_add_del_cloud_filter_big_buf(vsi,
6066	filter: cfilter,
6067	add: false);
6068	else
6069	ret = i40e_add_del_cloud_filter(vsi, filter: cfilter,
6070	add: false);
6071	last_aq_status = pf->hw.aq.asq_last_status;
6072	if (ret)
6073	dev_info(&pf->pdev->dev,
6074	"Failed to delete cloud filter, err %pe aq_err %s\n",
6075	ERR_PTR(ret),
6076	i40e_aq_str(&pf->hw, last_aq_status));
6077	kfree(objp: cfilter);
6078	}
6079
6080	/* delete VSI from FW */
6081	ret = i40e_aq_delete_element(hw: &vsi->back->hw, seid: ch->seid,
6082	NULL);
6083	if (ret)
6084	dev_err(&vsi->back->pdev->dev,
6085	"unable to remove channel (%d) for parent VSI(%d)\n",
6086	ch->seid, p_vsi->seid);
6087	kfree(objp: ch);
6088	}
6089	INIT_LIST_HEAD(list: &vsi->ch_list);
6090	}
6091
6092	/**
6093	* i40e_get_max_queues_for_channel
6094	* @vsi: ptr to VSI to which channels are associated with
6095	*
6096	* Helper function which returns max value among the queue counts set on the
6097	* channels/TCs created.
6098	**/
6099	static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
6100	{
6101	struct i40e_channel *ch, *ch_tmp;
6102	int max = 0;
6103
6104	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
6105	if (!ch->initialized)
6106	continue;
6107	if (ch->num_queue_pairs > max)
6108	max = ch->num_queue_pairs;
6109	}
6110
6111	return max;
6112	}
6113
6114	/**
6115	* i40e_validate_num_queues - validate num_queues w.r.t channel
6116	* @pf: ptr to PF device
6117	* @num_queues: number of queues
6118	* @vsi: the parent VSI
6119	* @reconfig_rss: indicates should the RSS be reconfigured or not
6120	*
6121	* This function validates number of queues in the context of new channel
6122	* which is being established and determines if RSS should be reconfigured
6123	* or not for parent VSI.
6124	**/
6125	static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
6126	struct i40e_vsi *vsi, bool *reconfig_rss)
6127	{
6128	int max_ch_queues;
6129
6130	if (!reconfig_rss)
6131	return -EINVAL;
6132
6133	*reconfig_rss = false;
6134	if (vsi->current_rss_size) {
6135	if (num_queues > vsi->current_rss_size) {
6136	dev_dbg(&pf->pdev->dev,
6137	"Error: num_queues (%d) > vsi's current_size(%d)\n",
6138	num_queues, vsi->current_rss_size);
6139	return -EINVAL;
6140	} else if ((num_queues < vsi->current_rss_size) &&
6141	(!is_power_of_2(n: num_queues))) {
6142	dev_dbg(&pf->pdev->dev,
6143	"Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
6144	num_queues, vsi->current_rss_size);
6145	return -EINVAL;
6146	}
6147	}
6148
6149	if (!is_power_of_2(n: num_queues)) {
6150	/* Find the max num_queues configured for channel if channel
6151	* exist.
6152	* if channel exist, then enforce 'num_queues' to be more than
6153	* max ever queues configured for channel.
6154	*/
6155	max_ch_queues = i40e_get_max_queues_for_channel(vsi);
6156	if (num_queues < max_ch_queues) {
6157	dev_dbg(&pf->pdev->dev,
6158	"Error: num_queues (%d) < max queues configured for channel(%d)\n",
6159	num_queues, max_ch_queues);
6160	return -EINVAL;
6161	}
6162	*reconfig_rss = true;
6163	}
6164
6165	return 0;
6166	}
6167
6168	/**
6169	* i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
6170	* @vsi: the VSI being setup
6171	* @rss_size: size of RSS, accordingly LUT gets reprogrammed
6172	*
6173	* This function reconfigures RSS by reprogramming LUTs using 'rss_size'
6174	**/
6175	static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
6176	{
6177	struct i40e_pf *pf = vsi->back;
6178	u8 seed[I40E_HKEY_ARRAY_SIZE];
6179	struct i40e_hw *hw = &pf->hw;
6180	int local_rss_size;
6181	u8 *lut;
6182	int ret;
6183
6184	if (!vsi->rss_size)
6185	return -EINVAL;
6186
6187	if (rss_size > vsi->rss_size)
6188	return -EINVAL;
6189
6190	local_rss_size = min_t(int, vsi->rss_size, rss_size);
6191	lut = kzalloc(size: vsi->rss_table_size, GFP_KERNEL);
6192	if (!lut)
6193	return -ENOMEM;
6194
6195	/* Ignoring user configured lut if there is one */
6196	i40e_fill_rss_lut(pf, lut, rss_table_size: vsi->rss_table_size, rss_size: local_rss_size);
6197
6198	/* Use user configured hash key if there is one, otherwise
6199	* use default.
6200	*/
6201	if (vsi->rss_hkey_user)
6202	memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
6203	else
6204	netdev_rss_key_fill(buffer: (void *)seed, I40E_HKEY_ARRAY_SIZE);
6205
6206	ret = i40e_config_rss(vsi, seed, lut, lut_size: vsi->rss_table_size);
6207	if (ret) {
6208	dev_info(&pf->pdev->dev,
6209	"Cannot set RSS lut, err %pe aq_err %s\n",
6210	ERR_PTR(ret),
6211	i40e_aq_str(hw, hw->aq.asq_last_status));
6212	kfree(objp: lut);
6213	return ret;
6214	}
6215	kfree(objp: lut);
6216
6217	/* Do the update w.r.t. storing rss_size */
6218	if (!vsi->orig_rss_size)
6219	vsi->orig_rss_size = vsi->rss_size;
6220	vsi->current_rss_size = local_rss_size;
6221
6222	return ret;
6223	}
6224
6225	/**
6226	* i40e_channel_setup_queue_map - Setup a channel queue map
6227	* @pf: ptr to PF device
6228	* @ctxt: VSI context structure
6229	* @ch: ptr to channel structure
6230	*
6231	* Setup queue map for a specific channel
6232	**/
6233	static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
6234	struct i40e_vsi_context *ctxt,
6235	struct i40e_channel *ch)
6236	{
6237	u16 qcount, qmap, sections = 0;
6238	u8 offset = 0;
6239	int pow;
6240
6241	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
6242	sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
6243
6244	qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
6245	ch->num_queue_pairs = qcount;
6246
6247	/* find the next higher power-of-2 of num queue pairs */
6248	pow = ilog2(qcount);
6249	if (!is_power_of_2(n: qcount))
6250	pow++;
6251
6252	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6253	(pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
6254
6255	/* Setup queue TC[0].qmap for given VSI context */
6256	ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
6257
6258	ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
6259	ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
6260	ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
6261	ctxt->info.valid_sections |= cpu_to_le16(sections);
6262	}
6263
6264	/**
6265	* i40e_add_channel - add a channel by adding VSI
6266	* @pf: ptr to PF device
6267	* @uplink_seid: underlying HW switching element (VEB) ID
6268	* @ch: ptr to channel structure
6269	*
6270	* Add a channel (VSI) using add_vsi and queue_map
6271	**/
6272	static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
6273	struct i40e_channel *ch)
6274	{
6275	struct i40e_hw *hw = &pf->hw;
6276	struct i40e_vsi_context ctxt;
6277	u8 enabled_tc = 0x1; /* TC0 enabled */
6278	int ret;
6279
6280	if (ch->type != I40E_VSI_VMDQ2) {
6281	dev_info(&pf->pdev->dev,
6282	"add new vsi failed, ch->type %d\n", ch->type);
6283	return -EINVAL;
6284	}
6285
6286	memset(&ctxt, 0, sizeof(ctxt));
6287	ctxt.pf_num = hw->pf_id;
6288	ctxt.vf_num = 0;
6289	ctxt.uplink_seid = uplink_seid;
6290	ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
6291	if (ch->type == I40E_VSI_VMDQ2)
6292	ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
6293
6294	if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
6295	ctxt.info.valid_sections |=
6296	cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6297	ctxt.info.switch_id =
6298	cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6299	}
6300
6301	/* Set queue map for a given VSI context */
6302	i40e_channel_setup_queue_map(pf, ctxt: &ctxt, ch);
6303
6304	/* Now time to create VSI */
6305	ret = i40e_aq_add_vsi(hw, vsi_ctx: &ctxt, NULL);
6306	if (ret) {
6307	dev_info(&pf->pdev->dev,
6308	"add new vsi failed, err %pe aq_err %s\n",
6309	ERR_PTR(ret),
6310	i40e_aq_str(&pf->hw,
6311	pf->hw.aq.asq_last_status));
6312	return -ENOENT;
6313	}
6314
6315	/* Success, update channel, set enabled_tc only if the channel
6316	* is not a macvlan
6317	*/
6318	ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
6319	ch->seid = ctxt.seid;
6320	ch->vsi_number = ctxt.vsi_number;
6321	ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);
6322
6323	/* copy just the sections touched not the entire info
6324	* since not all sections are valid as returned by
6325	* update vsi params
6326	*/
6327	ch->info.mapping_flags = ctxt.info.mapping_flags;
6328	memcpy(&ch->info.queue_mapping,
6329	&ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
6330	memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
6331	sizeof(ctxt.info.tc_mapping));
6332
6333	return 0;
6334	}
6335
6336	static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
6337	u8 *bw_share)
6338	{
6339	struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
6340	int ret;
6341	int i;
6342
6343	memset(&bw_data, 0, sizeof(bw_data));
6344	bw_data.tc_valid_bits = ch->enabled_tc;
6345	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6346	bw_data.tc_bw_credits[i] = bw_share[i];
6347
6348	ret = i40e_aq_config_vsi_tc_bw(hw: &vsi->back->hw, seid: ch->seid,
6349	bw_data: &bw_data, NULL);
6350	if (ret) {
6351	dev_info(&vsi->back->pdev->dev,
6352	"Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
6353	vsi->back->hw.aq.asq_last_status, ch->seid);
6354	return -EINVAL;
6355	}
6356
6357	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6358	ch->info.qs_handle[i] = bw_data.qs_handles[i];
6359
6360	return 0;
6361	}
6362
6363	/**
6364	* i40e_channel_config_tx_ring - config TX ring associated with new channel
6365	* @pf: ptr to PF device
6366	* @vsi: the VSI being setup
6367	* @ch: ptr to channel structure
6368	*
6369	* Configure TX rings associated with channel (VSI) since queues are being
6370	* from parent VSI.
6371	**/
6372	static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
6373	struct i40e_vsi *vsi,
6374	struct i40e_channel *ch)
6375	{
6376	u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
6377	int ret;
6378	int i;
6379
6380	/* Enable ETS TCs with equal BW Share for now across all VSIs */
6381	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6382	if (ch->enabled_tc & BIT(i))
6383	bw_share[i] = 1;
6384	}
6385
6386	/* configure BW for new VSI */
6387	ret = i40e_channel_config_bw(vsi, ch, bw_share);
6388	if (ret) {
6389	dev_info(&vsi->back->pdev->dev,
6390	"Failed configuring TC map %d for channel (seid %u)\n",
6391	ch->enabled_tc, ch->seid);
6392	return ret;
6393	}
6394
6395	for (i = 0; i < ch->num_queue_pairs; i++) {
6396	struct i40e_ring *tx_ring, *rx_ring;
6397	u16 pf_q;
6398
6399	pf_q = ch->base_queue + i;
6400
6401	/* Get to TX ring ptr of main VSI, for re-setup TX queue
6402	* context
6403	*/
6404	tx_ring = vsi->tx_rings[pf_q];
6405	tx_ring->ch = ch;
6406
6407	/* Get the RX ring ptr */
6408	rx_ring = vsi->rx_rings[pf_q];
6409	rx_ring->ch = ch;
6410	}
6411
6412	return 0;
6413	}
6414
6415	/**
6416	* i40e_setup_hw_channel - setup new channel
6417	* @pf: ptr to PF device
6418	* @vsi: the VSI being setup
6419	* @ch: ptr to channel structure
6420	* @uplink_seid: underlying HW switching element (VEB) ID
6421	* @type: type of channel to be created (VMDq2/VF)
6422	*
6423	* Setup new channel (VSI) based on specified type (VMDq2/VF)
6424	* and configures TX rings accordingly
6425	**/
6426	static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
6427	struct i40e_vsi *vsi,
6428	struct i40e_channel *ch,
6429	u16 uplink_seid, u8 type)
6430	{
6431	int ret;
6432
6433	ch->initialized = false;
6434	ch->base_queue = vsi->next_base_queue;
6435	ch->type = type;
6436
6437	/* Proceed with creation of channel (VMDq2) VSI */
6438	ret = i40e_add_channel(pf, uplink_seid, ch);
6439	if (ret) {
6440	dev_info(&pf->pdev->dev,
6441	"failed to add_channel using uplink_seid %u\n",
6442	uplink_seid);
6443	return ret;
6444	}
6445
6446	/* Mark the successful creation of channel */
6447	ch->initialized = true;
6448
6449	/* Reconfigure TX queues using QTX_CTL register */
6450	ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6451	if (ret) {
6452	dev_info(&pf->pdev->dev,
6453	"failed to configure TX rings for channel %u\n",
6454	ch->seid);
6455	return ret;
6456	}
6457
6458	/* update 'next_base_queue' */
6459	vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6460	dev_dbg(&pf->pdev->dev,
6461	"Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6462	ch->seid, ch->vsi_number, ch->stat_counter_idx,
6463	ch->num_queue_pairs,
6464	vsi->next_base_queue);
6465	return ret;
6466	}
6467
6468	/**
6469	* i40e_setup_channel - setup new channel using uplink element
6470	* @pf: ptr to PF device
6471	* @vsi: pointer to the VSI to set up the channel within
6472	* @ch: ptr to channel structure
6473	*
6474	* Setup new channel (VSI) based on specified type (VMDq2/VF)
6475	* and uplink switching element (uplink_seid)
6476	**/
6477	static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6478	struct i40e_channel *ch)
6479	{
6480	u8 vsi_type;
6481	u16 seid;
6482	int ret;
6483
6484	if (vsi->type == I40E_VSI_MAIN) {
6485	vsi_type = I40E_VSI_VMDQ2;
6486	} else {
6487	dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6488	vsi->type);
6489	return false;
6490	}
6491
6492	/* underlying switching element */
6493	seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6494
6495	/* create channel (VSI), configure TX rings */
6496	ret = i40e_setup_hw_channel(pf, vsi, ch, uplink_seid: seid, type: vsi_type);
6497	if (ret) {
6498	dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6499	return false;
6500	}
6501
6502	return ch->initialized ? true : false;
6503	}
6504
6505	/**
6506	* i40e_validate_and_set_switch_mode - sets up switch mode correctly
6507	* @vsi: ptr to VSI which has PF backing
6508	*
6509	* Sets up switch mode correctly if it needs to be changed and perform
6510	* what are allowed modes.
6511	**/
6512	static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6513	{
6514	u8 mode;
6515	struct i40e_pf *pf = vsi->back;
6516	struct i40e_hw *hw = &pf->hw;
6517	int ret;
6518
6519	ret = i40e_get_capabilities(pf, list_type: i40e_aqc_opc_list_dev_capabilities);
6520	if (ret)
6521	return -EINVAL;
6522
6523	if (hw->dev_caps.switch_mode) {
6524	/* if switch mode is set, support mode2 (non-tunneled for
6525	* cloud filter) for now
6526	*/
6527	u32 switch_mode = hw->dev_caps.switch_mode &
6528	I40E_SWITCH_MODE_MASK;
6529	if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6530	if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6531	return 0;
6532	dev_err(&pf->pdev->dev,
6533	"Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6534	hw->dev_caps.switch_mode);
6535	return -EINVAL;
6536	}
6537	}
6538
6539	/* Set Bit 7 to be valid */
6540	mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6541
6542	/* Set L4type for TCP support */
6543	mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6544
6545	/* Set cloud filter mode */
6546	mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6547
6548	/* Prep mode field for set_switch_config */
6549	ret = i40e_aq_set_switch_config(hw, flags: pf->last_sw_conf_flags,
6550	valid_flags: pf->last_sw_conf_valid_flags,
6551	mode, NULL);
6552	if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6553	dev_err(&pf->pdev->dev,
6554	"couldn't set switch config bits, err %pe aq_err %s\n",
6555	ERR_PTR(ret),
6556	i40e_aq_str(hw,
6557	hw->aq.asq_last_status));
6558
6559	return ret;
6560	}
6561
6562	/**
6563	* i40e_create_queue_channel - function to create channel
6564	* @vsi: VSI to be configured
6565	* @ch: ptr to channel (it contains channel specific params)
6566	*
6567	* This function creates channel (VSI) using num_queues specified by user,
6568	* reconfigs RSS if needed.
6569	**/
6570	int i40e_create_queue_channel(struct i40e_vsi *vsi,
6571	struct i40e_channel *ch)
6572	{
6573	struct i40e_pf *pf = vsi->back;
6574	bool reconfig_rss;
6575	int err;
6576
6577	if (!ch)
6578	return -EINVAL;
6579
6580	if (!ch->num_queue_pairs) {
6581	dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6582	ch->num_queue_pairs);
6583	return -EINVAL;
6584	}
6585
6586	/* validate user requested num_queues for channel */
6587	err = i40e_validate_num_queues(pf, num_queues: ch->num_queue_pairs, vsi,
6588	reconfig_rss: &reconfig_rss);
6589	if (err) {
6590	dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6591	ch->num_queue_pairs);
6592	return -EINVAL;
6593	}
6594
6595	/* By default we are in VEPA mode, if this is the first VF/VMDq
6596	* VSI to be added switch to VEB mode.
6597	*/
6598
6599	if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
6600	set_bit(nr: I40E_FLAG_VEB_MODE_ENA, addr: pf->flags);
6601
6602	if (vsi->type == I40E_VSI_MAIN) {
6603	if (i40e_is_tc_mqprio_enabled(pf))
6604	i40e_do_reset(pf, I40E_PF_RESET_FLAG, lock_acquired: true);
6605	else
6606	i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
6607	}
6608	/* now onwards for main VSI, number of queues will be value
6609	* of TC0's queue count
6610	*/
6611	}
6612
6613	/* By this time, vsi->cnt_q_avail shall be set to non-zero and
6614	* it should be more than num_queues
6615	*/
6616	if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6617	dev_dbg(&pf->pdev->dev,
6618	"Error: cnt_q_avail (%u) less than num_queues %d\n",
6619	vsi->cnt_q_avail, ch->num_queue_pairs);
6620	return -EINVAL;
6621	}
6622
6623	/* reconfig_rss only if vsi type is MAIN_VSI */
6624	if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6625	err = i40e_vsi_reconfig_rss(vsi, rss_size: ch->num_queue_pairs);
6626	if (err) {
6627	dev_info(&pf->pdev->dev,
6628	"Error: unable to reconfig rss for num_queues (%u)\n",
6629	ch->num_queue_pairs);
6630	return -EINVAL;
6631	}
6632	}
6633
6634	if (!i40e_setup_channel(pf, vsi, ch)) {
6635	dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6636	return -EINVAL;
6637	}
6638
6639	dev_info(&pf->pdev->dev,
6640	"Setup channel (id:%u) utilizing num_queues %d\n",
6641	ch->seid, ch->num_queue_pairs);
6642
6643	/* configure VSI for BW limit */
6644	if (ch->max_tx_rate) {
6645	u64 credits = ch->max_tx_rate;
6646
6647	if (i40e_set_bw_limit(vsi, seid: ch->seid, max_tx_rate: ch->max_tx_rate))
6648	return -EINVAL;
6649
6650	do_div(credits, I40E_BW_CREDIT_DIVISOR);
6651	dev_dbg(&pf->pdev->dev,
6652	"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6653	ch->max_tx_rate,
6654	credits,
6655	ch->seid);
6656	}
6657
6658	/* in case of VF, this will be main SRIOV VSI */
6659	ch->parent_vsi = vsi;
6660
6661	/* and update main_vsi's count for queue_available to use */
6662	vsi->cnt_q_avail -= ch->num_queue_pairs;
6663
6664	return 0;
6665	}
6666
6667	/**
6668	* i40e_configure_queue_channels - Add queue channel for the given TCs
6669	* @vsi: VSI to be configured
6670	*
6671	* Configures queue channel mapping to the given TCs
6672	**/
6673	static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6674	{
6675	struct i40e_channel *ch;
6676	u64 max_rate = 0;
6677	int ret = 0, i;
6678
6679	/* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6680	vsi->tc_seid_map[0] = vsi->seid;
6681	for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6682	if (vsi->tc_config.enabled_tc & BIT(i)) {
6683	ch = kzalloc(size: sizeof(*ch), GFP_KERNEL);
6684	if (!ch) {
6685	ret = -ENOMEM;
6686	goto err_free;
6687	}
6688
6689	INIT_LIST_HEAD(list: &ch->list);
6690	ch->num_queue_pairs =
6691	vsi->tc_config.tc_info[i].qcount;
6692	ch->base_queue =
6693	vsi->tc_config.tc_info[i].qoffset;
6694
6695	/* Bandwidth limit through tc interface is in bytes/s,
6696	* change to Mbit/s
6697	*/
6698	max_rate = vsi->mqprio_qopt.max_rate[i];
6699	do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6700	ch->max_tx_rate = max_rate;
6701
6702	list_add_tail(new: &ch->list, head: &vsi->ch_list);
6703
6704	ret = i40e_create_queue_channel(vsi, ch);
6705	if (ret) {
6706	dev_err(&vsi->back->pdev->dev,
6707	"Failed creating queue channel with TC%d: queues %d\n",
6708	i, ch->num_queue_pairs);
6709	goto err_free;
6710	}
6711	vsi->tc_seid_map[i] = ch->seid;
6712	}
6713	}
6714
6715	/* reset to reconfigure TX queue contexts */
6716	i40e_do_reset(pf: vsi->back, I40E_PF_RESET_FLAG, lock_acquired: true);
6717	return ret;
6718
6719	err_free:
6720	i40e_remove_queue_channels(vsi);
6721	return ret;
6722	}
6723
6724	/**
6725	* i40e_veb_config_tc - Configure TCs for given VEB
6726	* @veb: given VEB
6727	* @enabled_tc: TC bitmap
6728	*
6729	* Configures given TC bitmap for VEB (switching) element
6730	**/
6731	int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6732	{
6733	struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6734	struct i40e_pf *pf = veb->pf;
6735	int ret = 0;
6736	int i;
6737
6738	/* No TCs or already enabled TCs just return */
6739	if (!enabled_tc || veb->enabled_tc == enabled_tc)
6740	return ret;
6741
6742	bw_data.tc_valid_bits = enabled_tc;
6743	/* bw_data.absolute_credits is not set (relative) */
6744
6745	/* Enable ETS TCs with equal BW Share for now */
6746	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6747	if (enabled_tc & BIT(i))
6748	bw_data.tc_bw_share_credits[i] = 1;
6749	}
6750
6751	ret = i40e_aq_config_switch_comp_bw_config(hw: &pf->hw, seid: veb->seid,
6752	bw_data: &bw_data, NULL);
6753	if (ret) {
6754	dev_info(&pf->pdev->dev,
6755	"VEB bw config failed, err %pe aq_err %s\n",
6756	ERR_PTR(ret),
6757	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6758	goto out;
6759	}
6760
6761	/* Update the BW information */
6762	ret = i40e_veb_get_bw_info(veb);
6763	if (ret) {
6764	dev_info(&pf->pdev->dev,
6765	"Failed getting veb bw config, err %pe aq_err %s\n",
6766	ERR_PTR(ret),
6767	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6768	}
6769
6770	out:
6771	return ret;
6772	}
6773
6774	#ifdef CONFIG_I40E_DCB
6775	/**
6776	* i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6777	* @pf: PF struct
6778	*
6779	* Reconfigure VEB/VSIs on a given PF; it is assumed that
6780	* the caller would've quiesce all the VSIs before calling
6781	* this function
6782	**/
6783	static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6784	{
6785	struct i40e_vsi *vsi;
6786	struct i40e_veb *veb;
6787	u8 tc_map = 0;
6788	int ret;
6789	int v;
6790
6791	/* Enable the TCs available on PF to all VEBs */
6792	tc_map = i40e_pf_get_tc_map(pf);
6793	if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS)
6794	return;
6795
6796	i40e_pf_for_each_veb(pf, v, veb) {
6797	ret = i40e_veb_config_tc(veb, enabled_tc: tc_map);
6798	if (ret) {
6799	dev_info(&pf->pdev->dev,
6800	"Failed configuring TC for VEB seid=%d\n",
6801	veb->seid);
6802	/* Will try to configure as many components */
6803	}
6804	}
6805
6806	/* Update each VSI */
6807	i40e_pf_for_each_vsi(pf, v, vsi) {
6808	/* - Enable all TCs for the LAN VSI
6809	* - For all others keep them at TC0 for now
6810	*/
6811	if (v == pf->lan_vsi)
6812	tc_map = i40e_pf_get_tc_map(pf);
6813	else
6814	tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6815
6816	ret = i40e_vsi_config_tc(vsi, enabled_tc: tc_map);
6817	if (ret) {
6818	dev_info(&pf->pdev->dev,
6819	"Failed configuring TC for VSI seid=%d\n",
6820	vsi->seid);
6821	/* Will try to configure as many components */
6822	} else {
6823	/* Re-configure VSI vectors based on updated TC map */
6824	i40e_vsi_map_rings_to_vectors(vsi);
6825	if (vsi->netdev)
6826	i40e_dcbnl_set_all(vsi);
6827	}
6828	}
6829	}
6830
6831	/**
6832	* i40e_resume_port_tx - Resume port Tx
6833	* @pf: PF struct
6834	*
6835	* Resume a port's Tx and issue a PF reset in case of failure to
6836	* resume.
6837	**/
6838	static int i40e_resume_port_tx(struct i40e_pf *pf)
6839	{
6840	struct i40e_hw *hw = &pf->hw;
6841	int ret;
6842
6843	ret = i40e_aq_resume_port_tx(hw, NULL);
6844	if (ret) {
6845	dev_info(&pf->pdev->dev,
6846	"Resume Port Tx failed, err %pe aq_err %s\n",
6847	ERR_PTR(ret),
6848	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6849	/* Schedule PF reset to recover */
6850	set_bit(nr: __I40E_PF_RESET_REQUESTED, addr: pf->state);
6851	i40e_service_event_schedule(pf);
6852	}
6853
6854	return ret;
6855	}
6856
6857	/**
6858	* i40e_suspend_port_tx - Suspend port Tx
6859	* @pf: PF struct
6860	*
6861	* Suspend a port's Tx and issue a PF reset in case of failure.
6862	**/
6863	static int i40e_suspend_port_tx(struct i40e_pf *pf)
6864	{
6865	struct i40e_hw *hw = &pf->hw;
6866	int ret;
6867
6868	ret = i40e_aq_suspend_port_tx(hw, seid: pf->mac_seid, NULL);
6869	if (ret) {
6870	dev_info(&pf->pdev->dev,
6871	"Suspend Port Tx failed, err %pe aq_err %s\n",
6872	ERR_PTR(ret),
6873	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6874	/* Schedule PF reset to recover */
6875	set_bit(nr: __I40E_PF_RESET_REQUESTED, addr: pf->state);
6876	i40e_service_event_schedule(pf);
6877	}
6878
6879	return ret;
6880	}
6881
6882	/**
6883	* i40e_hw_set_dcb_config - Program new DCBX settings into HW
6884	* @pf: PF being configured
6885	* @new_cfg: New DCBX configuration
6886	*
6887	* Program DCB settings into HW and reconfigure VEB/VSIs on
6888	* given PF. Uses "Set LLDP MIB" AQC to program the hardware.
6889	**/
6890	static int i40e_hw_set_dcb_config(struct i40e_pf *pf,
6891	struct i40e_dcbx_config *new_cfg)
6892	{
6893	struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config;
6894	int ret;
6895
6896	/* Check if need reconfiguration */
6897	if (!memcmp(p: &new_cfg, q: &old_cfg, size: sizeof(new_cfg))) {
6898	dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n");
6899	return 0;
6900	}
6901
6902	/* Config change disable all VSIs */
6903	i40e_pf_quiesce_all_vsi(pf);
6904
6905	/* Copy the new config to the current config */
6906	*old_cfg = *new_cfg;
6907	old_cfg->etsrec = old_cfg->etscfg;
6908	ret = i40e_set_dcb_config(hw: &pf->hw);
6909	if (ret) {
6910	dev_info(&pf->pdev->dev,
6911	"Set DCB Config failed, err %pe aq_err %s\n",
6912	ERR_PTR(ret),
6913	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6914	goto out;
6915	}
6916
6917	/* Changes in configuration update VEB/VSI */
6918	i40e_dcb_reconfigure(pf);
6919	out:
6920	/* In case of reset do not try to resume anything */
6921	if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {
6922	/* Re-start the VSIs if disabled */
6923	ret = i40e_resume_port_tx(pf);
6924	/* In case of error no point in resuming VSIs */
6925	if (ret)
6926	goto err;
6927	i40e_pf_unquiesce_all_vsi(pf);
6928	}
6929	err:
6930	return ret;
6931	}
6932
6933	/**
6934	* i40e_hw_dcb_config - Program new DCBX settings into HW
6935	* @pf: PF being configured
6936	* @new_cfg: New DCBX configuration
6937	*
6938	* Program DCB settings into HW and reconfigure VEB/VSIs on
6939	* given PF
6940	**/
6941	int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)
6942	{
6943	struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6944	u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0};
6945	u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS];
6946	struct i40e_dcbx_config *old_cfg;
6947	u8 mode[I40E_MAX_TRAFFIC_CLASS];
6948	struct i40e_rx_pb_config pb_cfg;
6949	struct i40e_hw *hw = &pf->hw;
6950	u8 num_ports = hw->num_ports;
6951	bool need_reconfig;
6952	int ret = -EINVAL;
6953	u8 lltc_map = 0;
6954	u8 tc_map = 0;
6955	u8 new_numtc;
6956	u8 i;
6957
6958	dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n");
6959	/* Un-pack information to Program ETS HW via shared API
6960	* numtc, tcmap
6961	* LLTC map
6962	* ETS/NON-ETS arbiter mode
6963	* max exponent (credit refills)
6964	* Total number of ports
6965	* PFC priority bit-map
6966	* Priority Table
6967	* BW % per TC
6968	* Arbiter mode between UPs sharing same TC
6969	* TSA table (ETS or non-ETS)
6970	* EEE enabled or not
6971	* MFS TC table
6972	*/
6973
6974	new_numtc = i40e_dcb_get_num_tc(dcbcfg: new_cfg);
6975
6976	memset(&ets_data, 0, sizeof(ets_data));
6977	for (i = 0; i < new_numtc; i++) {
6978	tc_map |= BIT(i);
6979	switch (new_cfg->etscfg.tsatable[i]) {
6980	case I40E_IEEE_TSA_ETS:
6981	prio_type[i] = I40E_DCB_PRIO_TYPE_ETS;
6982	ets_data.tc_bw_share_credits[i] =
6983	new_cfg->etscfg.tcbwtable[i];
6984	break;
6985	case I40E_IEEE_TSA_STRICT:
6986	prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT;
6987	lltc_map |= BIT(i);
6988	ets_data.tc_bw_share_credits[i] =
6989	I40E_DCB_STRICT_PRIO_CREDITS;
6990	break;
6991	default:
6992	/* Invalid TSA type */
6993	need_reconfig = false;
6994	goto out;
6995	}
6996	}
6997
6998	old_cfg = &hw->local_dcbx_config;
6999	/* Check if need reconfiguration */
7000	need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg);
7001
7002	/* If needed, enable/disable frame tagging, disable all VSIs
7003	* and suspend port tx
7004	*/
7005	if (need_reconfig) {
7006	/* Enable DCB tagging only when more than one TC */
7007	if (new_numtc > 1)
7008	set_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
7009	else
7010	clear_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
7011
7012	set_bit(nr: __I40E_PORT_SUSPENDED, addr: pf->state);
7013	/* Reconfiguration needed quiesce all VSIs */
7014	i40e_pf_quiesce_all_vsi(pf);
7015	ret = i40e_suspend_port_tx(pf);
7016	if (ret)
7017	goto err;
7018	}
7019
7020	/* Configure Port ETS Tx Scheduler */
7021	ets_data.tc_valid_bits = tc_map;
7022	ets_data.tc_strict_priority_flags = lltc_map;
7023	ret = i40e_aq_config_switch_comp_ets
7024	(hw, seid: pf->mac_seid, ets_data: &ets_data,
7025	opcode: i40e_aqc_opc_modify_switching_comp_ets, NULL);
7026	if (ret) {
7027	dev_info(&pf->pdev->dev,
7028	"Modify Port ETS failed, err %pe aq_err %s\n",
7029	ERR_PTR(ret),
7030	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7031	goto out;
7032	}
7033
7034	/* Configure Rx ETS HW */
7035	memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode));
7036	i40e_dcb_hw_set_num_tc(hw, num_tc: new_numtc);
7037	i40e_dcb_hw_rx_fifo_config(hw, ets_mode: I40E_DCB_ARB_MODE_ROUND_ROBIN,
7038	non_ets_mode: I40E_DCB_ARB_MODE_STRICT_PRIORITY,
7039	I40E_DCB_DEFAULT_MAX_EXPONENT,
7040	lltc_map);
7041	i40e_dcb_hw_rx_cmd_monitor_config(hw, num_tc: new_numtc, num_ports);
7042	i40e_dcb_hw_rx_ets_bw_config(hw, bw_share: new_cfg->etscfg.tcbwtable, mode,
7043	prio_type);
7044	i40e_dcb_hw_pfc_config(hw, pfc_en: new_cfg->pfc.pfcenable,
7045	prio_tc: new_cfg->etscfg.prioritytable);
7046	i40e_dcb_hw_rx_up2tc_config(hw, prio_tc: new_cfg->etscfg.prioritytable);
7047
7048	/* Configure Rx Packet Buffers in HW */
7049	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7050	mfs_tc[i] = pf->vsi[pf->lan_vsi]->netdev->mtu;
7051	mfs_tc[i] += I40E_PACKET_HDR_PAD;
7052	}
7053
7054	i40e_dcb_hw_calculate_pool_sizes(hw, num_ports,
7055	eee_enabled: false, pfc_en: new_cfg->pfc.pfcenable,
7056	mfs_tc, pb_cfg: &pb_cfg);
7057	i40e_dcb_hw_rx_pb_config(hw, old_pb_cfg: &pf->pb_cfg, new_pb_cfg: &pb_cfg);
7058
7059	/* Update the local Rx Packet buffer config */
7060	pf->pb_cfg = pb_cfg;
7061
7062	/* Inform the FW about changes to DCB configuration */
7063	ret = i40e_aq_dcb_updated(hw: &pf->hw, NULL);
7064	if (ret) {
7065	dev_info(&pf->pdev->dev,
7066	"DCB Updated failed, err %pe aq_err %s\n",
7067	ERR_PTR(ret),
7068	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7069	goto out;
7070	}
7071
7072	/* Update the port DCBx configuration */
7073	*old_cfg = *new_cfg;
7074
7075	/* Changes in configuration update VEB/VSI */
7076	i40e_dcb_reconfigure(pf);
7077	out:
7078	/* Re-start the VSIs if disabled */
7079	if (need_reconfig) {
7080	ret = i40e_resume_port_tx(pf);
7081
7082	clear_bit(nr: __I40E_PORT_SUSPENDED, addr: pf->state);
7083	/* In case of error no point in resuming VSIs */
7084	if (ret)
7085	goto err;
7086
7087	/* Wait for the PF's queues to be disabled */
7088	ret = i40e_pf_wait_queues_disabled(pf);
7089	if (ret) {
7090	/* Schedule PF reset to recover */
7091	set_bit(nr: __I40E_PF_RESET_REQUESTED, addr: pf->state);
7092	i40e_service_event_schedule(pf);
7093	goto err;
7094	} else {
7095	i40e_pf_unquiesce_all_vsi(pf);
7096	set_bit(nr: __I40E_CLIENT_SERVICE_REQUESTED, addr: pf->state);
7097	set_bit(nr: __I40E_CLIENT_L2_CHANGE, addr: pf->state);
7098	}
7099	/* registers are set, lets apply */
7100	if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps))
7101	ret = i40e_hw_set_dcb_config(pf, new_cfg);
7102	}
7103
7104	err:
7105	return ret;
7106	}
7107
7108	/**
7109	* i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW
7110	* @pf: PF being queried
7111	*
7112	* Set default DCB configuration in case DCB is to be done in SW.
7113	**/
7114	int i40e_dcb_sw_default_config(struct i40e_pf *pf)
7115	{
7116	struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config;
7117	struct i40e_aqc_configure_switching_comp_ets_data ets_data;
7118	struct i40e_hw *hw = &pf->hw;
7119	int err;
7120
7121	if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) {
7122	/* Update the local cached instance with TC0 ETS */
7123	memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));
7124	pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7125	pf->tmp_cfg.etscfg.maxtcs = 0;
7126	pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7127	pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;
7128	pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING;
7129	pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
7130	/* FW needs one App to configure HW */
7131	pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS;
7132	pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE;
7133	pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO;
7134	pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE;
7135
7136	return i40e_hw_set_dcb_config(pf, new_cfg: &pf->tmp_cfg);
7137	}
7138
7139	memset(&ets_data, 0, sizeof(ets_data));
7140	ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */
7141	ets_data.tc_strict_priority_flags = 0; /* ETS */
7142	ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */
7143
7144	/* Enable ETS on the Physical port */
7145	err = i40e_aq_config_switch_comp_ets
7146	(hw, seid: pf->mac_seid, ets_data: &ets_data,
7147	opcode: i40e_aqc_opc_enable_switching_comp_ets, NULL);
7148	if (err) {
7149	dev_info(&pf->pdev->dev,
7150	"Enable Port ETS failed, err %pe aq_err %s\n",
7151	ERR_PTR(err),
7152	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7153	err = -ENOENT;
7154	goto out;
7155	}
7156
7157	/* Update the local cached instance with TC0 ETS */
7158	dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7159	dcb_cfg->etscfg.cbs = 0;
7160	dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS;
7161	dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7162
7163	out:
7164	return err;
7165	}
7166
7167	/**
7168	* i40e_init_pf_dcb - Initialize DCB configuration
7169	* @pf: PF being configured
7170	*
7171	* Query the current DCB configuration and cache it
7172	* in the hardware structure
7173	**/
7174	static int i40e_init_pf_dcb(struct i40e_pf *pf)
7175	{
7176	struct i40e_hw *hw = &pf->hw;
7177	int err;
7178
7179	/* Do not enable DCB for SW1 and SW2 images even if the FW is capable
7180	* Also do not enable DCBx if FW LLDP agent is disabled
7181	*/
7182	if (test_bit(I40E_HW_CAP_NO_DCB_SUPPORT, pf->hw.caps)) {
7183	dev_info(&pf->pdev->dev, "DCB is not supported.\n");
7184	err = -EOPNOTSUPP;
7185	goto out;
7186	}
7187	if (test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) {
7188	dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");
7189	err = i40e_dcb_sw_default_config(pf);
7190	if (err) {
7191	dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n");
7192	goto out;
7193	}
7194	dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n");
7195	pf->dcbx_cap = DCB_CAP_DCBX_HOST |
7196	DCB_CAP_DCBX_VER_IEEE;
7197	/* at init capable but disabled */
7198	set_bit(nr: I40E_FLAG_DCB_CAPABLE, addr: pf->flags);
7199	clear_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
7200	goto out;
7201	}
7202	err = i40e_init_dcb(hw, enable_mib_change: true);
7203	if (!err) {
7204	/* Device/Function is not DCBX capable */
7205	if ((!hw->func_caps.dcb) ||
7206	(hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
7207	dev_info(&pf->pdev->dev,
7208	"DCBX offload is not supported or is disabled for this PF.\n");
7209	} else {
7210	/* When status is not DISABLED then DCBX in FW */
7211	pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
7212	DCB_CAP_DCBX_VER_IEEE;
7213
7214	set_bit(nr: I40E_FLAG_DCB_CAPABLE, addr: pf->flags);
7215	/* Enable DCB tagging only when more than one TC
7216	* or explicitly disable if only one TC
7217	*/
7218	if (i40e_dcb_get_num_tc(dcbcfg: &hw->local_dcbx_config) > 1)
7219	set_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
7220	else
7221	clear_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
7222	dev_dbg(&pf->pdev->dev,
7223	"DCBX offload is supported for this PF.\n");
7224	}
7225	} else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
7226	dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
7227	set_bit(nr: I40E_FLAG_FW_LLDP_DIS, addr: pf->flags);
7228	} else {
7229	dev_info(&pf->pdev->dev,
7230	"Query for DCB configuration failed, err %pe aq_err %s\n",
7231	ERR_PTR(err),
7232	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7233	}
7234
7235	out:
7236	return err;
7237	}
7238	#endif /* CONFIG_I40E_DCB */
7239
7240	/**
7241	* i40e_print_link_message - print link up or down
7242	* @vsi: the VSI for which link needs a message
7243	* @isup: true of link is up, false otherwise
7244	*/
7245	void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
7246	{
7247	enum i40e_aq_link_speed new_speed;
7248	struct i40e_pf *pf = vsi->back;
7249	char *speed = "Unknown";
7250	char *fc = "Unknown";
7251	char *fec = "";
7252	char *req_fec = "";
7253	char *an = "";
7254
7255	if (isup)
7256	new_speed = pf->hw.phy.link_info.link_speed;
7257	else
7258	new_speed = I40E_LINK_SPEED_UNKNOWN;
7259
7260	if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
7261	return;
7262	vsi->current_isup = isup;
7263	vsi->current_speed = new_speed;
7264	if (!isup) {
7265	netdev_info(dev: vsi->netdev, format: "NIC Link is Down\n");
7266	return;
7267	}
7268
7269	/* Warn user if link speed on NPAR enabled partition is not at
7270	* least 10GB
7271	*/
7272	if (pf->hw.func_caps.npar_enable &&
7273	(pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
7274	pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
7275	netdev_warn(dev: vsi->netdev,
7276	format: "The partition detected link speed that is less than 10Gbps\n");
7277
7278	switch (pf->hw.phy.link_info.link_speed) {
7279	case I40E_LINK_SPEED_40GB:
7280	speed = "40 G";
7281	break;
7282	case I40E_LINK_SPEED_20GB:
7283	speed = "20 G";
7284	break;
7285	case I40E_LINK_SPEED_25GB:
7286	speed = "25 G";
7287	break;
7288	case I40E_LINK_SPEED_10GB:
7289	speed = "10 G";
7290	break;
7291	case I40E_LINK_SPEED_5GB:
7292	speed = "5 G";
7293	break;
7294	case I40E_LINK_SPEED_2_5GB:
7295	speed = "2.5 G";
7296	break;
7297	case I40E_LINK_SPEED_1GB:
7298	speed = "1000 M";
7299	break;
7300	case I40E_LINK_SPEED_100MB:
7301	speed = "100 M";
7302	break;
7303	default:
7304	break;
7305	}
7306
7307	switch (pf->hw.fc.current_mode) {
7308	case I40E_FC_FULL:
7309	fc = "RX/TX";
7310	break;
7311	case I40E_FC_TX_PAUSE:
7312	fc = "TX";
7313	break;
7314	case I40E_FC_RX_PAUSE:
7315	fc = "RX";
7316	break;
7317	default:
7318	fc = "None";
7319	break;
7320	}
7321
7322	if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
7323	req_fec = "None";
7324	fec = "None";
7325	an = "False";
7326
7327	if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7328	an = "True";
7329
7330	if (pf->hw.phy.link_info.fec_info &
7331	I40E_AQ_CONFIG_FEC_KR_ENA)
7332	fec = "CL74 FC-FEC/BASE-R";
7333	else if (pf->hw.phy.link_info.fec_info &
7334	I40E_AQ_CONFIG_FEC_RS_ENA)
7335	fec = "CL108 RS-FEC";
7336
7337	/* 'CL108 RS-FEC' should be displayed when RS is requested, or
7338	* both RS and FC are requested
7339	*/
7340	if (vsi->back->hw.phy.link_info.req_fec_info &
7341	(I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
7342	if (vsi->back->hw.phy.link_info.req_fec_info &
7343	I40E_AQ_REQUEST_FEC_RS)
7344	req_fec = "CL108 RS-FEC";
7345	else
7346	req_fec = "CL74 FC-FEC/BASE-R";
7347	}
7348	netdev_info(dev: vsi->netdev,
7349	format: "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7350	speed, req_fec, fec, an, fc);
7351	} else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {
7352	req_fec = "None";
7353	fec = "None";
7354	an = "False";
7355
7356	if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7357	an = "True";
7358
7359	if (pf->hw.phy.link_info.fec_info &
7360	I40E_AQ_CONFIG_FEC_KR_ENA)
7361	fec = "CL74 FC-FEC/BASE-R";
7362
7363	if (pf->hw.phy.link_info.req_fec_info &
7364	I40E_AQ_REQUEST_FEC_KR)
7365	req_fec = "CL74 FC-FEC/BASE-R";
7366
7367	netdev_info(dev: vsi->netdev,
7368	format: "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7369	speed, req_fec, fec, an, fc);
7370	} else {
7371	netdev_info(dev: vsi->netdev,
7372	format: "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",
7373	speed, fc);
7374	}
7375
7376	}
7377
7378	/**
7379	* i40e_up_complete - Finish the last steps of bringing up a connection
7380	* @vsi: the VSI being configured
7381	**/
7382	static int i40e_up_complete(struct i40e_vsi *vsi)
7383	{
7384	struct i40e_pf *pf = vsi->back;
7385	int err;
7386
7387	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
7388	i40e_vsi_configure_msix(vsi);
7389	else
7390	i40e_configure_msi_and_legacy(vsi);
7391
7392	/* start rings */
7393	err = i40e_vsi_start_rings(vsi);
7394	if (err)
7395	return err;
7396
7397	clear_bit(nr: __I40E_VSI_DOWN, addr: vsi->state);
7398	i40e_napi_enable_all(vsi);
7399	i40e_vsi_enable_irq(vsi);
7400
7401	if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
7402	(vsi->netdev)) {
7403	i40e_print_link_message(vsi, isup: true);
7404	netif_tx_start_all_queues(dev: vsi->netdev);
7405	netif_carrier_on(dev: vsi->netdev);
7406	}
7407
7408	/* replay FDIR SB filters */
7409	if (vsi->type == I40E_VSI_FDIR) {
7410	/* reset fd counters */
7411	pf->fd_add_err = 0;
7412	pf->fd_atr_cnt = 0;
7413	i40e_fdir_filter_restore(vsi);
7414	}
7415
7416	/* On the next run of the service_task, notify any clients of the new
7417	* opened netdev
7418	*/
7419	set_bit(nr: __I40E_CLIENT_SERVICE_REQUESTED, addr: pf->state);
7420	i40e_service_event_schedule(pf);
7421
7422	return 0;
7423	}
7424
7425	/**
7426	* i40e_vsi_reinit_locked - Reset the VSI
7427	* @vsi: the VSI being configured
7428	*
7429	* Rebuild the ring structs after some configuration
7430	* has changed, e.g. MTU size.
7431	**/
7432	static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
7433	{
7434	struct i40e_pf *pf = vsi->back;
7435
7436	while (test_and_set_bit(nr: __I40E_CONFIG_BUSY, addr: pf->state))
7437	usleep_range(min: 1000, max: 2000);
7438	i40e_down(vsi);
7439
7440	i40e_up(vsi);
7441	clear_bit(nr: __I40E_CONFIG_BUSY, addr: pf->state);
7442	}
7443
7444	/**
7445	* i40e_force_link_state - Force the link status
7446	* @pf: board private structure
7447	* @is_up: whether the link state should be forced up or down
7448	**/
7449	static int i40e_force_link_state(struct i40e_pf *pf, bool is_up)
7450	{
7451	struct i40e_aq_get_phy_abilities_resp abilities;
7452	struct i40e_aq_set_phy_config config = {0};
7453	bool non_zero_phy_type = is_up;
7454	struct i40e_hw *hw = &pf->hw;
7455	u64 mask;
7456	u8 speed;
7457	int err;
7458
7459	/* Card might've been put in an unstable state by other drivers
7460	* and applications, which causes incorrect speed values being
7461	* set on startup. In order to clear speed registers, we call
7462	* get_phy_capabilities twice, once to get initial state of
7463	* available speeds, and once to get current PHY config.
7464	*/
7465	err = i40e_aq_get_phy_capabilities(hw, qualified_modules: false, report_init: true, abilities: &abilities,
7466	NULL);
7467	if (err) {
7468	dev_err(&pf->pdev->dev,
7469	"failed to get phy cap., ret = %pe last_status = %s\n",
7470	ERR_PTR(err),
7471	i40e_aq_str(hw, hw->aq.asq_last_status));
7472	return err;
7473	}
7474	speed = abilities.link_speed;
7475
7476	/* Get the current phy config */
7477	err = i40e_aq_get_phy_capabilities(hw, qualified_modules: false, report_init: false, abilities: &abilities,
7478	NULL);
7479	if (err) {
7480	dev_err(&pf->pdev->dev,
7481	"failed to get phy cap., ret = %pe last_status = %s\n",
7482	ERR_PTR(err),
7483	i40e_aq_str(hw, hw->aq.asq_last_status));
7484	return err;
7485	}
7486
7487	/* If link needs to go up, but was not forced to go down,
7488	* and its speed values are OK, no need for a flap
7489	* if non_zero_phy_type was set, still need to force up
7490	*/
7491	if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags))
7492	non_zero_phy_type = true;
7493	else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
7494	return 0;
7495
7496	/* To force link we need to set bits for all supported PHY types,
7497	* but there are now more than 32, so we need to split the bitmap
7498	* across two fields.
7499	*/
7500	mask = I40E_PHY_TYPES_BITMASK;
7501	config.phy_type =
7502	non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
7503	config.phy_type_ext =
7504	non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;
7505	/* Copy the old settings, except of phy_type */
7506	config.abilities = abilities.abilities;
7507	if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) {
7508	if (is_up)
7509	config.abilities |= I40E_AQ_PHY_ENABLE_LINK;
7510	else
7511	config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK);
7512	}
7513	if (abilities.link_speed != 0)
7514	config.link_speed = abilities.link_speed;
7515	else
7516	config.link_speed = speed;
7517	config.eee_capability = abilities.eee_capability;
7518	config.eeer = abilities.eeer_val;
7519	config.low_power_ctrl = abilities.d3_lpan;
7520	config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
7521	I40E_AQ_PHY_FEC_CONFIG_MASK;
7522	err = i40e_aq_set_phy_config(hw, config: &config, NULL);
7523
7524	if (err) {
7525	dev_err(&pf->pdev->dev,
7526	"set phy config ret = %pe last_status = %s\n",
7527	ERR_PTR(err),
7528	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7529	return err;
7530	}
7531
7532	/* Update the link info */
7533	err = i40e_update_link_info(hw);
7534	if (err) {
7535	/* Wait a little bit (on 40G cards it sometimes takes a really
7536	* long time for link to come back from the atomic reset)
7537	* and try once more
7538	*/
7539	msleep(msecs: 1000);
7540	i40e_update_link_info(hw);
7541	}
7542
7543	i40e_aq_set_link_restart_an(hw, enable_link: is_up, NULL);
7544
7545	return 0;
7546	}
7547
7548	/**
7549	* i40e_up - Bring the connection back up after being down
7550	* @vsi: the VSI being configured
7551	**/
7552	int i40e_up(struct i40e_vsi *vsi)
7553	{
7554	int err;
7555
7556	if (vsi->type == I40E_VSI_MAIN &&
7557	(test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||
7558	test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))
7559	i40e_force_link_state(pf: vsi->back, is_up: true);
7560
7561	err = i40e_vsi_configure(vsi);
7562	if (!err)
7563	err = i40e_up_complete(vsi);
7564
7565	return err;
7566	}
7567
7568	/**
7569	* i40e_down - Shutdown the connection processing
7570	* @vsi: the VSI being stopped
7571	**/
7572	void i40e_down(struct i40e_vsi *vsi)
7573	{
7574	int i;
7575
7576	/* It is assumed that the caller of this function
7577	* sets the vsi->state __I40E_VSI_DOWN bit.
7578	*/
7579	if (vsi->netdev) {
7580	netif_carrier_off(dev: vsi->netdev);
7581	netif_tx_disable(dev: vsi->netdev);
7582	}
7583	i40e_vsi_disable_irq(vsi);
7584	i40e_vsi_stop_rings(vsi);
7585	if (vsi->type == I40E_VSI_MAIN &&
7586	(test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||
7587	test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))
7588	i40e_force_link_state(pf: vsi->back, is_up: false);
7589	i40e_napi_disable_all(vsi);
7590
7591	for (i = 0; i < vsi->num_queue_pairs; i++) {
7592	i40e_clean_tx_ring(tx_ring: vsi->tx_rings[i]);
7593	if (i40e_enabled_xdp_vsi(vsi)) {
7594	/* Make sure that in-progress ndo_xdp_xmit and
7595	* ndo_xsk_wakeup calls are completed.
7596	*/
7597	synchronize_rcu();
7598	i40e_clean_tx_ring(tx_ring: vsi->xdp_rings[i]);
7599	}
7600	i40e_clean_rx_ring(rx_ring: vsi->rx_rings[i]);
7601	}
7602
7603	}
7604
7605	/**
7606	* i40e_validate_mqprio_qopt- validate queue mapping info
7607	* @vsi: the VSI being configured
7608	* @mqprio_qopt: queue parametrs
7609	**/
7610	static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
7611	struct tc_mqprio_qopt_offload *mqprio_qopt)
7612	{
7613	u64 sum_max_rate = 0;
7614	u64 max_rate = 0;
7615	int i;
7616
7617	if (mqprio_qopt->qopt.offset[0] != 0 ||
7618	mqprio_qopt->qopt.num_tc < 1 ||
7619	mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
7620	return -EINVAL;
7621	for (i = 0; ; i++) {
7622	if (!mqprio_qopt->qopt.count[i])
7623	return -EINVAL;
7624	if (mqprio_qopt->min_rate[i]) {
7625	dev_err(&vsi->back->pdev->dev,
7626	"Invalid min tx rate (greater than 0) specified\n");
7627	return -EINVAL;
7628	}
7629	max_rate = mqprio_qopt->max_rate[i];
7630	do_div(max_rate, I40E_BW_MBPS_DIVISOR);
7631	sum_max_rate += max_rate;
7632
7633	if (i >= mqprio_qopt->qopt.num_tc - 1)
7634	break;
7635	if (mqprio_qopt->qopt.offset[i + 1] !=
7636	(mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7637	return -EINVAL;
7638	}
7639	if (vsi->num_queue_pairs <
7640	(mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
7641	dev_err(&vsi->back->pdev->dev,
7642	"Failed to create traffic channel, insufficient number of queues.\n");
7643	return -EINVAL;
7644	}
7645	if (sum_max_rate > i40e_get_link_speed(vsi)) {
7646	dev_err(&vsi->back->pdev->dev,
7647	"Invalid max tx rate specified\n");
7648	return -EINVAL;
7649	}
7650	return 0;
7651	}
7652
7653	/**
7654	* i40e_vsi_set_default_tc_config - set default values for tc configuration
7655	* @vsi: the VSI being configured
7656	**/
7657	static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
7658	{
7659	u16 qcount;
7660	int i;
7661
7662	/* Only TC0 is enabled */
7663	vsi->tc_config.numtc = 1;
7664	vsi->tc_config.enabled_tc = 1;
7665	qcount = min_t(int, vsi->alloc_queue_pairs,
7666	i40e_pf_get_max_q_per_tc(vsi->back));
7667	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7668	/* For the TC that is not enabled set the offset to default
7669	* queue and allocate one queue for the given TC.
7670	*/
7671	vsi->tc_config.tc_info[i].qoffset = 0;
7672	if (i == 0)
7673	vsi->tc_config.tc_info[i].qcount = qcount;
7674	else
7675	vsi->tc_config.tc_info[i].qcount = 1;
7676	vsi->tc_config.tc_info[i].netdev_tc = 0;
7677	}
7678	}
7679
7680	/**
7681	* i40e_del_macvlan_filter
7682	* @hw: pointer to the HW structure
7683	* @seid: seid of the channel VSI
7684	* @macaddr: the mac address to apply as a filter
7685	* @aq_err: store the admin Q error
7686	*
7687	* This function deletes a mac filter on the channel VSI which serves as the
7688	* macvlan. Returns 0 on success.
7689	**/
7690	static int i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
7691	const u8 *macaddr, int *aq_err)
7692	{
7693	struct i40e_aqc_remove_macvlan_element_data element;
7694	int status;
7695
7696	memset(&element, 0, sizeof(element));
7697	ether_addr_copy(dst: element.mac_addr, src: macaddr);
7698	element.vlan_tag = 0;
7699	element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
7700	status = i40e_aq_remove_macvlan(hw, vsi_id: seid, mv_list: &element, count: 1, NULL);
7701	*aq_err = hw->aq.asq_last_status;
7702
7703	return status;
7704	}
7705
7706	/**
7707	* i40e_add_macvlan_filter
7708	* @hw: pointer to the HW structure
7709	* @seid: seid of the channel VSI
7710	* @macaddr: the mac address to apply as a filter
7711	* @aq_err: store the admin Q error
7712	*
7713	* This function adds a mac filter on the channel VSI which serves as the
7714	* macvlan. Returns 0 on success.
7715	**/
7716	static int i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
7717	const u8 *macaddr, int *aq_err)
7718	{
7719	struct i40e_aqc_add_macvlan_element_data element;
7720	u16 cmd_flags = 0;
7721	int status;
7722
7723	ether_addr_copy(dst: element.mac_addr, src: macaddr);
7724	element.vlan_tag = 0;
7725	element.queue_number = 0;
7726	element.match_method = I40E_AQC_MM_ERR_NO_RES;
7727	cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
7728	element.flags = cpu_to_le16(cmd_flags);
7729	status = i40e_aq_add_macvlan(hw, vsi_id: seid, mv_list: &element, count: 1, NULL);
7730	*aq_err = hw->aq.asq_last_status;
7731
7732	return status;
7733	}
7734
7735	/**
7736	* i40e_reset_ch_rings - Reset the queue contexts in a channel
7737	* @vsi: the VSI we want to access
7738	* @ch: the channel we want to access
7739	*/
7740	static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
7741	{
7742	struct i40e_ring *tx_ring, *rx_ring;
7743	u16 pf_q;
7744	int i;
7745
7746	for (i = 0; i < ch->num_queue_pairs; i++) {
7747	pf_q = ch->base_queue + i;
7748	tx_ring = vsi->tx_rings[pf_q];
7749	tx_ring->ch = NULL;
7750	rx_ring = vsi->rx_rings[pf_q];
7751	rx_ring->ch = NULL;
7752	}
7753	}
7754
7755	/**
7756	* i40e_free_macvlan_channels
7757	* @vsi: the VSI we want to access
7758	*
7759	* This function frees the Qs of the channel VSI from
7760	* the stack and also deletes the channel VSIs which
7761	* serve as macvlans.
7762	*/
7763	static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
7764	{
7765	struct i40e_channel *ch, *ch_tmp;
7766	int ret;
7767
7768	if (list_empty(head: &vsi->macvlan_list))
7769	return;
7770
7771	list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7772	struct i40e_vsi *parent_vsi;
7773
7774	if (i40e_is_channel_macvlan(ch)) {
7775	i40e_reset_ch_rings(vsi, ch);
7776	clear_bit(nr: ch->fwd->bit_no, addr: vsi->fwd_bitmask);
7777	netdev_unbind_sb_channel(dev: vsi->netdev, sb_dev: ch->fwd->netdev);
7778	netdev_set_sb_channel(dev: ch->fwd->netdev, channel: 0);
7779	kfree(objp: ch->fwd);
7780	ch->fwd = NULL;
7781	}
7782
7783	list_del(entry: &ch->list);
7784	parent_vsi = ch->parent_vsi;
7785	if (!parent_vsi || !ch->initialized) {
7786	kfree(objp: ch);
7787	continue;
7788	}
7789
7790	/* remove the VSI */
7791	ret = i40e_aq_delete_element(hw: &vsi->back->hw, seid: ch->seid,
7792	NULL);
7793	if (ret)
7794	dev_err(&vsi->back->pdev->dev,
7795	"unable to remove channel (%d) for parent VSI(%d)\n",
7796	ch->seid, parent_vsi->seid);
7797	kfree(objp: ch);
7798	}
7799	vsi->macvlan_cnt = 0;
7800	}
7801
7802	/**
7803	* i40e_fwd_ring_up - bring the macvlan device up
7804	* @vsi: the VSI we want to access
7805	* @vdev: macvlan netdevice
7806	* @fwd: the private fwd structure
7807	*/
7808	static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7809	struct i40e_fwd_adapter *fwd)
7810	{
7811	struct i40e_channel *ch = NULL, *ch_tmp, *iter;
7812	int ret = 0, num_tc = 1, i, aq_err;
7813	struct i40e_pf *pf = vsi->back;
7814	struct i40e_hw *hw = &pf->hw;
7815
7816	/* Go through the list and find an available channel */
7817	list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) {
7818	if (!i40e_is_channel_macvlan(ch: iter)) {
7819	iter->fwd = fwd;
7820	/* record configuration for macvlan interface in vdev */
7821	for (i = 0; i < num_tc; i++)
7822	netdev_bind_sb_channel_queue(dev: vsi->netdev, sb_dev: vdev,
7823	tc: i,
7824	count: iter->num_queue_pairs,
7825	offset: iter->base_queue);
7826	for (i = 0; i < iter->num_queue_pairs; i++) {
7827	struct i40e_ring *tx_ring, *rx_ring;
7828	u16 pf_q;
7829
7830	pf_q = iter->base_queue + i;
7831
7832	/* Get to TX ring ptr */
7833	tx_ring = vsi->tx_rings[pf_q];
7834	tx_ring->ch = iter;
7835
7836	/* Get the RX ring ptr */
7837	rx_ring = vsi->rx_rings[pf_q];
7838	rx_ring->ch = iter;
7839	}
7840	ch = iter;
7841	break;
7842	}
7843	}
7844
7845	if (!ch)
7846	return -EINVAL;
7847
7848	/* Guarantee all rings are updated before we update the
7849	* MAC address filter.
7850	*/
7851	wmb();
7852
7853	/* Add a mac filter */
7854	ret = i40e_add_macvlan_filter(hw, seid: ch->seid, macaddr: vdev->dev_addr, aq_err: &aq_err);
7855	if (ret) {
7856	/* if we cannot add the MAC rule then disable the offload */
7857	macvlan_release_l2fw_offload(dev: vdev);
7858	for (i = 0; i < ch->num_queue_pairs; i++) {
7859	struct i40e_ring *rx_ring;
7860	u16 pf_q;
7861
7862	pf_q = ch->base_queue + i;
7863	rx_ring = vsi->rx_rings[pf_q];
7864	rx_ring->netdev = NULL;
7865	}
7866	dev_info(&pf->pdev->dev,
7867	"Error adding mac filter on macvlan err %pe, aq_err %s\n",
7868	ERR_PTR(ret),
7869	i40e_aq_str(hw, aq_err));
7870	netdev_err(dev: vdev, format: "L2fwd offload disabled to L2 filter error\n");
7871	}
7872
7873	return ret;
7874	}
7875
7876	/**
7877	* i40e_setup_macvlans - create the channels which will be macvlans
7878	* @vsi: the VSI we want to access
7879	* @macvlan_cnt: no. of macvlans to be setup
7880	* @qcnt: no. of Qs per macvlan
7881	* @vdev: macvlan netdevice
7882	*/
7883	static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7884	struct net_device *vdev)
7885	{
7886	struct i40e_pf *pf = vsi->back;
7887	struct i40e_hw *hw = &pf->hw;
7888	struct i40e_vsi_context ctxt;
7889	u16 sections, qmap, num_qps;
7890	struct i40e_channel *ch;
7891	int i, pow, ret = 0;
7892	u8 offset = 0;
7893
7894	if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7895	return -EINVAL;
7896
7897	num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7898
7899	/* find the next higher power-of-2 of num queue pairs */
7900	pow = fls(roundup_pow_of_two(num_qps) - 1);
7901
7902	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7903	(pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7904
7905	/* Setup context bits for the main VSI */
7906	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7907	sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7908	memset(&ctxt, 0, sizeof(ctxt));
7909	ctxt.seid = vsi->seid;
7910	ctxt.pf_num = vsi->back->hw.pf_id;
7911	ctxt.vf_num = 0;
7912	ctxt.uplink_seid = vsi->uplink_seid;
7913	ctxt.info = vsi->info;
7914	ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7915	ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7916	ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7917	ctxt.info.valid_sections |= cpu_to_le16(sections);
7918
7919	/* Reconfigure RSS for main VSI with new max queue count */
7920	vsi->rss_size = max_t(u16, num_qps, qcnt);
7921	ret = i40e_vsi_config_rss(vsi);
7922	if (ret) {
7923	dev_info(&pf->pdev->dev,
7924	"Failed to reconfig RSS for num_queues (%u)\n",
7925	vsi->rss_size);
7926	return ret;
7927	}
7928	vsi->reconfig_rss = true;
7929	dev_dbg(&vsi->back->pdev->dev,
7930	"Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7931	vsi->next_base_queue = num_qps;
7932	vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7933
7934	/* Update the VSI after updating the VSI queue-mapping
7935	* information
7936	*/
7937	ret = i40e_aq_update_vsi_params(hw, vsi_ctx: &ctxt, NULL);
7938	if (ret) {
7939	dev_info(&pf->pdev->dev,
7940	"Update vsi tc config failed, err %pe aq_err %s\n",
7941	ERR_PTR(ret),
7942	i40e_aq_str(hw, hw->aq.asq_last_status));
7943	return ret;
7944	}
7945	/* update the local VSI info with updated queue map */
7946	i40e_vsi_update_queue_map(vsi, ctxt: &ctxt);
7947	vsi->info.valid_sections = 0;
7948
7949	/* Create channels for macvlans */
7950	INIT_LIST_HEAD(list: &vsi->macvlan_list);
7951	for (i = 0; i < macvlan_cnt; i++) {
7952	ch = kzalloc(size: sizeof(*ch), GFP_KERNEL);
7953	if (!ch) {
7954	ret = -ENOMEM;
7955	goto err_free;
7956	}
7957	INIT_LIST_HEAD(list: &ch->list);
7958	ch->num_queue_pairs = qcnt;
7959	if (!i40e_setup_channel(pf, vsi, ch)) {
7960	ret = -EINVAL;
7961	kfree(objp: ch);
7962	goto err_free;
7963	}
7964	ch->parent_vsi = vsi;
7965	vsi->cnt_q_avail -= ch->num_queue_pairs;
7966	vsi->macvlan_cnt++;
7967	list_add_tail(new: &ch->list, head: &vsi->macvlan_list);
7968	}
7969
7970	return ret;
7971
7972	err_free:
7973	dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
7974	i40e_free_macvlan_channels(vsi);
7975
7976	return ret;
7977	}
7978
7979	/**
7980	* i40e_fwd_add - configure macvlans
7981	* @netdev: net device to configure
7982	* @vdev: macvlan netdevice
7983	**/
7984	static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
7985	{
7986	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
7987	u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
7988	struct i40e_vsi *vsi = np->vsi;
7989	struct i40e_pf *pf = vsi->back;
7990	struct i40e_fwd_adapter *fwd;
7991	int avail_macvlan, ret;
7992
7993	if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
7994	netdev_info(dev: netdev, format: "Macvlans are not supported when DCB is enabled\n");
7995	return ERR_PTR(error: -EINVAL);
7996	}
7997	if (i40e_is_tc_mqprio_enabled(pf)) {
7998	netdev_info(dev: netdev, format: "Macvlans are not supported when HW TC offload is on\n");
7999	return ERR_PTR(error: -EINVAL);
8000	}
8001	if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
8002	netdev_info(dev: netdev, format: "Not enough vectors available to support macvlans\n");
8003	return ERR_PTR(error: -EINVAL);
8004	}
8005
8006	/* The macvlan device has to be a single Q device so that the
8007	* tc_to_txq field can be reused to pick the tx queue.
8008	*/
8009	if (netif_is_multiqueue(dev: vdev))
8010	return ERR_PTR(error: -ERANGE);
8011
8012	if (!vsi->macvlan_cnt) {
8013	/* reserve bit 0 for the pf device */
8014	set_bit(nr: 0, addr: vsi->fwd_bitmask);
8015
8016	/* Try to reserve as many queues as possible for macvlans. First
8017	* reserve 3/4th of max vectors, then half, then quarter and
8018	* calculate Qs per macvlan as you go
8019	*/
8020	vectors = pf->num_lan_msix;
8021	if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
8022	/* allocate 4 Qs per macvlan and 32 Qs to the PF*/
8023	q_per_macvlan = 4;
8024	macvlan_cnt = (vectors - 32) / 4;
8025	} else if (vectors <= 64 && vectors > 32) {
8026	/* allocate 2 Qs per macvlan and 16 Qs to the PF*/
8027	q_per_macvlan = 2;
8028	macvlan_cnt = (vectors - 16) / 2;
8029	} else if (vectors <= 32 && vectors > 16) {
8030	/* allocate 1 Q per macvlan and 16 Qs to the PF*/
8031	q_per_macvlan = 1;
8032	macvlan_cnt = vectors - 16;
8033	} else if (vectors <= 16 && vectors > 8) {
8034	/* allocate 1 Q per macvlan and 8 Qs to the PF */
8035	q_per_macvlan = 1;
8036	macvlan_cnt = vectors - 8;
8037	} else {
8038	/* allocate 1 Q per macvlan and 1 Q to the PF */
8039	q_per_macvlan = 1;
8040	macvlan_cnt = vectors - 1;
8041	}
8042
8043	if (macvlan_cnt == 0)
8044	return ERR_PTR(error: -EBUSY);
8045
8046	/* Quiesce VSI queues */
8047	i40e_quiesce_vsi(vsi);
8048
8049	/* sets up the macvlans but does not "enable" them */
8050	ret = i40e_setup_macvlans(vsi, macvlan_cnt, qcnt: q_per_macvlan,
8051	vdev);
8052	if (ret)
8053	return ERR_PTR(error: ret);
8054
8055	/* Unquiesce VSI */
8056	i40e_unquiesce_vsi(vsi);
8057	}
8058	avail_macvlan = find_first_zero_bit(addr: vsi->fwd_bitmask,
8059	size: vsi->macvlan_cnt);
8060	if (avail_macvlan >= I40E_MAX_MACVLANS)
8061	return ERR_PTR(error: -EBUSY);
8062
8063	/* create the fwd struct */
8064	fwd = kzalloc(size: sizeof(*fwd), GFP_KERNEL);
8065	if (!fwd)
8066	return ERR_PTR(error: -ENOMEM);
8067
8068	set_bit(nr: avail_macvlan, addr: vsi->fwd_bitmask);
8069	fwd->bit_no = avail_macvlan;
8070	netdev_set_sb_channel(dev: vdev, channel: avail_macvlan);
8071	fwd->netdev = vdev;
8072
8073	if (!netif_running(dev: netdev))
8074	return fwd;
8075
8076	/* Set fwd ring up */
8077	ret = i40e_fwd_ring_up(vsi, vdev, fwd);
8078	if (ret) {
8079	/* unbind the queues and drop the subordinate channel config */
8080	netdev_unbind_sb_channel(dev: netdev, sb_dev: vdev);
8081	netdev_set_sb_channel(dev: vdev, channel: 0);
8082
8083	kfree(objp: fwd);
8084	return ERR_PTR(error: -EINVAL);
8085	}
8086
8087	return fwd;
8088	}
8089
8090	/**
8091	* i40e_del_all_macvlans - Delete all the mac filters on the channels
8092	* @vsi: the VSI we want to access
8093	*/
8094	static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
8095	{
8096	struct i40e_channel *ch, *ch_tmp;
8097	struct i40e_pf *pf = vsi->back;
8098	struct i40e_hw *hw = &pf->hw;
8099	int aq_err, ret = 0;
8100
8101	if (list_empty(head: &vsi->macvlan_list))
8102	return;
8103
8104	list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8105	if (i40e_is_channel_macvlan(ch)) {
8106	ret = i40e_del_macvlan_filter(hw, seid: ch->seid,
8107	macaddr: i40e_channel_mac(ch),
8108	aq_err: &aq_err);
8109	if (!ret) {
8110	/* Reset queue contexts */
8111	i40e_reset_ch_rings(vsi, ch);
8112	clear_bit(nr: ch->fwd->bit_no, addr: vsi->fwd_bitmask);
8113	netdev_unbind_sb_channel(dev: vsi->netdev,
8114	sb_dev: ch->fwd->netdev);
8115	netdev_set_sb_channel(dev: ch->fwd->netdev, channel: 0);
8116	kfree(objp: ch->fwd);
8117	ch->fwd = NULL;
8118	}
8119	}
8120	}
8121	}
8122
8123	/**
8124	* i40e_fwd_del - delete macvlan interfaces
8125	* @netdev: net device to configure
8126	* @vdev: macvlan netdevice
8127	*/
8128	static void i40e_fwd_del(struct net_device *netdev, void *vdev)
8129	{
8130	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
8131	struct i40e_fwd_adapter *fwd = vdev;
8132	struct i40e_channel *ch, *ch_tmp;
8133	struct i40e_vsi *vsi = np->vsi;
8134	struct i40e_pf *pf = vsi->back;
8135	struct i40e_hw *hw = &pf->hw;
8136	int aq_err, ret = 0;
8137
8138	/* Find the channel associated with the macvlan and del mac filter */
8139	list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8140	if (i40e_is_channel_macvlan(ch) &&
8141	ether_addr_equal(addr1: i40e_channel_mac(ch),
8142	addr2: fwd->netdev->dev_addr)) {
8143	ret = i40e_del_macvlan_filter(hw, seid: ch->seid,
8144	macaddr: i40e_channel_mac(ch),
8145	aq_err: &aq_err);
8146	if (!ret) {
8147	/* Reset queue contexts */
8148	i40e_reset_ch_rings(vsi, ch);
8149	clear_bit(nr: ch->fwd->bit_no, addr: vsi->fwd_bitmask);
8150	netdev_unbind_sb_channel(dev: netdev, sb_dev: fwd->netdev);
8151	netdev_set_sb_channel(dev: fwd->netdev, channel: 0);
8152	kfree(objp: ch->fwd);
8153	ch->fwd = NULL;
8154	} else {
8155	dev_info(&pf->pdev->dev,
8156	"Error deleting mac filter on macvlan err %pe, aq_err %s\n",
8157	ERR_PTR(ret),
8158	i40e_aq_str(hw, aq_err));
8159	}
8160	break;
8161	}
8162	}
8163	}
8164
8165	/**
8166	* i40e_setup_tc - configure multiple traffic classes
8167	* @netdev: net device to configure
8168	* @type_data: tc offload data
8169	**/
8170	static int i40e_setup_tc(struct net_device *netdev, void *type_data)
8171	{
8172	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
8173	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
8174	struct i40e_vsi *vsi = np->vsi;
8175	struct i40e_pf *pf = vsi->back;
8176	u8 enabled_tc = 0, num_tc, hw;
8177	bool need_reset = false;
8178	int old_queue_pairs;
8179	int ret = -EINVAL;
8180	u16 mode;
8181	int i;
8182
8183	old_queue_pairs = vsi->num_queue_pairs;
8184	num_tc = mqprio_qopt->qopt.num_tc;
8185	hw = mqprio_qopt->qopt.hw;
8186	mode = mqprio_qopt->mode;
8187	if (!hw) {
8188	clear_bit(nr: I40E_FLAG_TC_MQPRIO_ENA, addr: pf->flags);
8189	memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8190	goto config_tc;
8191	}
8192
8193	/* Check if MFP enabled */
8194	if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
8195	netdev_info(dev: netdev,
8196	format: "Configuring TC not supported in MFP mode\n");
8197	return ret;
8198	}
8199	switch (mode) {
8200	case TC_MQPRIO_MODE_DCB:
8201	clear_bit(nr: I40E_FLAG_TC_MQPRIO_ENA, addr: pf->flags);
8202
8203	/* Check if DCB enabled to continue */
8204	if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8205	netdev_info(dev: netdev,
8206	format: "DCB is not enabled for adapter\n");
8207	return ret;
8208	}
8209
8210	/* Check whether tc count is within enabled limit */
8211	if (num_tc > i40e_pf_get_num_tc(pf)) {
8212	netdev_info(dev: netdev,
8213	format: "TC count greater than enabled on link for adapter\n");
8214	return ret;
8215	}
8216	break;
8217	case TC_MQPRIO_MODE_CHANNEL:
8218	if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8219	netdev_info(dev: netdev,
8220	format: "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
8221	return ret;
8222	}
8223	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
8224	return ret;
8225	ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
8226	if (ret)
8227	return ret;
8228	memcpy(&vsi->mqprio_qopt, mqprio_qopt,
8229	sizeof(*mqprio_qopt));
8230	set_bit(nr: I40E_FLAG_TC_MQPRIO_ENA, addr: pf->flags);
8231	clear_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
8232	break;
8233	default:
8234	return -EINVAL;
8235	}
8236
8237	config_tc:
8238	/* Generate TC map for number of tc requested */
8239	for (i = 0; i < num_tc; i++)
8240	enabled_tc |= BIT(i);
8241
8242	/* Requesting same TC configuration as already enabled */
8243	if (enabled_tc == vsi->tc_config.enabled_tc &&
8244	mode != TC_MQPRIO_MODE_CHANNEL)
8245	return 0;
8246
8247	/* Quiesce VSI queues */
8248	i40e_quiesce_vsi(vsi);
8249
8250	if (!hw && !i40e_is_tc_mqprio_enabled(pf))
8251	i40e_remove_queue_channels(vsi);
8252
8253	/* Configure VSI for enabled TCs */
8254	ret = i40e_vsi_config_tc(vsi, enabled_tc);
8255	if (ret) {
8256	netdev_info(dev: netdev, format: "Failed configuring TC for VSI seid=%d\n",
8257	vsi->seid);
8258	need_reset = true;
8259	goto exit;
8260	} else if (enabled_tc &&
8261	(!is_power_of_2(n: vsi->tc_config.tc_info[0].qcount))) {
8262	netdev_info(dev: netdev,
8263	format: "Failed to create channel. Override queues (%u) not power of 2\n",
8264	vsi->tc_config.tc_info[0].qcount);
8265	ret = -EINVAL;
8266	need_reset = true;
8267	goto exit;
8268	}
8269
8270	dev_info(&vsi->back->pdev->dev,
8271	"Setup channel (id:%u) utilizing num_queues %d\n",
8272	vsi->seid, vsi->tc_config.tc_info[0].qcount);
8273
8274	if (i40e_is_tc_mqprio_enabled(pf)) {
8275	if (vsi->mqprio_qopt.max_rate[0]) {
8276	u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
8277	max_tx_rate: vsi->mqprio_qopt.max_rate[0]);
8278
8279	ret = i40e_set_bw_limit(vsi, seid: vsi->seid, max_tx_rate);
8280	if (!ret) {
8281	u64 credits = max_tx_rate;
8282
8283	do_div(credits, I40E_BW_CREDIT_DIVISOR);
8284	dev_dbg(&vsi->back->pdev->dev,
8285	"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
8286	max_tx_rate,
8287	credits,
8288	vsi->seid);
8289	} else {
8290	need_reset = true;
8291	goto exit;
8292	}
8293	}
8294	ret = i40e_configure_queue_channels(vsi);
8295	if (ret) {
8296	vsi->num_queue_pairs = old_queue_pairs;
8297	netdev_info(dev: netdev,
8298	format: "Failed configuring queue channels\n");
8299	need_reset = true;
8300	goto exit;
8301	}
8302	}
8303
8304	exit:
8305	/* Reset the configuration data to defaults, only TC0 is enabled */
8306	if (need_reset) {
8307	i40e_vsi_set_default_tc_config(vsi);
8308	need_reset = false;
8309	}
8310
8311	/* Unquiesce VSI */
8312	i40e_unquiesce_vsi(vsi);
8313	return ret;
8314	}
8315
8316	/**
8317	* i40e_set_cld_element - sets cloud filter element data
8318	* @filter: cloud filter rule
8319	* @cld: ptr to cloud filter element data
8320	*
8321	* This is helper function to copy data into cloud filter element
8322	**/
8323	static inline void
8324	i40e_set_cld_element(struct i40e_cloud_filter *filter,
8325	struct i40e_aqc_cloud_filters_element_data *cld)
8326	{
8327	u32 ipa;
8328	int i;
8329
8330	memset(cld, 0, sizeof(*cld));
8331	ether_addr_copy(dst: cld->outer_mac, src: filter->dst_mac);
8332	ether_addr_copy(dst: cld->inner_mac, src: filter->src_mac);
8333
8334	if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
8335	return;
8336
8337	if (filter->n_proto == ETH_P_IPV6) {
8338	#define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1)
8339	for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {
8340	ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
8341
8342	*(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);
8343	}
8344	} else {
8345	ipa = be32_to_cpu(filter->dst_ipv4);
8346
8347	memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
8348	}
8349
8350	cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
8351
8352	/* tenant_id is not supported by FW now, once the support is enabled
8353	* fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
8354	*/
8355	if (filter->tenant_id)
8356	return;
8357	}
8358
8359	/**
8360	* i40e_add_del_cloud_filter - Add/del cloud filter
8361	* @vsi: pointer to VSI
8362	* @filter: cloud filter rule
8363	* @add: if true, add, if false, delete
8364	*
8365	* Add or delete a cloud filter for a specific flow spec.
8366	* Returns 0 if the filter were successfully added.
8367	**/
8368	int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
8369	struct i40e_cloud_filter *filter, bool add)
8370	{
8371	struct i40e_aqc_cloud_filters_element_data cld_filter;
8372	struct i40e_pf *pf = vsi->back;
8373	int ret;
8374	static const u16 flag_table[128] = {
8375	[I40E_CLOUD_FILTER_FLAGS_OMAC] =
8376	I40E_AQC_ADD_CLOUD_FILTER_OMAC,
8377	[I40E_CLOUD_FILTER_FLAGS_IMAC] =
8378	I40E_AQC_ADD_CLOUD_FILTER_IMAC,
8379	[I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] =
8380	I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
8381	[I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
8382	I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
8383	[I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
8384	I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
8385	[I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
8386	I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
8387	[I40E_CLOUD_FILTER_FLAGS_IIP] =
8388	I40E_AQC_ADD_CLOUD_FILTER_IIP,
8389	};
8390
8391	if (filter->flags >= ARRAY_SIZE(flag_table))
8392	return -EIO;
8393
8394	memset(&cld_filter, 0, sizeof(cld_filter));
8395
8396	/* copy element needed to add cloud filter from filter */
8397	i40e_set_cld_element(filter, cld: &cld_filter);
8398
8399	if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
8400	cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
8401	I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
8402
8403	if (filter->n_proto == ETH_P_IPV6)
8404	cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8405	I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8406	else
8407	cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8408	I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8409
8410	if (add)
8411	ret = i40e_aq_add_cloud_filters(hw: &pf->hw, vsi: filter->seid,
8412	filters: &cld_filter, filter_count: 1);
8413	else
8414	ret = i40e_aq_rem_cloud_filters(hw: &pf->hw, vsi: filter->seid,
8415	filters: &cld_filter, filter_count: 1);
8416	if (ret)
8417	dev_dbg(&pf->pdev->dev,
8418	"Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
8419	add ? "add" : "delete", filter->dst_port, ret,
8420	pf->hw.aq.asq_last_status);
8421	else
8422	dev_info(&pf->pdev->dev,
8423	"%s cloud filter for VSI: %d\n",
8424	add ? "Added" : "Deleted", filter->seid);
8425	return ret;
8426	}
8427
8428	/**
8429	* i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
8430	* @vsi: pointer to VSI
8431	* @filter: cloud filter rule
8432	* @add: if true, add, if false, delete
8433	*
8434	* Add or delete a cloud filter for a specific flow spec using big buffer.
8435	* Returns 0 if the filter were successfully added.
8436	**/
8437	int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
8438	struct i40e_cloud_filter *filter,
8439	bool add)
8440	{
8441	struct i40e_aqc_cloud_filters_element_bb cld_filter;
8442	struct i40e_pf *pf = vsi->back;
8443	int ret;
8444
8445	/* Both (src/dst) valid mac_addr are not supported */
8446	if ((is_valid_ether_addr(addr: filter->dst_mac) &&
8447	is_valid_ether_addr(addr: filter->src_mac)) ||
8448	(is_multicast_ether_addr(addr: filter->dst_mac) &&
8449	is_multicast_ether_addr(addr: filter->src_mac)))
8450	return -EOPNOTSUPP;
8451
8452	/* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
8453	* ports are not supported via big buffer now.
8454	*/
8455	if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
8456	return -EOPNOTSUPP;
8457
8458	/* adding filter using src_port/src_ip is not supported at this stage */
8459	if (filter->src_port ||
8460	(filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8461	!ipv6_addr_any(a: &filter->ip.v6.src_ip6))
8462	return -EOPNOTSUPP;
8463
8464	memset(&cld_filter, 0, sizeof(cld_filter));
8465
8466	/* copy element needed to add cloud filter from filter */
8467	i40e_set_cld_element(filter, cld: &cld_filter.element);
8468
8469	if (is_valid_ether_addr(addr: filter->dst_mac) ||
8470	is_valid_ether_addr(addr: filter->src_mac) ||
8471	is_multicast_ether_addr(addr: filter->dst_mac) ||
8472	is_multicast_ether_addr(addr: filter->src_mac)) {
8473	/* MAC + IP : unsupported mode */
8474	if (filter->dst_ipv4)
8475	return -EOPNOTSUPP;
8476
8477	/* since we validated that L4 port must be valid before
8478	* we get here, start with respective "flags" value
8479	* and update if vlan is present or not
8480	*/
8481	cld_filter.element.flags =
8482	cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
8483
8484	if (filter->vlan_id) {
8485	cld_filter.element.flags =
8486	cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
8487	}
8488
8489	} else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8490	!ipv6_addr_any(a: &filter->ip.v6.dst_ip6)) {
8491	cld_filter.element.flags =
8492	cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
8493	if (filter->n_proto == ETH_P_IPV6)
8494	cld_filter.element.flags |=
8495	cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8496	else
8497	cld_filter.element.flags |=
8498	cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8499	} else {
8500	dev_err(&pf->pdev->dev,
8501	"either mac or ip has to be valid for cloud filter\n");
8502	return -EINVAL;
8503	}
8504
8505	/* Now copy L4 port in Byte 6..7 in general fields */
8506	cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
8507	be16_to_cpu(filter->dst_port);
8508
8509	if (add) {
8510	/* Validate current device switch mode, change if necessary */
8511	ret = i40e_validate_and_set_switch_mode(vsi);
8512	if (ret) {
8513	dev_err(&pf->pdev->dev,
8514	"failed to set switch mode, ret %d\n",
8515	ret);
8516	return ret;
8517	}
8518
8519	ret = i40e_aq_add_cloud_filters_bb(hw: &pf->hw, seid: filter->seid,
8520	filters: &cld_filter, filter_count: 1);
8521	} else {
8522	ret = i40e_aq_rem_cloud_filters_bb(hw: &pf->hw, seid: filter->seid,
8523	filters: &cld_filter, filter_count: 1);
8524	}
8525
8526	if (ret)
8527	dev_dbg(&pf->pdev->dev,
8528	"Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
8529	add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
8530	else
8531	dev_info(&pf->pdev->dev,
8532	"%s cloud filter for VSI: %d, L4 port: %d\n",
8533	add ? "add" : "delete", filter->seid,
8534	ntohs(filter->dst_port));
8535	return ret;
8536	}
8537
8538	/**
8539	* i40e_parse_cls_flower - Parse tc flower filters provided by kernel
8540	* @vsi: Pointer to VSI
8541	* @f: Pointer to struct flow_cls_offload
8542	* @filter: Pointer to cloud filter structure
8543	*
8544	**/
8545	static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
8546	struct flow_cls_offload *f,
8547	struct i40e_cloud_filter *filter)
8548	{
8549	struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: f);
8550	struct flow_dissector *dissector = rule->match.dissector;
8551	u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
8552	struct i40e_pf *pf = vsi->back;
8553	u8 field_flags = 0;
8554
8555	if (dissector->used_keys &
8556	~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
8557	BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
8558	BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
8559	BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
8560	BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
8561	BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
8562	BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
8563	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
8564	dev_err(&pf->pdev->dev, "Unsupported key used: 0x%llx\n",
8565	dissector->used_keys);
8566	return -EOPNOTSUPP;
8567	}
8568
8569	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_ENC_KEYID)) {
8570	struct flow_match_enc_keyid match;
8571
8572	flow_rule_match_enc_keyid(rule, out: &match);
8573	if (match.mask->keyid != 0)
8574	field_flags |= I40E_CLOUD_FIELD_TEN_ID;
8575
8576	filter->tenant_id = be32_to_cpu(match.key->keyid);
8577	}
8578
8579	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_BASIC)) {
8580	struct flow_match_basic match;
8581
8582	flow_rule_match_basic(rule, out: &match);
8583	n_proto_key = ntohs(match.key->n_proto);
8584	n_proto_mask = ntohs(match.mask->n_proto);
8585
8586	if (n_proto_key == ETH_P_ALL) {
8587	n_proto_key = 0;
8588	n_proto_mask = 0;
8589	}
8590	filter->n_proto = n_proto_key & n_proto_mask;
8591	filter->ip_proto = match.key->ip_proto;
8592	}
8593
8594	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
8595	struct flow_match_eth_addrs match;
8596
8597	flow_rule_match_eth_addrs(rule, out: &match);
8598
8599	/* use is_broadcast and is_zero to check for all 0xf or 0 */
8600	if (!is_zero_ether_addr(addr: match.mask->dst)) {
8601	if (is_broadcast_ether_addr(addr: match.mask->dst)) {
8602	field_flags |= I40E_CLOUD_FIELD_OMAC;
8603	} else {
8604	dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
8605	match.mask->dst);
8606	return -EIO;
8607	}
8608	}
8609
8610	if (!is_zero_ether_addr(addr: match.mask->src)) {
8611	if (is_broadcast_ether_addr(addr: match.mask->src)) {
8612	field_flags |= I40E_CLOUD_FIELD_IMAC;
8613	} else {
8614	dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
8615	match.mask->src);
8616	return -EIO;
8617	}
8618	}
8619	ether_addr_copy(dst: filter->dst_mac, src: match.key->dst);
8620	ether_addr_copy(dst: filter->src_mac, src: match.key->src);
8621	}
8622
8623	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_VLAN)) {
8624	struct flow_match_vlan match;
8625
8626	flow_rule_match_vlan(rule, out: &match);
8627	if (match.mask->vlan_id) {
8628	if (match.mask->vlan_id == VLAN_VID_MASK) {
8629	field_flags |= I40E_CLOUD_FIELD_IVLAN;
8630
8631	} else {
8632	dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
8633	match.mask->vlan_id);
8634	return -EIO;
8635	}
8636	}
8637
8638	filter->vlan_id = cpu_to_be16(match.key->vlan_id);
8639	}
8640
8641	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_CONTROL)) {
8642	struct flow_match_control match;
8643
8644	flow_rule_match_control(rule, out: &match);
8645	addr_type = match.key->addr_type;
8646	}
8647
8648	if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
8649	struct flow_match_ipv4_addrs match;
8650
8651	flow_rule_match_ipv4_addrs(rule, out: &match);
8652	if (match.mask->dst) {
8653	if (match.mask->dst == cpu_to_be32(0xffffffff)) {
8654	field_flags |= I40E_CLOUD_FIELD_IIP;
8655	} else {
8656	dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
8657	&match.mask->dst);
8658	return -EIO;
8659	}
8660	}
8661
8662	if (match.mask->src) {
8663	if (match.mask->src == cpu_to_be32(0xffffffff)) {
8664	field_flags |= I40E_CLOUD_FIELD_IIP;
8665	} else {
8666	dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
8667	&match.mask->src);
8668	return -EIO;
8669	}
8670	}
8671
8672	if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
8673	dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
8674	return -EIO;
8675	}
8676	filter->dst_ipv4 = match.key->dst;
8677	filter->src_ipv4 = match.key->src;
8678	}
8679
8680	if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
8681	struct flow_match_ipv6_addrs match;
8682
8683	flow_rule_match_ipv6_addrs(rule, out: &match);
8684
8685	/* src and dest IPV6 address should not be LOOPBACK
8686	* (0:0:0:0:0:0:0:1), which can be represented as ::1
8687	*/
8688	if (ipv6_addr_loopback(a: &match.key->dst) ||
8689	ipv6_addr_loopback(a: &match.key->src)) {
8690	dev_err(&pf->pdev->dev,
8691	"Bad ipv6, addr is LOOPBACK\n");
8692	return -EIO;
8693	}
8694	if (!ipv6_addr_any(a: &match.mask->dst) ||
8695	!ipv6_addr_any(a: &match.mask->src))
8696	field_flags |= I40E_CLOUD_FIELD_IIP;
8697
8698	memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
8699	sizeof(filter->src_ipv6));
8700	memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
8701	sizeof(filter->dst_ipv6));
8702	}
8703
8704	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_PORTS)) {
8705	struct flow_match_ports match;
8706
8707	flow_rule_match_ports(rule, out: &match);
8708	if (match.mask->src) {
8709	if (match.mask->src == cpu_to_be16(0xffff)) {
8710	field_flags |= I40E_CLOUD_FIELD_IIP;
8711	} else {
8712	dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
8713	be16_to_cpu(match.mask->src));
8714	return -EIO;
8715	}
8716	}
8717
8718	if (match.mask->dst) {
8719	if (match.mask->dst == cpu_to_be16(0xffff)) {
8720	field_flags |= I40E_CLOUD_FIELD_IIP;
8721	} else {
8722	dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
8723	be16_to_cpu(match.mask->dst));
8724	return -EIO;
8725	}
8726	}
8727
8728	filter->dst_port = match.key->dst;
8729	filter->src_port = match.key->src;
8730
8731	switch (filter->ip_proto) {
8732	case IPPROTO_TCP:
8733	case IPPROTO_UDP:
8734	break;
8735	default:
8736	dev_err(&pf->pdev->dev,
8737	"Only UDP and TCP transport are supported\n");
8738	return -EINVAL;
8739	}
8740	}
8741	filter->flags = field_flags;
8742	return 0;
8743	}
8744
8745	/**
8746	* i40e_handle_tclass: Forward to a traffic class on the device
8747	* @vsi: Pointer to VSI
8748	* @tc: traffic class index on the device
8749	* @filter: Pointer to cloud filter structure
8750	*
8751	**/
8752	static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
8753	struct i40e_cloud_filter *filter)
8754	{
8755	struct i40e_channel *ch, *ch_tmp;
8756
8757	/* direct to a traffic class on the same device */
8758	if (tc == 0) {
8759	filter->seid = vsi->seid;
8760	return 0;
8761	} else if (vsi->tc_config.enabled_tc & BIT(tc)) {
8762	if (!filter->dst_port) {
8763	dev_err(&vsi->back->pdev->dev,
8764	"Specify destination port to direct to traffic class that is not default\n");
8765	return -EINVAL;
8766	}
8767	if (list_empty(head: &vsi->ch_list))
8768	return -EINVAL;
8769	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
8770	list) {
8771	if (ch->seid == vsi->tc_seid_map[tc])
8772	filter->seid = ch->seid;
8773	}
8774	return 0;
8775	}
8776	dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
8777	return -EINVAL;
8778	}
8779
8780	/**
8781	* i40e_configure_clsflower - Configure tc flower filters
8782	* @vsi: Pointer to VSI
8783	* @cls_flower: Pointer to struct flow_cls_offload
8784	*
8785	**/
8786	static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8787	struct flow_cls_offload *cls_flower)
8788	{
8789	int tc = tc_classid_to_hwtc(dev: vsi->netdev, classid: cls_flower->classid);
8790	struct i40e_cloud_filter *filter = NULL;
8791	struct i40e_pf *pf = vsi->back;
8792	int err = 0;
8793
8794	if (tc < 0) {
8795	dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8796	return -EOPNOTSUPP;
8797	}
8798
8799	if (!tc) {
8800	dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination");
8801	return -EINVAL;
8802	}
8803
8804	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8805	test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8806	return -EBUSY;
8807
8808	if (pf->fdir_pf_active_filters ||
8809	(!hlist_empty(h: &pf->fdir_filter_list))) {
8810	dev_err(&vsi->back->pdev->dev,
8811	"Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8812	return -EINVAL;
8813	}
8814
8815	if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags)) {
8816	dev_err(&vsi->back->pdev->dev,
8817	"Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8818	clear_bit(nr: I40E_FLAG_FD_SB_ENA, addr: vsi->back->flags);
8819	clear_bit(nr: I40E_FLAG_FD_SB_TO_CLOUD_FILTER, addr: vsi->back->flags);
8820	}
8821
8822	filter = kzalloc(size: sizeof(*filter), GFP_KERNEL);
8823	if (!filter)
8824	return -ENOMEM;
8825
8826	filter->cookie = cls_flower->cookie;
8827
8828	err = i40e_parse_cls_flower(vsi, f: cls_flower, filter);
8829	if (err < 0)
8830	goto err;
8831
8832	err = i40e_handle_tclass(vsi, tc, filter);
8833	if (err < 0)
8834	goto err;
8835
8836	/* Add cloud filter */
8837	if (filter->dst_port)
8838	err = i40e_add_del_cloud_filter_big_buf(vsi, filter, add: true);
8839	else
8840	err = i40e_add_del_cloud_filter(vsi, filter, add: true);
8841
8842	if (err) {
8843	dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",
8844	err);
8845	goto err;
8846	}
8847
8848	/* add filter to the ordered list */
8849	INIT_HLIST_NODE(h: &filter->cloud_node);
8850
8851	hlist_add_head(n: &filter->cloud_node, h: &pf->cloud_filter_list);
8852
8853	pf->num_cloud_filters++;
8854
8855	return err;
8856	err:
8857	kfree(objp: filter);
8858	return err;
8859	}
8860
8861	/**
8862	* i40e_find_cloud_filter - Find the could filter in the list
8863	* @vsi: Pointer to VSI
8864	* @cookie: filter specific cookie
8865	*
8866	**/
8867	static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8868	unsigned long *cookie)
8869	{
8870	struct i40e_cloud_filter *filter = NULL;
8871	struct hlist_node *node2;
8872
8873	hlist_for_each_entry_safe(filter, node2,
8874	&vsi->back->cloud_filter_list, cloud_node)
8875	if (!memcmp(p: cookie, q: &filter->cookie, size: sizeof(filter->cookie)))
8876	return filter;
8877	return NULL;
8878	}
8879
8880	/**
8881	* i40e_delete_clsflower - Remove tc flower filters
8882	* @vsi: Pointer to VSI
8883	* @cls_flower: Pointer to struct flow_cls_offload
8884	*
8885	**/
8886	static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8887	struct flow_cls_offload *cls_flower)
8888	{
8889	struct i40e_cloud_filter *filter = NULL;
8890	struct i40e_pf *pf = vsi->back;
8891	int err = 0;
8892
8893	filter = i40e_find_cloud_filter(vsi, cookie: &cls_flower->cookie);
8894
8895	if (!filter)
8896	return -EINVAL;
8897
8898	hash_del(node: &filter->cloud_node);
8899
8900	if (filter->dst_port)
8901	err = i40e_add_del_cloud_filter_big_buf(vsi, filter, add: false);
8902	else
8903	err = i40e_add_del_cloud_filter(vsi, filter, add: false);
8904
8905	kfree(objp: filter);
8906	if (err) {
8907	dev_err(&pf->pdev->dev,
8908	"Failed to delete cloud filter, err %pe\n",
8909	ERR_PTR(err));
8910	return i40e_aq_rc_to_posix(aq_ret: err, aq_rc: pf->hw.aq.asq_last_status);
8911	}
8912
8913	pf->num_cloud_filters--;
8914	if (!pf->num_cloud_filters)
8915	if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&
8916	!test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {
8917	set_bit(nr: I40E_FLAG_FD_SB_ENA, addr: pf->flags);
8918	clear_bit(nr: I40E_FLAG_FD_SB_TO_CLOUD_FILTER, addr: pf->flags);
8919	clear_bit(nr: I40E_FLAG_FD_SB_INACTIVE, addr: pf->flags);
8920	}
8921	return 0;
8922	}
8923
8924	/**
8925	* i40e_setup_tc_cls_flower - flower classifier offloads
8926	* @np: net device to configure
8927	* @cls_flower: offload data
8928	**/
8929	static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8930	struct flow_cls_offload *cls_flower)
8931	{
8932	struct i40e_vsi *vsi = np->vsi;
8933
8934	switch (cls_flower->command) {
8935	case FLOW_CLS_REPLACE:
8936	return i40e_configure_clsflower(vsi, cls_flower);
8937	case FLOW_CLS_DESTROY:
8938	return i40e_delete_clsflower(vsi, cls_flower);
8939	case FLOW_CLS_STATS:
8940	return -EOPNOTSUPP;
8941	default:
8942	return -EOPNOTSUPP;
8943	}
8944	}
8945
8946	static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8947	void *cb_priv)
8948	{
8949	struct i40e_netdev_priv *np = cb_priv;
8950
8951	if (!tc_cls_can_offload_and_chain0(dev: np->vsi->netdev, common: type_data))
8952	return -EOPNOTSUPP;
8953
8954	switch (type) {
8955	case TC_SETUP_CLSFLOWER:
8956	return i40e_setup_tc_cls_flower(np, cls_flower: type_data);
8957
8958	default:
8959	return -EOPNOTSUPP;
8960	}
8961	}
8962
8963	static LIST_HEAD(i40e_block_cb_list);
8964
8965	static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8966	void *type_data)
8967	{
8968	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
8969
8970	switch (type) {
8971	case TC_SETUP_QDISC_MQPRIO:
8972	return i40e_setup_tc(netdev, type_data);
8973	case TC_SETUP_BLOCK:
8974	return flow_block_cb_setup_simple(f: type_data,
8975	driver_list: &i40e_block_cb_list,
8976	cb: i40e_setup_tc_block_cb,
8977	cb_ident: np, cb_priv: np, ingress_only: true);
8978	default:
8979	return -EOPNOTSUPP;
8980	}
8981	}
8982
8983	/**
8984	* i40e_open - Called when a network interface is made active
8985	* @netdev: network interface device structure
8986	*
8987	* The open entry point is called when a network interface is made
8988	* active by the system (IFF_UP). At this point all resources needed
8989	* for transmit and receive operations are allocated, the interrupt
8990	* handler is registered with the OS, the netdev watchdog subtask is
8991	* enabled, and the stack is notified that the interface is ready.
8992	*
8993	* Returns 0 on success, negative value on failure
8994	**/
8995	int i40e_open(struct net_device *netdev)
8996	{
8997	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
8998	struct i40e_vsi *vsi = np->vsi;
8999	struct i40e_pf *pf = vsi->back;
9000	int err;
9001
9002	/* disallow open during test or if eeprom is broken */
9003	if (test_bit(__I40E_TESTING, pf->state) ||
9004	test_bit(__I40E_BAD_EEPROM, pf->state))
9005	return -EBUSY;
9006
9007	netif_carrier_off(dev: netdev);
9008
9009	if (i40e_force_link_state(pf, is_up: true))
9010	return -EAGAIN;
9011
9012	err = i40e_vsi_open(vsi);
9013	if (err)
9014	return err;
9015
9016	/* configure global TSO hardware offload settings */
9017	wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
9018	TCP_FLAG_FIN) >> 16);
9019	wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
9020	TCP_FLAG_FIN |
9021	TCP_FLAG_CWR) >> 16);
9022	wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
9023	udp_tunnel_get_rx_info(dev: netdev);
9024
9025	return 0;
9026	}
9027
9028	/**
9029	* i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues
9030	* @vsi: vsi structure
9031	*
9032	* This updates netdev's number of tx/rx queues
9033	*
9034	* Returns status of setting tx/rx queues
9035	**/
9036	static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi)
9037	{
9038	int ret;
9039
9040	ret = netif_set_real_num_rx_queues(dev: vsi->netdev,
9041	rxq: vsi->num_queue_pairs);
9042	if (ret)
9043	return ret;
9044
9045	return netif_set_real_num_tx_queues(dev: vsi->netdev,
9046	txq: vsi->num_queue_pairs);
9047	}
9048
9049	/**
9050	* i40e_vsi_open -
9051	* @vsi: the VSI to open
9052	*
9053	* Finish initialization of the VSI.
9054	*
9055	* Returns 0 on success, negative value on failure
9056	*
9057	* Note: expects to be called while under rtnl_lock()
9058	**/
9059	int i40e_vsi_open(struct i40e_vsi *vsi)
9060	{
9061	struct i40e_pf *pf = vsi->back;
9062	char int_name[I40E_INT_NAME_STR_LEN];
9063	int err;
9064
9065	/* allocate descriptors */
9066	err = i40e_vsi_setup_tx_resources(vsi);
9067	if (err)
9068	goto err_setup_tx;
9069	err = i40e_vsi_setup_rx_resources(vsi);
9070	if (err)
9071	goto err_setup_rx;
9072
9073	err = i40e_vsi_configure(vsi);
9074	if (err)
9075	goto err_setup_rx;
9076
9077	if (vsi->netdev) {
9078	snprintf(buf: int_name, size: sizeof(int_name) - 1, fmt: "%s-%s",
9079	dev_driver_string(dev: &pf->pdev->dev), vsi->netdev->name);
9080	err = i40e_vsi_request_irq(vsi, basename: int_name);
9081	if (err)
9082	goto err_setup_rx;
9083
9084	/* Notify the stack of the actual queue counts. */
9085	err = i40e_netif_set_realnum_tx_rx_queues(vsi);
9086	if (err)
9087	goto err_set_queues;
9088
9089	} else if (vsi->type == I40E_VSI_FDIR) {
9090	snprintf(buf: int_name, size: sizeof(int_name) - 1, fmt: "%s-%s:fdir",
9091	dev_driver_string(dev: &pf->pdev->dev),
9092	dev_name(dev: &pf->pdev->dev));
9093	err = i40e_vsi_request_irq(vsi, basename: int_name);
9094	if (err)
9095	goto err_setup_rx;
9096
9097	} else {
9098	err = -EINVAL;
9099	goto err_setup_rx;
9100	}
9101
9102	err = i40e_up_complete(vsi);
9103	if (err)
9104	goto err_up_complete;
9105
9106	return 0;
9107
9108	err_up_complete:
9109	i40e_down(vsi);
9110	err_set_queues:
9111	i40e_vsi_free_irq(vsi);
9112	err_setup_rx:
9113	i40e_vsi_free_rx_resources(vsi);
9114	err_setup_tx:
9115	i40e_vsi_free_tx_resources(vsi);
9116	if (vsi == pf->vsi[pf->lan_vsi])
9117	i40e_do_reset(pf, I40E_PF_RESET_FLAG, lock_acquired: true);
9118
9119	return err;
9120	}
9121
9122	/**
9123	* i40e_fdir_filter_exit - Cleans up the Flow Director accounting
9124	* @pf: Pointer to PF
9125	*
9126	* This function destroys the hlist where all the Flow Director
9127	* filters were saved.
9128	**/
9129	static void i40e_fdir_filter_exit(struct i40e_pf *pf)
9130	{
9131	struct i40e_fdir_filter *filter;
9132	struct i40e_flex_pit *pit_entry, *tmp;
9133	struct hlist_node *node2;
9134
9135	hlist_for_each_entry_safe(filter, node2,
9136	&pf->fdir_filter_list, fdir_node) {
9137	hlist_del(n: &filter->fdir_node);
9138	kfree(objp: filter);
9139	}
9140
9141	list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
9142	list_del(entry: &pit_entry->list);
9143	kfree(objp: pit_entry);
9144	}
9145	INIT_LIST_HEAD(list: &pf->l3_flex_pit_list);
9146
9147	list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
9148	list_del(entry: &pit_entry->list);
9149	kfree(objp: pit_entry);
9150	}
9151	INIT_LIST_HEAD(list: &pf->l4_flex_pit_list);
9152
9153	pf->fdir_pf_active_filters = 0;
9154	i40e_reset_fdir_filter_cnt(pf);
9155
9156	/* Reprogram the default input set for TCP/IPv4 */
9157	i40e_write_fd_input_set(pf, addr: I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9158	I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9159	I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9160
9161	/* Reprogram the default input set for TCP/IPv6 */
9162	i40e_write_fd_input_set(pf, addr: I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
9163	I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9164	I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9165
9166	/* Reprogram the default input set for UDP/IPv4 */
9167	i40e_write_fd_input_set(pf, addr: I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
9168	I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9169	I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9170
9171	/* Reprogram the default input set for UDP/IPv6 */
9172	i40e_write_fd_input_set(pf, addr: I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
9173	I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9174	I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9175
9176	/* Reprogram the default input set for SCTP/IPv4 */
9177	i40e_write_fd_input_set(pf, addr: I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
9178	I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9179	I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9180
9181	/* Reprogram the default input set for SCTP/IPv6 */
9182	i40e_write_fd_input_set(pf, addr: I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
9183	I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9184	I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9185
9186	/* Reprogram the default input set for Other/IPv4 */
9187	i40e_write_fd_input_set(pf, addr: I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
9188	I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9189
9190	i40e_write_fd_input_set(pf, addr: I40E_FILTER_PCTYPE_FRAG_IPV4,
9191	I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9192
9193	/* Reprogram the default input set for Other/IPv6 */
9194	i40e_write_fd_input_set(pf, addr: I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
9195	I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9196
9197	i40e_write_fd_input_set(pf, addr: I40E_FILTER_PCTYPE_FRAG_IPV6,
9198	I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9199	}
9200
9201	/**
9202	* i40e_cloud_filter_exit - Cleans up the cloud filters
9203	* @pf: Pointer to PF
9204	*
9205	* This function destroys the hlist where all the cloud filters
9206	* were saved.
9207	**/
9208	static void i40e_cloud_filter_exit(struct i40e_pf *pf)
9209	{
9210	struct i40e_cloud_filter *cfilter;
9211	struct hlist_node *node;
9212
9213	hlist_for_each_entry_safe(cfilter, node,
9214	&pf->cloud_filter_list, cloud_node) {
9215	hlist_del(n: &cfilter->cloud_node);
9216	kfree(objp: cfilter);
9217	}
9218	pf->num_cloud_filters = 0;
9219
9220	if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&
9221	!test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {
9222	set_bit(nr: I40E_FLAG_FD_SB_ENA, addr: pf->flags);
9223	clear_bit(nr: I40E_FLAG_FD_SB_TO_CLOUD_FILTER, addr: pf->flags);
9224	clear_bit(nr: I40E_FLAG_FD_SB_INACTIVE, addr: pf->flags);
9225	}
9226	}
9227
9228	/**
9229	* i40e_close - Disables a network interface
9230	* @netdev: network interface device structure
9231	*
9232	* The close entry point is called when an interface is de-activated
9233	* by the OS. The hardware is still under the driver's control, but
9234	* this netdev interface is disabled.
9235	*
9236	* Returns 0, this is not allowed to fail
9237	**/
9238	int i40e_close(struct net_device *netdev)
9239	{
9240	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
9241	struct i40e_vsi *vsi = np->vsi;
9242
9243	i40e_vsi_close(vsi);
9244
9245	return 0;
9246	}
9247
9248	/**
9249	* i40e_do_reset - Start a PF or Core Reset sequence
9250	* @pf: board private structure
9251	* @reset_flags: which reset is requested
9252	* @lock_acquired: indicates whether or not the lock has been acquired
9253	* before this function was called.
9254	*
9255	* The essential difference in resets is that the PF Reset
9256	* doesn't clear the packet buffers, doesn't reset the PE
9257	* firmware, and doesn't bother the other PFs on the chip.
9258	**/
9259	void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
9260	{
9261	struct i40e_vsi *vsi;
9262	u32 val;
9263	int i;
9264
9265	/* do the biggest reset indicated */
9266	if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
9267
9268	/* Request a Global Reset
9269	*
9270	* This will start the chip's countdown to the actual full
9271	* chip reset event, and a warning interrupt to be sent
9272	* to all PFs, including the requestor. Our handler
9273	* for the warning interrupt will deal with the shutdown
9274	* and recovery of the switch setup.
9275	*/
9276	dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
9277	val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9278	val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
9279	wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9280
9281	} else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
9282
9283	/* Request a Core Reset
9284	*
9285	* Same as Global Reset, except does *not* include the MAC/PHY
9286	*/
9287	dev_dbg(&pf->pdev->dev, "CoreR requested\n");
9288	val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9289	val |= I40E_GLGEN_RTRIG_CORER_MASK;
9290	wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9291	i40e_flush(&pf->hw);
9292
9293	} else if (reset_flags & I40E_PF_RESET_FLAG) {
9294
9295	/* Request a PF Reset
9296	*
9297	* Resets only the PF-specific registers
9298	*
9299	* This goes directly to the tear-down and rebuild of
9300	* the switch, since we need to do all the recovery as
9301	* for the Core Reset.
9302	*/
9303	dev_dbg(&pf->pdev->dev, "PFR requested\n");
9304	i40e_handle_reset_warning(pf, lock_acquired);
9305
9306	} else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
9307	/* Request a PF Reset
9308	*
9309	* Resets PF and reinitializes PFs VSI.
9310	*/
9311	i40e_prep_for_reset(pf);
9312	i40e_reset_and_rebuild(pf, reinit: true, lock_acquired);
9313	dev_info(&pf->pdev->dev,
9314	test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?
9315	"FW LLDP is disabled\n" :
9316	"FW LLDP is enabled\n");
9317
9318	} else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
9319	/* Find the VSI(s) that requested a re-init */
9320	dev_info(&pf->pdev->dev, "VSI reinit requested\n");
9321
9322	i40e_pf_for_each_vsi(pf, i, vsi) {
9323	if (test_and_clear_bit(nr: __I40E_VSI_REINIT_REQUESTED,
9324	addr: vsi->state))
9325	i40e_vsi_reinit_locked(vsi);
9326	}
9327	} else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
9328	/* Find the VSI(s) that needs to be brought down */
9329	dev_info(&pf->pdev->dev, "VSI down requested\n");
9330
9331	i40e_pf_for_each_vsi(pf, i, vsi) {
9332	if (test_and_clear_bit(nr: __I40E_VSI_DOWN_REQUESTED,
9333	addr: vsi->state)) {
9334	set_bit(nr: __I40E_VSI_DOWN, addr: vsi->state);
9335	i40e_down(vsi);
9336	}
9337	}
9338	} else {
9339	dev_info(&pf->pdev->dev,
9340	"bad reset request 0x%08x\n", reset_flags);
9341	}
9342	}
9343
9344	#ifdef CONFIG_I40E_DCB
9345	/**
9346	* i40e_dcb_need_reconfig - Check if DCB needs reconfig
9347	* @pf: board private structure
9348	* @old_cfg: current DCB config
9349	* @new_cfg: new DCB config
9350	**/
9351	bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
9352	struct i40e_dcbx_config *old_cfg,
9353	struct i40e_dcbx_config *new_cfg)
9354	{
9355	bool need_reconfig = false;
9356
9357	/* Check if ETS configuration has changed */
9358	if (memcmp(p: &new_cfg->etscfg,
9359	q: &old_cfg->etscfg,
9360	size: sizeof(new_cfg->etscfg))) {
9361	/* If Priority Table has changed reconfig is needed */
9362	if (memcmp(p: &new_cfg->etscfg.prioritytable,
9363	q: &old_cfg->etscfg.prioritytable,
9364	size: sizeof(new_cfg->etscfg.prioritytable))) {
9365	need_reconfig = true;
9366	dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
9367	}
9368
9369	if (memcmp(p: &new_cfg->etscfg.tcbwtable,
9370	q: &old_cfg->etscfg.tcbwtable,
9371	size: sizeof(new_cfg->etscfg.tcbwtable)))
9372	dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
9373
9374	if (memcmp(p: &new_cfg->etscfg.tsatable,
9375	q: &old_cfg->etscfg.tsatable,
9376	size: sizeof(new_cfg->etscfg.tsatable)))
9377	dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
9378	}
9379
9380	/* Check if PFC configuration has changed */
9381	if (memcmp(p: &new_cfg->pfc,
9382	q: &old_cfg->pfc,
9383	size: sizeof(new_cfg->pfc))) {
9384	need_reconfig = true;
9385	dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
9386	}
9387
9388	/* Check if APP Table has changed */
9389	if (memcmp(p: &new_cfg->app,
9390	q: &old_cfg->app,
9391	size: sizeof(new_cfg->app))) {
9392	need_reconfig = true;
9393	dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
9394	}
9395
9396	dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
9397	return need_reconfig;
9398	}
9399
9400	/**
9401	* i40e_handle_lldp_event - Handle LLDP Change MIB event
9402	* @pf: board private structure
9403	* @e: event info posted on ARQ
9404	**/
9405	static int i40e_handle_lldp_event(struct i40e_pf *pf,
9406	struct i40e_arq_event_info *e)
9407	{
9408	struct i40e_aqc_lldp_get_mib *mib =
9409	(struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
9410	struct i40e_hw *hw = &pf->hw;
9411	struct i40e_dcbx_config tmp_dcbx_cfg;
9412	bool need_reconfig = false;
9413	int ret = 0;
9414	u8 type;
9415
9416	/* X710-T*L 2.5G and 5G speeds don't support DCB */
9417	if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9418	(hw->phy.link_info.link_speed &
9419	~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) &&
9420	!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
9421	/* let firmware decide if the DCB should be disabled */
9422	set_bit(nr: I40E_FLAG_DCB_CAPABLE, addr: pf->flags);
9423
9424	/* Not DCB capable or capability disabled */
9425	if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
9426	return ret;
9427
9428	/* Ignore if event is not for Nearest Bridge */
9429	type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
9430	& I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
9431	dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
9432	if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
9433	return ret;
9434
9435	/* Check MIB Type and return if event for Remote MIB update */
9436	type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
9437	dev_dbg(&pf->pdev->dev,
9438	"LLDP event mib type %s\n", type ? "remote" : "local");
9439	if (type == I40E_AQ_LLDP_MIB_REMOTE) {
9440	/* Update the remote cached instance and return */
9441	ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
9442	I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
9443	dcbcfg: &hw->remote_dcbx_config);
9444	goto exit;
9445	}
9446
9447	/* Store the old configuration */
9448	tmp_dcbx_cfg = hw->local_dcbx_config;
9449
9450	/* Reset the old DCBx configuration data */
9451	memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
9452	/* Get updated DCBX data from firmware */
9453	ret = i40e_get_dcb_config(hw: &pf->hw);
9454	if (ret) {
9455	/* X710-T*L 2.5G and 5G speeds don't support DCB */
9456	if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9457	(hw->phy.link_info.link_speed &
9458	(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
9459	dev_warn(&pf->pdev->dev,
9460	"DCB is not supported for X710-T*L 2.5/5G speeds\n");
9461	clear_bit(nr: I40E_FLAG_DCB_CAPABLE, addr: pf->flags);
9462	} else {
9463	dev_info(&pf->pdev->dev,
9464	"Failed querying DCB configuration data from firmware, err %pe aq_err %s\n",
9465	ERR_PTR(ret),
9466	i40e_aq_str(&pf->hw,
9467	pf->hw.aq.asq_last_status));
9468	}
9469	goto exit;
9470	}
9471
9472	/* No change detected in DCBX configs */
9473	if (!memcmp(p: &tmp_dcbx_cfg, q: &hw->local_dcbx_config,
9474	size: sizeof(tmp_dcbx_cfg))) {
9475	dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
9476	goto exit;
9477	}
9478
9479	need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg: &tmp_dcbx_cfg,
9480	new_cfg: &hw->local_dcbx_config);
9481
9482	i40e_dcbnl_flush_apps(pf, old_cfg: &tmp_dcbx_cfg, new_cfg: &hw->local_dcbx_config);
9483
9484	if (!need_reconfig)
9485	goto exit;
9486
9487	/* Enable DCB tagging only when more than one TC */
9488	if (i40e_dcb_get_num_tc(dcbcfg: &hw->local_dcbx_config) > 1)
9489	set_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
9490	else
9491	clear_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
9492
9493	set_bit(nr: __I40E_PORT_SUSPENDED, addr: pf->state);
9494	/* Reconfiguration needed quiesce all VSIs */
9495	i40e_pf_quiesce_all_vsi(pf);
9496
9497	/* Changes in configuration update VEB/VSI */
9498	i40e_dcb_reconfigure(pf);
9499
9500	ret = i40e_resume_port_tx(pf);
9501
9502	clear_bit(nr: __I40E_PORT_SUSPENDED, addr: pf->state);
9503	/* In case of error no point in resuming VSIs */
9504	if (ret)
9505	goto exit;
9506
9507	/* Wait for the PF's queues to be disabled */
9508	ret = i40e_pf_wait_queues_disabled(pf);
9509	if (ret) {
9510	/* Schedule PF reset to recover */
9511	set_bit(nr: __I40E_PF_RESET_REQUESTED, addr: pf->state);
9512	i40e_service_event_schedule(pf);
9513	} else {
9514	i40e_pf_unquiesce_all_vsi(pf);
9515	set_bit(nr: __I40E_CLIENT_SERVICE_REQUESTED, addr: pf->state);
9516	set_bit(nr: __I40E_CLIENT_L2_CHANGE, addr: pf->state);
9517	}
9518
9519	exit:
9520	return ret;
9521	}
9522	#endif /* CONFIG_I40E_DCB */
9523
9524	/**
9525	* i40e_do_reset_safe - Protected reset path for userland calls.
9526	* @pf: board private structure
9527	* @reset_flags: which reset is requested
9528	*
9529	**/
9530	void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
9531	{
9532	rtnl_lock();
9533	i40e_do_reset(pf, reset_flags, lock_acquired: true);
9534	rtnl_unlock();
9535	}
9536
9537	/**
9538	* i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
9539	* @pf: board private structure
9540	* @e: event info posted on ARQ
9541	*
9542	* Handler for LAN Queue Overflow Event generated by the firmware for PF
9543	* and VF queues
9544	**/
9545	static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
9546	struct i40e_arq_event_info *e)
9547	{
9548	struct i40e_aqc_lan_overflow *data =
9549	(struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
9550	u32 queue = le32_to_cpu(data->prtdcb_rupto);
9551	u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
9552	struct i40e_hw *hw = &pf->hw;
9553	struct i40e_vf *vf;
9554	u16 vf_id;
9555
9556	dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
9557	queue, qtx_ctl);
9558
9559	if (FIELD_GET(I40E_QTX_CTL_PFVF_Q_MASK, qtx_ctl) !=
9560	I40E_QTX_CTL_VF_QUEUE)
9561	return;
9562
9563	/* Queue belongs to VF, find the VF and issue VF reset */
9564	vf_id = FIELD_GET(I40E_QTX_CTL_VFVM_INDX_MASK, qtx_ctl);
9565	vf_id -= hw->func_caps.vf_base_id;
9566	vf = &pf->vf[vf_id];
9567	i40e_vc_notify_vf_reset(vf);
9568	/* Allow VF to process pending reset notification */
9569	msleep(msecs: 20);
9570	i40e_reset_vf(vf, flr: false);
9571	}
9572
9573	/**
9574	* i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
9575	* @pf: board private structure
9576	**/
9577	u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
9578	{
9579	u32 val, fcnt_prog;
9580
9581	val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9582	fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
9583	return fcnt_prog;
9584	}
9585
9586	/**
9587	* i40e_get_current_fd_count - Get total FD filters programmed for this PF
9588	* @pf: board private structure
9589	**/
9590	u32 i40e_get_current_fd_count(struct i40e_pf *pf)
9591	{
9592	u32 val, fcnt_prog;
9593
9594	val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9595	fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
9596	FIELD_GET(I40E_PFQF_FDSTAT_BEST_CNT_MASK, val);
9597	return fcnt_prog;
9598	}
9599
9600	/**
9601	* i40e_get_global_fd_count - Get total FD filters programmed on device
9602	* @pf: board private structure
9603	**/
9604	u32 i40e_get_global_fd_count(struct i40e_pf *pf)
9605	{
9606	u32 val, fcnt_prog;
9607
9608	val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
9609	fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
9610	FIELD_GET(I40E_GLQF_FDCNT_0_BESTCNT_MASK, val);
9611	return fcnt_prog;
9612	}
9613
9614	/**
9615	* i40e_reenable_fdir_sb - Restore FDir SB capability
9616	* @pf: board private structure
9617	**/
9618	static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
9619	{
9620	if (test_and_clear_bit(nr: __I40E_FD_SB_AUTO_DISABLED, addr: pf->state))
9621	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
9622	(I40E_DEBUG_FD & pf->hw.debug_mask))
9623	dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
9624	}
9625
9626	/**
9627	* i40e_reenable_fdir_atr - Restore FDir ATR capability
9628	* @pf: board private structure
9629	**/
9630	static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
9631	{
9632	if (test_and_clear_bit(nr: __I40E_FD_ATR_AUTO_DISABLED, addr: pf->state)) {
9633	/* ATR uses the same filtering logic as SB rules. It only
9634	* functions properly if the input set mask is at the default
9635	* settings. It is safe to restore the default input set
9636	* because there are no active TCPv4 filter rules.
9637	*/
9638	i40e_write_fd_input_set(pf, addr: I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9639	I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9640	I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9641
9642	if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
9643	(I40E_DEBUG_FD & pf->hw.debug_mask))
9644	dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
9645	}
9646	}
9647
9648	/**
9649	* i40e_delete_invalid_filter - Delete an invalid FDIR filter
9650	* @pf: board private structure
9651	* @filter: FDir filter to remove
9652	*/
9653	static void i40e_delete_invalid_filter(struct i40e_pf *pf,
9654	struct i40e_fdir_filter *filter)
9655	{
9656	/* Update counters */
9657	pf->fdir_pf_active_filters--;
9658	pf->fd_inv = 0;
9659
9660	switch (filter->flow_type) {
9661	case TCP_V4_FLOW:
9662	pf->fd_tcp4_filter_cnt--;
9663	break;
9664	case UDP_V4_FLOW:
9665	pf->fd_udp4_filter_cnt--;
9666	break;
9667	case SCTP_V4_FLOW:
9668	pf->fd_sctp4_filter_cnt--;
9669	break;
9670	case TCP_V6_FLOW:
9671	pf->fd_tcp6_filter_cnt--;
9672	break;
9673	case UDP_V6_FLOW:
9674	pf->fd_udp6_filter_cnt--;
9675	break;
9676	case SCTP_V6_FLOW:
9677	pf->fd_udp6_filter_cnt--;
9678	break;
9679	case IP_USER_FLOW:
9680	switch (filter->ipl4_proto) {
9681	case IPPROTO_TCP:
9682	pf->fd_tcp4_filter_cnt--;
9683	break;
9684	case IPPROTO_UDP:
9685	pf->fd_udp4_filter_cnt--;
9686	break;
9687	case IPPROTO_SCTP:
9688	pf->fd_sctp4_filter_cnt--;
9689	break;
9690	case IPPROTO_IP:
9691	pf->fd_ip4_filter_cnt--;
9692	break;
9693	}
9694	break;
9695	case IPV6_USER_FLOW:
9696	switch (filter->ipl4_proto) {
9697	case IPPROTO_TCP:
9698	pf->fd_tcp6_filter_cnt--;
9699	break;
9700	case IPPROTO_UDP:
9701	pf->fd_udp6_filter_cnt--;
9702	break;
9703	case IPPROTO_SCTP:
9704	pf->fd_sctp6_filter_cnt--;
9705	break;
9706	case IPPROTO_IP:
9707	pf->fd_ip6_filter_cnt--;
9708	break;
9709	}
9710	break;
9711	}
9712
9713	/* Remove the filter from the list and free memory */
9714	hlist_del(n: &filter->fdir_node);
9715	kfree(objp: filter);
9716	}
9717
9718	/**
9719	* i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
9720	* @pf: board private structure
9721	**/
9722	void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
9723	{
9724	struct i40e_fdir_filter *filter;
9725	u32 fcnt_prog, fcnt_avail;
9726	struct hlist_node *node;
9727
9728	if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9729	return;
9730
9731	/* Check if we have enough room to re-enable FDir SB capability. */
9732	fcnt_prog = i40e_get_global_fd_count(pf);
9733	fcnt_avail = pf->fdir_pf_filter_count;
9734	if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
9735	(pf->fd_add_err == 0) ||
9736	(i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
9737	i40e_reenable_fdir_sb(pf);
9738
9739	/* We should wait for even more space before re-enabling ATR.
9740	* Additionally, we cannot enable ATR as long as we still have TCP SB
9741	* rules active.
9742	*/
9743	if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
9744	pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0)
9745	i40e_reenable_fdir_atr(pf);
9746
9747	/* if hw had a problem adding a filter, delete it */
9748	if (pf->fd_inv > 0) {
9749	hlist_for_each_entry_safe(filter, node,
9750	&pf->fdir_filter_list, fdir_node)
9751	if (filter->fd_id == pf->fd_inv)
9752	i40e_delete_invalid_filter(pf, filter);
9753	}
9754	}
9755
9756	#define I40E_MIN_FD_FLUSH_INTERVAL 10
9757	#define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
9758	/**
9759	* i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
9760	* @pf: board private structure
9761	**/
9762	static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
9763	{
9764	unsigned long min_flush_time;
9765	int flush_wait_retry = 50;
9766	bool disable_atr = false;
9767	int fd_room;
9768	int reg;
9769
9770	if (!time_after(jiffies, pf->fd_flush_timestamp +
9771	(I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
9772	return;
9773
9774	/* If the flush is happening too quick and we have mostly SB rules we
9775	* should not re-enable ATR for some time.
9776	*/
9777	min_flush_time = pf->fd_flush_timestamp +
9778	(I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
9779	fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
9780
9781	if (!(time_after(jiffies, min_flush_time)) &&
9782	(fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
9783	if (I40E_DEBUG_FD & pf->hw.debug_mask)
9784	dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
9785	disable_atr = true;
9786	}
9787
9788	pf->fd_flush_timestamp = jiffies;
9789	set_bit(nr: __I40E_FD_ATR_AUTO_DISABLED, addr: pf->state);
9790	/* flush all filters */
9791	wr32(&pf->hw, I40E_PFQF_CTL_1,
9792	I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
9793	i40e_flush(&pf->hw);
9794	pf->fd_flush_cnt++;
9795	pf->fd_add_err = 0;
9796	do {
9797	/* Check FD flush status every 5-6msec */
9798	usleep_range(min: 5000, max: 6000);
9799	reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
9800	if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
9801	break;
9802	} while (flush_wait_retry--);
9803	if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
9804	dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
9805	} else {
9806	/* replay sideband filters */
9807	i40e_fdir_filter_restore(vsi: pf->vsi[pf->lan_vsi]);
9808	if (!disable_atr && !pf->fd_tcp4_filter_cnt)
9809	clear_bit(nr: __I40E_FD_ATR_AUTO_DISABLED, addr: pf->state);
9810	clear_bit(nr: __I40E_FD_FLUSH_REQUESTED, addr: pf->state);
9811	if (I40E_DEBUG_FD & pf->hw.debug_mask)
9812	dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
9813	}
9814	}
9815
9816	/**
9817	* i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed
9818	* @pf: board private structure
9819	**/
9820	u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
9821	{
9822	return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
9823	}
9824
9825	/**
9826	* i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
9827	* @pf: board private structure
9828	**/
9829	static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
9830	{
9831
9832	/* if interface is down do nothing */
9833	if (test_bit(__I40E_DOWN, pf->state))
9834	return;
9835
9836	if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9837	i40e_fdir_flush_and_replay(pf);
9838
9839	i40e_fdir_check_and_reenable(pf);
9840
9841	}
9842
9843	/**
9844	* i40e_vsi_link_event - notify VSI of a link event
9845	* @vsi: vsi to be notified
9846	* @link_up: link up or down
9847	**/
9848	static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
9849	{
9850	if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
9851	return;
9852
9853	switch (vsi->type) {
9854	case I40E_VSI_MAIN:
9855	if (!vsi->netdev || !vsi->netdev_registered)
9856	break;
9857
9858	if (link_up) {
9859	netif_carrier_on(dev: vsi->netdev);
9860	netif_tx_wake_all_queues(dev: vsi->netdev);
9861	} else {
9862	netif_carrier_off(dev: vsi->netdev);
9863	netif_tx_stop_all_queues(dev: vsi->netdev);
9864	}
9865	break;
9866
9867	case I40E_VSI_SRIOV:
9868	case I40E_VSI_VMDQ2:
9869	case I40E_VSI_CTRL:
9870	case I40E_VSI_IWARP:
9871	case I40E_VSI_MIRROR:
9872	default:
9873	/* there is no notification for other VSIs */
9874	break;
9875	}
9876	}
9877
9878	/**
9879	* i40e_veb_link_event - notify elements on the veb of a link event
9880	* @veb: veb to be notified
9881	* @link_up: link up or down
9882	**/
9883	static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9884	{
9885	struct i40e_vsi *vsi;
9886	struct i40e_pf *pf;
9887	int i;
9888
9889	if (!veb || !veb->pf)
9890	return;
9891	pf = veb->pf;
9892
9893	/* Send link event to contained VSIs */
9894	i40e_pf_for_each_vsi(pf, i, vsi)
9895	if (vsi->uplink_seid == veb->seid)
9896	i40e_vsi_link_event(vsi, link_up);
9897	}
9898
9899	/**
9900	* i40e_link_event - Update netif_carrier status
9901	* @pf: board private structure
9902	**/
9903	static void i40e_link_event(struct i40e_pf *pf)
9904	{
9905	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9906	u8 new_link_speed, old_link_speed;
9907	bool new_link, old_link;
9908	int status;
9909	#ifdef CONFIG_I40E_DCB
9910	int err;
9911	#endif /* CONFIG_I40E_DCB */
9912
9913	/* set this to force the get_link_status call to refresh state */
9914	pf->hw.phy.get_link_info = true;
9915	old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9916	status = i40e_get_link_status(hw: &pf->hw, link_up: &new_link);
9917
9918	/* On success, disable temp link polling */
9919	if (status == 0) {
9920	clear_bit(nr: __I40E_TEMP_LINK_POLLING, addr: pf->state);
9921	} else {
9922	/* Enable link polling temporarily until i40e_get_link_status
9923	* returns 0
9924	*/
9925	set_bit(nr: __I40E_TEMP_LINK_POLLING, addr: pf->state);
9926	dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9927	status);
9928	return;
9929	}
9930
9931	old_link_speed = pf->hw.phy.link_info_old.link_speed;
9932	new_link_speed = pf->hw.phy.link_info.link_speed;
9933
9934	if (new_link == old_link &&
9935	new_link_speed == old_link_speed &&
9936	(test_bit(__I40E_VSI_DOWN, vsi->state) ||
9937	new_link == netif_carrier_ok(dev: vsi->netdev)))
9938	return;
9939
9940	i40e_print_link_message(vsi, isup: new_link);
9941
9942	/* Notify the base of the switch tree connected to
9943	* the link. Floating VEBs are not notified.
9944	*/
9945	if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
9946	i40e_veb_link_event(veb: pf->veb[pf->lan_veb], link_up: new_link);
9947	else
9948	i40e_vsi_link_event(vsi, link_up: new_link);
9949
9950	if (pf->vf)
9951	i40e_vc_notify_link_state(pf);
9952
9953	if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
9954	i40e_ptp_set_increment(pf);
9955	#ifdef CONFIG_I40E_DCB
9956	if (new_link == old_link)
9957	return;
9958	/* Not SW DCB so firmware will take care of default settings */
9959	if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)
9960	return;
9961
9962	/* We cover here only link down, as after link up in case of SW DCB
9963	* SW LLDP agent will take care of setting it up
9964	*/
9965	if (!new_link) {
9966	dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n");
9967	memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));
9968	err = i40e_dcb_sw_default_config(pf);
9969	if (err) {
9970	clear_bit(nr: I40E_FLAG_DCB_CAPABLE, addr: pf->flags);
9971	clear_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
9972	} else {
9973	pf->dcbx_cap = DCB_CAP_DCBX_HOST |
9974	DCB_CAP_DCBX_VER_IEEE;
9975	set_bit(nr: I40E_FLAG_DCB_CAPABLE, addr: pf->flags);
9976	clear_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
9977	}
9978	}
9979	#endif /* CONFIG_I40E_DCB */
9980	}
9981
9982	/**
9983	* i40e_watchdog_subtask - periodic checks not using event driven response
9984	* @pf: board private structure
9985	**/
9986	static void i40e_watchdog_subtask(struct i40e_pf *pf)
9987	{
9988	struct i40e_vsi *vsi;
9989	struct i40e_veb *veb;
9990	int i;
9991
9992	/* if interface is down do nothing */
9993	if (test_bit(__I40E_DOWN, pf->state) ||
9994	test_bit(__I40E_CONFIG_BUSY, pf->state))
9995	return;
9996
9997	/* make sure we don't do these things too often */
9998	if (time_before(jiffies, (pf->service_timer_previous +
9999	pf->service_timer_period)))
10000	return;
10001	pf->service_timer_previous = jiffies;
10002
10003	if (test_bit(I40E_FLAG_LINK_POLLING_ENA, pf->flags) ||
10004	test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
10005	i40e_link_event(pf);
10006
10007	/* Update the stats for active netdevs so the network stack
10008	* can look at updated numbers whenever it cares to
10009	*/
10010	i40e_pf_for_each_vsi(pf, i, vsi)
10011	if (vsi->netdev)
10012	i40e_update_stats(vsi);
10013
10014	if (test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags)) {
10015	/* Update the stats for the active switching components */
10016	i40e_pf_for_each_veb(pf, i, veb)
10017	i40e_update_veb_stats(veb);
10018	}
10019
10020	i40e_ptp_rx_hang(pf);
10021	i40e_ptp_tx_hang(pf);
10022	}
10023
10024	/**
10025	* i40e_reset_subtask - Set up for resetting the device and driver
10026	* @pf: board private structure
10027	**/
10028	static void i40e_reset_subtask(struct i40e_pf *pf)
10029	{
10030	u32 reset_flags = 0;
10031
10032	if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
10033	reset_flags |= BIT(__I40E_REINIT_REQUESTED);
10034	clear_bit(nr: __I40E_REINIT_REQUESTED, addr: pf->state);
10035	}
10036	if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
10037	reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
10038	clear_bit(nr: __I40E_PF_RESET_REQUESTED, addr: pf->state);
10039	}
10040	if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
10041	reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
10042	clear_bit(nr: __I40E_CORE_RESET_REQUESTED, addr: pf->state);
10043	}
10044	if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
10045	reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
10046	clear_bit(nr: __I40E_GLOBAL_RESET_REQUESTED, addr: pf->state);
10047	}
10048	if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
10049	reset_flags |= BIT(__I40E_DOWN_REQUESTED);
10050	clear_bit(nr: __I40E_DOWN_REQUESTED, addr: pf->state);
10051	}
10052
10053	/* If there's a recovery already waiting, it takes
10054	* precedence before starting a new reset sequence.
10055	*/
10056	if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
10057	i40e_prep_for_reset(pf);
10058	i40e_reset(pf);
10059	i40e_rebuild(pf, reinit: false, lock_acquired: false);
10060	}
10061
10062	/* If we're already down or resetting, just bail */
10063	if (reset_flags &&
10064	!test_bit(__I40E_DOWN, pf->state) &&
10065	!test_bit(__I40E_CONFIG_BUSY, pf->state)) {
10066	i40e_do_reset(pf, reset_flags, lock_acquired: false);
10067	}
10068	}
10069
10070	/**
10071	* i40e_handle_link_event - Handle link event
10072	* @pf: board private structure
10073	* @e: event info posted on ARQ
10074	**/
10075	static void i40e_handle_link_event(struct i40e_pf *pf,
10076	struct i40e_arq_event_info *e)
10077	{
10078	struct i40e_aqc_get_link_status *status =
10079	(struct i40e_aqc_get_link_status *)&e->desc.params.raw;
10080
10081	/* Do a new status request to re-enable LSE reporting
10082	* and load new status information into the hw struct
10083	* This completely ignores any state information
10084	* in the ARQ event info, instead choosing to always
10085	* issue the AQ update link status command.
10086	*/
10087	i40e_link_event(pf);
10088
10089	/* Check if module meets thermal requirements */
10090	if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
10091	dev_err(&pf->pdev->dev,
10092	"Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
10093	dev_err(&pf->pdev->dev,
10094	"Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10095	} else {
10096	/* check for unqualified module, if link is down, suppress
10097	* the message if link was forced to be down.
10098	*/
10099	if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
10100	(!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
10101	(!(status->link_info & I40E_AQ_LINK_UP)) &&
10102	(!test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))) {
10103	dev_err(&pf->pdev->dev,
10104	"Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
10105	dev_err(&pf->pdev->dev,
10106	"Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10107	}
10108	}
10109	}
10110
10111	/**
10112	* i40e_clean_adminq_subtask - Clean the AdminQ rings
10113	* @pf: board private structure
10114	**/
10115	static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
10116	{
10117	struct i40e_arq_event_info event;
10118	struct i40e_hw *hw = &pf->hw;
10119	u16 pending, i = 0;
10120	u16 opcode;
10121	u32 oldval;
10122	int ret;
10123	u32 val;
10124
10125	/* Do not run clean AQ when PF reset fails */
10126	if (test_bit(__I40E_RESET_FAILED, pf->state))
10127	return;
10128
10129	/* check for error indications */
10130	val = rd32(&pf->hw, I40E_PF_ARQLEN);
10131	oldval = val;
10132	if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
10133	if (hw->debug_mask & I40E_DEBUG_AQ)
10134	dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
10135	val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
10136	}
10137	if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
10138	if (hw->debug_mask & I40E_DEBUG_AQ)
10139	dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
10140	val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
10141	pf->arq_overflows++;
10142	}
10143	if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
10144	if (hw->debug_mask & I40E_DEBUG_AQ)
10145	dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
10146	val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
10147	}
10148	if (oldval != val)
10149	wr32(&pf->hw, I40E_PF_ARQLEN, val);
10150
10151	val = rd32(&pf->hw, I40E_PF_ATQLEN);
10152	oldval = val;
10153	if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
10154	if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10155	dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
10156	val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
10157	}
10158	if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
10159	if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10160	dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
10161	val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
10162	}
10163	if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
10164	if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10165	dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
10166	val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
10167	}
10168	if (oldval != val)
10169	wr32(&pf->hw, I40E_PF_ATQLEN, val);
10170
10171	event.buf_len = I40E_MAX_AQ_BUF_SIZE;
10172	event.msg_buf = kzalloc(size: event.buf_len, GFP_KERNEL);
10173	if (!event.msg_buf)
10174	return;
10175
10176	do {
10177	ret = i40e_clean_arq_element(hw, e: &event, events_pending: &pending);
10178	if (ret == -EALREADY)
10179	break;
10180	else if (ret) {
10181	dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
10182	break;
10183	}
10184
10185	opcode = le16_to_cpu(event.desc.opcode);
10186	switch (opcode) {
10187
10188	case i40e_aqc_opc_get_link_status:
10189	rtnl_lock();
10190	i40e_handle_link_event(pf, e: &event);
10191	rtnl_unlock();
10192	break;
10193	case i40e_aqc_opc_send_msg_to_pf:
10194	ret = i40e_vc_process_vf_msg(pf,
10195	le16_to_cpu(event.desc.retval),
10196	le32_to_cpu(event.desc.cookie_high),
10197	le32_to_cpu(event.desc.cookie_low),
10198	msg: event.msg_buf,
10199	msglen: event.msg_len);
10200	break;
10201	case i40e_aqc_opc_lldp_update_mib:
10202	dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
10203	#ifdef CONFIG_I40E_DCB
10204	rtnl_lock();
10205	i40e_handle_lldp_event(pf, e: &event);
10206	rtnl_unlock();
10207	#endif /* CONFIG_I40E_DCB */
10208	break;
10209	case i40e_aqc_opc_event_lan_overflow:
10210	dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
10211	i40e_handle_lan_overflow_event(pf, e: &event);
10212	break;
10213	case i40e_aqc_opc_send_msg_to_peer:
10214	dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
10215	break;
10216	case i40e_aqc_opc_nvm_erase:
10217	case i40e_aqc_opc_nvm_update:
10218	case i40e_aqc_opc_oem_post_update:
10219	i40e_debug(&pf->hw, I40E_DEBUG_NVM,
10220	"ARQ NVM operation 0x%04x completed\n",
10221	opcode);
10222	break;
10223	default:
10224	dev_info(&pf->pdev->dev,
10225	"ARQ: Unknown event 0x%04x ignored\n",
10226	opcode);
10227	break;
10228	}
10229	} while (i++ < I40E_AQ_WORK_LIMIT);
10230
10231	if (i < I40E_AQ_WORK_LIMIT)
10232	clear_bit(nr: __I40E_ADMINQ_EVENT_PENDING, addr: pf->state);
10233
10234	/* re-enable Admin queue interrupt cause */
10235	val = rd32(hw, I40E_PFINT_ICR0_ENA);
10236	val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
10237	wr32(hw, I40E_PFINT_ICR0_ENA, val);
10238	i40e_flush(hw);
10239
10240	kfree(objp: event.msg_buf);
10241	}
10242
10243	/**
10244	* i40e_verify_eeprom - make sure eeprom is good to use
10245	* @pf: board private structure
10246	**/
10247	static void i40e_verify_eeprom(struct i40e_pf *pf)
10248	{
10249	int err;
10250
10251	err = i40e_diag_eeprom_test(hw: &pf->hw);
10252	if (err) {
10253	/* retry in case of garbage read */
10254	err = i40e_diag_eeprom_test(hw: &pf->hw);
10255	if (err) {
10256	dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
10257	err);
10258	set_bit(nr: __I40E_BAD_EEPROM, addr: pf->state);
10259	}
10260	}
10261
10262	if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
10263	dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
10264	clear_bit(nr: __I40E_BAD_EEPROM, addr: pf->state);
10265	}
10266	}
10267
10268	/**
10269	* i40e_enable_pf_switch_lb
10270	* @pf: pointer to the PF structure
10271	*
10272	* enable switch loop back or die - no point in a return value
10273	**/
10274	static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
10275	{
10276	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10277	struct i40e_vsi_context ctxt;
10278	int ret;
10279
10280	ctxt.seid = pf->main_vsi_seid;
10281	ctxt.pf_num = pf->hw.pf_id;
10282	ctxt.vf_num = 0;
10283	ret = i40e_aq_get_vsi_params(hw: &pf->hw, vsi_ctx: &ctxt, NULL);
10284	if (ret) {
10285	dev_info(&pf->pdev->dev,
10286	"couldn't get PF vsi config, err %pe aq_err %s\n",
10287	ERR_PTR(ret),
10288	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10289	return;
10290	}
10291	ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10292	ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10293	ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10294
10295	ret = i40e_aq_update_vsi_params(hw: &vsi->back->hw, vsi_ctx: &ctxt, NULL);
10296	if (ret) {
10297	dev_info(&pf->pdev->dev,
10298	"update vsi switch failed, err %pe aq_err %s\n",
10299	ERR_PTR(ret),
10300	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10301	}
10302	}
10303
10304	/**
10305	* i40e_disable_pf_switch_lb
10306	* @pf: pointer to the PF structure
10307	*
10308	* disable switch loop back or die - no point in a return value
10309	**/
10310	static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
10311	{
10312	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10313	struct i40e_vsi_context ctxt;
10314	int ret;
10315
10316	ctxt.seid = pf->main_vsi_seid;
10317	ctxt.pf_num = pf->hw.pf_id;
10318	ctxt.vf_num = 0;
10319	ret = i40e_aq_get_vsi_params(hw: &pf->hw, vsi_ctx: &ctxt, NULL);
10320	if (ret) {
10321	dev_info(&pf->pdev->dev,
10322	"couldn't get PF vsi config, err %pe aq_err %s\n",
10323	ERR_PTR(ret),
10324	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10325	return;
10326	}
10327	ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10328	ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10329	ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10330
10331	ret = i40e_aq_update_vsi_params(hw: &vsi->back->hw, vsi_ctx: &ctxt, NULL);
10332	if (ret) {
10333	dev_info(&pf->pdev->dev,
10334	"update vsi switch failed, err %pe aq_err %s\n",
10335	ERR_PTR(ret),
10336	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10337	}
10338	}
10339
10340	/**
10341	* i40e_config_bridge_mode - Configure the HW bridge mode
10342	* @veb: pointer to the bridge instance
10343	*
10344	* Configure the loop back mode for the LAN VSI that is downlink to the
10345	* specified HW bridge instance. It is expected this function is called
10346	* when a new HW bridge is instantiated.
10347	**/
10348	static void i40e_config_bridge_mode(struct i40e_veb *veb)
10349	{
10350	struct i40e_pf *pf = veb->pf;
10351
10352	if (pf->hw.debug_mask & I40E_DEBUG_LAN)
10353	dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
10354	veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
10355	if (veb->bridge_mode & BRIDGE_MODE_VEPA)
10356	i40e_disable_pf_switch_lb(pf);
10357	else
10358	i40e_enable_pf_switch_lb(pf);
10359	}
10360
10361	/**
10362	* i40e_reconstitute_veb - rebuild the VEB and VSIs connected to it
10363	* @veb: pointer to the VEB instance
10364	*
10365	* This is a function that builds the attached VSIs. We track the connections
10366	* through our own index numbers because the seid's from the HW could change
10367	* across the reset.
10368	**/
10369	static int i40e_reconstitute_veb(struct i40e_veb *veb)
10370	{
10371	struct i40e_vsi *ctl_vsi = NULL;
10372	struct i40e_pf *pf = veb->pf;
10373	struct i40e_vsi *vsi;
10374	int v, ret;
10375
10376	/* As we do not maintain PV (port virtualizer) switch element then
10377	* there can be only one non-floating VEB that have uplink to MAC SEID
10378	* and its control VSI is the main one.
10379	*/
10380	if (WARN_ON(veb->uplink_seid && veb->uplink_seid != pf->mac_seid)) {
10381	dev_err(&pf->pdev->dev,
10382	"Invalid uplink SEID for VEB %d\n", veb->idx);
10383	return -ENOENT;
10384	}
10385
10386	if (veb->uplink_seid == pf->mac_seid) {
10387	/* Check that the LAN VSI has VEB owning flag set */
10388	ctl_vsi = pf->vsi[pf->lan_vsi];
10389
10390	if (WARN_ON(ctl_vsi->veb_idx != veb->idx ||
10391	!(ctl_vsi->flags & I40E_VSI_FLAG_VEB_OWNER))) {
10392	dev_err(&pf->pdev->dev,
10393	"Invalid control VSI for VEB %d\n", veb->idx);
10394	return -ENOENT;
10395	}
10396
10397	/* Add the control VSI to switch */
10398	ret = i40e_add_vsi(vsi: ctl_vsi);
10399	if (ret) {
10400	dev_err(&pf->pdev->dev,
10401	"Rebuild of owner VSI for VEB %d failed: %d\n",
10402	veb->idx, ret);
10403	return ret;
10404	}
10405
10406	i40e_vsi_reset_stats(vsi: ctl_vsi);
10407	}
10408
10409	/* create the VEB in the switch and move the VSI onto the VEB */
10410	ret = i40e_add_veb(veb, vsi: ctl_vsi);
10411	if (ret)
10412	return ret;
10413
10414	if (veb->uplink_seid) {
10415	if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))
10416	veb->bridge_mode = BRIDGE_MODE_VEB;
10417	else
10418	veb->bridge_mode = BRIDGE_MODE_VEPA;
10419	i40e_config_bridge_mode(veb);
10420	}
10421
10422	/* create the remaining VSIs attached to this VEB */
10423	i40e_pf_for_each_vsi(pf, v, vsi) {
10424	if (vsi == ctl_vsi)
10425	continue;
10426
10427	if (vsi->veb_idx == veb->idx) {
10428	vsi->uplink_seid = veb->seid;
10429	ret = i40e_add_vsi(vsi);
10430	if (ret) {
10431	dev_info(&pf->pdev->dev,
10432	"rebuild of vsi_idx %d failed: %d\n",
10433	v, ret);
10434	return ret;
10435	}
10436	i40e_vsi_reset_stats(vsi);
10437	}
10438	}
10439
10440	return ret;
10441	}
10442
10443	/**
10444	* i40e_get_capabilities - get info about the HW
10445	* @pf: the PF struct
10446	* @list_type: AQ capability to be queried
10447	**/
10448	static int i40e_get_capabilities(struct i40e_pf *pf,
10449	enum i40e_admin_queue_opc list_type)
10450	{
10451	struct i40e_aqc_list_capabilities_element_resp *cap_buf;
10452	u16 data_size;
10453	int buf_len;
10454	int err;
10455
10456	buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
10457	do {
10458	cap_buf = kzalloc(size: buf_len, GFP_KERNEL);
10459	if (!cap_buf)
10460	return -ENOMEM;
10461
10462	/* this loads the data into the hw struct for us */
10463	err = i40e_aq_discover_capabilities(hw: &pf->hw, buff: cap_buf, buff_size: buf_len,
10464	data_size: &data_size, list_type_opc: list_type,
10465	NULL);
10466	/* data loaded, buffer no longer needed */
10467	kfree(objp: cap_buf);
10468
10469	if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
10470	/* retry with a larger buffer */
10471	buf_len = data_size;
10472	} else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {
10473	dev_info(&pf->pdev->dev,
10474	"capability discovery failed, err %pe aq_err %s\n",
10475	ERR_PTR(err),
10476	i40e_aq_str(&pf->hw,
10477	pf->hw.aq.asq_last_status));
10478	return -ENODEV;
10479	}
10480	} while (err);
10481
10482	if (pf->hw.debug_mask & I40E_DEBUG_USER) {
10483	if (list_type == i40e_aqc_opc_list_func_capabilities) {
10484	dev_info(&pf->pdev->dev,
10485	"pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
10486	pf->hw.pf_id, pf->hw.func_caps.num_vfs,
10487	pf->hw.func_caps.num_msix_vectors,
10488	pf->hw.func_caps.num_msix_vectors_vf,
10489	pf->hw.func_caps.fd_filters_guaranteed,
10490	pf->hw.func_caps.fd_filters_best_effort,
10491	pf->hw.func_caps.num_tx_qp,
10492	pf->hw.func_caps.num_vsis);
10493	} else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
10494	dev_info(&pf->pdev->dev,
10495	"switch_mode=0x%04x, function_valid=0x%08x\n",
10496	pf->hw.dev_caps.switch_mode,
10497	pf->hw.dev_caps.valid_functions);
10498	dev_info(&pf->pdev->dev,
10499	"SR-IOV=%d, num_vfs for all function=%u\n",
10500	pf->hw.dev_caps.sr_iov_1_1,
10501	pf->hw.dev_caps.num_vfs);
10502	dev_info(&pf->pdev->dev,
10503	"num_vsis=%u, num_rx:%u, num_tx=%u\n",
10504	pf->hw.dev_caps.num_vsis,
10505	pf->hw.dev_caps.num_rx_qp,
10506	pf->hw.dev_caps.num_tx_qp);
10507	}
10508	}
10509	if (list_type == i40e_aqc_opc_list_func_capabilities) {
10510	#define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
10511	+ pf->hw.func_caps.num_vfs)
10512	if (pf->hw.revision_id == 0 &&
10513	pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
10514	dev_info(&pf->pdev->dev,
10515	"got num_vsis %d, setting num_vsis to %d\n",
10516	pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
10517	pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
10518	}
10519	}
10520	return 0;
10521	}
10522
10523	static int i40e_vsi_clear(struct i40e_vsi *vsi);
10524
10525	/**
10526	* i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
10527	* @pf: board private structure
10528	**/
10529	static void i40e_fdir_sb_setup(struct i40e_pf *pf)
10530	{
10531	struct i40e_vsi *vsi;
10532
10533	/* quick workaround for an NVM issue that leaves a critical register
10534	* uninitialized
10535	*/
10536	if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
10537	static const u32 hkey[] = {
10538	0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
10539	0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
10540	0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
10541	0x95b3a76d};
10542	int i;
10543
10544	for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
10545	wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
10546	}
10547
10548	if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
10549	return;
10550
10551	/* find existing VSI and see if it needs configuring */
10552	vsi = i40e_find_vsi_by_type(pf, type: I40E_VSI_FDIR);
10553
10554	/* create a new VSI if none exists */
10555	if (!vsi) {
10556	vsi = i40e_vsi_setup(pf, type: I40E_VSI_FDIR,
10557	uplink: pf->vsi[pf->lan_vsi]->seid, param1: 0);
10558	if (!vsi) {
10559	dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
10560	clear_bit(nr: I40E_FLAG_FD_SB_ENA, addr: pf->flags);
10561	set_bit(nr: I40E_FLAG_FD_SB_INACTIVE, addr: pf->flags);
10562	return;
10563	}
10564	}
10565
10566	i40e_vsi_setup_irqhandler(vsi, irq_handler: i40e_fdir_clean_ring);
10567	}
10568
10569	/**
10570	* i40e_fdir_teardown - release the Flow Director resources
10571	* @pf: board private structure
10572	**/
10573	static void i40e_fdir_teardown(struct i40e_pf *pf)
10574	{
10575	struct i40e_vsi *vsi;
10576
10577	i40e_fdir_filter_exit(pf);
10578	vsi = i40e_find_vsi_by_type(pf, type: I40E_VSI_FDIR);
10579	if (vsi)
10580	i40e_vsi_release(vsi);
10581	}
10582
10583	/**
10584	* i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
10585	* @vsi: PF main vsi
10586	* @seid: seid of main or channel VSIs
10587	*
10588	* Rebuilds cloud filters associated with main VSI and channel VSIs if they
10589	* existed before reset
10590	**/
10591	static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
10592	{
10593	struct i40e_cloud_filter *cfilter;
10594	struct i40e_pf *pf = vsi->back;
10595	struct hlist_node *node;
10596	int ret;
10597
10598	/* Add cloud filters back if they exist */
10599	hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
10600	cloud_node) {
10601	if (cfilter->seid != seid)
10602	continue;
10603
10604	if (cfilter->dst_port)
10605	ret = i40e_add_del_cloud_filter_big_buf(vsi, filter: cfilter,
10606	add: true);
10607	else
10608	ret = i40e_add_del_cloud_filter(vsi, filter: cfilter, add: true);
10609
10610	if (ret) {
10611	dev_dbg(&pf->pdev->dev,
10612	"Failed to rebuild cloud filter, err %pe aq_err %s\n",
10613	ERR_PTR(ret),
10614	i40e_aq_str(&pf->hw,
10615	pf->hw.aq.asq_last_status));
10616	return ret;
10617	}
10618	}
10619	return 0;
10620	}
10621
10622	/**
10623	* i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
10624	* @vsi: PF main vsi
10625	*
10626	* Rebuilds channel VSIs if they existed before reset
10627	**/
10628	static int i40e_rebuild_channels(struct i40e_vsi *vsi)
10629	{
10630	struct i40e_channel *ch, *ch_tmp;
10631	int ret;
10632
10633	if (list_empty(head: &vsi->ch_list))
10634	return 0;
10635
10636	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
10637	if (!ch->initialized)
10638	break;
10639	/* Proceed with creation of channel (VMDq2) VSI */
10640	ret = i40e_add_channel(pf: vsi->back, uplink_seid: vsi->uplink_seid, ch);
10641	if (ret) {
10642	dev_info(&vsi->back->pdev->dev,
10643	"failed to rebuild channels using uplink_seid %u\n",
10644	vsi->uplink_seid);
10645	return ret;
10646	}
10647	/* Reconfigure TX queues using QTX_CTL register */
10648	ret = i40e_channel_config_tx_ring(pf: vsi->back, vsi, ch);
10649	if (ret) {
10650	dev_info(&vsi->back->pdev->dev,
10651	"failed to configure TX rings for channel %u\n",
10652	ch->seid);
10653	return ret;
10654	}
10655	/* update 'next_base_queue' */
10656	vsi->next_base_queue = vsi->next_base_queue +
10657	ch->num_queue_pairs;
10658	if (ch->max_tx_rate) {
10659	u64 credits = ch->max_tx_rate;
10660
10661	if (i40e_set_bw_limit(vsi, seid: ch->seid,
10662	max_tx_rate: ch->max_tx_rate))
10663	return -EINVAL;
10664
10665	do_div(credits, I40E_BW_CREDIT_DIVISOR);
10666	dev_dbg(&vsi->back->pdev->dev,
10667	"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10668	ch->max_tx_rate,
10669	credits,
10670	ch->seid);
10671	}
10672	ret = i40e_rebuild_cloud_filters(vsi, seid: ch->seid);
10673	if (ret) {
10674	dev_dbg(&vsi->back->pdev->dev,
10675	"Failed to rebuild cloud filters for channel VSI %u\n",
10676	ch->seid);
10677	return ret;
10678	}
10679	}
10680	return 0;
10681	}
10682
10683	/**
10684	* i40e_clean_xps_state - clean xps state for every tx_ring
10685	* @vsi: ptr to the VSI
10686	**/
10687	static void i40e_clean_xps_state(struct i40e_vsi *vsi)
10688	{
10689	int i;
10690
10691	if (vsi->tx_rings)
10692	for (i = 0; i < vsi->num_queue_pairs; i++)
10693	if (vsi->tx_rings[i])
10694	clear_bit(nr: __I40E_TX_XPS_INIT_DONE,
10695	addr: vsi->tx_rings[i]->state);
10696	}
10697
10698	/**
10699	* i40e_prep_for_reset - prep for the core to reset
10700	* @pf: board private structure
10701	*
10702	* Close up the VFs and other things in prep for PF Reset.
10703	**/
10704	static void i40e_prep_for_reset(struct i40e_pf *pf)
10705	{
10706	struct i40e_hw *hw = &pf->hw;
10707	struct i40e_vsi *vsi;
10708	int ret = 0;
10709	u32 v;
10710
10711	clear_bit(nr: __I40E_RESET_INTR_RECEIVED, addr: pf->state);
10712	if (test_and_set_bit(nr: __I40E_RESET_RECOVERY_PENDING, addr: pf->state))
10713	return;
10714	if (i40e_check_asq_alive(hw: &pf->hw))
10715	i40e_vc_notify_reset(pf);
10716
10717	dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
10718
10719	/* quiesce the VSIs and their queues that are not already DOWN */
10720	i40e_pf_quiesce_all_vsi(pf);
10721
10722	i40e_pf_for_each_vsi(pf, v, vsi) {
10723	i40e_clean_xps_state(vsi);
10724	vsi->seid = 0;
10725	}
10726
10727	i40e_shutdown_adminq(hw: &pf->hw);
10728
10729	/* call shutdown HMC */
10730	if (hw->hmc.hmc_obj) {
10731	ret = i40e_shutdown_lan_hmc(hw);
10732	if (ret)
10733	dev_warn(&pf->pdev->dev,
10734	"shutdown_lan_hmc failed: %d\n", ret);
10735	}
10736
10737	/* Save the current PTP time so that we can restore the time after the
10738	* reset completes.
10739	*/
10740	i40e_ptp_save_hw_time(pf);
10741	}
10742
10743	/**
10744	* i40e_send_version - update firmware with driver version
10745	* @pf: PF struct
10746	*/
10747	static void i40e_send_version(struct i40e_pf *pf)
10748	{
10749	struct i40e_driver_version dv;
10750
10751	dv.major_version = 0xff;
10752	dv.minor_version = 0xff;
10753	dv.build_version = 0xff;
10754	dv.subbuild_version = 0;
10755	strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
10756	i40e_aq_send_driver_version(hw: &pf->hw, dv: &dv, NULL);
10757	}
10758
10759	/**
10760	* i40e_get_oem_version - get OEM specific version information
10761	* @hw: pointer to the hardware structure
10762	**/
10763	static void i40e_get_oem_version(struct i40e_hw *hw)
10764	{
10765	u16 block_offset = 0xffff;
10766	u16 block_length = 0;
10767	u16 capabilities = 0;
10768	u16 gen_snap = 0;
10769	u16 release = 0;
10770
10771	#define I40E_SR_NVM_OEM_VERSION_PTR 0x1B
10772	#define I40E_NVM_OEM_LENGTH_OFFSET 0x00
10773	#define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01
10774	#define I40E_NVM_OEM_GEN_OFFSET 0x02
10775	#define I40E_NVM_OEM_RELEASE_OFFSET 0x03
10776	#define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F
10777	#define I40E_NVM_OEM_LENGTH 3
10778
10779	/* Check if pointer to OEM version block is valid. */
10780	i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, data: &block_offset);
10781	if (block_offset == 0xffff)
10782	return;
10783
10784	/* Check if OEM version block has correct length. */
10785	i40e_read_nvm_word(hw, offset: block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
10786	data: &block_length);
10787	if (block_length < I40E_NVM_OEM_LENGTH)
10788	return;
10789
10790	/* Check if OEM version format is as expected. */
10791	i40e_read_nvm_word(hw, offset: block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
10792	data: &capabilities);
10793	if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
10794	return;
10795
10796	i40e_read_nvm_word(hw, offset: block_offset + I40E_NVM_OEM_GEN_OFFSET,
10797	data: &gen_snap);
10798	i40e_read_nvm_word(hw, offset: block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
10799	data: &release);
10800	hw->nvm.oem_ver =
10801	FIELD_PREP(I40E_OEM_GEN_MASK | I40E_OEM_SNAP_MASK, gen_snap) |
10802	FIELD_PREP(I40E_OEM_RELEASE_MASK, release);
10803	hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
10804	}
10805
10806	/**
10807	* i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
10808	* @pf: board private structure
10809	**/
10810	static int i40e_reset(struct i40e_pf *pf)
10811	{
10812	struct i40e_hw *hw = &pf->hw;
10813	int ret;
10814
10815	ret = i40e_pf_reset(hw);
10816	if (ret) {
10817	dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
10818	set_bit(nr: __I40E_RESET_FAILED, addr: pf->state);
10819	clear_bit(nr: __I40E_RESET_RECOVERY_PENDING, addr: pf->state);
10820	} else {
10821	pf->pfr_count++;
10822	}
10823	return ret;
10824	}
10825
10826	/**
10827	* i40e_rebuild - rebuild using a saved config
10828	* @pf: board private structure
10829	* @reinit: if the Main VSI needs to re-initialized.
10830	* @lock_acquired: indicates whether or not the lock has been acquired
10831	* before this function was called.
10832	**/
10833	static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
10834	{
10835	const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf);
10836	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10837	struct i40e_hw *hw = &pf->hw;
10838	struct i40e_veb *veb;
10839	int ret;
10840	u32 val;
10841	int v;
10842
10843	if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10844	is_recovery_mode_reported)
10845	i40e_set_ethtool_ops(netdev: pf->vsi[pf->lan_vsi]->netdev);
10846
10847	if (test_bit(__I40E_DOWN, pf->state) &&
10848	!test_bit(__I40E_RECOVERY_MODE, pf->state))
10849	goto clear_recovery;
10850	dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
10851
10852	/* rebuild the basics for the AdminQ, HMC, and initial HW switch */
10853	ret = i40e_init_adminq(hw: &pf->hw);
10854	if (ret) {
10855	dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %pe aq_err %s\n",
10856	ERR_PTR(ret),
10857	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10858	goto clear_recovery;
10859	}
10860	i40e_get_oem_version(hw: &pf->hw);
10861
10862	if (test_and_clear_bit(nr: __I40E_EMP_RESET_INTR_RECEIVED, addr: pf->state)) {
10863	/* The following delay is necessary for firmware update. */
10864	mdelay(1000);
10865	}
10866
10867	/* re-verify the eeprom if we just had an EMP reset */
10868	if (test_and_clear_bit(nr: __I40E_EMP_RESET_INTR_RECEIVED, addr: pf->state))
10869	i40e_verify_eeprom(pf);
10870
10871	/* if we are going out of or into recovery mode we have to act
10872	* accordingly with regard to resources initialization
10873	* and deinitialization
10874	*/
10875	if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10876	if (i40e_get_capabilities(pf,
10877	list_type: i40e_aqc_opc_list_func_capabilities))
10878	goto end_unlock;
10879
10880	if (is_recovery_mode_reported) {
10881	/* we're staying in recovery mode so we'll reinitialize
10882	* misc vector here
10883	*/
10884	if (i40e_setup_misc_vector_for_recovery_mode(pf))
10885	goto end_unlock;
10886	} else {
10887	if (!lock_acquired)
10888	rtnl_lock();
10889	/* we're going out of recovery mode so we'll free
10890	* the IRQ allocated specifically for recovery mode
10891	* and restore the interrupt scheme
10892	*/
10893	free_irq(pf->pdev->irq, pf);
10894	i40e_clear_interrupt_scheme(pf);
10895	if (i40e_restore_interrupt_scheme(pf))
10896	goto end_unlock;
10897	}
10898
10899	/* tell the firmware that we're starting */
10900	i40e_send_version(pf);
10901
10902	/* bail out in case recovery mode was detected, as there is
10903	* no need for further configuration.
10904	*/
10905	goto end_unlock;
10906	}
10907
10908	i40e_clear_pxe_mode(hw);
10909	ret = i40e_get_capabilities(pf, list_type: i40e_aqc_opc_list_func_capabilities);
10910	if (ret)
10911	goto end_core_reset;
10912
10913	ret = i40e_init_lan_hmc(hw, txq_num: hw->func_caps.num_tx_qp,
10914	rxq_num: hw->func_caps.num_rx_qp, fcoe_cntx_num: 0, fcoe_filt_num: 0);
10915	if (ret) {
10916	dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10917	goto end_core_reset;
10918	}
10919	ret = i40e_configure_lan_hmc(hw, model: I40E_HMC_MODEL_DIRECT_ONLY);
10920	if (ret) {
10921	dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10922	goto end_core_reset;
10923	}
10924
10925	#ifdef CONFIG_I40E_DCB
10926	/* Enable FW to write a default DCB config on link-up
10927	* unless I40E_FLAG_TC_MQPRIO was enabled or DCB
10928	* is not supported with new link speed
10929	*/
10930	if (i40e_is_tc_mqprio_enabled(pf)) {
10931	i40e_aq_set_dcb_parameters(hw, dcb_enable: false, NULL);
10932	} else {
10933	if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
10934	(hw->phy.link_info.link_speed &
10935	(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
10936	i40e_aq_set_dcb_parameters(hw, dcb_enable: false, NULL);
10937	dev_warn(&pf->pdev->dev,
10938	"DCB is not supported for X710-T*L 2.5/5G speeds\n");
10939	clear_bit(nr: I40E_FLAG_DCB_CAPABLE, addr: pf->flags);
10940	} else {
10941	i40e_aq_set_dcb_parameters(hw, dcb_enable: true, NULL);
10942	ret = i40e_init_pf_dcb(pf);
10943	if (ret) {
10944	dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",
10945	ret);
10946	clear_bit(nr: I40E_FLAG_DCB_CAPABLE, addr: pf->flags);
10947	/* Continue without DCB enabled */
10948	}
10949	}
10950	}
10951
10952	#endif /* CONFIG_I40E_DCB */
10953	if (!lock_acquired)
10954	rtnl_lock();
10955	ret = i40e_setup_pf_switch(pf, reinit, lock_acquired: true);
10956	if (ret)
10957	goto end_unlock;
10958
10959	/* The driver only wants link up/down and module qualification
10960	* reports from firmware. Note the negative logic.
10961	*/
10962	ret = i40e_aq_set_phy_int_mask(hw: &pf->hw,
10963	mask: ~(I40E_AQ_EVENT_LINK_UPDOWN |
10964	I40E_AQ_EVENT_MEDIA_NA |
10965	I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
10966	if (ret)
10967	dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",
10968	ERR_PTR(ret),
10969	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10970
10971	/* Rebuild the VSIs and VEBs that existed before reset.
10972	* They are still in our local switch element arrays, so only
10973	* need to rebuild the switch model in the HW.
10974	*
10975	* If there were VEBs but the reconstitution failed, we'll try
10976	* to recover minimal use by getting the basic PF VSI working.
10977	*/
10978	if (vsi->uplink_seid != pf->mac_seid) {
10979	dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
10980
10981	/* Rebuild VEBs */
10982	i40e_pf_for_each_veb(pf, v, veb) {
10983	ret = i40e_reconstitute_veb(veb);
10984	if (!ret)
10985	continue;
10986
10987	/* If Main VEB failed, we're in deep doodoo,
10988	* so give up rebuilding the switch and set up
10989	* for minimal rebuild of PF VSI.
10990	* If orphan failed, we'll report the error
10991	* but try to keep going.
10992	*/
10993	if (veb->uplink_seid == pf->mac_seid) {
10994	dev_info(&pf->pdev->dev,
10995	"rebuild of switch failed: %d, will try to set up simple PF connection\n",
10996	ret);
10997	vsi->uplink_seid = pf->mac_seid;
10998	break;
10999	} else if (veb->uplink_seid == 0) {
11000	dev_info(&pf->pdev->dev,
11001	"rebuild of orphan VEB failed: %d\n",
11002	ret);
11003	}
11004	}
11005	}
11006
11007	if (vsi->uplink_seid == pf->mac_seid) {
11008	dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
11009	/* no VEB, so rebuild only the Main VSI */
11010	ret = i40e_add_vsi(vsi);
11011	if (ret) {
11012	dev_info(&pf->pdev->dev,
11013	"rebuild of Main VSI failed: %d\n", ret);
11014	goto end_unlock;
11015	}
11016	}
11017
11018	if (vsi->mqprio_qopt.max_rate[0]) {
11019	u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
11020	max_tx_rate: vsi->mqprio_qopt.max_rate[0]);
11021	u64 credits = 0;
11022
11023	ret = i40e_set_bw_limit(vsi, seid: vsi->seid, max_tx_rate);
11024	if (ret)
11025	goto end_unlock;
11026
11027	credits = max_tx_rate;
11028	do_div(credits, I40E_BW_CREDIT_DIVISOR);
11029	dev_dbg(&vsi->back->pdev->dev,
11030	"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
11031	max_tx_rate,
11032	credits,
11033	vsi->seid);
11034	}
11035
11036	ret = i40e_rebuild_cloud_filters(vsi, seid: vsi->seid);
11037	if (ret)
11038	goto end_unlock;
11039
11040	/* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
11041	* for this main VSI if they exist
11042	*/
11043	ret = i40e_rebuild_channels(vsi);
11044	if (ret)
11045	goto end_unlock;
11046
11047	/* Reconfigure hardware for allowing smaller MSS in the case
11048	* of TSO, so that we avoid the MDD being fired and causing
11049	* a reset in the case of small MSS+TSO.
11050	*/
11051	#define I40E_REG_MSS 0x000E64DC
11052	#define I40E_REG_MSS_MIN_MASK 0x3FF0000
11053	#define I40E_64BYTE_MSS 0x400000
11054	val = rd32(hw, I40E_REG_MSS);
11055	if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11056	val &= ~I40E_REG_MSS_MIN_MASK;
11057	val |= I40E_64BYTE_MSS;
11058	wr32(hw, I40E_REG_MSS, val);
11059	}
11060
11061	if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {
11062	msleep(msecs: 75);
11063	ret = i40e_aq_set_link_restart_an(hw: &pf->hw, enable_link: true, NULL);
11064	if (ret)
11065	dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",
11066	ERR_PTR(ret),
11067	i40e_aq_str(&pf->hw,
11068	pf->hw.aq.asq_last_status));
11069	}
11070	/* reinit the misc interrupt */
11071	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
11072	ret = i40e_setup_misc_vector(pf);
11073	if (ret)
11074	goto end_unlock;
11075	}
11076
11077	/* Add a filter to drop all Flow control frames from any VSI from being
11078	* transmitted. By doing so we stop a malicious VF from sending out
11079	* PAUSE or PFC frames and potentially controlling traffic for other
11080	* PF/VF VSIs.
11081	* The FW can still send Flow control frames if enabled.
11082	*/
11083	i40e_add_filter_to_drop_tx_flow_control_frames(hw: &pf->hw,
11084	vsi_seid: pf->main_vsi_seid);
11085
11086	/* restart the VSIs that were rebuilt and running before the reset */
11087	i40e_pf_unquiesce_all_vsi(pf);
11088
11089	/* Release the RTNL lock before we start resetting VFs */
11090	if (!lock_acquired)
11091	rtnl_unlock();
11092
11093	/* Restore promiscuous settings */
11094	ret = i40e_set_promiscuous(pf, promisc: pf->cur_promisc);
11095	if (ret)
11096	dev_warn(&pf->pdev->dev,
11097	"Failed to restore promiscuous setting: %s, err %pe aq_err %s\n",
11098	pf->cur_promisc ? "on" : "off",
11099	ERR_PTR(ret),
11100	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11101
11102	i40e_reset_all_vfs(pf, flr: true);
11103
11104	/* tell the firmware that we're starting */
11105	i40e_send_version(pf);
11106
11107	/* We've already released the lock, so don't do it again */
11108	goto end_core_reset;
11109
11110	end_unlock:
11111	if (!lock_acquired)
11112	rtnl_unlock();
11113	end_core_reset:
11114	clear_bit(nr: __I40E_RESET_FAILED, addr: pf->state);
11115	clear_recovery:
11116	clear_bit(nr: __I40E_RESET_RECOVERY_PENDING, addr: pf->state);
11117	clear_bit(nr: __I40E_TIMEOUT_RECOVERY_PENDING, addr: pf->state);
11118	}
11119
11120	/**
11121	* i40e_reset_and_rebuild - reset and rebuild using a saved config
11122	* @pf: board private structure
11123	* @reinit: if the Main VSI needs to re-initialized.
11124	* @lock_acquired: indicates whether or not the lock has been acquired
11125	* before this function was called.
11126	**/
11127	static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
11128	bool lock_acquired)
11129	{
11130	int ret;
11131
11132	if (test_bit(__I40E_IN_REMOVE, pf->state))
11133	return;
11134	/* Now we wait for GRST to settle out.
11135	* We don't have to delete the VEBs or VSIs from the hw switch
11136	* because the reset will make them disappear.
11137	*/
11138	ret = i40e_reset(pf);
11139	if (!ret)
11140	i40e_rebuild(pf, reinit, lock_acquired);
11141	}
11142
11143	/**
11144	* i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
11145	* @pf: board private structure
11146	*
11147	* Close up the VFs and other things in prep for a Core Reset,
11148	* then get ready to rebuild the world.
11149	* @lock_acquired: indicates whether or not the lock has been acquired
11150	* before this function was called.
11151	**/
11152	static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
11153	{
11154	i40e_prep_for_reset(pf);
11155	i40e_reset_and_rebuild(pf, reinit: false, lock_acquired);
11156	}
11157
11158	/**
11159	* i40e_handle_mdd_event
11160	* @pf: pointer to the PF structure
11161	*
11162	* Called from the MDD irq handler to identify possibly malicious vfs
11163	**/
11164	static void i40e_handle_mdd_event(struct i40e_pf *pf)
11165	{
11166	struct i40e_hw *hw = &pf->hw;
11167	bool mdd_detected = false;
11168	struct i40e_vf *vf;
11169	u32 reg;
11170	int i;
11171
11172	if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
11173	return;
11174
11175	/* find what triggered the MDD event */
11176	reg = rd32(hw, I40E_GL_MDET_TX);
11177	if (reg & I40E_GL_MDET_TX_VALID_MASK) {
11178	u8 pf_num = FIELD_GET(I40E_GL_MDET_TX_PF_NUM_MASK, reg);
11179	u16 vf_num = FIELD_GET(I40E_GL_MDET_TX_VF_NUM_MASK, reg);
11180	u8 event = FIELD_GET(I40E_GL_MDET_TX_EVENT_MASK, reg);
11181	u16 queue = FIELD_GET(I40E_GL_MDET_TX_QUEUE_MASK, reg) -
11182	pf->hw.func_caps.base_queue;
11183	if (netif_msg_tx_err(pf))
11184	dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
11185	event, queue, pf_num, vf_num);
11186	wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
11187	mdd_detected = true;
11188	}
11189	reg = rd32(hw, I40E_GL_MDET_RX);
11190	if (reg & I40E_GL_MDET_RX_VALID_MASK) {
11191	u8 func = FIELD_GET(I40E_GL_MDET_RX_FUNCTION_MASK, reg);
11192	u8 event = FIELD_GET(I40E_GL_MDET_RX_EVENT_MASK, reg);
11193	u16 queue = FIELD_GET(I40E_GL_MDET_RX_QUEUE_MASK, reg) -
11194	pf->hw.func_caps.base_queue;
11195	if (netif_msg_rx_err(pf))
11196	dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
11197	event, queue, func);
11198	wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
11199	mdd_detected = true;
11200	}
11201
11202	if (mdd_detected) {
11203	reg = rd32(hw, I40E_PF_MDET_TX);
11204	if (reg & I40E_PF_MDET_TX_VALID_MASK) {
11205	wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
11206	dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
11207	}
11208	reg = rd32(hw, I40E_PF_MDET_RX);
11209	if (reg & I40E_PF_MDET_RX_VALID_MASK) {
11210	wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
11211	dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
11212	}
11213	}
11214
11215	/* see if one of the VFs needs its hand slapped */
11216	for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
11217	vf = &(pf->vf[i]);
11218	reg = rd32(hw, I40E_VP_MDET_TX(i));
11219	if (reg & I40E_VP_MDET_TX_VALID_MASK) {
11220	wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
11221	vf->num_mdd_events++;
11222	dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
11223	i);
11224	dev_info(&pf->pdev->dev,
11225	"Use PF Control I/F to re-enable the VF\n");
11226	set_bit(nr: I40E_VF_STATE_DISABLED, addr: &vf->vf_states);
11227	}
11228
11229	reg = rd32(hw, I40E_VP_MDET_RX(i));
11230	if (reg & I40E_VP_MDET_RX_VALID_MASK) {
11231	wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
11232	vf->num_mdd_events++;
11233	dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
11234	i);
11235	dev_info(&pf->pdev->dev,
11236	"Use PF Control I/F to re-enable the VF\n");
11237	set_bit(nr: I40E_VF_STATE_DISABLED, addr: &vf->vf_states);
11238	}
11239	}
11240
11241	/* re-enable mdd interrupt cause */
11242	clear_bit(nr: __I40E_MDD_EVENT_PENDING, addr: pf->state);
11243	reg = rd32(hw, I40E_PFINT_ICR0_ENA);
11244	reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
11245	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
11246	i40e_flush(hw);
11247	}
11248
11249	/**
11250	* i40e_service_task - Run the driver's async subtasks
11251	* @work: pointer to work_struct containing our data
11252	**/
11253	static void i40e_service_task(struct work_struct *work)
11254	{
11255	struct i40e_pf *pf = container_of(work,
11256	struct i40e_pf,
11257	service_task);
11258	unsigned long start_time = jiffies;
11259
11260	/* don't bother with service tasks if a reset is in progress */
11261	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
11262	test_bit(__I40E_SUSPENDED, pf->state))
11263	return;
11264
11265	if (test_and_set_bit(nr: __I40E_SERVICE_SCHED, addr: pf->state))
11266	return;
11267
11268	if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
11269	i40e_detect_recover_hung(vsi: pf->vsi[pf->lan_vsi]);
11270	i40e_sync_filters_subtask(pf);
11271	i40e_reset_subtask(pf);
11272	i40e_handle_mdd_event(pf);
11273	i40e_vc_process_vflr_event(pf);
11274	i40e_watchdog_subtask(pf);
11275	i40e_fdir_reinit_subtask(pf);
11276	if (test_and_clear_bit(nr: __I40E_CLIENT_RESET, addr: pf->state)) {
11277	/* Client subtask will reopen next time through. */
11278	i40e_notify_client_of_netdev_close(vsi: pf->vsi[pf->lan_vsi],
11279	reset: true);
11280	} else {
11281	i40e_client_subtask(pf);
11282	if (test_and_clear_bit(nr: __I40E_CLIENT_L2_CHANGE,
11283	addr: pf->state))
11284	i40e_notify_client_of_l2_param_changes(
11285	vsi: pf->vsi[pf->lan_vsi]);
11286	}
11287	i40e_sync_filters_subtask(pf);
11288	} else {
11289	i40e_reset_subtask(pf);
11290	}
11291
11292	i40e_clean_adminq_subtask(pf);
11293
11294	/* flush memory to make sure state is correct before next watchdog */
11295	smp_mb__before_atomic();
11296	clear_bit(nr: __I40E_SERVICE_SCHED, addr: pf->state);
11297
11298	/* If the tasks have taken longer than one timer cycle or there
11299	* is more work to be done, reschedule the service task now
11300	* rather than wait for the timer to tick again.
11301	*/
11302	if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
11303	test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) ||
11304	test_bit(__I40E_MDD_EVENT_PENDING, pf->state) ||
11305	test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
11306	i40e_service_event_schedule(pf);
11307	}
11308
11309	/**
11310	* i40e_service_timer - timer callback
11311	* @t: timer list pointer
11312	**/
11313	static void i40e_service_timer(struct timer_list *t)
11314	{
11315	struct i40e_pf *pf = from_timer(pf, t, service_timer);
11316
11317	mod_timer(timer: &pf->service_timer,
11318	expires: round_jiffies(j: jiffies + pf->service_timer_period));
11319	i40e_service_event_schedule(pf);
11320	}
11321
11322	/**
11323	* i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
11324	* @vsi: the VSI being configured
11325	**/
11326	static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
11327	{
11328	struct i40e_pf *pf = vsi->back;
11329
11330	switch (vsi->type) {
11331	case I40E_VSI_MAIN:
11332	vsi->alloc_queue_pairs = pf->num_lan_qps;
11333	if (!vsi->num_tx_desc)
11334	vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11335	I40E_REQ_DESCRIPTOR_MULTIPLE);
11336	if (!vsi->num_rx_desc)
11337	vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11338	I40E_REQ_DESCRIPTOR_MULTIPLE);
11339	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
11340	vsi->num_q_vectors = pf->num_lan_msix;
11341	else
11342	vsi->num_q_vectors = 1;
11343
11344	break;
11345
11346	case I40E_VSI_FDIR:
11347	vsi->alloc_queue_pairs = 1;
11348	vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11349	I40E_REQ_DESCRIPTOR_MULTIPLE);
11350	vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11351	I40E_REQ_DESCRIPTOR_MULTIPLE);
11352	vsi->num_q_vectors = pf->num_fdsb_msix;
11353	break;
11354
11355	case I40E_VSI_VMDQ2:
11356	vsi->alloc_queue_pairs = pf->num_vmdq_qps;
11357	if (!vsi->num_tx_desc)
11358	vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11359	I40E_REQ_DESCRIPTOR_MULTIPLE);
11360	if (!vsi->num_rx_desc)
11361	vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11362	I40E_REQ_DESCRIPTOR_MULTIPLE);
11363	vsi->num_q_vectors = pf->num_vmdq_msix;
11364	break;
11365
11366	case I40E_VSI_SRIOV:
11367	vsi->alloc_queue_pairs = pf->num_vf_qps;
11368	if (!vsi->num_tx_desc)
11369	vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11370	I40E_REQ_DESCRIPTOR_MULTIPLE);
11371	if (!vsi->num_rx_desc)
11372	vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11373	I40E_REQ_DESCRIPTOR_MULTIPLE);
11374	break;
11375
11376	default:
11377	WARN_ON(1);
11378	return -ENODATA;
11379	}
11380
11381	if (is_kdump_kernel()) {
11382	vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS;
11383	vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS;
11384	}
11385
11386	return 0;
11387	}
11388
11389	/**
11390	* i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
11391	* @vsi: VSI pointer
11392	* @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
11393	*
11394	* On error: returns error code (negative)
11395	* On success: returns 0
11396	**/
11397	static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
11398	{
11399	struct i40e_ring **next_rings;
11400	int size;
11401	int ret = 0;
11402
11403	/* allocate memory for both Tx, XDP Tx and Rx ring pointers */
11404	size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
11405	(i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
11406	vsi->tx_rings = kzalloc(size, GFP_KERNEL);
11407	if (!vsi->tx_rings)
11408	return -ENOMEM;
11409	next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
11410	if (i40e_enabled_xdp_vsi(vsi)) {
11411	vsi->xdp_rings = next_rings;
11412	next_rings += vsi->alloc_queue_pairs;
11413	}
11414	vsi->rx_rings = next_rings;
11415
11416	if (alloc_qvectors) {
11417	/* allocate memory for q_vector pointers */
11418	size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
11419	vsi->q_vectors = kzalloc(size, GFP_KERNEL);
11420	if (!vsi->q_vectors) {
11421	ret = -ENOMEM;
11422	goto err_vectors;
11423	}
11424	}
11425	return ret;
11426
11427	err_vectors:
11428	kfree(objp: vsi->tx_rings);
11429	return ret;
11430	}
11431
11432	/**
11433	* i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
11434	* @pf: board private structure
11435	* @type: type of VSI
11436	*
11437	* On error: returns error code (negative)
11438	* On success: returns vsi index in PF (positive)
11439	**/
11440	static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
11441	{
11442	int ret = -ENODEV;
11443	struct i40e_vsi *vsi;
11444	int vsi_idx;
11445	int i;
11446
11447	/* Need to protect the allocation of the VSIs at the PF level */
11448	mutex_lock(&pf->switch_mutex);
11449
11450	/* VSI list may be fragmented if VSI creation/destruction has
11451	* been happening. We can afford to do a quick scan to look
11452	* for any free VSIs in the list.
11453	*
11454	* find next empty vsi slot, looping back around if necessary
11455	*/
11456	i = pf->next_vsi;
11457	while (i < pf->num_alloc_vsi && pf->vsi[i])
11458	i++;
11459	if (i >= pf->num_alloc_vsi) {
11460	i = 0;
11461	while (i < pf->next_vsi && pf->vsi[i])
11462	i++;
11463	}
11464
11465	if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
11466	vsi_idx = i; /* Found one! */
11467	} else {
11468	ret = -ENODEV;
11469	goto unlock_pf; /* out of VSI slots! */
11470	}
11471	pf->next_vsi = ++i;
11472
11473	vsi = kzalloc(size: sizeof(*vsi), GFP_KERNEL);
11474	if (!vsi) {
11475	ret = -ENOMEM;
11476	goto unlock_pf;
11477	}
11478	vsi->type = type;
11479	vsi->back = pf;
11480	set_bit(nr: __I40E_VSI_DOWN, addr: vsi->state);
11481	vsi->flags = 0;
11482	vsi->idx = vsi_idx;
11483	vsi->int_rate_limit = 0;
11484	vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
11485	pf->rss_table_size : 64;
11486	vsi->netdev_registered = false;
11487	vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
11488	hash_init(vsi->mac_filter_hash);
11489	vsi->irqs_ready = false;
11490
11491	if (type == I40E_VSI_MAIN) {
11492	vsi->af_xdp_zc_qps = bitmap_zalloc(nbits: pf->num_lan_qps, GFP_KERNEL);
11493	if (!vsi->af_xdp_zc_qps)
11494	goto err_rings;
11495	}
11496
11497	ret = i40e_set_num_rings_in_vsi(vsi);
11498	if (ret)
11499	goto err_rings;
11500
11501	ret = i40e_vsi_alloc_arrays(vsi, alloc_qvectors: true);
11502	if (ret)
11503	goto err_rings;
11504
11505	/* Setup default MSIX irq handler for VSI */
11506	i40e_vsi_setup_irqhandler(vsi, irq_handler: i40e_msix_clean_rings);
11507
11508	/* Initialize VSI lock */
11509	spin_lock_init(&vsi->mac_filter_hash_lock);
11510	pf->vsi[vsi_idx] = vsi;
11511	ret = vsi_idx;
11512	goto unlock_pf;
11513
11514	err_rings:
11515	bitmap_free(bitmap: vsi->af_xdp_zc_qps);
11516	pf->next_vsi = i - 1;
11517	kfree(objp: vsi);
11518	unlock_pf:
11519	mutex_unlock(lock: &pf->switch_mutex);
11520	return ret;
11521	}
11522
11523	/**
11524	* i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
11525	* @vsi: VSI pointer
11526	* @free_qvectors: a bool to specify if q_vectors need to be freed.
11527	*
11528	* On error: returns error code (negative)
11529	* On success: returns 0
11530	**/
11531	static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
11532	{
11533	/* free the ring and vector containers */
11534	if (free_qvectors) {
11535	kfree(objp: vsi->q_vectors);
11536	vsi->q_vectors = NULL;
11537	}
11538	kfree(objp: vsi->tx_rings);
11539	vsi->tx_rings = NULL;
11540	vsi->rx_rings = NULL;
11541	vsi->xdp_rings = NULL;
11542	}
11543
11544	/**
11545	* i40e_clear_rss_config_user - clear the user configured RSS hash keys
11546	* and lookup table
11547	* @vsi: Pointer to VSI structure
11548	*/
11549	static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
11550	{
11551	if (!vsi)
11552	return;
11553
11554	kfree(objp: vsi->rss_hkey_user);
11555	vsi->rss_hkey_user = NULL;
11556
11557	kfree(objp: vsi->rss_lut_user);
11558	vsi->rss_lut_user = NULL;
11559	}
11560
11561	/**
11562	* i40e_vsi_clear - Deallocate the VSI provided
11563	* @vsi: the VSI being un-configured
11564	**/
11565	static int i40e_vsi_clear(struct i40e_vsi *vsi)
11566	{
11567	struct i40e_pf *pf;
11568
11569	if (!vsi)
11570	return 0;
11571
11572	if (!vsi->back)
11573	goto free_vsi;
11574	pf = vsi->back;
11575
11576	mutex_lock(&pf->switch_mutex);
11577	if (!pf->vsi[vsi->idx]) {
11578	dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
11579	vsi->idx, vsi->idx, vsi->type);
11580	goto unlock_vsi;
11581	}
11582
11583	if (pf->vsi[vsi->idx] != vsi) {
11584	dev_err(&pf->pdev->dev,
11585	"pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
11586	pf->vsi[vsi->idx]->idx,
11587	pf->vsi[vsi->idx]->type,
11588	vsi->idx, vsi->type);
11589	goto unlock_vsi;
11590	}
11591
11592	/* updates the PF for this cleared vsi */
11593	i40e_put_lump(pile: pf->qp_pile, index: vsi->base_queue, id: vsi->idx);
11594	i40e_put_lump(pile: pf->irq_pile, index: vsi->base_vector, id: vsi->idx);
11595
11596	bitmap_free(bitmap: vsi->af_xdp_zc_qps);
11597	i40e_vsi_free_arrays(vsi, free_qvectors: true);
11598	i40e_clear_rss_config_user(vsi);
11599
11600	pf->vsi[vsi->idx] = NULL;
11601	if (vsi->idx < pf->next_vsi)
11602	pf->next_vsi = vsi->idx;
11603
11604	unlock_vsi:
11605	mutex_unlock(lock: &pf->switch_mutex);
11606	free_vsi:
11607	kfree(objp: vsi);
11608
11609	return 0;
11610	}
11611
11612	/**
11613	* i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
11614	* @vsi: the VSI being cleaned
11615	**/
11616	static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
11617	{
11618	int i;
11619
11620	if (vsi->tx_rings && vsi->tx_rings[0]) {
11621	for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11622	kfree_rcu(vsi->tx_rings[i], rcu);
11623	WRITE_ONCE(vsi->tx_rings[i], NULL);
11624	WRITE_ONCE(vsi->rx_rings[i], NULL);
11625	if (vsi->xdp_rings)
11626	WRITE_ONCE(vsi->xdp_rings[i], NULL);
11627	}
11628	}
11629	}
11630
11631	/**
11632	* i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
11633	* @vsi: the VSI being configured
11634	**/
11635	static int i40e_alloc_rings(struct i40e_vsi *vsi)
11636	{
11637	int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
11638	struct i40e_pf *pf = vsi->back;
11639	struct i40e_ring *ring;
11640
11641	/* Set basic values in the rings to be used later during open() */
11642	for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11643	/* allocate space for both Tx and Rx in one shot */
11644	ring = kcalloc(n: qpv, size: sizeof(struct i40e_ring), GFP_KERNEL);
11645	if (!ring)
11646	goto err_out;
11647
11648	ring->queue_index = i;
11649	ring->reg_idx = vsi->base_queue + i;
11650	ring->ring_active = false;
11651	ring->vsi = vsi;
11652	ring->netdev = vsi->netdev;
11653	ring->dev = &pf->pdev->dev;
11654	ring->count = vsi->num_tx_desc;
11655	ring->size = 0;
11656	ring->dcb_tc = 0;
11657	if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))
11658	ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11659	ring->itr_setting = pf->tx_itr_default;
11660	WRITE_ONCE(vsi->tx_rings[i], ring++);
11661
11662	if (!i40e_enabled_xdp_vsi(vsi))
11663	goto setup_rx;
11664
11665	ring->queue_index = vsi->alloc_queue_pairs + i;
11666	ring->reg_idx = vsi->base_queue + ring->queue_index;
11667	ring->ring_active = false;
11668	ring->vsi = vsi;
11669	ring->netdev = NULL;
11670	ring->dev = &pf->pdev->dev;
11671	ring->count = vsi->num_tx_desc;
11672	ring->size = 0;
11673	ring->dcb_tc = 0;
11674	if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))
11675	ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11676	set_ring_xdp(ring);
11677	ring->itr_setting = pf->tx_itr_default;
11678	WRITE_ONCE(vsi->xdp_rings[i], ring++);
11679
11680	setup_rx:
11681	ring->queue_index = i;
11682	ring->reg_idx = vsi->base_queue + i;
11683	ring->ring_active = false;
11684	ring->vsi = vsi;
11685	ring->netdev = vsi->netdev;
11686	ring->dev = &pf->pdev->dev;
11687	ring->count = vsi->num_rx_desc;
11688	ring->size = 0;
11689	ring->dcb_tc = 0;
11690	ring->itr_setting = pf->rx_itr_default;
11691	WRITE_ONCE(vsi->rx_rings[i], ring);
11692	}
11693
11694	return 0;
11695
11696	err_out:
11697	i40e_vsi_clear_rings(vsi);
11698	return -ENOMEM;
11699	}
11700
11701	/**
11702	* i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
11703	* @pf: board private structure
11704	* @vectors: the number of MSI-X vectors to request
11705	*
11706	* Returns the number of vectors reserved, or error
11707	**/
11708	static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
11709	{
11710	vectors = pci_enable_msix_range(dev: pf->pdev, entries: pf->msix_entries,
11711	I40E_MIN_MSIX, maxvec: vectors);
11712	if (vectors < 0) {
11713	dev_info(&pf->pdev->dev,
11714	"MSI-X vector reservation failed: %d\n", vectors);
11715	vectors = 0;
11716	}
11717
11718	return vectors;
11719	}
11720
11721	/**
11722	* i40e_init_msix - Setup the MSIX capability
11723	* @pf: board private structure
11724	*
11725	* Work with the OS to set up the MSIX vectors needed.
11726	*
11727	* Returns the number of vectors reserved or negative on failure
11728	**/
11729	static int i40e_init_msix(struct i40e_pf *pf)
11730	{
11731	struct i40e_hw *hw = &pf->hw;
11732	int cpus, extra_vectors;
11733	int vectors_left;
11734	int v_budget, i;
11735	int v_actual;
11736	int iwarp_requested = 0;
11737
11738	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
11739	return -ENODEV;
11740
11741	/* The number of vectors we'll request will be comprised of:
11742	* - Add 1 for "other" cause for Admin Queue events, etc.
11743	* - The number of LAN queue pairs
11744	* - Queues being used for RSS.
11745	* We don't need as many as max_rss_size vectors.
11746	* use rss_size instead in the calculation since that
11747	* is governed by number of cpus in the system.
11748	* - assumes symmetric Tx/Rx pairing
11749	* - The number of VMDq pairs
11750	* - The CPU count within the NUMA node if iWARP is enabled
11751	* Once we count this up, try the request.
11752	*
11753	* If we can't get what we want, we'll simplify to nearly nothing
11754	* and try again. If that still fails, we punt.
11755	*/
11756	vectors_left = hw->func_caps.num_msix_vectors;
11757	v_budget = 0;
11758
11759	/* reserve one vector for miscellaneous handler */
11760	if (vectors_left) {
11761	v_budget++;
11762	vectors_left--;
11763	}
11764
11765	/* reserve some vectors for the main PF traffic queues. Initially we
11766	* only reserve at most 50% of the available vectors, in the case that
11767	* the number of online CPUs is large. This ensures that we can enable
11768	* extra features as well. Once we've enabled the other features, we
11769	* will use any remaining vectors to reach as close as we can to the
11770	* number of online CPUs.
11771	*/
11772	cpus = num_online_cpus();
11773	pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
11774	vectors_left -= pf->num_lan_msix;
11775
11776	/* reserve one vector for sideband flow director */
11777	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
11778	if (vectors_left) {
11779	pf->num_fdsb_msix = 1;
11780	v_budget++;
11781	vectors_left--;
11782	} else {
11783	pf->num_fdsb_msix = 0;
11784	}
11785	}
11786
11787	/* can we reserve enough for iWARP? */
11788	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11789	iwarp_requested = pf->num_iwarp_msix;
11790
11791	if (!vectors_left)
11792	pf->num_iwarp_msix = 0;
11793	else if (vectors_left < pf->num_iwarp_msix)
11794	pf->num_iwarp_msix = 1;
11795	v_budget += pf->num_iwarp_msix;
11796	vectors_left -= pf->num_iwarp_msix;
11797	}
11798
11799	/* any vectors left over go for VMDq support */
11800	if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags)) {
11801	if (!vectors_left) {
11802	pf->num_vmdq_msix = 0;
11803	pf->num_vmdq_qps = 0;
11804	} else {
11805	int vmdq_vecs_wanted =
11806	pf->num_vmdq_vsis * pf->num_vmdq_qps;
11807	int vmdq_vecs =
11808	min_t(int, vectors_left, vmdq_vecs_wanted);
11809
11810	/* if we're short on vectors for what's desired, we limit
11811	* the queues per vmdq. If this is still more than are
11812	* available, the user will need to change the number of
11813	* queues/vectors used by the PF later with the ethtool
11814	* channels command
11815	*/
11816	if (vectors_left < vmdq_vecs_wanted) {
11817	pf->num_vmdq_qps = 1;
11818	vmdq_vecs_wanted = pf->num_vmdq_vsis;
11819	vmdq_vecs = min_t(int,
11820	vectors_left,
11821	vmdq_vecs_wanted);
11822	}
11823	pf->num_vmdq_msix = pf->num_vmdq_qps;
11824
11825	v_budget += vmdq_vecs;
11826	vectors_left -= vmdq_vecs;
11827	}
11828	}
11829
11830	/* On systems with a large number of SMP cores, we previously limited
11831	* the number of vectors for num_lan_msix to be at most 50% of the
11832	* available vectors, to allow for other features. Now, we add back
11833	* the remaining vectors. However, we ensure that the total
11834	* num_lan_msix will not exceed num_online_cpus(). To do this, we
11835	* calculate the number of vectors we can add without going over the
11836	* cap of CPUs. For systems with a small number of CPUs this will be
11837	* zero.
11838	*/
11839	extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11840	pf->num_lan_msix += extra_vectors;
11841	vectors_left -= extra_vectors;
11842
11843	WARN(vectors_left < 0,
11844	"Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11845
11846	v_budget += pf->num_lan_msix;
11847	pf->msix_entries = kcalloc(n: v_budget, size: sizeof(struct msix_entry),
11848	GFP_KERNEL);
11849	if (!pf->msix_entries)
11850	return -ENOMEM;
11851
11852	for (i = 0; i < v_budget; i++)
11853	pf->msix_entries[i].entry = i;
11854	v_actual = i40e_reserve_msix_vectors(pf, vectors: v_budget);
11855
11856	if (v_actual < I40E_MIN_MSIX) {
11857	clear_bit(nr: I40E_FLAG_MSIX_ENA, addr: pf->flags);
11858	kfree(objp: pf->msix_entries);
11859	pf->msix_entries = NULL;
11860	pci_disable_msix(dev: pf->pdev);
11861	return -ENODEV;
11862
11863	} else if (v_actual == I40E_MIN_MSIX) {
11864	/* Adjust for minimal MSIX use */
11865	pf->num_vmdq_vsis = 0;
11866	pf->num_vmdq_qps = 0;
11867	pf->num_lan_qps = 1;
11868	pf->num_lan_msix = 1;
11869
11870	} else if (v_actual != v_budget) {
11871	/* If we have limited resources, we will start with no vectors
11872	* for the special features and then allocate vectors to some
11873	* of these features based on the policy and at the end disable
11874	* the features that did not get any vectors.
11875	*/
11876	int vec;
11877
11878	dev_info(&pf->pdev->dev,
11879	"MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11880	v_actual, v_budget);
11881	/* reserve the misc vector */
11882	vec = v_actual - 1;
11883
11884	/* Scale vector usage down */
11885	pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */
11886	pf->num_vmdq_vsis = 1;
11887	pf->num_vmdq_qps = 1;
11888
11889	/* partition out the remaining vectors */
11890	switch (vec) {
11891	case 2:
11892	pf->num_lan_msix = 1;
11893	break;
11894	case 3:
11895	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11896	pf->num_lan_msix = 1;
11897	pf->num_iwarp_msix = 1;
11898	} else {
11899	pf->num_lan_msix = 2;
11900	}
11901	break;
11902	default:
11903	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11904	pf->num_iwarp_msix = min_t(int, (vec / 3),
11905	iwarp_requested);
11906	pf->num_vmdq_vsis = min_t(int, (vec / 3),
11907	I40E_DEFAULT_NUM_VMDQ_VSI);
11908	} else {
11909	pf->num_vmdq_vsis = min_t(int, (vec / 2),
11910	I40E_DEFAULT_NUM_VMDQ_VSI);
11911	}
11912	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
11913	pf->num_fdsb_msix = 1;
11914	vec--;
11915	}
11916	pf->num_lan_msix = min_t(int,
11917	(vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
11918	pf->num_lan_msix);
11919	pf->num_lan_qps = pf->num_lan_msix;
11920	break;
11921	}
11922	}
11923
11924	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && pf->num_fdsb_msix == 0) {
11925	dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
11926	clear_bit(nr: I40E_FLAG_FD_SB_ENA, addr: pf->flags);
11927	set_bit(nr: I40E_FLAG_FD_SB_INACTIVE, addr: pf->flags);
11928	}
11929	if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && pf->num_vmdq_msix == 0) {
11930	dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
11931	clear_bit(nr: I40E_FLAG_VMDQ_ENA, addr: pf->flags);
11932	}
11933
11934	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&
11935	pf->num_iwarp_msix == 0) {
11936	dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
11937	clear_bit(nr: I40E_FLAG_IWARP_ENA, addr: pf->flags);
11938	}
11939	i40e_debug(&pf->hw, I40E_DEBUG_INIT,
11940	"MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
11941	pf->num_lan_msix,
11942	pf->num_vmdq_msix * pf->num_vmdq_vsis,
11943	pf->num_fdsb_msix,
11944	pf->num_iwarp_msix);
11945
11946	return v_actual;
11947	}
11948
11949	/**
11950	* i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
11951	* @vsi: the VSI being configured
11952	* @v_idx: index of the vector in the vsi struct
11953	*
11954	* We allocate one q_vector. If allocation fails we return -ENOMEM.
11955	**/
11956	static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
11957	{
11958	struct i40e_q_vector *q_vector;
11959
11960	/* allocate q_vector */
11961	q_vector = kzalloc(size: sizeof(struct i40e_q_vector), GFP_KERNEL);
11962	if (!q_vector)
11963	return -ENOMEM;
11964
11965	q_vector->vsi = vsi;
11966	q_vector->v_idx = v_idx;
11967	cpumask_copy(dstp: &q_vector->affinity_mask, cpu_possible_mask);
11968
11969	if (vsi->netdev)
11970	netif_napi_add(dev: vsi->netdev, napi: &q_vector->napi, poll: i40e_napi_poll);
11971
11972	/* tie q_vector and vsi together */
11973	vsi->q_vectors[v_idx] = q_vector;
11974
11975	return 0;
11976	}
11977
11978	/**
11979	* i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
11980	* @vsi: the VSI being configured
11981	*
11982	* We allocate one q_vector per queue interrupt. If allocation fails we
11983	* return -ENOMEM.
11984	**/
11985	static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
11986	{
11987	struct i40e_pf *pf = vsi->back;
11988	int err, v_idx, num_q_vectors;
11989
11990	/* if not MSIX, give the one vector only to the LAN VSI */
11991	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
11992	num_q_vectors = vsi->num_q_vectors;
11993	else if (vsi == pf->vsi[pf->lan_vsi])
11994	num_q_vectors = 1;
11995	else
11996	return -EINVAL;
11997
11998	for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
11999	err = i40e_vsi_alloc_q_vector(vsi, v_idx);
12000	if (err)
12001	goto err_out;
12002	}
12003
12004	return 0;
12005
12006	err_out:
12007	while (v_idx--)
12008	i40e_free_q_vector(vsi, v_idx);
12009
12010	return err;
12011	}
12012
12013	/**
12014	* i40e_init_interrupt_scheme - Determine proper interrupt scheme
12015	* @pf: board private structure to initialize
12016	**/
12017	static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
12018	{
12019	int vectors = 0;
12020	ssize_t size;
12021
12022	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
12023	vectors = i40e_init_msix(pf);
12024	if (vectors < 0) {
12025	clear_bit(nr: I40E_FLAG_MSIX_ENA, addr: pf->flags);
12026	clear_bit(nr: I40E_FLAG_IWARP_ENA, addr: pf->flags);
12027	clear_bit(nr: I40E_FLAG_RSS_ENA, addr: pf->flags);
12028	clear_bit(nr: I40E_FLAG_DCB_CAPABLE, addr: pf->flags);
12029	clear_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
12030	clear_bit(nr: I40E_FLAG_SRIOV_ENA, addr: pf->flags);
12031	clear_bit(nr: I40E_FLAG_FD_SB_ENA, addr: pf->flags);
12032	clear_bit(nr: I40E_FLAG_FD_ATR_ENA, addr: pf->flags);
12033	clear_bit(nr: I40E_FLAG_VMDQ_ENA, addr: pf->flags);
12034	set_bit(nr: I40E_FLAG_FD_SB_INACTIVE, addr: pf->flags);
12035
12036	/* rework the queue expectations without MSIX */
12037	i40e_determine_queue_usage(pf);
12038	}
12039	}
12040
12041	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
12042	test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {
12043	dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
12044	vectors = pci_enable_msi(dev: pf->pdev);
12045	if (vectors < 0) {
12046	dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
12047	vectors);
12048	clear_bit(nr: I40E_FLAG_MSI_ENA, addr: pf->flags);
12049	}
12050	vectors = 1; /* one MSI or Legacy vector */
12051	}
12052
12053	if (!test_bit(I40E_FLAG_MSI_ENA, pf->flags) &&
12054	!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
12055	dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
12056
12057	/* set up vector assignment tracking */
12058	size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
12059	pf->irq_pile = kzalloc(size, GFP_KERNEL);
12060	if (!pf->irq_pile)
12061	return -ENOMEM;
12062
12063	pf->irq_pile->num_entries = vectors;
12064
12065	/* track first vector for misc interrupts, ignore return */
12066	(void)i40e_get_lump(pf, pile: pf->irq_pile, needed: 1, I40E_PILE_VALID_BIT - 1);
12067
12068	return 0;
12069	}
12070
12071	/**
12072	* i40e_restore_interrupt_scheme - Restore the interrupt scheme
12073	* @pf: private board data structure
12074	*
12075	* Restore the interrupt scheme that was cleared when we suspended the
12076	* device. This should be called during resume to re-allocate the q_vectors
12077	* and reacquire IRQs.
12078	*/
12079	static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
12080	{
12081	struct i40e_vsi *vsi;
12082	int err, i;
12083
12084	/* We cleared the MSI and MSI-X flags when disabling the old interrupt
12085	* scheme. We need to re-enabled them here in order to attempt to
12086	* re-acquire the MSI or MSI-X vectors
12087	*/
12088	set_bit(nr: I40E_FLAG_MSI_ENA, addr: pf->flags);
12089	set_bit(nr: I40E_FLAG_MSIX_ENA, addr: pf->flags);
12090
12091	err = i40e_init_interrupt_scheme(pf);
12092	if (err)
12093	return err;
12094
12095	/* Now that we've re-acquired IRQs, we need to remap the vectors and
12096	* rings together again.
12097	*/
12098	i40e_pf_for_each_vsi(pf, i, vsi) {
12099	err = i40e_vsi_alloc_q_vectors(vsi);
12100	if (err)
12101	goto err_unwind;
12102
12103	i40e_vsi_map_rings_to_vectors(vsi);
12104	}
12105
12106	err = i40e_setup_misc_vector(pf);
12107	if (err)
12108	goto err_unwind;
12109
12110	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))
12111	i40e_client_update_msix_info(pf);
12112
12113	return 0;
12114
12115	err_unwind:
12116	while (i--) {
12117	if (pf->vsi[i])
12118	i40e_vsi_free_q_vectors(vsi: pf->vsi[i]);
12119	}
12120
12121	return err;
12122	}
12123
12124	/**
12125	* i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
12126	* non queue events in recovery mode
12127	* @pf: board private structure
12128	*
12129	* This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage
12130	* the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
12131	* This is handled differently than in recovery mode since no Tx/Rx resources
12132	* are being allocated.
12133	**/
12134	static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
12135	{
12136	int err;
12137
12138	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
12139	err = i40e_setup_misc_vector(pf);
12140
12141	if (err) {
12142	dev_info(&pf->pdev->dev,
12143	"MSI-X misc vector request failed, error %d\n",
12144	err);
12145	return err;
12146	}
12147	} else {
12148	u32 flags = test_bit(I40E_FLAG_MSI_ENA, pf->flags) ? 0 : IRQF_SHARED;
12149
12150	err = request_irq(irq: pf->pdev->irq, handler: i40e_intr, flags,
12151	name: pf->int_name, dev: pf);
12152
12153	if (err) {
12154	dev_info(&pf->pdev->dev,
12155	"MSI/legacy misc vector request failed, error %d\n",
12156	err);
12157	return err;
12158	}
12159	i40e_enable_misc_int_causes(pf);
12160	i40e_irq_dynamic_enable_icr0(pf);
12161	}
12162
12163	return 0;
12164	}
12165
12166	/**
12167	* i40e_setup_misc_vector - Setup the misc vector to handle non queue events
12168	* @pf: board private structure
12169	*
12170	* This sets up the handler for MSIX 0, which is used to manage the
12171	* non-queue interrupts, e.g. AdminQ and errors. This is not used
12172	* when in MSI or Legacy interrupt mode.
12173	**/
12174	static int i40e_setup_misc_vector(struct i40e_pf *pf)
12175	{
12176	struct i40e_hw *hw = &pf->hw;
12177	int err = 0;
12178
12179	/* Only request the IRQ once, the first time through. */
12180	if (!test_and_set_bit(nr: __I40E_MISC_IRQ_REQUESTED, addr: pf->state)) {
12181	err = request_irq(irq: pf->msix_entries[0].vector,
12182	handler: i40e_intr, flags: 0, name: pf->int_name, dev: pf);
12183	if (err) {
12184	clear_bit(nr: __I40E_MISC_IRQ_REQUESTED, addr: pf->state);
12185	dev_info(&pf->pdev->dev,
12186	"request_irq for %s failed: %d\n",
12187	pf->int_name, err);
12188	return -EFAULT;
12189	}
12190	}
12191
12192	i40e_enable_misc_int_causes(pf);
12193
12194	/* associate no queues to the misc vector */
12195	wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
12196	wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
12197
12198	i40e_flush(hw);
12199
12200	i40e_irq_dynamic_enable_icr0(pf);
12201
12202	return err;
12203	}
12204
12205	/**
12206	* i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
12207	* @vsi: Pointer to vsi structure
12208	* @seed: Buffter to store the hash keys
12209	* @lut: Buffer to store the lookup table entries
12210	* @lut_size: Size of buffer to store the lookup table entries
12211	*
12212	* Return 0 on success, negative on failure
12213	*/
12214	static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
12215	u8 *lut, u16 lut_size)
12216	{
12217	struct i40e_pf *pf = vsi->back;
12218	struct i40e_hw *hw = &pf->hw;
12219	int ret = 0;
12220
12221	if (seed) {
12222	ret = i40e_aq_get_rss_key(hw, seid: vsi->id,
12223	key: (struct i40e_aqc_get_set_rss_key_data *)seed);
12224	if (ret) {
12225	dev_info(&pf->pdev->dev,
12226	"Cannot get RSS key, err %pe aq_err %s\n",
12227	ERR_PTR(ret),
12228	i40e_aq_str(&pf->hw,
12229	pf->hw.aq.asq_last_status));
12230	return ret;
12231	}
12232	}
12233
12234	if (lut) {
12235	bool pf_lut = vsi->type == I40E_VSI_MAIN;
12236
12237	ret = i40e_aq_get_rss_lut(hw, seid: vsi->id, pf_lut, lut, lut_size);
12238	if (ret) {
12239	dev_info(&pf->pdev->dev,
12240	"Cannot get RSS lut, err %pe aq_err %s\n",
12241	ERR_PTR(ret),
12242	i40e_aq_str(&pf->hw,
12243	pf->hw.aq.asq_last_status));
12244	return ret;
12245	}
12246	}
12247
12248	return ret;
12249	}
12250
12251	/**
12252	* i40e_config_rss_reg - Configure RSS keys and lut by writing registers
12253	* @vsi: Pointer to vsi structure
12254	* @seed: RSS hash seed
12255	* @lut: Lookup table
12256	* @lut_size: Lookup table size
12257	*
12258	* Returns 0 on success, negative on failure
12259	**/
12260	static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
12261	const u8 *lut, u16 lut_size)
12262	{
12263	struct i40e_pf *pf = vsi->back;
12264	struct i40e_hw *hw = &pf->hw;
12265	u16 vf_id = vsi->vf_id;
12266	u8 i;
12267
12268	/* Fill out hash function seed */
12269	if (seed) {
12270	u32 *seed_dw = (u32 *)seed;
12271
12272	if (vsi->type == I40E_VSI_MAIN) {
12273	for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12274	wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
12275	} else if (vsi->type == I40E_VSI_SRIOV) {
12276	for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
12277	wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
12278	} else {
12279	dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
12280	}
12281	}
12282
12283	if (lut) {
12284	u32 *lut_dw = (u32 *)lut;
12285
12286	if (vsi->type == I40E_VSI_MAIN) {
12287	if (lut_size != I40E_HLUT_ARRAY_SIZE)
12288	return -EINVAL;
12289	for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12290	wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
12291	} else if (vsi->type == I40E_VSI_SRIOV) {
12292	if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
12293	return -EINVAL;
12294	for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12295	wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
12296	} else {
12297	dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12298	}
12299	}
12300	i40e_flush(hw);
12301
12302	return 0;
12303	}
12304
12305	/**
12306	* i40e_get_rss_reg - Get the RSS keys and lut by reading registers
12307	* @vsi: Pointer to VSI structure
12308	* @seed: Buffer to store the keys
12309	* @lut: Buffer to store the lookup table entries
12310	* @lut_size: Size of buffer to store the lookup table entries
12311	*
12312	* Returns 0 on success, negative on failure
12313	*/
12314	static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
12315	u8 *lut, u16 lut_size)
12316	{
12317	struct i40e_pf *pf = vsi->back;
12318	struct i40e_hw *hw = &pf->hw;
12319	u16 i;
12320
12321	if (seed) {
12322	u32 *seed_dw = (u32 *)seed;
12323
12324	for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12325	seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
12326	}
12327	if (lut) {
12328	u32 *lut_dw = (u32 *)lut;
12329
12330	if (lut_size != I40E_HLUT_ARRAY_SIZE)
12331	return -EINVAL;
12332	for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12333	lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
12334	}
12335
12336	return 0;
12337	}
12338
12339	/**
12340	* i40e_config_rss - Configure RSS keys and lut
12341	* @vsi: Pointer to VSI structure
12342	* @seed: RSS hash seed
12343	* @lut: Lookup table
12344	* @lut_size: Lookup table size
12345	*
12346	* Returns 0 on success, negative on failure
12347	*/
12348	int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12349	{
12350	struct i40e_pf *pf = vsi->back;
12351
12352	if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
12353	return i40e_config_rss_aq(vsi, seed, lut, lut_size);
12354	else
12355	return i40e_config_rss_reg(vsi, seed, lut, lut_size);
12356	}
12357
12358	/**
12359	* i40e_get_rss - Get RSS keys and lut
12360	* @vsi: Pointer to VSI structure
12361	* @seed: Buffer to store the keys
12362	* @lut: Buffer to store the lookup table entries
12363	* @lut_size: Size of buffer to store the lookup table entries
12364	*
12365	* Returns 0 on success, negative on failure
12366	*/
12367	int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12368	{
12369	struct i40e_pf *pf = vsi->back;
12370
12371	if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
12372	return i40e_get_rss_aq(vsi, seed, lut, lut_size);
12373	else
12374	return i40e_get_rss_reg(vsi, seed, lut, lut_size);
12375	}
12376
12377	/**
12378	* i40e_fill_rss_lut - Fill the RSS lookup table with default values
12379	* @pf: Pointer to board private structure
12380	* @lut: Lookup table
12381	* @rss_table_size: Lookup table size
12382	* @rss_size: Range of queue number for hashing
12383	*/
12384	void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
12385	u16 rss_table_size, u16 rss_size)
12386	{
12387	u16 i;
12388
12389	for (i = 0; i < rss_table_size; i++)
12390	lut[i] = i % rss_size;
12391	}
12392
12393	/**
12394	* i40e_pf_config_rss - Prepare for RSS if used
12395	* @pf: board private structure
12396	**/
12397	static int i40e_pf_config_rss(struct i40e_pf *pf)
12398	{
12399	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12400	u8 seed[I40E_HKEY_ARRAY_SIZE];
12401	u8 *lut;
12402	struct i40e_hw *hw = &pf->hw;
12403	u32 reg_val;
12404	u64 hena;
12405	int ret;
12406
12407	/* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
12408	hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
12409	((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
12410	hena |= i40e_pf_get_default_rss_hena(pf);
12411
12412	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), reg_val: (u32)hena);
12413	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), reg_val: (u32)(hena >> 32));
12414
12415	/* Determine the RSS table size based on the hardware capabilities */
12416	reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
12417	reg_val = (pf->rss_table_size == 512) ?
12418	(reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
12419	(reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
12420	i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
12421
12422	/* Determine the RSS size of the VSI */
12423	if (!vsi->rss_size) {
12424	u16 qcount;
12425	/* If the firmware does something weird during VSI init, we
12426	* could end up with zero TCs. Check for that to avoid
12427	* divide-by-zero. It probably won't pass traffic, but it also
12428	* won't panic.
12429	*/
12430	qcount = vsi->num_queue_pairs /
12431	(vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
12432	vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12433	}
12434	if (!vsi->rss_size)
12435	return -EINVAL;
12436
12437	lut = kzalloc(size: vsi->rss_table_size, GFP_KERNEL);
12438	if (!lut)
12439	return -ENOMEM;
12440
12441	/* Use user configured lut if there is one, otherwise use default */
12442	if (vsi->rss_lut_user)
12443	memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
12444	else
12445	i40e_fill_rss_lut(pf, lut, rss_table_size: vsi->rss_table_size, rss_size: vsi->rss_size);
12446
12447	/* Use user configured hash key if there is one, otherwise
12448	* use default.
12449	*/
12450	if (vsi->rss_hkey_user)
12451	memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
12452	else
12453	netdev_rss_key_fill(buffer: (void *)seed, I40E_HKEY_ARRAY_SIZE);
12454	ret = i40e_config_rss(vsi, seed, lut, lut_size: vsi->rss_table_size);
12455	kfree(objp: lut);
12456
12457	return ret;
12458	}
12459
12460	/**
12461	* i40e_reconfig_rss_queues - change number of queues for rss and rebuild
12462	* @pf: board private structure
12463	* @queue_count: the requested queue count for rss.
12464	*
12465	* returns 0 if rss is not enabled, if enabled returns the final rss queue
12466	* count which may be different from the requested queue count.
12467	* Note: expects to be called while under rtnl_lock()
12468	**/
12469	int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
12470	{
12471	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12472	int new_rss_size;
12473
12474	if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags))
12475	return 0;
12476
12477	queue_count = min_t(int, queue_count, num_online_cpus());
12478	new_rss_size = min_t(int, queue_count, pf->rss_size_max);
12479
12480	if (queue_count != vsi->num_queue_pairs) {
12481	u16 qcount;
12482
12483	vsi->req_queue_pairs = queue_count;
12484	i40e_prep_for_reset(pf);
12485	if (test_bit(__I40E_IN_REMOVE, pf->state))
12486	return pf->alloc_rss_size;
12487
12488	pf->alloc_rss_size = new_rss_size;
12489
12490	i40e_reset_and_rebuild(pf, reinit: true, lock_acquired: true);
12491
12492	/* Discard the user configured hash keys and lut, if less
12493	* queues are enabled.
12494	*/
12495	if (queue_count < vsi->rss_size) {
12496	i40e_clear_rss_config_user(vsi);
12497	dev_dbg(&pf->pdev->dev,
12498	"discard user configured hash keys and lut\n");
12499	}
12500
12501	/* Reset vsi->rss_size, as number of enabled queues changed */
12502	qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
12503	vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12504
12505	i40e_pf_config_rss(pf);
12506	}
12507	dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n",
12508	vsi->req_queue_pairs, pf->rss_size_max);
12509	return pf->alloc_rss_size;
12510	}
12511
12512	/**
12513	* i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
12514	* @pf: board private structure
12515	**/
12516	int i40e_get_partition_bw_setting(struct i40e_pf *pf)
12517	{
12518	bool min_valid, max_valid;
12519	u32 max_bw, min_bw;
12520	int status;
12521
12522	status = i40e_read_bw_from_alt_ram(hw: &pf->hw, max_bw: &max_bw, min_bw: &min_bw,
12523	min_valid: &min_valid, max_valid: &max_valid);
12524
12525	if (!status) {
12526	if (min_valid)
12527	pf->min_bw = min_bw;
12528	if (max_valid)
12529	pf->max_bw = max_bw;
12530	}
12531
12532	return status;
12533	}
12534
12535	/**
12536	* i40e_set_partition_bw_setting - Set BW settings for this PF partition
12537	* @pf: board private structure
12538	**/
12539	int i40e_set_partition_bw_setting(struct i40e_pf *pf)
12540	{
12541	struct i40e_aqc_configure_partition_bw_data bw_data;
12542	int status;
12543
12544	memset(&bw_data, 0, sizeof(bw_data));
12545
12546	/* Set the valid bit for this PF */
12547	bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
12548	bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
12549	bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
12550
12551	/* Set the new bandwidths */
12552	status = i40e_aq_configure_partition_bw(hw: &pf->hw, bw_data: &bw_data, NULL);
12553
12554	return status;
12555	}
12556
12557	/**
12558	* i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
12559	* @pf: board private structure
12560	**/
12561	int i40e_commit_partition_bw_setting(struct i40e_pf *pf)
12562	{
12563	/* Commit temporary BW setting to permanent NVM image */
12564	enum i40e_admin_queue_err last_aq_status;
12565	u16 nvm_word;
12566	int ret;
12567
12568	if (pf->hw.partition_id != 1) {
12569	dev_info(&pf->pdev->dev,
12570	"Commit BW only works on partition 1! This is partition %d",
12571	pf->hw.partition_id);
12572	ret = -EOPNOTSUPP;
12573	goto bw_commit_out;
12574	}
12575
12576	/* Acquire NVM for read access */
12577	ret = i40e_acquire_nvm(hw: &pf->hw, access: I40E_RESOURCE_READ);
12578	last_aq_status = pf->hw.aq.asq_last_status;
12579	if (ret) {
12580	dev_info(&pf->pdev->dev,
12581	"Cannot acquire NVM for read access, err %pe aq_err %s\n",
12582	ERR_PTR(ret),
12583	i40e_aq_str(&pf->hw, last_aq_status));
12584	goto bw_commit_out;
12585	}
12586
12587	/* Read word 0x10 of NVM - SW compatibility word 1 */
12588	ret = i40e_aq_read_nvm(hw: &pf->hw,
12589	I40E_SR_NVM_CONTROL_WORD,
12590	offset: 0x10, length: sizeof(nvm_word), data: &nvm_word,
12591	last_command: false, NULL);
12592	/* Save off last admin queue command status before releasing
12593	* the NVM
12594	*/
12595	last_aq_status = pf->hw.aq.asq_last_status;
12596	i40e_release_nvm(hw: &pf->hw);
12597	if (ret) {
12598	dev_info(&pf->pdev->dev, "NVM read error, err %pe aq_err %s\n",
12599	ERR_PTR(ret),
12600	i40e_aq_str(&pf->hw, last_aq_status));
12601	goto bw_commit_out;
12602	}
12603
12604	/* Wait a bit for NVM release to complete */
12605	msleep(msecs: 50);
12606
12607	/* Acquire NVM for write access */
12608	ret = i40e_acquire_nvm(hw: &pf->hw, access: I40E_RESOURCE_WRITE);
12609	last_aq_status = pf->hw.aq.asq_last_status;
12610	if (ret) {
12611	dev_info(&pf->pdev->dev,
12612	"Cannot acquire NVM for write access, err %pe aq_err %s\n",
12613	ERR_PTR(ret),
12614	i40e_aq_str(&pf->hw, last_aq_status));
12615	goto bw_commit_out;
12616	}
12617	/* Write it back out unchanged to initiate update NVM,
12618	* which will force a write of the shadow (alt) RAM to
12619	* the NVM - thus storing the bandwidth values permanently.
12620	*/
12621	ret = i40e_aq_update_nvm(hw: &pf->hw,
12622	I40E_SR_NVM_CONTROL_WORD,
12623	offset: 0x10, length: sizeof(nvm_word),
12624	data: &nvm_word, last_command: true, preservation_flags: 0, NULL);
12625	/* Save off last admin queue command status before releasing
12626	* the NVM
12627	*/
12628	last_aq_status = pf->hw.aq.asq_last_status;
12629	i40e_release_nvm(hw: &pf->hw);
12630	if (ret)
12631	dev_info(&pf->pdev->dev,
12632	"BW settings NOT SAVED, err %pe aq_err %s\n",
12633	ERR_PTR(ret),
12634	i40e_aq_str(&pf->hw, last_aq_status));
12635	bw_commit_out:
12636
12637	return ret;
12638	}
12639
12640	/**
12641	* i40e_is_total_port_shutdown_enabled - read NVM and return value
12642	* if total port shutdown feature is enabled for this PF
12643	* @pf: board private structure
12644	**/
12645	static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf)
12646	{
12647	#define I40E_TOTAL_PORT_SHUTDOWN_ENABLED BIT(4)
12648	#define I40E_FEATURES_ENABLE_PTR 0x2A
12649	#define I40E_CURRENT_SETTING_PTR 0x2B
12650	#define I40E_LINK_BEHAVIOR_WORD_OFFSET 0x2D
12651	#define I40E_LINK_BEHAVIOR_WORD_LENGTH 0x1
12652	#define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED BIT(0)
12653	#define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH 4
12654	u16 sr_emp_sr_settings_ptr = 0;
12655	u16 features_enable = 0;
12656	u16 link_behavior = 0;
12657	int read_status = 0;
12658	bool ret = false;
12659
12660	read_status = i40e_read_nvm_word(hw: &pf->hw,
12661	I40E_SR_EMP_SR_SETTINGS_PTR,
12662	data: &sr_emp_sr_settings_ptr);
12663	if (read_status)
12664	goto err_nvm;
12665	read_status = i40e_read_nvm_word(hw: &pf->hw,
12666	offset: sr_emp_sr_settings_ptr +
12667	I40E_FEATURES_ENABLE_PTR,
12668	data: &features_enable);
12669	if (read_status)
12670	goto err_nvm;
12671	if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) {
12672	read_status = i40e_read_nvm_module_data(hw: &pf->hw,
12673	I40E_SR_EMP_SR_SETTINGS_PTR,
12674	I40E_CURRENT_SETTING_PTR,
12675	I40E_LINK_BEHAVIOR_WORD_OFFSET,
12676	I40E_LINK_BEHAVIOR_WORD_LENGTH,
12677	data_ptr: &link_behavior);
12678	if (read_status)
12679	goto err_nvm;
12680	link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);
12681	ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior;
12682	}
12683	return ret;
12684
12685	err_nvm:
12686	dev_warn(&pf->pdev->dev,
12687	"total-port-shutdown feature is off due to read nvm error: %pe\n",
12688	ERR_PTR(read_status));
12689	return ret;
12690	}
12691
12692	/**
12693	* i40e_sw_init - Initialize general software structures (struct i40e_pf)
12694	* @pf: board private structure to initialize
12695	*
12696	* i40e_sw_init initializes the Adapter private data structure.
12697	* Fields are initialized based on PCI device information and
12698	* OS network device settings (MTU size).
12699	**/
12700	static int i40e_sw_init(struct i40e_pf *pf)
12701	{
12702	int err = 0;
12703	int size;
12704	u16 pow;
12705
12706	/* Set default capability flags */
12707	bitmap_zero(dst: pf->flags, nbits: I40E_PF_FLAGS_NBITS);
12708	set_bit(nr: I40E_FLAG_MSI_ENA, addr: pf->flags);
12709	set_bit(nr: I40E_FLAG_MSIX_ENA, addr: pf->flags);
12710
12711	/* Set default ITR */
12712	pf->rx_itr_default = I40E_ITR_RX_DEF;
12713	pf->tx_itr_default = I40E_ITR_TX_DEF;
12714
12715	/* Depending on PF configurations, it is possible that the RSS
12716	* maximum might end up larger than the available queues
12717	*/
12718	pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
12719	pf->alloc_rss_size = 1;
12720	pf->rss_table_size = pf->hw.func_caps.rss_table_size;
12721	pf->rss_size_max = min_t(int, pf->rss_size_max,
12722	pf->hw.func_caps.num_tx_qp);
12723
12724	/* find the next higher power-of-2 of num cpus */
12725	pow = roundup_pow_of_two(num_online_cpus());
12726	pf->rss_size_max = min_t(int, pf->rss_size_max, pow);
12727
12728	if (pf->hw.func_caps.rss) {
12729	set_bit(nr: I40E_FLAG_RSS_ENA, addr: pf->flags);
12730	pf->alloc_rss_size = min_t(int, pf->rss_size_max,
12731	num_online_cpus());
12732	}
12733
12734	/* MFP mode enabled */
12735	if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
12736	set_bit(nr: I40E_FLAG_MFP_ENA, addr: pf->flags);
12737	dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
12738	if (i40e_get_partition_bw_setting(pf)) {
12739	dev_warn(&pf->pdev->dev,
12740	"Could not get partition bw settings\n");
12741	} else {
12742	dev_info(&pf->pdev->dev,
12743	"Partition BW Min = %8.8x, Max = %8.8x\n",
12744	pf->min_bw, pf->max_bw);
12745
12746	/* nudge the Tx scheduler */
12747	i40e_set_partition_bw_setting(pf);
12748	}
12749	}
12750
12751	if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
12752	(pf->hw.func_caps.fd_filters_best_effort > 0)) {
12753	set_bit(nr: I40E_FLAG_FD_ATR_ENA, addr: pf->flags);
12754	if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&
12755	pf->hw.num_partitions > 1)
12756	dev_info(&pf->pdev->dev,
12757	"Flow Director Sideband mode Disabled in MFP mode\n");
12758	else
12759	set_bit(nr: I40E_FLAG_FD_SB_ENA, addr: pf->flags);
12760	pf->fdir_pf_filter_count =
12761	pf->hw.func_caps.fd_filters_guaranteed;
12762	pf->hw.fdir_shared_filter_count =
12763	pf->hw.func_caps.fd_filters_best_effort;
12764	}
12765
12766	/* Enable HW ATR eviction if possible */
12767	if (test_bit(I40E_HW_CAP_ATR_EVICT, pf->hw.caps))
12768	set_bit(nr: I40E_FLAG_HW_ATR_EVICT_ENA, addr: pf->flags);
12769
12770	if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
12771	pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
12772	set_bit(nr: I40E_FLAG_VMDQ_ENA, addr: pf->flags);
12773	pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
12774	}
12775
12776	if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
12777	set_bit(nr: I40E_FLAG_IWARP_ENA, addr: pf->flags);
12778	/* IWARP needs one extra vector for CQP just like MISC.*/
12779	pf->num_iwarp_msix = (int)num_online_cpus() + 1;
12780	}
12781	/* Stopping FW LLDP engine is supported on XL710 and X722
12782	* starting from FW versions determined in i40e_init_adminq.
12783	* Stopping the FW LLDP engine is not supported on XL710
12784	* if NPAR is functioning so unset this hw flag in this case.
12785	*/
12786	if (pf->hw.mac.type == I40E_MAC_XL710 &&
12787	pf->hw.func_caps.npar_enable)
12788	clear_bit(nr: I40E_HW_CAP_FW_LLDP_STOPPABLE, addr: pf->hw.caps);
12789
12790	#ifdef CONFIG_PCI_IOV
12791	if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12792	pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12793	set_bit(nr: I40E_FLAG_SRIOV_ENA, addr: pf->flags);
12794	pf->num_req_vfs = min_t(int,
12795	pf->hw.func_caps.num_vfs,
12796	I40E_MAX_VF_COUNT);
12797	}
12798	#endif /* CONFIG_PCI_IOV */
12799	pf->lan_veb = I40E_NO_VEB;
12800	pf->lan_vsi = I40E_NO_VSI;
12801
12802	/* By default FW has this off for performance reasons */
12803	clear_bit(nr: I40E_FLAG_VEB_STATS_ENA, addr: pf->flags);
12804
12805	/* set up queue assignment tracking */
12806	size = sizeof(struct i40e_lump_tracking)
12807	+ (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12808	pf->qp_pile = kzalloc(size, GFP_KERNEL);
12809	if (!pf->qp_pile) {
12810	err = -ENOMEM;
12811	goto sw_init_done;
12812	}
12813	pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12814
12815	pf->tx_timeout_recovery_level = 1;
12816
12817	if (pf->hw.mac.type != I40E_MAC_X722 &&
12818	i40e_is_total_port_shutdown_enabled(pf)) {
12819	/* Link down on close must be on when total port shutdown
12820	* is enabled for a given port
12821	*/
12822	set_bit(nr: I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, addr: pf->flags);
12823	set_bit(nr: I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, addr: pf->flags);
12824	dev_info(&pf->pdev->dev,
12825	"total-port-shutdown was enabled, link-down-on-close is forced on\n");
12826	}
12827	mutex_init(&pf->switch_mutex);
12828
12829	sw_init_done:
12830	return err;
12831	}
12832
12833	/**
12834	* i40e_set_ntuple - set the ntuple feature flag and take action
12835	* @pf: board private structure to initialize
12836	* @features: the feature set that the stack is suggesting
12837	*
12838	* returns a bool to indicate if reset needs to happen
12839	**/
12840	bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12841	{
12842	bool need_reset = false;
12843
12844	/* Check if Flow Director n-tuple support was enabled or disabled. If
12845	* the state changed, we need to reset.
12846	*/
12847	if (features & NETIF_F_NTUPLE) {
12848	/* Enable filters and mark for reset */
12849	if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
12850	need_reset = true;
12851	/* enable FD_SB only if there is MSI-X vector and no cloud
12852	* filters exist
12853	*/
12854	if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12855	set_bit(nr: I40E_FLAG_FD_SB_ENA, addr: pf->flags);
12856	clear_bit(nr: I40E_FLAG_FD_SB_INACTIVE, addr: pf->flags);
12857	}
12858	} else {
12859	/* turn off filters, mark for reset and clear SW filter list */
12860	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
12861	need_reset = true;
12862	i40e_fdir_filter_exit(pf);
12863	}
12864	clear_bit(nr: I40E_FLAG_FD_SB_ENA, addr: pf->flags);
12865	clear_bit(nr: __I40E_FD_SB_AUTO_DISABLED, addr: pf->state);
12866	set_bit(nr: I40E_FLAG_FD_SB_INACTIVE, addr: pf->flags);
12867
12868	/* reset fd counters */
12869	pf->fd_add_err = 0;
12870	pf->fd_atr_cnt = 0;
12871	/* if ATR was auto disabled it can be re-enabled. */
12872	if (test_and_clear_bit(nr: __I40E_FD_ATR_AUTO_DISABLED, addr: pf->state))
12873	if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
12874	(I40E_DEBUG_FD & pf->hw.debug_mask))
12875	dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12876	}
12877	return need_reset;
12878	}
12879
12880	/**
12881	* i40e_clear_rss_lut - clear the rx hash lookup table
12882	* @vsi: the VSI being configured
12883	**/
12884	static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12885	{
12886	struct i40e_pf *pf = vsi->back;
12887	struct i40e_hw *hw = &pf->hw;
12888	u16 vf_id = vsi->vf_id;
12889	u8 i;
12890
12891	if (vsi->type == I40E_VSI_MAIN) {
12892	for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12893	wr32(hw, I40E_PFQF_HLUT(i), 0);
12894	} else if (vsi->type == I40E_VSI_SRIOV) {
12895	for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12896	i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), reg_val: 0);
12897	} else {
12898	dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12899	}
12900	}
12901
12902	/**
12903	* i40e_set_loopback - turn on/off loopback mode on underlying PF
12904	* @vsi: ptr to VSI
12905	* @ena: flag to indicate the on/off setting
12906	*/
12907	static int i40e_set_loopback(struct i40e_vsi *vsi, bool ena)
12908	{
12909	bool if_running = netif_running(dev: vsi->netdev) &&
12910	!test_and_set_bit(nr: __I40E_VSI_DOWN, addr: vsi->state);
12911	int ret;
12912
12913	if (if_running)
12914	i40e_down(vsi);
12915
12916	ret = i40e_aq_set_mac_loopback(hw: &vsi->back->hw, ena_lpbk: ena, NULL);
12917	if (ret)
12918	netdev_err(dev: vsi->netdev, format: "Failed to toggle loopback state\n");
12919	if (if_running)
12920	i40e_up(vsi);
12921
12922	return ret;
12923	}
12924
12925	/**
12926	* i40e_set_features - set the netdev feature flags
12927	* @netdev: ptr to the netdev being adjusted
12928	* @features: the feature set that the stack is suggesting
12929	* Note: expects to be called while under rtnl_lock()
12930	**/
12931	static int i40e_set_features(struct net_device *netdev,
12932	netdev_features_t features)
12933	{
12934	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
12935	struct i40e_vsi *vsi = np->vsi;
12936	struct i40e_pf *pf = vsi->back;
12937	bool need_reset;
12938
12939	if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12940	i40e_pf_config_rss(pf);
12941	else if (!(features & NETIF_F_RXHASH) &&
12942	netdev->features & NETIF_F_RXHASH)
12943	i40e_clear_rss_lut(vsi);
12944
12945	if (features & NETIF_F_HW_VLAN_CTAG_RX)
12946	i40e_vlan_stripping_enable(vsi);
12947	else
12948	i40e_vlan_stripping_disable(vsi);
12949
12950	if (!(features & NETIF_F_HW_TC) &&
12951	(netdev->features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
12952	dev_err(&pf->pdev->dev,
12953	"Offloaded tc filters active, can't turn hw_tc_offload off");
12954	return -EINVAL;
12955	}
12956
12957	if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
12958	i40e_del_all_macvlans(vsi);
12959
12960	need_reset = i40e_set_ntuple(pf, features);
12961
12962	if (need_reset)
12963	i40e_do_reset(pf, I40E_PF_RESET_FLAG, lock_acquired: true);
12964
12965	if ((features ^ netdev->features) & NETIF_F_LOOPBACK)
12966	return i40e_set_loopback(vsi, ena: !!(features & NETIF_F_LOOPBACK));
12967
12968	return 0;
12969	}
12970
12971	static int i40e_udp_tunnel_set_port(struct net_device *netdev,
12972	unsigned int table, unsigned int idx,
12973	struct udp_tunnel_info *ti)
12974	{
12975	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
12976	struct i40e_hw *hw = &np->vsi->back->hw;
12977	u8 type, filter_index;
12978	int ret;
12979
12980	type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :
12981	I40E_AQC_TUNNEL_TYPE_NGE;
12982
12983	ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), protocol_index: type, filter_index: &filter_index,
12984	NULL);
12985	if (ret) {
12986	netdev_info(dev: netdev, format: "add UDP port failed, err %pe aq_err %s\n",
12987	ERR_PTR(error: ret),
12988	i40e_aq_str(hw, aq_err: hw->aq.asq_last_status));
12989	return -EIO;
12990	}
12991
12992	udp_tunnel_nic_set_port_priv(dev: netdev, table, idx, priv: filter_index);
12993	return 0;
12994	}
12995
12996	static int i40e_udp_tunnel_unset_port(struct net_device *netdev,
12997	unsigned int table, unsigned int idx,
12998	struct udp_tunnel_info *ti)
12999	{
13000	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
13001	struct i40e_hw *hw = &np->vsi->back->hw;
13002	int ret;
13003
13004	ret = i40e_aq_del_udp_tunnel(hw, index: ti->hw_priv, NULL);
13005	if (ret) {
13006	netdev_info(dev: netdev, format: "delete UDP port failed, err %pe aq_err %s\n",
13007	ERR_PTR(error: ret),
13008	i40e_aq_str(hw, aq_err: hw->aq.asq_last_status));
13009	return -EIO;
13010	}
13011
13012	return 0;
13013	}
13014
13015	static int i40e_get_phys_port_id(struct net_device *netdev,
13016	struct netdev_phys_item_id *ppid)
13017	{
13018	struct i40e_netdev_priv *np = netdev_priv(dev: netdev);
13019	struct i40e_pf *pf = np->vsi->back;
13020	struct i40e_hw *hw = &pf->hw;
13021
13022	if (!test_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps))
13023	return -EOPNOTSUPP;
13024
13025	ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
13026	memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
13027
13028	return 0;
13029	}
13030
13031	/**
13032	* i40e_ndo_fdb_add - add an entry to the hardware database
13033	* @ndm: the input from the stack
13034	* @tb: pointer to array of nladdr (unused)
13035	* @dev: the net device pointer
13036	* @addr: the MAC address entry being added
13037	* @vid: VLAN ID
13038	* @flags: instructions from stack about fdb operation
13039	* @extack: netlink extended ack, unused currently
13040	*/
13041	static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
13042	struct net_device *dev,
13043	const unsigned char *addr, u16 vid,
13044	u16 flags,
13045	struct netlink_ext_ack *extack)
13046	{
13047	struct i40e_netdev_priv *np = netdev_priv(dev);
13048	struct i40e_pf *pf = np->vsi->back;
13049	int err = 0;
13050
13051	if (!test_bit(I40E_FLAG_SRIOV_ENA, pf->flags))
13052	return -EOPNOTSUPP;
13053
13054	if (vid) {
13055	pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
13056	return -EINVAL;
13057	}
13058
13059	/* Hardware does not support aging addresses so if a
13060	* ndm_state is given only allow permanent addresses
13061	*/
13062	if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
13063	netdev_info(dev, format: "FDB only supports static addresses\n");
13064	return -EINVAL;
13065	}
13066
13067	if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
13068	err = dev_uc_add_excl(dev, addr);
13069	else if (is_multicast_ether_addr(addr))
13070	err = dev_mc_add_excl(dev, addr);
13071	else
13072	err = -EINVAL;
13073
13074	/* Only return duplicate errors if NLM_F_EXCL is set */
13075	if (err == -EEXIST && !(flags & NLM_F_EXCL))
13076	err = 0;
13077
13078	return err;
13079	}
13080
13081	/**
13082	* i40e_ndo_bridge_setlink - Set the hardware bridge mode
13083	* @dev: the netdev being configured
13084	* @nlh: RTNL message
13085	* @flags: bridge flags
13086	* @extack: netlink extended ack
13087	*
13088	* Inserts a new hardware bridge if not already created and
13089	* enables the bridging mode requested (VEB or VEPA). If the
13090	* hardware bridge has already been inserted and the request
13091	* is to change the mode then that requires a PF reset to
13092	* allow rebuild of the components with required hardware
13093	* bridge mode enabled.
13094	*
13095	* Note: expects to be called while under rtnl_lock()
13096	**/
13097	static int i40e_ndo_bridge_setlink(struct net_device *dev,
13098	struct nlmsghdr *nlh,
13099	u16 flags,
13100	struct netlink_ext_ack *extack)
13101	{
13102	struct i40e_netdev_priv *np = netdev_priv(dev);
13103	struct i40e_vsi *vsi = np->vsi;
13104	struct i40e_pf *pf = vsi->back;
13105	struct nlattr *attr, *br_spec;
13106	struct i40e_veb *veb;
13107	int rem;
13108
13109	/* Only for PF VSI for now */
13110	if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
13111	return -EOPNOTSUPP;
13112
13113	/* Find the HW bridge for PF VSI */
13114	veb = i40e_pf_get_veb_by_seid(pf, seid: vsi->uplink_seid);
13115
13116	br_spec = nlmsg_find_attr(nlh, hdrlen: sizeof(struct ifinfomsg), attrtype: IFLA_AF_SPEC);
13117	if (!br_spec)
13118	return -EINVAL;
13119
13120	nla_for_each_nested(attr, br_spec, rem) {
13121	__u16 mode;
13122
13123	if (nla_type(nla: attr) != IFLA_BRIDGE_MODE)
13124	continue;
13125
13126	mode = nla_get_u16(nla: attr);
13127	if ((mode != BRIDGE_MODE_VEPA) &&
13128	(mode != BRIDGE_MODE_VEB))
13129	return -EINVAL;
13130
13131	/* Insert a new HW bridge */
13132	if (!veb) {
13133	veb = i40e_veb_setup(pf, flags: 0, uplink_seid: vsi->uplink_seid, downlink_seid: vsi->seid,
13134	enabled_tc: vsi->tc_config.enabled_tc);
13135	if (veb) {
13136	veb->bridge_mode = mode;
13137	i40e_config_bridge_mode(veb);
13138	} else {
13139	/* No Bridge HW offload available */
13140	return -ENOENT;
13141	}
13142	break;
13143	} else if (mode != veb->bridge_mode) {
13144	/* Existing HW bridge but different mode needs reset */
13145	veb->bridge_mode = mode;
13146	/* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
13147	if (mode == BRIDGE_MODE_VEB)
13148	set_bit(nr: I40E_FLAG_VEB_MODE_ENA, addr: pf->flags);
13149	else
13150	clear_bit(nr: I40E_FLAG_VEB_MODE_ENA, addr: pf->flags);
13151	i40e_do_reset(pf, I40E_PF_RESET_FLAG, lock_acquired: true);
13152	break;
13153	}
13154	}
13155
13156	return 0;
13157	}
13158
13159	/**
13160	* i40e_ndo_bridge_getlink - Get the hardware bridge mode
13161	* @skb: skb buff
13162	* @pid: process id
13163	* @seq: RTNL message seq #
13164	* @dev: the netdev being configured
13165	* @filter_mask: unused
13166	* @nlflags: netlink flags passed in
13167	*
13168	* Return the mode in which the hardware bridge is operating in
13169	* i.e VEB or VEPA.
13170	**/
13171	static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
13172	struct net_device *dev,
13173	u32 __always_unused filter_mask,
13174	int nlflags)
13175	{
13176	struct i40e_netdev_priv *np = netdev_priv(dev);
13177	struct i40e_vsi *vsi = np->vsi;
13178	struct i40e_pf *pf = vsi->back;
13179	struct i40e_veb *veb;
13180
13181	/* Only for PF VSI for now */
13182	if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
13183	return -EOPNOTSUPP;
13184
13185	/* Find the HW bridge for the PF VSI */
13186	veb = i40e_pf_get_veb_by_seid(pf, seid: vsi->uplink_seid);
13187	if (!veb)
13188	return 0;
13189
13190	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, mode: veb->bridge_mode,
13191	flags: 0, mask: 0, nlflags, filter_mask, NULL);
13192	}
13193
13194	/**
13195	* i40e_features_check - Validate encapsulated packet conforms to limits
13196	* @skb: skb buff
13197	* @dev: This physical port's netdev
13198	* @features: Offload features that the stack believes apply
13199	**/
13200	static netdev_features_t i40e_features_check(struct sk_buff *skb,
13201	struct net_device *dev,
13202	netdev_features_t features)
13203	{
13204	size_t len;
13205
13206	/* No point in doing any of this if neither checksum nor GSO are
13207	* being requested for this frame. We can rule out both by just
13208	* checking for CHECKSUM_PARTIAL
13209	*/
13210	if (skb->ip_summed != CHECKSUM_PARTIAL)
13211	return features;
13212
13213	/* We cannot support GSO if the MSS is going to be less than
13214	* 64 bytes. If it is then we need to drop support for GSO.
13215	*/
13216	if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
13217	features &= ~NETIF_F_GSO_MASK;
13218
13219	/* MACLEN can support at most 63 words */
13220	len = skb_network_offset(skb);
13221	if (len & ~(63 * 2))
13222	goto out_err;
13223
13224	/* IPLEN and EIPLEN can support at most 127 dwords */
13225	len = skb_network_header_len(skb);
13226	if (len & ~(127 * 4))
13227	goto out_err;
13228
13229	if (skb->encapsulation) {
13230	/* L4TUNLEN can support 127 words */
13231	len = skb_inner_network_header(skb) - skb_transport_header(skb);
13232	if (len & ~(127 * 2))
13233	goto out_err;
13234
13235	/* IPLEN can support at most 127 dwords */
13236	len = skb_inner_transport_header(skb) -
13237	skb_inner_network_header(skb);
13238	if (len & ~(127 * 4))
13239	goto out_err;
13240	}
13241
13242	/* No need to validate L4LEN as TCP is the only protocol with a
13243	* flexible value and we support all possible values supported
13244	* by TCP, which is at most 15 dwords
13245	*/
13246
13247	return features;
13248	out_err:
13249	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
13250	}
13251
13252	/**
13253	* i40e_xdp_setup - add/remove an XDP program
13254	* @vsi: VSI to changed
13255	* @prog: XDP program
13256	* @extack: netlink extended ack
13257	**/
13258	static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,
13259	struct netlink_ext_ack *extack)
13260	{
13261	int frame_size = i40e_max_vsi_frame_size(vsi, xdp_prog: prog);
13262	struct i40e_pf *pf = vsi->back;
13263	struct bpf_prog *old_prog;
13264	bool need_reset;
13265	int i;
13266
13267	/* Don't allow frames that span over multiple buffers */
13268	if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) {
13269	NL_SET_ERR_MSG_MOD(extack, "MTU too large for linear frames and XDP prog does not support frags");
13270	return -EINVAL;
13271	}
13272
13273	/* When turning XDP on->off/off->on we reset and rebuild the rings. */
13274	need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
13275
13276	if (need_reset)
13277	i40e_prep_for_reset(pf);
13278
13279	/* VSI shall be deleted in a moment, just return EINVAL */
13280	if (test_bit(__I40E_IN_REMOVE, pf->state))
13281	return -EINVAL;
13282
13283	old_prog = xchg(&vsi->xdp_prog, prog);
13284
13285	if (need_reset) {
13286	if (!prog) {
13287	xdp_features_clear_redirect_target(dev: vsi->netdev);
13288	/* Wait until ndo_xsk_wakeup completes. */
13289	synchronize_rcu();
13290	}
13291	i40e_reset_and_rebuild(pf, reinit: true, lock_acquired: true);
13292	}
13293
13294	if (!i40e_enabled_xdp_vsi(vsi) && prog) {
13295	if (i40e_realloc_rx_bi_zc(vsi, zc: true))
13296	return -ENOMEM;
13297	} else if (i40e_enabled_xdp_vsi(vsi) && !prog) {
13298	if (i40e_realloc_rx_bi_zc(vsi, zc: false))
13299	return -ENOMEM;
13300	}
13301
13302	for (i = 0; i < vsi->num_queue_pairs; i++)
13303	WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
13304
13305	if (old_prog)
13306	bpf_prog_put(prog: old_prog);
13307
13308	/* Kick start the NAPI context if there is an AF_XDP socket open
13309	* on that queue id. This so that receiving will start.
13310	*/
13311	if (need_reset && prog) {
13312	for (i = 0; i < vsi->num_queue_pairs; i++)
13313	if (vsi->xdp_rings[i]->xsk_pool)
13314	(void)i40e_xsk_wakeup(dev: vsi->netdev, queue_id: i,
13315	XDP_WAKEUP_RX);
13316	xdp_features_set_redirect_target(dev: vsi->netdev, support_sg: true);
13317	}
13318
13319	return 0;
13320	}
13321
13322	/**
13323	* i40e_enter_busy_conf - Enters busy config state
13324	* @vsi: vsi
13325	*
13326	* Returns 0 on success, <0 for failure.
13327	**/
13328	static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
13329	{
13330	struct i40e_pf *pf = vsi->back;
13331	int timeout = 50;
13332
13333	while (test_and_set_bit(nr: __I40E_CONFIG_BUSY, addr: pf->state)) {
13334	timeout--;
13335	if (!timeout)
13336	return -EBUSY;
13337	usleep_range(min: 1000, max: 2000);
13338	}
13339
13340	return 0;
13341	}
13342
13343	/**
13344	* i40e_exit_busy_conf - Exits busy config state
13345	* @vsi: vsi
13346	**/
13347	static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
13348	{
13349	struct i40e_pf *pf = vsi->back;
13350
13351	clear_bit(nr: __I40E_CONFIG_BUSY, addr: pf->state);
13352	}
13353
13354	/**
13355	* i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
13356	* @vsi: vsi
13357	* @queue_pair: queue pair
13358	**/
13359	static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
13360	{
13361	memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
13362	sizeof(vsi->rx_rings[queue_pair]->rx_stats));
13363	memset(&vsi->tx_rings[queue_pair]->stats, 0,
13364	sizeof(vsi->tx_rings[queue_pair]->stats));
13365	if (i40e_enabled_xdp_vsi(vsi)) {
13366	memset(&vsi->xdp_rings[queue_pair]->stats, 0,
13367	sizeof(vsi->xdp_rings[queue_pair]->stats));
13368	}
13369	}
13370
13371	/**
13372	* i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
13373	* @vsi: vsi
13374	* @queue_pair: queue pair
13375	**/
13376	static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
13377	{
13378	i40e_clean_tx_ring(tx_ring: vsi->tx_rings[queue_pair]);
13379	if (i40e_enabled_xdp_vsi(vsi)) {
13380	/* Make sure that in-progress ndo_xdp_xmit calls are
13381	* completed.
13382	*/
13383	synchronize_rcu();
13384	i40e_clean_tx_ring(tx_ring: vsi->xdp_rings[queue_pair]);
13385	}
13386	i40e_clean_rx_ring(rx_ring: vsi->rx_rings[queue_pair]);
13387	}
13388
13389	/**
13390	* i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
13391	* @vsi: vsi
13392	* @queue_pair: queue pair
13393	* @enable: true for enable, false for disable
13394	**/
13395	static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
13396	bool enable)
13397	{
13398	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13399	struct i40e_q_vector *q_vector = rxr->q_vector;
13400
13401	if (!vsi->netdev)
13402	return;
13403
13404	/* All rings in a qp belong to the same qvector. */
13405	if (q_vector->rx.ring || q_vector->tx.ring) {
13406	if (enable)
13407	napi_enable(n: &q_vector->napi);
13408	else
13409	napi_disable(n: &q_vector->napi);
13410	}
13411	}
13412
13413	/**
13414	* i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
13415	* @vsi: vsi
13416	* @queue_pair: queue pair
13417	* @enable: true for enable, false for disable
13418	*
13419	* Returns 0 on success, <0 on failure.
13420	**/
13421	static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
13422	bool enable)
13423	{
13424	struct i40e_pf *pf = vsi->back;
13425	int pf_q, ret = 0;
13426
13427	pf_q = vsi->base_queue + queue_pair;
13428	ret = i40e_control_wait_tx_q(seid: vsi->seid, pf, pf_q,
13429	is_xdp: false /*is xdp*/, enable);
13430	if (ret) {
13431	dev_info(&pf->pdev->dev,
13432	"VSI seid %d Tx ring %d %sable timeout\n",
13433	vsi->seid, pf_q, (enable ? "en" : "dis"));
13434	return ret;
13435	}
13436
13437	i40e_control_rx_q(pf, pf_q, enable);
13438	ret = i40e_pf_rxq_wait(pf, pf_q, enable);
13439	if (ret) {
13440	dev_info(&pf->pdev->dev,
13441	"VSI seid %d Rx ring %d %sable timeout\n",
13442	vsi->seid, pf_q, (enable ? "en" : "dis"));
13443	return ret;
13444	}
13445
13446	/* Due to HW errata, on Rx disable only, the register can
13447	* indicate done before it really is. Needs 50ms to be sure
13448	*/
13449	if (!enable)
13450	mdelay(50);
13451
13452	if (!i40e_enabled_xdp_vsi(vsi))
13453	return ret;
13454
13455	ret = i40e_control_wait_tx_q(seid: vsi->seid, pf,
13456	pf_q: pf_q + vsi->alloc_queue_pairs,
13457	is_xdp: true /*is xdp*/, enable);
13458	if (ret) {
13459	dev_info(&pf->pdev->dev,
13460	"VSI seid %d XDP Tx ring %d %sable timeout\n",
13461	vsi->seid, pf_q, (enable ? "en" : "dis"));
13462	}
13463
13464	return ret;
13465	}
13466
13467	/**
13468	* i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
13469	* @vsi: vsi
13470	* @queue_pair: queue_pair
13471	**/
13472	static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
13473	{
13474	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13475	struct i40e_pf *pf = vsi->back;
13476	struct i40e_hw *hw = &pf->hw;
13477
13478	/* All rings in a qp belong to the same qvector. */
13479	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
13480	i40e_irq_dynamic_enable(vsi, vector: rxr->q_vector->v_idx);
13481	else
13482	i40e_irq_dynamic_enable_icr0(pf);
13483
13484	i40e_flush(hw);
13485	}
13486
13487	/**
13488	* i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
13489	* @vsi: vsi
13490	* @queue_pair: queue_pair
13491	**/
13492	static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
13493	{
13494	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13495	struct i40e_pf *pf = vsi->back;
13496	struct i40e_hw *hw = &pf->hw;
13497
13498	/* For simplicity, instead of removing the qp interrupt causes
13499	* from the interrupt linked list, we simply disable the interrupt, and
13500	* leave the list intact.
13501	*
13502	* All rings in a qp belong to the same qvector.
13503	*/
13504	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
13505	u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
13506
13507	wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
13508	i40e_flush(hw);
13509	synchronize_irq(irq: pf->msix_entries[intpf].vector);
13510	} else {
13511	/* Legacy and MSI mode - this stops all interrupt handling */
13512	wr32(hw, I40E_PFINT_ICR0_ENA, 0);
13513	wr32(hw, I40E_PFINT_DYN_CTL0, 0);
13514	i40e_flush(hw);
13515	synchronize_irq(irq: pf->pdev->irq);
13516	}
13517	}
13518
13519	/**
13520	* i40e_queue_pair_disable - Disables a queue pair
13521	* @vsi: vsi
13522	* @queue_pair: queue pair
13523	*
13524	* Returns 0 on success, <0 on failure.
13525	**/
13526	int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
13527	{
13528	int err;
13529
13530	err = i40e_enter_busy_conf(vsi);
13531	if (err)
13532	return err;
13533
13534	i40e_queue_pair_disable_irq(vsi, queue_pair);
13535	i40e_queue_pair_toggle_napi(vsi, queue_pair, enable: false /* off */);
13536	err = i40e_queue_pair_toggle_rings(vsi, queue_pair, enable: false /* off */);
13537	i40e_clean_rx_ring(rx_ring: vsi->rx_rings[queue_pair]);
13538	i40e_queue_pair_clean_rings(vsi, queue_pair);
13539	i40e_queue_pair_reset_stats(vsi, queue_pair);
13540
13541	return err;
13542	}
13543
13544	/**
13545	* i40e_queue_pair_enable - Enables a queue pair
13546	* @vsi: vsi
13547	* @queue_pair: queue pair
13548	*
13549	* Returns 0 on success, <0 on failure.
13550	**/
13551	int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
13552	{
13553	int err;
13554
13555	err = i40e_configure_tx_ring(ring: vsi->tx_rings[queue_pair]);
13556	if (err)
13557	return err;
13558
13559	if (i40e_enabled_xdp_vsi(vsi)) {
13560	err = i40e_configure_tx_ring(ring: vsi->xdp_rings[queue_pair]);
13561	if (err)
13562	return err;
13563	}
13564
13565	err = i40e_configure_rx_ring(ring: vsi->rx_rings[queue_pair]);
13566	if (err)
13567	return err;
13568
13569	err = i40e_queue_pair_toggle_rings(vsi, queue_pair, enable: true /* on */);
13570	i40e_queue_pair_toggle_napi(vsi, queue_pair, enable: true /* on */);
13571	i40e_queue_pair_enable_irq(vsi, queue_pair);
13572
13573	i40e_exit_busy_conf(vsi);
13574
13575	return err;
13576	}
13577
13578	/**
13579	* i40e_xdp - implements ndo_bpf for i40e
13580	* @dev: netdevice
13581	* @xdp: XDP command
13582	**/
13583	static int i40e_xdp(struct net_device *dev,
13584	struct netdev_bpf *xdp)
13585	{
13586	struct i40e_netdev_priv *np = netdev_priv(dev);
13587	struct i40e_vsi *vsi = np->vsi;
13588
13589	if (vsi->type != I40E_VSI_MAIN)
13590	return -EINVAL;
13591
13592	switch (xdp->command) {
13593	case XDP_SETUP_PROG:
13594	return i40e_xdp_setup(vsi, prog: xdp->prog, extack: xdp->extack);
13595	case XDP_SETUP_XSK_POOL:
13596	return i40e_xsk_pool_setup(vsi, pool: xdp->xsk.pool,
13597	qid: xdp->xsk.queue_id);
13598	default:
13599	return -EINVAL;
13600	}
13601	}
13602
13603	static const struct net_device_ops i40e_netdev_ops = {
13604	.ndo_open = i40e_open,
13605	.ndo_stop = i40e_close,
13606	.ndo_start_xmit = i40e_lan_xmit_frame,
13607	.ndo_get_stats64 = i40e_get_netdev_stats_struct,
13608	.ndo_set_rx_mode = i40e_set_rx_mode,
13609	.ndo_validate_addr = eth_validate_addr,
13610	.ndo_set_mac_address = i40e_set_mac,
13611	.ndo_change_mtu = i40e_change_mtu,
13612	.ndo_eth_ioctl = i40e_ioctl,
13613	.ndo_tx_timeout = i40e_tx_timeout,
13614	.ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid,
13615	.ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid,
13616	#ifdef CONFIG_NET_POLL_CONTROLLER
13617	.ndo_poll_controller = i40e_netpoll,
13618	#endif
13619	.ndo_setup_tc = __i40e_setup_tc,
13620	.ndo_select_queue = i40e_lan_select_queue,
13621	.ndo_set_features = i40e_set_features,
13622	.ndo_set_vf_mac = i40e_ndo_set_vf_mac,
13623	.ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan,
13624	.ndo_get_vf_stats = i40e_get_vf_stats,
13625	.ndo_set_vf_rate = i40e_ndo_set_vf_bw,
13626	.ndo_get_vf_config = i40e_ndo_get_vf_config,
13627	.ndo_set_vf_link_state = i40e_ndo_set_vf_link_state,
13628	.ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk,
13629	.ndo_set_vf_trust = i40e_ndo_set_vf_trust,
13630	.ndo_get_phys_port_id = i40e_get_phys_port_id,
13631	.ndo_fdb_add = i40e_ndo_fdb_add,
13632	.ndo_features_check = i40e_features_check,
13633	.ndo_bridge_getlink = i40e_ndo_bridge_getlink,
13634	.ndo_bridge_setlink = i40e_ndo_bridge_setlink,
13635	.ndo_bpf = i40e_xdp,
13636	.ndo_xdp_xmit = i40e_xdp_xmit,
13637	.ndo_xsk_wakeup = i40e_xsk_wakeup,
13638	.ndo_dfwd_add_station = i40e_fwd_add,
13639	.ndo_dfwd_del_station = i40e_fwd_del,
13640	};
13641
13642	/**
13643	* i40e_config_netdev - Setup the netdev flags
13644	* @vsi: the VSI being configured
13645	*
13646	* Returns 0 on success, negative value on failure
13647	**/
13648	static int i40e_config_netdev(struct i40e_vsi *vsi)
13649	{
13650	struct i40e_pf *pf = vsi->back;
13651	struct i40e_hw *hw = &pf->hw;
13652	struct i40e_netdev_priv *np;
13653	struct net_device *netdev;
13654	u8 broadcast[ETH_ALEN];
13655	u8 mac_addr[ETH_ALEN];
13656	int etherdev_size;
13657	netdev_features_t hw_enc_features;
13658	netdev_features_t hw_features;
13659
13660	etherdev_size = sizeof(struct i40e_netdev_priv);
13661	netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
13662	if (!netdev)
13663	return -ENOMEM;
13664
13665	vsi->netdev = netdev;
13666	np = netdev_priv(dev: netdev);
13667	np->vsi = vsi;
13668
13669	hw_enc_features = NETIF_F_SG |
13670	NETIF_F_HW_CSUM |
13671	NETIF_F_HIGHDMA |
13672	NETIF_F_SOFT_FEATURES |
13673	NETIF_F_TSO |
13674	NETIF_F_TSO_ECN |
13675	NETIF_F_TSO6 |
13676	NETIF_F_GSO_GRE |
13677	NETIF_F_GSO_GRE_CSUM |
13678	NETIF_F_GSO_PARTIAL |
13679	NETIF_F_GSO_IPXIP4 |
13680	NETIF_F_GSO_IPXIP6 |
13681	NETIF_F_GSO_UDP_TUNNEL |
13682	NETIF_F_GSO_UDP_TUNNEL_CSUM |
13683	NETIF_F_GSO_UDP_L4 |
13684	NETIF_F_SCTP_CRC |
13685	NETIF_F_RXHASH |
13686	NETIF_F_RXCSUM |
13687	0;
13688
13689	if (!test_bit(I40E_HW_CAP_OUTER_UDP_CSUM, pf->hw.caps))
13690	netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
13691
13692	netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;
13693
13694	netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
13695
13696	netdev->hw_enc_features |= hw_enc_features;
13697
13698	/* record features VLANs can make use of */
13699	netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
13700
13701	#define I40E_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
13702	NETIF_F_GSO_GRE_CSUM | \
13703	NETIF_F_GSO_IPXIP4 | \
13704	NETIF_F_GSO_IPXIP6 | \
13705	NETIF_F_GSO_UDP_TUNNEL | \
13706	NETIF_F_GSO_UDP_TUNNEL_CSUM)
13707
13708	netdev->gso_partial_features = I40E_GSO_PARTIAL_FEATURES;
13709	netdev->features |= NETIF_F_GSO_PARTIAL |
13710	I40E_GSO_PARTIAL_FEATURES;
13711
13712	netdev->mpls_features |= NETIF_F_SG;
13713	netdev->mpls_features |= NETIF_F_HW_CSUM;
13714	netdev->mpls_features |= NETIF_F_TSO;
13715	netdev->mpls_features |= NETIF_F_TSO6;
13716	netdev->mpls_features |= I40E_GSO_PARTIAL_FEATURES;
13717
13718	/* enable macvlan offloads */
13719	netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
13720
13721	hw_features = hw_enc_features |
13722	NETIF_F_HW_VLAN_CTAG_TX |
13723	NETIF_F_HW_VLAN_CTAG_RX;
13724
13725	if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
13726	hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
13727
13728	netdev->hw_features |= hw_features | NETIF_F_LOOPBACK;
13729
13730	netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
13731	netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
13732
13733	netdev->features &= ~NETIF_F_HW_TC;
13734
13735	if (vsi->type == I40E_VSI_MAIN) {
13736	SET_NETDEV_DEV(netdev, &pf->pdev->dev);
13737	ether_addr_copy(dst: mac_addr, src: hw->mac.perm_addr);
13738	/* The following steps are necessary for two reasons. First,
13739	* some older NVM configurations load a default MAC-VLAN
13740	* filter that will accept any tagged packet, and we want to
13741	* replace this with a normal filter. Additionally, it is
13742	* possible our MAC address was provided by the platform using
13743	* Open Firmware or similar.
13744	*
13745	* Thus, we need to remove the default filter and install one
13746	* specific to the MAC address.
13747	*/
13748	i40e_rm_default_mac_filter(vsi, macaddr: mac_addr);
13749	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
13750	i40e_add_mac_filter(vsi, macaddr: mac_addr);
13751	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
13752
13753	netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
13754	NETDEV_XDP_ACT_REDIRECT |
13755	NETDEV_XDP_ACT_XSK_ZEROCOPY |
13756	NETDEV_XDP_ACT_RX_SG;
13757	netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD;
13758	} else {
13759	/* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
13760	* are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
13761	* the end, which is 4 bytes long, so force truncation of the
13762	* original name by IFNAMSIZ - 4
13763	*/
13764	snprintf(buf: netdev->name, IFNAMSIZ, fmt: "%.*sv%%d",
13765	IFNAMSIZ - 4,
13766	pf->vsi[pf->lan_vsi]->netdev->name);
13767	eth_random_addr(addr: mac_addr);
13768
13769	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
13770	i40e_add_mac_filter(vsi, macaddr: mac_addr);
13771	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
13772	}
13773
13774	/* Add the broadcast filter so that we initially will receive
13775	* broadcast packets. Note that when a new VLAN is first added the
13776	* driver will convert all filters marked I40E_VLAN_ANY into VLAN
13777	* specific filters as part of transitioning into "vlan" operation.
13778	* When more VLANs are added, the driver will copy each existing MAC
13779	* filter and add it for the new VLAN.
13780	*
13781	* Broadcast filters are handled specially by
13782	* i40e_sync_filters_subtask, as the driver must to set the broadcast
13783	* promiscuous bit instead of adding this directly as a MAC/VLAN
13784	* filter. The subtask will update the correct broadcast promiscuous
13785	* bits as VLANs become active or inactive.
13786	*/
13787	eth_broadcast_addr(addr: broadcast);
13788	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
13789	i40e_add_mac_filter(vsi, macaddr: broadcast);
13790	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
13791
13792	eth_hw_addr_set(dev: netdev, addr: mac_addr);
13793	ether_addr_copy(dst: netdev->perm_addr, src: mac_addr);
13794
13795	/* i40iw_net_event() reads 16 bytes from neigh->primary_key */
13796	netdev->neigh_priv_len = sizeof(u32) * 4;
13797
13798	netdev->priv_flags |= IFF_UNICAST_FLT;
13799	netdev->priv_flags |= IFF_SUPP_NOFCS;
13800	/* Setup netdev TC information */
13801	i40e_vsi_config_netdev_tc(vsi, enabled_tc: vsi->tc_config.enabled_tc);
13802
13803	netdev->netdev_ops = &i40e_netdev_ops;
13804	netdev->watchdog_timeo = 5 * HZ;
13805	i40e_set_ethtool_ops(netdev);
13806
13807	/* MTU range: 68 - 9706 */
13808	netdev->min_mtu = ETH_MIN_MTU;
13809	netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13810
13811	return 0;
13812	}
13813
13814	/**
13815	* i40e_vsi_delete - Delete a VSI from the switch
13816	* @vsi: the VSI being removed
13817	*
13818	* Returns 0 on success, negative value on failure
13819	**/
13820	static void i40e_vsi_delete(struct i40e_vsi *vsi)
13821	{
13822	/* remove default VSI is not allowed */
13823	if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13824	return;
13825
13826	i40e_aq_delete_element(hw: &vsi->back->hw, seid: vsi->seid, NULL);
13827	}
13828
13829	/**
13830	* i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13831	* @vsi: the VSI being queried
13832	*
13833	* Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
13834	**/
13835	int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13836	{
13837	struct i40e_veb *veb;
13838	struct i40e_pf *pf = vsi->back;
13839
13840	/* Uplink is not a bridge so default to VEB */
13841	if (vsi->veb_idx >= I40E_MAX_VEB)
13842	return 1;
13843
13844	veb = pf->veb[vsi->veb_idx];
13845	if (!veb) {
13846	dev_info(&pf->pdev->dev,
13847	"There is no veb associated with the bridge\n");
13848	return -ENOENT;
13849	}
13850
13851	/* Uplink is a bridge in VEPA mode */
13852	if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13853	return 0;
13854	} else {
13855	/* Uplink is a bridge in VEB mode */
13856	return 1;
13857	}
13858
13859	/* VEPA is now default bridge, so return 0 */
13860	return 0;
13861	}
13862
13863	/**
13864	* i40e_add_vsi - Add a VSI to the switch
13865	* @vsi: the VSI being configured
13866	*
13867	* This initializes a VSI context depending on the VSI type to be added and
13868	* passes it down to the add_vsi aq command.
13869	**/
13870	static int i40e_add_vsi(struct i40e_vsi *vsi)
13871	{
13872	int ret = -ENODEV;
13873	struct i40e_pf *pf = vsi->back;
13874	struct i40e_hw *hw = &pf->hw;
13875	struct i40e_vsi_context ctxt;
13876	struct i40e_mac_filter *f;
13877	struct hlist_node *h;
13878	int bkt;
13879
13880	u8 enabled_tc = 0x1; /* TC0 enabled */
13881	int f_count = 0;
13882
13883	memset(&ctxt, 0, sizeof(ctxt));
13884	switch (vsi->type) {
13885	case I40E_VSI_MAIN:
13886	/* The PF's main VSI is already setup as part of the
13887	* device initialization, so we'll not bother with
13888	* the add_vsi call, but we will retrieve the current
13889	* VSI context.
13890	*/
13891	ctxt.seid = pf->main_vsi_seid;
13892	ctxt.pf_num = pf->hw.pf_id;
13893	ctxt.vf_num = 0;
13894	ret = i40e_aq_get_vsi_params(hw: &pf->hw, vsi_ctx: &ctxt, NULL);
13895	ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13896	if (ret) {
13897	dev_info(&pf->pdev->dev,
13898	"couldn't get PF vsi config, err %pe aq_err %s\n",
13899	ERR_PTR(ret),
13900	i40e_aq_str(&pf->hw,
13901	pf->hw.aq.asq_last_status));
13902	return -ENOENT;
13903	}
13904	vsi->info = ctxt.info;
13905	vsi->info.valid_sections = 0;
13906
13907	vsi->seid = ctxt.seid;
13908	vsi->id = ctxt.vsi_number;
13909
13910	enabled_tc = i40e_pf_get_tc_map(pf);
13911
13912	/* Source pruning is enabled by default, so the flag is
13913	* negative logic - if it's set, we need to fiddle with
13914	* the VSI to disable source pruning.
13915	*/
13916	if (test_bit(I40E_FLAG_SOURCE_PRUNING_DIS, pf->flags)) {
13917	memset(&ctxt, 0, sizeof(ctxt));
13918	ctxt.seid = pf->main_vsi_seid;
13919	ctxt.pf_num = pf->hw.pf_id;
13920	ctxt.vf_num = 0;
13921	ctxt.info.valid_sections |=
13922	cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13923	ctxt.info.switch_id =
13924	cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13925	ret = i40e_aq_update_vsi_params(hw, vsi_ctx: &ctxt, NULL);
13926	if (ret) {
13927	dev_info(&pf->pdev->dev,
13928	"update vsi failed, err %d aq_err %s\n",
13929	ret,
13930	i40e_aq_str(&pf->hw,
13931	pf->hw.aq.asq_last_status));
13932	ret = -ENOENT;
13933	goto err;
13934	}
13935	}
13936
13937	/* MFP mode setup queue map and update VSI */
13938	if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&
13939	!(pf->hw.func_caps.iscsi)) { /* NIC type PF */
13940	memset(&ctxt, 0, sizeof(ctxt));
13941	ctxt.seid = pf->main_vsi_seid;
13942	ctxt.pf_num = pf->hw.pf_id;
13943	ctxt.vf_num = 0;
13944	i40e_vsi_setup_queue_map(vsi, ctxt: &ctxt, enabled_tc, is_add: false);
13945	ret = i40e_aq_update_vsi_params(hw, vsi_ctx: &ctxt, NULL);
13946	if (ret) {
13947	dev_info(&pf->pdev->dev,
13948	"update vsi failed, err %pe aq_err %s\n",
13949	ERR_PTR(ret),
13950	i40e_aq_str(&pf->hw,
13951	pf->hw.aq.asq_last_status));
13952	ret = -ENOENT;
13953	goto err;
13954	}
13955	/* update the local VSI info queue map */
13956	i40e_vsi_update_queue_map(vsi, ctxt: &ctxt);
13957	vsi->info.valid_sections = 0;
13958	} else {
13959	/* Default/Main VSI is only enabled for TC0
13960	* reconfigure it to enable all TCs that are
13961	* available on the port in SFP mode.
13962	* For MFP case the iSCSI PF would use this
13963	* flow to enable LAN+iSCSI TC.
13964	*/
13965	ret = i40e_vsi_config_tc(vsi, enabled_tc);
13966	if (ret) {
13967	/* Single TC condition is not fatal,
13968	* message and continue
13969	*/
13970	dev_info(&pf->pdev->dev,
13971	"failed to configure TCs for main VSI tc_map 0x%08x, err %pe aq_err %s\n",
13972	enabled_tc,
13973	ERR_PTR(ret),
13974	i40e_aq_str(&pf->hw,
13975	pf->hw.aq.asq_last_status));
13976	}
13977	}
13978	break;
13979
13980	case I40E_VSI_FDIR:
13981	ctxt.pf_num = hw->pf_id;
13982	ctxt.vf_num = 0;
13983	ctxt.uplink_seid = vsi->uplink_seid;
13984	ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13985	ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13986	if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags) &&
13987	(i40e_is_vsi_uplink_mode_veb(vsi))) {
13988	ctxt.info.valid_sections |=
13989	cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13990	ctxt.info.switch_id =
13991	cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13992	}
13993	i40e_vsi_setup_queue_map(vsi, ctxt: &ctxt, enabled_tc, is_add: true);
13994	break;
13995
13996	case I40E_VSI_VMDQ2:
13997	ctxt.pf_num = hw->pf_id;
13998	ctxt.vf_num = 0;
13999	ctxt.uplink_seid = vsi->uplink_seid;
14000	ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14001	ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
14002
14003	/* This VSI is connected to VEB so the switch_id
14004	* should be set to zero by default.
14005	*/
14006	if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14007	ctxt.info.valid_sections |=
14008	cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14009	ctxt.info.switch_id =
14010	cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14011	}
14012
14013	/* Setup the VSI tx/rx queue map for TC0 only for now */
14014	i40e_vsi_setup_queue_map(vsi, ctxt: &ctxt, enabled_tc, is_add: true);
14015	break;
14016
14017	case I40E_VSI_SRIOV:
14018	ctxt.pf_num = hw->pf_id;
14019	ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
14020	ctxt.uplink_seid = vsi->uplink_seid;
14021	ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14022	ctxt.flags = I40E_AQ_VSI_TYPE_VF;
14023
14024	/* This VSI is connected to VEB so the switch_id
14025	* should be set to zero by default.
14026	*/
14027	if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14028	ctxt.info.valid_sections |=
14029	cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14030	ctxt.info.switch_id =
14031	cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14032	}
14033
14034	if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {
14035	ctxt.info.valid_sections |=
14036	cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
14037	ctxt.info.queueing_opt_flags |=
14038	(I40E_AQ_VSI_QUE_OPT_TCP_ENA |
14039	I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
14040	}
14041
14042	ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
14043	ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
14044	if (pf->vf[vsi->vf_id].spoofchk) {
14045	ctxt.info.valid_sections |=
14046	cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
14047	ctxt.info.sec_flags |=
14048	(I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
14049	I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
14050	}
14051	/* Setup the VSI tx/rx queue map for TC0 only for now */
14052	i40e_vsi_setup_queue_map(vsi, ctxt: &ctxt, enabled_tc, is_add: true);
14053	break;
14054
14055	case I40E_VSI_IWARP:
14056	/* send down message to iWARP */
14057	break;
14058
14059	default:
14060	return -ENODEV;
14061	}
14062
14063	if (vsi->type != I40E_VSI_MAIN) {
14064	ret = i40e_aq_add_vsi(hw, vsi_ctx: &ctxt, NULL);
14065	if (ret) {
14066	dev_info(&vsi->back->pdev->dev,
14067	"add vsi failed, err %pe aq_err %s\n",
14068	ERR_PTR(ret),
14069	i40e_aq_str(&pf->hw,
14070	pf->hw.aq.asq_last_status));
14071	ret = -ENOENT;
14072	goto err;
14073	}
14074	vsi->info = ctxt.info;
14075	vsi->info.valid_sections = 0;
14076	vsi->seid = ctxt.seid;
14077	vsi->id = ctxt.vsi_number;
14078	}
14079
14080	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
14081	vsi->active_filters = 0;
14082	/* If macvlan filters already exist, force them to get loaded */
14083	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
14084	f->state = I40E_FILTER_NEW;
14085	f_count++;
14086	}
14087	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
14088	clear_bit(nr: __I40E_VSI_OVERFLOW_PROMISC, addr: vsi->state);
14089
14090	if (f_count) {
14091	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
14092	set_bit(nr: __I40E_MACVLAN_SYNC_PENDING, addr: pf->state);
14093	}
14094
14095	/* Update VSI BW information */
14096	ret = i40e_vsi_get_bw_info(vsi);
14097	if (ret) {
14098	dev_info(&pf->pdev->dev,
14099	"couldn't get vsi bw info, err %pe aq_err %s\n",
14100	ERR_PTR(ret),
14101	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14102	/* VSI is already added so not tearing that up */
14103	ret = 0;
14104	}
14105
14106	err:
14107	return ret;
14108	}
14109
14110	/**
14111	* i40e_vsi_release - Delete a VSI and free its resources
14112	* @vsi: the VSI being removed
14113	*
14114	* Returns 0 on success or < 0 on error
14115	**/
14116	int i40e_vsi_release(struct i40e_vsi *vsi)
14117	{
14118	struct i40e_mac_filter *f;
14119	struct hlist_node *h;
14120	struct i40e_veb *veb;
14121	struct i40e_pf *pf;
14122	u16 uplink_seid;
14123	int i, n, bkt;
14124
14125	pf = vsi->back;
14126
14127	/* release of a VEB-owner or last VSI is not allowed */
14128	if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
14129	dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
14130	vsi->seid, vsi->uplink_seid);
14131	return -ENODEV;
14132	}
14133	if (vsi == pf->vsi[pf->lan_vsi] &&
14134	!test_bit(__I40E_DOWN, pf->state)) {
14135	dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
14136	return -ENODEV;
14137	}
14138	set_bit(nr: __I40E_VSI_RELEASING, addr: vsi->state);
14139	uplink_seid = vsi->uplink_seid;
14140
14141	if (vsi->type != I40E_VSI_SRIOV) {
14142	if (vsi->netdev_registered) {
14143	vsi->netdev_registered = false;
14144	if (vsi->netdev) {
14145	/* results in a call to i40e_close() */
14146	unregister_netdev(dev: vsi->netdev);
14147	}
14148	} else {
14149	i40e_vsi_close(vsi);
14150	}
14151	i40e_vsi_disable_irq(vsi);
14152	}
14153
14154	if (vsi->type == I40E_VSI_MAIN)
14155	i40e_devlink_destroy_port(pf);
14156
14157	spin_lock_bh(lock: &vsi->mac_filter_hash_lock);
14158
14159	/* clear the sync flag on all filters */
14160	if (vsi->netdev) {
14161	__dev_uc_unsync(dev: vsi->netdev, NULL);
14162	__dev_mc_unsync(dev: vsi->netdev, NULL);
14163	}
14164
14165	/* make sure any remaining filters are marked for deletion */
14166	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
14167	__i40e_del_filter(vsi, f);
14168
14169	spin_unlock_bh(lock: &vsi->mac_filter_hash_lock);
14170
14171	i40e_sync_vsi_filters(vsi);
14172
14173	i40e_vsi_delete(vsi);
14174	i40e_vsi_free_q_vectors(vsi);
14175	if (vsi->netdev) {
14176	free_netdev(dev: vsi->netdev);
14177	vsi->netdev = NULL;
14178	}
14179	i40e_vsi_clear_rings(vsi);
14180	i40e_vsi_clear(vsi);
14181
14182	/* If this was the last thing on the VEB, except for the
14183	* controlling VSI, remove the VEB, which puts the controlling
14184	* VSI onto the uplink port.
14185	*
14186	* Well, okay, there's one more exception here: don't remove
14187	* the floating VEBs yet. We'll wait for an explicit remove request
14188	* from up the network stack.
14189	*/
14190	veb = i40e_pf_get_veb_by_seid(pf, seid: uplink_seid);
14191	if (veb && veb->uplink_seid) {
14192	n = 0;
14193
14194	/* Count non-controlling VSIs present on the VEB */
14195	i40e_pf_for_each_vsi(pf, i, vsi)
14196	if (vsi->uplink_seid == uplink_seid &&
14197	(vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14198	n++;
14199
14200	/* If there is no VSI except the control one then release
14201	* the VEB and put the control VSI onto VEB uplink.
14202	*/
14203	if (!n)
14204	i40e_veb_release(veb);
14205	}
14206
14207	return 0;
14208	}
14209
14210	/**
14211	* i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
14212	* @vsi: ptr to the VSI
14213	*
14214	* This should only be called after i40e_vsi_mem_alloc() which allocates the
14215	* corresponding SW VSI structure and initializes num_queue_pairs for the
14216	* newly allocated VSI.
14217	*
14218	* Returns 0 on success or negative on failure
14219	**/
14220	static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
14221	{
14222	int ret = -ENOENT;
14223	struct i40e_pf *pf = vsi->back;
14224
14225	if (vsi->q_vectors[0]) {
14226	dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
14227	vsi->seid);
14228	return -EEXIST;
14229	}
14230
14231	if (vsi->base_vector) {
14232	dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
14233	vsi->seid, vsi->base_vector);
14234	return -EEXIST;
14235	}
14236
14237	ret = i40e_vsi_alloc_q_vectors(vsi);
14238	if (ret) {
14239	dev_info(&pf->pdev->dev,
14240	"failed to allocate %d q_vector for VSI %d, ret=%d\n",
14241	vsi->num_q_vectors, vsi->seid, ret);
14242	vsi->num_q_vectors = 0;
14243	goto vector_setup_out;
14244	}
14245
14246	/* In Legacy mode, we do not have to get any other vector since we
14247	* piggyback on the misc/ICR0 for queue interrupts.
14248	*/
14249	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
14250	return ret;
14251	if (vsi->num_q_vectors)
14252	vsi->base_vector = i40e_get_lump(pf, pile: pf->irq_pile,
14253	needed: vsi->num_q_vectors, id: vsi->idx);
14254	if (vsi->base_vector < 0) {
14255	dev_info(&pf->pdev->dev,
14256	"failed to get tracking for %d vectors for VSI %d, err=%d\n",
14257	vsi->num_q_vectors, vsi->seid, vsi->base_vector);
14258	i40e_vsi_free_q_vectors(vsi);
14259	ret = -ENOENT;
14260	goto vector_setup_out;
14261	}
14262
14263	vector_setup_out:
14264	return ret;
14265	}
14266
14267	/**
14268	* i40e_vsi_reinit_setup - return and reallocate resources for a VSI
14269	* @vsi: pointer to the vsi.
14270	*
14271	* This re-allocates a vsi's queue resources.
14272	*
14273	* Returns pointer to the successfully allocated and configured VSI sw struct
14274	* on success, otherwise returns NULL on failure.
14275	**/
14276	static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
14277	{
14278	u16 alloc_queue_pairs;
14279	struct i40e_pf *pf;
14280	u8 enabled_tc;
14281	int ret;
14282
14283	if (!vsi)
14284	return NULL;
14285
14286	pf = vsi->back;
14287
14288	i40e_put_lump(pile: pf->qp_pile, index: vsi->base_queue, id: vsi->idx);
14289	i40e_vsi_clear_rings(vsi);
14290
14291	i40e_vsi_free_arrays(vsi, free_qvectors: false);
14292	i40e_set_num_rings_in_vsi(vsi);
14293	ret = i40e_vsi_alloc_arrays(vsi, alloc_qvectors: false);
14294	if (ret)
14295	goto err_vsi;
14296
14297	alloc_queue_pairs = vsi->alloc_queue_pairs *
14298	(i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14299
14300	ret = i40e_get_lump(pf, pile: pf->qp_pile, needed: alloc_queue_pairs, id: vsi->idx);
14301	if (ret < 0) {
14302	dev_info(&pf->pdev->dev,
14303	"failed to get tracking for %d queues for VSI %d err %d\n",
14304	alloc_queue_pairs, vsi->seid, ret);
14305	goto err_vsi;
14306	}
14307	vsi->base_queue = ret;
14308
14309	/* Update the FW view of the VSI. Force a reset of TC and queue
14310	* layout configurations.
14311	*/
14312	enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
14313	pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
14314	pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
14315	i40e_vsi_config_tc(vsi: pf->vsi[pf->lan_vsi], enabled_tc);
14316	if (vsi->type == I40E_VSI_MAIN)
14317	i40e_rm_default_mac_filter(vsi, macaddr: pf->hw.mac.perm_addr);
14318
14319	/* assign it some queues */
14320	ret = i40e_alloc_rings(vsi);
14321	if (ret)
14322	goto err_rings;
14323
14324	/* map all of the rings to the q_vectors */
14325	i40e_vsi_map_rings_to_vectors(vsi);
14326	return vsi;
14327
14328	err_rings:
14329	i40e_vsi_free_q_vectors(vsi);
14330	if (vsi->netdev_registered) {
14331	vsi->netdev_registered = false;
14332	unregister_netdev(dev: vsi->netdev);
14333	free_netdev(dev: vsi->netdev);
14334	vsi->netdev = NULL;
14335	}
14336	if (vsi->type == I40E_VSI_MAIN)
14337	i40e_devlink_destroy_port(pf);
14338	i40e_aq_delete_element(hw: &pf->hw, seid: vsi->seid, NULL);
14339	err_vsi:
14340	i40e_vsi_clear(vsi);
14341	return NULL;
14342	}
14343
14344	/**
14345	* i40e_vsi_setup - Set up a VSI by a given type
14346	* @pf: board private structure
14347	* @type: VSI type
14348	* @uplink_seid: the switch element to link to
14349	* @param1: usage depends upon VSI type. For VF types, indicates VF id
14350	*
14351	* This allocates the sw VSI structure and its queue resources, then add a VSI
14352	* to the identified VEB.
14353	*
14354	* Returns pointer to the successfully allocated and configure VSI sw struct on
14355	* success, otherwise returns NULL on failure.
14356	**/
14357	struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
14358	u16 uplink_seid, u32 param1)
14359	{
14360	struct i40e_vsi *vsi = NULL;
14361	struct i40e_veb *veb = NULL;
14362	u16 alloc_queue_pairs;
14363	int v_idx;
14364	int ret;
14365
14366	/* The requested uplink_seid must be either
14367	* - the PF's port seid
14368	* no VEB is needed because this is the PF
14369	* or this is a Flow Director special case VSI
14370	* - seid of an existing VEB
14371	* - seid of a VSI that owns an existing VEB
14372	* - seid of a VSI that doesn't own a VEB
14373	* a new VEB is created and the VSI becomes the owner
14374	* - seid of the PF VSI, which is what creates the first VEB
14375	* this is a special case of the previous
14376	*
14377	* Find which uplink_seid we were given and create a new VEB if needed
14378	*/
14379	veb = i40e_pf_get_veb_by_seid(pf, seid: uplink_seid);
14380	if (!veb && uplink_seid != pf->mac_seid) {
14381	vsi = i40e_pf_get_vsi_by_seid(pf, seid: uplink_seid);
14382	if (!vsi) {
14383	dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
14384	uplink_seid);
14385	return NULL;
14386	}
14387
14388	if (vsi->uplink_seid == pf->mac_seid)
14389	veb = i40e_veb_setup(pf, flags: 0, uplink_seid: pf->mac_seid, downlink_seid: vsi->seid,
14390	enabled_tc: vsi->tc_config.enabled_tc);
14391	else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14392	veb = i40e_veb_setup(pf, flags: 0, uplink_seid: vsi->uplink_seid, downlink_seid: vsi->seid,
14393	enabled_tc: vsi->tc_config.enabled_tc);
14394	if (veb) {
14395	if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
14396	dev_info(&vsi->back->pdev->dev,
14397	"New VSI creation error, uplink seid of LAN VSI expected.\n");
14398	return NULL;
14399	}
14400	/* We come up by default in VEPA mode if SRIOV is not
14401	* already enabled, in which case we can't force VEPA
14402	* mode.
14403	*/
14404	if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
14405	veb->bridge_mode = BRIDGE_MODE_VEPA;
14406	clear_bit(nr: I40E_FLAG_VEB_MODE_ENA, addr: pf->flags);
14407	}
14408	i40e_config_bridge_mode(veb);
14409	}
14410	veb = i40e_pf_get_veb_by_seid(pf, seid: vsi->uplink_seid);
14411	if (!veb) {
14412	dev_info(&pf->pdev->dev, "couldn't add VEB\n");
14413	return NULL;
14414	}
14415
14416	vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14417	uplink_seid = veb->seid;
14418	}
14419
14420	/* get vsi sw struct */
14421	v_idx = i40e_vsi_mem_alloc(pf, type);
14422	if (v_idx < 0)
14423	goto err_alloc;
14424	vsi = pf->vsi[v_idx];
14425	if (!vsi)
14426	goto err_alloc;
14427	vsi->type = type;
14428	vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
14429
14430	if (type == I40E_VSI_MAIN)
14431	pf->lan_vsi = v_idx;
14432	else if (type == I40E_VSI_SRIOV)
14433	vsi->vf_id = param1;
14434	/* assign it some queues */
14435	alloc_queue_pairs = vsi->alloc_queue_pairs *
14436	(i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14437
14438	ret = i40e_get_lump(pf, pile: pf->qp_pile, needed: alloc_queue_pairs, id: vsi->idx);
14439	if (ret < 0) {
14440	dev_info(&pf->pdev->dev,
14441	"failed to get tracking for %d queues for VSI %d err=%d\n",
14442	alloc_queue_pairs, vsi->seid, ret);
14443	goto err_vsi;
14444	}
14445	vsi->base_queue = ret;
14446
14447	/* get a VSI from the hardware */
14448	vsi->uplink_seid = uplink_seid;
14449	ret = i40e_add_vsi(vsi);
14450	if (ret)
14451	goto err_vsi;
14452
14453	switch (vsi->type) {
14454	/* setup the netdev if needed */
14455	case I40E_VSI_MAIN:
14456	case I40E_VSI_VMDQ2:
14457	ret = i40e_config_netdev(vsi);
14458	if (ret)
14459	goto err_netdev;
14460	ret = i40e_netif_set_realnum_tx_rx_queues(vsi);
14461	if (ret)
14462	goto err_netdev;
14463	if (vsi->type == I40E_VSI_MAIN) {
14464	ret = i40e_devlink_create_port(pf);
14465	if (ret)
14466	goto err_netdev;
14467	SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port);
14468	}
14469	ret = register_netdev(dev: vsi->netdev);
14470	if (ret)
14471	goto err_dl_port;
14472	vsi->netdev_registered = true;
14473	netif_carrier_off(dev: vsi->netdev);
14474	#ifdef CONFIG_I40E_DCB
14475	/* Setup DCB netlink interface */
14476	i40e_dcbnl_setup(vsi);
14477	#endif /* CONFIG_I40E_DCB */
14478	fallthrough;
14479	case I40E_VSI_FDIR:
14480	/* set up vectors and rings if needed */
14481	ret = i40e_vsi_setup_vectors(vsi);
14482	if (ret)
14483	goto err_msix;
14484
14485	ret = i40e_alloc_rings(vsi);
14486	if (ret)
14487	goto err_rings;
14488
14489	/* map all of the rings to the q_vectors */
14490	i40e_vsi_map_rings_to_vectors(vsi);
14491
14492	i40e_vsi_reset_stats(vsi);
14493	break;
14494	default:
14495	/* no netdev or rings for the other VSI types */
14496	break;
14497	}
14498
14499	if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps) &&
14500	vsi->type == I40E_VSI_VMDQ2) {
14501	ret = i40e_vsi_config_rss(vsi);
14502	if (ret)
14503	goto err_config;
14504	}
14505	return vsi;
14506
14507	err_config:
14508	i40e_vsi_clear_rings(vsi);
14509	err_rings:
14510	i40e_vsi_free_q_vectors(vsi);
14511	err_msix:
14512	if (vsi->netdev_registered) {
14513	vsi->netdev_registered = false;
14514	unregister_netdev(dev: vsi->netdev);
14515	free_netdev(dev: vsi->netdev);
14516	vsi->netdev = NULL;
14517	}
14518	err_dl_port:
14519	if (vsi->type == I40E_VSI_MAIN)
14520	i40e_devlink_destroy_port(pf);
14521	err_netdev:
14522	i40e_aq_delete_element(hw: &pf->hw, seid: vsi->seid, NULL);
14523	err_vsi:
14524	i40e_vsi_clear(vsi);
14525	err_alloc:
14526	return NULL;
14527	}
14528
14529	/**
14530	* i40e_veb_get_bw_info - Query VEB BW information
14531	* @veb: the veb to query
14532	*
14533	* Query the Tx scheduler BW configuration data for given VEB
14534	**/
14535	static int i40e_veb_get_bw_info(struct i40e_veb *veb)
14536	{
14537	struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
14538	struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
14539	struct i40e_pf *pf = veb->pf;
14540	struct i40e_hw *hw = &pf->hw;
14541	u32 tc_bw_max;
14542	int ret = 0;
14543	int i;
14544
14545	ret = i40e_aq_query_switch_comp_bw_config(hw, seid: veb->seid,
14546	bw_data: &bw_data, NULL);
14547	if (ret) {
14548	dev_info(&pf->pdev->dev,
14549	"query veb bw config failed, err %pe aq_err %s\n",
14550	ERR_PTR(ret),
14551	i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14552	goto out;
14553	}
14554
14555	ret = i40e_aq_query_switch_comp_ets_config(hw, seid: veb->seid,
14556	bw_data: &ets_data, NULL);
14557	if (ret) {
14558	dev_info(&pf->pdev->dev,
14559	"query veb bw ets config failed, err %pe aq_err %s\n",
14560	ERR_PTR(ret),
14561	i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14562	goto out;
14563	}
14564
14565	veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
14566	veb->bw_max_quanta = ets_data.tc_bw_max;
14567	veb->is_abs_credits = bw_data.absolute_credits_enable;
14568	veb->enabled_tc = ets_data.tc_valid_bits;
14569	tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
14570	(le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
14571	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
14572	veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
14573	veb->bw_tc_limit_credits[i] =
14574	le16_to_cpu(bw_data.tc_bw_limits[i]);
14575	veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
14576	}
14577
14578	out:
14579	return ret;
14580	}
14581
14582	/**
14583	* i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
14584	* @pf: board private structure
14585	*
14586	* On error: returns error code (negative)
14587	* On success: returns vsi index in PF (positive)
14588	**/
14589	static int i40e_veb_mem_alloc(struct i40e_pf *pf)
14590	{
14591	int ret = -ENOENT;
14592	struct i40e_veb *veb;
14593	int i;
14594
14595	/* Need to protect the allocation of switch elements at the PF level */
14596	mutex_lock(&pf->switch_mutex);
14597
14598	/* VEB list may be fragmented if VEB creation/destruction has
14599	* been happening. We can afford to do a quick scan to look
14600	* for any free slots in the list.
14601	*
14602	* find next empty veb slot, looping back around if necessary
14603	*/
14604	i = 0;
14605	while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
14606	i++;
14607	if (i >= I40E_MAX_VEB) {
14608	ret = -ENOMEM;
14609	goto err_alloc_veb; /* out of VEB slots! */
14610	}
14611
14612	veb = kzalloc(size: sizeof(*veb), GFP_KERNEL);
14613	if (!veb) {
14614	ret = -ENOMEM;
14615	goto err_alloc_veb;
14616	}
14617	veb->pf = pf;
14618	veb->idx = i;
14619	veb->enabled_tc = 1;
14620
14621	pf->veb[i] = veb;
14622	ret = i;
14623	err_alloc_veb:
14624	mutex_unlock(lock: &pf->switch_mutex);
14625	return ret;
14626	}
14627
14628	/**
14629	* i40e_switch_branch_release - Delete a branch of the switch tree
14630	* @branch: where to start deleting
14631	*
14632	* This uses recursion to find the tips of the branch to be
14633	* removed, deleting until we get back to and can delete this VEB.
14634	**/
14635	static void i40e_switch_branch_release(struct i40e_veb *branch)
14636	{
14637	struct i40e_pf *pf = branch->pf;
14638	u16 branch_seid = branch->seid;
14639	u16 veb_idx = branch->idx;
14640	struct i40e_vsi *vsi;
14641	struct i40e_veb *veb;
14642	int i;
14643
14644	/* release any VEBs on this VEB - RECURSION */
14645	i40e_pf_for_each_veb(pf, i, veb)
14646	if (veb->uplink_seid == branch->seid)
14647	i40e_switch_branch_release(branch: veb);
14648
14649	/* Release the VSIs on this VEB, but not the owner VSI.
14650	*
14651	* NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
14652	* the VEB itself, so don't use (*branch) after this loop.
14653	*/
14654	i40e_pf_for_each_vsi(pf, i, vsi)
14655	if (vsi->uplink_seid == branch_seid &&
14656	(vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14657	i40e_vsi_release(vsi);
14658
14659	/* There's one corner case where the VEB might not have been
14660	* removed, so double check it here and remove it if needed.
14661	* This case happens if the veb was created from the debugfs
14662	* commands and no VSIs were added to it.
14663	*/
14664	if (pf->veb[veb_idx])
14665	i40e_veb_release(veb: pf->veb[veb_idx]);
14666	}
14667
14668	/**
14669	* i40e_veb_clear - remove veb struct
14670	* @veb: the veb to remove
14671	**/
14672	static void i40e_veb_clear(struct i40e_veb *veb)
14673	{
14674	if (!veb)
14675	return;
14676
14677	if (veb->pf) {
14678	struct i40e_pf *pf = veb->pf;
14679
14680	mutex_lock(&pf->switch_mutex);
14681	if (pf->veb[veb->idx] == veb)
14682	pf->veb[veb->idx] = NULL;
14683	mutex_unlock(lock: &pf->switch_mutex);
14684	}
14685
14686	kfree(objp: veb);
14687	}
14688
14689	/**
14690	* i40e_veb_release - Delete a VEB and free its resources
14691	* @veb: the VEB being removed
14692	**/
14693	void i40e_veb_release(struct i40e_veb *veb)
14694	{
14695	struct i40e_vsi *vsi, *vsi_it;
14696	struct i40e_pf *pf;
14697	int i, n = 0;
14698
14699	pf = veb->pf;
14700
14701	/* find the remaining VSI and check for extras */
14702	i40e_pf_for_each_vsi(pf, i, vsi_it)
14703	if (vsi_it->uplink_seid == veb->seid) {
14704	if (vsi_it->flags & I40E_VSI_FLAG_VEB_OWNER)
14705	vsi = vsi_it;
14706	n++;
14707	}
14708
14709	/* Floating VEB has to be empty and regular one must have
14710	* single owner VSI.
14711	*/
14712	if ((veb->uplink_seid && n != 1) || (!veb->uplink_seid && n != 0)) {
14713	dev_info(&pf->pdev->dev,
14714	"can't remove VEB %d with %d VSIs left\n",
14715	veb->seid, n);
14716	return;
14717	}
14718
14719	/* For regular VEB move the owner VSI to uplink port */
14720	if (veb->uplink_seid) {
14721	vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
14722	vsi->uplink_seid = veb->uplink_seid;
14723	vsi->veb_idx = I40E_NO_VEB;
14724	}
14725
14726	i40e_aq_delete_element(hw: &pf->hw, seid: veb->seid, NULL);
14727	i40e_veb_clear(veb);
14728	}
14729
14730	/**
14731	* i40e_add_veb - create the VEB in the switch
14732	* @veb: the VEB to be instantiated
14733	* @vsi: the controlling VSI
14734	**/
14735	static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
14736	{
14737	struct i40e_pf *pf = veb->pf;
14738	bool enable_stats = !!test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);
14739	int ret;
14740
14741	ret = i40e_aq_add_veb(hw: &pf->hw, uplink_seid: veb->uplink_seid, downlink_seid: vsi ? vsi->seid : 0,
14742	enabled_tc: veb->enabled_tc, default_port: vsi ? false : true,
14743	pveb_seid: &veb->seid, enable_stats, NULL);
14744
14745	/* get a VEB from the hardware */
14746	if (ret) {
14747	dev_info(&pf->pdev->dev,
14748	"couldn't add VEB, err %pe aq_err %s\n",
14749	ERR_PTR(ret),
14750	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14751	return -EPERM;
14752	}
14753
14754	/* get statistics counter */
14755	ret = i40e_aq_get_veb_parameters(hw: &pf->hw, veb_seid: veb->seid, NULL, NULL,
14756	statistic_index: &veb->stats_idx, NULL, NULL, NULL);
14757	if (ret) {
14758	dev_info(&pf->pdev->dev,
14759	"couldn't get VEB statistics idx, err %pe aq_err %s\n",
14760	ERR_PTR(ret),
14761	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14762	return -EPERM;
14763	}
14764	ret = i40e_veb_get_bw_info(veb);
14765	if (ret) {
14766	dev_info(&pf->pdev->dev,
14767	"couldn't get VEB bw info, err %pe aq_err %s\n",
14768	ERR_PTR(ret),
14769	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14770	i40e_aq_delete_element(hw: &pf->hw, seid: veb->seid, NULL);
14771	return -ENOENT;
14772	}
14773
14774	if (vsi) {
14775	vsi->uplink_seid = veb->seid;
14776	vsi->veb_idx = veb->idx;
14777	vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14778	}
14779
14780	return 0;
14781	}
14782
14783	/**
14784	* i40e_veb_setup - Set up a VEB
14785	* @pf: board private structure
14786	* @flags: VEB setup flags
14787	* @uplink_seid: the switch element to link to
14788	* @vsi_seid: the initial VSI seid
14789	* @enabled_tc: Enabled TC bit-map
14790	*
14791	* This allocates the sw VEB structure and links it into the switch
14792	* It is possible and legal for this to be a duplicate of an already
14793	* existing VEB. It is also possible for both uplink and vsi seids
14794	* to be zero, in order to create a floating VEB.
14795	*
14796	* Returns pointer to the successfully allocated VEB sw struct on
14797	* success, otherwise returns NULL on failure.
14798	**/
14799	struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
14800	u16 uplink_seid, u16 vsi_seid,
14801	u8 enabled_tc)
14802	{
14803	struct i40e_vsi *vsi = NULL;
14804	struct i40e_veb *veb;
14805	int veb_idx;
14806	int ret;
14807
14808	/* if one seid is 0, the other must be 0 to create a floating relay */
14809	if ((uplink_seid == 0 || vsi_seid == 0) &&
14810	(uplink_seid + vsi_seid != 0)) {
14811	dev_info(&pf->pdev->dev,
14812	"one, not both seid's are 0: uplink=%d vsi=%d\n",
14813	uplink_seid, vsi_seid);
14814	return NULL;
14815	}
14816
14817	/* make sure there is such a vsi and uplink */
14818	if (vsi_seid) {
14819	vsi = i40e_pf_get_vsi_by_seid(pf, seid: vsi_seid);
14820	if (!vsi) {
14821	dev_err(&pf->pdev->dev, "vsi seid %d not found\n",
14822	vsi_seid);
14823	return NULL;
14824	}
14825	}
14826
14827	/* get veb sw struct */
14828	veb_idx = i40e_veb_mem_alloc(pf);
14829	if (veb_idx < 0)
14830	goto err_alloc;
14831	veb = pf->veb[veb_idx];
14832	veb->flags = flags;
14833	veb->uplink_seid = uplink_seid;
14834	veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14835
14836	/* create the VEB in the switch */
14837	ret = i40e_add_veb(veb, vsi);
14838	if (ret)
14839	goto err_veb;
14840
14841	if (vsi && vsi->idx == pf->lan_vsi)
14842	pf->lan_veb = veb->idx;
14843
14844	return veb;
14845
14846	err_veb:
14847	i40e_veb_clear(veb);
14848	err_alloc:
14849	return NULL;
14850	}
14851
14852	/**
14853	* i40e_setup_pf_switch_element - set PF vars based on switch type
14854	* @pf: board private structure
14855	* @ele: element we are building info from
14856	* @num_reported: total number of elements
14857	* @printconfig: should we print the contents
14858	*
14859	* helper function to assist in extracting a few useful SEID values.
14860	**/
14861	static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14862	struct i40e_aqc_switch_config_element_resp *ele,
14863	u16 num_reported, bool printconfig)
14864	{
14865	u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14866	u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14867	u8 element_type = ele->element_type;
14868	u16 seid = le16_to_cpu(ele->seid);
14869	struct i40e_veb *veb;
14870
14871	if (printconfig)
14872	dev_info(&pf->pdev->dev,
14873	"type=%d seid=%d uplink=%d downlink=%d\n",
14874	element_type, seid, uplink_seid, downlink_seid);
14875
14876	switch (element_type) {
14877	case I40E_SWITCH_ELEMENT_TYPE_MAC:
14878	pf->mac_seid = seid;
14879	break;
14880	case I40E_SWITCH_ELEMENT_TYPE_VEB:
14881	/* Main VEB? */
14882	if (uplink_seid != pf->mac_seid)
14883	break;
14884	if (pf->lan_veb >= I40E_MAX_VEB) {
14885	int v;
14886
14887	/* find existing or else empty VEB */
14888	veb = i40e_pf_get_veb_by_seid(pf, seid);
14889	if (veb) {
14890	pf->lan_veb = veb->idx;
14891	} else {
14892	v = i40e_veb_mem_alloc(pf);
14893	if (v < 0)
14894	break;
14895	pf->lan_veb = v;
14896	}
14897	}
14898	if (pf->lan_veb >= I40E_MAX_VEB)
14899	break;
14900
14901	pf->veb[pf->lan_veb]->seid = seid;
14902	pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
14903	pf->veb[pf->lan_veb]->pf = pf;
14904	break;
14905	case I40E_SWITCH_ELEMENT_TYPE_VSI:
14906	if (num_reported != 1)
14907	break;
14908	/* This is immediately after a reset so we can assume this is
14909	* the PF's VSI
14910	*/
14911	pf->mac_seid = uplink_seid;
14912	pf->main_vsi_seid = seid;
14913	if (printconfig)
14914	dev_info(&pf->pdev->dev,
14915	"pf_seid=%d main_vsi_seid=%d\n",
14916	downlink_seid, pf->main_vsi_seid);
14917	break;
14918	case I40E_SWITCH_ELEMENT_TYPE_PF:
14919	case I40E_SWITCH_ELEMENT_TYPE_VF:
14920	case I40E_SWITCH_ELEMENT_TYPE_EMP:
14921	case I40E_SWITCH_ELEMENT_TYPE_BMC:
14922	case I40E_SWITCH_ELEMENT_TYPE_PE:
14923	case I40E_SWITCH_ELEMENT_TYPE_PA:
14924	/* ignore these for now */
14925	break;
14926	default:
14927	dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
14928	element_type, seid);
14929	break;
14930	}
14931	}
14932
14933	/**
14934	* i40e_fetch_switch_configuration - Get switch config from firmware
14935	* @pf: board private structure
14936	* @printconfig: should we print the contents
14937	*
14938	* Get the current switch configuration from the device and
14939	* extract a few useful SEID values.
14940	**/
14941	int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
14942	{
14943	struct i40e_aqc_get_switch_config_resp *sw_config;
14944	u16 next_seid = 0;
14945	int ret = 0;
14946	u8 *aq_buf;
14947	int i;
14948
14949	aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
14950	if (!aq_buf)
14951	return -ENOMEM;
14952
14953	sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
14954	do {
14955	u16 num_reported, num_total;
14956
14957	ret = i40e_aq_get_switch_config(hw: &pf->hw, buf: sw_config,
14958	I40E_AQ_LARGE_BUF,
14959	start_seid: &next_seid, NULL);
14960	if (ret) {
14961	dev_info(&pf->pdev->dev,
14962	"get switch config failed err %d aq_err %s\n",
14963	ret,
14964	i40e_aq_str(&pf->hw,
14965	pf->hw.aq.asq_last_status));
14966	kfree(objp: aq_buf);
14967	return -ENOENT;
14968	}
14969
14970	num_reported = le16_to_cpu(sw_config->header.num_reported);
14971	num_total = le16_to_cpu(sw_config->header.num_total);
14972
14973	if (printconfig)
14974	dev_info(&pf->pdev->dev,
14975	"header: %d reported %d total\n",
14976	num_reported, num_total);
14977
14978	for (i = 0; i < num_reported; i++) {
14979	struct i40e_aqc_switch_config_element_resp *ele =
14980	&sw_config->element[i];
14981
14982	i40e_setup_pf_switch_element(pf, ele, num_reported,
14983	printconfig);
14984	}
14985	} while (next_seid != 0);
14986
14987	kfree(objp: aq_buf);
14988	return ret;
14989	}
14990
14991	/**
14992	* i40e_setup_pf_switch - Setup the HW switch on startup or after reset
14993	* @pf: board private structure
14994	* @reinit: if the Main VSI needs to re-initialized.
14995	* @lock_acquired: indicates whether or not the lock has been acquired
14996	*
14997	* Returns 0 on success, negative value on failure
14998	**/
14999	static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired)
15000	{
15001	u16 flags = 0;
15002	int ret;
15003
15004	/* find out what's out there already */
15005	ret = i40e_fetch_switch_configuration(pf, printconfig: false);
15006	if (ret) {
15007	dev_info(&pf->pdev->dev,
15008	"couldn't fetch switch config, err %pe aq_err %s\n",
15009	ERR_PTR(ret),
15010	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15011	return ret;
15012	}
15013	i40e_pf_reset_stats(pf);
15014
15015	/* set the switch config bit for the whole device to
15016	* support limited promisc or true promisc
15017	* when user requests promisc. The default is limited
15018	* promisc.
15019	*/
15020
15021	if ((pf->hw.pf_id == 0) &&
15022	!test_bit(I40E_FLAG_TRUE_PROMISC_ENA, pf->flags)) {
15023	flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15024	pf->last_sw_conf_flags = flags;
15025	}
15026
15027	if (pf->hw.pf_id == 0) {
15028	u16 valid_flags;
15029
15030	valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15031	ret = i40e_aq_set_switch_config(hw: &pf->hw, flags, valid_flags, mode: 0,
15032	NULL);
15033	if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
15034	dev_info(&pf->pdev->dev,
15035	"couldn't set switch config bits, err %pe aq_err %s\n",
15036	ERR_PTR(ret),
15037	i40e_aq_str(&pf->hw,
15038	pf->hw.aq.asq_last_status));
15039	/* not a fatal problem, just keep going */
15040	}
15041	pf->last_sw_conf_valid_flags = valid_flags;
15042	}
15043
15044	/* first time setup */
15045	if (pf->lan_vsi == I40E_NO_VSI || reinit) {
15046	struct i40e_vsi *vsi = NULL;
15047	u16 uplink_seid;
15048
15049	/* Set up the PF VSI associated with the PF's main VSI
15050	* that is already in the HW switch
15051	*/
15052	if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
15053	uplink_seid = pf->veb[pf->lan_veb]->seid;
15054	else
15055	uplink_seid = pf->mac_seid;
15056	if (pf->lan_vsi == I40E_NO_VSI)
15057	vsi = i40e_vsi_setup(pf, type: I40E_VSI_MAIN, uplink_seid, param1: 0);
15058	else if (reinit)
15059	vsi = i40e_vsi_reinit_setup(vsi: pf->vsi[pf->lan_vsi]);
15060	if (!vsi) {
15061	dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
15062	i40e_cloud_filter_exit(pf);
15063	i40e_fdir_teardown(pf);
15064	return -EAGAIN;
15065	}
15066	} else {
15067	/* force a reset of TC and queue layout configurations */
15068	u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
15069
15070	pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
15071	pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
15072	i40e_vsi_config_tc(vsi: pf->vsi[pf->lan_vsi], enabled_tc);
15073	}
15074	i40e_vlan_stripping_disable(vsi: pf->vsi[pf->lan_vsi]);
15075
15076	i40e_fdir_sb_setup(pf);
15077
15078	/* Setup static PF queue filter control settings */
15079	ret = i40e_setup_pf_filter_control(pf);
15080	if (ret) {
15081	dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
15082	ret);
15083	/* Failure here should not stop continuing other steps */
15084	}
15085
15086	/* enable RSS in the HW, even for only one queue, as the stack can use
15087	* the hash
15088	*/
15089	if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))
15090	i40e_pf_config_rss(pf);
15091
15092	/* fill in link information and enable LSE reporting */
15093	i40e_link_event(pf);
15094
15095	i40e_ptp_init(pf);
15096
15097	if (!lock_acquired)
15098	rtnl_lock();
15099
15100	/* repopulate tunnel port filters */
15101	udp_tunnel_nic_reset_ntf(dev: pf->vsi[pf->lan_vsi]->netdev);
15102
15103	if (!lock_acquired)
15104	rtnl_unlock();
15105
15106	return ret;
15107	}
15108
15109	/**
15110	* i40e_determine_queue_usage - Work out queue distribution
15111	* @pf: board private structure
15112	**/
15113	static void i40e_determine_queue_usage(struct i40e_pf *pf)
15114	{
15115	int queues_left;
15116	int q_max;
15117
15118	pf->num_lan_qps = 0;
15119
15120	/* Find the max queues to be put into basic use. We'll always be
15121	* using TC0, whether or not DCB is running, and TC0 will get the
15122	* big RSS set.
15123	*/
15124	queues_left = pf->hw.func_caps.num_tx_qp;
15125
15126	if ((queues_left == 1) ||
15127	!test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
15128	/* one qp for PF, no queues for anything else */
15129	queues_left = 0;
15130	pf->alloc_rss_size = pf->num_lan_qps = 1;
15131
15132	/* make sure all the fancies are disabled */
15133	clear_bit(nr: I40E_FLAG_RSS_ENA, addr: pf->flags);
15134	clear_bit(nr: I40E_FLAG_IWARP_ENA, addr: pf->flags);
15135	clear_bit(nr: I40E_FLAG_FD_SB_ENA, addr: pf->flags);
15136	clear_bit(nr: I40E_FLAG_FD_ATR_ENA, addr: pf->flags);
15137	clear_bit(nr: I40E_FLAG_DCB_CAPABLE, addr: pf->flags);
15138	clear_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
15139	clear_bit(nr: I40E_FLAG_SRIOV_ENA, addr: pf->flags);
15140	clear_bit(nr: I40E_FLAG_VMDQ_ENA, addr: pf->flags);
15141	set_bit(nr: I40E_FLAG_FD_SB_INACTIVE, addr: pf->flags);
15142	} else if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags) &&
15143	!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
15144	!test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
15145	!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) {
15146	/* one qp for PF */
15147	pf->alloc_rss_size = pf->num_lan_qps = 1;
15148	queues_left -= pf->num_lan_qps;
15149
15150	clear_bit(nr: I40E_FLAG_RSS_ENA, addr: pf->flags);
15151	clear_bit(nr: I40E_FLAG_IWARP_ENA, addr: pf->flags);
15152	clear_bit(nr: I40E_FLAG_FD_SB_ENA, addr: pf->flags);
15153	clear_bit(nr: I40E_FLAG_FD_ATR_ENA, addr: pf->flags);
15154	clear_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
15155	clear_bit(nr: I40E_FLAG_VMDQ_ENA, addr: pf->flags);
15156	set_bit(nr: I40E_FLAG_FD_SB_INACTIVE, addr: pf->flags);
15157	} else {
15158	/* Not enough queues for all TCs */
15159	if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags) &&
15160	queues_left < I40E_MAX_TRAFFIC_CLASS) {
15161	clear_bit(nr: I40E_FLAG_DCB_CAPABLE, addr: pf->flags);
15162	clear_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
15163	dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
15164	}
15165
15166	/* limit lan qps to the smaller of qps, cpus or msix */
15167	q_max = max_t(int, pf->rss_size_max, num_online_cpus());
15168	q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
15169	q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
15170	pf->num_lan_qps = q_max;
15171
15172	queues_left -= pf->num_lan_qps;
15173	}
15174
15175	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
15176	if (queues_left > 1) {
15177	queues_left -= 1; /* save 1 queue for FD */
15178	} else {
15179	clear_bit(nr: I40E_FLAG_FD_SB_ENA, addr: pf->flags);
15180	set_bit(nr: I40E_FLAG_FD_SB_INACTIVE, addr: pf->flags);
15181	dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
15182	}
15183	}
15184
15185	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
15186	pf->num_vf_qps && pf->num_req_vfs && queues_left) {
15187	pf->num_req_vfs = min_t(int, pf->num_req_vfs,
15188	(queues_left / pf->num_vf_qps));
15189	queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
15190	}
15191
15192	if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) &&
15193	pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
15194	pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
15195	(queues_left / pf->num_vmdq_qps));
15196	queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
15197	}
15198
15199	pf->queues_left = queues_left;
15200	dev_dbg(&pf->pdev->dev,
15201	"qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
15202	pf->hw.func_caps.num_tx_qp,
15203	!!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags),
15204	pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
15205	pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
15206	queues_left);
15207	}
15208
15209	/**
15210	* i40e_setup_pf_filter_control - Setup PF static filter control
15211	* @pf: PF to be setup
15212	*
15213	* i40e_setup_pf_filter_control sets up a PF's initial filter control
15214	* settings. If PE/FCoE are enabled then it will also set the per PF
15215	* based filter sizes required for them. It also enables Flow director,
15216	* ethertype and macvlan type filter settings for the pf.
15217	*
15218	* Returns 0 on success, negative on failure
15219	**/
15220	static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
15221	{
15222	struct i40e_filter_control_settings *settings = &pf->filter_settings;
15223
15224	settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
15225
15226	/* Flow Director is enabled */
15227	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) ||
15228	test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
15229	settings->enable_fdir = true;
15230
15231	/* Ethtype and MACVLAN filters enabled for PF */
15232	settings->enable_ethtype = true;
15233	settings->enable_macvlan = true;
15234
15235	if (i40e_set_filter_control(hw: &pf->hw, settings))
15236	return -ENOENT;
15237
15238	return 0;
15239	}
15240
15241	#define INFO_STRING_LEN 255
15242	#define REMAIN(__x) (INFO_STRING_LEN - (__x))
15243	static void i40e_print_features(struct i40e_pf *pf)
15244	{
15245	struct i40e_hw *hw = &pf->hw;
15246	char *buf;
15247	int i;
15248
15249	buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
15250	if (!buf)
15251	return;
15252
15253	i = snprintf(buf, INFO_STRING_LEN, fmt: "Features: PF-id[%d]", hw->pf_id);
15254	#ifdef CONFIG_PCI_IOV
15255	i += scnprintf(buf: &buf[i], REMAIN(i), fmt: " VFs: %d", pf->num_req_vfs);
15256	#endif
15257	i += scnprintf(buf: &buf[i], REMAIN(i), fmt: " VSIs: %d QP: %d",
15258	pf->hw.func_caps.num_vsis,
15259	pf->vsi[pf->lan_vsi]->num_queue_pairs);
15260	if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))
15261	i += scnprintf(buf: &buf[i], REMAIN(i), fmt: " RSS");
15262	if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
15263	i += scnprintf(buf: &buf[i], REMAIN(i), fmt: " FD_ATR");
15264	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
15265	i += scnprintf(buf: &buf[i], REMAIN(i), fmt: " FD_SB");
15266	i += scnprintf(buf: &buf[i], REMAIN(i), fmt: " NTUPLE");
15267	}
15268	if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
15269	i += scnprintf(buf: &buf[i], REMAIN(i), fmt: " DCB");
15270	i += scnprintf(buf: &buf[i], REMAIN(i), fmt: " VxLAN");
15271	i += scnprintf(buf: &buf[i], REMAIN(i), fmt: " Geneve");
15272	if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
15273	i += scnprintf(buf: &buf[i], REMAIN(i), fmt: " PTP");
15274	if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))
15275	i += scnprintf(buf: &buf[i], REMAIN(i), fmt: " VEB");
15276	else
15277	i += scnprintf(buf: &buf[i], REMAIN(i), fmt: " VEPA");
15278
15279	dev_info(&pf->pdev->dev, "%s\n", buf);
15280	kfree(objp: buf);
15281	WARN_ON(i > INFO_STRING_LEN);
15282	}
15283
15284	/**
15285	* i40e_get_platform_mac_addr - get platform-specific MAC address
15286	* @pdev: PCI device information struct
15287	* @pf: board private structure
15288	*
15289	* Look up the MAC address for the device. First we'll try
15290	* eth_platform_get_mac_address, which will check Open Firmware, or arch
15291	* specific fallback. Otherwise, we'll default to the stored value in
15292	* firmware.
15293	**/
15294	static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
15295	{
15296	if (eth_platform_get_mac_address(dev: &pdev->dev, mac_addr: pf->hw.mac.addr))
15297	i40e_get_mac_addr(hw: &pf->hw, mac_addr: pf->hw.mac.addr);
15298	}
15299
15300	/**
15301	* i40e_set_fec_in_flags - helper function for setting FEC options in flags
15302	* @fec_cfg: FEC option to set in flags
15303	* @flags: ptr to flags in which we set FEC option
15304	**/
15305	void i40e_set_fec_in_flags(u8 fec_cfg, unsigned long *flags)
15306	{
15307	if (fec_cfg & I40E_AQ_SET_FEC_AUTO) {
15308	set_bit(nr: I40E_FLAG_RS_FEC, addr: flags);
15309	set_bit(nr: I40E_FLAG_BASE_R_FEC, addr: flags);
15310	}
15311	if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
15312	(fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
15313	set_bit(nr: I40E_FLAG_RS_FEC, addr: flags);
15314	clear_bit(nr: I40E_FLAG_BASE_R_FEC, addr: flags);
15315	}
15316	if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
15317	(fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
15318	set_bit(nr: I40E_FLAG_BASE_R_FEC, addr: flags);
15319	clear_bit(nr: I40E_FLAG_RS_FEC, addr: flags);
15320	}
15321	if (fec_cfg == 0) {
15322	clear_bit(nr: I40E_FLAG_RS_FEC, addr: flags);
15323	clear_bit(nr: I40E_FLAG_BASE_R_FEC, addr: flags);
15324	}
15325	}
15326
15327	/**
15328	* i40e_check_recovery_mode - check if we are running transition firmware
15329	* @pf: board private structure
15330	*
15331	* Check registers indicating the firmware runs in recovery mode. Sets the
15332	* appropriate driver state.
15333	*
15334	* Returns true if the recovery mode was detected, false otherwise
15335	**/
15336	static bool i40e_check_recovery_mode(struct i40e_pf *pf)
15337	{
15338	u32 val = rd32(&pf->hw, I40E_GL_FWSTS);
15339
15340	if (val & I40E_GL_FWSTS_FWS1B_MASK) {
15341	dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
15342	dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
15343	set_bit(nr: __I40E_RECOVERY_MODE, addr: pf->state);
15344
15345	return true;
15346	}
15347	if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15348	dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n");
15349
15350	return false;
15351	}
15352
15353	/**
15354	* i40e_pf_loop_reset - perform reset in a loop.
15355	* @pf: board private structure
15356	*
15357	* This function is useful when a NIC is about to enter recovery mode.
15358	* When a NIC's internal data structures are corrupted the NIC's
15359	* firmware is going to enter recovery mode.
15360	* Right after a POR it takes about 7 minutes for firmware to enter
15361	* recovery mode. Until that time a NIC is in some kind of intermediate
15362	* state. After that time period the NIC almost surely enters
15363	* recovery mode. The only way for a driver to detect intermediate
15364	* state is to issue a series of pf-resets and check a return value.
15365	* If a PF reset returns success then the firmware could be in recovery
15366	* mode so the caller of this code needs to check for recovery mode
15367	* if this function returns success. There is a little chance that
15368	* firmware will hang in intermediate state forever.
15369	* Since waiting 7 minutes is quite a lot of time this function waits
15370	* 10 seconds and then gives up by returning an error.
15371	*
15372	* Return 0 on success, negative on failure.
15373	**/
15374	static int i40e_pf_loop_reset(struct i40e_pf *pf)
15375	{
15376	/* wait max 10 seconds for PF reset to succeed */
15377	const unsigned long time_end = jiffies + 10 * HZ;
15378	struct i40e_hw *hw = &pf->hw;
15379	int ret;
15380
15381	ret = i40e_pf_reset(hw);
15382	while (ret != 0 && time_before(jiffies, time_end)) {
15383	usleep_range(min: 10000, max: 20000);
15384	ret = i40e_pf_reset(hw);
15385	}
15386
15387	if (ret == 0)
15388	pf->pfr_count++;
15389	else
15390	dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);
15391
15392	return ret;
15393	}
15394
15395	/**
15396	* i40e_check_fw_empr - check if FW issued unexpected EMP Reset
15397	* @pf: board private structure
15398	*
15399	* Check FW registers to determine if FW issued unexpected EMP Reset.
15400	* Every time when unexpected EMP Reset occurs the FW increments
15401	* a counter of unexpected EMP Resets. When the counter reaches 10
15402	* the FW should enter the Recovery mode
15403	*
15404	* Returns true if FW issued unexpected EMP Reset
15405	**/
15406	static bool i40e_check_fw_empr(struct i40e_pf *pf)
15407	{
15408	const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &
15409	I40E_GL_FWSTS_FWS1B_MASK;
15410	return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) &&
15411	(fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10);
15412	}
15413
15414	/**
15415	* i40e_handle_resets - handle EMP resets and PF resets
15416	* @pf: board private structure
15417	*
15418	* Handle both EMP resets and PF resets and conclude whether there are
15419	* any issues regarding these resets. If there are any issues then
15420	* generate log entry.
15421	*
15422	* Return 0 if NIC is healthy or negative value when there are issues
15423	* with resets
15424	**/
15425	static int i40e_handle_resets(struct i40e_pf *pf)
15426	{
15427	const int pfr = i40e_pf_loop_reset(pf);
15428	const bool is_empr = i40e_check_fw_empr(pf);
15429
15430	if (is_empr || pfr != 0)
15431	dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n");
15432
15433	return is_empr ? -EIO : pfr;
15434	}
15435
15436	/**
15437	* i40e_init_recovery_mode - initialize subsystems needed in recovery mode
15438	* @pf: board private structure
15439	* @hw: ptr to the hardware info
15440	*
15441	* This function does a minimal setup of all subsystems needed for running
15442	* recovery mode.
15443	*
15444	* Returns 0 on success, negative on failure
15445	**/
15446	static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
15447	{
15448	struct i40e_vsi *vsi;
15449	int err;
15450	int v_idx;
15451
15452	pci_set_drvdata(pdev: pf->pdev, data: pf);
15453	pci_save_state(dev: pf->pdev);
15454
15455	/* set up periodic task facility */
15456	timer_setup(&pf->service_timer, i40e_service_timer, 0);
15457	pf->service_timer_period = HZ;
15458
15459	INIT_WORK(&pf->service_task, i40e_service_task);
15460	clear_bit(nr: __I40E_SERVICE_SCHED, addr: pf->state);
15461
15462	err = i40e_init_interrupt_scheme(pf);
15463	if (err)
15464	goto err_switch_setup;
15465
15466	/* The number of VSIs reported by the FW is the minimum guaranteed
15467	* to us; HW supports far more and we share the remaining pool with
15468	* the other PFs. We allocate space for more than the guarantee with
15469	* the understanding that we might not get them all later.
15470	*/
15471	if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15472	pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15473	else
15474	pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15475
15476	/* Set up the vsi struct and our local tracking of the MAIN PF vsi. */
15477	pf->vsi = kcalloc(n: pf->num_alloc_vsi, size: sizeof(struct i40e_vsi *),
15478	GFP_KERNEL);
15479	if (!pf->vsi) {
15480	err = -ENOMEM;
15481	goto err_switch_setup;
15482	}
15483
15484	/* We allocate one VSI which is needed as absolute minimum
15485	* in order to register the netdev
15486	*/
15487	v_idx = i40e_vsi_mem_alloc(pf, type: I40E_VSI_MAIN);
15488	if (v_idx < 0) {
15489	err = v_idx;
15490	goto err_switch_setup;
15491	}
15492	pf->lan_vsi = v_idx;
15493	vsi = pf->vsi[v_idx];
15494	if (!vsi) {
15495	err = -EFAULT;
15496	goto err_switch_setup;
15497	}
15498	vsi->alloc_queue_pairs = 1;
15499	err = i40e_config_netdev(vsi);
15500	if (err)
15501	goto err_switch_setup;
15502	err = register_netdev(dev: vsi->netdev);
15503	if (err)
15504	goto err_switch_setup;
15505	vsi->netdev_registered = true;
15506	i40e_dbg_pf_init(pf);
15507
15508	err = i40e_setup_misc_vector_for_recovery_mode(pf);
15509	if (err)
15510	goto err_switch_setup;
15511
15512	/* tell the firmware that we're starting */
15513	i40e_send_version(pf);
15514
15515	/* since everything's happy, start the service_task timer */
15516	mod_timer(timer: &pf->service_timer,
15517	expires: round_jiffies(j: jiffies + pf->service_timer_period));
15518
15519	return 0;
15520
15521	err_switch_setup:
15522	i40e_reset_interrupt_capability(pf);
15523	timer_shutdown_sync(timer: &pf->service_timer);
15524	i40e_shutdown_adminq(hw);
15525	iounmap(addr: hw->hw_addr);
15526	pci_release_mem_regions(pdev: pf->pdev);
15527	pci_disable_device(dev: pf->pdev);
15528	i40e_free_pf(pf);
15529
15530	return err;
15531	}
15532
15533	/**
15534	* i40e_set_subsystem_device_id - set subsystem device id
15535	* @hw: pointer to the hardware info
15536	*
15537	* Set PCI subsystem device id either from a pci_dev structure or
15538	* a specific FW register.
15539	**/
15540	static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw)
15541	{
15542	struct i40e_pf *pf = i40e_hw_to_pf(hw);
15543
15544	hw->subsystem_device_id = pf->pdev->subsystem_device ?
15545	pf->pdev->subsystem_device :
15546	(ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX);
15547	}
15548
15549	/**
15550	* i40e_probe - Device initialization routine
15551	* @pdev: PCI device information struct
15552	* @ent: entry in i40e_pci_tbl
15553	*
15554	* i40e_probe initializes a PF identified by a pci_dev structure.
15555	* The OS initialization, configuring of the PF private structure,
15556	* and a hardware reset occur.
15557	*
15558	* Returns 0 on success, negative on failure
15559	**/
15560	static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
15561	{
15562	struct i40e_aq_get_phy_abilities_resp abilities;
15563	#ifdef CONFIG_I40E_DCB
15564	enum i40e_get_fw_lldp_status_resp lldp_status;
15565	#endif /* CONFIG_I40E_DCB */
15566	struct i40e_vsi *vsi;
15567	struct i40e_pf *pf;
15568	struct i40e_hw *hw;
15569	u16 wol_nvm_bits;
15570	char nvm_ver[32];
15571	u16 link_status;
15572	#ifdef CONFIG_I40E_DCB
15573	int status;
15574	#endif /* CONFIG_I40E_DCB */
15575	int err;
15576	u32 val;
15577
15578	err = pci_enable_device_mem(dev: pdev);
15579	if (err)
15580	return err;
15581
15582	/* set up for high or low dma */
15583	err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
15584	if (err) {
15585	dev_err(&pdev->dev,
15586	"DMA configuration failed: 0x%x\n", err);
15587	goto err_dma;
15588	}
15589
15590	/* set up pci connections */
15591	err = pci_request_mem_regions(pdev, name: i40e_driver_name);
15592	if (err) {
15593	dev_info(&pdev->dev,
15594	"pci_request_selected_regions failed %d\n", err);
15595	goto err_pci_reg;
15596	}
15597
15598	pci_set_master(dev: pdev);
15599
15600	/* Now that we have a PCI connection, we need to do the
15601	* low level device setup. This is primarily setting up
15602	* the Admin Queue structures and then querying for the
15603	* device's current profile information.
15604	*/
15605	pf = i40e_alloc_pf(dev: &pdev->dev);
15606	if (!pf) {
15607	err = -ENOMEM;
15608	goto err_pf_alloc;
15609	}
15610	pf->next_vsi = 0;
15611	pf->pdev = pdev;
15612	set_bit(nr: __I40E_DOWN, addr: pf->state);
15613
15614	hw = &pf->hw;
15615
15616	pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
15617	I40E_MAX_CSR_SPACE);
15618	/* We believe that the highest register to read is
15619	* I40E_GLGEN_STAT_CLEAR, so we check if the BAR size
15620	* is not less than that before mapping to prevent a
15621	* kernel panic.
15622	*/
15623	if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
15624	dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
15625	pf->ioremap_len);
15626	err = -ENOMEM;
15627	goto err_ioremap;
15628	}
15629	hw->hw_addr = ioremap(pci_resource_start(pdev, 0), size: pf->ioremap_len);
15630	if (!hw->hw_addr) {
15631	err = -EIO;
15632	dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
15633	(unsigned int)pci_resource_start(pdev, 0),
15634	pf->ioremap_len, err);
15635	goto err_ioremap;
15636	}
15637	hw->vendor_id = pdev->vendor;
15638	hw->device_id = pdev->device;
15639	pci_read_config_byte(dev: pdev, PCI_REVISION_ID, val: &hw->revision_id);
15640	hw->subsystem_vendor_id = pdev->subsystem_vendor;
15641	i40e_set_subsystem_device_id(hw);
15642	hw->bus.device = PCI_SLOT(pdev->devfn);
15643	hw->bus.func = PCI_FUNC(pdev->devfn);
15644	hw->bus.bus_id = pdev->bus->number;
15645
15646	/* Select something other than the 802.1ad ethertype for the
15647	* switch to use internally and drop on ingress.
15648	*/
15649	hw->switch_tag = 0xffff;
15650	hw->first_tag = ETH_P_8021AD;
15651	hw->second_tag = ETH_P_8021Q;
15652
15653	INIT_LIST_HEAD(list: &pf->l3_flex_pit_list);
15654	INIT_LIST_HEAD(list: &pf->l4_flex_pit_list);
15655	INIT_LIST_HEAD(list: &pf->ddp_old_prof);
15656
15657	/* set up the locks for the AQ, do this only once in probe
15658	* and destroy them only once in remove
15659	*/
15660	mutex_init(&hw->aq.asq_mutex);
15661	mutex_init(&hw->aq.arq_mutex);
15662
15663	pf->msg_enable = netif_msg_init(debug_value: debug,
15664	NETIF_MSG_DRV |
15665	NETIF_MSG_PROBE |
15666	NETIF_MSG_LINK);
15667	if (debug < -1)
15668	pf->hw.debug_mask = debug;
15669
15670	/* do a special CORER for clearing PXE mode once at init */
15671	if (hw->revision_id == 0 &&
15672	(rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
15673	wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
15674	i40e_flush(hw);
15675	msleep(msecs: 200);
15676	pf->corer_count++;
15677
15678	i40e_clear_pxe_mode(hw);
15679	}
15680
15681	/* Reset here to make sure all is clean and to define PF 'n' */
15682	i40e_clear_hw(hw);
15683
15684	err = i40e_set_mac_type(hw);
15685	if (err) {
15686	dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15687	err);
15688	goto err_pf_reset;
15689	}
15690
15691	err = i40e_handle_resets(pf);
15692	if (err)
15693	goto err_pf_reset;
15694
15695	i40e_check_recovery_mode(pf);
15696
15697	if (is_kdump_kernel()) {
15698	hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN;
15699	hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN;
15700	} else {
15701	hw->aq.num_arq_entries = I40E_AQ_LEN;
15702	hw->aq.num_asq_entries = I40E_AQ_LEN;
15703	}
15704	hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15705	hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15706
15707	snprintf(buf: pf->int_name, size: sizeof(pf->int_name) - 1,
15708	fmt: "%s-%s:misc",
15709	dev_driver_string(dev: &pf->pdev->dev), dev_name(dev: &pdev->dev));
15710
15711	err = i40e_init_shared_code(hw);
15712	if (err) {
15713	dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15714	err);
15715	goto err_pf_reset;
15716	}
15717
15718	/* set up a default setting for link flow control */
15719	pf->hw.fc.requested_mode = I40E_FC_NONE;
15720
15721	err = i40e_init_adminq(hw);
15722	if (err) {
15723	if (err == -EIO)
15724	dev_info(&pdev->dev,
15725	"The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
15726	hw->aq.api_maj_ver,
15727	hw->aq.api_min_ver,
15728	I40E_FW_API_VERSION_MAJOR,
15729	I40E_FW_MINOR_VERSION(hw));
15730	else
15731	dev_info(&pdev->dev,
15732	"The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
15733
15734	goto err_pf_reset;
15735	}
15736	i40e_get_oem_version(hw);
15737	i40e_get_pba_string(hw);
15738
15739	/* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */
15740	i40e_nvm_version_str(hw, buf: nvm_ver, len: sizeof(nvm_ver));
15741	dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
15742	hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
15743	hw->aq.api_maj_ver, hw->aq.api_min_ver, nvm_ver,
15744	hw->vendor_id, hw->device_id, hw->subsystem_vendor_id,
15745	hw->subsystem_device_id);
15746
15747	if (i40e_is_aq_api_ver_ge(hw, I40E_FW_API_VERSION_MAJOR,
15748	I40E_FW_MINOR_VERSION(hw) + 1))
15749	dev_dbg(&pdev->dev,
15750	"The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n",
15751	hw->aq.api_maj_ver,
15752	hw->aq.api_min_ver,
15753	I40E_FW_API_VERSION_MAJOR,
15754	I40E_FW_MINOR_VERSION(hw));
15755	else if (i40e_is_aq_api_ver_lt(hw, maj: 1, min: 4))
15756	dev_info(&pdev->dev,
15757	"The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
15758	hw->aq.api_maj_ver,
15759	hw->aq.api_min_ver,
15760	I40E_FW_API_VERSION_MAJOR,
15761	I40E_FW_MINOR_VERSION(hw));
15762
15763	i40e_verify_eeprom(pf);
15764
15765	/* Rev 0 hardware was never productized */
15766	if (hw->revision_id < 1)
15767	dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
15768
15769	i40e_clear_pxe_mode(hw);
15770
15771	err = i40e_get_capabilities(pf, list_type: i40e_aqc_opc_list_func_capabilities);
15772	if (err)
15773	goto err_adminq_setup;
15774
15775	err = i40e_sw_init(pf);
15776	if (err) {
15777	dev_info(&pdev->dev, "sw_init failed: %d\n", err);
15778	goto err_sw_init;
15779	}
15780
15781	if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15782	return i40e_init_recovery_mode(pf, hw);
15783
15784	err = i40e_init_lan_hmc(hw, txq_num: hw->func_caps.num_tx_qp,
15785	rxq_num: hw->func_caps.num_rx_qp, fcoe_cntx_num: 0, fcoe_filt_num: 0);
15786	if (err) {
15787	dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
15788	goto err_init_lan_hmc;
15789	}
15790
15791	err = i40e_configure_lan_hmc(hw, model: I40E_HMC_MODEL_DIRECT_ONLY);
15792	if (err) {
15793	dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
15794	err = -ENOENT;
15795	goto err_configure_lan_hmc;
15796	}
15797
15798	/* Disable LLDP for NICs that have firmware versions lower than v4.3.
15799	* Ignore error return codes because if it was already disabled via
15800	* hardware settings this will fail
15801	*/
15802	if (test_bit(I40E_HW_CAP_STOP_FW_LLDP, pf->hw.caps)) {
15803	dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
15804	i40e_aq_stop_lldp(hw, shutdown_agent: true, persist: false, NULL);
15805	}
15806
15807	/* allow a platform config to override the HW addr */
15808	i40e_get_platform_mac_addr(pdev, pf);
15809
15810	if (!is_valid_ether_addr(addr: hw->mac.addr)) {
15811	dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
15812	err = -EIO;
15813	goto err_mac_addr;
15814	}
15815	dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
15816	ether_addr_copy(dst: hw->mac.perm_addr, src: hw->mac.addr);
15817	i40e_get_port_mac_addr(hw, mac_addr: hw->mac.port_addr);
15818	if (is_valid_ether_addr(addr: hw->mac.port_addr))
15819	set_bit(nr: I40E_HW_CAP_PORT_ID_VALID, addr: pf->hw.caps);
15820
15821	i40e_ptp_alloc_pins(pf);
15822	pci_set_drvdata(pdev, data: pf);
15823	pci_save_state(dev: pdev);
15824
15825	#ifdef CONFIG_I40E_DCB
15826	status = i40e_get_fw_lldp_status(hw: &pf->hw, lldp_status: &lldp_status);
15827	(!status &&
15828	lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ?
15829	(clear_bit(nr: I40E_FLAG_FW_LLDP_DIS, addr: pf->flags)) :
15830	(set_bit(nr: I40E_FLAG_FW_LLDP_DIS, addr: pf->flags));
15831	dev_info(&pdev->dev,
15832	test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?
15833	"FW LLDP is disabled\n" :
15834	"FW LLDP is enabled\n");
15835
15836	/* Enable FW to write default DCB config on link-up */
15837	i40e_aq_set_dcb_parameters(hw, dcb_enable: true, NULL);
15838
15839	err = i40e_init_pf_dcb(pf);
15840	if (err) {
15841	dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
15842	clear_bit(nr: I40E_FLAG_DCB_CAPABLE, addr: pf->flags);
15843	clear_bit(nr: I40E_FLAG_DCB_ENA, addr: pf->flags);
15844	/* Continue without DCB enabled */
15845	}
15846	#endif /* CONFIG_I40E_DCB */
15847
15848	/* set up periodic task facility */
15849	timer_setup(&pf->service_timer, i40e_service_timer, 0);
15850	pf->service_timer_period = HZ;
15851
15852	INIT_WORK(&pf->service_task, i40e_service_task);
15853	clear_bit(nr: __I40E_SERVICE_SCHED, addr: pf->state);
15854
15855	/* NVM bit on means WoL disabled for the port */
15856	i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, data: &wol_nvm_bits);
15857	if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
15858	pf->wol_en = false;
15859	else
15860	pf->wol_en = true;
15861	device_set_wakeup_enable(dev: &pf->pdev->dev, enable: pf->wol_en);
15862
15863	/* set up the main switch operations */
15864	i40e_determine_queue_usage(pf);
15865	err = i40e_init_interrupt_scheme(pf);
15866	if (err)
15867	goto err_switch_setup;
15868
15869	/* Reduce Tx and Rx pairs for kdump
15870	* When MSI-X is enabled, it's not allowed to use more TC queue
15871	* pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus
15872	* vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1.
15873	*/
15874	if (is_kdump_kernel())
15875	pf->num_lan_msix = 1;
15876
15877	pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;
15878	pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;
15879	pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
15880	pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;
15881	pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;
15882	pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |
15883	UDP_TUNNEL_TYPE_GENEVE;
15884
15885	/* The number of VSIs reported by the FW is the minimum guaranteed
15886	* to us; HW supports far more and we share the remaining pool with
15887	* the other PFs. We allocate space for more than the guarantee with
15888	* the understanding that we might not get them all later.
15889	*/
15890	if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15891	pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15892	else
15893	pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15894	if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
15895	dev_warn(&pf->pdev->dev,
15896	"limiting the VSI count due to UDP tunnel limitation %d > %d\n",
15897	pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
15898	pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
15899	}
15900
15901	/* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
15902	pf->vsi = kcalloc(n: pf->num_alloc_vsi, size: sizeof(struct i40e_vsi *),
15903	GFP_KERNEL);
15904	if (!pf->vsi) {
15905	err = -ENOMEM;
15906	goto err_switch_setup;
15907	}
15908
15909	#ifdef CONFIG_PCI_IOV
15910	/* prep for VF support */
15911	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
15912	test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
15913	!test_bit(__I40E_BAD_EEPROM, pf->state)) {
15914	if (pci_num_vf(dev: pdev))
15915	set_bit(nr: I40E_FLAG_VEB_MODE_ENA, addr: pf->flags);
15916	}
15917	#endif
15918	err = i40e_setup_pf_switch(pf, reinit: false, lock_acquired: false);
15919	if (err) {
15920	dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
15921	goto err_vsis;
15922	}
15923	INIT_LIST_HEAD(list: &pf->vsi[pf->lan_vsi]->ch_list);
15924
15925	/* if FDIR VSI was set up, start it now */
15926	vsi = i40e_find_vsi_by_type(pf, type: I40E_VSI_FDIR);
15927	if (vsi)
15928	i40e_vsi_open(vsi);
15929
15930	/* The driver only wants link up/down and module qualification
15931	* reports from firmware. Note the negative logic.
15932	*/
15933	err = i40e_aq_set_phy_int_mask(hw: &pf->hw,
15934	mask: ~(I40E_AQ_EVENT_LINK_UPDOWN |
15935	I40E_AQ_EVENT_MEDIA_NA |
15936	I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
15937	if (err)
15938	dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",
15939	ERR_PTR(err),
15940	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15941
15942	/* Reconfigure hardware for allowing smaller MSS in the case
15943	* of TSO, so that we avoid the MDD being fired and causing
15944	* a reset in the case of small MSS+TSO.
15945	*/
15946	val = rd32(hw, I40E_REG_MSS);
15947	if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
15948	val &= ~I40E_REG_MSS_MIN_MASK;
15949	val |= I40E_64BYTE_MSS;
15950	wr32(hw, I40E_REG_MSS, val);
15951	}
15952
15953	if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {
15954	msleep(msecs: 75);
15955	err = i40e_aq_set_link_restart_an(hw: &pf->hw, enable_link: true, NULL);
15956	if (err)
15957	dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",
15958	ERR_PTR(err),
15959	i40e_aq_str(&pf->hw,
15960	pf->hw.aq.asq_last_status));
15961	}
15962	/* The main driver is (mostly) up and happy. We need to set this state
15963	* before setting up the misc vector or we get a race and the vector
15964	* ends up disabled forever.
15965	*/
15966	clear_bit(nr: __I40E_DOWN, addr: pf->state);
15967
15968	/* In case of MSIX we are going to setup the misc vector right here
15969	* to handle admin queue events etc. In case of legacy and MSI
15970	* the misc functionality and queue processing is combined in
15971	* the same vector and that gets setup at open.
15972	*/
15973	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
15974	err = i40e_setup_misc_vector(pf);
15975	if (err) {
15976	dev_info(&pdev->dev,
15977	"setup of misc vector failed: %d\n", err);
15978	i40e_cloud_filter_exit(pf);
15979	i40e_fdir_teardown(pf);
15980	goto err_vsis;
15981	}
15982	}
15983
15984	#ifdef CONFIG_PCI_IOV
15985	/* prep for VF support */
15986	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
15987	test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
15988	!test_bit(__I40E_BAD_EEPROM, pf->state)) {
15989	/* disable link interrupts for VFs */
15990	val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
15991	val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
15992	wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
15993	i40e_flush(hw);
15994
15995	if (pci_num_vf(dev: pdev)) {
15996	dev_info(&pdev->dev,
15997	"Active VFs found, allocating resources.\n");
15998	err = i40e_alloc_vfs(pf, num_alloc_vfs: pci_num_vf(dev: pdev));
15999	if (err)
16000	dev_info(&pdev->dev,
16001	"Error %d allocating resources for existing VFs\n",
16002	err);
16003	}
16004	}
16005	#endif /* CONFIG_PCI_IOV */
16006
16007	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16008	pf->iwarp_base_vector = i40e_get_lump(pf, pile: pf->irq_pile,
16009	needed: pf->num_iwarp_msix,
16010	I40E_IWARP_IRQ_PILE_ID);
16011	if (pf->iwarp_base_vector < 0) {
16012	dev_info(&pdev->dev,
16013	"failed to get tracking for %d vectors for IWARP err=%d\n",
16014	pf->num_iwarp_msix, pf->iwarp_base_vector);
16015	clear_bit(nr: I40E_FLAG_IWARP_ENA, addr: pf->flags);
16016	}
16017	}
16018
16019	i40e_dbg_pf_init(pf);
16020
16021	/* tell the firmware that we're starting */
16022	i40e_send_version(pf);
16023
16024	/* since everything's happy, start the service_task timer */
16025	mod_timer(timer: &pf->service_timer,
16026	expires: round_jiffies(j: jiffies + pf->service_timer_period));
16027
16028	/* add this PF to client device list and launch a client service task */
16029	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16030	err = i40e_lan_add_device(pf);
16031	if (err)
16032	dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
16033	err);
16034	}
16035
16036	#define PCI_SPEED_SIZE 8
16037	#define PCI_WIDTH_SIZE 8
16038	/* Devices on the IOSF bus do not have this information
16039	* and will report PCI Gen 1 x 1 by default so don't bother
16040	* checking them.
16041	*/
16042	if (!test_bit(I40E_HW_CAP_NO_PCI_LINK_CHECK, pf->hw.caps)) {
16043	char speed[PCI_SPEED_SIZE] = "Unknown";
16044	char width[PCI_WIDTH_SIZE] = "Unknown";
16045
16046	/* Get the negotiated link width and speed from PCI config
16047	* space
16048	*/
16049	pcie_capability_read_word(dev: pf->pdev, PCI_EXP_LNKSTA,
16050	val: &link_status);
16051
16052	i40e_set_pci_config_data(hw, link_status);
16053
16054	switch (hw->bus.speed) {
16055	case i40e_bus_speed_8000:
16056	strscpy(speed, "8.0", PCI_SPEED_SIZE); break;
16057	case i40e_bus_speed_5000:
16058	strscpy(speed, "5.0", PCI_SPEED_SIZE); break;
16059	case i40e_bus_speed_2500:
16060	strscpy(speed, "2.5", PCI_SPEED_SIZE); break;
16061	default:
16062	break;
16063	}
16064	switch (hw->bus.width) {
16065	case i40e_bus_width_pcie_x8:
16066	strscpy(width, "8", PCI_WIDTH_SIZE); break;
16067	case i40e_bus_width_pcie_x4:
16068	strscpy(width, "4", PCI_WIDTH_SIZE); break;
16069	case i40e_bus_width_pcie_x2:
16070	strscpy(width, "2", PCI_WIDTH_SIZE); break;
16071	case i40e_bus_width_pcie_x1:
16072	strscpy(width, "1", PCI_WIDTH_SIZE); break;
16073	default:
16074	break;
16075	}
16076
16077	dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
16078	speed, width);
16079
16080	if (hw->bus.width < i40e_bus_width_pcie_x8 ||
16081	hw->bus.speed < i40e_bus_speed_8000) {
16082	dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
16083	dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
16084	}
16085	}
16086
16087	/* get the requested speeds from the fw */
16088	err = i40e_aq_get_phy_capabilities(hw, qualified_modules: false, report_init: false, abilities: &abilities, NULL);
16089	if (err)
16090	dev_dbg(&pf->pdev->dev, "get requested speeds ret = %pe last_status = %s\n",
16091	ERR_PTR(err),
16092	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16093	pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
16094
16095	/* set the FEC config due to the board capabilities */
16096	i40e_set_fec_in_flags(fec_cfg: abilities.fec_cfg_curr_mod_ext_info, flags: pf->flags);
16097
16098	/* get the supported phy types from the fw */
16099	err = i40e_aq_get_phy_capabilities(hw, qualified_modules: false, report_init: true, abilities: &abilities, NULL);
16100	if (err)
16101	dev_dbg(&pf->pdev->dev, "get supported phy types ret = %pe last_status = %s\n",
16102	ERR_PTR(err),
16103	i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16104
16105	/* make sure the MFS hasn't been set lower than the default */
16106	#define MAX_FRAME_SIZE_DEFAULT 0x2600
16107	val = FIELD_GET(I40E_PRTGL_SAH_MFS_MASK,
16108	rd32(&pf->hw, I40E_PRTGL_SAH));
16109	if (val < MAX_FRAME_SIZE_DEFAULT)
16110	dev_warn(&pdev->dev, "MFS for port %x has been set below the default: %x\n",
16111	pf->hw.port, val);
16112
16113	/* Add a filter to drop all Flow control frames from any VSI from being
16114	* transmitted. By doing so we stop a malicious VF from sending out
16115	* PAUSE or PFC frames and potentially controlling traffic for other
16116	* PF/VF VSIs.
16117	* The FW can still send Flow control frames if enabled.
16118	*/
16119	i40e_add_filter_to_drop_tx_flow_control_frames(hw: &pf->hw,
16120	vsi_seid: pf->main_vsi_seid);
16121
16122	if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
16123	(pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
16124	set_bit(nr: I40E_HW_CAP_PHY_CONTROLS_LEDS, addr: pf->hw.caps);
16125	if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
16126	set_bit(nr: I40E_HW_CAP_CRT_RETIMER, addr: pf->hw.caps);
16127	/* print a string summarizing features */
16128	i40e_print_features(pf);
16129
16130	i40e_devlink_register(pf);
16131
16132	return 0;
16133
16134	/* Unwind what we've done if something failed in the setup */
16135	err_vsis:
16136	set_bit(nr: __I40E_DOWN, addr: pf->state);
16137	i40e_clear_interrupt_scheme(pf);
16138	kfree(objp: pf->vsi);
16139	err_switch_setup:
16140	i40e_reset_interrupt_capability(pf);
16141	timer_shutdown_sync(timer: &pf->service_timer);
16142	err_mac_addr:
16143	err_configure_lan_hmc:
16144	(void)i40e_shutdown_lan_hmc(hw);
16145	err_init_lan_hmc:
16146	kfree(objp: pf->qp_pile);
16147	err_sw_init:
16148	err_adminq_setup:
16149	err_pf_reset:
16150	iounmap(addr: hw->hw_addr);
16151	err_ioremap:
16152	i40e_free_pf(pf);
16153	err_pf_alloc:
16154	pci_release_mem_regions(pdev);
16155	err_pci_reg:
16156	err_dma:
16157	pci_disable_device(dev: pdev);
16158	return err;
16159	}
16160
16161	/**
16162	* i40e_remove - Device removal routine
16163	* @pdev: PCI device information struct
16164	*
16165	* i40e_remove is called by the PCI subsystem to alert the driver
16166	* that is should release a PCI device. This could be caused by a
16167	* Hot-Plug event, or because the driver is going to be removed from
16168	* memory.
16169	**/
16170	static void i40e_remove(struct pci_dev *pdev)
16171	{
16172	struct i40e_pf *pf = pci_get_drvdata(pdev);
16173	struct i40e_hw *hw = &pf->hw;
16174	struct i40e_vsi *vsi;
16175	struct i40e_veb *veb;
16176	int ret_code;
16177	int i;
16178
16179	i40e_devlink_unregister(pf);
16180
16181	i40e_dbg_pf_exit(pf);
16182
16183	i40e_ptp_stop(pf);
16184
16185	/* Disable RSS in hw */
16186	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), reg_val: 0);
16187	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), reg_val: 0);
16188
16189	/* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE
16190	* flags, once they are set, i40e_rebuild should not be called as
16191	* i40e_prep_for_reset always returns early.
16192	*/
16193	while (test_and_set_bit(nr: __I40E_RESET_RECOVERY_PENDING, addr: pf->state))
16194	usleep_range(min: 1000, max: 2000);
16195	set_bit(nr: __I40E_IN_REMOVE, addr: pf->state);
16196
16197	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) {
16198	set_bit(nr: __I40E_VF_RESETS_DISABLED, addr: pf->state);
16199	i40e_free_vfs(pf);
16200	clear_bit(nr: I40E_FLAG_SRIOV_ENA, addr: pf->flags);
16201	}
16202	/* no more scheduling of any task */
16203	set_bit(nr: __I40E_SUSPENDED, addr: pf->state);
16204	set_bit(nr: __I40E_DOWN, addr: pf->state);
16205	if (pf->service_timer.function)
16206	timer_shutdown_sync(timer: &pf->service_timer);
16207	if (pf->service_task.func)
16208	cancel_work_sync(work: &pf->service_task);
16209
16210	if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
16211	struct i40e_vsi *vsi = pf->vsi[0];
16212
16213	/* We know that we have allocated only one vsi for this PF,
16214	* it was just for registering netdevice, so the interface
16215	* could be visible in the 'ifconfig' output
16216	*/
16217	unregister_netdev(dev: vsi->netdev);
16218	free_netdev(dev: vsi->netdev);
16219
16220	goto unmap;
16221	}
16222
16223	/* Client close must be called explicitly here because the timer
16224	* has been stopped.
16225	*/
16226	i40e_notify_client_of_netdev_close(vsi: pf->vsi[pf->lan_vsi], reset: false);
16227
16228	i40e_fdir_teardown(pf);
16229
16230	/* If there is a switch structure or any orphans, remove them.
16231	* This will leave only the PF's VSI remaining.
16232	*/
16233	i40e_pf_for_each_veb(pf, i, veb)
16234	if (veb->uplink_seid == pf->mac_seid ||
16235	veb->uplink_seid == 0)
16236	i40e_switch_branch_release(branch: veb);
16237
16238	/* Now we can shutdown the PF's VSIs, just before we kill
16239	* adminq and hmc.
16240	*/
16241	i40e_pf_for_each_vsi(pf, i, vsi) {
16242	i40e_vsi_close(vsi);
16243	i40e_vsi_release(vsi);
16244	pf->vsi[i] = NULL;
16245	}
16246
16247	i40e_cloud_filter_exit(pf);
16248
16249	/* remove attached clients */
16250	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16251	ret_code = i40e_lan_del_device(pf);
16252	if (ret_code)
16253	dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
16254	ret_code);
16255	}
16256
16257	/* shutdown and destroy the HMC */
16258	if (hw->hmc.hmc_obj) {
16259	ret_code = i40e_shutdown_lan_hmc(hw);
16260	if (ret_code)
16261	dev_warn(&pdev->dev,
16262	"Failed to destroy the HMC resources: %d\n",
16263	ret_code);
16264	}
16265
16266	unmap:
16267	/* Free MSI/legacy interrupt 0 when in recovery mode. */
16268	if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16269	!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
16270	free_irq(pf->pdev->irq, pf);
16271
16272	/* shutdown the adminq */
16273	i40e_shutdown_adminq(hw);
16274
16275	/* destroy the locks only once, here */
16276	mutex_destroy(lock: &hw->aq.arq_mutex);
16277	mutex_destroy(lock: &hw->aq.asq_mutex);
16278
16279	/* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
16280	rtnl_lock();
16281	i40e_clear_interrupt_scheme(pf);
16282	i40e_pf_for_each_vsi(pf, i, vsi) {
16283	if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
16284	i40e_vsi_clear_rings(vsi);
16285
16286	i40e_vsi_clear(vsi);
16287	pf->vsi[i] = NULL;
16288	}
16289	rtnl_unlock();
16290
16291	i40e_pf_for_each_veb(pf, i, veb) {
16292	kfree(objp: veb);
16293	pf->veb[i] = NULL;
16294	}
16295
16296	kfree(objp: pf->qp_pile);
16297	kfree(objp: pf->vsi);
16298
16299	iounmap(addr: hw->hw_addr);
16300	i40e_free_pf(pf);
16301	pci_release_mem_regions(pdev);
16302
16303	pci_disable_device(dev: pdev);
16304	}
16305
16306	/**
16307	* i40e_pci_error_detected - warning that something funky happened in PCI land
16308	* @pdev: PCI device information struct
16309	* @error: the type of PCI error
16310	*
16311	* Called to warn that something happened and the error handling steps
16312	* are in progress. Allows the driver to quiesce things, be ready for
16313	* remediation.
16314	**/
16315	static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
16316	pci_channel_state_t error)
16317	{
16318	struct i40e_pf *pf = pci_get_drvdata(pdev);
16319
16320	dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
16321
16322	if (!pf) {
16323	dev_info(&pdev->dev,
16324	"Cannot recover - error happened during device probe\n");
16325	return PCI_ERS_RESULT_DISCONNECT;
16326	}
16327
16328	/* shutdown all operations */
16329	if (!test_bit(__I40E_SUSPENDED, pf->state))
16330	i40e_prep_for_reset(pf);
16331
16332	/* Request a slot reset */
16333	return PCI_ERS_RESULT_NEED_RESET;
16334	}
16335
16336	/**
16337	* i40e_pci_error_slot_reset - a PCI slot reset just happened
16338	* @pdev: PCI device information struct
16339	*
16340	* Called to find if the driver can work with the device now that
16341	* the pci slot has been reset. If a basic connection seems good
16342	* (registers are readable and have sane content) then return a
16343	* happy little PCI_ERS_RESULT_xxx.
16344	**/
16345	static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
16346	{
16347	struct i40e_pf *pf = pci_get_drvdata(pdev);
16348	pci_ers_result_t result;
16349	u32 reg;
16350
16351	dev_dbg(&pdev->dev, "%s\n", __func__);
16352	if (pci_enable_device_mem(dev: pdev)) {
16353	dev_info(&pdev->dev,
16354	"Cannot re-enable PCI device after reset.\n");
16355	result = PCI_ERS_RESULT_DISCONNECT;
16356	} else {
16357	pci_set_master(dev: pdev);
16358	pci_restore_state(dev: pdev);
16359	pci_save_state(dev: pdev);
16360	pci_wake_from_d3(dev: pdev, enable: false);
16361
16362	reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
16363	if (reg == 0)
16364	result = PCI_ERS_RESULT_RECOVERED;
16365	else
16366	result = PCI_ERS_RESULT_DISCONNECT;
16367	}
16368
16369	return result;
16370	}
16371
16372	/**
16373	* i40e_pci_error_reset_prepare - prepare device driver for pci reset
16374	* @pdev: PCI device information struct
16375	*/
16376	static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
16377	{
16378	struct i40e_pf *pf = pci_get_drvdata(pdev);
16379
16380	i40e_prep_for_reset(pf);
16381	}
16382
16383	/**
16384	* i40e_pci_error_reset_done - pci reset done, device driver reset can begin
16385	* @pdev: PCI device information struct
16386	*/
16387	static void i40e_pci_error_reset_done(struct pci_dev *pdev)
16388	{
16389	struct i40e_pf *pf = pci_get_drvdata(pdev);
16390
16391	if (test_bit(__I40E_IN_REMOVE, pf->state))
16392	return;
16393
16394	i40e_reset_and_rebuild(pf, reinit: false, lock_acquired: false);
16395	#ifdef CONFIG_PCI_IOV
16396	i40e_restore_all_vfs_msi_state(pdev);
16397	#endif /* CONFIG_PCI_IOV */
16398	}
16399
16400	/**
16401	* i40e_pci_error_resume - restart operations after PCI error recovery
16402	* @pdev: PCI device information struct
16403	*
16404	* Called to allow the driver to bring things back up after PCI error
16405	* and/or reset recovery has finished.
16406	**/
16407	static void i40e_pci_error_resume(struct pci_dev *pdev)
16408	{
16409	struct i40e_pf *pf = pci_get_drvdata(pdev);
16410
16411	dev_dbg(&pdev->dev, "%s\n", __func__);
16412	if (test_bit(__I40E_SUSPENDED, pf->state))
16413	return;
16414
16415	i40e_handle_reset_warning(pf, lock_acquired: false);
16416	}
16417
16418	/**
16419	* i40e_enable_mc_magic_wake - enable multicast magic packet wake up
16420	* using the mac_address_write admin q function
16421	* @pf: pointer to i40e_pf struct
16422	**/
16423	static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
16424	{
16425	struct i40e_hw *hw = &pf->hw;
16426	u8 mac_addr[6];
16427	u16 flags = 0;
16428	int ret;
16429
16430	/* Get current MAC address in case it's an LAA */
16431	if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
16432	ether_addr_copy(dst: mac_addr,
16433	src: pf->vsi[pf->lan_vsi]->netdev->dev_addr);
16434	} else {
16435	dev_err(&pf->pdev->dev,
16436	"Failed to retrieve MAC address; using default\n");
16437	ether_addr_copy(dst: mac_addr, src: hw->mac.addr);
16438	}
16439
16440	/* The FW expects the mac address write cmd to first be called with
16441	* one of these flags before calling it again with the multicast
16442	* enable flags.
16443	*/
16444	flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
16445
16446	if (hw->func_caps.flex10_enable && hw->partition_id != 1)
16447	flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
16448
16449	ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16450	if (ret) {
16451	dev_err(&pf->pdev->dev,
16452	"Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
16453	return;
16454	}
16455
16456	flags = I40E_AQC_MC_MAG_EN
16457	| I40E_AQC_WOL_PRESERVE_ON_PFR
16458	| I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
16459	ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16460	if (ret)
16461	dev_err(&pf->pdev->dev,
16462	"Failed to enable Multicast Magic Packet wake up\n");
16463	}
16464
16465	/**
16466	* i40e_shutdown - PCI callback for shutting down
16467	* @pdev: PCI device information struct
16468	**/
16469	static void i40e_shutdown(struct pci_dev *pdev)
16470	{
16471	struct i40e_pf *pf = pci_get_drvdata(pdev);
16472	struct i40e_hw *hw = &pf->hw;
16473
16474	set_bit(nr: __I40E_SUSPENDED, addr: pf->state);
16475	set_bit(nr: __I40E_DOWN, addr: pf->state);
16476
16477	del_timer_sync(timer: &pf->service_timer);
16478	cancel_work_sync(work: &pf->service_task);
16479	i40e_cloud_filter_exit(pf);
16480	i40e_fdir_teardown(pf);
16481
16482	/* Client close must be called explicitly here because the timer
16483	* has been stopped.
16484	*/
16485	i40e_notify_client_of_netdev_close(vsi: pf->vsi[pf->lan_vsi], reset: false);
16486
16487	if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&
16488	pf->wol_en)
16489	i40e_enable_mc_magic_wake(pf);
16490
16491	i40e_prep_for_reset(pf);
16492
16493	wr32(hw, I40E_PFPM_APM,
16494	(pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16495	wr32(hw, I40E_PFPM_WUFC,
16496	(pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16497
16498	/* Free MSI/legacy interrupt 0 when in recovery mode. */
16499	if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16500	!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
16501	free_irq(pf->pdev->irq, pf);
16502
16503	/* Since we're going to destroy queues during the
16504	* i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16505	* whole section
16506	*/
16507	rtnl_lock();
16508	i40e_clear_interrupt_scheme(pf);
16509	rtnl_unlock();
16510
16511	if (system_state == SYSTEM_POWER_OFF) {
16512	pci_wake_from_d3(dev: pdev, enable: pf->wol_en);
16513	pci_set_power_state(dev: pdev, PCI_D3hot);
16514	}
16515	}
16516
16517	/**
16518	* i40e_suspend - PM callback for moving to D3
16519	* @dev: generic device information structure
16520	**/
16521	static int __maybe_unused i40e_suspend(struct device *dev)
16522	{
16523	struct i40e_pf *pf = dev_get_drvdata(dev);
16524	struct i40e_hw *hw = &pf->hw;
16525
16526	/* If we're already suspended, then there is nothing to do */
16527	if (test_and_set_bit(nr: __I40E_SUSPENDED, addr: pf->state))
16528	return 0;
16529
16530	set_bit(nr: __I40E_DOWN, addr: pf->state);
16531
16532	/* Ensure service task will not be running */
16533	del_timer_sync(timer: &pf->service_timer);
16534	cancel_work_sync(work: &pf->service_task);
16535
16536	/* Client close must be called explicitly here because the timer
16537	* has been stopped.
16538	*/
16539	i40e_notify_client_of_netdev_close(vsi: pf->vsi[pf->lan_vsi], reset: false);
16540
16541	if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&
16542	pf->wol_en)
16543	i40e_enable_mc_magic_wake(pf);
16544
16545	/* Since we're going to destroy queues during the
16546	* i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16547	* whole section
16548	*/
16549	rtnl_lock();
16550
16551	i40e_prep_for_reset(pf);
16552
16553	wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16554	wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16555
16556	/* Clear the interrupt scheme and release our IRQs so that the system
16557	* can safely hibernate even when there are a large number of CPUs.
16558	* Otherwise hibernation might fail when mapping all the vectors back
16559	* to CPU0.
16560	*/
16561	i40e_clear_interrupt_scheme(pf);
16562
16563	rtnl_unlock();
16564
16565	return 0;
16566	}
16567
16568	/**
16569	* i40e_resume - PM callback for waking up from D3
16570	* @dev: generic device information structure
16571	**/
16572	static int __maybe_unused i40e_resume(struct device *dev)
16573	{
16574	struct i40e_pf *pf = dev_get_drvdata(dev);
16575	int err;
16576
16577	/* If we're not suspended, then there is nothing to do */
16578	if (!test_bit(__I40E_SUSPENDED, pf->state))
16579	return 0;
16580
16581	/* We need to hold the RTNL lock prior to restoring interrupt schemes,
16582	* since we're going to be restoring queues
16583	*/
16584	rtnl_lock();
16585
16586	/* We cleared the interrupt scheme when we suspended, so we need to
16587	* restore it now to resume device functionality.
16588	*/
16589	err = i40e_restore_interrupt_scheme(pf);
16590	if (err) {
16591	dev_err(dev, "Cannot restore interrupt scheme: %d\n",
16592	err);
16593	}
16594
16595	clear_bit(nr: __I40E_DOWN, addr: pf->state);
16596	i40e_reset_and_rebuild(pf, reinit: false, lock_acquired: true);
16597
16598	rtnl_unlock();
16599
16600	/* Clear suspended state last after everything is recovered */
16601	clear_bit(nr: __I40E_SUSPENDED, addr: pf->state);
16602
16603	/* Restart the service task */
16604	mod_timer(timer: &pf->service_timer,
16605	expires: round_jiffies(j: jiffies + pf->service_timer_period));
16606
16607	return 0;
16608	}
16609
16610	static const struct pci_error_handlers i40e_err_handler = {
16611	.error_detected = i40e_pci_error_detected,
16612	.slot_reset = i40e_pci_error_slot_reset,
16613	.reset_prepare = i40e_pci_error_reset_prepare,
16614	.reset_done = i40e_pci_error_reset_done,
16615	.resume = i40e_pci_error_resume,
16616	};
16617
16618	static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
16619
16620	static struct pci_driver i40e_driver = {
16621	.name = i40e_driver_name,
16622	.id_table = i40e_pci_tbl,
16623	.probe = i40e_probe,
16624	.remove = i40e_remove,
16625	.driver = {
16626	.pm = &i40e_pm_ops,
16627	},
16628	.shutdown = i40e_shutdown,
16629	.err_handler = &i40e_err_handler,
16630	.sriov_configure = i40e_pci_sriov_configure,
16631	};
16632
16633	/**
16634	* i40e_init_module - Driver registration routine
16635	*
16636	* i40e_init_module is the first routine called when the driver is
16637	* loaded. All it does is register with the PCI subsystem.
16638	**/
16639	static int __init i40e_init_module(void)
16640	{
16641	int err;
16642
16643	pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string);
16644	pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
16645
16646	/* There is no need to throttle the number of active tasks because
16647	* each device limits its own task using a state bit for scheduling
16648	* the service task, and the device tasks do not interfere with each
16649	* other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
16650	* since we need to be able to guarantee forward progress even under
16651	* memory pressure.
16652	*/
16653	i40e_wq = alloc_workqueue(fmt: "%s", flags: WQ_MEM_RECLAIM, max_active: 0, i40e_driver_name);
16654	if (!i40e_wq) {
16655	pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
16656	return -ENOMEM;
16657	}
16658
16659	i40e_dbg_init();
16660	err = pci_register_driver(&i40e_driver);
16661	if (err) {
16662	destroy_workqueue(wq: i40e_wq);
16663	i40e_dbg_exit();
16664	return err;
16665	}
16666
16667	return 0;
16668	}
16669	module_init(i40e_init_module);
16670
16671	/**
16672	* i40e_exit_module - Driver exit cleanup routine
16673	*
16674	* i40e_exit_module is called just before the driver is removed
16675	* from memory.
16676	**/
16677	static void __exit i40e_exit_module(void)
16678	{
16679	pci_unregister_driver(dev: &i40e_driver);
16680	destroy_workqueue(wq: i40e_wq);
16681	ida_destroy(ida: &i40e_client_ida);
16682	i40e_dbg_exit();
16683	}
16684	module_exit(i40e_exit_module);
16685