1	// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2	/* QLogic qede NIC Driver
3	* Copyright (c) 2015-2017 QLogic Corporation
4	* Copyright (c) 2019-2020 Marvell International Ltd.
5	*/
6
7	#include <linux/crash_dump.h>
8	#include <linux/module.h>
9	#include <linux/pci.h>
10	#include <linux/device.h>
11	#include <linux/netdevice.h>
12	#include <linux/etherdevice.h>
13	#include <linux/skbuff.h>
14	#include <linux/errno.h>
15	#include <linux/list.h>
16	#include <linux/string.h>
17	#include <linux/dma-mapping.h>
18	#include <linux/interrupt.h>
19	#include <asm/byteorder.h>
20	#include <asm/param.h>
21	#include <linux/io.h>
22	#include <linux/netdev_features.h>
23	#include <linux/udp.h>
24	#include <linux/tcp.h>
25	#include <net/udp_tunnel.h>
26	#include <linux/ip.h>
27	#include <net/ipv6.h>
28	#include <net/tcp.h>
29	#include <linux/if_ether.h>
30	#include <linux/if_vlan.h>
31	#include <linux/pkt_sched.h>
32	#include <linux/ethtool.h>
33	#include <linux/in.h>
34	#include <linux/random.h>
35	#include <net/ip6_checksum.h>
36	#include <linux/bitops.h>
37	#include <linux/vmalloc.h>
38	#include "qede.h"
39	#include "qede_ptp.h"
40
41	MODULE_DESCRIPTION("QLogic FastLinQ 4xxxx Ethernet Driver");
42	MODULE_LICENSE("GPL");
43
44	static uint debug;
45	module_param(debug, uint, 0);
46	MODULE_PARM_DESC(debug, " Default debug msglevel");
47
48	static const struct qed_eth_ops *qed_ops;
49
50	#define CHIP_NUM_57980S_40 0x1634
51	#define CHIP_NUM_57980S_10 0x1666
52	#define CHIP_NUM_57980S_MF 0x1636
53	#define CHIP_NUM_57980S_100 0x1644
54	#define CHIP_NUM_57980S_50 0x1654
55	#define CHIP_NUM_57980S_25 0x1656
56	#define CHIP_NUM_57980S_IOV 0x1664
57	#define CHIP_NUM_AH 0x8070
58	#define CHIP_NUM_AH_IOV 0x8090
59
60	#ifndef PCI_DEVICE_ID_NX2_57980E
61	#define PCI_DEVICE_ID_57980S_40 CHIP_NUM_57980S_40
62	#define PCI_DEVICE_ID_57980S_10 CHIP_NUM_57980S_10
63	#define PCI_DEVICE_ID_57980S_MF CHIP_NUM_57980S_MF
64	#define PCI_DEVICE_ID_57980S_100 CHIP_NUM_57980S_100
65	#define PCI_DEVICE_ID_57980S_50 CHIP_NUM_57980S_50
66	#define PCI_DEVICE_ID_57980S_25 CHIP_NUM_57980S_25
67	#define PCI_DEVICE_ID_57980S_IOV CHIP_NUM_57980S_IOV
68	#define PCI_DEVICE_ID_AH CHIP_NUM_AH
69	#define PCI_DEVICE_ID_AH_IOV CHIP_NUM_AH_IOV
70
71	#endif
72
73	enum qede_pci_private {
74	QEDE_PRIVATE_PF,
75	QEDE_PRIVATE_VF
76	};
77
78	static const struct pci_device_id qede_pci_tbl[] = {
79	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_40), QEDE_PRIVATE_PF},
80	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_10), QEDE_PRIVATE_PF},
81	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_MF), QEDE_PRIVATE_PF},
82	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_100), QEDE_PRIVATE_PF},
83	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_50), QEDE_PRIVATE_PF},
84	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_25), QEDE_PRIVATE_PF},
85	#ifdef CONFIG_QED_SRIOV
86	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_IOV), QEDE_PRIVATE_VF},
87	#endif
88	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_AH), QEDE_PRIVATE_PF},
89	#ifdef CONFIG_QED_SRIOV
90	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_AH_IOV), QEDE_PRIVATE_VF},
91	#endif
92	{ 0 }
93	};
94
95	MODULE_DEVICE_TABLE(pci, qede_pci_tbl);
96
97	static int qede_probe(struct pci_dev *pdev, const struct pci_device_id *id);
98	static pci_ers_result_t
99	qede_io_error_detected(struct pci_dev *pdev, pci_channel_state_t state);
100
101	#define TX_TIMEOUT (5 * HZ)
102
103	/* Utilize last protocol index for XDP */
104	#define XDP_PI 11
105
106	static void qede_remove(struct pci_dev *pdev);
107	static void qede_shutdown(struct pci_dev *pdev);
108	static void qede_link_update(void *dev, struct qed_link_output *link);
109	static void qede_schedule_recovery_handler(void *dev);
110	static void qede_recovery_handler(struct qede_dev *edev);
111	static void qede_schedule_hw_err_handler(void *dev,
112	enum qed_hw_err_type err_type);
113	static void qede_get_eth_tlv_data(void *edev, void *data);
114	static void qede_get_generic_tlv_data(void *edev,
115	struct qed_generic_tlvs *data);
116	static void qede_generic_hw_err_handler(struct qede_dev *edev);
117	#ifdef CONFIG_QED_SRIOV
118	static int qede_set_vf_vlan(struct net_device *ndev, int vf, u16 vlan, u8 qos,
119	__be16 vlan_proto)
120	{
121	struct qede_dev *edev = netdev_priv(dev: ndev);
122
123	if (vlan > 4095) {
124	DP_NOTICE(edev, "Illegal vlan value %d\n", vlan);
125	return -EINVAL;
126	}
127
128	if (vlan_proto != htons(ETH_P_8021Q))
129	return -EPROTONOSUPPORT;
130
131	DP_VERBOSE(edev, QED_MSG_IOV, "Setting Vlan 0x%04x to VF [%d]\n",
132	vlan, vf);
133
134	return edev->ops->iov->set_vlan(edev->cdev, vlan, vf);
135	}
136
137	static int qede_set_vf_mac(struct net_device *ndev, int vfidx, u8 *mac)
138	{
139	struct qede_dev *edev = netdev_priv(dev: ndev);
140
141	DP_VERBOSE(edev, QED_MSG_IOV, "Setting MAC %pM to VF [%d]\n", mac, vfidx);
142
143	if (!is_valid_ether_addr(addr: mac)) {
144	DP_VERBOSE(edev, QED_MSG_IOV, "MAC address isn't valid\n");
145	return -EINVAL;
146	}
147
148	return edev->ops->iov->set_mac(edev->cdev, mac, vfidx);
149	}
150
151	static int qede_sriov_configure(struct pci_dev *pdev, int num_vfs_param)
152	{
153	struct qede_dev *edev = netdev_priv(dev: pci_get_drvdata(pdev));
154	struct qed_dev_info *qed_info = &edev->dev_info.common;
155	struct qed_update_vport_params *vport_params;
156	int rc;
157
158	vport_params = vzalloc(size: sizeof(*vport_params));
159	if (!vport_params)
160	return -ENOMEM;
161	DP_VERBOSE(edev, QED_MSG_IOV, "Requested %d VFs\n", num_vfs_param);
162
163	rc = edev->ops->iov->configure(edev->cdev, num_vfs_param);
164
165	/* Enable/Disable Tx switching for PF */
166	if ((rc == num_vfs_param) && netif_running(dev: edev->ndev) &&
167	!qed_info->b_inter_pf_switch && qed_info->tx_switching) {
168	vport_params->vport_id = 0;
169	vport_params->update_tx_switching_flg = 1;
170	vport_params->tx_switching_flg = num_vfs_param ? 1 : 0;
171	edev->ops->vport_update(edev->cdev, vport_params);
172	}
173
174	vfree(addr: vport_params);
175	return rc;
176	}
177	#endif
178
179	static int __maybe_unused qede_suspend(struct device *dev)
180	{
181	dev_info(dev, "Device does not support suspend operation\n");
182
183	return -EOPNOTSUPP;
184	}
185
186	static DEFINE_SIMPLE_DEV_PM_OPS(qede_pm_ops, qede_suspend, NULL);
187
188	static const struct pci_error_handlers qede_err_handler = {
189	.error_detected = qede_io_error_detected,
190	};
191
192	static struct pci_driver qede_pci_driver = {
193	.name = "qede",
194	.id_table = qede_pci_tbl,
195	.probe = qede_probe,
196	.remove = qede_remove,
197	.shutdown = qede_shutdown,
198	#ifdef CONFIG_QED_SRIOV
199	.sriov_configure = qede_sriov_configure,
200	#endif
201	.err_handler = &qede_err_handler,
202	.driver.pm = &qede_pm_ops,
203	};
204
205	static struct qed_eth_cb_ops qede_ll_ops = {
206	{
207	#ifdef CONFIG_RFS_ACCEL
208	.arfs_filter_op = qede_arfs_filter_op,
209	#endif
210	.link_update = qede_link_update,
211	.schedule_recovery_handler = qede_schedule_recovery_handler,
212	.schedule_hw_err_handler = qede_schedule_hw_err_handler,
213	.get_generic_tlv_data = qede_get_generic_tlv_data,
214	.get_protocol_tlv_data = qede_get_eth_tlv_data,
215	},
216	.force_mac = qede_force_mac,
217	.ports_update = qede_udp_ports_update,
218	};
219
220	static int qede_netdev_event(struct notifier_block *this, unsigned long event,
221	void *ptr)
222	{
223	struct net_device *ndev = netdev_notifier_info_to_dev(info: ptr);
224	struct ethtool_drvinfo drvinfo;
225	struct qede_dev *edev;
226
227	if (event != NETDEV_CHANGENAME && event != NETDEV_CHANGEADDR)
228	goto done;
229
230	/* Check whether this is a qede device */
231	if (!ndev || !ndev->ethtool_ops || !ndev->ethtool_ops->get_drvinfo)
232	goto done;
233
234	memset(&drvinfo, 0, sizeof(drvinfo));
235	ndev->ethtool_ops->get_drvinfo(ndev, &drvinfo);
236	if (strcmp(drvinfo.driver, "qede"))
237	goto done;
238	edev = netdev_priv(dev: ndev);
239
240	switch (event) {
241	case NETDEV_CHANGENAME:
242	/* Notify qed of the name change */
243	if (!edev->ops || !edev->ops->common)
244	goto done;
245	edev->ops->common->set_name(edev->cdev, edev->ndev->name);
246	break;
247	case NETDEV_CHANGEADDR:
248	edev = netdev_priv(dev: ndev);
249	qede_rdma_event_changeaddr(edr: edev);
250	break;
251	}
252
253	done:
254	return NOTIFY_DONE;
255	}
256
257	static struct notifier_block qede_netdev_notifier = {
258	.notifier_call = qede_netdev_event,
259	};
260
261	static
262	int __init qede_init(void)
263	{
264	int ret;
265
266	pr_info("qede init: QLogic FastLinQ 4xxxx Ethernet Driver qede\n");
267
268	qede_forced_speed_maps_init();
269
270	qed_ops = qed_get_eth_ops();
271	if (!qed_ops) {
272	pr_notice("Failed to get qed ethtool operations\n");
273	return -EINVAL;
274	}
275
276	/* Must register notifier before pci ops, since we might miss
277	* interface rename after pci probe and netdev registration.
278	*/
279	ret = register_netdevice_notifier(nb: &qede_netdev_notifier);
280	if (ret) {
281	pr_notice("Failed to register netdevice_notifier\n");
282	qed_put_eth_ops();
283	return -EINVAL;
284	}
285
286	ret = pci_register_driver(&qede_pci_driver);
287	if (ret) {
288	pr_notice("Failed to register driver\n");
289	unregister_netdevice_notifier(nb: &qede_netdev_notifier);
290	qed_put_eth_ops();
291	return -EINVAL;
292	}
293
294	return 0;
295	}
296
297	static void __exit qede_cleanup(void)
298	{
299	if (debug & QED_LOG_INFO_MASK)
300	pr_info("qede_cleanup called\n");
301
302	unregister_netdevice_notifier(nb: &qede_netdev_notifier);
303	pci_unregister_driver(dev: &qede_pci_driver);
304	qed_put_eth_ops();
305	}
306
307	module_init(qede_init);
308	module_exit(qede_cleanup);
309
310	static int qede_open(struct net_device *ndev);
311	static int qede_close(struct net_device *ndev);
312
313	void qede_fill_by_demand_stats(struct qede_dev *edev)
314	{
315	struct qede_stats_common *p_common = &edev->stats.common;
316	struct qed_eth_stats stats;
317
318	edev->ops->get_vport_stats(edev->cdev, &stats);
319
320	spin_lock(lock: &edev->stats_lock);
321
322	p_common->no_buff_discards = stats.common.no_buff_discards;
323	p_common->packet_too_big_discard = stats.common.packet_too_big_discard;
324	p_common->ttl0_discard = stats.common.ttl0_discard;
325	p_common->rx_ucast_bytes = stats.common.rx_ucast_bytes;
326	p_common->rx_mcast_bytes = stats.common.rx_mcast_bytes;
327	p_common->rx_bcast_bytes = stats.common.rx_bcast_bytes;
328	p_common->rx_ucast_pkts = stats.common.rx_ucast_pkts;
329	p_common->rx_mcast_pkts = stats.common.rx_mcast_pkts;
330	p_common->rx_bcast_pkts = stats.common.rx_bcast_pkts;
331	p_common->mftag_filter_discards = stats.common.mftag_filter_discards;
332	p_common->mac_filter_discards = stats.common.mac_filter_discards;
333	p_common->gft_filter_drop = stats.common.gft_filter_drop;
334
335	p_common->tx_ucast_bytes = stats.common.tx_ucast_bytes;
336	p_common->tx_mcast_bytes = stats.common.tx_mcast_bytes;
337	p_common->tx_bcast_bytes = stats.common.tx_bcast_bytes;
338	p_common->tx_ucast_pkts = stats.common.tx_ucast_pkts;
339	p_common->tx_mcast_pkts = stats.common.tx_mcast_pkts;
340	p_common->tx_bcast_pkts = stats.common.tx_bcast_pkts;
341	p_common->tx_err_drop_pkts = stats.common.tx_err_drop_pkts;
342	p_common->coalesced_pkts = stats.common.tpa_coalesced_pkts;
343	p_common->coalesced_events = stats.common.tpa_coalesced_events;
344	p_common->coalesced_aborts_num = stats.common.tpa_aborts_num;
345	p_common->non_coalesced_pkts = stats.common.tpa_not_coalesced_pkts;
346	p_common->coalesced_bytes = stats.common.tpa_coalesced_bytes;
347
348	p_common->rx_64_byte_packets = stats.common.rx_64_byte_packets;
349	p_common->rx_65_to_127_byte_packets =
350	stats.common.rx_65_to_127_byte_packets;
351	p_common->rx_128_to_255_byte_packets =
352	stats.common.rx_128_to_255_byte_packets;
353	p_common->rx_256_to_511_byte_packets =
354	stats.common.rx_256_to_511_byte_packets;
355	p_common->rx_512_to_1023_byte_packets =
356	stats.common.rx_512_to_1023_byte_packets;
357	p_common->rx_1024_to_1518_byte_packets =
358	stats.common.rx_1024_to_1518_byte_packets;
359	p_common->rx_crc_errors = stats.common.rx_crc_errors;
360	p_common->rx_mac_crtl_frames = stats.common.rx_mac_crtl_frames;
361	p_common->rx_pause_frames = stats.common.rx_pause_frames;
362	p_common->rx_pfc_frames = stats.common.rx_pfc_frames;
363	p_common->rx_align_errors = stats.common.rx_align_errors;
364	p_common->rx_carrier_errors = stats.common.rx_carrier_errors;
365	p_common->rx_oversize_packets = stats.common.rx_oversize_packets;
366	p_common->rx_jabbers = stats.common.rx_jabbers;
367	p_common->rx_undersize_packets = stats.common.rx_undersize_packets;
368	p_common->rx_fragments = stats.common.rx_fragments;
369	p_common->tx_64_byte_packets = stats.common.tx_64_byte_packets;
370	p_common->tx_65_to_127_byte_packets =
371	stats.common.tx_65_to_127_byte_packets;
372	p_common->tx_128_to_255_byte_packets =
373	stats.common.tx_128_to_255_byte_packets;
374	p_common->tx_256_to_511_byte_packets =
375	stats.common.tx_256_to_511_byte_packets;
376	p_common->tx_512_to_1023_byte_packets =
377	stats.common.tx_512_to_1023_byte_packets;
378	p_common->tx_1024_to_1518_byte_packets =
379	stats.common.tx_1024_to_1518_byte_packets;
380	p_common->tx_pause_frames = stats.common.tx_pause_frames;
381	p_common->tx_pfc_frames = stats.common.tx_pfc_frames;
382	p_common->brb_truncates = stats.common.brb_truncates;
383	p_common->brb_discards = stats.common.brb_discards;
384	p_common->tx_mac_ctrl_frames = stats.common.tx_mac_ctrl_frames;
385	p_common->link_change_count = stats.common.link_change_count;
386	p_common->ptp_skip_txts = edev->ptp_skip_txts;
387
388	if (QEDE_IS_BB(edev)) {
389	struct qede_stats_bb *p_bb = &edev->stats.bb;
390
391	p_bb->rx_1519_to_1522_byte_packets =
392	stats.bb.rx_1519_to_1522_byte_packets;
393	p_bb->rx_1519_to_2047_byte_packets =
394	stats.bb.rx_1519_to_2047_byte_packets;
395	p_bb->rx_2048_to_4095_byte_packets =
396	stats.bb.rx_2048_to_4095_byte_packets;
397	p_bb->rx_4096_to_9216_byte_packets =
398	stats.bb.rx_4096_to_9216_byte_packets;
399	p_bb->rx_9217_to_16383_byte_packets =
400	stats.bb.rx_9217_to_16383_byte_packets;
401	p_bb->tx_1519_to_2047_byte_packets =
402	stats.bb.tx_1519_to_2047_byte_packets;
403	p_bb->tx_2048_to_4095_byte_packets =
404	stats.bb.tx_2048_to_4095_byte_packets;
405	p_bb->tx_4096_to_9216_byte_packets =
406	stats.bb.tx_4096_to_9216_byte_packets;
407	p_bb->tx_9217_to_16383_byte_packets =
408	stats.bb.tx_9217_to_16383_byte_packets;
409	p_bb->tx_lpi_entry_count = stats.bb.tx_lpi_entry_count;
410	p_bb->tx_total_collisions = stats.bb.tx_total_collisions;
411	} else {
412	struct qede_stats_ah *p_ah = &edev->stats.ah;
413
414	p_ah->rx_1519_to_max_byte_packets =
415	stats.ah.rx_1519_to_max_byte_packets;
416	p_ah->tx_1519_to_max_byte_packets =
417	stats.ah.tx_1519_to_max_byte_packets;
418	}
419
420	spin_unlock(lock: &edev->stats_lock);
421	}
422
423	static void qede_get_stats64(struct net_device *dev,
424	struct rtnl_link_stats64 *stats)
425	{
426	struct qede_dev *edev = netdev_priv(dev);
427	struct qede_stats_common *p_common;
428
429	p_common = &edev->stats.common;
430
431	spin_lock(lock: &edev->stats_lock);
432
433	stats->rx_packets = p_common->rx_ucast_pkts + p_common->rx_mcast_pkts +
434	p_common->rx_bcast_pkts;
435	stats->tx_packets = p_common->tx_ucast_pkts + p_common->tx_mcast_pkts +
436	p_common->tx_bcast_pkts;
437
438	stats->rx_bytes = p_common->rx_ucast_bytes + p_common->rx_mcast_bytes +
439	p_common->rx_bcast_bytes;
440	stats->tx_bytes = p_common->tx_ucast_bytes + p_common->tx_mcast_bytes +
441	p_common->tx_bcast_bytes;
442
443	stats->tx_errors = p_common->tx_err_drop_pkts;
444	stats->multicast = p_common->rx_mcast_pkts + p_common->rx_bcast_pkts;
445
446	stats->rx_fifo_errors = p_common->no_buff_discards;
447
448	if (QEDE_IS_BB(edev))
449	stats->collisions = edev->stats.bb.tx_total_collisions;
450	stats->rx_crc_errors = p_common->rx_crc_errors;
451	stats->rx_frame_errors = p_common->rx_align_errors;
452
453	spin_unlock(lock: &edev->stats_lock);
454	}
455
456	#ifdef CONFIG_QED_SRIOV
457	static int qede_get_vf_config(struct net_device *dev, int vfidx,
458	struct ifla_vf_info *ivi)
459	{
460	struct qede_dev *edev = netdev_priv(dev);
461
462	if (!edev->ops)
463	return -EINVAL;
464
465	return edev->ops->iov->get_config(edev->cdev, vfidx, ivi);
466	}
467
468	static int qede_set_vf_rate(struct net_device *dev, int vfidx,
469	int min_tx_rate, int max_tx_rate)
470	{
471	struct qede_dev *edev = netdev_priv(dev);
472
473	return edev->ops->iov->set_rate(edev->cdev, vfidx, min_tx_rate,
474	max_tx_rate);
475	}
476
477	static int qede_set_vf_spoofchk(struct net_device *dev, int vfidx, bool val)
478	{
479	struct qede_dev *edev = netdev_priv(dev);
480
481	if (!edev->ops)
482	return -EINVAL;
483
484	return edev->ops->iov->set_spoof(edev->cdev, vfidx, val);
485	}
486
487	static int qede_set_vf_link_state(struct net_device *dev, int vfidx,
488	int link_state)
489	{
490	struct qede_dev *edev = netdev_priv(dev);
491
492	if (!edev->ops)
493	return -EINVAL;
494
495	return edev->ops->iov->set_link_state(edev->cdev, vfidx, link_state);
496	}
497
498	static int qede_set_vf_trust(struct net_device *dev, int vfidx, bool setting)
499	{
500	struct qede_dev *edev = netdev_priv(dev);
501
502	if (!edev->ops)
503	return -EINVAL;
504
505	return edev->ops->iov->set_trust(edev->cdev, vfidx, setting);
506	}
507	#endif
508
509	static int qede_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
510	{
511	struct qede_dev *edev = netdev_priv(dev);
512
513	if (!netif_running(dev))
514	return -EAGAIN;
515
516	switch (cmd) {
517	case SIOCSHWTSTAMP:
518	return qede_ptp_hw_ts(edev, req: ifr);
519	default:
520	DP_VERBOSE(edev, QED_MSG_DEBUG,
521	"default IOCTL cmd 0x%x\n", cmd);
522	return -EOPNOTSUPP;
523	}
524
525	return 0;
526	}
527
528	static void qede_fp_sb_dump(struct qede_dev *edev, struct qede_fastpath *fp)
529	{
530	char *p_sb = (char *)fp->sb_info->sb_virt;
531	u32 sb_size, i;
532
533	sb_size = sizeof(struct status_block);
534
535	for (i = 0; i < sb_size; i += 8)
536	DP_NOTICE(edev,
537	"%02hhX %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX\n",
538	p_sb[i], p_sb[i + 1], p_sb[i + 2], p_sb[i + 3],
539	p_sb[i + 4], p_sb[i + 5], p_sb[i + 6], p_sb[i + 7]);
540	}
541
542	static void
543	qede_txq_fp_log_metadata(struct qede_dev *edev,
544	struct qede_fastpath *fp, struct qede_tx_queue *txq)
545	{
546	struct qed_chain *p_chain = &txq->tx_pbl;
547
548	/* Dump txq/fp/sb ids etc. other metadata */
549	DP_NOTICE(edev,
550	"fpid 0x%x sbid 0x%x txqid [0x%x] ndev_qid [0x%x] cos [0x%x] p_chain %p cap %d size %d jiffies %lu HZ 0x%x\n",
551	fp->id, fp->sb_info->igu_sb_id, txq->index, txq->ndev_txq_id, txq->cos,
552	p_chain, p_chain->capacity, p_chain->size, jiffies, HZ);
553
554	/* Dump all the relevant prod/cons indexes */
555	DP_NOTICE(edev,
556	"hw cons %04x sw_tx_prod=0x%x, sw_tx_cons=0x%x, bd_prod 0x%x bd_cons 0x%x\n",
557	le16_to_cpu(*txq->hw_cons_ptr), txq->sw_tx_prod, txq->sw_tx_cons,
558	qed_chain_get_prod_idx(p_chain), qed_chain_get_cons_idx(p_chain));
559	}
560
561	static void
562	qede_tx_log_print(struct qede_dev *edev, struct qede_fastpath *fp, struct qede_tx_queue *txq)
563	{
564	struct qed_sb_info_dbg sb_dbg;
565	int rc;
566
567	/* sb info */
568	qede_fp_sb_dump(edev, fp);
569
570	memset(&sb_dbg, 0, sizeof(sb_dbg));
571	rc = edev->ops->common->get_sb_info(edev->cdev, fp->sb_info, (u16)fp->id, &sb_dbg);
572
573	DP_NOTICE(edev, "IGU: prod %08x cons %08x CAU Tx %04x\n",
574	sb_dbg.igu_prod, sb_dbg.igu_cons, sb_dbg.pi[TX_PI(txq->cos)]);
575
576	/* report to mfw */
577	edev->ops->common->mfw_report(edev->cdev,
578	"Txq[%d]: FW cons [host] %04x, SW cons %04x, SW prod %04x [Jiffies %lu]\n",
579	txq->index, le16_to_cpu(*txq->hw_cons_ptr),
580	qed_chain_get_cons_idx(chain: &txq->tx_pbl),
581	qed_chain_get_prod_idx(chain: &txq->tx_pbl), jiffies);
582	if (!rc)
583	edev->ops->common->mfw_report(edev->cdev,
584	"Txq[%d]: SB[0x%04x] - IGU: prod %08x cons %08x CAU Tx %04x\n",
585	txq->index, fp->sb_info->igu_sb_id,
586	sb_dbg.igu_prod, sb_dbg.igu_cons,
587	sb_dbg.pi[TX_PI(txq->cos)]);
588	}
589
590	static void qede_tx_timeout(struct net_device *dev, unsigned int txqueue)
591	{
592	struct qede_dev *edev = netdev_priv(dev);
593	int i;
594
595	netif_carrier_off(dev);
596	DP_NOTICE(edev, "TX timeout on queue %u!\n", txqueue);
597
598	for_each_queue(i) {
599	struct qede_tx_queue *txq;
600	struct qede_fastpath *fp;
601	int cos;
602
603	fp = &edev->fp_array[i];
604	if (!(fp->type & QEDE_FASTPATH_TX))
605	continue;
606
607	for_each_cos_in_txq(edev, cos) {
608	txq = &fp->txq[cos];
609
610	/* Dump basic metadata for all queues */
611	qede_txq_fp_log_metadata(edev, fp, txq);
612
613	if (qed_chain_get_cons_idx(chain: &txq->tx_pbl) !=
614	qed_chain_get_prod_idx(chain: &txq->tx_pbl))
615	qede_tx_log_print(edev, fp, txq);
616	}
617	}
618
619	if (IS_VF(edev))
620	return;
621
622	if (test_and_set_bit(QEDE_ERR_IS_HANDLED, addr: &edev->err_flags) ||
623	edev->state == QEDE_STATE_RECOVERY) {
624	DP_INFO(edev,
625	"Avoid handling a Tx timeout while another HW error is being handled\n");
626	return;
627	}
628
629	set_bit(QEDE_ERR_GET_DBG_INFO, addr: &edev->err_flags);
630	set_bit(QEDE_SP_HW_ERR, addr: &edev->sp_flags);
631	schedule_delayed_work(dwork: &edev->sp_task, delay: 0);
632	}
633
634	static int qede_setup_tc(struct net_device *ndev, u8 num_tc)
635	{
636	struct qede_dev *edev = netdev_priv(dev: ndev);
637	int cos, count, offset;
638
639	if (num_tc > edev->dev_info.num_tc)
640	return -EINVAL;
641
642	netdev_reset_tc(dev: ndev);
643	netdev_set_num_tc(dev: ndev, num_tc);
644
645	for_each_cos_in_txq(edev, cos) {
646	count = QEDE_TSS_COUNT(edev);
647	offset = cos * QEDE_TSS_COUNT(edev);
648	netdev_set_tc_queue(dev: ndev, tc: cos, count, offset);
649	}
650
651	return 0;
652	}
653
654	static int
655	qede_set_flower(struct qede_dev *edev, struct flow_cls_offload *f,
656	__be16 proto)
657	{
658	switch (f->command) {
659	case FLOW_CLS_REPLACE:
660	return qede_add_tc_flower_fltr(edev, proto, f);
661	case FLOW_CLS_DESTROY:
662	return qede_delete_flow_filter(edev, cookie: f->cookie);
663	default:
664	return -EOPNOTSUPP;
665	}
666	}
667
668	static int qede_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
669	void *cb_priv)
670	{
671	struct flow_cls_offload *f;
672	struct qede_dev *edev = cb_priv;
673
674	if (!tc_cls_can_offload_and_chain0(dev: edev->ndev, common: type_data))
675	return -EOPNOTSUPP;
676
677	switch (type) {
678	case TC_SETUP_CLSFLOWER:
679	f = type_data;
680	return qede_set_flower(edev, f, proto: f->common.protocol);
681	default:
682	return -EOPNOTSUPP;
683	}
684	}
685
686	static LIST_HEAD(qede_block_cb_list);
687
688	static int
689	qede_setup_tc_offload(struct net_device *dev, enum tc_setup_type type,
690	void *type_data)
691	{
692	struct qede_dev *edev = netdev_priv(dev);
693	struct tc_mqprio_qopt *mqprio;
694
695	switch (type) {
696	case TC_SETUP_BLOCK:
697	return flow_block_cb_setup_simple(f: type_data,
698	driver_list: &qede_block_cb_list,
699	cb: qede_setup_tc_block_cb,
700	cb_ident: edev, cb_priv: edev, ingress_only: true);
701	case TC_SETUP_QDISC_MQPRIO:
702	mqprio = type_data;
703
704	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
705	return qede_setup_tc(ndev: dev, num_tc: mqprio->num_tc);
706	default:
707	return -EOPNOTSUPP;
708	}
709	}
710
711	static const struct net_device_ops qede_netdev_ops = {
712	.ndo_open = qede_open,
713	.ndo_stop = qede_close,
714	.ndo_start_xmit = qede_start_xmit,
715	.ndo_select_queue = qede_select_queue,
716	.ndo_set_rx_mode = qede_set_rx_mode,
717	.ndo_set_mac_address = qede_set_mac_addr,
718	.ndo_validate_addr = eth_validate_addr,
719	.ndo_change_mtu = qede_change_mtu,
720	.ndo_eth_ioctl = qede_ioctl,
721	.ndo_tx_timeout = qede_tx_timeout,
722	#ifdef CONFIG_QED_SRIOV
723	.ndo_set_vf_mac = qede_set_vf_mac,
724	.ndo_set_vf_vlan = qede_set_vf_vlan,
725	.ndo_set_vf_trust = qede_set_vf_trust,
726	#endif
727	.ndo_vlan_rx_add_vid = qede_vlan_rx_add_vid,
728	.ndo_vlan_rx_kill_vid = qede_vlan_rx_kill_vid,
729	.ndo_fix_features = qede_fix_features,
730	.ndo_set_features = qede_set_features,
731	.ndo_get_stats64 = qede_get_stats64,
732	#ifdef CONFIG_QED_SRIOV
733	.ndo_set_vf_link_state = qede_set_vf_link_state,
734	.ndo_set_vf_spoofchk = qede_set_vf_spoofchk,
735	.ndo_get_vf_config = qede_get_vf_config,
736	.ndo_set_vf_rate = qede_set_vf_rate,
737	#endif
738	.ndo_features_check = qede_features_check,
739	.ndo_bpf = qede_xdp,
740	#ifdef CONFIG_RFS_ACCEL
741	.ndo_rx_flow_steer = qede_rx_flow_steer,
742	#endif
743	.ndo_xdp_xmit = qede_xdp_transmit,
744	.ndo_setup_tc = qede_setup_tc_offload,
745	};
746
747	static const struct net_device_ops qede_netdev_vf_ops = {
748	.ndo_open = qede_open,
749	.ndo_stop = qede_close,
750	.ndo_start_xmit = qede_start_xmit,
751	.ndo_select_queue = qede_select_queue,
752	.ndo_set_rx_mode = qede_set_rx_mode,
753	.ndo_set_mac_address = qede_set_mac_addr,
754	.ndo_validate_addr = eth_validate_addr,
755	.ndo_change_mtu = qede_change_mtu,
756	.ndo_vlan_rx_add_vid = qede_vlan_rx_add_vid,
757	.ndo_vlan_rx_kill_vid = qede_vlan_rx_kill_vid,
758	.ndo_fix_features = qede_fix_features,
759	.ndo_set_features = qede_set_features,
760	.ndo_get_stats64 = qede_get_stats64,
761	.ndo_features_check = qede_features_check,
762	};
763
764	static const struct net_device_ops qede_netdev_vf_xdp_ops = {
765	.ndo_open = qede_open,
766	.ndo_stop = qede_close,
767	.ndo_start_xmit = qede_start_xmit,
768	.ndo_select_queue = qede_select_queue,
769	.ndo_set_rx_mode = qede_set_rx_mode,
770	.ndo_set_mac_address = qede_set_mac_addr,
771	.ndo_validate_addr = eth_validate_addr,
772	.ndo_change_mtu = qede_change_mtu,
773	.ndo_vlan_rx_add_vid = qede_vlan_rx_add_vid,
774	.ndo_vlan_rx_kill_vid = qede_vlan_rx_kill_vid,
775	.ndo_fix_features = qede_fix_features,
776	.ndo_set_features = qede_set_features,
777	.ndo_get_stats64 = qede_get_stats64,
778	.ndo_features_check = qede_features_check,
779	.ndo_bpf = qede_xdp,
780	.ndo_xdp_xmit = qede_xdp_transmit,
781	};
782
783	/* -------------------------------------------------------------------------
784	* START OF PROBE / REMOVE
785	* -------------------------------------------------------------------------
786	*/
787
788	static struct qede_dev *qede_alloc_etherdev(struct qed_dev *cdev,
789	struct pci_dev *pdev,
790	struct qed_dev_eth_info *info,
791	u32 dp_module, u8 dp_level)
792	{
793	struct net_device *ndev;
794	struct qede_dev *edev;
795
796	ndev = alloc_etherdev_mqs(sizeof_priv: sizeof(*edev),
797	txqs: info->num_queues * info->num_tc,
798	rxqs: info->num_queues);
799	if (!ndev) {
800	pr_err("etherdev allocation failed\n");
801	return NULL;
802	}
803
804	edev = netdev_priv(dev: ndev);
805	edev->ndev = ndev;
806	edev->cdev = cdev;
807	edev->pdev = pdev;
808	edev->dp_module = dp_module;
809	edev->dp_level = dp_level;
810	edev->ops = qed_ops;
811
812	if (is_kdump_kernel()) {
813	edev->q_num_rx_buffers = NUM_RX_BDS_KDUMP_MIN;
814	edev->q_num_tx_buffers = NUM_TX_BDS_KDUMP_MIN;
815	} else {
816	edev->q_num_rx_buffers = NUM_RX_BDS_DEF;
817	edev->q_num_tx_buffers = NUM_TX_BDS_DEF;
818	}
819
820	DP_INFO(edev, "Allocated netdev with %d tx queues and %d rx queues\n",
821	info->num_queues, info->num_queues);
822
823	SET_NETDEV_DEV(ndev, &pdev->dev);
824
825	memset(&edev->stats, 0, sizeof(edev->stats));
826	memcpy(&edev->dev_info, info, sizeof(*info));
827
828	/* As ethtool doesn't have the ability to show WoL behavior as
829	* 'default', if device supports it declare it's enabled.
830	*/
831	if (edev->dev_info.common.wol_support)
832	edev->wol_enabled = true;
833
834	INIT_LIST_HEAD(list: &edev->vlan_list);
835
836	return edev;
837	}
838
839	static void qede_init_ndev(struct qede_dev *edev)
840	{
841	struct net_device *ndev = edev->ndev;
842	struct pci_dev *pdev = edev->pdev;
843	bool udp_tunnel_enable = false;
844	netdev_features_t hw_features;
845
846	pci_set_drvdata(pdev, data: ndev);
847
848	ndev->mem_start = edev->dev_info.common.pci_mem_start;
849	ndev->base_addr = ndev->mem_start;
850	ndev->mem_end = edev->dev_info.common.pci_mem_end;
851	ndev->irq = edev->dev_info.common.pci_irq;
852
853	ndev->watchdog_timeo = TX_TIMEOUT;
854
855	if (IS_VF(edev)) {
856	if (edev->dev_info.xdp_supported)
857	ndev->netdev_ops = &qede_netdev_vf_xdp_ops;
858	else
859	ndev->netdev_ops = &qede_netdev_vf_ops;
860	} else {
861	ndev->netdev_ops = &qede_netdev_ops;
862	}
863
864	qede_set_ethtool_ops(netdev: ndev);
865
866	ndev->priv_flags |= IFF_UNICAST_FLT;
867
868	/* user-changeble features */
869	hw_features = NETIF_F_GRO | NETIF_F_GRO_HW | NETIF_F_SG |
870	NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
871	NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_HW_TC;
872
873	if (edev->dev_info.common.b_arfs_capable)
874	hw_features |= NETIF_F_NTUPLE;
875
876	if (edev->dev_info.common.vxlan_enable ||
877	edev->dev_info.common.geneve_enable)
878	udp_tunnel_enable = true;
879
880	if (udp_tunnel_enable || edev->dev_info.common.gre_enable) {
881	hw_features |= NETIF_F_TSO_ECN;
882	ndev->hw_enc_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
883	NETIF_F_SG | NETIF_F_TSO |
884	NETIF_F_TSO_ECN | NETIF_F_TSO6 |
885	NETIF_F_RXCSUM;
886	}
887
888	if (udp_tunnel_enable) {
889	hw_features |= (NETIF_F_GSO_UDP_TUNNEL |
890	NETIF_F_GSO_UDP_TUNNEL_CSUM);
891	ndev->hw_enc_features |= (NETIF_F_GSO_UDP_TUNNEL |
892	NETIF_F_GSO_UDP_TUNNEL_CSUM);
893
894	qede_set_udp_tunnels(edev);
895	}
896
897	if (edev->dev_info.common.gre_enable) {
898	hw_features |= (NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM);
899	ndev->hw_enc_features |= (NETIF_F_GSO_GRE |
900	NETIF_F_GSO_GRE_CSUM);
901	}
902
903	ndev->vlan_features = hw_features | NETIF_F_RXHASH | NETIF_F_RXCSUM |
904	NETIF_F_HIGHDMA;
905	ndev->features = hw_features | NETIF_F_RXHASH | NETIF_F_RXCSUM |
906	NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HIGHDMA |
907	NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_TX;
908
909	ndev->hw_features = hw_features;
910
911	ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
912	NETDEV_XDP_ACT_NDO_XMIT;
913
914	/* MTU range: 46 - 9600 */
915	ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
916	ndev->max_mtu = QEDE_MAX_JUMBO_PACKET_SIZE;
917
918	/* Set network device HW mac */
919	eth_hw_addr_set(dev: edev->ndev, addr: edev->dev_info.common.hw_mac);
920
921	ndev->mtu = edev->dev_info.common.mtu;
922	}
923
924	/* This function converts from 32b param to two params of level and module
925	* Input 32b decoding:
926	* b31 - enable all NOTICE prints. NOTICE prints are for deviation from the
927	* 'happy' flow, e.g. memory allocation failed.
928	* b30 - enable all INFO prints. INFO prints are for major steps in the flow
929	* and provide important parameters.
930	* b29-b0 - per-module bitmap, where each bit enables VERBOSE prints of that
931	* module. VERBOSE prints are for tracking the specific flow in low level.
932	*
933	* Notice that the level should be that of the lowest required logs.
934	*/
935	void qede_config_debug(uint debug, u32 *p_dp_module, u8 *p_dp_level)
936	{
937	*p_dp_level = QED_LEVEL_NOTICE;
938	*p_dp_module = 0;
939
940	if (debug & QED_LOG_VERBOSE_MASK) {
941	*p_dp_level = QED_LEVEL_VERBOSE;
942	*p_dp_module = (debug & 0x3FFFFFFF);
943	} else if (debug & QED_LOG_INFO_MASK) {
944	*p_dp_level = QED_LEVEL_INFO;
945	} else if (debug & QED_LOG_NOTICE_MASK) {
946	*p_dp_level = QED_LEVEL_NOTICE;
947	}
948	}
949
950	static void qede_free_fp_array(struct qede_dev *edev)
951	{
952	if (edev->fp_array) {
953	struct qede_fastpath *fp;
954	int i;
955
956	for_each_queue(i) {
957	fp = &edev->fp_array[i];
958
959	kfree(objp: fp->sb_info);
960	/* Handle mem alloc failure case where qede_init_fp
961	* didn't register xdp_rxq_info yet.
962	* Implicit only (fp->type & QEDE_FASTPATH_RX)
963	*/
964	if (fp->rxq && xdp_rxq_info_is_reg(xdp_rxq: &fp->rxq->xdp_rxq))
965	xdp_rxq_info_unreg(xdp_rxq: &fp->rxq->xdp_rxq);
966	kfree(objp: fp->rxq);
967	kfree(objp: fp->xdp_tx);
968	kfree(objp: fp->txq);
969	}
970	kfree(objp: edev->fp_array);
971	}
972
973	edev->num_queues = 0;
974	edev->fp_num_tx = 0;
975	edev->fp_num_rx = 0;
976	}
977
978	static int qede_alloc_fp_array(struct qede_dev *edev)
979	{
980	u8 fp_combined, fp_rx = edev->fp_num_rx;
981	struct qede_fastpath *fp;
982	int i;
983
984	edev->fp_array = kcalloc(QEDE_QUEUE_CNT(edev),
985	size: sizeof(*edev->fp_array), GFP_KERNEL);
986	if (!edev->fp_array) {
987	DP_NOTICE(edev, "fp array allocation failed\n");
988	goto err;
989	}
990
991	if (!edev->coal_entry) {
992	edev->coal_entry = kcalloc(QEDE_MAX_RSS_CNT(edev),
993	size: sizeof(*edev->coal_entry),
994	GFP_KERNEL);
995	if (!edev->coal_entry) {
996	DP_ERR(edev, "coalesce entry allocation failed\n");
997	goto err;
998	}
999	}
1000
1001	fp_combined = QEDE_QUEUE_CNT(edev) - fp_rx - edev->fp_num_tx;
1002
1003	/* Allocate the FP elements for Rx queues followed by combined and then
1004	* the Tx. This ordering should be maintained so that the respective
1005	* queues (Rx or Tx) will be together in the fastpath array and the
1006	* associated ids will be sequential.
1007	*/
1008	for_each_queue(i) {
1009	fp = &edev->fp_array[i];
1010
1011	fp->sb_info = kzalloc(size: sizeof(*fp->sb_info), GFP_KERNEL);
1012	if (!fp->sb_info) {
1013	DP_NOTICE(edev, "sb info struct allocation failed\n");
1014	goto err;
1015	}
1016
1017	if (fp_rx) {
1018	fp->type = QEDE_FASTPATH_RX;
1019	fp_rx--;
1020	} else if (fp_combined) {
1021	fp->type = QEDE_FASTPATH_COMBINED;
1022	fp_combined--;
1023	} else {
1024	fp->type = QEDE_FASTPATH_TX;
1025	}
1026
1027	if (fp->type & QEDE_FASTPATH_TX) {
1028	fp->txq = kcalloc(n: edev->dev_info.num_tc,
1029	size: sizeof(*fp->txq), GFP_KERNEL);
1030	if (!fp->txq)
1031	goto err;
1032	}
1033
1034	if (fp->type & QEDE_FASTPATH_RX) {
1035	fp->rxq = kzalloc(size: sizeof(*fp->rxq), GFP_KERNEL);
1036	if (!fp->rxq)
1037	goto err;
1038
1039	if (edev->xdp_prog) {
1040	fp->xdp_tx = kzalloc(size: sizeof(*fp->xdp_tx),
1041	GFP_KERNEL);
1042	if (!fp->xdp_tx)
1043	goto err;
1044	fp->type |= QEDE_FASTPATH_XDP;
1045	}
1046	}
1047	}
1048
1049	return 0;
1050	err:
1051	qede_free_fp_array(edev);
1052	return -ENOMEM;
1053	}
1054
1055	/* The qede lock is used to protect driver state change and driver flows that
1056	* are not reentrant.
1057	*/
1058	void __qede_lock(struct qede_dev *edev)
1059	{
1060	mutex_lock(&edev->qede_lock);
1061	}
1062
1063	void __qede_unlock(struct qede_dev *edev)
1064	{
1065	mutex_unlock(lock: &edev->qede_lock);
1066	}
1067
1068	/* This version of the lock should be used when acquiring the RTNL lock is also
1069	* needed in addition to the internal qede lock.
1070	*/
1071	static void qede_lock(struct qede_dev *edev)
1072	{
1073	rtnl_lock();
1074	__qede_lock(edev);
1075	}
1076
1077	static void qede_unlock(struct qede_dev *edev)
1078	{
1079	__qede_unlock(edev);
1080	rtnl_unlock();
1081	}
1082
1083	static void qede_periodic_task(struct work_struct *work)
1084	{
1085	struct qede_dev *edev = container_of(work, struct qede_dev,
1086	periodic_task.work);
1087
1088	qede_fill_by_demand_stats(edev);
1089	schedule_delayed_work(dwork: &edev->periodic_task, delay: edev->stats_coal_ticks);
1090	}
1091
1092	static void qede_init_periodic_task(struct qede_dev *edev)
1093	{
1094	INIT_DELAYED_WORK(&edev->periodic_task, qede_periodic_task);
1095	spin_lock_init(&edev->stats_lock);
1096	edev->stats_coal_usecs = USEC_PER_SEC;
1097	edev->stats_coal_ticks = usecs_to_jiffies(USEC_PER_SEC);
1098	}
1099
1100	static void qede_sp_task(struct work_struct *work)
1101	{
1102	struct qede_dev *edev = container_of(work, struct qede_dev,
1103	sp_task.work);
1104
1105	/* Disable execution of this deferred work once
1106	* qede removal is in progress, this stop any future
1107	* scheduling of sp_task.
1108	*/
1109	if (test_bit(QEDE_SP_DISABLE, &edev->sp_flags))
1110	return;
1111
1112	/* The locking scheme depends on the specific flag:
1113	* In case of QEDE_SP_RECOVERY, acquiring the RTNL lock is required to
1114	* ensure that ongoing flows are ended and new ones are not started.
1115	* In other cases - only the internal qede lock should be acquired.
1116	*/
1117
1118	if (test_and_clear_bit(QEDE_SP_RECOVERY, addr: &edev->sp_flags)) {
1119	cancel_delayed_work_sync(dwork: &edev->periodic_task);
1120	#ifdef CONFIG_QED_SRIOV
1121	/* SRIOV must be disabled outside the lock to avoid a deadlock.
1122	* The recovery of the active VFs is currently not supported.
1123	*/
1124	if (pci_num_vf(dev: edev->pdev))
1125	qede_sriov_configure(pdev: edev->pdev, num_vfs_param: 0);
1126	#endif
1127	qede_lock(edev);
1128	qede_recovery_handler(edev);
1129	qede_unlock(edev);
1130	}
1131
1132	__qede_lock(edev);
1133
1134	if (test_and_clear_bit(QEDE_SP_RX_MODE, addr: &edev->sp_flags))
1135	if (edev->state == QEDE_STATE_OPEN)
1136	qede_config_rx_mode(ndev: edev->ndev);
1137
1138	#ifdef CONFIG_RFS_ACCEL
1139	if (test_and_clear_bit(QEDE_SP_ARFS_CONFIG, addr: &edev->sp_flags)) {
1140	if (edev->state == QEDE_STATE_OPEN)
1141	qede_process_arfs_filters(edev, free_fltr: false);
1142	}
1143	#endif
1144	if (test_and_clear_bit(QEDE_SP_HW_ERR, addr: &edev->sp_flags))
1145	qede_generic_hw_err_handler(edev);
1146	__qede_unlock(edev);
1147
1148	if (test_and_clear_bit(QEDE_SP_AER, addr: &edev->sp_flags)) {
1149	#ifdef CONFIG_QED_SRIOV
1150	/* SRIOV must be disabled outside the lock to avoid a deadlock.
1151	* The recovery of the active VFs is currently not supported.
1152	*/
1153	if (pci_num_vf(dev: edev->pdev))
1154	qede_sriov_configure(pdev: edev->pdev, num_vfs_param: 0);
1155	#endif
1156	edev->ops->common->recovery_process(edev->cdev);
1157	}
1158	}
1159
1160	static void qede_update_pf_params(struct qed_dev *cdev)
1161	{
1162	struct qed_pf_params pf_params;
1163	u16 num_cons;
1164
1165	/* 64 rx + 64 tx + 64 XDP */
1166	memset(&pf_params, 0, sizeof(struct qed_pf_params));
1167
1168	/* 1 rx + 1 xdp + max tx cos */
1169	num_cons = QED_MIN_L2_CONS;
1170
1171	pf_params.eth_pf_params.num_cons = (MAX_SB_PER_PF_MIMD - 1) * num_cons;
1172
1173	/* Same for VFs - make sure they'll have sufficient connections
1174	* to support XDP Tx queues.
1175	*/
1176	pf_params.eth_pf_params.num_vf_cons = 48;
1177
1178	pf_params.eth_pf_params.num_arfs_filters = QEDE_RFS_MAX_FLTR;
1179	qed_ops->common->update_pf_params(cdev, &pf_params);
1180	}
1181
1182	#define QEDE_FW_VER_STR_SIZE 80
1183
1184	static void qede_log_probe(struct qede_dev *edev)
1185	{
1186	struct qed_dev_info *p_dev_info = &edev->dev_info.common;
1187	u8 buf[QEDE_FW_VER_STR_SIZE];
1188	size_t left_size;
1189
1190	snprintf(buf, QEDE_FW_VER_STR_SIZE,
1191	fmt: "Storm FW %d.%d.%d.%d, Management FW %d.%d.%d.%d",
1192	p_dev_info->fw_major, p_dev_info->fw_minor, p_dev_info->fw_rev,
1193	p_dev_info->fw_eng,
1194	(p_dev_info->mfw_rev & QED_MFW_VERSION_3_MASK) >>
1195	QED_MFW_VERSION_3_OFFSET,
1196	(p_dev_info->mfw_rev & QED_MFW_VERSION_2_MASK) >>
1197	QED_MFW_VERSION_2_OFFSET,
1198	(p_dev_info->mfw_rev & QED_MFW_VERSION_1_MASK) >>
1199	QED_MFW_VERSION_1_OFFSET,
1200	(p_dev_info->mfw_rev & QED_MFW_VERSION_0_MASK) >>
1201	QED_MFW_VERSION_0_OFFSET);
1202
1203	left_size = QEDE_FW_VER_STR_SIZE - strlen(buf);
1204	if (p_dev_info->mbi_version && left_size)
1205	snprintf(buf: buf + strlen(buf), size: left_size,
1206	fmt: " [MBI %d.%d.%d]",
1207	(p_dev_info->mbi_version & QED_MBI_VERSION_2_MASK) >>
1208	QED_MBI_VERSION_2_OFFSET,
1209	(p_dev_info->mbi_version & QED_MBI_VERSION_1_MASK) >>
1210	QED_MBI_VERSION_1_OFFSET,
1211	(p_dev_info->mbi_version & QED_MBI_VERSION_0_MASK) >>
1212	QED_MBI_VERSION_0_OFFSET);
1213
1214	pr_info("qede %02x:%02x.%02x: %s [%s]\n", edev->pdev->bus->number,
1215	PCI_SLOT(edev->pdev->devfn), PCI_FUNC(edev->pdev->devfn),
1216	buf, edev->ndev->name);
1217	}
1218
1219	enum qede_probe_mode {
1220	QEDE_PROBE_NORMAL,
1221	QEDE_PROBE_RECOVERY,
1222	};
1223
1224	static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level,
1225	bool is_vf, enum qede_probe_mode mode)
1226	{
1227	struct qed_probe_params probe_params;
1228	struct qed_slowpath_params sp_params;
1229	struct qed_dev_eth_info dev_info;
1230	struct qede_dev *edev;
1231	struct qed_dev *cdev;
1232	int rc;
1233
1234	if (unlikely(dp_level & QED_LEVEL_INFO))
1235	pr_notice("Starting qede probe\n");
1236
1237	memset(&probe_params, 0, sizeof(probe_params));
1238	probe_params.protocol = QED_PROTOCOL_ETH;
1239	probe_params.dp_module = dp_module;
1240	probe_params.dp_level = dp_level;
1241	probe_params.is_vf = is_vf;
1242	probe_params.recov_in_prog = (mode == QEDE_PROBE_RECOVERY);
1243	cdev = qed_ops->common->probe(pdev, &probe_params);
1244	if (!cdev) {
1245	rc = -ENODEV;
1246	goto err0;
1247	}
1248
1249	qede_update_pf_params(cdev);
1250
1251	/* Start the Slowpath-process */
1252	memset(&sp_params, 0, sizeof(sp_params));
1253	sp_params.int_mode = QED_INT_MODE_MSIX;
1254	strscpy(sp_params.name, "qede LAN", QED_DRV_VER_STR_SIZE);
1255	rc = qed_ops->common->slowpath_start(cdev, &sp_params);
1256	if (rc) {
1257	pr_notice("Cannot start slowpath\n");
1258	goto err1;
1259	}
1260
1261	/* Learn information crucial for qede to progress */
1262	rc = qed_ops->fill_dev_info(cdev, &dev_info);
1263	if (rc)
1264	goto err2;
1265
1266	if (mode != QEDE_PROBE_RECOVERY) {
1267	edev = qede_alloc_etherdev(cdev, pdev, info: &dev_info, dp_module,
1268	dp_level);
1269	if (!edev) {
1270	rc = -ENOMEM;
1271	goto err2;
1272	}
1273
1274	edev->devlink = qed_ops->common->devlink_register(cdev);
1275	if (IS_ERR(ptr: edev->devlink)) {
1276	DP_NOTICE(edev, "Cannot register devlink\n");
1277	rc = PTR_ERR(ptr: edev->devlink);
1278	edev->devlink = NULL;
1279	goto err3;
1280	}
1281	} else {
1282	struct net_device *ndev = pci_get_drvdata(pdev);
1283	struct qed_devlink *qdl;
1284
1285	edev = netdev_priv(dev: ndev);
1286	qdl = devlink_priv(devlink: edev->devlink);
1287	qdl->cdev = cdev;
1288	edev->cdev = cdev;
1289	memset(&edev->stats, 0, sizeof(edev->stats));
1290	memcpy(&edev->dev_info, &dev_info, sizeof(dev_info));
1291	}
1292
1293	if (is_vf)
1294	set_bit(nr: QEDE_FLAGS_IS_VF, addr: &edev->flags);
1295
1296	qede_init_ndev(edev);
1297
1298	rc = qede_rdma_dev_add(dev: edev, recovery: (mode == QEDE_PROBE_RECOVERY));
1299	if (rc)
1300	goto err3;
1301
1302	if (mode != QEDE_PROBE_RECOVERY) {
1303	/* Prepare the lock prior to the registration of the netdev,
1304	* as once it's registered we might reach flows requiring it
1305	* [it's even possible to reach a flow needing it directly
1306	* from there, although it's unlikely].
1307	*/
1308	INIT_DELAYED_WORK(&edev->sp_task, qede_sp_task);
1309	mutex_init(&edev->qede_lock);
1310	qede_init_periodic_task(edev);
1311
1312	rc = register_netdev(dev: edev->ndev);
1313	if (rc) {
1314	DP_NOTICE(edev, "Cannot register net-device\n");
1315	goto err4;
1316	}
1317	}
1318
1319	edev->ops->common->set_name(cdev, edev->ndev->name);
1320
1321	/* PTP not supported on VFs */
1322	if (!is_vf)
1323	qede_ptp_enable(edev);
1324
1325	edev->ops->register_ops(cdev, &qede_ll_ops, edev);
1326
1327	#ifdef CONFIG_DCB
1328	if (!IS_VF(edev))
1329	qede_set_dcbnl_ops(ndev: edev->ndev);
1330	#endif
1331
1332	edev->rx_copybreak = QEDE_RX_HDR_SIZE;
1333
1334	qede_log_probe(edev);
1335
1336	/* retain user config (for example - after recovery) */
1337	if (edev->stats_coal_usecs)
1338	schedule_delayed_work(dwork: &edev->periodic_task, delay: 0);
1339
1340	return 0;
1341
1342	err4:
1343	qede_rdma_dev_remove(dev: edev, recovery: (mode == QEDE_PROBE_RECOVERY));
1344	err3:
1345	if (mode != QEDE_PROBE_RECOVERY)
1346	free_netdev(dev: edev->ndev);
1347	else
1348	edev->cdev = NULL;
1349	err2:
1350	qed_ops->common->slowpath_stop(cdev);
1351	err1:
1352	qed_ops->common->remove(cdev);
1353	err0:
1354	return rc;
1355	}
1356
1357	static int qede_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1358	{
1359	bool is_vf = false;
1360	u32 dp_module = 0;
1361	u8 dp_level = 0;
1362
1363	switch ((enum qede_pci_private)id->driver_data) {
1364	case QEDE_PRIVATE_VF:
1365	if (debug & QED_LOG_VERBOSE_MASK)
1366	dev_err(&pdev->dev, "Probing a VF\n");
1367	is_vf = true;
1368	break;
1369	default:
1370	if (debug & QED_LOG_VERBOSE_MASK)
1371	dev_err(&pdev->dev, "Probing a PF\n");
1372	}
1373
1374	qede_config_debug(debug, p_dp_module: &dp_module, p_dp_level: &dp_level);
1375
1376	return __qede_probe(pdev, dp_module, dp_level, is_vf,
1377	mode: QEDE_PROBE_NORMAL);
1378	}
1379
1380	enum qede_remove_mode {
1381	QEDE_REMOVE_NORMAL,
1382	QEDE_REMOVE_RECOVERY,
1383	};
1384
1385	static void __qede_remove(struct pci_dev *pdev, enum qede_remove_mode mode)
1386	{
1387	struct net_device *ndev = pci_get_drvdata(pdev);
1388	struct qede_dev *edev;
1389	struct qed_dev *cdev;
1390
1391	if (!ndev) {
1392	dev_info(&pdev->dev, "Device has already been removed\n");
1393	return;
1394	}
1395
1396	edev = netdev_priv(dev: ndev);
1397	cdev = edev->cdev;
1398
1399	DP_INFO(edev, "Starting qede_remove\n");
1400
1401	qede_rdma_dev_remove(dev: edev, recovery: (mode == QEDE_REMOVE_RECOVERY));
1402
1403	if (mode != QEDE_REMOVE_RECOVERY) {
1404	set_bit(QEDE_SP_DISABLE, addr: &edev->sp_flags);
1405	unregister_netdev(dev: ndev);
1406
1407	cancel_delayed_work_sync(dwork: &edev->sp_task);
1408	cancel_delayed_work_sync(dwork: &edev->periodic_task);
1409
1410	edev->ops->common->set_power_state(cdev, PCI_D0);
1411
1412	pci_set_drvdata(pdev, NULL);
1413	}
1414
1415	qede_ptp_disable(edev);
1416
1417	/* Use global ops since we've freed edev */
1418	qed_ops->common->slowpath_stop(cdev);
1419	if (system_state == SYSTEM_POWER_OFF)
1420	return;
1421
1422	if (mode != QEDE_REMOVE_RECOVERY && edev->devlink) {
1423	qed_ops->common->devlink_unregister(edev->devlink);
1424	edev->devlink = NULL;
1425	}
1426	qed_ops->common->remove(cdev);
1427	edev->cdev = NULL;
1428
1429	/* Since this can happen out-of-sync with other flows,
1430	* don't release the netdevice until after slowpath stop
1431	* has been called to guarantee various other contexts
1432	* [e.g., QED register callbacks] won't break anything when
1433	* accessing the netdevice.
1434	*/
1435	if (mode != QEDE_REMOVE_RECOVERY) {
1436	kfree(objp: edev->coal_entry);
1437	free_netdev(dev: ndev);
1438	}
1439
1440	dev_info(&pdev->dev, "Ending qede_remove successfully\n");
1441	}
1442
1443	static void qede_remove(struct pci_dev *pdev)
1444	{
1445	__qede_remove(pdev, mode: QEDE_REMOVE_NORMAL);
1446	}
1447
1448	static void qede_shutdown(struct pci_dev *pdev)
1449	{
1450	__qede_remove(pdev, mode: QEDE_REMOVE_NORMAL);
1451	}
1452
1453	/* -------------------------------------------------------------------------
1454	* START OF LOAD / UNLOAD
1455	* -------------------------------------------------------------------------
1456	*/
1457
1458	static int qede_set_num_queues(struct qede_dev *edev)
1459	{
1460	int rc;
1461	u16 rss_num;
1462
1463	/* Setup queues according to possible resources*/
1464	if (edev->req_queues)
1465	rss_num = edev->req_queues;
1466	else
1467	rss_num = netif_get_num_default_rss_queues() *
1468	edev->dev_info.common.num_hwfns;
1469
1470	rss_num = min_t(u16, QEDE_MAX_RSS_CNT(edev), rss_num);
1471
1472	rc = edev->ops->common->set_fp_int(edev->cdev, rss_num);
1473	if (rc > 0) {
1474	/* Managed to request interrupts for our queues */
1475	edev->num_queues = rc;
1476	DP_INFO(edev, "Managed %d [of %d] RSS queues\n",
1477	QEDE_QUEUE_CNT(edev), rss_num);
1478	rc = 0;
1479	}
1480
1481	edev->fp_num_tx = edev->req_num_tx;
1482	edev->fp_num_rx = edev->req_num_rx;
1483
1484	return rc;
1485	}
1486
1487	static void qede_free_mem_sb(struct qede_dev *edev, struct qed_sb_info *sb_info,
1488	u16 sb_id)
1489	{
1490	if (sb_info->sb_virt) {
1491	edev->ops->common->sb_release(edev->cdev, sb_info, sb_id,
1492	QED_SB_TYPE_L2_QUEUE);
1493	dma_free_coherent(dev: &edev->pdev->dev, size: sizeof(*sb_info->sb_virt),
1494	cpu_addr: (void *)sb_info->sb_virt, dma_handle: sb_info->sb_phys);
1495	memset(sb_info, 0, sizeof(*sb_info));
1496	}
1497	}
1498
1499	/* This function allocates fast-path status block memory */
1500	static int qede_alloc_mem_sb(struct qede_dev *edev,
1501	struct qed_sb_info *sb_info, u16 sb_id)
1502	{
1503	struct status_block *sb_virt;
1504	dma_addr_t sb_phys;
1505	int rc;
1506
1507	sb_virt = dma_alloc_coherent(dev: &edev->pdev->dev,
1508	size: sizeof(*sb_virt), dma_handle: &sb_phys, GFP_KERNEL);
1509	if (!sb_virt) {
1510	DP_ERR(edev, "Status block allocation failed\n");
1511	return -ENOMEM;
1512	}
1513
1514	rc = edev->ops->common->sb_init(edev->cdev, sb_info,
1515	sb_virt, sb_phys, sb_id,
1516	QED_SB_TYPE_L2_QUEUE);
1517	if (rc) {
1518	DP_ERR(edev, "Status block initialization failed\n");
1519	dma_free_coherent(dev: &edev->pdev->dev, size: sizeof(*sb_virt),
1520	cpu_addr: sb_virt, dma_handle: sb_phys);
1521	return rc;
1522	}
1523
1524	return 0;
1525	}
1526
1527	static void qede_free_rx_buffers(struct qede_dev *edev,
1528	struct qede_rx_queue *rxq)
1529	{
1530	u16 i;
1531
1532	for (i = rxq->sw_rx_cons; i != rxq->sw_rx_prod; i++) {
1533	struct sw_rx_data *rx_buf;
1534	struct page *data;
1535
1536	rx_buf = &rxq->sw_rx_ring[i & NUM_RX_BDS_MAX];
1537	data = rx_buf->data;
1538
1539	dma_unmap_page(&edev->pdev->dev,
1540	rx_buf->mapping, PAGE_SIZE, rxq->data_direction);
1541
1542	rx_buf->data = NULL;
1543	__free_page(data);
1544	}
1545	}
1546
1547	static void qede_free_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
1548	{
1549	/* Free rx buffers */
1550	qede_free_rx_buffers(edev, rxq);
1551
1552	/* Free the parallel SW ring */
1553	kfree(objp: rxq->sw_rx_ring);
1554
1555	/* Free the real RQ ring used by FW */
1556	edev->ops->common->chain_free(edev->cdev, &rxq->rx_bd_ring);
1557	edev->ops->common->chain_free(edev->cdev, &rxq->rx_comp_ring);
1558	}
1559
1560	static void qede_set_tpa_param(struct qede_rx_queue *rxq)
1561	{
1562	int i;
1563
1564	for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) {
1565	struct qede_agg_info *tpa_info = &rxq->tpa_info[i];
1566
1567	tpa_info->state = QEDE_AGG_STATE_NONE;
1568	}
1569	}
1570
1571	/* This function allocates all memory needed per Rx queue */
1572	static int qede_alloc_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq)
1573	{
1574	struct qed_chain_init_params params = {
1575	.cnt_type = QED_CHAIN_CNT_TYPE_U16,
1576	.num_elems = RX_RING_SIZE,
1577	};
1578	struct qed_dev *cdev = edev->cdev;
1579	int i, rc, size;
1580
1581	rxq->num_rx_buffers = edev->q_num_rx_buffers;
1582
1583	rxq->rx_buf_size = NET_IP_ALIGN + ETH_OVERHEAD + edev->ndev->mtu;
1584
1585	rxq->rx_headroom = edev->xdp_prog ? XDP_PACKET_HEADROOM : NET_SKB_PAD;
1586	size = rxq->rx_headroom +
1587	SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1588
1589	/* Make sure that the headroom and payload fit in a single page */
1590	if (rxq->rx_buf_size + size > PAGE_SIZE)
1591	rxq->rx_buf_size = PAGE_SIZE - size;
1592
1593	/* Segment size to split a page in multiple equal parts,
1594	* unless XDP is used in which case we'd use the entire page.
1595	*/
1596	if (!edev->xdp_prog) {
1597	size = size + rxq->rx_buf_size;
1598	rxq->rx_buf_seg_size = roundup_pow_of_two(size);
1599	} else {
1600	rxq->rx_buf_seg_size = PAGE_SIZE;
1601	edev->ndev->features &= ~NETIF_F_GRO_HW;
1602	}
1603
1604	/* Allocate the parallel driver ring for Rx buffers */
1605	size = sizeof(*rxq->sw_rx_ring) * RX_RING_SIZE;
1606	rxq->sw_rx_ring = kzalloc(size, GFP_KERNEL);
1607	if (!rxq->sw_rx_ring) {
1608	DP_ERR(edev, "Rx buffers ring allocation failed\n");
1609	rc = -ENOMEM;
1610	goto err;
1611	}
1612
1613	/* Allocate FW Rx ring */
1614	params.mode = QED_CHAIN_MODE_NEXT_PTR;
1615	params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
1616	params.elem_size = sizeof(struct eth_rx_bd);
1617
1618	rc = edev->ops->common->chain_alloc(cdev, &rxq->rx_bd_ring, &params);
1619	if (rc)
1620	goto err;
1621
1622	/* Allocate FW completion ring */
1623	params.mode = QED_CHAIN_MODE_PBL;
1624	params.intended_use = QED_CHAIN_USE_TO_CONSUME;
1625	params.elem_size = sizeof(union eth_rx_cqe);
1626
1627	rc = edev->ops->common->chain_alloc(cdev, &rxq->rx_comp_ring, &params);
1628	if (rc)
1629	goto err;
1630
1631	/* Allocate buffers for the Rx ring */
1632	rxq->filled_buffers = 0;
1633	for (i = 0; i < rxq->num_rx_buffers; i++) {
1634	rc = qede_alloc_rx_buffer(rxq, allow_lazy: false);
1635	if (rc) {
1636	DP_ERR(edev,
1637	"Rx buffers allocation failed at index %d\n", i);
1638	goto err;
1639	}
1640	}
1641
1642	edev->gro_disable = !(edev->ndev->features & NETIF_F_GRO_HW);
1643	if (!edev->gro_disable)
1644	qede_set_tpa_param(rxq);
1645	err:
1646	return rc;
1647	}
1648
1649	static void qede_free_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq)
1650	{
1651	/* Free the parallel SW ring */
1652	if (txq->is_xdp)
1653	kfree(objp: txq->sw_tx_ring.xdp);
1654	else
1655	kfree(objp: txq->sw_tx_ring.skbs);
1656
1657	/* Free the real RQ ring used by FW */
1658	edev->ops->common->chain_free(edev->cdev, &txq->tx_pbl);
1659	}
1660
1661	/* This function allocates all memory needed per Tx queue */
1662	static int qede_alloc_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq)
1663	{
1664	struct qed_chain_init_params params = {
1665	.mode = QED_CHAIN_MODE_PBL,
1666	.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1667	.cnt_type = QED_CHAIN_CNT_TYPE_U16,
1668	.num_elems = edev->q_num_tx_buffers,
1669	.elem_size = sizeof(union eth_tx_bd_types),
1670	};
1671	int size, rc;
1672
1673	txq->num_tx_buffers = edev->q_num_tx_buffers;
1674
1675	/* Allocate the parallel driver ring for Tx buffers */
1676	if (txq->is_xdp) {
1677	size = sizeof(*txq->sw_tx_ring.xdp) * txq->num_tx_buffers;
1678	txq->sw_tx_ring.xdp = kzalloc(size, GFP_KERNEL);
1679	if (!txq->sw_tx_ring.xdp)
1680	goto err;
1681	} else {
1682	size = sizeof(*txq->sw_tx_ring.skbs) * txq->num_tx_buffers;
1683	txq->sw_tx_ring.skbs = kzalloc(size, GFP_KERNEL);
1684	if (!txq->sw_tx_ring.skbs)
1685	goto err;
1686	}
1687
1688	rc = edev->ops->common->chain_alloc(edev->cdev, &txq->tx_pbl, &params);
1689	if (rc)
1690	goto err;
1691
1692	return 0;
1693
1694	err:
1695	qede_free_mem_txq(edev, txq);
1696	return -ENOMEM;
1697	}
1698
1699	/* This function frees all memory of a single fp */
1700	static void qede_free_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp)
1701	{
1702	qede_free_mem_sb(edev, sb_info: fp->sb_info, sb_id: fp->id);
1703
1704	if (fp->type & QEDE_FASTPATH_RX)
1705	qede_free_mem_rxq(edev, rxq: fp->rxq);
1706
1707	if (fp->type & QEDE_FASTPATH_XDP)
1708	qede_free_mem_txq(edev, txq: fp->xdp_tx);
1709
1710	if (fp->type & QEDE_FASTPATH_TX) {
1711	int cos;
1712
1713	for_each_cos_in_txq(edev, cos)
1714	qede_free_mem_txq(edev, txq: &fp->txq[cos]);
1715	}
1716	}
1717
1718	/* This function allocates all memory needed for a single fp (i.e. an entity
1719	* which contains status block, one rx queue and/or multiple per-TC tx queues.
1720	*/
1721	static int qede_alloc_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp)
1722	{
1723	int rc = 0;
1724
1725	rc = qede_alloc_mem_sb(edev, sb_info: fp->sb_info, sb_id: fp->id);
1726	if (rc)
1727	goto out;
1728
1729	if (fp->type & QEDE_FASTPATH_RX) {
1730	rc = qede_alloc_mem_rxq(edev, rxq: fp->rxq);
1731	if (rc)
1732	goto out;
1733	}
1734
1735	if (fp->type & QEDE_FASTPATH_XDP) {
1736	rc = qede_alloc_mem_txq(edev, txq: fp->xdp_tx);
1737	if (rc)
1738	goto out;
1739	}
1740
1741	if (fp->type & QEDE_FASTPATH_TX) {
1742	int cos;
1743
1744	for_each_cos_in_txq(edev, cos) {
1745	rc = qede_alloc_mem_txq(edev, txq: &fp->txq[cos]);
1746	if (rc)
1747	goto out;
1748	}
1749	}
1750
1751	out:
1752	return rc;
1753	}
1754
1755	static void qede_free_mem_load(struct qede_dev *edev)
1756	{
1757	int i;
1758
1759	for_each_queue(i) {
1760	struct qede_fastpath *fp = &edev->fp_array[i];
1761
1762	qede_free_mem_fp(edev, fp);
1763	}
1764	}
1765
1766	/* This function allocates all qede memory at NIC load. */
1767	static int qede_alloc_mem_load(struct qede_dev *edev)
1768	{
1769	int rc = 0, queue_id;
1770
1771	for (queue_id = 0; queue_id < QEDE_QUEUE_CNT(edev); queue_id++) {
1772	struct qede_fastpath *fp = &edev->fp_array[queue_id];
1773
1774	rc = qede_alloc_mem_fp(edev, fp);
1775	if (rc) {
1776	DP_ERR(edev,
1777	"Failed to allocate memory for fastpath - rss id = %d\n",
1778	queue_id);
1779	qede_free_mem_load(edev);
1780	return rc;
1781	}
1782	}
1783
1784	return 0;
1785	}
1786
1787	static void qede_empty_tx_queue(struct qede_dev *edev,
1788	struct qede_tx_queue *txq)
1789	{
1790	unsigned int pkts_compl = 0, bytes_compl = 0;
1791	struct netdev_queue *netdev_txq;
1792	int rc, len = 0;
1793
1794	netdev_txq = netdev_get_tx_queue(dev: edev->ndev, index: txq->ndev_txq_id);
1795
1796	while (qed_chain_get_cons_idx(chain: &txq->tx_pbl) !=
1797	qed_chain_get_prod_idx(chain: &txq->tx_pbl)) {
1798	DP_VERBOSE(edev, NETIF_MSG_IFDOWN,
1799	"Freeing a packet on tx queue[%d]: chain_cons 0x%x, chain_prod 0x%x\n",
1800	txq->index, qed_chain_get_cons_idx(&txq->tx_pbl),
1801	qed_chain_get_prod_idx(&txq->tx_pbl));
1802
1803	rc = qede_free_tx_pkt(edev, txq, len: &len);
1804	if (rc) {
1805	DP_NOTICE(edev,
1806	"Failed to free a packet on tx queue[%d]: chain_cons 0x%x, chain_prod 0x%x\n",
1807	txq->index,
1808	qed_chain_get_cons_idx(&txq->tx_pbl),
1809	qed_chain_get_prod_idx(&txq->tx_pbl));
1810	break;
1811	}
1812
1813	bytes_compl += len;
1814	pkts_compl++;
1815	txq->sw_tx_cons++;
1816	}
1817
1818	netdev_tx_completed_queue(dev_queue: netdev_txq, pkts: pkts_compl, bytes: bytes_compl);
1819	}
1820
1821	static void qede_empty_tx_queues(struct qede_dev *edev)
1822	{
1823	int i;
1824
1825	for_each_queue(i)
1826	if (edev->fp_array[i].type & QEDE_FASTPATH_TX) {
1827	int cos;
1828
1829	for_each_cos_in_txq(edev, cos) {
1830	struct qede_fastpath *fp;
1831
1832	fp = &edev->fp_array[i];
1833	qede_empty_tx_queue(edev,
1834	txq: &fp->txq[cos]);
1835	}
1836	}
1837	}
1838
1839	/* This function inits fp content and resets the SB, RXQ and TXQ structures */
1840	static void qede_init_fp(struct qede_dev *edev)
1841	{
1842	int queue_id, rxq_index = 0, txq_index = 0;
1843	struct qede_fastpath *fp;
1844	bool init_xdp = false;
1845
1846	for_each_queue(queue_id) {
1847	fp = &edev->fp_array[queue_id];
1848
1849	fp->edev = edev;
1850	fp->id = queue_id;
1851
1852	if (fp->type & QEDE_FASTPATH_XDP) {
1853	fp->xdp_tx->index = QEDE_TXQ_IDX_TO_XDP(edev,
1854	rxq_index);
1855	fp->xdp_tx->is_xdp = 1;
1856
1857	spin_lock_init(&fp->xdp_tx->xdp_tx_lock);
1858	init_xdp = true;
1859	}
1860
1861	if (fp->type & QEDE_FASTPATH_RX) {
1862	fp->rxq->rxq_id = rxq_index++;
1863
1864	/* Determine how to map buffers for this queue */
1865	if (fp->type & QEDE_FASTPATH_XDP)
1866	fp->rxq->data_direction = DMA_BIDIRECTIONAL;
1867	else
1868	fp->rxq->data_direction = DMA_FROM_DEVICE;
1869	fp->rxq->dev = &edev->pdev->dev;
1870
1871	/* Driver have no error path from here */
1872	WARN_ON(xdp_rxq_info_reg(&fp->rxq->xdp_rxq, edev->ndev,
1873	fp->rxq->rxq_id, 0) < 0);
1874
1875	if (xdp_rxq_info_reg_mem_model(xdp_rxq: &fp->rxq->xdp_rxq,
1876	type: MEM_TYPE_PAGE_ORDER0,
1877	NULL)) {
1878	DP_NOTICE(edev,
1879	"Failed to register XDP memory model\n");
1880	}
1881	}
1882
1883	if (fp->type & QEDE_FASTPATH_TX) {
1884	int cos;
1885
1886	for_each_cos_in_txq(edev, cos) {
1887	struct qede_tx_queue *txq = &fp->txq[cos];
1888	u16 ndev_tx_id;
1889
1890	txq->cos = cos;
1891	txq->index = txq_index;
1892	ndev_tx_id = QEDE_TXQ_TO_NDEV_TXQ_ID(edev, txq);
1893	txq->ndev_txq_id = ndev_tx_id;
1894
1895	if (edev->dev_info.is_legacy)
1896	txq->is_legacy = true;
1897	txq->dev = &edev->pdev->dev;
1898	}
1899
1900	txq_index++;
1901	}
1902
1903	snprintf(buf: fp->name, size: sizeof(fp->name), fmt: "%s-fp-%d",
1904	edev->ndev->name, queue_id);
1905	}
1906
1907	if (init_xdp) {
1908	edev->total_xdp_queues = QEDE_RSS_COUNT(edev);
1909	DP_INFO(edev, "Total XDP queues: %u\n", edev->total_xdp_queues);
1910	}
1911	}
1912
1913	static int qede_set_real_num_queues(struct qede_dev *edev)
1914	{
1915	int rc = 0;
1916
1917	rc = netif_set_real_num_tx_queues(dev: edev->ndev,
1918	QEDE_TSS_COUNT(edev) *
1919	edev->dev_info.num_tc);
1920	if (rc) {
1921	DP_NOTICE(edev, "Failed to set real number of Tx queues\n");
1922	return rc;
1923	}
1924
1925	rc = netif_set_real_num_rx_queues(dev: edev->ndev, QEDE_RSS_COUNT(edev));
1926	if (rc) {
1927	DP_NOTICE(edev, "Failed to set real number of Rx queues\n");
1928	return rc;
1929	}
1930
1931	return 0;
1932	}
1933
1934	static void qede_napi_disable_remove(struct qede_dev *edev)
1935	{
1936	int i;
1937
1938	for_each_queue(i) {
1939	napi_disable(n: &edev->fp_array[i].napi);
1940
1941	netif_napi_del(napi: &edev->fp_array[i].napi);
1942	}
1943	}
1944
1945	static void qede_napi_add_enable(struct qede_dev *edev)
1946	{
1947	int i;
1948
1949	/* Add NAPI objects */
1950	for_each_queue(i) {
1951	netif_napi_add(dev: edev->ndev, napi: &edev->fp_array[i].napi, poll: qede_poll);
1952	napi_enable(n: &edev->fp_array[i].napi);
1953	}
1954	}
1955
1956	static void qede_sync_free_irqs(struct qede_dev *edev)
1957	{
1958	int i;
1959
1960	for (i = 0; i < edev->int_info.used_cnt; i++) {
1961	if (edev->int_info.msix_cnt) {
1962	free_irq(edev->int_info.msix[i].vector,
1963	&edev->fp_array[i]);
1964	} else {
1965	edev->ops->common->simd_handler_clean(edev->cdev, i);
1966	}
1967	}
1968
1969	edev->int_info.used_cnt = 0;
1970	edev->int_info.msix_cnt = 0;
1971	}
1972
1973	static int qede_req_msix_irqs(struct qede_dev *edev)
1974	{
1975	int i, rc;
1976
1977	/* Sanitize number of interrupts == number of prepared RSS queues */
1978	if (QEDE_QUEUE_CNT(edev) > edev->int_info.msix_cnt) {
1979	DP_ERR(edev,
1980	"Interrupt mismatch: %d RSS queues > %d MSI-x vectors\n",
1981	QEDE_QUEUE_CNT(edev), edev->int_info.msix_cnt);
1982	return -EINVAL;
1983	}
1984
1985	for (i = 0; i < QEDE_QUEUE_CNT(edev); i++) {
1986	#ifdef CONFIG_RFS_ACCEL
1987	struct qede_fastpath *fp = &edev->fp_array[i];
1988
1989	if (edev->ndev->rx_cpu_rmap && (fp->type & QEDE_FASTPATH_RX)) {
1990	rc = irq_cpu_rmap_add(rmap: edev->ndev->rx_cpu_rmap,
1991	irq: edev->int_info.msix[i].vector);
1992	if (rc) {
1993	DP_ERR(edev, "Failed to add CPU rmap\n");
1994	qede_free_arfs(edev);
1995	}
1996	}
1997	#endif
1998	rc = request_irq(irq: edev->int_info.msix[i].vector,
1999	handler: qede_msix_fp_int, flags: 0, name: edev->fp_array[i].name,
2000	dev: &edev->fp_array[i]);
2001	if (rc) {
2002	DP_ERR(edev, "Request fp %d irq failed\n", i);
2003	#ifdef CONFIG_RFS_ACCEL
2004	if (edev->ndev->rx_cpu_rmap)
2005	free_irq_cpu_rmap(rmap: edev->ndev->rx_cpu_rmap);
2006
2007	edev->ndev->rx_cpu_rmap = NULL;
2008	#endif
2009	qede_sync_free_irqs(edev);
2010	return rc;
2011	}
2012	DP_VERBOSE(edev, NETIF_MSG_INTR,
2013	"Requested fp irq for %s [entry %d]. Cookie is at %p\n",
2014	edev->fp_array[i].name, i,
2015	&edev->fp_array[i]);
2016	edev->int_info.used_cnt++;
2017	}
2018
2019	return 0;
2020	}
2021
2022	static void qede_simd_fp_handler(void *cookie)
2023	{
2024	struct qede_fastpath *fp = (struct qede_fastpath *)cookie;
2025
2026	napi_schedule_irqoff(n: &fp->napi);
2027	}
2028
2029	static int qede_setup_irqs(struct qede_dev *edev)
2030	{
2031	int i, rc = 0;
2032
2033	/* Learn Interrupt configuration */
2034	rc = edev->ops->common->get_fp_int(edev->cdev, &edev->int_info);
2035	if (rc)
2036	return rc;
2037
2038	if (edev->int_info.msix_cnt) {
2039	rc = qede_req_msix_irqs(edev);
2040	if (rc)
2041	return rc;
2042	edev->ndev->irq = edev->int_info.msix[0].vector;
2043	} else {
2044	const struct qed_common_ops *ops;
2045
2046	/* qed should learn receive the RSS ids and callbacks */
2047	ops = edev->ops->common;
2048	for (i = 0; i < QEDE_QUEUE_CNT(edev); i++)
2049	ops->simd_handler_config(edev->cdev,
2050	&edev->fp_array[i], i,
2051	qede_simd_fp_handler);
2052	edev->int_info.used_cnt = QEDE_QUEUE_CNT(edev);
2053	}
2054	return 0;
2055	}
2056
2057	static int qede_drain_txq(struct qede_dev *edev,
2058	struct qede_tx_queue *txq, bool allow_drain)
2059	{
2060	int rc, cnt = 1000;
2061
2062	while (txq->sw_tx_cons != txq->sw_tx_prod) {
2063	if (!cnt) {
2064	if (allow_drain) {
2065	DP_NOTICE(edev,
2066	"Tx queue[%d] is stuck, requesting MCP to drain\n",
2067	txq->index);
2068	rc = edev->ops->common->drain(edev->cdev);
2069	if (rc)
2070	return rc;
2071	return qede_drain_txq(edev, txq, allow_drain: false);
2072	}
2073	DP_NOTICE(edev,
2074	"Timeout waiting for tx queue[%d]: PROD=%d, CONS=%d\n",
2075	txq->index, txq->sw_tx_prod,
2076	txq->sw_tx_cons);
2077	return -ENODEV;
2078	}
2079	cnt--;
2080	usleep_range(min: 1000, max: 2000);
2081	barrier();
2082	}
2083
2084	/* FW finished processing, wait for HW to transmit all tx packets */
2085	usleep_range(min: 1000, max: 2000);
2086
2087	return 0;
2088	}
2089
2090	static int qede_stop_txq(struct qede_dev *edev,
2091	struct qede_tx_queue *txq, int rss_id)
2092	{
2093	/* delete doorbell from doorbell recovery mechanism */
2094	edev->ops->common->db_recovery_del(edev->cdev, txq->doorbell_addr,
2095	&txq->tx_db);
2096
2097	return edev->ops->q_tx_stop(edev->cdev, rss_id, txq->handle);
2098	}
2099
2100	static int qede_stop_queues(struct qede_dev *edev)
2101	{
2102	struct qed_update_vport_params *vport_update_params;
2103	struct qed_dev *cdev = edev->cdev;
2104	struct qede_fastpath *fp;
2105	int rc, i;
2106
2107	/* Disable the vport */
2108	vport_update_params = vzalloc(size: sizeof(*vport_update_params));
2109	if (!vport_update_params)
2110	return -ENOMEM;
2111
2112	vport_update_params->vport_id = 0;
2113	vport_update_params->update_vport_active_flg = 1;
2114	vport_update_params->vport_active_flg = 0;
2115	vport_update_params->update_rss_flg = 0;
2116
2117	rc = edev->ops->vport_update(cdev, vport_update_params);
2118	vfree(addr: vport_update_params);
2119
2120	if (rc) {
2121	DP_ERR(edev, "Failed to update vport\n");
2122	return rc;
2123	}
2124
2125	/* Flush Tx queues. If needed, request drain from MCP */
2126	for_each_queue(i) {
2127	fp = &edev->fp_array[i];
2128
2129	if (fp->type & QEDE_FASTPATH_TX) {
2130	int cos;
2131
2132	for_each_cos_in_txq(edev, cos) {
2133	rc = qede_drain_txq(edev, txq: &fp->txq[cos], allow_drain: true);
2134	if (rc)
2135	return rc;
2136	}
2137	}
2138
2139	if (fp->type & QEDE_FASTPATH_XDP) {
2140	rc = qede_drain_txq(edev, txq: fp->xdp_tx, allow_drain: true);
2141	if (rc)
2142	return rc;
2143	}
2144	}
2145
2146	/* Stop all Queues in reverse order */
2147	for (i = QEDE_QUEUE_CNT(edev) - 1; i >= 0; i--) {
2148	fp = &edev->fp_array[i];
2149
2150	/* Stop the Tx Queue(s) */
2151	if (fp->type & QEDE_FASTPATH_TX) {
2152	int cos;
2153
2154	for_each_cos_in_txq(edev, cos) {
2155	rc = qede_stop_txq(edev, txq: &fp->txq[cos], rss_id: i);
2156	if (rc)
2157	return rc;
2158	}
2159	}
2160
2161	/* Stop the Rx Queue */
2162	if (fp->type & QEDE_FASTPATH_RX) {
2163	rc = edev->ops->q_rx_stop(cdev, i, fp->rxq->handle);
2164	if (rc) {
2165	DP_ERR(edev, "Failed to stop RXQ #%d\n", i);
2166	return rc;
2167	}
2168	}
2169
2170	/* Stop the XDP forwarding queue */
2171	if (fp->type & QEDE_FASTPATH_XDP) {
2172	rc = qede_stop_txq(edev, txq: fp->xdp_tx, rss_id: i);
2173	if (rc)
2174	return rc;
2175
2176	bpf_prog_put(prog: fp->rxq->xdp_prog);
2177	}
2178	}
2179
2180	/* Stop the vport */
2181	rc = edev->ops->vport_stop(cdev, 0);
2182	if (rc)
2183	DP_ERR(edev, "Failed to stop VPORT\n");
2184
2185	return rc;
2186	}
2187
2188	static int qede_start_txq(struct qede_dev *edev,
2189	struct qede_fastpath *fp,
2190	struct qede_tx_queue *txq, u8 rss_id, u16 sb_idx)
2191	{
2192	dma_addr_t phys_table = qed_chain_get_pbl_phys(chain: &txq->tx_pbl);
2193	u32 page_cnt = qed_chain_get_page_cnt(chain: &txq->tx_pbl);
2194	struct qed_queue_start_common_params params;
2195	struct qed_txq_start_ret_params ret_params;
2196	int rc;
2197
2198	memset(&params, 0, sizeof(params));
2199	memset(&ret_params, 0, sizeof(ret_params));
2200
2201	/* Let the XDP queue share the queue-zone with one of the regular txq.
2202	* We don't really care about its coalescing.
2203	*/
2204	if (txq->is_xdp)
2205	params.queue_id = QEDE_TXQ_XDP_TO_IDX(edev, txq);
2206	else
2207	params.queue_id = txq->index;
2208
2209	params.p_sb = fp->sb_info;
2210	params.sb_idx = sb_idx;
2211	params.tc = txq->cos;
2212
2213	rc = edev->ops->q_tx_start(edev->cdev, rss_id, &params, phys_table,
2214	page_cnt, &ret_params);
2215	if (rc) {
2216	DP_ERR(edev, "Start TXQ #%d failed %d\n", txq->index, rc);
2217	return rc;
2218	}
2219
2220	txq->doorbell_addr = ret_params.p_doorbell;
2221	txq->handle = ret_params.p_handle;
2222
2223	/* Determine the FW consumer address associated */
2224	txq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[sb_idx];
2225
2226	/* Prepare the doorbell parameters */
2227	SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_DEST, DB_DEST_XCM);
2228	SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD, DB_AGG_CMD_SET);
2229	SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_VAL_SEL,
2230	DQ_XCM_ETH_TX_BD_PROD_CMD);
2231	txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD;
2232
2233	/* register doorbell with doorbell recovery mechanism */
2234	rc = edev->ops->common->db_recovery_add(edev->cdev, txq->doorbell_addr,
2235	&txq->tx_db, DB_REC_WIDTH_32B,
2236	DB_REC_KERNEL);
2237
2238	return rc;
2239	}
2240
2241	static int qede_start_queues(struct qede_dev *edev, bool clear_stats)
2242	{
2243	int vlan_removal_en = 1;
2244	struct qed_dev *cdev = edev->cdev;
2245	struct qed_dev_info *qed_info = &edev->dev_info.common;
2246	struct qed_update_vport_params *vport_update_params;
2247	struct qed_queue_start_common_params q_params;
2248	struct qed_start_vport_params start = {0};
2249	int rc, i;
2250
2251	if (!edev->num_queues) {
2252	DP_ERR(edev,
2253	"Cannot update V-VPORT as active as there are no Rx queues\n");
2254	return -EINVAL;
2255	}
2256
2257	vport_update_params = vzalloc(size: sizeof(*vport_update_params));
2258	if (!vport_update_params)
2259	return -ENOMEM;
2260
2261	start.handle_ptp_pkts = !!(edev->ptp);
2262	start.gro_enable = !edev->gro_disable;
2263	start.mtu = edev->ndev->mtu;
2264	start.vport_id = 0;
2265	start.drop_ttl0 = true;
2266	start.remove_inner_vlan = vlan_removal_en;
2267	start.clear_stats = clear_stats;
2268
2269	rc = edev->ops->vport_start(cdev, &start);
2270
2271	if (rc) {
2272	DP_ERR(edev, "Start V-PORT failed %d\n", rc);
2273	goto out;
2274	}
2275
2276	DP_VERBOSE(edev, NETIF_MSG_IFUP,
2277	"Start vport ramrod passed, vport_id = %d, MTU = %d, vlan_removal_en = %d\n",
2278	start.vport_id, edev->ndev->mtu + 0xe, vlan_removal_en);
2279
2280	for_each_queue(i) {
2281	struct qede_fastpath *fp = &edev->fp_array[i];
2282	dma_addr_t p_phys_table;
2283	u32 page_cnt;
2284
2285	if (fp->type & QEDE_FASTPATH_RX) {
2286	struct qed_rxq_start_ret_params ret_params;
2287	struct qede_rx_queue *rxq = fp->rxq;
2288	__le16 *val;
2289
2290	memset(&ret_params, 0, sizeof(ret_params));
2291	memset(&q_params, 0, sizeof(q_params));
2292	q_params.queue_id = rxq->rxq_id;
2293	q_params.vport_id = 0;
2294	q_params.p_sb = fp->sb_info;
2295	q_params.sb_idx = RX_PI;
2296
2297	p_phys_table =
2298	qed_chain_get_pbl_phys(chain: &rxq->rx_comp_ring);
2299	page_cnt = qed_chain_get_page_cnt(chain: &rxq->rx_comp_ring);
2300
2301	rc = edev->ops->q_rx_start(cdev, i, &q_params,
2302	rxq->rx_buf_size,
2303	rxq->rx_bd_ring.p_phys_addr,
2304	p_phys_table,
2305	page_cnt, &ret_params);
2306	if (rc) {
2307	DP_ERR(edev, "Start RXQ #%d failed %d\n", i,
2308	rc);
2309	goto out;
2310	}
2311
2312	/* Use the return parameters */
2313	rxq->hw_rxq_prod_addr = ret_params.p_prod;
2314	rxq->handle = ret_params.p_handle;
2315
2316	val = &fp->sb_info->sb_virt->pi_array[RX_PI];
2317	rxq->hw_cons_ptr = val;
2318
2319	qede_update_rx_prod(edev, rxq);
2320	}
2321
2322	if (fp->type & QEDE_FASTPATH_XDP) {
2323	rc = qede_start_txq(edev, fp, txq: fp->xdp_tx, rss_id: i, XDP_PI);
2324	if (rc)
2325	goto out;
2326
2327	bpf_prog_add(prog: edev->xdp_prog, i: 1);
2328	fp->rxq->xdp_prog = edev->xdp_prog;
2329	}
2330
2331	if (fp->type & QEDE_FASTPATH_TX) {
2332	int cos;
2333
2334	for_each_cos_in_txq(edev, cos) {
2335	rc = qede_start_txq(edev, fp, txq: &fp->txq[cos], rss_id: i,
2336	TX_PI(cos));
2337	if (rc)
2338	goto out;
2339	}
2340	}
2341	}
2342
2343	/* Prepare and send the vport enable */
2344	vport_update_params->vport_id = start.vport_id;
2345	vport_update_params->update_vport_active_flg = 1;
2346	vport_update_params->vport_active_flg = 1;
2347
2348	if ((qed_info->b_inter_pf_switch || pci_num_vf(dev: edev->pdev)) &&
2349	qed_info->tx_switching) {
2350	vport_update_params->update_tx_switching_flg = 1;
2351	vport_update_params->tx_switching_flg = 1;
2352	}
2353
2354	qede_fill_rss_params(edev, rss: &vport_update_params->rss_params,
2355	update: &vport_update_params->update_rss_flg);
2356
2357	rc = edev->ops->vport_update(cdev, vport_update_params);
2358	if (rc)
2359	DP_ERR(edev, "Update V-PORT failed %d\n", rc);
2360
2361	out:
2362	vfree(addr: vport_update_params);
2363	return rc;
2364	}
2365
2366	enum qede_unload_mode {
2367	QEDE_UNLOAD_NORMAL,
2368	QEDE_UNLOAD_RECOVERY,
2369	};
2370
2371	static void qede_unload(struct qede_dev *edev, enum qede_unload_mode mode,
2372	bool is_locked)
2373	{
2374	struct qed_link_params link_params;
2375	int rc;
2376
2377	DP_INFO(edev, "Starting qede unload\n");
2378
2379	if (!is_locked)
2380	__qede_lock(edev);
2381
2382	clear_bit(nr: QEDE_FLAGS_LINK_REQUESTED, addr: &edev->flags);
2383
2384	if (mode != QEDE_UNLOAD_RECOVERY)
2385	edev->state = QEDE_STATE_CLOSED;
2386
2387	qede_rdma_dev_event_close(dev: edev);
2388
2389	/* Close OS Tx */
2390	netif_tx_disable(dev: edev->ndev);
2391	netif_carrier_off(dev: edev->ndev);
2392
2393	if (mode != QEDE_UNLOAD_RECOVERY) {
2394	/* Reset the link */
2395	memset(&link_params, 0, sizeof(link_params));
2396	link_params.link_up = false;
2397	edev->ops->common->set_link(edev->cdev, &link_params);
2398
2399	rc = qede_stop_queues(edev);
2400	if (rc) {
2401	#ifdef CONFIG_RFS_ACCEL
2402	if (edev->dev_info.common.b_arfs_capable) {
2403	qede_poll_for_freeing_arfs_filters(edev);
2404	if (edev->ndev->rx_cpu_rmap)
2405	free_irq_cpu_rmap(rmap: edev->ndev->rx_cpu_rmap);
2406
2407	edev->ndev->rx_cpu_rmap = NULL;
2408	}
2409	#endif
2410	qede_sync_free_irqs(edev);
2411	goto out;
2412	}
2413
2414	DP_INFO(edev, "Stopped Queues\n");
2415	}
2416
2417	qede_vlan_mark_nonconfigured(edev);
2418	edev->ops->fastpath_stop(edev->cdev);
2419
2420	if (edev->dev_info.common.b_arfs_capable) {
2421	qede_poll_for_freeing_arfs_filters(edev);
2422	qede_free_arfs(edev);
2423	}
2424
2425	/* Release the interrupts */
2426	qede_sync_free_irqs(edev);
2427	edev->ops->common->set_fp_int(edev->cdev, 0);
2428
2429	qede_napi_disable_remove(edev);
2430
2431	if (mode == QEDE_UNLOAD_RECOVERY)
2432	qede_empty_tx_queues(edev);
2433
2434	qede_free_mem_load(edev);
2435	qede_free_fp_array(edev);
2436
2437	out:
2438	if (!is_locked)
2439	__qede_unlock(edev);
2440
2441	if (mode != QEDE_UNLOAD_RECOVERY)
2442	DP_NOTICE(edev, "Link is down\n");
2443
2444	edev->ptp_skip_txts = 0;
2445
2446	DP_INFO(edev, "Ending qede unload\n");
2447	}
2448
2449	enum qede_load_mode {
2450	QEDE_LOAD_NORMAL,
2451	QEDE_LOAD_RELOAD,
2452	QEDE_LOAD_RECOVERY,
2453	};
2454
2455	static int qede_load(struct qede_dev *edev, enum qede_load_mode mode,
2456	bool is_locked)
2457	{
2458	struct qed_link_params link_params;
2459	struct ethtool_coalesce coal = {};
2460	u8 num_tc;
2461	int rc, i;
2462
2463	DP_INFO(edev, "Starting qede load\n");
2464
2465	if (!is_locked)
2466	__qede_lock(edev);
2467
2468	rc = qede_set_num_queues(edev);
2469	if (rc)
2470	goto out;
2471
2472	rc = qede_alloc_fp_array(edev);
2473	if (rc)
2474	goto out;
2475
2476	qede_init_fp(edev);
2477
2478	rc = qede_alloc_mem_load(edev);
2479	if (rc)
2480	goto err1;
2481	DP_INFO(edev, "Allocated %d Rx, %d Tx queues\n",
2482	QEDE_RSS_COUNT(edev), QEDE_TSS_COUNT(edev));
2483
2484	rc = qede_set_real_num_queues(edev);
2485	if (rc)
2486	goto err2;
2487
2488	if (qede_alloc_arfs(edev)) {
2489	edev->ndev->features &= ~NETIF_F_NTUPLE;
2490	edev->dev_info.common.b_arfs_capable = false;
2491	}
2492
2493	qede_napi_add_enable(edev);
2494	DP_INFO(edev, "Napi added and enabled\n");
2495
2496	rc = qede_setup_irqs(edev);
2497	if (rc)
2498	goto err3;
2499	DP_INFO(edev, "Setup IRQs succeeded\n");
2500
2501	rc = qede_start_queues(edev, clear_stats: mode != QEDE_LOAD_RELOAD);
2502	if (rc)
2503	goto err4;
2504	DP_INFO(edev, "Start VPORT, RXQ and TXQ succeeded\n");
2505
2506	num_tc = netdev_get_num_tc(dev: edev->ndev);
2507	num_tc = num_tc ? num_tc : edev->dev_info.num_tc;
2508	qede_setup_tc(ndev: edev->ndev, num_tc);
2509
2510	/* Program un-configured VLANs */
2511	qede_configure_vlan_filters(edev);
2512
2513	set_bit(nr: QEDE_FLAGS_LINK_REQUESTED, addr: &edev->flags);
2514
2515	/* Ask for link-up using current configuration */
2516	memset(&link_params, 0, sizeof(link_params));
2517	link_params.link_up = true;
2518	edev->ops->common->set_link(edev->cdev, &link_params);
2519
2520	edev->state = QEDE_STATE_OPEN;
2521
2522	coal.rx_coalesce_usecs = QED_DEFAULT_RX_USECS;
2523	coal.tx_coalesce_usecs = QED_DEFAULT_TX_USECS;
2524
2525	for_each_queue(i) {
2526	if (edev->coal_entry[i].isvalid) {
2527	coal.rx_coalesce_usecs = edev->coal_entry[i].rxc;
2528	coal.tx_coalesce_usecs = edev->coal_entry[i].txc;
2529	}
2530	__qede_unlock(edev);
2531	qede_set_per_coalesce(dev: edev->ndev, queue: i, coal: &coal);
2532	__qede_lock(edev);
2533	}
2534	DP_INFO(edev, "Ending successfully qede load\n");
2535
2536	goto out;
2537	err4:
2538	qede_sync_free_irqs(edev);
2539	err3:
2540	qede_napi_disable_remove(edev);
2541	err2:
2542	qede_free_mem_load(edev);
2543	err1:
2544	edev->ops->common->set_fp_int(edev->cdev, 0);
2545	qede_free_fp_array(edev);
2546	edev->num_queues = 0;
2547	edev->fp_num_tx = 0;
2548	edev->fp_num_rx = 0;
2549	out:
2550	if (!is_locked)
2551	__qede_unlock(edev);
2552
2553	return rc;
2554	}
2555
2556	/* 'func' should be able to run between unload and reload assuming interface
2557	* is actually running, or afterwards in case it's currently DOWN.
2558	*/
2559	void qede_reload(struct qede_dev *edev,
2560	struct qede_reload_args *args, bool is_locked)
2561	{
2562	if (!is_locked)
2563	__qede_lock(edev);
2564
2565	/* Since qede_lock is held, internal state wouldn't change even
2566	* if netdev state would start transitioning. Check whether current
2567	* internal configuration indicates device is up, then reload.
2568	*/
2569	if (edev->state == QEDE_STATE_OPEN) {
2570	qede_unload(edev, mode: QEDE_UNLOAD_NORMAL, is_locked: true);
2571	if (args)
2572	args->func(edev, args);
2573	qede_load(edev, mode: QEDE_LOAD_RELOAD, is_locked: true);
2574
2575	/* Since no one is going to do it for us, re-configure */
2576	qede_config_rx_mode(ndev: edev->ndev);
2577	} else if (args) {
2578	args->func(edev, args);
2579	}
2580
2581	if (!is_locked)
2582	__qede_unlock(edev);
2583	}
2584
2585	/* called with rtnl_lock */
2586	static int qede_open(struct net_device *ndev)
2587	{
2588	struct qede_dev *edev = netdev_priv(dev: ndev);
2589	int rc;
2590
2591	netif_carrier_off(dev: ndev);
2592
2593	edev->ops->common->set_power_state(edev->cdev, PCI_D0);
2594
2595	rc = qede_load(edev, mode: QEDE_LOAD_NORMAL, is_locked: false);
2596	if (rc)
2597	return rc;
2598
2599	udp_tunnel_nic_reset_ntf(dev: ndev);
2600
2601	edev->ops->common->update_drv_state(edev->cdev, true);
2602
2603	return 0;
2604	}
2605
2606	static int qede_close(struct net_device *ndev)
2607	{
2608	struct qede_dev *edev = netdev_priv(dev: ndev);
2609
2610	qede_unload(edev, mode: QEDE_UNLOAD_NORMAL, is_locked: false);
2611
2612	if (edev->cdev)
2613	edev->ops->common->update_drv_state(edev->cdev, false);
2614
2615	return 0;
2616	}
2617
2618	static void qede_link_update(void *dev, struct qed_link_output *link)
2619	{
2620	struct qede_dev *edev = dev;
2621
2622	if (!test_bit(QEDE_FLAGS_LINK_REQUESTED, &edev->flags)) {
2623	DP_VERBOSE(edev, NETIF_MSG_LINK, "Interface is not ready\n");
2624	return;
2625	}
2626
2627	if (link->link_up) {
2628	if (!netif_carrier_ok(dev: edev->ndev)) {
2629	DP_NOTICE(edev, "Link is up\n");
2630	netif_tx_start_all_queues(dev: edev->ndev);
2631	netif_carrier_on(dev: edev->ndev);
2632	qede_rdma_dev_event_open(dev: edev);
2633	}
2634	} else {
2635	if (netif_carrier_ok(dev: edev->ndev)) {
2636	DP_NOTICE(edev, "Link is down\n");
2637	netif_tx_disable(dev: edev->ndev);
2638	netif_carrier_off(dev: edev->ndev);
2639	qede_rdma_dev_event_close(dev: edev);
2640	}
2641	}
2642	}
2643
2644	static void qede_schedule_recovery_handler(void *dev)
2645	{
2646	struct qede_dev *edev = dev;
2647
2648	if (edev->state == QEDE_STATE_RECOVERY) {
2649	DP_NOTICE(edev,
2650	"Avoid scheduling a recovery handling since already in recovery state\n");
2651	return;
2652	}
2653
2654	set_bit(QEDE_SP_RECOVERY, addr: &edev->sp_flags);
2655	schedule_delayed_work(dwork: &edev->sp_task, delay: 0);
2656
2657	DP_INFO(edev, "Scheduled a recovery handler\n");
2658	}
2659
2660	static void qede_recovery_failed(struct qede_dev *edev)
2661	{
2662	netdev_err(dev: edev->ndev, format: "Recovery handling has failed. Power cycle is needed.\n");
2663
2664	netif_device_detach(dev: edev->ndev);
2665
2666	if (edev->cdev)
2667	edev->ops->common->set_power_state(edev->cdev, PCI_D3hot);
2668	}
2669
2670	static void qede_recovery_handler(struct qede_dev *edev)
2671	{
2672	u32 curr_state = edev->state;
2673	int rc;
2674
2675	DP_NOTICE(edev, "Starting a recovery process\n");
2676
2677	/* No need to acquire first the qede_lock since is done by qede_sp_task
2678	* before calling this function.
2679	*/
2680	edev->state = QEDE_STATE_RECOVERY;
2681
2682	edev->ops->common->recovery_prolog(edev->cdev);
2683
2684	if (curr_state == QEDE_STATE_OPEN)
2685	qede_unload(edev, mode: QEDE_UNLOAD_RECOVERY, is_locked: true);
2686
2687	__qede_remove(pdev: edev->pdev, mode: QEDE_REMOVE_RECOVERY);
2688
2689	rc = __qede_probe(pdev: edev->pdev, dp_module: edev->dp_module, dp_level: edev->dp_level,
2690	IS_VF(edev), mode: QEDE_PROBE_RECOVERY);
2691	if (rc) {
2692	edev->cdev = NULL;
2693	goto err;
2694	}
2695
2696	if (curr_state == QEDE_STATE_OPEN) {
2697	rc = qede_load(edev, mode: QEDE_LOAD_RECOVERY, is_locked: true);
2698	if (rc)
2699	goto err;
2700
2701	qede_config_rx_mode(ndev: edev->ndev);
2702	udp_tunnel_nic_reset_ntf(dev: edev->ndev);
2703	}
2704
2705	edev->state = curr_state;
2706
2707	DP_NOTICE(edev, "Recovery handling is done\n");
2708
2709	return;
2710
2711	err:
2712	qede_recovery_failed(edev);
2713	}
2714
2715	static void qede_atomic_hw_err_handler(struct qede_dev *edev)
2716	{
2717	struct qed_dev *cdev = edev->cdev;
2718
2719	DP_NOTICE(edev,
2720	"Generic non-sleepable HW error handling started - err_flags 0x%lx\n",
2721	edev->err_flags);
2722
2723	/* Get a call trace of the flow that led to the error */
2724	WARN_ON(test_bit(QEDE_ERR_WARN, &edev->err_flags));
2725
2726	/* Prevent HW attentions from being reasserted */
2727	if (test_bit(QEDE_ERR_ATTN_CLR_EN, &edev->err_flags))
2728	edev->ops->common->attn_clr_enable(cdev, true);
2729
2730	DP_NOTICE(edev, "Generic non-sleepable HW error handling is done\n");
2731	}
2732
2733	static void qede_generic_hw_err_handler(struct qede_dev *edev)
2734	{
2735	DP_NOTICE(edev,
2736	"Generic sleepable HW error handling started - err_flags 0x%lx\n",
2737	edev->err_flags);
2738
2739	if (edev->devlink) {
2740	DP_NOTICE(edev, "Reporting fatal error to devlink\n");
2741	edev->ops->common->report_fatal_error(edev->devlink, edev->last_err_type);
2742	}
2743
2744	clear_bit(QEDE_ERR_IS_HANDLED, addr: &edev->err_flags);
2745
2746	DP_NOTICE(edev, "Generic sleepable HW error handling is done\n");
2747	}
2748
2749	static void qede_set_hw_err_flags(struct qede_dev *edev,
2750	enum qed_hw_err_type err_type)
2751	{
2752	unsigned long err_flags = 0;
2753
2754	switch (err_type) {
2755	case QED_HW_ERR_DMAE_FAIL:
2756	set_bit(QEDE_ERR_WARN, addr: &err_flags);
2757	fallthrough;
2758	case QED_HW_ERR_MFW_RESP_FAIL:
2759	case QED_HW_ERR_HW_ATTN:
2760	case QED_HW_ERR_RAMROD_FAIL:
2761	case QED_HW_ERR_FW_ASSERT:
2762	set_bit(QEDE_ERR_ATTN_CLR_EN, addr: &err_flags);
2763	set_bit(QEDE_ERR_GET_DBG_INFO, addr: &err_flags);
2764	/* make this error as recoverable and start recovery*/
2765	set_bit(QEDE_ERR_IS_RECOVERABLE, addr: &err_flags);
2766	break;
2767
2768	default:
2769	DP_NOTICE(edev, "Unexpected HW error [%d]\n", err_type);
2770	break;
2771	}
2772
2773	edev->err_flags |= err_flags;
2774	}
2775
2776	static void qede_schedule_hw_err_handler(void *dev,
2777	enum qed_hw_err_type err_type)
2778	{
2779	struct qede_dev *edev = dev;
2780
2781	/* Fan failure cannot be masked by handling of another HW error or by a
2782	* concurrent recovery process.
2783	*/
2784	if ((test_and_set_bit(QEDE_ERR_IS_HANDLED, addr: &edev->err_flags) ||
2785	edev->state == QEDE_STATE_RECOVERY) &&
2786	err_type != QED_HW_ERR_FAN_FAIL) {
2787	DP_INFO(edev,
2788	"Avoid scheduling an error handling while another HW error is being handled\n");
2789	return;
2790	}
2791
2792	if (err_type >= QED_HW_ERR_LAST) {
2793	DP_NOTICE(edev, "Unknown HW error [%d]\n", err_type);
2794	clear_bit(QEDE_ERR_IS_HANDLED, addr: &edev->err_flags);
2795	return;
2796	}
2797
2798	edev->last_err_type = err_type;
2799	qede_set_hw_err_flags(edev, err_type);
2800	qede_atomic_hw_err_handler(edev);
2801	set_bit(QEDE_SP_HW_ERR, addr: &edev->sp_flags);
2802	schedule_delayed_work(dwork: &edev->sp_task, delay: 0);
2803
2804	DP_INFO(edev, "Scheduled a error handler [err_type %d]\n", err_type);
2805	}
2806
2807	static bool qede_is_txq_full(struct qede_dev *edev, struct qede_tx_queue *txq)
2808	{
2809	struct netdev_queue *netdev_txq;
2810
2811	netdev_txq = netdev_get_tx_queue(dev: edev->ndev, index: txq->ndev_txq_id);
2812	if (netif_xmit_stopped(dev_queue: netdev_txq))
2813	return true;
2814
2815	return false;
2816	}
2817
2818	static void qede_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data)
2819	{
2820	struct qede_dev *edev = dev;
2821	struct netdev_hw_addr *ha;
2822	int i;
2823
2824	if (edev->ndev->features & NETIF_F_IP_CSUM)
2825	data->feat_flags |= QED_TLV_IP_CSUM;
2826	if (edev->ndev->features & NETIF_F_TSO)
2827	data->feat_flags |= QED_TLV_LSO;
2828
2829	ether_addr_copy(dst: data->mac[0], src: edev->ndev->dev_addr);
2830	eth_zero_addr(addr: data->mac[1]);
2831	eth_zero_addr(addr: data->mac[2]);
2832	/* Copy the first two UC macs */
2833	netif_addr_lock_bh(dev: edev->ndev);
2834	i = 1;
2835	netdev_for_each_uc_addr(ha, edev->ndev) {
2836	ether_addr_copy(dst: data->mac[i++], src: ha->addr);
2837	if (i == QED_TLV_MAC_COUNT)
2838	break;
2839	}
2840
2841	netif_addr_unlock_bh(dev: edev->ndev);
2842	}
2843
2844	static void qede_get_eth_tlv_data(void *dev, void *data)
2845	{
2846	struct qed_mfw_tlv_eth *etlv = data;
2847	struct qede_dev *edev = dev;
2848	struct qede_fastpath *fp;
2849	int i;
2850
2851	etlv->lso_maxoff_size = 0XFFFF;
2852	etlv->lso_maxoff_size_set = true;
2853	etlv->lso_minseg_size = (u16)ETH_TX_LSO_WINDOW_MIN_LEN;
2854	etlv->lso_minseg_size_set = true;
2855	etlv->prom_mode = !!(edev->ndev->flags & IFF_PROMISC);
2856	etlv->prom_mode_set = true;
2857	etlv->tx_descr_size = QEDE_TSS_COUNT(edev);
2858	etlv->tx_descr_size_set = true;
2859	etlv->rx_descr_size = QEDE_RSS_COUNT(edev);
2860	etlv->rx_descr_size_set = true;
2861	etlv->iov_offload = QED_MFW_TLV_IOV_OFFLOAD_VEB;
2862	etlv->iov_offload_set = true;
2863
2864	/* Fill information regarding queues; Should be done under the qede
2865	* lock to guarantee those don't change beneath our feet.
2866	*/
2867	etlv->txqs_empty = true;
2868	etlv->rxqs_empty = true;
2869	etlv->num_txqs_full = 0;
2870	etlv->num_rxqs_full = 0;
2871
2872	__qede_lock(edev);
2873	for_each_queue(i) {
2874	fp = &edev->fp_array[i];
2875	if (fp->type & QEDE_FASTPATH_TX) {
2876	struct qede_tx_queue *txq = QEDE_FP_TC0_TXQ(fp);
2877
2878	if (txq->sw_tx_cons != txq->sw_tx_prod)
2879	etlv->txqs_empty = false;
2880	if (qede_is_txq_full(edev, txq))
2881	etlv->num_txqs_full++;
2882	}
2883	if (fp->type & QEDE_FASTPATH_RX) {
2884	if (qede_has_rx_work(rxq: fp->rxq))
2885	etlv->rxqs_empty = false;
2886
2887	/* This one is a bit tricky; Firmware might stop
2888	* placing packets if ring is not yet full.
2889	* Give an approximation.
2890	*/
2891	if (le16_to_cpu(*fp->rxq->hw_cons_ptr) -
2892	qed_chain_get_cons_idx(chain: &fp->rxq->rx_comp_ring) >
2893	RX_RING_SIZE - 100)
2894	etlv->num_rxqs_full++;
2895	}
2896	}
2897	__qede_unlock(edev);
2898
2899	etlv->txqs_empty_set = true;
2900	etlv->rxqs_empty_set = true;
2901	etlv->num_txqs_full_set = true;
2902	etlv->num_rxqs_full_set = true;
2903	}
2904
2905	/**
2906	* qede_io_error_detected(): Called when PCI error is detected
2907	*
2908	* @pdev: Pointer to PCI device
2909	* @state: The current pci connection state
2910	*
2911	*Return: pci_ers_result_t.
2912	*
2913	* This function is called after a PCI bus error affecting
2914	* this device has been detected.
2915	*/
2916	static pci_ers_result_t
2917	qede_io_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
2918	{
2919	struct net_device *dev = pci_get_drvdata(pdev);
2920	struct qede_dev *edev = netdev_priv(dev);
2921
2922	if (!edev)
2923	return PCI_ERS_RESULT_NONE;
2924
2925	DP_NOTICE(edev, "IO error detected [%d]\n", state);
2926
2927	__qede_lock(edev);
2928	if (edev->state == QEDE_STATE_RECOVERY) {
2929	DP_NOTICE(edev, "Device already in the recovery state\n");
2930	__qede_unlock(edev);
2931	return PCI_ERS_RESULT_NONE;
2932	}
2933
2934	/* PF handles the recovery of its VFs */
2935	if (IS_VF(edev)) {
2936	DP_VERBOSE(edev, QED_MSG_IOV,
2937	"VF recovery is handled by its PF\n");
2938	__qede_unlock(edev);
2939	return PCI_ERS_RESULT_RECOVERED;
2940	}
2941
2942	/* Close OS Tx */
2943	netif_tx_disable(dev: edev->ndev);
2944	netif_carrier_off(dev: edev->ndev);
2945
2946	set_bit(QEDE_SP_AER, addr: &edev->sp_flags);
2947	schedule_delayed_work(dwork: &edev->sp_task, delay: 0);
2948
2949	__qede_unlock(edev);
2950
2951	return PCI_ERS_RESULT_CAN_RECOVER;
2952	}
2953