1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright(c) 2009 - 2018 Intel Corporation. */
3
4	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5
6	#include <linux/bitfield.h>
7	#include <linux/delay.h>
8	#include <linux/ethtool.h>
9	#include <linux/if_vlan.h>
10	#include <linux/init.h>
11	#include <linux/ipv6.h>
12	#include <linux/mii.h>
13	#include <linux/module.h>
14	#include <linux/netdevice.h>
15	#include <linux/pagemap.h>
16	#include <linux/pci.h>
17	#include <linux/prefetch.h>
18	#include <linux/sctp.h>
19	#include <linux/slab.h>
20	#include <linux/tcp.h>
21	#include <linux/types.h>
22	#include <linux/vmalloc.h>
23	#include <net/checksum.h>
24	#include <net/ip6_checksum.h>
25	#include "igbvf.h"
26
27	char igbvf_driver_name[] = "igbvf";
28	static const char igbvf_driver_string[] =
29	"Intel(R) Gigabit Virtual Function Network Driver";
30	static const char igbvf_copyright[] =
31	"Copyright (c) 2009 - 2012 Intel Corporation.";
32
33	#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
34	static int debug = -1;
35	module_param(debug, int, 0);
36	MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
37
38	static int igbvf_poll(struct napi_struct *napi, int budget);
39	static void igbvf_reset(struct igbvf_adapter *);
40	static void igbvf_set_interrupt_capability(struct igbvf_adapter *);
41	static void igbvf_reset_interrupt_capability(struct igbvf_adapter *);
42
43	static struct igbvf_info igbvf_vf_info = {
44	.mac = e1000_vfadapt,
45	.flags = 0,
46	.pba = 10,
47	.init_ops = e1000_init_function_pointers_vf,
48	};
49
50	static struct igbvf_info igbvf_i350_vf_info = {
51	.mac = e1000_vfadapt_i350,
52	.flags = 0,
53	.pba = 10,
54	.init_ops = e1000_init_function_pointers_vf,
55	};
56
57	static const struct igbvf_info *igbvf_info_tbl[] = {
58	[board_vf] = &igbvf_vf_info,
59	[board_i350_vf] = &igbvf_i350_vf_info,
60	};
61
62	/**
63	* igbvf_desc_unused - calculate if we have unused descriptors
64	* @ring: address of receive ring structure
65	**/
66	static int igbvf_desc_unused(struct igbvf_ring *ring)
67	{
68	if (ring->next_to_clean > ring->next_to_use)
69	return ring->next_to_clean - ring->next_to_use - 1;
70
71	return ring->count + ring->next_to_clean - ring->next_to_use - 1;
72	}
73
74	/**
75	* igbvf_receive_skb - helper function to handle Rx indications
76	* @adapter: board private structure
77	* @netdev: pointer to netdev struct
78	* @skb: skb to indicate to stack
79	* @status: descriptor status field as written by hardware
80	* @vlan: descriptor vlan field as written by hardware (no le/be conversion)
81	* @skb: pointer to sk_buff to be indicated to stack
82	**/
83	static void igbvf_receive_skb(struct igbvf_adapter *adapter,
84	struct net_device *netdev,
85	struct sk_buff *skb,
86	u32 status, __le16 vlan)
87	{
88	u16 vid;
89
90	if (status & E1000_RXD_STAT_VP) {
91	if ((adapter->flags & IGBVF_FLAG_RX_LB_VLAN_BSWAP) &&
92	(status & E1000_RXDEXT_STATERR_LB))
93	vid = be16_to_cpu((__force __be16)vlan) & E1000_RXD_SPC_VLAN_MASK;
94	else
95	vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
96	if (test_bit(vid, adapter->active_vlans))
97	__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci: vid);
98	}
99
100	napi_gro_receive(napi: &adapter->rx_ring->napi, skb);
101	}
102
103	static inline void igbvf_rx_checksum_adv(struct igbvf_adapter *adapter,
104	u32 status_err, struct sk_buff *skb)
105	{
106	skb_checksum_none_assert(skb);
107
108	/* Ignore Checksum bit is set or checksum is disabled through ethtool */
109	if ((status_err & E1000_RXD_STAT_IXSM) ||
110	(adapter->flags & IGBVF_FLAG_RX_CSUM_DISABLED))
111	return;
112
113	/* TCP/UDP checksum error bit is set */
114	if (status_err &
115	(E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
116	/* let the stack verify checksum errors */
117	adapter->hw_csum_err++;
118	return;
119	}
120
121	/* It must be a TCP or UDP packet with a valid checksum */
122	if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
123	skb->ip_summed = CHECKSUM_UNNECESSARY;
124
125	adapter->hw_csum_good++;
126	}
127
128	/**
129	* igbvf_alloc_rx_buffers - Replace used receive buffers; packet split
130	* @rx_ring: address of ring structure to repopulate
131	* @cleaned_count: number of buffers to repopulate
132	**/
133	static void igbvf_alloc_rx_buffers(struct igbvf_ring *rx_ring,
134	int cleaned_count)
135	{
136	struct igbvf_adapter *adapter = rx_ring->adapter;
137	struct net_device *netdev = adapter->netdev;
138	struct pci_dev *pdev = adapter->pdev;
139	union e1000_adv_rx_desc *rx_desc;
140	struct igbvf_buffer *buffer_info;
141	struct sk_buff *skb;
142	unsigned int i;
143	int bufsz;
144
145	i = rx_ring->next_to_use;
146	buffer_info = &rx_ring->buffer_info[i];
147
148	if (adapter->rx_ps_hdr_size)
149	bufsz = adapter->rx_ps_hdr_size;
150	else
151	bufsz = adapter->rx_buffer_len;
152
153	while (cleaned_count--) {
154	rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);
155
156	if (adapter->rx_ps_hdr_size && !buffer_info->page_dma) {
157	if (!buffer_info->page) {
158	buffer_info->page = alloc_page(GFP_ATOMIC);
159	if (!buffer_info->page) {
160	adapter->alloc_rx_buff_failed++;
161	goto no_buffers;
162	}
163	buffer_info->page_offset = 0;
164	} else {
165	buffer_info->page_offset ^= PAGE_SIZE / 2;
166	}
167	buffer_info->page_dma =
168	dma_map_page(&pdev->dev, buffer_info->page,
169	buffer_info->page_offset,
170	PAGE_SIZE / 2,
171	DMA_FROM_DEVICE);
172	if (dma_mapping_error(dev: &pdev->dev,
173	dma_addr: buffer_info->page_dma)) {
174	__free_page(buffer_info->page);
175	buffer_info->page = NULL;
176	dev_err(&pdev->dev, "RX DMA map failed\n");
177	break;
178	}
179	}
180
181	if (!buffer_info->skb) {
182	skb = netdev_alloc_skb_ip_align(dev: netdev, length: bufsz);
183	if (!skb) {
184	adapter->alloc_rx_buff_failed++;
185	goto no_buffers;
186	}
187
188	buffer_info->skb = skb;
189	buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
190	bufsz,
191	DMA_FROM_DEVICE);
192	if (dma_mapping_error(dev: &pdev->dev, dma_addr: buffer_info->dma)) {
193	dev_kfree_skb(buffer_info->skb);
194	buffer_info->skb = NULL;
195	dev_err(&pdev->dev, "RX DMA map failed\n");
196	goto no_buffers;
197	}
198	}
199	/* Refresh the desc even if buffer_addrs didn't change because
200	* each write-back erases this info.
201	*/
202	if (adapter->rx_ps_hdr_size) {
203	rx_desc->read.pkt_addr =
204	cpu_to_le64(buffer_info->page_dma);
205	rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
206	} else {
207	rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
208	rx_desc->read.hdr_addr = 0;
209	}
210
211	i++;
212	if (i == rx_ring->count)
213	i = 0;
214	buffer_info = &rx_ring->buffer_info[i];
215	}
216
217	no_buffers:
218	if (rx_ring->next_to_use != i) {
219	rx_ring->next_to_use = i;
220	if (i == 0)
221	i = (rx_ring->count - 1);
222	else
223	i--;
224
225	/* Force memory writes to complete before letting h/w
226	* know there are new descriptors to fetch. (Only
227	* applicable for weak-ordered memory model archs,
228	* such as IA-64).
229	*/
230	wmb();
231	writel(val: i, addr: adapter->hw.hw_addr + rx_ring->tail);
232	}
233	}
234
235	/**
236	* igbvf_clean_rx_irq - Send received data up the network stack; legacy
237	* @adapter: board private structure
238	* @work_done: output parameter used to indicate completed work
239	* @work_to_do: input parameter setting limit of work
240	*
241	* the return value indicates whether actual cleaning was done, there
242	* is no guarantee that everything was cleaned
243	**/
244	static bool igbvf_clean_rx_irq(struct igbvf_adapter *adapter,
245	int *work_done, int work_to_do)
246	{
247	struct igbvf_ring *rx_ring = adapter->rx_ring;
248	struct net_device *netdev = adapter->netdev;
249	struct pci_dev *pdev = adapter->pdev;
250	union e1000_adv_rx_desc *rx_desc, *next_rxd;
251	struct igbvf_buffer *buffer_info, *next_buffer;
252	struct sk_buff *skb;
253	bool cleaned = false;
254	int cleaned_count = 0;
255	unsigned int total_bytes = 0, total_packets = 0;
256	unsigned int i;
257	u32 length, hlen, staterr;
258
259	i = rx_ring->next_to_clean;
260	rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);
261	staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
262
263	while (staterr & E1000_RXD_STAT_DD) {
264	if (*work_done >= work_to_do)
265	break;
266	(*work_done)++;
267	rmb(); /* read descriptor and rx_buffer_info after status DD */
268
269	buffer_info = &rx_ring->buffer_info[i];
270
271	/* HW will not DMA in data larger than the given buffer, even
272	* if it parses the (NFS, of course) header to be larger. In
273	* that case, it fills the header buffer and spills the rest
274	* into the page.
275	*/
276	hlen = le16_get_bits(v: rx_desc->wb.lower.lo_dword.hs_rss.hdr_info,
277	E1000_RXDADV_HDRBUFLEN_MASK);
278	if (hlen > adapter->rx_ps_hdr_size)
279	hlen = adapter->rx_ps_hdr_size;
280
281	length = le16_to_cpu(rx_desc->wb.upper.length);
282	cleaned = true;
283	cleaned_count++;
284
285	skb = buffer_info->skb;
286	prefetch(skb->data - NET_IP_ALIGN);
287	buffer_info->skb = NULL;
288	if (!adapter->rx_ps_hdr_size) {
289	dma_unmap_single(&pdev->dev, buffer_info->dma,
290	adapter->rx_buffer_len,
291	DMA_FROM_DEVICE);
292	buffer_info->dma = 0;
293	skb_put(skb, len: length);
294	goto send_up;
295	}
296
297	if (!skb_shinfo(skb)->nr_frags) {
298	dma_unmap_single(&pdev->dev, buffer_info->dma,
299	adapter->rx_ps_hdr_size,
300	DMA_FROM_DEVICE);
301	buffer_info->dma = 0;
302	skb_put(skb, len: hlen);
303	}
304
305	if (length) {
306	dma_unmap_page(&pdev->dev, buffer_info->page_dma,
307	PAGE_SIZE / 2,
308	DMA_FROM_DEVICE);
309	buffer_info->page_dma = 0;
310
311	skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
312	page: buffer_info->page,
313	off: buffer_info->page_offset,
314	size: length);
315
316	if ((adapter->rx_buffer_len > (PAGE_SIZE / 2)) ||
317	(page_count(page: buffer_info->page) != 1))
318	buffer_info->page = NULL;
319	else
320	get_page(page: buffer_info->page);
321
322	skb->len += length;
323	skb->data_len += length;
324	skb->truesize += PAGE_SIZE / 2;
325	}
326	send_up:
327	i++;
328	if (i == rx_ring->count)
329	i = 0;
330	next_rxd = IGBVF_RX_DESC_ADV(*rx_ring, i);
331	prefetch(next_rxd);
332	next_buffer = &rx_ring->buffer_info[i];
333
334	if (!(staterr & E1000_RXD_STAT_EOP)) {
335	buffer_info->skb = next_buffer->skb;
336	buffer_info->dma = next_buffer->dma;
337	next_buffer->skb = skb;
338	next_buffer->dma = 0;
339	goto next_desc;
340	}
341
342	if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
343	dev_kfree_skb_irq(skb);
344	goto next_desc;
345	}
346
347	total_bytes += skb->len;
348	total_packets++;
349
350	igbvf_rx_checksum_adv(adapter, status_err: staterr, skb);
351
352	skb->protocol = eth_type_trans(skb, dev: netdev);
353
354	igbvf_receive_skb(adapter, netdev, skb, status: staterr,
355	vlan: rx_desc->wb.upper.vlan);
356
357	next_desc:
358	rx_desc->wb.upper.status_error = 0;
359
360	/* return some buffers to hardware, one at a time is too slow */
361	if (cleaned_count >= IGBVF_RX_BUFFER_WRITE) {
362	igbvf_alloc_rx_buffers(rx_ring, cleaned_count);
363	cleaned_count = 0;
364	}
365
366	/* use prefetched values */
367	rx_desc = next_rxd;
368	buffer_info = next_buffer;
369
370	staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
371	}
372
373	rx_ring->next_to_clean = i;
374	cleaned_count = igbvf_desc_unused(ring: rx_ring);
375
376	if (cleaned_count)
377	igbvf_alloc_rx_buffers(rx_ring, cleaned_count);
378
379	adapter->total_rx_packets += total_packets;
380	adapter->total_rx_bytes += total_bytes;
381	netdev->stats.rx_bytes += total_bytes;
382	netdev->stats.rx_packets += total_packets;
383	return cleaned;
384	}
385
386	static void igbvf_put_txbuf(struct igbvf_adapter *adapter,
387	struct igbvf_buffer *buffer_info)
388	{
389	if (buffer_info->dma) {
390	if (buffer_info->mapped_as_page)
391	dma_unmap_page(&adapter->pdev->dev,
392	buffer_info->dma,
393	buffer_info->length,
394	DMA_TO_DEVICE);
395	else
396	dma_unmap_single(&adapter->pdev->dev,
397	buffer_info->dma,
398	buffer_info->length,
399	DMA_TO_DEVICE);
400	buffer_info->dma = 0;
401	}
402	if (buffer_info->skb) {
403	dev_kfree_skb_any(skb: buffer_info->skb);
404	buffer_info->skb = NULL;
405	}
406	buffer_info->time_stamp = 0;
407	}
408
409	/**
410	* igbvf_setup_tx_resources - allocate Tx resources (Descriptors)
411	* @adapter: board private structure
412	* @tx_ring: ring being initialized
413	*
414	* Return 0 on success, negative on failure
415	**/
416	int igbvf_setup_tx_resources(struct igbvf_adapter *adapter,
417	struct igbvf_ring *tx_ring)
418	{
419	struct pci_dev *pdev = adapter->pdev;
420	int size;
421
422	size = sizeof(struct igbvf_buffer) * tx_ring->count;
423	tx_ring->buffer_info = vzalloc(size);
424	if (!tx_ring->buffer_info)
425	goto err;
426
427	/* round up to nearest 4K */
428	tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
429	tx_ring->size = ALIGN(tx_ring->size, 4096);
430
431	tx_ring->desc = dma_alloc_coherent(dev: &pdev->dev, size: tx_ring->size,
432	dma_handle: &tx_ring->dma, GFP_KERNEL);
433	if (!tx_ring->desc)
434	goto err;
435
436	tx_ring->adapter = adapter;
437	tx_ring->next_to_use = 0;
438	tx_ring->next_to_clean = 0;
439
440	return 0;
441	err:
442	vfree(addr: tx_ring->buffer_info);
443	dev_err(&adapter->pdev->dev,
444	"Unable to allocate memory for the transmit descriptor ring\n");
445	return -ENOMEM;
446	}
447
448	/**
449	* igbvf_setup_rx_resources - allocate Rx resources (Descriptors)
450	* @adapter: board private structure
451	* @rx_ring: ring being initialized
452	*
453	* Returns 0 on success, negative on failure
454	**/
455	int igbvf_setup_rx_resources(struct igbvf_adapter *adapter,
456	struct igbvf_ring *rx_ring)
457	{
458	struct pci_dev *pdev = adapter->pdev;
459	int size, desc_len;
460
461	size = sizeof(struct igbvf_buffer) * rx_ring->count;
462	rx_ring->buffer_info = vzalloc(size);
463	if (!rx_ring->buffer_info)
464	goto err;
465
466	desc_len = sizeof(union e1000_adv_rx_desc);
467
468	/* Round up to nearest 4K */
469	rx_ring->size = rx_ring->count * desc_len;
470	rx_ring->size = ALIGN(rx_ring->size, 4096);
471
472	rx_ring->desc = dma_alloc_coherent(dev: &pdev->dev, size: rx_ring->size,
473	dma_handle: &rx_ring->dma, GFP_KERNEL);
474	if (!rx_ring->desc)
475	goto err;
476
477	rx_ring->next_to_clean = 0;
478	rx_ring->next_to_use = 0;
479
480	rx_ring->adapter = adapter;
481
482	return 0;
483
484	err:
485	vfree(addr: rx_ring->buffer_info);
486	rx_ring->buffer_info = NULL;
487	dev_err(&adapter->pdev->dev,
488	"Unable to allocate memory for the receive descriptor ring\n");
489	return -ENOMEM;
490	}
491
492	/**
493	* igbvf_clean_tx_ring - Free Tx Buffers
494	* @tx_ring: ring to be cleaned
495	**/
496	static void igbvf_clean_tx_ring(struct igbvf_ring *tx_ring)
497	{
498	struct igbvf_adapter *adapter = tx_ring->adapter;
499	struct igbvf_buffer *buffer_info;
500	unsigned long size;
501	unsigned int i;
502
503	if (!tx_ring->buffer_info)
504	return;
505
506	/* Free all the Tx ring sk_buffs */
507	for (i = 0; i < tx_ring->count; i++) {
508	buffer_info = &tx_ring->buffer_info[i];
509	igbvf_put_txbuf(adapter, buffer_info);
510	}
511
512	size = sizeof(struct igbvf_buffer) * tx_ring->count;
513	memset(tx_ring->buffer_info, 0, size);
514
515	/* Zero out the descriptor ring */
516	memset(tx_ring->desc, 0, tx_ring->size);
517
518	tx_ring->next_to_use = 0;
519	tx_ring->next_to_clean = 0;
520
521	writel(val: 0, addr: adapter->hw.hw_addr + tx_ring->head);
522	writel(val: 0, addr: adapter->hw.hw_addr + tx_ring->tail);
523	}
524
525	/**
526	* igbvf_free_tx_resources - Free Tx Resources per Queue
527	* @tx_ring: ring to free resources from
528	*
529	* Free all transmit software resources
530	**/
531	void igbvf_free_tx_resources(struct igbvf_ring *tx_ring)
532	{
533	struct pci_dev *pdev = tx_ring->adapter->pdev;
534
535	igbvf_clean_tx_ring(tx_ring);
536
537	vfree(addr: tx_ring->buffer_info);
538	tx_ring->buffer_info = NULL;
539
540	dma_free_coherent(dev: &pdev->dev, size: tx_ring->size, cpu_addr: tx_ring->desc,
541	dma_handle: tx_ring->dma);
542
543	tx_ring->desc = NULL;
544	}
545
546	/**
547	* igbvf_clean_rx_ring - Free Rx Buffers per Queue
548	* @rx_ring: ring structure pointer to free buffers from
549	**/
550	static void igbvf_clean_rx_ring(struct igbvf_ring *rx_ring)
551	{
552	struct igbvf_adapter *adapter = rx_ring->adapter;
553	struct igbvf_buffer *buffer_info;
554	struct pci_dev *pdev = adapter->pdev;
555	unsigned long size;
556	unsigned int i;
557
558	if (!rx_ring->buffer_info)
559	return;
560
561	/* Free all the Rx ring sk_buffs */
562	for (i = 0; i < rx_ring->count; i++) {
563	buffer_info = &rx_ring->buffer_info[i];
564	if (buffer_info->dma) {
565	if (adapter->rx_ps_hdr_size) {
566	dma_unmap_single(&pdev->dev, buffer_info->dma,
567	adapter->rx_ps_hdr_size,
568	DMA_FROM_DEVICE);
569	} else {
570	dma_unmap_single(&pdev->dev, buffer_info->dma,
571	adapter->rx_buffer_len,
572	DMA_FROM_DEVICE);
573	}
574	buffer_info->dma = 0;
575	}
576
577	if (buffer_info->skb) {
578	dev_kfree_skb(buffer_info->skb);
579	buffer_info->skb = NULL;
580	}
581
582	if (buffer_info->page) {
583	if (buffer_info->page_dma)
584	dma_unmap_page(&pdev->dev,
585	buffer_info->page_dma,
586	PAGE_SIZE / 2,
587	DMA_FROM_DEVICE);
588	put_page(page: buffer_info->page);
589	buffer_info->page = NULL;
590	buffer_info->page_dma = 0;
591	buffer_info->page_offset = 0;
592	}
593	}
594
595	size = sizeof(struct igbvf_buffer) * rx_ring->count;
596	memset(rx_ring->buffer_info, 0, size);
597
598	/* Zero out the descriptor ring */
599	memset(rx_ring->desc, 0, rx_ring->size);
600
601	rx_ring->next_to_clean = 0;
602	rx_ring->next_to_use = 0;
603
604	writel(val: 0, addr: adapter->hw.hw_addr + rx_ring->head);
605	writel(val: 0, addr: adapter->hw.hw_addr + rx_ring->tail);
606	}
607
608	/**
609	* igbvf_free_rx_resources - Free Rx Resources
610	* @rx_ring: ring to clean the resources from
611	*
612	* Free all receive software resources
613	**/
614
615	void igbvf_free_rx_resources(struct igbvf_ring *rx_ring)
616	{
617	struct pci_dev *pdev = rx_ring->adapter->pdev;
618
619	igbvf_clean_rx_ring(rx_ring);
620
621	vfree(addr: rx_ring->buffer_info);
622	rx_ring->buffer_info = NULL;
623
624	dma_free_coherent(dev: &pdev->dev, size: rx_ring->size, cpu_addr: rx_ring->desc,
625	dma_handle: rx_ring->dma);
626	rx_ring->desc = NULL;
627	}
628
629	/**
630	* igbvf_update_itr - update the dynamic ITR value based on statistics
631	* @adapter: pointer to adapter
632	* @itr_setting: current adapter->itr
633	* @packets: the number of packets during this measurement interval
634	* @bytes: the number of bytes during this measurement interval
635	*
636	* Stores a new ITR value based on packets and byte counts during the last
637	* interrupt. The advantage of per interrupt computation is faster updates
638	* and more accurate ITR for the current traffic pattern. Constants in this
639	* function were computed based on theoretical maximum wire speed and thresholds
640	* were set based on testing data as well as attempting to minimize response
641	* time while increasing bulk throughput.
642	**/
643	static enum latency_range igbvf_update_itr(struct igbvf_adapter *adapter,
644	enum latency_range itr_setting,
645	int packets, int bytes)
646	{
647	enum latency_range retval = itr_setting;
648
649	if (packets == 0)
650	goto update_itr_done;
651
652	switch (itr_setting) {
653	case lowest_latency:
654	/* handle TSO and jumbo frames */
655	if (bytes/packets > 8000)
656	retval = bulk_latency;
657	else if ((packets < 5) && (bytes > 512))
658	retval = low_latency;
659	break;
660	case low_latency: /* 50 usec aka 20000 ints/s */
661	if (bytes > 10000) {
662	/* this if handles the TSO accounting */
663	if (bytes/packets > 8000)
664	retval = bulk_latency;
665	else if ((packets < 10) || ((bytes/packets) > 1200))
666	retval = bulk_latency;
667	else if ((packets > 35))
668	retval = lowest_latency;
669	} else if (bytes/packets > 2000) {
670	retval = bulk_latency;
671	} else if (packets <= 2 && bytes < 512) {
672	retval = lowest_latency;
673	}
674	break;
675	case bulk_latency: /* 250 usec aka 4000 ints/s */
676	if (bytes > 25000) {
677	if (packets > 35)
678	retval = low_latency;
679	} else if (bytes < 6000) {
680	retval = low_latency;
681	}
682	break;
683	default:
684	break;
685	}
686
687	update_itr_done:
688	return retval;
689	}
690
691	static int igbvf_range_to_itr(enum latency_range current_range)
692	{
693	int new_itr;
694
695	switch (current_range) {
696	/* counts and packets in update_itr are dependent on these numbers */
697	case lowest_latency:
698	new_itr = IGBVF_70K_ITR;
699	break;
700	case low_latency:
701	new_itr = IGBVF_20K_ITR;
702	break;
703	case bulk_latency:
704	new_itr = IGBVF_4K_ITR;
705	break;
706	default:
707	new_itr = IGBVF_START_ITR;
708	break;
709	}
710	return new_itr;
711	}
712
713	static void igbvf_set_itr(struct igbvf_adapter *adapter)
714	{
715	u32 new_itr;
716
717	adapter->tx_ring->itr_range =
718	igbvf_update_itr(adapter,
719	itr_setting: adapter->tx_ring->itr_val,
720	packets: adapter->total_tx_packets,
721	bytes: adapter->total_tx_bytes);
722
723	/* conservative mode (itr 3) eliminates the lowest_latency setting */
724	if (adapter->requested_itr == 3 &&
725	adapter->tx_ring->itr_range == lowest_latency)
726	adapter->tx_ring->itr_range = low_latency;
727
728	new_itr = igbvf_range_to_itr(current_range: adapter->tx_ring->itr_range);
729
730	if (new_itr != adapter->tx_ring->itr_val) {
731	u32 current_itr = adapter->tx_ring->itr_val;
732	/* this attempts to bias the interrupt rate towards Bulk
733	* by adding intermediate steps when interrupt rate is
734	* increasing
735	*/
736	new_itr = new_itr > current_itr ?
737	min(current_itr + (new_itr >> 2), new_itr) :
738	new_itr;
739	adapter->tx_ring->itr_val = new_itr;
740
741	adapter->tx_ring->set_itr = 1;
742	}
743
744	adapter->rx_ring->itr_range =
745	igbvf_update_itr(adapter, itr_setting: adapter->rx_ring->itr_val,
746	packets: adapter->total_rx_packets,
747	bytes: adapter->total_rx_bytes);
748	if (adapter->requested_itr == 3 &&
749	adapter->rx_ring->itr_range == lowest_latency)
750	adapter->rx_ring->itr_range = low_latency;
751
752	new_itr = igbvf_range_to_itr(current_range: adapter->rx_ring->itr_range);
753
754	if (new_itr != adapter->rx_ring->itr_val) {
755	u32 current_itr = adapter->rx_ring->itr_val;
756
757	new_itr = new_itr > current_itr ?
758	min(current_itr + (new_itr >> 2), new_itr) :
759	new_itr;
760	adapter->rx_ring->itr_val = new_itr;
761
762	adapter->rx_ring->set_itr = 1;
763	}
764	}
765
766	/**
767	* igbvf_clean_tx_irq - Reclaim resources after transmit completes
768	* @tx_ring: ring structure to clean descriptors from
769	*
770	* returns true if ring is completely cleaned
771	**/
772	static bool igbvf_clean_tx_irq(struct igbvf_ring *tx_ring)
773	{
774	struct igbvf_adapter *adapter = tx_ring->adapter;
775	struct net_device *netdev = adapter->netdev;
776	struct igbvf_buffer *buffer_info;
777	struct sk_buff *skb;
778	union e1000_adv_tx_desc *tx_desc, *eop_desc;
779	unsigned int total_bytes = 0, total_packets = 0;
780	unsigned int i, count = 0;
781	bool cleaned = false;
782
783	i = tx_ring->next_to_clean;
784	buffer_info = &tx_ring->buffer_info[i];
785	eop_desc = buffer_info->next_to_watch;
786
787	do {
788	/* if next_to_watch is not set then there is no work pending */
789	if (!eop_desc)
790	break;
791
792	/* prevent any other reads prior to eop_desc */
793	smp_rmb();
794
795	/* if DD is not set pending work has not been completed */
796	if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
797	break;
798
799	/* clear next_to_watch to prevent false hangs */
800	buffer_info->next_to_watch = NULL;
801
802	for (cleaned = false; !cleaned; count++) {
803	tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
804	cleaned = (tx_desc == eop_desc);
805	skb = buffer_info->skb;
806
807	if (skb) {
808	unsigned int segs, bytecount;
809
810	/* gso_segs is currently only valid for tcp */
811	segs = skb_shinfo(skb)->gso_segs ?: 1;
812	/* multiply data chunks by size of headers */
813	bytecount = ((segs - 1) * skb_headlen(skb)) +
814	skb->len;
815	total_packets += segs;
816	total_bytes += bytecount;
817	}
818
819	igbvf_put_txbuf(adapter, buffer_info);
820	tx_desc->wb.status = 0;
821
822	i++;
823	if (i == tx_ring->count)
824	i = 0;
825
826	buffer_info = &tx_ring->buffer_info[i];
827	}
828
829	eop_desc = buffer_info->next_to_watch;
830	} while (count < tx_ring->count);
831
832	tx_ring->next_to_clean = i;
833
834	if (unlikely(count && netif_carrier_ok(netdev) &&
835	igbvf_desc_unused(tx_ring) >= IGBVF_TX_QUEUE_WAKE)) {
836	/* Make sure that anybody stopping the queue after this
837	* sees the new next_to_clean.
838	*/
839	smp_mb();
840	if (netif_queue_stopped(dev: netdev) &&
841	!(test_bit(__IGBVF_DOWN, &adapter->state))) {
842	netif_wake_queue(dev: netdev);
843	++adapter->restart_queue;
844	}
845	}
846
847	netdev->stats.tx_bytes += total_bytes;
848	netdev->stats.tx_packets += total_packets;
849	return count < tx_ring->count;
850	}
851
852	static irqreturn_t igbvf_msix_other(int irq, void *data)
853	{
854	struct net_device *netdev = data;
855	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
856	struct e1000_hw *hw = &adapter->hw;
857
858	adapter->int_counter1++;
859
860	hw->mac.get_link_status = 1;
861	if (!test_bit(__IGBVF_DOWN, &adapter->state))
862	mod_timer(timer: &adapter->watchdog_timer, expires: jiffies + 1);
863
864	ew32(EIMS, adapter->eims_other);
865
866	return IRQ_HANDLED;
867	}
868
869	static irqreturn_t igbvf_intr_msix_tx(int irq, void *data)
870	{
871	struct net_device *netdev = data;
872	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
873	struct e1000_hw *hw = &adapter->hw;
874	struct igbvf_ring *tx_ring = adapter->tx_ring;
875
876	if (tx_ring->set_itr) {
877	writel(val: tx_ring->itr_val,
878	addr: adapter->hw.hw_addr + tx_ring->itr_register);
879	adapter->tx_ring->set_itr = 0;
880	}
881
882	adapter->total_tx_bytes = 0;
883	adapter->total_tx_packets = 0;
884
885	/* auto mask will automatically re-enable the interrupt when we write
886	* EICS
887	*/
888	if (!igbvf_clean_tx_irq(tx_ring))
889	/* Ring was not completely cleaned, so fire another interrupt */
890	ew32(EICS, tx_ring->eims_value);
891	else
892	ew32(EIMS, tx_ring->eims_value);
893
894	return IRQ_HANDLED;
895	}
896
897	static irqreturn_t igbvf_intr_msix_rx(int irq, void *data)
898	{
899	struct net_device *netdev = data;
900	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
901
902	adapter->int_counter0++;
903
904	/* Write the ITR value calculated at the end of the
905	* previous interrupt.
906	*/
907	if (adapter->rx_ring->set_itr) {
908	writel(val: adapter->rx_ring->itr_val,
909	addr: adapter->hw.hw_addr + adapter->rx_ring->itr_register);
910	adapter->rx_ring->set_itr = 0;
911	}
912
913	if (napi_schedule_prep(n: &adapter->rx_ring->napi)) {
914	adapter->total_rx_bytes = 0;
915	adapter->total_rx_packets = 0;
916	__napi_schedule(n: &adapter->rx_ring->napi);
917	}
918
919	return IRQ_HANDLED;
920	}
921
922	#define IGBVF_NO_QUEUE -1
923
924	static void igbvf_assign_vector(struct igbvf_adapter *adapter, int rx_queue,
925	int tx_queue, int msix_vector)
926	{
927	struct e1000_hw *hw = &adapter->hw;
928	u32 ivar, index;
929
930	/* 82576 uses a table-based method for assigning vectors.
931	* Each queue has a single entry in the table to which we write
932	* a vector number along with a "valid" bit. Sadly, the layout
933	* of the table is somewhat counterintuitive.
934	*/
935	if (rx_queue > IGBVF_NO_QUEUE) {
936	index = (rx_queue >> 1);
937	ivar = array_er32(IVAR0, index);
938	if (rx_queue & 0x1) {
939	/* vector goes into third byte of register */
940	ivar = ivar & 0xFF00FFFF;
941	ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
942	} else {
943	/* vector goes into low byte of register */
944	ivar = ivar & 0xFFFFFF00;
945	ivar |= msix_vector | E1000_IVAR_VALID;
946	}
947	adapter->rx_ring[rx_queue].eims_value = BIT(msix_vector);
948	array_ew32(IVAR0, index, ivar);
949	}
950	if (tx_queue > IGBVF_NO_QUEUE) {
951	index = (tx_queue >> 1);
952	ivar = array_er32(IVAR0, index);
953	if (tx_queue & 0x1) {
954	/* vector goes into high byte of register */
955	ivar = ivar & 0x00FFFFFF;
956	ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
957	} else {
958	/* vector goes into second byte of register */
959	ivar = ivar & 0xFFFF00FF;
960	ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
961	}
962	adapter->tx_ring[tx_queue].eims_value = BIT(msix_vector);
963	array_ew32(IVAR0, index, ivar);
964	}
965	}
966
967	/**
968	* igbvf_configure_msix - Configure MSI-X hardware
969	* @adapter: board private structure
970	*
971	* igbvf_configure_msix sets up the hardware to properly
972	* generate MSI-X interrupts.
973	**/
974	static void igbvf_configure_msix(struct igbvf_adapter *adapter)
975	{
976	u32 tmp;
977	struct e1000_hw *hw = &adapter->hw;
978	struct igbvf_ring *tx_ring = adapter->tx_ring;
979	struct igbvf_ring *rx_ring = adapter->rx_ring;
980	int vector = 0;
981
982	adapter->eims_enable_mask = 0;
983
984	igbvf_assign_vector(adapter, IGBVF_NO_QUEUE, tx_queue: 0, msix_vector: vector++);
985	adapter->eims_enable_mask |= tx_ring->eims_value;
986	writel(val: tx_ring->itr_val, addr: hw->hw_addr + tx_ring->itr_register);
987	igbvf_assign_vector(adapter, rx_queue: 0, IGBVF_NO_QUEUE, msix_vector: vector++);
988	adapter->eims_enable_mask |= rx_ring->eims_value;
989	writel(val: rx_ring->itr_val, addr: hw->hw_addr + rx_ring->itr_register);
990
991	/* set vector for other causes, i.e. link changes */
992
993	tmp = (vector++ | E1000_IVAR_VALID);
994
995	ew32(IVAR_MISC, tmp);
996
997	adapter->eims_enable_mask = GENMASK(vector - 1, 0);
998	adapter->eims_other = BIT(vector - 1);
999	e1e_flush();
1000	}
1001
1002	static void igbvf_reset_interrupt_capability(struct igbvf_adapter *adapter)
1003	{
1004	if (adapter->msix_entries) {
1005	pci_disable_msix(dev: adapter->pdev);
1006	kfree(objp: adapter->msix_entries);
1007	adapter->msix_entries = NULL;
1008	}
1009	}
1010
1011	/**
1012	* igbvf_set_interrupt_capability - set MSI or MSI-X if supported
1013	* @adapter: board private structure
1014	*
1015	* Attempt to configure interrupts using the best available
1016	* capabilities of the hardware and kernel.
1017	**/
1018	static void igbvf_set_interrupt_capability(struct igbvf_adapter *adapter)
1019	{
1020	int err = -ENOMEM;
1021	int i;
1022
1023	/* we allocate 3 vectors, 1 for Tx, 1 for Rx, one for PF messages */
1024	adapter->msix_entries = kcalloc(n: 3, size: sizeof(struct msix_entry),
1025	GFP_KERNEL);
1026	if (adapter->msix_entries) {
1027	for (i = 0; i < 3; i++)
1028	adapter->msix_entries[i].entry = i;
1029
1030	err = pci_enable_msix_range(dev: adapter->pdev,
1031	entries: adapter->msix_entries, minvec: 3, maxvec: 3);
1032	}
1033
1034	if (err < 0) {
1035	/* MSI-X failed */
1036	dev_err(&adapter->pdev->dev,
1037	"Failed to initialize MSI-X interrupts.\n");
1038	igbvf_reset_interrupt_capability(adapter);
1039	}
1040	}
1041
1042	/**
1043	* igbvf_request_msix - Initialize MSI-X interrupts
1044	* @adapter: board private structure
1045	*
1046	* igbvf_request_msix allocates MSI-X vectors and requests interrupts from the
1047	* kernel.
1048	**/
1049	static int igbvf_request_msix(struct igbvf_adapter *adapter)
1050	{
1051	struct net_device *netdev = adapter->netdev;
1052	int err = 0, vector = 0;
1053
1054	if (strlen(netdev->name) < (IFNAMSIZ - 5)) {
1055	sprintf(buf: adapter->tx_ring->name, fmt: "%s-tx-0", netdev->name);
1056	sprintf(buf: adapter->rx_ring->name, fmt: "%s-rx-0", netdev->name);
1057	} else {
1058	memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
1059	memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
1060	}
1061
1062	err = request_irq(irq: adapter->msix_entries[vector].vector,
1063	handler: igbvf_intr_msix_tx, flags: 0, name: adapter->tx_ring->name,
1064	dev: netdev);
1065	if (err)
1066	goto out;
1067
1068	adapter->tx_ring->itr_register = E1000_EITR(vector);
1069	adapter->tx_ring->itr_val = adapter->current_itr;
1070	vector++;
1071
1072	err = request_irq(irq: adapter->msix_entries[vector].vector,
1073	handler: igbvf_intr_msix_rx, flags: 0, name: adapter->rx_ring->name,
1074	dev: netdev);
1075	if (err)
1076	goto free_irq_tx;
1077
1078	adapter->rx_ring->itr_register = E1000_EITR(vector);
1079	adapter->rx_ring->itr_val = adapter->current_itr;
1080	vector++;
1081
1082	err = request_irq(irq: adapter->msix_entries[vector].vector,
1083	handler: igbvf_msix_other, flags: 0, name: netdev->name, dev: netdev);
1084	if (err)
1085	goto free_irq_rx;
1086
1087	igbvf_configure_msix(adapter);
1088	return 0;
1089	free_irq_rx:
1090	free_irq(adapter->msix_entries[--vector].vector, netdev);
1091	free_irq_tx:
1092	free_irq(adapter->msix_entries[--vector].vector, netdev);
1093	out:
1094	return err;
1095	}
1096
1097	/**
1098	* igbvf_alloc_queues - Allocate memory for all rings
1099	* @adapter: board private structure to initialize
1100	**/
1101	static int igbvf_alloc_queues(struct igbvf_adapter *adapter)
1102	{
1103	struct net_device *netdev = adapter->netdev;
1104
1105	adapter->tx_ring = kzalloc(size: sizeof(struct igbvf_ring), GFP_KERNEL);
1106	if (!adapter->tx_ring)
1107	return -ENOMEM;
1108
1109	adapter->rx_ring = kzalloc(size: sizeof(struct igbvf_ring), GFP_KERNEL);
1110	if (!adapter->rx_ring) {
1111	kfree(objp: adapter->tx_ring);
1112	return -ENOMEM;
1113	}
1114
1115	netif_napi_add(dev: netdev, napi: &adapter->rx_ring->napi, poll: igbvf_poll);
1116
1117	return 0;
1118	}
1119
1120	/**
1121	* igbvf_request_irq - initialize interrupts
1122	* @adapter: board private structure
1123	*
1124	* Attempts to configure interrupts using the best available
1125	* capabilities of the hardware and kernel.
1126	**/
1127	static int igbvf_request_irq(struct igbvf_adapter *adapter)
1128	{
1129	int err = -1;
1130
1131	/* igbvf supports msi-x only */
1132	if (adapter->msix_entries)
1133	err = igbvf_request_msix(adapter);
1134
1135	if (!err)
1136	return err;
1137
1138	dev_err(&adapter->pdev->dev,
1139	"Unable to allocate interrupt, Error: %d\n", err);
1140
1141	return err;
1142	}
1143
1144	static void igbvf_free_irq(struct igbvf_adapter *adapter)
1145	{
1146	struct net_device *netdev = adapter->netdev;
1147	int vector;
1148
1149	if (adapter->msix_entries) {
1150	for (vector = 0; vector < 3; vector++)
1151	free_irq(adapter->msix_entries[vector].vector, netdev);
1152	}
1153	}
1154
1155	/**
1156	* igbvf_irq_disable - Mask off interrupt generation on the NIC
1157	* @adapter: board private structure
1158	**/
1159	static void igbvf_irq_disable(struct igbvf_adapter *adapter)
1160	{
1161	struct e1000_hw *hw = &adapter->hw;
1162
1163	ew32(EIMC, ~0);
1164
1165	if (adapter->msix_entries)
1166	ew32(EIAC, 0);
1167	}
1168
1169	/**
1170	* igbvf_irq_enable - Enable default interrupt generation settings
1171	* @adapter: board private structure
1172	**/
1173	static void igbvf_irq_enable(struct igbvf_adapter *adapter)
1174	{
1175	struct e1000_hw *hw = &adapter->hw;
1176
1177	ew32(EIAC, adapter->eims_enable_mask);
1178	ew32(EIAM, adapter->eims_enable_mask);
1179	ew32(EIMS, adapter->eims_enable_mask);
1180	}
1181
1182	/**
1183	* igbvf_poll - NAPI Rx polling callback
1184	* @napi: struct associated with this polling callback
1185	* @budget: amount of packets driver is allowed to process this poll
1186	**/
1187	static int igbvf_poll(struct napi_struct *napi, int budget)
1188	{
1189	struct igbvf_ring *rx_ring = container_of(napi, struct igbvf_ring, napi);
1190	struct igbvf_adapter *adapter = rx_ring->adapter;
1191	struct e1000_hw *hw = &adapter->hw;
1192	int work_done = 0;
1193
1194	igbvf_clean_rx_irq(adapter, work_done: &work_done, work_to_do: budget);
1195
1196	if (work_done == budget)
1197	return budget;
1198
1199	/* Exit the polling mode, but don't re-enable interrupts if stack might
1200	* poll us due to busy-polling
1201	*/
1202	if (likely(napi_complete_done(napi, work_done))) {
1203	if (adapter->requested_itr & 3)
1204	igbvf_set_itr(adapter);
1205
1206	if (!test_bit(__IGBVF_DOWN, &adapter->state))
1207	ew32(EIMS, adapter->rx_ring->eims_value);
1208	}
1209
1210	return work_done;
1211	}
1212
1213	/**
1214	* igbvf_set_rlpml - set receive large packet maximum length
1215	* @adapter: board private structure
1216	*
1217	* Configure the maximum size of packets that will be received
1218	*/
1219	static void igbvf_set_rlpml(struct igbvf_adapter *adapter)
1220	{
1221	int max_frame_size;
1222	struct e1000_hw *hw = &adapter->hw;
1223
1224	max_frame_size = adapter->max_frame_size + VLAN_TAG_SIZE;
1225
1226	spin_lock_bh(lock: &hw->mbx_lock);
1227
1228	e1000_rlpml_set_vf(hw, max_frame_size);
1229
1230	spin_unlock_bh(lock: &hw->mbx_lock);
1231	}
1232
1233	static int igbvf_vlan_rx_add_vid(struct net_device *netdev,
1234	__be16 proto, u16 vid)
1235	{
1236	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
1237	struct e1000_hw *hw = &adapter->hw;
1238
1239	spin_lock_bh(lock: &hw->mbx_lock);
1240
1241	if (hw->mac.ops.set_vfta(hw, vid, true)) {
1242	dev_warn(&adapter->pdev->dev, "Vlan id %d\n is not added", vid);
1243	spin_unlock_bh(lock: &hw->mbx_lock);
1244	return -EINVAL;
1245	}
1246
1247	spin_unlock_bh(lock: &hw->mbx_lock);
1248
1249	set_bit(nr: vid, addr: adapter->active_vlans);
1250	return 0;
1251	}
1252
1253	static int igbvf_vlan_rx_kill_vid(struct net_device *netdev,
1254	__be16 proto, u16 vid)
1255	{
1256	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
1257	struct e1000_hw *hw = &adapter->hw;
1258
1259	spin_lock_bh(lock: &hw->mbx_lock);
1260
1261	if (hw->mac.ops.set_vfta(hw, vid, false)) {
1262	dev_err(&adapter->pdev->dev,
1263	"Failed to remove vlan id %d\n", vid);
1264	spin_unlock_bh(lock: &hw->mbx_lock);
1265	return -EINVAL;
1266	}
1267
1268	spin_unlock_bh(lock: &hw->mbx_lock);
1269
1270	clear_bit(nr: vid, addr: adapter->active_vlans);
1271	return 0;
1272	}
1273
1274	static void igbvf_restore_vlan(struct igbvf_adapter *adapter)
1275	{
1276	u16 vid;
1277
1278	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1279	igbvf_vlan_rx_add_vid(netdev: adapter->netdev, htons(ETH_P_8021Q), vid);
1280	}
1281
1282	/**
1283	* igbvf_configure_tx - Configure Transmit Unit after Reset
1284	* @adapter: board private structure
1285	*
1286	* Configure the Tx unit of the MAC after a reset.
1287	**/
1288	static void igbvf_configure_tx(struct igbvf_adapter *adapter)
1289	{
1290	struct e1000_hw *hw = &adapter->hw;
1291	struct igbvf_ring *tx_ring = adapter->tx_ring;
1292	u64 tdba;
1293	u32 txdctl, dca_txctrl;
1294
1295	/* disable transmits */
1296	txdctl = er32(TXDCTL(0));
1297	ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
1298	e1e_flush();
1299	msleep(msecs: 10);
1300
1301	/* Setup the HW Tx Head and Tail descriptor pointers */
1302	ew32(TDLEN(0), tx_ring->count * sizeof(union e1000_adv_tx_desc));
1303	tdba = tx_ring->dma;
1304	ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));
1305	ew32(TDBAH(0), (tdba >> 32));
1306	ew32(TDH(0), 0);
1307	ew32(TDT(0), 0);
1308	tx_ring->head = E1000_TDH(0);
1309	tx_ring->tail = E1000_TDT(0);
1310
1311	/* Turn off Relaxed Ordering on head write-backs. The writebacks
1312	* MUST be delivered in order or it will completely screw up
1313	* our bookkeeping.
1314	*/
1315	dca_txctrl = er32(DCA_TXCTRL(0));
1316	dca_txctrl &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
1317	ew32(DCA_TXCTRL(0), dca_txctrl);
1318
1319	/* enable transmits */
1320	txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
1321	ew32(TXDCTL(0), txdctl);
1322
1323	/* Setup Transmit Descriptor Settings for eop descriptor */
1324	adapter->txd_cmd = E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_IFCS;
1325
1326	/* enable Report Status bit */
1327	adapter->txd_cmd |= E1000_ADVTXD_DCMD_RS;
1328	}
1329
1330	/**
1331	* igbvf_setup_srrctl - configure the receive control registers
1332	* @adapter: Board private structure
1333	**/
1334	static void igbvf_setup_srrctl(struct igbvf_adapter *adapter)
1335	{
1336	struct e1000_hw *hw = &adapter->hw;
1337	u32 srrctl = 0;
1338
1339	srrctl &= ~(E1000_SRRCTL_DESCTYPE_MASK |
1340	E1000_SRRCTL_BSIZEHDR_MASK |
1341	E1000_SRRCTL_BSIZEPKT_MASK);
1342
1343	/* Enable queue drop to avoid head of line blocking */
1344	srrctl |= E1000_SRRCTL_DROP_EN;
1345
1346	/* Setup buffer sizes */
1347	srrctl |= ALIGN(adapter->rx_buffer_len, 1024) >>
1348	E1000_SRRCTL_BSIZEPKT_SHIFT;
1349
1350	if (adapter->rx_buffer_len < 2048) {
1351	adapter->rx_ps_hdr_size = 0;
1352	srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
1353	} else {
1354	adapter->rx_ps_hdr_size = 128;
1355	srrctl |= adapter->rx_ps_hdr_size <<
1356	E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
1357	srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
1358	}
1359
1360	ew32(SRRCTL(0), srrctl);
1361	}
1362
1363	/**
1364	* igbvf_configure_rx - Configure Receive Unit after Reset
1365	* @adapter: board private structure
1366	*
1367	* Configure the Rx unit of the MAC after a reset.
1368	**/
1369	static void igbvf_configure_rx(struct igbvf_adapter *adapter)
1370	{
1371	struct e1000_hw *hw = &adapter->hw;
1372	struct igbvf_ring *rx_ring = adapter->rx_ring;
1373	u64 rdba;
1374	u32 rxdctl;
1375
1376	/* disable receives */
1377	rxdctl = er32(RXDCTL(0));
1378	ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
1379	e1e_flush();
1380	msleep(msecs: 10);
1381
1382	/* Setup the HW Rx Head and Tail Descriptor Pointers and
1383	* the Base and Length of the Rx Descriptor Ring
1384	*/
1385	rdba = rx_ring->dma;
1386	ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));
1387	ew32(RDBAH(0), (rdba >> 32));
1388	ew32(RDLEN(0), rx_ring->count * sizeof(union e1000_adv_rx_desc));
1389	rx_ring->head = E1000_RDH(0);
1390	rx_ring->tail = E1000_RDT(0);
1391	ew32(RDH(0), 0);
1392	ew32(RDT(0), 0);
1393
1394	rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
1395	rxdctl &= 0xFFF00000;
1396	rxdctl |= IGBVF_RX_PTHRESH;
1397	rxdctl |= IGBVF_RX_HTHRESH << 8;
1398	rxdctl |= IGBVF_RX_WTHRESH << 16;
1399
1400	igbvf_set_rlpml(adapter);
1401
1402	/* enable receives */
1403	ew32(RXDCTL(0), rxdctl);
1404	}
1405
1406	/**
1407	* igbvf_set_multi - Multicast and Promiscuous mode set
1408	* @netdev: network interface device structure
1409	*
1410	* The set_multi entry point is called whenever the multicast address
1411	* list or the network interface flags are updated. This routine is
1412	* responsible for configuring the hardware for proper multicast,
1413	* promiscuous mode, and all-multi behavior.
1414	**/
1415	static void igbvf_set_multi(struct net_device *netdev)
1416	{
1417	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
1418	struct e1000_hw *hw = &adapter->hw;
1419	struct netdev_hw_addr *ha;
1420	u8 *mta_list = NULL;
1421	int i;
1422
1423	if (!netdev_mc_empty(netdev)) {
1424	mta_list = kmalloc_array(netdev_mc_count(netdev), ETH_ALEN,
1425	GFP_ATOMIC);
1426	if (!mta_list)
1427	return;
1428	}
1429
1430	/* prepare a packed array of only addresses. */
1431	i = 0;
1432	netdev_for_each_mc_addr(ha, netdev)
1433	memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
1434
1435	spin_lock_bh(lock: &hw->mbx_lock);
1436
1437	hw->mac.ops.update_mc_addr_list(hw, mta_list, i, 0, 0);
1438
1439	spin_unlock_bh(lock: &hw->mbx_lock);
1440	kfree(objp: mta_list);
1441	}
1442
1443	/**
1444	* igbvf_set_uni - Configure unicast MAC filters
1445	* @netdev: network interface device structure
1446	*
1447	* This routine is responsible for configuring the hardware for proper
1448	* unicast filters.
1449	**/
1450	static int igbvf_set_uni(struct net_device *netdev)
1451	{
1452	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
1453	struct e1000_hw *hw = &adapter->hw;
1454
1455	if (netdev_uc_count(netdev) > IGBVF_MAX_MAC_FILTERS) {
1456	pr_err("Too many unicast filters - No Space\n");
1457	return -ENOSPC;
1458	}
1459
1460	spin_lock_bh(lock: &hw->mbx_lock);
1461
1462	/* Clear all unicast MAC filters */
1463	hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_CLR, NULL);
1464
1465	spin_unlock_bh(lock: &hw->mbx_lock);
1466
1467	if (!netdev_uc_empty(netdev)) {
1468	struct netdev_hw_addr *ha;
1469
1470	/* Add MAC filters one by one */
1471	netdev_for_each_uc_addr(ha, netdev) {
1472	spin_lock_bh(lock: &hw->mbx_lock);
1473
1474	hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_ADD,
1475	ha->addr);
1476
1477	spin_unlock_bh(lock: &hw->mbx_lock);
1478	udelay(200);
1479	}
1480	}
1481
1482	return 0;
1483	}
1484
1485	static void igbvf_set_rx_mode(struct net_device *netdev)
1486	{
1487	igbvf_set_multi(netdev);
1488	igbvf_set_uni(netdev);
1489	}
1490
1491	/**
1492	* igbvf_configure - configure the hardware for Rx and Tx
1493	* @adapter: private board structure
1494	**/
1495	static void igbvf_configure(struct igbvf_adapter *adapter)
1496	{
1497	igbvf_set_rx_mode(netdev: adapter->netdev);
1498
1499	igbvf_restore_vlan(adapter);
1500
1501	igbvf_configure_tx(adapter);
1502	igbvf_setup_srrctl(adapter);
1503	igbvf_configure_rx(adapter);
1504	igbvf_alloc_rx_buffers(rx_ring: adapter->rx_ring,
1505	cleaned_count: igbvf_desc_unused(ring: adapter->rx_ring));
1506	}
1507
1508	/* igbvf_reset - bring the hardware into a known good state
1509	* @adapter: private board structure
1510	*
1511	* This function boots the hardware and enables some settings that
1512	* require a configuration cycle of the hardware - those cannot be
1513	* set/changed during runtime. After reset the device needs to be
1514	* properly configured for Rx, Tx etc.
1515	*/
1516	static void igbvf_reset(struct igbvf_adapter *adapter)
1517	{
1518	struct e1000_mac_info *mac = &adapter->hw.mac;
1519	struct net_device *netdev = adapter->netdev;
1520	struct e1000_hw *hw = &adapter->hw;
1521
1522	spin_lock_bh(lock: &hw->mbx_lock);
1523
1524	/* Allow time for pending master requests to run */
1525	if (mac->ops.reset_hw(hw))
1526	dev_info(&adapter->pdev->dev, "PF still resetting\n");
1527
1528	mac->ops.init_hw(hw);
1529
1530	spin_unlock_bh(lock: &hw->mbx_lock);
1531
1532	if (is_valid_ether_addr(addr: adapter->hw.mac.addr)) {
1533	eth_hw_addr_set(dev: netdev, addr: adapter->hw.mac.addr);
1534	memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1535	netdev->addr_len);
1536	}
1537
1538	adapter->last_reset = jiffies;
1539	}
1540
1541	int igbvf_up(struct igbvf_adapter *adapter)
1542	{
1543	struct e1000_hw *hw = &adapter->hw;
1544
1545	/* hardware has been reset, we need to reload some things */
1546	igbvf_configure(adapter);
1547
1548	clear_bit(nr: __IGBVF_DOWN, addr: &adapter->state);
1549
1550	napi_enable(n: &adapter->rx_ring->napi);
1551	if (adapter->msix_entries)
1552	igbvf_configure_msix(adapter);
1553
1554	/* Clear any pending interrupts. */
1555	er32(EICR);
1556	igbvf_irq_enable(adapter);
1557
1558	/* start the watchdog */
1559	hw->mac.get_link_status = 1;
1560	mod_timer(timer: &adapter->watchdog_timer, expires: jiffies + 1);
1561
1562	return 0;
1563	}
1564
1565	void igbvf_down(struct igbvf_adapter *adapter)
1566	{
1567	struct net_device *netdev = adapter->netdev;
1568	struct e1000_hw *hw = &adapter->hw;
1569	u32 rxdctl, txdctl;
1570
1571	/* signal that we're down so the interrupt handler does not
1572	* reschedule our watchdog timer
1573	*/
1574	set_bit(nr: __IGBVF_DOWN, addr: &adapter->state);
1575
1576	/* disable receives in the hardware */
1577	rxdctl = er32(RXDCTL(0));
1578	ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
1579
1580	netif_carrier_off(dev: netdev);
1581	netif_stop_queue(dev: netdev);
1582
1583	/* disable transmits in the hardware */
1584	txdctl = er32(TXDCTL(0));
1585	ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
1586
1587	/* flush both disables and wait for them to finish */
1588	e1e_flush();
1589	msleep(msecs: 10);
1590
1591	napi_disable(n: &adapter->rx_ring->napi);
1592
1593	igbvf_irq_disable(adapter);
1594
1595	del_timer_sync(timer: &adapter->watchdog_timer);
1596
1597	/* record the stats before reset*/
1598	igbvf_update_stats(adapter);
1599
1600	adapter->link_speed = 0;
1601	adapter->link_duplex = 0;
1602
1603	igbvf_reset(adapter);
1604	igbvf_clean_tx_ring(tx_ring: adapter->tx_ring);
1605	igbvf_clean_rx_ring(rx_ring: adapter->rx_ring);
1606	}
1607
1608	void igbvf_reinit_locked(struct igbvf_adapter *adapter)
1609	{
1610	might_sleep();
1611	while (test_and_set_bit(nr: __IGBVF_RESETTING, addr: &adapter->state))
1612	usleep_range(min: 1000, max: 2000);
1613	igbvf_down(adapter);
1614	igbvf_up(adapter);
1615	clear_bit(nr: __IGBVF_RESETTING, addr: &adapter->state);
1616	}
1617
1618	/**
1619	* igbvf_sw_init - Initialize general software structures (struct igbvf_adapter)
1620	* @adapter: board private structure to initialize
1621	*
1622	* igbvf_sw_init initializes the Adapter private data structure.
1623	* Fields are initialized based on PCI device information and
1624	* OS network device settings (MTU size).
1625	**/
1626	static int igbvf_sw_init(struct igbvf_adapter *adapter)
1627	{
1628	struct net_device *netdev = adapter->netdev;
1629	s32 rc;
1630
1631	adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;
1632	adapter->rx_ps_hdr_size = 0;
1633	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1634	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1635
1636	adapter->tx_int_delay = 8;
1637	adapter->tx_abs_int_delay = 32;
1638	adapter->rx_int_delay = 0;
1639	adapter->rx_abs_int_delay = 8;
1640	adapter->requested_itr = 3;
1641	adapter->current_itr = IGBVF_START_ITR;
1642
1643	/* Set various function pointers */
1644	adapter->ei->init_ops(&adapter->hw);
1645
1646	rc = adapter->hw.mac.ops.init_params(&adapter->hw);
1647	if (rc)
1648	return rc;
1649
1650	rc = adapter->hw.mbx.ops.init_params(&adapter->hw);
1651	if (rc)
1652	return rc;
1653
1654	igbvf_set_interrupt_capability(adapter);
1655
1656	if (igbvf_alloc_queues(adapter))
1657	return -ENOMEM;
1658
1659	spin_lock_init(&adapter->tx_queue_lock);
1660
1661	/* Explicitly disable IRQ since the NIC can be in any state. */
1662	igbvf_irq_disable(adapter);
1663
1664	spin_lock_init(&adapter->stats_lock);
1665	spin_lock_init(&adapter->hw.mbx_lock);
1666
1667	set_bit(nr: __IGBVF_DOWN, addr: &adapter->state);
1668	return 0;
1669	}
1670
1671	static void igbvf_initialize_last_counter_stats(struct igbvf_adapter *adapter)
1672	{
1673	struct e1000_hw *hw = &adapter->hw;
1674
1675	adapter->stats.last_gprc = er32(VFGPRC);
1676	adapter->stats.last_gorc = er32(VFGORC);
1677	adapter->stats.last_gptc = er32(VFGPTC);
1678	adapter->stats.last_gotc = er32(VFGOTC);
1679	adapter->stats.last_mprc = er32(VFMPRC);
1680	adapter->stats.last_gotlbc = er32(VFGOTLBC);
1681	adapter->stats.last_gptlbc = er32(VFGPTLBC);
1682	adapter->stats.last_gorlbc = er32(VFGORLBC);
1683	adapter->stats.last_gprlbc = er32(VFGPRLBC);
1684
1685	adapter->stats.base_gprc = er32(VFGPRC);
1686	adapter->stats.base_gorc = er32(VFGORC);
1687	adapter->stats.base_gptc = er32(VFGPTC);
1688	adapter->stats.base_gotc = er32(VFGOTC);
1689	adapter->stats.base_mprc = er32(VFMPRC);
1690	adapter->stats.base_gotlbc = er32(VFGOTLBC);
1691	adapter->stats.base_gptlbc = er32(VFGPTLBC);
1692	adapter->stats.base_gorlbc = er32(VFGORLBC);
1693	adapter->stats.base_gprlbc = er32(VFGPRLBC);
1694	}
1695
1696	/**
1697	* igbvf_open - Called when a network interface is made active
1698	* @netdev: network interface device structure
1699	*
1700	* Returns 0 on success, negative value on failure
1701	*
1702	* The open entry point is called when a network interface is made
1703	* active by the system (IFF_UP). At this point all resources needed
1704	* for transmit and receive operations are allocated, the interrupt
1705	* handler is registered with the OS, the watchdog timer is started,
1706	* and the stack is notified that the interface is ready.
1707	**/
1708	static int igbvf_open(struct net_device *netdev)
1709	{
1710	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
1711	struct e1000_hw *hw = &adapter->hw;
1712	int err;
1713
1714	/* disallow open during test */
1715	if (test_bit(__IGBVF_TESTING, &adapter->state))
1716	return -EBUSY;
1717
1718	/* allocate transmit descriptors */
1719	err = igbvf_setup_tx_resources(adapter, tx_ring: adapter->tx_ring);
1720	if (err)
1721	goto err_setup_tx;
1722
1723	/* allocate receive descriptors */
1724	err = igbvf_setup_rx_resources(adapter, rx_ring: adapter->rx_ring);
1725	if (err)
1726	goto err_setup_rx;
1727
1728	/* before we allocate an interrupt, we must be ready to handle it.
1729	* Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1730	* as soon as we call pci_request_irq, so we have to setup our
1731	* clean_rx handler before we do so.
1732	*/
1733	igbvf_configure(adapter);
1734
1735	err = igbvf_request_irq(adapter);
1736	if (err)
1737	goto err_req_irq;
1738
1739	/* From here on the code is the same as igbvf_up() */
1740	clear_bit(nr: __IGBVF_DOWN, addr: &adapter->state);
1741
1742	napi_enable(n: &adapter->rx_ring->napi);
1743
1744	/* clear any pending interrupts */
1745	er32(EICR);
1746
1747	igbvf_irq_enable(adapter);
1748
1749	/* start the watchdog */
1750	hw->mac.get_link_status = 1;
1751	mod_timer(timer: &adapter->watchdog_timer, expires: jiffies + 1);
1752
1753	return 0;
1754
1755	err_req_irq:
1756	igbvf_free_rx_resources(rx_ring: adapter->rx_ring);
1757	err_setup_rx:
1758	igbvf_free_tx_resources(tx_ring: adapter->tx_ring);
1759	err_setup_tx:
1760	igbvf_reset(adapter);
1761
1762	return err;
1763	}
1764
1765	/**
1766	* igbvf_close - Disables a network interface
1767	* @netdev: network interface device structure
1768	*
1769	* Returns 0, this is not allowed to fail
1770	*
1771	* The close entry point is called when an interface is de-activated
1772	* by the OS. The hardware is still under the drivers control, but
1773	* needs to be disabled. A global MAC reset is issued to stop the
1774	* hardware, and all transmit and receive resources are freed.
1775	**/
1776	static int igbvf_close(struct net_device *netdev)
1777	{
1778	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
1779
1780	WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));
1781	igbvf_down(adapter);
1782
1783	igbvf_free_irq(adapter);
1784
1785	igbvf_free_tx_resources(tx_ring: adapter->tx_ring);
1786	igbvf_free_rx_resources(rx_ring: adapter->rx_ring);
1787
1788	return 0;
1789	}
1790
1791	/**
1792	* igbvf_set_mac - Change the Ethernet Address of the NIC
1793	* @netdev: network interface device structure
1794	* @p: pointer to an address structure
1795	*
1796	* Returns 0 on success, negative on failure
1797	**/
1798	static int igbvf_set_mac(struct net_device *netdev, void *p)
1799	{
1800	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
1801	struct e1000_hw *hw = &adapter->hw;
1802	struct sockaddr *addr = p;
1803
1804	if (!is_valid_ether_addr(addr: addr->sa_data))
1805	return -EADDRNOTAVAIL;
1806
1807	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
1808
1809	spin_lock_bh(lock: &hw->mbx_lock);
1810
1811	hw->mac.ops.rar_set(hw, hw->mac.addr, 0);
1812
1813	spin_unlock_bh(lock: &hw->mbx_lock);
1814
1815	if (!ether_addr_equal(addr1: addr->sa_data, addr2: hw->mac.addr))
1816	return -EADDRNOTAVAIL;
1817
1818	eth_hw_addr_set(dev: netdev, addr: addr->sa_data);
1819
1820	return 0;
1821	}
1822
1823	#define UPDATE_VF_COUNTER(reg, name) \
1824	{ \
1825	u32 current_counter = er32(reg); \
1826	if (current_counter < adapter->stats.last_##name) \
1827	adapter->stats.name += 0x100000000LL; \
1828	adapter->stats.last_##name = current_counter; \
1829	adapter->stats.name &= 0xFFFFFFFF00000000LL; \
1830	adapter->stats.name |= current_counter; \
1831	}
1832
1833	/**
1834	* igbvf_update_stats - Update the board statistics counters
1835	* @adapter: board private structure
1836	**/
1837	void igbvf_update_stats(struct igbvf_adapter *adapter)
1838	{
1839	struct e1000_hw *hw = &adapter->hw;
1840	struct pci_dev *pdev = adapter->pdev;
1841
1842	/* Prevent stats update while adapter is being reset, link is down
1843	* or if the pci connection is down.
1844	*/
1845	if (adapter->link_speed == 0)
1846	return;
1847
1848	if (test_bit(__IGBVF_RESETTING, &adapter->state))
1849	return;
1850
1851	if (pci_channel_offline(pdev))
1852	return;
1853
1854	UPDATE_VF_COUNTER(VFGPRC, gprc);
1855	UPDATE_VF_COUNTER(VFGORC, gorc);
1856	UPDATE_VF_COUNTER(VFGPTC, gptc);
1857	UPDATE_VF_COUNTER(VFGOTC, gotc);
1858	UPDATE_VF_COUNTER(VFMPRC, mprc);
1859	UPDATE_VF_COUNTER(VFGOTLBC, gotlbc);
1860	UPDATE_VF_COUNTER(VFGPTLBC, gptlbc);
1861	UPDATE_VF_COUNTER(VFGORLBC, gorlbc);
1862	UPDATE_VF_COUNTER(VFGPRLBC, gprlbc);
1863
1864	/* Fill out the OS statistics structure */
1865	adapter->netdev->stats.multicast = adapter->stats.mprc;
1866	}
1867
1868	static void igbvf_print_link_info(struct igbvf_adapter *adapter)
1869	{
1870	dev_info(&adapter->pdev->dev, "Link is Up %d Mbps %s Duplex\n",
1871	adapter->link_speed,
1872	adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");
1873	}
1874
1875	static bool igbvf_has_link(struct igbvf_adapter *adapter)
1876	{
1877	struct e1000_hw *hw = &adapter->hw;
1878	s32 ret_val = E1000_SUCCESS;
1879	bool link_active;
1880
1881	/* If interface is down, stay link down */
1882	if (test_bit(__IGBVF_DOWN, &adapter->state))
1883	return false;
1884
1885	spin_lock_bh(lock: &hw->mbx_lock);
1886
1887	ret_val = hw->mac.ops.check_for_link(hw);
1888
1889	spin_unlock_bh(lock: &hw->mbx_lock);
1890
1891	link_active = !hw->mac.get_link_status;
1892
1893	/* if check for link returns error we will need to reset */
1894	if (ret_val && time_after(jiffies, adapter->last_reset + (10 * HZ)))
1895	schedule_work(work: &adapter->reset_task);
1896
1897	return link_active;
1898	}
1899
1900	/**
1901	* igbvf_watchdog - Timer Call-back
1902	* @t: timer list pointer containing private struct
1903	**/
1904	static void igbvf_watchdog(struct timer_list *t)
1905	{
1906	struct igbvf_adapter *adapter = from_timer(adapter, t, watchdog_timer);
1907
1908	/* Do the rest outside of interrupt context */
1909	schedule_work(work: &adapter->watchdog_task);
1910	}
1911
1912	static void igbvf_watchdog_task(struct work_struct *work)
1913	{
1914	struct igbvf_adapter *adapter = container_of(work,
1915	struct igbvf_adapter,
1916	watchdog_task);
1917	struct net_device *netdev = adapter->netdev;
1918	struct e1000_mac_info *mac = &adapter->hw.mac;
1919	struct igbvf_ring *tx_ring = adapter->tx_ring;
1920	struct e1000_hw *hw = &adapter->hw;
1921	u32 link;
1922	int tx_pending = 0;
1923
1924	link = igbvf_has_link(adapter);
1925
1926	if (link) {
1927	if (!netif_carrier_ok(dev: netdev)) {
1928	mac->ops.get_link_up_info(&adapter->hw,
1929	&adapter->link_speed,
1930	&adapter->link_duplex);
1931	igbvf_print_link_info(adapter);
1932
1933	netif_carrier_on(dev: netdev);
1934	netif_wake_queue(dev: netdev);
1935	}
1936	} else {
1937	if (netif_carrier_ok(dev: netdev)) {
1938	adapter->link_speed = 0;
1939	adapter->link_duplex = 0;
1940	dev_info(&adapter->pdev->dev, "Link is Down\n");
1941	netif_carrier_off(dev: netdev);
1942	netif_stop_queue(dev: netdev);
1943	}
1944	}
1945
1946	if (netif_carrier_ok(dev: netdev)) {
1947	igbvf_update_stats(adapter);
1948	} else {
1949	tx_pending = (igbvf_desc_unused(ring: tx_ring) + 1 <
1950	tx_ring->count);
1951	if (tx_pending) {
1952	/* We've lost link, so the controller stops DMA,
1953	* but we've got queued Tx work that's never going
1954	* to get done, so reset controller to flush Tx.
1955	* (Do the reset outside of interrupt context).
1956	*/
1957	adapter->tx_timeout_count++;
1958	schedule_work(work: &adapter->reset_task);
1959	}
1960	}
1961
1962	/* Cause software interrupt to ensure Rx ring is cleaned */
1963	ew32(EICS, adapter->rx_ring->eims_value);
1964
1965	/* Reset the timer */
1966	if (!test_bit(__IGBVF_DOWN, &adapter->state))
1967	mod_timer(timer: &adapter->watchdog_timer,
1968	expires: round_jiffies(j: jiffies + (2 * HZ)));
1969	}
1970
1971	#define IGBVF_TX_FLAGS_CSUM 0x00000001
1972	#define IGBVF_TX_FLAGS_VLAN 0x00000002
1973	#define IGBVF_TX_FLAGS_TSO 0x00000004
1974	#define IGBVF_TX_FLAGS_IPV4 0x00000008
1975	#define IGBVF_TX_FLAGS_VLAN_MASK 0xffff0000
1976	#define IGBVF_TX_FLAGS_VLAN_SHIFT 16
1977
1978	static void igbvf_tx_ctxtdesc(struct igbvf_ring *tx_ring, u32 vlan_macip_lens,
1979	u32 type_tucmd, u32 mss_l4len_idx)
1980	{
1981	struct e1000_adv_tx_context_desc *context_desc;
1982	struct igbvf_buffer *buffer_info;
1983	u16 i = tx_ring->next_to_use;
1984
1985	context_desc = IGBVF_TX_CTXTDESC_ADV(*tx_ring, i);
1986	buffer_info = &tx_ring->buffer_info[i];
1987
1988	i++;
1989	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1990
1991	/* set bits to identify this as an advanced context descriptor */
1992	type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
1993
1994	context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
1995	context_desc->seqnum_seed = 0;
1996	context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
1997	context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
1998
1999	buffer_info->time_stamp = jiffies;
2000	buffer_info->dma = 0;
2001	}
2002
2003	static int igbvf_tso(struct igbvf_ring *tx_ring,
2004	struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2005	{
2006	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
2007	union {
2008	struct iphdr *v4;
2009	struct ipv6hdr *v6;
2010	unsigned char *hdr;
2011	} ip;
2012	union {
2013	struct tcphdr *tcp;
2014	unsigned char *hdr;
2015	} l4;
2016	u32 paylen, l4_offset;
2017	int err;
2018
2019	if (skb->ip_summed != CHECKSUM_PARTIAL)
2020	return 0;
2021
2022	if (!skb_is_gso(skb))
2023	return 0;
2024
2025	err = skb_cow_head(skb, headroom: 0);
2026	if (err < 0)
2027	return err;
2028
2029	ip.hdr = skb_network_header(skb);
2030	l4.hdr = skb_checksum_start(skb);
2031
2032	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2033	type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
2034
2035	/* initialize outer IP header fields */
2036	if (ip.v4->version == 4) {
2037	unsigned char *csum_start = skb_checksum_start(skb);
2038	unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
2039
2040	/* IP header will have to cancel out any data that
2041	* is not a part of the outer IP header
2042	*/
2043	ip.v4->check = csum_fold(sum: csum_partial(buff: trans_start,
2044	len: csum_start - trans_start,
2045	sum: 0));
2046	type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
2047
2048	ip.v4->tot_len = 0;
2049	} else {
2050	ip.v6->payload_len = 0;
2051	}
2052
2053	/* determine offset of inner transport header */
2054	l4_offset = l4.hdr - skb->data;
2055
2056	/* compute length of segmentation header */
2057	*hdr_len = (l4.tcp->doff * 4) + l4_offset;
2058
2059	/* remove payload length from inner checksum */
2060	paylen = skb->len - l4_offset;
2061	csum_replace_by_diff(sum: &l4.tcp->check, diff: (__force __wsum)htonl(paylen));
2062
2063	/* MSS L4LEN IDX */
2064	mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
2065	mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
2066
2067	/* VLAN MACLEN IPLEN */
2068	vlan_macip_lens = l4.hdr - ip.hdr;
2069	vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
2070	vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;
2071
2072	igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
2073
2074	return 1;
2075	}
2076
2077	static bool igbvf_tx_csum(struct igbvf_ring *tx_ring, struct sk_buff *skb,
2078	u32 tx_flags, __be16 protocol)
2079	{
2080	u32 vlan_macip_lens = 0;
2081	u32 type_tucmd = 0;
2082
2083	if (skb->ip_summed != CHECKSUM_PARTIAL) {
2084	csum_failed:
2085	if (!(tx_flags & IGBVF_TX_FLAGS_VLAN))
2086	return false;
2087	goto no_csum;
2088	}
2089
2090	switch (skb->csum_offset) {
2091	case offsetof(struct tcphdr, check):
2092	type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
2093	fallthrough;
2094	case offsetof(struct udphdr, check):
2095	break;
2096	case offsetof(struct sctphdr, checksum):
2097	/* validate that this is actually an SCTP request */
2098	if (skb_csum_is_sctp(skb)) {
2099	type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
2100	break;
2101	}
2102	fallthrough;
2103	default:
2104	skb_checksum_help(skb);
2105	goto csum_failed;
2106	}
2107
2108	vlan_macip_lens = skb_checksum_start_offset(skb) -
2109	skb_network_offset(skb);
2110	no_csum:
2111	vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
2112	vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;
2113
2114	igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx: 0);
2115	return true;
2116	}
2117
2118	static int igbvf_maybe_stop_tx(struct net_device *netdev, int size)
2119	{
2120	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2121
2122	/* there is enough descriptors then we don't need to worry */
2123	if (igbvf_desc_unused(ring: adapter->tx_ring) >= size)
2124	return 0;
2125
2126	netif_stop_queue(dev: netdev);
2127
2128	/* Herbert's original patch had:
2129	* smp_mb__after_netif_stop_queue();
2130	* but since that doesn't exist yet, just open code it.
2131	*/
2132	smp_mb();
2133
2134	/* We need to check again just in case room has been made available */
2135	if (igbvf_desc_unused(ring: adapter->tx_ring) < size)
2136	return -EBUSY;
2137
2138	netif_wake_queue(dev: netdev);
2139
2140	++adapter->restart_queue;
2141	return 0;
2142	}
2143
2144	#define IGBVF_MAX_TXD_PWR 16
2145	#define IGBVF_MAX_DATA_PER_TXD (1u << IGBVF_MAX_TXD_PWR)
2146
2147	static inline int igbvf_tx_map_adv(struct igbvf_adapter *adapter,
2148	struct igbvf_ring *tx_ring,
2149	struct sk_buff *skb)
2150	{
2151	struct igbvf_buffer *buffer_info;
2152	struct pci_dev *pdev = adapter->pdev;
2153	unsigned int len = skb_headlen(skb);
2154	unsigned int count = 0, i;
2155	unsigned int f;
2156
2157	i = tx_ring->next_to_use;
2158
2159	buffer_info = &tx_ring->buffer_info[i];
2160	BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);
2161	buffer_info->length = len;
2162	/* set time_stamp *before* dma to help avoid a possible race */
2163	buffer_info->time_stamp = jiffies;
2164	buffer_info->mapped_as_page = false;
2165	buffer_info->dma = dma_map_single(&pdev->dev, skb->data, len,
2166	DMA_TO_DEVICE);
2167	if (dma_mapping_error(dev: &pdev->dev, dma_addr: buffer_info->dma))
2168	goto dma_error;
2169
2170	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2171	const skb_frag_t *frag;
2172
2173	count++;
2174	i++;
2175	if (i == tx_ring->count)
2176	i = 0;
2177
2178	frag = &skb_shinfo(skb)->frags[f];
2179	len = skb_frag_size(frag);
2180
2181	buffer_info = &tx_ring->buffer_info[i];
2182	BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);
2183	buffer_info->length = len;
2184	buffer_info->time_stamp = jiffies;
2185	buffer_info->mapped_as_page = true;
2186	buffer_info->dma = skb_frag_dma_map(dev: &pdev->dev, frag, offset: 0, size: len,
2187	dir: DMA_TO_DEVICE);
2188	if (dma_mapping_error(dev: &pdev->dev, dma_addr: buffer_info->dma))
2189	goto dma_error;
2190	}
2191
2192	tx_ring->buffer_info[i].skb = skb;
2193
2194	return ++count;
2195
2196	dma_error:
2197	dev_err(&pdev->dev, "TX DMA map failed\n");
2198
2199	/* clear timestamp and dma mappings for failed buffer_info mapping */
2200	buffer_info->dma = 0;
2201	buffer_info->time_stamp = 0;
2202	buffer_info->length = 0;
2203	buffer_info->mapped_as_page = false;
2204	if (count)
2205	count--;
2206
2207	/* clear timestamp and dma mappings for remaining portion of packet */
2208	while (count--) {
2209	if (i == 0)
2210	i += tx_ring->count;
2211	i--;
2212	buffer_info = &tx_ring->buffer_info[i];
2213	igbvf_put_txbuf(adapter, buffer_info);
2214	}
2215
2216	return 0;
2217	}
2218
2219	static inline void igbvf_tx_queue_adv(struct igbvf_adapter *adapter,
2220	struct igbvf_ring *tx_ring,
2221	int tx_flags, int count,
2222	unsigned int first, u32 paylen,
2223	u8 hdr_len)
2224	{
2225	union e1000_adv_tx_desc *tx_desc = NULL;
2226	struct igbvf_buffer *buffer_info;
2227	u32 olinfo_status = 0, cmd_type_len;
2228	unsigned int i;
2229
2230	cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
2231	E1000_ADVTXD_DCMD_DEXT);
2232
2233	if (tx_flags & IGBVF_TX_FLAGS_VLAN)
2234	cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
2235
2236	if (tx_flags & IGBVF_TX_FLAGS_TSO) {
2237	cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
2238
2239	/* insert tcp checksum */
2240	olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2241
2242	/* insert ip checksum */
2243	if (tx_flags & IGBVF_TX_FLAGS_IPV4)
2244	olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
2245
2246	} else if (tx_flags & IGBVF_TX_FLAGS_CSUM) {
2247	olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2248	}
2249
2250	olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
2251
2252	i = tx_ring->next_to_use;
2253	while (count--) {
2254	buffer_info = &tx_ring->buffer_info[i];
2255	tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
2256	tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
2257	tx_desc->read.cmd_type_len =
2258	cpu_to_le32(cmd_type_len | buffer_info->length);
2259	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2260	i++;
2261	if (i == tx_ring->count)
2262	i = 0;
2263	}
2264
2265	tx_desc->read.cmd_type_len |= cpu_to_le32(adapter->txd_cmd);
2266	/* Force memory writes to complete before letting h/w
2267	* know there are new descriptors to fetch. (Only
2268	* applicable for weak-ordered memory model archs,
2269	* such as IA-64).
2270	*/
2271	wmb();
2272
2273	tx_ring->buffer_info[first].next_to_watch = tx_desc;
2274	tx_ring->next_to_use = i;
2275	writel(val: i, addr: adapter->hw.hw_addr + tx_ring->tail);
2276	}
2277
2278	static netdev_tx_t igbvf_xmit_frame_ring_adv(struct sk_buff *skb,
2279	struct net_device *netdev,
2280	struct igbvf_ring *tx_ring)
2281	{
2282	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2283	unsigned int first, tx_flags = 0;
2284	u8 hdr_len = 0;
2285	int count = 0;
2286	int tso = 0;
2287	__be16 protocol = vlan_get_protocol(skb);
2288
2289	if (test_bit(__IGBVF_DOWN, &adapter->state)) {
2290	dev_kfree_skb_any(skb);
2291	return NETDEV_TX_OK;
2292	}
2293
2294	if (skb->len <= 0) {
2295	dev_kfree_skb_any(skb);
2296	return NETDEV_TX_OK;
2297	}
2298
2299	/* need: count + 4 desc gap to keep tail from touching
2300	* + 2 desc gap to keep tail from touching head,
2301	* + 1 desc for skb->data,
2302	* + 1 desc for context descriptor,
2303	* head, otherwise try next time
2304	*/
2305	if (igbvf_maybe_stop_tx(netdev, skb_shinfo(skb)->nr_frags + 4)) {
2306	/* this is a hard error */
2307	return NETDEV_TX_BUSY;
2308	}
2309
2310	if (skb_vlan_tag_present(skb)) {
2311	tx_flags |= IGBVF_TX_FLAGS_VLAN;
2312	tx_flags |= (skb_vlan_tag_get(skb) <<
2313	IGBVF_TX_FLAGS_VLAN_SHIFT);
2314	}
2315
2316	if (protocol == htons(ETH_P_IP))
2317	tx_flags |= IGBVF_TX_FLAGS_IPV4;
2318
2319	first = tx_ring->next_to_use;
2320
2321	tso = igbvf_tso(tx_ring, skb, tx_flags, hdr_len: &hdr_len);
2322	if (unlikely(tso < 0)) {
2323	dev_kfree_skb_any(skb);
2324	return NETDEV_TX_OK;
2325	}
2326
2327	if (tso)
2328	tx_flags |= IGBVF_TX_FLAGS_TSO;
2329	else if (igbvf_tx_csum(tx_ring, skb, tx_flags, protocol) &&
2330	(skb->ip_summed == CHECKSUM_PARTIAL))
2331	tx_flags |= IGBVF_TX_FLAGS_CSUM;
2332
2333	/* count reflects descriptors mapped, if 0 then mapping error
2334	* has occurred and we need to rewind the descriptor queue
2335	*/
2336	count = igbvf_tx_map_adv(adapter, tx_ring, skb);
2337
2338	if (count) {
2339	igbvf_tx_queue_adv(adapter, tx_ring, tx_flags, count,
2340	first, paylen: skb->len, hdr_len);
2341	/* Make sure there is space in the ring for the next send. */
2342	igbvf_maybe_stop_tx(netdev, MAX_SKB_FRAGS + 4);
2343	} else {
2344	dev_kfree_skb_any(skb);
2345	tx_ring->buffer_info[first].time_stamp = 0;
2346	tx_ring->next_to_use = first;
2347	}
2348
2349	return NETDEV_TX_OK;
2350	}
2351
2352	static netdev_tx_t igbvf_xmit_frame(struct sk_buff *skb,
2353	struct net_device *netdev)
2354	{
2355	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2356	struct igbvf_ring *tx_ring;
2357
2358	if (test_bit(__IGBVF_DOWN, &adapter->state)) {
2359	dev_kfree_skb_any(skb);
2360	return NETDEV_TX_OK;
2361	}
2362
2363	tx_ring = &adapter->tx_ring[0];
2364
2365	return igbvf_xmit_frame_ring_adv(skb, netdev, tx_ring);
2366	}
2367
2368	/**
2369	* igbvf_tx_timeout - Respond to a Tx Hang
2370	* @netdev: network interface device structure
2371	* @txqueue: queue timing out (unused)
2372	**/
2373	static void igbvf_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
2374	{
2375	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2376
2377	/* Do the reset outside of interrupt context */
2378	adapter->tx_timeout_count++;
2379	schedule_work(work: &adapter->reset_task);
2380	}
2381
2382	static void igbvf_reset_task(struct work_struct *work)
2383	{
2384	struct igbvf_adapter *adapter;
2385
2386	adapter = container_of(work, struct igbvf_adapter, reset_task);
2387
2388	igbvf_reinit_locked(adapter);
2389	}
2390
2391	/**
2392	* igbvf_change_mtu - Change the Maximum Transfer Unit
2393	* @netdev: network interface device structure
2394	* @new_mtu: new value for maximum frame size
2395	*
2396	* Returns 0 on success, negative on failure
2397	**/
2398	static int igbvf_change_mtu(struct net_device *netdev, int new_mtu)
2399	{
2400	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2401	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
2402
2403	while (test_and_set_bit(nr: __IGBVF_RESETTING, addr: &adapter->state))
2404	usleep_range(min: 1000, max: 2000);
2405	/* igbvf_down has a dependency on max_frame_size */
2406	adapter->max_frame_size = max_frame;
2407	if (netif_running(dev: netdev))
2408	igbvf_down(adapter);
2409
2410	/* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
2411	* means we reserve 2 more, this pushes us to allocate from the next
2412	* larger slab size.
2413	* i.e. RXBUFFER_2048 --> size-4096 slab
2414	* However with the new *_jumbo_rx* routines, jumbo receives will use
2415	* fragmented skbs
2416	*/
2417
2418	if (max_frame <= 1024)
2419	adapter->rx_buffer_len = 1024;
2420	else if (max_frame <= 2048)
2421	adapter->rx_buffer_len = 2048;
2422	else
2423	#if (PAGE_SIZE / 2) > 16384
2424	adapter->rx_buffer_len = 16384;
2425	#else
2426	adapter->rx_buffer_len = PAGE_SIZE / 2;
2427	#endif
2428
2429	/* adjust allocation if LPE protects us, and we aren't using SBP */
2430	if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
2431	(max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))
2432	adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN +
2433	ETH_FCS_LEN;
2434
2435	netdev_dbg(netdev, "changing MTU from %d to %d\n",
2436	netdev->mtu, new_mtu);
2437	netdev->mtu = new_mtu;
2438
2439	if (netif_running(dev: netdev))
2440	igbvf_up(adapter);
2441	else
2442	igbvf_reset(adapter);
2443
2444	clear_bit(nr: __IGBVF_RESETTING, addr: &adapter->state);
2445
2446	return 0;
2447	}
2448
2449	static int igbvf_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2450	{
2451	switch (cmd) {
2452	default:
2453	return -EOPNOTSUPP;
2454	}
2455	}
2456
2457	static int igbvf_suspend(struct device *dev_d)
2458	{
2459	struct net_device *netdev = dev_get_drvdata(dev: dev_d);
2460	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2461
2462	netif_device_detach(dev: netdev);
2463
2464	if (netif_running(dev: netdev)) {
2465	WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));
2466	igbvf_down(adapter);
2467	igbvf_free_irq(adapter);
2468	}
2469
2470	return 0;
2471	}
2472
2473	static int __maybe_unused igbvf_resume(struct device *dev_d)
2474	{
2475	struct pci_dev *pdev = to_pci_dev(dev_d);
2476	struct net_device *netdev = pci_get_drvdata(pdev);
2477	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2478	u32 err;
2479
2480	pci_set_master(dev: pdev);
2481
2482	if (netif_running(dev: netdev)) {
2483	err = igbvf_request_irq(adapter);
2484	if (err)
2485	return err;
2486	}
2487
2488	igbvf_reset(adapter);
2489
2490	if (netif_running(dev: netdev))
2491	igbvf_up(adapter);
2492
2493	netif_device_attach(dev: netdev);
2494
2495	return 0;
2496	}
2497
2498	static void igbvf_shutdown(struct pci_dev *pdev)
2499	{
2500	igbvf_suspend(dev_d: &pdev->dev);
2501	}
2502
2503	#ifdef CONFIG_NET_POLL_CONTROLLER
2504	/* Polling 'interrupt' - used by things like netconsole to send skbs
2505	* without having to re-enable interrupts. It's not called while
2506	* the interrupt routine is executing.
2507	*/
2508	static void igbvf_netpoll(struct net_device *netdev)
2509	{
2510	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2511
2512	disable_irq(irq: adapter->pdev->irq);
2513
2514	igbvf_clean_tx_irq(tx_ring: adapter->tx_ring);
2515
2516	enable_irq(irq: adapter->pdev->irq);
2517	}
2518	#endif
2519
2520	/**
2521	* igbvf_io_error_detected - called when PCI error is detected
2522	* @pdev: Pointer to PCI device
2523	* @state: The current pci connection state
2524	*
2525	* This function is called after a PCI bus error affecting
2526	* this device has been detected.
2527	*/
2528	static pci_ers_result_t igbvf_io_error_detected(struct pci_dev *pdev,
2529	pci_channel_state_t state)
2530	{
2531	struct net_device *netdev = pci_get_drvdata(pdev);
2532	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2533
2534	netif_device_detach(dev: netdev);
2535
2536	if (state == pci_channel_io_perm_failure)
2537	return PCI_ERS_RESULT_DISCONNECT;
2538
2539	if (netif_running(dev: netdev))
2540	igbvf_down(adapter);
2541	pci_disable_device(dev: pdev);
2542
2543	/* Request a slot reset. */
2544	return PCI_ERS_RESULT_NEED_RESET;
2545	}
2546
2547	/**
2548	* igbvf_io_slot_reset - called after the pci bus has been reset.
2549	* @pdev: Pointer to PCI device
2550	*
2551	* Restart the card from scratch, as if from a cold-boot. Implementation
2552	* resembles the first-half of the igbvf_resume routine.
2553	*/
2554	static pci_ers_result_t igbvf_io_slot_reset(struct pci_dev *pdev)
2555	{
2556	struct net_device *netdev = pci_get_drvdata(pdev);
2557	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2558
2559	if (pci_enable_device_mem(dev: pdev)) {
2560	dev_err(&pdev->dev,
2561	"Cannot re-enable PCI device after reset.\n");
2562	return PCI_ERS_RESULT_DISCONNECT;
2563	}
2564	pci_set_master(dev: pdev);
2565
2566	igbvf_reset(adapter);
2567
2568	return PCI_ERS_RESULT_RECOVERED;
2569	}
2570
2571	/**
2572	* igbvf_io_resume - called when traffic can start flowing again.
2573	* @pdev: Pointer to PCI device
2574	*
2575	* This callback is called when the error recovery driver tells us that
2576	* its OK to resume normal operation. Implementation resembles the
2577	* second-half of the igbvf_resume routine.
2578	*/
2579	static void igbvf_io_resume(struct pci_dev *pdev)
2580	{
2581	struct net_device *netdev = pci_get_drvdata(pdev);
2582	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2583
2584	if (netif_running(dev: netdev)) {
2585	if (igbvf_up(adapter)) {
2586	dev_err(&pdev->dev,
2587	"can't bring device back up after reset\n");
2588	return;
2589	}
2590	}
2591
2592	netif_device_attach(dev: netdev);
2593	}
2594
2595	/**
2596	* igbvf_io_prepare - prepare device driver for PCI reset
2597	* @pdev: PCI device information struct
2598	*/
2599	static void igbvf_io_prepare(struct pci_dev *pdev)
2600	{
2601	struct net_device *netdev = pci_get_drvdata(pdev);
2602	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2603
2604	while (test_and_set_bit(nr: __IGBVF_RESETTING, addr: &adapter->state))
2605	usleep_range(min: 1000, max: 2000);
2606	igbvf_down(adapter);
2607	}
2608
2609	/**
2610	* igbvf_io_reset_done - PCI reset done, device driver reset can begin
2611	* @pdev: PCI device information struct
2612	*/
2613	static void igbvf_io_reset_done(struct pci_dev *pdev)
2614	{
2615	struct net_device *netdev = pci_get_drvdata(pdev);
2616	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2617
2618	igbvf_up(adapter);
2619	clear_bit(nr: __IGBVF_RESETTING, addr: &adapter->state);
2620	}
2621
2622	static void igbvf_print_device_info(struct igbvf_adapter *adapter)
2623	{
2624	struct e1000_hw *hw = &adapter->hw;
2625	struct net_device *netdev = adapter->netdev;
2626	struct pci_dev *pdev = adapter->pdev;
2627
2628	if (hw->mac.type == e1000_vfadapt_i350)
2629	dev_info(&pdev->dev, "Intel(R) I350 Virtual Function\n");
2630	else
2631	dev_info(&pdev->dev, "Intel(R) 82576 Virtual Function\n");
2632	dev_info(&pdev->dev, "Address: %pM\n", netdev->dev_addr);
2633	}
2634
2635	static int igbvf_set_features(struct net_device *netdev,
2636	netdev_features_t features)
2637	{
2638	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2639
2640	if (features & NETIF_F_RXCSUM)
2641	adapter->flags &= ~IGBVF_FLAG_RX_CSUM_DISABLED;
2642	else
2643	adapter->flags |= IGBVF_FLAG_RX_CSUM_DISABLED;
2644
2645	return 0;
2646	}
2647
2648	#define IGBVF_MAX_MAC_HDR_LEN 127
2649	#define IGBVF_MAX_NETWORK_HDR_LEN 511
2650
2651	static netdev_features_t
2652	igbvf_features_check(struct sk_buff *skb, struct net_device *dev,
2653	netdev_features_t features)
2654	{
2655	unsigned int network_hdr_len, mac_hdr_len;
2656
2657	/* Make certain the headers can be described by a context descriptor */
2658	mac_hdr_len = skb_network_offset(skb);
2659	if (unlikely(mac_hdr_len > IGBVF_MAX_MAC_HDR_LEN))
2660	return features & ~(NETIF_F_HW_CSUM |
2661	NETIF_F_SCTP_CRC |
2662	NETIF_F_HW_VLAN_CTAG_TX |
2663	NETIF_F_TSO |
2664	NETIF_F_TSO6);
2665
2666	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2667	if (unlikely(network_hdr_len > IGBVF_MAX_NETWORK_HDR_LEN))
2668	return features & ~(NETIF_F_HW_CSUM |
2669	NETIF_F_SCTP_CRC |
2670	NETIF_F_TSO |
2671	NETIF_F_TSO6);
2672
2673	/* We can only support IPV4 TSO in tunnels if we can mangle the
2674	* inner IP ID field, so strip TSO if MANGLEID is not supported.
2675	*/
2676	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2677	features &= ~NETIF_F_TSO;
2678
2679	return features;
2680	}
2681
2682	static const struct net_device_ops igbvf_netdev_ops = {
2683	.ndo_open = igbvf_open,
2684	.ndo_stop = igbvf_close,
2685	.ndo_start_xmit = igbvf_xmit_frame,
2686	.ndo_set_rx_mode = igbvf_set_rx_mode,
2687	.ndo_set_mac_address = igbvf_set_mac,
2688	.ndo_change_mtu = igbvf_change_mtu,
2689	.ndo_eth_ioctl = igbvf_ioctl,
2690	.ndo_tx_timeout = igbvf_tx_timeout,
2691	.ndo_vlan_rx_add_vid = igbvf_vlan_rx_add_vid,
2692	.ndo_vlan_rx_kill_vid = igbvf_vlan_rx_kill_vid,
2693	#ifdef CONFIG_NET_POLL_CONTROLLER
2694	.ndo_poll_controller = igbvf_netpoll,
2695	#endif
2696	.ndo_set_features = igbvf_set_features,
2697	.ndo_features_check = igbvf_features_check,
2698	};
2699
2700	/**
2701	* igbvf_probe - Device Initialization Routine
2702	* @pdev: PCI device information struct
2703	* @ent: entry in igbvf_pci_tbl
2704	*
2705	* Returns 0 on success, negative on failure
2706	*
2707	* igbvf_probe initializes an adapter identified by a pci_dev structure.
2708	* The OS initialization, configuring of the adapter private structure,
2709	* and a hardware reset occur.
2710	**/
2711	static int igbvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2712	{
2713	struct net_device *netdev;
2714	struct igbvf_adapter *adapter;
2715	struct e1000_hw *hw;
2716	const struct igbvf_info *ei = igbvf_info_tbl[ent->driver_data];
2717	static int cards_found;
2718	int err;
2719
2720	err = pci_enable_device_mem(dev: pdev);
2721	if (err)
2722	return err;
2723
2724	err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
2725	if (err) {
2726	dev_err(&pdev->dev,
2727	"No usable DMA configuration, aborting\n");
2728	goto err_dma;
2729	}
2730
2731	err = pci_request_regions(pdev, igbvf_driver_name);
2732	if (err)
2733	goto err_pci_reg;
2734
2735	pci_set_master(dev: pdev);
2736
2737	err = -ENOMEM;
2738	netdev = alloc_etherdev(sizeof(struct igbvf_adapter));
2739	if (!netdev)
2740	goto err_alloc_etherdev;
2741
2742	SET_NETDEV_DEV(netdev, &pdev->dev);
2743
2744	pci_set_drvdata(pdev, data: netdev);
2745	adapter = netdev_priv(dev: netdev);
2746	hw = &adapter->hw;
2747	adapter->netdev = netdev;
2748	adapter->pdev = pdev;
2749	adapter->ei = ei;
2750	adapter->pba = ei->pba;
2751	adapter->flags = ei->flags;
2752	adapter->hw.back = adapter;
2753	adapter->hw.mac.type = ei->mac;
2754	adapter->msg_enable = netif_msg_init(debug_value: debug, DEFAULT_MSG_ENABLE);
2755
2756	/* PCI config space info */
2757
2758	hw->vendor_id = pdev->vendor;
2759	hw->device_id = pdev->device;
2760	hw->subsystem_vendor_id = pdev->subsystem_vendor;
2761	hw->subsystem_device_id = pdev->subsystem_device;
2762	hw->revision_id = pdev->revision;
2763
2764	err = -EIO;
2765	adapter->hw.hw_addr = ioremap(pci_resource_start(pdev, 0),
2766	pci_resource_len(pdev, 0));
2767
2768	if (!adapter->hw.hw_addr)
2769	goto err_ioremap;
2770
2771	if (ei->get_variants) {
2772	err = ei->get_variants(adapter);
2773	if (err)
2774	goto err_get_variants;
2775	}
2776
2777	/* setup adapter struct */
2778	err = igbvf_sw_init(adapter);
2779	if (err)
2780	goto err_sw_init;
2781
2782	/* construct the net_device struct */
2783	netdev->netdev_ops = &igbvf_netdev_ops;
2784
2785	igbvf_set_ethtool_ops(netdev);
2786	netdev->watchdog_timeo = 5 * HZ;
2787	strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
2788
2789	adapter->bd_number = cards_found++;
2790
2791	netdev->hw_features = NETIF_F_SG |
2792	NETIF_F_TSO |
2793	NETIF_F_TSO6 |
2794	NETIF_F_RXCSUM |
2795	NETIF_F_HW_CSUM |
2796	NETIF_F_SCTP_CRC;
2797
2798	#define IGBVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
2799	NETIF_F_GSO_GRE_CSUM | \
2800	NETIF_F_GSO_IPXIP4 | \
2801	NETIF_F_GSO_IPXIP6 | \
2802	NETIF_F_GSO_UDP_TUNNEL | \
2803	NETIF_F_GSO_UDP_TUNNEL_CSUM)
2804
2805	netdev->gso_partial_features = IGBVF_GSO_PARTIAL_FEATURES;
2806	netdev->hw_features |= NETIF_F_GSO_PARTIAL |
2807	IGBVF_GSO_PARTIAL_FEATURES;
2808
2809	netdev->features = netdev->hw_features | NETIF_F_HIGHDMA;
2810
2811	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
2812	netdev->mpls_features |= NETIF_F_HW_CSUM;
2813	netdev->hw_enc_features |= netdev->vlan_features;
2814
2815	/* set this bit last since it cannot be part of vlan_features */
2816	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
2817	NETIF_F_HW_VLAN_CTAG_RX |
2818	NETIF_F_HW_VLAN_CTAG_TX;
2819
2820	/* MTU range: 68 - 9216 */
2821	netdev->min_mtu = ETH_MIN_MTU;
2822	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
2823
2824	spin_lock_bh(lock: &hw->mbx_lock);
2825
2826	/*reset the controller to put the device in a known good state */
2827	err = hw->mac.ops.reset_hw(hw);
2828	if (err) {
2829	dev_info(&pdev->dev,
2830	"PF still in reset state. Is the PF interface up?\n");
2831	} else {
2832	err = hw->mac.ops.read_mac_addr(hw);
2833	if (err)
2834	dev_info(&pdev->dev, "Error reading MAC address.\n");
2835	else if (is_zero_ether_addr(addr: adapter->hw.mac.addr))
2836	dev_info(&pdev->dev,
2837	"MAC address not assigned by administrator.\n");
2838	eth_hw_addr_set(dev: netdev, addr: adapter->hw.mac.addr);
2839	}
2840
2841	spin_unlock_bh(lock: &hw->mbx_lock);
2842
2843	if (!is_valid_ether_addr(addr: netdev->dev_addr)) {
2844	dev_info(&pdev->dev, "Assigning random MAC address.\n");
2845	eth_hw_addr_random(dev: netdev);
2846	memcpy(adapter->hw.mac.addr, netdev->dev_addr,
2847	netdev->addr_len);
2848	}
2849
2850	timer_setup(&adapter->watchdog_timer, igbvf_watchdog, 0);
2851
2852	INIT_WORK(&adapter->reset_task, igbvf_reset_task);
2853	INIT_WORK(&adapter->watchdog_task, igbvf_watchdog_task);
2854
2855	/* ring size defaults */
2856	adapter->rx_ring->count = 1024;
2857	adapter->tx_ring->count = 1024;
2858
2859	/* reset the hardware with the new settings */
2860	igbvf_reset(adapter);
2861
2862	/* set hardware-specific flags */
2863	if (adapter->hw.mac.type == e1000_vfadapt_i350)
2864	adapter->flags |= IGBVF_FLAG_RX_LB_VLAN_BSWAP;
2865
2866	strcpy(p: netdev->name, q: "eth%d");
2867	err = register_netdev(dev: netdev);
2868	if (err)
2869	goto err_hw_init;
2870
2871	/* tell the stack to leave us alone until igbvf_open() is called */
2872	netif_carrier_off(dev: netdev);
2873	netif_stop_queue(dev: netdev);
2874
2875	igbvf_print_device_info(adapter);
2876
2877	igbvf_initialize_last_counter_stats(adapter);
2878
2879	return 0;
2880
2881	err_hw_init:
2882	netif_napi_del(napi: &adapter->rx_ring->napi);
2883	kfree(objp: adapter->tx_ring);
2884	kfree(objp: adapter->rx_ring);
2885	err_sw_init:
2886	igbvf_reset_interrupt_capability(adapter);
2887	err_get_variants:
2888	iounmap(addr: adapter->hw.hw_addr);
2889	err_ioremap:
2890	free_netdev(dev: netdev);
2891	err_alloc_etherdev:
2892	pci_release_regions(pdev);
2893	err_pci_reg:
2894	err_dma:
2895	pci_disable_device(dev: pdev);
2896	return err;
2897	}
2898
2899	/**
2900	* igbvf_remove - Device Removal Routine
2901	* @pdev: PCI device information struct
2902	*
2903	* igbvf_remove is called by the PCI subsystem to alert the driver
2904	* that it should release a PCI device. The could be caused by a
2905	* Hot-Plug event, or because the driver is going to be removed from
2906	* memory.
2907	**/
2908	static void igbvf_remove(struct pci_dev *pdev)
2909	{
2910	struct net_device *netdev = pci_get_drvdata(pdev);
2911	struct igbvf_adapter *adapter = netdev_priv(dev: netdev);
2912	struct e1000_hw *hw = &adapter->hw;
2913
2914	/* The watchdog timer may be rescheduled, so explicitly
2915	* disable it from being rescheduled.
2916	*/
2917	set_bit(nr: __IGBVF_DOWN, addr: &adapter->state);
2918	del_timer_sync(timer: &adapter->watchdog_timer);
2919
2920	cancel_work_sync(work: &adapter->reset_task);
2921	cancel_work_sync(work: &adapter->watchdog_task);
2922
2923	unregister_netdev(dev: netdev);
2924
2925	igbvf_reset_interrupt_capability(adapter);
2926
2927	/* it is important to delete the NAPI struct prior to freeing the
2928	* Rx ring so that you do not end up with null pointer refs
2929	*/
2930	netif_napi_del(napi: &adapter->rx_ring->napi);
2931	kfree(objp: adapter->tx_ring);
2932	kfree(objp: adapter->rx_ring);
2933
2934	iounmap(addr: hw->hw_addr);
2935	if (hw->flash_address)
2936	iounmap(addr: hw->flash_address);
2937	pci_release_regions(pdev);
2938
2939	free_netdev(dev: netdev);
2940
2941	pci_disable_device(dev: pdev);
2942	}
2943
2944	/* PCI Error Recovery (ERS) */
2945	static const struct pci_error_handlers igbvf_err_handler = {
2946	.error_detected = igbvf_io_error_detected,
2947	.slot_reset = igbvf_io_slot_reset,
2948	.resume = igbvf_io_resume,
2949	.reset_prepare = igbvf_io_prepare,
2950	.reset_done = igbvf_io_reset_done,
2951	};
2952
2953	static const struct pci_device_id igbvf_pci_tbl[] = {
2954	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_VF), board_vf },
2955	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_VF), board_i350_vf },
2956	{ } /* terminate list */
2957	};
2958	MODULE_DEVICE_TABLE(pci, igbvf_pci_tbl);
2959
2960	static SIMPLE_DEV_PM_OPS(igbvf_pm_ops, igbvf_suspend, igbvf_resume);
2961
2962	/* PCI Device API Driver */
2963	static struct pci_driver igbvf_driver = {
2964	.name = igbvf_driver_name,
2965	.id_table = igbvf_pci_tbl,
2966	.probe = igbvf_probe,
2967	.remove = igbvf_remove,
2968	.driver.pm = &igbvf_pm_ops,
2969	.shutdown = igbvf_shutdown,
2970	.err_handler = &igbvf_err_handler
2971	};
2972
2973	/**
2974	* igbvf_init_module - Driver Registration Routine
2975	*
2976	* igbvf_init_module is the first routine called when the driver is
2977	* loaded. All it does is register with the PCI subsystem.
2978	**/
2979	static int __init igbvf_init_module(void)
2980	{
2981	int ret;
2982
2983	pr_info("%s\n", igbvf_driver_string);
2984	pr_info("%s\n", igbvf_copyright);
2985
2986	ret = pci_register_driver(&igbvf_driver);
2987
2988	return ret;
2989	}
2990	module_init(igbvf_init_module);
2991
2992	/**
2993	* igbvf_exit_module - Driver Exit Cleanup Routine
2994	*
2995	* igbvf_exit_module is called just before the driver is removed
2996	* from memory.
2997	**/
2998	static void __exit igbvf_exit_module(void)
2999	{
3000	pci_unregister_driver(dev: &igbvf_driver);
3001	}
3002	module_exit(igbvf_exit_module);
3003
3004	MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
3005	MODULE_DESCRIPTION("Intel(R) Gigabit Virtual Function Network Driver");
3006	MODULE_LICENSE("GPL v2");
3007
3008	/* netdev.c */
3009