1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Tehuti Networks(R) Network Driver
4	* ethtool interface implementation
5	* Copyright (C) 2007 Tehuti Networks Ltd. All rights reserved
6	*/
7
8	/*
9	* RX HW/SW interaction overview
10	* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
11	* There are 2 types of RX communication channels between driver and NIC.
12	* 1) RX Free Fifo - RXF - holds descriptors of empty buffers to accept incoming
13	* traffic. This Fifo is filled by SW and is readen by HW. Each descriptor holds
14	* info about buffer's location, size and ID. An ID field is used to identify a
15	* buffer when it's returned with data via RXD Fifo (see below)
16	* 2) RX Data Fifo - RXD - holds descriptors of full buffers. This Fifo is
17	* filled by HW and is readen by SW. Each descriptor holds status and ID.
18	* HW pops descriptor from RXF Fifo, stores ID, fills buffer with incoming data,
19	* via dma moves it into host memory, builds new RXD descriptor with same ID,
20	* pushes it into RXD Fifo and raises interrupt to indicate new RX data.
21	*
22	* Current NIC configuration (registers + firmware) makes NIC use 2 RXF Fifos.
23	* One holds 1.5K packets and another - 26K packets. Depending on incoming
24	* packet size, HW desides on a RXF Fifo to pop buffer from. When packet is
25	* filled with data, HW builds new RXD descriptor for it and push it into single
26	* RXD Fifo.
27	*
28	* RX SW Data Structures
29	* ~~~~~~~~~~~~~~~~~~~~~
30	* skb db - used to keep track of all skbs owned by SW and their dma addresses.
31	* For RX case, ownership lasts from allocating new empty skb for RXF until
32	* accepting full skb from RXD and passing it to OS. Each RXF Fifo has its own
33	* skb db. Implemented as array with bitmask.
34	* fifo - keeps info about fifo's size and location, relevant HW registers,
35	* usage and skb db. Each RXD and RXF Fifo has its own fifo structure.
36	* Implemented as simple struct.
37	*
38	* RX SW Execution Flow
39	* ~~~~~~~~~~~~~~~~~~~~
40	* Upon initialization (ifconfig up) driver creates RX fifos and initializes
41	* relevant registers. At the end of init phase, driver enables interrupts.
42	* NIC sees that there is no RXF buffers and raises
43	* RD_INTR interrupt, isr fills skbs and Rx begins.
44	* Driver has two receive operation modes:
45	* NAPI - interrupt-driven mixed with polling
46	* interrupt-driven only
47	*
48	* Interrupt-driven only flow is following. When buffer is ready, HW raises
49	* interrupt and isr is called. isr collects all available packets
50	* (bdx_rx_receive), refills skbs (bdx_rx_alloc_skbs) and exit.
51
52	* Rx buffer allocation note
53	* ~~~~~~~~~~~~~~~~~~~~~~~~~
54	* Driver cares to feed such amount of RxF descriptors that respective amount of
55	* RxD descriptors can not fill entire RxD fifo. The main reason is lack of
56	* overflow check in Bordeaux for RxD fifo free/used size.
57	* FIXME: this is NOT fully implemented, more work should be done
58	*
59	*/
60
61	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
62
63	#include "tehuti.h"
64
65	static const struct pci_device_id bdx_pci_tbl[] = {
66	{ PCI_VDEVICE(TEHUTI, 0x3009), },
67	{ PCI_VDEVICE(TEHUTI, 0x3010), },
68	{ PCI_VDEVICE(TEHUTI, 0x3014), },
69	{ 0 }
70	};
71
72	MODULE_DEVICE_TABLE(pci, bdx_pci_tbl);
73
74	/* Definitions needed by ISR or NAPI functions */
75	static void bdx_rx_alloc_skbs(struct bdx_priv *priv, struct rxf_fifo *f);
76	static void bdx_tx_cleanup(struct bdx_priv *priv);
77	static int bdx_rx_receive(struct bdx_priv *priv, struct rxd_fifo *f, int budget);
78
79	/* Definitions needed by FW loading */
80	static void bdx_tx_push_desc_safe(struct bdx_priv *priv, void *data, int size);
81
82	/* Definitions needed by hw_start */
83	static int bdx_tx_init(struct bdx_priv *priv);
84	static int bdx_rx_init(struct bdx_priv *priv);
85
86	/* Definitions needed by bdx_close */
87	static void bdx_rx_free(struct bdx_priv *priv);
88	static void bdx_tx_free(struct bdx_priv *priv);
89
90	/* Definitions needed by bdx_probe */
91	static void bdx_set_ethtool_ops(struct net_device *netdev);
92
93	/*************************************************************************
94	* Print Info *
95	*************************************************************************/
96
97	static void print_hw_id(struct pci_dev *pdev)
98	{
99	struct pci_nic *nic = pci_get_drvdata(pdev);
100	u16 pci_link_status = 0;
101	u16 pci_ctrl = 0;
102
103	pci_read_config_word(dev: pdev, PCI_LINK_STATUS_REG, val: &pci_link_status);
104	pci_read_config_word(dev: pdev, PCI_DEV_CTRL_REG, val: &pci_ctrl);
105
106	pr_info("%s%s\n", BDX_NIC_NAME,
107	nic->port_num == 1 ? "" : ", 2-Port");
108	pr_info("srom 0x%x fpga %d build %u lane# %d max_pl 0x%x mrrs 0x%x\n",
109	readl(nic->regs + SROM_VER), readl(nic->regs + FPGA_VER) & 0xFFF,
110	readl(nic->regs + FPGA_SEED),
111	GET_LINK_STATUS_LANES(pci_link_status),
112	GET_DEV_CTRL_MAXPL(pci_ctrl), GET_DEV_CTRL_MRRS(pci_ctrl));
113	}
114
115	static void print_fw_id(struct pci_nic *nic)
116	{
117	pr_info("fw 0x%x\n", readl(nic->regs + FW_VER));
118	}
119
120	static void print_eth_id(struct net_device *ndev)
121	{
122	netdev_info(dev: ndev, format: "%s, Port %c\n",
123	BDX_NIC_NAME, (ndev->if_port == 0) ? 'A' : 'B');
124
125	}
126
127	/*************************************************************************
128	* Code *
129	*************************************************************************/
130
131	#define bdx_enable_interrupts(priv) \
132	do { WRITE_REG(priv, regIMR, IR_RUN); } while (0)
133	#define bdx_disable_interrupts(priv) \
134	do { WRITE_REG(priv, regIMR, 0); } while (0)
135
136	/**
137	* bdx_fifo_init - create TX/RX descriptor fifo for host-NIC communication.
138	* @priv: NIC private structure
139	* @f: fifo to initialize
140	* @fsz_type: fifo size type: 0-4KB, 1-8KB, 2-16KB, 3-32KB
141	* @reg_CFG0: offsets of registers relative to base address
142	* @reg_CFG1: offsets of registers relative to base address
143	* @reg_RPTR: offsets of registers relative to base address
144	* @reg_WPTR: offsets of registers relative to base address
145	*
146	* 1K extra space is allocated at the end of the fifo to simplify
147	* processing of descriptors that wraps around fifo's end
148	*
149	* Returns 0 on success, negative value on failure
150	*
151	*/
152	static int
153	bdx_fifo_init(struct bdx_priv *priv, struct fifo *f, int fsz_type,
154	u16 reg_CFG0, u16 reg_CFG1, u16 reg_RPTR, u16 reg_WPTR)
155	{
156	u16 memsz = FIFO_SIZE * (1 << fsz_type);
157
158	memset(f, 0, sizeof(struct fifo));
159	/* dma_alloc_coherent gives us 4k-aligned memory */
160	f->va = dma_alloc_coherent(dev: &priv->pdev->dev, size: memsz + FIFO_EXTRA_SPACE,
161	dma_handle: &f->da, GFP_ATOMIC);
162	if (!f->va) {
163	pr_err("dma_alloc_coherent failed\n");
164	RET(-ENOMEM);
165	}
166	f->reg_CFG0 = reg_CFG0;
167	f->reg_CFG1 = reg_CFG1;
168	f->reg_RPTR = reg_RPTR;
169	f->reg_WPTR = reg_WPTR;
170	f->rptr = 0;
171	f->wptr = 0;
172	f->memsz = memsz;
173	f->size_mask = memsz - 1;
174	WRITE_REG(priv, reg_CFG0, (u32) ((f->da & TX_RX_CFG0_BASE) | fsz_type));
175	WRITE_REG(priv, reg_CFG1, H32_64(f->da));
176
177	RET(0);
178	}
179
180	/**
181	* bdx_fifo_free - free all resources used by fifo
182	* @priv: NIC private structure
183	* @f: fifo to release
184	*/
185	static void bdx_fifo_free(struct bdx_priv *priv, struct fifo *f)
186	{
187	ENTER;
188	if (f->va) {
189	dma_free_coherent(dev: &priv->pdev->dev,
190	size: f->memsz + FIFO_EXTRA_SPACE, cpu_addr: f->va, dma_handle: f->da);
191	f->va = NULL;
192	}
193	RET();
194	}
195
196	/**
197	* bdx_link_changed - notifies OS about hw link state.
198	* @priv: hw adapter structure
199	*/
200	static void bdx_link_changed(struct bdx_priv *priv)
201	{
202	u32 link = READ_REG(priv, regMAC_LNK_STAT) & MAC_LINK_STAT;
203
204	if (!link) {
205	if (netif_carrier_ok(dev: priv->ndev)) {
206	netif_stop_queue(dev: priv->ndev);
207	netif_carrier_off(dev: priv->ndev);
208	netdev_err(dev: priv->ndev, format: "Link Down\n");
209	}
210	} else {
211	if (!netif_carrier_ok(dev: priv->ndev)) {
212	netif_wake_queue(dev: priv->ndev);
213	netif_carrier_on(dev: priv->ndev);
214	netdev_err(dev: priv->ndev, format: "Link Up\n");
215	}
216	}
217	}
218
219	static void bdx_isr_extra(struct bdx_priv *priv, u32 isr)
220	{
221	if (isr & IR_RX_FREE_0) {
222	bdx_rx_alloc_skbs(priv, f: &priv->rxf_fifo0);
223	DBG("RX_FREE_0\n");
224	}
225
226	if (isr & IR_LNKCHG0)
227	bdx_link_changed(priv);
228
229	if (isr & IR_PCIE_LINK)
230	netdev_err(dev: priv->ndev, format: "PCI-E Link Fault\n");
231
232	if (isr & IR_PCIE_TOUT)
233	netdev_err(dev: priv->ndev, format: "PCI-E Time Out\n");
234
235	}
236
237	/**
238	* bdx_isr_napi - Interrupt Service Routine for Bordeaux NIC
239	* @irq: interrupt number
240	* @dev: network device
241	*
242	* Return IRQ_NONE if it was not our interrupt, IRQ_HANDLED - otherwise
243	*
244	* It reads ISR register to know interrupt reasons, and proceed them one by one.
245	* Reasons of interest are:
246	* RX_DESC - new packet has arrived and RXD fifo holds its descriptor
247	* RX_FREE - number of free Rx buffers in RXF fifo gets low
248	* TX_FREE - packet was transmited and RXF fifo holds its descriptor
249	*/
250
251	static irqreturn_t bdx_isr_napi(int irq, void *dev)
252	{
253	struct net_device *ndev = dev;
254	struct bdx_priv *priv = netdev_priv(dev: ndev);
255	u32 isr;
256
257	ENTER;
258	isr = (READ_REG(priv, regISR) & IR_RUN);
259	if (unlikely(!isr)) {
260	bdx_enable_interrupts(priv);
261	return IRQ_NONE; /* Not our interrupt */
262	}
263
264	if (isr & IR_EXTRA)
265	bdx_isr_extra(priv, isr);
266
267	if (isr & (IR_RX_DESC_0 | IR_TX_FREE_0)) {
268	if (likely(napi_schedule_prep(&priv->napi))) {
269	__napi_schedule(n: &priv->napi);
270	RET(IRQ_HANDLED);
271	} else {
272	/* NOTE: we get here if intr has slipped into window
273	* between these lines in bdx_poll:
274	* bdx_enable_interrupts(priv);
275	* return 0;
276	* currently intrs are disabled (since we read ISR),
277	* and we have failed to register next poll.
278	* so we read the regs to trigger chip
279	* and allow further interupts. */
280	READ_REG(priv, regTXF_WPTR_0);
281	READ_REG(priv, regRXD_WPTR_0);
282	}
283	}
284
285	bdx_enable_interrupts(priv);
286	RET(IRQ_HANDLED);
287	}
288
289	static int bdx_poll(struct napi_struct *napi, int budget)
290	{
291	struct bdx_priv *priv = container_of(napi, struct bdx_priv, napi);
292	int work_done;
293
294	ENTER;
295	bdx_tx_cleanup(priv);
296	work_done = bdx_rx_receive(priv, f: &priv->rxd_fifo0, budget);
297	if ((work_done < budget) ||
298	(priv->napi_stop++ >= 30)) {
299	DBG("rx poll is done. backing to isr-driven\n");
300
301	/* from time to time we exit to let NAPI layer release
302	* device lock and allow waiting tasks (eg rmmod) to advance) */
303	priv->napi_stop = 0;
304
305	napi_complete_done(n: napi, work_done);
306	bdx_enable_interrupts(priv);
307	}
308	return work_done;
309	}
310
311	/**
312	* bdx_fw_load - loads firmware to NIC
313	* @priv: NIC private structure
314	*
315	* Firmware is loaded via TXD fifo, so it must be initialized first.
316	* Firware must be loaded once per NIC not per PCI device provided by NIC (NIC
317	* can have few of them). So all drivers use semaphore register to choose one
318	* that will actually load FW to NIC.
319	*/
320
321	static int bdx_fw_load(struct bdx_priv *priv)
322	{
323	const struct firmware *fw = NULL;
324	int master, i;
325	int rc;
326
327	ENTER;
328	master = READ_REG(priv, regINIT_SEMAPHORE);
329	if (!READ_REG(priv, regINIT_STATUS) && master) {
330	rc = request_firmware(fw: &fw, name: "tehuti/bdx.bin", device: &priv->pdev->dev);
331	if (rc)
332	goto out;
333	bdx_tx_push_desc_safe(priv, data: (char *)fw->data, size: fw->size);
334	mdelay(100);
335	}
336	for (i = 0; i < 200; i++) {
337	if (READ_REG(priv, regINIT_STATUS)) {
338	rc = 0;
339	goto out;
340	}
341	mdelay(2);
342	}
343	rc = -EIO;
344	out:
345	if (master)
346	WRITE_REG(priv, regINIT_SEMAPHORE, 1);
347
348	release_firmware(fw);
349
350	if (rc) {
351	netdev_err(dev: priv->ndev, format: "firmware loading failed\n");
352	if (rc == -EIO)
353	DBG("VPC = 0x%x VIC = 0x%x INIT_STATUS = 0x%x i=%d\n",
354	READ_REG(priv, regVPC),
355	READ_REG(priv, regVIC),
356	READ_REG(priv, regINIT_STATUS), i);
357	RET(rc);
358	} else {
359	DBG("%s: firmware loading success\n", priv->ndev->name);
360	RET(0);
361	}
362	}
363
364	static void bdx_restore_mac(struct net_device *ndev, struct bdx_priv *priv)
365	{
366	u32 val;
367
368	ENTER;
369	DBG("mac0=%x mac1=%x mac2=%x\n",
370	READ_REG(priv, regUNC_MAC0_A),
371	READ_REG(priv, regUNC_MAC1_A), READ_REG(priv, regUNC_MAC2_A));
372
373	val = (ndev->dev_addr[0] << 8) | (ndev->dev_addr[1]);
374	WRITE_REG(priv, regUNC_MAC2_A, val);
375	val = (ndev->dev_addr[2] << 8) | (ndev->dev_addr[3]);
376	WRITE_REG(priv, regUNC_MAC1_A, val);
377	val = (ndev->dev_addr[4] << 8) | (ndev->dev_addr[5]);
378	WRITE_REG(priv, regUNC_MAC0_A, val);
379
380	DBG("mac0=%x mac1=%x mac2=%x\n",
381	READ_REG(priv, regUNC_MAC0_A),
382	READ_REG(priv, regUNC_MAC1_A), READ_REG(priv, regUNC_MAC2_A));
383	RET();
384	}
385
386	/**
387	* bdx_hw_start - inits registers and starts HW's Rx and Tx engines
388	* @priv: NIC private structure
389	*/
390	static int bdx_hw_start(struct bdx_priv *priv)
391	{
392	int rc = -EIO;
393	struct net_device *ndev = priv->ndev;
394
395	ENTER;
396	bdx_link_changed(priv);
397
398	/* 10G overall max length (vlan, eth&ip header, ip payload, crc) */
399	WRITE_REG(priv, regFRM_LENGTH, 0X3FE0);
400	WRITE_REG(priv, regPAUSE_QUANT, 0x96);
401	WRITE_REG(priv, regRX_FIFO_SECTION, 0x800010);
402	WRITE_REG(priv, regTX_FIFO_SECTION, 0xE00010);
403	WRITE_REG(priv, regRX_FULLNESS, 0);
404	WRITE_REG(priv, regTX_FULLNESS, 0);
405	WRITE_REG(priv, regCTRLST,
406	regCTRLST_BASE | regCTRLST_RX_ENA | regCTRLST_TX_ENA);
407
408	WRITE_REG(priv, regVGLB, 0);
409	WRITE_REG(priv, regMAX_FRAME_A,
410	priv->rxf_fifo0.m.pktsz & MAX_FRAME_AB_VAL);
411
412	DBG("RDINTCM=%08x\n", priv->rdintcm); /*NOTE: test script uses this */
413	WRITE_REG(priv, regRDINTCM0, priv->rdintcm);
414	WRITE_REG(priv, regRDINTCM2, 0); /*cpu_to_le32(rcm.val)); */
415
416	DBG("TDINTCM=%08x\n", priv->tdintcm); /*NOTE: test script uses this */
417	WRITE_REG(priv, regTDINTCM0, priv->tdintcm); /* old val = 0x300064 */
418
419	/* Enable timer interrupt once in 2 secs. */
420	/*WRITE_REG(priv, regGTMR0, ((GTMR_SEC * 2) & GTMR_DATA)); */
421	bdx_restore_mac(ndev: priv->ndev, priv);
422
423	WRITE_REG(priv, regGMAC_RXF_A, GMAC_RX_FILTER_OSEN |
424	GMAC_RX_FILTER_AM | GMAC_RX_FILTER_AB);
425
426	#define BDX_IRQ_TYPE ((priv->nic->irq_type == IRQ_MSI) ? 0 : IRQF_SHARED)
427
428	rc = request_irq(irq: priv->pdev->irq, handler: bdx_isr_napi, BDX_IRQ_TYPE,
429	name: ndev->name, dev: ndev);
430	if (rc)
431	goto err_irq;
432	bdx_enable_interrupts(priv);
433
434	RET(0);
435
436	err_irq:
437	RET(rc);
438	}
439
440	static void bdx_hw_stop(struct bdx_priv *priv)
441	{
442	ENTER;
443	bdx_disable_interrupts(priv);
444	free_irq(priv->pdev->irq, priv->ndev);
445
446	netif_carrier_off(dev: priv->ndev);
447	netif_stop_queue(dev: priv->ndev);
448
449	RET();
450	}
451
452	static int bdx_hw_reset_direct(void __iomem *regs)
453	{
454	u32 val, i;
455	ENTER;
456
457	/* reset sequences: read, write 1, read, write 0 */
458	val = readl(addr: regs + regCLKPLL);
459	writel(val: (val | CLKPLL_SFTRST) + 0x8, addr: regs + regCLKPLL);
460	udelay(50);
461	val = readl(addr: regs + regCLKPLL);
462	writel(val: val & ~CLKPLL_SFTRST, addr: regs + regCLKPLL);
463
464	/* check that the PLLs are locked and reset ended */
465	for (i = 0; i < 70; i++, mdelay(10))
466	if ((readl(addr: regs + regCLKPLL) & CLKPLL_LKD) == CLKPLL_LKD) {
467	/* do any PCI-E read transaction */
468	readl(addr: regs + regRXD_CFG0_0);
469	return 0;
470	}
471	pr_err("HW reset failed\n");
472	return 1; /* failure */
473	}
474
475	static int bdx_hw_reset(struct bdx_priv *priv)
476	{
477	u32 val, i;
478	ENTER;
479
480	if (priv->port == 0) {
481	/* reset sequences: read, write 1, read, write 0 */
482	val = READ_REG(priv, regCLKPLL);
483	WRITE_REG(priv, regCLKPLL, (val | CLKPLL_SFTRST) + 0x8);
484	udelay(50);
485	val = READ_REG(priv, regCLKPLL);
486	WRITE_REG(priv, regCLKPLL, val & ~CLKPLL_SFTRST);
487	}
488	/* check that the PLLs are locked and reset ended */
489	for (i = 0; i < 70; i++, mdelay(10))
490	if ((READ_REG(priv, regCLKPLL) & CLKPLL_LKD) == CLKPLL_LKD) {
491	/* do any PCI-E read transaction */
492	READ_REG(priv, regRXD_CFG0_0);
493	return 0;
494	}
495	pr_err("HW reset failed\n");
496	return 1; /* failure */
497	}
498
499	static int bdx_sw_reset(struct bdx_priv *priv)
500	{
501	int i;
502
503	ENTER;
504	/* 1. load MAC (obsolete) */
505	/* 2. disable Rx (and Tx) */
506	WRITE_REG(priv, regGMAC_RXF_A, 0);
507	mdelay(100);
508	/* 3. disable port */
509	WRITE_REG(priv, regDIS_PORT, 1);
510	/* 4. disable queue */
511	WRITE_REG(priv, regDIS_QU, 1);
512	/* 5. wait until hw is disabled */
513	for (i = 0; i < 50; i++) {
514	if (READ_REG(priv, regRST_PORT) & 1)
515	break;
516	mdelay(10);
517	}
518	if (i == 50)
519	netdev_err(dev: priv->ndev, format: "SW reset timeout. continuing anyway\n");
520
521	/* 6. disable intrs */
522	WRITE_REG(priv, regRDINTCM0, 0);
523	WRITE_REG(priv, regTDINTCM0, 0);
524	WRITE_REG(priv, regIMR, 0);
525	READ_REG(priv, regISR);
526
527	/* 7. reset queue */
528	WRITE_REG(priv, regRST_QU, 1);
529	/* 8. reset port */
530	WRITE_REG(priv, regRST_PORT, 1);
531	/* 9. zero all read and write pointers */
532	for (i = regTXD_WPTR_0; i <= regTXF_RPTR_3; i += 0x10)
533	DBG("%x = %x\n", i, READ_REG(priv, i) & TXF_WPTR_WR_PTR);
534	for (i = regTXD_WPTR_0; i <= regTXF_RPTR_3; i += 0x10)
535	WRITE_REG(priv, i, 0);
536	/* 10. unseet port disable */
537	WRITE_REG(priv, regDIS_PORT, 0);
538	/* 11. unset queue disable */
539	WRITE_REG(priv, regDIS_QU, 0);
540	/* 12. unset queue reset */
541	WRITE_REG(priv, regRST_QU, 0);
542	/* 13. unset port reset */
543	WRITE_REG(priv, regRST_PORT, 0);
544	/* 14. enable Rx */
545	/* skiped. will be done later */
546	/* 15. save MAC (obsolete) */
547	for (i = regTXD_WPTR_0; i <= regTXF_RPTR_3; i += 0x10)
548	DBG("%x = %x\n", i, READ_REG(priv, i) & TXF_WPTR_WR_PTR);
549
550	RET(0);
551	}
552
553	/* bdx_reset - performs right type of reset depending on hw type */
554	static int bdx_reset(struct bdx_priv *priv)
555	{
556	ENTER;
557	RET((priv->pdev->device == 0x3009)
558	? bdx_hw_reset(priv)
559	: bdx_sw_reset(priv));
560	}
561
562	/**
563	* bdx_close - Disables a network interface
564	* @ndev: network interface device structure
565	*
566	* Returns 0, this is not allowed to fail
567	*
568	* The close entry point is called when an interface is de-activated
569	* by the OS. The hardware is still under the drivers control, but
570	* needs to be disabled. A global MAC reset is issued to stop the
571	* hardware, and all transmit and receive resources are freed.
572	**/
573	static int bdx_close(struct net_device *ndev)
574	{
575	struct bdx_priv *priv = NULL;
576
577	ENTER;
578	priv = netdev_priv(dev: ndev);
579
580	napi_disable(n: &priv->napi);
581
582	bdx_reset(priv);
583	bdx_hw_stop(priv);
584	bdx_rx_free(priv);
585	bdx_tx_free(priv);
586	RET(0);
587	}
588
589	/**
590	* bdx_open - Called when a network interface is made active
591	* @ndev: network interface device structure
592	*
593	* Returns 0 on success, negative value on failure
594	*
595	* The open entry point is called when a network interface is made
596	* active by the system (IFF_UP). At this point all resources needed
597	* for transmit and receive operations are allocated, the interrupt
598	* handler is registered with the OS, the watchdog timer is started,
599	* and the stack is notified that the interface is ready.
600	**/
601	static int bdx_open(struct net_device *ndev)
602	{
603	struct bdx_priv *priv;
604	int rc;
605
606	ENTER;
607	priv = netdev_priv(dev: ndev);
608	bdx_reset(priv);
609	if (netif_running(dev: ndev))
610	netif_stop_queue(dev: priv->ndev);
611
612	if ((rc = bdx_tx_init(priv)) ||
613	(rc = bdx_rx_init(priv)) ||
614	(rc = bdx_fw_load(priv)))
615	goto err;
616
617	bdx_rx_alloc_skbs(priv, f: &priv->rxf_fifo0);
618
619	rc = bdx_hw_start(priv);
620	if (rc)
621	goto err;
622
623	napi_enable(n: &priv->napi);
624
625	print_fw_id(nic: priv->nic);
626
627	RET(0);
628
629	err:
630	bdx_close(ndev);
631	RET(rc);
632	}
633
634	static int bdx_range_check(struct bdx_priv *priv, u32 offset)
635	{
636	return (offset > (u32) (BDX_REGS_SIZE / priv->nic->port_num)) ?
637	-EINVAL : 0;
638	}
639
640	static int bdx_siocdevprivate(struct net_device *ndev, struct ifreq *ifr,
641	void __user *udata, int cmd)
642	{
643	struct bdx_priv *priv = netdev_priv(dev: ndev);
644	u32 data[3];
645	int error;
646
647	ENTER;
648
649	DBG("jiffies=%ld cmd=%d\n", jiffies, cmd);
650	if (cmd != SIOCDEVPRIVATE) {
651	error = copy_from_user(to: data, from: udata, n: sizeof(data));
652	if (error) {
653	pr_err("can't copy from user\n");
654	RET(-EFAULT);
655	}
656	DBG("%d 0x%x 0x%x\n", data[0], data[1], data[2]);
657	} else {
658	return -EOPNOTSUPP;
659	}
660
661	if (!capable(CAP_SYS_RAWIO))
662	return -EPERM;
663
664	switch (data[0]) {
665
666	case BDX_OP_READ:
667	error = bdx_range_check(priv, offset: data[1]);
668	if (error < 0)
669	return error;
670	data[2] = READ_REG(priv, data[1]);
671	DBG("read_reg(0x%x)=0x%x (dec %d)\n", data[1], data[2],
672	data[2]);
673	error = copy_to_user(to: udata, from: data, n: sizeof(data));
674	if (error)
675	RET(-EFAULT);
676	break;
677
678	case BDX_OP_WRITE:
679	error = bdx_range_check(priv, offset: data[1]);
680	if (error < 0)
681	return error;
682	WRITE_REG(priv, data[1], data[2]);
683	DBG("write_reg(0x%x, 0x%x)\n", data[1], data[2]);
684	break;
685
686	default:
687	RET(-EOPNOTSUPP);
688	}
689	return 0;
690	}
691
692	/**
693	* __bdx_vlan_rx_vid - private helper for adding/killing VLAN vid
694	* @ndev: network device
695	* @vid: VLAN vid
696	* @enable: enable or disable vlan
697	*
698	* Passes VLAN filter table to hardware
699	*/
700	static void __bdx_vlan_rx_vid(struct net_device *ndev, uint16_t vid, int enable)
701	{
702	struct bdx_priv *priv = netdev_priv(dev: ndev);
703	u32 reg, bit, val;
704
705	ENTER;
706	DBG2("vid=%d value=%d\n", (int)vid, enable);
707	if (unlikely(vid >= 4096)) {
708	pr_err("invalid VID: %u (> 4096)\n", vid);
709	RET();
710	}
711	reg = regVLAN_0 + (vid / 32) * 4;
712	bit = 1 << vid % 32;
713	val = READ_REG(priv, reg);
714	DBG2("reg=%x, val=%x, bit=%d\n", reg, val, bit);
715	if (enable)
716	val |= bit;
717	else
718	val &= ~bit;
719	DBG2("new val %x\n", val);
720	WRITE_REG(priv, reg, val);
721	RET();
722	}
723
724	/**
725	* bdx_vlan_rx_add_vid - kernel hook for adding VLAN vid to hw filtering table
726	* @ndev: network device
727	* @proto: unused
728	* @vid: VLAN vid to add
729	*/
730	static int bdx_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid)
731	{
732	__bdx_vlan_rx_vid(ndev, vid, enable: 1);
733	return 0;
734	}
735
736	/**
737	* bdx_vlan_rx_kill_vid - kernel hook for killing VLAN vid in hw filtering table
738	* @ndev: network device
739	* @proto: unused
740	* @vid: VLAN vid to kill
741	*/
742	static int bdx_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
743	{
744	__bdx_vlan_rx_vid(ndev, vid, enable: 0);
745	return 0;
746	}
747
748	/**
749	* bdx_change_mtu - Change the Maximum Transfer Unit
750	* @ndev: network interface device structure
751	* @new_mtu: new value for maximum frame size
752	*
753	* Returns 0 on success, negative on failure
754	*/
755	static int bdx_change_mtu(struct net_device *ndev, int new_mtu)
756	{
757	ENTER;
758
759	ndev->mtu = new_mtu;
760	if (netif_running(dev: ndev)) {
761	bdx_close(ndev);
762	bdx_open(ndev);
763	}
764	RET(0);
765	}
766
767	static void bdx_setmulti(struct net_device *ndev)
768	{
769	struct bdx_priv *priv = netdev_priv(dev: ndev);
770
771	u32 rxf_val =
772	GMAC_RX_FILTER_AM | GMAC_RX_FILTER_AB | GMAC_RX_FILTER_OSEN;
773	int i;
774
775	ENTER;
776	/* IMF - imperfect (hash) rx multicat filter */
777	/* PMF - perfect rx multicat filter */
778
779	/* FIXME: RXE(OFF) */
780	if (ndev->flags & IFF_PROMISC) {
781	rxf_val |= GMAC_RX_FILTER_PRM;
782	} else if (ndev->flags & IFF_ALLMULTI) {
783	/* set IMF to accept all multicast frmaes */
784	for (i = 0; i < MAC_MCST_HASH_NUM; i++)
785	WRITE_REG(priv, regRX_MCST_HASH0 + i * 4, ~0);
786	} else if (!netdev_mc_empty(ndev)) {
787	u8 hash;
788	struct netdev_hw_addr *ha;
789	u32 reg, val;
790
791	/* set IMF to deny all multicast frames */
792	for (i = 0; i < MAC_MCST_HASH_NUM; i++)
793	WRITE_REG(priv, regRX_MCST_HASH0 + i * 4, 0);
794	/* set PMF to deny all multicast frames */
795	for (i = 0; i < MAC_MCST_NUM; i++) {
796	WRITE_REG(priv, regRX_MAC_MCST0 + i * 8, 0);
797	WRITE_REG(priv, regRX_MAC_MCST1 + i * 8, 0);
798	}
799
800	/* use PMF to accept first MAC_MCST_NUM (15) addresses */
801	/* TBD: sort addresses and write them in ascending order
802	* into RX_MAC_MCST regs. we skip this phase now and accept ALL
803	* multicast frames throu IMF */
804	/* accept the rest of addresses throu IMF */
805	netdev_for_each_mc_addr(ha, ndev) {
806	hash = 0;
807	for (i = 0; i < ETH_ALEN; i++)
808	hash ^= ha->addr[i];
809	reg = regRX_MCST_HASH0 + ((hash >> 5) << 2);
810	val = READ_REG(priv, reg);
811	val |= (1 << (hash % 32));
812	WRITE_REG(priv, reg, val);
813	}
814
815	} else {
816	DBG("only own mac %d\n", netdev_mc_count(ndev));
817	rxf_val |= GMAC_RX_FILTER_AB;
818	}
819	WRITE_REG(priv, regGMAC_RXF_A, rxf_val);
820	/* enable RX */
821	/* FIXME: RXE(ON) */
822	RET();
823	}
824
825	static int bdx_set_mac(struct net_device *ndev, void *p)
826	{
827	struct bdx_priv *priv = netdev_priv(dev: ndev);
828	struct sockaddr *addr = p;
829
830	ENTER;
831	/*
832	if (netif_running(dev))
833	return -EBUSY
834	*/
835	eth_hw_addr_set(dev: ndev, addr: addr->sa_data);
836	bdx_restore_mac(ndev, priv);
837	RET(0);
838	}
839
840	static int bdx_read_mac(struct bdx_priv *priv)
841	{
842	u16 macAddress[3], i;
843	u8 addr[ETH_ALEN];
844	ENTER;
845
846	macAddress[2] = READ_REG(priv, regUNC_MAC0_A);
847	macAddress[2] = READ_REG(priv, regUNC_MAC0_A);
848	macAddress[1] = READ_REG(priv, regUNC_MAC1_A);
849	macAddress[1] = READ_REG(priv, regUNC_MAC1_A);
850	macAddress[0] = READ_REG(priv, regUNC_MAC2_A);
851	macAddress[0] = READ_REG(priv, regUNC_MAC2_A);
852	for (i = 0; i < 3; i++) {
853	addr[i * 2 + 1] = macAddress[i];
854	addr[i * 2] = macAddress[i] >> 8;
855	}
856	eth_hw_addr_set(dev: priv->ndev, addr);
857	RET(0);
858	}
859
860	static u64 bdx_read_l2stat(struct bdx_priv *priv, int reg)
861	{
862	u64 val;
863
864	val = READ_REG(priv, reg);
865	val |= ((u64) READ_REG(priv, reg + 8)) << 32;
866	return val;
867	}
868
869	/*Do the statistics-update work*/
870	static void bdx_update_stats(struct bdx_priv *priv)
871	{
872	struct bdx_stats *stats = &priv->hw_stats;
873	u64 *stats_vector = (u64 *) stats;
874	int i;
875	int addr;
876
877	/*Fill HW structure */
878	addr = 0x7200;
879	/*First 12 statistics - 0x7200 - 0x72B0 */
880	for (i = 0; i < 12; i++) {
881	stats_vector[i] = bdx_read_l2stat(priv, reg: addr);
882	addr += 0x10;
883	}
884	BDX_ASSERT(addr != 0x72C0);
885	/* 0x72C0-0x72E0 RSRV */
886	addr = 0x72F0;
887	for (; i < 16; i++) {
888	stats_vector[i] = bdx_read_l2stat(priv, reg: addr);
889	addr += 0x10;
890	}
891	BDX_ASSERT(addr != 0x7330);
892	/* 0x7330-0x7360 RSRV */
893	addr = 0x7370;
894	for (; i < 19; i++) {
895	stats_vector[i] = bdx_read_l2stat(priv, reg: addr);
896	addr += 0x10;
897	}
898	BDX_ASSERT(addr != 0x73A0);
899	/* 0x73A0-0x73B0 RSRV */
900	addr = 0x73C0;
901	for (; i < 23; i++) {
902	stats_vector[i] = bdx_read_l2stat(priv, reg: addr);
903	addr += 0x10;
904	}
905	BDX_ASSERT(addr != 0x7400);
906	BDX_ASSERT((sizeof(struct bdx_stats) / sizeof(u64)) != i);
907	}
908
909	static void print_rxdd(struct rxd_desc *rxdd, u32 rxd_val1, u16 len,
910	u16 rxd_vlan);
911	static void print_rxfd(struct rxf_desc *rxfd);
912
913	/*************************************************************************
914	* Rx DB *
915	*************************************************************************/
916
917	static void bdx_rxdb_destroy(struct rxdb *db)
918	{
919	vfree(addr: db);
920	}
921
922	static struct rxdb *bdx_rxdb_create(int nelem)
923	{
924	struct rxdb *db;
925	int i;
926
927	db = vmalloc(size: sizeof(struct rxdb)
928	+ (nelem * sizeof(int))
929	+ (nelem * sizeof(struct rx_map)));
930	if (likely(db != NULL)) {
931	db->stack = (int *)(db + 1);
932	db->elems = (void *)(db->stack + nelem);
933	db->nelem = nelem;
934	db->top = nelem;
935	for (i = 0; i < nelem; i++)
936	db->stack[i] = nelem - i - 1; /* to make first allocs
937	close to db struct*/
938	}
939
940	return db;
941	}
942
943	static inline int bdx_rxdb_alloc_elem(struct rxdb *db)
944	{
945	BDX_ASSERT(db->top <= 0);
946	return db->stack[--(db->top)];
947	}
948
949	static inline void *bdx_rxdb_addr_elem(struct rxdb *db, int n)
950	{
951	BDX_ASSERT((n < 0) || (n >= db->nelem));
952	return db->elems + n;
953	}
954
955	static inline int bdx_rxdb_available(struct rxdb *db)
956	{
957	return db->top;
958	}
959
960	static inline void bdx_rxdb_free_elem(struct rxdb *db, int n)
961	{
962	BDX_ASSERT((n >= db->nelem) || (n < 0));
963	db->stack[(db->top)++] = n;
964	}
965
966	/*************************************************************************
967	* Rx Init *
968	*************************************************************************/
969
970	/**
971	* bdx_rx_init - initialize RX all related HW and SW resources
972	* @priv: NIC private structure
973	*
974	* Returns 0 on success, negative value on failure
975	*
976	* It creates rxf and rxd fifos, update relevant HW registers, preallocate
977	* skb for rx. It assumes that Rx is desabled in HW
978	* funcs are grouped for better cache usage
979	*
980	* RxD fifo is smaller than RxF fifo by design. Upon high load, RxD will be
981	* filled and packets will be dropped by nic without getting into host or
982	* cousing interrupt. Anyway, in that condition, host has no chance to process
983	* all packets, but dropping in nic is cheaper, since it takes 0 cpu cycles
984	*/
985
986	/* TBD: ensure proper packet size */
987
988	static int bdx_rx_init(struct bdx_priv *priv)
989	{
990	ENTER;
991
992	if (bdx_fifo_init(priv, f: &priv->rxd_fifo0.m, fsz_type: priv->rxd_size,
993	regRXD_CFG0_0, regRXD_CFG1_0,
994	regRXD_RPTR_0, regRXD_WPTR_0))
995	goto err_mem;
996	if (bdx_fifo_init(priv, f: &priv->rxf_fifo0.m, fsz_type: priv->rxf_size,
997	regRXF_CFG0_0, regRXF_CFG1_0,
998	regRXF_RPTR_0, regRXF_WPTR_0))
999	goto err_mem;
1000	priv->rxdb = bdx_rxdb_create(nelem: priv->rxf_fifo0.m.memsz /
1001	sizeof(struct rxf_desc));
1002	if (!priv->rxdb)
1003	goto err_mem;
1004
1005	priv->rxf_fifo0.m.pktsz = priv->ndev->mtu + VLAN_ETH_HLEN;
1006	return 0;
1007
1008	err_mem:
1009	netdev_err(dev: priv->ndev, format: "Rx init failed\n");
1010	return -ENOMEM;
1011	}
1012
1013	/**
1014	* bdx_rx_free_skbs - frees and unmaps all skbs allocated for the fifo
1015	* @priv: NIC private structure
1016	* @f: RXF fifo
1017	*/
1018	static void bdx_rx_free_skbs(struct bdx_priv *priv, struct rxf_fifo *f)
1019	{
1020	struct rx_map *dm;
1021	struct rxdb *db = priv->rxdb;
1022	u16 i;
1023
1024	ENTER;
1025	DBG("total=%d free=%d busy=%d\n", db->nelem, bdx_rxdb_available(db),
1026	db->nelem - bdx_rxdb_available(db));
1027	while (bdx_rxdb_available(db) > 0) {
1028	i = bdx_rxdb_alloc_elem(db);
1029	dm = bdx_rxdb_addr_elem(db, n: i);
1030	dm->dma = 0;
1031	}
1032	for (i = 0; i < db->nelem; i++) {
1033	dm = bdx_rxdb_addr_elem(db, n: i);
1034	if (dm->dma) {
1035	dma_unmap_single(&priv->pdev->dev, dm->dma,
1036	f->m.pktsz, DMA_FROM_DEVICE);
1037	dev_kfree_skb(dm->skb);
1038	}
1039	}
1040	}
1041
1042	/**
1043	* bdx_rx_free - release all Rx resources
1044	* @priv: NIC private structure
1045	*
1046	* It assumes that Rx is desabled in HW
1047	*/
1048	static void bdx_rx_free(struct bdx_priv *priv)
1049	{
1050	ENTER;
1051	if (priv->rxdb) {
1052	bdx_rx_free_skbs(priv, f: &priv->rxf_fifo0);
1053	bdx_rxdb_destroy(db: priv->rxdb);
1054	priv->rxdb = NULL;
1055	}
1056	bdx_fifo_free(priv, f: &priv->rxf_fifo0.m);
1057	bdx_fifo_free(priv, f: &priv->rxd_fifo0.m);
1058
1059	RET();
1060	}
1061
1062	/*************************************************************************
1063	* Rx Engine *
1064	*************************************************************************/
1065
1066	/**
1067	* bdx_rx_alloc_skbs - fill rxf fifo with new skbs
1068	* @priv: nic's private structure
1069	* @f: RXF fifo that needs skbs
1070	*
1071	* It allocates skbs, build rxf descs and push it (rxf descr) into rxf fifo.
1072	* skb's virtual and physical addresses are stored in skb db.
1073	* To calculate free space, func uses cached values of RPTR and WPTR
1074	* When needed, it also updates RPTR and WPTR.
1075	*/
1076
1077	/* TBD: do not update WPTR if no desc were written */
1078
1079	static void bdx_rx_alloc_skbs(struct bdx_priv *priv, struct rxf_fifo *f)
1080	{
1081	struct sk_buff *skb;
1082	struct rxf_desc *rxfd;
1083	struct rx_map *dm;
1084	int dno, delta, idx;
1085	struct rxdb *db = priv->rxdb;
1086
1087	ENTER;
1088	dno = bdx_rxdb_available(db) - 1;
1089	while (dno > 0) {
1090	skb = netdev_alloc_skb(dev: priv->ndev, length: f->m.pktsz + NET_IP_ALIGN);
1091	if (!skb)
1092	break;
1093
1094	skb_reserve(skb, NET_IP_ALIGN);
1095
1096	idx = bdx_rxdb_alloc_elem(db);
1097	dm = bdx_rxdb_addr_elem(db, n: idx);
1098	dm->dma = dma_map_single(&priv->pdev->dev, skb->data,
1099	f->m.pktsz, DMA_FROM_DEVICE);
1100	dm->skb = skb;
1101	rxfd = (struct rxf_desc *)(f->m.va + f->m.wptr);
1102	rxfd->info = CPU_CHIP_SWAP32(0x10003); /* INFO=1 BC=3 */
1103	rxfd->va_lo = idx;
1104	rxfd->pa_lo = CPU_CHIP_SWAP32(L32_64(dm->dma));
1105	rxfd->pa_hi = CPU_CHIP_SWAP32(H32_64(dm->dma));
1106	rxfd->len = CPU_CHIP_SWAP32(f->m.pktsz);
1107	print_rxfd(rxfd);
1108
1109	f->m.wptr += sizeof(struct rxf_desc);
1110	delta = f->m.wptr - f->m.memsz;
1111	if (unlikely(delta >= 0)) {
1112	f->m.wptr = delta;
1113	if (delta > 0) {
1114	memcpy(f->m.va, f->m.va + f->m.memsz, delta);
1115	DBG("wrapped descriptor\n");
1116	}
1117	}
1118	dno--;
1119	}
1120	/*TBD: to do - delayed rxf wptr like in txd */
1121	WRITE_REG(priv, f->m.reg_WPTR, f->m.wptr & TXF_WPTR_WR_PTR);
1122	RET();
1123	}
1124
1125	static inline void
1126	NETIF_RX_MUX(struct bdx_priv *priv, u32 rxd_val1, u16 rxd_vlan,
1127	struct sk_buff *skb)
1128	{
1129	ENTER;
1130	DBG("rxdd->flags.bits.vtag=%d\n", GET_RXD_VTAG(rxd_val1));
1131	if (GET_RXD_VTAG(rxd_val1)) {
1132	DBG("%s: vlan rcv vlan '%x' vtag '%x'\n",
1133	priv->ndev->name,
1134	GET_RXD_VLAN_ID(rxd_vlan),
1135	GET_RXD_VTAG(rxd_val1));
1136	__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), GET_RXD_VLAN_TCI(rxd_vlan));
1137	}
1138	netif_receive_skb(skb);
1139	}
1140
1141	static void bdx_recycle_skb(struct bdx_priv *priv, struct rxd_desc *rxdd)
1142	{
1143	struct rxf_desc *rxfd;
1144	struct rx_map *dm;
1145	struct rxf_fifo *f;
1146	struct rxdb *db;
1147	int delta;
1148
1149	ENTER;
1150	DBG("priv=%p rxdd=%p\n", priv, rxdd);
1151	f = &priv->rxf_fifo0;
1152	db = priv->rxdb;
1153	DBG("db=%p f=%p\n", db, f);
1154	dm = bdx_rxdb_addr_elem(db, n: rxdd->va_lo);
1155	DBG("dm=%p\n", dm);
1156	rxfd = (struct rxf_desc *)(f->m.va + f->m.wptr);
1157	rxfd->info = CPU_CHIP_SWAP32(0x10003); /* INFO=1 BC=3 */
1158	rxfd->va_lo = rxdd->va_lo;
1159	rxfd->pa_lo = CPU_CHIP_SWAP32(L32_64(dm->dma));
1160	rxfd->pa_hi = CPU_CHIP_SWAP32(H32_64(dm->dma));
1161	rxfd->len = CPU_CHIP_SWAP32(f->m.pktsz);
1162	print_rxfd(rxfd);
1163
1164	f->m.wptr += sizeof(struct rxf_desc);
1165	delta = f->m.wptr - f->m.memsz;
1166	if (unlikely(delta >= 0)) {
1167	f->m.wptr = delta;
1168	if (delta > 0) {
1169	memcpy(f->m.va, f->m.va + f->m.memsz, delta);
1170	DBG("wrapped descriptor\n");
1171	}
1172	}
1173	RET();
1174	}
1175
1176	/**
1177	* bdx_rx_receive - receives full packets from RXD fifo and pass them to OS
1178	* NOTE: a special treatment is given to non-continuous descriptors
1179	* that start near the end, wraps around and continue at the beginning. a second
1180	* part is copied right after the first, and then descriptor is interpreted as
1181	* normal. fifo has an extra space to allow such operations
1182	* @priv: nic's private structure
1183	* @f: RXF fifo that needs skbs
1184	* @budget: maximum number of packets to receive
1185	*/
1186
1187	/* TBD: replace memcpy func call by explicite inline asm */
1188
1189	static int bdx_rx_receive(struct bdx_priv *priv, struct rxd_fifo *f, int budget)
1190	{
1191	struct net_device *ndev = priv->ndev;
1192	struct sk_buff *skb, *skb2;
1193	struct rxd_desc *rxdd;
1194	struct rx_map *dm;
1195	struct rxf_fifo *rxf_fifo;
1196	int tmp_len, size;
1197	int done = 0;
1198	int max_done = BDX_MAX_RX_DONE;
1199	struct rxdb *db = NULL;
1200	/* Unmarshalled descriptor - copy of descriptor in host order */
1201	u32 rxd_val1;
1202	u16 len;
1203	u16 rxd_vlan;
1204
1205	ENTER;
1206	max_done = budget;
1207
1208	f->m.wptr = READ_REG(priv, f->m.reg_WPTR) & TXF_WPTR_WR_PTR;
1209
1210	size = f->m.wptr - f->m.rptr;
1211	if (size < 0)
1212	size = f->m.memsz + size; /* size is negative :-) */
1213
1214	while (size > 0) {
1215
1216	rxdd = (struct rxd_desc *)(f->m.va + f->m.rptr);
1217	rxd_val1 = CPU_CHIP_SWAP32(rxdd->rxd_val1);
1218
1219	len = CPU_CHIP_SWAP16(rxdd->len);
1220
1221	rxd_vlan = CPU_CHIP_SWAP16(rxdd->rxd_vlan);
1222
1223	print_rxdd(rxdd, rxd_val1, len, rxd_vlan);
1224
1225	tmp_len = GET_RXD_BC(rxd_val1) << 3;
1226	BDX_ASSERT(tmp_len <= 0);
1227	size -= tmp_len;
1228	if (size < 0) /* test for partially arrived descriptor */
1229	break;
1230
1231	f->m.rptr += tmp_len;
1232
1233	tmp_len = f->m.rptr - f->m.memsz;
1234	if (unlikely(tmp_len >= 0)) {
1235	f->m.rptr = tmp_len;
1236	if (tmp_len > 0) {
1237	DBG("wrapped desc rptr=%d tmp_len=%d\n",
1238	f->m.rptr, tmp_len);
1239	memcpy(f->m.va + f->m.memsz, f->m.va, tmp_len);
1240	}
1241	}
1242
1243	if (unlikely(GET_RXD_ERR(rxd_val1))) {
1244	DBG("rxd_err = 0x%x\n", GET_RXD_ERR(rxd_val1));
1245	ndev->stats.rx_errors++;
1246	bdx_recycle_skb(priv, rxdd);
1247	continue;
1248	}
1249
1250	rxf_fifo = &priv->rxf_fifo0;
1251	db = priv->rxdb;
1252	dm = bdx_rxdb_addr_elem(db, n: rxdd->va_lo);
1253	skb = dm->skb;
1254
1255	if (len < BDX_COPYBREAK &&
1256	(skb2 = netdev_alloc_skb(dev: priv->ndev, length: len + NET_IP_ALIGN))) {
1257	skb_reserve(skb: skb2, NET_IP_ALIGN);
1258	/*skb_put(skb2, len); */
1259	dma_sync_single_for_cpu(dev: &priv->pdev->dev, addr: dm->dma,
1260	size: rxf_fifo->m.pktsz,
1261	dir: DMA_FROM_DEVICE);
1262	memcpy(skb2->data, skb->data, len);
1263	bdx_recycle_skb(priv, rxdd);
1264	skb = skb2;
1265	} else {
1266	dma_unmap_single(&priv->pdev->dev, dm->dma,
1267	rxf_fifo->m.pktsz, DMA_FROM_DEVICE);
1268	bdx_rxdb_free_elem(db, n: rxdd->va_lo);
1269	}
1270
1271	ndev->stats.rx_bytes += len;
1272
1273	skb_put(skb, len);
1274	skb->protocol = eth_type_trans(skb, dev: ndev);
1275
1276	/* Non-IP packets aren't checksum-offloaded */
1277	if (GET_RXD_PKT_ID(rxd_val1) == 0)
1278	skb_checksum_none_assert(skb);
1279	else
1280	skb->ip_summed = CHECKSUM_UNNECESSARY;
1281
1282	NETIF_RX_MUX(priv, rxd_val1, rxd_vlan, skb);
1283
1284	if (++done >= max_done)
1285	break;
1286	}
1287
1288	ndev->stats.rx_packets += done;
1289
1290	/* FIXME: do smth to minimize pci accesses */
1291	WRITE_REG(priv, f->m.reg_RPTR, f->m.rptr & TXF_WPTR_WR_PTR);
1292
1293	bdx_rx_alloc_skbs(priv, f: &priv->rxf_fifo0);
1294
1295	RET(done);
1296	}
1297
1298	/*************************************************************************
1299	* Debug / Temprorary Code *
1300	*************************************************************************/
1301	static void print_rxdd(struct rxd_desc *rxdd, u32 rxd_val1, u16 len,
1302	u16 rxd_vlan)
1303	{
1304	DBG("ERROR: rxdd bc %d rxfq %d to %d type %d err %d rxp %d pkt_id %d vtag %d len %d vlan_id %d cfi %d prio %d va_lo %d va_hi %d\n",
1305	GET_RXD_BC(rxd_val1), GET_RXD_RXFQ(rxd_val1), GET_RXD_TO(rxd_val1),
1306	GET_RXD_TYPE(rxd_val1), GET_RXD_ERR(rxd_val1),
1307	GET_RXD_RXP(rxd_val1), GET_RXD_PKT_ID(rxd_val1),
1308	GET_RXD_VTAG(rxd_val1), len, GET_RXD_VLAN_ID(rxd_vlan),
1309	GET_RXD_CFI(rxd_vlan), GET_RXD_PRIO(rxd_vlan), rxdd->va_lo,
1310	rxdd->va_hi);
1311	}
1312
1313	static void print_rxfd(struct rxf_desc *rxfd)
1314	{
1315	DBG("=== RxF desc CHIP ORDER/ENDIANNESS =============\n"
1316	"info 0x%x va_lo %u pa_lo 0x%x pa_hi 0x%x len 0x%x\n",
1317	rxfd->info, rxfd->va_lo, rxfd->pa_lo, rxfd->pa_hi, rxfd->len);
1318	}
1319
1320	/*
1321	* TX HW/SW interaction overview
1322	* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1323	* There are 2 types of TX communication channels between driver and NIC.
1324	* 1) TX Free Fifo - TXF - holds ack descriptors for sent packets
1325	* 2) TX Data Fifo - TXD - holds descriptors of full buffers.
1326	*
1327	* Currently NIC supports TSO, checksuming and gather DMA
1328	* UFO and IP fragmentation is on the way
1329	*
1330	* RX SW Data Structures
1331	* ~~~~~~~~~~~~~~~~~~~~~
1332	* txdb - used to keep track of all skbs owned by SW and their dma addresses.
1333	* For TX case, ownership lasts from geting packet via hard_xmit and until HW
1334	* acknowledges sent by TXF descriptors.
1335	* Implemented as cyclic buffer.
1336	* fifo - keeps info about fifo's size and location, relevant HW registers,
1337	* usage and skb db. Each RXD and RXF Fifo has its own fifo structure.
1338	* Implemented as simple struct.
1339	*
1340	* TX SW Execution Flow
1341	* ~~~~~~~~~~~~~~~~~~~~
1342	* OS calls driver's hard_xmit method with packet to sent.
1343	* Driver creates DMA mappings, builds TXD descriptors and kicks HW
1344	* by updating TXD WPTR.
1345	* When packet is sent, HW write us TXF descriptor and SW frees original skb.
1346	* To prevent TXD fifo overflow without reading HW registers every time,
1347	* SW deploys "tx level" technique.
1348	* Upon strart up, tx level is initialized to TXD fifo length.
1349	* For every sent packet, SW gets its TXD descriptor sizei
1350	* (from precalculated array) and substructs it from tx level.
1351	* The size is also stored in txdb. When TXF ack arrives, SW fetch size of
1352	* original TXD descriptor from txdb and adds it to tx level.
1353	* When Tx level drops under some predefined treshhold, the driver
1354	* stops the TX queue. When TX level rises above that level,
1355	* the tx queue is enabled again.
1356	*
1357	* This technique avoids eccessive reading of RPTR and WPTR registers.
1358	* As our benchmarks shows, it adds 1.5 Gbit/sec to NIS's throuput.
1359	*/
1360
1361	/**
1362	* __bdx_tx_db_ptr_next - helper function, increment read/write pointer + wrap
1363	* @db: tx data base
1364	* @pptr: read or write pointer
1365	*/
1366	static inline void __bdx_tx_db_ptr_next(struct txdb *db, struct tx_map **pptr)
1367	{
1368	BDX_ASSERT(db == NULL || pptr == NULL); /* sanity */
1369
1370	BDX_ASSERT(*pptr != db->rptr && /* expect either read */
1371	*pptr != db->wptr); /* or write pointer */
1372
1373	BDX_ASSERT(*pptr < db->start || /* pointer has to be */
1374	*pptr >= db->end); /* in range */
1375
1376	++*pptr;
1377	if (unlikely(*pptr == db->end))
1378	*pptr = db->start;
1379	}
1380
1381	/**
1382	* bdx_tx_db_inc_rptr - increment read pointer
1383	* @db: tx data base
1384	*/
1385	static inline void bdx_tx_db_inc_rptr(struct txdb *db)
1386	{
1387	BDX_ASSERT(db->rptr == db->wptr); /* can't read from empty db */
1388	__bdx_tx_db_ptr_next(db, pptr: &db->rptr);
1389	}
1390
1391	/**
1392	* bdx_tx_db_inc_wptr - increment write pointer
1393	* @db: tx data base
1394	*/
1395	static inline void bdx_tx_db_inc_wptr(struct txdb *db)
1396	{
1397	__bdx_tx_db_ptr_next(db, pptr: &db->wptr);
1398	BDX_ASSERT(db->rptr == db->wptr); /* we can not get empty db as
1399	a result of write */
1400	}
1401
1402	/**
1403	* bdx_tx_db_init - creates and initializes tx db
1404	* @d: tx data base
1405	* @sz_type: size of tx fifo
1406	*
1407	* Returns 0 on success, error code otherwise
1408	*/
1409	static int bdx_tx_db_init(struct txdb *d, int sz_type)
1410	{
1411	int memsz = FIFO_SIZE * (1 << (sz_type + 1));
1412
1413	d->start = vmalloc(size: memsz);
1414	if (!d->start)
1415	return -ENOMEM;
1416
1417	/*
1418	* In order to differentiate between db is empty and db is full
1419	* states at least one element should always be empty in order to
1420	* avoid rptr == wptr which means db is empty
1421	*/
1422	d->size = memsz / sizeof(struct tx_map) - 1;
1423	d->end = d->start + d->size + 1; /* just after last element */
1424
1425	/* all dbs are created equally empty */
1426	d->rptr = d->start;
1427	d->wptr = d->start;
1428
1429	return 0;
1430	}
1431
1432	/**
1433	* bdx_tx_db_close - closes tx db and frees all memory
1434	* @d: tx data base
1435	*/
1436	static void bdx_tx_db_close(struct txdb *d)
1437	{
1438	BDX_ASSERT(d == NULL);
1439
1440	vfree(addr: d->start);
1441	d->start = NULL;
1442	}
1443
1444	/*************************************************************************
1445	* Tx Engine *
1446	*************************************************************************/
1447
1448	/* sizes of tx desc (including padding if needed) as function
1449	* of skb's frag number */
1450	static struct {
1451	u16 bytes;
1452	u16 qwords; /* qword = 64 bit */
1453	} txd_sizes[MAX_SKB_FRAGS + 1];
1454
1455	/**
1456	* bdx_tx_map_skb - creates and stores dma mappings for skb's data blocks
1457	* @priv: NIC private structure
1458	* @skb: socket buffer to map
1459	* @txdd: TX descriptor to use
1460	*
1461	* It makes dma mappings for skb's data blocks and writes them to PBL of
1462	* new tx descriptor. It also stores them in the tx db, so they could be
1463	* unmaped after data was sent. It is reponsibility of a caller to make
1464	* sure that there is enough space in the tx db. Last element holds pointer
1465	* to skb itself and marked with zero length
1466	*/
1467	static inline void
1468	bdx_tx_map_skb(struct bdx_priv *priv, struct sk_buff *skb,
1469	struct txd_desc *txdd)
1470	{
1471	struct txdb *db = &priv->txdb;
1472	struct pbl *pbl = &txdd->pbl[0];
1473	int nr_frags = skb_shinfo(skb)->nr_frags;
1474	int i;
1475
1476	db->wptr->len = skb_headlen(skb);
1477	db->wptr->addr.dma = dma_map_single(&priv->pdev->dev, skb->data,
1478	db->wptr->len, DMA_TO_DEVICE);
1479	pbl->len = CPU_CHIP_SWAP32(db->wptr->len);
1480	pbl->pa_lo = CPU_CHIP_SWAP32(L32_64(db->wptr->addr.dma));
1481	pbl->pa_hi = CPU_CHIP_SWAP32(H32_64(db->wptr->addr.dma));
1482	DBG("=== pbl len: 0x%x ================\n", pbl->len);
1483	DBG("=== pbl pa_lo: 0x%x ================\n", pbl->pa_lo);
1484	DBG("=== pbl pa_hi: 0x%x ================\n", pbl->pa_hi);
1485	bdx_tx_db_inc_wptr(db);
1486
1487	for (i = 0; i < nr_frags; i++) {
1488	const skb_frag_t *frag;
1489
1490	frag = &skb_shinfo(skb)->frags[i];
1491	db->wptr->len = skb_frag_size(frag);
1492	db->wptr->addr.dma = skb_frag_dma_map(dev: &priv->pdev->dev, frag,
1493	offset: 0, size: skb_frag_size(frag),
1494	dir: DMA_TO_DEVICE);
1495
1496	pbl++;
1497	pbl->len = CPU_CHIP_SWAP32(db->wptr->len);
1498	pbl->pa_lo = CPU_CHIP_SWAP32(L32_64(db->wptr->addr.dma));
1499	pbl->pa_hi = CPU_CHIP_SWAP32(H32_64(db->wptr->addr.dma));
1500	bdx_tx_db_inc_wptr(db);
1501	}
1502
1503	/* add skb clean up info. */
1504	db->wptr->len = -txd_sizes[nr_frags].bytes;
1505	db->wptr->addr.skb = skb;
1506	bdx_tx_db_inc_wptr(db);
1507	}
1508
1509	/* init_txd_sizes - precalculate sizes of descriptors for skbs up to 16 frags
1510	* number of frags is used as index to fetch correct descriptors size,
1511	* instead of calculating it each time */
1512	static void __init init_txd_sizes(void)
1513	{
1514	int i, lwords;
1515
1516	/* 7 - is number of lwords in txd with one phys buffer
1517	* 3 - is number of lwords used for every additional phys buffer */
1518	for (i = 0; i < MAX_SKB_FRAGS + 1; i++) {
1519	lwords = 7 + (i * 3);
1520	if (lwords & 1)
1521	lwords++; /* pad it with 1 lword */
1522	txd_sizes[i].qwords = lwords >> 1;
1523	txd_sizes[i].bytes = lwords << 2;
1524	}
1525	}
1526
1527	/* bdx_tx_init - initialize all Tx related stuff.
1528	* Namely, TXD and TXF fifos, database etc */
1529	static int bdx_tx_init(struct bdx_priv *priv)
1530	{
1531	if (bdx_fifo_init(priv, f: &priv->txd_fifo0.m, fsz_type: priv->txd_size,
1532	regTXD_CFG0_0,
1533	regTXD_CFG1_0, regTXD_RPTR_0, regTXD_WPTR_0))
1534	goto err_mem;
1535	if (bdx_fifo_init(priv, f: &priv->txf_fifo0.m, fsz_type: priv->txf_size,
1536	regTXF_CFG0_0,
1537	regTXF_CFG1_0, regTXF_RPTR_0, regTXF_WPTR_0))
1538	goto err_mem;
1539
1540	/* The TX db has to keep mappings for all packets sent (on TxD)
1541	* and not yet reclaimed (on TxF) */
1542	if (bdx_tx_db_init(d: &priv->txdb, max(priv->txd_size, priv->txf_size)))
1543	goto err_mem;
1544
1545	priv->tx_level = BDX_MAX_TX_LEVEL;
1546	#ifdef BDX_DELAY_WPTR
1547	priv->tx_update_mark = priv->tx_level - 1024;
1548	#endif
1549	return 0;
1550
1551	err_mem:
1552	netdev_err(dev: priv->ndev, format: "Tx init failed\n");
1553	return -ENOMEM;
1554	}
1555
1556	/**
1557	* bdx_tx_space - calculates available space in TX fifo
1558	* @priv: NIC private structure
1559	*
1560	* Returns available space in TX fifo in bytes
1561	*/
1562	static inline int bdx_tx_space(struct bdx_priv *priv)
1563	{
1564	struct txd_fifo *f = &priv->txd_fifo0;
1565	int fsize;
1566
1567	f->m.rptr = READ_REG(priv, f->m.reg_RPTR) & TXF_WPTR_WR_PTR;
1568	fsize = f->m.rptr - f->m.wptr;
1569	if (fsize <= 0)
1570	fsize = f->m.memsz + fsize;
1571	return fsize;
1572	}
1573
1574	/**
1575	* bdx_tx_transmit - send packet to NIC
1576	* @skb: packet to send
1577	* @ndev: network device assigned to NIC
1578	* Return codes:
1579	* o NETDEV_TX_OK everything ok.
1580	* o NETDEV_TX_BUSY Cannot transmit packet, try later
1581	* Usually a bug, means queue start/stop flow control is broken in
1582	* the driver. Note: the driver must NOT put the skb in its DMA ring.
1583	*/
1584	static netdev_tx_t bdx_tx_transmit(struct sk_buff *skb,
1585	struct net_device *ndev)
1586	{
1587	struct bdx_priv *priv = netdev_priv(dev: ndev);
1588	struct txd_fifo *f = &priv->txd_fifo0;
1589	int txd_checksum = 7; /* full checksum */
1590	int txd_lgsnd = 0;
1591	int txd_vlan_id = 0;
1592	int txd_vtag = 0;
1593	int txd_mss = 0;
1594
1595	int nr_frags = skb_shinfo(skb)->nr_frags;
1596	struct txd_desc *txdd;
1597	int len;
1598	unsigned long flags;
1599
1600	ENTER;
1601	local_irq_save(flags);
1602	spin_lock(lock: &priv->tx_lock);
1603
1604	/* build tx descriptor */
1605	BDX_ASSERT(f->m.wptr >= f->m.memsz); /* started with valid wptr */
1606	txdd = (struct txd_desc *)(f->m.va + f->m.wptr);
1607	if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL))
1608	txd_checksum = 0;
1609
1610	if (skb_shinfo(skb)->gso_size) {
1611	txd_mss = skb_shinfo(skb)->gso_size;
1612	txd_lgsnd = 1;
1613	DBG("skb %p skb len %d gso size = %d\n", skb, skb->len,
1614	txd_mss);
1615	}
1616
1617	if (skb_vlan_tag_present(skb)) {
1618	/*Cut VLAN ID to 12 bits */
1619	txd_vlan_id = skb_vlan_tag_get(skb) & BITS_MASK(12);
1620	txd_vtag = 1;
1621	}
1622
1623	txdd->length = CPU_CHIP_SWAP16(skb->len);
1624	txdd->mss = CPU_CHIP_SWAP16(txd_mss);
1625	txdd->txd_val1 =
1626	CPU_CHIP_SWAP32(TXD_W1_VAL
1627	(txd_sizes[nr_frags].qwords, txd_checksum, txd_vtag,
1628	txd_lgsnd, txd_vlan_id));
1629	DBG("=== TxD desc =====================\n");
1630	DBG("=== w1: 0x%x ================\n", txdd->txd_val1);
1631	DBG("=== w2: mss 0x%x len 0x%x\n", txdd->mss, txdd->length);
1632
1633	bdx_tx_map_skb(priv, skb, txdd);
1634
1635	/* increment TXD write pointer. In case of
1636	fifo wrapping copy reminder of the descriptor
1637	to the beginning */
1638	f->m.wptr += txd_sizes[nr_frags].bytes;
1639	len = f->m.wptr - f->m.memsz;
1640	if (unlikely(len >= 0)) {
1641	f->m.wptr = len;
1642	if (len > 0) {
1643	BDX_ASSERT(len > f->m.memsz);
1644	memcpy(f->m.va, f->m.va + f->m.memsz, len);
1645	}
1646	}
1647	BDX_ASSERT(f->m.wptr >= f->m.memsz); /* finished with valid wptr */
1648
1649	priv->tx_level -= txd_sizes[nr_frags].bytes;
1650	BDX_ASSERT(priv->tx_level <= 0 || priv->tx_level > BDX_MAX_TX_LEVEL);
1651	#ifdef BDX_DELAY_WPTR
1652	if (priv->tx_level > priv->tx_update_mark) {
1653	/* Force memory writes to complete before letting h/w
1654	know there are new descriptors to fetch.
1655	(might be needed on platforms like IA64)
1656	wmb(); */
1657	WRITE_REG(priv, f->m.reg_WPTR, f->m.wptr & TXF_WPTR_WR_PTR);
1658	} else {
1659	if (priv->tx_noupd++ > BDX_NO_UPD_PACKETS) {
1660	priv->tx_noupd = 0;
1661	WRITE_REG(priv, f->m.reg_WPTR,
1662	f->m.wptr & TXF_WPTR_WR_PTR);
1663	}
1664	}
1665	#else
1666	/* Force memory writes to complete before letting h/w
1667	know there are new descriptors to fetch.
1668	(might be needed on platforms like IA64)
1669	wmb(); */
1670	WRITE_REG(priv, f->m.reg_WPTR, f->m.wptr & TXF_WPTR_WR_PTR);
1671
1672	#endif
1673	#ifdef BDX_LLTX
1674	netif_trans_update(dev: ndev); /* NETIF_F_LLTX driver :( */
1675	#endif
1676	ndev->stats.tx_packets++;
1677	ndev->stats.tx_bytes += skb->len;
1678
1679	if (priv->tx_level < BDX_MIN_TX_LEVEL) {
1680	DBG("%s: %s: TX Q STOP level %d\n",
1681	BDX_DRV_NAME, ndev->name, priv->tx_level);
1682	netif_stop_queue(dev: ndev);
1683	}
1684
1685	spin_unlock_irqrestore(lock: &priv->tx_lock, flags);
1686	return NETDEV_TX_OK;
1687	}
1688
1689	/**
1690	* bdx_tx_cleanup - clean TXF fifo, run in the context of IRQ.
1691	* @priv: bdx adapter
1692	*
1693	* It scans TXF fifo for descriptors, frees DMA mappings and reports to OS
1694	* that those packets were sent
1695	*/
1696	static void bdx_tx_cleanup(struct bdx_priv *priv)
1697	{
1698	struct txf_fifo *f = &priv->txf_fifo0;
1699	struct txdb *db = &priv->txdb;
1700	int tx_level = 0;
1701
1702	ENTER;
1703	f->m.wptr = READ_REG(priv, f->m.reg_WPTR) & TXF_WPTR_MASK;
1704	BDX_ASSERT(f->m.rptr >= f->m.memsz); /* started with valid rptr */
1705
1706	while (f->m.wptr != f->m.rptr) {
1707	f->m.rptr += BDX_TXF_DESC_SZ;
1708	f->m.rptr &= f->m.size_mask;
1709
1710	/* unmap all the fragments */
1711	/* first has to come tx_maps containing dma */
1712	BDX_ASSERT(db->rptr->len == 0);
1713	do {
1714	BDX_ASSERT(db->rptr->addr.dma == 0);
1715	dma_unmap_page(&priv->pdev->dev, db->rptr->addr.dma,
1716	db->rptr->len, DMA_TO_DEVICE);
1717	bdx_tx_db_inc_rptr(db);
1718	} while (db->rptr->len > 0);
1719	tx_level -= db->rptr->len; /* '-' koz len is negative */
1720
1721	/* now should come skb pointer - free it */
1722	dev_consume_skb_irq(skb: db->rptr->addr.skb);
1723	bdx_tx_db_inc_rptr(db);
1724	}
1725
1726	/* let h/w know which TXF descriptors were cleaned */
1727	BDX_ASSERT((f->m.wptr & TXF_WPTR_WR_PTR) >= f->m.memsz);
1728	WRITE_REG(priv, f->m.reg_RPTR, f->m.rptr & TXF_WPTR_WR_PTR);
1729
1730	/* We reclaimed resources, so in case the Q is stopped by xmit callback,
1731	* we resume the transmission and use tx_lock to synchronize with xmit.*/
1732	spin_lock(lock: &priv->tx_lock);
1733	priv->tx_level += tx_level;
1734	BDX_ASSERT(priv->tx_level <= 0 || priv->tx_level > BDX_MAX_TX_LEVEL);
1735	#ifdef BDX_DELAY_WPTR
1736	if (priv->tx_noupd) {
1737	priv->tx_noupd = 0;
1738	WRITE_REG(priv, priv->txd_fifo0.m.reg_WPTR,
1739	priv->txd_fifo0.m.wptr & TXF_WPTR_WR_PTR);
1740	}
1741	#endif
1742
1743	if (unlikely(netif_queue_stopped(priv->ndev) &&
1744	netif_carrier_ok(priv->ndev) &&
1745	(priv->tx_level >= BDX_MIN_TX_LEVEL))) {
1746	DBG("%s: %s: TX Q WAKE level %d\n",
1747	BDX_DRV_NAME, priv->ndev->name, priv->tx_level);
1748	netif_wake_queue(dev: priv->ndev);
1749	}
1750	spin_unlock(lock: &priv->tx_lock);
1751	}
1752
1753	/**
1754	* bdx_tx_free_skbs - frees all skbs from TXD fifo.
1755	* @priv: NIC private structure
1756	*
1757	* It gets called when OS stops this dev, eg upon "ifconfig down" or rmmod
1758	*/
1759	static void bdx_tx_free_skbs(struct bdx_priv *priv)
1760	{
1761	struct txdb *db = &priv->txdb;
1762
1763	ENTER;
1764	while (db->rptr != db->wptr) {
1765	if (likely(db->rptr->len))
1766	dma_unmap_page(&priv->pdev->dev, db->rptr->addr.dma,
1767	db->rptr->len, DMA_TO_DEVICE);
1768	else
1769	dev_kfree_skb(db->rptr->addr.skb);
1770	bdx_tx_db_inc_rptr(db);
1771	}
1772	RET();
1773	}
1774
1775	/* bdx_tx_free - frees all Tx resources */
1776	static void bdx_tx_free(struct bdx_priv *priv)
1777	{
1778	ENTER;
1779	bdx_tx_free_skbs(priv);
1780	bdx_fifo_free(priv, f: &priv->txd_fifo0.m);
1781	bdx_fifo_free(priv, f: &priv->txf_fifo0.m);
1782	bdx_tx_db_close(d: &priv->txdb);
1783	}
1784
1785	/**
1786	* bdx_tx_push_desc - push descriptor to TxD fifo
1787	* @priv: NIC private structure
1788	* @data: desc's data
1789	* @size: desc's size
1790	*
1791	* Pushes desc to TxD fifo and overlaps it if needed.
1792	* NOTE: this func does not check for available space. this is responsibility
1793	* of the caller. Neither does it check that data size is smaller than
1794	* fifo size.
1795	*/
1796	static void bdx_tx_push_desc(struct bdx_priv *priv, void *data, int size)
1797	{
1798	struct txd_fifo *f = &priv->txd_fifo0;
1799	int i = f->m.memsz - f->m.wptr;
1800
1801	if (size == 0)
1802	return;
1803
1804	if (i > size) {
1805	memcpy(f->m.va + f->m.wptr, data, size);
1806	f->m.wptr += size;
1807	} else {
1808	memcpy(f->m.va + f->m.wptr, data, i);
1809	f->m.wptr = size - i;
1810	memcpy(f->m.va, data + i, f->m.wptr);
1811	}
1812	WRITE_REG(priv, f->m.reg_WPTR, f->m.wptr & TXF_WPTR_WR_PTR);
1813	}
1814
1815	/**
1816	* bdx_tx_push_desc_safe - push descriptor to TxD fifo in a safe way
1817	* @priv: NIC private structure
1818	* @data: desc's data
1819	* @size: desc's size
1820	*
1821	* NOTE: this func does check for available space and, if necessary, waits for
1822	* NIC to read existing data before writing new one.
1823	*/
1824	static void bdx_tx_push_desc_safe(struct bdx_priv *priv, void *data, int size)
1825	{
1826	int timer = 0;
1827	ENTER;
1828
1829	while (size > 0) {
1830	/* we substruct 8 because when fifo is full rptr == wptr
1831	which also means that fifo is empty, we can understand
1832	the difference, but could hw do the same ??? :) */
1833	int avail = bdx_tx_space(priv) - 8;
1834	if (avail <= 0) {
1835	if (timer++ > 300) { /* prevent endless loop */
1836	DBG("timeout while writing desc to TxD fifo\n");
1837	break;
1838	}
1839	udelay(50); /* give hw a chance to clean fifo */
1840	continue;
1841	}
1842	avail = min(avail, size);
1843	DBG("about to push %d bytes starting %p size %d\n", avail,
1844	data, size);
1845	bdx_tx_push_desc(priv, data, size: avail);
1846	size -= avail;
1847	data += avail;
1848	}
1849	RET();
1850	}
1851
1852	static const struct net_device_ops bdx_netdev_ops = {
1853	.ndo_open = bdx_open,
1854	.ndo_stop = bdx_close,
1855	.ndo_start_xmit = bdx_tx_transmit,
1856	.ndo_validate_addr = eth_validate_addr,
1857	.ndo_siocdevprivate = bdx_siocdevprivate,
1858	.ndo_set_rx_mode = bdx_setmulti,
1859	.ndo_change_mtu = bdx_change_mtu,
1860	.ndo_set_mac_address = bdx_set_mac,
1861	.ndo_vlan_rx_add_vid = bdx_vlan_rx_add_vid,
1862	.ndo_vlan_rx_kill_vid = bdx_vlan_rx_kill_vid,
1863	};
1864
1865	/**
1866	* bdx_probe - Device Initialization Routine
1867	* @pdev: PCI device information struct
1868	* @ent: entry in bdx_pci_tbl
1869	*
1870	* Returns 0 on success, negative on failure
1871	*
1872	* bdx_probe initializes an adapter identified by a pci_dev structure.
1873	* The OS initialization, configuring of the adapter private structure,
1874	* and a hardware reset occur.
1875	*
1876	* functions and their order used as explained in
1877	* /usr/src/linux/Documentation/DMA-{API,mapping}.txt
1878	*
1879	*/
1880
1881	/* TBD: netif_msg should be checked and implemented. I disable it for now */
1882	static int
1883	bdx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1884	{
1885	struct net_device *ndev;
1886	struct bdx_priv *priv;
1887	unsigned long pciaddr;
1888	u32 regionSize;
1889	struct pci_nic *nic;
1890	int err, port;
1891
1892	ENTER;
1893
1894	nic = vmalloc(size: sizeof(*nic));
1895	if (!nic)
1896	RET(-ENOMEM);
1897
1898	/************** pci *****************/
1899	err = pci_enable_device(dev: pdev);
1900	if (err) /* it triggers interrupt, dunno why. */
1901	goto err_pci; /* it's not a problem though */
1902
1903	err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
1904	if (err) {
1905	pr_err("No usable DMA configuration, aborting\n");
1906	goto err_dma;
1907	}
1908
1909	err = pci_request_regions(pdev, BDX_DRV_NAME);
1910	if (err)
1911	goto err_dma;
1912
1913	pci_set_master(dev: pdev);
1914
1915	pciaddr = pci_resource_start(pdev, 0);
1916	if (!pciaddr) {
1917	err = -EIO;
1918	pr_err("no MMIO resource\n");
1919	goto err_out_res;
1920	}
1921	regionSize = pci_resource_len(pdev, 0);
1922	if (regionSize < BDX_REGS_SIZE) {
1923	err = -EIO;
1924	pr_err("MMIO resource (%x) too small\n", regionSize);
1925	goto err_out_res;
1926	}
1927
1928	nic->regs = ioremap(offset: pciaddr, size: regionSize);
1929	if (!nic->regs) {
1930	err = -EIO;
1931	pr_err("ioremap failed\n");
1932	goto err_out_res;
1933	}
1934
1935	if (pdev->irq < 2) {
1936	err = -EIO;
1937	pr_err("invalid irq (%d)\n", pdev->irq);
1938	goto err_out_iomap;
1939	}
1940	pci_set_drvdata(pdev, data: nic);
1941
1942	if (pdev->device == 0x3014)
1943	nic->port_num = 2;
1944	else
1945	nic->port_num = 1;
1946
1947	print_hw_id(pdev);
1948
1949	bdx_hw_reset_direct(regs: nic->regs);
1950
1951	nic->irq_type = IRQ_INTX;
1952	#ifdef BDX_MSI
1953	if ((readl(nic->regs + FPGA_VER) & 0xFFF) >= 378) {
1954	err = pci_enable_msi(pdev);
1955	if (err)
1956	pr_err("Can't enable msi. error is %d\n", err);
1957	else
1958	nic->irq_type = IRQ_MSI;
1959	} else
1960	DBG("HW does not support MSI\n");
1961	#endif
1962
1963	/************** netdev **************/
1964	for (port = 0; port < nic->port_num; port++) {
1965	ndev = alloc_etherdev(sizeof(struct bdx_priv));
1966	if (!ndev) {
1967	err = -ENOMEM;
1968	goto err_out_iomap;
1969	}
1970
1971	ndev->netdev_ops = &bdx_netdev_ops;
1972	ndev->tx_queue_len = BDX_NDEV_TXQ_LEN;
1973
1974	bdx_set_ethtool_ops(netdev: ndev); /* ethtool interface */
1975
1976	/* these fields are used for info purposes only
1977	* so we can have them same for all ports of the board */
1978	ndev->if_port = port;
1979	ndev->features = NETIF_F_IP_CSUM | NETIF_F_SG | NETIF_F_TSO |
1980	NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1981	NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_RXCSUM |
1982	NETIF_F_HIGHDMA;
1983
1984	ndev->hw_features = NETIF_F_IP_CSUM | NETIF_F_SG |
1985	NETIF_F_TSO | NETIF_F_HW_VLAN_CTAG_TX;
1986
1987	/************** priv ****************/
1988	priv = nic->priv[port] = netdev_priv(dev: ndev);
1989
1990	priv->pBdxRegs = nic->regs + port * 0x8000;
1991	priv->port = port;
1992	priv->pdev = pdev;
1993	priv->ndev = ndev;
1994	priv->nic = nic;
1995	priv->msg_enable = BDX_DEF_MSG_ENABLE;
1996
1997	netif_napi_add(dev: ndev, napi: &priv->napi, poll: bdx_poll);
1998
1999	if ((readl(addr: nic->regs + FPGA_VER) & 0xFFF) == 308) {
2000	DBG("HW statistics not supported\n");
2001	priv->stats_flag = 0;
2002	} else {
2003	priv->stats_flag = 1;
2004	}
2005
2006	/* Initialize fifo sizes. */
2007	priv->txd_size = 2;
2008	priv->txf_size = 2;
2009	priv->rxd_size = 2;
2010	priv->rxf_size = 3;
2011
2012	/* Initialize the initial coalescing registers. */
2013	priv->rdintcm = INT_REG_VAL(0x20, 1, 4, 12);
2014	priv->tdintcm = INT_REG_VAL(0x20, 1, 0, 12);
2015
2016	/* ndev->xmit_lock spinlock is not used.
2017	* Private priv->tx_lock is used for synchronization
2018	* between transmit and TX irq cleanup. In addition
2019	* set multicast list callback has to use priv->tx_lock.
2020	*/
2021	#ifdef BDX_LLTX
2022	ndev->features |= NETIF_F_LLTX;
2023	#endif
2024	/* MTU range: 60 - 16384 */
2025	ndev->min_mtu = ETH_ZLEN;
2026	ndev->max_mtu = BDX_MAX_MTU;
2027
2028	spin_lock_init(&priv->tx_lock);
2029
2030	/*bdx_hw_reset(priv); */
2031	if (bdx_read_mac(priv)) {
2032	pr_err("load MAC address failed\n");
2033	err = -EFAULT;
2034	goto err_out_iomap;
2035	}
2036	SET_NETDEV_DEV(ndev, &pdev->dev);
2037	err = register_netdev(dev: ndev);
2038	if (err) {
2039	pr_err("register_netdev failed\n");
2040	goto err_out_free;
2041	}
2042	netif_carrier_off(dev: ndev);
2043	netif_stop_queue(dev: ndev);
2044
2045	print_eth_id(ndev);
2046	}
2047	RET(0);
2048
2049	err_out_free:
2050	free_netdev(dev: ndev);
2051	err_out_iomap:
2052	iounmap(addr: nic->regs);
2053	err_out_res:
2054	pci_release_regions(pdev);
2055	err_dma:
2056	pci_disable_device(dev: pdev);
2057	err_pci:
2058	vfree(addr: nic);
2059
2060	RET(err);
2061	}
2062
2063	/****************** Ethtool interface *********************/
2064	/* get strings for statistics counters */
2065	static const char
2066	bdx_stat_names[][ETH_GSTRING_LEN] = {
2067	"InUCast", /* 0x7200 */
2068	"InMCast", /* 0x7210 */
2069	"InBCast", /* 0x7220 */
2070	"InPkts", /* 0x7230 */
2071	"InErrors", /* 0x7240 */
2072	"InDropped", /* 0x7250 */
2073	"FrameTooLong", /* 0x7260 */
2074	"FrameSequenceErrors", /* 0x7270 */
2075	"InVLAN", /* 0x7280 */
2076	"InDroppedDFE", /* 0x7290 */
2077	"InDroppedIntFull", /* 0x72A0 */
2078	"InFrameAlignErrors", /* 0x72B0 */
2079
2080	/* 0x72C0-0x72E0 RSRV */
2081
2082	"OutUCast", /* 0x72F0 */
2083	"OutMCast", /* 0x7300 */
2084	"OutBCast", /* 0x7310 */
2085	"OutPkts", /* 0x7320 */
2086
2087	/* 0x7330-0x7360 RSRV */
2088
2089	"OutVLAN", /* 0x7370 */
2090	"InUCastOctects", /* 0x7380 */
2091	"OutUCastOctects", /* 0x7390 */
2092
2093	/* 0x73A0-0x73B0 RSRV */
2094
2095	"InBCastOctects", /* 0x73C0 */
2096	"OutBCastOctects", /* 0x73D0 */
2097	"InOctects", /* 0x73E0 */
2098	"OutOctects", /* 0x73F0 */
2099	};
2100
2101	/*
2102	* bdx_get_link_ksettings - get device-specific settings
2103	* @netdev
2104	* @ecmd
2105	*/
2106	static int bdx_get_link_ksettings(struct net_device *netdev,
2107	struct ethtool_link_ksettings *ecmd)
2108	{
2109	ethtool_link_ksettings_zero_link_mode(ecmd, supported);
2110	ethtool_link_ksettings_add_link_mode(ecmd, supported,
2111	10000baseT_Full);
2112	ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE);
2113	ethtool_link_ksettings_zero_link_mode(ecmd, advertising);
2114	ethtool_link_ksettings_add_link_mode(ecmd, advertising,
2115	10000baseT_Full);
2116	ethtool_link_ksettings_add_link_mode(ecmd, advertising, FIBRE);
2117
2118	ecmd->base.speed = SPEED_10000;
2119	ecmd->base.duplex = DUPLEX_FULL;
2120	ecmd->base.port = PORT_FIBRE;
2121	ecmd->base.autoneg = AUTONEG_DISABLE;
2122
2123	return 0;
2124	}
2125
2126	/*
2127	* bdx_get_drvinfo - report driver information
2128	* @netdev
2129	* @drvinfo
2130	*/
2131	static void
2132	bdx_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
2133	{
2134	struct bdx_priv *priv = netdev_priv(dev: netdev);
2135
2136	strscpy(p: drvinfo->driver, BDX_DRV_NAME, size: sizeof(drvinfo->driver));
2137	strscpy(p: drvinfo->version, BDX_DRV_VERSION, size: sizeof(drvinfo->version));
2138	strscpy(p: drvinfo->fw_version, q: "N/A", size: sizeof(drvinfo->fw_version));
2139	strscpy(p: drvinfo->bus_info, q: pci_name(pdev: priv->pdev),
2140	size: sizeof(drvinfo->bus_info));
2141	}
2142
2143	/*
2144	* bdx_get_coalesce - get interrupt coalescing parameters
2145	* @netdev
2146	* @ecoal
2147	*/
2148	static int bdx_get_coalesce(struct net_device *netdev,
2149	struct ethtool_coalesce *ecoal,
2150	struct kernel_ethtool_coalesce *kernel_coal,
2151	struct netlink_ext_ack *extack)
2152	{
2153	u32 rdintcm;
2154	u32 tdintcm;
2155	struct bdx_priv *priv = netdev_priv(dev: netdev);
2156
2157	rdintcm = priv->rdintcm;
2158	tdintcm = priv->tdintcm;
2159
2160	/* PCK_TH measures in multiples of FIFO bytes
2161	We translate to packets */
2162	ecoal->rx_coalesce_usecs = GET_INT_COAL(rdintcm) * INT_COAL_MULT;
2163	ecoal->rx_max_coalesced_frames =
2164	((GET_PCK_TH(rdintcm) * PCK_TH_MULT) / sizeof(struct rxf_desc));
2165
2166	ecoal->tx_coalesce_usecs = GET_INT_COAL(tdintcm) * INT_COAL_MULT;
2167	ecoal->tx_max_coalesced_frames =
2168	((GET_PCK_TH(tdintcm) * PCK_TH_MULT) / BDX_TXF_DESC_SZ);
2169
2170	/* adaptive parameters ignored */
2171	return 0;
2172	}
2173
2174	/*
2175	* bdx_set_coalesce - set interrupt coalescing parameters
2176	* @netdev
2177	* @ecoal
2178	*/
2179	static int bdx_set_coalesce(struct net_device *netdev,
2180	struct ethtool_coalesce *ecoal,
2181	struct kernel_ethtool_coalesce *kernel_coal,
2182	struct netlink_ext_ack *extack)
2183	{
2184	u32 rdintcm;
2185	u32 tdintcm;
2186	struct bdx_priv *priv = netdev_priv(dev: netdev);
2187	int rx_coal;
2188	int tx_coal;
2189	int rx_max_coal;
2190	int tx_max_coal;
2191
2192	/* Check for valid input */
2193	rx_coal = ecoal->rx_coalesce_usecs / INT_COAL_MULT;
2194	tx_coal = ecoal->tx_coalesce_usecs / INT_COAL_MULT;
2195	rx_max_coal = ecoal->rx_max_coalesced_frames;
2196	tx_max_coal = ecoal->tx_max_coalesced_frames;
2197
2198	/* Translate from packets to multiples of FIFO bytes */
2199	rx_max_coal =
2200	(((rx_max_coal * sizeof(struct rxf_desc)) + PCK_TH_MULT - 1)
2201	/ PCK_TH_MULT);
2202	tx_max_coal =
2203	(((tx_max_coal * BDX_TXF_DESC_SZ) + PCK_TH_MULT - 1)
2204	/ PCK_TH_MULT);
2205
2206	if ((rx_coal > 0x7FFF) || (tx_coal > 0x7FFF) ||
2207	(rx_max_coal > 0xF) || (tx_max_coal > 0xF))
2208	return -EINVAL;
2209
2210	rdintcm = INT_REG_VAL(rx_coal, GET_INT_COAL_RC(priv->rdintcm),
2211	GET_RXF_TH(priv->rdintcm), rx_max_coal);
2212	tdintcm = INT_REG_VAL(tx_coal, GET_INT_COAL_RC(priv->tdintcm), 0,
2213	tx_max_coal);
2214
2215	priv->rdintcm = rdintcm;
2216	priv->tdintcm = tdintcm;
2217
2218	WRITE_REG(priv, regRDINTCM0, rdintcm);
2219	WRITE_REG(priv, regTDINTCM0, tdintcm);
2220
2221	return 0;
2222	}
2223
2224	/* Convert RX fifo size to number of pending packets */
2225	static inline int bdx_rx_fifo_size_to_packets(int rx_size)
2226	{
2227	return (FIFO_SIZE * (1 << rx_size)) / sizeof(struct rxf_desc);
2228	}
2229
2230	/* Convert TX fifo size to number of pending packets */
2231	static inline int bdx_tx_fifo_size_to_packets(int tx_size)
2232	{
2233	return (FIFO_SIZE * (1 << tx_size)) / BDX_TXF_DESC_SZ;
2234	}
2235
2236	/*
2237	* bdx_get_ringparam - report ring sizes
2238	* @netdev
2239	* @ring
2240	* @kernel_ring
2241	* @extack
2242	*/
2243	static void
2244	bdx_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring,
2245	struct kernel_ethtool_ringparam *kernel_ring,
2246	struct netlink_ext_ack *extack)
2247	{
2248	struct bdx_priv *priv = netdev_priv(dev: netdev);
2249
2250	/*max_pending - the maximum-sized FIFO we allow */
2251	ring->rx_max_pending = bdx_rx_fifo_size_to_packets(rx_size: 3);
2252	ring->tx_max_pending = bdx_tx_fifo_size_to_packets(tx_size: 3);
2253	ring->rx_pending = bdx_rx_fifo_size_to_packets(rx_size: priv->rxf_size);
2254	ring->tx_pending = bdx_tx_fifo_size_to_packets(tx_size: priv->txd_size);
2255	}
2256
2257	/*
2258	* bdx_set_ringparam - set ring sizes
2259	* @netdev
2260	* @ring
2261	* @kernel_ring
2262	* @extack
2263	*/
2264	static int
2265	bdx_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring,
2266	struct kernel_ethtool_ringparam *kernel_ring,
2267	struct netlink_ext_ack *extack)
2268	{
2269	struct bdx_priv *priv = netdev_priv(dev: netdev);
2270	int rx_size = 0;
2271	int tx_size = 0;
2272
2273	for (; rx_size < 4; rx_size++) {
2274	if (bdx_rx_fifo_size_to_packets(rx_size) >= ring->rx_pending)
2275	break;
2276	}
2277	if (rx_size == 4)
2278	rx_size = 3;
2279
2280	for (; tx_size < 4; tx_size++) {
2281	if (bdx_tx_fifo_size_to_packets(tx_size) >= ring->tx_pending)
2282	break;
2283	}
2284	if (tx_size == 4)
2285	tx_size = 3;
2286
2287	/*Is there anything to do? */
2288	if ((rx_size == priv->rxf_size) &&
2289	(tx_size == priv->txd_size))
2290	return 0;
2291
2292	priv->rxf_size = rx_size;
2293	if (rx_size > 1)
2294	priv->rxd_size = rx_size - 1;
2295	else
2296	priv->rxd_size = rx_size;
2297
2298	priv->txf_size = priv->txd_size = tx_size;
2299
2300	if (netif_running(dev: netdev)) {
2301	bdx_close(ndev: netdev);
2302	bdx_open(ndev: netdev);
2303	}
2304	return 0;
2305	}
2306
2307	/*
2308	* bdx_get_strings - return a set of strings that describe the requested objects
2309	* @netdev
2310	* @data
2311	*/
2312	static void bdx_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
2313	{
2314	switch (stringset) {
2315	case ETH_SS_STATS:
2316	memcpy(data, *bdx_stat_names, sizeof(bdx_stat_names));
2317	break;
2318	}
2319	}
2320
2321	/*
2322	* bdx_get_sset_count - return number of statistics or tests
2323	* @netdev
2324	*/
2325	static int bdx_get_sset_count(struct net_device *netdev, int stringset)
2326	{
2327	struct bdx_priv *priv = netdev_priv(dev: netdev);
2328
2329	switch (stringset) {
2330	case ETH_SS_STATS:
2331	BDX_ASSERT(ARRAY_SIZE(bdx_stat_names)
2332	!= sizeof(struct bdx_stats) / sizeof(u64));
2333	return (priv->stats_flag) ? ARRAY_SIZE(bdx_stat_names) : 0;
2334	}
2335
2336	return -EINVAL;
2337	}
2338
2339	/*
2340	* bdx_get_ethtool_stats - return device's hardware L2 statistics
2341	* @netdev
2342	* @stats
2343	* @data
2344	*/
2345	static void bdx_get_ethtool_stats(struct net_device *netdev,
2346	struct ethtool_stats *stats, u64 *data)
2347	{
2348	struct bdx_priv *priv = netdev_priv(dev: netdev);
2349
2350	if (priv->stats_flag) {
2351
2352	/* Update stats from HW */
2353	bdx_update_stats(priv);
2354
2355	/* Copy data to user buffer */
2356	memcpy(data, &priv->hw_stats, sizeof(priv->hw_stats));
2357	}
2358	}
2359
2360	/*
2361	* bdx_set_ethtool_ops - ethtool interface implementation
2362	* @netdev
2363	*/
2364	static void bdx_set_ethtool_ops(struct net_device *netdev)
2365	{
2366	static const struct ethtool_ops bdx_ethtool_ops = {
2367	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
2368	ETHTOOL_COALESCE_MAX_FRAMES,
2369	.get_drvinfo = bdx_get_drvinfo,
2370	.get_link = ethtool_op_get_link,
2371	.get_coalesce = bdx_get_coalesce,
2372	.set_coalesce = bdx_set_coalesce,
2373	.get_ringparam = bdx_get_ringparam,
2374	.set_ringparam = bdx_set_ringparam,
2375	.get_strings = bdx_get_strings,
2376	.get_sset_count = bdx_get_sset_count,
2377	.get_ethtool_stats = bdx_get_ethtool_stats,
2378	.get_link_ksettings = bdx_get_link_ksettings,
2379	};
2380
2381	netdev->ethtool_ops = &bdx_ethtool_ops;
2382	}
2383
2384	/**
2385	* bdx_remove - Device Removal Routine
2386	* @pdev: PCI device information struct
2387	*
2388	* bdx_remove is called by the PCI subsystem to alert the driver
2389	* that it should release a PCI device. The could be caused by a
2390	* Hot-Plug event, or because the driver is going to be removed from
2391	* memory.
2392	**/
2393	static void bdx_remove(struct pci_dev *pdev)
2394	{
2395	struct pci_nic *nic = pci_get_drvdata(pdev);
2396	struct net_device *ndev;
2397	int port;
2398
2399	for (port = 0; port < nic->port_num; port++) {
2400	ndev = nic->priv[port]->ndev;
2401	unregister_netdev(dev: ndev);
2402	free_netdev(dev: ndev);
2403	}
2404
2405	/*bdx_hw_reset_direct(nic->regs); */
2406	#ifdef BDX_MSI
2407	if (nic->irq_type == IRQ_MSI)
2408	pci_disable_msi(pdev);
2409	#endif
2410
2411	iounmap(addr: nic->regs);
2412	pci_release_regions(pdev);
2413	pci_disable_device(dev: pdev);
2414	vfree(addr: nic);
2415
2416	RET();
2417	}
2418
2419	static struct pci_driver bdx_pci_driver = {
2420	.name = BDX_DRV_NAME,
2421	.id_table = bdx_pci_tbl,
2422	.probe = bdx_probe,
2423	.remove = bdx_remove,
2424	};
2425
2426	/*
2427	* print_driver_id - print parameters of the driver build
2428	*/
2429	static void __init print_driver_id(void)
2430	{
2431	pr_info("%s, %s\n", BDX_DRV_DESC, BDX_DRV_VERSION);
2432	pr_info("Options: hw_csum %s\n", BDX_MSI_STRING);
2433	}
2434
2435	static int __init bdx_module_init(void)
2436	{
2437	ENTER;
2438	init_txd_sizes();
2439	print_driver_id();
2440	RET(pci_register_driver(&bdx_pci_driver));
2441	}
2442
2443	module_init(bdx_module_init);
2444
2445	static void __exit bdx_module_exit(void)
2446	{
2447	ENTER;
2448	pci_unregister_driver(dev: &bdx_pci_driver);
2449	RET();
2450	}
2451
2452	module_exit(bdx_module_exit);
2453
2454	MODULE_LICENSE("GPL");
2455	MODULE_AUTHOR(DRIVER_AUTHOR);
2456	MODULE_DESCRIPTION(BDX_DRV_DESC);
2457	MODULE_FIRMWARE("tehuti/bdx.bin");
2458