1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) 2006-2007 PA Semi, Inc
4	*
5	* Driver for the PA Semi PWRficient onchip 1G/10G Ethernet MACs
6	*/
7
8	#include <linux/module.h>
9	#include <linux/pci.h>
10	#include <linux/slab.h>
11	#include <linux/interrupt.h>
12	#include <linux/dmaengine.h>
13	#include <linux/delay.h>
14	#include <linux/netdevice.h>
15	#include <linux/of_mdio.h>
16	#include <linux/etherdevice.h>
17	#include <asm/dma-mapping.h>
18	#include <linux/in.h>
19	#include <linux/skbuff.h>
20
21	#include <linux/ip.h>
22	#include <net/checksum.h>
23	#include <linux/prefetch.h>
24
25	#include <asm/irq.h>
26	#include <asm/firmware.h>
27	#include <asm/pasemi_dma.h>
28
29	#include "pasemi_mac.h"
30
31	/* We have our own align, since ppc64 in general has it at 0 because
32	* of design flaws in some of the server bridge chips. However, for
33	* PWRficient doing the unaligned copies is more expensive than doing
34	* unaligned DMA, so make sure the data is aligned instead.
35	*/
36	#define LOCAL_SKB_ALIGN 2
37
38	/* TODO list
39	*
40	* - Multicast support
41	* - Large MTU support
42	* - Multiqueue RX/TX
43	*/
44
45	#define PE_MIN_MTU (ETH_ZLEN + ETH_HLEN)
46	#define PE_MAX_MTU 9000
47	#define PE_DEF_MTU ETH_DATA_LEN
48
49	#define DEFAULT_MSG_ENABLE \
50	(NETIF_MSG_DRV | \
51	NETIF_MSG_PROBE | \
52	NETIF_MSG_LINK | \
53	NETIF_MSG_TIMER | \
54	NETIF_MSG_IFDOWN | \
55	NETIF_MSG_IFUP | \
56	NETIF_MSG_RX_ERR | \
57	NETIF_MSG_TX_ERR)
58
59	MODULE_LICENSE("GPL");
60	MODULE_AUTHOR ("Olof Johansson <olof@lixom.net>");
61	MODULE_DESCRIPTION("PA Semi PWRficient Ethernet driver");
62
63	static int debug = -1; /* -1 == use DEFAULT_MSG_ENABLE as value */
64	module_param(debug, int, 0);
65	MODULE_PARM_DESC(debug, "PA Semi MAC bitmapped debugging message enable value");
66
67	extern const struct ethtool_ops pasemi_mac_ethtool_ops;
68
69	static int translation_enabled(void)
70	{
71	#if defined(CONFIG_PPC_PASEMI_IOMMU_DMA_FORCE)
72	return 1;
73	#else
74	return firmware_has_feature(FW_FEATURE_LPAR);
75	#endif
76	}
77
78	static void write_iob_reg(unsigned int reg, unsigned int val)
79	{
80	pasemi_write_iob_reg(reg, val);
81	}
82
83	static unsigned int read_mac_reg(const struct pasemi_mac *mac, unsigned int reg)
84	{
85	return pasemi_read_mac_reg(mac->dma_if, reg);
86	}
87
88	static void write_mac_reg(const struct pasemi_mac *mac, unsigned int reg,
89	unsigned int val)
90	{
91	pasemi_write_mac_reg(mac->dma_if, reg, val);
92	}
93
94	static unsigned int read_dma_reg(unsigned int reg)
95	{
96	return pasemi_read_dma_reg(reg);
97	}
98
99	static void write_dma_reg(unsigned int reg, unsigned int val)
100	{
101	pasemi_write_dma_reg(reg, val);
102	}
103
104	static struct pasemi_mac_rxring *rx_ring(const struct pasemi_mac *mac)
105	{
106	return mac->rx;
107	}
108
109	static struct pasemi_mac_txring *tx_ring(const struct pasemi_mac *mac)
110	{
111	return mac->tx;
112	}
113
114	static inline void prefetch_skb(const struct sk_buff *skb)
115	{
116	const void *d = skb;
117
118	prefetch(d);
119	prefetch(d+64);
120	prefetch(d+128);
121	prefetch(d+192);
122	}
123
124	static int mac_to_intf(struct pasemi_mac *mac)
125	{
126	struct pci_dev *pdev = mac->pdev;
127	u32 tmp;
128	int nintf, off, i, j;
129	int devfn = pdev->devfn;
130
131	tmp = read_dma_reg(reg: PAS_DMA_CAP_IFI);
132	nintf = (tmp & PAS_DMA_CAP_IFI_NIN_M) >> PAS_DMA_CAP_IFI_NIN_S;
133	off = (tmp & PAS_DMA_CAP_IFI_IOFF_M) >> PAS_DMA_CAP_IFI_IOFF_S;
134
135	/* IOFF contains the offset to the registers containing the
136	* DMA interface-to-MAC-pci-id mappings, and NIN contains number
137	* of total interfaces. Each register contains 4 devfns.
138	* Just do a linear search until we find the devfn of the MAC
139	* we're trying to look up.
140	*/
141
142	for (i = 0; i < (nintf+3)/4; i++) {
143	tmp = read_dma_reg(reg: off+4*i);
144	for (j = 0; j < 4; j++) {
145	if (((tmp >> (8*j)) & 0xff) == devfn)
146	return i*4 + j;
147	}
148	}
149	return -1;
150	}
151
152	static void pasemi_mac_intf_disable(struct pasemi_mac *mac)
153	{
154	unsigned int flags;
155
156	flags = read_mac_reg(mac, reg: PAS_MAC_CFG_PCFG);
157	flags &= ~PAS_MAC_CFG_PCFG_PE;
158	write_mac_reg(mac, reg: PAS_MAC_CFG_PCFG, val: flags);
159	}
160
161	static void pasemi_mac_intf_enable(struct pasemi_mac *mac)
162	{
163	unsigned int flags;
164
165	flags = read_mac_reg(mac, reg: PAS_MAC_CFG_PCFG);
166	flags |= PAS_MAC_CFG_PCFG_PE;
167	write_mac_reg(mac, reg: PAS_MAC_CFG_PCFG, val: flags);
168	}
169
170	static int pasemi_get_mac_addr(struct pasemi_mac *mac)
171	{
172	struct pci_dev *pdev = mac->pdev;
173	struct device_node *dn = pci_device_to_OF_node(pdev);
174	int len;
175	const u8 *maddr;
176	u8 addr[ETH_ALEN];
177
178	if (!dn) {
179	dev_dbg(&pdev->dev,
180	"No device node for mac, not configuring\n");
181	return -ENOENT;
182	}
183
184	maddr = of_get_property(node: dn, name: "local-mac-address", lenp: &len);
185
186	if (maddr && len == ETH_ALEN) {
187	memcpy(mac->mac_addr, maddr, ETH_ALEN);
188	return 0;
189	}
190
191	/* Some old versions of firmware mistakenly uses mac-address
192	* (and as a string) instead of a byte array in local-mac-address.
193	*/
194
195	if (maddr == NULL)
196	maddr = of_get_property(node: dn, name: "mac-address", NULL);
197
198	if (maddr == NULL) {
199	dev_warn(&pdev->dev,
200	"no mac address in device tree, not configuring\n");
201	return -ENOENT;
202	}
203
204	if (!mac_pton(s: maddr, mac: addr)) {
205	dev_warn(&pdev->dev,
206	"can't parse mac address, not configuring\n");
207	return -EINVAL;
208	}
209
210	memcpy(mac->mac_addr, addr, ETH_ALEN);
211
212	return 0;
213	}
214
215	static int pasemi_mac_set_mac_addr(struct net_device *dev, void *p)
216	{
217	struct pasemi_mac *mac = netdev_priv(dev);
218	struct sockaddr *addr = p;
219	unsigned int adr0, adr1;
220
221	if (!is_valid_ether_addr(addr: addr->sa_data))
222	return -EADDRNOTAVAIL;
223
224	eth_hw_addr_set(dev, addr: addr->sa_data);
225
226	adr0 = dev->dev_addr[2] << 24 |
227	dev->dev_addr[3] << 16 |
228	dev->dev_addr[4] << 8 |
229	dev->dev_addr[5];
230	adr1 = read_mac_reg(mac, reg: PAS_MAC_CFG_ADR1);
231	adr1 &= ~0xffff;
232	adr1 |= dev->dev_addr[0] << 8 | dev->dev_addr[1];
233
234	pasemi_mac_intf_disable(mac);
235	write_mac_reg(mac, reg: PAS_MAC_CFG_ADR0, val: adr0);
236	write_mac_reg(mac, reg: PAS_MAC_CFG_ADR1, val: adr1);
237	pasemi_mac_intf_enable(mac);
238
239	return 0;
240	}
241
242	static int pasemi_mac_unmap_tx_skb(struct pasemi_mac *mac,
243	const int nfrags,
244	struct sk_buff *skb,
245	const dma_addr_t *dmas)
246	{
247	int f;
248	struct pci_dev *pdev = mac->dma_pdev;
249
250	dma_unmap_single(&pdev->dev, dmas[0], skb_headlen(skb), DMA_TO_DEVICE);
251
252	for (f = 0; f < nfrags; f++) {
253	const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
254
255	dma_unmap_page(&pdev->dev, dmas[f + 1], skb_frag_size(frag),
256	DMA_TO_DEVICE);
257	}
258	dev_kfree_skb_irq(skb);
259
260	/* Freed descriptor slot + main SKB ptr + nfrags additional ptrs,
261	* aligned up to a power of 2
262	*/
263	return (nfrags + 3) & ~1;
264	}
265
266	static struct pasemi_mac_csring *pasemi_mac_setup_csring(struct pasemi_mac *mac)
267	{
268	struct pasemi_mac_csring *ring;
269	u32 val;
270	unsigned int cfg;
271	int chno;
272
273	ring = pasemi_dma_alloc_chan(TXCHAN, sizeof(struct pasemi_mac_csring),
274	offsetof(struct pasemi_mac_csring, chan));
275
276	if (!ring) {
277	dev_err(&mac->pdev->dev, "Can't allocate checksum channel\n");
278	goto out_chan;
279	}
280
281	chno = ring->chan.chno;
282
283	ring->size = CS_RING_SIZE;
284	ring->next_to_fill = 0;
285
286	/* Allocate descriptors */
287	if (pasemi_dma_alloc_ring(&ring->chan, CS_RING_SIZE))
288	goto out_ring_desc;
289
290	write_dma_reg(reg: PAS_DMA_TXCHAN_BASEL(chno),
291	val: PAS_DMA_TXCHAN_BASEL_BRBL(ring->chan.ring_dma));
292	val = PAS_DMA_TXCHAN_BASEU_BRBH(ring->chan.ring_dma >> 32);
293	val |= PAS_DMA_TXCHAN_BASEU_SIZ(CS_RING_SIZE >> 3);
294
295	write_dma_reg(reg: PAS_DMA_TXCHAN_BASEU(chno), val);
296
297	ring->events[0] = pasemi_dma_alloc_flag();
298	ring->events[1] = pasemi_dma_alloc_flag();
299	if (ring->events[0] < 0 || ring->events[1] < 0)
300	goto out_flags;
301
302	pasemi_dma_clear_flag(ring->events[0]);
303	pasemi_dma_clear_flag(ring->events[1]);
304
305	ring->fun = pasemi_dma_alloc_fun();
306	if (ring->fun < 0)
307	goto out_fun;
308
309	cfg = PAS_DMA_TXCHAN_CFG_TY_FUNC | PAS_DMA_TXCHAN_CFG_UP |
310	PAS_DMA_TXCHAN_CFG_TATTR(ring->fun) |
311	PAS_DMA_TXCHAN_CFG_LPSQ | PAS_DMA_TXCHAN_CFG_LPDQ;
312
313	if (translation_enabled())
314	cfg |= PAS_DMA_TXCHAN_CFG_TRD | PAS_DMA_TXCHAN_CFG_TRR;
315
316	write_dma_reg(reg: PAS_DMA_TXCHAN_CFG(chno), val: cfg);
317
318	/* enable channel */
319	pasemi_dma_start_chan(&ring->chan, PAS_DMA_TXCHAN_TCMDSTA_SZ |
320	PAS_DMA_TXCHAN_TCMDSTA_DB |
321	PAS_DMA_TXCHAN_TCMDSTA_DE |
322	PAS_DMA_TXCHAN_TCMDSTA_DA);
323
324	return ring;
325
326	out_fun:
327	out_flags:
328	if (ring->events[0] >= 0)
329	pasemi_dma_free_flag(ring->events[0]);
330	if (ring->events[1] >= 0)
331	pasemi_dma_free_flag(ring->events[1]);
332	pasemi_dma_free_ring(&ring->chan);
333	out_ring_desc:
334	pasemi_dma_free_chan(&ring->chan);
335	out_chan:
336
337	return NULL;
338	}
339
340	static void pasemi_mac_setup_csrings(struct pasemi_mac *mac)
341	{
342	int i;
343	mac->cs[0] = pasemi_mac_setup_csring(mac);
344	if (mac->type == MAC_TYPE_XAUI)
345	mac->cs[1] = pasemi_mac_setup_csring(mac);
346	else
347	mac->cs[1] = 0;
348
349	for (i = 0; i < MAX_CS; i++)
350	if (mac->cs[i])
351	mac->num_cs++;
352	}
353
354	static void pasemi_mac_free_csring(struct pasemi_mac_csring *csring)
355	{
356	pasemi_dma_stop_chan(&csring->chan);
357	pasemi_dma_free_flag(csring->events[0]);
358	pasemi_dma_free_flag(csring->events[1]);
359	pasemi_dma_free_ring(&csring->chan);
360	pasemi_dma_free_chan(&csring->chan);
361	pasemi_dma_free_fun(csring->fun);
362	}
363
364	static int pasemi_mac_setup_rx_resources(const struct net_device *dev)
365	{
366	struct pasemi_mac_rxring *ring;
367	struct pasemi_mac *mac = netdev_priv(dev);
368	int chno;
369	unsigned int cfg;
370
371	ring = pasemi_dma_alloc_chan(RXCHAN, sizeof(struct pasemi_mac_rxring),
372	offsetof(struct pasemi_mac_rxring, chan));
373
374	if (!ring) {
375	dev_err(&mac->pdev->dev, "Can't allocate RX channel\n");
376	goto out_chan;
377	}
378	chno = ring->chan.chno;
379
380	spin_lock_init(&ring->lock);
381
382	ring->size = RX_RING_SIZE;
383	ring->ring_info = kcalloc(RX_RING_SIZE,
384	size: sizeof(struct pasemi_mac_buffer),
385	GFP_KERNEL);
386
387	if (!ring->ring_info)
388	goto out_ring_info;
389
390	/* Allocate descriptors */
391	if (pasemi_dma_alloc_ring(&ring->chan, RX_RING_SIZE))
392	goto out_ring_desc;
393
394	ring->buffers = dma_alloc_coherent(dev: &mac->dma_pdev->dev,
395	RX_RING_SIZE * sizeof(u64),
396	dma_handle: &ring->buf_dma, GFP_KERNEL);
397	if (!ring->buffers)
398	goto out_ring_desc;
399
400	write_dma_reg(reg: PAS_DMA_RXCHAN_BASEL(chno),
401	val: PAS_DMA_RXCHAN_BASEL_BRBL(ring->chan.ring_dma));
402
403	write_dma_reg(reg: PAS_DMA_RXCHAN_BASEU(chno),
404	val: PAS_DMA_RXCHAN_BASEU_BRBH(ring->chan.ring_dma >> 32) |
405	PAS_DMA_RXCHAN_BASEU_SIZ(RX_RING_SIZE >> 3));
406
407	cfg = PAS_DMA_RXCHAN_CFG_HBU(2);
408
409	if (translation_enabled())
410	cfg |= PAS_DMA_RXCHAN_CFG_CTR;
411
412	write_dma_reg(reg: PAS_DMA_RXCHAN_CFG(chno), val: cfg);
413
414	write_dma_reg(reg: PAS_DMA_RXINT_BASEL(mac->dma_if),
415	val: PAS_DMA_RXINT_BASEL_BRBL(ring->buf_dma));
416
417	write_dma_reg(reg: PAS_DMA_RXINT_BASEU(mac->dma_if),
418	val: PAS_DMA_RXINT_BASEU_BRBH(ring->buf_dma >> 32) |
419	PAS_DMA_RXINT_BASEU_SIZ(RX_RING_SIZE >> 3));
420
421	cfg = PAS_DMA_RXINT_CFG_DHL(2) | PAS_DMA_RXINT_CFG_L2 |
422	PAS_DMA_RXINT_CFG_LW | PAS_DMA_RXINT_CFG_RBP |
423	PAS_DMA_RXINT_CFG_HEN;
424
425	if (translation_enabled())
426	cfg |= PAS_DMA_RXINT_CFG_ITRR | PAS_DMA_RXINT_CFG_ITR;
427
428	write_dma_reg(reg: PAS_DMA_RXINT_CFG(mac->dma_if), val: cfg);
429
430	ring->next_to_fill = 0;
431	ring->next_to_clean = 0;
432	ring->mac = mac;
433	mac->rx = ring;
434
435	return 0;
436
437	out_ring_desc:
438	kfree(objp: ring->ring_info);
439	out_ring_info:
440	pasemi_dma_free_chan(&ring->chan);
441	out_chan:
442	return -ENOMEM;
443	}
444
445	static struct pasemi_mac_txring *
446	pasemi_mac_setup_tx_resources(const struct net_device *dev)
447	{
448	struct pasemi_mac *mac = netdev_priv(dev);
449	u32 val;
450	struct pasemi_mac_txring *ring;
451	unsigned int cfg;
452	int chno;
453
454	ring = pasemi_dma_alloc_chan(TXCHAN, sizeof(struct pasemi_mac_txring),
455	offsetof(struct pasemi_mac_txring, chan));
456
457	if (!ring) {
458	dev_err(&mac->pdev->dev, "Can't allocate TX channel\n");
459	goto out_chan;
460	}
461
462	chno = ring->chan.chno;
463
464	spin_lock_init(&ring->lock);
465
466	ring->size = TX_RING_SIZE;
467	ring->ring_info = kcalloc(TX_RING_SIZE,
468	size: sizeof(struct pasemi_mac_buffer),
469	GFP_KERNEL);
470	if (!ring->ring_info)
471	goto out_ring_info;
472
473	/* Allocate descriptors */
474	if (pasemi_dma_alloc_ring(&ring->chan, TX_RING_SIZE))
475	goto out_ring_desc;
476
477	write_dma_reg(reg: PAS_DMA_TXCHAN_BASEL(chno),
478	val: PAS_DMA_TXCHAN_BASEL_BRBL(ring->chan.ring_dma));
479	val = PAS_DMA_TXCHAN_BASEU_BRBH(ring->chan.ring_dma >> 32);
480	val |= PAS_DMA_TXCHAN_BASEU_SIZ(TX_RING_SIZE >> 3);
481
482	write_dma_reg(reg: PAS_DMA_TXCHAN_BASEU(chno), val);
483
484	cfg = PAS_DMA_TXCHAN_CFG_TY_IFACE |
485	PAS_DMA_TXCHAN_CFG_TATTR(mac->dma_if) |
486	PAS_DMA_TXCHAN_CFG_UP |
487	PAS_DMA_TXCHAN_CFG_WT(4);
488
489	if (translation_enabled())
490	cfg |= PAS_DMA_TXCHAN_CFG_TRD | PAS_DMA_TXCHAN_CFG_TRR;
491
492	write_dma_reg(reg: PAS_DMA_TXCHAN_CFG(chno), val: cfg);
493
494	ring->next_to_fill = 0;
495	ring->next_to_clean = 0;
496	ring->mac = mac;
497
498	return ring;
499
500	out_ring_desc:
501	kfree(objp: ring->ring_info);
502	out_ring_info:
503	pasemi_dma_free_chan(&ring->chan);
504	out_chan:
505	return NULL;
506	}
507
508	static void pasemi_mac_free_tx_resources(struct pasemi_mac *mac)
509	{
510	struct pasemi_mac_txring *txring = tx_ring(mac);
511	unsigned int i, j;
512	struct pasemi_mac_buffer *info;
513	dma_addr_t dmas[MAX_SKB_FRAGS+1];
514	int freed, nfrags;
515	int start, limit;
516
517	start = txring->next_to_clean;
518	limit = txring->next_to_fill;
519
520	/* Compensate for when fill has wrapped and clean has not */
521	if (start > limit)
522	limit += TX_RING_SIZE;
523
524	for (i = start; i < limit; i += freed) {
525	info = &txring->ring_info[(i+1) & (TX_RING_SIZE-1)];
526	if (info->dma && info->skb) {
527	nfrags = skb_shinfo(info->skb)->nr_frags;
528	for (j = 0; j <= nfrags; j++)
529	dmas[j] = txring->ring_info[(i+1+j) &
530	(TX_RING_SIZE-1)].dma;
531	freed = pasemi_mac_unmap_tx_skb(mac, nfrags,
532	skb: info->skb, dmas);
533	} else {
534	freed = 2;
535	}
536	}
537
538	kfree(objp: txring->ring_info);
539	pasemi_dma_free_chan(&txring->chan);
540
541	}
542
543	static void pasemi_mac_free_rx_buffers(struct pasemi_mac *mac)
544	{
545	struct pasemi_mac_rxring *rx = rx_ring(mac);
546	unsigned int i;
547	struct pasemi_mac_buffer *info;
548
549	for (i = 0; i < RX_RING_SIZE; i++) {
550	info = &RX_DESC_INFO(rx, i);
551	if (info->skb && info->dma) {
552	dma_unmap_single(&mac->dma_pdev->dev, info->dma,
553	info->skb->len, DMA_FROM_DEVICE);
554	dev_kfree_skb_any(skb: info->skb);
555	}
556	info->dma = 0;
557	info->skb = NULL;
558	}
559
560	for (i = 0; i < RX_RING_SIZE; i++)
561	RX_BUFF(rx, i) = 0;
562	}
563
564	static void pasemi_mac_free_rx_resources(struct pasemi_mac *mac)
565	{
566	pasemi_mac_free_rx_buffers(mac);
567
568	dma_free_coherent(dev: &mac->dma_pdev->dev, RX_RING_SIZE * sizeof(u64),
569	cpu_addr: rx_ring(mac)->buffers, dma_handle: rx_ring(mac)->buf_dma);
570
571	kfree(objp: rx_ring(mac)->ring_info);
572	pasemi_dma_free_chan(&rx_ring(mac)->chan);
573	mac->rx = NULL;
574	}
575
576	static void pasemi_mac_replenish_rx_ring(struct net_device *dev,
577	const int limit)
578	{
579	const struct pasemi_mac *mac = netdev_priv(dev);
580	struct pasemi_mac_rxring *rx = rx_ring(mac);
581	int fill, count;
582
583	if (limit <= 0)
584	return;
585
586	fill = rx_ring(mac)->next_to_fill;
587	for (count = 0; count < limit; count++) {
588	struct pasemi_mac_buffer *info = &RX_DESC_INFO(rx, fill);
589	u64 *buff = &RX_BUFF(rx, fill);
590	struct sk_buff *skb;
591	dma_addr_t dma;
592
593	/* Entry in use? */
594	WARN_ON(*buff);
595
596	skb = netdev_alloc_skb(dev, length: mac->bufsz);
597	skb_reserve(skb, LOCAL_SKB_ALIGN);
598
599	if (unlikely(!skb))
600	break;
601
602	dma = dma_map_single(&mac->dma_pdev->dev, skb->data,
603	mac->bufsz - LOCAL_SKB_ALIGN,
604	DMA_FROM_DEVICE);
605
606	if (dma_mapping_error(dev: &mac->dma_pdev->dev, dma_addr: dma)) {
607	dev_kfree_skb_irq(skb: info->skb);
608	break;
609	}
610
611	info->skb = skb;
612	info->dma = dma;
613	*buff = XCT_RXB_LEN(mac->bufsz) | XCT_RXB_ADDR(dma);
614	fill++;
615	}
616
617	wmb();
618
619	write_dma_reg(reg: PAS_DMA_RXINT_INCR(mac->dma_if), val: count);
620
621	rx_ring(mac)->next_to_fill = (rx_ring(mac)->next_to_fill + count) &
622	(RX_RING_SIZE - 1);
623	}
624
625	static void pasemi_mac_restart_rx_intr(const struct pasemi_mac *mac)
626	{
627	struct pasemi_mac_rxring *rx = rx_ring(mac);
628	unsigned int reg, pcnt;
629	/* Re-enable packet count interrupts: finally
630	* ack the packet count interrupt we got in rx_intr.
631	*/
632
633	pcnt = *rx->chan.status & PAS_STATUS_PCNT_M;
634
635	reg = PAS_IOB_DMA_RXCH_RESET_PCNT(pcnt) | PAS_IOB_DMA_RXCH_RESET_PINTC;
636
637	if (*rx->chan.status & PAS_STATUS_TIMER)
638	reg |= PAS_IOB_DMA_RXCH_RESET_TINTC;
639
640	write_iob_reg(reg: PAS_IOB_DMA_RXCH_RESET(mac->rx->chan.chno), val: reg);
641	}
642
643	static void pasemi_mac_restart_tx_intr(const struct pasemi_mac *mac)
644	{
645	unsigned int reg, pcnt;
646
647	/* Re-enable packet count interrupts */
648	pcnt = *tx_ring(mac)->chan.status & PAS_STATUS_PCNT_M;
649
650	reg = PAS_IOB_DMA_TXCH_RESET_PCNT(pcnt) | PAS_IOB_DMA_TXCH_RESET_PINTC;
651
652	write_iob_reg(reg: PAS_IOB_DMA_TXCH_RESET(tx_ring(mac)->chan.chno), val: reg);
653	}
654
655
656	static inline void pasemi_mac_rx_error(const struct pasemi_mac *mac,
657	const u64 macrx)
658	{
659	unsigned int rcmdsta, ccmdsta;
660	struct pasemi_dmachan *chan = &rx_ring(mac)->chan;
661
662	if (!netif_msg_rx_err(mac))
663	return;
664
665	rcmdsta = read_dma_reg(reg: PAS_DMA_RXINT_RCMDSTA(mac->dma_if));
666	ccmdsta = read_dma_reg(reg: PAS_DMA_RXCHAN_CCMDSTA(chan->chno));
667
668	printk(KERN_ERR "pasemi_mac: rx error. macrx %016llx, rx status %llx\n",
669	macrx, *chan->status);
670
671	printk(KERN_ERR "pasemi_mac: rcmdsta %08x ccmdsta %08x\n",
672	rcmdsta, ccmdsta);
673	}
674
675	static inline void pasemi_mac_tx_error(const struct pasemi_mac *mac,
676	const u64 mactx)
677	{
678	unsigned int cmdsta;
679	struct pasemi_dmachan *chan = &tx_ring(mac)->chan;
680
681	if (!netif_msg_tx_err(mac))
682	return;
683
684	cmdsta = read_dma_reg(reg: PAS_DMA_TXCHAN_TCMDSTA(chan->chno));
685
686	printk(KERN_ERR "pasemi_mac: tx error. mactx 0x%016llx, "\
687	"tx status 0x%016llx\n", mactx, *chan->status);
688
689	printk(KERN_ERR "pasemi_mac: tcmdsta 0x%08x\n", cmdsta);
690	}
691
692	static int pasemi_mac_clean_rx(struct pasemi_mac_rxring *rx,
693	const int limit)
694	{
695	const struct pasemi_dmachan *chan = &rx->chan;
696	struct pasemi_mac *mac = rx->mac;
697	struct pci_dev *pdev = mac->dma_pdev;
698	unsigned int n;
699	int count, buf_index, tot_bytes, packets;
700	struct pasemi_mac_buffer *info;
701	struct sk_buff *skb;
702	unsigned int len;
703	u64 macrx, eval;
704	dma_addr_t dma;
705
706	tot_bytes = 0;
707	packets = 0;
708
709	spin_lock(lock: &rx->lock);
710
711	n = rx->next_to_clean;
712
713	prefetch(&RX_DESC(rx, n));
714
715	for (count = 0; count < limit; count++) {
716	macrx = RX_DESC(rx, n);
717	prefetch(&RX_DESC(rx, n+4));
718
719	if ((macrx & XCT_MACRX_E) ||
720	(*chan->status & PAS_STATUS_ERROR))
721	pasemi_mac_rx_error(mac, macrx);
722
723	if (!(macrx & XCT_MACRX_O))
724	break;
725
726	info = NULL;
727
728	BUG_ON(!(macrx & XCT_MACRX_RR_8BRES));
729
730	eval = (RX_DESC(rx, n+1) & XCT_RXRES_8B_EVAL_M) >>
731	XCT_RXRES_8B_EVAL_S;
732	buf_index = eval-1;
733
734	dma = (RX_DESC(rx, n+2) & XCT_PTR_ADDR_M);
735	info = &RX_DESC_INFO(rx, buf_index);
736
737	skb = info->skb;
738
739	prefetch_skb(skb);
740
741	len = (macrx & XCT_MACRX_LLEN_M) >> XCT_MACRX_LLEN_S;
742
743	dma_unmap_single(&pdev->dev, dma,
744	mac->bufsz - LOCAL_SKB_ALIGN,
745	DMA_FROM_DEVICE);
746
747	if (macrx & XCT_MACRX_CRC) {
748	/* CRC error flagged */
749	mac->netdev->stats.rx_errors++;
750	mac->netdev->stats.rx_crc_errors++;
751	/* No need to free skb, it'll be reused */
752	goto next;
753	}
754
755	info->skb = NULL;
756	info->dma = 0;
757
758	if (likely((macrx & XCT_MACRX_HTY_M) == XCT_MACRX_HTY_IPV4_OK)) {
759	skb->ip_summed = CHECKSUM_UNNECESSARY;
760	skb->csum = (macrx & XCT_MACRX_CSUM_M) >>
761	XCT_MACRX_CSUM_S;
762	} else {
763	skb_checksum_none_assert(skb);
764	}
765
766	packets++;
767	tot_bytes += len;
768
769	/* Don't include CRC */
770	skb_put(skb, len: len-4);
771
772	skb->protocol = eth_type_trans(skb, dev: mac->netdev);
773	napi_gro_receive(napi: &mac->napi, skb);
774
775	next:
776	RX_DESC(rx, n) = 0;
777	RX_DESC(rx, n+1) = 0;
778
779	/* Need to zero it out since hardware doesn't, since the
780	* replenish loop uses it to tell when it's done.
781	*/
782	RX_BUFF(rx, buf_index) = 0;
783
784	n += 4;
785	}
786
787	if (n > RX_RING_SIZE) {
788	/* Errata 5971 workaround: L2 target of headers */
789	write_iob_reg(PAS_IOB_COM_PKTHDRCNT, 0);
790	n &= (RX_RING_SIZE-1);
791	}
792
793	rx_ring(mac)->next_to_clean = n;
794
795	/* Increase is in number of 16-byte entries, and since each descriptor
796	* with an 8BRES takes up 3x8 bytes (padded to 4x8), increase with
797	* count*2.
798	*/
799	write_dma_reg(reg: PAS_DMA_RXCHAN_INCR(mac->rx->chan.chno), val: count << 1);
800
801	pasemi_mac_replenish_rx_ring(dev: mac->netdev, limit: count);
802
803	mac->netdev->stats.rx_bytes += tot_bytes;
804	mac->netdev->stats.rx_packets += packets;
805
806	spin_unlock(lock: &rx_ring(mac)->lock);
807
808	return count;
809	}
810
811	/* Can't make this too large or we blow the kernel stack limits */
812	#define TX_CLEAN_BATCHSIZE (128/MAX_SKB_FRAGS)
813
814	static int pasemi_mac_clean_tx(struct pasemi_mac_txring *txring)
815	{
816	struct pasemi_dmachan *chan = &txring->chan;
817	struct pasemi_mac *mac = txring->mac;
818	int i, j;
819	unsigned int start, descr_count, buf_count, batch_limit;
820	unsigned int ring_limit;
821	unsigned int total_count;
822	unsigned long flags;
823	struct sk_buff *skbs[TX_CLEAN_BATCHSIZE];
824	dma_addr_t dmas[TX_CLEAN_BATCHSIZE][MAX_SKB_FRAGS+1];
825	int nf[TX_CLEAN_BATCHSIZE];
826	int nr_frags;
827
828	total_count = 0;
829	batch_limit = TX_CLEAN_BATCHSIZE;
830	restart:
831	spin_lock_irqsave(&txring->lock, flags);
832
833	start = txring->next_to_clean;
834	ring_limit = txring->next_to_fill;
835
836	prefetch(&TX_DESC_INFO(txring, start+1).skb);
837
838	/* Compensate for when fill has wrapped but clean has not */
839	if (start > ring_limit)
840	ring_limit += TX_RING_SIZE;
841
842	buf_count = 0;
843	descr_count = 0;
844
845	for (i = start;
846	descr_count < batch_limit && i < ring_limit;
847	i += buf_count) {
848	u64 mactx = TX_DESC(txring, i);
849	struct sk_buff *skb;
850
851	if ((mactx & XCT_MACTX_E) ||
852	(*chan->status & PAS_STATUS_ERROR))
853	pasemi_mac_tx_error(mac, mactx);
854
855	/* Skip over control descriptors */
856	if (!(mactx & XCT_MACTX_LLEN_M)) {
857	TX_DESC(txring, i) = 0;
858	TX_DESC(txring, i+1) = 0;
859	buf_count = 2;
860	continue;
861	}
862
863	skb = TX_DESC_INFO(txring, i+1).skb;
864	nr_frags = TX_DESC_INFO(txring, i).dma;
865
866	if (unlikely(mactx & XCT_MACTX_O))
867	/* Not yet transmitted */
868	break;
869
870	buf_count = 2 + nr_frags;
871	/* Since we always fill with an even number of entries, make
872	* sure we skip any unused one at the end as well.
873	*/
874	if (buf_count & 1)
875	buf_count++;
876
877	for (j = 0; j <= nr_frags; j++)
878	dmas[descr_count][j] = TX_DESC_INFO(txring, i+1+j).dma;
879
880	skbs[descr_count] = skb;
881	nf[descr_count] = nr_frags;
882
883	TX_DESC(txring, i) = 0;
884	TX_DESC(txring, i+1) = 0;
885
886	descr_count++;
887	}
888	txring->next_to_clean = i & (TX_RING_SIZE-1);
889
890	spin_unlock_irqrestore(lock: &txring->lock, flags);
891	netif_wake_queue(dev: mac->netdev);
892
893	for (i = 0; i < descr_count; i++)
894	pasemi_mac_unmap_tx_skb(mac, nfrags: nf[i], skb: skbs[i], dmas: dmas[i]);
895
896	total_count += descr_count;
897
898	/* If the batch was full, try to clean more */
899	if (descr_count == batch_limit)
900	goto restart;
901
902	return total_count;
903	}
904
905
906	static irqreturn_t pasemi_mac_rx_intr(int irq, void *data)
907	{
908	const struct pasemi_mac_rxring *rxring = data;
909	struct pasemi_mac *mac = rxring->mac;
910	const struct pasemi_dmachan *chan = &rxring->chan;
911	unsigned int reg;
912
913	if (!(*chan->status & PAS_STATUS_CAUSE_M))
914	return IRQ_NONE;
915
916	/* Don't reset packet count so it won't fire again but clear
917	* all others.
918	*/
919
920	reg = 0;
921	if (*chan->status & PAS_STATUS_SOFT)
922	reg |= PAS_IOB_DMA_RXCH_RESET_SINTC;
923	if (*chan->status & PAS_STATUS_ERROR)
924	reg |= PAS_IOB_DMA_RXCH_RESET_DINTC;
925
926	napi_schedule(n: &mac->napi);
927
928	write_iob_reg(reg: PAS_IOB_DMA_RXCH_RESET(chan->chno), val: reg);
929
930	return IRQ_HANDLED;
931	}
932
933	#define TX_CLEAN_INTERVAL HZ
934
935	static void pasemi_mac_tx_timer(struct timer_list *t)
936	{
937	struct pasemi_mac_txring *txring = from_timer(txring, t, clean_timer);
938	struct pasemi_mac *mac = txring->mac;
939
940	pasemi_mac_clean_tx(txring);
941
942	mod_timer(timer: &txring->clean_timer, expires: jiffies + TX_CLEAN_INTERVAL);
943
944	pasemi_mac_restart_tx_intr(mac);
945	}
946
947	static irqreturn_t pasemi_mac_tx_intr(int irq, void *data)
948	{
949	struct pasemi_mac_txring *txring = data;
950	const struct pasemi_dmachan *chan = &txring->chan;
951	struct pasemi_mac *mac = txring->mac;
952	unsigned int reg;
953
954	if (!(*chan->status & PAS_STATUS_CAUSE_M))
955	return IRQ_NONE;
956
957	reg = 0;
958
959	if (*chan->status & PAS_STATUS_SOFT)
960	reg |= PAS_IOB_DMA_TXCH_RESET_SINTC;
961	if (*chan->status & PAS_STATUS_ERROR)
962	reg |= PAS_IOB_DMA_TXCH_RESET_DINTC;
963
964	mod_timer(timer: &txring->clean_timer, expires: jiffies + (TX_CLEAN_INTERVAL)*2);
965
966	napi_schedule(n: &mac->napi);
967
968	if (reg)
969	write_iob_reg(reg: PAS_IOB_DMA_TXCH_RESET(chan->chno), val: reg);
970
971	return IRQ_HANDLED;
972	}
973
974	static void pasemi_adjust_link(struct net_device *dev)
975	{
976	struct pasemi_mac *mac = netdev_priv(dev);
977	int msg;
978	unsigned int flags;
979	unsigned int new_flags;
980
981	if (!dev->phydev->link) {
982	/* If no link, MAC speed settings don't matter. Just report
983	* link down and return.
984	*/
985	if (mac->link && netif_msg_link(mac))
986	printk(KERN_INFO "%s: Link is down.\n", dev->name);
987
988	netif_carrier_off(dev);
989	pasemi_mac_intf_disable(mac);
990	mac->link = 0;
991
992	return;
993	} else {
994	pasemi_mac_intf_enable(mac);
995	netif_carrier_on(dev);
996	}
997
998	flags = read_mac_reg(mac, reg: PAS_MAC_CFG_PCFG);
999	new_flags = flags & ~(PAS_MAC_CFG_PCFG_HD | PAS_MAC_CFG_PCFG_SPD_M |
1000	PAS_MAC_CFG_PCFG_TSR_M);
1001
1002	if (!dev->phydev->duplex)
1003	new_flags |= PAS_MAC_CFG_PCFG_HD;
1004
1005	switch (dev->phydev->speed) {
1006	case 1000:
1007	new_flags |= PAS_MAC_CFG_PCFG_SPD_1G |
1008	PAS_MAC_CFG_PCFG_TSR_1G;
1009	break;
1010	case 100:
1011	new_flags |= PAS_MAC_CFG_PCFG_SPD_100M |
1012	PAS_MAC_CFG_PCFG_TSR_100M;
1013	break;
1014	case 10:
1015	new_flags |= PAS_MAC_CFG_PCFG_SPD_10M |
1016	PAS_MAC_CFG_PCFG_TSR_10M;
1017	break;
1018	default:
1019	printk("Unsupported speed %d\n", dev->phydev->speed);
1020	}
1021
1022	/* Print on link or speed/duplex change */
1023	msg = mac->link != dev->phydev->link || flags != new_flags;
1024
1025	mac->duplex = dev->phydev->duplex;
1026	mac->speed = dev->phydev->speed;
1027	mac->link = dev->phydev->link;
1028
1029	if (new_flags != flags)
1030	write_mac_reg(mac, reg: PAS_MAC_CFG_PCFG, val: new_flags);
1031
1032	if (msg && netif_msg_link(mac))
1033	printk(KERN_INFO "%s: Link is up at %d Mbps, %s duplex.\n",
1034	dev->name, mac->speed, mac->duplex ? "full" : "half");
1035	}
1036
1037	static int pasemi_mac_phy_init(struct net_device *dev)
1038	{
1039	struct pasemi_mac *mac = netdev_priv(dev);
1040	struct device_node *dn, *phy_dn;
1041	struct phy_device *phydev;
1042
1043	dn = pci_device_to_OF_node(pdev: mac->pdev);
1044	phy_dn = of_parse_phandle(np: dn, phandle_name: "phy-handle", index: 0);
1045
1046	mac->link = 0;
1047	mac->speed = 0;
1048	mac->duplex = -1;
1049
1050	phydev = of_phy_connect(dev, phy_np: phy_dn, hndlr: &pasemi_adjust_link, flags: 0,
1051	iface: PHY_INTERFACE_MODE_SGMII);
1052
1053	of_node_put(node: phy_dn);
1054	if (!phydev) {
1055	printk(KERN_ERR "%s: Could not attach to phy\n", dev->name);
1056	return -ENODEV;
1057	}
1058
1059	return 0;
1060	}
1061
1062
1063	static int pasemi_mac_open(struct net_device *dev)
1064	{
1065	struct pasemi_mac *mac = netdev_priv(dev);
1066	unsigned int flags;
1067	int i, ret;
1068
1069	flags = PAS_MAC_CFG_TXP_FCE | PAS_MAC_CFG_TXP_FPC(3) |
1070	PAS_MAC_CFG_TXP_SL(3) | PAS_MAC_CFG_TXP_COB(0xf) |
1071	PAS_MAC_CFG_TXP_TIFT(8) | PAS_MAC_CFG_TXP_TIFG(12);
1072
1073	write_mac_reg(mac, reg: PAS_MAC_CFG_TXP, val: flags);
1074
1075	ret = pasemi_mac_setup_rx_resources(dev);
1076	if (ret)
1077	goto out_rx_resources;
1078
1079	mac->tx = pasemi_mac_setup_tx_resources(dev);
1080
1081	if (!mac->tx) {
1082	ret = -ENOMEM;
1083	goto out_tx_ring;
1084	}
1085
1086	/* We might already have allocated rings in case mtu was changed
1087	* before interface was brought up.
1088	*/
1089	if (dev->mtu > 1500 && !mac->num_cs) {
1090	pasemi_mac_setup_csrings(mac);
1091	if (!mac->num_cs) {
1092	ret = -ENOMEM;
1093	goto out_tx_ring;
1094	}
1095	}
1096
1097	/* Zero out rmon counters */
1098	for (i = 0; i < 32; i++)
1099	write_mac_reg(mac, PAS_MAC_RMON(i), val: 0);
1100
1101	/* 0x3ff with 33MHz clock is about 31us */
1102	write_iob_reg(PAS_IOB_DMA_COM_TIMEOUTCFG,
1103	PAS_IOB_DMA_COM_TIMEOUTCFG_TCNT(0x3ff));
1104
1105	write_iob_reg(reg: PAS_IOB_DMA_RXCH_CFG(mac->rx->chan.chno),
1106	val: PAS_IOB_DMA_RXCH_CFG_CNTTH(256));
1107
1108	write_iob_reg(reg: PAS_IOB_DMA_TXCH_CFG(mac->tx->chan.chno),
1109	val: PAS_IOB_DMA_TXCH_CFG_CNTTH(32));
1110
1111	write_mac_reg(mac, reg: PAS_MAC_IPC_CHNL,
1112	PAS_MAC_IPC_CHNL_DCHNO(mac->rx->chan.chno) |
1113	PAS_MAC_IPC_CHNL_BCH(mac->rx->chan.chno));
1114
1115	/* enable rx if */
1116	write_dma_reg(PAS_DMA_RXINT_RCMDSTA(mac->dma_if),
1117	PAS_DMA_RXINT_RCMDSTA_EN |
1118	PAS_DMA_RXINT_RCMDSTA_DROPS_M |
1119	PAS_DMA_RXINT_RCMDSTA_BP |
1120	PAS_DMA_RXINT_RCMDSTA_OO |
1121	PAS_DMA_RXINT_RCMDSTA_BT);
1122
1123	/* enable rx channel */
1124	pasemi_dma_start_chan(&rx_ring(mac)->chan, PAS_DMA_RXCHAN_CCMDSTA_DU |
1125	PAS_DMA_RXCHAN_CCMDSTA_OD |
1126	PAS_DMA_RXCHAN_CCMDSTA_FD |
1127	PAS_DMA_RXCHAN_CCMDSTA_DT);
1128
1129	/* enable tx channel */
1130	pasemi_dma_start_chan(&tx_ring(mac)->chan, PAS_DMA_TXCHAN_TCMDSTA_SZ |
1131	PAS_DMA_TXCHAN_TCMDSTA_DB |
1132	PAS_DMA_TXCHAN_TCMDSTA_DE |
1133	PAS_DMA_TXCHAN_TCMDSTA_DA);
1134
1135	pasemi_mac_replenish_rx_ring(dev, RX_RING_SIZE);
1136
1137	write_dma_reg(reg: PAS_DMA_RXCHAN_INCR(rx_ring(mac)->chan.chno),
1138	RX_RING_SIZE>>1);
1139
1140	/* Clear out any residual packet count state from firmware */
1141	pasemi_mac_restart_rx_intr(mac);
1142	pasemi_mac_restart_tx_intr(mac);
1143
1144	flags = PAS_MAC_CFG_PCFG_S1 | PAS_MAC_CFG_PCFG_PR | PAS_MAC_CFG_PCFG_CE;
1145
1146	if (mac->type == MAC_TYPE_GMAC)
1147	flags |= PAS_MAC_CFG_PCFG_TSR_1G | PAS_MAC_CFG_PCFG_SPD_1G;
1148	else
1149	flags |= PAS_MAC_CFG_PCFG_TSR_10G | PAS_MAC_CFG_PCFG_SPD_10G;
1150
1151	/* Enable interface in MAC */
1152	write_mac_reg(mac, reg: PAS_MAC_CFG_PCFG, val: flags);
1153
1154	ret = pasemi_mac_phy_init(dev);
1155	if (ret) {
1156	/* Since we won't get link notification, just enable RX */
1157	pasemi_mac_intf_enable(mac);
1158	if (mac->type == MAC_TYPE_GMAC) {
1159	/* Warn for missing PHY on SGMII (1Gig) ports */
1160	dev_warn(&mac->pdev->dev,
1161	"PHY init failed: %d.\n", ret);
1162	dev_warn(&mac->pdev->dev,
1163	"Defaulting to 1Gbit full duplex\n");
1164	}
1165	}
1166
1167	netif_start_queue(dev);
1168	napi_enable(n: &mac->napi);
1169
1170	snprintf(buf: mac->tx_irq_name, size: sizeof(mac->tx_irq_name), fmt: "%s tx",
1171	dev->name);
1172
1173	ret = request_irq(irq: mac->tx->chan.irq, handler: pasemi_mac_tx_intr, flags: 0,
1174	name: mac->tx_irq_name, dev: mac->tx);
1175	if (ret) {
1176	dev_err(&mac->pdev->dev, "request_irq of irq %d failed: %d\n",
1177	mac->tx->chan.irq, ret);
1178	goto out_tx_int;
1179	}
1180
1181	snprintf(buf: mac->rx_irq_name, size: sizeof(mac->rx_irq_name), fmt: "%s rx",
1182	dev->name);
1183
1184	ret = request_irq(irq: mac->rx->chan.irq, handler: pasemi_mac_rx_intr, flags: 0,
1185	name: mac->rx_irq_name, dev: mac->rx);
1186	if (ret) {
1187	dev_err(&mac->pdev->dev, "request_irq of irq %d failed: %d\n",
1188	mac->rx->chan.irq, ret);
1189	goto out_rx_int;
1190	}
1191
1192	if (dev->phydev)
1193	phy_start(phydev: dev->phydev);
1194
1195	timer_setup(&mac->tx->clean_timer, pasemi_mac_tx_timer, 0);
1196	mod_timer(timer: &mac->tx->clean_timer, expires: jiffies + HZ);
1197
1198	return 0;
1199
1200	out_rx_int:
1201	free_irq(mac->tx->chan.irq, mac->tx);
1202	out_tx_int:
1203	napi_disable(n: &mac->napi);
1204	netif_stop_queue(dev);
1205	out_tx_ring:
1206	if (mac->tx)
1207	pasemi_mac_free_tx_resources(mac);
1208	pasemi_mac_free_rx_resources(mac);
1209	out_rx_resources:
1210
1211	return ret;
1212	}
1213
1214	#define MAX_RETRIES 5000
1215
1216	static void pasemi_mac_pause_txchan(struct pasemi_mac *mac)
1217	{
1218	unsigned int sta, retries;
1219	int txch = tx_ring(mac)->chan.chno;
1220
1221	write_dma_reg(PAS_DMA_TXCHAN_TCMDSTA(txch),
1222	PAS_DMA_TXCHAN_TCMDSTA_ST);
1223
1224	for (retries = 0; retries < MAX_RETRIES; retries++) {
1225	sta = read_dma_reg(reg: PAS_DMA_TXCHAN_TCMDSTA(txch));
1226	if (!(sta & PAS_DMA_TXCHAN_TCMDSTA_ACT))
1227	break;
1228	cond_resched();
1229	}
1230
1231	if (sta & PAS_DMA_TXCHAN_TCMDSTA_ACT)
1232	dev_err(&mac->dma_pdev->dev,
1233	"Failed to stop tx channel, tcmdsta %08x\n", sta);
1234
1235	write_dma_reg(reg: PAS_DMA_TXCHAN_TCMDSTA(txch), val: 0);
1236	}
1237
1238	static void pasemi_mac_pause_rxchan(struct pasemi_mac *mac)
1239	{
1240	unsigned int sta, retries;
1241	int rxch = rx_ring(mac)->chan.chno;
1242
1243	write_dma_reg(PAS_DMA_RXCHAN_CCMDSTA(rxch),
1244	PAS_DMA_RXCHAN_CCMDSTA_ST);
1245	for (retries = 0; retries < MAX_RETRIES; retries++) {
1246	sta = read_dma_reg(reg: PAS_DMA_RXCHAN_CCMDSTA(rxch));
1247	if (!(sta & PAS_DMA_RXCHAN_CCMDSTA_ACT))
1248	break;
1249	cond_resched();
1250	}
1251
1252	if (sta & PAS_DMA_RXCHAN_CCMDSTA_ACT)
1253	dev_err(&mac->dma_pdev->dev,
1254	"Failed to stop rx channel, ccmdsta 08%x\n", sta);
1255	write_dma_reg(reg: PAS_DMA_RXCHAN_CCMDSTA(rxch), val: 0);
1256	}
1257
1258	static void pasemi_mac_pause_rxint(struct pasemi_mac *mac)
1259	{
1260	unsigned int sta, retries;
1261
1262	write_dma_reg(PAS_DMA_RXINT_RCMDSTA(mac->dma_if),
1263	PAS_DMA_RXINT_RCMDSTA_ST);
1264	for (retries = 0; retries < MAX_RETRIES; retries++) {
1265	sta = read_dma_reg(reg: PAS_DMA_RXINT_RCMDSTA(mac->dma_if));
1266	if (!(sta & PAS_DMA_RXINT_RCMDSTA_ACT))
1267	break;
1268	cond_resched();
1269	}
1270
1271	if (sta & PAS_DMA_RXINT_RCMDSTA_ACT)
1272	dev_err(&mac->dma_pdev->dev,
1273	"Failed to stop rx interface, rcmdsta %08x\n", sta);
1274	write_dma_reg(reg: PAS_DMA_RXINT_RCMDSTA(mac->dma_if), val: 0);
1275	}
1276
1277	static int pasemi_mac_close(struct net_device *dev)
1278	{
1279	struct pasemi_mac *mac = netdev_priv(dev);
1280	unsigned int sta;
1281	int rxch, txch, i;
1282
1283	rxch = rx_ring(mac)->chan.chno;
1284	txch = tx_ring(mac)->chan.chno;
1285
1286	if (dev->phydev) {
1287	phy_stop(phydev: dev->phydev);
1288	phy_disconnect(phydev: dev->phydev);
1289	}
1290
1291	del_timer_sync(timer: &mac->tx->clean_timer);
1292
1293	netif_stop_queue(dev);
1294	napi_disable(n: &mac->napi);
1295
1296	sta = read_dma_reg(reg: PAS_DMA_RXINT_RCMDSTA(mac->dma_if));
1297	if (sta & (PAS_DMA_RXINT_RCMDSTA_BP |
1298	PAS_DMA_RXINT_RCMDSTA_OO |
1299	PAS_DMA_RXINT_RCMDSTA_BT))
1300	printk(KERN_DEBUG "pasemi_mac: rcmdsta error: 0x%08x\n", sta);
1301
1302	sta = read_dma_reg(reg: PAS_DMA_RXCHAN_CCMDSTA(rxch));
1303	if (sta & (PAS_DMA_RXCHAN_CCMDSTA_DU |
1304	PAS_DMA_RXCHAN_CCMDSTA_OD |
1305	PAS_DMA_RXCHAN_CCMDSTA_FD |
1306	PAS_DMA_RXCHAN_CCMDSTA_DT))
1307	printk(KERN_DEBUG "pasemi_mac: ccmdsta error: 0x%08x\n", sta);
1308
1309	sta = read_dma_reg(reg: PAS_DMA_TXCHAN_TCMDSTA(txch));
1310	if (sta & (PAS_DMA_TXCHAN_TCMDSTA_SZ | PAS_DMA_TXCHAN_TCMDSTA_DB |
1311	PAS_DMA_TXCHAN_TCMDSTA_DE | PAS_DMA_TXCHAN_TCMDSTA_DA))
1312	printk(KERN_DEBUG "pasemi_mac: tcmdsta error: 0x%08x\n", sta);
1313
1314	/* Clean out any pending buffers */
1315	pasemi_mac_clean_tx(txring: tx_ring(mac));
1316	pasemi_mac_clean_rx(rx: rx_ring(mac), RX_RING_SIZE);
1317
1318	pasemi_mac_pause_txchan(mac);
1319	pasemi_mac_pause_rxint(mac);
1320	pasemi_mac_pause_rxchan(mac);
1321	pasemi_mac_intf_disable(mac);
1322
1323	free_irq(mac->tx->chan.irq, mac->tx);
1324	free_irq(mac->rx->chan.irq, mac->rx);
1325
1326	for (i = 0; i < mac->num_cs; i++) {
1327	pasemi_mac_free_csring(csring: mac->cs[i]);
1328	mac->cs[i] = NULL;
1329	}
1330
1331	mac->num_cs = 0;
1332
1333	/* Free resources */
1334	pasemi_mac_free_rx_resources(mac);
1335	pasemi_mac_free_tx_resources(mac);
1336
1337	return 0;
1338	}
1339
1340	static void pasemi_mac_queue_csdesc(const struct sk_buff *skb,
1341	const dma_addr_t *map,
1342	const unsigned int *map_size,
1343	struct pasemi_mac_txring *txring,
1344	struct pasemi_mac_csring *csring)
1345	{
1346	u64 fund;
1347	dma_addr_t cs_dest;
1348	const int nh_off = skb_network_offset(skb);
1349	const int nh_len = skb_network_header_len(skb);
1350	const int nfrags = skb_shinfo(skb)->nr_frags;
1351	int cs_size, i, fill, hdr, evt;
1352	dma_addr_t csdma;
1353
1354	fund = XCT_FUN_ST | XCT_FUN_RR_8BRES |
1355	XCT_FUN_O | XCT_FUN_FUN(csring->fun) |
1356	XCT_FUN_CRM_SIG | XCT_FUN_LLEN(skb->len - nh_off) |
1357	XCT_FUN_SHL(nh_len >> 2) | XCT_FUN_SE;
1358
1359	switch (ip_hdr(skb)->protocol) {
1360	case IPPROTO_TCP:
1361	fund |= XCT_FUN_SIG_TCP4;
1362	/* TCP checksum is 16 bytes into the header */
1363	cs_dest = map[0] + skb_transport_offset(skb) + 16;
1364	break;
1365	case IPPROTO_UDP:
1366	fund |= XCT_FUN_SIG_UDP4;
1367	/* UDP checksum is 6 bytes into the header */
1368	cs_dest = map[0] + skb_transport_offset(skb) + 6;
1369	break;
1370	default:
1371	BUG();
1372	}
1373
1374	/* Do the checksum offloaded */
1375	fill = csring->next_to_fill;
1376	hdr = fill;
1377
1378	CS_DESC(csring, fill++) = fund;
1379	/* Room for 8BRES. Checksum result is really 2 bytes into it */
1380	csdma = csring->chan.ring_dma + (fill & (CS_RING_SIZE-1)) * 8 + 2;
1381	CS_DESC(csring, fill++) = 0;
1382
1383	CS_DESC(csring, fill) = XCT_PTR_LEN(map_size[0]-nh_off) | XCT_PTR_ADDR(map[0]+nh_off);
1384	for (i = 1; i <= nfrags; i++)
1385	CS_DESC(csring, fill+i) = XCT_PTR_LEN(map_size[i]) | XCT_PTR_ADDR(map[i]);
1386
1387	fill += i;
1388	if (fill & 1)
1389	fill++;
1390
1391	/* Copy the result into the TCP packet */
1392	CS_DESC(csring, fill++) = XCT_FUN_O | XCT_FUN_FUN(csring->fun) |
1393	XCT_FUN_LLEN(2) | XCT_FUN_SE;
1394	CS_DESC(csring, fill++) = XCT_PTR_LEN(2) | XCT_PTR_ADDR(cs_dest) | XCT_PTR_T;
1395	CS_DESC(csring, fill++) = XCT_PTR_LEN(2) | XCT_PTR_ADDR(csdma);
1396	fill++;
1397
1398	evt = !csring->last_event;
1399	csring->last_event = evt;
1400
1401	/* Event handshaking with MAC TX */
1402	CS_DESC(csring, fill++) = CTRL_CMD_T | CTRL_CMD_META_EVT | CTRL_CMD_O |
1403	CTRL_CMD_ETYPE_SET | CTRL_CMD_REG(csring->events[evt]);
1404	CS_DESC(csring, fill++) = 0;
1405	CS_DESC(csring, fill++) = CTRL_CMD_T | CTRL_CMD_META_EVT | CTRL_CMD_O |
1406	CTRL_CMD_ETYPE_WCLR | CTRL_CMD_REG(csring->events[!evt]);
1407	CS_DESC(csring, fill++) = 0;
1408	csring->next_to_fill = fill & (CS_RING_SIZE-1);
1409
1410	cs_size = fill - hdr;
1411	write_dma_reg(reg: PAS_DMA_TXCHAN_INCR(csring->chan.chno), val: (cs_size) >> 1);
1412
1413	/* TX-side event handshaking */
1414	fill = txring->next_to_fill;
1415	TX_DESC(txring, fill++) = CTRL_CMD_T | CTRL_CMD_META_EVT | CTRL_CMD_O |
1416	CTRL_CMD_ETYPE_WSET | CTRL_CMD_REG(csring->events[evt]);
1417	TX_DESC(txring, fill++) = 0;
1418	TX_DESC(txring, fill++) = CTRL_CMD_T | CTRL_CMD_META_EVT | CTRL_CMD_O |
1419	CTRL_CMD_ETYPE_CLR | CTRL_CMD_REG(csring->events[!evt]);
1420	TX_DESC(txring, fill++) = 0;
1421	txring->next_to_fill = fill;
1422
1423	write_dma_reg(reg: PAS_DMA_TXCHAN_INCR(txring->chan.chno), val: 2);
1424	}
1425
1426	static netdev_tx_t pasemi_mac_start_tx(struct sk_buff *skb, struct net_device *dev)
1427	{
1428	struct pasemi_mac * const mac = netdev_priv(dev);
1429	struct pasemi_mac_txring * const txring = tx_ring(mac);
1430	struct pasemi_mac_csring *csring;
1431	u64 dflags = 0;
1432	u64 mactx;
1433	dma_addr_t map[MAX_SKB_FRAGS+1];
1434	unsigned int map_size[MAX_SKB_FRAGS+1];
1435	unsigned long flags;
1436	int i, nfrags;
1437	int fill;
1438	const int nh_off = skb_network_offset(skb);
1439	const int nh_len = skb_network_header_len(skb);
1440
1441	prefetch(&txring->ring_info);
1442
1443	dflags = XCT_MACTX_O | XCT_MACTX_ST | XCT_MACTX_CRC_PAD;
1444
1445	nfrags = skb_shinfo(skb)->nr_frags;
1446
1447	map[0] = dma_map_single(&mac->dma_pdev->dev, skb->data,
1448	skb_headlen(skb), DMA_TO_DEVICE);
1449	map_size[0] = skb_headlen(skb);
1450	if (dma_mapping_error(dev: &mac->dma_pdev->dev, dma_addr: map[0]))
1451	goto out_err_nolock;
1452
1453	for (i = 0; i < nfrags; i++) {
1454	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1455
1456	map[i + 1] = skb_frag_dma_map(dev: &mac->dma_pdev->dev, frag, offset: 0,
1457	size: skb_frag_size(frag), dir: DMA_TO_DEVICE);
1458	map_size[i+1] = skb_frag_size(frag);
1459	if (dma_mapping_error(dev: &mac->dma_pdev->dev, dma_addr: map[i + 1])) {
1460	nfrags = i;
1461	goto out_err_nolock;
1462	}
1463	}
1464
1465	if (skb->ip_summed == CHECKSUM_PARTIAL && skb->len <= 1540) {
1466	switch (ip_hdr(skb)->protocol) {
1467	case IPPROTO_TCP:
1468	dflags |= XCT_MACTX_CSUM_TCP;
1469	dflags |= XCT_MACTX_IPH(nh_len >> 2);
1470	dflags |= XCT_MACTX_IPO(nh_off);
1471	break;
1472	case IPPROTO_UDP:
1473	dflags |= XCT_MACTX_CSUM_UDP;
1474	dflags |= XCT_MACTX_IPH(nh_len >> 2);
1475	dflags |= XCT_MACTX_IPO(nh_off);
1476	break;
1477	default:
1478	WARN_ON(1);
1479	}
1480	}
1481
1482	mactx = dflags | XCT_MACTX_LLEN(skb->len);
1483
1484	spin_lock_irqsave(&txring->lock, flags);
1485
1486	/* Avoid stepping on the same cache line that the DMA controller
1487	* is currently about to send, so leave at least 8 words available.
1488	* Total free space needed is mactx + fragments + 8
1489	*/
1490	if (RING_AVAIL(txring) < nfrags + 14) {
1491	/* no room -- stop the queue and wait for tx intr */
1492	netif_stop_queue(dev);
1493	goto out_err;
1494	}
1495
1496	/* Queue up checksum + event descriptors, if needed */
1497	if (mac->num_cs && skb->ip_summed == CHECKSUM_PARTIAL && skb->len > 1540) {
1498	csring = mac->cs[mac->last_cs];
1499	mac->last_cs = (mac->last_cs + 1) % mac->num_cs;
1500
1501	pasemi_mac_queue_csdesc(skb, map, map_size, txring, csring);
1502	}
1503
1504	fill = txring->next_to_fill;
1505	TX_DESC(txring, fill) = mactx;
1506	TX_DESC_INFO(txring, fill).dma = nfrags;
1507	fill++;
1508	TX_DESC_INFO(txring, fill).skb = skb;
1509	for (i = 0; i <= nfrags; i++) {
1510	TX_DESC(txring, fill+i) =
1511	XCT_PTR_LEN(map_size[i]) | XCT_PTR_ADDR(map[i]);
1512	TX_DESC_INFO(txring, fill+i).dma = map[i];
1513	}
1514
1515	/* We have to add an even number of 8-byte entries to the ring
1516	* even if the last one is unused. That means always an odd number
1517	* of pointers + one mactx descriptor.
1518	*/
1519	if (nfrags & 1)
1520	nfrags++;
1521
1522	txring->next_to_fill = (fill + nfrags + 1) & (TX_RING_SIZE-1);
1523
1524	dev->stats.tx_packets++;
1525	dev->stats.tx_bytes += skb->len;
1526
1527	spin_unlock_irqrestore(lock: &txring->lock, flags);
1528
1529	write_dma_reg(reg: PAS_DMA_TXCHAN_INCR(txring->chan.chno), val: (nfrags+2) >> 1);
1530
1531	return NETDEV_TX_OK;
1532
1533	out_err:
1534	spin_unlock_irqrestore(lock: &txring->lock, flags);
1535	out_err_nolock:
1536	while (nfrags--)
1537	dma_unmap_single(&mac->dma_pdev->dev, map[nfrags],
1538	map_size[nfrags], DMA_TO_DEVICE);
1539
1540	return NETDEV_TX_BUSY;
1541	}
1542
1543	static void pasemi_mac_set_rx_mode(struct net_device *dev)
1544	{
1545	const struct pasemi_mac *mac = netdev_priv(dev);
1546	unsigned int flags;
1547
1548	flags = read_mac_reg(mac, reg: PAS_MAC_CFG_PCFG);
1549
1550	/* Set promiscuous */
1551	if (dev->flags & IFF_PROMISC)
1552	flags |= PAS_MAC_CFG_PCFG_PR;
1553	else
1554	flags &= ~PAS_MAC_CFG_PCFG_PR;
1555
1556	write_mac_reg(mac, reg: PAS_MAC_CFG_PCFG, val: flags);
1557	}
1558
1559
1560	static int pasemi_mac_poll(struct napi_struct *napi, int budget)
1561	{
1562	struct pasemi_mac *mac = container_of(napi, struct pasemi_mac, napi);
1563	int pkts;
1564
1565	pasemi_mac_clean_tx(txring: tx_ring(mac));
1566	pkts = pasemi_mac_clean_rx(rx: rx_ring(mac), limit: budget);
1567	if (pkts < budget) {
1568	/* all done, no more packets present */
1569	napi_complete_done(n: napi, work_done: pkts);
1570
1571	pasemi_mac_restart_rx_intr(mac);
1572	pasemi_mac_restart_tx_intr(mac);
1573	}
1574	return pkts;
1575	}
1576
1577	#ifdef CONFIG_NET_POLL_CONTROLLER
1578	/*
1579	* Polling 'interrupt' - used by things like netconsole to send skbs
1580	* without having to re-enable interrupts. It's not called while
1581	* the interrupt routine is executing.
1582	*/
1583	static void pasemi_mac_netpoll(struct net_device *dev)
1584	{
1585	const struct pasemi_mac *mac = netdev_priv(dev);
1586
1587	disable_irq(irq: mac->tx->chan.irq);
1588	pasemi_mac_tx_intr(irq: mac->tx->chan.irq, data: mac->tx);
1589	enable_irq(irq: mac->tx->chan.irq);
1590
1591	disable_irq(irq: mac->rx->chan.irq);
1592	pasemi_mac_rx_intr(irq: mac->rx->chan.irq, data: mac->rx);
1593	enable_irq(irq: mac->rx->chan.irq);
1594	}
1595	#endif
1596
1597	static int pasemi_mac_change_mtu(struct net_device *dev, int new_mtu)
1598	{
1599	struct pasemi_mac *mac = netdev_priv(dev);
1600	unsigned int reg;
1601	unsigned int rcmdsta = 0;
1602	int running;
1603	int ret = 0;
1604
1605	running = netif_running(dev);
1606
1607	if (running) {
1608	/* Need to stop the interface, clean out all already
1609	* received buffers, free all unused buffers on the RX
1610	* interface ring, then finally re-fill the rx ring with
1611	* the new-size buffers and restart.
1612	*/
1613
1614	napi_disable(n: &mac->napi);
1615	netif_tx_disable(dev);
1616	pasemi_mac_intf_disable(mac);
1617
1618	rcmdsta = read_dma_reg(reg: PAS_DMA_RXINT_RCMDSTA(mac->dma_if));
1619	pasemi_mac_pause_rxint(mac);
1620	pasemi_mac_clean_rx(rx: rx_ring(mac), RX_RING_SIZE);
1621	pasemi_mac_free_rx_buffers(mac);
1622
1623	}
1624
1625	/* Setup checksum channels if large MTU and none already allocated */
1626	if (new_mtu > PE_DEF_MTU && !mac->num_cs) {
1627	pasemi_mac_setup_csrings(mac);
1628	if (!mac->num_cs) {
1629	ret = -ENOMEM;
1630	goto out;
1631	}
1632	}
1633
1634	/* Change maxf, i.e. what size frames are accepted.
1635	* Need room for ethernet header and CRC word
1636	*/
1637	reg = read_mac_reg(mac, reg: PAS_MAC_CFG_MACCFG);
1638	reg &= ~PAS_MAC_CFG_MACCFG_MAXF_M;
1639	reg |= PAS_MAC_CFG_MACCFG_MAXF(new_mtu + ETH_HLEN + 4);
1640	write_mac_reg(mac, reg: PAS_MAC_CFG_MACCFG, val: reg);
1641
1642	dev->mtu = new_mtu;
1643	/* MTU + ETH_HLEN + VLAN_HLEN + 2 64B cachelines */
1644	mac->bufsz = new_mtu + ETH_HLEN + ETH_FCS_LEN + LOCAL_SKB_ALIGN + 128;
1645
1646	out:
1647	if (running) {
1648	write_dma_reg(PAS_DMA_RXINT_RCMDSTA(mac->dma_if),
1649	rcmdsta | PAS_DMA_RXINT_RCMDSTA_EN);
1650
1651	rx_ring(mac)->next_to_fill = 0;
1652	pasemi_mac_replenish_rx_ring(dev, RX_RING_SIZE-1);
1653
1654	napi_enable(n: &mac->napi);
1655	netif_start_queue(dev);
1656	pasemi_mac_intf_enable(mac);
1657	}
1658
1659	return ret;
1660	}
1661
1662	static const struct net_device_ops pasemi_netdev_ops = {
1663	.ndo_open = pasemi_mac_open,
1664	.ndo_stop = pasemi_mac_close,
1665	.ndo_start_xmit = pasemi_mac_start_tx,
1666	.ndo_set_rx_mode = pasemi_mac_set_rx_mode,
1667	.ndo_set_mac_address = pasemi_mac_set_mac_addr,
1668	.ndo_change_mtu = pasemi_mac_change_mtu,
1669	.ndo_validate_addr = eth_validate_addr,
1670	#ifdef CONFIG_NET_POLL_CONTROLLER
1671	.ndo_poll_controller = pasemi_mac_netpoll,
1672	#endif
1673	};
1674
1675	static int
1676	pasemi_mac_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1677	{
1678	struct net_device *dev;
1679	struct pasemi_mac *mac;
1680	int err, ret;
1681
1682	err = pci_enable_device(dev: pdev);
1683	if (err)
1684	return err;
1685
1686	dev = alloc_etherdev(sizeof(struct pasemi_mac));
1687	if (dev == NULL) {
1688	err = -ENOMEM;
1689	goto out_disable_device;
1690	}
1691
1692	pci_set_drvdata(pdev, data: dev);
1693	SET_NETDEV_DEV(dev, &pdev->dev);
1694
1695	mac = netdev_priv(dev);
1696
1697	mac->pdev = pdev;
1698	mac->netdev = dev;
1699
1700	netif_napi_add(dev, napi: &mac->napi, poll: pasemi_mac_poll);
1701
1702	dev->features = NETIF_F_IP_CSUM | NETIF_F_LLTX | NETIF_F_SG |
1703	NETIF_F_HIGHDMA | NETIF_F_GSO;
1704
1705	mac->dma_pdev = pci_get_device(PCI_VENDOR_ID_PASEMI, device: 0xa007, NULL);
1706	if (!mac->dma_pdev) {
1707	dev_err(&mac->pdev->dev, "Can't find DMA Controller\n");
1708	err = -ENODEV;
1709	goto out;
1710	}
1711	dma_set_mask(dev: &mac->dma_pdev->dev, DMA_BIT_MASK(64));
1712
1713	mac->iob_pdev = pci_get_device(PCI_VENDOR_ID_PASEMI, device: 0xa001, NULL);
1714	if (!mac->iob_pdev) {
1715	dev_err(&mac->pdev->dev, "Can't find I/O Bridge\n");
1716	err = -ENODEV;
1717	goto out;
1718	}
1719
1720	/* get mac addr from device tree */
1721	if (pasemi_get_mac_addr(mac) || !is_valid_ether_addr(addr: mac->mac_addr)) {
1722	err = -ENODEV;
1723	goto out;
1724	}
1725	eth_hw_addr_set(dev, addr: mac->mac_addr);
1726
1727	ret = mac_to_intf(mac);
1728	if (ret < 0) {
1729	dev_err(&mac->pdev->dev, "Can't map DMA interface\n");
1730	err = -ENODEV;
1731	goto out;
1732	}
1733	mac->dma_if = ret;
1734
1735	switch (pdev->device) {
1736	case 0xa005:
1737	mac->type = MAC_TYPE_GMAC;
1738	break;
1739	case 0xa006:
1740	mac->type = MAC_TYPE_XAUI;
1741	break;
1742	default:
1743	err = -ENODEV;
1744	goto out;
1745	}
1746
1747	dev->netdev_ops = &pasemi_netdev_ops;
1748	dev->mtu = PE_DEF_MTU;
1749
1750	/* MTU range: 64 - 9000 */
1751	dev->min_mtu = PE_MIN_MTU;
1752	dev->max_mtu = PE_MAX_MTU;
1753
1754	/* 1500 MTU + ETH_HLEN + VLAN_HLEN + 2 64B cachelines */
1755	mac->bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + LOCAL_SKB_ALIGN + 128;
1756
1757	dev->ethtool_ops = &pasemi_mac_ethtool_ops;
1758
1759	if (err)
1760	goto out;
1761
1762	mac->msg_enable = netif_msg_init(debug_value: debug, DEFAULT_MSG_ENABLE);
1763
1764	/* Enable most messages by default */
1765	mac->msg_enable = (NETIF_MSG_IFUP << 1 ) - 1;
1766
1767	err = register_netdev(dev);
1768
1769	if (err) {
1770	dev_err(&mac->pdev->dev, "register_netdev failed with error %d\n",
1771	err);
1772	goto out;
1773	} else if (netif_msg_probe(mac)) {
1774	printk(KERN_INFO "%s: PA Semi %s: intf %d, hw addr %pM\n",
1775	dev->name, mac->type == MAC_TYPE_GMAC ? "GMAC" : "XAUI",
1776	mac->dma_if, dev->dev_addr);
1777	}
1778
1779	return err;
1780
1781	out:
1782	pci_dev_put(dev: mac->iob_pdev);
1783	pci_dev_put(dev: mac->dma_pdev);
1784
1785	free_netdev(dev);
1786	out_disable_device:
1787	pci_disable_device(dev: pdev);
1788	return err;
1789
1790	}
1791
1792	static void pasemi_mac_remove(struct pci_dev *pdev)
1793	{
1794	struct net_device *netdev = pci_get_drvdata(pdev);
1795	struct pasemi_mac *mac;
1796
1797	if (!netdev)
1798	return;
1799
1800	mac = netdev_priv(dev: netdev);
1801
1802	unregister_netdev(dev: netdev);
1803
1804	pci_disable_device(dev: pdev);
1805	pci_dev_put(dev: mac->dma_pdev);
1806	pci_dev_put(dev: mac->iob_pdev);
1807
1808	pasemi_dma_free_chan(&mac->tx->chan);
1809	pasemi_dma_free_chan(&mac->rx->chan);
1810
1811	free_netdev(dev: netdev);
1812	}
1813
1814	static const struct pci_device_id pasemi_mac_pci_tbl[] = {
1815	{ PCI_DEVICE(PCI_VENDOR_ID_PASEMI, 0xa005) },
1816	{ PCI_DEVICE(PCI_VENDOR_ID_PASEMI, 0xa006) },
1817	{ },
1818	};
1819
1820	MODULE_DEVICE_TABLE(pci, pasemi_mac_pci_tbl);
1821
1822	static struct pci_driver pasemi_mac_driver = {
1823	.name = "pasemi_mac",
1824	.id_table = pasemi_mac_pci_tbl,
1825	.probe = pasemi_mac_probe,
1826	.remove = pasemi_mac_remove,
1827	};
1828
1829	static void __exit pasemi_mac_cleanup_module(void)
1830	{
1831	pci_unregister_driver(dev: &pasemi_mac_driver);
1832	}
1833
1834	static int pasemi_mac_init_module(void)
1835	{
1836	int err;
1837
1838	err = pasemi_dma_init();
1839	if (err)
1840	return err;
1841
1842	return pci_register_driver(&pasemi_mac_driver);
1843	}
1844
1845	module_init(pasemi_mac_init_module);
1846	module_exit(pasemi_mac_cleanup_module);
1847