1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Huawei HiNIC PCI Express Linux driver |
4 | * Copyright(c) 2017 Huawei Technologies Co., Ltd |
5 | */ |
6 | |
7 | #include <linux/kernel.h> |
8 | #include <linux/module.h> |
9 | #include <linux/moduleparam.h> |
10 | #include <linux/pci.h> |
11 | #include <linux/device.h> |
12 | #include <linux/errno.h> |
13 | #include <linux/types.h> |
14 | #include <linux/etherdevice.h> |
15 | #include <linux/netdevice.h> |
16 | #include <linux/slab.h> |
17 | #include <linux/if_vlan.h> |
18 | #include <linux/semaphore.h> |
19 | #include <linux/workqueue.h> |
20 | #include <net/ip.h> |
21 | #include <net/devlink.h> |
22 | #include <linux/bitops.h> |
23 | #include <linux/bitmap.h> |
24 | #include <linux/delay.h> |
25 | #include <linux/err.h> |
26 | |
27 | #include "hinic_debugfs.h" |
28 | #include "hinic_hw_qp.h" |
29 | #include "hinic_hw_dev.h" |
30 | #include "hinic_devlink.h" |
31 | #include "hinic_port.h" |
32 | #include "hinic_tx.h" |
33 | #include "hinic_rx.h" |
34 | #include "hinic_dev.h" |
35 | #include "hinic_sriov.h" |
36 | |
37 | MODULE_AUTHOR("Huawei Technologies CO., Ltd" ); |
38 | MODULE_DESCRIPTION("Huawei Intelligent NIC driver" ); |
39 | MODULE_LICENSE("GPL" ); |
40 | |
41 | static unsigned int tx_weight = 64; |
42 | module_param(tx_weight, uint, 0644); |
43 | MODULE_PARM_DESC(tx_weight, "Number Tx packets for NAPI budget (default=64)" ); |
44 | |
45 | static unsigned int rx_weight = 64; |
46 | module_param(rx_weight, uint, 0644); |
47 | MODULE_PARM_DESC(rx_weight, "Number Rx packets for NAPI budget (default=64)" ); |
48 | |
49 | #define HINIC_DEV_ID_QUAD_PORT_25GE 0x1822 |
50 | #define HINIC_DEV_ID_DUAL_PORT_100GE 0x0200 |
51 | #define HINIC_DEV_ID_DUAL_PORT_100GE_MEZZ 0x0205 |
52 | #define HINIC_DEV_ID_QUAD_PORT_25GE_MEZZ 0x0210 |
53 | #define HINIC_DEV_ID_VF 0x375e |
54 | |
55 | #define HINIC_WQ_NAME "hinic_dev" |
56 | |
57 | #define MSG_ENABLE_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | \ |
58 | NETIF_MSG_IFUP | \ |
59 | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR) |
60 | |
61 | #define HINIC_LRO_MAX_WQE_NUM_DEFAULT 8 |
62 | |
63 | #define HINIC_LRO_RX_TIMER_DEFAULT 16 |
64 | |
65 | #define work_to_rx_mode_work(work) \ |
66 | container_of(work, struct hinic_rx_mode_work, work) |
67 | |
68 | #define rx_mode_work_to_nic_dev(rx_mode_work) \ |
69 | container_of(rx_mode_work, struct hinic_dev, rx_mode_work) |
70 | |
71 | #define HINIC_WAIT_SRIOV_CFG_TIMEOUT 15000 |
72 | |
73 | #define HINIC_DEAULT_TXRX_MSIX_PENDING_LIMIT 2 |
74 | #define HINIC_DEAULT_TXRX_MSIX_COALESC_TIMER_CFG 32 |
75 | #define HINIC_DEAULT_TXRX_MSIX_RESEND_TIMER_CFG 7 |
76 | |
77 | static int change_mac_addr(struct net_device *netdev, const u8 *addr); |
78 | |
79 | static int set_features(struct hinic_dev *nic_dev, |
80 | netdev_features_t pre_features, |
81 | netdev_features_t features, bool force_change); |
82 | |
83 | static void gather_rx_stats(struct hinic_rxq_stats *nic_rx_stats, struct hinic_rxq *rxq) |
84 | { |
85 | struct hinic_rxq_stats rx_stats; |
86 | |
87 | hinic_rxq_get_stats(rxq, stats: &rx_stats); |
88 | |
89 | nic_rx_stats->bytes += rx_stats.bytes; |
90 | nic_rx_stats->pkts += rx_stats.pkts; |
91 | nic_rx_stats->errors += rx_stats.errors; |
92 | nic_rx_stats->csum_errors += rx_stats.csum_errors; |
93 | nic_rx_stats->other_errors += rx_stats.other_errors; |
94 | } |
95 | |
96 | static void gather_tx_stats(struct hinic_txq_stats *nic_tx_stats, struct hinic_txq *txq) |
97 | { |
98 | struct hinic_txq_stats tx_stats; |
99 | |
100 | hinic_txq_get_stats(txq, stats: &tx_stats); |
101 | |
102 | nic_tx_stats->bytes += tx_stats.bytes; |
103 | nic_tx_stats->pkts += tx_stats.pkts; |
104 | nic_tx_stats->tx_busy += tx_stats.tx_busy; |
105 | nic_tx_stats->tx_wake += tx_stats.tx_wake; |
106 | nic_tx_stats->tx_dropped += tx_stats.tx_dropped; |
107 | nic_tx_stats->big_frags_pkts += tx_stats.big_frags_pkts; |
108 | } |
109 | |
110 | static void gather_nic_stats(struct hinic_dev *nic_dev, |
111 | struct hinic_rxq_stats *nic_rx_stats, |
112 | struct hinic_txq_stats *nic_tx_stats) |
113 | { |
114 | int i, num_qps = hinic_hwdev_num_qps(hwdev: nic_dev->hwdev); |
115 | |
116 | for (i = 0; i < num_qps; i++) |
117 | gather_rx_stats(nic_rx_stats, rxq: &nic_dev->rxqs[i]); |
118 | |
119 | for (i = 0; i < num_qps; i++) |
120 | gather_tx_stats(nic_tx_stats, txq: &nic_dev->txqs[i]); |
121 | } |
122 | |
123 | /** |
124 | * create_txqs - Create the Logical Tx Queues of specific NIC device |
125 | * @nic_dev: the specific NIC device |
126 | * |
127 | * Return 0 - Success, negative - Failure |
128 | **/ |
129 | static int create_txqs(struct hinic_dev *nic_dev) |
130 | { |
131 | int err, i, j, num_txqs = hinic_hwdev_num_qps(hwdev: nic_dev->hwdev); |
132 | struct net_device *netdev = nic_dev->netdev; |
133 | |
134 | if (nic_dev->txqs) |
135 | return -EINVAL; |
136 | |
137 | nic_dev->txqs = devm_kcalloc(dev: &netdev->dev, n: num_txqs, |
138 | size: sizeof(*nic_dev->txqs), GFP_KERNEL); |
139 | if (!nic_dev->txqs) |
140 | return -ENOMEM; |
141 | |
142 | hinic_sq_dbgfs_init(nic_dev); |
143 | |
144 | for (i = 0; i < num_txqs; i++) { |
145 | struct hinic_sq *sq = hinic_hwdev_get_sq(hwdev: nic_dev->hwdev, i); |
146 | |
147 | err = hinic_init_txq(txq: &nic_dev->txqs[i], sq, netdev); |
148 | if (err) { |
149 | netif_err(nic_dev, drv, netdev, |
150 | "Failed to init Txq\n" ); |
151 | goto err_init_txq; |
152 | } |
153 | |
154 | err = hinic_sq_debug_add(dev: nic_dev, sq_id: i); |
155 | if (err) { |
156 | netif_err(nic_dev, drv, netdev, |
157 | "Failed to add SQ%d debug\n" , i); |
158 | goto err_add_sq_dbg; |
159 | } |
160 | } |
161 | |
162 | return 0; |
163 | |
164 | err_add_sq_dbg: |
165 | hinic_clean_txq(txq: &nic_dev->txqs[i]); |
166 | err_init_txq: |
167 | for (j = 0; j < i; j++) { |
168 | hinic_sq_debug_rem(sq: nic_dev->txqs[j].sq); |
169 | hinic_clean_txq(txq: &nic_dev->txqs[j]); |
170 | } |
171 | |
172 | hinic_sq_dbgfs_uninit(nic_dev); |
173 | |
174 | devm_kfree(dev: &netdev->dev, p: nic_dev->txqs); |
175 | return err; |
176 | } |
177 | |
178 | static void enable_txqs_napi(struct hinic_dev *nic_dev) |
179 | { |
180 | int num_txqs = hinic_hwdev_num_qps(hwdev: nic_dev->hwdev); |
181 | int i; |
182 | |
183 | for (i = 0; i < num_txqs; i++) |
184 | napi_enable(n: &nic_dev->txqs[i].napi); |
185 | } |
186 | |
187 | static void disable_txqs_napi(struct hinic_dev *nic_dev) |
188 | { |
189 | int num_txqs = hinic_hwdev_num_qps(hwdev: nic_dev->hwdev); |
190 | int i; |
191 | |
192 | for (i = 0; i < num_txqs; i++) |
193 | napi_disable(n: &nic_dev->txqs[i].napi); |
194 | } |
195 | |
196 | /** |
197 | * free_txqs - Free the Logical Tx Queues of specific NIC device |
198 | * @nic_dev: the specific NIC device |
199 | **/ |
200 | static void free_txqs(struct hinic_dev *nic_dev) |
201 | { |
202 | int i, num_txqs = hinic_hwdev_num_qps(hwdev: nic_dev->hwdev); |
203 | struct net_device *netdev = nic_dev->netdev; |
204 | |
205 | if (!nic_dev->txqs) |
206 | return; |
207 | |
208 | for (i = 0; i < num_txqs; i++) { |
209 | hinic_sq_debug_rem(sq: nic_dev->txqs[i].sq); |
210 | hinic_clean_txq(txq: &nic_dev->txqs[i]); |
211 | } |
212 | |
213 | hinic_sq_dbgfs_uninit(nic_dev); |
214 | |
215 | devm_kfree(dev: &netdev->dev, p: nic_dev->txqs); |
216 | nic_dev->txqs = NULL; |
217 | } |
218 | |
219 | /** |
220 | * create_rxqs - Create the Logical Rx Queues of specific NIC device |
221 | * @nic_dev: the specific NIC device |
222 | * |
223 | * Return 0 - Success, negative - Failure |
224 | **/ |
225 | static int create_rxqs(struct hinic_dev *nic_dev) |
226 | { |
227 | int err, i, j, num_rxqs = hinic_hwdev_num_qps(hwdev: nic_dev->hwdev); |
228 | struct net_device *netdev = nic_dev->netdev; |
229 | |
230 | if (nic_dev->rxqs) |
231 | return -EINVAL; |
232 | |
233 | nic_dev->rxqs = devm_kcalloc(dev: &netdev->dev, n: num_rxqs, |
234 | size: sizeof(*nic_dev->rxqs), GFP_KERNEL); |
235 | if (!nic_dev->rxqs) |
236 | return -ENOMEM; |
237 | |
238 | hinic_rq_dbgfs_init(nic_dev); |
239 | |
240 | for (i = 0; i < num_rxqs; i++) { |
241 | struct hinic_rq *rq = hinic_hwdev_get_rq(hwdev: nic_dev->hwdev, i); |
242 | |
243 | err = hinic_init_rxq(rxq: &nic_dev->rxqs[i], rq, netdev); |
244 | if (err) { |
245 | netif_err(nic_dev, drv, netdev, |
246 | "Failed to init rxq\n" ); |
247 | goto err_init_rxq; |
248 | } |
249 | |
250 | err = hinic_rq_debug_add(dev: nic_dev, rq_id: i); |
251 | if (err) { |
252 | netif_err(nic_dev, drv, netdev, |
253 | "Failed to add RQ%d debug\n" , i); |
254 | goto err_add_rq_dbg; |
255 | } |
256 | } |
257 | |
258 | return 0; |
259 | |
260 | err_add_rq_dbg: |
261 | hinic_clean_rxq(rxq: &nic_dev->rxqs[i]); |
262 | err_init_rxq: |
263 | for (j = 0; j < i; j++) { |
264 | hinic_rq_debug_rem(rq: nic_dev->rxqs[j].rq); |
265 | hinic_clean_rxq(rxq: &nic_dev->rxqs[j]); |
266 | } |
267 | |
268 | hinic_rq_dbgfs_uninit(nic_dev); |
269 | |
270 | devm_kfree(dev: &netdev->dev, p: nic_dev->rxqs); |
271 | return err; |
272 | } |
273 | |
274 | /** |
275 | * free_rxqs - Free the Logical Rx Queues of specific NIC device |
276 | * @nic_dev: the specific NIC device |
277 | **/ |
278 | static void free_rxqs(struct hinic_dev *nic_dev) |
279 | { |
280 | int i, num_rxqs = hinic_hwdev_num_qps(hwdev: nic_dev->hwdev); |
281 | struct net_device *netdev = nic_dev->netdev; |
282 | |
283 | if (!nic_dev->rxqs) |
284 | return; |
285 | |
286 | for (i = 0; i < num_rxqs; i++) { |
287 | hinic_rq_debug_rem(rq: nic_dev->rxqs[i].rq); |
288 | hinic_clean_rxq(rxq: &nic_dev->rxqs[i]); |
289 | } |
290 | |
291 | hinic_rq_dbgfs_uninit(nic_dev); |
292 | |
293 | devm_kfree(dev: &netdev->dev, p: nic_dev->rxqs); |
294 | nic_dev->rxqs = NULL; |
295 | } |
296 | |
297 | static int hinic_configure_max_qnum(struct hinic_dev *nic_dev) |
298 | { |
299 | return hinic_set_max_qnum(nic_dev, num_rqs: nic_dev->hwdev->nic_cap.max_qps); |
300 | } |
301 | |
302 | static int (struct hinic_dev *nic_dev) |
303 | { |
304 | u8 [HINIC_RSS_KEY_SIZE]; |
305 | u8 tmpl_idx = nic_dev->rss_tmpl_idx; |
306 | u32 *indir_tbl; |
307 | int err, i; |
308 | |
309 | indir_tbl = kcalloc(HINIC_RSS_INDIR_SIZE, size: sizeof(u32), GFP_KERNEL); |
310 | if (!indir_tbl) |
311 | return -ENOMEM; |
312 | |
313 | netdev_rss_key_fill(buffer: default_rss_key, len: sizeof(default_rss_key)); |
314 | for (i = 0; i < HINIC_RSS_INDIR_SIZE; i++) |
315 | indir_tbl[i] = ethtool_rxfh_indir_default(index: i, n_rx_rings: nic_dev->num_rss); |
316 | |
317 | err = hinic_rss_set_template_tbl(nic_dev, template_id: tmpl_idx, temp: default_rss_key); |
318 | if (err) |
319 | goto out; |
320 | |
321 | err = hinic_rss_set_indir_tbl(nic_dev, tmpl_idx, indir_table: indir_tbl); |
322 | if (err) |
323 | goto out; |
324 | |
325 | err = hinic_set_rss_type(nic_dev, tmpl_idx, rss_type: nic_dev->rss_type); |
326 | if (err) |
327 | goto out; |
328 | |
329 | err = hinic_rss_set_hash_engine(nic_dev, template_id: tmpl_idx, |
330 | type: nic_dev->rss_hash_engine); |
331 | if (err) |
332 | goto out; |
333 | |
334 | err = hinic_rss_cfg(nic_dev, rss_en: 1, template_id: tmpl_idx); |
335 | if (err) |
336 | goto out; |
337 | |
338 | out: |
339 | kfree(objp: indir_tbl); |
340 | return err; |
341 | } |
342 | |
343 | static void (struct hinic_dev *nic_dev) |
344 | { |
345 | hinic_rss_cfg(nic_dev, rss_en: 0, template_id: nic_dev->rss_tmpl_idx); |
346 | } |
347 | |
348 | static void (struct hinic_dev *nic_dev) |
349 | { |
350 | nic_dev->rss_hash_engine = HINIC_RSS_HASH_ENGINE_TYPE_XOR; |
351 | nic_dev->rss_type.tcp_ipv6_ext = 1; |
352 | nic_dev->rss_type.ipv6_ext = 1; |
353 | nic_dev->rss_type.tcp_ipv6 = 1; |
354 | nic_dev->rss_type.ipv6 = 1; |
355 | nic_dev->rss_type.tcp_ipv4 = 1; |
356 | nic_dev->rss_type.ipv4 = 1; |
357 | nic_dev->rss_type.udp_ipv6 = 1; |
358 | nic_dev->rss_type.udp_ipv4 = 1; |
359 | } |
360 | |
361 | static void (struct hinic_dev *nic_dev) |
362 | { |
363 | struct net_device *netdev = nic_dev->netdev; |
364 | struct hinic_hwdev *hwdev = nic_dev->hwdev; |
365 | struct hinic_hwif *hwif = hwdev->hwif; |
366 | struct pci_dev *pdev = hwif->pdev; |
367 | int i, node, err = 0; |
368 | u16 num_cpus = 0; |
369 | |
370 | if (nic_dev->max_qps <= 1) { |
371 | nic_dev->flags &= ~HINIC_RSS_ENABLE; |
372 | nic_dev->rss_limit = nic_dev->max_qps; |
373 | nic_dev->num_qps = nic_dev->max_qps; |
374 | nic_dev->num_rss = nic_dev->max_qps; |
375 | |
376 | return; |
377 | } |
378 | |
379 | err = hinic_rss_template_alloc(nic_dev, tmpl_idx: &nic_dev->rss_tmpl_idx); |
380 | if (err) { |
381 | netif_err(nic_dev, drv, netdev, |
382 | "Failed to alloc tmpl_idx for rss, can't enable rss for this function\n" ); |
383 | nic_dev->flags &= ~HINIC_RSS_ENABLE; |
384 | nic_dev->max_qps = 1; |
385 | nic_dev->rss_limit = nic_dev->max_qps; |
386 | nic_dev->num_qps = nic_dev->max_qps; |
387 | nic_dev->num_rss = nic_dev->max_qps; |
388 | |
389 | return; |
390 | } |
391 | |
392 | nic_dev->flags |= HINIC_RSS_ENABLE; |
393 | |
394 | for (i = 0; i < num_online_cpus(); i++) { |
395 | node = cpu_to_node(cpu: i); |
396 | if (node == dev_to_node(dev: &pdev->dev)) |
397 | num_cpus++; |
398 | } |
399 | |
400 | if (!num_cpus) |
401 | num_cpus = num_online_cpus(); |
402 | |
403 | nic_dev->num_qps = hinic_hwdev_num_qps(hwdev); |
404 | nic_dev->num_qps = min_t(u16, nic_dev->num_qps, num_cpus); |
405 | |
406 | nic_dev->rss_limit = nic_dev->num_qps; |
407 | nic_dev->num_rss = nic_dev->num_qps; |
408 | |
409 | hinic_init_rss_parameters(nic_dev); |
410 | err = hinic_rss_init(nic_dev); |
411 | if (err) |
412 | netif_err(nic_dev, drv, netdev, "Failed to init rss\n" ); |
413 | } |
414 | |
415 | int hinic_open(struct net_device *netdev) |
416 | { |
417 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
418 | enum hinic_port_link_state link_state; |
419 | int err, ret; |
420 | |
421 | if (!(nic_dev->flags & HINIC_INTF_UP)) { |
422 | err = hinic_hwdev_ifup(hwdev: nic_dev->hwdev, sq_depth: nic_dev->sq_depth, |
423 | rq_depth: nic_dev->rq_depth); |
424 | if (err) { |
425 | netif_err(nic_dev, drv, netdev, |
426 | "Failed - HW interface up\n" ); |
427 | return err; |
428 | } |
429 | } |
430 | |
431 | err = create_txqs(nic_dev); |
432 | if (err) { |
433 | netif_err(nic_dev, drv, netdev, |
434 | "Failed to create Tx queues\n" ); |
435 | goto err_create_txqs; |
436 | } |
437 | |
438 | enable_txqs_napi(nic_dev); |
439 | |
440 | err = create_rxqs(nic_dev); |
441 | if (err) { |
442 | netif_err(nic_dev, drv, netdev, |
443 | "Failed to create Rx queues\n" ); |
444 | goto err_create_rxqs; |
445 | } |
446 | |
447 | hinic_enable_rss(nic_dev); |
448 | |
449 | err = hinic_configure_max_qnum(nic_dev); |
450 | if (err) { |
451 | netif_err(nic_dev, drv, nic_dev->netdev, |
452 | "Failed to configure the maximum number of queues\n" ); |
453 | goto err_port_state; |
454 | } |
455 | |
456 | netif_set_real_num_tx_queues(dev: netdev, txq: nic_dev->num_qps); |
457 | netif_set_real_num_rx_queues(dev: netdev, rxq: nic_dev->num_qps); |
458 | |
459 | err = hinic_port_set_state(nic_dev, state: HINIC_PORT_ENABLE); |
460 | if (err) { |
461 | netif_err(nic_dev, drv, netdev, |
462 | "Failed to set port state\n" ); |
463 | goto err_port_state; |
464 | } |
465 | |
466 | err = hinic_port_set_func_state(nic_dev, state: HINIC_FUNC_PORT_ENABLE); |
467 | if (err) { |
468 | netif_err(nic_dev, drv, netdev, |
469 | "Failed to set func port state\n" ); |
470 | goto err_func_port_state; |
471 | } |
472 | |
473 | down(sem: &nic_dev->mgmt_lock); |
474 | |
475 | err = hinic_port_link_state(nic_dev, link_state: &link_state); |
476 | if (err) { |
477 | netif_err(nic_dev, drv, netdev, "Failed to get link state\n" ); |
478 | goto err_port_link; |
479 | } |
480 | |
481 | if (!HINIC_IS_VF(nic_dev->hwdev->hwif)) |
482 | hinic_notify_all_vfs_link_changed(hwdev: nic_dev->hwdev, link_status: link_state); |
483 | |
484 | if (link_state == HINIC_LINK_STATE_UP) { |
485 | nic_dev->flags |= HINIC_LINK_UP; |
486 | nic_dev->cable_unplugged = false; |
487 | nic_dev->module_unrecognized = false; |
488 | } |
489 | |
490 | nic_dev->flags |= HINIC_INTF_UP; |
491 | |
492 | if ((nic_dev->flags & (HINIC_LINK_UP | HINIC_INTF_UP)) == |
493 | (HINIC_LINK_UP | HINIC_INTF_UP)) { |
494 | netif_info(nic_dev, drv, netdev, "link + intf UP\n" ); |
495 | netif_carrier_on(dev: netdev); |
496 | netif_tx_wake_all_queues(dev: netdev); |
497 | } |
498 | |
499 | up(sem: &nic_dev->mgmt_lock); |
500 | |
501 | netif_info(nic_dev, drv, netdev, "HINIC_INTF is UP\n" ); |
502 | return 0; |
503 | |
504 | err_port_link: |
505 | up(sem: &nic_dev->mgmt_lock); |
506 | ret = hinic_port_set_func_state(nic_dev, state: HINIC_FUNC_PORT_DISABLE); |
507 | if (ret) |
508 | netif_warn(nic_dev, drv, netdev, |
509 | "Failed to revert func port state\n" ); |
510 | |
511 | err_func_port_state: |
512 | ret = hinic_port_set_state(nic_dev, state: HINIC_PORT_DISABLE); |
513 | if (ret) |
514 | netif_warn(nic_dev, drv, netdev, |
515 | "Failed to revert port state\n" ); |
516 | err_port_state: |
517 | free_rxqs(nic_dev); |
518 | if (nic_dev->flags & HINIC_RSS_ENABLE) { |
519 | hinic_rss_deinit(nic_dev); |
520 | hinic_rss_template_free(nic_dev, tmpl_idx: nic_dev->rss_tmpl_idx); |
521 | } |
522 | |
523 | err_create_rxqs: |
524 | disable_txqs_napi(nic_dev); |
525 | free_txqs(nic_dev); |
526 | |
527 | err_create_txqs: |
528 | if (!(nic_dev->flags & HINIC_INTF_UP)) |
529 | hinic_hwdev_ifdown(hwdev: nic_dev->hwdev); |
530 | return err; |
531 | } |
532 | |
533 | int hinic_close(struct net_device *netdev) |
534 | { |
535 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
536 | unsigned int flags; |
537 | |
538 | /* Disable txq napi firstly to aviod rewaking txq in free_tx_poll */ |
539 | disable_txqs_napi(nic_dev); |
540 | |
541 | down(sem: &nic_dev->mgmt_lock); |
542 | |
543 | flags = nic_dev->flags; |
544 | nic_dev->flags &= ~HINIC_INTF_UP; |
545 | |
546 | netif_carrier_off(dev: netdev); |
547 | netif_tx_disable(dev: netdev); |
548 | |
549 | up(sem: &nic_dev->mgmt_lock); |
550 | |
551 | if (!HINIC_IS_VF(nic_dev->hwdev->hwif)) |
552 | hinic_notify_all_vfs_link_changed(hwdev: nic_dev->hwdev, link_status: 0); |
553 | |
554 | hinic_port_set_state(nic_dev, state: HINIC_PORT_DISABLE); |
555 | |
556 | hinic_port_set_func_state(nic_dev, state: HINIC_FUNC_PORT_DISABLE); |
557 | |
558 | if (nic_dev->flags & HINIC_RSS_ENABLE) { |
559 | hinic_rss_deinit(nic_dev); |
560 | hinic_rss_template_free(nic_dev, tmpl_idx: nic_dev->rss_tmpl_idx); |
561 | } |
562 | |
563 | free_rxqs(nic_dev); |
564 | free_txqs(nic_dev); |
565 | |
566 | if (flags & HINIC_INTF_UP) |
567 | hinic_hwdev_ifdown(hwdev: nic_dev->hwdev); |
568 | |
569 | netif_info(nic_dev, drv, netdev, "HINIC_INTF is DOWN\n" ); |
570 | return 0; |
571 | } |
572 | |
573 | static int hinic_change_mtu(struct net_device *netdev, int new_mtu) |
574 | { |
575 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
576 | int err; |
577 | |
578 | netif_info(nic_dev, drv, netdev, "set_mtu = %d\n" , new_mtu); |
579 | |
580 | err = hinic_port_set_mtu(nic_dev, new_mtu); |
581 | if (err) |
582 | netif_err(nic_dev, drv, netdev, "Failed to set port mtu\n" ); |
583 | else |
584 | netdev->mtu = new_mtu; |
585 | |
586 | return err; |
587 | } |
588 | |
589 | /** |
590 | * change_mac_addr - change the main mac address of network device |
591 | * @netdev: network device |
592 | * @addr: mac address to set |
593 | * |
594 | * Return 0 - Success, negative - Failure |
595 | **/ |
596 | static int change_mac_addr(struct net_device *netdev, const u8 *addr) |
597 | { |
598 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
599 | u16 vid = 0; |
600 | int err; |
601 | |
602 | if (!is_valid_ether_addr(addr)) |
603 | return -EADDRNOTAVAIL; |
604 | |
605 | netif_info(nic_dev, drv, netdev, "change mac addr = %02x %02x %02x %02x %02x %02x\n" , |
606 | addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]); |
607 | |
608 | down(sem: &nic_dev->mgmt_lock); |
609 | |
610 | do { |
611 | err = hinic_port_del_mac(nic_dev, addr: netdev->dev_addr, vlan_id: vid); |
612 | if (err) { |
613 | netif_err(nic_dev, drv, netdev, |
614 | "Failed to delete mac\n" ); |
615 | break; |
616 | } |
617 | |
618 | err = hinic_port_add_mac(nic_dev, addr, vlan_id: vid); |
619 | if (err) { |
620 | netif_err(nic_dev, drv, netdev, "Failed to add mac\n" ); |
621 | break; |
622 | } |
623 | |
624 | vid = find_next_bit(addr: nic_dev->vlan_bitmap, VLAN_N_VID, offset: vid + 1); |
625 | } while (vid != VLAN_N_VID); |
626 | |
627 | up(sem: &nic_dev->mgmt_lock); |
628 | return err; |
629 | } |
630 | |
631 | static int hinic_set_mac_addr(struct net_device *netdev, void *addr) |
632 | { |
633 | unsigned char new_mac[ETH_ALEN]; |
634 | struct sockaddr *saddr = addr; |
635 | int err; |
636 | |
637 | memcpy(new_mac, saddr->sa_data, ETH_ALEN); |
638 | |
639 | err = change_mac_addr(netdev, addr: new_mac); |
640 | if (!err) |
641 | eth_hw_addr_set(dev: netdev, addr: new_mac); |
642 | |
643 | return err; |
644 | } |
645 | |
646 | /** |
647 | * add_mac_addr - add mac address to network device |
648 | * @netdev: network device |
649 | * @addr: mac address to add |
650 | * |
651 | * Return 0 - Success, negative - Failure |
652 | **/ |
653 | static int add_mac_addr(struct net_device *netdev, const u8 *addr) |
654 | { |
655 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
656 | u16 vid = 0; |
657 | int err; |
658 | |
659 | netif_info(nic_dev, drv, netdev, "set mac addr = %02x %02x %02x %02x %02x %02x\n" , |
660 | addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]); |
661 | |
662 | down(sem: &nic_dev->mgmt_lock); |
663 | |
664 | do { |
665 | err = hinic_port_add_mac(nic_dev, addr, vlan_id: vid); |
666 | if (err) { |
667 | netif_err(nic_dev, drv, netdev, "Failed to add mac\n" ); |
668 | break; |
669 | } |
670 | |
671 | vid = find_next_bit(addr: nic_dev->vlan_bitmap, VLAN_N_VID, offset: vid + 1); |
672 | } while (vid != VLAN_N_VID); |
673 | |
674 | up(sem: &nic_dev->mgmt_lock); |
675 | return err; |
676 | } |
677 | |
678 | /** |
679 | * remove_mac_addr - remove mac address from network device |
680 | * @netdev: network device |
681 | * @addr: mac address to remove |
682 | * |
683 | * Return 0 - Success, negative - Failure |
684 | **/ |
685 | static int remove_mac_addr(struct net_device *netdev, const u8 *addr) |
686 | { |
687 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
688 | u16 vid = 0; |
689 | int err; |
690 | |
691 | if (!is_valid_ether_addr(addr)) |
692 | return -EADDRNOTAVAIL; |
693 | |
694 | netif_info(nic_dev, drv, netdev, "remove mac addr = %02x %02x %02x %02x %02x %02x\n" , |
695 | addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]); |
696 | |
697 | down(sem: &nic_dev->mgmt_lock); |
698 | |
699 | do { |
700 | err = hinic_port_del_mac(nic_dev, addr, vlan_id: vid); |
701 | if (err) { |
702 | netif_err(nic_dev, drv, netdev, |
703 | "Failed to delete mac\n" ); |
704 | break; |
705 | } |
706 | |
707 | vid = find_next_bit(addr: nic_dev->vlan_bitmap, VLAN_N_VID, offset: vid + 1); |
708 | } while (vid != VLAN_N_VID); |
709 | |
710 | up(sem: &nic_dev->mgmt_lock); |
711 | return err; |
712 | } |
713 | |
714 | static int hinic_vlan_rx_add_vid(struct net_device *netdev, |
715 | __always_unused __be16 proto, u16 vid) |
716 | { |
717 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
718 | int ret, err; |
719 | |
720 | netif_info(nic_dev, drv, netdev, "add vid = %d\n" , vid); |
721 | |
722 | down(sem: &nic_dev->mgmt_lock); |
723 | |
724 | err = hinic_port_add_vlan(nic_dev, vlan_id: vid); |
725 | if (err) { |
726 | netif_err(nic_dev, drv, netdev, "Failed to add vlan\n" ); |
727 | goto err_vlan_add; |
728 | } |
729 | |
730 | err = hinic_port_add_mac(nic_dev, addr: netdev->dev_addr, vlan_id: vid); |
731 | if (err && err != HINIC_PF_SET_VF_ALREADY) { |
732 | netif_err(nic_dev, drv, netdev, "Failed to set mac\n" ); |
733 | goto err_add_mac; |
734 | } |
735 | |
736 | bitmap_set(map: nic_dev->vlan_bitmap, start: vid, nbits: 1); |
737 | |
738 | up(sem: &nic_dev->mgmt_lock); |
739 | return 0; |
740 | |
741 | err_add_mac: |
742 | ret = hinic_port_del_vlan(nic_dev, vlan_id: vid); |
743 | if (ret) |
744 | netif_err(nic_dev, drv, netdev, |
745 | "Failed to revert by removing vlan\n" ); |
746 | |
747 | err_vlan_add: |
748 | up(sem: &nic_dev->mgmt_lock); |
749 | return err; |
750 | } |
751 | |
752 | static int hinic_vlan_rx_kill_vid(struct net_device *netdev, |
753 | __always_unused __be16 proto, u16 vid) |
754 | { |
755 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
756 | int err; |
757 | |
758 | netif_info(nic_dev, drv, netdev, "remove vid = %d\n" , vid); |
759 | |
760 | down(sem: &nic_dev->mgmt_lock); |
761 | |
762 | err = hinic_port_del_vlan(nic_dev, vlan_id: vid); |
763 | if (err) { |
764 | netif_err(nic_dev, drv, netdev, "Failed to delete vlan\n" ); |
765 | goto err_del_vlan; |
766 | } |
767 | |
768 | bitmap_clear(map: nic_dev->vlan_bitmap, start: vid, nbits: 1); |
769 | |
770 | up(sem: &nic_dev->mgmt_lock); |
771 | return 0; |
772 | |
773 | err_del_vlan: |
774 | up(sem: &nic_dev->mgmt_lock); |
775 | return err; |
776 | } |
777 | |
778 | static void set_rx_mode(struct work_struct *work) |
779 | { |
780 | struct hinic_rx_mode_work *rx_mode_work = work_to_rx_mode_work(work); |
781 | struct hinic_dev *nic_dev = rx_mode_work_to_nic_dev(rx_mode_work); |
782 | |
783 | hinic_port_set_rx_mode(nic_dev, rx_mode: rx_mode_work->rx_mode); |
784 | |
785 | __dev_uc_sync(dev: nic_dev->netdev, sync: add_mac_addr, unsync: remove_mac_addr); |
786 | __dev_mc_sync(dev: nic_dev->netdev, sync: add_mac_addr, unsync: remove_mac_addr); |
787 | } |
788 | |
789 | static void hinic_set_rx_mode(struct net_device *netdev) |
790 | { |
791 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
792 | struct hinic_rx_mode_work *rx_mode_work; |
793 | u32 rx_mode; |
794 | |
795 | rx_mode_work = &nic_dev->rx_mode_work; |
796 | |
797 | rx_mode = HINIC_RX_MODE_UC | |
798 | HINIC_RX_MODE_MC | |
799 | HINIC_RX_MODE_BC; |
800 | |
801 | if (netdev->flags & IFF_PROMISC) { |
802 | if (!HINIC_IS_VF(nic_dev->hwdev->hwif)) |
803 | rx_mode |= HINIC_RX_MODE_PROMISC; |
804 | } else if (netdev->flags & IFF_ALLMULTI) { |
805 | rx_mode |= HINIC_RX_MODE_MC_ALL; |
806 | } |
807 | |
808 | rx_mode_work->rx_mode = rx_mode; |
809 | |
810 | queue_work(wq: nic_dev->workq, work: &rx_mode_work->work); |
811 | } |
812 | |
813 | static void hinic_tx_timeout(struct net_device *netdev, unsigned int txqueue) |
814 | { |
815 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
816 | u16 sw_pi, hw_ci, sw_ci; |
817 | struct hinic_sq *sq; |
818 | u16 num_sqs, q_id; |
819 | |
820 | num_sqs = hinic_hwdev_num_qps(hwdev: nic_dev->hwdev); |
821 | |
822 | netif_err(nic_dev, drv, netdev, "Tx timeout\n" ); |
823 | |
824 | for (q_id = 0; q_id < num_sqs; q_id++) { |
825 | if (!netif_xmit_stopped(dev_queue: netdev_get_tx_queue(dev: netdev, index: q_id))) |
826 | continue; |
827 | |
828 | sq = hinic_hwdev_get_sq(hwdev: nic_dev->hwdev, i: q_id); |
829 | sw_pi = atomic_read(v: &sq->wq->prod_idx) & sq->wq->mask; |
830 | hw_ci = be16_to_cpu(*(u16 *)(sq->hw_ci_addr)) & sq->wq->mask; |
831 | sw_ci = atomic_read(v: &sq->wq->cons_idx) & sq->wq->mask; |
832 | netif_err(nic_dev, drv, netdev, "Txq%d: sw_pi: %d, hw_ci: %d, sw_ci: %d, napi->state: 0x%lx\n" , |
833 | q_id, sw_pi, hw_ci, sw_ci, |
834 | nic_dev->txqs[q_id].napi.state); |
835 | } |
836 | } |
837 | |
838 | static void hinic_get_stats64(struct net_device *netdev, |
839 | struct rtnl_link_stats64 *stats) |
840 | { |
841 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
842 | struct hinic_rxq_stats nic_rx_stats = {}; |
843 | struct hinic_txq_stats nic_tx_stats = {}; |
844 | |
845 | if (nic_dev->flags & HINIC_INTF_UP) |
846 | gather_nic_stats(nic_dev, nic_rx_stats: &nic_rx_stats, nic_tx_stats: &nic_tx_stats); |
847 | |
848 | stats->rx_bytes = nic_rx_stats.bytes; |
849 | stats->rx_packets = nic_rx_stats.pkts; |
850 | stats->rx_errors = nic_rx_stats.errors; |
851 | |
852 | stats->tx_bytes = nic_tx_stats.bytes; |
853 | stats->tx_packets = nic_tx_stats.pkts; |
854 | stats->tx_errors = nic_tx_stats.tx_dropped; |
855 | } |
856 | |
857 | static int hinic_set_features(struct net_device *netdev, |
858 | netdev_features_t features) |
859 | { |
860 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
861 | |
862 | return set_features(nic_dev, pre_features: nic_dev->netdev->features, |
863 | features, force_change: false); |
864 | } |
865 | |
866 | static netdev_features_t hinic_fix_features(struct net_device *netdev, |
867 | netdev_features_t features) |
868 | { |
869 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
870 | |
871 | /* If Rx checksum is disabled, then LRO should also be disabled */ |
872 | if (!(features & NETIF_F_RXCSUM)) { |
873 | netif_info(nic_dev, drv, netdev, "disabling LRO as RXCSUM is off\n" ); |
874 | features &= ~NETIF_F_LRO; |
875 | } |
876 | |
877 | return features; |
878 | } |
879 | |
880 | static const struct net_device_ops hinic_netdev_ops = { |
881 | .ndo_open = hinic_open, |
882 | .ndo_stop = hinic_close, |
883 | .ndo_change_mtu = hinic_change_mtu, |
884 | .ndo_set_mac_address = hinic_set_mac_addr, |
885 | .ndo_validate_addr = eth_validate_addr, |
886 | .ndo_vlan_rx_add_vid = hinic_vlan_rx_add_vid, |
887 | .ndo_vlan_rx_kill_vid = hinic_vlan_rx_kill_vid, |
888 | .ndo_set_rx_mode = hinic_set_rx_mode, |
889 | .ndo_start_xmit = hinic_xmit_frame, |
890 | .ndo_tx_timeout = hinic_tx_timeout, |
891 | .ndo_get_stats64 = hinic_get_stats64, |
892 | .ndo_fix_features = hinic_fix_features, |
893 | .ndo_set_features = hinic_set_features, |
894 | .ndo_set_vf_mac = hinic_ndo_set_vf_mac, |
895 | .ndo_set_vf_vlan = hinic_ndo_set_vf_vlan, |
896 | .ndo_get_vf_config = hinic_ndo_get_vf_config, |
897 | .ndo_set_vf_trust = hinic_ndo_set_vf_trust, |
898 | .ndo_set_vf_rate = hinic_ndo_set_vf_bw, |
899 | .ndo_set_vf_spoofchk = hinic_ndo_set_vf_spoofchk, |
900 | .ndo_set_vf_link_state = hinic_ndo_set_vf_link_state, |
901 | }; |
902 | |
903 | static const struct net_device_ops hinicvf_netdev_ops = { |
904 | .ndo_open = hinic_open, |
905 | .ndo_stop = hinic_close, |
906 | .ndo_change_mtu = hinic_change_mtu, |
907 | .ndo_set_mac_address = hinic_set_mac_addr, |
908 | .ndo_validate_addr = eth_validate_addr, |
909 | .ndo_vlan_rx_add_vid = hinic_vlan_rx_add_vid, |
910 | .ndo_vlan_rx_kill_vid = hinic_vlan_rx_kill_vid, |
911 | .ndo_set_rx_mode = hinic_set_rx_mode, |
912 | .ndo_start_xmit = hinic_xmit_frame, |
913 | .ndo_tx_timeout = hinic_tx_timeout, |
914 | .ndo_get_stats64 = hinic_get_stats64, |
915 | .ndo_fix_features = hinic_fix_features, |
916 | .ndo_set_features = hinic_set_features, |
917 | }; |
918 | |
919 | static void netdev_features_init(struct net_device *netdev) |
920 | { |
921 | netdev->hw_features = NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_IP_CSUM | |
922 | NETIF_F_IPV6_CSUM | NETIF_F_TSO | NETIF_F_TSO6 | |
923 | NETIF_F_RXCSUM | NETIF_F_LRO | |
924 | NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | |
925 | NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_UDP_TUNNEL_CSUM; |
926 | |
927 | netdev->vlan_features = netdev->hw_features; |
928 | |
929 | netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; |
930 | |
931 | netdev->hw_enc_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SCTP_CRC | |
932 | NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN | |
933 | NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_UDP_TUNNEL; |
934 | } |
935 | |
936 | static void hinic_refresh_nic_cfg(struct hinic_dev *nic_dev) |
937 | { |
938 | struct hinic_nic_cfg *nic_cfg = &nic_dev->hwdev->func_to_io.nic_cfg; |
939 | struct hinic_pause_config pause_info = {0}; |
940 | struct hinic_port_cap port_cap = {0}; |
941 | |
942 | if (hinic_port_get_cap(nic_dev, port_cap: &port_cap)) |
943 | return; |
944 | |
945 | mutex_lock(&nic_cfg->cfg_mutex); |
946 | if (nic_cfg->pause_set || !port_cap.autoneg_state) { |
947 | nic_cfg->auto_neg = port_cap.autoneg_state; |
948 | pause_info.auto_neg = nic_cfg->auto_neg; |
949 | pause_info.rx_pause = nic_cfg->rx_pause; |
950 | pause_info.tx_pause = nic_cfg->tx_pause; |
951 | hinic_set_hw_pause_info(hwdev: nic_dev->hwdev, pause_info: &pause_info); |
952 | } |
953 | mutex_unlock(lock: &nic_cfg->cfg_mutex); |
954 | } |
955 | |
956 | /** |
957 | * link_status_event_handler - link event handler |
958 | * @handle: nic device for the handler |
959 | * @buf_in: input buffer |
960 | * @in_size: input size |
961 | * @buf_out: output buffer |
962 | * @out_size: returned output size |
963 | **/ |
964 | static void link_status_event_handler(void *handle, void *buf_in, u16 in_size, |
965 | void *buf_out, u16 *out_size) |
966 | { |
967 | struct hinic_port_link_status *link_status, *ret_link_status; |
968 | struct hinic_dev *nic_dev = handle; |
969 | |
970 | link_status = buf_in; |
971 | |
972 | if (link_status->link == HINIC_LINK_STATE_UP) { |
973 | down(sem: &nic_dev->mgmt_lock); |
974 | |
975 | nic_dev->flags |= HINIC_LINK_UP; |
976 | nic_dev->cable_unplugged = false; |
977 | nic_dev->module_unrecognized = false; |
978 | |
979 | if ((nic_dev->flags & (HINIC_LINK_UP | HINIC_INTF_UP)) == |
980 | (HINIC_LINK_UP | HINIC_INTF_UP)) { |
981 | netif_carrier_on(dev: nic_dev->netdev); |
982 | netif_tx_wake_all_queues(dev: nic_dev->netdev); |
983 | } |
984 | |
985 | up(sem: &nic_dev->mgmt_lock); |
986 | |
987 | if (!HINIC_IS_VF(nic_dev->hwdev->hwif)) |
988 | hinic_refresh_nic_cfg(nic_dev); |
989 | |
990 | netif_info(nic_dev, drv, nic_dev->netdev, "HINIC_Link is UP\n" ); |
991 | } else { |
992 | down(sem: &nic_dev->mgmt_lock); |
993 | |
994 | nic_dev->flags &= ~HINIC_LINK_UP; |
995 | |
996 | netif_carrier_off(dev: nic_dev->netdev); |
997 | netif_tx_disable(dev: nic_dev->netdev); |
998 | |
999 | up(sem: &nic_dev->mgmt_lock); |
1000 | |
1001 | netif_info(nic_dev, drv, nic_dev->netdev, "HINIC_Link is DOWN\n" ); |
1002 | } |
1003 | |
1004 | if (!HINIC_IS_VF(nic_dev->hwdev->hwif)) |
1005 | hinic_notify_all_vfs_link_changed(hwdev: nic_dev->hwdev, |
1006 | link_status: link_status->link); |
1007 | |
1008 | ret_link_status = buf_out; |
1009 | ret_link_status->status = 0; |
1010 | |
1011 | *out_size = sizeof(*ret_link_status); |
1012 | } |
1013 | |
1014 | static void cable_plug_event(void *handle, |
1015 | void *buf_in, u16 in_size, |
1016 | void *buf_out, u16 *out_size) |
1017 | { |
1018 | struct hinic_cable_plug_event *plug_event = buf_in; |
1019 | struct hinic_dev *nic_dev = handle; |
1020 | |
1021 | nic_dev->cable_unplugged = plug_event->plugged ? false : true; |
1022 | |
1023 | *out_size = sizeof(*plug_event); |
1024 | plug_event = buf_out; |
1025 | plug_event->status = 0; |
1026 | } |
1027 | |
1028 | static void link_err_event(void *handle, |
1029 | void *buf_in, u16 in_size, |
1030 | void *buf_out, u16 *out_size) |
1031 | { |
1032 | struct hinic_link_err_event *link_err = buf_in; |
1033 | struct hinic_dev *nic_dev = handle; |
1034 | |
1035 | if (link_err->err_type >= LINK_ERR_NUM) |
1036 | netif_info(nic_dev, link, nic_dev->netdev, |
1037 | "Link failed, Unknown error type: 0x%x\n" , |
1038 | link_err->err_type); |
1039 | else |
1040 | nic_dev->module_unrecognized = true; |
1041 | |
1042 | *out_size = sizeof(*link_err); |
1043 | link_err = buf_out; |
1044 | link_err->status = 0; |
1045 | } |
1046 | |
1047 | static int set_features(struct hinic_dev *nic_dev, |
1048 | netdev_features_t pre_features, |
1049 | netdev_features_t features, bool force_change) |
1050 | { |
1051 | netdev_features_t changed = force_change ? ~0 : pre_features ^ features; |
1052 | u32 csum_en = HINIC_RX_CSUM_OFFLOAD_EN; |
1053 | netdev_features_t failed_features = 0; |
1054 | int ret = 0; |
1055 | int err = 0; |
1056 | |
1057 | if (changed & NETIF_F_TSO) { |
1058 | ret = hinic_port_set_tso(nic_dev, state: (features & NETIF_F_TSO) ? |
1059 | HINIC_TSO_ENABLE : HINIC_TSO_DISABLE); |
1060 | if (ret) { |
1061 | err = ret; |
1062 | failed_features |= NETIF_F_TSO; |
1063 | } |
1064 | } |
1065 | |
1066 | if (changed & NETIF_F_RXCSUM) { |
1067 | ret = hinic_set_rx_csum_offload(nic_dev, en: csum_en); |
1068 | if (ret) { |
1069 | err = ret; |
1070 | failed_features |= NETIF_F_RXCSUM; |
1071 | } |
1072 | } |
1073 | |
1074 | if (changed & NETIF_F_LRO) { |
1075 | ret = hinic_set_rx_lro_state(nic_dev, |
1076 | lro_en: !!(features & NETIF_F_LRO), |
1077 | HINIC_LRO_RX_TIMER_DEFAULT, |
1078 | HINIC_LRO_MAX_WQE_NUM_DEFAULT); |
1079 | if (ret) { |
1080 | err = ret; |
1081 | failed_features |= NETIF_F_LRO; |
1082 | } |
1083 | } |
1084 | |
1085 | if (changed & NETIF_F_HW_VLAN_CTAG_RX) { |
1086 | ret = hinic_set_rx_vlan_offload(nic_dev, |
1087 | en: !!(features & |
1088 | NETIF_F_HW_VLAN_CTAG_RX)); |
1089 | if (ret) { |
1090 | err = ret; |
1091 | failed_features |= NETIF_F_HW_VLAN_CTAG_RX; |
1092 | } |
1093 | } |
1094 | |
1095 | if (changed & NETIF_F_HW_VLAN_CTAG_FILTER) { |
1096 | ret = hinic_set_vlan_fliter(nic_dev, |
1097 | en: !!(features & |
1098 | NETIF_F_HW_VLAN_CTAG_FILTER)); |
1099 | if (ret) { |
1100 | err = ret; |
1101 | failed_features |= NETIF_F_HW_VLAN_CTAG_FILTER; |
1102 | } |
1103 | } |
1104 | |
1105 | if (err) { |
1106 | nic_dev->netdev->features = features ^ failed_features; |
1107 | return -EIO; |
1108 | } |
1109 | |
1110 | return 0; |
1111 | } |
1112 | |
1113 | static int hinic_init_intr_coalesce(struct hinic_dev *nic_dev) |
1114 | { |
1115 | u64 size; |
1116 | u16 i; |
1117 | |
1118 | size = sizeof(struct hinic_intr_coal_info) * nic_dev->max_qps; |
1119 | nic_dev->rx_intr_coalesce = kzalloc(size, GFP_KERNEL); |
1120 | if (!nic_dev->rx_intr_coalesce) |
1121 | return -ENOMEM; |
1122 | nic_dev->tx_intr_coalesce = kzalloc(size, GFP_KERNEL); |
1123 | if (!nic_dev->tx_intr_coalesce) { |
1124 | kfree(objp: nic_dev->rx_intr_coalesce); |
1125 | return -ENOMEM; |
1126 | } |
1127 | |
1128 | for (i = 0; i < nic_dev->max_qps; i++) { |
1129 | nic_dev->rx_intr_coalesce[i].pending_limt = |
1130 | HINIC_DEAULT_TXRX_MSIX_PENDING_LIMIT; |
1131 | nic_dev->rx_intr_coalesce[i].coalesce_timer_cfg = |
1132 | HINIC_DEAULT_TXRX_MSIX_COALESC_TIMER_CFG; |
1133 | nic_dev->rx_intr_coalesce[i].resend_timer_cfg = |
1134 | HINIC_DEAULT_TXRX_MSIX_RESEND_TIMER_CFG; |
1135 | nic_dev->tx_intr_coalesce[i].pending_limt = |
1136 | HINIC_DEAULT_TXRX_MSIX_PENDING_LIMIT; |
1137 | nic_dev->tx_intr_coalesce[i].coalesce_timer_cfg = |
1138 | HINIC_DEAULT_TXRX_MSIX_COALESC_TIMER_CFG; |
1139 | nic_dev->tx_intr_coalesce[i].resend_timer_cfg = |
1140 | HINIC_DEAULT_TXRX_MSIX_RESEND_TIMER_CFG; |
1141 | } |
1142 | |
1143 | return 0; |
1144 | } |
1145 | |
1146 | static void hinic_free_intr_coalesce(struct hinic_dev *nic_dev) |
1147 | { |
1148 | kfree(objp: nic_dev->tx_intr_coalesce); |
1149 | kfree(objp: nic_dev->rx_intr_coalesce); |
1150 | } |
1151 | |
1152 | /** |
1153 | * nic_dev_init - Initialize the NIC device |
1154 | * @pdev: the NIC pci device |
1155 | * |
1156 | * Return 0 - Success, negative - Failure |
1157 | **/ |
1158 | static int nic_dev_init(struct pci_dev *pdev) |
1159 | { |
1160 | struct hinic_rx_mode_work *rx_mode_work; |
1161 | struct hinic_dev *nic_dev; |
1162 | struct net_device *netdev; |
1163 | struct hinic_hwdev *hwdev; |
1164 | struct devlink *devlink; |
1165 | u8 addr[ETH_ALEN]; |
1166 | int err, num_qps; |
1167 | |
1168 | devlink = hinic_devlink_alloc(dev: &pdev->dev); |
1169 | if (!devlink) { |
1170 | dev_err(&pdev->dev, "Hinic devlink alloc failed\n" ); |
1171 | return -ENOMEM; |
1172 | } |
1173 | |
1174 | hwdev = hinic_init_hwdev(pdev, devlink); |
1175 | if (IS_ERR(ptr: hwdev)) { |
1176 | dev_err(&pdev->dev, "Failed to initialize HW device\n" ); |
1177 | hinic_devlink_free(devlink); |
1178 | return PTR_ERR(ptr: hwdev); |
1179 | } |
1180 | |
1181 | num_qps = hinic_hwdev_num_qps(hwdev); |
1182 | if (num_qps <= 0) { |
1183 | dev_err(&pdev->dev, "Invalid number of QPS\n" ); |
1184 | err = -EINVAL; |
1185 | goto err_num_qps; |
1186 | } |
1187 | |
1188 | netdev = alloc_etherdev_mq(sizeof(*nic_dev), num_qps); |
1189 | if (!netdev) { |
1190 | dev_err(&pdev->dev, "Failed to allocate Ethernet device\n" ); |
1191 | err = -ENOMEM; |
1192 | goto err_alloc_etherdev; |
1193 | } |
1194 | |
1195 | if (!HINIC_IS_VF(hwdev->hwif)) |
1196 | netdev->netdev_ops = &hinic_netdev_ops; |
1197 | else |
1198 | netdev->netdev_ops = &hinicvf_netdev_ops; |
1199 | |
1200 | netdev->max_mtu = HINIC_MAX_MTU_SIZE; |
1201 | netdev->min_mtu = HINIC_MIN_MTU_SIZE; |
1202 | |
1203 | nic_dev = netdev_priv(dev: netdev); |
1204 | nic_dev->netdev = netdev; |
1205 | nic_dev->hwdev = hwdev; |
1206 | nic_dev->msg_enable = MSG_ENABLE_DEFAULT; |
1207 | nic_dev->flags = 0; |
1208 | nic_dev->txqs = NULL; |
1209 | nic_dev->rxqs = NULL; |
1210 | nic_dev->tx_weight = tx_weight; |
1211 | nic_dev->rx_weight = rx_weight; |
1212 | nic_dev->sq_depth = HINIC_SQ_DEPTH; |
1213 | nic_dev->rq_depth = HINIC_RQ_DEPTH; |
1214 | nic_dev->sriov_info.hwdev = hwdev; |
1215 | nic_dev->sriov_info.pdev = pdev; |
1216 | nic_dev->max_qps = num_qps; |
1217 | nic_dev->devlink = devlink; |
1218 | |
1219 | hinic_set_ethtool_ops(netdev); |
1220 | |
1221 | sema_init(sem: &nic_dev->mgmt_lock, val: 1); |
1222 | |
1223 | nic_dev->vlan_bitmap = devm_bitmap_zalloc(dev: &pdev->dev, VLAN_N_VID, |
1224 | GFP_KERNEL); |
1225 | if (!nic_dev->vlan_bitmap) { |
1226 | err = -ENOMEM; |
1227 | goto err_vlan_bitmap; |
1228 | } |
1229 | |
1230 | nic_dev->workq = create_singlethread_workqueue(HINIC_WQ_NAME); |
1231 | if (!nic_dev->workq) { |
1232 | err = -ENOMEM; |
1233 | goto err_workq; |
1234 | } |
1235 | |
1236 | pci_set_drvdata(pdev, data: netdev); |
1237 | |
1238 | err = hinic_port_get_mac(nic_dev, addr); |
1239 | if (err) { |
1240 | dev_err(&pdev->dev, "Failed to get mac address\n" ); |
1241 | goto err_get_mac; |
1242 | } |
1243 | eth_hw_addr_set(dev: netdev, addr); |
1244 | |
1245 | if (!is_valid_ether_addr(addr: netdev->dev_addr)) { |
1246 | if (!HINIC_IS_VF(nic_dev->hwdev->hwif)) { |
1247 | dev_err(&pdev->dev, "Invalid MAC address\n" ); |
1248 | err = -EIO; |
1249 | goto err_add_mac; |
1250 | } |
1251 | |
1252 | dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n" , |
1253 | netdev->dev_addr); |
1254 | eth_hw_addr_random(dev: netdev); |
1255 | } |
1256 | |
1257 | err = hinic_port_add_mac(nic_dev, addr: netdev->dev_addr, vlan_id: 0); |
1258 | if (err && err != HINIC_PF_SET_VF_ALREADY) { |
1259 | dev_err(&pdev->dev, "Failed to add mac\n" ); |
1260 | goto err_add_mac; |
1261 | } |
1262 | |
1263 | err = hinic_port_set_mtu(nic_dev, new_mtu: netdev->mtu); |
1264 | if (err) { |
1265 | dev_err(&pdev->dev, "Failed to set mtu\n" ); |
1266 | goto err_set_mtu; |
1267 | } |
1268 | |
1269 | rx_mode_work = &nic_dev->rx_mode_work; |
1270 | INIT_WORK(&rx_mode_work->work, set_rx_mode); |
1271 | |
1272 | netdev_features_init(netdev); |
1273 | |
1274 | netif_carrier_off(dev: netdev); |
1275 | |
1276 | hinic_hwdev_cb_register(hwdev: nic_dev->hwdev, cmd: HINIC_MGMT_MSG_CMD_LINK_STATUS, |
1277 | handle: nic_dev, handler: link_status_event_handler); |
1278 | hinic_hwdev_cb_register(hwdev: nic_dev->hwdev, |
1279 | cmd: HINIC_MGMT_MSG_CMD_CABLE_PLUG_EVENT, |
1280 | handle: nic_dev, handler: cable_plug_event); |
1281 | hinic_hwdev_cb_register(hwdev: nic_dev->hwdev, |
1282 | cmd: HINIC_MGMT_MSG_CMD_LINK_ERR_EVENT, |
1283 | handle: nic_dev, handler: link_err_event); |
1284 | |
1285 | err = set_features(nic_dev, pre_features: 0, features: nic_dev->netdev->features, force_change: true); |
1286 | if (err) |
1287 | goto err_set_features; |
1288 | |
1289 | /* enable pause and disable pfc by default */ |
1290 | err = hinic_dcb_set_pfc(hwdev: nic_dev->hwdev, pfc_en: 0, pfc_bitmap: 0); |
1291 | if (err) |
1292 | goto err_set_pfc; |
1293 | |
1294 | SET_NETDEV_DEV(netdev, &pdev->dev); |
1295 | |
1296 | err = hinic_init_intr_coalesce(nic_dev); |
1297 | if (err) { |
1298 | dev_err(&pdev->dev, "Failed to init_intr_coalesce\n" ); |
1299 | goto err_init_intr; |
1300 | } |
1301 | |
1302 | hinic_dbg_init(nic_dev); |
1303 | |
1304 | hinic_func_tbl_dbgfs_init(nic_dev); |
1305 | |
1306 | err = hinic_func_table_debug_add(dev: nic_dev); |
1307 | if (err) { |
1308 | dev_err(&pdev->dev, "Failed to add func_table debug\n" ); |
1309 | goto err_add_func_table_dbg; |
1310 | } |
1311 | |
1312 | err = register_netdev(dev: netdev); |
1313 | if (err) { |
1314 | dev_err(&pdev->dev, "Failed to register netdev\n" ); |
1315 | goto err_reg_netdev; |
1316 | } |
1317 | |
1318 | return 0; |
1319 | |
1320 | err_reg_netdev: |
1321 | hinic_func_table_debug_rem(dev: nic_dev); |
1322 | err_add_func_table_dbg: |
1323 | hinic_func_tbl_dbgfs_uninit(nic_dev); |
1324 | hinic_dbg_uninit(nic_dev); |
1325 | hinic_free_intr_coalesce(nic_dev); |
1326 | err_init_intr: |
1327 | err_set_pfc: |
1328 | err_set_features: |
1329 | hinic_hwdev_cb_unregister(hwdev: nic_dev->hwdev, |
1330 | cmd: HINIC_MGMT_MSG_CMD_LINK_ERR_EVENT); |
1331 | hinic_hwdev_cb_unregister(hwdev: nic_dev->hwdev, |
1332 | cmd: HINIC_MGMT_MSG_CMD_CABLE_PLUG_EVENT); |
1333 | hinic_hwdev_cb_unregister(hwdev: nic_dev->hwdev, |
1334 | cmd: HINIC_MGMT_MSG_CMD_LINK_STATUS); |
1335 | cancel_work_sync(work: &rx_mode_work->work); |
1336 | |
1337 | err_set_mtu: |
1338 | hinic_port_del_mac(nic_dev, addr: netdev->dev_addr, vlan_id: 0); |
1339 | err_add_mac: |
1340 | err_get_mac: |
1341 | pci_set_drvdata(pdev, NULL); |
1342 | destroy_workqueue(wq: nic_dev->workq); |
1343 | err_workq: |
1344 | err_vlan_bitmap: |
1345 | free_netdev(dev: netdev); |
1346 | |
1347 | err_alloc_etherdev: |
1348 | err_num_qps: |
1349 | hinic_free_hwdev(hwdev); |
1350 | hinic_devlink_free(devlink); |
1351 | return err; |
1352 | } |
1353 | |
1354 | static int hinic_probe(struct pci_dev *pdev, |
1355 | const struct pci_device_id *id) |
1356 | { |
1357 | int err = pci_enable_device(dev: pdev); |
1358 | |
1359 | if (err) |
1360 | return dev_err_probe(dev: &pdev->dev, err, fmt: "Failed to enable PCI device\n" ); |
1361 | |
1362 | err = pci_request_regions(pdev, HINIC_DRV_NAME); |
1363 | if (err) { |
1364 | dev_err(&pdev->dev, "Failed to request PCI regions\n" ); |
1365 | goto err_pci_regions; |
1366 | } |
1367 | |
1368 | pci_set_master(dev: pdev); |
1369 | |
1370 | err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64)); |
1371 | if (err) { |
1372 | dev_err(&pdev->dev, "Failed to set DMA mask\n" ); |
1373 | goto err_dma_mask; |
1374 | } |
1375 | |
1376 | err = nic_dev_init(pdev); |
1377 | if (err) { |
1378 | dev_err(&pdev->dev, "Failed to initialize NIC device\n" ); |
1379 | goto err_nic_dev_init; |
1380 | } |
1381 | |
1382 | dev_info(&pdev->dev, "HiNIC driver - probed\n" ); |
1383 | return 0; |
1384 | |
1385 | err_nic_dev_init: |
1386 | err_dma_mask: |
1387 | pci_release_regions(pdev); |
1388 | |
1389 | err_pci_regions: |
1390 | pci_disable_device(dev: pdev); |
1391 | return err; |
1392 | } |
1393 | |
1394 | static void wait_sriov_cfg_complete(struct hinic_dev *nic_dev) |
1395 | { |
1396 | struct hinic_sriov_info *sriov_info = &nic_dev->sriov_info; |
1397 | u32 loop_cnt = 0; |
1398 | |
1399 | set_bit(nr: HINIC_FUNC_REMOVE, addr: &sriov_info->state); |
1400 | usleep_range(min: 9900, max: 10000); |
1401 | |
1402 | while (loop_cnt < HINIC_WAIT_SRIOV_CFG_TIMEOUT) { |
1403 | if (!test_bit(HINIC_SRIOV_ENABLE, &sriov_info->state) && |
1404 | !test_bit(HINIC_SRIOV_DISABLE, &sriov_info->state)) |
1405 | return; |
1406 | |
1407 | usleep_range(min: 9900, max: 10000); |
1408 | loop_cnt++; |
1409 | } |
1410 | } |
1411 | |
1412 | static void hinic_remove(struct pci_dev *pdev) |
1413 | { |
1414 | struct net_device *netdev = pci_get_drvdata(pdev); |
1415 | struct hinic_dev *nic_dev = netdev_priv(dev: netdev); |
1416 | struct devlink *devlink = nic_dev->devlink; |
1417 | struct hinic_rx_mode_work *rx_mode_work; |
1418 | |
1419 | if (!HINIC_IS_VF(nic_dev->hwdev->hwif)) { |
1420 | wait_sriov_cfg_complete(nic_dev); |
1421 | hinic_pci_sriov_disable(dev: pdev); |
1422 | } |
1423 | |
1424 | unregister_netdev(dev: netdev); |
1425 | |
1426 | hinic_func_table_debug_rem(dev: nic_dev); |
1427 | |
1428 | hinic_func_tbl_dbgfs_uninit(nic_dev); |
1429 | |
1430 | hinic_dbg_uninit(nic_dev); |
1431 | |
1432 | hinic_free_intr_coalesce(nic_dev); |
1433 | |
1434 | hinic_port_del_mac(nic_dev, addr: netdev->dev_addr, vlan_id: 0); |
1435 | |
1436 | hinic_hwdev_cb_unregister(hwdev: nic_dev->hwdev, |
1437 | cmd: HINIC_MGMT_MSG_CMD_LINK_ERR_EVENT); |
1438 | hinic_hwdev_cb_unregister(hwdev: nic_dev->hwdev, |
1439 | cmd: HINIC_MGMT_MSG_CMD_CABLE_PLUG_EVENT); |
1440 | hinic_hwdev_cb_unregister(hwdev: nic_dev->hwdev, |
1441 | cmd: HINIC_MGMT_MSG_CMD_LINK_STATUS); |
1442 | |
1443 | rx_mode_work = &nic_dev->rx_mode_work; |
1444 | cancel_work_sync(work: &rx_mode_work->work); |
1445 | |
1446 | pci_set_drvdata(pdev, NULL); |
1447 | |
1448 | destroy_workqueue(wq: nic_dev->workq); |
1449 | |
1450 | hinic_free_hwdev(hwdev: nic_dev->hwdev); |
1451 | |
1452 | free_netdev(dev: netdev); |
1453 | |
1454 | hinic_devlink_free(devlink); |
1455 | |
1456 | pci_release_regions(pdev); |
1457 | pci_disable_device(dev: pdev); |
1458 | |
1459 | dev_info(&pdev->dev, "HiNIC driver - removed\n" ); |
1460 | } |
1461 | |
1462 | static void hinic_shutdown(struct pci_dev *pdev) |
1463 | { |
1464 | pci_disable_device(dev: pdev); |
1465 | } |
1466 | |
1467 | static const struct pci_device_id hinic_pci_table[] = { |
1468 | { PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_QUAD_PORT_25GE), 0}, |
1469 | { PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_DUAL_PORT_100GE), 0}, |
1470 | { PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_DUAL_PORT_100GE_MEZZ), 0}, |
1471 | { PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_QUAD_PORT_25GE_MEZZ), 0}, |
1472 | { PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_VF), 0}, |
1473 | { 0, 0} |
1474 | }; |
1475 | MODULE_DEVICE_TABLE(pci, hinic_pci_table); |
1476 | |
1477 | static struct pci_driver hinic_driver = { |
1478 | .name = HINIC_DRV_NAME, |
1479 | .id_table = hinic_pci_table, |
1480 | .probe = hinic_probe, |
1481 | .remove = hinic_remove, |
1482 | .shutdown = hinic_shutdown, |
1483 | .sriov_configure = hinic_pci_sriov_configure, |
1484 | }; |
1485 | |
1486 | static int __init hinic_module_init(void) |
1487 | { |
1488 | int ret; |
1489 | |
1490 | hinic_dbg_register_debugfs(HINIC_DRV_NAME); |
1491 | |
1492 | ret = pci_register_driver(&hinic_driver); |
1493 | if (ret) |
1494 | hinic_dbg_unregister_debugfs(); |
1495 | |
1496 | return ret; |
1497 | } |
1498 | |
1499 | static void __exit hinic_module_exit(void) |
1500 | { |
1501 | pci_unregister_driver(dev: &hinic_driver); |
1502 | hinic_dbg_unregister_debugfs(); |
1503 | } |
1504 | |
1505 | module_init(hinic_module_init); |
1506 | module_exit(hinic_module_exit); |
1507 | |