1// SPDX-License-Identifier: GPL-2.0-only
2/* Atlantic Network Driver
3 *
4 * Copyright (C) 2014-2019 aQuantia Corporation
5 * Copyright (C) 2019-2020 Marvell International Ltd.
6 */
7
8/* File aq_main.c: Main file for aQuantia Linux driver. */
9
10#include "aq_main.h"
11#include "aq_nic.h"
12#include "aq_pci_func.h"
13#include "aq_ethtool.h"
14#include "aq_ptp.h"
15#include "aq_filters.h"
16#include "aq_hw_utils.h"
17#include "aq_vec.h"
18
19#include <linux/netdevice.h>
20#include <linux/module.h>
21#include <linux/ip.h>
22#include <linux/udp.h>
23#include <net/pkt_cls.h>
24#include <net/pkt_sched.h>
25#include <linux/filter.h>
26
27MODULE_LICENSE("GPL v2");
28MODULE_AUTHOR(AQ_CFG_DRV_AUTHOR);
29MODULE_DESCRIPTION(AQ_CFG_DRV_DESC);
30
31DEFINE_STATIC_KEY_FALSE(aq_xdp_locking_key);
32EXPORT_SYMBOL(aq_xdp_locking_key);
33
34static const char aq_ndev_driver_name[] = AQ_CFG_DRV_NAME;
35
36static const struct net_device_ops aq_ndev_ops;
37
38static struct workqueue_struct *aq_ndev_wq;
39
40void aq_ndev_schedule_work(struct work_struct *work)
41{
42 queue_work(wq: aq_ndev_wq, work);
43}
44
45struct net_device *aq_ndev_alloc(void)
46{
47 struct net_device *ndev = NULL;
48 struct aq_nic_s *aq_nic = NULL;
49
50 ndev = alloc_etherdev_mq(sizeof(struct aq_nic_s), AQ_HW_QUEUES_MAX);
51 if (!ndev)
52 return NULL;
53
54 aq_nic = netdev_priv(dev: ndev);
55 aq_nic->ndev = ndev;
56 ndev->netdev_ops = &aq_ndev_ops;
57 ndev->ethtool_ops = &aq_ethtool_ops;
58
59 return ndev;
60}
61
62int aq_ndev_open(struct net_device *ndev)
63{
64 struct aq_nic_s *aq_nic = netdev_priv(dev: ndev);
65 int err = 0;
66
67 err = aq_nic_init(self: aq_nic);
68 if (err < 0)
69 goto err_exit;
70
71 err = aq_reapply_rxnfc_all_rules(aq_nic);
72 if (err < 0)
73 goto err_exit;
74
75 err = aq_filters_vlans_update(aq_nic);
76 if (err < 0)
77 goto err_exit;
78
79 err = aq_nic_start(self: aq_nic);
80 if (err < 0) {
81 aq_nic_stop(self: aq_nic);
82 goto err_exit;
83 }
84
85err_exit:
86 if (err < 0)
87 aq_nic_deinit(self: aq_nic, link_down: true);
88
89 return err;
90}
91
92int aq_ndev_close(struct net_device *ndev)
93{
94 struct aq_nic_s *aq_nic = netdev_priv(dev: ndev);
95 int err = 0;
96
97 err = aq_nic_stop(self: aq_nic);
98 aq_nic_deinit(self: aq_nic, link_down: true);
99
100 return err;
101}
102
103static netdev_tx_t aq_ndev_start_xmit(struct sk_buff *skb, struct net_device *ndev)
104{
105 struct aq_nic_s *aq_nic = netdev_priv(dev: ndev);
106
107#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
108 if (unlikely(aq_utils_obj_test(&aq_nic->flags, AQ_NIC_PTP_DPATH_UP))) {
109 /* Hardware adds the Timestamp for PTPv2 802.AS1
110 * and PTPv2 IPv4 UDP.
111 * We have to push even general 320 port messages to the ptp
112 * queue explicitly. This is a limitation of current firmware
113 * and hardware PTP design of the chip. Otherwise ptp stream
114 * will fail to sync
115 */
116 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) ||
117 unlikely((ip_hdr(skb)->version == 4) &&
118 (ip_hdr(skb)->protocol == IPPROTO_UDP) &&
119 ((udp_hdr(skb)->dest == htons(319)) ||
120 (udp_hdr(skb)->dest == htons(320)))) ||
121 unlikely(eth_hdr(skb)->h_proto == htons(ETH_P_1588)))
122 return aq_ptp_xmit(aq_nic, skb);
123 }
124#endif
125
126 skb_tx_timestamp(skb);
127 return aq_nic_xmit(self: aq_nic, skb);
128}
129
130static int aq_ndev_change_mtu(struct net_device *ndev, int new_mtu)
131{
132 int new_frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN;
133 struct aq_nic_s *aq_nic = netdev_priv(dev: ndev);
134 struct bpf_prog *prog;
135 int err;
136
137 prog = READ_ONCE(aq_nic->xdp_prog);
138 if (prog && !prog->aux->xdp_has_frags &&
139 new_frame_size > AQ_CFG_RX_FRAME_MAX) {
140 netdev_err(dev: ndev, format: "Illegal MTU %d for XDP prog without frags\n",
141 ndev->mtu);
142 return -EOPNOTSUPP;
143 }
144
145 err = aq_nic_set_mtu(self: aq_nic, new_mtu: new_mtu + ETH_HLEN);
146
147 if (err < 0)
148 goto err_exit;
149 ndev->mtu = new_mtu;
150
151err_exit:
152 return err;
153}
154
155static int aq_ndev_set_features(struct net_device *ndev,
156 netdev_features_t features)
157{
158 bool is_vlan_tx_insert = !!(features & NETIF_F_HW_VLAN_CTAG_TX);
159 bool is_vlan_rx_strip = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
160 struct aq_nic_s *aq_nic = netdev_priv(dev: ndev);
161 bool need_ndev_restart = false;
162 struct aq_nic_cfg_s *aq_cfg;
163 bool is_lro = false;
164 int err = 0;
165
166 aq_cfg = aq_nic_get_cfg(self: aq_nic);
167
168 if (!(features & NETIF_F_NTUPLE)) {
169 if (aq_nic->ndev->features & NETIF_F_NTUPLE) {
170 err = aq_clear_rxnfc_all_rules(aq_nic);
171 if (unlikely(err))
172 goto err_exit;
173 }
174 }
175 if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER)) {
176 if (aq_nic->ndev->features & NETIF_F_HW_VLAN_CTAG_FILTER) {
177 err = aq_filters_vlan_offload_off(aq_nic);
178 if (unlikely(err))
179 goto err_exit;
180 }
181 }
182
183 aq_cfg->features = features;
184
185 if (aq_cfg->aq_hw_caps->hw_features & NETIF_F_LRO) {
186 is_lro = features & NETIF_F_LRO;
187
188 if (aq_cfg->is_lro != is_lro) {
189 aq_cfg->is_lro = is_lro;
190 need_ndev_restart = true;
191 }
192 }
193
194 if ((aq_nic->ndev->features ^ features) & NETIF_F_RXCSUM) {
195 err = aq_nic->aq_hw_ops->hw_set_offload(aq_nic->aq_hw,
196 aq_cfg);
197
198 if (unlikely(err))
199 goto err_exit;
200 }
201
202 if (aq_cfg->is_vlan_rx_strip != is_vlan_rx_strip) {
203 aq_cfg->is_vlan_rx_strip = is_vlan_rx_strip;
204 need_ndev_restart = true;
205 }
206 if (aq_cfg->is_vlan_tx_insert != is_vlan_tx_insert) {
207 aq_cfg->is_vlan_tx_insert = is_vlan_tx_insert;
208 need_ndev_restart = true;
209 }
210
211 if (need_ndev_restart && netif_running(dev: ndev)) {
212 aq_ndev_close(ndev);
213 aq_ndev_open(ndev);
214 }
215
216err_exit:
217 return err;
218}
219
220static netdev_features_t aq_ndev_fix_features(struct net_device *ndev,
221 netdev_features_t features)
222{
223 struct aq_nic_s *aq_nic = netdev_priv(dev: ndev);
224 struct bpf_prog *prog;
225
226 if (!(features & NETIF_F_RXCSUM))
227 features &= ~NETIF_F_LRO;
228
229 prog = READ_ONCE(aq_nic->xdp_prog);
230 if (prog && !prog->aux->xdp_has_frags &&
231 aq_nic->xdp_prog && features & NETIF_F_LRO) {
232 netdev_err(dev: ndev, format: "LRO is not supported with single buffer XDP, disabling\n");
233 features &= ~NETIF_F_LRO;
234 }
235
236 return features;
237}
238
239static int aq_ndev_set_mac_address(struct net_device *ndev, void *addr)
240{
241 struct aq_nic_s *aq_nic = netdev_priv(dev: ndev);
242 int err = 0;
243
244 err = eth_mac_addr(dev: ndev, p: addr);
245 if (err < 0)
246 goto err_exit;
247 err = aq_nic_set_mac(self: aq_nic, ndev);
248 if (err < 0)
249 goto err_exit;
250
251err_exit:
252 return err;
253}
254
255static void aq_ndev_set_multicast_settings(struct net_device *ndev)
256{
257 struct aq_nic_s *aq_nic = netdev_priv(dev: ndev);
258
259 (void)aq_nic_set_multicast_list(self: aq_nic, ndev);
260}
261
262#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
263static int aq_ndev_config_hwtstamp(struct aq_nic_s *aq_nic,
264 struct hwtstamp_config *config)
265{
266 switch (config->tx_type) {
267 case HWTSTAMP_TX_OFF:
268 case HWTSTAMP_TX_ON:
269 break;
270 default:
271 return -ERANGE;
272 }
273
274 switch (config->rx_filter) {
275 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
276 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
277 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
278 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
279 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
280 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
281 case HWTSTAMP_FILTER_PTP_V2_SYNC:
282 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
283 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
284 break;
285 case HWTSTAMP_FILTER_PTP_V2_EVENT:
286 case HWTSTAMP_FILTER_NONE:
287 break;
288 default:
289 return -ERANGE;
290 }
291
292 return aq_ptp_hwtstamp_config_set(aq_ptp: aq_nic->aq_ptp, config);
293}
294#endif
295
296static int aq_ndev_hwtstamp_set(struct aq_nic_s *aq_nic, struct ifreq *ifr)
297{
298 struct hwtstamp_config config;
299#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
300 int ret_val;
301#endif
302
303 if (!aq_nic->aq_ptp)
304 return -EOPNOTSUPP;
305
306 if (copy_from_user(to: &config, from: ifr->ifr_data, n: sizeof(config)))
307 return -EFAULT;
308#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
309 ret_val = aq_ndev_config_hwtstamp(aq_nic, config: &config);
310 if (ret_val)
311 return ret_val;
312#endif
313
314 return copy_to_user(to: ifr->ifr_data, from: &config, n: sizeof(config)) ?
315 -EFAULT : 0;
316}
317
318#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
319static int aq_ndev_hwtstamp_get(struct aq_nic_s *aq_nic, struct ifreq *ifr)
320{
321 struct hwtstamp_config config;
322
323 if (!aq_nic->aq_ptp)
324 return -EOPNOTSUPP;
325
326 aq_ptp_hwtstamp_config_get(aq_ptp: aq_nic->aq_ptp, config: &config);
327 return copy_to_user(to: ifr->ifr_data, from: &config, n: sizeof(config)) ?
328 -EFAULT : 0;
329}
330#endif
331
332static int aq_ndev_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
333{
334 struct aq_nic_s *aq_nic = netdev_priv(dev: netdev);
335
336 switch (cmd) {
337 case SIOCSHWTSTAMP:
338 return aq_ndev_hwtstamp_set(aq_nic, ifr);
339
340#if IS_REACHABLE(CONFIG_PTP_1588_CLOCK)
341 case SIOCGHWTSTAMP:
342 return aq_ndev_hwtstamp_get(aq_nic, ifr);
343#endif
344 }
345
346 return -EOPNOTSUPP;
347}
348
349static int aq_ndo_vlan_rx_add_vid(struct net_device *ndev, __be16 proto,
350 u16 vid)
351{
352 struct aq_nic_s *aq_nic = netdev_priv(dev: ndev);
353
354 if (!aq_nic->aq_hw_ops->hw_filter_vlan_set)
355 return -EOPNOTSUPP;
356
357 set_bit(nr: vid, addr: aq_nic->active_vlans);
358
359 return aq_filters_vlans_update(aq_nic);
360}
361
362static int aq_ndo_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto,
363 u16 vid)
364{
365 struct aq_nic_s *aq_nic = netdev_priv(dev: ndev);
366
367 if (!aq_nic->aq_hw_ops->hw_filter_vlan_set)
368 return -EOPNOTSUPP;
369
370 clear_bit(nr: vid, addr: aq_nic->active_vlans);
371
372 if (-ENOENT == aq_del_fvlan_by_vlan(aq_nic, vlan_id: vid))
373 return aq_filters_vlans_update(aq_nic);
374
375 return 0;
376}
377
378static int aq_validate_mqprio_opt(struct aq_nic_s *self,
379 struct tc_mqprio_qopt_offload *mqprio,
380 const unsigned int num_tc)
381{
382 const bool has_min_rate = !!(mqprio->flags & TC_MQPRIO_F_MIN_RATE);
383 struct aq_nic_cfg_s *aq_nic_cfg = aq_nic_get_cfg(self);
384 const unsigned int tcs_max = min_t(u8, aq_nic_cfg->aq_hw_caps->tcs_max,
385 AQ_CFG_TCS_MAX);
386
387 if (num_tc > tcs_max) {
388 netdev_err(dev: self->ndev, format: "Too many TCs requested\n");
389 return -EOPNOTSUPP;
390 }
391
392 if (num_tc != 0 && !is_power_of_2(n: num_tc)) {
393 netdev_err(dev: self->ndev, format: "TC count should be power of 2\n");
394 return -EOPNOTSUPP;
395 }
396
397 if (has_min_rate && !ATL_HW_IS_CHIP_FEATURE(self->aq_hw, ANTIGUA)) {
398 netdev_err(dev: self->ndev, format: "Min tx rate is not supported\n");
399 return -EOPNOTSUPP;
400 }
401
402 return 0;
403}
404
405static int aq_ndo_setup_tc(struct net_device *dev, enum tc_setup_type type,
406 void *type_data)
407{
408 struct tc_mqprio_qopt_offload *mqprio = type_data;
409 struct aq_nic_s *aq_nic = netdev_priv(dev);
410 bool has_min_rate;
411 bool has_max_rate;
412 int err;
413 int i;
414
415 if (type != TC_SETUP_QDISC_MQPRIO)
416 return -EOPNOTSUPP;
417
418 has_min_rate = !!(mqprio->flags & TC_MQPRIO_F_MIN_RATE);
419 has_max_rate = !!(mqprio->flags & TC_MQPRIO_F_MAX_RATE);
420
421 err = aq_validate_mqprio_opt(self: aq_nic, mqprio, num_tc: mqprio->qopt.num_tc);
422 if (err)
423 return err;
424
425 for (i = 0; i < mqprio->qopt.num_tc; i++) {
426 if (has_max_rate) {
427 u64 max_rate = mqprio->max_rate[i];
428
429 do_div(max_rate, AQ_MBPS_DIVISOR);
430 aq_nic_setup_tc_max_rate(self: aq_nic, tc: i, max_rate: (u32)max_rate);
431 }
432
433 if (has_min_rate) {
434 u64 min_rate = mqprio->min_rate[i];
435
436 do_div(min_rate, AQ_MBPS_DIVISOR);
437 aq_nic_setup_tc_min_rate(self: aq_nic, tc: i, min_rate: (u32)min_rate);
438 }
439 }
440
441 return aq_nic_setup_tc_mqprio(self: aq_nic, tcs: mqprio->qopt.num_tc,
442 prio_tc_map: mqprio->qopt.prio_tc_map);
443}
444
445static int aq_xdp_setup(struct net_device *ndev, struct bpf_prog *prog,
446 struct netlink_ext_ack *extack)
447{
448 bool need_update, running = netif_running(dev: ndev);
449 struct aq_nic_s *aq_nic = netdev_priv(dev: ndev);
450 struct bpf_prog *old_prog;
451
452 if (prog && !prog->aux->xdp_has_frags) {
453 if (ndev->mtu > AQ_CFG_RX_FRAME_MAX) {
454 NL_SET_ERR_MSG_MOD(extack,
455 "prog does not support XDP frags");
456 return -EOPNOTSUPP;
457 }
458
459 if (prog && ndev->features & NETIF_F_LRO) {
460 netdev_err(dev: ndev,
461 format: "LRO is not supported with single buffer XDP, disabling\n");
462 ndev->features &= ~NETIF_F_LRO;
463 }
464 }
465
466 need_update = !!aq_nic->xdp_prog != !!prog;
467 if (running && need_update)
468 aq_ndev_close(ndev);
469
470 old_prog = xchg(&aq_nic->xdp_prog, prog);
471 if (old_prog)
472 bpf_prog_put(prog: old_prog);
473
474 if (!old_prog && prog)
475 static_branch_inc(&aq_xdp_locking_key);
476 else if (old_prog && !prog)
477 static_branch_dec(&aq_xdp_locking_key);
478
479 if (running && need_update)
480 return aq_ndev_open(ndev);
481
482 return 0;
483}
484
485static int aq_xdp(struct net_device *dev, struct netdev_bpf *xdp)
486{
487 switch (xdp->command) {
488 case XDP_SETUP_PROG:
489 return aq_xdp_setup(ndev: dev, prog: xdp->prog, extack: xdp->extack);
490 default:
491 return -EINVAL;
492 }
493}
494
495static const struct net_device_ops aq_ndev_ops = {
496 .ndo_open = aq_ndev_open,
497 .ndo_stop = aq_ndev_close,
498 .ndo_start_xmit = aq_ndev_start_xmit,
499 .ndo_set_rx_mode = aq_ndev_set_multicast_settings,
500 .ndo_change_mtu = aq_ndev_change_mtu,
501 .ndo_set_mac_address = aq_ndev_set_mac_address,
502 .ndo_set_features = aq_ndev_set_features,
503 .ndo_fix_features = aq_ndev_fix_features,
504 .ndo_eth_ioctl = aq_ndev_ioctl,
505 .ndo_vlan_rx_add_vid = aq_ndo_vlan_rx_add_vid,
506 .ndo_vlan_rx_kill_vid = aq_ndo_vlan_rx_kill_vid,
507 .ndo_setup_tc = aq_ndo_setup_tc,
508 .ndo_bpf = aq_xdp,
509 .ndo_xdp_xmit = aq_xdp_xmit,
510};
511
512static int __init aq_ndev_init_module(void)
513{
514 int ret;
515
516 aq_ndev_wq = create_singlethread_workqueue(aq_ndev_driver_name);
517 if (!aq_ndev_wq) {
518 pr_err("Failed to create workqueue\n");
519 return -ENOMEM;
520 }
521
522 ret = aq_pci_func_register_driver();
523 if (ret) {
524 destroy_workqueue(wq: aq_ndev_wq);
525 return ret;
526 }
527
528 return 0;
529}
530
531static void __exit aq_ndev_exit_module(void)
532{
533 aq_pci_func_unregister_driver();
534
535 if (aq_ndev_wq) {
536 destroy_workqueue(wq: aq_ndev_wq);
537 aq_ndev_wq = NULL;
538 }
539}
540
541module_init(aq_ndev_init_module);
542module_exit(aq_ndev_exit_module);
543

source code of linux/drivers/net/ethernet/aquantia/atlantic/aq_main.c