1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ |
3 | |
4 | #include <linux/etherdevice.h> |
5 | #include <linux/io-64-nonatomic-hi-lo.h> |
6 | #include <linux/lockdep.h> |
7 | #include <net/dst_metadata.h> |
8 | |
9 | #include "nfpcore/nfp_cpp.h" |
10 | #include "nfpcore/nfp_nsp.h" |
11 | #include "nfp_app.h" |
12 | #include "nfp_main.h" |
13 | #include "nfp_net.h" |
14 | #include "nfp_net_ctrl.h" |
15 | #include "nfp_net_repr.h" |
16 | #include "nfp_net_sriov.h" |
17 | #include "nfp_port.h" |
18 | |
19 | struct net_device * |
20 | nfp_repr_get_locked(struct nfp_app *app, struct nfp_reprs *set, unsigned int id) |
21 | { |
22 | return rcu_dereference_protected(set->reprs[id], |
23 | nfp_app_is_locked(app)); |
24 | } |
25 | |
26 | static void |
27 | nfp_repr_inc_tx_stats(struct net_device *netdev, unsigned int len, |
28 | int tx_status) |
29 | { |
30 | struct nfp_repr *repr = netdev_priv(dev: netdev); |
31 | struct nfp_repr_pcpu_stats *stats; |
32 | |
33 | if (unlikely(tx_status != NET_XMIT_SUCCESS && |
34 | tx_status != NET_XMIT_CN)) { |
35 | this_cpu_inc(repr->stats->tx_drops); |
36 | return; |
37 | } |
38 | |
39 | stats = this_cpu_ptr(repr->stats); |
40 | u64_stats_update_begin(syncp: &stats->syncp); |
41 | stats->tx_packets++; |
42 | stats->tx_bytes += len; |
43 | u64_stats_update_end(syncp: &stats->syncp); |
44 | } |
45 | |
46 | void nfp_repr_inc_rx_stats(struct net_device *netdev, unsigned int len) |
47 | { |
48 | struct nfp_repr *repr = netdev_priv(dev: netdev); |
49 | struct nfp_repr_pcpu_stats *stats; |
50 | |
51 | stats = this_cpu_ptr(repr->stats); |
52 | u64_stats_update_begin(syncp: &stats->syncp); |
53 | stats->rx_packets++; |
54 | stats->rx_bytes += len; |
55 | u64_stats_update_end(syncp: &stats->syncp); |
56 | } |
57 | |
58 | static void |
59 | nfp_repr_phy_port_get_stats64(struct nfp_port *port, |
60 | struct rtnl_link_stats64 *stats) |
61 | { |
62 | u8 __iomem *mem = port->eth_stats; |
63 | |
64 | stats->tx_packets = readq(addr: mem + NFP_MAC_STATS_TX_FRAMES_TRANSMITTED_OK); |
65 | stats->tx_bytes = readq(addr: mem + NFP_MAC_STATS_TX_OUT_OCTETS); |
66 | stats->tx_dropped = readq(addr: mem + NFP_MAC_STATS_TX_OUT_ERRORS); |
67 | |
68 | stats->rx_packets = readq(addr: mem + NFP_MAC_STATS_RX_FRAMES_RECEIVED_OK); |
69 | stats->rx_bytes = readq(addr: mem + NFP_MAC_STATS_RX_IN_OCTETS); |
70 | stats->rx_dropped = readq(addr: mem + NFP_MAC_STATS_RX_IN_ERRORS); |
71 | } |
72 | |
73 | static void |
74 | nfp_repr_vnic_get_stats64(struct nfp_port *port, |
75 | struct rtnl_link_stats64 *stats) |
76 | { |
77 | /* TX and RX stats are flipped as we are returning the stats as seen |
78 | * at the switch port corresponding to the VF. |
79 | */ |
80 | stats->tx_packets = readq(addr: port->vnic + NFP_NET_CFG_STATS_RX_FRAMES); |
81 | stats->tx_bytes = readq(addr: port->vnic + NFP_NET_CFG_STATS_RX_OCTETS); |
82 | stats->tx_dropped = readq(addr: port->vnic + NFP_NET_CFG_STATS_RX_DISCARDS); |
83 | |
84 | stats->rx_packets = readq(addr: port->vnic + NFP_NET_CFG_STATS_TX_FRAMES); |
85 | stats->rx_bytes = readq(addr: port->vnic + NFP_NET_CFG_STATS_TX_OCTETS); |
86 | stats->rx_dropped = readq(addr: port->vnic + NFP_NET_CFG_STATS_TX_DISCARDS); |
87 | } |
88 | |
89 | static void |
90 | nfp_repr_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) |
91 | { |
92 | struct nfp_repr *repr = netdev_priv(dev: netdev); |
93 | |
94 | if (WARN_ON(!repr->port)) |
95 | return; |
96 | |
97 | switch (repr->port->type) { |
98 | case NFP_PORT_PHYS_PORT: |
99 | if (!__nfp_port_get_eth_port(port: repr->port)) |
100 | break; |
101 | nfp_repr_phy_port_get_stats64(port: repr->port, stats); |
102 | break; |
103 | case NFP_PORT_PF_PORT: |
104 | case NFP_PORT_VF_PORT: |
105 | nfp_repr_vnic_get_stats64(port: repr->port, stats); |
106 | break; |
107 | default: |
108 | break; |
109 | } |
110 | } |
111 | |
112 | static bool |
113 | nfp_repr_has_offload_stats(const struct net_device *dev, int attr_id) |
114 | { |
115 | switch (attr_id) { |
116 | case IFLA_OFFLOAD_XSTATS_CPU_HIT: |
117 | return true; |
118 | } |
119 | |
120 | return false; |
121 | } |
122 | |
123 | static int |
124 | nfp_repr_get_host_stats64(const struct net_device *netdev, |
125 | struct rtnl_link_stats64 *stats) |
126 | { |
127 | struct nfp_repr *repr = netdev_priv(dev: netdev); |
128 | int i; |
129 | |
130 | for_each_possible_cpu(i) { |
131 | u64 tbytes, tpkts, tdrops, rbytes, rpkts; |
132 | struct nfp_repr_pcpu_stats *repr_stats; |
133 | unsigned int start; |
134 | |
135 | repr_stats = per_cpu_ptr(repr->stats, i); |
136 | do { |
137 | start = u64_stats_fetch_begin(syncp: &repr_stats->syncp); |
138 | tbytes = repr_stats->tx_bytes; |
139 | tpkts = repr_stats->tx_packets; |
140 | tdrops = repr_stats->tx_drops; |
141 | rbytes = repr_stats->rx_bytes; |
142 | rpkts = repr_stats->rx_packets; |
143 | } while (u64_stats_fetch_retry(syncp: &repr_stats->syncp, start)); |
144 | |
145 | stats->tx_bytes += tbytes; |
146 | stats->tx_packets += tpkts; |
147 | stats->tx_dropped += tdrops; |
148 | stats->rx_bytes += rbytes; |
149 | stats->rx_packets += rpkts; |
150 | } |
151 | |
152 | return 0; |
153 | } |
154 | |
155 | static int |
156 | nfp_repr_get_offload_stats(int attr_id, const struct net_device *dev, |
157 | void *stats) |
158 | { |
159 | switch (attr_id) { |
160 | case IFLA_OFFLOAD_XSTATS_CPU_HIT: |
161 | return nfp_repr_get_host_stats64(netdev: dev, stats); |
162 | } |
163 | |
164 | return -EINVAL; |
165 | } |
166 | |
167 | static int nfp_repr_change_mtu(struct net_device *netdev, int new_mtu) |
168 | { |
169 | struct nfp_repr *repr = netdev_priv(dev: netdev); |
170 | int err; |
171 | |
172 | err = nfp_app_check_mtu(app: repr->app, netdev, new_mtu); |
173 | if (err) |
174 | return err; |
175 | |
176 | err = nfp_app_repr_change_mtu(app: repr->app, netdev, new_mtu); |
177 | if (err) |
178 | return err; |
179 | |
180 | netdev->mtu = new_mtu; |
181 | |
182 | return 0; |
183 | } |
184 | |
185 | static netdev_tx_t nfp_repr_xmit(struct sk_buff *skb, struct net_device *netdev) |
186 | { |
187 | struct nfp_repr *repr = netdev_priv(dev: netdev); |
188 | unsigned int len = skb->len; |
189 | int ret; |
190 | |
191 | skb_dst_drop(skb); |
192 | dst_hold(dst: (struct dst_entry *)repr->dst); |
193 | skb_dst_set(skb, dst: (struct dst_entry *)repr->dst); |
194 | skb->dev = repr->dst->u.port_info.lower_dev; |
195 | |
196 | ret = dev_queue_xmit(skb); |
197 | nfp_repr_inc_tx_stats(netdev, len, tx_status: ret); |
198 | |
199 | return NETDEV_TX_OK; |
200 | } |
201 | |
202 | static int nfp_repr_stop(struct net_device *netdev) |
203 | { |
204 | struct nfp_repr *repr = netdev_priv(dev: netdev); |
205 | int err; |
206 | |
207 | err = nfp_app_repr_stop(app: repr->app, repr); |
208 | if (err) |
209 | return err; |
210 | |
211 | nfp_port_configure(netdev, configed: false); |
212 | return 0; |
213 | } |
214 | |
215 | static int nfp_repr_open(struct net_device *netdev) |
216 | { |
217 | struct nfp_repr *repr = netdev_priv(dev: netdev); |
218 | int err; |
219 | |
220 | err = nfp_port_configure(netdev, configed: true); |
221 | if (err) |
222 | return err; |
223 | |
224 | err = nfp_app_repr_open(app: repr->app, repr); |
225 | if (err) |
226 | goto err_port_disable; |
227 | |
228 | return 0; |
229 | |
230 | err_port_disable: |
231 | nfp_port_configure(netdev, configed: false); |
232 | return err; |
233 | } |
234 | |
235 | static netdev_features_t |
236 | nfp_repr_fix_features(struct net_device *netdev, netdev_features_t features) |
237 | { |
238 | struct nfp_repr *repr = netdev_priv(dev: netdev); |
239 | netdev_features_t old_features = features; |
240 | netdev_features_t lower_features; |
241 | struct net_device *lower_dev; |
242 | |
243 | lower_dev = repr->dst->u.port_info.lower_dev; |
244 | |
245 | lower_features = lower_dev->features; |
246 | if (lower_features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) |
247 | lower_features |= NETIF_F_HW_CSUM; |
248 | |
249 | features = netdev_intersect_features(f1: features, f2: lower_features); |
250 | features |= old_features & (NETIF_F_SOFT_FEATURES | NETIF_F_HW_TC); |
251 | features |= NETIF_F_LLTX; |
252 | |
253 | return features; |
254 | } |
255 | |
256 | const struct net_device_ops nfp_repr_netdev_ops = { |
257 | .ndo_init = nfp_app_ndo_init, |
258 | .ndo_uninit = nfp_app_ndo_uninit, |
259 | .ndo_open = nfp_repr_open, |
260 | .ndo_stop = nfp_repr_stop, |
261 | .ndo_start_xmit = nfp_repr_xmit, |
262 | .ndo_change_mtu = nfp_repr_change_mtu, |
263 | .ndo_get_stats64 = nfp_repr_get_stats64, |
264 | .ndo_has_offload_stats = nfp_repr_has_offload_stats, |
265 | .ndo_get_offload_stats = nfp_repr_get_offload_stats, |
266 | .ndo_get_phys_port_name = nfp_port_get_phys_port_name, |
267 | .ndo_setup_tc = nfp_port_setup_tc, |
268 | .ndo_set_vf_mac = nfp_app_set_vf_mac, |
269 | .ndo_set_vf_vlan = nfp_app_set_vf_vlan, |
270 | .ndo_set_vf_spoofchk = nfp_app_set_vf_spoofchk, |
271 | .ndo_set_vf_trust = nfp_app_set_vf_trust, |
272 | .ndo_get_vf_config = nfp_app_get_vf_config, |
273 | .ndo_set_vf_link_state = nfp_app_set_vf_link_state, |
274 | .ndo_fix_features = nfp_repr_fix_features, |
275 | .ndo_set_features = nfp_port_set_features, |
276 | .ndo_set_mac_address = eth_mac_addr, |
277 | .ndo_get_port_parent_id = nfp_port_get_port_parent_id, |
278 | }; |
279 | |
280 | void |
281 | nfp_repr_transfer_features(struct net_device *netdev, struct net_device *lower) |
282 | { |
283 | struct nfp_repr *repr = netdev_priv(dev: netdev); |
284 | |
285 | if (repr->dst->u.port_info.lower_dev != lower) |
286 | return; |
287 | |
288 | netif_inherit_tso_max(to: netdev, from: lower); |
289 | |
290 | netdev_update_features(dev: netdev); |
291 | } |
292 | |
293 | static void nfp_repr_clean(struct nfp_repr *repr) |
294 | { |
295 | unregister_netdev(dev: repr->netdev); |
296 | nfp_app_repr_clean(app: repr->app, netdev: repr->netdev); |
297 | dst_release(dst: (struct dst_entry *)repr->dst); |
298 | nfp_port_free(port: repr->port); |
299 | } |
300 | |
301 | static struct lock_class_key nfp_repr_netdev_xmit_lock_key; |
302 | |
303 | static void nfp_repr_set_lockdep_class_one(struct net_device *dev, |
304 | struct netdev_queue *txq, |
305 | void *_unused) |
306 | { |
307 | lockdep_set_class(&txq->_xmit_lock, &nfp_repr_netdev_xmit_lock_key); |
308 | } |
309 | |
310 | static void nfp_repr_set_lockdep_class(struct net_device *dev) |
311 | { |
312 | netdev_for_each_tx_queue(dev, f: nfp_repr_set_lockdep_class_one, NULL); |
313 | } |
314 | |
315 | int nfp_repr_init(struct nfp_app *app, struct net_device *netdev, |
316 | u32 cmsg_port_id, struct nfp_port *port, |
317 | struct net_device *pf_netdev) |
318 | { |
319 | struct nfp_repr *repr = netdev_priv(dev: netdev); |
320 | struct nfp_net *nn = netdev_priv(dev: pf_netdev); |
321 | u32 repr_cap = nn->tlv_caps.repr_cap; |
322 | int err; |
323 | |
324 | nfp_repr_set_lockdep_class(dev: netdev); |
325 | |
326 | repr->port = port; |
327 | repr->dst = metadata_dst_alloc(optslen: 0, type: METADATA_HW_PORT_MUX, GFP_KERNEL); |
328 | if (!repr->dst) |
329 | return -ENOMEM; |
330 | repr->dst->u.port_info.port_id = cmsg_port_id; |
331 | repr->dst->u.port_info.lower_dev = pf_netdev; |
332 | |
333 | netdev->netdev_ops = &nfp_repr_netdev_ops; |
334 | netdev->ethtool_ops = &nfp_port_ethtool_ops; |
335 | |
336 | netdev->max_mtu = pf_netdev->max_mtu; |
337 | |
338 | /* Set features the lower device can support with representors */ |
339 | if (repr_cap & NFP_NET_CFG_CTRL_LIVE_ADDR) |
340 | netdev->priv_flags |= IFF_LIVE_ADDR_CHANGE; |
341 | |
342 | netdev->hw_features = NETIF_F_HIGHDMA; |
343 | if (repr_cap & NFP_NET_CFG_CTRL_RXCSUM_ANY) |
344 | netdev->hw_features |= NETIF_F_RXCSUM; |
345 | if (repr_cap & NFP_NET_CFG_CTRL_TXCSUM) |
346 | netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; |
347 | if (repr_cap & NFP_NET_CFG_CTRL_GATHER) |
348 | netdev->hw_features |= NETIF_F_SG; |
349 | if ((repr_cap & NFP_NET_CFG_CTRL_LSO && nn->fw_ver.major > 2) || |
350 | repr_cap & NFP_NET_CFG_CTRL_LSO2) |
351 | netdev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; |
352 | if (repr_cap & NFP_NET_CFG_CTRL_RSS_ANY) |
353 | netdev->hw_features |= NETIF_F_RXHASH; |
354 | if (repr_cap & NFP_NET_CFG_CTRL_VXLAN) { |
355 | if (repr_cap & NFP_NET_CFG_CTRL_LSO) |
356 | netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL; |
357 | } |
358 | if (repr_cap & NFP_NET_CFG_CTRL_NVGRE) { |
359 | if (repr_cap & NFP_NET_CFG_CTRL_LSO) |
360 | netdev->hw_features |= NETIF_F_GSO_GRE; |
361 | } |
362 | if (repr_cap & (NFP_NET_CFG_CTRL_VXLAN | NFP_NET_CFG_CTRL_NVGRE)) |
363 | netdev->hw_enc_features = netdev->hw_features; |
364 | |
365 | netdev->vlan_features = netdev->hw_features; |
366 | |
367 | if (repr_cap & NFP_NET_CFG_CTRL_RXVLAN_ANY) |
368 | netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; |
369 | if (repr_cap & NFP_NET_CFG_CTRL_TXVLAN_ANY) { |
370 | if (repr_cap & NFP_NET_CFG_CTRL_LSO2) |
371 | netdev_warn(dev: netdev, format: "Device advertises both TSO2 and TXVLAN. Refusing to enable TXVLAN.\n" ); |
372 | else |
373 | netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; |
374 | } |
375 | if (repr_cap & NFP_NET_CFG_CTRL_CTAG_FILTER) |
376 | netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; |
377 | if (repr_cap & NFP_NET_CFG_CTRL_RXQINQ) |
378 | netdev->hw_features |= NETIF_F_HW_VLAN_STAG_RX; |
379 | |
380 | netdev->features = netdev->hw_features; |
381 | |
382 | /* C-Tag strip and S-Tag strip can't be supported simultaneously, |
383 | * so enable C-Tag strip and disable S-Tag strip by default. |
384 | */ |
385 | netdev->features &= ~NETIF_F_HW_VLAN_STAG_RX; |
386 | netif_set_tso_max_segs(dev: netdev, NFP_NET_LSO_MAX_SEGS); |
387 | |
388 | netdev->priv_flags |= IFF_NO_QUEUE | IFF_DISABLE_NETPOLL; |
389 | netdev->features |= NETIF_F_LLTX; |
390 | |
391 | if (nfp_app_has_tc(app)) { |
392 | netdev->features |= NETIF_F_HW_TC; |
393 | netdev->hw_features |= NETIF_F_HW_TC; |
394 | } |
395 | |
396 | err = nfp_app_repr_init(app, netdev); |
397 | if (err) |
398 | goto err_clean; |
399 | |
400 | err = register_netdev(dev: netdev); |
401 | if (err) |
402 | goto err_repr_clean; |
403 | |
404 | return 0; |
405 | |
406 | err_repr_clean: |
407 | nfp_app_repr_clean(app, netdev); |
408 | err_clean: |
409 | dst_release(dst: (struct dst_entry *)repr->dst); |
410 | return err; |
411 | } |
412 | |
413 | static void __nfp_repr_free(struct nfp_repr *repr) |
414 | { |
415 | free_percpu(pdata: repr->stats); |
416 | free_netdev(dev: repr->netdev); |
417 | } |
418 | |
419 | void nfp_repr_free(struct net_device *netdev) |
420 | { |
421 | __nfp_repr_free(repr: netdev_priv(dev: netdev)); |
422 | } |
423 | |
424 | struct net_device * |
425 | nfp_repr_alloc_mqs(struct nfp_app *app, unsigned int txqs, unsigned int rxqs) |
426 | { |
427 | struct net_device *netdev; |
428 | struct nfp_repr *repr; |
429 | |
430 | netdev = alloc_etherdev_mqs(sizeof_priv: sizeof(*repr), txqs, rxqs); |
431 | if (!netdev) |
432 | return NULL; |
433 | |
434 | netif_carrier_off(dev: netdev); |
435 | |
436 | repr = netdev_priv(dev: netdev); |
437 | repr->netdev = netdev; |
438 | repr->app = app; |
439 | |
440 | repr->stats = netdev_alloc_pcpu_stats(struct nfp_repr_pcpu_stats); |
441 | if (!repr->stats) |
442 | goto err_free_netdev; |
443 | |
444 | return netdev; |
445 | |
446 | err_free_netdev: |
447 | free_netdev(dev: netdev); |
448 | return NULL; |
449 | } |
450 | |
451 | void nfp_repr_clean_and_free(struct nfp_repr *repr) |
452 | { |
453 | nfp_info(repr->app->cpp, "Destroying Representor(%s)\n" , |
454 | repr->netdev->name); |
455 | nfp_repr_clean(repr); |
456 | __nfp_repr_free(repr); |
457 | } |
458 | |
459 | void nfp_reprs_clean_and_free(struct nfp_app *app, struct nfp_reprs *reprs) |
460 | { |
461 | struct net_device *netdev; |
462 | unsigned int i; |
463 | |
464 | for (i = 0; i < reprs->num_reprs; i++) { |
465 | netdev = nfp_repr_get_locked(app, set: reprs, id: i); |
466 | if (netdev) |
467 | nfp_repr_clean_and_free(repr: netdev_priv(dev: netdev)); |
468 | } |
469 | |
470 | kfree(objp: reprs); |
471 | } |
472 | |
473 | void |
474 | nfp_reprs_clean_and_free_by_type(struct nfp_app *app, enum nfp_repr_type type) |
475 | { |
476 | struct net_device *netdev; |
477 | struct nfp_reprs *reprs; |
478 | int i; |
479 | |
480 | reprs = rcu_dereference_protected(app->reprs[type], |
481 | nfp_app_is_locked(app)); |
482 | if (!reprs) |
483 | return; |
484 | |
485 | /* Preclean must happen before we remove the reprs reference from the |
486 | * app below. |
487 | */ |
488 | for (i = 0; i < reprs->num_reprs; i++) { |
489 | netdev = nfp_repr_get_locked(app, set: reprs, id: i); |
490 | if (netdev) |
491 | nfp_app_repr_preclean(app, netdev); |
492 | } |
493 | |
494 | reprs = nfp_app_reprs_set(app, type, NULL); |
495 | |
496 | synchronize_rcu(); |
497 | nfp_reprs_clean_and_free(app, reprs); |
498 | } |
499 | |
500 | struct nfp_reprs *nfp_reprs_alloc(unsigned int num_reprs) |
501 | { |
502 | struct nfp_reprs *reprs; |
503 | |
504 | reprs = kzalloc(struct_size(reprs, reprs, num_reprs), GFP_KERNEL); |
505 | if (!reprs) |
506 | return NULL; |
507 | reprs->num_reprs = num_reprs; |
508 | |
509 | return reprs; |
510 | } |
511 | |
512 | int nfp_reprs_resync_phys_ports(struct nfp_app *app) |
513 | { |
514 | struct net_device *netdev; |
515 | struct nfp_reprs *reprs; |
516 | struct nfp_repr *repr; |
517 | int i; |
518 | |
519 | reprs = nfp_reprs_get_locked(app, type: NFP_REPR_TYPE_PHYS_PORT); |
520 | if (!reprs) |
521 | return 0; |
522 | |
523 | for (i = 0; i < reprs->num_reprs; i++) { |
524 | netdev = nfp_repr_get_locked(app, set: reprs, id: i); |
525 | if (!netdev) |
526 | continue; |
527 | |
528 | repr = netdev_priv(dev: netdev); |
529 | if (repr->port->type != NFP_PORT_INVALID) |
530 | continue; |
531 | |
532 | nfp_app_repr_preclean(app, netdev); |
533 | rtnl_lock(); |
534 | rcu_assign_pointer(reprs->reprs[i], NULL); |
535 | rtnl_unlock(); |
536 | synchronize_rcu(); |
537 | nfp_repr_clean(repr); |
538 | } |
539 | |
540 | return 0; |
541 | } |
542 | |