1 | /* |
2 | * Copyright (c) 2007 Mellanox Technologies. All rights reserved. |
3 | * |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU |
6 | * General Public License (GPL) Version 2, available from the file |
7 | * COPYING in the main directory of this source tree, or the |
8 | * OpenIB.org BSD license below: |
9 | * |
10 | * Redistribution and use in source and binary forms, with or |
11 | * without modification, are permitted provided that the following |
12 | * conditions are met: |
13 | * |
14 | * - Redistributions of source code must retain the above |
15 | * copyright notice, this list of conditions and the following |
16 | * disclaimer. |
17 | * |
18 | * - Redistributions in binary form must reproduce the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer in the documentation and/or other materials |
21 | * provided with the distribution. |
22 | * |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
30 | * SOFTWARE. |
31 | * |
32 | */ |
33 | |
34 | #include <linux/bpf.h> |
35 | #include <linux/etherdevice.h> |
36 | #include <linux/filter.h> |
37 | #include <linux/tcp.h> |
38 | #include <linux/if_vlan.h> |
39 | #include <linux/delay.h> |
40 | #include <linux/slab.h> |
41 | #include <linux/hash.h> |
42 | #include <net/ip.h> |
43 | #include <net/vxlan.h> |
44 | #include <net/devlink.h> |
45 | #include <net/rps.h> |
46 | |
47 | #include <linux/mlx4/driver.h> |
48 | #include <linux/mlx4/device.h> |
49 | #include <linux/mlx4/cmd.h> |
50 | #include <linux/mlx4/cq.h> |
51 | |
52 | #include "mlx4_en.h" |
53 | #include "en_port.h" |
54 | |
55 | #define MLX4_EN_MAX_XDP_MTU ((int)(PAGE_SIZE - ETH_HLEN - (2 * VLAN_HLEN) - \ |
56 | XDP_PACKET_HEADROOM - \ |
57 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))) |
58 | |
59 | int mlx4_en_setup_tc(struct net_device *dev, u8 up) |
60 | { |
61 | struct mlx4_en_priv *priv = netdev_priv(dev); |
62 | int i; |
63 | unsigned int offset = 0; |
64 | |
65 | if (up && up != MLX4_EN_NUM_UP_HIGH) |
66 | return -EINVAL; |
67 | |
68 | netdev_set_num_tc(dev, num_tc: up); |
69 | netif_set_real_num_tx_queues(dev, txq: priv->tx_ring_num[TX]); |
70 | /* Partition Tx queues evenly amongst UP's */ |
71 | for (i = 0; i < up; i++) { |
72 | netdev_set_tc_queue(dev, tc: i, count: priv->num_tx_rings_p_up, offset); |
73 | offset += priv->num_tx_rings_p_up; |
74 | } |
75 | |
76 | #ifdef CONFIG_MLX4_EN_DCB |
77 | if (!mlx4_is_slave(dev: priv->mdev->dev)) { |
78 | if (up) { |
79 | if (priv->dcbx_cap) |
80 | priv->flags |= MLX4_EN_FLAG_DCB_ENABLED; |
81 | } else { |
82 | priv->flags &= ~MLX4_EN_FLAG_DCB_ENABLED; |
83 | priv->cee_config.pfc_state = false; |
84 | } |
85 | } |
86 | #endif /* CONFIG_MLX4_EN_DCB */ |
87 | |
88 | return 0; |
89 | } |
90 | |
91 | int mlx4_en_alloc_tx_queue_per_tc(struct net_device *dev, u8 tc) |
92 | { |
93 | struct mlx4_en_priv *priv = netdev_priv(dev); |
94 | struct mlx4_en_dev *mdev = priv->mdev; |
95 | struct mlx4_en_port_profile new_prof; |
96 | struct mlx4_en_priv *tmp; |
97 | int total_count; |
98 | int port_up = 0; |
99 | int err = 0; |
100 | |
101 | tmp = kzalloc(size: sizeof(*tmp), GFP_KERNEL); |
102 | if (!tmp) |
103 | return -ENOMEM; |
104 | |
105 | mutex_lock(&mdev->state_lock); |
106 | memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile)); |
107 | new_prof.num_up = (tc == 0) ? MLX4_EN_NUM_UP_LOW : |
108 | MLX4_EN_NUM_UP_HIGH; |
109 | new_prof.tx_ring_num[TX] = new_prof.num_tx_rings_p_up * |
110 | new_prof.num_up; |
111 | total_count = new_prof.tx_ring_num[TX] + new_prof.tx_ring_num[TX_XDP]; |
112 | if (total_count > MAX_TX_RINGS) { |
113 | err = -EINVAL; |
114 | en_err(priv, |
115 | "Total number of TX and XDP rings (%d) exceeds the maximum supported (%d)\n" , |
116 | total_count, MAX_TX_RINGS); |
117 | goto out; |
118 | } |
119 | err = mlx4_en_try_alloc_resources(priv, tmp, prof: &new_prof, carry_xdp_prog: true); |
120 | if (err) |
121 | goto out; |
122 | |
123 | if (priv->port_up) { |
124 | port_up = 1; |
125 | mlx4_en_stop_port(dev, detach: 1); |
126 | } |
127 | |
128 | mlx4_en_safe_replace_resources(priv, tmp); |
129 | if (port_up) { |
130 | err = mlx4_en_start_port(dev); |
131 | if (err) { |
132 | en_err(priv, "Failed starting port for setup TC\n" ); |
133 | goto out; |
134 | } |
135 | } |
136 | |
137 | err = mlx4_en_setup_tc(dev, up: tc); |
138 | out: |
139 | mutex_unlock(lock: &mdev->state_lock); |
140 | kfree(objp: tmp); |
141 | return err; |
142 | } |
143 | |
144 | static int __mlx4_en_setup_tc(struct net_device *dev, enum tc_setup_type type, |
145 | void *type_data) |
146 | { |
147 | struct tc_mqprio_qopt *mqprio = type_data; |
148 | |
149 | if (type != TC_SETUP_QDISC_MQPRIO) |
150 | return -EOPNOTSUPP; |
151 | |
152 | if (mqprio->num_tc && mqprio->num_tc != MLX4_EN_NUM_UP_HIGH) |
153 | return -EINVAL; |
154 | |
155 | mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; |
156 | |
157 | return mlx4_en_alloc_tx_queue_per_tc(dev, tc: mqprio->num_tc); |
158 | } |
159 | |
160 | #ifdef CONFIG_RFS_ACCEL |
161 | |
162 | struct mlx4_en_filter { |
163 | struct list_head next; |
164 | struct work_struct work; |
165 | |
166 | u8 ip_proto; |
167 | __be32 src_ip; |
168 | __be32 dst_ip; |
169 | __be16 src_port; |
170 | __be16 dst_port; |
171 | |
172 | int rxq_index; |
173 | struct mlx4_en_priv *priv; |
174 | u32 flow_id; /* RFS infrastructure id */ |
175 | int id; /* mlx4_en driver id */ |
176 | u64 reg_id; /* Flow steering API id */ |
177 | u8 activated; /* Used to prevent expiry before filter |
178 | * is attached |
179 | */ |
180 | struct hlist_node filter_chain; |
181 | }; |
182 | |
183 | static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv); |
184 | |
185 | static enum mlx4_net_trans_rule_id mlx4_ip_proto_to_trans_rule_id(u8 ip_proto) |
186 | { |
187 | switch (ip_proto) { |
188 | case IPPROTO_UDP: |
189 | return MLX4_NET_TRANS_RULE_ID_UDP; |
190 | case IPPROTO_TCP: |
191 | return MLX4_NET_TRANS_RULE_ID_TCP; |
192 | default: |
193 | return MLX4_NET_TRANS_RULE_NUM; |
194 | } |
195 | }; |
196 | |
197 | /* Must not acquire state_lock, as its corresponding work_sync |
198 | * is done under it. |
199 | */ |
200 | static void mlx4_en_filter_work(struct work_struct *work) |
201 | { |
202 | struct mlx4_en_filter *filter = container_of(work, |
203 | struct mlx4_en_filter, |
204 | work); |
205 | struct mlx4_en_priv *priv = filter->priv; |
206 | struct mlx4_spec_list spec_tcp_udp = { |
207 | .id = mlx4_ip_proto_to_trans_rule_id(ip_proto: filter->ip_proto), |
208 | { |
209 | .tcp_udp = { |
210 | .dst_port = filter->dst_port, |
211 | .dst_port_msk = (__force __be16)-1, |
212 | .src_port = filter->src_port, |
213 | .src_port_msk = (__force __be16)-1, |
214 | }, |
215 | }, |
216 | }; |
217 | struct mlx4_spec_list spec_ip = { |
218 | .id = MLX4_NET_TRANS_RULE_ID_IPV4, |
219 | { |
220 | .ipv4 = { |
221 | .dst_ip = filter->dst_ip, |
222 | .dst_ip_msk = (__force __be32)-1, |
223 | .src_ip = filter->src_ip, |
224 | .src_ip_msk = (__force __be32)-1, |
225 | }, |
226 | }, |
227 | }; |
228 | struct mlx4_spec_list spec_eth = { |
229 | .id = MLX4_NET_TRANS_RULE_ID_ETH, |
230 | }; |
231 | struct mlx4_net_trans_rule rule = { |
232 | .list = LIST_HEAD_INIT(rule.list), |
233 | .queue_mode = MLX4_NET_TRANS_Q_LIFO, |
234 | .exclusive = 1, |
235 | .allow_loopback = 1, |
236 | .promisc_mode = MLX4_FS_REGULAR, |
237 | .port = priv->port, |
238 | .priority = MLX4_DOMAIN_RFS, |
239 | }; |
240 | int rc; |
241 | __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); |
242 | |
243 | if (spec_tcp_udp.id >= MLX4_NET_TRANS_RULE_NUM) { |
244 | en_warn(priv, "RFS: ignoring unsupported ip protocol (%d)\n" , |
245 | filter->ip_proto); |
246 | goto ignore; |
247 | } |
248 | list_add_tail(new: &spec_eth.list, head: &rule.list); |
249 | list_add_tail(new: &spec_ip.list, head: &rule.list); |
250 | list_add_tail(new: &spec_tcp_udp.list, head: &rule.list); |
251 | |
252 | rule.qpn = priv->rss_map.qps[filter->rxq_index].qpn; |
253 | memcpy(spec_eth.eth.dst_mac, priv->dev->dev_addr, ETH_ALEN); |
254 | memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); |
255 | |
256 | filter->activated = 0; |
257 | |
258 | if (filter->reg_id) { |
259 | rc = mlx4_flow_detach(dev: priv->mdev->dev, reg_id: filter->reg_id); |
260 | if (rc && rc != -ENOENT) |
261 | en_err(priv, "Error detaching flow. rc = %d\n" , rc); |
262 | } |
263 | |
264 | rc = mlx4_flow_attach(dev: priv->mdev->dev, rule: &rule, reg_id: &filter->reg_id); |
265 | if (rc) |
266 | en_err(priv, "Error attaching flow. err = %d\n" , rc); |
267 | |
268 | ignore: |
269 | mlx4_en_filter_rfs_expire(priv); |
270 | |
271 | filter->activated = 1; |
272 | } |
273 | |
274 | static inline struct hlist_head * |
275 | filter_hash_bucket(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, |
276 | __be16 src_port, __be16 dst_port) |
277 | { |
278 | unsigned long l; |
279 | int bucket_idx; |
280 | |
281 | l = (__force unsigned long)src_port | |
282 | ((__force unsigned long)dst_port << 2); |
283 | l ^= (__force unsigned long)(src_ip ^ dst_ip); |
284 | |
285 | bucket_idx = hash_long(l, MLX4_EN_FILTER_HASH_SHIFT); |
286 | |
287 | return &priv->filter_hash[bucket_idx]; |
288 | } |
289 | |
290 | static struct mlx4_en_filter * |
291 | mlx4_en_filter_alloc(struct mlx4_en_priv *priv, int rxq_index, __be32 src_ip, |
292 | __be32 dst_ip, u8 ip_proto, __be16 src_port, |
293 | __be16 dst_port, u32 flow_id) |
294 | { |
295 | struct mlx4_en_filter *filter; |
296 | |
297 | filter = kzalloc(size: sizeof(struct mlx4_en_filter), GFP_ATOMIC); |
298 | if (!filter) |
299 | return NULL; |
300 | |
301 | filter->priv = priv; |
302 | filter->rxq_index = rxq_index; |
303 | INIT_WORK(&filter->work, mlx4_en_filter_work); |
304 | |
305 | filter->src_ip = src_ip; |
306 | filter->dst_ip = dst_ip; |
307 | filter->ip_proto = ip_proto; |
308 | filter->src_port = src_port; |
309 | filter->dst_port = dst_port; |
310 | |
311 | filter->flow_id = flow_id; |
312 | |
313 | filter->id = priv->last_filter_id++ % RPS_NO_FILTER; |
314 | |
315 | list_add_tail(new: &filter->next, head: &priv->filters); |
316 | hlist_add_head(n: &filter->filter_chain, |
317 | h: filter_hash_bucket(priv, src_ip, dst_ip, src_port, |
318 | dst_port)); |
319 | |
320 | return filter; |
321 | } |
322 | |
323 | static void mlx4_en_filter_free(struct mlx4_en_filter *filter) |
324 | { |
325 | struct mlx4_en_priv *priv = filter->priv; |
326 | int rc; |
327 | |
328 | list_del(entry: &filter->next); |
329 | |
330 | rc = mlx4_flow_detach(dev: priv->mdev->dev, reg_id: filter->reg_id); |
331 | if (rc && rc != -ENOENT) |
332 | en_err(priv, "Error detaching flow. rc = %d\n" , rc); |
333 | |
334 | kfree(objp: filter); |
335 | } |
336 | |
337 | static inline struct mlx4_en_filter * |
338 | mlx4_en_filter_find(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, |
339 | u8 ip_proto, __be16 src_port, __be16 dst_port) |
340 | { |
341 | struct mlx4_en_filter *filter; |
342 | struct mlx4_en_filter *ret = NULL; |
343 | |
344 | hlist_for_each_entry(filter, |
345 | filter_hash_bucket(priv, src_ip, dst_ip, |
346 | src_port, dst_port), |
347 | filter_chain) { |
348 | if (filter->src_ip == src_ip && |
349 | filter->dst_ip == dst_ip && |
350 | filter->ip_proto == ip_proto && |
351 | filter->src_port == src_port && |
352 | filter->dst_port == dst_port) { |
353 | ret = filter; |
354 | break; |
355 | } |
356 | } |
357 | |
358 | return ret; |
359 | } |
360 | |
361 | static int |
362 | mlx4_en_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, |
363 | u16 rxq_index, u32 flow_id) |
364 | { |
365 | struct mlx4_en_priv *priv = netdev_priv(dev: net_dev); |
366 | struct mlx4_en_filter *filter; |
367 | const struct iphdr *ip; |
368 | const __be16 *ports; |
369 | u8 ip_proto; |
370 | __be32 src_ip; |
371 | __be32 dst_ip; |
372 | __be16 src_port; |
373 | __be16 dst_port; |
374 | int nhoff = skb_network_offset(skb); |
375 | int ret = 0; |
376 | |
377 | if (skb->encapsulation) |
378 | return -EPROTONOSUPPORT; |
379 | |
380 | if (skb->protocol != htons(ETH_P_IP)) |
381 | return -EPROTONOSUPPORT; |
382 | |
383 | ip = (const struct iphdr *)(skb->data + nhoff); |
384 | if (ip_is_fragment(iph: ip)) |
385 | return -EPROTONOSUPPORT; |
386 | |
387 | if ((ip->protocol != IPPROTO_TCP) && (ip->protocol != IPPROTO_UDP)) |
388 | return -EPROTONOSUPPORT; |
389 | ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); |
390 | |
391 | ip_proto = ip->protocol; |
392 | src_ip = ip->saddr; |
393 | dst_ip = ip->daddr; |
394 | src_port = ports[0]; |
395 | dst_port = ports[1]; |
396 | |
397 | spin_lock_bh(lock: &priv->filters_lock); |
398 | filter = mlx4_en_filter_find(priv, src_ip, dst_ip, ip_proto, |
399 | src_port, dst_port); |
400 | if (filter) { |
401 | if (filter->rxq_index == rxq_index) |
402 | goto out; |
403 | |
404 | filter->rxq_index = rxq_index; |
405 | } else { |
406 | filter = mlx4_en_filter_alloc(priv, rxq_index, |
407 | src_ip, dst_ip, ip_proto, |
408 | src_port, dst_port, flow_id); |
409 | if (!filter) { |
410 | ret = -ENOMEM; |
411 | goto err; |
412 | } |
413 | } |
414 | |
415 | queue_work(wq: priv->mdev->workqueue, work: &filter->work); |
416 | |
417 | out: |
418 | ret = filter->id; |
419 | err: |
420 | spin_unlock_bh(lock: &priv->filters_lock); |
421 | |
422 | return ret; |
423 | } |
424 | |
425 | void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv) |
426 | { |
427 | struct mlx4_en_filter *filter, *tmp; |
428 | LIST_HEAD(del_list); |
429 | |
430 | spin_lock_bh(lock: &priv->filters_lock); |
431 | list_for_each_entry_safe(filter, tmp, &priv->filters, next) { |
432 | list_move(list: &filter->next, head: &del_list); |
433 | hlist_del(n: &filter->filter_chain); |
434 | } |
435 | spin_unlock_bh(lock: &priv->filters_lock); |
436 | |
437 | list_for_each_entry_safe(filter, tmp, &del_list, next) { |
438 | cancel_work_sync(work: &filter->work); |
439 | mlx4_en_filter_free(filter); |
440 | } |
441 | } |
442 | |
443 | static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv) |
444 | { |
445 | struct mlx4_en_filter *filter = NULL, *tmp, *last_filter = NULL; |
446 | LIST_HEAD(del_list); |
447 | int i = 0; |
448 | |
449 | spin_lock_bh(lock: &priv->filters_lock); |
450 | list_for_each_entry_safe(filter, tmp, &priv->filters, next) { |
451 | if (i > MLX4_EN_FILTER_EXPIRY_QUOTA) |
452 | break; |
453 | |
454 | if (filter->activated && |
455 | !work_pending(&filter->work) && |
456 | rps_may_expire_flow(dev: priv->dev, |
457 | rxq_index: filter->rxq_index, flow_id: filter->flow_id, |
458 | filter_id: filter->id)) { |
459 | list_move(list: &filter->next, head: &del_list); |
460 | hlist_del(n: &filter->filter_chain); |
461 | } else |
462 | last_filter = filter; |
463 | |
464 | i++; |
465 | } |
466 | |
467 | if (last_filter && (&last_filter->next != priv->filters.next)) |
468 | list_move(list: &priv->filters, head: &last_filter->next); |
469 | |
470 | spin_unlock_bh(lock: &priv->filters_lock); |
471 | |
472 | list_for_each_entry_safe(filter, tmp, &del_list, next) |
473 | mlx4_en_filter_free(filter); |
474 | } |
475 | #endif |
476 | |
477 | static int mlx4_en_vlan_rx_add_vid(struct net_device *dev, |
478 | __be16 proto, u16 vid) |
479 | { |
480 | struct mlx4_en_priv *priv = netdev_priv(dev); |
481 | struct mlx4_en_dev *mdev = priv->mdev; |
482 | int err; |
483 | int idx; |
484 | |
485 | en_dbg(HW, priv, "adding VLAN:%d\n" , vid); |
486 | |
487 | set_bit(nr: vid, addr: priv->active_vlans); |
488 | |
489 | /* Add VID to port VLAN filter */ |
490 | mutex_lock(&mdev->state_lock); |
491 | if (mdev->device_up && priv->port_up) { |
492 | err = mlx4_SET_VLAN_FLTR(dev: mdev->dev, priv); |
493 | if (err) { |
494 | en_err(priv, "Failed configuring VLAN filter\n" ); |
495 | goto out; |
496 | } |
497 | } |
498 | err = mlx4_register_vlan(dev: mdev->dev, port: priv->port, vlan: vid, index: &idx); |
499 | if (err) |
500 | en_dbg(HW, priv, "Failed adding vlan %d\n" , vid); |
501 | |
502 | out: |
503 | mutex_unlock(lock: &mdev->state_lock); |
504 | return err; |
505 | } |
506 | |
507 | static int mlx4_en_vlan_rx_kill_vid(struct net_device *dev, |
508 | __be16 proto, u16 vid) |
509 | { |
510 | struct mlx4_en_priv *priv = netdev_priv(dev); |
511 | struct mlx4_en_dev *mdev = priv->mdev; |
512 | int err = 0; |
513 | |
514 | en_dbg(HW, priv, "Killing VID:%d\n" , vid); |
515 | |
516 | clear_bit(nr: vid, addr: priv->active_vlans); |
517 | |
518 | /* Remove VID from port VLAN filter */ |
519 | mutex_lock(&mdev->state_lock); |
520 | mlx4_unregister_vlan(dev: mdev->dev, port: priv->port, vlan: vid); |
521 | |
522 | if (mdev->device_up && priv->port_up) { |
523 | err = mlx4_SET_VLAN_FLTR(dev: mdev->dev, priv); |
524 | if (err) |
525 | en_err(priv, "Failed configuring VLAN filter\n" ); |
526 | } |
527 | mutex_unlock(lock: &mdev->state_lock); |
528 | |
529 | return err; |
530 | } |
531 | |
532 | static void mlx4_en_u64_to_mac(struct net_device *dev, u64 src_mac) |
533 | { |
534 | u8 addr[ETH_ALEN]; |
535 | |
536 | u64_to_ether_addr(u: src_mac, addr); |
537 | eth_hw_addr_set(dev, addr); |
538 | } |
539 | |
540 | |
541 | static int mlx4_en_tunnel_steer_add(struct mlx4_en_priv *priv, |
542 | const unsigned char *addr, |
543 | int qpn, u64 *reg_id) |
544 | { |
545 | int err; |
546 | |
547 | if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN || |
548 | priv->mdev->dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC) |
549 | return 0; /* do nothing */ |
550 | |
551 | err = mlx4_tunnel_steer_add(dev: priv->mdev->dev, addr, port: priv->port, qpn, |
552 | prio: MLX4_DOMAIN_NIC, reg_id); |
553 | if (err) { |
554 | en_err(priv, "failed to add vxlan steering rule, err %d\n" , err); |
555 | return err; |
556 | } |
557 | en_dbg(DRV, priv, "added vxlan steering rule, mac %pM reg_id %llx\n" , addr, *reg_id); |
558 | return 0; |
559 | } |
560 | |
561 | |
562 | static int mlx4_en_uc_steer_add(struct mlx4_en_priv *priv, |
563 | const unsigned char *mac, int *qpn, u64 *reg_id) |
564 | { |
565 | struct mlx4_en_dev *mdev = priv->mdev; |
566 | struct mlx4_dev *dev = mdev->dev; |
567 | int err; |
568 | |
569 | switch (dev->caps.steering_mode) { |
570 | case MLX4_STEERING_MODE_B0: { |
571 | struct mlx4_qp qp; |
572 | u8 gid[16] = {0}; |
573 | |
574 | qp.qpn = *qpn; |
575 | memcpy(&gid[10], mac, ETH_ALEN); |
576 | gid[5] = priv->port; |
577 | |
578 | err = mlx4_unicast_attach(dev, qp: &qp, gid, block_mcast_loopback: 0, prot: MLX4_PROT_ETH); |
579 | break; |
580 | } |
581 | case MLX4_STEERING_MODE_DEVICE_MANAGED: { |
582 | struct mlx4_spec_list spec_eth = { {NULL} }; |
583 | __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); |
584 | |
585 | struct mlx4_net_trans_rule rule = { |
586 | .queue_mode = MLX4_NET_TRANS_Q_FIFO, |
587 | .exclusive = 0, |
588 | .allow_loopback = 1, |
589 | .promisc_mode = MLX4_FS_REGULAR, |
590 | .priority = MLX4_DOMAIN_NIC, |
591 | }; |
592 | |
593 | rule.port = priv->port; |
594 | rule.qpn = *qpn; |
595 | INIT_LIST_HEAD(list: &rule.list); |
596 | |
597 | spec_eth.id = MLX4_NET_TRANS_RULE_ID_ETH; |
598 | memcpy(spec_eth.eth.dst_mac, mac, ETH_ALEN); |
599 | memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); |
600 | list_add_tail(new: &spec_eth.list, head: &rule.list); |
601 | |
602 | err = mlx4_flow_attach(dev, rule: &rule, reg_id); |
603 | break; |
604 | } |
605 | default: |
606 | return -EINVAL; |
607 | } |
608 | if (err) |
609 | en_warn(priv, "Failed Attaching Unicast\n" ); |
610 | |
611 | return err; |
612 | } |
613 | |
614 | static void mlx4_en_uc_steer_release(struct mlx4_en_priv *priv, |
615 | const unsigned char *mac, |
616 | int qpn, u64 reg_id) |
617 | { |
618 | struct mlx4_en_dev *mdev = priv->mdev; |
619 | struct mlx4_dev *dev = mdev->dev; |
620 | |
621 | switch (dev->caps.steering_mode) { |
622 | case MLX4_STEERING_MODE_B0: { |
623 | struct mlx4_qp qp; |
624 | u8 gid[16] = {0}; |
625 | |
626 | qp.qpn = qpn; |
627 | memcpy(&gid[10], mac, ETH_ALEN); |
628 | gid[5] = priv->port; |
629 | |
630 | mlx4_unicast_detach(dev, qp: &qp, gid, prot: MLX4_PROT_ETH); |
631 | break; |
632 | } |
633 | case MLX4_STEERING_MODE_DEVICE_MANAGED: { |
634 | mlx4_flow_detach(dev, reg_id); |
635 | break; |
636 | } |
637 | default: |
638 | en_err(priv, "Invalid steering mode.\n" ); |
639 | } |
640 | } |
641 | |
642 | static int mlx4_en_get_qp(struct mlx4_en_priv *priv) |
643 | { |
644 | struct mlx4_en_dev *mdev = priv->mdev; |
645 | struct mlx4_dev *dev = mdev->dev; |
646 | int index = 0; |
647 | int err = 0; |
648 | int *qpn = &priv->base_qpn; |
649 | u64 mac = ether_addr_to_u64(addr: priv->dev->dev_addr); |
650 | |
651 | en_dbg(DRV, priv, "Registering MAC: %pM for adding\n" , |
652 | priv->dev->dev_addr); |
653 | index = mlx4_register_mac(dev, port: priv->port, mac); |
654 | if (index < 0) { |
655 | err = index; |
656 | en_err(priv, "Failed adding MAC: %pM\n" , |
657 | priv->dev->dev_addr); |
658 | return err; |
659 | } |
660 | |
661 | en_info(priv, "Steering Mode %d\n" , dev->caps.steering_mode); |
662 | |
663 | if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { |
664 | int base_qpn = mlx4_get_base_qpn(dev, port: priv->port); |
665 | *qpn = base_qpn + index; |
666 | return 0; |
667 | } |
668 | |
669 | err = mlx4_qp_reserve_range(dev, cnt: 1, align: 1, base: qpn, flags: MLX4_RESERVE_A0_QP, |
670 | usage: MLX4_RES_USAGE_DRIVER); |
671 | en_dbg(DRV, priv, "Reserved qp %d\n" , *qpn); |
672 | if (err) { |
673 | en_err(priv, "Failed to reserve qp for mac registration\n" ); |
674 | mlx4_unregister_mac(dev, port: priv->port, mac); |
675 | return err; |
676 | } |
677 | |
678 | return 0; |
679 | } |
680 | |
681 | static void mlx4_en_put_qp(struct mlx4_en_priv *priv) |
682 | { |
683 | struct mlx4_en_dev *mdev = priv->mdev; |
684 | struct mlx4_dev *dev = mdev->dev; |
685 | int qpn = priv->base_qpn; |
686 | |
687 | if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { |
688 | u64 mac = ether_addr_to_u64(addr: priv->dev->dev_addr); |
689 | en_dbg(DRV, priv, "Registering MAC: %pM for deleting\n" , |
690 | priv->dev->dev_addr); |
691 | mlx4_unregister_mac(dev, port: priv->port, mac); |
692 | } else { |
693 | en_dbg(DRV, priv, "Releasing qp: port %d, qpn %d\n" , |
694 | priv->port, qpn); |
695 | mlx4_qp_release_range(dev, base_qpn: qpn, cnt: 1); |
696 | priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC; |
697 | } |
698 | } |
699 | |
700 | static int mlx4_en_replace_mac(struct mlx4_en_priv *priv, int qpn, |
701 | unsigned char *new_mac, unsigned char *prev_mac) |
702 | { |
703 | struct mlx4_en_dev *mdev = priv->mdev; |
704 | struct mlx4_dev *dev = mdev->dev; |
705 | int err = 0; |
706 | u64 new_mac_u64 = ether_addr_to_u64(addr: new_mac); |
707 | |
708 | if (dev->caps.steering_mode != MLX4_STEERING_MODE_A0) { |
709 | struct hlist_head *bucket; |
710 | unsigned int mac_hash; |
711 | struct mlx4_mac_entry *entry; |
712 | struct hlist_node *tmp; |
713 | u64 prev_mac_u64 = ether_addr_to_u64(addr: prev_mac); |
714 | |
715 | bucket = &priv->mac_hash[prev_mac[MLX4_EN_MAC_HASH_IDX]]; |
716 | hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { |
717 | if (ether_addr_equal_64bits(addr1: entry->mac, addr2: prev_mac)) { |
718 | mlx4_en_uc_steer_release(priv, mac: entry->mac, |
719 | qpn, reg_id: entry->reg_id); |
720 | mlx4_unregister_mac(dev, port: priv->port, |
721 | mac: prev_mac_u64); |
722 | hlist_del_rcu(n: &entry->hlist); |
723 | synchronize_rcu(); |
724 | memcpy(entry->mac, new_mac, ETH_ALEN); |
725 | entry->reg_id = 0; |
726 | mac_hash = new_mac[MLX4_EN_MAC_HASH_IDX]; |
727 | hlist_add_head_rcu(n: &entry->hlist, |
728 | h: &priv->mac_hash[mac_hash]); |
729 | mlx4_register_mac(dev, port: priv->port, mac: new_mac_u64); |
730 | err = mlx4_en_uc_steer_add(priv, mac: new_mac, |
731 | qpn: &qpn, |
732 | reg_id: &entry->reg_id); |
733 | if (err) |
734 | return err; |
735 | if (priv->tunnel_reg_id) { |
736 | mlx4_flow_detach(dev: priv->mdev->dev, reg_id: priv->tunnel_reg_id); |
737 | priv->tunnel_reg_id = 0; |
738 | } |
739 | err = mlx4_en_tunnel_steer_add(priv, addr: new_mac, qpn, |
740 | reg_id: &priv->tunnel_reg_id); |
741 | return err; |
742 | } |
743 | } |
744 | return -EINVAL; |
745 | } |
746 | |
747 | return __mlx4_replace_mac(dev, port: priv->port, qpn, new_mac: new_mac_u64); |
748 | } |
749 | |
750 | static void mlx4_en_update_user_mac(struct mlx4_en_priv *priv, |
751 | unsigned char new_mac[ETH_ALEN + 2]) |
752 | { |
753 | struct mlx4_en_dev *mdev = priv->mdev; |
754 | int err; |
755 | |
756 | if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_USER_MAC_EN)) |
757 | return; |
758 | |
759 | err = mlx4_SET_PORT_user_mac(dev: mdev->dev, port: priv->port, user_mac: new_mac); |
760 | if (err) |
761 | en_err(priv, "Failed to pass user MAC(%pM) to Firmware for port %d, with error %d\n" , |
762 | new_mac, priv->port, err); |
763 | } |
764 | |
765 | static int mlx4_en_do_set_mac(struct mlx4_en_priv *priv, |
766 | unsigned char new_mac[ETH_ALEN + 2]) |
767 | { |
768 | int err = 0; |
769 | |
770 | if (priv->port_up) { |
771 | /* Remove old MAC and insert the new one */ |
772 | err = mlx4_en_replace_mac(priv, qpn: priv->base_qpn, |
773 | new_mac, prev_mac: priv->current_mac); |
774 | if (err) |
775 | en_err(priv, "Failed changing HW MAC address\n" ); |
776 | } else |
777 | en_dbg(HW, priv, "Port is down while registering mac, exiting...\n" ); |
778 | |
779 | if (!err) |
780 | memcpy(priv->current_mac, new_mac, sizeof(priv->current_mac)); |
781 | |
782 | return err; |
783 | } |
784 | |
785 | static int mlx4_en_set_mac(struct net_device *dev, void *addr) |
786 | { |
787 | struct mlx4_en_priv *priv = netdev_priv(dev); |
788 | struct mlx4_en_dev *mdev = priv->mdev; |
789 | struct sockaddr *saddr = addr; |
790 | unsigned char new_mac[ETH_ALEN + 2]; |
791 | int err; |
792 | |
793 | if (!is_valid_ether_addr(addr: saddr->sa_data)) |
794 | return -EADDRNOTAVAIL; |
795 | |
796 | mutex_lock(&mdev->state_lock); |
797 | memcpy(new_mac, saddr->sa_data, ETH_ALEN); |
798 | err = mlx4_en_do_set_mac(priv, new_mac); |
799 | if (err) |
800 | goto out; |
801 | |
802 | eth_hw_addr_set(dev, addr: saddr->sa_data); |
803 | mlx4_en_update_user_mac(priv, new_mac); |
804 | out: |
805 | mutex_unlock(lock: &mdev->state_lock); |
806 | |
807 | return err; |
808 | } |
809 | |
810 | static void mlx4_en_clear_list(struct net_device *dev) |
811 | { |
812 | struct mlx4_en_priv *priv = netdev_priv(dev); |
813 | struct mlx4_en_mc_list *tmp, *mc_to_del; |
814 | |
815 | list_for_each_entry_safe(mc_to_del, tmp, &priv->mc_list, list) { |
816 | list_del(entry: &mc_to_del->list); |
817 | kfree(objp: mc_to_del); |
818 | } |
819 | } |
820 | |
821 | static void mlx4_en_cache_mclist(struct net_device *dev) |
822 | { |
823 | struct mlx4_en_priv *priv = netdev_priv(dev); |
824 | struct netdev_hw_addr *ha; |
825 | struct mlx4_en_mc_list *tmp; |
826 | |
827 | mlx4_en_clear_list(dev); |
828 | netdev_for_each_mc_addr(ha, dev) { |
829 | tmp = kzalloc(size: sizeof(struct mlx4_en_mc_list), GFP_ATOMIC); |
830 | if (!tmp) { |
831 | mlx4_en_clear_list(dev); |
832 | return; |
833 | } |
834 | memcpy(tmp->addr, ha->addr, ETH_ALEN); |
835 | list_add_tail(new: &tmp->list, head: &priv->mc_list); |
836 | } |
837 | } |
838 | |
839 | static void update_mclist_flags(struct mlx4_en_priv *priv, |
840 | struct list_head *dst, |
841 | struct list_head *src) |
842 | { |
843 | struct mlx4_en_mc_list *dst_tmp, *src_tmp, *new_mc; |
844 | bool found; |
845 | |
846 | /* Find all the entries that should be removed from dst, |
847 | * These are the entries that are not found in src |
848 | */ |
849 | list_for_each_entry(dst_tmp, dst, list) { |
850 | found = false; |
851 | list_for_each_entry(src_tmp, src, list) { |
852 | if (ether_addr_equal(addr1: dst_tmp->addr, addr2: src_tmp->addr)) { |
853 | found = true; |
854 | break; |
855 | } |
856 | } |
857 | if (!found) |
858 | dst_tmp->action = MCLIST_REM; |
859 | } |
860 | |
861 | /* Add entries that exist in src but not in dst |
862 | * mark them as need to add |
863 | */ |
864 | list_for_each_entry(src_tmp, src, list) { |
865 | found = false; |
866 | list_for_each_entry(dst_tmp, dst, list) { |
867 | if (ether_addr_equal(addr1: dst_tmp->addr, addr2: src_tmp->addr)) { |
868 | dst_tmp->action = MCLIST_NONE; |
869 | found = true; |
870 | break; |
871 | } |
872 | } |
873 | if (!found) { |
874 | new_mc = kmemdup(p: src_tmp, |
875 | size: sizeof(struct mlx4_en_mc_list), |
876 | GFP_KERNEL); |
877 | if (!new_mc) |
878 | return; |
879 | |
880 | new_mc->action = MCLIST_ADD; |
881 | list_add_tail(new: &new_mc->list, head: dst); |
882 | } |
883 | } |
884 | } |
885 | |
886 | static void mlx4_en_set_rx_mode(struct net_device *dev) |
887 | { |
888 | struct mlx4_en_priv *priv = netdev_priv(dev); |
889 | |
890 | if (!priv->port_up) |
891 | return; |
892 | |
893 | queue_work(wq: priv->mdev->workqueue, work: &priv->rx_mode_task); |
894 | } |
895 | |
896 | static void mlx4_en_set_promisc_mode(struct mlx4_en_priv *priv, |
897 | struct mlx4_en_dev *mdev) |
898 | { |
899 | int err = 0; |
900 | |
901 | if (!(priv->flags & MLX4_EN_FLAG_PROMISC)) { |
902 | if (netif_msg_rx_status(priv)) |
903 | en_warn(priv, "Entering promiscuous mode\n" ); |
904 | priv->flags |= MLX4_EN_FLAG_PROMISC; |
905 | |
906 | /* Enable promiscouos mode */ |
907 | switch (mdev->dev->caps.steering_mode) { |
908 | case MLX4_STEERING_MODE_DEVICE_MANAGED: |
909 | err = mlx4_flow_steer_promisc_add(dev: mdev->dev, |
910 | port: priv->port, |
911 | qpn: priv->base_qpn, |
912 | mode: MLX4_FS_ALL_DEFAULT); |
913 | if (err) |
914 | en_err(priv, "Failed enabling promiscuous mode\n" ); |
915 | priv->flags |= MLX4_EN_FLAG_MC_PROMISC; |
916 | break; |
917 | |
918 | case MLX4_STEERING_MODE_B0: |
919 | err = mlx4_unicast_promisc_add(dev: mdev->dev, |
920 | qpn: priv->base_qpn, |
921 | port: priv->port); |
922 | if (err) |
923 | en_err(priv, "Failed enabling unicast promiscuous mode\n" ); |
924 | |
925 | /* Add the default qp number as multicast |
926 | * promisc |
927 | */ |
928 | if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { |
929 | err = mlx4_multicast_promisc_add(dev: mdev->dev, |
930 | qpn: priv->base_qpn, |
931 | port: priv->port); |
932 | if (err) |
933 | en_err(priv, "Failed enabling multicast promiscuous mode\n" ); |
934 | priv->flags |= MLX4_EN_FLAG_MC_PROMISC; |
935 | } |
936 | break; |
937 | |
938 | case MLX4_STEERING_MODE_A0: |
939 | err = mlx4_SET_PORT_qpn_calc(dev: mdev->dev, |
940 | port: priv->port, |
941 | base_qpn: priv->base_qpn, |
942 | promisc: 1); |
943 | if (err) |
944 | en_err(priv, "Failed enabling promiscuous mode\n" ); |
945 | break; |
946 | } |
947 | |
948 | /* Disable port multicast filter (unconditionally) */ |
949 | err = mlx4_SET_MCAST_FLTR(dev: mdev->dev, port: priv->port, mac: 0, |
950 | clear: 0, mode: MLX4_MCAST_DISABLE); |
951 | if (err) |
952 | en_err(priv, "Failed disabling multicast filter\n" ); |
953 | } |
954 | } |
955 | |
956 | static void mlx4_en_clear_promisc_mode(struct mlx4_en_priv *priv, |
957 | struct mlx4_en_dev *mdev) |
958 | { |
959 | int err = 0; |
960 | |
961 | if (netif_msg_rx_status(priv)) |
962 | en_warn(priv, "Leaving promiscuous mode\n" ); |
963 | priv->flags &= ~MLX4_EN_FLAG_PROMISC; |
964 | |
965 | /* Disable promiscouos mode */ |
966 | switch (mdev->dev->caps.steering_mode) { |
967 | case MLX4_STEERING_MODE_DEVICE_MANAGED: |
968 | err = mlx4_flow_steer_promisc_remove(dev: mdev->dev, |
969 | port: priv->port, |
970 | mode: MLX4_FS_ALL_DEFAULT); |
971 | if (err) |
972 | en_err(priv, "Failed disabling promiscuous mode\n" ); |
973 | priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; |
974 | break; |
975 | |
976 | case MLX4_STEERING_MODE_B0: |
977 | err = mlx4_unicast_promisc_remove(dev: mdev->dev, |
978 | qpn: priv->base_qpn, |
979 | port: priv->port); |
980 | if (err) |
981 | en_err(priv, "Failed disabling unicast promiscuous mode\n" ); |
982 | /* Disable Multicast promisc */ |
983 | if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { |
984 | err = mlx4_multicast_promisc_remove(dev: mdev->dev, |
985 | qpn: priv->base_qpn, |
986 | port: priv->port); |
987 | if (err) |
988 | en_err(priv, "Failed disabling multicast promiscuous mode\n" ); |
989 | priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; |
990 | } |
991 | break; |
992 | |
993 | case MLX4_STEERING_MODE_A0: |
994 | err = mlx4_SET_PORT_qpn_calc(dev: mdev->dev, |
995 | port: priv->port, |
996 | base_qpn: priv->base_qpn, promisc: 0); |
997 | if (err) |
998 | en_err(priv, "Failed disabling promiscuous mode\n" ); |
999 | break; |
1000 | } |
1001 | } |
1002 | |
1003 | static void mlx4_en_do_multicast(struct mlx4_en_priv *priv, |
1004 | struct net_device *dev, |
1005 | struct mlx4_en_dev *mdev) |
1006 | { |
1007 | struct mlx4_en_mc_list *mclist, *tmp; |
1008 | u64 mcast_addr = 0; |
1009 | u8 mc_list[16] = {0}; |
1010 | int err = 0; |
1011 | |
1012 | /* Enable/disable the multicast filter according to IFF_ALLMULTI */ |
1013 | if (dev->flags & IFF_ALLMULTI) { |
1014 | err = mlx4_SET_MCAST_FLTR(dev: mdev->dev, port: priv->port, mac: 0, |
1015 | clear: 0, mode: MLX4_MCAST_DISABLE); |
1016 | if (err) |
1017 | en_err(priv, "Failed disabling multicast filter\n" ); |
1018 | |
1019 | /* Add the default qp number as multicast promisc */ |
1020 | if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { |
1021 | switch (mdev->dev->caps.steering_mode) { |
1022 | case MLX4_STEERING_MODE_DEVICE_MANAGED: |
1023 | err = mlx4_flow_steer_promisc_add(dev: mdev->dev, |
1024 | port: priv->port, |
1025 | qpn: priv->base_qpn, |
1026 | mode: MLX4_FS_MC_DEFAULT); |
1027 | break; |
1028 | |
1029 | case MLX4_STEERING_MODE_B0: |
1030 | err = mlx4_multicast_promisc_add(dev: mdev->dev, |
1031 | qpn: priv->base_qpn, |
1032 | port: priv->port); |
1033 | break; |
1034 | |
1035 | case MLX4_STEERING_MODE_A0: |
1036 | break; |
1037 | } |
1038 | if (err) |
1039 | en_err(priv, "Failed entering multicast promisc mode\n" ); |
1040 | priv->flags |= MLX4_EN_FLAG_MC_PROMISC; |
1041 | } |
1042 | } else { |
1043 | /* Disable Multicast promisc */ |
1044 | if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { |
1045 | switch (mdev->dev->caps.steering_mode) { |
1046 | case MLX4_STEERING_MODE_DEVICE_MANAGED: |
1047 | err = mlx4_flow_steer_promisc_remove(dev: mdev->dev, |
1048 | port: priv->port, |
1049 | mode: MLX4_FS_MC_DEFAULT); |
1050 | break; |
1051 | |
1052 | case MLX4_STEERING_MODE_B0: |
1053 | err = mlx4_multicast_promisc_remove(dev: mdev->dev, |
1054 | qpn: priv->base_qpn, |
1055 | port: priv->port); |
1056 | break; |
1057 | |
1058 | case MLX4_STEERING_MODE_A0: |
1059 | break; |
1060 | } |
1061 | if (err) |
1062 | en_err(priv, "Failed disabling multicast promiscuous mode\n" ); |
1063 | priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; |
1064 | } |
1065 | |
1066 | err = mlx4_SET_MCAST_FLTR(dev: mdev->dev, port: priv->port, mac: 0, |
1067 | clear: 0, mode: MLX4_MCAST_DISABLE); |
1068 | if (err) |
1069 | en_err(priv, "Failed disabling multicast filter\n" ); |
1070 | |
1071 | /* Flush mcast filter and init it with broadcast address */ |
1072 | mlx4_SET_MCAST_FLTR(dev: mdev->dev, port: priv->port, ETH_BCAST, |
1073 | clear: 1, mode: MLX4_MCAST_CONFIG); |
1074 | |
1075 | /* Update multicast list - we cache all addresses so they won't |
1076 | * change while HW is updated holding the command semaphore |
1077 | */ |
1078 | netif_addr_lock_bh(dev); |
1079 | mlx4_en_cache_mclist(dev); |
1080 | netif_addr_unlock_bh(dev); |
1081 | list_for_each_entry(mclist, &priv->mc_list, list) { |
1082 | mcast_addr = ether_addr_to_u64(addr: mclist->addr); |
1083 | mlx4_SET_MCAST_FLTR(dev: mdev->dev, port: priv->port, |
1084 | mac: mcast_addr, clear: 0, mode: MLX4_MCAST_CONFIG); |
1085 | } |
1086 | err = mlx4_SET_MCAST_FLTR(dev: mdev->dev, port: priv->port, mac: 0, |
1087 | clear: 0, mode: MLX4_MCAST_ENABLE); |
1088 | if (err) |
1089 | en_err(priv, "Failed enabling multicast filter\n" ); |
1090 | |
1091 | update_mclist_flags(priv, dst: &priv->curr_list, src: &priv->mc_list); |
1092 | list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) { |
1093 | if (mclist->action == MCLIST_REM) { |
1094 | /* detach this address and delete from list */ |
1095 | memcpy(&mc_list[10], mclist->addr, ETH_ALEN); |
1096 | mc_list[5] = priv->port; |
1097 | err = mlx4_multicast_detach(dev: mdev->dev, |
1098 | qp: priv->rss_map.indir_qp, |
1099 | gid: mc_list, |
1100 | protocol: MLX4_PROT_ETH, |
1101 | reg_id: mclist->reg_id); |
1102 | if (err) |
1103 | en_err(priv, "Fail to detach multicast address\n" ); |
1104 | |
1105 | if (mclist->tunnel_reg_id) { |
1106 | err = mlx4_flow_detach(dev: priv->mdev->dev, reg_id: mclist->tunnel_reg_id); |
1107 | if (err) |
1108 | en_err(priv, "Failed to detach multicast address\n" ); |
1109 | } |
1110 | |
1111 | /* remove from list */ |
1112 | list_del(entry: &mclist->list); |
1113 | kfree(objp: mclist); |
1114 | } else if (mclist->action == MCLIST_ADD) { |
1115 | /* attach the address */ |
1116 | memcpy(&mc_list[10], mclist->addr, ETH_ALEN); |
1117 | /* needed for B0 steering support */ |
1118 | mc_list[5] = priv->port; |
1119 | err = mlx4_multicast_attach(dev: mdev->dev, |
1120 | qp: priv->rss_map.indir_qp, |
1121 | gid: mc_list, |
1122 | port: priv->port, block_mcast_loopback: 0, |
1123 | protocol: MLX4_PROT_ETH, |
1124 | reg_id: &mclist->reg_id); |
1125 | if (err) |
1126 | en_err(priv, "Fail to attach multicast address\n" ); |
1127 | |
1128 | err = mlx4_en_tunnel_steer_add(priv, addr: &mc_list[10], qpn: priv->base_qpn, |
1129 | reg_id: &mclist->tunnel_reg_id); |
1130 | if (err) |
1131 | en_err(priv, "Failed to attach multicast address\n" ); |
1132 | } |
1133 | } |
1134 | } |
1135 | } |
1136 | |
1137 | static void mlx4_en_do_uc_filter(struct mlx4_en_priv *priv, |
1138 | struct net_device *dev, |
1139 | struct mlx4_en_dev *mdev) |
1140 | { |
1141 | struct netdev_hw_addr *ha; |
1142 | struct mlx4_mac_entry *entry; |
1143 | struct hlist_node *tmp; |
1144 | bool found; |
1145 | u64 mac; |
1146 | int err = 0; |
1147 | struct hlist_head *bucket; |
1148 | unsigned int i; |
1149 | int removed = 0; |
1150 | u32 prev_flags; |
1151 | |
1152 | /* Note that we do not need to protect our mac_hash traversal with rcu, |
1153 | * since all modification code is protected by mdev->state_lock |
1154 | */ |
1155 | |
1156 | /* find what to remove */ |
1157 | for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) { |
1158 | bucket = &priv->mac_hash[i]; |
1159 | hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { |
1160 | found = false; |
1161 | netdev_for_each_uc_addr(ha, dev) { |
1162 | if (ether_addr_equal_64bits(addr1: entry->mac, |
1163 | addr2: ha->addr)) { |
1164 | found = true; |
1165 | break; |
1166 | } |
1167 | } |
1168 | |
1169 | /* MAC address of the port is not in uc list */ |
1170 | if (ether_addr_equal_64bits(addr1: entry->mac, |
1171 | addr2: priv->current_mac)) |
1172 | found = true; |
1173 | |
1174 | if (!found) { |
1175 | mac = ether_addr_to_u64(addr: entry->mac); |
1176 | mlx4_en_uc_steer_release(priv, mac: entry->mac, |
1177 | qpn: priv->base_qpn, |
1178 | reg_id: entry->reg_id); |
1179 | mlx4_unregister_mac(dev: mdev->dev, port: priv->port, mac); |
1180 | |
1181 | hlist_del_rcu(n: &entry->hlist); |
1182 | kfree_rcu(entry, rcu); |
1183 | en_dbg(DRV, priv, "Removed MAC %pM on port:%d\n" , |
1184 | entry->mac, priv->port); |
1185 | ++removed; |
1186 | } |
1187 | } |
1188 | } |
1189 | |
1190 | /* if we didn't remove anything, there is no use in trying to add |
1191 | * again once we are in a forced promisc mode state |
1192 | */ |
1193 | if ((priv->flags & MLX4_EN_FLAG_FORCE_PROMISC) && 0 == removed) |
1194 | return; |
1195 | |
1196 | prev_flags = priv->flags; |
1197 | priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC; |
1198 | |
1199 | /* find what to add */ |
1200 | netdev_for_each_uc_addr(ha, dev) { |
1201 | found = false; |
1202 | bucket = &priv->mac_hash[ha->addr[MLX4_EN_MAC_HASH_IDX]]; |
1203 | hlist_for_each_entry(entry, bucket, hlist) { |
1204 | if (ether_addr_equal_64bits(addr1: entry->mac, addr2: ha->addr)) { |
1205 | found = true; |
1206 | break; |
1207 | } |
1208 | } |
1209 | |
1210 | if (!found) { |
1211 | entry = kmalloc(size: sizeof(*entry), GFP_KERNEL); |
1212 | if (!entry) { |
1213 | en_err(priv, "Failed adding MAC %pM on port:%d (out of memory)\n" , |
1214 | ha->addr, priv->port); |
1215 | priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC; |
1216 | break; |
1217 | } |
1218 | mac = ether_addr_to_u64(addr: ha->addr); |
1219 | memcpy(entry->mac, ha->addr, ETH_ALEN); |
1220 | err = mlx4_register_mac(dev: mdev->dev, port: priv->port, mac); |
1221 | if (err < 0) { |
1222 | en_err(priv, "Failed registering MAC %pM on port %d: %d\n" , |
1223 | ha->addr, priv->port, err); |
1224 | kfree(objp: entry); |
1225 | priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC; |
1226 | break; |
1227 | } |
1228 | err = mlx4_en_uc_steer_add(priv, mac: ha->addr, |
1229 | qpn: &priv->base_qpn, |
1230 | reg_id: &entry->reg_id); |
1231 | if (err) { |
1232 | en_err(priv, "Failed adding MAC %pM on port %d: %d\n" , |
1233 | ha->addr, priv->port, err); |
1234 | mlx4_unregister_mac(dev: mdev->dev, port: priv->port, mac); |
1235 | kfree(objp: entry); |
1236 | priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC; |
1237 | break; |
1238 | } else { |
1239 | unsigned int mac_hash; |
1240 | en_dbg(DRV, priv, "Added MAC %pM on port:%d\n" , |
1241 | ha->addr, priv->port); |
1242 | mac_hash = ha->addr[MLX4_EN_MAC_HASH_IDX]; |
1243 | bucket = &priv->mac_hash[mac_hash]; |
1244 | hlist_add_head_rcu(n: &entry->hlist, h: bucket); |
1245 | } |
1246 | } |
1247 | } |
1248 | |
1249 | if (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC) { |
1250 | en_warn(priv, "Forcing promiscuous mode on port:%d\n" , |
1251 | priv->port); |
1252 | } else if (prev_flags & MLX4_EN_FLAG_FORCE_PROMISC) { |
1253 | en_warn(priv, "Stop forcing promiscuous mode on port:%d\n" , |
1254 | priv->port); |
1255 | } |
1256 | } |
1257 | |
1258 | static void mlx4_en_do_set_rx_mode(struct work_struct *work) |
1259 | { |
1260 | struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, |
1261 | rx_mode_task); |
1262 | struct mlx4_en_dev *mdev = priv->mdev; |
1263 | struct net_device *dev = priv->dev; |
1264 | |
1265 | mutex_lock(&mdev->state_lock); |
1266 | if (!mdev->device_up) { |
1267 | en_dbg(HW, priv, "Card is not up, ignoring rx mode change.\n" ); |
1268 | goto out; |
1269 | } |
1270 | if (!priv->port_up) { |
1271 | en_dbg(HW, priv, "Port is down, ignoring rx mode change.\n" ); |
1272 | goto out; |
1273 | } |
1274 | |
1275 | if (!netif_carrier_ok(dev)) { |
1276 | if (!mlx4_en_QUERY_PORT(mdev, port: priv->port)) { |
1277 | if (priv->port_state.link_state) { |
1278 | netif_carrier_on(dev); |
1279 | en_dbg(LINK, priv, "Link Up\n" ); |
1280 | } |
1281 | } |
1282 | } |
1283 | |
1284 | if (dev->priv_flags & IFF_UNICAST_FLT) |
1285 | mlx4_en_do_uc_filter(priv, dev, mdev); |
1286 | |
1287 | /* Promsicuous mode: disable all filters */ |
1288 | if ((dev->flags & IFF_PROMISC) || |
1289 | (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC)) { |
1290 | mlx4_en_set_promisc_mode(priv, mdev); |
1291 | goto out; |
1292 | } |
1293 | |
1294 | /* Not in promiscuous mode */ |
1295 | if (priv->flags & MLX4_EN_FLAG_PROMISC) |
1296 | mlx4_en_clear_promisc_mode(priv, mdev); |
1297 | |
1298 | mlx4_en_do_multicast(priv, dev, mdev); |
1299 | out: |
1300 | mutex_unlock(lock: &mdev->state_lock); |
1301 | } |
1302 | |
1303 | static int (struct mlx4_en_priv *priv) |
1304 | { |
1305 | u64 reg_id; |
1306 | int err = 0; |
1307 | int *qpn = &priv->base_qpn; |
1308 | struct mlx4_mac_entry *entry; |
1309 | |
1310 | err = mlx4_en_uc_steer_add(priv, mac: priv->dev->dev_addr, qpn, reg_id: ®_id); |
1311 | if (err) |
1312 | return err; |
1313 | |
1314 | err = mlx4_en_tunnel_steer_add(priv, addr: priv->dev->dev_addr, qpn: *qpn, |
1315 | reg_id: &priv->tunnel_reg_id); |
1316 | if (err) |
1317 | goto tunnel_err; |
1318 | |
1319 | entry = kmalloc(size: sizeof(*entry), GFP_KERNEL); |
1320 | if (!entry) { |
1321 | err = -ENOMEM; |
1322 | goto alloc_err; |
1323 | } |
1324 | |
1325 | memcpy(entry->mac, priv->dev->dev_addr, sizeof(entry->mac)); |
1326 | memcpy(priv->current_mac, entry->mac, sizeof(priv->current_mac)); |
1327 | entry->reg_id = reg_id; |
1328 | hlist_add_head_rcu(n: &entry->hlist, |
1329 | h: &priv->mac_hash[entry->mac[MLX4_EN_MAC_HASH_IDX]]); |
1330 | |
1331 | return 0; |
1332 | |
1333 | alloc_err: |
1334 | if (priv->tunnel_reg_id) |
1335 | mlx4_flow_detach(dev: priv->mdev->dev, reg_id: priv->tunnel_reg_id); |
1336 | |
1337 | tunnel_err: |
1338 | mlx4_en_uc_steer_release(priv, mac: priv->dev->dev_addr, qpn: *qpn, reg_id); |
1339 | return err; |
1340 | } |
1341 | |
1342 | static void (struct mlx4_en_priv *priv) |
1343 | { |
1344 | u64 mac; |
1345 | unsigned int i; |
1346 | int qpn = priv->base_qpn; |
1347 | struct hlist_head *bucket; |
1348 | struct hlist_node *tmp; |
1349 | struct mlx4_mac_entry *entry; |
1350 | |
1351 | for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) { |
1352 | bucket = &priv->mac_hash[i]; |
1353 | hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { |
1354 | mac = ether_addr_to_u64(addr: entry->mac); |
1355 | en_dbg(DRV, priv, "Registering MAC:%pM for deleting\n" , |
1356 | entry->mac); |
1357 | mlx4_en_uc_steer_release(priv, mac: entry->mac, |
1358 | qpn, reg_id: entry->reg_id); |
1359 | |
1360 | mlx4_unregister_mac(dev: priv->mdev->dev, port: priv->port, mac); |
1361 | hlist_del_rcu(n: &entry->hlist); |
1362 | kfree_rcu(entry, rcu); |
1363 | } |
1364 | } |
1365 | |
1366 | if (priv->tunnel_reg_id) { |
1367 | mlx4_flow_detach(dev: priv->mdev->dev, reg_id: priv->tunnel_reg_id); |
1368 | priv->tunnel_reg_id = 0; |
1369 | } |
1370 | } |
1371 | |
1372 | static void mlx4_en_tx_timeout(struct net_device *dev, unsigned int txqueue) |
1373 | { |
1374 | struct mlx4_en_priv *priv = netdev_priv(dev); |
1375 | struct mlx4_en_dev *mdev = priv->mdev; |
1376 | struct mlx4_en_tx_ring *tx_ring = priv->tx_ring[TX][txqueue]; |
1377 | |
1378 | if (netif_msg_timer(priv)) |
1379 | en_warn(priv, "Tx timeout called on port:%d\n" , priv->port); |
1380 | |
1381 | en_warn(priv, "TX timeout on queue: %d, QP: 0x%x, CQ: 0x%x, Cons: 0x%x, Prod: 0x%x\n" , |
1382 | txqueue, tx_ring->qpn, tx_ring->sp_cqn, |
1383 | tx_ring->cons, tx_ring->prod); |
1384 | |
1385 | priv->port_stats.tx_timeout++; |
1386 | if (!test_and_set_bit(nr: MLX4_EN_STATE_FLAG_RESTARTING, addr: &priv->state)) { |
1387 | en_dbg(DRV, priv, "Scheduling port restart\n" ); |
1388 | queue_work(wq: mdev->workqueue, work: &priv->restart_task); |
1389 | } |
1390 | } |
1391 | |
1392 | |
1393 | static void |
1394 | mlx4_en_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) |
1395 | { |
1396 | struct mlx4_en_priv *priv = netdev_priv(dev); |
1397 | |
1398 | spin_lock_bh(lock: &priv->stats_lock); |
1399 | mlx4_en_fold_software_stats(dev); |
1400 | netdev_stats_to_stats64(stats64: stats, netdev_stats: &dev->stats); |
1401 | spin_unlock_bh(lock: &priv->stats_lock); |
1402 | } |
1403 | |
1404 | static void mlx4_en_set_default_moderation(struct mlx4_en_priv *priv) |
1405 | { |
1406 | struct mlx4_en_cq *cq; |
1407 | int i, t; |
1408 | |
1409 | /* If we haven't received a specific coalescing setting |
1410 | * (module param), we set the moderation parameters as follows: |
1411 | * - moder_cnt is set to the number of mtu sized packets to |
1412 | * satisfy our coalescing target. |
1413 | * - moder_time is set to a fixed value. |
1414 | */ |
1415 | priv->rx_frames = MLX4_EN_RX_COAL_TARGET; |
1416 | priv->rx_usecs = MLX4_EN_RX_COAL_TIME; |
1417 | priv->tx_frames = MLX4_EN_TX_COAL_PKTS; |
1418 | priv->tx_usecs = MLX4_EN_TX_COAL_TIME; |
1419 | en_dbg(INTR, priv, "Default coalescing params for mtu:%d - rx_frames:%d rx_usecs:%d\n" , |
1420 | priv->dev->mtu, priv->rx_frames, priv->rx_usecs); |
1421 | |
1422 | /* Setup cq moderation params */ |
1423 | for (i = 0; i < priv->rx_ring_num; i++) { |
1424 | cq = priv->rx_cq[i]; |
1425 | cq->moder_cnt = priv->rx_frames; |
1426 | cq->moder_time = priv->rx_usecs; |
1427 | priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; |
1428 | priv->last_moder_packets[i] = 0; |
1429 | priv->last_moder_bytes[i] = 0; |
1430 | } |
1431 | |
1432 | for (t = 0 ; t < MLX4_EN_NUM_TX_TYPES; t++) { |
1433 | for (i = 0; i < priv->tx_ring_num[t]; i++) { |
1434 | cq = priv->tx_cq[t][i]; |
1435 | cq->moder_cnt = priv->tx_frames; |
1436 | cq->moder_time = priv->tx_usecs; |
1437 | } |
1438 | } |
1439 | |
1440 | /* Reset auto-moderation params */ |
1441 | priv->pkt_rate_low = MLX4_EN_RX_RATE_LOW; |
1442 | priv->rx_usecs_low = MLX4_EN_RX_COAL_TIME_LOW; |
1443 | priv->pkt_rate_high = MLX4_EN_RX_RATE_HIGH; |
1444 | priv->rx_usecs_high = MLX4_EN_RX_COAL_TIME_HIGH; |
1445 | priv->sample_interval = MLX4_EN_SAMPLE_INTERVAL; |
1446 | priv->adaptive_rx_coal = 1; |
1447 | priv->last_moder_jiffies = 0; |
1448 | priv->last_moder_tx_packets = 0; |
1449 | } |
1450 | |
1451 | static void mlx4_en_auto_moderation(struct mlx4_en_priv *priv) |
1452 | { |
1453 | unsigned long period = (unsigned long) (jiffies - priv->last_moder_jiffies); |
1454 | u32 pkt_rate_high, pkt_rate_low; |
1455 | struct mlx4_en_cq *cq; |
1456 | unsigned long packets; |
1457 | unsigned long rate; |
1458 | unsigned long avg_pkt_size; |
1459 | unsigned long rx_packets; |
1460 | unsigned long rx_bytes; |
1461 | unsigned long rx_pkt_diff; |
1462 | int moder_time; |
1463 | int ring, err; |
1464 | |
1465 | if (!priv->adaptive_rx_coal || period < priv->sample_interval * HZ) |
1466 | return; |
1467 | |
1468 | pkt_rate_low = READ_ONCE(priv->pkt_rate_low); |
1469 | pkt_rate_high = READ_ONCE(priv->pkt_rate_high); |
1470 | |
1471 | for (ring = 0; ring < priv->rx_ring_num; ring++) { |
1472 | rx_packets = READ_ONCE(priv->rx_ring[ring]->packets); |
1473 | rx_bytes = READ_ONCE(priv->rx_ring[ring]->bytes); |
1474 | |
1475 | rx_pkt_diff = rx_packets - priv->last_moder_packets[ring]; |
1476 | packets = rx_pkt_diff; |
1477 | rate = packets * HZ / period; |
1478 | avg_pkt_size = packets ? (rx_bytes - |
1479 | priv->last_moder_bytes[ring]) / packets : 0; |
1480 | |
1481 | /* Apply auto-moderation only when packet rate |
1482 | * exceeds a rate that it matters */ |
1483 | if (rate > (MLX4_EN_RX_RATE_THRESH / priv->rx_ring_num) && |
1484 | avg_pkt_size > MLX4_EN_AVG_PKT_SMALL) { |
1485 | if (rate <= pkt_rate_low) |
1486 | moder_time = priv->rx_usecs_low; |
1487 | else if (rate >= pkt_rate_high) |
1488 | moder_time = priv->rx_usecs_high; |
1489 | else |
1490 | moder_time = (rate - pkt_rate_low) * |
1491 | (priv->rx_usecs_high - priv->rx_usecs_low) / |
1492 | (pkt_rate_high - pkt_rate_low) + |
1493 | priv->rx_usecs_low; |
1494 | } else { |
1495 | moder_time = priv->rx_usecs_low; |
1496 | } |
1497 | |
1498 | cq = priv->rx_cq[ring]; |
1499 | if (moder_time != priv->last_moder_time[ring] || |
1500 | cq->moder_cnt != priv->rx_frames) { |
1501 | priv->last_moder_time[ring] = moder_time; |
1502 | cq->moder_time = moder_time; |
1503 | cq->moder_cnt = priv->rx_frames; |
1504 | err = mlx4_en_set_cq_moder(priv, cq); |
1505 | if (err) |
1506 | en_err(priv, "Failed modifying moderation for cq:%d\n" , |
1507 | ring); |
1508 | } |
1509 | priv->last_moder_packets[ring] = rx_packets; |
1510 | priv->last_moder_bytes[ring] = rx_bytes; |
1511 | } |
1512 | |
1513 | priv->last_moder_jiffies = jiffies; |
1514 | } |
1515 | |
1516 | static void mlx4_en_do_get_stats(struct work_struct *work) |
1517 | { |
1518 | struct delayed_work *delay = to_delayed_work(work); |
1519 | struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, |
1520 | stats_task); |
1521 | struct mlx4_en_dev *mdev = priv->mdev; |
1522 | int err; |
1523 | |
1524 | mutex_lock(&mdev->state_lock); |
1525 | if (mdev->device_up) { |
1526 | if (priv->port_up) { |
1527 | err = mlx4_en_DUMP_ETH_STATS(mdev, port: priv->port, reset: 0); |
1528 | if (err) |
1529 | en_dbg(HW, priv, "Could not update stats\n" ); |
1530 | |
1531 | mlx4_en_auto_moderation(priv); |
1532 | } |
1533 | |
1534 | queue_delayed_work(wq: mdev->workqueue, dwork: &priv->stats_task, STATS_DELAY); |
1535 | } |
1536 | if (mdev->mac_removed[MLX4_MAX_PORTS + 1 - priv->port]) { |
1537 | mlx4_en_do_set_mac(priv, new_mac: priv->current_mac); |
1538 | mdev->mac_removed[MLX4_MAX_PORTS + 1 - priv->port] = 0; |
1539 | } |
1540 | mutex_unlock(lock: &mdev->state_lock); |
1541 | } |
1542 | |
1543 | /* mlx4_en_service_task - Run service task for tasks that needed to be done |
1544 | * periodically |
1545 | */ |
1546 | static void mlx4_en_service_task(struct work_struct *work) |
1547 | { |
1548 | struct delayed_work *delay = to_delayed_work(work); |
1549 | struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, |
1550 | service_task); |
1551 | struct mlx4_en_dev *mdev = priv->mdev; |
1552 | |
1553 | mutex_lock(&mdev->state_lock); |
1554 | if (mdev->device_up) { |
1555 | if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) |
1556 | mlx4_en_ptp_overflow_check(mdev); |
1557 | |
1558 | mlx4_en_recover_from_oom(priv); |
1559 | queue_delayed_work(wq: mdev->workqueue, dwork: &priv->service_task, |
1560 | SERVICE_TASK_DELAY); |
1561 | } |
1562 | mutex_unlock(lock: &mdev->state_lock); |
1563 | } |
1564 | |
1565 | static void mlx4_en_linkstate(struct mlx4_en_priv *priv) |
1566 | { |
1567 | struct mlx4_en_port_state *port_state = &priv->port_state; |
1568 | struct mlx4_en_dev *mdev = priv->mdev; |
1569 | struct net_device *dev = priv->dev; |
1570 | bool up; |
1571 | |
1572 | if (mlx4_en_QUERY_PORT(mdev, port: priv->port)) |
1573 | port_state->link_state = MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN; |
1574 | |
1575 | up = port_state->link_state == MLX4_PORT_STATE_DEV_EVENT_PORT_UP; |
1576 | if (up == netif_carrier_ok(dev)) |
1577 | netif_carrier_event(dev); |
1578 | if (!up) { |
1579 | en_info(priv, "Link Down\n" ); |
1580 | netif_carrier_off(dev); |
1581 | } else { |
1582 | en_info(priv, "Link Up\n" ); |
1583 | netif_carrier_on(dev); |
1584 | } |
1585 | } |
1586 | |
1587 | static void mlx4_en_linkstate_work(struct work_struct *work) |
1588 | { |
1589 | struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, |
1590 | linkstate_task); |
1591 | struct mlx4_en_dev *mdev = priv->mdev; |
1592 | |
1593 | mutex_lock(&mdev->state_lock); |
1594 | mlx4_en_linkstate(priv); |
1595 | mutex_unlock(lock: &mdev->state_lock); |
1596 | } |
1597 | |
1598 | static int mlx4_en_init_affinity_hint(struct mlx4_en_priv *priv, int ring_idx) |
1599 | { |
1600 | struct mlx4_en_rx_ring *ring = priv->rx_ring[ring_idx]; |
1601 | int numa_node = priv->mdev->dev->numa_node; |
1602 | |
1603 | if (!zalloc_cpumask_var(mask: &ring->affinity_mask, GFP_KERNEL)) |
1604 | return -ENOMEM; |
1605 | |
1606 | cpumask_set_cpu(cpu: cpumask_local_spread(i: ring_idx, node: numa_node), |
1607 | dstp: ring->affinity_mask); |
1608 | return 0; |
1609 | } |
1610 | |
1611 | static void mlx4_en_free_affinity_hint(struct mlx4_en_priv *priv, int ring_idx) |
1612 | { |
1613 | free_cpumask_var(mask: priv->rx_ring[ring_idx]->affinity_mask); |
1614 | } |
1615 | |
1616 | static void mlx4_en_init_recycle_ring(struct mlx4_en_priv *priv, |
1617 | int tx_ring_idx) |
1618 | { |
1619 | struct mlx4_en_tx_ring *tx_ring = priv->tx_ring[TX_XDP][tx_ring_idx]; |
1620 | int rr_index = tx_ring_idx; |
1621 | |
1622 | tx_ring->free_tx_desc = mlx4_en_recycle_tx_desc; |
1623 | tx_ring->recycle_ring = priv->rx_ring[rr_index]; |
1624 | en_dbg(DRV, priv, "Set tx_ring[%d][%d]->recycle_ring = rx_ring[%d]\n" , |
1625 | TX_XDP, tx_ring_idx, rr_index); |
1626 | } |
1627 | |
1628 | int mlx4_en_start_port(struct net_device *dev) |
1629 | { |
1630 | struct mlx4_en_priv *priv = netdev_priv(dev); |
1631 | struct mlx4_en_dev *mdev = priv->mdev; |
1632 | struct mlx4_en_cq *cq; |
1633 | struct mlx4_en_tx_ring *tx_ring; |
1634 | int rx_index = 0; |
1635 | int err = 0; |
1636 | int i, t; |
1637 | int j; |
1638 | u8 mc_list[16] = {0}; |
1639 | |
1640 | if (priv->port_up) { |
1641 | en_dbg(DRV, priv, "start port called while port already up\n" ); |
1642 | return 0; |
1643 | } |
1644 | |
1645 | INIT_LIST_HEAD(list: &priv->mc_list); |
1646 | INIT_LIST_HEAD(list: &priv->curr_list); |
1647 | INIT_LIST_HEAD(list: &priv->ethtool_list); |
1648 | memset(&priv->ethtool_rules[0], 0, |
1649 | sizeof(struct ethtool_flow_id) * MAX_NUM_OF_FS_RULES); |
1650 | |
1651 | /* Calculate Rx buf size */ |
1652 | dev->mtu = min(dev->mtu, priv->max_mtu); |
1653 | mlx4_en_calc_rx_buf(dev); |
1654 | en_dbg(DRV, priv, "Rx buf size:%d\n" , priv->rx_skb_size); |
1655 | |
1656 | /* Configure rx cq's and rings */ |
1657 | err = mlx4_en_activate_rx_rings(priv); |
1658 | if (err) { |
1659 | en_err(priv, "Failed to activate RX rings\n" ); |
1660 | return err; |
1661 | } |
1662 | for (i = 0; i < priv->rx_ring_num; i++) { |
1663 | cq = priv->rx_cq[i]; |
1664 | |
1665 | err = mlx4_en_init_affinity_hint(priv, ring_idx: i); |
1666 | if (err) { |
1667 | en_err(priv, "Failed preparing IRQ affinity hint\n" ); |
1668 | goto cq_err; |
1669 | } |
1670 | |
1671 | err = mlx4_en_activate_cq(priv, cq, cq_idx: i); |
1672 | if (err) { |
1673 | en_err(priv, "Failed activating Rx CQ\n" ); |
1674 | mlx4_en_free_affinity_hint(priv, ring_idx: i); |
1675 | goto cq_err; |
1676 | } |
1677 | |
1678 | for (j = 0; j < cq->size; j++) { |
1679 | struct mlx4_cqe *cqe = NULL; |
1680 | |
1681 | cqe = mlx4_en_get_cqe(buf: cq->buf, idx: j, cqe_sz: priv->cqe_size) + |
1682 | priv->cqe_factor; |
1683 | cqe->owner_sr_opcode = MLX4_CQE_OWNER_MASK; |
1684 | } |
1685 | |
1686 | err = mlx4_en_set_cq_moder(priv, cq); |
1687 | if (err) { |
1688 | en_err(priv, "Failed setting cq moderation parameters\n" ); |
1689 | mlx4_en_deactivate_cq(priv, cq); |
1690 | mlx4_en_free_affinity_hint(priv, ring_idx: i); |
1691 | goto cq_err; |
1692 | } |
1693 | mlx4_en_arm_cq(priv, cq); |
1694 | priv->rx_ring[i]->cqn = cq->mcq.cqn; |
1695 | ++rx_index; |
1696 | } |
1697 | |
1698 | /* Set qp number */ |
1699 | en_dbg(DRV, priv, "Getting qp number for port %d\n" , priv->port); |
1700 | err = mlx4_en_get_qp(priv); |
1701 | if (err) { |
1702 | en_err(priv, "Failed getting eth qp\n" ); |
1703 | goto cq_err; |
1704 | } |
1705 | mdev->mac_removed[priv->port] = 0; |
1706 | |
1707 | priv->counter_index = |
1708 | mlx4_get_default_counter_index(dev: mdev->dev, port: priv->port); |
1709 | |
1710 | err = mlx4_en_config_rss_steer(priv); |
1711 | if (err) { |
1712 | en_err(priv, "Failed configuring rss steering\n" ); |
1713 | goto mac_err; |
1714 | } |
1715 | |
1716 | err = mlx4_en_create_drop_qp(priv); |
1717 | if (err) |
1718 | goto rss_err; |
1719 | |
1720 | /* Configure tx cq's and rings */ |
1721 | for (t = 0 ; t < MLX4_EN_NUM_TX_TYPES; t++) { |
1722 | u8 num_tx_rings_p_up = t == TX ? |
1723 | priv->num_tx_rings_p_up : priv->tx_ring_num[t]; |
1724 | |
1725 | for (i = 0; i < priv->tx_ring_num[t]; i++) { |
1726 | /* Configure cq */ |
1727 | cq = priv->tx_cq[t][i]; |
1728 | err = mlx4_en_activate_cq(priv, cq, cq_idx: i); |
1729 | if (err) { |
1730 | en_err(priv, "Failed allocating Tx CQ\n" ); |
1731 | goto tx_err; |
1732 | } |
1733 | err = mlx4_en_set_cq_moder(priv, cq); |
1734 | if (err) { |
1735 | en_err(priv, "Failed setting cq moderation parameters\n" ); |
1736 | mlx4_en_deactivate_cq(priv, cq); |
1737 | goto tx_err; |
1738 | } |
1739 | en_dbg(DRV, priv, |
1740 | "Resetting index of collapsed CQ:%d to -1\n" , i); |
1741 | cq->buf->wqe_index = cpu_to_be16(0xffff); |
1742 | |
1743 | /* Configure ring */ |
1744 | tx_ring = priv->tx_ring[t][i]; |
1745 | err = mlx4_en_activate_tx_ring(priv, ring: tx_ring, |
1746 | cq: cq->mcq.cqn, |
1747 | user_prio: i / num_tx_rings_p_up); |
1748 | if (err) { |
1749 | en_err(priv, "Failed allocating Tx ring\n" ); |
1750 | mlx4_en_deactivate_cq(priv, cq); |
1751 | goto tx_err; |
1752 | } |
1753 | clear_bit(nr: MLX4_EN_TX_RING_STATE_RECOVERING, addr: &tx_ring->state); |
1754 | if (t != TX_XDP) { |
1755 | tx_ring->tx_queue = netdev_get_tx_queue(dev, index: i); |
1756 | tx_ring->recycle_ring = NULL; |
1757 | |
1758 | /* Arm CQ for TX completions */ |
1759 | mlx4_en_arm_cq(priv, cq); |
1760 | |
1761 | } else { |
1762 | mlx4_en_init_tx_xdp_ring_descs(priv, ring: tx_ring); |
1763 | mlx4_en_init_recycle_ring(priv, tx_ring_idx: i); |
1764 | /* XDP TX CQ should never be armed */ |
1765 | } |
1766 | |
1767 | /* Set initial ownership of all Tx TXBBs to SW (1) */ |
1768 | for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE) |
1769 | *((u32 *)(tx_ring->buf + j)) = 0xffffffff; |
1770 | } |
1771 | } |
1772 | |
1773 | /* Configure port */ |
1774 | err = mlx4_SET_PORT_general(dev: mdev->dev, port: priv->port, |
1775 | mtu: priv->rx_skb_size + ETH_FCS_LEN, |
1776 | pptx: priv->prof->tx_pause, |
1777 | pfctx: priv->prof->tx_ppp, |
1778 | pprx: priv->prof->rx_pause, |
1779 | pfcrx: priv->prof->rx_ppp); |
1780 | if (err) { |
1781 | en_err(priv, "Failed setting port general configurations for port %d, with error %d\n" , |
1782 | priv->port, err); |
1783 | goto tx_err; |
1784 | } |
1785 | |
1786 | err = mlx4_SET_PORT_user_mtu(dev: mdev->dev, port: priv->port, user_mtu: dev->mtu); |
1787 | if (err) { |
1788 | en_err(priv, "Failed to pass user MTU(%d) to Firmware for port %d, with error %d\n" , |
1789 | dev->mtu, priv->port, err); |
1790 | goto tx_err; |
1791 | } |
1792 | |
1793 | /* Set default qp number */ |
1794 | err = mlx4_SET_PORT_qpn_calc(dev: mdev->dev, port: priv->port, base_qpn: priv->base_qpn, promisc: 0); |
1795 | if (err) { |
1796 | en_err(priv, "Failed setting default qp numbers\n" ); |
1797 | goto tx_err; |
1798 | } |
1799 | |
1800 | if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) { |
1801 | err = mlx4_SET_PORT_VXLAN(dev: mdev->dev, port: priv->port, steering: VXLAN_STEER_BY_OUTER_MAC, enable: 1); |
1802 | if (err) { |
1803 | en_err(priv, "Failed setting port L2 tunnel configuration, err %d\n" , |
1804 | err); |
1805 | goto tx_err; |
1806 | } |
1807 | } |
1808 | |
1809 | /* Init port */ |
1810 | en_dbg(HW, priv, "Initializing port\n" ); |
1811 | err = mlx4_INIT_PORT(dev: mdev->dev, port: priv->port); |
1812 | if (err) { |
1813 | en_err(priv, "Failed Initializing port\n" ); |
1814 | goto tx_err; |
1815 | } |
1816 | |
1817 | /* Set Unicast and VXLAN steering rules */ |
1818 | if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0 && |
1819 | mlx4_en_set_rss_steer_rules(priv)) |
1820 | mlx4_warn(mdev, "Failed setting steering rules\n" ); |
1821 | |
1822 | /* Attach rx QP to broadcast address */ |
1823 | eth_broadcast_addr(addr: &mc_list[10]); |
1824 | mc_list[5] = priv->port; /* needed for B0 steering support */ |
1825 | if (mlx4_multicast_attach(dev: mdev->dev, qp: priv->rss_map.indir_qp, gid: mc_list, |
1826 | port: priv->port, block_mcast_loopback: 0, protocol: MLX4_PROT_ETH, |
1827 | reg_id: &priv->broadcast_id)) |
1828 | mlx4_warn(mdev, "Failed Attaching Broadcast\n" ); |
1829 | |
1830 | /* Must redo promiscuous mode setup. */ |
1831 | priv->flags &= ~(MLX4_EN_FLAG_PROMISC | MLX4_EN_FLAG_MC_PROMISC); |
1832 | |
1833 | /* Schedule multicast task to populate multicast list */ |
1834 | queue_work(wq: mdev->workqueue, work: &priv->rx_mode_task); |
1835 | |
1836 | if (priv->mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) |
1837 | udp_tunnel_nic_reset_ntf(dev); |
1838 | |
1839 | priv->port_up = true; |
1840 | |
1841 | /* Process all completions if exist to prevent |
1842 | * the queues freezing if they are full |
1843 | */ |
1844 | for (i = 0; i < priv->rx_ring_num; i++) { |
1845 | local_bh_disable(); |
1846 | napi_schedule(n: &priv->rx_cq[i]->napi); |
1847 | local_bh_enable(); |
1848 | } |
1849 | |
1850 | clear_bit(nr: MLX4_EN_STATE_FLAG_RESTARTING, addr: &priv->state); |
1851 | netif_tx_start_all_queues(dev); |
1852 | netif_device_attach(dev); |
1853 | |
1854 | return 0; |
1855 | |
1856 | tx_err: |
1857 | if (t == MLX4_EN_NUM_TX_TYPES) { |
1858 | t--; |
1859 | i = priv->tx_ring_num[t]; |
1860 | } |
1861 | while (t >= 0) { |
1862 | while (i--) { |
1863 | mlx4_en_deactivate_tx_ring(priv, ring: priv->tx_ring[t][i]); |
1864 | mlx4_en_deactivate_cq(priv, cq: priv->tx_cq[t][i]); |
1865 | } |
1866 | if (!t--) |
1867 | break; |
1868 | i = priv->tx_ring_num[t]; |
1869 | } |
1870 | mlx4_en_destroy_drop_qp(priv); |
1871 | : |
1872 | mlx4_en_release_rss_steer(priv); |
1873 | mac_err: |
1874 | mlx4_en_put_qp(priv); |
1875 | cq_err: |
1876 | while (rx_index--) { |
1877 | mlx4_en_deactivate_cq(priv, cq: priv->rx_cq[rx_index]); |
1878 | mlx4_en_free_affinity_hint(priv, ring_idx: rx_index); |
1879 | } |
1880 | for (i = 0; i < priv->rx_ring_num; i++) |
1881 | mlx4_en_deactivate_rx_ring(priv, ring: priv->rx_ring[i]); |
1882 | |
1883 | return err; /* need to close devices */ |
1884 | } |
1885 | |
1886 | |
1887 | void mlx4_en_stop_port(struct net_device *dev, int detach) |
1888 | { |
1889 | struct mlx4_en_priv *priv = netdev_priv(dev); |
1890 | struct mlx4_en_dev *mdev = priv->mdev; |
1891 | struct mlx4_en_mc_list *mclist, *tmp; |
1892 | struct ethtool_flow_id *flow, *tmp_flow; |
1893 | int i, t; |
1894 | u8 mc_list[16] = {0}; |
1895 | |
1896 | if (!priv->port_up) { |
1897 | en_dbg(DRV, priv, "stop port called while port already down\n" ); |
1898 | return; |
1899 | } |
1900 | |
1901 | /* close port*/ |
1902 | mlx4_CLOSE_PORT(dev: mdev->dev, port: priv->port); |
1903 | |
1904 | /* Synchronize with tx routine */ |
1905 | netif_tx_lock_bh(dev); |
1906 | if (detach) |
1907 | netif_device_detach(dev); |
1908 | netif_tx_stop_all_queues(dev); |
1909 | netif_tx_unlock_bh(dev); |
1910 | |
1911 | netif_tx_disable(dev); |
1912 | |
1913 | spin_lock_bh(lock: &priv->stats_lock); |
1914 | mlx4_en_fold_software_stats(dev); |
1915 | /* Set port as not active */ |
1916 | priv->port_up = false; |
1917 | spin_unlock_bh(lock: &priv->stats_lock); |
1918 | |
1919 | priv->counter_index = MLX4_SINK_COUNTER_INDEX(mdev->dev); |
1920 | |
1921 | /* Promsicuous mode */ |
1922 | if (mdev->dev->caps.steering_mode == |
1923 | MLX4_STEERING_MODE_DEVICE_MANAGED) { |
1924 | priv->flags &= ~(MLX4_EN_FLAG_PROMISC | |
1925 | MLX4_EN_FLAG_MC_PROMISC); |
1926 | mlx4_flow_steer_promisc_remove(dev: mdev->dev, |
1927 | port: priv->port, |
1928 | mode: MLX4_FS_ALL_DEFAULT); |
1929 | mlx4_flow_steer_promisc_remove(dev: mdev->dev, |
1930 | port: priv->port, |
1931 | mode: MLX4_FS_MC_DEFAULT); |
1932 | } else if (priv->flags & MLX4_EN_FLAG_PROMISC) { |
1933 | priv->flags &= ~MLX4_EN_FLAG_PROMISC; |
1934 | |
1935 | /* Disable promiscouos mode */ |
1936 | mlx4_unicast_promisc_remove(dev: mdev->dev, qpn: priv->base_qpn, |
1937 | port: priv->port); |
1938 | |
1939 | /* Disable Multicast promisc */ |
1940 | if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { |
1941 | mlx4_multicast_promisc_remove(dev: mdev->dev, qpn: priv->base_qpn, |
1942 | port: priv->port); |
1943 | priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; |
1944 | } |
1945 | } |
1946 | |
1947 | /* Detach All multicasts */ |
1948 | eth_broadcast_addr(addr: &mc_list[10]); |
1949 | mc_list[5] = priv->port; /* needed for B0 steering support */ |
1950 | mlx4_multicast_detach(dev: mdev->dev, qp: priv->rss_map.indir_qp, gid: mc_list, |
1951 | protocol: MLX4_PROT_ETH, reg_id: priv->broadcast_id); |
1952 | list_for_each_entry(mclist, &priv->curr_list, list) { |
1953 | memcpy(&mc_list[10], mclist->addr, ETH_ALEN); |
1954 | mc_list[5] = priv->port; |
1955 | mlx4_multicast_detach(dev: mdev->dev, qp: priv->rss_map.indir_qp, |
1956 | gid: mc_list, protocol: MLX4_PROT_ETH, reg_id: mclist->reg_id); |
1957 | if (mclist->tunnel_reg_id) |
1958 | mlx4_flow_detach(dev: mdev->dev, reg_id: mclist->tunnel_reg_id); |
1959 | } |
1960 | mlx4_en_clear_list(dev); |
1961 | list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) { |
1962 | list_del(entry: &mclist->list); |
1963 | kfree(objp: mclist); |
1964 | } |
1965 | |
1966 | /* Flush multicast filter */ |
1967 | mlx4_SET_MCAST_FLTR(dev: mdev->dev, port: priv->port, mac: 0, clear: 1, mode: MLX4_MCAST_CONFIG); |
1968 | |
1969 | /* Remove flow steering rules for the port*/ |
1970 | if (mdev->dev->caps.steering_mode == |
1971 | MLX4_STEERING_MODE_DEVICE_MANAGED) { |
1972 | ASSERT_RTNL(); |
1973 | list_for_each_entry_safe(flow, tmp_flow, |
1974 | &priv->ethtool_list, list) { |
1975 | mlx4_flow_detach(dev: mdev->dev, reg_id: flow->id); |
1976 | list_del(entry: &flow->list); |
1977 | } |
1978 | } |
1979 | |
1980 | mlx4_en_destroy_drop_qp(priv); |
1981 | |
1982 | /* Free TX Rings */ |
1983 | for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { |
1984 | for (i = 0; i < priv->tx_ring_num[t]; i++) { |
1985 | mlx4_en_deactivate_tx_ring(priv, ring: priv->tx_ring[t][i]); |
1986 | mlx4_en_deactivate_cq(priv, cq: priv->tx_cq[t][i]); |
1987 | } |
1988 | } |
1989 | msleep(msecs: 10); |
1990 | |
1991 | for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) |
1992 | for (i = 0; i < priv->tx_ring_num[t]; i++) |
1993 | mlx4_en_free_tx_buf(dev, ring: priv->tx_ring[t][i]); |
1994 | |
1995 | if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0) |
1996 | mlx4_en_delete_rss_steer_rules(priv); |
1997 | |
1998 | /* Free RSS qps */ |
1999 | mlx4_en_release_rss_steer(priv); |
2000 | |
2001 | /* Unregister Mac address for the port */ |
2002 | mlx4_en_put_qp(priv); |
2003 | if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_REASSIGN_MAC_EN)) |
2004 | mdev->mac_removed[priv->port] = 1; |
2005 | |
2006 | /* Free RX Rings */ |
2007 | for (i = 0; i < priv->rx_ring_num; i++) { |
2008 | struct mlx4_en_cq *cq = priv->rx_cq[i]; |
2009 | |
2010 | napi_synchronize(n: &cq->napi); |
2011 | mlx4_en_deactivate_rx_ring(priv, ring: priv->rx_ring[i]); |
2012 | mlx4_en_deactivate_cq(priv, cq); |
2013 | |
2014 | mlx4_en_free_affinity_hint(priv, ring_idx: i); |
2015 | } |
2016 | } |
2017 | |
2018 | static void mlx4_en_restart(struct work_struct *work) |
2019 | { |
2020 | struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, |
2021 | restart_task); |
2022 | struct mlx4_en_dev *mdev = priv->mdev; |
2023 | struct net_device *dev = priv->dev; |
2024 | |
2025 | en_dbg(DRV, priv, "Watchdog task called for port %d\n" , priv->port); |
2026 | |
2027 | rtnl_lock(); |
2028 | mutex_lock(&mdev->state_lock); |
2029 | if (priv->port_up) { |
2030 | mlx4_en_stop_port(dev, detach: 1); |
2031 | if (mlx4_en_start_port(dev)) |
2032 | en_err(priv, "Failed restarting port %d\n" , priv->port); |
2033 | } |
2034 | mutex_unlock(lock: &mdev->state_lock); |
2035 | rtnl_unlock(); |
2036 | } |
2037 | |
2038 | static void mlx4_en_clear_stats(struct net_device *dev) |
2039 | { |
2040 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2041 | struct mlx4_en_dev *mdev = priv->mdev; |
2042 | struct mlx4_en_tx_ring **tx_ring; |
2043 | int i; |
2044 | |
2045 | if (!mlx4_is_slave(dev: mdev->dev)) |
2046 | if (mlx4_en_DUMP_ETH_STATS(mdev, port: priv->port, reset: 1)) |
2047 | en_dbg(HW, priv, "Failed dumping statistics\n" ); |
2048 | |
2049 | memset(&priv->pkstats, 0, sizeof(priv->pkstats)); |
2050 | memset(&priv->port_stats, 0, sizeof(priv->port_stats)); |
2051 | memset(&priv->rx_flowstats, 0, sizeof(priv->rx_flowstats)); |
2052 | memset(&priv->tx_flowstats, 0, sizeof(priv->tx_flowstats)); |
2053 | memset(&priv->rx_priority_flowstats, 0, |
2054 | sizeof(priv->rx_priority_flowstats)); |
2055 | memset(&priv->tx_priority_flowstats, 0, |
2056 | sizeof(priv->tx_priority_flowstats)); |
2057 | memset(&priv->pf_stats, 0, sizeof(priv->pf_stats)); |
2058 | |
2059 | tx_ring = priv->tx_ring[TX]; |
2060 | for (i = 0; i < priv->tx_ring_num[TX]; i++) { |
2061 | tx_ring[i]->bytes = 0; |
2062 | tx_ring[i]->packets = 0; |
2063 | tx_ring[i]->tx_csum = 0; |
2064 | tx_ring[i]->tx_dropped = 0; |
2065 | tx_ring[i]->queue_stopped = 0; |
2066 | tx_ring[i]->wake_queue = 0; |
2067 | tx_ring[i]->tso_packets = 0; |
2068 | tx_ring[i]->xmit_more = 0; |
2069 | } |
2070 | for (i = 0; i < priv->rx_ring_num; i++) { |
2071 | priv->rx_ring[i]->bytes = 0; |
2072 | priv->rx_ring[i]->packets = 0; |
2073 | priv->rx_ring[i]->csum_ok = 0; |
2074 | priv->rx_ring[i]->csum_none = 0; |
2075 | priv->rx_ring[i]->csum_complete = 0; |
2076 | } |
2077 | } |
2078 | |
2079 | static int mlx4_en_open(struct net_device *dev) |
2080 | { |
2081 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2082 | struct mlx4_en_dev *mdev = priv->mdev; |
2083 | int err = 0; |
2084 | |
2085 | mutex_lock(&mdev->state_lock); |
2086 | |
2087 | if (!mdev->device_up) { |
2088 | en_err(priv, "Cannot open - device down/disabled\n" ); |
2089 | err = -EBUSY; |
2090 | goto out; |
2091 | } |
2092 | |
2093 | /* Reset HW statistics and SW counters */ |
2094 | mlx4_en_clear_stats(dev); |
2095 | |
2096 | err = mlx4_en_start_port(dev); |
2097 | if (err) { |
2098 | en_err(priv, "Failed starting port:%d\n" , priv->port); |
2099 | goto out; |
2100 | } |
2101 | mlx4_en_linkstate(priv); |
2102 | out: |
2103 | mutex_unlock(lock: &mdev->state_lock); |
2104 | return err; |
2105 | } |
2106 | |
2107 | |
2108 | static int mlx4_en_close(struct net_device *dev) |
2109 | { |
2110 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2111 | struct mlx4_en_dev *mdev = priv->mdev; |
2112 | |
2113 | en_dbg(IFDOWN, priv, "Close port called\n" ); |
2114 | |
2115 | mutex_lock(&mdev->state_lock); |
2116 | |
2117 | mlx4_en_stop_port(dev, detach: 0); |
2118 | netif_carrier_off(dev); |
2119 | |
2120 | mutex_unlock(lock: &mdev->state_lock); |
2121 | return 0; |
2122 | } |
2123 | |
2124 | static void mlx4_en_free_resources(struct mlx4_en_priv *priv) |
2125 | { |
2126 | int i, t; |
2127 | |
2128 | #ifdef CONFIG_RFS_ACCEL |
2129 | priv->dev->rx_cpu_rmap = NULL; |
2130 | #endif |
2131 | |
2132 | for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { |
2133 | for (i = 0; i < priv->tx_ring_num[t]; i++) { |
2134 | if (priv->tx_ring[t] && priv->tx_ring[t][i]) |
2135 | mlx4_en_destroy_tx_ring(priv, |
2136 | pring: &priv->tx_ring[t][i]); |
2137 | if (priv->tx_cq[t] && priv->tx_cq[t][i]) |
2138 | mlx4_en_destroy_cq(priv, pcq: &priv->tx_cq[t][i]); |
2139 | } |
2140 | kfree(objp: priv->tx_ring[t]); |
2141 | kfree(objp: priv->tx_cq[t]); |
2142 | } |
2143 | |
2144 | for (i = 0; i < priv->rx_ring_num; i++) { |
2145 | if (priv->rx_ring[i]) |
2146 | mlx4_en_destroy_rx_ring(priv, pring: &priv->rx_ring[i], |
2147 | size: priv->prof->rx_ring_size, stride: priv->stride); |
2148 | if (priv->rx_cq[i]) |
2149 | mlx4_en_destroy_cq(priv, pcq: &priv->rx_cq[i]); |
2150 | } |
2151 | |
2152 | } |
2153 | |
2154 | static int mlx4_en_alloc_resources(struct mlx4_en_priv *priv) |
2155 | { |
2156 | struct mlx4_en_port_profile *prof = priv->prof; |
2157 | int i, t; |
2158 | int node; |
2159 | |
2160 | /* Create tx Rings */ |
2161 | for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { |
2162 | for (i = 0; i < priv->tx_ring_num[t]; i++) { |
2163 | node = cpu_to_node(cpu: i % num_online_cpus()); |
2164 | if (mlx4_en_create_cq(priv, pcq: &priv->tx_cq[t][i], |
2165 | entries: prof->tx_ring_size, ring: i, mode: t, node)) |
2166 | goto err; |
2167 | |
2168 | if (mlx4_en_create_tx_ring(priv, pring: &priv->tx_ring[t][i], |
2169 | size: prof->tx_ring_size, |
2170 | TXBB_SIZE, node, queue_index: i)) |
2171 | goto err; |
2172 | } |
2173 | } |
2174 | |
2175 | /* Create rx Rings */ |
2176 | for (i = 0; i < priv->rx_ring_num; i++) { |
2177 | node = cpu_to_node(cpu: i % num_online_cpus()); |
2178 | if (mlx4_en_create_cq(priv, pcq: &priv->rx_cq[i], |
2179 | entries: prof->rx_ring_size, ring: i, mode: RX, node)) |
2180 | goto err; |
2181 | |
2182 | if (mlx4_en_create_rx_ring(priv, pring: &priv->rx_ring[i], |
2183 | size: prof->rx_ring_size, stride: priv->stride, |
2184 | node, queue_index: i)) |
2185 | goto err; |
2186 | |
2187 | } |
2188 | |
2189 | #ifdef CONFIG_RFS_ACCEL |
2190 | priv->dev->rx_cpu_rmap = mlx4_get_cpu_rmap(dev: priv->mdev->dev, port: priv->port); |
2191 | #endif |
2192 | |
2193 | return 0; |
2194 | |
2195 | err: |
2196 | en_err(priv, "Failed to allocate NIC resources\n" ); |
2197 | for (i = 0; i < priv->rx_ring_num; i++) { |
2198 | if (priv->rx_ring[i]) |
2199 | mlx4_en_destroy_rx_ring(priv, pring: &priv->rx_ring[i], |
2200 | size: prof->rx_ring_size, |
2201 | stride: priv->stride); |
2202 | if (priv->rx_cq[i]) |
2203 | mlx4_en_destroy_cq(priv, pcq: &priv->rx_cq[i]); |
2204 | } |
2205 | for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { |
2206 | for (i = 0; i < priv->tx_ring_num[t]; i++) { |
2207 | if (priv->tx_ring[t][i]) |
2208 | mlx4_en_destroy_tx_ring(priv, |
2209 | pring: &priv->tx_ring[t][i]); |
2210 | if (priv->tx_cq[t][i]) |
2211 | mlx4_en_destroy_cq(priv, pcq: &priv->tx_cq[t][i]); |
2212 | } |
2213 | } |
2214 | return -ENOMEM; |
2215 | } |
2216 | |
2217 | |
2218 | static int mlx4_en_copy_priv(struct mlx4_en_priv *dst, |
2219 | struct mlx4_en_priv *src, |
2220 | struct mlx4_en_port_profile *prof) |
2221 | { |
2222 | int t; |
2223 | |
2224 | memcpy(&dst->hwtstamp_config, &prof->hwtstamp_config, |
2225 | sizeof(dst->hwtstamp_config)); |
2226 | dst->num_tx_rings_p_up = prof->num_tx_rings_p_up; |
2227 | dst->rx_ring_num = prof->rx_ring_num; |
2228 | dst->flags = prof->flags; |
2229 | dst->mdev = src->mdev; |
2230 | dst->port = src->port; |
2231 | dst->dev = src->dev; |
2232 | dst->prof = prof; |
2233 | dst->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + |
2234 | DS_SIZE * MLX4_EN_MAX_RX_FRAGS); |
2235 | |
2236 | for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { |
2237 | dst->tx_ring_num[t] = prof->tx_ring_num[t]; |
2238 | if (!dst->tx_ring_num[t]) |
2239 | continue; |
2240 | |
2241 | dst->tx_ring[t] = kcalloc(MAX_TX_RINGS, |
2242 | size: sizeof(struct mlx4_en_tx_ring *), |
2243 | GFP_KERNEL); |
2244 | if (!dst->tx_ring[t]) |
2245 | goto err_free_tx; |
2246 | |
2247 | dst->tx_cq[t] = kcalloc(MAX_TX_RINGS, |
2248 | size: sizeof(struct mlx4_en_cq *), |
2249 | GFP_KERNEL); |
2250 | if (!dst->tx_cq[t]) { |
2251 | kfree(objp: dst->tx_ring[t]); |
2252 | goto err_free_tx; |
2253 | } |
2254 | } |
2255 | |
2256 | return 0; |
2257 | |
2258 | err_free_tx: |
2259 | while (t--) { |
2260 | kfree(objp: dst->tx_ring[t]); |
2261 | kfree(objp: dst->tx_cq[t]); |
2262 | } |
2263 | return -ENOMEM; |
2264 | } |
2265 | |
2266 | static void mlx4_en_update_priv(struct mlx4_en_priv *dst, |
2267 | struct mlx4_en_priv *src) |
2268 | { |
2269 | int t; |
2270 | memcpy(dst->rx_ring, src->rx_ring, |
2271 | sizeof(struct mlx4_en_rx_ring *) * src->rx_ring_num); |
2272 | memcpy(dst->rx_cq, src->rx_cq, |
2273 | sizeof(struct mlx4_en_cq *) * src->rx_ring_num); |
2274 | memcpy(&dst->hwtstamp_config, &src->hwtstamp_config, |
2275 | sizeof(dst->hwtstamp_config)); |
2276 | for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { |
2277 | dst->tx_ring_num[t] = src->tx_ring_num[t]; |
2278 | dst->tx_ring[t] = src->tx_ring[t]; |
2279 | dst->tx_cq[t] = src->tx_cq[t]; |
2280 | } |
2281 | dst->num_tx_rings_p_up = src->num_tx_rings_p_up; |
2282 | dst->rx_ring_num = src->rx_ring_num; |
2283 | memcpy(dst->prof, src->prof, sizeof(struct mlx4_en_port_profile)); |
2284 | } |
2285 | |
2286 | int mlx4_en_try_alloc_resources(struct mlx4_en_priv *priv, |
2287 | struct mlx4_en_priv *tmp, |
2288 | struct mlx4_en_port_profile *prof, |
2289 | bool carry_xdp_prog) |
2290 | { |
2291 | struct bpf_prog *xdp_prog; |
2292 | int i, t, ret; |
2293 | |
2294 | ret = mlx4_en_copy_priv(dst: tmp, src: priv, prof); |
2295 | if (ret) { |
2296 | en_warn(priv, "%s: mlx4_en_copy_priv() failed, return\n" , |
2297 | __func__); |
2298 | return ret; |
2299 | } |
2300 | |
2301 | if (mlx4_en_alloc_resources(priv: tmp)) { |
2302 | en_warn(priv, |
2303 | "%s: Resource allocation failed, using previous configuration\n" , |
2304 | __func__); |
2305 | for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { |
2306 | kfree(objp: tmp->tx_ring[t]); |
2307 | kfree(objp: tmp->tx_cq[t]); |
2308 | } |
2309 | return -ENOMEM; |
2310 | } |
2311 | |
2312 | /* All rx_rings has the same xdp_prog. Pick the first one. */ |
2313 | xdp_prog = rcu_dereference_protected( |
2314 | priv->rx_ring[0]->xdp_prog, |
2315 | lockdep_is_held(&priv->mdev->state_lock)); |
2316 | |
2317 | if (xdp_prog && carry_xdp_prog) { |
2318 | bpf_prog_add(prog: xdp_prog, i: tmp->rx_ring_num); |
2319 | for (i = 0; i < tmp->rx_ring_num; i++) |
2320 | rcu_assign_pointer(tmp->rx_ring[i]->xdp_prog, |
2321 | xdp_prog); |
2322 | } |
2323 | |
2324 | return 0; |
2325 | } |
2326 | |
2327 | void mlx4_en_safe_replace_resources(struct mlx4_en_priv *priv, |
2328 | struct mlx4_en_priv *tmp) |
2329 | { |
2330 | mlx4_en_free_resources(priv); |
2331 | mlx4_en_update_priv(dst: priv, src: tmp); |
2332 | } |
2333 | |
2334 | void mlx4_en_destroy_netdev(struct net_device *dev) |
2335 | { |
2336 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2337 | struct mlx4_en_dev *mdev = priv->mdev; |
2338 | |
2339 | en_dbg(DRV, priv, "Destroying netdev on port:%d\n" , priv->port); |
2340 | |
2341 | /* Unregister device - this will close the port if it was up */ |
2342 | if (priv->registered) |
2343 | unregister_netdev(dev); |
2344 | |
2345 | if (priv->allocated) |
2346 | mlx4_free_hwq_res(mdev: mdev->dev, wqres: &priv->res, MLX4_EN_PAGE_SIZE); |
2347 | |
2348 | cancel_delayed_work(dwork: &priv->stats_task); |
2349 | cancel_delayed_work(dwork: &priv->service_task); |
2350 | /* flush any pending task for this netdev */ |
2351 | flush_workqueue(mdev->workqueue); |
2352 | |
2353 | if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) |
2354 | mlx4_en_remove_timestamp(mdev); |
2355 | |
2356 | /* Detach the netdev so tasks would not attempt to access it */ |
2357 | mutex_lock(&mdev->state_lock); |
2358 | mdev->pndev[priv->port] = NULL; |
2359 | mdev->upper[priv->port] = NULL; |
2360 | |
2361 | #ifdef CONFIG_RFS_ACCEL |
2362 | mlx4_en_cleanup_filters(priv); |
2363 | #endif |
2364 | |
2365 | mlx4_en_free_resources(priv); |
2366 | mutex_unlock(lock: &mdev->state_lock); |
2367 | |
2368 | free_netdev(dev); |
2369 | } |
2370 | |
2371 | static bool mlx4_en_check_xdp_mtu(struct net_device *dev, int mtu) |
2372 | { |
2373 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2374 | |
2375 | if (mtu > MLX4_EN_MAX_XDP_MTU) { |
2376 | en_err(priv, "mtu:%d > max:%d when XDP prog is attached\n" , |
2377 | mtu, MLX4_EN_MAX_XDP_MTU); |
2378 | return false; |
2379 | } |
2380 | |
2381 | return true; |
2382 | } |
2383 | |
2384 | static int mlx4_en_change_mtu(struct net_device *dev, int new_mtu) |
2385 | { |
2386 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2387 | struct mlx4_en_dev *mdev = priv->mdev; |
2388 | int err = 0; |
2389 | |
2390 | en_dbg(DRV, priv, "Change MTU called - current:%d new:%d\n" , |
2391 | dev->mtu, new_mtu); |
2392 | |
2393 | if (priv->tx_ring_num[TX_XDP] && |
2394 | !mlx4_en_check_xdp_mtu(dev, mtu: new_mtu)) |
2395 | return -EOPNOTSUPP; |
2396 | |
2397 | dev->mtu = new_mtu; |
2398 | |
2399 | if (netif_running(dev)) { |
2400 | mutex_lock(&mdev->state_lock); |
2401 | if (!mdev->device_up) { |
2402 | /* NIC is probably restarting - let restart task reset |
2403 | * the port */ |
2404 | en_dbg(DRV, priv, "Change MTU called with card down!?\n" ); |
2405 | } else { |
2406 | mlx4_en_stop_port(dev, detach: 1); |
2407 | err = mlx4_en_start_port(dev); |
2408 | if (err) { |
2409 | en_err(priv, "Failed restarting port:%d\n" , |
2410 | priv->port); |
2411 | if (!test_and_set_bit(nr: MLX4_EN_STATE_FLAG_RESTARTING, |
2412 | addr: &priv->state)) |
2413 | queue_work(wq: mdev->workqueue, work: &priv->restart_task); |
2414 | } |
2415 | } |
2416 | mutex_unlock(lock: &mdev->state_lock); |
2417 | } |
2418 | return 0; |
2419 | } |
2420 | |
2421 | static int mlx4_en_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) |
2422 | { |
2423 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2424 | struct mlx4_en_dev *mdev = priv->mdev; |
2425 | struct hwtstamp_config config; |
2426 | |
2427 | if (copy_from_user(to: &config, from: ifr->ifr_data, n: sizeof(config))) |
2428 | return -EFAULT; |
2429 | |
2430 | /* device doesn't support time stamping */ |
2431 | if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS)) |
2432 | return -EINVAL; |
2433 | |
2434 | /* TX HW timestamp */ |
2435 | switch (config.tx_type) { |
2436 | case HWTSTAMP_TX_OFF: |
2437 | case HWTSTAMP_TX_ON: |
2438 | break; |
2439 | default: |
2440 | return -ERANGE; |
2441 | } |
2442 | |
2443 | /* RX HW timestamp */ |
2444 | switch (config.rx_filter) { |
2445 | case HWTSTAMP_FILTER_NONE: |
2446 | break; |
2447 | case HWTSTAMP_FILTER_ALL: |
2448 | case HWTSTAMP_FILTER_SOME: |
2449 | case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: |
2450 | case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: |
2451 | case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: |
2452 | case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: |
2453 | case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: |
2454 | case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: |
2455 | case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: |
2456 | case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: |
2457 | case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: |
2458 | case HWTSTAMP_FILTER_PTP_V2_EVENT: |
2459 | case HWTSTAMP_FILTER_PTP_V2_SYNC: |
2460 | case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: |
2461 | case HWTSTAMP_FILTER_NTP_ALL: |
2462 | config.rx_filter = HWTSTAMP_FILTER_ALL; |
2463 | break; |
2464 | default: |
2465 | return -ERANGE; |
2466 | } |
2467 | |
2468 | if (mlx4_en_reset_config(dev, ts_config: config, new_features: dev->features)) { |
2469 | config.tx_type = HWTSTAMP_TX_OFF; |
2470 | config.rx_filter = HWTSTAMP_FILTER_NONE; |
2471 | } |
2472 | |
2473 | return copy_to_user(to: ifr->ifr_data, from: &config, |
2474 | n: sizeof(config)) ? -EFAULT : 0; |
2475 | } |
2476 | |
2477 | static int mlx4_en_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) |
2478 | { |
2479 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2480 | |
2481 | return copy_to_user(to: ifr->ifr_data, from: &priv->hwtstamp_config, |
2482 | n: sizeof(priv->hwtstamp_config)) ? -EFAULT : 0; |
2483 | } |
2484 | |
2485 | static int mlx4_en_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) |
2486 | { |
2487 | switch (cmd) { |
2488 | case SIOCSHWTSTAMP: |
2489 | return mlx4_en_hwtstamp_set(dev, ifr); |
2490 | case SIOCGHWTSTAMP: |
2491 | return mlx4_en_hwtstamp_get(dev, ifr); |
2492 | default: |
2493 | return -EOPNOTSUPP; |
2494 | } |
2495 | } |
2496 | |
2497 | static netdev_features_t mlx4_en_fix_features(struct net_device *netdev, |
2498 | netdev_features_t features) |
2499 | { |
2500 | struct mlx4_en_priv *en_priv = netdev_priv(dev: netdev); |
2501 | struct mlx4_en_dev *mdev = en_priv->mdev; |
2502 | |
2503 | /* Since there is no support for separate RX C-TAG/S-TAG vlan accel |
2504 | * enable/disable make sure S-TAG flag is always in same state as |
2505 | * C-TAG. |
2506 | */ |
2507 | if (features & NETIF_F_HW_VLAN_CTAG_RX && |
2508 | !(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) |
2509 | features |= NETIF_F_HW_VLAN_STAG_RX; |
2510 | else |
2511 | features &= ~NETIF_F_HW_VLAN_STAG_RX; |
2512 | |
2513 | return features; |
2514 | } |
2515 | |
2516 | static int mlx4_en_set_features(struct net_device *netdev, |
2517 | netdev_features_t features) |
2518 | { |
2519 | struct mlx4_en_priv *priv = netdev_priv(dev: netdev); |
2520 | bool reset = false; |
2521 | int ret = 0; |
2522 | |
2523 | if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_RXFCS)) { |
2524 | en_info(priv, "Turn %s RX-FCS\n" , |
2525 | (features & NETIF_F_RXFCS) ? "ON" : "OFF" ); |
2526 | reset = true; |
2527 | } |
2528 | |
2529 | if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_RXALL)) { |
2530 | u8 ignore_fcs_value = (features & NETIF_F_RXALL) ? 1 : 0; |
2531 | |
2532 | en_info(priv, "Turn %s RX-ALL\n" , |
2533 | ignore_fcs_value ? "ON" : "OFF" ); |
2534 | ret = mlx4_SET_PORT_fcs_check(dev: priv->mdev->dev, |
2535 | port: priv->port, ignore_fcs_value); |
2536 | if (ret) |
2537 | return ret; |
2538 | } |
2539 | |
2540 | if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_CTAG_RX)) { |
2541 | en_info(priv, "Turn %s RX vlan strip offload\n" , |
2542 | (features & NETIF_F_HW_VLAN_CTAG_RX) ? "ON" : "OFF" ); |
2543 | reset = true; |
2544 | } |
2545 | |
2546 | if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_CTAG_TX)) |
2547 | en_info(priv, "Turn %s TX vlan strip offload\n" , |
2548 | (features & NETIF_F_HW_VLAN_CTAG_TX) ? "ON" : "OFF" ); |
2549 | |
2550 | if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_STAG_TX)) |
2551 | en_info(priv, "Turn %s TX S-VLAN strip offload\n" , |
2552 | (features & NETIF_F_HW_VLAN_STAG_TX) ? "ON" : "OFF" ); |
2553 | |
2554 | if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_LOOPBACK)) { |
2555 | en_info(priv, "Turn %s loopback\n" , |
2556 | (features & NETIF_F_LOOPBACK) ? "ON" : "OFF" ); |
2557 | mlx4_en_update_loopback_state(dev: netdev, features); |
2558 | } |
2559 | |
2560 | if (reset) { |
2561 | ret = mlx4_en_reset_config(dev: netdev, ts_config: priv->hwtstamp_config, |
2562 | new_features: features); |
2563 | if (ret) |
2564 | return ret; |
2565 | } |
2566 | |
2567 | return 0; |
2568 | } |
2569 | |
2570 | static int mlx4_en_set_vf_mac(struct net_device *dev, int queue, u8 *mac) |
2571 | { |
2572 | struct mlx4_en_priv *en_priv = netdev_priv(dev); |
2573 | struct mlx4_en_dev *mdev = en_priv->mdev; |
2574 | |
2575 | return mlx4_set_vf_mac(dev: mdev->dev, port: en_priv->port, vf: queue, mac); |
2576 | } |
2577 | |
2578 | static int mlx4_en_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, |
2579 | __be16 vlan_proto) |
2580 | { |
2581 | struct mlx4_en_priv *en_priv = netdev_priv(dev); |
2582 | struct mlx4_en_dev *mdev = en_priv->mdev; |
2583 | |
2584 | return mlx4_set_vf_vlan(dev: mdev->dev, port: en_priv->port, vf, vlan, qos, |
2585 | proto: vlan_proto); |
2586 | } |
2587 | |
2588 | static int mlx4_en_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, |
2589 | int max_tx_rate) |
2590 | { |
2591 | struct mlx4_en_priv *en_priv = netdev_priv(dev); |
2592 | struct mlx4_en_dev *mdev = en_priv->mdev; |
2593 | |
2594 | return mlx4_set_vf_rate(dev: mdev->dev, port: en_priv->port, vf, min_tx_rate, |
2595 | max_tx_rate); |
2596 | } |
2597 | |
2598 | static int mlx4_en_set_vf_spoofchk(struct net_device *dev, int vf, bool setting) |
2599 | { |
2600 | struct mlx4_en_priv *en_priv = netdev_priv(dev); |
2601 | struct mlx4_en_dev *mdev = en_priv->mdev; |
2602 | |
2603 | return mlx4_set_vf_spoofchk(dev: mdev->dev, port: en_priv->port, vf, setting); |
2604 | } |
2605 | |
2606 | static int mlx4_en_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivf) |
2607 | { |
2608 | struct mlx4_en_priv *en_priv = netdev_priv(dev); |
2609 | struct mlx4_en_dev *mdev = en_priv->mdev; |
2610 | |
2611 | return mlx4_get_vf_config(dev: mdev->dev, port: en_priv->port, vf, ivf); |
2612 | } |
2613 | |
2614 | static int mlx4_en_set_vf_link_state(struct net_device *dev, int vf, int link_state) |
2615 | { |
2616 | struct mlx4_en_priv *en_priv = netdev_priv(dev); |
2617 | struct mlx4_en_dev *mdev = en_priv->mdev; |
2618 | |
2619 | return mlx4_set_vf_link_state(dev: mdev->dev, port: en_priv->port, vf, link_state); |
2620 | } |
2621 | |
2622 | static int mlx4_en_get_vf_stats(struct net_device *dev, int vf, |
2623 | struct ifla_vf_stats *vf_stats) |
2624 | { |
2625 | struct mlx4_en_priv *en_priv = netdev_priv(dev); |
2626 | struct mlx4_en_dev *mdev = en_priv->mdev; |
2627 | |
2628 | return mlx4_get_vf_stats(dev: mdev->dev, port: en_priv->port, vf_idx: vf, vf_stats); |
2629 | } |
2630 | |
2631 | #define PORT_ID_BYTE_LEN 8 |
2632 | static int mlx4_en_get_phys_port_id(struct net_device *dev, |
2633 | struct netdev_phys_item_id *ppid) |
2634 | { |
2635 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2636 | struct mlx4_dev *mdev = priv->mdev->dev; |
2637 | int i; |
2638 | u64 phys_port_id = mdev->caps.phys_port_id[priv->port]; |
2639 | |
2640 | if (!phys_port_id) |
2641 | return -EOPNOTSUPP; |
2642 | |
2643 | ppid->id_len = sizeof(phys_port_id); |
2644 | for (i = PORT_ID_BYTE_LEN - 1; i >= 0; --i) { |
2645 | ppid->id[i] = phys_port_id & 0xff; |
2646 | phys_port_id >>= 8; |
2647 | } |
2648 | return 0; |
2649 | } |
2650 | |
2651 | static int mlx4_udp_tunnel_sync(struct net_device *dev, unsigned int table) |
2652 | { |
2653 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2654 | struct udp_tunnel_info ti; |
2655 | int ret; |
2656 | |
2657 | udp_tunnel_nic_get_port(dev, table, idx: 0, ti: &ti); |
2658 | priv->vxlan_port = ti.port; |
2659 | |
2660 | ret = mlx4_config_vxlan_port(dev: priv->mdev->dev, udp_port: priv->vxlan_port); |
2661 | if (ret) |
2662 | return ret; |
2663 | |
2664 | return mlx4_SET_PORT_VXLAN(dev: priv->mdev->dev, port: priv->port, |
2665 | steering: VXLAN_STEER_BY_OUTER_MAC, |
2666 | enable: !!priv->vxlan_port); |
2667 | } |
2668 | |
2669 | static const struct udp_tunnel_nic_info mlx4_udp_tunnels = { |
2670 | .sync_table = mlx4_udp_tunnel_sync, |
2671 | .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | |
2672 | UDP_TUNNEL_NIC_INFO_IPV4_ONLY, |
2673 | .tables = { |
2674 | { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, |
2675 | }, |
2676 | }; |
2677 | |
2678 | static netdev_features_t mlx4_en_features_check(struct sk_buff *skb, |
2679 | struct net_device *dev, |
2680 | netdev_features_t features) |
2681 | { |
2682 | features = vlan_features_check(skb, features); |
2683 | features = vxlan_features_check(skb, features); |
2684 | |
2685 | /* The ConnectX-3 doesn't support outer IPv6 checksums but it does |
2686 | * support inner IPv6 checksums and segmentation so we need to |
2687 | * strip that feature if this is an IPv6 encapsulated frame. |
2688 | */ |
2689 | if (skb->encapsulation && |
2690 | (skb->ip_summed == CHECKSUM_PARTIAL)) { |
2691 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2692 | |
2693 | if (!priv->vxlan_port || |
2694 | (ip_hdr(skb)->version != 4) || |
2695 | (udp_hdr(skb)->dest != priv->vxlan_port)) |
2696 | features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); |
2697 | } |
2698 | |
2699 | return features; |
2700 | } |
2701 | |
2702 | static int mlx4_en_set_tx_maxrate(struct net_device *dev, int queue_index, u32 maxrate) |
2703 | { |
2704 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2705 | struct mlx4_en_tx_ring *tx_ring = priv->tx_ring[TX][queue_index]; |
2706 | struct mlx4_update_qp_params params; |
2707 | int err; |
2708 | |
2709 | if (!(priv->mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_QP_RATE_LIMIT)) |
2710 | return -EOPNOTSUPP; |
2711 | |
2712 | /* rate provided to us in Mbs, check if it fits into 12 bits, if not use Gbs */ |
2713 | if (maxrate >> 12) { |
2714 | params.rate_unit = MLX4_QP_RATE_LIMIT_GBS; |
2715 | params.rate_val = maxrate / 1000; |
2716 | } else if (maxrate) { |
2717 | params.rate_unit = MLX4_QP_RATE_LIMIT_MBS; |
2718 | params.rate_val = maxrate; |
2719 | } else { /* zero serves to revoke the QP rate-limitation */ |
2720 | params.rate_unit = 0; |
2721 | params.rate_val = 0; |
2722 | } |
2723 | |
2724 | err = mlx4_update_qp(dev: priv->mdev->dev, qpn: tx_ring->qpn, attr: MLX4_UPDATE_QP_RATE_LIMIT, |
2725 | params: ¶ms); |
2726 | return err; |
2727 | } |
2728 | |
2729 | static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog) |
2730 | { |
2731 | struct mlx4_en_priv *priv = netdev_priv(dev); |
2732 | struct mlx4_en_dev *mdev = priv->mdev; |
2733 | struct mlx4_en_port_profile new_prof; |
2734 | struct bpf_prog *old_prog; |
2735 | struct mlx4_en_priv *tmp; |
2736 | int tx_changed = 0; |
2737 | int xdp_ring_num; |
2738 | int port_up = 0; |
2739 | int err; |
2740 | int i; |
2741 | |
2742 | xdp_ring_num = prog ? priv->rx_ring_num : 0; |
2743 | |
2744 | /* No need to reconfigure buffers when simply swapping the |
2745 | * program for a new one. |
2746 | */ |
2747 | if (priv->tx_ring_num[TX_XDP] == xdp_ring_num) { |
2748 | if (prog) |
2749 | bpf_prog_add(prog, i: priv->rx_ring_num - 1); |
2750 | |
2751 | mutex_lock(&mdev->state_lock); |
2752 | for (i = 0; i < priv->rx_ring_num; i++) { |
2753 | old_prog = rcu_dereference_protected( |
2754 | priv->rx_ring[i]->xdp_prog, |
2755 | lockdep_is_held(&mdev->state_lock)); |
2756 | rcu_assign_pointer(priv->rx_ring[i]->xdp_prog, prog); |
2757 | if (old_prog) |
2758 | bpf_prog_put(prog: old_prog); |
2759 | } |
2760 | mutex_unlock(lock: &mdev->state_lock); |
2761 | return 0; |
2762 | } |
2763 | |
2764 | if (!mlx4_en_check_xdp_mtu(dev, mtu: dev->mtu)) |
2765 | return -EOPNOTSUPP; |
2766 | |
2767 | tmp = kzalloc(size: sizeof(*tmp), GFP_KERNEL); |
2768 | if (!tmp) |
2769 | return -ENOMEM; |
2770 | |
2771 | if (prog) |
2772 | bpf_prog_add(prog, i: priv->rx_ring_num - 1); |
2773 | |
2774 | mutex_lock(&mdev->state_lock); |
2775 | memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile)); |
2776 | new_prof.tx_ring_num[TX_XDP] = xdp_ring_num; |
2777 | |
2778 | if (priv->tx_ring_num[TX] + xdp_ring_num > MAX_TX_RINGS) { |
2779 | tx_changed = 1; |
2780 | new_prof.tx_ring_num[TX] = |
2781 | MAX_TX_RINGS - ALIGN(xdp_ring_num, priv->prof->num_up); |
2782 | en_warn(priv, "Reducing the number of TX rings, to not exceed the max total rings number.\n" ); |
2783 | } |
2784 | |
2785 | err = mlx4_en_try_alloc_resources(priv, tmp, prof: &new_prof, carry_xdp_prog: false); |
2786 | if (err) { |
2787 | if (prog) |
2788 | bpf_prog_sub(prog, i: priv->rx_ring_num - 1); |
2789 | goto unlock_out; |
2790 | } |
2791 | |
2792 | if (priv->port_up) { |
2793 | port_up = 1; |
2794 | mlx4_en_stop_port(dev, detach: 1); |
2795 | } |
2796 | |
2797 | mlx4_en_safe_replace_resources(priv, tmp); |
2798 | if (tx_changed) |
2799 | netif_set_real_num_tx_queues(dev, txq: priv->tx_ring_num[TX]); |
2800 | |
2801 | for (i = 0; i < priv->rx_ring_num; i++) { |
2802 | old_prog = rcu_dereference_protected( |
2803 | priv->rx_ring[i]->xdp_prog, |
2804 | lockdep_is_held(&mdev->state_lock)); |
2805 | rcu_assign_pointer(priv->rx_ring[i]->xdp_prog, prog); |
2806 | if (old_prog) |
2807 | bpf_prog_put(prog: old_prog); |
2808 | } |
2809 | |
2810 | if (port_up) { |
2811 | err = mlx4_en_start_port(dev); |
2812 | if (err) { |
2813 | en_err(priv, "Failed starting port %d for XDP change\n" , |
2814 | priv->port); |
2815 | if (!test_and_set_bit(nr: MLX4_EN_STATE_FLAG_RESTARTING, addr: &priv->state)) |
2816 | queue_work(wq: mdev->workqueue, work: &priv->restart_task); |
2817 | } |
2818 | } |
2819 | |
2820 | unlock_out: |
2821 | mutex_unlock(lock: &mdev->state_lock); |
2822 | kfree(objp: tmp); |
2823 | return err; |
2824 | } |
2825 | |
2826 | static int mlx4_xdp(struct net_device *dev, struct netdev_bpf *xdp) |
2827 | { |
2828 | switch (xdp->command) { |
2829 | case XDP_SETUP_PROG: |
2830 | return mlx4_xdp_set(dev, prog: xdp->prog); |
2831 | default: |
2832 | return -EINVAL; |
2833 | } |
2834 | } |
2835 | |
2836 | static const struct net_device_ops mlx4_netdev_ops = { |
2837 | .ndo_open = mlx4_en_open, |
2838 | .ndo_stop = mlx4_en_close, |
2839 | .ndo_start_xmit = mlx4_en_xmit, |
2840 | .ndo_select_queue = mlx4_en_select_queue, |
2841 | .ndo_get_stats64 = mlx4_en_get_stats64, |
2842 | .ndo_set_rx_mode = mlx4_en_set_rx_mode, |
2843 | .ndo_set_mac_address = mlx4_en_set_mac, |
2844 | .ndo_validate_addr = eth_validate_addr, |
2845 | .ndo_change_mtu = mlx4_en_change_mtu, |
2846 | .ndo_eth_ioctl = mlx4_en_ioctl, |
2847 | .ndo_tx_timeout = mlx4_en_tx_timeout, |
2848 | .ndo_vlan_rx_add_vid = mlx4_en_vlan_rx_add_vid, |
2849 | .ndo_vlan_rx_kill_vid = mlx4_en_vlan_rx_kill_vid, |
2850 | .ndo_set_features = mlx4_en_set_features, |
2851 | .ndo_fix_features = mlx4_en_fix_features, |
2852 | .ndo_setup_tc = __mlx4_en_setup_tc, |
2853 | #ifdef CONFIG_RFS_ACCEL |
2854 | .ndo_rx_flow_steer = mlx4_en_filter_rfs, |
2855 | #endif |
2856 | .ndo_get_phys_port_id = mlx4_en_get_phys_port_id, |
2857 | .ndo_features_check = mlx4_en_features_check, |
2858 | .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, |
2859 | .ndo_bpf = mlx4_xdp, |
2860 | }; |
2861 | |
2862 | static const struct net_device_ops mlx4_netdev_ops_master = { |
2863 | .ndo_open = mlx4_en_open, |
2864 | .ndo_stop = mlx4_en_close, |
2865 | .ndo_start_xmit = mlx4_en_xmit, |
2866 | .ndo_select_queue = mlx4_en_select_queue, |
2867 | .ndo_get_stats64 = mlx4_en_get_stats64, |
2868 | .ndo_set_rx_mode = mlx4_en_set_rx_mode, |
2869 | .ndo_set_mac_address = mlx4_en_set_mac, |
2870 | .ndo_validate_addr = eth_validate_addr, |
2871 | .ndo_change_mtu = mlx4_en_change_mtu, |
2872 | .ndo_tx_timeout = mlx4_en_tx_timeout, |
2873 | .ndo_vlan_rx_add_vid = mlx4_en_vlan_rx_add_vid, |
2874 | .ndo_vlan_rx_kill_vid = mlx4_en_vlan_rx_kill_vid, |
2875 | .ndo_set_vf_mac = mlx4_en_set_vf_mac, |
2876 | .ndo_set_vf_vlan = mlx4_en_set_vf_vlan, |
2877 | .ndo_set_vf_rate = mlx4_en_set_vf_rate, |
2878 | .ndo_set_vf_spoofchk = mlx4_en_set_vf_spoofchk, |
2879 | .ndo_set_vf_link_state = mlx4_en_set_vf_link_state, |
2880 | .ndo_get_vf_stats = mlx4_en_get_vf_stats, |
2881 | .ndo_get_vf_config = mlx4_en_get_vf_config, |
2882 | .ndo_set_features = mlx4_en_set_features, |
2883 | .ndo_fix_features = mlx4_en_fix_features, |
2884 | .ndo_setup_tc = __mlx4_en_setup_tc, |
2885 | #ifdef CONFIG_RFS_ACCEL |
2886 | .ndo_rx_flow_steer = mlx4_en_filter_rfs, |
2887 | #endif |
2888 | .ndo_get_phys_port_id = mlx4_en_get_phys_port_id, |
2889 | .ndo_features_check = mlx4_en_features_check, |
2890 | .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, |
2891 | .ndo_bpf = mlx4_xdp, |
2892 | }; |
2893 | |
2894 | static const struct xdp_metadata_ops mlx4_xdp_metadata_ops = { |
2895 | .xmo_rx_timestamp = mlx4_en_xdp_rx_timestamp, |
2896 | .xmo_rx_hash = mlx4_en_xdp_rx_hash, |
2897 | }; |
2898 | |
2899 | int mlx4_en_netdev_event(struct notifier_block *this, |
2900 | unsigned long event, void *ptr) |
2901 | { |
2902 | struct net_device *ndev = netdev_notifier_info_to_dev(info: ptr); |
2903 | u8 port = 0; |
2904 | struct mlx4_en_dev *mdev; |
2905 | struct mlx4_dev *dev; |
2906 | int i, num_eth_ports = 0; |
2907 | bool do_bond = true; |
2908 | u8 v2p_port1 = 0; |
2909 | u8 v2p_port2 = 0; |
2910 | |
2911 | if (!net_eq(net1: dev_net(dev: ndev), net2: &init_net)) |
2912 | return NOTIFY_DONE; |
2913 | |
2914 | mdev = container_of(this, struct mlx4_en_dev, netdev_nb); |
2915 | dev = mdev->dev; |
2916 | |
2917 | /* Go into this mode only when two network devices set on two ports |
2918 | * of the same mlx4 device are slaves of the same bonding master |
2919 | */ |
2920 | mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { |
2921 | ++num_eth_ports; |
2922 | if (!port && (mdev->pndev[i] == ndev)) |
2923 | port = i; |
2924 | mdev->upper[i] = mdev->pndev[i] ? |
2925 | netdev_master_upper_dev_get(dev: mdev->pndev[i]) : NULL; |
2926 | /* condition not met: network device is a slave */ |
2927 | if (!mdev->upper[i]) |
2928 | do_bond = false; |
2929 | if (num_eth_ports < 2) |
2930 | continue; |
2931 | /* condition not met: same master */ |
2932 | if (mdev->upper[i] != mdev->upper[i-1]) |
2933 | do_bond = false; |
2934 | } |
2935 | /* condition not met: 2 salves */ |
2936 | do_bond = (num_eth_ports == 2) ? do_bond : false; |
2937 | |
2938 | /* handle only events that come with enough info */ |
2939 | if ((do_bond && (event != NETDEV_BONDING_INFO)) || !port) |
2940 | return NOTIFY_DONE; |
2941 | |
2942 | if (do_bond) { |
2943 | struct netdev_notifier_bonding_info *notifier_info = ptr; |
2944 | struct netdev_bonding_info *bonding_info = |
2945 | ¬ifier_info->bonding_info; |
2946 | |
2947 | /* required mode 1, 2 or 4 */ |
2948 | if ((bonding_info->master.bond_mode != BOND_MODE_ACTIVEBACKUP) && |
2949 | (bonding_info->master.bond_mode != BOND_MODE_XOR) && |
2950 | (bonding_info->master.bond_mode != BOND_MODE_8023AD)) |
2951 | do_bond = false; |
2952 | |
2953 | /* require exactly 2 slaves */ |
2954 | if (bonding_info->master.num_slaves != 2) |
2955 | do_bond = false; |
2956 | |
2957 | /* calc v2p */ |
2958 | if (do_bond) { |
2959 | if (bonding_info->master.bond_mode == |
2960 | BOND_MODE_ACTIVEBACKUP) { |
2961 | /* in active-backup mode virtual ports are |
2962 | * mapped to the physical port of the active |
2963 | * slave */ |
2964 | if (bonding_info->slave.state == |
2965 | BOND_STATE_BACKUP) { |
2966 | if (port == 1) { |
2967 | v2p_port1 = 2; |
2968 | v2p_port2 = 2; |
2969 | } else { |
2970 | v2p_port1 = 1; |
2971 | v2p_port2 = 1; |
2972 | } |
2973 | } else { /* BOND_STATE_ACTIVE */ |
2974 | if (port == 1) { |
2975 | v2p_port1 = 1; |
2976 | v2p_port2 = 1; |
2977 | } else { |
2978 | v2p_port1 = 2; |
2979 | v2p_port2 = 2; |
2980 | } |
2981 | } |
2982 | } else { /* Active-Active */ |
2983 | /* in active-active mode a virtual port is |
2984 | * mapped to the native physical port if and only |
2985 | * if the physical port is up */ |
2986 | __s8 link = bonding_info->slave.link; |
2987 | |
2988 | if (port == 1) |
2989 | v2p_port2 = 2; |
2990 | else |
2991 | v2p_port1 = 1; |
2992 | if ((link == BOND_LINK_UP) || |
2993 | (link == BOND_LINK_FAIL)) { |
2994 | if (port == 1) |
2995 | v2p_port1 = 1; |
2996 | else |
2997 | v2p_port2 = 2; |
2998 | } else { /* BOND_LINK_DOWN || BOND_LINK_BACK */ |
2999 | if (port == 1) |
3000 | v2p_port1 = 2; |
3001 | else |
3002 | v2p_port2 = 1; |
3003 | } |
3004 | } |
3005 | } |
3006 | } |
3007 | |
3008 | mlx4_queue_bond_work(dev, is_bonded: do_bond, v2p_p1: v2p_port1, v2p_p2: v2p_port2); |
3009 | |
3010 | return NOTIFY_DONE; |
3011 | } |
3012 | |
3013 | void mlx4_en_update_pfc_stats_bitmap(struct mlx4_dev *dev, |
3014 | struct mlx4_en_stats_bitmap *stats_bitmap, |
3015 | u8 rx_ppp, u8 rx_pause, |
3016 | u8 tx_ppp, u8 tx_pause) |
3017 | { |
3018 | int last_i = NUM_MAIN_STATS + NUM_PORT_STATS + NUM_PF_STATS; |
3019 | |
3020 | if (!mlx4_is_slave(dev) && |
3021 | (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN)) { |
3022 | mutex_lock(&stats_bitmap->mutex); |
3023 | bitmap_clear(map: stats_bitmap->bitmap, start: last_i, NUM_FLOW_STATS); |
3024 | |
3025 | if (rx_ppp) |
3026 | bitmap_set(map: stats_bitmap->bitmap, start: last_i, |
3027 | NUM_FLOW_PRIORITY_STATS_RX); |
3028 | last_i += NUM_FLOW_PRIORITY_STATS_RX; |
3029 | |
3030 | if (rx_pause && !(rx_ppp)) |
3031 | bitmap_set(map: stats_bitmap->bitmap, start: last_i, |
3032 | NUM_FLOW_STATS_RX); |
3033 | last_i += NUM_FLOW_STATS_RX; |
3034 | |
3035 | if (tx_ppp) |
3036 | bitmap_set(map: stats_bitmap->bitmap, start: last_i, |
3037 | NUM_FLOW_PRIORITY_STATS_TX); |
3038 | last_i += NUM_FLOW_PRIORITY_STATS_TX; |
3039 | |
3040 | if (tx_pause && !(tx_ppp)) |
3041 | bitmap_set(map: stats_bitmap->bitmap, start: last_i, |
3042 | NUM_FLOW_STATS_TX); |
3043 | last_i += NUM_FLOW_STATS_TX; |
3044 | |
3045 | mutex_unlock(lock: &stats_bitmap->mutex); |
3046 | } |
3047 | } |
3048 | |
3049 | void mlx4_en_set_stats_bitmap(struct mlx4_dev *dev, |
3050 | struct mlx4_en_stats_bitmap *stats_bitmap, |
3051 | u8 rx_ppp, u8 rx_pause, |
3052 | u8 tx_ppp, u8 tx_pause) |
3053 | { |
3054 | int last_i = 0; |
3055 | |
3056 | mutex_init(&stats_bitmap->mutex); |
3057 | bitmap_zero(dst: stats_bitmap->bitmap, NUM_ALL_STATS); |
3058 | |
3059 | if (mlx4_is_slave(dev)) { |
3060 | bitmap_set(map: stats_bitmap->bitmap, start: last_i + |
3061 | MLX4_FIND_NETDEV_STAT(rx_packets), nbits: 1); |
3062 | bitmap_set(map: stats_bitmap->bitmap, start: last_i + |
3063 | MLX4_FIND_NETDEV_STAT(tx_packets), nbits: 1); |
3064 | bitmap_set(map: stats_bitmap->bitmap, start: last_i + |
3065 | MLX4_FIND_NETDEV_STAT(rx_bytes), nbits: 1); |
3066 | bitmap_set(map: stats_bitmap->bitmap, start: last_i + |
3067 | MLX4_FIND_NETDEV_STAT(tx_bytes), nbits: 1); |
3068 | bitmap_set(map: stats_bitmap->bitmap, start: last_i + |
3069 | MLX4_FIND_NETDEV_STAT(rx_dropped), nbits: 1); |
3070 | bitmap_set(map: stats_bitmap->bitmap, start: last_i + |
3071 | MLX4_FIND_NETDEV_STAT(tx_dropped), nbits: 1); |
3072 | } else { |
3073 | bitmap_set(map: stats_bitmap->bitmap, start: last_i, NUM_MAIN_STATS); |
3074 | } |
3075 | last_i += NUM_MAIN_STATS; |
3076 | |
3077 | bitmap_set(map: stats_bitmap->bitmap, start: last_i, NUM_PORT_STATS); |
3078 | last_i += NUM_PORT_STATS; |
3079 | |
3080 | if (mlx4_is_master(dev)) |
3081 | bitmap_set(map: stats_bitmap->bitmap, start: last_i, |
3082 | NUM_PF_STATS); |
3083 | last_i += NUM_PF_STATS; |
3084 | |
3085 | mlx4_en_update_pfc_stats_bitmap(dev, stats_bitmap, |
3086 | rx_ppp, rx_pause, |
3087 | tx_ppp, tx_pause); |
3088 | last_i += NUM_FLOW_STATS; |
3089 | |
3090 | if (!mlx4_is_slave(dev)) |
3091 | bitmap_set(map: stats_bitmap->bitmap, start: last_i, NUM_PKT_STATS); |
3092 | last_i += NUM_PKT_STATS; |
3093 | |
3094 | bitmap_set(map: stats_bitmap->bitmap, start: last_i, NUM_XDP_STATS); |
3095 | last_i += NUM_XDP_STATS; |
3096 | |
3097 | if (!mlx4_is_slave(dev)) |
3098 | bitmap_set(map: stats_bitmap->bitmap, start: last_i, NUM_PHY_STATS); |
3099 | last_i += NUM_PHY_STATS; |
3100 | } |
3101 | |
3102 | int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, |
3103 | struct mlx4_en_port_profile *prof) |
3104 | { |
3105 | struct net_device *dev; |
3106 | struct mlx4_en_priv *priv; |
3107 | int i, t; |
3108 | int err; |
3109 | |
3110 | dev = alloc_etherdev_mqs(sizeof_priv: sizeof(struct mlx4_en_priv), |
3111 | MAX_TX_RINGS, MAX_RX_RINGS); |
3112 | if (dev == NULL) |
3113 | return -ENOMEM; |
3114 | |
3115 | netif_set_real_num_tx_queues(dev, txq: prof->tx_ring_num[TX]); |
3116 | netif_set_real_num_rx_queues(dev, rxq: prof->rx_ring_num); |
3117 | |
3118 | SET_NETDEV_DEV(dev, &mdev->dev->persist->pdev->dev); |
3119 | dev->dev_port = port - 1; |
3120 | |
3121 | /* |
3122 | * Initialize driver private data |
3123 | */ |
3124 | |
3125 | priv = netdev_priv(dev); |
3126 | memset(priv, 0, sizeof(struct mlx4_en_priv)); |
3127 | priv->counter_index = MLX4_SINK_COUNTER_INDEX(mdev->dev); |
3128 | spin_lock_init(&priv->stats_lock); |
3129 | INIT_WORK(&priv->rx_mode_task, mlx4_en_do_set_rx_mode); |
3130 | INIT_WORK(&priv->restart_task, mlx4_en_restart); |
3131 | INIT_WORK(&priv->linkstate_task, mlx4_en_linkstate_work); |
3132 | INIT_DELAYED_WORK(&priv->stats_task, mlx4_en_do_get_stats); |
3133 | INIT_DELAYED_WORK(&priv->service_task, mlx4_en_service_task); |
3134 | #ifdef CONFIG_RFS_ACCEL |
3135 | INIT_LIST_HEAD(list: &priv->filters); |
3136 | spin_lock_init(&priv->filters_lock); |
3137 | #endif |
3138 | |
3139 | priv->dev = dev; |
3140 | priv->mdev = mdev; |
3141 | priv->ddev = &mdev->pdev->dev; |
3142 | priv->prof = prof; |
3143 | priv->port = port; |
3144 | priv->port_up = false; |
3145 | priv->flags = prof->flags; |
3146 | priv->pflags = MLX4_EN_PRIV_FLAGS_BLUEFLAME; |
3147 | priv->ctrl_flags = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | |
3148 | MLX4_WQE_CTRL_SOLICITED); |
3149 | priv->num_tx_rings_p_up = mdev->profile.max_num_tx_rings_p_up; |
3150 | priv->tx_work_limit = MLX4_EN_DEFAULT_TX_WORK; |
3151 | netdev_rss_key_fill(buffer: priv->rss_key, len: sizeof(priv->rss_key)); |
3152 | |
3153 | for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { |
3154 | priv->tx_ring_num[t] = prof->tx_ring_num[t]; |
3155 | if (!priv->tx_ring_num[t]) |
3156 | continue; |
3157 | |
3158 | priv->tx_ring[t] = kcalloc(MAX_TX_RINGS, |
3159 | size: sizeof(struct mlx4_en_tx_ring *), |
3160 | GFP_KERNEL); |
3161 | if (!priv->tx_ring[t]) { |
3162 | err = -ENOMEM; |
3163 | goto out; |
3164 | } |
3165 | priv->tx_cq[t] = kcalloc(MAX_TX_RINGS, |
3166 | size: sizeof(struct mlx4_en_cq *), |
3167 | GFP_KERNEL); |
3168 | if (!priv->tx_cq[t]) { |
3169 | err = -ENOMEM; |
3170 | goto out; |
3171 | } |
3172 | } |
3173 | priv->rx_ring_num = prof->rx_ring_num; |
3174 | priv->cqe_factor = (mdev->dev->caps.cqe_size == 64) ? 1 : 0; |
3175 | priv->cqe_size = mdev->dev->caps.cqe_size; |
3176 | priv->mac_index = -1; |
3177 | priv->msg_enable = MLX4_EN_MSG_LEVEL; |
3178 | #ifdef CONFIG_MLX4_EN_DCB |
3179 | if (!mlx4_is_slave(dev: priv->mdev->dev)) { |
3180 | u8 prio; |
3181 | |
3182 | for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; ++prio) { |
3183 | priv->ets.prio_tc[prio] = prio; |
3184 | priv->ets.tc_tsa[prio] = IEEE_8021QAZ_TSA_VENDOR; |
3185 | } |
3186 | |
3187 | priv->dcbx_cap = DCB_CAP_DCBX_VER_CEE | DCB_CAP_DCBX_HOST | |
3188 | DCB_CAP_DCBX_VER_IEEE; |
3189 | priv->flags |= MLX4_EN_DCB_ENABLED; |
3190 | priv->cee_config.pfc_state = false; |
3191 | |
3192 | for (i = 0; i < MLX4_EN_NUM_UP_HIGH; i++) |
3193 | priv->cee_config.dcb_pfc[i] = pfc_disabled; |
3194 | |
3195 | if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) { |
3196 | dev->dcbnl_ops = &mlx4_en_dcbnl_ops; |
3197 | } else { |
3198 | en_info(priv, "enabling only PFC DCB ops\n" ); |
3199 | dev->dcbnl_ops = &mlx4_en_dcbnl_pfc_ops; |
3200 | } |
3201 | } |
3202 | #endif |
3203 | |
3204 | for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) |
3205 | INIT_HLIST_HEAD(&priv->mac_hash[i]); |
3206 | |
3207 | /* Query for default mac and max mtu */ |
3208 | priv->max_mtu = mdev->dev->caps.eth_mtu_cap[priv->port]; |
3209 | |
3210 | if (mdev->dev->caps.rx_checksum_flags_port[priv->port] & |
3211 | MLX4_RX_CSUM_MODE_VAL_NON_TCP_UDP) |
3212 | priv->flags |= MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP; |
3213 | |
3214 | /* Set default MAC */ |
3215 | dev->addr_len = ETH_ALEN; |
3216 | mlx4_en_u64_to_mac(dev, src_mac: mdev->dev->caps.def_mac[priv->port]); |
3217 | if (!is_valid_ether_addr(addr: dev->dev_addr)) { |
3218 | en_err(priv, "Port: %d, invalid mac burned: %pM, quitting\n" , |
3219 | priv->port, dev->dev_addr); |
3220 | err = -EINVAL; |
3221 | goto out; |
3222 | } else if (mlx4_is_slave(dev: priv->mdev->dev) && |
3223 | (priv->mdev->dev->port_random_macs & 1 << priv->port)) { |
3224 | /* Random MAC was assigned in mlx4_slave_cap |
3225 | * in mlx4_core module |
3226 | */ |
3227 | dev->addr_assign_type |= NET_ADDR_RANDOM; |
3228 | en_warn(priv, "Assigned random MAC address %pM\n" , dev->dev_addr); |
3229 | } |
3230 | |
3231 | memcpy(priv->current_mac, dev->dev_addr, sizeof(priv->current_mac)); |
3232 | |
3233 | priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + |
3234 | DS_SIZE * MLX4_EN_MAX_RX_FRAGS); |
3235 | err = mlx4_en_alloc_resources(priv); |
3236 | if (err) |
3237 | goto out; |
3238 | |
3239 | /* Initialize time stamping config */ |
3240 | priv->hwtstamp_config.flags = 0; |
3241 | priv->hwtstamp_config.tx_type = HWTSTAMP_TX_OFF; |
3242 | priv->hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; |
3243 | |
3244 | /* Allocate page for receive rings */ |
3245 | err = mlx4_alloc_hwq_res(dev: mdev->dev, wqres: &priv->res, |
3246 | MLX4_EN_PAGE_SIZE); |
3247 | if (err) { |
3248 | en_err(priv, "Failed to allocate page for rx qps\n" ); |
3249 | goto out; |
3250 | } |
3251 | priv->allocated = 1; |
3252 | |
3253 | /* |
3254 | * Initialize netdev entry points |
3255 | */ |
3256 | if (mlx4_is_master(dev: priv->mdev->dev)) |
3257 | dev->netdev_ops = &mlx4_netdev_ops_master; |
3258 | else |
3259 | dev->netdev_ops = &mlx4_netdev_ops; |
3260 | dev->xdp_metadata_ops = &mlx4_xdp_metadata_ops; |
3261 | dev->watchdog_timeo = MLX4_EN_WATCHDOG_TIMEOUT; |
3262 | netif_set_real_num_tx_queues(dev, txq: priv->tx_ring_num[TX]); |
3263 | netif_set_real_num_rx_queues(dev, rxq: priv->rx_ring_num); |
3264 | |
3265 | dev->ethtool_ops = &mlx4_en_ethtool_ops; |
3266 | |
3267 | /* |
3268 | * Set driver features |
3269 | */ |
3270 | dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; |
3271 | if (mdev->LSO_support) |
3272 | dev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; |
3273 | |
3274 | if (mdev->dev->caps.tunnel_offload_mode == |
3275 | MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) { |
3276 | dev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | |
3277 | NETIF_F_GSO_UDP_TUNNEL_CSUM | |
3278 | NETIF_F_GSO_PARTIAL; |
3279 | dev->features |= NETIF_F_GSO_UDP_TUNNEL | |
3280 | NETIF_F_GSO_UDP_TUNNEL_CSUM | |
3281 | NETIF_F_GSO_PARTIAL; |
3282 | dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; |
3283 | dev->hw_enc_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | |
3284 | NETIF_F_RXCSUM | |
3285 | NETIF_F_TSO | NETIF_F_TSO6 | |
3286 | NETIF_F_GSO_UDP_TUNNEL | |
3287 | NETIF_F_GSO_UDP_TUNNEL_CSUM | |
3288 | NETIF_F_GSO_PARTIAL; |
3289 | |
3290 | dev->udp_tunnel_nic_info = &mlx4_udp_tunnels; |
3291 | } |
3292 | |
3293 | dev->vlan_features = dev->hw_features; |
3294 | |
3295 | dev->hw_features |= NETIF_F_RXCSUM | NETIF_F_RXHASH; |
3296 | dev->features = dev->hw_features | NETIF_F_HIGHDMA | |
3297 | NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | |
3298 | NETIF_F_HW_VLAN_CTAG_FILTER; |
3299 | dev->hw_features |= NETIF_F_LOOPBACK | |
3300 | NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; |
3301 | |
3302 | if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) { |
3303 | dev->features |= NETIF_F_HW_VLAN_STAG_RX | |
3304 | NETIF_F_HW_VLAN_STAG_FILTER; |
3305 | dev->hw_features |= NETIF_F_HW_VLAN_STAG_RX; |
3306 | } |
3307 | |
3308 | if (mlx4_is_slave(dev: mdev->dev)) { |
3309 | bool vlan_offload_disabled; |
3310 | int phv; |
3311 | |
3312 | err = get_phv_bit(dev: mdev->dev, port, phv: &phv); |
3313 | if (!err && phv) { |
3314 | dev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; |
3315 | priv->pflags |= MLX4_EN_PRIV_FLAGS_PHV; |
3316 | } |
3317 | err = mlx4_get_is_vlan_offload_disabled(dev: mdev->dev, port, |
3318 | vlan_offload_disabled: &vlan_offload_disabled); |
3319 | if (!err && vlan_offload_disabled) { |
3320 | dev->hw_features &= ~(NETIF_F_HW_VLAN_CTAG_TX | |
3321 | NETIF_F_HW_VLAN_CTAG_RX | |
3322 | NETIF_F_HW_VLAN_STAG_TX | |
3323 | NETIF_F_HW_VLAN_STAG_RX); |
3324 | dev->features &= ~(NETIF_F_HW_VLAN_CTAG_TX | |
3325 | NETIF_F_HW_VLAN_CTAG_RX | |
3326 | NETIF_F_HW_VLAN_STAG_TX | |
3327 | NETIF_F_HW_VLAN_STAG_RX); |
3328 | } |
3329 | } else { |
3330 | if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PHV_EN && |
3331 | !(mdev->dev->caps.flags2 & |
3332 | MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) |
3333 | dev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; |
3334 | } |
3335 | |
3336 | if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) |
3337 | dev->hw_features |= NETIF_F_RXFCS; |
3338 | |
3339 | if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_IGNORE_FCS) |
3340 | dev->hw_features |= NETIF_F_RXALL; |
3341 | |
3342 | if (mdev->dev->caps.steering_mode == |
3343 | MLX4_STEERING_MODE_DEVICE_MANAGED && |
3344 | mdev->dev->caps.dmfs_high_steer_mode != MLX4_STEERING_DMFS_A0_STATIC) |
3345 | dev->hw_features |= NETIF_F_NTUPLE; |
3346 | |
3347 | if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0) |
3348 | dev->priv_flags |= IFF_UNICAST_FLT; |
3349 | |
3350 | /* Setting a default hash function value */ |
3351 | if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP) { |
3352 | priv->rss_hash_fn = ETH_RSS_HASH_TOP; |
3353 | } else if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_XOR) { |
3354 | priv->rss_hash_fn = ETH_RSS_HASH_XOR; |
3355 | } else { |
3356 | en_warn(priv, |
3357 | "No RSS hash capabilities exposed, using Toeplitz\n" ); |
3358 | priv->rss_hash_fn = ETH_RSS_HASH_TOP; |
3359 | } |
3360 | |
3361 | dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT; |
3362 | |
3363 | /* MTU range: 68 - hw-specific max */ |
3364 | dev->min_mtu = ETH_MIN_MTU; |
3365 | dev->max_mtu = priv->max_mtu; |
3366 | |
3367 | /* supports LSOv2 packets. */ |
3368 | netif_set_tso_max_size(dev, GSO_MAX_SIZE); |
3369 | |
3370 | mdev->pndev[port] = dev; |
3371 | mdev->upper[port] = NULL; |
3372 | |
3373 | netif_carrier_off(dev); |
3374 | mlx4_en_set_default_moderation(priv); |
3375 | |
3376 | en_warn(priv, "Using %d TX rings\n" , prof->tx_ring_num[TX]); |
3377 | en_warn(priv, "Using %d RX rings\n" , prof->rx_ring_num); |
3378 | |
3379 | mlx4_en_update_loopback_state(dev: priv->dev, features: priv->dev->features); |
3380 | |
3381 | /* Configure port */ |
3382 | mlx4_en_calc_rx_buf(dev); |
3383 | err = mlx4_SET_PORT_general(dev: mdev->dev, port: priv->port, |
3384 | mtu: priv->rx_skb_size + ETH_FCS_LEN, |
3385 | pptx: prof->tx_pause, pfctx: prof->tx_ppp, |
3386 | pprx: prof->rx_pause, pfcrx: prof->rx_ppp); |
3387 | if (err) { |
3388 | en_err(priv, "Failed setting port general configurations for port %d, with error %d\n" , |
3389 | priv->port, err); |
3390 | goto out; |
3391 | } |
3392 | |
3393 | if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) { |
3394 | err = mlx4_SET_PORT_VXLAN(dev: mdev->dev, port: priv->port, steering: VXLAN_STEER_BY_OUTER_MAC, enable: 1); |
3395 | if (err) { |
3396 | en_err(priv, "Failed setting port L2 tunnel configuration, err %d\n" , |
3397 | err); |
3398 | goto out; |
3399 | } |
3400 | } |
3401 | |
3402 | /* Init port */ |
3403 | en_warn(priv, "Initializing port\n" ); |
3404 | err = mlx4_INIT_PORT(dev: mdev->dev, port: priv->port); |
3405 | if (err) { |
3406 | en_err(priv, "Failed Initializing port\n" ); |
3407 | goto out; |
3408 | } |
3409 | queue_delayed_work(wq: mdev->workqueue, dwork: &priv->stats_task, STATS_DELAY); |
3410 | |
3411 | /* Initialize time stamp mechanism */ |
3412 | if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) |
3413 | mlx4_en_init_timestamp(mdev); |
3414 | |
3415 | queue_delayed_work(wq: mdev->workqueue, dwork: &priv->service_task, |
3416 | SERVICE_TASK_DELAY); |
3417 | |
3418 | mlx4_en_set_stats_bitmap(dev: mdev->dev, stats_bitmap: &priv->stats_bitmap, |
3419 | rx_ppp: mdev->profile.prof[priv->port].rx_ppp, |
3420 | rx_pause: mdev->profile.prof[priv->port].rx_pause, |
3421 | tx_ppp: mdev->profile.prof[priv->port].tx_ppp, |
3422 | tx_pause: mdev->profile.prof[priv->port].tx_pause); |
3423 | |
3424 | SET_NETDEV_DEVLINK_PORT(dev, |
3425 | mlx4_get_devlink_port(mdev->dev, priv->port)); |
3426 | err = register_netdev(dev); |
3427 | if (err) { |
3428 | en_err(priv, "Netdev registration failed for port %d\n" , port); |
3429 | goto out; |
3430 | } |
3431 | |
3432 | priv->registered = 1; |
3433 | |
3434 | return 0; |
3435 | |
3436 | out: |
3437 | mlx4_en_destroy_netdev(dev); |
3438 | return err; |
3439 | } |
3440 | |
3441 | int mlx4_en_reset_config(struct net_device *dev, |
3442 | struct hwtstamp_config ts_config, |
3443 | netdev_features_t features) |
3444 | { |
3445 | struct mlx4_en_priv *priv = netdev_priv(dev); |
3446 | struct mlx4_en_dev *mdev = priv->mdev; |
3447 | struct mlx4_en_port_profile new_prof; |
3448 | struct mlx4_en_priv *tmp; |
3449 | int port_up = 0; |
3450 | int err = 0; |
3451 | |
3452 | if (priv->hwtstamp_config.tx_type == ts_config.tx_type && |
3453 | priv->hwtstamp_config.rx_filter == ts_config.rx_filter && |
3454 | !DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX) && |
3455 | !DEV_FEATURE_CHANGED(dev, features, NETIF_F_RXFCS)) |
3456 | return 0; /* Nothing to change */ |
3457 | |
3458 | if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX) && |
3459 | (features & NETIF_F_HW_VLAN_CTAG_RX) && |
3460 | (priv->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE)) { |
3461 | en_warn(priv, "Can't turn ON rx vlan offload while time-stamping rx filter is ON\n" ); |
3462 | return -EINVAL; |
3463 | } |
3464 | |
3465 | tmp = kzalloc(size: sizeof(*tmp), GFP_KERNEL); |
3466 | if (!tmp) |
3467 | return -ENOMEM; |
3468 | |
3469 | mutex_lock(&mdev->state_lock); |
3470 | |
3471 | memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile)); |
3472 | memcpy(&new_prof.hwtstamp_config, &ts_config, sizeof(ts_config)); |
3473 | |
3474 | err = mlx4_en_try_alloc_resources(priv, tmp, prof: &new_prof, carry_xdp_prog: true); |
3475 | if (err) |
3476 | goto out; |
3477 | |
3478 | if (priv->port_up) { |
3479 | port_up = 1; |
3480 | mlx4_en_stop_port(dev, detach: 1); |
3481 | } |
3482 | |
3483 | mlx4_en_safe_replace_resources(priv, tmp); |
3484 | |
3485 | if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX)) { |
3486 | if (features & NETIF_F_HW_VLAN_CTAG_RX) |
3487 | dev->features |= NETIF_F_HW_VLAN_CTAG_RX; |
3488 | else |
3489 | dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; |
3490 | } else if (ts_config.rx_filter == HWTSTAMP_FILTER_NONE) { |
3491 | /* RX time-stamping is OFF, update the RX vlan offload |
3492 | * to the latest wanted state |
3493 | */ |
3494 | if (dev->wanted_features & NETIF_F_HW_VLAN_CTAG_RX) |
3495 | dev->features |= NETIF_F_HW_VLAN_CTAG_RX; |
3496 | else |
3497 | dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; |
3498 | } |
3499 | |
3500 | if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_RXFCS)) { |
3501 | if (features & NETIF_F_RXFCS) |
3502 | dev->features |= NETIF_F_RXFCS; |
3503 | else |
3504 | dev->features &= ~NETIF_F_RXFCS; |
3505 | } |
3506 | |
3507 | /* RX vlan offload and RX time-stamping can't co-exist ! |
3508 | * Regardless of the caller's choice, |
3509 | * Turn Off RX vlan offload in case of time-stamping is ON |
3510 | */ |
3511 | if (ts_config.rx_filter != HWTSTAMP_FILTER_NONE) { |
3512 | if (dev->features & NETIF_F_HW_VLAN_CTAG_RX) |
3513 | en_warn(priv, "Turning off RX vlan offload since RX time-stamping is ON\n" ); |
3514 | dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; |
3515 | } |
3516 | |
3517 | if (port_up) { |
3518 | err = mlx4_en_start_port(dev); |
3519 | if (err) |
3520 | en_err(priv, "Failed starting port\n" ); |
3521 | } |
3522 | |
3523 | if (!err) |
3524 | err = mlx4_en_moderation_update(priv); |
3525 | out: |
3526 | mutex_unlock(lock: &mdev->state_lock); |
3527 | kfree(objp: tmp); |
3528 | if (!err) |
3529 | netdev_features_change(dev); |
3530 | return err; |
3531 | } |
3532 | |