1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* |
3 | * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved. |
4 | */ |
5 | |
6 | #include <linux/ethtool.h> |
7 | #include <linux/pci.h> |
8 | |
9 | #include "ena_netdev.h" |
10 | #include "ena_xdp.h" |
11 | |
12 | struct ena_stats { |
13 | char name[ETH_GSTRING_LEN]; |
14 | int stat_offset; |
15 | }; |
16 | |
17 | #define ENA_STAT_ENA_COM_ENTRY(stat) { \ |
18 | .name = #stat, \ |
19 | .stat_offset = offsetof(struct ena_com_stats_admin, stat) / sizeof(u64) \ |
20 | } |
21 | |
22 | #define ENA_STAT_ENTRY(stat, stat_type) { \ |
23 | .name = #stat, \ |
24 | .stat_offset = offsetof(struct ena_stats_##stat_type, stat) / sizeof(u64) \ |
25 | } |
26 | |
27 | #define ENA_STAT_HW_ENTRY(stat, stat_type) { \ |
28 | .name = #stat, \ |
29 | .stat_offset = offsetof(struct ena_admin_##stat_type, stat) / sizeof(u64) \ |
30 | } |
31 | |
32 | #define ENA_STAT_RX_ENTRY(stat) \ |
33 | ENA_STAT_ENTRY(stat, rx) |
34 | |
35 | #define ENA_STAT_TX_ENTRY(stat) \ |
36 | ENA_STAT_ENTRY(stat, tx) |
37 | |
38 | #define ENA_STAT_GLOBAL_ENTRY(stat) \ |
39 | ENA_STAT_ENTRY(stat, dev) |
40 | |
41 | #define ENA_STAT_ENI_ENTRY(stat) \ |
42 | ENA_STAT_HW_ENTRY(stat, eni_stats) |
43 | |
44 | static const struct ena_stats ena_stats_global_strings[] = { |
45 | ENA_STAT_GLOBAL_ENTRY(tx_timeout), |
46 | ENA_STAT_GLOBAL_ENTRY(suspend), |
47 | ENA_STAT_GLOBAL_ENTRY(resume), |
48 | ENA_STAT_GLOBAL_ENTRY(wd_expired), |
49 | ENA_STAT_GLOBAL_ENTRY(interface_up), |
50 | ENA_STAT_GLOBAL_ENTRY(interface_down), |
51 | ENA_STAT_GLOBAL_ENTRY(admin_q_pause), |
52 | }; |
53 | |
54 | static const struct ena_stats ena_stats_eni_strings[] = { |
55 | ENA_STAT_ENI_ENTRY(bw_in_allowance_exceeded), |
56 | ENA_STAT_ENI_ENTRY(bw_out_allowance_exceeded), |
57 | ENA_STAT_ENI_ENTRY(pps_allowance_exceeded), |
58 | ENA_STAT_ENI_ENTRY(conntrack_allowance_exceeded), |
59 | ENA_STAT_ENI_ENTRY(linklocal_allowance_exceeded), |
60 | }; |
61 | |
62 | static const struct ena_stats ena_stats_tx_strings[] = { |
63 | ENA_STAT_TX_ENTRY(cnt), |
64 | ENA_STAT_TX_ENTRY(bytes), |
65 | ENA_STAT_TX_ENTRY(queue_stop), |
66 | ENA_STAT_TX_ENTRY(queue_wakeup), |
67 | ENA_STAT_TX_ENTRY(dma_mapping_err), |
68 | ENA_STAT_TX_ENTRY(linearize), |
69 | ENA_STAT_TX_ENTRY(linearize_failed), |
70 | ENA_STAT_TX_ENTRY(napi_comp), |
71 | ENA_STAT_TX_ENTRY(tx_poll), |
72 | ENA_STAT_TX_ENTRY(doorbells), |
73 | ENA_STAT_TX_ENTRY(prepare_ctx_err), |
74 | ENA_STAT_TX_ENTRY(bad_req_id), |
75 | ENA_STAT_TX_ENTRY(llq_buffer_copy), |
76 | ENA_STAT_TX_ENTRY(missed_tx), |
77 | ENA_STAT_TX_ENTRY(unmask_interrupt), |
78 | }; |
79 | |
80 | static const struct ena_stats ena_stats_rx_strings[] = { |
81 | ENA_STAT_RX_ENTRY(cnt), |
82 | ENA_STAT_RX_ENTRY(bytes), |
83 | ENA_STAT_RX_ENTRY(rx_copybreak_pkt), |
84 | ENA_STAT_RX_ENTRY(csum_good), |
85 | ENA_STAT_RX_ENTRY(refil_partial), |
86 | ENA_STAT_RX_ENTRY(csum_bad), |
87 | ENA_STAT_RX_ENTRY(page_alloc_fail), |
88 | ENA_STAT_RX_ENTRY(skb_alloc_fail), |
89 | ENA_STAT_RX_ENTRY(dma_mapping_err), |
90 | ENA_STAT_RX_ENTRY(bad_desc_num), |
91 | ENA_STAT_RX_ENTRY(bad_req_id), |
92 | ENA_STAT_RX_ENTRY(empty_rx_ring), |
93 | ENA_STAT_RX_ENTRY(csum_unchecked), |
94 | ENA_STAT_RX_ENTRY(xdp_aborted), |
95 | ENA_STAT_RX_ENTRY(xdp_drop), |
96 | ENA_STAT_RX_ENTRY(xdp_pass), |
97 | ENA_STAT_RX_ENTRY(xdp_tx), |
98 | ENA_STAT_RX_ENTRY(xdp_invalid), |
99 | ENA_STAT_RX_ENTRY(xdp_redirect), |
100 | }; |
101 | |
102 | static const struct ena_stats ena_stats_ena_com_strings[] = { |
103 | ENA_STAT_ENA_COM_ENTRY(aborted_cmd), |
104 | ENA_STAT_ENA_COM_ENTRY(submitted_cmd), |
105 | ENA_STAT_ENA_COM_ENTRY(completed_cmd), |
106 | ENA_STAT_ENA_COM_ENTRY(out_of_space), |
107 | ENA_STAT_ENA_COM_ENTRY(no_completion), |
108 | }; |
109 | |
110 | #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) |
111 | #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) |
112 | #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) |
113 | #define ENA_STATS_ARRAY_ENA_COM ARRAY_SIZE(ena_stats_ena_com_strings) |
114 | #define ENA_STATS_ARRAY_ENI(adapter) ARRAY_SIZE(ena_stats_eni_strings) |
115 | |
116 | static void ena_safe_update_stat(u64 *src, u64 *dst, |
117 | struct u64_stats_sync *syncp) |
118 | { |
119 | unsigned int start; |
120 | |
121 | do { |
122 | start = u64_stats_fetch_begin(syncp); |
123 | *(dst) = *src; |
124 | } while (u64_stats_fetch_retry(syncp, start)); |
125 | } |
126 | |
127 | static void ena_queue_stats(struct ena_adapter *adapter, u64 **data) |
128 | { |
129 | const struct ena_stats *ena_stats; |
130 | struct ena_ring *ring; |
131 | |
132 | u64 *ptr; |
133 | int i, j; |
134 | |
135 | for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) { |
136 | /* Tx stats */ |
137 | ring = &adapter->tx_ring[i]; |
138 | |
139 | for (j = 0; j < ENA_STATS_ARRAY_TX; j++) { |
140 | ena_stats = &ena_stats_tx_strings[j]; |
141 | |
142 | ptr = (u64 *)&ring->tx_stats + ena_stats->stat_offset; |
143 | |
144 | ena_safe_update_stat(src: ptr, dst: (*data)++, syncp: &ring->syncp); |
145 | } |
146 | /* XDP TX queues don't have a RX queue counterpart */ |
147 | if (!ENA_IS_XDP_INDEX(adapter, i)) { |
148 | /* Rx stats */ |
149 | ring = &adapter->rx_ring[i]; |
150 | |
151 | for (j = 0; j < ENA_STATS_ARRAY_RX; j++) { |
152 | ena_stats = &ena_stats_rx_strings[j]; |
153 | |
154 | ptr = (u64 *)&ring->rx_stats + |
155 | ena_stats->stat_offset; |
156 | |
157 | ena_safe_update_stat(src: ptr, dst: (*data)++, syncp: &ring->syncp); |
158 | } |
159 | } |
160 | } |
161 | } |
162 | |
163 | static void ena_dev_admin_queue_stats(struct ena_adapter *adapter, u64 **data) |
164 | { |
165 | const struct ena_stats *ena_stats; |
166 | u64 *ptr; |
167 | int i; |
168 | |
169 | for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) { |
170 | ena_stats = &ena_stats_ena_com_strings[i]; |
171 | |
172 | ptr = (u64 *)&adapter->ena_dev->admin_queue.stats + |
173 | ena_stats->stat_offset; |
174 | |
175 | *(*data)++ = *ptr; |
176 | } |
177 | } |
178 | |
179 | static void ena_get_stats(struct ena_adapter *adapter, |
180 | u64 *data, |
181 | bool eni_stats_needed) |
182 | { |
183 | const struct ena_stats *ena_stats; |
184 | u64 *ptr; |
185 | int i; |
186 | |
187 | for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) { |
188 | ena_stats = &ena_stats_global_strings[i]; |
189 | |
190 | ptr = (u64 *)&adapter->dev_stats + ena_stats->stat_offset; |
191 | |
192 | ena_safe_update_stat(src: ptr, dst: data++, syncp: &adapter->syncp); |
193 | } |
194 | |
195 | if (eni_stats_needed) { |
196 | ena_update_hw_stats(adapter); |
197 | for (i = 0; i < ENA_STATS_ARRAY_ENI(adapter); i++) { |
198 | ena_stats = &ena_stats_eni_strings[i]; |
199 | |
200 | ptr = (u64 *)&adapter->eni_stats + |
201 | ena_stats->stat_offset; |
202 | |
203 | ena_safe_update_stat(src: ptr, dst: data++, syncp: &adapter->syncp); |
204 | } |
205 | } |
206 | |
207 | ena_queue_stats(adapter, data: &data); |
208 | ena_dev_admin_queue_stats(adapter, data: &data); |
209 | } |
210 | |
211 | static void ena_get_ethtool_stats(struct net_device *netdev, |
212 | struct ethtool_stats *stats, |
213 | u64 *data) |
214 | { |
215 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
216 | struct ena_com_dev *dev = adapter->ena_dev; |
217 | |
218 | ena_get_stats(adapter, data, eni_stats_needed: ena_com_get_cap(ena_dev: dev, cap_id: ENA_ADMIN_ENI_STATS)); |
219 | } |
220 | |
221 | static int ena_get_sw_stats_count(struct ena_adapter *adapter) |
222 | { |
223 | return adapter->num_io_queues * (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX) |
224 | + adapter->xdp_num_queues * ENA_STATS_ARRAY_TX |
225 | + ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM; |
226 | } |
227 | |
228 | static int ena_get_hw_stats_count(struct ena_adapter *adapter) |
229 | { |
230 | bool supported = ena_com_get_cap(ena_dev: adapter->ena_dev, cap_id: ENA_ADMIN_ENI_STATS); |
231 | |
232 | return ENA_STATS_ARRAY_ENI(adapter) * supported; |
233 | } |
234 | |
235 | int ena_get_sset_count(struct net_device *netdev, int sset) |
236 | { |
237 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
238 | |
239 | switch (sset) { |
240 | case ETH_SS_STATS: |
241 | return ena_get_sw_stats_count(adapter) + |
242 | ena_get_hw_stats_count(adapter); |
243 | } |
244 | |
245 | return -EOPNOTSUPP; |
246 | } |
247 | |
248 | static void ena_queue_strings(struct ena_adapter *adapter, u8 **data) |
249 | { |
250 | const struct ena_stats *ena_stats; |
251 | bool is_xdp; |
252 | int i, j; |
253 | |
254 | for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) { |
255 | is_xdp = ENA_IS_XDP_INDEX(adapter, i); |
256 | /* Tx stats */ |
257 | for (j = 0; j < ENA_STATS_ARRAY_TX; j++) { |
258 | ena_stats = &ena_stats_tx_strings[j]; |
259 | |
260 | ethtool_sprintf(data, |
261 | fmt: "queue_%u_%s_%s" , i, |
262 | is_xdp ? "xdp_tx" : "tx" , |
263 | ena_stats->name); |
264 | } |
265 | |
266 | /* In XDP there isn't an RX queue counterpart */ |
267 | if (is_xdp) |
268 | continue; |
269 | |
270 | for (j = 0; j < ENA_STATS_ARRAY_RX; j++) { |
271 | ena_stats = &ena_stats_rx_strings[j]; |
272 | |
273 | ethtool_sprintf(data, fmt: "queue_%u_rx_%s" , i, ena_stats->name); |
274 | } |
275 | } |
276 | } |
277 | |
278 | static void ena_com_dev_strings(u8 **data) |
279 | { |
280 | const struct ena_stats *ena_stats; |
281 | int i; |
282 | |
283 | for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) { |
284 | ena_stats = &ena_stats_ena_com_strings[i]; |
285 | |
286 | ethtool_sprintf(data, |
287 | fmt: "ena_admin_q_%s" , ena_stats->name); |
288 | } |
289 | } |
290 | |
291 | static void ena_get_strings(struct ena_adapter *adapter, |
292 | u8 *data, |
293 | bool eni_stats_needed) |
294 | { |
295 | const struct ena_stats *ena_stats; |
296 | int i; |
297 | |
298 | for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) { |
299 | ena_stats = &ena_stats_global_strings[i]; |
300 | ethtool_puts(data: &data, str: ena_stats->name); |
301 | } |
302 | |
303 | if (eni_stats_needed) { |
304 | for (i = 0; i < ENA_STATS_ARRAY_ENI(adapter); i++) { |
305 | ena_stats = &ena_stats_eni_strings[i]; |
306 | ethtool_puts(data: &data, str: ena_stats->name); |
307 | } |
308 | } |
309 | |
310 | ena_queue_strings(adapter, data: &data); |
311 | ena_com_dev_strings(data: &data); |
312 | } |
313 | |
314 | static void ena_get_ethtool_strings(struct net_device *netdev, |
315 | u32 sset, |
316 | u8 *data) |
317 | { |
318 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
319 | struct ena_com_dev *dev = adapter->ena_dev; |
320 | |
321 | switch (sset) { |
322 | case ETH_SS_STATS: |
323 | ena_get_strings(adapter, data, eni_stats_needed: ena_com_get_cap(ena_dev: dev, cap_id: ENA_ADMIN_ENI_STATS)); |
324 | break; |
325 | } |
326 | } |
327 | |
328 | static int ena_get_link_ksettings(struct net_device *netdev, |
329 | struct ethtool_link_ksettings *link_ksettings) |
330 | { |
331 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
332 | struct ena_com_dev *ena_dev = adapter->ena_dev; |
333 | struct ena_admin_get_feature_link_desc *link; |
334 | struct ena_admin_get_feat_resp feat_resp; |
335 | int rc; |
336 | |
337 | rc = ena_com_get_link_params(ena_dev, resp: &feat_resp); |
338 | if (rc) |
339 | return rc; |
340 | |
341 | link = &feat_resp.u.link; |
342 | link_ksettings->base.speed = link->speed; |
343 | |
344 | if (link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) { |
345 | ethtool_link_ksettings_add_link_mode(link_ksettings, |
346 | supported, Autoneg); |
347 | ethtool_link_ksettings_add_link_mode(link_ksettings, |
348 | supported, Autoneg); |
349 | } |
350 | |
351 | link_ksettings->base.autoneg = |
352 | (link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) ? |
353 | AUTONEG_ENABLE : AUTONEG_DISABLE; |
354 | |
355 | link_ksettings->base.duplex = DUPLEX_FULL; |
356 | |
357 | return 0; |
358 | } |
359 | |
360 | static int ena_get_coalesce(struct net_device *net_dev, |
361 | struct ethtool_coalesce *coalesce, |
362 | struct kernel_ethtool_coalesce *kernel_coal, |
363 | struct netlink_ext_ack *extack) |
364 | { |
365 | struct ena_adapter *adapter = netdev_priv(dev: net_dev); |
366 | struct ena_com_dev *ena_dev = adapter->ena_dev; |
367 | |
368 | if (!ena_com_interrupt_moderation_supported(ena_dev)) |
369 | return -EOPNOTSUPP; |
370 | |
371 | coalesce->tx_coalesce_usecs = |
372 | ena_com_get_nonadaptive_moderation_interval_tx(ena_dev) * |
373 | ena_dev->intr_delay_resolution; |
374 | |
375 | coalesce->rx_coalesce_usecs = |
376 | ena_com_get_nonadaptive_moderation_interval_rx(ena_dev) |
377 | * ena_dev->intr_delay_resolution; |
378 | |
379 | coalesce->use_adaptive_rx_coalesce = |
380 | ena_com_get_adaptive_moderation_enabled(ena_dev); |
381 | |
382 | return 0; |
383 | } |
384 | |
385 | static void ena_update_tx_rings_nonadaptive_intr_moderation(struct ena_adapter *adapter) |
386 | { |
387 | unsigned int val; |
388 | int i; |
389 | |
390 | val = ena_com_get_nonadaptive_moderation_interval_tx(ena_dev: adapter->ena_dev); |
391 | |
392 | for (i = 0; i < adapter->num_io_queues; i++) |
393 | adapter->tx_ring[i].smoothed_interval = val; |
394 | } |
395 | |
396 | static void ena_update_rx_rings_nonadaptive_intr_moderation(struct ena_adapter *adapter) |
397 | { |
398 | unsigned int val; |
399 | int i; |
400 | |
401 | val = ena_com_get_nonadaptive_moderation_interval_rx(ena_dev: adapter->ena_dev); |
402 | |
403 | for (i = 0; i < adapter->num_io_queues; i++) |
404 | adapter->rx_ring[i].smoothed_interval = val; |
405 | } |
406 | |
407 | static int ena_set_coalesce(struct net_device *net_dev, |
408 | struct ethtool_coalesce *coalesce, |
409 | struct kernel_ethtool_coalesce *kernel_coal, |
410 | struct netlink_ext_ack *extack) |
411 | { |
412 | struct ena_adapter *adapter = netdev_priv(dev: net_dev); |
413 | struct ena_com_dev *ena_dev = adapter->ena_dev; |
414 | int rc; |
415 | |
416 | if (!ena_com_interrupt_moderation_supported(ena_dev)) |
417 | return -EOPNOTSUPP; |
418 | |
419 | rc = ena_com_update_nonadaptive_moderation_interval_tx(ena_dev, |
420 | tx_coalesce_usecs: coalesce->tx_coalesce_usecs); |
421 | if (rc) |
422 | return rc; |
423 | |
424 | ena_update_tx_rings_nonadaptive_intr_moderation(adapter); |
425 | |
426 | rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev, |
427 | rx_coalesce_usecs: coalesce->rx_coalesce_usecs); |
428 | if (rc) |
429 | return rc; |
430 | |
431 | ena_update_rx_rings_nonadaptive_intr_moderation(adapter); |
432 | |
433 | if (coalesce->use_adaptive_rx_coalesce && |
434 | !ena_com_get_adaptive_moderation_enabled(ena_dev)) |
435 | ena_com_enable_adaptive_moderation(ena_dev); |
436 | |
437 | if (!coalesce->use_adaptive_rx_coalesce && |
438 | ena_com_get_adaptive_moderation_enabled(ena_dev)) |
439 | ena_com_disable_adaptive_moderation(ena_dev); |
440 | |
441 | return 0; |
442 | } |
443 | |
444 | static u32 ena_get_msglevel(struct net_device *netdev) |
445 | { |
446 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
447 | |
448 | return adapter->msg_enable; |
449 | } |
450 | |
451 | static void ena_set_msglevel(struct net_device *netdev, u32 value) |
452 | { |
453 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
454 | |
455 | adapter->msg_enable = value; |
456 | } |
457 | |
458 | static void ena_get_drvinfo(struct net_device *dev, |
459 | struct ethtool_drvinfo *info) |
460 | { |
461 | struct ena_adapter *adapter = netdev_priv(dev); |
462 | |
463 | strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); |
464 | strscpy(info->bus_info, pci_name(adapter->pdev), |
465 | sizeof(info->bus_info)); |
466 | } |
467 | |
468 | static void ena_get_ringparam(struct net_device *netdev, |
469 | struct ethtool_ringparam *ring, |
470 | struct kernel_ethtool_ringparam *kernel_ring, |
471 | struct netlink_ext_ack *extack) |
472 | { |
473 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
474 | |
475 | ring->tx_max_pending = adapter->max_tx_ring_size; |
476 | ring->rx_max_pending = adapter->max_rx_ring_size; |
477 | if (adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { |
478 | bool large_llq_supported = adapter->large_llq_header_supported; |
479 | |
480 | kernel_ring->tx_push = true; |
481 | kernel_ring->tx_push_buf_len = adapter->ena_dev->tx_max_header_size; |
482 | if (large_llq_supported) |
483 | kernel_ring->tx_push_buf_max_len = ENA_LLQ_LARGE_HEADER; |
484 | else |
485 | kernel_ring->tx_push_buf_max_len = ENA_LLQ_HEADER; |
486 | } else { |
487 | kernel_ring->tx_push = false; |
488 | kernel_ring->tx_push_buf_max_len = 0; |
489 | kernel_ring->tx_push_buf_len = 0; |
490 | } |
491 | |
492 | ring->tx_pending = adapter->tx_ring[0].ring_size; |
493 | ring->rx_pending = adapter->rx_ring[0].ring_size; |
494 | } |
495 | |
496 | static int ena_set_ringparam(struct net_device *netdev, |
497 | struct ethtool_ringparam *ring, |
498 | struct kernel_ethtool_ringparam *kernel_ring, |
499 | struct netlink_ext_ack *extack) |
500 | { |
501 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
502 | u32 new_tx_size, new_rx_size, new_tx_push_buf_len; |
503 | bool changed = false; |
504 | |
505 | new_tx_size = ring->tx_pending < ENA_MIN_RING_SIZE ? |
506 | ENA_MIN_RING_SIZE : ring->tx_pending; |
507 | new_tx_size = rounddown_pow_of_two(new_tx_size); |
508 | |
509 | new_rx_size = ring->rx_pending < ENA_MIN_RING_SIZE ? |
510 | ENA_MIN_RING_SIZE : ring->rx_pending; |
511 | new_rx_size = rounddown_pow_of_two(new_rx_size); |
512 | |
513 | changed |= new_tx_size != adapter->requested_tx_ring_size || |
514 | new_rx_size != adapter->requested_rx_ring_size; |
515 | |
516 | /* This value is ignored if LLQ is not supported */ |
517 | new_tx_push_buf_len = adapter->ena_dev->tx_max_header_size; |
518 | |
519 | if ((adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) != |
520 | kernel_ring->tx_push) { |
521 | NL_SET_ERR_MSG_MOD(extack, "Push mode state cannot be modified" ); |
522 | return -EINVAL; |
523 | } |
524 | |
525 | /* Validate that the push buffer is supported on the underlying device */ |
526 | if (kernel_ring->tx_push_buf_len) { |
527 | enum ena_admin_placement_policy_type placement; |
528 | |
529 | new_tx_push_buf_len = kernel_ring->tx_push_buf_len; |
530 | |
531 | placement = adapter->ena_dev->tx_mem_queue_type; |
532 | if (placement == ENA_ADMIN_PLACEMENT_POLICY_HOST) |
533 | return -EOPNOTSUPP; |
534 | |
535 | if (new_tx_push_buf_len != ENA_LLQ_HEADER && |
536 | new_tx_push_buf_len != ENA_LLQ_LARGE_HEADER) { |
537 | bool large_llq_sup = adapter->large_llq_header_supported; |
538 | char large_llq_size_str[40]; |
539 | |
540 | snprintf(buf: large_llq_size_str, size: 40, fmt: ", %lu" , ENA_LLQ_LARGE_HEADER); |
541 | |
542 | NL_SET_ERR_MSG_FMT_MOD(extack, |
543 | "Supported tx push buff values: [%lu%s]" , |
544 | ENA_LLQ_HEADER, |
545 | large_llq_sup ? large_llq_size_str : "" ); |
546 | |
547 | return -EINVAL; |
548 | } |
549 | |
550 | changed |= new_tx_push_buf_len != adapter->ena_dev->tx_max_header_size; |
551 | } |
552 | |
553 | if (!changed) |
554 | return 0; |
555 | |
556 | return ena_update_queue_params(adapter, new_tx_size, new_rx_size, |
557 | new_llq_header_len: new_tx_push_buf_len); |
558 | } |
559 | |
560 | static u32 ena_flow_hash_to_flow_type(u16 hash_fields) |
561 | { |
562 | u32 data = 0; |
563 | |
564 | if (hash_fields & ENA_ADMIN_RSS_L2_DA) |
565 | data |= RXH_L2DA; |
566 | |
567 | if (hash_fields & ENA_ADMIN_RSS_L3_DA) |
568 | data |= RXH_IP_DST; |
569 | |
570 | if (hash_fields & ENA_ADMIN_RSS_L3_SA) |
571 | data |= RXH_IP_SRC; |
572 | |
573 | if (hash_fields & ENA_ADMIN_RSS_L4_DP) |
574 | data |= RXH_L4_B_2_3; |
575 | |
576 | if (hash_fields & ENA_ADMIN_RSS_L4_SP) |
577 | data |= RXH_L4_B_0_1; |
578 | |
579 | return data; |
580 | } |
581 | |
582 | static u16 ena_flow_data_to_flow_hash(u32 hash_fields) |
583 | { |
584 | u16 data = 0; |
585 | |
586 | if (hash_fields & RXH_L2DA) |
587 | data |= ENA_ADMIN_RSS_L2_DA; |
588 | |
589 | if (hash_fields & RXH_IP_DST) |
590 | data |= ENA_ADMIN_RSS_L3_DA; |
591 | |
592 | if (hash_fields & RXH_IP_SRC) |
593 | data |= ENA_ADMIN_RSS_L3_SA; |
594 | |
595 | if (hash_fields & RXH_L4_B_2_3) |
596 | data |= ENA_ADMIN_RSS_L4_DP; |
597 | |
598 | if (hash_fields & RXH_L4_B_0_1) |
599 | data |= ENA_ADMIN_RSS_L4_SP; |
600 | |
601 | return data; |
602 | } |
603 | |
604 | static int (struct ena_com_dev *ena_dev, |
605 | struct ethtool_rxnfc *cmd) |
606 | { |
607 | enum ena_admin_flow_hash_proto proto; |
608 | u16 hash_fields; |
609 | int rc; |
610 | |
611 | cmd->data = 0; |
612 | |
613 | switch (cmd->flow_type) { |
614 | case TCP_V4_FLOW: |
615 | proto = ENA_ADMIN_RSS_TCP4; |
616 | break; |
617 | case UDP_V4_FLOW: |
618 | proto = ENA_ADMIN_RSS_UDP4; |
619 | break; |
620 | case TCP_V6_FLOW: |
621 | proto = ENA_ADMIN_RSS_TCP6; |
622 | break; |
623 | case UDP_V6_FLOW: |
624 | proto = ENA_ADMIN_RSS_UDP6; |
625 | break; |
626 | case IPV4_FLOW: |
627 | proto = ENA_ADMIN_RSS_IP4; |
628 | break; |
629 | case IPV6_FLOW: |
630 | proto = ENA_ADMIN_RSS_IP6; |
631 | break; |
632 | case ETHER_FLOW: |
633 | proto = ENA_ADMIN_RSS_NOT_IP; |
634 | break; |
635 | case AH_V4_FLOW: |
636 | case ESP_V4_FLOW: |
637 | case AH_V6_FLOW: |
638 | case ESP_V6_FLOW: |
639 | case SCTP_V4_FLOW: |
640 | case AH_ESP_V4_FLOW: |
641 | return -EOPNOTSUPP; |
642 | default: |
643 | return -EINVAL; |
644 | } |
645 | |
646 | rc = ena_com_get_hash_ctrl(ena_dev, proto, fields: &hash_fields); |
647 | if (rc) |
648 | return rc; |
649 | |
650 | cmd->data = ena_flow_hash_to_flow_type(hash_fields); |
651 | |
652 | return 0; |
653 | } |
654 | |
655 | static int (struct ena_com_dev *ena_dev, |
656 | struct ethtool_rxnfc *cmd) |
657 | { |
658 | enum ena_admin_flow_hash_proto proto; |
659 | u16 hash_fields; |
660 | |
661 | switch (cmd->flow_type) { |
662 | case TCP_V4_FLOW: |
663 | proto = ENA_ADMIN_RSS_TCP4; |
664 | break; |
665 | case UDP_V4_FLOW: |
666 | proto = ENA_ADMIN_RSS_UDP4; |
667 | break; |
668 | case TCP_V6_FLOW: |
669 | proto = ENA_ADMIN_RSS_TCP6; |
670 | break; |
671 | case UDP_V6_FLOW: |
672 | proto = ENA_ADMIN_RSS_UDP6; |
673 | break; |
674 | case IPV4_FLOW: |
675 | proto = ENA_ADMIN_RSS_IP4; |
676 | break; |
677 | case IPV6_FLOW: |
678 | proto = ENA_ADMIN_RSS_IP6; |
679 | break; |
680 | case ETHER_FLOW: |
681 | proto = ENA_ADMIN_RSS_NOT_IP; |
682 | break; |
683 | case AH_V4_FLOW: |
684 | case ESP_V4_FLOW: |
685 | case AH_V6_FLOW: |
686 | case ESP_V6_FLOW: |
687 | case SCTP_V4_FLOW: |
688 | case AH_ESP_V4_FLOW: |
689 | return -EOPNOTSUPP; |
690 | default: |
691 | return -EINVAL; |
692 | } |
693 | |
694 | hash_fields = ena_flow_data_to_flow_hash(hash_fields: cmd->data); |
695 | |
696 | return ena_com_fill_hash_ctrl(ena_dev, proto, hash_fields); |
697 | } |
698 | |
699 | static int ena_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info) |
700 | { |
701 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
702 | int rc = 0; |
703 | |
704 | switch (info->cmd) { |
705 | case ETHTOOL_SRXFH: |
706 | rc = ena_set_rss_hash(ena_dev: adapter->ena_dev, cmd: info); |
707 | break; |
708 | case ETHTOOL_SRXCLSRLDEL: |
709 | case ETHTOOL_SRXCLSRLINS: |
710 | default: |
711 | netif_err(adapter, drv, netdev, |
712 | "Command parameter %d is not supported\n" , info->cmd); |
713 | rc = -EOPNOTSUPP; |
714 | } |
715 | |
716 | return rc; |
717 | } |
718 | |
719 | static int ena_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info, |
720 | u32 *rules) |
721 | { |
722 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
723 | int rc = 0; |
724 | |
725 | switch (info->cmd) { |
726 | case ETHTOOL_GRXRINGS: |
727 | info->data = adapter->num_io_queues; |
728 | rc = 0; |
729 | break; |
730 | case ETHTOOL_GRXFH: |
731 | rc = ena_get_rss_hash(ena_dev: adapter->ena_dev, cmd: info); |
732 | break; |
733 | case ETHTOOL_GRXCLSRLCNT: |
734 | case ETHTOOL_GRXCLSRULE: |
735 | case ETHTOOL_GRXCLSRLALL: |
736 | default: |
737 | netif_err(adapter, drv, netdev, |
738 | "Command parameter %d is not supported\n" , info->cmd); |
739 | rc = -EOPNOTSUPP; |
740 | } |
741 | |
742 | return rc; |
743 | } |
744 | |
745 | static u32 ena_get_rxfh_indir_size(struct net_device *netdev) |
746 | { |
747 | return ENA_RX_RSS_TABLE_SIZE; |
748 | } |
749 | |
750 | static u32 ena_get_rxfh_key_size(struct net_device *netdev) |
751 | { |
752 | return ENA_HASH_KEY_SIZE; |
753 | } |
754 | |
755 | static int ena_indirection_table_set(struct ena_adapter *adapter, |
756 | const u32 *indir) |
757 | { |
758 | struct ena_com_dev *ena_dev = adapter->ena_dev; |
759 | int i, rc; |
760 | |
761 | for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { |
762 | rc = ena_com_indirect_table_fill_entry(ena_dev, |
763 | entry_idx: i, |
764 | ENA_IO_RXQ_IDX(indir[i])); |
765 | if (unlikely(rc)) { |
766 | netif_err(adapter, drv, adapter->netdev, |
767 | "Cannot fill indirect table (index is too large)\n" ); |
768 | return rc; |
769 | } |
770 | } |
771 | |
772 | rc = ena_com_indirect_table_set(ena_dev); |
773 | if (rc) { |
774 | netif_err(adapter, drv, adapter->netdev, |
775 | "Cannot set indirect table\n" ); |
776 | return rc == -EPERM ? -EOPNOTSUPP : rc; |
777 | } |
778 | return rc; |
779 | } |
780 | |
781 | static int ena_indirection_table_get(struct ena_adapter *adapter, u32 *indir) |
782 | { |
783 | struct ena_com_dev *ena_dev = adapter->ena_dev; |
784 | int i, rc; |
785 | |
786 | if (!indir) |
787 | return 0; |
788 | |
789 | rc = ena_com_indirect_table_get(ena_dev, ind_tbl: indir); |
790 | if (rc) |
791 | return rc; |
792 | |
793 | /* Our internal representation of the indices is: even indices |
794 | * for Tx and uneven indices for Rx. We need to convert the Rx |
795 | * indices to be consecutive |
796 | */ |
797 | for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) |
798 | indir[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(indir[i]); |
799 | |
800 | return rc; |
801 | } |
802 | |
803 | static int ena_get_rxfh(struct net_device *netdev, |
804 | struct ethtool_rxfh_param *rxfh) |
805 | { |
806 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
807 | enum ena_admin_hash_functions ena_func; |
808 | u8 func; |
809 | int rc; |
810 | |
811 | rc = ena_indirection_table_get(adapter, indir: rxfh->indir); |
812 | if (rc) |
813 | return rc; |
814 | |
815 | /* We call this function in order to check if the device |
816 | * supports getting/setting the hash function. |
817 | */ |
818 | rc = ena_com_get_hash_function(ena_dev: adapter->ena_dev, func: &ena_func); |
819 | if (rc) { |
820 | if (rc == -EOPNOTSUPP) |
821 | rc = 0; |
822 | |
823 | return rc; |
824 | } |
825 | |
826 | rc = ena_com_get_hash_key(ena_dev: adapter->ena_dev, key: rxfh->key); |
827 | if (rc) |
828 | return rc; |
829 | |
830 | switch (ena_func) { |
831 | case ENA_ADMIN_TOEPLITZ: |
832 | func = ETH_RSS_HASH_TOP; |
833 | break; |
834 | case ENA_ADMIN_CRC32: |
835 | func = ETH_RSS_HASH_CRC32; |
836 | break; |
837 | default: |
838 | netif_err(adapter, drv, netdev, |
839 | "Command parameter is not supported\n" ); |
840 | return -EOPNOTSUPP; |
841 | } |
842 | |
843 | rxfh->hfunc = func; |
844 | |
845 | return 0; |
846 | } |
847 | |
848 | static int ena_set_rxfh(struct net_device *netdev, |
849 | struct ethtool_rxfh_param *rxfh, |
850 | struct netlink_ext_ack *extack) |
851 | { |
852 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
853 | struct ena_com_dev *ena_dev = adapter->ena_dev; |
854 | enum ena_admin_hash_functions func = 0; |
855 | int rc; |
856 | |
857 | if (rxfh->indir) { |
858 | rc = ena_indirection_table_set(adapter, indir: rxfh->indir); |
859 | if (rc) |
860 | return rc; |
861 | } |
862 | |
863 | switch (rxfh->hfunc) { |
864 | case ETH_RSS_HASH_NO_CHANGE: |
865 | func = ena_com_get_current_hash_function(ena_dev); |
866 | break; |
867 | case ETH_RSS_HASH_TOP: |
868 | func = ENA_ADMIN_TOEPLITZ; |
869 | break; |
870 | case ETH_RSS_HASH_CRC32: |
871 | func = ENA_ADMIN_CRC32; |
872 | break; |
873 | default: |
874 | netif_err(adapter, drv, netdev, "Unsupported hfunc %d\n" , |
875 | rxfh->hfunc); |
876 | return -EOPNOTSUPP; |
877 | } |
878 | |
879 | if (rxfh->key || func) { |
880 | rc = ena_com_fill_hash_function(ena_dev, func, key: rxfh->key, |
881 | ENA_HASH_KEY_SIZE, |
882 | init_val: 0xFFFFFFFF); |
883 | if (unlikely(rc)) { |
884 | netif_err(adapter, drv, netdev, "Cannot fill key\n" ); |
885 | return rc == -EPERM ? -EOPNOTSUPP : rc; |
886 | } |
887 | } |
888 | |
889 | return 0; |
890 | } |
891 | |
892 | static void ena_get_channels(struct net_device *netdev, |
893 | struct ethtool_channels *channels) |
894 | { |
895 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
896 | |
897 | channels->max_combined = adapter->max_num_io_queues; |
898 | channels->combined_count = adapter->num_io_queues; |
899 | } |
900 | |
901 | static int ena_set_channels(struct net_device *netdev, |
902 | struct ethtool_channels *channels) |
903 | { |
904 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
905 | u32 count = channels->combined_count; |
906 | /* The check for max value is already done in ethtool */ |
907 | if (count < ENA_MIN_NUM_IO_QUEUES) |
908 | return -EINVAL; |
909 | |
910 | if (!ena_xdp_legal_queue_count(adapter, queues: count)) { |
911 | if (ena_xdp_present(adapter)) |
912 | return -EINVAL; |
913 | |
914 | xdp_clear_features_flag(dev: netdev); |
915 | } else { |
916 | xdp_set_features_flag(dev: netdev, |
917 | val: NETDEV_XDP_ACT_BASIC | |
918 | NETDEV_XDP_ACT_REDIRECT); |
919 | } |
920 | |
921 | return ena_update_queue_count(adapter, new_channel_count: count); |
922 | } |
923 | |
924 | static int ena_get_tunable(struct net_device *netdev, |
925 | const struct ethtool_tunable *tuna, void *data) |
926 | { |
927 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
928 | int ret = 0; |
929 | |
930 | switch (tuna->id) { |
931 | case ETHTOOL_RX_COPYBREAK: |
932 | *(u32 *)data = adapter->rx_copybreak; |
933 | break; |
934 | default: |
935 | ret = -EINVAL; |
936 | break; |
937 | } |
938 | |
939 | return ret; |
940 | } |
941 | |
942 | static int ena_set_tunable(struct net_device *netdev, |
943 | const struct ethtool_tunable *tuna, |
944 | const void *data) |
945 | { |
946 | struct ena_adapter *adapter = netdev_priv(dev: netdev); |
947 | int ret = 0; |
948 | u32 len; |
949 | |
950 | switch (tuna->id) { |
951 | case ETHTOOL_RX_COPYBREAK: |
952 | len = *(u32 *)data; |
953 | ret = ena_set_rx_copybreak(adapter, rx_copybreak: len); |
954 | break; |
955 | default: |
956 | ret = -EINVAL; |
957 | break; |
958 | } |
959 | |
960 | return ret; |
961 | } |
962 | |
963 | static const struct ethtool_ops ena_ethtool_ops = { |
964 | .supported_coalesce_params = ETHTOOL_COALESCE_USECS | |
965 | ETHTOOL_COALESCE_USE_ADAPTIVE_RX, |
966 | .supported_ring_params = ETHTOOL_RING_USE_TX_PUSH_BUF_LEN | |
967 | ETHTOOL_RING_USE_TX_PUSH, |
968 | .get_link_ksettings = ena_get_link_ksettings, |
969 | .get_drvinfo = ena_get_drvinfo, |
970 | .get_msglevel = ena_get_msglevel, |
971 | .set_msglevel = ena_set_msglevel, |
972 | .get_link = ethtool_op_get_link, |
973 | .get_coalesce = ena_get_coalesce, |
974 | .set_coalesce = ena_set_coalesce, |
975 | .get_ringparam = ena_get_ringparam, |
976 | .set_ringparam = ena_set_ringparam, |
977 | .get_sset_count = ena_get_sset_count, |
978 | .get_strings = ena_get_ethtool_strings, |
979 | .get_ethtool_stats = ena_get_ethtool_stats, |
980 | .get_rxnfc = ena_get_rxnfc, |
981 | .set_rxnfc = ena_set_rxnfc, |
982 | .get_rxfh_indir_size = ena_get_rxfh_indir_size, |
983 | .get_rxfh_key_size = ena_get_rxfh_key_size, |
984 | .get_rxfh = ena_get_rxfh, |
985 | .set_rxfh = ena_set_rxfh, |
986 | .get_channels = ena_get_channels, |
987 | .set_channels = ena_set_channels, |
988 | .get_tunable = ena_get_tunable, |
989 | .set_tunable = ena_set_tunable, |
990 | .get_ts_info = ethtool_op_get_ts_info, |
991 | }; |
992 | |
993 | void ena_set_ethtool_ops(struct net_device *netdev) |
994 | { |
995 | netdev->ethtool_ops = &ena_ethtool_ops; |
996 | } |
997 | |
998 | static void ena_dump_stats_ex(struct ena_adapter *adapter, u8 *buf) |
999 | { |
1000 | struct net_device *netdev = adapter->netdev; |
1001 | u8 *strings_buf; |
1002 | u64 *data_buf; |
1003 | int strings_num; |
1004 | int i, rc; |
1005 | |
1006 | strings_num = ena_get_sw_stats_count(adapter); |
1007 | if (strings_num <= 0) { |
1008 | netif_err(adapter, drv, netdev, "Can't get stats num\n" ); |
1009 | return; |
1010 | } |
1011 | |
1012 | strings_buf = devm_kcalloc(dev: &adapter->pdev->dev, |
1013 | ETH_GSTRING_LEN, size: strings_num, |
1014 | GFP_ATOMIC); |
1015 | if (!strings_buf) { |
1016 | netif_err(adapter, drv, netdev, |
1017 | "Failed to allocate strings_buf\n" ); |
1018 | return; |
1019 | } |
1020 | |
1021 | data_buf = devm_kcalloc(dev: &adapter->pdev->dev, |
1022 | n: strings_num, size: sizeof(u64), |
1023 | GFP_ATOMIC); |
1024 | if (!data_buf) { |
1025 | netif_err(adapter, drv, netdev, |
1026 | "Failed to allocate data buf\n" ); |
1027 | devm_kfree(dev: &adapter->pdev->dev, p: strings_buf); |
1028 | return; |
1029 | } |
1030 | |
1031 | ena_get_strings(adapter, data: strings_buf, eni_stats_needed: false); |
1032 | ena_get_stats(adapter, data: data_buf, eni_stats_needed: false); |
1033 | |
1034 | /* If there is a buffer, dump stats, otherwise print them to dmesg */ |
1035 | if (buf) |
1036 | for (i = 0; i < strings_num; i++) { |
1037 | rc = snprintf(buf, ETH_GSTRING_LEN + sizeof(u64), |
1038 | fmt: "%s %llu\n" , |
1039 | strings_buf + i * ETH_GSTRING_LEN, |
1040 | data_buf[i]); |
1041 | buf += rc; |
1042 | } |
1043 | else |
1044 | for (i = 0; i < strings_num; i++) |
1045 | netif_err(adapter, drv, netdev, "%s: %llu\n" , |
1046 | strings_buf + i * ETH_GSTRING_LEN, |
1047 | data_buf[i]); |
1048 | |
1049 | devm_kfree(dev: &adapter->pdev->dev, p: strings_buf); |
1050 | devm_kfree(dev: &adapter->pdev->dev, p: data_buf); |
1051 | } |
1052 | |
1053 | void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf) |
1054 | { |
1055 | if (!buf) |
1056 | return; |
1057 | |
1058 | ena_dump_stats_ex(adapter, buf); |
1059 | } |
1060 | |
1061 | void ena_dump_stats_to_dmesg(struct ena_adapter *adapter) |
1062 | { |
1063 | ena_dump_stats_ex(adapter, NULL); |
1064 | } |
1065 | |