1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2018 Netronome Systems, Inc. */ |
3 | |
4 | #include <linux/rtnetlink.h> |
5 | #include <net/pkt_cls.h> |
6 | #include <net/pkt_sched.h> |
7 | #include <net/red.h> |
8 | |
9 | #include "../nfpcore/nfp_cpp.h" |
10 | #include "../nfp_app.h" |
11 | #include "../nfp_main.h" |
12 | #include "../nfp_net.h" |
13 | #include "../nfp_port.h" |
14 | #include "main.h" |
15 | |
16 | static bool nfp_abm_qdisc_is_red(struct nfp_qdisc *qdisc) |
17 | { |
18 | return qdisc->type == NFP_QDISC_RED || qdisc->type == NFP_QDISC_GRED; |
19 | } |
20 | |
21 | static bool nfp_abm_qdisc_child_valid(struct nfp_qdisc *qdisc, unsigned int id) |
22 | { |
23 | return qdisc->children[id] && |
24 | qdisc->children[id] != NFP_QDISC_UNTRACKED; |
25 | } |
26 | |
27 | static void *nfp_abm_qdisc_tree_deref_slot(void __rcu **slot) |
28 | { |
29 | return rtnl_dereference(*slot); |
30 | } |
31 | |
32 | static void |
33 | nfp_abm_stats_propagate(struct nfp_alink_stats *parent, |
34 | struct nfp_alink_stats *child) |
35 | { |
36 | parent->tx_pkts += child->tx_pkts; |
37 | parent->tx_bytes += child->tx_bytes; |
38 | parent->backlog_pkts += child->backlog_pkts; |
39 | parent->backlog_bytes += child->backlog_bytes; |
40 | parent->overlimits += child->overlimits; |
41 | parent->drops += child->drops; |
42 | } |
43 | |
44 | static void |
45 | nfp_abm_stats_update_red(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc, |
46 | unsigned int queue) |
47 | { |
48 | struct nfp_cpp *cpp = alink->abm->app->cpp; |
49 | unsigned int i; |
50 | int err; |
51 | |
52 | if (!qdisc->offloaded) |
53 | return; |
54 | |
55 | for (i = 0; i < qdisc->red.num_bands; i++) { |
56 | err = nfp_abm_ctrl_read_q_stats(alink, band: i, queue, |
57 | stats: &qdisc->red.band[i].stats); |
58 | if (err) |
59 | nfp_err(cpp, "RED stats (%d, %d) read failed with error %d\n" , |
60 | i, queue, err); |
61 | |
62 | err = nfp_abm_ctrl_read_q_xstats(alink, band: i, queue, |
63 | xstats: &qdisc->red.band[i].xstats); |
64 | if (err) |
65 | nfp_err(cpp, "RED xstats (%d, %d) read failed with error %d\n" , |
66 | i, queue, err); |
67 | } |
68 | } |
69 | |
70 | static void |
71 | nfp_abm_stats_update_mq(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc) |
72 | { |
73 | unsigned int i; |
74 | |
75 | if (qdisc->type != NFP_QDISC_MQ) |
76 | return; |
77 | |
78 | for (i = 0; i < alink->total_queues; i++) |
79 | if (nfp_abm_qdisc_child_valid(qdisc, id: i)) |
80 | nfp_abm_stats_update_red(alink, qdisc: qdisc->children[i], queue: i); |
81 | } |
82 | |
83 | static void __nfp_abm_stats_update(struct nfp_abm_link *alink, u64 time_now) |
84 | { |
85 | alink->last_stats_update = time_now; |
86 | if (alink->root_qdisc) |
87 | nfp_abm_stats_update_mq(alink, qdisc: alink->root_qdisc); |
88 | } |
89 | |
90 | static void nfp_abm_stats_update(struct nfp_abm_link *alink) |
91 | { |
92 | u64 now; |
93 | |
94 | /* Limit the frequency of updates - stats of non-leaf qdiscs are a sum |
95 | * of all their leafs, so we would read the same stat multiple times |
96 | * for every dump. |
97 | */ |
98 | now = ktime_get(); |
99 | if (now - alink->last_stats_update < NFP_ABM_STATS_REFRESH_IVAL) |
100 | return; |
101 | |
102 | __nfp_abm_stats_update(alink, time_now: now); |
103 | } |
104 | |
105 | static void |
106 | nfp_abm_qdisc_unlink_children(struct nfp_qdisc *qdisc, |
107 | unsigned int start, unsigned int end) |
108 | { |
109 | unsigned int i; |
110 | |
111 | for (i = start; i < end; i++) |
112 | if (nfp_abm_qdisc_child_valid(qdisc, id: i)) { |
113 | qdisc->children[i]->use_cnt--; |
114 | qdisc->children[i] = NULL; |
115 | } |
116 | } |
117 | |
118 | static void |
119 | nfp_abm_qdisc_offload_stop(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc) |
120 | { |
121 | unsigned int i; |
122 | |
123 | /* Don't complain when qdisc is getting unlinked */ |
124 | if (qdisc->use_cnt) |
125 | nfp_warn(alink->abm->app->cpp, "Offload of '%08x' stopped\n" , |
126 | qdisc->handle); |
127 | |
128 | if (!nfp_abm_qdisc_is_red(qdisc)) |
129 | return; |
130 | |
131 | for (i = 0; i < qdisc->red.num_bands; i++) { |
132 | qdisc->red.band[i].stats.backlog_pkts = 0; |
133 | qdisc->red.band[i].stats.backlog_bytes = 0; |
134 | } |
135 | } |
136 | |
137 | static int |
138 | __nfp_abm_stats_init(struct nfp_abm_link *alink, unsigned int band, |
139 | unsigned int queue, struct nfp_alink_stats *prev_stats, |
140 | struct nfp_alink_xstats *prev_xstats) |
141 | { |
142 | u64 backlog_pkts, backlog_bytes; |
143 | int err; |
144 | |
145 | /* Don't touch the backlog, backlog can only be reset after it has |
146 | * been reported back to the tc qdisc stats. |
147 | */ |
148 | backlog_pkts = prev_stats->backlog_pkts; |
149 | backlog_bytes = prev_stats->backlog_bytes; |
150 | |
151 | err = nfp_abm_ctrl_read_q_stats(alink, band, queue, stats: prev_stats); |
152 | if (err) { |
153 | nfp_err(alink->abm->app->cpp, |
154 | "RED stats init (%d, %d) failed with error %d\n" , |
155 | band, queue, err); |
156 | return err; |
157 | } |
158 | |
159 | err = nfp_abm_ctrl_read_q_xstats(alink, band, queue, xstats: prev_xstats); |
160 | if (err) { |
161 | nfp_err(alink->abm->app->cpp, |
162 | "RED xstats init (%d, %d) failed with error %d\n" , |
163 | band, queue, err); |
164 | return err; |
165 | } |
166 | |
167 | prev_stats->backlog_pkts = backlog_pkts; |
168 | prev_stats->backlog_bytes = backlog_bytes; |
169 | return 0; |
170 | } |
171 | |
172 | static int |
173 | nfp_abm_stats_init(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc, |
174 | unsigned int queue) |
175 | { |
176 | unsigned int i; |
177 | int err; |
178 | |
179 | for (i = 0; i < qdisc->red.num_bands; i++) { |
180 | err = __nfp_abm_stats_init(alink, band: i, queue, |
181 | prev_stats: &qdisc->red.band[i].prev_stats, |
182 | prev_xstats: &qdisc->red.band[i].prev_xstats); |
183 | if (err) |
184 | return err; |
185 | } |
186 | |
187 | return 0; |
188 | } |
189 | |
190 | static void |
191 | nfp_abm_offload_compile_red(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc, |
192 | unsigned int queue) |
193 | { |
194 | bool good_red, good_gred; |
195 | unsigned int i; |
196 | |
197 | good_red = qdisc->type == NFP_QDISC_RED && |
198 | qdisc->params_ok && |
199 | qdisc->use_cnt == 1 && |
200 | !alink->has_prio && |
201 | !qdisc->children[0]; |
202 | good_gred = qdisc->type == NFP_QDISC_GRED && |
203 | qdisc->params_ok && |
204 | qdisc->use_cnt == 1; |
205 | qdisc->offload_mark = good_red || good_gred; |
206 | |
207 | /* If we are starting offload init prev_stats */ |
208 | if (qdisc->offload_mark && !qdisc->offloaded) |
209 | if (nfp_abm_stats_init(alink, qdisc, queue)) |
210 | qdisc->offload_mark = false; |
211 | |
212 | if (!qdisc->offload_mark) |
213 | return; |
214 | |
215 | for (i = 0; i < alink->abm->num_bands; i++) { |
216 | enum nfp_abm_q_action act; |
217 | |
218 | nfp_abm_ctrl_set_q_lvl(alink, band: i, queue, |
219 | val: qdisc->red.band[i].threshold); |
220 | act = qdisc->red.band[i].ecn ? |
221 | NFP_ABM_ACT_MARK_DROP : NFP_ABM_ACT_DROP; |
222 | nfp_abm_ctrl_set_q_act(alink, band: i, queue, act); |
223 | } |
224 | } |
225 | |
226 | static void |
227 | nfp_abm_offload_compile_mq(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc) |
228 | { |
229 | unsigned int i; |
230 | |
231 | qdisc->offload_mark = qdisc->type == NFP_QDISC_MQ; |
232 | if (!qdisc->offload_mark) |
233 | return; |
234 | |
235 | for (i = 0; i < alink->total_queues; i++) { |
236 | struct nfp_qdisc *child = qdisc->children[i]; |
237 | |
238 | if (!nfp_abm_qdisc_child_valid(qdisc, id: i)) |
239 | continue; |
240 | |
241 | nfp_abm_offload_compile_red(alink, qdisc: child, queue: i); |
242 | } |
243 | } |
244 | |
245 | void nfp_abm_qdisc_offload_update(struct nfp_abm_link *alink) |
246 | { |
247 | struct nfp_abm *abm = alink->abm; |
248 | struct radix_tree_iter iter; |
249 | struct nfp_qdisc *qdisc; |
250 | void __rcu **slot; |
251 | size_t i; |
252 | |
253 | /* Mark all thresholds as unconfigured */ |
254 | for (i = 0; i < abm->num_bands; i++) |
255 | __bitmap_set(map: abm->threshold_undef, |
256 | start: i * NFP_NET_MAX_RX_RINGS + alink->queue_base, |
257 | len: alink->total_queues); |
258 | |
259 | /* Clear offload marks */ |
260 | radix_tree_for_each_slot(slot, &alink->qdiscs, &iter, 0) { |
261 | qdisc = nfp_abm_qdisc_tree_deref_slot(slot); |
262 | qdisc->offload_mark = false; |
263 | } |
264 | |
265 | if (alink->root_qdisc) |
266 | nfp_abm_offload_compile_mq(alink, qdisc: alink->root_qdisc); |
267 | |
268 | /* Refresh offload status */ |
269 | radix_tree_for_each_slot(slot, &alink->qdiscs, &iter, 0) { |
270 | qdisc = nfp_abm_qdisc_tree_deref_slot(slot); |
271 | if (!qdisc->offload_mark && qdisc->offloaded) |
272 | nfp_abm_qdisc_offload_stop(alink, qdisc); |
273 | qdisc->offloaded = qdisc->offload_mark; |
274 | } |
275 | |
276 | /* Reset the unconfigured thresholds */ |
277 | for (i = 0; i < abm->num_thresholds; i++) |
278 | if (test_bit(i, abm->threshold_undef)) |
279 | __nfp_abm_ctrl_set_q_lvl(abm, id: i, NFP_ABM_LVL_INFINITY); |
280 | |
281 | __nfp_abm_stats_update(alink, time_now: ktime_get()); |
282 | } |
283 | |
284 | static void |
285 | nfp_abm_qdisc_clear_mq(struct net_device *netdev, struct nfp_abm_link *alink, |
286 | struct nfp_qdisc *qdisc) |
287 | { |
288 | struct radix_tree_iter iter; |
289 | unsigned int mq_refs = 0; |
290 | void __rcu **slot; |
291 | |
292 | if (!qdisc->use_cnt) |
293 | return; |
294 | /* MQ doesn't notify well on destruction, we need special handling of |
295 | * MQ's children. |
296 | */ |
297 | if (qdisc->type == NFP_QDISC_MQ && |
298 | qdisc == alink->root_qdisc && |
299 | netdev->reg_state == NETREG_UNREGISTERING) |
300 | return; |
301 | |
302 | /* Count refs held by MQ instances and clear pointers */ |
303 | radix_tree_for_each_slot(slot, &alink->qdiscs, &iter, 0) { |
304 | struct nfp_qdisc *mq = nfp_abm_qdisc_tree_deref_slot(slot); |
305 | unsigned int i; |
306 | |
307 | if (mq->type != NFP_QDISC_MQ || mq->netdev != netdev) |
308 | continue; |
309 | for (i = 0; i < mq->num_children; i++) |
310 | if (mq->children[i] == qdisc) { |
311 | mq->children[i] = NULL; |
312 | mq_refs++; |
313 | } |
314 | } |
315 | |
316 | WARN(qdisc->use_cnt != mq_refs, "non-zero qdisc use count: %d (- %d)\n" , |
317 | qdisc->use_cnt, mq_refs); |
318 | } |
319 | |
320 | static void |
321 | nfp_abm_qdisc_free(struct net_device *netdev, struct nfp_abm_link *alink, |
322 | struct nfp_qdisc *qdisc) |
323 | { |
324 | struct nfp_port *port = nfp_port_from_netdev(netdev); |
325 | |
326 | if (!qdisc) |
327 | return; |
328 | nfp_abm_qdisc_clear_mq(netdev, alink, qdisc); |
329 | WARN_ON(radix_tree_delete(&alink->qdiscs, |
330 | TC_H_MAJ(qdisc->handle)) != qdisc); |
331 | |
332 | kfree(objp: qdisc->children); |
333 | kfree(objp: qdisc); |
334 | |
335 | port->tc_offload_cnt--; |
336 | } |
337 | |
338 | static struct nfp_qdisc * |
339 | nfp_abm_qdisc_alloc(struct net_device *netdev, struct nfp_abm_link *alink, |
340 | enum nfp_qdisc_type type, u32 parent_handle, u32 handle, |
341 | unsigned int children) |
342 | { |
343 | struct nfp_port *port = nfp_port_from_netdev(netdev); |
344 | struct nfp_qdisc *qdisc; |
345 | int err; |
346 | |
347 | qdisc = kzalloc(size: sizeof(*qdisc), GFP_KERNEL); |
348 | if (!qdisc) |
349 | return NULL; |
350 | |
351 | if (children) { |
352 | qdisc->children = kcalloc(n: children, size: sizeof(void *), GFP_KERNEL); |
353 | if (!qdisc->children) |
354 | goto err_free_qdisc; |
355 | } |
356 | |
357 | qdisc->netdev = netdev; |
358 | qdisc->type = type; |
359 | qdisc->parent_handle = parent_handle; |
360 | qdisc->handle = handle; |
361 | qdisc->num_children = children; |
362 | |
363 | err = radix_tree_insert(&alink->qdiscs, TC_H_MAJ(qdisc->handle), qdisc); |
364 | if (err) { |
365 | nfp_err(alink->abm->app->cpp, |
366 | "Qdisc insertion into radix tree failed: %d\n" , err); |
367 | goto err_free_child_tbl; |
368 | } |
369 | |
370 | port->tc_offload_cnt++; |
371 | return qdisc; |
372 | |
373 | err_free_child_tbl: |
374 | kfree(objp: qdisc->children); |
375 | err_free_qdisc: |
376 | kfree(objp: qdisc); |
377 | return NULL; |
378 | } |
379 | |
380 | static struct nfp_qdisc * |
381 | nfp_abm_qdisc_find(struct nfp_abm_link *alink, u32 handle) |
382 | { |
383 | return radix_tree_lookup(&alink->qdiscs, TC_H_MAJ(handle)); |
384 | } |
385 | |
386 | static int |
387 | nfp_abm_qdisc_replace(struct net_device *netdev, struct nfp_abm_link *alink, |
388 | enum nfp_qdisc_type type, u32 parent_handle, u32 handle, |
389 | unsigned int children, struct nfp_qdisc **qdisc) |
390 | { |
391 | *qdisc = nfp_abm_qdisc_find(alink, handle); |
392 | if (*qdisc) { |
393 | if (WARN_ON((*qdisc)->type != type)) |
394 | return -EINVAL; |
395 | return 1; |
396 | } |
397 | |
398 | *qdisc = nfp_abm_qdisc_alloc(netdev, alink, type, parent_handle, handle, |
399 | children); |
400 | return *qdisc ? 0 : -ENOMEM; |
401 | } |
402 | |
403 | static void |
404 | nfp_abm_qdisc_destroy(struct net_device *netdev, struct nfp_abm_link *alink, |
405 | u32 handle) |
406 | { |
407 | struct nfp_qdisc *qdisc; |
408 | |
409 | qdisc = nfp_abm_qdisc_find(alink, handle); |
410 | if (!qdisc) |
411 | return; |
412 | |
413 | /* We don't get TC_SETUP_ROOT_QDISC w/ MQ when netdev is unregistered */ |
414 | if (alink->root_qdisc == qdisc) |
415 | qdisc->use_cnt--; |
416 | |
417 | nfp_abm_qdisc_unlink_children(qdisc, start: 0, end: qdisc->num_children); |
418 | nfp_abm_qdisc_free(netdev, alink, qdisc); |
419 | |
420 | if (alink->root_qdisc == qdisc) { |
421 | alink->root_qdisc = NULL; |
422 | /* Only root change matters, other changes are acted upon on |
423 | * the graft notification. |
424 | */ |
425 | nfp_abm_qdisc_offload_update(alink); |
426 | } |
427 | } |
428 | |
429 | static int |
430 | nfp_abm_qdisc_graft(struct nfp_abm_link *alink, u32 handle, u32 child_handle, |
431 | unsigned int id) |
432 | { |
433 | struct nfp_qdisc *parent, *child; |
434 | |
435 | parent = nfp_abm_qdisc_find(alink, handle); |
436 | if (!parent) |
437 | return 0; |
438 | |
439 | if (WARN(id >= parent->num_children, |
440 | "graft child out of bound %d >= %d\n" , |
441 | id, parent->num_children)) |
442 | return -EINVAL; |
443 | |
444 | nfp_abm_qdisc_unlink_children(qdisc: parent, start: id, end: id + 1); |
445 | |
446 | child = nfp_abm_qdisc_find(alink, handle: child_handle); |
447 | if (child) |
448 | child->use_cnt++; |
449 | else |
450 | child = NFP_QDISC_UNTRACKED; |
451 | parent->children[id] = child; |
452 | |
453 | nfp_abm_qdisc_offload_update(alink); |
454 | |
455 | return 0; |
456 | } |
457 | |
458 | static void |
459 | nfp_abm_stats_calculate(struct nfp_alink_stats *new, |
460 | struct nfp_alink_stats *old, |
461 | struct gnet_stats_basic_sync *bstats, |
462 | struct gnet_stats_queue *qstats) |
463 | { |
464 | _bstats_update(bstats, bytes: new->tx_bytes - old->tx_bytes, |
465 | packets: new->tx_pkts - old->tx_pkts); |
466 | qstats->qlen += new->backlog_pkts - old->backlog_pkts; |
467 | qstats->backlog += new->backlog_bytes - old->backlog_bytes; |
468 | qstats->overlimits += new->overlimits - old->overlimits; |
469 | qstats->drops += new->drops - old->drops; |
470 | } |
471 | |
472 | static void |
473 | nfp_abm_stats_red_calculate(struct nfp_alink_xstats *new, |
474 | struct nfp_alink_xstats *old, |
475 | struct red_stats *stats) |
476 | { |
477 | stats->forced_mark += new->ecn_marked - old->ecn_marked; |
478 | stats->pdrop += new->pdrop - old->pdrop; |
479 | } |
480 | |
481 | static int |
482 | nfp_abm_gred_stats(struct nfp_abm_link *alink, u32 handle, |
483 | struct tc_gred_qopt_offload_stats *stats) |
484 | { |
485 | struct nfp_qdisc *qdisc; |
486 | unsigned int i; |
487 | |
488 | nfp_abm_stats_update(alink); |
489 | |
490 | qdisc = nfp_abm_qdisc_find(alink, handle); |
491 | if (!qdisc) |
492 | return -EOPNOTSUPP; |
493 | /* If the qdisc offload has stopped we may need to adjust the backlog |
494 | * counters back so carry on even if qdisc is not currently offloaded. |
495 | */ |
496 | |
497 | for (i = 0; i < qdisc->red.num_bands; i++) { |
498 | if (!stats->xstats[i]) |
499 | continue; |
500 | |
501 | nfp_abm_stats_calculate(new: &qdisc->red.band[i].stats, |
502 | old: &qdisc->red.band[i].prev_stats, |
503 | bstats: &stats->bstats[i], qstats: &stats->qstats[i]); |
504 | qdisc->red.band[i].prev_stats = qdisc->red.band[i].stats; |
505 | |
506 | nfp_abm_stats_red_calculate(new: &qdisc->red.band[i].xstats, |
507 | old: &qdisc->red.band[i].prev_xstats, |
508 | stats: stats->xstats[i]); |
509 | qdisc->red.band[i].prev_xstats = qdisc->red.band[i].xstats; |
510 | } |
511 | |
512 | return qdisc->offloaded ? 0 : -EOPNOTSUPP; |
513 | } |
514 | |
515 | static bool |
516 | nfp_abm_gred_check_params(struct nfp_abm_link *alink, |
517 | struct tc_gred_qopt_offload *opt) |
518 | { |
519 | struct nfp_cpp *cpp = alink->abm->app->cpp; |
520 | struct nfp_abm *abm = alink->abm; |
521 | unsigned int i; |
522 | |
523 | if (opt->set.grio_on || opt->set.wred_on) { |
524 | nfp_warn(cpp, "GRED offload failed - GRIO and WRED not supported (p:%08x h:%08x)\n" , |
525 | opt->parent, opt->handle); |
526 | return false; |
527 | } |
528 | if (opt->set.dp_def != alink->def_band) { |
529 | nfp_warn(cpp, "GRED offload failed - default band must be %d (p:%08x h:%08x)\n" , |
530 | alink->def_band, opt->parent, opt->handle); |
531 | return false; |
532 | } |
533 | if (opt->set.dp_cnt != abm->num_bands) { |
534 | nfp_warn(cpp, "GRED offload failed - band count must be %d (p:%08x h:%08x)\n" , |
535 | abm->num_bands, opt->parent, opt->handle); |
536 | return false; |
537 | } |
538 | |
539 | for (i = 0; i < abm->num_bands; i++) { |
540 | struct tc_gred_vq_qopt_offload_params *band = &opt->set.tab[i]; |
541 | |
542 | if (!band->present) |
543 | return false; |
544 | if (!band->is_ecn && !nfp_abm_has_drop(abm)) { |
545 | nfp_warn(cpp, "GRED offload failed - drop is not supported (ECN option required) (p:%08x h:%08x vq:%d)\n" , |
546 | opt->parent, opt->handle, i); |
547 | return false; |
548 | } |
549 | if (band->is_ecn && !nfp_abm_has_mark(abm)) { |
550 | nfp_warn(cpp, "GRED offload failed - ECN marking not supported (p:%08x h:%08x vq:%d)\n" , |
551 | opt->parent, opt->handle, i); |
552 | return false; |
553 | } |
554 | if (band->is_harddrop) { |
555 | nfp_warn(cpp, "GRED offload failed - harddrop is not supported (p:%08x h:%08x vq:%d)\n" , |
556 | opt->parent, opt->handle, i); |
557 | return false; |
558 | } |
559 | if (band->min != band->max) { |
560 | nfp_warn(cpp, "GRED offload failed - threshold mismatch (p:%08x h:%08x vq:%d)\n" , |
561 | opt->parent, opt->handle, i); |
562 | return false; |
563 | } |
564 | if (band->min > S32_MAX) { |
565 | nfp_warn(cpp, "GRED offload failed - threshold too large %d > %d (p:%08x h:%08x vq:%d)\n" , |
566 | band->min, S32_MAX, opt->parent, opt->handle, |
567 | i); |
568 | return false; |
569 | } |
570 | } |
571 | |
572 | return true; |
573 | } |
574 | |
575 | static int |
576 | nfp_abm_gred_replace(struct net_device *netdev, struct nfp_abm_link *alink, |
577 | struct tc_gred_qopt_offload *opt) |
578 | { |
579 | struct nfp_qdisc *qdisc; |
580 | unsigned int i; |
581 | int ret; |
582 | |
583 | ret = nfp_abm_qdisc_replace(netdev, alink, type: NFP_QDISC_GRED, parent_handle: opt->parent, |
584 | handle: opt->handle, children: 0, qdisc: &qdisc); |
585 | if (ret < 0) |
586 | return ret; |
587 | |
588 | qdisc->params_ok = nfp_abm_gred_check_params(alink, opt); |
589 | if (qdisc->params_ok) { |
590 | qdisc->red.num_bands = opt->set.dp_cnt; |
591 | for (i = 0; i < qdisc->red.num_bands; i++) { |
592 | qdisc->red.band[i].ecn = opt->set.tab[i].is_ecn; |
593 | qdisc->red.band[i].threshold = opt->set.tab[i].min; |
594 | } |
595 | } |
596 | |
597 | if (qdisc->use_cnt) |
598 | nfp_abm_qdisc_offload_update(alink); |
599 | |
600 | return 0; |
601 | } |
602 | |
603 | int nfp_abm_setup_tc_gred(struct net_device *netdev, struct nfp_abm_link *alink, |
604 | struct tc_gred_qopt_offload *opt) |
605 | { |
606 | switch (opt->command) { |
607 | case TC_GRED_REPLACE: |
608 | return nfp_abm_gred_replace(netdev, alink, opt); |
609 | case TC_GRED_DESTROY: |
610 | nfp_abm_qdisc_destroy(netdev, alink, handle: opt->handle); |
611 | return 0; |
612 | case TC_GRED_STATS: |
613 | return nfp_abm_gred_stats(alink, handle: opt->handle, stats: &opt->stats); |
614 | default: |
615 | return -EOPNOTSUPP; |
616 | } |
617 | } |
618 | |
619 | static int |
620 | nfp_abm_red_xstats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt) |
621 | { |
622 | struct nfp_qdisc *qdisc; |
623 | |
624 | nfp_abm_stats_update(alink); |
625 | |
626 | qdisc = nfp_abm_qdisc_find(alink, handle: opt->handle); |
627 | if (!qdisc || !qdisc->offloaded) |
628 | return -EOPNOTSUPP; |
629 | |
630 | nfp_abm_stats_red_calculate(new: &qdisc->red.band[0].xstats, |
631 | old: &qdisc->red.band[0].prev_xstats, |
632 | stats: opt->xstats); |
633 | qdisc->red.band[0].prev_xstats = qdisc->red.band[0].xstats; |
634 | return 0; |
635 | } |
636 | |
637 | static int |
638 | nfp_abm_red_stats(struct nfp_abm_link *alink, u32 handle, |
639 | struct tc_qopt_offload_stats *stats) |
640 | { |
641 | struct nfp_qdisc *qdisc; |
642 | |
643 | nfp_abm_stats_update(alink); |
644 | |
645 | qdisc = nfp_abm_qdisc_find(alink, handle); |
646 | if (!qdisc) |
647 | return -EOPNOTSUPP; |
648 | /* If the qdisc offload has stopped we may need to adjust the backlog |
649 | * counters back so carry on even if qdisc is not currently offloaded. |
650 | */ |
651 | |
652 | nfp_abm_stats_calculate(new: &qdisc->red.band[0].stats, |
653 | old: &qdisc->red.band[0].prev_stats, |
654 | bstats: stats->bstats, qstats: stats->qstats); |
655 | qdisc->red.band[0].prev_stats = qdisc->red.band[0].stats; |
656 | |
657 | return qdisc->offloaded ? 0 : -EOPNOTSUPP; |
658 | } |
659 | |
660 | static bool |
661 | nfp_abm_red_check_params(struct nfp_abm_link *alink, |
662 | struct tc_red_qopt_offload *opt) |
663 | { |
664 | struct nfp_cpp *cpp = alink->abm->app->cpp; |
665 | struct nfp_abm *abm = alink->abm; |
666 | |
667 | if (!opt->set.is_ecn && !nfp_abm_has_drop(abm)) { |
668 | nfp_warn(cpp, "RED offload failed - drop is not supported (ECN option required) (p:%08x h:%08x)\n" , |
669 | opt->parent, opt->handle); |
670 | return false; |
671 | } |
672 | if (opt->set.is_ecn && !nfp_abm_has_mark(abm)) { |
673 | nfp_warn(cpp, "RED offload failed - ECN marking not supported (p:%08x h:%08x)\n" , |
674 | opt->parent, opt->handle); |
675 | return false; |
676 | } |
677 | if (opt->set.is_harddrop) { |
678 | nfp_warn(cpp, "RED offload failed - harddrop is not supported (p:%08x h:%08x)\n" , |
679 | opt->parent, opt->handle); |
680 | return false; |
681 | } |
682 | if (opt->set.min != opt->set.max) { |
683 | nfp_warn(cpp, "RED offload failed - unsupported min/max parameters (p:%08x h:%08x)\n" , |
684 | opt->parent, opt->handle); |
685 | return false; |
686 | } |
687 | if (opt->set.min > NFP_ABM_LVL_INFINITY) { |
688 | nfp_warn(cpp, "RED offload failed - threshold too large %d > %d (p:%08x h:%08x)\n" , |
689 | opt->set.min, NFP_ABM_LVL_INFINITY, opt->parent, |
690 | opt->handle); |
691 | return false; |
692 | } |
693 | |
694 | return true; |
695 | } |
696 | |
697 | static int |
698 | nfp_abm_red_replace(struct net_device *netdev, struct nfp_abm_link *alink, |
699 | struct tc_red_qopt_offload *opt) |
700 | { |
701 | struct nfp_qdisc *qdisc; |
702 | int ret; |
703 | |
704 | ret = nfp_abm_qdisc_replace(netdev, alink, type: NFP_QDISC_RED, parent_handle: opt->parent, |
705 | handle: opt->handle, children: 1, qdisc: &qdisc); |
706 | if (ret < 0) |
707 | return ret; |
708 | |
709 | /* If limit != 0 child gets reset */ |
710 | if (opt->set.limit) { |
711 | if (nfp_abm_qdisc_child_valid(qdisc, id: 0)) |
712 | qdisc->children[0]->use_cnt--; |
713 | qdisc->children[0] = NULL; |
714 | } else { |
715 | /* Qdisc was just allocated without a limit will use noop_qdisc, |
716 | * i.e. a block hole. |
717 | */ |
718 | if (!ret) |
719 | qdisc->children[0] = NFP_QDISC_UNTRACKED; |
720 | } |
721 | |
722 | qdisc->params_ok = nfp_abm_red_check_params(alink, opt); |
723 | if (qdisc->params_ok) { |
724 | qdisc->red.num_bands = 1; |
725 | qdisc->red.band[0].ecn = opt->set.is_ecn; |
726 | qdisc->red.band[0].threshold = opt->set.min; |
727 | } |
728 | |
729 | if (qdisc->use_cnt == 1) |
730 | nfp_abm_qdisc_offload_update(alink); |
731 | |
732 | return 0; |
733 | } |
734 | |
735 | int nfp_abm_setup_tc_red(struct net_device *netdev, struct nfp_abm_link *alink, |
736 | struct tc_red_qopt_offload *opt) |
737 | { |
738 | switch (opt->command) { |
739 | case TC_RED_REPLACE: |
740 | return nfp_abm_red_replace(netdev, alink, opt); |
741 | case TC_RED_DESTROY: |
742 | nfp_abm_qdisc_destroy(netdev, alink, handle: opt->handle); |
743 | return 0; |
744 | case TC_RED_STATS: |
745 | return nfp_abm_red_stats(alink, handle: opt->handle, stats: &opt->stats); |
746 | case TC_RED_XSTATS: |
747 | return nfp_abm_red_xstats(alink, opt); |
748 | case TC_RED_GRAFT: |
749 | return nfp_abm_qdisc_graft(alink, handle: opt->handle, |
750 | child_handle: opt->child_handle, id: 0); |
751 | default: |
752 | return -EOPNOTSUPP; |
753 | } |
754 | } |
755 | |
756 | static int |
757 | nfp_abm_mq_create(struct net_device *netdev, struct nfp_abm_link *alink, |
758 | struct tc_mq_qopt_offload *opt) |
759 | { |
760 | struct nfp_qdisc *qdisc; |
761 | int ret; |
762 | |
763 | ret = nfp_abm_qdisc_replace(netdev, alink, type: NFP_QDISC_MQ, |
764 | TC_H_ROOT, handle: opt->handle, children: alink->total_queues, |
765 | qdisc: &qdisc); |
766 | if (ret < 0) |
767 | return ret; |
768 | |
769 | qdisc->params_ok = true; |
770 | qdisc->offloaded = true; |
771 | nfp_abm_qdisc_offload_update(alink); |
772 | return 0; |
773 | } |
774 | |
775 | static int |
776 | nfp_abm_mq_stats(struct nfp_abm_link *alink, u32 handle, |
777 | struct tc_qopt_offload_stats *stats) |
778 | { |
779 | struct nfp_qdisc *qdisc, *red; |
780 | unsigned int i, j; |
781 | |
782 | qdisc = nfp_abm_qdisc_find(alink, handle); |
783 | if (!qdisc) |
784 | return -EOPNOTSUPP; |
785 | |
786 | nfp_abm_stats_update(alink); |
787 | |
788 | /* MQ stats are summed over the children in the core, so we need |
789 | * to add up the unreported child values. |
790 | */ |
791 | memset(&qdisc->mq.stats, 0, sizeof(qdisc->mq.stats)); |
792 | memset(&qdisc->mq.prev_stats, 0, sizeof(qdisc->mq.prev_stats)); |
793 | |
794 | for (i = 0; i < qdisc->num_children; i++) { |
795 | if (!nfp_abm_qdisc_child_valid(qdisc, id: i)) |
796 | continue; |
797 | |
798 | if (!nfp_abm_qdisc_is_red(qdisc: qdisc->children[i])) |
799 | continue; |
800 | red = qdisc->children[i]; |
801 | |
802 | for (j = 0; j < red->red.num_bands; j++) { |
803 | nfp_abm_stats_propagate(parent: &qdisc->mq.stats, |
804 | child: &red->red.band[j].stats); |
805 | nfp_abm_stats_propagate(parent: &qdisc->mq.prev_stats, |
806 | child: &red->red.band[j].prev_stats); |
807 | } |
808 | } |
809 | |
810 | nfp_abm_stats_calculate(new: &qdisc->mq.stats, old: &qdisc->mq.prev_stats, |
811 | bstats: stats->bstats, qstats: stats->qstats); |
812 | |
813 | return qdisc->offloaded ? 0 : -EOPNOTSUPP; |
814 | } |
815 | |
816 | int nfp_abm_setup_tc_mq(struct net_device *netdev, struct nfp_abm_link *alink, |
817 | struct tc_mq_qopt_offload *opt) |
818 | { |
819 | switch (opt->command) { |
820 | case TC_MQ_CREATE: |
821 | return nfp_abm_mq_create(netdev, alink, opt); |
822 | case TC_MQ_DESTROY: |
823 | nfp_abm_qdisc_destroy(netdev, alink, handle: opt->handle); |
824 | return 0; |
825 | case TC_MQ_STATS: |
826 | return nfp_abm_mq_stats(alink, handle: opt->handle, stats: &opt->stats); |
827 | case TC_MQ_GRAFT: |
828 | return nfp_abm_qdisc_graft(alink, handle: opt->handle, |
829 | child_handle: opt->graft_params.child_handle, |
830 | id: opt->graft_params.queue); |
831 | default: |
832 | return -EOPNOTSUPP; |
833 | } |
834 | } |
835 | |
836 | int nfp_abm_setup_root(struct net_device *netdev, struct nfp_abm_link *alink, |
837 | struct tc_root_qopt_offload *opt) |
838 | { |
839 | if (opt->ingress) |
840 | return -EOPNOTSUPP; |
841 | if (alink->root_qdisc) |
842 | alink->root_qdisc->use_cnt--; |
843 | alink->root_qdisc = nfp_abm_qdisc_find(alink, handle: opt->handle); |
844 | if (alink->root_qdisc) |
845 | alink->root_qdisc->use_cnt++; |
846 | |
847 | nfp_abm_qdisc_offload_update(alink); |
848 | |
849 | return 0; |
850 | } |
851 | |