1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2019 Netronome Systems, Inc. */ |
3 | |
4 | #include <linux/hash.h> |
5 | #include <linux/hashtable.h> |
6 | #include <linux/jhash.h> |
7 | #include <linux/math64.h> |
8 | #include <linux/vmalloc.h> |
9 | #include <net/pkt_cls.h> |
10 | #include <net/pkt_sched.h> |
11 | |
12 | #include "cmsg.h" |
13 | #include "main.h" |
14 | #include "../nfp_port.h" |
15 | |
16 | #define NFP_FL_QOS_UPDATE msecs_to_jiffies(1000) |
17 | #define NFP_FL_QOS_PPS BIT(15) |
18 | #define NFP_FL_QOS_METER BIT(10) |
19 | |
20 | struct nfp_police_cfg_head { |
21 | __be32 flags_opts; |
22 | union { |
23 | __be32 meter_id; |
24 | __be32 port; |
25 | }; |
26 | }; |
27 | |
28 | enum NFP_FL_QOS_TYPES { |
29 | NFP_FL_QOS_TYPE_BPS, |
30 | NFP_FL_QOS_TYPE_PPS, |
31 | NFP_FL_QOS_TYPE_MAX, |
32 | }; |
33 | |
34 | /* Police cmsg for configuring a trTCM traffic conditioner (8W/32B) |
35 | * See RFC 2698 for more details. |
36 | * ---------------------------------------------------------------- |
37 | * 3 2 1 |
38 | * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 |
39 | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
40 | * | Reserved |p| Reserved | |
41 | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
42 | * | Port Ingress | |
43 | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
44 | * | Token Bucket Peak | |
45 | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
46 | * | Token Bucket Committed | |
47 | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
48 | * | Peak Burst Size | |
49 | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
50 | * | Committed Burst Size | |
51 | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
52 | * | Peak Information Rate | |
53 | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
54 | * | Committed Information Rate | |
55 | * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
56 | * Word[0](FLag options): |
57 | * [15] p(pps) 1 for pps, 0 for bps |
58 | * |
59 | * Meter control message |
60 | * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 |
61 | * +-------------------------------+-+---+-----+-+---------+-+---+-+ |
62 | * | Reserved |p| Y |TYPE |E|TSHFV |P| PC|R| |
63 | * +-------------------------------+-+---+-----+-+---------+-+---+-+ |
64 | * | meter ID | |
65 | * +-------------------------------+-------------------------------+ |
66 | * |
67 | */ |
68 | struct nfp_police_config { |
69 | struct nfp_police_cfg_head head; |
70 | __be32 bkt_tkn_p; |
71 | __be32 bkt_tkn_c; |
72 | __be32 pbs; |
73 | __be32 cbs; |
74 | __be32 pir; |
75 | __be32 cir; |
76 | }; |
77 | |
78 | struct nfp_police_stats_reply { |
79 | struct nfp_police_cfg_head head; |
80 | __be64 pass_bytes; |
81 | __be64 pass_pkts; |
82 | __be64 drop_bytes; |
83 | __be64 drop_pkts; |
84 | }; |
85 | |
86 | int nfp_flower_offload_one_police(struct nfp_app *app, bool ingress, |
87 | bool pps, u32 id, u32 rate, u32 burst) |
88 | { |
89 | struct nfp_police_config *config; |
90 | struct sk_buff *skb; |
91 | |
92 | skb = nfp_flower_cmsg_alloc(app, size: sizeof(struct nfp_police_config), |
93 | type: NFP_FLOWER_CMSG_TYPE_QOS_MOD, GFP_KERNEL); |
94 | if (!skb) |
95 | return -ENOMEM; |
96 | |
97 | config = nfp_flower_cmsg_get_data(skb); |
98 | memset(config, 0, sizeof(struct nfp_police_config)); |
99 | if (pps) |
100 | config->head.flags_opts |= cpu_to_be32(NFP_FL_QOS_PPS); |
101 | if (!ingress) |
102 | config->head.flags_opts |= cpu_to_be32(NFP_FL_QOS_METER); |
103 | |
104 | if (ingress) |
105 | config->head.port = cpu_to_be32(id); |
106 | else |
107 | config->head.meter_id = cpu_to_be32(id); |
108 | |
109 | config->bkt_tkn_p = cpu_to_be32(burst); |
110 | config->bkt_tkn_c = cpu_to_be32(burst); |
111 | config->pbs = cpu_to_be32(burst); |
112 | config->cbs = cpu_to_be32(burst); |
113 | config->pir = cpu_to_be32(rate); |
114 | config->cir = cpu_to_be32(rate); |
115 | nfp_ctrl_tx(nn: app->ctrl, skb); |
116 | |
117 | return 0; |
118 | } |
119 | |
120 | static int nfp_policer_validate(const struct flow_action *action, |
121 | const struct flow_action_entry *act, |
122 | struct netlink_ext_ack *extack, |
123 | bool ingress) |
124 | { |
125 | if (act->police.exceed.act_id != FLOW_ACTION_DROP) { |
126 | NL_SET_ERR_MSG_MOD(extack, |
127 | "Offload not supported when exceed action is not drop" ); |
128 | return -EOPNOTSUPP; |
129 | } |
130 | |
131 | if (ingress) { |
132 | if (act->police.notexceed.act_id != FLOW_ACTION_CONTINUE && |
133 | act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { |
134 | NL_SET_ERR_MSG_MOD(extack, |
135 | "Offload not supported when conform action is not continue or ok" ); |
136 | return -EOPNOTSUPP; |
137 | } |
138 | } else { |
139 | if (act->police.notexceed.act_id != FLOW_ACTION_PIPE && |
140 | act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { |
141 | NL_SET_ERR_MSG_MOD(extack, |
142 | "Offload not supported when conform action is not pipe or ok" ); |
143 | return -EOPNOTSUPP; |
144 | } |
145 | } |
146 | |
147 | if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT && |
148 | !flow_action_is_last_entry(action, entry: act)) { |
149 | NL_SET_ERR_MSG_MOD(extack, |
150 | "Offload not supported when conform action is ok, but action is not last" ); |
151 | return -EOPNOTSUPP; |
152 | } |
153 | |
154 | if (act->police.peakrate_bytes_ps || |
155 | act->police.avrate || act->police.overhead) { |
156 | NL_SET_ERR_MSG_MOD(extack, |
157 | "Offload not supported when peakrate/avrate/overhead is configured" ); |
158 | return -EOPNOTSUPP; |
159 | } |
160 | |
161 | return 0; |
162 | } |
163 | |
164 | static int |
165 | nfp_flower_install_rate_limiter(struct nfp_app *app, struct net_device *netdev, |
166 | struct tc_cls_matchall_offload *flow, |
167 | struct netlink_ext_ack *extack) |
168 | { |
169 | struct flow_action_entry *paction = &flow->rule->action.entries[0]; |
170 | u32 action_num = flow->rule->action.num_entries; |
171 | struct nfp_flower_priv *fl_priv = app->priv; |
172 | struct flow_action_entry *action = NULL; |
173 | struct nfp_flower_repr_priv *repr_priv; |
174 | u32 netdev_port_id, i; |
175 | struct nfp_repr *repr; |
176 | bool pps_support; |
177 | u32 bps_num = 0; |
178 | u32 pps_num = 0; |
179 | u32 burst; |
180 | bool pps; |
181 | u64 rate; |
182 | int err; |
183 | |
184 | if (!nfp_netdev_is_nfp_repr(netdev)) { |
185 | NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port" ); |
186 | return -EOPNOTSUPP; |
187 | } |
188 | repr = netdev_priv(dev: netdev); |
189 | repr_priv = repr->app_priv; |
190 | netdev_port_id = nfp_repr_get_port_id(netdev); |
191 | pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS); |
192 | |
193 | if (repr_priv->block_shared) { |
194 | NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on shared blocks" ); |
195 | return -EOPNOTSUPP; |
196 | } |
197 | |
198 | if (repr->port->type != NFP_PORT_VF_PORT) { |
199 | NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on non-VF ports" ); |
200 | return -EOPNOTSUPP; |
201 | } |
202 | |
203 | if (pps_support) { |
204 | if (action_num > 2 || action_num == 0) { |
205 | NL_SET_ERR_MSG_MOD(extack, |
206 | "unsupported offload: qos rate limit offload only support action number 1 or 2" ); |
207 | return -EOPNOTSUPP; |
208 | } |
209 | } else { |
210 | if (!flow_offload_has_one_action(action: &flow->rule->action)) { |
211 | NL_SET_ERR_MSG_MOD(extack, |
212 | "unsupported offload: qos rate limit offload requires a single action" ); |
213 | return -EOPNOTSUPP; |
214 | } |
215 | } |
216 | |
217 | if (flow->common.prio != 1) { |
218 | NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload requires highest priority" ); |
219 | return -EOPNOTSUPP; |
220 | } |
221 | |
222 | for (i = 0 ; i < action_num; i++) { |
223 | action = paction + i; |
224 | if (action->id != FLOW_ACTION_POLICE) { |
225 | NL_SET_ERR_MSG_MOD(extack, |
226 | "unsupported offload: qos rate limit offload requires police action" ); |
227 | return -EOPNOTSUPP; |
228 | } |
229 | |
230 | err = nfp_policer_validate(action: &flow->rule->action, act: action, extack, ingress: true); |
231 | if (err) |
232 | return err; |
233 | |
234 | if (action->police.rate_bytes_ps > 0) { |
235 | if (bps_num++) { |
236 | NL_SET_ERR_MSG_MOD(extack, |
237 | "unsupported offload: qos rate limit offload only support one BPS action" ); |
238 | return -EOPNOTSUPP; |
239 | } |
240 | } |
241 | if (action->police.rate_pkt_ps > 0) { |
242 | if (!pps_support) { |
243 | NL_SET_ERR_MSG_MOD(extack, |
244 | "unsupported offload: FW does not support PPS action" ); |
245 | return -EOPNOTSUPP; |
246 | } |
247 | if (pps_num++) { |
248 | NL_SET_ERR_MSG_MOD(extack, |
249 | "unsupported offload: qos rate limit offload only support one PPS action" ); |
250 | return -EOPNOTSUPP; |
251 | } |
252 | } |
253 | } |
254 | |
255 | for (i = 0 ; i < action_num; i++) { |
256 | /* Set QoS data for this interface */ |
257 | action = paction + i; |
258 | if (action->police.rate_bytes_ps > 0) { |
259 | rate = action->police.rate_bytes_ps; |
260 | burst = action->police.burst; |
261 | } else if (action->police.rate_pkt_ps > 0) { |
262 | rate = action->police.rate_pkt_ps; |
263 | burst = action->police.burst_pkt; |
264 | } else { |
265 | NL_SET_ERR_MSG_MOD(extack, |
266 | "unsupported offload: qos rate limit is not BPS or PPS" ); |
267 | continue; |
268 | } |
269 | |
270 | if (rate != 0) { |
271 | pps = false; |
272 | if (action->police.rate_pkt_ps > 0) |
273 | pps = true; |
274 | nfp_flower_offload_one_police(app: repr->app, ingress: true, |
275 | pps, id: netdev_port_id, |
276 | rate, burst); |
277 | } |
278 | } |
279 | repr_priv->qos_table.netdev_port_id = netdev_port_id; |
280 | fl_priv->qos_rate_limiters++; |
281 | if (fl_priv->qos_rate_limiters == 1) |
282 | schedule_delayed_work(dwork: &fl_priv->qos_stats_work, |
283 | NFP_FL_QOS_UPDATE); |
284 | |
285 | return 0; |
286 | } |
287 | |
288 | static int |
289 | nfp_flower_remove_rate_limiter(struct nfp_app *app, struct net_device *netdev, |
290 | struct tc_cls_matchall_offload *flow, |
291 | struct netlink_ext_ack *extack) |
292 | { |
293 | struct nfp_flower_priv *fl_priv = app->priv; |
294 | struct nfp_flower_repr_priv *repr_priv; |
295 | struct nfp_police_config *config; |
296 | u32 netdev_port_id, i; |
297 | struct nfp_repr *repr; |
298 | struct sk_buff *skb; |
299 | bool pps_support; |
300 | |
301 | if (!nfp_netdev_is_nfp_repr(netdev)) { |
302 | NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port" ); |
303 | return -EOPNOTSUPP; |
304 | } |
305 | repr = netdev_priv(dev: netdev); |
306 | |
307 | netdev_port_id = nfp_repr_get_port_id(netdev); |
308 | repr_priv = repr->app_priv; |
309 | pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS); |
310 | |
311 | if (!repr_priv->qos_table.netdev_port_id) { |
312 | NL_SET_ERR_MSG_MOD(extack, "unsupported offload: cannot remove qos entry that does not exist" ); |
313 | return -EOPNOTSUPP; |
314 | } |
315 | |
316 | memset(&repr_priv->qos_table, 0, sizeof(struct nfp_fl_qos)); |
317 | fl_priv->qos_rate_limiters--; |
318 | if (!fl_priv->qos_rate_limiters) |
319 | cancel_delayed_work_sync(dwork: &fl_priv->qos_stats_work); |
320 | for (i = 0 ; i < NFP_FL_QOS_TYPE_MAX; i++) { |
321 | if (i == NFP_FL_QOS_TYPE_PPS && !pps_support) |
322 | break; |
323 | /* 0:bps 1:pps |
324 | * Clear QoS data for this interface. |
325 | * There is no need to check if a specific QOS_TYPE was |
326 | * configured as the firmware handles clearing a QoS entry |
327 | * safely, even if it wasn't explicitly added. |
328 | */ |
329 | skb = nfp_flower_cmsg_alloc(app: repr->app, size: sizeof(struct nfp_police_config), |
330 | type: NFP_FLOWER_CMSG_TYPE_QOS_DEL, GFP_KERNEL); |
331 | if (!skb) |
332 | return -ENOMEM; |
333 | |
334 | config = nfp_flower_cmsg_get_data(skb); |
335 | memset(config, 0, sizeof(struct nfp_police_config)); |
336 | if (i == NFP_FL_QOS_TYPE_PPS) |
337 | config->head.flags_opts = cpu_to_be32(NFP_FL_QOS_PPS); |
338 | config->head.port = cpu_to_be32(netdev_port_id); |
339 | nfp_ctrl_tx(nn: repr->app->ctrl, skb); |
340 | } |
341 | |
342 | return 0; |
343 | } |
344 | |
345 | void nfp_flower_stats_rlim_reply(struct nfp_app *app, struct sk_buff *skb) |
346 | { |
347 | struct nfp_flower_priv *fl_priv = app->priv; |
348 | struct nfp_flower_repr_priv *repr_priv; |
349 | struct nfp_police_stats_reply *msg; |
350 | struct nfp_stat_pair *curr_stats; |
351 | struct nfp_stat_pair *prev_stats; |
352 | struct net_device *netdev; |
353 | struct nfp_repr *repr; |
354 | u32 netdev_port_id; |
355 | |
356 | msg = nfp_flower_cmsg_get_data(skb); |
357 | if (be32_to_cpu(msg->head.flags_opts) & NFP_FL_QOS_METER) |
358 | return nfp_act_stats_reply(app, pmsg: msg); |
359 | |
360 | netdev_port_id = be32_to_cpu(msg->head.port); |
361 | rcu_read_lock(); |
362 | netdev = nfp_app_dev_get(app, id: netdev_port_id, NULL); |
363 | if (!netdev) |
364 | goto exit_unlock_rcu; |
365 | |
366 | repr = netdev_priv(dev: netdev); |
367 | repr_priv = repr->app_priv; |
368 | curr_stats = &repr_priv->qos_table.curr_stats; |
369 | prev_stats = &repr_priv->qos_table.prev_stats; |
370 | |
371 | spin_lock_bh(lock: &fl_priv->qos_stats_lock); |
372 | curr_stats->pkts = be64_to_cpu(msg->pass_pkts) + |
373 | be64_to_cpu(msg->drop_pkts); |
374 | curr_stats->bytes = be64_to_cpu(msg->pass_bytes) + |
375 | be64_to_cpu(msg->drop_bytes); |
376 | |
377 | if (!repr_priv->qos_table.last_update) { |
378 | prev_stats->pkts = curr_stats->pkts; |
379 | prev_stats->bytes = curr_stats->bytes; |
380 | } |
381 | |
382 | repr_priv->qos_table.last_update = jiffies; |
383 | spin_unlock_bh(lock: &fl_priv->qos_stats_lock); |
384 | |
385 | exit_unlock_rcu: |
386 | rcu_read_unlock(); |
387 | } |
388 | |
389 | static void |
390 | nfp_flower_stats_rlim_request(struct nfp_flower_priv *fl_priv, |
391 | u32 id, bool ingress) |
392 | { |
393 | struct nfp_police_cfg_head *head; |
394 | struct sk_buff *skb; |
395 | |
396 | skb = nfp_flower_cmsg_alloc(app: fl_priv->app, |
397 | size: sizeof(struct nfp_police_cfg_head), |
398 | type: NFP_FLOWER_CMSG_TYPE_QOS_STATS, |
399 | GFP_ATOMIC); |
400 | if (!skb) |
401 | return; |
402 | head = nfp_flower_cmsg_get_data(skb); |
403 | |
404 | memset(head, 0, sizeof(struct nfp_police_cfg_head)); |
405 | if (ingress) { |
406 | head->port = cpu_to_be32(id); |
407 | } else { |
408 | head->flags_opts = cpu_to_be32(NFP_FL_QOS_METER); |
409 | head->meter_id = cpu_to_be32(id); |
410 | } |
411 | |
412 | nfp_ctrl_tx(nn: fl_priv->app->ctrl, skb); |
413 | } |
414 | |
415 | static void |
416 | nfp_flower_stats_rlim_request_all(struct nfp_flower_priv *fl_priv) |
417 | { |
418 | struct nfp_reprs *repr_set; |
419 | int i; |
420 | |
421 | rcu_read_lock(); |
422 | repr_set = rcu_dereference(fl_priv->app->reprs[NFP_REPR_TYPE_VF]); |
423 | if (!repr_set) |
424 | goto exit_unlock_rcu; |
425 | |
426 | for (i = 0; i < repr_set->num_reprs; i++) { |
427 | struct net_device *netdev; |
428 | |
429 | netdev = rcu_dereference(repr_set->reprs[i]); |
430 | if (netdev) { |
431 | struct nfp_repr *priv = netdev_priv(dev: netdev); |
432 | struct nfp_flower_repr_priv *repr_priv; |
433 | u32 netdev_port_id; |
434 | |
435 | repr_priv = priv->app_priv; |
436 | netdev_port_id = repr_priv->qos_table.netdev_port_id; |
437 | if (!netdev_port_id) |
438 | continue; |
439 | |
440 | nfp_flower_stats_rlim_request(fl_priv, |
441 | id: netdev_port_id, ingress: true); |
442 | } |
443 | } |
444 | |
445 | exit_unlock_rcu: |
446 | rcu_read_unlock(); |
447 | } |
448 | |
449 | static void update_stats_cache(struct work_struct *work) |
450 | { |
451 | struct delayed_work *delayed_work; |
452 | struct nfp_flower_priv *fl_priv; |
453 | |
454 | delayed_work = to_delayed_work(work); |
455 | fl_priv = container_of(delayed_work, struct nfp_flower_priv, |
456 | qos_stats_work); |
457 | |
458 | nfp_flower_stats_rlim_request_all(fl_priv); |
459 | nfp_flower_stats_meter_request_all(fl_priv); |
460 | |
461 | schedule_delayed_work(dwork: &fl_priv->qos_stats_work, NFP_FL_QOS_UPDATE); |
462 | } |
463 | |
464 | static int |
465 | nfp_flower_stats_rate_limiter(struct nfp_app *app, struct net_device *netdev, |
466 | struct tc_cls_matchall_offload *flow, |
467 | struct netlink_ext_ack *extack) |
468 | { |
469 | struct nfp_flower_priv *fl_priv = app->priv; |
470 | struct nfp_flower_repr_priv *repr_priv; |
471 | struct nfp_stat_pair *curr_stats; |
472 | struct nfp_stat_pair *prev_stats; |
473 | u64 diff_bytes, diff_pkts; |
474 | struct nfp_repr *repr; |
475 | |
476 | if (!nfp_netdev_is_nfp_repr(netdev)) { |
477 | NL_SET_ERR_MSG_MOD(extack, "unsupported offload: qos rate limit offload not supported on higher level port" ); |
478 | return -EOPNOTSUPP; |
479 | } |
480 | repr = netdev_priv(dev: netdev); |
481 | |
482 | repr_priv = repr->app_priv; |
483 | if (!repr_priv->qos_table.netdev_port_id) { |
484 | NL_SET_ERR_MSG_MOD(extack, "unsupported offload: cannot find qos entry for stats update" ); |
485 | return -EOPNOTSUPP; |
486 | } |
487 | |
488 | spin_lock_bh(lock: &fl_priv->qos_stats_lock); |
489 | curr_stats = &repr_priv->qos_table.curr_stats; |
490 | prev_stats = &repr_priv->qos_table.prev_stats; |
491 | diff_pkts = curr_stats->pkts - prev_stats->pkts; |
492 | diff_bytes = curr_stats->bytes - prev_stats->bytes; |
493 | prev_stats->pkts = curr_stats->pkts; |
494 | prev_stats->bytes = curr_stats->bytes; |
495 | spin_unlock_bh(lock: &fl_priv->qos_stats_lock); |
496 | |
497 | flow_stats_update(flow_stats: &flow->stats, bytes: diff_bytes, pkts: diff_pkts, drops: 0, |
498 | lastused: repr_priv->qos_table.last_update, |
499 | used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED); |
500 | return 0; |
501 | } |
502 | |
503 | void nfp_flower_qos_init(struct nfp_app *app) |
504 | { |
505 | struct nfp_flower_priv *fl_priv = app->priv; |
506 | |
507 | spin_lock_init(&fl_priv->qos_stats_lock); |
508 | mutex_init(&fl_priv->meter_stats_lock); |
509 | nfp_init_meter_table(app); |
510 | |
511 | INIT_DELAYED_WORK(&fl_priv->qos_stats_work, &update_stats_cache); |
512 | } |
513 | |
514 | void nfp_flower_qos_cleanup(struct nfp_app *app) |
515 | { |
516 | struct nfp_flower_priv *fl_priv = app->priv; |
517 | |
518 | cancel_delayed_work_sync(dwork: &fl_priv->qos_stats_work); |
519 | } |
520 | |
521 | int nfp_flower_setup_qos_offload(struct nfp_app *app, struct net_device *netdev, |
522 | struct tc_cls_matchall_offload *flow) |
523 | { |
524 | struct netlink_ext_ack *extack = flow->common.extack; |
525 | struct nfp_flower_priv *fl_priv = app->priv; |
526 | int ret; |
527 | |
528 | if (!(fl_priv->flower_ext_feats & NFP_FL_FEATS_VF_RLIM)) { |
529 | NL_SET_ERR_MSG_MOD(extack, "unsupported offload: loaded firmware does not support qos rate limit offload" ); |
530 | return -EOPNOTSUPP; |
531 | } |
532 | |
533 | mutex_lock(&fl_priv->nfp_fl_lock); |
534 | switch (flow->command) { |
535 | case TC_CLSMATCHALL_REPLACE: |
536 | ret = nfp_flower_install_rate_limiter(app, netdev, flow, extack); |
537 | break; |
538 | case TC_CLSMATCHALL_DESTROY: |
539 | ret = nfp_flower_remove_rate_limiter(app, netdev, flow, extack); |
540 | break; |
541 | case TC_CLSMATCHALL_STATS: |
542 | ret = nfp_flower_stats_rate_limiter(app, netdev, flow, extack); |
543 | break; |
544 | default: |
545 | ret = -EOPNOTSUPP; |
546 | break; |
547 | } |
548 | mutex_unlock(lock: &fl_priv->nfp_fl_lock); |
549 | |
550 | return ret; |
551 | } |
552 | |
553 | /* Offload tc action, currently only for tc police */ |
554 | |
555 | static const struct rhashtable_params stats_meter_table_params = { |
556 | .key_offset = offsetof(struct nfp_meter_entry, meter_id), |
557 | .head_offset = offsetof(struct nfp_meter_entry, ht_node), |
558 | .key_len = sizeof(u32), |
559 | }; |
560 | |
561 | struct nfp_meter_entry * |
562 | nfp_flower_search_meter_entry(struct nfp_app *app, u32 meter_id) |
563 | { |
564 | struct nfp_flower_priv *priv = app->priv; |
565 | |
566 | return rhashtable_lookup_fast(ht: &priv->meter_table, key: &meter_id, |
567 | params: stats_meter_table_params); |
568 | } |
569 | |
570 | static struct nfp_meter_entry * |
571 | nfp_flower_add_meter_entry(struct nfp_app *app, u32 meter_id) |
572 | { |
573 | struct nfp_meter_entry *meter_entry = NULL; |
574 | struct nfp_flower_priv *priv = app->priv; |
575 | |
576 | meter_entry = rhashtable_lookup_fast(ht: &priv->meter_table, |
577 | key: &meter_id, |
578 | params: stats_meter_table_params); |
579 | if (meter_entry) |
580 | return meter_entry; |
581 | |
582 | meter_entry = kzalloc(size: sizeof(*meter_entry), GFP_KERNEL); |
583 | if (!meter_entry) |
584 | return NULL; |
585 | |
586 | meter_entry->meter_id = meter_id; |
587 | meter_entry->used = jiffies; |
588 | if (rhashtable_insert_fast(ht: &priv->meter_table, obj: &meter_entry->ht_node, |
589 | params: stats_meter_table_params)) { |
590 | kfree(objp: meter_entry); |
591 | return NULL; |
592 | } |
593 | |
594 | priv->qos_rate_limiters++; |
595 | if (priv->qos_rate_limiters == 1) |
596 | schedule_delayed_work(dwork: &priv->qos_stats_work, |
597 | NFP_FL_QOS_UPDATE); |
598 | |
599 | return meter_entry; |
600 | } |
601 | |
602 | static void nfp_flower_del_meter_entry(struct nfp_app *app, u32 meter_id) |
603 | { |
604 | struct nfp_meter_entry *meter_entry = NULL; |
605 | struct nfp_flower_priv *priv = app->priv; |
606 | |
607 | meter_entry = rhashtable_lookup_fast(ht: &priv->meter_table, key: &meter_id, |
608 | params: stats_meter_table_params); |
609 | if (!meter_entry) |
610 | return; |
611 | |
612 | rhashtable_remove_fast(ht: &priv->meter_table, |
613 | obj: &meter_entry->ht_node, |
614 | params: stats_meter_table_params); |
615 | kfree(objp: meter_entry); |
616 | priv->qos_rate_limiters--; |
617 | if (!priv->qos_rate_limiters) |
618 | cancel_delayed_work_sync(dwork: &priv->qos_stats_work); |
619 | } |
620 | |
621 | int nfp_flower_setup_meter_entry(struct nfp_app *app, |
622 | const struct flow_action_entry *action, |
623 | enum nfp_meter_op op, |
624 | u32 meter_id) |
625 | { |
626 | struct nfp_flower_priv *fl_priv = app->priv; |
627 | struct nfp_meter_entry *meter_entry = NULL; |
628 | int err = 0; |
629 | |
630 | mutex_lock(&fl_priv->meter_stats_lock); |
631 | |
632 | switch (op) { |
633 | case NFP_METER_DEL: |
634 | nfp_flower_del_meter_entry(app, meter_id); |
635 | goto exit_unlock; |
636 | case NFP_METER_ADD: |
637 | meter_entry = nfp_flower_add_meter_entry(app, meter_id); |
638 | break; |
639 | default: |
640 | err = -EOPNOTSUPP; |
641 | goto exit_unlock; |
642 | } |
643 | |
644 | if (!meter_entry) { |
645 | err = -ENOMEM; |
646 | goto exit_unlock; |
647 | } |
648 | |
649 | if (action->police.rate_bytes_ps > 0) { |
650 | meter_entry->bps = true; |
651 | meter_entry->rate = action->police.rate_bytes_ps; |
652 | meter_entry->burst = action->police.burst; |
653 | } else { |
654 | meter_entry->bps = false; |
655 | meter_entry->rate = action->police.rate_pkt_ps; |
656 | meter_entry->burst = action->police.burst_pkt; |
657 | } |
658 | |
659 | exit_unlock: |
660 | mutex_unlock(lock: &fl_priv->meter_stats_lock); |
661 | return err; |
662 | } |
663 | |
664 | int nfp_init_meter_table(struct nfp_app *app) |
665 | { |
666 | struct nfp_flower_priv *priv = app->priv; |
667 | |
668 | return rhashtable_init(ht: &priv->meter_table, params: &stats_meter_table_params); |
669 | } |
670 | |
671 | void |
672 | nfp_flower_stats_meter_request_all(struct nfp_flower_priv *fl_priv) |
673 | { |
674 | struct nfp_meter_entry *meter_entry = NULL; |
675 | struct rhashtable_iter iter; |
676 | |
677 | mutex_lock(&fl_priv->meter_stats_lock); |
678 | rhashtable_walk_enter(ht: &fl_priv->meter_table, iter: &iter); |
679 | rhashtable_walk_start(iter: &iter); |
680 | |
681 | while ((meter_entry = rhashtable_walk_next(iter: &iter)) != NULL) { |
682 | if (IS_ERR(ptr: meter_entry)) |
683 | continue; |
684 | nfp_flower_stats_rlim_request(fl_priv, |
685 | id: meter_entry->meter_id, ingress: false); |
686 | } |
687 | |
688 | rhashtable_walk_stop(iter: &iter); |
689 | rhashtable_walk_exit(iter: &iter); |
690 | mutex_unlock(lock: &fl_priv->meter_stats_lock); |
691 | } |
692 | |
693 | static int |
694 | nfp_act_install_actions(struct nfp_app *app, struct flow_offload_action *fl_act, |
695 | struct netlink_ext_ack *extack) |
696 | { |
697 | struct flow_action_entry *paction = &fl_act->action.entries[0]; |
698 | u32 action_num = fl_act->action.num_entries; |
699 | struct nfp_flower_priv *fl_priv = app->priv; |
700 | struct flow_action_entry *action = NULL; |
701 | u32 burst, i, meter_id; |
702 | bool pps_support, pps; |
703 | bool add = false; |
704 | u64 rate; |
705 | int err; |
706 | |
707 | pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS); |
708 | |
709 | for (i = 0 ; i < action_num; i++) { |
710 | /* Set qos associate data for this interface */ |
711 | action = paction + i; |
712 | if (action->id != FLOW_ACTION_POLICE) { |
713 | NL_SET_ERR_MSG_MOD(extack, |
714 | "unsupported offload: qos rate limit offload requires police action" ); |
715 | continue; |
716 | } |
717 | |
718 | err = nfp_policer_validate(action: &fl_act->action, act: action, extack, ingress: false); |
719 | if (err) |
720 | return err; |
721 | |
722 | if (action->police.rate_bytes_ps > 0) { |
723 | rate = action->police.rate_bytes_ps; |
724 | burst = action->police.burst; |
725 | } else if (action->police.rate_pkt_ps > 0 && pps_support) { |
726 | rate = action->police.rate_pkt_ps; |
727 | burst = action->police.burst_pkt; |
728 | } else { |
729 | NL_SET_ERR_MSG_MOD(extack, |
730 | "unsupported offload: unsupported qos rate limit" ); |
731 | continue; |
732 | } |
733 | |
734 | if (rate != 0) { |
735 | meter_id = action->hw_index; |
736 | if (nfp_flower_setup_meter_entry(app, action, op: NFP_METER_ADD, meter_id)) |
737 | continue; |
738 | |
739 | pps = false; |
740 | if (action->police.rate_pkt_ps > 0) |
741 | pps = true; |
742 | nfp_flower_offload_one_police(app, ingress: false, pps, id: meter_id, |
743 | rate, burst); |
744 | add = true; |
745 | } |
746 | } |
747 | |
748 | return add ? 0 : -EOPNOTSUPP; |
749 | } |
750 | |
751 | static int |
752 | nfp_act_remove_actions(struct nfp_app *app, struct flow_offload_action *fl_act, |
753 | struct netlink_ext_ack *extack) |
754 | { |
755 | struct nfp_meter_entry *meter_entry = NULL; |
756 | struct nfp_police_config *config; |
757 | struct sk_buff *skb; |
758 | u32 meter_id; |
759 | bool pps; |
760 | |
761 | /* Delete qos associate data for this interface */ |
762 | if (fl_act->id != FLOW_ACTION_POLICE) { |
763 | NL_SET_ERR_MSG_MOD(extack, |
764 | "unsupported offload: qos rate limit offload requires police action" ); |
765 | return -EOPNOTSUPP; |
766 | } |
767 | |
768 | meter_id = fl_act->index; |
769 | meter_entry = nfp_flower_search_meter_entry(app, meter_id); |
770 | if (!meter_entry) { |
771 | NL_SET_ERR_MSG_MOD(extack, |
772 | "no meter entry when delete the action index." ); |
773 | return -ENOENT; |
774 | } |
775 | pps = !meter_entry->bps; |
776 | |
777 | skb = nfp_flower_cmsg_alloc(app, size: sizeof(struct nfp_police_config), |
778 | type: NFP_FLOWER_CMSG_TYPE_QOS_DEL, GFP_KERNEL); |
779 | if (!skb) |
780 | return -ENOMEM; |
781 | |
782 | config = nfp_flower_cmsg_get_data(skb); |
783 | memset(config, 0, sizeof(struct nfp_police_config)); |
784 | config->head.flags_opts = cpu_to_be32(NFP_FL_QOS_METER); |
785 | config->head.meter_id = cpu_to_be32(meter_id); |
786 | if (pps) |
787 | config->head.flags_opts |= cpu_to_be32(NFP_FL_QOS_PPS); |
788 | |
789 | nfp_ctrl_tx(nn: app->ctrl, skb); |
790 | nfp_flower_setup_meter_entry(app, NULL, op: NFP_METER_DEL, meter_id); |
791 | |
792 | return 0; |
793 | } |
794 | |
795 | void |
796 | nfp_act_stats_reply(struct nfp_app *app, void *pmsg) |
797 | { |
798 | struct nfp_flower_priv *fl_priv = app->priv; |
799 | struct nfp_meter_entry *meter_entry = NULL; |
800 | struct nfp_police_stats_reply *msg = pmsg; |
801 | u32 meter_id; |
802 | |
803 | meter_id = be32_to_cpu(msg->head.meter_id); |
804 | mutex_lock(&fl_priv->meter_stats_lock); |
805 | |
806 | meter_entry = nfp_flower_search_meter_entry(app, meter_id); |
807 | if (!meter_entry) |
808 | goto exit_unlock; |
809 | |
810 | meter_entry->stats.curr.pkts = be64_to_cpu(msg->pass_pkts) + |
811 | be64_to_cpu(msg->drop_pkts); |
812 | meter_entry->stats.curr.bytes = be64_to_cpu(msg->pass_bytes) + |
813 | be64_to_cpu(msg->drop_bytes); |
814 | meter_entry->stats.curr.drops = be64_to_cpu(msg->drop_pkts); |
815 | if (!meter_entry->stats.update) { |
816 | meter_entry->stats.prev.pkts = meter_entry->stats.curr.pkts; |
817 | meter_entry->stats.prev.bytes = meter_entry->stats.curr.bytes; |
818 | meter_entry->stats.prev.drops = meter_entry->stats.curr.drops; |
819 | } |
820 | |
821 | meter_entry->stats.update = jiffies; |
822 | |
823 | exit_unlock: |
824 | mutex_unlock(lock: &fl_priv->meter_stats_lock); |
825 | } |
826 | |
827 | static int |
828 | nfp_act_stats_actions(struct nfp_app *app, struct flow_offload_action *fl_act, |
829 | struct netlink_ext_ack *extack) |
830 | { |
831 | struct nfp_flower_priv *fl_priv = app->priv; |
832 | struct nfp_meter_entry *meter_entry = NULL; |
833 | u64 diff_bytes, diff_pkts, diff_drops; |
834 | int err = 0; |
835 | |
836 | if (fl_act->id != FLOW_ACTION_POLICE) { |
837 | NL_SET_ERR_MSG_MOD(extack, |
838 | "unsupported offload: qos rate limit offload requires police action" ); |
839 | return -EOPNOTSUPP; |
840 | } |
841 | |
842 | mutex_lock(&fl_priv->meter_stats_lock); |
843 | meter_entry = nfp_flower_search_meter_entry(app, meter_id: fl_act->index); |
844 | if (!meter_entry) { |
845 | err = -ENOENT; |
846 | goto exit_unlock; |
847 | } |
848 | diff_pkts = meter_entry->stats.curr.pkts > meter_entry->stats.prev.pkts ? |
849 | meter_entry->stats.curr.pkts - meter_entry->stats.prev.pkts : 0; |
850 | diff_bytes = meter_entry->stats.curr.bytes > meter_entry->stats.prev.bytes ? |
851 | meter_entry->stats.curr.bytes - meter_entry->stats.prev.bytes : 0; |
852 | diff_drops = meter_entry->stats.curr.drops > meter_entry->stats.prev.drops ? |
853 | meter_entry->stats.curr.drops - meter_entry->stats.prev.drops : 0; |
854 | |
855 | flow_stats_update(flow_stats: &fl_act->stats, bytes: diff_bytes, pkts: diff_pkts, drops: diff_drops, |
856 | lastused: meter_entry->stats.update, |
857 | used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED); |
858 | |
859 | meter_entry->stats.prev.pkts = meter_entry->stats.curr.pkts; |
860 | meter_entry->stats.prev.bytes = meter_entry->stats.curr.bytes; |
861 | meter_entry->stats.prev.drops = meter_entry->stats.curr.drops; |
862 | |
863 | exit_unlock: |
864 | mutex_unlock(lock: &fl_priv->meter_stats_lock); |
865 | return err; |
866 | } |
867 | |
868 | int nfp_setup_tc_act_offload(struct nfp_app *app, |
869 | struct flow_offload_action *fl_act) |
870 | { |
871 | struct netlink_ext_ack *extack = fl_act->extack; |
872 | struct nfp_flower_priv *fl_priv = app->priv; |
873 | |
874 | if (!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_METER)) |
875 | return -EOPNOTSUPP; |
876 | |
877 | switch (fl_act->command) { |
878 | case FLOW_ACT_REPLACE: |
879 | return nfp_act_install_actions(app, fl_act, extack); |
880 | case FLOW_ACT_DESTROY: |
881 | return nfp_act_remove_actions(app, fl_act, extack); |
882 | case FLOW_ACT_STATS: |
883 | return nfp_act_stats_actions(app, fl_act, extack); |
884 | default: |
885 | return -EOPNOTSUPP; |
886 | } |
887 | } |
888 | |