1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ |
3 | |
4 | #include <linux/hash.h> |
5 | #include <linux/hashtable.h> |
6 | #include <linux/jhash.h> |
7 | #include <linux/math64.h> |
8 | #include <linux/vmalloc.h> |
9 | #include <net/pkt_cls.h> |
10 | |
11 | #include "cmsg.h" |
12 | #include "conntrack.h" |
13 | #include "main.h" |
14 | #include "../nfp_app.h" |
15 | |
16 | struct nfp_mask_id_table { |
17 | struct hlist_node link; |
18 | u32 hash_key; |
19 | u32 ref_cnt; |
20 | u8 mask_id; |
21 | }; |
22 | |
23 | struct nfp_fl_flow_table_cmp_arg { |
24 | struct net_device *netdev; |
25 | unsigned long cookie; |
26 | }; |
27 | |
28 | struct nfp_fl_stats_ctx_to_flow { |
29 | struct rhash_head ht_node; |
30 | u32 stats_cxt; |
31 | struct nfp_fl_payload *flow; |
32 | }; |
33 | |
34 | static const struct rhashtable_params stats_ctx_table_params = { |
35 | .key_offset = offsetof(struct nfp_fl_stats_ctx_to_flow, stats_cxt), |
36 | .head_offset = offsetof(struct nfp_fl_stats_ctx_to_flow, ht_node), |
37 | .key_len = sizeof(u32), |
38 | }; |
39 | |
40 | static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id) |
41 | { |
42 | struct nfp_flower_priv *priv = app->priv; |
43 | struct circ_buf *ring; |
44 | |
45 | ring = &priv->stats_ids.free_list; |
46 | /* Check if buffer is full, stats_ring_size must be power of 2 */ |
47 | if (!CIRC_SPACE(ring->head, ring->tail, priv->stats_ring_size)) |
48 | return -ENOBUFS; |
49 | |
50 | /* Each increment of head represents size of NFP_FL_STATS_ELEM_RS */ |
51 | memcpy(&ring->buf[ring->head * NFP_FL_STATS_ELEM_RS], |
52 | &stats_context_id, NFP_FL_STATS_ELEM_RS); |
53 | ring->head = (ring->head + 1) & (priv->stats_ring_size - 1); |
54 | |
55 | return 0; |
56 | } |
57 | |
58 | static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id) |
59 | { |
60 | struct nfp_flower_priv *priv = app->priv; |
61 | u32 freed_stats_id, temp_stats_id; |
62 | struct circ_buf *ring; |
63 | |
64 | ring = &priv->stats_ids.free_list; |
65 | freed_stats_id = priv->stats_ring_size; |
66 | /* Check for unallocated entries first. */ |
67 | if (priv->stats_ids.init_unalloc > 0) { |
68 | *stats_context_id = |
69 | FIELD_PREP(NFP_FL_STAT_ID_STAT, |
70 | priv->stats_ids.init_unalloc - 1) | |
71 | FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, |
72 | priv->active_mem_unit); |
73 | |
74 | if (++priv->active_mem_unit == priv->total_mem_units) { |
75 | priv->stats_ids.init_unalloc--; |
76 | priv->active_mem_unit = 0; |
77 | } |
78 | |
79 | return 0; |
80 | } |
81 | |
82 | /* Check if buffer is empty. */ |
83 | if (ring->head == ring->tail) { |
84 | *stats_context_id = freed_stats_id; |
85 | return -ENOENT; |
86 | } |
87 | |
88 | /* Each increment of tail represents size of NFP_FL_STATS_ELEM_RS */ |
89 | memcpy(&temp_stats_id, &ring->buf[ring->tail * NFP_FL_STATS_ELEM_RS], |
90 | NFP_FL_STATS_ELEM_RS); |
91 | *stats_context_id = temp_stats_id; |
92 | memcpy(&ring->buf[ring->tail * NFP_FL_STATS_ELEM_RS], &freed_stats_id, |
93 | NFP_FL_STATS_ELEM_RS); |
94 | /* stats_ring_size must be power of 2 */ |
95 | ring->tail = (ring->tail + 1) & (priv->stats_ring_size - 1); |
96 | |
97 | return 0; |
98 | } |
99 | |
100 | /* Must be called with either RTNL or rcu_read_lock */ |
101 | struct nfp_fl_payload * |
102 | nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie, |
103 | struct net_device *netdev) |
104 | { |
105 | struct nfp_fl_flow_table_cmp_arg flower_cmp_arg; |
106 | struct nfp_flower_priv *priv = app->priv; |
107 | |
108 | flower_cmp_arg.netdev = netdev; |
109 | flower_cmp_arg.cookie = tc_flower_cookie; |
110 | |
111 | return rhashtable_lookup_fast(ht: &priv->flow_table, key: &flower_cmp_arg, |
112 | params: nfp_flower_table_params); |
113 | } |
114 | |
115 | void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb) |
116 | { |
117 | unsigned int msg_len = nfp_flower_cmsg_get_data_len(skb); |
118 | struct nfp_flower_priv *priv = app->priv; |
119 | struct nfp_fl_stats_frame *stats; |
120 | unsigned char *msg; |
121 | u32 ctx_id; |
122 | int i; |
123 | |
124 | msg = nfp_flower_cmsg_get_data(skb); |
125 | |
126 | spin_lock(lock: &priv->stats_lock); |
127 | for (i = 0; i < msg_len / sizeof(*stats); i++) { |
128 | stats = (struct nfp_fl_stats_frame *)msg + i; |
129 | ctx_id = be32_to_cpu(stats->stats_con_id); |
130 | priv->stats[ctx_id].pkts += be32_to_cpu(stats->pkt_count); |
131 | priv->stats[ctx_id].bytes += be64_to_cpu(stats->byte_count); |
132 | priv->stats[ctx_id].used = jiffies; |
133 | } |
134 | spin_unlock(lock: &priv->stats_lock); |
135 | } |
136 | |
137 | static int nfp_release_mask_id(struct nfp_app *app, u8 mask_id) |
138 | { |
139 | struct nfp_flower_priv *priv = app->priv; |
140 | struct circ_buf *ring; |
141 | |
142 | ring = &priv->mask_ids.mask_id_free_list; |
143 | /* Checking if buffer is full, |
144 | * NFP_FLOWER_MASK_ENTRY_RS must be power of 2 |
145 | */ |
146 | if (CIRC_SPACE(ring->head, ring->tail, NFP_FLOWER_MASK_ENTRY_RS) == 0) |
147 | return -ENOBUFS; |
148 | |
149 | /* Each increment of head represents size of |
150 | * NFP_FLOWER_MASK_ELEMENT_RS |
151 | */ |
152 | memcpy(&ring->buf[ring->head * NFP_FLOWER_MASK_ELEMENT_RS], &mask_id, |
153 | NFP_FLOWER_MASK_ELEMENT_RS); |
154 | ring->head = (ring->head + 1) & (NFP_FLOWER_MASK_ENTRY_RS - 1); |
155 | |
156 | priv->mask_ids.last_used[mask_id] = ktime_get(); |
157 | |
158 | return 0; |
159 | } |
160 | |
161 | static int nfp_mask_alloc(struct nfp_app *app, u8 *mask_id) |
162 | { |
163 | struct nfp_flower_priv *priv = app->priv; |
164 | ktime_t reuse_timeout; |
165 | struct circ_buf *ring; |
166 | u8 temp_id, freed_id; |
167 | |
168 | ring = &priv->mask_ids.mask_id_free_list; |
169 | freed_id = NFP_FLOWER_MASK_ENTRY_RS - 1; |
170 | /* Checking for unallocated entries first. */ |
171 | if (priv->mask_ids.init_unallocated > 0) { |
172 | *mask_id = priv->mask_ids.init_unallocated; |
173 | priv->mask_ids.init_unallocated--; |
174 | return 0; |
175 | } |
176 | |
177 | /* Checking if buffer is empty. */ |
178 | if (ring->head == ring->tail) |
179 | goto err_not_found; |
180 | |
181 | /* Each increment of tail represents size of |
182 | * NFP_FLOWER_MASK_ELEMENT_RS |
183 | */ |
184 | memcpy(&temp_id, &ring->buf[ring->tail * NFP_FLOWER_MASK_ELEMENT_RS], |
185 | NFP_FLOWER_MASK_ELEMENT_RS); |
186 | *mask_id = temp_id; |
187 | |
188 | reuse_timeout = ktime_add_ns(priv->mask_ids.last_used[*mask_id], |
189 | NFP_FL_MASK_REUSE_TIME_NS); |
190 | |
191 | if (ktime_before(cmp1: ktime_get(), cmp2: reuse_timeout)) |
192 | goto err_not_found; |
193 | |
194 | memcpy(&ring->buf[ring->tail * NFP_FLOWER_MASK_ELEMENT_RS], &freed_id, |
195 | NFP_FLOWER_MASK_ELEMENT_RS); |
196 | /* NFP_FLOWER_MASK_ENTRY_RS must be power of 2 */ |
197 | ring->tail = (ring->tail + 1) & (NFP_FLOWER_MASK_ENTRY_RS - 1); |
198 | |
199 | return 0; |
200 | |
201 | err_not_found: |
202 | *mask_id = freed_id; |
203 | return -ENOENT; |
204 | } |
205 | |
206 | static int |
207 | nfp_add_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) |
208 | { |
209 | struct nfp_flower_priv *priv = app->priv; |
210 | struct nfp_mask_id_table *mask_entry; |
211 | unsigned long hash_key; |
212 | u8 mask_id; |
213 | |
214 | if (nfp_mask_alloc(app, mask_id: &mask_id)) |
215 | return -ENOENT; |
216 | |
217 | mask_entry = kmalloc(size: sizeof(*mask_entry), GFP_KERNEL); |
218 | if (!mask_entry) { |
219 | nfp_release_mask_id(app, mask_id); |
220 | return -ENOMEM; |
221 | } |
222 | |
223 | INIT_HLIST_NODE(h: &mask_entry->link); |
224 | mask_entry->mask_id = mask_id; |
225 | hash_key = jhash(key: mask_data, length: mask_len, initval: priv->mask_id_seed); |
226 | mask_entry->hash_key = hash_key; |
227 | mask_entry->ref_cnt = 1; |
228 | hash_add(priv->mask_table, &mask_entry->link, hash_key); |
229 | |
230 | return mask_id; |
231 | } |
232 | |
233 | static struct nfp_mask_id_table * |
234 | nfp_search_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) |
235 | { |
236 | struct nfp_flower_priv *priv = app->priv; |
237 | struct nfp_mask_id_table *mask_entry; |
238 | unsigned long hash_key; |
239 | |
240 | hash_key = jhash(key: mask_data, length: mask_len, initval: priv->mask_id_seed); |
241 | |
242 | hash_for_each_possible(priv->mask_table, mask_entry, link, hash_key) |
243 | if (mask_entry->hash_key == hash_key) |
244 | return mask_entry; |
245 | |
246 | return NULL; |
247 | } |
248 | |
249 | static int |
250 | nfp_find_in_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) |
251 | { |
252 | struct nfp_mask_id_table *mask_entry; |
253 | |
254 | mask_entry = nfp_search_mask_table(app, mask_data, mask_len); |
255 | if (!mask_entry) |
256 | return -ENOENT; |
257 | |
258 | mask_entry->ref_cnt++; |
259 | |
260 | /* Casting u8 to int for later use. */ |
261 | return mask_entry->mask_id; |
262 | } |
263 | |
264 | static bool |
265 | nfp_check_mask_add(struct nfp_app *app, char *mask_data, u32 mask_len, |
266 | u8 *meta_flags, u8 *mask_id) |
267 | { |
268 | int id; |
269 | |
270 | id = nfp_find_in_mask_table(app, mask_data, mask_len); |
271 | if (id < 0) { |
272 | id = nfp_add_mask_table(app, mask_data, mask_len); |
273 | if (id < 0) |
274 | return false; |
275 | *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; |
276 | } |
277 | *mask_id = id; |
278 | |
279 | return true; |
280 | } |
281 | |
282 | static bool |
283 | nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len, |
284 | u8 *meta_flags, u8 *mask_id) |
285 | { |
286 | struct nfp_mask_id_table *mask_entry; |
287 | |
288 | mask_entry = nfp_search_mask_table(app, mask_data, mask_len); |
289 | if (!mask_entry) |
290 | return false; |
291 | |
292 | *mask_id = mask_entry->mask_id; |
293 | mask_entry->ref_cnt--; |
294 | if (!mask_entry->ref_cnt) { |
295 | hash_del(node: &mask_entry->link); |
296 | nfp_release_mask_id(app, mask_id: *mask_id); |
297 | kfree(objp: mask_entry); |
298 | if (meta_flags) |
299 | *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; |
300 | } |
301 | |
302 | return true; |
303 | } |
304 | |
305 | int nfp_compile_flow_metadata(struct nfp_app *app, u32 cookie, |
306 | struct nfp_fl_payload *nfp_flow, |
307 | struct net_device *netdev, |
308 | struct netlink_ext_ack *extack) |
309 | { |
310 | struct nfp_fl_stats_ctx_to_flow *ctx_entry; |
311 | struct nfp_flower_priv *priv = app->priv; |
312 | struct nfp_fl_payload *check_entry; |
313 | u8 new_mask_id; |
314 | u32 stats_cxt; |
315 | int err; |
316 | |
317 | err = nfp_get_stats_entry(app, stats_context_id: &stats_cxt); |
318 | if (err) { |
319 | NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate new stats context" ); |
320 | return err; |
321 | } |
322 | |
323 | nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt); |
324 | nfp_flow->meta.host_cookie = cpu_to_be64(cookie); |
325 | nfp_flow->ingress_dev = netdev; |
326 | |
327 | ctx_entry = kzalloc(size: sizeof(*ctx_entry), GFP_KERNEL); |
328 | if (!ctx_entry) { |
329 | err = -ENOMEM; |
330 | goto err_release_stats; |
331 | } |
332 | |
333 | ctx_entry->stats_cxt = stats_cxt; |
334 | ctx_entry->flow = nfp_flow; |
335 | |
336 | if (rhashtable_insert_fast(ht: &priv->stats_ctx_table, obj: &ctx_entry->ht_node, |
337 | params: stats_ctx_table_params)) { |
338 | err = -ENOMEM; |
339 | goto err_free_ctx_entry; |
340 | } |
341 | |
342 | /* Do not allocate a mask-id for pre_tun_rules. These flows are used to |
343 | * configure the pre_tun table and are never actually send to the |
344 | * firmware as an add-flow message. This causes the mask-id allocation |
345 | * on the firmware to get out of sync if allocated here. |
346 | */ |
347 | new_mask_id = 0; |
348 | if (!nfp_flow->pre_tun_rule.dev && |
349 | !nfp_check_mask_add(app, mask_data: nfp_flow->mask_data, |
350 | mask_len: nfp_flow->meta.mask_len, |
351 | meta_flags: &nfp_flow->meta.flags, mask_id: &new_mask_id)) { |
352 | NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate a new mask id" ); |
353 | err = -ENOENT; |
354 | goto err_remove_rhash; |
355 | } |
356 | |
357 | nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); |
358 | priv->flower_version++; |
359 | |
360 | /* Update flow payload with mask ids. */ |
361 | nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; |
362 | priv->stats[stats_cxt].pkts = 0; |
363 | priv->stats[stats_cxt].bytes = 0; |
364 | priv->stats[stats_cxt].used = jiffies; |
365 | |
366 | check_entry = nfp_flower_search_fl_table(app, tc_flower_cookie: cookie, netdev); |
367 | if (check_entry) { |
368 | NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot offload duplicate flow entry" ); |
369 | err = -EEXIST; |
370 | goto err_remove_mask; |
371 | } |
372 | |
373 | return 0; |
374 | |
375 | err_remove_mask: |
376 | if (!nfp_flow->pre_tun_rule.dev) |
377 | nfp_check_mask_remove(app, mask_data: nfp_flow->mask_data, |
378 | mask_len: nfp_flow->meta.mask_len, |
379 | NULL, mask_id: &new_mask_id); |
380 | err_remove_rhash: |
381 | WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table, |
382 | &ctx_entry->ht_node, |
383 | stats_ctx_table_params)); |
384 | err_free_ctx_entry: |
385 | kfree(objp: ctx_entry); |
386 | err_release_stats: |
387 | nfp_release_stats_entry(app, stats_context_id: stats_cxt); |
388 | |
389 | return err; |
390 | } |
391 | |
392 | void __nfp_modify_flow_metadata(struct nfp_flower_priv *priv, |
393 | struct nfp_fl_payload *nfp_flow) |
394 | { |
395 | nfp_flow->meta.flags &= ~NFP_FL_META_FLAG_MANAGE_MASK; |
396 | nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); |
397 | priv->flower_version++; |
398 | } |
399 | |
400 | int nfp_modify_flow_metadata(struct nfp_app *app, |
401 | struct nfp_fl_payload *nfp_flow) |
402 | { |
403 | struct nfp_fl_stats_ctx_to_flow *ctx_entry; |
404 | struct nfp_flower_priv *priv = app->priv; |
405 | u8 new_mask_id = 0; |
406 | u32 temp_ctx_id; |
407 | |
408 | __nfp_modify_flow_metadata(priv, nfp_flow); |
409 | |
410 | if (!nfp_flow->pre_tun_rule.dev) |
411 | nfp_check_mask_remove(app, mask_data: nfp_flow->mask_data, |
412 | mask_len: nfp_flow->meta.mask_len, meta_flags: &nfp_flow->meta.flags, |
413 | mask_id: &new_mask_id); |
414 | |
415 | /* Update flow payload with mask ids. */ |
416 | nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; |
417 | |
418 | /* Release the stats ctx id and ctx to flow table entry. */ |
419 | temp_ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id); |
420 | |
421 | ctx_entry = rhashtable_lookup_fast(ht: &priv->stats_ctx_table, key: &temp_ctx_id, |
422 | params: stats_ctx_table_params); |
423 | if (!ctx_entry) |
424 | return -ENOENT; |
425 | |
426 | WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table, |
427 | &ctx_entry->ht_node, |
428 | stats_ctx_table_params)); |
429 | kfree(objp: ctx_entry); |
430 | |
431 | return nfp_release_stats_entry(app, stats_context_id: temp_ctx_id); |
432 | } |
433 | |
434 | struct nfp_fl_payload * |
435 | nfp_flower_get_fl_payload_from_ctx(struct nfp_app *app, u32 ctx_id) |
436 | { |
437 | struct nfp_fl_stats_ctx_to_flow *ctx_entry; |
438 | struct nfp_flower_priv *priv = app->priv; |
439 | |
440 | ctx_entry = rhashtable_lookup_fast(ht: &priv->stats_ctx_table, key: &ctx_id, |
441 | params: stats_ctx_table_params); |
442 | if (!ctx_entry) |
443 | return NULL; |
444 | |
445 | return ctx_entry->flow; |
446 | } |
447 | |
448 | static int nfp_fl_obj_cmpfn(struct rhashtable_compare_arg *arg, |
449 | const void *obj) |
450 | { |
451 | const struct nfp_fl_flow_table_cmp_arg *cmp_arg = arg->key; |
452 | const struct nfp_fl_payload *flow_entry = obj; |
453 | |
454 | if (flow_entry->ingress_dev == cmp_arg->netdev) |
455 | return flow_entry->tc_flower_cookie != cmp_arg->cookie; |
456 | |
457 | return 1; |
458 | } |
459 | |
460 | static u32 nfp_fl_obj_hashfn(const void *data, u32 len, u32 seed) |
461 | { |
462 | const struct nfp_fl_payload *flower_entry = data; |
463 | |
464 | return jhash2(k: (u32 *)&flower_entry->tc_flower_cookie, |
465 | length: sizeof(flower_entry->tc_flower_cookie) / sizeof(u32), |
466 | initval: seed); |
467 | } |
468 | |
469 | static u32 nfp_fl_key_hashfn(const void *data, u32 len, u32 seed) |
470 | { |
471 | const struct nfp_fl_flow_table_cmp_arg *cmp_arg = data; |
472 | |
473 | return jhash2(k: (u32 *)&cmp_arg->cookie, |
474 | length: sizeof(cmp_arg->cookie) / sizeof(u32), initval: seed); |
475 | } |
476 | |
477 | const struct rhashtable_params nfp_flower_table_params = { |
478 | .head_offset = offsetof(struct nfp_fl_payload, fl_node), |
479 | .hashfn = nfp_fl_key_hashfn, |
480 | .obj_cmpfn = nfp_fl_obj_cmpfn, |
481 | .obj_hashfn = nfp_fl_obj_hashfn, |
482 | .automatic_shrinking = true, |
483 | }; |
484 | |
485 | const struct rhashtable_params merge_table_params = { |
486 | .key_offset = offsetof(struct nfp_merge_info, parent_ctx), |
487 | .head_offset = offsetof(struct nfp_merge_info, ht_node), |
488 | .key_len = sizeof(u64), |
489 | }; |
490 | |
491 | const struct rhashtable_params nfp_zone_table_params = { |
492 | .head_offset = offsetof(struct nfp_fl_ct_zone_entry, hash_node), |
493 | .key_len = sizeof(u16), |
494 | .key_offset = offsetof(struct nfp_fl_ct_zone_entry, zone), |
495 | .automatic_shrinking = false, |
496 | }; |
497 | |
498 | const struct rhashtable_params nfp_ct_map_params = { |
499 | .head_offset = offsetof(struct nfp_fl_ct_map_entry, hash_node), |
500 | .key_len = sizeof(unsigned long), |
501 | .key_offset = offsetof(struct nfp_fl_ct_map_entry, cookie), |
502 | .automatic_shrinking = true, |
503 | }; |
504 | |
505 | const struct rhashtable_params neigh_table_params = { |
506 | .key_offset = offsetof(struct nfp_neigh_entry, neigh_cookie), |
507 | .head_offset = offsetof(struct nfp_neigh_entry, ht_node), |
508 | .key_len = sizeof(unsigned long), |
509 | }; |
510 | |
511 | int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count, |
512 | unsigned int host_num_mems) |
513 | { |
514 | struct nfp_flower_priv *priv = app->priv; |
515 | int err, stats_size; |
516 | |
517 | hash_init(priv->mask_table); |
518 | |
519 | err = rhashtable_init(ht: &priv->flow_table, params: &nfp_flower_table_params); |
520 | if (err) |
521 | return err; |
522 | |
523 | err = rhashtable_init(ht: &priv->stats_ctx_table, params: &stats_ctx_table_params); |
524 | if (err) |
525 | goto err_free_flow_table; |
526 | |
527 | err = rhashtable_init(ht: &priv->merge_table, params: &merge_table_params); |
528 | if (err) |
529 | goto err_free_stats_ctx_table; |
530 | |
531 | mutex_init(&priv->nfp_fl_lock); |
532 | |
533 | err = rhashtable_init(ht: &priv->ct_zone_table, params: &nfp_zone_table_params); |
534 | if (err) |
535 | goto err_free_merge_table; |
536 | |
537 | err = rhashtable_init(ht: &priv->ct_map_table, params: &nfp_ct_map_params); |
538 | if (err) |
539 | goto err_free_ct_zone_table; |
540 | |
541 | err = rhashtable_init(ht: &priv->neigh_table, params: &neigh_table_params); |
542 | if (err) |
543 | goto err_free_ct_map_table; |
544 | |
545 | INIT_LIST_HEAD(list: &priv->predt_list); |
546 | |
547 | get_random_bytes(buf: &priv->mask_id_seed, len: sizeof(priv->mask_id_seed)); |
548 | |
549 | /* Init ring buffer and unallocated mask_ids. */ |
550 | priv->mask_ids.mask_id_free_list.buf = |
551 | kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, |
552 | NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL); |
553 | if (!priv->mask_ids.mask_id_free_list.buf) |
554 | goto err_free_neigh_table; |
555 | |
556 | priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1; |
557 | |
558 | /* Init timestamps for mask id*/ |
559 | priv->mask_ids.last_used = |
560 | kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, |
561 | size: sizeof(*priv->mask_ids.last_used), GFP_KERNEL); |
562 | if (!priv->mask_ids.last_used) |
563 | goto err_free_mask_id; |
564 | |
565 | /* Init ring buffer and unallocated stats_ids. */ |
566 | priv->stats_ids.free_list.buf = |
567 | vmalloc(array_size(NFP_FL_STATS_ELEM_RS, |
568 | priv->stats_ring_size)); |
569 | if (!priv->stats_ids.free_list.buf) |
570 | goto err_free_last_used; |
571 | |
572 | priv->stats_ids.init_unalloc = div_u64(dividend: host_ctx_count, divisor: host_num_mems); |
573 | |
574 | stats_size = FIELD_PREP(NFP_FL_STAT_ID_STAT, host_ctx_count) | |
575 | FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, host_num_mems - 1); |
576 | priv->stats = kvmalloc_array(n: stats_size, size: sizeof(struct nfp_fl_stats), |
577 | GFP_KERNEL); |
578 | if (!priv->stats) |
579 | goto err_free_ring_buf; |
580 | |
581 | spin_lock_init(&priv->stats_lock); |
582 | spin_lock_init(&priv->predt_lock); |
583 | |
584 | return 0; |
585 | |
586 | err_free_ring_buf: |
587 | vfree(addr: priv->stats_ids.free_list.buf); |
588 | err_free_last_used: |
589 | kfree(objp: priv->mask_ids.last_used); |
590 | err_free_mask_id: |
591 | kfree(objp: priv->mask_ids.mask_id_free_list.buf); |
592 | err_free_neigh_table: |
593 | rhashtable_destroy(ht: &priv->neigh_table); |
594 | err_free_ct_map_table: |
595 | rhashtable_destroy(ht: &priv->ct_map_table); |
596 | err_free_ct_zone_table: |
597 | rhashtable_destroy(ht: &priv->ct_zone_table); |
598 | err_free_merge_table: |
599 | rhashtable_destroy(ht: &priv->merge_table); |
600 | err_free_stats_ctx_table: |
601 | rhashtable_destroy(ht: &priv->stats_ctx_table); |
602 | err_free_flow_table: |
603 | rhashtable_destroy(ht: &priv->flow_table); |
604 | return -ENOMEM; |
605 | } |
606 | |
607 | static void nfp_zone_table_entry_destroy(struct nfp_fl_ct_zone_entry *zt) |
608 | { |
609 | if (!zt) |
610 | return; |
611 | |
612 | if (!list_empty(head: &zt->pre_ct_list)) { |
613 | struct rhashtable *m_table = &zt->priv->ct_map_table; |
614 | struct nfp_fl_ct_flow_entry *entry, *tmp; |
615 | struct nfp_fl_ct_map_entry *map; |
616 | |
617 | WARN_ONCE(1, "pre_ct_list not empty as expected, cleaning up\n" ); |
618 | list_for_each_entry_safe(entry, tmp, &zt->pre_ct_list, |
619 | list_node) { |
620 | map = rhashtable_lookup_fast(ht: m_table, |
621 | key: &entry->cookie, |
622 | params: nfp_ct_map_params); |
623 | WARN_ON_ONCE(rhashtable_remove_fast(m_table, |
624 | &map->hash_node, |
625 | nfp_ct_map_params)); |
626 | nfp_fl_ct_clean_flow_entry(entry); |
627 | kfree(objp: map); |
628 | } |
629 | } |
630 | |
631 | if (!list_empty(head: &zt->post_ct_list)) { |
632 | struct rhashtable *m_table = &zt->priv->ct_map_table; |
633 | struct nfp_fl_ct_flow_entry *entry, *tmp; |
634 | struct nfp_fl_ct_map_entry *map; |
635 | |
636 | WARN_ONCE(1, "post_ct_list not empty as expected, cleaning up\n" ); |
637 | list_for_each_entry_safe(entry, tmp, &zt->post_ct_list, |
638 | list_node) { |
639 | map = rhashtable_lookup_fast(ht: m_table, |
640 | key: &entry->cookie, |
641 | params: nfp_ct_map_params); |
642 | WARN_ON_ONCE(rhashtable_remove_fast(m_table, |
643 | &map->hash_node, |
644 | nfp_ct_map_params)); |
645 | nfp_fl_ct_clean_flow_entry(entry); |
646 | kfree(objp: map); |
647 | } |
648 | } |
649 | |
650 | if (zt->nft) { |
651 | nf_flow_table_offload_del_cb(flow_table: zt->nft, |
652 | cb: nfp_fl_ct_handle_nft_flow, |
653 | cb_priv: zt); |
654 | zt->nft = NULL; |
655 | } |
656 | |
657 | if (!list_empty(head: &zt->nft_flows_list)) { |
658 | struct rhashtable *m_table = &zt->priv->ct_map_table; |
659 | struct nfp_fl_ct_flow_entry *entry, *tmp; |
660 | struct nfp_fl_ct_map_entry *map; |
661 | |
662 | WARN_ONCE(1, "nft_flows_list not empty as expected, cleaning up\n" ); |
663 | list_for_each_entry_safe(entry, tmp, &zt->nft_flows_list, |
664 | list_node) { |
665 | map = rhashtable_lookup_fast(ht: m_table, |
666 | key: &entry->cookie, |
667 | params: nfp_ct_map_params); |
668 | WARN_ON_ONCE(rhashtable_remove_fast(m_table, |
669 | &map->hash_node, |
670 | nfp_ct_map_params)); |
671 | nfp_fl_ct_clean_flow_entry(entry); |
672 | kfree(objp: map); |
673 | } |
674 | } |
675 | |
676 | rhashtable_free_and_destroy(ht: &zt->tc_merge_tb, |
677 | free_fn: nfp_check_rhashtable_empty, NULL); |
678 | rhashtable_free_and_destroy(ht: &zt->nft_merge_tb, |
679 | free_fn: nfp_check_rhashtable_empty, NULL); |
680 | |
681 | kfree(objp: zt); |
682 | } |
683 | |
684 | static void nfp_free_zone_table_entry(void *ptr, void *arg) |
685 | { |
686 | struct nfp_fl_ct_zone_entry *zt = ptr; |
687 | |
688 | nfp_zone_table_entry_destroy(zt); |
689 | } |
690 | |
691 | static void nfp_free_map_table_entry(void *ptr, void *arg) |
692 | { |
693 | struct nfp_fl_ct_map_entry *map = ptr; |
694 | |
695 | if (!map) |
696 | return; |
697 | |
698 | kfree(objp: map); |
699 | } |
700 | |
701 | void nfp_flower_metadata_cleanup(struct nfp_app *app) |
702 | { |
703 | struct nfp_flower_priv *priv = app->priv; |
704 | |
705 | if (!priv) |
706 | return; |
707 | |
708 | rhashtable_free_and_destroy(ht: &priv->flow_table, |
709 | free_fn: nfp_check_rhashtable_empty, NULL); |
710 | rhashtable_free_and_destroy(ht: &priv->stats_ctx_table, |
711 | free_fn: nfp_check_rhashtable_empty, NULL); |
712 | rhashtable_free_and_destroy(ht: &priv->merge_table, |
713 | free_fn: nfp_check_rhashtable_empty, NULL); |
714 | rhashtable_free_and_destroy(ht: &priv->ct_zone_table, |
715 | free_fn: nfp_free_zone_table_entry, NULL); |
716 | nfp_zone_table_entry_destroy(zt: priv->ct_zone_wc); |
717 | |
718 | rhashtable_free_and_destroy(ht: &priv->ct_map_table, |
719 | free_fn: nfp_free_map_table_entry, NULL); |
720 | rhashtable_free_and_destroy(ht: &priv->neigh_table, |
721 | free_fn: nfp_check_rhashtable_empty, NULL); |
722 | kvfree(addr: priv->stats); |
723 | kfree(objp: priv->mask_ids.mask_id_free_list.buf); |
724 | kfree(objp: priv->mask_ids.last_used); |
725 | vfree(addr: priv->stats_ids.free_list.buf); |
726 | } |
727 | |