1	// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2	/* Copyright (C) 2021 Corigine, Inc. */
3
4	#include <net/tc_act/tc_csum.h>
5	#include <net/tc_act/tc_ct.h>
6
7	#include "conntrack.h"
8	#include "../nfp_port.h"
9
10	const struct rhashtable_params nfp_tc_ct_merge_params = {
11	.head_offset = offsetof(struct nfp_fl_ct_tc_merge,
12	hash_node),
13	.key_len = sizeof(unsigned long) * 2,
14	.key_offset = offsetof(struct nfp_fl_ct_tc_merge, cookie),
15	.automatic_shrinking = true,
16	};
17
18	const struct rhashtable_params nfp_nft_ct_merge_params = {
19	.head_offset = offsetof(struct nfp_fl_nft_tc_merge,
20	hash_node),
21	.key_len = sizeof(unsigned long) * 3,
22	.key_offset = offsetof(struct nfp_fl_nft_tc_merge, cookie),
23	.automatic_shrinking = true,
24	};
25
26	static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
27	enum flow_action_id act_id);
28
29	/**
30	* get_hashentry() - Wrapper around hashtable lookup.
31	* @ht: hashtable where entry could be found
32	* @key: key to lookup
33	* @params: hashtable params
34	* @size: size of entry to allocate if not in table
35	*
36	* Returns an entry from a hashtable. If entry does not exist
37	* yet allocate the memory for it and return the new entry.
38	*/
39	static void *get_hashentry(struct rhashtable *ht, void *key,
40	const struct rhashtable_params params, size_t size)
41	{
42	void *result;
43
44	result = rhashtable_lookup_fast(ht, key, params);
45
46	if (result)
47	return result;
48
49	result = kzalloc(size, GFP_KERNEL);
50	if (!result)
51	return ERR_PTR(error: -ENOMEM);
52
53	return result;
54	}
55
56	bool is_pre_ct_flow(struct flow_cls_offload *flow)
57	{
58	struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: flow);
59	struct flow_dissector *dissector = rule->match.dissector;
60	struct flow_action_entry *act;
61	struct flow_match_ct ct;
62	int i;
63
64	if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) {
65	flow_rule_match_ct(rule, out: &ct);
66	if (ct.key->ct_state)
67	return false;
68	}
69
70	if (flow->common.chain_index)
71	return false;
72
73	flow_action_for_each(i, act, &flow->rule->action) {
74	if (act->id == FLOW_ACTION_CT) {
75	/* The pre_ct rule only have the ct or ct nat action, cannot
76	* contains other ct action e.g ct commit and so on.
77	*/
78	if ((!act->ct.action || act->ct.action == TCA_CT_ACT_NAT))
79	return true;
80	else
81	return false;
82	}
83	}
84
85	return false;
86	}
87
88	bool is_post_ct_flow(struct flow_cls_offload *flow)
89	{
90	struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: flow);
91	struct flow_dissector *dissector = rule->match.dissector;
92	struct flow_action_entry *act;
93	bool exist_ct_clear = false;
94	struct flow_match_ct ct;
95	int i;
96
97	if (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT)) {
98	flow_rule_match_ct(rule, out: &ct);
99	if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)
100	return true;
101	} else {
102	/* post ct entry cannot contains any ct action except ct_clear. */
103	flow_action_for_each(i, act, &flow->rule->action) {
104	if (act->id == FLOW_ACTION_CT) {
105	/* ignore ct clear action. */
106	if (act->ct.action == TCA_CT_ACT_CLEAR) {
107	exist_ct_clear = true;
108	continue;
109	}
110
111	return false;
112	}
113	}
114	/* when do nat with ct, the post ct entry ignore the ct status,
115	* will match the nat field(sip/dip) instead. In this situation,
116	* the flow chain index is not zero and contains ct clear action.
117	*/
118	if (flow->common.chain_index && exist_ct_clear)
119	return true;
120	}
121
122	return false;
123	}
124
125	/**
126	* get_mangled_key() - Mangle the key if mangle act exists
127	* @rule: rule that carries the actions
128	* @buf: pointer to key to be mangled
129	* @offset: used to adjust mangled offset in L2/L3/L4 header
130	* @key_sz: key size
131	* @htype: mangling type
132	*
133	* Returns buf where the mangled key stores.
134	*/
135	static void *get_mangled_key(struct flow_rule *rule, void *buf,
136	u32 offset, size_t key_sz,
137	enum flow_action_mangle_base htype)
138	{
139	struct flow_action_entry *act;
140	u32 *val = (u32 *)buf;
141	u32 off, msk, key;
142	int i;
143
144	flow_action_for_each(i, act, &rule->action) {
145	if (act->id == FLOW_ACTION_MANGLE &&
146	act->mangle.htype == htype) {
147	off = act->mangle.offset - offset;
148	msk = act->mangle.mask;
149	key = act->mangle.val;
150
151	/* Mangling is supposed to be u32 aligned */
152	if (off % 4 || off >= key_sz)
153	continue;
154
155	val[off >> 2] &= msk;
156	val[off >> 2] |= key;
157	}
158	}
159
160	return buf;
161	}
162
163	/* Only tos and ttl are involved in flow_match_ip structure, which
164	* doesn't conform to the layout of ip/ipv6 header definition. So
165	* they need particular process here: fill them into the ip/ipv6
166	* header, so that mangling actions can work directly.
167	*/
168	#define NFP_IPV4_TOS_MASK GENMASK(23, 16)
169	#define NFP_IPV4_TTL_MASK GENMASK(31, 24)
170	#define NFP_IPV6_TCLASS_MASK GENMASK(27, 20)
171	#define NFP_IPV6_HLIMIT_MASK GENMASK(7, 0)
172	static void *get_mangled_tos_ttl(struct flow_rule *rule, void *buf,
173	bool is_v6)
174	{
175	struct flow_match_ip match;
176	/* IPv4's ttl field is in third dword. */
177	__be32 ip_hdr[3];
178	u32 tmp, hdr_len;
179
180	flow_rule_match_ip(rule, out: &match);
181
182	if (is_v6) {
183	tmp = FIELD_PREP(NFP_IPV6_TCLASS_MASK, match.key->tos);
184	ip_hdr[0] = cpu_to_be32(tmp);
185	tmp = FIELD_PREP(NFP_IPV6_HLIMIT_MASK, match.key->ttl);
186	ip_hdr[1] = cpu_to_be32(tmp);
187	hdr_len = 2 * sizeof(__be32);
188	} else {
189	tmp = FIELD_PREP(NFP_IPV4_TOS_MASK, match.key->tos);
190	ip_hdr[0] = cpu_to_be32(tmp);
191	tmp = FIELD_PREP(NFP_IPV4_TTL_MASK, match.key->ttl);
192	ip_hdr[2] = cpu_to_be32(tmp);
193	hdr_len = 3 * sizeof(__be32);
194	}
195
196	get_mangled_key(rule, buf: ip_hdr, offset: 0, key_sz: hdr_len,
197	htype: is_v6 ? FLOW_ACT_MANGLE_HDR_TYPE_IP6 :
198	FLOW_ACT_MANGLE_HDR_TYPE_IP4);
199
200	match.key = buf;
201
202	if (is_v6) {
203	tmp = be32_to_cpu(ip_hdr[0]);
204	match.key->tos = FIELD_GET(NFP_IPV6_TCLASS_MASK, tmp);
205	tmp = be32_to_cpu(ip_hdr[1]);
206	match.key->ttl = FIELD_GET(NFP_IPV6_HLIMIT_MASK, tmp);
207	} else {
208	tmp = be32_to_cpu(ip_hdr[0]);
209	match.key->tos = FIELD_GET(NFP_IPV4_TOS_MASK, tmp);
210	tmp = be32_to_cpu(ip_hdr[2]);
211	match.key->ttl = FIELD_GET(NFP_IPV4_TTL_MASK, tmp);
212	}
213
214	return buf;
215	}
216
217	/* Note entry1 and entry2 are not swappable. only skip ip and
218	* tport merge check for pre_ct and post_ct when pre_ct do nat.
219	*/
220	static bool nfp_ct_merge_check_cannot_skip(struct nfp_fl_ct_flow_entry *entry1,
221	struct nfp_fl_ct_flow_entry *entry2)
222	{
223	/* only pre_ct have NFP_FL_ACTION_DO_NAT flag. */
224	if ((entry1->flags & NFP_FL_ACTION_DO_NAT) &&
225	entry2->type == CT_TYPE_POST_CT)
226	return false;
227
228	return true;
229	}
230
231	/* Note entry1 and entry2 are not swappable, entry1 should be
232	* the former flow whose mangle action need be taken into account
233	* if existed, and entry2 should be the latter flow whose action
234	* we don't care.
235	*/
236	static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1,
237	struct nfp_fl_ct_flow_entry *entry2)
238	{
239	unsigned long long ovlp_keys;
240	bool out, is_v6 = false;
241	u8 ip_proto = 0;
242	ovlp_keys = entry1->rule->match.dissector->used_keys &
243	entry2->rule->match.dissector->used_keys;
244	/* Temporary buffer for mangling keys, 64 is enough to cover max
245	* struct size of key in various fields that may be mangled.
246	* Supported fields to mangle:
247	* mac_src/mac_dst(struct flow_match_eth_addrs, 12B)
248	* nw_tos/nw_ttl(struct flow_match_ip, 2B)
249	* nw_src/nw_dst(struct flow_match_ipv4/6_addrs, 32B)
250	* tp_src/tp_dst(struct flow_match_ports, 4B)
251	*/
252	char buf[64];
253
254	if (entry1->netdev && entry2->netdev &&
255	entry1->netdev != entry2->netdev)
256	return -EINVAL;
257
258	/* Check the overlapped fields one by one, the unmasked part
259	* should not conflict with each other.
260	*/
261	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL)) {
262	struct flow_match_control match1, match2;
263
264	flow_rule_match_control(rule: entry1->rule, out: &match1);
265	flow_rule_match_control(rule: entry2->rule, out: &match2);
266	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
267	if (out)
268	goto check_failed;
269	}
270
271	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_BASIC)) {
272	struct flow_match_basic match1, match2;
273
274	flow_rule_match_basic(rule: entry1->rule, out: &match1);
275	flow_rule_match_basic(rule: entry2->rule, out: &match2);
276
277	/* n_proto field is a must in ct-related flows,
278	* it should be either ipv4 or ipv6.
279	*/
280	is_v6 = match1.key->n_proto == htons(ETH_P_IPV6);
281	/* ip_proto field is a must when port field is cared */
282	ip_proto = match1.key->ip_proto;
283
284	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
285	if (out)
286	goto check_failed;
287	}
288
289	/* if pre ct entry do nat, the nat ip exists in nft entry,
290	* will be do merge check when do nft and post ct merge,
291	* so skip this ip merge check here.
292	*/
293	if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) &&
294	nfp_ct_merge_check_cannot_skip(entry1, entry2)) {
295	struct flow_match_ipv4_addrs match1, match2;
296
297	flow_rule_match_ipv4_addrs(rule: entry1->rule, out: &match1);
298	flow_rule_match_ipv4_addrs(rule: entry2->rule, out: &match2);
299
300	memcpy(buf, match1.key, sizeof(*match1.key));
301	match1.key = get_mangled_key(rule: entry1->rule, buf,
302	offsetof(struct iphdr, saddr),
303	key_sz: sizeof(*match1.key),
304	htype: FLOW_ACT_MANGLE_HDR_TYPE_IP4);
305
306	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
307	if (out)
308	goto check_failed;
309	}
310
311	/* if pre ct entry do nat, the nat ip exists in nft entry,
312	* will be do merge check when do nft and post ct merge,
313	* so skip this ip merge check here.
314	*/
315	if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) &&
316	nfp_ct_merge_check_cannot_skip(entry1, entry2)) {
317	struct flow_match_ipv6_addrs match1, match2;
318
319	flow_rule_match_ipv6_addrs(rule: entry1->rule, out: &match1);
320	flow_rule_match_ipv6_addrs(rule: entry2->rule, out: &match2);
321
322	memcpy(buf, match1.key, sizeof(*match1.key));
323	match1.key = get_mangled_key(rule: entry1->rule, buf,
324	offsetof(struct ipv6hdr, saddr),
325	key_sz: sizeof(*match1.key),
326	htype: FLOW_ACT_MANGLE_HDR_TYPE_IP6);
327
328	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
329	if (out)
330	goto check_failed;
331	}
332
333	/* if pre ct entry do nat, the nat tport exists in nft entry,
334	* will be do merge check when do nft and post ct merge,
335	* so skip this tport merge check here.
336	*/
337	if ((ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_PORTS)) &&
338	nfp_ct_merge_check_cannot_skip(entry1, entry2)) {
339	enum flow_action_mangle_base htype = FLOW_ACT_MANGLE_UNSPEC;
340	struct flow_match_ports match1, match2;
341
342	flow_rule_match_ports(rule: entry1->rule, out: &match1);
343	flow_rule_match_ports(rule: entry2->rule, out: &match2);
344
345	if (ip_proto == IPPROTO_UDP)
346	htype = FLOW_ACT_MANGLE_HDR_TYPE_UDP;
347	else if (ip_proto == IPPROTO_TCP)
348	htype = FLOW_ACT_MANGLE_HDR_TYPE_TCP;
349
350	memcpy(buf, match1.key, sizeof(*match1.key));
351	match1.key = get_mangled_key(rule: entry1->rule, buf, offset: 0,
352	key_sz: sizeof(*match1.key), htype);
353
354	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
355	if (out)
356	goto check_failed;
357	}
358
359	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
360	struct flow_match_eth_addrs match1, match2;
361
362	flow_rule_match_eth_addrs(rule: entry1->rule, out: &match1);
363	flow_rule_match_eth_addrs(rule: entry2->rule, out: &match2);
364
365	memcpy(buf, match1.key, sizeof(*match1.key));
366	match1.key = get_mangled_key(rule: entry1->rule, buf, offset: 0,
367	key_sz: sizeof(*match1.key),
368	htype: FLOW_ACT_MANGLE_HDR_TYPE_ETH);
369
370	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
371	if (out)
372	goto check_failed;
373	}
374
375	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_VLAN)) {
376	struct flow_match_vlan match1, match2;
377
378	flow_rule_match_vlan(rule: entry1->rule, out: &match1);
379	flow_rule_match_vlan(rule: entry2->rule, out: &match2);
380	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
381	if (out)
382	goto check_failed;
383	}
384
385	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_MPLS)) {
386	struct flow_match_mpls match1, match2;
387
388	flow_rule_match_mpls(rule: entry1->rule, out: &match1);
389	flow_rule_match_mpls(rule: entry2->rule, out: &match2);
390	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
391	if (out)
392	goto check_failed;
393	}
394
395	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_TCP)) {
396	struct flow_match_tcp match1, match2;
397
398	flow_rule_match_tcp(rule: entry1->rule, out: &match1);
399	flow_rule_match_tcp(rule: entry2->rule, out: &match2);
400	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
401	if (out)
402	goto check_failed;
403	}
404
405	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_IP)) {
406	struct flow_match_ip match1, match2;
407
408	flow_rule_match_ip(rule: entry1->rule, out: &match1);
409	flow_rule_match_ip(rule: entry2->rule, out: &match2);
410
411	match1.key = get_mangled_tos_ttl(rule: entry1->rule, buf, is_v6);
412	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
413	if (out)
414	goto check_failed;
415	}
416
417	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID)) {
418	struct flow_match_enc_keyid match1, match2;
419
420	flow_rule_match_enc_keyid(rule: entry1->rule, out: &match1);
421	flow_rule_match_enc_keyid(rule: entry2->rule, out: &match2);
422	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
423	if (out)
424	goto check_failed;
425	}
426
427	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
428	struct flow_match_ipv4_addrs match1, match2;
429
430	flow_rule_match_enc_ipv4_addrs(rule: entry1->rule, out: &match1);
431	flow_rule_match_enc_ipv4_addrs(rule: entry2->rule, out: &match2);
432	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
433	if (out)
434	goto check_failed;
435	}
436
437	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
438	struct flow_match_ipv6_addrs match1, match2;
439
440	flow_rule_match_enc_ipv6_addrs(rule: entry1->rule, out: &match1);
441	flow_rule_match_enc_ipv6_addrs(rule: entry2->rule, out: &match2);
442	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
443	if (out)
444	goto check_failed;
445	}
446
447	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
448	struct flow_match_control match1, match2;
449
450	flow_rule_match_enc_control(rule: entry1->rule, out: &match1);
451	flow_rule_match_enc_control(rule: entry2->rule, out: &match2);
452	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
453	if (out)
454	goto check_failed;
455	}
456
457	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP)) {
458	struct flow_match_ip match1, match2;
459
460	flow_rule_match_enc_ip(rule: entry1->rule, out: &match1);
461	flow_rule_match_enc_ip(rule: entry2->rule, out: &match2);
462	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
463	if (out)
464	goto check_failed;
465	}
466
467	if (ovlp_keys & BIT_ULL(FLOW_DISSECTOR_KEY_ENC_OPTS)) {
468	struct flow_match_enc_opts match1, match2;
469
470	flow_rule_match_enc_opts(rule: entry1->rule, out: &match1);
471	flow_rule_match_enc_opts(rule: entry2->rule, out: &match2);
472	COMPARE_UNMASKED_FIELDS(match1, match2, &out);
473	if (out)
474	goto check_failed;
475	}
476
477	return 0;
478
479	check_failed:
480	return -EINVAL;
481	}
482
483	static int nfp_ct_check_vlan_merge(struct flow_action_entry *a_in,
484	struct flow_rule *rule)
485	{
486	struct flow_match_vlan match;
487
488	if (unlikely(flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)))
489	return -EOPNOTSUPP;
490
491	/* post_ct does not match VLAN KEY, can be merged. */
492	if (likely(!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)))
493	return 0;
494
495	switch (a_in->id) {
496	/* pre_ct has pop vlan, post_ct cannot match VLAN KEY, cannot be merged. */
497	case FLOW_ACTION_VLAN_POP:
498	return -EOPNOTSUPP;
499
500	case FLOW_ACTION_VLAN_PUSH:
501	case FLOW_ACTION_VLAN_MANGLE:
502	flow_rule_match_vlan(rule, out: &match);
503	/* different vlan id, cannot be merged. */
504	if ((match.key->vlan_id & match.mask->vlan_id) ^
505	(a_in->vlan.vid & match.mask->vlan_id))
506	return -EOPNOTSUPP;
507
508	/* different tpid, cannot be merged. */
509	if ((match.key->vlan_tpid & match.mask->vlan_tpid) ^
510	(a_in->vlan.proto & match.mask->vlan_tpid))
511	return -EOPNOTSUPP;
512
513	/* different priority, cannot be merged. */
514	if ((match.key->vlan_priority & match.mask->vlan_priority) ^
515	(a_in->vlan.prio & match.mask->vlan_priority))
516	return -EOPNOTSUPP;
517
518	break;
519	default:
520	return -EOPNOTSUPP;
521	}
522
523	return 0;
524	}
525
526	/* Extra check for multiple ct-zones merge
527	* currently surpport nft entries merge check in different zones
528	*/
529	static int nfp_ct_merge_extra_check(struct nfp_fl_ct_flow_entry *nft_entry,
530	struct nfp_fl_ct_tc_merge *tc_m_entry)
531	{
532	struct nfp_fl_nft_tc_merge *prev_nft_m_entry;
533	struct nfp_fl_ct_flow_entry *pre_ct_entry;
534
535	pre_ct_entry = tc_m_entry->pre_ct_parent;
536	prev_nft_m_entry = pre_ct_entry->prev_m_entries[pre_ct_entry->num_prev_m_entries - 1];
537
538	return nfp_ct_merge_check(entry1: prev_nft_m_entry->nft_parent, entry2: nft_entry);
539	}
540
541	static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry,
542	struct nfp_fl_ct_flow_entry *post_ct_entry,
543	struct nfp_fl_ct_flow_entry *nft_entry)
544	{
545	struct flow_action_entry *act;
546	int i, err;
547
548	/* Check for pre_ct->action conflicts */
549	flow_action_for_each(i, act, &pre_ct_entry->rule->action) {
550	switch (act->id) {
551	case FLOW_ACTION_VLAN_PUSH:
552	case FLOW_ACTION_VLAN_POP:
553	case FLOW_ACTION_VLAN_MANGLE:
554	err = nfp_ct_check_vlan_merge(a_in: act, rule: post_ct_entry->rule);
555	if (err)
556	return err;
557	break;
558	case FLOW_ACTION_MPLS_PUSH:
559	case FLOW_ACTION_MPLS_POP:
560	case FLOW_ACTION_MPLS_MANGLE:
561	return -EOPNOTSUPP;
562	default:
563	break;
564	}
565	}
566
567	/* Check for nft->action conflicts */
568	flow_action_for_each(i, act, &nft_entry->rule->action) {
569	switch (act->id) {
570	case FLOW_ACTION_VLAN_PUSH:
571	case FLOW_ACTION_VLAN_POP:
572	case FLOW_ACTION_VLAN_MANGLE:
573	case FLOW_ACTION_MPLS_PUSH:
574	case FLOW_ACTION_MPLS_POP:
575	case FLOW_ACTION_MPLS_MANGLE:
576	return -EOPNOTSUPP;
577	default:
578	break;
579	}
580	}
581	return 0;
582	}
583
584	static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry,
585	struct nfp_fl_ct_flow_entry *nft_entry)
586	{
587	struct flow_dissector *dissector = post_ct_entry->rule->match.dissector;
588	struct flow_action_entry *ct_met;
589	struct flow_match_ct ct;
590	int i;
591
592	ct_met = get_flow_act(rule: nft_entry->rule, act_id: FLOW_ACTION_CT_METADATA);
593	if (ct_met && (dissector->used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CT))) {
594	u32 *act_lbl;
595
596	act_lbl = ct_met->ct_metadata.labels;
597	flow_rule_match_ct(rule: post_ct_entry->rule, out: &ct);
598	for (i = 0; i < 4; i++) {
599	if ((ct.key->ct_labels[i] & ct.mask->ct_labels[i]) ^
600	(act_lbl[i] & ct.mask->ct_labels[i]))
601	return -EINVAL;
602	}
603
604	if ((ct.key->ct_mark & ct.mask->ct_mark) ^
605	(ct_met->ct_metadata.mark & ct.mask->ct_mark))
606	return -EINVAL;
607
608	return 0;
609	} else {
610	/* post_ct with ct clear action will not match the
611	* ct status when nft is nat entry.
612	*/
613	if (nft_entry->flags & NFP_FL_ACTION_DO_MANGLE)
614	return 0;
615	}
616
617	return -EINVAL;
618	}
619
620	static int
621	nfp_fl_calc_key_layers_sz(struct nfp_fl_key_ls in_key_ls, uint16_t *map)
622	{
623	int key_size;
624
625	/* This field must always be present */
626	key_size = sizeof(struct nfp_flower_meta_tci);
627	map[FLOW_PAY_META_TCI] = 0;
628
629	if (in_key_ls.key_layer & NFP_FLOWER_LAYER_EXT_META) {
630	map[FLOW_PAY_EXT_META] = key_size;
631	key_size += sizeof(struct nfp_flower_ext_meta);
632	}
633	if (in_key_ls.key_layer & NFP_FLOWER_LAYER_PORT) {
634	map[FLOW_PAY_INPORT] = key_size;
635	key_size += sizeof(struct nfp_flower_in_port);
636	}
637	if (in_key_ls.key_layer & NFP_FLOWER_LAYER_MAC) {
638	map[FLOW_PAY_MAC_MPLS] = key_size;
639	key_size += sizeof(struct nfp_flower_mac_mpls);
640	}
641	if (in_key_ls.key_layer & NFP_FLOWER_LAYER_TP) {
642	map[FLOW_PAY_L4] = key_size;
643	key_size += sizeof(struct nfp_flower_tp_ports);
644	}
645	if (in_key_ls.key_layer & NFP_FLOWER_LAYER_IPV4) {
646	map[FLOW_PAY_IPV4] = key_size;
647	key_size += sizeof(struct nfp_flower_ipv4);
648	}
649	if (in_key_ls.key_layer & NFP_FLOWER_LAYER_IPV6) {
650	map[FLOW_PAY_IPV6] = key_size;
651	key_size += sizeof(struct nfp_flower_ipv6);
652	}
653
654	if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_QINQ) {
655	map[FLOW_PAY_QINQ] = key_size;
656	key_size += sizeof(struct nfp_flower_vlan);
657	}
658
659	if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GRE) {
660	map[FLOW_PAY_GRE] = key_size;
661	if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6)
662	key_size += sizeof(struct nfp_flower_ipv6_gre_tun);
663	else
664	key_size += sizeof(struct nfp_flower_ipv4_gre_tun);
665	}
666
667	if ((in_key_ls.key_layer & NFP_FLOWER_LAYER_VXLAN) ||
668	(in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GENEVE)) {
669	map[FLOW_PAY_UDP_TUN] = key_size;
670	if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6)
671	key_size += sizeof(struct nfp_flower_ipv6_udp_tun);
672	else
673	key_size += sizeof(struct nfp_flower_ipv4_udp_tun);
674	}
675
676	if (in_key_ls.key_layer_two & NFP_FLOWER_LAYER2_GENEVE_OP) {
677	map[FLOW_PAY_GENEVE_OPT] = key_size;
678	key_size += sizeof(struct nfp_flower_geneve_options);
679	}
680
681	return key_size;
682	}
683
684	/* get the csum flag according the ip proto and mangle action. */
685	static void nfp_fl_get_csum_flag(struct flow_action_entry *a_in, u8 ip_proto, u32 *csum)
686	{
687	if (a_in->id != FLOW_ACTION_MANGLE)
688	return;
689
690	switch (a_in->mangle.htype) {
691	case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
692	*csum |= TCA_CSUM_UPDATE_FLAG_IPV4HDR;
693	if (ip_proto == IPPROTO_TCP)
694	*csum |= TCA_CSUM_UPDATE_FLAG_TCP;
695	else if (ip_proto == IPPROTO_UDP)
696	*csum |= TCA_CSUM_UPDATE_FLAG_UDP;
697	break;
698	case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
699	*csum |= TCA_CSUM_UPDATE_FLAG_TCP;
700	break;
701	case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
702	*csum |= TCA_CSUM_UPDATE_FLAG_UDP;
703	break;
704	default:
705	break;
706	}
707	}
708
709	static int nfp_fl_merge_actions_offload(struct flow_rule **rules,
710	struct nfp_flower_priv *priv,
711	struct net_device *netdev,
712	struct nfp_fl_payload *flow_pay,
713	int num_rules)
714	{
715	enum flow_action_hw_stats tmp_stats = FLOW_ACTION_HW_STATS_DONT_CARE;
716	struct flow_action_entry *a_in;
717	int i, j, id, num_actions = 0;
718	struct flow_rule *a_rule;
719	int err = 0, offset = 0;
720
721	for (i = 0; i < num_rules; i++)
722	num_actions += rules[i]->action.num_entries;
723
724	/* Add one action to make sure there is enough room to add an checksum action
725	* when do nat.
726	*/
727	a_rule = flow_rule_alloc(num_actions: num_actions + (num_rules / 2));
728	if (!a_rule)
729	return -ENOMEM;
730
731	/* post_ct entry have one action at least. */
732	if (rules[num_rules - 1]->action.num_entries != 0)
733	tmp_stats = rules[num_rules - 1]->action.entries[0].hw_stats;
734
735	/* Actions need a BASIC dissector. */
736	a_rule->match = rules[0]->match;
737
738	/* Copy actions */
739	for (j = 0; j < num_rules; j++) {
740	u32 csum_updated = 0;
741	u8 ip_proto = 0;
742
743	if (flow_rule_match_key(rule: rules[j], key: FLOW_DISSECTOR_KEY_BASIC)) {
744	struct flow_match_basic match;
745
746	/* ip_proto is the only field that is needed in later compile_action,
747	* needed to set the correct checksum flags. It doesn't really matter
748	* which input rule's ip_proto field we take as the earlier merge checks
749	* would have made sure that they don't conflict. We do not know which
750	* of the subflows would have the ip_proto filled in, so we need to iterate
751	* through the subflows and assign the proper subflow to a_rule
752	*/
753	flow_rule_match_basic(rule: rules[j], out: &match);
754	if (match.mask->ip_proto) {
755	a_rule->match = rules[j]->match;
756	ip_proto = match.key->ip_proto;
757	}
758	}
759
760	for (i = 0; i < rules[j]->action.num_entries; i++) {
761	a_in = &rules[j]->action.entries[i];
762	id = a_in->id;
763
764	/* Ignore CT related actions as these would already have
765	* been taken care of by previous checks, and we do not send
766	* any CT actions to the firmware.
767	*/
768	switch (id) {
769	case FLOW_ACTION_CT:
770	case FLOW_ACTION_GOTO:
771	case FLOW_ACTION_CT_METADATA:
772	continue;
773	default:
774	/* nft entry is generated by tc ct, which mangle action do not care
775	* the stats, inherit the post entry stats to meet the
776	* flow_action_hw_stats_check.
777	* nft entry flow rules are at odd array index.
778	*/
779	if (j & 0x01) {
780	if (a_in->hw_stats == FLOW_ACTION_HW_STATS_DONT_CARE)
781	a_in->hw_stats = tmp_stats;
782	nfp_fl_get_csum_flag(a_in, ip_proto, csum: &csum_updated);
783	}
784	memcpy(&a_rule->action.entries[offset++],
785	a_in, sizeof(struct flow_action_entry));
786	break;
787	}
788	}
789	/* nft entry have mangle action, but do not have checksum action when do NAT,
790	* hardware will automatically fix IPv4 and TCP/UDP checksum. so add an csum action
791	* to meet csum action check.
792	*/
793	if (csum_updated) {
794	struct flow_action_entry *csum_action;
795
796	csum_action = &a_rule->action.entries[offset++];
797	csum_action->id = FLOW_ACTION_CSUM;
798	csum_action->csum_flags = csum_updated;
799	csum_action->hw_stats = tmp_stats;
800	}
801	}
802
803	/* Some actions would have been ignored, so update the num_entries field */
804	a_rule->action.num_entries = offset;
805	err = nfp_flower_compile_action(app: priv->app, rule: a_rule, netdev, nfp_flow: flow_pay, NULL);
806	kfree(objp: a_rule);
807
808	return err;
809	}
810
811	static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry)
812	{
813	enum nfp_flower_tun_type tun_type = NFP_FL_TUNNEL_NONE;
814	struct nfp_fl_ct_zone_entry *zt = m_entry->zt;
815	struct flow_rule *rules[NFP_MAX_ENTRY_RULES];
816	struct nfp_fl_ct_flow_entry *pre_ct_entry;
817	struct nfp_fl_key_ls key_layer, tmp_layer;
818	struct nfp_flower_priv *priv = zt->priv;
819	u16 key_map[_FLOW_PAY_LAYERS_MAX];
820	struct nfp_fl_payload *flow_pay;
821	u8 *key, *msk, *kdata, *mdata;
822	struct nfp_port *port = NULL;
823	int num_rules, err, i, j = 0;
824	struct net_device *netdev;
825	bool qinq_sup;
826	u32 port_id;
827	u16 offset;
828
829	netdev = m_entry->netdev;
830	qinq_sup = !!(priv->flower_ext_feats & NFP_FL_FEATS_VLAN_QINQ);
831
832	pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent;
833	num_rules = pre_ct_entry->num_prev_m_entries * 2 + _CT_TYPE_MAX;
834
835	for (i = 0; i < pre_ct_entry->num_prev_m_entries; i++) {
836	rules[j++] = pre_ct_entry->prev_m_entries[i]->tc_m_parent->pre_ct_parent->rule;
837	rules[j++] = pre_ct_entry->prev_m_entries[i]->nft_parent->rule;
838	}
839
840	rules[j++] = m_entry->tc_m_parent->pre_ct_parent->rule;
841	rules[j++] = m_entry->nft_parent->rule;
842	rules[j++] = m_entry->tc_m_parent->post_ct_parent->rule;
843
844	memset(&key_layer, 0, sizeof(struct nfp_fl_key_ls));
845	memset(&key_map, 0, sizeof(key_map));
846
847	/* Calculate the resultant key layer and size for offload */
848	for (i = 0; i < num_rules; i++) {
849	err = nfp_flower_calculate_key_layers(app: priv->app,
850	netdev: m_entry->netdev,
851	ret_key_ls: &tmp_layer, flow: rules[i],
852	tun_type: &tun_type, NULL);
853	if (err)
854	return err;
855
856	key_layer.key_layer |= tmp_layer.key_layer;
857	key_layer.key_layer_two |= tmp_layer.key_layer_two;
858	}
859	key_layer.key_size = nfp_fl_calc_key_layers_sz(in_key_ls: key_layer, map: key_map);
860
861	flow_pay = nfp_flower_allocate_new(key_layer: &key_layer);
862	if (!flow_pay)
863	return -ENOMEM;
864
865	memset(flow_pay->unmasked_data, 0, key_layer.key_size);
866	memset(flow_pay->mask_data, 0, key_layer.key_size);
867
868	kdata = flow_pay->unmasked_data;
869	mdata = flow_pay->mask_data;
870
871	offset = key_map[FLOW_PAY_META_TCI];
872	key = kdata + offset;
873	msk = mdata + offset;
874	nfp_flower_compile_meta(ext: (struct nfp_flower_meta_tci *)key,
875	msk: (struct nfp_flower_meta_tci *)msk,
876	key_type: key_layer.key_layer);
877
878	if (NFP_FLOWER_LAYER_EXT_META & key_layer.key_layer) {
879	offset = key_map[FLOW_PAY_EXT_META];
880	key = kdata + offset;
881	msk = mdata + offset;
882	nfp_flower_compile_ext_meta(frame: (struct nfp_flower_ext_meta *)key,
883	key_ext: key_layer.key_layer_two);
884	nfp_flower_compile_ext_meta(frame: (struct nfp_flower_ext_meta *)msk,
885	key_ext: key_layer.key_layer_two);
886	}
887
888	/* Using in_port from the -trk rule. The tc merge checks should already
889	* be checking that the ingress netdevs are the same
890	*/
891	port_id = nfp_flower_get_port_id_from_netdev(app: priv->app, netdev);
892	offset = key_map[FLOW_PAY_INPORT];
893	key = kdata + offset;
894	msk = mdata + offset;
895	err = nfp_flower_compile_port(frame: (struct nfp_flower_in_port *)key,
896	cmsg_port: port_id, mask_version: false, tun_type, NULL);
897	if (err)
898	goto ct_offload_err;
899	err = nfp_flower_compile_port(frame: (struct nfp_flower_in_port *)msk,
900	cmsg_port: port_id, mask_version: true, tun_type, NULL);
901	if (err)
902	goto ct_offload_err;
903
904	/* This following part works on the assumption that previous checks has
905	* already filtered out flows that has different values for the different
906	* layers. Here we iterate through all three rules and merge their respective
907	* masked value(cared bits), basic method is:
908	* final_key = (r1_key & r1_mask) | (r2_key & r2_mask) | (r3_key & r3_mask)
909	* final_mask = r1_mask | r2_mask | r3_mask
910	* If none of the rules contains a match that is also fine, that simply means
911	* that the layer is not present.
912	*/
913	if (!qinq_sup) {
914	for (i = 0; i < num_rules; i++) {
915	offset = key_map[FLOW_PAY_META_TCI];
916	key = kdata + offset;
917	msk = mdata + offset;
918	nfp_flower_compile_tci(ext: (struct nfp_flower_meta_tci *)key,
919	msk: (struct nfp_flower_meta_tci *)msk,
920	rule: rules[i]);
921	}
922	}
923
924	if (NFP_FLOWER_LAYER_MAC & key_layer.key_layer) {
925	offset = key_map[FLOW_PAY_MAC_MPLS];
926	key = kdata + offset;
927	msk = mdata + offset;
928	for (i = 0; i < num_rules; i++) {
929	nfp_flower_compile_mac(ext: (struct nfp_flower_mac_mpls *)key,
930	msk: (struct nfp_flower_mac_mpls *)msk,
931	rule: rules[i]);
932	err = nfp_flower_compile_mpls(ext: (struct nfp_flower_mac_mpls *)key,
933	msk: (struct nfp_flower_mac_mpls *)msk,
934	rule: rules[i], NULL);
935	if (err)
936	goto ct_offload_err;
937	}
938	}
939
940	if (NFP_FLOWER_LAYER_IPV4 & key_layer.key_layer) {
941	offset = key_map[FLOW_PAY_IPV4];
942	key = kdata + offset;
943	msk = mdata + offset;
944	for (i = 0; i < num_rules; i++) {
945	nfp_flower_compile_ipv4(ext: (struct nfp_flower_ipv4 *)key,
946	msk: (struct nfp_flower_ipv4 *)msk,
947	rule: rules[i]);
948	}
949	}
950
951	if (NFP_FLOWER_LAYER_IPV6 & key_layer.key_layer) {
952	offset = key_map[FLOW_PAY_IPV6];
953	key = kdata + offset;
954	msk = mdata + offset;
955	for (i = 0; i < num_rules; i++) {
956	nfp_flower_compile_ipv6(ext: (struct nfp_flower_ipv6 *)key,
957	msk: (struct nfp_flower_ipv6 *)msk,
958	rule: rules[i]);
959	}
960	}
961
962	if (NFP_FLOWER_LAYER_TP & key_layer.key_layer) {
963	offset = key_map[FLOW_PAY_L4];
964	key = kdata + offset;
965	msk = mdata + offset;
966	for (i = 0; i < num_rules; i++) {
967	nfp_flower_compile_tport(ext: (struct nfp_flower_tp_ports *)key,
968	msk: (struct nfp_flower_tp_ports *)msk,
969	rule: rules[i]);
970	}
971	}
972
973	if (NFP_FLOWER_LAYER2_QINQ & key_layer.key_layer_two) {
974	offset = key_map[FLOW_PAY_QINQ];
975	key = kdata + offset;
976	msk = mdata + offset;
977	for (i = 0; i < num_rules; i++) {
978	nfp_flower_compile_vlan(ext: (struct nfp_flower_vlan *)key,
979	msk: (struct nfp_flower_vlan *)msk,
980	rule: rules[i]);
981	}
982	}
983
984	if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_GRE) {
985	offset = key_map[FLOW_PAY_GRE];
986	key = kdata + offset;
987	msk = mdata + offset;
988	if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6) {
989	struct nfp_flower_ipv6_gre_tun *gre_match;
990	struct nfp_ipv6_addr_entry *entry;
991	struct in6_addr *dst;
992
993	for (i = 0; i < num_rules; i++) {
994	nfp_flower_compile_ipv6_gre_tun(ext: (void *)key,
995	msk: (void *)msk, rule: rules[i]);
996	}
997	gre_match = (struct nfp_flower_ipv6_gre_tun *)key;
998	dst = &gre_match->ipv6.dst;
999
1000	entry = nfp_tunnel_add_ipv6_off(app: priv->app, ipv6: dst);
1001	if (!entry) {
1002	err = -ENOMEM;
1003	goto ct_offload_err;
1004	}
1005
1006	flow_pay->nfp_tun_ipv6 = entry;
1007	} else {
1008	__be32 dst;
1009
1010	for (i = 0; i < num_rules; i++) {
1011	nfp_flower_compile_ipv4_gre_tun(ext: (void *)key,
1012	msk: (void *)msk, rule: rules[i]);
1013	}
1014	dst = ((struct nfp_flower_ipv4_gre_tun *)key)->ipv4.dst;
1015
1016	/* Store the tunnel destination in the rule data.
1017	* This must be present and be an exact match.
1018	*/
1019	flow_pay->nfp_tun_ipv4_addr = dst;
1020	nfp_tunnel_add_ipv4_off(app: priv->app, ipv4: dst);
1021	}
1022	}
1023
1024	if (key_layer.key_layer & NFP_FLOWER_LAYER_VXLAN ||
1025	key_layer.key_layer_two & NFP_FLOWER_LAYER2_GENEVE) {
1026	offset = key_map[FLOW_PAY_UDP_TUN];
1027	key = kdata + offset;
1028	msk = mdata + offset;
1029	if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_TUN_IPV6) {
1030	struct nfp_flower_ipv6_udp_tun *udp_match;
1031	struct nfp_ipv6_addr_entry *entry;
1032	struct in6_addr *dst;
1033
1034	for (i = 0; i < num_rules; i++) {
1035	nfp_flower_compile_ipv6_udp_tun(ext: (void *)key,
1036	msk: (void *)msk, rule: rules[i]);
1037	}
1038	udp_match = (struct nfp_flower_ipv6_udp_tun *)key;
1039	dst = &udp_match->ipv6.dst;
1040
1041	entry = nfp_tunnel_add_ipv6_off(app: priv->app, ipv6: dst);
1042	if (!entry) {
1043	err = -ENOMEM;
1044	goto ct_offload_err;
1045	}
1046
1047	flow_pay->nfp_tun_ipv6 = entry;
1048	} else {
1049	__be32 dst;
1050
1051	for (i = 0; i < num_rules; i++) {
1052	nfp_flower_compile_ipv4_udp_tun(ext: (void *)key,
1053	msk: (void *)msk, rule: rules[i]);
1054	}
1055	dst = ((struct nfp_flower_ipv4_udp_tun *)key)->ipv4.dst;
1056
1057	/* Store the tunnel destination in the rule data.
1058	* This must be present and be an exact match.
1059	*/
1060	flow_pay->nfp_tun_ipv4_addr = dst;
1061	nfp_tunnel_add_ipv4_off(app: priv->app, ipv4: dst);
1062	}
1063
1064	if (key_layer.key_layer_two & NFP_FLOWER_LAYER2_GENEVE_OP) {
1065	offset = key_map[FLOW_PAY_GENEVE_OPT];
1066	key = kdata + offset;
1067	msk = mdata + offset;
1068	for (i = 0; i < num_rules; i++)
1069	nfp_flower_compile_geneve_opt(ext: key, msk, rule: rules[i]);
1070	}
1071	}
1072
1073	/* Merge actions into flow_pay */
1074	err = nfp_fl_merge_actions_offload(rules, priv, netdev, flow_pay, num_rules);
1075	if (err)
1076	goto ct_offload_err;
1077
1078	/* Use the pointer address as the cookie, but set the last bit to 1.
1079	* This is to avoid the 'is_merge_flow' check from detecting this as
1080	* an already merged flow. This works since address alignment means
1081	* that the last bit for pointer addresses will be 0.
1082	*/
1083	flow_pay->tc_flower_cookie = ((unsigned long)flow_pay) | 0x1;
1084	err = nfp_compile_flow_metadata(app: priv->app, cookie: flow_pay->tc_flower_cookie,
1085	nfp_flow: flow_pay, netdev, NULL);
1086	if (err)
1087	goto ct_offload_err;
1088
1089	if (nfp_netdev_is_nfp_repr(netdev))
1090	port = nfp_port_from_netdev(netdev);
1091
1092	err = rhashtable_insert_fast(ht: &priv->flow_table, obj: &flow_pay->fl_node,
1093	params: nfp_flower_table_params);
1094	if (err)
1095	goto ct_release_offload_meta_err;
1096
1097	err = nfp_flower_xmit_flow(app: priv->app, nfp_flow: flow_pay,
1098	mtype: NFP_FLOWER_CMSG_TYPE_FLOW_ADD);
1099	if (err)
1100	goto ct_remove_rhash_err;
1101
1102	m_entry->tc_flower_cookie = flow_pay->tc_flower_cookie;
1103	m_entry->flow_pay = flow_pay;
1104
1105	if (port)
1106	port->tc_offload_cnt++;
1107
1108	return err;
1109
1110	ct_remove_rhash_err:
1111	WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
1112	&flow_pay->fl_node,
1113	nfp_flower_table_params));
1114	ct_release_offload_meta_err:
1115	nfp_modify_flow_metadata(app: priv->app, nfp_flow: flow_pay);
1116	ct_offload_err:
1117	if (flow_pay->nfp_tun_ipv4_addr)
1118	nfp_tunnel_del_ipv4_off(app: priv->app, ipv4: flow_pay->nfp_tun_ipv4_addr);
1119	if (flow_pay->nfp_tun_ipv6)
1120	nfp_tunnel_put_ipv6_off(app: priv->app, entry: flow_pay->nfp_tun_ipv6);
1121	kfree(objp: flow_pay->action_data);
1122	kfree(objp: flow_pay->mask_data);
1123	kfree(objp: flow_pay->unmasked_data);
1124	kfree(objp: flow_pay);
1125	return err;
1126	}
1127
1128	static int nfp_fl_ct_del_offload(struct nfp_app *app, unsigned long cookie,
1129	struct net_device *netdev)
1130	{
1131	struct nfp_flower_priv *priv = app->priv;
1132	struct nfp_fl_payload *flow_pay;
1133	struct nfp_port *port = NULL;
1134	int err = 0;
1135
1136	if (nfp_netdev_is_nfp_repr(netdev))
1137	port = nfp_port_from_netdev(netdev);
1138
1139	flow_pay = nfp_flower_search_fl_table(app, tc_flower_cookie: cookie, netdev);
1140	if (!flow_pay)
1141	return -ENOENT;
1142
1143	err = nfp_modify_flow_metadata(app, nfp_flow: flow_pay);
1144	if (err)
1145	goto err_free_merge_flow;
1146
1147	if (flow_pay->nfp_tun_ipv4_addr)
1148	nfp_tunnel_del_ipv4_off(app, ipv4: flow_pay->nfp_tun_ipv4_addr);
1149
1150	if (flow_pay->nfp_tun_ipv6)
1151	nfp_tunnel_put_ipv6_off(app, entry: flow_pay->nfp_tun_ipv6);
1152
1153	if (!flow_pay->in_hw) {
1154	err = 0;
1155	goto err_free_merge_flow;
1156	}
1157
1158	err = nfp_flower_xmit_flow(app, nfp_flow: flow_pay,
1159	mtype: NFP_FLOWER_CMSG_TYPE_FLOW_DEL);
1160
1161	err_free_merge_flow:
1162	nfp_flower_del_linked_merge_flows(app, sub_flow: flow_pay);
1163	if (port)
1164	port->tc_offload_cnt--;
1165	kfree(objp: flow_pay->action_data);
1166	kfree(objp: flow_pay->mask_data);
1167	kfree(objp: flow_pay->unmasked_data);
1168	WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
1169	&flow_pay->fl_node,
1170	nfp_flower_table_params));
1171	kfree_rcu(flow_pay, rcu);
1172	return err;
1173	}
1174
1175	static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt,
1176	struct nfp_fl_ct_flow_entry *nft_entry,
1177	struct nfp_fl_ct_tc_merge *tc_m_entry)
1178	{
1179	struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
1180	struct nfp_fl_nft_tc_merge *nft_m_entry;
1181	unsigned long new_cookie[3];
1182	int err;
1183
1184	pre_ct_entry = tc_m_entry->pre_ct_parent;
1185	post_ct_entry = tc_m_entry->post_ct_parent;
1186
1187	err = nfp_ct_merge_act_check(pre_ct_entry, post_ct_entry, nft_entry);
1188	if (err)
1189	return err;
1190
1191	/* Check that the two tc flows are also compatible with
1192	* the nft entry. No need to check the pre_ct and post_ct
1193	* entries as that was already done during pre_merge.
1194	* The nft entry does not have a chain populated, so
1195	* skip this check.
1196	*/
1197	err = nfp_ct_merge_check(entry1: pre_ct_entry, entry2: nft_entry);
1198	if (err)
1199	return err;
1200	err = nfp_ct_merge_check(entry1: nft_entry, entry2: post_ct_entry);
1201	if (err)
1202	return err;
1203	err = nfp_ct_check_meta(post_ct_entry, nft_entry);
1204	if (err)
1205	return err;
1206
1207	if (pre_ct_entry->num_prev_m_entries > 0) {
1208	err = nfp_ct_merge_extra_check(nft_entry, tc_m_entry);
1209	if (err)
1210	return err;
1211	}
1212
1213	/* Combine tc_merge and nft cookies for this cookie. */
1214	new_cookie[0] = tc_m_entry->cookie[0];
1215	new_cookie[1] = tc_m_entry->cookie[1];
1216	new_cookie[2] = nft_entry->cookie;
1217	nft_m_entry = get_hashentry(ht: &zt->nft_merge_tb,
1218	key: &new_cookie,
1219	params: nfp_nft_ct_merge_params,
1220	size: sizeof(*nft_m_entry));
1221
1222	if (IS_ERR(ptr: nft_m_entry))
1223	return PTR_ERR(ptr: nft_m_entry);
1224
1225	/* nft_m_entry already present, not merging again */
1226	if (!memcmp(p: &new_cookie, q: nft_m_entry->cookie, size: sizeof(new_cookie)))
1227	return 0;
1228
1229	memcpy(&nft_m_entry->cookie, &new_cookie, sizeof(new_cookie));
1230	nft_m_entry->zt = zt;
1231	nft_m_entry->tc_m_parent = tc_m_entry;
1232	nft_m_entry->nft_parent = nft_entry;
1233	nft_m_entry->tc_flower_cookie = 0;
1234	/* Copy the netdev from the pre_ct entry. When the tc_m_entry was created
1235	* it only combined them if the netdevs were the same, so can use any of them.
1236	*/
1237	nft_m_entry->netdev = pre_ct_entry->netdev;
1238
1239	/* Add this entry to the tc_m_list and nft_flow lists */
1240	list_add(new: &nft_m_entry->tc_merge_list, head: &tc_m_entry->children);
1241	list_add(new: &nft_m_entry->nft_flow_list, head: &nft_entry->children);
1242
1243	err = rhashtable_insert_fast(ht: &zt->nft_merge_tb, obj: &nft_m_entry->hash_node,
1244	params: nfp_nft_ct_merge_params);
1245	if (err)
1246	goto err_nft_ct_merge_insert;
1247
1248	zt->nft_merge_count++;
1249
1250	if (post_ct_entry->goto_chain_index > 0)
1251	return nfp_fl_create_new_pre_ct(m_entry: nft_m_entry);
1252
1253	/* Generate offload structure and send to nfp */
1254	err = nfp_fl_ct_add_offload(m_entry: nft_m_entry);
1255	if (err)
1256	goto err_nft_ct_offload;
1257
1258	return err;
1259
1260	err_nft_ct_offload:
1261	nfp_fl_ct_del_offload(app: zt->priv->app, cookie: nft_m_entry->tc_flower_cookie,
1262	netdev: nft_m_entry->netdev);
1263	err_nft_ct_merge_insert:
1264	list_del(entry: &nft_m_entry->tc_merge_list);
1265	list_del(entry: &nft_m_entry->nft_flow_list);
1266	kfree(objp: nft_m_entry);
1267	return err;
1268	}
1269
1270	static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt,
1271	struct nfp_fl_ct_flow_entry *ct_entry1,
1272	struct nfp_fl_ct_flow_entry *ct_entry2)
1273	{
1274	struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry;
1275	struct nfp_fl_ct_flow_entry *nft_entry, *nft_tmp;
1276	struct nfp_fl_ct_tc_merge *m_entry;
1277	unsigned long new_cookie[2];
1278	int err;
1279
1280	if (ct_entry1->type == CT_TYPE_PRE_CT) {
1281	pre_ct_entry = ct_entry1;
1282	post_ct_entry = ct_entry2;
1283	} else {
1284	post_ct_entry = ct_entry1;
1285	pre_ct_entry = ct_entry2;
1286	}
1287
1288	/* Checks that the chain_index of the filter matches the
1289	* chain_index of the GOTO action.
1290	*/
1291	if (post_ct_entry->chain_index != pre_ct_entry->goto_chain_index)
1292	return -EINVAL;
1293
1294	err = nfp_ct_merge_check(entry1: pre_ct_entry, entry2: post_ct_entry);
1295	if (err)
1296	return err;
1297
1298	new_cookie[0] = pre_ct_entry->cookie;
1299	new_cookie[1] = post_ct_entry->cookie;
1300	m_entry = get_hashentry(ht: &zt->tc_merge_tb, key: &new_cookie,
1301	params: nfp_tc_ct_merge_params, size: sizeof(*m_entry));
1302	if (IS_ERR(ptr: m_entry))
1303	return PTR_ERR(ptr: m_entry);
1304
1305	/* m_entry already present, not merging again */
1306	if (!memcmp(p: &new_cookie, q: m_entry->cookie, size: sizeof(new_cookie)))
1307	return 0;
1308
1309	memcpy(&m_entry->cookie, &new_cookie, sizeof(new_cookie));
1310	m_entry->zt = zt;
1311	m_entry->post_ct_parent = post_ct_entry;
1312	m_entry->pre_ct_parent = pre_ct_entry;
1313
1314	/* Add this entry to the pre_ct and post_ct lists */
1315	list_add(new: &m_entry->post_ct_list, head: &post_ct_entry->children);
1316	list_add(new: &m_entry->pre_ct_list, head: &pre_ct_entry->children);
1317	INIT_LIST_HEAD(list: &m_entry->children);
1318
1319	err = rhashtable_insert_fast(ht: &zt->tc_merge_tb, obj: &m_entry->hash_node,
1320	params: nfp_tc_ct_merge_params);
1321	if (err)
1322	goto err_ct_tc_merge_insert;
1323	zt->tc_merge_count++;
1324
1325	/* Merge with existing nft flows */
1326	list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list,
1327	list_node) {
1328	nfp_ct_do_nft_merge(zt, nft_entry, tc_m_entry: m_entry);
1329	}
1330
1331	return 0;
1332
1333	err_ct_tc_merge_insert:
1334	list_del(entry: &m_entry->post_ct_list);
1335	list_del(entry: &m_entry->pre_ct_list);
1336	kfree(objp: m_entry);
1337	return err;
1338	}
1339
1340	static struct
1341	nfp_fl_ct_zone_entry *get_nfp_zone_entry(struct nfp_flower_priv *priv,
1342	u16 zone, bool wildcarded)
1343	{
1344	struct nfp_fl_ct_zone_entry *zt;
1345	int err;
1346
1347	if (wildcarded && priv->ct_zone_wc)
1348	return priv->ct_zone_wc;
1349
1350	if (!wildcarded) {
1351	zt = get_hashentry(ht: &priv->ct_zone_table, key: &zone,
1352	params: nfp_zone_table_params, size: sizeof(*zt));
1353
1354	/* If priv is set this is an existing entry, just return it */
1355	if (IS_ERR(ptr: zt) || zt->priv)
1356	return zt;
1357	} else {
1358	zt = kzalloc(size: sizeof(*zt), GFP_KERNEL);
1359	if (!zt)
1360	return ERR_PTR(error: -ENOMEM);
1361	}
1362
1363	zt->zone = zone;
1364	zt->priv = priv;
1365	zt->nft = NULL;
1366
1367	/* init the various hash tables and lists */
1368	INIT_LIST_HEAD(list: &zt->pre_ct_list);
1369	INIT_LIST_HEAD(list: &zt->post_ct_list);
1370	INIT_LIST_HEAD(list: &zt->nft_flows_list);
1371
1372	err = rhashtable_init(ht: &zt->tc_merge_tb, params: &nfp_tc_ct_merge_params);
1373	if (err)
1374	goto err_tc_merge_tb_init;
1375
1376	err = rhashtable_init(ht: &zt->nft_merge_tb, params: &nfp_nft_ct_merge_params);
1377	if (err)
1378	goto err_nft_merge_tb_init;
1379
1380	if (wildcarded) {
1381	priv->ct_zone_wc = zt;
1382	} else {
1383	err = rhashtable_insert_fast(ht: &priv->ct_zone_table,
1384	obj: &zt->hash_node,
1385	params: nfp_zone_table_params);
1386	if (err)
1387	goto err_zone_insert;
1388	}
1389
1390	return zt;
1391
1392	err_zone_insert:
1393	rhashtable_destroy(ht: &zt->nft_merge_tb);
1394	err_nft_merge_tb_init:
1395	rhashtable_destroy(ht: &zt->tc_merge_tb);
1396	err_tc_merge_tb_init:
1397	kfree(objp: zt);
1398	return ERR_PTR(error: err);
1399	}
1400
1401	static struct net_device *get_netdev_from_rule(struct flow_rule *rule)
1402	{
1403	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_META)) {
1404	struct flow_match_meta match;
1405
1406	flow_rule_match_meta(rule, out: &match);
1407	if (match.key->ingress_ifindex & match.mask->ingress_ifindex)
1408	return __dev_get_by_index(net: &init_net,
1409	ifindex: match.key->ingress_ifindex);
1410	}
1411
1412	return NULL;
1413	}
1414
1415	static void nfp_nft_ct_translate_mangle_action(struct flow_action_entry *mangle_action)
1416	{
1417	if (mangle_action->id != FLOW_ACTION_MANGLE)
1418	return;
1419
1420	switch (mangle_action->mangle.htype) {
1421	case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
1422	case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
1423	mangle_action->mangle.val = (__force u32)cpu_to_be32(mangle_action->mangle.val);
1424	mangle_action->mangle.mask = (__force u32)cpu_to_be32(mangle_action->mangle.mask);
1425	return;
1426
1427	/* Both struct tcphdr and struct udphdr start with
1428	* __be16 source;
1429	* __be16 dest;
1430	* so we can use the same code for both.
1431	*/
1432	case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
1433	case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
1434	if (mangle_action->mangle.offset == offsetof(struct tcphdr, source)) {
1435	mangle_action->mangle.val =
1436	(__force u32)cpu_to_be32(mangle_action->mangle.val << 16);
1437	/* The mask of mangle action is inverse mask,
1438	* so clear the dest tp port with 0xFFFF to
1439	* instead of rotate-left operation.
1440	*/
1441	mangle_action->mangle.mask =
1442	(__force u32)cpu_to_be32(mangle_action->mangle.mask << 16 | 0xFFFF);
1443	}
1444	if (mangle_action->mangle.offset == offsetof(struct tcphdr, dest)) {
1445	mangle_action->mangle.offset = 0;
1446	mangle_action->mangle.val =
1447	(__force u32)cpu_to_be32(mangle_action->mangle.val);
1448	mangle_action->mangle.mask =
1449	(__force u32)cpu_to_be32(mangle_action->mangle.mask);
1450	}
1451	return;
1452
1453	default:
1454	return;
1455	}
1456	}
1457
1458	static int nfp_nft_ct_set_flow_flag(struct flow_action_entry *act,
1459	struct nfp_fl_ct_flow_entry *entry)
1460	{
1461	switch (act->id) {
1462	case FLOW_ACTION_CT:
1463	if (act->ct.action == TCA_CT_ACT_NAT)
1464	entry->flags |= NFP_FL_ACTION_DO_NAT;
1465	break;
1466
1467	case FLOW_ACTION_MANGLE:
1468	entry->flags |= NFP_FL_ACTION_DO_MANGLE;
1469	break;
1470
1471	default:
1472	break;
1473	}
1474
1475	return 0;
1476	}
1477
1478	static struct
1479	nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,
1480	struct net_device *netdev,
1481	struct flow_cls_offload *flow,
1482	bool is_nft, struct netlink_ext_ack *extack)
1483	{
1484	struct nf_flow_match *nft_match = NULL;
1485	struct nfp_fl_ct_flow_entry *entry;
1486	struct nfp_fl_ct_map_entry *map;
1487	struct flow_action_entry *act;
1488	int err, i;
1489
1490	entry = kzalloc(size: sizeof(*entry), GFP_KERNEL);
1491	if (!entry)
1492	return ERR_PTR(error: -ENOMEM);
1493
1494	entry->rule = flow_rule_alloc(num_actions: flow->rule->action.num_entries);
1495	if (!entry->rule) {
1496	err = -ENOMEM;
1497	goto err_pre_ct_rule;
1498	}
1499
1500	/* nft flows gets destroyed after callback return, so need
1501	* to do a full copy instead of just a reference.
1502	*/
1503	if (is_nft) {
1504	nft_match = kzalloc(size: sizeof(*nft_match), GFP_KERNEL);
1505	if (!nft_match) {
1506	err = -ENOMEM;
1507	goto err_pre_ct_act;
1508	}
1509	memcpy(&nft_match->dissector, flow->rule->match.dissector,
1510	sizeof(nft_match->dissector));
1511	memcpy(&nft_match->mask, flow->rule->match.mask,
1512	sizeof(nft_match->mask));
1513	memcpy(&nft_match->key, flow->rule->match.key,
1514	sizeof(nft_match->key));
1515	entry->rule->match.dissector = &nft_match->dissector;
1516	entry->rule->match.mask = &nft_match->mask;
1517	entry->rule->match.key = &nft_match->key;
1518
1519	if (!netdev)
1520	netdev = get_netdev_from_rule(rule: entry->rule);
1521	} else {
1522	entry->rule->match.dissector = flow->rule->match.dissector;
1523	entry->rule->match.mask = flow->rule->match.mask;
1524	entry->rule->match.key = flow->rule->match.key;
1525	}
1526
1527	entry->zt = zt;
1528	entry->netdev = netdev;
1529	entry->cookie = flow->cookie > 0 ? flow->cookie : (unsigned long)entry;
1530	entry->chain_index = flow->common.chain_index;
1531	entry->tun_offset = NFP_FL_CT_NO_TUN;
1532
1533	/* Copy over action data. Unfortunately we do not get a handle to the
1534	* original tcf_action data, and the flow objects gets destroyed, so we
1535	* cannot just save a pointer to this either, so need to copy over the
1536	* data unfortunately.
1537	*/
1538	entry->rule->action.num_entries = flow->rule->action.num_entries;
1539	flow_action_for_each(i, act, &flow->rule->action) {
1540	struct flow_action_entry *new_act;
1541
1542	new_act = &entry->rule->action.entries[i];
1543	memcpy(new_act, act, sizeof(struct flow_action_entry));
1544	/* nft entry mangle field is host byte order, need translate to
1545	* network byte order.
1546	*/
1547	if (is_nft)
1548	nfp_nft_ct_translate_mangle_action(mangle_action: new_act);
1549
1550	nfp_nft_ct_set_flow_flag(act: new_act, entry);
1551	/* Entunnel is a special case, need to allocate and copy
1552	* tunnel info.
1553	*/
1554	if (act->id == FLOW_ACTION_TUNNEL_ENCAP) {
1555	struct ip_tunnel_info *tun = act->tunnel;
1556	size_t tun_size = sizeof(*tun) + tun->options_len;
1557
1558	new_act->tunnel = kmemdup(p: tun, size: tun_size, GFP_ATOMIC);
1559	if (!new_act->tunnel) {
1560	err = -ENOMEM;
1561	goto err_pre_ct_tun_cp;
1562	}
1563	entry->tun_offset = i;
1564	}
1565	}
1566
1567	INIT_LIST_HEAD(list: &entry->children);
1568
1569	if (flow->cookie == 0)
1570	return entry;
1571
1572	/* Now add a ct map entry to flower-priv */
1573	map = get_hashentry(ht: &zt->priv->ct_map_table, key: &flow->cookie,
1574	params: nfp_ct_map_params, size: sizeof(*map));
1575	if (IS_ERR(ptr: map)) {
1576	NL_SET_ERR_MSG_MOD(extack,
1577	"offload error: ct map entry creation failed");
1578	err = -ENOMEM;
1579	goto err_ct_flow_insert;
1580	}
1581	map->cookie = flow->cookie;
1582	map->ct_entry = entry;
1583	err = rhashtable_insert_fast(ht: &zt->priv->ct_map_table,
1584	obj: &map->hash_node,
1585	params: nfp_ct_map_params);
1586	if (err) {
1587	NL_SET_ERR_MSG_MOD(extack,
1588	"offload error: ct map entry table add failed");
1589	goto err_map_insert;
1590	}
1591
1592	return entry;
1593
1594	err_map_insert:
1595	kfree(objp: map);
1596	err_ct_flow_insert:
1597	if (entry->tun_offset != NFP_FL_CT_NO_TUN)
1598	kfree(objp: entry->rule->action.entries[entry->tun_offset].tunnel);
1599	err_pre_ct_tun_cp:
1600	kfree(objp: nft_match);
1601	err_pre_ct_act:
1602	kfree(objp: entry->rule);
1603	err_pre_ct_rule:
1604	kfree(objp: entry);
1605	return ERR_PTR(error: err);
1606	}
1607
1608	static void cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge *m_entry)
1609	{
1610	struct nfp_fl_ct_zone_entry *zt;
1611	int err;
1612
1613	zt = m_entry->zt;
1614
1615	/* Flow is in HW, need to delete */
1616	if (m_entry->tc_flower_cookie) {
1617	err = nfp_fl_ct_del_offload(app: zt->priv->app, cookie: m_entry->tc_flower_cookie,
1618	netdev: m_entry->netdev);
1619	if (err)
1620	return;
1621	}
1622
1623	WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb,
1624	&m_entry->hash_node,
1625	nfp_nft_ct_merge_params));
1626	zt->nft_merge_count--;
1627	list_del(entry: &m_entry->tc_merge_list);
1628	list_del(entry: &m_entry->nft_flow_list);
1629
1630	if (m_entry->next_pre_ct_entry) {
1631	struct nfp_fl_ct_map_entry pre_ct_map_ent;
1632
1633	pre_ct_map_ent.ct_entry = m_entry->next_pre_ct_entry;
1634	pre_ct_map_ent.cookie = 0;
1635	nfp_fl_ct_del_flow(ct_map_ent: &pre_ct_map_ent);
1636	}
1637
1638	kfree(objp: m_entry);
1639	}
1640
1641	static void nfp_free_nft_merge_children(void *entry, bool is_nft_flow)
1642	{
1643	struct nfp_fl_nft_tc_merge *m_entry, *tmp;
1644
1645	/* These post entries are parts of two lists, one is a list of nft_entries
1646	* and the other is of from a list of tc_merge structures. Iterate
1647	* through the relevant list and cleanup the entries.
1648	*/
1649
1650	if (is_nft_flow) {
1651	/* Need to iterate through list of nft_flow entries */
1652	struct nfp_fl_ct_flow_entry *ct_entry = entry;
1653
1654	list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
1655	nft_flow_list) {
1656	cleanup_nft_merge_entry(m_entry);
1657	}
1658	} else {
1659	/* Need to iterate through list of tc_merged_flow entries */
1660	struct nfp_fl_ct_tc_merge *ct_entry = entry;
1661
1662	list_for_each_entry_safe(m_entry, tmp, &ct_entry->children,
1663	tc_merge_list) {
1664	cleanup_nft_merge_entry(m_entry);
1665	}
1666	}
1667	}
1668
1669	static void nfp_del_tc_merge_entry(struct nfp_fl_ct_tc_merge *m_ent)
1670	{
1671	struct nfp_fl_ct_zone_entry *zt;
1672	int err;
1673
1674	zt = m_ent->zt;
1675	err = rhashtable_remove_fast(ht: &zt->tc_merge_tb,
1676	obj: &m_ent->hash_node,
1677	params: nfp_tc_ct_merge_params);
1678	if (err)
1679	pr_warn("WARNING: could not remove merge_entry from hashtable\n");
1680	zt->tc_merge_count--;
1681	list_del(entry: &m_ent->post_ct_list);
1682	list_del(entry: &m_ent->pre_ct_list);
1683
1684	if (!list_empty(head: &m_ent->children))
1685	nfp_free_nft_merge_children(entry: m_ent, is_nft_flow: false);
1686	kfree(objp: m_ent);
1687	}
1688
1689	static void nfp_free_tc_merge_children(struct nfp_fl_ct_flow_entry *entry)
1690	{
1691	struct nfp_fl_ct_tc_merge *m_ent, *tmp;
1692
1693	switch (entry->type) {
1694	case CT_TYPE_PRE_CT:
1695	list_for_each_entry_safe(m_ent, tmp, &entry->children, pre_ct_list) {
1696	nfp_del_tc_merge_entry(m_ent);
1697	}
1698	break;
1699	case CT_TYPE_POST_CT:
1700	list_for_each_entry_safe(m_ent, tmp, &entry->children, post_ct_list) {
1701	nfp_del_tc_merge_entry(m_ent);
1702	}
1703	break;
1704	default:
1705	break;
1706	}
1707	}
1708
1709	void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry)
1710	{
1711	list_del(entry: &entry->list_node);
1712
1713	if (!list_empty(head: &entry->children)) {
1714	if (entry->type == CT_TYPE_NFT)
1715	nfp_free_nft_merge_children(entry, is_nft_flow: true);
1716	else
1717	nfp_free_tc_merge_children(entry);
1718	}
1719
1720	if (entry->tun_offset != NFP_FL_CT_NO_TUN)
1721	kfree(objp: entry->rule->action.entries[entry->tun_offset].tunnel);
1722
1723	if (entry->type == CT_TYPE_NFT) {
1724	struct nf_flow_match *nft_match;
1725
1726	nft_match = container_of(entry->rule->match.dissector,
1727	struct nf_flow_match, dissector);
1728	kfree(objp: nft_match);
1729	}
1730
1731	kfree(objp: entry->rule);
1732	kfree(objp: entry);
1733	}
1734
1735	static struct flow_action_entry *get_flow_act_ct(struct flow_rule *rule)
1736	{
1737	struct flow_action_entry *act;
1738	int i;
1739
1740	/* More than one ct action may be present in a flow rule,
1741	* Return the first one that is not a CT clear action
1742	*/
1743	flow_action_for_each(i, act, &rule->action) {
1744	if (act->id == FLOW_ACTION_CT && act->ct.action != TCA_CT_ACT_CLEAR)
1745	return act;
1746	}
1747
1748	return NULL;
1749	}
1750
1751	static struct flow_action_entry *get_flow_act(struct flow_rule *rule,
1752	enum flow_action_id act_id)
1753	{
1754	struct flow_action_entry *act = NULL;
1755	int i;
1756
1757	flow_action_for_each(i, act, &rule->action) {
1758	if (act->id == act_id)
1759	return act;
1760	}
1761	return NULL;
1762	}
1763
1764	static void
1765	nfp_ct_merge_tc_entries(struct nfp_fl_ct_flow_entry *ct_entry1,
1766	struct nfp_fl_ct_zone_entry *zt_src,
1767	struct nfp_fl_ct_zone_entry *zt_dst)
1768	{
1769	struct nfp_fl_ct_flow_entry *ct_entry2, *ct_tmp;
1770	struct list_head *ct_list;
1771
1772	if (ct_entry1->type == CT_TYPE_PRE_CT)
1773	ct_list = &zt_src->post_ct_list;
1774	else if (ct_entry1->type == CT_TYPE_POST_CT)
1775	ct_list = &zt_src->pre_ct_list;
1776	else
1777	return;
1778
1779	list_for_each_entry_safe(ct_entry2, ct_tmp, ct_list,
1780	list_node) {
1781	nfp_ct_do_tc_merge(zt: zt_dst, ct_entry1: ct_entry2, ct_entry2: ct_entry1);
1782	}
1783	}
1784
1785	static void
1786	nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry *nft_entry,
1787	struct nfp_fl_ct_zone_entry *zt)
1788	{
1789	struct nfp_fl_ct_tc_merge *tc_merge_entry;
1790	struct rhashtable_iter iter;
1791
1792	rhashtable_walk_enter(ht: &zt->tc_merge_tb, iter: &iter);
1793	rhashtable_walk_start(iter: &iter);
1794	while ((tc_merge_entry = rhashtable_walk_next(iter: &iter)) != NULL) {
1795	if (IS_ERR(ptr: tc_merge_entry))
1796	continue;
1797	rhashtable_walk_stop(iter: &iter);
1798	nfp_ct_do_nft_merge(zt, nft_entry, tc_m_entry: tc_merge_entry);
1799	rhashtable_walk_start(iter: &iter);
1800	}
1801	rhashtable_walk_stop(iter: &iter);
1802	rhashtable_walk_exit(iter: &iter);
1803	}
1804
1805	int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv,
1806	struct net_device *netdev,
1807	struct flow_cls_offload *flow,
1808	struct netlink_ext_ack *extack,
1809	struct nfp_fl_nft_tc_merge *m_entry)
1810	{
1811	struct flow_action_entry *ct_act, *ct_goto;
1812	struct nfp_fl_ct_flow_entry *ct_entry;
1813	struct nfp_fl_ct_zone_entry *zt;
1814	int err;
1815
1816	ct_act = get_flow_act_ct(rule: flow->rule);
1817	if (!ct_act) {
1818	NL_SET_ERR_MSG_MOD(extack,
1819	"unsupported offload: Conntrack action empty in conntrack offload");
1820	return -EOPNOTSUPP;
1821	}
1822
1823	ct_goto = get_flow_act(rule: flow->rule, act_id: FLOW_ACTION_GOTO);
1824	if (!ct_goto) {
1825	NL_SET_ERR_MSG_MOD(extack,
1826	"unsupported offload: Conntrack requires ACTION_GOTO");
1827	return -EOPNOTSUPP;
1828	}
1829
1830	zt = get_nfp_zone_entry(priv, zone: ct_act->ct.zone, wildcarded: false);
1831	if (IS_ERR(ptr: zt)) {
1832	NL_SET_ERR_MSG_MOD(extack,
1833	"offload error: Could not create zone table entry");
1834	return PTR_ERR(ptr: zt);
1835	}
1836
1837	if (!zt->nft) {
1838	zt->nft = ct_act->ct.flow_table;
1839	err = nf_flow_table_offload_add_cb(flow_table: zt->nft, cb: nfp_fl_ct_handle_nft_flow, cb_priv: zt);
1840	if (err) {
1841	NL_SET_ERR_MSG_MOD(extack,
1842	"offload error: Could not register nft_callback");
1843	return err;
1844	}
1845	}
1846
1847	/* Add entry to pre_ct_list */
1848	ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, is_nft: false, extack);
1849	if (IS_ERR(ptr: ct_entry))
1850	return PTR_ERR(ptr: ct_entry);
1851	ct_entry->type = CT_TYPE_PRE_CT;
1852	ct_entry->chain_index = flow->common.chain_index;
1853	ct_entry->goto_chain_index = ct_goto->chain_index;
1854
1855	if (m_entry) {
1856	struct nfp_fl_ct_flow_entry *pre_ct_entry;
1857	int i;
1858
1859	pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent;
1860	for (i = 0; i < pre_ct_entry->num_prev_m_entries; i++)
1861	ct_entry->prev_m_entries[i] = pre_ct_entry->prev_m_entries[i];
1862	ct_entry->prev_m_entries[i++] = m_entry;
1863	ct_entry->num_prev_m_entries = i;
1864
1865	m_entry->next_pre_ct_entry = ct_entry;
1866	}
1867
1868	list_add(new: &ct_entry->list_node, head: &zt->pre_ct_list);
1869	zt->pre_ct_count++;
1870
1871	nfp_ct_merge_tc_entries(ct_entry1: ct_entry, zt_src: zt, zt_dst: zt);
1872
1873	/* Need to check and merge with tables in the wc_zone as well */
1874	if (priv->ct_zone_wc)
1875	nfp_ct_merge_tc_entries(ct_entry1: ct_entry, zt_src: priv->ct_zone_wc, zt_dst: zt);
1876
1877	return 0;
1878	}
1879
1880	int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv,
1881	struct net_device *netdev,
1882	struct flow_cls_offload *flow,
1883	struct netlink_ext_ack *extack)
1884	{
1885	struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: flow);
1886	struct nfp_fl_ct_flow_entry *ct_entry;
1887	struct flow_action_entry *ct_goto;
1888	struct nfp_fl_ct_zone_entry *zt;
1889	struct flow_action_entry *act;
1890	bool wildcarded = false;
1891	struct flow_match_ct ct;
1892	int i;
1893
1894	flow_action_for_each(i, act, &rule->action) {
1895	switch (act->id) {
1896	case FLOW_ACTION_REDIRECT:
1897	case FLOW_ACTION_REDIRECT_INGRESS:
1898	case FLOW_ACTION_MIRRED:
1899	case FLOW_ACTION_MIRRED_INGRESS:
1900	if (act->dev->rtnl_link_ops &&
1901	!strcmp(act->dev->rtnl_link_ops->kind, "openvswitch")) {
1902	NL_SET_ERR_MSG_MOD(extack,
1903	"unsupported offload: out port is openvswitch internal port");
1904	return -EOPNOTSUPP;
1905	}
1906	break;
1907	default:
1908	break;
1909	}
1910	}
1911
1912	flow_rule_match_ct(rule, out: &ct);
1913	if (!ct.mask->ct_zone) {
1914	wildcarded = true;
1915	} else if (ct.mask->ct_zone != U16_MAX) {
1916	NL_SET_ERR_MSG_MOD(extack,
1917	"unsupported offload: partially wildcarded ct_zone is not supported");
1918	return -EOPNOTSUPP;
1919	}
1920
1921	zt = get_nfp_zone_entry(priv, zone: ct.key->ct_zone, wildcarded);
1922	if (IS_ERR(ptr: zt)) {
1923	NL_SET_ERR_MSG_MOD(extack,
1924	"offload error: Could not create zone table entry");
1925	return PTR_ERR(ptr: zt);
1926	}
1927
1928	/* Add entry to post_ct_list */
1929	ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, is_nft: false, extack);
1930	if (IS_ERR(ptr: ct_entry))
1931	return PTR_ERR(ptr: ct_entry);
1932
1933	ct_entry->type = CT_TYPE_POST_CT;
1934	ct_entry->chain_index = flow->common.chain_index;
1935	ct_goto = get_flow_act(rule: flow->rule, act_id: FLOW_ACTION_GOTO);
1936	ct_entry->goto_chain_index = ct_goto ? ct_goto->chain_index : 0;
1937	list_add(new: &ct_entry->list_node, head: &zt->post_ct_list);
1938	zt->post_ct_count++;
1939
1940	if (wildcarded) {
1941	/* Iterate through all zone tables if not empty, look for merges with
1942	* pre_ct entries and merge them.
1943	*/
1944	struct rhashtable_iter iter;
1945	struct nfp_fl_ct_zone_entry *zone_table;
1946
1947	rhashtable_walk_enter(ht: &priv->ct_zone_table, iter: &iter);
1948	rhashtable_walk_start(iter: &iter);
1949	while ((zone_table = rhashtable_walk_next(iter: &iter)) != NULL) {
1950	if (IS_ERR(ptr: zone_table))
1951	continue;
1952	rhashtable_walk_stop(iter: &iter);
1953	nfp_ct_merge_tc_entries(ct_entry1: ct_entry, zt_src: zone_table, zt_dst: zone_table);
1954	rhashtable_walk_start(iter: &iter);
1955	}
1956	rhashtable_walk_stop(iter: &iter);
1957	rhashtable_walk_exit(iter: &iter);
1958	} else {
1959	nfp_ct_merge_tc_entries(ct_entry1: ct_entry, zt_src: zt, zt_dst: zt);
1960	}
1961
1962	return 0;
1963	}
1964
1965	int nfp_fl_create_new_pre_ct(struct nfp_fl_nft_tc_merge *m_entry)
1966	{
1967	struct nfp_fl_ct_flow_entry *pre_ct_entry, *post_ct_entry;
1968	struct flow_cls_offload new_pre_ct_flow;
1969	int err;
1970
1971	pre_ct_entry = m_entry->tc_m_parent->pre_ct_parent;
1972	if (pre_ct_entry->num_prev_m_entries >= NFP_MAX_RECIRC_CT_ZONES - 1)
1973	return -1;
1974
1975	post_ct_entry = m_entry->tc_m_parent->post_ct_parent;
1976	memset(&new_pre_ct_flow, 0, sizeof(struct flow_cls_offload));
1977	new_pre_ct_flow.rule = post_ct_entry->rule;
1978	new_pre_ct_flow.common.chain_index = post_ct_entry->chain_index;
1979
1980	err = nfp_fl_ct_handle_pre_ct(priv: pre_ct_entry->zt->priv,
1981	netdev: pre_ct_entry->netdev,
1982	flow: &new_pre_ct_flow, NULL,
1983	m_entry);
1984	return err;
1985	}
1986
1987	static void
1988	nfp_fl_ct_sub_stats(struct nfp_fl_nft_tc_merge *nft_merge,
1989	enum ct_entry_type type, u64 *m_pkts,
1990	u64 *m_bytes, u64 *m_used)
1991	{
1992	struct nfp_flower_priv *priv = nft_merge->zt->priv;
1993	struct nfp_fl_payload *nfp_flow;
1994	u32 ctx_id;
1995
1996	nfp_flow = nft_merge->flow_pay;
1997	if (!nfp_flow)
1998	return;
1999
2000	ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id);
2001	*m_pkts += priv->stats[ctx_id].pkts;
2002	*m_bytes += priv->stats[ctx_id].bytes;
2003	*m_used = max_t(u64, *m_used, priv->stats[ctx_id].used);
2004
2005	/* If request is for a sub_flow which is part of a tunnel merged
2006	* flow then update stats from tunnel merged flows first.
2007	*/
2008	if (!list_empty(head: &nfp_flow->linked_flows))
2009	nfp_flower_update_merge_stats(app: priv->app, sub_flow: nfp_flow);
2010
2011	if (type != CT_TYPE_NFT) {
2012	/* Update nft cached stats */
2013	flow_stats_update(flow_stats: &nft_merge->nft_parent->stats,
2014	bytes: priv->stats[ctx_id].bytes,
2015	pkts: priv->stats[ctx_id].pkts,
2016	drops: 0, lastused: priv->stats[ctx_id].used,
2017	used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED);
2018	} else {
2019	/* Update pre_ct cached stats */
2020	flow_stats_update(flow_stats: &nft_merge->tc_m_parent->pre_ct_parent->stats,
2021	bytes: priv->stats[ctx_id].bytes,
2022	pkts: priv->stats[ctx_id].pkts,
2023	drops: 0, lastused: priv->stats[ctx_id].used,
2024	used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED);
2025	/* Update post_ct cached stats */
2026	flow_stats_update(flow_stats: &nft_merge->tc_m_parent->post_ct_parent->stats,
2027	bytes: priv->stats[ctx_id].bytes,
2028	pkts: priv->stats[ctx_id].pkts,
2029	drops: 0, lastused: priv->stats[ctx_id].used,
2030	used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED);
2031	}
2032
2033	/* Update previous pre_ct/post_ct/nft flow stats */
2034	if (nft_merge->tc_m_parent->pre_ct_parent->num_prev_m_entries > 0) {
2035	struct nfp_fl_nft_tc_merge *tmp_nft_merge;
2036	int i;
2037
2038	for (i = 0; i < nft_merge->tc_m_parent->pre_ct_parent->num_prev_m_entries; i++) {
2039	tmp_nft_merge = nft_merge->tc_m_parent->pre_ct_parent->prev_m_entries[i];
2040	flow_stats_update(flow_stats: &tmp_nft_merge->tc_m_parent->pre_ct_parent->stats,
2041	bytes: priv->stats[ctx_id].bytes,
2042	pkts: priv->stats[ctx_id].pkts,
2043	drops: 0, lastused: priv->stats[ctx_id].used,
2044	used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED);
2045	flow_stats_update(flow_stats: &tmp_nft_merge->tc_m_parent->post_ct_parent->stats,
2046	bytes: priv->stats[ctx_id].bytes,
2047	pkts: priv->stats[ctx_id].pkts,
2048	drops: 0, lastused: priv->stats[ctx_id].used,
2049	used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED);
2050	flow_stats_update(flow_stats: &tmp_nft_merge->nft_parent->stats,
2051	bytes: priv->stats[ctx_id].bytes,
2052	pkts: priv->stats[ctx_id].pkts,
2053	drops: 0, lastused: priv->stats[ctx_id].used,
2054	used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED);
2055	}
2056	}
2057
2058	/* Reset stats from the nfp */
2059	priv->stats[ctx_id].pkts = 0;
2060	priv->stats[ctx_id].bytes = 0;
2061	}
2062
2063	int nfp_fl_ct_stats(struct flow_cls_offload *flow,
2064	struct nfp_fl_ct_map_entry *ct_map_ent)
2065	{
2066	struct nfp_fl_ct_flow_entry *ct_entry = ct_map_ent->ct_entry;
2067	struct nfp_fl_nft_tc_merge *nft_merge, *nft_m_tmp;
2068	struct nfp_fl_ct_tc_merge *tc_merge, *tc_m_tmp;
2069
2070	u64 pkts = 0, bytes = 0, used = 0;
2071	u64 m_pkts, m_bytes, m_used;
2072
2073	spin_lock_bh(lock: &ct_entry->zt->priv->stats_lock);
2074
2075	if (ct_entry->type == CT_TYPE_PRE_CT) {
2076	/* Iterate tc_merge entries associated with this flow */
2077	list_for_each_entry_safe(tc_merge, tc_m_tmp, &ct_entry->children,
2078	pre_ct_list) {
2079	m_pkts = 0;
2080	m_bytes = 0;
2081	m_used = 0;
2082	/* Iterate nft_merge entries associated with this tc_merge flow */
2083	list_for_each_entry_safe(nft_merge, nft_m_tmp, &tc_merge->children,
2084	tc_merge_list) {
2085	nfp_fl_ct_sub_stats(nft_merge, type: CT_TYPE_PRE_CT,
2086	m_pkts: &m_pkts, m_bytes: &m_bytes, m_used: &m_used);
2087	}
2088	pkts += m_pkts;
2089	bytes += m_bytes;
2090	used = max_t(u64, used, m_used);
2091	/* Update post_ct partner */
2092	flow_stats_update(flow_stats: &tc_merge->post_ct_parent->stats,
2093	bytes: m_bytes, pkts: m_pkts, drops: 0, lastused: m_used,
2094	used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED);
2095	}
2096	} else if (ct_entry->type == CT_TYPE_POST_CT) {
2097	/* Iterate tc_merge entries associated with this flow */
2098	list_for_each_entry_safe(tc_merge, tc_m_tmp, &ct_entry->children,
2099	post_ct_list) {
2100	m_pkts = 0;
2101	m_bytes = 0;
2102	m_used = 0;
2103	/* Iterate nft_merge entries associated with this tc_merge flow */
2104	list_for_each_entry_safe(nft_merge, nft_m_tmp, &tc_merge->children,
2105	tc_merge_list) {
2106	nfp_fl_ct_sub_stats(nft_merge, type: CT_TYPE_POST_CT,
2107	m_pkts: &m_pkts, m_bytes: &m_bytes, m_used: &m_used);
2108	}
2109	pkts += m_pkts;
2110	bytes += m_bytes;
2111	used = max_t(u64, used, m_used);
2112	/* Update pre_ct partner */
2113	flow_stats_update(flow_stats: &tc_merge->pre_ct_parent->stats,
2114	bytes: m_bytes, pkts: m_pkts, drops: 0, lastused: m_used,
2115	used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED);
2116	}
2117	} else {
2118	/* Iterate nft_merge entries associated with this nft flow */
2119	list_for_each_entry_safe(nft_merge, nft_m_tmp, &ct_entry->children,
2120	nft_flow_list) {
2121	nfp_fl_ct_sub_stats(nft_merge, type: CT_TYPE_NFT,
2122	m_pkts: &pkts, m_bytes: &bytes, m_used: &used);
2123	}
2124	}
2125
2126	/* Add stats from this request to stats potentially cached by
2127	* previous requests.
2128	*/
2129	flow_stats_update(flow_stats: &ct_entry->stats, bytes, pkts, drops: 0, lastused: used,
2130	used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED);
2131	/* Finally update the flow stats from the original stats request */
2132	flow_stats_update(flow_stats: &flow->stats, bytes: ct_entry->stats.bytes,
2133	pkts: ct_entry->stats.pkts, drops: 0,
2134	lastused: ct_entry->stats.lastused,
2135	used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED);
2136	/* Stats has been synced to original flow, can now clear
2137	* the cache.
2138	*/
2139	ct_entry->stats.pkts = 0;
2140	ct_entry->stats.bytes = 0;
2141	spin_unlock_bh(lock: &ct_entry->zt->priv->stats_lock);
2142
2143	return 0;
2144	}
2145
2146	static bool
2147	nfp_fl_ct_offload_nft_supported(struct flow_cls_offload *flow)
2148	{
2149	struct flow_rule *flow_rule = flow->rule;
2150	struct flow_action *flow_action =
2151	&flow_rule->action;
2152	struct flow_action_entry *act;
2153	int i;
2154
2155	flow_action_for_each(i, act, flow_action) {
2156	if (act->id == FLOW_ACTION_CT_METADATA) {
2157	enum ip_conntrack_info ctinfo =
2158	act->ct_metadata.cookie & NFCT_INFOMASK;
2159
2160	return ctinfo != IP_CT_NEW;
2161	}
2162	}
2163
2164	return false;
2165	}
2166
2167	static int
2168	nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry *zt, struct flow_cls_offload *flow)
2169	{
2170	struct nfp_fl_ct_map_entry *ct_map_ent;
2171	struct nfp_fl_ct_flow_entry *ct_entry;
2172	struct netlink_ext_ack *extack = NULL;
2173
2174	extack = flow->common.extack;
2175	switch (flow->command) {
2176	case FLOW_CLS_REPLACE:
2177	if (!nfp_fl_ct_offload_nft_supported(flow))
2178	return -EOPNOTSUPP;
2179
2180	/* Netfilter can request offload multiple times for the same
2181	* flow - protect against adding duplicates.
2182	*/
2183	ct_map_ent = rhashtable_lookup_fast(ht: &zt->priv->ct_map_table, key: &flow->cookie,
2184	params: nfp_ct_map_params);
2185	if (!ct_map_ent) {
2186	ct_entry = nfp_fl_ct_add_flow(zt, NULL, flow, is_nft: true, extack);
2187	if (IS_ERR(ptr: ct_entry))
2188	return PTR_ERR(ptr: ct_entry);
2189	ct_entry->type = CT_TYPE_NFT;
2190	list_add(new: &ct_entry->list_node, head: &zt->nft_flows_list);
2191	zt->nft_flows_count++;
2192	nfp_ct_merge_nft_with_tc(nft_entry: ct_entry, zt);
2193	}
2194	return 0;
2195	case FLOW_CLS_DESTROY:
2196	ct_map_ent = rhashtable_lookup_fast(ht: &zt->priv->ct_map_table, key: &flow->cookie,
2197	params: nfp_ct_map_params);
2198	return nfp_fl_ct_del_flow(ct_map_ent);
2199	case FLOW_CLS_STATS:
2200	ct_map_ent = rhashtable_lookup_fast(ht: &zt->priv->ct_map_table, key: &flow->cookie,
2201	params: nfp_ct_map_params);
2202	if (ct_map_ent)
2203	return nfp_fl_ct_stats(flow, ct_map_ent);
2204	break;
2205	default:
2206	break;
2207	}
2208	return -EINVAL;
2209	}
2210
2211	int nfp_fl_ct_handle_nft_flow(enum tc_setup_type type, void *type_data, void *cb_priv)
2212	{
2213	struct flow_cls_offload *flow = type_data;
2214	struct nfp_fl_ct_zone_entry *zt = cb_priv;
2215	int err = -EOPNOTSUPP;
2216
2217	switch (type) {
2218	case TC_SETUP_CLSFLOWER:
2219	while (!mutex_trylock(lock: &zt->priv->nfp_fl_lock)) {
2220	if (!zt->nft) /* avoid deadlock */
2221	return err;
2222	msleep(msecs: 20);
2223	}
2224	err = nfp_fl_ct_offload_nft_flow(zt, flow);
2225	mutex_unlock(lock: &zt->priv->nfp_fl_lock);
2226	break;
2227	default:
2228	return -EOPNOTSUPP;
2229	}
2230	return err;
2231	}
2232
2233	static void
2234	nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry *zt)
2235	{
2236	struct nfp_fl_ct_flow_entry *nft_entry, *ct_tmp;
2237	struct nfp_fl_ct_map_entry *ct_map_ent;
2238
2239	list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list,
2240	list_node) {
2241	ct_map_ent = rhashtable_lookup_fast(ht: &zt->priv->ct_map_table,
2242	key: &nft_entry->cookie,
2243	params: nfp_ct_map_params);
2244	nfp_fl_ct_del_flow(ct_map_ent);
2245	}
2246	}
2247
2248	int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent)
2249	{
2250	struct nfp_fl_ct_flow_entry *ct_entry;
2251	struct nfp_fl_ct_zone_entry *zt;
2252	struct rhashtable *m_table;
2253	struct nf_flowtable *nft;
2254
2255	if (!ct_map_ent)
2256	return -ENOENT;
2257
2258	zt = ct_map_ent->ct_entry->zt;
2259	ct_entry = ct_map_ent->ct_entry;
2260	m_table = &zt->priv->ct_map_table;
2261
2262	switch (ct_entry->type) {
2263	case CT_TYPE_PRE_CT:
2264	zt->pre_ct_count--;
2265	if (ct_map_ent->cookie > 0)
2266	rhashtable_remove_fast(ht: m_table, obj: &ct_map_ent->hash_node,
2267	params: nfp_ct_map_params);
2268	nfp_fl_ct_clean_flow_entry(entry: ct_entry);
2269	if (ct_map_ent->cookie > 0)
2270	kfree(objp: ct_map_ent);
2271
2272	if (!zt->pre_ct_count && zt->nft) {
2273	nft = zt->nft;
2274	zt->nft = NULL; /* avoid deadlock */
2275	nf_flow_table_offload_del_cb(flow_table: nft,
2276	cb: nfp_fl_ct_handle_nft_flow,
2277	cb_priv: zt);
2278	nfp_fl_ct_clean_nft_entries(zt);
2279	}
2280	break;
2281	case CT_TYPE_POST_CT:
2282	zt->post_ct_count--;
2283	rhashtable_remove_fast(ht: m_table, obj: &ct_map_ent->hash_node,
2284	params: nfp_ct_map_params);
2285	nfp_fl_ct_clean_flow_entry(entry: ct_entry);
2286	kfree(objp: ct_map_ent);
2287	break;
2288	case CT_TYPE_NFT:
2289	zt->nft_flows_count--;
2290	rhashtable_remove_fast(ht: m_table, obj: &ct_map_ent->hash_node,
2291	params: nfp_ct_map_params);
2292	nfp_fl_ct_clean_flow_entry(entry: ct_map_ent->ct_entry);
2293	kfree(objp: ct_map_ent);
2294	break;
2295	default:
2296	break;
2297	}
2298
2299	return 0;
2300	}
2301