1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2	/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
3
4	#include <net/macsec.h>
5	#include <linux/mlx5/qp.h>
6	#include <linux/if_vlan.h>
7	#include <linux/mlx5/fs_helpers.h>
8	#include <linux/mlx5/macsec.h>
9	#include "fs_core.h"
10	#include "lib/macsec_fs.h"
11	#include "mlx5_core.h"
12
13	/* MACsec TX flow steering */
14	#define CRYPTO_NUM_MAXSEC_FTE BIT(15)
15	#define CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE 1
16
17	#define TX_CRYPTO_TABLE_LEVEL 0
18	#define TX_CRYPTO_TABLE_NUM_GROUPS 3
19	#define TX_CRYPTO_TABLE_MKE_GROUP_SIZE 1
20	#define TX_CRYPTO_TABLE_SA_GROUP_SIZE \
21	(CRYPTO_NUM_MAXSEC_FTE - (TX_CRYPTO_TABLE_MKE_GROUP_SIZE + \
22	CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE))
23	#define TX_CHECK_TABLE_LEVEL 1
24	#define TX_CHECK_TABLE_NUM_FTE 2
25	#define RX_CRYPTO_TABLE_LEVEL 0
26	#define RX_CHECK_TABLE_LEVEL 1
27	#define RX_ROCE_TABLE_LEVEL 2
28	#define RX_CHECK_TABLE_NUM_FTE 3
29	#define RX_ROCE_TABLE_NUM_FTE 2
30	#define RX_CRYPTO_TABLE_NUM_GROUPS 3
31	#define RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE \
32	((CRYPTO_NUM_MAXSEC_FTE - CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE) / 2)
33	#define RX_CRYPTO_TABLE_SA_RULE_WITHOUT_SCI_GROUP_SIZE \
34	(CRYPTO_NUM_MAXSEC_FTE - RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE)
35	#define RX_NUM_OF_RULES_PER_SA 2
36
37	#define RDMA_RX_ROCE_IP_TABLE_LEVEL 0
38	#define RDMA_RX_ROCE_MACSEC_OP_TABLE_LEVEL 1
39
40	#define MLX5_MACSEC_TAG_LEN 8 /* SecTAG length with ethertype and without the optional SCI */
41	#define MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK 0x23
42	#define MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET 0x8
43	#define MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET 0x5
44	#define MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT (0x1 << MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET)
45	#define MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI 0x8
46	#define MLX5_SECTAG_HEADER_SIZE_WITH_SCI (MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI + MACSEC_SCI_LEN)
47
48	/* MACsec RX flow steering */
49	#define MLX5_ETH_WQE_FT_META_MACSEC_MASK 0x3E
50
51	/* MACsec fs_id handling for steering */
52	#define macsec_fs_set_tx_fs_id(fs_id) (MLX5_ETH_WQE_FT_META_MACSEC | (fs_id) << 2)
53	#define macsec_fs_set_rx_fs_id(fs_id) ((fs_id) | BIT(30))
54
55	struct mlx5_sectag_header {
56	__be16 ethertype;
57	u8 tci_an;
58	u8 sl;
59	u32 pn;
60	u8 sci[MACSEC_SCI_LEN]; /* optional */
61	} __packed;
62
63	struct mlx5_roce_macsec_tx_rule {
64	u32 fs_id;
65	u16 gid_idx;
66	struct list_head entry;
67	struct mlx5_flow_handle *rule;
68	struct mlx5_modify_hdr *meta_modhdr;
69	};
70
71	struct mlx5_macsec_tx_rule {
72	struct mlx5_flow_handle *rule;
73	struct mlx5_pkt_reformat *pkt_reformat;
74	u32 fs_id;
75	};
76
77	struct mlx5_macsec_flow_table {
78	int num_groups;
79	struct mlx5_flow_table *t;
80	struct mlx5_flow_group **g;
81	};
82
83	struct mlx5_macsec_tables {
84	struct mlx5_macsec_flow_table ft_crypto;
85	struct mlx5_flow_handle *crypto_miss_rule;
86
87	struct mlx5_flow_table *ft_check;
88	struct mlx5_flow_group *ft_check_group;
89	struct mlx5_fc *check_miss_rule_counter;
90	struct mlx5_flow_handle *check_miss_rule;
91	struct mlx5_fc *check_rule_counter;
92
93	u32 refcnt;
94	};
95
96	struct mlx5_fs_id {
97	u32 id;
98	refcount_t refcnt;
99	sci_t sci;
100	struct rhash_head hash;
101	};
102
103	struct mlx5_macsec_device {
104	struct list_head macsec_devices_list_entry;
105	void *macdev;
106	struct xarray tx_id_xa;
107	struct xarray rx_id_xa;
108	};
109
110	struct mlx5_macsec_tx {
111	struct mlx5_flow_handle *crypto_mke_rule;
112	struct mlx5_flow_handle *check_rule;
113
114	struct ida tx_halloc;
115
116	struct mlx5_macsec_tables tables;
117
118	struct mlx5_flow_table *ft_rdma_tx;
119	};
120
121	struct mlx5_roce_macsec_rx_rule {
122	u32 fs_id;
123	u16 gid_idx;
124	struct mlx5_flow_handle *op;
125	struct mlx5_flow_handle *ip;
126	struct list_head entry;
127	};
128
129	struct mlx5_macsec_rx_rule {
130	struct mlx5_flow_handle *rule[RX_NUM_OF_RULES_PER_SA];
131	struct mlx5_modify_hdr *meta_modhdr;
132	};
133
134	struct mlx5_macsec_miss {
135	struct mlx5_flow_group *g;
136	struct mlx5_flow_handle *rule;
137	};
138
139	struct mlx5_macsec_rx_roce {
140	/* Flow table/rules in NIC domain, to check if it's a RoCE packet */
141	struct mlx5_flow_group *g;
142	struct mlx5_flow_table *ft;
143	struct mlx5_flow_handle *rule;
144	struct mlx5_modify_hdr *copy_modify_hdr;
145	struct mlx5_macsec_miss nic_miss;
146
147	/* Flow table/rule in RDMA domain, to check dgid */
148	struct mlx5_flow_table *ft_ip_check;
149	struct mlx5_flow_table *ft_macsec_op_check;
150	struct mlx5_macsec_miss miss;
151	};
152
153	struct mlx5_macsec_rx {
154	struct mlx5_flow_handle *check_rule[2];
155	struct mlx5_pkt_reformat *check_rule_pkt_reformat[2];
156
157	struct mlx5_macsec_tables tables;
158	struct mlx5_macsec_rx_roce roce;
159	};
160
161	union mlx5_macsec_rule {
162	struct mlx5_macsec_tx_rule tx_rule;
163	struct mlx5_macsec_rx_rule rx_rule;
164	};
165
166	static const struct rhashtable_params rhash_sci = {
167	.key_len = sizeof_field(struct mlx5_fs_id, sci),
168	.key_offset = offsetof(struct mlx5_fs_id, sci),
169	.head_offset = offsetof(struct mlx5_fs_id, hash),
170	.automatic_shrinking = true,
171	.min_size = 1,
172	};
173
174	static const struct rhashtable_params rhash_fs_id = {
175	.key_len = sizeof_field(struct mlx5_fs_id, id),
176	.key_offset = offsetof(struct mlx5_fs_id, id),
177	.head_offset = offsetof(struct mlx5_fs_id, hash),
178	.automatic_shrinking = true,
179	.min_size = 1,
180	};
181
182	struct mlx5_macsec_fs {
183	struct mlx5_core_dev *mdev;
184	struct mlx5_macsec_tx *tx_fs;
185	struct mlx5_macsec_rx *rx_fs;
186
187	/* Stats manage */
188	struct mlx5_macsec_stats stats;
189
190	/* Tx sci -> fs id mapping handling */
191	struct rhashtable sci_hash; /* sci -> mlx5_fs_id */
192
193	/* RX fs_id -> mlx5_fs_id mapping handling */
194	struct rhashtable fs_id_hash; /* fs_id -> mlx5_fs_id */
195
196	/* TX & RX fs_id lists per macsec device */
197	struct list_head macsec_devices_list;
198	};
199
200	static void macsec_fs_destroy_groups(struct mlx5_macsec_flow_table *ft)
201	{
202	int i;
203
204	for (i = ft->num_groups - 1; i >= 0; i--) {
205	if (!IS_ERR_OR_NULL(ptr: ft->g[i]))
206	mlx5_destroy_flow_group(fg: ft->g[i]);
207	ft->g[i] = NULL;
208	}
209	ft->num_groups = 0;
210	}
211
212	static void macsec_fs_destroy_flow_table(struct mlx5_macsec_flow_table *ft)
213	{
214	macsec_fs_destroy_groups(ft);
215	kfree(objp: ft->g);
216	mlx5_destroy_flow_table(ft: ft->t);
217	ft->t = NULL;
218	}
219
220	static void macsec_fs_tx_destroy(struct mlx5_macsec_fs *macsec_fs)
221	{
222	struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs;
223	struct mlx5_macsec_tables *tx_tables;
224
225	if (mlx5_is_macsec_roce_supported(mdev: macsec_fs->mdev))
226	mlx5_destroy_flow_table(ft: tx_fs->ft_rdma_tx);
227
228	tx_tables = &tx_fs->tables;
229
230	/* Tx check table */
231	if (tx_fs->check_rule) {
232	mlx5_del_flow_rules(fr: tx_fs->check_rule);
233	tx_fs->check_rule = NULL;
234	}
235
236	if (tx_tables->check_miss_rule) {
237	mlx5_del_flow_rules(fr: tx_tables->check_miss_rule);
238	tx_tables->check_miss_rule = NULL;
239	}
240
241	if (tx_tables->ft_check_group) {
242	mlx5_destroy_flow_group(fg: tx_tables->ft_check_group);
243	tx_tables->ft_check_group = NULL;
244	}
245
246	if (tx_tables->ft_check) {
247	mlx5_destroy_flow_table(ft: tx_tables->ft_check);
248	tx_tables->ft_check = NULL;
249	}
250
251	/* Tx crypto table */
252	if (tx_fs->crypto_mke_rule) {
253	mlx5_del_flow_rules(fr: tx_fs->crypto_mke_rule);
254	tx_fs->crypto_mke_rule = NULL;
255	}
256
257	if (tx_tables->crypto_miss_rule) {
258	mlx5_del_flow_rules(fr: tx_tables->crypto_miss_rule);
259	tx_tables->crypto_miss_rule = NULL;
260	}
261
262	macsec_fs_destroy_flow_table(ft: &tx_tables->ft_crypto);
263	}
264
265	static int macsec_fs_tx_create_crypto_table_groups(struct mlx5_macsec_flow_table *ft)
266	{
267	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
268	int mclen = MLX5_ST_SZ_BYTES(fte_match_param);
269	int ix = 0;
270	u32 *in;
271	int err;
272	u8 *mc;
273
274	ft->g = kcalloc(TX_CRYPTO_TABLE_NUM_GROUPS, size: sizeof(*ft->g), GFP_KERNEL);
275	if (!ft->g)
276	return -ENOMEM;
277	in = kvzalloc(size: inlen, GFP_KERNEL);
278
279	if (!in) {
280	kfree(objp: ft->g);
281	ft->g = NULL;
282	return -ENOMEM;
283	}
284
285	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
286
287	/* Flow Group for MKE match */
288	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
289	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
290
291	MLX5_SET_CFG(in, start_flow_index, ix);
292	ix += TX_CRYPTO_TABLE_MKE_GROUP_SIZE;
293	MLX5_SET_CFG(in, end_flow_index, ix - 1);
294	ft->g[ft->num_groups] = mlx5_create_flow_group(ft: ft->t, in);
295	if (IS_ERR(ptr: ft->g[ft->num_groups]))
296	goto err;
297	ft->num_groups++;
298
299	/* Flow Group for SA rules */
300	memset(in, 0, inlen);
301	memset(mc, 0, mclen);
302	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_MISC_PARAMETERS_2);
303	MLX5_SET(fte_match_param, mc, misc_parameters_2.metadata_reg_a,
304	MLX5_ETH_WQE_FT_META_MACSEC_MASK);
305
306	MLX5_SET_CFG(in, start_flow_index, ix);
307	ix += TX_CRYPTO_TABLE_SA_GROUP_SIZE;
308	MLX5_SET_CFG(in, end_flow_index, ix - 1);
309	ft->g[ft->num_groups] = mlx5_create_flow_group(ft: ft->t, in);
310	if (IS_ERR(ptr: ft->g[ft->num_groups]))
311	goto err;
312	ft->num_groups++;
313
314	/* Flow Group for l2 traps */
315	memset(in, 0, inlen);
316	memset(mc, 0, mclen);
317	MLX5_SET_CFG(in, start_flow_index, ix);
318	ix += CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE;
319	MLX5_SET_CFG(in, end_flow_index, ix - 1);
320	ft->g[ft->num_groups] = mlx5_create_flow_group(ft: ft->t, in);
321	if (IS_ERR(ptr: ft->g[ft->num_groups]))
322	goto err;
323	ft->num_groups++;
324
325	kvfree(addr: in);
326	return 0;
327
328	err:
329	err = PTR_ERR(ptr: ft->g[ft->num_groups]);
330	ft->g[ft->num_groups] = NULL;
331	kvfree(addr: in);
332
333	return err;
334	}
335
336	static struct mlx5_flow_table
337	*macsec_fs_auto_group_table_create(struct mlx5_flow_namespace *ns, int flags,
338	int level, int max_fte)
339	{
340	struct mlx5_flow_table_attr ft_attr = {};
341	struct mlx5_flow_table *fdb = NULL;
342
343	/* reserve entry for the match all miss group and rule */
344	ft_attr.autogroup.num_reserved_entries = 1;
345	ft_attr.autogroup.max_num_groups = 1;
346	ft_attr.prio = 0;
347	ft_attr.flags = flags;
348	ft_attr.level = level;
349	ft_attr.max_fte = max_fte;
350
351	fdb = mlx5_create_auto_grouped_flow_table(ns, ft_attr: &ft_attr);
352
353	return fdb;
354	}
355
356	enum {
357	RDMA_TX_MACSEC_LEVEL = 0,
358	};
359
360	static int macsec_fs_tx_roce_create(struct mlx5_macsec_fs *macsec_fs)
361	{
362	struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs;
363	struct mlx5_core_dev *mdev = macsec_fs->mdev;
364	struct mlx5_flow_namespace *ns;
365	struct mlx5_flow_table *ft;
366	int err;
367
368	if (!mlx5_is_macsec_roce_supported(mdev)) {
369	mlx5_core_dbg(mdev, "Failed to init RoCE MACsec, capabilities not supported\n");
370	return 0;
371	}
372
373	ns = mlx5_get_flow_namespace(dev: mdev, type: MLX5_FLOW_NAMESPACE_RDMA_TX_MACSEC);
374	if (!ns)
375	return -ENOMEM;
376
377	/* Tx RoCE crypto table */
378	ft = macsec_fs_auto_group_table_create(ns, flags: 0, level: RDMA_TX_MACSEC_LEVEL, CRYPTO_NUM_MAXSEC_FTE);
379	if (IS_ERR(ptr: ft)) {
380	err = PTR_ERR(ptr: ft);
381	mlx5_core_err(mdev, "Failed to create MACsec RoCE Tx crypto table err(%d)\n", err);
382	return err;
383	}
384	tx_fs->ft_rdma_tx = ft;
385
386	return 0;
387	}
388
389	static int macsec_fs_tx_create(struct mlx5_macsec_fs *macsec_fs)
390	{
391	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
392	struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs;
393	struct mlx5_core_dev *mdev = macsec_fs->mdev;
394	struct mlx5_flow_table_attr ft_attr = {};
395	struct mlx5_flow_destination dest = {};
396	struct mlx5_macsec_tables *tx_tables;
397	struct mlx5_flow_act flow_act = {};
398	struct mlx5_macsec_flow_table *ft_crypto;
399	struct mlx5_flow_table *flow_table;
400	struct mlx5_flow_group *flow_group;
401	struct mlx5_flow_namespace *ns;
402	struct mlx5_flow_handle *rule;
403	struct mlx5_flow_spec *spec;
404	u32 *flow_group_in;
405	int err;
406
407	ns = mlx5_get_flow_namespace(dev: mdev, type: MLX5_FLOW_NAMESPACE_EGRESS_MACSEC);
408	if (!ns)
409	return -ENOMEM;
410
411	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
412	if (!spec)
413	return -ENOMEM;
414
415	flow_group_in = kvzalloc(size: inlen, GFP_KERNEL);
416	if (!flow_group_in) {
417	err = -ENOMEM;
418	goto out_spec;
419	}
420
421	tx_tables = &tx_fs->tables;
422	ft_crypto = &tx_tables->ft_crypto;
423
424	/* Tx crypto table */
425	ft_attr.flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
426	ft_attr.level = TX_CRYPTO_TABLE_LEVEL;
427	ft_attr.max_fte = CRYPTO_NUM_MAXSEC_FTE;
428
429	flow_table = mlx5_create_flow_table(ns, ft_attr: &ft_attr);
430	if (IS_ERR(ptr: flow_table)) {
431	err = PTR_ERR(ptr: flow_table);
432	mlx5_core_err(mdev, "Failed to create MACsec Tx crypto table err(%d)\n", err);
433	goto out_flow_group;
434	}
435	ft_crypto->t = flow_table;
436
437	/* Tx crypto table groups */
438	err = macsec_fs_tx_create_crypto_table_groups(ft: ft_crypto);
439	if (err) {
440	mlx5_core_err(mdev,
441	"Failed to create default flow group for MACsec Tx crypto table err(%d)\n",
442	err);
443	goto err;
444	}
445
446	/* Tx crypto table MKE rule - MKE packets shouldn't be offloaded */
447	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
448
449	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
450	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ETH_P_PAE);
451	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW;
452
453	rule = mlx5_add_flow_rules(ft: ft_crypto->t, spec, flow_act: &flow_act, NULL, num_dest: 0);
454	if (IS_ERR(ptr: rule)) {
455	err = PTR_ERR(ptr: rule);
456	mlx5_core_err(mdev, "Failed to add MACsec TX MKE rule, err=%d\n", err);
457	goto err;
458	}
459	tx_fs->crypto_mke_rule = rule;
460
461	/* Tx crypto table Default miss rule */
462	memset(&flow_act, 0, sizeof(flow_act));
463	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW;
464	rule = mlx5_add_flow_rules(ft: ft_crypto->t, NULL, flow_act: &flow_act, NULL, num_dest: 0);
465	if (IS_ERR(ptr: rule)) {
466	err = PTR_ERR(ptr: rule);
467	mlx5_core_err(mdev, "Failed to add MACsec Tx table default miss rule %d\n", err);
468	goto err;
469	}
470	tx_tables->crypto_miss_rule = rule;
471
472	/* Tx check table */
473	flow_table = macsec_fs_auto_group_table_create(ns, flags: 0, TX_CHECK_TABLE_LEVEL,
474	TX_CHECK_TABLE_NUM_FTE);
475	if (IS_ERR(ptr: flow_table)) {
476	err = PTR_ERR(ptr: flow_table);
477	mlx5_core_err(mdev, "Fail to create MACsec TX check table, err(%d)\n", err);
478	goto err;
479	}
480	tx_tables->ft_check = flow_table;
481
482	/* Tx check table Default miss group/rule */
483	memset(flow_group_in, 0, inlen);
484	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_table->max_fte - 1);
485	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_table->max_fte - 1);
486	flow_group = mlx5_create_flow_group(ft: tx_tables->ft_check, in: flow_group_in);
487	if (IS_ERR(ptr: flow_group)) {
488	err = PTR_ERR(ptr: flow_group);
489	mlx5_core_err(mdev,
490	"Failed to create default flow group for MACsec Tx crypto table err(%d)\n",
491	err);
492	goto err;
493	}
494	tx_tables->ft_check_group = flow_group;
495
496	/* Tx check table default drop rule */
497	memset(&dest, 0, sizeof(struct mlx5_flow_destination));
498	memset(&flow_act, 0, sizeof(flow_act));
499	dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
500	dest.counter_id = mlx5_fc_id(counter: tx_tables->check_miss_rule_counter);
501	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT;
502	rule = mlx5_add_flow_rules(ft: tx_tables->ft_check, NULL, flow_act: &flow_act, dest: &dest, num_dest: 1);
503	if (IS_ERR(ptr: rule)) {
504	err = PTR_ERR(ptr: rule);
505	mlx5_core_err(mdev, "Failed to added MACsec tx check drop rule, err(%d)\n", err);
506	goto err;
507	}
508	tx_tables->check_miss_rule = rule;
509
510	/* Tx check table rule */
511	memset(spec, 0, sizeof(struct mlx5_flow_spec));
512	memset(&dest, 0, sizeof(struct mlx5_flow_destination));
513	memset(&flow_act, 0, sizeof(flow_act));
514
515	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4);
516	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 0);
517	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
518
519	flow_act.flags = FLOW_ACT_NO_APPEND;
520	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | MLX5_FLOW_CONTEXT_ACTION_COUNT;
521	dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
522	dest.counter_id = mlx5_fc_id(counter: tx_tables->check_rule_counter);
523	rule = mlx5_add_flow_rules(ft: tx_tables->ft_check, spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
524	if (IS_ERR(ptr: rule)) {
525	err = PTR_ERR(ptr: rule);
526	mlx5_core_err(mdev, "Failed to add MACsec check rule, err=%d\n", err);
527	goto err;
528	}
529	tx_fs->check_rule = rule;
530
531	err = macsec_fs_tx_roce_create(macsec_fs);
532	if (err)
533	goto err;
534
535	kvfree(addr: flow_group_in);
536	kvfree(addr: spec);
537	return 0;
538
539	err:
540	macsec_fs_tx_destroy(macsec_fs);
541	out_flow_group:
542	kvfree(addr: flow_group_in);
543	out_spec:
544	kvfree(addr: spec);
545	return err;
546	}
547
548	static int macsec_fs_tx_ft_get(struct mlx5_macsec_fs *macsec_fs)
549	{
550	struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs;
551	struct mlx5_macsec_tables *tx_tables;
552	int err = 0;
553
554	tx_tables = &tx_fs->tables;
555	if (tx_tables->refcnt)
556	goto out;
557
558	err = macsec_fs_tx_create(macsec_fs);
559	if (err)
560	return err;
561
562	out:
563	tx_tables->refcnt++;
564	return err;
565	}
566
567	static void macsec_fs_tx_ft_put(struct mlx5_macsec_fs *macsec_fs)
568	{
569	struct mlx5_macsec_tables *tx_tables = &macsec_fs->tx_fs->tables;
570
571	if (--tx_tables->refcnt)
572	return;
573
574	macsec_fs_tx_destroy(macsec_fs);
575	}
576
577	static int macsec_fs_tx_setup_fte(struct mlx5_macsec_fs *macsec_fs,
578	struct mlx5_flow_spec *spec,
579	struct mlx5_flow_act *flow_act,
580	u32 macsec_obj_id,
581	u32 *fs_id)
582	{
583	struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs;
584	int err = 0;
585	u32 id;
586
587	err = ida_alloc_range(&tx_fs->tx_halloc, min: 1,
588	MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES,
589	GFP_KERNEL);
590	if (err < 0)
591	return err;
592
593	id = err;
594	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
595
596	/* Metadata match */
597	MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_a,
598	MLX5_ETH_WQE_FT_META_MACSEC_MASK);
599	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_a,
600	macsec_fs_set_tx_fs_id(id));
601
602	*fs_id = id;
603	flow_act->crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC;
604	flow_act->crypto.obj_id = macsec_obj_id;
605
606	mlx5_core_dbg(macsec_fs->mdev, "Tx fte: macsec obj_id %u, fs_id %u\n", macsec_obj_id, id);
607	return 0;
608	}
609
610	static void macsec_fs_tx_create_sectag_header(const struct macsec_context *ctx,
611	char *reformatbf,
612	size_t *reformat_size)
613	{
614	const struct macsec_secy *secy = ctx->secy;
615	bool sci_present = macsec_send_sci(secy);
616	struct mlx5_sectag_header sectag = {};
617	const struct macsec_tx_sc *tx_sc;
618
619	tx_sc = &secy->tx_sc;
620	sectag.ethertype = htons(ETH_P_MACSEC);
621
622	if (sci_present) {
623	sectag.tci_an |= MACSEC_TCI_SC;
624	memcpy(&sectag.sci, &secy->sci,
625	sizeof(sectag.sci));
626	} else {
627	if (tx_sc->end_station)
628	sectag.tci_an |= MACSEC_TCI_ES;
629	if (tx_sc->scb)
630	sectag.tci_an |= MACSEC_TCI_SCB;
631	}
632
633	/* With GCM, C/E clear for !encrypt, both set for encrypt */
634	if (tx_sc->encrypt)
635	sectag.tci_an |= MACSEC_TCI_CONFID;
636	else if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN)
637	sectag.tci_an |= MACSEC_TCI_C;
638
639	sectag.tci_an |= tx_sc->encoding_sa;
640
641	*reformat_size = MLX5_MACSEC_TAG_LEN + (sci_present ? MACSEC_SCI_LEN : 0);
642
643	memcpy(reformatbf, &sectag, *reformat_size);
644	}
645
646	static bool macsec_fs_is_macsec_device_empty(struct mlx5_macsec_device *macsec_device)
647	{
648	if (xa_empty(xa: &macsec_device->tx_id_xa) &&
649	xa_empty(xa: &macsec_device->rx_id_xa))
650	return true;
651
652	return false;
653	}
654
655	static void macsec_fs_id_del(struct list_head *macsec_devices_list, u32 fs_id,
656	void *macdev, struct rhashtable *hash_table, bool is_tx)
657	{
658	const struct rhashtable_params *rhash = (is_tx) ? &rhash_sci : &rhash_fs_id;
659	struct mlx5_macsec_device *iter, *macsec_device = NULL;
660	struct mlx5_fs_id *fs_id_found;
661	struct xarray *fs_id_xa;
662
663	list_for_each_entry(iter, macsec_devices_list, macsec_devices_list_entry) {
664	if (iter->macdev == macdev) {
665	macsec_device = iter;
666	break;
667	}
668	}
669	WARN_ON(!macsec_device);
670
671	fs_id_xa = (is_tx) ? &macsec_device->tx_id_xa :
672	&macsec_device->rx_id_xa;
673	xa_lock(fs_id_xa);
674	fs_id_found = xa_load(fs_id_xa, index: fs_id);
675	WARN_ON(!fs_id_found);
676
677	if (!refcount_dec_and_test(r: &fs_id_found->refcnt)) {
678	xa_unlock(fs_id_xa);
679	return;
680	}
681
682	if (fs_id_found->id) {
683	/* Make sure ongoing datapath readers sees a valid SA */
684	rhashtable_remove_fast(ht: hash_table, obj: &fs_id_found->hash, params: *rhash);
685	fs_id_found->id = 0;
686	}
687	xa_unlock(fs_id_xa);
688
689	xa_erase(fs_id_xa, index: fs_id);
690
691	kfree(objp: fs_id_found);
692
693	if (macsec_fs_is_macsec_device_empty(macsec_device)) {
694	list_del(entry: &macsec_device->macsec_devices_list_entry);
695	kfree(objp: macsec_device);
696	}
697	}
698
699	static int macsec_fs_id_add(struct list_head *macsec_devices_list, u32 fs_id,
700	void *macdev, struct rhashtable *hash_table, sci_t sci,
701	bool is_tx)
702	{
703	const struct rhashtable_params *rhash = (is_tx) ? &rhash_sci : &rhash_fs_id;
704	struct mlx5_macsec_device *iter, *macsec_device = NULL;
705	struct mlx5_fs_id *fs_id_iter;
706	struct xarray *fs_id_xa;
707	int err;
708
709	if (!is_tx) {
710	rcu_read_lock();
711	fs_id_iter = rhashtable_lookup(ht: hash_table, key: &fs_id, params: rhash_fs_id);
712	if (fs_id_iter) {
713	refcount_inc(r: &fs_id_iter->refcnt);
714	rcu_read_unlock();
715	return 0;
716	}
717	rcu_read_unlock();
718	}
719
720	fs_id_iter = kzalloc(size: sizeof(*fs_id_iter), GFP_KERNEL);
721	if (!fs_id_iter)
722	return -ENOMEM;
723
724	list_for_each_entry(iter, macsec_devices_list, macsec_devices_list_entry) {
725	if (iter->macdev == macdev) {
726	macsec_device = iter;
727	break;
728	}
729	}
730
731	if (!macsec_device) { /* first time adding a SA to that device */
732	macsec_device = kzalloc(size: sizeof(*macsec_device), GFP_KERNEL);
733	if (!macsec_device) {
734	err = -ENOMEM;
735	goto err_alloc_dev;
736	}
737	macsec_device->macdev = macdev;
738	xa_init(xa: &macsec_device->tx_id_xa);
739	xa_init(xa: &macsec_device->rx_id_xa);
740	list_add(new: &macsec_device->macsec_devices_list_entry, head: macsec_devices_list);
741	}
742
743	fs_id_xa = (is_tx) ? &macsec_device->tx_id_xa :
744	&macsec_device->rx_id_xa;
745	fs_id_iter->id = fs_id;
746	refcount_set(r: &fs_id_iter->refcnt, n: 1);
747	fs_id_iter->sci = sci;
748	err = xa_err(entry: xa_store(fs_id_xa, index: fs_id, entry: fs_id_iter, GFP_KERNEL));
749	if (err)
750	goto err_store_id;
751
752	err = rhashtable_insert_fast(ht: hash_table, obj: &fs_id_iter->hash, params: *rhash);
753	if (err)
754	goto err_hash_insert;
755
756	return 0;
757
758	err_hash_insert:
759	xa_erase(fs_id_xa, index: fs_id);
760	err_store_id:
761	if (macsec_fs_is_macsec_device_empty(macsec_device)) {
762	list_del(entry: &macsec_device->macsec_devices_list_entry);
763	kfree(objp: macsec_device);
764	}
765	err_alloc_dev:
766	kfree(objp: fs_id_iter);
767	return err;
768	}
769
770	static void macsec_fs_tx_del_rule(struct mlx5_macsec_fs *macsec_fs,
771	struct mlx5_macsec_tx_rule *tx_rule,
772	void *macdev)
773	{
774	macsec_fs_id_del(macsec_devices_list: &macsec_fs->macsec_devices_list, fs_id: tx_rule->fs_id, macdev,
775	hash_table: &macsec_fs->sci_hash, is_tx: true);
776
777	if (tx_rule->rule) {
778	mlx5_del_flow_rules(fr: tx_rule->rule);
779	tx_rule->rule = NULL;
780	}
781
782	if (tx_rule->pkt_reformat) {
783	mlx5_packet_reformat_dealloc(dev: macsec_fs->mdev, reformat: tx_rule->pkt_reformat);
784	tx_rule->pkt_reformat = NULL;
785	}
786
787	if (tx_rule->fs_id) {
788	ida_free(&macsec_fs->tx_fs->tx_halloc, id: tx_rule->fs_id);
789	tx_rule->fs_id = 0;
790	}
791
792	kfree(objp: tx_rule);
793
794	macsec_fs_tx_ft_put(macsec_fs);
795	}
796
797	#define MLX5_REFORMAT_PARAM_ADD_MACSEC_OFFSET_4_BYTES 1
798
799	static union mlx5_macsec_rule *
800	macsec_fs_tx_add_rule(struct mlx5_macsec_fs *macsec_fs,
801	const struct macsec_context *macsec_ctx,
802	struct mlx5_macsec_rule_attrs *attrs, u32 *fs_id)
803	{
804	char reformatbf[MLX5_MACSEC_TAG_LEN + MACSEC_SCI_LEN];
805	struct mlx5_pkt_reformat_params reformat_params = {};
806	struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs;
807	struct mlx5_core_dev *mdev = macsec_fs->mdev;
808	union mlx5_macsec_rule *macsec_rule = NULL;
809	struct mlx5_flow_destination dest = {};
810	struct mlx5_macsec_tables *tx_tables;
811	struct mlx5_macsec_tx_rule *tx_rule;
812	struct mlx5_flow_act flow_act = {};
813	struct mlx5_flow_handle *rule;
814	struct mlx5_flow_spec *spec;
815	size_t reformat_size;
816	int err = 0;
817
818	tx_tables = &tx_fs->tables;
819
820	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
821	if (!spec)
822	return NULL;
823
824	err = macsec_fs_tx_ft_get(macsec_fs);
825	if (err)
826	goto out_spec;
827
828	macsec_rule = kzalloc(size: sizeof(*macsec_rule), GFP_KERNEL);
829	if (!macsec_rule) {
830	macsec_fs_tx_ft_put(macsec_fs);
831	goto out_spec;
832	}
833
834	tx_rule = &macsec_rule->tx_rule;
835
836	/* Tx crypto table crypto rule */
837	macsec_fs_tx_create_sectag_header(ctx: macsec_ctx, reformatbf, reformat_size: &reformat_size);
838
839	reformat_params.type = MLX5_REFORMAT_TYPE_ADD_MACSEC;
840	reformat_params.size = reformat_size;
841	reformat_params.data = reformatbf;
842
843	if (is_vlan_dev(dev: macsec_ctx->netdev))
844	reformat_params.param_0 = MLX5_REFORMAT_PARAM_ADD_MACSEC_OFFSET_4_BYTES;
845
846	flow_act.pkt_reformat = mlx5_packet_reformat_alloc(dev: mdev,
847	params: &reformat_params,
848	ns_type: MLX5_FLOW_NAMESPACE_EGRESS_MACSEC);
849	if (IS_ERR(ptr: flow_act.pkt_reformat)) {
850	err = PTR_ERR(ptr: flow_act.pkt_reformat);
851	mlx5_core_err(mdev, "Failed to allocate MACsec Tx reformat context err=%d\n", err);
852	goto err;
853	}
854	tx_rule->pkt_reformat = flow_act.pkt_reformat;
855
856	err = macsec_fs_tx_setup_fte(macsec_fs, spec, flow_act: &flow_act, macsec_obj_id: attrs->macsec_obj_id, fs_id);
857	if (err) {
858	mlx5_core_err(mdev,
859	"Failed to add packet reformat for MACsec TX crypto rule, err=%d\n",
860	err);
861	goto err;
862	}
863
864	tx_rule->fs_id = *fs_id;
865
866	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
867	MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT |
868	MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
869	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
870	dest.ft = tx_tables->ft_check;
871	rule = mlx5_add_flow_rules(ft: tx_tables->ft_crypto.t, spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
872	if (IS_ERR(ptr: rule)) {
873	err = PTR_ERR(ptr: rule);
874	mlx5_core_err(mdev, "Failed to add MACsec TX crypto rule, err=%d\n", err);
875	goto err;
876	}
877	tx_rule->rule = rule;
878
879	err = macsec_fs_id_add(macsec_devices_list: &macsec_fs->macsec_devices_list, fs_id: *fs_id, macdev: macsec_ctx->secy->netdev,
880	hash_table: &macsec_fs->sci_hash, sci: attrs->sci, is_tx: true);
881	if (err) {
882	mlx5_core_err(mdev, "Failed to save fs_id, err=%d\n", err);
883	goto err;
884	}
885
886	goto out_spec;
887
888	err:
889	macsec_fs_tx_del_rule(macsec_fs, tx_rule, macdev: macsec_ctx->secy->netdev);
890	macsec_rule = NULL;
891	out_spec:
892	kvfree(addr: spec);
893
894	return macsec_rule;
895	}
896
897	static void macsec_fs_tx_cleanup(struct mlx5_macsec_fs *macsec_fs)
898	{
899	struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs;
900	struct mlx5_core_dev *mdev = macsec_fs->mdev;
901	struct mlx5_macsec_tables *tx_tables;
902
903	if (!tx_fs)
904	return;
905
906	tx_tables = &tx_fs->tables;
907	if (tx_tables->refcnt) {
908	mlx5_core_err(mdev,
909	"Can't destroy MACsec offload tx_fs, refcnt(%u) isn't 0\n",
910	tx_tables->refcnt);
911	return;
912	}
913
914	ida_destroy(ida: &tx_fs->tx_halloc);
915
916	if (tx_tables->check_miss_rule_counter) {
917	mlx5_fc_destroy(dev: mdev, counter: tx_tables->check_miss_rule_counter);
918	tx_tables->check_miss_rule_counter = NULL;
919	}
920
921	if (tx_tables->check_rule_counter) {
922	mlx5_fc_destroy(dev: mdev, counter: tx_tables->check_rule_counter);
923	tx_tables->check_rule_counter = NULL;
924	}
925
926	kfree(objp: tx_fs);
927	macsec_fs->tx_fs = NULL;
928	}
929
930	static int macsec_fs_tx_init(struct mlx5_macsec_fs *macsec_fs)
931	{
932	struct mlx5_core_dev *mdev = macsec_fs->mdev;
933	struct mlx5_macsec_tables *tx_tables;
934	struct mlx5_macsec_tx *tx_fs;
935	struct mlx5_fc *flow_counter;
936	int err;
937
938	tx_fs = kzalloc(size: sizeof(*tx_fs), GFP_KERNEL);
939	if (!tx_fs)
940	return -ENOMEM;
941
942	tx_tables = &tx_fs->tables;
943
944	flow_counter = mlx5_fc_create(dev: mdev, aging: false);
945	if (IS_ERR(ptr: flow_counter)) {
946	err = PTR_ERR(ptr: flow_counter);
947	mlx5_core_err(mdev,
948	"Failed to create MACsec Tx encrypt flow counter, err(%d)\n",
949	err);
950	goto err_encrypt_counter;
951	}
952	tx_tables->check_rule_counter = flow_counter;
953
954	flow_counter = mlx5_fc_create(dev: mdev, aging: false);
955	if (IS_ERR(ptr: flow_counter)) {
956	err = PTR_ERR(ptr: flow_counter);
957	mlx5_core_err(mdev,
958	"Failed to create MACsec Tx drop flow counter, err(%d)\n",
959	err);
960	goto err_drop_counter;
961	}
962	tx_tables->check_miss_rule_counter = flow_counter;
963
964	ida_init(ida: &tx_fs->tx_halloc);
965	INIT_LIST_HEAD(list: &macsec_fs->macsec_devices_list);
966
967	macsec_fs->tx_fs = tx_fs;
968
969	return 0;
970
971	err_drop_counter:
972	mlx5_fc_destroy(dev: mdev, counter: tx_tables->check_rule_counter);
973	tx_tables->check_rule_counter = NULL;
974
975	err_encrypt_counter:
976	kfree(objp: tx_fs);
977	macsec_fs->tx_fs = NULL;
978
979	return err;
980	}
981
982	static void macsec_fs_rx_roce_miss_destroy(struct mlx5_macsec_miss *miss)
983	{
984	mlx5_del_flow_rules(fr: miss->rule);
985	mlx5_destroy_flow_group(fg: miss->g);
986	}
987
988	static void macsec_fs_rdma_rx_destroy(struct mlx5_macsec_rx_roce *roce, struct mlx5_core_dev *mdev)
989	{
990	if (!mlx5_is_macsec_roce_supported(mdev))
991	return;
992
993	mlx5_del_flow_rules(fr: roce->nic_miss.rule);
994	mlx5_del_flow_rules(fr: roce->rule);
995	mlx5_modify_header_dealloc(dev: mdev, modify_hdr: roce->copy_modify_hdr);
996	mlx5_destroy_flow_group(fg: roce->nic_miss.g);
997	mlx5_destroy_flow_group(fg: roce->g);
998	mlx5_destroy_flow_table(ft: roce->ft);
999
1000	macsec_fs_rx_roce_miss_destroy(miss: &roce->miss);
1001	mlx5_destroy_flow_table(ft: roce->ft_macsec_op_check);
1002	mlx5_destroy_flow_table(ft: roce->ft_ip_check);
1003	}
1004
1005	static void macsec_fs_rx_destroy(struct mlx5_macsec_fs *macsec_fs)
1006	{
1007	struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs;
1008	struct mlx5_macsec_tables *rx_tables;
1009	int i;
1010
1011	/* Rx check table */
1012	for (i = 1; i >= 0; --i) {
1013	if (rx_fs->check_rule[i]) {
1014	mlx5_del_flow_rules(fr: rx_fs->check_rule[i]);
1015	rx_fs->check_rule[i] = NULL;
1016	}
1017
1018	if (rx_fs->check_rule_pkt_reformat[i]) {
1019	mlx5_packet_reformat_dealloc(dev: macsec_fs->mdev,
1020	reformat: rx_fs->check_rule_pkt_reformat[i]);
1021	rx_fs->check_rule_pkt_reformat[i] = NULL;
1022	}
1023	}
1024
1025	rx_tables = &rx_fs->tables;
1026
1027	if (rx_tables->check_miss_rule) {
1028	mlx5_del_flow_rules(fr: rx_tables->check_miss_rule);
1029	rx_tables->check_miss_rule = NULL;
1030	}
1031
1032	if (rx_tables->ft_check_group) {
1033	mlx5_destroy_flow_group(fg: rx_tables->ft_check_group);
1034	rx_tables->ft_check_group = NULL;
1035	}
1036
1037	if (rx_tables->ft_check) {
1038	mlx5_destroy_flow_table(ft: rx_tables->ft_check);
1039	rx_tables->ft_check = NULL;
1040	}
1041
1042	/* Rx crypto table */
1043	if (rx_tables->crypto_miss_rule) {
1044	mlx5_del_flow_rules(fr: rx_tables->crypto_miss_rule);
1045	rx_tables->crypto_miss_rule = NULL;
1046	}
1047
1048	macsec_fs_destroy_flow_table(ft: &rx_tables->ft_crypto);
1049
1050	macsec_fs_rdma_rx_destroy(roce: &macsec_fs->rx_fs->roce, mdev: macsec_fs->mdev);
1051	}
1052
1053	static int macsec_fs_rx_create_crypto_table_groups(struct mlx5_macsec_flow_table *ft)
1054	{
1055	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1056	int mclen = MLX5_ST_SZ_BYTES(fte_match_param);
1057	int ix = 0;
1058	u32 *in;
1059	int err;
1060	u8 *mc;
1061
1062	ft->g = kcalloc(RX_CRYPTO_TABLE_NUM_GROUPS, size: sizeof(*ft->g), GFP_KERNEL);
1063	if (!ft->g)
1064	return -ENOMEM;
1065
1066	in = kvzalloc(size: inlen, GFP_KERNEL);
1067	if (!in) {
1068	kfree(objp: ft->g);
1069	return -ENOMEM;
1070	}
1071
1072	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
1073
1074	/* Flow group for SA rule with SCI */
1075	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS |
1076	MLX5_MATCH_MISC_PARAMETERS_5);
1077	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1078
1079	MLX5_SET(fte_match_param, mc, misc_parameters_5.macsec_tag_0,
1080	MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK <<
1081	MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
1082	MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters_5.macsec_tag_2);
1083	MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters_5.macsec_tag_3);
1084
1085	MLX5_SET_CFG(in, start_flow_index, ix);
1086	ix += RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE;
1087	MLX5_SET_CFG(in, end_flow_index, ix - 1);
1088	ft->g[ft->num_groups] = mlx5_create_flow_group(ft: ft->t, in);
1089	if (IS_ERR(ptr: ft->g[ft->num_groups]))
1090	goto err;
1091	ft->num_groups++;
1092
1093	/* Flow group for SA rule without SCI */
1094	memset(in, 0, inlen);
1095	memset(mc, 0, mclen);
1096	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS |
1097	MLX5_MATCH_MISC_PARAMETERS_5);
1098	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.smac_47_16);
1099	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.smac_15_0);
1100	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
1101
1102	MLX5_SET(fte_match_param, mc, misc_parameters_5.macsec_tag_0,
1103	MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
1104
1105	MLX5_SET_CFG(in, start_flow_index, ix);
1106	ix += RX_CRYPTO_TABLE_SA_RULE_WITHOUT_SCI_GROUP_SIZE;
1107	MLX5_SET_CFG(in, end_flow_index, ix - 1);
1108	ft->g[ft->num_groups] = mlx5_create_flow_group(ft: ft->t, in);
1109	if (IS_ERR(ptr: ft->g[ft->num_groups]))
1110	goto err;
1111	ft->num_groups++;
1112
1113	/* Flow Group for l2 traps */
1114	memset(in, 0, inlen);
1115	memset(mc, 0, mclen);
1116	MLX5_SET_CFG(in, start_flow_index, ix);
1117	ix += CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE;
1118	MLX5_SET_CFG(in, end_flow_index, ix - 1);
1119	ft->g[ft->num_groups] = mlx5_create_flow_group(ft: ft->t, in);
1120	if (IS_ERR(ptr: ft->g[ft->num_groups]))
1121	goto err;
1122	ft->num_groups++;
1123
1124	kvfree(addr: in);
1125	return 0;
1126
1127	err:
1128	err = PTR_ERR(ptr: ft->g[ft->num_groups]);
1129	ft->g[ft->num_groups] = NULL;
1130	kvfree(addr: in);
1131
1132	return err;
1133	}
1134
1135	static int macsec_fs_rx_create_check_decap_rule(struct mlx5_macsec_fs *macsec_fs,
1136	struct mlx5_flow_destination *dest,
1137	struct mlx5_flow_act *flow_act,
1138	struct mlx5_flow_spec *spec,
1139	int reformat_param_size)
1140	{
1141	int rule_index = (reformat_param_size == MLX5_SECTAG_HEADER_SIZE_WITH_SCI) ? 0 : 1;
1142	u8 mlx5_reformat_buf[MLX5_SECTAG_HEADER_SIZE_WITH_SCI];
1143	struct mlx5_pkt_reformat_params reformat_params = {};
1144	struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs;
1145	struct mlx5_core_dev *mdev = macsec_fs->mdev;
1146	struct mlx5_flow_destination roce_dest[2];
1147	struct mlx5_macsec_tables *rx_tables;
1148	struct mlx5_flow_handle *rule;
1149	int err = 0, dstn = 0;
1150
1151	rx_tables = &rx_fs->tables;
1152
1153	/* Rx check table decap 16B rule */
1154	memset(dest, 0, sizeof(*dest));
1155	memset(flow_act, 0, sizeof(*flow_act));
1156	memset(spec, 0, sizeof(*spec));
1157
1158	reformat_params.type = MLX5_REFORMAT_TYPE_DEL_MACSEC;
1159	reformat_params.size = reformat_param_size;
1160	reformat_params.data = mlx5_reformat_buf;
1161	flow_act->pkt_reformat = mlx5_packet_reformat_alloc(dev: mdev,
1162	params: &reformat_params,
1163	ns_type: MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC);
1164	if (IS_ERR(ptr: flow_act->pkt_reformat)) {
1165	err = PTR_ERR(ptr: flow_act->pkt_reformat);
1166	mlx5_core_err(mdev, "Failed to allocate MACsec Rx reformat context err=%d\n", err);
1167	return err;
1168	}
1169	rx_fs->check_rule_pkt_reformat[rule_index] = flow_act->pkt_reformat;
1170
1171	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1172	/* MACsec syndrome match */
1173	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.macsec_syndrome);
1174	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.macsec_syndrome, 0);
1175	/* ASO return reg syndrome match */
1176	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4);
1177	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 0);
1178
1179	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5;
1180	/* Sectag TCI SC present bit*/
1181	MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_0,
1182	MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
1183
1184	if (reformat_param_size == MLX5_SECTAG_HEADER_SIZE_WITH_SCI)
1185	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_0,
1186	MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT <<
1187	MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
1188
1189	flow_act->flags = FLOW_ACT_NO_APPEND;
1190
1191	if (rx_fs->roce.ft) {
1192	flow_act->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1193	roce_dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1194	roce_dest[dstn].ft = rx_fs->roce.ft;
1195	dstn++;
1196	} else {
1197	flow_act->action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
1198	}
1199
1200	flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT |
1201	MLX5_FLOW_CONTEXT_ACTION_COUNT;
1202	roce_dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1203	roce_dest[dstn].counter_id = mlx5_fc_id(counter: rx_tables->check_rule_counter);
1204	rule = mlx5_add_flow_rules(ft: rx_tables->ft_check, spec, flow_act, dest: roce_dest, num_dest: dstn + 1);
1205
1206	if (IS_ERR(ptr: rule)) {
1207	err = PTR_ERR(ptr: rule);
1208	mlx5_core_err(mdev, "Failed to add MACsec Rx check rule, err=%d\n", err);
1209	return err;
1210	}
1211
1212	rx_fs->check_rule[rule_index] = rule;
1213
1214	return 0;
1215	}
1216
1217	static int macsec_fs_rx_roce_miss_create(struct mlx5_core_dev *mdev,
1218	struct mlx5_macsec_rx_roce *roce)
1219	{
1220	struct mlx5_flow_act flow_act = {};
1221	struct mlx5_flow_group *flow_group;
1222	struct mlx5_flow_handle *rule;
1223	u32 *flow_group_in;
1224	int err;
1225
1226	flow_group_in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
1227	if (!flow_group_in)
1228	return -ENOMEM;
1229
1230	/* IP check ft has no miss rule since we use default miss action which is go to next PRIO */
1231	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index,
1232	roce->ft_macsec_op_check->max_fte - 1);
1233	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1234	roce->ft_macsec_op_check->max_fte - 1);
1235	flow_group = mlx5_create_flow_group(ft: roce->ft_macsec_op_check, in: flow_group_in);
1236	if (IS_ERR(ptr: flow_group)) {
1237	err = PTR_ERR(ptr: flow_group);
1238	mlx5_core_err(mdev,
1239	"Failed to create miss flow group for MACsec RoCE operation check table err(%d)\n",
1240	err);
1241	goto err_macsec_op_miss_group;
1242	}
1243	roce->miss.g = flow_group;
1244
1245	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
1246	rule = mlx5_add_flow_rules(ft: roce->ft_macsec_op_check, NULL, flow_act: &flow_act, NULL, num_dest: 0);
1247	if (IS_ERR(ptr: rule)) {
1248	err = PTR_ERR(ptr: rule);
1249	mlx5_core_err(mdev, "Failed to add miss rule to MACsec RoCE operation check table err(%d)\n",
1250	err);
1251	goto err_macsec_op_rule;
1252	}
1253	roce->miss.rule = rule;
1254
1255	kvfree(addr: flow_group_in);
1256	return 0;
1257
1258	err_macsec_op_rule:
1259	mlx5_destroy_flow_group(fg: roce->miss.g);
1260	err_macsec_op_miss_group:
1261	kvfree(addr: flow_group_in);
1262	return err;
1263	}
1264
1265	#define MLX5_RX_ROCE_GROUP_SIZE BIT(0)
1266
1267	static int macsec_fs_rx_roce_jump_to_rdma_groups_create(struct mlx5_core_dev *mdev,
1268	struct mlx5_macsec_rx_roce *roce)
1269	{
1270	struct mlx5_flow_group *g;
1271	void *outer_headers_c;
1272	int ix = 0;
1273	u32 *in;
1274	int err;
1275	u8 *mc;
1276
1277	in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL);
1278	if (!in)
1279	return -ENOMEM;
1280
1281	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
1282	outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
1283	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
1284	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
1285
1286	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
1287	MLX5_SET_CFG(in, start_flow_index, ix);
1288	ix += MLX5_RX_ROCE_GROUP_SIZE;
1289	MLX5_SET_CFG(in, end_flow_index, ix - 1);
1290	g = mlx5_create_flow_group(ft: roce->ft, in);
1291	if (IS_ERR(ptr: g)) {
1292	err = PTR_ERR(ptr: g);
1293	mlx5_core_err(mdev, "Failed to create main flow group for MACsec RoCE NIC UDP table err(%d)\n",
1294	err);
1295	goto err_udp_group;
1296	}
1297	roce->g = g;
1298
1299	memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in));
1300	MLX5_SET_CFG(in, start_flow_index, ix);
1301	ix += MLX5_RX_ROCE_GROUP_SIZE;
1302	MLX5_SET_CFG(in, end_flow_index, ix - 1);
1303	g = mlx5_create_flow_group(ft: roce->ft, in);
1304	if (IS_ERR(ptr: g)) {
1305	err = PTR_ERR(ptr: g);
1306	mlx5_core_err(mdev, "Failed to create miss flow group for MACsec RoCE NIC UDP table err(%d)\n",
1307	err);
1308	goto err_udp_miss_group;
1309	}
1310	roce->nic_miss.g = g;
1311
1312	kvfree(addr: in);
1313	return 0;
1314
1315	err_udp_miss_group:
1316	mlx5_destroy_flow_group(fg: roce->g);
1317	err_udp_group:
1318	kvfree(addr: in);
1319	return err;
1320	}
1321
1322	static int macsec_fs_rx_roce_jump_to_rdma_rules_create(struct mlx5_macsec_fs *macsec_fs,
1323	struct mlx5_macsec_rx_roce *roce)
1324	{
1325	u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
1326	struct mlx5_core_dev *mdev = macsec_fs->mdev;
1327	struct mlx5_flow_destination dst = {};
1328	struct mlx5_modify_hdr *modify_hdr;
1329	MLX5_DECLARE_FLOW_ACT(flow_act);
1330	struct mlx5_flow_handle *rule;
1331	struct mlx5_flow_spec *spec;
1332	int err;
1333
1334	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
1335	if (!spec)
1336	return -ENOMEM;
1337
1338	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1339	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
1340	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP);
1341	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport);
1342	MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, ROCE_V2_UDP_DPORT);
1343
1344	MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY);
1345	MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B);
1346	MLX5_SET(copy_action_in, action, src_offset, 0);
1347	MLX5_SET(copy_action_in, action, length, 32);
1348	MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_C_5);
1349	MLX5_SET(copy_action_in, action, dst_offset, 0);
1350
1351	modify_hdr = mlx5_modify_header_alloc(dev: macsec_fs->mdev, ns_type: MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC,
1352	num_actions: 1, modify_actions: action);
1353
1354	if (IS_ERR(ptr: modify_hdr)) {
1355	err = PTR_ERR(ptr: modify_hdr);
1356	mlx5_core_err(mdev,
1357	"Failed to alloc macsec copy modify_header_id err(%d)\n", err);
1358	goto err_alloc_hdr;
1359	}
1360
1361	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1362	flow_act.modify_hdr = modify_hdr;
1363	dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
1364	dst.ft = roce->ft_ip_check;
1365	rule = mlx5_add_flow_rules(ft: roce->ft, spec, flow_act: &flow_act, dest: &dst, num_dest: 1);
1366	if (IS_ERR(ptr: rule)) {
1367	err = PTR_ERR(ptr: rule);
1368	mlx5_core_err(mdev, "Failed to add rule to MACsec RoCE NIC UDP table err(%d)\n",
1369	err);
1370	goto err_add_rule;
1371	}
1372	roce->rule = rule;
1373	roce->copy_modify_hdr = modify_hdr;
1374
1375	memset(&flow_act, 0, sizeof(flow_act));
1376	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
1377	rule = mlx5_add_flow_rules(ft: roce->ft, NULL, flow_act: &flow_act, NULL, num_dest: 0);
1378	if (IS_ERR(ptr: rule)) {
1379	err = PTR_ERR(ptr: rule);
1380	mlx5_core_err(mdev, "Failed to add miss rule to MACsec RoCE NIC UDP table err(%d)\n",
1381	err);
1382	goto err_add_rule2;
1383	}
1384	roce->nic_miss.rule = rule;
1385
1386	kvfree(addr: spec);
1387	return 0;
1388
1389	err_add_rule2:
1390	mlx5_del_flow_rules(fr: roce->rule);
1391	err_add_rule:
1392	mlx5_modify_header_dealloc(dev: macsec_fs->mdev, modify_hdr);
1393	err_alloc_hdr:
1394	kvfree(addr: spec);
1395	return err;
1396	}
1397
1398	static int macsec_fs_rx_roce_jump_to_rdma_create(struct mlx5_macsec_fs *macsec_fs,
1399	struct mlx5_macsec_rx_roce *roce)
1400	{
1401	int err;
1402
1403	err = macsec_fs_rx_roce_jump_to_rdma_groups_create(mdev: macsec_fs->mdev, roce);
1404	if (err)
1405	return err;
1406
1407	err = macsec_fs_rx_roce_jump_to_rdma_rules_create(macsec_fs, roce);
1408	if (err)
1409	goto err;
1410
1411	return 0;
1412	err:
1413	mlx5_destroy_flow_group(fg: roce->nic_miss.g);
1414	mlx5_destroy_flow_group(fg: roce->g);
1415	return err;
1416	}
1417
1418	static int macsec_fs_rx_roce_create(struct mlx5_macsec_fs *macsec_fs)
1419	{
1420	struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs;
1421	struct mlx5_core_dev *mdev = macsec_fs->mdev;
1422	struct mlx5_flow_table_attr ft_attr = {};
1423	struct mlx5_flow_namespace *ns;
1424	struct mlx5_flow_table *ft;
1425	int err = 0;
1426
1427	if (!mlx5_is_macsec_roce_supported(mdev: macsec_fs->mdev)) {
1428	mlx5_core_dbg(mdev, "Failed to init RoCE MACsec, capabilities not supported\n");
1429	return 0;
1430	}
1431
1432	ns = mlx5_get_flow_namespace(dev: macsec_fs->mdev, type: MLX5_FLOW_NAMESPACE_RDMA_RX_MACSEC);
1433	if (!ns)
1434	return -ENOMEM;
1435
1436	ft = macsec_fs_auto_group_table_create(ns, flags: 0, RDMA_RX_ROCE_IP_TABLE_LEVEL,
1437	CRYPTO_NUM_MAXSEC_FTE);
1438	if (IS_ERR(ptr: ft)) {
1439	err = PTR_ERR(ptr: ft);
1440	mlx5_core_err(mdev,
1441	"Failed to create MACsec IP check RoCE table err(%d)\n", err);
1442	return err;
1443	}
1444	rx_fs->roce.ft_ip_check = ft;
1445
1446	ft = macsec_fs_auto_group_table_create(ns, flags: 0, RDMA_RX_ROCE_MACSEC_OP_TABLE_LEVEL,
1447	CRYPTO_NUM_MAXSEC_FTE);
1448	if (IS_ERR(ptr: ft)) {
1449	err = PTR_ERR(ptr: ft);
1450	mlx5_core_err(mdev,
1451	"Failed to create MACsec operation check RoCE table err(%d)\n",
1452	err);
1453	goto err_macsec_op;
1454	}
1455	rx_fs->roce.ft_macsec_op_check = ft;
1456
1457	err = macsec_fs_rx_roce_miss_create(mdev, roce: &rx_fs->roce);
1458	if (err)
1459	goto err_miss_create;
1460
1461	ns = mlx5_get_flow_namespace(dev: macsec_fs->mdev, type: MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC);
1462	if (!ns) {
1463	err = -EOPNOTSUPP;
1464	goto err_ns;
1465	}
1466
1467	ft_attr.level = RX_ROCE_TABLE_LEVEL;
1468	ft_attr.max_fte = RX_ROCE_TABLE_NUM_FTE;
1469	ft = mlx5_create_flow_table(ns, ft_attr: &ft_attr);
1470	if (IS_ERR(ptr: ft)) {
1471	err = PTR_ERR(ptr: ft);
1472	mlx5_core_err(mdev,
1473	"Failed to create MACsec jump to RX RoCE, NIC table err(%d)\n", err);
1474	goto err_ns;
1475	}
1476	rx_fs->roce.ft = ft;
1477
1478	err = macsec_fs_rx_roce_jump_to_rdma_create(macsec_fs, roce: &rx_fs->roce);
1479	if (err)
1480	goto err_udp_ft;
1481
1482	return 0;
1483
1484	err_udp_ft:
1485	mlx5_destroy_flow_table(ft: rx_fs->roce.ft);
1486	err_ns:
1487	macsec_fs_rx_roce_miss_destroy(miss: &rx_fs->roce.miss);
1488	err_miss_create:
1489	mlx5_destroy_flow_table(ft: rx_fs->roce.ft_macsec_op_check);
1490	err_macsec_op:
1491	mlx5_destroy_flow_table(ft: rx_fs->roce.ft_ip_check);
1492	return err;
1493	}
1494
1495	static int macsec_fs_rx_create(struct mlx5_macsec_fs *macsec_fs)
1496	{
1497	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1498	struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs;
1499	struct mlx5_core_dev *mdev = macsec_fs->mdev;
1500	struct mlx5_macsec_flow_table *ft_crypto;
1501	struct mlx5_flow_table_attr ft_attr = {};
1502	struct mlx5_flow_destination dest = {};
1503	struct mlx5_macsec_tables *rx_tables;
1504	struct mlx5_flow_table *flow_table;
1505	struct mlx5_flow_group *flow_group;
1506	struct mlx5_flow_act flow_act = {};
1507	struct mlx5_flow_namespace *ns;
1508	struct mlx5_flow_handle *rule;
1509	struct mlx5_flow_spec *spec;
1510	u32 *flow_group_in;
1511	int err;
1512
1513	ns = mlx5_get_flow_namespace(dev: mdev, type: MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC);
1514	if (!ns)
1515	return -ENOMEM;
1516
1517	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
1518	if (!spec)
1519	return -ENOMEM;
1520
1521	flow_group_in = kvzalloc(size: inlen, GFP_KERNEL);
1522	if (!flow_group_in) {
1523	err = -ENOMEM;
1524	goto free_spec;
1525	}
1526
1527	rx_tables = &rx_fs->tables;
1528	ft_crypto = &rx_tables->ft_crypto;
1529
1530	err = macsec_fs_rx_roce_create(macsec_fs);
1531	if (err)
1532	goto out_flow_group;
1533
1534	/* Rx crypto table */
1535	ft_attr.level = RX_CRYPTO_TABLE_LEVEL;
1536	ft_attr.max_fte = CRYPTO_NUM_MAXSEC_FTE;
1537
1538	flow_table = mlx5_create_flow_table(ns, ft_attr: &ft_attr);
1539	if (IS_ERR(ptr: flow_table)) {
1540	err = PTR_ERR(ptr: flow_table);
1541	mlx5_core_err(mdev, "Failed to create MACsec Rx crypto table err(%d)\n", err);
1542	goto err;
1543	}
1544	ft_crypto->t = flow_table;
1545
1546	/* Rx crypto table groups */
1547	err = macsec_fs_rx_create_crypto_table_groups(ft: ft_crypto);
1548	if (err) {
1549	mlx5_core_err(mdev,
1550	"Failed to create default flow group for MACsec Tx crypto table err(%d)\n",
1551	err);
1552	goto err;
1553	}
1554
1555	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
1556	rule = mlx5_add_flow_rules(ft: ft_crypto->t, NULL, flow_act: &flow_act, NULL, num_dest: 0);
1557	if (IS_ERR(ptr: rule)) {
1558	err = PTR_ERR(ptr: rule);
1559	mlx5_core_err(mdev,
1560	"Failed to add MACsec Rx crypto table default miss rule %d\n",
1561	err);
1562	goto err;
1563	}
1564	rx_tables->crypto_miss_rule = rule;
1565
1566	/* Rx check table */
1567	flow_table = macsec_fs_auto_group_table_create(ns,
1568	flags: MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT,
1569	RX_CHECK_TABLE_LEVEL,
1570	RX_CHECK_TABLE_NUM_FTE);
1571	if (IS_ERR(ptr: flow_table)) {
1572	err = PTR_ERR(ptr: flow_table);
1573	mlx5_core_err(mdev, "Fail to create MACsec RX check table, err(%d)\n", err);
1574	goto err;
1575	}
1576	rx_tables->ft_check = flow_table;
1577
1578	/* Rx check table Default miss group/rule */
1579	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_table->max_fte - 1);
1580	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_table->max_fte - 1);
1581	flow_group = mlx5_create_flow_group(ft: rx_tables->ft_check, in: flow_group_in);
1582	if (IS_ERR(ptr: flow_group)) {
1583	err = PTR_ERR(ptr: flow_group);
1584	mlx5_core_err(mdev,
1585	"Failed to create default flow group for MACsec Rx check table err(%d)\n",
1586	err);
1587	goto err;
1588	}
1589	rx_tables->ft_check_group = flow_group;
1590
1591	/* Rx check table default drop rule */
1592	memset(&flow_act, 0, sizeof(flow_act));
1593
1594	dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1595	dest.counter_id = mlx5_fc_id(counter: rx_tables->check_miss_rule_counter);
1596	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT;
1597	rule = mlx5_add_flow_rules(ft: rx_tables->ft_check, NULL, flow_act: &flow_act, dest: &dest, num_dest: 1);
1598	if (IS_ERR(ptr: rule)) {
1599	err = PTR_ERR(ptr: rule);
1600	mlx5_core_err(mdev, "Failed to added MACsec Rx check drop rule, err(%d)\n", err);
1601	goto err;
1602	}
1603	rx_tables->check_miss_rule = rule;
1604
1605	/* Rx check table decap rules */
1606	err = macsec_fs_rx_create_check_decap_rule(macsec_fs, dest: &dest, flow_act: &flow_act, spec,
1607	MLX5_SECTAG_HEADER_SIZE_WITH_SCI);
1608	if (err)
1609	goto err;
1610
1611	err = macsec_fs_rx_create_check_decap_rule(macsec_fs, dest: &dest, flow_act: &flow_act, spec,
1612	MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI);
1613	if (err)
1614	goto err;
1615
1616	goto out_flow_group;
1617
1618	err:
1619	macsec_fs_rx_destroy(macsec_fs);
1620	out_flow_group:
1621	kvfree(addr: flow_group_in);
1622	free_spec:
1623	kvfree(addr: spec);
1624	return err;
1625	}
1626
1627	static int macsec_fs_rx_ft_get(struct mlx5_macsec_fs *macsec_fs)
1628	{
1629	struct mlx5_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables;
1630	int err = 0;
1631
1632	if (rx_tables->refcnt)
1633	goto out;
1634
1635	err = macsec_fs_rx_create(macsec_fs);
1636	if (err)
1637	return err;
1638
1639	out:
1640	rx_tables->refcnt++;
1641	return err;
1642	}
1643
1644	static void macsec_fs_rx_ft_put(struct mlx5_macsec_fs *macsec_fs)
1645	{
1646	struct mlx5_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables;
1647
1648	if (--rx_tables->refcnt)
1649	return;
1650
1651	macsec_fs_rx_destroy(macsec_fs);
1652	}
1653
1654	static void macsec_fs_rx_del_rule(struct mlx5_macsec_fs *macsec_fs,
1655	struct mlx5_macsec_rx_rule *rx_rule,
1656	void *macdev, u32 fs_id)
1657	{
1658	int i;
1659
1660	macsec_fs_id_del(macsec_devices_list: &macsec_fs->macsec_devices_list, fs_id, macdev,
1661	hash_table: &macsec_fs->fs_id_hash, is_tx: false);
1662
1663	for (i = 0; i < RX_NUM_OF_RULES_PER_SA; ++i) {
1664	if (rx_rule->rule[i]) {
1665	mlx5_del_flow_rules(fr: rx_rule->rule[i]);
1666	rx_rule->rule[i] = NULL;
1667	}
1668	}
1669
1670	if (rx_rule->meta_modhdr) {
1671	mlx5_modify_header_dealloc(dev: macsec_fs->mdev, modify_hdr: rx_rule->meta_modhdr);
1672	rx_rule->meta_modhdr = NULL;
1673	}
1674
1675	kfree(objp: rx_rule);
1676
1677	macsec_fs_rx_ft_put(macsec_fs);
1678	}
1679
1680	static void macsec_fs_rx_setup_fte(struct mlx5_flow_spec *spec,
1681	struct mlx5_flow_act *flow_act,
1682	struct mlx5_macsec_rule_attrs *attrs,
1683	bool sci_present)
1684	{
1685	u8 tci_an = (sci_present << MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET) | attrs->assoc_num;
1686	struct mlx5_flow_act_crypto_params *crypto_params = &flow_act->crypto;
1687	__be32 *sci_p = (__be32 *)(&attrs->sci);
1688
1689	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1690
1691	/* MACsec ethertype */
1692	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
1693	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ETH_P_MACSEC);
1694
1695	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5;
1696
1697	/* Sectag AN + TCI SC present bit*/
1698	MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_0,
1699	MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
1700	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_0,
1701	tci_an << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
1702
1703	if (sci_present) {
1704	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1705	misc_parameters_5.macsec_tag_2);
1706	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_2,
1707	be32_to_cpu(sci_p[0]));
1708
1709	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1710	misc_parameters_5.macsec_tag_3);
1711	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_3,
1712	be32_to_cpu(sci_p[1]));
1713	} else {
1714	/* When SCI isn't present in the Sectag, need to match the source */
1715	/* MAC address only if the SCI contains the default MACsec PORT */
1716	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_47_16);
1717	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_15_0);
1718	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.smac_47_16),
1719	sci_p, ETH_ALEN);
1720	}
1721
1722	crypto_params->type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC;
1723	crypto_params->obj_id = attrs->macsec_obj_id;
1724	}
1725
1726	static union mlx5_macsec_rule *
1727	macsec_fs_rx_add_rule(struct mlx5_macsec_fs *macsec_fs,
1728	const struct macsec_context *macsec_ctx,
1729	struct mlx5_macsec_rule_attrs *attrs,
1730	u32 fs_id)
1731	{
1732	u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
1733	struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs;
1734	struct mlx5_core_dev *mdev = macsec_fs->mdev;
1735	union mlx5_macsec_rule *macsec_rule = NULL;
1736	struct mlx5_modify_hdr *modify_hdr = NULL;
1737	struct mlx5_macsec_flow_table *ft_crypto;
1738	struct mlx5_flow_destination dest = {};
1739	struct mlx5_macsec_tables *rx_tables;
1740	struct mlx5_macsec_rx_rule *rx_rule;
1741	struct mlx5_flow_act flow_act = {};
1742	struct mlx5_flow_handle *rule;
1743	struct mlx5_flow_spec *spec;
1744	int err = 0;
1745
1746	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
1747	if (!spec)
1748	return NULL;
1749
1750	err = macsec_fs_rx_ft_get(macsec_fs);
1751	if (err)
1752	goto out_spec;
1753
1754	macsec_rule = kzalloc(size: sizeof(*macsec_rule), GFP_KERNEL);
1755	if (!macsec_rule) {
1756	macsec_fs_rx_ft_put(macsec_fs);
1757	goto out_spec;
1758	}
1759
1760	rx_rule = &macsec_rule->rx_rule;
1761	rx_tables = &rx_fs->tables;
1762	ft_crypto = &rx_tables->ft_crypto;
1763
1764	/* Set bit[31 - 30] macsec marker - 0x01 */
1765	/* Set bit[15-0] fs id */
1766	MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET);
1767	MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B);
1768	MLX5_SET(set_action_in, action, data, macsec_fs_set_rx_fs_id(fs_id));
1769	MLX5_SET(set_action_in, action, offset, 0);
1770	MLX5_SET(set_action_in, action, length, 32);
1771
1772	modify_hdr = mlx5_modify_header_alloc(dev: mdev, ns_type: MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC,
1773	num_actions: 1, modify_actions: action);
1774	if (IS_ERR(ptr: modify_hdr)) {
1775	err = PTR_ERR(ptr: modify_hdr);
1776	mlx5_core_err(mdev, "Fail to alloc MACsec set modify_header_id err=%d\n", err);
1777	modify_hdr = NULL;
1778	goto err;
1779	}
1780	rx_rule->meta_modhdr = modify_hdr;
1781
1782	/* Rx crypto table with SCI rule */
1783	macsec_fs_rx_setup_fte(spec, flow_act: &flow_act, attrs, sci_present: true);
1784
1785	flow_act.modify_hdr = modify_hdr;
1786	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1787	MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT |
1788	MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1789
1790	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1791	dest.ft = rx_tables->ft_check;
1792	rule = mlx5_add_flow_rules(ft: ft_crypto->t, spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1793	if (IS_ERR(ptr: rule)) {
1794	err = PTR_ERR(ptr: rule);
1795	mlx5_core_err(mdev,
1796	"Failed to add SA with SCI rule to Rx crypto rule, err=%d\n",
1797	err);
1798	goto err;
1799	}
1800	rx_rule->rule[0] = rule;
1801
1802	/* Rx crypto table without SCI rule */
1803	if ((cpu_to_be64((__force u64)attrs->sci) & 0xFFFF) == ntohs(MACSEC_PORT_ES)) {
1804	memset(spec, 0, sizeof(struct mlx5_flow_spec));
1805	memset(&dest, 0, sizeof(struct mlx5_flow_destination));
1806	memset(&flow_act, 0, sizeof(flow_act));
1807
1808	macsec_fs_rx_setup_fte(spec, flow_act: &flow_act, attrs, sci_present: false);
1809
1810	flow_act.modify_hdr = modify_hdr;
1811	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1812	MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT |
1813	MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1814
1815	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1816	dest.ft = rx_tables->ft_check;
1817	rule = mlx5_add_flow_rules(ft: ft_crypto->t, spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1818	if (IS_ERR(ptr: rule)) {
1819	err = PTR_ERR(ptr: rule);
1820	mlx5_core_err(mdev,
1821	"Failed to add SA without SCI rule to Rx crypto rule, err=%d\n",
1822	err);
1823	goto err;
1824	}
1825	rx_rule->rule[1] = rule;
1826	}
1827
1828	err = macsec_fs_id_add(macsec_devices_list: &macsec_fs->macsec_devices_list, fs_id, macdev: macsec_ctx->secy->netdev,
1829	hash_table: &macsec_fs->fs_id_hash, sci: attrs->sci, is_tx: false);
1830	if (err) {
1831	mlx5_core_err(mdev, "Failed to save fs_id, err=%d\n", err);
1832	goto err;
1833	}
1834
1835	kvfree(addr: spec);
1836	return macsec_rule;
1837
1838	err:
1839	macsec_fs_rx_del_rule(macsec_fs, rx_rule, macdev: macsec_ctx->secy->netdev, fs_id);
1840	macsec_rule = NULL;
1841	out_spec:
1842	kvfree(addr: spec);
1843	return macsec_rule;
1844	}
1845
1846	static int macsec_fs_rx_init(struct mlx5_macsec_fs *macsec_fs)
1847	{
1848	struct mlx5_core_dev *mdev = macsec_fs->mdev;
1849	struct mlx5_macsec_tables *rx_tables;
1850	struct mlx5_macsec_rx *rx_fs;
1851	struct mlx5_fc *flow_counter;
1852	int err;
1853
1854	rx_fs = kzalloc(size: sizeof(*rx_fs), GFP_KERNEL);
1855	if (!rx_fs)
1856	return -ENOMEM;
1857
1858	flow_counter = mlx5_fc_create(dev: mdev, aging: false);
1859	if (IS_ERR(ptr: flow_counter)) {
1860	err = PTR_ERR(ptr: flow_counter);
1861	mlx5_core_err(mdev,
1862	"Failed to create MACsec Rx encrypt flow counter, err(%d)\n",
1863	err);
1864	goto err_encrypt_counter;
1865	}
1866
1867	rx_tables = &rx_fs->tables;
1868	rx_tables->check_rule_counter = flow_counter;
1869
1870	flow_counter = mlx5_fc_create(dev: mdev, aging: false);
1871	if (IS_ERR(ptr: flow_counter)) {
1872	err = PTR_ERR(ptr: flow_counter);
1873	mlx5_core_err(mdev,
1874	"Failed to create MACsec Rx drop flow counter, err(%d)\n",
1875	err);
1876	goto err_drop_counter;
1877	}
1878	rx_tables->check_miss_rule_counter = flow_counter;
1879
1880	macsec_fs->rx_fs = rx_fs;
1881
1882	return 0;
1883
1884	err_drop_counter:
1885	mlx5_fc_destroy(dev: mdev, counter: rx_tables->check_rule_counter);
1886	rx_tables->check_rule_counter = NULL;
1887
1888	err_encrypt_counter:
1889	kfree(objp: rx_fs);
1890	macsec_fs->rx_fs = NULL;
1891
1892	return err;
1893	}
1894
1895	static void macsec_fs_rx_cleanup(struct mlx5_macsec_fs *macsec_fs)
1896	{
1897	struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs;
1898	struct mlx5_core_dev *mdev = macsec_fs->mdev;
1899	struct mlx5_macsec_tables *rx_tables;
1900
1901	if (!rx_fs)
1902	return;
1903
1904	rx_tables = &rx_fs->tables;
1905
1906	if (rx_tables->refcnt) {
1907	mlx5_core_err(mdev,
1908	"Can't destroy MACsec offload rx_fs, refcnt(%u) isn't 0\n",
1909	rx_tables->refcnt);
1910	return;
1911	}
1912
1913	if (rx_tables->check_miss_rule_counter) {
1914	mlx5_fc_destroy(dev: mdev, counter: rx_tables->check_miss_rule_counter);
1915	rx_tables->check_miss_rule_counter = NULL;
1916	}
1917
1918	if (rx_tables->check_rule_counter) {
1919	mlx5_fc_destroy(dev: mdev, counter: rx_tables->check_rule_counter);
1920	rx_tables->check_rule_counter = NULL;
1921	}
1922
1923	kfree(objp: rx_fs);
1924	macsec_fs->rx_fs = NULL;
1925	}
1926
1927	static void set_ipaddr_spec_v4(struct sockaddr_in *in, struct mlx5_flow_spec *spec, bool is_dst_ip)
1928	{
1929	MLX5_SET(fte_match_param, spec->match_value,
1930	outer_headers.ip_version, MLX5_FS_IPV4_VERSION);
1931
1932	if (is_dst_ip) {
1933	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1934	outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
1935	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
1936	outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
1937	&in->sin_addr.s_addr, 4);
1938	} else {
1939	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1940	outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
1941	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
1942	outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4),
1943	&in->sin_addr.s_addr, 4);
1944	}
1945	}
1946
1947	static void set_ipaddr_spec_v6(struct sockaddr_in6 *in6, struct mlx5_flow_spec *spec,
1948	bool is_dst_ip)
1949	{
1950	MLX5_SET(fte_match_param, spec->match_value,
1951	outer_headers.ip_version, MLX5_FS_IPV6_VERSION);
1952
1953	if (is_dst_ip) {
1954	memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1955	outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
1956	0xff, 16);
1957	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
1958	outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
1959	&in6->sin6_addr, 16);
1960	} else {
1961	memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1962	outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
1963	0xff, 16);
1964	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
1965	outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
1966	&in6->sin6_addr, 16);
1967	}
1968	}
1969
1970	static void set_ipaddr_spec(const struct sockaddr *addr,
1971	struct mlx5_flow_spec *spec, bool is_dst_ip)
1972	{
1973	struct sockaddr_in6 *in6;
1974
1975	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
1976	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1977	outer_headers.ip_version);
1978
1979	if (addr->sa_family == AF_INET) {
1980	struct sockaddr_in *in = (struct sockaddr_in *)addr;
1981
1982	set_ipaddr_spec_v4(in, spec, is_dst_ip);
1983	return;
1984	}
1985
1986	in6 = (struct sockaddr_in6 *)addr;
1987	set_ipaddr_spec_v6(in6, spec, is_dst_ip);
1988	}
1989
1990	static void macsec_fs_del_roce_rule_rx(struct mlx5_roce_macsec_rx_rule *rx_rule)
1991	{
1992	mlx5_del_flow_rules(fr: rx_rule->op);
1993	mlx5_del_flow_rules(fr: rx_rule->ip);
1994	list_del(entry: &rx_rule->entry);
1995	kfree(objp: rx_rule);
1996	}
1997
1998	static void macsec_fs_del_roce_rules_rx(struct mlx5_macsec_fs *macsec_fs, u32 fs_id,
1999	struct list_head *rx_rules_list)
2000	{
2001	struct mlx5_roce_macsec_rx_rule *rx_rule, *next;
2002
2003	if (!mlx5_is_macsec_roce_supported(mdev: macsec_fs->mdev))
2004	return;
2005
2006	list_for_each_entry_safe(rx_rule, next, rx_rules_list, entry) {
2007	if (rx_rule->fs_id == fs_id)
2008	macsec_fs_del_roce_rule_rx(rx_rule);
2009	}
2010	}
2011
2012	static void macsec_fs_del_roce_rule_tx(struct mlx5_core_dev *mdev,
2013	struct mlx5_roce_macsec_tx_rule *tx_rule)
2014	{
2015	mlx5_del_flow_rules(fr: tx_rule->rule);
2016	mlx5_modify_header_dealloc(dev: mdev, modify_hdr: tx_rule->meta_modhdr);
2017	list_del(entry: &tx_rule->entry);
2018	kfree(objp: tx_rule);
2019	}
2020
2021	static void macsec_fs_del_roce_rules_tx(struct mlx5_macsec_fs *macsec_fs, u32 fs_id,
2022	struct list_head *tx_rules_list)
2023	{
2024	struct mlx5_roce_macsec_tx_rule *tx_rule, *next;
2025
2026	if (!mlx5_is_macsec_roce_supported(mdev: macsec_fs->mdev))
2027	return;
2028
2029	list_for_each_entry_safe(tx_rule, next, tx_rules_list, entry) {
2030	if (tx_rule->fs_id == fs_id)
2031	macsec_fs_del_roce_rule_tx(mdev: macsec_fs->mdev, tx_rule);
2032	}
2033	}
2034
2035	void mlx5_macsec_fs_get_stats_fill(struct mlx5_macsec_fs *macsec_fs, void *macsec_stats)
2036	{
2037	struct mlx5_macsec_stats *stats = (struct mlx5_macsec_stats *)macsec_stats;
2038	struct mlx5_macsec_tables *tx_tables = &macsec_fs->tx_fs->tables;
2039	struct mlx5_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables;
2040	struct mlx5_core_dev *mdev = macsec_fs->mdev;
2041
2042	if (tx_tables->check_rule_counter)
2043	mlx5_fc_query(dev: mdev, counter: tx_tables->check_rule_counter,
2044	packets: &stats->macsec_tx_pkts, bytes: &stats->macsec_tx_bytes);
2045
2046	if (tx_tables->check_miss_rule_counter)
2047	mlx5_fc_query(dev: mdev, counter: tx_tables->check_miss_rule_counter,
2048	packets: &stats->macsec_tx_pkts_drop, bytes: &stats->macsec_tx_bytes_drop);
2049
2050	if (rx_tables->check_rule_counter)
2051	mlx5_fc_query(dev: mdev, counter: rx_tables->check_rule_counter,
2052	packets: &stats->macsec_rx_pkts, bytes: &stats->macsec_rx_bytes);
2053
2054	if (rx_tables->check_miss_rule_counter)
2055	mlx5_fc_query(dev: mdev, counter: rx_tables->check_miss_rule_counter,
2056	packets: &stats->macsec_rx_pkts_drop, bytes: &stats->macsec_rx_bytes_drop);
2057	}
2058
2059	struct mlx5_macsec_stats *mlx5_macsec_fs_get_stats(struct mlx5_macsec_fs *macsec_fs)
2060	{
2061	if (!macsec_fs)
2062	return NULL;
2063
2064	return &macsec_fs->stats;
2065	}
2066
2067	u32 mlx5_macsec_fs_get_fs_id_from_hashtable(struct mlx5_macsec_fs *macsec_fs, sci_t *sci)
2068	{
2069	struct mlx5_fs_id *mlx5_fs_id;
2070	u32 fs_id = 0;
2071
2072	rcu_read_lock();
2073	mlx5_fs_id = rhashtable_lookup(ht: &macsec_fs->sci_hash, key: sci, params: rhash_sci);
2074	if (mlx5_fs_id)
2075	fs_id = mlx5_fs_id->id;
2076	rcu_read_unlock();
2077
2078	return fs_id;
2079	}
2080
2081	union mlx5_macsec_rule *
2082	mlx5_macsec_fs_add_rule(struct mlx5_macsec_fs *macsec_fs,
2083	const struct macsec_context *macsec_ctx,
2084	struct mlx5_macsec_rule_attrs *attrs,
2085	u32 *sa_fs_id)
2086	{
2087	struct mlx5_macsec_event_data data = {.macsec_fs = macsec_fs,
2088	.macdev = macsec_ctx->secy->netdev,
2089	.is_tx =
2090	(attrs->action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT)
2091	};
2092	union mlx5_macsec_rule *macsec_rule;
2093	u32 tx_new_fs_id;
2094
2095	macsec_rule = (attrs->action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ?
2096	macsec_fs_tx_add_rule(macsec_fs, macsec_ctx, attrs, fs_id: &tx_new_fs_id) :
2097	macsec_fs_rx_add_rule(macsec_fs, macsec_ctx, attrs, fs_id: *sa_fs_id);
2098
2099	data.fs_id = (data.is_tx) ? tx_new_fs_id : *sa_fs_id;
2100	if (macsec_rule)
2101	blocking_notifier_call_chain(nh: &macsec_fs->mdev->macsec_nh,
2102	val: MLX5_DRIVER_EVENT_MACSEC_SA_ADDED,
2103	v: &data);
2104
2105	return macsec_rule;
2106	}
2107
2108	void mlx5_macsec_fs_del_rule(struct mlx5_macsec_fs *macsec_fs,
2109	union mlx5_macsec_rule *macsec_rule,
2110	int action, void *macdev, u32 sa_fs_id)
2111	{
2112	struct mlx5_macsec_event_data data = {.macsec_fs = macsec_fs,
2113	.macdev = macdev,
2114	.is_tx = (action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT)
2115	};
2116
2117	data.fs_id = (data.is_tx) ? macsec_rule->tx_rule.fs_id : sa_fs_id;
2118	blocking_notifier_call_chain(nh: &macsec_fs->mdev->macsec_nh,
2119	val: MLX5_DRIVER_EVENT_MACSEC_SA_DELETED,
2120	v: &data);
2121
2122	(action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ?
2123	macsec_fs_tx_del_rule(macsec_fs, tx_rule: &macsec_rule->tx_rule, macdev) :
2124	macsec_fs_rx_del_rule(macsec_fs, rx_rule: &macsec_rule->rx_rule, macdev, fs_id: sa_fs_id);
2125	}
2126
2127	static int mlx5_macsec_fs_add_roce_rule_rx(struct mlx5_macsec_fs *macsec_fs, u32 fs_id, u16 gid_idx,
2128	const struct sockaddr *addr,
2129	struct list_head *rx_rules_list)
2130	{
2131	struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs;
2132	struct mlx5_roce_macsec_rx_rule *rx_rule;
2133	struct mlx5_flow_destination dest = {};
2134	struct mlx5_flow_act flow_act = {};
2135	struct mlx5_flow_handle *new_rule;
2136	struct mlx5_flow_spec *spec;
2137	int err = 0;
2138
2139	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
2140	if (!spec)
2141	return -ENOMEM;
2142
2143	rx_rule = kzalloc(size: sizeof(*rx_rule), GFP_KERNEL);
2144	if (!rx_rule) {
2145	err = -ENOMEM;
2146	goto out;
2147	}
2148
2149	set_ipaddr_spec(addr, spec, is_dst_ip: true);
2150
2151	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2152	dest.ft = rx_fs->roce.ft_macsec_op_check;
2153	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
2154	new_rule = mlx5_add_flow_rules(ft: rx_fs->roce.ft_ip_check, spec, flow_act: &flow_act,
2155	dest: &dest, num_dest: 1);
2156	if (IS_ERR(ptr: new_rule)) {
2157	err = PTR_ERR(ptr: new_rule);
2158	goto ip_rule_err;
2159	}
2160	rx_rule->ip = new_rule;
2161
2162	memset(&flow_act, 0, sizeof(flow_act));
2163	memset(spec, 0, sizeof(*spec));
2164
2165	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
2166	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_5);
2167	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_5,
2168	macsec_fs_set_rx_fs_id(fs_id));
2169	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW;
2170	new_rule = mlx5_add_flow_rules(ft: rx_fs->roce.ft_macsec_op_check, spec, flow_act: &flow_act,
2171	NULL, num_dest: 0);
2172	if (IS_ERR(ptr: new_rule)) {
2173	err = PTR_ERR(ptr: new_rule);
2174	goto op_rule_err;
2175	}
2176	rx_rule->op = new_rule;
2177	rx_rule->gid_idx = gid_idx;
2178	rx_rule->fs_id = fs_id;
2179	list_add_tail(new: &rx_rule->entry, head: rx_rules_list);
2180
2181	goto out;
2182
2183	op_rule_err:
2184	mlx5_del_flow_rules(fr: rx_rule->ip);
2185	rx_rule->ip = NULL;
2186	ip_rule_err:
2187	kfree(objp: rx_rule);
2188	out:
2189	kvfree(addr: spec);
2190	return err;
2191	}
2192
2193	static int mlx5_macsec_fs_add_roce_rule_tx(struct mlx5_macsec_fs *macsec_fs, u32 fs_id, u16 gid_idx,
2194	const struct sockaddr *addr,
2195	struct list_head *tx_rules_list)
2196	{
2197	u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
2198	struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs;
2199	struct mlx5_core_dev *mdev = macsec_fs->mdev;
2200	struct mlx5_modify_hdr *modify_hdr = NULL;
2201	struct mlx5_roce_macsec_tx_rule *tx_rule;
2202	struct mlx5_flow_destination dest = {};
2203	struct mlx5_flow_act flow_act = {};
2204	struct mlx5_flow_handle *new_rule;
2205	struct mlx5_flow_spec *spec;
2206	int err = 0;
2207
2208	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
2209	if (!spec)
2210	return -ENOMEM;
2211
2212	tx_rule = kzalloc(size: sizeof(*tx_rule), GFP_KERNEL);
2213	if (!tx_rule) {
2214	err = -ENOMEM;
2215	goto out;
2216	}
2217
2218	set_ipaddr_spec(addr, spec, is_dst_ip: false);
2219
2220	MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET);
2221	MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_A);
2222	MLX5_SET(set_action_in, action, data, macsec_fs_set_tx_fs_id(fs_id));
2223	MLX5_SET(set_action_in, action, offset, 0);
2224	MLX5_SET(set_action_in, action, length, 32);
2225
2226	modify_hdr = mlx5_modify_header_alloc(dev: mdev, ns_type: MLX5_FLOW_NAMESPACE_RDMA_TX_MACSEC,
2227	num_actions: 1, modify_actions: action);
2228	if (IS_ERR(ptr: modify_hdr)) {
2229	err = PTR_ERR(ptr: modify_hdr);
2230	mlx5_core_err(mdev, "Fail to alloc ROCE MACsec set modify_header_id err=%d\n",
2231	err);
2232	modify_hdr = NULL;
2233	goto modify_hdr_err;
2234	}
2235	tx_rule->meta_modhdr = modify_hdr;
2236
2237	flow_act.modify_hdr = modify_hdr;
2238	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
2239
2240	dest.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
2241	dest.ft = tx_fs->tables.ft_crypto.t;
2242	new_rule = mlx5_add_flow_rules(ft: tx_fs->ft_rdma_tx, spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
2243	if (IS_ERR(ptr: new_rule)) {
2244	err = PTR_ERR(ptr: new_rule);
2245	mlx5_core_err(mdev, "Failed to add ROCE TX rule, err=%d\n", err);
2246	goto rule_err;
2247	}
2248	tx_rule->rule = new_rule;
2249	tx_rule->gid_idx = gid_idx;
2250	tx_rule->fs_id = fs_id;
2251	list_add_tail(new: &tx_rule->entry, head: tx_rules_list);
2252
2253	goto out;
2254
2255	rule_err:
2256	mlx5_modify_header_dealloc(dev: mdev, modify_hdr: tx_rule->meta_modhdr);
2257	modify_hdr_err:
2258	kfree(objp: tx_rule);
2259	out:
2260	kvfree(addr: spec);
2261	return err;
2262	}
2263
2264	void mlx5_macsec_del_roce_rule(u16 gid_idx, struct mlx5_macsec_fs *macsec_fs,
2265	struct list_head *tx_rules_list, struct list_head *rx_rules_list)
2266	{
2267	struct mlx5_roce_macsec_rx_rule *rx_rule, *next_rx;
2268	struct mlx5_roce_macsec_tx_rule *tx_rule, *next_tx;
2269
2270	list_for_each_entry_safe(tx_rule, next_tx, tx_rules_list, entry) {
2271	if (tx_rule->gid_idx == gid_idx)
2272	macsec_fs_del_roce_rule_tx(mdev: macsec_fs->mdev, tx_rule);
2273	}
2274
2275	list_for_each_entry_safe(rx_rule, next_rx, rx_rules_list, entry) {
2276	if (rx_rule->gid_idx == gid_idx)
2277	macsec_fs_del_roce_rule_rx(rx_rule);
2278	}
2279	}
2280	EXPORT_SYMBOL_GPL(mlx5_macsec_del_roce_rule);
2281
2282	int mlx5_macsec_add_roce_rule(void *macdev, const struct sockaddr *addr, u16 gid_idx,
2283	struct list_head *tx_rules_list, struct list_head *rx_rules_list,
2284	struct mlx5_macsec_fs *macsec_fs)
2285	{
2286	struct mlx5_macsec_device *iter, *macsec_device = NULL;
2287	struct mlx5_core_dev *mdev = macsec_fs->mdev;
2288	struct mlx5_fs_id *fs_id_iter;
2289	unsigned long index = 0;
2290	int err;
2291
2292	list_for_each_entry(iter, &macsec_fs->macsec_devices_list, macsec_devices_list_entry) {
2293	if (iter->macdev == macdev) {
2294	macsec_device = iter;
2295	break;
2296	}
2297	}
2298
2299	if (!macsec_device)
2300	return 0;
2301
2302	xa_for_each(&macsec_device->tx_id_xa, index, fs_id_iter) {
2303	err = mlx5_macsec_fs_add_roce_rule_tx(macsec_fs, fs_id: fs_id_iter->id, gid_idx, addr,
2304	tx_rules_list);
2305	if (err) {
2306	mlx5_core_err(mdev, "MACsec offload: Failed to add roce TX rule\n");
2307	goto out;
2308	}
2309	}
2310
2311	index = 0;
2312	xa_for_each(&macsec_device->rx_id_xa, index, fs_id_iter) {
2313	err = mlx5_macsec_fs_add_roce_rule_rx(macsec_fs, fs_id: fs_id_iter->id, gid_idx, addr,
2314	rx_rules_list);
2315	if (err) {
2316	mlx5_core_err(mdev, "MACsec offload: Failed to add roce TX rule\n");
2317	goto out;
2318	}
2319	}
2320
2321	return 0;
2322	out:
2323	mlx5_macsec_del_roce_rule(gid_idx, macsec_fs, tx_rules_list, rx_rules_list);
2324	return err;
2325	}
2326	EXPORT_SYMBOL_GPL(mlx5_macsec_add_roce_rule);
2327
2328	void mlx5_macsec_add_roce_sa_rules(u32 fs_id, const struct sockaddr *addr, u16 gid_idx,
2329	struct list_head *tx_rules_list,
2330	struct list_head *rx_rules_list,
2331	struct mlx5_macsec_fs *macsec_fs, bool is_tx)
2332	{
2333	(is_tx) ?
2334	mlx5_macsec_fs_add_roce_rule_tx(macsec_fs, fs_id, gid_idx, addr,
2335	tx_rules_list) :
2336	mlx5_macsec_fs_add_roce_rule_rx(macsec_fs, fs_id, gid_idx, addr,
2337	rx_rules_list);
2338	}
2339	EXPORT_SYMBOL_GPL(mlx5_macsec_add_roce_sa_rules);
2340
2341	void mlx5_macsec_del_roce_sa_rules(u32 fs_id, struct mlx5_macsec_fs *macsec_fs,
2342	struct list_head *tx_rules_list,
2343	struct list_head *rx_rules_list, bool is_tx)
2344	{
2345	(is_tx) ?
2346	macsec_fs_del_roce_rules_tx(macsec_fs, fs_id, tx_rules_list) :
2347	macsec_fs_del_roce_rules_rx(macsec_fs, fs_id, rx_rules_list);
2348	}
2349	EXPORT_SYMBOL_GPL(mlx5_macsec_del_roce_sa_rules);
2350
2351	void mlx5_macsec_fs_cleanup(struct mlx5_macsec_fs *macsec_fs)
2352	{
2353	macsec_fs_rx_cleanup(macsec_fs);
2354	macsec_fs_tx_cleanup(macsec_fs);
2355	rhashtable_destroy(ht: &macsec_fs->fs_id_hash);
2356	rhashtable_destroy(ht: &macsec_fs->sci_hash);
2357	kfree(objp: macsec_fs);
2358	}
2359
2360	struct mlx5_macsec_fs *
2361	mlx5_macsec_fs_init(struct mlx5_core_dev *mdev)
2362	{
2363	struct mlx5_macsec_fs *macsec_fs;
2364	int err;
2365
2366	macsec_fs = kzalloc(size: sizeof(*macsec_fs), GFP_KERNEL);
2367	if (!macsec_fs)
2368	return NULL;
2369
2370	macsec_fs->mdev = mdev;
2371
2372	err = rhashtable_init(ht: &macsec_fs->sci_hash, params: &rhash_sci);
2373	if (err) {
2374	mlx5_core_err(mdev, "MACsec offload: Failed to init SCI hash table, err=%d\n",
2375	err);
2376	goto err_hash;
2377	}
2378
2379	err = rhashtable_init(ht: &macsec_fs->fs_id_hash, params: &rhash_fs_id);
2380	if (err) {
2381	mlx5_core_err(mdev, "MACsec offload: Failed to init FS_ID hash table, err=%d\n",
2382	err);
2383	goto sci_hash_cleanup;
2384	}
2385
2386	err = macsec_fs_tx_init(macsec_fs);
2387	if (err) {
2388	mlx5_core_err(mdev, "MACsec offload: Failed to init tx_fs, err=%d\n", err);
2389	goto fs_id_hash_cleanup;
2390	}
2391
2392	err = macsec_fs_rx_init(macsec_fs);
2393	if (err) {
2394	mlx5_core_err(mdev, "MACsec offload: Failed to init tx_fs, err=%d\n", err);
2395	goto tx_cleanup;
2396	}
2397
2398	BLOCKING_INIT_NOTIFIER_HEAD(&mdev->macsec_nh);
2399
2400	return macsec_fs;
2401
2402	tx_cleanup:
2403	macsec_fs_tx_cleanup(macsec_fs);
2404	fs_id_hash_cleanup:
2405	rhashtable_destroy(ht: &macsec_fs->fs_id_hash);
2406	sci_hash_cleanup:
2407	rhashtable_destroy(ht: &macsec_fs->sci_hash);
2408	err_hash:
2409	kfree(objp: macsec_fs);
2410	return NULL;
2411	}
2412