1	/*
2	* Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/
32
33	#include <linux/etherdevice.h>
34	#include <linux/idr.h>
35	#include <linux/mlx5/driver.h>
36	#include <linux/mlx5/mlx5_ifc.h>
37	#include <linux/mlx5/vport.h>
38	#include <linux/mlx5/fs.h>
39	#include "mlx5_core.h"
40	#include "eswitch.h"
41	#include "esw/indir_table.h"
42	#include "esw/acl/ofld.h"
43	#include "rdma.h"
44	#include "en.h"
45	#include "fs_core.h"
46	#include "lib/mlx5.h"
47	#include "lib/devcom.h"
48	#include "lib/eq.h"
49	#include "lib/fs_chains.h"
50	#include "en_tc.h"
51	#include "en/mapping.h"
52	#include "devlink.h"
53	#include "lag/lag.h"
54	#include "en/tc/post_meter.h"
55
56	#define mlx5_esw_for_each_rep(esw, i, rep) \
57	xa_for_each(&((esw)->offloads.vport_reps), i, rep)
58
59	/* There are two match-all miss flows, one for unicast dst mac and
60	* one for multicast.
61	*/
62	#define MLX5_ESW_MISS_FLOWS (2)
63	#define UPLINK_REP_INDEX 0
64
65	#define MLX5_ESW_VPORT_TBL_SIZE 128
66	#define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
67
68	#define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
69
70	static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
71	.max_fte = MLX5_ESW_VPORT_TBL_SIZE,
72	.max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
73	.flags = 0,
74	};
75
76	static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
77	u16 vport_num)
78	{
79	return xa_load(&esw->offloads.vport_reps, index: vport_num);
80	}
81
82	static void
83	mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw,
84	struct mlx5_flow_spec *spec,
85	struct mlx5_esw_flow_attr *attr)
86	{
87	if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep)
88	return;
89
90	if (attr->int_port) {
91	spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(int_port: attr->int_port);
92
93	return;
94	}
95
96	spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ?
97	MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK :
98	MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
99	}
100
101	/* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
102	* are not needed as well in the following process. So clear them all for simplicity.
103	*/
104	void
105	mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec)
106	{
107	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
108	void *misc2;
109
110	misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
111	MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
112
113	misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
114	MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
115
116	if (!memchr_inv(p: misc2, c: 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2)))
117	spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2;
118	}
119	}
120
121	static void
122	mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw,
123	struct mlx5_flow_spec *spec,
124	struct mlx5_flow_attr *attr,
125	struct mlx5_eswitch *src_esw,
126	u16 vport)
127	{
128	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
129	u32 metadata;
130	void *misc2;
131	void *misc;
132
133	/* Use metadata matching because vport is not represented by single
134	* VHCA in dual-port RoCE mode, and matching on source vport may fail.
135	*/
136	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
137	if (mlx5_esw_indir_table_decap_vport(attr))
138	vport = mlx5_esw_indir_table_decap_vport(attr);
139
140	if (!attr->chain && esw_attr && esw_attr->int_port)
141	metadata =
142	mlx5e_tc_int_port_get_metadata_for_match(int_port: esw_attr->int_port);
143	else
144	metadata =
145	mlx5_eswitch_get_vport_metadata_for_match(esw: src_esw, vport_num: vport);
146
147	misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
148	MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata);
149
150	misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
151	MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
152	mlx5_eswitch_get_vport_metadata_mask());
153
154	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
155	} else {
156	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
157	MLX5_SET(fte_match_set_misc, misc, source_port, vport);
158
159	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
160	MLX5_SET(fte_match_set_misc, misc,
161	source_eswitch_owner_vhca_id,
162	MLX5_CAP_GEN(src_esw->dev, vhca_id));
163
164	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
165	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
166	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
167	MLX5_SET_TO_ONES(fte_match_set_misc, misc,
168	source_eswitch_owner_vhca_id);
169
170	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
171	}
172	}
173
174	static int
175	esw_setup_decap_indir(struct mlx5_eswitch *esw,
176	struct mlx5_flow_attr *attr)
177	{
178	struct mlx5_flow_table *ft;
179
180	if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
181	return -EOPNOTSUPP;
182
183	ft = mlx5_esw_indir_table_get(esw, attr,
184	vport: mlx5_esw_indir_table_decap_vport(attr), decap: true);
185	return PTR_ERR_OR_ZERO(ptr: ft);
186	}
187
188	static void
189	esw_cleanup_decap_indir(struct mlx5_eswitch *esw,
190	struct mlx5_flow_attr *attr)
191	{
192	if (mlx5_esw_indir_table_decap_vport(attr))
193	mlx5_esw_indir_table_put(esw,
194	vport: mlx5_esw_indir_table_decap_vport(attr),
195	decap: true);
196	}
197
198	static int
199	esw_setup_mtu_dest(struct mlx5_flow_destination *dest,
200	struct mlx5e_meter_attr *meter,
201	int i)
202	{
203	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE;
204	dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN;
205	dest[i].range.min = 0;
206	dest[i].range.max = meter->params.mtu;
207	dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(post_meter: meter->post_meter);
208	dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(post_meter: meter->post_meter);
209
210	return 0;
211	}
212
213	static int
214	esw_setup_sampler_dest(struct mlx5_flow_destination *dest,
215	struct mlx5_flow_act *flow_act,
216	u32 sampler_id,
217	int i)
218	{
219	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
220	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
221	dest[i].sampler_id = sampler_id;
222
223	return 0;
224	}
225
226	static int
227	esw_setup_ft_dest(struct mlx5_flow_destination *dest,
228	struct mlx5_flow_act *flow_act,
229	struct mlx5_eswitch *esw,
230	struct mlx5_flow_attr *attr,
231	int i)
232	{
233	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
234	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
235	dest[i].ft = attr->dest_ft;
236
237	if (mlx5_esw_indir_table_decap_vport(attr))
238	return esw_setup_decap_indir(esw, attr);
239	return 0;
240	}
241
242	static void
243	esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
244	struct mlx5_fs_chains *chains, int i)
245	{
246	if (mlx5_chains_ignore_flow_level_supported(chains))
247	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
248	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
249	dest[i].ft = mlx5_chains_get_tc_end_ft(chains);
250	}
251
252	static void
253	esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
254	struct mlx5_eswitch *esw, int i)
255	{
256	if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level))
257	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
258	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
259	dest[i].ft = mlx5_eswitch_get_slow_fdb(esw);
260	}
261
262	static int
263	esw_setup_chain_dest(struct mlx5_flow_destination *dest,
264	struct mlx5_flow_act *flow_act,
265	struct mlx5_fs_chains *chains,
266	u32 chain, u32 prio, u32 level,
267	int i)
268	{
269	struct mlx5_flow_table *ft;
270
271	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
272	ft = mlx5_chains_get_table(chains, chain, prio, level);
273	if (IS_ERR(ptr: ft))
274	return PTR_ERR(ptr: ft);
275
276	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
277	dest[i].ft = ft;
278	return 0;
279	}
280
281	static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr,
282	int from, int to)
283	{
284	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
285	struct mlx5_fs_chains *chains = esw_chains(esw);
286	int i;
287
288	for (i = from; i < to; i++)
289	if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
290	mlx5_chains_put_table(chains, chain: 0, prio: 1, level: 0);
291	else if (mlx5_esw_indir_table_needed(esw, attr, vport_num: esw_attr->dests[i].vport,
292	dest_mdev: esw_attr->dests[i].mdev))
293	mlx5_esw_indir_table_put(esw, vport: esw_attr->dests[i].vport, decap: false);
294	}
295
296	static bool
297	esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr)
298	{
299	int i;
300
301	for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
302	if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
303	return true;
304	return false;
305	}
306
307	static int
308	esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest,
309	struct mlx5_flow_act *flow_act,
310	struct mlx5_eswitch *esw,
311	struct mlx5_fs_chains *chains,
312	struct mlx5_flow_attr *attr,
313	int *i)
314	{
315	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
316	int err;
317
318	if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
319	return -EOPNOTSUPP;
320
321	/* flow steering cannot handle more than one dest with the same ft
322	* in a single flow
323	*/
324	if (esw_attr->out_count - esw_attr->split_count > 1)
325	return -EOPNOTSUPP;
326
327	err = esw_setup_chain_dest(dest, flow_act, chains, chain: attr->dest_chain, prio: 1, level: 0, i: *i);
328	if (err)
329	return err;
330
331	if (esw_attr->dests[esw_attr->split_count].pkt_reformat) {
332	flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
333	flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat;
334	}
335	(*i)++;
336
337	return 0;
338	}
339
340	static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw,
341	struct mlx5_flow_attr *attr)
342	{
343	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
344
345	esw_put_dest_tables_loop(esw, attr, from: esw_attr->split_count, to: esw_attr->out_count);
346	}
347
348	static bool
349	esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
350	{
351	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
352	bool result = false;
353	int i;
354
355	/* Indirect table is supported only for flows with in_port uplink
356	* and the destination is vport on the same eswitch as the uplink,
357	* return false in case at least one of destinations doesn't meet
358	* this criteria.
359	*/
360	for (i = esw_attr->split_count; i < esw_attr->out_count; i++) {
361	if (esw_attr->dests[i].vport_valid &&
362	mlx5_esw_indir_table_needed(esw, attr, vport_num: esw_attr->dests[i].vport,
363	dest_mdev: esw_attr->dests[i].mdev)) {
364	result = true;
365	} else {
366	result = false;
367	break;
368	}
369	}
370	return result;
371	}
372
373	static int
374	esw_setup_indir_table(struct mlx5_flow_destination *dest,
375	struct mlx5_flow_act *flow_act,
376	struct mlx5_eswitch *esw,
377	struct mlx5_flow_attr *attr,
378	int *i)
379	{
380	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
381	int j, err;
382
383	if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
384	return -EOPNOTSUPP;
385
386	for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
387	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
388	dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
389
390	dest[*i].ft = mlx5_esw_indir_table_get(esw, attr,
391	vport: esw_attr->dests[j].vport, decap: false);
392	if (IS_ERR(ptr: dest[*i].ft)) {
393	err = PTR_ERR(ptr: dest[*i].ft);
394	goto err_indir_tbl_get;
395	}
396	}
397
398	if (mlx5_esw_indir_table_decap_vport(attr)) {
399	err = esw_setup_decap_indir(esw, attr);
400	if (err)
401	goto err_indir_tbl_get;
402	}
403
404	return 0;
405
406	err_indir_tbl_get:
407	esw_put_dest_tables_loop(esw, attr, from: esw_attr->split_count, to: j);
408	return err;
409	}
410
411	static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
412	{
413	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
414
415	esw_put_dest_tables_loop(esw, attr, from: esw_attr->split_count, to: esw_attr->out_count);
416	esw_cleanup_decap_indir(esw, attr);
417	}
418
419	static void
420	esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level)
421	{
422	mlx5_chains_put_table(chains, chain, prio, level);
423	}
424
425	static bool esw_same_vhca_id(struct mlx5_core_dev *mdev1, struct mlx5_core_dev *mdev2)
426	{
427	return MLX5_CAP_GEN(mdev1, vhca_id) == MLX5_CAP_GEN(mdev2, vhca_id);
428	}
429
430	static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch *esw,
431	struct mlx5_esw_flow_attr *esw_attr,
432	int attr_idx)
433	{
434	if (esw->offloads.ft_ipsec_tx_pol &&
435	esw_attr->dests[attr_idx].vport_valid &&
436	esw_attr->dests[attr_idx].vport == MLX5_VPORT_UPLINK &&
437	/* To be aligned with software, encryption is needed only for tunnel device */
438	(esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) &&
439	esw_attr->dests[attr_idx].vport != esw_attr->in_rep->vport &&
440	esw_same_vhca_id(mdev1: esw_attr->dests[attr_idx].mdev, mdev2: esw->dev))
441	return true;
442
443	return false;
444	}
445
446	static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch *esw,
447	struct mlx5_esw_flow_attr *esw_attr)
448	{
449	int i;
450
451	if (!esw->offloads.ft_ipsec_tx_pol)
452	return true;
453
454	for (i = 0; i < esw_attr->split_count; i++)
455	if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx: i))
456	return false;
457
458	for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
459	if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx: i) &&
460	(esw_attr->out_count - esw_attr->split_count > 1))
461	return false;
462
463	return true;
464	}
465
466	static void
467	esw_setup_dest_fwd_vport(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
468	struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
469	int attr_idx, int dest_idx, bool pkt_reformat)
470	{
471	dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
472	dest[dest_idx].vport.num = esw_attr->dests[attr_idx].vport;
473	if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
474	dest[dest_idx].vport.vhca_id =
475	MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id);
476	dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
477	if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK &&
478	mlx5_lag_is_mpesw(dev: esw->dev))
479	dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK;
480	}
481	if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
482	if (pkt_reformat) {
483	flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
484	flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
485	}
486	dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID;
487	dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
488	}
489	}
490
491	static void
492	esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
493	struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
494	int attr_idx, int dest_idx, bool pkt_reformat)
495	{
496	dest[dest_idx].ft = esw->offloads.ft_ipsec_tx_pol;
497	dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
498	if (pkt_reformat &&
499	esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
500	flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
501	flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
502	}
503	}
504
505	static void
506	esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
507	struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
508	int attr_idx, int dest_idx, bool pkt_reformat)
509	{
510	if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx))
511	esw_setup_dest_fwd_ipsec(dest, flow_act, esw, esw_attr,
512	attr_idx, dest_idx, pkt_reformat);
513	else
514	esw_setup_dest_fwd_vport(dest, flow_act, esw, esw_attr,
515	attr_idx, dest_idx, pkt_reformat);
516	}
517
518	static int
519	esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
520	struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
521	int i)
522	{
523	int j;
524
525	for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++)
526	esw_setup_vport_dest(dest, flow_act, esw, esw_attr, attr_idx: j, dest_idx: i, pkt_reformat: true);
527	return i;
528	}
529
530	static bool
531	esw_src_port_rewrite_supported(struct mlx5_eswitch *esw)
532	{
533	return MLX5_CAP_GEN(esw->dev, reg_c_preserve) &&
534	mlx5_eswitch_vport_match_metadata_enabled(esw) &&
535	MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level);
536	}
537
538	static bool
539	esw_dests_to_int_external(struct mlx5_flow_destination *dests, int max_dest)
540	{
541	bool internal_dest = false, external_dest = false;
542	int i;
543
544	for (i = 0; i < max_dest; i++) {
545	if (dests[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT &&
546	dests[i].type != MLX5_FLOW_DESTINATION_TYPE_UPLINK)
547	continue;
548
549	/* Uplink dest is external, but considered as internal
550	* if there is reformat because firmware uses LB+hairpin to support it.
551	*/
552	if (dests[i].vport.num == MLX5_VPORT_UPLINK &&
553	!(dests[i].vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID))
554	external_dest = true;
555	else
556	internal_dest = true;
557
558	if (internal_dest && external_dest)
559	return true;
560	}
561
562	return false;
563	}
564
565	static int
566	esw_setup_dests(struct mlx5_flow_destination *dest,
567	struct mlx5_flow_act *flow_act,
568	struct mlx5_eswitch *esw,
569	struct mlx5_flow_attr *attr,
570	struct mlx5_flow_spec *spec,
571	int *i)
572	{
573	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
574	struct mlx5_fs_chains *chains = esw_chains(esw);
575	int err = 0;
576
577	if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) &&
578	esw_src_port_rewrite_supported(esw))
579	attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE;
580
581	if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) {
582	esw_setup_slow_path_dest(dest, flow_act, esw, i: *i);
583	(*i)++;
584	goto out;
585	}
586
587	if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) {
588	esw_setup_sampler_dest(dest, flow_act, sampler_id: attr->sample_attr.sampler_id, i: *i);
589	(*i)++;
590	} else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) {
591	esw_setup_accept_dest(dest, flow_act, chains, i: *i);
592	(*i)++;
593	} else if (attr->flags & MLX5_ATTR_FLAG_MTU) {
594	err = esw_setup_mtu_dest(dest, meter: &attr->meter_attr, i: *i);
595	(*i)++;
596	} else if (esw_is_indir_table(esw, attr)) {
597	err = esw_setup_indir_table(dest, flow_act, esw, attr, i);
598	} else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) {
599	err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i);
600	} else {
601	*i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, i: *i);
602
603	if (attr->dest_ft) {
604	err = esw_setup_ft_dest(dest, flow_act, esw, attr, i: *i);
605	(*i)++;
606	} else if (attr->dest_chain) {
607	err = esw_setup_chain_dest(dest, flow_act, chains, chain: attr->dest_chain,
608	prio: 1, level: 0, i: *i);
609	(*i)++;
610	}
611	}
612
613	out:
614	return err;
615	}
616
617	static void
618	esw_cleanup_dests(struct mlx5_eswitch *esw,
619	struct mlx5_flow_attr *attr)
620	{
621	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
622	struct mlx5_fs_chains *chains = esw_chains(esw);
623
624	if (attr->dest_ft) {
625	esw_cleanup_decap_indir(esw, attr);
626	} else if (!mlx5e_tc_attr_flags_skip(attr_flags: attr->flags)) {
627	if (attr->dest_chain)
628	esw_cleanup_chain_dest(chains, chain: attr->dest_chain, prio: 1, level: 0);
629	else if (esw_is_indir_table(esw, attr))
630	esw_cleanup_indir_table(esw, attr);
631	else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
632	esw_cleanup_chain_src_port_rewrite(esw, attr);
633	}
634	}
635
636	static void
637	esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act)
638	{
639	struct mlx5e_flow_meter_handle *meter;
640
641	meter = attr->meter_attr.meter;
642	flow_act->exe_aso.type = attr->exe_aso_type;
643	flow_act->exe_aso.object_id = meter->obj_id;
644	flow_act->exe_aso.flow_meter.meter_idx = meter->idx;
645	flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN;
646	/* use metadata reg 5 for packet color */
647	flow_act->exe_aso.return_reg_id = 5;
648	}
649
650	struct mlx5_flow_handle *
651	mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
652	struct mlx5_flow_spec *spec,
653	struct mlx5_flow_attr *attr)
654	{
655	struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
656	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
657	struct mlx5_fs_chains *chains = esw_chains(esw);
658	bool split = !!(esw_attr->split_count);
659	struct mlx5_vport_tbl_attr fwd_attr;
660	struct mlx5_flow_destination *dest;
661	struct mlx5_flow_handle *rule;
662	struct mlx5_flow_table *fdb;
663	int i = 0;
664
665	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
666	return ERR_PTR(error: -EOPNOTSUPP);
667
668	if (!mlx5_eswitch_vlan_actions_supported(dev: esw->dev, vlan_depth: 1))
669	return ERR_PTR(error: -EOPNOTSUPP);
670
671	if (!esw_flow_dests_fwd_ipsec_check(esw, esw_attr))
672	return ERR_PTR(error: -EOPNOTSUPP);
673
674	dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, size: sizeof(*dest), GFP_KERNEL);
675	if (!dest)
676	return ERR_PTR(error: -ENOMEM);
677
678	flow_act.action = attr->action;
679
680	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
681	flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]);
682	flow_act.vlan[0].vid = esw_attr->vlan_vid[0];
683	flow_act.vlan[0].prio = esw_attr->vlan_prio[0];
684	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
685	flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]);
686	flow_act.vlan[1].vid = esw_attr->vlan_vid[1];
687	flow_act.vlan[1].prio = esw_attr->vlan_prio[1];
688	}
689	}
690
691	mlx5_eswitch_set_rule_flow_source(esw, spec, attr: esw_attr);
692
693	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
694	int err;
695
696	err = esw_setup_dests(dest, flow_act: &flow_act, esw, attr, spec, i: &i);
697	if (err) {
698	rule = ERR_PTR(error: err);
699	goto err_create_goto_table;
700	}
701
702	/* Header rewrite with combined wire+loopback in FDB is not allowed */
703	if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) &&
704	esw_dests_to_int_external(dests: dest, max_dest: i)) {
705	esw_warn(esw->dev,
706	"FDB: Header rewrite with forwarding to both internal and external dests is not allowed\n");
707	rule = ERR_PTR(error: -EINVAL);
708	goto err_esw_get;
709	}
710	}
711
712	if (esw_attr->decap_pkt_reformat)
713	flow_act.pkt_reformat = esw_attr->decap_pkt_reformat;
714
715	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
716	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
717	dest[i].counter_id = mlx5_fc_id(counter: attr->counter);
718	i++;
719	}
720
721	if (attr->outer_match_level != MLX5_MATCH_NONE)
722	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
723	if (attr->inner_match_level != MLX5_MATCH_NONE)
724	spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS;
725
726	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
727	flow_act.modify_hdr = attr->modify_hdr;
728
729	if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) &&
730	attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)
731	esw_setup_meter(attr, flow_act: &flow_act);
732
733	if (split) {
734	fwd_attr.chain = attr->chain;
735	fwd_attr.prio = attr->prio;
736	fwd_attr.vport = esw_attr->in_rep->vport;
737	fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
738
739	fdb = mlx5_esw_vporttbl_get(esw, attr: &fwd_attr);
740	} else {
741	if (attr->chain || attr->prio)
742	fdb = mlx5_chains_get_table(chains, chain: attr->chain,
743	prio: attr->prio, level: 0);
744	else
745	fdb = attr->ft;
746
747	if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT))
748	mlx5_eswitch_set_rule_source_port(esw, spec, attr,
749	src_esw: esw_attr->in_mdev->priv.eswitch,
750	vport: esw_attr->in_rep->vport);
751	}
752	if (IS_ERR(ptr: fdb)) {
753	rule = ERR_CAST(ptr: fdb);
754	goto err_esw_get;
755	}
756
757	if (!i) {
758	kfree(objp: dest);
759	dest = NULL;
760	}
761
762	if (mlx5_eswitch_termtbl_required(esw, attr, flow_act: &flow_act, spec))
763	rule = mlx5_eswitch_add_termtbl_rule(esw, ft: fdb, spec, attr: esw_attr,
764	flow_act: &flow_act, dest, num_dest: i);
765	else
766	rule = mlx5_add_flow_rules(ft: fdb, spec, flow_act: &flow_act, dest, num_dest: i);
767	if (IS_ERR(ptr: rule))
768	goto err_add_rule;
769	else
770	atomic64_inc(v: &esw->offloads.num_flows);
771
772	kfree(objp: dest);
773	return rule;
774
775	err_add_rule:
776	if (split)
777	mlx5_esw_vporttbl_put(esw, attr: &fwd_attr);
778	else if (attr->chain || attr->prio)
779	mlx5_chains_put_table(chains, chain: attr->chain, prio: attr->prio, level: 0);
780	err_esw_get:
781	esw_cleanup_dests(esw, attr);
782	err_create_goto_table:
783	kfree(objp: dest);
784	return rule;
785	}
786
787	struct mlx5_flow_handle *
788	mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
789	struct mlx5_flow_spec *spec,
790	struct mlx5_flow_attr *attr)
791	{
792	struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
793	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
794	struct mlx5_fs_chains *chains = esw_chains(esw);
795	struct mlx5_vport_tbl_attr fwd_attr;
796	struct mlx5_flow_destination *dest;
797	struct mlx5_flow_table *fast_fdb;
798	struct mlx5_flow_table *fwd_fdb;
799	struct mlx5_flow_handle *rule;
800	int i, err = 0;
801
802	dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, size: sizeof(*dest), GFP_KERNEL);
803	if (!dest)
804	return ERR_PTR(error: -ENOMEM);
805
806	fast_fdb = mlx5_chains_get_table(chains, chain: attr->chain, prio: attr->prio, level: 0);
807	if (IS_ERR(ptr: fast_fdb)) {
808	rule = ERR_CAST(ptr: fast_fdb);
809	goto err_get_fast;
810	}
811
812	fwd_attr.chain = attr->chain;
813	fwd_attr.prio = attr->prio;
814	fwd_attr.vport = esw_attr->in_rep->vport;
815	fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
816	fwd_fdb = mlx5_esw_vporttbl_get(esw, attr: &fwd_attr);
817	if (IS_ERR(ptr: fwd_fdb)) {
818	rule = ERR_CAST(ptr: fwd_fdb);
819	goto err_get_fwd;
820	}
821
822	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
823	for (i = 0; i < esw_attr->split_count; i++) {
824	if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
825	/* Source port rewrite (forward to ovs internal port or statck device) isn't
826	* supported in the rule of split action.
827	*/
828	err = -EOPNOTSUPP;
829	else
830	esw_setup_vport_dest(dest, flow_act: &flow_act, esw, esw_attr, attr_idx: i, dest_idx: i, pkt_reformat: false);
831
832	if (err) {
833	rule = ERR_PTR(error: err);
834	goto err_chain_src_rewrite;
835	}
836	}
837	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
838	dest[i].ft = fwd_fdb;
839	i++;
840
841	mlx5_eswitch_set_rule_source_port(esw, spec, attr,
842	src_esw: esw_attr->in_mdev->priv.eswitch,
843	vport: esw_attr->in_rep->vport);
844
845	if (attr->outer_match_level != MLX5_MATCH_NONE)
846	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
847
848	flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
849	rule = mlx5_add_flow_rules(ft: fast_fdb, spec, flow_act: &flow_act, dest, num_dest: i);
850
851	if (IS_ERR(ptr: rule)) {
852	i = esw_attr->split_count;
853	goto err_chain_src_rewrite;
854	}
855
856	atomic64_inc(v: &esw->offloads.num_flows);
857
858	kfree(objp: dest);
859	return rule;
860	err_chain_src_rewrite:
861	mlx5_esw_vporttbl_put(esw, attr: &fwd_attr);
862	err_get_fwd:
863	mlx5_chains_put_table(chains, chain: attr->chain, prio: attr->prio, level: 0);
864	err_get_fast:
865	kfree(objp: dest);
866	return rule;
867	}
868
869	static void
870	__mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
871	struct mlx5_flow_handle *rule,
872	struct mlx5_flow_attr *attr,
873	bool fwd_rule)
874	{
875	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
876	struct mlx5_fs_chains *chains = esw_chains(esw);
877	bool split = (esw_attr->split_count > 0);
878	struct mlx5_vport_tbl_attr fwd_attr;
879	int i;
880
881	mlx5_del_flow_rules(fr: rule);
882
883	if (!mlx5e_tc_attr_flags_skip(attr_flags: attr->flags)) {
884	/* unref the term table */
885	for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
886	if (esw_attr->dests[i].termtbl)
887	mlx5_eswitch_termtbl_put(esw, tt: esw_attr->dests[i].termtbl);
888	}
889	}
890
891	atomic64_dec(v: &esw->offloads.num_flows);
892
893	if (fwd_rule || split) {
894	fwd_attr.chain = attr->chain;
895	fwd_attr.prio = attr->prio;
896	fwd_attr.vport = esw_attr->in_rep->vport;
897	fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
898	}
899
900	if (fwd_rule) {
901	mlx5_esw_vporttbl_put(esw, attr: &fwd_attr);
902	mlx5_chains_put_table(chains, chain: attr->chain, prio: attr->prio, level: 0);
903	} else {
904	if (split)
905	mlx5_esw_vporttbl_put(esw, attr: &fwd_attr);
906	else if (attr->chain || attr->prio)
907	mlx5_chains_put_table(chains, chain: attr->chain, prio: attr->prio, level: 0);
908	esw_cleanup_dests(esw, attr);
909	}
910	}
911
912	void
913	mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
914	struct mlx5_flow_handle *rule,
915	struct mlx5_flow_attr *attr)
916	{
917	__mlx5_eswitch_del_rule(esw, rule, attr, fwd_rule: false);
918	}
919
920	void
921	mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw,
922	struct mlx5_flow_handle *rule,
923	struct mlx5_flow_attr *attr)
924	{
925	__mlx5_eswitch_del_rule(esw, rule, attr, fwd_rule: true);
926	}
927
928	struct mlx5_flow_handle *
929	mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw,
930	struct mlx5_eswitch *from_esw,
931	struct mlx5_eswitch_rep *rep,
932	u32 sqn)
933	{
934	struct mlx5_flow_act flow_act = {0};
935	struct mlx5_flow_destination dest = {};
936	struct mlx5_flow_handle *flow_rule;
937	struct mlx5_flow_spec *spec;
938	void *misc;
939	u16 vport;
940
941	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
942	if (!spec) {
943	flow_rule = ERR_PTR(error: -ENOMEM);
944	goto out;
945	}
946
947	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
948	MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn);
949
950	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
951	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn);
952
953	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
954
955	/* source vport is the esw manager */
956	vport = from_esw->manager_vport;
957
958	if (mlx5_eswitch_vport_match_metadata_enabled(esw: on_esw)) {
959	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
960	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
961	mlx5_eswitch_get_vport_metadata_for_match(from_esw, vport));
962
963	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
964	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
965	mlx5_eswitch_get_vport_metadata_mask());
966
967	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
968	} else {
969	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
970	MLX5_SET(fte_match_set_misc, misc, source_port, vport);
971
972	if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
973	MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
974	MLX5_CAP_GEN(from_esw->dev, vhca_id));
975
976	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
977	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
978
979	if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
980	MLX5_SET_TO_ONES(fte_match_set_misc, misc,
981	source_eswitch_owner_vhca_id);
982
983	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
984	}
985
986	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
987	dest.vport.num = rep->vport;
988	dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
989	dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
990	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
991
992	if (rep->vport == MLX5_VPORT_UPLINK &&
993	on_esw == from_esw && on_esw->offloads.ft_ipsec_tx_pol) {
994	dest.ft = on_esw->offloads.ft_ipsec_tx_pol;
995	flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL;
996	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
997	} else {
998	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
999	dest.vport.num = rep->vport;
1000	dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
1001	dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1002	}
1003
1004	if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) &&
1005	rep->vport == MLX5_VPORT_UPLINK)
1006	spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
1007
1008	flow_rule = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw: on_esw),
1009	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1010	if (IS_ERR(ptr: flow_rule))
1011	esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n",
1012	PTR_ERR(flow_rule));
1013	out:
1014	kvfree(addr: spec);
1015	return flow_rule;
1016	}
1017	EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule);
1018
1019	void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
1020	{
1021	mlx5_del_flow_rules(fr: rule);
1022	}
1023
1024	void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule)
1025	{
1026	if (rule)
1027	mlx5_del_flow_rules(fr: rule);
1028	}
1029
1030	struct mlx5_flow_handle *
1031	mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num)
1032	{
1033	struct mlx5_flow_destination dest = {};
1034	struct mlx5_flow_act flow_act = {0};
1035	struct mlx5_flow_handle *flow_rule;
1036	struct mlx5_flow_spec *spec;
1037
1038	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
1039	if (!spec)
1040	return ERR_PTR(error: -ENOMEM);
1041
1042	MLX5_SET(fte_match_param, spec->match_criteria,
1043	misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
1044	MLX5_SET(fte_match_param, spec->match_criteria,
1045	misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1046	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1,
1047	ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK);
1048
1049	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1050	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1051	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1052
1053	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0,
1054	mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
1055	dest.vport.num = vport_num;
1056
1057	flow_rule = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw),
1058	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1059	if (IS_ERR(ptr: flow_rule))
1060	esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n",
1061	vport_num, PTR_ERR(flow_rule));
1062
1063	kvfree(addr: spec);
1064	return flow_rule;
1065	}
1066
1067	static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw)
1068	{
1069	return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
1070	MLX5_FDB_TO_VPORT_REG_C_1;
1071	}
1072
1073	static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable)
1074	{
1075	u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {};
1076	u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {};
1077	u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {};
1078	u8 curr, wanted;
1079	int err;
1080
1081	if (!mlx5_eswitch_reg_c1_loopback_supported(esw) &&
1082	!mlx5_eswitch_vport_match_metadata_enabled(esw))
1083	return 0;
1084
1085	MLX5_SET(query_esw_vport_context_in, in, opcode,
1086	MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT);
1087	err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out);
1088	if (err)
1089	return err;
1090
1091	curr = MLX5_GET(query_esw_vport_context_out, out,
1092	esw_vport_context.fdb_to_vport_reg_c_id);
1093	wanted = MLX5_FDB_TO_VPORT_REG_C_0;
1094	if (mlx5_eswitch_reg_c1_loopback_supported(esw))
1095	wanted |= MLX5_FDB_TO_VPORT_REG_C_1;
1096
1097	if (enable)
1098	curr |= wanted;
1099	else
1100	curr &= ~wanted;
1101
1102	MLX5_SET(modify_esw_vport_context_in, min,
1103	esw_vport_context.fdb_to_vport_reg_c_id, curr);
1104	MLX5_SET(modify_esw_vport_context_in, min,
1105	field_select.fdb_to_vport_reg_c_id, 1);
1106
1107	err = mlx5_eswitch_modify_esw_vport_context(dev: esw->dev, vport: 0, other_vport: false, in: min);
1108	if (!err) {
1109	if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1))
1110	esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1111	else
1112	esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1113	}
1114
1115	return err;
1116	}
1117
1118	static void peer_miss_rules_setup(struct mlx5_eswitch *esw,
1119	struct mlx5_core_dev *peer_dev,
1120	struct mlx5_flow_spec *spec,
1121	struct mlx5_flow_destination *dest)
1122	{
1123	void *misc;
1124
1125	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1126	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1127	misc_parameters_2);
1128	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1129	mlx5_eswitch_get_vport_metadata_mask());
1130
1131	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1132	} else {
1133	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1134	misc_parameters);
1135
1136	MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1137	MLX5_CAP_GEN(peer_dev, vhca_id));
1138
1139	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1140
1141	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1142	misc_parameters);
1143	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1144	MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1145	source_eswitch_owner_vhca_id);
1146	}
1147
1148	dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1149	dest->vport.num = peer_dev->priv.eswitch->manager_vport;
1150	dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
1151	dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1152	}
1153
1154	static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw,
1155	struct mlx5_eswitch *peer_esw,
1156	struct mlx5_flow_spec *spec,
1157	u16 vport)
1158	{
1159	void *misc;
1160
1161	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1162	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1163	misc_parameters_2);
1164	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1165	mlx5_eswitch_get_vport_metadata_for_match(peer_esw,
1166	vport));
1167	} else {
1168	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1169	misc_parameters);
1170	MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1171	}
1172	}
1173
1174	static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1175	struct mlx5_core_dev *peer_dev)
1176	{
1177	struct mlx5_flow_destination dest = {};
1178	struct mlx5_flow_act flow_act = {0};
1179	struct mlx5_flow_handle **flows;
1180	/* total vports is the same for both e-switches */
1181	int nvports = esw->total_vports;
1182	struct mlx5_flow_handle *flow;
1183	struct mlx5_flow_spec *spec;
1184	struct mlx5_vport *vport;
1185	int err, pfindex;
1186	unsigned long i;
1187	void *misc;
1188
1189	if (!MLX5_VPORT_MANAGER(esw->dev) && !mlx5_core_is_ecpf_esw_manager(dev: esw->dev))
1190	return 0;
1191
1192	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
1193	if (!spec)
1194	return -ENOMEM;
1195
1196	peer_miss_rules_setup(esw, peer_dev, spec, dest: &dest);
1197
1198	flows = kvcalloc(n: nvports, size: sizeof(*flows), GFP_KERNEL);
1199	if (!flows) {
1200	err = -ENOMEM;
1201	goto alloc_flows_err;
1202	}
1203
1204	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1205	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1206	misc_parameters);
1207
1208	if (mlx5_core_is_ecpf_esw_manager(dev: esw->dev)) {
1209	vport = mlx5_eswitch_get_vport(esw, vport_num: MLX5_VPORT_PF);
1210	esw_set_peer_miss_rule_source_port(esw, peer_esw: peer_dev->priv.eswitch,
1211	spec, vport: MLX5_VPORT_PF);
1212
1213	flow = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw),
1214	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1215	if (IS_ERR(ptr: flow)) {
1216	err = PTR_ERR(ptr: flow);
1217	goto add_pf_flow_err;
1218	}
1219	flows[vport->index] = flow;
1220	}
1221
1222	if (mlx5_ecpf_vport_exists(dev: esw->dev)) {
1223	vport = mlx5_eswitch_get_vport(esw, vport_num: MLX5_VPORT_ECPF);
1224	MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF);
1225	flow = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw),
1226	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1227	if (IS_ERR(ptr: flow)) {
1228	err = PTR_ERR(ptr: flow);
1229	goto add_ecpf_flow_err;
1230	}
1231	flows[vport->index] = flow;
1232	}
1233
1234	mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1235	esw_set_peer_miss_rule_source_port(esw,
1236	peer_esw: peer_dev->priv.eswitch,
1237	spec, vport: vport->vport);
1238
1239	flow = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw),
1240	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1241	if (IS_ERR(ptr: flow)) {
1242	err = PTR_ERR(ptr: flow);
1243	goto add_vf_flow_err;
1244	}
1245	flows[vport->index] = flow;
1246	}
1247
1248	if (mlx5_core_ec_sriov_enabled(dev: esw->dev)) {
1249	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) {
1250	if (i >= mlx5_core_max_ec_vfs(dev: peer_dev))
1251	break;
1252	esw_set_peer_miss_rule_source_port(esw, peer_esw: peer_dev->priv.eswitch,
1253	spec, vport: vport->vport);
1254	flow = mlx5_add_flow_rules(ft: esw->fdb_table.offloads.slow_fdb,
1255	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1256	if (IS_ERR(ptr: flow)) {
1257	err = PTR_ERR(ptr: flow);
1258	goto add_ec_vf_flow_err;
1259	}
1260	flows[vport->index] = flow;
1261	}
1262	}
1263
1264	pfindex = mlx5_get_dev_index(dev: peer_dev);
1265	if (pfindex >= MLX5_MAX_PORTS) {
1266	esw_warn(esw->dev, "Peer dev index(%d) is over the max num defined(%d)\n",
1267	pfindex, MLX5_MAX_PORTS);
1268	err = -EINVAL;
1269	goto add_ec_vf_flow_err;
1270	}
1271	esw->fdb_table.offloads.peer_miss_rules[pfindex] = flows;
1272
1273	kvfree(addr: spec);
1274	return 0;
1275
1276	add_ec_vf_flow_err:
1277	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) {
1278	if (!flows[vport->index])
1279	continue;
1280	mlx5_del_flow_rules(fr: flows[vport->index]);
1281	}
1282	add_vf_flow_err:
1283	mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1284	if (!flows[vport->index])
1285	continue;
1286	mlx5_del_flow_rules(fr: flows[vport->index]);
1287	}
1288	if (mlx5_ecpf_vport_exists(dev: esw->dev)) {
1289	vport = mlx5_eswitch_get_vport(esw, vport_num: MLX5_VPORT_ECPF);
1290	mlx5_del_flow_rules(fr: flows[vport->index]);
1291	}
1292	add_ecpf_flow_err:
1293	if (mlx5_core_is_ecpf_esw_manager(dev: esw->dev)) {
1294	vport = mlx5_eswitch_get_vport(esw, vport_num: MLX5_VPORT_PF);
1295	mlx5_del_flow_rules(fr: flows[vport->index]);
1296	}
1297	add_pf_flow_err:
1298	esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err);
1299	kvfree(addr: flows);
1300	alloc_flows_err:
1301	kvfree(addr: spec);
1302	return err;
1303	}
1304
1305	static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1306	struct mlx5_core_dev *peer_dev)
1307	{
1308	u16 peer_index = mlx5_get_dev_index(dev: peer_dev);
1309	struct mlx5_flow_handle **flows;
1310	struct mlx5_vport *vport;
1311	unsigned long i;
1312
1313	flows = esw->fdb_table.offloads.peer_miss_rules[peer_index];
1314	if (!flows)
1315	return;
1316
1317	if (mlx5_core_ec_sriov_enabled(dev: esw->dev)) {
1318	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) {
1319	/* The flow for a particular vport could be NULL if the other ECPF
1320	* has fewer or no VFs enabled
1321	*/
1322	if (!flows[vport->index])
1323	continue;
1324	mlx5_del_flow_rules(fr: flows[vport->index]);
1325	}
1326	}
1327
1328	mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev))
1329	mlx5_del_flow_rules(fr: flows[vport->index]);
1330
1331	if (mlx5_ecpf_vport_exists(dev: esw->dev)) {
1332	vport = mlx5_eswitch_get_vport(esw, vport_num: MLX5_VPORT_ECPF);
1333	mlx5_del_flow_rules(fr: flows[vport->index]);
1334	}
1335
1336	if (mlx5_core_is_ecpf_esw_manager(dev: esw->dev)) {
1337	vport = mlx5_eswitch_get_vport(esw, vport_num: MLX5_VPORT_PF);
1338	mlx5_del_flow_rules(fr: flows[vport->index]);
1339	}
1340
1341	kvfree(addr: flows);
1342	esw->fdb_table.offloads.peer_miss_rules[peer_index] = NULL;
1343	}
1344
1345	static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
1346	{
1347	struct mlx5_flow_act flow_act = {0};
1348	struct mlx5_flow_destination dest = {};
1349	struct mlx5_flow_handle *flow_rule = NULL;
1350	struct mlx5_flow_spec *spec;
1351	void *headers_c;
1352	void *headers_v;
1353	int err = 0;
1354	u8 *dmac_c;
1355	u8 *dmac_v;
1356
1357	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
1358	if (!spec) {
1359	err = -ENOMEM;
1360	goto out;
1361	}
1362
1363	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1364	headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1365	outer_headers);
1366	dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c,
1367	outer_headers.dmac_47_16);
1368	dmac_c[0] = 0x01;
1369
1370	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1371	dest.vport.num = esw->manager_vport;
1372	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1373
1374	flow_rule = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw),
1375	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1376	if (IS_ERR(ptr: flow_rule)) {
1377	err = PTR_ERR(ptr: flow_rule);
1378	esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);
1379	goto out;
1380	}
1381
1382	esw->fdb_table.offloads.miss_rule_uni = flow_rule;
1383
1384	headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1385	outer_headers);
1386	dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
1387	outer_headers.dmac_47_16);
1388	dmac_v[0] = 0x01;
1389	flow_rule = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw),
1390	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1391	if (IS_ERR(ptr: flow_rule)) {
1392	err = PTR_ERR(ptr: flow_rule);
1393	esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
1394	mlx5_del_flow_rules(fr: esw->fdb_table.offloads.miss_rule_uni);
1395	goto out;
1396	}
1397
1398	esw->fdb_table.offloads.miss_rule_multi = flow_rule;
1399
1400	out:
1401	kvfree(addr: spec);
1402	return err;
1403	}
1404
1405	struct mlx5_flow_handle *
1406	esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
1407	{
1408	struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
1409	struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore;
1410	struct mlx5_flow_context *flow_context;
1411	struct mlx5_flow_handle *flow_rule;
1412	struct mlx5_flow_destination dest;
1413	struct mlx5_flow_spec *spec;
1414	void *misc;
1415
1416	if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1417	return ERR_PTR(error: -EOPNOTSUPP);
1418
1419	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
1420	if (!spec)
1421	return ERR_PTR(error: -ENOMEM);
1422
1423	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1424	misc_parameters_2);
1425	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1426	ESW_REG_C0_USER_DATA_METADATA_MASK);
1427	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1428	misc_parameters_2);
1429	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag);
1430	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1431	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1432	MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1433	flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id;
1434
1435	flow_context = &spec->flow_context;
1436	flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
1437	flow_context->flow_tag = tag;
1438	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1439	dest.ft = esw->offloads.ft_offloads;
1440
1441	flow_rule = mlx5_add_flow_rules(ft, spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1442	kvfree(addr: spec);
1443
1444	if (IS_ERR(ptr: flow_rule))
1445	esw_warn(esw->dev,
1446	"Failed to create restore rule for tag: %d, err(%d)\n",
1447	tag, (int)PTR_ERR(flow_rule));
1448
1449	return flow_rule;
1450	}
1451
1452	#define MAX_PF_SQ 256
1453	#define MAX_SQ_NVPORTS 32
1454
1455	void
1456	mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch *esw,
1457	u32 *flow_group_in,
1458	int match_params)
1459	{
1460	void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1461	flow_group_in,
1462	match_criteria);
1463
1464	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1465	MLX5_SET(create_flow_group_in, flow_group_in,
1466	match_criteria_enable,
1467	MLX5_MATCH_MISC_PARAMETERS_2 | match_params);
1468
1469	MLX5_SET(fte_match_param, match_criteria,
1470	misc_parameters_2.metadata_reg_c_0,
1471	mlx5_eswitch_get_vport_metadata_mask());
1472	} else {
1473	MLX5_SET(create_flow_group_in, flow_group_in,
1474	match_criteria_enable,
1475	MLX5_MATCH_MISC_PARAMETERS | match_params);
1476
1477	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1478	misc_parameters.source_port);
1479	}
1480	}
1481
1482	#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
1483	static void esw_vport_tbl_put(struct mlx5_eswitch *esw)
1484	{
1485	struct mlx5_vport_tbl_attr attr;
1486	struct mlx5_vport *vport;
1487	unsigned long i;
1488
1489	attr.chain = 0;
1490	attr.prio = 1;
1491	mlx5_esw_for_each_vport(esw, i, vport) {
1492	attr.vport = vport->vport;
1493	attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1494	mlx5_esw_vporttbl_put(esw, attr: &attr);
1495	}
1496	}
1497
1498	static int esw_vport_tbl_get(struct mlx5_eswitch *esw)
1499	{
1500	struct mlx5_vport_tbl_attr attr;
1501	struct mlx5_flow_table *fdb;
1502	struct mlx5_vport *vport;
1503	unsigned long i;
1504
1505	attr.chain = 0;
1506	attr.prio = 1;
1507	mlx5_esw_for_each_vport(esw, i, vport) {
1508	attr.vport = vport->vport;
1509	attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1510	fdb = mlx5_esw_vporttbl_get(esw, attr: &attr);
1511	if (IS_ERR(ptr: fdb))
1512	goto out;
1513	}
1514	return 0;
1515
1516	out:
1517	esw_vport_tbl_put(esw);
1518	return PTR_ERR(ptr: fdb);
1519	}
1520
1521	#define fdb_modify_header_fwd_to_table_supported(esw) \
1522	(MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
1523	static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags)
1524	{
1525	struct mlx5_core_dev *dev = esw->dev;
1526
1527	if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level))
1528	*flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
1529
1530	if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) &&
1531	esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
1532	*flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1533	esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1534	} else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1535	*flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1536	esw_warn(dev, "Tc chains and priorities offload aren't supported\n");
1537	} else if (!fdb_modify_header_fwd_to_table_supported(esw)) {
1538	/* Disabled when ttl workaround is needed, e.g
1539	* when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1540	*/
1541	esw_warn(dev,
1542	"Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1543	*flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1544	} else {
1545	*flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1546	esw_info(dev, "Supported tc chains and prios offload\n");
1547	}
1548
1549	if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1550	*flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED;
1551	}
1552
1553	static int
1554	esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1555	{
1556	struct mlx5_core_dev *dev = esw->dev;
1557	struct mlx5_flow_table *nf_ft, *ft;
1558	struct mlx5_chains_attr attr = {};
1559	struct mlx5_fs_chains *chains;
1560	int err;
1561
1562	esw_init_chains_offload_flags(esw, flags: &attr.flags);
1563	attr.ns = MLX5_FLOW_NAMESPACE_FDB;
1564	attr.max_grp_num = esw->params.large_group_num;
1565	attr.default_ft = miss_fdb;
1566	attr.mapping = esw->offloads.reg_c0_obj_pool;
1567
1568	chains = mlx5_chains_create(dev, attr: &attr);
1569	if (IS_ERR(ptr: chains)) {
1570	err = PTR_ERR(ptr: chains);
1571	esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
1572	return err;
1573	}
1574	mlx5_chains_print_info(chains);
1575
1576	esw->fdb_table.offloads.esw_chains_priv = chains;
1577
1578	/* Create tc_end_ft which is the always created ft chain */
1579	nf_ft = mlx5_chains_get_table(chains, chain: mlx5_chains_get_nf_ft_chain(chains),
1580	prio: 1, level: 0);
1581	if (IS_ERR(ptr: nf_ft)) {
1582	err = PTR_ERR(ptr: nf_ft);
1583	goto nf_ft_err;
1584	}
1585
1586	/* Always open the root for fast path */
1587	ft = mlx5_chains_get_table(chains, chain: 0, prio: 1, level: 0);
1588	if (IS_ERR(ptr: ft)) {
1589	err = PTR_ERR(ptr: ft);
1590	goto level_0_err;
1591	}
1592
1593	/* Open level 1 for split fdb rules now if prios isn't supported */
1594	if (!mlx5_chains_prios_supported(chains)) {
1595	err = esw_vport_tbl_get(esw);
1596	if (err)
1597	goto level_1_err;
1598	}
1599
1600	mlx5_chains_set_end_ft(chains, ft: nf_ft);
1601
1602	return 0;
1603
1604	level_1_err:
1605	mlx5_chains_put_table(chains, chain: 0, prio: 1, level: 0);
1606	level_0_err:
1607	mlx5_chains_put_table(chains, chain: mlx5_chains_get_nf_ft_chain(chains), prio: 1, level: 0);
1608	nf_ft_err:
1609	mlx5_chains_destroy(chains);
1610	esw->fdb_table.offloads.esw_chains_priv = NULL;
1611
1612	return err;
1613	}
1614
1615	static void
1616	esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1617	{
1618	if (!mlx5_chains_prios_supported(chains))
1619	esw_vport_tbl_put(esw);
1620	mlx5_chains_put_table(chains, chain: 0, prio: 1, level: 0);
1621	mlx5_chains_put_table(chains, chain: mlx5_chains_get_nf_ft_chain(chains), prio: 1, level: 0);
1622	mlx5_chains_destroy(chains);
1623	}
1624
1625	#else /* CONFIG_MLX5_CLS_ACT */
1626
1627	static int
1628	esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1629	{ return 0; }
1630
1631	static void
1632	esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1633	{}
1634
1635	#endif
1636
1637	static int
1638	esw_create_send_to_vport_group(struct mlx5_eswitch *esw,
1639	struct mlx5_flow_table *fdb,
1640	u32 *flow_group_in,
1641	int *ix)
1642	{
1643	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1644	struct mlx5_flow_group *g;
1645	void *match_criteria;
1646	int count, err = 0;
1647
1648	memset(flow_group_in, 0, inlen);
1649
1650	mlx5_esw_set_flow_group_source_port(esw, flow_group_in, match_params: MLX5_MATCH_MISC_PARAMETERS);
1651
1652	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1653	MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
1654
1655	if (!mlx5_eswitch_vport_match_metadata_enabled(esw) &&
1656	MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1657	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1658	misc_parameters.source_eswitch_owner_vhca_id);
1659	MLX5_SET(create_flow_group_in, flow_group_in,
1660	source_eswitch_owner_vhca_id_valid, 1);
1661	}
1662
1663	/* See comment at table_size calculation */
1664	count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ);
1665	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1666	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1);
1667	*ix += count;
1668
1669	g = mlx5_create_flow_group(ft: fdb, in: flow_group_in);
1670	if (IS_ERR(ptr: g)) {
1671	err = PTR_ERR(ptr: g);
1672	esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err);
1673	goto out;
1674	}
1675	esw->fdb_table.offloads.send_to_vport_grp = g;
1676
1677	out:
1678	return err;
1679	}
1680
1681	static int
1682	esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw,
1683	struct mlx5_flow_table *fdb,
1684	u32 *flow_group_in,
1685	int *ix)
1686	{
1687	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1688	struct mlx5_flow_group *g;
1689	void *match_criteria;
1690	int err = 0;
1691
1692	if (!esw_src_port_rewrite_supported(esw))
1693	return 0;
1694
1695	memset(flow_group_in, 0, inlen);
1696
1697	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1698	MLX5_MATCH_MISC_PARAMETERS_2);
1699
1700	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1701
1702	MLX5_SET(fte_match_param, match_criteria,
1703	misc_parameters_2.metadata_reg_c_0,
1704	mlx5_eswitch_get_vport_metadata_mask());
1705	MLX5_SET(fte_match_param, match_criteria,
1706	misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1707
1708	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1709	MLX5_SET(create_flow_group_in, flow_group_in,
1710	end_flow_index, *ix + esw->total_vports - 1);
1711	*ix += esw->total_vports;
1712
1713	g = mlx5_create_flow_group(ft: fdb, in: flow_group_in);
1714	if (IS_ERR(ptr: g)) {
1715	err = PTR_ERR(ptr: g);
1716	esw_warn(esw->dev,
1717	"Failed to create send-to-vport meta flow group err(%d)\n", err);
1718	goto send_vport_meta_err;
1719	}
1720	esw->fdb_table.offloads.send_to_vport_meta_grp = g;
1721
1722	return 0;
1723
1724	send_vport_meta_err:
1725	return err;
1726	}
1727
1728	static int
1729	esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw,
1730	struct mlx5_flow_table *fdb,
1731	u32 *flow_group_in,
1732	int *ix)
1733	{
1734	int max_peer_ports = (esw->total_vports - 1) * (MLX5_MAX_PORTS - 1);
1735	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1736	struct mlx5_flow_group *g;
1737	void *match_criteria;
1738	int err = 0;
1739
1740	if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
1741	return 0;
1742
1743	memset(flow_group_in, 0, inlen);
1744
1745	mlx5_esw_set_flow_group_source_port(esw, flow_group_in, match_params: 0);
1746
1747	if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1748	match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1749	flow_group_in,
1750	match_criteria);
1751
1752	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1753	misc_parameters.source_eswitch_owner_vhca_id);
1754
1755	MLX5_SET(create_flow_group_in, flow_group_in,
1756	source_eswitch_owner_vhca_id_valid, 1);
1757	}
1758
1759	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1760	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1761	*ix + max_peer_ports);
1762	*ix += max_peer_ports + 1;
1763
1764	g = mlx5_create_flow_group(ft: fdb, in: flow_group_in);
1765	if (IS_ERR(ptr: g)) {
1766	err = PTR_ERR(ptr: g);
1767	esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err);
1768	goto out;
1769	}
1770	esw->fdb_table.offloads.peer_miss_grp = g;
1771
1772	out:
1773	return err;
1774	}
1775
1776	static int
1777	esw_create_miss_group(struct mlx5_eswitch *esw,
1778	struct mlx5_flow_table *fdb,
1779	u32 *flow_group_in,
1780	int *ix)
1781	{
1782	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1783	struct mlx5_flow_group *g;
1784	void *match_criteria;
1785	int err = 0;
1786	u8 *dmac;
1787
1788	memset(flow_group_in, 0, inlen);
1789
1790	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1791	MLX5_MATCH_OUTER_HEADERS);
1792	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1793	match_criteria);
1794	dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
1795	outer_headers.dmac_47_16);
1796	dmac[0] = 0x01;
1797
1798	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1799	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1800	*ix + MLX5_ESW_MISS_FLOWS);
1801
1802	g = mlx5_create_flow_group(ft: fdb, in: flow_group_in);
1803	if (IS_ERR(ptr: g)) {
1804	err = PTR_ERR(ptr: g);
1805	esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err);
1806	goto miss_err;
1807	}
1808	esw->fdb_table.offloads.miss_grp = g;
1809
1810	err = esw_add_fdb_miss_rule(esw);
1811	if (err)
1812	goto miss_rule_err;
1813
1814	return 0;
1815
1816	miss_rule_err:
1817	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.miss_grp);
1818	miss_err:
1819	return err;
1820	}
1821
1822	static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
1823	{
1824	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1825	struct mlx5_flow_table_attr ft_attr = {};
1826	struct mlx5_core_dev *dev = esw->dev;
1827	struct mlx5_flow_namespace *root_ns;
1828	struct mlx5_flow_table *fdb = NULL;
1829	int table_size, ix = 0, err = 0;
1830	u32 flags = 0, *flow_group_in;
1831
1832	esw_debug(esw->dev, "Create offloads FDB Tables\n");
1833
1834	flow_group_in = kvzalloc(size: inlen, GFP_KERNEL);
1835	if (!flow_group_in)
1836	return -ENOMEM;
1837
1838	root_ns = mlx5_get_flow_namespace(dev, type: MLX5_FLOW_NAMESPACE_FDB);
1839	if (!root_ns) {
1840	esw_warn(dev, "Failed to get FDB flow namespace\n");
1841	err = -EOPNOTSUPP;
1842	goto ns_err;
1843	}
1844	esw->fdb_table.offloads.ns = root_ns;
1845	err = mlx5_flow_namespace_set_mode(ns: root_ns,
1846	mode: esw->dev->priv.steering->mode);
1847	if (err) {
1848	esw_warn(dev, "Failed to set FDB namespace steering mode\n");
1849	goto ns_err;
1850	}
1851
1852	/* To be strictly correct:
1853	* MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ)
1854	* should be:
1855	* esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1856	* peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ
1857	* but as the peer device might not be in switchdev mode it's not
1858	* possible. We use the fact that by default FW sets max vfs and max sfs
1859	* to the same value on both devices. If it needs to be changed in the future note
1860	* the peer miss group should also be created based on the number of
1861	* total vports of the peer (currently is also uses esw->total_vports).
1862	*/
1863	table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) +
1864	esw->total_vports * MLX5_MAX_PORTS + MLX5_ESW_MISS_FLOWS;
1865
1866	/* create the slow path fdb with encap set, so further table instances
1867	* can be created at run time while VFs are probed if the FW allows that.
1868	*/
1869	if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1870	flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
1871	MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
1872
1873	ft_attr.flags = flags;
1874	ft_attr.max_fte = table_size;
1875	ft_attr.prio = FDB_SLOW_PATH;
1876
1877	fdb = mlx5_create_flow_table(ns: root_ns, ft_attr: &ft_attr);
1878	if (IS_ERR(ptr: fdb)) {
1879	err = PTR_ERR(ptr: fdb);
1880	esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
1881	goto slow_fdb_err;
1882	}
1883	esw->fdb_table.offloads.slow_fdb = fdb;
1884
1885	/* Create empty TC-miss managed table. This allows plugging in following
1886	* priorities without directly exposing their level 0 table to
1887	* eswitch_offloads and passing it as miss_fdb to following call to
1888	* esw_chains_create().
1889	*/
1890	memset(&ft_attr, 0, sizeof(ft_attr));
1891	ft_attr.prio = FDB_TC_MISS;
1892	esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(ns: root_ns, ft_attr: &ft_attr);
1893	if (IS_ERR(ptr: esw->fdb_table.offloads.tc_miss_table)) {
1894	err = PTR_ERR(ptr: esw->fdb_table.offloads.tc_miss_table);
1895	esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err);
1896	goto tc_miss_table_err;
1897	}
1898
1899	err = esw_chains_create(esw, miss_fdb: esw->fdb_table.offloads.tc_miss_table);
1900	if (err) {
1901	esw_warn(dev, "Failed to open fdb chains err(%d)\n", err);
1902	goto fdb_chains_err;
1903	}
1904
1905	err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, ix: &ix);
1906	if (err)
1907	goto send_vport_err;
1908
1909	err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, ix: &ix);
1910	if (err)
1911	goto send_vport_meta_err;
1912
1913	err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, ix: &ix);
1914	if (err)
1915	goto peer_miss_err;
1916
1917	err = esw_create_miss_group(esw, fdb, flow_group_in, ix: &ix);
1918	if (err)
1919	goto miss_err;
1920
1921	kvfree(addr: flow_group_in);
1922	return 0;
1923
1924	miss_err:
1925	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1926	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.peer_miss_grp);
1927	peer_miss_err:
1928	if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1929	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.send_to_vport_meta_grp);
1930	send_vport_meta_err:
1931	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.send_to_vport_grp);
1932	send_vport_err:
1933	esw_chains_destroy(esw, esw_chains(esw));
1934	fdb_chains_err:
1935	mlx5_destroy_flow_table(ft: esw->fdb_table.offloads.tc_miss_table);
1936	tc_miss_table_err:
1937	mlx5_destroy_flow_table(ft: mlx5_eswitch_get_slow_fdb(esw));
1938	slow_fdb_err:
1939	/* Holds true only as long as DMFS is the default */
1940	mlx5_flow_namespace_set_mode(ns: root_ns, mode: MLX5_FLOW_STEERING_MODE_DMFS);
1941	ns_err:
1942	kvfree(addr: flow_group_in);
1943	return err;
1944	}
1945
1946	static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
1947	{
1948	if (!mlx5_eswitch_get_slow_fdb(esw))
1949	return;
1950
1951	esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
1952	mlx5_del_flow_rules(fr: esw->fdb_table.offloads.miss_rule_multi);
1953	mlx5_del_flow_rules(fr: esw->fdb_table.offloads.miss_rule_uni);
1954	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.send_to_vport_grp);
1955	if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1956	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.send_to_vport_meta_grp);
1957	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1958	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.peer_miss_grp);
1959	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.miss_grp);
1960
1961	esw_chains_destroy(esw, esw_chains(esw));
1962
1963	mlx5_destroy_flow_table(ft: esw->fdb_table.offloads.tc_miss_table);
1964	mlx5_destroy_flow_table(ft: mlx5_eswitch_get_slow_fdb(esw));
1965	/* Holds true only as long as DMFS is the default */
1966	mlx5_flow_namespace_set_mode(ns: esw->fdb_table.offloads.ns,
1967	mode: MLX5_FLOW_STEERING_MODE_DMFS);
1968	atomic64_set(v: &esw->user_count, i: 0);
1969	}
1970
1971	static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw)
1972	{
1973	int nvports;
1974
1975	nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS;
1976	if (mlx5e_tc_int_port_supported(esw))
1977	nvports += MLX5E_TC_MAX_INT_PORT_NUM;
1978
1979	return nvports;
1980	}
1981
1982	static int esw_create_offloads_table(struct mlx5_eswitch *esw)
1983	{
1984	struct mlx5_flow_table_attr ft_attr = {};
1985	struct mlx5_core_dev *dev = esw->dev;
1986	struct mlx5_flow_table *ft_offloads;
1987	struct mlx5_flow_namespace *ns;
1988	int err = 0;
1989
1990	ns = mlx5_get_flow_namespace(dev, type: MLX5_FLOW_NAMESPACE_OFFLOADS);
1991	if (!ns) {
1992	esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
1993	return -EOPNOTSUPP;
1994	}
1995
1996	ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) +
1997	MLX5_ESW_FT_OFFLOADS_DROP_RULE;
1998	ft_attr.prio = 1;
1999
2000	ft_offloads = mlx5_create_flow_table(ns, ft_attr: &ft_attr);
2001	if (IS_ERR(ptr: ft_offloads)) {
2002	err = PTR_ERR(ptr: ft_offloads);
2003	esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err);
2004	return err;
2005	}
2006
2007	esw->offloads.ft_offloads = ft_offloads;
2008	return 0;
2009	}
2010
2011	static void esw_destroy_offloads_table(struct mlx5_eswitch *esw)
2012	{
2013	struct mlx5_esw_offload *offloads = &esw->offloads;
2014
2015	mlx5_destroy_flow_table(ft: offloads->ft_offloads);
2016	}
2017
2018	static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
2019	{
2020	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2021	struct mlx5_flow_group *g;
2022	u32 *flow_group_in;
2023	int nvports;
2024	int err = 0;
2025
2026	nvports = esw_get_nr_ft_offloads_steering_src_ports(esw);
2027	flow_group_in = kvzalloc(size: inlen, GFP_KERNEL);
2028	if (!flow_group_in)
2029	return -ENOMEM;
2030
2031	mlx5_esw_set_flow_group_source_port(esw, flow_group_in, match_params: 0);
2032
2033	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2034	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1);
2035
2036	g = mlx5_create_flow_group(ft: esw->offloads.ft_offloads, in: flow_group_in);
2037
2038	if (IS_ERR(ptr: g)) {
2039	err = PTR_ERR(ptr: g);
2040	mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err);
2041	goto out;
2042	}
2043
2044	esw->offloads.vport_rx_group = g;
2045	out:
2046	kvfree(addr: flow_group_in);
2047	return err;
2048	}
2049
2050	static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
2051	{
2052	mlx5_destroy_flow_group(fg: esw->offloads.vport_rx_group);
2053	}
2054
2055	static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw)
2056	{
2057	/* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
2058	* for the drop rule, which is placed at the end of the table.
2059	* So return the total of vport and int_port as rule index.
2060	*/
2061	return esw_get_nr_ft_offloads_steering_src_ports(esw);
2062	}
2063
2064	static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw)
2065	{
2066	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2067	struct mlx5_flow_group *g;
2068	u32 *flow_group_in;
2069	int flow_index;
2070	int err = 0;
2071
2072	flow_index = esw_create_vport_rx_drop_rule_index(esw);
2073
2074	flow_group_in = kvzalloc(size: inlen, GFP_KERNEL);
2075	if (!flow_group_in)
2076	return -ENOMEM;
2077
2078	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index);
2079	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index);
2080
2081	g = mlx5_create_flow_group(ft: esw->offloads.ft_offloads, in: flow_group_in);
2082
2083	if (IS_ERR(ptr: g)) {
2084	err = PTR_ERR(ptr: g);
2085	mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err);
2086	goto out;
2087	}
2088
2089	esw->offloads.vport_rx_drop_group = g;
2090	out:
2091	kvfree(addr: flow_group_in);
2092	return err;
2093	}
2094
2095	static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw)
2096	{
2097	if (esw->offloads.vport_rx_drop_group)
2098	mlx5_destroy_flow_group(fg: esw->offloads.vport_rx_drop_group);
2099	}
2100
2101	void
2102	mlx5_esw_set_spec_source_port(struct mlx5_eswitch *esw,
2103	u16 vport,
2104	struct mlx5_flow_spec *spec)
2105	{
2106	void *misc;
2107
2108	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
2109	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
2110	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2111	mlx5_eswitch_get_vport_metadata_for_match(esw, vport));
2112
2113	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
2114	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2115	mlx5_eswitch_get_vport_metadata_mask());
2116
2117	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
2118	} else {
2119	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
2120	MLX5_SET(fte_match_set_misc, misc, source_port, vport);
2121
2122	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2123	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2124
2125	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2126	}
2127	}
2128
2129	struct mlx5_flow_handle *
2130	mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
2131	struct mlx5_flow_destination *dest)
2132	{
2133	struct mlx5_flow_act flow_act = {0};
2134	struct mlx5_flow_handle *flow_rule;
2135	struct mlx5_flow_spec *spec;
2136
2137	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
2138	if (!spec) {
2139	flow_rule = ERR_PTR(error: -ENOMEM);
2140	goto out;
2141	}
2142
2143	mlx5_esw_set_spec_source_port(esw, vport, spec);
2144
2145	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2146	flow_rule = mlx5_add_flow_rules(ft: esw->offloads.ft_offloads, spec,
2147	flow_act: &flow_act, dest, num_dest: 1);
2148	if (IS_ERR(ptr: flow_rule)) {
2149	esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule));
2150	goto out;
2151	}
2152
2153	out:
2154	kvfree(addr: spec);
2155	return flow_rule;
2156	}
2157
2158	static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2159	{
2160	struct mlx5_flow_act flow_act = {};
2161	struct mlx5_flow_handle *flow_rule;
2162
2163	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2164	flow_rule = mlx5_add_flow_rules(ft: esw->offloads.ft_offloads, NULL,
2165	flow_act: &flow_act, NULL, num_dest: 0);
2166	if (IS_ERR(ptr: flow_rule)) {
2167	esw_warn(esw->dev,
2168	"fs offloads: Failed to add vport rx drop rule err %ld\n",
2169	PTR_ERR(flow_rule));
2170	return PTR_ERR(ptr: flow_rule);
2171	}
2172
2173	esw->offloads.vport_rx_drop_rule = flow_rule;
2174
2175	return 0;
2176	}
2177
2178	static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2179	{
2180	if (esw->offloads.vport_rx_drop_rule)
2181	mlx5_del_flow_rules(fr: esw->offloads.vport_rx_drop_rule);
2182	}
2183
2184	static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode)
2185	{
2186	u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2;
2187	struct mlx5_core_dev *dev = esw->dev;
2188	struct mlx5_vport *vport;
2189	unsigned long i;
2190
2191	if (!MLX5_CAP_GEN(dev, vport_group_manager))
2192	return -EOPNOTSUPP;
2193
2194	if (!mlx5_esw_is_fdb_created(esw))
2195	return -EOPNOTSUPP;
2196
2197	switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
2198	case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
2199	mlx5_mode = MLX5_INLINE_MODE_NONE;
2200	goto out;
2201	case MLX5_CAP_INLINE_MODE_L2:
2202	mlx5_mode = MLX5_INLINE_MODE_L2;
2203	goto out;
2204	case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
2205	goto query_vports;
2206	}
2207
2208	query_vports:
2209	mlx5_query_nic_vport_min_inline(mdev: dev, vport: esw->first_host_vport, min_inline: &prev_mlx5_mode);
2210	mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
2211	mlx5_query_nic_vport_min_inline(mdev: dev, vport: vport->vport, min_inline: &mlx5_mode);
2212	if (prev_mlx5_mode != mlx5_mode)
2213	return -EINVAL;
2214	prev_mlx5_mode = mlx5_mode;
2215	}
2216
2217	out:
2218	*mode = mlx5_mode;
2219	return 0;
2220	}
2221
2222	static void esw_destroy_restore_table(struct mlx5_eswitch *esw)
2223	{
2224	struct mlx5_esw_offload *offloads = &esw->offloads;
2225
2226	if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2227	return;
2228
2229	mlx5_modify_header_dealloc(dev: esw->dev, modify_hdr: offloads->restore_copy_hdr_id);
2230	mlx5_destroy_flow_group(fg: offloads->restore_group);
2231	mlx5_destroy_flow_table(ft: offloads->ft_offloads_restore);
2232	}
2233
2234	static int esw_create_restore_table(struct mlx5_eswitch *esw)
2235	{
2236	u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
2237	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2238	struct mlx5_flow_table_attr ft_attr = {};
2239	struct mlx5_core_dev *dev = esw->dev;
2240	struct mlx5_flow_namespace *ns;
2241	struct mlx5_modify_hdr *mod_hdr;
2242	void *match_criteria, *misc;
2243	struct mlx5_flow_table *ft;
2244	struct mlx5_flow_group *g;
2245	u32 *flow_group_in;
2246	int err = 0;
2247
2248	if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2249	return 0;
2250
2251	ns = mlx5_get_flow_namespace(dev, type: MLX5_FLOW_NAMESPACE_OFFLOADS);
2252	if (!ns) {
2253	esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
2254	return -EOPNOTSUPP;
2255	}
2256
2257	flow_group_in = kvzalloc(size: inlen, GFP_KERNEL);
2258	if (!flow_group_in) {
2259	err = -ENOMEM;
2260	goto out_free;
2261	}
2262
2263	ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS;
2264	ft = mlx5_create_flow_table(ns, ft_attr: &ft_attr);
2265	if (IS_ERR(ptr: ft)) {
2266	err = PTR_ERR(ptr: ft);
2267	esw_warn(esw->dev, "Failed to create restore table, err %d\n",
2268	err);
2269	goto out_free;
2270	}
2271
2272	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2273	match_criteria);
2274	misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
2275	misc_parameters_2);
2276
2277	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2278	ESW_REG_C0_USER_DATA_METADATA_MASK);
2279	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2280	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
2281	ft_attr.max_fte - 1);
2282	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2283	MLX5_MATCH_MISC_PARAMETERS_2);
2284	g = mlx5_create_flow_group(ft, in: flow_group_in);
2285	if (IS_ERR(ptr: g)) {
2286	err = PTR_ERR(ptr: g);
2287	esw_warn(dev, "Failed to create restore flow group, err: %d\n",
2288	err);
2289	goto err_group;
2290	}
2291
2292	MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY);
2293	MLX5_SET(copy_action_in, modact, src_field,
2294	MLX5_ACTION_IN_FIELD_METADATA_REG_C_1);
2295	MLX5_SET(copy_action_in, modact, dst_field,
2296	MLX5_ACTION_IN_FIELD_METADATA_REG_B);
2297	mod_hdr = mlx5_modify_header_alloc(dev: esw->dev,
2298	ns_type: MLX5_FLOW_NAMESPACE_KERNEL, num_actions: 1,
2299	modify_actions: modact);
2300	if (IS_ERR(ptr: mod_hdr)) {
2301	err = PTR_ERR(ptr: mod_hdr);
2302	esw_warn(dev, "Failed to create restore mod header, err: %d\n",
2303	err);
2304	goto err_mod_hdr;
2305	}
2306
2307	esw->offloads.ft_offloads_restore = ft;
2308	esw->offloads.restore_group = g;
2309	esw->offloads.restore_copy_hdr_id = mod_hdr;
2310
2311	kvfree(addr: flow_group_in);
2312
2313	return 0;
2314
2315	err_mod_hdr:
2316	mlx5_destroy_flow_group(fg: g);
2317	err_group:
2318	mlx5_destroy_flow_table(ft);
2319	out_free:
2320	kvfree(addr: flow_group_in);
2321
2322	return err;
2323	}
2324
2325	static int esw_offloads_start(struct mlx5_eswitch *esw,
2326	struct netlink_ext_ack *extack)
2327	{
2328	int err;
2329
2330	esw->mode = MLX5_ESWITCH_OFFLOADS;
2331	err = mlx5_eswitch_enable_locked(esw, num_vfs: esw->dev->priv.sriov.num_vfs);
2332	if (err) {
2333	NL_SET_ERR_MSG_MOD(extack,
2334	"Failed setting eswitch to offloads");
2335	esw->mode = MLX5_ESWITCH_LEGACY;
2336	mlx5_rescan_drivers(dev: esw->dev);
2337	return err;
2338	}
2339	if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) {
2340	if (mlx5_eswitch_inline_mode_get(esw,
2341	mode: &esw->offloads.inline_mode)) {
2342	esw->offloads.inline_mode = MLX5_INLINE_MODE_L2;
2343	NL_SET_ERR_MSG_MOD(extack,
2344	"Inline mode is different between vports");
2345	}
2346	}
2347	return 0;
2348	}
2349
2350	static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport)
2351	{
2352	struct mlx5_eswitch_rep *rep;
2353	int rep_type;
2354	int err;
2355
2356	rep = kzalloc(size: sizeof(*rep), GFP_KERNEL);
2357	if (!rep)
2358	return -ENOMEM;
2359
2360	rep->vport = vport->vport;
2361	rep->vport_index = vport->index;
2362	for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2363	atomic_set(v: &rep->rep_data[rep_type].state, i: REP_UNREGISTERED);
2364
2365	err = xa_insert(xa: &esw->offloads.vport_reps, index: rep->vport, entry: rep, GFP_KERNEL);
2366	if (err)
2367	goto insert_err;
2368
2369	return 0;
2370
2371	insert_err:
2372	kfree(objp: rep);
2373	return err;
2374	}
2375
2376	static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw,
2377	struct mlx5_eswitch_rep *rep)
2378	{
2379	xa_erase(&esw->offloads.vport_reps, index: rep->vport);
2380	kfree(objp: rep);
2381	}
2382
2383	static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
2384	{
2385	struct mlx5_eswitch_rep *rep;
2386	unsigned long i;
2387
2388	mlx5_esw_for_each_rep(esw, i, rep)
2389	mlx5_esw_offloads_rep_cleanup(esw, rep);
2390	xa_destroy(&esw->offloads.vport_reps);
2391	}
2392
2393	static int esw_offloads_init_reps(struct mlx5_eswitch *esw)
2394	{
2395	struct mlx5_vport *vport;
2396	unsigned long i;
2397	int err;
2398
2399	xa_init(xa: &esw->offloads.vport_reps);
2400
2401	mlx5_esw_for_each_vport(esw, i, vport) {
2402	err = mlx5_esw_offloads_rep_init(esw, vport);
2403	if (err)
2404	goto err;
2405	}
2406	return 0;
2407
2408	err:
2409	esw_offloads_cleanup_reps(esw);
2410	return err;
2411	}
2412
2413	static int esw_port_metadata_set(struct devlink *devlink, u32 id,
2414	struct devlink_param_gset_ctx *ctx)
2415	{
2416	struct mlx5_core_dev *dev = devlink_priv(devlink);
2417	struct mlx5_eswitch *esw = dev->priv.eswitch;
2418	int err = 0;
2419
2420	down_write(sem: &esw->mode_lock);
2421	if (mlx5_esw_is_fdb_created(esw)) {
2422	err = -EBUSY;
2423	goto done;
2424	}
2425	if (!mlx5_esw_vport_match_metadata_supported(esw)) {
2426	err = -EOPNOTSUPP;
2427	goto done;
2428	}
2429	if (ctx->val.vbool)
2430	esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2431	else
2432	esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
2433	done:
2434	up_write(sem: &esw->mode_lock);
2435	return err;
2436	}
2437
2438	static int esw_port_metadata_get(struct devlink *devlink, u32 id,
2439	struct devlink_param_gset_ctx *ctx)
2440	{
2441	struct mlx5_core_dev *dev = devlink_priv(devlink);
2442
2443	ctx->val.vbool = mlx5_eswitch_vport_match_metadata_enabled(esw: dev->priv.eswitch);
2444	return 0;
2445	}
2446
2447	static int esw_port_metadata_validate(struct devlink *devlink, u32 id,
2448	union devlink_param_value val,
2449	struct netlink_ext_ack *extack)
2450	{
2451	struct mlx5_core_dev *dev = devlink_priv(devlink);
2452	u8 esw_mode;
2453
2454	esw_mode = mlx5_eswitch_mode(dev);
2455	if (esw_mode == MLX5_ESWITCH_OFFLOADS) {
2456	NL_SET_ERR_MSG_MOD(extack,
2457	"E-Switch must either disabled or non switchdev mode");
2458	return -EBUSY;
2459	}
2460	return 0;
2461	}
2462
2463	static const struct devlink_param esw_devlink_params[] = {
2464	DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA,
2465	"esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL,
2466	BIT(DEVLINK_PARAM_CMODE_RUNTIME),
2467	esw_port_metadata_get,
2468	esw_port_metadata_set,
2469	esw_port_metadata_validate),
2470	};
2471
2472	int esw_offloads_init(struct mlx5_eswitch *esw)
2473	{
2474	int err;
2475
2476	err = esw_offloads_init_reps(esw);
2477	if (err)
2478	return err;
2479
2480	if (MLX5_ESWITCH_MANAGER(esw->dev) &&
2481	mlx5_esw_vport_match_metadata_supported(esw))
2482	esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2483
2484	err = devl_params_register(devlink: priv_to_devlink(priv: esw->dev),
2485	params: esw_devlink_params,
2486	ARRAY_SIZE(esw_devlink_params));
2487	if (err)
2488	goto err_params;
2489
2490	return 0;
2491
2492	err_params:
2493	esw_offloads_cleanup_reps(esw);
2494	return err;
2495	}
2496
2497	void esw_offloads_cleanup(struct mlx5_eswitch *esw)
2498	{
2499	devl_params_unregister(devlink: priv_to_devlink(priv: esw->dev),
2500	params: esw_devlink_params,
2501	ARRAY_SIZE(esw_devlink_params));
2502	esw_offloads_cleanup_reps(esw);
2503	}
2504
2505	static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
2506	struct mlx5_eswitch_rep *rep, u8 rep_type)
2507	{
2508	if (atomic_cmpxchg(v: &rep->rep_data[rep_type].state,
2509	old: REP_LOADED, new: REP_REGISTERED) == REP_LOADED)
2510	esw->offloads.rep_ops[rep_type]->unload(rep);
2511	}
2512
2513	static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type)
2514	{
2515	struct mlx5_eswitch_rep *rep;
2516	unsigned long i;
2517
2518	mlx5_esw_for_each_rep(esw, i, rep)
2519	__esw_offloads_unload_rep(esw, rep, rep_type);
2520	}
2521
2522	static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num)
2523	{
2524	struct mlx5_eswitch_rep *rep;
2525	int rep_type;
2526	int err;
2527
2528	rep = mlx5_eswitch_get_rep(esw, vport_num);
2529	for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2530	if (atomic_cmpxchg(v: &rep->rep_data[rep_type].state,
2531	old: REP_REGISTERED, new: REP_LOADED) == REP_REGISTERED) {
2532	err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
2533	if (err)
2534	goto err_reps;
2535	}
2536
2537	return 0;
2538
2539	err_reps:
2540	atomic_set(v: &rep->rep_data[rep_type].state, i: REP_REGISTERED);
2541	for (--rep_type; rep_type >= 0; rep_type--)
2542	__esw_offloads_unload_rep(esw, rep, rep_type);
2543	return err;
2544	}
2545
2546	static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num)
2547	{
2548	struct mlx5_eswitch_rep *rep;
2549	int rep_type;
2550
2551	rep = mlx5_eswitch_get_rep(esw, vport_num);
2552	for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--)
2553	__esw_offloads_unload_rep(esw, rep, rep_type);
2554	}
2555
2556	int mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2557	{
2558	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2559	return 0;
2560
2561	return mlx5_esw_offloads_pf_vf_devlink_port_init(esw, vport);
2562	}
2563
2564	void mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2565	{
2566	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2567	return;
2568
2569	mlx5_esw_offloads_pf_vf_devlink_port_cleanup(esw, vport);
2570	}
2571
2572	int mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
2573	struct mlx5_devlink_port *dl_port,
2574	u32 controller, u32 sfnum)
2575	{
2576	return mlx5_esw_offloads_sf_devlink_port_init(esw, vport, dl_port, controller, sfnum);
2577	}
2578
2579	void mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2580	{
2581	mlx5_esw_offloads_sf_devlink_port_cleanup(esw, vport);
2582	}
2583
2584	int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2585	{
2586	int err;
2587
2588	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2589	return 0;
2590
2591	err = mlx5_esw_offloads_devlink_port_register(esw, vport);
2592	if (err)
2593	return err;
2594
2595	err = mlx5_esw_offloads_rep_load(esw, vport_num: vport->vport);
2596	if (err)
2597	goto load_err;
2598	return err;
2599
2600	load_err:
2601	mlx5_esw_offloads_devlink_port_unregister(esw, vport);
2602	return err;
2603	}
2604
2605	void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2606	{
2607	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2608	return;
2609
2610	mlx5_esw_offloads_rep_unload(esw, vport_num: vport->vport);
2611
2612	mlx5_esw_offloads_devlink_port_unregister(esw, vport);
2613	}
2614
2615	static int esw_set_slave_root_fdb(struct mlx5_core_dev *master,
2616	struct mlx5_core_dev *slave)
2617	{
2618	u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
2619	u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {};
2620	struct mlx5_flow_root_namespace *root;
2621	struct mlx5_flow_namespace *ns;
2622	int err;
2623
2624	MLX5_SET(set_flow_table_root_in, in, opcode,
2625	MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
2626	MLX5_SET(set_flow_table_root_in, in, table_type,
2627	FS_FT_FDB);
2628
2629	if (master) {
2630	ns = mlx5_get_flow_namespace(dev: master,
2631	type: MLX5_FLOW_NAMESPACE_FDB);
2632	root = find_root(node: &ns->node);
2633	mutex_lock(&root->chain_lock);
2634	MLX5_SET(set_flow_table_root_in, in,
2635	table_eswitch_owner_vhca_id_valid, 1);
2636	MLX5_SET(set_flow_table_root_in, in,
2637	table_eswitch_owner_vhca_id,
2638	MLX5_CAP_GEN(master, vhca_id));
2639	MLX5_SET(set_flow_table_root_in, in, table_id,
2640	root->root_ft->id);
2641	} else {
2642	ns = mlx5_get_flow_namespace(dev: slave,
2643	type: MLX5_FLOW_NAMESPACE_FDB);
2644	root = find_root(node: &ns->node);
2645	mutex_lock(&root->chain_lock);
2646	MLX5_SET(set_flow_table_root_in, in, table_id,
2647	root->root_ft->id);
2648	}
2649
2650	err = mlx5_cmd_exec(dev: slave, in, in_size: sizeof(in), out, out_size: sizeof(out));
2651	mutex_unlock(lock: &root->chain_lock);
2652
2653	return err;
2654	}
2655
2656	static int __esw_set_master_egress_rule(struct mlx5_core_dev *master,
2657	struct mlx5_core_dev *slave,
2658	struct mlx5_vport *vport,
2659	struct mlx5_flow_table *acl)
2660	{
2661	u16 slave_index = MLX5_CAP_GEN(slave, vhca_id);
2662	struct mlx5_flow_handle *flow_rule = NULL;
2663	struct mlx5_flow_destination dest = {};
2664	struct mlx5_flow_act flow_act = {};
2665	struct mlx5_flow_spec *spec;
2666	int err = 0;
2667	void *misc;
2668
2669	spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
2670	if (!spec)
2671	return -ENOMEM;
2672
2673	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2674	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2675	misc_parameters);
2676	MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK);
2677	MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, slave_index);
2678
2679	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2680	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2681	MLX5_SET_TO_ONES(fte_match_set_misc, misc,
2682	source_eswitch_owner_vhca_id);
2683
2684	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2685	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
2686	dest.vport.num = slave->priv.eswitch->manager_vport;
2687	dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id);
2688	dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
2689
2690	flow_rule = mlx5_add_flow_rules(ft: acl, spec, flow_act: &flow_act,
2691	dest: &dest, num_dest: 1);
2692	if (IS_ERR(ptr: flow_rule)) {
2693	err = PTR_ERR(ptr: flow_rule);
2694	} else {
2695	err = xa_insert(xa: &vport->egress.offloads.bounce_rules,
2696	index: slave_index, entry: flow_rule, GFP_KERNEL);
2697	if (err)
2698	mlx5_del_flow_rules(fr: flow_rule);
2699	}
2700
2701	kvfree(addr: spec);
2702	return err;
2703	}
2704
2705	static int esw_master_egress_create_resources(struct mlx5_eswitch *esw,
2706	struct mlx5_flow_namespace *egress_ns,
2707	struct mlx5_vport *vport, size_t count)
2708	{
2709	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2710	struct mlx5_flow_table_attr ft_attr = {
2711	.max_fte = count, .prio = 0, .level = 0,
2712	};
2713	struct mlx5_flow_table *acl;
2714	struct mlx5_flow_group *g;
2715	void *match_criteria;
2716	u32 *flow_group_in;
2717	int err;
2718
2719	if (vport->egress.acl)
2720	return 0;
2721
2722	flow_group_in = kvzalloc(size: inlen, GFP_KERNEL);
2723	if (!flow_group_in)
2724	return -ENOMEM;
2725
2726	if (vport->vport || mlx5_core_is_ecpf(dev: esw->dev))
2727	ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT;
2728
2729	acl = mlx5_create_vport_flow_table(ns: egress_ns, ft_attr: &ft_attr, vport: vport->vport);
2730	if (IS_ERR(ptr: acl)) {
2731	err = PTR_ERR(ptr: acl);
2732	goto out;
2733	}
2734
2735	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2736	match_criteria);
2737	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2738	misc_parameters.source_port);
2739	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2740	misc_parameters.source_eswitch_owner_vhca_id);
2741	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2742	MLX5_MATCH_MISC_PARAMETERS);
2743
2744	MLX5_SET(create_flow_group_in, flow_group_in,
2745	source_eswitch_owner_vhca_id_valid, 1);
2746	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2747	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, count);
2748
2749	g = mlx5_create_flow_group(ft: acl, in: flow_group_in);
2750	if (IS_ERR(ptr: g)) {
2751	err = PTR_ERR(ptr: g);
2752	goto err_group;
2753	}
2754
2755	vport->egress.acl = acl;
2756	vport->egress.offloads.bounce_grp = g;
2757	vport->egress.type = VPORT_EGRESS_ACL_TYPE_SHARED_FDB;
2758	xa_init_flags(xa: &vport->egress.offloads.bounce_rules, XA_FLAGS_ALLOC);
2759
2760	kvfree(addr: flow_group_in);
2761
2762	return 0;
2763
2764	err_group:
2765	mlx5_destroy_flow_table(ft: acl);
2766	out:
2767	kvfree(addr: flow_group_in);
2768	return err;
2769	}
2770
2771	static void esw_master_egress_destroy_resources(struct mlx5_vport *vport)
2772	{
2773	if (!xa_empty(xa: &vport->egress.offloads.bounce_rules))
2774	return;
2775	mlx5_destroy_flow_group(fg: vport->egress.offloads.bounce_grp);
2776	vport->egress.offloads.bounce_grp = NULL;
2777	mlx5_destroy_flow_table(ft: vport->egress.acl);
2778	vport->egress.acl = NULL;
2779	}
2780
2781	static int esw_set_master_egress_rule(struct mlx5_core_dev *master,
2782	struct mlx5_core_dev *slave, size_t count)
2783	{
2784	struct mlx5_eswitch *esw = master->priv.eswitch;
2785	u16 slave_index = MLX5_CAP_GEN(slave, vhca_id);
2786	struct mlx5_flow_namespace *egress_ns;
2787	struct mlx5_vport *vport;
2788	int err;
2789
2790	vport = mlx5_eswitch_get_vport(esw, vport_num: esw->manager_vport);
2791	if (IS_ERR(ptr: vport))
2792	return PTR_ERR(ptr: vport);
2793
2794	egress_ns = mlx5_get_flow_vport_acl_namespace(dev: master,
2795	type: MLX5_FLOW_NAMESPACE_ESW_EGRESS,
2796	vport: vport->index);
2797	if (!egress_ns)
2798	return -EINVAL;
2799
2800	if (vport->egress.acl && vport->egress.type != VPORT_EGRESS_ACL_TYPE_SHARED_FDB)
2801	return 0;
2802
2803	err = esw_master_egress_create_resources(esw, egress_ns, vport, count);
2804	if (err)
2805	return err;
2806
2807	if (xa_load(&vport->egress.offloads.bounce_rules, index: slave_index))
2808	return -EINVAL;
2809
2810	err = __esw_set_master_egress_rule(master, slave, vport, acl: vport->egress.acl);
2811	if (err)
2812	goto err_rule;
2813
2814	return 0;
2815
2816	err_rule:
2817	esw_master_egress_destroy_resources(vport);
2818	return err;
2819	}
2820
2821	static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev,
2822	struct mlx5_core_dev *slave_dev)
2823	{
2824	struct mlx5_vport *vport;
2825
2826	vport = mlx5_eswitch_get_vport(esw: dev->priv.eswitch,
2827	vport_num: dev->priv.eswitch->manager_vport);
2828
2829	esw_acl_egress_ofld_bounce_rule_destroy(vport, MLX5_CAP_GEN(slave_dev, vhca_id));
2830
2831	if (xa_empty(xa: &vport->egress.offloads.bounce_rules)) {
2832	esw_acl_egress_ofld_cleanup(vport);
2833	xa_destroy(&vport->egress.offloads.bounce_rules);
2834	}
2835	}
2836
2837	int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch *master_esw,
2838	struct mlx5_eswitch *slave_esw, int max_slaves)
2839	{
2840	int err;
2841
2842	err = esw_set_slave_root_fdb(master: master_esw->dev,
2843	slave: slave_esw->dev);
2844	if (err)
2845	return err;
2846
2847	err = esw_set_master_egress_rule(master: master_esw->dev,
2848	slave: slave_esw->dev, count: max_slaves);
2849	if (err)
2850	goto err_acl;
2851
2852	return err;
2853
2854	err_acl:
2855	esw_set_slave_root_fdb(NULL, slave: slave_esw->dev);
2856	return err;
2857	}
2858
2859	void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw,
2860	struct mlx5_eswitch *slave_esw)
2861	{
2862	esw_set_slave_root_fdb(NULL, slave: slave_esw->dev);
2863	esw_unset_master_egress_rule(dev: master_esw->dev, slave_dev: slave_esw->dev);
2864	}
2865
2866	#define ESW_OFFLOADS_DEVCOM_PAIR (0)
2867	#define ESW_OFFLOADS_DEVCOM_UNPAIR (1)
2868
2869	static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw,
2870	struct mlx5_eswitch *peer_esw)
2871	{
2872	const struct mlx5_eswitch_rep_ops *ops;
2873	struct mlx5_eswitch_rep *rep;
2874	unsigned long i;
2875	u8 rep_type;
2876
2877	mlx5_esw_for_each_rep(esw, i, rep) {
2878	rep_type = NUM_REP_TYPES;
2879	while (rep_type--) {
2880	ops = esw->offloads.rep_ops[rep_type];
2881	if (atomic_read(v: &rep->rep_data[rep_type].state) == REP_LOADED &&
2882	ops->event)
2883	ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, peer_esw);
2884	}
2885	}
2886	}
2887
2888	static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw,
2889	struct mlx5_eswitch *peer_esw)
2890	{
2891	#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
2892	mlx5e_tc_clean_fdb_peer_flows(esw);
2893	#endif
2894	mlx5_esw_offloads_rep_event_unpair(esw, peer_esw);
2895	esw_del_fdb_peer_miss_rules(esw, peer_dev: peer_esw->dev);
2896	}
2897
2898	static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
2899	struct mlx5_eswitch *peer_esw)
2900	{
2901	const struct mlx5_eswitch_rep_ops *ops;
2902	struct mlx5_eswitch_rep *rep;
2903	unsigned long i;
2904	u8 rep_type;
2905	int err;
2906
2907	err = esw_add_fdb_peer_miss_rules(esw, peer_dev: peer_esw->dev);
2908	if (err)
2909	return err;
2910
2911	mlx5_esw_for_each_rep(esw, i, rep) {
2912	for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2913	ops = esw->offloads.rep_ops[rep_type];
2914	if (atomic_read(v: &rep->rep_data[rep_type].state) == REP_LOADED &&
2915	ops->event) {
2916	err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw);
2917	if (err)
2918	goto err_out;
2919	}
2920	}
2921	}
2922
2923	return 0;
2924
2925	err_out:
2926	mlx5_esw_offloads_unpair(esw, peer_esw);
2927	return err;
2928	}
2929
2930	static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw,
2931	struct mlx5_eswitch *peer_esw,
2932	bool pair)
2933	{
2934	u16 peer_vhca_id = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
2935	u16 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id);
2936	struct mlx5_flow_root_namespace *peer_ns;
2937	struct mlx5_flow_root_namespace *ns;
2938	int err;
2939
2940	peer_ns = peer_esw->dev->priv.steering->fdb_root_ns;
2941	ns = esw->dev->priv.steering->fdb_root_ns;
2942
2943	if (pair) {
2944	err = mlx5_flow_namespace_set_peer(ns, peer_ns, peer_vhca_id);
2945	if (err)
2946	return err;
2947
2948	err = mlx5_flow_namespace_set_peer(ns: peer_ns, peer_ns: ns, peer_vhca_id: vhca_id);
2949	if (err) {
2950	mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id);
2951	return err;
2952	}
2953	} else {
2954	mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id);
2955	mlx5_flow_namespace_set_peer(ns: peer_ns, NULL, peer_vhca_id: vhca_id);
2956	}
2957
2958	return 0;
2959	}
2960
2961	static int mlx5_esw_offloads_devcom_event(int event,
2962	void *my_data,
2963	void *event_data)
2964	{
2965	struct mlx5_eswitch *esw = my_data;
2966	struct mlx5_eswitch *peer_esw = event_data;
2967	u16 esw_i, peer_esw_i;
2968	bool esw_paired;
2969	int err;
2970
2971	peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
2972	esw_i = MLX5_CAP_GEN(esw->dev, vhca_id);
2973	esw_paired = !!xa_load(&esw->paired, index: peer_esw_i);
2974
2975	switch (event) {
2976	case ESW_OFFLOADS_DEVCOM_PAIR:
2977	if (mlx5_eswitch_vport_match_metadata_enabled(esw) !=
2978	mlx5_eswitch_vport_match_metadata_enabled(esw: peer_esw))
2979	break;
2980
2981	if (esw_paired)
2982	break;
2983
2984	err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, pair: true);
2985	if (err)
2986	goto err_out;
2987
2988	err = mlx5_esw_offloads_pair(esw, peer_esw);
2989	if (err)
2990	goto err_peer;
2991
2992	err = mlx5_esw_offloads_pair(esw: peer_esw, peer_esw: esw);
2993	if (err)
2994	goto err_pair;
2995
2996	err = xa_insert(xa: &esw->paired, index: peer_esw_i, entry: peer_esw, GFP_KERNEL);
2997	if (err)
2998	goto err_xa;
2999
3000	err = xa_insert(xa: &peer_esw->paired, index: esw_i, entry: esw, GFP_KERNEL);
3001	if (err)
3002	goto err_peer_xa;
3003
3004	esw->num_peers++;
3005	peer_esw->num_peers++;
3006	mlx5_devcom_comp_set_ready(devcom: esw->devcom, ready: true);
3007	break;
3008
3009	case ESW_OFFLOADS_DEVCOM_UNPAIR:
3010	if (!esw_paired)
3011	break;
3012
3013	peer_esw->num_peers--;
3014	esw->num_peers--;
3015	if (!esw->num_peers && !peer_esw->num_peers)
3016	mlx5_devcom_comp_set_ready(devcom: esw->devcom, ready: false);
3017	xa_erase(&peer_esw->paired, index: esw_i);
3018	xa_erase(&esw->paired, index: peer_esw_i);
3019	mlx5_esw_offloads_unpair(esw: peer_esw, peer_esw: esw);
3020	mlx5_esw_offloads_unpair(esw, peer_esw);
3021	mlx5_esw_offloads_set_ns_peer(esw, peer_esw, pair: false);
3022	break;
3023	}
3024
3025	return 0;
3026
3027	err_peer_xa:
3028	xa_erase(&esw->paired, index: peer_esw_i);
3029	err_xa:
3030	mlx5_esw_offloads_unpair(esw: peer_esw, peer_esw: esw);
3031	err_pair:
3032	mlx5_esw_offloads_unpair(esw, peer_esw);
3033	err_peer:
3034	mlx5_esw_offloads_set_ns_peer(esw, peer_esw, pair: false);
3035	err_out:
3036	mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d",
3037	event, err);
3038	return err;
3039	}
3040
3041	void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key)
3042	{
3043	int i;
3044
3045	for (i = 0; i < MLX5_MAX_PORTS; i++)
3046	INIT_LIST_HEAD(list: &esw->offloads.peer_flows[i]);
3047	mutex_init(&esw->offloads.peer_mutex);
3048
3049	if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
3050	return;
3051
3052	if ((MLX5_VPORT_MANAGER(esw->dev) || mlx5_core_is_ecpf_esw_manager(dev: esw->dev)) &&
3053	!mlx5_lag_is_supported(dev: esw->dev))
3054	return;
3055
3056	xa_init(xa: &esw->paired);
3057	esw->num_peers = 0;
3058	esw->devcom = mlx5_devcom_register_component(devc: esw->dev->priv.devc,
3059	id: MLX5_DEVCOM_ESW_OFFLOADS,
3060	key,
3061	handler: mlx5_esw_offloads_devcom_event,
3062	data: esw);
3063	if (IS_ERR(ptr: esw->devcom))
3064	return;
3065
3066	mlx5_devcom_send_event(devcom: esw->devcom,
3067	ESW_OFFLOADS_DEVCOM_PAIR,
3068	ESW_OFFLOADS_DEVCOM_UNPAIR,
3069	event_data: esw);
3070	}
3071
3072	void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
3073	{
3074	if (IS_ERR_OR_NULL(ptr: esw->devcom))
3075	return;
3076
3077	mlx5_devcom_send_event(devcom: esw->devcom,
3078	ESW_OFFLOADS_DEVCOM_UNPAIR,
3079	ESW_OFFLOADS_DEVCOM_UNPAIR,
3080	event_data: esw);
3081
3082	mlx5_devcom_unregister_component(devcom: esw->devcom);
3083	xa_destroy(&esw->paired);
3084	esw->devcom = NULL;
3085	}
3086
3087	bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw)
3088	{
3089	return mlx5_devcom_comp_is_ready(devcom: esw->devcom);
3090	}
3091
3092	bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw)
3093	{
3094	if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl))
3095	return false;
3096
3097	if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
3098	MLX5_FDB_TO_VPORT_REG_C_0))
3099	return false;
3100
3101	return true;
3102	}
3103
3104	#define MLX5_ESW_METADATA_RSVD_UPLINK 1
3105
3106	/* Share the same metadata for uplink's. This is fine because:
3107	* (a) In shared FDB mode (LAG) both uplink's are treated the
3108	* same and tagged with the same metadata.
3109	* (b) In non shared FDB mode, packets from physical port0
3110	* cannot hit eswitch of PF1 and vice versa.
3111	*/
3112	static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw)
3113	{
3114	return MLX5_ESW_METADATA_RSVD_UPLINK;
3115	}
3116
3117	u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw)
3118	{
3119	u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1;
3120	/* Reserve 0xf for internal port offload */
3121	u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2;
3122	u32 pf_num;
3123	int id;
3124
3125	/* Only 4 bits of pf_num */
3126	pf_num = mlx5_get_dev_index(dev: esw->dev);
3127	if (pf_num > max_pf_num)
3128	return 0;
3129
3130	/* Metadata is 4 bits of PFNUM and 12 bits of unique id */
3131	/* Use only non-zero vport_id (2-4095) for all PF's */
3132	id = ida_alloc_range(&esw->offloads.vport_metadata_ida,
3133	MLX5_ESW_METADATA_RSVD_UPLINK + 1,
3134	max: vport_end_ida, GFP_KERNEL);
3135	if (id < 0)
3136	return 0;
3137	id = (pf_num << ESW_VPORT_BITS) | id;
3138	return id;
3139	}
3140
3141	void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata)
3142	{
3143	u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1;
3144
3145	/* Metadata contains only 12 bits of actual ida id */
3146	ida_free(&esw->offloads.vport_metadata_ida, id: metadata & vport_bit_mask);
3147	}
3148
3149	static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw,
3150	struct mlx5_vport *vport)
3151	{
3152	if (vport->vport == MLX5_VPORT_UPLINK)
3153	vport->default_metadata = mlx5_esw_match_metadata_reserved(esw);
3154	else
3155	vport->default_metadata = mlx5_esw_match_metadata_alloc(esw);
3156
3157	vport->metadata = vport->default_metadata;
3158	return vport->metadata ? 0 : -ENOSPC;
3159	}
3160
3161	static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw,
3162	struct mlx5_vport *vport)
3163	{
3164	if (!vport->default_metadata)
3165	return;
3166
3167	if (vport->vport == MLX5_VPORT_UPLINK)
3168	return;
3169
3170	WARN_ON(vport->metadata != vport->default_metadata);
3171	mlx5_esw_match_metadata_free(esw, metadata: vport->default_metadata);
3172	}
3173
3174	static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw)
3175	{
3176	struct mlx5_vport *vport;
3177	unsigned long i;
3178
3179	if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3180	return;
3181
3182	mlx5_esw_for_each_vport(esw, i, vport)
3183	esw_offloads_vport_metadata_cleanup(esw, vport);
3184	}
3185
3186	static int esw_offloads_metadata_init(struct mlx5_eswitch *esw)
3187	{
3188	struct mlx5_vport *vport;
3189	unsigned long i;
3190	int err;
3191
3192	if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3193	return 0;
3194
3195	mlx5_esw_for_each_vport(esw, i, vport) {
3196	err = esw_offloads_vport_metadata_setup(esw, vport);
3197	if (err)
3198	goto metadata_err;
3199	}
3200
3201	return 0;
3202
3203	metadata_err:
3204	esw_offloads_metadata_uninit(esw);
3205	return err;
3206	}
3207
3208	int
3209	esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw,
3210	struct mlx5_vport *vport)
3211	{
3212	int err;
3213
3214	err = esw_acl_ingress_ofld_setup(esw, vport);
3215	if (err)
3216	return err;
3217
3218	err = esw_acl_egress_ofld_setup(esw, vport);
3219	if (err)
3220	goto egress_err;
3221
3222	return 0;
3223
3224	egress_err:
3225	esw_acl_ingress_ofld_cleanup(esw, vport);
3226	return err;
3227	}
3228
3229	void
3230	esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw,
3231	struct mlx5_vport *vport)
3232	{
3233	esw_acl_egress_ofld_cleanup(vport);
3234	esw_acl_ingress_ofld_cleanup(esw, vport);
3235	}
3236
3237	static int esw_create_offloads_acl_tables(struct mlx5_eswitch *esw)
3238	{
3239	struct mlx5_vport *uplink, *manager;
3240	int ret;
3241
3242	uplink = mlx5_eswitch_get_vport(esw, vport_num: MLX5_VPORT_UPLINK);
3243	if (IS_ERR(ptr: uplink))
3244	return PTR_ERR(ptr: uplink);
3245
3246	ret = esw_vport_create_offloads_acl_tables(esw, vport: uplink);
3247	if (ret)
3248	return ret;
3249
3250	manager = mlx5_eswitch_get_vport(esw, vport_num: esw->manager_vport);
3251	if (IS_ERR(ptr: manager)) {
3252	ret = PTR_ERR(ptr: manager);
3253	goto err_manager;
3254	}
3255
3256	ret = esw_vport_create_offloads_acl_tables(esw, vport: manager);
3257	if (ret)
3258	goto err_manager;
3259
3260	return 0;
3261
3262	err_manager:
3263	esw_vport_destroy_offloads_acl_tables(esw, vport: uplink);
3264	return ret;
3265	}
3266
3267	static void esw_destroy_offloads_acl_tables(struct mlx5_eswitch *esw)
3268	{
3269	struct mlx5_vport *vport;
3270
3271	vport = mlx5_eswitch_get_vport(esw, vport_num: esw->manager_vport);
3272	if (!IS_ERR(ptr: vport))
3273	esw_vport_destroy_offloads_acl_tables(esw, vport);
3274
3275	vport = mlx5_eswitch_get_vport(esw, vport_num: MLX5_VPORT_UPLINK);
3276	if (!IS_ERR(ptr: vport))
3277	esw_vport_destroy_offloads_acl_tables(esw, vport);
3278	}
3279
3280	int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw)
3281	{
3282	struct mlx5_eswitch_rep *rep;
3283	unsigned long i;
3284	int ret;
3285
3286	if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS)
3287	return 0;
3288
3289	rep = mlx5_eswitch_get_rep(esw, vport_num: MLX5_VPORT_UPLINK);
3290	if (atomic_read(v: &rep->rep_data[REP_ETH].state) != REP_LOADED)
3291	return 0;
3292
3293	ret = mlx5_esw_offloads_rep_load(esw, vport_num: MLX5_VPORT_UPLINK);
3294	if (ret)
3295	return ret;
3296
3297	mlx5_esw_for_each_rep(esw, i, rep) {
3298	if (atomic_read(v: &rep->rep_data[REP_ETH].state) == REP_LOADED)
3299	mlx5_esw_offloads_rep_load(esw, vport_num: rep->vport);
3300	}
3301
3302	return 0;
3303	}
3304
3305	static int esw_offloads_steering_init(struct mlx5_eswitch *esw)
3306	{
3307	struct mlx5_esw_indir_table *indir;
3308	int err;
3309
3310	memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
3311	mutex_init(&esw->fdb_table.offloads.vports.lock);
3312	hash_init(esw->fdb_table.offloads.vports.table);
3313	atomic64_set(v: &esw->user_count, i: 0);
3314
3315	indir = mlx5_esw_indir_table_init();
3316	if (IS_ERR(ptr: indir)) {
3317	err = PTR_ERR(ptr: indir);
3318	goto create_indir_err;
3319	}
3320	esw->fdb_table.offloads.indir = indir;
3321
3322	err = esw_create_offloads_acl_tables(esw);
3323	if (err)
3324	goto create_acl_err;
3325
3326	err = esw_create_offloads_table(esw);
3327	if (err)
3328	goto create_offloads_err;
3329
3330	err = esw_create_restore_table(esw);
3331	if (err)
3332	goto create_restore_err;
3333
3334	err = esw_create_offloads_fdb_tables(esw);
3335	if (err)
3336	goto create_fdb_err;
3337
3338	err = esw_create_vport_rx_group(esw);
3339	if (err)
3340	goto create_fg_err;
3341
3342	err = esw_create_vport_rx_drop_group(esw);
3343	if (err)
3344	goto create_rx_drop_fg_err;
3345
3346	err = esw_create_vport_rx_drop_rule(esw);
3347	if (err)
3348	goto create_rx_drop_rule_err;
3349
3350	return 0;
3351
3352	create_rx_drop_rule_err:
3353	esw_destroy_vport_rx_drop_group(esw);
3354	create_rx_drop_fg_err:
3355	esw_destroy_vport_rx_group(esw);
3356	create_fg_err:
3357	esw_destroy_offloads_fdb_tables(esw);
3358	create_fdb_err:
3359	esw_destroy_restore_table(esw);
3360	create_restore_err:
3361	esw_destroy_offloads_table(esw);
3362	create_offloads_err:
3363	esw_destroy_offloads_acl_tables(esw);
3364	create_acl_err:
3365	mlx5_esw_indir_table_destroy(indir: esw->fdb_table.offloads.indir);
3366	create_indir_err:
3367	mutex_destroy(lock: &esw->fdb_table.offloads.vports.lock);
3368	return err;
3369	}
3370
3371	static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
3372	{
3373	esw_destroy_vport_rx_drop_rule(esw);
3374	esw_destroy_vport_rx_drop_group(esw);
3375	esw_destroy_vport_rx_group(esw);
3376	esw_destroy_offloads_fdb_tables(esw);
3377	esw_destroy_restore_table(esw);
3378	esw_destroy_offloads_table(esw);
3379	esw_destroy_offloads_acl_tables(esw);
3380	mlx5_esw_indir_table_destroy(indir: esw->fdb_table.offloads.indir);
3381	mutex_destroy(lock: &esw->fdb_table.offloads.vports.lock);
3382	}
3383
3384	static void
3385	esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out)
3386	{
3387	struct devlink *devlink;
3388	bool host_pf_disabled;
3389	u16 new_num_vfs;
3390
3391	new_num_vfs = MLX5_GET(query_esw_functions_out, out,
3392	host_params_context.host_num_of_vfs);
3393	host_pf_disabled = MLX5_GET(query_esw_functions_out, out,
3394	host_params_context.host_pf_disabled);
3395
3396	if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled)
3397	return;
3398
3399	devlink = priv_to_devlink(priv: esw->dev);
3400	devl_lock(devlink);
3401	/* Number of VFs can only change from "0 to x" or "x to 0". */
3402	if (esw->esw_funcs.num_vfs > 0) {
3403	mlx5_eswitch_unload_vf_vports(esw, num_vfs: esw->esw_funcs.num_vfs);
3404	} else {
3405	int err;
3406
3407	err = mlx5_eswitch_load_vf_vports(esw, num_vfs: new_num_vfs,
3408	enabled_events: MLX5_VPORT_UC_ADDR_CHANGE);
3409	if (err) {
3410	devl_unlock(devlink);
3411	return;
3412	}
3413	}
3414	esw->esw_funcs.num_vfs = new_num_vfs;
3415	devl_unlock(devlink);
3416	}
3417
3418	static void esw_functions_changed_event_handler(struct work_struct *work)
3419	{
3420	struct mlx5_host_work *host_work;
3421	struct mlx5_eswitch *esw;
3422	const u32 *out;
3423
3424	host_work = container_of(work, struct mlx5_host_work, work);
3425	esw = host_work->esw;
3426
3427	out = mlx5_esw_query_functions(dev: esw->dev);
3428	if (IS_ERR(ptr: out))
3429	goto out;
3430
3431	esw_vfs_changed_event_handler(esw, out);
3432	kvfree(addr: out);
3433	out:
3434	kfree(objp: host_work);
3435	}
3436
3437	int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
3438	{
3439	struct mlx5_esw_functions *esw_funcs;
3440	struct mlx5_host_work *host_work;
3441	struct mlx5_eswitch *esw;
3442
3443	host_work = kzalloc(size: sizeof(*host_work), GFP_ATOMIC);
3444	if (!host_work)
3445	return NOTIFY_DONE;
3446
3447	esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
3448	esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
3449
3450	host_work->esw = esw;
3451
3452	INIT_WORK(&host_work->work, esw_functions_changed_event_handler);
3453	queue_work(wq: esw->work_queue, work: &host_work->work);
3454
3455	return NOTIFY_OK;
3456	}
3457
3458	static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
3459	{
3460	const u32 *query_host_out;
3461
3462	if (!mlx5_core_is_ecpf_esw_manager(dev: esw->dev))
3463	return 0;
3464
3465	query_host_out = mlx5_esw_query_functions(dev: esw->dev);
3466	if (IS_ERR(ptr: query_host_out))
3467	return PTR_ERR(ptr: query_host_out);
3468
3469	/* Mark non local controller with non zero controller number. */
3470	esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out,
3471	host_params_context.host_number);
3472	kvfree(addr: query_host_out);
3473	return 0;
3474	}
3475
3476	bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller)
3477	{
3478	/* Local controller is always valid */
3479	if (controller == 0)
3480	return true;
3481
3482	if (!mlx5_core_is_ecpf_esw_manager(dev: esw->dev))
3483	return false;
3484
3485	/* External host number starts with zero in device */
3486	return (controller == esw->offloads.host_number + 1);
3487	}
3488
3489	int esw_offloads_enable(struct mlx5_eswitch *esw)
3490	{
3491	struct mapping_ctx *reg_c0_obj_pool;
3492	struct mlx5_vport *vport;
3493	unsigned long i;
3494	u64 mapping_id;
3495	int err;
3496
3497	mutex_init(&esw->offloads.termtbl_mutex);
3498	mlx5_rdma_enable_roce(dev: esw->dev);
3499
3500	err = mlx5_esw_host_number_init(esw);
3501	if (err)
3502	goto err_metadata;
3503
3504	err = esw_offloads_metadata_init(esw);
3505	if (err)
3506	goto err_metadata;
3507
3508	err = esw_set_passing_vport_metadata(esw, enable: true);
3509	if (err)
3510	goto err_vport_metadata;
3511
3512	mapping_id = mlx5_query_nic_system_image_guid(mdev: esw->dev);
3513
3514	reg_c0_obj_pool = mapping_create_for_id(id: mapping_id, type: MAPPING_TYPE_CHAIN,
3515	data_size: sizeof(struct mlx5_mapped_obj),
3516	ESW_REG_C0_USER_DATA_METADATA_MASK,
3517	delayed_removal: true);
3518
3519	if (IS_ERR(ptr: reg_c0_obj_pool)) {
3520	err = PTR_ERR(ptr: reg_c0_obj_pool);
3521	goto err_pool;
3522	}
3523	esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool;
3524
3525	err = esw_offloads_steering_init(esw);
3526	if (err)
3527	goto err_steering_init;
3528
3529	/* Representor will control the vport link state */
3530	mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs)
3531	vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3532	if (mlx5_core_ec_sriov_enabled(dev: esw->dev))
3533	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs)
3534	vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3535
3536	/* Uplink vport rep must load first. */
3537	err = mlx5_esw_offloads_rep_load(esw, vport_num: MLX5_VPORT_UPLINK);
3538	if (err)
3539	goto err_uplink;
3540
3541	err = mlx5_eswitch_enable_pf_vf_vports(esw, enabled_events: MLX5_VPORT_UC_ADDR_CHANGE);
3542	if (err)
3543	goto err_vports;
3544
3545	return 0;
3546
3547	err_vports:
3548	mlx5_esw_offloads_rep_unload(esw, vport_num: MLX5_VPORT_UPLINK);
3549	err_uplink:
3550	esw_offloads_steering_cleanup(esw);
3551	err_steering_init:
3552	mapping_destroy(ctx: reg_c0_obj_pool);
3553	err_pool:
3554	esw_set_passing_vport_metadata(esw, enable: false);
3555	err_vport_metadata:
3556	esw_offloads_metadata_uninit(esw);
3557	err_metadata:
3558	mlx5_rdma_disable_roce(dev: esw->dev);
3559	mutex_destroy(lock: &esw->offloads.termtbl_mutex);
3560	return err;
3561	}
3562
3563	static int esw_offloads_stop(struct mlx5_eswitch *esw,
3564	struct netlink_ext_ack *extack)
3565	{
3566	int err;
3567
3568	esw->mode = MLX5_ESWITCH_LEGACY;
3569
3570	/* If changing from switchdev to legacy mode without sriov enabled,
3571	* no need to create legacy fdb.
3572	*/
3573	if (!mlx5_core_is_pf(dev: esw->dev) || !mlx5_sriov_is_enabled(dev: esw->dev))
3574	return 0;
3575
3576	err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS);
3577	if (err)
3578	NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy");
3579
3580	return err;
3581	}
3582
3583	void esw_offloads_disable(struct mlx5_eswitch *esw)
3584	{
3585	mlx5_eswitch_disable_pf_vf_vports(esw);
3586	mlx5_esw_offloads_rep_unload(esw, vport_num: MLX5_VPORT_UPLINK);
3587	esw_set_passing_vport_metadata(esw, enable: false);
3588	esw_offloads_steering_cleanup(esw);
3589	mapping_destroy(ctx: esw->offloads.reg_c0_obj_pool);
3590	esw_offloads_metadata_uninit(esw);
3591	mlx5_rdma_disable_roce(dev: esw->dev);
3592	mutex_destroy(lock: &esw->offloads.termtbl_mutex);
3593	}
3594
3595	static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
3596	{
3597	switch (mode) {
3598	case DEVLINK_ESWITCH_MODE_LEGACY:
3599	*mlx5_mode = MLX5_ESWITCH_LEGACY;
3600	break;
3601	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3602	*mlx5_mode = MLX5_ESWITCH_OFFLOADS;
3603	break;
3604	default:
3605	return -EINVAL;
3606	}
3607
3608	return 0;
3609	}
3610
3611	static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode)
3612	{
3613	switch (mlx5_mode) {
3614	case MLX5_ESWITCH_LEGACY:
3615	*mode = DEVLINK_ESWITCH_MODE_LEGACY;
3616	break;
3617	case MLX5_ESWITCH_OFFLOADS:
3618	*mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
3619	break;
3620	default:
3621	return -EINVAL;
3622	}
3623
3624	return 0;
3625	}
3626
3627	static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode)
3628	{
3629	switch (mode) {
3630	case DEVLINK_ESWITCH_INLINE_MODE_NONE:
3631	*mlx5_mode = MLX5_INLINE_MODE_NONE;
3632	break;
3633	case DEVLINK_ESWITCH_INLINE_MODE_LINK:
3634	*mlx5_mode = MLX5_INLINE_MODE_L2;
3635	break;
3636	case DEVLINK_ESWITCH_INLINE_MODE_NETWORK:
3637	*mlx5_mode = MLX5_INLINE_MODE_IP;
3638	break;
3639	case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT:
3640	*mlx5_mode = MLX5_INLINE_MODE_TCP_UDP;
3641	break;
3642	default:
3643	return -EINVAL;
3644	}
3645
3646	return 0;
3647	}
3648
3649	static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode)
3650	{
3651	switch (mlx5_mode) {
3652	case MLX5_INLINE_MODE_NONE:
3653	*mode = DEVLINK_ESWITCH_INLINE_MODE_NONE;
3654	break;
3655	case MLX5_INLINE_MODE_L2:
3656	*mode = DEVLINK_ESWITCH_INLINE_MODE_LINK;
3657	break;
3658	case MLX5_INLINE_MODE_IP:
3659	*mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK;
3660	break;
3661	case MLX5_INLINE_MODE_TCP_UDP:
3662	*mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT;
3663	break;
3664	default:
3665	return -EINVAL;
3666	}
3667
3668	return 0;
3669	}
3670
3671	int mlx5_eswitch_block_mode(struct mlx5_core_dev *dev)
3672	{
3673	struct mlx5_eswitch *esw = dev->priv.eswitch;
3674	int err;
3675
3676	if (!mlx5_esw_allowed(esw))
3677	return 0;
3678
3679	/* Take TC into account */
3680	err = mlx5_esw_try_lock(esw);
3681	if (err < 0)
3682	return err;
3683
3684	esw->offloads.num_block_mode++;
3685	mlx5_esw_unlock(esw);
3686	return 0;
3687	}
3688
3689	void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev)
3690	{
3691	struct mlx5_eswitch *esw = dev->priv.eswitch;
3692
3693	if (!mlx5_esw_allowed(esw))
3694	return;
3695
3696	down_write(sem: &esw->mode_lock);
3697	esw->offloads.num_block_mode--;
3698	up_write(sem: &esw->mode_lock);
3699	}
3700
3701	int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
3702	struct netlink_ext_ack *extack)
3703	{
3704	u16 cur_mlx5_mode, mlx5_mode = 0;
3705	struct mlx5_eswitch *esw;
3706	int err = 0;
3707
3708	esw = mlx5_devlink_eswitch_get(devlink);
3709	if (IS_ERR(ptr: esw))
3710	return PTR_ERR(ptr: esw);
3711
3712	if (esw_mode_from_devlink(mode, mlx5_mode: &mlx5_mode))
3713	return -EINVAL;
3714
3715	if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && mlx5_get_sd(dev: esw->dev)) {
3716	NL_SET_ERR_MSG_MOD(extack,
3717	"Can't change E-Switch mode to switchdev when multi-PF netdev (Socket Direct) is configured.");
3718	return -EPERM;
3719	}
3720
3721	mlx5_lag_disable_change(dev: esw->dev);
3722	err = mlx5_esw_try_lock(esw);
3723	if (err < 0) {
3724	NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy");
3725	goto enable_lag;
3726	}
3727	cur_mlx5_mode = err;
3728	err = 0;
3729
3730	if (cur_mlx5_mode == mlx5_mode)
3731	goto unlock;
3732
3733	if (esw->offloads.num_block_mode) {
3734	NL_SET_ERR_MSG_MOD(extack,
3735	"Can't change eswitch mode when IPsec SA and/or policies are configured");
3736	err = -EOPNOTSUPP;
3737	goto unlock;
3738	}
3739
3740	esw->eswitch_operation_in_progress = true;
3741	up_write(sem: &esw->mode_lock);
3742
3743	mlx5_eswitch_disable_locked(esw);
3744	if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) {
3745	if (mlx5_devlink_trap_get_num_active(dev: esw->dev)) {
3746	NL_SET_ERR_MSG_MOD(extack,
3747	"Can't change mode while devlink traps are active");
3748	err = -EOPNOTSUPP;
3749	goto skip;
3750	}
3751	err = esw_offloads_start(esw, extack);
3752	} else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) {
3753	err = esw_offloads_stop(esw, extack);
3754	mlx5_rescan_drivers(dev: esw->dev);
3755	} else {
3756	err = -EINVAL;
3757	}
3758
3759	skip:
3760	down_write(sem: &esw->mode_lock);
3761	esw->eswitch_operation_in_progress = false;
3762	unlock:
3763	mlx5_esw_unlock(esw);
3764	enable_lag:
3765	mlx5_lag_enable_change(dev: esw->dev);
3766	return err;
3767	}
3768
3769	int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3770	{
3771	struct mlx5_eswitch *esw;
3772
3773	esw = mlx5_devlink_eswitch_get(devlink);
3774	if (IS_ERR(ptr: esw))
3775	return PTR_ERR(ptr: esw);
3776
3777	return esw_mode_to_devlink(mlx5_mode: esw->mode, mode);
3778	}
3779
3780	static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode,
3781	struct netlink_ext_ack *extack)
3782	{
3783	struct mlx5_core_dev *dev = esw->dev;
3784	struct mlx5_vport *vport;
3785	u16 err_vport_num = 0;
3786	unsigned long i;
3787	int err = 0;
3788
3789	mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3790	err = mlx5_modify_nic_vport_min_inline(mdev: dev, vport: vport->vport, min_inline: mlx5_mode);
3791	if (err) {
3792	err_vport_num = vport->vport;
3793	NL_SET_ERR_MSG_MOD(extack,
3794	"Failed to set min inline on vport");
3795	goto revert_inline_mode;
3796	}
3797	}
3798	if (mlx5_core_ec_sriov_enabled(dev: esw->dev)) {
3799	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
3800	err = mlx5_modify_nic_vport_min_inline(mdev: dev, vport: vport->vport, min_inline: mlx5_mode);
3801	if (err) {
3802	err_vport_num = vport->vport;
3803	NL_SET_ERR_MSG_MOD(extack,
3804	"Failed to set min inline on vport");
3805	goto revert_ec_vf_inline_mode;
3806	}
3807	}
3808	}
3809	return 0;
3810
3811	revert_ec_vf_inline_mode:
3812	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
3813	if (vport->vport == err_vport_num)
3814	break;
3815	mlx5_modify_nic_vport_min_inline(mdev: dev,
3816	vport: vport->vport,
3817	min_inline: esw->offloads.inline_mode);
3818	}
3819	revert_inline_mode:
3820	mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3821	if (vport->vport == err_vport_num)
3822	break;
3823	mlx5_modify_nic_vport_min_inline(mdev: dev,
3824	vport: vport->vport,
3825	min_inline: esw->offloads.inline_mode);
3826	}
3827	return err;
3828	}
3829
3830	int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
3831	struct netlink_ext_ack *extack)
3832	{
3833	struct mlx5_core_dev *dev = devlink_priv(devlink);
3834	struct mlx5_eswitch *esw;
3835	u8 mlx5_mode;
3836	int err;
3837
3838	esw = mlx5_devlink_eswitch_get(devlink);
3839	if (IS_ERR(ptr: esw))
3840	return PTR_ERR(ptr: esw);
3841
3842	down_write(sem: &esw->mode_lock);
3843
3844	switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
3845	case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
3846	if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) {
3847	err = 0;
3848	goto out;
3849	}
3850
3851	fallthrough;
3852	case MLX5_CAP_INLINE_MODE_L2:
3853	NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set");
3854	err = -EOPNOTSUPP;
3855	goto out;
3856	case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
3857	break;
3858	}
3859
3860	if (atomic64_read(v: &esw->offloads.num_flows) > 0) {
3861	NL_SET_ERR_MSG_MOD(extack,
3862	"Can't set inline mode when flows are configured");
3863	err = -EOPNOTSUPP;
3864	goto out;
3865	}
3866
3867	err = esw_inline_mode_from_devlink(mode, mlx5_mode: &mlx5_mode);
3868	if (err)
3869	goto out;
3870
3871	esw->eswitch_operation_in_progress = true;
3872	up_write(sem: &esw->mode_lock);
3873
3874	err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack);
3875	if (!err)
3876	esw->offloads.inline_mode = mlx5_mode;
3877
3878	down_write(sem: &esw->mode_lock);
3879	esw->eswitch_operation_in_progress = false;
3880	up_write(sem: &esw->mode_lock);
3881	return 0;
3882
3883	out:
3884	up_write(sem: &esw->mode_lock);
3885	return err;
3886	}
3887
3888	int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
3889	{
3890	struct mlx5_eswitch *esw;
3891
3892	esw = mlx5_devlink_eswitch_get(devlink);
3893	if (IS_ERR(ptr: esw))
3894	return PTR_ERR(ptr: esw);
3895
3896	return esw_inline_mode_to_devlink(mlx5_mode: esw->offloads.inline_mode, mode);
3897	}
3898
3899	bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev)
3900	{
3901	struct mlx5_eswitch *esw = dev->priv.eswitch;
3902
3903	if (!mlx5_esw_allowed(esw))
3904	return true;
3905
3906	down_write(sem: &esw->mode_lock);
3907	if (esw->mode != MLX5_ESWITCH_LEGACY &&
3908	esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
3909	up_write(sem: &esw->mode_lock);
3910	return false;
3911	}
3912
3913	esw->offloads.num_block_encap++;
3914	up_write(sem: &esw->mode_lock);
3915	return true;
3916	}
3917
3918	void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev)
3919	{
3920	struct mlx5_eswitch *esw = dev->priv.eswitch;
3921
3922	if (!mlx5_esw_allowed(esw))
3923	return;
3924
3925	down_write(sem: &esw->mode_lock);
3926	esw->offloads.num_block_encap--;
3927	up_write(sem: &esw->mode_lock);
3928	}
3929
3930	int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
3931	enum devlink_eswitch_encap_mode encap,
3932	struct netlink_ext_ack *extack)
3933	{
3934	struct mlx5_core_dev *dev = devlink_priv(devlink);
3935	struct mlx5_eswitch *esw;
3936	int err = 0;
3937
3938	esw = mlx5_devlink_eswitch_get(devlink);
3939	if (IS_ERR(ptr: esw))
3940	return PTR_ERR(ptr: esw);
3941
3942	down_write(sem: &esw->mode_lock);
3943
3944	if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
3945	(!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
3946	!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) {
3947	err = -EOPNOTSUPP;
3948	goto unlock;
3949	}
3950
3951	if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) {
3952	err = -EOPNOTSUPP;
3953	goto unlock;
3954	}
3955
3956	if (esw->mode == MLX5_ESWITCH_LEGACY) {
3957	esw->offloads.encap = encap;
3958	goto unlock;
3959	}
3960
3961	if (esw->offloads.encap == encap)
3962	goto unlock;
3963
3964	if (atomic64_read(v: &esw->offloads.num_flows) > 0) {
3965	NL_SET_ERR_MSG_MOD(extack,
3966	"Can't set encapsulation when flows are configured");
3967	err = -EOPNOTSUPP;
3968	goto unlock;
3969	}
3970
3971	if (esw->offloads.num_block_encap) {
3972	NL_SET_ERR_MSG_MOD(extack,
3973	"Can't set encapsulation when IPsec SA and/or policies are configured");
3974	err = -EOPNOTSUPP;
3975	goto unlock;
3976	}
3977
3978	esw->eswitch_operation_in_progress = true;
3979	up_write(sem: &esw->mode_lock);
3980
3981	esw_destroy_offloads_fdb_tables(esw);
3982
3983	esw->offloads.encap = encap;
3984
3985	err = esw_create_offloads_fdb_tables(esw);
3986
3987	if (err) {
3988	NL_SET_ERR_MSG_MOD(extack,
3989	"Failed re-creating fast FDB table");
3990	esw->offloads.encap = !encap;
3991	(void)esw_create_offloads_fdb_tables(esw);
3992	}
3993
3994	down_write(sem: &esw->mode_lock);
3995	esw->eswitch_operation_in_progress = false;
3996
3997	unlock:
3998	up_write(sem: &esw->mode_lock);
3999	return err;
4000	}
4001
4002	int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
4003	enum devlink_eswitch_encap_mode *encap)
4004	{
4005	struct mlx5_eswitch *esw;
4006
4007	esw = mlx5_devlink_eswitch_get(devlink);
4008	if (IS_ERR(ptr: esw))
4009	return PTR_ERR(ptr: esw);
4010
4011	*encap = esw->offloads.encap;
4012	return 0;
4013	}
4014
4015	static bool
4016	mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
4017	{
4018	/* Currently, only ECPF based device has representor for host PF. */
4019	if (vport_num == MLX5_VPORT_PF &&
4020	!mlx5_core_is_ecpf_esw_manager(dev: esw->dev))
4021	return false;
4022
4023	if (vport_num == MLX5_VPORT_ECPF &&
4024	!mlx5_ecpf_vport_exists(dev: esw->dev))
4025	return false;
4026
4027	return true;
4028	}
4029
4030	void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
4031	const struct mlx5_eswitch_rep_ops *ops,
4032	u8 rep_type)
4033	{
4034	struct mlx5_eswitch_rep_data *rep_data;
4035	struct mlx5_eswitch_rep *rep;
4036	unsigned long i;
4037
4038	esw->offloads.rep_ops[rep_type] = ops;
4039	mlx5_esw_for_each_rep(esw, i, rep) {
4040	if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) {
4041	rep->esw = esw;
4042	rep_data = &rep->rep_data[rep_type];
4043	atomic_set(v: &rep_data->state, i: REP_REGISTERED);
4044	}
4045	}
4046	}
4047	EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
4048
4049	void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
4050	{
4051	struct mlx5_eswitch_rep *rep;
4052	unsigned long i;
4053
4054	if (esw->mode == MLX5_ESWITCH_OFFLOADS)
4055	__unload_reps_all_vport(esw, rep_type);
4056
4057	mlx5_esw_for_each_rep(esw, i, rep)
4058	atomic_set(v: &rep->rep_data[rep_type].state, i: REP_UNREGISTERED);
4059	}
4060	EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
4061
4062	void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
4063	{
4064	struct mlx5_eswitch_rep *rep;
4065
4066	rep = mlx5_eswitch_get_rep(esw, vport_num: MLX5_VPORT_UPLINK);
4067	return rep->rep_data[rep_type].priv;
4068	}
4069
4070	void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
4071	u16 vport,
4072	u8 rep_type)
4073	{
4074	struct mlx5_eswitch_rep *rep;
4075
4076	rep = mlx5_eswitch_get_rep(esw, vport_num: vport);
4077
4078	if (atomic_read(v: &rep->rep_data[rep_type].state) == REP_LOADED &&
4079	esw->offloads.rep_ops[rep_type]->get_proto_dev)
4080	return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep);
4081	return NULL;
4082	}
4083	EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev);
4084
4085	void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type)
4086	{
4087	return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type);
4088	}
4089	EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev);
4090
4091	struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw,
4092	u16 vport)
4093	{
4094	return mlx5_eswitch_get_rep(esw, vport_num: vport);
4095	}
4096	EXPORT_SYMBOL(mlx5_eswitch_vport_rep);
4097
4098	bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw)
4099	{
4100	return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED);
4101	}
4102	EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled);
4103
4104	bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw)
4105	{
4106	return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA);
4107	}
4108	EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled);
4109
4110	u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw,
4111	u16 vport_num)
4112	{
4113	struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
4114
4115	if (WARN_ON_ONCE(IS_ERR(vport)))
4116	return 0;
4117
4118	return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS);
4119	}
4120	EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match);
4121
4122	static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id)
4123	{
4124	int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4125	void *query_ctx;
4126	void *hca_caps;
4127	int err;
4128
4129	*vhca_id = 0;
4130
4131	query_ctx = kzalloc(size: query_out_sz, GFP_KERNEL);
4132	if (!query_ctx)
4133	return -ENOMEM;
4134
4135	err = mlx5_vport_get_other_func_general_cap(esw->dev, vport_num, query_ctx);
4136	if (err)
4137	goto out_free;
4138
4139	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4140	*vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id);
4141
4142	out_free:
4143	kfree(objp: query_ctx);
4144	return err;
4145	}
4146
4147	int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num)
4148	{
4149	u16 *old_entry, *vhca_map_entry, vhca_id;
4150	int err;
4151
4152	err = mlx5_esw_query_vport_vhca_id(esw, vport_num, vhca_id: &vhca_id);
4153	if (err) {
4154	esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n",
4155	vport_num, err);
4156	return err;
4157	}
4158
4159	vhca_map_entry = kmalloc(size: sizeof(*vhca_map_entry), GFP_KERNEL);
4160	if (!vhca_map_entry)
4161	return -ENOMEM;
4162
4163	*vhca_map_entry = vport_num;
4164	old_entry = xa_store(&esw->offloads.vhca_map, index: vhca_id, entry: vhca_map_entry, GFP_KERNEL);
4165	if (xa_is_err(entry: old_entry)) {
4166	kfree(objp: vhca_map_entry);
4167	return xa_err(entry: old_entry);
4168	}
4169	kfree(objp: old_entry);
4170	return 0;
4171	}
4172
4173	void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num)
4174	{
4175	u16 *vhca_map_entry, vhca_id;
4176	int err;
4177
4178	err = mlx5_esw_query_vport_vhca_id(esw, vport_num, vhca_id: &vhca_id);
4179	if (err)
4180	esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n",
4181	vport_num, err);
4182
4183	vhca_map_entry = xa_erase(&esw->offloads.vhca_map, index: vhca_id);
4184	kfree(objp: vhca_map_entry);
4185	}
4186
4187	int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num)
4188	{
4189	u16 *res = xa_load(&esw->offloads.vhca_map, index: vhca_id);
4190
4191	if (!res)
4192	return -ENOENT;
4193
4194	*vport_num = *res;
4195	return 0;
4196	}
4197
4198	u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw,
4199	u16 vport_num)
4200	{
4201	struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
4202
4203	if (WARN_ON_ONCE(IS_ERR(vport)))
4204	return 0;
4205
4206	return vport->metadata;
4207	}
4208	EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set);
4209
4210	int mlx5_devlink_port_fn_hw_addr_get(struct devlink_port *port,
4211	u8 *hw_addr, int *hw_addr_len,
4212	struct netlink_ext_ack *extack)
4213	{
4214	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4215	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4216
4217	mutex_lock(&esw->state_lock);
4218	ether_addr_copy(dst: hw_addr, src: vport->info.mac);
4219	*hw_addr_len = ETH_ALEN;
4220	mutex_unlock(lock: &esw->state_lock);
4221	return 0;
4222	}
4223
4224	int mlx5_devlink_port_fn_hw_addr_set(struct devlink_port *port,
4225	const u8 *hw_addr, int hw_addr_len,
4226	struct netlink_ext_ack *extack)
4227	{
4228	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4229	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4230
4231	return mlx5_eswitch_set_vport_mac(esw, vport: vport->vport, mac: hw_addr);
4232	}
4233
4234	int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled,
4235	struct netlink_ext_ack *extack)
4236	{
4237	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4238	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4239
4240	if (!MLX5_CAP_GEN(esw->dev, migration)) {
4241	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration");
4242	return -EOPNOTSUPP;
4243	}
4244
4245	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4246	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4247	return -EOPNOTSUPP;
4248	}
4249
4250	mutex_lock(&esw->state_lock);
4251	*is_enabled = vport->info.mig_enabled;
4252	mutex_unlock(lock: &esw->state_lock);
4253	return 0;
4254	}
4255
4256	int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable,
4257	struct netlink_ext_ack *extack)
4258	{
4259	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4260	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4261	int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4262	void *query_ctx;
4263	void *hca_caps;
4264	int err;
4265
4266	if (!MLX5_CAP_GEN(esw->dev, migration)) {
4267	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration");
4268	return -EOPNOTSUPP;
4269	}
4270
4271	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4272	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4273	return -EOPNOTSUPP;
4274	}
4275
4276	mutex_lock(&esw->state_lock);
4277
4278	if (vport->info.mig_enabled == enable) {
4279	err = 0;
4280	goto out;
4281	}
4282
4283	query_ctx = kzalloc(size: query_out_sz, GFP_KERNEL);
4284	if (!query_ctx) {
4285	err = -ENOMEM;
4286	goto out;
4287	}
4288
4289	err = mlx5_vport_get_other_func_cap(dev: esw->dev, vport: vport->vport, out: query_ctx,
4290	opmod: MLX5_CAP_GENERAL_2);
4291	if (err) {
4292	NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4293	goto out_free;
4294	}
4295
4296	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4297	MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, enable);
4298
4299	err = mlx5_vport_set_other_func_cap(dev: esw->dev, hca_cap: hca_caps, vport: vport->vport,
4300	opmod: MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2);
4301	if (err) {
4302	NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap");
4303	goto out_free;
4304	}
4305
4306	vport->info.mig_enabled = enable;
4307
4308	out_free:
4309	kfree(objp: query_ctx);
4310	out:
4311	mutex_unlock(lock: &esw->state_lock);
4312	return err;
4313	}
4314
4315	int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled,
4316	struct netlink_ext_ack *extack)
4317	{
4318	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4319	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4320
4321	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4322	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4323	return -EOPNOTSUPP;
4324	}
4325
4326	mutex_lock(&esw->state_lock);
4327	*is_enabled = vport->info.roce_enabled;
4328	mutex_unlock(lock: &esw->state_lock);
4329	return 0;
4330	}
4331
4332	int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable,
4333	struct netlink_ext_ack *extack)
4334	{
4335	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4336	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4337	int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4338	u16 vport_num = vport->vport;
4339	void *query_ctx;
4340	void *hca_caps;
4341	int err;
4342
4343	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4344	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4345	return -EOPNOTSUPP;
4346	}
4347
4348	mutex_lock(&esw->state_lock);
4349
4350	if (vport->info.roce_enabled == enable) {
4351	err = 0;
4352	goto out;
4353	}
4354
4355	query_ctx = kzalloc(size: query_out_sz, GFP_KERNEL);
4356	if (!query_ctx) {
4357	err = -ENOMEM;
4358	goto out;
4359	}
4360
4361	err = mlx5_vport_get_other_func_cap(dev: esw->dev, vport: vport_num, out: query_ctx,
4362	opmod: MLX5_CAP_GENERAL);
4363	if (err) {
4364	NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4365	goto out_free;
4366	}
4367
4368	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4369	MLX5_SET(cmd_hca_cap, hca_caps, roce, enable);
4370
4371	err = mlx5_vport_set_other_func_cap(dev: esw->dev, hca_cap: hca_caps, vport: vport_num,
4372	opmod: MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
4373	if (err) {
4374	NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap");
4375	goto out_free;
4376	}
4377
4378	vport->info.roce_enabled = enable;
4379
4380	out_free:
4381	kfree(objp: query_ctx);
4382	out:
4383	mutex_unlock(lock: &esw->state_lock);
4384	return err;
4385	}
4386
4387	int
4388	mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule,
4389	struct mlx5_esw_flow_attr *esw_attr, int attr_idx)
4390	{
4391	struct mlx5_flow_destination new_dest = {};
4392	struct mlx5_flow_destination old_dest = {};
4393
4394	if (!esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx))
4395	return 0;
4396
4397	esw_setup_dest_fwd_ipsec(dest: &old_dest, NULL, esw, esw_attr, attr_idx, dest_idx: 0, pkt_reformat: false);
4398	esw_setup_dest_fwd_vport(dest: &new_dest, NULL, esw, esw_attr, attr_idx, dest_idx: 0, pkt_reformat: false);
4399
4400	return mlx5_modify_rule_destination(handler: rule, new_dest: &new_dest, old_dest: &old_dest);
4401	}
4402
4403	#ifdef CONFIG_XFRM_OFFLOAD
4404	int mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port *port, bool *is_enabled,
4405	struct netlink_ext_ack *extack)
4406	{
4407	struct mlx5_eswitch *esw;
4408	struct mlx5_vport *vport;
4409	int err = 0;
4410
4411	esw = mlx5_devlink_eswitch_get(devlink: port->devlink);
4412	if (IS_ERR(ptr: esw))
4413	return PTR_ERR(ptr: esw);
4414
4415	if (!mlx5_esw_ipsec_vf_offload_supported(dev: esw->dev)) {
4416	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPSec crypto");
4417	return -EOPNOTSUPP;
4418	}
4419
4420	vport = mlx5_devlink_port_vport_get(dl_port: port);
4421
4422	mutex_lock(&esw->state_lock);
4423	if (!vport->enabled) {
4424	err = -EOPNOTSUPP;
4425	goto unlock;
4426	}
4427
4428	*is_enabled = vport->info.ipsec_crypto_enabled;
4429	unlock:
4430	mutex_unlock(lock: &esw->state_lock);
4431	return err;
4432	}
4433
4434	int mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port *port, bool enable,
4435	struct netlink_ext_ack *extack)
4436	{
4437	struct mlx5_eswitch *esw;
4438	struct mlx5_vport *vport;
4439	u16 vport_num;
4440	int err;
4441
4442	esw = mlx5_devlink_eswitch_get(devlink: port->devlink);
4443	if (IS_ERR(ptr: esw))
4444	return PTR_ERR(ptr: esw);
4445
4446	vport_num = mlx5_esw_devlink_port_index_to_vport_num(dl_port_index: port->index);
4447	err = mlx5_esw_ipsec_vf_crypto_offload_supported(dev: esw->dev, vport_num);
4448	if (err) {
4449	NL_SET_ERR_MSG_MOD(extack,
4450	"Device doesn't support IPsec crypto");
4451	return err;
4452	}
4453
4454	vport = mlx5_devlink_port_vport_get(dl_port: port);
4455
4456	mutex_lock(&esw->state_lock);
4457	if (!vport->enabled) {
4458	err = -EOPNOTSUPP;
4459	NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled");
4460	goto unlock;
4461	}
4462
4463	if (vport->info.ipsec_crypto_enabled == enable)
4464	goto unlock;
4465
4466	if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) {
4467	err = -EBUSY;
4468	goto unlock;
4469	}
4470
4471	err = mlx5_esw_ipsec_vf_crypto_offload_set(esw, vport, enable);
4472	if (err) {
4473	NL_SET_ERR_MSG_MOD(extack, "Failed to set IPsec crypto");
4474	goto unlock;
4475	}
4476
4477	vport->info.ipsec_crypto_enabled = enable;
4478	if (enable)
4479	esw->enabled_ipsec_vf_count++;
4480	else
4481	esw->enabled_ipsec_vf_count--;
4482	unlock:
4483	mutex_unlock(lock: &esw->state_lock);
4484	return err;
4485	}
4486
4487	int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_enabled,
4488	struct netlink_ext_ack *extack)
4489	{
4490	struct mlx5_eswitch *esw;
4491	struct mlx5_vport *vport;
4492	int err = 0;
4493
4494	esw = mlx5_devlink_eswitch_get(devlink: port->devlink);
4495	if (IS_ERR(ptr: esw))
4496	return PTR_ERR(ptr: esw);
4497
4498	if (!mlx5_esw_ipsec_vf_offload_supported(dev: esw->dev)) {
4499	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPsec packet");
4500	return -EOPNOTSUPP;
4501	}
4502
4503	vport = mlx5_devlink_port_vport_get(dl_port: port);
4504
4505	mutex_lock(&esw->state_lock);
4506	if (!vport->enabled) {
4507	err = -EOPNOTSUPP;
4508	goto unlock;
4509	}
4510
4511	*is_enabled = vport->info.ipsec_packet_enabled;
4512	unlock:
4513	mutex_unlock(lock: &esw->state_lock);
4514	return err;
4515	}
4516
4517	int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port,
4518	bool enable,
4519	struct netlink_ext_ack *extack)
4520	{
4521	struct mlx5_eswitch *esw;
4522	struct mlx5_vport *vport;
4523	u16 vport_num;
4524	int err;
4525
4526	esw = mlx5_devlink_eswitch_get(devlink: port->devlink);
4527	if (IS_ERR(ptr: esw))
4528	return PTR_ERR(ptr: esw);
4529
4530	vport_num = mlx5_esw_devlink_port_index_to_vport_num(dl_port_index: port->index);
4531	err = mlx5_esw_ipsec_vf_packet_offload_supported(dev: esw->dev, vport_num);
4532	if (err) {
4533	NL_SET_ERR_MSG_MOD(extack,
4534	"Device doesn't support IPsec packet mode");
4535	return err;
4536	}
4537
4538	vport = mlx5_devlink_port_vport_get(dl_port: port);
4539	mutex_lock(&esw->state_lock);
4540	if (!vport->enabled) {
4541	err = -EOPNOTSUPP;
4542	NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled");
4543	goto unlock;
4544	}
4545
4546	if (vport->info.ipsec_packet_enabled == enable)
4547	goto unlock;
4548
4549	if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) {
4550	err = -EBUSY;
4551	goto unlock;
4552	}
4553
4554	err = mlx5_esw_ipsec_vf_packet_offload_set(esw, vport, enable);
4555	if (err) {
4556	NL_SET_ERR_MSG_MOD(extack,
4557	"Failed to set IPsec packet mode");
4558	goto unlock;
4559	}
4560
4561	vport->info.ipsec_packet_enabled = enable;
4562	if (enable)
4563	esw->enabled_ipsec_vf_count++;
4564	else
4565	esw->enabled_ipsec_vf_count--;
4566	unlock:
4567	mutex_unlock(lock: &esw->state_lock);
4568	return err;
4569	}
4570	#endif /* CONFIG_XFRM_OFFLOAD */
4571