1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* net/sched/cls_flower.c Flower classifier
4	*
5	* Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
6	*/
7
8	#include <linux/kernel.h>
9	#include <linux/init.h>
10	#include <linux/module.h>
11	#include <linux/rhashtable.h>
12	#include <linux/workqueue.h>
13	#include <linux/refcount.h>
14	#include <linux/bitfield.h>
15
16	#include <linux/if_ether.h>
17	#include <linux/in6.h>
18	#include <linux/ip.h>
19	#include <linux/mpls.h>
20	#include <linux/ppp_defs.h>
21
22	#include <net/sch_generic.h>
23	#include <net/pkt_cls.h>
24	#include <net/pkt_sched.h>
25	#include <net/ip.h>
26	#include <net/flow_dissector.h>
27	#include <net/geneve.h>
28	#include <net/vxlan.h>
29	#include <net/erspan.h>
30	#include <net/gtp.h>
31	#include <net/tc_wrapper.h>
32
33	#include <net/dst.h>
34	#include <net/dst_metadata.h>
35
36	#include <uapi/linux/netfilter/nf_conntrack_common.h>
37
38	#define TCA_FLOWER_KEY_CT_FLAGS_MAX \
39	((__TCA_FLOWER_KEY_CT_FLAGS_MAX - 1) << 1)
40	#define TCA_FLOWER_KEY_CT_FLAGS_MASK \
41	(TCA_FLOWER_KEY_CT_FLAGS_MAX - 1)
42
43	struct fl_flow_key {
44	struct flow_dissector_key_meta meta;
45	struct flow_dissector_key_control control;
46	struct flow_dissector_key_control enc_control;
47	struct flow_dissector_key_basic basic;
48	struct flow_dissector_key_eth_addrs eth;
49	struct flow_dissector_key_vlan vlan;
50	struct flow_dissector_key_vlan cvlan;
51	union {
52	struct flow_dissector_key_ipv4_addrs ipv4;
53	struct flow_dissector_key_ipv6_addrs ipv6;
54	};
55	struct flow_dissector_key_ports tp;
56	struct flow_dissector_key_icmp icmp;
57	struct flow_dissector_key_arp arp;
58	struct flow_dissector_key_keyid enc_key_id;
59	union {
60	struct flow_dissector_key_ipv4_addrs enc_ipv4;
61	struct flow_dissector_key_ipv6_addrs enc_ipv6;
62	};
63	struct flow_dissector_key_ports enc_tp;
64	struct flow_dissector_key_mpls mpls;
65	struct flow_dissector_key_tcp tcp;
66	struct flow_dissector_key_ip ip;
67	struct flow_dissector_key_ip enc_ip;
68	struct flow_dissector_key_enc_opts enc_opts;
69	struct flow_dissector_key_ports_range tp_range;
70	struct flow_dissector_key_ct ct;
71	struct flow_dissector_key_hash hash;
72	struct flow_dissector_key_num_of_vlans num_of_vlans;
73	struct flow_dissector_key_pppoe pppoe;
74	struct flow_dissector_key_l2tpv3 l2tpv3;
75	struct flow_dissector_key_ipsec ipsec;
76	struct flow_dissector_key_cfm cfm;
77	} __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */
78
79	struct fl_flow_mask_range {
80	unsigned short int start;
81	unsigned short int end;
82	};
83
84	struct fl_flow_mask {
85	struct fl_flow_key key;
86	struct fl_flow_mask_range range;
87	u32 flags;
88	struct rhash_head ht_node;
89	struct rhashtable ht;
90	struct rhashtable_params filter_ht_params;
91	struct flow_dissector dissector;
92	struct list_head filters;
93	struct rcu_work rwork;
94	struct list_head list;
95	refcount_t refcnt;
96	};
97
98	struct fl_flow_tmplt {
99	struct fl_flow_key dummy_key;
100	struct fl_flow_key mask;
101	struct flow_dissector dissector;
102	struct tcf_chain *chain;
103	};
104
105	struct cls_fl_head {
106	struct rhashtable ht;
107	spinlock_t masks_lock; /* Protect masks list */
108	struct list_head masks;
109	struct list_head hw_filters;
110	struct rcu_work rwork;
111	struct idr handle_idr;
112	};
113
114	struct cls_fl_filter {
115	struct fl_flow_mask *mask;
116	struct rhash_head ht_node;
117	struct fl_flow_key mkey;
118	struct tcf_exts exts;
119	struct tcf_result res;
120	struct fl_flow_key key;
121	struct list_head list;
122	struct list_head hw_list;
123	u32 handle;
124	u32 flags;
125	u32 in_hw_count;
126	u8 needs_tc_skb_ext:1;
127	struct rcu_work rwork;
128	struct net_device *hw_dev;
129	/* Flower classifier is unlocked, which means that its reference counter
130	* can be changed concurrently without any kind of external
131	* synchronization. Use atomic reference counter to be concurrency-safe.
132	*/
133	refcount_t refcnt;
134	bool deleted;
135	};
136
137	static const struct rhashtable_params mask_ht_params = {
138	.key_offset = offsetof(struct fl_flow_mask, key),
139	.key_len = sizeof(struct fl_flow_key),
140	.head_offset = offsetof(struct fl_flow_mask, ht_node),
141	.automatic_shrinking = true,
142	};
143
144	static unsigned short int fl_mask_range(const struct fl_flow_mask *mask)
145	{
146	return mask->range.end - mask->range.start;
147	}
148
149	static void fl_mask_update_range(struct fl_flow_mask *mask)
150	{
151	const u8 *bytes = (const u8 *) &mask->key;
152	size_t size = sizeof(mask->key);
153	size_t i, first = 0, last;
154
155	for (i = 0; i < size; i++) {
156	if (bytes[i]) {
157	first = i;
158	break;
159	}
160	}
161	last = first;
162	for (i = size - 1; i != first; i--) {
163	if (bytes[i]) {
164	last = i;
165	break;
166	}
167	}
168	mask->range.start = rounddown(first, sizeof(long));
169	mask->range.end = roundup(last + 1, sizeof(long));
170	}
171
172	static void *fl_key_get_start(struct fl_flow_key *key,
173	const struct fl_flow_mask *mask)
174	{
175	return (u8 *) key + mask->range.start;
176	}
177
178	static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key,
179	struct fl_flow_mask *mask)
180	{
181	const long *lkey = fl_key_get_start(key, mask);
182	const long *lmask = fl_key_get_start(key: &mask->key, mask);
183	long *lmkey = fl_key_get_start(key: mkey, mask);
184	int i;
185
186	for (i = 0; i < fl_mask_range(mask); i += sizeof(long))
187	*lmkey++ = *lkey++ & *lmask++;
188	}
189
190	static bool fl_mask_fits_tmplt(struct fl_flow_tmplt *tmplt,
191	struct fl_flow_mask *mask)
192	{
193	const long *lmask = fl_key_get_start(key: &mask->key, mask);
194	const long *ltmplt;
195	int i;
196
197	if (!tmplt)
198	return true;
199	ltmplt = fl_key_get_start(key: &tmplt->mask, mask);
200	for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) {
201	if (~*ltmplt++ & *lmask++)
202	return false;
203	}
204	return true;
205	}
206
207	static void fl_clear_masked_range(struct fl_flow_key *key,
208	struct fl_flow_mask *mask)
209	{
210	memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask));
211	}
212
213	static bool fl_range_port_dst_cmp(struct cls_fl_filter *filter,
214	struct fl_flow_key *key,
215	struct fl_flow_key *mkey)
216	{
217	u16 min_mask, max_mask, min_val, max_val;
218
219	min_mask = ntohs(filter->mask->key.tp_range.tp_min.dst);
220	max_mask = ntohs(filter->mask->key.tp_range.tp_max.dst);
221	min_val = ntohs(filter->key.tp_range.tp_min.dst);
222	max_val = ntohs(filter->key.tp_range.tp_max.dst);
223
224	if (min_mask && max_mask) {
225	if (ntohs(key->tp_range.tp.dst) < min_val ||
226	ntohs(key->tp_range.tp.dst) > max_val)
227	return false;
228
229	/* skb does not have min and max values */
230	mkey->tp_range.tp_min.dst = filter->mkey.tp_range.tp_min.dst;
231	mkey->tp_range.tp_max.dst = filter->mkey.tp_range.tp_max.dst;
232	}
233	return true;
234	}
235
236	static bool fl_range_port_src_cmp(struct cls_fl_filter *filter,
237	struct fl_flow_key *key,
238	struct fl_flow_key *mkey)
239	{
240	u16 min_mask, max_mask, min_val, max_val;
241
242	min_mask = ntohs(filter->mask->key.tp_range.tp_min.src);
243	max_mask = ntohs(filter->mask->key.tp_range.tp_max.src);
244	min_val = ntohs(filter->key.tp_range.tp_min.src);
245	max_val = ntohs(filter->key.tp_range.tp_max.src);
246
247	if (min_mask && max_mask) {
248	if (ntohs(key->tp_range.tp.src) < min_val ||
249	ntohs(key->tp_range.tp.src) > max_val)
250	return false;
251
252	/* skb does not have min and max values */
253	mkey->tp_range.tp_min.src = filter->mkey.tp_range.tp_min.src;
254	mkey->tp_range.tp_max.src = filter->mkey.tp_range.tp_max.src;
255	}
256	return true;
257	}
258
259	static struct cls_fl_filter *__fl_lookup(struct fl_flow_mask *mask,
260	struct fl_flow_key *mkey)
261	{
262	return rhashtable_lookup_fast(ht: &mask->ht, key: fl_key_get_start(key: mkey, mask),
263	params: mask->filter_ht_params);
264	}
265
266	static struct cls_fl_filter *fl_lookup_range(struct fl_flow_mask *mask,
267	struct fl_flow_key *mkey,
268	struct fl_flow_key *key)
269	{
270	struct cls_fl_filter *filter, *f;
271
272	list_for_each_entry_rcu(filter, &mask->filters, list) {
273	if (!fl_range_port_dst_cmp(filter, key, mkey))
274	continue;
275
276	if (!fl_range_port_src_cmp(filter, key, mkey))
277	continue;
278
279	f = __fl_lookup(mask, mkey);
280	if (f)
281	return f;
282	}
283	return NULL;
284	}
285
286	static noinline_for_stack
287	struct cls_fl_filter *fl_mask_lookup(struct fl_flow_mask *mask, struct fl_flow_key *key)
288	{
289	struct fl_flow_key mkey;
290
291	fl_set_masked_key(mkey: &mkey, key, mask);
292	if ((mask->flags & TCA_FLOWER_MASK_FLAGS_RANGE))
293	return fl_lookup_range(mask, mkey: &mkey, key);
294
295	return __fl_lookup(mask, mkey: &mkey);
296	}
297
298	static u16 fl_ct_info_to_flower_map[] = {
299	[IP_CT_ESTABLISHED] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
300	TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED,
301	[IP_CT_RELATED] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
302	TCA_FLOWER_KEY_CT_FLAGS_RELATED,
303	[IP_CT_ESTABLISHED_REPLY] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
304	TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED |
305	TCA_FLOWER_KEY_CT_FLAGS_REPLY,
306	[IP_CT_RELATED_REPLY] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
307	TCA_FLOWER_KEY_CT_FLAGS_RELATED |
308	TCA_FLOWER_KEY_CT_FLAGS_REPLY,
309	[IP_CT_NEW] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
310	TCA_FLOWER_KEY_CT_FLAGS_NEW,
311	};
312
313	TC_INDIRECT_SCOPE int fl_classify(struct sk_buff *skb,
314	const struct tcf_proto *tp,
315	struct tcf_result *res)
316	{
317	struct cls_fl_head *head = rcu_dereference_bh(tp->root);
318	bool post_ct = tc_skb_cb(skb)->post_ct;
319	u16 zone = tc_skb_cb(skb)->zone;
320	struct fl_flow_key skb_key;
321	struct fl_flow_mask *mask;
322	struct cls_fl_filter *f;
323
324	list_for_each_entry_rcu(mask, &head->masks, list) {
325	flow_dissector_init_keys(key_control: &skb_key.control, key_basic: &skb_key.basic);
326	fl_clear_masked_range(key: &skb_key, mask);
327
328	skb_flow_dissect_meta(skb, flow_dissector: &mask->dissector, target_container: &skb_key);
329	/* skb_flow_dissect() does not set n_proto in case an unknown
330	* protocol, so do it rather here.
331	*/
332	skb_key.basic.n_proto = skb_protocol(skb, skip_vlan: false);
333	skb_flow_dissect_tunnel_info(skb, flow_dissector: &mask->dissector, target_container: &skb_key);
334	skb_flow_dissect_ct(skb, flow_dissector: &mask->dissector, target_container: &skb_key,
335	ctinfo_map: fl_ct_info_to_flower_map,
336	ARRAY_SIZE(fl_ct_info_to_flower_map),
337	post_ct, zone);
338	skb_flow_dissect_hash(skb, flow_dissector: &mask->dissector, target_container: &skb_key);
339	skb_flow_dissect(skb, flow_dissector: &mask->dissector, target_container: &skb_key,
340	FLOW_DISSECTOR_F_STOP_BEFORE_ENCAP);
341
342	f = fl_mask_lookup(mask, key: &skb_key);
343	if (f && !tc_skip_sw(flags: f->flags)) {
344	*res = f->res;
345	return tcf_exts_exec(skb, exts: &f->exts, res);
346	}
347	}
348	return -1;
349	}
350
351	static int fl_init(struct tcf_proto *tp)
352	{
353	struct cls_fl_head *head;
354
355	head = kzalloc(size: sizeof(*head), GFP_KERNEL);
356	if (!head)
357	return -ENOBUFS;
358
359	spin_lock_init(&head->masks_lock);
360	INIT_LIST_HEAD_RCU(list: &head->masks);
361	INIT_LIST_HEAD(list: &head->hw_filters);
362	rcu_assign_pointer(tp->root, head);
363	idr_init(idr: &head->handle_idr);
364
365	return rhashtable_init(ht: &head->ht, params: &mask_ht_params);
366	}
367
368	static void fl_mask_free(struct fl_flow_mask *mask, bool mask_init_done)
369	{
370	/* temporary masks don't have their filters list and ht initialized */
371	if (mask_init_done) {
372	WARN_ON(!list_empty(&mask->filters));
373	rhashtable_destroy(ht: &mask->ht);
374	}
375	kfree(objp: mask);
376	}
377
378	static void fl_mask_free_work(struct work_struct *work)
379	{
380	struct fl_flow_mask *mask = container_of(to_rcu_work(work),
381	struct fl_flow_mask, rwork);
382
383	fl_mask_free(mask, mask_init_done: true);
384	}
385
386	static void fl_uninit_mask_free_work(struct work_struct *work)
387	{
388	struct fl_flow_mask *mask = container_of(to_rcu_work(work),
389	struct fl_flow_mask, rwork);
390
391	fl_mask_free(mask, mask_init_done: false);
392	}
393
394	static bool fl_mask_put(struct cls_fl_head *head, struct fl_flow_mask *mask)
395	{
396	if (!refcount_dec_and_test(r: &mask->refcnt))
397	return false;
398
399	rhashtable_remove_fast(ht: &head->ht, obj: &mask->ht_node, params: mask_ht_params);
400
401	spin_lock(lock: &head->masks_lock);
402	list_del_rcu(entry: &mask->list);
403	spin_unlock(lock: &head->masks_lock);
404
405	tcf_queue_work(rwork: &mask->rwork, func: fl_mask_free_work);
406
407	return true;
408	}
409
410	static struct cls_fl_head *fl_head_dereference(struct tcf_proto *tp)
411	{
412	/* Flower classifier only changes root pointer during init and destroy.
413	* Users must obtain reference to tcf_proto instance before calling its
414	* API, so tp->root pointer is protected from concurrent call to
415	* fl_destroy() by reference counting.
416	*/
417	return rcu_dereference_raw(tp->root);
418	}
419
420	static void __fl_destroy_filter(struct cls_fl_filter *f)
421	{
422	if (f->needs_tc_skb_ext)
423	tc_skb_ext_tc_disable();
424	tcf_exts_destroy(exts: &f->exts);
425	tcf_exts_put_net(exts: &f->exts);
426	kfree(objp: f);
427	}
428
429	static void fl_destroy_filter_work(struct work_struct *work)
430	{
431	struct cls_fl_filter *f = container_of(to_rcu_work(work),
432	struct cls_fl_filter, rwork);
433
434	__fl_destroy_filter(f);
435	}
436
437	static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f,
438	bool rtnl_held, struct netlink_ext_ack *extack)
439	{
440	struct tcf_block *block = tp->chain->block;
441	struct flow_cls_offload cls_flower = {};
442
443	tc_cls_common_offload_init(cls_common: &cls_flower.common, tp, flags: f->flags, extack);
444	cls_flower.command = FLOW_CLS_DESTROY;
445	cls_flower.cookie = (unsigned long) f;
446
447	tc_setup_cb_destroy(block, tp, type: TC_SETUP_CLSFLOWER, type_data: &cls_flower, err_stop: false,
448	flags: &f->flags, in_hw_count: &f->in_hw_count, rtnl_held);
449
450	}
451
452	static int fl_hw_replace_filter(struct tcf_proto *tp,
453	struct cls_fl_filter *f, bool rtnl_held,
454	struct netlink_ext_ack *extack)
455	{
456	struct tcf_block *block = tp->chain->block;
457	struct flow_cls_offload cls_flower = {};
458	bool skip_sw = tc_skip_sw(flags: f->flags);
459	int err = 0;
460
461	cls_flower.rule = flow_rule_alloc(num_actions: tcf_exts_num_actions(exts: &f->exts));
462	if (!cls_flower.rule)
463	return -ENOMEM;
464
465	tc_cls_common_offload_init(cls_common: &cls_flower.common, tp, flags: f->flags, extack);
466	cls_flower.command = FLOW_CLS_REPLACE;
467	cls_flower.cookie = (unsigned long) f;
468	cls_flower.rule->match.dissector = &f->mask->dissector;
469	cls_flower.rule->match.mask = &f->mask->key;
470	cls_flower.rule->match.key = &f->mkey;
471	cls_flower.classid = f->res.classid;
472
473	err = tc_setup_offload_action(flow_action: &cls_flower.rule->action, exts: &f->exts,
474	extack: cls_flower.common.extack);
475	if (err) {
476	kfree(objp: cls_flower.rule);
477
478	return skip_sw ? err : 0;
479	}
480
481	err = tc_setup_cb_add(block, tp, type: TC_SETUP_CLSFLOWER, type_data: &cls_flower,
482	err_stop: skip_sw, flags: &f->flags, in_hw_count: &f->in_hw_count, rtnl_held);
483	tc_cleanup_offload_action(flow_action: &cls_flower.rule->action);
484	kfree(objp: cls_flower.rule);
485
486	if (err) {
487	fl_hw_destroy_filter(tp, f, rtnl_held, NULL);
488	return err;
489	}
490
491	if (skip_sw && !(f->flags & TCA_CLS_FLAGS_IN_HW))
492	return -EINVAL;
493
494	return 0;
495	}
496
497	static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f,
498	bool rtnl_held)
499	{
500	struct tcf_block *block = tp->chain->block;
501	struct flow_cls_offload cls_flower = {};
502
503	tc_cls_common_offload_init(cls_common: &cls_flower.common, tp, flags: f->flags, NULL);
504	cls_flower.command = FLOW_CLS_STATS;
505	cls_flower.cookie = (unsigned long) f;
506	cls_flower.classid = f->res.classid;
507
508	tc_setup_cb_call(block, type: TC_SETUP_CLSFLOWER, type_data: &cls_flower, err_stop: false,
509	rtnl_held);
510
511	tcf_exts_hw_stats_update(exts: &f->exts, stats: &cls_flower.stats, use_act_stats: cls_flower.use_act_stats);
512	}
513
514	static void __fl_put(struct cls_fl_filter *f)
515	{
516	if (!refcount_dec_and_test(r: &f->refcnt))
517	return;
518
519	if (tcf_exts_get_net(exts: &f->exts))
520	tcf_queue_work(rwork: &f->rwork, func: fl_destroy_filter_work);
521	else
522	__fl_destroy_filter(f);
523	}
524
525	static struct cls_fl_filter *__fl_get(struct cls_fl_head *head, u32 handle)
526	{
527	struct cls_fl_filter *f;
528
529	rcu_read_lock();
530	f = idr_find(&head->handle_idr, id: handle);
531	if (f && !refcount_inc_not_zero(r: &f->refcnt))
532	f = NULL;
533	rcu_read_unlock();
534
535	return f;
536	}
537
538	static struct tcf_exts *fl_get_exts(const struct tcf_proto *tp, u32 handle)
539	{
540	struct cls_fl_head *head = rcu_dereference_bh(tp->root);
541	struct cls_fl_filter *f;
542
543	f = idr_find(&head->handle_idr, id: handle);
544	return f ? &f->exts : NULL;
545	}
546
547	static int __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f,
548	bool *last, bool rtnl_held,
549	struct netlink_ext_ack *extack)
550	{
551	struct cls_fl_head *head = fl_head_dereference(tp);
552
553	*last = false;
554
555	spin_lock(lock: &tp->lock);
556	if (f->deleted) {
557	spin_unlock(lock: &tp->lock);
558	return -ENOENT;
559	}
560
561	f->deleted = true;
562	rhashtable_remove_fast(ht: &f->mask->ht, obj: &f->ht_node,
563	params: f->mask->filter_ht_params);
564	idr_remove(&head->handle_idr, id: f->handle);
565	list_del_rcu(entry: &f->list);
566	spin_unlock(lock: &tp->lock);
567
568	*last = fl_mask_put(head, mask: f->mask);
569	if (!tc_skip_hw(flags: f->flags))
570	fl_hw_destroy_filter(tp, f, rtnl_held, extack);
571	tcf_unbind_filter(tp, r: &f->res);
572	__fl_put(f);
573
574	return 0;
575	}
576
577	static void fl_destroy_sleepable(struct work_struct *work)
578	{
579	struct cls_fl_head *head = container_of(to_rcu_work(work),
580	struct cls_fl_head,
581	rwork);
582
583	rhashtable_destroy(ht: &head->ht);
584	kfree(objp: head);
585	module_put(THIS_MODULE);
586	}
587
588	static void fl_destroy(struct tcf_proto *tp, bool rtnl_held,
589	struct netlink_ext_ack *extack)
590	{
591	struct cls_fl_head *head = fl_head_dereference(tp);
592	struct fl_flow_mask *mask, *next_mask;
593	struct cls_fl_filter *f, *next;
594	bool last;
595
596	list_for_each_entry_safe(mask, next_mask, &head->masks, list) {
597	list_for_each_entry_safe(f, next, &mask->filters, list) {
598	__fl_delete(tp, f, last: &last, rtnl_held, extack);
599	if (last)
600	break;
601	}
602	}
603	idr_destroy(&head->handle_idr);
604
605	__module_get(THIS_MODULE);
606	tcf_queue_work(rwork: &head->rwork, func: fl_destroy_sleepable);
607	}
608
609	static void fl_put(struct tcf_proto *tp, void *arg)
610	{
611	struct cls_fl_filter *f = arg;
612
613	__fl_put(f);
614	}
615
616	static void *fl_get(struct tcf_proto *tp, u32 handle)
617	{
618	struct cls_fl_head *head = fl_head_dereference(tp);
619
620	return __fl_get(head, handle);
621	}
622
623	static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = {
624	[TCA_FLOWER_UNSPEC] = { .strict_start_type =
625	TCA_FLOWER_L2_MISS },
626	[TCA_FLOWER_CLASSID] = { .type = NLA_U32 },
627	[TCA_FLOWER_INDEV] = { .type = NLA_STRING,
628	.len = IFNAMSIZ },
629	[TCA_FLOWER_KEY_ETH_DST] = { .len = ETH_ALEN },
630	[TCA_FLOWER_KEY_ETH_DST_MASK] = { .len = ETH_ALEN },
631	[TCA_FLOWER_KEY_ETH_SRC] = { .len = ETH_ALEN },
632	[TCA_FLOWER_KEY_ETH_SRC_MASK] = { .len = ETH_ALEN },
633	[TCA_FLOWER_KEY_ETH_TYPE] = { .type = NLA_U16 },
634	[TCA_FLOWER_KEY_IP_PROTO] = { .type = NLA_U8 },
635	[TCA_FLOWER_KEY_IPV4_SRC] = { .type = NLA_U32 },
636	[TCA_FLOWER_KEY_IPV4_SRC_MASK] = { .type = NLA_U32 },
637	[TCA_FLOWER_KEY_IPV4_DST] = { .type = NLA_U32 },
638	[TCA_FLOWER_KEY_IPV4_DST_MASK] = { .type = NLA_U32 },
639	[TCA_FLOWER_KEY_IPV6_SRC] = { .len = sizeof(struct in6_addr) },
640	[TCA_FLOWER_KEY_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
641	[TCA_FLOWER_KEY_IPV6_DST] = { .len = sizeof(struct in6_addr) },
642	[TCA_FLOWER_KEY_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
643	[TCA_FLOWER_KEY_TCP_SRC] = { .type = NLA_U16 },
644	[TCA_FLOWER_KEY_TCP_DST] = { .type = NLA_U16 },
645	[TCA_FLOWER_KEY_UDP_SRC] = { .type = NLA_U16 },
646	[TCA_FLOWER_KEY_UDP_DST] = { .type = NLA_U16 },
647	[TCA_FLOWER_KEY_VLAN_ID] = { .type = NLA_U16 },
648	[TCA_FLOWER_KEY_VLAN_PRIO] = { .type = NLA_U8 },
649	[TCA_FLOWER_KEY_VLAN_ETH_TYPE] = { .type = NLA_U16 },
650	[TCA_FLOWER_KEY_ENC_KEY_ID] = { .type = NLA_U32 },
651	[TCA_FLOWER_KEY_ENC_IPV4_SRC] = { .type = NLA_U32 },
652	[TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 },
653	[TCA_FLOWER_KEY_ENC_IPV4_DST] = { .type = NLA_U32 },
654	[TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 },
655	[TCA_FLOWER_KEY_ENC_IPV6_SRC] = { .len = sizeof(struct in6_addr) },
656	[TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
657	[TCA_FLOWER_KEY_ENC_IPV6_DST] = { .len = sizeof(struct in6_addr) },
658	[TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
659	[TCA_FLOWER_KEY_TCP_SRC_MASK] = { .type = NLA_U16 },
660	[TCA_FLOWER_KEY_TCP_DST_MASK] = { .type = NLA_U16 },
661	[TCA_FLOWER_KEY_UDP_SRC_MASK] = { .type = NLA_U16 },
662	[TCA_FLOWER_KEY_UDP_DST_MASK] = { .type = NLA_U16 },
663	[TCA_FLOWER_KEY_SCTP_SRC_MASK] = { .type = NLA_U16 },
664	[TCA_FLOWER_KEY_SCTP_DST_MASK] = { .type = NLA_U16 },
665	[TCA_FLOWER_KEY_SCTP_SRC] = { .type = NLA_U16 },
666	[TCA_FLOWER_KEY_SCTP_DST] = { .type = NLA_U16 },
667	[TCA_FLOWER_KEY_ENC_UDP_SRC_PORT] = { .type = NLA_U16 },
668	[TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK] = { .type = NLA_U16 },
669	[TCA_FLOWER_KEY_ENC_UDP_DST_PORT] = { .type = NLA_U16 },
670	[TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK] = { .type = NLA_U16 },
671	[TCA_FLOWER_KEY_FLAGS] = { .type = NLA_U32 },
672	[TCA_FLOWER_KEY_FLAGS_MASK] = { .type = NLA_U32 },
673	[TCA_FLOWER_KEY_ICMPV4_TYPE] = { .type = NLA_U8 },
674	[TCA_FLOWER_KEY_ICMPV4_TYPE_MASK] = { .type = NLA_U8 },
675	[TCA_FLOWER_KEY_ICMPV4_CODE] = { .type = NLA_U8 },
676	[TCA_FLOWER_KEY_ICMPV4_CODE_MASK] = { .type = NLA_U8 },
677	[TCA_FLOWER_KEY_ICMPV6_TYPE] = { .type = NLA_U8 },
678	[TCA_FLOWER_KEY_ICMPV6_TYPE_MASK] = { .type = NLA_U8 },
679	[TCA_FLOWER_KEY_ICMPV6_CODE] = { .type = NLA_U8 },
680	[TCA_FLOWER_KEY_ICMPV6_CODE_MASK] = { .type = NLA_U8 },
681	[TCA_FLOWER_KEY_ARP_SIP] = { .type = NLA_U32 },
682	[TCA_FLOWER_KEY_ARP_SIP_MASK] = { .type = NLA_U32 },
683	[TCA_FLOWER_KEY_ARP_TIP] = { .type = NLA_U32 },
684	[TCA_FLOWER_KEY_ARP_TIP_MASK] = { .type = NLA_U32 },
685	[TCA_FLOWER_KEY_ARP_OP] = { .type = NLA_U8 },
686	[TCA_FLOWER_KEY_ARP_OP_MASK] = { .type = NLA_U8 },
687	[TCA_FLOWER_KEY_ARP_SHA] = { .len = ETH_ALEN },
688	[TCA_FLOWER_KEY_ARP_SHA_MASK] = { .len = ETH_ALEN },
689	[TCA_FLOWER_KEY_ARP_THA] = { .len = ETH_ALEN },
690	[TCA_FLOWER_KEY_ARP_THA_MASK] = { .len = ETH_ALEN },
691	[TCA_FLOWER_KEY_MPLS_TTL] = { .type = NLA_U8 },
692	[TCA_FLOWER_KEY_MPLS_BOS] = { .type = NLA_U8 },
693	[TCA_FLOWER_KEY_MPLS_TC] = { .type = NLA_U8 },
694	[TCA_FLOWER_KEY_MPLS_LABEL] = { .type = NLA_U32 },
695	[TCA_FLOWER_KEY_MPLS_OPTS] = { .type = NLA_NESTED },
696	[TCA_FLOWER_KEY_TCP_FLAGS] = { .type = NLA_U16 },
697	[TCA_FLOWER_KEY_TCP_FLAGS_MASK] = { .type = NLA_U16 },
698	[TCA_FLOWER_KEY_IP_TOS] = { .type = NLA_U8 },
699	[TCA_FLOWER_KEY_IP_TOS_MASK] = { .type = NLA_U8 },
700	[TCA_FLOWER_KEY_IP_TTL] = { .type = NLA_U8 },
701	[TCA_FLOWER_KEY_IP_TTL_MASK] = { .type = NLA_U8 },
702	[TCA_FLOWER_KEY_CVLAN_ID] = { .type = NLA_U16 },
703	[TCA_FLOWER_KEY_CVLAN_PRIO] = { .type = NLA_U8 },
704	[TCA_FLOWER_KEY_CVLAN_ETH_TYPE] = { .type = NLA_U16 },
705	[TCA_FLOWER_KEY_ENC_IP_TOS] = { .type = NLA_U8 },
706	[TCA_FLOWER_KEY_ENC_IP_TOS_MASK] = { .type = NLA_U8 },
707	[TCA_FLOWER_KEY_ENC_IP_TTL] = { .type = NLA_U8 },
708	[TCA_FLOWER_KEY_ENC_IP_TTL_MASK] = { .type = NLA_U8 },
709	[TCA_FLOWER_KEY_ENC_OPTS] = { .type = NLA_NESTED },
710	[TCA_FLOWER_KEY_ENC_OPTS_MASK] = { .type = NLA_NESTED },
711	[TCA_FLOWER_KEY_CT_STATE] =
712	NLA_POLICY_MASK(NLA_U16, TCA_FLOWER_KEY_CT_FLAGS_MASK),
713	[TCA_FLOWER_KEY_CT_STATE_MASK] =
714	NLA_POLICY_MASK(NLA_U16, TCA_FLOWER_KEY_CT_FLAGS_MASK),
715	[TCA_FLOWER_KEY_CT_ZONE] = { .type = NLA_U16 },
716	[TCA_FLOWER_KEY_CT_ZONE_MASK] = { .type = NLA_U16 },
717	[TCA_FLOWER_KEY_CT_MARK] = { .type = NLA_U32 },
718	[TCA_FLOWER_KEY_CT_MARK_MASK] = { .type = NLA_U32 },
719	[TCA_FLOWER_KEY_CT_LABELS] = { .type = NLA_BINARY,
720	.len = 128 / BITS_PER_BYTE },
721	[TCA_FLOWER_KEY_CT_LABELS_MASK] = { .type = NLA_BINARY,
722	.len = 128 / BITS_PER_BYTE },
723	[TCA_FLOWER_FLAGS] = { .type = NLA_U32 },
724	[TCA_FLOWER_KEY_HASH] = { .type = NLA_U32 },
725	[TCA_FLOWER_KEY_HASH_MASK] = { .type = NLA_U32 },
726	[TCA_FLOWER_KEY_NUM_OF_VLANS] = { .type = NLA_U8 },
727	[TCA_FLOWER_KEY_PPPOE_SID] = { .type = NLA_U16 },
728	[TCA_FLOWER_KEY_PPP_PROTO] = { .type = NLA_U16 },
729	[TCA_FLOWER_KEY_L2TPV3_SID] = { .type = NLA_U32 },
730	[TCA_FLOWER_KEY_SPI] = { .type = NLA_U32 },
731	[TCA_FLOWER_KEY_SPI_MASK] = { .type = NLA_U32 },
732	[TCA_FLOWER_L2_MISS] = NLA_POLICY_MAX(NLA_U8, 1),
733	[TCA_FLOWER_KEY_CFM] = { .type = NLA_NESTED },
734	};
735
736	static const struct nla_policy
737	enc_opts_policy[TCA_FLOWER_KEY_ENC_OPTS_MAX + 1] = {
738	[TCA_FLOWER_KEY_ENC_OPTS_UNSPEC] = {
739	.strict_start_type = TCA_FLOWER_KEY_ENC_OPTS_VXLAN },
740	[TCA_FLOWER_KEY_ENC_OPTS_GENEVE] = { .type = NLA_NESTED },
741	[TCA_FLOWER_KEY_ENC_OPTS_VXLAN] = { .type = NLA_NESTED },
742	[TCA_FLOWER_KEY_ENC_OPTS_ERSPAN] = { .type = NLA_NESTED },
743	[TCA_FLOWER_KEY_ENC_OPTS_GTP] = { .type = NLA_NESTED },
744	};
745
746	static const struct nla_policy
747	geneve_opt_policy[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1] = {
748	[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] = { .type = NLA_U16 },
749	[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] = { .type = NLA_U8 },
750	[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA] = { .type = NLA_BINARY,
751	.len = 128 },
752	};
753
754	static const struct nla_policy
755	vxlan_opt_policy[TCA_FLOWER_KEY_ENC_OPT_VXLAN_MAX + 1] = {
756	[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP] = { .type = NLA_U32 },
757	};
758
759	static const struct nla_policy
760	erspan_opt_policy[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX + 1] = {
761	[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER] = { .type = NLA_U8 },
762	[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX] = { .type = NLA_U32 },
763	[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR] = { .type = NLA_U8 },
764	[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID] = { .type = NLA_U8 },
765	};
766
767	static const struct nla_policy
768	gtp_opt_policy[TCA_FLOWER_KEY_ENC_OPT_GTP_MAX + 1] = {
769	[TCA_FLOWER_KEY_ENC_OPT_GTP_PDU_TYPE] = { .type = NLA_U8 },
770	[TCA_FLOWER_KEY_ENC_OPT_GTP_QFI] = { .type = NLA_U8 },
771	};
772
773	static const struct nla_policy
774	mpls_stack_entry_policy[TCA_FLOWER_KEY_MPLS_OPT_LSE_MAX + 1] = {
775	[TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH] = { .type = NLA_U8 },
776	[TCA_FLOWER_KEY_MPLS_OPT_LSE_TTL] = { .type = NLA_U8 },
777	[TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS] = { .type = NLA_U8 },
778	[TCA_FLOWER_KEY_MPLS_OPT_LSE_TC] = { .type = NLA_U8 },
779	[TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL] = { .type = NLA_U32 },
780	};
781
782	static const struct nla_policy
783	cfm_opt_policy[TCA_FLOWER_KEY_CFM_OPT_MAX + 1] = {
784	[TCA_FLOWER_KEY_CFM_MD_LEVEL] = NLA_POLICY_MAX(NLA_U8,
785	FLOW_DIS_CFM_MDL_MAX),
786	[TCA_FLOWER_KEY_CFM_OPCODE] = { .type = NLA_U8 },
787	};
788
789	static void fl_set_key_val(struct nlattr **tb,
790	void *val, int val_type,
791	void *mask, int mask_type, int len)
792	{
793	if (!tb[val_type])
794	return;
795	nla_memcpy(dest: val, src: tb[val_type], count: len);
796	if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type])
797	memset(mask, 0xff, len);
798	else
799	nla_memcpy(dest: mask, src: tb[mask_type], count: len);
800	}
801
802	static int fl_set_key_spi(struct nlattr **tb, struct fl_flow_key *key,
803	struct fl_flow_key *mask,
804	struct netlink_ext_ack *extack)
805	{
806	if (key->basic.ip_proto != IPPROTO_ESP &&
807	key->basic.ip_proto != IPPROTO_AH) {
808	NL_SET_ERR_MSG(extack,
809	"Protocol must be either ESP or AH");
810	return -EINVAL;
811	}
812
813	fl_set_key_val(tb, val: &key->ipsec.spi,
814	val_type: TCA_FLOWER_KEY_SPI,
815	mask: &mask->ipsec.spi, mask_type: TCA_FLOWER_KEY_SPI_MASK,
816	len: sizeof(key->ipsec.spi));
817	return 0;
818	}
819
820	static int fl_set_key_port_range(struct nlattr **tb, struct fl_flow_key *key,
821	struct fl_flow_key *mask,
822	struct netlink_ext_ack *extack)
823	{
824	fl_set_key_val(tb, val: &key->tp_range.tp_min.dst,
825	val_type: TCA_FLOWER_KEY_PORT_DST_MIN, mask: &mask->tp_range.tp_min.dst,
826	mask_type: TCA_FLOWER_UNSPEC, len: sizeof(key->tp_range.tp_min.dst));
827	fl_set_key_val(tb, val: &key->tp_range.tp_max.dst,
828	val_type: TCA_FLOWER_KEY_PORT_DST_MAX, mask: &mask->tp_range.tp_max.dst,
829	mask_type: TCA_FLOWER_UNSPEC, len: sizeof(key->tp_range.tp_max.dst));
830	fl_set_key_val(tb, val: &key->tp_range.tp_min.src,
831	val_type: TCA_FLOWER_KEY_PORT_SRC_MIN, mask: &mask->tp_range.tp_min.src,
832	mask_type: TCA_FLOWER_UNSPEC, len: sizeof(key->tp_range.tp_min.src));
833	fl_set_key_val(tb, val: &key->tp_range.tp_max.src,
834	val_type: TCA_FLOWER_KEY_PORT_SRC_MAX, mask: &mask->tp_range.tp_max.src,
835	mask_type: TCA_FLOWER_UNSPEC, len: sizeof(key->tp_range.tp_max.src));
836
837	if (mask->tp_range.tp_min.dst != mask->tp_range.tp_max.dst) {
838	NL_SET_ERR_MSG(extack,
839	"Both min and max destination ports must be specified");
840	return -EINVAL;
841	}
842	if (mask->tp_range.tp_min.src != mask->tp_range.tp_max.src) {
843	NL_SET_ERR_MSG(extack,
844	"Both min and max source ports must be specified");
845	return -EINVAL;
846	}
847	if (mask->tp_range.tp_min.dst && mask->tp_range.tp_max.dst &&
848	ntohs(key->tp_range.tp_max.dst) <=
849	ntohs(key->tp_range.tp_min.dst)) {
850	NL_SET_ERR_MSG_ATTR(extack,
851	tb[TCA_FLOWER_KEY_PORT_DST_MIN],
852	"Invalid destination port range (min must be strictly smaller than max)");
853	return -EINVAL;
854	}
855	if (mask->tp_range.tp_min.src && mask->tp_range.tp_max.src &&
856	ntohs(key->tp_range.tp_max.src) <=
857	ntohs(key->tp_range.tp_min.src)) {
858	NL_SET_ERR_MSG_ATTR(extack,
859	tb[TCA_FLOWER_KEY_PORT_SRC_MIN],
860	"Invalid source port range (min must be strictly smaller than max)");
861	return -EINVAL;
862	}
863
864	return 0;
865	}
866
867	static int fl_set_key_mpls_lse(const struct nlattr *nla_lse,
868	struct flow_dissector_key_mpls *key_val,
869	struct flow_dissector_key_mpls *key_mask,
870	struct netlink_ext_ack *extack)
871	{
872	struct nlattr *tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_MAX + 1];
873	struct flow_dissector_mpls_lse *lse_mask;
874	struct flow_dissector_mpls_lse *lse_val;
875	u8 lse_index;
876	u8 depth;
877	int err;
878
879	err = nla_parse_nested(tb, TCA_FLOWER_KEY_MPLS_OPT_LSE_MAX, nla: nla_lse,
880	policy: mpls_stack_entry_policy, extack);
881	if (err < 0)
882	return err;
883
884	if (!tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH]) {
885	NL_SET_ERR_MSG(extack, "Missing MPLS option \"depth\"");
886	return -EINVAL;
887	}
888
889	depth = nla_get_u8(nla: tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH]);
890
891	/* LSE depth starts at 1, for consistency with terminology used by
892	* RFC 3031 (section 3.9), where depth 0 refers to unlabeled packets.
893	*/
894	if (depth < 1 || depth > FLOW_DIS_MPLS_MAX) {
895	NL_SET_ERR_MSG_ATTR(extack,
896	tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH],
897	"Invalid MPLS depth");
898	return -EINVAL;
899	}
900	lse_index = depth - 1;
901
902	dissector_set_mpls_lse(mpls: key_val, lse_index);
903	dissector_set_mpls_lse(mpls: key_mask, lse_index);
904
905	lse_val = &key_val->ls[lse_index];
906	lse_mask = &key_mask->ls[lse_index];
907
908	if (tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TTL]) {
909	lse_val->mpls_ttl = nla_get_u8(nla: tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TTL]);
910	lse_mask->mpls_ttl = MPLS_TTL_MASK;
911	}
912	if (tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS]) {
913	u8 bos = nla_get_u8(nla: tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS]);
914
915	if (bos & ~MPLS_BOS_MASK) {
916	NL_SET_ERR_MSG_ATTR(extack,
917	tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS],
918	"Bottom Of Stack (BOS) must be 0 or 1");
919	return -EINVAL;
920	}
921	lse_val->mpls_bos = bos;
922	lse_mask->mpls_bos = MPLS_BOS_MASK;
923	}
924	if (tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TC]) {
925	u8 tc = nla_get_u8(nla: tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TC]);
926
927	if (tc & ~MPLS_TC_MASK) {
928	NL_SET_ERR_MSG_ATTR(extack,
929	tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TC],
930	"Traffic Class (TC) must be between 0 and 7");
931	return -EINVAL;
932	}
933	lse_val->mpls_tc = tc;
934	lse_mask->mpls_tc = MPLS_TC_MASK;
935	}
936	if (tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL]) {
937	u32 label = nla_get_u32(nla: tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL]);
938
939	if (label & ~MPLS_LABEL_MASK) {
940	NL_SET_ERR_MSG_ATTR(extack,
941	tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL],
942	"Label must be between 0 and 1048575");
943	return -EINVAL;
944	}
945	lse_val->mpls_label = label;
946	lse_mask->mpls_label = MPLS_LABEL_MASK;
947	}
948
949	return 0;
950	}
951
952	static int fl_set_key_mpls_opts(const struct nlattr *nla_mpls_opts,
953	struct flow_dissector_key_mpls *key_val,
954	struct flow_dissector_key_mpls *key_mask,
955	struct netlink_ext_ack *extack)
956	{
957	struct nlattr *nla_lse;
958	int rem;
959	int err;
960
961	if (!(nla_mpls_opts->nla_type & NLA_F_NESTED)) {
962	NL_SET_ERR_MSG_ATTR(extack, nla_mpls_opts,
963	"NLA_F_NESTED is missing");
964	return -EINVAL;
965	}
966
967	nla_for_each_nested(nla_lse, nla_mpls_opts, rem) {
968	if (nla_type(nla: nla_lse) != TCA_FLOWER_KEY_MPLS_OPTS_LSE) {
969	NL_SET_ERR_MSG_ATTR(extack, nla_lse,
970	"Invalid MPLS option type");
971	return -EINVAL;
972	}
973
974	err = fl_set_key_mpls_lse(nla_lse, key_val, key_mask, extack);
975	if (err < 0)
976	return err;
977	}
978	if (rem) {
979	NL_SET_ERR_MSG(extack,
980	"Bytes leftover after parsing MPLS options");
981	return -EINVAL;
982	}
983
984	return 0;
985	}
986
987	static int fl_set_key_mpls(struct nlattr **tb,
988	struct flow_dissector_key_mpls *key_val,
989	struct flow_dissector_key_mpls *key_mask,
990	struct netlink_ext_ack *extack)
991	{
992	struct flow_dissector_mpls_lse *lse_mask;
993	struct flow_dissector_mpls_lse *lse_val;
994
995	if (tb[TCA_FLOWER_KEY_MPLS_OPTS]) {
996	if (tb[TCA_FLOWER_KEY_MPLS_TTL] ||
997	tb[TCA_FLOWER_KEY_MPLS_BOS] ||
998	tb[TCA_FLOWER_KEY_MPLS_TC] ||
999	tb[TCA_FLOWER_KEY_MPLS_LABEL]) {
1000	NL_SET_ERR_MSG_ATTR(extack,
1001	tb[TCA_FLOWER_KEY_MPLS_OPTS],
1002	"MPLS label, Traffic Class, Bottom Of Stack and Time To Live must be encapsulated in the MPLS options attribute");
1003	return -EBADMSG;
1004	}
1005
1006	return fl_set_key_mpls_opts(nla_mpls_opts: tb[TCA_FLOWER_KEY_MPLS_OPTS],
1007	key_val, key_mask, extack);
1008	}
1009
1010	lse_val = &key_val->ls[0];
1011	lse_mask = &key_mask->ls[0];
1012
1013	if (tb[TCA_FLOWER_KEY_MPLS_TTL]) {
1014	lse_val->mpls_ttl = nla_get_u8(nla: tb[TCA_FLOWER_KEY_MPLS_TTL]);
1015	lse_mask->mpls_ttl = MPLS_TTL_MASK;
1016	dissector_set_mpls_lse(mpls: key_val, lse_index: 0);
1017	dissector_set_mpls_lse(mpls: key_mask, lse_index: 0);
1018	}
1019	if (tb[TCA_FLOWER_KEY_MPLS_BOS]) {
1020	u8 bos = nla_get_u8(nla: tb[TCA_FLOWER_KEY_MPLS_BOS]);
1021
1022	if (bos & ~MPLS_BOS_MASK) {
1023	NL_SET_ERR_MSG_ATTR(extack,
1024	tb[TCA_FLOWER_KEY_MPLS_BOS],
1025	"Bottom Of Stack (BOS) must be 0 or 1");
1026	return -EINVAL;
1027	}
1028	lse_val->mpls_bos = bos;
1029	lse_mask->mpls_bos = MPLS_BOS_MASK;
1030	dissector_set_mpls_lse(mpls: key_val, lse_index: 0);
1031	dissector_set_mpls_lse(mpls: key_mask, lse_index: 0);
1032	}
1033	if (tb[TCA_FLOWER_KEY_MPLS_TC]) {
1034	u8 tc = nla_get_u8(nla: tb[TCA_FLOWER_KEY_MPLS_TC]);
1035
1036	if (tc & ~MPLS_TC_MASK) {
1037	NL_SET_ERR_MSG_ATTR(extack,
1038	tb[TCA_FLOWER_KEY_MPLS_TC],
1039	"Traffic Class (TC) must be between 0 and 7");
1040	return -EINVAL;
1041	}
1042	lse_val->mpls_tc = tc;
1043	lse_mask->mpls_tc = MPLS_TC_MASK;
1044	dissector_set_mpls_lse(mpls: key_val, lse_index: 0);
1045	dissector_set_mpls_lse(mpls: key_mask, lse_index: 0);
1046	}
1047	if (tb[TCA_FLOWER_KEY_MPLS_LABEL]) {
1048	u32 label = nla_get_u32(nla: tb[TCA_FLOWER_KEY_MPLS_LABEL]);
1049
1050	if (label & ~MPLS_LABEL_MASK) {
1051	NL_SET_ERR_MSG_ATTR(extack,
1052	tb[TCA_FLOWER_KEY_MPLS_LABEL],
1053	"Label must be between 0 and 1048575");
1054	return -EINVAL;
1055	}
1056	lse_val->mpls_label = label;
1057	lse_mask->mpls_label = MPLS_LABEL_MASK;
1058	dissector_set_mpls_lse(mpls: key_val, lse_index: 0);
1059	dissector_set_mpls_lse(mpls: key_mask, lse_index: 0);
1060	}
1061	return 0;
1062	}
1063
1064	static void fl_set_key_vlan(struct nlattr **tb,
1065	__be16 ethertype,
1066	int vlan_id_key, int vlan_prio_key,
1067	int vlan_next_eth_type_key,
1068	struct flow_dissector_key_vlan *key_val,
1069	struct flow_dissector_key_vlan *key_mask)
1070	{
1071	#define VLAN_PRIORITY_MASK 0x7
1072
1073	if (tb[vlan_id_key]) {
1074	key_val->vlan_id =
1075	nla_get_u16(nla: tb[vlan_id_key]) & VLAN_VID_MASK;
1076	key_mask->vlan_id = VLAN_VID_MASK;
1077	}
1078	if (tb[vlan_prio_key]) {
1079	key_val->vlan_priority =
1080	nla_get_u8(nla: tb[vlan_prio_key]) &
1081	VLAN_PRIORITY_MASK;
1082	key_mask->vlan_priority = VLAN_PRIORITY_MASK;
1083	}
1084	if (ethertype) {
1085	key_val->vlan_tpid = ethertype;
1086	key_mask->vlan_tpid = cpu_to_be16(~0);
1087	}
1088	if (tb[vlan_next_eth_type_key]) {
1089	key_val->vlan_eth_type =
1090	nla_get_be16(nla: tb[vlan_next_eth_type_key]);
1091	key_mask->vlan_eth_type = cpu_to_be16(~0);
1092	}
1093	}
1094
1095	static void fl_set_key_pppoe(struct nlattr **tb,
1096	struct flow_dissector_key_pppoe *key_val,
1097	struct flow_dissector_key_pppoe *key_mask,
1098	struct fl_flow_key *key,
1099	struct fl_flow_key *mask)
1100	{
1101	/* key_val::type must be set to ETH_P_PPP_SES
1102	* because ETH_P_PPP_SES was stored in basic.n_proto
1103	* which might get overwritten by ppp_proto
1104	* or might be set to 0, the role of key_val::type
1105	* is similar to vlan_key::tpid
1106	*/
1107	key_val->type = htons(ETH_P_PPP_SES);
1108	key_mask->type = cpu_to_be16(~0);
1109
1110	if (tb[TCA_FLOWER_KEY_PPPOE_SID]) {
1111	key_val->session_id =
1112	nla_get_be16(nla: tb[TCA_FLOWER_KEY_PPPOE_SID]);
1113	key_mask->session_id = cpu_to_be16(~0);
1114	}
1115	if (tb[TCA_FLOWER_KEY_PPP_PROTO]) {
1116	key_val->ppp_proto =
1117	nla_get_be16(nla: tb[TCA_FLOWER_KEY_PPP_PROTO]);
1118	key_mask->ppp_proto = cpu_to_be16(~0);
1119
1120	if (key_val->ppp_proto == htons(PPP_IP)) {
1121	key->basic.n_proto = htons(ETH_P_IP);
1122	mask->basic.n_proto = cpu_to_be16(~0);
1123	} else if (key_val->ppp_proto == htons(PPP_IPV6)) {
1124	key->basic.n_proto = htons(ETH_P_IPV6);
1125	mask->basic.n_proto = cpu_to_be16(~0);
1126	} else if (key_val->ppp_proto == htons(PPP_MPLS_UC)) {
1127	key->basic.n_proto = htons(ETH_P_MPLS_UC);
1128	mask->basic.n_proto = cpu_to_be16(~0);
1129	} else if (key_val->ppp_proto == htons(PPP_MPLS_MC)) {
1130	key->basic.n_proto = htons(ETH_P_MPLS_MC);
1131	mask->basic.n_proto = cpu_to_be16(~0);
1132	}
1133	} else {
1134	key->basic.n_proto = 0;
1135	mask->basic.n_proto = cpu_to_be16(0);
1136	}
1137	}
1138
1139	static void fl_set_key_flag(u32 flower_key, u32 flower_mask,
1140	u32 *dissector_key, u32 *dissector_mask,
1141	u32 flower_flag_bit, u32 dissector_flag_bit)
1142	{
1143	if (flower_mask & flower_flag_bit) {
1144	*dissector_mask |= dissector_flag_bit;
1145	if (flower_key & flower_flag_bit)
1146	*dissector_key |= dissector_flag_bit;
1147	}
1148	}
1149
1150	static int fl_set_key_flags(struct nlattr **tb, u32 *flags_key,
1151	u32 *flags_mask, struct netlink_ext_ack *extack)
1152	{
1153	u32 key, mask;
1154
1155	/* mask is mandatory for flags */
1156	if (!tb[TCA_FLOWER_KEY_FLAGS_MASK]) {
1157	NL_SET_ERR_MSG(extack, "Missing flags mask");
1158	return -EINVAL;
1159	}
1160
1161	key = be32_to_cpu(nla_get_be32(tb[TCA_FLOWER_KEY_FLAGS]));
1162	mask = be32_to_cpu(nla_get_be32(tb[TCA_FLOWER_KEY_FLAGS_MASK]));
1163
1164	*flags_key = 0;
1165	*flags_mask = 0;
1166
1167	fl_set_key_flag(flower_key: key, flower_mask: mask, dissector_key: flags_key, dissector_mask: flags_mask,
1168	flower_flag_bit: TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
1169	fl_set_key_flag(flower_key: key, flower_mask: mask, dissector_key: flags_key, dissector_mask: flags_mask,
1170	flower_flag_bit: TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST,
1171	FLOW_DIS_FIRST_FRAG);
1172
1173	return 0;
1174	}
1175
1176	static void fl_set_key_ip(struct nlattr **tb, bool encap,
1177	struct flow_dissector_key_ip *key,
1178	struct flow_dissector_key_ip *mask)
1179	{
1180	int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS;
1181	int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL;
1182	int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK;
1183	int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK;
1184
1185	fl_set_key_val(tb, val: &key->tos, val_type: tos_key, mask: &mask->tos, mask_type: tos_mask, len: sizeof(key->tos));
1186	fl_set_key_val(tb, val: &key->ttl, val_type: ttl_key, mask: &mask->ttl, mask_type: ttl_mask, len: sizeof(key->ttl));
1187	}
1188
1189	static int fl_set_geneve_opt(const struct nlattr *nla, struct fl_flow_key *key,
1190	int depth, int option_len,
1191	struct netlink_ext_ack *extack)
1192	{
1193	struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1];
1194	struct nlattr *class = NULL, *type = NULL, *data = NULL;
1195	struct geneve_opt *opt;
1196	int err, data_len = 0;
1197
1198	if (option_len > sizeof(struct geneve_opt))
1199	data_len = option_len - sizeof(struct geneve_opt);
1200
1201	if (key->enc_opts.len > FLOW_DIS_TUN_OPTS_MAX - 4)
1202	return -ERANGE;
1203
1204	opt = (struct geneve_opt *)&key->enc_opts.data[key->enc_opts.len];
1205	memset(opt, 0xff, option_len);
1206	opt->length = data_len / 4;
1207	opt->r1 = 0;
1208	opt->r2 = 0;
1209	opt->r3 = 0;
1210
1211	/* If no mask has been prodived we assume an exact match. */
1212	if (!depth)
1213	return sizeof(struct geneve_opt) + data_len;
1214
1215	if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_GENEVE) {
1216	NL_SET_ERR_MSG(extack, "Non-geneve option type for mask");
1217	return -EINVAL;
1218	}
1219
1220	err = nla_parse_nested_deprecated(tb,
1221	TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX,
1222	nla, policy: geneve_opt_policy, extack);
1223	if (err < 0)
1224	return err;
1225
1226	/* We are not allowed to omit any of CLASS, TYPE or DATA
1227	* fields from the key.
1228	*/
1229	if (!option_len &&
1230	(!tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] ||
1231	!tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] ||
1232	!tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA])) {
1233	NL_SET_ERR_MSG(extack, "Missing tunnel key geneve option class, type or data");
1234	return -EINVAL;
1235	}
1236
1237	/* Omitting any of CLASS, TYPE or DATA fields is allowed
1238	* for the mask.
1239	*/
1240	if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]) {
1241	int new_len = key->enc_opts.len;
1242
1243	data = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA];
1244	data_len = nla_len(nla: data);
1245	if (data_len < 4) {
1246	NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is less than 4 bytes long");
1247	return -ERANGE;
1248	}
1249	if (data_len % 4) {
1250	NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is not a multiple of 4 bytes long");
1251	return -ERANGE;
1252	}
1253
1254	new_len += sizeof(struct geneve_opt) + data_len;
1255	BUILD_BUG_ON(FLOW_DIS_TUN_OPTS_MAX != IP_TUNNEL_OPTS_MAX);
1256	if (new_len > FLOW_DIS_TUN_OPTS_MAX) {
1257	NL_SET_ERR_MSG(extack, "Tunnel options exceeds max size");
1258	return -ERANGE;
1259	}
1260	opt->length = data_len / 4;
1261	memcpy(opt->opt_data, nla_data(data), data_len);
1262	}
1263
1264	if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]) {
1265	class = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS];
1266	opt->opt_class = nla_get_be16(nla: class);
1267	}
1268
1269	if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]) {
1270	type = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE];
1271	opt->type = nla_get_u8(nla: type);
1272	}
1273
1274	return sizeof(struct geneve_opt) + data_len;
1275	}
1276
1277	static int fl_set_vxlan_opt(const struct nlattr *nla, struct fl_flow_key *key,
1278	int depth, int option_len,
1279	struct netlink_ext_ack *extack)
1280	{
1281	struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_MAX + 1];
1282	struct vxlan_metadata *md;
1283	int err;
1284
1285	md = (struct vxlan_metadata *)&key->enc_opts.data[key->enc_opts.len];
1286	memset(md, 0xff, sizeof(*md));
1287
1288	if (!depth)
1289	return sizeof(*md);
1290
1291	if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_VXLAN) {
1292	NL_SET_ERR_MSG(extack, "Non-vxlan option type for mask");
1293	return -EINVAL;
1294	}
1295
1296	err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_VXLAN_MAX, nla,
1297	policy: vxlan_opt_policy, extack);
1298	if (err < 0)
1299	return err;
1300
1301	if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP]) {
1302	NL_SET_ERR_MSG(extack, "Missing tunnel key vxlan option gbp");
1303	return -EINVAL;
1304	}
1305
1306	if (tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP]) {
1307	md->gbp = nla_get_u32(nla: tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP]);
1308	md->gbp &= VXLAN_GBP_MASK;
1309	}
1310
1311	return sizeof(*md);
1312	}
1313
1314	static int fl_set_erspan_opt(const struct nlattr *nla, struct fl_flow_key *key,
1315	int depth, int option_len,
1316	struct netlink_ext_ack *extack)
1317	{
1318	struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX + 1];
1319	struct erspan_metadata *md;
1320	int err;
1321
1322	md = (struct erspan_metadata *)&key->enc_opts.data[key->enc_opts.len];
1323	memset(md, 0xff, sizeof(*md));
1324	md->version = 1;
1325
1326	if (!depth)
1327	return sizeof(*md);
1328
1329	if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_ERSPAN) {
1330	NL_SET_ERR_MSG(extack, "Non-erspan option type for mask");
1331	return -EINVAL;
1332	}
1333
1334	err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX, nla,
1335	policy: erspan_opt_policy, extack);
1336	if (err < 0)
1337	return err;
1338
1339	if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER]) {
1340	NL_SET_ERR_MSG(extack, "Missing tunnel key erspan option ver");
1341	return -EINVAL;
1342	}
1343
1344	if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER])
1345	md->version = nla_get_u8(nla: tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER]);
1346
1347	if (md->version == 1) {
1348	if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX]) {
1349	NL_SET_ERR_MSG(extack, "Missing tunnel key erspan option index");
1350	return -EINVAL;
1351	}
1352	if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX]) {
1353	nla = tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX];
1354	memset(&md->u, 0x00, sizeof(md->u));
1355	md->u.index = nla_get_be32(nla);
1356	}
1357	} else if (md->version == 2) {
1358	if (!option_len && (!tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR] ||
1359	!tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID])) {
1360	NL_SET_ERR_MSG(extack, "Missing tunnel key erspan option dir or hwid");
1361	return -EINVAL;
1362	}
1363	if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR]) {
1364	nla = tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR];
1365	md->u.md2.dir = nla_get_u8(nla);
1366	}
1367	if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID]) {
1368	nla = tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID];
1369	set_hwid(md2: &md->u.md2, hwid: nla_get_u8(nla));
1370	}
1371	} else {
1372	NL_SET_ERR_MSG(extack, "Tunnel key erspan option ver is incorrect");
1373	return -EINVAL;
1374	}
1375
1376	return sizeof(*md);
1377	}
1378
1379	static int fl_set_gtp_opt(const struct nlattr *nla, struct fl_flow_key *key,
1380	int depth, int option_len,
1381	struct netlink_ext_ack *extack)
1382	{
1383	struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_GTP_MAX + 1];
1384	struct gtp_pdu_session_info *sinfo;
1385	u8 len = key->enc_opts.len;
1386	int err;
1387
1388	sinfo = (struct gtp_pdu_session_info *)&key->enc_opts.data[len];
1389	memset(sinfo, 0xff, option_len);
1390
1391	if (!depth)
1392	return sizeof(*sinfo);
1393
1394	if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_GTP) {
1395	NL_SET_ERR_MSG_MOD(extack, "Non-gtp option type for mask");
1396	return -EINVAL;
1397	}
1398
1399	err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_GTP_MAX, nla,
1400	policy: gtp_opt_policy, extack);
1401	if (err < 0)
1402	return err;
1403
1404	if (!option_len &&
1405	(!tb[TCA_FLOWER_KEY_ENC_OPT_GTP_PDU_TYPE] ||
1406	!tb[TCA_FLOWER_KEY_ENC_OPT_GTP_QFI])) {
1407	NL_SET_ERR_MSG_MOD(extack,
1408	"Missing tunnel key gtp option pdu type or qfi");
1409	return -EINVAL;
1410	}
1411
1412	if (tb[TCA_FLOWER_KEY_ENC_OPT_GTP_PDU_TYPE])
1413	sinfo->pdu_type =
1414	nla_get_u8(nla: tb[TCA_FLOWER_KEY_ENC_OPT_GTP_PDU_TYPE]);
1415
1416	if (tb[TCA_FLOWER_KEY_ENC_OPT_GTP_QFI])
1417	sinfo->qfi = nla_get_u8(nla: tb[TCA_FLOWER_KEY_ENC_OPT_GTP_QFI]);
1418
1419	return sizeof(*sinfo);
1420	}
1421
1422	static int fl_set_enc_opt(struct nlattr **tb, struct fl_flow_key *key,
1423	struct fl_flow_key *mask,
1424	struct netlink_ext_ack *extack)
1425	{
1426	const struct nlattr *nla_enc_key, *nla_opt_key, *nla_opt_msk = NULL;
1427	int err, option_len, key_depth, msk_depth = 0;
1428
1429	err = nla_validate_nested_deprecated(start: tb[TCA_FLOWER_KEY_ENC_OPTS],
1430	TCA_FLOWER_KEY_ENC_OPTS_MAX,
1431	policy: enc_opts_policy, extack);
1432	if (err)
1433	return err;
1434
1435	nla_enc_key = nla_data(nla: tb[TCA_FLOWER_KEY_ENC_OPTS]);
1436
1437	if (tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]) {
1438	err = nla_validate_nested_deprecated(start: tb[TCA_FLOWER_KEY_ENC_OPTS_MASK],
1439	TCA_FLOWER_KEY_ENC_OPTS_MAX,
1440	policy: enc_opts_policy, extack);
1441	if (err)
1442	return err;
1443
1444	nla_opt_msk = nla_data(nla: tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]);
1445	msk_depth = nla_len(nla: tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]);
1446	if (!nla_ok(nla: nla_opt_msk, remaining: msk_depth)) {
1447	NL_SET_ERR_MSG(extack, "Invalid nested attribute for masks");
1448	return -EINVAL;
1449	}
1450	}
1451
1452	nla_for_each_attr(nla_opt_key, nla_enc_key,
1453	nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS]), key_depth) {
1454	switch (nla_type(nla: nla_opt_key)) {
1455	case TCA_FLOWER_KEY_ENC_OPTS_GENEVE:
1456	if (key->enc_opts.dst_opt_type &&
1457	key->enc_opts.dst_opt_type != TUNNEL_GENEVE_OPT) {
1458	NL_SET_ERR_MSG(extack, "Duplicate type for geneve options");
1459	return -EINVAL;
1460	}
1461	option_len = 0;
1462	key->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT;
1463	option_len = fl_set_geneve_opt(nla: nla_opt_key, key,
1464	depth: key_depth, option_len,
1465	extack);
1466	if (option_len < 0)
1467	return option_len;
1468
1469	key->enc_opts.len += option_len;
1470	/* At the same time we need to parse through the mask
1471	* in order to verify exact and mask attribute lengths.
1472	*/
1473	mask->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT;
1474	option_len = fl_set_geneve_opt(nla: nla_opt_msk, key: mask,
1475	depth: msk_depth, option_len,
1476	extack);
1477	if (option_len < 0)
1478	return option_len;
1479
1480	mask->enc_opts.len += option_len;
1481	if (key->enc_opts.len != mask->enc_opts.len) {
1482	NL_SET_ERR_MSG(extack, "Key and mask miss aligned");
1483	return -EINVAL;
1484	}
1485	break;
1486	case TCA_FLOWER_KEY_ENC_OPTS_VXLAN:
1487	if (key->enc_opts.dst_opt_type) {
1488	NL_SET_ERR_MSG(extack, "Duplicate type for vxlan options");
1489	return -EINVAL;
1490	}
1491	option_len = 0;
1492	key->enc_opts.dst_opt_type = TUNNEL_VXLAN_OPT;
1493	option_len = fl_set_vxlan_opt(nla: nla_opt_key, key,
1494	depth: key_depth, option_len,
1495	extack);
1496	if (option_len < 0)
1497	return option_len;
1498
1499	key->enc_opts.len += option_len;
1500	/* At the same time we need to parse through the mask
1501	* in order to verify exact and mask attribute lengths.
1502	*/
1503	mask->enc_opts.dst_opt_type = TUNNEL_VXLAN_OPT;
1504	option_len = fl_set_vxlan_opt(nla: nla_opt_msk, key: mask,
1505	depth: msk_depth, option_len,
1506	extack);
1507	if (option_len < 0)
1508	return option_len;
1509
1510	mask->enc_opts.len += option_len;
1511	if (key->enc_opts.len != mask->enc_opts.len) {
1512	NL_SET_ERR_MSG(extack, "Key and mask miss aligned");
1513	return -EINVAL;
1514	}
1515	break;
1516	case TCA_FLOWER_KEY_ENC_OPTS_ERSPAN:
1517	if (key->enc_opts.dst_opt_type) {
1518	NL_SET_ERR_MSG(extack, "Duplicate type for erspan options");
1519	return -EINVAL;
1520	}
1521	option_len = 0;
1522	key->enc_opts.dst_opt_type = TUNNEL_ERSPAN_OPT;
1523	option_len = fl_set_erspan_opt(nla: nla_opt_key, key,
1524	depth: key_depth, option_len,
1525	extack);
1526	if (option_len < 0)
1527	return option_len;
1528
1529	key->enc_opts.len += option_len;
1530	/* At the same time we need to parse through the mask
1531	* in order to verify exact and mask attribute lengths.
1532	*/
1533	mask->enc_opts.dst_opt_type = TUNNEL_ERSPAN_OPT;
1534	option_len = fl_set_erspan_opt(nla: nla_opt_msk, key: mask,
1535	depth: msk_depth, option_len,
1536	extack);
1537	if (option_len < 0)
1538	return option_len;
1539
1540	mask->enc_opts.len += option_len;
1541	if (key->enc_opts.len != mask->enc_opts.len) {
1542	NL_SET_ERR_MSG(extack, "Key and mask miss aligned");
1543	return -EINVAL;
1544	}
1545	break;
1546	case TCA_FLOWER_KEY_ENC_OPTS_GTP:
1547	if (key->enc_opts.dst_opt_type) {
1548	NL_SET_ERR_MSG_MOD(extack,
1549	"Duplicate type for gtp options");
1550	return -EINVAL;
1551	}
1552	option_len = 0;
1553	key->enc_opts.dst_opt_type = TUNNEL_GTP_OPT;
1554	option_len = fl_set_gtp_opt(nla: nla_opt_key, key,
1555	depth: key_depth, option_len,
1556	extack);
1557	if (option_len < 0)
1558	return option_len;
1559
1560	key->enc_opts.len += option_len;
1561	/* At the same time we need to parse through the mask
1562	* in order to verify exact and mask attribute lengths.
1563	*/
1564	mask->enc_opts.dst_opt_type = TUNNEL_GTP_OPT;
1565	option_len = fl_set_gtp_opt(nla: nla_opt_msk, key: mask,
1566	depth: msk_depth, option_len,
1567	extack);
1568	if (option_len < 0)
1569	return option_len;
1570
1571	mask->enc_opts.len += option_len;
1572	if (key->enc_opts.len != mask->enc_opts.len) {
1573	NL_SET_ERR_MSG_MOD(extack,
1574	"Key and mask miss aligned");
1575	return -EINVAL;
1576	}
1577	break;
1578	default:
1579	NL_SET_ERR_MSG(extack, "Unknown tunnel option type");
1580	return -EINVAL;
1581	}
1582
1583	if (!msk_depth)
1584	continue;
1585
1586	if (!nla_ok(nla: nla_opt_msk, remaining: msk_depth)) {
1587	NL_SET_ERR_MSG(extack, "A mask attribute is invalid");
1588	return -EINVAL;
1589	}
1590	nla_opt_msk = nla_next(nla: nla_opt_msk, remaining: &msk_depth);
1591	}
1592
1593	return 0;
1594	}
1595
1596	static int fl_validate_ct_state(u16 state, struct nlattr *tb,
1597	struct netlink_ext_ack *extack)
1598	{
1599	if (state && !(state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED)) {
1600	NL_SET_ERR_MSG_ATTR(extack, tb,
1601	"no trk, so no other flag can be set");
1602	return -EINVAL;
1603	}
1604
1605	if (state & TCA_FLOWER_KEY_CT_FLAGS_NEW &&
1606	state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED) {
1607	NL_SET_ERR_MSG_ATTR(extack, tb,
1608	"new and est are mutually exclusive");
1609	return -EINVAL;
1610	}
1611
1612	if (state & TCA_FLOWER_KEY_CT_FLAGS_INVALID &&
1613	state & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
1614	TCA_FLOWER_KEY_CT_FLAGS_INVALID)) {
1615	NL_SET_ERR_MSG_ATTR(extack, tb,
1616	"when inv is set, only trk may be set");
1617	return -EINVAL;
1618	}
1619
1620	if (state & TCA_FLOWER_KEY_CT_FLAGS_NEW &&
1621	state & TCA_FLOWER_KEY_CT_FLAGS_REPLY) {
1622	NL_SET_ERR_MSG_ATTR(extack, tb,
1623	"new and rpl are mutually exclusive");
1624	return -EINVAL;
1625	}
1626
1627	return 0;
1628	}
1629
1630	static int fl_set_key_ct(struct nlattr **tb,
1631	struct flow_dissector_key_ct *key,
1632	struct flow_dissector_key_ct *mask,
1633	struct netlink_ext_ack *extack)
1634	{
1635	if (tb[TCA_FLOWER_KEY_CT_STATE]) {
1636	int err;
1637
1638	if (!IS_ENABLED(CONFIG_NF_CONNTRACK)) {
1639	NL_SET_ERR_MSG(extack, "Conntrack isn't enabled");
1640	return -EOPNOTSUPP;
1641	}
1642	fl_set_key_val(tb, val: &key->ct_state, val_type: TCA_FLOWER_KEY_CT_STATE,
1643	mask: &mask->ct_state, mask_type: TCA_FLOWER_KEY_CT_STATE_MASK,
1644	len: sizeof(key->ct_state));
1645
1646	err = fl_validate_ct_state(state: key->ct_state & mask->ct_state,
1647	tb: tb[TCA_FLOWER_KEY_CT_STATE_MASK],
1648	extack);
1649	if (err)
1650	return err;
1651
1652	}
1653	if (tb[TCA_FLOWER_KEY_CT_ZONE]) {
1654	if (!IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) {
1655	NL_SET_ERR_MSG(extack, "Conntrack zones isn't enabled");
1656	return -EOPNOTSUPP;
1657	}
1658	fl_set_key_val(tb, val: &key->ct_zone, val_type: TCA_FLOWER_KEY_CT_ZONE,
1659	mask: &mask->ct_zone, mask_type: TCA_FLOWER_KEY_CT_ZONE_MASK,
1660	len: sizeof(key->ct_zone));
1661	}
1662	if (tb[TCA_FLOWER_KEY_CT_MARK]) {
1663	if (!IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)) {
1664	NL_SET_ERR_MSG(extack, "Conntrack mark isn't enabled");
1665	return -EOPNOTSUPP;
1666	}
1667	fl_set_key_val(tb, val: &key->ct_mark, val_type: TCA_FLOWER_KEY_CT_MARK,
1668	mask: &mask->ct_mark, mask_type: TCA_FLOWER_KEY_CT_MARK_MASK,
1669	len: sizeof(key->ct_mark));
1670	}
1671	if (tb[TCA_FLOWER_KEY_CT_LABELS]) {
1672	if (!IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)) {
1673	NL_SET_ERR_MSG(extack, "Conntrack labels aren't enabled");
1674	return -EOPNOTSUPP;
1675	}
1676	fl_set_key_val(tb, val: key->ct_labels, val_type: TCA_FLOWER_KEY_CT_LABELS,
1677	mask: mask->ct_labels, mask_type: TCA_FLOWER_KEY_CT_LABELS_MASK,
1678	len: sizeof(key->ct_labels));
1679	}
1680
1681	return 0;
1682	}
1683
1684	static bool is_vlan_key(struct nlattr *tb, __be16 *ethertype,
1685	struct fl_flow_key *key, struct fl_flow_key *mask,
1686	int vthresh)
1687	{
1688	const bool good_num_of_vlans = key->num_of_vlans.num_of_vlans > vthresh;
1689
1690	if (!tb) {
1691	*ethertype = 0;
1692	return good_num_of_vlans;
1693	}
1694
1695	*ethertype = nla_get_be16(nla: tb);
1696	if (good_num_of_vlans || eth_type_vlan(ethertype: *ethertype))
1697	return true;
1698
1699	key->basic.n_proto = *ethertype;
1700	mask->basic.n_proto = cpu_to_be16(~0);
1701	return false;
1702	}
1703
1704	static void fl_set_key_cfm_md_level(struct nlattr **tb,
1705	struct fl_flow_key *key,
1706	struct fl_flow_key *mask,
1707	struct netlink_ext_ack *extack)
1708	{
1709	u8 level;
1710
1711	if (!tb[TCA_FLOWER_KEY_CFM_MD_LEVEL])
1712	return;
1713
1714	level = nla_get_u8(nla: tb[TCA_FLOWER_KEY_CFM_MD_LEVEL]);
1715	key->cfm.mdl_ver = FIELD_PREP(FLOW_DIS_CFM_MDL_MASK, level);
1716	mask->cfm.mdl_ver = FLOW_DIS_CFM_MDL_MASK;
1717	}
1718
1719	static void fl_set_key_cfm_opcode(struct nlattr **tb,
1720	struct fl_flow_key *key,
1721	struct fl_flow_key *mask,
1722	struct netlink_ext_ack *extack)
1723	{
1724	fl_set_key_val(tb, val: &key->cfm.opcode, val_type: TCA_FLOWER_KEY_CFM_OPCODE,
1725	mask: &mask->cfm.opcode, mask_type: TCA_FLOWER_UNSPEC,
1726	len: sizeof(key->cfm.opcode));
1727	}
1728
1729	static int fl_set_key_cfm(struct nlattr **tb,
1730	struct fl_flow_key *key,
1731	struct fl_flow_key *mask,
1732	struct netlink_ext_ack *extack)
1733	{
1734	struct nlattr *nla_cfm_opt[TCA_FLOWER_KEY_CFM_OPT_MAX + 1];
1735	int err;
1736
1737	if (!tb[TCA_FLOWER_KEY_CFM])
1738	return 0;
1739
1740	err = nla_parse_nested(tb: nla_cfm_opt, TCA_FLOWER_KEY_CFM_OPT_MAX,
1741	nla: tb[TCA_FLOWER_KEY_CFM], policy: cfm_opt_policy, extack);
1742	if (err < 0)
1743	return err;
1744
1745	fl_set_key_cfm_opcode(tb: nla_cfm_opt, key, mask, extack);
1746	fl_set_key_cfm_md_level(tb: nla_cfm_opt, key, mask, extack);
1747
1748	return 0;
1749	}
1750
1751	static int fl_set_key(struct net *net, struct nlattr **tb,
1752	struct fl_flow_key *key, struct fl_flow_key *mask,
1753	struct netlink_ext_ack *extack)
1754	{
1755	__be16 ethertype;
1756	int ret = 0;
1757
1758	if (tb[TCA_FLOWER_INDEV]) {
1759	int err = tcf_change_indev(net, indev_tlv: tb[TCA_FLOWER_INDEV], extack);
1760	if (err < 0)
1761	return err;
1762	key->meta.ingress_ifindex = err;
1763	mask->meta.ingress_ifindex = 0xffffffff;
1764	}
1765
1766	fl_set_key_val(tb, val: &key->meta.l2_miss, val_type: TCA_FLOWER_L2_MISS,
1767	mask: &mask->meta.l2_miss, mask_type: TCA_FLOWER_UNSPEC,
1768	len: sizeof(key->meta.l2_miss));
1769
1770	fl_set_key_val(tb, val: key->eth.dst, val_type: TCA_FLOWER_KEY_ETH_DST,
1771	mask: mask->eth.dst, mask_type: TCA_FLOWER_KEY_ETH_DST_MASK,
1772	len: sizeof(key->eth.dst));
1773	fl_set_key_val(tb, val: key->eth.src, val_type: TCA_FLOWER_KEY_ETH_SRC,
1774	mask: mask->eth.src, mask_type: TCA_FLOWER_KEY_ETH_SRC_MASK,
1775	len: sizeof(key->eth.src));
1776	fl_set_key_val(tb, val: &key->num_of_vlans,
1777	val_type: TCA_FLOWER_KEY_NUM_OF_VLANS,
1778	mask: &mask->num_of_vlans,
1779	mask_type: TCA_FLOWER_UNSPEC,
1780	len: sizeof(key->num_of_vlans));
1781
1782	if (is_vlan_key(tb: tb[TCA_FLOWER_KEY_ETH_TYPE], ethertype: &ethertype, key, mask, vthresh: 0)) {
1783	fl_set_key_vlan(tb, ethertype, vlan_id_key: TCA_FLOWER_KEY_VLAN_ID,
1784	vlan_prio_key: TCA_FLOWER_KEY_VLAN_PRIO,
1785	vlan_next_eth_type_key: TCA_FLOWER_KEY_VLAN_ETH_TYPE,
1786	key_val: &key->vlan, key_mask: &mask->vlan);
1787
1788	if (is_vlan_key(tb: tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE],
1789	ethertype: &ethertype, key, mask, vthresh: 1)) {
1790	fl_set_key_vlan(tb, ethertype,
1791	vlan_id_key: TCA_FLOWER_KEY_CVLAN_ID,
1792	vlan_prio_key: TCA_FLOWER_KEY_CVLAN_PRIO,
1793	vlan_next_eth_type_key: TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
1794	key_val: &key->cvlan, key_mask: &mask->cvlan);
1795	fl_set_key_val(tb, val: &key->basic.n_proto,
1796	val_type: TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
1797	mask: &mask->basic.n_proto,
1798	mask_type: TCA_FLOWER_UNSPEC,
1799	len: sizeof(key->basic.n_proto));
1800	}
1801	}
1802
1803	if (key->basic.n_proto == htons(ETH_P_PPP_SES))
1804	fl_set_key_pppoe(tb, key_val: &key->pppoe, key_mask: &mask->pppoe, key, mask);
1805
1806	if (key->basic.n_proto == htons(ETH_P_IP) ||
1807	key->basic.n_proto == htons(ETH_P_IPV6)) {
1808	fl_set_key_val(tb, val: &key->basic.ip_proto, val_type: TCA_FLOWER_KEY_IP_PROTO,
1809	mask: &mask->basic.ip_proto, mask_type: TCA_FLOWER_UNSPEC,
1810	len: sizeof(key->basic.ip_proto));
1811	fl_set_key_ip(tb, encap: false, key: &key->ip, mask: &mask->ip);
1812	}
1813
1814	if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) {
1815	key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1816	mask->control.addr_type = ~0;
1817	fl_set_key_val(tb, val: &key->ipv4.src, val_type: TCA_FLOWER_KEY_IPV4_SRC,
1818	mask: &mask->ipv4.src, mask_type: TCA_FLOWER_KEY_IPV4_SRC_MASK,
1819	len: sizeof(key->ipv4.src));
1820	fl_set_key_val(tb, val: &key->ipv4.dst, val_type: TCA_FLOWER_KEY_IPV4_DST,
1821	mask: &mask->ipv4.dst, mask_type: TCA_FLOWER_KEY_IPV4_DST_MASK,
1822	len: sizeof(key->ipv4.dst));
1823	} else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) {
1824	key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1825	mask->control.addr_type = ~0;
1826	fl_set_key_val(tb, val: &key->ipv6.src, val_type: TCA_FLOWER_KEY_IPV6_SRC,
1827	mask: &mask->ipv6.src, mask_type: TCA_FLOWER_KEY_IPV6_SRC_MASK,
1828	len: sizeof(key->ipv6.src));
1829	fl_set_key_val(tb, val: &key->ipv6.dst, val_type: TCA_FLOWER_KEY_IPV6_DST,
1830	mask: &mask->ipv6.dst, mask_type: TCA_FLOWER_KEY_IPV6_DST_MASK,
1831	len: sizeof(key->ipv6.dst));
1832	}
1833
1834	if (key->basic.ip_proto == IPPROTO_TCP) {
1835	fl_set_key_val(tb, val: &key->tp.src, val_type: TCA_FLOWER_KEY_TCP_SRC,
1836	mask: &mask->tp.src, mask_type: TCA_FLOWER_KEY_TCP_SRC_MASK,
1837	len: sizeof(key->tp.src));
1838	fl_set_key_val(tb, val: &key->tp.dst, val_type: TCA_FLOWER_KEY_TCP_DST,
1839	mask: &mask->tp.dst, mask_type: TCA_FLOWER_KEY_TCP_DST_MASK,
1840	len: sizeof(key->tp.dst));
1841	fl_set_key_val(tb, val: &key->tcp.flags, val_type: TCA_FLOWER_KEY_TCP_FLAGS,
1842	mask: &mask->tcp.flags, mask_type: TCA_FLOWER_KEY_TCP_FLAGS_MASK,
1843	len: sizeof(key->tcp.flags));
1844	} else if (key->basic.ip_proto == IPPROTO_UDP) {
1845	fl_set_key_val(tb, val: &key->tp.src, val_type: TCA_FLOWER_KEY_UDP_SRC,
1846	mask: &mask->tp.src, mask_type: TCA_FLOWER_KEY_UDP_SRC_MASK,
1847	len: sizeof(key->tp.src));
1848	fl_set_key_val(tb, val: &key->tp.dst, val_type: TCA_FLOWER_KEY_UDP_DST,
1849	mask: &mask->tp.dst, mask_type: TCA_FLOWER_KEY_UDP_DST_MASK,
1850	len: sizeof(key->tp.dst));
1851	} else if (key->basic.ip_proto == IPPROTO_SCTP) {
1852	fl_set_key_val(tb, val: &key->tp.src, val_type: TCA_FLOWER_KEY_SCTP_SRC,
1853	mask: &mask->tp.src, mask_type: TCA_FLOWER_KEY_SCTP_SRC_MASK,
1854	len: sizeof(key->tp.src));
1855	fl_set_key_val(tb, val: &key->tp.dst, val_type: TCA_FLOWER_KEY_SCTP_DST,
1856	mask: &mask->tp.dst, mask_type: TCA_FLOWER_KEY_SCTP_DST_MASK,
1857	len: sizeof(key->tp.dst));
1858	} else if (key->basic.n_proto == htons(ETH_P_IP) &&
1859	key->basic.ip_proto == IPPROTO_ICMP) {
1860	fl_set_key_val(tb, val: &key->icmp.type, val_type: TCA_FLOWER_KEY_ICMPV4_TYPE,
1861	mask: &mask->icmp.type,
1862	mask_type: TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,
1863	len: sizeof(key->icmp.type));
1864	fl_set_key_val(tb, val: &key->icmp.code, val_type: TCA_FLOWER_KEY_ICMPV4_CODE,
1865	mask: &mask->icmp.code,
1866	mask_type: TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
1867	len: sizeof(key->icmp.code));
1868	} else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
1869	key->basic.ip_proto == IPPROTO_ICMPV6) {
1870	fl_set_key_val(tb, val: &key->icmp.type, val_type: TCA_FLOWER_KEY_ICMPV6_TYPE,
1871	mask: &mask->icmp.type,
1872	mask_type: TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,
1873	len: sizeof(key->icmp.type));
1874	fl_set_key_val(tb, val: &key->icmp.code, val_type: TCA_FLOWER_KEY_ICMPV6_CODE,
1875	mask: &mask->icmp.code,
1876	mask_type: TCA_FLOWER_KEY_ICMPV6_CODE_MASK,
1877	len: sizeof(key->icmp.code));
1878	} else if (key->basic.n_proto == htons(ETH_P_MPLS_UC) ||
1879	key->basic.n_proto == htons(ETH_P_MPLS_MC)) {
1880	ret = fl_set_key_mpls(tb, key_val: &key->mpls, key_mask: &mask->mpls, extack);
1881	if (ret)
1882	return ret;
1883	} else if (key->basic.n_proto == htons(ETH_P_ARP) ||
1884	key->basic.n_proto == htons(ETH_P_RARP)) {
1885	fl_set_key_val(tb, val: &key->arp.sip, val_type: TCA_FLOWER_KEY_ARP_SIP,
1886	mask: &mask->arp.sip, mask_type: TCA_FLOWER_KEY_ARP_SIP_MASK,
1887	len: sizeof(key->arp.sip));
1888	fl_set_key_val(tb, val: &key->arp.tip, val_type: TCA_FLOWER_KEY_ARP_TIP,
1889	mask: &mask->arp.tip, mask_type: TCA_FLOWER_KEY_ARP_TIP_MASK,
1890	len: sizeof(key->arp.tip));
1891	fl_set_key_val(tb, val: &key->arp.op, val_type: TCA_FLOWER_KEY_ARP_OP,
1892	mask: &mask->arp.op, mask_type: TCA_FLOWER_KEY_ARP_OP_MASK,
1893	len: sizeof(key->arp.op));
1894	fl_set_key_val(tb, val: key->arp.sha, val_type: TCA_FLOWER_KEY_ARP_SHA,
1895	mask: mask->arp.sha, mask_type: TCA_FLOWER_KEY_ARP_SHA_MASK,
1896	len: sizeof(key->arp.sha));
1897	fl_set_key_val(tb, val: key->arp.tha, val_type: TCA_FLOWER_KEY_ARP_THA,
1898	mask: mask->arp.tha, mask_type: TCA_FLOWER_KEY_ARP_THA_MASK,
1899	len: sizeof(key->arp.tha));
1900	} else if (key->basic.ip_proto == IPPROTO_L2TP) {
1901	fl_set_key_val(tb, val: &key->l2tpv3.session_id,
1902	val_type: TCA_FLOWER_KEY_L2TPV3_SID,
1903	mask: &mask->l2tpv3.session_id, mask_type: TCA_FLOWER_UNSPEC,
1904	len: sizeof(key->l2tpv3.session_id));
1905	} else if (key->basic.n_proto == htons(ETH_P_CFM)) {
1906	ret = fl_set_key_cfm(tb, key, mask, extack);
1907	if (ret)
1908	return ret;
1909	}
1910
1911	if (key->basic.ip_proto == IPPROTO_TCP ||
1912	key->basic.ip_proto == IPPROTO_UDP ||
1913	key->basic.ip_proto == IPPROTO_SCTP) {
1914	ret = fl_set_key_port_range(tb, key, mask, extack);
1915	if (ret)
1916	return ret;
1917	}
1918
1919	if (tb[TCA_FLOWER_KEY_SPI]) {
1920	ret = fl_set_key_spi(tb, key, mask, extack);
1921	if (ret)
1922	return ret;
1923	}
1924
1925	if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] ||
1926	tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) {
1927	key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1928	mask->enc_control.addr_type = ~0;
1929	fl_set_key_val(tb, val: &key->enc_ipv4.src,
1930	val_type: TCA_FLOWER_KEY_ENC_IPV4_SRC,
1931	mask: &mask->enc_ipv4.src,
1932	mask_type: TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
1933	len: sizeof(key->enc_ipv4.src));
1934	fl_set_key_val(tb, val: &key->enc_ipv4.dst,
1935	val_type: TCA_FLOWER_KEY_ENC_IPV4_DST,
1936	mask: &mask->enc_ipv4.dst,
1937	mask_type: TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
1938	len: sizeof(key->enc_ipv4.dst));
1939	}
1940
1941	if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] ||
1942	tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) {
1943	key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1944	mask->enc_control.addr_type = ~0;
1945	fl_set_key_val(tb, val: &key->enc_ipv6.src,
1946	val_type: TCA_FLOWER_KEY_ENC_IPV6_SRC,
1947	mask: &mask->enc_ipv6.src,
1948	mask_type: TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
1949	len: sizeof(key->enc_ipv6.src));
1950	fl_set_key_val(tb, val: &key->enc_ipv6.dst,
1951	val_type: TCA_FLOWER_KEY_ENC_IPV6_DST,
1952	mask: &mask->enc_ipv6.dst,
1953	mask_type: TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
1954	len: sizeof(key->enc_ipv6.dst));
1955	}
1956
1957	fl_set_key_val(tb, val: &key->enc_key_id.keyid, val_type: TCA_FLOWER_KEY_ENC_KEY_ID,
1958	mask: &mask->enc_key_id.keyid, mask_type: TCA_FLOWER_UNSPEC,
1959	len: sizeof(key->enc_key_id.keyid));
1960
1961	fl_set_key_val(tb, val: &key->enc_tp.src, val_type: TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
1962	mask: &mask->enc_tp.src, mask_type: TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
1963	len: sizeof(key->enc_tp.src));
1964
1965	fl_set_key_val(tb, val: &key->enc_tp.dst, val_type: TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
1966	mask: &mask->enc_tp.dst, mask_type: TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
1967	len: sizeof(key->enc_tp.dst));
1968
1969	fl_set_key_ip(tb, encap: true, key: &key->enc_ip, mask: &mask->enc_ip);
1970
1971	fl_set_key_val(tb, val: &key->hash.hash, val_type: TCA_FLOWER_KEY_HASH,
1972	mask: &mask->hash.hash, mask_type: TCA_FLOWER_KEY_HASH_MASK,
1973	len: sizeof(key->hash.hash));
1974
1975	if (tb[TCA_FLOWER_KEY_ENC_OPTS]) {
1976	ret = fl_set_enc_opt(tb, key, mask, extack);
1977	if (ret)
1978	return ret;
1979	}
1980
1981	ret = fl_set_key_ct(tb, key: &key->ct, mask: &mask->ct, extack);
1982	if (ret)
1983	return ret;
1984
1985	if (tb[TCA_FLOWER_KEY_FLAGS])
1986	ret = fl_set_key_flags(tb, flags_key: &key->control.flags,
1987	flags_mask: &mask->control.flags, extack);
1988
1989	return ret;
1990	}
1991
1992	static void fl_mask_copy(struct fl_flow_mask *dst,
1993	struct fl_flow_mask *src)
1994	{
1995	const void *psrc = fl_key_get_start(key: &src->key, mask: src);
1996	void *pdst = fl_key_get_start(key: &dst->key, mask: src);
1997
1998	memcpy(pdst, psrc, fl_mask_range(src));
1999	dst->range = src->range;
2000	}
2001
2002	static const struct rhashtable_params fl_ht_params = {
2003	.key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */
2004	.head_offset = offsetof(struct cls_fl_filter, ht_node),
2005	.automatic_shrinking = true,
2006	};
2007
2008	static int fl_init_mask_hashtable(struct fl_flow_mask *mask)
2009	{
2010	mask->filter_ht_params = fl_ht_params;
2011	mask->filter_ht_params.key_len = fl_mask_range(mask);
2012	mask->filter_ht_params.key_offset += mask->range.start;
2013
2014	return rhashtable_init(ht: &mask->ht, params: &mask->filter_ht_params);
2015	}
2016
2017	#define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member)
2018	#define FL_KEY_MEMBER_SIZE(member) sizeof_field(struct fl_flow_key, member)
2019
2020	#define FL_KEY_IS_MASKED(mask, member) \
2021	memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member), \
2022	0, FL_KEY_MEMBER_SIZE(member)) \
2023
2024	#define FL_KEY_SET(keys, cnt, id, member) \
2025	do { \
2026	keys[cnt].key_id = id; \
2027	keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member); \
2028	cnt++; \
2029	} while(0);
2030
2031	#define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member) \
2032	do { \
2033	if (FL_KEY_IS_MASKED(mask, member)) \
2034	FL_KEY_SET(keys, cnt, id, member); \
2035	} while(0);
2036
2037	static void fl_init_dissector(struct flow_dissector *dissector,
2038	struct fl_flow_key *mask)
2039	{
2040	struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX];
2041	size_t cnt = 0;
2042
2043	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2044	FLOW_DISSECTOR_KEY_META, meta);
2045	FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control);
2046	FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic);
2047	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2048	FLOW_DISSECTOR_KEY_ETH_ADDRS, eth);
2049	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2050	FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
2051	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2052	FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
2053	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2054	FLOW_DISSECTOR_KEY_PORTS, tp);
2055	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2056	FLOW_DISSECTOR_KEY_PORTS_RANGE, tp_range);
2057	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2058	FLOW_DISSECTOR_KEY_IP, ip);
2059	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2060	FLOW_DISSECTOR_KEY_TCP, tcp);
2061	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2062	FLOW_DISSECTOR_KEY_ICMP, icmp);
2063	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2064	FLOW_DISSECTOR_KEY_ARP, arp);
2065	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2066	FLOW_DISSECTOR_KEY_MPLS, mpls);
2067	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2068	FLOW_DISSECTOR_KEY_VLAN, vlan);
2069	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2070	FLOW_DISSECTOR_KEY_CVLAN, cvlan);
2071	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2072	FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
2073	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2074	FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4);
2075	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2076	FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6);
2077	if (FL_KEY_IS_MASKED(mask, enc_ipv4) ||
2078	FL_KEY_IS_MASKED(mask, enc_ipv6))
2079	FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL,
2080	enc_control);
2081	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2082	FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp);
2083	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2084	FLOW_DISSECTOR_KEY_ENC_IP, enc_ip);
2085	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2086	FLOW_DISSECTOR_KEY_ENC_OPTS, enc_opts);
2087	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2088	FLOW_DISSECTOR_KEY_CT, ct);
2089	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2090	FLOW_DISSECTOR_KEY_HASH, hash);
2091	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2092	FLOW_DISSECTOR_KEY_NUM_OF_VLANS, num_of_vlans);
2093	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2094	FLOW_DISSECTOR_KEY_PPPOE, pppoe);
2095	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2096	FLOW_DISSECTOR_KEY_L2TPV3, l2tpv3);
2097	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2098	FLOW_DISSECTOR_KEY_IPSEC, ipsec);
2099	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
2100	FLOW_DISSECTOR_KEY_CFM, cfm);
2101
2102	skb_flow_dissector_init(flow_dissector: dissector, key: keys, key_count: cnt);
2103	}
2104
2105	static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head,
2106	struct fl_flow_mask *mask)
2107	{
2108	struct fl_flow_mask *newmask;
2109	int err;
2110
2111	newmask = kzalloc(size: sizeof(*newmask), GFP_KERNEL);
2112	if (!newmask)
2113	return ERR_PTR(error: -ENOMEM);
2114
2115	fl_mask_copy(dst: newmask, src: mask);
2116
2117	if ((newmask->key.tp_range.tp_min.dst &&
2118	newmask->key.tp_range.tp_max.dst) ||
2119	(newmask->key.tp_range.tp_min.src &&
2120	newmask->key.tp_range.tp_max.src))
2121	newmask->flags |= TCA_FLOWER_MASK_FLAGS_RANGE;
2122
2123	err = fl_init_mask_hashtable(mask: newmask);
2124	if (err)
2125	goto errout_free;
2126
2127	fl_init_dissector(dissector: &newmask->dissector, mask: &newmask->key);
2128
2129	INIT_LIST_HEAD_RCU(list: &newmask->filters);
2130
2131	refcount_set(r: &newmask->refcnt, n: 1);
2132	err = rhashtable_replace_fast(ht: &head->ht, obj_old: &mask->ht_node,
2133	obj_new: &newmask->ht_node, params: mask_ht_params);
2134	if (err)
2135	goto errout_destroy;
2136
2137	spin_lock(lock: &head->masks_lock);
2138	list_add_tail_rcu(new: &newmask->list, head: &head->masks);
2139	spin_unlock(lock: &head->masks_lock);
2140
2141	return newmask;
2142
2143	errout_destroy:
2144	rhashtable_destroy(ht: &newmask->ht);
2145	errout_free:
2146	kfree(objp: newmask);
2147
2148	return ERR_PTR(error: err);
2149	}
2150
2151	static int fl_check_assign_mask(struct cls_fl_head *head,
2152	struct cls_fl_filter *fnew,
2153	struct cls_fl_filter *fold,
2154	struct fl_flow_mask *mask)
2155	{
2156	struct fl_flow_mask *newmask;
2157	int ret = 0;
2158
2159	rcu_read_lock();
2160
2161	/* Insert mask as temporary node to prevent concurrent creation of mask
2162	* with same key. Any concurrent lookups with same key will return
2163	* -EAGAIN because mask's refcnt is zero.
2164	*/
2165	fnew->mask = rhashtable_lookup_get_insert_fast(ht: &head->ht,
2166	obj: &mask->ht_node,
2167	params: mask_ht_params);
2168	if (!fnew->mask) {
2169	rcu_read_unlock();
2170
2171	if (fold) {
2172	ret = -EINVAL;
2173	goto errout_cleanup;
2174	}
2175
2176	newmask = fl_create_new_mask(head, mask);
2177	if (IS_ERR(ptr: newmask)) {
2178	ret = PTR_ERR(ptr: newmask);
2179	goto errout_cleanup;
2180	}
2181
2182	fnew->mask = newmask;
2183	return 0;
2184	} else if (IS_ERR(ptr: fnew->mask)) {
2185	ret = PTR_ERR(ptr: fnew->mask);
2186	} else if (fold && fold->mask != fnew->mask) {
2187	ret = -EINVAL;
2188	} else if (!refcount_inc_not_zero(r: &fnew->mask->refcnt)) {
2189	/* Mask was deleted concurrently, try again */
2190	ret = -EAGAIN;
2191	}
2192	rcu_read_unlock();
2193	return ret;
2194
2195	errout_cleanup:
2196	rhashtable_remove_fast(ht: &head->ht, obj: &mask->ht_node,
2197	params: mask_ht_params);
2198	return ret;
2199	}
2200
2201	static bool fl_needs_tc_skb_ext(const struct fl_flow_key *mask)
2202	{
2203	return mask->meta.l2_miss;
2204	}
2205
2206	static int fl_ht_insert_unique(struct cls_fl_filter *fnew,
2207	struct cls_fl_filter *fold,
2208	bool *in_ht)
2209	{
2210	struct fl_flow_mask *mask = fnew->mask;
2211	int err;
2212
2213	err = rhashtable_lookup_insert_fast(ht: &mask->ht,
2214	obj: &fnew->ht_node,
2215	params: mask->filter_ht_params);
2216	if (err) {
2217	*in_ht = false;
2218	/* It is okay if filter with same key exists when
2219	* overwriting.
2220	*/
2221	return fold && err == -EEXIST ? 0 : err;
2222	}
2223
2224	*in_ht = true;
2225	return 0;
2226	}
2227
2228	static int fl_change(struct net *net, struct sk_buff *in_skb,
2229	struct tcf_proto *tp, unsigned long base,
2230	u32 handle, struct nlattr **tca,
2231	void **arg, u32 flags,
2232	struct netlink_ext_ack *extack)
2233	{
2234	struct cls_fl_head *head = fl_head_dereference(tp);
2235	bool rtnl_held = !(flags & TCA_ACT_FLAGS_NO_RTNL);
2236	struct cls_fl_filter *fold = *arg;
2237	bool bound_to_filter = false;
2238	struct cls_fl_filter *fnew;
2239	struct fl_flow_mask *mask;
2240	struct nlattr **tb;
2241	bool in_ht;
2242	int err;
2243
2244	if (!tca[TCA_OPTIONS]) {
2245	err = -EINVAL;
2246	goto errout_fold;
2247	}
2248
2249	mask = kzalloc(size: sizeof(struct fl_flow_mask), GFP_KERNEL);
2250	if (!mask) {
2251	err = -ENOBUFS;
2252	goto errout_fold;
2253	}
2254
2255	tb = kcalloc(TCA_FLOWER_MAX + 1, size: sizeof(struct nlattr *), GFP_KERNEL);
2256	if (!tb) {
2257	err = -ENOBUFS;
2258	goto errout_mask_alloc;
2259	}
2260
2261	err = nla_parse_nested_deprecated(tb, TCA_FLOWER_MAX,
2262	nla: tca[TCA_OPTIONS], policy: fl_policy, NULL);
2263	if (err < 0)
2264	goto errout_tb;
2265
2266	if (fold && handle && fold->handle != handle) {
2267	err = -EINVAL;
2268	goto errout_tb;
2269	}
2270
2271	fnew = kzalloc(size: sizeof(*fnew), GFP_KERNEL);
2272	if (!fnew) {
2273	err = -ENOBUFS;
2274	goto errout_tb;
2275	}
2276	INIT_LIST_HEAD(list: &fnew->hw_list);
2277	refcount_set(r: &fnew->refcnt, n: 1);
2278
2279	if (tb[TCA_FLOWER_FLAGS]) {
2280	fnew->flags = nla_get_u32(nla: tb[TCA_FLOWER_FLAGS]);
2281
2282	if (!tc_flags_valid(flags: fnew->flags)) {
2283	kfree(objp: fnew);
2284	err = -EINVAL;
2285	goto errout_tb;
2286	}
2287	}
2288
2289	if (!fold) {
2290	spin_lock(lock: &tp->lock);
2291	if (!handle) {
2292	handle = 1;
2293	err = idr_alloc_u32(&head->handle_idr, NULL, id: &handle,
2294	INT_MAX, GFP_ATOMIC);
2295	} else {
2296	err = idr_alloc_u32(&head->handle_idr, NULL, id: &handle,
2297	max: handle, GFP_ATOMIC);
2298
2299	/* Filter with specified handle was concurrently
2300	* inserted after initial check in cls_api. This is not
2301	* necessarily an error if NLM_F_EXCL is not set in
2302	* message flags. Returning EAGAIN will cause cls_api to
2303	* try to update concurrently inserted rule.
2304	*/
2305	if (err == -ENOSPC)
2306	err = -EAGAIN;
2307	}
2308	spin_unlock(lock: &tp->lock);
2309
2310	if (err) {
2311	kfree(objp: fnew);
2312	goto errout_tb;
2313	}
2314	}
2315	fnew->handle = handle;
2316
2317	err = tcf_exts_init_ex(exts: &fnew->exts, net, action: TCA_FLOWER_ACT, police: 0, tp, handle,
2318	used_action_miss: !tc_skip_hw(flags: fnew->flags));
2319	if (err < 0)
2320	goto errout_idr;
2321
2322	err = tcf_exts_validate_ex(net, tp, tb, rate_tlv: tca[TCA_RATE],
2323	exts: &fnew->exts, flags, fl_flags: fnew->flags,
2324	extack);
2325	if (err < 0)
2326	goto errout_idr;
2327
2328	if (tb[TCA_FLOWER_CLASSID]) {
2329	fnew->res.classid = nla_get_u32(nla: tb[TCA_FLOWER_CLASSID]);
2330	if (flags & TCA_ACT_FLAGS_NO_RTNL)
2331	rtnl_lock();
2332	tcf_bind_filter(tp, r: &fnew->res, base);
2333	if (flags & TCA_ACT_FLAGS_NO_RTNL)
2334	rtnl_unlock();
2335	bound_to_filter = true;
2336	}
2337
2338	err = fl_set_key(net, tb, key: &fnew->key, mask: &mask->key, extack);
2339	if (err)
2340	goto unbind_filter;
2341
2342	fl_mask_update_range(mask);
2343	fl_set_masked_key(mkey: &fnew->mkey, key: &fnew->key, mask);
2344
2345	if (!fl_mask_fits_tmplt(tmplt: tp->chain->tmplt_priv, mask)) {
2346	NL_SET_ERR_MSG_MOD(extack, "Mask does not fit the template");
2347	err = -EINVAL;
2348	goto unbind_filter;
2349	}
2350
2351	/* Enable tc skb extension if filter matches on data extracted from
2352	* this extension.
2353	*/
2354	if (fl_needs_tc_skb_ext(mask: &mask->key)) {
2355	fnew->needs_tc_skb_ext = 1;
2356	tc_skb_ext_tc_enable();
2357	}
2358
2359	err = fl_check_assign_mask(head, fnew, fold, mask);
2360	if (err)
2361	goto unbind_filter;
2362
2363	err = fl_ht_insert_unique(fnew, fold, in_ht: &in_ht);
2364	if (err)
2365	goto errout_mask;
2366
2367	if (!tc_skip_hw(flags: fnew->flags)) {
2368	err = fl_hw_replace_filter(tp, f: fnew, rtnl_held, extack);
2369	if (err)
2370	goto errout_ht;
2371	}
2372
2373	if (!tc_in_hw(flags: fnew->flags))
2374	fnew->flags |= TCA_CLS_FLAGS_NOT_IN_HW;
2375
2376	spin_lock(lock: &tp->lock);
2377
2378	/* tp was deleted concurrently. -EAGAIN will cause caller to lookup
2379	* proto again or create new one, if necessary.
2380	*/
2381	if (tp->deleting) {
2382	err = -EAGAIN;
2383	goto errout_hw;
2384	}
2385
2386	if (fold) {
2387	/* Fold filter was deleted concurrently. Retry lookup. */
2388	if (fold->deleted) {
2389	err = -EAGAIN;
2390	goto errout_hw;
2391	}
2392
2393	fnew->handle = handle;
2394
2395	if (!in_ht) {
2396	struct rhashtable_params params =
2397	fnew->mask->filter_ht_params;
2398
2399	err = rhashtable_insert_fast(ht: &fnew->mask->ht,
2400	obj: &fnew->ht_node,
2401	params);
2402	if (err)
2403	goto errout_hw;
2404	in_ht = true;
2405	}
2406
2407	refcount_inc(r: &fnew->refcnt);
2408	rhashtable_remove_fast(ht: &fold->mask->ht,
2409	obj: &fold->ht_node,
2410	params: fold->mask->filter_ht_params);
2411	idr_replace(&head->handle_idr, fnew, id: fnew->handle);
2412	list_replace_rcu(old: &fold->list, new: &fnew->list);
2413	fold->deleted = true;
2414
2415	spin_unlock(lock: &tp->lock);
2416
2417	fl_mask_put(head, mask: fold->mask);
2418	if (!tc_skip_hw(flags: fold->flags))
2419	fl_hw_destroy_filter(tp, f: fold, rtnl_held, NULL);
2420	tcf_unbind_filter(tp, r: &fold->res);
2421	/* Caller holds reference to fold, so refcnt is always > 0
2422	* after this.
2423	*/
2424	refcount_dec(r: &fold->refcnt);
2425	__fl_put(f: fold);
2426	} else {
2427	idr_replace(&head->handle_idr, fnew, id: fnew->handle);
2428
2429	refcount_inc(r: &fnew->refcnt);
2430	list_add_tail_rcu(new: &fnew->list, head: &fnew->mask->filters);
2431	spin_unlock(lock: &tp->lock);
2432	}
2433
2434	*arg = fnew;
2435
2436	kfree(objp: tb);
2437	tcf_queue_work(rwork: &mask->rwork, func: fl_uninit_mask_free_work);
2438	return 0;
2439
2440	errout_ht:
2441	spin_lock(lock: &tp->lock);
2442	errout_hw:
2443	fnew->deleted = true;
2444	spin_unlock(lock: &tp->lock);
2445	if (!tc_skip_hw(flags: fnew->flags))
2446	fl_hw_destroy_filter(tp, f: fnew, rtnl_held, NULL);
2447	if (in_ht)
2448	rhashtable_remove_fast(ht: &fnew->mask->ht, obj: &fnew->ht_node,
2449	params: fnew->mask->filter_ht_params);
2450	errout_mask:
2451	fl_mask_put(head, mask: fnew->mask);
2452
2453	unbind_filter:
2454	if (bound_to_filter) {
2455	if (flags & TCA_ACT_FLAGS_NO_RTNL)
2456	rtnl_lock();
2457	tcf_unbind_filter(tp, r: &fnew->res);
2458	if (flags & TCA_ACT_FLAGS_NO_RTNL)
2459	rtnl_unlock();
2460	}
2461
2462	errout_idr:
2463	if (!fold)
2464	idr_remove(&head->handle_idr, id: fnew->handle);
2465	__fl_put(f: fnew);
2466	errout_tb:
2467	kfree(objp: tb);
2468	errout_mask_alloc:
2469	tcf_queue_work(rwork: &mask->rwork, func: fl_uninit_mask_free_work);
2470	errout_fold:
2471	if (fold)
2472	__fl_put(f: fold);
2473	return err;
2474	}
2475
2476	static int fl_delete(struct tcf_proto *tp, void *arg, bool *last,
2477	bool rtnl_held, struct netlink_ext_ack *extack)
2478	{
2479	struct cls_fl_head *head = fl_head_dereference(tp);
2480	struct cls_fl_filter *f = arg;
2481	bool last_on_mask;
2482	int err = 0;
2483
2484	err = __fl_delete(tp, f, last: &last_on_mask, rtnl_held, extack);
2485	*last = list_empty(head: &head->masks);
2486	__fl_put(f);
2487
2488	return err;
2489	}
2490
2491	static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg,
2492	bool rtnl_held)
2493	{
2494	struct cls_fl_head *head = fl_head_dereference(tp);
2495	unsigned long id = arg->cookie, tmp;
2496	struct cls_fl_filter *f;
2497
2498	arg->count = arg->skip;
2499
2500	rcu_read_lock();
2501	idr_for_each_entry_continue_ul(&head->handle_idr, f, tmp, id) {
2502	/* don't return filters that are being deleted */
2503	if (!f || !refcount_inc_not_zero(r: &f->refcnt))
2504	continue;
2505	rcu_read_unlock();
2506
2507	if (arg->fn(tp, f, arg) < 0) {
2508	__fl_put(f);
2509	arg->stop = 1;
2510	rcu_read_lock();
2511	break;
2512	}
2513	__fl_put(f);
2514	arg->count++;
2515	rcu_read_lock();
2516	}
2517	rcu_read_unlock();
2518	arg->cookie = id;
2519	}
2520
2521	static struct cls_fl_filter *
2522	fl_get_next_hw_filter(struct tcf_proto *tp, struct cls_fl_filter *f, bool add)
2523	{
2524	struct cls_fl_head *head = fl_head_dereference(tp);
2525
2526	spin_lock(lock: &tp->lock);
2527	if (list_empty(head: &head->hw_filters)) {
2528	spin_unlock(lock: &tp->lock);
2529	return NULL;
2530	}
2531
2532	if (!f)
2533	f = list_entry(&head->hw_filters, struct cls_fl_filter,
2534	hw_list);
2535	list_for_each_entry_continue(f, &head->hw_filters, hw_list) {
2536	if (!(add && f->deleted) && refcount_inc_not_zero(r: &f->refcnt)) {
2537	spin_unlock(lock: &tp->lock);
2538	return f;
2539	}
2540	}
2541
2542	spin_unlock(lock: &tp->lock);
2543	return NULL;
2544	}
2545
2546	static int fl_reoffload(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb,
2547	void *cb_priv, struct netlink_ext_ack *extack)
2548	{
2549	struct tcf_block *block = tp->chain->block;
2550	struct flow_cls_offload cls_flower = {};
2551	struct cls_fl_filter *f = NULL;
2552	int err;
2553
2554	/* hw_filters list can only be changed by hw offload functions after
2555	* obtaining rtnl lock. Make sure it is not changed while reoffload is
2556	* iterating it.
2557	*/
2558	ASSERT_RTNL();
2559
2560	while ((f = fl_get_next_hw_filter(tp, f, add))) {
2561	cls_flower.rule =
2562	flow_rule_alloc(num_actions: tcf_exts_num_actions(exts: &f->exts));
2563	if (!cls_flower.rule) {
2564	__fl_put(f);
2565	return -ENOMEM;
2566	}
2567
2568	tc_cls_common_offload_init(cls_common: &cls_flower.common, tp, flags: f->flags,
2569	extack);
2570	cls_flower.command = add ?
2571	FLOW_CLS_REPLACE : FLOW_CLS_DESTROY;
2572	cls_flower.cookie = (unsigned long)f;
2573	cls_flower.rule->match.dissector = &f->mask->dissector;
2574	cls_flower.rule->match.mask = &f->mask->key;
2575	cls_flower.rule->match.key = &f->mkey;
2576
2577	err = tc_setup_offload_action(flow_action: &cls_flower.rule->action, exts: &f->exts,
2578	extack: cls_flower.common.extack);
2579	if (err) {
2580	kfree(objp: cls_flower.rule);
2581	if (tc_skip_sw(flags: f->flags)) {
2582	__fl_put(f);
2583	return err;
2584	}
2585	goto next_flow;
2586	}
2587
2588	cls_flower.classid = f->res.classid;
2589
2590	err = tc_setup_cb_reoffload(block, tp, add, cb,
2591	type: TC_SETUP_CLSFLOWER, type_data: &cls_flower,
2592	cb_priv, flags: &f->flags,
2593	in_hw_count: &f->in_hw_count);
2594	tc_cleanup_offload_action(flow_action: &cls_flower.rule->action);
2595	kfree(objp: cls_flower.rule);
2596
2597	if (err) {
2598	__fl_put(f);
2599	return err;
2600	}
2601	next_flow:
2602	__fl_put(f);
2603	}
2604
2605	return 0;
2606	}
2607
2608	static void fl_hw_add(struct tcf_proto *tp, void *type_data)
2609	{
2610	struct flow_cls_offload *cls_flower = type_data;
2611	struct cls_fl_filter *f =
2612	(struct cls_fl_filter *) cls_flower->cookie;
2613	struct cls_fl_head *head = fl_head_dereference(tp);
2614
2615	spin_lock(lock: &tp->lock);
2616	list_add(new: &f->hw_list, head: &head->hw_filters);
2617	spin_unlock(lock: &tp->lock);
2618	}
2619
2620	static void fl_hw_del(struct tcf_proto *tp, void *type_data)
2621	{
2622	struct flow_cls_offload *cls_flower = type_data;
2623	struct cls_fl_filter *f =
2624	(struct cls_fl_filter *) cls_flower->cookie;
2625
2626	spin_lock(lock: &tp->lock);
2627	if (!list_empty(head: &f->hw_list))
2628	list_del_init(entry: &f->hw_list);
2629	spin_unlock(lock: &tp->lock);
2630	}
2631
2632	static int fl_hw_create_tmplt(struct tcf_chain *chain,
2633	struct fl_flow_tmplt *tmplt)
2634	{
2635	struct flow_cls_offload cls_flower = {};
2636	struct tcf_block *block = chain->block;
2637
2638	cls_flower.rule = flow_rule_alloc(num_actions: 0);
2639	if (!cls_flower.rule)
2640	return -ENOMEM;
2641
2642	cls_flower.common.chain_index = chain->index;
2643	cls_flower.command = FLOW_CLS_TMPLT_CREATE;
2644	cls_flower.cookie = (unsigned long) tmplt;
2645	cls_flower.rule->match.dissector = &tmplt->dissector;
2646	cls_flower.rule->match.mask = &tmplt->mask;
2647	cls_flower.rule->match.key = &tmplt->dummy_key;
2648
2649	/* We don't care if driver (any of them) fails to handle this
2650	* call. It serves just as a hint for it.
2651	*/
2652	tc_setup_cb_call(block, type: TC_SETUP_CLSFLOWER, type_data: &cls_flower, err_stop: false, rtnl_held: true);
2653	kfree(objp: cls_flower.rule);
2654
2655	return 0;
2656	}
2657
2658	static void fl_hw_destroy_tmplt(struct tcf_chain *chain,
2659	struct fl_flow_tmplt *tmplt)
2660	{
2661	struct flow_cls_offload cls_flower = {};
2662	struct tcf_block *block = chain->block;
2663
2664	cls_flower.common.chain_index = chain->index;
2665	cls_flower.command = FLOW_CLS_TMPLT_DESTROY;
2666	cls_flower.cookie = (unsigned long) tmplt;
2667
2668	tc_setup_cb_call(block, type: TC_SETUP_CLSFLOWER, type_data: &cls_flower, err_stop: false, rtnl_held: true);
2669	}
2670
2671	static void *fl_tmplt_create(struct net *net, struct tcf_chain *chain,
2672	struct nlattr **tca,
2673	struct netlink_ext_ack *extack)
2674	{
2675	struct fl_flow_tmplt *tmplt;
2676	struct nlattr **tb;
2677	int err;
2678
2679	if (!tca[TCA_OPTIONS])
2680	return ERR_PTR(error: -EINVAL);
2681
2682	tb = kcalloc(TCA_FLOWER_MAX + 1, size: sizeof(struct nlattr *), GFP_KERNEL);
2683	if (!tb)
2684	return ERR_PTR(error: -ENOBUFS);
2685	err = nla_parse_nested_deprecated(tb, TCA_FLOWER_MAX,
2686	nla: tca[TCA_OPTIONS], policy: fl_policy, NULL);
2687	if (err)
2688	goto errout_tb;
2689
2690	tmplt = kzalloc(size: sizeof(*tmplt), GFP_KERNEL);
2691	if (!tmplt) {
2692	err = -ENOMEM;
2693	goto errout_tb;
2694	}
2695	tmplt->chain = chain;
2696	err = fl_set_key(net, tb, key: &tmplt->dummy_key, mask: &tmplt->mask, extack);
2697	if (err)
2698	goto errout_tmplt;
2699
2700	fl_init_dissector(dissector: &tmplt->dissector, mask: &tmplt->mask);
2701
2702	err = fl_hw_create_tmplt(chain, tmplt);
2703	if (err)
2704	goto errout_tmplt;
2705
2706	kfree(objp: tb);
2707	return tmplt;
2708
2709	errout_tmplt:
2710	kfree(objp: tmplt);
2711	errout_tb:
2712	kfree(objp: tb);
2713	return ERR_PTR(error: err);
2714	}
2715
2716	static void fl_tmplt_destroy(void *tmplt_priv)
2717	{
2718	struct fl_flow_tmplt *tmplt = tmplt_priv;
2719
2720	fl_hw_destroy_tmplt(chain: tmplt->chain, tmplt);
2721	kfree(objp: tmplt);
2722	}
2723
2724	static int fl_dump_key_val(struct sk_buff *skb,
2725	void *val, int val_type,
2726	void *mask, int mask_type, int len)
2727	{
2728	int err;
2729
2730	if (!memchr_inv(p: mask, c: 0, size: len))
2731	return 0;
2732	err = nla_put(skb, attrtype: val_type, attrlen: len, data: val);
2733	if (err)
2734	return err;
2735	if (mask_type != TCA_FLOWER_UNSPEC) {
2736	err = nla_put(skb, attrtype: mask_type, attrlen: len, data: mask);
2737	if (err)
2738	return err;
2739	}
2740	return 0;
2741	}
2742
2743	static int fl_dump_key_port_range(struct sk_buff *skb, struct fl_flow_key *key,
2744	struct fl_flow_key *mask)
2745	{
2746	if (fl_dump_key_val(skb, val: &key->tp_range.tp_min.dst,
2747	val_type: TCA_FLOWER_KEY_PORT_DST_MIN,
2748	mask: &mask->tp_range.tp_min.dst, mask_type: TCA_FLOWER_UNSPEC,
2749	len: sizeof(key->tp_range.tp_min.dst)) ||
2750	fl_dump_key_val(skb, val: &key->tp_range.tp_max.dst,
2751	val_type: TCA_FLOWER_KEY_PORT_DST_MAX,
2752	mask: &mask->tp_range.tp_max.dst, mask_type: TCA_FLOWER_UNSPEC,
2753	len: sizeof(key->tp_range.tp_max.dst)) ||
2754	fl_dump_key_val(skb, val: &key->tp_range.tp_min.src,
2755	val_type: TCA_FLOWER_KEY_PORT_SRC_MIN,
2756	mask: &mask->tp_range.tp_min.src, mask_type: TCA_FLOWER_UNSPEC,
2757	len: sizeof(key->tp_range.tp_min.src)) ||
2758	fl_dump_key_val(skb, val: &key->tp_range.tp_max.src,
2759	val_type: TCA_FLOWER_KEY_PORT_SRC_MAX,
2760	mask: &mask->tp_range.tp_max.src, mask_type: TCA_FLOWER_UNSPEC,
2761	len: sizeof(key->tp_range.tp_max.src)))
2762	return -1;
2763
2764	return 0;
2765	}
2766
2767	static int fl_dump_key_mpls_opt_lse(struct sk_buff *skb,
2768	struct flow_dissector_key_mpls *mpls_key,
2769	struct flow_dissector_key_mpls *mpls_mask,
2770	u8 lse_index)
2771	{
2772	struct flow_dissector_mpls_lse *lse_mask = &mpls_mask->ls[lse_index];
2773	struct flow_dissector_mpls_lse *lse_key = &mpls_key->ls[lse_index];
2774	int err;
2775
2776	err = nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH,
2777	value: lse_index + 1);
2778	if (err)
2779	return err;
2780
2781	if (lse_mask->mpls_ttl) {
2782	err = nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_MPLS_OPT_LSE_TTL,
2783	value: lse_key->mpls_ttl);
2784	if (err)
2785	return err;
2786	}
2787	if (lse_mask->mpls_bos) {
2788	err = nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS,
2789	value: lse_key->mpls_bos);
2790	if (err)
2791	return err;
2792	}
2793	if (lse_mask->mpls_tc) {
2794	err = nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_MPLS_OPT_LSE_TC,
2795	value: lse_key->mpls_tc);
2796	if (err)
2797	return err;
2798	}
2799	if (lse_mask->mpls_label) {
2800	err = nla_put_u32(skb, attrtype: TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL,
2801	value: lse_key->mpls_label);
2802	if (err)
2803	return err;
2804	}
2805
2806	return 0;
2807	}
2808
2809	static int fl_dump_key_mpls_opts(struct sk_buff *skb,
2810	struct flow_dissector_key_mpls *mpls_key,
2811	struct flow_dissector_key_mpls *mpls_mask)
2812	{
2813	struct nlattr *opts;
2814	struct nlattr *lse;
2815	u8 lse_index;
2816	int err;
2817
2818	opts = nla_nest_start(skb, attrtype: TCA_FLOWER_KEY_MPLS_OPTS);
2819	if (!opts)
2820	return -EMSGSIZE;
2821
2822	for (lse_index = 0; lse_index < FLOW_DIS_MPLS_MAX; lse_index++) {
2823	if (!(mpls_mask->used_lses & 1 << lse_index))
2824	continue;
2825
2826	lse = nla_nest_start(skb, attrtype: TCA_FLOWER_KEY_MPLS_OPTS_LSE);
2827	if (!lse) {
2828	err = -EMSGSIZE;
2829	goto err_opts;
2830	}
2831
2832	err = fl_dump_key_mpls_opt_lse(skb, mpls_key, mpls_mask,
2833	lse_index);
2834	if (err)
2835	goto err_opts_lse;
2836	nla_nest_end(skb, start: lse);
2837	}
2838	nla_nest_end(skb, start: opts);
2839
2840	return 0;
2841
2842	err_opts_lse:
2843	nla_nest_cancel(skb, start: lse);
2844	err_opts:
2845	nla_nest_cancel(skb, start: opts);
2846
2847	return err;
2848	}
2849
2850	static int fl_dump_key_mpls(struct sk_buff *skb,
2851	struct flow_dissector_key_mpls *mpls_key,
2852	struct flow_dissector_key_mpls *mpls_mask)
2853	{
2854	struct flow_dissector_mpls_lse *lse_mask;
2855	struct flow_dissector_mpls_lse *lse_key;
2856	int err;
2857
2858	if (!mpls_mask->used_lses)
2859	return 0;
2860
2861	lse_mask = &mpls_mask->ls[0];
2862	lse_key = &mpls_key->ls[0];
2863
2864	/* For backward compatibility, don't use the MPLS nested attributes if
2865	* the rule can be expressed using the old attributes.
2866	*/
2867	if (mpls_mask->used_lses & ~1 ||
2868	(!lse_mask->mpls_ttl && !lse_mask->mpls_bos &&
2869	!lse_mask->mpls_tc && !lse_mask->mpls_label))
2870	return fl_dump_key_mpls_opts(skb, mpls_key, mpls_mask);
2871
2872	if (lse_mask->mpls_ttl) {
2873	err = nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_MPLS_TTL,
2874	value: lse_key->mpls_ttl);
2875	if (err)
2876	return err;
2877	}
2878	if (lse_mask->mpls_tc) {
2879	err = nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_MPLS_TC,
2880	value: lse_key->mpls_tc);
2881	if (err)
2882	return err;
2883	}
2884	if (lse_mask->mpls_label) {
2885	err = nla_put_u32(skb, attrtype: TCA_FLOWER_KEY_MPLS_LABEL,
2886	value: lse_key->mpls_label);
2887	if (err)
2888	return err;
2889	}
2890	if (lse_mask->mpls_bos) {
2891	err = nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_MPLS_BOS,
2892	value: lse_key->mpls_bos);
2893	if (err)
2894	return err;
2895	}
2896	return 0;
2897	}
2898
2899	static int fl_dump_key_ip(struct sk_buff *skb, bool encap,
2900	struct flow_dissector_key_ip *key,
2901	struct flow_dissector_key_ip *mask)
2902	{
2903	int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS;
2904	int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL;
2905	int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK;
2906	int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK;
2907
2908	if (fl_dump_key_val(skb, val: &key->tos, val_type: tos_key, mask: &mask->tos, mask_type: tos_mask, len: sizeof(key->tos)) ||
2909	fl_dump_key_val(skb, val: &key->ttl, val_type: ttl_key, mask: &mask->ttl, mask_type: ttl_mask, len: sizeof(key->ttl)))
2910	return -1;
2911
2912	return 0;
2913	}
2914
2915	static int fl_dump_key_vlan(struct sk_buff *skb,
2916	int vlan_id_key, int vlan_prio_key,
2917	struct flow_dissector_key_vlan *vlan_key,
2918	struct flow_dissector_key_vlan *vlan_mask)
2919	{
2920	int err;
2921
2922	if (!memchr_inv(p: vlan_mask, c: 0, size: sizeof(*vlan_mask)))
2923	return 0;
2924	if (vlan_mask->vlan_id) {
2925	err = nla_put_u16(skb, attrtype: vlan_id_key,
2926	value: vlan_key->vlan_id);
2927	if (err)
2928	return err;
2929	}
2930	if (vlan_mask->vlan_priority) {
2931	err = nla_put_u8(skb, attrtype: vlan_prio_key,
2932	value: vlan_key->vlan_priority);
2933	if (err)
2934	return err;
2935	}
2936	return 0;
2937	}
2938
2939	static void fl_get_key_flag(u32 dissector_key, u32 dissector_mask,
2940	u32 *flower_key, u32 *flower_mask,
2941	u32 flower_flag_bit, u32 dissector_flag_bit)
2942	{
2943	if (dissector_mask & dissector_flag_bit) {
2944	*flower_mask |= flower_flag_bit;
2945	if (dissector_key & dissector_flag_bit)
2946	*flower_key |= flower_flag_bit;
2947	}
2948	}
2949
2950	static int fl_dump_key_flags(struct sk_buff *skb, u32 flags_key, u32 flags_mask)
2951	{
2952	u32 key, mask;
2953	__be32 _key, _mask;
2954	int err;
2955
2956	if (!memchr_inv(p: &flags_mask, c: 0, size: sizeof(flags_mask)))
2957	return 0;
2958
2959	key = 0;
2960	mask = 0;
2961
2962	fl_get_key_flag(dissector_key: flags_key, dissector_mask: flags_mask, flower_key: &key, flower_mask: &mask,
2963	flower_flag_bit: TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
2964	fl_get_key_flag(dissector_key: flags_key, dissector_mask: flags_mask, flower_key: &key, flower_mask: &mask,
2965	flower_flag_bit: TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST,
2966	FLOW_DIS_FIRST_FRAG);
2967
2968	_key = cpu_to_be32(key);
2969	_mask = cpu_to_be32(mask);
2970
2971	err = nla_put(skb, attrtype: TCA_FLOWER_KEY_FLAGS, attrlen: 4, data: &_key);
2972	if (err)
2973	return err;
2974
2975	return nla_put(skb, attrtype: TCA_FLOWER_KEY_FLAGS_MASK, attrlen: 4, data: &_mask);
2976	}
2977
2978	static int fl_dump_key_geneve_opt(struct sk_buff *skb,
2979	struct flow_dissector_key_enc_opts *enc_opts)
2980	{
2981	struct geneve_opt *opt;
2982	struct nlattr *nest;
2983	int opt_off = 0;
2984
2985	nest = nla_nest_start_noflag(skb, attrtype: TCA_FLOWER_KEY_ENC_OPTS_GENEVE);
2986	if (!nest)
2987	goto nla_put_failure;
2988
2989	while (enc_opts->len > opt_off) {
2990	opt = (struct geneve_opt *)&enc_opts->data[opt_off];
2991
2992	if (nla_put_be16(skb, attrtype: TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS,
2993	value: opt->opt_class))
2994	goto nla_put_failure;
2995	if (nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE,
2996	value: opt->type))
2997	goto nla_put_failure;
2998	if (nla_put(skb, attrtype: TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA,
2999	attrlen: opt->length * 4, data: opt->opt_data))
3000	goto nla_put_failure;
3001
3002	opt_off += sizeof(struct geneve_opt) + opt->length * 4;
3003	}
3004	nla_nest_end(skb, start: nest);
3005	return 0;
3006
3007	nla_put_failure:
3008	nla_nest_cancel(skb, start: nest);
3009	return -EMSGSIZE;
3010	}
3011
3012	static int fl_dump_key_vxlan_opt(struct sk_buff *skb,
3013	struct flow_dissector_key_enc_opts *enc_opts)
3014	{
3015	struct vxlan_metadata *md;
3016	struct nlattr *nest;
3017
3018	nest = nla_nest_start_noflag(skb, attrtype: TCA_FLOWER_KEY_ENC_OPTS_VXLAN);
3019	if (!nest)
3020	goto nla_put_failure;
3021
3022	md = (struct vxlan_metadata *)&enc_opts->data[0];
3023	if (nla_put_u32(skb, attrtype: TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP, value: md->gbp))
3024	goto nla_put_failure;
3025
3026	nla_nest_end(skb, start: nest);
3027	return 0;
3028
3029	nla_put_failure:
3030	nla_nest_cancel(skb, start: nest);
3031	return -EMSGSIZE;
3032	}
3033
3034	static int fl_dump_key_erspan_opt(struct sk_buff *skb,
3035	struct flow_dissector_key_enc_opts *enc_opts)
3036	{
3037	struct erspan_metadata *md;
3038	struct nlattr *nest;
3039
3040	nest = nla_nest_start_noflag(skb, attrtype: TCA_FLOWER_KEY_ENC_OPTS_ERSPAN);
3041	if (!nest)
3042	goto nla_put_failure;
3043
3044	md = (struct erspan_metadata *)&enc_opts->data[0];
3045	if (nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER, value: md->version))
3046	goto nla_put_failure;
3047
3048	if (md->version == 1 &&
3049	nla_put_be32(skb, attrtype: TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX, value: md->u.index))
3050	goto nla_put_failure;
3051
3052	if (md->version == 2 &&
3053	(nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR,
3054	value: md->u.md2.dir) ||
3055	nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID,
3056	value: get_hwid(md2: &md->u.md2))))
3057	goto nla_put_failure;
3058
3059	nla_nest_end(skb, start: nest);
3060	return 0;
3061
3062	nla_put_failure:
3063	nla_nest_cancel(skb, start: nest);
3064	return -EMSGSIZE;
3065	}
3066
3067	static int fl_dump_key_gtp_opt(struct sk_buff *skb,
3068	struct flow_dissector_key_enc_opts *enc_opts)
3069
3070	{
3071	struct gtp_pdu_session_info *session_info;
3072	struct nlattr *nest;
3073
3074	nest = nla_nest_start_noflag(skb, attrtype: TCA_FLOWER_KEY_ENC_OPTS_GTP);
3075	if (!nest)
3076	goto nla_put_failure;
3077
3078	session_info = (struct gtp_pdu_session_info *)&enc_opts->data[0];
3079
3080	if (nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_ENC_OPT_GTP_PDU_TYPE,
3081	value: session_info->pdu_type))
3082	goto nla_put_failure;
3083
3084	if (nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_ENC_OPT_GTP_QFI, value: session_info->qfi))
3085	goto nla_put_failure;
3086
3087	nla_nest_end(skb, start: nest);
3088	return 0;
3089
3090	nla_put_failure:
3091	nla_nest_cancel(skb, start: nest);
3092	return -EMSGSIZE;
3093	}
3094
3095	static int fl_dump_key_ct(struct sk_buff *skb,
3096	struct flow_dissector_key_ct *key,
3097	struct flow_dissector_key_ct *mask)
3098	{
3099	if (IS_ENABLED(CONFIG_NF_CONNTRACK) &&
3100	fl_dump_key_val(skb, val: &key->ct_state, val_type: TCA_FLOWER_KEY_CT_STATE,
3101	mask: &mask->ct_state, mask_type: TCA_FLOWER_KEY_CT_STATE_MASK,
3102	len: sizeof(key->ct_state)))
3103	goto nla_put_failure;
3104
3105	if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) &&
3106	fl_dump_key_val(skb, val: &key->ct_zone, val_type: TCA_FLOWER_KEY_CT_ZONE,
3107	mask: &mask->ct_zone, mask_type: TCA_FLOWER_KEY_CT_ZONE_MASK,
3108	len: sizeof(key->ct_zone)))
3109	goto nla_put_failure;
3110
3111	if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) &&
3112	fl_dump_key_val(skb, val: &key->ct_mark, val_type: TCA_FLOWER_KEY_CT_MARK,
3113	mask: &mask->ct_mark, mask_type: TCA_FLOWER_KEY_CT_MARK_MASK,
3114	len: sizeof(key->ct_mark)))
3115	goto nla_put_failure;
3116
3117	if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) &&
3118	fl_dump_key_val(skb, val: &key->ct_labels, val_type: TCA_FLOWER_KEY_CT_LABELS,
3119	mask: &mask->ct_labels, mask_type: TCA_FLOWER_KEY_CT_LABELS_MASK,
3120	len: sizeof(key->ct_labels)))
3121	goto nla_put_failure;
3122
3123	return 0;
3124
3125	nla_put_failure:
3126	return -EMSGSIZE;
3127	}
3128
3129	static int fl_dump_key_cfm(struct sk_buff *skb,
3130	struct flow_dissector_key_cfm *key,
3131	struct flow_dissector_key_cfm *mask)
3132	{
3133	struct nlattr *opts;
3134	int err;
3135	u8 mdl;
3136
3137	if (!memchr_inv(p: mask, c: 0, size: sizeof(*mask)))
3138	return 0;
3139
3140	opts = nla_nest_start(skb, attrtype: TCA_FLOWER_KEY_CFM);
3141	if (!opts)
3142	return -EMSGSIZE;
3143
3144	if (FIELD_GET(FLOW_DIS_CFM_MDL_MASK, mask->mdl_ver)) {
3145	mdl = FIELD_GET(FLOW_DIS_CFM_MDL_MASK, key->mdl_ver);
3146	err = nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_CFM_MD_LEVEL, value: mdl);
3147	if (err)
3148	goto err_cfm_opts;
3149	}
3150
3151	if (mask->opcode) {
3152	err = nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_CFM_OPCODE, value: key->opcode);
3153	if (err)
3154	goto err_cfm_opts;
3155	}
3156
3157	nla_nest_end(skb, start: opts);
3158
3159	return 0;
3160
3161	err_cfm_opts:
3162	nla_nest_cancel(skb, start: opts);
3163	return err;
3164	}
3165
3166	static int fl_dump_key_options(struct sk_buff *skb, int enc_opt_type,
3167	struct flow_dissector_key_enc_opts *enc_opts)
3168	{
3169	struct nlattr *nest;
3170	int err;
3171
3172	if (!enc_opts->len)
3173	return 0;
3174
3175	nest = nla_nest_start_noflag(skb, attrtype: enc_opt_type);
3176	if (!nest)
3177	goto nla_put_failure;
3178
3179	switch (enc_opts->dst_opt_type) {
3180	case TUNNEL_GENEVE_OPT:
3181	err = fl_dump_key_geneve_opt(skb, enc_opts);
3182	if (err)
3183	goto nla_put_failure;
3184	break;
3185	case TUNNEL_VXLAN_OPT:
3186	err = fl_dump_key_vxlan_opt(skb, enc_opts);
3187	if (err)
3188	goto nla_put_failure;
3189	break;
3190	case TUNNEL_ERSPAN_OPT:
3191	err = fl_dump_key_erspan_opt(skb, enc_opts);
3192	if (err)
3193	goto nla_put_failure;
3194	break;
3195	case TUNNEL_GTP_OPT:
3196	err = fl_dump_key_gtp_opt(skb, enc_opts);
3197	if (err)
3198	goto nla_put_failure;
3199	break;
3200	default:
3201	goto nla_put_failure;
3202	}
3203	nla_nest_end(skb, start: nest);
3204	return 0;
3205
3206	nla_put_failure:
3207	nla_nest_cancel(skb, start: nest);
3208	return -EMSGSIZE;
3209	}
3210
3211	static int fl_dump_key_enc_opt(struct sk_buff *skb,
3212	struct flow_dissector_key_enc_opts *key_opts,
3213	struct flow_dissector_key_enc_opts *msk_opts)
3214	{
3215	int err;
3216
3217	err = fl_dump_key_options(skb, enc_opt_type: TCA_FLOWER_KEY_ENC_OPTS, enc_opts: key_opts);
3218	if (err)
3219	return err;
3220
3221	return fl_dump_key_options(skb, enc_opt_type: TCA_FLOWER_KEY_ENC_OPTS_MASK, enc_opts: msk_opts);
3222	}
3223
3224	static int fl_dump_key(struct sk_buff *skb, struct net *net,
3225	struct fl_flow_key *key, struct fl_flow_key *mask)
3226	{
3227	if (mask->meta.ingress_ifindex) {
3228	struct net_device *dev;
3229
3230	dev = __dev_get_by_index(net, ifindex: key->meta.ingress_ifindex);
3231	if (dev && nla_put_string(skb, attrtype: TCA_FLOWER_INDEV, str: dev->name))
3232	goto nla_put_failure;
3233	}
3234
3235	if (fl_dump_key_val(skb, val: &key->meta.l2_miss,
3236	val_type: TCA_FLOWER_L2_MISS, mask: &mask->meta.l2_miss,
3237	mask_type: TCA_FLOWER_UNSPEC, len: sizeof(key->meta.l2_miss)))
3238	goto nla_put_failure;
3239
3240	if (fl_dump_key_val(skb, val: key->eth.dst, val_type: TCA_FLOWER_KEY_ETH_DST,
3241	mask: mask->eth.dst, mask_type: TCA_FLOWER_KEY_ETH_DST_MASK,
3242	len: sizeof(key->eth.dst)) ||
3243	fl_dump_key_val(skb, val: key->eth.src, val_type: TCA_FLOWER_KEY_ETH_SRC,
3244	mask: mask->eth.src, mask_type: TCA_FLOWER_KEY_ETH_SRC_MASK,
3245	len: sizeof(key->eth.src)) ||
3246	fl_dump_key_val(skb, val: &key->basic.n_proto, val_type: TCA_FLOWER_KEY_ETH_TYPE,
3247	mask: &mask->basic.n_proto, mask_type: TCA_FLOWER_UNSPEC,
3248	len: sizeof(key->basic.n_proto)))
3249	goto nla_put_failure;
3250
3251	if (mask->num_of_vlans.num_of_vlans) {
3252	if (nla_put_u8(skb, attrtype: TCA_FLOWER_KEY_NUM_OF_VLANS, value: key->num_of_vlans.num_of_vlans))
3253	goto nla_put_failure;
3254	}
3255
3256	if (fl_dump_key_mpls(skb, mpls_key: &key->mpls, mpls_mask: &mask->mpls))
3257	goto nla_put_failure;
3258
3259	if (fl_dump_key_vlan(skb, vlan_id_key: TCA_FLOWER_KEY_VLAN_ID,
3260	vlan_prio_key: TCA_FLOWER_KEY_VLAN_PRIO, vlan_key: &key->vlan, vlan_mask: &mask->vlan))
3261	goto nla_put_failure;
3262
3263	if (fl_dump_key_vlan(skb, vlan_id_key: TCA_FLOWER_KEY_CVLAN_ID,
3264	vlan_prio_key: TCA_FLOWER_KEY_CVLAN_PRIO,
3265	vlan_key: &key->cvlan, vlan_mask: &mask->cvlan) ||
3266	(mask->cvlan.vlan_tpid &&
3267	nla_put_be16(skb, attrtype: TCA_FLOWER_KEY_VLAN_ETH_TYPE,
3268	value: key->cvlan.vlan_tpid)))
3269	goto nla_put_failure;
3270
3271	if (mask->basic.n_proto) {
3272	if (mask->cvlan.vlan_eth_type) {
3273	if (nla_put_be16(skb, attrtype: TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
3274	value: key->basic.n_proto))
3275	goto nla_put_failure;
3276	} else if (mask->vlan.vlan_eth_type) {
3277	if (nla_put_be16(skb, attrtype: TCA_FLOWER_KEY_VLAN_ETH_TYPE,
3278	value: key->vlan.vlan_eth_type))
3279	goto nla_put_failure;
3280	}
3281	}
3282
3283	if ((key->basic.n_proto == htons(ETH_P_IP) ||
3284	key->basic.n_proto == htons(ETH_P_IPV6)) &&
3285	(fl_dump_key_val(skb, val: &key->basic.ip_proto, val_type: TCA_FLOWER_KEY_IP_PROTO,
3286	mask: &mask->basic.ip_proto, mask_type: TCA_FLOWER_UNSPEC,
3287	len: sizeof(key->basic.ip_proto)) ||
3288	fl_dump_key_ip(skb, encap: false, key: &key->ip, mask: &mask->ip)))
3289	goto nla_put_failure;
3290
3291	if (mask->pppoe.session_id) {
3292	if (nla_put_be16(skb, attrtype: TCA_FLOWER_KEY_PPPOE_SID,
3293	value: key->pppoe.session_id))
3294	goto nla_put_failure;
3295	}
3296	if (mask->basic.n_proto && mask->pppoe.ppp_proto) {
3297	if (nla_put_be16(skb, attrtype: TCA_FLOWER_KEY_PPP_PROTO,
3298	value: key->pppoe.ppp_proto))
3299	goto nla_put_failure;
3300	}
3301
3302	if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
3303	(fl_dump_key_val(skb, val: &key->ipv4.src, val_type: TCA_FLOWER_KEY_IPV4_SRC,
3304	mask: &mask->ipv4.src, mask_type: TCA_FLOWER_KEY_IPV4_SRC_MASK,
3305	len: sizeof(key->ipv4.src)) ||
3306	fl_dump_key_val(skb, val: &key->ipv4.dst, val_type: TCA_FLOWER_KEY_IPV4_DST,
3307	mask: &mask->ipv4.dst, mask_type: TCA_FLOWER_KEY_IPV4_DST_MASK,
3308	len: sizeof(key->ipv4.dst))))
3309	goto nla_put_failure;
3310	else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
3311	(fl_dump_key_val(skb, val: &key->ipv6.src, val_type: TCA_FLOWER_KEY_IPV6_SRC,
3312	mask: &mask->ipv6.src, mask_type: TCA_FLOWER_KEY_IPV6_SRC_MASK,
3313	len: sizeof(key->ipv6.src)) ||
3314	fl_dump_key_val(skb, val: &key->ipv6.dst, val_type: TCA_FLOWER_KEY_IPV6_DST,
3315	mask: &mask->ipv6.dst, mask_type: TCA_FLOWER_KEY_IPV6_DST_MASK,
3316	len: sizeof(key->ipv6.dst))))
3317	goto nla_put_failure;
3318
3319	if (key->basic.ip_proto == IPPROTO_TCP &&
3320	(fl_dump_key_val(skb, val: &key->tp.src, val_type: TCA_FLOWER_KEY_TCP_SRC,
3321	mask: &mask->tp.src, mask_type: TCA_FLOWER_KEY_TCP_SRC_MASK,
3322	len: sizeof(key->tp.src)) ||
3323	fl_dump_key_val(skb, val: &key->tp.dst, val_type: TCA_FLOWER_KEY_TCP_DST,
3324	mask: &mask->tp.dst, mask_type: TCA_FLOWER_KEY_TCP_DST_MASK,
3325	len: sizeof(key->tp.dst)) ||
3326	fl_dump_key_val(skb, val: &key->tcp.flags, val_type: TCA_FLOWER_KEY_TCP_FLAGS,
3327	mask: &mask->tcp.flags, mask_type: TCA_FLOWER_KEY_TCP_FLAGS_MASK,
3328	len: sizeof(key->tcp.flags))))
3329	goto nla_put_failure;
3330	else if (key->basic.ip_proto == IPPROTO_UDP &&
3331	(fl_dump_key_val(skb, val: &key->tp.src, val_type: TCA_FLOWER_KEY_UDP_SRC,
3332	mask: &mask->tp.src, mask_type: TCA_FLOWER_KEY_UDP_SRC_MASK,
3333	len: sizeof(key->tp.src)) ||
3334	fl_dump_key_val(skb, val: &key->tp.dst, val_type: TCA_FLOWER_KEY_UDP_DST,
3335	mask: &mask->tp.dst, mask_type: TCA_FLOWER_KEY_UDP_DST_MASK,
3336	len: sizeof(key->tp.dst))))
3337	goto nla_put_failure;
3338	else if (key->basic.ip_proto == IPPROTO_SCTP &&
3339	(fl_dump_key_val(skb, val: &key->tp.src, val_type: TCA_FLOWER_KEY_SCTP_SRC,
3340	mask: &mask->tp.src, mask_type: TCA_FLOWER_KEY_SCTP_SRC_MASK,
3341	len: sizeof(key->tp.src)) ||
3342	fl_dump_key_val(skb, val: &key->tp.dst, val_type: TCA_FLOWER_KEY_SCTP_DST,
3343	mask: &mask->tp.dst, mask_type: TCA_FLOWER_KEY_SCTP_DST_MASK,
3344	len: sizeof(key->tp.dst))))
3345	goto nla_put_failure;
3346	else if (key->basic.n_proto == htons(ETH_P_IP) &&
3347	key->basic.ip_proto == IPPROTO_ICMP &&
3348	(fl_dump_key_val(skb, val: &key->icmp.type,
3349	val_type: TCA_FLOWER_KEY_ICMPV4_TYPE, mask: &mask->icmp.type,
3350	mask_type: TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,
3351	len: sizeof(key->icmp.type)) ||
3352	fl_dump_key_val(skb, val: &key->icmp.code,
3353	val_type: TCA_FLOWER_KEY_ICMPV4_CODE, mask: &mask->icmp.code,
3354	mask_type: TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
3355	len: sizeof(key->icmp.code))))
3356	goto nla_put_failure;
3357	else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
3358	key->basic.ip_proto == IPPROTO_ICMPV6 &&
3359	(fl_dump_key_val(skb, val: &key->icmp.type,
3360	val_type: TCA_FLOWER_KEY_ICMPV6_TYPE, mask: &mask->icmp.type,
3361	mask_type: TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,
3362	len: sizeof(key->icmp.type)) ||
3363	fl_dump_key_val(skb, val: &key->icmp.code,
3364	val_type: TCA_FLOWER_KEY_ICMPV6_CODE, mask: &mask->icmp.code,
3365	mask_type: TCA_FLOWER_KEY_ICMPV6_CODE_MASK,
3366	len: sizeof(key->icmp.code))))
3367	goto nla_put_failure;
3368	else if ((key->basic.n_proto == htons(ETH_P_ARP) ||
3369	key->basic.n_proto == htons(ETH_P_RARP)) &&
3370	(fl_dump_key_val(skb, val: &key->arp.sip,
3371	val_type: TCA_FLOWER_KEY_ARP_SIP, mask: &mask->arp.sip,
3372	mask_type: TCA_FLOWER_KEY_ARP_SIP_MASK,
3373	len: sizeof(key->arp.sip)) ||
3374	fl_dump_key_val(skb, val: &key->arp.tip,
3375	val_type: TCA_FLOWER_KEY_ARP_TIP, mask: &mask->arp.tip,
3376	mask_type: TCA_FLOWER_KEY_ARP_TIP_MASK,
3377	len: sizeof(key->arp.tip)) ||
3378	fl_dump_key_val(skb, val: &key->arp.op,
3379	val_type: TCA_FLOWER_KEY_ARP_OP, mask: &mask->arp.op,
3380	mask_type: TCA_FLOWER_KEY_ARP_OP_MASK,
3381	len: sizeof(key->arp.op)) ||
3382	fl_dump_key_val(skb, val: key->arp.sha, val_type: TCA_FLOWER_KEY_ARP_SHA,
3383	mask: mask->arp.sha, mask_type: TCA_FLOWER_KEY_ARP_SHA_MASK,
3384	len: sizeof(key->arp.sha)) ||
3385	fl_dump_key_val(skb, val: key->arp.tha, val_type: TCA_FLOWER_KEY_ARP_THA,
3386	mask: mask->arp.tha, mask_type: TCA_FLOWER_KEY_ARP_THA_MASK,
3387	len: sizeof(key->arp.tha))))
3388	goto nla_put_failure;
3389	else if (key->basic.ip_proto == IPPROTO_L2TP &&
3390	fl_dump_key_val(skb, val: &key->l2tpv3.session_id,
3391	val_type: TCA_FLOWER_KEY_L2TPV3_SID,
3392	mask: &mask->l2tpv3.session_id,
3393	mask_type: TCA_FLOWER_UNSPEC,
3394	len: sizeof(key->l2tpv3.session_id)))
3395	goto nla_put_failure;
3396
3397	if (key->ipsec.spi &&
3398	fl_dump_key_val(skb, val: &key->ipsec.spi, val_type: TCA_FLOWER_KEY_SPI,
3399	mask: &mask->ipsec.spi, mask_type: TCA_FLOWER_KEY_SPI_MASK,
3400	len: sizeof(key->ipsec.spi)))
3401	goto nla_put_failure;
3402
3403	if ((key->basic.ip_proto == IPPROTO_TCP ||
3404	key->basic.ip_proto == IPPROTO_UDP ||
3405	key->basic.ip_proto == IPPROTO_SCTP) &&
3406	fl_dump_key_port_range(skb, key, mask))
3407	goto nla_put_failure;
3408
3409	if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
3410	(fl_dump_key_val(skb, val: &key->enc_ipv4.src,
3411	val_type: TCA_FLOWER_KEY_ENC_IPV4_SRC, mask: &mask->enc_ipv4.src,
3412	mask_type: TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
3413	len: sizeof(key->enc_ipv4.src)) ||
3414	fl_dump_key_val(skb, val: &key->enc_ipv4.dst,
3415	val_type: TCA_FLOWER_KEY_ENC_IPV4_DST, mask: &mask->enc_ipv4.dst,
3416	mask_type: TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
3417	len: sizeof(key->enc_ipv4.dst))))
3418	goto nla_put_failure;
3419	else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
3420	(fl_dump_key_val(skb, val: &key->enc_ipv6.src,
3421	val_type: TCA_FLOWER_KEY_ENC_IPV6_SRC, mask: &mask->enc_ipv6.src,
3422	mask_type: TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
3423	len: sizeof(key->enc_ipv6.src)) ||
3424	fl_dump_key_val(skb, val: &key->enc_ipv6.dst,
3425	val_type: TCA_FLOWER_KEY_ENC_IPV6_DST,
3426	mask: &mask->enc_ipv6.dst,
3427	mask_type: TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
3428	len: sizeof(key->enc_ipv6.dst))))
3429	goto nla_put_failure;
3430
3431	if (fl_dump_key_val(skb, val: &key->enc_key_id, val_type: TCA_FLOWER_KEY_ENC_KEY_ID,
3432	mask: &mask->enc_key_id, mask_type: TCA_FLOWER_UNSPEC,
3433	len: sizeof(key->enc_key_id)) ||
3434	fl_dump_key_val(skb, val: &key->enc_tp.src,
3435	val_type: TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
3436	mask: &mask->enc_tp.src,
3437	mask_type: TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
3438	len: sizeof(key->enc_tp.src)) ||
3439	fl_dump_key_val(skb, val: &key->enc_tp.dst,
3440	val_type: TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
3441	mask: &mask->enc_tp.dst,
3442	mask_type: TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
3443	len: sizeof(key->enc_tp.dst)) ||
3444	fl_dump_key_ip(skb, encap: true, key: &key->enc_ip, mask: &mask->enc_ip) ||
3445	fl_dump_key_enc_opt(skb, key_opts: &key->enc_opts, msk_opts: &mask->enc_opts))
3446	goto nla_put_failure;
3447
3448	if (fl_dump_key_ct(skb, key: &key->ct, mask: &mask->ct))
3449	goto nla_put_failure;
3450
3451	if (fl_dump_key_flags(skb, flags_key: key->control.flags, flags_mask: mask->control.flags))
3452	goto nla_put_failure;
3453
3454	if (fl_dump_key_val(skb, val: &key->hash.hash, val_type: TCA_FLOWER_KEY_HASH,
3455	mask: &mask->hash.hash, mask_type: TCA_FLOWER_KEY_HASH_MASK,
3456	len: sizeof(key->hash.hash)))
3457	goto nla_put_failure;
3458
3459	if (fl_dump_key_cfm(skb, key: &key->cfm, mask: &mask->cfm))
3460	goto nla_put_failure;
3461
3462	return 0;
3463
3464	nla_put_failure:
3465	return -EMSGSIZE;
3466	}
3467
3468	static int fl_dump(struct net *net, struct tcf_proto *tp, void *fh,
3469	struct sk_buff *skb, struct tcmsg *t, bool rtnl_held)
3470	{
3471	struct cls_fl_filter *f = fh;
3472	struct nlattr *nest;
3473	struct fl_flow_key *key, *mask;
3474	bool skip_hw;
3475
3476	if (!f)
3477	return skb->len;
3478
3479	t->tcm_handle = f->handle;
3480
3481	nest = nla_nest_start_noflag(skb, attrtype: TCA_OPTIONS);
3482	if (!nest)
3483	goto nla_put_failure;
3484
3485	spin_lock(lock: &tp->lock);
3486
3487	if (f->res.classid &&
3488	nla_put_u32(skb, attrtype: TCA_FLOWER_CLASSID, value: f->res.classid))
3489	goto nla_put_failure_locked;
3490
3491	key = &f->key;
3492	mask = &f->mask->key;
3493	skip_hw = tc_skip_hw(flags: f->flags);
3494
3495	if (fl_dump_key(skb, net, key, mask))
3496	goto nla_put_failure_locked;
3497
3498	if (f->flags && nla_put_u32(skb, attrtype: TCA_FLOWER_FLAGS, value: f->flags))
3499	goto nla_put_failure_locked;
3500
3501	spin_unlock(lock: &tp->lock);
3502
3503	if (!skip_hw)
3504	fl_hw_update_stats(tp, f, rtnl_held);
3505
3506	if (nla_put_u32(skb, attrtype: TCA_FLOWER_IN_HW_COUNT, value: f->in_hw_count))
3507	goto nla_put_failure;
3508
3509	if (tcf_exts_dump(skb, exts: &f->exts))
3510	goto nla_put_failure;
3511
3512	nla_nest_end(skb, start: nest);
3513
3514	if (tcf_exts_dump_stats(skb, exts: &f->exts) < 0)
3515	goto nla_put_failure;
3516
3517	return skb->len;
3518
3519	nla_put_failure_locked:
3520	spin_unlock(lock: &tp->lock);
3521	nla_put_failure:
3522	nla_nest_cancel(skb, start: nest);
3523	return -1;
3524	}
3525
3526	static int fl_terse_dump(struct net *net, struct tcf_proto *tp, void *fh,
3527	struct sk_buff *skb, struct tcmsg *t, bool rtnl_held)
3528	{
3529	struct cls_fl_filter *f = fh;
3530	struct nlattr *nest;
3531	bool skip_hw;
3532
3533	if (!f)
3534	return skb->len;
3535
3536	t->tcm_handle = f->handle;
3537
3538	nest = nla_nest_start_noflag(skb, attrtype: TCA_OPTIONS);
3539	if (!nest)
3540	goto nla_put_failure;
3541
3542	spin_lock(lock: &tp->lock);
3543
3544	skip_hw = tc_skip_hw(flags: f->flags);
3545
3546	if (f->flags && nla_put_u32(skb, attrtype: TCA_FLOWER_FLAGS, value: f->flags))
3547	goto nla_put_failure_locked;
3548
3549	spin_unlock(lock: &tp->lock);
3550
3551	if (!skip_hw)
3552	fl_hw_update_stats(tp, f, rtnl_held);
3553
3554	if (tcf_exts_terse_dump(skb, exts: &f->exts))
3555	goto nla_put_failure;
3556
3557	nla_nest_end(skb, start: nest);
3558
3559	return skb->len;
3560
3561	nla_put_failure_locked:
3562	spin_unlock(lock: &tp->lock);
3563	nla_put_failure:
3564	nla_nest_cancel(skb, start: nest);
3565	return -1;
3566	}
3567
3568	static int fl_tmplt_dump(struct sk_buff *skb, struct net *net, void *tmplt_priv)
3569	{
3570	struct fl_flow_tmplt *tmplt = tmplt_priv;
3571	struct fl_flow_key *key, *mask;
3572	struct nlattr *nest;
3573
3574	nest = nla_nest_start_noflag(skb, attrtype: TCA_OPTIONS);
3575	if (!nest)
3576	goto nla_put_failure;
3577
3578	key = &tmplt->dummy_key;
3579	mask = &tmplt->mask;
3580
3581	if (fl_dump_key(skb, net, key, mask))
3582	goto nla_put_failure;
3583
3584	nla_nest_end(skb, start: nest);
3585
3586	return skb->len;
3587
3588	nla_put_failure:
3589	nla_nest_cancel(skb, start: nest);
3590	return -EMSGSIZE;
3591	}
3592
3593	static void fl_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
3594	unsigned long base)
3595	{
3596	struct cls_fl_filter *f = fh;
3597
3598	tc_cls_bind_class(classid, cl, q, res: &f->res, base);
3599	}
3600
3601	static bool fl_delete_empty(struct tcf_proto *tp)
3602	{
3603	struct cls_fl_head *head = fl_head_dereference(tp);
3604
3605	spin_lock(lock: &tp->lock);
3606	tp->deleting = idr_is_empty(idr: &head->handle_idr);
3607	spin_unlock(lock: &tp->lock);
3608
3609	return tp->deleting;
3610	}
3611
3612	static struct tcf_proto_ops cls_fl_ops __read_mostly = {
3613	.kind = "flower",
3614	.classify = fl_classify,
3615	.init = fl_init,
3616	.destroy = fl_destroy,
3617	.get = fl_get,
3618	.put = fl_put,
3619	.change = fl_change,
3620	.delete = fl_delete,
3621	.delete_empty = fl_delete_empty,
3622	.walk = fl_walk,
3623	.reoffload = fl_reoffload,
3624	.hw_add = fl_hw_add,
3625	.hw_del = fl_hw_del,
3626	.dump = fl_dump,
3627	.terse_dump = fl_terse_dump,
3628	.bind_class = fl_bind_class,
3629	.tmplt_create = fl_tmplt_create,
3630	.tmplt_destroy = fl_tmplt_destroy,
3631	.tmplt_dump = fl_tmplt_dump,
3632	.get_exts = fl_get_exts,
3633	.owner = THIS_MODULE,
3634	.flags = TCF_PROTO_OPS_DOIT_UNLOCKED,
3635	};
3636
3637	static int __init cls_fl_init(void)
3638	{
3639	return register_tcf_proto_ops(ops: &cls_fl_ops);
3640	}
3641
3642	static void __exit cls_fl_exit(void)
3643	{
3644	unregister_tcf_proto_ops(ops: &cls_fl_ops);
3645	}
3646
3647	module_init(cls_fl_init);
3648	module_exit(cls_fl_exit);
3649
3650	MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
3651	MODULE_DESCRIPTION("Flower classifier");
3652	MODULE_LICENSE("GPL v2");
3653