1/* SPDX-License-Identifier: GPL-2.0 */
2#include <linux/init.h>
3#include <linux/module.h>
4#include <linux/netfilter.h>
5#include <net/flow_offload.h>
6#include <net/netfilter/nf_tables.h>
7#include <net/netfilter/nf_tables_offload.h>
8#include <net/pkt_cls.h>
9
10static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
11{
12 struct nft_flow_rule *flow;
13
14 flow = kzalloc(size: sizeof(struct nft_flow_rule), GFP_KERNEL);
15 if (!flow)
16 return NULL;
17
18 flow->rule = flow_rule_alloc(num_actions);
19 if (!flow->rule) {
20 kfree(objp: flow);
21 return NULL;
22 }
23
24 flow->rule->match.dissector = &flow->match.dissector;
25 flow->rule->match.mask = &flow->match.mask;
26 flow->rule->match.key = &flow->match.key;
27
28 return flow;
29}
30
31void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow,
32 enum flow_dissector_key_id addr_type)
33{
34 struct nft_flow_match *match = &flow->match;
35 struct nft_flow_key *mask = &match->mask;
36 struct nft_flow_key *key = &match->key;
37
38 if (match->dissector.used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL))
39 return;
40
41 key->control.addr_type = addr_type;
42 mask->control.addr_type = 0xffff;
43 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL);
44 match->dissector.offset[FLOW_DISSECTOR_KEY_CONTROL] =
45 offsetof(struct nft_flow_key, control);
46}
47
48struct nft_offload_ethertype {
49 __be16 value;
50 __be16 mask;
51};
52
53static void nft_flow_rule_transfer_vlan(struct nft_offload_ctx *ctx,
54 struct nft_flow_rule *flow)
55{
56 struct nft_flow_match *match = &flow->match;
57 struct nft_offload_ethertype ethertype = {
58 .value = match->key.basic.n_proto,
59 .mask = match->mask.basic.n_proto,
60 };
61
62 if (match->dissector.used_keys & BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) &&
63 (match->key.vlan.vlan_tpid == htons(ETH_P_8021Q) ||
64 match->key.vlan.vlan_tpid == htons(ETH_P_8021AD))) {
65 match->key.basic.n_proto = match->key.cvlan.vlan_tpid;
66 match->mask.basic.n_proto = match->mask.cvlan.vlan_tpid;
67 match->key.cvlan.vlan_tpid = match->key.vlan.vlan_tpid;
68 match->mask.cvlan.vlan_tpid = match->mask.vlan.vlan_tpid;
69 match->key.vlan.vlan_tpid = ethertype.value;
70 match->mask.vlan.vlan_tpid = ethertype.mask;
71 match->dissector.offset[FLOW_DISSECTOR_KEY_CVLAN] =
72 offsetof(struct nft_flow_key, cvlan);
73 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN);
74 } else if (match->dissector.used_keys &
75 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) &&
76 (match->key.basic.n_proto == htons(ETH_P_8021Q) ||
77 match->key.basic.n_proto == htons(ETH_P_8021AD))) {
78 match->key.basic.n_proto = match->key.vlan.vlan_tpid;
79 match->mask.basic.n_proto = match->mask.vlan.vlan_tpid;
80 match->key.vlan.vlan_tpid = ethertype.value;
81 match->mask.vlan.vlan_tpid = ethertype.mask;
82 match->dissector.offset[FLOW_DISSECTOR_KEY_VLAN] =
83 offsetof(struct nft_flow_key, vlan);
84 match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_VLAN);
85 }
86}
87
88struct nft_flow_rule *nft_flow_rule_create(struct net *net,
89 const struct nft_rule *rule)
90{
91 struct nft_offload_ctx *ctx;
92 struct nft_flow_rule *flow;
93 int num_actions = 0, err;
94 struct nft_expr *expr;
95
96 expr = nft_expr_first(rule);
97 while (nft_expr_more(rule, expr)) {
98 if (expr->ops->offload_action &&
99 expr->ops->offload_action(expr))
100 num_actions++;
101
102 expr = nft_expr_next(expr);
103 }
104
105 if (num_actions == 0)
106 return ERR_PTR(error: -EOPNOTSUPP);
107
108 flow = nft_flow_rule_alloc(num_actions);
109 if (!flow)
110 return ERR_PTR(error: -ENOMEM);
111
112 expr = nft_expr_first(rule);
113
114 ctx = kzalloc(size: sizeof(struct nft_offload_ctx), GFP_KERNEL);
115 if (!ctx) {
116 err = -ENOMEM;
117 goto err_out;
118 }
119 ctx->net = net;
120 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
121
122 while (nft_expr_more(rule, expr)) {
123 if (!expr->ops->offload) {
124 err = -EOPNOTSUPP;
125 goto err_out;
126 }
127 err = expr->ops->offload(ctx, flow, expr);
128 if (err < 0)
129 goto err_out;
130
131 expr = nft_expr_next(expr);
132 }
133 nft_flow_rule_transfer_vlan(ctx, flow);
134
135 flow->proto = ctx->dep.l3num;
136 kfree(objp: ctx);
137
138 return flow;
139err_out:
140 kfree(objp: ctx);
141 nft_flow_rule_destroy(flow);
142
143 return ERR_PTR(error: err);
144}
145
146void nft_flow_rule_destroy(struct nft_flow_rule *flow)
147{
148 struct flow_action_entry *entry;
149 int i;
150
151 flow_action_for_each(i, entry, &flow->rule->action) {
152 switch (entry->id) {
153 case FLOW_ACTION_REDIRECT:
154 case FLOW_ACTION_MIRRED:
155 dev_put(dev: entry->dev);
156 break;
157 default:
158 break;
159 }
160 }
161 kfree(objp: flow->rule);
162 kfree(objp: flow);
163}
164
165void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
166 enum nft_offload_dep_type type)
167{
168 ctx->dep.type = type;
169}
170
171void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
172 const void *data, u32 len)
173{
174 switch (ctx->dep.type) {
175 case NFT_OFFLOAD_DEP_NETWORK:
176 WARN_ON(len != sizeof(__u16));
177 memcpy(&ctx->dep.l3num, data, sizeof(__u16));
178 break;
179 case NFT_OFFLOAD_DEP_TRANSPORT:
180 WARN_ON(len != sizeof(__u8));
181 memcpy(&ctx->dep.protonum, data, sizeof(__u8));
182 break;
183 default:
184 break;
185 }
186 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
187}
188
189static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
190 __be16 proto, int priority,
191 struct netlink_ext_ack *extack)
192{
193 common->protocol = proto;
194 common->prio = priority;
195 common->extack = extack;
196}
197
198static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
199 struct list_head *cb_list)
200{
201 struct flow_block_cb *block_cb;
202 int err;
203
204 list_for_each_entry(block_cb, cb_list, list) {
205 err = block_cb->cb(type, type_data, block_cb->cb_priv);
206 if (err < 0)
207 return err;
208 }
209 return 0;
210}
211
212static int nft_chain_offload_priority(const struct nft_base_chain *basechain)
213{
214 if (basechain->ops.priority <= 0 ||
215 basechain->ops.priority > USHRT_MAX)
216 return -1;
217
218 return 0;
219}
220
221bool nft_chain_offload_support(const struct nft_base_chain *basechain)
222{
223 struct net_device *dev;
224 struct nft_hook *hook;
225
226 if (nft_chain_offload_priority(basechain) < 0)
227 return false;
228
229 list_for_each_entry(hook, &basechain->hook_list, list) {
230 if (hook->ops.pf != NFPROTO_NETDEV ||
231 hook->ops.hooknum != NF_NETDEV_INGRESS)
232 return false;
233
234 dev = hook->ops.dev;
235 if (!dev->netdev_ops->ndo_setup_tc && !flow_indr_dev_exists())
236 return false;
237 }
238
239 return true;
240}
241
242static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
243 const struct nft_base_chain *basechain,
244 const struct nft_rule *rule,
245 const struct nft_flow_rule *flow,
246 struct netlink_ext_ack *extack,
247 enum flow_cls_command command)
248{
249 __be16 proto = ETH_P_ALL;
250
251 memset(cls_flow, 0, sizeof(*cls_flow));
252
253 if (flow)
254 proto = flow->proto;
255
256 nft_flow_offload_common_init(common: &cls_flow->common, proto,
257 priority: basechain->ops.priority, extack);
258 cls_flow->command = command;
259 cls_flow->cookie = (unsigned long) rule;
260 if (flow)
261 cls_flow->rule = flow->rule;
262}
263
264static int nft_flow_offload_cmd(const struct nft_chain *chain,
265 const struct nft_rule *rule,
266 struct nft_flow_rule *flow,
267 enum flow_cls_command command,
268 struct flow_cls_offload *cls_flow)
269{
270 struct netlink_ext_ack extack = {};
271 struct nft_base_chain *basechain;
272
273 if (!nft_is_base_chain(chain))
274 return -EOPNOTSUPP;
275
276 basechain = nft_base_chain(chain);
277 nft_flow_cls_offload_setup(cls_flow, basechain, rule, flow, extack: &extack,
278 command);
279
280 return nft_setup_cb_call(type: TC_SETUP_CLSFLOWER, type_data: cls_flow,
281 cb_list: &basechain->flow_block.cb_list);
282}
283
284static int nft_flow_offload_rule(const struct nft_chain *chain,
285 struct nft_rule *rule,
286 struct nft_flow_rule *flow,
287 enum flow_cls_command command)
288{
289 struct flow_cls_offload cls_flow;
290
291 return nft_flow_offload_cmd(chain, rule, flow, command, cls_flow: &cls_flow);
292}
293
294int nft_flow_rule_stats(const struct nft_chain *chain,
295 const struct nft_rule *rule)
296{
297 struct flow_cls_offload cls_flow = {};
298 struct nft_expr *expr, *next;
299 int err;
300
301 err = nft_flow_offload_cmd(chain, rule, NULL, command: FLOW_CLS_STATS,
302 cls_flow: &cls_flow);
303 if (err < 0)
304 return err;
305
306 nft_rule_for_each_expr(expr, next, rule) {
307 if (expr->ops->offload_stats)
308 expr->ops->offload_stats(expr, &cls_flow.stats);
309 }
310
311 return 0;
312}
313
314static int nft_flow_offload_bind(struct flow_block_offload *bo,
315 struct nft_base_chain *basechain)
316{
317 list_splice(list: &bo->cb_list, head: &basechain->flow_block.cb_list);
318 return 0;
319}
320
321static int nft_flow_offload_unbind(struct flow_block_offload *bo,
322 struct nft_base_chain *basechain)
323{
324 struct flow_block_cb *block_cb, *next;
325 struct flow_cls_offload cls_flow;
326 struct netlink_ext_ack extack;
327 struct nft_chain *chain;
328 struct nft_rule *rule;
329
330 chain = &basechain->chain;
331 list_for_each_entry(rule, &chain->rules, list) {
332 memset(&extack, 0, sizeof(extack));
333 nft_flow_cls_offload_setup(cls_flow: &cls_flow, basechain, rule, NULL,
334 extack: &extack, command: FLOW_CLS_DESTROY);
335 nft_setup_cb_call(type: TC_SETUP_CLSFLOWER, type_data: &cls_flow, cb_list: &bo->cb_list);
336 }
337
338 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
339 list_del(entry: &block_cb->list);
340 flow_block_cb_free(block_cb);
341 }
342
343 return 0;
344}
345
346static int nft_block_setup(struct nft_base_chain *basechain,
347 struct flow_block_offload *bo,
348 enum flow_block_command cmd)
349{
350 int err;
351
352 switch (cmd) {
353 case FLOW_BLOCK_BIND:
354 err = nft_flow_offload_bind(bo, basechain);
355 break;
356 case FLOW_BLOCK_UNBIND:
357 err = nft_flow_offload_unbind(bo, basechain);
358 break;
359 default:
360 WARN_ON_ONCE(1);
361 err = -EOPNOTSUPP;
362 }
363
364 return err;
365}
366
367static void nft_flow_block_offload_init(struct flow_block_offload *bo,
368 struct net *net,
369 enum flow_block_command cmd,
370 struct nft_base_chain *basechain,
371 struct netlink_ext_ack *extack)
372{
373 memset(bo, 0, sizeof(*bo));
374 bo->net = net;
375 bo->block = &basechain->flow_block;
376 bo->command = cmd;
377 bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
378 bo->extack = extack;
379 bo->cb_list_head = &basechain->flow_block.cb_list;
380 INIT_LIST_HEAD(list: &bo->cb_list);
381}
382
383static int nft_block_offload_cmd(struct nft_base_chain *chain,
384 struct net_device *dev,
385 enum flow_block_command cmd)
386{
387 struct netlink_ext_ack extack = {};
388 struct flow_block_offload bo;
389 int err;
390
391 nft_flow_block_offload_init(bo: &bo, net: dev_net(dev), cmd, basechain: chain, extack: &extack);
392
393 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
394 if (err < 0)
395 return err;
396
397 return nft_block_setup(basechain: chain, bo: &bo, cmd);
398}
399
400static void nft_indr_block_cleanup(struct flow_block_cb *block_cb)
401{
402 struct nft_base_chain *basechain = block_cb->indr.data;
403 struct net_device *dev = block_cb->indr.dev;
404 struct netlink_ext_ack extack = {};
405 struct nftables_pernet *nft_net;
406 struct net *net = dev_net(dev);
407 struct flow_block_offload bo;
408
409 nft_flow_block_offload_init(bo: &bo, net: dev_net(dev), cmd: FLOW_BLOCK_UNBIND,
410 basechain, extack: &extack);
411 nft_net = nft_pernet(net);
412 mutex_lock(&nft_net->commit_mutex);
413 list_del(entry: &block_cb->driver_list);
414 list_move(list: &block_cb->list, head: &bo.cb_list);
415 nft_flow_offload_unbind(bo: &bo, basechain);
416 mutex_unlock(lock: &nft_net->commit_mutex);
417}
418
419static int nft_indr_block_offload_cmd(struct nft_base_chain *basechain,
420 struct net_device *dev,
421 enum flow_block_command cmd)
422{
423 struct netlink_ext_ack extack = {};
424 struct flow_block_offload bo;
425 int err;
426
427 nft_flow_block_offload_init(bo: &bo, net: dev_net(dev), cmd, basechain, extack: &extack);
428
429 err = flow_indr_dev_setup_offload(dev, NULL, type: TC_SETUP_BLOCK, data: basechain, bo: &bo,
430 cleanup: nft_indr_block_cleanup);
431 if (err < 0)
432 return err;
433
434 if (list_empty(head: &bo.cb_list))
435 return -EOPNOTSUPP;
436
437 return nft_block_setup(basechain, bo: &bo, cmd);
438}
439
440static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
441 struct net_device *dev,
442 enum flow_block_command cmd)
443{
444 int err;
445
446 if (dev->netdev_ops->ndo_setup_tc)
447 err = nft_block_offload_cmd(chain: basechain, dev, cmd);
448 else
449 err = nft_indr_block_offload_cmd(basechain, dev, cmd);
450
451 return err;
452}
453
454static int nft_flow_block_chain(struct nft_base_chain *basechain,
455 const struct net_device *this_dev,
456 enum flow_block_command cmd)
457{
458 struct net_device *dev;
459 struct nft_hook *hook;
460 int err, i = 0;
461
462 list_for_each_entry(hook, &basechain->hook_list, list) {
463 dev = hook->ops.dev;
464 if (this_dev && this_dev != dev)
465 continue;
466
467 err = nft_chain_offload_cmd(basechain, dev, cmd);
468 if (err < 0 && cmd == FLOW_BLOCK_BIND) {
469 if (!this_dev)
470 goto err_flow_block;
471
472 return err;
473 }
474 i++;
475 }
476
477 return 0;
478
479err_flow_block:
480 list_for_each_entry(hook, &basechain->hook_list, list) {
481 if (i-- <= 0)
482 break;
483
484 dev = hook->ops.dev;
485 nft_chain_offload_cmd(basechain, dev, cmd: FLOW_BLOCK_UNBIND);
486 }
487 return err;
488}
489
490static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
491 enum flow_block_command cmd)
492{
493 struct nft_base_chain *basechain;
494 u8 policy;
495
496 if (!nft_is_base_chain(chain))
497 return -EOPNOTSUPP;
498
499 basechain = nft_base_chain(chain);
500 policy = ppolicy ? *ppolicy : basechain->policy;
501
502 /* Only default policy to accept is supported for now. */
503 if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
504 return -EOPNOTSUPP;
505
506 return nft_flow_block_chain(basechain, NULL, cmd);
507}
508
509static void nft_flow_rule_offload_abort(struct net *net,
510 struct nft_trans *trans)
511{
512 struct nftables_pernet *nft_net = nft_pernet(net);
513 int err = 0;
514
515 list_for_each_entry_continue_reverse(trans, &nft_net->commit_list, list) {
516 if (trans->ctx.family != NFPROTO_NETDEV)
517 continue;
518
519 switch (trans->msg_type) {
520 case NFT_MSG_NEWCHAIN:
521 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
522 nft_trans_chain_update(trans))
523 continue;
524
525 err = nft_flow_offload_chain(chain: trans->ctx.chain, NULL,
526 cmd: FLOW_BLOCK_UNBIND);
527 break;
528 case NFT_MSG_DELCHAIN:
529 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
530 continue;
531
532 err = nft_flow_offload_chain(chain: trans->ctx.chain, NULL,
533 cmd: FLOW_BLOCK_BIND);
534 break;
535 case NFT_MSG_NEWRULE:
536 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
537 continue;
538
539 err = nft_flow_offload_rule(chain: trans->ctx.chain,
540 nft_trans_rule(trans),
541 NULL, command: FLOW_CLS_DESTROY);
542 break;
543 case NFT_MSG_DELRULE:
544 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
545 continue;
546
547 err = nft_flow_offload_rule(chain: trans->ctx.chain,
548 nft_trans_rule(trans),
549 nft_trans_flow_rule(trans),
550 command: FLOW_CLS_REPLACE);
551 break;
552 }
553
554 if (WARN_ON_ONCE(err))
555 break;
556 }
557}
558
559int nft_flow_rule_offload_commit(struct net *net)
560{
561 struct nftables_pernet *nft_net = nft_pernet(net);
562 struct nft_trans *trans;
563 int err = 0;
564 u8 policy;
565
566 list_for_each_entry(trans, &nft_net->commit_list, list) {
567 if (trans->ctx.family != NFPROTO_NETDEV)
568 continue;
569
570 switch (trans->msg_type) {
571 case NFT_MSG_NEWCHAIN:
572 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
573 nft_trans_chain_update(trans))
574 continue;
575
576 policy = nft_trans_chain_policy(trans);
577 err = nft_flow_offload_chain(chain: trans->ctx.chain, ppolicy: &policy,
578 cmd: FLOW_BLOCK_BIND);
579 break;
580 case NFT_MSG_DELCHAIN:
581 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
582 continue;
583
584 policy = nft_trans_chain_policy(trans);
585 err = nft_flow_offload_chain(chain: trans->ctx.chain, ppolicy: &policy,
586 cmd: FLOW_BLOCK_UNBIND);
587 break;
588 case NFT_MSG_NEWRULE:
589 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
590 continue;
591
592 if (trans->ctx.flags & NLM_F_REPLACE ||
593 !(trans->ctx.flags & NLM_F_APPEND)) {
594 err = -EOPNOTSUPP;
595 break;
596 }
597 err = nft_flow_offload_rule(chain: trans->ctx.chain,
598 nft_trans_rule(trans),
599 nft_trans_flow_rule(trans),
600 command: FLOW_CLS_REPLACE);
601 break;
602 case NFT_MSG_DELRULE:
603 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
604 continue;
605
606 err = nft_flow_offload_rule(chain: trans->ctx.chain,
607 nft_trans_rule(trans),
608 NULL, command: FLOW_CLS_DESTROY);
609 break;
610 }
611
612 if (err) {
613 nft_flow_rule_offload_abort(net, trans);
614 break;
615 }
616 }
617
618 return err;
619}
620
621static struct nft_chain *__nft_offload_get_chain(const struct nftables_pernet *nft_net,
622 struct net_device *dev)
623{
624 struct nft_base_chain *basechain;
625 struct nft_hook *hook, *found;
626 const struct nft_table *table;
627 struct nft_chain *chain;
628
629 list_for_each_entry(table, &nft_net->tables, list) {
630 if (table->family != NFPROTO_NETDEV)
631 continue;
632
633 list_for_each_entry(chain, &table->chains, list) {
634 if (!nft_is_base_chain(chain) ||
635 !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
636 continue;
637
638 found = NULL;
639 basechain = nft_base_chain(chain);
640 list_for_each_entry(hook, &basechain->hook_list, list) {
641 if (hook->ops.dev != dev)
642 continue;
643
644 found = hook;
645 break;
646 }
647 if (!found)
648 continue;
649
650 return chain;
651 }
652 }
653
654 return NULL;
655}
656
657static int nft_offload_netdev_event(struct notifier_block *this,
658 unsigned long event, void *ptr)
659{
660 struct net_device *dev = netdev_notifier_info_to_dev(info: ptr);
661 struct nftables_pernet *nft_net;
662 struct net *net = dev_net(dev);
663 struct nft_chain *chain;
664
665 if (event != NETDEV_UNREGISTER)
666 return NOTIFY_DONE;
667
668 nft_net = nft_pernet(net);
669 mutex_lock(&nft_net->commit_mutex);
670 chain = __nft_offload_get_chain(nft_net, dev);
671 if (chain)
672 nft_flow_block_chain(basechain: nft_base_chain(chain), this_dev: dev,
673 cmd: FLOW_BLOCK_UNBIND);
674
675 mutex_unlock(lock: &nft_net->commit_mutex);
676
677 return NOTIFY_DONE;
678}
679
680static struct notifier_block nft_offload_netdev_notifier = {
681 .notifier_call = nft_offload_netdev_event,
682};
683
684int nft_offload_init(void)
685{
686 return register_netdevice_notifier(nb: &nft_offload_netdev_notifier);
687}
688
689void nft_offload_exit(void)
690{
691 unregister_netdevice_notifier(nb: &nft_offload_netdev_notifier);
692}
693

source code of linux/net/netfilter/nf_tables_offload.c