1// SPDX-License-Identifier: GPL-2.0-only
2/* Unstable Conntrack Helpers for XDP and TC-BPF hook
3 *
4 * These are called from the XDP and SCHED_CLS BPF programs. Note that it is
5 * allowed to break compatibility for these functions since the interface they
6 * are exposed through to BPF programs is explicitly unstable.
7 */
8
9#include <linux/bpf_verifier.h>
10#include <linux/bpf.h>
11#include <linux/btf.h>
12#include <linux/filter.h>
13#include <linux/mutex.h>
14#include <linux/types.h>
15#include <linux/btf_ids.h>
16#include <linux/net_namespace.h>
17#include <net/xdp.h>
18#include <net/netfilter/nf_conntrack_bpf.h>
19#include <net/netfilter/nf_conntrack_core.h>
20
21/* bpf_ct_opts - Options for CT lookup helpers
22 *
23 * Members:
24 * @netns_id - Specify the network namespace for lookup
25 * Values:
26 * BPF_F_CURRENT_NETNS (-1)
27 * Use namespace associated with ctx (xdp_md, __sk_buff)
28 * [0, S32_MAX]
29 * Network Namespace ID
30 * @error - Out parameter, set for any errors encountered
31 * Values:
32 * -EINVAL - Passed NULL for bpf_tuple pointer
33 * -EINVAL - opts->reserved is not 0
34 * -EINVAL - netns_id is less than -1
35 * -EINVAL - opts__sz isn't NF_BPF_CT_OPTS_SZ (12)
36 * -EPROTO - l4proto isn't one of IPPROTO_TCP or IPPROTO_UDP
37 * -ENONET - No network namespace found for netns_id
38 * -ENOENT - Conntrack lookup could not find entry for tuple
39 * -EAFNOSUPPORT - tuple__sz isn't one of sizeof(tuple->ipv4)
40 * or sizeof(tuple->ipv6)
41 * @l4proto - Layer 4 protocol
42 * Values:
43 * IPPROTO_TCP, IPPROTO_UDP
44 * @dir: - connection tracking tuple direction.
45 * @reserved - Reserved member, will be reused for more options in future
46 * Values:
47 * 0
48 */
49struct bpf_ct_opts {
50 s32 netns_id;
51 s32 error;
52 u8 l4proto;
53 u8 dir;
54 u8 reserved[2];
55};
56
57enum {
58 NF_BPF_CT_OPTS_SZ = 12,
59};
60
61static int bpf_nf_ct_tuple_parse(struct bpf_sock_tuple *bpf_tuple,
62 u32 tuple_len, u8 protonum, u8 dir,
63 struct nf_conntrack_tuple *tuple)
64{
65 union nf_inet_addr *src = dir ? &tuple->dst.u3 : &tuple->src.u3;
66 union nf_inet_addr *dst = dir ? &tuple->src.u3 : &tuple->dst.u3;
67 union nf_conntrack_man_proto *sport = dir ? (void *)&tuple->dst.u
68 : &tuple->src.u;
69 union nf_conntrack_man_proto *dport = dir ? &tuple->src.u
70 : (void *)&tuple->dst.u;
71
72 if (unlikely(protonum != IPPROTO_TCP && protonum != IPPROTO_UDP))
73 return -EPROTO;
74
75 memset(tuple, 0, sizeof(*tuple));
76
77 switch (tuple_len) {
78 case sizeof(bpf_tuple->ipv4):
79 tuple->src.l3num = AF_INET;
80 src->ip = bpf_tuple->ipv4.saddr;
81 sport->tcp.port = bpf_tuple->ipv4.sport;
82 dst->ip = bpf_tuple->ipv4.daddr;
83 dport->tcp.port = bpf_tuple->ipv4.dport;
84 break;
85 case sizeof(bpf_tuple->ipv6):
86 tuple->src.l3num = AF_INET6;
87 memcpy(src->ip6, bpf_tuple->ipv6.saddr, sizeof(bpf_tuple->ipv6.saddr));
88 sport->tcp.port = bpf_tuple->ipv6.sport;
89 memcpy(dst->ip6, bpf_tuple->ipv6.daddr, sizeof(bpf_tuple->ipv6.daddr));
90 dport->tcp.port = bpf_tuple->ipv6.dport;
91 break;
92 default:
93 return -EAFNOSUPPORT;
94 }
95 tuple->dst.protonum = protonum;
96 tuple->dst.dir = dir;
97
98 return 0;
99}
100
101static struct nf_conn *
102__bpf_nf_ct_alloc_entry(struct net *net, struct bpf_sock_tuple *bpf_tuple,
103 u32 tuple_len, struct bpf_ct_opts *opts, u32 opts_len,
104 u32 timeout)
105{
106 struct nf_conntrack_tuple otuple, rtuple;
107 struct nf_conn *ct;
108 int err;
109
110 if (!opts || !bpf_tuple || opts->reserved[0] || opts->reserved[1] ||
111 opts_len != NF_BPF_CT_OPTS_SZ)
112 return ERR_PTR(error: -EINVAL);
113
114 if (unlikely(opts->netns_id < BPF_F_CURRENT_NETNS))
115 return ERR_PTR(error: -EINVAL);
116
117 err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, protonum: opts->l4proto,
118 dir: IP_CT_DIR_ORIGINAL, tuple: &otuple);
119 if (err < 0)
120 return ERR_PTR(error: err);
121
122 err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, protonum: opts->l4proto,
123 dir: IP_CT_DIR_REPLY, tuple: &rtuple);
124 if (err < 0)
125 return ERR_PTR(error: err);
126
127 if (opts->netns_id >= 0) {
128 net = get_net_ns_by_id(net, id: opts->netns_id);
129 if (unlikely(!net))
130 return ERR_PTR(error: -ENONET);
131 }
132
133 ct = nf_conntrack_alloc(net, zone: &nf_ct_zone_dflt, orig: &otuple, repl: &rtuple,
134 GFP_ATOMIC);
135 if (IS_ERR(ptr: ct))
136 goto out;
137
138 memset(&ct->proto, 0, sizeof(ct->proto));
139 __nf_ct_set_timeout(ct, timeout: timeout * HZ);
140
141out:
142 if (opts->netns_id >= 0)
143 put_net(net);
144
145 return ct;
146}
147
148static struct nf_conn *__bpf_nf_ct_lookup(struct net *net,
149 struct bpf_sock_tuple *bpf_tuple,
150 u32 tuple_len, struct bpf_ct_opts *opts,
151 u32 opts_len)
152{
153 struct nf_conntrack_tuple_hash *hash;
154 struct nf_conntrack_tuple tuple;
155 struct nf_conn *ct;
156 int err;
157
158 if (!opts || !bpf_tuple || opts->reserved[0] || opts->reserved[1] ||
159 opts_len != NF_BPF_CT_OPTS_SZ)
160 return ERR_PTR(error: -EINVAL);
161 if (unlikely(opts->l4proto != IPPROTO_TCP && opts->l4proto != IPPROTO_UDP))
162 return ERR_PTR(error: -EPROTO);
163 if (unlikely(opts->netns_id < BPF_F_CURRENT_NETNS))
164 return ERR_PTR(error: -EINVAL);
165
166 err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, protonum: opts->l4proto,
167 dir: IP_CT_DIR_ORIGINAL, tuple: &tuple);
168 if (err < 0)
169 return ERR_PTR(error: err);
170
171 if (opts->netns_id >= 0) {
172 net = get_net_ns_by_id(net, id: opts->netns_id);
173 if (unlikely(!net))
174 return ERR_PTR(error: -ENONET);
175 }
176
177 hash = nf_conntrack_find_get(net, zone: &nf_ct_zone_dflt, tuple: &tuple);
178 if (opts->netns_id >= 0)
179 put_net(net);
180 if (!hash)
181 return ERR_PTR(error: -ENOENT);
182
183 ct = nf_ct_tuplehash_to_ctrack(hash);
184 opts->dir = NF_CT_DIRECTION(hash);
185
186 return ct;
187}
188
189BTF_ID_LIST(btf_nf_conn_ids)
190BTF_ID(struct, nf_conn)
191BTF_ID(struct, nf_conn___init)
192
193/* Check writes into `struct nf_conn` */
194static int _nf_conntrack_btf_struct_access(struct bpf_verifier_log *log,
195 const struct bpf_reg_state *reg,
196 int off, int size)
197{
198 const struct btf_type *ncit, *nct, *t;
199 size_t end;
200
201 ncit = btf_type_by_id(btf: reg->btf, type_id: btf_nf_conn_ids[1]);
202 nct = btf_type_by_id(btf: reg->btf, type_id: btf_nf_conn_ids[0]);
203 t = btf_type_by_id(btf: reg->btf, type_id: reg->btf_id);
204 if (t != nct && t != ncit) {
205 bpf_log(log, fmt: "only read is supported\n");
206 return -EACCES;
207 }
208
209 /* `struct nf_conn` and `struct nf_conn___init` have the same layout
210 * so we are safe to simply merge offset checks here
211 */
212 switch (off) {
213#if defined(CONFIG_NF_CONNTRACK_MARK)
214 case offsetof(struct nf_conn, mark):
215 end = offsetofend(struct nf_conn, mark);
216 break;
217#endif
218 default:
219 bpf_log(log, fmt: "no write support to nf_conn at off %d\n", off);
220 return -EACCES;
221 }
222
223 if (off + size > end) {
224 bpf_log(log,
225 fmt: "write access at off %d with size %d beyond the member of nf_conn ended at %zu\n",
226 off, size, end);
227 return -EACCES;
228 }
229
230 return 0;
231}
232
233__bpf_kfunc_start_defs();
234
235/* bpf_xdp_ct_alloc - Allocate a new CT entry
236 *
237 * Parameters:
238 * @xdp_ctx - Pointer to ctx (xdp_md) in XDP program
239 * Cannot be NULL
240 * @bpf_tuple - Pointer to memory representing the tuple to look up
241 * Cannot be NULL
242 * @tuple__sz - Length of the tuple structure
243 * Must be one of sizeof(bpf_tuple->ipv4) or
244 * sizeof(bpf_tuple->ipv6)
245 * @opts - Additional options for allocation (documented above)
246 * Cannot be NULL
247 * @opts__sz - Length of the bpf_ct_opts structure
248 * Must be NF_BPF_CT_OPTS_SZ (12)
249 */
250__bpf_kfunc struct nf_conn___init *
251bpf_xdp_ct_alloc(struct xdp_md *xdp_ctx, struct bpf_sock_tuple *bpf_tuple,
252 u32 tuple__sz, struct bpf_ct_opts *opts, u32 opts__sz)
253{
254 struct xdp_buff *ctx = (struct xdp_buff *)xdp_ctx;
255 struct nf_conn *nfct;
256
257 nfct = __bpf_nf_ct_alloc_entry(net: dev_net(dev: ctx->rxq->dev), bpf_tuple, tuple_len: tuple__sz,
258 opts, opts_len: opts__sz, timeout: 10);
259 if (IS_ERR(ptr: nfct)) {
260 if (opts)
261 opts->error = PTR_ERR(ptr: nfct);
262 return NULL;
263 }
264
265 return (struct nf_conn___init *)nfct;
266}
267
268/* bpf_xdp_ct_lookup - Lookup CT entry for the given tuple, and acquire a
269 * reference to it
270 *
271 * Parameters:
272 * @xdp_ctx - Pointer to ctx (xdp_md) in XDP program
273 * Cannot be NULL
274 * @bpf_tuple - Pointer to memory representing the tuple to look up
275 * Cannot be NULL
276 * @tuple__sz - Length of the tuple structure
277 * Must be one of sizeof(bpf_tuple->ipv4) or
278 * sizeof(bpf_tuple->ipv6)
279 * @opts - Additional options for lookup (documented above)
280 * Cannot be NULL
281 * @opts__sz - Length of the bpf_ct_opts structure
282 * Must be NF_BPF_CT_OPTS_SZ (12)
283 */
284__bpf_kfunc struct nf_conn *
285bpf_xdp_ct_lookup(struct xdp_md *xdp_ctx, struct bpf_sock_tuple *bpf_tuple,
286 u32 tuple__sz, struct bpf_ct_opts *opts, u32 opts__sz)
287{
288 struct xdp_buff *ctx = (struct xdp_buff *)xdp_ctx;
289 struct net *caller_net;
290 struct nf_conn *nfct;
291
292 caller_net = dev_net(dev: ctx->rxq->dev);
293 nfct = __bpf_nf_ct_lookup(net: caller_net, bpf_tuple, tuple_len: tuple__sz, opts, opts_len: opts__sz);
294 if (IS_ERR(ptr: nfct)) {
295 if (opts)
296 opts->error = PTR_ERR(ptr: nfct);
297 return NULL;
298 }
299 return nfct;
300}
301
302/* bpf_skb_ct_alloc - Allocate a new CT entry
303 *
304 * Parameters:
305 * @skb_ctx - Pointer to ctx (__sk_buff) in TC program
306 * Cannot be NULL
307 * @bpf_tuple - Pointer to memory representing the tuple to look up
308 * Cannot be NULL
309 * @tuple__sz - Length of the tuple structure
310 * Must be one of sizeof(bpf_tuple->ipv4) or
311 * sizeof(bpf_tuple->ipv6)
312 * @opts - Additional options for allocation (documented above)
313 * Cannot be NULL
314 * @opts__sz - Length of the bpf_ct_opts structure
315 * Must be NF_BPF_CT_OPTS_SZ (12)
316 */
317__bpf_kfunc struct nf_conn___init *
318bpf_skb_ct_alloc(struct __sk_buff *skb_ctx, struct bpf_sock_tuple *bpf_tuple,
319 u32 tuple__sz, struct bpf_ct_opts *opts, u32 opts__sz)
320{
321 struct sk_buff *skb = (struct sk_buff *)skb_ctx;
322 struct nf_conn *nfct;
323 struct net *net;
324
325 net = skb->dev ? dev_net(dev: skb->dev) : sock_net(sk: skb->sk);
326 nfct = __bpf_nf_ct_alloc_entry(net, bpf_tuple, tuple_len: tuple__sz, opts, opts_len: opts__sz, timeout: 10);
327 if (IS_ERR(ptr: nfct)) {
328 if (opts)
329 opts->error = PTR_ERR(ptr: nfct);
330 return NULL;
331 }
332
333 return (struct nf_conn___init *)nfct;
334}
335
336/* bpf_skb_ct_lookup - Lookup CT entry for the given tuple, and acquire a
337 * reference to it
338 *
339 * Parameters:
340 * @skb_ctx - Pointer to ctx (__sk_buff) in TC program
341 * Cannot be NULL
342 * @bpf_tuple - Pointer to memory representing the tuple to look up
343 * Cannot be NULL
344 * @tuple__sz - Length of the tuple structure
345 * Must be one of sizeof(bpf_tuple->ipv4) or
346 * sizeof(bpf_tuple->ipv6)
347 * @opts - Additional options for lookup (documented above)
348 * Cannot be NULL
349 * @opts__sz - Length of the bpf_ct_opts structure
350 * Must be NF_BPF_CT_OPTS_SZ (12)
351 */
352__bpf_kfunc struct nf_conn *
353bpf_skb_ct_lookup(struct __sk_buff *skb_ctx, struct bpf_sock_tuple *bpf_tuple,
354 u32 tuple__sz, struct bpf_ct_opts *opts, u32 opts__sz)
355{
356 struct sk_buff *skb = (struct sk_buff *)skb_ctx;
357 struct net *caller_net;
358 struct nf_conn *nfct;
359
360 caller_net = skb->dev ? dev_net(dev: skb->dev) : sock_net(sk: skb->sk);
361 nfct = __bpf_nf_ct_lookup(net: caller_net, bpf_tuple, tuple_len: tuple__sz, opts, opts_len: opts__sz);
362 if (IS_ERR(ptr: nfct)) {
363 if (opts)
364 opts->error = PTR_ERR(ptr: nfct);
365 return NULL;
366 }
367 return nfct;
368}
369
370/* bpf_ct_insert_entry - Add the provided entry into a CT map
371 *
372 * This must be invoked for referenced PTR_TO_BTF_ID.
373 *
374 * @nfct - Pointer to referenced nf_conn___init object, obtained
375 * using bpf_xdp_ct_alloc or bpf_skb_ct_alloc.
376 */
377__bpf_kfunc struct nf_conn *bpf_ct_insert_entry(struct nf_conn___init *nfct_i)
378{
379 struct nf_conn *nfct = (struct nf_conn *)nfct_i;
380 int err;
381
382 if (!nf_ct_is_confirmed(ct: nfct))
383 nfct->timeout += nfct_time_stamp;
384 nfct->status |= IPS_CONFIRMED;
385 err = nf_conntrack_hash_check_insert(ct: nfct);
386 if (err < 0) {
387 nf_conntrack_free(ct: nfct);
388 return NULL;
389 }
390 return nfct;
391}
392
393/* bpf_ct_release - Release acquired nf_conn object
394 *
395 * This must be invoked for referenced PTR_TO_BTF_ID, and the verifier rejects
396 * the program if any references remain in the program in all of the explored
397 * states.
398 *
399 * Parameters:
400 * @nf_conn - Pointer to referenced nf_conn object, obtained using
401 * bpf_xdp_ct_lookup or bpf_skb_ct_lookup.
402 */
403__bpf_kfunc void bpf_ct_release(struct nf_conn *nfct)
404{
405 nf_ct_put(ct: nfct);
406}
407
408/* bpf_ct_set_timeout - Set timeout of allocated nf_conn
409 *
410 * Sets the default timeout of newly allocated nf_conn before insertion.
411 * This helper must be invoked for refcounted pointer to nf_conn___init.
412 *
413 * Parameters:
414 * @nfct - Pointer to referenced nf_conn object, obtained using
415 * bpf_xdp_ct_alloc or bpf_skb_ct_alloc.
416 * @timeout - Timeout in msecs.
417 */
418__bpf_kfunc void bpf_ct_set_timeout(struct nf_conn___init *nfct, u32 timeout)
419{
420 __nf_ct_set_timeout(ct: (struct nf_conn *)nfct, timeout: msecs_to_jiffies(m: timeout));
421}
422
423/* bpf_ct_change_timeout - Change timeout of inserted nf_conn
424 *
425 * Change timeout associated of the inserted or looked up nf_conn.
426 * This helper must be invoked for refcounted pointer to nf_conn.
427 *
428 * Parameters:
429 * @nfct - Pointer to referenced nf_conn object, obtained using
430 * bpf_ct_insert_entry, bpf_xdp_ct_lookup, or bpf_skb_ct_lookup.
431 * @timeout - New timeout in msecs.
432 */
433__bpf_kfunc int bpf_ct_change_timeout(struct nf_conn *nfct, u32 timeout)
434{
435 return __nf_ct_change_timeout(ct: nfct, cta_timeout: msecs_to_jiffies(m: timeout));
436}
437
438/* bpf_ct_set_status - Set status field of allocated nf_conn
439 *
440 * Set the status field of the newly allocated nf_conn before insertion.
441 * This must be invoked for referenced PTR_TO_BTF_ID to nf_conn___init.
442 *
443 * Parameters:
444 * @nfct - Pointer to referenced nf_conn object, obtained using
445 * bpf_xdp_ct_alloc or bpf_skb_ct_alloc.
446 * @status - New status value.
447 */
448__bpf_kfunc int bpf_ct_set_status(const struct nf_conn___init *nfct, u32 status)
449{
450 return nf_ct_change_status_common(ct: (struct nf_conn *)nfct, status);
451}
452
453/* bpf_ct_change_status - Change status of inserted nf_conn
454 *
455 * Change the status field of the provided connection tracking entry.
456 * This must be invoked for referenced PTR_TO_BTF_ID to nf_conn.
457 *
458 * Parameters:
459 * @nfct - Pointer to referenced nf_conn object, obtained using
460 * bpf_ct_insert_entry, bpf_xdp_ct_lookup or bpf_skb_ct_lookup.
461 * @status - New status value.
462 */
463__bpf_kfunc int bpf_ct_change_status(struct nf_conn *nfct, u32 status)
464{
465 return nf_ct_change_status_common(ct: nfct, status);
466}
467
468__bpf_kfunc_end_defs();
469
470BTF_KFUNCS_START(nf_ct_kfunc_set)
471BTF_ID_FLAGS(func, bpf_xdp_ct_alloc, KF_ACQUIRE | KF_RET_NULL)
472BTF_ID_FLAGS(func, bpf_xdp_ct_lookup, KF_ACQUIRE | KF_RET_NULL)
473BTF_ID_FLAGS(func, bpf_skb_ct_alloc, KF_ACQUIRE | KF_RET_NULL)
474BTF_ID_FLAGS(func, bpf_skb_ct_lookup, KF_ACQUIRE | KF_RET_NULL)
475BTF_ID_FLAGS(func, bpf_ct_insert_entry, KF_ACQUIRE | KF_RET_NULL | KF_RELEASE)
476BTF_ID_FLAGS(func, bpf_ct_release, KF_RELEASE)
477BTF_ID_FLAGS(func, bpf_ct_set_timeout, KF_TRUSTED_ARGS)
478BTF_ID_FLAGS(func, bpf_ct_change_timeout, KF_TRUSTED_ARGS)
479BTF_ID_FLAGS(func, bpf_ct_set_status, KF_TRUSTED_ARGS)
480BTF_ID_FLAGS(func, bpf_ct_change_status, KF_TRUSTED_ARGS)
481BTF_KFUNCS_END(nf_ct_kfunc_set)
482
483static const struct btf_kfunc_id_set nf_conntrack_kfunc_set = {
484 .owner = THIS_MODULE,
485 .set = &nf_ct_kfunc_set,
486};
487
488int register_nf_conntrack_bpf(void)
489{
490 int ret;
491
492 ret = register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_XDP, s: &nf_conntrack_kfunc_set);
493 ret = ret ?: register_btf_kfunc_id_set(prog_type: BPF_PROG_TYPE_SCHED_CLS, s: &nf_conntrack_kfunc_set);
494 if (!ret) {
495 mutex_lock(&nf_conn_btf_access_lock);
496 nfct_btf_struct_access = _nf_conntrack_btf_struct_access;
497 mutex_unlock(lock: &nf_conn_btf_access_lock);
498 }
499
500 return ret;
501}
502
503void cleanup_nf_conntrack_bpf(void)
504{
505 mutex_lock(&nf_conn_btf_access_lock);
506 nfct_btf_struct_access = NULL;
507 mutex_unlock(lock: &nf_conn_btf_access_lock);
508}
509

source code of linux/net/netfilter/nf_conntrack_bpf.c