1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2008 Patrick McHardy <kaber@trash.net> |
4 | * |
5 | * Development of this code funded by Astaro AG (http://www.astaro.com/) |
6 | */ |
7 | |
8 | #include <linux/kernel.h> |
9 | #include <linux/module.h> |
10 | #include <linux/init.h> |
11 | #include <linux/list.h> |
12 | #include <linux/rculist.h> |
13 | #include <linux/skbuff.h> |
14 | #include <linux/netlink.h> |
15 | #include <linux/netfilter.h> |
16 | #include <linux/static_key.h> |
17 | #include <linux/netfilter/nfnetlink.h> |
18 | #include <linux/netfilter/nf_tables.h> |
19 | #include <net/netfilter/nf_tables_core.h> |
20 | #include <net/netfilter/nf_tables.h> |
21 | #include <net/netfilter/nf_log.h> |
22 | #include <net/netfilter/nft_meta.h> |
23 | |
24 | #if defined(CONFIG_RETPOLINE) && defined(CONFIG_X86) |
25 | |
26 | static struct static_key_false nf_tables_skip_direct_calls; |
27 | |
28 | static bool nf_skip_indirect_calls(void) |
29 | { |
30 | return static_branch_likely(&nf_tables_skip_direct_calls); |
31 | } |
32 | |
33 | static void __init nf_skip_indirect_calls_enable(void) |
34 | { |
35 | if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE)) |
36 | static_branch_enable(&nf_tables_skip_direct_calls); |
37 | } |
38 | #else |
39 | static inline bool nf_skip_indirect_calls(void) { return false; } |
40 | |
41 | static inline void nf_skip_indirect_calls_enable(void) { } |
42 | #endif |
43 | |
44 | static noinline void __nft_trace_packet(const struct nft_pktinfo *pkt, |
45 | const struct nft_verdict *verdict, |
46 | const struct nft_rule_dp *rule, |
47 | struct nft_traceinfo *info, |
48 | enum nft_trace_types type) |
49 | { |
50 | if (!info->trace || !info->nf_trace) |
51 | return; |
52 | |
53 | info->type = type; |
54 | |
55 | nft_trace_notify(pkt, verdict, rule, info); |
56 | } |
57 | |
58 | static inline void nft_trace_packet(const struct nft_pktinfo *pkt, |
59 | struct nft_verdict *verdict, |
60 | struct nft_traceinfo *info, |
61 | const struct nft_rule_dp *rule, |
62 | enum nft_trace_types type) |
63 | { |
64 | if (static_branch_unlikely(&nft_trace_enabled)) { |
65 | info->nf_trace = pkt->skb->nf_trace; |
66 | __nft_trace_packet(pkt, verdict, rule, info, type); |
67 | } |
68 | } |
69 | |
70 | static inline void nft_trace_copy_nftrace(const struct nft_pktinfo *pkt, |
71 | struct nft_traceinfo *info) |
72 | { |
73 | if (static_branch_unlikely(&nft_trace_enabled)) |
74 | info->nf_trace = pkt->skb->nf_trace; |
75 | } |
76 | |
77 | static void nft_bitwise_fast_eval(const struct nft_expr *expr, |
78 | struct nft_regs *regs) |
79 | { |
80 | const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr); |
81 | u32 *src = ®s->data[priv->sreg]; |
82 | u32 *dst = ®s->data[priv->dreg]; |
83 | |
84 | *dst = (*src & priv->mask) ^ priv->xor; |
85 | } |
86 | |
87 | static void nft_cmp_fast_eval(const struct nft_expr *expr, |
88 | struct nft_regs *regs) |
89 | { |
90 | const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr); |
91 | |
92 | if (((regs->data[priv->sreg] & priv->mask) == priv->data) ^ priv->inv) |
93 | return; |
94 | regs->verdict.code = NFT_BREAK; |
95 | } |
96 | |
97 | static void nft_cmp16_fast_eval(const struct nft_expr *expr, |
98 | struct nft_regs *regs) |
99 | { |
100 | const struct nft_cmp16_fast_expr *priv = nft_expr_priv(expr); |
101 | const u64 *reg_data = (const u64 *)®s->data[priv->sreg]; |
102 | const u64 *mask = (const u64 *)&priv->mask; |
103 | const u64 *data = (const u64 *)&priv->data; |
104 | |
105 | if (((reg_data[0] & mask[0]) == data[0] && |
106 | ((reg_data[1] & mask[1]) == data[1])) ^ priv->inv) |
107 | return; |
108 | regs->verdict.code = NFT_BREAK; |
109 | } |
110 | |
111 | static noinline void __nft_trace_verdict(const struct nft_pktinfo *pkt, |
112 | struct nft_traceinfo *info, |
113 | const struct nft_rule_dp *rule, |
114 | const struct nft_regs *regs) |
115 | { |
116 | enum nft_trace_types type; |
117 | |
118 | switch (regs->verdict.code & NF_VERDICT_MASK) { |
119 | case NFT_CONTINUE: |
120 | case NFT_RETURN: |
121 | type = NFT_TRACETYPE_RETURN; |
122 | break; |
123 | case NF_STOLEN: |
124 | type = NFT_TRACETYPE_RULE; |
125 | /* can't access skb->nf_trace; use copy */ |
126 | break; |
127 | default: |
128 | type = NFT_TRACETYPE_RULE; |
129 | |
130 | if (info->trace) |
131 | info->nf_trace = pkt->skb->nf_trace; |
132 | break; |
133 | } |
134 | |
135 | __nft_trace_packet(pkt, verdict: ®s->verdict, rule, info, type); |
136 | } |
137 | |
138 | static inline void nft_trace_verdict(const struct nft_pktinfo *pkt, |
139 | struct nft_traceinfo *info, |
140 | const struct nft_rule_dp *rule, |
141 | const struct nft_regs *regs) |
142 | { |
143 | if (static_branch_unlikely(&nft_trace_enabled)) |
144 | __nft_trace_verdict(pkt, info, rule, regs); |
145 | } |
146 | |
147 | static bool nft_payload_fast_eval(const struct nft_expr *expr, |
148 | struct nft_regs *regs, |
149 | const struct nft_pktinfo *pkt) |
150 | { |
151 | const struct nft_payload *priv = nft_expr_priv(expr); |
152 | const struct sk_buff *skb = pkt->skb; |
153 | u32 *dest = ®s->data[priv->dreg]; |
154 | unsigned char *ptr; |
155 | |
156 | if (priv->base == NFT_PAYLOAD_NETWORK_HEADER) |
157 | ptr = skb_network_header(skb); |
158 | else { |
159 | if (!(pkt->flags & NFT_PKTINFO_L4PROTO)) |
160 | return false; |
161 | ptr = skb_network_header(skb) + nft_thoff(pkt); |
162 | } |
163 | |
164 | ptr += priv->offset; |
165 | |
166 | if (unlikely(ptr + priv->len > skb_tail_pointer(skb))) |
167 | return false; |
168 | |
169 | *dest = 0; |
170 | if (priv->len == 2) |
171 | *(u16 *)dest = *(u16 *)ptr; |
172 | else if (priv->len == 4) |
173 | *(u32 *)dest = *(u32 *)ptr; |
174 | else |
175 | *(u8 *)dest = *(u8 *)ptr; |
176 | return true; |
177 | } |
178 | |
179 | DEFINE_STATIC_KEY_FALSE(nft_counters_enabled); |
180 | |
181 | static noinline void nft_update_chain_stats(const struct nft_chain *chain, |
182 | const struct nft_pktinfo *pkt) |
183 | { |
184 | struct nft_base_chain *base_chain; |
185 | struct nft_stats __percpu *pstats; |
186 | struct nft_stats *stats; |
187 | |
188 | base_chain = nft_base_chain(chain); |
189 | |
190 | pstats = READ_ONCE(base_chain->stats); |
191 | if (pstats) { |
192 | local_bh_disable(); |
193 | stats = this_cpu_ptr(pstats); |
194 | u64_stats_update_begin(syncp: &stats->syncp); |
195 | stats->pkts++; |
196 | stats->bytes += pkt->skb->len; |
197 | u64_stats_update_end(syncp: &stats->syncp); |
198 | local_bh_enable(); |
199 | } |
200 | } |
201 | |
202 | struct nft_jumpstack { |
203 | const struct nft_rule_dp *rule; |
204 | }; |
205 | |
206 | static void expr_call_ops_eval(const struct nft_expr *expr, |
207 | struct nft_regs *regs, |
208 | struct nft_pktinfo *pkt) |
209 | { |
210 | #ifdef CONFIG_RETPOLINE |
211 | unsigned long e; |
212 | |
213 | if (nf_skip_indirect_calls()) |
214 | goto indirect_call; |
215 | |
216 | e = (unsigned long)expr->ops->eval; |
217 | #define X(e, fun) \ |
218 | do { if ((e) == (unsigned long)(fun)) \ |
219 | return fun(expr, regs, pkt); } while (0) |
220 | |
221 | X(e, nft_payload_eval); |
222 | X(e, nft_cmp_eval); |
223 | X(e, nft_counter_eval); |
224 | X(e, nft_meta_get_eval); |
225 | X(e, nft_lookup_eval); |
226 | #if IS_ENABLED(CONFIG_NFT_CT) |
227 | X(e, nft_ct_get_fast_eval); |
228 | #endif |
229 | X(e, nft_range_eval); |
230 | X(e, nft_immediate_eval); |
231 | X(e, nft_byteorder_eval); |
232 | X(e, nft_dynset_eval); |
233 | X(e, nft_rt_get_eval); |
234 | X(e, nft_bitwise_eval); |
235 | X(e, nft_objref_eval); |
236 | X(e, nft_objref_map_eval); |
237 | #undef X |
238 | indirect_call: |
239 | #endif /* CONFIG_RETPOLINE */ |
240 | expr->ops->eval(expr, regs, pkt); |
241 | } |
242 | |
243 | #define nft_rule_expr_first(rule) (struct nft_expr *)&rule->data[0] |
244 | #define nft_rule_expr_next(expr) ((void *)expr) + expr->ops->size |
245 | #define nft_rule_expr_last(rule) (struct nft_expr *)&rule->data[rule->dlen] |
246 | |
247 | #define nft_rule_dp_for_each_expr(expr, last, rule) \ |
248 | for ((expr) = nft_rule_expr_first(rule), (last) = nft_rule_expr_last(rule); \ |
249 | (expr) != (last); \ |
250 | (expr) = nft_rule_expr_next(expr)) |
251 | |
252 | unsigned int |
253 | nft_do_chain(struct nft_pktinfo *pkt, void *priv) |
254 | { |
255 | const struct nft_chain *chain = priv, *basechain = chain; |
256 | const struct net *net = nft_net(pkt); |
257 | const struct nft_expr *expr, *last; |
258 | const struct nft_rule_dp *rule; |
259 | struct nft_regs regs = {}; |
260 | unsigned int stackptr = 0; |
261 | struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE]; |
262 | bool genbit = READ_ONCE(net->nft.gencursor); |
263 | struct nft_rule_blob *blob; |
264 | struct nft_traceinfo info; |
265 | |
266 | info.trace = false; |
267 | if (static_branch_unlikely(&nft_trace_enabled)) |
268 | nft_trace_init(info: &info, pkt, basechain); |
269 | do_chain: |
270 | if (genbit) |
271 | blob = rcu_dereference(chain->blob_gen_1); |
272 | else |
273 | blob = rcu_dereference(chain->blob_gen_0); |
274 | |
275 | rule = (struct nft_rule_dp *)blob->data; |
276 | next_rule: |
277 | regs.verdict.code = NFT_CONTINUE; |
278 | for (; !rule->is_last ; rule = nft_rule_next(rule)) { |
279 | nft_rule_dp_for_each_expr(expr, last, rule) { |
280 | if (expr->ops == &nft_cmp_fast_ops) |
281 | nft_cmp_fast_eval(expr, regs: ®s); |
282 | else if (expr->ops == &nft_cmp16_fast_ops) |
283 | nft_cmp16_fast_eval(expr, regs: ®s); |
284 | else if (expr->ops == &nft_bitwise_fast_ops) |
285 | nft_bitwise_fast_eval(expr, regs: ®s); |
286 | else if (expr->ops != &nft_payload_fast_ops || |
287 | !nft_payload_fast_eval(expr, regs: ®s, pkt)) |
288 | expr_call_ops_eval(expr, regs: ®s, pkt); |
289 | |
290 | if (regs.verdict.code != NFT_CONTINUE) |
291 | break; |
292 | } |
293 | |
294 | switch (regs.verdict.code) { |
295 | case NFT_BREAK: |
296 | regs.verdict.code = NFT_CONTINUE; |
297 | nft_trace_copy_nftrace(pkt, info: &info); |
298 | continue; |
299 | case NFT_CONTINUE: |
300 | nft_trace_packet(pkt, verdict: ®s.verdict, info: &info, rule, |
301 | type: NFT_TRACETYPE_RULE); |
302 | continue; |
303 | } |
304 | break; |
305 | } |
306 | |
307 | nft_trace_verdict(pkt, info: &info, rule, regs: ®s); |
308 | |
309 | switch (regs.verdict.code & NF_VERDICT_MASK) { |
310 | case NF_ACCEPT: |
311 | case NF_QUEUE: |
312 | case NF_STOLEN: |
313 | return regs.verdict.code; |
314 | case NF_DROP: |
315 | return NF_DROP_REASON(skb: pkt->skb, reason: SKB_DROP_REASON_NETFILTER_DROP, EPERM); |
316 | } |
317 | |
318 | switch (regs.verdict.code) { |
319 | case NFT_JUMP: |
320 | if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE)) |
321 | return NF_DROP; |
322 | jumpstack[stackptr].rule = nft_rule_next(rule); |
323 | stackptr++; |
324 | fallthrough; |
325 | case NFT_GOTO: |
326 | chain = regs.verdict.chain; |
327 | goto do_chain; |
328 | case NFT_CONTINUE: |
329 | case NFT_RETURN: |
330 | break; |
331 | default: |
332 | WARN_ON_ONCE(1); |
333 | } |
334 | |
335 | if (stackptr > 0) { |
336 | stackptr--; |
337 | rule = jumpstack[stackptr].rule; |
338 | goto next_rule; |
339 | } |
340 | |
341 | nft_trace_packet(pkt, verdict: ®s.verdict, info: &info, NULL, type: NFT_TRACETYPE_POLICY); |
342 | |
343 | if (static_branch_unlikely(&nft_counters_enabled)) |
344 | nft_update_chain_stats(chain: basechain, pkt); |
345 | |
346 | if (nft_base_chain(chain: basechain)->policy == NF_DROP) |
347 | return NF_DROP_REASON(skb: pkt->skb, reason: SKB_DROP_REASON_NETFILTER_DROP, EPERM); |
348 | |
349 | return nft_base_chain(chain: basechain)->policy; |
350 | } |
351 | EXPORT_SYMBOL_GPL(nft_do_chain); |
352 | |
353 | static struct nft_expr_type *nft_basic_types[] = { |
354 | &nft_imm_type, |
355 | &nft_cmp_type, |
356 | &nft_lookup_type, |
357 | &nft_bitwise_type, |
358 | &nft_byteorder_type, |
359 | &nft_payload_type, |
360 | &nft_dynset_type, |
361 | &nft_range_type, |
362 | &nft_meta_type, |
363 | &nft_rt_type, |
364 | &nft_exthdr_type, |
365 | &nft_last_type, |
366 | &nft_counter_type, |
367 | &nft_objref_type, |
368 | &nft_inner_type, |
369 | }; |
370 | |
371 | static struct nft_object_type *nft_basic_objects[] = { |
372 | #ifdef CONFIG_NETWORK_SECMARK |
373 | &nft_secmark_obj_type, |
374 | #endif |
375 | &nft_counter_obj_type, |
376 | }; |
377 | |
378 | int __init nf_tables_core_module_init(void) |
379 | { |
380 | int err, i, j = 0; |
381 | |
382 | nft_counter_init_seqcount(); |
383 | |
384 | for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) { |
385 | err = nft_register_obj(obj_type: nft_basic_objects[i]); |
386 | if (err) |
387 | goto err; |
388 | } |
389 | |
390 | for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) { |
391 | err = nft_register_expr(nft_basic_types[j]); |
392 | if (err) |
393 | goto err; |
394 | } |
395 | |
396 | nf_skip_indirect_calls_enable(); |
397 | |
398 | return 0; |
399 | |
400 | err: |
401 | while (j-- > 0) |
402 | nft_unregister_expr(nft_basic_types[j]); |
403 | |
404 | while (i-- > 0) |
405 | nft_unregister_obj(obj_type: nft_basic_objects[i]); |
406 | |
407 | return err; |
408 | } |
409 | |
410 | void nf_tables_core_module_exit(void) |
411 | { |
412 | int i; |
413 | |
414 | i = ARRAY_SIZE(nft_basic_types); |
415 | while (i-- > 0) |
416 | nft_unregister_expr(nft_basic_types[i]); |
417 | |
418 | i = ARRAY_SIZE(nft_basic_objects); |
419 | while (i-- > 0) |
420 | nft_unregister_obj(obj_type: nft_basic_objects[i]); |
421 | } |
422 | |