1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/kernel.h> |
3 | #include <linux/init.h> |
4 | #include <linux/module.h> |
5 | #include <linux/netfilter.h> |
6 | #include <linux/rhashtable.h> |
7 | #include <linux/ip.h> |
8 | #include <linux/ipv6.h> |
9 | #include <linux/netdevice.h> |
10 | #include <linux/if_ether.h> |
11 | #include <net/gso.h> |
12 | #include <net/ip.h> |
13 | #include <net/ipv6.h> |
14 | #include <net/ip6_route.h> |
15 | #include <net/neighbour.h> |
16 | #include <net/netfilter/nf_flow_table.h> |
17 | #include <net/netfilter/nf_conntrack_acct.h> |
18 | /* For layer 4 checksum field offset. */ |
19 | #include <linux/tcp.h> |
20 | #include <linux/udp.h> |
21 | |
22 | static int nf_flow_state_check(struct flow_offload *flow, int proto, |
23 | struct sk_buff *skb, unsigned int thoff) |
24 | { |
25 | struct tcphdr *tcph; |
26 | |
27 | if (proto != IPPROTO_TCP) |
28 | return 0; |
29 | |
30 | tcph = (void *)(skb_network_header(skb) + thoff); |
31 | if (unlikely(tcph->fin || tcph->rst)) { |
32 | flow_offload_teardown(flow); |
33 | return -1; |
34 | } |
35 | |
36 | return 0; |
37 | } |
38 | |
39 | static void nf_flow_nat_ip_tcp(struct sk_buff *skb, unsigned int thoff, |
40 | __be32 addr, __be32 new_addr) |
41 | { |
42 | struct tcphdr *tcph; |
43 | |
44 | tcph = (void *)(skb_network_header(skb) + thoff); |
45 | inet_proto_csum_replace4(sum: &tcph->check, skb, from: addr, to: new_addr, pseudohdr: true); |
46 | } |
47 | |
48 | static void nf_flow_nat_ip_udp(struct sk_buff *skb, unsigned int thoff, |
49 | __be32 addr, __be32 new_addr) |
50 | { |
51 | struct udphdr *udph; |
52 | |
53 | udph = (void *)(skb_network_header(skb) + thoff); |
54 | if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) { |
55 | inet_proto_csum_replace4(sum: &udph->check, skb, from: addr, |
56 | to: new_addr, pseudohdr: true); |
57 | if (!udph->check) |
58 | udph->check = CSUM_MANGLED_0; |
59 | } |
60 | } |
61 | |
62 | static void nf_flow_nat_ip_l4proto(struct sk_buff *skb, struct iphdr *iph, |
63 | unsigned int thoff, __be32 addr, |
64 | __be32 new_addr) |
65 | { |
66 | switch (iph->protocol) { |
67 | case IPPROTO_TCP: |
68 | nf_flow_nat_ip_tcp(skb, thoff, addr, new_addr); |
69 | break; |
70 | case IPPROTO_UDP: |
71 | nf_flow_nat_ip_udp(skb, thoff, addr, new_addr); |
72 | break; |
73 | } |
74 | } |
75 | |
76 | static void nf_flow_snat_ip(const struct flow_offload *flow, |
77 | struct sk_buff *skb, struct iphdr *iph, |
78 | unsigned int thoff, enum flow_offload_tuple_dir dir) |
79 | { |
80 | __be32 addr, new_addr; |
81 | |
82 | switch (dir) { |
83 | case FLOW_OFFLOAD_DIR_ORIGINAL: |
84 | addr = iph->saddr; |
85 | new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr; |
86 | iph->saddr = new_addr; |
87 | break; |
88 | case FLOW_OFFLOAD_DIR_REPLY: |
89 | addr = iph->daddr; |
90 | new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr; |
91 | iph->daddr = new_addr; |
92 | break; |
93 | } |
94 | csum_replace4(sum: &iph->check, from: addr, to: new_addr); |
95 | |
96 | nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr); |
97 | } |
98 | |
99 | static void nf_flow_dnat_ip(const struct flow_offload *flow, |
100 | struct sk_buff *skb, struct iphdr *iph, |
101 | unsigned int thoff, enum flow_offload_tuple_dir dir) |
102 | { |
103 | __be32 addr, new_addr; |
104 | |
105 | switch (dir) { |
106 | case FLOW_OFFLOAD_DIR_ORIGINAL: |
107 | addr = iph->daddr; |
108 | new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr; |
109 | iph->daddr = new_addr; |
110 | break; |
111 | case FLOW_OFFLOAD_DIR_REPLY: |
112 | addr = iph->saddr; |
113 | new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr; |
114 | iph->saddr = new_addr; |
115 | break; |
116 | } |
117 | csum_replace4(sum: &iph->check, from: addr, to: new_addr); |
118 | |
119 | nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr); |
120 | } |
121 | |
122 | static void nf_flow_nat_ip(const struct flow_offload *flow, struct sk_buff *skb, |
123 | unsigned int thoff, enum flow_offload_tuple_dir dir, |
124 | struct iphdr *iph) |
125 | { |
126 | if (test_bit(NF_FLOW_SNAT, &flow->flags)) { |
127 | nf_flow_snat_port(flow, skb, thoff, protocol: iph->protocol, dir); |
128 | nf_flow_snat_ip(flow, skb, iph, thoff, dir); |
129 | } |
130 | if (test_bit(NF_FLOW_DNAT, &flow->flags)) { |
131 | nf_flow_dnat_port(flow, skb, thoff, protocol: iph->protocol, dir); |
132 | nf_flow_dnat_ip(flow, skb, iph, thoff, dir); |
133 | } |
134 | } |
135 | |
136 | static bool ip_has_options(unsigned int thoff) |
137 | { |
138 | return thoff != sizeof(struct iphdr); |
139 | } |
140 | |
141 | static void nf_flow_tuple_encap(struct sk_buff *skb, |
142 | struct flow_offload_tuple *tuple) |
143 | { |
144 | struct vlan_ethhdr *veth; |
145 | struct pppoe_hdr *phdr; |
146 | int i = 0; |
147 | |
148 | if (skb_vlan_tag_present(skb)) { |
149 | tuple->encap[i].id = skb_vlan_tag_get(skb); |
150 | tuple->encap[i].proto = skb->vlan_proto; |
151 | i++; |
152 | } |
153 | switch (skb->protocol) { |
154 | case htons(ETH_P_8021Q): |
155 | veth = (struct vlan_ethhdr *)skb_mac_header(skb); |
156 | tuple->encap[i].id = ntohs(veth->h_vlan_TCI); |
157 | tuple->encap[i].proto = skb->protocol; |
158 | break; |
159 | case htons(ETH_P_PPP_SES): |
160 | phdr = (struct pppoe_hdr *)skb_mac_header(skb); |
161 | tuple->encap[i].id = ntohs(phdr->sid); |
162 | tuple->encap[i].proto = skb->protocol; |
163 | break; |
164 | } |
165 | } |
166 | |
167 | struct nf_flowtable_ctx { |
168 | const struct net_device *in; |
169 | u32 offset; |
170 | u32 hdrsize; |
171 | }; |
172 | |
173 | static int nf_flow_tuple_ip(struct nf_flowtable_ctx *ctx, struct sk_buff *skb, |
174 | struct flow_offload_tuple *tuple) |
175 | { |
176 | struct flow_ports *ports; |
177 | unsigned int thoff; |
178 | struct iphdr *iph; |
179 | u8 ipproto; |
180 | |
181 | if (!pskb_may_pull(skb, len: sizeof(*iph) + ctx->offset)) |
182 | return -1; |
183 | |
184 | iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset); |
185 | thoff = (iph->ihl * 4); |
186 | |
187 | if (ip_is_fragment(iph) || |
188 | unlikely(ip_has_options(thoff))) |
189 | return -1; |
190 | |
191 | thoff += ctx->offset; |
192 | |
193 | ipproto = iph->protocol; |
194 | switch (ipproto) { |
195 | case IPPROTO_TCP: |
196 | ctx->hdrsize = sizeof(struct tcphdr); |
197 | break; |
198 | case IPPROTO_UDP: |
199 | ctx->hdrsize = sizeof(struct udphdr); |
200 | break; |
201 | #ifdef CONFIG_NF_CT_PROTO_GRE |
202 | case IPPROTO_GRE: |
203 | ctx->hdrsize = sizeof(struct gre_base_hdr); |
204 | break; |
205 | #endif |
206 | default: |
207 | return -1; |
208 | } |
209 | |
210 | if (iph->ttl <= 1) |
211 | return -1; |
212 | |
213 | if (!pskb_may_pull(skb, len: thoff + ctx->hdrsize)) |
214 | return -1; |
215 | |
216 | switch (ipproto) { |
217 | case IPPROTO_TCP: |
218 | case IPPROTO_UDP: |
219 | ports = (struct flow_ports *)(skb_network_header(skb) + thoff); |
220 | tuple->src_port = ports->source; |
221 | tuple->dst_port = ports->dest; |
222 | break; |
223 | case IPPROTO_GRE: { |
224 | struct gre_base_hdr *greh; |
225 | |
226 | greh = (struct gre_base_hdr *)(skb_network_header(skb) + thoff); |
227 | if ((greh->flags & GRE_VERSION) != GRE_VERSION_0) |
228 | return -1; |
229 | break; |
230 | } |
231 | } |
232 | |
233 | iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset); |
234 | |
235 | tuple->src_v4.s_addr = iph->saddr; |
236 | tuple->dst_v4.s_addr = iph->daddr; |
237 | tuple->l3proto = AF_INET; |
238 | tuple->l4proto = ipproto; |
239 | tuple->iifidx = ctx->in->ifindex; |
240 | nf_flow_tuple_encap(skb, tuple); |
241 | |
242 | return 0; |
243 | } |
244 | |
245 | /* Based on ip_exceeds_mtu(). */ |
246 | static bool nf_flow_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu) |
247 | { |
248 | if (skb->len <= mtu) |
249 | return false; |
250 | |
251 | if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) |
252 | return false; |
253 | |
254 | return true; |
255 | } |
256 | |
257 | static inline bool nf_flow_dst_check(struct flow_offload_tuple *tuple) |
258 | { |
259 | if (tuple->xmit_type != FLOW_OFFLOAD_XMIT_NEIGH && |
260 | tuple->xmit_type != FLOW_OFFLOAD_XMIT_XFRM) |
261 | return true; |
262 | |
263 | return dst_check(dst: tuple->dst_cache, cookie: tuple->dst_cookie); |
264 | } |
265 | |
266 | static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb, |
267 | const struct nf_hook_state *state, |
268 | struct dst_entry *dst) |
269 | { |
270 | skb_orphan(skb); |
271 | skb_dst_set_noref(skb, dst); |
272 | dst_output(net: state->net, sk: state->sk, skb); |
273 | return NF_STOLEN; |
274 | } |
275 | |
276 | static bool nf_flow_skb_encap_protocol(const struct sk_buff *skb, __be16 proto, |
277 | u32 *offset) |
278 | { |
279 | struct vlan_ethhdr *veth; |
280 | |
281 | switch (skb->protocol) { |
282 | case htons(ETH_P_8021Q): |
283 | veth = (struct vlan_ethhdr *)skb_mac_header(skb); |
284 | if (veth->h_vlan_encapsulated_proto == proto) { |
285 | *offset += VLAN_HLEN; |
286 | return true; |
287 | } |
288 | break; |
289 | case htons(ETH_P_PPP_SES): |
290 | if (nf_flow_pppoe_proto(skb) == proto) { |
291 | *offset += PPPOE_SES_HLEN; |
292 | return true; |
293 | } |
294 | break; |
295 | } |
296 | |
297 | return false; |
298 | } |
299 | |
300 | static void nf_flow_encap_pop(struct sk_buff *skb, |
301 | struct flow_offload_tuple_rhash *tuplehash) |
302 | { |
303 | struct vlan_hdr *vlan_hdr; |
304 | int i; |
305 | |
306 | for (i = 0; i < tuplehash->tuple.encap_num; i++) { |
307 | if (skb_vlan_tag_present(skb)) { |
308 | __vlan_hwaccel_clear_tag(skb); |
309 | continue; |
310 | } |
311 | switch (skb->protocol) { |
312 | case htons(ETH_P_8021Q): |
313 | vlan_hdr = (struct vlan_hdr *)skb->data; |
314 | __skb_pull(skb, VLAN_HLEN); |
315 | vlan_set_encap_proto(skb, vhdr: vlan_hdr); |
316 | skb_reset_network_header(skb); |
317 | break; |
318 | case htons(ETH_P_PPP_SES): |
319 | skb->protocol = nf_flow_pppoe_proto(skb); |
320 | skb_pull(skb, PPPOE_SES_HLEN); |
321 | skb_reset_network_header(skb); |
322 | break; |
323 | } |
324 | } |
325 | } |
326 | |
327 | static unsigned int nf_flow_queue_xmit(struct net *net, struct sk_buff *skb, |
328 | const struct flow_offload_tuple_rhash *tuplehash, |
329 | unsigned short type) |
330 | { |
331 | struct net_device *outdev; |
332 | |
333 | outdev = dev_get_by_index_rcu(net, ifindex: tuplehash->tuple.out.ifidx); |
334 | if (!outdev) |
335 | return NF_DROP; |
336 | |
337 | skb->dev = outdev; |
338 | dev_hard_header(skb, dev: skb->dev, type, daddr: tuplehash->tuple.out.h_dest, |
339 | saddr: tuplehash->tuple.out.h_source, len: skb->len); |
340 | dev_queue_xmit(skb); |
341 | |
342 | return NF_STOLEN; |
343 | } |
344 | |
345 | static struct flow_offload_tuple_rhash * |
346 | nf_flow_offload_lookup(struct nf_flowtable_ctx *ctx, |
347 | struct nf_flowtable *flow_table, struct sk_buff *skb) |
348 | { |
349 | struct flow_offload_tuple tuple = {}; |
350 | |
351 | if (skb->protocol != htons(ETH_P_IP) && |
352 | !nf_flow_skb_encap_protocol(skb, htons(ETH_P_IP), offset: &ctx->offset)) |
353 | return NULL; |
354 | |
355 | if (nf_flow_tuple_ip(ctx, skb, tuple: &tuple) < 0) |
356 | return NULL; |
357 | |
358 | return flow_offload_lookup(flow_table, tuple: &tuple); |
359 | } |
360 | |
361 | static int nf_flow_offload_forward(struct nf_flowtable_ctx *ctx, |
362 | struct nf_flowtable *flow_table, |
363 | struct flow_offload_tuple_rhash *tuplehash, |
364 | struct sk_buff *skb) |
365 | { |
366 | enum flow_offload_tuple_dir dir; |
367 | struct flow_offload *flow; |
368 | unsigned int thoff, mtu; |
369 | struct iphdr *iph; |
370 | |
371 | dir = tuplehash->tuple.dir; |
372 | flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); |
373 | |
374 | mtu = flow->tuplehash[dir].tuple.mtu + ctx->offset; |
375 | if (unlikely(nf_flow_exceeds_mtu(skb, mtu))) |
376 | return 0; |
377 | |
378 | iph = (struct iphdr *)(skb_network_header(skb) + ctx->offset); |
379 | thoff = (iph->ihl * 4) + ctx->offset; |
380 | if (nf_flow_state_check(flow, proto: iph->protocol, skb, thoff)) |
381 | return 0; |
382 | |
383 | if (!nf_flow_dst_check(tuple: &tuplehash->tuple)) { |
384 | flow_offload_teardown(flow); |
385 | return 0; |
386 | } |
387 | |
388 | if (skb_try_make_writable(skb, write_len: thoff + ctx->hdrsize)) |
389 | return -1; |
390 | |
391 | flow_offload_refresh(flow_table, flow, force: false); |
392 | |
393 | nf_flow_encap_pop(skb, tuplehash); |
394 | thoff -= ctx->offset; |
395 | |
396 | iph = ip_hdr(skb); |
397 | nf_flow_nat_ip(flow, skb, thoff, dir, iph); |
398 | |
399 | ip_decrease_ttl(iph); |
400 | skb_clear_tstamp(skb); |
401 | |
402 | if (flow_table->flags & NF_FLOWTABLE_COUNTER) |
403 | nf_ct_acct_update(ct: flow->ct, dir: tuplehash->tuple.dir, bytes: skb->len); |
404 | |
405 | return 1; |
406 | } |
407 | |
408 | unsigned int |
409 | nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, |
410 | const struct nf_hook_state *state) |
411 | { |
412 | struct flow_offload_tuple_rhash *tuplehash; |
413 | struct nf_flowtable *flow_table = priv; |
414 | enum flow_offload_tuple_dir dir; |
415 | struct nf_flowtable_ctx ctx = { |
416 | .in = state->in, |
417 | }; |
418 | struct flow_offload *flow; |
419 | struct net_device *outdev; |
420 | struct rtable *rt; |
421 | __be32 nexthop; |
422 | int ret; |
423 | |
424 | tuplehash = nf_flow_offload_lookup(ctx: &ctx, flow_table, skb); |
425 | if (!tuplehash) |
426 | return NF_ACCEPT; |
427 | |
428 | ret = nf_flow_offload_forward(ctx: &ctx, flow_table, tuplehash, skb); |
429 | if (ret < 0) |
430 | return NF_DROP; |
431 | else if (ret == 0) |
432 | return NF_ACCEPT; |
433 | |
434 | if (unlikely(tuplehash->tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)) { |
435 | rt = (struct rtable *)tuplehash->tuple.dst_cache; |
436 | memset(skb->cb, 0, sizeof(struct inet_skb_parm)); |
437 | IPCB(skb)->iif = skb->dev->ifindex; |
438 | IPCB(skb)->flags = IPSKB_FORWARDED; |
439 | return nf_flow_xmit_xfrm(skb, state, dst: &rt->dst); |
440 | } |
441 | |
442 | dir = tuplehash->tuple.dir; |
443 | flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); |
444 | |
445 | switch (tuplehash->tuple.xmit_type) { |
446 | case FLOW_OFFLOAD_XMIT_NEIGH: |
447 | rt = (struct rtable *)tuplehash->tuple.dst_cache; |
448 | outdev = rt->dst.dev; |
449 | skb->dev = outdev; |
450 | nexthop = rt_nexthop(rt, daddr: flow->tuplehash[!dir].tuple.src_v4.s_addr); |
451 | skb_dst_set_noref(skb, dst: &rt->dst); |
452 | neigh_xmit(fam: NEIGH_ARP_TABLE, outdev, &nexthop, skb); |
453 | ret = NF_STOLEN; |
454 | break; |
455 | case FLOW_OFFLOAD_XMIT_DIRECT: |
456 | ret = nf_flow_queue_xmit(net: state->net, skb, tuplehash, ETH_P_IP); |
457 | if (ret == NF_DROP) |
458 | flow_offload_teardown(flow); |
459 | break; |
460 | default: |
461 | WARN_ON_ONCE(1); |
462 | ret = NF_DROP; |
463 | break; |
464 | } |
465 | |
466 | return ret; |
467 | } |
468 | EXPORT_SYMBOL_GPL(nf_flow_offload_ip_hook); |
469 | |
470 | static void nf_flow_nat_ipv6_tcp(struct sk_buff *skb, unsigned int thoff, |
471 | struct in6_addr *addr, |
472 | struct in6_addr *new_addr, |
473 | struct ipv6hdr *ip6h) |
474 | { |
475 | struct tcphdr *tcph; |
476 | |
477 | tcph = (void *)(skb_network_header(skb) + thoff); |
478 | inet_proto_csum_replace16(sum: &tcph->check, skb, from: addr->s6_addr32, |
479 | to: new_addr->s6_addr32, pseudohdr: true); |
480 | } |
481 | |
482 | static void nf_flow_nat_ipv6_udp(struct sk_buff *skb, unsigned int thoff, |
483 | struct in6_addr *addr, |
484 | struct in6_addr *new_addr) |
485 | { |
486 | struct udphdr *udph; |
487 | |
488 | udph = (void *)(skb_network_header(skb) + thoff); |
489 | if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) { |
490 | inet_proto_csum_replace16(sum: &udph->check, skb, from: addr->s6_addr32, |
491 | to: new_addr->s6_addr32, pseudohdr: true); |
492 | if (!udph->check) |
493 | udph->check = CSUM_MANGLED_0; |
494 | } |
495 | } |
496 | |
497 | static void nf_flow_nat_ipv6_l4proto(struct sk_buff *skb, struct ipv6hdr *ip6h, |
498 | unsigned int thoff, struct in6_addr *addr, |
499 | struct in6_addr *new_addr) |
500 | { |
501 | switch (ip6h->nexthdr) { |
502 | case IPPROTO_TCP: |
503 | nf_flow_nat_ipv6_tcp(skb, thoff, addr, new_addr, ip6h); |
504 | break; |
505 | case IPPROTO_UDP: |
506 | nf_flow_nat_ipv6_udp(skb, thoff, addr, new_addr); |
507 | break; |
508 | } |
509 | } |
510 | |
511 | static void nf_flow_snat_ipv6(const struct flow_offload *flow, |
512 | struct sk_buff *skb, struct ipv6hdr *ip6h, |
513 | unsigned int thoff, |
514 | enum flow_offload_tuple_dir dir) |
515 | { |
516 | struct in6_addr addr, new_addr; |
517 | |
518 | switch (dir) { |
519 | case FLOW_OFFLOAD_DIR_ORIGINAL: |
520 | addr = ip6h->saddr; |
521 | new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6; |
522 | ip6h->saddr = new_addr; |
523 | break; |
524 | case FLOW_OFFLOAD_DIR_REPLY: |
525 | addr = ip6h->daddr; |
526 | new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6; |
527 | ip6h->daddr = new_addr; |
528 | break; |
529 | } |
530 | |
531 | nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, addr: &addr, new_addr: &new_addr); |
532 | } |
533 | |
534 | static void nf_flow_dnat_ipv6(const struct flow_offload *flow, |
535 | struct sk_buff *skb, struct ipv6hdr *ip6h, |
536 | unsigned int thoff, |
537 | enum flow_offload_tuple_dir dir) |
538 | { |
539 | struct in6_addr addr, new_addr; |
540 | |
541 | switch (dir) { |
542 | case FLOW_OFFLOAD_DIR_ORIGINAL: |
543 | addr = ip6h->daddr; |
544 | new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6; |
545 | ip6h->daddr = new_addr; |
546 | break; |
547 | case FLOW_OFFLOAD_DIR_REPLY: |
548 | addr = ip6h->saddr; |
549 | new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6; |
550 | ip6h->saddr = new_addr; |
551 | break; |
552 | } |
553 | |
554 | nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, addr: &addr, new_addr: &new_addr); |
555 | } |
556 | |
557 | static void nf_flow_nat_ipv6(const struct flow_offload *flow, |
558 | struct sk_buff *skb, |
559 | enum flow_offload_tuple_dir dir, |
560 | struct ipv6hdr *ip6h) |
561 | { |
562 | unsigned int thoff = sizeof(*ip6h); |
563 | |
564 | if (test_bit(NF_FLOW_SNAT, &flow->flags)) { |
565 | nf_flow_snat_port(flow, skb, thoff, protocol: ip6h->nexthdr, dir); |
566 | nf_flow_snat_ipv6(flow, skb, ip6h, thoff, dir); |
567 | } |
568 | if (test_bit(NF_FLOW_DNAT, &flow->flags)) { |
569 | nf_flow_dnat_port(flow, skb, thoff, protocol: ip6h->nexthdr, dir); |
570 | nf_flow_dnat_ipv6(flow, skb, ip6h, thoff, dir); |
571 | } |
572 | } |
573 | |
574 | static int nf_flow_tuple_ipv6(struct nf_flowtable_ctx *ctx, struct sk_buff *skb, |
575 | struct flow_offload_tuple *tuple) |
576 | { |
577 | struct flow_ports *ports; |
578 | struct ipv6hdr *ip6h; |
579 | unsigned int thoff; |
580 | u8 nexthdr; |
581 | |
582 | thoff = sizeof(*ip6h) + ctx->offset; |
583 | if (!pskb_may_pull(skb, len: thoff)) |
584 | return -1; |
585 | |
586 | ip6h = (struct ipv6hdr *)(skb_network_header(skb) + ctx->offset); |
587 | |
588 | nexthdr = ip6h->nexthdr; |
589 | switch (nexthdr) { |
590 | case IPPROTO_TCP: |
591 | ctx->hdrsize = sizeof(struct tcphdr); |
592 | break; |
593 | case IPPROTO_UDP: |
594 | ctx->hdrsize = sizeof(struct udphdr); |
595 | break; |
596 | #ifdef CONFIG_NF_CT_PROTO_GRE |
597 | case IPPROTO_GRE: |
598 | ctx->hdrsize = sizeof(struct gre_base_hdr); |
599 | break; |
600 | #endif |
601 | default: |
602 | return -1; |
603 | } |
604 | |
605 | if (ip6h->hop_limit <= 1) |
606 | return -1; |
607 | |
608 | if (!pskb_may_pull(skb, len: thoff + ctx->hdrsize)) |
609 | return -1; |
610 | |
611 | switch (nexthdr) { |
612 | case IPPROTO_TCP: |
613 | case IPPROTO_UDP: |
614 | ports = (struct flow_ports *)(skb_network_header(skb) + thoff); |
615 | tuple->src_port = ports->source; |
616 | tuple->dst_port = ports->dest; |
617 | break; |
618 | case IPPROTO_GRE: { |
619 | struct gre_base_hdr *greh; |
620 | |
621 | greh = (struct gre_base_hdr *)(skb_network_header(skb) + thoff); |
622 | if ((greh->flags & GRE_VERSION) != GRE_VERSION_0) |
623 | return -1; |
624 | break; |
625 | } |
626 | } |
627 | |
628 | ip6h = (struct ipv6hdr *)(skb_network_header(skb) + ctx->offset); |
629 | |
630 | tuple->src_v6 = ip6h->saddr; |
631 | tuple->dst_v6 = ip6h->daddr; |
632 | tuple->l3proto = AF_INET6; |
633 | tuple->l4proto = nexthdr; |
634 | tuple->iifidx = ctx->in->ifindex; |
635 | nf_flow_tuple_encap(skb, tuple); |
636 | |
637 | return 0; |
638 | } |
639 | |
640 | static int nf_flow_offload_ipv6_forward(struct nf_flowtable_ctx *ctx, |
641 | struct nf_flowtable *flow_table, |
642 | struct flow_offload_tuple_rhash *tuplehash, |
643 | struct sk_buff *skb) |
644 | { |
645 | enum flow_offload_tuple_dir dir; |
646 | struct flow_offload *flow; |
647 | unsigned int thoff, mtu; |
648 | struct ipv6hdr *ip6h; |
649 | |
650 | dir = tuplehash->tuple.dir; |
651 | flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); |
652 | |
653 | mtu = flow->tuplehash[dir].tuple.mtu + ctx->offset; |
654 | if (unlikely(nf_flow_exceeds_mtu(skb, mtu))) |
655 | return 0; |
656 | |
657 | ip6h = (struct ipv6hdr *)(skb_network_header(skb) + ctx->offset); |
658 | thoff = sizeof(*ip6h) + ctx->offset; |
659 | if (nf_flow_state_check(flow, proto: ip6h->nexthdr, skb, thoff)) |
660 | return 0; |
661 | |
662 | if (!nf_flow_dst_check(tuple: &tuplehash->tuple)) { |
663 | flow_offload_teardown(flow); |
664 | return 0; |
665 | } |
666 | |
667 | if (skb_try_make_writable(skb, write_len: thoff + ctx->hdrsize)) |
668 | return -1; |
669 | |
670 | flow_offload_refresh(flow_table, flow, force: false); |
671 | |
672 | nf_flow_encap_pop(skb, tuplehash); |
673 | |
674 | ip6h = ipv6_hdr(skb); |
675 | nf_flow_nat_ipv6(flow, skb, dir, ip6h); |
676 | |
677 | ip6h->hop_limit--; |
678 | skb_clear_tstamp(skb); |
679 | |
680 | if (flow_table->flags & NF_FLOWTABLE_COUNTER) |
681 | nf_ct_acct_update(ct: flow->ct, dir: tuplehash->tuple.dir, bytes: skb->len); |
682 | |
683 | return 1; |
684 | } |
685 | |
686 | static struct flow_offload_tuple_rhash * |
687 | nf_flow_offload_ipv6_lookup(struct nf_flowtable_ctx *ctx, |
688 | struct nf_flowtable *flow_table, |
689 | struct sk_buff *skb) |
690 | { |
691 | struct flow_offload_tuple tuple = {}; |
692 | |
693 | if (skb->protocol != htons(ETH_P_IPV6) && |
694 | !nf_flow_skb_encap_protocol(skb, htons(ETH_P_IPV6), offset: &ctx->offset)) |
695 | return NULL; |
696 | |
697 | if (nf_flow_tuple_ipv6(ctx, skb, tuple: &tuple) < 0) |
698 | return NULL; |
699 | |
700 | return flow_offload_lookup(flow_table, tuple: &tuple); |
701 | } |
702 | |
703 | unsigned int |
704 | nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, |
705 | const struct nf_hook_state *state) |
706 | { |
707 | struct flow_offload_tuple_rhash *tuplehash; |
708 | struct nf_flowtable *flow_table = priv; |
709 | enum flow_offload_tuple_dir dir; |
710 | struct nf_flowtable_ctx ctx = { |
711 | .in = state->in, |
712 | }; |
713 | const struct in6_addr *nexthop; |
714 | struct flow_offload *flow; |
715 | struct net_device *outdev; |
716 | struct rt6_info *rt; |
717 | int ret; |
718 | |
719 | tuplehash = nf_flow_offload_ipv6_lookup(ctx: &ctx, flow_table, skb); |
720 | if (tuplehash == NULL) |
721 | return NF_ACCEPT; |
722 | |
723 | ret = nf_flow_offload_ipv6_forward(ctx: &ctx, flow_table, tuplehash, skb); |
724 | if (ret < 0) |
725 | return NF_DROP; |
726 | else if (ret == 0) |
727 | return NF_ACCEPT; |
728 | |
729 | if (unlikely(tuplehash->tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)) { |
730 | rt = (struct rt6_info *)tuplehash->tuple.dst_cache; |
731 | memset(skb->cb, 0, sizeof(struct inet6_skb_parm)); |
732 | IP6CB(skb)->iif = skb->dev->ifindex; |
733 | IP6CB(skb)->flags = IP6SKB_FORWARDED; |
734 | return nf_flow_xmit_xfrm(skb, state, dst: &rt->dst); |
735 | } |
736 | |
737 | dir = tuplehash->tuple.dir; |
738 | flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]); |
739 | |
740 | switch (tuplehash->tuple.xmit_type) { |
741 | case FLOW_OFFLOAD_XMIT_NEIGH: |
742 | rt = (struct rt6_info *)tuplehash->tuple.dst_cache; |
743 | outdev = rt->dst.dev; |
744 | skb->dev = outdev; |
745 | nexthop = rt6_nexthop(rt, daddr: &flow->tuplehash[!dir].tuple.src_v6); |
746 | skb_dst_set_noref(skb, dst: &rt->dst); |
747 | neigh_xmit(fam: NEIGH_ND_TABLE, outdev, nexthop, skb); |
748 | ret = NF_STOLEN; |
749 | break; |
750 | case FLOW_OFFLOAD_XMIT_DIRECT: |
751 | ret = nf_flow_queue_xmit(net: state->net, skb, tuplehash, ETH_P_IPV6); |
752 | if (ret == NF_DROP) |
753 | flow_offload_teardown(flow); |
754 | break; |
755 | default: |
756 | WARN_ON_ONCE(1); |
757 | ret = NF_DROP; |
758 | break; |
759 | } |
760 | |
761 | return ret; |
762 | } |
763 | EXPORT_SYMBOL_GPL(nf_flow_offload_ipv6_hook); |
764 | |