1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * IPV6 GSO/GRO offload support |
4 | * Linux INET6 implementation |
5 | * |
6 | * UDPv6 GSO support |
7 | */ |
8 | #include <linux/skbuff.h> |
9 | #include <linux/netdevice.h> |
10 | #include <linux/indirect_call_wrapper.h> |
11 | #include <net/protocol.h> |
12 | #include <net/ipv6.h> |
13 | #include <net/udp.h> |
14 | #include <net/ip6_checksum.h> |
15 | #include "ip6_offload.h" |
16 | #include <net/gro.h> |
17 | #include <net/gso.h> |
18 | |
19 | static struct sk_buff *udp6_ufo_fragment(struct sk_buff *skb, |
20 | netdev_features_t features) |
21 | { |
22 | struct sk_buff *segs = ERR_PTR(error: -EINVAL); |
23 | unsigned int mss; |
24 | unsigned int unfrag_ip6hlen, unfrag_len; |
25 | struct frag_hdr *fptr; |
26 | u8 *packet_start, *prevhdr; |
27 | u8 nexthdr; |
28 | u8 frag_hdr_sz = sizeof(struct frag_hdr); |
29 | __wsum csum; |
30 | int tnl_hlen; |
31 | int err; |
32 | |
33 | if (skb->encapsulation && skb_shinfo(skb)->gso_type & |
34 | (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM)) |
35 | segs = skb_udp_tunnel_segment(skb, features, is_ipv6: true); |
36 | else { |
37 | const struct ipv6hdr *ipv6h; |
38 | struct udphdr *uh; |
39 | |
40 | if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4))) |
41 | goto out; |
42 | |
43 | if (!pskb_may_pull(skb, len: sizeof(struct udphdr))) |
44 | goto out; |
45 | |
46 | if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) |
47 | return __udp_gso_segment(gso_skb: skb, features, is_ipv6: true); |
48 | |
49 | mss = skb_shinfo(skb)->gso_size; |
50 | if (unlikely(skb->len <= mss)) |
51 | goto out; |
52 | |
53 | /* Do software UFO. Complete and fill in the UDP checksum as HW cannot |
54 | * do checksum of UDP packets sent as multiple IP fragments. |
55 | */ |
56 | |
57 | uh = udp_hdr(skb); |
58 | ipv6h = ipv6_hdr(skb); |
59 | |
60 | uh->check = 0; |
61 | csum = skb_checksum(skb, offset: 0, len: skb->len, csum: 0); |
62 | uh->check = udp_v6_check(len: skb->len, saddr: &ipv6h->saddr, |
63 | daddr: &ipv6h->daddr, base: csum); |
64 | if (uh->check == 0) |
65 | uh->check = CSUM_MANGLED_0; |
66 | |
67 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
68 | |
69 | /* If there is no outer header we can fake a checksum offload |
70 | * due to the fact that we have already done the checksum in |
71 | * software prior to segmenting the frame. |
72 | */ |
73 | if (!skb->encap_hdr_csum) |
74 | features |= NETIF_F_HW_CSUM; |
75 | |
76 | /* Check if there is enough headroom to insert fragment header. */ |
77 | tnl_hlen = skb_tnl_header_len(inner_skb: skb); |
78 | if (skb->mac_header < (tnl_hlen + frag_hdr_sz)) { |
79 | if (gso_pskb_expand_head(skb, extra: tnl_hlen + frag_hdr_sz)) |
80 | goto out; |
81 | } |
82 | |
83 | /* Find the unfragmentable header and shift it left by frag_hdr_sz |
84 | * bytes to insert fragment header. |
85 | */ |
86 | err = ip6_find_1stfragopt(skb, nexthdr: &prevhdr); |
87 | if (err < 0) |
88 | return ERR_PTR(error: err); |
89 | unfrag_ip6hlen = err; |
90 | nexthdr = *prevhdr; |
91 | *prevhdr = NEXTHDR_FRAGMENT; |
92 | unfrag_len = (skb_network_header(skb) - skb_mac_header(skb)) + |
93 | unfrag_ip6hlen + tnl_hlen; |
94 | packet_start = (u8 *) skb->head + SKB_GSO_CB(skb)->mac_offset; |
95 | memmove(packet_start-frag_hdr_sz, packet_start, unfrag_len); |
96 | |
97 | SKB_GSO_CB(skb)->mac_offset -= frag_hdr_sz; |
98 | skb->mac_header -= frag_hdr_sz; |
99 | skb->network_header -= frag_hdr_sz; |
100 | |
101 | fptr = (struct frag_hdr *)(skb_network_header(skb) + unfrag_ip6hlen); |
102 | fptr->nexthdr = nexthdr; |
103 | fptr->reserved = 0; |
104 | fptr->identification = ipv6_proxy_select_ident(net: dev_net(dev: skb->dev), skb); |
105 | |
106 | /* Fragment the skb. ipv6 header and the remaining fields of the |
107 | * fragment header are updated in ipv6_gso_segment() |
108 | */ |
109 | segs = skb_segment(skb, features); |
110 | } |
111 | |
112 | out: |
113 | return segs; |
114 | } |
115 | |
116 | static struct sock *udp6_gro_lookup_skb(struct sk_buff *skb, __be16 sport, |
117 | __be16 dport) |
118 | { |
119 | const struct ipv6hdr *iph = skb_gro_network_header(skb); |
120 | struct net *net = dev_net(dev: skb->dev); |
121 | int iif, sdif; |
122 | |
123 | inet6_get_iif_sdif(skb, iif: &iif, sdif: &sdif); |
124 | |
125 | return __udp6_lib_lookup(net, saddr: &iph->saddr, sport, |
126 | daddr: &iph->daddr, dport, dif: iif, |
127 | sdif, tbl: net->ipv4.udp_table, NULL); |
128 | } |
129 | |
130 | INDIRECT_CALLABLE_SCOPE |
131 | struct sk_buff *udp6_gro_receive(struct list_head *head, struct sk_buff *skb) |
132 | { |
133 | struct udphdr *uh = udp_gro_udphdr(skb); |
134 | struct sock *sk = NULL; |
135 | struct sk_buff *pp; |
136 | |
137 | if (unlikely(!uh)) |
138 | goto flush; |
139 | |
140 | /* Don't bother verifying checksum if we're going to flush anyway. */ |
141 | if (NAPI_GRO_CB(skb)->flush) |
142 | goto skip; |
143 | |
144 | if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check, |
145 | ip6_gro_compute_pseudo)) |
146 | goto flush; |
147 | else if (uh->check) |
148 | skb_gro_checksum_try_convert(skb, IPPROTO_UDP, |
149 | ip6_gro_compute_pseudo); |
150 | |
151 | skip: |
152 | NAPI_GRO_CB(skb)->is_ipv6 = 1; |
153 | |
154 | if (static_branch_unlikely(&udpv6_encap_needed_key)) |
155 | sk = udp6_gro_lookup_skb(skb, sport: uh->source, dport: uh->dest); |
156 | |
157 | pp = udp_gro_receive(head, skb, uh, sk); |
158 | return pp; |
159 | |
160 | flush: |
161 | NAPI_GRO_CB(skb)->flush = 1; |
162 | return NULL; |
163 | } |
164 | |
165 | INDIRECT_CALLABLE_SCOPE int udp6_gro_complete(struct sk_buff *skb, int nhoff) |
166 | { |
167 | const struct ipv6hdr *ipv6h = ipv6_hdr(skb); |
168 | struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); |
169 | |
170 | /* do fraglist only if there is no outer UDP encap (or we already processed it) */ |
171 | if (NAPI_GRO_CB(skb)->is_flist && !NAPI_GRO_CB(skb)->encap_mark) { |
172 | uh->len = htons(skb->len - nhoff); |
173 | |
174 | skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4); |
175 | skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count; |
176 | |
177 | if (skb->ip_summed == CHECKSUM_UNNECESSARY) { |
178 | if (skb->csum_level < SKB_MAX_CSUM_LEVEL) |
179 | skb->csum_level++; |
180 | } else { |
181 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
182 | skb->csum_level = 0; |
183 | } |
184 | |
185 | return 0; |
186 | } |
187 | |
188 | if (uh->check) |
189 | uh->check = ~udp_v6_check(len: skb->len - nhoff, saddr: &ipv6h->saddr, |
190 | daddr: &ipv6h->daddr, base: 0); |
191 | |
192 | return udp_gro_complete(skb, nhoff, lookup: udp6_lib_lookup_skb); |
193 | } |
194 | |
195 | static const struct net_offload udpv6_offload = { |
196 | .callbacks = { |
197 | .gso_segment = udp6_ufo_fragment, |
198 | .gro_receive = udp6_gro_receive, |
199 | .gro_complete = udp6_gro_complete, |
200 | }, |
201 | }; |
202 | |
203 | int udpv6_offload_init(void) |
204 | { |
205 | return inet6_add_offload(prot: &udpv6_offload, IPPROTO_UDP); |
206 | } |
207 | |
208 | int udpv6_offload_exit(void) |
209 | { |
210 | return inet6_del_offload(prot: &udpv6_offload, IPPROTO_UDP); |
211 | } |
212 | |