1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * IPV4 GSO/GRO offload support |
4 | * Linux INET implementation |
5 | * |
6 | * UDPv4 GSO support |
7 | */ |
8 | |
9 | #include <linux/skbuff.h> |
10 | #include <net/gro.h> |
11 | #include <net/gso.h> |
12 | #include <net/udp.h> |
13 | #include <net/protocol.h> |
14 | #include <net/inet_common.h> |
15 | |
16 | static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb, |
17 | netdev_features_t features, |
18 | struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, |
19 | netdev_features_t features), |
20 | __be16 new_protocol, bool is_ipv6) |
21 | { |
22 | int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb); |
23 | bool remcsum, need_csum, offload_csum, gso_partial; |
24 | struct sk_buff *segs = ERR_PTR(error: -EINVAL); |
25 | struct udphdr *uh = udp_hdr(skb); |
26 | u16 mac_offset = skb->mac_header; |
27 | __be16 protocol = skb->protocol; |
28 | u16 mac_len = skb->mac_len; |
29 | int udp_offset, outer_hlen; |
30 | __wsum partial; |
31 | bool need_ipsec; |
32 | |
33 | if (unlikely(!pskb_may_pull(skb, tnl_hlen))) |
34 | goto out; |
35 | |
36 | /* Adjust partial header checksum to negate old length. |
37 | * We cannot rely on the value contained in uh->len as it is |
38 | * possible that the actual value exceeds the boundaries of the |
39 | * 16 bit length field due to the header being added outside of an |
40 | * IP or IPv6 frame that was already limited to 64K - 1. |
41 | */ |
42 | if (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) |
43 | partial = (__force __wsum)uh->len; |
44 | else |
45 | partial = (__force __wsum)htonl(skb->len); |
46 | partial = csum_sub(csum: csum_unfold(n: uh->check), addend: partial); |
47 | |
48 | /* setup inner skb. */ |
49 | skb->encapsulation = 0; |
50 | SKB_GSO_CB(skb)->encap_level = 0; |
51 | __skb_pull(skb, len: tnl_hlen); |
52 | skb_reset_mac_header(skb); |
53 | skb_set_network_header(skb, offset: skb_inner_network_offset(skb)); |
54 | skb_set_transport_header(skb, offset: skb_inner_transport_offset(skb)); |
55 | skb->mac_len = skb_inner_network_offset(skb); |
56 | skb->protocol = new_protocol; |
57 | |
58 | need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM); |
59 | skb->encap_hdr_csum = need_csum; |
60 | |
61 | remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM); |
62 | skb->remcsum_offload = remcsum; |
63 | |
64 | need_ipsec = skb_dst(skb) && dst_xfrm(dst: skb_dst(skb)); |
65 | /* Try to offload checksum if possible */ |
66 | offload_csum = !!(need_csum && |
67 | !need_ipsec && |
68 | (skb->dev->features & |
69 | (is_ipv6 ? (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM) : |
70 | (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM)))); |
71 | |
72 | features &= skb->dev->hw_enc_features; |
73 | if (need_csum) |
74 | features &= ~NETIF_F_SCTP_CRC; |
75 | |
76 | /* The only checksum offload we care about from here on out is the |
77 | * outer one so strip the existing checksum feature flags and |
78 | * instead set the flag based on our outer checksum offload value. |
79 | */ |
80 | if (remcsum) { |
81 | features &= ~NETIF_F_CSUM_MASK; |
82 | if (!need_csum || offload_csum) |
83 | features |= NETIF_F_HW_CSUM; |
84 | } |
85 | |
86 | /* segment inner packet. */ |
87 | segs = gso_inner_segment(skb, features); |
88 | if (IS_ERR_OR_NULL(ptr: segs)) { |
89 | skb_gso_error_unwind(skb, protocol, pulled_hlen: tnl_hlen, mac_offset, |
90 | mac_len); |
91 | goto out; |
92 | } |
93 | |
94 | gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL); |
95 | |
96 | outer_hlen = skb_tnl_header_len(inner_skb: skb); |
97 | udp_offset = outer_hlen - tnl_hlen; |
98 | skb = segs; |
99 | do { |
100 | unsigned int len; |
101 | |
102 | if (remcsum) |
103 | skb->ip_summed = CHECKSUM_NONE; |
104 | |
105 | /* Set up inner headers if we are offloading inner checksum */ |
106 | if (skb->ip_summed == CHECKSUM_PARTIAL) { |
107 | skb_reset_inner_headers(skb); |
108 | skb->encapsulation = 1; |
109 | } |
110 | |
111 | skb->mac_len = mac_len; |
112 | skb->protocol = protocol; |
113 | |
114 | __skb_push(skb, len: outer_hlen); |
115 | skb_reset_mac_header(skb); |
116 | skb_set_network_header(skb, offset: mac_len); |
117 | skb_set_transport_header(skb, offset: udp_offset); |
118 | len = skb->len - udp_offset; |
119 | uh = udp_hdr(skb); |
120 | |
121 | /* If we are only performing partial GSO the inner header |
122 | * will be using a length value equal to only one MSS sized |
123 | * segment instead of the entire frame. |
124 | */ |
125 | if (gso_partial && skb_is_gso(skb)) { |
126 | uh->len = htons(skb_shinfo(skb)->gso_size + |
127 | SKB_GSO_CB(skb)->data_offset + |
128 | skb->head - (unsigned char *)uh); |
129 | } else { |
130 | uh->len = htons(len); |
131 | } |
132 | |
133 | if (!need_csum) |
134 | continue; |
135 | |
136 | uh->check = ~csum_fold(sum: csum_add(csum: partial, |
137 | addend: (__force __wsum)htonl(len))); |
138 | |
139 | if (skb->encapsulation || !offload_csum) { |
140 | uh->check = gso_make_checksum(skb, res: ~uh->check); |
141 | if (uh->check == 0) |
142 | uh->check = CSUM_MANGLED_0; |
143 | } else { |
144 | skb->ip_summed = CHECKSUM_PARTIAL; |
145 | skb->csum_start = skb_transport_header(skb) - skb->head; |
146 | skb->csum_offset = offsetof(struct udphdr, check); |
147 | } |
148 | } while ((skb = skb->next)); |
149 | out: |
150 | return segs; |
151 | } |
152 | |
153 | struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, |
154 | netdev_features_t features, |
155 | bool is_ipv6) |
156 | { |
157 | const struct net_offload __rcu **offloads; |
158 | __be16 protocol = skb->protocol; |
159 | const struct net_offload *ops; |
160 | struct sk_buff *segs = ERR_PTR(error: -EINVAL); |
161 | struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, |
162 | netdev_features_t features); |
163 | |
164 | rcu_read_lock(); |
165 | |
166 | switch (skb->inner_protocol_type) { |
167 | case ENCAP_TYPE_ETHER: |
168 | protocol = skb->inner_protocol; |
169 | gso_inner_segment = skb_mac_gso_segment; |
170 | break; |
171 | case ENCAP_TYPE_IPPROTO: |
172 | offloads = is_ipv6 ? inet6_offloads : inet_offloads; |
173 | ops = rcu_dereference(offloads[skb->inner_ipproto]); |
174 | if (!ops || !ops->callbacks.gso_segment) |
175 | goto out_unlock; |
176 | gso_inner_segment = ops->callbacks.gso_segment; |
177 | break; |
178 | default: |
179 | goto out_unlock; |
180 | } |
181 | |
182 | segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment, |
183 | new_protocol: protocol, is_ipv6); |
184 | |
185 | out_unlock: |
186 | rcu_read_unlock(); |
187 | |
188 | return segs; |
189 | } |
190 | EXPORT_SYMBOL(skb_udp_tunnel_segment); |
191 | |
192 | static void __udpv4_gso_segment_csum(struct sk_buff *seg, |
193 | __be32 *oldip, __be32 *newip, |
194 | __be16 *oldport, __be16 *newport) |
195 | { |
196 | struct udphdr *uh; |
197 | struct iphdr *iph; |
198 | |
199 | if (*oldip == *newip && *oldport == *newport) |
200 | return; |
201 | |
202 | uh = udp_hdr(skb: seg); |
203 | iph = ip_hdr(skb: seg); |
204 | |
205 | if (uh->check) { |
206 | inet_proto_csum_replace4(sum: &uh->check, skb: seg, from: *oldip, to: *newip, |
207 | pseudohdr: true); |
208 | inet_proto_csum_replace2(sum: &uh->check, skb: seg, from: *oldport, to: *newport, |
209 | pseudohdr: false); |
210 | if (!uh->check) |
211 | uh->check = CSUM_MANGLED_0; |
212 | } |
213 | *oldport = *newport; |
214 | |
215 | csum_replace4(sum: &iph->check, from: *oldip, to: *newip); |
216 | *oldip = *newip; |
217 | } |
218 | |
219 | static struct sk_buff *__udpv4_gso_segment_list_csum(struct sk_buff *segs) |
220 | { |
221 | struct sk_buff *seg; |
222 | struct udphdr *uh, *uh2; |
223 | struct iphdr *iph, *iph2; |
224 | |
225 | seg = segs; |
226 | uh = udp_hdr(skb: seg); |
227 | iph = ip_hdr(skb: seg); |
228 | |
229 | if ((udp_hdr(skb: seg)->dest == udp_hdr(skb: seg->next)->dest) && |
230 | (udp_hdr(skb: seg)->source == udp_hdr(skb: seg->next)->source) && |
231 | (ip_hdr(skb: seg)->daddr == ip_hdr(skb: seg->next)->daddr) && |
232 | (ip_hdr(skb: seg)->saddr == ip_hdr(skb: seg->next)->saddr)) |
233 | return segs; |
234 | |
235 | while ((seg = seg->next)) { |
236 | uh2 = udp_hdr(skb: seg); |
237 | iph2 = ip_hdr(skb: seg); |
238 | |
239 | __udpv4_gso_segment_csum(seg, |
240 | oldip: &iph2->saddr, newip: &iph->saddr, |
241 | oldport: &uh2->source, newport: &uh->source); |
242 | __udpv4_gso_segment_csum(seg, |
243 | oldip: &iph2->daddr, newip: &iph->daddr, |
244 | oldport: &uh2->dest, newport: &uh->dest); |
245 | } |
246 | |
247 | return segs; |
248 | } |
249 | |
250 | static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb, |
251 | netdev_features_t features, |
252 | bool is_ipv6) |
253 | { |
254 | unsigned int mss = skb_shinfo(skb)->gso_size; |
255 | |
256 | skb = skb_segment_list(skb, features, offset: skb_mac_header_len(skb)); |
257 | if (IS_ERR(ptr: skb)) |
258 | return skb; |
259 | |
260 | udp_hdr(skb)->len = htons(sizeof(struct udphdr) + mss); |
261 | |
262 | return is_ipv6 ? skb : __udpv4_gso_segment_list_csum(segs: skb); |
263 | } |
264 | |
265 | struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, |
266 | netdev_features_t features, bool is_ipv6) |
267 | { |
268 | struct sock *sk = gso_skb->sk; |
269 | unsigned int sum_truesize = 0; |
270 | struct sk_buff *segs, *seg; |
271 | struct udphdr *uh; |
272 | unsigned int mss; |
273 | bool copy_dtor; |
274 | __sum16 check; |
275 | __be16 newlen; |
276 | |
277 | mss = skb_shinfo(gso_skb)->gso_size; |
278 | if (gso_skb->len <= sizeof(*uh) + mss) |
279 | return ERR_PTR(error: -EINVAL); |
280 | |
281 | if (skb_gso_ok(skb: gso_skb, features: features | NETIF_F_GSO_ROBUST)) { |
282 | /* Packet is from an untrusted source, reset gso_segs. */ |
283 | skb_shinfo(gso_skb)->gso_segs = DIV_ROUND_UP(gso_skb->len - sizeof(*uh), |
284 | mss); |
285 | return NULL; |
286 | } |
287 | |
288 | if (skb_shinfo(gso_skb)->gso_type & SKB_GSO_FRAGLIST) |
289 | return __udp_gso_segment_list(skb: gso_skb, features, is_ipv6); |
290 | |
291 | skb_pull(skb: gso_skb, len: sizeof(*uh)); |
292 | |
293 | /* clear destructor to avoid skb_segment assigning it to tail */ |
294 | copy_dtor = gso_skb->destructor == sock_wfree; |
295 | if (copy_dtor) |
296 | gso_skb->destructor = NULL; |
297 | |
298 | segs = skb_segment(skb: gso_skb, features); |
299 | if (IS_ERR_OR_NULL(ptr: segs)) { |
300 | if (copy_dtor) |
301 | gso_skb->destructor = sock_wfree; |
302 | return segs; |
303 | } |
304 | |
305 | /* GSO partial and frag_list segmentation only requires splitting |
306 | * the frame into an MSS multiple and possibly a remainder, both |
307 | * cases return a GSO skb. So update the mss now. |
308 | */ |
309 | if (skb_is_gso(skb: segs)) |
310 | mss *= skb_shinfo(segs)->gso_segs; |
311 | |
312 | seg = segs; |
313 | uh = udp_hdr(skb: seg); |
314 | |
315 | /* preserve TX timestamp flags and TS key for first segment */ |
316 | skb_shinfo(seg)->tskey = skb_shinfo(gso_skb)->tskey; |
317 | skb_shinfo(seg)->tx_flags |= |
318 | (skb_shinfo(gso_skb)->tx_flags & SKBTX_ANY_TSTAMP); |
319 | |
320 | /* compute checksum adjustment based on old length versus new */ |
321 | newlen = htons(sizeof(*uh) + mss); |
322 | check = csum16_add(csum: csum16_sub(csum: uh->check, addend: uh->len), addend: newlen); |
323 | |
324 | for (;;) { |
325 | if (copy_dtor) { |
326 | seg->destructor = sock_wfree; |
327 | seg->sk = sk; |
328 | sum_truesize += seg->truesize; |
329 | } |
330 | |
331 | if (!seg->next) |
332 | break; |
333 | |
334 | uh->len = newlen; |
335 | uh->check = check; |
336 | |
337 | if (seg->ip_summed == CHECKSUM_PARTIAL) |
338 | gso_reset_checksum(skb: seg, res: ~check); |
339 | else |
340 | uh->check = gso_make_checksum(skb: seg, res: ~check) ? : |
341 | CSUM_MANGLED_0; |
342 | |
343 | seg = seg->next; |
344 | uh = udp_hdr(skb: seg); |
345 | } |
346 | |
347 | /* last packet can be partial gso_size, account for that in checksum */ |
348 | newlen = htons(skb_tail_pointer(seg) - skb_transport_header(seg) + |
349 | seg->data_len); |
350 | check = csum16_add(csum: csum16_sub(csum: uh->check, addend: uh->len), addend: newlen); |
351 | |
352 | uh->len = newlen; |
353 | uh->check = check; |
354 | |
355 | if (seg->ip_summed == CHECKSUM_PARTIAL) |
356 | gso_reset_checksum(skb: seg, res: ~check); |
357 | else |
358 | uh->check = gso_make_checksum(skb: seg, res: ~check) ? : CSUM_MANGLED_0; |
359 | |
360 | /* update refcount for the packet */ |
361 | if (copy_dtor) { |
362 | int delta = sum_truesize - gso_skb->truesize; |
363 | |
364 | /* In some pathological cases, delta can be negative. |
365 | * We need to either use refcount_add() or refcount_sub_and_test() |
366 | */ |
367 | if (likely(delta >= 0)) |
368 | refcount_add(i: delta, r: &sk->sk_wmem_alloc); |
369 | else |
370 | WARN_ON_ONCE(refcount_sub_and_test(-delta, &sk->sk_wmem_alloc)); |
371 | } |
372 | return segs; |
373 | } |
374 | EXPORT_SYMBOL_GPL(__udp_gso_segment); |
375 | |
376 | static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, |
377 | netdev_features_t features) |
378 | { |
379 | struct sk_buff *segs = ERR_PTR(error: -EINVAL); |
380 | unsigned int mss; |
381 | __wsum csum; |
382 | struct udphdr *uh; |
383 | struct iphdr *iph; |
384 | |
385 | if (skb->encapsulation && |
386 | (skb_shinfo(skb)->gso_type & |
387 | (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) { |
388 | segs = skb_udp_tunnel_segment(skb, features, false); |
389 | goto out; |
390 | } |
391 | |
392 | if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4))) |
393 | goto out; |
394 | |
395 | if (!pskb_may_pull(skb, len: sizeof(struct udphdr))) |
396 | goto out; |
397 | |
398 | if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) |
399 | return __udp_gso_segment(skb, features, false); |
400 | |
401 | mss = skb_shinfo(skb)->gso_size; |
402 | if (unlikely(skb->len <= mss)) |
403 | goto out; |
404 | |
405 | /* Do software UFO. Complete and fill in the UDP checksum as |
406 | * HW cannot do checksum of UDP packets sent as multiple |
407 | * IP fragments. |
408 | */ |
409 | |
410 | uh = udp_hdr(skb); |
411 | iph = ip_hdr(skb); |
412 | |
413 | uh->check = 0; |
414 | csum = skb_checksum(skb, offset: 0, len: skb->len, csum: 0); |
415 | uh->check = udp_v4_check(len: skb->len, saddr: iph->saddr, daddr: iph->daddr, base: csum); |
416 | if (uh->check == 0) |
417 | uh->check = CSUM_MANGLED_0; |
418 | |
419 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
420 | |
421 | /* If there is no outer header we can fake a checksum offload |
422 | * due to the fact that we have already done the checksum in |
423 | * software prior to segmenting the frame. |
424 | */ |
425 | if (!skb->encap_hdr_csum) |
426 | features |= NETIF_F_HW_CSUM; |
427 | |
428 | /* Fragment the skb. IP headers of the fragments are updated in |
429 | * inet_gso_segment() |
430 | */ |
431 | segs = skb_segment(skb, features); |
432 | out: |
433 | return segs; |
434 | } |
435 | |
436 | static int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) |
437 | { |
438 | if (unlikely(p->len + skb->len >= 65536)) |
439 | return -E2BIG; |
440 | |
441 | if (NAPI_GRO_CB(p)->last == p) |
442 | skb_shinfo(p)->frag_list = skb; |
443 | else |
444 | NAPI_GRO_CB(p)->last->next = skb; |
445 | |
446 | skb_pull(skb, len: skb_gro_offset(skb)); |
447 | |
448 | NAPI_GRO_CB(p)->last = skb; |
449 | NAPI_GRO_CB(p)->count++; |
450 | p->data_len += skb->len; |
451 | |
452 | /* sk owenrship - if any - completely transferred to the aggregated packet */ |
453 | skb->destructor = NULL; |
454 | p->truesize += skb->truesize; |
455 | p->len += skb->len; |
456 | |
457 | NAPI_GRO_CB(skb)->same_flow = 1; |
458 | |
459 | return 0; |
460 | } |
461 | |
462 | |
463 | #define UDP_GRO_CNT_MAX 64 |
464 | static struct sk_buff *udp_gro_receive_segment(struct list_head *head, |
465 | struct sk_buff *skb) |
466 | { |
467 | struct udphdr *uh = udp_gro_udphdr(skb); |
468 | struct sk_buff *pp = NULL; |
469 | struct udphdr *uh2; |
470 | struct sk_buff *p; |
471 | unsigned int ulen; |
472 | int ret = 0; |
473 | |
474 | /* requires non zero csum, for symmetry with GSO */ |
475 | if (!uh->check) { |
476 | NAPI_GRO_CB(skb)->flush = 1; |
477 | return NULL; |
478 | } |
479 | |
480 | /* Do not deal with padded or malicious packets, sorry ! */ |
481 | ulen = ntohs(uh->len); |
482 | if (ulen <= sizeof(*uh) || ulen != skb_gro_len(skb)) { |
483 | NAPI_GRO_CB(skb)->flush = 1; |
484 | return NULL; |
485 | } |
486 | /* pull encapsulating udp header */ |
487 | skb_gro_pull(skb, len: sizeof(struct udphdr)); |
488 | |
489 | list_for_each_entry(p, head, list) { |
490 | if (!NAPI_GRO_CB(p)->same_flow) |
491 | continue; |
492 | |
493 | uh2 = udp_hdr(skb: p); |
494 | |
495 | /* Match ports only, as csum is always non zero */ |
496 | if ((*(u32 *)&uh->source != *(u32 *)&uh2->source)) { |
497 | NAPI_GRO_CB(p)->same_flow = 0; |
498 | continue; |
499 | } |
500 | |
501 | if (NAPI_GRO_CB(skb)->is_flist != NAPI_GRO_CB(p)->is_flist) { |
502 | NAPI_GRO_CB(skb)->flush = 1; |
503 | return p; |
504 | } |
505 | |
506 | /* Terminate the flow on len mismatch or if it grow "too much". |
507 | * Under small packet flood GRO count could elsewhere grow a lot |
508 | * leading to excessive truesize values. |
509 | * On len mismatch merge the first packet shorter than gso_size, |
510 | * otherwise complete the GRO packet. |
511 | */ |
512 | if (ulen > ntohs(uh2->len)) { |
513 | pp = p; |
514 | } else { |
515 | if (NAPI_GRO_CB(skb)->is_flist) { |
516 | if (!pskb_may_pull(skb, len: skb_gro_offset(skb))) { |
517 | NAPI_GRO_CB(skb)->flush = 1; |
518 | return NULL; |
519 | } |
520 | if ((skb->ip_summed != p->ip_summed) || |
521 | (skb->csum_level != p->csum_level)) { |
522 | NAPI_GRO_CB(skb)->flush = 1; |
523 | return NULL; |
524 | } |
525 | ret = skb_gro_receive_list(p, skb); |
526 | } else { |
527 | skb_gro_postpull_rcsum(skb, start: uh, |
528 | len: sizeof(struct udphdr)); |
529 | |
530 | ret = skb_gro_receive(p, skb); |
531 | } |
532 | } |
533 | |
534 | if (ret || ulen != ntohs(uh2->len) || |
535 | NAPI_GRO_CB(p)->count >= UDP_GRO_CNT_MAX) |
536 | pp = p; |
537 | |
538 | return pp; |
539 | } |
540 | |
541 | /* mismatch, but we never need to flush */ |
542 | return NULL; |
543 | } |
544 | |
545 | struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, |
546 | struct udphdr *uh, struct sock *sk) |
547 | { |
548 | struct sk_buff *pp = NULL; |
549 | struct sk_buff *p; |
550 | struct udphdr *uh2; |
551 | unsigned int off = skb_gro_offset(skb); |
552 | int flush = 1; |
553 | |
554 | /* we can do L4 aggregation only if the packet can't land in a tunnel |
555 | * otherwise we could corrupt the inner stream |
556 | */ |
557 | NAPI_GRO_CB(skb)->is_flist = 0; |
558 | if (!sk || !udp_sk(sk)->gro_receive) { |
559 | if (skb->dev->features & NETIF_F_GRO_FRAGLIST) |
560 | NAPI_GRO_CB(skb)->is_flist = sk ? !udp_test_bit(GRO_ENABLED, sk) : 1; |
561 | |
562 | if ((!sk && (skb->dev->features & NETIF_F_GRO_UDP_FWD)) || |
563 | (sk && udp_test_bit(GRO_ENABLED, sk)) || NAPI_GRO_CB(skb)->is_flist) |
564 | return call_gro_receive(cb: udp_gro_receive_segment, head, skb); |
565 | |
566 | /* no GRO, be sure flush the current packet */ |
567 | goto out; |
568 | } |
569 | |
570 | if (NAPI_GRO_CB(skb)->encap_mark || |
571 | (uh->check && skb->ip_summed != CHECKSUM_PARTIAL && |
572 | NAPI_GRO_CB(skb)->csum_cnt == 0 && |
573 | !NAPI_GRO_CB(skb)->csum_valid)) |
574 | goto out; |
575 | |
576 | /* mark that this skb passed once through the tunnel gro layer */ |
577 | NAPI_GRO_CB(skb)->encap_mark = 1; |
578 | |
579 | flush = 0; |
580 | |
581 | list_for_each_entry(p, head, list) { |
582 | if (!NAPI_GRO_CB(p)->same_flow) |
583 | continue; |
584 | |
585 | uh2 = (struct udphdr *)(p->data + off); |
586 | |
587 | /* Match ports and either checksums are either both zero |
588 | * or nonzero. |
589 | */ |
590 | if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) || |
591 | (!uh->check ^ !uh2->check)) { |
592 | NAPI_GRO_CB(p)->same_flow = 0; |
593 | continue; |
594 | } |
595 | } |
596 | |
597 | skb_gro_pull(skb, len: sizeof(struct udphdr)); /* pull encapsulating udp header */ |
598 | skb_gro_postpull_rcsum(skb, start: uh, len: sizeof(struct udphdr)); |
599 | pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb); |
600 | |
601 | out: |
602 | skb_gro_flush_final(skb, pp, flush); |
603 | return pp; |
604 | } |
605 | EXPORT_SYMBOL(udp_gro_receive); |
606 | |
607 | static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport, |
608 | __be16 dport) |
609 | { |
610 | const struct iphdr *iph = skb_gro_network_header(skb); |
611 | struct net *net = dev_net(dev: skb->dev); |
612 | int iif, sdif; |
613 | |
614 | inet_get_iif_sdif(skb, iif: &iif, sdif: &sdif); |
615 | |
616 | return __udp4_lib_lookup(net, saddr: iph->saddr, sport, |
617 | daddr: iph->daddr, dport, dif: iif, |
618 | sdif, tbl: net->ipv4.udp_table, NULL); |
619 | } |
620 | |
621 | INDIRECT_CALLABLE_SCOPE |
622 | struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb) |
623 | { |
624 | struct udphdr *uh = udp_gro_udphdr(skb); |
625 | struct sock *sk = NULL; |
626 | struct sk_buff *pp; |
627 | |
628 | if (unlikely(!uh)) |
629 | goto flush; |
630 | |
631 | /* Don't bother verifying checksum if we're going to flush anyway. */ |
632 | if (NAPI_GRO_CB(skb)->flush) |
633 | goto skip; |
634 | |
635 | if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check, |
636 | inet_gro_compute_pseudo)) |
637 | goto flush; |
638 | else if (uh->check) |
639 | skb_gro_checksum_try_convert(skb, IPPROTO_UDP, |
640 | inet_gro_compute_pseudo); |
641 | skip: |
642 | NAPI_GRO_CB(skb)->is_ipv6 = 0; |
643 | |
644 | if (static_branch_unlikely(&udp_encap_needed_key)) |
645 | sk = udp4_gro_lookup_skb(skb, sport: uh->source, dport: uh->dest); |
646 | |
647 | pp = udp_gro_receive(head, skb, uh, sk); |
648 | return pp; |
649 | |
650 | flush: |
651 | NAPI_GRO_CB(skb)->flush = 1; |
652 | return NULL; |
653 | } |
654 | |
655 | static int udp_gro_complete_segment(struct sk_buff *skb) |
656 | { |
657 | struct udphdr *uh = udp_hdr(skb); |
658 | |
659 | skb->csum_start = (unsigned char *)uh - skb->head; |
660 | skb->csum_offset = offsetof(struct udphdr, check); |
661 | skb->ip_summed = CHECKSUM_PARTIAL; |
662 | |
663 | skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count; |
664 | skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4; |
665 | |
666 | if (skb->encapsulation) |
667 | skb->inner_transport_header = skb->transport_header; |
668 | |
669 | return 0; |
670 | } |
671 | |
672 | int udp_gro_complete(struct sk_buff *skb, int nhoff, |
673 | udp_lookup_t lookup) |
674 | { |
675 | __be16 newlen = htons(skb->len - nhoff); |
676 | struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); |
677 | struct sock *sk; |
678 | int err; |
679 | |
680 | uh->len = newlen; |
681 | |
682 | sk = INDIRECT_CALL_INET(lookup, udp6_lib_lookup_skb, |
683 | udp4_lib_lookup_skb, skb, uh->source, uh->dest); |
684 | if (sk && udp_sk(sk)->gro_complete) { |
685 | skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM |
686 | : SKB_GSO_UDP_TUNNEL; |
687 | |
688 | /* clear the encap mark, so that inner frag_list gro_complete |
689 | * can take place |
690 | */ |
691 | NAPI_GRO_CB(skb)->encap_mark = 0; |
692 | |
693 | /* Set encapsulation before calling into inner gro_complete() |
694 | * functions to make them set up the inner offsets. |
695 | */ |
696 | skb->encapsulation = 1; |
697 | err = udp_sk(sk)->gro_complete(sk, skb, |
698 | nhoff + sizeof(struct udphdr)); |
699 | } else { |
700 | err = udp_gro_complete_segment(skb); |
701 | } |
702 | |
703 | if (skb->remcsum_offload) |
704 | skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM; |
705 | |
706 | return err; |
707 | } |
708 | EXPORT_SYMBOL(udp_gro_complete); |
709 | |
710 | INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff) |
711 | { |
712 | const struct iphdr *iph = ip_hdr(skb); |
713 | struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); |
714 | |
715 | /* do fraglist only if there is no outer UDP encap (or we already processed it) */ |
716 | if (NAPI_GRO_CB(skb)->is_flist && !NAPI_GRO_CB(skb)->encap_mark) { |
717 | uh->len = htons(skb->len - nhoff); |
718 | |
719 | skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4); |
720 | skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count; |
721 | |
722 | if (skb->ip_summed == CHECKSUM_UNNECESSARY) { |
723 | if (skb->csum_level < SKB_MAX_CSUM_LEVEL) |
724 | skb->csum_level++; |
725 | } else { |
726 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
727 | skb->csum_level = 0; |
728 | } |
729 | |
730 | return 0; |
731 | } |
732 | |
733 | if (uh->check) |
734 | uh->check = ~udp_v4_check(len: skb->len - nhoff, saddr: iph->saddr, |
735 | daddr: iph->daddr, base: 0); |
736 | |
737 | return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb); |
738 | } |
739 | |
740 | static const struct net_offload udpv4_offload = { |
741 | .callbacks = { |
742 | .gso_segment = udp4_ufo_fragment, |
743 | .gro_receive = udp4_gro_receive, |
744 | .gro_complete = udp4_gro_complete, |
745 | }, |
746 | }; |
747 | |
748 | int __init udpv4_offload_init(void) |
749 | { |
750 | return inet_add_offload(prot: &udpv4_offload, IPPROTO_UDP); |
751 | } |
752 | |