1/*
2 * Routines having to do with the 'struct sk_buff' memory handlers.
3 *
4 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
5 * Florian La Roche <rzsfl@rz.uni-sb.de>
6 *
7 * Fixes:
8 * Alan Cox : Fixed the worst of the load
9 * balancer bugs.
10 * Dave Platt : Interrupt stacking fix.
11 * Richard Kooijman : Timestamp fixes.
12 * Alan Cox : Changed buffer format.
13 * Alan Cox : destructor hook for AF_UNIX etc.
14 * Linus Torvalds : Better skb_clone.
15 * Alan Cox : Added skb_copy.
16 * Alan Cox : Added all the changed routines Linus
17 * only put in the headers
18 * Ray VanTassle : Fixed --skb->lock in free
19 * Alan Cox : skb_copy copy arp field
20 * Andi Kleen : slabified it.
21 * Robert Olsson : Removed skb_head_pool
22 *
23 * NOTE:
24 * The __skb_ routines should be called with interrupts
25 * disabled, or you better be *real* sure that the operation is atomic
26 * with respect to whatever list is being frobbed (e.g. via lock_sock()
27 * or via disabling bottom half handlers, etc).
28 *
29 * This program is free software; you can redistribute it and/or
30 * modify it under the terms of the GNU General Public License
31 * as published by the Free Software Foundation; either version
32 * 2 of the License, or (at your option) any later version.
33 */
34
35/*
36 * The functions in this file will not compile correctly with gcc 2.4.x
37 */
38
39#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
40
41#include <linux/module.h>
42#include <linux/types.h>
43#include <linux/kernel.h>
44#include <linux/mm.h>
45#include <linux/interrupt.h>
46#include <linux/in.h>
47#include <linux/inet.h>
48#include <linux/slab.h>
49#include <linux/tcp.h>
50#include <linux/udp.h>
51#include <linux/sctp.h>
52#include <linux/netdevice.h>
53#ifdef CONFIG_NET_CLS_ACT
54#include <net/pkt_sched.h>
55#endif
56#include <linux/string.h>
57#include <linux/skbuff.h>
58#include <linux/splice.h>
59#include <linux/cache.h>
60#include <linux/rtnetlink.h>
61#include <linux/init.h>
62#include <linux/scatterlist.h>
63#include <linux/errqueue.h>
64#include <linux/prefetch.h>
65#include <linux/if_vlan.h>
66
67#include <net/protocol.h>
68#include <net/dst.h>
69#include <net/sock.h>
70#include <net/checksum.h>
71#include <net/ip6_checksum.h>
72#include <net/xfrm.h>
73
74#include <linux/uaccess.h>
75#include <trace/events/skb.h>
76#include <linux/highmem.h>
77#include <linux/capability.h>
78#include <linux/user_namespace.h>
79
80struct kmem_cache *skbuff_head_cache __ro_after_init;
81static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
82#ifdef CONFIG_SKB_EXTENSIONS
83static struct kmem_cache *skbuff_ext_cache __ro_after_init;
84#endif
85int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
86EXPORT_SYMBOL(sysctl_max_skb_frags);
87
88/**
89 * skb_panic - private function for out-of-line support
90 * @skb: buffer
91 * @sz: size
92 * @addr: address
93 * @msg: skb_over_panic or skb_under_panic
94 *
95 * Out-of-line support for skb_put() and skb_push().
96 * Called via the wrapper skb_over_panic() or skb_under_panic().
97 * Keep out of line to prevent kernel bloat.
98 * __builtin_return_address is not used because it is not always reliable.
99 */
100static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
101 const char msg[])
102{
103 pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
104 msg, addr, skb->len, sz, skb->head, skb->data,
105 (unsigned long)skb->tail, (unsigned long)skb->end,
106 skb->dev ? skb->dev->name : "<NULL>");
107 BUG();
108}
109
110static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
111{
112 skb_panic(skb, sz, addr, __func__);
113}
114
115static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
116{
117 skb_panic(skb, sz, addr, __func__);
118}
119
120/*
121 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
122 * the caller if emergency pfmemalloc reserves are being used. If it is and
123 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
124 * may be used. Otherwise, the packet data may be discarded until enough
125 * memory is free
126 */
127#define kmalloc_reserve(size, gfp, node, pfmemalloc) \
128 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
129
130static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
131 unsigned long ip, bool *pfmemalloc)
132{
133 void *obj;
134 bool ret_pfmemalloc = false;
135
136 /*
137 * Try a regular allocation, when that fails and we're not entitled
138 * to the reserves, fail.
139 */
140 obj = kmalloc_node_track_caller(size,
141 flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
142 node);
143 if (obj || !(gfp_pfmemalloc_allowed(flags)))
144 goto out;
145
146 /* Try again but now we are using pfmemalloc reserves */
147 ret_pfmemalloc = true;
148 obj = kmalloc_node_track_caller(size, flags, node);
149
150out:
151 if (pfmemalloc)
152 *pfmemalloc = ret_pfmemalloc;
153
154 return obj;
155}
156
157/* Allocate a new skbuff. We do this ourselves so we can fill in a few
158 * 'private' fields and also do memory statistics to find all the
159 * [BEEP] leaks.
160 *
161 */
162
163/**
164 * __alloc_skb - allocate a network buffer
165 * @size: size to allocate
166 * @gfp_mask: allocation mask
167 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
168 * instead of head cache and allocate a cloned (child) skb.
169 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
170 * allocations in case the data is required for writeback
171 * @node: numa node to allocate memory on
172 *
173 * Allocate a new &sk_buff. The returned buffer has no headroom and a
174 * tail room of at least size bytes. The object has a reference count
175 * of one. The return is the buffer. On a failure the return is %NULL.
176 *
177 * Buffers may only be allocated from interrupts using a @gfp_mask of
178 * %GFP_ATOMIC.
179 */
180struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
181 int flags, int node)
182{
183 struct kmem_cache *cache;
184 struct skb_shared_info *shinfo;
185 struct sk_buff *skb;
186 u8 *data;
187 bool pfmemalloc;
188
189 cache = (flags & SKB_ALLOC_FCLONE)
190 ? skbuff_fclone_cache : skbuff_head_cache;
191
192 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
193 gfp_mask |= __GFP_MEMALLOC;
194
195 /* Get the HEAD */
196 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
197 if (!skb)
198 goto out;
199 prefetchw(skb);
200
201 /* We do our best to align skb_shared_info on a separate cache
202 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
203 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
204 * Both skb->head and skb_shared_info are cache line aligned.
205 */
206 size = SKB_DATA_ALIGN(size);
207 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
208 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
209 if (!data)
210 goto nodata;
211 /* kmalloc(size) might give us more room than requested.
212 * Put skb_shared_info exactly at the end of allocated zone,
213 * to allow max possible filling before reallocation.
214 */
215 size = SKB_WITH_OVERHEAD(ksize(data));
216 prefetchw(data + size);
217
218 /*
219 * Only clear those fields we need to clear, not those that we will
220 * actually initialise below. Hence, don't put any more fields after
221 * the tail pointer in struct sk_buff!
222 */
223 memset(skb, 0, offsetof(struct sk_buff, tail));
224 /* Account for allocated memory : skb + skb->head */
225 skb->truesize = SKB_TRUESIZE(size);
226 skb->pfmemalloc = pfmemalloc;
227 refcount_set(&skb->users, 1);
228 skb->head = data;
229 skb->data = data;
230 skb_reset_tail_pointer(skb);
231 skb->end = skb->tail + size;
232 skb->mac_header = (typeof(skb->mac_header))~0U;
233 skb->transport_header = (typeof(skb->transport_header))~0U;
234
235 /* make sure we initialize shinfo sequentially */
236 shinfo = skb_shinfo(skb);
237 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
238 atomic_set(&shinfo->dataref, 1);
239
240 if (flags & SKB_ALLOC_FCLONE) {
241 struct sk_buff_fclones *fclones;
242
243 fclones = container_of(skb, struct sk_buff_fclones, skb1);
244
245 skb->fclone = SKB_FCLONE_ORIG;
246 refcount_set(&fclones->fclone_ref, 1);
247
248 fclones->skb2.fclone = SKB_FCLONE_CLONE;
249 }
250out:
251 return skb;
252nodata:
253 kmem_cache_free(cache, skb);
254 skb = NULL;
255 goto out;
256}
257EXPORT_SYMBOL(__alloc_skb);
258
259/**
260 * __build_skb - build a network buffer
261 * @data: data buffer provided by caller
262 * @frag_size: size of data, or 0 if head was kmalloced
263 *
264 * Allocate a new &sk_buff. Caller provides space holding head and
265 * skb_shared_info. @data must have been allocated by kmalloc() only if
266 * @frag_size is 0, otherwise data should come from the page allocator
267 * or vmalloc()
268 * The return is the new skb buffer.
269 * On a failure the return is %NULL, and @data is not freed.
270 * Notes :
271 * Before IO, driver allocates only data buffer where NIC put incoming frame
272 * Driver should add room at head (NET_SKB_PAD) and
273 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
274 * After IO, driver calls build_skb(), to allocate sk_buff and populate it
275 * before giving packet to stack.
276 * RX rings only contains data buffers, not full skbs.
277 */
278struct sk_buff *__build_skb(void *data, unsigned int frag_size)
279{
280 struct skb_shared_info *shinfo;
281 struct sk_buff *skb;
282 unsigned int size = frag_size ? : ksize(data);
283
284 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
285 if (!skb)
286 return NULL;
287
288 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
289
290 memset(skb, 0, offsetof(struct sk_buff, tail));
291 skb->truesize = SKB_TRUESIZE(size);
292 refcount_set(&skb->users, 1);
293 skb->head = data;
294 skb->data = data;
295 skb_reset_tail_pointer(skb);
296 skb->end = skb->tail + size;
297 skb->mac_header = (typeof(skb->mac_header))~0U;
298 skb->transport_header = (typeof(skb->transport_header))~0U;
299
300 /* make sure we initialize shinfo sequentially */
301 shinfo = skb_shinfo(skb);
302 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
303 atomic_set(&shinfo->dataref, 1);
304
305 return skb;
306}
307
308/* build_skb() is wrapper over __build_skb(), that specifically
309 * takes care of skb->head and skb->pfmemalloc
310 * This means that if @frag_size is not zero, then @data must be backed
311 * by a page fragment, not kmalloc() or vmalloc()
312 */
313struct sk_buff *build_skb(void *data, unsigned int frag_size)
314{
315 struct sk_buff *skb = __build_skb(data, frag_size);
316
317 if (skb && frag_size) {
318 skb->head_frag = 1;
319 if (page_is_pfmemalloc(virt_to_head_page(data)))
320 skb->pfmemalloc = 1;
321 }
322 return skb;
323}
324EXPORT_SYMBOL(build_skb);
325
326#define NAPI_SKB_CACHE_SIZE 64
327
328struct napi_alloc_cache {
329 struct page_frag_cache page;
330 unsigned int skb_count;
331 void *skb_cache[NAPI_SKB_CACHE_SIZE];
332};
333
334static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
335static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
336
337static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
338{
339 struct page_frag_cache *nc;
340 unsigned long flags;
341 void *data;
342
343 local_irq_save(flags);
344 nc = this_cpu_ptr(&netdev_alloc_cache);
345 data = page_frag_alloc(nc, fragsz, gfp_mask);
346 local_irq_restore(flags);
347 return data;
348}
349
350/**
351 * netdev_alloc_frag - allocate a page fragment
352 * @fragsz: fragment size
353 *
354 * Allocates a frag from a page for receive buffer.
355 * Uses GFP_ATOMIC allocations.
356 */
357void *netdev_alloc_frag(unsigned int fragsz)
358{
359 fragsz = SKB_DATA_ALIGN(fragsz);
360
361 return __netdev_alloc_frag(fragsz, GFP_ATOMIC);
362}
363EXPORT_SYMBOL(netdev_alloc_frag);
364
365static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
366{
367 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
368
369 return page_frag_alloc(&nc->page, fragsz, gfp_mask);
370}
371
372void *napi_alloc_frag(unsigned int fragsz)
373{
374 fragsz = SKB_DATA_ALIGN(fragsz);
375
376 return __napi_alloc_frag(fragsz, GFP_ATOMIC);
377}
378EXPORT_SYMBOL(napi_alloc_frag);
379
380/**
381 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
382 * @dev: network device to receive on
383 * @len: length to allocate
384 * @gfp_mask: get_free_pages mask, passed to alloc_skb
385 *
386 * Allocate a new &sk_buff and assign it a usage count of one. The
387 * buffer has NET_SKB_PAD headroom built in. Users should allocate
388 * the headroom they think they need without accounting for the
389 * built in space. The built in space is used for optimisations.
390 *
391 * %NULL is returned if there is no free memory.
392 */
393struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
394 gfp_t gfp_mask)
395{
396 struct page_frag_cache *nc;
397 unsigned long flags;
398 struct sk_buff *skb;
399 bool pfmemalloc;
400 void *data;
401
402 len += NET_SKB_PAD;
403
404 if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
405 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
406 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
407 if (!skb)
408 goto skb_fail;
409 goto skb_success;
410 }
411
412 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
413 len = SKB_DATA_ALIGN(len);
414
415 if (sk_memalloc_socks())
416 gfp_mask |= __GFP_MEMALLOC;
417
418 local_irq_save(flags);
419
420 nc = this_cpu_ptr(&netdev_alloc_cache);
421 data = page_frag_alloc(nc, len, gfp_mask);
422 pfmemalloc = nc->pfmemalloc;
423
424 local_irq_restore(flags);
425
426 if (unlikely(!data))
427 return NULL;
428
429 skb = __build_skb(data, len);
430 if (unlikely(!skb)) {
431 skb_free_frag(data);
432 return NULL;
433 }
434
435 /* use OR instead of assignment to avoid clearing of bits in mask */
436 if (pfmemalloc)
437 skb->pfmemalloc = 1;
438 skb->head_frag = 1;
439
440skb_success:
441 skb_reserve(skb, NET_SKB_PAD);
442 skb->dev = dev;
443
444skb_fail:
445 return skb;
446}
447EXPORT_SYMBOL(__netdev_alloc_skb);
448
449/**
450 * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
451 * @napi: napi instance this buffer was allocated for
452 * @len: length to allocate
453 * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
454 *
455 * Allocate a new sk_buff for use in NAPI receive. This buffer will
456 * attempt to allocate the head from a special reserved region used
457 * only for NAPI Rx allocation. By doing this we can save several
458 * CPU cycles by avoiding having to disable and re-enable IRQs.
459 *
460 * %NULL is returned if there is no free memory.
461 */
462struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
463 gfp_t gfp_mask)
464{
465 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
466 struct sk_buff *skb;
467 void *data;
468
469 len += NET_SKB_PAD + NET_IP_ALIGN;
470
471 if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
472 (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
473 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
474 if (!skb)
475 goto skb_fail;
476 goto skb_success;
477 }
478
479 len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
480 len = SKB_DATA_ALIGN(len);
481
482 if (sk_memalloc_socks())
483 gfp_mask |= __GFP_MEMALLOC;
484
485 data = page_frag_alloc(&nc->page, len, gfp_mask);
486 if (unlikely(!data))
487 return NULL;
488
489 skb = __build_skb(data, len);
490 if (unlikely(!skb)) {
491 skb_free_frag(data);
492 return NULL;
493 }
494
495 /* use OR instead of assignment to avoid clearing of bits in mask */
496 if (nc->page.pfmemalloc)
497 skb->pfmemalloc = 1;
498 skb->head_frag = 1;
499
500skb_success:
501 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
502 skb->dev = napi->dev;
503
504skb_fail:
505 return skb;
506}
507EXPORT_SYMBOL(__napi_alloc_skb);
508
509void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
510 int size, unsigned int truesize)
511{
512 skb_fill_page_desc(skb, i, page, off, size);
513 skb->len += size;
514 skb->data_len += size;
515 skb->truesize += truesize;
516}
517EXPORT_SYMBOL(skb_add_rx_frag);
518
519void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
520 unsigned int truesize)
521{
522 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
523
524 skb_frag_size_add(frag, size);
525 skb->len += size;
526 skb->data_len += size;
527 skb->truesize += truesize;
528}
529EXPORT_SYMBOL(skb_coalesce_rx_frag);
530
531static void skb_drop_list(struct sk_buff **listp)
532{
533 kfree_skb_list(*listp);
534 *listp = NULL;
535}
536
537static inline void skb_drop_fraglist(struct sk_buff *skb)
538{
539 skb_drop_list(&skb_shinfo(skb)->frag_list);
540}
541
542static void skb_clone_fraglist(struct sk_buff *skb)
543{
544 struct sk_buff *list;
545
546 skb_walk_frags(skb, list)
547 skb_get(list);
548}
549
550static void skb_free_head(struct sk_buff *skb)
551{
552 unsigned char *head = skb->head;
553
554 if (skb->head_frag)
555 skb_free_frag(head);
556 else
557 kfree(head);
558}
559
560static void skb_release_data(struct sk_buff *skb)
561{
562 struct skb_shared_info *shinfo = skb_shinfo(skb);
563 int i;
564
565 if (skb->cloned &&
566 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
567 &shinfo->dataref))
568 return;
569
570 for (i = 0; i < shinfo->nr_frags; i++)
571 __skb_frag_unref(&shinfo->frags[i]);
572
573 if (shinfo->frag_list)
574 kfree_skb_list(shinfo->frag_list);
575
576 skb_zcopy_clear(skb, true);
577 skb_free_head(skb);
578}
579
580/*
581 * Free an skbuff by memory without cleaning the state.
582 */
583static void kfree_skbmem(struct sk_buff *skb)
584{
585 struct sk_buff_fclones *fclones;
586
587 switch (skb->fclone) {
588 case SKB_FCLONE_UNAVAILABLE:
589 kmem_cache_free(skbuff_head_cache, skb);
590 return;
591
592 case SKB_FCLONE_ORIG:
593 fclones = container_of(skb, struct sk_buff_fclones, skb1);
594
595 /* We usually free the clone (TX completion) before original skb
596 * This test would have no chance to be true for the clone,
597 * while here, branch prediction will be good.
598 */
599 if (refcount_read(&fclones->fclone_ref) == 1)
600 goto fastpath;
601 break;
602
603 default: /* SKB_FCLONE_CLONE */
604 fclones = container_of(skb, struct sk_buff_fclones, skb2);
605 break;
606 }
607 if (!refcount_dec_and_test(&fclones->fclone_ref))
608 return;
609fastpath:
610 kmem_cache_free(skbuff_fclone_cache, fclones);
611}
612
613void skb_release_head_state(struct sk_buff *skb)
614{
615 skb_dst_drop(skb);
616 if (skb->destructor) {
617 WARN_ON(in_irq());
618 skb->destructor(skb);
619 }
620#if IS_ENABLED(CONFIG_NF_CONNTRACK)
621 nf_conntrack_put(skb_nfct(skb));
622#endif
623 skb_ext_put(skb);
624}
625
626/* Free everything but the sk_buff shell. */
627static void skb_release_all(struct sk_buff *skb)
628{
629 skb_release_head_state(skb);
630 if (likely(skb->head))
631 skb_release_data(skb);
632}
633
634/**
635 * __kfree_skb - private function
636 * @skb: buffer
637 *
638 * Free an sk_buff. Release anything attached to the buffer.
639 * Clean the state. This is an internal helper function. Users should
640 * always call kfree_skb
641 */
642
643void __kfree_skb(struct sk_buff *skb)
644{
645 skb_release_all(skb);
646 kfree_skbmem(skb);
647}
648EXPORT_SYMBOL(__kfree_skb);
649
650/**
651 * kfree_skb - free an sk_buff
652 * @skb: buffer to free
653 *
654 * Drop a reference to the buffer and free it if the usage count has
655 * hit zero.
656 */
657void kfree_skb(struct sk_buff *skb)
658{
659 if (!skb_unref(skb))
660 return;
661
662 trace_kfree_skb(skb, __builtin_return_address(0));
663 __kfree_skb(skb);
664}
665EXPORT_SYMBOL(kfree_skb);
666
667void kfree_skb_list(struct sk_buff *segs)
668{
669 while (segs) {
670 struct sk_buff *next = segs->next;
671
672 kfree_skb(segs);
673 segs = next;
674 }
675}
676EXPORT_SYMBOL(kfree_skb_list);
677
678/**
679 * skb_tx_error - report an sk_buff xmit error
680 * @skb: buffer that triggered an error
681 *
682 * Report xmit error if a device callback is tracking this skb.
683 * skb must be freed afterwards.
684 */
685void skb_tx_error(struct sk_buff *skb)
686{
687 skb_zcopy_clear(skb, true);
688}
689EXPORT_SYMBOL(skb_tx_error);
690
691/**
692 * consume_skb - free an skbuff
693 * @skb: buffer to free
694 *
695 * Drop a ref to the buffer and free it if the usage count has hit zero
696 * Functions identically to kfree_skb, but kfree_skb assumes that the frame
697 * is being dropped after a failure and notes that
698 */
699void consume_skb(struct sk_buff *skb)
700{
701 if (!skb_unref(skb))
702 return;
703
704 trace_consume_skb(skb);
705 __kfree_skb(skb);
706}
707EXPORT_SYMBOL(consume_skb);
708
709/**
710 * consume_stateless_skb - free an skbuff, assuming it is stateless
711 * @skb: buffer to free
712 *
713 * Alike consume_skb(), but this variant assumes that this is the last
714 * skb reference and all the head states have been already dropped
715 */
716void __consume_stateless_skb(struct sk_buff *skb)
717{
718 trace_consume_skb(skb);
719 skb_release_data(skb);
720 kfree_skbmem(skb);
721}
722
723void __kfree_skb_flush(void)
724{
725 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
726
727 /* flush skb_cache if containing objects */
728 if (nc->skb_count) {
729 kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
730 nc->skb_cache);
731 nc->skb_count = 0;
732 }
733}
734
735static inline void _kfree_skb_defer(struct sk_buff *skb)
736{
737 struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
738
739 /* drop skb->head and call any destructors for packet */
740 skb_release_all(skb);
741
742 /* record skb to CPU local list */
743 nc->skb_cache[nc->skb_count++] = skb;
744
745#ifdef CONFIG_SLUB
746 /* SLUB writes into objects when freeing */
747 prefetchw(skb);
748#endif
749
750 /* flush skb_cache if it is filled */
751 if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
752 kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE,
753 nc->skb_cache);
754 nc->skb_count = 0;
755 }
756}
757void __kfree_skb_defer(struct sk_buff *skb)
758{
759 _kfree_skb_defer(skb);
760}
761
762void napi_consume_skb(struct sk_buff *skb, int budget)
763{
764 if (unlikely(!skb))
765 return;
766
767 /* Zero budget indicate non-NAPI context called us, like netpoll */
768 if (unlikely(!budget)) {
769 dev_consume_skb_any(skb);
770 return;
771 }
772
773 if (!skb_unref(skb))
774 return;
775
776 /* if reaching here SKB is ready to free */
777 trace_consume_skb(skb);
778
779 /* if SKB is a clone, don't handle this case */
780 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
781 __kfree_skb(skb);
782 return;
783 }
784
785 _kfree_skb_defer(skb);
786}
787EXPORT_SYMBOL(napi_consume_skb);
788
789/* Make sure a field is enclosed inside headers_start/headers_end section */
790#define CHECK_SKB_FIELD(field) \
791 BUILD_BUG_ON(offsetof(struct sk_buff, field) < \
792 offsetof(struct sk_buff, headers_start)); \
793 BUILD_BUG_ON(offsetof(struct sk_buff, field) > \
794 offsetof(struct sk_buff, headers_end)); \
795
796static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
797{
798 new->tstamp = old->tstamp;
799 /* We do not copy old->sk */
800 new->dev = old->dev;
801 memcpy(new->cb, old->cb, sizeof(old->cb));
802 skb_dst_copy(new, old);
803 __skb_ext_copy(new, old);
804 __nf_copy(new, old, false);
805
806 /* Note : this field could be in headers_start/headers_end section
807 * It is not yet because we do not want to have a 16 bit hole
808 */
809 new->queue_mapping = old->queue_mapping;
810
811 memcpy(&new->headers_start, &old->headers_start,
812 offsetof(struct sk_buff, headers_end) -
813 offsetof(struct sk_buff, headers_start));
814 CHECK_SKB_FIELD(protocol);
815 CHECK_SKB_FIELD(csum);
816 CHECK_SKB_FIELD(hash);
817 CHECK_SKB_FIELD(priority);
818 CHECK_SKB_FIELD(skb_iif);
819 CHECK_SKB_FIELD(vlan_proto);
820 CHECK_SKB_FIELD(vlan_tci);
821 CHECK_SKB_FIELD(transport_header);
822 CHECK_SKB_FIELD(network_header);
823 CHECK_SKB_FIELD(mac_header);
824 CHECK_SKB_FIELD(inner_protocol);
825 CHECK_SKB_FIELD(inner_transport_header);
826 CHECK_SKB_FIELD(inner_network_header);
827 CHECK_SKB_FIELD(inner_mac_header);
828 CHECK_SKB_FIELD(mark);
829#ifdef CONFIG_NETWORK_SECMARK
830 CHECK_SKB_FIELD(secmark);
831#endif
832#ifdef CONFIG_NET_RX_BUSY_POLL
833 CHECK_SKB_FIELD(napi_id);
834#endif
835#ifdef CONFIG_XPS
836 CHECK_SKB_FIELD(sender_cpu);
837#endif
838#ifdef CONFIG_NET_SCHED
839 CHECK_SKB_FIELD(tc_index);
840#endif
841
842}
843
844/*
845 * You should not add any new code to this function. Add it to
846 * __copy_skb_header above instead.
847 */
848static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
849{
850#define C(x) n->x = skb->x
851
852 n->next = n->prev = NULL;
853 n->sk = NULL;
854 __copy_skb_header(n, skb);
855
856 C(len);
857 C(data_len);
858 C(mac_len);
859 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
860 n->cloned = 1;
861 n->nohdr = 0;
862 n->peeked = 0;
863 C(pfmemalloc);
864 n->destructor = NULL;
865 C(tail);
866 C(end);
867 C(head);
868 C(head_frag);
869 C(data);
870 C(truesize);
871 refcount_set(&n->users, 1);
872
873 atomic_inc(&(skb_shinfo(skb)->dataref));
874 skb->cloned = 1;
875
876 return n;
877#undef C
878}
879
880/**
881 * skb_morph - morph one skb into another
882 * @dst: the skb to receive the contents
883 * @src: the skb to supply the contents
884 *
885 * This is identical to skb_clone except that the target skb is
886 * supplied by the user.
887 *
888 * The target skb is returned upon exit.
889 */
890struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
891{
892 skb_release_all(dst);
893 return __skb_clone(dst, src);
894}
895EXPORT_SYMBOL_GPL(skb_morph);
896
897int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
898{
899 unsigned long max_pg, num_pg, new_pg, old_pg;
900 struct user_struct *user;
901
902 if (capable(CAP_IPC_LOCK) || !size)
903 return 0;
904
905 num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */
906 max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
907 user = mmp->user ? : current_user();
908
909 do {
910 old_pg = atomic_long_read(&user->locked_vm);
911 new_pg = old_pg + num_pg;
912 if (new_pg > max_pg)
913 return -ENOBUFS;
914 } while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) !=
915 old_pg);
916
917 if (!mmp->user) {
918 mmp->user = get_uid(user);
919 mmp->num_pg = num_pg;
920 } else {
921 mmp->num_pg += num_pg;
922 }
923
924 return 0;
925}
926EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
927
928void mm_unaccount_pinned_pages(struct mmpin *mmp)
929{
930 if (mmp->user) {
931 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
932 free_uid(mmp->user);
933 }
934}
935EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
936
937struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size)
938{
939 struct ubuf_info *uarg;
940 struct sk_buff *skb;
941
942 WARN_ON_ONCE(!in_task());
943
944 skb = sock_omalloc(sk, 0, GFP_KERNEL);
945 if (!skb)
946 return NULL;
947
948 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
949 uarg = (void *)skb->cb;
950 uarg->mmp.user = NULL;
951
952 if (mm_account_pinned_pages(&uarg->mmp, size)) {
953 kfree_skb(skb);
954 return NULL;
955 }
956
957 uarg->callback = sock_zerocopy_callback;
958 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
959 uarg->len = 1;
960 uarg->bytelen = size;
961 uarg->zerocopy = 1;
962 refcount_set(&uarg->refcnt, 1);
963 sock_hold(sk);
964
965 return uarg;
966}
967EXPORT_SYMBOL_GPL(sock_zerocopy_alloc);
968
969static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)
970{
971 return container_of((void *)uarg, struct sk_buff, cb);
972}
973
974struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size,
975 struct ubuf_info *uarg)
976{
977 if (uarg) {
978 const u32 byte_limit = 1 << 19; /* limit to a few TSO */
979 u32 bytelen, next;
980
981 /* realloc only when socket is locked (TCP, UDP cork),
982 * so uarg->len and sk_zckey access is serialized
983 */
984 if (!sock_owned_by_user(sk)) {
985 WARN_ON_ONCE(1);
986 return NULL;
987 }
988
989 bytelen = uarg->bytelen + size;
990 if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {
991 /* TCP can create new skb to attach new uarg */
992 if (sk->sk_type == SOCK_STREAM)
993 goto new_alloc;
994 return NULL;
995 }
996
997 next = (u32)atomic_read(&sk->sk_zckey);
998 if ((u32)(uarg->id + uarg->len) == next) {
999 if (mm_account_pinned_pages(&uarg->mmp, size))
1000 return NULL;
1001 uarg->len++;
1002 uarg->bytelen = bytelen;
1003 atomic_set(&sk->sk_zckey, ++next);
1004 sock_zerocopy_get(uarg);
1005 return uarg;
1006 }
1007 }
1008
1009new_alloc:
1010 return sock_zerocopy_alloc(sk, size);
1011}
1012EXPORT_SYMBOL_GPL(sock_zerocopy_realloc);
1013
1014static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
1015{
1016 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
1017 u32 old_lo, old_hi;
1018 u64 sum_len;
1019
1020 old_lo = serr->ee.ee_info;
1021 old_hi = serr->ee.ee_data;
1022 sum_len = old_hi - old_lo + 1ULL + len;
1023
1024 if (sum_len >= (1ULL << 32))
1025 return false;
1026
1027 if (lo != old_hi + 1)
1028 return false;
1029
1030 serr->ee.ee_data += len;
1031 return true;
1032}
1033
1034void sock_zerocopy_callback(struct ubuf_info *uarg, bool success)
1035{
1036 struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1037 struct sock_exterr_skb *serr;
1038 struct sock *sk = skb->sk;
1039 struct sk_buff_head *q;
1040 unsigned long flags;
1041 u32 lo, hi;
1042 u16 len;
1043
1044 mm_unaccount_pinned_pages(&uarg->mmp);
1045
1046 /* if !len, there was only 1 call, and it was aborted
1047 * so do not queue a completion notification
1048 */
1049 if (!uarg->len || sock_flag(sk, SOCK_DEAD))
1050 goto release;
1051
1052 len = uarg->len;
1053 lo = uarg->id;
1054 hi = uarg->id + len - 1;
1055
1056 serr = SKB_EXT_ERR(skb);
1057 memset(serr, 0, sizeof(*serr));
1058 serr->ee.ee_errno = 0;
1059 serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
1060 serr->ee.ee_data = hi;
1061 serr->ee.ee_info = lo;
1062 if (!success)
1063 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
1064
1065 q = &sk->sk_error_queue;
1066 spin_lock_irqsave(&q->lock, flags);
1067 tail = skb_peek_tail(q);
1068 if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
1069 !skb_zerocopy_notify_extend(tail, lo, len)) {
1070 __skb_queue_tail(q, skb);
1071 skb = NULL;
1072 }
1073 spin_unlock_irqrestore(&q->lock, flags);
1074
1075 sk->sk_error_report(sk);
1076
1077release:
1078 consume_skb(skb);
1079 sock_put(sk);
1080}
1081EXPORT_SYMBOL_GPL(sock_zerocopy_callback);
1082
1083void sock_zerocopy_put(struct ubuf_info *uarg)
1084{
1085 if (uarg && refcount_dec_and_test(&uarg->refcnt)) {
1086 if (uarg->callback)
1087 uarg->callback(uarg, uarg->zerocopy);
1088 else
1089 consume_skb(skb_from_uarg(uarg));
1090 }
1091}
1092EXPORT_SYMBOL_GPL(sock_zerocopy_put);
1093
1094void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1095{
1096 if (uarg) {
1097 struct sock *sk = skb_from_uarg(uarg)->sk;
1098
1099 atomic_dec(&sk->sk_zckey);
1100 uarg->len--;
1101
1102 if (have_uref)
1103 sock_zerocopy_put(uarg);
1104 }
1105}
1106EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort);
1107
1108extern int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb,
1109 struct iov_iter *from, size_t length);
1110
1111int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len)
1112{
1113 return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len);
1114}
1115EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram);
1116
1117int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1118 struct msghdr *msg, int len,
1119 struct ubuf_info *uarg)
1120{
1121 struct ubuf_info *orig_uarg = skb_zcopy(skb);
1122 struct iov_iter orig_iter = msg->msg_iter;
1123 int err, orig_len = skb->len;
1124
1125 /* An skb can only point to one uarg. This edge case happens when
1126 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
1127 */
1128 if (orig_uarg && uarg != orig_uarg)
1129 return -EEXIST;
1130
1131 err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);
1132 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
1133 struct sock *save_sk = skb->sk;
1134
1135 /* Streams do not free skb on error. Reset to prev state. */
1136 msg->msg_iter = orig_iter;
1137 skb->sk = sk;
1138 ___pskb_trim(skb, orig_len);
1139 skb->sk = save_sk;
1140 return err;
1141 }
1142
1143 skb_zcopy_set(skb, uarg, NULL);
1144 return skb->len - orig_len;
1145}
1146EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
1147
1148static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
1149 gfp_t gfp_mask)
1150{
1151 if (skb_zcopy(orig)) {
1152 if (skb_zcopy(nskb)) {
1153 /* !gfp_mask callers are verified to !skb_zcopy(nskb) */
1154 if (!gfp_mask) {
1155 WARN_ON_ONCE(1);
1156 return -ENOMEM;
1157 }
1158 if (skb_uarg(nskb) == skb_uarg(orig))
1159 return 0;
1160 if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1161 return -EIO;
1162 }
1163 skb_zcopy_set(nskb, skb_uarg(orig), NULL);
1164 }
1165 return 0;
1166}
1167
1168/**
1169 * skb_copy_ubufs - copy userspace skb frags buffers to kernel
1170 * @skb: the skb to modify
1171 * @gfp_mask: allocation priority
1172 *
1173 * This must be called on SKBTX_DEV_ZEROCOPY skb.
1174 * It will copy all frags into kernel and drop the reference
1175 * to userspace pages.
1176 *
1177 * If this function is called from an interrupt gfp_mask() must be
1178 * %GFP_ATOMIC.
1179 *
1180 * Returns 0 on success or a negative error code on failure
1181 * to allocate kernel memory to copy to.
1182 */
1183int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
1184{
1185 int num_frags = skb_shinfo(skb)->nr_frags;
1186 struct page *page, *head = NULL;
1187 int i, new_frags;
1188 u32 d_off;
1189
1190 if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
1191 return -EINVAL;
1192
1193 if (!num_frags)
1194 goto release;
1195
1196 new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT;
1197 for (i = 0; i < new_frags; i++) {
1198 page = alloc_page(gfp_mask);
1199 if (!page) {
1200 while (head) {
1201 struct page *next = (struct page *)page_private(head);
1202 put_page(head);
1203 head = next;
1204 }
1205 return -ENOMEM;
1206 }
1207 set_page_private(page, (unsigned long)head);
1208 head = page;
1209 }
1210
1211 page = head;
1212 d_off = 0;
1213 for (i = 0; i < num_frags; i++) {
1214 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1215 u32 p_off, p_len, copied;
1216 struct page *p;
1217 u8 *vaddr;
1218
1219 skb_frag_foreach_page(f, f->page_offset, skb_frag_size(f),
1220 p, p_off, p_len, copied) {
1221 u32 copy, done = 0;
1222 vaddr = kmap_atomic(p);
1223
1224 while (done < p_len) {
1225 if (d_off == PAGE_SIZE) {
1226 d_off = 0;
1227 page = (struct page *)page_private(page);
1228 }
1229 copy = min_t(u32, PAGE_SIZE - d_off, p_len - done);
1230 memcpy(page_address(page) + d_off,
1231 vaddr + p_off + done, copy);
1232 done += copy;
1233 d_off += copy;
1234 }
1235 kunmap_atomic(vaddr);
1236 }
1237 }
1238
1239 /* skb frags release userspace buffers */
1240 for (i = 0; i < num_frags; i++)
1241 skb_frag_unref(skb, i);
1242
1243 /* skb frags point to kernel buffers */
1244 for (i = 0; i < new_frags - 1; i++) {
1245 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE);
1246 head = (struct page *)page_private(head);
1247 }
1248 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
1249 skb_shinfo(skb)->nr_frags = new_frags;
1250
1251release:
1252 skb_zcopy_clear(skb, false);
1253 return 0;
1254}
1255EXPORT_SYMBOL_GPL(skb_copy_ubufs);
1256
1257/**
1258 * skb_clone - duplicate an sk_buff
1259 * @skb: buffer to clone
1260 * @gfp_mask: allocation priority
1261 *
1262 * Duplicate an &sk_buff. The new one is not owned by a socket. Both
1263 * copies share the same packet data but not structure. The new
1264 * buffer has a reference count of 1. If the allocation fails the
1265 * function returns %NULL otherwise the new buffer is returned.
1266 *
1267 * If this function is called from an interrupt gfp_mask() must be
1268 * %GFP_ATOMIC.
1269 */
1270
1271struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
1272{
1273 struct sk_buff_fclones *fclones = container_of(skb,
1274 struct sk_buff_fclones,
1275 skb1);
1276 struct sk_buff *n;
1277
1278 if (skb_orphan_frags(skb, gfp_mask))
1279 return NULL;
1280
1281 if (skb->fclone == SKB_FCLONE_ORIG &&
1282 refcount_read(&fclones->fclone_ref) == 1) {
1283 n = &fclones->skb2;
1284 refcount_set(&fclones->fclone_ref, 2);
1285 } else {
1286 if (skb_pfmemalloc(skb))
1287 gfp_mask |= __GFP_MEMALLOC;
1288
1289 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
1290 if (!n)
1291 return NULL;
1292
1293 n->fclone = SKB_FCLONE_UNAVAILABLE;
1294 }
1295
1296 return __skb_clone(n, skb);
1297}
1298EXPORT_SYMBOL(skb_clone);
1299
1300void skb_headers_offset_update(struct sk_buff *skb, int off)
1301{
1302 /* Only adjust this if it actually is csum_start rather than csum */
1303 if (skb->ip_summed == CHECKSUM_PARTIAL)
1304 skb->csum_start += off;
1305 /* {transport,network,mac}_header and tail are relative to skb->head */
1306 skb->transport_header += off;
1307 skb->network_header += off;
1308 if (skb_mac_header_was_set(skb))
1309 skb->mac_header += off;
1310 skb->inner_transport_header += off;
1311 skb->inner_network_header += off;
1312 skb->inner_mac_header += off;
1313}
1314EXPORT_SYMBOL(skb_headers_offset_update);
1315
1316void skb_copy_header(struct sk_buff *new, const struct sk_buff *old)
1317{
1318 __copy_skb_header(new, old);
1319
1320 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
1321 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
1322 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
1323}
1324EXPORT_SYMBOL(skb_copy_header);
1325
1326static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
1327{
1328 if (skb_pfmemalloc(skb))
1329 return SKB_ALLOC_RX;
1330 return 0;
1331}
1332
1333/**
1334 * skb_copy - create private copy of an sk_buff
1335 * @skb: buffer to copy
1336 * @gfp_mask: allocation priority
1337 *
1338 * Make a copy of both an &sk_buff and its data. This is used when the
1339 * caller wishes to modify the data and needs a private copy of the
1340 * data to alter. Returns %NULL on failure or the pointer to the buffer
1341 * on success. The returned buffer has a reference count of 1.
1342 *
1343 * As by-product this function converts non-linear &sk_buff to linear
1344 * one, so that &sk_buff becomes completely private and caller is allowed
1345 * to modify all the data of returned buffer. This means that this
1346 * function is not recommended for use in circumstances when only
1347 * header is going to be modified. Use pskb_copy() instead.
1348 */
1349
1350struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
1351{
1352 int headerlen = skb_headroom(skb);
1353 unsigned int size = skb_end_offset(skb) + skb->data_len;
1354 struct sk_buff *n = __alloc_skb(size, gfp_mask,
1355 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
1356
1357 if (!n)
1358 return NULL;
1359
1360 /* Set the data pointer */
1361 skb_reserve(n, headerlen);
1362 /* Set the tail pointer and length */
1363 skb_put(n, skb->len);
1364
1365 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
1366
1367 skb_copy_header(n, skb);
1368 return n;
1369}
1370EXPORT_SYMBOL(skb_copy);
1371
1372/**
1373 * __pskb_copy_fclone - create copy of an sk_buff with private head.
1374 * @skb: buffer to copy
1375 * @headroom: headroom of new skb
1376 * @gfp_mask: allocation priority
1377 * @fclone: if true allocate the copy of the skb from the fclone
1378 * cache instead of the head cache; it is recommended to set this
1379 * to true for the cases where the copy will likely be cloned
1380 *
1381 * Make a copy of both an &sk_buff and part of its data, located
1382 * in header. Fragmented data remain shared. This is used when
1383 * the caller wishes to modify only header of &sk_buff and needs
1384 * private copy of the header to alter. Returns %NULL on failure
1385 * or the pointer to the buffer on success.
1386 * The returned buffer has a reference count of 1.
1387 */
1388
1389struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1390 gfp_t gfp_mask, bool fclone)
1391{
1392 unsigned int size = skb_headlen(skb) + headroom;
1393 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
1394 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
1395
1396 if (!n)
1397 goto out;
1398
1399 /* Set the data pointer */
1400 skb_reserve(n, headroom);
1401 /* Set the tail pointer and length */
1402 skb_put(n, skb_headlen(skb));
1403 /* Copy the bytes */
1404 skb_copy_from_linear_data(skb, n->data, n->len);
1405
1406 n->truesize += skb->data_len;
1407 n->data_len = skb->data_len;
1408 n->len = skb->len;
1409
1410 if (skb_shinfo(skb)->nr_frags) {
1411 int i;
1412
1413 if (skb_orphan_frags(skb, gfp_mask) ||
1414 skb_zerocopy_clone(n, skb, gfp_mask)) {
1415 kfree_skb(n);
1416 n = NULL;
1417 goto out;
1418 }
1419 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1420 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
1421 skb_frag_ref(skb, i);
1422 }
1423 skb_shinfo(n)->nr_frags = i;
1424 }
1425
1426 if (skb_has_frag_list(skb)) {
1427 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1428 skb_clone_fraglist(n);
1429 }
1430
1431 skb_copy_header(n, skb);
1432out:
1433 return n;
1434}
1435EXPORT_SYMBOL(__pskb_copy_fclone);
1436
1437/**
1438 * pskb_expand_head - reallocate header of &sk_buff
1439 * @skb: buffer to reallocate
1440 * @nhead: room to add at head
1441 * @ntail: room to add at tail
1442 * @gfp_mask: allocation priority
1443 *
1444 * Expands (or creates identical copy, if @nhead and @ntail are zero)
1445 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
1446 * reference count of 1. Returns zero in the case of success or error,
1447 * if expansion failed. In the last case, &sk_buff is not changed.
1448 *
1449 * All the pointers pointing into skb header may change and must be
1450 * reloaded after call to this function.
1451 */
1452
1453int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
1454 gfp_t gfp_mask)
1455{
1456 int i, osize = skb_end_offset(skb);
1457 int size = osize + nhead + ntail;
1458 long off;
1459 u8 *data;
1460
1461 BUG_ON(nhead < 0);
1462
1463 BUG_ON(skb_shared(skb));
1464
1465 size = SKB_DATA_ALIGN(size);
1466
1467 if (skb_pfmemalloc(skb))
1468 gfp_mask |= __GFP_MEMALLOC;
1469 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
1470 gfp_mask, NUMA_NO_NODE, NULL);
1471 if (!data)
1472 goto nodata;
1473 size = SKB_WITH_OVERHEAD(ksize(data));
1474
1475 /* Copy only real data... and, alas, header. This should be
1476 * optimized for the cases when header is void.
1477 */
1478 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1479
1480 memcpy((struct skb_shared_info *)(data + size),
1481 skb_shinfo(skb),
1482 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
1483
1484 /*
1485 * if shinfo is shared we must drop the old head gracefully, but if it
1486 * is not we can just drop the old head and let the existing refcount
1487 * be since all we did is relocate the values
1488 */
1489 if (skb_cloned(skb)) {
1490 if (skb_orphan_frags(skb, gfp_mask))
1491 goto nofrags;
1492 if (skb_zcopy(skb))
1493 refcount_inc(&skb_uarg(skb)->refcnt);
1494 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1495 skb_frag_ref(skb, i);
1496
1497 if (skb_has_frag_list(skb))
1498 skb_clone_fraglist(skb);
1499
1500 skb_release_data(skb);
1501 } else {
1502 skb_free_head(skb);
1503 }
1504 off = (data + nhead) - skb->head;
1505
1506 skb->head = data;
1507 skb->head_frag = 0;
1508 skb->data += off;
1509#ifdef NET_SKBUFF_DATA_USES_OFFSET
1510 skb->end = size;
1511 off = nhead;
1512#else
1513 skb->end = skb->head + size;
1514#endif
1515 skb->tail += off;
1516 skb_headers_offset_update(skb, nhead);
1517 skb->cloned = 0;
1518 skb->hdr_len = 0;
1519 skb->nohdr = 0;
1520 atomic_set(&skb_shinfo(skb)->dataref, 1);
1521
1522 skb_metadata_clear(skb);
1523
1524 /* It is not generally safe to change skb->truesize.
1525 * For the moment, we really care of rx path, or
1526 * when skb is orphaned (not attached to a socket).
1527 */
1528 if (!skb->sk || skb->destructor == sock_edemux)
1529 skb->truesize += size - osize;
1530
1531 return 0;
1532
1533nofrags:
1534 kfree(data);
1535nodata:
1536 return -ENOMEM;
1537}
1538EXPORT_SYMBOL(pskb_expand_head);
1539
1540/* Make private copy of skb with writable head and some headroom */
1541
1542struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1543{
1544 struct sk_buff *skb2;
1545 int delta = headroom - skb_headroom(skb);
1546
1547 if (delta <= 0)
1548 skb2 = pskb_copy(skb, GFP_ATOMIC);
1549 else {
1550 skb2 = skb_clone(skb, GFP_ATOMIC);
1551 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
1552 GFP_ATOMIC)) {
1553 kfree_skb(skb2);
1554 skb2 = NULL;
1555 }
1556 }
1557 return skb2;
1558}
1559EXPORT_SYMBOL(skb_realloc_headroom);
1560
1561/**
1562 * skb_copy_expand - copy and expand sk_buff
1563 * @skb: buffer to copy
1564 * @newheadroom: new free bytes at head
1565 * @newtailroom: new free bytes at tail
1566 * @gfp_mask: allocation priority
1567 *
1568 * Make a copy of both an &sk_buff and its data and while doing so
1569 * allocate additional space.
1570 *
1571 * This is used when the caller wishes to modify the data and needs a
1572 * private copy of the data to alter as well as more space for new fields.
1573 * Returns %NULL on failure or the pointer to the buffer
1574 * on success. The returned buffer has a reference count of 1.
1575 *
1576 * You must pass %GFP_ATOMIC as the allocation priority if this function
1577 * is called from an interrupt.
1578 */
1579struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
1580 int newheadroom, int newtailroom,
1581 gfp_t gfp_mask)
1582{
1583 /*
1584 * Allocate the copy buffer
1585 */
1586 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
1587 gfp_mask, skb_alloc_rx_flag(skb),
1588 NUMA_NO_NODE);
1589 int oldheadroom = skb_headroom(skb);
1590 int head_copy_len, head_copy_off;
1591
1592 if (!n)
1593 return NULL;
1594
1595 skb_reserve(n, newheadroom);
1596
1597 /* Set the tail pointer and length */
1598 skb_put(n, skb->len);
1599
1600 head_copy_len = oldheadroom;
1601 head_copy_off = 0;
1602 if (newheadroom <= head_copy_len)
1603 head_copy_len = newheadroom;
1604 else
1605 head_copy_off = newheadroom - head_copy_len;
1606
1607 /* Copy the linear header and data. */
1608 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1609 skb->len + head_copy_len));
1610
1611 skb_copy_header(n, skb);
1612
1613 skb_headers_offset_update(n, newheadroom - oldheadroom);
1614
1615 return n;
1616}
1617EXPORT_SYMBOL(skb_copy_expand);
1618
1619/**
1620 * __skb_pad - zero pad the tail of an skb
1621 * @skb: buffer to pad
1622 * @pad: space to pad
1623 * @free_on_error: free buffer on error
1624 *
1625 * Ensure that a buffer is followed by a padding area that is zero
1626 * filled. Used by network drivers which may DMA or transfer data
1627 * beyond the buffer end onto the wire.
1628 *
1629 * May return error in out of memory cases. The skb is freed on error
1630 * if @free_on_error is true.
1631 */
1632
1633int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
1634{
1635 int err;
1636 int ntail;
1637
1638 /* If the skbuff is non linear tailroom is always zero.. */
1639 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
1640 memset(skb->data+skb->len, 0, pad);
1641 return 0;
1642 }
1643
1644 ntail = skb->data_len + pad - (skb->end - skb->tail);
1645 if (likely(skb_cloned(skb) || ntail > 0)) {
1646 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1647 if (unlikely(err))
1648 goto free_skb;
1649 }
1650
1651 /* FIXME: The use of this function with non-linear skb's really needs
1652 * to be audited.
1653 */
1654 err = skb_linearize(skb);
1655 if (unlikely(err))
1656 goto free_skb;
1657
1658 memset(skb->data + skb->len, 0, pad);
1659 return 0;
1660
1661free_skb:
1662 if (free_on_error)
1663 kfree_skb(skb);
1664 return err;
1665}
1666EXPORT_SYMBOL(__skb_pad);
1667
1668/**
1669 * pskb_put - add data to the tail of a potentially fragmented buffer
1670 * @skb: start of the buffer to use
1671 * @tail: tail fragment of the buffer to use
1672 * @len: amount of data to add
1673 *
1674 * This function extends the used data area of the potentially
1675 * fragmented buffer. @tail must be the last fragment of @skb -- or
1676 * @skb itself. If this would exceed the total buffer size the kernel
1677 * will panic. A pointer to the first byte of the extra data is
1678 * returned.
1679 */
1680
1681void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
1682{
1683 if (tail != skb) {
1684 skb->data_len += len;
1685 skb->len += len;
1686 }
1687 return skb_put(tail, len);
1688}
1689EXPORT_SYMBOL_GPL(pskb_put);
1690
1691/**
1692 * skb_put - add data to a buffer
1693 * @skb: buffer to use
1694 * @len: amount of data to add
1695 *
1696 * This function extends the used data area of the buffer. If this would
1697 * exceed the total buffer size the kernel will panic. A pointer to the
1698 * first byte of the extra data is returned.
1699 */
1700void *skb_put(struct sk_buff *skb, unsigned int len)
1701{
1702 void *tmp = skb_tail_pointer(skb);
1703 SKB_LINEAR_ASSERT(skb);
1704 skb->tail += len;
1705 skb->len += len;
1706 if (unlikely(skb->tail > skb->end))
1707 skb_over_panic(skb, len, __builtin_return_address(0));
1708 return tmp;
1709}
1710EXPORT_SYMBOL(skb_put);
1711
1712/**
1713 * skb_push - add data to the start of a buffer
1714 * @skb: buffer to use
1715 * @len: amount of data to add
1716 *
1717 * This function extends the used data area of the buffer at the buffer
1718 * start. If this would exceed the total buffer headroom the kernel will
1719 * panic. A pointer to the first byte of the extra data is returned.
1720 */
1721void *skb_push(struct sk_buff *skb, unsigned int len)
1722{
1723 skb->data -= len;
1724 skb->len += len;
1725 if (unlikely(skb->data < skb->head))
1726 skb_under_panic(skb, len, __builtin_return_address(0));
1727 return skb->data;
1728}
1729EXPORT_SYMBOL(skb_push);
1730
1731/**
1732 * skb_pull - remove data from the start of a buffer
1733 * @skb: buffer to use
1734 * @len: amount of data to remove
1735 *
1736 * This function removes data from the start of a buffer, returning
1737 * the memory to the headroom. A pointer to the next data in the buffer
1738 * is returned. Once the data has been pulled future pushes will overwrite
1739 * the old data.
1740 */
1741void *skb_pull(struct sk_buff *skb, unsigned int len)
1742{
1743 return skb_pull_inline(skb, len);
1744}
1745EXPORT_SYMBOL(skb_pull);
1746
1747/**
1748 * skb_trim - remove end from a buffer
1749 * @skb: buffer to alter
1750 * @len: new length
1751 *
1752 * Cut the length of a buffer down by removing data from the tail. If
1753 * the buffer is already under the length specified it is not modified.
1754 * The skb must be linear.
1755 */
1756void skb_trim(struct sk_buff *skb, unsigned int len)
1757{
1758 if (skb->len > len)
1759 __skb_trim(skb, len);
1760}
1761EXPORT_SYMBOL(skb_trim);
1762
1763/* Trims skb to length len. It can change skb pointers.
1764 */
1765
1766int ___pskb_trim(struct sk_buff *skb, unsigned int len)
1767{
1768 struct sk_buff **fragp;
1769 struct sk_buff *frag;
1770 int offset = skb_headlen(skb);
1771 int nfrags = skb_shinfo(skb)->nr_frags;
1772 int i;
1773 int err;
1774
1775 if (skb_cloned(skb) &&
1776 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1777 return err;
1778
1779 i = 0;
1780 if (offset >= len)
1781 goto drop_pages;
1782
1783 for (; i < nfrags; i++) {
1784 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
1785
1786 if (end < len) {
1787 offset = end;
1788 continue;
1789 }
1790
1791 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
1792
1793drop_pages:
1794 skb_shinfo(skb)->nr_frags = i;
1795
1796 for (; i < nfrags; i++)
1797 skb_frag_unref(skb, i);
1798
1799 if (skb_has_frag_list(skb))
1800 skb_drop_fraglist(skb);
1801 goto done;
1802 }
1803
1804 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1805 fragp = &frag->next) {
1806 int end = offset + frag->len;
1807
1808 if (skb_shared(frag)) {
1809 struct sk_buff *nfrag;
1810
1811 nfrag = skb_clone(frag, GFP_ATOMIC);
1812 if (unlikely(!nfrag))
1813 return -ENOMEM;
1814
1815 nfrag->next = frag->next;
1816 consume_skb(frag);
1817 frag = nfrag;
1818 *fragp = frag;
1819 }
1820
1821 if (end < len) {
1822 offset = end;
1823 continue;
1824 }
1825
1826 if (end > len &&
1827 unlikely((err = pskb_trim(frag, len - offset))))
1828 return err;
1829
1830 if (frag->next)
1831 skb_drop_list(&frag->next);
1832 break;
1833 }
1834
1835done:
1836 if (len > skb_headlen(skb)) {
1837 skb->data_len -= skb->len - len;
1838 skb->len = len;
1839 } else {
1840 skb->len = len;
1841 skb->data_len = 0;
1842 skb_set_tail_pointer(skb, len);
1843 }
1844
1845 if (!skb->sk || skb->destructor == sock_edemux)
1846 skb_condense(skb);
1847 return 0;
1848}
1849EXPORT_SYMBOL(___pskb_trim);
1850
1851/* Note : use pskb_trim_rcsum() instead of calling this directly
1852 */
1853int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
1854{
1855 if (skb->ip_summed == CHECKSUM_COMPLETE) {
1856 int delta = skb->len - len;
1857
1858 skb->csum = csum_block_sub(skb->csum,
1859 skb_checksum(skb, len, delta, 0),
1860 len);
1861 }
1862 return __pskb_trim(skb, len);
1863}
1864EXPORT_SYMBOL(pskb_trim_rcsum_slow);
1865
1866/**
1867 * __pskb_pull_tail - advance tail of skb header
1868 * @skb: buffer to reallocate
1869 * @delta: number of bytes to advance tail
1870 *
1871 * The function makes a sense only on a fragmented &sk_buff,
1872 * it expands header moving its tail forward and copying necessary
1873 * data from fragmented part.
1874 *
1875 * &sk_buff MUST have reference count of 1.
1876 *
1877 * Returns %NULL (and &sk_buff does not change) if pull failed
1878 * or value of new tail of skb in the case of success.
1879 *
1880 * All the pointers pointing into skb header may change and must be
1881 * reloaded after call to this function.
1882 */
1883
1884/* Moves tail of skb head forward, copying data from fragmented part,
1885 * when it is necessary.
1886 * 1. It may fail due to malloc failure.
1887 * 2. It may change skb pointers.
1888 *
1889 * It is pretty complicated. Luckily, it is called only in exceptional cases.
1890 */
1891void *__pskb_pull_tail(struct sk_buff *skb, int delta)
1892{
1893 /* If skb has not enough free space at tail, get new one
1894 * plus 128 bytes for future expansions. If we have enough
1895 * room at tail, reallocate without expansion only if skb is cloned.
1896 */
1897 int i, k, eat = (skb->tail + delta) - skb->end;
1898
1899 if (eat > 0 || skb_cloned(skb)) {
1900 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
1901 GFP_ATOMIC))
1902 return NULL;
1903 }
1904
1905 BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
1906 skb_tail_pointer(skb), delta));
1907
1908 /* Optimization: no fragments, no reasons to preestimate
1909 * size of pulled pages. Superb.
1910 */
1911 if (!skb_has_frag_list(skb))
1912 goto pull_pages;
1913
1914 /* Estimate size of pulled pages. */
1915 eat = delta;
1916 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1917 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1918
1919 if (size >= eat)
1920 goto pull_pages;
1921 eat -= size;
1922 }
1923
1924 /* If we need update frag list, we are in troubles.
1925 * Certainly, it is possible to add an offset to skb data,
1926 * but taking into account that pulling is expected to
1927 * be very rare operation, it is worth to fight against
1928 * further bloating skb head and crucify ourselves here instead.
1929 * Pure masohism, indeed. 8)8)
1930 */
1931 if (eat) {
1932 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1933 struct sk_buff *clone = NULL;
1934 struct sk_buff *insp = NULL;
1935
1936 do {
1937 if (list->len <= eat) {
1938 /* Eaten as whole. */
1939 eat -= list->len;
1940 list = list->next;
1941 insp = list;
1942 } else {
1943 /* Eaten partially. */
1944
1945 if (skb_shared(list)) {
1946 /* Sucks! We need to fork list. :-( */
1947 clone = skb_clone(list, GFP_ATOMIC);
1948 if (!clone)
1949 return NULL;
1950 insp = list->next;
1951 list = clone;
1952 } else {
1953 /* This may be pulled without
1954 * problems. */
1955 insp = list;
1956 }
1957 if (!pskb_pull(list, eat)) {
1958 kfree_skb(clone);
1959 return NULL;
1960 }
1961 break;
1962 }
1963 } while (eat);
1964
1965 /* Free pulled out fragments. */
1966 while ((list = skb_shinfo(skb)->frag_list) != insp) {
1967 skb_shinfo(skb)->frag_list = list->next;
1968 kfree_skb(list);
1969 }
1970 /* And insert new clone at head. */
1971 if (clone) {
1972 clone->next = list;
1973 skb_shinfo(skb)->frag_list = clone;
1974 }
1975 }
1976 /* Success! Now we may commit changes to skb data. */
1977
1978pull_pages:
1979 eat = delta;
1980 k = 0;
1981 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1982 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1983
1984 if (size <= eat) {
1985 skb_frag_unref(skb, i);
1986 eat -= size;
1987 } else {
1988 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1989 if (eat) {
1990 skb_shinfo(skb)->frags[k].page_offset += eat;
1991 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
1992 if (!i)
1993 goto end;
1994 eat = 0;
1995 }
1996 k++;
1997 }
1998 }
1999 skb_shinfo(skb)->nr_frags = k;
2000
2001end:
2002 skb->tail += delta;
2003 skb->data_len -= delta;
2004
2005 if (!skb->data_len)
2006 skb_zcopy_clear(skb, false);
2007
2008 return skb_tail_pointer(skb);
2009}
2010EXPORT_SYMBOL(__pskb_pull_tail);
2011
2012/**
2013 * skb_copy_bits - copy bits from skb to kernel buffer
2014 * @skb: source skb
2015 * @offset: offset in source
2016 * @to: destination buffer
2017 * @len: number of bytes to copy
2018 *
2019 * Copy the specified number of bytes from the source skb to the
2020 * destination buffer.
2021 *
2022 * CAUTION ! :
2023 * If its prototype is ever changed,
2024 * check arch/{*}/net/{*}.S files,
2025 * since it is called from BPF assembly code.
2026 */
2027int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
2028{
2029 int start = skb_headlen(skb);
2030 struct sk_buff *frag_iter;
2031 int i, copy;
2032
2033 if (offset > (int)skb->len - len)
2034 goto fault;
2035
2036 /* Copy header. */
2037 if ((copy = start - offset) > 0) {
2038 if (copy > len)
2039 copy = len;
2040 skb_copy_from_linear_data_offset(skb, offset, to, copy);
2041 if ((len -= copy) == 0)
2042 return 0;
2043 offset += copy;
2044 to += copy;
2045 }
2046
2047 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2048 int end;
2049 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
2050
2051 WARN_ON(start > offset + len);
2052
2053 end = start + skb_frag_size(f);
2054 if ((copy = end - offset) > 0) {
2055 u32 p_off, p_len, copied;
2056 struct page *p;
2057 u8 *vaddr;
2058
2059 if (copy > len)
2060 copy = len;
2061
2062 skb_frag_foreach_page(f,
2063 f->page_offset + offset - start,
2064 copy, p, p_off, p_len, copied) {
2065 vaddr = kmap_atomic(p);
2066 memcpy(to + copied, vaddr + p_off, p_len);
2067 kunmap_atomic(vaddr);
2068 }
2069
2070 if ((len -= copy) == 0)
2071 return 0;
2072 offset += copy;
2073 to += copy;
2074 }
2075 start = end;
2076 }
2077
2078 skb_walk_frags(skb, frag_iter) {
2079 int end;
2080
2081 WARN_ON(start > offset + len);
2082
2083 end = start + frag_iter->len;
2084 if ((copy = end - offset) > 0) {
2085 if (copy > len)
2086 copy = len;
2087 if (skb_copy_bits(frag_iter, offset - start, to, copy))
2088 goto fault;
2089 if ((len -= copy) == 0)
2090 return 0;
2091 offset += copy;
2092 to += copy;
2093 }
2094 start = end;
2095 }
2096
2097 if (!len)
2098 return 0;
2099
2100fault:
2101 return -EFAULT;
2102}
2103EXPORT_SYMBOL(skb_copy_bits);
2104
2105/*
2106 * Callback from splice_to_pipe(), if we need to release some pages
2107 * at the end of the spd in case we error'ed out in filling the pipe.
2108 */
2109static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
2110{
2111 put_page(spd->pages[i]);
2112}
2113
2114static struct page *linear_to_page(struct page *page, unsigned int *len,
2115 unsigned int *offset,
2116 struct sock *sk)
2117{
2118 struct page_frag *pfrag = sk_page_frag(sk);
2119
2120 if (!sk_page_frag_refill(sk, pfrag))
2121 return NULL;
2122
2123 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
2124
2125 memcpy(page_address(pfrag->page) + pfrag->offset,
2126 page_address(page) + *offset, *len);
2127 *offset = pfrag->offset;
2128 pfrag->offset += *len;
2129
2130 return pfrag->page;
2131}
2132
2133static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
2134 struct page *page,
2135 unsigned int offset)
2136{
2137 return spd->nr_pages &&
2138 spd->pages[spd->nr_pages - 1] == page &&
2139 (spd->partial[spd->nr_pages - 1].offset +
2140 spd->partial[spd->nr_pages - 1].len == offset);
2141}
2142
2143/*
2144 * Fill page/offset/length into spd, if it can hold more pages.
2145 */
2146static bool spd_fill_page(struct splice_pipe_desc *spd,
2147 struct pipe_inode_info *pipe, struct page *page,
2148 unsigned int *len, unsigned int offset,
2149 bool linear,
2150 struct sock *sk)
2151{
2152 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
2153 return true;
2154
2155 if (linear) {
2156 page = linear_to_page(page, len, &offset, sk);
2157 if (!page)
2158 return true;
2159 }
2160 if (spd_can_coalesce(spd, page, offset)) {
2161 spd->partial[spd->nr_pages - 1].len += *len;
2162 return false;
2163 }
2164 get_page(page);
2165 spd->pages[spd->nr_pages] = page;
2166 spd->partial[spd->nr_pages].len = *len;
2167 spd->partial[spd->nr_pages].offset = offset;
2168 spd->nr_pages++;
2169
2170 return false;
2171}
2172
2173static bool __splice_segment(struct page *page, unsigned int poff,
2174 unsigned int plen, unsigned int *off,
2175 unsigned int *len,
2176 struct splice_pipe_desc *spd, bool linear,
2177 struct sock *sk,
2178 struct pipe_inode_info *pipe)
2179{
2180 if (!*len)
2181 return true;
2182
2183 /* skip this segment if already processed */
2184 if (*off >= plen) {
2185 *off -= plen;
2186 return false;
2187 }
2188
2189 /* ignore any bits we already processed */
2190 poff += *off;
2191 plen -= *off;
2192 *off = 0;
2193
2194 do {
2195 unsigned int flen = min(*len, plen);
2196
2197 if (spd_fill_page(spd, pipe, page, &flen, poff,
2198 linear, sk))
2199 return true;
2200 poff += flen;
2201 plen -= flen;
2202 *len -= flen;
2203 } while (*len && plen);
2204
2205 return false;
2206}
2207
2208/*
2209 * Map linear and fragment data from the skb to spd. It reports true if the
2210 * pipe is full or if we already spliced the requested length.
2211 */
2212static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
2213 unsigned int *offset, unsigned int *len,
2214 struct splice_pipe_desc *spd, struct sock *sk)
2215{
2216 int seg;
2217 struct sk_buff *iter;
2218
2219 /* map the linear part :
2220 * If skb->head_frag is set, this 'linear' part is backed by a
2221 * fragment, and if the head is not shared with any clones then
2222 * we can avoid a copy since we own the head portion of this page.
2223 */
2224 if (__splice_segment(virt_to_page(skb->data),
2225 (unsigned long) skb->data & (PAGE_SIZE - 1),
2226 skb_headlen(skb),
2227 offset, len, spd,
2228 skb_head_is_locked(skb),
2229 sk, pipe))
2230 return true;
2231
2232 /*
2233 * then map the fragments
2234 */
2235 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
2236 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
2237
2238 if (__splice_segment(skb_frag_page(f),
2239 f->page_offset, skb_frag_size(f),
2240 offset, len, spd, false, sk, pipe))
2241 return true;
2242 }
2243
2244 skb_walk_frags(skb, iter) {
2245 if (*offset >= iter->len) {
2246 *offset -= iter->len;
2247 continue;
2248 }
2249 /* __skb_splice_bits() only fails if the output has no room
2250 * left, so no point in going over the frag_list for the error
2251 * case.
2252 */
2253 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
2254 return true;
2255 }
2256
2257 return false;
2258}
2259
2260/*
2261 * Map data from the skb to a pipe. Should handle both the linear part,
2262 * the fragments, and the frag list.
2263 */
2264int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
2265 struct pipe_inode_info *pipe, unsigned int tlen,
2266 unsigned int flags)
2267{
2268 struct partial_page partial[MAX_SKB_FRAGS];
2269 struct page *pages[MAX_SKB_FRAGS];
2270 struct splice_pipe_desc spd = {
2271 .pages = pages,
2272 .partial = partial,
2273 .nr_pages_max = MAX_SKB_FRAGS,
2274 .ops = &nosteal_pipe_buf_ops,
2275 .spd_release = sock_spd_release,
2276 };
2277 int ret = 0;
2278
2279 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
2280
2281 if (spd.nr_pages)
2282 ret = splice_to_pipe(pipe, &spd);
2283
2284 return ret;
2285}
2286EXPORT_SYMBOL_GPL(skb_splice_bits);
2287
2288/* Send skb data on a socket. Socket must be locked. */
2289int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
2290 int len)
2291{
2292 unsigned int orig_len = len;
2293 struct sk_buff *head = skb;
2294 unsigned short fragidx;
2295 int slen, ret;
2296
2297do_frag_list:
2298
2299 /* Deal with head data */
2300 while (offset < skb_headlen(skb) && len) {
2301 struct kvec kv;
2302 struct msghdr msg;
2303
2304 slen = min_t(int, len, skb_headlen(skb) - offset);
2305 kv.iov_base = skb->data + offset;
2306 kv.iov_len = slen;
2307 memset(&msg, 0, sizeof(msg));
2308
2309 ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);
2310 if (ret <= 0)
2311 goto error;
2312
2313 offset += ret;
2314 len -= ret;
2315 }
2316
2317 /* All the data was skb head? */
2318 if (!len)
2319 goto out;
2320
2321 /* Make offset relative to start of frags */
2322 offset -= skb_headlen(skb);
2323
2324 /* Find where we are in frag list */
2325 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2326 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
2327
2328 if (offset < frag->size)
2329 break;
2330
2331 offset -= frag->size;
2332 }
2333
2334 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2335 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
2336
2337 slen = min_t(size_t, len, frag->size - offset);
2338
2339 while (slen) {
2340 ret = kernel_sendpage_locked(sk, frag->page.p,
2341 frag->page_offset + offset,
2342 slen, MSG_DONTWAIT);
2343 if (ret <= 0)
2344 goto error;
2345
2346 len -= ret;
2347 offset += ret;
2348 slen -= ret;
2349 }
2350
2351 offset = 0;
2352 }
2353
2354 if (len) {
2355 /* Process any frag lists */
2356
2357 if (skb == head) {
2358 if (skb_has_frag_list(skb)) {
2359 skb = skb_shinfo(skb)->frag_list;
2360 goto do_frag_list;
2361 }
2362 } else if (skb->next) {
2363 skb = skb->next;
2364 goto do_frag_list;
2365 }
2366 }
2367
2368out:
2369 return orig_len - len;
2370
2371error:
2372 return orig_len == len ? ret : orig_len - len;
2373}
2374EXPORT_SYMBOL_GPL(skb_send_sock_locked);
2375
2376/**
2377 * skb_store_bits - store bits from kernel buffer to skb
2378 * @skb: destination buffer
2379 * @offset: offset in destination
2380 * @from: source buffer
2381 * @len: number of bytes to copy
2382 *
2383 * Copy the specified number of bytes from the source buffer to the
2384 * destination skb. This function handles all the messy bits of
2385 * traversing fragment lists and such.
2386 */
2387
2388int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
2389{
2390 int start = skb_headlen(skb);
2391 struct sk_buff *frag_iter;
2392 int i, copy;
2393
2394 if (offset > (int)skb->len - len)
2395 goto fault;
2396
2397 if ((copy = start - offset) > 0) {
2398 if (copy > len)
2399 copy = len;
2400 skb_copy_to_linear_data_offset(skb, offset, from, copy);
2401 if ((len -= copy) == 0)
2402 return 0;
2403 offset += copy;
2404 from += copy;
2405 }
2406
2407 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2408 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2409 int end;
2410
2411 WARN_ON(start > offset + len);
2412
2413 end = start + skb_frag_size(frag);
2414 if ((copy = end - offset) > 0) {
2415 u32 p_off, p_len, copied;
2416 struct page *p;
2417 u8 *vaddr;
2418
2419 if (copy > len)
2420 copy = len;
2421
2422 skb_frag_foreach_page(frag,
2423 frag->page_offset + offset - start,
2424 copy, p, p_off, p_len, copied) {
2425 vaddr = kmap_atomic(p);
2426 memcpy(vaddr + p_off, from + copied, p_len);
2427 kunmap_atomic(vaddr);
2428 }
2429
2430 if ((len -= copy) == 0)
2431 return 0;
2432 offset += copy;
2433 from += copy;
2434 }
2435 start = end;
2436 }
2437
2438 skb_walk_frags(skb, frag_iter) {
2439 int end;
2440
2441 WARN_ON(start > offset + len);
2442
2443 end = start + frag_iter->len;
2444 if ((copy = end - offset) > 0) {
2445 if (copy > len)
2446 copy = len;
2447 if (skb_store_bits(frag_iter, offset - start,
2448 from, copy))
2449 goto fault;
2450 if ((len -= copy) == 0)
2451 return 0;
2452 offset += copy;
2453 from += copy;
2454 }
2455 start = end;
2456 }
2457 if (!len)
2458 return 0;
2459
2460fault:
2461 return -EFAULT;
2462}
2463EXPORT_SYMBOL(skb_store_bits);
2464
2465/* Checksum skb data. */
2466__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
2467 __wsum csum, const struct skb_checksum_ops *ops)
2468{
2469 int start = skb_headlen(skb);
2470 int i, copy = start - offset;
2471 struct sk_buff *frag_iter;
2472 int pos = 0;
2473
2474 /* Checksum header. */
2475 if (copy > 0) {
2476 if (copy > len)
2477 copy = len;
2478 csum = ops->update(skb->data + offset, copy, csum);
2479 if ((len -= copy) == 0)
2480 return csum;
2481 offset += copy;
2482 pos = copy;
2483 }
2484
2485 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2486 int end;
2487 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2488
2489 WARN_ON(start > offset + len);
2490
2491 end = start + skb_frag_size(frag);
2492 if ((copy = end - offset) > 0) {
2493 u32 p_off, p_len, copied;
2494 struct page *p;
2495 __wsum csum2;
2496 u8 *vaddr;
2497
2498 if (copy > len)
2499 copy = len;
2500
2501 skb_frag_foreach_page(frag,
2502 frag->page_offset + offset - start,
2503 copy, p, p_off, p_len, copied) {
2504 vaddr = kmap_atomic(p);
2505 csum2 = ops->update(vaddr + p_off, p_len, 0);
2506 kunmap_atomic(vaddr);
2507 csum = ops->combine(csum, csum2, pos, p_len);
2508 pos += p_len;
2509 }
2510
2511 if (!(len -= copy))
2512 return csum;
2513 offset += copy;
2514 }
2515 start = end;
2516 }
2517
2518 skb_walk_frags(skb, frag_iter) {
2519 int end;
2520
2521 WARN_ON(start > offset + len);
2522
2523 end = start + frag_iter->len;
2524 if ((copy = end - offset) > 0) {
2525 __wsum csum2;
2526 if (copy > len)
2527 copy = len;
2528 csum2 = __skb_checksum(frag_iter, offset - start,
2529 copy, 0, ops);
2530 csum = ops->combine(csum, csum2, pos, copy);
2531 if ((len -= copy) == 0)
2532 return csum;
2533 offset += copy;
2534 pos += copy;
2535 }
2536 start = end;
2537 }
2538 BUG_ON(len);
2539
2540 return csum;
2541}
2542EXPORT_SYMBOL(__skb_checksum);
2543
2544__wsum skb_checksum(const struct sk_buff *skb, int offset,
2545 int len, __wsum csum)
2546{
2547 const struct skb_checksum_ops ops = {
2548 .update = csum_partial_ext,
2549 .combine = csum_block_add_ext,
2550 };
2551
2552 return __skb_checksum(skb, offset, len, csum, &ops);
2553}
2554EXPORT_SYMBOL(skb_checksum);
2555
2556/* Both of above in one bottle. */
2557
2558__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2559 u8 *to, int len, __wsum csum)
2560{
2561 int start = skb_headlen(skb);
2562 int i, copy = start - offset;
2563 struct sk_buff *frag_iter;
2564 int pos = 0;
2565
2566 /* Copy header. */
2567 if (copy > 0) {
2568 if (copy > len)
2569 copy = len;
2570 csum = csum_partial_copy_nocheck(skb->data + offset, to,
2571 copy, csum);
2572 if ((len -= copy) == 0)
2573 return csum;
2574 offset += copy;
2575 to += copy;
2576 pos = copy;
2577 }
2578
2579 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2580 int end;
2581
2582 WARN_ON(start > offset + len);
2583
2584 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2585 if ((copy = end - offset) > 0) {
2586 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2587 u32 p_off, p_len, copied;
2588 struct page *p;
2589 __wsum csum2;
2590 u8 *vaddr;
2591
2592 if (copy > len)
2593 copy = len;
2594
2595 skb_frag_foreach_page(frag,
2596 frag->page_offset + offset - start,
2597 copy, p, p_off, p_len, copied) {
2598 vaddr = kmap_atomic(p);
2599 csum2 = csum_partial_copy_nocheck(vaddr + p_off,
2600 to + copied,
2601 p_len, 0);
2602 kunmap_atomic(vaddr);
2603 csum = csum_block_add(csum, csum2, pos);
2604 pos += p_len;
2605 }
2606
2607 if (!(len -= copy))
2608 return csum;
2609 offset += copy;
2610 to += copy;
2611 }
2612 start = end;
2613 }
2614
2615 skb_walk_frags(skb, frag_iter) {
2616 __wsum csum2;
2617 int end;
2618
2619 WARN_ON(start > offset + len);
2620
2621 end = start + frag_iter->len;
2622 if ((copy = end - offset) > 0) {
2623 if (copy > len)
2624 copy = len;
2625 csum2 = skb_copy_and_csum_bits(frag_iter,
2626 offset - start,
2627 to, copy, 0);
2628 csum = csum_block_add(csum, csum2, pos);
2629 if ((len -= copy) == 0)
2630 return csum;
2631 offset += copy;
2632 to += copy;
2633 pos += copy;
2634 }
2635 start = end;
2636 }
2637 BUG_ON(len);
2638 return csum;
2639}
2640EXPORT_SYMBOL(skb_copy_and_csum_bits);
2641
2642__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
2643{
2644 __sum16 sum;
2645
2646 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
2647 /* See comments in __skb_checksum_complete(). */
2648 if (likely(!sum)) {
2649 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
2650 !skb->csum_complete_sw)
2651 netdev_rx_csum_fault(skb->dev, skb);
2652 }
2653 if (!skb_shared(skb))
2654 skb->csum_valid = !sum;
2655 return sum;
2656}
2657EXPORT_SYMBOL(__skb_checksum_complete_head);
2658
2659/* This function assumes skb->csum already holds pseudo header's checksum,
2660 * which has been changed from the hardware checksum, for example, by
2661 * __skb_checksum_validate_complete(). And, the original skb->csum must
2662 * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
2663 *
2664 * It returns non-zero if the recomputed checksum is still invalid, otherwise
2665 * zero. The new checksum is stored back into skb->csum unless the skb is
2666 * shared.
2667 */
2668__sum16 __skb_checksum_complete(struct sk_buff *skb)
2669{
2670 __wsum csum;
2671 __sum16 sum;
2672
2673 csum = skb_checksum(skb, 0, skb->len, 0);
2674
2675 sum = csum_fold(csum_add(skb->csum, csum));
2676 /* This check is inverted, because we already knew the hardware
2677 * checksum is invalid before calling this function. So, if the
2678 * re-computed checksum is valid instead, then we have a mismatch
2679 * between the original skb->csum and skb_checksum(). This means either
2680 * the original hardware checksum is incorrect or we screw up skb->csum
2681 * when moving skb->data around.
2682 */
2683 if (likely(!sum)) {
2684 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
2685 !skb->csum_complete_sw)
2686 netdev_rx_csum_fault(skb->dev, skb);
2687 }
2688
2689 if (!skb_shared(skb)) {
2690 /* Save full packet checksum */
2691 skb->csum = csum;
2692 skb->ip_summed = CHECKSUM_COMPLETE;
2693 skb->csum_complete_sw = 1;
2694 skb->csum_valid = !sum;
2695 }
2696
2697 return sum;
2698}
2699EXPORT_SYMBOL(__skb_checksum_complete);
2700
2701static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
2702{
2703 net_warn_ratelimited(
2704 "%s: attempt to compute crc32c without libcrc32c.ko\n",
2705 __func__);
2706 return 0;
2707}
2708
2709static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
2710 int offset, int len)
2711{
2712 net_warn_ratelimited(
2713 "%s: attempt to compute crc32c without libcrc32c.ko\n",
2714 __func__);
2715 return 0;
2716}
2717
2718static const struct skb_checksum_ops default_crc32c_ops = {
2719 .update = warn_crc32c_csum_update,
2720 .combine = warn_crc32c_csum_combine,
2721};
2722
2723const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
2724 &default_crc32c_ops;
2725EXPORT_SYMBOL(crc32c_csum_stub);
2726
2727 /**
2728 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
2729 * @from: source buffer
2730 *
2731 * Calculates the amount of linear headroom needed in the 'to' skb passed
2732 * into skb_zerocopy().
2733 */
2734unsigned int
2735skb_zerocopy_headlen(const struct sk_buff *from)
2736{
2737 unsigned int hlen = 0;
2738
2739 if (!from->head_frag ||
2740 skb_headlen(from) < L1_CACHE_BYTES ||
2741 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
2742 hlen = skb_headlen(from);
2743
2744 if (skb_has_frag_list(from))
2745 hlen = from->len;
2746
2747 return hlen;
2748}
2749EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
2750
2751/**
2752 * skb_zerocopy - Zero copy skb to skb
2753 * @to: destination buffer
2754 * @from: source buffer
2755 * @len: number of bytes to copy from source buffer
2756 * @hlen: size of linear headroom in destination buffer
2757 *
2758 * Copies up to `len` bytes from `from` to `to` by creating references
2759 * to the frags in the source buffer.
2760 *
2761 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the
2762 * headroom in the `to` buffer.
2763 *
2764 * Return value:
2765 * 0: everything is OK
2766 * -ENOMEM: couldn't orphan frags of @from due to lack of memory
2767 * -EFAULT: skb_copy_bits() found some problem with skb geometry
2768 */
2769int
2770skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
2771{
2772 int i, j = 0;
2773 int plen = 0; /* length of skb->head fragment */
2774 int ret;
2775 struct page *page;
2776 unsigned int offset;
2777
2778 BUG_ON(!from->head_frag && !hlen);
2779
2780 /* dont bother with small payloads */
2781 if (len <= skb_tailroom(to))
2782 return skb_copy_bits(from, 0, skb_put(to, len), len);
2783
2784 if (hlen) {
2785 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
2786 if (unlikely(ret))
2787 return ret;
2788 len -= hlen;
2789 } else {
2790 plen = min_t(int, skb_headlen(from), len);
2791 if (plen) {
2792 page = virt_to_head_page(from->head);
2793 offset = from->data - (unsigned char *)page_address(page);
2794 __skb_fill_page_desc(to, 0, page, offset, plen);
2795 get_page(page);
2796 j = 1;
2797 len -= plen;
2798 }
2799 }
2800
2801 to->truesize += len + plen;
2802 to->len += len + plen;
2803 to->data_len += len + plen;
2804
2805 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
2806 skb_tx_error(from);
2807 return -ENOMEM;
2808 }
2809 skb_zerocopy_clone(to, from, GFP_ATOMIC);
2810
2811 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
2812 if (!len)
2813 break;
2814 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
2815 skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len);
2816 len -= skb_shinfo(to)->frags[j].size;
2817 skb_frag_ref(to, j);
2818 j++;
2819 }
2820 skb_shinfo(to)->nr_frags = j;
2821
2822 return 0;
2823}
2824EXPORT_SYMBOL_GPL(skb_zerocopy);
2825
2826void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
2827{
2828 __wsum csum;
2829 long csstart;
2830
2831 if (skb->ip_summed == CHECKSUM_PARTIAL)
2832 csstart = skb_checksum_start_offset(skb);
2833 else
2834 csstart = skb_headlen(skb);
2835
2836 BUG_ON(csstart > skb_headlen(skb));
2837
2838 skb_copy_from_linear_data(skb, to, csstart);
2839
2840 csum = 0;
2841 if (csstart != skb->len)
2842 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
2843 skb->len - csstart, 0);
2844
2845 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2846 long csstuff = csstart + skb->csum_offset;
2847
2848 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
2849 }
2850}
2851EXPORT_SYMBOL(skb_copy_and_csum_dev);
2852
2853/**
2854 * skb_dequeue - remove from the head of the queue
2855 * @list: list to dequeue from
2856 *
2857 * Remove the head of the list. The list lock is taken so the function
2858 * may be used safely with other locking list functions. The head item is
2859 * returned or %NULL if the list is empty.
2860 */
2861
2862struct sk_buff *skb_dequeue(struct sk_buff_head *list)
2863{
2864 unsigned long flags;
2865 struct sk_buff *result;
2866
2867 spin_lock_irqsave(&list->lock, flags);
2868 result = __skb_dequeue(list);
2869 spin_unlock_irqrestore(&list->lock, flags);
2870 return result;
2871}
2872EXPORT_SYMBOL(skb_dequeue);
2873
2874/**
2875 * skb_dequeue_tail - remove from the tail of the queue
2876 * @list: list to dequeue from
2877 *
2878 * Remove the tail of the list. The list lock is taken so the function
2879 * may be used safely with other locking list functions. The tail item is
2880 * returned or %NULL if the list is empty.
2881 */
2882struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
2883{
2884 unsigned long flags;
2885 struct sk_buff *result;
2886
2887 spin_lock_irqsave(&list->lock, flags);
2888 result = __skb_dequeue_tail(list);
2889 spin_unlock_irqrestore(&list->lock, flags);
2890 return result;
2891}
2892EXPORT_SYMBOL(skb_dequeue_tail);
2893
2894/**
2895 * skb_queue_purge - empty a list
2896 * @list: list to empty
2897 *
2898 * Delete all buffers on an &sk_buff list. Each buffer is removed from
2899 * the list and one reference dropped. This function takes the list
2900 * lock and is atomic with respect to other list locking functions.
2901 */
2902void skb_queue_purge(struct sk_buff_head *list)
2903{
2904 struct sk_buff *skb;
2905 while ((skb = skb_dequeue(list)) != NULL)
2906 kfree_skb(skb);
2907}
2908EXPORT_SYMBOL(skb_queue_purge);
2909
2910/**
2911 * skb_rbtree_purge - empty a skb rbtree
2912 * @root: root of the rbtree to empty
2913 * Return value: the sum of truesizes of all purged skbs.
2914 *
2915 * Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
2916 * the list and one reference dropped. This function does not take
2917 * any lock. Synchronization should be handled by the caller (e.g., TCP
2918 * out-of-order queue is protected by the socket lock).
2919 */
2920unsigned int skb_rbtree_purge(struct rb_root *root)
2921{
2922 struct rb_node *p = rb_first(root);
2923 unsigned int sum = 0;
2924
2925 while (p) {
2926 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
2927
2928 p = rb_next(p);
2929 rb_erase(&skb->rbnode, root);
2930 sum += skb->truesize;
2931 kfree_skb(skb);
2932 }
2933 return sum;
2934}
2935
2936/**
2937 * skb_queue_head - queue a buffer at the list head
2938 * @list: list to use
2939 * @newsk: buffer to queue
2940 *
2941 * Queue a buffer at the start of the list. This function takes the
2942 * list lock and can be used safely with other locking &sk_buff functions
2943 * safely.
2944 *
2945 * A buffer cannot be placed on two lists at the same time.
2946 */
2947void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
2948{
2949 unsigned long flags;
2950
2951 spin_lock_irqsave(&list->lock, flags);
2952 __skb_queue_head(list, newsk);
2953 spin_unlock_irqrestore(&list->lock, flags);
2954}
2955EXPORT_SYMBOL(skb_queue_head);
2956
2957/**
2958 * skb_queue_tail - queue a buffer at the list tail
2959 * @list: list to use
2960 * @newsk: buffer to queue
2961 *
2962 * Queue a buffer at the tail of the list. This function takes the
2963 * list lock and can be used safely with other locking &sk_buff functions
2964 * safely.
2965 *
2966 * A buffer cannot be placed on two lists at the same time.
2967 */
2968void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
2969{
2970 unsigned long flags;
2971
2972 spin_lock_irqsave(&list->lock, flags);
2973 __skb_queue_tail(list, newsk);
2974 spin_unlock_irqrestore(&list->lock, flags);
2975}
2976EXPORT_SYMBOL(skb_queue_tail);
2977
2978/**
2979 * skb_unlink - remove a buffer from a list
2980 * @skb: buffer to remove
2981 * @list: list to use
2982 *
2983 * Remove a packet from a list. The list locks are taken and this
2984 * function is atomic with respect to other list locked calls
2985 *
2986 * You must know what list the SKB is on.
2987 */
2988void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
2989{
2990 unsigned long flags;
2991
2992 spin_lock_irqsave(&list->lock, flags);
2993 __skb_unlink(skb, list);
2994 spin_unlock_irqrestore(&list->lock, flags);
2995}
2996EXPORT_SYMBOL(skb_unlink);
2997
2998/**
2999 * skb_append - append a buffer
3000 * @old: buffer to insert after
3001 * @newsk: buffer to insert
3002 * @list: list to use
3003 *
3004 * Place a packet after a given packet in a list. The list locks are taken
3005 * and this function is atomic with respect to other list locked calls.
3006 * A buffer cannot be placed on two lists at the same time.
3007 */
3008void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
3009{
3010 unsigned long flags;
3011
3012 spin_lock_irqsave(&list->lock, flags);
3013 __skb_queue_after(list, old, newsk);
3014 spin_unlock_irqrestore(&list->lock, flags);
3015}
3016EXPORT_SYMBOL(skb_append);
3017
3018static inline void skb_split_inside_header(struct sk_buff *skb,
3019 struct sk_buff* skb1,
3020 const u32 len, const int pos)
3021{
3022 int i;
3023
3024 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
3025 pos - len);
3026 /* And move data appendix as is. */
3027 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
3028 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
3029
3030 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
3031 skb_shinfo(skb)->nr_frags = 0;
3032 skb1->data_len = skb->data_len;
3033 skb1->len += skb1->data_len;
3034 skb->data_len = 0;
3035 skb->len = len;
3036 skb_set_tail_pointer(skb, len);
3037}
3038
3039static inline void skb_split_no_header(struct sk_buff *skb,
3040 struct sk_buff* skb1,
3041 const u32 len, int pos)
3042{
3043 int i, k = 0;
3044 const int nfrags = skb_shinfo(skb)->nr_frags;
3045
3046 skb_shinfo(skb)->nr_frags = 0;
3047 skb1->len = skb1->data_len = skb->len - len;
3048 skb->len = len;
3049 skb->data_len = len - pos;
3050
3051 for (i = 0; i < nfrags; i++) {
3052 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
3053
3054 if (pos + size > len) {
3055 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
3056
3057 if (pos < len) {
3058 /* Split frag.
3059 * We have two variants in this case:
3060 * 1. Move all the frag to the second
3061 * part, if it is possible. F.e.
3062 * this approach is mandatory for TUX,
3063 * where splitting is expensive.
3064 * 2. Split is accurately. We make this.
3065 */
3066 skb_frag_ref(skb, i);
3067 skb_shinfo(skb1)->frags[0].page_offset += len - pos;
3068 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
3069 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
3070 skb_shinfo(skb)->nr_frags++;
3071 }
3072 k++;
3073 } else
3074 skb_shinfo(skb)->nr_frags++;
3075 pos += size;
3076 }
3077 skb_shinfo(skb1)->nr_frags = k;
3078}
3079
3080/**
3081 * skb_split - Split fragmented skb to two parts at length len.
3082 * @skb: the buffer to split
3083 * @skb1: the buffer to receive the second part
3084 * @len: new length for skb
3085 */
3086void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
3087{
3088 int pos = skb_headlen(skb);
3089
3090 skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags &
3091 SKBTX_SHARED_FRAG;
3092 skb_zerocopy_clone(skb1, skb, 0);
3093 if (len < pos) /* Split line is inside header. */
3094 skb_split_inside_header(skb, skb1, len, pos);
3095 else /* Second chunk has no header, nothing to copy. */
3096 skb_split_no_header(skb, skb1, len, pos);
3097}
3098EXPORT_SYMBOL(skb_split);
3099
3100/* Shifting from/to a cloned skb is a no-go.
3101 *
3102 * Caller cannot keep skb_shinfo related pointers past calling here!
3103 */
3104static int skb_prepare_for_shift(struct sk_buff *skb)
3105{
3106 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3107}
3108
3109/**
3110 * skb_shift - Shifts paged data partially from skb to another
3111 * @tgt: buffer into which tail data gets added
3112 * @skb: buffer from which the paged data comes from
3113 * @shiftlen: shift up to this many bytes
3114 *
3115 * Attempts to shift up to shiftlen worth of bytes, which may be less than
3116 * the length of the skb, from skb to tgt. Returns number bytes shifted.
3117 * It's up to caller to free skb if everything was shifted.
3118 *
3119 * If @tgt runs out of frags, the whole operation is aborted.
3120 *
3121 * Skb cannot include anything else but paged data while tgt is allowed
3122 * to have non-paged data as well.
3123 *
3124 * TODO: full sized shift could be optimized but that would need
3125 * specialized skb free'er to handle frags without up-to-date nr_frags.
3126 */
3127int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
3128{
3129 int from, to, merge, todo;
3130 struct skb_frag_struct *fragfrom, *fragto;
3131
3132 BUG_ON(shiftlen > skb->len);
3133
3134 if (skb_headlen(skb))
3135 return 0;
3136 if (skb_zcopy(tgt) || skb_zcopy(skb))
3137 return 0;
3138
3139 todo = shiftlen;
3140 from = 0;
3141 to = skb_shinfo(tgt)->nr_frags;
3142 fragfrom = &skb_shinfo(skb)->frags[from];
3143
3144 /* Actual merge is delayed until the point when we know we can
3145 * commit all, so that we don't have to undo partial changes
3146 */
3147 if (!to ||
3148 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
3149 fragfrom->page_offset)) {
3150 merge = -1;
3151 } else {
3152 merge = to - 1;
3153
3154 todo -= skb_frag_size(fragfrom);
3155 if (todo < 0) {
3156 if (skb_prepare_for_shift(skb) ||
3157 skb_prepare_for_shift(tgt))
3158 return 0;
3159
3160 /* All previous frag pointers might be stale! */
3161 fragfrom = &skb_shinfo(skb)->frags[from];
3162 fragto = &skb_shinfo(tgt)->frags[merge];
3163
3164 skb_frag_size_add(fragto, shiftlen);
3165 skb_frag_size_sub(fragfrom, shiftlen);
3166 fragfrom->page_offset += shiftlen;
3167
3168 goto onlymerged;
3169 }
3170
3171 from++;
3172 }
3173
3174 /* Skip full, not-fitting skb to avoid expensive operations */
3175 if ((shiftlen == skb->len) &&
3176 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
3177 return 0;
3178
3179 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
3180 return 0;
3181
3182 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
3183 if (to == MAX_SKB_FRAGS)
3184 return 0;
3185
3186 fragfrom = &skb_shinfo(skb)->frags[from];
3187 fragto = &skb_shinfo(tgt)->frags[to];
3188
3189 if (todo >= skb_frag_size(fragfrom)) {
3190 *fragto = *fragfrom;
3191 todo -= skb_frag_size(fragfrom);
3192 from++;
3193 to++;
3194
3195 } else {
3196 __skb_frag_ref(fragfrom);
3197 fragto->page = fragfrom->page;
3198 fragto->page_offset = fragfrom->page_offset;
3199 skb_frag_size_set(fragto, todo);
3200
3201 fragfrom->page_offset += todo;
3202 skb_frag_size_sub(fragfrom, todo);
3203 todo = 0;
3204
3205 to++;
3206 break;
3207 }
3208 }
3209
3210 /* Ready to "commit" this state change to tgt */
3211 skb_shinfo(tgt)->nr_frags = to;
3212
3213 if (merge >= 0) {
3214 fragfrom = &skb_shinfo(skb)->frags[0];
3215 fragto = &skb_shinfo(tgt)->frags[merge];
3216
3217 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
3218 __skb_frag_unref(fragfrom);
3219 }
3220
3221 /* Reposition in the original skb */
3222 to = 0;
3223 while (from < skb_shinfo(skb)->nr_frags)
3224 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
3225 skb_shinfo(skb)->nr_frags = to;
3226
3227 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
3228
3229onlymerged:
3230 /* Most likely the tgt won't ever need its checksum anymore, skb on
3231 * the other hand might need it if it needs to be resent
3232 */
3233 tgt->ip_summed = CHECKSUM_PARTIAL;
3234 skb->ip_summed = CHECKSUM_PARTIAL;
3235
3236 /* Yak, is it really working this way? Some helper please? */
3237 skb->len -= shiftlen;
3238 skb->data_len -= shiftlen;
3239 skb->truesize -= shiftlen;
3240 tgt->len += shiftlen;
3241 tgt->data_len += shiftlen;
3242 tgt->truesize += shiftlen;
3243
3244 return shiftlen;
3245}
3246
3247/**
3248 * skb_prepare_seq_read - Prepare a sequential read of skb data
3249 * @skb: the buffer to read
3250 * @from: lower offset of data to be read
3251 * @to: upper offset of data to be read
3252 * @st: state variable
3253 *
3254 * Initializes the specified state variable. Must be called before
3255 * invoking skb_seq_read() for the first time.
3256 */
3257void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
3258 unsigned int to, struct skb_seq_state *st)
3259{
3260 st->lower_offset = from;
3261 st->upper_offset = to;
3262 st->root_skb = st->cur_skb = skb;
3263 st->frag_idx = st->stepped_offset = 0;
3264 st->frag_data = NULL;
3265}
3266EXPORT_SYMBOL(skb_prepare_seq_read);
3267
3268/**
3269 * skb_seq_read - Sequentially read skb data
3270 * @consumed: number of bytes consumed by the caller so far
3271 * @data: destination pointer for data to be returned
3272 * @st: state variable
3273 *
3274 * Reads a block of skb data at @consumed relative to the
3275 * lower offset specified to skb_prepare_seq_read(). Assigns
3276 * the head of the data block to @data and returns the length
3277 * of the block or 0 if the end of the skb data or the upper
3278 * offset has been reached.
3279 *
3280 * The caller is not required to consume all of the data
3281 * returned, i.e. @consumed is typically set to the number
3282 * of bytes already consumed and the next call to
3283 * skb_seq_read() will return the remaining part of the block.
3284 *
3285 * Note 1: The size of each block of data returned can be arbitrary,
3286 * this limitation is the cost for zerocopy sequential
3287 * reads of potentially non linear data.
3288 *
3289 * Note 2: Fragment lists within fragments are not implemented
3290 * at the moment, state->root_skb could be replaced with
3291 * a stack for this purpose.
3292 */
3293unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
3294 struct skb_seq_state *st)
3295{
3296 unsigned int block_limit, abs_offset = consumed + st->lower_offset;
3297 skb_frag_t *frag;
3298
3299 if (unlikely(abs_offset >= st->upper_offset)) {
3300 if (st->frag_data) {
3301 kunmap_atomic(st->frag_data);
3302 st->frag_data = NULL;
3303 }
3304 return 0;
3305 }
3306
3307next_skb:
3308 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
3309
3310 if (abs_offset < block_limit && !st->frag_data) {
3311 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
3312 return block_limit - abs_offset;
3313 }
3314
3315 if (st->frag_idx == 0 && !st->frag_data)
3316 st->stepped_offset += skb_headlen(st->cur_skb);
3317
3318 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
3319 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
3320 block_limit = skb_frag_size(frag) + st->stepped_offset;
3321
3322 if (abs_offset < block_limit) {
3323 if (!st->frag_data)
3324 st->frag_data = kmap_atomic(skb_frag_page(frag));
3325
3326 *data = (u8 *) st->frag_data + frag->page_offset +
3327 (abs_offset - st->stepped_offset);
3328
3329 return block_limit - abs_offset;
3330 }
3331
3332 if (st->frag_data) {
3333 kunmap_atomic(st->frag_data);
3334 st->frag_data = NULL;
3335 }
3336
3337 st->frag_idx++;
3338 st->stepped_offset += skb_frag_size(frag);
3339 }
3340
3341 if (st->frag_data) {
3342 kunmap_atomic(st->frag_data);
3343 st->frag_data = NULL;
3344 }
3345
3346 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
3347 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
3348 st->frag_idx = 0;
3349 goto next_skb;
3350 } else if (st->cur_skb->next) {
3351 st->cur_skb = st->cur_skb->next;
3352 st->frag_idx = 0;
3353 goto next_skb;
3354 }
3355
3356 return 0;
3357}
3358EXPORT_SYMBOL(skb_seq_read);
3359
3360/**
3361 * skb_abort_seq_read - Abort a sequential read of skb data
3362 * @st: state variable
3363 *
3364 * Must be called if skb_seq_read() was not called until it
3365 * returned 0.
3366 */
3367void skb_abort_seq_read(struct skb_seq_state *st)
3368{
3369 if (st->frag_data)
3370 kunmap_atomic(st->frag_data);
3371}
3372EXPORT_SYMBOL(skb_abort_seq_read);
3373
3374#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
3375
3376static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
3377 struct ts_config *conf,
3378 struct ts_state *state)
3379{
3380 return skb_seq_read(offset, text, TS_SKB_CB(state));
3381}
3382
3383static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
3384{
3385 skb_abort_seq_read(TS_SKB_CB(state));
3386}
3387
3388/**
3389 * skb_find_text - Find a text pattern in skb data
3390 * @skb: the buffer to look in
3391 * @from: search offset
3392 * @to: search limit