Warning: That file was not part of the compilation database. It may have many parsing errors.

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/mm/slab.c
4 * Written by Mark Hemment, 1996/97.
5 * (markhe@nextd.demon.co.uk)
6 *
7 * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
8 *
9 * Major cleanup, different bufctl logic, per-cpu arrays
10 * (c) 2000 Manfred Spraul
11 *
12 * Cleanup, make the head arrays unconditional, preparation for NUMA
13 * (c) 2002 Manfred Spraul
14 *
15 * An implementation of the Slab Allocator as described in outline in;
16 * UNIX Internals: The New Frontiers by Uresh Vahalia
17 * Pub: Prentice Hall ISBN 0-13-101908-2
18 * or with a little more detail in;
19 * The Slab Allocator: An Object-Caching Kernel Memory Allocator
20 * Jeff Bonwick (Sun Microsystems).
21 * Presented at: USENIX Summer 1994 Technical Conference
22 *
23 * The memory is organized in caches, one cache for each object type.
24 * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
25 * Each cache consists out of many slabs (they are small (usually one
26 * page long) and always contiguous), and each slab contains multiple
27 * initialized objects.
28 *
29 * This means, that your constructor is used only for newly allocated
30 * slabs and you must pass objects with the same initializations to
31 * kmem_cache_free.
32 *
33 * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
34 * normal). If you need a special memory type, then must create a new
35 * cache for that memory type.
36 *
37 * In order to reduce fragmentation, the slabs are sorted in 3 groups:
38 * full slabs with 0 free objects
39 * partial slabs
40 * empty slabs with no allocated objects
41 *
42 * If partial slabs exist, then new allocations come from these slabs,
43 * otherwise from empty slabs or new slabs are allocated.
44 *
45 * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
46 * during kmem_cache_destroy(). The caller must prevent concurrent allocs.
47 *
48 * Each cache has a short per-cpu head array, most allocs
49 * and frees go into that array, and if that array overflows, then 1/2
50 * of the entries in the array are given back into the global cache.
51 * The head array is strictly LIFO and should improve the cache hit rates.
52 * On SMP, it additionally reduces the spinlock operations.
53 *
54 * The c_cpuarray may not be read with enabled local interrupts -
55 * it's changed with a smp_call_function().
56 *
57 * SMP synchronization:
58 * constructors and destructors are called without any locking.
59 * Several members in struct kmem_cache and struct slab never change, they
60 * are accessed without any locking.
61 * The per-cpu arrays are never accessed from the wrong cpu, no locking,
62 * and local interrupts are disabled so slab code is preempt-safe.
63 * The non-constant members are protected with a per-cache irq spinlock.
64 *
65 * Many thanks to Mark Hemment, who wrote another per-cpu slab patch
66 * in 2000 - many ideas in the current implementation are derived from
67 * his patch.
68 *
69 * Further notes from the original documentation:
70 *
71 * 11 April '97. Started multi-threading - markhe
72 * The global cache-chain is protected by the mutex 'slab_mutex'.
73 * The sem is only needed when accessing/extending the cache-chain, which
74 * can never happen inside an interrupt (kmem_cache_create(),
75 * kmem_cache_shrink() and kmem_cache_reap()).
76 *
77 * At present, each engine can be growing a cache. This should be blocked.
78 *
79 * 15 March 2005. NUMA slab allocator.
80 * Shai Fultheim <shai@scalex86.org>.
81 * Shobhit Dayal <shobhit@calsoftinc.com>
82 * Alok N Kataria <alokk@calsoftinc.com>
83 * Christoph Lameter <christoph@lameter.com>
84 *
85 * Modified the slab allocator to be node aware on NUMA systems.
86 * Each node has its own list of partial, free and full slabs.
87 * All object allocations for a node occur from node specific slab lists.
88 */
89
90#include <linux/slab.h>
91#include <linux/mm.h>
92#include <linux/poison.h>
93#include <linux/swap.h>
94#include <linux/cache.h>
95#include <linux/interrupt.h>
96#include <linux/init.h>
97#include <linux/compiler.h>
98#include <linux/cpuset.h>
99#include <linux/proc_fs.h>
100#include <linux/seq_file.h>
101#include <linux/notifier.h>
102#include <linux/kallsyms.h>
103#include <linux/cpu.h>
104#include <linux/sysctl.h>
105#include <linux/module.h>
106#include <linux/rcupdate.h>
107#include <linux/string.h>
108#include <linux/uaccess.h>
109#include <linux/nodemask.h>
110#include <linux/kmemleak.h>
111#include <linux/mempolicy.h>
112#include <linux/mutex.h>
113#include <linux/fault-inject.h>
114#include <linux/rtmutex.h>
115#include <linux/reciprocal_div.h>
116#include <linux/debugobjects.h>
117#include <linux/memory.h>
118#include <linux/prefetch.h>
119#include <linux/sched/task_stack.h>
120
121#include <net/sock.h>
122
123#include <asm/cacheflush.h>
124#include <asm/tlbflush.h>
125#include <asm/page.h>
126
127#include <trace/events/kmem.h>
128
129#include "internal.h"
130
131#include "slab.h"
132
133/*
134 * DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
135 * 0 for faster, smaller code (especially in the critical paths).
136 *
137 * STATS - 1 to collect stats for /proc/slabinfo.
138 * 0 for faster, smaller code (especially in the critical paths).
139 *
140 * FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
141 */
142
143#ifdef CONFIG_DEBUG_SLAB
144#define DEBUG 1
145#define STATS 1
146#define FORCED_DEBUG 1
147#else
148#define DEBUG 0
149#define STATS 0
150#define FORCED_DEBUG 0
151#endif
152
153/* Shouldn't this be in a header file somewhere? */
154#define BYTES_PER_WORD sizeof(void *)
155#define REDZONE_ALIGN max(BYTES_PER_WORD, __alignof__(unsigned long long))
156
157#ifndef ARCH_KMALLOC_FLAGS
158#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
159#endif
160
161#define FREELIST_BYTE_INDEX (((PAGE_SIZE >> BITS_PER_BYTE) \
162 <= SLAB_OBJ_MIN_SIZE) ? 1 : 0)
163
164#if FREELIST_BYTE_INDEX
165typedef unsigned char freelist_idx_t;
166#else
167typedef unsigned short freelist_idx_t;
168#endif
169
170#define SLAB_OBJ_MAX_NUM ((1 << sizeof(freelist_idx_t) * BITS_PER_BYTE) - 1)
171
172/*
173 * struct array_cache
174 *
175 * Purpose:
176 * - LIFO ordering, to hand out cache-warm objects from _alloc
177 * - reduce the number of linked list operations
178 * - reduce spinlock operations
179 *
180 * The limit is stored in the per-cpu structure to reduce the data cache
181 * footprint.
182 *
183 */
184struct array_cache {
185 unsigned int avail;
186 unsigned int limit;
187 unsigned int batchcount;
188 unsigned int touched;
189 void *entry[]; /*
190 * Must have this definition in here for the proper
191 * alignment of array_cache. Also simplifies accessing
192 * the entries.
193 */
194};
195
196struct alien_cache {
197 spinlock_t lock;
198 struct array_cache ac;
199};
200
201/*
202 * Need this for bootstrapping a per node allocator.
203 */
204#define NUM_INIT_LISTS (2 * MAX_NUMNODES)
205static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS];
206#define CACHE_CACHE 0
207#define SIZE_NODE (MAX_NUMNODES)
208
209static int drain_freelist(struct kmem_cache *cache,
210 struct kmem_cache_node *n, int tofree);
211static void free_block(struct kmem_cache *cachep, void **objpp, int len,
212 int node, struct list_head *list);
213static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list);
214static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
215static void cache_reap(struct work_struct *unused);
216
217static inline void fixup_objfreelist_debug(struct kmem_cache *cachep,
218 void **list);
219static inline void fixup_slab_list(struct kmem_cache *cachep,
220 struct kmem_cache_node *n, struct page *page,
221 void **list);
222static int slab_early_init = 1;
223
224#define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node))
225
226static void kmem_cache_node_init(struct kmem_cache_node *parent)
227{
228 INIT_LIST_HEAD(&parent->slabs_full);
229 INIT_LIST_HEAD(&parent->slabs_partial);
230 INIT_LIST_HEAD(&parent->slabs_free);
231 parent->total_slabs = 0;
232 parent->free_slabs = 0;
233 parent->shared = NULL;
234 parent->alien = NULL;
235 parent->colour_next = 0;
236 spin_lock_init(&parent->list_lock);
237 parent->free_objects = 0;
238 parent->free_touched = 0;
239}
240
241#define MAKE_LIST(cachep, listp, slab, nodeid) \
242 do { \
243 INIT_LIST_HEAD(listp); \
244 list_splice(&get_node(cachep, nodeid)->slab, listp); \
245 } while (0)
246
247#define MAKE_ALL_LISTS(cachep, ptr, nodeid) \
248 do { \
249 MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
250 MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
251 MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
252 } while (0)
253
254#define CFLGS_OBJFREELIST_SLAB ((slab_flags_t __force)0x40000000U)
255#define CFLGS_OFF_SLAB ((slab_flags_t __force)0x80000000U)
256#define OBJFREELIST_SLAB(x) ((x)->flags & CFLGS_OBJFREELIST_SLAB)
257#define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB)
258
259#define BATCHREFILL_LIMIT 16
260/*
261 * Optimization question: fewer reaps means less probability for unnessary
262 * cpucache drain/refill cycles.
263 *
264 * OTOH the cpuarrays can contain lots of objects,
265 * which could lock up otherwise freeable slabs.
266 */
267#define REAPTIMEOUT_AC (2*HZ)
268#define REAPTIMEOUT_NODE (4*HZ)
269
270#if STATS
271#define STATS_INC_ACTIVE(x) ((x)->num_active++)
272#define STATS_DEC_ACTIVE(x) ((x)->num_active--)
273#define STATS_INC_ALLOCED(x) ((x)->num_allocations++)
274#define STATS_INC_GROWN(x) ((x)->grown++)
275#define STATS_ADD_REAPED(x,y) ((x)->reaped += (y))
276#define STATS_SET_HIGH(x) \
277 do { \
278 if ((x)->num_active > (x)->high_mark) \
279 (x)->high_mark = (x)->num_active; \
280 } while (0)
281#define STATS_INC_ERR(x) ((x)->errors++)
282#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
283#define STATS_INC_NODEFREES(x) ((x)->node_frees++)
284#define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++)
285#define STATS_SET_FREEABLE(x, i) \
286 do { \
287 if ((x)->max_freeable < i) \
288 (x)->max_freeable = i; \
289 } while (0)
290#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
291#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
292#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
293#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
294#else
295#define STATS_INC_ACTIVE(x) do { } while (0)
296#define STATS_DEC_ACTIVE(x) do { } while (0)
297#define STATS_INC_ALLOCED(x) do { } while (0)
298#define STATS_INC_GROWN(x) do { } while (0)
299#define STATS_ADD_REAPED(x,y) do { (void)(y); } while (0)
300#define STATS_SET_HIGH(x) do { } while (0)
301#define STATS_INC_ERR(x) do { } while (0)
302#define STATS_INC_NODEALLOCS(x) do { } while (0)
303#define STATS_INC_NODEFREES(x) do { } while (0)
304#define STATS_INC_ACOVERFLOW(x) do { } while (0)
305#define STATS_SET_FREEABLE(x, i) do { } while (0)
306#define STATS_INC_ALLOCHIT(x) do { } while (0)
307#define STATS_INC_ALLOCMISS(x) do { } while (0)
308#define STATS_INC_FREEHIT(x) do { } while (0)
309#define STATS_INC_FREEMISS(x) do { } while (0)
310#endif
311
312#if DEBUG
313
314/*
315 * memory layout of objects:
316 * 0 : objp
317 * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
318 * the end of an object is aligned with the end of the real
319 * allocation. Catches writes behind the end of the allocation.
320 * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
321 * redzone word.
322 * cachep->obj_offset: The real object.
323 * cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
324 * cachep->size - 1* BYTES_PER_WORD: last caller address
325 * [BYTES_PER_WORD long]
326 */
327static int obj_offset(struct kmem_cache *cachep)
328{
329 return cachep->obj_offset;
330}
331
332static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
333{
334 BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
335 return (unsigned long long*) (objp + obj_offset(cachep) -
336 sizeof(unsigned long long));
337}
338
339static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp)
340{
341 BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
342 if (cachep->flags & SLAB_STORE_USER)
343 return (unsigned long long *)(objp + cachep->size -
344 sizeof(unsigned long long) -
345 REDZONE_ALIGN);
346 return (unsigned long long *) (objp + cachep->size -
347 sizeof(unsigned long long));
348}
349
350static void **dbg_userword(struct kmem_cache *cachep, void *objp)
351{
352 BUG_ON(!(cachep->flags & SLAB_STORE_USER));
353 return (void **)(objp + cachep->size - BYTES_PER_WORD);
354}
355
356#else
357
358#define obj_offset(x) 0
359#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
360#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
361#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
362
363#endif
364
365#ifdef CONFIG_DEBUG_SLAB_LEAK
366
367static inline bool is_store_user_clean(struct kmem_cache *cachep)
368{
369 return atomic_read(&cachep->store_user_clean) == 1;
370}
371
372static inline void set_store_user_clean(struct kmem_cache *cachep)
373{
374 atomic_set(&cachep->store_user_clean, 1);
375}
376
377static inline void set_store_user_dirty(struct kmem_cache *cachep)
378{
379 if (is_store_user_clean(cachep))
380 atomic_set(&cachep->store_user_clean, 0);
381}
382
383#else
384static inline void set_store_user_dirty(struct kmem_cache *cachep) {}
385
386#endif
387
388/*
389 * Do not go above this order unless 0 objects fit into the slab or
390 * overridden on the command line.
391 */
392#define SLAB_MAX_ORDER_HI 1
393#define SLAB_MAX_ORDER_LO 0
394static int slab_max_order = SLAB_MAX_ORDER_LO;
395static bool slab_max_order_set __initdata;
396
397static inline struct kmem_cache *virt_to_cache(const void *obj)
398{
399 struct page *page = virt_to_head_page(obj);
400 return page->slab_cache;
401}
402
403static inline void *index_to_obj(struct kmem_cache *cache, struct page *page,
404 unsigned int idx)
405{
406 return page->s_mem + cache->size * idx;
407}
408
409#define BOOT_CPUCACHE_ENTRIES 1
410/* internal cache of cache description objs */
411static struct kmem_cache kmem_cache_boot = {
412 .batchcount = 1,
413 .limit = BOOT_CPUCACHE_ENTRIES,
414 .shared = 1,
415 .size = sizeof(struct kmem_cache),
416 .name = "kmem_cache",
417};
418
419static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
420
421static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
422{
423 return this_cpu_ptr(cachep->cpu_cache);
424}
425
426/*
427 * Calculate the number of objects and left-over bytes for a given buffer size.
428 */
429static unsigned int cache_estimate(unsigned long gfporder, size_t buffer_size,
430 slab_flags_t flags, size_t *left_over)
431{
432 unsigned int num;
433 size_t slab_size = PAGE_SIZE << gfporder;
434
435 /*
436 * The slab management structure can be either off the slab or
437 * on it. For the latter case, the memory allocated for a
438 * slab is used for:
439 *
440 * - @buffer_size bytes for each object
441 * - One freelist_idx_t for each object
442 *
443 * We don't need to consider alignment of freelist because
444 * freelist will be at the end of slab page. The objects will be
445 * at the correct alignment.
446 *
447 * If the slab management structure is off the slab, then the
448 * alignment will already be calculated into the size. Because
449 * the slabs are all pages aligned, the objects will be at the
450 * correct alignment when allocated.
451 */
452 if (flags & (CFLGS_OBJFREELIST_SLAB | CFLGS_OFF_SLAB)) {
453 num = slab_size / buffer_size;
454 *left_over = slab_size % buffer_size;
455 } else {
456 num = slab_size / (buffer_size + sizeof(freelist_idx_t));
457 *left_over = slab_size %
458 (buffer_size + sizeof(freelist_idx_t));
459 }
460
461 return num;
462}
463
464#if DEBUG
465#define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)
466
467static void __slab_error(const char *function, struct kmem_cache *cachep,
468 char *msg)
469{
470 pr_err("slab error in %s(): cache `%s': %s\n",
471 function, cachep->name, msg);
472 dump_stack();
473 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
474}
475#endif
476
477/*
478 * By default on NUMA we use alien caches to stage the freeing of
479 * objects allocated from other nodes. This causes massive memory
480 * inefficiencies when using fake NUMA setup to split memory into a
481 * large number of small nodes, so it can be disabled on the command
482 * line
483 */
484
485static int use_alien_caches __read_mostly = 1;
486static int __init noaliencache_setup(char *s)
487{
488 use_alien_caches = 0;
489 return 1;
490}
491__setup("noaliencache", noaliencache_setup);
492
493static int __init slab_max_order_setup(char *str)
494{
495 get_option(&str, &slab_max_order);
496 slab_max_order = slab_max_order < 0 ? 0 :
497 min(slab_max_order, MAX_ORDER - 1);
498 slab_max_order_set = true;
499
500 return 1;
501}
502__setup("slab_max_order=", slab_max_order_setup);
503
504#ifdef CONFIG_NUMA
505/*
506 * Special reaping functions for NUMA systems called from cache_reap().
507 * These take care of doing round robin flushing of alien caches (containing
508 * objects freed on different nodes from which they were allocated) and the
509 * flushing of remote pcps by calling drain_node_pages.
510 */
511static DEFINE_PER_CPU(unsigned long, slab_reap_node);
512
513static void init_reap_node(int cpu)
514{
515 per_cpu(slab_reap_node, cpu) = next_node_in(cpu_to_mem(cpu),
516 node_online_map);
517}
518
519static void next_reap_node(void)
520{
521 int node = __this_cpu_read(slab_reap_node);
522
523 node = next_node_in(node, node_online_map);
524 __this_cpu_write(slab_reap_node, node);
525}
526
527#else
528#define init_reap_node(cpu) do { } while (0)
529#define next_reap_node(void) do { } while (0)
530#endif
531
532/*
533 * Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz
534 * via the workqueue/eventd.
535 * Add the CPU number into the expiration time to minimize the possibility of
536 * the CPUs getting into lockstep and contending for the global cache chain
537 * lock.
538 */
539static void start_cpu_timer(int cpu)
540{
541 struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
542
543 if (reap_work->work.func == NULL) {
544 init_reap_node(cpu);
545 INIT_DEFERRABLE_WORK(reap_work, cache_reap);
546 schedule_delayed_work_on(cpu, reap_work,
547 __round_jiffies_relative(HZ, cpu));
548 }
549}
550
551static void init_arraycache(struct array_cache *ac, int limit, int batch)
552{
553 if (ac) {
554 ac->avail = 0;
555 ac->limit = limit;
556 ac->batchcount = batch;
557 ac->touched = 0;
558 }
559}
560
561static struct array_cache *alloc_arraycache(int node, int entries,
562 int batchcount, gfp_t gfp)
563{
564 size_t memsize = sizeof(void *) * entries + sizeof(struct array_cache);
565 struct array_cache *ac = NULL;
566
567 ac = kmalloc_node(memsize, gfp, node);
568 /*
569 * The array_cache structures contain pointers to free object.
570 * However, when such objects are allocated or transferred to another
571 * cache the pointers are not cleared and they could be counted as
572 * valid references during a kmemleak scan. Therefore, kmemleak must
573 * not scan such objects.
574 */
575 kmemleak_no_scan(ac);
576 init_arraycache(ac, entries, batchcount);
577 return ac;
578}
579
580static noinline void cache_free_pfmemalloc(struct kmem_cache *cachep,
581 struct page *page, void *objp)
582{
583 struct kmem_cache_node *n;
584 int page_node;
585 LIST_HEAD(list);
586
587 page_node = page_to_nid(page);
588 n = get_node(cachep, page_node);
589
590 spin_lock(&n->list_lock);
591 free_block(cachep, &objp, 1, page_node, &list);
592 spin_unlock(&n->list_lock);
593
594 slabs_destroy(cachep, &list);
595}
596
597/*
598 * Transfer objects in one arraycache to another.
599 * Locking must be handled by the caller.
600 *
601 * Return the number of entries transferred.
602 */
603static int transfer_objects(struct array_cache *to,
604 struct array_cache *from, unsigned int max)
605{
606 /* Figure out how many entries to transfer */
607 int nr = min3(from->avail, max, to->limit - to->avail);
608
609 if (!nr)
610 return 0;
611
612 memcpy(to->entry + to->avail, from->entry + from->avail -nr,
613 sizeof(void *) *nr);
614
615 from->avail -= nr;
616 to->avail += nr;
617 return nr;
618}
619
620#ifndef CONFIG_NUMA
621
622#define drain_alien_cache(cachep, alien) do { } while (0)
623#define reap_alien(cachep, n) do { } while (0)
624
625static inline struct alien_cache **alloc_alien_cache(int node,
626 int limit, gfp_t gfp)
627{
628 return NULL;
629}
630
631static inline void free_alien_cache(struct alien_cache **ac_ptr)
632{
633}
634
635static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
636{
637 return 0;
638}
639
640static inline void *alternate_node_alloc(struct kmem_cache *cachep,
641 gfp_t flags)
642{
643 return NULL;
644}
645
646static inline void *____cache_alloc_node(struct kmem_cache *cachep,
647 gfp_t flags, int nodeid)
648{
649 return NULL;
650}
651
652static inline gfp_t gfp_exact_node(gfp_t flags)
653{
654 return flags & ~__GFP_NOFAIL;
655}
656
657#else /* CONFIG_NUMA */
658
659static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int);
660static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
661
662static struct alien_cache *__alloc_alien_cache(int node, int entries,
663 int batch, gfp_t gfp)
664{
665 size_t memsize = sizeof(void *) * entries + sizeof(struct alien_cache);
666 struct alien_cache *alc = NULL;
667
668 alc = kmalloc_node(memsize, gfp, node);
669 if (alc) {
670 kmemleak_no_scan(alc);
671 init_arraycache(&alc->ac, entries, batch);
672 spin_lock_init(&alc->lock);
673 }
674 return alc;
675}
676
677static struct alien_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
678{
679 struct alien_cache **alc_ptr;
680 int i;
681
682 if (limit > 1)
683 limit = 12;
684 alc_ptr = kcalloc_node(nr_node_ids, sizeof(void *), gfp, node);
685 if (!alc_ptr)
686 return NULL;
687
688 for_each_node(i) {
689 if (i == node || !node_online(i))
690 continue;
691 alc_ptr[i] = __alloc_alien_cache(node, limit, 0xbaadf00d, gfp);
692 if (!alc_ptr[i]) {
693 for (i--; i >= 0; i--)
694 kfree(alc_ptr[i]);
695 kfree(alc_ptr);
696 return NULL;
697 }
698 }
699 return alc_ptr;
700}
701
702static void free_alien_cache(struct alien_cache **alc_ptr)
703{
704 int i;
705
706 if (!alc_ptr)
707 return;
708 for_each_node(i)
709 kfree(alc_ptr[i]);
710 kfree(alc_ptr);
711}
712
713static void __drain_alien_cache(struct kmem_cache *cachep,
714 struct array_cache *ac, int node,
715 struct list_head *list)
716{
717 struct kmem_cache_node *n = get_node(cachep, node);
718
719 if (ac->avail) {
720 spin_lock(&n->list_lock);
721 /*
722 * Stuff objects into the remote nodes shared array first.
723 * That way we could avoid the overhead of putting the objects
724 * into the free lists and getting them back later.
725 */
726 if (n->shared)
727 transfer_objects(n->shared, ac, ac->limit);
728
729 free_block(cachep, ac->entry, ac->avail, node, list);
730 ac->avail = 0;
731 spin_unlock(&n->list_lock);
732 }
733}
734
735/*
736 * Called from cache_reap() to regularly drain alien caches round robin.
737 */
738static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n)
739{
740 int node = __this_cpu_read(slab_reap_node);
741
742 if (n->alien) {
743 struct alien_cache *alc = n->alien[node];
744 struct array_cache *ac;
745
746 if (alc) {
747 ac = &alc->ac;
748 if (ac->avail && spin_trylock_irq(&alc->lock)) {
749 LIST_HEAD(list);
750
751 __drain_alien_cache(cachep, ac, node, &list);
752 spin_unlock_irq(&alc->lock);
753 slabs_destroy(cachep, &list);
754 }
755 }
756 }
757}
758
759static void drain_alien_cache(struct kmem_cache *cachep,
760 struct alien_cache **alien)
761{
762 int i = 0;
763 struct alien_cache *alc;
764 struct array_cache *ac;
765 unsigned long flags;
766
767 for_each_online_node(i) {
768 alc = alien[i];
769 if (alc) {
770 LIST_HEAD(list);
771
772 ac = &alc->ac;
773 spin_lock_irqsave(&alc->lock, flags);
774 __drain_alien_cache(cachep, ac, i, &list);
775 spin_unlock_irqrestore(&alc->lock, flags);
776 slabs_destroy(cachep, &list);
777 }
778 }
779}
780
781static int __cache_free_alien(struct kmem_cache *cachep, void *objp,
782 int node, int page_node)
783{
784 struct kmem_cache_node *n;
785 struct alien_cache *alien = NULL;
786 struct array_cache *ac;
787 LIST_HEAD(list);
788
789 n = get_node(cachep, node);
790 STATS_INC_NODEFREES(cachep);
791 if (n->alien && n->alien[page_node]) {
792 alien = n->alien[page_node];
793 ac = &alien->ac;
794 spin_lock(&alien->lock);
795 if (unlikely(ac->avail == ac->limit)) {
796 STATS_INC_ACOVERFLOW(cachep);
797 __drain_alien_cache(cachep, ac, page_node, &list);
798 }
799 ac->entry[ac->avail++] = objp;
800 spin_unlock(&alien->lock);
801 slabs_destroy(cachep, &list);
802 } else {
803 n = get_node(cachep, page_node);
804 spin_lock(&n->list_lock);
805 free_block(cachep, &objp, 1, page_node, &list);
806 spin_unlock(&n->list_lock);
807 slabs_destroy(cachep, &list);
808 }
809 return 1;
810}
811
812static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
813{
814 int page_node = page_to_nid(virt_to_page(objp));
815 int node = numa_mem_id();
816 /*
817 * Make sure we are not freeing a object from another node to the array
818 * cache on this cpu.
819 */
820 if (likely(node == page_node))
821 return 0;
822
823 return __cache_free_alien(cachep, objp, node, page_node);
824}
825
826/*
827 * Construct gfp mask to allocate from a specific node but do not reclaim or
828 * warn about failures.
829 */
830static inline gfp_t gfp_exact_node(gfp_t flags)
831{
832 return (flags | __GFP_THISNODE | __GFP_NOWARN) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
833}
834#endif
835
836static int init_cache_node(struct kmem_cache *cachep, int node, gfp_t gfp)
837{
838 struct kmem_cache_node *n;
839
840 /*
841 * Set up the kmem_cache_node for cpu before we can
842 * begin anything. Make sure some other cpu on this
843 * node has not already allocated this
844 */
845 n = get_node(cachep, node);
846 if (n) {
847 spin_lock_irq(&n->list_lock);
848 n->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount +
849 cachep->num;
850 spin_unlock_irq(&n->list_lock);
851
852 return 0;
853 }
854
855 n = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node);
856 if (!n)
857 return -ENOMEM;
858
859 kmem_cache_node_init(n);
860 n->next_reap = jiffies + REAPTIMEOUT_NODE +
861 ((unsigned long)cachep) % REAPTIMEOUT_NODE;
862
863 n->free_limit =
864 (1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num;
865
866 /*
867 * The kmem_cache_nodes don't come and go as CPUs
868 * come and go. slab_mutex is sufficient
869 * protection here.
870 */
871 cachep->node[node] = n;
872
873 return 0;
874}
875
876#if (defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)) || defined(CONFIG_SMP)
877/*
878 * Allocates and initializes node for a node on each slab cache, used for
879 * either memory or cpu hotplug. If memory is being hot-added, the kmem_cache_node
880 * will be allocated off-node since memory is not yet online for the new node.
881 * When hotplugging memory or a cpu, existing node are not replaced if
882 * already in use.
883 *
884 * Must hold slab_mutex.
885 */
886static int init_cache_node_node(int node)
887{
888 int ret;
889 struct kmem_cache *cachep;
890
891 list_for_each_entry(cachep, &slab_caches, list) {
892 ret = init_cache_node(cachep, node, GFP_KERNEL);
893 if (ret)
894 return ret;
895 }
896
897 return 0;
898}
899#endif
900
901static int setup_kmem_cache_node(struct kmem_cache *cachep,
902 int node, gfp_t gfp, bool force_change)
903{
904 int ret = -ENOMEM;
905 struct kmem_cache_node *n;
906 struct array_cache *old_shared = NULL;
907 struct array_cache *new_shared = NULL;
908 struct alien_cache **new_alien = NULL;
909 LIST_HEAD(list);
910
911 if (use_alien_caches) {
912 new_alien = alloc_alien_cache(node, cachep->limit, gfp);
913 if (!new_alien)
914 goto fail;
915 }
916
917 if (cachep->shared) {
918 new_shared = alloc_arraycache(node,
919 cachep->shared * cachep->batchcount, 0xbaadf00d, gfp);
920 if (!new_shared)
921 goto fail;
922 }
923
924 ret = init_cache_node(cachep, node, gfp);
925 if (ret)
926 goto fail;
927
928 n = get_node(cachep, node);
929 spin_lock_irq(&n->list_lock);
930 if (n->shared && force_change) {
931 free_block(cachep, n->shared->entry,
932 n->shared->avail, node, &list);
933 n->shared->avail = 0;
934 }
935
936 if (!n->shared || force_change) {
937 old_shared = n->shared;
938 n->shared = new_shared;
939 new_shared = NULL;
940 }
941
942 if (!n->alien) {
943 n->alien = new_alien;
944 new_alien = NULL;
945 }
946
947 spin_unlock_irq(&n->list_lock);
948 slabs_destroy(cachep, &list);
949
950 /*
951 * To protect lockless access to n->shared during irq disabled context.
952 * If n->shared isn't NULL in irq disabled context, accessing to it is
953 * guaranteed to be valid until irq is re-enabled, because it will be
954 * freed after synchronize_rcu().
955 */
956 if (old_shared && force_change)
957 synchronize_rcu();
958
959fail:
960 kfree(old_shared);
961 kfree(new_shared);
962 free_alien_cache(new_alien);
963
964 return ret;
965}
966
967#ifdef CONFIG_SMP
968
969static void cpuup_canceled(long cpu)
970{
971 struct kmem_cache *cachep;
972 struct kmem_cache_node *n = NULL;
973 int node = cpu_to_mem(cpu);
974 const struct cpumask *mask = cpumask_of_node(node);
975
976 list_for_each_entry(cachep, &slab_caches, list) {
977 struct array_cache *nc;
978 struct array_cache *shared;
979 struct alien_cache **alien;
980 LIST_HEAD(list);
981
982 n = get_node(cachep, node);
983 if (!n)
984 continue;
985
986 spin_lock_irq(&n->list_lock);
987
988 /* Free limit for this kmem_cache_node */
989 n->free_limit -= cachep->batchcount;
990
991 /* cpu is dead; no one can alloc from it. */
992 nc = per_cpu_ptr(cachep->cpu_cache, cpu);
993 if (nc) {
994 free_block(cachep, nc->entry, nc->avail, node, &list);
995 nc->avail = 0;
996 }
997
998 if (!cpumask_empty(mask)) {
999 spin_unlock_irq(&n->list_lock);
1000 goto free_slab;
1001 }
1002
1003 shared = n->shared;
1004 if (shared) {
1005 free_block(cachep, shared->entry,
1006 shared->avail, node, &list);
1007 n->shared = NULL;
1008 }
1009
1010 alien = n->alien;
1011 n->alien = NULL;
1012
1013 spin_unlock_irq(&n->list_lock);
1014
1015 kfree(shared);
1016 if (alien) {
1017 drain_alien_cache(cachep, alien);
1018 free_alien_cache(alien);
1019 }
1020
1021free_slab:
1022 slabs_destroy(cachep, &list);
1023 }
1024 /*
1025 * In the previous loop, all the objects were freed to
1026 * the respective cache's slabs, now we can go ahead and
1027 * shrink each nodelist to its limit.
1028 */
1029 list_for_each_entry(cachep, &slab_caches, list) {
1030 n = get_node(cachep, node);
1031 if (!n)
1032 continue;
1033 drain_freelist(cachep, n, INT_MAX);
1034 }
1035}
1036
1037static int cpuup_prepare(long cpu)
1038{
1039 struct kmem_cache *cachep;
1040 int node = cpu_to_mem(cpu);
1041 int err;
1042
1043 /*
1044 * We need to do this right in the beginning since
1045 * alloc_arraycache's are going to use this list.
1046 * kmalloc_node allows us to add the slab to the right
1047 * kmem_cache_node and not this cpu's kmem_cache_node
1048 */
1049 err = init_cache_node_node(node);
1050 if (err < 0)
1051 goto bad;
1052
1053 /*
1054 * Now we can go ahead with allocating the shared arrays and
1055 * array caches
1056 */
1057 list_for_each_entry(cachep, &slab_caches, list) {
1058 err = setup_kmem_cache_node(cachep, node, GFP_KERNEL, false);
1059 if (err)
1060 goto bad;
1061 }
1062
1063 return 0;
1064bad:
1065 cpuup_canceled(cpu);
1066 return -ENOMEM;
1067}
1068
1069int slab_prepare_cpu(unsigned int cpu)
1070{
1071 int err;
1072
1073 mutex_lock(&slab_mutex);
1074 err = cpuup_prepare(cpu);
1075 mutex_unlock(&slab_mutex);
1076 return err;
1077}
1078
1079/*
1080 * This is called for a failed online attempt and for a successful
1081 * offline.
1082 *
1083 * Even if all the cpus of a node are down, we don't free the
1084 * kmem_list3 of any cache. This to avoid a race between cpu_down, and
1085 * a kmalloc allocation from another cpu for memory from the node of
1086 * the cpu going down. The list3 structure is usually allocated from
1087 * kmem_cache_create() and gets destroyed at kmem_cache_destroy().
1088 */
1089int slab_dead_cpu(unsigned int cpu)
1090{
1091 mutex_lock(&slab_mutex);
1092 cpuup_canceled(cpu);
1093 mutex_unlock(&slab_mutex);
1094 return 0;
1095}
1096#endif
1097
1098static int slab_online_cpu(unsigned int cpu)
1099{
1100 start_cpu_timer(cpu);
1101 return 0;
1102}
1103
1104static int slab_offline_cpu(unsigned int cpu)
1105{
1106 /*
1107 * Shutdown cache reaper. Note that the slab_mutex is held so
1108 * that if cache_reap() is invoked it cannot do anything
1109 * expensive but will only modify reap_work and reschedule the
1110 * timer.
1111 */
1112 cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu));
1113 /* Now the cache_reaper is guaranteed to be not running. */
1114 per_cpu(slab_reap_work, cpu).work.func = NULL;
1115 return 0;
1116}
1117
1118#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
1119/*
1120 * Drains freelist for a node on each slab cache, used for memory hot-remove.
1121 * Returns -EBUSY if all objects cannot be drained so that the node is not
1122 * removed.
1123 *
1124 * Must hold slab_mutex.
1125 */
1126static int __meminit drain_cache_node_node(int node)
1127{
1128 struct kmem_cache *cachep;
1129 int ret = 0;
1130
1131 list_for_each_entry(cachep, &slab_caches, list) {
1132 struct kmem_cache_node *n;
1133
1134 n = get_node(cachep, node);
1135 if (!n)
1136 continue;
1137
1138 drain_freelist(cachep, n, INT_MAX);
1139
1140 if (!list_empty(&n->slabs_full) ||
1141 !list_empty(&n->slabs_partial)) {
1142 ret = -EBUSY;
1143 break;
1144 }
1145 }
1146 return ret;
1147}
1148
1149static int __meminit slab_memory_callback(struct notifier_block *self,
1150 unsigned long action, void *arg)
1151{
1152 struct memory_notify *mnb = arg;
1153 int ret = 0;
1154 int nid;
1155
1156 nid = mnb->status_change_nid;
1157 if (nid < 0)
1158 goto out;
1159
1160 switch (action) {
1161 case MEM_GOING_ONLINE:
1162 mutex_lock(&slab_mutex);
1163 ret = init_cache_node_node(nid);
1164 mutex_unlock(&slab_mutex);
1165 break;
1166 case MEM_GOING_OFFLINE:
1167 mutex_lock(&slab_mutex);
1168 ret = drain_cache_node_node(nid);
1169 mutex_unlock(&slab_mutex);
1170 break;
1171 case MEM_ONLINE:
1172 case MEM_OFFLINE:
1173 case MEM_CANCEL_ONLINE:
1174 case MEM_CANCEL_OFFLINE:
1175 break;
1176 }
1177out:
1178 return notifier_from_errno(ret);
1179}
1180#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */
1181
1182/*
1183 * swap the static kmem_cache_node with kmalloced memory
1184 */
1185static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list,
1186 int nodeid)
1187{
1188 struct kmem_cache_node *ptr;
1189
1190 ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);
1191 BUG_ON(!ptr);
1192
1193 memcpy(ptr, list, sizeof(struct kmem_cache_node));
1194 /*
1195 * Do not assume that spinlocks can be initialized via memcpy:
1196 */
1197 spin_lock_init(&ptr->list_lock);
1198
1199 MAKE_ALL_LISTS(cachep, ptr, nodeid);
1200 cachep->node[nodeid] = ptr;
1201}
1202
1203/*
1204 * For setting up all the kmem_cache_node for cache whose buffer_size is same as
1205 * size of kmem_cache_node.
1206 */
1207static void __init set_up_node(struct kmem_cache *cachep, int index)
1208{
1209 int node;
1210
1211 for_each_online_node(node) {
1212 cachep->node[node] = &init_kmem_cache_node[index + node];
1213 cachep->node[node]->next_reap = jiffies +
1214 REAPTIMEOUT_NODE +
1215 ((unsigned long)cachep) % REAPTIMEOUT_NODE;
1216 }
1217}
1218
1219/*
1220 * Initialisation. Called after the page allocator have been initialised and
1221 * before smp_init().
1222 */
1223void __init kmem_cache_init(void)
1224{
1225 int i;
1226
1227 kmem_cache = &kmem_cache_boot;
1228
1229 if (!IS_ENABLED(CONFIG_NUMA) || num_possible_nodes() == 1)
1230 use_alien_caches = 0;
1231
1232 for (i = 0; i < NUM_INIT_LISTS; i++)
1233 kmem_cache_node_init(&init_kmem_cache_node[i]);
1234
1235 /*
1236 * Fragmentation resistance on low memory - only use bigger
1237 * page orders on machines with more than 32MB of memory if
1238 * not overridden on the command line.
1239 */
1240 if (!slab_max_order_set && totalram_pages() > (32 << 20) >> PAGE_SHIFT)
1241 slab_max_order = SLAB_MAX_ORDER_HI;
1242
1243 /* Bootstrap is tricky, because several objects are allocated
1244 * from caches that do not exist yet:
1245 * 1) initialize the kmem_cache cache: it contains the struct
1246 * kmem_cache structures of all caches, except kmem_cache itself:
1247 * kmem_cache is statically allocated.
1248 * Initially an __init data area is used for the head array and the
1249 * kmem_cache_node structures, it's replaced with a kmalloc allocated
1250 * array at the end of the bootstrap.
1251 * 2) Create the first kmalloc cache.
1252 * The struct kmem_cache for the new cache is allocated normally.
1253 * An __init data area is used for the head array.
1254 * 3) Create the remaining kmalloc caches, with minimally sized
1255 * head arrays.
1256 * 4) Replace the __init data head arrays for kmem_cache and the first
1257 * kmalloc cache with kmalloc allocated arrays.
1258 * 5) Replace the __init data for kmem_cache_node for kmem_cache and
1259 * the other cache's with kmalloc allocated memory.
1260 * 6) Resize the head arrays of the kmalloc caches to their final sizes.
1261 */
1262
1263 /* 1) create the kmem_cache */
1264
1265 /*
1266 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
1267 */
1268 create_boot_cache(kmem_cache, "kmem_cache",
1269 offsetof(struct kmem_cache, node) +
1270 nr_node_ids * sizeof(struct kmem_cache_node *),
1271 SLAB_HWCACHE_ALIGN, 0, 0);
1272 list_add(&kmem_cache->list, &slab_caches);
1273 memcg_link_cache(kmem_cache);
1274 slab_state = PARTIAL;
1275
1276 /*
1277 * Initialize the caches that provide memory for the kmem_cache_node
1278 * structures first. Without this, further allocations will bug.
1279 */
1280 kmalloc_caches[KMALLOC_NORMAL][INDEX_NODE] = create_kmalloc_cache(
1281 kmalloc_info[INDEX_NODE].name,
1282 kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS,
1283 0, kmalloc_size(INDEX_NODE));
1284 slab_state = PARTIAL_NODE;
1285 setup_kmalloc_cache_index_table();
1286
1287 slab_early_init = 0;
1288
1289 /* 5) Replace the bootstrap kmem_cache_node */
1290 {
1291 int nid;
1292
1293 for_each_online_node(nid) {
1294 init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid);
1295
1296 init_list(kmalloc_caches[KMALLOC_NORMAL][INDEX_NODE],
1297 &init_kmem_cache_node[SIZE_NODE + nid], nid);
1298 }
1299 }
1300
1301 create_kmalloc_caches(ARCH_KMALLOC_FLAGS);
1302}
1303
1304void __init kmem_cache_init_late(void)
1305{
1306 struct kmem_cache *cachep;
1307
1308 /* 6) resize the head arrays to their final sizes */
1309 mutex_lock(&slab_mutex);
1310 list_for_each_entry(cachep, &slab_caches, list)
1311 if (enable_cpucache(cachep, GFP_NOWAIT))
1312 BUG();
1313 mutex_unlock(&slab_mutex);
1314
1315 /* Done! */
1316 slab_state = FULL;
1317
1318#ifdef CONFIG_NUMA
1319 /*
1320 * Register a memory hotplug callback that initializes and frees
1321 * node.
1322 */
1323 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
1324#endif
1325
1326 /*
1327 * The reap timers are started later, with a module init call: That part
1328 * of the kernel is not yet operational.
1329 */
1330}
1331
1332static int __init cpucache_init(void)
1333{
1334 int ret;
1335
1336 /*
1337 * Register the timers that return unneeded pages to the page allocator
1338 */
1339 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "SLAB online",
1340 slab_online_cpu, slab_offline_cpu);
1341 WARN_ON(ret < 0);
1342
1343 return 0;
1344}
1345__initcall(cpucache_init);
1346
1347static noinline void
1348slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
1349{
1350#if DEBUG
1351 struct kmem_cache_node *n;
1352 unsigned long flags;
1353 int node;
1354 static DEFINE_RATELIMIT_STATE(slab_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
1355 DEFAULT_RATELIMIT_BURST);
1356
1357 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slab_oom_rs))
1358 return;
1359
1360 pr_warn("SLAB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
1361 nodeid, gfpflags, &gfpflags);
1362 pr_warn(" cache: %s, object size: %d, order: %d\n",
1363 cachep->name, cachep->size, cachep->gfporder);
1364
1365 for_each_kmem_cache_node(cachep, node, n) {
1366 unsigned long total_slabs, free_slabs, free_objs;
1367
1368 spin_lock_irqsave(&n->list_lock, flags);
1369 total_slabs = n->total_slabs;
1370 free_slabs = n->free_slabs;
1371 free_objs = n->free_objects;
1372 spin_unlock_irqrestore(&n->list_lock, flags);
1373
1374 pr_warn(" node %d: slabs: %ld/%ld, objs: %ld/%ld\n",
1375 node, total_slabs - free_slabs, total_slabs,
1376 (total_slabs * cachep->num) - free_objs,
1377 total_slabs * cachep->num);
1378 }
1379#endif
1380}
1381
1382/*
1383 * Interface to system's page allocator. No need to hold the
1384 * kmem_cache_node ->list_lock.
1385 *
1386 * If we requested dmaable memory, we will get it. Even if we
1387 * did not request dmaable memory, we might get it, but that
1388 * would be relatively rare and ignorable.
1389 */
1390static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
1391 int nodeid)
1392{
1393 struct page *page;
1394 int nr_pages;
1395
1396 flags |= cachep->allocflags;
1397
1398 page = __alloc_pages_node(nodeid, flags, cachep->gfporder);
1399 if (!page) {
1400 slab_out_of_memory(cachep, flags, nodeid);
1401 return NULL;
1402 }
1403
1404 if (memcg_charge_slab(page, flags, cachep->gfporder, cachep)) {
1405 __free_pages(page, cachep->gfporder);
1406 return NULL;
1407 }
1408
1409 nr_pages = (1 << cachep->gfporder);
1410 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1411 mod_lruvec_page_state(page, NR_SLAB_RECLAIMABLE, nr_pages);
1412 else
1413 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE, nr_pages);
1414
1415 __SetPageSlab(page);
1416 /* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
1417 if (sk_memalloc_socks() && page_is_pfmemalloc(page))
1418 SetPageSlabPfmemalloc(page);
1419
1420 return page;
1421}
1422
1423/*
1424 * Interface to system's page release.
1425 */
1426static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
1427{
1428 int order = cachep->gfporder;
1429 unsigned long nr_freed = (1 << order);
1430
1431 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1432 mod_lruvec_page_state(page, NR_SLAB_RECLAIMABLE, -nr_freed);
1433 else
1434 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE, -nr_freed);
1435
1436 BUG_ON(!PageSlab(page));
1437 __ClearPageSlabPfmemalloc(page);
1438 __ClearPageSlab(page);
1439 page_mapcount_reset(page);
1440 page->mapping = NULL;
1441
1442 if (current->reclaim_state)
1443 current->reclaim_state->reclaimed_slab += nr_freed;
1444 memcg_uncharge_slab(page, order, cachep);
1445 __free_pages(page, order);
1446}
1447
1448static void kmem_rcu_free(struct rcu_head *head)
1449{
1450 struct kmem_cache *cachep;
1451 struct page *page;
1452
1453 page = container_of(head, struct page, rcu_head);
1454 cachep = page->slab_cache;
1455
1456 kmem_freepages(cachep, page);
1457}
1458
1459#if DEBUG
1460static bool is_debug_pagealloc_cache(struct kmem_cache *cachep)
1461{
1462 if (debug_pagealloc_enabled() && OFF_SLAB(cachep) &&
1463 (cachep->size % PAGE_SIZE) == 0)
1464 return true;
1465
1466 return false;
1467}
1468
1469#ifdef CONFIG_DEBUG_PAGEALLOC
1470static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr,
1471 unsigned long caller)
1472{
1473 int size = cachep->object_size;
1474
1475 addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
1476
1477 if (size < 5 * sizeof(unsigned long))
1478 return;
1479
1480 *addr++ = 0x12345678;
1481 *addr++ = caller;
1482 *addr++ = smp_processor_id();
1483 size -= 3 * sizeof(unsigned long);
1484 {
1485 unsigned long *sptr = &caller;
1486 unsigned long svalue;
1487
1488 while (!kstack_end(sptr)) {
1489 svalue = *sptr++;
1490 if (kernel_text_address(svalue)) {
1491 *addr++ = svalue;
1492 size -= sizeof(unsigned long);
1493 if (size <= sizeof(unsigned long))
1494 break;
1495 }
1496 }
1497
1498 }
1499 *addr++ = 0x87654321;
1500}
1501
1502static void slab_kernel_map(struct kmem_cache *cachep, void *objp,
1503 int map, unsigned long caller)
1504{
1505 if (!is_debug_pagealloc_cache(cachep))
1506 return;
1507
1508 if (caller)
1509 store_stackinfo(cachep, objp, caller);
1510
1511 kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map);
1512}
1513
1514#else
1515static inline void slab_kernel_map(struct kmem_cache *cachep, void *objp,
1516 int map, unsigned long caller) {}
1517
1518#endif
1519
1520static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val)
1521{
1522 int size = cachep->object_size;
1523 addr = &((char *)addr)[obj_offset(cachep)];
1524
1525 memset(addr, val, size);
1526 *(unsigned char *)(addr + size - 1) = POISON_END;
1527}
1528
1529static void dump_line(char *data, int offset, int limit)
1530{
1531 int i;
1532 unsigned char error = 0;
1533 int bad_count = 0;
1534
1535 pr_err("%03x: ", offset);
1536 for (i = 0; i < limit; i++) {
1537 if (data[offset + i] != POISON_FREE) {
1538 error = data[offset + i];
1539 bad_count++;
1540 }
1541 }
1542 print_hex_dump(KERN_CONT, "", 0, 16, 1,
1543 &data[offset], limit, 1);
1544
1545 if (bad_count == 1) {
1546 error ^= POISON_FREE;
1547 if (!(error & (error - 1))) {
1548 pr_err("Single bit error detected. Probably bad RAM.\n");
1549#ifdef CONFIG_X86
1550 pr_err("Run memtest86+ or a similar memory test tool.\n");
1551#else
1552 pr_err("Run a memory test tool.\n");
1553#endif
1554 }
1555 }
1556}
1557#endif
1558
1559#if DEBUG
1560
1561static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)
1562{
1563 int i, size;
1564 char *realobj;
1565
1566 if (cachep->flags & SLAB_RED_ZONE) {
1567 pr_err("Redzone: 0x%llx/0x%llx\n",
1568 *dbg_redzone1(cachep, objp),
1569 *dbg_redzone2(cachep, objp));
1570 }
1571
1572 if (cachep->flags & SLAB_STORE_USER)
1573 pr_err("Last user: (%pSR)\n", *dbg_userword(cachep, objp));
1574 realobj = (char *)objp + obj_offset(cachep);
1575 size = cachep->object_size;
1576 for (i = 0; i < size && lines; i += 16, lines--) {
1577 int limit;
1578 limit = 16;
1579 if (i + limit > size)
1580 limit = size - i;
1581 dump_line(realobj, i, limit);
1582 }
1583}
1584
1585static void check_poison_obj(struct kmem_cache *cachep, void *objp)
1586{
1587 char *realobj;
1588 int size, i;
1589 int lines = 0;
1590
1591 if (is_debug_pagealloc_cache(cachep))
1592 return;
1593
1594 realobj = (char *)objp + obj_offset(cachep);
1595 size = cachep->object_size;
1596
1597 for (i = 0; i < size; i++) {
1598 char exp = POISON_FREE;
1599 if (i == size - 1)
1600 exp = POISON_END;
1601 if (realobj[i] != exp) {
1602 int limit;
1603 /* Mismatch ! */
1604 /* Print header */
1605 if (lines == 0) {
1606 pr_err("Slab corruption (%s): %s start=%px, len=%d\n",
1607 print_tainted(), cachep->name,
1608 realobj, size);
1609 print_objinfo(cachep, objp, 0);
1610 }
1611 /* Hexdump the affected line */
1612 i = (i / 16) * 16;
1613 limit = 16;
1614 if (i + limit > size)
1615 limit = size - i;
1616 dump_line(realobj, i, limit);
1617 i += 16;
1618 lines++;
1619 /* Limit to 5 lines */
1620 if (lines > 5)
1621 break;
1622 }
1623 }
1624 if (lines != 0) {
1625 /* Print some data about the neighboring objects, if they
1626 * exist:
1627 */
1628 struct page *page = virt_to_head_page(objp);
1629 unsigned int objnr;
1630
1631 objnr = obj_to_index(cachep, page, objp);
1632 if (objnr) {
1633 objp = index_to_obj(cachep, page, objnr - 1);
1634 realobj = (char *)objp + obj_offset(cachep);
1635 pr_err("Prev obj: start=%px, len=%d\n", realobj, size);
1636 print_objinfo(cachep, objp, 2);
1637 }
1638 if (objnr + 1 < cachep->num) {
1639 objp = index_to_obj(cachep, page, objnr + 1);
1640 realobj = (char *)objp + obj_offset(cachep);
1641 pr_err("Next obj: start=%px, len=%d\n", realobj, size);
1642 print_objinfo(cachep, objp, 2);
1643 }
1644 }
1645}
1646#endif
1647
1648#if DEBUG
1649static void slab_destroy_debugcheck(struct kmem_cache *cachep,
1650 struct page *page)
1651{
1652 int i;
1653
1654 if (OBJFREELIST_SLAB(cachep) && cachep->flags & SLAB_POISON) {
1655 poison_obj(cachep, page->freelist - obj_offset(cachep),
1656 POISON_FREE);
1657 }
1658
1659 for (i = 0; i < cachep->num; i++) {
1660 void *objp = index_to_obj(cachep, page, i);
1661
1662 if (cachep->flags & SLAB_POISON) {
1663 check_poison_obj(cachep, objp);
1664 slab_kernel_map(cachep, objp, 1, 0);
1665 }
1666 if (cachep->flags & SLAB_RED_ZONE) {
1667 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
1668 slab_error(cachep, "start of a freed object was overwritten");
1669 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
1670 slab_error(cachep, "end of a freed object was overwritten");
1671 }
1672 }
1673}
1674#else
1675static void slab_destroy_debugcheck(struct kmem_cache *cachep,
1676 struct page *page)
1677{
1678}
1679#endif
1680
1681/**
1682 * slab_destroy - destroy and release all objects in a slab
1683 * @cachep: cache pointer being destroyed
1684 * @page: page pointer being destroyed
1685 *
1686 * Destroy all the objs in a slab page, and release the mem back to the system.
1687 * Before calling the slab page must have been unlinked from the cache. The
1688 * kmem_cache_node ->list_lock is not held/needed.
1689 */
1690static void slab_destroy(struct kmem_cache *cachep, struct page *page)
1691{
1692 void *freelist;
1693
1694 freelist = page->freelist;
1695 slab_destroy_debugcheck(cachep, page);
1696 if (unlikely(cachep->flags & SLAB_TYPESAFE_BY_RCU))
1697 call_rcu(&page->rcu_head, kmem_rcu_free);
1698 else
1699 kmem_freepages(cachep, page);
1700
1701 /*
1702 * From now on, we don't use freelist
1703 * although actual page can be freed in rcu context
1704 */
1705 if (OFF_SLAB(cachep))
1706 kmem_cache_free(cachep->freelist_cache, freelist);
1707}
1708
1709static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list)
1710{
1711 struct page *page, *n;
1712
1713 list_for_each_entry_safe(page, n, list, lru) {
1714 list_del(&page->lru);
1715 slab_destroy(cachep, page);
1716 }
1717}
1718
1719/**
1720 * calculate_slab_order - calculate size (page order) of slabs
1721 * @cachep: pointer to the cache that is being created
1722 * @size: size of objects to be created in this cache.
1723 * @flags: slab allocation flags
1724 *
1725 * Also calculates the number of objects per slab.
1726 *
1727 * This could be made much more intelligent. For now, try to avoid using
1728 * high order pages for slabs. When the gfp() functions are more friendly
1729 * towards high-order requests, this should be changed.
1730 *
1731 * Return: number of left-over bytes in a slab
1732 */
1733static size_t calculate_slab_order(struct kmem_cache *cachep,
1734 size_t size, slab_flags_t flags)
1735{
1736 size_t left_over = 0;
1737 int gfporder;
1738
1739 for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) {
1740 unsigned int num;
1741 size_t remainder;
1742
1743 num = cache_estimate(gfporder, size, flags, &remainder);
1744 if (!num)
1745 continue;
1746
1747 /* Can't handle number of objects more than SLAB_OBJ_MAX_NUM */
1748 if (num > SLAB_OBJ_MAX_NUM)
1749 break;
1750
1751 if (flags & CFLGS_OFF_SLAB) {
1752 struct kmem_cache *freelist_cache;
1753 size_t freelist_size;
1754
1755 freelist_size = num * sizeof(freelist_idx_t);
1756 freelist_cache = kmalloc_slab(freelist_size, 0u);
1757 if (!freelist_cache)
1758 continue;
1759
1760 /*
1761 * Needed to avoid possible looping condition
1762 * in cache_grow_begin()
1763 */
1764 if (OFF_SLAB(freelist_cache))
1765 continue;
1766
1767 /* check if off slab has enough benefit */
1768 if (freelist_cache->size > cachep->size / 2)
1769 continue;
1770 }
1771
1772 /* Found something acceptable - save it away */
1773 cachep->num = num;
1774 cachep->gfporder = gfporder;
1775 left_over = remainder;
1776
1777 /*
1778 * A VFS-reclaimable slab tends to have most allocations
1779 * as GFP_NOFS and we really don't want to have to be allocating
1780 * higher-order pages when we are unable to shrink dcache.
1781 */
1782 if (flags & SLAB_RECLAIM_ACCOUNT)
1783 break;
1784
1785 /*
1786 * Large number of objects is good, but very large slabs are
1787 * currently bad for the gfp()s.
1788 */
1789 if (gfporder >= slab_max_order)
1790 break;
1791
1792 /*
1793 * Acceptable internal fragmentation?
1794 */
1795 if (left_over * 8 <= (PAGE_SIZE << gfporder))
1796 break;
1797 }
1798 return left_over;
1799}
1800
1801static struct array_cache __percpu *alloc_kmem_cache_cpus(
1802 struct kmem_cache *cachep, int entries, int batchcount)
1803{
1804 int cpu;
1805 size_t size;
1806 struct array_cache __percpu *cpu_cache;
1807
1808 size = sizeof(void *) * entries + sizeof(struct array_cache);
1809 cpu_cache = __alloc_percpu(size, sizeof(void *));
1810
1811 if (!cpu_cache)
1812 return NULL;
1813
1814 for_each_possible_cpu(cpu) {
1815 init_arraycache(per_cpu_ptr(cpu_cache, cpu),
1816 entries, batchcount);
1817 }
1818
1819 return cpu_cache;
1820}
1821
1822static int __ref setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
1823{
1824 if (slab_state >= FULL)
1825 return enable_cpucache(cachep, gfp);
1826
1827 cachep->cpu_cache = alloc_kmem_cache_cpus(cachep, 1, 1);
1828 if (!cachep->cpu_cache)
1829 return 1;
1830
1831 if (slab_state == DOWN) {
1832 /* Creation of first cache (kmem_cache). */
1833 set_up_node(kmem_cache, CACHE_CACHE);
1834 } else if (slab_state == PARTIAL) {
1835 /* For kmem_cache_node */
1836 set_up_node(cachep, SIZE_NODE);
1837 } else {
1838 int node;
1839
1840 for_each_online_node(node) {
1841 cachep->node[node] = kmalloc_node(
1842 sizeof(struct kmem_cache_node), gfp, node);
1843 BUG_ON(!cachep->node[node]);
1844 kmem_cache_node_init(cachep->node[node]);
1845 }
1846 }
1847
1848 cachep->node[numa_mem_id()]->next_reap =
1849 jiffies + REAPTIMEOUT_NODE +
1850 ((unsigned long)cachep) % REAPTIMEOUT_NODE;
1851
1852 cpu_cache_get(cachep)->avail = 0;
1853 cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
1854 cpu_cache_get(cachep)->batchcount = 1;
1855 cpu_cache_get(cachep)->touched = 0;
1856 cachep->batchcount = 1;
1857 cachep->limit = BOOT_CPUCACHE_ENTRIES;
1858 return 0;
1859}
1860
1861slab_flags_t kmem_cache_flags(unsigned int object_size,
1862 slab_flags_t flags, const char *name,
1863 void (*ctor)(void *))
1864{
1865 return flags;
1866}
1867
1868struct kmem_cache *
1869__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
1870 slab_flags_t flags, void (*ctor)(void *))
1871{
1872 struct kmem_cache *cachep;
1873
1874 cachep = find_mergeable(size, align, flags, name, ctor);
1875 if (cachep) {
1876 cachep->refcount++;
1877
1878 /*
1879 * Adjust the object sizes so that we clear
1880 * the complete object on kzalloc.
1881 */
1882 cachep->object_size = max_t(int, cachep->object_size, size);
1883 }
1884 return cachep;
1885}
1886
1887static bool set_objfreelist_slab_cache(struct kmem_cache *cachep,
1888 size_t size, slab_flags_t flags)
1889{
1890 size_t left;
1891
1892 cachep->num = 0;
1893
1894 if (cachep->ctor || flags & SLAB_TYPESAFE_BY_RCU)
1895 return false;
1896
1897 left = calculate_slab_order(cachep, size,
1898 flags | CFLGS_OBJFREELIST_SLAB);
1899 if (!cachep->num)
1900 return false;
1901
1902 if (cachep->num * sizeof(freelist_idx_t) > cachep->object_size)
1903 return false;
1904
1905 cachep->colour = left / cachep->colour_off;
1906
1907 return true;
1908}
1909
1910static bool set_off_slab_cache(struct kmem_cache *cachep,
1911 size_t size, slab_flags_t flags)
1912{
1913 size_t left;
1914
1915 cachep->num = 0;
1916
1917 /*
1918 * Always use on-slab management when SLAB_NOLEAKTRACE
1919 * to avoid recursive calls into kmemleak.
1920 */
1921 if (flags & SLAB_NOLEAKTRACE)
1922 return false;
1923
1924 /*
1925 * Size is large, assume best to place the slab management obj
1926 * off-slab (should allow better packing of objs).
1927 */
1928 left = calculate_slab_order(cachep, size, flags | CFLGS_OFF_SLAB);
1929 if (!cachep->num)
1930 return false;
1931
1932 /*
1933 * If the slab has been placed off-slab, and we have enough space then
1934 * move it on-slab. This is at the expense of any extra colouring.
1935 */
1936 if (left >= cachep->num * sizeof(freelist_idx_t))
1937 return false;
1938
1939 cachep->colour = left / cachep->colour_off;
1940
1941 return true;
1942}
1943
1944static bool set_on_slab_cache(struct kmem_cache *cachep,
1945 size_t size, slab_flags_t flags)
1946{
1947 size_t left;
1948
1949 cachep->num = 0;
1950
1951 left = calculate_slab_order(cachep, size, flags);
1952 if (!cachep->num)
1953 return false;
1954
1955 cachep->colour = left / cachep->colour_off;
1956
1957 return true;
1958}
1959
1960/**
1961 * __kmem_cache_create - Create a cache.
1962 * @cachep: cache management descriptor
1963 * @flags: SLAB flags
1964 *
1965 * Returns a ptr to the cache on success, NULL on failure.
1966 * Cannot be called within a int, but can be interrupted.
1967 * The @ctor is run when new pages are allocated by the cache.
1968 *
1969 * The flags are
1970 *
1971 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
1972 * to catch references to uninitialised memory.
1973 *
1974 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
1975 * for buffer overruns.
1976 *
1977 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
1978 * cacheline. This can be beneficial if you're counting cycles as closely
1979 * as davem.
1980 *
1981 * Return: a pointer to the created cache or %NULL in case of error
1982 */
1983int __kmem_cache_create(struct kmem_cache *cachep, slab_flags_t flags)
1984{
1985 size_t ralign = BYTES_PER_WORD;
1986 gfp_t gfp;
1987 int err;
1988 unsigned int size = cachep->size;
1989
1990#if DEBUG
1991#if FORCED_DEBUG
1992 /*
1993 * Enable redzoning and last user accounting, except for caches with
1994 * large objects, if the increased size would increase the object size
1995 * above the next power of two: caches with object sizes just above a
1996 * power of two have a significant amount of internal fragmentation.
1997 */
1998 if (size < 4096 || fls(size - 1) == fls(size-1 + REDZONE_ALIGN +
1999 2 * sizeof(unsigned long long)))
2000 flags |= SLAB_RED_ZONE | SLAB_STORE_USER;
2001 if (!(flags & SLAB_TYPESAFE_BY_RCU))
2002 flags |= SLAB_POISON;
2003#endif
2004#endif
2005
2006 /*
2007 * Check that size is in terms of words. This is needed to avoid
2008 * unaligned accesses for some archs when redzoning is used, and makes
2009 * sure any on-slab bufctl's are also correctly aligned.
2010 */
2011 size = ALIGN(size, BYTES_PER_WORD);
2012
2013 if (flags & SLAB_RED_ZONE) {
2014 ralign = REDZONE_ALIGN;
2015 /* If redzoning, ensure that the second redzone is suitably
2016 * aligned, by adjusting the object size accordingly. */
2017 size = ALIGN(size, REDZONE_ALIGN);
2018 }
2019
2020 /* 3) caller mandated alignment */
2021 if (ralign < cachep->align) {
2022 ralign = cachep->align;
2023 }
2024 /* disable debug if necessary */
2025 if (ralign > __alignof__(unsigned long long))
2026 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
2027 /*
2028 * 4) Store it.
2029 */
2030 cachep->align = ralign;
2031 cachep->colour_off = cache_line_size();
2032 /* Offset must be a multiple of the alignment. */
2033 if (cachep->colour_off < cachep->align)
2034 cachep->colour_off = cachep->align;
2035
2036 if (slab_is_available())
2037 gfp = GFP_KERNEL;
2038 else
2039 gfp = GFP_NOWAIT;
2040
2041#if DEBUG
2042
2043 /*
2044 * Both debugging options require word-alignment which is calculated
2045 * into align above.
2046 */
2047 if (flags & SLAB_RED_ZONE) {
2048 /* add space for red zone words */
2049 cachep->obj_offset += sizeof(unsigned long long);
2050 size += 2 * sizeof(unsigned long long);
2051 }
2052 if (flags & SLAB_STORE_USER) {
2053 /* user store requires one word storage behind the end of
2054 * the real object. But if the second red zone needs to be
2055 * aligned to 64 bits, we must allow that much space.
2056 */
2057 if (flags & SLAB_RED_ZONE)
2058 size += REDZONE_ALIGN;
2059 else
2060 size += BYTES_PER_WORD;
2061 }
2062#endif
2063
2064 kasan_cache_create(cachep, &size, &flags);
2065
2066 size = ALIGN(size, cachep->align);
2067 /*
2068 * We should restrict the number of objects in a slab to implement
2069 * byte sized index. Refer comment on SLAB_OBJ_MIN_SIZE definition.
2070 */
2071 if (FREELIST_BYTE_INDEX && size < SLAB_OBJ_MIN_SIZE)
2072 size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align);
2073
2074#if DEBUG
2075 /*
2076 * To activate debug pagealloc, off-slab management is necessary
2077 * requirement. In early phase of initialization, small sized slab
2078 * doesn't get initialized so it would not be possible. So, we need
2079 * to check size >= 256. It guarantees that all necessary small
2080 * sized slab is initialized in current slab initialization sequence.
2081 */
2082 if (debug_pagealloc_enabled() && (flags & SLAB_POISON) &&
2083 size >= 256 && cachep->object_size > cache_line_size()) {
2084 if (size < PAGE_SIZE || size % PAGE_SIZE == 0) {
2085 size_t tmp_size = ALIGN(size, PAGE_SIZE);
2086
2087 if (set_off_slab_cache(cachep, tmp_size, flags)) {
2088 flags |= CFLGS_OFF_SLAB;
2089 cachep->obj_offset += tmp_size - size;
2090 size = tmp_size;
2091 goto done;
2092 }
2093 }
2094 }
2095#endif
2096
2097 if (set_objfreelist_slab_cache(cachep, size, flags)) {
2098 flags |= CFLGS_OBJFREELIST_SLAB;
2099 goto done;
2100 }
2101
2102 if (set_off_slab_cache(cachep, size, flags)) {
2103 flags |= CFLGS_OFF_SLAB;
2104 goto done;
2105 }
2106
2107 if (set_on_slab_cache(cachep, size, flags))
2108 goto done;
2109
2110 return -E2BIG;
2111
2112done:
2113 cachep->freelist_size = cachep->num * sizeof(freelist_idx_t);
2114 cachep->flags = flags;
2115 cachep->allocflags = __GFP_COMP;
2116 if (flags & SLAB_CACHE_DMA)
2117 cachep->allocflags |= GFP_DMA;
2118 if (flags & SLAB_RECLAIM_ACCOUNT)
2119 cachep->allocflags |= __GFP_RECLAIMABLE;
2120 cachep->size = size;
2121 cachep->reciprocal_buffer_size = reciprocal_value(size);
2122
2123#if DEBUG
2124 /*
2125 * If we're going to use the generic kernel_map_pages()
2126 * poisoning, then it's going to smash the contents of
2127 * the redzone and userword anyhow, so switch them off.
2128 */
2129 if (IS_ENABLED(CONFIG_PAGE_POISONING) &&
2130 (cachep->flags & SLAB_POISON) &&
2131 is_debug_pagealloc_cache(cachep))
2132 cachep->flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
2133#endif
2134
2135 if (OFF_SLAB(cachep)) {
2136 cachep->freelist_cache =
2137 kmalloc_slab(cachep->freelist_size, 0u);
2138 }
2139
2140 err = setup_cpu_cache(cachep, gfp);
2141 if (err) {
2142 __kmem_cache_release(cachep);
2143 return err;
2144 }
2145
2146 return 0;
2147}
2148
2149#if DEBUG
2150static void check_irq_off(void)
2151{
2152 BUG_ON(!irqs_disabled());
2153}
2154
2155static void check_irq_on(void)
2156{
2157 BUG_ON(irqs_disabled());
2158}
2159
2160static void check_mutex_acquired(void)
2161{
2162 BUG_ON(!mutex_is_locked(&slab_mutex));
2163}
2164
2165static void check_spinlock_acquired(struct kmem_cache *cachep)
2166{
2167#ifdef CONFIG_SMP
2168 check_irq_off();
2169 assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock);
2170#endif
2171}
2172
2173static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
2174{
2175#ifdef CONFIG_SMP
2176 check_irq_off();
2177 assert_spin_locked(&get_node(cachep, node)->list_lock);
2178#endif
2179}
2180
2181#else
2182#define check_irq_off() do { } while(0)
2183#define check_irq_on() do { } while(0)
2184#define check_mutex_acquired() do { } while(0)
2185#define check_spinlock_acquired(x) do { } while(0)
2186#define check_spinlock_acquired_node(x, y) do { } while(0)
2187#endif
2188
2189static void drain_array_locked(struct kmem_cache *cachep, struct array_cache *ac,
2190 int node, bool free_all, struct list_head *list)
2191{
2192 int tofree;
2193
2194 if (!ac || !ac->avail)
2195 return;
2196
2197 tofree = free_all ? ac->avail : (ac->limit + 4) / 5;
2198 if (tofree > ac->avail)
2199 tofree = (ac->avail + 1) / 2;
2200
2201 free_block(cachep, ac->entry, tofree, node, list);
2202 ac->avail -= tofree;
2203 memmove(ac->entry, &(ac->entry[tofree]), sizeof(void *) * ac->avail);
2204}
2205
2206static void do_drain(void *arg)
2207{
2208 struct kmem_cache *cachep = arg;
2209 struct array_cache *ac;
2210 int node = numa_mem_id();
2211 struct kmem_cache_node *n;
2212 LIST_HEAD(list);
2213
2214 check_irq_off();
2215 ac = cpu_cache_get(cachep);
2216 n = get_node(cachep, node);
2217 spin_lock(&n->list_lock);
2218 free_block(cachep, ac->entry, ac->avail, node, &list);
2219 spin_unlock(&n->list_lock);
2220 slabs_destroy(cachep, &list);
2221 ac->avail = 0;
2222}
2223
2224static void drain_cpu_caches(struct kmem_cache *cachep)
2225{
2226 struct kmem_cache_node *n;
2227 int node;
2228 LIST_HEAD(list);
2229
2230 on_each_cpu(do_drain, cachep, 1);
2231 check_irq_on();
2232 for_each_kmem_cache_node(cachep, node, n)
2233 if (n->alien)
2234 drain_alien_cache(cachep, n->alien);
2235
2236 for_each_kmem_cache_node(cachep, node, n) {
2237 spin_lock_irq(&n->list_lock);
2238 drain_array_locked(cachep, n->shared, node, true, &list);
2239 spin_unlock_irq(&n->list_lock);
2240
2241 slabs_destroy(cachep, &list);
2242 }
2243}
2244
2245/*
2246 * Remove slabs from the list of free slabs.
2247 * Specify the number of slabs to drain in tofree.
2248 *
2249 * Returns the actual number of slabs released.
2250 */
2251static int drain_freelist(struct kmem_cache *cache,
2252 struct kmem_cache_node *n, int tofree)
2253{
2254 struct list_head *p;
2255 int nr_freed;
2256 struct page *page;
2257
2258 nr_freed = 0;
2259 while (nr_freed < tofree && !list_empty(&n->slabs_free)) {
2260
2261 spin_lock_irq(&n->list_lock);
2262 p = n->slabs_free.prev;
2263 if (p == &n->slabs_free) {
2264 spin_unlock_irq(&n->list_lock);
2265 goto out;
2266 }
2267
2268 page = list_entry(p, struct page, lru);
2269 list_del(&page->lru);
2270 n->free_slabs--;
2271 n->total_slabs--;
2272 /*
2273 * Safe to drop the lock. The slab is no longer linked
2274 * to the cache.
2275 */
2276 n->free_objects -= cache->num;
2277 spin_unlock_irq(&n->list_lock);
2278 slab_destroy(cache, page);
2279 nr_freed++;
2280 }
2281out:
2282 return nr_freed;
2283}
2284
2285bool __kmem_cache_empty(struct kmem_cache *s)
2286{
2287 int node;
2288 struct kmem_cache_node *n;
2289
2290 for_each_kmem_cache_node(s, node, n)
2291 if (!list_empty(&n->slabs_full) ||
2292 !list_empty(&n->slabs_partial))
2293 return false;
2294 return true;
2295}
2296
2297int __kmem_cache_shrink(struct kmem_cache *cachep)
2298{
2299 int ret = 0;
2300 int node;
2301 struct kmem_cache_node *n;
2302
2303 drain_cpu_caches(cachep);
2304
2305 check_irq_on();
2306 for_each_kmem_cache_node(cachep, node, n) {
2307 drain_freelist(cachep, n, INT_MAX);
2308
2309 ret += !list_empty(&n->slabs_full) ||
2310 !list_empty(&n->slabs_partial);
2311 }
2312 return (ret ? 1 : 0);
2313}
2314
2315#ifdef CONFIG_MEMCG
2316void __kmemcg_cache_deactivate(struct kmem_cache *cachep)
2317{
2318 __kmem_cache_shrink(cachep);
2319}
2320#endif
2321
2322int __kmem_cache_shutdown(struct kmem_cache *cachep)
2323{
2324 return __kmem_cache_shrink(cachep);
2325}
2326
2327void __kmem_cache_release(struct kmem_cache *cachep)
2328{
2329 int i;
2330 struct kmem_cache_node *n;
2331
2332 cache_random_seq_destroy(cachep);
2333
2334 free_percpu(cachep->cpu_cache);
2335
2336 /* NUMA: free the node structures */
2337 for_each_kmem_cache_node(cachep, i, n) {
2338 kfree(n->shared);
2339 free_alien_cache(n->alien);
2340 kfree(n);
2341 cachep->node[i] = NULL;
2342 }
2343}
2344
2345/*
2346 * Get the memory for a slab management obj.
2347 *
2348 * For a slab cache when the slab descriptor is off-slab, the
2349 * slab descriptor can't come from the same cache which is being created,
2350 * Because if it is the case, that means we defer the creation of
2351 * the kmalloc_{dma,}_cache of size sizeof(slab descriptor) to this point.
2352 * And we eventually call down to __kmem_cache_create(), which
2353 * in turn looks up in the kmalloc_{dma,}_caches for the disired-size one.
2354 * This is a "chicken-and-egg" problem.
2355 *
2356 * So the off-slab slab descriptor shall come from the kmalloc_{dma,}_caches,
2357 * which are all initialized during kmem_cache_init().
2358 */
2359static void *alloc_slabmgmt(struct kmem_cache *cachep,
2360 struct page *page, int colour_off,
2361 gfp_t local_flags, int nodeid)
2362{
2363 void *freelist;
2364 void *addr = page_address(page);
2365
2366 page->s_mem = addr + colour_off;
2367 page->active = 0;
2368
2369 if (OBJFREELIST_SLAB(cachep))
2370 freelist = NULL;
2371 else if (OFF_SLAB(cachep)) {
2372 /* Slab management obj is off-slab. */
2373 freelist = kmem_cache_alloc_node(cachep->freelist_cache,
2374 local_flags, nodeid);
2375 freelist = kasan_reset_tag(freelist);
2376 if (!freelist)
2377 return NULL;
2378 } else {
2379 /* We will use last bytes at the slab for freelist */
2380 freelist = addr + (PAGE_SIZE << cachep->gfporder) -
2381 cachep->freelist_size;
2382 }
2383
2384 return freelist;
2385}
2386
2387static inline freelist_idx_t get_free_obj(struct page *page, unsigned int idx)
2388{
2389 return ((freelist_idx_t *)page->freelist)[idx];
2390}
2391
2392static inline void set_free_obj(struct page *page,
2393 unsigned int idx, freelist_idx_t val)
2394{
2395 ((freelist_idx_t *)(page->freelist))[idx] = val;
2396}
2397
2398static void cache_init_objs_debug(struct kmem_cache *cachep, struct page *page)
2399{
2400#if DEBUG
2401 int i;
2402
2403 for (i = 0; i < cachep->num; i++) {
2404 void *objp = index_to_obj(cachep, page, i);
2405
2406 if (cachep->flags & SLAB_STORE_USER)
2407 *dbg_userword(cachep, objp) = NULL;
2408
2409 if (cachep->flags & SLAB_RED_ZONE) {
2410 *dbg_redzone1(cachep, objp) = RED_INACTIVE;
2411 *dbg_redzone2(cachep, objp) = RED_INACTIVE;
2412 }
2413 /*
2414 * Constructors are not allowed to allocate memory from the same
2415 * cache which they are a constructor for. Otherwise, deadlock.
2416 * They must also be threaded.
2417 */
2418 if (cachep->ctor && !(cachep->flags & SLAB_POISON)) {
2419 kasan_unpoison_object_data(cachep,
2420 objp + obj_offset(cachep));
2421 cachep->ctor(objp + obj_offset(cachep));
2422 kasan_poison_object_data(
2423 cachep, objp + obj_offset(cachep));
2424 }
2425
2426 if (cachep->flags & SLAB_RED_ZONE) {
2427 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2428 slab_error(cachep, "constructor overwrote the end of an object");
2429 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2430 slab_error(cachep, "constructor overwrote the start of an object");
2431 }
2432 /* need to poison the objs? */
2433 if (cachep->flags & SLAB_POISON) {
2434 poison_obj(cachep, objp, POISON_FREE);
2435 slab_kernel_map(cachep, objp, 0, 0);
2436 }
2437 }
2438#endif
2439}
2440
2441#ifdef CONFIG_SLAB_FREELIST_RANDOM
2442/* Hold information during a freelist initialization */
2443union freelist_init_state {
2444 struct {
2445 unsigned int pos;
2446 unsigned int *list;
2447 unsigned int count;
2448 };
2449 struct rnd_state rnd_state;
2450};
2451
2452/*
2453 * Initialize the state based on the randomization methode available.
2454 * return true if the pre-computed list is available, false otherwize.
2455 */
2456static bool freelist_state_initialize(union freelist_init_state *state,
2457 struct kmem_cache *cachep,
2458 unsigned int count)
2459{
2460 bool ret;
2461 unsigned int rand;
2462
2463 /* Use best entropy available to define a random shift */
2464 rand = get_random_int();
2465
2466 /* Use a random state if the pre-computed list is not available */
2467 if (!cachep->random_seq) {
2468 prandom_seed_state(&state->rnd_state, rand);
2469 ret = false;
2470 } else {
2471 state->list = cachep->random_seq;
2472 state->count = count;
2473 state->pos = rand % count;
2474 ret = true;
2475 }
2476 return ret;
2477}
2478
2479/* Get the next entry on the list and randomize it using a random shift */
2480static freelist_idx_t next_random_slot(union freelist_init_state *state)
2481{
2482 if (state->pos >= state->count)
2483 state->pos = 0;
2484 return state->list[state->pos++];
2485}
2486
2487/* Swap two freelist entries */
2488static void swap_free_obj(struct page *page, unsigned int a, unsigned int b)
2489{
2490 swap(((freelist_idx_t *)page->freelist)[a],
2491 ((freelist_idx_t *)page->freelist)[b]);
2492}
2493
2494/*
2495 * Shuffle the freelist initialization state based on pre-computed lists.
2496 * return true if the list was successfully shuffled, false otherwise.
2497 */
2498static bool shuffle_freelist(struct kmem_cache *cachep, struct page *page)
2499{
2500 unsigned int objfreelist = 0, i, rand, count = cachep->num;
2501 union freelist_init_state state;
2502 bool precomputed;
2503
2504 if (count < 2)
2505 return false;
2506
2507 precomputed = freelist_state_initialize(&state, cachep, count);
2508
2509 /* Take a random entry as the objfreelist */
2510 if (OBJFREELIST_SLAB(cachep)) {
2511 if (!precomputed)
2512 objfreelist = count - 1;
2513 else
2514 objfreelist = next_random_slot(&state);
2515 page->freelist = index_to_obj(cachep, page, objfreelist) +
2516 obj_offset(cachep);
2517 count--;
2518 }
2519
2520 /*
2521 * On early boot, generate the list dynamically.
2522 * Later use a pre-computed list for speed.
2523 */
2524 if (!precomputed) {
2525 for (i = 0; i < count; i++)
2526 set_free_obj(page, i, i);
2527
2528 /* Fisher-Yates shuffle */
2529 for (i = count - 1; i > 0; i--) {
2530 rand = prandom_u32_state(&state.rnd_state);
2531 rand %= (i + 1);
2532 swap_free_obj(page, i, rand);
2533 }
2534 } else {
2535 for (i = 0; i < count; i++)
2536 set_free_obj(page, i, next_random_slot(&state));
2537 }
2538
2539 if (OBJFREELIST_SLAB(cachep))
2540 set_free_obj(page, cachep->num - 1, objfreelist);
2541
2542 return true;
2543}
2544#else
2545static inline bool shuffle_freelist(struct kmem_cache *cachep,
2546 struct page *page)
2547{
2548 return false;
2549}
2550#endif /* CONFIG_SLAB_FREELIST_RANDOM */
2551
2552static void cache_init_objs(struct kmem_cache *cachep,
2553 struct page *page)
2554{
2555 int i;
2556 void *objp;
2557 bool shuffled;
2558
2559 cache_init_objs_debug(cachep, page);
2560
2561 /* Try to randomize the freelist if enabled */
2562 shuffled = shuffle_freelist(cachep, page);
2563
2564 if (!shuffled && OBJFREELIST_SLAB(cachep)) {
2565 page->freelist = index_to_obj(cachep, page, cachep->num - 1) +
2566 obj_offset(cachep);
2567 }
2568
2569 for (i = 0; i < cachep->num; i++) {
2570 objp = index_to_obj(cachep, page, i);
2571 objp = kasan_init_slab_obj(cachep, objp);
2572
2573 /* constructor could break poison info */
2574 if (DEBUG == 0 && cachep->ctor) {
2575 kasan_unpoison_object_data(cachep, objp);
2576 cachep->ctor(objp);
2577 kasan_poison_object_data(cachep, objp);
2578 }
2579
2580 if (!shuffled)
2581 set_free_obj(page, i, i);
2582 }
2583}
2584
2585static void *slab_get_obj(struct kmem_cache *cachep, struct page *page)
2586{
2587 void *objp;
2588
2589 objp = index_to_obj(cachep, page, get_free_obj(page, page->active));
2590 page->active++;
2591
2592#if DEBUG
2593 if (cachep->flags & SLAB_STORE_USER)
2594 set_store_user_dirty(cachep);
2595#endif
2596
2597 return objp;
2598}
2599
2600static void slab_put_obj(struct kmem_cache *cachep,
2601 struct page *page, void *objp)
2602{
2603 unsigned int objnr = obj_to_index(cachep, page, objp);
2604#if DEBUG
2605 unsigned int i;
2606
2607 /* Verify double free bug */
2608 for (i = page->active; i < cachep->num; i++) {
2609 if (get_free_obj(page, i) == objnr) {
2610 pr_err("slab: double free detected in cache '%s', objp %px\n",
2611 cachep->name, objp);
2612 BUG();
2613 }
2614 }
2615#endif
2616 page->active--;
2617 if (!page->freelist)
2618 page->freelist = objp + obj_offset(cachep);
2619
2620 set_free_obj(page, page->active, objnr);
2621}
2622
2623/*
2624 * Map pages beginning at addr to the given cache and slab. This is required
2625 * for the slab allocator to be able to lookup the cache and slab of a
2626 * virtual address for kfree, ksize, and slab debugging.
2627 */
2628static void slab_map_pages(struct kmem_cache *cache, struct page *page,
2629 void *freelist)
2630{
2631 page->slab_cache = cache;
2632 page->freelist = freelist;
2633}
2634
2635/*
2636 * Grow (by 1) the number of slabs within a cache. This is called by
2637 * kmem_cache_alloc() when there are no active objs left in a cache.
2638 */
2639static struct page *cache_grow_begin(struct kmem_cache *cachep,
2640 gfp_t flags, int nodeid)
2641{
2642 void *freelist;
2643 size_t offset;
2644 gfp_t local_flags;
2645 int page_node;
2646 struct kmem_cache_node *n;
2647 struct page *page;
2648
2649 /*
2650 * Be lazy and only check for valid flags here, keeping it out of the
2651 * critical path in kmem_cache_alloc().
2652 */
2653 if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
2654 gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
2655 flags &= ~GFP_SLAB_BUG_MASK;
2656 pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
2657 invalid_mask, &invalid_mask, flags, &flags);
2658 dump_stack();
2659 }
2660 WARN_ON_ONCE(cachep->ctor && (flags & __GFP_ZERO));
2661 local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
2662
2663 check_irq_off();
2664 if (gfpflags_allow_blocking(local_flags))
2665 local_irq_enable();
2666
2667 /*
2668 * Get mem for the objs. Attempt to allocate a physical page from
2669 * 'nodeid'.
2670 */
2671 page = kmem_getpages(cachep, local_flags, nodeid);
2672 if (!page)
2673 goto failed;
2674
2675 page_node = page_to_nid(page);
2676 n = get_node(cachep, page_node);
2677
2678 /* Get colour for the slab, and cal the next value. */
2679 n->colour_next++;
2680 if (n->colour_next >= cachep->colour)
2681 n->colour_next = 0;
2682
2683 offset = n->colour_next;
2684 if (offset >= cachep->colour)
2685 offset = 0;
2686
2687 offset *= cachep->colour_off;
2688
2689 /*
2690 * Call kasan_poison_slab() before calling alloc_slabmgmt(), so
2691 * page_address() in the latter returns a non-tagged pointer,
2692 * as it should be for slab pages.
2693 */
2694 kasan_poison_slab(page);
2695
2696 /* Get slab management. */
2697 freelist = alloc_slabmgmt(cachep, page, offset,
2698 local_flags & ~GFP_CONSTRAINT_MASK, page_node);
2699 if (OFF_SLAB(cachep) && !freelist)
2700 goto opps1;
2701
2702 slab_map_pages(cachep, page, freelist);
2703
2704 cache_init_objs(cachep, page);
2705
2706 if (gfpflags_allow_blocking(local_flags))
2707 local_irq_disable();
2708
2709 return page;
2710
2711opps1:
2712 kmem_freepages(cachep, page);
2713failed:
2714 if (gfpflags_allow_blocking(local_flags))
2715 local_irq_disable();
2716 return NULL;
2717}
2718
2719static void cache_grow_end(struct kmem_cache *cachep, struct page *page)
2720{
2721 struct kmem_cache_node *n;
2722 void *list = NULL;
2723
2724 check_irq_off();
2725
2726 if (!page)
2727 return;
2728
2729 INIT_LIST_HEAD(&page->lru);
2730 n = get_node(cachep, page_to_nid(page));
2731
2732 spin_lock(&n->list_lock);
2733 n->total_slabs++;
2734 if (!page->active) {
2735 list_add_tail(&page->lru, &(n->slabs_free));
2736 n->free_slabs++;
2737 } else
2738 fixup_slab_list(cachep, n, page, &list);
2739
2740 STATS_INC_GROWN(cachep);
2741 n->free_objects += cachep->num - page->active;
2742 spin_unlock(&n->list_lock);
2743
2744 fixup_objfreelist_debug(cachep, &list);
2745}
2746
2747#if DEBUG
2748
2749/*
2750 * Perform extra freeing checks:
2751 * - detect bad pointers.
2752 * - POISON/RED_ZONE checking
2753 */
2754static void kfree_debugcheck(const void *objp)
2755{
2756 if (!virt_addr_valid(objp)) {
2757 pr_err("kfree_debugcheck: out of range ptr %lxh\n",
2758 (unsigned long)objp);
2759 BUG();
2760 }
2761}
2762
2763static inline void verify_redzone_free(struct kmem_cache *cache, void *obj)
2764{
2765 unsigned long long redzone1, redzone2;
2766
2767 redzone1 = *dbg_redzone1(cache, obj);
2768 redzone2 = *dbg_redzone2(cache, obj);
2769
2770 /*
2771 * Redzone is ok.
2772 */
2773 if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
2774 return;
2775
2776 if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
2777 slab_error(cache, "double free detected");
2778 else
2779 slab_error(cache, "memory outside object was overwritten");
2780
2781 pr_err("%px: redzone 1:0x%llx, redzone 2:0x%llx\n",
2782 obj, redzone1, redzone2);
2783}
2784
2785static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
2786 unsigned long caller)
2787{
2788 unsigned int objnr;
2789 struct page *page;
2790
2791 BUG_ON(virt_to_cache(objp) != cachep);
2792
2793 objp -= obj_offset(cachep);
2794 kfree_debugcheck(objp);
2795 page = virt_to_head_page(objp);
2796
2797 if (cachep->flags & SLAB_RED_ZONE) {
2798 verify_redzone_free(cachep, objp);
2799 *dbg_redzone1(cachep, objp) = RED_INACTIVE;
2800 *dbg_redzone2(cachep, objp) = RED_INACTIVE;
2801 }
2802 if (cachep->flags & SLAB_STORE_USER) {
2803 set_store_user_dirty(cachep);
2804 *dbg_userword(cachep, objp) = (void *)caller;
2805 }
2806
2807 objnr = obj_to_index(cachep, page, objp);
2808
2809 BUG_ON(objnr >= cachep->num);
2810 BUG_ON(objp != index_to_obj(cachep, page, objnr));
2811
2812 if (cachep->flags & SLAB_POISON) {
2813 poison_obj(cachep, objp, POISON_FREE);
2814 slab_kernel_map(cachep, objp, 0, caller);
2815 }
2816 return objp;
2817}
2818
2819#else
2820#define kfree_debugcheck(x) do { } while(0)
2821#define cache_free_debugcheck(x,objp,z) (objp)
2822#endif
2823
2824static inline void fixup_objfreelist_debug(struct kmem_cache *cachep,
2825 void **list)
2826{
2827#if DEBUG
2828 void *next = *list;
2829 void *objp;
2830
2831 while (next) {
2832 objp = next - obj_offset(cachep);
2833 next = *(void **)next;
2834 poison_obj(cachep, objp, POISON_FREE);
2835 }
2836#endif
2837}
2838
2839static inline void fixup_slab_list(struct kmem_cache *cachep,
2840 struct kmem_cache_node *n, struct page *page,
2841 void **list)
2842{
2843 /* move slabp to correct slabp list: */
2844 list_del(&page->lru);
2845 if (page->active == cachep->num) {
2846 list_add(&page->lru, &n->slabs_full);
2847 if (OBJFREELIST_SLAB(cachep)) {
2848#if DEBUG
2849 /* Poisoning will be done without holding the lock */
2850 if (cachep->flags & SLAB_POISON) {
2851 void **objp = page->freelist;
2852
2853 *objp = *list;
2854 *list = objp;
2855 }
2856#endif
2857 page->freelist = NULL;
2858 }
2859 } else
2860 list_add(&page->lru, &n->slabs_partial);
2861}
2862
2863/* Try to find non-pfmemalloc slab if needed */
2864static noinline struct page *get_valid_first_slab(struct kmem_cache_node *n,
2865 struct page *page, bool pfmemalloc)
2866{
2867 if (!page)
2868 return NULL;
2869
2870 if (pfmemalloc)
2871 return page;
2872
2873 if (!PageSlabPfmemalloc(page))
2874 return page;
2875
2876 /* No need to keep pfmemalloc slab if we have enough free objects */
2877 if (n->free_objects > n->free_limit) {
2878 ClearPageSlabPfmemalloc(page);
2879 return page;
2880 }
2881
2882 /* Move pfmemalloc slab to the end of list to speed up next search */
2883 list_del(&page->lru);
2884 if (!page->active) {
2885 list_add_tail(&page->lru, &n->slabs_free);
2886 n->free_slabs++;
2887 } else
2888 list_add_tail(&page->lru, &n->slabs_partial);
2889
2890 list_for_each_entry(page, &n->slabs_partial, lru) {
2891 if (!PageSlabPfmemalloc(page))
2892 return page;
2893 }
2894
2895 n->free_touched = 1;
2896 list_for_each_entry(page, &n->slabs_free, lru) {
2897 if (!PageSlabPfmemalloc(page)) {
2898 n->free_slabs--;
2899 return page;
2900 }
2901 }
2902
2903 return NULL;
2904}
2905
2906static struct page *get_first_slab(struct kmem_cache_node *n, bool pfmemalloc)
2907{
2908 struct page *page;
2909
2910 assert_spin_locked(&n->list_lock);
2911 page = list_first_entry_or_null(&n->slabs_partial, struct page, lru);
2912 if (!page) {
2913 n->free_touched = 1;
2914 page = list_first_entry_or_null(&n->slabs_free, struct page,
2915 lru);
2916 if (page)
2917 n->free_slabs--;
2918 }
2919
2920 if (sk_memalloc_socks())
2921 page = get_valid_first_slab(n, page, pfmemalloc);
2922
2923 return page;
2924}
2925
2926static noinline void *cache_alloc_pfmemalloc(struct kmem_cache *cachep,
2927 struct kmem_cache_node *n, gfp_t flags)
2928{
2929 struct page *page;
2930 void *obj;
2931 void *list = NULL;
2932
2933 if (!gfp_pfmemalloc_allowed(flags))
2934 return NULL;
2935
2936 spin_lock(&n->list_lock);
2937 page = get_first_slab(n, true);
2938 if (!page) {
2939 spin_unlock(&n->list_lock);
2940 return NULL;
2941 }
2942
2943 obj = slab_get_obj(cachep, page);
2944 n->free_objects--;
2945
2946 fixup_slab_list(cachep, n, page, &list);
2947
2948 spin_unlock(&n->list_lock);
2949 fixup_objfreelist_debug(cachep, &list);
2950
2951 return obj;
2952}
2953
2954/*
2955 * Slab list should be fixed up by fixup_slab_list() for existing slab
2956 * or cache_grow_end() for new slab
2957 */
2958static __always_inline int alloc_block(struct kmem_cache *cachep,
2959 struct array_cache *ac, struct page *page, int batchcount)
2960{
2961 /*
2962 * There must be at least one object available for
2963 * allocation.
2964 */
2965 BUG_ON(page->active >= cachep->num);
2966
2967 while (page->active < cachep->num && batchcount--) {
2968 STATS_INC_ALLOCED(cachep);
2969 STATS_INC_ACTIVE(cachep);
2970 STATS_SET_HIGH(cachep);
2971
2972 ac->entry[ac->avail++] = slab_get_obj(cachep, page);
2973 }
2974
2975 return batchcount;
2976}
2977
2978static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)
2979{
2980 int batchcount;
2981 struct kmem_cache_node *n;
2982 struct array_cache *ac, *shared;
2983 int node;
2984 void *list = NULL;
2985 struct page *page;
2986
2987 check_irq_off();
2988 node = numa_mem_id();
2989
2990 ac = cpu_cache_get(cachep);
2991 batchcount = ac->batchcount;
2992 if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
2993 /*
2994 * If there was little recent activity on this cache, then
2995 * perform only a partial refill. Otherwise we could generate
2996 * refill bouncing.
2997 */
2998 batchcount = BATCHREFILL_LIMIT;
2999 }
3000 n = get_node(cachep, node);
3001
3002 BUG_ON(ac->avail > 0 || !n);
3003 shared = READ_ONCE(n->shared);
3004 if (!n->free_objects && (!shared || !shared->avail))
3005 goto direct_grow;
3006
3007 spin_lock(&n->list_lock);
3008 shared = READ_ONCE(n->shared);
3009
3010 /* See if we can refill from the shared array */
3011 if (shared && transfer_objects(ac, shared, batchcount)) {
3012 shared->touched = 1;
3013 goto alloc_done;
3014 }
3015
3016 while (batchcount > 0) {
3017 /* Get slab alloc is to come from. */
3018 page = get_first_slab(n, false);
3019 if (!page)
3020 goto must_grow;
3021
3022 check_spinlock_acquired(cachep);
3023
3024 batchcount = alloc_block(cachep, ac, page, batchcount);
3025 fixup_slab_list(cachep, n, page, &list);
3026 }
3027
3028must_grow:
3029 n->free_objects -= ac->avail;
3030alloc_done:
3031 spin_unlock(&n->list_lock);
3032 fixup_objfreelist_debug(cachep, &list);
3033
3034direct_grow:
3035 if (unlikely(!ac->avail)) {
3036 /* Check if we can use obj in pfmemalloc slab */
3037 if (sk_memalloc_socks()) {
3038 void *obj = cache_alloc_pfmemalloc(cachep, n, flags);
3039
3040 if (obj)
3041 return obj;
3042 }
3043
3044 page = cache_grow_begin(cachep, gfp_exact_node(flags), node);
3045
3046 /*
3047 * cache_grow_begin() can reenable interrupts,
3048 * then ac could change.
3049 */
3050 ac = cpu_cache_get(cachep);
3051 if (!ac->avail && page)
3052 alloc_block(cachep, ac, page, batchcount);
3053 cache_grow_end(cachep, page);
3054
3055 if (!ac->avail)
3056 return NULL;
3057 }
3058 ac->touched = 1;
3059
3060 return ac->entry[--ac->avail];
3061}
3062
3063static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
3064 gfp_t flags)
3065{
3066 might_sleep_if(gfpflags_allow_blocking(flags));
3067}
3068
3069#if DEBUG
3070static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
3071 gfp_t flags, void *objp, unsigned long caller)
3072{
3073 WARN_ON_ONCE(cachep->ctor && (flags & __GFP_ZERO));
3074 if (!objp)
3075 return objp;
3076 if (cachep->flags & SLAB_POISON) {
3077 check_poison_obj(cachep, objp);
3078 slab_kernel_map(cachep, objp, 1, 0);
3079 poison_obj(cachep, objp, POISON_INUSE);
3080 }
3081 if (cachep->flags & SLAB_STORE_USER)
3082 *dbg_userword(cachep, objp) = (void *)caller;
3083
3084 if (cachep->flags & SLAB_RED_ZONE) {
3085 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
3086 *dbg_redzone2(cachep, objp) != RED_INACTIVE) {
3087 slab_error(cachep, "double free, or memory outside object was overwritten");
3088 pr_err("%px: redzone 1:0x%llx, redzone 2:0x%llx\n",
3089 objp, *dbg_redzone1(cachep, objp),
3090 *dbg_redzone2(cachep, objp));
3091 }
3092 *dbg_redzone1(cachep, objp) = RED_ACTIVE;
3093 *dbg_redzone2(cachep, objp) = RED_ACTIVE;
3094 }
3095
3096 objp += obj_offset(cachep);
3097 if (cachep->ctor && cachep->flags & SLAB_POISON)
3098 cachep->ctor(objp);
3099 if (ARCH_SLAB_MINALIGN &&
3100 ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
3101 pr_err("0x%px: not aligned to ARCH_SLAB_MINALIGN=%d\n",
3102 objp, (int)ARCH_SLAB_MINALIGN);
3103 }
3104 return objp;
3105}
3106#else
3107#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
3108#endif
3109
3110static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3111{
3112 void *objp;
3113 struct array_cache *ac;
3114
3115 check_irq_off();
3116
3117 ac = cpu_cache_get(cachep);
3118 if (likely(ac->avail)) {
3119 ac->touched = 1;
3120 objp = ac->entry[--ac->avail];
3121
3122 STATS_INC_ALLOCHIT(cachep);
3123 goto out;
3124 }
3125
3126 STATS_INC_ALLOCMISS(cachep);
3127 objp = cache_alloc_refill(cachep, flags);
3128 /*
3129 * the 'ac' may be updated by cache_alloc_refill(),
3130 * and kmemleak_erase() requires its correct value.
3131 */
3132 ac = cpu_cache_get(cachep);
3133
3134out:
3135 /*
3136 * To avoid a false negative, if an object that is in one of the
3137 * per-CPU caches is leaked, we need to make sure kmemleak doesn't
3138 * treat the array pointers as a reference to the object.
3139 */
3140 if (objp)
3141 kmemleak_erase(&ac->entry[ac->avail]);
3142 return objp;
3143}
3144
3145#ifdef CONFIG_NUMA
3146/*
3147 * Try allocating on another node if PFA_SPREAD_SLAB is a mempolicy is set.
3148 *
3149 * If we are in_interrupt, then process context, including cpusets and
3150 * mempolicy, may not apply and should not be used for allocation policy.
3151 */
3152static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
3153{
3154 int nid_alloc, nid_here;
3155
3156 if (in_interrupt() || (flags & __GFP_THISNODE))
3157 return NULL;
3158 nid_alloc = nid_here = numa_mem_id();
3159 if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
3160 nid_alloc = cpuset_slab_spread_node();
3161 else if (current->mempolicy)
3162 nid_alloc = mempolicy_slab_node();
3163 if (nid_alloc != nid_here)
3164 return ____cache_alloc_node(cachep, flags, nid_alloc);
3165 return NULL;
3166}
3167
3168/*
3169 * Fallback function if there was no memory available and no objects on a
3170 * certain node and fall back is permitted. First we scan all the
3171 * available node for available objects. If that fails then we
3172 * perform an allocation without specifying a node. This allows the page
3173 * allocator to do its reclaim / fallback magic. We then insert the
3174 * slab into the proper nodelist and then allocate from it.
3175 */
3176static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
3177{
3178 struct zonelist *zonelist;
3179 struct zoneref *z;
3180 struct zone *zone;
3181 enum zone_type high_zoneidx = gfp_zone(flags);
3182 void *obj = NULL;
3183 struct page *page;
3184 int nid;
3185 unsigned int cpuset_mems_cookie;
3186
3187 if (flags & __GFP_THISNODE)
3188 return NULL;
3189
3190retry_cpuset:
3191 cpuset_mems_cookie = read_mems_allowed_begin();
3192 zonelist = node_zonelist(mempolicy_slab_node(), flags);
3193
3194retry:
3195 /*
3196 * Look through allowed nodes for objects available
3197 * from existing per node queues.
3198 */
3199 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
3200 nid = zone_to_nid(zone);
3201
3202 if (cpuset_zone_allowed(zone, flags) &&
3203 get_node(cache, nid) &&
3204 get_node(cache, nid)->free_objects) {
3205 obj = ____cache_alloc_node(cache,
3206 gfp_exact_node(flags), nid);
3207 if (obj)
3208 break;
3209 }
3210 }
3211
3212 if (!obj) {
3213 /*
3214 * This allocation will be performed within the constraints
3215 * of the current cpuset / memory policy requirements.
3216 * We may trigger various forms of reclaim on the allowed
3217 * set and go into memory reserves if necessary.
3218 */
3219 page = cache_grow_begin(cache, flags, numa_mem_id());
3220 cache_grow_end(cache, page);
3221 if (page) {
3222 nid = page_to_nid(page);
3223 obj = ____cache_alloc_node(cache,
3224 gfp_exact_node(flags), nid);
3225
3226 /*
3227 * Another processor may allocate the objects in
3228 * the slab since we are not holding any locks.
3229 */
3230 if (!obj)
3231 goto retry;
3232 }
3233 }
3234
3235 if (unlikely(!obj && read_mems_allowed_retry(cpuset_mems_cookie)))
3236 goto retry_cpuset;
3237 return obj;
3238}
3239
3240/*
3241 * A interface to enable slab creation on nodeid
3242 */
3243static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
3244 int nodeid)
3245{
3246 struct page *page;
3247 struct kmem_cache_node *n;
3248 void *obj = NULL;
3249 void *list = NULL;
3250
3251 VM_BUG_ON(nodeid < 0 || nodeid >= MAX_NUMNODES);
3252 n = get_node(cachep, nodeid);
3253 BUG_ON(!n);
3254
3255 check_irq_off();
3256 spin_lock(&n->list_lock);
3257 page = get_first_slab(n, false);
3258 if (!page)
3259 goto must_grow;
3260
3261 check_spinlock_acquired_node(cachep, nodeid);
3262
3263 STATS_INC_NODEALLOCS(cachep);
3264 STATS_INC_ACTIVE(cachep);
3265 STATS_SET_HIGH(cachep);
3266
3267 BUG_ON(page->active == cachep->num);
3268
3269 obj = slab_get_obj(cachep, page);
3270 n->free_objects--;
3271
3272 fixup_slab_list(cachep, n, page, &list);
3273
3274 spin_unlock(&n->list_lock);
3275 fixup_objfreelist_debug(cachep, &list);
3276 return obj;
3277
3278must_grow:
3279 spin_unlock(&n->list_lock);
3280 page = cache_grow_begin(cachep, gfp_exact_node(flags), nodeid);
3281 if (page) {
3282 /* This slab isn't counted yet so don't update free_objects */
3283 obj = slab_get_obj(cachep, page);
3284 }
3285 cache_grow_end(cachep, page);
3286
3287 return obj ? obj : fallback_alloc(cachep, flags);
3288}
3289
3290static __always_inline void *
3291slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
3292 unsigned long caller)
3293{
3294 unsigned long save_flags;
3295 void *ptr;
3296 int slab_node = numa_mem_id();
3297
3298 flags &= gfp_allowed_mask;
3299 cachep = slab_pre_alloc_hook(cachep, flags);
3300 if (unlikely(!cachep))
3301 return NULL;
3302
3303 cache_alloc_debugcheck_before(cachep, flags);
3304 local_irq_save(save_flags);
3305
3306 if (nodeid == NUMA_NO_NODE)
3307 nodeid = slab_node;
3308
3309 if (unlikely(!get_node(cachep, nodeid))) {
3310 /* Node not bootstrapped yet */
3311 ptr = fallback_alloc(cachep, flags);
3312 goto out;
3313 }
3314
3315 if (nodeid == slab_node) {
3316 /*
3317 * Use the locally cached objects if possible.
3318 * However ____cache_alloc does not allow fallback
3319 * to other nodes. It may fail while we still have
3320 * objects on other nodes available.
3321 */
3322 ptr = ____cache_alloc(cachep, flags);
3323 if (ptr)
3324 goto out;
3325 }
3326 /* ___cache_alloc_node can fall back to other nodes */
3327 ptr = ____cache_alloc_node(cachep, flags, nodeid);
3328 out:
3329 local_irq_restore(save_flags);
3330 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
3331
3332 if (unlikely(flags & __GFP_ZERO) && ptr)
3333 memset(ptr, 0, cachep->object_size);
3334
3335 slab_post_alloc_hook(cachep, flags, 1, &ptr);
3336 return ptr;
3337}
3338
3339static __always_inline void *
3340__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
3341{
3342 void *objp;
3343
3344 if (current->mempolicy || cpuset_do_slab_mem_spread()) {
3345 objp = alternate_node_alloc(cache, flags);
3346 if (objp)
3347 goto out;
3348 }
3349 objp = ____cache_alloc(cache, flags);
3350
3351 /*
3352 * We may just have run out of memory on the local node.
3353 * ____cache_alloc_node() knows how to locate memory on other nodes
3354 */
3355 if (!objp)
3356 objp = ____cache_alloc_node(cache, flags, numa_mem_id());
3357
3358 out:
3359 return objp;
3360}
3361#else
3362
3363static __always_inline void *
3364__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3365{
3366 return ____cache_alloc(cachep, flags);
3367}
3368
3369#endif /* CONFIG_NUMA */
3370
3371static __always_inline void *
3372slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
3373{
3374 unsigned long save_flags;
3375 void *objp;
3376
3377 flags &= gfp_allowed_mask;
3378 cachep = slab_pre_alloc_hook(cachep, flags);
3379 if (unlikely(!cachep))
3380 return NULL;
3381
3382 cache_alloc_debugcheck_before(cachep, flags);
3383 local_irq_save(save_flags);
3384 objp = __do_cache_alloc(cachep, flags);
3385 local_irq_restore(save_flags);
3386 objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
3387 prefetchw(objp);
3388
3389 if (unlikely(flags & __GFP_ZERO) && objp)
3390 memset(objp, 0, cachep->object_size);
3391
3392 slab_post_alloc_hook(cachep, flags, 1, &objp);
3393 return objp;
3394}
3395
3396/*
3397 * Caller needs to acquire correct kmem_cache_node's list_lock
3398 * @list: List of detached free slabs should be freed by caller
3399 */
3400static void free_block(struct kmem_cache *cachep, void **objpp,
3401 int nr_objects, int node, struct list_head *list)
3402{
3403 int i;
3404 struct kmem_cache_node *n = get_node(cachep, node);
3405 struct page *page;
3406
3407 n->free_objects += nr_objects;
3408
3409 for (i = 0; i < nr_objects; i++) {
3410 void *objp;
3411 struct page *page;
3412
3413 objp = objpp[i];
3414
3415 page = virt_to_head_page(objp);
3416 list_del(&page->lru);
3417 check_spinlock_acquired_node(cachep, node);
3418 slab_put_obj(cachep, page, objp);
3419 STATS_DEC_ACTIVE(cachep);
3420
3421 /* fixup slab chains */
3422 if (page->active == 0) {
3423 list_add(&page->lru, &n->slabs_free);
3424 n->free_slabs++;
3425 } else {
3426 /* Unconditionally move a slab to the end of the
3427 * partial list on free - maximum time for the
3428 * other objects to be freed, too.
3429 */
3430 list_add_tail(&page->lru, &n->slabs_partial);
3431 }
3432 }
3433
3434 while (n->free_objects > n->free_limit && !list_empty(&n->slabs_free)) {
3435 n->free_objects -= cachep->num;
3436
3437 page = list_last_entry(&n->slabs_free, struct page, lru);
3438 list_move(&page->lru, list);
3439 n->free_slabs--;
3440 n->total_slabs--;
3441 }
3442}
3443
3444static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
3445{
3446 int batchcount;
3447 struct kmem_cache_node *n;
3448 int node = numa_mem_id();
3449 LIST_HEAD(list);
3450
3451 batchcount = ac->batchcount;
3452
3453 check_irq_off();
3454 n = get_node(cachep, node);
3455 spin_lock(&n->list_lock);
3456 if (n->shared) {
3457 struct array_cache *shared_array = n->shared;
3458 int max = shared_array->limit - shared_array->avail;
3459 if (max) {
3460 if (batchcount > max)
3461 batchcount = max;
3462 memcpy(&(shared_array->entry[shared_array->avail]),
3463 ac->entry, sizeof(void *) * batchcount);
3464 shared_array->avail += batchcount;
3465 goto free_done;
3466 }
3467 }
3468
3469 free_block(cachep, ac->entry, batchcount, node, &list);
3470free_done:
3471#if STATS
3472 {
3473 int i = 0;
3474 struct page *page;
3475
3476 list_for_each_entry(page, &n->slabs_free, lru) {
3477 BUG_ON(page->active);
3478
3479 i++;
3480 }
3481 STATS_SET_FREEABLE(cachep, i);
3482 }
3483#endif
3484 spin_unlock(&n->list_lock);
3485 slabs_destroy(cachep, &list);
3486 ac->avail -= batchcount;
3487 memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail);
3488}
3489
3490/*
3491 * Release an obj back to its cache. If the obj has a constructed state, it must
3492 * be in this state _before_ it is released. Called with disabled ints.
3493 */
3494static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp,
3495 unsigned long caller)
3496{
3497 /* Put the object into the quarantine, don't touch it for now. */
3498 if (kasan_slab_free(cachep, objp, _RET_IP_))
3499 return;
3500
3501 ___cache_free(cachep, objp, caller);
3502}
3503
3504void ___cache_free(struct kmem_cache *cachep, void *objp,
3505 unsigned long caller)
3506{
3507 struct array_cache *ac = cpu_cache_get(cachep);
3508
3509 check_irq_off();
3510 kmemleak_free_recursive(objp, cachep->flags);
3511 objp = cache_free_debugcheck(cachep, objp, caller);
3512
3513 /*
3514 * Skip calling cache_free_alien() when the platform is not numa.
3515 * This will avoid cache misses that happen while accessing slabp (which
3516 * is per page memory reference) to get nodeid. Instead use a global
3517 * variable to skip the call, which is mostly likely to be present in
3518 * the cache.
3519 */
3520 if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
3521 return;
3522
3523 if (ac->avail < ac->limit) {
3524 STATS_INC_FREEHIT(cachep);
3525 } else {
3526 STATS_INC_FREEMISS(cachep);
3527 cache_flusharray(cachep, ac);
3528 }
3529
3530 if (sk_memalloc_socks()) {
3531 struct page *page = virt_to_head_page(objp);
3532
3533 if (unlikely(PageSlabPfmemalloc(page))) {
3534 cache_free_pfmemalloc(cachep, page, objp);
3535 return;
3536 }
3537 }
3538
3539 ac->entry[ac->avail++] = objp;
3540}
3541
3542/**
3543 * kmem_cache_alloc - Allocate an object
3544 * @cachep: The cache to allocate from.
3545 * @flags: See kmalloc().
3546 *
3547 * Allocate an object from this cache. The flags are only relevant
3548 * if the cache has no available objects.
3549 *
3550 * Return: pointer to the new object or %NULL in case of error
3551 */
3552void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3553{
3554 void *ret = slab_alloc(cachep, flags, _RET_IP_);
3555
3556 trace_kmem_cache_alloc(_RET_IP_, ret,
3557 cachep->object_size, cachep->size, flags);
3558
3559 return ret;
3560}
3561EXPORT_SYMBOL(kmem_cache_alloc);
3562
3563static __always_inline void
3564cache_alloc_debugcheck_after_bulk(struct kmem_cache *s, gfp_t flags,
3565 size_t size, void **p, unsigned long caller)
3566{
3567 size_t i;
3568
3569 for (i = 0; i < size; i++)
3570 p[i] = cache_alloc_debugcheck_after(s, flags, p[i], caller);
3571}
3572
3573int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
3574 void **p)
3575{
3576 size_t i;
3577
3578 s = slab_pre_alloc_hook(s, flags);
3579 if (!s)
3580 return 0;
3581
3582 cache_alloc_debugcheck_before(s, flags);
3583
3584 local_irq_disable();
3585 for (i = 0; i < size; i++) {
3586 void *objp = __do_cache_alloc(s, flags);
3587
3588 if (unlikely(!objp))
3589 goto error;
3590 p[i] = objp;
3591 }
3592 local_irq_enable();
3593
3594 cache_alloc_debugcheck_after_bulk(s, flags, size, p, _RET_IP_);
3595
3596 /* Clear memory outside IRQ disabled section */
3597 if (unlikely(flags & __GFP_ZERO))
3598 for (i = 0; i < size; i++)
3599 memset(p[i], 0, s->object_size);
3600
3601 slab_post_alloc_hook(s, flags, size, p);
3602 /* FIXME: Trace call missing. Christoph would like a bulk variant */
3603 return size;
3604error:
3605 local_irq_enable();
3606 cache_alloc_debugcheck_after_bulk(s, flags, i, p, _RET_IP_);
3607 slab_post_alloc_hook(s, flags, i, p);
3608 __kmem_cache_free_bulk(s, i, p);
3609 return 0;
3610}
3611EXPORT_SYMBOL(kmem_cache_alloc_bulk);
3612
3613#ifdef CONFIG_TRACING
3614void *
3615kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
3616{
3617 void *ret;
3618
3619 ret = slab_alloc(cachep, flags, _RET_IP_);
3620
3621 ret = kasan_kmalloc(cachep, ret, size, flags);
3622 trace_kmalloc(_RET_IP_, ret,
3623 size, cachep->size, flags);
3624 return ret;
3625}
3626EXPORT_SYMBOL(kmem_cache_alloc_trace);
3627#endif
3628
3629#ifdef CONFIG_NUMA
3630/**
3631 * kmem_cache_alloc_node - Allocate an object on the specified node
3632 * @cachep: The cache to allocate from.
3633 * @flags: See kmalloc().
3634 * @nodeid: node number of the target node.
3635 *
3636 * Identical to kmem_cache_alloc but it will allocate memory on the given
3637 * node, which can improve the performance for cpu bound structures.
3638 *
3639 * Fallback to other node is possible if __GFP_THISNODE is not set.
3640 *
3641 * Return: pointer to the new object or %NULL in case of error
3642 */
3643void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
3644{
3645 void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
3646
3647 trace_kmem_cache_alloc_node(_RET_IP_, ret,
3648 cachep->object_size, cachep->size,
3649 flags, nodeid);
3650
3651 return ret;
3652}
3653EXPORT_SYMBOL(kmem_cache_alloc_node);
3654
3655#ifdef CONFIG_TRACING
3656void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
3657 gfp_t flags,
3658 int nodeid,
3659 size_t size)
3660{
3661 void *ret;
3662
3663 ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
3664
3665 ret = kasan_kmalloc(cachep, ret, size, flags);
3666 trace_kmalloc_node(_RET_IP_, ret,
3667 size, cachep->size,
3668 flags, nodeid);
3669 return ret;
3670}
3671EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
3672#endif
3673
3674static __always_inline void *
3675__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
3676{
3677 struct kmem_cache *cachep;
3678 void *ret;
3679
3680 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
3681 return NULL;
3682 cachep = kmalloc_slab(size, flags);
3683 if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3684 return cachep;
3685 ret = kmem_cache_alloc_node_trace(cachep, flags, node, size);
3686 ret = kasan_kmalloc(cachep, ret, size, flags);
3687
3688 return ret;
3689}
3690
3691void *__kmalloc_node(size_t size, gfp_t flags, int node)
3692{
3693 return __do_kmalloc_node(size, flags, node, _RET_IP_);
3694}
3695EXPORT_SYMBOL(__kmalloc_node);
3696
3697void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
3698 int node, unsigned long caller)
3699{
3700 return __do_kmalloc_node(size, flags, node, caller);
3701}
3702EXPORT_SYMBOL(__kmalloc_node_track_caller);
3703#endif /* CONFIG_NUMA */
3704
3705/**
3706 * __do_kmalloc - allocate memory
3707 * @size: how many bytes of memory are required.
3708 * @flags: the type of memory to allocate (see kmalloc).
3709 * @caller: function caller for debug tracking of the caller
3710 *
3711 * Return: pointer to the allocated memory or %NULL in case of error
3712 */
3713static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3714 unsigned long caller)
3715{
3716 struct kmem_cache *cachep;
3717 void *ret;
3718
3719 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
3720 return NULL;
3721 cachep = kmalloc_slab(size, flags);
3722 if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3723 return cachep;
3724 ret = slab_alloc(cachep, flags, caller);
3725
3726 ret = kasan_kmalloc(cachep, ret, size, flags);
3727 trace_kmalloc(caller, ret,
3728 size, cachep->size, flags);
3729
3730 return ret;
3731}
3732
3733void *__kmalloc(size_t size, gfp_t flags)
3734{
3735 return __do_kmalloc(size, flags, _RET_IP_);
3736}
3737EXPORT_SYMBOL(__kmalloc);
3738
3739void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
3740{
3741 return __do_kmalloc(size, flags, caller);
3742}
3743EXPORT_SYMBOL(__kmalloc_track_caller);
3744
3745/**
3746 * kmem_cache_free - Deallocate an object
3747 * @cachep: The cache the allocation was from.
3748 * @objp: The previously allocated object.
3749 *
3750 * Free an object which was previously allocated from this
3751 * cache.
3752 */
3753void kmem_cache_free(struct kmem_cache *cachep, void *objp)
3754{
3755 unsigned long flags;
3756 cachep = cache_from_obj(cachep, objp);
3757 if (!cachep)
3758 return;
3759
3760 local_irq_save(flags);
3761 debug_check_no_locks_freed(objp, cachep->object_size);
3762 if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
3763 debug_check_no_obj_freed(objp, cachep->object_size);
3764 __cache_free(cachep, objp, _RET_IP_);
3765 local_irq_restore(flags);
3766
3767 trace_kmem_cache_free(_RET_IP_, objp);
3768}
3769EXPORT_SYMBOL(kmem_cache_free);
3770
3771void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
3772{
3773 struct kmem_cache *s;
3774 size_t i;
3775
3776 local_irq_disable();
3777 for (i = 0; i < size; i++) {
3778 void *objp = p[i];
3779
3780 if (!orig_s) /* called via kfree_bulk */
3781 s = virt_to_cache(objp);
3782 else
3783 s = cache_from_obj(orig_s, objp);
3784
3785 debug_check_no_locks_freed(objp, s->object_size);
3786 if (!(s->flags & SLAB_DEBUG_OBJECTS))
3787 debug_check_no_obj_freed(objp, s->object_size);
3788
3789 __cache_free(s, objp, _RET_IP_);
3790 }
3791 local_irq_enable();
3792
3793 /* FIXME: add tracing */
3794}
3795EXPORT_SYMBOL(kmem_cache_free_bulk);
3796
3797/**
3798 * kfree - free previously allocated memory
3799 * @objp: pointer returned by kmalloc.
3800 *
3801 * If @objp is NULL, no operation is performed.
3802 *
3803 * Don't free memory not originally allocated by kmalloc()
3804 * or you will run into trouble.
3805 */
3806void kfree(const void *objp)
3807{
3808 struct kmem_cache *c;
3809 unsigned long flags;
3810
3811 trace_kfree(_RET_IP_, objp);
3812
3813 if (unlikely(ZERO_OR_NULL_PTR(objp)))
3814 return;
3815 local_irq_save(flags);
3816 kfree_debugcheck(objp);
3817 c = virt_to_cache(objp);
3818 debug_check_no_locks_freed(objp, c->object_size);
3819
3820 debug_check_no_obj_freed(objp, c->object_size);
3821 __cache_free(c, (void *)objp, _RET_IP_);
3822 local_irq_restore(flags);
3823}
3824EXPORT_SYMBOL(kfree);
3825
3826/*
3827 * This initializes kmem_cache_node or resizes various caches for all nodes.
3828 */
3829static int setup_kmem_cache_nodes(struct kmem_cache *cachep, gfp_t gfp)
3830{
3831 int ret;
3832 int node;
3833 struct kmem_cache_node *n;
3834
3835 for_each_online_node(node) {
3836 ret = setup_kmem_cache_node(cachep, node, gfp, true);
3837 if (ret)
3838 goto fail;
3839
3840 }
3841
3842 return 0;
3843
3844fail:
3845 if (!cachep->list.next) {
3846 /* Cache is not active yet. Roll back what we did */
3847 node--;
3848 while (node >= 0) {
3849 n = get_node(cachep, node);
3850 if (n) {
3851 kfree(n->shared);
3852 free_alien_cache(n->alien);
3853 kfree(n);
3854 cachep->node[node] = NULL;
3855 }
3856 node--;
3857 }
3858 }
3859 return -ENOMEM;
3860}
3861
3862/* Always called with the slab_mutex held */
3863static int __do_tune_cpucache(struct kmem_cache *cachep, int limit,
3864 int batchcount, int shared, gfp_t gfp)
3865{
3866 struct array_cache __percpu *cpu_cache, *prev;
3867 int cpu;
3868
3869 cpu_cache = alloc_kmem_cache_cpus(cachep, limit, batchcount);
3870 if (!cpu_cache)
3871 return -ENOMEM;
3872
3873 prev = cachep->cpu_cache;
3874 cachep->cpu_cache = cpu_cache;
3875 /*
3876 * Without a previous cpu_cache there's no need to synchronize remote
3877 * cpus, so skip the IPIs.
3878 */
3879 if (prev)
3880 kick_all_cpus_sync();
3881
3882 check_irq_on();
3883 cachep->batchcount = batchcount;
3884 cachep->limit = limit;
3885 cachep->shared = shared;
3886
3887 if (!prev)
3888 goto setup_node;
3889
3890 for_each_online_cpu(cpu) {
3891 LIST_HEAD(list);
3892 int node;
3893 struct kmem_cache_node *n;
3894 struct array_cache *ac = per_cpu_ptr(prev, cpu);
3895
3896 node = cpu_to_mem(cpu);
3897 n = get_node(cachep, node);
3898 spin_lock_irq(&n->list_lock);
3899 free_block(cachep, ac->entry, ac->avail, node, &list);
3900 spin_unlock_irq(&n->list_lock);
3901 slabs_destroy(cachep, &list);
3902 }
3903 free_percpu(prev);
3904
3905setup_node:
3906 return setup_kmem_cache_nodes(cachep, gfp);
3907}
3908
3909static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
3910 int batchcount, int shared, gfp_t gfp)
3911{
3912 int ret;
3913 struct kmem_cache *c;
3914
3915 ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
3916
3917 if (slab_state < FULL)
3918 return ret;
3919
3920 if ((ret < 0) || !is_root_cache(cachep))
3921 return ret;
3922
3923 lockdep_assert_held(&slab_mutex);
3924 for_each_memcg_cache(c, cachep) {
3925 /* return value determined by the root cache only */
3926 __do_tune_cpucache(c, limit, batchcount, shared, gfp);
3927 }
3928
3929 return ret;
3930}
3931
3932/* Called with slab_mutex held always */
3933static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
3934{
3935 int err;
3936 int limit = 0;
3937 int shared = 0;
3938 int batchcount = 0;
3939
3940 err = cache_random_seq_create(cachep, cachep->num, gfp);
3941 if (err)
3942 goto end;
3943
3944 if (!is_root_cache(cachep)) {
3945 struct kmem_cache *root = memcg_root_cache(cachep);
3946 limit = root->limit;
3947 shared = root->shared;
3948 batchcount = root->batchcount;
3949 }
3950
3951 if (limit && shared && batchcount)
3952 goto skip_setup;
3953 /*
3954 * The head array serves three purposes:
3955 * - create a LIFO ordering, i.e. return objects that are cache-warm
3956 * - reduce the number of spinlock operations.
3957 * - reduce the number of linked list operations on the slab and
3958 * bufctl chains: array operations are cheaper.
3959 * The numbers are guessed, we should auto-tune as described by
3960 * Bonwick.
3961 */
3962 if (cachep->size > 131072)
3963 limit = 1;
3964 else if (cachep->size > PAGE_SIZE)
3965 limit = 8;
3966 else if (cachep->size > 1024)
3967 limit = 24;
3968 else if (cachep->size > 256)
3969 limit = 54;
3970 else
3971 limit = 120;
3972
3973 /*
3974 * CPU bound tasks (e.g. network routing) can exhibit cpu bound
3975 * allocation behaviour: Most allocs on one cpu, most free operations
3976 * on another cpu. For these cases, an efficient object passing between
3977 * cpus is necessary. This is provided by a shared array. The array
3978 * replaces Bonwick's magazine layer.
3979 * On uniprocessor, it's functionally equivalent (but less efficient)
3980 * to a larger limit. Thus disabled by default.
3981 */
3982 shared = 0;
3983 if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1)
3984 shared = 8;
3985
3986#if DEBUG
3987 /*
3988 * With debugging enabled, large batchcount lead to excessively long
3989 * periods with disabled local interrupts. Limit the batchcount
3990 */
3991 if (limit > 32)
3992 limit = 32;
3993#endif
3994 batchcount = (limit + 1) / 2;
3995skip_setup:
3996 err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
3997end:
3998 if (err)
3999 pr_err("enable_cpucache failed for %s, error %d\n",
4000 cachep->name, -err);
4001 return err;
4002}
4003
4004/*
4005 * Drain an array if it contains any elements taking the node lock only if
4006 * necessary. Note that the node listlock also protects the array_cache
4007 * if drain_array() is used on the shared array.
4008 */
4009static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
4010 struct array_cache *ac, int node)
4011{
4012 LIST_HEAD(list);
4013
4014 /* ac from n->shared can be freed if we don't hold the slab_mutex. */
4015 check_mutex_acquired();
4016
4017 if (!ac || !ac->avail)
4018 return;
4019
4020 if (ac->touched) {
4021 ac->touched = 0;
4022 return;
4023 }
4024
4025 spin_lock_irq(&n->list_lock);
4026 drain_array_locked(cachep, ac, node, false, &list);
4027 spin_unlock_irq(&n->list_lock);
4028
4029 slabs_destroy(cachep, &list);
4030}
4031
4032/**
4033 * cache_reap - Reclaim memory from caches.
4034 * @w: work descriptor
4035 *
4036 * Called from workqueue/eventd every few seconds.
4037 * Purpose:
4038 * - clear the per-cpu caches for this CPU.
4039 * - return freeable pages to the main free memory pool.
4040 *
4041 * If we cannot acquire the cache chain mutex then just give up - we'll try
4042 * again on the next iteration.
4043 */
4044static void cache_reap(struct work_struct *w)
4045{
4046 struct kmem_cache *searchp;
4047 struct kmem_cache_node *n;
4048 int node = numa_mem_id();
4049 struct delayed_work *work = to_delayed_work(w);
4050
4051 if (!mutex_trylock(&slab_mutex))
4052 /* Give up. Setup the next iteration. */
4053 goto out;
4054
4055 list_for_each_entry(searchp, &slab_caches, list) {
4056 check_irq_on();
4057
4058 /*
4059 * We only take the node lock if absolutely necessary and we
4060 * have established with reasonable certainty that
4061 * we can do some work if the lock was obtained.
4062 */
4063 n = get_node(searchp, node);
4064
4065 reap_alien(searchp, n);
4066
4067 drain_array(searchp, n, cpu_cache_get(searchp), node);
4068
4069 /*
4070 * These are racy checks but it does not matter
4071 * if we skip one check or scan twice.
4072 */
4073 if (time_after(n->next_reap, jiffies))
4074 goto next;
4075
4076 n->next_reap = jiffies + REAPTIMEOUT_NODE;
4077
4078 drain_array(searchp, n, n->shared, node);
4079
4080 if (n->free_touched)
4081 n->free_touched = 0;
4082 else {
4083 int freed;
4084
4085 freed = drain_freelist(searchp, n, (n->free_limit +
4086 5 * searchp->num - 1) / (5 * searchp->num));
4087 STATS_ADD_REAPED(searchp, freed);
4088 }
4089next:
4090 cond_resched();
4091 }
4092 check_irq_on();
4093 mutex_unlock(&slab_mutex);
4094 next_reap_node();
4095out:
4096 /* Set up the next iteration */
4097 schedule_delayed_work_on(smp_processor_id(), work,
4098 round_jiffies_relative(REAPTIMEOUT_AC));
4099}
4100
4101void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
4102{
4103 unsigned long active_objs, num_objs, active_slabs;
4104 unsigned long total_slabs = 0, free_objs = 0, shared_avail = 0;
4105 unsigned long free_slabs = 0;
4106 int node;
4107 struct kmem_cache_node *n;
4108
4109 for_each_kmem_cache_node(cachep, node, n) {
4110 check_irq_on();
4111 spin_lock_irq(&n->list_lock);
4112
4113 total_slabs += n->total_slabs;
4114 free_slabs += n->free_slabs;
4115 free_objs += n->free_objects;
4116
4117 if (n->shared)
4118 shared_avail += n->shared->avail;
4119
4120 spin_unlock_irq(&n->list_lock);
4121 }
4122 num_objs = total_slabs * cachep->num;
4123 active_slabs = total_slabs - free_slabs;
4124 active_objs = num_objs - free_objs;
4125
4126 sinfo->active_objs = active_objs;
4127 sinfo->num_objs = num_objs;
4128 sinfo->active_slabs = active_slabs;
4129 sinfo->num_slabs = total_slabs;
4130 sinfo->shared_avail = shared_avail;
4131 sinfo->limit = cachep->limit;
4132 sinfo->batchcount = cachep->batchcount;
4133 sinfo->shared = cachep->shared;
4134 sinfo->objects_per_slab = cachep->num;
4135 sinfo->cache_order = cachep->gfporder;
4136}
4137
4138void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep)
4139{
4140#if STATS
4141 { /* node stats */
4142 unsigned long high = cachep->high_mark;
4143 unsigned long allocs = cachep->num_allocations;
4144 unsigned long grown = cachep->grown;
4145 unsigned long reaped = cachep->reaped;
4146 unsigned long errors = cachep->errors;
4147 unsigned long max_freeable = cachep->max_freeable;
4148 unsigned long node_allocs = cachep->node_allocs;
4149 unsigned long node_frees = cachep->node_frees;
4150 unsigned long overflows = cachep->node_overflow;
4151
4152 seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu %4lu %4lu %4lu %4lu %4lu",
4153 allocs, high, grown,
4154 reaped, errors, max_freeable, node_allocs,
4155 node_frees, overflows);
4156 }
4157 /* cpu stats */
4158 {
4159 unsigned long allochit = atomic_read(&cachep->allochit);
4160 unsigned long allocmiss = atomic_read(&cachep->allocmiss);
4161 unsigned long freehit = atomic_read(&cachep->freehit);
4162 unsigned long freemiss = atomic_read(&cachep->freemiss);
4163
4164 seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
4165 allochit, allocmiss, freehit, freemiss);
4166 }
4167#endif
4168}
4169
4170#define MAX_SLABINFO_WRITE 128
4171/**
4172 * slabinfo_write - Tuning for the slab allocator
4173 * @file: unused
4174 * @buffer: user buffer
4175 * @count: data length
4176 * @ppos: unused
4177 *
4178 * Return: %0 on success, negative error code otherwise.
4179 */
4180ssize_t slabinfo_write(struct file *file, const char __user *buffer,
4181 size_t count, loff_t *ppos)
4182{
4183 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
4184 int limit, batchcount, shared, res;
4185 struct kmem_cache *cachep;
4186
4187 if (count > MAX_SLABINFO_WRITE)
4188 return -EINVAL;
4189 if (copy_from_user(&kbuf, buffer, count))
4190 return -EFAULT;
4191 kbuf[MAX_SLABINFO_WRITE] = '\0';
4192
4193 tmp = strchr(kbuf, ' ');
4194 if (!tmp)
4195 return -EINVAL;
4196 *tmp = '\0';
4197 tmp++;
4198 if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
4199 return -EINVAL;
4200
4201 /* Find the cache in the chain of caches. */
4202 mutex_lock(&slab_mutex);
4203 res = -EINVAL;
4204 list_for_each_entry(cachep, &slab_caches, list) {
4205 if (!strcmp(cachep->name, kbuf)) {
4206 if (limit < 1 || batchcount < 1 ||
4207 batchcount > limit || shared < 0) {
4208 res = 0;
4209 } else {
4210 res = do_tune_cpucache(cachep, limit,
4211 batchcount, shared,
4212 GFP_KERNEL);
4213 }
4214 break;
4215 }
4216 }
4217 mutex_unlock(&slab_mutex);
4218 if (res >= 0)
4219 res = count;
4220 return res;
4221}
4222
4223#ifdef CONFIG_DEBUG_SLAB_LEAK
4224
4225static inline int add_caller(unsigned long *n, unsigned long v)
4226{
4227 unsigned long *p;
4228 int l;
4229 if (!v)
4230 return 1;
4231 l = n[1];
4232 p = n + 2;
4233 while (l) {
4234 int i = l/2;
4235 unsigned long *q = p + 2 * i;
4236 if (*q == v) {
4237 q[1]++;
4238 return 1;
4239 }
4240 if (*q > v) {
4241 l = i;
4242 } else {
4243 p = q + 2;
4244 l -= i + 1;
4245 }
4246 }
4247 if (++n[1] == n[0])
4248 return 0;
4249 memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n));
4250 p[0] = v;
4251 p[1] = 1;
4252 return 1;
4253}
4254
4255static void handle_slab(unsigned long *n, struct kmem_cache *c,
4256 struct page *page)
4257{
4258 void *p;
4259 int i, j;
4260 unsigned long v;
4261
4262 if (n[0] == n[1])
4263 return;
4264 for (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) {
4265 bool active = true;
4266
4267 for (j = page->active; j < c->num; j++) {
4268 if (get_free_obj(page, j) == i) {
4269 active = false;
4270 break;
4271 }
4272 }
4273
4274 if (!active)
4275 continue;
4276
4277 /*
4278 * probe_kernel_read() is used for DEBUG_PAGEALLOC. page table
4279 * mapping is established when actual object allocation and
4280 * we could mistakenly access the unmapped object in the cpu
4281 * cache.
4282 */
4283 if (probe_kernel_read(&v, dbg_userword(c, p), sizeof(v)))
4284 continue;
4285
4286 if (!add_caller(n, v))
4287 return;
4288 }
4289}
4290
4291static void show_symbol(struct seq_file *m, unsigned long address)
4292{
4293#ifdef CONFIG_KALLSYMS
4294 unsigned long offset, size;
4295 char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN];
4296
4297 if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) {
4298 seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
4299 if (modname[0])
4300 seq_printf(m, " [%s]", modname);
4301 return;
4302 }
4303#endif
4304 seq_printf(m, "%px", (void *)address);
4305}
4306
4307static int leaks_show(struct seq_file *m, void *p)
4308{
4309 struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
4310 struct page *page;
4311 struct kmem_cache_node *n;
4312 const char *name;
4313 unsigned long *x = m->private;
4314 int node;
4315 int i;
4316
4317 if (!(cachep->flags & SLAB_STORE_USER))
4318 return 0;
4319 if (!(cachep->flags & SLAB_RED_ZONE))
4320 return 0;
4321
4322 /*
4323 * Set store_user_clean and start to grab stored user information
4324 * for all objects on this cache. If some alloc/free requests comes
4325 * during the processing, information would be wrong so restart
4326 * whole processing.
4327 */
4328 do {
4329 set_store_user_clean(cachep);
4330 drain_cpu_caches(cachep);
4331
4332 x[1] = 0;
4333
4334 for_each_kmem_cache_node(cachep, node, n) {
4335
4336 check_irq_on();
4337 spin_lock_irq(&n->list_lock);
4338
4339 list_for_each_entry(page, &n->slabs_full, lru)
4340 handle_slab(x, cachep, page);
4341 list_for_each_entry(page, &n->slabs_partial, lru)
4342 handle_slab(x, cachep, page);
4343 spin_unlock_irq(&n->list_lock);
4344 }
4345 } while (!is_store_user_clean(cachep));
4346
4347 name = cachep->name;
4348 if (x[0] == x[1]) {
4349 /* Increase the buffer size */
4350 mutex_unlock(&slab_mutex);
4351 m->private = kcalloc(x[0] * 4, sizeof(unsigned long),
4352 GFP_KERNEL);
4353 if (!m->private) {
4354 /* Too bad, we are really out */
4355 m->private = x;
4356 mutex_lock(&slab_mutex);
4357 return -ENOMEM;
4358 }
4359 *(unsigned long *)m->private = x[0] * 2;
4360 kfree(x);
4361 mutex_lock(&slab_mutex);
4362 /* Now make sure this entry will be retried */
4363 m->count = m->size;
4364 return 0;
4365 }
4366 for (i = 0; i < x[1]; i++) {
4367 seq_printf(m, "%s: %lu ", name, x[2*i+3]);
4368 show_symbol(m, x[2*i+2]);
4369 seq_putc(m, '\n');
4370 }
4371
4372 return 0;
4373}
4374
4375static const struct seq_operations slabstats_op = {
4376 .start = slab_start,
4377 .next = slab_next,
4378 .stop = slab_stop,
4379 .show = leaks_show,
4380};
4381
4382static int slabstats_open(struct inode *inode, struct file *file)
4383{
4384 unsigned long *n;
4385
4386 n = __seq_open_private(file, &slabstats_op, PAGE_SIZE);
4387 if (!n)
4388 return -ENOMEM;
4389
4390 *n = PAGE_SIZE / (2 * sizeof(unsigned long));
4391
4392 return 0;
4393}
4394
4395static const struct file_operations proc_slabstats_operations = {
4396 .open = slabstats_open,
4397 .read = seq_read,
4398 .llseek = seq_lseek,
4399 .release = seq_release_private,
4400};
4401#endif
4402
4403static int __init slab_proc_init(void)
4404{
4405#ifdef CONFIG_DEBUG_SLAB_LEAK
4406 proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
4407#endif
4408 return 0;
4409}
4410module_init(slab_proc_init);
4411
4412#ifdef CONFIG_HARDENED_USERCOPY
4413/*
4414 * Rejects incorrectly sized objects and objects that are to be copied
4415 * to/from userspace but do not fall entirely within the containing slab
4416 * cache's usercopy region.
4417 *
4418 * Returns NULL if check passes, otherwise const char * to name of cache
4419 * to indicate an error.
4420 */
4421void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
4422 bool to_user)
4423{
4424 struct kmem_cache *cachep;
4425 unsigned int objnr;
4426 unsigned long offset;
4427
4428 ptr = kasan_reset_tag(ptr);
4429
4430 /* Find and validate object. */
4431 cachep = page->slab_cache;
4432 objnr = obj_to_index(cachep, page, (void *)ptr);
4433 BUG_ON(objnr >= cachep->num);
4434
4435 /* Find offset within object. */
4436 offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
4437
4438 /* Allow address range falling entirely within usercopy region. */
4439 if (offset >= cachep->useroffset &&
4440 offset - cachep->useroffset <= cachep->usersize &&
4441 n <= cachep->useroffset - offset + cachep->usersize)
4442 return;
4443
4444 /*
4445 * If the copy is still within the allocated object, produce
4446 * a warning instead of rejecting the copy. This is intended
4447 * to be a temporary method to find any missing usercopy
4448 * whitelists.
4449 */
4450 if (usercopy_fallback &&
4451 offset <= cachep->object_size &&
4452 n <= cachep->object_size - offset) {
4453 usercopy_warn("SLAB object", cachep->name, to_user, offset, n);
4454 return;
4455 }
4456
4457 usercopy_abort("SLAB object", cachep->name, to_user, offset, n);
4458}
4459#endif /* CONFIG_HARDENED_USERCOPY */
4460
4461/**
4462 * ksize - get the actual amount of memory allocated for a given object
4463 * @objp: Pointer to the object
4464 *
4465 * kmalloc may internally round up allocations and return more memory
4466 * than requested. ksize() can be used to determine the actual amount of
4467 * memory allocated. The caller may use this additional memory, even though
4468 * a smaller amount of memory was initially specified with the kmalloc call.
4469 * The caller must guarantee that objp points to a valid object previously
4470 * allocated with either kmalloc() or kmem_cache_alloc(). The object
4471 * must not be freed during the duration of the call.
4472 *
4473 * Return: size of the actual memory used by @objp in bytes
4474 */
4475size_t ksize(const void *objp)
4476{
4477 size_t size;
4478
4479 BUG_ON(!objp);
4480 if (unlikely(objp == ZERO_SIZE_PTR))
4481 return 0;
4482
4483 size = virt_to_cache(objp)->object_size;
4484 /* We assume that ksize callers could use the whole allocated area,
4485 * so we need to unpoison this area.
4486 */
4487 kasan_unpoison_shadow(objp, size);
4488
4489 return size;
4490}
4491EXPORT_SYMBOL(ksize);
4492

Warning: That file was not part of the compilation database. It may have many parsing errors.