page_alloc.c source code [linux/mm/page_alloc.c]


1	/*
2	* linux/mm/page_alloc.c
3	*
4	* Manages the free list, the system allocates free pages here.
5	* Note that kmalloc() lives in slab.c
6	*
7	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
8	* Swap reorganised 29.12.95, Stephen Tweedie
9	* Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
10	* Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
11	* Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
12	* Zone balancing, Kanoj Sarcar, SGI, Jan 2000
13	* Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
14	* (lots of bits borrowed from Ingo Molnar & Andrew Morton)
15	*/
16
17	#include <linux/stddef.h>
18	#include <linux/mm.h>
19	#include <linux/highmem.h>
20	#include <linux/swap.h>
21	#include <linux/interrupt.h>
22	#include <linux/pagemap.h>
23	#include <linux/jiffies.h>
24	#include <linux/memblock.h>
25	#include <linux/compiler.h>
26	#include <linux/kernel.h>
27	#include <linux/kasan.h>
28	#include <linux/module.h>
29	#include <linux/suspend.h>
30	#include <linux/pagevec.h>
31	#include <linux/blkdev.h>
32	#include <linux/slab.h>
33	#include <linux/ratelimit.h>
34	#include <linux/oom.h>
35	#include <linux/topology.h>
36	#include <linux/sysctl.h>
37	#include <linux/cpu.h>
38	#include <linux/cpuset.h>
39	#include <linux/memory_hotplug.h>
40	#include <linux/nodemask.h>
41	#include <linux/vmalloc.h>
42	#include <linux/vmstat.h>
43	#include <linux/mempolicy.h>
44	#include <linux/memremap.h>
45	#include <linux/stop_machine.h>
46	#include <linux/sort.h>
47	#include <linux/pfn.h>
48	#include <linux/backing-dev.h>
49	#include <linux/fault-inject.h>
50	#include <linux/page-isolation.h>
51	#include <linux/page_ext.h>
52	#include <linux/debugobjects.h>
53	#include <linux/kmemleak.h>
54	#include <linux/compaction.h>
55	#include <trace/events/kmem.h>
56	#include <trace/events/oom.h>
57	#include <linux/prefetch.h>
58	#include <linux/mm_inline.h>
59	#include <linux/migrate.h>
60	#include <linux/hugetlb.h>
61	#include <linux/sched/rt.h>
62	#include <linux/sched/mm.h>
63	#include <linux/page_owner.h>
64	#include <linux/kthread.h>
65	#include <linux/memcontrol.h>
66	#include <linux/ftrace.h>
67	#include <linux/lockdep.h>
68	#include <linux/nmi.h>
69	#include <linux/psi.h>
70
71	#include <asm/sections.h>
72	#include <asm/tlbflush.h>
73	#include <asm/div64.h>
74	#include "internal.h"
75
76	/ prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields /
77	static DEFINE_MUTEX(pcp_batch_high_lock);
78	#define MIN_PERCPU_PAGELIST_FRACTION (8)
79
80	#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
81	DEFINE_PER_CPU(int, numa_node);
82	EXPORT_PER_CPU_SYMBOL(numa_node);
83	#endif
84
85	DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
86
87	#ifdef CONFIG_HAVE_MEMORYLESS_NODES
88	/*
89	* N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
90	* It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
91	* Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
92	* defined in <linux/topology.h>.
93	*/
94	DEFINE_PER_CPU(int, _numa_mem_); / Kernel "local memory" node /
95	EXPORT_PER_CPU_SYMBOL(_numa_mem_);
96	int _node_numa_mem_[MAX_NUMNODES];
97	#endif
98
99	/ work_structs for global per-cpu drains /
100	struct pcpu_drain {
101	struct zone *zone;
102	struct work_struct work;
103	};
104	DEFINE_MUTEX(pcpu_drain_mutex);
105	DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain);
106
107	#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
108	volatile unsigned long latent_entropy __latent_entropy;
109	EXPORT_SYMBOL(latent_entropy);
110	#endif
111
112	/*
113	* Array of node states.
114	*/
115	nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
116	[N_POSSIBLE] = NODE_MASK_ALL,
117	[N_ONLINE] = { { [`0`] = `1UL` } },
118	#ifndef CONFIG_NUMA
119	[N_NORMAL_MEMORY] = { { [`0`] = `1UL` } },
120	#ifdef CONFIG_HIGHMEM
121	[N_HIGH_MEMORY] = { { [`0`] = `1UL` } },
122	#endif
123	[N_MEMORY] = { { [`0`] = `1UL` } },
124	[N_CPU] = { { [`0`] = `1UL` } },
125	#endif /* NUMA */
126	};
127	EXPORT_SYMBOL(node_states);
128
129	atomic_long_t _totalram_pages __read_mostly;
130	EXPORT_SYMBOL(_totalram_pages);
131	unsigned long totalreserve_pages __read_mostly;
132	unsigned long totalcma_pages __read_mostly;
133
134	int percpu_pagelist_fraction;
135	gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
136
137	/*
138	* A cached value of the page's pageblock's migratetype, used when the page is
139	* put on a pcplist. Used to avoid the pageblock migratetype lookup when
140	* freeing from pcplists in most cases, at the cost of possibly becoming stale.
141	* Also the migratetype set in the page does not necessarily match the pcplist
142	* index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
143	* other index - this ensures that it will be put on the correct CMA freelist.
144	*/
145	static inline int get_pcppage_migratetype(struct page *page)
146	{
147	return page->index;
148	}
149
150	static inline void set_pcppage_migratetype(struct page page, int* migratetype)
151	{
152	page->index = migratetype;
153	}
154
155	#ifdef CONFIG_PM_SLEEP
156	/*
157	* The following functions are used by the suspend/hibernate code to temporarily
158	* change gfp_allowed_mask in order to avoid using I/O during memory allocations
159	* while devices are suspended. To avoid races with the suspend/hibernate code,
160	* they should always be called with system_transition_mutex held
161	* (gfp_allowed_mask also should only be modified with system_transition_mutex
162	* held, unless the suspend/hibernate code is guaranteed not to run in parallel
163	* with that modification).
164	*/
165
166	static gfp_t saved_gfp_mask;
167
168	void pm_restore_gfp_mask(void)
169	{
170	WARN_ON(!mutex_is_locked(&system_transition_mutex));
171	if (saved_gfp_mask) {
172	gfp_allowed_mask = saved_gfp_mask;
173	saved_gfp_mask = `0`;
174	}
175	}
176
177	void pm_restrict_gfp_mask(void)
178	{
179	WARN_ON(!mutex_is_locked(&system_transition_mutex));
180	WARN_ON(saved_gfp_mask);
181	saved_gfp_mask = gfp_allowed_mask;
182	gfp_allowed_mask &= ~(__GFP_IO \| __GFP_FS);
183	}
184
185	bool pm_suspended_storage(void)
186	{
187	if ((gfp_allowed_mask & (__GFP_IO \| __GFP_FS)) == (__GFP_IO \| __GFP_FS))
188	return false;
189	return true;
190	}
191	#endif /* CONFIG_PM_SLEEP */
192
193	#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
194	unsigned int pageblock_order __read_mostly;
195	#endif
196
197	static void __free_pages_ok(struct page page, unsigned* int order);
198
199	/*
200	* results with 256, 32 in the lowmem_reserve sysctl:
201	* 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
202	* 1G machine -> (16M dma, 784M normal, 224M high)
203	* NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
204	* HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
205	* HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
206	*
207	* TBD: should special case ZONE_DMA32 machines here - in those we normally
208	* don't need any ZONE_NORMAL reservation
209	*/
210	int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
211	#ifdef CONFIG_ZONE_DMA
212	[ZONE_DMA] = `256`,
213	#endif
214	#ifdef CONFIG_ZONE_DMA32
215	[ZONE_DMA32] = `256`,
216	#endif
217	[ZONE_NORMAL] = `32`,
218	#ifdef CONFIG_HIGHMEM
219	[ZONE_HIGHMEM] = `0`,
220	#endif
221	[ZONE_MOVABLE] = `0`,
222	};
223
224	EXPORT_SYMBOL(totalram_pages);
225
226	static char * const zone_names[MAX_NR_ZONES] = {
227	#ifdef CONFIG_ZONE_DMA
228	"DMA",
229	#endif
230	#ifdef CONFIG_ZONE_DMA32
231	"DMA32",
232	#endif
233	"Normal",
234	#ifdef CONFIG_HIGHMEM
235	"HighMem",
236	#endif
237	"Movable",
238	#ifdef CONFIG_ZONE_DEVICE
239	"Device",
240	#endif
241	};
242
243	const char * const migratetype_names[MIGRATE_TYPES] = {
244	"Unmovable",
245	"Movable",
246	"Reclaimable",
247	"HighAtomic",
248	#ifdef CONFIG_CMA
249	"CMA",
250	#endif
251	#ifdef CONFIG_MEMORY_ISOLATION
252	"Isolate",
253	#endif
254	};
255
256	compound_page_dtor * const compound_page_dtors[] = {
257	NULL,
258	free_compound_page,
259	#ifdef CONFIG_HUGETLB_PAGE
260	free_huge_page,
261	#endif
262	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
263	free_transhuge_page,
264	#endif
265	};
266
267	int min_free_kbytes = `1024`;
268	int user_min_free_kbytes = -`1`;
269	int watermark_boost_factor __read_mostly = `15000`;
270	int watermark_scale_factor = `10`;
271
272	static unsigned long nr_kernel_pages __initdata;
273	static unsigned long nr_all_pages __initdata;
274	static unsigned long dma_reserve __initdata;
275
276	#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
277	static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
278	static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
279	static unsigned long required_kernelcore __initdata;
280	static unsigned long required_kernelcore_percent __initdata;
281	static unsigned long required_movablecore __initdata;
282	static unsigned long required_movablecore_percent __initdata;
283	static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
284	static bool mirrored_kernelcore __meminitdata;
285
286	/ movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from /
287	int movable_zone;
288	EXPORT_SYMBOL(movable_zone);
289	#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
290
291	#if MAX_NUMNODES > 1
292	unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;
293	unsigned int nr_online_nodes __read_mostly = `1`;
294	EXPORT_SYMBOL(nr_node_ids);
295	EXPORT_SYMBOL(nr_online_nodes);
296	#endif
297
298	int page_group_by_mobility_disabled __read_mostly;
299
300	#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
301	/*
302	* During boot we initialize deferred pages on-demand, as needed, but once
303	* page_alloc_init_late() has finished, the deferred pages are all initialized,
304	* and we can permanently disable that path.
305	*/
306	static DEFINE_STATIC_KEY_TRUE(deferred_pages);
307
308	/*
309	* Calling kasan_free_pages() only after deferred memory initialization
310	* has completed. Poisoning pages during deferred memory init will greatly
311	* lengthen the process and cause problem in large memory systems as the
312	* deferred pages initialization is done with interrupt disabled.
313	*
314	* Assuming that there will be no reference to those newly initialized
315	* pages before they are ever allocated, this should have no effect on
316	* KASAN memory tracking as the poison will be properly inserted at page
317	* allocation time. The only corner case is when pages are allocated by
318	* on-demand allocation and then freed again before the deferred pages
319	* initialization is done, but this is not likely to happen.
320	*/
321	static inline void kasan_free_nondeferred_pages(struct page page, int* order)
322	{
323	if (!static_branch_unlikely(&deferred_pages))
324	kasan_free_pages(page, order);
325	}
326
327	/ Returns true if the struct page for the pfn is uninitialised /
328	static inline bool __meminit early_page_uninitialised(unsigned long pfn)
329	{
330	int nid = early_pfn_to_nid(pfn);
331
332	if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
333	return true;
334
335	return false;
336	}
337
338	/*
339	* Returns true when the remaining initialisation should be deferred until
340	* later in the boot cycle when it can be parallelised.
341	*/
342	static bool __meminit
343	defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
344	{
345	static unsigned long prev_end_pfn, nr_initialised;
346
347	/*
348	* prev_end_pfn static that contains the end of previous zone
349	* No need to protect because called very early in boot before smp_init.
350	*/
351	if (prev_end_pfn != end_pfn) {
352	prev_end_pfn = end_pfn;
353	nr_initialised = `0`;
354	}
355
356	/ Always populate low zones for address-constrained allocations /
357	if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
358	return false;
359
360	/*
361	* We start only with one section of pages, more pages are added as
362	* needed until the rest of deferred pages are initialized.
363	*/
364	nr_initialised++;
365	if ((nr_initialised > PAGES_PER_SECTION) &&
366	(pfn & (PAGES_PER_SECTION - `1`)) == `0`) {
367	NODE_DATA(nid)->first_deferred_pfn = pfn;
368	return true;
369	}
370	return false;
371	}
372	#else
373	#define kasan_free_nondeferred_pages(p, o) kasan_free_pages(p, o)
374
375	static inline bool early_page_uninitialised(unsigned long pfn)
376	{
377	return false;
378	}
379
380	static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
381	{
382	return false;
383	}
384	#endif
385
386	/ Return a pointer to the bitmap storing bits affecting a block of pages /
387	static inline unsigned long get_pageblock_bitmap(struct* page *page,
388	unsigned long pfn)
389	{
390	#ifdef CONFIG_SPARSEMEM
391	return __pfn_to_section(pfn)->pageblock_flags;
392	#else
393	return page_zone(page)->pageblock_flags;
394	#endif /* CONFIG_SPARSEMEM */
395	}
396
397	static inline int pfn_to_bitidx(struct page page, unsigned* long pfn)
398	{
399	#ifdef CONFIG_SPARSEMEM
400	pfn &= (PAGES_PER_SECTION-`1`);
401	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
402	#else
403	pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
404	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
405	#endif /* CONFIG_SPARSEMEM */
406	}
407
408	/**
409	* get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
410	* @page: The page within the block of interest
411	* @pfn: The target page frame number
412	* @end_bitidx: The last bit of interest to retrieve
413	* @mask: mask of bits that the caller is interested in
414	*
415	* Return: pageblock_bits flags
416	*/
417	static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,
418	unsigned long pfn,
419	unsigned long end_bitidx,
420	unsigned long mask)
421	{
422	unsigned long *bitmap;
423	unsigned long bitidx, word_bitidx;
424	unsigned long word;
425
426	bitmap = get_pageblock_bitmap(page, pfn);
427	bitidx = pfn_to_bitidx(page, pfn);
428	word_bitidx = bitidx / BITS_PER_LONG;
429	bitidx &= (BITS_PER_LONG-`1`);
430
431	word = bitmap[word_bitidx];
432	bitidx += end_bitidx;
433	return (word >> (BITS_PER_LONG - bitidx - `1`)) & mask;
434	}
435
436	unsigned long get_pfnblock_flags_mask(struct page page, unsigned* long pfn,
437	unsigned long end_bitidx,
438	unsigned long mask)
439	{
440	return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);
441	}
442
443	static __always_inline int get_pfnblock_migratetype(struct page page, unsigned* long pfn)
444	{
445	return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);
446	}
447
448	/**
449	* set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
450	* @page: The page within the block of interest
451	* @flags: The flags to set
452	* @pfn: The target page frame number
453	* @end_bitidx: The last bit of interest
454	* @mask: mask of bits that the caller is interested in
455	*/
456	void set_pfnblock_flags_mask(struct page page, unsigned* long flags,
457	unsigned long pfn,
458	unsigned long end_bitidx,
459	unsigned long mask)
460	{
461	unsigned long *bitmap;
462	unsigned long bitidx, word_bitidx;
463	unsigned long old_word, word;
464
465	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != `4`);
466	BUILD_BUG_ON(MIGRATE_TYPES > (`1` << PB_migratetype_bits));
467
468	bitmap = get_pageblock_bitmap(page, pfn);
469	bitidx = pfn_to_bitidx(page, pfn);
470	word_bitidx = bitidx / BITS_PER_LONG;
471	bitidx &= (BITS_PER_LONG-`1`);
472
473	VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
474
475	bitidx += end_bitidx;
476	mask <<= (BITS_PER_LONG - bitidx - `1`);
477	flags <<= (BITS_PER_LONG - bitidx - `1`);
478
479	word = READ_ONCE(bitmap[word_bitidx]);
480	for (;;) {
481	old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) \| flags);
482	if (word == old_word)
483	break;
484	word = old_word;
485	}
486	}
487
488	void set_pageblock_migratetype(struct page page, int* migratetype)
489	{
490	if (unlikely(page_group_by_mobility_disabled &&
491	migratetype < MIGRATE_PCPTYPES))
492	migratetype = MIGRATE_UNMOVABLE;
493
494	set_pageblock_flags_group(page, (unsigned long)migratetype,
495	PB_migrate, PB_migrate_end);
496	}
497
498	#ifdef CONFIG_DEBUG_VM
499	static int page_outside_zone_boundaries(struct zone zone, struct* page *page)
500	{
501	int ret = `0`;
502	unsigned seq;
503	unsigned long pfn = page_to_pfn(page);
504	unsigned long sp, start_pfn;
505
506	do {
507	seq = zone_span_seqbegin(zone);
508	start_pfn = zone->zone_start_pfn;
509	sp = zone->spanned_pages;
510	if (!zone_spans_pfn(zone, pfn))
511	ret = `1`;
512	} while (zone_span_seqretry(zone, seq));
513
514	if (ret)
515	pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
516	pfn, zone_to_nid(zone), zone->name,
517	start_pfn, start_pfn + sp);
518
519	return ret;
520	}
521
522	static int page_is_consistent(struct zone zone, struct* page *page)
523	{
524	if (!pfn_valid_within(page_to_pfn(page)))
525	return `0`;
526	if (zone != page_zone(page))
527	return `0`;
528
529	return `1`;
530	}
531	/*
532	* Temporary debugging check for pages not lying within a given zone.
533	*/
534	static int __maybe_unused bad_range(struct zone zone, struct* page *page)
535	{
536	if (page_outside_zone_boundaries(zone, page))
537	return `1`;
538	if (!page_is_consistent(zone, page))
539	return `1`;
540
541	return `0`;
542	}
543	#else
544	static inline int __maybe_unused bad_range(struct zone zone, struct* page *page)
545	{
546	return `0`;
547	}
548	#endif
549
550	static void bad_page(struct page page, const* char *reason,
551	unsigned long bad_flags)
552	{
553	static unsigned long resume;
554	static unsigned long nr_shown;
555	static unsigned long nr_unshown;
556
557	/*
558	* Allow a burst of 60 reports, then keep quiet for that minute;
559	* or allow a steady drip of one report per second.
560	*/
561	if (nr_shown == `60`) {
562	if (time_before(jiffies, resume)) {
563	nr_unshown++;
564	goto out;
565	}
566	if (nr_unshown) {
567	pr_alert(
568	"BUG: Bad page state: %lu messages suppressed\n",
569	nr_unshown);
570	nr_unshown = `0`;
571	}
572	nr_shown = `0`;
573	}
574	if (nr_shown++ == `0`)
575	resume = jiffies + `60` * HZ;
576
577	pr_alert("BUG: Bad page state in process %s pfn:%05lx\n",
578	current->comm, page_to_pfn(page));
579	__dump_page(page, reason);
580	bad_flags &= page->flags;
581	if (bad_flags)
582	pr_alert("bad because of flags: %#lx(%pGp)\n",
583	bad_flags, &bad_flags);
584	dump_page_owner(page);
585
586	print_modules();
587	dump_stack();
588	out:
589	/ Leave bad fields for debug, except PageBuddy could make trouble /
590	page_mapcount_reset(page); / remove PageBuddy /
591	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
592	}
593
594	/*
595	* Higher-order pages are called "compound pages". They are structured thusly:
596	*
597	* The first PAGE_SIZE page is called the "head page" and have PG_head set.
598	*
599	* The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
600	* in bit 0 of page->compound_head. The rest of bits is pointer to head page.
601	*
602	* The first tail page's ->compound_dtor holds the offset in array of compound
603	* page destructors. See compound_page_dtors.
604	*
605	* The first tail page's ->compound_order holds the order of allocation.
606	* This usage means that zero-order pages may not be compound.
607	*/
608
609	void free_compound_page(struct page *page)
610	{
611	__free_pages_ok(page, compound_order(page));
612	}
613
614	void prep_compound_page(struct page page, unsigned* int order)
615	{
616	int i;
617	int nr_pages = `1` << order;
618
619	set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
620	set_compound_order(page, order);
621	__SetPageHead(page);
622	for (i = `1`; i < nr_pages; i++) {
623	struct page *p = page + i;
624	set_page_count(p, `0`);
625	p->mapping = TAIL_MAPPING;
626	set_compound_head(p, page);
627	}
628	atomic_set(compound_mapcount_ptr(page), -`1`);
629	}
630
631	#ifdef CONFIG_DEBUG_PAGEALLOC
632	unsigned int _debug_guardpage_minorder;
633	bool _debug_pagealloc_enabled __read_mostly
634	= IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
635	EXPORT_SYMBOL(_debug_pagealloc_enabled);
636	bool _debug_guardpage_enabled __read_mostly;
637
638	static int __init early_debug_pagealloc(char *buf)
639	{
640	if (!buf)
641	return -EINVAL;
642	return kstrtobool(buf, &_debug_pagealloc_enabled);
643	}
644	early_param("debug_pagealloc", early_debug_pagealloc);
645
646	static bool need_debug_guardpage(void)
647	{
648	/ If we don't use debug_pagealloc, we don't need guard page /
649	if (!debug_pagealloc_enabled())
650	return false;
651
652	if (!debug_guardpage_minorder())
653	return false;
654
655	return true;
656	}
657
658	static void init_debug_guardpage(void)
659	{
660	if (!debug_pagealloc_enabled())
661	return;
662
663	if (!debug_guardpage_minorder())
664	return;
665
666	_debug_guardpage_enabled = true;
667	}
668
669	struct page_ext_operations debug_guardpage_ops = {
670	.need = need_debug_guardpage,
671	.init = init_debug_guardpage,
672	};
673
674	static int __init debug_guardpage_minorder_setup(char *buf)
675	{
676	unsigned long res;
677
678	if (kstrtoul(buf, `10`, &res) < `0` \|\| res > MAX_ORDER / `2`) {
679	pr_err("Bad debug_guardpage_minorder value\n");
680	return `0`;
681	}
682	_debug_guardpage_minorder = res;
683	pr_info("Setting debug_guardpage_minorder to %lu\n", res);
684	return `0`;
685	}
686	early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
687
688	static inline bool set_page_guard(struct zone zone, struct* page *page,
689	unsigned int order, int migratetype)
690	{
691	struct page_ext *page_ext;
692
693	if (!debug_guardpage_enabled())
694	return false;
695
696	if (order >= debug_guardpage_minorder())
697	return false;
698
699	page_ext = lookup_page_ext(page);
700	if (unlikely(!page_ext))
701	return false;
702
703	__set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
704
705	INIT_LIST_HEAD(&page->lru);
706	set_page_private(page, order);
707	/ Guard pages are not available for any usage /
708	__mod_zone_freepage_state(zone, -(`1` << order), migratetype);
709
710	return true;
711	}
712
713	static inline void clear_page_guard(struct zone zone, struct* page *page,
714	unsigned int order, int migratetype)
715	{
716	struct page_ext *page_ext;
717
718	if (!debug_guardpage_enabled())
719	return;
720
721	page_ext = lookup_page_ext(page);
722	if (unlikely(!page_ext))
723	return;
724
725	__clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
726
727	set_page_private(page, `0`);
728	if (!is_migrate_isolate(migratetype))
729	__mod_zone_freepage_state(zone, (`1` << order), migratetype);
730	}
731	#else
732	struct page_ext_operations debug_guardpage_ops;
733	static inline bool set_page_guard(struct zone zone, struct* page *page,
734	unsigned int order, int migratetype) { return false; }
735	static inline void clear_page_guard(struct zone zone, struct* page *page,
736	unsigned int order, int migratetype) {}
737	#endif
738
739	static inline void set_page_order(struct page page, unsigned* int order)
740	{
741	set_page_private(page, order);
742	__SetPageBuddy(page);
743	}
744
745	static inline void rmv_page_order(struct page *page)
746	{
747	__ClearPageBuddy(page);
748	set_page_private(page, `0`);
749	}
750
751	/*
752	* This function checks whether a page is free && is the buddy
753	* we can coalesce a page and its buddy if
754	* (a) the buddy is not in a hole (check before calling!) &&
755	* (b) the buddy is in the buddy system &&
756	* (c) a page and its buddy have the same order &&
757	* (d) a page and its buddy are in the same zone.
758	*
759	* For recording whether a page is in the buddy system, we set PageBuddy.
760	* Setting, clearing, and testing PageBuddy is serialized by zone->lock.
761	*
762	* For recording page's order, we use page_private(page).
763	*/
764	static inline int page_is_buddy(struct page page, struct* page *buddy,
765	unsigned int order)
766	{
767	if (page_is_guard(buddy) && page_order(buddy) == order) {
768	if (page_zone_id(page) != page_zone_id(buddy))
769	return `0`;
770
771	VM_BUG_ON_PAGE(page_count(buddy) != `0`, buddy);
772
773	return `1`;
774	}
775
776	if (PageBuddy(buddy) && page_order(buddy) == order) {
777	/*
778	* zone check is done late to avoid uselessly
779	* calculating zone/node ids for pages that could
780	* never merge.
781	*/
782	if (page_zone_id(page) != page_zone_id(buddy))
783	return `0`;
784
785	VM_BUG_ON_PAGE(page_count(buddy) != `0`, buddy);
786
787	return `1`;
788	}
789	return `0`;
790	}
791
792	#ifdef CONFIG_COMPACTION
793	static inline struct capture_control task_capc(struct* zone *zone)
794	{
795	struct capture_control *capc = current->capture_control;
796
797	return capc &&
798	!(current->flags & PF_KTHREAD) &&
799	!capc->page &&
800	capc->cc->zone == zone &&
801	capc->cc->direct_compaction ? capc : NULL;
802	}
803
804	static inline bool
805	compaction_capture(struct capture_control capc, struct* page *page,
806	int order, int migratetype)
807	{
808	if (!capc \|\| order != capc->cc->order)
809	return false;
810
811	/ Do not accidentally pollute CMA or isolated regions/
812	if (is_migrate_cma(migratetype) \|\|
813	is_migrate_isolate(migratetype))
814	return false;
815
816	/*
817	* Do not let lower order allocations polluate a movable pageblock.
818	* This might let an unmovable request use a reclaimable pageblock
819	* and vice-versa but no more than normal fallback logic which can
820	* have trouble finding a high-order free page.
821	*/
822	if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
823	return false;
824
825	capc->page = page;
826	return true;
827	}
828
829	#else
830	static inline struct capture_control task_capc(struct* zone *zone)
831	{
832	return NULL;
833	}
834
835	static inline bool
836	compaction_capture(struct capture_control capc, struct* page *page,
837	int order, int migratetype)
838	{
839	return false;
840	}
841	#endif /* CONFIG_COMPACTION */
842
843	/*
844	* Freeing function for a buddy system allocator.
845	*
846	* The concept of a buddy system is to maintain direct-mapped table
847	* (containing bit values) for memory blocks of various "orders".
848	* The bottom level table contains the map for the smallest allocatable
849	* units of memory (here, pages), and each level above it describes
850	* pairs of units from the levels below, hence, "buddies".
851	* At a high level, all that happens here is marking the table entry
852	* at the bottom level available, and propagating the changes upward
853	* as necessary, plus some accounting needed to play nicely with other
854	* parts of the VM system.
855	* At each level, we keep a list of pages, which are heads of continuous
856	* free pages of length of (1 << order) and marked with PageBuddy.
857	* Page's order is recorded in page_private(page) field.
858	* So when we are allocating or freeing one, we can derive the state of the
859	* other. That is, if we allocate a small block, and both were
860	* free, the remainder of the region must be split into blocks.
861	* If a block is freed, and its buddy is also free, then this
862	* triggers coalescing into a block of larger size.
863	*
864	* -- nyc
865	*/
866
867	static inline void __free_one_page(struct page *page,
868	unsigned long pfn,
869	struct zone zone, unsigned* int order,
870	int migratetype)
871	{
872	unsigned long combined_pfn;
873	unsigned long uninitialized_var(buddy_pfn);
874	struct page *buddy;
875	unsigned int max_order;
876	struct capture_control *capc = task_capc(zone);
877
878	max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + `1`);
879
880	VM_BUG_ON(!zone_is_initialized(zone));
881	VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
882
883	VM_BUG_ON(migratetype == -`1`);
884	if (likely(!is_migrate_isolate(migratetype)))
885	__mod_zone_freepage_state(zone, `1` << order, migratetype);
886
887	VM_BUG_ON_PAGE(pfn & ((`1` << order) - `1`), page);
888	VM_BUG_ON_PAGE(bad_range(zone, page), page);
889
890	continue_merging:
891	while (order < max_order - `1`) {
892	if (compaction_capture(capc, page, order, migratetype)) {
893	__mod_zone_freepage_state(zone, -(`1` << order),
894	migratetype);
895	return;
896	}
897	buddy_pfn = __find_buddy_pfn(pfn, order);
898	buddy = page + (buddy_pfn - pfn);
899
900	if (!pfn_valid_within(buddy_pfn))
901	goto done_merging;
902	if (!page_is_buddy(page, buddy, order))
903	goto done_merging;
904	/*
905	* Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
906	* merge with it and move up one order.
907	*/
908	if (page_is_guard(buddy)) {
909	clear_page_guard(zone, buddy, order, migratetype);
910	} else {
911	list_del(&buddy->lru);
912	zone->free_area[order].nr_free--;
913	rmv_page_order(buddy);
914	}
915	combined_pfn = buddy_pfn & pfn;
916	page = page + (combined_pfn - pfn);
917	pfn = combined_pfn;
918	order++;
919	}
920	if (max_order < MAX_ORDER) {
921	/ If we are here, it means order is >= pageblock_order.*
922	* We want to prevent merge between freepages on isolate
923	* pageblock and normal pageblock. Without this, pageblock
924	* isolation could cause incorrect freepage or CMA accounting.
925	*
926	* We don't want to hit this code for the more frequent
927	* low-order merging.
928	*/
929	if (unlikely(has_isolate_pageblock(zone))) {
930	int buddy_mt;
931
932	buddy_pfn = __find_buddy_pfn(pfn, order);
933	buddy = page + (buddy_pfn - pfn);
934	buddy_mt = get_pageblock_migratetype(buddy);
935
936	if (migratetype != buddy_mt
937	&& (is_migrate_isolate(migratetype) \|\|
938	is_migrate_isolate(buddy_mt)))
939	goto done_merging;
940	}
941	max_order++;
942	goto continue_merging;
943	}
944
945	done_merging:
946	set_page_order(page, order);
947
948	/*
949	* If this is not the largest possible page, check if the buddy
950	* of the next-highest order is free. If it is, it's possible
951	* that pages are being freed that will coalesce soon. In case,
952	* that is happening, add the free page to the tail of the list
953	* so it's less likely to be used soon and more likely to be merged
954	* as a higher order page
955	*/
956	if ((order < MAX_ORDER-`2`) && pfn_valid_within(buddy_pfn)) {
957	struct page higher_page, higher_buddy;
958	combined_pfn = buddy_pfn & pfn;
959	higher_page = page + (combined_pfn - pfn);
960	buddy_pfn = __find_buddy_pfn(combined_pfn, order + `1`);
961	higher_buddy = higher_page + (buddy_pfn - combined_pfn);
962	if (pfn_valid_within(buddy_pfn) &&
963	page_is_buddy(higher_page, higher_buddy, order + `1`)) {
964	list_add_tail(&page->lru,
965	&zone->free_area[order].free_list[migratetype]);
966	goto out;
967	}
968	}
969
970	list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
971	out:
972	zone->free_area[order].nr_free++;
973	}
974
975	/*
976	* A bad page could be due to a number of fields. Instead of multiple branches,
977	* try and check multiple fields with one check. The caller must do a detailed
978	* check if necessary.
979	*/
980	static inline bool page_expected_state(struct page *page,
981	unsigned long check_flags)
982	{
983	if (unlikely(atomic_read(&page->_mapcount) != -`1`))
984	return false;
985
986	if (unlikely((unsigned long)page->mapping \|
987	page_ref_count(page) \|
988	#ifdef CONFIG_MEMCG
989	(unsigned long)page->mem_cgroup \|
990	#endif
991	(page->flags & check_flags)))
992	return false;
993
994	return true;
995	}
996
997	static void free_pages_check_bad(struct page *page)
998	{
999	const char *bad_reason;
1000	unsigned long bad_flags;
1001
1002	bad_reason = NULL;
1003	bad_flags = `0`;
1004
1005	if (unlikely(atomic_read(&page->_mapcount) != -`1`))
1006	bad_reason = "nonzero mapcount";
1007	if (unlikely(page->mapping != NULL))
1008	bad_reason = "non-NULL mapping";
1009	if (unlikely(page_ref_count(page) != `0`))
1010	bad_reason = "nonzero _refcount";
1011	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
1012	bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
1013	bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
1014	}
1015	#ifdef CONFIG_MEMCG
1016	if (unlikely(page->mem_cgroup))
1017	bad_reason = "page still charged to cgroup";
1018	#endif
1019	bad_page(page, bad_reason, bad_flags);
1020	}
1021
1022	static inline int free_pages_check(struct page *page)
1023	{
1024	if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
1025	return `0`;
1026
1027	/ Something has gone sideways, find it /
1028	free_pages_check_bad(page);
1029	return `1`;
1030	}
1031
1032	static int free_tail_pages_check(struct page head_page, struct* page *page)
1033	{
1034	int ret = `1`;
1035
1036	/*
1037	* We rely page->lru.next never has bit 0 set, unless the page
1038	* is PageTail(). Let's make sure that's true even for poisoned ->lru.
1039	*/
1040	BUILD_BUG_ON((unsigned long)LIST_POISON1 & `1`);
1041
1042	if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
1043	ret = `0`;
1044	goto out;
1045	}
1046	switch (page - head_page) {
1047	case `1`:
1048	/ the first tail page: ->mapping may be compound_mapcount() /
1049	if (unlikely(compound_mapcount(page))) {
1050	bad_page(page, "nonzero compound_mapcount", `0`);
1051	goto out;
1052	}
1053	break;
1054	case `2`:
1055	/*
1056	* the second tail page: ->mapping is
1057	* deferred_list.next -- ignore value.
1058	*/
1059	break;
1060	default:
1061	if (page->mapping != TAIL_MAPPING) {
1062	bad_page(page, "corrupted mapping in tail page", `0`);
1063	goto out;
1064	}
1065	break;
1066	}
1067	if (unlikely(!PageTail(page))) {
1068	bad_page(page, "PageTail not set", `0`);
1069	goto out;
1070	}
1071	if (unlikely(compound_head(page) != head_page)) {
1072	bad_page(page, "compound_head not consistent", `0`);
1073	goto out;
1074	}
1075	ret = `0`;
1076	out:
1077	page->mapping = NULL;
1078	clear_compound_head(page);
1079	return ret;
1080	}
1081
1082	static __always_inline bool free_pages_prepare(struct page *page,
1083	unsigned int order, bool check_free)
1084	{
1085	int bad = `0`;
1086
1087	VM_BUG_ON_PAGE(PageTail(page), page);
1088
1089	trace_mm_page_free(page, order);
1090
1091	/*
1092	* Check tail pages before head page information is cleared to
1093	* avoid checking PageCompound for order-0 pages.
1094	*/
1095	if (unlikely(order)) {
1096	bool compound = PageCompound(page);
1097	int i;
1098
1099	VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1100
1101	if (compound)
1102	ClearPageDoubleMap(page);
1103	for (i = `1`; i < (`1` << order); i++) {
1104	if (compound)
1105	bad += free_tail_pages_check(page, page + i);
1106	if (unlikely(free_pages_check(page + i))) {
1107	bad++;
1108	continue;
1109	}
1110	(page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1111	}
1112	}
1113	if (PageMappingFlags(page))
1114	page->mapping = NULL;
1115	if (memcg_kmem_enabled() && PageKmemcg(page))
1116	__memcg_kmem_uncharge(page, order);
1117	if (check_free)
1118	bad += free_pages_check(page);
1119	if (bad)
1120	return false;
1121
1122	page_cpupid_reset_last(page);
1123	page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1124	reset_page_owner(page, order);
1125
1126	if (!PageHighMem(page)) {
1127	debug_check_no_locks_freed(page_address(page),
1128	PAGE_SIZE << order);
1129	debug_check_no_obj_freed(page_address(page),
1130	PAGE_SIZE << order);
1131	}
1132	arch_free_page(page, order);
1133	kernel_poison_pages(page, `1` << order, `0`);
1134	kernel_map_pages(page, `1` << order, `0`);
1135	kasan_free_nondeferred_pages(page, order);
1136
1137	return true;
1138	}
1139
1140	#ifdef CONFIG_DEBUG_VM
1141	static inline bool free_pcp_prepare(struct page *page)
1142	{
1143	return free_pages_prepare(page, `0`, true);
1144	}
1145
1146	static inline bool bulkfree_pcp_prepare(struct page *page)
1147	{
1148	return false;
1149	}
1150	#else
1151	static bool free_pcp_prepare(struct page *page)
1152	{
1153	return free_pages_prepare(page, `0`, false);
1154	}
1155
1156	static bool bulkfree_pcp_prepare(struct page *page)
1157	{
1158	return free_pages_check(page);
1159	}
1160	#endif /* CONFIG_DEBUG_VM */
1161
1162	static inline void prefetch_buddy(struct page *page)
1163	{
1164	unsigned long pfn = page_to_pfn(page);
1165	unsigned long buddy_pfn = __find_buddy_pfn(pfn, `0`);
1166	struct page *buddy = page + (buddy_pfn - pfn);
1167
1168	prefetch(buddy);
1169	}
1170
1171	/*
1172	* Frees a number of pages from the PCP lists
1173	* Assumes all pages on list are in same zone, and of same order.
1174	* count is the number of pages to free.
1175	*
1176	* If the zone was previously in an "all pages pinned" state then look to
1177	* see if this freeing clears that state.
1178	*
1179	* And clear the zone's pages_scanned counter, to hold off the "all pages are
1180	* pinned" detection logic.
1181	*/
1182	static void free_pcppages_bulk(struct zone zone, int* count,
1183	struct per_cpu_pages *pcp)
1184	{
1185	int migratetype = `0`;
1186	int batch_free = `0`;
1187	int prefetch_nr = `0`;
1188	bool isolated_pageblocks;
1189	struct page page, tmp;
1190	LIST_HEAD(head);
1191
1192	while (count) {
1193	struct list_head *list;
1194
1195	/*
1196	* Remove pages from lists in a round-robin fashion. A
1197	* batch_free count is maintained that is incremented when an
1198	* empty list is encountered. This is so more pages are freed
1199	* off fuller lists instead of spinning excessively around empty
1200	* lists
1201	*/
1202	do {
1203	batch_free++;
1204	if (++migratetype == MIGRATE_PCPTYPES)
1205	migratetype = `0`;
1206	list = &pcp->lists[migratetype];
1207	} while (list_empty(list));
1208
1209	/ This is the only non-empty list. Free them all. /
1210	if (batch_free == MIGRATE_PCPTYPES)
1211	batch_free = count;
1212
1213	do {
1214	page = list_last_entry(list, struct page, lru);
1215	/ must delete to avoid corrupting pcp list /
1216	list_del(&page->lru);
1217	pcp->count--;
1218
1219	if (bulkfree_pcp_prepare(page))
1220	continue;
1221
1222	list_add_tail(&page->lru, &head);
1223
1224	/*
1225	* We are going to put the page back to the global
1226	* pool, prefetch its buddy to speed up later access
1227	* under zone->lock. It is believed the overhead of
1228	* an additional test and calculating buddy_pfn here
1229	* can be offset by reduced memory latency later. To
1230	* avoid excessive prefetching due to large count, only
1231	* prefetch buddy for the first pcp->batch nr of pages.
1232	*/
1233	if (prefetch_nr++ < pcp->batch)
1234	prefetch_buddy(page);
1235	} while (--count && --batch_free && !list_empty(list));
1236	}
1237
1238	spin_lock(&zone->lock);
1239	isolated_pageblocks = has_isolate_pageblock(zone);
1240
1241	/*
1242	* Use safe version since after __free_one_page(),
1243	* page->lru.next will not point to original list.
1244	*/
1245	list_for_each_entry_safe(page, tmp, &head, lru) {
1246	int mt = get_pcppage_migratetype(page);
1247	/ MIGRATE_ISOLATE page should not go to pcplists /
1248	VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1249	/ Pageblock could have been isolated meanwhile /
1250	if (unlikely(isolated_pageblocks))
1251	mt = get_pageblock_migratetype(page);
1252
1253	__free_one_page(page, page_to_pfn(page), zone, `0`, mt);
1254	trace_mm_page_pcpu_drain(page, `0`, mt);
1255	}
1256	spin_unlock(&zone->lock);
1257	}
1258
1259	static void free_one_page(struct zone *zone,
1260	struct page page, unsigned* long pfn,
1261	unsigned int order,
1262	int migratetype)
1263	{
1264	spin_lock(&zone->lock);
1265	if (unlikely(has_isolate_pageblock(zone) \|\|
1266	is_migrate_isolate(migratetype))) {
1267	migratetype = get_pfnblock_migratetype(page, pfn);
1268	}
1269	__free_one_page(page, pfn, zone, order, migratetype);
1270	spin_unlock(&zone->lock);
1271	}
1272
1273	static void __meminit __init_single_page(struct page page, unsigned* long pfn,
1274	unsigned long zone, int nid)
1275	{
1276	mm_zero_struct_page(page);
1277	set_page_links(page, zone, nid, pfn);
1278	init_page_count(page);
1279	page_mapcount_reset(page);
1280	page_cpupid_reset_last(page);
1281	page_kasan_tag_reset(page);
1282
1283	INIT_LIST_HEAD(&page->lru);
1284	#ifdef WANT_PAGE_VIRTUAL
1285	/ The shift won't overflow because ZONE_NORMAL is below 4G. /
1286	if (!is_highmem_idx(zone))
1287	set_page_address(page, __va(pfn << PAGE_SHIFT));
1288	#endif
1289	}
1290
1291	#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1292	static void __meminit init_reserved_page(unsigned long pfn)
1293	{
1294	pg_data_t *pgdat;
1295	int nid, zid;
1296
1297	if (!early_page_uninitialised(pfn))
1298	return;
1299
1300	nid = early_pfn_to_nid(pfn);
1301	pgdat = NODE_DATA(nid);
1302
1303	for (zid = `0`; zid < MAX_NR_ZONES; zid++) {
1304	struct zone *zone = &pgdat->node_zones[zid];
1305
1306	if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
1307	break;
1308	}
1309	__init_single_page(pfn_to_page(pfn), pfn, zid, nid);
1310	}
1311	#else
1312	static inline void init_reserved_page(unsigned long pfn)
1313	{
1314	}
1315	#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1316
1317	/*
1318	* Initialised pages do not have PageReserved set. This function is
1319	* called for each range allocated by the bootmem allocator and
1320	* marks the pages PageReserved. The remaining valid pages are later
1321	* sent to the buddy page allocator.
1322	*/
1323	void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
1324	{
1325	unsigned long start_pfn = PFN_DOWN(start);
1326	unsigned long end_pfn = PFN_UP(end);
1327
1328	for (; start_pfn < end_pfn; start_pfn++) {
1329	if (pfn_valid(start_pfn)) {
1330	struct page *page = pfn_to_page(start_pfn);
1331
1332	init_reserved_page(start_pfn);
1333
1334	/ Avoid false-positive PageTail() /
1335	INIT_LIST_HEAD(&page->lru);
1336
1337	/*
1338	* no need for atomic set_bit because the struct
1339	* page is not visible yet so nobody should
1340	* access it yet.
1341	*/
1342	__SetPageReserved(page);
1343	}
1344	}
1345	}
1346
1347	static void __free_pages_ok(struct page page, unsigned* int order)
1348	{
1349	unsigned long flags;
1350	int migratetype;
1351	unsigned long pfn = page_to_pfn(page);
1352
1353	if (!free_pages_prepare(page, order, true))
1354	return;
1355
1356	migratetype = get_pfnblock_migratetype(page, pfn);
1357	local_irq_save(flags);
1358	__count_vm_events(PGFREE, `1` << order);
1359	free_one_page(page_zone(page), page, pfn, order, migratetype);
1360	local_irq_restore(flags);
1361	}
1362
1363	void __free_pages_core(struct page page, unsigned* int order)
1364	{
1365	unsigned int nr_pages = `1` << order;
1366	struct page *p = page;
1367	unsigned int loop;
1368
1369	prefetchw(p);
1370	for (loop = `0`; loop < (nr_pages - `1`); loop++, p++) {
1371	prefetchw(p + `1`);
1372	__ClearPageReserved(p);
1373	set_page_count(p, `0`);
1374	}
1375	__ClearPageReserved(p);
1376	set_page_count(p, `0`);
1377
1378	atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
1379	set_page_refcounted(page);
1380	__free_pages(page, order);
1381	}
1382
1383	#if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) \|\| \
1384	defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
1385
1386	static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
1387
1388	int __meminit early_pfn_to_nid(unsigned long pfn)
1389	{
1390	static DEFINE_SPINLOCK(early_pfn_lock);
1391	int nid;
1392
1393	spin_lock(&early_pfn_lock);
1394	nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1395	if (nid < `0`)
1396	nid = first_online_node;
1397	spin_unlock(&early_pfn_lock);
1398
1399	return nid;
1400	}
1401	#endif
1402
1403	#ifdef CONFIG_NODES_SPAN_OTHER_NODES
1404	static inline bool __meminit __maybe_unused
1405	meminit_pfn_in_nid(unsigned long pfn, int node,
1406	struct mminit_pfnnid_cache *state)
1407	{
1408	int nid;
1409
1410	nid = __early_pfn_to_nid(pfn, state);
1411	if (nid >= `0` && nid != node)
1412	return false;
1413	return true;
1414	}
1415
1416	/ Only safe to use early in boot when initialisation is single-threaded /
1417	static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1418	{
1419	return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache);
1420	}
1421
1422	#else
1423
1424	static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1425	{
1426	return true;
1427	}
1428	static inline bool __meminit __maybe_unused
1429	meminit_pfn_in_nid(unsigned long pfn, int node,
1430	struct mminit_pfnnid_cache *state)
1431	{
1432	return true;
1433	}
1434	#endif
1435
1436
1437	void __init memblock_free_pages(struct page page, unsigned* long pfn,
1438	unsigned int order)
1439	{
1440	if (early_page_uninitialised(pfn))
1441	return;
1442	__free_pages_core(page, order);
1443	}
1444
1445	/*
1446	* Check that the whole (or subset of) a pageblock given by the interval of
1447	* [start_pfn, end_pfn) is valid and within the same zone, before scanning it
1448	* with the migration of free compaction scanner. The scanners then need to
1449	* use only pfn_valid_within() check for arches that allow holes within
1450	* pageblocks.
1451	*
1452	* Return struct page pointer of start_pfn, or NULL if checks were not passed.
1453	*
1454	* It's possible on some configurations to have a setup like node0 node1 node0
1455	* i.e. it's possible that all pages within a zones range of pages do not
1456	* belong to a single zone. We assume that a border between node0 and node1
1457	* can occur within a single pageblock, but not a node0 node1 node0
1458	* interleaving within a single pageblock. It is therefore sufficient to check
1459	* the first and last page of a pageblock and avoid checking each individual
1460	* page in a pageblock.
1461	*/
1462	struct page __pageblock_pfn_to_page(unsigned* long start_pfn,
1463	unsigned long end_pfn, struct zone *zone)
1464	{
1465	struct page *start_page;
1466	struct page *end_page;
1467
1468	/ end_pfn is one past the range we are checking /
1469	end_pfn--;
1470
1471	if (!pfn_valid(start_pfn) \|\| !pfn_valid(end_pfn))
1472	return NULL;
1473
1474	start_page = pfn_to_online_page(start_pfn);
1475	if (!start_page)
1476	return NULL;
1477
1478	if (page_zone(start_page) != zone)
1479	return NULL;
1480
1481	end_page = pfn_to_page(end_pfn);
1482
1483	/ This gives a shorter code than deriving page_zone(end_page) /
1484	if (page_zone_id(start_page) != page_zone_id(end_page))
1485	return NULL;
1486
1487	return start_page;
1488	}
1489
1490	void set_zone_contiguous(struct zone *zone)
1491	{
1492	unsigned long block_start_pfn = zone->zone_start_pfn;
1493	unsigned long block_end_pfn;
1494
1495	block_end_pfn = ALIGN(block_start_pfn + `1`, pageblock_nr_pages);
1496	for (; block_start_pfn < zone_end_pfn(zone);
1497	block_start_pfn = block_end_pfn,
1498	block_end_pfn += pageblock_nr_pages) {
1499
1500	block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
1501
1502	if (!__pageblock_pfn_to_page(block_start_pfn,
1503	block_end_pfn, zone))
1504	return;
1505	}
1506
1507	/ We confirm that there is no hole /
1508	zone->contiguous = true;
1509	}
1510
1511	void clear_zone_contiguous(struct zone *zone)
1512	{
1513	zone->contiguous = false;
1514	}
1515
1516	#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1517	static void __init deferred_free_range(unsigned long pfn,
1518	unsigned long nr_pages)
1519	{
1520	struct page *page;
1521	unsigned long i;
1522
1523	if (!nr_pages)
1524	return;
1525
1526	page = pfn_to_page(pfn);
1527
1528	/ Free a large naturally-aligned chunk if possible /
1529	if (nr_pages == pageblock_nr_pages &&
1530	(pfn & (pageblock_nr_pages - `1`)) == `0`) {
1531	set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1532	__free_pages_core(page, pageblock_order);
1533	return;
1534	}
1535
1536	for (i = `0`; i < nr_pages; i++, page++, pfn++) {
1537	if ((pfn & (pageblock_nr_pages - `1`)) == `0`)
1538	set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1539	__free_pages_core(page, `0`);
1540	}
1541	}
1542
1543	/ Completion tracking for deferred_init_memmap() threads /
1544	static atomic_t pgdat_init_n_undone __initdata;
1545	static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1546
1547	static inline void __init pgdat_init_report_one_done(void)
1548	{
1549	if (atomic_dec_and_test(&pgdat_init_n_undone))
1550	complete(&pgdat_init_all_done_comp);
1551	}
1552
1553	/*
1554	* Returns true if page needs to be initialized or freed to buddy allocator.
1555	*
1556	* First we check if pfn is valid on architectures where it is possible to have
1557	* holes within pageblock_nr_pages. On systems where it is not possible, this
1558	* function is optimized out.
1559	*
1560	* Then, we check if a current large page is valid by only checking the validity
1561	* of the head pfn.
1562	*
1563	* Finally, meminit_pfn_in_nid is checked on systems where pfns can interleave
1564	* within a node: a pfn is between start and end of a node, but does not belong
1565	* to this memory node.
1566	*/
1567	static inline bool __init
1568	deferred_pfn_valid(int nid, unsigned long pfn,
1569	struct mminit_pfnnid_cache *nid_init_state)
1570	{
1571	if (!pfn_valid_within(pfn))
1572	return false;
1573	if (!(pfn & (pageblock_nr_pages - `1`)) && !pfn_valid(pfn))
1574	return false;
1575	if (!meminit_pfn_in_nid(pfn, nid, nid_init_state))
1576	return false;
1577	return true;
1578	}
1579
1580	/*
1581	* Free pages to buddy allocator. Try to free aligned pages in
1582	* pageblock_nr_pages sizes.
1583	*/
1584	static void __init deferred_free_pages(int nid, int zid, unsigned long pfn,
1585	unsigned long end_pfn)
1586	{
1587	struct mminit_pfnnid_cache nid_init_state = { };
1588	unsigned long nr_pgmask = pageblock_nr_pages - `1`;
1589	unsigned long nr_free = `0`;
1590
1591	for (; pfn < end_pfn; pfn++) {
1592	if (!deferred_pfn_valid(nid, pfn, &nid_init_state)) {
1593	deferred_free_range(pfn - nr_free, nr_free);
1594	nr_free = `0`;
1595	} else if (!(pfn & nr_pgmask)) {
1596	deferred_free_range(pfn - nr_free, nr_free);
1597	nr_free = `1`;
1598	touch_nmi_watchdog();
1599	} else {
1600	nr_free++;
1601	}
1602	}
1603	/ Free the last block of pages to allocator /
1604	deferred_free_range(pfn - nr_free, nr_free);
1605	}
1606
1607	/*
1608	* Initialize struct pages. We minimize pfn page lookups and scheduler checks
1609	* by performing it only once every pageblock_nr_pages.
1610	* Return number of pages initialized.
1611	*/
1612	static unsigned long __init deferred_init_pages(int nid, int zid,
1613	unsigned long pfn,
1614	unsigned long end_pfn)
1615	{
1616	struct mminit_pfnnid_cache nid_init_state = { };
1617	unsigned long nr_pgmask = pageblock_nr_pages - `1`;
1618	unsigned long nr_pages = `0`;
1619	struct page *page = NULL;
1620
1621	for (; pfn < end_pfn; pfn++) {
1622	if (!deferred_pfn_valid(nid, pfn, &nid_init_state)) {
1623	page = NULL;
1624	continue;
1625	} else if (!page \|\| !(pfn & nr_pgmask)) {
1626	page = pfn_to_page(pfn);
1627	touch_nmi_watchdog();
1628	} else {
1629	page++;
1630	}
1631	__init_single_page(page, pfn, zid, nid);
1632	nr_pages++;
1633	}
1634	return (nr_pages);
1635	}
1636
1637	/ Initialise remaining memory on a node /
1638	static int __init deferred_init_memmap(void *data)
1639	{
1640	pg_data_t *pgdat = data;
1641	int nid = pgdat->node_id;
1642	unsigned long start = jiffies;
1643	unsigned long nr_pages = `0`;
1644	unsigned long spfn, epfn, first_init_pfn, flags;
1645	phys_addr_t spa, epa;
1646	int zid;
1647	struct zone *zone;
1648	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
1649	u64 i;
1650
1651	/ Bind memory initialisation thread to a local node if possible /
1652	if (!cpumask_empty(cpumask))
1653	set_cpus_allowed_ptr(current, cpumask);
1654
1655	pgdat_resize_lock(pgdat, &flags);
1656	first_init_pfn = pgdat->first_deferred_pfn;
1657	if (first_init_pfn == ULONG_MAX) {
1658	pgdat_resize_unlock(pgdat, &flags);
1659	pgdat_init_report_one_done();
1660	return `0`;
1661	}
1662
1663	/ Sanity check boundaries /
1664	BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
1665	BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
1666	pgdat->first_deferred_pfn = ULONG_MAX;
1667
1668	/ Only the highest zone is deferred so find it /
1669	for (zid = `0`; zid < MAX_NR_ZONES; zid++) {
1670	zone = pgdat->node_zones + zid;
1671	if (first_init_pfn < zone_end_pfn(zone))
1672	break;
1673	}
1674	first_init_pfn = max(zone->zone_start_pfn, first_init_pfn);
1675
1676	/*
1677	* Initialize and free pages. We do it in two loops: first we initialize
1678	* struct page, than free to buddy allocator, because while we are
1679	* freeing pages we can access pages that are ahead (computing buddy
1680	* page in __free_one_page()).
1681	*/
1682	for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
1683	spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
1684	epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa));
1685	nr_pages += deferred_init_pages(nid, zid, spfn, epfn);
1686	}
1687	for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
1688	spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
1689	epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa));
1690	deferred_free_pages(nid, zid, spfn, epfn);
1691	}
1692	pgdat_resize_unlock(pgdat, &flags);
1693
1694	/ Sanity check that the next zone really is unpopulated /
1695	WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
1696
1697	pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages,
1698	jiffies_to_msecs(jiffies - start));
1699
1700	pgdat_init_report_one_done();
1701	return `0`;
1702	}
1703
1704	/*
1705	* If this zone has deferred pages, try to grow it by initializing enough
1706	* deferred pages to satisfy the allocation specified by order, rounded up to
1707	* the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments
1708	* of SECTION_SIZE bytes by initializing struct pages in increments of
1709	* PAGES_PER_SECTION * sizeof(struct page) bytes.
1710	*
1711	* Return true when zone was grown, otherwise return false. We return true even
1712	* when we grow less than requested, to let the caller decide if there are
1713	* enough pages to satisfy the allocation.
1714	*
1715	* Note: We use noinline because this function is needed only during boot, and
1716	* it is called from a __ref function _deferred_grow_zone. This way we are
1717	* making sure that it is not inlined into permanent text section.
1718	*/
1719	static noinline bool __init
1720	deferred_grow_zone(struct zone zone, unsigned* int order)
1721	{
1722	int zid = zone_idx(zone);
1723	int nid = zone_to_nid(zone);
1724	pg_data_t *pgdat = NODE_DATA(nid);
1725	unsigned long nr_pages_needed = ALIGN(`1` << order, PAGES_PER_SECTION);
1726	unsigned long nr_pages = `0`;
1727	unsigned long first_init_pfn, spfn, epfn, t, flags;
1728	unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
1729	phys_addr_t spa, epa;
1730	u64 i;
1731
1732	/ Only the last zone may have deferred pages /
1733	if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
1734	return false;
1735
1736	pgdat_resize_lock(pgdat, &flags);
1737
1738	/*
1739	* If deferred pages have been initialized while we were waiting for
1740	* the lock, return true, as the zone was grown. The caller will retry
1741	* this zone. We won't return to this function since the caller also
1742	* has this static branch.
1743	*/
1744	if (!static_branch_unlikely(&deferred_pages)) {
1745	pgdat_resize_unlock(pgdat, &flags);
1746	return true;
1747	}
1748
1749	/*
1750	* If someone grew this zone while we were waiting for spinlock, return
1751	* true, as there might be enough pages already.
1752	*/
1753	if (first_deferred_pfn != pgdat->first_deferred_pfn) {
1754	pgdat_resize_unlock(pgdat, &flags);
1755	return true;
1756	}
1757
1758	first_init_pfn = max(zone->zone_start_pfn, first_deferred_pfn);
1759
1760	if (first_init_pfn >= pgdat_end_pfn(pgdat)) {
1761	pgdat_resize_unlock(pgdat, &flags);
1762	return false;
1763	}
1764
1765	for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
1766	spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
1767	epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa));
1768
1769	while (spfn < epfn && nr_pages < nr_pages_needed) {
1770	t = ALIGN(spfn + PAGES_PER_SECTION, PAGES_PER_SECTION);
1771	first_deferred_pfn = min(t, epfn);
1772	nr_pages += deferred_init_pages(nid, zid, spfn,
1773	first_deferred_pfn);
1774	spfn = first_deferred_pfn;
1775	}
1776
1777	if (nr_pages >= nr_pages_needed)
1778	break;
1779	}
1780
1781	for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
1782	spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
1783	epfn = min_t(unsigned long, first_deferred_pfn, PFN_DOWN(epa));
1784	deferred_free_pages(nid, zid, spfn, epfn);
1785
1786	if (first_deferred_pfn == epfn)
1787	break;
1788	}
1789	pgdat->first_deferred_pfn = first_deferred_pfn;
1790	pgdat_resize_unlock(pgdat, &flags);
1791
1792	return nr_pages > `0`;
1793	}
1794
1795	/*
1796	* deferred_grow_zone() is __init, but it is called from
1797	* get_page_from_freelist() during early boot until deferred_pages permanently
1798	* disables this call. This is why we have refdata wrapper to avoid warning,
1799	* and to ensure that the function body gets unloaded.
1800	*/
1801	static bool __ref
1802	_deferred_grow_zone(struct zone zone, unsigned* int order)
1803	{
1804	return deferred_grow_zone(zone, order);
1805	}
1806
1807	#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1808
1809	void __init page_alloc_init_late(void)
1810	{
1811	struct zone *zone;
1812
1813	#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1814	int nid;
1815
1816	/ There will be num_node_state(N_MEMORY) threads /
1817	atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
1818	for_each_node_state(nid, N_MEMORY) {
1819	kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
1820	}
1821
1822	/ Block until all are initialised /
1823	wait_for_completion(&pgdat_init_all_done_comp);
1824
1825	/*
1826	* We initialized the rest of the deferred pages. Permanently disable
1827	* on-demand struct page initialization.
1828	*/
1829	static_branch_disable(&deferred_pages);
1830
1831	/ Reinit limits that are based on free pages after the kernel is up /
1832	files_maxfiles_init();
1833	#endif
1834	#ifdef CONFIG_ARCH_DISCARD_MEMBLOCK
1835	/ Discard memblock private memory /
1836	memblock_discard();
1837	#endif
1838
1839	for_each_populated_zone(zone)
1840	set_zone_contiguous(zone);
1841	}
1842
1843	#ifdef CONFIG_CMA
1844	/ Free whole pageblock and set its migration type to MIGRATE_CMA. /
1845	void __init init_cma_reserved_pageblock(struct page *page)
1846	{
1847	unsigned i = pageblock_nr_pages;
1848	struct page *p = page;
1849
1850	do {
1851	__ClearPageReserved(p);
1852	set_page_count(p, `0`);
1853	} while (++p, --i);
1854
1855	set_pageblock_migratetype(page, MIGRATE_CMA);
1856
1857	if (pageblock_order >= MAX_ORDER) {
1858	i = pageblock_nr_pages;
1859	p = page;
1860	do {
1861	set_page_refcounted(p);
1862	__free_pages(p, MAX_ORDER - `1`);
1863	p += MAX_ORDER_NR_PAGES;
1864	} while (i -= MAX_ORDER_NR_PAGES);
1865	} else {
1866	set_page_refcounted(page);
1867	__free_pages(page, pageblock_order);
1868	}
1869
1870	adjust_managed_page_count(page, pageblock_nr_pages);
1871	}
1872	#endif
1873
1874	/*
1875	* The order of subdivision here is critical for the IO subsystem.
1876	* Please do not alter this order without good reasons and regression
1877	* testing. Specifically, as large blocks of memory are subdivided,
1878	* the order in which smaller blocks are delivered depends on the order
1879	* they're subdivided in this function. This is the primary factor
1880	* influencing the order in which pages are delivered to the IO
1881	* subsystem according to empirical testing, and this is also justified
1882	* by considering the behavior of a buddy system containing a single
1883	* large block of memory acted on by a series of small allocations.
1884	* This behavior is a critical factor in sglist merging's success.
1885	*
1886	* -- nyc
1887	*/
1888	static inline void expand(struct zone zone, struct* page *page,
1889	int low, int high, struct free_area *area,
1890	int migratetype)
1891	{
1892	unsigned long size = `1` << high;
1893
1894	while (high > low) {
1895	area--;
1896	high--;
1897	size >>= `1`;
1898	VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
1899
1900	/*
1901	* Mark as guard pages (or page), that will allow to
1902	* merge back to allocator when buddy will be freed.
1903	* Corresponding page table entries will not be touched,
1904	* pages will stay not present in virtual address space
1905	*/
1906	if (set_page_guard(zone, &page[size], high, migratetype))
1907	continue;
1908
1909	list_add(&page[size].lru, &area->free_list[migratetype]);
1910	area->nr_free++;
1911	set_page_order(&page[size], high);
1912	}
1913	}
1914
1915	static void check_new_page_bad(struct page *page)
1916	{
1917	const char *bad_reason = NULL;
1918	unsigned long bad_flags = `0`;
1919
1920	if (unlikely(atomic_read(&page->_mapcount) != -`1`))
1921	bad_reason = "nonzero mapcount";
1922	if (unlikely(page->mapping != NULL))
1923	bad_reason = "non-NULL mapping";
1924	if (unlikely(page_ref_count(page) != `0`))
1925	bad_reason = "nonzero _count";
1926	if (unlikely(page->flags & __PG_HWPOISON)) {
1927	bad_reason = "HWPoisoned (hardware-corrupted)";
1928	bad_flags = __PG_HWPOISON;
1929	/ Don't complain about hwpoisoned pages /
1930	page_mapcount_reset(page); / remove PageBuddy /
1931	return;
1932	}
1933	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
1934	bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
1935	bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
1936	}
1937	#ifdef CONFIG_MEMCG
1938	if (unlikely(page->mem_cgroup))
1939	bad_reason = "page still charged to cgroup";
1940	#endif
1941	bad_page(page, bad_reason, bad_flags);
1942	}
1943
1944	/*
1945	* This page is about to be returned from the page allocator
1946	*/
1947	static inline int check_new_page(struct page *page)
1948	{
1949	if (likely(page_expected_state(page,
1950	PAGE_FLAGS_CHECK_AT_PREP\|__PG_HWPOISON)))
1951	return `0`;
1952
1953	check_new_page_bad(page);
1954	return `1`;
1955	}
1956
1957	static inline bool free_pages_prezeroed(void)
1958	{
1959	return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
1960	page_poisoning_enabled();
1961	}
1962
1963	#ifdef CONFIG_DEBUG_VM
1964	static bool check_pcp_refill(struct page *page)
1965	{
1966	return false;
1967	}
1968
1969	static bool check_new_pcp(struct page *page)
1970	{
1971	return check_new_page(page);
1972	}
1973	#else
1974	static bool check_pcp_refill(struct page *page)
1975	{
1976	return check_new_page(page);
1977	}
1978	static bool check_new_pcp(struct page *page)
1979	{
1980	return false;
1981	}
1982	#endif /* CONFIG_DEBUG_VM */
1983
1984	static bool check_new_pages(struct page page, unsigned* int order)
1985	{
1986	int i;
1987	for (i = `0`; i < (`1` << order); i++) {
1988	struct page *p = page + i;
1989
1990	if (unlikely(check_new_page(p)))
1991	return true;
1992	}
1993
1994	return false;
1995	}
1996
1997	inline void post_alloc_hook(struct page page, unsigned* int order,
1998	gfp_t gfp_flags)
1999	{
2000	set_page_private(page, `0`);
2001	set_page_refcounted(page);
2002
2003	arch_alloc_page(page, order);
2004	kernel_map_pages(page, `1` << order, `1`);
2005	kasan_alloc_pages(page, order);
2006	kernel_poison_pages(page, `1` << order, `1`);
2007	set_page_owner(page, order, gfp_flags);
2008	}
2009
2010	static void prep_new_page(struct page page, unsigned* int order, gfp_t gfp_flags,
2011	unsigned int alloc_flags)
2012	{
2013	int i;
2014
2015	post_alloc_hook(page, order, gfp_flags);
2016
2017	if (!free_pages_prezeroed() && (gfp_flags & __GFP_ZERO))
2018	for (i = `0`; i < (`1` << order); i++)
2019	clear_highpage(page + i);
2020
2021	if (order && (gfp_flags & __GFP_COMP))
2022	prep_compound_page(page, order);
2023
2024	/*
2025	* page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
2026	* allocate the page. The expectation is that the caller is taking
2027	* steps that will free more memory. The caller should avoid the page
2028	* being used for !PFMEMALLOC purposes.
2029	*/
2030	if (alloc_flags & ALLOC_NO_WATERMARKS)
2031	set_page_pfmemalloc(page);
2032	else
2033	clear_page_pfmemalloc(page);
2034	}
2035
2036	/*
2037	* Go through the free lists for the given migratetype and remove
2038	* the smallest available page from the freelists
2039	*/
2040	static __always_inline
2041	struct page __rmqueue_smallest(struct* zone zone, unsigned* int order,
2042	int migratetype)
2043	{
2044	unsigned int current_order;
2045	struct free_area *area;
2046	struct page *page;
2047
2048	/ Find a page of the appropriate size in the preferred list /
2049	for (current_order = order; current_order < MAX_ORDER; ++current_order) {
2050	area = &(zone->free_area[current_order]);
2051	page = list_first_entry_or_null(&area->free_list[migratetype],
2052	struct page, lru);
2053	if (!page)
2054	continue;
2055	list_del(&page->lru);
2056	rmv_page_order(page);
2057	area->nr_free--;
2058	expand(zone, page, order, current_order, area, migratetype);
2059	set_pcppage_migratetype(page, migratetype);
2060	return page;
2061	}
2062
2063	return NULL;
2064	}
2065
2066
2067	/*
2068	* This array describes the order lists are fallen back to when
2069	* the free lists for the desirable migrate type are depleted
2070	*/
2071	static int fallbacks[MIGRATE_TYPES][`4`] = {
2072	[MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
2073	[MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
2074	[MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
2075	#ifdef CONFIG_CMA
2076	[MIGRATE_CMA] = { MIGRATE_TYPES }, / Never used /
2077	#endif
2078	#ifdef CONFIG_MEMORY_ISOLATION
2079	[MIGRATE_ISOLATE] = { MIGRATE_TYPES }, / Never used /
2080	#endif
2081	};
2082
2083	#ifdef CONFIG_CMA
2084	static __always_inline struct page __rmqueue_cma_fallback(struct* zone *zone,
2085	unsigned int order)
2086	{
2087	return __rmqueue_smallest(zone, order, MIGRATE_CMA);
2088	}
2089	#else
2090	static inline struct page __rmqueue_cma_fallback(struct* zone *zone,
2091	unsigned int order) { return NULL; }
2092	#endif
2093
2094	/*
2095	* Move the free pages in a range to the free lists of the requested type.
2096	* Note that start_page and end_pages are not aligned on a pageblock
2097	* boundary. If alignment is required, use move_freepages_block()
2098	*/
2099	static int move_freepages(struct zone *zone,
2100	struct page start_page, struct* page *end_page,
2101	int migratetype, int *num_movable)
2102	{
2103	struct page *page;
2104	unsigned int order;
2105	int pages_moved = `0`;
2106
2107	#ifndef CONFIG_HOLES_IN_ZONE
2108	/*
2109	* page_zone is not safe to call in this context when
2110	* CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
2111	* anyway as we check zone boundaries in move_freepages_block().
2112	* Remove at a later date when no bug reports exist related to
2113	* grouping pages by mobility
2114	*/
2115	VM_BUG_ON(pfn_valid(page_to_pfn(start_page)) &&
2116	pfn_valid(page_to_pfn(end_page)) &&
2117	page_zone(start_page) != page_zone(end_page));
2118	#endif
2119	for (page = start_page; page <= end_page;) {
2120	if (!pfn_valid_within(page_to_pfn(page))) {
2121	page++;
2122	continue;
2123	}
2124
2125	/ Make sure we are not inadvertently changing nodes /
2126	VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2127
2128	if (!PageBuddy(page)) {
2129	/*
2130	* We assume that pages that could be isolated for
2131	* migration are movable. But we don't actually try
2132	* isolating, as that would be expensive.
2133	*/
2134	if (num_movable &&
2135	(PageLRU(page) \|\| __PageMovable(page)))
2136	(*num_movable)++;
2137
2138	page++;
2139	continue;
2140	}
2141
2142	order = page_order(page);
2143	list_move(&page->lru,
2144	&zone->free_area[order].free_list[migratetype]);
2145	page += `1` << order;
2146	pages_moved += `1` << order;
2147	}
2148
2149	return pages_moved;
2150	}
2151
2152	int move_freepages_block(struct zone zone, struct* page *page,
2153	int migratetype, int *num_movable)
2154	{
2155	unsigned long start_pfn, end_pfn;
2156	struct page start_page, end_page;
2157
2158	if (num_movable)
2159	*num_movable = `0`;
2160
2161	start_pfn = page_to_pfn(page);
2162	start_pfn = start_pfn & ~(pageblock_nr_pages-`1`);
2163	start_page = pfn_to_page(start_pfn);
2164	end_page = start_page + pageblock_nr_pages - `1`;
2165	end_pfn = start_pfn + pageblock_nr_pages - `1`;
2166
2167	/ Do not cross zone boundaries /
2168	if (!zone_spans_pfn(zone, start_pfn))
2169	start_page = page;
2170	if (!zone_spans_pfn(zone, end_pfn))
2171	return `0`;
2172
2173	return move_freepages(zone, start_page, end_page, migratetype,
2174	num_movable);
2175	}
2176
2177	static void change_pageblock_range(struct page *pageblock_page,
2178	int start_order, int migratetype)
2179	{
2180	int nr_pageblocks = `1` << (start_order - pageblock_order);
2181
2182	while (nr_pageblocks--) {
2183	set_pageblock_migratetype(pageblock_page, migratetype);
2184	pageblock_page += pageblock_nr_pages;
2185	}
2186	}
2187
2188	/*
2189	* When we are falling back to another migratetype during allocation, try to
2190	* steal extra free pages from the same pageblocks to satisfy further
2191	* allocations, instead of polluting multiple pageblocks.
2192	*
2193	* If we are stealing a relatively large buddy page, it is likely there will
2194	* be more free pages in the pageblock, so try to steal them all. For
2195	* reclaimable and unmovable allocations, we steal regardless of page size,
2196	* as fragmentation caused by those allocations polluting movable pageblocks
2197	* is worse than movable allocations stealing from unmovable and reclaimable
2198	* pageblocks.
2199	*/
2200	static bool can_steal_fallback(unsigned int order, int start_mt)
2201	{
2202	/*
2203	* Leaving this order check is intended, although there is
2204	* relaxed order check in next check. The reason is that
2205	* we can actually steal whole pageblock if this condition met,
2206	* but, below check doesn't guarantee it and that is just heuristic
2207	* so could be changed anytime.
2208	*/
2209	if (order >= pageblock_order)
2210	return true;
2211
2212	if (order >= pageblock_order / `2` \|\|
2213	start_mt == MIGRATE_RECLAIMABLE \|\|
2214	start_mt == MIGRATE_UNMOVABLE \|\|
2215	page_group_by_mobility_disabled)
2216	return true;
2217
2218	return false;
2219	}
2220
2221	static inline void boost_watermark(struct zone *zone)
2222	{
2223	unsigned long max_boost;
2224
2225	if (!watermark_boost_factor)
2226	return;
2227
2228	max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
2229	watermark_boost_factor, `10000`);
2230
2231	/*
2232	* high watermark may be uninitialised if fragmentation occurs
2233	* very early in boot so do not boost. We do not fall
2234	* through and boost by pageblock_nr_pages as failing
2235	* allocations that early means that reclaim is not going
2236	* to help and it may even be impossible to reclaim the
2237	* boosted watermark resulting in a hang.
2238	*/
2239	if (!max_boost)
2240	return;
2241
2242	max_boost = max(pageblock_nr_pages, max_boost);
2243
2244	zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
2245	max_boost);
2246	}
2247
2248	/*
2249	* This function implements actual steal behaviour. If order is large enough,
2250	* we can steal whole pageblock. If not, we first move freepages in this
2251	* pageblock to our migratetype and determine how many already-allocated pages
2252	* are there in the pageblock with a compatible migratetype. If at least half
2253	* of pages are free or compatible, we can change migratetype of the pageblock
2254	* itself, so pages freed in the future will be put on the correct free list.
2255	*/
2256	static void steal_suitable_fallback(struct zone zone, struct* page *page,
2257	unsigned int alloc_flags, int start_type, bool whole_block)
2258	{
2259	unsigned int current_order = page_order(page);
2260	struct free_area *area;
2261	int free_pages, movable_pages, alike_pages;
2262	int old_block_type;
2263
2264	old_block_type = get_pageblock_migratetype(page);
2265
2266	/*
2267	* This can happen due to races and we want to prevent broken
2268	* highatomic accounting.
2269	*/
2270	if (is_migrate_highatomic(old_block_type))
2271	goto single_page;
2272
2273	/ Take ownership for orders >= pageblock_order /
2274	if (current_order >= pageblock_order) {
2275	change_pageblock_range(page, current_order, start_type);
2276	goto single_page;
2277	}
2278
2279	/*
2280	* Boost watermarks to increase reclaim pressure to reduce the
2281	* likelihood of future fallbacks. Wake kswapd now as the node
2282	* may be balanced overall and kswapd will not wake naturally.
2283	*/
2284	boost_watermark(zone);
2285	if (alloc_flags & ALLOC_KSWAPD)
2286	set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
2287
2288	/ We are not allowed to try stealing from the whole block /
2289	if (!whole_block)
2290	goto single_page;
2291
2292	free_pages = move_freepages_block(zone, page, start_type,
2293	&movable_pages);
2294	/*
2295	* Determine how many pages are compatible with our allocation.
2296	* For movable allocation, it's the number of movable pages which
2297	* we just obtained. For other types it's a bit more tricky.
2298	*/
2299	if (start_type == MIGRATE_MOVABLE) {
2300	alike_pages = movable_pages;
2301	} else {
2302	/*
2303	* If we are falling back a RECLAIMABLE or UNMOVABLE allocation
2304	* to MOVABLE pageblock, consider all non-movable pages as
2305	* compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
2306	* vice versa, be conservative since we can't distinguish the
2307	* exact migratetype of non-movable pages.
2308	*/
2309	if (old_block_type == MIGRATE_MOVABLE)
2310	alike_pages = pageblock_nr_pages
2311	- (free_pages + movable_pages);
2312	else
2313	alike_pages = `0`;
2314	}
2315
2316	/ moving whole block can fail due to zone boundary conditions /
2317	if (!free_pages)
2318	goto single_page;
2319
2320	/*
2321	* If a sufficient number of pages in the block are either free or of
2322	* comparable migratability as our allocation, claim the whole block.
2323	*/
2324	if (free_pages + alike_pages >= (`1` << (pageblock_order-`1`)) \|\|
2325	page_group_by_mobility_disabled)
2326	set_pageblock_migratetype(page, start_type);
2327
2328	return;
2329
2330	single_page:
2331	area = &zone->free_area[current_order];
2332	list_move(&page->lru, &area->free_list[start_type]);
2333	}
2334
2335	/*
2336	* Check whether there is a suitable fallback freepage with requested order.
2337	* If only_stealable is true, this function returns fallback_mt only if
2338	* we can steal other freepages all together. This would help to reduce
2339	* fragmentation due to mixed migratetype pages in one pageblock.
2340	*/
2341	int find_suitable_fallback(struct free_area area, unsigned* int order,
2342	int migratetype, bool only_stealable, bool *can_steal)
2343	{
2344	int i;
2345	int fallback_mt;
2346
2347	if (area->nr_free == `0`)
2348	return -`1`;
2349
2350	*can_steal = false;
2351	for (i = `0`;; i++) {
2352	fallback_mt = fallbacks[migratetype][i];
2353	if (fallback_mt == MIGRATE_TYPES)
2354	break;
2355
2356	if (list_empty(&area->free_list[fallback_mt]))
2357	continue;
2358
2359	if (can_steal_fallback(order, migratetype))
2360	*can_steal = true;
2361
2362	if (!only_stealable)
2363	return fallback_mt;
2364
2365	if (*can_steal)
2366	return fallback_mt;
2367	}
2368
2369	return -`1`;
2370	}
2371
2372	/*
2373	* Reserve a pageblock for exclusive use of high-order atomic allocations if
2374	* there are no empty page blocks that contain a page with a suitable order
2375	*/
2376	static void reserve_highatomic_pageblock(struct page page, struct* zone *zone,
2377	unsigned int alloc_order)
2378	{
2379	int mt;
2380	unsigned long max_managed, flags;
2381
2382	/*
2383	* Limit the number reserved to 1 pageblock or roughly 1% of a zone.
2384	* Check is race-prone but harmless.
2385	*/
2386	max_managed = (zone_managed_pages(zone) / `100`) + pageblock_nr_pages;
2387	if (zone->nr_reserved_highatomic >= max_managed)
2388	return;
2389
2390	spin_lock_irqsave(&zone->lock, flags);
2391
2392	/ Recheck the nr_reserved_highatomic limit under the lock /
2393	if (zone->nr_reserved_highatomic >= max_managed)
2394	goto out_unlock;
2395
2396	/ Yoink! /
2397	mt = get_pageblock_migratetype(page);
2398	if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
2399	&& !is_migrate_cma(mt)) {
2400	zone->nr_reserved_highatomic += pageblock_nr_pages;
2401	set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
2402	move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
2403	}
2404
2405	out_unlock:
2406	spin_unlock_irqrestore(&zone->lock, flags);
2407	}
2408
2409	/*
2410	* Used when an allocation is about to fail under memory pressure. This
2411	* potentially hurts the reliability of high-order allocations when under
2412	* intense memory pressure but failed atomic allocations should be easier
2413	* to recover from than an OOM.
2414	*
2415	* If @force is true, try to unreserve a pageblock even though highatomic
2416	* pageblock is exhausted.
2417	*/
2418	static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
2419	bool force)
2420	{
2421	struct zonelist *zonelist = ac->zonelist;
2422	unsigned long flags;
2423	struct zoneref *z;
2424	struct zone *zone;
2425	struct page *page;
2426	int order;
2427	bool ret;
2428
2429	for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
2430	ac->nodemask) {
2431	/*
2432	* Preserve at least one pageblock unless memory pressure
2433	* is really high.
2434	*/
2435	if (!force && zone->nr_reserved_highatomic <=
2436	pageblock_nr_pages)
2437	continue;
2438
2439	spin_lock_irqsave(&zone->lock, flags);
2440	for (order = `0`; order < MAX_ORDER; order++) {
2441	struct free_area *area = &(zone->free_area[order]);
2442
2443	page = list_first_entry_or_null(
2444	&area->free_list[MIGRATE_HIGHATOMIC],
2445	struct page, lru);
2446	if (!page)
2447	continue;
2448
2449	/*
2450	* In page freeing path, migratetype change is racy so
2451	* we can counter several free pages in a pageblock
2452	* in this loop althoug we changed the pageblock type
2453	* from highatomic to ac->migratetype. So we should
2454	* adjust the count once.
2455	*/
2456	if (is_migrate_highatomic_page(page)) {
2457	/*
2458	* It should never happen but changes to
2459	* locking could inadvertently allow a per-cpu
2460	* drain to add pages to MIGRATE_HIGHATOMIC
2461	* while unreserving so be safe and watch for
2462	* underflows.
2463	*/
2464	zone->nr_reserved_highatomic -= min(
2465	pageblock_nr_pages,
2466	zone->nr_reserved_highatomic);
2467	}
2468
2469	/*
2470	* Convert to ac->migratetype and avoid the normal
2471	* pageblock stealing heuristics. Minimally, the caller
2472	* is doing the work and needs the pages. More
2473	* importantly, if the block was always converted to
2474	* MIGRATE_UNMOVABLE or another type then the number
2475	* of pageblocks that cannot be completely freed
2476	* may increase.
2477	*/
2478	set_pageblock_migratetype(page, ac->migratetype);
2479	ret = move_freepages_block(zone, page, ac->migratetype,
2480	NULL);
2481	if (ret) {
2482	spin_unlock_irqrestore(&zone->lock, flags);
2483	return ret;
2484	}
2485	}
2486	spin_unlock_irqrestore(&zone->lock, flags);
2487	}
2488
2489	return false;
2490	}
2491
2492	/*
2493	* Try finding a free buddy page on the fallback list and put it on the free
2494	* list of requested migratetype, possibly along with other pages from the same
2495	* block, depending on fragmentation avoidance heuristics. Returns true if
2496	* fallback was found so that __rmqueue_smallest() can grab it.
2497	*
2498	* The use of signed ints for order and current_order is a deliberate
2499	* deviation from the rest of this file, to make the for loop
2500	* condition simpler.
2501	*/
2502	static __always_inline bool
2503	__rmqueue_fallback(struct zone zone, int* order, int start_migratetype,
2504	unsigned int alloc_flags)
2505	{
2506	struct free_area *area;
2507	int current_order;
2508	int min_order = order;
2509	struct page *page;
2510	int fallback_mt;
2511	bool can_steal;
2512
2513	/*
2514	* Do not steal pages from freelists belonging to other pageblocks
2515	* i.e. orders < pageblock_order. If there are no local zones free,
2516	* the zonelists will be reiterated without ALLOC_NOFRAGMENT.
2517	*/
2518	if (alloc_flags & ALLOC_NOFRAGMENT)
2519	min_order = pageblock_order;
2520
2521	/*
2522	* Find the largest available free page in the other list. This roughly
2523	* approximates finding the pageblock with the most free pages, which
2524	* would be too costly to do exactly.
2525	*/
2526	for (current_order = MAX_ORDER - `1`; current_order >= min_order;
2527	--current_order) {
2528	area = &(zone->free_area[current_order]);
2529	fallback_mt = find_suitable_fallback(area, current_order,
2530	start_migratetype, false, &can_steal);
2531	if (fallback_mt == -`1`)
2532	continue;
2533
2534	/*
2535	* We cannot steal all free pages from the pageblock and the
2536	* requested migratetype is movable. In that case it's better to
2537	* steal and split the smallest available page instead of the
2538	* largest available page, because even if the next movable
2539	* allocation falls back into a different pageblock than this
2540	* one, it won't cause permanent fragmentation.
2541	*/
2542	if (!can_steal && start_migratetype == MIGRATE_MOVABLE
2543	&& current_order > order)
2544	goto find_smallest;
2545
2546	goto do_steal;
2547	}
2548
2549	return false;
2550
2551	find_smallest:
2552	for (current_order = order; current_order < MAX_ORDER;
2553	current_order++) {
2554	area = &(zone->free_area[current_order]);
2555	fallback_mt = find_suitable_fallback(area, current_order,
2556	start_migratetype, false, &can_steal);
2557	if (fallback_mt != -`1`)
2558	break;
2559	}
2560
2561	/*
2562	* This should not happen - we already found a suitable fallback
2563	* when looking for the largest page.
2564	*/
2565	VM_BUG_ON(current_order == MAX_ORDER);
2566
2567	do_steal:
2568	page = list_first_entry(&area->free_list[fallback_mt],
2569	struct page, lru);
2570
2571	steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
2572	can_steal);
2573
2574	trace_mm_page_alloc_extfrag(page, order, current_order,
2575	start_migratetype, fallback_mt);
2576
2577	return true;
2578
2579	}
2580
2581	/*
2582	* Do the hard work of removing an element from the buddy allocator.
2583	* Call me with the zone->lock already held.
2584	*/
2585	static __always_inline struct page *
2586	__rmqueue(struct zone zone, unsigned* int order, int migratetype,
2587	unsigned int alloc_flags)
2588	{
2589	struct page *page;
2590
2591	retry:
2592	page = __rmqueue_smallest(zone, order, migratetype);
2593	if (unlikely(!page)) {
2594	if (migratetype == MIGRATE_MOVABLE)
2595	page = __rmqueue_cma_fallback(zone, order);
2596
2597	if (!page && __rmqueue_fallback(zone, order, migratetype,
2598	alloc_flags))
2599	goto retry;
2600	}
2601
2602	trace_mm_page_alloc_zone_locked(page, order, migratetype);
2603	return page;
2604	}
2605
2606	/*
2607	* Obtain a specified number of elements from the buddy allocator, all under
2608	* a single hold of the lock, for efficiency. Add them to the supplied list.
2609	* Returns the number of new pages which were placed at *list.
2610	*/
2611	static int rmqueue_bulk(struct zone zone, unsigned* int order,
2612	unsigned long count, struct list_head *list,
2613	int migratetype, unsigned int alloc_flags)
2614	{
2615	int i, alloced = `0`;
2616
2617	spin_lock(&zone->lock);
2618	for (i = `0`; i < count; ++i) {
2619	struct page *page = __rmqueue(zone, order, migratetype,
2620	alloc_flags);
2621	if (unlikely(page == NULL))
2622	break;
2623
2624	if (unlikely(check_pcp_refill(page)))
2625	continue;
2626
2627	/*
2628	* Split buddy pages returned by expand() are received here in
2629	* physical page order. The page is added to the tail of
2630	* caller's list. From the callers perspective, the linked list
2631	* is ordered by page number under some conditions. This is
2632	* useful for IO devices that can forward direction from the
2633	* head, thus also in the physical page order. This is useful
2634	* for IO devices that can merge IO requests if the physical
2635	* pages are ordered properly.
2636	*/
2637	list_add_tail(&page->lru, list);
2638	alloced++;
2639	if (is_migrate_cma(get_pcppage_migratetype(page)))
2640	__mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
2641	-(`1` << order));
2642	}
2643
2644	/*
2645	* i pages were removed from the buddy list even if some leak due
2646	* to check_pcp_refill failing so adjust NR_FREE_PAGES based
2647	* on i. Do not confuse with 'alloced' which is the number of
2648	* pages added to the pcp list.
2649	*/
2650	__mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
2651	spin_unlock(&zone->lock);
2652	return alloced;
2653	}
2654
2655	#ifdef CONFIG_NUMA
2656	/*
2657	* Called from the vmstat counter updater to drain pagesets of this
2658	* currently executing processor on remote nodes after they have
2659	* expired.
2660	*
2661	* Note that this function must be called with the thread pinned to
2662	* a single processor.
2663	*/
2664	void drain_zone_pages(struct zone zone, struct* per_cpu_pages *pcp)
2665	{
2666	unsigned long flags;
2667	int to_drain, batch;
2668
2669	local_irq_save(flags);
2670	batch = READ_ONCE(pcp->batch);
2671	to_drain = min(pcp->count, batch);
2672	if (to_drain > `0`)
2673	free_pcppages_bulk(zone, to_drain, pcp);
2674	local_irq_restore(flags);
2675	}
2676	#endif
2677
2678	/*
2679	* Drain pcplists of the indicated processor and zone.
2680	*
2681	* The processor must either be the current processor and the
2682	* thread pinned to the current processor or a processor that
2683	* is not online.
2684	*/
2685	static void drain_pages_zone(unsigned int cpu, struct zone *zone)
2686	{
2687	unsigned long flags;
2688	struct per_cpu_pageset *pset;
2689	struct per_cpu_pages *pcp;
2690
2691	local_irq_save(flags);
2692	pset = per_cpu_ptr(zone->pageset, cpu);
2693
2694	pcp = &pset->pcp;
2695	if (pcp->count)
2696	free_pcppages_bulk(zone, pcp->count, pcp);
2697	local_irq_restore(flags);
2698	}
2699
2700	/*
2701	* Drain pcplists of all zones on the indicated processor.
2702	*
2703	* The processor must either be the current processor and the
2704	* thread pinned to the current processor or a processor that
2705	* is not online.
2706	*/
2707	static void drain_pages(unsigned int cpu)
2708	{
2709	struct zone *zone;
2710
2711	for_each_populated_zone(zone) {
2712	drain_pages_zone(cpu, zone);
2713	}
2714	}
2715
2716	/*
2717	* Spill all of this CPU's per-cpu pages back into the buddy allocator.
2718	*
2719	* The CPU has to be pinned. When zone parameter is non-NULL, spill just
2720	* the single zone's pages.
2721	*/
2722	void drain_local_pages(struct zone *zone)
2723	{
2724	int cpu = smp_processor_id();
2725
2726	if (zone)
2727	drain_pages_zone(cpu, zone);
2728	else
2729	drain_pages(cpu);
2730	}
2731
2732	static void drain_local_pages_wq(struct work_struct *work)
2733	{
2734	struct pcpu_drain *drain;
2735
2736	drain = container_of(work, struct pcpu_drain, work);
2737
2738	/*
2739	* drain_all_pages doesn't use proper cpu hotplug protection so
2740	* we can race with cpu offline when the WQ can move this from
2741	* a cpu pinned worker to an unbound one. We can operate on a different
2742	* cpu which is allright but we also have to make sure to not move to
2743	* a different one.
2744	*/
2745	preempt_disable();
2746	drain_local_pages(drain->zone);
2747	preempt_enable();
2748	}
2749
2750	/*
2751	* Spill all the per-cpu pages from all CPUs back into the buddy allocator.
2752	*
2753	* When zone parameter is non-NULL, spill just the single zone's pages.
2754	*
2755	* Note that this can be extremely slow as the draining happens in a workqueue.
2756	*/
2757	void drain_all_pages(struct zone *zone)
2758	{
2759	int cpu;
2760
2761	/*
2762	* Allocate in the BSS so we wont require allocation in
2763	* direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
2764	*/
2765	static cpumask_t cpus_with_pcps;
2766
2767	/*
2768	* Make sure nobody triggers this path before mm_percpu_wq is fully
2769	* initialized.
2770	*/
2771	if (WARN_ON_ONCE(!mm_percpu_wq))
2772	return;
2773
2774	/*
2775	* Do not drain if one is already in progress unless it's specific to
2776	* a zone. Such callers are primarily CMA and memory hotplug and need
2777	* the drain to be complete when the call returns.
2778	*/
2779	if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
2780	if (!zone)
2781	return;
2782	mutex_lock(&pcpu_drain_mutex);
2783	}
2784
2785	/*
2786	* We don't care about racing with CPU hotplug event
2787	* as offline notification will cause the notified
2788	* cpu to drain that CPU pcps and on_each_cpu_mask
2789	* disables preemption as part of its processing
2790	*/
2791	for_each_online_cpu(cpu) {
2792	struct per_cpu_pageset *pcp;
2793	struct zone *z;
2794	bool has_pcps = false;
2795
2796	if (zone) {
2797	pcp = per_cpu_ptr(zone->pageset, cpu);
2798	if (pcp->pcp.count)
2799	has_pcps = true;
2800	} else {
2801	for_each_populated_zone(z) {
2802	pcp = per_cpu_ptr(z->pageset, cpu);
2803	if (pcp->pcp.count) {
2804	has_pcps = true;
2805	break;
2806	}
2807	}
2808	}
2809
2810	if (has_pcps)
2811	cpumask_set_cpu(cpu, &cpus_with_pcps);
2812	else
2813	cpumask_clear_cpu(cpu, &cpus_with_pcps);
2814	}
2815
2816	for_each_cpu(cpu, &cpus_with_pcps) {
2817	struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu);
2818
2819	drain->zone = zone;
2820	INIT_WORK(&drain->work, drain_local_pages_wq);
2821	queue_work_on(cpu, mm_percpu_wq, &drain->work);
2822	}
2823	for_each_cpu(cpu, &cpus_with_pcps)
2824	flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work);
2825
2826	mutex_unlock(&pcpu_drain_mutex);
2827	}
2828
2829	#ifdef CONFIG_HIBERNATION
2830
2831	/*
2832	* Touch the watchdog for every WD_PAGE_COUNT pages.
2833	*/
2834	#define WD_PAGE_COUNT (128*1024)
2835
2836	void mark_free_pages(struct zone *zone)
2837	{
2838	unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
2839	unsigned long flags;
2840	unsigned int order, t;
2841	struct page *page;
2842
2843	if (zone_is_empty(zone))
2844	return;
2845
2846	spin_lock_irqsave(&zone->lock, flags);
2847
2848	max_zone_pfn = zone_end_pfn(zone);
2849	for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
2850	if (pfn_valid(pfn)) {
2851	page = pfn_to_page(pfn);
2852
2853	if (!--page_count) {
2854	touch_nmi_watchdog();
2855	page_count = WD_PAGE_COUNT;
2856	}
2857
2858	if (page_zone(page) != zone)
2859	continue;
2860
2861	if (!swsusp_page_is_forbidden(page))
2862	swsusp_unset_page_free(page);
2863	}
2864
2865	for_each_migratetype_order(order, t) {
2866	list_for_each_entry(page,
2867	&zone->free_area[order].free_list[t], lru) {
2868	unsigned long i;
2869
2870	pfn = page_to_pfn(page);
2871	for (i = `0`; i < (`1UL` << order); i++) {
2872	if (!--page_count) {
2873	touch_nmi_watchdog();
2874	page_count = WD_PAGE_COUNT;
2875	}
2876	swsusp_set_page_free(pfn_to_page(pfn + i));
2877	}
2878	}
2879	}
2880	spin_unlock_irqrestore(&zone->lock, flags);
2881	}
2882	#endif /* CONFIG_PM */
2883
2884	static bool free_unref_page_prepare(struct page page, unsigned* long pfn)
2885	{
2886	int migratetype;
2887
2888	if (!free_pcp_prepare(page))
2889	return false;
2890
2891	migratetype = get_pfnblock_migratetype(page, pfn);
2892	set_pcppage_migratetype(page, migratetype);
2893	return true;
2894	}
2895
2896	static void free_unref_page_commit(struct page page, unsigned* long pfn)
2897	{
2898	struct zone *zone = page_zone(page);
2899	struct per_cpu_pages *pcp;
2900	int migratetype;
2901
2902	migratetype = get_pcppage_migratetype(page);
2903	__count_vm_event(PGFREE);
2904
2905	/*
2906	* We only track unmovable, reclaimable and movable on pcp lists.
2907	* Free ISOLATE pages back to the allocator because they are being
2908	* offlined but treat HIGHATOMIC as movable pages so we can get those
2909	* areas back if necessary. Otherwise, we may have to free
2910	* excessively into the page allocator
2911	*/
2912	if (migratetype >= MIGRATE_PCPTYPES) {
2913	if (unlikely(is_migrate_isolate(migratetype))) {
2914	free_one_page(zone, page, pfn, `0`, migratetype);
2915	return;
2916	}
2917	migratetype = MIGRATE_MOVABLE;
2918	}
2919
2920	pcp = &this_cpu_ptr(zone->pageset)->pcp;
2921	list_add(&page->lru, &pcp->lists[migratetype]);
2922	pcp->count++;
2923	if (pcp->count >= pcp->high) {
2924	unsigned long batch = READ_ONCE(pcp->batch);
2925	free_pcppages_bulk(zone, batch, pcp);
2926	}
2927	}
2928
2929	/*
2930	* Free a 0-order page
2931	*/
2932	void free_unref_page(struct page *page)
2933	{
2934	unsigned long flags;
2935	unsigned long pfn = page_to_pfn(page);
2936
2937	if (!free_unref_page_prepare(page, pfn))
2938	return;
2939
2940	local_irq_save(flags);
2941	free_unref_page_commit(page, pfn);
2942	local_irq_restore(flags);
2943	}
2944
2945	/*
2946	* Free a list of 0-order pages
2947	*/
2948	void free_unref_page_list(struct list_head *list)
2949	{
2950	struct page page, next;
2951	unsigned long flags, pfn;
2952	int batch_count = `0`;
2953
2954	/ Prepare pages for freeing /
2955	list_for_each_entry_safe(page, next, list, lru) {
2956	pfn = page_to_pfn(page);
2957	if (!free_unref_page_prepare(page, pfn))
2958	list_del(&page->lru);
2959	set_page_private(page, pfn);
2960	}
2961
2962	local_irq_save(flags);
2963	list_for_each_entry_safe(page, next, list, lru) {
2964	unsigned long pfn = page_private(page);
2965
2966	set_page_private(page, `0`);
2967	trace_mm_page_free_batched(page);
2968	free_unref_page_commit(page, pfn);
2969
2970	/*
2971	* Guard against excessive IRQ disabled times when we get
2972	* a large list of pages to free.
2973	*/
2974	if (++batch_count == SWAP_CLUSTER_MAX) {
2975	local_irq_restore(flags);
2976	batch_count = `0`;
2977	local_irq_save(flags);
2978	}
2979	}
2980	local_irq_restore(flags);
2981	}
2982
2983	/*
2984	* split_page takes a non-compound higher-order page, and splits it into
2985	* n (1<<order) sub-pages: page[0..n]
2986	* Each sub-page must be freed individually.
2987	*
2988	* Note: this is probably too low level an operation for use in drivers.
2989	* Please consult with lkml before using this in your driver.
2990	*/
2991	void split_page(struct page page, unsigned* int order)
2992	{
2993	int i;
2994
2995	VM_BUG_ON_PAGE(PageCompound(page), page);
2996	VM_BUG_ON_PAGE(!page_count(page), page);
2997
2998	for (i = `1`; i < (`1` << order); i++)
2999	set_page_refcounted(page + i);
3000	split_page_owner(page, order);
3001	}
3002	EXPORT_SYMBOL_GPL(split_page);
3003
3004	int __isolate_free_page(struct page page, unsigned* int order)
3005	{
3006	unsigned long watermark;
3007	struct zone *zone;
3008	int mt;
3009
3010	BUG_ON(!PageBuddy(page));
3011
3012	zone = page_zone(page);
3013	mt = get_pageblock_migratetype(page);
3014
3015	if (!is_migrate_isolate(mt)) {
3016	/*
3017	* Obey watermarks as if the page was being allocated. We can
3018	* emulate a high-order watermark check with a raised order-0
3019	* watermark, because we already know our high-order page
3020	* exists.
3021	*/
3022	watermark = zone->_watermark[WMARK_MIN] + (`1UL` << order);
3023	if (!zone_watermark_ok(zone, `0`, watermark, `0`, ALLOC_CMA))
3024	return `0`;
3025
3026	__mod_zone_freepage_state(zone, -(`1UL` << order), mt);
3027	}
3028
3029	/ Remove page from free list /
3030	list_del(&page->lru);
3031	zone->free_area[order].nr_free--;
3032	rmv_page_order(page);
3033
3034	/*
3035	* Set the pageblock if the isolated page is at least half of a
3036	* pageblock
3037	*/
3038	if (order >= pageblock_order - `1`) {
3039	struct page *endpage = page + (`1` << order) - `1`;
3040	for (; page < endpage; page += pageblock_nr_pages) {
3041	int mt = get_pageblock_migratetype(page);
3042	if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
3043	&& !is_migrate_highatomic(mt))
3044	set_pageblock_migratetype(page,
3045	MIGRATE_MOVABLE);
3046	}
3047	}
3048
3049
3050	return `1UL` << order;
3051	}
3052
3053	/*
3054	* Update NUMA hit/miss statistics
3055	*
3056	* Must be called with interrupts disabled.
3057	*/
3058	static inline void zone_statistics(struct zone preferred_zone, struct* zone *z)
3059	{
3060	#ifdef CONFIG_NUMA
3061	enum numa_stat_item local_stat = NUMA_LOCAL;
3062
3063	/ skip numa counters update if numa stats is disabled /
3064	if (!static_branch_likely(&vm_numa_stat_key))
3065	return;
3066
3067	if (zone_to_nid(z) != numa_node_id())
3068	local_stat = NUMA_OTHER;
3069
3070	if (zone_to_nid(z) == zone_to_nid(preferred_zone))
3071	__inc_numa_state(z, NUMA_HIT);
3072	else {
3073	__inc_numa_state(z, NUMA_MISS);
3074	__inc_numa_state(preferred_zone, NUMA_FOREIGN);
3075	}
3076	__inc_numa_state(z, local_stat);
3077	#endif
3078	}
3079
3080	/ Remove page from the per-cpu list, caller must protect the list /
3081	static struct page __rmqueue_pcplist(struct* zone zone, int* migratetype,
3082	unsigned int alloc_flags,
3083	struct per_cpu_pages *pcp,
3084	struct list_head *list)
3085	{
3086	struct page *page;
3087
3088	do {
3089	if (list_empty(list)) {
3090	pcp->count += rmqueue_bulk(zone, `0`,
3091	pcp->batch, list,
3092	migratetype, alloc_flags);
3093	if (unlikely(list_empty(list)))
3094	return NULL;
3095	}
3096
3097	page = list_first_entry(list, struct page, lru);
3098	list_del(&page->lru);
3099	pcp->count--;
3100	} while (check_new_pcp(page));
3101
3102	return page;
3103	}
3104
3105	/ Lock and remove page from the per-cpu list /
3106	static struct page rmqueue_pcplist(struct* zone *preferred_zone,
3107	struct zone zone, unsigned* int order,
3108	gfp_t gfp_flags, int migratetype,
3109	unsigned int alloc_flags)
3110	{
3111	struct per_cpu_pages *pcp;
3112	struct list_head *list;
3113	struct page *page;
3114	unsigned long flags;
3115
3116	local_irq_save(flags);
3117	pcp = &this_cpu_ptr(zone->pageset)->pcp;
3118	list = &pcp->lists[migratetype];
3119	page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list);
3120	if (page) {
3121	__count_zid_vm_events(PGALLOC, page_zonenum(page), `1` << order);
3122	zone_statistics(preferred_zone, zone);
3123	}
3124	local_irq_restore(flags);
3125	return page;
3126	}
3127
3128	/*
3129	* Allocate a page from the given zone. Use pcplists for order-0 allocations.
3130	*/
3131	static inline
3132	struct page rmqueue(struct* zone *preferred_zone,
3133	struct zone zone, unsigned* int order,
3134	gfp_t gfp_flags, unsigned int alloc_flags,
3135	int migratetype)
3136	{
3137	unsigned long flags;
3138	struct page *page;
3139
3140	if (likely(order == `0`)) {
3141	page = rmqueue_pcplist(preferred_zone, zone, order,
3142	gfp_flags, migratetype, alloc_flags);
3143	goto out;
3144	}
3145
3146	/*
3147	* We most definitely don't want callers attempting to
3148	* allocate greater than order-1 page units with __GFP_NOFAIL.
3149	*/
3150	WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > `1`));
3151	spin_lock_irqsave(&zone->lock, flags);
3152
3153	do {
3154	page = NULL;
3155	if (alloc_flags & ALLOC_HARDER) {
3156	page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
3157	if (page)
3158	trace_mm_page_alloc_zone_locked(page, order, migratetype);
3159	}
3160	if (!page)
3161	page = __rmqueue(zone, order, migratetype, alloc_flags);
3162	} while (page && check_new_pages(page, order));
3163	spin_unlock(&zone->lock);
3164	if (!page)
3165	goto failed;
3166	__mod_zone_freepage_state(zone, -(`1` << order),
3167	get_pcppage_migratetype(page));
3168
3169	__count_zid_vm_events(PGALLOC, page_zonenum(page), `1` << order);
3170	zone_statistics(preferred_zone, zone);
3171	local_irq_restore(flags);
3172
3173	out:
3174	/ Separate test+clear to avoid unnecessary atomics /
3175	if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) {
3176	clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
3177	wakeup_kswapd(zone, `0`, `0`, zone_idx(zone));
3178	}
3179
3180	VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
3181	return page;
3182
3183	failed:
3184	local_irq_restore(flags);
3185	return NULL;
3186	}
3187
3188	#ifdef CONFIG_FAIL_PAGE_ALLOC
3189
3190	static struct {
3191	struct fault_attr attr;
3192
3193	bool ignore_gfp_highmem;
3194	bool ignore_gfp_reclaim;
3195	u32 min_order;
3196	} fail_page_alloc = {
3197	.attr = FAULT_ATTR_INITIALIZER,
3198	.ignore_gfp_reclaim = true,
3199	.ignore_gfp_highmem = true,
3200	.min_order = `1`,
3201	};
3202
3203	static int __init setup_fail_page_alloc(char *str)
3204	{
3205	return setup_fault_attr(&fail_page_alloc.attr, str);
3206	}
3207	__setup("fail_page_alloc=", setup_fail_page_alloc);
3208
3209	static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3210	{
3211	if (order < fail_page_alloc.min_order)
3212	return false;
3213	if (gfp_mask & __GFP_NOFAIL)
3214	return false;
3215	if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
3216	return false;
3217	if (fail_page_alloc.ignore_gfp_reclaim &&
3218	(gfp_mask & __GFP_DIRECT_RECLAIM))
3219	return false;
3220
3221	return should_fail(&fail_page_alloc.attr, `1` << order);
3222	}
3223
3224	#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
3225
3226	static int __init fail_page_alloc_debugfs(void)
3227	{
3228	umode_t mode = S_IFREG \| `0600`;
3229	struct dentry *dir;
3230
3231	dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
3232	&fail_page_alloc.attr);
3233
3234	debugfs_create_bool("ignore-gfp-wait", mode, dir,
3235	&fail_page_alloc.ignore_gfp_reclaim);
3236	debugfs_create_bool("ignore-gfp-highmem", mode, dir,
3237	&fail_page_alloc.ignore_gfp_highmem);
3238	debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order);
3239
3240	return `0`;
3241	}
3242
3243	late_initcall(fail_page_alloc_debugfs);
3244
3245	#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
3246
3247	#else /* CONFIG_FAIL_PAGE_ALLOC */
3248
3249	static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3250	{
3251	return false;
3252	}
3253
3254	#endif /* CONFIG_FAIL_PAGE_ALLOC */
3255
3256	static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3257	{
3258	return __should_fail_alloc_page(gfp_mask, order);
3259	}
3260	ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE);
3261
3262	/*
3263	* Return true if free base pages are above 'mark'. For high-order checks it
3264	* will return true of the order-0 watermark is reached and there is at least
3265	* one free page of a suitable size. Checking now avoids taking the zone lock
3266	* to check in the allocation paths if no pages are free.
3267	*/
3268	bool __zone_watermark_ok(struct zone z, unsigned* int order, unsigned long mark,
3269	int classzone_idx, unsigned int alloc_flags,
3270	long free_pages)
3271	{
3272	long min = mark;
3273	int o;
3274	const bool alloc_harder = (alloc_flags & (ALLOC_HARDER\|ALLOC_OOM));
3275
3276	/ free_pages may go negative - that's OK /
3277	free_pages -= (`1` << order) - `1`;
3278
3279	if (alloc_flags & ALLOC_HIGH)
3280	min -= min / `2`;
3281
3282	/*
3283	* If the caller does not have rights to ALLOC_HARDER then subtract
3284	* the high-atomic reserves. This will over-estimate the size of the
3285	* atomic reserve but it avoids a search.
3286	*/
3287	if (likely(!alloc_harder)) {
3288	free_pages -= z->nr_reserved_highatomic;
3289	} else {
3290	/*
3291	* OOM victims can try even harder than normal ALLOC_HARDER
3292	* users on the grounds that it's definitely going to be in
3293	* the exit path shortly and free memory. Any allocation it
3294	* makes during the free path will be small and short-lived.
3295	*/
3296	if (alloc_flags & ALLOC_OOM)
3297	min -= min / `2`;
3298	else
3299	min -= min / `4`;
3300	}
3301
3302
3303	#ifdef CONFIG_CMA
3304	/ If allocation can't use CMA areas don't use free CMA pages /
3305	if (!(alloc_flags & ALLOC_CMA))
3306	free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
3307	#endif
3308
3309	/*
3310	* Check watermarks for an order-0 allocation request. If these
3311	* are not met, then a high-order request also cannot go ahead
3312	* even if a suitable page happened to be free.
3313	*/
3314	if (free_pages <= min + z->lowmem_reserve[classzone_idx])
3315	return false;
3316
3317	/ If this is an order-0 request then the watermark is fine /
3318	if (!order)
3319	return true;
3320
3321	/ For a high-order request, check at least one suitable page is free /
3322	for (o = order; o < MAX_ORDER; o++) {
3323	struct free_area *area = &z->free_area[o];
3324	int mt;
3325
3326	if (!area->nr_free)
3327	continue;
3328
3329	for (mt = `0`; mt < MIGRATE_PCPTYPES; mt++) {
3330	if (!list_empty(&area->free_list[mt]))
3331	return true;
3332	}
3333
3334	#ifdef CONFIG_CMA
3335	if ((alloc_flags & ALLOC_CMA) &&
3336	!list_empty(&area->free_list[MIGRATE_CMA])) {
3337	return true;
3338	}
3339	#endif
3340	if (alloc_harder &&
3341	!list_empty(&area->free_list[MIGRATE_HIGHATOMIC]))
3342	return true;
3343	}
3344	return false;
3345	}
3346
3347	bool zone_watermark_ok(struct zone z, unsigned* int order, unsigned long mark,
3348	int classzone_idx, unsigned int alloc_flags)
3349	{
3350	return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
3351	zone_page_state(z, NR_FREE_PAGES));
3352	}
3353
3354	static inline bool zone_watermark_fast(struct zone z, unsigned* int order,
3355	unsigned long mark, int classzone_idx, unsigned int alloc_flags)
3356	{
3357	long free_pages = zone_page_state(z, NR_FREE_PAGES);
3358	long cma_pages = `0`;
3359
3360	#ifdef CONFIG_CMA
3361	/ If allocation can't use CMA areas don't use free CMA pages /
3362	if (!(alloc_flags & ALLOC_CMA))
3363	cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES);
3364	#endif
3365
3366	/*
3367	* Fast check for order-0 only. If this fails then the reserves
3368	* need to be calculated. There is a corner case where the check
3369	* passes but only the high-order atomic reserve are free. If
3370	* the caller is !atomic then it'll uselessly search the free
3371	* list. That corner case is then slower but it is harmless.
3372	*/
3373	if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx])
3374	return true;
3375
3376	return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
3377	free_pages);
3378	}
3379
3380	bool zone_watermark_ok_safe(struct zone z, unsigned* int order,
3381	unsigned long mark, int classzone_idx)
3382	{
3383	long free_pages = zone_page_state(z, NR_FREE_PAGES);
3384
3385	if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
3386	free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
3387
3388	return __zone_watermark_ok(z, order, mark, classzone_idx, `0`,
3389	free_pages);
3390	}
3391
3392	#ifdef CONFIG_NUMA
3393	static bool zone_allows_reclaim(struct zone local_zone, struct* zone *zone)
3394	{
3395	return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
3396	RECLAIM_DISTANCE;
3397	}
3398	#else /* CONFIG_NUMA */
3399	static bool zone_allows_reclaim(struct zone local_zone, struct* zone *zone)
3400	{
3401	return true;
3402	}
3403	#endif /* CONFIG_NUMA */
3404
3405	/*
3406	* The restriction on ZONE_DMA32 as being a suitable zone to use to avoid
3407	* fragmentation is subtle. If the preferred zone was HIGHMEM then
3408	* premature use of a lower zone may cause lowmem pressure problems that
3409	* are worse than fragmentation. If the next zone is ZONE_DMA then it is
3410	* probably too small. It only makes sense to spread allocations to avoid
3411	* fragmentation between the Normal and DMA32 zones.
3412	*/
3413	static inline unsigned int
3414	alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
3415	{
3416	unsigned int alloc_flags = `0`;
3417
3418	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
3419	alloc_flags \|= ALLOC_KSWAPD;
3420
3421	#ifdef CONFIG_ZONE_DMA32
3422	if (zone_idx(zone) != ZONE_NORMAL)
3423	goto out;
3424
3425	/*
3426	* If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and
3427	* the pointer is within zone->zone_pgdat->node_zones[]. Also assume
3428	* on UMA that if Normal is populated then so is DMA32.
3429	*/
3430	BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != `1`);
3431	if (nr_online_nodes > `1` && !populated_zone(--zone))
3432	goto out;
3433
3434	out:
3435	#endif /* CONFIG_ZONE_DMA32 */
3436	return alloc_flags;
3437	}
3438
3439	/*
3440	* get_page_from_freelist goes through the zonelist trying to allocate
3441	* a page.
3442	*/
3443	static struct page *
3444	get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
3445	const struct alloc_context *ac)
3446	{
3447	struct zoneref *z;
3448	struct zone *zone;
3449	struct pglist_data *last_pgdat_dirty_limit = NULL;
3450	bool no_fallback;
3451
3452	retry:
3453	/*
3454	* Scan zonelist, looking for a zone with enough free.
3455	* See also __cpuset_node_allowed() comment in kernel/cpuset.c.
3456	*/
3457	no_fallback = alloc_flags & ALLOC_NOFRAGMENT;
3458	z = ac->preferred_zoneref;
3459	for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3460	ac->nodemask) {
3461	struct page *page;
3462	unsigned long mark;
3463
3464	if (cpusets_enabled() &&
3465	(alloc_flags & ALLOC_CPUSET) &&
3466	!__cpuset_zone_allowed(zone, gfp_mask))
3467	continue;
3468	/*
3469	* When allocating a page cache page for writing, we
3470	* want to get it from a node that is within its dirty
3471	* limit, such that no single node holds more than its
3472	* proportional share of globally allowed dirty pages.
3473	* The dirty limits take into account the node's
3474	* lowmem reserves and high watermark so that kswapd
3475	* should be able to balance it without having to
3476	* write pages from its LRU list.
3477	*
3478	* XXX: For now, allow allocations to potentially
3479	* exceed the per-node dirty limit in the slowpath
3480	* (spread_dirty_pages unset) before going into reclaim,
3481	* which is important when on a NUMA setup the allowed
3482	* nodes are together not big enough to reach the
3483	* global limit. The proper fix for these situations
3484	* will require awareness of nodes in the
3485	* dirty-throttling and the flusher threads.
3486	*/
3487	if (ac->spread_dirty_pages) {
3488	if (last_pgdat_dirty_limit == zone->zone_pgdat)
3489	continue;
3490
3491	if (!node_dirty_ok(zone->zone_pgdat)) {
3492	last_pgdat_dirty_limit = zone->zone_pgdat;
3493	continue;
3494	}
3495	}
3496
3497	if (no_fallback && nr_online_nodes > `1` &&
3498	zone != ac->preferred_zoneref->zone) {
3499	int local_nid;
3500
3501	/*
3502	* If moving to a remote node, retry but allow
3503	* fragmenting fallbacks. Locality is more important
3504	* than fragmentation avoidance.
3505	*/
3506	local_nid = zone_to_nid(ac->preferred_zoneref->zone);
3507	if (zone_to_nid(zone) != local_nid) {
3508	alloc_flags &= ~ALLOC_NOFRAGMENT;
3509	goto retry;
3510	}
3511	}
3512
3513	mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
3514	if (!zone_watermark_fast(zone, order, mark,
3515	ac_classzone_idx(ac), alloc_flags)) {
3516	int ret;
3517
3518	#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3519	/*
3520	* Watermark failed for this zone, but see if we can
3521	* grow this zone if it contains deferred pages.
3522	*/
3523	if (static_branch_unlikely(&deferred_pages)) {
3524	if (_deferred_grow_zone(zone, order))
3525	goto try_this_zone;
3526	}
3527	#endif
3528	/ Checked here to keep the fast path fast /
3529	BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
3530	if (alloc_flags & ALLOC_NO_WATERMARKS)
3531	goto try_this_zone;
3532
3533	if (node_reclaim_mode == `0` \|\|
3534	!zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
3535	continue;
3536
3537	ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
3538	switch (ret) {
3539	case NODE_RECLAIM_NOSCAN:
3540	/ did not scan /
3541	continue;
3542	case NODE_RECLAIM_FULL:
3543	/ scanned but unreclaimable /
3544	continue;
3545	default:
3546	/ did we reclaim enough /
3547	if (zone_watermark_ok(zone, order, mark,
3548	ac_classzone_idx(ac), alloc_flags))
3549	goto try_this_zone;
3550
3551	continue;
3552	}
3553	}
3554
3555	try_this_zone:
3556	page = rmqueue(ac->preferred_zoneref->zone, zone, order,
3557	gfp_mask, alloc_flags, ac->migratetype);
3558	if (page) {
3559	prep_new_page(page, order, gfp_mask, alloc_flags);
3560
3561	/*
3562	* If this is a high-order atomic allocation then check
3563	* if the pageblock should be reserved for the future
3564	*/
3565	if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
3566	reserve_highatomic_pageblock(page, zone, order);
3567
3568	return page;
3569	} else {
3570	#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3571	/ Try again if zone has deferred pages /
3572	if (static_branch_unlikely(&deferred_pages)) {
3573	if (_deferred_grow_zone(zone, order))
3574	goto try_this_zone;
3575	}
3576	#endif
3577	}
3578	}
3579
3580	/*
3581	* It's possible on a UMA machine to get through all zones that are
3582	* fragmented. If avoiding fragmentation, reset and try again.
3583	*/
3584	if (no_fallback) {
3585	alloc_flags &= ~ALLOC_NOFRAGMENT;
3586	goto retry;
3587	}
3588
3589	return NULL;
3590	}
3591
3592	static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
3593	{
3594	unsigned int filter = SHOW_MEM_FILTER_NODES;
3595	static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, `1`);
3596
3597	if (!__ratelimit(&show_mem_rs))
3598	return;
3599
3600	/*
3601	* This documents exceptions given to allocations in certain
3602	* contexts that are allowed to allocate outside current's set
3603	* of allowed nodes.
3604	*/
3605	if (!(gfp_mask & __GFP_NOMEMALLOC))
3606	if (tsk_is_oom_victim(current) \|\|
3607	(current->flags & (PF_MEMALLOC \| PF_EXITING)))
3608	filter &= ~SHOW_MEM_FILTER_NODES;
3609	if (in_interrupt() \|\| !(gfp_mask & __GFP_DIRECT_RECLAIM))
3610	filter &= ~SHOW_MEM_FILTER_NODES;
3611
3612	show_mem(filter, nodemask);
3613	}
3614
3615	void warn_alloc(gfp_t gfp_mask, nodemask_t nodemask, const* char *fmt, ...)
3616	{
3617	struct va_format vaf;
3618	va_list args;
3619	static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL,
3620	DEFAULT_RATELIMIT_BURST);
3621
3622	if ((gfp_mask & __GFP_NOWARN) \|\| !__ratelimit(&nopage_rs))
3623	return;
3624
3625	va_start(args, fmt);
3626	vaf.fmt = fmt;
3627	vaf.va = &args;
3628	pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl",
3629	current->comm, &vaf, gfp_mask, &gfp_mask,
3630	nodemask_pr_args(nodemask));
3631	va_end(args);
3632
3633	cpuset_print_current_mems_allowed();
3634	pr_cont("\n");
3635	dump_stack();
3636	warn_alloc_show_mem(gfp_mask, nodemask);
3637	}
3638
3639	static inline struct page *
3640	__alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
3641	unsigned int alloc_flags,
3642	const struct alloc_context *ac)
3643	{
3644	struct page *page;
3645
3646	page = get_page_from_freelist(gfp_mask, order,
3647	alloc_flags\|ALLOC_CPUSET, ac);
3648	/*
3649	* fallback to ignore cpuset restriction if our nodes
3650	* are depleted
3651	*/
3652	if (!page)
3653	page = get_page_from_freelist(gfp_mask, order,
3654	alloc_flags, ac);
3655
3656	return page;
3657	}
3658
3659	static inline struct page *
3660	__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
3661	const struct alloc_context ac, unsigned* long *did_some_progress)
3662	{
3663	struct oom_control oc = {
3664	.zonelist = ac->zonelist,
3665	.nodemask = ac->nodemask,
3666	.memcg = NULL,
3667	.gfp_mask = gfp_mask,
3668	.order = order,
3669	};
3670	struct page *page;
3671
3672	*did_some_progress = `0`;
3673
3674	/*
3675	* Acquire the oom lock. If that fails, somebody else is
3676	* making progress for us.
3677	*/
3678	if (!mutex_trylock(&oom_lock)) {
3679	*did_some_progress = `1`;
3680	schedule_timeout_uninterruptible(`1`);
3681	return NULL;
3682	}
3683
3684	/*
3685	* Go through the zonelist yet one more time, keep very high watermark
3686	* here, this is only to catch a parallel oom killing, we must fail if
3687	* we're still under heavy pressure. But make sure that this reclaim
3688	* attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
3689	* allocation which will never fail due to oom_lock already held.
3690	*/
3691	page = get_page_from_freelist((gfp_mask \| __GFP_HARDWALL) &
3692	~__GFP_DIRECT_RECLAIM, order,
3693	ALLOC_WMARK_HIGH\|ALLOC_CPUSET, ac);
3694	if (page)
3695	goto out;
3696
3697	/ Coredumps can quickly deplete all memory reserves /
3698	if (current->flags & PF_DUMPCORE)
3699	goto out;
3700	/ The OOM killer will not help higher order allocs /
3701	if (order > PAGE_ALLOC_COSTLY_ORDER)
3702	goto out;
3703	/*
3704	* We have already exhausted all our reclaim opportunities without any
3705	* success so it is time to admit defeat. We will skip the OOM killer
3706	* because it is very likely that the caller has a more reasonable
3707	* fallback than shooting a random task.
3708	*/
3709	if (gfp_mask & __GFP_RETRY_MAYFAIL)
3710	goto out;
3711	/ The OOM killer does not needlessly kill tasks for lowmem /
3712	if (ac->high_zoneidx < ZONE_NORMAL)
3713	goto out;
3714	if (pm_suspended_storage())
3715	goto out;
3716	/*
3717	* XXX: GFP_NOFS allocations should rather fail than rely on
3718	* other request to make a forward progress.
3719	* We are in an unfortunate situation where out_of_memory cannot
3720	* do much for this context but let's try it to at least get
3721	* access to memory reserved if the current task is killed (see
3722	* out_of_memory). Once filesystems are ready to handle allocation
3723	* failures more gracefully we should just bail out here.
3724	*/
3725
3726	/ The OOM killer may not free memory on a specific node /
3727	if (gfp_mask & __GFP_THISNODE)
3728	goto out;
3729
3730	/ Exhausted what can be done so it's blame time /
3731	if (out_of_memory(&oc) \|\| WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
3732	*did_some_progress = `1`;
3733
3734	/*
3735	* Help non-failing allocations by giving them access to memory
3736	* reserves
3737	*/
3738	if (gfp_mask & __GFP_NOFAIL)
3739	page = __alloc_pages_cpuset_fallback(gfp_mask, order,
3740	ALLOC_NO_WATERMARKS, ac);
3741	}
3742	out:
3743	mutex_unlock(&oom_lock);
3744	return page;
3745	}
3746
3747	/*
3748	* Maximum number of compaction retries wit a progress before OOM
3749	* killer is consider as the only way to move forward.
3750	*/
3751	#define MAX_COMPACT_RETRIES 16
3752
3753	#ifdef CONFIG_COMPACTION
3754	/ Try memory compaction for high-order allocations before reclaim /
3755	static struct page *
3756	__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3757	unsigned int alloc_flags, const struct alloc_context *ac,
3758	enum compact_priority prio, enum compact_result *compact_result)
3759	{
3760	struct page *page = NULL;
3761	unsigned long pflags;
3762	unsigned int noreclaim_flag;
3763
3764	if (!order)
3765	return NULL;
3766
3767	psi_memstall_enter(&pflags);
3768	noreclaim_flag = memalloc_noreclaim_save();
3769
3770	*compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
3771	prio, &page);
3772
3773	memalloc_noreclaim_restore(noreclaim_flag);
3774	psi_memstall_leave(&pflags);
3775
3776	if (*compact_result <= COMPACT_INACTIVE) {
3777	WARN_ON_ONCE(page);
3778	return NULL;
3779	}
3780
3781	/*
3782	* At least in one zone compaction wasn't deferred or skipped, so let's
3783	* count a compaction stall
3784	*/
3785	count_vm_event(COMPACTSTALL);
3786
3787	/ Prep a captured page if available /
3788	if (page)
3789	prep_new_page(page, order, gfp_mask, alloc_flags);
3790
3791	/ Try get a page from the freelist if available /
3792	if (!page)
3793	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
3794
3795	if (page) {
3796	struct zone *zone = page_zone(page);
3797
3798	zone->compact_blockskip_flush = false;
3799	compaction_defer_reset(zone, order, true);
3800	count_vm_event(COMPACTSUCCESS);
3801	return page;
3802	}
3803
3804	/*
3805	* It's bad if compaction run occurs and fails. The most likely reason
3806	* is that pages exist, but not enough to satisfy watermarks.
3807	*/
3808	count_vm_event(COMPACTFAIL);
3809
3810	cond_resched();
3811
3812	return NULL;
3813	}
3814
3815	static inline bool
3816	should_compact_retry(struct alloc_context ac, int* order, int alloc_flags,
3817	enum compact_result compact_result,
3818	enum compact_priority *compact_priority,
3819	int *compaction_retries)
3820	{
3821	int max_retries = MAX_COMPACT_RETRIES;
3822	int min_priority;
3823	bool ret = false;
3824	int retries = *compaction_retries;
3825	enum compact_priority priority = *compact_priority;
3826
3827	if (!order)
3828	return false;
3829
3830	if (compaction_made_progress(compact_result))
3831	(*compaction_retries)++;
3832
3833	/*
3834	* compaction considers all the zone as desperately out of memory
3835	* so it doesn't really make much sense to retry except when the
3836	* failure could be caused by insufficient priority
3837	*/
3838	if (compaction_failed(compact_result))
3839	goto check_priority;
3840
3841	/*
3842	* make sure the compaction wasn't deferred or didn't bail out early
3843	* due to locks contention before we declare that we should give up.
3844	* But do not retry if the given zonelist is not suitable for
3845	* compaction.
3846	*/
3847	if (compaction_withdrawn(compact_result)) {
3848	ret = compaction_zonelist_suitable(ac, order, alloc_flags);
3849	goto out;
3850	}
3851
3852	/*
3853	* !costly requests are much more important than __GFP_RETRY_MAYFAIL
3854	* costly ones because they are de facto nofail and invoke OOM
3855	* killer to move on while costly can fail and users are ready
3856	* to cope with that. 1/4 retries is rather arbitrary but we
3857	* would need much more detailed feedback from compaction to
3858	* make a better decision.
3859	*/
3860	if (order > PAGE_ALLOC_COSTLY_ORDER)
3861	max_retries /= `4`;
3862	if (*compaction_retries <= max_retries) {
3863	ret = true;
3864	goto out;
3865	}
3866
3867	/*
3868	* Make sure there are attempts at the highest priority if we exhausted
3869	* all retries or failed at the lower priorities.
3870	*/
3871	check_priority:
3872	min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
3873	MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
3874
3875	if (*compact_priority > min_priority) {
3876	(*compact_priority)--;
3877	*compaction_retries = `0`;
3878	ret = true;
3879	}
3880	out:
3881	trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
3882	return ret;
3883	}
3884	#else
3885	static inline struct page *
3886	__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3887	unsigned int alloc_flags, const struct alloc_context *ac,
3888	enum compact_priority prio, enum compact_result *compact_result)
3889	{
3890	*compact_result = COMPACT_SKIPPED;
3891	return NULL;
3892	}
3893
3894	static inline bool
3895	should_compact_retry(struct alloc_context ac, unsigned* int order, int alloc_flags,
3896	enum compact_result compact_result,
3897	enum compact_priority *compact_priority,
3898	int *compaction_retries)
3899	{
3900	struct zone *zone;
3901	struct zoneref *z;
3902
3903	if (!order \|\| order > PAGE_ALLOC_COSTLY_ORDER)
3904	return false;
3905
3906	/*
3907	* There are setups with compaction disabled which would prefer to loop
3908	* inside the allocator rather than hit the oom killer prematurely.
3909	* Let's give them a good hope and keep retrying while the order-0
3910	* watermarks are OK.
3911	*/
3912	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3913	ac->nodemask) {
3914	if (zone_watermark_ok(zone, `0`, min_wmark_pages(zone),
3915	ac_classzone_idx(ac), alloc_flags))
3916	return true;
3917	}
3918	return false;
3919	}
3920	#endif /* CONFIG_COMPACTION */
3921
3922	#ifdef CONFIG_LOCKDEP
3923	static struct lockdep_map __fs_reclaim_map =
3924	STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
3925
3926	static bool __need_fs_reclaim(gfp_t gfp_mask)
3927	{
3928	gfp_mask = current_gfp_context(gfp_mask);
3929
3930	/ no reclaim without waiting on it /
3931	if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
3932	return false;
3933
3934	/ this guy won't enter reclaim /
3935	if (current->flags & PF_MEMALLOC)
3936	return false;
3937
3938	/ We're only interested __GFP_FS allocations for now /
3939	if (!(gfp_mask & __GFP_FS))
3940	return false;
3941
3942	if (gfp_mask & __GFP_NOLOCKDEP)
3943	return false;
3944
3945	return true;
3946	}
3947
3948	void __fs_reclaim_acquire(void)
3949	{
3950	lock_map_acquire(&__fs_reclaim_map);
3951	}
3952
3953	void __fs_reclaim_release(void)
3954	{
3955	lock_map_release(&__fs_reclaim_map);
3956	}
3957
3958	void fs_reclaim_acquire(gfp_t gfp_mask)
3959	{
3960	if (__need_fs_reclaim(gfp_mask))
3961	__fs_reclaim_acquire();
3962	}
3963	EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
3964
3965	void fs_reclaim_release(gfp_t gfp_mask)
3966	{
3967	if (__need_fs_reclaim(gfp_mask))
3968	__fs_reclaim_release();
3969	}
3970	EXPORT_SYMBOL_GPL(fs_reclaim_release);
3971	#endif
3972
3973	/ Perform direct synchronous page reclaim /
3974	static int
3975	__perform_reclaim(gfp_t gfp_mask, unsigned int order,
3976	const struct alloc_context *ac)
3977	{
3978	struct reclaim_state reclaim_state;
3979	int progress;
3980	unsigned int noreclaim_flag;
3981	unsigned long pflags;
3982
3983	cond_resched();
3984
3985	/ We now go into synchronous reclaim /
3986	cpuset_memory_pressure_bump();
3987	psi_memstall_enter(&pflags);
3988	fs_reclaim_acquire(gfp_mask);
3989	noreclaim_flag = memalloc_noreclaim_save();
3990	reclaim_state.reclaimed_slab = `0`;
3991	current->reclaim_state = &reclaim_state;
3992
3993	progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
3994	ac->nodemask);
3995
3996	current->reclaim_state = NULL;
3997	memalloc_noreclaim_restore(noreclaim_flag);
3998	fs_reclaim_release(gfp_mask);
3999	psi_memstall_leave(&pflags);
4000
4001	cond_resched();
4002
4003	return progress;
4004	}
4005
4006	/ The really slow allocator path where we enter direct reclaim /
4007	static inline struct page *
4008	__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
4009	unsigned int alloc_flags, const struct alloc_context *ac,
4010	unsigned long *did_some_progress)
4011	{
4012	struct page *page = NULL;
4013	bool drained = false;
4014
4015	*did_some_progress = __perform_reclaim(gfp_mask, order, ac);
4016	if (unlikely(!(*did_some_progress)))
4017	return NULL;
4018
4019	retry:
4020	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4021
4022	/*
4023	* If an allocation failed after direct reclaim, it could be because
4024	* pages are pinned on the per-cpu lists or in high alloc reserves.
4025	* Shrink them them and try again
4026	*/
4027	if (!page && !drained) {
4028	unreserve_highatomic_pageblock(ac, false);
4029	drain_all_pages(NULL);
4030	drained = true;
4031	goto retry;
4032	}
4033
4034	return page;
4035	}
4036
4037	static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
4038	const struct alloc_context *ac)
4039	{
4040	struct zoneref *z;
4041	struct zone *zone;

Browse the source code of linux/mm/page_alloc.c