1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/mm/vmscan.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 *
7 * Swap reorganised 29.12.95, Stephen Tweedie.
8 * kswapd added: 7.1.96 sct
9 * Removed kswapd_ctl limits, and swap out as many pages as needed
10 * to bring the system back to freepages.high: 2.4.97, Rik van Riel.
11 * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
12 * Multiqueue VM started 5.8.00, Rik van Riel.
13 */
14
15#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16
17#include <linux/mm.h>
18#include <linux/sched/mm.h>
19#include <linux/module.h>
20#include <linux/gfp.h>
21#include <linux/kernel_stat.h>
22#include <linux/swap.h>
23#include <linux/pagemap.h>
24#include <linux/init.h>
25#include <linux/highmem.h>
26#include <linux/vmpressure.h>
27#include <linux/vmstat.h>
28#include <linux/file.h>
29#include <linux/writeback.h>
30#include <linux/blkdev.h>
31#include <linux/buffer_head.h> /* for try_to_release_page(),
32 buffer_heads_over_limit */
33#include <linux/mm_inline.h>
34#include <linux/backing-dev.h>
35#include <linux/rmap.h>
36#include <linux/topology.h>
37#include <linux/cpu.h>
38#include <linux/cpuset.h>
39#include <linux/compaction.h>
40#include <linux/notifier.h>
41#include <linux/rwsem.h>
42#include <linux/delay.h>
43#include <linux/kthread.h>
44#include <linux/freezer.h>
45#include <linux/memcontrol.h>
46#include <linux/delayacct.h>
47#include <linux/sysctl.h>
48#include <linux/oom.h>
49#include <linux/pagevec.h>
50#include <linux/prefetch.h>
51#include <linux/printk.h>
52#include <linux/dax.h>
53#include <linux/psi.h>
54
55#include <asm/tlbflush.h>
56#include <asm/div64.h>
57
58#include <linux/swapops.h>
59#include <linux/balloon_compaction.h>
60
61#include "internal.h"
62
63#define CREATE_TRACE_POINTS
64#include <trace/events/vmscan.h>
65
66struct scan_control {
67 /* How many pages shrink_list() should reclaim */
68 unsigned long nr_to_reclaim;
69
70 /*
71 * Nodemask of nodes allowed by the caller. If NULL, all nodes
72 * are scanned.
73 */
74 nodemask_t *nodemask;
75
76 /*
77 * The memory cgroup that hit its limit and as a result is the
78 * primary target of this reclaim invocation.
79 */
80 struct mem_cgroup *target_mem_cgroup;
81
82 /* Writepage batching in laptop mode; RECLAIM_WRITE */
83 unsigned int may_writepage:1;
84
85 /* Can mapped pages be reclaimed? */
86 unsigned int may_unmap:1;
87
88 /* Can pages be swapped as part of reclaim? */
89 unsigned int may_swap:1;
90
91 /* e.g. boosted watermark reclaim leaves slabs alone */
92 unsigned int may_shrinkslab:1;
93
94 /*
95 * Cgroups are not reclaimed below their configured memory.low,
96 * unless we threaten to OOM. If any cgroups are skipped due to
97 * memory.low and nothing was reclaimed, go back for memory.low.
98 */
99 unsigned int memcg_low_reclaim:1;
100 unsigned int memcg_low_skipped:1;
101
102 unsigned int hibernation_mode:1;
103
104 /* One of the zones is ready for compaction */
105 unsigned int compaction_ready:1;
106
107 /* Allocation order */
108 s8 order;
109
110 /* Scan (total_size >> priority) pages at once */
111 s8 priority;
112
113 /* The highest zone to isolate pages for reclaim from */
114 s8 reclaim_idx;
115
116 /* This context's GFP mask */
117 gfp_t gfp_mask;
118
119 /* Incremented by the number of inactive pages that were scanned */
120 unsigned long nr_scanned;
121
122 /* Number of pages freed so far during a call to shrink_zones() */
123 unsigned long nr_reclaimed;
124
125 struct {
126 unsigned int dirty;
127 unsigned int unqueued_dirty;
128 unsigned int congested;
129 unsigned int writeback;
130 unsigned int immediate;
131 unsigned int file_taken;
132 unsigned int taken;
133 } nr;
134};
135
136#ifdef ARCH_HAS_PREFETCH
137#define prefetch_prev_lru_page(_page, _base, _field) \
138 do { \
139 if ((_page)->lru.prev != _base) { \
140 struct page *prev; \
141 \
142 prev = lru_to_page(&(_page->lru)); \
143 prefetch(&prev->_field); \
144 } \
145 } while (0)
146#else
147#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
148#endif
149
150#ifdef ARCH_HAS_PREFETCHW
151#define prefetchw_prev_lru_page(_page, _base, _field) \
152 do { \
153 if ((_page)->lru.prev != _base) { \
154 struct page *prev; \
155 \
156 prev = lru_to_page(&(_page->lru)); \
157 prefetchw(&prev->_field); \
158 } \
159 } while (0)
160#else
161#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
162#endif
163
164/*
165 * From 0 .. 100. Higher means more swappy.
166 */
167int vm_swappiness = 60;
168/*
169 * The total number of pages which are beyond the high watermark within all
170 * zones.
171 */
172unsigned long vm_total_pages;
173
174static LIST_HEAD(shrinker_list);
175static DECLARE_RWSEM(shrinker_rwsem);
176
177#ifdef CONFIG_MEMCG_KMEM
178
179/*
180 * We allow subsystems to populate their shrinker-related
181 * LRU lists before register_shrinker_prepared() is called
182 * for the shrinker, since we don't want to impose
183 * restrictions on their internal registration order.
184 * In this case shrink_slab_memcg() may find corresponding
185 * bit is set in the shrinkers map.
186 *
187 * This value is used by the function to detect registering
188 * shrinkers and to skip do_shrink_slab() calls for them.
189 */
190#define SHRINKER_REGISTERING ((struct shrinker *)~0UL)
191
192static DEFINE_IDR(shrinker_idr);
193static int shrinker_nr_max;
194
195static int prealloc_memcg_shrinker(struct shrinker *shrinker)
196{
197 int id, ret = -ENOMEM;
198
199 down_write(&shrinker_rwsem);
200 /* This may call shrinker, so it must use down_read_trylock() */
201 id = idr_alloc(&shrinker_idr, SHRINKER_REGISTERING, 0, 0, GFP_KERNEL);
202 if (id < 0)
203 goto unlock;
204
205 if (id >= shrinker_nr_max) {
206 if (memcg_expand_shrinker_maps(id)) {
207 idr_remove(&shrinker_idr, id);
208 goto unlock;
209 }
210
211 shrinker_nr_max = id + 1;
212 }
213 shrinker->id = id;
214 ret = 0;
215unlock:
216 up_write(&shrinker_rwsem);
217 return ret;
218}
219
220static void unregister_memcg_shrinker(struct shrinker *shrinker)
221{
222 int id = shrinker->id;
223
224 BUG_ON(id < 0);
225
226 down_write(&shrinker_rwsem);
227 idr_remove(&shrinker_idr, id);
228 up_write(&shrinker_rwsem);
229}
230#else /* CONFIG_MEMCG_KMEM */
231static int prealloc_memcg_shrinker(struct shrinker *shrinker)
232{
233 return 0;
234}
235
236static void unregister_memcg_shrinker(struct shrinker *shrinker)
237{
238}
239#endif /* CONFIG_MEMCG_KMEM */
240
241#ifdef CONFIG_MEMCG
242static bool global_reclaim(struct scan_control *sc)
243{
244 return !sc->target_mem_cgroup;
245}
246
247/**
248 * sane_reclaim - is the usual dirty throttling mechanism operational?
249 * @sc: scan_control in question
250 *
251 * The normal page dirty throttling mechanism in balance_dirty_pages() is
252 * completely broken with the legacy memcg and direct stalling in
253 * shrink_page_list() is used for throttling instead, which lacks all the
254 * niceties such as fairness, adaptive pausing, bandwidth proportional
255 * allocation and configurability.
256 *
257 * This function tests whether the vmscan currently in progress can assume
258 * that the normal dirty throttling mechanism is operational.
259 */
260static bool sane_reclaim(struct scan_control *sc)
261{
262 struct mem_cgroup *memcg = sc->target_mem_cgroup;
263
264 if (!memcg)
265 return true;
266#ifdef CONFIG_CGROUP_WRITEBACK
267 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
268 return true;
269#endif
270 return false;
271}
272
273static void set_memcg_congestion(pg_data_t *pgdat,
274 struct mem_cgroup *memcg,
275 bool congested)
276{
277 struct mem_cgroup_per_node *mn;
278
279 if (!memcg)
280 return;
281
282 mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
283 WRITE_ONCE(mn->congested, congested);
284}
285
286static bool memcg_congested(pg_data_t *pgdat,
287 struct mem_cgroup *memcg)
288{
289 struct mem_cgroup_per_node *mn;
290
291 mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
292 return READ_ONCE(mn->congested);
293
294}
295#else
296static bool global_reclaim(struct scan_control *sc)
297{
298 return true;
299}
300
301static bool sane_reclaim(struct scan_control *sc)
302{
303 return true;
304}
305
306static inline void set_memcg_congestion(struct pglist_data *pgdat,
307 struct mem_cgroup *memcg, bool congested)
308{
309}
310
311static inline bool memcg_congested(struct pglist_data *pgdat,
312 struct mem_cgroup *memcg)
313{
314 return false;
315
316}
317#endif
318
319/*
320 * This misses isolated pages which are not accounted for to save counters.
321 * As the data only determines if reclaim or compaction continues, it is
322 * not expected that isolated pages will be a dominating factor.
323 */
324unsigned long zone_reclaimable_pages(struct zone *zone)
325{
326 unsigned long nr;
327
328 nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
329 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
330 if (get_nr_swap_pages() > 0)
331 nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
332 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);
333
334 return nr;
335}
336
337/**
338 * lruvec_lru_size - Returns the number of pages on the given LRU list.
339 * @lruvec: lru vector
340 * @lru: lru to use
341 * @zone_idx: zones to consider (use MAX_NR_ZONES for the whole LRU list)
342 */
343unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)
344{
345 unsigned long lru_size;
346 int zid;
347
348 if (!mem_cgroup_disabled())
349 lru_size = mem_cgroup_get_lru_size(lruvec, lru);
350 else
351 lru_size = node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);
352
353 for (zid = zone_idx + 1; zid < MAX_NR_ZONES; zid++) {
354 struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
355 unsigned long size;
356
357 if (!managed_zone(zone))
358 continue;
359
360 if (!mem_cgroup_disabled())
361 size = mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
362 else
363 size = zone_page_state(&lruvec_pgdat(lruvec)->node_zones[zid],
364 NR_ZONE_LRU_BASE + lru);
365 lru_size -= min(size, lru_size);
366 }
367
368 return lru_size;
369
370}
371
372/*
373 * Add a shrinker callback to be called from the vm.
374 */
375int prealloc_shrinker(struct shrinker *shrinker)
376{
377 unsigned int size = sizeof(*shrinker->nr_deferred);
378
379 if (shrinker->flags & SHRINKER_NUMA_AWARE)
380 size *= nr_node_ids;
381
382 shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
383 if (!shrinker->nr_deferred)
384 return -ENOMEM;
385
386 if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
387 if (prealloc_memcg_shrinker(shrinker))
388 goto free_deferred;
389 }
390
391 return 0;
392
393free_deferred:
394 kfree(shrinker->nr_deferred);
395 shrinker->nr_deferred = NULL;
396 return -ENOMEM;
397}
398
399void free_prealloced_shrinker(struct shrinker *shrinker)
400{
401 if (!shrinker->nr_deferred)
402 return;
403
404 if (shrinker->flags & SHRINKER_MEMCG_AWARE)
405 unregister_memcg_shrinker(shrinker);
406
407 kfree(shrinker->nr_deferred);
408 shrinker->nr_deferred = NULL;
409}
410
411void register_shrinker_prepared(struct shrinker *shrinker)
412{
413 down_write(&shrinker_rwsem);
414 list_add_tail(&shrinker->list, &shrinker_list);
415#ifdef CONFIG_MEMCG_KMEM
416 if (shrinker->flags & SHRINKER_MEMCG_AWARE)
417 idr_replace(&shrinker_idr, shrinker, shrinker->id);
418#endif
419 up_write(&shrinker_rwsem);
420}
421
422int register_shrinker(struct shrinker *shrinker)
423{
424 int err = prealloc_shrinker(shrinker);
425
426 if (err)
427 return err;
428 register_shrinker_prepared(shrinker);
429 return 0;
430}
431EXPORT_SYMBOL(register_shrinker);
432
433/*
434 * Remove one
435 */
436void unregister_shrinker(struct shrinker *shrinker)
437{
438 if (!shrinker->nr_deferred)
439 return;
440 if (shrinker->flags & SHRINKER_MEMCG_AWARE)
441 unregister_memcg_shrinker(shrinker);
442 down_write(&shrinker_rwsem);
443 list_del(&shrinker->list);
444 up_write(&shrinker_rwsem);
445 kfree(shrinker->nr_deferred);
446 shrinker->nr_deferred = NULL;
447}
448EXPORT_SYMBOL(unregister_shrinker);
449
450#define SHRINK_BATCH 128
451
452static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
453 struct shrinker *shrinker, int priority)
454{
455 unsigned long freed = 0;
456 unsigned long long delta;
457 long total_scan;
458 long freeable;
459 long nr;
460 long new_nr;
461 int nid = shrinkctl->nid;
462 long batch_size = shrinker->batch ? shrinker->batch
463 : SHRINK_BATCH;
464 long scanned = 0, next_deferred;
465
466 if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
467 nid = 0;
468
469 freeable = shrinker->count_objects(shrinker, shrinkctl);
470 if (freeable == 0 || freeable == SHRINK_EMPTY)
471 return freeable;
472
473 /*
474 * copy the current shrinker scan count into a local variable
475 * and zero it so that other concurrent shrinker invocations
476 * don't also do this scanning work.
477 */
478 nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
479
480 total_scan = nr;
481 if (shrinker->seeks) {
482 delta = freeable >> priority;
483 delta *= 4;
484 do_div(delta, shrinker->seeks);
485 } else {
486 /*
487 * These objects don't require any IO to create. Trim
488 * them aggressively under memory pressure to keep
489 * them from causing refetches in the IO caches.
490 */
491 delta = freeable / 2;
492 }
493
494 total_scan += delta;
495 if (total_scan < 0) {
496 pr_err("shrink_slab: %pF negative objects to delete nr=%ld\n",
497 shrinker->scan_objects, total_scan);
498 total_scan = freeable;
499 next_deferred = nr;
500 } else
501 next_deferred = total_scan;
502
503 /*
504 * We need to avoid excessive windup on filesystem shrinkers
505 * due to large numbers of GFP_NOFS allocations causing the
506 * shrinkers to return -1 all the time. This results in a large
507 * nr being built up so when a shrink that can do some work
508 * comes along it empties the entire cache due to nr >>>
509 * freeable. This is bad for sustaining a working set in
510 * memory.
511 *
512 * Hence only allow the shrinker to scan the entire cache when
513 * a large delta change is calculated directly.
514 */
515 if (delta < freeable / 4)
516 total_scan = min(total_scan, freeable / 2);
517
518 /*
519 * Avoid risking looping forever due to too large nr value:
520 * never try to free more than twice the estimate number of
521 * freeable entries.
522 */
523 if (total_scan > freeable * 2)
524 total_scan = freeable * 2;
525
526 trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
527 freeable, delta, total_scan, priority);
528
529 /*
530 * Normally, we should not scan less than batch_size objects in one
531 * pass to avoid too frequent shrinker calls, but if the slab has less
532 * than batch_size objects in total and we are really tight on memory,
533 * we will try to reclaim all available objects, otherwise we can end
534 * up failing allocations although there are plenty of reclaimable
535 * objects spread over several slabs with usage less than the
536 * batch_size.
537 *
538 * We detect the "tight on memory" situations by looking at the total
539 * number of objects we want to scan (total_scan). If it is greater
540 * than the total number of objects on slab (freeable), we must be
541 * scanning at high prio and therefore should try to reclaim as much as
542 * possible.
543 */
544 while (total_scan >= batch_size ||
545 total_scan >= freeable) {
546 unsigned long ret;
547 unsigned long nr_to_scan = min(batch_size, total_scan);
548
549 shrinkctl->nr_to_scan = nr_to_scan;
550 shrinkctl->nr_scanned = nr_to_scan;
551 ret = shrinker->scan_objects(shrinker, shrinkctl);
552 if (ret == SHRINK_STOP)
553 break;
554 freed += ret;
555
556 count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);
557 total_scan -= shrinkctl->nr_scanned;
558 scanned += shrinkctl->nr_scanned;
559
560 cond_resched();
561 }
562
563 if (next_deferred >= scanned)
564 next_deferred -= scanned;
565 else
566 next_deferred = 0;
567 /*
568 * move the unused scan count back into the shrinker in a
569 * manner that handles concurrent updates. If we exhausted the
570 * scan, there is no need to do an update.
571 */
572 if (next_deferred > 0)
573 new_nr = atomic_long_add_return(next_deferred,
574 &shrinker->nr_deferred[nid]);
575 else
576 new_nr = atomic_long_read(&shrinker->nr_deferred[nid]);
577
578 trace_mm_shrink_slab_end(shrinker, nid, freed, nr, new_nr, total_scan);
579 return freed;
580}
581
582#ifdef CONFIG_MEMCG_KMEM
583static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
584 struct mem_cgroup *memcg, int priority)
585{
586 struct memcg_shrinker_map *map;
587 unsigned long ret, freed = 0;
588 int i;
589
590 if (!memcg_kmem_enabled() || !mem_cgroup_online(memcg))
591 return 0;
592
593 if (!down_read_trylock(&shrinker_rwsem))
594 return 0;
595
596 map = rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_map,
597 true);
598 if (unlikely(!map))
599 goto unlock;
600
601 for_each_set_bit(i, map->map, shrinker_nr_max) {
602 struct shrink_control sc = {
603 .gfp_mask = gfp_mask,
604 .nid = nid,
605 .memcg = memcg,
606 };
607 struct shrinker *shrinker;
608
609 shrinker = idr_find(&shrinker_idr, i);
610 if (unlikely(!shrinker || shrinker == SHRINKER_REGISTERING)) {
611 if (!shrinker)
612 clear_bit(i, map->map);
613 continue;
614 }
615
616 ret = do_shrink_slab(&sc, shrinker, priority);
617 if (ret == SHRINK_EMPTY) {
618 clear_bit(i, map->map);
619 /*
620 * After the shrinker reported that it had no objects to
621 * free, but before we cleared the corresponding bit in
622 * the memcg shrinker map, a new object might have been
623 * added. To make sure, we have the bit set in this
624 * case, we invoke the shrinker one more time and reset
625 * the bit if it reports that it is not empty anymore.
626 * The memory barrier here pairs with the barrier in
627 * memcg_set_shrinker_bit():
628 *
629 * list_lru_add() shrink_slab_memcg()
630 * list_add_tail() clear_bit()
631 * <MB> <MB>
632 * set_bit() do_shrink_slab()
633 */
634 smp_mb__after_atomic();
635 ret = do_shrink_slab(&sc, shrinker, priority);
636 if (ret == SHRINK_EMPTY)
637 ret = 0;
638 else
639 memcg_set_shrinker_bit(memcg, nid, i);
640 }
641 freed += ret;
642
643 if (rwsem_is_contended(&shrinker_rwsem)) {
644 freed = freed ? : 1;
645 break;
646 }
647 }
648unlock:
649 up_read(&shrinker_rwsem);
650 return freed;
651}
652#else /* CONFIG_MEMCG_KMEM */
653static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
654 struct mem_cgroup *memcg, int priority)
655{
656 return 0;
657}
658#endif /* CONFIG_MEMCG_KMEM */
659
660/**
661 * shrink_slab - shrink slab caches
662 * @gfp_mask: allocation context
663 * @nid: node whose slab caches to target
664 * @memcg: memory cgroup whose slab caches to target
665 * @priority: the reclaim priority
666 *
667 * Call the shrink functions to age shrinkable caches.
668 *
669 * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
670 * unaware shrinkers will receive a node id of 0 instead.
671 *
672 * @memcg specifies the memory cgroup to target. Unaware shrinkers
673 * are called only if it is the root cgroup.
674 *
675 * @priority is sc->priority, we take the number of objects and >> by priority
676 * in order to get the scan target.
677 *
678 * Returns the number of reclaimed slab objects.
679 */
680static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
681 struct mem_cgroup *memcg,
682 int priority)
683{
684 unsigned long ret, freed = 0;
685 struct shrinker *shrinker;
686
687 if (!mem_cgroup_is_root(memcg))
688 return shrink_slab_memcg(gfp_mask, nid, memcg, priority);
689
690 if (!down_read_trylock(&shrinker_rwsem))
691 goto out;
692
693 list_for_each_entry(shrinker, &shrinker_list, list) {
694 struct shrink_control sc = {
695 .gfp_mask = gfp_mask,
696 .nid = nid,
697 .memcg = memcg,
698 };
699
700 ret = do_shrink_slab(&sc, shrinker, priority);
701 if (ret == SHRINK_EMPTY)
702 ret = 0;
703 freed += ret;
704 /*
705 * Bail out if someone want to register a new shrinker to
706 * prevent the regsitration from being stalled for long periods
707 * by parallel ongoing shrinking.
708 */
709 if (rwsem_is_contended(&shrinker_rwsem)) {
710 freed = freed ? : 1;
711 break;
712 }
713 }
714
715 up_read(&shrinker_rwsem);
716out:
717 cond_resched();
718 return freed;
719}
720
721void drop_slab_node(int nid)
722{
723 unsigned long freed;
724
725 do {
726 struct mem_cgroup *memcg = NULL;
727
728 freed = 0;
729 memcg = mem_cgroup_iter(NULL, NULL, NULL);
730 do {
731 freed += shrink_slab(GFP_KERNEL, nid, memcg, 0);
732 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
733 } while (freed > 10);
734}
735
736void drop_slab(void)
737{
738 int nid;
739
740 for_each_online_node(nid)
741 drop_slab_node(nid);
742}
743
744static inline int is_page_cache_freeable(struct page *page)
745{
746 /*
747 * A freeable page cache page is referenced only by the caller
748 * that isolated the page, the page cache and optional buffer
749 * heads at page->private.
750 */
751 int page_cache_pins = PageTransHuge(page) && PageSwapCache(page) ?
752 HPAGE_PMD_NR : 1;
753 return page_count(page) - page_has_private(page) == 1 + page_cache_pins;
754}
755
756static int may_write_to_inode(struct inode *inode, struct scan_control *sc)
757{
758 if (current->flags & PF_SWAPWRITE)
759 return 1;
760 if (!inode_write_congested(inode))
761 return 1;
762 if (inode_to_bdi(inode) == current->backing_dev_info)
763 return 1;
764 return 0;
765}
766
767/*
768 * We detected a synchronous write error writing a page out. Probably
769 * -ENOSPC. We need to propagate that into the address_space for a subsequent
770 * fsync(), msync() or close().
771 *
772 * The tricky part is that after writepage we cannot touch the mapping: nothing
773 * prevents it from being freed up. But we have a ref on the page and once
774 * that page is locked, the mapping is pinned.
775 *
776 * We're allowed to run sleeping lock_page() here because we know the caller has
777 * __GFP_FS.
778 */
779static void handle_write_error(struct address_space *mapping,
780 struct page *page, int error)
781{
782 lock_page(page);
783 if (page_mapping(page) == mapping)
784 mapping_set_error(mapping, error);
785 unlock_page(page);
786}
787
788/* possible outcome of pageout() */
789typedef enum {
790 /* failed to write page out, page is locked */
791 PAGE_KEEP,
792 /* move page to the active list, page is locked */
793 PAGE_ACTIVATE,
794 /* page has been sent to the disk successfully, page is unlocked */
795 PAGE_SUCCESS,
796 /* page is clean and locked */
797 PAGE_CLEAN,
798} pageout_t;
799
800/*
801 * pageout is called by shrink_page_list() for each dirty page.
802 * Calls ->writepage().
803 */
804static pageout_t pageout(struct page *page, struct address_space *mapping,
805 struct scan_control *sc)
806{
807 /*
808 * If the page is dirty, only perform writeback if that write
809 * will be non-blocking. To prevent this allocation from being
810 * stalled by pagecache activity. But note that there may be
811 * stalls if we need to run get_block(). We could test
812 * PagePrivate for that.
813 *
814 * If this process is currently in __generic_file_write_iter() against
815 * this page's queue, we can perform writeback even if that
816 * will block.
817 *
818 * If the page is swapcache, write it back even if that would
819 * block, for some throttling. This happens by accident, because
820 * swap_backing_dev_info is bust: it doesn't reflect the
821 * congestion state of the swapdevs. Easy to fix, if needed.
822 */
823 if (!is_page_cache_freeable(page))
824 return PAGE_KEEP;
825 if (!mapping) {
826 /*
827 * Some data journaling orphaned pages can have
828 * page->mapping == NULL while being dirty with clean buffers.
829 */
830 if (page_has_private(page)) {
831 if (try_to_free_buffers(page)) {
832 ClearPageDirty(page);
833 pr_info("%s: orphaned page\n", __func__);
834 return PAGE_CLEAN;
835 }
836 }
837 return PAGE_KEEP;
838 }
839 if (mapping->a_ops->writepage == NULL)
840 return PAGE_ACTIVATE;
841 if (!may_write_to_inode(mapping->host, sc))
842 return PAGE_KEEP;
843
844 if (clear_page_dirty_for_io(page)) {
845 int res;
846 struct writeback_control wbc = {
847 .sync_mode = WB_SYNC_NONE,
848 .nr_to_write = SWAP_CLUSTER_MAX,
849 .range_start = 0,
850 .range_end = LLONG_MAX,
851 .for_reclaim = 1,
852 };
853
854 SetPageReclaim(page);
855 res = mapping->a_ops->writepage(page, &wbc);
856 if (res < 0)
857 handle_write_error(mapping, page, res);
858 if (res == AOP_WRITEPAGE_ACTIVATE) {
859 ClearPageReclaim(page);
860 return PAGE_ACTIVATE;
861 }
862
863 if (!PageWriteback(page)) {
864 /* synchronous write or broken a_ops? */
865 ClearPageReclaim(page);
866 }
867 trace_mm_vmscan_writepage(page);
868 inc_node_page_state(page, NR_VMSCAN_WRITE);
869 return PAGE_SUCCESS;
870 }
871
872 return PAGE_CLEAN;
873}
874
875/*
876 * Same as remove_mapping, but if the page is removed from the mapping, it
877 * gets returned with a refcount of 0.
878 */
879static int __remove_mapping(struct address_space *mapping, struct page *page,
880 bool reclaimed)
881{
882 unsigned long flags;
883 int refcount;
884
885 BUG_ON(!PageLocked(page));
886 BUG_ON(mapping != page_mapping(page));
887
888 xa_lock_irqsave(&mapping->i_pages, flags);
889 /*
890 * The non racy check for a busy page.
891 *
892 * Must be careful with the order of the tests. When someone has
893 * a ref to the page, it may be possible that they dirty it then
894 * drop the reference. So if PageDirty is tested before page_count
895 * here, then the following race may occur:
896 *
897 * get_user_pages(&page);
898 * [user mapping goes away]
899 * write_to(page);
900 * !PageDirty(page) [good]
901 * SetPageDirty(page);
902 * put_page(page);
903 * !page_count(page) [good, discard it]
904 *
905 * [oops, our write_to data is lost]
906 *
907 * Reversing the order of the tests ensures such a situation cannot
908 * escape unnoticed. The smp_rmb is needed to ensure the page->flags
909 * load is not satisfied before that of page->_refcount.
910 *
911 * Note that if SetPageDirty is always performed via set_page_dirty,
912 * and thus under the i_pages lock, then this ordering is not required.
913 */
914 if (unlikely(PageTransHuge(page)) && PageSwapCache(page))
915 refcount = 1 + HPAGE_PMD_NR;
916 else
917 refcount = 2;
918 if (!page_ref_freeze(page, refcount))
919 goto cannot_free;
920 /* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */
921 if (unlikely(PageDirty(page))) {
922 page_ref_unfreeze(page, refcount);
923 goto cannot_free;
924 }
925
926 if (PageSwapCache(page)) {
927 swp_entry_t swap = { .val = page_private(page) };
928 mem_cgroup_swapout(page, swap);
929 __delete_from_swap_cache(page, swap);
930 xa_unlock_irqrestore(&mapping->i_pages, flags);
931 put_swap_page(page, swap);
932 } else {
933 void (*freepage)(struct page *);
934 void *shadow = NULL;
935
936 freepage = mapping->a_ops->freepage;
937 /*
938 * Remember a shadow entry for reclaimed file cache in
939 * order to detect refaults, thus thrashing, later on.
940 *
941 * But don't store shadows in an address space that is
942 * already exiting. This is not just an optizimation,
943 * inode reclaim needs to empty out the radix tree or
944 * the nodes are lost. Don't plant shadows behind its
945 * back.
946 *
947 * We also don't store shadows for DAX mappings because the
948 * only page cache pages found in these are zero pages
949 * covering holes, and because we don't want to mix DAX
950 * exceptional entries and shadow exceptional entries in the
951 * same address_space.
952 */
953 if (reclaimed && page_is_file_cache(page) &&
954 !mapping_exiting(mapping) && !dax_mapping(mapping))
955 shadow = workingset_eviction(page);
956 __delete_from_page_cache(page, shadow);
957 xa_unlock_irqrestore(&mapping->i_pages, flags);
958
959 if (freepage != NULL)
960 freepage(page);
961 }
962
963 return 1;
964
965cannot_free:
966 xa_unlock_irqrestore(&mapping->i_pages, flags);
967 return 0;
968}
969
970/*
971 * Attempt to detach a locked page from its ->mapping. If it is dirty or if
972 * someone else has a ref on the page, abort and return 0. If it was
973 * successfully detached, return 1. Assumes the caller has a single ref on
974 * this page.
975 */
976int remove_mapping(struct address_space *mapping, struct page *page)
977{
978 if (__remove_mapping(mapping, page, false)) {
979 /*
980 * Unfreezing the refcount with 1 rather than 2 effectively
981 * drops the pagecache ref for us without requiring another
982 * atomic operation.
983 */
984 page_ref_unfreeze(page, 1);
985 return 1;
986 }
987 return 0;
988}
989
990/**
991 * putback_lru_page - put previously isolated page onto appropriate LRU list
992 * @page: page to be put back to appropriate lru list
993 *
994 * Add previously isolated @page to appropriate LRU list.
995 * Page may still be unevictable for other reasons.
996 *
997 * lru_lock must not be held, interrupts must be enabled.
998 */
999void putback_lru_page(struct page *page)
1000{
1001 lru_cache_add(page);
1002 put_page(page); /* drop ref from isolate */
1003}
1004
1005enum page_references {
1006 PAGEREF_RECLAIM,
1007 PAGEREF_RECLAIM_CLEAN,
1008 PAGEREF_KEEP,
1009 PAGEREF_ACTIVATE,
1010};
1011
1012static enum page_references page_check_references(struct page *page,
1013 struct scan_control *sc)
1014{
1015 int referenced_ptes, referenced_page;
1016 unsigned long vm_flags;
1017
1018 referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
1019 &vm_flags);
1020 referenced_page = TestClearPageReferenced(page);
1021
1022 /*
1023 * Mlock lost the isolation race with us. Let try_to_unmap()
1024 * move the page to the unevictable list.
1025 */
1026 if (vm_flags & VM_LOCKED)
1027 return PAGEREF_RECLAIM;
1028
1029 if (referenced_ptes) {
1030 if (PageSwapBacked(page))
1031 return PAGEREF_ACTIVATE;
1032 /*
1033 * All mapped pages start out with page table
1034 * references from the instantiating fault, so we need
1035 * to look twice if a mapped file page is used more
1036 * than once.
1037 *
1038 * Mark it and spare it for another trip around the
1039 * inactive list. Another page table reference will
1040 * lead to its activation.
1041 *
1042 * Note: the mark is set for activated pages as well
1043 * so that recently deactivated but used pages are
1044 * quickly recovered.
1045 */
1046 SetPageReferenced(page);
1047
1048 if (referenced_page || referenced_ptes > 1)
1049 return PAGEREF_ACTIVATE;
1050
1051 /*
1052 * Activate file-backed executable pages after first usage.
1053 */
1054 if (vm_flags & VM_EXEC)
1055 return PAGEREF_ACTIVATE;
1056
1057 return PAGEREF_KEEP;
1058 }
1059
1060 /* Reclaim if clean, defer dirty pages to writeback */
1061 if (referenced_page && !PageSwapBacked(page))
1062 return PAGEREF_RECLAIM_CLEAN;
1063
1064 return PAGEREF_RECLAIM;
1065}
1066
1067/* Check if a page is dirty or under writeback */
1068static void page_check_dirty_writeback(struct page *page,
1069 bool *dirty, bool *writeback)
1070{
1071 struct address_space *mapping;
1072
1073 /*
1074 * Anonymous pages are not handled by flushers and must be written
1075 * from reclaim context. Do not stall reclaim based on them
1076 */
1077 if (!page_is_file_cache(page) ||
1078 (PageAnon(page) && !PageSwapBacked(page))) {
1079 *dirty = false;
1080 *writeback = false;
1081 return;
1082 }
1083
1084 /* By default assume that the page flags are accurate */
1085 *dirty = PageDirty(page);
1086 *writeback = PageWriteback(page);
1087
1088 /* Verify dirty/writeback state if the filesystem supports it */
1089 if (!page_has_private(page))
1090 return;
1091
1092 mapping = page_mapping(page);
1093 if (mapping && mapping->a_ops->is_dirty_writeback)
1094 mapping->a_ops->is_dirty_writeback(page, dirty, writeback);
1095}
1096
1097/*
1098 * shrink_page_list() returns the number of reclaimed pages
1099 */
1100static unsigned long shrink_page_list(struct list_head *page_list,
1101 struct pglist_data *pgdat,
1102 struct scan_control *sc,
1103 enum ttu_flags ttu_flags,
1104 struct reclaim_stat *stat,
1105 bool force_reclaim)
1106{
1107 LIST_HEAD(ret_pages);
1108 LIST_HEAD(free_pages);
1109 unsigned nr_reclaimed = 0;
1110
1111 memset(stat, 0, sizeof(*stat));
1112 cond_resched();
1113
1114 while (!list_empty(page_list)) {
1115 struct address_space *mapping;
1116 struct page *page;
1117 int may_enter_fs;
1118 enum page_references references = PAGEREF_RECLAIM_CLEAN;
1119 bool dirty, writeback;
1120
1121 cond_resched();
1122
1123 page = lru_to_page(page_list);
1124 list_del(&page->lru);
1125
1126 if (!trylock_page(page))
1127 goto keep;
1128
1129 VM_BUG_ON_PAGE(PageActive(page), page);
1130
1131 sc->nr_scanned++;
1132
1133 if (unlikely(!page_evictable(page)))
1134 goto activate_locked;
1135
1136 if (!sc->may_unmap && page_mapped(page))
1137 goto keep_locked;
1138
1139 /* Double the slab pressure for mapped and swapcache pages */
1140 if ((page_mapped(page) || PageSwapCache(page)) &&
1141 !(PageAnon(page) && !PageSwapBacked(page)))
1142 sc->nr_scanned++;
1143
1144 may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
1145 (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
1146
1147 /*
1148 * The number of dirty pages determines if a node is marked
1149 * reclaim_congested which affects wait_iff_congested. kswapd
1150 * will stall and start writing pages if the tail of the LRU
1151 * is all dirty unqueued pages.
1152 */
1153 page_check_dirty_writeback(page, &dirty, &writeback);
1154 if (dirty || writeback)
1155 stat->nr_dirty++;
1156
1157 if (dirty && !writeback)
1158 stat->nr_unqueued_dirty++;
1159
1160 /*
1161 * Treat this page as congested if the underlying BDI is or if
1162 * pages are cycling through the LRU so quickly that the
1163 * pages marked for immediate reclaim are making it to the
1164 * end of the LRU a second time.
1165 */
1166 mapping = page_mapping(page);
1167 if (((dirty || writeback) && mapping &&
1168 inode_write_congested(mapping->host)) ||
1169 (writeback && PageReclaim(page)))
1170 stat->nr_congested++;
1171
1172 /*
1173 * If a page at the tail of the LRU is under writeback, there
1174 * are three cases to consider.
1175 *
1176 * 1) If reclaim is encountering an excessive number of pages
1177 * under writeback and this page is both under writeback and
1178 * PageReclaim then it indicates that pages are being queued
1179 * for IO but are being recycled through the LRU before the
1180 * IO can complete. Waiting on the page itself risks an
1181 * indefinite stall if it is impossible to writeback the
1182 * page due to IO error or disconnected storage so instead
1183 * note that the LRU is being scanned too quickly and the
1184 * caller can stall after page list has been processed.
1185 *
1186 * 2) Global or new memcg reclaim encounters a page that is
1187 * not marked for immediate reclaim, or the caller does not
1188 * have __GFP_FS (or __GFP_IO if it's simply going to swap,
1189 * not to fs). In this case mark the page for immediate
1190 * reclaim and continue scanning.
1191 *
1192 * Require may_enter_fs because we would wait on fs, which
1193 * may not have submitted IO yet. And the loop driver might
1194 * enter reclaim, and deadlock if it waits on a page for
1195 * which it is needed to do the write (loop masks off
1196 * __GFP_IO|__GFP_FS for this reason); but more thought
1197 * would probably show more reasons.
1198 *
1199 * 3) Legacy memcg encounters a page that is already marked
1200 * PageReclaim. memcg does not have any dirty pages
1201 * throttling so we could easily OOM just because too many
1202 * pages are in writeback and there is nothing else to
1203 * reclaim. Wait for the writeback to complete.
1204 *
1205 * In cases 1) and 2) we activate the pages to get them out of
1206 * the way while we continue scanning for clean pages on the
1207 * inactive list and refilling from the active list. The
1208 * observation here is that waiting for disk writes is more
1209 * expensive than potentially causing reloads down the line.
1210 * Since they're marked for immediate reclaim, they won't put
1211 * memory pressure on the cache working set any longer than it
1212 * takes to write them to disk.
1213 */
1214 if (PageWriteback(page)) {
1215 /* Case 1 above */
1216 if (current_is_kswapd() &&
1217 PageReclaim(page) &&
1218 test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1219 stat->nr_immediate++;
1220 goto activate_locked;
1221
1222 /* Case 2 above */
1223 } else if (sane_reclaim(sc) ||
1224 !PageReclaim(page) || !may_enter_fs) {
1225 /*
1226 * This is slightly racy - end_page_writeback()
1227 * might have just cleared PageReclaim, then
1228 * setting PageReclaim here end up interpreted
1229 * as PageReadahead - but that does not matter
1230 * enough to care. What we do want is for this
1231 * page to have PageReclaim set next time memcg
1232 * reclaim reaches the tests above, so it will
1233 * then wait_on_page_writeback() to avoid OOM;
1234 * and it's also appropriate in global reclaim.
1235 */
1236 SetPageReclaim(page);
1237 stat->nr_writeback++;
1238 goto activate_locked;
1239
1240 /* Case 3 above */
1241 } else {
1242 unlock_page(page);
1243 wait_on_page_writeback(page);
1244 /* then go back and try same page again */
1245 list_add_tail(&page->lru, page_list);
1246 continue;
1247 }
1248 }
1249
1250 if (!force_reclaim)
1251 references = page_check_references(page, sc);
1252
1253 switch (references) {
1254 case PAGEREF_ACTIVATE:
1255 goto activate_locked;
1256 case PAGEREF_KEEP:
1257 stat->nr_ref_keep++;
1258 goto keep_locked;
1259 case PAGEREF_RECLAIM:
1260 case PAGEREF_RECLAIM_CLEAN:
1261 ; /* try to reclaim the page below */
1262 }
1263
1264 /*
1265 * Anonymous process memory has backing store?
1266 * Try to allocate it some swap space here.
1267 * Lazyfree page could be freed directly
1268 */
1269 if (PageAnon(page) && PageSwapBacked(page)) {
1270 if (!PageSwapCache(page)) {
1271 if (!(sc->gfp_mask & __GFP_IO))
1272 goto keep_locked;
1273 if (PageTransHuge(page)) {
1274 /* cannot split THP, skip it */
1275 if (!can_split_huge_page(page, NULL))
1276 goto activate_locked;
1277 /*
1278 * Split pages without a PMD map right
1279 * away. Chances are some or all of the
1280 * tail pages can be freed without IO.
1281 */
1282 if (!compound_mapcount(page) &&
1283 split_huge_page_to_list(page,
1284 page_list))
1285 goto activate_locked;
1286 }
1287 if (!add_to_swap(page)) {
1288 if (!PageTransHuge(page))
1289 goto activate_locked;
1290 /* Fallback to swap normal pages */
1291 if (split_huge_page_to_list(page,
1292 page_list))
1293 goto activate_locked;
1294#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1295 count_vm_event(THP_SWPOUT_FALLBACK);
1296#endif
1297 if (!add_to_swap(page))
1298 goto activate_locked;
1299 }
1300
1301 may_enter_fs = 1;
1302
1303 /* Adding to swap updated mapping */
1304 mapping = page_mapping(page);
1305 }
1306 } else if (unlikely(PageTransHuge(page))) {
1307 /* Split file THP */
1308 if (split_huge_page_to_list(page, page_list))
1309 goto keep_locked;
1310 }
1311
1312 /*
1313 * The page is mapped into the page tables of one or more
1314 * processes. Try to unmap it here.
1315 */
1316 if (page_mapped(page)) {
1317 enum ttu_flags flags = ttu_flags | TTU_BATCH_FLUSH;
1318
1319 if (unlikely(PageTransHuge(page)))
1320 flags |= TTU_SPLIT_HUGE_PMD;
1321 if (!try_to_unmap(page, flags)) {
1322 stat->nr_unmap_fail++;
1323 goto activate_locked;
1324 }
1325 }
1326
1327 if (PageDirty(page)) {
1328 /*
1329 * Only kswapd can writeback filesystem pages
1330 * to avoid risk of stack overflow. But avoid
1331 * injecting inefficient single-page IO into
1332 * flusher writeback as much as possible: only
1333 * write pages when we've encountered many
1334 * dirty pages, and when we've already scanned
1335 * the rest of the LRU for clean pages and see
1336 * the same dirty pages again (PageReclaim).
1337 */
1338 if (page_is_file_cache(page) &&
1339 (!current_is_kswapd() || !PageReclaim(page) ||
1340 !test_bit(PGDAT_DIRTY, &pgdat->flags))) {
1341 /*
1342 * Immediately reclaim when written back.
1343 * Similar in principal to deactivate_page()
1344 * except we already have the page isolated
1345 * and know it's dirty
1346 */
1347 inc_node_page_state(page, NR_VMSCAN_IMMEDIATE);
1348 SetPageReclaim(page);
1349
1350 goto activate_locked;
1351 }
1352
1353 if (references == PAGEREF_RECLAIM_CLEAN)
1354 goto keep_locked;
1355 if (!may_enter_fs)
1356 goto keep_locked;
1357 if (!sc->may_writepage)
1358 goto keep_locked;
1359
1360 /*
1361 * Page is dirty. Flush the TLB if a writable entry
1362 * potentially exists to avoid CPU writes after IO
1363 * starts and then write it out here.
1364 */
1365 try_to_unmap_flush_dirty();
1366 switch (pageout(page, mapping, sc)) {
1367 case PAGE_KEEP:
1368 goto keep_locked;
1369 case PAGE_ACTIVATE:
1370 goto activate_locked;
1371 case PAGE_SUCCESS:
1372 if (PageWriteback(page))
1373 goto keep;
1374 if (PageDirty(page))
1375 goto keep;
1376
1377 /*
1378 * A synchronous write - probably a ramdisk. Go
1379 * ahead and try to reclaim the page.
1380 */
1381 if (!trylock_page(page))
1382 goto keep;
1383 if (PageDirty(page) || PageWriteback(page))
1384 goto keep_locked;
1385 mapping = page_mapping(page);
1386 case PAGE_CLEAN:
1387 ; /* try to free the page below */
1388 }
1389 }
1390
1391 /*
1392 * If the page has buffers, try to free the buffer mappings
1393 * associated with this page. If we succeed we try to free
1394 * the page as well.
1395 *
1396 * We do this even if the page is PageDirty().
1397 * try_to_release_page() does not perform I/O, but it is
1398 * possible for a page to have PageDirty set, but it is actually
1399 * clean (all its buffers are clean). This happens if the
1400 * buffers were written out directly, with submit_bh(). ext3
1401 * will do this, as well as the blockdev mapping.
1402 * try_to_release_page() will discover that cleanness and will
1403 * drop the buffers and mark the page clean - it can be freed.
1404 *
1405 * Rarely, pages can have buffers and no ->mapping. These are
1406 * the pages which were not successfully invalidated in
1407 * truncate_complete_page(). We try to drop those buffers here
1408 * and if that worked, and the page is no longer mapped into
1409 * process address space (page_count == 1) it can be freed.
1410 * Otherwise, leave the page on the LRU so it is swappable.
1411 */
1412 if (page_has_private(page)) {
1413 if (!try_to_release_page(page, sc->gfp_mask))
1414 goto activate_locked;
1415 if (!mapping && page_count(page) == 1) {
1416 unlock_page(page);
1417 if (put_page_testzero(page))
1418 goto free_it;
1419 else {
1420 /*
1421 * rare race with speculative reference.
1422 * the speculative reference will free
1423 * this page shortly, so we may
1424 * increment nr_reclaimed here (and
1425 * leave it off the LRU).
1426 */
1427 nr_reclaimed++;
1428 continue;
1429 }
1430 }
1431 }
1432
1433 if (PageAnon(page) && !PageSwapBacked(page)) {
1434 /* follow __remove_mapping for reference */
1435 if (!page_ref_freeze(page, 1))
1436 goto keep_locked;
1437 if (PageDirty(page)) {
1438 page_ref_unfreeze(page, 1);
1439 goto keep_locked;
1440 }
1441
1442 count_vm_event(PGLAZYFREED);
1443 count_memcg_page_event(page, PGLAZYFREED);
1444 } else if (!mapping || !__remove_mapping(mapping, page, true))
1445 goto keep_locked;
1446
1447 unlock_page(page);
1448free_it:
1449 nr_reclaimed++;
1450
1451 /*
1452 * Is there need to periodically free_page_list? It would
1453 * appear not as the counts should be low
1454 */
1455 if (unlikely(PageTransHuge(page))) {
1456 mem_cgroup_uncharge(page);
1457 (*get_compound_page_dtor(page))(page);
1458 } else
1459 list_add(&page->lru, &free_pages);
1460 continue;
1461
1462activate_locked:
1463 /* Not a candidate for swapping, so reclaim swap space. */
1464 if (PageSwapCache(page) && (mem_cgroup_swap_full(page) ||
1465 PageMlocked(page)))
1466 try_to_free_swap(page);
1467 VM_BUG_ON_PAGE(PageActive(page), page);
1468 if (!PageMlocked(page)) {
1469 SetPageActive(page);
1470 stat->nr_activate++;
1471 count_memcg_page_event(page, PGACTIVATE);
1472 }
1473keep_locked:
1474 unlock_page(page);
1475keep:
1476 list_add(&page->lru, &ret_pages);
1477 VM_BUG_ON_PAGE(PageLRU(page) || PageUnevictable(page), page);
1478 }
1479
1480 mem_cgroup_uncharge_list(&free_pages);
1481 try_to_unmap_flush();
1482 free_unref_page_list(&free_pages);
1483
1484 list_splice(&ret_pages, page_list);
1485 count_vm_events(PGACTIVATE, stat->nr_activate);
1486
1487 return nr_reclaimed;
1488}
1489
1490unsigned long reclaim_clean_pages_from_list(struct zone *zone,
1491 struct list_head *page_list)
1492{
1493 struct scan_control sc = {
1494 .gfp_mask = GFP_KERNEL,
1495 .priority = DEF_PRIORITY,
1496 .may_unmap = 1,
1497 };
1498 struct reclaim_stat dummy_stat;
1499 unsigned long ret;
1500 struct page *page, *next;
1501 LIST_HEAD(clean_pages);
1502
1503 list_for_each_entry_safe(page, next, page_list, lru) {
1504 if (page_is_file_cache(page) && !PageDirty(page) &&
1505 !__PageMovable(page)) {
1506 ClearPageActive(page);
1507 list_move(&page->lru, &clean_pages);
1508 }
1509 }
1510
1511 ret = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
1512 TTU_IGNORE_ACCESS, &dummy_stat, true);
1513 list_splice(&clean_pages, page_list);
1514 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, -ret);
1515 return ret;
1516}
1517
1518/*
1519 * Attempt to remove the specified page from its LRU. Only take this page
1520 * if it is of the appropriate PageActive status. Pages which are being
1521 * freed elsewhere are also ignored.
1522 *
1523 * page: page to consider
1524 * mode: one of the LRU isolation modes defined above
1525 *
1526 * returns 0 on success, -ve errno on failure.
1527 */
1528int __isolate_lru_page(struct page *page, isolate_mode_t mode)
1529{
1530 int ret = -EINVAL;
1531
1532 /* Only take pages on the LRU. */
1533 if (!PageLRU(page))
1534 return ret;
1535
1536 /* Compaction should not handle unevictable pages but CMA can do so */
1537 if (PageUnevictable(page) && !(mode & ISOLATE_UNEVICTABLE))
1538 return ret;
1539
1540 ret = -EBUSY;
1541
1542 /*
1543 * To minimise LRU disruption, the caller can indicate that it only
1544 * wants to isolate pages it will be able to operate on without
1545 * blocking - clean pages for the most part.
1546 *
1547 * ISOLATE_ASYNC_MIGRATE is used to indicate that it only wants to pages
1548 * that it is possible to migrate without blocking
1549 */
1550 if (mode & ISOLATE_ASYNC_MIGRATE) {
1551 /* All the caller can do on PageWriteback is block */
1552 if (PageWriteback(page))
1553 return ret;
1554
1555 if (PageDirty(page)) {
1556 struct address_space *mapping;
1557 bool migrate_dirty;
1558
1559 /*
1560 * Only pages without mappings or that have a
1561 * ->migratepage callback are possible to migrate
1562 * without blocking. However, we can be racing with
1563 * truncation so it's necessary to lock the page
1564 * to stabilise the mapping as truncation holds
1565 * the page lock until after the page is removed
1566 * from the page cache.
1567 */
1568 if (!trylock_page(page))
1569 return ret;
1570
1571 mapping = page_mapping(page);
1572 migrate_dirty = !mapping || mapping->a_ops->migratepage;
1573 unlock_page(page);
1574 if (!migrate_dirty)
1575 return ret;
1576 }
1577 }
1578
1579 if ((mode & ISOLATE_UNMAPPED) && page_mapped(page))
1580 return ret;
1581
1582 if (likely(get_page_unless_zero(page))) {
1583 /*
1584 * Be careful not to clear PageLRU until after we're
1585 * sure the page is not being freed elsewhere -- the
1586 * page release code relies on it.
1587 */
1588 ClearPageLRU(page);
1589 ret = 0;
1590 }
1591
1592 return ret;
1593}
1594
1595
1596/*
1597 * Update LRU sizes after isolating pages. The LRU size updates must
1598 * be complete before mem_cgroup_update_lru_size due to a santity check.
1599 */
1600static __always_inline void update_lru_sizes(struct lruvec *lruvec,
1601 enum lru_list lru, unsigned long *nr_zone_taken)
1602{
1603 int zid;
1604
1605 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1606 if (!nr_zone_taken[zid])
1607 continue;
1608
1609 __update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1610#ifdef CONFIG_MEMCG
1611 mem_cgroup_update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1612#endif
1613 }
1614
1615}
1616
1617/**
1618 * pgdat->lru_lock is heavily contended. Some of the functions that
1619 * shrink the lists perform better by taking out a batch of pages
1620 * and working on them outside the LRU lock.
1621 *
1622 * For pagecache intensive workloads, this function is the hottest
1623 * spot in the kernel (apart from copy_*_user functions).
1624 *
1625 * Appropriate locks must be held before calling this function.
1626 *
1627 * @nr_to_scan: The number of eligible pages to look through on the list.
1628 * @lruvec: The LRU vector to pull pages from.
1629 * @dst: The temp list to put pages on to.
1630 * @nr_scanned: The number of pages that were scanned.
1631 * @sc: The scan_control struct for this reclaim session
1632 * @mode: One of the LRU isolation modes
1633 * @lru: LRU list id for isolating
1634 *
1635 * returns how many pages were moved onto *@dst.
1636 */
1637static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
1638 struct lruvec *lruvec, struct list_head *dst,
1639 unsigned long *nr_scanned, struct scan_control *sc,
1640 enum lru_list lru)
1641{
1642 struct list_head *src = &lruvec->lists[lru];
1643 unsigned long nr_taken = 0;
1644 unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 };
1645 unsigned long nr_skipped[MAX_NR_ZONES] = { 0, };
1646 unsigned long skipped = 0;
1647 unsigned long scan, total_scan, nr_pages;
1648 LIST_HEAD(pages_skipped);
1649 isolate_mode_t mode = (sc->may_unmap ? 0 : ISOLATE_UNMAPPED);
1650
1651 scan = 0;
1652 for (total_scan = 0;
1653 scan < nr_to_scan && nr_taken < nr_to_scan && !list_empty(src);
1654 total_scan++) {
1655 struct page *page;
1656
1657 page = lru_to_page(src);
1658 prefetchw_prev_lru_page(page, src, flags);
1659
1660 VM_BUG_ON_PAGE(!PageLRU(page), page);
1661
1662 if (page_zonenum(page) > sc->reclaim_idx) {
1663 list_move(&page->lru, &pages_skipped);
1664 nr_skipped[page_zonenum(page)]++;
1665 continue;
1666 }
1667
1668 /*
1669 * Do not count skipped pages because that makes the function
1670 * return with no isolated pages if the LRU mostly contains
1671 * ineligible pages. This causes the VM to not reclaim any
1672 * pages, triggering a premature OOM.
1673 */
1674 scan++;
1675 switch (__isolate_lru_page(page, mode)) {
1676 case 0:
1677 nr_pages = hpage_nr_pages(page);
1678 nr_taken += nr_pages;
1679 nr_zone_taken[page_zonenum(page)] += nr_pages;
1680 list_move(&page->lru, dst);
1681 break;
1682
1683 case -EBUSY:
1684 /* else it is being freed elsewhere */
1685 list_move(&page->lru, src);
1686 continue;
1687
1688 default:
1689 BUG();
1690 }
1691 }
1692
1693 /*
1694 * Splice any skipped pages to the start of the LRU list. Note that
1695 * this disrupts the LRU order when reclaiming for lower zones but
1696 * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX
1697 * scanning would soon rescan the same pages to skip and put the
1698 * system at risk of premature OOM.
1699 */
1700 if (!list_empty(&pages_skipped)) {
1701 int zid;
1702
1703 list_splice(&pages_skipped, src);
1704 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1705 if (!nr_skipped[zid])
1706 continue;
1707
1708 __count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]);
1709 skipped += nr_skipped[zid];
1710 }
1711 }
1712 *nr_scanned = total_scan;
1713 trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
1714 total_scan, skipped, nr_taken, mode, lru);
1715 update_lru_sizes(lruvec, lru, nr_zone_taken);
1716 return nr_taken;
1717}
1718
1719/**
1720 * isolate_lru_page - tries to isolate a page from its LRU list
1721 * @page: page to isolate from its LRU list
1722 *
1723 * Isolates a @page from an LRU list, clears PageLRU and adjusts the
1724 * vmstat statistic corresponding to whatever LRU list the page was on.
1725 *
1726 * Returns 0 if the page was removed from an LRU list.
1727 * Returns -EBUSY if the page was not on an LRU list.
1728 *
1729 * The returned page will have PageLRU() cleared. If it was found on
1730 * the active list, it will have PageActive set. If it was found on
1731 * the unevictable list, it will have the PageUnevictable bit set. That flag
1732 * may need to be cleared by the caller before letting the page go.
1733 *
1734 * The vmstat statistic corresponding to the list on which the page was
1735 * found will be decremented.
1736 *
1737 * Restrictions:
1738 *
1739 * (1) Must be called with an elevated refcount on the page. This is a
1740 * fundamentnal difference from isolate_lru_pages (which is called
1741 * without a stable reference).
1742 * (2) the lru_lock must not be held.
1743 * (3) interrupts must be enabled.
1744 */
1745int isolate_lru_page(struct page *page)
1746{
1747 int ret = -EBUSY;
1748
1749 VM_BUG_ON_PAGE(!page_count(page), page);
1750 WARN_RATELIMIT(PageTail(page), "trying to isolate tail page");
1751
1752 if (PageLRU(page)) {
1753 pg_data_t *pgdat = page_pgdat(page);
1754 struct lruvec *lruvec;
1755
1756 spin_lock_irq(&pgdat->lru_lock);
1757 lruvec = mem_cgroup_page_lruvec(page, pgdat);
1758 if (PageLRU(page)) {
1759 int lru = page_lru(page);
1760 get_page(page);
1761 ClearPageLRU(page);
1762 del_page_from_lru_list(page, lruvec, lru);
1763 ret = 0;
1764 }
1765 spin_unlock_irq(&pgdat->lru_lock);
1766 }
1767 return ret;
1768}
1769
1770/*
1771 * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
1772 * then get resheduled. When there are massive number of tasks doing page
1773 * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
1774 * the LRU list will go small and be scanned faster than necessary, leading to
1775 * unnecessary swapping, thrashing and OOM.
1776 */
1777static int too_many_isolated(struct pglist_data *pgdat, int file,
1778 struct scan_control *sc)
1779{
1780 unsigned long inactive, isolated;
1781
1782 if (current_is_kswapd())
1783 return 0;
1784
1785 if (!sane_reclaim(sc))
1786 return 0;
1787
1788 if (file) {
1789 inactive = node_page_state(pgdat, NR_INACTIVE_FILE);
1790 isolated = node_page_state(pgdat, NR_ISOLATED_FILE);
1791 } else {
1792 inactive = node_page_state(pgdat, NR_INACTIVE_ANON);
1793 isolated = node_page_state(pgdat, NR_ISOLATED_ANON);
1794 }
1795
1796 /*
1797 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
1798 * won't get blocked by normal direct-reclaimers, forming a circular
1799 * deadlock.
1800 */
1801 if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
1802 inactive >>= 3;
1803
1804 return isolated > inactive;
1805}
1806
1807static noinline_for_stack void
1808putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)
1809{
1810 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1811 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1812 LIST_HEAD(pages_to_free);
1813
1814 /*
1815 * Put back any unfreeable pages.
1816 */
1817 while (!list_empty(page_list)) {
1818 struct page *page = lru_to_page(page_list);
1819 int lru;
1820
1821 VM_BUG_ON_PAGE(PageLRU(page), page);
1822 list_del(&page->lru);
1823 if (unlikely(!page_evictable(page))) {
1824 spin_unlock_irq(&pgdat->lru_lock);
1825 putback_lru_page(page);
1826 spin_lock_irq(&pgdat->lru_lock);
1827 continue;
1828 }
1829
1830 lruvec = mem_cgroup_page_lruvec(page, pgdat);
1831
1832 SetPageLRU(page);
1833 lru = page_lru(page);
1834 add_page_to_lru_list(page, lruvec, lru);
1835
1836 if (is_active_lru(lru)) {
1837 int file = is_file_lru(lru);
1838 int numpages = hpage_nr_pages(page);
1839 reclaim_stat->recent_rotated[file] += numpages;
1840 }
1841 if (put_page_testzero(page)) {
1842 __ClearPageLRU(page);
1843 __ClearPageActive(page);
1844 del_page_from_lru_list(page, lruvec, lru);
1845
1846 if (unlikely(PageCompound(page))) {
1847 spin_unlock_irq(&pgdat->lru_lock);
1848 mem_cgroup_uncharge(page);
1849 (*get_compound_page_dtor(page))(page);
1850 spin_lock_irq(&pgdat->lru_lock);
1851 } else
1852 list_add(&page->lru, &pages_to_free);
1853 }
1854 }
1855
1856 /*
1857 * To save our caller's stack, now use input list for pages to free.
1858 */
1859 list_splice(&pages_to_free, page_list);
1860}
1861
1862/*
1863 * If a kernel thread (such as nfsd for loop-back mounts) services
1864 * a backing device by writing to the page cache it sets PF_LESS_THROTTLE.
1865 * In that case we should only throttle if the backing device it is
1866 * writing to is congested. In other cases it is safe to throttle.
1867 */
1868static int current_may_throttle(void)
1869{
1870 return !(current->flags & PF_LESS_THROTTLE) ||
1871 current->backing_dev_info == NULL ||
1872 bdi_write_congested(current->backing_dev_info);
1873}
1874
1875/*
1876 * shrink_inactive_list() is a helper for shrink_node(). It returns the number
1877 * of reclaimed pages
1878 */
1879static noinline_for_stack unsigned long
1880shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
1881 struct scan_control *sc, enum lru_list lru)
1882{
1883 LIST_HEAD(page_list);
1884 unsigned long nr_scanned;
1885 unsigned long nr_reclaimed = 0;
1886 unsigned long nr_taken;
1887 struct reclaim_stat stat;
1888 int file = is_file_lru(lru);
1889 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1890 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1891 bool stalled = false;
1892
1893 while (unlikely(too_many_isolated(pgdat, file, sc))) {
1894 if (stalled)
1895 return 0;
1896
1897 /* wait a bit for the reclaimer. */
1898 msleep(100);
1899 stalled = true;
1900
1901 /* We are about to die and free our memory. Return now. */
1902 if (fatal_signal_pending(current))
1903 return SWAP_CLUSTER_MAX;
1904 }
1905
1906 lru_add_drain();
1907
1908 spin_lock_irq(&pgdat->lru_lock);
1909
1910 nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
1911 &nr_scanned, sc, lru);
1912
1913 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
1914 reclaim_stat->recent_scanned[file] += nr_taken;
1915
1916 if (current_is_kswapd()) {
1917 if (global_reclaim(sc))
1918 __count_vm_events(PGSCAN_KSWAPD, nr_scanned);
1919 count_memcg_events(lruvec_memcg(lruvec), PGSCAN_KSWAPD,
1920 nr_scanned);
1921 } else {
1922 if (global_reclaim(sc))
1923 __count_vm_events(PGSCAN_DIRECT, nr_scanned);
1924 count_memcg_events(lruvec_memcg(lruvec), PGSCAN_DIRECT,
1925 nr_scanned);
1926 }
1927 spin_unlock_irq(&pgdat->lru_lock);
1928
1929 if (nr_taken == 0)
1930 return 0;
1931
1932 nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, 0,
1933 &stat, false);
1934
1935 spin_lock_irq(&pgdat->lru_lock);
1936
1937 if (current_is_kswapd()) {
1938 if (global_reclaim(sc))
1939 __count_vm_events(PGSTEAL_KSWAPD, nr_reclaimed);
1940 count_memcg_events(lruvec_memcg(lruvec), PGSTEAL_KSWAPD,
1941 nr_reclaimed);
1942 } else {
1943 if (global_reclaim(sc))
1944 __count_vm_events(PGSTEAL_DIRECT, nr_reclaimed);
1945 count_memcg_events(lruvec_memcg(lruvec), PGSTEAL_DIRECT,
1946 nr_reclaimed);
1947 }
1948
1949 putback_inactive_pages(lruvec, &page_list);
1950
1951 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
1952
1953 spin_unlock_irq(&pgdat->lru_lock);
1954
1955 mem_cgroup_uncharge_list(&page_list);
1956 free_unref_page_list(&page_list);
1957
1958 /*
1959 * If dirty pages are scanned that are not queued for IO, it
1960 * implies that flushers are not doing their job. This can
1961 * happen when memory pressure pushes dirty pages to the end of
1962 * the LRU before the dirty limits are breached and the dirty
1963 * data has expired. It can also happen when the proportion of
1964 * dirty pages grows not through writes but through memory
1965 * pressure reclaiming all the clean cache. And in some cases,
1966 * the flushers simply cannot keep up with the allocation
1967 * rate. Nudge the flusher threads in case they are asleep.
1968 */
1969 if (stat.nr_unqueued_dirty == nr_taken)
1970 wakeup_flusher_threads(WB_REASON_VMSCAN);
1971
1972 sc->nr.dirty += stat.nr_dirty;
1973 sc->nr.congested += stat.nr_congested;
1974 sc->nr.unqueued_dirty += stat.nr_unqueued_dirty;
1975 sc->nr.writeback += stat.nr_writeback;
1976 sc->nr.immediate += stat.nr_immediate;
1977 sc->nr.taken += nr_taken;
1978 if (file)
1979 sc->nr.file_taken += nr_taken;
1980
1981 trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
1982 nr_scanned, nr_reclaimed, &stat, sc->priority, file);
1983 return nr_reclaimed;
1984}
1985
1986/*
1987 * This moves pages from the active list to the inactive list.
1988 *
1989 * We move them the other way if the page is referenced by one or more
1990 * processes, from rmap.
1991 *
1992 * If the pages are mostly unmapped, the processing is fast and it is
1993 * appropriate to hold pgdat->lru_lock across the whole operation. But if
1994 * the pages are mapped, the processing is slow (page_referenced()) so we
1995 * should drop pgdat->lru_lock around each page. It's impossible to balance
1996 * this, so instead we remove the pages from the LRU while processing them.
1997 * It is safe to rely on PG_active against the non-LRU pages in here because
1998 * nobody will play with that bit on a non-LRU page.
1999 *
2000 * The downside is that we have to touch page->_refcount against each page.
2001 * But we had to alter page->flags anyway.
2002 *
2003 * Returns the number of pages moved to the given lru.
2004 */
2005
2006static unsigned move_active_pages_to_lru(struct lruvec *lruvec,
2007 struct list_head *list,
2008 struct list_head *pages_to_free,
2009 enum lru_list lru)
2010{
2011 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2012 struct page *page;
2013 int nr_pages;
2014 int nr_moved = 0;
2015
2016 while (!list_empty(list)) {
2017 page = lru_to_page(list);
2018 lruvec = mem_cgroup_page_lruvec(page, pgdat);
2019
2020 VM_BUG_ON_PAGE(PageLRU(page), page);
2021 SetPageLRU(page);
2022
2023 nr_pages = hpage_nr_pages(page);
2024 update_lru_size(lruvec, lru, page_zonenum(page), nr_pages);
2025 list_move(&page->lru, &lruvec->lists[lru]);
2026
2027 if (put_page_testzero(page)) {
2028 __ClearPageLRU(page);
2029 __ClearPageActive(page);
2030 del_page_from_lru_list(page, lruvec, lru);
2031
2032 if (unlikely(PageCompound(page))) {
2033 spin_unlock_irq(&pgdat->lru_lock);
2034 mem_cgroup_uncharge(page);
2035 (*get_compound_page_dtor(page))(page);
2036 spin_lock_irq(&pgdat->lru_lock);
2037 } else
2038 list_add(&page->lru, pages_to_free);
2039 } else {
2040 nr_moved += nr_pages;
2041 }
2042 }
2043
2044 if (!is_active_lru(lru)) {
2045 __count_vm_events(PGDEACTIVATE, nr_moved);
2046 count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
2047 nr_moved);
2048 }
2049
2050 return nr_moved;
2051}
2052
2053static void shrink_active_list(unsigned long nr_to_scan,
2054 struct lruvec *lruvec,
2055 struct scan_control *sc,
2056 enum lru_list lru)
2057{
2058 unsigned long nr_taken;
2059 unsigned long nr_scanned;
2060 unsigned long vm_flags;
2061 LIST_HEAD(l_hold); /* The pages which were snipped off */
2062 LIST_HEAD(l_active);
2063 LIST_HEAD(l_inactive);
2064 struct page *page;
2065 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2066 unsigned nr_deactivate, nr_activate;
2067 unsigned nr_rotated = 0;
2068 int file = is_file_lru(lru);
2069 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2070
2071 lru_add_drain();
2072
2073 spin_lock_irq(&pgdat->lru_lock);
2074
2075 nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
2076 &nr_scanned, sc, lru);
2077
2078 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2079 reclaim_stat->recent_scanned[file] += nr_taken;
2080
2081 __count_vm_events(PGREFILL, nr_scanned);
2082 count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
2083
2084 spin_unlock_irq(&pgdat->lru_lock);
2085
2086 while (!list_empty(&l_hold)) {
2087 cond_resched();
2088 page = lru_to_page(&l_hold);
2089 list_del(&page->lru);
2090
2091 if (unlikely(!page_evictable(page))) {
2092 putback_lru_page(page);
2093 continue;
2094 }
2095
2096 if (unlikely(buffer_heads_over_limit)) {
2097 if (page_has_private(page) && trylock_page(page)) {
2098 if (page_has_private(page))
2099 try_to_release_page(page, 0);
2100 unlock_page(page);
2101 }
2102 }
2103
2104 if (page_referenced(page, 0, sc->target_mem_cgroup,
2105 &vm_flags)) {
2106 nr_rotated += hpage_nr_pages(page);
2107 /*
2108 * Identify referenced, file-backed active pages and
2109 * give them one more trip around the active list. So
2110 * that executable code get better chances to stay in
2111 * memory under moderate memory pressure. Anon pages
2112 * are not likely to be evicted by use-once streaming
2113 * IO, plus JVM can create lots of anon VM_EXEC pages,
2114 * so we ignore them here.
2115 */
2116 if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) {
2117 list_add(&page->lru, &l_active);
2118 continue;
2119 }
2120 }
2121
2122 ClearPageActive(page); /* we are de-activating */
2123 SetPageWorkingset(page);
2124 list_add(&page->lru, &l_inactive);
2125 }
2126
2127 /*
2128 * Move pages back to the lru list.
2129 */
2130 spin_lock_irq(&pgdat->lru_lock);
2131 /*
2132 * Count referenced pages from currently used mappings as rotated,
2133 * even though only some of them are actually re-activated. This
2134 * helps balance scan pressure between file and anonymous pages in
2135 * get_scan_count.
2136 */
2137 reclaim_stat->recent_rotated[file] += nr_rotated;
2138
2139 nr_activate = move_active_pages_to_lru(lruvec, &l_active, &l_hold, lru);
2140 nr_deactivate = move_active_pages_to_lru(lruvec, &l_inactive, &l_hold, lru - LRU_ACTIVE);
2141 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2142 spin_unlock_irq(&pgdat->lru_lock);
2143
2144 mem_cgroup_uncharge_list(&l_hold);
2145 free_unref_page_list(&l_hold);
2146 trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
2147 nr_deactivate, nr_rotated, sc->priority, file);
2148}
2149
2150/*
2151 * The inactive anon list should be small enough that the VM never has
2152 * to do too much work.
2153 *
2154 * The inactive file list should be small enough to leave most memory
2155 * to the established workingset on the scan-resistant active list,
2156 * but large enough to avoid thrashing the aggregate readahead window.
2157 *
2158 * Both inactive lists should also be large enough that each inactive
2159 * page has a chance to be referenced again before it is reclaimed.
2160 *
2161 * If that fails and refaulting is observed, the inactive list grows.
2162 *
2163 * The inactive_ratio is the target ratio of ACTIVE to INACTIVE pages
2164 * on this LRU, maintained by the pageout code. An inactive_ratio
2165 * of 3 means 3:1 or 25% of the pages are kept on the inactive list.
2166 *
2167 * total target max
2168 * memory ratio inactive
2169 * -------------------------------------
2170 * 10MB 1 5MB
2171 * 100MB 1 50MB
2172 * 1GB 3 250MB
2173 * 10GB 10 0.9GB
2174 * 100GB 31 3GB
2175 * 1TB 101 10GB
2176 * 10TB 320 32GB
2177 */
2178static bool inactive_list_is_low(struct lruvec *lruvec, bool file,
2179 struct mem_cgroup *memcg,
2180 struct scan_control *sc, bool actual_reclaim)
2181{
2182 enum lru_list active_lru = file * LRU_FILE + LRU_ACTIVE;
2183 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2184 enum lru_list inactive_lru = file * LRU_FILE;
2185 unsigned long inactive, active;
2186 unsigned long inactive_ratio;
2187 unsigned long refaults;
2188 unsigned long gb;
2189
2190 /*
2191 * If we don't have swap space, anonymous page deactivation
2192 * is pointless.
2193 */
2194 if (!file && !total_swap_pages)
2195 return false;
2196
2197 inactive = lruvec_lru_size(lruvec, inactive_lru, sc->reclaim_idx);
2198 active = lruvec_lru_size(lruvec, active_lru, sc->reclaim_idx);
2199
2200 if (memcg)
2201 refaults = memcg_page_state(memcg, WORKINGSET_ACTIVATE);
2202 else
2203 refaults = node_page_state(pgdat, WORKINGSET_ACTIVATE);
2204
2205 /*
2206 * When refaults are being observed, it means a new workingset
2207 * is being established. Disable active list protection to get
2208 * rid of the stale workingset quickly.
2209 */
2210 if (file && actual_reclaim && lruvec->refaults != refaults) {
2211 inactive_ratio = 0;
2212 } else {
2213 gb = (inactive + active) >> (30 - PAGE_SHIFT);
2214 if (gb)
2215 inactive_ratio = int_sqrt(10 * gb);
2216 else
2217 inactive_ratio = 1;
2218 }
2219
2220 if (actual_reclaim)
2221 trace_mm_vmscan_inactive_list_is_low(pgdat->node_id, sc->reclaim_idx,
2222 lruvec_lru_size(lruvec, inactive_lru, MAX_NR_ZONES), inactive,
2223 lruvec_lru_size(lruvec, active_lru, MAX_NR_ZONES), active,
2224 inactive_ratio, file);
2225
2226 return inactive * inactive_ratio < active;
2227}
2228
2229static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2230 struct lruvec *lruvec, struct mem_cgroup *memcg,
2231 struct scan_control *sc)
2232{
2233 if (is_active_lru(lru)) {
2234 if (inactive_list_is_low(lruvec, is_file_lru(lru),
2235 memcg, sc, true))
2236 shrink_active_list(nr_to_scan, lruvec, sc, lru);
2237 return 0;
2238 }
2239
2240 return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2241}
2242
2243enum scan_balance {
2244 SCAN_EQUAL,
2245 SCAN_FRACT,
2246 SCAN_ANON,
2247 SCAN_FILE,
2248};
2249
2250/*
2251 * Determine how aggressively the anon and file LRU lists should be
2252 * scanned. The relative value of each set of LRU lists is determined
2253 * by looking at the fraction of the pages scanned we did rotate back
2254 * onto the active list instead of evict.
2255 *
2256 * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
2257 * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
2258 */
2259static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg,
2260 struct scan_control *sc, unsigned long *nr,
2261 unsigned long *lru_pages)
2262{
2263 int swappiness = mem_cgroup_swappiness(memcg);
2264 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2265 u64 fraction[2];
2266 u64 denominator = 0; /* gcc */
2267 struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2268 unsigned long anon_prio, file_prio;
2269 enum scan_balance scan_balance;
2270 unsigned long anon, file;
2271 unsigned long ap, fp;
2272 enum lru_list lru;
2273
2274 /* If we have no swap space, do not bother scanning anon pages. */
2275 if (!sc->may_swap || mem_cgroup_get_nr_swap_pages(memcg) <= 0) {
2276 scan_balance = SCAN_FILE;
2277 goto out;
2278 }
2279
2280 /*
2281 * Global reclaim will swap to prevent OOM even with no
2282 * swappiness, but memcg users want to use this knob to
2283 * disable swapping for individual groups completely when
2284 * using the memory controller's swap limit feature would be
2285 * too expensive.
2286 */
2287 if (!global_reclaim(sc) && !swappiness) {
2288 scan_balance = SCAN_FILE;
2289 goto out;
2290 }
2291
2292 /*
2293 * Do not apply any pressure balancing cleverness when the
2294 * system is close to OOM, scan both anon and file equally
2295 * (unless the swappiness setting disagrees with swapping).
2296 */
2297 if (!sc->priority && swappiness) {
2298 scan_balance = SCAN_EQUAL;
2299 goto out;
2300 }
2301
2302 /*
2303 * Prevent the reclaimer from falling into the cache trap: as
2304 * cache pages start out inactive, every cache fault will tip
2305 * the scan balance towards the file LRU. And as the file LRU
2306 * shrinks, so does the window for rotation from references.
2307 * This means we have a runaway feedback loop where a tiny
2308 * thrashing file LRU becomes infinitely more attractive than
2309 * anon pages. Try to detect this based on file LRU size.
2310 */
2311 if (global_reclaim(sc)) {
2312 unsigned long pgdatfile;
2313 unsigned long pgdatfree;
2314 int z;
2315 unsigned long total_high_wmark = 0;
2316
2317 pgdatfree = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2318 pgdatfile = node_page_state(pgdat, NR_ACTIVE_FILE) +
2319 node_page_state(pgdat, NR_INACTIVE_FILE);
2320
2321 for (z = 0; z < MAX_NR_ZONES; z++) {
2322 struct zone *zone = &pgdat->node_zones[z];
2323 if (!managed_zone(zone))
2324 continue;
2325
2326 total_high_wmark += high_wmark_pages(zone);
2327 }
2328
2329 if (unlikely(pgdatfile + pgdatfree <= total_high_wmark)) {
2330 /*
2331 * Force SCAN_ANON if there are enough inactive
2332 * anonymous pages on the LRU in eligible zones.
2333 * Otherwise, the small LRU gets thrashed.
2334 */
2335 if (!inactive_list_is_low(lruvec, false, memcg, sc, false) &&
2336 lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, sc->reclaim_idx)
2337 >> sc->priority) {
2338 scan_balance = SCAN_ANON;
2339 goto out;
2340 }
2341 }
2342 }
2343
2344 /*
2345 * If there is enough inactive page cache, i.e. if the size of the
2346 * inactive list is greater than that of the active list *and* the
2347 * inactive list actually has some pages to scan on this priority, we
2348 * do not reclaim anything from the anonymous working set right now.
2349 * Without the second condition we could end up never scanning an
2350 * lruvec even if it has plenty of old anonymous pages unless the
2351 * system is under heavy pressure.
2352 */
2353 if (!inactive_list_is_low(lruvec, true, memcg, sc, false) &&
2354 lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, sc->reclaim_idx) >> sc->priority) {
2355 scan_balance = SCAN_FILE;
2356 goto out;
2357 }
2358
2359 scan_balance = SCAN_FRACT;
2360
2361 /*
2362 * With swappiness at 100, anonymous and file have the same priority.
2363 * This scanning priority is essentially the inverse of IO cost.
2364 */
2365 anon_prio = swappiness;
2366 file_prio = 200 - anon_prio;
2367
2368 /*
2369 * OK, so we have swap space and a fair amount of page cache
2370 * pages. We use the recently rotated / recently scanned
2371 * ratios to determine how valuable each cache is.
2372 *
2373 * Because workloads change over time (and to avoid overflow)
2374 * we keep these statistics as a floating average, which ends
2375 * up weighing recent references more than old ones.
2376 *
2377 * anon in [0], file in [1]
2378 */
2379
2380 anon = lruvec_lru_size(lruvec, LRU_ACTIVE_ANON, MAX_NR_ZONES) +
2381 lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, MAX_NR_ZONES);
2382 file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES) +
2383 lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, MAX_NR_ZONES);
2384
2385 spin_lock_irq(&pgdat->lru_lock);
2386 if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
2387 reclaim_stat->recent_scanned[0] /= 2;
2388 reclaim_stat->recent_rotated[0] /= 2;
2389 }
2390
2391 if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) {
2392 reclaim_stat->recent_scanned[1] /= 2;
2393 reclaim_stat->recent_rotated[1] /= 2;
2394 }
2395
2396 /*
2397 * The amount of pressure on anon vs file pages is inversely
2398 * proportional to the fraction of recently scanned pages on
2399 * each list that were recently referenced and in active use.
2400 */
2401 ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1);
2402 ap /= reclaim_stat->recent_rotated[0] + 1;
2403
2404 fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
2405 fp /= reclaim_stat->recent_rotated[1] + 1;
2406 spin_unlock_irq(&pgdat->lru_lock);
2407
2408 fraction[0] = ap;
2409 fraction[1] = fp;
2410 denominator = ap + fp + 1;
2411out:
2412 *lru_pages = 0;
2413 for_each_evictable_lru(lru) {
2414 int file = is_file_lru(lru);
2415 unsigned long size;
2416 unsigned long scan;
2417
2418 size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
2419 scan = size >> sc->priority;
2420 /*
2421 * If the cgroup's already been deleted, make sure to
2422 * scrape out the remaining cache.
2423 */
2424 if (!scan && !mem_cgroup_online(memcg))
2425 scan = min(size, SWAP_CLUSTER_MAX);
2426
2427 switch (scan_balance) {
2428 case SCAN_EQUAL:
2429 /* Scan lists relative to size */
2430 break;
2431 case SCAN_FRACT:
2432 /*
2433 * Scan types proportional to swappiness and
2434 * their relative recent reclaim efficiency.
2435 * Make sure we don't miss the last page
2436 * because of a round-off error.
2437 */
2438 scan = DIV64_U64_ROUND_UP(scan * fraction[file],
2439 denominator);
2440 break;
2441 case SCAN_FILE:
2442 case SCAN_ANON:
2443 /* Scan one type exclusively */
2444 if ((scan_balance == SCAN_FILE) != file) {
2445 size = 0;
2446 scan = 0;
2447 }
2448 break;
2449 default:
2450 /* Look ma, no brain */
2451 BUG();
2452 }
2453
2454 *lru_pages += size;
2455 nr[lru] = scan;
2456 }
2457}
2458
2459/*
2460 * This is a basic per-node page freer. Used by both kswapd and direct reclaim.
2461 */
2462static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memcg,
2463 struct scan_control *sc, unsigned long *lru_pages)
2464{
2465 struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
2466 unsigned long nr[NR_LRU_LISTS];
2467 unsigned long targets[NR_LRU_LISTS];
2468 unsigned long nr_to_scan;
2469 enum lru_list lru;
2470 unsigned long nr_reclaimed = 0;
2471 unsigned long nr_to_reclaim = sc->nr_to_reclaim;
2472 struct blk_plug plug;
2473 bool scan_adjusted;
2474
2475 get_scan_count(lruvec, memcg, sc, nr, lru_pages);
2476
2477 /* Record the original scan target for proportional adjustments later */
2478 memcpy(targets, nr, sizeof(nr));
2479
2480 /*
2481 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
2482 * event that can occur when there is little memory pressure e.g.
2483 * multiple streaming readers/writers. Hence, we do not abort scanning
2484 * when the requested number of pages are reclaimed when scanning at
2485 * DEF_PRIORITY on the assumption that the fact we are direct
2486 * reclaiming implies that kswapd is not keeping up and it is best to
2487 * do a batch of work at once. For memcg reclaim one check is made to
2488 * abort proportional reclaim if either the file or anon lru has already
2489 * dropped to zero at the first pass.
2490 */
2491 scan_adjusted = (global_reclaim(sc) && !current_is_kswapd() &&
2492 sc->priority == DEF_PRIORITY);
2493
2494 blk_start_plug(&plug);
2495 while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
2496 nr[LRU_INACTIVE_FILE]) {
2497 unsigned long nr_anon, nr_file, percentage;
2498 unsigned long nr_scanned;
2499
2500 for_each_evictable_lru(lru) {
2501 if (nr[lru]) {
2502 nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
2503 nr[lru] -= nr_to_scan;
2504
2505 nr_reclaimed += shrink_list(lru, nr_to_scan,
2506 lruvec, memcg, sc);
2507 }
2508 }
2509
2510 cond_resched();
2511
2512 if (nr_reclaimed < nr_to_reclaim || scan_adjusted)
2513 continue;
2514
2515 /*
2516 * For kswapd and memcg, reclaim at least the number of pages
2517 * requested. Ensure that the anon and file LRUs are scanned
2518 * proportionally what was requested by get_scan_count(). We
2519 * stop reclaiming one LRU and reduce the amount scanning
2520 * proportional to the original scan target.
2521 */
2522 nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
2523 nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
2524
2525 /*
2526 * It's just vindictive to attack the larger once the smaller
2527 * has gone to zero. And given the way we stop scanning the
2528 * smaller below, this makes sure that we only make one nudge
2529 * towards proportionality once we've got nr_to_reclaim.
2530 */
2531 if (!nr_file || !nr_anon)
2532 break;
2533
2534 if (nr_file > nr_anon) {
2535 unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
2536 targets[LRU_ACTIVE_ANON] + 1;
2537 lru = LRU_BASE;
2538 percentage = nr_anon * 100 / scan_target;
2539 } else {
2540 unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
2541 targets[LRU_ACTIVE_FILE] + 1;
2542 lru = LRU_FILE;
2543 percentage = nr_file * 100 / scan_target;
2544 }
2545
2546 /* Stop scanning the smaller of the LRU */
2547 nr[lru] = 0;
2548 nr[lru + LRU_ACTIVE] = 0;
2549
2550 /*
2551 * Recalculate the other LRU scan count based on its original
2552 * scan target and the percentage scanning already complete
2553 */
2554 lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
2555 nr_scanned = targets[lru] - nr[lru];
2556 nr[lru] = targets[lru] * (100 - percentage) / 100;
2557 nr[lru] -= min(nr[lru], nr_scanned);
2558
2559 lru += LRU_ACTIVE;
2560 nr_scanned = targets[lru] - nr[lru];
2561 nr[lru] = targets[lru] * (100 - percentage) / 100;
2562 nr[lru] -= min(nr[lru], nr_scanned);
2563
2564 scan_adjusted = true;
2565 }
2566 blk_finish_plug(&plug);
2567 sc->nr_reclaimed += nr_reclaimed;
2568
2569 /*
2570 * Even if we did not try to evict anon pages at all, we want to
2571 * rebalance the anon lru active/inactive ratio.
2572 */
2573 if (inactive_list_is_low(lruvec, false, memcg, sc, true))
2574 shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
2575 sc, LRU_ACTIVE_ANON);
2576}
2577
2578/* Use reclaim/compaction for costly allocs or under memory pressure */
2579static bool in_reclaim_compaction(struct scan_control *sc)
2580{
2581 if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
2582 (sc->order > PAGE_ALLOC_COSTLY_ORDER ||
2583 sc->priority < DEF_PRIORITY - 2))
2584 return true;
2585
2586 return false;
2587}
2588
2589/*
2590 * Reclaim/compaction is used for high-order allocation requests. It reclaims
2591 * order-0 pages before compacting the zone. should_continue_reclaim() returns
2592 * true if more pages should be reclaimed such that when the page allocator
2593 * calls try_to_compact_zone() that it will have enough free pages to succeed.
2594 * It will give up earlier than that if there is difficulty reclaiming pages.
2595 */
2596static inline bool should_continue_reclaim(struct pglist_data *pgdat,
2597 unsigned long nr_reclaimed,
2598 unsigned long nr_scanned,
2599 struct scan_control *sc)
2600{
2601 unsigned long pages_for_compaction;
2602 unsigned long inactive_lru_pages;
2603 int z;
2604
2605 /* If not in reclaim/compaction mode, stop */
2606 if (!in_reclaim_compaction(sc))
2607 return false;
2608
2609 /* Consider stopping depending on scan and reclaim activity */
2610 if (sc->gfp_mask & __GFP_RETRY_MAYFAIL) {
2611 /*
2612 * For __GFP_RETRY_MAYFAIL allocations, stop reclaiming if the
2613 * full LRU list has been scanned and we are still failing
2614 * to reclaim pages. This full LRU scan is potentially
2615 * expensive but a __GFP_RETRY_MAYFAIL caller really wants to succeed
2616 */
2617 if (!nr_reclaimed && !nr_scanned)
2618 return false;
2619 } else {
2620 /*
2621 * For non-__GFP_RETRY_MAYFAIL allocations which can presumably
2622 * fail without consequence, stop if we failed to reclaim
2623 * any pages from the last SWAP_CLUSTER_MAX number of
2624 * pages that were scanned. This will return to the
2625 * caller faster at the risk reclaim/compaction and
2626 * the resulting allocation attempt fails
2627 */
2628 if (!nr_reclaimed)
2629 return false;
2630 }
2631
2632 /*
2633 * If we have not reclaimed enough pages for compaction and the
2634 * inactive lists are large enough, continue reclaiming
2635 */
2636 pages_for_compaction = compact_gap(sc->order);
2637 inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
2638 if (get_nr_swap_pages() > 0)
2639 inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
2640 if (sc->nr_reclaimed < pages_for_compaction &&
2641 inactive_lru_pages > pages_for_compaction)
2642 return true;
2643
2644 /* If compaction would go ahead or the allocation would succeed, stop */
2645 for (z = 0; z <= sc->reclaim_idx; z++) {
2646 struct zone *zone = &pgdat->node_zones[z];
2647 if (!managed_zone(zone))
2648 continue;
2649
2650 switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) {
2651 case COMPACT_SUCCESS:
2652 case COMPACT_CONTINUE:
2653 return false;
2654 default:
2655 /* check next zone */
2656 ;
2657 }
2658 }
2659 return true;
2660}
2661
2662static bool pgdat_memcg_congested(pg_data_t *pgdat, struct mem_cgroup *memcg)
2663{
2664 return test_bit(PGDAT_CONGESTED, &pgdat->flags) ||
2665 (memcg && memcg_congested(pgdat, memcg));
2666}
2667
2668static bool shrink_node(pg_data_t *pgdat, struct scan_control *sc)
2669{
2670 struct reclaim_state *reclaim_state = current->reclaim_state;
2671 unsigned long nr_reclaimed, nr_scanned;
2672 bool reclaimable = false;
2673
2674 do {
2675 struct mem_cgroup *root = sc->target_mem_cgroup;
2676 struct mem_cgroup_reclaim_cookie reclaim = {
2677 .pgdat = pgdat,
2678 .priority = sc->priority,
2679 };
2680 unsigned long node_lru_pages = 0;
2681 struct mem_cgroup *memcg;
2682
2683 memset(&sc->nr, 0, sizeof(sc->nr));
2684
2685 nr_reclaimed = sc->nr_reclaimed;
2686 nr_scanned = sc->nr_scanned;
2687
2688 memcg = mem_cgroup_iter(root, NULL, &reclaim);
2689 do {
2690 unsigned long lru_pages;
2691 unsigned long reclaimed;
2692 unsigned long scanned;
2693
2694 switch (mem_cgroup_protected(root, memcg)) {
2695 case MEMCG_PROT_MIN:
2696 /*
2697 * Hard protection.
2698 * If there is no reclaimable memory, OOM.
2699 */
2700 continue;
2701 case MEMCG_PROT_LOW:
2702 /*
2703 * Soft protection.
2704 * Respect the protection only as long as
2705 * there is an unprotected supply
2706 * of reclaimable memory from other cgroups.
2707 */
2708 if (!sc->memcg_low_reclaim) {
2709 sc->memcg_low_skipped = 1;
2710 continue;
2711 }
2712 memcg_memory_event(memcg, MEMCG_LOW);
2713 break;
2714 case MEMCG_PROT_NONE:
2715 break;
2716 }
2717
2718 reclaimed = sc->nr_reclaimed;
2719 scanned = sc->nr_scanned;
2720 shrink_node_memcg(pgdat, memcg, sc, &lru_pages);
2721 node_lru_pages += lru_pages;
2722
2723 if (sc->may_shrinkslab) {
2724 shrink_slab(sc->gfp_mask, pgdat->node_id,
2725 memcg, sc->priority);
2726 }
2727
2728 /* Record the group's reclaim efficiency */
2729 vmpressure(sc->gfp_mask, memcg, false,
2730 sc->nr_scanned - scanned,
2731 sc->nr_reclaimed - reclaimed);
2732
2733 /*
2734 * Kswapd have to scan all memory cgroups to fulfill
2735 * the overall scan target for the node.
2736 *
2737 * Limit reclaim, on the other hand, only cares about
2738 * nr_to_reclaim pages to be reclaimed and it will
2739 * retry with decreasing priority if one round over the
2740 * whole hierarchy is not sufficient.
2741 */
2742 if (!current_is_kswapd() &&
2743 sc->nr_reclaimed >= sc->nr_to_reclaim) {
2744 mem_cgroup_iter_break(root, memcg);
2745 break;
2746 }
2747 } while ((memcg = mem_cgroup_iter(root, memcg, &reclaim)));
2748
2749 if (reclaim_state) {
2750 sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2751 reclaim_state->reclaimed_slab = 0;
2752 }
2753
2754 /* Record the subtree's reclaim efficiency */
2755 vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
2756 sc->nr_scanned - nr_scanned,
2757 sc->nr_reclaimed - nr_reclaimed);
2758
2759 if (sc->nr_reclaimed - nr_reclaimed)
2760 reclaimable = true;
2761
2762 if (current_is_kswapd()) {
2763 /*
2764 * If reclaim is isolating dirty pages under writeback,
2765 * it implies that the long-lived page allocation rate
2766 * is exceeding the page laundering rate. Either the
2767 * global limits are not being effective at throttling
2768 * processes due to the page distribution throughout
2769 * zones or there is heavy usage of a slow backing
2770 * device. The only option is to throttle from reclaim
2771 * context which is not ideal as there is no guarantee
2772 * the dirtying process is throttled in the same way
2773 * balance_dirty_pages() manages.
2774 *
2775 * Once a node is flagged PGDAT_WRITEBACK, kswapd will
2776 * count the number of pages under pages flagged for
2777 * immediate reclaim and stall if any are encountered
2778 * in the nr_immediate check below.
2779 */
2780 if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
2781 set_bit(PGDAT_WRITEBACK, &pgdat->flags);
2782
2783 /*
2784 * Tag a node as congested if all the dirty pages
2785 * scanned were backed by a congested BDI and
2786 * wait_iff_congested will stall.
2787 */
2788 if (sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
2789 set_bit(PGDAT_CONGESTED, &pgdat->flags);
2790
2791 /* Allow kswapd to start writing pages during reclaim.*/
2792 if (sc->nr.unqueued_dirty == sc->nr.file_taken)
2793 set_bit(PGDAT_DIRTY, &pgdat->flags);
2794
2795 /*
2796 * If kswapd scans pages marked marked for immediate
2797 * reclaim and under writeback (nr_immediate), it
2798 * implies that pages are cycling through the LRU
2799 * faster than they are written so also forcibly stall.
2800 */
2801 if (sc->nr.immediate)
2802 congestion_wait(BLK_RW_ASYNC, HZ/10);
2803 }
2804
2805 /*
2806 * Legacy memcg will stall in page writeback so avoid forcibly
2807 * stalling in wait_iff_congested().
2808 */
2809 if (!global_reclaim(sc) && sane_reclaim(sc) &&
2810 sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
2811 set_memcg_congestion(pgdat, root, true);
2812
2813 /*
2814 * Stall direct reclaim for IO completions if underlying BDIs
2815 * and node is congested. Allow kswapd to continue until it
2816 * starts encountering unqueued dirty pages or cycling through
2817 * the LRU too quickly.
2818 */
2819 if (!sc->hibernation_mode && !current_is_kswapd() &&
2820 current_may_throttle() && pgdat_memcg_congested(pgdat, root))
2821 wait_iff_congested(BLK_RW_ASYNC, HZ/10);
2822
2823 } while (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
2824 sc->nr_scanned - nr_scanned, sc));
2825
2826 /*
2827 * Kswapd gives up on balancing particular nodes after too
2828 * many failures to reclaim anything from them and goes to
2829 * sleep. On reclaim progress, reset the failure counter. A
2830 * successful direct reclaim run will revive a dormant kswapd.
2831 */
2832 if (reclaimable)
2833 pgdat->kswapd_failures = 0;
2834
2835 return reclaimable;
2836}
2837
2838/*
2839 * Returns true if compaction should go ahead for a costly-order request, or
2840 * the allocation would already succeed without compaction. Return false if we
2841 * should reclaim first.
2842 */
2843static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
2844{
2845 unsigned long watermark;
2846 enum compact_result suitable;
2847
2848 suitable = compaction_suitable(zone, sc->order, 0, sc->reclaim_idx);
2849 if (suitable == COMPACT_SUCCESS)
2850 /* Allocation should succeed already. Don't reclaim. */
2851 return true;
2852 if (suitable == COMPACT_SKIPPED)
2853 /* Compaction cannot yet proceed. Do reclaim. */
2854 return false;
2855
2856 /*
2857 * Compaction is already possible, but it takes time to run and there
2858 * are potentially other callers using the pages just freed. So proceed
2859 * with reclaim to make a buffer of free pages available to give
2860 * compaction a reasonable chance of completing and allocating the page.
2861 * Note that we won't actually reclaim the whole buffer in one attempt
2862 * as the target watermark in should_continue_reclaim() is lower. But if
2863 * we are already above the high+gap watermark, don't reclaim at all.
2864 */
2865 watermark = high_wmark_pages(zone) + compact_gap(sc->order);
2866
2867 return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx);
2868}
2869
2870/*
2871 * This is the direct reclaim path, for page-allocating processes. We only
2872 * try to reclaim pages from zones which will satisfy the caller's allocation
2873 * request.
2874 *
2875 * If a zone is deemed to be full of pinned pages then just give it a light
2876 * scan then give up on it.
2877 */
2878static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
2879{
2880 struct zoneref *z;
2881 struct zone *zone;
2882 unsigned long nr_soft_reclaimed;
2883 unsigned long nr_soft_scanned;
2884 gfp_t orig_mask;
2885 pg_data_t *last_pgdat = NULL;
2886
2887 /*
2888 * If the number of buffer_heads in the machine exceeds the maximum
2889 * allowed level, force direct reclaim to scan the highmem zone as
2890 * highmem pages could be pinning lowmem pages storing buffer_heads
2891 */
2892 orig_mask = sc->gfp_mask;
2893 if (buffer_heads_over_limit) {
2894 sc->gfp_mask |= __GFP_HIGHMEM;
2895 sc->reclaim_idx = gfp_zone(sc->gfp_mask);
2896 }
2897
2898 for_each_zone_zonelist_nodemask(zone, z, zonelist,
2899 sc->reclaim_idx, sc->nodemask) {
2900 /*
2901 * Take care memory controller reclaiming has small influence
2902 * to global LRU.
2903 */
2904 if (global_reclaim(sc)) {
2905 if (!cpuset_zone_allowed(zone,
2906 GFP_KERNEL | __GFP_HARDWALL))
2907 continue;
2908
2909 /*
2910 * If we already have plenty of memory free for
2911 * compaction in this zone, don't free any more.
2912 * Even though compaction is invoked for any
2913 * non-zero order, only frequent costly order
2914 * reclamation is disruptive enough to become a
2915 * noticeable problem, like transparent huge
2916 * page allocations.
2917 */
2918 if (IS_ENABLED(CONFIG_COMPACTION) &&
2919 sc->order > PAGE_ALLOC_COSTLY_ORDER &&
2920 compaction_ready(zone, sc)) {
2921 sc->compaction_ready = true;
2922 continue;
2923 }
2924
2925 /*
2926 * Shrink each node in the zonelist once. If the
2927 * zonelist is ordered by zone (not the default) then a
2928 * node may be shrunk multiple times but in that case
2929 * the user prefers lower zones being preserved.
2930 */
2931 if (zone->zone_pgdat == last_pgdat)
2932 continue;
2933
2934 /*
2935 * This steals pages from memory cgroups over softlimit
2936 * and returns the number of reclaimed pages and
2937 * scanned pages. This works for global memory pressure
2938 * and balancing, not for a memcg's limit.
2939 */
2940 nr_soft_scanned = 0;
2941 nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat,
2942 sc->order, sc->gfp_mask,
2943 &nr_soft_scanned);
2944 sc->nr_reclaimed += nr_soft_reclaimed;
2945 sc->nr_scanned += nr_soft_scanned;
2946 /* need some check for avoid more shrink_zone() */
2947 }
2948
2949 /* See comment about same check for global reclaim above */
2950 if (zone->zone_pgdat == last_pgdat)
2951 continue;
2952 last_pgdat = zone->zone_pgdat;
2953 shrink_node(zone->zone_pgdat, sc);
2954 }
2955
2956 /*
2957 * Restore to original mask to avoid the impact on the caller if we
2958 * promoted it to __GFP_HIGHMEM.
2959 */
2960 sc->gfp_mask = orig_mask;
2961}
2962
2963static void snapshot_refaults(struct mem_cgroup *root_memcg, pg_data_t *pgdat)
2964{
2965 struct mem_cgroup *memcg;
2966
2967 memcg = mem_cgroup_iter(root_memcg, NULL, NULL);
2968 do {
2969 unsigned long refaults;
2970 struct lruvec *lruvec;
2971
2972 if (memcg)
2973 refaults = memcg_page_state(memcg, WORKINGSET_ACTIVATE);
2974 else
2975 refaults = node_page_state(pgdat, WORKINGSET_ACTIVATE);
2976
2977 lruvec = mem_cgroup_lruvec(pgdat, memcg);
2978 lruvec->refaults = refaults;
2979 } while ((memcg = mem_cgroup_iter(root_memcg, memcg, NULL)));
2980}
2981
2982/*
2983 * This is the main entry point to direct page reclaim.
2984 *
2985 * If a full scan of the inactive list fails to free enough memory then we
2986 * are "out of memory" and something needs to be killed.
2987 *
2988 * If the caller is !__GFP_FS then the probability of a failure is reasonably
2989 * high - the zone may be full of dirty or under-writeback pages, which this
2990 * caller can't do much about. We kick the writeback threads and take explicit
2991 * naps in the hope that some of these pages can be written. But if the
2992 * allocating task holds filesystem locks which prevent writeout this might not
2993 * work, and the allocation attempt will fail.
2994 *
2995 * returns: 0, if no pages reclaimed
2996 * else, the number of pages reclaimed
2997 */
2998static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
2999 struct scan_control *sc)
3000{
3001 int initial_priority = sc->priority;
3002 pg_data_t *last_pgdat;
3003 struct zoneref *z;
3004 struct zone *zone;
3005retry:
3006 delayacct_freepages_start();
3007
3008 if (global_reclaim(sc))
3009 __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
3010
3011 do {
3012 vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
3013 sc->priority);
3014 sc->nr_scanned = 0;
3015 shrink_zones(zonelist, sc);
3016
3017 if (sc->nr_reclaimed >= sc->nr_to_reclaim)
3018 break;
3019
3020 if (sc->compaction_ready)
3021 break;
3022
3023 /*
3024 * If we're getting trouble reclaiming, start doing
3025 * writepage even in laptop mode.
3026 */
3027 if (sc->priority < DEF_PRIORITY - 2)
3028 sc->may_writepage = 1;
3029 } while (--sc->priority >= 0);
3030
3031 last_pgdat = NULL;
3032 for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx,
3033 sc->nodemask) {
3034 if (zone->zone_pgdat == last_pgdat)
3035 continue;
3036 last_pgdat = zone->zone_pgdat;
3037 snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);
3038 set_memcg_congestion(last_pgdat, sc->target_mem_cgroup, false);
3039 }
3040
3041 delayacct_freepages_end();
3042
3043 if (sc->nr_reclaimed)
3044 return sc->nr_reclaimed;
3045
3046 /* Aborted reclaim to try compaction? don't OOM, then */
3047 if (sc->compaction_ready)
3048 return 1;
3049
3050 /* Untapped cgroup reserves? Don't OOM, retry. */
3051 if (sc->memcg_low_skipped) {
3052 sc->priority = initial_priority;
3053 sc->memcg_low_reclaim = 1;
3054 sc->memcg_low_skipped = 0;
3055 goto retry;
3056 }
3057
3058 return 0;
3059}
3060
3061static bool allow_direct_reclaim(pg_data_t *pgdat)
3062{
3063 struct zone *zone;
3064 unsigned long pfmemalloc_reserve = 0;
3065 unsigned long free_pages = 0;
3066 int i;
3067 bool wmark_ok;
3068
3069 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3070 return true;
3071
3072 for (i = 0; i <= ZONE_NORMAL; i++) {
3073 zone = &pgdat->node_zones[i];
3074 if (!managed_zone(zone))
3075 continue;
3076
3077 if (!zone_reclaimable_pages(zone))
3078 continue;
3079
3080 pfmemalloc_reserve += min_wmark_pages(zone);
3081 free_pages += zone_page_state(zone, NR_FREE_PAGES);
3082 }
3083
3084 /* If there are no reserves (unexpected config) then do not throttle */
3085 if (!pfmemalloc_reserve)
3086 return true;
3087
3088 wmark_ok = free_pages > pfmemalloc_reserve / 2;
3089
3090 /* kswapd must be awake if processes are being throttled */
3091 if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
3092 pgdat->kswapd_classzone_idx = min(pgdat->kswapd_classzone_idx,
3093 (enum zone_type)ZONE_NORMAL);
3094 wake_up_interruptible(&pgdat->kswapd_wait);
3095 }
3096
3097 return wmark_ok;
3098}
3099
3100/*
3101 * Throttle direct reclaimers if backing storage is backed by the network
3102 * and the PFMEMALLOC reserve for the preferred node is getting dangerously
3103 * depleted. kswapd will continue to make progress and wake the processes
3104 * when the low watermark is reached.
3105 *
3106 * Returns true if a fatal signal was delivered during throttling. If this
3107 * happens, the page allocator should not consider triggering the OOM killer.
3108 */
3109static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
3110 nodemask_t *nodemask)
3111{
3112 struct zoneref *z;
3113 struct zone *zone;
3114 pg_data_t *pgdat = NULL;
3115
3116 /*
3117 * Kernel threads should not be throttled as they may be indirectly
3118 * responsible for cleaning pages necessary for reclaim to make forward
3119 * progress. kjournald for example may enter direct reclaim while
3120 * committing a transaction where throttling it could forcing other
3121 * processes to block on log_wait_commit().
3122 */
3123 if (current->flags & PF_KTHREAD)
3124 goto out;
3125
3126 /*
3127 * If a fatal signal is pending, this process should not throttle.
3128 * It should return quickly so it can exit and free its memory
3129 */
3130 if (fatal_signal_pending(current))
3131 goto out;
3132
3133 /*
3134 * Check if the pfmemalloc reserves are ok by finding the first node
3135 * with a usable ZONE_NORMAL or lower zone. The expectation is that
3136 * GFP_KERNEL will be required for allocating network buffers when
3137 * swapping over the network so ZONE_HIGHMEM is unusable.
3138 *
3139 * Throttling is based on the first usable node and throttled processes
3140 * wait on a queue until kswapd makes progress and wakes them. There
3141 * is an affinity then between processes waking up and where reclaim
3142 * progress has been made assuming the process wakes on the same node.
3143 * More importantly, processes running on remote nodes will not compete
3144 * for remote pfmemalloc reserves and processes on different nodes
3145 * should make reasonable progress.
3146 */
3147 for_each_zone_zonelist_nodemask(zone, z, zonelist,
3148 gfp_zone(gfp_mask), nodemask) {
3149 if (zone_idx(zone) > ZONE_NORMAL)
3150 continue;
3151
3152 /* Throttle based on the first usable node */
3153 pgdat = zone->zone_pgdat;
3154 if (allow_direct_reclaim(pgdat))
3155 goto out;
3156 break;
3157 }
3158
3159 /* If no zone was usable by the allocation flags then do not throttle */
3160 if (!pgdat)
3161 goto out;
3162
3163 /* Account for the throttling */
3164 count_vm_event(PGSCAN_DIRECT_THROTTLE);
3165
3166 /*
3167 * If the caller cannot enter the filesystem, it's possible that it
3168 * is due to the caller holding an FS lock or performing a journal
3169 * transaction in the case of a filesystem like ext[3|4]. In this case,
3170 * it is not safe to block on pfmemalloc_wait as kswapd could be
3171 * blocked waiting on the same lock. Instead, throttle for up to a
3172 * second before continuing.
3173 */
3174 if (!(gfp_mask & __GFP_FS)) {
3175 wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
3176 allow_direct_reclaim(pgdat), HZ);
3177
3178 goto check_pending;
3179 }
3180
3181 /* Throttle until kswapd wakes the process */
3182 wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
3183 allow_direct_reclaim(pgdat));
3184
3185check_pending:
3186 if (fatal_signal_pending(current))
3187 return true;
3188
3189out:
3190 return false;
3191}
3192
3193unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
3194 gfp_t gfp_mask, nodemask_t *nodemask)
3195{
3196 unsigned long nr_reclaimed;
3197 struct scan_control sc = {
3198 .nr_to_reclaim = SWAP_CLUSTER_MAX,
3199 .gfp_mask = current_gfp_context(gfp_mask),
3200 .reclaim_idx = gfp_zone(gfp_mask),
3201 .order = order,
3202 .nodemask = nodemask,
3203 .priority = DEF_PRIORITY,
3204 .may_writepage = !laptop_mode,
3205 .may_unmap = 1,
3206 .may_swap = 1,
3207 .may_shrinkslab = 1,
3208 };
3209
3210 /*
3211 * scan_control uses s8 fields for order, priority, and reclaim_idx.
3212 * Confirm they are large enough for max values.
3213 */
3214 BUILD_BUG_ON(MAX_ORDER > S8_MAX);
3215 BUILD_BUG_ON(DEF_PRIORITY > S8_MAX);
3216 BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX);
3217
3218 /*
3219 * Do not enter reclaim if fatal signal was delivered while throttled.
3220 * 1 is returned so that the page allocator does not OOM kill at this
3221 * point.
3222 */
3223 if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
3224 return 1;
3225
3226 trace_mm_vmscan_direct_reclaim_begin(order,
3227 sc.may_writepage,
3228 sc.gfp_mask,
3229 sc.reclaim_idx);
3230
3231 nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3232
3233 trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
3234
3235 return nr_reclaimed;
3236}
3237
3238#ifdef CONFIG_MEMCG
3239
3240unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
3241 gfp_t gfp_mask, bool noswap,
3242 pg_data_t *pgdat,
3243 unsigned long *nr_scanned)
3244{
3245 struct scan_control sc = {
3246 .nr_to_reclaim = SWAP_CLUSTER_MAX,
3247 .target_mem_cgroup = memcg,
3248 .may_writepage = !laptop_mode,
3249 .may_unmap = 1,
3250 .reclaim_idx = MAX_NR_ZONES - 1,
3251 .may_swap = !noswap,
3252 .may_shrinkslab = 1,
3253 };
3254 unsigned long lru_pages;
3255
3256 sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
3257 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
3258
3259 trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
3260 sc.may_writepage,
3261 sc.gfp_mask,
3262 sc.reclaim_idx);
3263
3264 /*
3265 * NOTE: Although we can get the priority field, using it
3266 * here is not a good idea, since it limits the pages we can scan.
3267 * if we don't reclaim here, the shrink_node from balance_pgdat
3268 * will pick up pages from other mem cgroup's as well. We hack
3269 * the priority and make it zero.
3270 */
3271 shrink_node_memcg(pgdat, memcg, &sc, &lru_pages);
3272
3273 trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
3274
3275 *nr_scanned = sc.nr_scanned;
3276 return sc.nr_reclaimed;
3277}
3278
3279unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
3280 unsigned long nr_pages,
3281 gfp_t gfp_mask,
3282 bool may_swap)
3283{
3284 struct zonelist *zonelist;
3285 unsigned long nr_reclaimed;
3286 unsigned long pflags;
3287 int nid;
3288 unsigned int noreclaim_flag;
3289 struct scan_control sc = {
3290 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3291 .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) |
3292 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
3293 .reclaim_idx = MAX_NR_ZONES - 1,
3294 .target_mem_cgroup = memcg,
3295 .priority = DEF_PRIORITY,
3296 .may_writepage = !laptop_mode,
3297 .may_unmap = 1,
3298 .may_swap = may_swap,
3299 .may_shrinkslab = 1,
3300 };
3301
3302 /*
3303 * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't
3304 * take care of from where we get pages. So the node where we start the
3305 * scan does not need to be the current node.
3306 */
3307 nid = mem_cgroup_select_victim_node(memcg);
3308
3309 zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK];
3310
3311 trace_mm_vmscan_memcg_reclaim_begin(0,
3312 sc.may_writepage,
3313 sc.gfp_mask,
3314 sc.reclaim_idx);
3315
3316 psi_memstall_enter(&pflags);
3317 noreclaim_flag = memalloc_noreclaim_save();
3318
3319 nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3320
3321 memalloc_noreclaim_restore(noreclaim_flag);
3322 psi_memstall_leave(&pflags);
3323
3324 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
3325
3326 return nr_reclaimed;
3327}
3328#endif
3329
3330static void age_active_anon(struct pglist_data *pgdat,
3331 struct scan_control *sc)
3332{
3333 struct mem_cgroup *memcg;
3334
3335 if (!total_swap_pages)
3336 return;
3337
3338 memcg = mem_cgroup_iter(NULL, NULL, NULL);
3339 do {
3340 struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
3341
3342 if (inactive_list_is_low(lruvec, false, memcg, sc, true))
3343 shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3344 sc, LRU_ACTIVE_ANON);
3345
3346 memcg = mem_cgroup_iter(NULL, memcg, NULL);
3347 } while (memcg);
3348}
3349
3350static bool pgdat_watermark_boosted(pg_data_t *pgdat, int classzone_idx)
3351{
3352 int i;
3353 struct zone *zone;
3354
3355 /*
3356 * Check for watermark boosts top-down as the higher zones
3357 * are more likely to be boosted. Both watermarks and boosts
3358 * should not be checked at the time time as reclaim would
3359 * start prematurely when there is no boosting and a lower
3360 * zone is balanced.
3361 */
3362 for (i = classzone_idx; i >= 0; i--) {
3363 zone = pgdat->node_zones + i;
3364 if (!managed_zone(zone))
3365 continue;
3366
3367 if (zone->watermark_boost)
3368 return true;
3369 }
3370
3371 return false;
3372}
3373
3374/*
3375 * Returns true if there is an eligible zone balanced for the request order
3376 * and classzone_idx
3377 */
3378static bool pgdat_balanced(pg_data_t *pgdat, int order, int classzone_idx)
3379{
3380 int i;
3381 unsigned long mark = -1;
3382 struct zone *zone;
3383
3384 /*
3385 * Check watermarks bottom-up as lower zones are more likely to
3386 * meet watermarks.
3387 */
3388 for (i = 0; i <= classzone_idx; i++) {
3389 zone = pgdat->node_zones + i;
3390
3391 if (!managed_zone(zone))
3392 continue;
3393
3394 mark = high_wmark_pages(zone);
3395 if (zone_watermark_ok_safe(zone, order, mark, classzone_idx))
3396 return true;
3397 }
3398
3399 /*
3400 * If a node has no populated zone within classzone_idx, it does not
3401 * need balancing by definition. This can happen if a zone-restricted
3402 * allocation tries to wake a remote kswapd.
3403 */
3404 if (mark == -1)
3405 return true;
3406
3407 return false;
3408}
3409
3410/* Clear pgdat state for congested, dirty or under writeback. */
3411static void clear_pgdat_congested(pg_data_t *pgdat)
3412{
3413 clear_bit(PGDAT_CONGESTED, &pgdat->flags);
3414 clear_bit(PGDAT_DIRTY, &pgdat->flags);
3415 clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
3416}
3417
3418/*
3419 * Prepare kswapd for sleeping. This verifies that there are no processes
3420 * waiting in throttle_direct_reclaim() and that watermarks have been met.
3421 *
3422 * Returns true if kswapd is ready to sleep
3423 */
3424static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, int classzone_idx)
3425{
3426 /*
3427 * The throttled processes are normally woken up in balance_pgdat() as
3428 * soon as allow_direct_reclaim() is true. But there is a potential
3429 * race between when kswapd checks the watermarks and a process gets
3430 * throttled. There is also a potential race if processes get
3431 * throttled, kswapd wakes, a large process exits thereby balancing the
3432 * zones, which causes kswapd to exit balance_pgdat() before reaching
3433 * the wake up checks. If kswapd is going to sleep, no process should
3434 * be sleeping on pfmemalloc_wait, so wake them now if necessary. If
3435 * the wake up is premature, processes will wake kswapd and get
3436 * throttled again. The difference from wake ups in balance_pgdat() is
3437 * that here we are under prepare_to_wait().
3438 */
3439 if (waitqueue_active(&pgdat->pfmemalloc_wait))
3440 wake_up_all(&pgdat->pfmemalloc_wait);
3441
3442 /* Hopeless node, leave it to direct reclaim */
3443 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3444 return true;
3445
3446 if (pgdat_balanced(pgdat, order, classzone_idx)) {
3447 clear_pgdat_congested(pgdat);
3448 return true;
3449 }
3450
3451 return false;
3452}
3453
3454/*
3455 * kswapd shrinks a node of pages that are at or below the highest usable
3456 * zone that is currently unbalanced.
3457 *
3458 * Returns true if kswapd scanned at least the requested number of pages to
3459 * reclaim or if the lack of progress was due to pages under writeback.
3460 * This is used to determine if the scanning priority needs to be raised.
3461 */
3462static bool kswapd_shrink_node(pg_data_t *pgdat,
3463 struct scan_control *sc)
3464{
3465 struct zone *zone;
3466 int z;
3467
3468 /* Reclaim a number of pages proportional to the number of zones */
3469 sc->nr_to_reclaim = 0;
3470 for (z = 0; z <= sc->reclaim_idx; z++) {
3471 zone = pgdat->node_zones + z;
3472 if (!managed_zone(zone))
3473 continue;
3474
3475 sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX);
3476 }
3477
3478 /*
3479 * Historically care was taken to put equal pressure on all zones but
3480 * now pressure is applied based on node LRU order.
3481 */
3482 shrink_node(pgdat, sc);
3483
3484 /*
3485 * Fragmentation may mean that the system cannot be rebalanced for
3486 * high-order allocations. If twice the allocation size has been
3487 * reclaimed then recheck watermarks only at order-0 to prevent
3488 * excessive reclaim. Assume that a process requested a high-order
3489 * can direct reclaim/compact.
3490 */
3491 if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order))
3492 sc->order = 0;
3493
3494 return sc->nr_scanned >= sc->nr_to_reclaim;
3495}
3496
3497/*
3498 * For kswapd, balance_pgdat() will reclaim pages across a node from zones
3499 * that are eligible for use by the caller until at least one zone is
3500 * balanced.
3501 *
3502 * Returns the order kswapd finished reclaiming at.
3503 *
3504 * kswapd scans the zones in the highmem->normal->dma direction. It skips
3505 * zones which have free_pages > high_wmark_pages(zone), but once a zone is
3506 * found to have free_pages <= high_wmark_pages(zone), any page in that zone
3507 * or lower is eligible for reclaim until at least one usable zone is
3508 * balanced.
3509 */
3510static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
3511{
3512 int i;
3513 unsigned long nr_soft_reclaimed;
3514 unsigned long nr_soft_scanned;
3515 unsigned long pflags;
3516 unsigned long nr_boost_reclaim;
3517 unsigned long zone_boosts[MAX_NR_ZONES] = { 0, };
3518 bool boosted;
3519 struct zone *zone;
3520 struct scan_control sc = {
3521 .gfp_mask = GFP_KERNEL,
3522 .order = order,
3523 .may_unmap = 1,
3524 };
3525
3526 psi_memstall_enter(&pflags);
3527 __fs_reclaim_acquire();
3528
3529 count_vm_event(PAGEOUTRUN);
3530
3531 /*
3532 * Account for the reclaim boost. Note that the zone boost is left in
3533 * place so that parallel allocations that are near the watermark will
3534 * stall or direct reclaim until kswapd is finished.
3535 */
3536 nr_boost_reclaim = 0;
3537 for (i = 0; i <= classzone_idx; i++) {
3538 zone = pgdat->node_zones + i;
3539 if (!managed_zone(zone))
3540 continue;
3541
3542 nr_boost_reclaim += zone->watermark_boost;
3543 zone_boosts[i] = zone->watermark_boost;
3544 }
3545 boosted = nr_boost_reclaim;
3546
3547restart:
3548 sc.priority = DEF_PRIORITY;
3549 do {
3550 unsigned long nr_reclaimed = sc.nr_reclaimed;
3551 bool raise_priority = true;
3552 bool balanced;
3553 bool ret;
3554
3555 sc.reclaim_idx = classzone_idx;
3556
3557 /*
3558 * If the number of buffer_heads exceeds the maximum allowed
3559 * then consider reclaiming from all zones. This has a dual
3560 * purpose -- on 64-bit systems it is expected that
3561 * buffer_heads are stripped during active rotation. On 32-bit
3562 * systems, highmem pages can pin lowmem memory and shrinking
3563 * buffers can relieve lowmem pressure. Reclaim may still not
3564 * go ahead if all eligible zones for the original allocation
3565 * request are balanced to avoid excessive reclaim from kswapd.
3566 */
3567 if (buffer_heads_over_limit) {
3568 for (i = MAX_NR_ZONES - 1; i >= 0; i--) {
3569 zone = pgdat->node_zones + i;
3570 if (!managed_zone(zone))
3571 continue;
3572
3573 sc.reclaim_idx = i;
3574 break;
3575 }
3576 }
3577
3578 /*
3579 * If the pgdat is imbalanced then ignore boosting and preserve
3580 * the watermarks for a later time and restart. Note that the
3581 * zone watermarks will be still reset at the end of balancing
3582 * on the grounds that the normal reclaim should be enough to
3583 * re-evaluate if boosting is required when kswapd next wakes.
3584 */
3585 balanced = pgdat_balanced(pgdat, sc.order, classzone_idx);
3586 if (!balanced && nr_boost_reclaim) {
3587 nr_boost_reclaim = 0;
3588 goto restart;
3589 }
3590
3591 /*
3592 * If boosting is not active then only reclaim if there are no
3593 * eligible zones. Note that sc.reclaim_idx is not used as
3594 * buffer_heads_over_limit may have adjusted it.
3595 */
3596 if (!nr_boost_reclaim && balanced)
3597 goto out;
3598
3599 /* Limit the priority of boosting to avoid reclaim writeback */
3600 if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2)
3601 raise_priority = false;
3602
3603 /*
3604 * Do not writeback or swap pages for boosted reclaim. The
3605 * intent is to relieve pressure not issue sub-optimal IO
3606 * from reclaim context. If no pages are reclaimed, the
3607 * reclaim will be aborted.
3608 */
3609 sc.may_writepage = !laptop_mode && !nr_boost_reclaim;
3610 sc.may_swap = !nr_boost_reclaim;
3611 sc.may_shrinkslab = !nr_boost_reclaim;
3612
3613 /*
3614 * Do some background aging of the anon list, to give
3615 * pages a chance to be referenced before reclaiming. All
3616 * pages are rotated regardless of classzone as this is
3617 * about consistent aging.
3618 */
3619 age_active_anon(pgdat, &sc);
3620
3621 /*
3622 * If we're getting trouble reclaiming, start doing writepage
3623 * even in laptop mode.
3624 */
3625 if (sc.priority < DEF_PRIORITY - 2)
3626 sc.may_writepage = 1;
3627
3628 /* Call soft limit reclaim before calling shrink_node. */
3629 sc.nr_scanned = 0;
3630 nr_soft_scanned = 0;
3631 nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order,
3632 sc.gfp_mask, &nr_soft_scanned);
3633 sc.nr_reclaimed += nr_soft_reclaimed;
3634
3635 /*
3636 * There should be no need to raise the scanning priority if
3637 * enough pages are already being scanned that that high
3638 * watermark would be met at 100% efficiency.
3639 */
3640 if (kswapd_shrink_node(pgdat, &sc))
3641 raise_priority = false;
3642
3643 /*
3644 * If the low watermark is met there is no need for processes
3645 * to be throttled on pfmemalloc_wait as they should not be
3646 * able to safely make forward progress. Wake them
3647 */
3648 if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
3649 allow_direct_reclaim(pgdat))
3650 wake_up_all(&pgdat->pfmemalloc_wait);
3651
3652 /* Check if kswapd should be suspending */
3653 __fs_reclaim_release();
3654 ret = try_to_freeze();
3655 __fs_reclaim_acquire();
3656 if (ret || kthread_should_stop())
3657 break;
3658
3659 /*
3660 * Raise priority if scanning rate is too low or there was no
3661 * progress in reclaiming pages
3662 */
3663 nr_reclaimed = sc.nr_reclaimed - nr_reclaimed;
3664 nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed);
3665
3666 /*
3667 * If reclaim made no progress for a boost, stop reclaim as
3668 * IO cannot be queued and it could be an infinite loop in
3669 * extreme circumstances.
3670 */
3671 if (nr_boost_reclaim && !nr_reclaimed)
3672 break;
3673
3674 if (raise_priority || !nr_reclaimed)
3675 sc.priority--;
3676 } while (sc.priority >= 1);
3677
3678 if (!sc.nr_reclaimed)
3679 pgdat->kswapd_failures++;
3680
3681out:
3682 /* If reclaim was boosted, account for the reclaim done in this pass */
3683 if (boosted) {
3684 unsigned long flags;
3685
3686 for (i = 0; i <= classzone_idx; i++) {
3687 if (!zone_boosts[i])
3688 continue;
3689
3690 /* Increments are under the zone lock */
3691 zone = pgdat->node_zones + i;
3692 spin_lock_irqsave(&zone->lock, flags);
3693 zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]);
3694 spin_unlock_irqrestore(&zone->lock, flags);
3695 }
3696
3697 /*
3698 * As there is now likely space, wakeup kcompact to defragment
3699 * pageblocks.
3700 */
3701 wakeup_kcompactd(pgdat, pageblock_order, classzone_idx);
3702 }
3703
3704 snapshot_refaults(NULL, pgdat);
3705 __fs_reclaim_release();
3706 psi_memstall_leave(&pflags);
3707 /*
3708 * Return the order kswapd stopped reclaiming at as
3709 * prepare_kswapd_sleep() takes it into account. If another caller
3710 * entered the allocator slow path while kswapd was awake, order will
3711 * remain at the higher level.
3712 */
3713 return sc.order;
3714}
3715
3716/*
3717 * pgdat->kswapd_classzone_idx is the highest zone index that a recent
3718 * allocation request woke kswapd for. When kswapd has not woken recently,
3719 * the value is MAX_NR_ZONES which is not a valid index. This compares a
3720 * given classzone and returns it or the highest classzone index kswapd
3721 * was recently woke for.
3722 */
3723static enum zone_type kswapd_classzone_idx(pg_data_t *pgdat,
3724 enum zone_type classzone_idx)
3725{
3726 if (pgdat->kswapd_classzone_idx == MAX_NR_ZONES)
3727 return classzone_idx;
3728
3729 return max(pgdat->kswapd_classzone_idx, classzone_idx);
3730}
3731
3732static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,
3733 unsigned int classzone_idx)
3734{
3735 long remaining = 0;
3736 DEFINE_WAIT(wait);
3737
3738 if (freezing(current) || kthread_should_stop())
3739 return;
3740
3741 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
3742
3743 /*
3744 * Try to sleep for a short interval. Note that kcompactd will only be
3745 * woken if it is possible to sleep for a short interval. This is
3746 * deliberate on the assumption that if reclaim cannot keep an
3747 * eligible zone balanced that it's also unlikely that compaction will
3748 * succeed.
3749 */
3750 if (prepare_kswapd_sleep(pgdat, reclaim_order, classzone_idx)) {
3751 /*
3752 * Compaction records what page blocks it recently failed to
3753 * isolate pages from and skips them in the future scanning.
3754 * When kswapd is going to sleep, it is reasonable to assume
3755 * that pages and compaction may succeed so reset the cache.
3756 */
3757 reset_isolation_suitable(pgdat);
3758
3759 /*
3760 * We have freed the memory, now we should compact it to make
3761 * allocation of the requested order possible.
3762 */
3763 wakeup_kcompactd(pgdat, alloc_order, classzone_idx);
3764
3765 remaining = schedule_timeout(HZ/10);
3766
3767 /*
3768 * If woken prematurely then reset kswapd_classzone_idx and
3769 * order. The values will either be from a wakeup request or
3770 * the previous request that slept prematurely.
3771 */
3772 if (remaining) {
3773 pgdat->kswapd_classzone_idx = kswapd_classzone_idx(pgdat, classzone_idx);
3774 pgdat->kswapd_order = max(pgdat->kswapd_order, reclaim_order);
3775 }
3776
3777 finish_wait(&pgdat->kswapd_wait, &wait);
3778 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
3779 }
3780
3781 /*
3782 * After a short sleep, check if it was a premature sleep. If not, then
3783 * go fully to sleep until explicitly woken up.
3784 */
3785 if (!remaining &&
3786 prepare_kswapd_sleep(pgdat, reclaim_order, classzone_idx)) {
3787 trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
3788
3789 /*
3790 * vmstat counters are not perfectly accurate and the estimated
3791 * value for counters such as NR_FREE_PAGES can deviate from the
3792 * true value by nr_online_cpus * threshold. To avoid the zone
3793 * watermarks being breached while under pressure, we reduce the
3794 * per-cpu vmstat threshold while kswapd is awake and restore
3795 * them before going back to sleep.
3796 */
3797 set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
3798
3799 if (!kthread_should_stop())
3800 schedule();
3801
3802 set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
3803 } else {
3804 if (remaining)
3805 count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
3806 else
3807 count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
3808 }
3809 finish_wait(&pgdat->kswapd_wait, &wait);
3810}
3811
3812/*
3813 * The background pageout daemon, started as a kernel thread
3814 * from the init process.
3815 *
3816 * This basically trickles out pages so that we have _some_
3817 * free memory available even if there is no other activity
3818 * that frees anything up. This is needed for things like routing
3819 * etc, where we otherwise might have all activity going on in
3820 * asynchronous contexts that cannot page things out.
3821 *
3822 * If there are applications that are active memory-allocators
3823 * (most normal use), this basically shouldn't matter.
3824 */
3825static int kswapd(void *p)
3826{
3827 unsigned int alloc_order, reclaim_order;
3828 unsigned int classzone_idx = MAX_NR_ZONES - 1;
3829 pg_data_t *pgdat = (pg_data_t*)p;
3830 struct task_struct *tsk = current;
3831
3832 struct reclaim_state reclaim_state = {
3833 .reclaimed_slab = 0,
3834 };
3835 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
3836
3837 if (!cpumask_empty(cpumask))
3838 set_cpus_allowed_ptr(tsk, cpumask);
3839 current->reclaim_state = &reclaim_state;
3840
3841 /*
3842 * Tell the memory management that we're a "memory allocator",
3843 * and that if we need more memory we should get access to it
3844 * regardless (see "__alloc_pages()"). "kswapd" should
3845 * never get caught in the normal page freeing logic.
3846 *
3847 * (Kswapd normally doesn't need memory anyway, but sometimes
3848 * you need a small amount of memory in order to be able to
3849 * page out something else, and this flag essentially protects
3850 * us from recursively trying to free more memory as we're
3851 * trying to free the first piece of memory in the first place).
3852 */
3853 tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
3854 set_freezable();
3855
3856 pgdat->kswapd_order = 0;
3857 pgdat->kswapd_classzone_idx = MAX_NR_ZONES;
3858 for ( ; ; ) {
3859 bool ret;
3860
3861 alloc_order = reclaim_order = pgdat->kswapd_order;
3862 classzone_idx = kswapd_classzone_idx(pgdat, classzone_idx);
3863
3864kswapd_try_sleep:
3865 kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,
3866 classzone_idx);
3867
3868 /* Read the new order and classzone_idx */
3869 alloc_order = reclaim_order = pgdat->kswapd_order;
3870 classzone_idx = kswapd_classzone_idx(pgdat, 0);
3871 pgdat->kswapd_order = 0;
3872 pgdat->kswapd_classzone_idx = MAX_NR_ZONES;
3873
3874 ret = try_to_freeze();
3875 if (kthread_should_stop())
3876 break;
3877
3878 /*
3879 * We can speed up thawing tasks if we don't call balance_pgdat
3880 * after returning from the refrigerator
3881 */
3882 if (ret)
3883 continue;
3884
3885 /*
3886 * Reclaim begins at the requested order but if a high-order
3887 * reclaim fails then kswapd falls back to reclaiming for
3888 * order-0. If that happens, kswapd will consider sleeping
3889 * for the order it finished reclaiming at (reclaim_order)
3890 * but kcompactd is woken to compact for the original
3891 * request (alloc_order).
3892 */
3893 trace_mm_vmscan_kswapd_wake(pgdat->node_id, classzone_idx,
3894 alloc_order);
3895 reclaim_order = balance_pgdat(pgdat, alloc_order, classzone_idx);
3896 if (reclaim_order < alloc_order)
3897 goto kswapd_try_sleep;
3898 }
3899
3900 tsk->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD);
3901 current->reclaim_state = NULL;
3902
3903 return 0;
3904}
3905
3906/*
3907 * A zone is low on free memory or too fragmented for high-order memory. If
3908 * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's
3909 * pgdat. It will wake up kcompactd after reclaiming memory. If kswapd reclaim
3910 * has failed or is not needed, still wake up kcompactd if only compaction is
3911 * needed.
3912 */
3913void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
3914 enum zone_type classzone_idx)
3915{
3916 pg_data_t *pgdat;
3917
3918 if (!managed_zone(zone))
3919 return;
3920
3921 if (!cpuset_zone_allowed(zone, gfp_flags))
3922 return;
3923 pgdat = zone->zone_pgdat;
3924 pgdat->kswapd_classzone_idx = kswapd_classzone_idx(pgdat,
3925 classzone_idx);
3926 pgdat->kswapd_order = max(pgdat->kswapd_order, order);
3927 if (!waitqueue_active(&pgdat->kswapd_wait))
3928 return;
3929
3930 /* Hopeless node, leave it to direct reclaim if possible */
3931 if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ||
3932 (pgdat_balanced(pgdat, order, classzone_idx) &&
3933 !pgdat_watermark_boosted(pgdat, classzone_idx))) {
3934 /*
3935 * There may be plenty of free memory available, but it's too
3936 * fragmented for high-order allocations. Wake up kcompactd
3937 * and rely on compaction_suitable() to determine if it's
3938 * needed. If it fails, it will defer subsequent attempts to
3939 * ratelimit its work.
3940 */
3941 if (!(gfp_flags & __GFP_DIRECT_RECLAIM))
3942 wakeup_kcompactd(pgdat, order, classzone_idx);
3943 return;
3944 }
3945
3946 trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, classzone_idx, order,
3947 gfp_flags);
3948 wake_up_interruptible(&pgdat->kswapd_wait);
3949}
3950
3951#ifdef CONFIG_HIBERNATION
3952/*
3953 * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
3954 * freed pages.
3955 *
3956 * Rather than trying to age LRUs the aim is to preserve the overall
3957 * LRU order by reclaiming preferentially
3958 * inactive > active > active referenced > active mapped
3959 */
3960unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
3961{
3962 struct reclaim_state reclaim_state;
3963 struct scan_control sc = {
3964 .nr_to_reclaim = nr_to_reclaim,
3965 .gfp_mask = GFP_HIGHUSER_MOVABLE,
3966 .reclaim_idx = MAX_NR_ZONES - 1,
3967 .priority = DEF_PRIORITY,
3968 .may_writepage = 1,
3969 .may_unmap = 1,
3970 .may_swap = 1,
3971 .hibernation_mode = 1,
3972 };
3973 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3974 struct task_struct *p = current;
3975 unsigned long nr_reclaimed;
3976 unsigned int noreclaim_flag;
3977
3978 fs_reclaim_acquire(sc.gfp_mask);
3979 noreclaim_flag = memalloc_noreclaim_save();
3980 reclaim_state.reclaimed_slab = 0;
3981 p->reclaim_state = &reclaim_state;
3982
3983 nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3984
3985 p->reclaim_state = NULL;
3986 memalloc_noreclaim_restore(noreclaim_flag);
3987 fs_reclaim_release(sc.gfp_mask);
3988
3989 return nr_reclaimed;
3990}
3991#endif /* CONFIG_HIBERNATION */
3992
3993/* It's optimal to keep kswapds on the same CPUs as their memory, but
3994 not required for correctness. So if the last cpu in a node goes
3995 away, we get changed to run anywhere: as the first one comes back,
3996 restore their cpu bindings. */
3997static int kswapd_cpu_online(unsigned int cpu)
3998{
3999 int nid;
4000
4001 for_each_node_state(nid, N_MEMORY) {
4002 pg_data_t *pgdat = NODE_DATA(nid);
4003 const struct cpumask *mask;
4004
4005 mask = cpumask_of_node(pgdat->node_id);
4006
4007 if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
4008 /* One of our CPUs online: restore mask */
4009 set_cpus_allowed_ptr(pgdat->kswapd, mask);
4010 }
4011 return 0;
4012}
4013
4014/*
4015 * This kswapd start function will be called by init and node-hot-add.
4016 * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
4017 */
4018int kswapd_run(int nid)
4019{
4020 pg_data_t *pgdat = NODE_DATA(nid);
4021 int ret = 0;
4022
4023 if (pgdat->kswapd)
4024 return 0;
4025
4026 pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
4027 if (IS_ERR(pgdat->kswapd)) {
4028 /* failure at boot is fatal */
4029 BUG_ON(system_state < SYSTEM_RUNNING);
4030 pr_err("Failed to start kswapd on node %d\n", nid);
4031 ret = PTR_ERR(pgdat->kswapd);
4032 pgdat->kswapd = NULL;
4033 }
4034 return ret;
4035}
4036
4037/*
4038 * Called by memory hotplug when all memory in a node is offlined. Caller must
4039 * hold mem_hotplug_begin/end().
4040 */
4041void kswapd_stop(int nid)
4042{
4043 struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
4044
4045 if (kswapd) {
4046 kthread_stop(kswapd);
4047 NODE_DATA(nid)->kswapd = NULL;
4048 }
4049}
4050
4051static int __init kswapd_init(void)
4052{
4053 int nid, ret;
4054
4055 swap_setup();
4056 for_each_node_state(nid, N_MEMORY)
4057 kswapd_run(nid);
4058 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
4059 "mm/vmscan:online", kswapd_cpu_online,
4060 NULL);
4061 WARN_ON(ret < 0);
4062 return 0;
4063}
4064
4065module_init(kswapd_init)
4066
4067#ifdef CONFIG_NUMA
4068/*
4069 * Node reclaim mode
4070 *
4071 * If non-zero call node_reclaim when the number of free pages falls below
4072 * the watermarks.
4073 */
4074int node_reclaim_mode __read_mostly;
4075
4076#define RECLAIM_OFF 0
4077#define RECLAIM_ZONE (1<<0) /* Run shrink_inactive_list on the zone */
4078#define RECLAIM_WRITE (1<<1) /* Writeout pages during reclaim */
4079#define RECLAIM_UNMAP (1<<2) /* Unmap pages during reclaim */
4080
4081/*
4082 * Priority for NODE_RECLAIM. This determines the fraction of pages
4083 * of a node considered for each zone_reclaim. 4 scans 1/16th of
4084 * a zone.
4085 */
4086#define NODE_RECLAIM_PRIORITY 4
4087
4088/*
4089 * Percentage of pages in a zone that must be unmapped for node_reclaim to
4090 * occur.
4091 */
4092int sysctl_min_unmapped_ratio = 1;
4093
4094/*
4095 * If the number of slab pages in a zone grows beyond this percentage then
4096 * slab reclaim needs to occur.
4097 */
4098int sysctl_min_slab_ratio = 5;
4099
4100static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)
4101{
4102 unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);
4103 unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +
4104 node_page_state(pgdat, NR_ACTIVE_FILE);
4105
4106 /*
4107 * It's possible for there to be more file mapped pages than
4108 * accounted for by the pages on the file LRU lists because
4109 * tmpfs pages accounted for as ANON can also be FILE_MAPPED
4110 */
4111 return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
4112}
4113
4114/* Work out how many page cache pages we can reclaim in this reclaim_mode */
4115static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)
4116{
4117 unsigned long nr_pagecache_reclaimable;
4118 unsigned long delta = 0;
4119
4120 /*
4121 * If RECLAIM_UNMAP is set, then all file pages are considered
4122 * potentially reclaimable. Otherwise, we have to worry about
4123 * pages like swapcache and node_unmapped_file_pages() provides
4124 * a better estimate
4125 */
4126 if (node_reclaim_mode & RECLAIM_UNMAP)
4127 nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);
4128 else
4129 nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);
4130
4131 /* If we can't clean pages, remove dirty pages from consideration */
4132 if (!(node_reclaim_mode & RECLAIM_WRITE))
4133 delta += node_page_state(pgdat, NR_FILE_DIRTY);
4134
4135 /* Watch for any possible underflows due to delta */
4136 if (unlikely(delta > nr_pagecache_reclaimable))
4137 delta = nr_pagecache_reclaimable;
4138
4139 return nr_pagecache_reclaimable - delta;
4140}
4141
4142/*
4143 * Try to free up some pages from this node through reclaim.
4144 */
4145static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
4146{
4147 /* Minimum pages needed in order to stay on node */
4148 const unsigned long nr_pages = 1 << order;
4149 struct task_struct *p = current;
4150 struct reclaim_state reclaim_state;
4151 unsigned int noreclaim_flag;
4152 struct scan_control sc = {
4153 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
4154 .gfp_mask = current_gfp_context(gfp_mask),
4155 .order = order,
4156 .priority = NODE_RECLAIM_PRIORITY,
4157 .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE),
4158 .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP),
4159 .may_swap = 1,
4160 .reclaim_idx = gfp_zone(gfp_mask),
4161 };
4162
4163 cond_resched();
4164 fs_reclaim_acquire(sc.gfp_mask);
4165 /*
4166 * We need to be able to allocate from the reserves for RECLAIM_UNMAP
4167 * and we also need to be able to write out pages for RECLAIM_WRITE
4168 * and RECLAIM_UNMAP.
4169 */
4170 noreclaim_flag = memalloc_noreclaim_save();
4171 p->flags |= PF_SWAPWRITE;
4172 reclaim_state.reclaimed_slab = 0;
4173 p->reclaim_state = &reclaim_state;
4174
4175 if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) {
4176 /*
4177 * Free memory by calling shrink node with increasing
4178 * priorities until we have enough memory freed.
4179 */
4180 do {
4181 shrink_node(pgdat, &sc);
4182 } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
4183 }
4184
4185 p->reclaim_state = NULL;
4186 current->flags &= ~PF_SWAPWRITE;
4187 memalloc_noreclaim_restore(noreclaim_flag);
4188 fs_reclaim_release(sc.gfp_mask);
4189 return sc.nr_reclaimed >= nr_pages;
4190}
4191
4192int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
4193{
4194 int ret;
4195
4196 /*
4197 * Node reclaim reclaims unmapped file backed pages and
4198 * slab pages if we are over the defined limits.
4199 *
4200 * A small portion of unmapped file backed pages is needed for
4201 * file I/O otherwise pages read by file I/O will be immediately
4202 * thrown out if the node is overallocated. So we do not reclaim
4203 * if less than a specified percentage of the node is used by
4204 * unmapped file backed pages.
4205 */
4206 if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
4207 node_page_state(pgdat, NR_SLAB_RECLAIMABLE) <= pgdat->min_slab_pages)
4208 return NODE_RECLAIM_FULL;
4209
4210 /*
4211 * Do not scan if the allocation should not be delayed.
4212 */
4213 if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC))
4214 return NODE_RECLAIM_NOSCAN;
4215
4216 /*
4217 * Only run node reclaim on the local node or on nodes that do not
4218 * have associated processors. This will favor the local processor
4219 * over remote processors and spread off node memory allocations
4220 * as wide as possible.
4221 */
4222 if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())
4223 return NODE_RECLAIM_NOSCAN;
4224
4225 if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags))
4226 return NODE_RECLAIM_NOSCAN;
4227
4228 ret = __node_reclaim(pgdat, gfp_mask, order);
4229 clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags);
4230
4231 if (!ret)
4232 count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
4233
4234 return ret;
4235}
4236#endif
4237
4238/*
4239 * page_evictable - test whether a page is evictable
4240 * @page: the page to test
4241 *
4242 * Test whether page is evictable--i.e., should be placed on active/inactive
4243 * lists vs unevictable list.
4244 *
4245 * Reasons page might not be evictable:
4246 * (1) page's mapping marked unevictable
4247 * (2) page is part of an mlocked VMA
4248 *
4249 */
4250int page_evictable(struct page *page)
4251{
4252 int ret;
4253
4254 /* Prevent address_space of inode and swap cache from being freed */
4255 rcu_read_lock();
4256 ret = !mapping_unevictable(page_mapping(page)) && !PageMlocked(page);
4257 rcu_read_unlock();
4258 return ret;
4259}
4260
4261/**
4262 * check_move_unevictable_pages - check pages for evictability and move to
4263 * appropriate zone lru list
4264 * @pvec: pagevec with lru pages to check
4265 *
4266 * Checks pages for evictability, if an evictable page is in the unevictable
4267 * lru list, moves it to the appropriate evictable lru list. This function
4268 * should be only used for lru pages.
4269 */
4270void check_move_unevictable_pages(struct pagevec *pvec)
4271{
4272 struct lruvec *lruvec;
4273 struct pglist_data *pgdat = NULL;
4274 int pgscanned = 0;
4275 int pgrescued = 0;
4276 int i;
4277
4278 for (i = 0; i < pvec->nr; i++) {
4279 struct page *page = pvec->pages[i];
4280 struct pglist_data *pagepgdat = page_pgdat(page);
4281
4282 pgscanned++;
4283 if (pagepgdat != pgdat) {
4284 if (pgdat)
4285 spin_unlock_irq(&pgdat->lru_lock);
4286 pgdat = pagepgdat;
4287 spin_lock_irq(&pgdat->lru_lock);
4288 }
4289 lruvec = mem_cgroup_page_lruvec(page, pgdat);
4290
4291 if (!PageLRU(page) || !PageUnevictable(page))
4292 continue;
4293
4294 if (page_evictable(page)) {
4295 enum lru_list lru = page_lru_base_type(page);
4296
4297 VM_BUG_ON_PAGE(PageActive(page), page);
4298 ClearPageUnevictable(page);
4299 del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE);
4300 add_page_to_lru_list(page, lruvec, lru);
4301 pgrescued++;
4302 }
4303 }
4304
4305 if (pgdat) {
4306 __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
4307 __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
4308 spin_unlock_irq(&pgdat->lru_lock);
4309 }
4310}
4311EXPORT_SYMBOL_GPL(check_move_unevictable_pages);
4312