1/* memcontrol.c - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
6 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
9 * Memory thresholds
10 * Copyright (C) 2009 Nokia Corporation
11 * Author: Kirill A. Shutemov
12 *
13 * Kernel Memory Controller
14 * Copyright (C) 2012 Parallels Inc. and Google Inc.
15 * Authors: Glauber Costa and Suleiman Souhlal
16 *
17 * Native page reclaim
18 * Charge lifetime sanitation
19 * Lockless page tracking & accounting
20 * Unified hierarchy configuration model
21 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
22 *
23 * This program is free software; you can redistribute it and/or modify
24 * it under the terms of the GNU General Public License as published by
25 * the Free Software Foundation; either version 2 of the License, or
26 * (at your option) any later version.
27 *
28 * This program is distributed in the hope that it will be useful,
29 * but WITHOUT ANY WARRANTY; without even the implied warranty of
30 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31 * GNU General Public License for more details.
32 */
33
34#include <linux/page_counter.h>
35#include <linux/memcontrol.h>
36#include <linux/cgroup.h>
37#include <linux/mm.h>
38#include <linux/sched/mm.h>
39#include <linux/shmem_fs.h>
40#include <linux/hugetlb.h>
41#include <linux/pagemap.h>
42#include <linux/vm_event_item.h>
43#include <linux/smp.h>
44#include <linux/page-flags.h>
45#include <linux/backing-dev.h>
46#include <linux/bit_spinlock.h>
47#include <linux/rcupdate.h>
48#include <linux/limits.h>
49#include <linux/export.h>
50#include <linux/mutex.h>
51#include <linux/rbtree.h>
52#include <linux/slab.h>
53#include <linux/swap.h>
54#include <linux/swapops.h>
55#include <linux/spinlock.h>
56#include <linux/eventfd.h>
57#include <linux/poll.h>
58#include <linux/sort.h>
59#include <linux/fs.h>
60#include <linux/seq_file.h>
61#include <linux/vmpressure.h>
62#include <linux/mm_inline.h>
63#include <linux/swap_cgroup.h>
64#include <linux/cpu.h>
65#include <linux/oom.h>
66#include <linux/lockdep.h>
67#include <linux/file.h>
68#include <linux/tracehook.h>
69#include "internal.h"
70#include <net/sock.h>
71#include <net/ip.h>
72#include "slab.h"
73
74#include <linux/uaccess.h>
75
76#include <trace/events/vmscan.h>
77
78struct cgroup_subsys memory_cgrp_subsys __read_mostly;
79EXPORT_SYMBOL(memory_cgrp_subsys);
80
81struct mem_cgroup *root_mem_cgroup __read_mostly;
82
83#define MEM_CGROUP_RECLAIM_RETRIES 5
84
85/* Socket memory accounting disabled? */
86static bool cgroup_memory_nosocket;
87
88/* Kernel memory accounting disabled? */
89static bool cgroup_memory_nokmem;
90
91/* Whether the swap controller is active */
92#ifdef CONFIG_MEMCG_SWAP
93int do_swap_account __read_mostly;
94#else
95#define do_swap_account 0
96#endif
97
98/* Whether legacy memory+swap accounting is active */
99static bool do_memsw_account(void)
100{
101 return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
102}
103
104static const char *const mem_cgroup_lru_names[] = {
105 "inactive_anon",
106 "active_anon",
107 "inactive_file",
108 "active_file",
109 "unevictable",
110};
111
112#define THRESHOLDS_EVENTS_TARGET 128
113#define SOFTLIMIT_EVENTS_TARGET 1024
114#define NUMAINFO_EVENTS_TARGET 1024
115
116/*
117 * Cgroups above their limits are maintained in a RB-Tree, independent of
118 * their hierarchy representation
119 */
120
121struct mem_cgroup_tree_per_node {
122 struct rb_root rb_root;
123 struct rb_node *rb_rightmost;
124 spinlock_t lock;
125};
126
127struct mem_cgroup_tree {
128 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
129};
130
131static struct mem_cgroup_tree soft_limit_tree __read_mostly;
132
133/* for OOM */
134struct mem_cgroup_eventfd_list {
135 struct list_head list;
136 struct eventfd_ctx *eventfd;
137};
138
139/*
140 * cgroup_event represents events which userspace want to receive.
141 */
142struct mem_cgroup_event {
143 /*
144 * memcg which the event belongs to.
145 */
146 struct mem_cgroup *memcg;
147 /*
148 * eventfd to signal userspace about the event.
149 */
150 struct eventfd_ctx *eventfd;
151 /*
152 * Each of these stored in a list by the cgroup.
153 */
154 struct list_head list;
155 /*
156 * register_event() callback will be used to add new userspace
157 * waiter for changes related to this event. Use eventfd_signal()
158 * on eventfd to send notification to userspace.
159 */
160 int (*register_event)(struct mem_cgroup *memcg,
161 struct eventfd_ctx *eventfd, const char *args);
162 /*
163 * unregister_event() callback will be called when userspace closes
164 * the eventfd or on cgroup removing. This callback must be set,
165 * if you want provide notification functionality.
166 */
167 void (*unregister_event)(struct mem_cgroup *memcg,
168 struct eventfd_ctx *eventfd);
169 /*
170 * All fields below needed to unregister event when
171 * userspace closes eventfd.
172 */
173 poll_table pt;
174 wait_queue_head_t *wqh;
175 wait_queue_entry_t wait;
176 struct work_struct remove;
177};
178
179static void mem_cgroup_threshold(struct mem_cgroup *memcg);
180static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
181
182/* Stuffs for move charges at task migration. */
183/*
184 * Types of charges to be moved.
185 */
186#define MOVE_ANON 0x1U
187#define MOVE_FILE 0x2U
188#define MOVE_MASK (MOVE_ANON | MOVE_FILE)
189
190/* "mc" and its members are protected by cgroup_mutex */
191static struct move_charge_struct {
192 spinlock_t lock; /* for from, to */
193 struct mm_struct *mm;
194 struct mem_cgroup *from;
195 struct mem_cgroup *to;
196 unsigned long flags;
197 unsigned long precharge;
198 unsigned long moved_charge;
199 unsigned long moved_swap;
200 struct task_struct *moving_task; /* a task moving charges */
201 wait_queue_head_t waitq; /* a waitq for other context */
202} mc = {
203 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
204 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
205};
206
207/*
208 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
209 * limit reclaim to prevent infinite loops, if they ever occur.
210 */
211#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
212#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
213
214enum charge_type {
215 MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
216 MEM_CGROUP_CHARGE_TYPE_ANON,
217 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
218 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
219 NR_CHARGE_TYPE,
220};
221
222/* for encoding cft->private value on file */
223enum res_type {
224 _MEM,
225 _MEMSWAP,
226 _OOM_TYPE,
227 _KMEM,
228 _TCP,
229};
230
231#define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
232#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
233#define MEMFILE_ATTR(val) ((val) & 0xffff)
234/* Used for OOM nofiier */
235#define OOM_CONTROL (0)
236
237/*
238 * Iteration constructs for visiting all cgroups (under a tree). If
239 * loops are exited prematurely (break), mem_cgroup_iter_break() must
240 * be used for reference counting.
241 */
242#define for_each_mem_cgroup_tree(iter, root) \
243 for (iter = mem_cgroup_iter(root, NULL, NULL); \
244 iter != NULL; \
245 iter = mem_cgroup_iter(root, iter, NULL))
246
247#define for_each_mem_cgroup(iter) \
248 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
249 iter != NULL; \
250 iter = mem_cgroup_iter(NULL, iter, NULL))
251
252static inline bool should_force_charge(void)
253{
254 return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
255 (current->flags & PF_EXITING);
256}
257
258/* Some nice accessors for the vmpressure. */
259struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
260{
261 if (!memcg)
262 memcg = root_mem_cgroup;
263 return &memcg->vmpressure;
264}
265
266struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
267{
268 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
269}
270
271#ifdef CONFIG_MEMCG_KMEM
272/*
273 * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
274 * The main reason for not using cgroup id for this:
275 * this works better in sparse environments, where we have a lot of memcgs,
276 * but only a few kmem-limited. Or also, if we have, for instance, 200
277 * memcgs, and none but the 200th is kmem-limited, we'd have to have a
278 * 200 entry array for that.
279 *
280 * The current size of the caches array is stored in memcg_nr_cache_ids. It
281 * will double each time we have to increase it.
282 */
283static DEFINE_IDA(memcg_cache_ida);
284int memcg_nr_cache_ids;
285
286/* Protects memcg_nr_cache_ids */
287static DECLARE_RWSEM(memcg_cache_ids_sem);
288
289void memcg_get_cache_ids(void)
290{
291 down_read(&memcg_cache_ids_sem);
292}
293
294void memcg_put_cache_ids(void)
295{
296 up_read(&memcg_cache_ids_sem);
297}
298
299/*
300 * MIN_SIZE is different than 1, because we would like to avoid going through
301 * the alloc/free process all the time. In a small machine, 4 kmem-limited
302 * cgroups is a reasonable guess. In the future, it could be a parameter or
303 * tunable, but that is strictly not necessary.
304 *
305 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
306 * this constant directly from cgroup, but it is understandable that this is
307 * better kept as an internal representation in cgroup.c. In any case, the
308 * cgrp_id space is not getting any smaller, and we don't have to necessarily
309 * increase ours as well if it increases.
310 */
311#define MEMCG_CACHES_MIN_SIZE 4
312#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
313
314/*
315 * A lot of the calls to the cache allocation functions are expected to be
316 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
317 * conditional to this static branch, we'll have to allow modules that does
318 * kmem_cache_alloc and the such to see this symbol as well
319 */
320DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
321EXPORT_SYMBOL(memcg_kmem_enabled_key);
322
323struct workqueue_struct *memcg_kmem_cache_wq;
324
325static int memcg_shrinker_map_size;
326static DEFINE_MUTEX(memcg_shrinker_map_mutex);
327
328static void memcg_free_shrinker_map_rcu(struct rcu_head *head)
329{
330 kvfree(container_of(head, struct memcg_shrinker_map, rcu));
331}
332
333static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
334 int size, int old_size)
335{
336 struct memcg_shrinker_map *new, *old;
337 int nid;
338
339 lockdep_assert_held(&memcg_shrinker_map_mutex);
340
341 for_each_node(nid) {
342 old = rcu_dereference_protected(
343 mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true);
344 /* Not yet online memcg */
345 if (!old)
346 return 0;
347
348 new = kvmalloc(sizeof(*new) + size, GFP_KERNEL);
349 if (!new)
350 return -ENOMEM;
351
352 /* Set all old bits, clear all new bits */
353 memset(new->map, (int)0xff, old_size);
354 memset((void *)new->map + old_size, 0, size - old_size);
355
356 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new);
357 call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);
358 }
359
360 return 0;
361}
362
363static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
364{
365 struct mem_cgroup_per_node *pn;
366 struct memcg_shrinker_map *map;
367 int nid;
368
369 if (mem_cgroup_is_root(memcg))
370 return;
371
372 for_each_node(nid) {
373 pn = mem_cgroup_nodeinfo(memcg, nid);
374 map = rcu_dereference_protected(pn->shrinker_map, true);
375 if (map)
376 kvfree(map);
377 rcu_assign_pointer(pn->shrinker_map, NULL);
378 }
379}
380
381static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
382{
383 struct memcg_shrinker_map *map;
384 int nid, size, ret = 0;
385
386 if (mem_cgroup_is_root(memcg))
387 return 0;
388
389 mutex_lock(&memcg_shrinker_map_mutex);
390 size = memcg_shrinker_map_size;
391 for_each_node(nid) {
392 map = kvzalloc(sizeof(*map) + size, GFP_KERNEL);
393 if (!map) {
394 memcg_free_shrinker_maps(memcg);
395 ret = -ENOMEM;
396 break;
397 }
398 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);
399 }
400 mutex_unlock(&memcg_shrinker_map_mutex);
401
402 return ret;
403}
404
405int memcg_expand_shrinker_maps(int new_id)
406{
407 int size, old_size, ret = 0;
408 struct mem_cgroup *memcg;
409
410 size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);
411 old_size = memcg_shrinker_map_size;
412 if (size <= old_size)
413 return 0;
414
415 mutex_lock(&memcg_shrinker_map_mutex);
416 if (!root_mem_cgroup)
417 goto unlock;
418
419 for_each_mem_cgroup(memcg) {
420 if (mem_cgroup_is_root(memcg))
421 continue;
422 ret = memcg_expand_one_shrinker_map(memcg, size, old_size);
423 if (ret)
424 goto unlock;
425 }
426unlock:
427 if (!ret)
428 memcg_shrinker_map_size = size;
429 mutex_unlock(&memcg_shrinker_map_mutex);
430 return ret;
431}
432
433void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
434{
435 if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
436 struct memcg_shrinker_map *map;
437
438 rcu_read_lock();
439 map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
440 /* Pairs with smp mb in shrink_slab() */
441 smp_mb__before_atomic();
442 set_bit(shrinker_id, map->map);
443 rcu_read_unlock();
444 }
445}
446
447#else /* CONFIG_MEMCG_KMEM */
448static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
449{
450 return 0;
451}
452static void memcg_free_shrinker_maps(struct mem_cgroup *memcg) { }
453#endif /* CONFIG_MEMCG_KMEM */
454
455/**
456 * mem_cgroup_css_from_page - css of the memcg associated with a page
457 * @page: page of interest
458 *
459 * If memcg is bound to the default hierarchy, css of the memcg associated
460 * with @page is returned. The returned css remains associated with @page
461 * until it is released.
462 *
463 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
464 * is returned.
465 */
466struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
467{
468 struct mem_cgroup *memcg;
469
470 memcg = page->mem_cgroup;
471
472 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
473 memcg = root_mem_cgroup;
474
475 return &memcg->css;
476}
477
478/**
479 * page_cgroup_ino - return inode number of the memcg a page is charged to
480 * @page: the page
481 *
482 * Look up the closest online ancestor of the memory cgroup @page is charged to
483 * and return its inode number or 0 if @page is not charged to any cgroup. It
484 * is safe to call this function without holding a reference to @page.
485 *
486 * Note, this function is inherently racy, because there is nothing to prevent
487 * the cgroup inode from getting torn down and potentially reallocated a moment
488 * after page_cgroup_ino() returns, so it only should be used by callers that
489 * do not care (such as procfs interfaces).
490 */
491ino_t page_cgroup_ino(struct page *page)
492{
493 struct mem_cgroup *memcg;
494 unsigned long ino = 0;
495
496 rcu_read_lock();
497 memcg = READ_ONCE(page->mem_cgroup);
498 while (memcg && !(memcg->css.flags & CSS_ONLINE))
499 memcg = parent_mem_cgroup(memcg);
500 if (memcg)
501 ino = cgroup_ino(memcg->css.cgroup);
502 rcu_read_unlock();
503 return ino;
504}
505
506static struct mem_cgroup_per_node *
507mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
508{
509 int nid = page_to_nid(page);
510
511 return memcg->nodeinfo[nid];
512}
513
514static struct mem_cgroup_tree_per_node *
515soft_limit_tree_node(int nid)
516{
517 return soft_limit_tree.rb_tree_per_node[nid];
518}
519
520static struct mem_cgroup_tree_per_node *
521soft_limit_tree_from_page(struct page *page)
522{
523 int nid = page_to_nid(page);
524
525 return soft_limit_tree.rb_tree_per_node[nid];
526}
527
528static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
529 struct mem_cgroup_tree_per_node *mctz,
530 unsigned long new_usage_in_excess)
531{
532 struct rb_node **p = &mctz->rb_root.rb_node;
533 struct rb_node *parent = NULL;
534 struct mem_cgroup_per_node *mz_node;
535 bool rightmost = true;
536
537 if (mz->on_tree)
538 return;
539
540 mz->usage_in_excess = new_usage_in_excess;
541 if (!mz->usage_in_excess)
542 return;
543 while (*p) {
544 parent = *p;
545 mz_node = rb_entry(parent, struct mem_cgroup_per_node,
546 tree_node);
547 if (mz->usage_in_excess < mz_node->usage_in_excess) {
548 p = &(*p)->rb_left;
549 rightmost = false;
550 }
551
552 /*
553 * We can't avoid mem cgroups that are over their soft
554 * limit by the same amount
555 */
556 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
557 p = &(*p)->rb_right;
558 }
559
560 if (rightmost)
561 mctz->rb_rightmost = &mz->tree_node;
562
563 rb_link_node(&mz->tree_node, parent, p);
564 rb_insert_color(&mz->tree_node, &mctz->rb_root);
565 mz->on_tree = true;
566}
567
568static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
569 struct mem_cgroup_tree_per_node *mctz)
570{
571 if (!mz->on_tree)
572 return;
573
574 if (&mz->tree_node == mctz->rb_rightmost)
575 mctz->rb_rightmost = rb_prev(&mz->tree_node);
576
577 rb_erase(&mz->tree_node, &mctz->rb_root);
578 mz->on_tree = false;
579}
580
581static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
582 struct mem_cgroup_tree_per_node *mctz)
583{
584 unsigned long flags;
585
586 spin_lock_irqsave(&mctz->lock, flags);
587 __mem_cgroup_remove_exceeded(mz, mctz);
588 spin_unlock_irqrestore(&mctz->lock, flags);
589}
590
591static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
592{
593 unsigned long nr_pages = page_counter_read(&memcg->memory);
594 unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
595 unsigned long excess = 0;
596
597 if (nr_pages > soft_limit)
598 excess = nr_pages - soft_limit;
599
600 return excess;
601}
602
603static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
604{
605 unsigned long excess;
606 struct mem_cgroup_per_node *mz;
607 struct mem_cgroup_tree_per_node *mctz;
608
609 mctz = soft_limit_tree_from_page(page);
610 if (!mctz)
611 return;
612 /*
613 * Necessary to update all ancestors when hierarchy is used.
614 * because their event counter is not touched.
615 */
616 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
617 mz = mem_cgroup_page_nodeinfo(memcg, page);
618 excess = soft_limit_excess(memcg);
619 /*
620 * We have to update the tree if mz is on RB-tree or
621 * mem is over its softlimit.
622 */
623 if (excess || mz->on_tree) {
624 unsigned long flags;
625
626 spin_lock_irqsave(&mctz->lock, flags);
627 /* if on-tree, remove it */
628 if (mz->on_tree)
629 __mem_cgroup_remove_exceeded(mz, mctz);
630 /*
631 * Insert again. mz->usage_in_excess will be updated.
632 * If excess is 0, no tree ops.
633 */
634 __mem_cgroup_insert_exceeded(mz, mctz, excess);
635 spin_unlock_irqrestore(&mctz->lock, flags);
636 }
637 }
638}
639
640static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
641{
642 struct mem_cgroup_tree_per_node *mctz;
643 struct mem_cgroup_per_node *mz;
644 int nid;
645
646 for_each_node(nid) {
647 mz = mem_cgroup_nodeinfo(memcg, nid);
648 mctz = soft_limit_tree_node(nid);
649 if (mctz)
650 mem_cgroup_remove_exceeded(mz, mctz);
651 }
652}
653
654static struct mem_cgroup_per_node *
655__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
656{
657 struct mem_cgroup_per_node *mz;
658
659retry:
660 mz = NULL;
661 if (!mctz->rb_rightmost)
662 goto done; /* Nothing to reclaim from */
663
664 mz = rb_entry(mctz->rb_rightmost,
665 struct mem_cgroup_per_node, tree_node);
666 /*
667 * Remove the node now but someone else can add it back,
668 * we will to add it back at the end of reclaim to its correct
669 * position in the tree.
670 */
671 __mem_cgroup_remove_exceeded(mz, mctz);
672 if (!soft_limit_excess(mz->memcg) ||
673 !css_tryget_online(&mz->memcg->css))
674 goto retry;
675done:
676 return mz;
677}
678
679static struct mem_cgroup_per_node *
680mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
681{
682 struct mem_cgroup_per_node *mz;
683
684 spin_lock_irq(&mctz->lock);
685 mz = __mem_cgroup_largest_soft_limit_node(mctz);
686 spin_unlock_irq(&mctz->lock);
687 return mz;
688}
689
690static unsigned long memcg_sum_events(struct mem_cgroup *memcg,
691 int event)
692{
693 return atomic_long_read(&memcg->events[event]);
694}
695
696static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
697 struct page *page,
698 bool compound, int nr_pages)
699{
700 /*
701 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
702 * counted as CACHE even if it's on ANON LRU.
703 */
704 if (PageAnon(page))
705 __mod_memcg_state(memcg, MEMCG_RSS, nr_pages);
706 else {
707 __mod_memcg_state(memcg, MEMCG_CACHE, nr_pages);
708 if (PageSwapBacked(page))
709 __mod_memcg_state(memcg, NR_SHMEM, nr_pages);
710 }
711
712 if (compound) {
713 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
714 __mod_memcg_state(memcg, MEMCG_RSS_HUGE, nr_pages);
715 }
716
717 /* pagein of a big page is an event. So, ignore page size */
718 if (nr_pages > 0)
719 __count_memcg_events(memcg, PGPGIN, 1);
720 else {
721 __count_memcg_events(memcg, PGPGOUT, 1);
722 nr_pages = -nr_pages; /* for event */
723 }
724
725 __this_cpu_add(memcg->stat_cpu->nr_page_events, nr_pages);
726}
727
728unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
729 int nid, unsigned int lru_mask)
730{
731 struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
732 unsigned long nr = 0;
733 enum lru_list lru;
734
735 VM_BUG_ON((unsigned)nid >= nr_node_ids);
736
737 for_each_lru(lru) {
738 if (!(BIT(lru) & lru_mask))
739 continue;
740 nr += mem_cgroup_get_lru_size(lruvec, lru);
741 }
742 return nr;
743}
744
745static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
746 unsigned int lru_mask)
747{
748 unsigned long nr = 0;
749 int nid;
750
751 for_each_node_state(nid, N_MEMORY)
752 nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
753 return nr;
754}
755
756static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
757 enum mem_cgroup_events_target target)
758{
759 unsigned long val, next;
760
761 val = __this_cpu_read(memcg->stat_cpu->nr_page_events);
762 next = __this_cpu_read(memcg->stat_cpu->targets[target]);
763 /* from time_after() in jiffies.h */
764 if ((long)(next - val) < 0) {
765 switch (target) {
766 case MEM_CGROUP_TARGET_THRESH:
767 next = val + THRESHOLDS_EVENTS_TARGET;
768 break;
769 case MEM_CGROUP_TARGET_SOFTLIMIT:
770 next = val + SOFTLIMIT_EVENTS_TARGET;
771 break;
772 case MEM_CGROUP_TARGET_NUMAINFO:
773 next = val + NUMAINFO_EVENTS_TARGET;
774 break;
775 default:
776 break;
777 }
778 __this_cpu_write(memcg->stat_cpu->targets[target], next);
779 return true;
780 }
781 return false;
782}
783
784/*
785 * Check events in order.
786 *
787 */
788static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
789{
790 /* threshold event is triggered in finer grain than soft limit */
791 if (unlikely(mem_cgroup_event_ratelimit(memcg,
792 MEM_CGROUP_TARGET_THRESH))) {
793 bool do_softlimit;
794 bool do_numainfo __maybe_unused;
795
796 do_softlimit = mem_cgroup_event_ratelimit(memcg,
797 MEM_CGROUP_TARGET_SOFTLIMIT);
798#if MAX_NUMNODES > 1
799 do_numainfo = mem_cgroup_event_ratelimit(memcg,
800 MEM_CGROUP_TARGET_NUMAINFO);
801#endif
802 mem_cgroup_threshold(memcg);
803 if (unlikely(do_softlimit))
804 mem_cgroup_update_tree(memcg, page);
805#if MAX_NUMNODES > 1
806 if (unlikely(do_numainfo))
807 atomic_inc(&memcg->numainfo_events);
808#endif
809 }
810}
811
812struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
813{
814 /*
815 * mm_update_next_owner() may clear mm->owner to NULL
816 * if it races with swapoff, page migration, etc.
817 * So this can be called with p == NULL.
818 */
819 if (unlikely(!p))
820 return NULL;
821
822 return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
823}
824EXPORT_SYMBOL(mem_cgroup_from_task);
825
826/**
827 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
828 * @mm: mm from which memcg should be extracted. It can be NULL.
829 *
830 * Obtain a reference on mm->memcg and returns it if successful. Otherwise
831 * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
832 * returned.
833 */
834struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
835{
836 struct mem_cgroup *memcg;
837
838 if (mem_cgroup_disabled())
839 return NULL;
840
841 rcu_read_lock();
842 do {
843 /*
844 * Page cache insertions can happen withou an
845 * actual mm context, e.g. during disk probing
846 * on boot, loopback IO, acct() writes etc.
847 */
848 if (unlikely(!mm))
849 memcg = root_mem_cgroup;
850 else {
851 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
852 if (unlikely(!memcg))
853 memcg = root_mem_cgroup;
854 }
855 } while (!css_tryget_online(&memcg->css));
856 rcu_read_unlock();
857 return memcg;
858}
859EXPORT_SYMBOL(get_mem_cgroup_from_mm);
860
861/**
862 * get_mem_cgroup_from_page: Obtain a reference on given page's memcg.
863 * @page: page from which memcg should be extracted.
864 *
865 * Obtain a reference on page->memcg and returns it if successful. Otherwise
866 * root_mem_cgroup is returned.
867 */
868struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
869{
870 struct mem_cgroup *memcg = page->mem_cgroup;
871
872 if (mem_cgroup_disabled())
873 return NULL;
874
875 rcu_read_lock();
876 if (!memcg || !css_tryget_online(&memcg->css))
877 memcg = root_mem_cgroup;
878 rcu_read_unlock();
879 return memcg;
880}
881EXPORT_SYMBOL(get_mem_cgroup_from_page);
882
883/**
884 * If current->active_memcg is non-NULL, do not fallback to current->mm->memcg.
885 */
886static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
887{
888 if (unlikely(current->active_memcg)) {
889 struct mem_cgroup *memcg = root_mem_cgroup;
890
891 rcu_read_lock();
892 if (css_tryget_online(&current->active_memcg->css))
893 memcg = current->active_memcg;
894 rcu_read_unlock();
895 return memcg;
896 }
897 return get_mem_cgroup_from_mm(current->mm);
898}
899
900/**
901 * mem_cgroup_iter - iterate over memory cgroup hierarchy
902 * @root: hierarchy root
903 * @prev: previously returned memcg, NULL on first invocation
904 * @reclaim: cookie for shared reclaim walks, NULL for full walks
905 *
906 * Returns references to children of the hierarchy below @root, or
907 * @root itself, or %NULL after a full round-trip.
908 *
909 * Caller must pass the return value in @prev on subsequent
910 * invocations for reference counting, or use mem_cgroup_iter_break()
911 * to cancel a hierarchy walk before the round-trip is complete.
912 *
913 * Reclaimers can specify a node and a priority level in @reclaim to
914 * divide up the memcgs in the hierarchy among all concurrent
915 * reclaimers operating on the same node and priority.
916 */
917struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
918 struct mem_cgroup *prev,
919 struct mem_cgroup_reclaim_cookie *reclaim)
920{
921 struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
922 struct cgroup_subsys_state *css = NULL;
923 struct mem_cgroup *memcg = NULL;
924 struct mem_cgroup *pos = NULL;
925
926 if (mem_cgroup_disabled())
927 return NULL;
928
929 if (!root)
930 root = root_mem_cgroup;
931
932 if (prev && !reclaim)
933 pos = prev;
934
935 if (!root->use_hierarchy && root != root_mem_cgroup) {
936 if (prev)
937 goto out;
938 return root;
939 }
940
941 rcu_read_lock();
942
943 if (reclaim) {
944 struct mem_cgroup_per_node *mz;
945
946 mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
947 iter = &mz->iter[reclaim->priority];
948
949 if (prev && reclaim->generation != iter->generation)
950 goto out_unlock;
951
952 while (1) {
953 pos = READ_ONCE(iter->position);
954 if (!pos || css_tryget(&pos->css))
955 break;
956 /*
957 * css reference reached zero, so iter->position will
958 * be cleared by ->css_released. However, we should not
959 * rely on this happening soon, because ->css_released
960 * is called from a work queue, and by busy-waiting we
961 * might block it. So we clear iter->position right
962 * away.
963 */
964 (void)cmpxchg(&iter->position, pos, NULL);
965 }
966 }
967
968 if (pos)
969 css = &pos->css;
970
971 for (;;) {
972 css = css_next_descendant_pre(css, &root->css);
973 if (!css) {
974 /*
975 * Reclaimers share the hierarchy walk, and a
976 * new one might jump in right at the end of
977 * the hierarchy - make sure they see at least
978 * one group and restart from the beginning.
979 */
980 if (!prev)
981 continue;
982 break;
983 }
984
985 /*
986 * Verify the css and acquire a reference. The root
987 * is provided by the caller, so we know it's alive
988 * and kicking, and don't take an extra reference.
989 */
990 memcg = mem_cgroup_from_css(css);
991
992 if (css == &root->css)
993 break;
994
995 if (css_tryget(css))
996 break;
997
998 memcg = NULL;
999 }
1000
1001 if (reclaim) {
1002 /*
1003 * The position could have already been updated by a competing
1004 * thread, so check that the value hasn't changed since we read
1005 * it to avoid reclaiming from the same cgroup twice.
1006 */
1007 (void)cmpxchg(&iter->position, pos, memcg);
1008
1009 if (pos)
1010 css_put(&pos->css);
1011
1012 if (!memcg)
1013 iter->generation++;
1014 else if (!prev)
1015 reclaim->generation = iter->generation;
1016 }
1017
1018out_unlock:
1019 rcu_read_unlock();
1020out:
1021 if (prev && prev != root)
1022 css_put(&prev->css);
1023
1024 return memcg;
1025}
1026
1027/**
1028 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1029 * @root: hierarchy root
1030 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1031 */
1032void mem_cgroup_iter_break(struct mem_cgroup *root,
1033 struct mem_cgroup *prev)
1034{
1035 if (!root)
1036 root = root_mem_cgroup;
1037 if (prev && prev != root)
1038 css_put(&prev->css);
1039}
1040
1041static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
1042{
1043 struct mem_cgroup *memcg = dead_memcg;
1044 struct mem_cgroup_reclaim_iter *iter;
1045 struct mem_cgroup_per_node *mz;
1046 int nid;
1047 int i;
1048
1049 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
1050 for_each_node(nid) {
1051 mz = mem_cgroup_nodeinfo(memcg, nid);
1052 for (i = 0; i <= DEF_PRIORITY; i++) {
1053 iter = &mz->iter[i];
1054 cmpxchg(&iter->position,
1055 dead_memcg, NULL);
1056 }
1057 }
1058 }
1059}
1060
1061/**
1062 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
1063 * @memcg: hierarchy root
1064 * @fn: function to call for each task
1065 * @arg: argument passed to @fn
1066 *
1067 * This function iterates over tasks attached to @memcg or to any of its
1068 * descendants and calls @fn for each task. If @fn returns a non-zero
1069 * value, the function breaks the iteration loop and returns the value.
1070 * Otherwise, it will iterate over all tasks and return 0.
1071 *
1072 * This function must not be called for the root memory cgroup.
1073 */
1074int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1075 int (*fn)(struct task_struct *, void *), void *arg)
1076{
1077 struct mem_cgroup *iter;
1078 int ret = 0;
1079
1080 BUG_ON(memcg == root_mem_cgroup);
1081
1082 for_each_mem_cgroup_tree(iter, memcg) {
1083 struct css_task_iter it;
1084 struct task_struct *task;
1085
1086 css_task_iter_start(&iter->css, 0, &it);
1087 while (!ret && (task = css_task_iter_next(&it)))
1088 ret = fn(task, arg);
1089 css_task_iter_end(&it);
1090 if (ret) {
1091 mem_cgroup_iter_break(memcg, iter);
1092 break;
1093 }
1094 }
1095 return ret;
1096}
1097
1098/**
1099 * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
1100 * @page: the page
1101 * @pgdat: pgdat of the page
1102 *
1103 * This function is only safe when following the LRU page isolation
1104 * and putback protocol: the LRU lock must be held, and the page must
1105 * either be PageLRU() or the caller must have isolated/allocated it.
1106 */
1107struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
1108{
1109 struct mem_cgroup_per_node *mz;
1110 struct mem_cgroup *memcg;
1111 struct lruvec *lruvec;
1112
1113 if (mem_cgroup_disabled()) {
1114 lruvec = &pgdat->lruvec;
1115 goto out;
1116 }
1117
1118 memcg = page->mem_cgroup;
1119 /*
1120 * Swapcache readahead pages are added to the LRU - and
1121 * possibly migrated - before they are charged.
1122 */
1123 if (!memcg)
1124 memcg = root_mem_cgroup;
1125
1126 mz = mem_cgroup_page_nodeinfo(memcg, page);
1127 lruvec = &mz->lruvec;
1128out:
1129 /*
1130 * Since a node can be onlined after the mem_cgroup was created,
1131 * we have to be prepared to initialize lruvec->zone here;
1132 * and if offlined then reonlined, we need to reinitialize it.
1133 */
1134 if (unlikely(lruvec->pgdat != pgdat))
1135 lruvec->pgdat = pgdat;
1136 return lruvec;
1137}
1138
1139/**
1140 * mem_cgroup_update_lru_size - account for adding or removing an lru page
1141 * @lruvec: mem_cgroup per zone lru vector
1142 * @lru: index of lru list the page is sitting on
1143 * @zid: zone id of the accounted pages
1144 * @nr_pages: positive when adding or negative when removing
1145 *
1146 * This function must be called under lru_lock, just before a page is added
1147 * to or just after a page is removed from an lru list (that ordering being
1148 * so as to allow it to check that lru_size 0 is consistent with list_empty).
1149 */
1150void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1151 int zid, int nr_pages)
1152{
1153 struct mem_cgroup_per_node *mz;
1154 unsigned long *lru_size;
1155 long size;
1156
1157 if (mem_cgroup_disabled())
1158 return;
1159
1160 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1161 lru_size = &mz->lru_zone_size[zid][lru];
1162
1163 if (nr_pages < 0)
1164 *lru_size += nr_pages;
1165
1166 size = *lru_size;
1167 if (WARN_ONCE(size < 0,
1168 "%s(%p, %d, %d): lru_size %ld\n",
1169 __func__, lruvec, lru, nr_pages, size)) {
1170 VM_BUG_ON(1);
1171 *lru_size = 0;
1172 }
1173
1174 if (nr_pages > 0)
1175 *lru_size += nr_pages;
1176}
1177
1178bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
1179{
1180 struct mem_cgroup *task_memcg;
1181 struct task_struct *p;
1182 bool ret;
1183
1184 p = find_lock_task_mm(task);
1185 if (p) {
1186 task_memcg = get_mem_cgroup_from_mm(p->mm);
1187 task_unlock(p);
1188 } else {
1189 /*
1190 * All threads may have already detached their mm's, but the oom
1191 * killer still needs to detect if they have already been oom
1192 * killed to prevent needlessly killing additional tasks.
1193 */
1194 rcu_read_lock();
1195 task_memcg = mem_cgroup_from_task(task);
1196 css_get(&task_memcg->css);
1197 rcu_read_unlock();
1198 }
1199 ret = mem_cgroup_is_descendant(task_memcg, memcg);
1200 css_put(&task_memcg->css);
1201 return ret;
1202}
1203
1204/**
1205 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1206 * @memcg: the memory cgroup
1207 *
1208 * Returns the maximum amount of memory @mem can be charged with, in
1209 * pages.
1210 */
1211static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1212{
1213 unsigned long margin = 0;
1214 unsigned long count;
1215 unsigned long limit;
1216
1217 count = page_counter_read(&memcg->memory);
1218 limit = READ_ONCE(memcg->memory.max);
1219 if (count < limit)
1220 margin = limit - count;
1221
1222 if (do_memsw_account()) {
1223 count = page_counter_read(&memcg->memsw);
1224 limit = READ_ONCE(memcg->memsw.max);
1225 if (count <= limit)
1226 margin = min(margin, limit - count);
1227 else
1228 margin = 0;
1229 }
1230
1231 return margin;
1232}
1233
1234/*
1235 * A routine for checking "mem" is under move_account() or not.
1236 *
1237 * Checking a cgroup is mc.from or mc.to or under hierarchy of
1238 * moving cgroups. This is for waiting at high-memory pressure
1239 * caused by "move".
1240 */
1241static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1242{
1243 struct mem_cgroup *from;
1244 struct mem_cgroup *to;
1245 bool ret = false;
1246 /*
1247 * Unlike task_move routines, we access mc.to, mc.from not under
1248 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1249 */
1250 spin_lock(&mc.lock);
1251 from = mc.from;
1252 to = mc.to;
1253 if (!from)
1254 goto unlock;
1255
1256 ret = mem_cgroup_is_descendant(from, memcg) ||
1257 mem_cgroup_is_descendant(to, memcg);
1258unlock:
1259 spin_unlock(&mc.lock);
1260 return ret;
1261}
1262
1263static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1264{
1265 if (mc.moving_task && current != mc.moving_task) {
1266 if (mem_cgroup_under_move(memcg)) {
1267 DEFINE_WAIT(wait);
1268 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1269 /* moving charge context might have finished. */
1270 if (mc.moving_task)
1271 schedule();
1272 finish_wait(&mc.waitq, &wait);
1273 return true;
1274 }
1275 }
1276 return false;
1277}
1278
1279static const unsigned int memcg1_stats[] = {
1280 MEMCG_CACHE,
1281 MEMCG_RSS,
1282 MEMCG_RSS_HUGE,
1283 NR_SHMEM,
1284 NR_FILE_MAPPED,
1285 NR_FILE_DIRTY,
1286 NR_WRITEBACK,
1287 MEMCG_SWAP,
1288};
1289
1290static const char *const memcg1_stat_names[] = {
1291 "cache",
1292 "rss",
1293 "rss_huge",
1294 "shmem",
1295 "mapped_file",
1296 "dirty",
1297 "writeback",
1298 "swap",
1299};
1300
1301#define K(x) ((x) << (PAGE_SHIFT-10))
1302/**
1303 * mem_cgroup_print_oom_context: Print OOM information relevant to
1304 * memory controller.
1305 * @memcg: The memory cgroup that went over limit
1306 * @p: Task that is going to be killed
1307 *
1308 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1309 * enabled
1310 */
1311void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1312{
1313 rcu_read_lock();
1314
1315 if (memcg) {
1316 pr_cont(",oom_memcg=");
1317 pr_cont_cgroup_path(memcg->css.cgroup);
1318 } else
1319 pr_cont(",global_oom");
1320 if (p) {
1321 pr_cont(",task_memcg=");
1322 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1323 }
1324 rcu_read_unlock();
1325}
1326
1327/**
1328 * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to
1329 * memory controller.
1330 * @memcg: The memory cgroup that went over limit
1331 */
1332void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
1333{
1334 struct mem_cgroup *iter;
1335 unsigned int i;
1336
1337 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1338 K((u64)page_counter_read(&memcg->memory)),
1339 K((u64)memcg->memory.max), memcg->memory.failcnt);
1340 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1341 K((u64)page_counter_read(&memcg->memsw)),
1342 K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1343 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1344 K((u64)page_counter_read(&memcg->kmem)),
1345 K((u64)memcg->kmem.max), memcg->kmem.failcnt);
1346
1347 for_each_mem_cgroup_tree(iter, memcg) {
1348 pr_info("Memory cgroup stats for ");
1349 pr_cont_cgroup_path(iter->css.cgroup);
1350 pr_cont(":");
1351
1352 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
1353 if (memcg1_stats[i] == MEMCG_SWAP && !do_swap_account)
1354 continue;
1355 pr_cont(" %s:%luKB", memcg1_stat_names[i],
1356 K(memcg_page_state(iter, memcg1_stats[i])));
1357 }
1358
1359 for (i = 0; i < NR_LRU_LISTS; i++)
1360 pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1361 K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1362
1363 pr_cont("\n");
1364 }
1365}
1366
1367/*
1368 * Return the memory (and swap, if configured) limit for a memcg.
1369 */
1370unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
1371{
1372 unsigned long max;
1373
1374 max = memcg->memory.max;
1375 if (mem_cgroup_swappiness(memcg)) {
1376 unsigned long memsw_max;
1377 unsigned long swap_max;
1378
1379 memsw_max = memcg->memsw.max;
1380 swap_max = memcg->swap.max;
1381 swap_max = min(swap_max, (unsigned long)total_swap_pages);
1382 max = min(max + swap_max, memsw_max);
1383 }
1384 return max;
1385}
1386
1387static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1388 int order)
1389{
1390 struct oom_control oc = {
1391 .zonelist = NULL,
1392 .nodemask = NULL,
1393 .memcg = memcg,
1394 .gfp_mask = gfp_mask,
1395 .order = order,
1396 };
1397 bool ret;
1398
1399 if (mutex_lock_killable(&oom_lock))
1400 return true;
1401 /*
1402 * A few threads which were not waiting at mutex_lock_killable() can
1403 * fail to bail out. Therefore, check again after holding oom_lock.
1404 */
1405 ret = should_force_charge() || out_of_memory(&oc);
1406 mutex_unlock(&oom_lock);
1407 return ret;
1408}
1409
1410#if MAX_NUMNODES > 1
1411
1412/**
1413 * test_mem_cgroup_node_reclaimable
1414 * @memcg: the target memcg
1415 * @nid: the node ID to be checked.
1416 * @noswap : specify true here if the user wants flle only information.
1417 *
1418 * This function returns whether the specified memcg contains any
1419 * reclaimable pages on a node. Returns true if there are any reclaimable
1420 * pages in the node.
1421 */
1422static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1423 int nid, bool noswap)
1424{
1425 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1426 return true;
1427 if (noswap || !total_swap_pages)
1428 return false;
1429 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1430 return true;
1431 return false;
1432
1433}
1434
1435/*
1436 * Always updating the nodemask is not very good - even if we have an empty
1437 * list or the wrong list here, we can start from some node and traverse all
1438 * nodes based on the zonelist. So update the list loosely once per 10 secs.
1439 *
1440 */
1441static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1442{
1443 int nid;
1444 /*
1445 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1446 * pagein/pageout changes since the last update.
1447 */
1448 if (!atomic_read(&memcg->numainfo_events))
1449 return;
1450 if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1451 return;
1452
1453 /* make a nodemask where this memcg uses memory from */
1454 memcg->scan_nodes = node_states[N_MEMORY];
1455
1456 for_each_node_mask(nid, node_states[N_MEMORY]) {
1457
1458 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1459 node_clear(nid, memcg->scan_nodes);
1460 }
1461
1462 atomic_set(&memcg->numainfo_events, 0);
1463 atomic_set(&memcg->numainfo_updating, 0);
1464}
1465
1466/*
1467 * Selecting a node where we start reclaim from. Because what we need is just
1468 * reducing usage counter, start from anywhere is O,K. Considering
1469 * memory reclaim from current node, there are pros. and cons.
1470 *
1471 * Freeing memory from current node means freeing memory from a node which
1472 * we'll use or we've used. So, it may make LRU bad. And if several threads
1473 * hit limits, it will see a contention on a node. But freeing from remote
1474 * node means more costs for memory reclaim because of memory latency.
1475 *
1476 * Now, we use round-robin. Better algorithm is welcomed.
1477 */
1478int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1479{
1480 int node;
1481
1482 mem_cgroup_may_update_nodemask(memcg);
1483 node = memcg->last_scanned_node;
1484
1485 node = next_node_in(node, memcg->scan_nodes);
1486 /*
1487 * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
1488 * last time it really checked all the LRUs due to rate limiting.
1489 * Fallback to the current node in that case for simplicity.
1490 */
1491 if (unlikely(node == MAX_NUMNODES))
1492 node = numa_node_id();
1493
1494 memcg->last_scanned_node = node;
1495 return node;
1496}
1497#else
1498int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1499{
1500 return 0;
1501}
1502#endif
1503
1504static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1505 pg_data_t *pgdat,
1506 gfp_t gfp_mask,
1507 unsigned long *total_scanned)
1508{
1509 struct mem_cgroup *victim = NULL;
1510 int total = 0;
1511 int loop = 0;
1512 unsigned long excess;
1513 unsigned long nr_scanned;
1514 struct mem_cgroup_reclaim_cookie reclaim = {
1515 .pgdat = pgdat,
1516 .priority = 0,
1517 };
1518
1519 excess = soft_limit_excess(root_memcg);
1520
1521 while (1) {
1522 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1523 if (!victim) {
1524 loop++;
1525 if (loop >= 2) {
1526 /*
1527 * If we have not been able to reclaim
1528 * anything, it might because there are
1529 * no reclaimable pages under this hierarchy
1530 */
1531 if (!total)
1532 break;
1533 /*
1534 * We want to do more targeted reclaim.
1535 * excess >> 2 is not to excessive so as to
1536 * reclaim too much, nor too less that we keep
1537 * coming back to reclaim from this cgroup
1538 */
1539 if (total >= (excess >> 2) ||
1540 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1541 break;
1542 }
1543 continue;
1544 }
1545 total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1546 pgdat, &nr_scanned);
1547 *total_scanned += nr_scanned;
1548 if (!soft_limit_excess(root_memcg))
1549 break;
1550 }
1551 mem_cgroup_iter_break(root_memcg, victim);
1552 return total;
1553}
1554
1555#ifdef CONFIG_LOCKDEP
1556static struct lockdep_map memcg_oom_lock_dep_map = {
1557 .name = "memcg_oom_lock",
1558};
1559#endif
1560
1561static DEFINE_SPINLOCK(memcg_oom_lock);
1562
1563/*
1564 * Check OOM-Killer is already running under our hierarchy.
1565 * If someone is running, return false.
1566 */
1567static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1568{
1569 struct mem_cgroup *iter, *failed = NULL;
1570
1571 spin_lock(&memcg_oom_lock);
1572
1573 for_each_mem_cgroup_tree(iter, memcg) {
1574 if (iter->oom_lock) {
1575 /*
1576 * this subtree of our hierarchy is already locked
1577 * so we cannot give a lock.
1578 */
1579 failed = iter;
1580 mem_cgroup_iter_break(memcg, iter);
1581 break;
1582 } else
1583 iter->oom_lock = true;
1584 }
1585
1586 if (failed) {
1587 /*
1588 * OK, we failed to lock the whole subtree so we have
1589 * to clean up what we set up to the failing subtree
1590 */
1591 for_each_mem_cgroup_tree(iter, memcg) {
1592 if (iter == failed) {
1593 mem_cgroup_iter_break(memcg, iter);
1594 break;
1595 }
1596 iter->oom_lock = false;
1597 }
1598 } else
1599 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1600
1601 spin_unlock(&memcg_oom_lock);
1602
1603 return !failed;
1604}
1605
1606static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1607{
1608 struct mem_cgroup *iter;
1609
1610 spin_lock(&memcg_oom_lock);
1611 mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1612 for_each_mem_cgroup_tree(iter, memcg)
1613 iter->oom_lock = false;
1614 spin_unlock(&memcg_oom_lock);
1615}
1616
1617static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1618{
1619 struct mem_cgroup *iter;
1620
1621 spin_lock(&memcg_oom_lock);
1622 for_each_mem_cgroup_tree(iter, memcg)
1623 iter->under_oom++;
1624 spin_unlock(&memcg_oom_lock);
1625}
1626
1627static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1628{
1629 struct mem_cgroup *iter;
1630
1631 /*
1632 * When a new child is created while the hierarchy is under oom,
1633 * mem_cgroup_oom_lock() may not be called. Watch for underflow.
1634 */
1635 spin_lock(&memcg_oom_lock);
1636 for_each_mem_cgroup_tree(iter, memcg)
1637 if (iter->under_oom > 0)
1638 iter->under_oom--;
1639 spin_unlock(&memcg_oom_lock);
1640}
1641
1642static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1643
1644struct oom_wait_info {
1645 struct mem_cgroup *memcg;
1646 wait_queue_entry_t wait;
1647};
1648
1649static int memcg_oom_wake_function(wait_queue_entry_t *wait,
1650 unsigned mode, int sync, void *arg)
1651{
1652 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1653 struct mem_cgroup *oom_wait_memcg;
1654 struct oom_wait_info *oom_wait_info;
1655
1656 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1657 oom_wait_memcg = oom_wait_info->memcg;
1658
1659 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1660 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1661 return 0;
1662 return autoremove_wake_function(wait, mode, sync, arg);
1663}
1664
1665static void memcg_oom_recover(struct mem_cgroup *memcg)
1666{
1667 /*
1668 * For the following lockless ->under_oom test, the only required
1669 * guarantee is that it must see the state asserted by an OOM when
1670 * this function is called as a result of userland actions
1671 * triggered by the notification of the OOM. This is trivially
1672 * achieved by invoking mem_cgroup_mark_under_oom() before
1673 * triggering notification.
1674 */
1675 if (memcg && memcg->under_oom)
1676 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1677}
1678
1679enum oom_status {
1680 OOM_SUCCESS,
1681 OOM_FAILED,
1682 OOM_ASYNC,
1683 OOM_SKIPPED
1684};
1685
1686static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1687{
1688 enum oom_status ret;
1689 bool locked;
1690
1691 if (order > PAGE_ALLOC_COSTLY_ORDER)
1692 return OOM_SKIPPED;
1693
1694 memcg_memory_event(memcg, MEMCG_OOM);
1695
1696 /*
1697 * We are in the middle of the charge context here, so we
1698 * don't want to block when potentially sitting on a callstack
1699 * that holds all kinds of filesystem and mm locks.
1700 *
1701 * cgroup1 allows disabling the OOM killer and waiting for outside
1702 * handling until the charge can succeed; remember the context and put
1703 * the task to sleep at the end of the page fault when all locks are
1704 * released.
1705 *
1706 * On the other hand, in-kernel OOM killer allows for an async victim
1707 * memory reclaim (oom_reaper) and that means that we are not solely
1708 * relying on the oom victim to make a forward progress and we can
1709 * invoke the oom killer here.
1710 *
1711 * Please note that mem_cgroup_out_of_memory might fail to find a
1712 * victim and then we have to bail out from the charge path.
1713 */
1714 if (memcg->oom_kill_disable) {
1715 if (!current->in_user_fault)
1716 return OOM_SKIPPED;
1717 css_get(&memcg->css);
1718 current->memcg_in_oom = memcg;
1719 current->memcg_oom_gfp_mask = mask;
1720 current->memcg_oom_order = order;
1721
1722 return OOM_ASYNC;
1723 }
1724
1725 mem_cgroup_mark_under_oom(memcg);
1726
1727 locked = mem_cgroup_oom_trylock(memcg);
1728
1729 if (locked)
1730 mem_cgroup_oom_notify(memcg);
1731
1732 mem_cgroup_unmark_under_oom(memcg);
1733 if (mem_cgroup_out_of_memory(memcg, mask, order))
1734 ret = OOM_SUCCESS;
1735 else
1736 ret = OOM_FAILED;
1737
1738 if (locked)
1739 mem_cgroup_oom_unlock(memcg);
1740
1741 return ret;
1742}
1743
1744/**
1745 * mem_cgroup_oom_synchronize - complete memcg OOM handling
1746 * @handle: actually kill/wait or just clean up the OOM state
1747 *
1748 * This has to be called at the end of a page fault if the memcg OOM
1749 * handler was enabled.
1750 *
1751 * Memcg supports userspace OOM handling where failed allocations must
1752 * sleep on a waitqueue until the userspace task resolves the
1753 * situation. Sleeping directly in the charge context with all kinds
1754 * of locks held is not a good idea, instead we remember an OOM state
1755 * in the task and mem_cgroup_oom_synchronize() has to be called at
1756 * the end of the page fault to complete the OOM handling.
1757 *
1758 * Returns %true if an ongoing memcg OOM situation was detected and
1759 * completed, %false otherwise.
1760 */
1761bool mem_cgroup_oom_synchronize(bool handle)
1762{
1763 struct mem_cgroup *memcg = current->memcg_in_oom;
1764 struct oom_wait_info owait;
1765 bool locked;
1766
1767 /* OOM is global, do not handle */
1768 if (!memcg)
1769 return false;
1770
1771 if (!handle)
1772 goto cleanup;
1773
1774 owait.memcg = memcg;
1775 owait.wait.flags = 0;
1776 owait.wait.func = memcg_oom_wake_function;
1777 owait.wait.private = current;
1778 INIT_LIST_HEAD(&owait.wait.entry);
1779
1780 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1781 mem_cgroup_mark_under_oom(memcg);
1782
1783 locked = mem_cgroup_oom_trylock(memcg);
1784
1785 if (locked)
1786 mem_cgroup_oom_notify(memcg);
1787
1788 if (locked && !memcg->oom_kill_disable) {
1789 mem_cgroup_unmark_under_oom(memcg);
1790 finish_wait(&memcg_oom_waitq, &owait.wait);
1791 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1792 current->memcg_oom_order);
1793 } else {
1794 schedule();
1795 mem_cgroup_unmark_under_oom(memcg);
1796 finish_wait(&memcg_oom_waitq, &owait.wait);
1797 }
1798
1799 if (locked) {
1800 mem_cgroup_oom_unlock(memcg);
1801 /*
1802 * There is no guarantee that an OOM-lock contender
1803 * sees the wakeups triggered by the OOM kill
1804 * uncharges. Wake any sleepers explicitely.
1805 */
1806 memcg_oom_recover(memcg);
1807 }
1808cleanup:
1809 current->memcg_in_oom = NULL;
1810 css_put(&memcg->css);
1811 return true;
1812}
1813
1814/**
1815 * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
1816 * @victim: task to be killed by the OOM killer
1817 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
1818 *
1819 * Returns a pointer to a memory cgroup, which has to be cleaned up
1820 * by killing all belonging OOM-killable tasks.
1821 *
1822 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
1823 */
1824struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
1825 struct mem_cgroup *oom_domain)
1826{
1827 struct mem_cgroup *oom_group = NULL;
1828 struct mem_cgroup *memcg;
1829
1830 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
1831 return NULL;
1832
1833 if (!oom_domain)
1834 oom_domain = root_mem_cgroup;
1835
1836 rcu_read_lock();
1837
1838 memcg = mem_cgroup_from_task(victim);
1839 if (memcg == root_mem_cgroup)
1840 goto out;
1841
1842 /*
1843 * Traverse the memory cgroup hierarchy from the victim task's
1844 * cgroup up to the OOMing cgroup (or root) to find the
1845 * highest-level memory cgroup with oom.group set.
1846 */
1847 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
1848 if (memcg->oom_group)
1849 oom_group = memcg;
1850
1851 if (memcg == oom_domain)
1852 break;
1853 }
1854
1855 if (oom_group)
1856 css_get(&oom_group->css);
1857out:
1858 rcu_read_unlock();
1859
1860 return oom_group;
1861}
1862
1863void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
1864{
1865 pr_info("Tasks in ");
1866 pr_cont_cgroup_path(memcg->css.cgroup);
1867 pr_cont(" are going to be killed due to memory.oom.group set\n");
1868}
1869
1870/**
1871 * lock_page_memcg - lock a page->mem_cgroup binding
1872 * @page: the page
1873 *
1874 * This function protects unlocked LRU pages from being moved to
1875 * another cgroup.
1876 *
1877 * It ensures lifetime of the returned memcg. Caller is responsible
1878 * for the lifetime of the page; __unlock_page_memcg() is available
1879 * when @page might get freed inside the locked section.
1880 */
1881struct mem_cgroup *lock_page_memcg(struct page *page)
1882{
1883 struct mem_cgroup *memcg;
1884 unsigned long flags;
1885
1886 /*
1887 * The RCU lock is held throughout the transaction. The fast
1888 * path can get away without acquiring the memcg->move_lock
1889 * because page moving starts with an RCU grace period.
1890 *
1891 * The RCU lock also protects the memcg from being freed when
1892 * the page state that is going to change is the only thing
1893 * preventing the page itself from being freed. E.g. writeback
1894 * doesn't hold a page reference and relies on PG_writeback to
1895 * keep off truncation, migration and so forth.
1896 */
1897 rcu_read_lock();
1898
1899 if (mem_cgroup_disabled())
1900 return NULL;
1901again:
1902 memcg = page->mem_cgroup;
1903 if (unlikely(!memcg))
1904 return NULL;
1905
1906 if (atomic_read(&memcg->moving_account) <= 0)
1907 return memcg;
1908
1909 spin_lock_irqsave(&memcg->move_lock, flags);
1910 if (memcg != page->mem_cgroup) {
1911 spin_unlock_irqrestore(&memcg->move_lock, flags);
1912 goto again;
1913 }
1914
1915 /*
1916 * When charge migration first begins, we can have locked and
1917 * unlocked page stat updates happening concurrently. Track
1918 * the task who has the lock for unlock_page_memcg().
1919 */
1920 memcg->move_lock_task = current;
1921 memcg->move_lock_flags = flags;
1922
1923 return memcg;
1924}
1925EXPORT_SYMBOL(lock_page_memcg);
1926
1927/**
1928 * __unlock_page_memcg - unlock and unpin a memcg
1929 * @memcg: the memcg
1930 *
1931 * Unlock and unpin a memcg returned by lock_page_memcg().
1932 */
1933void __unlock_page_memcg(struct mem_cgroup *memcg)
1934{
1935 if (memcg && memcg->move_lock_task == current) {
1936 unsigned long flags = memcg->move_lock_flags;
1937
1938 memcg->move_lock_task = NULL;
1939 memcg->move_lock_flags = 0;
1940
1941 spin_unlock_irqrestore(&memcg->move_lock, flags);
1942 }
1943
1944 rcu_read_unlock();
1945}
1946
1947/**
1948 * unlock_page_memcg - unlock a page->mem_cgroup binding
1949 * @page: the page
1950 */
1951void unlock_page_memcg(struct page *page)
1952{
1953 __unlock_page_memcg(page->mem_cgroup);
1954}
1955EXPORT_SYMBOL(unlock_page_memcg);
1956
1957struct memcg_stock_pcp {
1958 struct mem_cgroup *cached; /* this never be root cgroup */
1959 unsigned int nr_pages;
1960 struct work_struct work;
1961 unsigned long flags;
1962#define FLUSHING_CACHED_CHARGE 0
1963};
1964static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1965static DEFINE_MUTEX(percpu_charge_mutex);
1966
1967/**
1968 * consume_stock: Try to consume stocked charge on this cpu.
1969 * @memcg: memcg to consume from.
1970 * @nr_pages: how many pages to charge.
1971 *
1972 * The charges will only happen if @memcg matches the current cpu's memcg
1973 * stock, and at least @nr_pages are available in that stock. Failure to
1974 * service an allocation will refill the stock.
1975 *
1976 * returns true if successful, false otherwise.
1977 */
1978static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1979{
1980 struct memcg_stock_pcp *stock;
1981 unsigned long flags;
1982 bool ret = false;
1983
1984 if (nr_pages > MEMCG_CHARGE_BATCH)
1985 return ret;
1986
1987 local_irq_save(flags);
1988
1989 stock = this_cpu_ptr(&memcg_stock);
1990 if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
1991 stock->nr_pages -= nr_pages;
1992 ret = true;
1993 }
1994
1995 local_irq_restore(flags);
1996
1997 return ret;
1998}
1999
2000/*
2001 * Returns stocks cached in percpu and reset cached information.
2002 */
2003static void drain_stock(struct memcg_stock_pcp *stock)
2004{
2005 struct mem_cgroup *old = stock->cached;
2006
2007 if (stock->nr_pages) {
2008 page_counter_uncharge(&old->memory, stock->nr_pages);
2009 if (do_memsw_account())
2010 page_counter_uncharge(&old->memsw, stock->nr_pages);
2011 css_put_many(&old->css, stock->nr_pages);
2012 stock->nr_pages = 0;
2013 }
2014 stock->cached = NULL;
2015}
2016
2017static void drain_local_stock(struct work_struct *dummy)
2018{
2019 struct memcg_stock_pcp *stock;
2020 unsigned long flags;
2021
2022 /*
2023 * The only protection from memory hotplug vs. drain_stock races is
2024 * that we always operate on local CPU stock here with IRQ disabled
2025 */
2026 local_irq_save(flags);
2027
2028 stock = this_cpu_ptr(&memcg_stock);
2029 drain_stock(stock);
2030 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2031
2032 local_irq_restore(flags);
2033}
2034
2035/*
2036 * Cache charges(val) to local per_cpu area.
2037 * This will be consumed by consume_stock() function, later.
2038 */
2039static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2040{
2041 struct memcg_stock_pcp *stock;
2042 unsigned long flags;
2043
2044 local_irq_save(flags);
2045
2046 stock = this_cpu_ptr(&memcg_stock);
2047 if (stock->cached != memcg) { /* reset if necessary */
2048 drain_stock(stock);
2049 stock->cached = memcg;
2050 }
2051 stock->nr_pages += nr_pages;
2052
2053 if (stock->nr_pages > MEMCG_CHARGE_BATCH)
2054 drain_stock(stock);
2055
2056 local_irq_restore(flags);
2057}
2058
2059/*
2060 * Drains all per-CPU charge caches for given root_memcg resp. subtree
2061 * of the hierarchy under it.
2062 */
2063static void drain_all_stock(struct mem_cgroup *root_memcg)
2064{
2065 int cpu, curcpu;
2066
2067 /* If someone's already draining, avoid adding running more workers. */
2068 if (!mutex_trylock(&percpu_charge_mutex))
2069 return;
2070 /*
2071 * Notify other cpus that system-wide "drain" is running
2072 * We do not care about races with the cpu hotplug because cpu down
2073 * as well as workers from this path always operate on the local
2074 * per-cpu data. CPU up doesn't touch memcg_stock at all.
2075 */
2076 curcpu = get_cpu();
2077 for_each_online_cpu(cpu) {
2078 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2079 struct mem_cgroup *memcg;
2080
2081 memcg = stock->cached;
2082 if (!memcg || !stock->nr_pages || !css_tryget(&memcg->css))
2083 continue;
2084 if (!mem_cgroup_is_descendant(memcg, root_memcg)) {
2085 css_put(&memcg->css);
2086 continue;
2087 }
2088 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2089 if (cpu == curcpu)
2090 drain_local_stock(&stock->work);
2091 else
2092 schedule_work_on(cpu, &stock->work);
2093 }
2094 css_put(&memcg->css);
2095 }
2096 put_cpu();
2097 mutex_unlock(&percpu_charge_mutex);
2098}
2099
2100static int memcg_hotplug_cpu_dead(unsigned int cpu)
2101{
2102 struct memcg_stock_pcp *stock;
2103 struct mem_cgroup *memcg;
2104
2105 stock = &per_cpu(memcg_stock, cpu);
2106 drain_stock(stock);
2107
2108 for_each_mem_cgroup(memcg) {
2109 int i;
2110
2111 for (i = 0; i < MEMCG_NR_STAT; i++) {
2112 int nid;
2113 long x;
2114
2115 x = this_cpu_xchg(memcg->stat_cpu->count[i], 0);
2116 if (x)
2117 atomic_long_add(x, &memcg->stat[i]);
2118
2119 if (i >= NR_VM_NODE_STAT_ITEMS)
2120 continue;
2121
2122 for_each_node(nid) {
2123 struct mem_cgroup_per_node *pn;
2124
2125 pn = mem_cgroup_nodeinfo(memcg, nid);
2126 x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
2127 if (x)
2128 atomic_long_add(x, &pn->lruvec_stat[i]);
2129 }
2130 }
2131
2132 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
2133 long x;
2134
2135 x = this_cpu_xchg(memcg->stat_cpu->events[i], 0);
2136 if (x)
2137 atomic_long_add(x, &memcg->events[i]);
2138 }
2139 }
2140
2141 return 0;
2142}
2143
2144static void reclaim_high(struct mem_cgroup *memcg,
2145 unsigned int nr_pages,
2146 gfp_t gfp_mask)
2147{
2148 do {
2149 if (page_counter_read(&memcg->memory) <= memcg->high)
2150 continue;
2151 memcg_memory_event(memcg, MEMCG_HIGH);
2152 try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
2153 } while ((memcg = parent_mem_cgroup(memcg)));
2154}
2155
2156static void high_work_func(struct work_struct *work)
2157{
2158 struct mem_cgroup *memcg;
2159
2160 memcg = container_of(work, struct mem_cgroup, high_work);
2161 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
2162}
2163
2164/*
2165 * Scheduled by try_charge() to be executed from the userland return path
2166 * and reclaims memory over the high limit.
2167 */
2168void mem_cgroup_handle_over_high(void)
2169{
2170 unsigned int nr_pages = current->memcg_nr_pages_over_high;
2171 struct mem_cgroup *memcg;
2172
2173 if (likely(!nr_pages))
2174 return;
2175
2176 memcg = get_mem_cgroup_from_mm(current->mm);
2177 reclaim_high(memcg, nr_pages, GFP_KERNEL);
2178 css_put(&memcg->css);
2179 current->memcg_nr_pages_over_high = 0;
2180}
2181
2182static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2183 unsigned int nr_pages)
2184{
2185 unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
2186 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2187 struct mem_cgroup *mem_over_limit;
2188 struct page_counter *counter;
2189 unsigned long nr_reclaimed;
2190 bool may_swap = true;
2191 bool drained = false;
2192 bool oomed = false;
2193 enum oom_status oom_status;
2194
2195 if (mem_cgroup_is_root(memcg))
2196 return 0;
2197retry:
2198 if (consume_stock(memcg, nr_pages))
2199 return 0;
2200
2201 if (!do_memsw_account() ||
2202 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2203 if (page_counter_try_charge(&memcg->memory, batch, &counter))
2204 goto done_restock;
2205 if (do_memsw_account())
2206 page_counter_uncharge(&memcg->memsw, batch);
2207 mem_over_limit = mem_cgroup_from_counter(counter, memory);
2208 } else {
2209 mem_over_limit = mem_cgroup_from_counter(counter, memsw);
2210 may_swap = false;
2211 }
2212
2213 if (batch > nr_pages) {
2214 batch = nr_pages;
2215 goto retry;
2216 }
2217
2218 /*
2219 * Unlike in global OOM situations, memcg is not in a physical
2220 * memory shortage. Allow dying and OOM-killed tasks to
2221 * bypass the last charges so that they can exit quickly and
2222 * free their memory.
2223 */
2224 if (unlikely(should_force_charge()))
2225 goto force;
2226
2227 /*
2228 * Prevent unbounded recursion when reclaim operations need to
2229 * allocate memory. This might exceed the limits temporarily,
2230 * but we prefer facilitating memory reclaim and getting back
2231 * under the limit over triggering OOM kills in these cases.
2232 */
2233 if (unlikely(current->flags & PF_MEMALLOC))
2234 goto force;
2235
2236 if (unlikely(task_in_memcg_oom(current)))
2237 goto nomem;
2238
2239 if (!gfpflags_allow_blocking(gfp_mask))
2240 goto nomem;
2241
2242 memcg_memory_event(mem_over_limit, MEMCG_MAX);
2243
2244 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
2245 gfp_mask, may_swap);
2246
2247 if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2248 goto retry;
2249
2250 if (!drained) {
2251 drain_all_stock(mem_over_limit);
2252 drained = true;
2253 goto retry;
2254 }
2255
2256 if (gfp_mask & __GFP_NORETRY)
2257 goto nomem;
2258 /*
2259 * Even though the limit is exceeded at this point, reclaim
2260 * may have been able to free some pages. Retry the charge
2261 * before killing the task.
2262 *
2263 * Only for regular pages, though: huge pages are rather
2264 * unlikely to succeed so close to the limit, and we fall back
2265 * to regular pages anyway in case of failure.
2266 */
2267 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
2268 goto retry;
2269 /*
2270 * At task move, charge accounts can be doubly counted. So, it's
2271 * better to wait until the end of task_move if something is going on.
2272 */
2273 if (mem_cgroup_wait_acct_move(mem_over_limit))
2274 goto retry;
2275
2276 if (nr_retries--)
2277 goto retry;
2278
2279 if (gfp_mask & __GFP_RETRY_MAYFAIL && oomed)
2280 goto nomem;
2281
2282 if (gfp_mask & __GFP_NOFAIL)
2283 goto force;
2284
2285 if (fatal_signal_pending(current))
2286 goto force;
2287
2288 /*
2289 * keep retrying as long as the memcg oom killer is able to make
2290 * a forward progress or bypass the charge if the oom killer
2291 * couldn't make any progress.
2292 */
2293 oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
2294 get_order(nr_pages * PAGE_SIZE));
2295 switch (oom_status) {
2296 case OOM_SUCCESS:
2297 nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2298 oomed = true;
2299 goto retry;
2300 case OOM_FAILED:
2301 goto force;
2302 default:
2303 goto nomem;
2304 }
2305nomem:
2306 if (!(gfp_mask & __GFP_NOFAIL))
2307 return -ENOMEM;
2308force:
2309 /*
2310 * The allocation either can't fail or will lead to more memory
2311 * being freed very soon. Allow memory usage go over the limit
2312 * temporarily by force charging it.
2313 */
2314 page_counter_charge(&memcg->memory, nr_pages);
2315 if (do_memsw_account())
2316 page_counter_charge(&memcg->memsw, nr_pages);
2317 css_get_many(&memcg->css, nr_pages);
2318
2319 return 0;
2320
2321done_restock:
2322 css_get_many(&memcg->css, batch);
2323 if (batch > nr_pages)
2324 refill_stock(memcg, batch - nr_pages);
2325
2326 /*
2327 * If the hierarchy is above the normal consumption range, schedule
2328 * reclaim on returning to userland. We can perform reclaim here
2329 * if __GFP_RECLAIM but let's always punt for simplicity and so that
2330 * GFP_KERNEL can consistently be used during reclaim. @memcg is
2331 * not recorded as it most likely matches current's and won't
2332 * change in the meantime. As high limit is checked again before
2333 * reclaim, the cost of mismatch is negligible.
2334 */
2335 do {
2336 if (page_counter_read(&memcg->memory) > memcg->high) {
2337 /* Don't bother a random interrupted task */
2338 if (in_interrupt()) {
2339 schedule_work(&memcg->high_work);
2340 break;
2341 }
2342 current->memcg_nr_pages_over_high += batch;
2343 set_notify_resume(current);
2344 break;
2345 }
2346 } while ((memcg = parent_mem_cgroup(memcg)));
2347
2348 return 0;
2349}
2350
2351static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2352{
2353 if (mem_cgroup_is_root(memcg))
2354 return;
2355
2356 page_counter_uncharge(&memcg->memory, nr_pages);
2357 if (do_memsw_account())
2358 page_counter_uncharge(&memcg->memsw, nr_pages);
2359
2360 css_put_many(&memcg->css, nr_pages);
2361}
2362
2363static void lock_page_lru(struct page *page, int *isolated)
2364{
2365 pg_data_t *pgdat = page_pgdat(page);
2366
2367 spin_lock_irq(&pgdat->lru_lock);
2368 if (PageLRU(page)) {
2369 struct lruvec *lruvec;
2370
2371 lruvec = mem_cgroup_page_lruvec(page, pgdat);
2372 ClearPageLRU(page);
2373 del_page_from_lru_list(page, lruvec, page_lru(page));
2374 *isolated = 1;
2375 } else
2376 *isolated = 0;
2377}
2378
2379static void unlock_page_lru(struct page *page, int isolated)
2380{
2381 pg_data_t *pgdat = page_pgdat(page);
2382
2383 if (isolated) {
2384 struct lruvec *lruvec;
2385
2386 lruvec = mem_cgroup_page_lruvec(page, pgdat);
2387 VM_BUG_ON_PAGE(PageLRU(page), page);
2388 SetPageLRU(page);
2389 add_page_to_lru_list(page, lruvec, page_lru(page));
2390 }
2391 spin_unlock_irq(&pgdat->lru_lock);
2392}
2393
2394static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2395 bool lrucare)
2396{
2397 int isolated;
2398
2399 VM_BUG_ON_PAGE(page->mem_cgroup, page);
2400
2401 /*
2402 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2403 * may already be on some other mem_cgroup's LRU. Take care of it.
2404 */
2405 if (lrucare)
2406 lock_page_lru(page, &isolated);
2407
2408 /*
2409 * Nobody should be changing or seriously looking at
2410 * page->mem_cgroup at this point:
2411 *
2412 * - the page is uncharged
2413 *
2414 * - the page is off-LRU
2415 *
2416 * - an anonymous fault has exclusive page access, except for
2417 * a locked page table
2418 *
2419 * - a page cache insertion, a swapin fault, or a migration
2420 * have the page locked
2421 */
2422 page->mem_cgroup = memcg;
2423
2424 if (lrucare)
2425 unlock_page_lru(page, isolated);
2426}
2427
2428#ifdef CONFIG_MEMCG_KMEM
2429static int memcg_alloc_cache_id(void)
2430{
2431 int id, size;
2432 int err;
2433
2434 id = ida_simple_get(&memcg_cache_ida,
2435 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2436 if (id < 0)
2437 return id;
2438
2439 if (id < memcg_nr_cache_ids)
2440 return id;
2441
2442 /*
2443 * There's no space for the new id in memcg_caches arrays,
2444 * so we have to grow them.
2445 */
2446 down_write(&memcg_cache_ids_sem);
2447
2448 size = 2 * (id + 1);
2449 if (size < MEMCG_CACHES_MIN_SIZE)
2450 size = MEMCG_CACHES_MIN_SIZE;
2451 else if (size > MEMCG_CACHES_MAX_SIZE)
2452 size = MEMCG_CACHES_MAX_SIZE;
2453
2454 err = memcg_update_all_caches(size);
2455 if (!err)
2456 err = memcg_update_all_list_lrus(size);
2457 if (!err)
2458 memcg_nr_cache_ids = size;
2459
2460 up_write(&memcg_cache_ids_sem);
2461
2462 if (err) {
2463 ida_simple_remove(&memcg_cache_ida, id);
2464 return err;
2465 }
2466 return id;
2467}
2468
2469static void memcg_free_cache_id(int id)
2470{
2471 ida_simple_remove(&memcg_cache_ida, id);
2472}
2473
2474struct memcg_kmem_cache_create_work {
2475 struct mem_cgroup *memcg;
2476 struct kmem_cache *cachep;
2477 struct work_struct work;
2478};
2479
2480static void memcg_kmem_cache_create_func(struct work_struct *w)
2481{
2482 struct memcg_kmem_cache_create_work *cw =
2483 container_of(w, struct memcg_kmem_cache_create_work, work);
2484 struct mem_cgroup *memcg = cw->memcg;
2485 struct kmem_cache *cachep = cw->cachep;
2486
2487 memcg_create_kmem_cache(memcg, cachep);
2488
2489 css_put(&memcg->css);
2490 kfree(cw);
2491}
2492
2493/*
2494 * Enqueue the creation of a per-memcg kmem_cache.
2495 */
2496static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2497 struct kmem_cache *cachep)
2498{
2499 struct memcg_kmem_cache_create_work *cw;
2500
2501 cw = kmalloc(sizeof(*cw), GFP_NOWAIT | __GFP_NOWARN);
2502 if (!cw)
2503 return;
2504
2505 css_get(&memcg->css);
2506
2507 cw->memcg = memcg;
2508 cw->cachep = cachep;
2509 INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2510
2511 queue_work(memcg_kmem_cache_wq, &cw->work);
2512}
2513
2514static inline bool memcg_kmem_bypass(void)
2515{
2516 if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
2517 return true;
2518 return false;
2519}
2520
2521/**
2522 * memcg_kmem_get_cache: select the correct per-memcg cache for allocation
2523 * @cachep: the original global kmem cache
2524 *
2525 * Return the kmem_cache we're supposed to use for a slab allocation.
2526 * We try to use the current memcg's version of the cache.
2527 *
2528 * If the cache does not exist yet, if we are the first user of it, we
2529 * create it asynchronously in a workqueue and let the current allocation
2530 * go through with the original cache.
2531 *
2532 * This function takes a reference to the cache it returns to assure it
2533 * won't get destroyed while we are working with it. Once the caller is
2534 * done with it, memcg_kmem_put_cache() must be called to release the
2535 * reference.
2536 */
2537struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
2538{
2539 struct mem_cgroup *memcg;
2540 struct kmem_cache *memcg_cachep;
2541 int kmemcg_id;
2542
2543 VM_BUG_ON(!is_root_cache(cachep));
2544
2545 if (memcg_kmem_bypass())
2546 return cachep;
2547
2548 memcg = get_mem_cgroup_from_current();
2549 kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2550 if (kmemcg_id < 0)
2551 goto out;
2552
2553 memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2554 if (likely(memcg_cachep))
2555 return memcg_cachep;
2556
2557 /*
2558 * If we are in a safe context (can wait, and not in interrupt
2559 * context), we could be be predictable and return right away.
2560 * This would guarantee that the allocation being performed
2561 * already belongs in the new cache.
2562 *
2563 * However, there are some clashes that can arrive from locking.
2564 * For instance, because we acquire the slab_mutex while doing
2565 * memcg_create_kmem_cache, this means no further allocation
2566 * could happen with the slab_mutex held. So it's better to
2567 * defer everything.
2568 */
2569 memcg_schedule_kmem_cache_create(memcg, cachep);
2570out:
2571 css_put(&memcg->css);
2572 return cachep;
2573}
2574
2575/**
2576 * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache
2577 * @cachep: the cache returned by memcg_kmem_get_cache
2578 */
2579void memcg_kmem_put_cache(struct kmem_cache *cachep)
2580{
2581 if (!is_root_cache(cachep))
2582 css_put(&cachep->memcg_params.memcg->css);
2583}
2584
2585/**
2586 * __memcg_kmem_charge_memcg: charge a kmem page
2587 * @page: page to charge
2588 * @gfp: reclaim mode
2589 * @order: allocation order
2590 * @memcg: memory cgroup to charge
2591 *
2592 * Returns 0 on success, an error code on failure.
2593 */
2594int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
2595 struct mem_cgroup *memcg)
2596{
2597 unsigned int nr_pages = 1 << order;
2598 struct page_counter *counter;
2599 int ret;
2600
2601 ret = try_charge(memcg, gfp, nr_pages);
2602 if (ret)
2603 return ret;
2604
2605 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2606 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2607 cancel_charge(memcg, nr_pages);
2608 return -ENOMEM;
2609 }
2610
2611 page->mem_cgroup = memcg;
2612
2613 return 0;
2614}
2615
2616/**
2617 * __memcg_kmem_charge: charge a kmem page to the current memory cgroup
2618 * @page: page to charge
2619 * @gfp: reclaim mode
2620 * @order: allocation order
2621 *
2622 * Returns 0 on success, an error code on failure.
2623 */
2624int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2625{
2626 struct mem_cgroup *memcg;
2627 int ret = 0;
2628
2629 if (memcg_kmem_bypass())
2630 return 0;
2631
2632 memcg = get_mem_cgroup_from_current();
2633 if (!mem_cgroup_is_root(memcg)) {
2634 ret = __memcg_kmem_charge_memcg(page, gfp, order, memcg);
2635 if (!ret)
2636 __SetPageKmemcg(page);
2637 }
2638 css_put(&memcg->css);
2639 return ret;
2640}
2641/**
2642 * __memcg_kmem_uncharge: uncharge a kmem page
2643 * @page: page to uncharge
2644 * @order: allocation order
2645 */
2646void __memcg_kmem_uncharge(struct page *page, int order)
2647{
2648 struct mem_cgroup *memcg = page->mem_cgroup;
2649 unsigned int nr_pages = 1 << order;
2650
2651 if (!memcg)
2652 return;
2653
2654 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2655
2656 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2657 page_counter_uncharge(&memcg->kmem, nr_pages);
2658
2659 page_counter_uncharge(&memcg->memory, nr_pages);
2660 if (do_memsw_account())
2661 page_counter_uncharge(&memcg->memsw, nr_pages);
2662
2663 page->mem_cgroup = NULL;
2664
2665 /* slab pages do not have PageKmemcg flag set */
2666 if (PageKmemcg(page))
2667 __ClearPageKmemcg(page);
2668
2669 css_put_many(&memcg->css, nr_pages);
2670}
2671#endif /* CONFIG_MEMCG_KMEM */
2672
2673#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2674
2675/*
2676 * Because tail pages are not marked as "used", set it. We're under
2677 * pgdat->lru_lock and migration entries setup in all page mappings.
2678 */
2679void mem_cgroup_split_huge_fixup(struct page *head)
2680{
2681 int i;
2682
2683 if (mem_cgroup_disabled())
2684 return;
2685
2686 for (i = 1; i < HPAGE_PMD_NR; i++)
2687 head[i].mem_cgroup = head->mem_cgroup;
2688
2689 __mod_memcg_state(head->mem_cgroup, MEMCG_RSS_HUGE, -HPAGE_PMD_NR);
2690}
2691#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2692
2693#ifdef CONFIG_MEMCG_SWAP
2694/**
2695 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2696 * @entry: swap entry to be moved
2697 * @from: mem_cgroup which the entry is moved from
2698 * @to: mem_cgroup which the entry is moved to
2699 *
2700 * It succeeds only when the swap_cgroup's record for this entry is the same
2701 * as the mem_cgroup's id of @from.
2702 *
2703 * Returns 0 on success, -EINVAL on failure.
2704 *
2705 * The caller must have charged to @to, IOW, called page_counter_charge() about
2706 * both res and memsw, and called css_get().
2707 */
2708static int mem_cgroup_move_swap_account(swp_entry_t entry,
2709 struct mem_cgroup *from, struct mem_cgroup *to)
2710{
2711 unsigned short old_id, new_id;
2712
2713 old_id = mem_cgroup_id(from);
2714 new_id = mem_cgroup_id(to);
2715
2716 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2717 mod_memcg_state(from, MEMCG_SWAP, -1);
2718 mod_memcg_state(to, MEMCG_SWAP, 1);
2719 return 0;
2720 }
2721 return -EINVAL;
2722}
2723#else
2724static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2725 struct mem_cgroup *from, struct mem_cgroup *to)
2726{
2727 return -EINVAL;
2728}
2729#endif
2730
2731static DEFINE_MUTEX(memcg_max_mutex);
2732
2733static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
2734 unsigned long max, bool memsw)
2735{
2736 bool enlarge = false;
2737 bool drained = false;
2738 int ret;
2739 bool limits_invariant;
2740 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
2741
2742 do {
2743 if (signal_pending(current)) {
2744 ret = -EINTR;
2745 break;
2746 }
2747
2748 mutex_lock(&memcg_max_mutex);
2749 /*
2750 * Make sure that the new limit (memsw or memory limit) doesn't
2751 * break our basic invariant rule memory.max <= memsw.max.
2752 */
2753 limits_invariant = memsw ? max >= memcg->memory.max :
2754 max <= memcg->memsw.max;
2755 if (!limits_invariant) {
2756 mutex_unlock(&memcg_max_mutex);
2757 ret = -EINVAL;
2758 break;
2759 }
2760 if (max > counter->max)
2761 enlarge = true;
2762 ret = page_counter_set_max(counter, max);
2763 mutex_unlock(&memcg_max_mutex);
2764
2765 if (!ret)
2766 break;
2767
2768 if (!drained) {
2769 drain_all_stock(memcg);
2770 drained = true;
2771 continue;
2772 }
2773
2774 if (!try_to_free_mem_cgroup_pages(memcg, 1,
2775 GFP_KERNEL, !memsw)) {
2776 ret = -EBUSY;
2777 break;
2778 }
2779 } while (true);
2780
2781 if (!ret && enlarge)
2782 memcg_oom_recover(memcg);
2783
2784 return ret;
2785}
2786
2787unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
2788 gfp_t gfp_mask,
2789 unsigned long *total_scanned)
2790{
2791 unsigned long nr_reclaimed = 0;
2792 struct mem_cgroup_per_node *mz, *next_mz = NULL;
2793 unsigned long reclaimed;
2794 int loop = 0;
2795 struct mem_cgroup_tree_per_node *mctz;
2796 unsigned long excess;
2797 unsigned long nr_scanned;
2798
2799 if (order > 0)
2800 return 0;
2801
2802 mctz = soft_limit_tree_node(pgdat->node_id);
2803
2804 /*
2805 * Do not even bother to check the largest node if the root
2806 * is empty. Do it lockless to prevent lock bouncing. Races
2807 * are acceptable as soft limit is best effort anyway.
2808 */
2809 if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
2810 return 0;
2811
2812 /*
2813 * This loop can run a while, specially if mem_cgroup's continuously
2814 * keep exceeding their soft limit and putting the system under
2815 * pressure
2816 */
2817 do {
2818 if (next_mz)
2819 mz = next_mz;
2820 else
2821 mz = mem_cgroup_largest_soft_limit_node(mctz);
2822 if (!mz)
2823 break;
2824
2825 nr_scanned = 0;
2826 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
2827 gfp_mask, &nr_scanned);
2828 nr_reclaimed += reclaimed;
2829 *total_scanned += nr_scanned;
2830 spin_lock_irq(&mctz->lock);
2831 __mem_cgroup_remove_exceeded(mz, mctz);
2832
2833 /*
2834 * If we failed to reclaim anything from this memory cgroup
2835 * it is time to move on to the next cgroup
2836 */
2837 next_mz = NULL;
2838 if (!reclaimed)
2839 next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
2840
2841 excess = soft_limit_excess(mz->memcg);
2842 /*
2843 * One school of thought says that we should not add
2844 * back the node to the tree if reclaim returns 0.
2845 * But our reclaim could return 0, simply because due
2846 * to priority we are exposing a smaller subset of
2847 * memory to reclaim from. Consider this as a longer
2848 * term TODO.
2849 */
2850 /* If excess == 0, no tree ops */
2851 __mem_cgroup_insert_exceeded(mz, mctz, excess);
2852 spin_unlock_irq(&mctz->lock);
2853 css_put(&mz->memcg->css);
2854 loop++;
2855 /*
2856 * Could not reclaim anything and there are no more
2857 * mem cgroups to try or we seem to be looping without
2858 * reclaiming anything.
2859 */
2860 if (!nr_reclaimed &&
2861 (next_mz == NULL ||
2862 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
2863 break;
2864 } while (!nr_reclaimed);
2865 if (next_mz)
2866 css_put(&next_mz->memcg->css);
2867 return nr_reclaimed;
2868}
2869
2870/*
2871 * Test whether @memcg has children, dead or alive. Note that this
2872 * function doesn't care whether @memcg has use_hierarchy enabled and
2873 * returns %true if there are child csses according to the cgroup
2874 * hierarchy. Testing use_hierarchy is the caller's responsiblity.
2875 */
2876static inline bool memcg_has_children(struct mem_cgroup *memcg)
2877{
2878 bool ret;
2879
2880 rcu_read_lock();
2881 ret = css_next_child(NULL, &memcg->css);
2882 rcu_read_unlock();
2883 return ret;
2884}
2885
2886/*
2887 * Reclaims as many pages from the given memcg as possible.
2888 *
2889 * Caller is responsible for holding css reference for memcg.
2890 */
2891static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
2892{
2893 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2894
2895 /* we call try-to-free pages for make this cgroup empty */
2896 lru_add_drain_all();
2897
2898 drain_all_stock(memcg);
2899
2900 /* try to free all pages in this cgroup */
2901 while (nr_retries && page_counter_read(&memcg->memory)) {
2902 int progress;
2903
2904 if (signal_pending(current))
2905 return -EINTR;
2906
2907 progress = try_to_free_mem_cgroup_pages(memcg, 1,
2908 GFP_KERNEL, true);
2909 if (!progress) {
2910 nr_retries--;
2911 /* maybe some writeback is necessary */
2912 congestion_wait(BLK_RW_ASYNC, HZ/10);
2913 }
2914
2915 }
2916
2917 return 0;
2918}
2919
2920static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
2921 char *buf, size_t nbytes,
2922 loff_t off)
2923{
2924 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2925
2926 if (mem_cgroup_is_root(memcg))
2927 return -EINVAL;
2928 return mem_cgroup_force_empty(memcg) ?: nbytes;
2929}
2930
2931static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
2932 struct cftype *cft)
2933{
2934 return mem_cgroup_from_css(css)->use_hierarchy;
2935}
2936
2937static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
2938 struct cftype *cft, u64 val)
2939{
2940 int retval = 0;
2941 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2942 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
2943
2944 if (memcg->use_hierarchy == val)
2945 return 0;
2946
2947 /*
2948 * If parent's use_hierarchy is set, we can't make any modifications
2949 * in the child subtrees. If it is unset, then the change can
2950 * occur, provided the current cgroup has no children.
2951 *
2952 * For the root cgroup, parent_mem is NULL, we allow value to be
2953 * set if there are no children.
2954 */
2955 if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
2956 (val == 1 || val == 0)) {
2957 if (!memcg_has_children(memcg))
2958 memcg->use_hierarchy = val;
2959 else
2960 retval = -EBUSY;
2961 } else
2962 retval = -EINVAL;
2963
2964 return retval;
2965}
2966
2967struct accumulated_stats {
2968 unsigned long stat[MEMCG_NR_STAT];
2969 unsigned long events[NR_VM_EVENT_ITEMS];
2970 unsigned long lru_pages[NR_LRU_LISTS];
2971 const unsigned int *stats_array;
2972 const unsigned int *events_array;
2973 int stats_size;
2974 int events_size;
2975};
2976
2977static void accumulate_memcg_tree(struct mem_cgroup *memcg,
2978 struct accumulated_stats *acc)
2979{
2980 struct mem_cgroup *mi;
2981 int i;
2982
2983 for_each_mem_cgroup_tree(mi, memcg) {
2984 for (i = 0; i < acc->stats_size; i++)
2985 acc->stat[i] += memcg_page_state(mi,
2986 acc->stats_array ? acc->stats_array[i] : i);
2987
2988 for (i = 0; i < acc->events_size; i++)
2989 acc->events[i] += memcg_sum_events(mi,
2990 acc->events_array ? acc->events_array[i] : i);
2991
2992 for (i = 0; i < NR_LRU_LISTS; i++)
2993 acc->lru_pages[i] +=
2994 mem_cgroup_nr_lru_pages(mi, BIT(i));
2995 }
2996}
2997
2998static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
2999{
3000 unsigned long val = 0;
3001
3002 if (mem_cgroup_is_root(memcg)) {
3003 struct mem_cgroup *iter;
3004
3005 for_each_mem_cgroup_tree(iter, memcg) {
3006 val += memcg_page_state(iter, MEMCG_CACHE);
3007 val += memcg_page_state(iter, MEMCG_RSS);
3008 if (swap)
3009 val += memcg_page_state(iter, MEMCG_SWAP);
3010 }
3011 } else {
3012 if (!swap)
3013 val = page_counter_read(&memcg->memory);
3014 else
3015 val = page_counter_read(&memcg->memsw);
3016 }
3017 return val;
3018}
3019
3020enum {
3021 RES_USAGE,
3022 RES_LIMIT,
3023 RES_MAX_USAGE,
3024 RES_FAILCNT,
3025 RES_SOFT_LIMIT,
3026};
3027
3028static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
3029 struct cftype *cft)
3030{
3031 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3032 struct page_counter *counter;
3033
3034 switch (MEMFILE_TYPE(cft->private)) {
3035 case _MEM:
3036 counter = &memcg->memory;
3037 break;
3038 case _MEMSWAP:
3039 counter = &memcg->memsw;
3040 break;
3041 case _KMEM:
3042 counter = &memcg->kmem;
3043 break;
3044 case _TCP:
3045 counter = &memcg->tcpmem;
3046 break;
3047 default:
3048 BUG();
3049 }
3050
3051 switch (MEMFILE_ATTR(cft->private)) {
3052 case RES_USAGE:
3053 if (counter == &memcg->memory)
3054 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
3055 if (counter == &memcg->memsw)
3056 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
3057 return (u64)page_counter_read(counter) * PAGE_SIZE;
3058 case RES_LIMIT:
3059 return (u64)counter->max * PAGE_SIZE;
3060 case RES_MAX_USAGE:
3061 return (u64)counter->watermark * PAGE_SIZE;
3062 case RES_FAILCNT:
3063 return counter->failcnt;
3064 case RES_SOFT_LIMIT:
3065 return (u64)memcg->soft_limit * PAGE_SIZE;
3066 default:
3067 BUG();
3068 }
3069}
3070
3071#ifdef CONFIG_MEMCG_KMEM
3072static int memcg_online_kmem(struct mem_cgroup *memcg)
3073{
3074 int memcg_id;
3075
3076 if (cgroup_memory_nokmem)
3077 return 0;
3078
3079 BUG_ON(memcg->kmemcg_id >= 0);
3080 BUG_ON(memcg->kmem_state);
3081
3082 memcg_id = memcg_alloc_cache_id();
3083 if (memcg_id < 0)
3084 return memcg_id;
3085
3086 static_branch_inc(&memcg_kmem_enabled_key);
3087 /*
3088 * A memory cgroup is considered kmem-online as soon as it gets
3089 * kmemcg_id. Setting the id after enabling static branching will
3090 * guarantee no one starts accounting before all call sites are
3091 * patched.
3092 */
3093 memcg->kmemcg_id = memcg_id;
3094 memcg->kmem_state = KMEM_ONLINE;
3095 INIT_LIST_HEAD(&memcg->kmem_caches);
3096
3097 return 0;
3098}
3099
3100static void memcg_offline_kmem(struct mem_cgroup *memcg)
3101{
3102 struct cgroup_subsys_state *css;
3103 struct mem_cgroup *parent, *child;
3104 int kmemcg_id;
3105
3106 if (memcg->kmem_state != KMEM_ONLINE)
3107 return;
3108 /*
3109 * Clear the online state before clearing memcg_caches array
3110 * entries. The slab_mutex in memcg_deactivate_kmem_caches()
3111 * guarantees that no cache will be created for this cgroup
3112 * after we are done (see memcg_create_kmem_cache()).
3113 */
3114 memcg->kmem_state = KMEM_ALLOCATED;
3115
3116 memcg_deactivate_kmem_caches(memcg);
3117
3118 kmemcg_id = memcg->kmemcg_id;
3119 BUG_ON(kmemcg_id < 0);
3120
3121 parent = parent_mem_cgroup(memcg);
3122 if (!parent)
3123 parent = root_mem_cgroup;
3124
3125 /*
3126 * Change kmemcg_id of this cgroup and all its descendants to the
3127 * parent's id, and then move all entries from this cgroup's list_lrus
3128 * to ones of the parent. After we have finished, all list_lrus
3129 * corresponding to this cgroup are guaranteed to remain empty. The
3130 * ordering is imposed by list_lru_node->lock taken by
3131 * memcg_drain_all_list_lrus().
3132 */
3133 rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
3134 css_for_each_descendant_pre(css, &memcg->css) {
3135 child = mem_cgroup_from_css(css);
3136 BUG_ON(child->kmemcg_id != kmemcg_id);
3137 child->kmemcg_id = parent->kmemcg_id;
3138 if (!memcg->use_hierarchy)
3139 break;
3140 }
3141 rcu_read_unlock();
3142
3143 memcg_drain_all_list_lrus(kmemcg_id, parent);
3144
3145 memcg_free_cache_id(kmemcg_id);
3146}
3147
3148static void memcg_free_kmem(struct mem_cgroup *memcg)
3149{
3150 /* css_alloc() failed, offlining didn't happen */
3151 if (unlikely(memcg->kmem_state == KMEM_ONLINE))
3152 memcg_offline_kmem(memcg);
3153
3154 if (memcg->kmem_state == KMEM_ALLOCATED) {
3155 memcg_destroy_kmem_caches(memcg);
3156 static_branch_dec(&memcg_kmem_enabled_key);
3157 WARN_ON(page_counter_read(&memcg->kmem));
3158 }
3159}
3160#else
3161static int memcg_online_kmem(struct mem_cgroup *memcg)
3162{
3163 return 0;
3164}
3165static void memcg_offline_kmem(struct mem_cgroup *memcg)
3166{
3167}
3168static void memcg_free_kmem(struct mem_cgroup *memcg)
3169{
3170}
3171#endif /* CONFIG_MEMCG_KMEM */
3172
3173static int memcg_update_kmem_max(struct mem_cgroup *memcg,
3174 unsigned long max)
3175{
3176 int ret;
3177
3178 mutex_lock(&memcg_max_mutex);
3179 ret = page_counter_set_max(&memcg->kmem, max);
3180 mutex_unlock(&memcg_max_mutex);
3181 return ret;
3182}
3183
3184static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
3185{
3186 int ret;
3187
3188 mutex_lock(&memcg_max_mutex);
3189
3190 ret = page_counter_set_max(&memcg->tcpmem, max);
3191 if (ret)
3192 goto out;
3193
3194 if (!memcg->tcpmem_active) {
3195 /*
3196 * The active flag needs to be written after the static_key
3197 * update. This is what guarantees that the socket activation
3198 * function is the last one to run. See mem_cgroup_sk_alloc()
3199 * for details, and note that we don't mark any socket as
3200 * belonging to this memcg until that flag is up.
3201 *
3202 * We need to do this, because static_keys will span multiple
3203 * sites, but we can't control their order. If we mark a socket
3204 * as accounted, but the accounting functions are not patched in
3205 * yet, we'll lose accounting.
3206 *
3207 * We never race with the readers in mem_cgroup_sk_alloc(),
3208 * because when this value change, the code to process it is not
3209 * patched in yet.
3210 */
3211 static_branch_inc(&memcg_sockets_enabled_key);
3212 memcg->tcpmem_active = true;
3213 }
3214out:
3215 mutex_unlock(&memcg_max_mutex);
3216 return ret;
3217}
3218
3219/*
3220 * The user of this function is...
3221 * RES_LIMIT.
3222 */
3223static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
3224 char *buf, size_t nbytes, loff_t off)
3225{
3226 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3227 unsigned long nr_pages;
3228 int ret;
3229
3230 buf = strstrip(buf);
3231 ret = page_counter_memparse(buf, "-1", &nr_pages);
3232 if (ret)
3233 return ret;
3234
3235 switch (MEMFILE_ATTR(of_cft(of)->private)) {
3236 case RES_LIMIT:
3237 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3238 ret = -EINVAL;
3239 break;
3240 }
3241 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3242 case _MEM:
3243 ret = mem_cgroup_resize_max(memcg, nr_pages, false);
3244 break;
3245 case _MEMSWAP:
3246 ret = mem_cgroup_resize_max(memcg, nr_pages, true);
3247 break;
3248 case _KMEM:
3249 ret = memcg_update_kmem_max(memcg, nr_pages);
3250 break;
3251 case _TCP:
3252 ret = memcg_update_tcp_max(memcg, nr_pages);
3253 break;
3254 }
3255 break;
3256 case RES_SOFT_LIMIT:
3257 memcg->soft_limit = nr_pages;
3258 ret = 0;
3259 break;
3260 }
3261 return ret ?: nbytes;
3262}
3263
3264static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3265 size_t nbytes, loff_t off)
3266{
3267 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3268 struct page_counter *counter;
3269
3270 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3271 case _MEM:
3272 counter = &memcg->memory;
3273 break;
3274 case _MEMSWAP:
3275 counter = &memcg->memsw;
3276 break;
3277 case _KMEM:
3278 counter = &memcg->kmem;
3279 break;
3280 case _TCP:
3281 counter = &memcg->tcpmem;
3282 break;
3283 default:
3284 BUG();
3285 }
3286
3287 switch (MEMFILE_ATTR(of_cft(of)->private)) {
3288 case RES_MAX_USAGE:
3289 page_counter_reset_watermark(counter);
3290 break;
3291 case RES_FAILCNT:
3292 counter->failcnt = 0;
3293 break;
3294 default:
3295 BUG();
3296 }
3297
3298 return nbytes;
3299}
3300
3301static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3302 struct cftype *cft)
3303{
3304 return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3305}
3306
3307#ifdef CONFIG_MMU
3308static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3309 struct cftype *cft, u64 val)
3310{
3311 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3312
3313 if (val & ~MOVE_MASK)
3314 return -EINVAL;
3315
3316 /*
3317 * No kind of locking is needed in here, because ->can_attach() will
3318 * check this value once in the beginning of the process, and then carry
3319 * on with stale data. This means that changes to this value will only
3320 * affect task migrations starting after the change.
3321 */
3322 memcg->move_charge_at_immigrate = val;
3323 return 0;
3324}
3325#else
3326static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3327 struct cftype *cft, u64 val)
3328{
3329 return -ENOSYS;
3330}
3331#endif
3332
3333#ifdef CONFIG_NUMA
3334static int memcg_numa_stat_show(struct seq_file *m, void *v)
3335{
3336 struct numa_stat {
3337 const char *name;
3338 unsigned int lru_mask;
3339 };
3340
3341 static const struct numa_stat stats[] = {
3342 { "total", LRU_ALL },
3343 { "file", LRU_ALL_FILE },
3344 { "anon", LRU_ALL_ANON },
3345 { "unevictable", BIT(LRU_UNEVICTABLE) },
3346 };
3347 const struct numa_stat *stat;
3348 int nid;
3349 unsigned long nr;
3350 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
3351
3352 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3353 nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
3354 seq_printf(m, "%s=%lu", stat->name, nr);
3355 for_each_node_state(nid, N_MEMORY) {
3356 nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
3357 stat->lru_mask);
3358 seq_printf(m, " N%d=%lu", nid, nr);
3359 }
3360 seq_putc(m, '\n');
3361 }
3362
3363 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3364 struct mem_cgroup *iter;
3365
3366 nr = 0;
3367 for_each_mem_cgroup_tree(iter, memcg)
3368 nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
3369 seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
3370 for_each_node_state(nid, N_MEMORY) {
3371 nr = 0;
3372 for_each_mem_cgroup_tree(iter, memcg)
3373 nr += mem_cgroup_node_nr_lru_pages(
3374 iter, nid, stat->lru_mask);
3375 seq_printf(m, " N%d=%lu", nid, nr);
3376 }
3377 seq_putc(m, '\n');
3378 }
3379
3380 return 0;
3381}
3382#endif /* CONFIG_NUMA */
3383
3384/* Universal VM events cgroup1 shows, original sort order */
3385static const unsigned int memcg1_events[] = {
3386 PGPGIN,
3387 PGPGOUT,
3388 PGFAULT,
3389 PGMAJFAULT,
3390};
3391
3392static const char *const memcg1_event_names[] = {
3393 "pgpgin",
3394 "pgpgout",
3395 "pgfault",
3396 "pgmajfault",
3397};
3398
3399static int memcg_stat_show(struct seq_file *m, void *v)
3400{
3401 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
3402 unsigned long memory, memsw;
3403 struct mem_cgroup *mi;
3404 unsigned int i;
3405 struct accumulated_stats acc;
3406
3407 BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
3408 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
3409
3410 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3411 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3412 continue;
3413 seq_printf(m, "%s %lu\n", memcg1_stat_names[i],
3414 memcg_page_state(memcg, memcg1_stats[i]) *
3415 PAGE_SIZE);
3416 }
3417
3418 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
3419 seq_printf(m, "%s %lu\n", memcg1_event_names[i],
3420 memcg_sum_events(memcg, memcg1_events[i]));
3421
3422 for (i = 0; i < NR_LRU_LISTS; i++)
3423 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3424 mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
3425
3426 /* Hierarchical information */
3427 memory = memsw = PAGE_COUNTER_MAX;
3428 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3429 memory = min(memory, mi->memory.max);
3430 memsw = min(memsw, mi->memsw.max);
3431 }
3432 seq_printf(m, "hierarchical_memory_limit %llu\n",
3433 (u64)memory * PAGE_SIZE);
3434 if (do_memsw_account())
3435 seq_printf(m, "hierarchical_memsw_limit %llu\n",
3436 (u64)memsw * PAGE_SIZE);
3437
3438 memset(&acc, 0, sizeof(acc));
3439 acc.stats_size = ARRAY_SIZE(memcg1_stats);
3440 acc.stats_array = memcg1_stats;
3441 acc.events_size = ARRAY_SIZE(memcg1_events);
3442 acc.events_array = memcg1_events;
3443 accumulate_memcg_tree(memcg, &acc);
3444
3445 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
3446 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
3447 continue;
3448 seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
3449 (u64)acc.stat[i] * PAGE_SIZE);
3450 }
3451
3452 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
3453 seq_printf(m, "total_%s %llu\n", memcg1_event_names[i],
3454 (u64)acc.events[i]);
3455
3456 for (i = 0; i < NR_LRU_LISTS; i++)
3457 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i],
3458 (u64)acc.lru_pages[i] * PAGE_SIZE);
3459
3460#ifdef CONFIG_DEBUG_VM
3461 {
3462 pg_data_t *pgdat;
3463 struct mem_cgroup_per_node *mz;
3464 struct zone_reclaim_stat *rstat;
3465 unsigned long recent_rotated[2] = {0, 0};
3466 unsigned long recent_scanned[2] = {0, 0};
3467
3468 for_each_online_pgdat(pgdat) {
3469 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
3470 rstat = &mz->lruvec.reclaim_stat;
3471
3472 recent_rotated[0] += rstat->recent_rotated[0];
3473 recent_rotated[1] += rstat->recent_rotated[1];
3474 recent_scanned[0] += rstat->recent_scanned[0];
3475 recent_scanned[1] += rstat->recent_scanned[1];
3476 }
3477 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
3478 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
3479 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
3480 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
3481 }
3482#endif
3483
3484 return 0;
3485}
3486
3487static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
3488 struct cftype *cft)
3489{
3490 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3491
3492 return mem_cgroup_swappiness(memcg);
3493}
3494
3495static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
3496 struct cftype *cft, u64 val)
3497{
3498 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3499
3500 if (val > 100)
3501 return -EINVAL;
3502
3503 if (css->parent)
3504 memcg->swappiness = val;
3505 else
3506 vm_swappiness = val;
3507
3508 return 0;
3509}
3510
3511static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3512{
3513 struct mem_cgroup_threshold_ary *t;
3514 unsigned long usage;
3515 int i;
3516
3517 rcu_read_lock();
3518 if (!swap)
3519 t = rcu_dereference(memcg->thresholds.primary);
3520 else
3521 t = rcu_dereference(memcg->memsw_thresholds.primary);
3522
3523 if (!t)
3524 goto unlock;
3525
3526 usage = mem_cgroup_usage(memcg, swap);
3527
3528 /*
3529 * current_threshold points to threshold just below or equal to usage.
3530 * If it's not true, a threshold was crossed after last
3531 * call of __mem_cgroup_threshold().
3532 */
3533 i = t->current_threshold;
3534
3535 /*
3536 * Iterate backward over array of thresholds starting from
3537 * current_threshold and check if a threshold is crossed.
3538 * If none of thresholds below usage is crossed, we read
3539 * only one element of the array here.
3540 */
3541 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3542 eventfd_signal(t->entries[i].eventfd, 1);
3543
3544 /* i = current_threshold + 1 */
3545 i++;
3546
3547 /*
3548 * Iterate forward over array of thresholds starting from
3549 * current_threshold+1 and check if a threshold is crossed.
3550 * If none of thresholds above usage is crossed, we read
3551 * only one element of the array here.
3552 */
3553 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3554 eventfd_signal(t->entries[i].eventfd, 1);
3555
3556 /* Update current_threshold */
3557 t->current_threshold = i - 1;
3558unlock:
3559 rcu_read_unlock();
3560}
3561
3562static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3563{
3564 while (memcg) {
3565 __mem_cgroup_threshold(memcg, false);
3566 if (do_memsw_account())
3567 __mem_cgroup_threshold(memcg, true);
3568
3569 memcg = parent_mem_cgroup(memcg);
3570 }
3571}
3572
3573static int compare_thresholds(const void *a, const void *b)
3574{
3575 const struct mem_cgroup_threshold *_a = a;
3576 const struct mem_cgroup_threshold *_b = b;
3577
3578 if (_a->threshold > _b->threshold)
3579 return 1;
3580
3581 if (_a->threshold < _b->threshold)
3582 return -1;
3583
3584 return 0;
3585}
3586
3587static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
3588{
3589 struct mem_cgroup_eventfd_list *ev;
3590
3591 spin_lock(&memcg_oom_lock);
3592
3593 list_for_each_entry(ev, &memcg->oom_notify, list)
3594 eventfd_signal(ev->eventfd, 1);
3595
3596 spin_unlock(&memcg_oom_lock);
3597 return 0;
3598}
3599
3600static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
3601{
3602 struct mem_cgroup *iter;
3603
3604 for_each_mem_cgroup_tree(iter, memcg)
3605 mem_cgroup_oom_notify_cb(iter);
3606}
3607
3608static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3609 struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3610{
3611 struct mem_cgroup_thresholds *thresholds;
3612 struct mem_cgroup_threshold_ary *new;
3613 unsigned long threshold;
3614 unsigned long usage;
3615 int i, size, ret;
3616
3617 ret = page_counter_memparse(args, "-1", &threshold);
3618 if (ret)
3619 return ret;
3620
3621 mutex_lock(&memcg->thresholds_lock);
3622
3623 if (type == _MEM) {
3624 thresholds = &memcg->thresholds;
3625 usage = mem_cgroup_usage(memcg, false);
3626 } else if (type == _MEMSWAP) {
3627 thresholds = &memcg->memsw_thresholds;
3628 usage = mem_cgroup_usage(memcg, true);
3629 } else
3630 BUG();
3631
3632 /* Check if a threshold crossed before adding a new one */
3633 if (thresholds->primary)
3634 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3635
3636 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3637
3638 /* Allocate memory for new array of thresholds */
3639 new = kmalloc(struct_size(new, entries, size), GFP_KERNEL);
3640 if (!new) {
3641 ret = -ENOMEM;
3642 goto unlock;
3643 }
3644 new->size = size;
3645
3646 /* Copy thresholds (if any) to new array */
3647 if (thresholds->primary) {
3648 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3649 sizeof(struct mem_cgroup_threshold));
3650 }
3651
3652 /* Add new threshold */
3653 new->entries[size - 1].eventfd = eventfd;
3654 new->entries[size - 1].threshold = threshold;
3655
3656 /* Sort thresholds. Registering of new threshold isn't time-critical */
3657 sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3658 compare_thresholds, NULL);
3659
3660 /* Find current threshold */
3661 new->current_threshold = -1;
3662 for (i = 0; i < size; i++) {
3663 if (new->entries[i].threshold <= usage) {
3664 /*
3665 * new->current_threshold will not be used until
3666 * rcu_assign_pointer(), so it's safe to increment
3667 * it here.
3668 */
3669 ++new->current_threshold;
3670 } else
3671 break;
3672 }
3673
3674 /* Free old spare buffer and save old primary buffer as spare */
3675 kfree(thresholds->spare);
3676 thresholds->spare = thresholds->primary;
3677
3678 rcu_assign_pointer(thresholds->primary, new);
3679
3680 /* To be sure that nobody uses thresholds */
3681 synchronize_rcu();
3682
3683unlock:
3684 mutex_unlock(&memcg->thresholds_lock);
3685
3686 return ret;
3687}
3688
3689static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3690 struct eventfd_ctx *eventfd, const char *args)
3691{
3692 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
3693}
3694
3695static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
3696 struct eventfd_ctx *eventfd, const char *args)
3697{
3698 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
3699}
3700
3701static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3702 struct eventfd_ctx *eventfd, enum res_type type)
3703{
3704 struct mem_cgroup_thresholds *thresholds;
3705 struct mem_cgroup_threshold_ary *new;
3706 unsigned long usage;
3707 int i, j, size;
3708
3709 mutex_lock(&memcg->thresholds_lock);
3710
3711 if (type == _MEM) {
3712 thresholds = &memcg->thresholds;
3713 usage = mem_cgroup_usage(memcg, false);
3714 } else if (type == _MEMSWAP) {
3715 thresholds = &memcg->memsw_thresholds;
3716 usage = mem_cgroup_usage(memcg, true);
3717 } else
3718 BUG();
3719
3720 if (!thresholds->primary)
3721 goto unlock;
3722
3723 /* Check if a threshold crossed before removing */
3724 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3725
3726 /* Calculate new number of threshold */
3727 size = 0;
3728 for (i = 0; i < thresholds->primary->size; i++) {
3729 if (thresholds->primary->entries[i].eventfd != eventfd)
3730 size++;
3731 }
3732
3733 new = thresholds->spare;
3734
3735 /* Set thresholds array to NULL if we don't have thresholds */
3736 if (!size) {
3737 kfree(new);
3738 new = NULL;
3739 goto swap_buffers;
3740 }
3741
3742 new->size = size;
3743
3744 /* Copy thresholds and find current threshold */
3745 new->current_threshold = -1;
3746 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3747 if (thresholds->primary->entries[i].eventfd == eventfd)
3748 continue;
3749
3750 new->entries[j] = thresholds->primary->entries[i];
3751 if (new->entries[j].threshold <= usage) {
3752 /*
3753 * new->current_threshold will not be used
3754 * until rcu_assign_pointer(), so it's safe to increment
3755 * it here.
3756 */
3757 ++new->current_threshold;
3758 }
3759 j++;
3760 }
3761
3762swap_buffers:
3763 /* Swap primary and spare array */
3764 thresholds->spare = thresholds->primary;
3765
3766 rcu_assign_pointer(thresholds->primary, new);
3767
3768 /* To be sure that nobody uses thresholds */
3769 synchronize_rcu();
3770
3771 /* If all events are unregistered, free the spare array */
3772 if (!new) {
3773 kfree(thresholds->spare);
3774 thresholds->spare = NULL;
3775 }
3776unlock:
3777 mutex_unlock(&memcg->thresholds_lock);
3778}
3779
3780static void mem_cgroup_usage_unregister_event(struct mem_cgroup *