1/*
2 * mm/rmap.c - physical to virtual reverse mappings
3 *
4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br>
5 * Released under the General Public License (GPL).
6 *
7 * Simple, low overhead reverse mapping scheme.
8 * Please try to keep this thing as modular as possible.
9 *
10 * Provides methods for unmapping each kind of mapped page:
11 * the anon methods track anonymous pages, and
12 * the file methods track pages belonging to an inode.
13 *
14 * Original design by Rik van Riel <riel@conectiva.com.br> 2001
15 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
16 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
17 * Contributions by Hugh Dickins 2003, 2004
18 */
19
20/*
21 * Lock ordering in mm:
22 *
23 * inode->i_mutex (while writing or truncating, not reading or faulting)
24 * mm->mmap_sem
25 * page->flags PG_locked (lock_page)
26 * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share)
27 * mapping->i_mmap_rwsem
28 * anon_vma->rwsem
29 * mm->page_table_lock or pte_lock
30 * pgdat->lru_lock (in mark_page_accessed, isolate_lru_page)
31 * swap_lock (in swap_duplicate, swap_info_get)
32 * mmlist_lock (in mmput, drain_mmlist and others)
33 * mapping->private_lock (in __set_page_dirty_buffers)
34 * mem_cgroup_{begin,end}_page_stat (memcg->move_lock)
35 * i_pages lock (widely used)
36 * inode->i_lock (in set_page_dirty's __mark_inode_dirty)
37 * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
38 * sb_lock (within inode_lock in fs/fs-writeback.c)
39 * i_pages lock (widely used, in set_page_dirty,
40 * in arch-dependent flush_dcache_mmap_lock,
41 * within bdi.wb->list_lock in __sync_single_inode)
42 *
43 * anon_vma->rwsem,mapping->i_mutex (memory_failure, collect_procs_anon)
44 * ->tasklist_lock
45 * pte map lock
46 */
47
48#include <linux/mm.h>
49#include <linux/sched/mm.h>
50#include <linux/sched/task.h>
51#include <linux/pagemap.h>
52#include <linux/swap.h>
53#include <linux/swapops.h>
54#include <linux/slab.h>
55#include <linux/init.h>
56#include <linux/ksm.h>
57#include <linux/rmap.h>
58#include <linux/rcupdate.h>
59#include <linux/export.h>
60#include <linux/memcontrol.h>
61#include <linux/mmu_notifier.h>
62#include <linux/migrate.h>
63#include <linux/hugetlb.h>
64#include <linux/backing-dev.h>
65#include <linux/page_idle.h>
66#include <linux/memremap.h>
67#include <linux/userfaultfd_k.h>
68
69#include <asm/tlbflush.h>
70
71#include <trace/events/tlb.h>
72
73#include "internal.h"
74
75static struct kmem_cache *anon_vma_cachep;
76static struct kmem_cache *anon_vma_chain_cachep;
77
78static inline struct anon_vma *anon_vma_alloc(void)
79{
80 struct anon_vma *anon_vma;
81
82 anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
83 if (anon_vma) {
84 atomic_set(&anon_vma->refcount, 1);
85 anon_vma->degree = 1; /* Reference for first vma */
86 anon_vma->parent = anon_vma;
87 /*
88 * Initialise the anon_vma root to point to itself. If called
89 * from fork, the root will be reset to the parents anon_vma.
90 */
91 anon_vma->root = anon_vma;
92 }
93
94 return anon_vma;
95}
96
97static inline void anon_vma_free(struct anon_vma *anon_vma)
98{
99 VM_BUG_ON(atomic_read(&anon_vma->refcount));
100
101 /*
102 * Synchronize against page_lock_anon_vma_read() such that
103 * we can safely hold the lock without the anon_vma getting
104 * freed.
105 *
106 * Relies on the full mb implied by the atomic_dec_and_test() from
107 * put_anon_vma() against the acquire barrier implied by
108 * down_read_trylock() from page_lock_anon_vma_read(). This orders:
109 *
110 * page_lock_anon_vma_read() VS put_anon_vma()
111 * down_read_trylock() atomic_dec_and_test()
112 * LOCK MB
113 * atomic_read() rwsem_is_locked()
114 *
115 * LOCK should suffice since the actual taking of the lock must
116 * happen _before_ what follows.
117 */
118 might_sleep();
119 if (rwsem_is_locked(&anon_vma->root->rwsem)) {
120 anon_vma_lock_write(anon_vma);
121 anon_vma_unlock_write(anon_vma);
122 }
123
124 kmem_cache_free(anon_vma_cachep, anon_vma);
125}
126
127static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp)
128{
129 return kmem_cache_alloc(anon_vma_chain_cachep, gfp);
130}
131
132static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
133{
134 kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
135}
136
137static void anon_vma_chain_link(struct vm_area_struct *vma,
138 struct anon_vma_chain *avc,
139 struct anon_vma *anon_vma)
140{
141 avc->vma = vma;
142 avc->anon_vma = anon_vma;
143 list_add(&avc->same_vma, &vma->anon_vma_chain);
144 anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
145}
146
147/**
148 * __anon_vma_prepare - attach an anon_vma to a memory region
149 * @vma: the memory region in question
150 *
151 * This makes sure the memory mapping described by 'vma' has
152 * an 'anon_vma' attached to it, so that we can associate the
153 * anonymous pages mapped into it with that anon_vma.
154 *
155 * The common case will be that we already have one, which
156 * is handled inline by anon_vma_prepare(). But if
157 * not we either need to find an adjacent mapping that we
158 * can re-use the anon_vma from (very common when the only
159 * reason for splitting a vma has been mprotect()), or we
160 * allocate a new one.
161 *
162 * Anon-vma allocations are very subtle, because we may have
163 * optimistically looked up an anon_vma in page_lock_anon_vma_read()
164 * and that may actually touch the spinlock even in the newly
165 * allocated vma (it depends on RCU to make sure that the
166 * anon_vma isn't actually destroyed).
167 *
168 * As a result, we need to do proper anon_vma locking even
169 * for the new allocation. At the same time, we do not want
170 * to do any locking for the common case of already having
171 * an anon_vma.
172 *
173 * This must be called with the mmap_sem held for reading.
174 */
175int __anon_vma_prepare(struct vm_area_struct *vma)
176{
177 struct mm_struct *mm = vma->vm_mm;
178 struct anon_vma *anon_vma, *allocated;
179 struct anon_vma_chain *avc;
180
181 might_sleep();
182
183 avc = anon_vma_chain_alloc(GFP_KERNEL);
184 if (!avc)
185 goto out_enomem;
186
187 anon_vma = find_mergeable_anon_vma(vma);
188 allocated = NULL;
189 if (!anon_vma) {
190 anon_vma = anon_vma_alloc();
191 if (unlikely(!anon_vma))
192 goto out_enomem_free_avc;
193 allocated = anon_vma;
194 }
195
196 anon_vma_lock_write(anon_vma);
197 /* page_table_lock to protect against threads */
198 spin_lock(&mm->page_table_lock);
199 if (likely(!vma->anon_vma)) {
200 vma->anon_vma = anon_vma;
201 anon_vma_chain_link(vma, avc, anon_vma);
202 /* vma reference or self-parent link for new root */
203 anon_vma->degree++;
204 allocated = NULL;
205 avc = NULL;
206 }
207 spin_unlock(&mm->page_table_lock);
208 anon_vma_unlock_write(anon_vma);
209
210 if (unlikely(allocated))
211 put_anon_vma(allocated);
212 if (unlikely(avc))
213 anon_vma_chain_free(avc);
214
215 return 0;
216
217 out_enomem_free_avc:
218 anon_vma_chain_free(avc);
219 out_enomem:
220 return -ENOMEM;
221}
222
223/*
224 * This is a useful helper function for locking the anon_vma root as
225 * we traverse the vma->anon_vma_chain, looping over anon_vma's that
226 * have the same vma.
227 *
228 * Such anon_vma's should have the same root, so you'd expect to see
229 * just a single mutex_lock for the whole traversal.
230 */
231static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)
232{
233 struct anon_vma *new_root = anon_vma->root;
234 if (new_root != root) {
235 if (WARN_ON_ONCE(root))
236 up_write(&root->rwsem);
237 root = new_root;
238 down_write(&root->rwsem);
239 }
240 return root;
241}
242
243static inline void unlock_anon_vma_root(struct anon_vma *root)
244{
245 if (root)
246 up_write(&root->rwsem);
247}
248
249/*
250 * Attach the anon_vmas from src to dst.
251 * Returns 0 on success, -ENOMEM on failure.
252 *
253 * If dst->anon_vma is NULL this function tries to find and reuse existing
254 * anon_vma which has no vmas and only one child anon_vma. This prevents
255 * degradation of anon_vma hierarchy to endless linear chain in case of
256 * constantly forking task. On the other hand, an anon_vma with more than one
257 * child isn't reused even if there was no alive vma, thus rmap walker has a
258 * good chance of avoiding scanning the whole hierarchy when it searches where
259 * page is mapped.
260 */
261int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
262{
263 struct anon_vma_chain *avc, *pavc;
264 struct anon_vma *root = NULL;
265
266 list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
267 struct anon_vma *anon_vma;
268
269 avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN);
270 if (unlikely(!avc)) {
271 unlock_anon_vma_root(root);
272 root = NULL;
273 avc = anon_vma_chain_alloc(GFP_KERNEL);
274 if (!avc)
275 goto enomem_failure;
276 }
277 anon_vma = pavc->anon_vma;
278 root = lock_anon_vma_root(root, anon_vma);
279 anon_vma_chain_link(dst, avc, anon_vma);
280
281 /*
282 * Reuse existing anon_vma if its degree lower than two,
283 * that means it has no vma and only one anon_vma child.
284 *
285 * Do not chose parent anon_vma, otherwise first child
286 * will always reuse it. Root anon_vma is never reused:
287 * it has self-parent reference and at least one child.
288 */
289 if (!dst->anon_vma && anon_vma != src->anon_vma &&
290 anon_vma->degree < 2)
291 dst->anon_vma = anon_vma;
292 }
293 if (dst->anon_vma)
294 dst->anon_vma->degree++;
295 unlock_anon_vma_root(root);
296 return 0;
297
298 enomem_failure:
299 /*
300 * dst->anon_vma is dropped here otherwise its degree can be incorrectly
301 * decremented in unlink_anon_vmas().
302 * We can safely do this because callers of anon_vma_clone() don't care
303 * about dst->anon_vma if anon_vma_clone() failed.
304 */
305 dst->anon_vma = NULL;
306 unlink_anon_vmas(dst);
307 return -ENOMEM;
308}
309
310/*
311 * Attach vma to its own anon_vma, as well as to the anon_vmas that
312 * the corresponding VMA in the parent process is attached to.
313 * Returns 0 on success, non-zero on failure.
314 */
315int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
316{
317 struct anon_vma_chain *avc;
318 struct anon_vma *anon_vma;
319 int error;
320
321 /* Don't bother if the parent process has no anon_vma here. */
322 if (!pvma->anon_vma)
323 return 0;
324
325 /* Drop inherited anon_vma, we'll reuse existing or allocate new. */
326 vma->anon_vma = NULL;
327
328 /*
329 * First, attach the new VMA to the parent VMA's anon_vmas,
330 * so rmap can find non-COWed pages in child processes.
331 */
332 error = anon_vma_clone(vma, pvma);
333 if (error)
334 return error;
335
336 /* An existing anon_vma has been reused, all done then. */
337 if (vma->anon_vma)
338 return 0;
339
340 /* Then add our own anon_vma. */
341 anon_vma = anon_vma_alloc();
342 if (!anon_vma)
343 goto out_error;
344 avc = anon_vma_chain_alloc(GFP_KERNEL);
345 if (!avc)
346 goto out_error_free_anon_vma;
347
348 /*
349 * The root anon_vma's spinlock is the lock actually used when we
350 * lock any of the anon_vmas in this anon_vma tree.
351 */
352 anon_vma->root = pvma->anon_vma->root;
353 anon_vma->parent = pvma->anon_vma;
354 /*
355 * With refcounts, an anon_vma can stay around longer than the
356 * process it belongs to. The root anon_vma needs to be pinned until
357 * this anon_vma is freed, because the lock lives in the root.
358 */
359 get_anon_vma(anon_vma->root);
360 /* Mark this anon_vma as the one where our new (COWed) pages go. */
361 vma->anon_vma = anon_vma;
362 anon_vma_lock_write(anon_vma);
363 anon_vma_chain_link(vma, avc, anon_vma);
364 anon_vma->parent->degree++;
365 anon_vma_unlock_write(anon_vma);
366
367 return 0;
368
369 out_error_free_anon_vma:
370 put_anon_vma(anon_vma);
371 out_error:
372 unlink_anon_vmas(vma);
373 return -ENOMEM;
374}
375
376void unlink_anon_vmas(struct vm_area_struct *vma)
377{
378 struct anon_vma_chain *avc, *next;
379 struct anon_vma *root = NULL;
380
381 /*
382 * Unlink each anon_vma chained to the VMA. This list is ordered
383 * from newest to oldest, ensuring the root anon_vma gets freed last.
384 */
385 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
386 struct anon_vma *anon_vma = avc->anon_vma;
387
388 root = lock_anon_vma_root(root, anon_vma);
389 anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
390
391 /*
392 * Leave empty anon_vmas on the list - we'll need
393 * to free them outside the lock.
394 */
395 if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) {
396 anon_vma->parent->degree--;
397 continue;
398 }
399
400 list_del(&avc->same_vma);
401 anon_vma_chain_free(avc);
402 }
403 if (vma->anon_vma)
404 vma->anon_vma->degree--;
405 unlock_anon_vma_root(root);
406
407 /*
408 * Iterate the list once more, it now only contains empty and unlinked
409 * anon_vmas, destroy them. Could not do before due to __put_anon_vma()
410 * needing to write-acquire the anon_vma->root->rwsem.
411 */
412 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
413 struct anon_vma *anon_vma = avc->anon_vma;
414
415 VM_WARN_ON(anon_vma->degree);
416 put_anon_vma(anon_vma);
417
418 list_del(&avc->same_vma);
419 anon_vma_chain_free(avc);
420 }
421}
422
423static void anon_vma_ctor(void *data)
424{
425 struct anon_vma *anon_vma = data;
426
427 init_rwsem(&anon_vma->rwsem);
428 atomic_set(&anon_vma->refcount, 0);
429 anon_vma->rb_root = RB_ROOT_CACHED;
430}
431
432void __init anon_vma_init(void)
433{
434 anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
435 0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT,
436 anon_vma_ctor);
437 anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain,
438 SLAB_PANIC|SLAB_ACCOUNT);
439}
440
441/*
442 * Getting a lock on a stable anon_vma from a page off the LRU is tricky!
443 *
444 * Since there is no serialization what so ever against page_remove_rmap()
445 * the best this function can do is return a locked anon_vma that might
446 * have been relevant to this page.
447 *
448 * The page might have been remapped to a different anon_vma or the anon_vma
449 * returned may already be freed (and even reused).
450 *
451 * In case it was remapped to a different anon_vma, the new anon_vma will be a
452 * child of the old anon_vma, and the anon_vma lifetime rules will therefore
453 * ensure that any anon_vma obtained from the page will still be valid for as
454 * long as we observe page_mapped() [ hence all those page_mapped() tests ].
455 *
456 * All users of this function must be very careful when walking the anon_vma
457 * chain and verify that the page in question is indeed mapped in it
458 * [ something equivalent to page_mapped_in_vma() ].
459 *
460 * Since anon_vma's slab is DESTROY_BY_RCU and we know from page_remove_rmap()
461 * that the anon_vma pointer from page->mapping is valid if there is a
462 * mapcount, we can dereference the anon_vma after observing those.
463 */
464struct anon_vma *page_get_anon_vma(struct page *page)
465{
466 struct anon_vma *anon_vma = NULL;
467 unsigned long anon_mapping;
468
469 rcu_read_lock();
470 anon_mapping = (unsigned long)READ_ONCE(page->mapping);
471 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
472 goto out;
473 if (!page_mapped(page))
474 goto out;
475
476 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
477 if (!atomic_inc_not_zero(&anon_vma->refcount)) {
478 anon_vma = NULL;
479 goto out;
480 }
481
482 /*
483 * If this page is still mapped, then its anon_vma cannot have been
484 * freed. But if it has been unmapped, we have no security against the
485 * anon_vma structure being freed and reused (for another anon_vma:
486 * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero()
487 * above cannot corrupt).
488 */
489 if (!page_mapped(page)) {
490 rcu_read_unlock();
491 put_anon_vma(anon_vma);
492 return NULL;
493 }
494out:
495 rcu_read_unlock();
496
497 return anon_vma;
498}
499
500/*
501 * Similar to page_get_anon_vma() except it locks the anon_vma.
502 *
503 * Its a little more complex as it tries to keep the fast path to a single
504 * atomic op -- the trylock. If we fail the trylock, we fall back to getting a
505 * reference like with page_get_anon_vma() and then block on the mutex.
506 */
507struct anon_vma *page_lock_anon_vma_read(struct page *page)
508{
509 struct anon_vma *anon_vma = NULL;
510 struct anon_vma *root_anon_vma;
511 unsigned long anon_mapping;
512
513 rcu_read_lock();
514 anon_mapping = (unsigned long)READ_ONCE(page->mapping);
515 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
516 goto out;
517 if (!page_mapped(page))
518 goto out;
519
520 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
521 root_anon_vma = READ_ONCE(anon_vma->root);
522 if (down_read_trylock(&root_anon_vma->rwsem)) {
523 /*
524 * If the page is still mapped, then this anon_vma is still
525 * its anon_vma, and holding the mutex ensures that it will
526 * not go away, see anon_vma_free().
527 */
528 if (!page_mapped(page)) {
529 up_read(&root_anon_vma->rwsem);
530 anon_vma = NULL;
531 }
532 goto out;
533 }
534
535 /* trylock failed, we got to sleep */
536 if (!atomic_inc_not_zero(&anon_vma->refcount)) {
537 anon_vma = NULL;
538 goto out;
539 }
540
541 if (!page_mapped(page)) {
542 rcu_read_unlock();
543 put_anon_vma(anon_vma);
544 return NULL;
545 }
546
547 /* we pinned the anon_vma, its safe to sleep */
548 rcu_read_unlock();
549 anon_vma_lock_read(anon_vma);
550
551 if (atomic_dec_and_test(&anon_vma->refcount)) {
552 /*
553 * Oops, we held the last refcount, release the lock
554 * and bail -- can't simply use put_anon_vma() because
555 * we'll deadlock on the anon_vma_lock_write() recursion.
556 */
557 anon_vma_unlock_read(anon_vma);
558 __put_anon_vma(anon_vma);
559 anon_vma = NULL;
560 }
561
562 return anon_vma;
563
564out:
565 rcu_read_unlock();
566 return anon_vma;
567}
568
569void page_unlock_anon_vma_read(struct anon_vma *anon_vma)
570{
571 anon_vma_unlock_read(anon_vma);
572}
573
574#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
575/*
576 * Flush TLB entries for recently unmapped pages from remote CPUs. It is
577 * important if a PTE was dirty when it was unmapped that it's flushed
578 * before any IO is initiated on the page to prevent lost writes. Similarly,
579 * it must be flushed before freeing to prevent data leakage.
580 */
581void try_to_unmap_flush(void)
582{
583 struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
584
585 if (!tlb_ubc->flush_required)
586 return;
587
588 arch_tlbbatch_flush(&tlb_ubc->arch);
589 tlb_ubc->flush_required = false;
590 tlb_ubc->writable = false;
591}
592
593/* Flush iff there are potentially writable TLB entries that can race with IO */
594void try_to_unmap_flush_dirty(void)
595{
596 struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
597
598 if (tlb_ubc->writable)
599 try_to_unmap_flush();
600}
601
602static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable)
603{
604 struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
605
606 arch_tlbbatch_add_mm(&tlb_ubc->arch, mm);
607 tlb_ubc->flush_required = true;
608
609 /*
610 * Ensure compiler does not re-order the setting of tlb_flush_batched
611 * before the PTE is cleared.
612 */
613 barrier();
614 mm->tlb_flush_batched = true;
615
616 /*
617 * If the PTE was dirty then it's best to assume it's writable. The
618 * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush()
619 * before the page is queued for IO.
620 */
621 if (writable)
622 tlb_ubc->writable = true;
623}
624
625/*
626 * Returns true if the TLB flush should be deferred to the end of a batch of
627 * unmap operations to reduce IPIs.
628 */
629static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
630{
631 bool should_defer = false;
632
633 if (!(flags & TTU_BATCH_FLUSH))
634 return false;
635
636 /* If remote CPUs need to be flushed then defer batch the flush */
637 if (cpumask_any_but(mm_cpumask(mm), get_cpu()) < nr_cpu_ids)
638 should_defer = true;
639 put_cpu();
640
641 return should_defer;
642}
643
644/*
645 * Reclaim unmaps pages under the PTL but do not flush the TLB prior to
646 * releasing the PTL if TLB flushes are batched. It's possible for a parallel
647 * operation such as mprotect or munmap to race between reclaim unmapping
648 * the page and flushing the page. If this race occurs, it potentially allows
649 * access to data via a stale TLB entry. Tracking all mm's that have TLB
650 * batching in flight would be expensive during reclaim so instead track
651 * whether TLB batching occurred in the past and if so then do a flush here
652 * if required. This will cost one additional flush per reclaim cycle paid
653 * by the first operation at risk such as mprotect and mumap.
654 *
655 * This must be called under the PTL so that an access to tlb_flush_batched
656 * that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise
657 * via the PTL.
658 */
659void flush_tlb_batched_pending(struct mm_struct *mm)
660{
661 if (mm->tlb_flush_batched) {
662 flush_tlb_mm(mm);
663
664 /*
665 * Do not allow the compiler to re-order the clearing of
666 * tlb_flush_batched before the tlb is flushed.
667 */
668 barrier();
669 mm->tlb_flush_batched = false;
670 }
671}
672#else
673static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable)
674{
675}
676
677static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
678{
679 return false;
680}
681#endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
682
683/*
684 * At what user virtual address is page expected in vma?
685 * Caller should check the page is actually part of the vma.
686 */
687unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
688{
689 unsigned long address;
690 if (PageAnon(page)) {
691 struct anon_vma *page__anon_vma = page_anon_vma(page);
692 /*
693 * Note: swapoff's unuse_vma() is more efficient with this
694 * check, and needs it to match anon_vma when KSM is active.
695 */
696 if (!vma->anon_vma || !page__anon_vma ||
697 vma->anon_vma->root != page__anon_vma->root)
698 return -EFAULT;
699 } else if (page->mapping) {
700 if (!vma->vm_file || vma->vm_file->f_mapping != page->mapping)
701 return -EFAULT;
702 } else
703 return -EFAULT;
704 address = __vma_address(page, vma);
705 if (unlikely(address < vma->vm_start || address >= vma->vm_end))
706 return -EFAULT;
707 return address;
708}
709
710pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
711{
712 pgd_t *pgd;
713 p4d_t *p4d;
714 pud_t *pud;
715 pmd_t *pmd = NULL;
716 pmd_t pmde;
717
718 pgd = pgd_offset(mm, address);
719 if (!pgd_present(*pgd))
720 goto out;
721
722 p4d = p4d_offset(pgd, address);
723 if (!p4d_present(*p4d))
724 goto out;
725
726 pud = pud_offset(p4d, address);
727 if (!pud_present(*pud))
728 goto out;
729
730 pmd = pmd_offset(pud, address);
731 /*
732 * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at()
733 * without holding anon_vma lock for write. So when looking for a
734 * genuine pmde (in which to find pte), test present and !THP together.
735 */
736 pmde = *pmd;
737 barrier();
738 if (!pmd_present(pmde) || pmd_trans_huge(pmde))
739 pmd = NULL;
740out:
741 return pmd;
742}
743
744struct page_referenced_arg {
745 int mapcount;
746 int referenced;
747 unsigned long vm_flags;
748 struct mem_cgroup *memcg;
749};
750/*
751 * arg: page_referenced_arg will be passed
752 */
753static bool page_referenced_one(struct page *page, struct vm_area_struct *vma,
754 unsigned long address, void *arg)
755{
756 struct page_referenced_arg *pra = arg;
757 struct page_vma_mapped_walk pvmw = {
758 .page = page,
759 .vma = vma,
760 .address = address,
761 };
762 int referenced = 0;
763
764 while (page_vma_mapped_walk(&pvmw)) {
765 address = pvmw.address;
766
767 if (vma->vm_flags & VM_LOCKED) {
768 page_vma_mapped_walk_done(&pvmw);
769 pra->vm_flags |= VM_LOCKED;
770 return false; /* To break the loop */
771 }
772
773 if (pvmw.pte) {
774 if (ptep_clear_flush_young_notify(vma, address,
775 pvmw.pte)) {
776 /*
777 * Don't treat a reference through
778 * a sequentially read mapping as such.
779 * If the page has been used in another mapping,
780 * we will catch it; if this other mapping is
781 * already gone, the unmap path will have set
782 * PG_referenced or activated the page.
783 */
784 if (likely(!(vma->vm_flags & VM_SEQ_READ)))
785 referenced++;
786 }
787 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
788 if (pmdp_clear_flush_young_notify(vma, address,
789 pvmw.pmd))
790 referenced++;
791 } else {
792 /* unexpected pmd-mapped page? */
793 WARN_ON_ONCE(1);
794 }
795
796 pra->mapcount--;
797 }
798
799 if (referenced)
800 clear_page_idle(page);
801 if (test_and_clear_page_young(page))
802 referenced++;
803
804 if (referenced) {
805 pra->referenced++;
806 pra->vm_flags |= vma->vm_flags;
807 }
808
809 if (!pra->mapcount)
810 return false; /* To break the loop */
811
812 return true;
813}
814
815static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg)
816{
817 struct page_referenced_arg *pra = arg;
818 struct mem_cgroup *memcg = pra->memcg;
819
820 if (!mm_match_cgroup(vma->vm_mm, memcg))
821 return true;
822
823 return false;
824}
825
826/**
827 * page_referenced - test if the page was referenced
828 * @page: the page to test
829 * @is_locked: caller holds lock on the page
830 * @memcg: target memory cgroup
831 * @vm_flags: collect encountered vma->vm_flags who actually referenced the page
832 *
833 * Quick test_and_clear_referenced for all mappings to a page,
834 * returns the number of ptes which referenced the page.
835 */
836int page_referenced(struct page *page,
837 int is_locked,
838 struct mem_cgroup *memcg,
839 unsigned long *vm_flags)
840{
841 int we_locked = 0;
842 struct page_referenced_arg pra = {
843 .mapcount = total_mapcount(page),
844 .memcg = memcg,
845 };
846 struct rmap_walk_control rwc = {
847 .rmap_one = page_referenced_one,
848 .arg = (void *)&pra,
849 .anon_lock = page_lock_anon_vma_read,
850 };
851
852 *vm_flags = 0;
853 if (!page_mapped(page))
854 return 0;
855
856 if (!page_rmapping(page))
857 return 0;
858
859 if (!is_locked && (!PageAnon(page) || PageKsm(page))) {
860 we_locked = trylock_page(page);
861 if (!we_locked)
862 return 1;
863 }
864
865 /*
866 * If we are reclaiming on behalf of a cgroup, skip
867 * counting on behalf of references from different
868 * cgroups
869 */
870 if (memcg) {
871 rwc.invalid_vma = invalid_page_referenced_vma;
872 }
873
874 rmap_walk(page, &rwc);
875 *vm_flags = pra.vm_flags;
876
877 if (we_locked)
878 unlock_page(page);
879
880 return pra.referenced;
881}
882
883static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
884 unsigned long address, void *arg)
885{
886 struct page_vma_mapped_walk pvmw = {
887 .page = page,
888 .vma = vma,
889 .address = address,
890 .flags = PVMW_SYNC,
891 };
892 struct mmu_notifier_range range;
893 int *cleaned = arg;
894
895 /*
896 * We have to assume the worse case ie pmd for invalidation. Note that
897 * the page can not be free from this function.
898 */
899 mmu_notifier_range_init(&range, vma->vm_mm, address,
900 min(vma->vm_end, address +
901 (PAGE_SIZE << compound_order(page))));
902 mmu_notifier_invalidate_range_start(&range);
903
904 while (page_vma_mapped_walk(&pvmw)) {
905 unsigned long cstart;
906 int ret = 0;
907
908 cstart = address = pvmw.address;
909 if (pvmw.pte) {
910 pte_t entry;
911 pte_t *pte = pvmw.pte;
912
913 if (!pte_dirty(*pte) && !pte_write(*pte))
914 continue;
915
916 flush_cache_page(vma, address, pte_pfn(*pte));
917 entry = ptep_clear_flush(vma, address, pte);
918 entry = pte_wrprotect(entry);
919 entry = pte_mkclean(entry);
920 set_pte_at(vma->vm_mm, address, pte, entry);
921 ret = 1;
922 } else {
923#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
924 pmd_t *pmd = pvmw.pmd;
925 pmd_t entry;
926
927 if (!pmd_dirty(*pmd) && !pmd_write(*pmd))
928 continue;
929
930 flush_cache_page(vma, address, page_to_pfn(page));
931 entry = pmdp_huge_clear_flush(vma, address, pmd);
932 entry = pmd_wrprotect(entry);
933 entry = pmd_mkclean(entry);
934 set_pmd_at(vma->vm_mm, address, pmd, entry);
935 cstart &= PMD_MASK;
936 ret = 1;
937#else
938 /* unexpected pmd-mapped page? */
939 WARN_ON_ONCE(1);
940#endif
941 }
942
943 /*
944 * No need to call mmu_notifier_invalidate_range() as we are
945 * downgrading page table protection not changing it to point
946 * to a new page.
947 *
948 * See Documentation/vm/mmu_notifier.rst
949 */
950 if (ret)
951 (*cleaned)++;
952 }
953
954 mmu_notifier_invalidate_range_end(&range);
955
956 return true;
957}
958
959static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)
960{
961 if (vma->vm_flags & VM_SHARED)
962 return false;
963
964 return true;
965}
966
967int page_mkclean(struct page *page)
968{
969 int cleaned = 0;
970 struct address_space *mapping;
971 struct rmap_walk_control rwc = {
972 .arg = (void *)&cleaned,
973 .rmap_one = page_mkclean_one,
974 .invalid_vma = invalid_mkclean_vma,
975 };
976
977 BUG_ON(!PageLocked(page));
978
979 if (!page_mapped(page))
980 return 0;
981
982 mapping = page_mapping(page);
983 if (!mapping)
984 return 0;
985
986 rmap_walk(page, &rwc);
987
988 return cleaned;
989}
990EXPORT_SYMBOL_GPL(page_mkclean);
991
992/**
993 * page_move_anon_rmap - move a page to our anon_vma
994 * @page: the page to move to our anon_vma
995 * @vma: the vma the page belongs to
996 *
997 * When a page belongs exclusively to one process after a COW event,
998 * that page can be moved into the anon_vma that belongs to just that
999 * process, so the rmap code will not search the parent or sibling
1000 * processes.
1001 */
1002void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma)
1003{
1004 struct anon_vma *anon_vma = vma->anon_vma;
1005
1006 page = compound_head(page);
1007
1008 VM_BUG_ON_PAGE(!PageLocked(page), page);
1009 VM_BUG_ON_VMA(!anon_vma, vma);
1010
1011 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
1012 /*
1013 * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written
1014 * simultaneously, so a concurrent reader (eg page_referenced()'s
1015 * PageAnon()) will not see one without the other.
1016 */
1017 WRITE_ONCE(page->mapping, (struct address_space *) anon_vma);
1018}
1019
1020/**
1021 * __page_set_anon_rmap - set up new anonymous rmap
1022 * @page: Page or Hugepage to add to rmap
1023 * @vma: VM area to add page to.
1024 * @address: User virtual address of the mapping
1025 * @exclusive: the page is exclusively owned by the current process
1026 */
1027static void __page_set_anon_rmap(struct page *page,
1028 struct vm_area_struct *vma, unsigned long address, int exclusive)
1029{
1030 struct anon_vma *anon_vma = vma->anon_vma;
1031
1032 BUG_ON(!anon_vma);
1033
1034 if (PageAnon(page))
1035 return;
1036
1037 /*
1038 * If the page isn't exclusively mapped into this vma,
1039 * we must use the _oldest_ possible anon_vma for the
1040 * page mapping!
1041 */
1042 if (!exclusive)
1043 anon_vma = anon_vma->root;
1044
1045 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
1046 page->mapping = (struct address_space *) anon_vma;
1047 page->index = linear_page_index(vma, address);
1048}
1049
1050/**
1051 * __page_check_anon_rmap - sanity check anonymous rmap addition
1052 * @page: the page to add the mapping to
1053 * @vma: the vm area in which the mapping is added
1054 * @address: the user virtual address mapped
1055 */
1056static void __page_check_anon_rmap(struct page *page,
1057 struct vm_area_struct *vma, unsigned long address)
1058{
1059#ifdef CONFIG_DEBUG_VM
1060 /*
1061 * The page's anon-rmap details (mapping and index) are guaranteed to
1062 * be set up correctly at this point.
1063 *
1064 * We have exclusion against page_add_anon_rmap because the caller
1065 * always holds the page locked, except if called from page_dup_rmap,
1066 * in which case the page is already known to be setup.
1067 *
1068 * We have exclusion against page_add_new_anon_rmap because those pages
1069 * are initially only visible via the pagetables, and the pte is locked
1070 * over the call to page_add_new_anon_rmap.
1071 */
1072 BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root);
1073 BUG_ON(page_to_pgoff(page) != linear_page_index(vma, address));
1074#endif
1075}
1076
1077/**
1078 * page_add_anon_rmap - add pte mapping to an anonymous page
1079 * @page: the page to add the mapping to
1080 * @vma: the vm area in which the mapping is added
1081 * @address: the user virtual address mapped
1082 * @compound: charge the page as compound or small page
1083 *
1084 * The caller needs to hold the pte lock, and the page must be locked in
1085 * the anon_vma case: to serialize mapping,index checking after setting,
1086 * and to ensure that PageAnon is not being upgraded racily to PageKsm
1087 * (but PageKsm is never downgraded to PageAnon).
1088 */
1089void page_add_anon_rmap(struct page *page,
1090 struct vm_area_struct *vma, unsigned long address, bool compound)
1091{
1092 do_page_add_anon_rmap(page, vma, address, compound ? RMAP_COMPOUND : 0);
1093}
1094
1095/*
1096 * Special version of the above for do_swap_page, which often runs
1097 * into pages that are exclusively owned by the current process.
1098 * Everybody else should continue to use page_add_anon_rmap above.
1099 */
1100void do_page_add_anon_rmap(struct page *page,
1101 struct vm_area_struct *vma, unsigned long address, int flags)
1102{
1103 bool compound = flags & RMAP_COMPOUND;
1104 bool first;
1105
1106 if (compound) {
1107 atomic_t *mapcount;
1108 VM_BUG_ON_PAGE(!PageLocked(page), page);
1109 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
1110 mapcount = compound_mapcount_ptr(page);
1111 first = atomic_inc_and_test(mapcount);
1112 } else {
1113 first = atomic_inc_and_test(&page->_mapcount);
1114 }
1115
1116 if (first) {
1117 int nr = compound ? hpage_nr_pages(page) : 1;
1118 /*
1119 * We use the irq-unsafe __{inc|mod}_zone_page_stat because
1120 * these counters are not modified in interrupt context, and
1121 * pte lock(a spinlock) is held, which implies preemption
1122 * disabled.
1123 */
1124 if (compound)
1125 __inc_node_page_state(page, NR_ANON_THPS);
1126 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr);
1127 }
1128 if (unlikely(PageKsm(page)))
1129 return;
1130
1131 VM_BUG_ON_PAGE(!PageLocked(page), page);
1132
1133 /* address might be in next vma when migration races vma_adjust */
1134 if (first)
1135 __page_set_anon_rmap(page, vma, address,
1136 flags & RMAP_EXCLUSIVE);
1137 else
1138 __page_check_anon_rmap(page, vma, address);
1139}
1140
1141/**
1142 * page_add_new_anon_rmap - add pte mapping to a new anonymous page
1143 * @page: the page to add the mapping to
1144 * @vma: the vm area in which the mapping is added
1145 * @address: the user virtual address mapped
1146 * @compound: charge the page as compound or small page
1147 *
1148 * Same as page_add_anon_rmap but must only be called on *new* pages.
1149 * This means the inc-and-test can be bypassed.
1150 * Page does not have to be locked.
1151 */
1152void page_add_new_anon_rmap(struct page *page,
1153 struct vm_area_struct *vma, unsigned long address, bool compound)
1154{
1155 int nr = compound ? hpage_nr_pages(page) : 1;
1156
1157 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
1158 __SetPageSwapBacked(page);
1159 if (compound) {
1160 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
1161 /* increment count (starts at -1) */
1162 atomic_set(compound_mapcount_ptr(page), 0);
1163 __inc_node_page_state(page, NR_ANON_THPS);
1164 } else {
1165 /* Anon THP always mapped first with PMD */
1166 VM_BUG_ON_PAGE(PageTransCompound(page), page);
1167 /* increment count (starts at -1) */
1168 atomic_set(&page->_mapcount, 0);
1169 }
1170 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr);
1171 __page_set_anon_rmap(page, vma, address, 1);
1172}
1173
1174/**
1175 * page_add_file_rmap - add pte mapping to a file page
1176 * @page: the page to add the mapping to
1177 * @compound: charge the page as compound or small page
1178 *
1179 * The caller needs to hold the pte lock.
1180 */
1181void page_add_file_rmap(struct page *page, bool compound)
1182{
1183 int i, nr = 1;
1184
1185 VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);
1186 lock_page_memcg(page);
1187 if (compound && PageTransHuge(page)) {
1188 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
1189 if (atomic_inc_and_test(&page[i]._mapcount))
1190 nr++;
1191 }
1192 if (!atomic_inc_and_test(compound_mapcount_ptr(page)))
1193 goto out;
1194 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
1195 __inc_node_page_state(page, NR_SHMEM_PMDMAPPED);
1196 } else {
1197 if (PageTransCompound(page) && page_mapping(page)) {
1198 VM_WARN_ON_ONCE(!PageLocked(page));
1199
1200 SetPageDoubleMap(compound_head(page));
1201 if (PageMlocked(page))
1202 clear_page_mlock(compound_head(page));
1203 }
1204 if (!atomic_inc_and_test(&page->_mapcount))
1205 goto out;
1206 }
1207 __mod_lruvec_page_state(page, NR_FILE_MAPPED, nr);
1208out:
1209 unlock_page_memcg(page);
1210}
1211
1212static void page_remove_file_rmap(struct page *page, bool compound)
1213{
1214 int i, nr = 1;
1215
1216 VM_BUG_ON_PAGE(compound && !PageHead(page), page);
1217 lock_page_memcg(page);
1218
1219 /* Hugepages are not counted in NR_FILE_MAPPED for now. */
1220 if (unlikely(PageHuge(page))) {
1221 /* hugetlb pages are always mapped with pmds */
1222 atomic_dec(compound_mapcount_ptr(page));
1223 goto out;
1224 }
1225
1226 /* page still mapped by someone else? */
1227 if (compound && PageTransHuge(page)) {
1228 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
1229 if (atomic_add_negative(-1, &page[i]._mapcount))
1230 nr++;
1231 }
1232 if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
1233 goto out;
1234 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
1235 __dec_node_page_state(page, NR_SHMEM_PMDMAPPED);
1236 } else {
1237 if (!atomic_add_negative(-1, &page->_mapcount))
1238 goto out;
1239 }
1240
1241 /*
1242 * We use the irq-unsafe __{inc|mod}_lruvec_page_state because
1243 * these counters are not modified in interrupt context, and
1244 * pte lock(a spinlock) is held, which implies preemption disabled.
1245 */
1246 __mod_lruvec_page_state(page, NR_FILE_MAPPED, -nr);
1247
1248 if (unlikely(PageMlocked(page)))
1249 clear_page_mlock(page);
1250out:
1251 unlock_page_memcg(page);
1252}
1253
1254static void page_remove_anon_compound_rmap(struct page *page)
1255{
1256 int i, nr;
1257
1258 if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
1259 return;
1260
1261 /* Hugepages are not counted in NR_ANON_PAGES for now. */
1262 if (unlikely(PageHuge(page)))
1263 return;
1264
1265 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1266 return;
1267
1268 __dec_node_page_state(page, NR_ANON_THPS);
1269
1270 if (TestClearPageDoubleMap(page)) {
1271 /*
1272 * Subpages can be mapped with PTEs too. Check how many of
1273 * themi are still mapped.
1274 */
1275 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
1276 if (atomic_add_negative(-1, &page[i]._mapcount))
1277 nr++;
1278 }
1279 } else {
1280 nr = HPAGE_PMD_NR;
1281 }
1282
1283 if (unlikely(PageMlocked(page)))
1284 clear_page_mlock(page);
1285
1286 if (nr) {
1287 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, -nr);
1288 deferred_split_huge_page(page);
1289 }
1290}
1291
1292/**
1293 * page_remove_rmap - take down pte mapping from a page
1294 * @page: page to remove mapping from
1295 * @compound: uncharge the page as compound or small page
1296 *
1297 * The caller needs to hold the pte lock.
1298 */
1299void page_remove_rmap(struct page *page, bool compound)
1300{
1301 if (!PageAnon(page))
1302 return page_remove_file_rmap(page, compound);
1303
1304 if (compound)
1305 return page_remove_anon_compound_rmap(page);
1306
1307 /* page still mapped by someone else? */
1308 if (!atomic_add_negative(-1, &page->_mapcount))
1309 return;
1310
1311 /*
1312 * We use the irq-unsafe __{inc|mod}_zone_page_stat because
1313 * these counters are not modified in interrupt context, and
1314 * pte lock(a spinlock) is held, which implies preemption disabled.
1315 */
1316 __dec_node_page_state(page, NR_ANON_MAPPED);
1317
1318 if (unlikely(PageMlocked(page)))
1319 clear_page_mlock(page);
1320
1321 if (PageTransCompound(page))
1322 deferred_split_huge_page(compound_head(page));
1323
1324 /*
1325 * It would be tidy to reset the PageAnon mapping here,
1326 * but that might overwrite a racing page_add_anon_rmap
1327 * which increments mapcount after us but sets mapping
1328 * before us: so leave the reset to free_unref_page,
1329 * and remember that it's only reliable while mapped.
1330 * Leaving it set also helps swapoff to reinstate ptes
1331 * faster for those pages still in swapcache.
1332 */
1333}
1334
1335/*
1336 * @arg: enum ttu_flags will be passed to this argument
1337 */
1338static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
1339 unsigned long address, void *arg)
1340{
1341 struct mm_struct *mm = vma->vm_mm;
1342 struct page_vma_mapped_walk pvmw = {
1343 .page = page,
1344 .vma = vma,
1345 .address = address,
1346 };
1347 pte_t pteval;
1348 struct page *subpage;
1349 bool ret = true;
1350 struct mmu_notifier_range range;
1351 enum ttu_flags flags = (enum ttu_flags)arg;
1352
1353 /* munlock has nothing to gain from examining un-locked vmas */
1354 if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED))
1355 return true;
1356
1357 if (IS_ENABLED(CONFIG_MIGRATION) && (flags & TTU_MIGRATION) &&
1358 is_zone_device_page(page) && !is_device_private_page(page))
1359 return true;
1360
1361 if (flags & TTU_SPLIT_HUGE_PMD) {
1362 split_huge_pmd_address(vma, address,
1363 flags & TTU_SPLIT_FREEZE, page);
1364 }
1365
1366 /*
1367 * For THP, we have to assume the worse case ie pmd for invalidation.
1368 * For hugetlb, it could be much worse if we need to do pud
1369 * invalidation in the case of pmd sharing.
1370 *
1371 * Note that the page can not be free in this function as call of
1372 * try_to_unmap() must hold a reference on the page.
1373 */
1374 mmu_notifier_range_init(&range, vma->vm_mm, address,
1375 min(vma->vm_end, address +
1376 (PAGE_SIZE << compound_order(page))));
1377 if (PageHuge(page)) {
1378 /*
1379 * If sharing is possible, start and end will be adjusted
1380 * accordingly.
1381 */
1382 adjust_range_if_pmd_sharing_possible(vma, &range.start,
1383 &range.end);
1384 }
1385 mmu_notifier_invalidate_range_start(&range);
1386
1387 while (page_vma_mapped_walk(&pvmw)) {
1388#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1389 /* PMD-mapped THP migration entry */
1390 if (!pvmw.pte && (flags & TTU_MIGRATION)) {
1391 VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
1392
1393 set_pmd_migration_entry(&pvmw, page);
1394 continue;
1395 }
1396#endif
1397
1398 /*
1399 * If the page is mlock()d, we cannot swap it out.
1400 * If it's recently referenced (perhaps page_referenced
1401 * skipped over this mm) then we should reactivate it.
1402 */
1403 if (!(flags & TTU_IGNORE_MLOCK)) {
1404 if (vma->vm_flags & VM_LOCKED) {
1405 /* PTE-mapped THP are never mlocked */
1406 if (!PageTransCompound(page)) {
1407 /*
1408 * Holding pte lock, we do *not* need
1409 * mmap_sem here
1410 */
1411 mlock_vma_page(page);
1412 }
1413 ret = false;
1414 page_vma_mapped_walk_done(&pvmw);
1415 break;
1416 }
1417 if (flags & TTU_MUNLOCK)
1418 continue;
1419 }
1420
1421 /* Unexpected PMD-mapped THP? */
1422 VM_BUG_ON_PAGE(!pvmw.pte, page);
1423
1424 subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
1425 address = pvmw.address;
1426
1427 if (PageHuge(page)) {
1428 if (huge_pmd_unshare(mm, &address, pvmw.pte)) {
1429 /*
1430 * huge_pmd_unshare unmapped an entire PMD
1431 * page. There is no way of knowing exactly
1432 * which PMDs may be cached for this mm, so
1433 * we must flush them all. start/end were
1434 * already adjusted above to cover this range.
1435 */
1436 flush_cache_range(vma, range.start, range.end);
1437 flush_tlb_range(vma, range.start, range.end);
1438 mmu_notifier_invalidate_range(mm, range.start,
1439 range.end);
1440
1441 /*
1442 * The ref count of the PMD page was dropped
1443 * which is part of the way map counting
1444 * is done for shared PMDs. Return 'true'
1445 * here. When there is no other sharing,
1446 * huge_pmd_unshare returns false and we will
1447 * unmap the actual page and drop map count
1448 * to zero.
1449 */
1450 page_vma_mapped_walk_done(&pvmw);
1451 break;
1452 }
1453 }
1454
1455 if (IS_ENABLED(CONFIG_MIGRATION) &&
1456 (flags & TTU_MIGRATION) &&
1457 is_zone_device_page(page)) {
1458 swp_entry_t entry;
1459 pte_t swp_pte;
1460
1461 pteval = ptep_get_and_clear(mm, pvmw.address, pvmw.pte);
1462
1463 /*
1464 * Store the pfn of the page in a special migration
1465 * pte. do_swap_page() will wait until the migration
1466 * pte is removed and then restart fault handling.
1467 */
1468 entry = make_migration_entry(page, 0);
1469 swp_pte = swp_entry_to_pte(entry);
1470 if (pte_soft_dirty(pteval))
1471 swp_pte = pte_swp_mksoft_dirty(swp_pte);
1472 set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
1473 /*
1474 * No need to invalidate here it will synchronize on
1475 * against the special swap migration pte.
1476 */
1477 goto discard;
1478 }
1479
1480 if (!(flags & TTU_IGNORE_ACCESS)) {
1481 if (ptep_clear_flush_young_notify(vma, address,
1482 pvmw.pte)) {
1483 ret = false;
1484 page_vma_mapped_walk_done(&pvmw);
1485 break;
1486 }
1487 }
1488
1489 /* Nuke the page table entry. */
1490 flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
1491 if (should_defer_flush(mm, flags)) {
1492 /*
1493 * We clear the PTE but do not flush so potentially
1494 * a remote CPU could still be writing to the page.
1495 * If the entry was previously clean then the
1496 * architecture must guarantee that a clear->dirty
1497 * transition on a cached TLB entry is written through
1498 * and traps if the PTE is unmapped.
1499 */
1500 pteval = ptep_get_and_clear(mm, address, pvmw.pte);
1501
1502 set_tlb_ubc_flush_pending(mm, pte_dirty(pteval));
1503 } else {
1504 pteval = ptep_clear_flush(vma, address, pvmw.pte);
1505 }
1506
1507 /* Move the dirty bit to the page. Now the pte is gone. */
1508 if (pte_dirty(pteval))
1509 set_page_dirty(page);
1510
1511 /* Update high watermark before we lower rss */
1512 update_hiwater_rss(mm);
1513
1514 if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) {
1515 pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
1516 if (PageHuge(page)) {
1517 int nr = 1 << compound_order(page);
1518 hugetlb_count_sub(nr, mm);
1519 set_huge_swap_pte_at(mm, address,
1520 pvmw.pte, pteval,
1521 vma_mmu_pagesize(vma));
1522 } else {
1523 dec_mm_counter(mm, mm_counter(page));
1524 set_pte_at(mm, address, pvmw.pte, pteval);
1525 }
1526
1527 } else if (pte_unused(pteval) && !userfaultfd_armed(vma)) {
1528 /*
1529 * The guest indicated that the page content is of no
1530 * interest anymore. Simply discard the pte, vmscan
1531 * will take care of the rest.
1532 * A future reference will then fault in a new zero
1533 * page. When userfaultfd is active, we must not drop
1534 * this page though, as its main user (postcopy
1535 * migration) will not expect userfaults on already
1536 * copied pages.
1537 */
1538 dec_mm_counter(mm, mm_counter(page));
1539 /* We have to invalidate as we cleared the pte */
1540 mmu_notifier_invalidate_range(mm, address,
1541 address + PAGE_SIZE);
1542 } else if (IS_ENABLED(CONFIG_MIGRATION) &&
1543 (flags & (TTU_MIGRATION|TTU_SPLIT_FREEZE))) {
1544 swp_entry_t entry;
1545 pte_t swp_pte;
1546
1547 if (arch_unmap_one(mm, vma, address, pteval) < 0) {
1548 set_pte_at(mm, address, pvmw.pte, pteval);
1549 ret = false;
1550 page_vma_mapped_walk_done(&pvmw);
1551 break;
1552 }
1553
1554 /*
1555 * Store the pfn of the page in a special migration
1556 * pte. do_swap_page() will wait until the migration
1557 * pte is removed and then restart fault handling.
1558 */
1559 entry = make_migration_entry(subpage,
1560 pte_write(pteval));
1561 swp_pte = swp_entry_to_pte(entry);
1562 if (pte_soft_dirty(pteval))
1563 swp_pte = pte_swp_mksoft_dirty(swp_pte);
1564 set_pte_at(mm, address, pvmw.pte, swp_pte);
1565 /*
1566 * No need to invalidate here it will synchronize on
1567 * against the special swap migration pte.
1568 */
1569 } else if (PageAnon(page)) {
1570 swp_entry_t entry = { .val = page_private(subpage) };
1571 pte_t swp_pte;
1572 /*
1573 * Store the swap location in the pte.
1574 * See handle_pte_fault() ...
1575 */
1576 if (unlikely(PageSwapBacked(page) != PageSwapCache(page))) {
1577 WARN_ON_ONCE(1);
1578 ret = false;
1579 /* We have to invalidate as we cleared the pte */
1580 mmu_notifier_invalidate_range(mm, address,
1581 address + PAGE_SIZE);
1582 page_vma_mapped_walk_done(&pvmw);
1583 break;
1584 }
1585
1586 /* MADV_FREE page check */
1587 if (!PageSwapBacked(page)) {
1588 if (!PageDirty(page)) {
1589 /* Invalidate as we cleared the pte */
1590 mmu_notifier_invalidate_range(mm,
1591 address, address + PAGE_SIZE);
1592 dec_mm_counter(mm, MM_ANONPAGES);
1593 goto discard;
1594 }
1595
1596 /*
1597 * If the page was redirtied, it cannot be
1598 * discarded. Remap the page to page table.
1599 */
1600 set_pte_at(mm, address, pvmw.pte, pteval);
1601 SetPageSwapBacked(page);
1602 ret = false;
1603 page_vma_mapped_walk_done(&pvmw);
1604 break;
1605 }
1606
1607 if (swap_duplicate(entry) < 0) {
1608 set_pte_at(mm, address, pvmw.pte, pteval);
1609 ret = false;
1610 page_vma_mapped_walk_done(&pvmw);
1611 break;
1612 }
1613 if (arch_unmap_one(mm, vma, address, pteval) < 0) {
1614 set_pte_at(mm, address, pvmw.pte, pteval);
1615 ret = false;
1616 page_vma_mapped_walk_done(&pvmw);
1617 break;
1618 }
1619 if (list_empty(&mm->mmlist)) {
1620 spin_lock(&mmlist_lock);
1621 if (list_empty(&mm->mmlist))
1622 list_add(&mm->mmlist, &init_mm.mmlist);
1623 spin_unlock(&mmlist_lock);
1624 }
1625 dec_mm_counter(mm, MM_ANONPAGES);
1626 inc_mm_counter(mm, MM_SWAPENTS);
1627 swp_pte = swp_entry_to_pte(entry);
1628 if (pte_soft_dirty(pteval))
1629 swp_pte = pte_swp_mksoft_dirty(swp_pte);
1630 set_pte_at(mm, address, pvmw.pte, swp_pte);
1631 /* Invalidate as we cleared the pte */
1632 mmu_notifier_invalidate_range(mm, address,
1633 address + PAGE_SIZE);
1634 } else {
1635 /*
1636 * This is a locked file-backed page, thus it cannot
1637 * be removed from the page cache and replaced by a new
1638 * page before mmu_notifier_invalidate_range_end, so no
1639 * concurrent thread might update its page table to
1640 * point at new page while a device still is using this
1641 * page.
1642 *
1643 * See Documentation/vm/mmu_notifier.rst
1644 */
1645 dec_mm_counter(mm, mm_counter_file(page));
1646 }
1647discard:
1648 /*
1649 * No need to call mmu_notifier_invalidate_range() it has be
1650 * done above for all cases requiring it to happen under page
1651 * table lock before mmu_notifier_invalidate_range_end()
1652 *
1653 * See Documentation/vm/mmu_notifier.rst
1654 */
1655 page_remove_rmap(subpage, PageHuge(page));
1656 put_page(page);
1657 }
1658
1659 mmu_notifier_invalidate_range_end(&range);
1660
1661 return ret;
1662}
1663
1664bool is_vma_temporary_stack(struct vm_area_struct *vma)
1665{
1666 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
1667
1668 if (!maybe_stack)
1669 return false;
1670
1671 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) ==
1672 VM_STACK_INCOMPLETE_SETUP)
1673 return true;
1674
1675 return false;
1676}
1677
1678static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg)
1679{
1680 return is_vma_temporary_stack(vma);
1681}
1682
1683static int page_mapcount_is_zero(struct page *page)
1684{
1685 return !total_mapcount(page);
1686}
1687
1688/**
1689 * try_to_unmap - try to remove all page table mappings to a page
1690 * @page: the page to get unmapped
1691 * @flags: action and flags
1692 *
1693 * Tries to remove all the page table entries which are mapping this
1694 * page, used in the pageout path. Caller must hold the page lock.
1695 *
1696 * If unmap is successful, return true. Otherwise, false.
1697 */
1698bool try_to_unmap(struct page *page, enum ttu_flags flags)
1699{
1700 struct rmap_walk_control rwc = {
1701 .rmap_one = try_to_unmap_one,
1702 .arg = (void *)flags,
1703 .done = page_mapcount_is_zero,
1704 .anon_lock = page_lock_anon_vma_read,
1705 };
1706
1707 /*
1708 * During exec, a temporary VMA is setup and later moved.
1709 * The VMA is moved under the anon_vma lock but not the
1710 * page tables leading to a race where migration cannot
1711 * find the migration ptes. Rather than increasing the
1712 * locking requirements of exec(), migration skips
1713 * temporary VMAs until after exec() completes.
1714 */
1715 if ((flags & (TTU_MIGRATION|TTU_SPLIT_FREEZE))
1716 && !PageKsm(page) && PageAnon(page))
1717 rwc.invalid_vma = invalid_migration_vma;
1718
1719 if (flags & TTU_RMAP_LOCKED)
1720 rmap_walk_locked(page, &rwc);
1721 else
1722 rmap_walk(page, &rwc);
1723
1724 return !page_mapcount(page) ? true : false;
1725}
1726
1727static int page_not_mapped(struct page *page)
1728{
1729 return !page_mapped(page);
1730};
1731
1732/**
1733 * try_to_munlock - try to munlock a page
1734 * @page: the page to be munlocked
1735 *
1736 * Called from munlock code. Checks all of the VMAs mapping the page
1737 * to make sure nobody else has this page mlocked. The page will be
1738 * returned with PG_mlocked cleared if no other vmas have it mlocked.
1739 */
1740
1741void try_to_munlock(struct page *page)
1742{
1743 struct rmap_walk_control rwc = {
1744 .rmap_one = try_to_unmap_one,
1745 .arg = (void *)TTU_MUNLOCK,
1746 .done = page_not_mapped,
1747 .anon_lock = page_lock_anon_vma_read,
1748
1749 };
1750
1751 VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page);
1752 VM_BUG_ON_PAGE(PageCompound(page) && PageDoubleMap(page), page);
1753
1754 rmap_walk(page, &rwc);
1755}
1756
1757void __put_anon_vma(struct anon_vma *anon_vma)
1758{
1759 struct anon_vma *root = anon_vma->root;
1760
1761 anon_vma_free(anon_vma);
1762 if (root != anon_vma && atomic_dec_and_test(&root->refcount))
1763 anon_vma_free(root);
1764}
1765
1766static struct anon_vma *rmap_walk_anon_lock(struct page *page,
1767 struct rmap_walk_control *rwc)
1768{
1769 struct anon_vma *anon_vma;
1770
1771 if (rwc->anon_lock)
1772 return rwc->anon_lock(page);
1773
1774 /*
1775 * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read()
1776 * because that depends on page_mapped(); but not all its usages
1777 * are holding mmap_sem. Users without mmap_sem are required to
1778 * take a reference count to prevent the anon_vma disappearing
1779 */
1780 anon_vma = page_anon_vma(page);
1781 if (!anon_vma)
1782 return NULL;
1783
1784 anon_vma_lock_read(anon_vma);
1785 return anon_vma;
1786}
1787
1788/*
1789 * rmap_walk_anon - do something to anonymous page using the object-based
1790 * rmap method
1791 * @page: the page to be handled
1792 * @rwc: control variable according to each walk type
1793 *
1794 * Find all the mappings of a page using the mapping pointer and the vma chains
1795 * contained in the anon_vma struct it points to.
1796 *
1797 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma
1798 * where the page was found will be held for write. So, we won't recheck
1799 * vm_flags for that VMA. That should be OK, because that vma shouldn't be
1800 * LOCKED.
1801 */
1802static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc,
1803 bool locked)
1804{
1805 struct anon_vma *anon_vma;
1806 pgoff_t pgoff_start, pgoff_end;
1807 struct anon_vma_chain *avc;
1808
1809 if (locked) {
1810 anon_vma = page_anon_vma(page);
1811 /* anon_vma disappear under us? */
1812 VM_BUG_ON_PAGE(!anon_vma, page);
1813 } else {
1814 anon_vma = rmap_walk_anon_lock(page, rwc);
1815 }
1816 if (!anon_vma)
1817 return;
1818
1819 pgoff_start = page_to_pgoff(page);
1820 pgoff_end = pgoff_start + hpage_nr_pages(page) - 1;
1821 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
1822 pgoff_start, pgoff_end) {
1823 struct vm_area_struct *vma = avc->vma;
1824 unsigned long address = vma_address(page, vma);
1825
1826 cond_resched();
1827
1828 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
1829 continue;
1830
1831 if (!rwc->rmap_one(page, vma, address, rwc->arg))
1832 break;
1833 if (rwc->done && rwc->done(page))
1834 break;
1835 }
1836
1837 if (!locked)
1838 anon_vma_unlock_read(anon_vma);
1839}
1840
1841/*
1842 * rmap_walk_file - do something to file page using the object-based rmap method
1843 * @page: the page to be handled
1844 * @rwc: control variable according to each walk type
1845 *
1846 * Find all the mappings of a page using the mapping pointer and the vma chains
1847 * contained in the address_space struct it points to.
1848 *
1849 * When called from try_to_munlock(), the mmap_sem of the mm containing the vma
1850 * where the page was found will be held for write. So, we won't recheck
1851 * vm_flags for that VMA. That should be OK, because that vma shouldn't be
1852 * LOCKED.
1853 */
1854static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc,
1855 bool locked)
1856{
1857 struct address_space *mapping = page_mapping(page);
1858 pgoff_t pgoff_start, pgoff_end;
1859 struct vm_area_struct *vma;
1860
1861 /*
1862 * The page lock not only makes sure that page->mapping cannot
1863 * suddenly be NULLified by truncation, it makes sure that the
1864 * structure at mapping cannot be freed and reused yet,
1865 * so we can safely take mapping->i_mmap_rwsem.
1866 */
1867 VM_BUG_ON_PAGE(!PageLocked(page), page);
1868
1869 if (!mapping)
1870 return;
1871
1872 pgoff_start = page_to_pgoff(page);
1873 pgoff_end = pgoff_start + hpage_nr_pages(page) - 1;
1874 if (!locked)
1875 i_mmap_lock_read(mapping);
1876 vma_interval_tree_foreach(vma, &mapping->i_mmap,
1877 pgoff_start, pgoff_end) {
1878 unsigned long address = vma_address(page, vma);
1879
1880 cond_resched();
1881
1882 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
1883 continue;
1884
1885 if (!rwc->rmap_one(page, vma, address, rwc->arg))
1886 goto done;
1887 if (rwc->done && rwc->done(page))
1888 goto done;
1889 }
1890
1891done:
1892 if (!locked)
1893 i_mmap_unlock_read(mapping);
1894}
1895
1896void rmap_walk(struct page *page, struct rmap_walk_control *rwc)
1897{
1898 if (unlikely(PageKsm(page)))
1899 rmap_walk_ksm(page, rwc);
1900 else if (PageAnon(page))
1901 rmap_walk_anon(page, rwc, false);
1902 else
1903 rmap_walk_file(page, rwc, false);
1904}
1905
1906/* Like rmap_walk, but caller holds relevant rmap lock */
1907void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc)
1908{
1909 /* no ksm support for now */
1910 VM_BUG_ON_PAGE(PageKsm(page), page);
1911 if (PageAnon(page))
1912 rmap_walk_anon(page, rwc, true);
1913 else
1914 rmap_walk_file(page, rwc, true);
1915}
1916
1917#ifdef CONFIG_HUGETLB_PAGE
1918/*
1919 * The following two functions are for anonymous (private mapped) hugepages.
1920 * Unlike common anonymous pages, anonymous hugepages have no accounting code
1921 * and no lru code, because we handle hugepages differently from common pages.
1922 */
1923void hugepage_add_anon_rmap(struct page *page,
1924 struct vm_area_struct *vma, unsigned long address)
1925{
1926 struct anon_vma *anon_vma = vma->anon_vma;
1927 int first;
1928
1929 BUG_ON(!PageLocked(page));
1930 BUG_ON(!anon_vma);
1931 /* address might be in next vma when migration races vma_adjust */
1932 first = atomic_inc_and_test(compound_mapcount_ptr(page));
1933 if (first)
1934 __page_set_anon_rmap(page, vma, address, 0);
1935}
1936
1937void hugepage_add_new_anon_rmap(struct page *page,
1938 struct vm_area_struct *vma, unsigned long address)
1939{
1940 BUG_ON(address < vma->vm_start || address >= vma->vm_end);
1941 atomic_set(compound_mapcount_ptr(page), 0);
1942 __page_set_anon_rmap(page, vma, address, 1);
1943}
1944#endif /* CONFIG_HUGETLB_PAGE */
1945