1// SPDX-License-Identifier: GPL-2.0
2/*
3 * mm/mprotect.c
4 *
5 * (C) Copyright 1994 Linus Torvalds
6 * (C) Copyright 2002 Christoph Hellwig
7 *
8 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
9 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
10 */
11
12#include <linux/mm.h>
13#include <linux/hugetlb.h>
14#include <linux/shm.h>
15#include <linux/mman.h>
16#include <linux/fs.h>
17#include <linux/highmem.h>
18#include <linux/security.h>
19#include <linux/mempolicy.h>
20#include <linux/personality.h>
21#include <linux/syscalls.h>
22#include <linux/swap.h>
23#include <linux/swapops.h>
24#include <linux/mmu_notifier.h>
25#include <linux/migrate.h>
26#include <linux/perf_event.h>
27#include <linux/pkeys.h>
28#include <linux/ksm.h>
29#include <linux/uaccess.h>
30#include <linux/mm_inline.h>
31#include <asm/pgtable.h>
32#include <asm/cacheflush.h>
33#include <asm/mmu_context.h>
34#include <asm/tlbflush.h>
35
36#include "internal.h"
37
38static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
39 unsigned long addr, unsigned long end, pgprot_t newprot,
40 int dirty_accountable, int prot_numa)
41{
42 struct mm_struct *mm = vma->vm_mm;
43 pte_t *pte, oldpte;
44 spinlock_t *ptl;
45 unsigned long pages = 0;
46 int target_node = NUMA_NO_NODE;
47
48 /*
49 * Can be called with only the mmap_sem for reading by
50 * prot_numa so we must check the pmd isn't constantly
51 * changing from under us from pmd_none to pmd_trans_huge
52 * and/or the other way around.
53 */
54 if (pmd_trans_unstable(pmd))
55 return 0;
56
57 /*
58 * The pmd points to a regular pte so the pmd can't change
59 * from under us even if the mmap_sem is only hold for
60 * reading.
61 */
62 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
63
64 /* Get target node for single threaded private VMAs */
65 if (prot_numa && !(vma->vm_flags & VM_SHARED) &&
66 atomic_read(&vma->vm_mm->mm_users) == 1)
67 target_node = numa_node_id();
68
69 flush_tlb_batched_pending(vma->vm_mm);
70 arch_enter_lazy_mmu_mode();
71 do {
72 oldpte = *pte;
73 if (pte_present(oldpte)) {
74 pte_t ptent;
75 bool preserve_write = prot_numa && pte_write(oldpte);
76
77 /*
78 * Avoid trapping faults against the zero or KSM
79 * pages. See similar comment in change_huge_pmd.
80 */
81 if (prot_numa) {
82 struct page *page;
83
84 page = vm_normal_page(vma, addr, oldpte);
85 if (!page || PageKsm(page))
86 continue;
87
88 /* Also skip shared copy-on-write pages */
89 if (is_cow_mapping(vma->vm_flags) &&
90 page_mapcount(page) != 1)
91 continue;
92
93 /*
94 * While migration can move some dirty pages,
95 * it cannot move them all from MIGRATE_ASYNC
96 * context.
97 */
98 if (page_is_file_cache(page) && PageDirty(page))
99 continue;
100
101 /* Avoid TLB flush if possible */
102 if (pte_protnone(oldpte))
103 continue;
104
105 /*
106 * Don't mess with PTEs if page is already on the node
107 * a single-threaded process is running on.
108 */
109 if (target_node == page_to_nid(page))
110 continue;
111 }
112
113 oldpte = ptep_modify_prot_start(vma, addr, pte);
114 ptent = pte_modify(oldpte, newprot);
115 if (preserve_write)
116 ptent = pte_mk_savedwrite(ptent);
117
118 /* Avoid taking write faults for known dirty pages */
119 if (dirty_accountable && pte_dirty(ptent) &&
120 (pte_soft_dirty(ptent) ||
121 !(vma->vm_flags & VM_SOFTDIRTY))) {
122 ptent = pte_mkwrite(ptent);
123 }
124 ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);
125 pages++;
126 } else if (IS_ENABLED(CONFIG_MIGRATION)) {
127 swp_entry_t entry = pte_to_swp_entry(oldpte);
128
129 if (is_write_migration_entry(entry)) {
130 pte_t newpte;
131 /*
132 * A protection check is difficult so
133 * just be safe and disable write
134 */
135 make_migration_entry_read(&entry);
136 newpte = swp_entry_to_pte(entry);
137 if (pte_swp_soft_dirty(oldpte))
138 newpte = pte_swp_mksoft_dirty(newpte);
139 set_pte_at(mm, addr, pte, newpte);
140
141 pages++;
142 }
143
144 if (is_write_device_private_entry(entry)) {
145 pte_t newpte;
146
147 /*
148 * We do not preserve soft-dirtiness. See
149 * copy_one_pte() for explanation.
150 */
151 make_device_private_entry_read(&entry);
152 newpte = swp_entry_to_pte(entry);
153 set_pte_at(mm, addr, pte, newpte);
154
155 pages++;
156 }
157 }
158 } while (pte++, addr += PAGE_SIZE, addr != end);
159 arch_leave_lazy_mmu_mode();
160 pte_unmap_unlock(pte - 1, ptl);
161
162 return pages;
163}
164
165static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
166 pud_t *pud, unsigned long addr, unsigned long end,
167 pgprot_t newprot, int dirty_accountable, int prot_numa)
168{
169 pmd_t *pmd;
170 unsigned long next;
171 unsigned long pages = 0;
172 unsigned long nr_huge_updates = 0;
173 struct mmu_notifier_range range;
174
175 range.start = 0;
176
177 pmd = pmd_offset(pud, addr);
178 do {
179 unsigned long this_pages;
180
181 next = pmd_addr_end(addr, end);
182 if (!is_swap_pmd(*pmd) && !pmd_trans_huge(*pmd) && !pmd_devmap(*pmd)
183 && pmd_none_or_clear_bad(pmd))
184 goto next;
185
186 /* invoke the mmu notifier if the pmd is populated */
187 if (!range.start) {
188 mmu_notifier_range_init(&range, vma->vm_mm, addr, end);
189 mmu_notifier_invalidate_range_start(&range);
190 }
191
192 if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
193 if (next - addr != HPAGE_PMD_SIZE) {
194 __split_huge_pmd(vma, pmd, addr, false, NULL);
195 } else {
196 int nr_ptes = change_huge_pmd(vma, pmd, addr,
197 newprot, prot_numa);
198
199 if (nr_ptes) {
200 if (nr_ptes == HPAGE_PMD_NR) {
201 pages += HPAGE_PMD_NR;
202 nr_huge_updates++;
203 }
204
205 /* huge pmd was handled */
206 goto next;
207 }
208 }
209 /* fall through, the trans huge pmd just split */
210 }
211 this_pages = change_pte_range(vma, pmd, addr, next, newprot,
212 dirty_accountable, prot_numa);
213 pages += this_pages;
214next:
215 cond_resched();
216 } while (pmd++, addr = next, addr != end);
217
218 if (range.start)
219 mmu_notifier_invalidate_range_end(&range);
220
221 if (nr_huge_updates)
222 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
223 return pages;
224}
225
226static inline unsigned long change_pud_range(struct vm_area_struct *vma,
227 p4d_t *p4d, unsigned long addr, unsigned long end,
228 pgprot_t newprot, int dirty_accountable, int prot_numa)
229{
230 pud_t *pud;
231 unsigned long next;
232 unsigned long pages = 0;
233
234 pud = pud_offset(p4d, addr);
235 do {
236 next = pud_addr_end(addr, end);
237 if (pud_none_or_clear_bad(pud))
238 continue;
239 pages += change_pmd_range(vma, pud, addr, next, newprot,
240 dirty_accountable, prot_numa);
241 } while (pud++, addr = next, addr != end);
242
243 return pages;
244}
245
246static inline unsigned long change_p4d_range(struct vm_area_struct *vma,
247 pgd_t *pgd, unsigned long addr, unsigned long end,
248 pgprot_t newprot, int dirty_accountable, int prot_numa)
249{
250 p4d_t *p4d;
251 unsigned long next;
252 unsigned long pages = 0;
253
254 p4d = p4d_offset(pgd, addr);
255 do {
256 next = p4d_addr_end(addr, end);
257 if (p4d_none_or_clear_bad(p4d))
258 continue;
259 pages += change_pud_range(vma, p4d, addr, next, newprot,
260 dirty_accountable, prot_numa);
261 } while (p4d++, addr = next, addr != end);
262
263 return pages;
264}
265
266static unsigned long change_protection_range(struct vm_area_struct *vma,
267 unsigned long addr, unsigned long end, pgprot_t newprot,
268 int dirty_accountable, int prot_numa)
269{
270 struct mm_struct *mm = vma->vm_mm;
271 pgd_t *pgd;
272 unsigned long next;
273 unsigned long start = addr;
274 unsigned long pages = 0;
275
276 BUG_ON(addr >= end);
277 pgd = pgd_offset(mm, addr);
278 flush_cache_range(vma, addr, end);
279 inc_tlb_flush_pending(mm);
280 do {
281 next = pgd_addr_end(addr, end);
282 if (pgd_none_or_clear_bad(pgd))
283 continue;
284 pages += change_p4d_range(vma, pgd, addr, next, newprot,
285 dirty_accountable, prot_numa);
286 } while (pgd++, addr = next, addr != end);
287
288 /* Only flush the TLB if we actually modified any entries: */
289 if (pages)
290 flush_tlb_range(vma, start, end);
291 dec_tlb_flush_pending(mm);
292
293 return pages;
294}
295
296unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
297 unsigned long end, pgprot_t newprot,
298 int dirty_accountable, int prot_numa)
299{
300 unsigned long pages;
301
302 if (is_vm_hugetlb_page(vma))
303 pages = hugetlb_change_protection(vma, start, end, newprot);
304 else
305 pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa);
306
307 return pages;
308}
309
310static int prot_none_pte_entry(pte_t *pte, unsigned long addr,
311 unsigned long next, struct mm_walk *walk)
312{
313 return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ?
314 0 : -EACCES;
315}
316
317static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask,
318 unsigned long addr, unsigned long next,
319 struct mm_walk *walk)
320{
321 return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ?
322 0 : -EACCES;
323}
324
325static int prot_none_test(unsigned long addr, unsigned long next,
326 struct mm_walk *walk)
327{
328 return 0;
329}
330
331static int prot_none_walk(struct vm_area_struct *vma, unsigned long start,
332 unsigned long end, unsigned long newflags)
333{
334 pgprot_t new_pgprot = vm_get_page_prot(newflags);
335 struct mm_walk prot_none_walk = {
336 .pte_entry = prot_none_pte_entry,
337 .hugetlb_entry = prot_none_hugetlb_entry,
338 .test_walk = prot_none_test,
339 .mm = current->mm,
340 .private = &new_pgprot,
341 };
342
343 return walk_page_range(start, end, &prot_none_walk);
344}
345
346int
347mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
348 unsigned long start, unsigned long end, unsigned long newflags)
349{
350 struct mm_struct *mm = vma->vm_mm;
351 unsigned long oldflags = vma->vm_flags;
352 long nrpages = (end - start) >> PAGE_SHIFT;
353 unsigned long charged = 0;
354 pgoff_t pgoff;
355 int error;
356 int dirty_accountable = 0;
357
358 if (newflags == oldflags) {
359 *pprev = vma;
360 return 0;
361 }
362
363 /*
364 * Do PROT_NONE PFN permission checks here when we can still
365 * bail out without undoing a lot of state. This is a rather
366 * uncommon case, so doesn't need to be very optimized.
367 */
368 if (arch_has_pfn_modify_check() &&
369 (vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
370 (newflags & (VM_READ|VM_WRITE|VM_EXEC)) == 0) {
371 error = prot_none_walk(vma, start, end, newflags);
372 if (error)
373 return error;
374 }
375
376 /*
377 * If we make a private mapping writable we increase our commit;
378 * but (without finer accounting) cannot reduce our commit if we
379 * make it unwritable again. hugetlb mapping were accounted for
380 * even if read-only so there is no need to account for them here
381 */
382 if (newflags & VM_WRITE) {
383 /* Check space limits when area turns into data. */
384 if (!may_expand_vm(mm, newflags, nrpages) &&
385 may_expand_vm(mm, oldflags, nrpages))
386 return -ENOMEM;
387 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB|
388 VM_SHARED|VM_NORESERVE))) {
389 charged = nrpages;
390 if (security_vm_enough_memory_mm(mm, charged))
391 return -ENOMEM;
392 newflags |= VM_ACCOUNT;
393 }
394 }
395
396 /*
397 * First try to merge with previous and/or next vma.
398 */
399 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
400 *pprev = vma_merge(mm, *pprev, start, end, newflags,
401 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
402 vma->vm_userfaultfd_ctx);
403 if (*pprev) {
404 vma = *pprev;
405 VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY);
406 goto success;
407 }
408
409 *pprev = vma;
410
411 if (start != vma->vm_start) {
412 error = split_vma(mm, vma, start, 1);
413 if (error)
414 goto fail;
415 }
416
417 if (end != vma->vm_end) {
418 error = split_vma(mm, vma, end, 0);
419 if (error)
420 goto fail;
421 }
422
423success:
424 /*
425 * vm_flags and vm_page_prot are protected by the mmap_sem
426 * held in write mode.
427 */
428 vma->vm_flags = newflags;
429 dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot);
430 vma_set_page_prot(vma);
431
432 change_protection(vma, start, end, vma->vm_page_prot,
433 dirty_accountable, 0);
434
435 /*
436 * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major
437 * fault on access.
438 */
439 if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED &&
440 (newflags & VM_WRITE)) {
441 populate_vma_page_range(vma, start, end, NULL);
442 }
443
444 vm_stat_account(mm, oldflags, -nrpages);
445 vm_stat_account(mm, newflags, nrpages);
446 perf_event_mmap(vma);
447 return 0;
448
449fail:
450 vm_unacct_memory(charged);
451 return error;
452}
453
454/*
455 * pkey==-1 when doing a legacy mprotect()
456 */
457static int do_mprotect_pkey(unsigned long start, size_t len,
458 unsigned long prot, int pkey)
459{
460 unsigned long nstart, end, tmp, reqprot;
461 struct vm_area_struct *vma, *prev;
462 int error = -EINVAL;
463 const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
464 const bool rier = (current->personality & READ_IMPLIES_EXEC) &&
465 (prot & PROT_READ);
466
467 prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
468 if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
469 return -EINVAL;
470
471 if (start & ~PAGE_MASK)
472 return -EINVAL;
473 if (!len)
474 return 0;
475 len = PAGE_ALIGN(len);
476 end = start + len;
477 if (end <= start)
478 return -ENOMEM;
479 if (!arch_validate_prot(prot, start))
480 return -EINVAL;
481
482 reqprot = prot;
483
484 if (down_write_killable(&current->mm->mmap_sem))
485 return -EINTR;
486
487 /*
488 * If userspace did not allocate the pkey, do not let
489 * them use it here.
490 */
491 error = -EINVAL;
492 if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey))
493 goto out;
494
495 vma = find_vma(current->mm, start);
496 error = -ENOMEM;
497 if (!vma)
498 goto out;
499 prev = vma->vm_prev;
500 if (unlikely(grows & PROT_GROWSDOWN)) {
501 if (vma->vm_start >= end)
502 goto out;
503 start = vma->vm_start;
504 error = -EINVAL;
505 if (!(vma->vm_flags & VM_GROWSDOWN))
506 goto out;
507 } else {
508 if (vma->vm_start > start)
509 goto out;
510 if (unlikely(grows & PROT_GROWSUP)) {
511 end = vma->vm_end;
512 error = -EINVAL;
513 if (!(vma->vm_flags & VM_GROWSUP))
514 goto out;
515 }
516 }
517 if (start > vma->vm_start)
518 prev = vma;
519
520 for (nstart = start ; ; ) {
521 unsigned long mask_off_old_flags;
522 unsigned long newflags;
523 int new_vma_pkey;
524
525 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
526
527 /* Does the application expect PROT_READ to imply PROT_EXEC */
528 if (rier && (vma->vm_flags & VM_MAYEXEC))
529 prot |= PROT_EXEC;
530
531 /*
532 * Each mprotect() call explicitly passes r/w/x permissions.
533 * If a permission is not passed to mprotect(), it must be
534 * cleared from the VMA.
535 */
536 mask_off_old_flags = VM_READ | VM_WRITE | VM_EXEC |
537 VM_FLAGS_CLEAR;
538
539 new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey);
540 newflags = calc_vm_prot_bits(prot, new_vma_pkey);
541 newflags |= (vma->vm_flags & ~mask_off_old_flags);
542
543 /* newflags >> 4 shift VM_MAY% in place of VM_% */
544 if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) {
545 error = -EACCES;
546 goto out;
547 }
548
549 error = security_file_mprotect(vma, reqprot, prot);
550 if (error)
551 goto out;
552
553 tmp = vma->vm_end;
554 if (tmp > end)
555 tmp = end;
556 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags);
557 if (error)
558 goto out;
559 nstart = tmp;
560
561 if (nstart < prev->vm_end)
562 nstart = prev->vm_end;
563 if (nstart >= end)
564 goto out;
565
566 vma = prev->vm_next;
567 if (!vma || vma->vm_start != nstart) {
568 error = -ENOMEM;
569 goto out;
570 }
571 prot = reqprot;
572 }
573out:
574 up_write(&current->mm->mmap_sem);
575 return error;
576}
577
578SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
579 unsigned long, prot)
580{
581 return do_mprotect_pkey(start, len, prot, -1);
582}
583
584#ifdef CONFIG_ARCH_HAS_PKEYS
585
586SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len,
587 unsigned long, prot, int, pkey)
588{
589 return do_mprotect_pkey(start, len, prot, pkey);
590}
591
592SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val)
593{
594 int pkey;
595 int ret;
596
597 /* No flags supported yet. */
598 if (flags)
599 return -EINVAL;
600 /* check for unsupported init values */
601 if (init_val & ~PKEY_ACCESS_MASK)
602 return -EINVAL;
603
604 down_write(&current->mm->mmap_sem);
605 pkey = mm_pkey_alloc(current->mm);
606
607 ret = -ENOSPC;
608 if (pkey == -1)
609 goto out;
610
611 ret = arch_set_user_pkey_access(current, pkey, init_val);
612 if (ret) {
613 mm_pkey_free(current->mm, pkey);
614 goto out;
615 }
616 ret = pkey;
617out:
618 up_write(&current->mm->mmap_sem);
619 return ret;
620}
621
622SYSCALL_DEFINE1(pkey_free, int, pkey)
623{
624 int ret;
625
626 down_write(&current->mm->mmap_sem);
627 ret = mm_pkey_free(current->mm, pkey);
628 up_write(&current->mm->mmap_sem);
629
630 /*
631 * We could provie warnings or errors if any VMA still
632 * has the pkey set here.
633 */
634 return ret;
635}
636
637#endif /* CONFIG_ARCH_HAS_PKEYS */
638