1/*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.
9 * Copyright (C) 2002-2011 Hugh Dickins.
10 * Copyright (C) 2011 Google Inc.
11 * Copyright (C) 2002-2005 VERITAS Software Corporation.
12 * Copyright (C) 2004 Andi Kleen, SuSE Labs
13 *
14 * Extended attribute support for tmpfs:
15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17 *
18 * tiny-shmem:
19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20 *
21 * This file is released under the GPL.
22 */
23
24#include <linux/fs.h>
25#include <linux/init.h>
26#include <linux/vfs.h>
27#include <linux/mount.h>
28#include <linux/ramfs.h>
29#include <linux/pagemap.h>
30#include <linux/file.h>
31#include <linux/mm.h>
32#include <linux/random.h>
33#include <linux/sched/signal.h>
34#include <linux/export.h>
35#include <linux/swap.h>
36#include <linux/uio.h>
37#include <linux/khugepaged.h>
38#include <linux/hugetlb.h>
39#include <linux/frontswap.h>
40
41#include <asm/tlbflush.h> /* for arch/microblaze update_mmu_cache() */
42
43static struct vfsmount *shm_mnt;
44
45#ifdef CONFIG_SHMEM
46/*
47 * This virtual memory filesystem is heavily based on the ramfs. It
48 * extends ramfs by the ability to use swap and honor resource limits
49 * which makes it a completely usable filesystem.
50 */
51
52#include <linux/xattr.h>
53#include <linux/exportfs.h>
54#include <linux/posix_acl.h>
55#include <linux/posix_acl_xattr.h>
56#include <linux/mman.h>
57#include <linux/string.h>
58#include <linux/slab.h>
59#include <linux/backing-dev.h>
60#include <linux/shmem_fs.h>
61#include <linux/writeback.h>
62#include <linux/blkdev.h>
63#include <linux/pagevec.h>
64#include <linux/percpu_counter.h>
65#include <linux/falloc.h>
66#include <linux/splice.h>
67#include <linux/security.h>
68#include <linux/swapops.h>
69#include <linux/mempolicy.h>
70#include <linux/namei.h>
71#include <linux/ctype.h>
72#include <linux/migrate.h>
73#include <linux/highmem.h>
74#include <linux/seq_file.h>
75#include <linux/magic.h>
76#include <linux/syscalls.h>
77#include <linux/fcntl.h>
78#include <uapi/linux/memfd.h>
79#include <linux/userfaultfd_k.h>
80#include <linux/rmap.h>
81#include <linux/uuid.h>
82
83#include <linux/uaccess.h>
84#include <asm/pgtable.h>
85
86#include "internal.h"
87
88#define BLOCKS_PER_PAGE (PAGE_SIZE/512)
89#define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT)
90
91/* Pretend that each entry is of this size in directory's i_size */
92#define BOGO_DIRENT_SIZE 20
93
94/* Symlink up to this size is kmalloc'ed instead of using a swappable page */
95#define SHORT_SYMLINK_LEN 128
96
97/*
98 * shmem_fallocate communicates with shmem_fault or shmem_writepage via
99 * inode->i_private (with i_mutex making sure that it has only one user at
100 * a time): we would prefer not to enlarge the shmem inode just for that.
101 */
102struct shmem_falloc {
103 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
104 pgoff_t start; /* start of range currently being fallocated */
105 pgoff_t next; /* the next page offset to be fallocated */
106 pgoff_t nr_falloced; /* how many new pages have been fallocated */
107 pgoff_t nr_unswapped; /* how often writepage refused to swap out */
108};
109
110#ifdef CONFIG_TMPFS
111static unsigned long shmem_default_max_blocks(void)
112{
113 return totalram_pages() / 2;
114}
115
116static unsigned long shmem_default_max_inodes(void)
117{
118 unsigned long nr_pages = totalram_pages();
119
120 return min(nr_pages - totalhigh_pages(), nr_pages / 2);
121}
122#endif
123
124static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
125static int shmem_replace_page(struct page **pagep, gfp_t gfp,
126 struct shmem_inode_info *info, pgoff_t index);
127static int shmem_swapin_page(struct inode *inode, pgoff_t index,
128 struct page **pagep, enum sgp_type sgp,
129 gfp_t gfp, struct vm_area_struct *vma,
130 vm_fault_t *fault_type);
131static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
132 struct page **pagep, enum sgp_type sgp,
133 gfp_t gfp, struct vm_area_struct *vma,
134 struct vm_fault *vmf, vm_fault_t *fault_type);
135
136int shmem_getpage(struct inode *inode, pgoff_t index,
137 struct page **pagep, enum sgp_type sgp)
138{
139 return shmem_getpage_gfp(inode, index, pagep, sgp,
140 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
141}
142
143static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
144{
145 return sb->s_fs_info;
146}
147
148/*
149 * shmem_file_setup pre-accounts the whole fixed size of a VM object,
150 * for shared memory and for shared anonymous (/dev/zero) mappings
151 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
152 * consistent with the pre-accounting of private mappings ...
153 */
154static inline int shmem_acct_size(unsigned long flags, loff_t size)
155{
156 return (flags & VM_NORESERVE) ?
157 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
158}
159
160static inline void shmem_unacct_size(unsigned long flags, loff_t size)
161{
162 if (!(flags & VM_NORESERVE))
163 vm_unacct_memory(VM_ACCT(size));
164}
165
166static inline int shmem_reacct_size(unsigned long flags,
167 loff_t oldsize, loff_t newsize)
168{
169 if (!(flags & VM_NORESERVE)) {
170 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
171 return security_vm_enough_memory_mm(current->mm,
172 VM_ACCT(newsize) - VM_ACCT(oldsize));
173 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
174 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
175 }
176 return 0;
177}
178
179/*
180 * ... whereas tmpfs objects are accounted incrementally as
181 * pages are allocated, in order to allow large sparse files.
182 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
183 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
184 */
185static inline int shmem_acct_block(unsigned long flags, long pages)
186{
187 if (!(flags & VM_NORESERVE))
188 return 0;
189
190 return security_vm_enough_memory_mm(current->mm,
191 pages * VM_ACCT(PAGE_SIZE));
192}
193
194static inline void shmem_unacct_blocks(unsigned long flags, long pages)
195{
196 if (flags & VM_NORESERVE)
197 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
198}
199
200static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
201{
202 struct shmem_inode_info *info = SHMEM_I(inode);
203 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
204
205 if (shmem_acct_block(info->flags, pages))
206 return false;
207
208 if (sbinfo->max_blocks) {
209 if (percpu_counter_compare(&sbinfo->used_blocks,
210 sbinfo->max_blocks - pages) > 0)
211 goto unacct;
212 percpu_counter_add(&sbinfo->used_blocks, pages);
213 }
214
215 return true;
216
217unacct:
218 shmem_unacct_blocks(info->flags, pages);
219 return false;
220}
221
222static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
223{
224 struct shmem_inode_info *info = SHMEM_I(inode);
225 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
226
227 if (sbinfo->max_blocks)
228 percpu_counter_sub(&sbinfo->used_blocks, pages);
229 shmem_unacct_blocks(info->flags, pages);
230}
231
232static const struct super_operations shmem_ops;
233static const struct address_space_operations shmem_aops;
234static const struct file_operations shmem_file_operations;
235static const struct inode_operations shmem_inode_operations;
236static const struct inode_operations shmem_dir_inode_operations;
237static const struct inode_operations shmem_special_inode_operations;
238static const struct vm_operations_struct shmem_vm_ops;
239static struct file_system_type shmem_fs_type;
240
241bool vma_is_shmem(struct vm_area_struct *vma)
242{
243 return vma->vm_ops == &shmem_vm_ops;
244}
245
246static LIST_HEAD(shmem_swaplist);
247static DEFINE_MUTEX(shmem_swaplist_mutex);
248
249static int shmem_reserve_inode(struct super_block *sb)
250{
251 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
252 if (sbinfo->max_inodes) {
253 spin_lock(&sbinfo->stat_lock);
254 if (!sbinfo->free_inodes) {
255 spin_unlock(&sbinfo->stat_lock);
256 return -ENOSPC;
257 }
258 sbinfo->free_inodes--;
259 spin_unlock(&sbinfo->stat_lock);
260 }
261 return 0;
262}
263
264static void shmem_free_inode(struct super_block *sb)
265{
266 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
267 if (sbinfo->max_inodes) {
268 spin_lock(&sbinfo->stat_lock);
269 sbinfo->free_inodes++;
270 spin_unlock(&sbinfo->stat_lock);
271 }
272}
273
274/**
275 * shmem_recalc_inode - recalculate the block usage of an inode
276 * @inode: inode to recalc
277 *
278 * We have to calculate the free blocks since the mm can drop
279 * undirtied hole pages behind our back.
280 *
281 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
282 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
283 *
284 * It has to be called with the spinlock held.
285 */
286static void shmem_recalc_inode(struct inode *inode)
287{
288 struct shmem_inode_info *info = SHMEM_I(inode);
289 long freed;
290
291 freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
292 if (freed > 0) {
293 info->alloced -= freed;
294 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
295 shmem_inode_unacct_blocks(inode, freed);
296 }
297}
298
299bool shmem_charge(struct inode *inode, long pages)
300{
301 struct shmem_inode_info *info = SHMEM_I(inode);
302 unsigned long flags;
303
304 if (!shmem_inode_acct_block(inode, pages))
305 return false;
306
307 /* nrpages adjustment first, then shmem_recalc_inode() when balanced */
308 inode->i_mapping->nrpages += pages;
309
310 spin_lock_irqsave(&info->lock, flags);
311 info->alloced += pages;
312 inode->i_blocks += pages * BLOCKS_PER_PAGE;
313 shmem_recalc_inode(inode);
314 spin_unlock_irqrestore(&info->lock, flags);
315
316 return true;
317}
318
319void shmem_uncharge(struct inode *inode, long pages)
320{
321 struct shmem_inode_info *info = SHMEM_I(inode);
322 unsigned long flags;
323
324 /* nrpages adjustment done by __delete_from_page_cache() or caller */
325
326 spin_lock_irqsave(&info->lock, flags);
327 info->alloced -= pages;
328 inode->i_blocks -= pages * BLOCKS_PER_PAGE;
329 shmem_recalc_inode(inode);
330 spin_unlock_irqrestore(&info->lock, flags);
331
332 shmem_inode_unacct_blocks(inode, pages);
333}
334
335/*
336 * Replace item expected in xarray by a new item, while holding xa_lock.
337 */
338static int shmem_replace_entry(struct address_space *mapping,
339 pgoff_t index, void *expected, void *replacement)
340{
341 XA_STATE(xas, &mapping->i_pages, index);
342 void *item;
343
344 VM_BUG_ON(!expected);
345 VM_BUG_ON(!replacement);
346 item = xas_load(&xas);
347 if (item != expected)
348 return -ENOENT;
349 xas_store(&xas, replacement);
350 return 0;
351}
352
353/*
354 * Sometimes, before we decide whether to proceed or to fail, we must check
355 * that an entry was not already brought back from swap by a racing thread.
356 *
357 * Checking page is not enough: by the time a SwapCache page is locked, it
358 * might be reused, and again be SwapCache, using the same swap as before.
359 */
360static bool shmem_confirm_swap(struct address_space *mapping,
361 pgoff_t index, swp_entry_t swap)
362{
363 return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
364}
365
366/*
367 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
368 *
369 * SHMEM_HUGE_NEVER:
370 * disables huge pages for the mount;
371 * SHMEM_HUGE_ALWAYS:
372 * enables huge pages for the mount;
373 * SHMEM_HUGE_WITHIN_SIZE:
374 * only allocate huge pages if the page will be fully within i_size,
375 * also respect fadvise()/madvise() hints;
376 * SHMEM_HUGE_ADVISE:
377 * only allocate huge pages if requested with fadvise()/madvise();
378 */
379
380#define SHMEM_HUGE_NEVER 0
381#define SHMEM_HUGE_ALWAYS 1
382#define SHMEM_HUGE_WITHIN_SIZE 2
383#define SHMEM_HUGE_ADVISE 3
384
385/*
386 * Special values.
387 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
388 *
389 * SHMEM_HUGE_DENY:
390 * disables huge on shm_mnt and all mounts, for emergency use;
391 * SHMEM_HUGE_FORCE:
392 * enables huge on shm_mnt and all mounts, w/o needing option, for testing;
393 *
394 */
395#define SHMEM_HUGE_DENY (-1)
396#define SHMEM_HUGE_FORCE (-2)
397
398#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
399/* ifdef here to avoid bloating shmem.o when not necessary */
400
401static int shmem_huge __read_mostly;
402
403#if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
404static int shmem_parse_huge(const char *str)
405{
406 if (!strcmp(str, "never"))
407 return SHMEM_HUGE_NEVER;
408 if (!strcmp(str, "always"))
409 return SHMEM_HUGE_ALWAYS;
410 if (!strcmp(str, "within_size"))
411 return SHMEM_HUGE_WITHIN_SIZE;
412 if (!strcmp(str, "advise"))
413 return SHMEM_HUGE_ADVISE;
414 if (!strcmp(str, "deny"))
415 return SHMEM_HUGE_DENY;
416 if (!strcmp(str, "force"))
417 return SHMEM_HUGE_FORCE;
418 return -EINVAL;
419}
420
421static const char *shmem_format_huge(int huge)
422{
423 switch (huge) {
424 case SHMEM_HUGE_NEVER:
425 return "never";
426 case SHMEM_HUGE_ALWAYS:
427 return "always";
428 case SHMEM_HUGE_WITHIN_SIZE:
429 return "within_size";
430 case SHMEM_HUGE_ADVISE:
431 return "advise";
432 case SHMEM_HUGE_DENY:
433 return "deny";
434 case SHMEM_HUGE_FORCE:
435 return "force";
436 default:
437 VM_BUG_ON(1);
438 return "bad_val";
439 }
440}
441#endif
442
443static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
444 struct shrink_control *sc, unsigned long nr_to_split)
445{
446 LIST_HEAD(list), *pos, *next;
447 LIST_HEAD(to_remove);
448 struct inode *inode;
449 struct shmem_inode_info *info;
450 struct page *page;
451 unsigned long batch = sc ? sc->nr_to_scan : 128;
452 int removed = 0, split = 0;
453
454 if (list_empty(&sbinfo->shrinklist))
455 return SHRINK_STOP;
456
457 spin_lock(&sbinfo->shrinklist_lock);
458 list_for_each_safe(pos, next, &sbinfo->shrinklist) {
459 info = list_entry(pos, struct shmem_inode_info, shrinklist);
460
461 /* pin the inode */
462 inode = igrab(&info->vfs_inode);
463
464 /* inode is about to be evicted */
465 if (!inode) {
466 list_del_init(&info->shrinklist);
467 removed++;
468 goto next;
469 }
470
471 /* Check if there's anything to gain */
472 if (round_up(inode->i_size, PAGE_SIZE) ==
473 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
474 list_move(&info->shrinklist, &to_remove);
475 removed++;
476 goto next;
477 }
478
479 list_move(&info->shrinklist, &list);
480next:
481 if (!--batch)
482 break;
483 }
484 spin_unlock(&sbinfo->shrinklist_lock);
485
486 list_for_each_safe(pos, next, &to_remove) {
487 info = list_entry(pos, struct shmem_inode_info, shrinklist);
488 inode = &info->vfs_inode;
489 list_del_init(&info->shrinklist);
490 iput(inode);
491 }
492
493 list_for_each_safe(pos, next, &list) {
494 int ret;
495
496 info = list_entry(pos, struct shmem_inode_info, shrinklist);
497 inode = &info->vfs_inode;
498
499 if (nr_to_split && split >= nr_to_split)
500 goto leave;
501
502 page = find_get_page(inode->i_mapping,
503 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT);
504 if (!page)
505 goto drop;
506
507 /* No huge page at the end of the file: nothing to split */
508 if (!PageTransHuge(page)) {
509 put_page(page);
510 goto drop;
511 }
512
513 /*
514 * Leave the inode on the list if we failed to lock
515 * the page at this time.
516 *
517 * Waiting for the lock may lead to deadlock in the
518 * reclaim path.
519 */
520 if (!trylock_page(page)) {
521 put_page(page);
522 goto leave;
523 }
524
525 ret = split_huge_page(page);
526 unlock_page(page);
527 put_page(page);
528
529 /* If split failed leave the inode on the list */
530 if (ret)
531 goto leave;
532
533 split++;
534drop:
535 list_del_init(&info->shrinklist);
536 removed++;
537leave:
538 iput(inode);
539 }
540
541 spin_lock(&sbinfo->shrinklist_lock);
542 list_splice_tail(&list, &sbinfo->shrinklist);
543 sbinfo->shrinklist_len -= removed;
544 spin_unlock(&sbinfo->shrinklist_lock);
545
546 return split;
547}
548
549static long shmem_unused_huge_scan(struct super_block *sb,
550 struct shrink_control *sc)
551{
552 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
553
554 if (!READ_ONCE(sbinfo->shrinklist_len))
555 return SHRINK_STOP;
556
557 return shmem_unused_huge_shrink(sbinfo, sc, 0);
558}
559
560static long shmem_unused_huge_count(struct super_block *sb,
561 struct shrink_control *sc)
562{
563 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
564 return READ_ONCE(sbinfo->shrinklist_len);
565}
566#else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */
567
568#define shmem_huge SHMEM_HUGE_DENY
569
570static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
571 struct shrink_control *sc, unsigned long nr_to_split)
572{
573 return 0;
574}
575#endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */
576
577static inline bool is_huge_enabled(struct shmem_sb_info *sbinfo)
578{
579 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
580 (shmem_huge == SHMEM_HUGE_FORCE || sbinfo->huge) &&
581 shmem_huge != SHMEM_HUGE_DENY)
582 return true;
583 return false;
584}
585
586/*
587 * Like add_to_page_cache_locked, but error if expected item has gone.
588 */
589static int shmem_add_to_page_cache(struct page *page,
590 struct address_space *mapping,
591 pgoff_t index, void *expected, gfp_t gfp)
592{
593 XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page));
594 unsigned long i = 0;
595 unsigned long nr = 1UL << compound_order(page);
596
597 VM_BUG_ON_PAGE(PageTail(page), page);
598 VM_BUG_ON_PAGE(index != round_down(index, nr), page);
599 VM_BUG_ON_PAGE(!PageLocked(page), page);
600 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
601 VM_BUG_ON(expected && PageTransHuge(page));
602
603 page_ref_add(page, nr);
604 page->mapping = mapping;
605 page->index = index;
606
607 do {
608 void *entry;
609 xas_lock_irq(&xas);
610 entry = xas_find_conflict(&xas);
611 if (entry != expected)
612 xas_set_err(&xas, -EEXIST);
613 xas_create_range(&xas);
614 if (xas_error(&xas))
615 goto unlock;
616next:
617 xas_store(&xas, page + i);
618 if (++i < nr) {
619 xas_next(&xas);
620 goto next;
621 }
622 if (PageTransHuge(page)) {
623 count_vm_event(THP_FILE_ALLOC);
624 __inc_node_page_state(page, NR_SHMEM_THPS);
625 }
626 mapping->nrpages += nr;
627 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
628 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr);
629unlock:
630 xas_unlock_irq(&xas);
631 } while (xas_nomem(&xas, gfp));
632
633 if (xas_error(&xas)) {
634 page->mapping = NULL;
635 page_ref_sub(page, nr);
636 return xas_error(&xas);
637 }
638
639 return 0;
640}
641
642/*
643 * Like delete_from_page_cache, but substitutes swap for page.
644 */
645static void shmem_delete_from_page_cache(struct page *page, void *radswap)
646{
647 struct address_space *mapping = page->mapping;
648 int error;
649
650 VM_BUG_ON_PAGE(PageCompound(page), page);
651
652 xa_lock_irq(&mapping->i_pages);
653 error = shmem_replace_entry(mapping, page->index, page, radswap);
654 page->mapping = NULL;
655 mapping->nrpages--;
656 __dec_node_page_state(page, NR_FILE_PAGES);
657 __dec_node_page_state(page, NR_SHMEM);
658 xa_unlock_irq(&mapping->i_pages);
659 put_page(page);
660 BUG_ON(error);
661}
662
663/*
664 * Remove swap entry from page cache, free the swap and its page cache.
665 */
666static int shmem_free_swap(struct address_space *mapping,
667 pgoff_t index, void *radswap)
668{
669 void *old;
670
671 old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
672 if (old != radswap)
673 return -ENOENT;
674 free_swap_and_cache(radix_to_swp_entry(radswap));
675 return 0;
676}
677
678/*
679 * Determine (in bytes) how many of the shmem object's pages mapped by the
680 * given offsets are swapped out.
681 *
682 * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
683 * as long as the inode doesn't go away and racy results are not a problem.
684 */
685unsigned long shmem_partial_swap_usage(struct address_space *mapping,
686 pgoff_t start, pgoff_t end)
687{
688 XA_STATE(xas, &mapping->i_pages, start);
689 struct page *page;
690 unsigned long swapped = 0;
691
692 rcu_read_lock();
693 xas_for_each(&xas, page, end - 1) {
694 if (xas_retry(&xas, page))
695 continue;
696 if (xa_is_value(page))
697 swapped++;
698
699 if (need_resched()) {
700 xas_pause(&xas);
701 cond_resched_rcu();
702 }
703 }
704
705 rcu_read_unlock();
706
707 return swapped << PAGE_SHIFT;
708}
709
710/*
711 * Determine (in bytes) how many of the shmem object's pages mapped by the
712 * given vma is swapped out.
713 *
714 * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
715 * as long as the inode doesn't go away and racy results are not a problem.
716 */
717unsigned long shmem_swap_usage(struct vm_area_struct *vma)
718{
719 struct inode *inode = file_inode(vma->vm_file);
720 struct shmem_inode_info *info = SHMEM_I(inode);
721 struct address_space *mapping = inode->i_mapping;
722 unsigned long swapped;
723
724 /* Be careful as we don't hold info->lock */
725 swapped = READ_ONCE(info->swapped);
726
727 /*
728 * The easier cases are when the shmem object has nothing in swap, or
729 * the vma maps it whole. Then we can simply use the stats that we
730 * already track.
731 */
732 if (!swapped)
733 return 0;
734
735 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
736 return swapped << PAGE_SHIFT;
737
738 /* Here comes the more involved part */
739 return shmem_partial_swap_usage(mapping,
740 linear_page_index(vma, vma->vm_start),
741 linear_page_index(vma, vma->vm_end));
742}
743
744/*
745 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
746 */
747void shmem_unlock_mapping(struct address_space *mapping)
748{
749 struct pagevec pvec;
750 pgoff_t indices[PAGEVEC_SIZE];
751 pgoff_t index = 0;
752
753 pagevec_init(&pvec);
754 /*
755 * Minor point, but we might as well stop if someone else SHM_LOCKs it.
756 */
757 while (!mapping_unevictable(mapping)) {
758 /*
759 * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
760 * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
761 */
762 pvec.nr = find_get_entries(mapping, index,
763 PAGEVEC_SIZE, pvec.pages, indices);
764 if (!pvec.nr)
765 break;
766 index = indices[pvec.nr - 1] + 1;
767 pagevec_remove_exceptionals(&pvec);
768 check_move_unevictable_pages(&pvec);
769 pagevec_release(&pvec);
770 cond_resched();
771 }
772}
773
774/*
775 * Remove range of pages and swap entries from page cache, and free them.
776 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
777 */
778static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
779 bool unfalloc)
780{
781 struct address_space *mapping = inode->i_mapping;
782 struct shmem_inode_info *info = SHMEM_I(inode);
783 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
784 pgoff_t end = (lend + 1) >> PAGE_SHIFT;
785 unsigned int partial_start = lstart & (PAGE_SIZE - 1);
786 unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1);
787 struct pagevec pvec;
788 pgoff_t indices[PAGEVEC_SIZE];
789 long nr_swaps_freed = 0;
790 pgoff_t index;
791 int i;
792
793 if (lend == -1)
794 end = -1; /* unsigned, so actually very big */
795
796 pagevec_init(&pvec);
797 index = start;
798 while (index < end) {
799 pvec.nr = find_get_entries(mapping, index,
800 min(end - index, (pgoff_t)PAGEVEC_SIZE),
801 pvec.pages, indices);
802 if (!pvec.nr)
803 break;
804 for (i = 0; i < pagevec_count(&pvec); i++) {
805 struct page *page = pvec.pages[i];
806
807 index = indices[i];
808 if (index >= end)
809 break;
810
811 if (xa_is_value(page)) {
812 if (unfalloc)
813 continue;
814 nr_swaps_freed += !shmem_free_swap(mapping,
815 index, page);
816 continue;
817 }
818
819 VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page);
820
821 if (!trylock_page(page))
822 continue;
823
824 if (PageTransTail(page)) {
825 /* Middle of THP: zero out the page */
826 clear_highpage(page);
827 unlock_page(page);
828 continue;
829 } else if (PageTransHuge(page)) {
830 if (index == round_down(end, HPAGE_PMD_NR)) {
831 /*
832 * Range ends in the middle of THP:
833 * zero out the page
834 */
835 clear_highpage(page);
836 unlock_page(page);
837 continue;
838 }
839 index += HPAGE_PMD_NR - 1;
840 i += HPAGE_PMD_NR - 1;
841 }
842
843 if (!unfalloc || !PageUptodate(page)) {
844 VM_BUG_ON_PAGE(PageTail(page), page);
845 if (page_mapping(page) == mapping) {
846 VM_BUG_ON_PAGE(PageWriteback(page), page);
847 truncate_inode_page(mapping, page);
848 }
849 }
850 unlock_page(page);
851 }
852 pagevec_remove_exceptionals(&pvec);
853 pagevec_release(&pvec);
854 cond_resched();
855 index++;
856 }
857
858 if (partial_start) {
859 struct page *page = NULL;
860 shmem_getpage(inode, start - 1, &page, SGP_READ);
861 if (page) {
862 unsigned int top = PAGE_SIZE;
863 if (start > end) {
864 top = partial_end;
865 partial_end = 0;
866 }
867 zero_user_segment(page, partial_start, top);
868 set_page_dirty(page);
869 unlock_page(page);
870 put_page(page);
871 }
872 }
873 if (partial_end) {
874 struct page *page = NULL;
875 shmem_getpage(inode, end, &page, SGP_READ);
876 if (page) {
877 zero_user_segment(page, 0, partial_end);
878 set_page_dirty(page);
879 unlock_page(page);
880 put_page(page);
881 }
882 }
883 if (start >= end)
884 return;
885
886 index = start;
887 while (index < end) {
888 cond_resched();
889
890 pvec.nr = find_get_entries(mapping, index,
891 min(end - index, (pgoff_t)PAGEVEC_SIZE),
892 pvec.pages, indices);
893 if (!pvec.nr) {
894 /* If all gone or hole-punch or unfalloc, we're done */
895 if (index == start || end != -1)
896 break;
897 /* But if truncating, restart to make sure all gone */
898 index = start;
899 continue;
900 }
901 for (i = 0; i < pagevec_count(&pvec); i++) {
902 struct page *page = pvec.pages[i];
903
904 index = indices[i];
905 if (index >= end)
906 break;
907
908 if (xa_is_value(page)) {
909 if (unfalloc)
910 continue;
911 if (shmem_free_swap(mapping, index, page)) {
912 /* Swap was replaced by page: retry */
913 index--;
914 break;
915 }
916 nr_swaps_freed++;
917 continue;
918 }
919
920 lock_page(page);
921
922 if (PageTransTail(page)) {
923 /* Middle of THP: zero out the page */
924 clear_highpage(page);
925 unlock_page(page);
926 /*
927 * Partial thp truncate due 'start' in middle
928 * of THP: don't need to look on these pages
929 * again on !pvec.nr restart.
930 */
931 if (index != round_down(end, HPAGE_PMD_NR))
932 start++;
933 continue;
934 } else if (PageTransHuge(page)) {
935 if (index == round_down(end, HPAGE_PMD_NR)) {
936 /*
937 * Range ends in the middle of THP:
938 * zero out the page
939 */
940 clear_highpage(page);
941 unlock_page(page);
942 continue;
943 }
944 index += HPAGE_PMD_NR - 1;
945 i += HPAGE_PMD_NR - 1;
946 }
947
948 if (!unfalloc || !PageUptodate(page)) {
949 VM_BUG_ON_PAGE(PageTail(page), page);
950 if (page_mapping(page) == mapping) {
951 VM_BUG_ON_PAGE(PageWriteback(page), page);
952 truncate_inode_page(mapping, page);
953 } else {
954 /* Page was replaced by swap: retry */
955 unlock_page(page);
956 index--;
957 break;
958 }
959 }
960 unlock_page(page);
961 }
962 pagevec_remove_exceptionals(&pvec);
963 pagevec_release(&pvec);
964 index++;
965 }
966
967 spin_lock_irq(&info->lock);
968 info->swapped -= nr_swaps_freed;
969 shmem_recalc_inode(inode);
970 spin_unlock_irq(&info->lock);
971}
972
973void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
974{
975 shmem_undo_range(inode, lstart, lend, false);
976 inode->i_ctime = inode->i_mtime = current_time(inode);
977}
978EXPORT_SYMBOL_GPL(shmem_truncate_range);
979
980static int shmem_getattr(const struct path *path, struct kstat *stat,
981 u32 request_mask, unsigned int query_flags)
982{
983 struct inode *inode = path->dentry->d_inode;
984 struct shmem_inode_info *info = SHMEM_I(inode);
985 struct shmem_sb_info *sb_info = SHMEM_SB(inode->i_sb);
986
987 if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
988 spin_lock_irq(&info->lock);
989 shmem_recalc_inode(inode);
990 spin_unlock_irq(&info->lock);
991 }
992 generic_fillattr(inode, stat);
993
994 if (is_huge_enabled(sb_info))
995 stat->blksize = HPAGE_PMD_SIZE;
996
997 return 0;
998}
999
1000static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
1001{
1002 struct inode *inode = d_inode(dentry);
1003 struct shmem_inode_info *info = SHMEM_I(inode);
1004 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1005 int error;
1006
1007 error = setattr_prepare(dentry, attr);
1008 if (error)
1009 return error;
1010
1011 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1012 loff_t oldsize = inode->i_size;
1013 loff_t newsize = attr->ia_size;
1014
1015 /* protected by i_mutex */
1016 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1017 (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1018 return -EPERM;
1019
1020 if (newsize != oldsize) {
1021 error = shmem_reacct_size(SHMEM_I(inode)->flags,
1022 oldsize, newsize);
1023 if (error)
1024 return error;
1025 i_size_write(inode, newsize);
1026 inode->i_ctime = inode->i_mtime = current_time(inode);
1027 }
1028 if (newsize <= oldsize) {
1029 loff_t holebegin = round_up(newsize, PAGE_SIZE);
1030 if (oldsize > holebegin)
1031 unmap_mapping_range(inode->i_mapping,
1032 holebegin, 0, 1);
1033 if (info->alloced)
1034 shmem_truncate_range(inode,
1035 newsize, (loff_t)-1);
1036 /* unmap again to remove racily COWed private pages */
1037 if (oldsize > holebegin)
1038 unmap_mapping_range(inode->i_mapping,
1039 holebegin, 0, 1);
1040
1041 /*
1042 * Part of the huge page can be beyond i_size: subject
1043 * to shrink under memory pressure.
1044 */
1045 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
1046 spin_lock(&sbinfo->shrinklist_lock);
1047 /*
1048 * _careful to defend against unlocked access to
1049 * ->shrink_list in shmem_unused_huge_shrink()
1050 */
1051 if (list_empty_careful(&info->shrinklist)) {
1052 list_add_tail(&info->shrinklist,
1053 &sbinfo->shrinklist);
1054 sbinfo->shrinklist_len++;
1055 }
1056 spin_unlock(&sbinfo->shrinklist_lock);
1057 }
1058 }
1059 }
1060
1061 setattr_copy(inode, attr);
1062 if (attr->ia_valid & ATTR_MODE)
1063 error = posix_acl_chmod(inode, inode->i_mode);
1064 return error;
1065}
1066
1067static void shmem_evict_inode(struct inode *inode)
1068{
1069 struct shmem_inode_info *info = SHMEM_I(inode);
1070 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1071
1072 if (inode->i_mapping->a_ops == &shmem_aops) {
1073 shmem_unacct_size(info->flags, inode->i_size);
1074 inode->i_size = 0;
1075 shmem_truncate_range(inode, 0, (loff_t)-1);
1076 if (!list_empty(&info->shrinklist)) {
1077 spin_lock(&sbinfo->shrinklist_lock);
1078 if (!list_empty(&info->shrinklist)) {
1079 list_del_init(&info->shrinklist);
1080 sbinfo->shrinklist_len--;
1081 }
1082 spin_unlock(&sbinfo->shrinklist_lock);
1083 }
1084 if (!list_empty(&info->swaplist)) {
1085 mutex_lock(&shmem_swaplist_mutex);
1086 list_del_init(&info->swaplist);
1087 mutex_unlock(&shmem_swaplist_mutex);
1088 }
1089 }
1090
1091 simple_xattrs_free(&info->xattrs);
1092 WARN_ON(inode->i_blocks);
1093 shmem_free_inode(inode->i_sb);
1094 clear_inode(inode);
1095}
1096
1097extern struct swap_info_struct *swap_info[];
1098
1099static int shmem_find_swap_entries(struct address_space *mapping,
1100 pgoff_t start, unsigned int nr_entries,
1101 struct page **entries, pgoff_t *indices,
1102 bool frontswap)
1103{
1104 XA_STATE(xas, &mapping->i_pages, start);
1105 struct page *page;
1106 unsigned int ret = 0;
1107
1108 if (!nr_entries)
1109 return 0;
1110
1111 rcu_read_lock();
1112 xas_for_each(&xas, page, ULONG_MAX) {
1113 if (xas_retry(&xas, page))
1114 continue;
1115
1116 if (!xa_is_value(page))
1117 continue;
1118
1119 if (frontswap) {
1120 swp_entry_t entry = radix_to_swp_entry(page);
1121
1122 if (!frontswap_test(swap_info[swp_type(entry)],
1123 swp_offset(entry)))
1124 continue;
1125 }
1126
1127 indices[ret] = xas.xa_index;
1128 entries[ret] = page;
1129
1130 if (need_resched()) {
1131 xas_pause(&xas);
1132 cond_resched_rcu();
1133 }
1134 if (++ret == nr_entries)
1135 break;
1136 }
1137 rcu_read_unlock();
1138
1139 return ret;
1140}
1141
1142/*
1143 * Move the swapped pages for an inode to page cache. Returns the count
1144 * of pages swapped in, or the error in case of failure.
1145 */
1146static int shmem_unuse_swap_entries(struct inode *inode, struct pagevec pvec,
1147 pgoff_t *indices)
1148{
1149 int i = 0;
1150 int ret = 0;
1151 int error = 0;
1152 struct address_space *mapping = inode->i_mapping;
1153
1154 for (i = 0; i < pvec.nr; i++) {
1155 struct page *page = pvec.pages[i];
1156
1157 if (!xa_is_value(page))
1158 continue;
1159 error = shmem_swapin_page(inode, indices[i],
1160 &page, SGP_CACHE,
1161 mapping_gfp_mask(mapping),
1162 NULL, NULL);
1163 if (error == 0) {
1164 unlock_page(page);
1165 put_page(page);
1166 ret++;
1167 }
1168 if (error == -ENOMEM)
1169 break;
1170 error = 0;
1171 }
1172 return error ? error : ret;
1173}
1174
1175/*
1176 * If swap found in inode, free it and move page from swapcache to filecache.
1177 */
1178static int shmem_unuse_inode(struct inode *inode, unsigned int type,
1179 bool frontswap, unsigned long *fs_pages_to_unuse)
1180{
1181 struct address_space *mapping = inode->i_mapping;
1182 pgoff_t start = 0;
1183 struct pagevec pvec;
1184 pgoff_t indices[PAGEVEC_SIZE];
1185 bool frontswap_partial = (frontswap && *fs_pages_to_unuse > 0);
1186 int ret = 0;
1187
1188 pagevec_init(&pvec);
1189 do {
1190 unsigned int nr_entries = PAGEVEC_SIZE;
1191
1192 if (frontswap_partial && *fs_pages_to_unuse < PAGEVEC_SIZE)
1193 nr_entries = *fs_pages_to_unuse;
1194
1195 pvec.nr = shmem_find_swap_entries(mapping, start, nr_entries,
1196 pvec.pages, indices,
1197 frontswap);
1198 if (pvec.nr == 0) {
1199 ret = 0;
1200 break;
1201 }
1202
1203 ret = shmem_unuse_swap_entries(inode, pvec, indices);
1204 if (ret < 0)
1205 break;
1206
1207 if (frontswap_partial) {
1208 *fs_pages_to_unuse -= ret;
1209 if (*fs_pages_to_unuse == 0) {
1210 ret = FRONTSWAP_PAGES_UNUSED;
1211 break;
1212 }
1213 }
1214
1215 start = indices[pvec.nr - 1];
1216 } while (true);
1217
1218 return ret;
1219}
1220
1221/*
1222 * Read all the shared memory data that resides in the swap
1223 * device 'type' back into memory, so the swap device can be
1224 * unused.
1225 */
1226int shmem_unuse(unsigned int type, bool frontswap,
1227 unsigned long *fs_pages_to_unuse)
1228{
1229 struct shmem_inode_info *info, *next;
1230 struct inode *inode;
1231 struct inode *prev_inode = NULL;
1232 int error = 0;
1233
1234 if (list_empty(&shmem_swaplist))
1235 return 0;
1236
1237 mutex_lock(&shmem_swaplist_mutex);
1238
1239 /*
1240 * The extra refcount on the inode is necessary to safely dereference
1241 * p->next after re-acquiring the lock. New shmem inodes with swap
1242 * get added to the end of the list and we will scan them all.
1243 */
1244 list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1245 if (!info->swapped) {
1246 list_del_init(&info->swaplist);
1247 continue;
1248 }
1249
1250 inode = igrab(&info->vfs_inode);
1251 if (!inode)
1252 continue;
1253
1254 mutex_unlock(&shmem_swaplist_mutex);
1255 if (prev_inode)
1256 iput(prev_inode);
1257 prev_inode = inode;
1258
1259 error = shmem_unuse_inode(inode, type, frontswap,
1260 fs_pages_to_unuse);
1261 cond_resched();
1262
1263 mutex_lock(&shmem_swaplist_mutex);
1264 next = list_next_entry(info, swaplist);
1265 if (!info->swapped)
1266 list_del_init(&info->swaplist);
1267 if (error)
1268 break;
1269 }
1270 mutex_unlock(&shmem_swaplist_mutex);
1271
1272 if (prev_inode)
1273 iput(prev_inode);
1274
1275 return error;
1276}
1277
1278/*
1279 * Move the page from the page cache to the swap cache.
1280 */
1281static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1282{
1283 struct shmem_inode_info *info;
1284 struct address_space *mapping;
1285 struct inode *inode;
1286 swp_entry_t swap;
1287 pgoff_t index;
1288
1289 VM_BUG_ON_PAGE(PageCompound(page), page);
1290 BUG_ON(!PageLocked(page));
1291 mapping = page->mapping;
1292 index = page->index;
1293 inode = mapping->host;
1294 info = SHMEM_I(inode);
1295 if (info->flags & VM_LOCKED)
1296 goto redirty;
1297 if (!total_swap_pages)
1298 goto redirty;
1299
1300 /*
1301 * Our capabilities prevent regular writeback or sync from ever calling
1302 * shmem_writepage; but a stacking filesystem might use ->writepage of
1303 * its underlying filesystem, in which case tmpfs should write out to
1304 * swap only in response to memory pressure, and not for the writeback
1305 * threads or sync.
1306 */
1307 if (!wbc->for_reclaim) {
1308 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */
1309 goto redirty;
1310 }
1311
1312 /*
1313 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1314 * value into swapfile.c, the only way we can correctly account for a
1315 * fallocated page arriving here is now to initialize it and write it.
1316 *
1317 * That's okay for a page already fallocated earlier, but if we have
1318 * not yet completed the fallocation, then (a) we want to keep track
1319 * of this page in case we have to undo it, and (b) it may not be a
1320 * good idea to continue anyway, once we're pushing into swap. So
1321 * reactivate the page, and let shmem_fallocate() quit when too many.
1322 */
1323 if (!PageUptodate(page)) {
1324 if (inode->i_private) {
1325 struct shmem_falloc *shmem_falloc;
1326 spin_lock(&inode->i_lock);
1327 shmem_falloc = inode->i_private;
1328 if (shmem_falloc &&
1329 !shmem_falloc->waitq &&
1330 index >= shmem_falloc->start &&
1331 index < shmem_falloc->next)
1332 shmem_falloc->nr_unswapped++;
1333 else
1334 shmem_falloc = NULL;
1335 spin_unlock(&inode->i_lock);
1336 if (shmem_falloc)
1337 goto redirty;
1338 }
1339 clear_highpage(page);
1340 flush_dcache_page(page);
1341 SetPageUptodate(page);
1342 }
1343
1344 swap = get_swap_page(page);
1345 if (!swap.val)
1346 goto redirty;
1347
1348 /*
1349 * Add inode to shmem_unuse()'s list of swapped-out inodes,
1350 * if it's not already there. Do it now before the page is
1351 * moved to swap cache, when its pagelock no longer protects
1352 * the inode from eviction. But don't unlock the mutex until
1353 * we've incremented swapped, because shmem_unuse_inode() will
1354 * prune a !swapped inode from the swaplist under this mutex.
1355 */
1356 mutex_lock(&shmem_swaplist_mutex);
1357 if (list_empty(&info->swaplist))
1358 list_add(&info->swaplist, &shmem_swaplist);
1359
1360 if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1361 spin_lock_irq(&info->lock);
1362 shmem_recalc_inode(inode);
1363 info->swapped++;
1364 spin_unlock_irq(&info->lock);
1365
1366 swap_shmem_alloc(swap);
1367 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
1368
1369 mutex_unlock(&shmem_swaplist_mutex);
1370 BUG_ON(page_mapped(page));
1371 swap_writepage(page, wbc);
1372 return 0;
1373 }
1374
1375 mutex_unlock(&shmem_swaplist_mutex);
1376 put_swap_page(page, swap);
1377redirty:
1378 set_page_dirty(page);
1379 if (wbc->for_reclaim)
1380 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */
1381 unlock_page(page);
1382 return 0;
1383}
1384
1385#if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1386static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1387{
1388 char buffer[64];
1389
1390 if (!mpol || mpol->mode == MPOL_DEFAULT)
1391 return; /* show nothing */
1392
1393 mpol_to_str(buffer, sizeof(buffer), mpol);
1394
1395 seq_printf(seq, ",mpol=%s", buffer);
1396}
1397
1398static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1399{
1400 struct mempolicy *mpol = NULL;
1401 if (sbinfo->mpol) {
1402 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */
1403 mpol = sbinfo->mpol;
1404 mpol_get(mpol);
1405 spin_unlock(&sbinfo->stat_lock);
1406 }
1407 return mpol;
1408}
1409#else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1410static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1411{
1412}
1413static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1414{
1415 return NULL;
1416}
1417#endif /* CONFIG_NUMA && CONFIG_TMPFS */
1418#ifndef CONFIG_NUMA
1419#define vm_policy vm_private_data
1420#endif
1421
1422static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1423 struct shmem_inode_info *info, pgoff_t index)
1424{
1425 /* Create a pseudo vma that just contains the policy */
1426 vma_init(vma, NULL);
1427 /* Bias interleave by inode number to distribute better across nodes */
1428 vma->vm_pgoff = index + info->vfs_inode.i_ino;
1429 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1430}
1431
1432static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1433{
1434 /* Drop reference taken by mpol_shared_policy_lookup() */
1435 mpol_cond_put(vma->vm_policy);
1436}
1437
1438static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1439 struct shmem_inode_info *info, pgoff_t index)
1440{
1441 struct vm_area_struct pvma;
1442 struct page *page;
1443 struct vm_fault vmf;
1444
1445 shmem_pseudo_vma_init(&pvma, info, index);
1446 vmf.vma = &pvma;
1447 vmf.address = 0;
1448 page = swap_cluster_readahead(swap, gfp, &vmf);
1449 shmem_pseudo_vma_destroy(&pvma);
1450
1451 return page;
1452}
1453
1454static struct page *shmem_alloc_hugepage(gfp_t gfp,
1455 struct shmem_inode_info *info, pgoff_t index)
1456{
1457 struct vm_area_struct pvma;
1458 struct address_space *mapping = info->vfs_inode.i_mapping;
1459 pgoff_t hindex;
1460 struct page *page;
1461
1462 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1463 return NULL;
1464
1465 hindex = round_down(index, HPAGE_PMD_NR);
1466 if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1467 XA_PRESENT))
1468 return NULL;
1469
1470 shmem_pseudo_vma_init(&pvma, info, hindex);
1471 page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN,
1472 HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true);
1473 shmem_pseudo_vma_destroy(&pvma);
1474 if (page)
1475 prep_transhuge_page(page);
1476 return page;
1477}
1478
1479static struct page *shmem_alloc_page(gfp_t gfp,
1480 struct shmem_inode_info *info, pgoff_t index)
1481{
1482 struct vm_area_struct pvma;
1483 struct page *page;
1484
1485 shmem_pseudo_vma_init(&pvma, info, index);
1486 page = alloc_page_vma(gfp, &pvma, 0);
1487 shmem_pseudo_vma_destroy(&pvma);
1488
1489 return page;
1490}
1491
1492static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1493 struct inode *inode,
1494 pgoff_t index, bool huge)
1495{
1496 struct shmem_inode_info *info = SHMEM_I(inode);
1497 struct page *page;
1498 int nr;
1499 int err = -ENOSPC;
1500
1501 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1502 huge = false;
1503 nr = huge ? HPAGE_PMD_NR : 1;
1504
1505 if (!shmem_inode_acct_block(inode, nr))
1506 goto failed;
1507
1508 if (huge)
1509 page = shmem_alloc_hugepage(gfp, info, index);
1510 else
1511 page = shmem_alloc_page(gfp, info, index);
1512 if (page) {
1513 __SetPageLocked(page);
1514 __SetPageSwapBacked(page);
1515 return page;
1516 }
1517
1518 err = -ENOMEM;
1519 shmem_inode_unacct_blocks(inode, nr);
1520failed:
1521 return ERR_PTR(err);
1522}
1523
1524/*
1525 * When a page is moved from swapcache to shmem filecache (either by the
1526 * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
1527 * shmem_unuse_inode()), it may have been read in earlier from swap, in
1528 * ignorance of the mapping it belongs to. If that mapping has special
1529 * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1530 * we may need to copy to a suitable page before moving to filecache.
1531 *
1532 * In a future release, this may well be extended to respect cpuset and
1533 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1534 * but for now it is a simple matter of zone.
1535 */
1536static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
1537{
1538 return page_zonenum(page) > gfp_zone(gfp);
1539}
1540
1541static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1542 struct shmem_inode_info *info, pgoff_t index)
1543{
1544 struct page *oldpage, *newpage;
1545 struct address_space *swap_mapping;
1546 swp_entry_t entry;
1547 pgoff_t swap_index;
1548 int error;
1549
1550 oldpage = *pagep;
1551 entry.val = page_private(oldpage);
1552 swap_index = swp_offset(entry);
1553 swap_mapping = page_mapping(oldpage);
1554
1555 /*
1556 * We have arrived here because our zones are constrained, so don't
1557 * limit chance of success by further cpuset and node constraints.
1558 */
1559 gfp &= ~GFP_CONSTRAINT_MASK;
1560 newpage = shmem_alloc_page(gfp, info, index);
1561 if (!newpage)
1562 return -ENOMEM;
1563
1564 get_page(newpage);
1565 copy_highpage(newpage, oldpage);
1566 flush_dcache_page(newpage);
1567
1568 __SetPageLocked(newpage);
1569 __SetPageSwapBacked(newpage);
1570 SetPageUptodate(newpage);
1571 set_page_private(newpage, entry.val);
1572 SetPageSwapCache(newpage);
1573
1574 /*
1575 * Our caller will very soon move newpage out of swapcache, but it's
1576 * a nice clean interface for us to replace oldpage by newpage there.
1577 */
1578 xa_lock_irq(&swap_mapping->i_pages);
1579 error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage);
1580 if (!error) {
1581 __inc_node_page_state(newpage, NR_FILE_PAGES);
1582 __dec_node_page_state(oldpage, NR_FILE_PAGES);
1583 }
1584 xa_unlock_irq(&swap_mapping->i_pages);
1585
1586 if (unlikely(error)) {
1587 /*
1588 * Is this possible? I think not, now that our callers check
1589 * both PageSwapCache and page_private after getting page lock;
1590 * but be defensive. Reverse old to newpage for clear and free.
1591 */
1592 oldpage = newpage;
1593 } else {
1594 mem_cgroup_migrate(oldpage, newpage);
1595 lru_cache_add_anon(newpage);
1596 *pagep = newpage;
1597 }
1598
1599 ClearPageSwapCache(oldpage);
1600 set_page_private(oldpage, 0);
1601
1602 unlock_page(oldpage);
1603 put_page(oldpage);
1604 put_page(oldpage);
1605 return error;
1606}
1607
1608/*
1609 * Swap in the page pointed to by *pagep.
1610 * Caller has to make sure that *pagep contains a valid swapped page.
1611 * Returns 0 and the page in pagep if success. On failure, returns the
1612 * the error code and NULL in *pagep.
1613 */
1614static int shmem_swapin_page(struct inode *inode, pgoff_t index,
1615 struct page **pagep, enum sgp_type sgp,
1616 gfp_t gfp, struct vm_area_struct *vma,
1617 vm_fault_t *fault_type)
1618{
1619 struct address_space *mapping = inode->i_mapping;
1620 struct shmem_inode_info *info = SHMEM_I(inode);
1621 struct mm_struct *charge_mm = vma ? vma->vm_mm : current->mm;
1622 struct mem_cgroup *memcg;
1623 struct page *page;
1624 swp_entry_t swap;
1625 int error;
1626
1627 VM_BUG_ON(!*pagep || !xa_is_value(*pagep));
1628 swap = radix_to_swp_entry(*pagep);
1629 *pagep = NULL;
1630
1631 /* Look it up and read it in.. */
1632 page = lookup_swap_cache(swap, NULL, 0);
1633 if (!page) {
1634 /* Or update major stats only when swapin succeeds?? */
1635 if (fault_type) {
1636 *fault_type |= VM_FAULT_MAJOR;
1637 count_vm_event(PGMAJFAULT);
1638 count_memcg_event_mm(charge_mm, PGMAJFAULT);
1639 }
1640 /* Here we actually start the io */
1641 page = shmem_swapin(swap, gfp, info, index);
1642 if (!page) {
1643 error = -ENOMEM;
1644 goto failed;
1645 }
1646 }
1647
1648 /* We have to do this with page locked to prevent races */
1649 lock_page(page);
1650 if (!PageSwapCache(page) || page_private(page) != swap.val ||
1651 !shmem_confirm_swap(mapping, index, swap)) {
1652 error = -EEXIST;
1653 goto unlock;
1654 }
1655 if (!PageUptodate(page)) {
1656 error = -EIO;
1657 goto failed;
1658 }
1659 wait_on_page_writeback(page);
1660
1661 if (shmem_should_replace_page(page, gfp)) {
1662 error = shmem_replace_page(&page, gfp, info, index);
1663 if (error)
1664 goto failed;
1665 }
1666
1667 error = mem_cgroup_try_charge_delay(page, charge_mm, gfp, &memcg,
1668 false);
1669 if (!error) {
1670 error = shmem_add_to_page_cache(page, mapping, index,
1671 swp_to_radix_entry(swap), gfp);
1672 /*
1673 * We already confirmed swap under page lock, and make
1674 * no memory allocation here, so usually no possibility
1675 * of error; but free_swap_and_cache() only trylocks a
1676 * page, so it is just possible that the entry has been
1677 * truncated or holepunched since swap was confirmed.
1678 * shmem_undo_range() will have done some of the
1679 * unaccounting, now delete_from_swap_cache() will do
1680 * the rest.
1681 */
1682 if (error) {
1683 mem_cgroup_cancel_charge(page, memcg, false);
1684 delete_from_swap_cache(page);
1685 }
1686 }
1687 if (error)
1688 goto failed;
1689
1690 mem_cgroup_commit_charge(page, memcg, true, false);
1691
1692 spin_lock_irq(&info->lock);
1693 info->swapped--;
1694 shmem_recalc_inode(inode);
1695 spin_unlock_irq(&info->lock);
1696
1697 if (sgp == SGP_WRITE)
1698 mark_page_accessed(page);
1699
1700 delete_from_swap_cache(page);
1701 set_page_dirty(page);
1702 swap_free(swap);
1703
1704 *pagep = page;
1705 return 0;
1706failed:
1707 if (!shmem_confirm_swap(mapping, index, swap))
1708 error = -EEXIST;
1709unlock:
1710 if (page) {
1711 unlock_page(page);
1712 put_page(page);
1713 }
1714
1715 return error;
1716}
1717
1718/*
1719 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1720 *
1721 * If we allocate a new one we do not mark it dirty. That's up to the
1722 * vm. If we swap it in we mark it dirty since we also free the swap
1723 * entry since a page cannot live in both the swap and page cache.
1724 *
1725 * fault_mm and fault_type are only supplied by shmem_fault:
1726 * otherwise they are NULL.
1727 */
1728static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1729 struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1730 struct vm_area_struct *vma, struct vm_fault *vmf,
1731 vm_fault_t *fault_type)
1732{
1733 struct address_space *mapping = inode->i_mapping;
1734 struct shmem_inode_info *info = SHMEM_I(inode);
1735 struct shmem_sb_info *sbinfo;
1736 struct mm_struct *charge_mm;
1737 struct mem_cgroup *memcg;
1738 struct page *page;
1739 enum sgp_type sgp_huge = sgp;
1740 pgoff_t hindex = index;
1741 int error;
1742 int once = 0;
1743 int alloced = 0;
1744
1745 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1746 return -EFBIG;
1747 if (sgp == SGP_NOHUGE || sgp == SGP_HUGE)
1748 sgp = SGP_CACHE;
1749repeat:
1750 if (sgp <= SGP_CACHE &&
1751 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1752 return -EINVAL;
1753 }
1754
1755 sbinfo = SHMEM_SB(inode->i_sb);
1756 charge_mm = vma ? vma->vm_mm : current->mm;
1757
1758 page = find_lock_entry(mapping, index);
1759 if (xa_is_value(page)) {
1760 error = shmem_swapin_page(inode, index, &page,
1761 sgp, gfp, vma, fault_type);
1762 if (error == -EEXIST)
1763 goto repeat;
1764
1765 *pagep = page;
1766 return error;
1767 }
1768
1769 if (page && sgp == SGP_WRITE)
1770 mark_page_accessed(page);
1771
1772 /* fallocated page? */
1773 if (page && !PageUptodate(page)) {
1774 if (sgp != SGP_READ)
1775 goto clear;
1776 unlock_page(page);
1777 put_page(page);
1778 page = NULL;
1779 }
1780 if (page || sgp == SGP_READ) {
1781 *pagep = page;
1782 return 0;
1783 }
1784
1785 /*
1786 * Fast cache lookup did not find it:
1787 * bring it back from swap or allocate.
1788 */
1789
1790 if (vma && userfaultfd_missing(vma)) {
1791 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1792 return 0;
1793 }
1794
1795 /* shmem_symlink() */
1796 if (mapping->a_ops != &shmem_aops)
1797 goto alloc_nohuge;
1798 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE)
1799 goto alloc_nohuge;
1800 if (shmem_huge == SHMEM_HUGE_FORCE)
1801 goto alloc_huge;
1802 switch (sbinfo->huge) {
1803 loff_t i_size;
1804 pgoff_t off;
1805 case SHMEM_HUGE_NEVER:
1806 goto alloc_nohuge;
1807 case SHMEM_HUGE_WITHIN_SIZE:
1808 off = round_up(index, HPAGE_PMD_NR);
1809 i_size = round_up(i_size_read(inode), PAGE_SIZE);
1810 if (i_size >= HPAGE_PMD_SIZE &&
1811 i_size >> PAGE_SHIFT >= off)
1812 goto alloc_huge;
1813 /* fallthrough */
1814 case SHMEM_HUGE_ADVISE:
1815 if (sgp_huge == SGP_HUGE)
1816 goto alloc_huge;
1817 /* TODO: implement fadvise() hints */
1818 goto alloc_nohuge;
1819 }
1820
1821alloc_huge:
1822 page = shmem_alloc_and_acct_page(gfp, inode, index, true);
1823 if (IS_ERR(page)) {
1824alloc_nohuge:
1825 page = shmem_alloc_and_acct_page(gfp, inode,
1826 index, false);
1827 }
1828 if (IS_ERR(page)) {
1829 int retry = 5;
1830
1831 error = PTR_ERR(page);
1832 page = NULL;
1833 if (error != -ENOSPC)
1834 goto unlock;
1835 /*
1836 * Try to reclaim some space by splitting a huge page
1837 * beyond i_size on the filesystem.
1838 */
1839 while (retry--) {
1840 int ret;
1841
1842 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1843 if (ret == SHRINK_STOP)
1844 break;
1845 if (ret)
1846 goto alloc_nohuge;
1847 }
1848 goto unlock;
1849 }
1850
1851 if (PageTransHuge(page))
1852 hindex = round_down(index, HPAGE_PMD_NR);
1853 else
1854 hindex = index;
1855
1856 if (sgp == SGP_WRITE)
1857 __SetPageReferenced(page);
1858
1859 error = mem_cgroup_try_charge_delay(page, charge_mm, gfp, &memcg,
1860 PageTransHuge(page));
1861 if (error)
1862 goto unacct;
1863 error = shmem_add_to_page_cache(page, mapping, hindex,
1864 NULL, gfp & GFP_RECLAIM_MASK);
1865 if (error) {
1866 mem_cgroup_cancel_charge(page, memcg,
1867 PageTransHuge(page));
1868 goto unacct;
1869 }
1870 mem_cgroup_commit_charge(page, memcg, false,
1871 PageTransHuge(page));
1872 lru_cache_add_anon(page);
1873
1874 spin_lock_irq(&info->lock);
1875 info->alloced += 1 << compound_order(page);
1876 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page);
1877 shmem_recalc_inode(inode);
1878 spin_unlock_irq(&info->lock);
1879 alloced = true;
1880
1881 if (PageTransHuge(page) &&
1882 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1883 hindex + HPAGE_PMD_NR - 1) {
1884 /*
1885 * Part of the huge page is beyond i_size: subject
1886 * to shrink under memory pressure.
1887 */
1888 spin_lock(&sbinfo->shrinklist_lock);
1889 /*
1890 * _careful to defend against unlocked access to
1891 * ->shrink_list in shmem_unused_huge_shrink()
1892 */
1893 if (list_empty_careful(&info->shrinklist)) {
1894 list_add_tail(&info->shrinklist,
1895 &sbinfo->shrinklist);
1896 sbinfo->shrinklist_len++;
1897 }
1898 spin_unlock(&sbinfo->shrinklist_lock);
1899 }
1900
1901 /*
1902 * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1903 */
1904 if (sgp == SGP_FALLOC)
1905 sgp = SGP_WRITE;
1906clear:
1907 /*
1908 * Let SGP_WRITE caller clear ends if write does not fill page;
1909 * but SGP_FALLOC on a page fallocated earlier must initialize
1910 * it now, lest undo on failure cancel our earlier guarantee.
1911 */
1912 if (sgp != SGP_WRITE && !PageUptodate(page)) {
1913 struct page *head = compound_head(page);
1914 int i;
1915
1916 for (i = 0; i < (1 << compound_order(head)); i++) {
1917 clear_highpage(head + i);
1918 flush_dcache_page(head + i);
1919 }
1920 SetPageUptodate(head);
1921 }
1922
1923 /* Perhaps the file has been truncated since we checked */
1924 if (sgp <= SGP_CACHE &&
1925 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1926 if (alloced) {
1927 ClearPageDirty(page);
1928 delete_from_page_cache(page);
1929 spin_lock_irq(&info->lock);
1930 shmem_recalc_inode(inode);
1931 spin_unlock_irq(&info->lock);
1932 }
1933 error = -EINVAL;
1934 goto unlock;
1935 }
1936 *pagep = page + index - hindex;
1937 return 0;
1938
1939 /*
1940 * Error recovery.
1941 */
1942unacct:
1943 shmem_inode_unacct_blocks(inode, 1 << compound_order(page));
1944
1945 if (PageTransHuge(page)) {
1946 unlock_page(page);
1947 put_page(page);
1948 goto alloc_nohuge;
1949 }
1950unlock:
1951 if (page) {
1952 unlock_page(page);
1953 put_page(page);
1954 }
1955 if (error == -ENOSPC && !once++) {
1956 spin_lock_irq(&info->lock);
1957 shmem_recalc_inode(inode);
1958 spin_unlock_irq(&info->lock);
1959 goto repeat;
1960 }
1961 if (error == -EEXIST)
1962 goto repeat;
1963 return error;
1964}
1965
1966/*
1967 * This is like autoremove_wake_function, but it removes the wait queue
1968 * entry unconditionally - even if something else had already woken the
1969 * target.
1970 */
1971static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
1972{
1973 int ret = default_wake_function(wait, mode, sync, key);
1974 list_del_init(&wait->entry);
1975 return ret;
1976}
1977
1978static vm_fault_t shmem_fault(struct vm_fault *vmf)
1979{
1980 struct vm_area_struct *vma = vmf->vma;
1981 struct inode *inode = file_inode(vma->vm_file);
1982 gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
1983 enum sgp_type sgp;
1984 int err;
1985 vm_fault_t ret = VM_FAULT_LOCKED;
1986
1987 /*
1988 * Trinity finds that probing a hole which tmpfs is punching can
1989 * prevent the hole-punch from ever completing: which in turn
1990 * locks writers out with its hold on i_mutex. So refrain from
1991 * faulting pages into the hole while it's being punched. Although
1992 * shmem_undo_range() does remove the additions, it may be unable to
1993 * keep up, as each new page needs its own unmap_mapping_range() call,
1994 * and the i_mmap tree grows ever slower to scan if new vmas are added.
1995 *
1996 * It does not matter if we sometimes reach this check just before the
1997 * hole-punch begins, so that one fault then races with the punch:
1998 * we just need to make racing faults a rare case.
1999 *
2000 * The implementation below would be much simpler if we just used a
2001 * standard mutex or completion: but we cannot take i_mutex in fault,
2002 * and bloating every shmem inode for this unlikely case would be sad.
2003 */
2004 if (unlikely(inode->i_private)) {
2005 struct shmem_falloc *shmem_falloc;
2006
2007 spin_lock(&inode->i_lock);
2008 shmem_falloc = inode->i_private;
2009 if (shmem_falloc &&
2010 shmem_falloc->waitq &&
2011 vmf->pgoff >= shmem_falloc->start &&
2012 vmf->pgoff < shmem_falloc->next) {
2013 wait_queue_head_t *shmem_falloc_waitq;
2014 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2015
2016 ret = VM_FAULT_NOPAGE;
2017 if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) &&
2018 !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
2019 /* It's polite to up mmap_sem if we can */
2020 up_read(&vma->vm_mm->mmap_sem);
2021 ret = VM_FAULT_RETRY;
2022 }
2023
2024 shmem_falloc_waitq = shmem_falloc->waitq;
2025 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
2026 TASK_UNINTERRUPTIBLE);
2027 spin_unlock(&inode->i_lock);
2028 schedule();
2029
2030 /*
2031 * shmem_falloc_waitq points into the shmem_fallocate()
2032 * stack of the hole-punching task: shmem_falloc_waitq
2033 * is usually invalid by the time we reach here, but
2034 * finish_wait() does not dereference it in that case;
2035 * though i_lock needed lest racing with wake_up_all().
2036 */
2037 spin_lock(&inode->i_lock);
2038 finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2039 spin_unlock(&inode->i_lock);
2040 return ret;
2041 }
2042 spin_unlock(&inode->i_lock);
2043 }
2044
2045 sgp = SGP_CACHE;
2046
2047 if ((vma->vm_flags & VM_NOHUGEPAGE) ||
2048 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
2049 sgp = SGP_NOHUGE;
2050 else if (vma->vm_flags & VM_HUGEPAGE)
2051 sgp = SGP_HUGE;
2052
2053 err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,
2054 gfp, vma, vmf, &ret);
2055 if (err)
2056 return vmf_error(err);
2057 return ret;
2058}
2059
2060unsigned long shmem_get_unmapped_area(struct file *file,
2061 unsigned long uaddr, unsigned long len,
2062 unsigned long pgoff, unsigned long flags)
2063{
2064 unsigned long (*get_area)(struct file *,
2065 unsigned long, unsigned long, unsigned long, unsigned long);
2066 unsigned long addr;
2067 unsigned long offset;
2068 unsigned long inflated_len;
2069 unsigned long inflated_addr;
2070 unsigned long inflated_offset;
2071
2072 if (len > TASK_SIZE)
2073 return -ENOMEM;
2074
2075 get_area = current->mm->get_unmapped_area;
2076 addr = get_area(file, uaddr, len, pgoff, flags);
2077
2078 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
2079 return addr;
2080 if (IS_ERR_VALUE(addr))
2081 return addr;
2082 if (addr & ~PAGE_MASK)
2083 return addr;
2084 if (addr > TASK_SIZE - len)
2085 return addr;
2086
2087 if (shmem_huge == SHMEM_HUGE_DENY)
2088 return addr;
2089 if (len < HPAGE_PMD_SIZE)
2090 return addr;
2091 if (flags & MAP_FIXED)
2092 return addr;
2093 /*
2094 * Our priority is to support MAP_SHARED mapped hugely;
2095 * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2096 * But if caller specified an address hint, respect that as before.
2097 */
2098 if (uaddr)
2099 return addr;
2100
2101 if (shmem_huge != SHMEM_HUGE_FORCE) {
2102 struct super_block *sb;
2103
2104 if (file) {
2105 VM_BUG_ON(file->f_op != &shmem_file_operations);
2106 sb = file_inode(file)->i_sb;
2107 } else {
2108 /*
2109 * Called directly from mm/mmap.c, or drivers/char/mem.c
2110 * for "/dev/zero", to create a shared anonymous object.
2111 */
2112 if (IS_ERR(shm_mnt))
2113 return addr;
2114 sb = shm_mnt->mnt_sb;
2115 }
2116 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2117 return addr;
2118 }
2119
2120 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2121 if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2122 return addr;
2123 if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2124 return addr;
2125
2126 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2127 if (inflated_len > TASK_SIZE)
2128 return addr;
2129 if (inflated_len < len)
2130 return addr;
2131
2132 inflated_addr = get_area(NULL, 0, inflated_len, 0, flags);
2133 if (IS_ERR_VALUE(inflated_addr))
2134 return addr;
2135 if (inflated_addr & ~PAGE_MASK)
2136 return addr;
2137
2138 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2139 inflated_addr += offset - inflated_offset;
2140 if (inflated_offset > offset)
2141 inflated_addr += HPAGE_PMD_SIZE;
2142
2143 if (inflated_addr > TASK_SIZE - len)
2144 return addr;
2145 return inflated_addr;
2146}
2147
2148#ifdef CONFIG_NUMA
2149static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2150{
2151 struct inode *inode = file_inode(vma->vm_file);
2152 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2153}
2154
2155static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2156 unsigned long addr)
2157{
2158 struct inode *inode = file_inode(vma->vm_file);
2159 pgoff_t index;
2160
2161 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2162 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2163}
2164#endif
2165
2166int shmem_lock(struct file *file, int lock, struct user_struct *user)
2167{
2168 struct inode *inode = file_inode(file);
2169 struct shmem_inode_info *info = SHMEM_I(inode);
2170 int retval = -ENOMEM;
2171
2172 spin_lock_irq(&info->lock);
2173 if (lock && !(info->flags & VM_LOCKED)) {
2174 if (!user_shm_lock(inode->i_size, user))
2175 goto out_nomem;
2176 info->flags |= VM_LOCKED;
2177 mapping_set_unevictable(file->f_mapping);
2178 }
2179 if (!lock && (info->flags & VM_LOCKED) && user) {
2180 user_shm_unlock(inode->i_size, user);
2181 info->flags &= ~VM_LOCKED;
2182 mapping_clear_unevictable(file->f_mapping);
2183 }
2184 retval = 0;
2185
2186out_nomem:
2187 spin_unlock_irq(&info->lock);
2188 return retval;
2189}
2190
2191static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2192{
2193 struct shmem_inode_info *info = SHMEM_I(file_inode(file));
2194
2195 if (info->seals & F_SEAL_FUTURE_WRITE) {
2196 /*
2197 * New PROT_WRITE and MAP_SHARED mmaps are not allowed when
2198 * "future write" seal active.
2199 */
2200 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
2201 return -EPERM;
2202
2203 /*
2204 * Since the F_SEAL_FUTURE_WRITE seals allow for a MAP_SHARED
2205 * read-only mapping, take care to not allow mprotect to revert
2206 * protections.
2207 */
2208 vma->vm_flags &= ~(VM_MAYWRITE);
2209 }
2210
2211 file_accessed(file);
2212 vma->vm_ops = &shmem_vm_ops;
2213 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
2214 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
2215 (vma->vm_end & HPAGE_PMD_MASK)) {
2216 khugepaged_enter(vma, vma->vm_flags);
2217 }
2218 return 0;
2219}
2220
2221static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
2222 umode_t mode, dev_t dev, unsigned long flags)
2223{
2224 struct inode *inode;
2225 struct shmem_inode_info *info;
2226 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2227
2228 if (shmem_reserve_inode(sb))
2229 return NULL;
2230
2231 inode = new_inode(sb);
2232 if (inode) {
2233 inode->i_ino = get_next_ino();
2234 inode_init_owner(inode, dir, mode);
2235 inode->i_blocks = 0;
2236 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2237 inode->i_generation = prandom_u32();
2238 info = SHMEM_I(inode);
2239 memset(info, 0, (char *)inode - (char *)info);
2240 spin_lock_init(&info->lock);
2241 info->seals = F_SEAL_SEAL;
2242 info->flags = flags & VM_NORESERVE;
2243 INIT_LIST_HEAD(&info->shrinklist);
2244 INIT_LIST_HEAD(&info->swaplist);
2245 simple_xattrs_init(&info->xattrs);
2246 cache_no_acl(inode);
2247
2248 switch (mode & S_IFMT) {
2249 default:
2250 inode->i_op = &shmem_special_inode_operations;
2251 init_special_inode(inode, mode, dev);
2252 break;
2253 case S_IFREG:
2254 inode->i_mapping->a_ops = &shmem_aops;
2255 inode->i_op = &shmem_inode_operations;
2256 inode->i_fop = &shmem_file_operations;
2257 mpol_shared_policy_init(&info->policy,
2258 shmem_get_sbmpol(sbinfo));
2259 break;
2260 case S_IFDIR:
2261 inc_nlink(inode);
2262 /* Some things misbehave if size == 0 on a directory */
2263 inode->i_size = 2 * BOGO_DIRENT_SIZE;
2264 inode->i_op = &shmem_dir_inode_operations;
2265 inode->i_fop = &simple_dir_operations;
2266 break;
2267 case S_IFLNK:
2268 /*
2269 * Must not load anything in the rbtree,
2270 * mpol_free_shared_policy will not be called.
2271 */
2272 mpol_shared_policy_init(&info->policy, NULL);
2273 break;
2274 }
2275
2276 lockdep_annotate_inode_mutex_key(inode);
2277 } else
2278 shmem_free_inode(sb);
2279 return inode;
2280}
2281
2282bool shmem_mapping(struct address_space *mapping)
2283{
2284 return mapping->a_ops == &shmem_aops;
2285}
2286
2287static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2288 pmd_t *dst_pmd,
2289 struct vm_area_struct *dst_vma,
2290 unsigned long dst_addr,
2291 unsigned long src_addr,
2292 bool zeropage,
2293 struct page **pagep)
2294{
2295 struct inode *inode = file_inode(dst_vma->vm_file);
2296 struct shmem_inode_info *info = SHMEM_I(inode);
2297 struct address_space *mapping = inode->i_mapping;
2298 gfp_t gfp = mapping_gfp_mask(mapping);
2299 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2300 struct mem_cgroup *memcg;
2301 spinlock_t *ptl;
2302 void *page_kaddr;
2303 struct page *page;
2304 pte_t _dst_pte, *dst_pte;
2305 int ret;
2306 pgoff_t offset, max_off;
2307
2308 ret = -ENOMEM;
2309 if (!shmem_inode_acct_block(inode, 1))
2310 goto out;
2311
2312 if (!*pagep) {
2313 page = shmem_alloc_page(gfp, info, pgoff);
2314 if (!page)
2315 goto out_unacct_blocks;
2316
2317 if (!zeropage) { /* mcopy_atomic */
2318 page_kaddr = kmap_atomic(page);
2319 ret = copy_from_user(page_kaddr,
2320 (const void __user *)src_addr,
2321 PAGE_SIZE);
2322 kunmap_atomic(page_kaddr);
2323
2324 /* fallback to copy_from_user outside mmap_sem */
2325 if (unlikely(ret)) {
2326 *pagep = page;
2327 shmem_inode_unacct_blocks(inode, 1);
2328 /* don't free the page */
2329 return -ENOENT;
2330 }
2331 } else { /* mfill_zeropage_atomic */
2332 clear_highpage(page);
2333 }
2334 } else {
2335 page = *pagep;
2336 *pagep = NULL;
2337 }
2338
2339 VM_BUG_ON(PageLocked(page) || PageSwapBacked(page));
2340 __SetPageLocked(page);
2341 __SetPageSwapBacked(page);
2342 __SetPageUptodate(page);
2343
2344 ret = -EFAULT;
2345 offset = linear_page_index(dst_vma, dst_addr);
2346 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2347 if (unlikely(offset >= max_off))
2348 goto out_release;
2349
2350 ret = mem_cgroup_try_charge_delay(page, dst_mm, gfp, &memcg, false);
2351 if (ret)
2352 goto out_release;
2353
2354 ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL,
2355 gfp & GFP_RECLAIM_MASK);
2356 if (ret)
2357 goto out_release_uncharge;
2358
2359 mem_cgroup_commit_charge(page, memcg, false, false);
2360
2361 _dst_pte = mk_pte(page, dst_vma->vm_page_prot);
2362 if (dst_vma->vm_flags & VM_WRITE)
2363 _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
2364 else {
2365 /*
2366 * We don't set the pte dirty if the vma has no
2367 * VM_WRITE permission, so mark the page dirty or it
2368 * could be freed from under us. We could do it
2369 * unconditionally before unlock_page(), but doing it
2370 * only if VM_WRITE is not set is faster.
2371 */
2372 set_page_dirty(page);
2373 }
2374
2375 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
2376
2377 ret = -EFAULT;
2378 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2379 if (unlikely(offset >= max_off))
2380 goto out_release_uncharge_unlock;
2381
2382 ret = -EEXIST;
2383 if (!pte_none(*dst_pte))
2384 goto out_release_uncharge_unlock;
2385
2386 lru_cache_add_anon(page);
2387
2388 spin_lock(&info->lock);
2389 info->alloced++;
2390 inode->i_blocks += BLOCKS_PER_PAGE;
2391 shmem_recalc_inode(inode);
2392 spin_unlock(&info->lock);
2393
2394 inc_mm_counter(dst_mm, mm_counter_file(page));
2395 page_add_file_rmap(page, false);
2396 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
2397
2398 /* No need to invalidate - it was non-present before */
2399 update_mmu_cache(dst_vma, dst_addr, dst_pte);
2400 pte_unmap_unlock(dst_pte, ptl);
2401 unlock_page(page);
2402 ret = 0;
2403out:
2404 return ret;
2405out_release_uncharge_unlock:
2406 pte_unmap_unlock(dst_pte, ptl);
2407 ClearPageDirty(page);
2408 delete_from_page_cache(page);
2409out_release_uncharge:
2410 mem_cgroup_cancel_charge(page, memcg, false);
2411out_release:
2412 unlock_page(page);
2413 put_page(page);
2414out_unacct_blocks:
2415 shmem_inode_unacct_blocks(inode, 1);
2416 goto out;
2417}
2418
2419int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm,
2420 pmd_t *dst_pmd,
2421 struct vm_area_struct *dst_vma,
2422 unsigned long dst_addr,
2423 unsigned long src_addr,
2424 struct page **pagep)
2425{
2426 return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2427 dst_addr, src_addr, false, pagep);
2428}
2429
2430int shmem_mfill_zeropage_pte(struct mm_struct *dst_mm,
2431 pmd_t *dst_pmd,
2432 struct vm_area_struct *dst_vma,
2433 unsigned long dst_addr)
2434{
2435 struct page *page = NULL;
2436
2437 return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2438 dst_addr, 0, true, &page);
2439}
2440
2441#ifdef CONFIG_TMPFS
2442static const struct inode_operations shmem_symlink_inode_operations;
2443static const struct inode_operations shmem_short_symlink_operations;
2444
2445#ifdef CONFIG_TMPFS_XATTR
2446static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2447#else
2448#define shmem_initxattrs NULL
2449#endif
2450
2451static int
2452shmem_write_begin(struct file *file, struct address_space *mapping,
2453 loff_t pos, unsigned len, unsigned flags,
2454 struct page **pagep, void **fsdata)
2455{
2456 struct inode *inode = mapping->host;
2457 struct shmem_inode_info *info = SHMEM_I(inode);
2458 pgoff_t index = pos >> PAGE_SHIFT;
2459
2460 /* i_mutex is held by caller */
2461 if (unlikely(info->seals & (F_SEAL_GROW |
2462 F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
2463 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
2464 return -EPERM;
2465 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2466 return -EPERM;
2467 }
2468
2469 return shmem_getpage(inode, index, pagep, SGP_WRITE);
2470}
2471
2472static int
2473shmem_write_end(struct file *file, struct address_space *mapping,
2474 loff_t pos, unsigned len, unsigned copied,
2475 struct page *page, void *fsdata)
2476{
2477 struct inode *inode = mapping->host;
2478
2479 if (pos + copied > inode->i_size)
2480 i_size_write(inode, pos + copied);
2481
2482 if (!PageUptodate(page)) {
2483 struct page *head = compound_head(page);
2484 if (PageTransCompound(page)) {
2485 int i;
2486
2487 for (i = 0; i < HPAGE_PMD_NR; i++) {
2488 if (head + i == page)
2489 continue;
2490 clear_highpage(head + i);
2491 flush_dcache_page(head + i);
2492 }
2493 }
2494 if (copied < PAGE_SIZE) {
2495 unsigned from = pos & (PAGE_SIZE - 1);
2496 zero_user_segments(page, 0, from,
2497 from + copied, PAGE_SIZE);
2498 }
2499 SetPageUptodate(head);
2500 }
2501 set_page_dirty(page);
2502 unlock_page(page);
2503 put_page(page);
2504
2505 return copied;
2506}
2507
2508static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2509{
2510 struct file *file = iocb->ki_filp;
2511 struct inode *inode = file_inode(file);
2512 struct address_space *mapping = inode->i_mapping;
2513 pgoff_t index;
2514 unsigned long offset;
2515 enum sgp_type sgp = SGP_READ;
2516 int error = 0;
2517 ssize_t retval = 0;
2518 loff_t *ppos = &iocb->ki_pos;
2519
2520 /*
2521 * Might this read be for a stacking filesystem? Then when reading
2522 * holes of a sparse file, we actually need to allocate those pages,
2523 * and even mark them dirty, so it cannot exceed the max_blocks limit.
2524 */
2525 if (!iter_is_iovec(to))
2526 sgp = SGP_CACHE;
2527
2528 index = *ppos >> PAGE_SHIFT;
2529 offset = *ppos & ~PAGE_MASK;
2530
2531 for (;;) {
2532 struct page *page = NULL;
2533 pgoff_t end_index;
2534 unsigned long nr, ret;
2535 loff_t i_size = i_size_read(inode);
2536
2537 end_index = i_size >> PAGE_SHIFT;
2538 if (index > end_index)
2539 break;
2540 if (index == end_index) {
2541 nr = i_size & ~PAGE_MASK;
2542 if (nr <= offset)
2543 break;
2544 }
2545
2546 error = shmem_getpage(inode, index, &page, sgp);
2547 if (error) {
2548 if (error == -EINVAL)
2549 error = 0;
2550 break;
2551 }
2552 if (page) {
2553 if (sgp == SGP_CACHE)
2554 set_page_dirty(page);
2555 unlock_page(page);
2556 }
2557
2558 /*
2559 * We must evaluate after, since reads (unlike writes)
2560 * are called without i_mutex protection against truncate
2561 */
2562 nr = PAGE_SIZE;
2563 i_size = i_size_read(inode);
2564 end_index = i_size >> PAGE_SHIFT;
2565 if (index == end_index) {
2566 nr = i_size & ~PAGE_MASK;
2567 if (nr <= offset) {
2568 if (page)
2569 put_page(page);
2570 break;
2571 }
2572 }
2573 nr -= offset;
2574
2575 if (page) {
2576 /*
2577 * If users can be writing to this page using arbitrary
2578 * virtual addresses, take care about potential aliasing
2579 * before reading the page on the kernel side.
2580 */
2581 if (mapping_writably_mapped(mapping))
2582 flush_dcache_page(page);
2583 /*
2584 * Mark the page accessed if we read the beginning.
2585 */
2586 if (!offset)
2587 mark_page_accessed(page);
2588 } else {
2589 page = ZERO_PAGE(0);
2590 get_page(page);
2591 }
2592
2593 /*
2594 * Ok, we have the page, and it's up-to-date, so
2595 * now we can copy it to user space...
2596 */
2597 ret = copy_page_to_iter(page, offset, nr, to);
2598 retval += ret;
2599 offset += ret;
2600 index += offset >> PAGE_SHIFT;
2601 offset &= ~PAGE_MASK;
2602
2603 put_page(page);
2604 if (!iov_iter_count(to))
2605 break;
2606 if (ret < nr) {
2607 error = -EFAULT;
2608 break;
2609 }
2610 cond_resched();
2611 }
2612
2613 *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2614 file_accessed(file);
2615 return retval ? retval : error;
2616}
2617
2618/*
2619 * llseek SEEK_DATA or SEEK_HOLE through the page cache.
2620 */
2621static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
2622 pgoff_t index, pgoff_t end, int whence)
2623{
2624 struct page *page;
2625 struct pagevec pvec;
2626 pgoff_t indices[PAGEVEC_SIZE];
2627 bool done = false;
2628 int i;
2629
2630 pagevec_init(&pvec);
2631 pvec.nr = 1; /* start small: we may be there already */
2632 while (!done) {
2633 pvec.nr = find_get_entries(mapping, index,
2634 pvec.nr, pvec.pages, indices);
2635 if (!pvec.nr) {
2636 if (whence == SEEK_DATA)
2637 index = end;
2638 break;
2639 }
2640 for (i = 0; i < pvec.nr; i++, index++) {
2641 if (index < indices[i]) {
2642 if (whence == SEEK_HOLE) {
2643 done = true;
2644 break;
2645 }
2646 index = indices[i];
2647 }
2648 page = pvec.pages[i];
2649 if (page && !xa_is_value(page)) {
2650 if (!PageUptodate(page))
2651 page = NULL;
2652 }
2653 if (index >= end ||
2654 (page && whence == SEEK_DATA) ||
2655 (!page && whence == SEEK_HOLE)) {
2656 done = true;
2657 break;
2658 }
2659 }
2660 pagevec_remove_exceptionals(&pvec);
2661 pagevec_release(&pvec);
2662 pvec.nr = PAGEVEC_SIZE;
2663 cond_resched();
2664 }
2665 return index;
2666}
2667
2668static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2669{
2670 struct address_space *mapping = file->f_mapping;
2671 struct inode *inode = mapping->host;
2672 pgoff_t start, end;
2673 loff_t new_offset;
2674
2675 if (whence != SEEK_DATA && whence != SEEK_HOLE)
2676 return generic_file_llseek_size(file, offset, whence,
2677 MAX_LFS_FILESIZE, i_size_read(inode));
2678 inode_lock(inode);
2679 /* We're holding i_mutex so we can access i_size directly */
2680
2681 if (offset < 0 || offset >= inode->i_size)
2682 offset = -ENXIO;
2683 else {
2684 start = offset >> PAGE_SHIFT;
2685 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
2686 new_offset = shmem_seek_hole_data(mapping, start, end, whence);
2687 new_offset <<= PAGE_SHIFT;
2688 if (new_offset > offset) {
2689 if (new_offset < inode->i_size)
2690 offset = new_offset;
2691 else if (whence == SEEK_DATA)
2692 offset = -ENXIO;
2693 else
2694 offset = inode->i_size;
2695 }
2696 }
2697
2698 if (offset >= 0)
2699 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2700 inode_unlock(inode);
2701 return offset;
2702}
2703
2704static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2705 loff_t len)
2706{
2707 struct inode *inode = file_inode(file);
2708 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2709 struct shmem_inode_info *info = SHMEM_I(inode);
2710 struct shmem_falloc shmem_falloc;
2711 pgoff_t start, index, end;
2712 int error;
2713
2714 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2715 return -EOPNOTSUPP;
2716
2717 inode_lock(inode);
2718
2719 if (mode & FALLOC_FL_PUNCH_HOLE) {
2720 struct address_space *mapping = file->f_mapping;
2721 loff_t unmap_start = round_up(offset, PAGE_SIZE);
2722 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2723 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2724
2725 /* protected by i_mutex */
2726 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
2727 error = -EPERM;
2728 goto out;
2729 }
2730
2731 shmem_falloc.waitq = &shmem_falloc_waitq;
2732 shmem_falloc.start = unmap_start >> PAGE_SHIFT;
2733 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2734 spin_lock(&inode->i_lock);
2735 inode->i_private = &shmem_falloc;
2736 spin_unlock(&inode->i_lock);
2737
2738 if ((u64)unmap_end > (u64)unmap_start)
2739 unmap_mapping_range(mapping, unmap_start,
2740 1 + unmap_end - unmap_start, 0);
2741 shmem_truncate_range(inode, offset, offset + len - 1);
2742 /* No need to unmap again: hole-punching leaves COWed pages */
2743
2744 spin_lock(&inode->i_lock);
2745 inode->i_private = NULL;
2746 wake_up_all(&shmem_falloc_waitq);
2747 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2748 spin_unlock(&inode->i_lock);
2749 error = 0;
2750 goto out;
2751 }
2752
2753 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2754 error = inode_newsize_ok(inode, offset + len);
2755 if (error)
2756 goto out;
2757
2758 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2759 error = -EPERM;
2760 goto out;
2761 }
2762
2763 start = offset >> PAGE_SHIFT;
2764 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2765 /* Try to avoid a swapstorm if len is impossible to satisfy */
2766 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2767 error = -ENOSPC;
2768 goto out;
2769 }
2770
2771 shmem_falloc.waitq = NULL;
2772 shmem_falloc.start = start;
2773 shmem_falloc.next = start;
2774 shmem_falloc.nr_falloced = 0;
2775 shmem_falloc.nr_unswapped = 0;
2776 spin_lock(&inode->i_lock);
2777 inode->i_private = &shmem_falloc;
2778 spin_unlock(&inode->i_lock);
2779
2780 for (index = start; index < end; index++) {
2781 struct page *page;
2782
2783 /*
2784 * Good, the fallocate(2) manpage permits EINTR: we may have
2785 * been interrupted because we are using up too much memory.
2786 */
2787 if (signal_pending(current))
2788 error = -EINTR;
2789 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2790 error = -ENOMEM;
2791 else
2792 error = shmem_getpage(inode, index, &page, SGP_FALLOC);
2793 if (error) {
2794 /* Remove the !PageUptodate pages we added */
2795 if (index > start) {
2796 shmem_undo_range(inode,
2797 (loff_t)start << PAGE_SHIFT,
2798 ((loff_t)index << PAGE_SHIFT) - 1, true);
2799 }
2800 goto undone;
2801 }
2802
2803 /*
2804 * Inform shmem_writepage() how far we have reached.
2805 * No need for lock or barrier: we have the page lock.
2806 */
2807 shmem_falloc.next++;
2808 if (!PageUptodate(page))
2809 shmem_falloc.nr_falloced++;
2810
2811 /*
2812 * If !PageUptodate, leave it that way so that freeable pages
2813 * can be recognized if we need to rollback on error later.
2814 * But set_page_dirty so that memory pressure will swap rather
2815 * than free the pages we are allocating (and SGP_CACHE pages
2816 * might still be clean: we now need to mark those dirty too).
2817 */
2818 set_page_dirty(page);
2819 unlock_page(page);
2820 put_page(page);
2821 cond_resched();
2822 }
2823
2824 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2825 i_size_write(inode, offset + len);
2826 inode->i_ctime = current_time(inode);
2827undone:
2828 spin_lock(&inode->i_lock);
2829 inode->i_private = NULL;
2830 spin_unlock(&inode->i_lock);
2831out:
2832 inode_unlock(inode);
2833 return error;
2834}
2835
2836static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2837{
2838 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2839
2840 buf->f_type = TMPFS_MAGIC;
2841 buf->f_bsize = PAGE_SIZE;
2842 buf->f_namelen = NAME_MAX;
2843 if (sbinfo->max_blocks) {
2844 buf->f_blocks = sbinfo->max_blocks;
2845 buf->f_bavail =
2846 buf->f_bfree = sbinfo->max_blocks -
2847 percpu_counter_sum(&sbinfo->used_blocks);
2848 }
2849 if (sbinfo->max_inodes) {
2850 buf->f_files = sbinfo->max_inodes;
2851 buf->f_ffree = sbinfo->free_inodes;
2852 }
2853 /* else leave those fields 0 like simple_statfs */
2854 return 0;
2855}
2856
2857/*
2858 * File creation. Allocate an inode, and we're done..
2859 */
2860static int
2861shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
2862{
2863 struct inode *inode;
2864 int error = -ENOSPC;
2865
2866 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2867 if (inode) {
2868 error = simple_acl_create(dir, inode);
2869 if (error)
2870 goto out_iput;
2871 error = security_inode_init_security(inode, dir,
2872 &dentry->d_name,
2873 shmem_initxattrs, NULL);
2874 if (error && error != -EOPNOTSUPP)
2875 goto out_iput;
2876
2877 error = 0;
2878 dir->i_size += BOGO_DIRENT_SIZE;
2879 dir->i_ctime = dir->i_mtime = current_time(dir);
2880 d_instantiate(dentry, inode);
2881 dget(dentry); /* Extra count - pin the dentry in core */
2882 }
2883 return error;
2884out_iput:
2885 iput(inode);
2886 return error;
2887}
2888
2889static int
2890shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
2891{
2892 struct inode *inode;
2893 int error = -ENOSPC;
2894
2895 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2896 if (inode) {
2897 error = security_inode_init_security(inode, dir,
2898 NULL,
2899 shmem_initxattrs, NULL);
2900 if (error && error != -EOPNOTSUPP)
2901 goto out_iput;
2902 error = simple_acl_create(dir, inode);
2903 if (error)
2904 goto out_iput;
2905 d_tmpfile(dentry, inode);
2906 }
2907 return error;
2908out_iput:
2909 iput(inode);
2910 return error;
2911}
2912
2913static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
2914{
2915 int error;
2916
2917 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
2918 return error;
2919 inc_nlink(dir);
2920 return 0;
2921}
2922
2923static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
2924 bool excl)
2925{
2926 return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
2927}
2928
2929/*
2930 * Link a file..
2931 */
2932static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2933{
2934 struct inode *inode = d_inode(old_dentry);
2935 int ret = 0;
2936
2937 /*
2938 * No ordinary (disk based) filesystem counts links as inodes;
2939 * but each new link needs a new dentry, pinning lowmem, and
2940 * tmpfs dentries cannot be pruned until they are unlinked.
2941 * But if an O_TMPFILE file is linked into the tmpfs, the
2942 * first link must skip that, to get the accounting right.
2943 */
2944 if (inode->i_nlink) {
2945 ret = shmem_reserve_inode(inode->i_sb);
2946 if (ret)
2947 goto out;
2948 }
2949
2950 dir->i_size += BOGO_DIRENT_SIZE;
2951 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2952 inc_nlink(inode);
2953 ihold(inode); /* New dentry reference */
2954 dget(dentry); /* Extra pinning count for the created dentry */
2955 d_instantiate(dentry, inode);
2956out:
2957 return ret;
2958}
2959
2960static int shmem_unlink(struct inode *dir, struct dentry *dentry)
2961{
2962 struct inode *inode = d_inode(dentry);
2963
2964 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
2965 shmem_free_inode(inode->i_sb);
2966
2967 dir->i_size -= BOGO_DIRENT_SIZE;
2968 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2969 drop_nlink(inode);
2970 dput(dentry); /* Undo the count from "create" - this does all the work */
2971 return 0;
2972}
2973
2974static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
2975{
2976 if (!simple_empty(dentry))
2977 return -ENOTEMPTY;
2978
2979 drop_nlink(d_inode(dentry));
2980 drop_nlink(dir);
2981 return shmem_unlink(dir, dentry);
2982}
2983
2984static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
2985{
2986 bool old_is_dir = d_is_dir(old_dentry);
2987 bool new_is_dir = d_is_dir(new_dentry);
2988
2989 if (old_dir != new_dir && old_is_dir != new_is_dir) {
2990 if (old_is_dir) {
2991 drop_nlink(old_dir);
2992 inc_nlink(new_dir);
2993 } else {
2994 drop_nlink(new_dir);
2995 inc_nlink(old_dir);
2996 }
2997 }
2998 old_dir->i_ctime = old_dir->i_mtime =
2999 new_dir->i_ctime = new_dir->i_mtime =
3000 d_inode(old_dentry)->i_ctime =
3001 d_inode(new_dentry)->i_ctime = current_time(old_dir);
3002
3003 return 0;
3004}
3005
3006static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry)
3007{
3008 struct dentry *whiteout;
3009 int error;
3010
3011 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
3012 if (!whiteout)
3013 return -ENOMEM;
3014
3015 error = shmem_mknod(old_dir, whiteout,
3016 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
3017 dput(whiteout);
3018 if (error)
3019 return error;
3020
3021 /*
3022 * Cheat and hash the whiteout while the old dentry is still in
3023 * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
3024 *
3025 * d_lookup() will consistently find one of them at this point,
3026 * not sure which one, but that isn't even important.
3027 */
3028 d_rehash(whiteout);
3029 return 0;
3030}
3031
3032/*
3033 * The VFS layer already does all the dentry stuff for rename,
3034 * we just have to decrement the usage count for the target if
3035 * it exists so that the VFS layer correctly free's it when it
3036 * gets overwritten.
3037 */
3038static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags)
3039{
3040 struct inode *inode = d_inode(old_dentry);
3041 int they_are_dirs = S_ISDIR(inode->i_mode);
3042
3043 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3044 return -EINVAL;
3045
3046 if (flags & RENAME_EXCHANGE)
3047 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry);
3048
3049 if (!simple_empty(new_dentry))
3050 return -ENOTEMPTY;
3051
3052 if (flags & RENAME_WHITEOUT) {
3053 int error;
3054
3055 error = shmem_whiteout(old_dir, old_dentry);
3056 if (error)
3057 return error;
3058 }
3059
3060 if (d_really_is_positive(new_dentry)) {
3061 (void) shmem_unlink(new_dir, new_dentry);
3062 if (they_are_dirs) {
3063 drop_nlink(d_inode(new_dentry));
3064 drop_nlink(old_dir);
3065 }
3066 } else if (they_are_dirs) {
3067 drop_nlink(old_dir);
3068 inc_nlink(new_dir);
3069 }
3070
3071 old_dir->i_size -= BOGO_DIRENT_SIZE;
3072 new_dir->i_size += BOGO_DIRENT_SIZE;
3073 old_dir->i_ctime = old_dir->i_mtime =
3074 new_dir->i_ctime = new_dir->i_mtime =
3075 inode->i_ctime = current_time(old_dir);
3076 return 0;
3077}
3078
3079static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
3080{
3081 int error;
3082 int len;
3083 struct inode *inode;
3084 struct page *page;
3085
3086 len = strlen(symname) + 1;
3087 if (len > PAGE_SIZE)
3088 return -ENAMETOOLONG;
3089
3090 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0,
3091 VM_NORESERVE);
3092 if (!inode)
3093 return -ENOSPC;
3094
3095 error = security_inode_init_security(inode, dir, &dentry->d_name,
3096 shmem_initxattrs, NULL);
3097 if (error) {
3098 if (error != -EOPNOTSUPP) {
3099 iput(inode);
3100 return error;
3101 }
3102 error = 0;
3103 }
3104
3105 inode->i_size = len-1;
3106 if (len <= SHORT_SYMLINK_LEN) {
3107 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3108 if (!inode->i_link) {
3109 iput(inode);
3110 return -ENOMEM;
3111 }
3112 inode->i_op = &shmem_short_symlink_operations;
3113 } else {
3114 inode_nohighmem(inode);
3115 error = shmem_getpage(inode, 0, &page, SGP_WRITE);
3116 if (error) {
3117 iput(inode);
3118 return error;
3119 }
3120 inode->i_mapping->a_ops = &shmem_aops;
3121 inode->i_op = &shmem_symlink_inode_operations;
3122 memcpy(page_address(page), symname, len);
3123 SetPageUptodate(page);
3124 set_page_dirty(page);
3125 unlock_page(page);
3126 put_page(page);
3127 }
3128 dir->i_size += BOGO_DIRENT_SIZE;
3129 dir->i_ctime = dir->i_mtime = current_time(dir);
3130 d_instantiate(dentry, inode);
3131 dget(dentry);
3132 return 0;
3133}
3134
3135static void shmem_put_link(void *arg)
3136{
3137 mark_page_accessed(arg);
3138 put_page(arg);
3139}
3140
3141static const char *shmem_get_link(struct dentry *dentry,
3142 struct inode *inode,
3143 struct delayed_call *done)
3144{
3145 struct page *page = NULL;
3146 int error;
3147 if (!dentry) {
3148 page = find_get_page(inode->i_mapping, 0);
3149 if (!page)
3150 return ERR_PTR(-ECHILD);
3151 if (!PageUptodate(page)) {
3152 put_page(page);
3153 return ERR_PTR(-ECHILD);
3154 }
3155 } else {
3156 error = shmem_getpage(inode, 0, &page, SGP_READ);
3157 if (error)
3158 return ERR_PTR(error);
3159 unlock_page(page);
3160 }
3161 set_delayed_call(done, shmem_put_link, page);
3162 return page_address(page);
3163}
3164
3165#ifdef CONFIG_TMPFS_XATTR
3166/*
3167 * Superblocks without xattr inode operations may get some security.* xattr
3168 * support from the LSM "for free". As soon as we have any other xattrs
3169 * like ACLs, we also need to implement the security.* handlers at
3170 * filesystem level, though.
3171 */
3172
3173/*
3174 * Callback for security_inode_init_security() for acquiring xattrs.
3175 */
3176static int shmem_initxattrs(struct inode *inode,
3177 const struct xattr *xattr_array,
3178 void *fs_info)
3179{
3180 struct shmem_inode_info *info = SHMEM_I(inode);
3181 const struct xattr *xattr;
3182 struct simple_xattr *new_xattr;
3183 size_t len;
3184
3185 for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3186 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3187 if (!new_xattr)
3188 return -ENOMEM;
3189
3190 len = strlen(xattr->name) + 1;
3191 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3192 GFP_KERNEL);
3193 if (!new_xattr->name) {
3194 kfree(new_xattr);
3195 return -ENOMEM;
3196 }
3197
3198 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3199 XATTR_SECURITY_PREFIX_LEN);
3200 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3201 xattr->name, len);
3202
3203 simple_xattr_list_add(&info->xattrs, new_xattr);
3204 }
3205
3206 return 0;
3207}
3208
3209static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3210 struct dentry *unused, struct inode *inode,
3211 const char *name, void *buffer, size_t size)
3212{
3213 struct shmem_inode_info *info = SHMEM_I(inode);
3214
3215 name = xattr_full_name(handler, name);
3216 return simple_xattr_get(&info->xattrs, name, buffer, size);
3217}
3218
3219static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3220 struct dentry *unused, struct inode *inode,
3221 const char *name, const void *value,
3222 size_t size, int flags)
3223{
3224 struct shmem_inode_info *info = SHMEM_I(inode);
3225
3226 name = xattr_full_name(handler, name);
3227 return simple_xattr_set(&info->xattrs, name, value, size, flags);
3228}
3229
3230static const struct xattr_handler shmem_security_xattr_handler = {
3231 .prefix = XATTR_SECURITY_PREFIX,
3232 .get = shmem_xattr_handler_get,
3233 .set = shmem_xattr_handler_set,
3234};
3235
3236static const struct xattr_handler shmem_trusted_xattr_handler = {
3237 .prefix = XATTR_TRUSTED_PREFIX,
3238 .get = shmem_xattr_handler_get,
3239 .set = shmem_xattr_handler_set,
3240};
3241
3242static const struct xattr_handler *shmem_xattr_handlers[] = {
3243#ifdef CONFIG_TMPFS_POSIX_ACL
3244 &posix_acl_access_xattr_handler,
3245 &posix_acl_default_xattr_handler,
3246#endif
3247 &shmem_security_xattr_handler,
3248 &shmem_trusted_xattr_handler,
3249 NULL
3250};
3251
3252static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3253{
3254 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3255 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3256}
3257#endif /* CONFIG_TMPFS_XATTR */
3258
3259static const struct inode_operations shmem_short_symlink_operations = {
3260 .get_link = simple_get_link,
3261#ifdef CONFIG_TMPFS_XATTR
3262 .listxattr = shmem_listxattr,
3263#endif
3264};
3265
3266static const struct inode_operations shmem_symlink_inode_operations = {
3267 .get_link = shmem_get_link,
3268#ifdef CONFIG_TMPFS_XATTR
3269 .listxattr = shmem_listxattr,
3270#endif
3271};
3272
3273static struct dentry *shmem_get_parent(struct dentry *child)
3274{
3275 return ERR_PTR(-ESTALE);
3276}
3277
3278static int shmem_match(struct inode *ino, void *vfh)
3279{
3280 __u32 *fh = vfh;
3281 __u64 inum = fh[2];
3282 inum = (inum << 32) | fh[1];
3283 return ino->i_ino == inum && fh[0] == ino->i_generation;
3284}
3285
3286/* Find any alias of inode, but prefer a hashed alias */
3287static struct dentry *shmem_find_alias(struct inode *inode)
3288{
3289 struct dentry *alias = d_find_alias(inode);
3290
3291 return alias ?: d_find_any_alias(inode);
3292}
3293
3294
3295static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3296 struct fid *fid, int fh_len, int fh_type)
3297{
3298 struct inode *inode;
3299 struct dentry *dentry = NULL;
3300 u64 inum;
3301
3302 if (fh_len < 3)
3303 return NULL;
3304
3305 inum = fid->raw[2];
3306 inum = (inum << 32) | fid->raw[1];
3307
3308 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3309 shmem_match, fid->raw);
3310 if (inode) {
3311 dentry = shmem_find_alias(inode);
3312 iput(inode);
3313 }
3314
3315 return dentry;
3316}
3317
3318static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3319 struct inode *parent)
3320{
3321 if (*len < 3) {
3322 *len = 3;
3323 return FILEID_INVALID;
3324 }
3325
3326 if (inode_unhashed(inode)) {
3327 /* Unfortunately insert_inode_hash is not idempotent,
3328 * so as we hash inodes here rather than at creation
3329 * time, we need a lock to ensure we only try
3330 * to do it once
3331 */
3332 static DEFINE_SPINLOCK(lock);
3333 spin_lock(&lock);
3334 if (inode_unhashed(inode))
3335 __insert_inode_hash(inode,
3336 inode->i_ino + inode->i_generation);
3337 spin_unlock(&lock);
3338 }
3339
3340 fh[0] = inode->i_generation;
3341 fh[1] = inode->i_ino;
3342 fh[2] = ((__u64)inode->i_ino) >> 32;
3343
3344 *len = 3;
3345 return 1;
3346}
3347
3348static const struct export_operations shmem_export_ops = {
3349 .get_parent = shmem_get_parent,
3350 .encode_fh = shmem_encode_fh,
3351 .fh_to_dentry = shmem_fh_to_dentry,
3352};
3353
3354static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
3355 bool remount)
3356{
3357 char *this_char, *value, *rest;
3358 struct mempolicy *mpol = NULL;
3359 uid_t uid;
3360 gid_t gid;
3361
3362 while (options != NULL) {
3363 this_char = options;
3364 for (;;) {
3365 /*
3366 * NUL-terminate this option: unfortunately,
3367 * mount options form a comma-separated list,
3368 * but mpol's nodelist may also contain commas.
3369 */
3370 options = strchr(options, ',');
3371 if (options == NULL)
3372 break;
3373 options++;
3374 if (!isdigit(*options)) {
3375 options[-1] = '\0';
3376 break;
3377 }
3378 }
3379 if (!*this_char)
3380 continue;
3381 if ((value = strchr(this_char,'=')) != NULL) {
3382 *value++ = 0;
3383 } else {
3384 pr_err("tmpfs: No value for mount option '%s'\n",
3385 this_char);
3386 goto error;
3387 }
3388
3389 if (!strcmp(this_char,"size")) {
3390 unsigned long long size;
3391 size = memparse(value,&rest);
3392 if (*rest == '%') {
3393 size <<= PAGE_SHIFT;
3394 size *= totalram_pages();
3395 do_div(size, 100);
3396 rest++;
3397 }
3398 if (*rest)
3399 goto bad_val;
3400 sbinfo->max_blocks =
3401 DIV_ROUND_UP(size, PAGE_SIZE);
3402 } else if (!strcmp(this_char,"nr_blocks")) {
3403 sbinfo->max_blocks = memparse(value, &rest);
3404 if (*rest)
3405 goto bad_val;
3406 } else if (!strcmp(this_char,"nr_inodes")) {
3407 sbinfo->max_inodes = memparse(value, &rest);
3408 if (*rest)
3409 goto bad_val;
3410 } else if (!strcmp(this_char,"mode")) {
3411 if (remount)
3412 continue;
3413 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
3414 if (*rest)
3415 goto bad_val;
3416 } else if (!strcmp(this_char,"uid")) {
3417 if (remount)
3418 continue;
3419 uid = simple_strtoul(value, &rest, 0);
3420 if (*rest)
3421 goto bad_val;
3422 sbinfo->uid = make_kuid(current_user_ns(), uid);
3423 if (!uid_valid(sbinfo->uid))
3424 goto bad_val;
3425 } else if (!strcmp(this_char,"gid")) {
3426 if (remount)
3427 continue;
3428 gid = simple_strtoul(value, &rest, 0);
3429 if (*rest)
3430 goto bad_val;
3431 sbinfo->gid = make_kgid(current_user_ns(), gid);
3432 if (!gid_valid(sbinfo->gid))
3433 goto bad_val;
3434#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3435 } else if (!strcmp(this_char, "huge")) {
3436 int huge;
3437 huge = shmem_parse_huge(value);
3438 if (huge < 0)
3439 goto bad_val;
3440 if (!has_transparent_hugepage() &&
3441 huge != SHMEM_HUGE_NEVER)
3442 goto bad_val;
3443 sbinfo->huge = huge;
3444#endif
3445#ifdef CONFIG_NUMA
3446 } else if (!strcmp(this_char,"mpol")) {
3447 mpol_put(mpol);
3448 mpol = NULL;
3449 if (mpol_parse_str(value, &mpol))
3450 goto bad_val;
3451#endif
3452 } else {
3453 pr_err("tmpfs: Bad mount option %s\n", this_char);
3454 goto error;
3455 }
3456 }
3457 sbinfo->mpol = mpol;
3458 return 0;
3459
3460bad_val:
3461 pr_err("tmpfs: Bad value '%s' for mount option '%s'\n",
3462 value, this_char);
3463error:
3464 mpol_put(mpol);
3465 return 1;
3466
3467}
3468
3469static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
3470{
3471 struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3472 struct shmem_sb_info config = *sbinfo;
3473 unsigned long inodes;
3474 int error = -EINVAL;
3475
3476 config.mpol = NULL;
3477 if (shmem_parse_options(data, &config, true))
3478 return error;
3479
3480 spin_lock(&sbinfo->stat_lock);
3481 inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3482 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
3483 goto out;
3484 if (config.max_inodes < inodes)
3485 goto out;
3486 /*
3487 * Those tests disallow limited->unlimited while any are in use;
3488 * but we must separately disallow unlimited->limited, because
3489 * in that case we have no record of how much is already in use.
3490 */
3491 if (config.max_blocks && !sbinfo->max_blocks)
3492 goto out;
3493 if (config.max_inodes && !sbinfo->max_inodes)
3494 goto out;
3495
3496 error = 0;
3497 sbinfo->huge = config.huge;
3498 sbinfo->max_blocks = config.max_blocks;
3499 sbinfo->max_inodes = config.max_inodes;
3500 sbinfo->free_inodes = config.max_inodes - inodes;
3501
3502 /*
3503 * Preserve previous mempolicy unless mpol remount option was specified.
3504 */
3505 if (config.mpol) {
3506 mpol_put(sbinfo->mpol);
3507 sbinfo->mpol = config.mpol; /* transfers initial ref */
3508 }
3509out:
3510 spin_unlock(&sbinfo->stat_lock);
3511 return error;
3512}
3513
3514static int shmem_show_options(struct seq_file *seq, struct dentry *root)
3515{
3516 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
3517
3518 if (sbinfo->max_blocks != shmem_default_max_blocks())
3519 seq_printf(seq, ",size=%luk",
3520 sbinfo->max_blocks << (PAGE_SHIFT - 10));
3521 if (sbinfo->max_inodes != shmem_default_max_inodes())
3522 seq_printf(seq, ",nr_inodes=%lu", sbinfo->