1/*
2 * mm/truncate.c - code for taking down pages from address_spaces
3 *
4 * Copyright (C) 2002, Linus Torvalds
5 *
6 * 10Sep2002 Andrew Morton
7 * Initial version.
8 */
9
10#include <linux/kernel.h>
11#include <linux/backing-dev.h>
12#include <linux/dax.h>
13#include <linux/gfp.h>
14#include <linux/mm.h>
15#include <linux/swap.h>
16#include <linux/export.h>
17#include <linux/pagemap.h>
18#include <linux/highmem.h>
19#include <linux/pagevec.h>
20#include <linux/task_io_accounting_ops.h>
21#include <linux/buffer_head.h> /* grr. try_to_release_page,
22 do_invalidatepage */
23#include <linux/shmem_fs.h>
24#include <linux/cleancache.h>
25#include <linux/rmap.h>
26#include "internal.h"
27
28/*
29 * Regular page slots are stabilized by the page lock even without the tree
30 * itself locked. These unlocked entries need verification under the tree
31 * lock.
32 */
33static inline void __clear_shadow_entry(struct address_space *mapping,
34 pgoff_t index, void *entry)
35{
36 XA_STATE(xas, &mapping->i_pages, index);
37
38 xas_set_update(&xas, workingset_update_node);
39 if (xas_load(&xas) != entry)
40 return;
41 xas_store(&xas, NULL);
42 mapping->nrexceptional--;
43}
44
45static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,
46 void *entry)
47{
48 xa_lock_irq(&mapping->i_pages);
49 __clear_shadow_entry(mapping, index, entry);
50 xa_unlock_irq(&mapping->i_pages);
51}
52
53/*
54 * Unconditionally remove exceptional entries. Usually called from truncate
55 * path. Note that the pagevec may be altered by this function by removing
56 * exceptional entries similar to what pagevec_remove_exceptionals does.
57 */
58static void truncate_exceptional_pvec_entries(struct address_space *mapping,
59 struct pagevec *pvec, pgoff_t *indices,
60 pgoff_t end)
61{
62 int i, j;
63 bool dax, lock;
64
65 /* Handled by shmem itself */
66 if (shmem_mapping(mapping))
67 return;
68
69 for (j = 0; j < pagevec_count(pvec); j++)
70 if (xa_is_value(pvec->pages[j]))
71 break;
72
73 if (j == pagevec_count(pvec))
74 return;
75
76 dax = dax_mapping(mapping);
77 lock = !dax && indices[j] < end;
78 if (lock)
79 xa_lock_irq(&mapping->i_pages);
80
81 for (i = j; i < pagevec_count(pvec); i++) {
82 struct page *page = pvec->pages[i];
83 pgoff_t index = indices[i];
84
85 if (!xa_is_value(page)) {
86 pvec->pages[j++] = page;
87 continue;
88 }
89
90 if (index >= end)
91 continue;
92
93 if (unlikely(dax)) {
94 dax_delete_mapping_entry(mapping, index);
95 continue;
96 }
97
98 __clear_shadow_entry(mapping, index, page);
99 }
100
101 if (lock)
102 xa_unlock_irq(&mapping->i_pages);
103 pvec->nr = j;
104}
105
106/*
107 * Invalidate exceptional entry if easily possible. This handles exceptional
108 * entries for invalidate_inode_pages().
109 */
110static int invalidate_exceptional_entry(struct address_space *mapping,
111 pgoff_t index, void *entry)
112{
113 /* Handled by shmem itself, or for DAX we do nothing. */
114 if (shmem_mapping(mapping) || dax_mapping(mapping))
115 return 1;
116 clear_shadow_entry(mapping, index, entry);
117 return 1;
118}
119
120/*
121 * Invalidate exceptional entry if clean. This handles exceptional entries for
122 * invalidate_inode_pages2() so for DAX it evicts only clean entries.
123 */
124static int invalidate_exceptional_entry2(struct address_space *mapping,
125 pgoff_t index, void *entry)
126{
127 /* Handled by shmem itself */
128 if (shmem_mapping(mapping))
129 return 1;
130 if (dax_mapping(mapping))
131 return dax_invalidate_mapping_entry_sync(mapping, index);
132 clear_shadow_entry(mapping, index, entry);
133 return 1;
134}
135
136/**
137 * do_invalidatepage - invalidate part or all of a page
138 * @page: the page which is affected
139 * @offset: start of the range to invalidate
140 * @length: length of the range to invalidate
141 *
142 * do_invalidatepage() is called when all or part of the page has become
143 * invalidated by a truncate operation.
144 *
145 * do_invalidatepage() does not have to release all buffers, but it must
146 * ensure that no dirty buffer is left outside @offset and that no I/O
147 * is underway against any of the blocks which are outside the truncation
148 * point. Because the caller is about to free (and possibly reuse) those
149 * blocks on-disk.
150 */
151void do_invalidatepage(struct page *page, unsigned int offset,
152 unsigned int length)
153{
154 void (*invalidatepage)(struct page *, unsigned int, unsigned int);
155
156 invalidatepage = page->mapping->a_ops->invalidatepage;
157#ifdef CONFIG_BLOCK
158 if (!invalidatepage)
159 invalidatepage = block_invalidatepage;
160#endif
161 if (invalidatepage)
162 (*invalidatepage)(page, offset, length);
163}
164
165/*
166 * If truncate cannot remove the fs-private metadata from the page, the page
167 * becomes orphaned. It will be left on the LRU and may even be mapped into
168 * user pagetables if we're racing with filemap_fault().
169 *
170 * We need to bale out if page->mapping is no longer equal to the original
171 * mapping. This happens a) when the VM reclaimed the page while we waited on
172 * its lock, b) when a concurrent invalidate_mapping_pages got there first and
173 * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
174 */
175static void
176truncate_cleanup_page(struct address_space *mapping, struct page *page)
177{
178 if (page_mapped(page)) {
179 pgoff_t nr = PageTransHuge(page) ? HPAGE_PMD_NR : 1;
180 unmap_mapping_pages(mapping, page->index, nr, false);
181 }
182
183 if (page_has_private(page))
184 do_invalidatepage(page, 0, PAGE_SIZE);
185
186 /*
187 * Some filesystems seem to re-dirty the page even after
188 * the VM has canceled the dirty bit (eg ext3 journaling).
189 * Hence dirty accounting check is placed after invalidation.
190 */
191 cancel_dirty_page(page);
192 ClearPageMappedToDisk(page);
193}
194
195/*
196 * This is for invalidate_mapping_pages(). That function can be called at
197 * any time, and is not supposed to throw away dirty pages. But pages can
198 * be marked dirty at any time too, so use remove_mapping which safely
199 * discards clean, unused pages.
200 *
201 * Returns non-zero if the page was successfully invalidated.
202 */
203static int
204invalidate_complete_page(struct address_space *mapping, struct page *page)
205{
206 int ret;
207
208 if (page->mapping != mapping)
209 return 0;
210
211 if (page_has_private(page) && !try_to_release_page(page, 0))
212 return 0;
213
214 ret = remove_mapping(mapping, page);
215
216 return ret;
217}
218
219int truncate_inode_page(struct address_space *mapping, struct page *page)
220{
221 VM_BUG_ON_PAGE(PageTail(page), page);
222
223 if (page->mapping != mapping)
224 return -EIO;
225
226 truncate_cleanup_page(mapping, page);
227 delete_from_page_cache(page);
228 return 0;
229}
230
231/*
232 * Used to get rid of pages on hardware memory corruption.
233 */
234int generic_error_remove_page(struct address_space *mapping, struct page *page)
235{
236 if (!mapping)
237 return -EINVAL;
238 /*
239 * Only punch for normal data pages for now.
240 * Handling other types like directories would need more auditing.
241 */
242 if (!S_ISREG(mapping->host->i_mode))
243 return -EIO;
244 return truncate_inode_page(mapping, page);
245}
246EXPORT_SYMBOL(generic_error_remove_page);
247
248/*
249 * Safely invalidate one page from its pagecache mapping.
250 * It only drops clean, unused pages. The page must be locked.
251 *
252 * Returns 1 if the page is successfully invalidated, otherwise 0.
253 */
254int invalidate_inode_page(struct page *page)
255{
256 struct address_space *mapping = page_mapping(page);
257 if (!mapping)
258 return 0;
259 if (PageDirty(page) || PageWriteback(page))
260 return 0;
261 if (page_mapped(page))
262 return 0;
263 return invalidate_complete_page(mapping, page);
264}
265
266/**
267 * truncate_inode_pages_range - truncate range of pages specified by start & end byte offsets
268 * @mapping: mapping to truncate
269 * @lstart: offset from which to truncate
270 * @lend: offset to which to truncate (inclusive)
271 *
272 * Truncate the page cache, removing the pages that are between
273 * specified offsets (and zeroing out partial pages
274 * if lstart or lend + 1 is not page aligned).
275 *
276 * Truncate takes two passes - the first pass is nonblocking. It will not
277 * block on page locks and it will not block on writeback. The second pass
278 * will wait. This is to prevent as much IO as possible in the affected region.
279 * The first pass will remove most pages, so the search cost of the second pass
280 * is low.
281 *
282 * We pass down the cache-hot hint to the page freeing code. Even if the
283 * mapping is large, it is probably the case that the final pages are the most
284 * recently touched, and freeing happens in ascending file offset order.
285 *
286 * Note that since ->invalidatepage() accepts range to invalidate
287 * truncate_inode_pages_range is able to handle cases where lend + 1 is not
288 * page aligned properly.
289 */
290void truncate_inode_pages_range(struct address_space *mapping,
291 loff_t lstart, loff_t lend)
292{
293 pgoff_t start; /* inclusive */
294 pgoff_t end; /* exclusive */
295 unsigned int partial_start; /* inclusive */
296 unsigned int partial_end; /* exclusive */
297 struct pagevec pvec;
298 pgoff_t indices[PAGEVEC_SIZE];
299 pgoff_t index;
300 int i;
301
302 if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
303 goto out;
304
305 /* Offsets within partial pages */
306 partial_start = lstart & (PAGE_SIZE - 1);
307 partial_end = (lend + 1) & (PAGE_SIZE - 1);
308
309 /*
310 * 'start' and 'end' always covers the range of pages to be fully
311 * truncated. Partial pages are covered with 'partial_start' at the
312 * start of the range and 'partial_end' at the end of the range.
313 * Note that 'end' is exclusive while 'lend' is inclusive.
314 */
315 start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
316 if (lend == -1)
317 /*
318 * lend == -1 indicates end-of-file so we have to set 'end'
319 * to the highest possible pgoff_t and since the type is
320 * unsigned we're using -1.
321 */
322 end = -1;
323 else
324 end = (lend + 1) >> PAGE_SHIFT;
325
326 pagevec_init(&pvec);
327 index = start;
328 while (index < end && pagevec_lookup_entries(&pvec, mapping, index,
329 min(end - index, (pgoff_t)PAGEVEC_SIZE),
330 indices)) {
331 /*
332 * Pagevec array has exceptional entries and we may also fail
333 * to lock some pages. So we store pages that can be deleted
334 * in a new pagevec.
335 */
336 struct pagevec locked_pvec;
337
338 pagevec_init(&locked_pvec);
339 for (i = 0; i < pagevec_count(&pvec); i++) {
340 struct page *page = pvec.pages[i];
341
342 /* We rely upon deletion not changing page->index */
343 index = indices[i];
344 if (index >= end)
345 break;
346
347 if (xa_is_value(page))
348 continue;
349
350 if (!trylock_page(page))
351 continue;
352 WARN_ON(page_to_index(page) != index);
353 if (PageWriteback(page)) {
354 unlock_page(page);
355 continue;
356 }
357 if (page->mapping != mapping) {
358 unlock_page(page);
359 continue;
360 }
361 pagevec_add(&locked_pvec, page);
362 }
363 for (i = 0; i < pagevec_count(&locked_pvec); i++)
364 truncate_cleanup_page(mapping, locked_pvec.pages[i]);
365 delete_from_page_cache_batch(mapping, &locked_pvec);
366 for (i = 0; i < pagevec_count(&locked_pvec); i++)
367 unlock_page(locked_pvec.pages[i]);
368 truncate_exceptional_pvec_entries(mapping, &pvec, indices, end);
369 pagevec_release(&pvec);
370 cond_resched();
371 index++;
372 }
373 if (partial_start) {
374 struct page *page = find_lock_page(mapping, start - 1);
375 if (page) {
376 unsigned int top = PAGE_SIZE;
377 if (start > end) {
378 /* Truncation within a single page */
379 top = partial_end;
380 partial_end = 0;
381 }
382 wait_on_page_writeback(page);
383 zero_user_segment(page, partial_start, top);
384 cleancache_invalidate_page(mapping, page);
385 if (page_has_private(page))
386 do_invalidatepage(page, partial_start,
387 top - partial_start);
388 unlock_page(page);
389 put_page(page);
390 }
391 }
392 if (partial_end) {
393 struct page *page = find_lock_page(mapping, end);
394 if (page) {
395 wait_on_page_writeback(page);
396 zero_user_segment(page, 0, partial_end);
397 cleancache_invalidate_page(mapping, page);
398 if (page_has_private(page))
399 do_invalidatepage(page, 0,
400 partial_end);
401 unlock_page(page);
402 put_page(page);
403 }
404 }
405 /*
406 * If the truncation happened within a single page no pages
407 * will be released, just zeroed, so we can bail out now.
408 */
409 if (start >= end)
410 goto out;
411
412 index = start;
413 for ( ; ; ) {
414 cond_resched();
415 if (!pagevec_lookup_entries(&pvec, mapping, index,
416 min(end - index, (pgoff_t)PAGEVEC_SIZE), indices)) {
417 /* If all gone from start onwards, we're done */
418 if (index == start)
419 break;
420 /* Otherwise restart to make sure all gone */
421 index = start;
422 continue;
423 }
424 if (index == start && indices[0] >= end) {
425 /* All gone out of hole to be punched, we're done */
426 pagevec_remove_exceptionals(&pvec);
427 pagevec_release(&pvec);
428 break;
429 }
430
431 for (i = 0; i < pagevec_count(&pvec); i++) {
432 struct page *page = pvec.pages[i];
433
434 /* We rely upon deletion not changing page->index */
435 index = indices[i];
436 if (index >= end) {
437 /* Restart punch to make sure all gone */
438 index = start - 1;
439 break;
440 }
441
442 if (xa_is_value(page))
443 continue;
444
445 lock_page(page);
446 WARN_ON(page_to_index(page) != index);
447 wait_on_page_writeback(page);
448 truncate_inode_page(mapping, page);
449 unlock_page(page);
450 }
451 truncate_exceptional_pvec_entries(mapping, &pvec, indices, end);
452 pagevec_release(&pvec);
453 index++;
454 }
455
456out:
457 cleancache_invalidate_inode(mapping);
458}
459EXPORT_SYMBOL(truncate_inode_pages_range);
460
461/**
462 * truncate_inode_pages - truncate *all* the pages from an offset
463 * @mapping: mapping to truncate
464 * @lstart: offset from which to truncate
465 *
466 * Called under (and serialised by) inode->i_mutex.
467 *
468 * Note: When this function returns, there can be a page in the process of
469 * deletion (inside __delete_from_page_cache()) in the specified range. Thus
470 * mapping->nrpages can be non-zero when this function returns even after
471 * truncation of the whole mapping.
472 */
473void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
474{
475 truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
476}
477EXPORT_SYMBOL(truncate_inode_pages);
478
479/**
480 * truncate_inode_pages_final - truncate *all* pages before inode dies
481 * @mapping: mapping to truncate
482 *
483 * Called under (and serialized by) inode->i_mutex.
484 *
485 * Filesystems have to use this in the .evict_inode path to inform the
486 * VM that this is the final truncate and the inode is going away.
487 */
488void truncate_inode_pages_final(struct address_space *mapping)
489{
490 unsigned long nrexceptional;
491 unsigned long nrpages;
492
493 /*
494 * Page reclaim can not participate in regular inode lifetime
495 * management (can't call iput()) and thus can race with the
496 * inode teardown. Tell it when the address space is exiting,
497 * so that it does not install eviction information after the
498 * final truncate has begun.
499 */
500 mapping_set_exiting(mapping);
501
502 /*
503 * When reclaim installs eviction entries, it increases
504 * nrexceptional first, then decreases nrpages. Make sure we see
505 * this in the right order or we might miss an entry.
506 */
507 nrpages = mapping->nrpages;
508 smp_rmb();
509 nrexceptional = mapping->nrexceptional;
510
511 if (nrpages || nrexceptional) {
512 /*
513 * As truncation uses a lockless tree lookup, cycle
514 * the tree lock to make sure any ongoing tree
515 * modification that does not see AS_EXITING is
516 * completed before starting the final truncate.
517 */
518 xa_lock_irq(&mapping->i_pages);
519 xa_unlock_irq(&mapping->i_pages);
520 }
521
522 /*
523 * Cleancache needs notification even if there are no pages or shadow
524 * entries.
525 */
526 truncate_inode_pages(mapping, 0);
527}
528EXPORT_SYMBOL(truncate_inode_pages_final);
529
530/**
531 * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
532 * @mapping: the address_space which holds the pages to invalidate
533 * @start: the offset 'from' which to invalidate
534 * @end: the offset 'to' which to invalidate (inclusive)
535 *
536 * This function only removes the unlocked pages, if you want to
537 * remove all the pages of one inode, you must call truncate_inode_pages.
538 *
539 * invalidate_mapping_pages() will not block on IO activity. It will not
540 * invalidate pages which are dirty, locked, under writeback or mapped into
541 * pagetables.
542 *
543 * Return: the number of the pages that were invalidated
544 */
545unsigned long invalidate_mapping_pages(struct address_space *mapping,
546 pgoff_t start, pgoff_t end)
547{
548 pgoff_t indices[PAGEVEC_SIZE];
549 struct pagevec pvec;
550 pgoff_t index = start;
551 unsigned long ret;
552 unsigned long count = 0;
553 int i;
554
555 pagevec_init(&pvec);
556 while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
557 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
558 indices)) {
559 for (i = 0; i < pagevec_count(&pvec); i++) {
560 struct page *page = pvec.pages[i];
561
562 /* We rely upon deletion not changing page->index */
563 index = indices[i];
564 if (index > end)
565 break;
566
567 if (xa_is_value(page)) {
568 invalidate_exceptional_entry(mapping, index,
569 page);
570 continue;
571 }
572
573 if (!trylock_page(page))
574 continue;
575
576 WARN_ON(page_to_index(page) != index);
577
578 /* Middle of THP: skip */
579 if (PageTransTail(page)) {
580 unlock_page(page);
581 continue;
582 } else if (PageTransHuge(page)) {
583 index += HPAGE_PMD_NR - 1;
584 i += HPAGE_PMD_NR - 1;
585 /*
586 * 'end' is in the middle of THP. Don't
587 * invalidate the page as the part outside of
588 * 'end' could be still useful.
589 */
590 if (index > end) {
591 unlock_page(page);
592 continue;
593 }
594 }
595
596 ret = invalidate_inode_page(page);
597 unlock_page(page);
598 /*
599 * Invalidation is a hint that the page is no longer
600 * of interest and try to speed up its reclaim.
601 */
602 if (!ret)
603 deactivate_file_page(page);
604 count += ret;
605 }
606 pagevec_remove_exceptionals(&pvec);
607 pagevec_release(&pvec);
608 cond_resched();
609 index++;
610 }
611 return count;
612}
613EXPORT_SYMBOL(invalidate_mapping_pages);
614
615/*
616 * This is like invalidate_complete_page(), except it ignores the page's
617 * refcount. We do this because invalidate_inode_pages2() needs stronger
618 * invalidation guarantees, and cannot afford to leave pages behind because
619 * shrink_page_list() has a temp ref on them, or because they're transiently
620 * sitting in the lru_cache_add() pagevecs.
621 */
622static int
623invalidate_complete_page2(struct address_space *mapping, struct page *page)
624{
625 unsigned long flags;
626
627 if (page->mapping != mapping)
628 return 0;
629
630 if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL))
631 return 0;
632
633 xa_lock_irqsave(&mapping->i_pages, flags);
634 if (PageDirty(page))
635 goto failed;
636
637 BUG_ON(page_has_private(page));
638 __delete_from_page_cache(page, NULL);
639 xa_unlock_irqrestore(&mapping->i_pages, flags);
640
641 if (mapping->a_ops->freepage)
642 mapping->a_ops->freepage(page);
643
644 put_page(page); /* pagecache ref */
645 return 1;
646failed:
647 xa_unlock_irqrestore(&mapping->i_pages, flags);
648 return 0;
649}
650
651static int do_launder_page(struct address_space *mapping, struct page *page)
652{
653 if (!PageDirty(page))
654 return 0;
655 if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)
656 return 0;
657 return mapping->a_ops->launder_page(page);
658}
659
660/**
661 * invalidate_inode_pages2_range - remove range of pages from an address_space
662 * @mapping: the address_space
663 * @start: the page offset 'from' which to invalidate
664 * @end: the page offset 'to' which to invalidate (inclusive)
665 *
666 * Any pages which are found to be mapped into pagetables are unmapped prior to
667 * invalidation.
668 *
669 * Return: -EBUSY if any pages could not be invalidated.
670 */
671int invalidate_inode_pages2_range(struct address_space *mapping,
672 pgoff_t start, pgoff_t end)
673{
674 pgoff_t indices[PAGEVEC_SIZE];
675 struct pagevec pvec;
676 pgoff_t index;
677 int i;
678 int ret = 0;
679 int ret2 = 0;
680 int did_range_unmap = 0;
681
682 if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
683 goto out;
684
685 pagevec_init(&pvec);
686 index = start;
687 while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
688 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
689 indices)) {
690 for (i = 0; i < pagevec_count(&pvec); i++) {
691 struct page *page = pvec.pages[i];
692
693 /* We rely upon deletion not changing page->index */
694 index = indices[i];
695 if (index > end)
696 break;
697
698 if (xa_is_value(page)) {
699 if (!invalidate_exceptional_entry2(mapping,
700 index, page))
701 ret = -EBUSY;
702 continue;
703 }
704
705 lock_page(page);
706 WARN_ON(page_to_index(page) != index);
707 if (page->mapping != mapping) {
708 unlock_page(page);
709 continue;
710 }
711 wait_on_page_writeback(page);
712 if (page_mapped(page)) {
713 if (!did_range_unmap) {
714 /*
715 * Zap the rest of the file in one hit.
716 */
717 unmap_mapping_pages(mapping, index,
718 (1 + end - index), false);
719 did_range_unmap = 1;
720 } else {
721 /*
722 * Just zap this page
723 */
724 unmap_mapping_pages(mapping, index,
725 1, false);
726 }
727 }
728 BUG_ON(page_mapped(page));
729 ret2 = do_launder_page(mapping, page);
730 if (ret2 == 0) {
731 if (!invalidate_complete_page2(mapping, page))
732 ret2 = -EBUSY;
733 }
734 if (ret2 < 0)
735 ret = ret2;
736 unlock_page(page);
737 }
738 pagevec_remove_exceptionals(&pvec);
739 pagevec_release(&pvec);
740 cond_resched();
741 index++;
742 }
743 /*
744 * For DAX we invalidate page tables after invalidating page cache. We
745 * could invalidate page tables while invalidating each entry however
746 * that would be expensive. And doing range unmapping before doesn't
747 * work as we have no cheap way to find whether page cache entry didn't
748 * get remapped later.
749 */
750 if (dax_mapping(mapping)) {
751 unmap_mapping_pages(mapping, start, end - start + 1, false);
752 }
753out:
754 cleancache_invalidate_inode(mapping);
755 return ret;
756}
757EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
758
759/**
760 * invalidate_inode_pages2 - remove all pages from an address_space
761 * @mapping: the address_space
762 *
763 * Any pages which are found to be mapped into pagetables are unmapped prior to
764 * invalidation.
765 *
766 * Return: -EBUSY if any pages could not be invalidated.
767 */
768int invalidate_inode_pages2(struct address_space *mapping)
769{
770 return invalidate_inode_pages2_range(mapping, 0, -1);
771}
772EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
773
774/**
775 * truncate_pagecache - unmap and remove pagecache that has been truncated
776 * @inode: inode
777 * @newsize: new file size
778 *
779 * inode's new i_size must already be written before truncate_pagecache
780 * is called.
781 *
782 * This function should typically be called before the filesystem
783 * releases resources associated with the freed range (eg. deallocates
784 * blocks). This way, pagecache will always stay logically coherent
785 * with on-disk format, and the filesystem would not have to deal with
786 * situations such as writepage being called for a page that has already
787 * had its underlying blocks deallocated.
788 */
789void truncate_pagecache(struct inode *inode, loff_t newsize)
790{
791 struct address_space *mapping = inode->i_mapping;
792 loff_t holebegin = round_up(newsize, PAGE_SIZE);
793
794 /*
795 * unmap_mapping_range is called twice, first simply for
796 * efficiency so that truncate_inode_pages does fewer
797 * single-page unmaps. However after this first call, and
798 * before truncate_inode_pages finishes, it is possible for
799 * private pages to be COWed, which remain after
800 * truncate_inode_pages finishes, hence the second
801 * unmap_mapping_range call must be made for correctness.
802 */
803 unmap_mapping_range(mapping, holebegin, 0, 1);
804 truncate_inode_pages(mapping, newsize);
805 unmap_mapping_range(mapping, holebegin, 0, 1);
806}
807EXPORT_SYMBOL(truncate_pagecache);
808
809/**
810 * truncate_setsize - update inode and pagecache for a new file size
811 * @inode: inode
812 * @newsize: new file size
813 *
814 * truncate_setsize updates i_size and performs pagecache truncation (if
815 * necessary) to @newsize. It will be typically be called from the filesystem's
816 * setattr function when ATTR_SIZE is passed in.
817 *
818 * Must be called with a lock serializing truncates and writes (generally
819 * i_mutex but e.g. xfs uses a different lock) and before all filesystem
820 * specific block truncation has been performed.
821 */
822void truncate_setsize(struct inode *inode, loff_t newsize)
823{
824 loff_t oldsize = inode->i_size;
825
826 i_size_write(inode, newsize);
827 if (newsize > oldsize)
828 pagecache_isize_extended(inode, oldsize, newsize);
829 truncate_pagecache(inode, newsize);
830}
831EXPORT_SYMBOL(truncate_setsize);
832
833/**
834 * pagecache_isize_extended - update pagecache after extension of i_size
835 * @inode: inode for which i_size was extended
836 * @from: original inode size
837 * @to: new inode size
838 *
839 * Handle extension of inode size either caused by extending truncate or by
840 * write starting after current i_size. We mark the page straddling current
841 * i_size RO so that page_mkwrite() is called on the nearest write access to
842 * the page. This way filesystem can be sure that page_mkwrite() is called on
843 * the page before user writes to the page via mmap after the i_size has been
844 * changed.
845 *
846 * The function must be called after i_size is updated so that page fault
847 * coming after we unlock the page will already see the new i_size.
848 * The function must be called while we still hold i_mutex - this not only
849 * makes sure i_size is stable but also that userspace cannot observe new
850 * i_size value before we are prepared to store mmap writes at new inode size.
851 */
852void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to)
853{
854 int bsize = i_blocksize(inode);
855 loff_t rounded_from;
856 struct page *page;
857 pgoff_t index;
858
859 WARN_ON(to > inode->i_size);
860
861 if (from >= to || bsize == PAGE_SIZE)
862 return;
863 /* Page straddling @from will not have any hole block created? */
864 rounded_from = round_up(from, bsize);
865 if (to <= rounded_from || !(rounded_from & (PAGE_SIZE - 1)))
866 return;
867
868 index = from >> PAGE_SHIFT;
869 page = find_lock_page(inode->i_mapping, index);
870 /* Page not cached? Nothing to do */
871 if (!page)
872 return;
873 /*
874 * See clear_page_dirty_for_io() for details why set_page_dirty()
875 * is needed.
876 */
877 if (page_mkclean(page))
878 set_page_dirty(page);
879 unlock_page(page);
880 put_page(page);
881}
882EXPORT_SYMBOL(pagecache_isize_extended);
883
884/**
885 * truncate_pagecache_range - unmap and remove pagecache that is hole-punched
886 * @inode: inode
887 * @lstart: offset of beginning of hole
888 * @lend: offset of last byte of hole
889 *
890 * This function should typically be called before the filesystem
891 * releases resources associated with the freed range (eg. deallocates
892 * blocks). This way, pagecache will always stay logically coherent
893 * with on-disk format, and the filesystem would not have to deal with
894 * situations such as writepage being called for a page that has already
895 * had its underlying blocks deallocated.
896 */
897void truncate_pagecache_range(struct inode *inode, loff_t lstart, loff_t lend)
898{
899 struct address_space *mapping = inode->i_mapping;
900 loff_t unmap_start = round_up(lstart, PAGE_SIZE);
901 loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1;
902 /*
903 * This rounding is currently just for example: unmap_mapping_range
904 * expands its hole outwards, whereas we want it to contract the hole
905 * inwards. However, existing callers of truncate_pagecache_range are
906 * doing their own page rounding first. Note that unmap_mapping_range
907 * allows holelen 0 for all, and we allow lend -1 for end of file.
908 */
909
910 /*
911 * Unlike in truncate_pagecache, unmap_mapping_range is called only
912 * once (before truncating pagecache), and without "even_cows" flag:
913 * hole-punching should not remove private COWed pages from the hole.
914 */
915 if ((u64)unmap_end > (u64)unmap_start)
916 unmap_mapping_range(mapping, unmap_start,
917 1 + unmap_end - unmap_start, 0);
918 truncate_inode_pages_range(mapping, lstart, lend);
919}
920EXPORT_SYMBOL(truncate_pagecache_range);
921