1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * mm/truncate.c - code for taking down pages from address_spaces
4 *
5 * Copyright (C) 2002, Linus Torvalds
6 *
7 * 10Sep2002 Andrew Morton
8 * Initial version.
9 */
10
11#include <linux/kernel.h>
12#include <linux/backing-dev.h>
13#include <linux/dax.h>
14#include <linux/gfp.h>
15#include <linux/mm.h>
16#include <linux/swap.h>
17#include <linux/export.h>
18#include <linux/pagemap.h>
19#include <linux/highmem.h>
20#include <linux/pagevec.h>
21#include <linux/task_io_accounting_ops.h>
22#include <linux/shmem_fs.h>
23#include <linux/rmap.h>
24#include "internal.h"
25
26/*
27 * Regular page slots are stabilized by the page lock even without the tree
28 * itself locked. These unlocked entries need verification under the tree
29 * lock.
30 */
31static inline void __clear_shadow_entry(struct address_space *mapping,
32 pgoff_t index, void *entry)
33{
34 XA_STATE(xas, &mapping->i_pages, index);
35
36 xas_set_update(xas: &xas, update: workingset_update_node);
37 if (xas_load(&xas) != entry)
38 return;
39 xas_store(&xas, NULL);
40}
41
42static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,
43 void *entry)
44{
45 spin_lock(lock: &mapping->host->i_lock);
46 xa_lock_irq(&mapping->i_pages);
47 __clear_shadow_entry(mapping, index, entry);
48 xa_unlock_irq(&mapping->i_pages);
49 if (mapping_shrinkable(mapping))
50 inode_add_lru(inode: mapping->host);
51 spin_unlock(lock: &mapping->host->i_lock);
52}
53
54/*
55 * Unconditionally remove exceptional entries. Usually called from truncate
56 * path. Note that the folio_batch may be altered by this function by removing
57 * exceptional entries similar to what folio_batch_remove_exceptionals() does.
58 */
59static void truncate_folio_batch_exceptionals(struct address_space *mapping,
60 struct folio_batch *fbatch, pgoff_t *indices)
61{
62 int i, j;
63 bool dax;
64
65 /* Handled by shmem itself */
66 if (shmem_mapping(mapping))
67 return;
68
69 for (j = 0; j < folio_batch_count(fbatch); j++)
70 if (xa_is_value(entry: fbatch->folios[j]))
71 break;
72
73 if (j == folio_batch_count(fbatch))
74 return;
75
76 dax = dax_mapping(mapping);
77 if (!dax) {
78 spin_lock(lock: &mapping->host->i_lock);
79 xa_lock_irq(&mapping->i_pages);
80 }
81
82 for (i = j; i < folio_batch_count(fbatch); i++) {
83 struct folio *folio = fbatch->folios[i];
84 pgoff_t index = indices[i];
85
86 if (!xa_is_value(entry: folio)) {
87 fbatch->folios[j++] = folio;
88 continue;
89 }
90
91 if (unlikely(dax)) {
92 dax_delete_mapping_entry(mapping, index);
93 continue;
94 }
95
96 __clear_shadow_entry(mapping, index, entry: folio);
97 }
98
99 if (!dax) {
100 xa_unlock_irq(&mapping->i_pages);
101 if (mapping_shrinkable(mapping))
102 inode_add_lru(inode: mapping->host);
103 spin_unlock(lock: &mapping->host->i_lock);
104 }
105 fbatch->nr = j;
106}
107
108/*
109 * Invalidate exceptional entry if easily possible. This handles exceptional
110 * entries for invalidate_inode_pages().
111 */
112static int invalidate_exceptional_entry(struct address_space *mapping,
113 pgoff_t index, void *entry)
114{
115 /* Handled by shmem itself, or for DAX we do nothing. */
116 if (shmem_mapping(mapping) || dax_mapping(mapping))
117 return 1;
118 clear_shadow_entry(mapping, index, entry);
119 return 1;
120}
121
122/*
123 * Invalidate exceptional entry if clean. This handles exceptional entries for
124 * invalidate_inode_pages2() so for DAX it evicts only clean entries.
125 */
126static int invalidate_exceptional_entry2(struct address_space *mapping,
127 pgoff_t index, void *entry)
128{
129 /* Handled by shmem itself */
130 if (shmem_mapping(mapping))
131 return 1;
132 if (dax_mapping(mapping))
133 return dax_invalidate_mapping_entry_sync(mapping, index);
134 clear_shadow_entry(mapping, index, entry);
135 return 1;
136}
137
138/**
139 * folio_invalidate - Invalidate part or all of a folio.
140 * @folio: The folio which is affected.
141 * @offset: start of the range to invalidate
142 * @length: length of the range to invalidate
143 *
144 * folio_invalidate() is called when all or part of the folio has become
145 * invalidated by a truncate operation.
146 *
147 * folio_invalidate() does not have to release all buffers, but it must
148 * ensure that no dirty buffer is left outside @offset and that no I/O
149 * is underway against any of the blocks which are outside the truncation
150 * point. Because the caller is about to free (and possibly reuse) those
151 * blocks on-disk.
152 */
153void folio_invalidate(struct folio *folio, size_t offset, size_t length)
154{
155 const struct address_space_operations *aops = folio->mapping->a_ops;
156
157 if (aops->invalidate_folio)
158 aops->invalidate_folio(folio, offset, length);
159}
160EXPORT_SYMBOL_GPL(folio_invalidate);
161
162/*
163 * If truncate cannot remove the fs-private metadata from the page, the page
164 * becomes orphaned. It will be left on the LRU and may even be mapped into
165 * user pagetables if we're racing with filemap_fault().
166 *
167 * We need to bail out if page->mapping is no longer equal to the original
168 * mapping. This happens a) when the VM reclaimed the page while we waited on
169 * its lock, b) when a concurrent invalidate_mapping_pages got there first and
170 * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
171 */
172static void truncate_cleanup_folio(struct folio *folio)
173{
174 if (folio_mapped(folio))
175 unmap_mapping_folio(folio);
176
177 if (folio_has_private(folio))
178 folio_invalidate(folio, 0, folio_size(folio));
179
180 /*
181 * Some filesystems seem to re-dirty the page even after
182 * the VM has canceled the dirty bit (eg ext3 journaling).
183 * Hence dirty accounting check is placed after invalidation.
184 */
185 folio_cancel_dirty(folio);
186 folio_clear_mappedtodisk(folio);
187}
188
189int truncate_inode_folio(struct address_space *mapping, struct folio *folio)
190{
191 if (folio->mapping != mapping)
192 return -EIO;
193
194 truncate_cleanup_folio(folio);
195 filemap_remove_folio(folio);
196 return 0;
197}
198
199/*
200 * Handle partial folios. The folio may be entirely within the
201 * range if a split has raced with us. If not, we zero the part of the
202 * folio that's within the [start, end] range, and then split the folio if
203 * it's large. split_page_range() will discard pages which now lie beyond
204 * i_size, and we rely on the caller to discard pages which lie within a
205 * newly created hole.
206 *
207 * Returns false if splitting failed so the caller can avoid
208 * discarding the entire folio which is stubbornly unsplit.
209 */
210bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end)
211{
212 loff_t pos = folio_pos(folio);
213 unsigned int offset, length;
214
215 if (pos < start)
216 offset = start - pos;
217 else
218 offset = 0;
219 length = folio_size(folio);
220 if (pos + length <= (u64)end)
221 length = length - offset;
222 else
223 length = end + 1 - pos - offset;
224
225 folio_wait_writeback(folio);
226 if (length == folio_size(folio)) {
227 truncate_inode_folio(mapping: folio->mapping, folio);
228 return true;
229 }
230
231 /*
232 * We may be zeroing pages we're about to discard, but it avoids
233 * doing a complex calculation here, and then doing the zeroing
234 * anyway if the page split fails.
235 */
236 folio_zero_range(folio, start: offset, length);
237
238 if (folio_has_private(folio))
239 folio_invalidate(folio, offset, length);
240 if (!folio_test_large(folio))
241 return true;
242 if (split_folio(folio) == 0)
243 return true;
244 if (folio_test_dirty(folio))
245 return false;
246 truncate_inode_folio(mapping: folio->mapping, folio);
247 return true;
248}
249
250/*
251 * Used to get rid of pages on hardware memory corruption.
252 */
253int generic_error_remove_page(struct address_space *mapping, struct page *page)
254{
255 VM_BUG_ON_PAGE(PageTail(page), page);
256
257 if (!mapping)
258 return -EINVAL;
259 /*
260 * Only punch for normal data pages for now.
261 * Handling other types like directories would need more auditing.
262 */
263 if (!S_ISREG(mapping->host->i_mode))
264 return -EIO;
265 return truncate_inode_folio(mapping, page_folio(page));
266}
267EXPORT_SYMBOL(generic_error_remove_page);
268
269static long mapping_evict_folio(struct address_space *mapping,
270 struct folio *folio)
271{
272 if (folio_test_dirty(folio) || folio_test_writeback(folio))
273 return 0;
274 /* The refcount will be elevated if any page in the folio is mapped */
275 if (folio_ref_count(folio) >
276 folio_nr_pages(folio) + folio_has_private(folio) + 1)
277 return 0;
278 if (!filemap_release_folio(folio, gfp: 0))
279 return 0;
280
281 return remove_mapping(mapping, folio);
282}
283
284/**
285 * invalidate_inode_page() - Remove an unused page from the pagecache.
286 * @page: The page to remove.
287 *
288 * Safely invalidate one page from its pagecache mapping.
289 * It only drops clean, unused pages.
290 *
291 * Context: Page must be locked.
292 * Return: The number of pages successfully removed.
293 */
294long invalidate_inode_page(struct page *page)
295{
296 struct folio *folio = page_folio(page);
297 struct address_space *mapping = folio_mapping(folio);
298
299 /* The page may have been truncated before it was locked */
300 if (!mapping)
301 return 0;
302 return mapping_evict_folio(mapping, folio);
303}
304
305/**
306 * truncate_inode_pages_range - truncate range of pages specified by start & end byte offsets
307 * @mapping: mapping to truncate
308 * @lstart: offset from which to truncate
309 * @lend: offset to which to truncate (inclusive)
310 *
311 * Truncate the page cache, removing the pages that are between
312 * specified offsets (and zeroing out partial pages
313 * if lstart or lend + 1 is not page aligned).
314 *
315 * Truncate takes two passes - the first pass is nonblocking. It will not
316 * block on page locks and it will not block on writeback. The second pass
317 * will wait. This is to prevent as much IO as possible in the affected region.
318 * The first pass will remove most pages, so the search cost of the second pass
319 * is low.
320 *
321 * We pass down the cache-hot hint to the page freeing code. Even if the
322 * mapping is large, it is probably the case that the final pages are the most
323 * recently touched, and freeing happens in ascending file offset order.
324 *
325 * Note that since ->invalidate_folio() accepts range to invalidate
326 * truncate_inode_pages_range is able to handle cases where lend + 1 is not
327 * page aligned properly.
328 */
329void truncate_inode_pages_range(struct address_space *mapping,
330 loff_t lstart, loff_t lend)
331{
332 pgoff_t start; /* inclusive */
333 pgoff_t end; /* exclusive */
334 struct folio_batch fbatch;
335 pgoff_t indices[PAGEVEC_SIZE];
336 pgoff_t index;
337 int i;
338 struct folio *folio;
339 bool same_folio;
340
341 if (mapping_empty(mapping))
342 return;
343
344 /*
345 * 'start' and 'end' always covers the range of pages to be fully
346 * truncated. Partial pages are covered with 'partial_start' at the
347 * start of the range and 'partial_end' at the end of the range.
348 * Note that 'end' is exclusive while 'lend' is inclusive.
349 */
350 start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
351 if (lend == -1)
352 /*
353 * lend == -1 indicates end-of-file so we have to set 'end'
354 * to the highest possible pgoff_t and since the type is
355 * unsigned we're using -1.
356 */
357 end = -1;
358 else
359 end = (lend + 1) >> PAGE_SHIFT;
360
361 folio_batch_init(fbatch: &fbatch);
362 index = start;
363 while (index < end && find_lock_entries(mapping, start: &index, end: end - 1,
364 fbatch: &fbatch, indices)) {
365 truncate_folio_batch_exceptionals(mapping, fbatch: &fbatch, indices);
366 for (i = 0; i < folio_batch_count(fbatch: &fbatch); i++)
367 truncate_cleanup_folio(folio: fbatch.folios[i]);
368 delete_from_page_cache_batch(mapping, fbatch: &fbatch);
369 for (i = 0; i < folio_batch_count(fbatch: &fbatch); i++)
370 folio_unlock(folio: fbatch.folios[i]);
371 folio_batch_release(fbatch: &fbatch);
372 cond_resched();
373 }
374
375 same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT);
376 folio = __filemap_get_folio(mapping, index: lstart >> PAGE_SHIFT, FGP_LOCK, gfp: 0);
377 if (!IS_ERR(ptr: folio)) {
378 same_folio = lend < folio_pos(folio) + folio_size(folio);
379 if (!truncate_inode_partial_folio(folio, start: lstart, end: lend)) {
380 start = folio_next_index(folio);
381 if (same_folio)
382 end = folio->index;
383 }
384 folio_unlock(folio);
385 folio_put(folio);
386 folio = NULL;
387 }
388
389 if (!same_folio) {
390 folio = __filemap_get_folio(mapping, index: lend >> PAGE_SHIFT,
391 FGP_LOCK, gfp: 0);
392 if (!IS_ERR(ptr: folio)) {
393 if (!truncate_inode_partial_folio(folio, start: lstart, end: lend))
394 end = folio->index;
395 folio_unlock(folio);
396 folio_put(folio);
397 }
398 }
399
400 index = start;
401 while (index < end) {
402 cond_resched();
403 if (!find_get_entries(mapping, start: &index, end: end - 1, fbatch: &fbatch,
404 indices)) {
405 /* If all gone from start onwards, we're done */
406 if (index == start)
407 break;
408 /* Otherwise restart to make sure all gone */
409 index = start;
410 continue;
411 }
412
413 for (i = 0; i < folio_batch_count(fbatch: &fbatch); i++) {
414 struct folio *folio = fbatch.folios[i];
415
416 /* We rely upon deletion not changing page->index */
417
418 if (xa_is_value(entry: folio))
419 continue;
420
421 folio_lock(folio);
422 VM_BUG_ON_FOLIO(!folio_contains(folio, indices[i]), folio);
423 folio_wait_writeback(folio);
424 truncate_inode_folio(mapping, folio);
425 folio_unlock(folio);
426 }
427 truncate_folio_batch_exceptionals(mapping, fbatch: &fbatch, indices);
428 folio_batch_release(fbatch: &fbatch);
429 }
430}
431EXPORT_SYMBOL(truncate_inode_pages_range);
432
433/**
434 * truncate_inode_pages - truncate *all* the pages from an offset
435 * @mapping: mapping to truncate
436 * @lstart: offset from which to truncate
437 *
438 * Called under (and serialised by) inode->i_rwsem and
439 * mapping->invalidate_lock.
440 *
441 * Note: When this function returns, there can be a page in the process of
442 * deletion (inside __filemap_remove_folio()) in the specified range. Thus
443 * mapping->nrpages can be non-zero when this function returns even after
444 * truncation of the whole mapping.
445 */
446void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
447{
448 truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
449}
450EXPORT_SYMBOL(truncate_inode_pages);
451
452/**
453 * truncate_inode_pages_final - truncate *all* pages before inode dies
454 * @mapping: mapping to truncate
455 *
456 * Called under (and serialized by) inode->i_rwsem.
457 *
458 * Filesystems have to use this in the .evict_inode path to inform the
459 * VM that this is the final truncate and the inode is going away.
460 */
461void truncate_inode_pages_final(struct address_space *mapping)
462{
463 /*
464 * Page reclaim can not participate in regular inode lifetime
465 * management (can't call iput()) and thus can race with the
466 * inode teardown. Tell it when the address space is exiting,
467 * so that it does not install eviction information after the
468 * final truncate has begun.
469 */
470 mapping_set_exiting(mapping);
471
472 if (!mapping_empty(mapping)) {
473 /*
474 * As truncation uses a lockless tree lookup, cycle
475 * the tree lock to make sure any ongoing tree
476 * modification that does not see AS_EXITING is
477 * completed before starting the final truncate.
478 */
479 xa_lock_irq(&mapping->i_pages);
480 xa_unlock_irq(&mapping->i_pages);
481 }
482
483 truncate_inode_pages(mapping, 0);
484}
485EXPORT_SYMBOL(truncate_inode_pages_final);
486
487/**
488 * mapping_try_invalidate - Invalidate all the evictable folios of one inode
489 * @mapping: the address_space which holds the folios to invalidate
490 * @start: the offset 'from' which to invalidate
491 * @end: the offset 'to' which to invalidate (inclusive)
492 * @nr_failed: How many folio invalidations failed
493 *
494 * This function is similar to invalidate_mapping_pages(), except that it
495 * returns the number of folios which could not be evicted in @nr_failed.
496 */
497unsigned long mapping_try_invalidate(struct address_space *mapping,
498 pgoff_t start, pgoff_t end, unsigned long *nr_failed)
499{
500 pgoff_t indices[PAGEVEC_SIZE];
501 struct folio_batch fbatch;
502 pgoff_t index = start;
503 unsigned long ret;
504 unsigned long count = 0;
505 int i;
506
507 folio_batch_init(fbatch: &fbatch);
508 while (find_lock_entries(mapping, start: &index, end, fbatch: &fbatch, indices)) {
509 for (i = 0; i < folio_batch_count(fbatch: &fbatch); i++) {
510 struct folio *folio = fbatch.folios[i];
511
512 /* We rely upon deletion not changing folio->index */
513
514 if (xa_is_value(entry: folio)) {
515 count += invalidate_exceptional_entry(mapping,
516 index: indices[i], entry: folio);
517 continue;
518 }
519
520 ret = mapping_evict_folio(mapping, folio);
521 folio_unlock(folio);
522 /*
523 * Invalidation is a hint that the folio is no longer
524 * of interest and try to speed up its reclaim.
525 */
526 if (!ret) {
527 deactivate_file_folio(folio);
528 /* Likely in the lru cache of a remote CPU */
529 if (nr_failed)
530 (*nr_failed)++;
531 }
532 count += ret;
533 }
534 folio_batch_remove_exceptionals(fbatch: &fbatch);
535 folio_batch_release(fbatch: &fbatch);
536 cond_resched();
537 }
538 return count;
539}
540
541/**
542 * invalidate_mapping_pages - Invalidate all clean, unlocked cache of one inode
543 * @mapping: the address_space which holds the cache to invalidate
544 * @start: the offset 'from' which to invalidate
545 * @end: the offset 'to' which to invalidate (inclusive)
546 *
547 * This function removes pages that are clean, unmapped and unlocked,
548 * as well as shadow entries. It will not block on IO activity.
549 *
550 * If you want to remove all the pages of one inode, regardless of
551 * their use and writeback state, use truncate_inode_pages().
552 *
553 * Return: The number of indices that had their contents invalidated
554 */
555unsigned long invalidate_mapping_pages(struct address_space *mapping,
556 pgoff_t start, pgoff_t end)
557{
558 return mapping_try_invalidate(mapping, start, end, NULL);
559}
560EXPORT_SYMBOL(invalidate_mapping_pages);
561
562/*
563 * This is like invalidate_inode_page(), except it ignores the page's
564 * refcount. We do this because invalidate_inode_pages2() needs stronger
565 * invalidation guarantees, and cannot afford to leave pages behind because
566 * shrink_page_list() has a temp ref on them, or because they're transiently
567 * sitting in the folio_add_lru() caches.
568 */
569static int invalidate_complete_folio2(struct address_space *mapping,
570 struct folio *folio)
571{
572 if (folio->mapping != mapping)
573 return 0;
574
575 if (!filemap_release_folio(folio, GFP_KERNEL))
576 return 0;
577
578 spin_lock(lock: &mapping->host->i_lock);
579 xa_lock_irq(&mapping->i_pages);
580 if (folio_test_dirty(folio))
581 goto failed;
582
583 BUG_ON(folio_has_private(folio));
584 __filemap_remove_folio(folio, NULL);
585 xa_unlock_irq(&mapping->i_pages);
586 if (mapping_shrinkable(mapping))
587 inode_add_lru(inode: mapping->host);
588 spin_unlock(lock: &mapping->host->i_lock);
589
590 filemap_free_folio(mapping, folio);
591 return 1;
592failed:
593 xa_unlock_irq(&mapping->i_pages);
594 spin_unlock(lock: &mapping->host->i_lock);
595 return 0;
596}
597
598static int folio_launder(struct address_space *mapping, struct folio *folio)
599{
600 if (!folio_test_dirty(folio))
601 return 0;
602 if (folio->mapping != mapping || mapping->a_ops->launder_folio == NULL)
603 return 0;
604 return mapping->a_ops->launder_folio(folio);
605}
606
607/**
608 * invalidate_inode_pages2_range - remove range of pages from an address_space
609 * @mapping: the address_space
610 * @start: the page offset 'from' which to invalidate
611 * @end: the page offset 'to' which to invalidate (inclusive)
612 *
613 * Any pages which are found to be mapped into pagetables are unmapped prior to
614 * invalidation.
615 *
616 * Return: -EBUSY if any pages could not be invalidated.
617 */
618int invalidate_inode_pages2_range(struct address_space *mapping,
619 pgoff_t start, pgoff_t end)
620{
621 pgoff_t indices[PAGEVEC_SIZE];
622 struct folio_batch fbatch;
623 pgoff_t index;
624 int i;
625 int ret = 0;
626 int ret2 = 0;
627 int did_range_unmap = 0;
628
629 if (mapping_empty(mapping))
630 return 0;
631
632 folio_batch_init(fbatch: &fbatch);
633 index = start;
634 while (find_get_entries(mapping, start: &index, end, fbatch: &fbatch, indices)) {
635 for (i = 0; i < folio_batch_count(fbatch: &fbatch); i++) {
636 struct folio *folio = fbatch.folios[i];
637
638 /* We rely upon deletion not changing folio->index */
639
640 if (xa_is_value(entry: folio)) {
641 if (!invalidate_exceptional_entry2(mapping,
642 index: indices[i], entry: folio))
643 ret = -EBUSY;
644 continue;
645 }
646
647 if (!did_range_unmap && folio_mapped(folio)) {
648 /*
649 * If folio is mapped, before taking its lock,
650 * zap the rest of the file in one hit.
651 */
652 unmap_mapping_pages(mapping, start: indices[i],
653 nr: (1 + end - indices[i]), even_cows: false);
654 did_range_unmap = 1;
655 }
656
657 folio_lock(folio);
658 if (unlikely(folio->mapping != mapping)) {
659 folio_unlock(folio);
660 continue;
661 }
662 VM_BUG_ON_FOLIO(!folio_contains(folio, indices[i]), folio);
663 folio_wait_writeback(folio);
664
665 if (folio_mapped(folio))
666 unmap_mapping_folio(folio);
667 BUG_ON(folio_mapped(folio));
668
669 ret2 = folio_launder(mapping, folio);
670 if (ret2 == 0) {
671 if (!invalidate_complete_folio2(mapping, folio))
672 ret2 = -EBUSY;
673 }
674 if (ret2 < 0)
675 ret = ret2;
676 folio_unlock(folio);
677 }
678 folio_batch_remove_exceptionals(fbatch: &fbatch);
679 folio_batch_release(fbatch: &fbatch);
680 cond_resched();
681 }
682 /*
683 * For DAX we invalidate page tables after invalidating page cache. We
684 * could invalidate page tables while invalidating each entry however
685 * that would be expensive. And doing range unmapping before doesn't
686 * work as we have no cheap way to find whether page cache entry didn't
687 * get remapped later.
688 */
689 if (dax_mapping(mapping)) {
690 unmap_mapping_pages(mapping, start, nr: end - start + 1, even_cows: false);
691 }
692 return ret;
693}
694EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
695
696/**
697 * invalidate_inode_pages2 - remove all pages from an address_space
698 * @mapping: the address_space
699 *
700 * Any pages which are found to be mapped into pagetables are unmapped prior to
701 * invalidation.
702 *
703 * Return: -EBUSY if any pages could not be invalidated.
704 */
705int invalidate_inode_pages2(struct address_space *mapping)
706{
707 return invalidate_inode_pages2_range(mapping, 0, -1);
708}
709EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
710
711/**
712 * truncate_pagecache - unmap and remove pagecache that has been truncated
713 * @inode: inode
714 * @newsize: new file size
715 *
716 * inode's new i_size must already be written before truncate_pagecache
717 * is called.
718 *
719 * This function should typically be called before the filesystem
720 * releases resources associated with the freed range (eg. deallocates
721 * blocks). This way, pagecache will always stay logically coherent
722 * with on-disk format, and the filesystem would not have to deal with
723 * situations such as writepage being called for a page that has already
724 * had its underlying blocks deallocated.
725 */
726void truncate_pagecache(struct inode *inode, loff_t newsize)
727{
728 struct address_space *mapping = inode->i_mapping;
729 loff_t holebegin = round_up(newsize, PAGE_SIZE);
730
731 /*
732 * unmap_mapping_range is called twice, first simply for
733 * efficiency so that truncate_inode_pages does fewer
734 * single-page unmaps. However after this first call, and
735 * before truncate_inode_pages finishes, it is possible for
736 * private pages to be COWed, which remain after
737 * truncate_inode_pages finishes, hence the second
738 * unmap_mapping_range call must be made for correctness.
739 */
740 unmap_mapping_range(mapping, holebegin, holelen: 0, even_cows: 1);
741 truncate_inode_pages(mapping, newsize);
742 unmap_mapping_range(mapping, holebegin, holelen: 0, even_cows: 1);
743}
744EXPORT_SYMBOL(truncate_pagecache);
745
746/**
747 * truncate_setsize - update inode and pagecache for a new file size
748 * @inode: inode
749 * @newsize: new file size
750 *
751 * truncate_setsize updates i_size and performs pagecache truncation (if
752 * necessary) to @newsize. It will be typically be called from the filesystem's
753 * setattr function when ATTR_SIZE is passed in.
754 *
755 * Must be called with a lock serializing truncates and writes (generally
756 * i_rwsem but e.g. xfs uses a different lock) and before all filesystem
757 * specific block truncation has been performed.
758 */
759void truncate_setsize(struct inode *inode, loff_t newsize)
760{
761 loff_t oldsize = inode->i_size;
762
763 i_size_write(inode, i_size: newsize);
764 if (newsize > oldsize)
765 pagecache_isize_extended(inode, from: oldsize, to: newsize);
766 truncate_pagecache(inode, newsize);
767}
768EXPORT_SYMBOL(truncate_setsize);
769
770/**
771 * pagecache_isize_extended - update pagecache after extension of i_size
772 * @inode: inode for which i_size was extended
773 * @from: original inode size
774 * @to: new inode size
775 *
776 * Handle extension of inode size either caused by extending truncate or by
777 * write starting after current i_size. We mark the page straddling current
778 * i_size RO so that page_mkwrite() is called on the nearest write access to
779 * the page. This way filesystem can be sure that page_mkwrite() is called on
780 * the page before user writes to the page via mmap after the i_size has been
781 * changed.
782 *
783 * The function must be called after i_size is updated so that page fault
784 * coming after we unlock the page will already see the new i_size.
785 * The function must be called while we still hold i_rwsem - this not only
786 * makes sure i_size is stable but also that userspace cannot observe new
787 * i_size value before we are prepared to store mmap writes at new inode size.
788 */
789void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to)
790{
791 int bsize = i_blocksize(node: inode);
792 loff_t rounded_from;
793 struct page *page;
794 pgoff_t index;
795
796 WARN_ON(to > inode->i_size);
797
798 if (from >= to || bsize == PAGE_SIZE)
799 return;
800 /* Page straddling @from will not have any hole block created? */
801 rounded_from = round_up(from, bsize);
802 if (to <= rounded_from || !(rounded_from & (PAGE_SIZE - 1)))
803 return;
804
805 index = from >> PAGE_SHIFT;
806 page = find_lock_page(mapping: inode->i_mapping, index);
807 /* Page not cached? Nothing to do */
808 if (!page)
809 return;
810 /*
811 * See clear_page_dirty_for_io() for details why set_page_dirty()
812 * is needed.
813 */
814 if (page_mkclean(page))
815 set_page_dirty(page);
816 unlock_page(page);
817 put_page(page);
818}
819EXPORT_SYMBOL(pagecache_isize_extended);
820
821/**
822 * truncate_pagecache_range - unmap and remove pagecache that is hole-punched
823 * @inode: inode
824 * @lstart: offset of beginning of hole
825 * @lend: offset of last byte of hole
826 *
827 * This function should typically be called before the filesystem
828 * releases resources associated with the freed range (eg. deallocates
829 * blocks). This way, pagecache will always stay logically coherent
830 * with on-disk format, and the filesystem would not have to deal with
831 * situations such as writepage being called for a page that has already
832 * had its underlying blocks deallocated.
833 */
834void truncate_pagecache_range(struct inode *inode, loff_t lstart, loff_t lend)
835{
836 struct address_space *mapping = inode->i_mapping;
837 loff_t unmap_start = round_up(lstart, PAGE_SIZE);
838 loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1;
839 /*
840 * This rounding is currently just for example: unmap_mapping_range
841 * expands its hole outwards, whereas we want it to contract the hole
842 * inwards. However, existing callers of truncate_pagecache_range are
843 * doing their own page rounding first. Note that unmap_mapping_range
844 * allows holelen 0 for all, and we allow lend -1 for end of file.
845 */
846
847 /*
848 * Unlike in truncate_pagecache, unmap_mapping_range is called only
849 * once (before truncating pagecache), and without "even_cows" flag:
850 * hole-punching should not remove private COWed pages from the hole.
851 */
852 if ((u64)unmap_end > (u64)unmap_start)
853 unmap_mapping_range(mapping, holebegin: unmap_start,
854 holelen: 1 + unmap_end - unmap_start, even_cows: 0);
855 truncate_inode_pages_range(mapping, lstart, lend);
856}
857EXPORT_SYMBOL(truncate_pagecache_range);
858

source code of linux/mm/truncate.c