1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * linux/mm/filemap.c |
4 | * |
5 | * Copyright (C) 1994-1999 Linus Torvalds |
6 | */ |
7 | |
8 | /* |
9 | * This file handles the generic file mmap semantics used by |
10 | * most "normal" filesystems (but you don't /have/ to use this: |
11 | * the NFS filesystem used to do this differently, for example) |
12 | */ |
13 | #include <linux/export.h> |
14 | #include <linux/compiler.h> |
15 | #include <linux/dax.h> |
16 | #include <linux/fs.h> |
17 | #include <linux/sched/signal.h> |
18 | #include <linux/uaccess.h> |
19 | #include <linux/capability.h> |
20 | #include <linux/kernel_stat.h> |
21 | #include <linux/gfp.h> |
22 | #include <linux/mm.h> |
23 | #include <linux/swap.h> |
24 | #include <linux/swapops.h> |
25 | #include <linux/syscalls.h> |
26 | #include <linux/mman.h> |
27 | #include <linux/pagemap.h> |
28 | #include <linux/file.h> |
29 | #include <linux/uio.h> |
30 | #include <linux/error-injection.h> |
31 | #include <linux/hash.h> |
32 | #include <linux/writeback.h> |
33 | #include <linux/backing-dev.h> |
34 | #include <linux/pagevec.h> |
35 | #include <linux/security.h> |
36 | #include <linux/cpuset.h> |
37 | #include <linux/hugetlb.h> |
38 | #include <linux/memcontrol.h> |
39 | #include <linux/shmem_fs.h> |
40 | #include <linux/rmap.h> |
41 | #include <linux/delayacct.h> |
42 | #include <linux/psi.h> |
43 | #include <linux/ramfs.h> |
44 | #include <linux/page_idle.h> |
45 | #include <linux/migrate.h> |
46 | #include <linux/pipe_fs_i.h> |
47 | #include <linux/splice.h> |
48 | #include <asm/pgalloc.h> |
49 | #include <asm/tlbflush.h> |
50 | #include "internal.h" |
51 | |
52 | #define CREATE_TRACE_POINTS |
53 | #include <trace/events/filemap.h> |
54 | |
55 | /* |
56 | * FIXME: remove all knowledge of the buffer layer from the core VM |
57 | */ |
58 | #include <linux/buffer_head.h> /* for try_to_free_buffers */ |
59 | |
60 | #include <asm/mman.h> |
61 | |
62 | #include "swap.h" |
63 | |
64 | /* |
65 | * Shared mappings implemented 30.11.1994. It's not fully working yet, |
66 | * though. |
67 | * |
68 | * Shared mappings now work. 15.8.1995 Bruno. |
69 | * |
70 | * finished 'unifying' the page and buffer cache and SMP-threaded the |
71 | * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com> |
72 | * |
73 | * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de> |
74 | */ |
75 | |
76 | /* |
77 | * Lock ordering: |
78 | * |
79 | * ->i_mmap_rwsem (truncate_pagecache) |
80 | * ->private_lock (__free_pte->block_dirty_folio) |
81 | * ->swap_lock (exclusive_swap_page, others) |
82 | * ->i_pages lock |
83 | * |
84 | * ->i_rwsem |
85 | * ->invalidate_lock (acquired by fs in truncate path) |
86 | * ->i_mmap_rwsem (truncate->unmap_mapping_range) |
87 | * |
88 | * ->mmap_lock |
89 | * ->i_mmap_rwsem |
90 | * ->page_table_lock or pte_lock (various, mainly in memory.c) |
91 | * ->i_pages lock (arch-dependent flush_dcache_mmap_lock) |
92 | * |
93 | * ->mmap_lock |
94 | * ->invalidate_lock (filemap_fault) |
95 | * ->lock_page (filemap_fault, access_process_vm) |
96 | * |
97 | * ->i_rwsem (generic_perform_write) |
98 | * ->mmap_lock (fault_in_readable->do_page_fault) |
99 | * |
100 | * bdi->wb.list_lock |
101 | * sb_lock (fs/fs-writeback.c) |
102 | * ->i_pages lock (__sync_single_inode) |
103 | * |
104 | * ->i_mmap_rwsem |
105 | * ->anon_vma.lock (vma_merge) |
106 | * |
107 | * ->anon_vma.lock |
108 | * ->page_table_lock or pte_lock (anon_vma_prepare and various) |
109 | * |
110 | * ->page_table_lock or pte_lock |
111 | * ->swap_lock (try_to_unmap_one) |
112 | * ->private_lock (try_to_unmap_one) |
113 | * ->i_pages lock (try_to_unmap_one) |
114 | * ->lruvec->lru_lock (follow_page->mark_page_accessed) |
115 | * ->lruvec->lru_lock (check_pte_range->isolate_lru_page) |
116 | * ->private_lock (page_remove_rmap->set_page_dirty) |
117 | * ->i_pages lock (page_remove_rmap->set_page_dirty) |
118 | * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) |
119 | * ->inode->i_lock (page_remove_rmap->set_page_dirty) |
120 | * ->memcg->move_lock (page_remove_rmap->folio_memcg_lock) |
121 | * bdi.wb->list_lock (zap_pte_range->set_page_dirty) |
122 | * ->inode->i_lock (zap_pte_range->set_page_dirty) |
123 | * ->private_lock (zap_pte_range->block_dirty_folio) |
124 | */ |
125 | |
126 | static void page_cache_delete(struct address_space *mapping, |
127 | struct folio *folio, void *shadow) |
128 | { |
129 | XA_STATE(xas, &mapping->i_pages, folio->index); |
130 | long nr = 1; |
131 | |
132 | mapping_set_update(&xas, mapping); |
133 | |
134 | xas_set_order(xas: &xas, index: folio->index, order: folio_order(folio)); |
135 | nr = folio_nr_pages(folio); |
136 | |
137 | VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); |
138 | |
139 | xas_store(&xas, entry: shadow); |
140 | xas_init_marks(&xas); |
141 | |
142 | folio->mapping = NULL; |
143 | /* Leave page->index set: truncation lookup relies upon it */ |
144 | mapping->nrpages -= nr; |
145 | } |
146 | |
147 | static void filemap_unaccount_folio(struct address_space *mapping, |
148 | struct folio *folio) |
149 | { |
150 | long nr; |
151 | |
152 | VM_BUG_ON_FOLIO(folio_mapped(folio), folio); |
153 | if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(folio_mapped(folio))) { |
154 | pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n" , |
155 | current->comm, folio_pfn(folio)); |
156 | dump_page(page: &folio->page, reason: "still mapped when deleted" ); |
157 | dump_stack(); |
158 | add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); |
159 | |
160 | if (mapping_exiting(mapping) && !folio_test_large(folio)) { |
161 | int mapcount = page_mapcount(page: &folio->page); |
162 | |
163 | if (folio_ref_count(folio) >= mapcount + 2) { |
164 | /* |
165 | * All vmas have already been torn down, so it's |
166 | * a good bet that actually the page is unmapped |
167 | * and we'd rather not leak it: if we're wrong, |
168 | * another bad page check should catch it later. |
169 | */ |
170 | page_mapcount_reset(page: &folio->page); |
171 | folio_ref_sub(folio, nr: mapcount); |
172 | } |
173 | } |
174 | } |
175 | |
176 | /* hugetlb folios do not participate in page cache accounting. */ |
177 | if (folio_test_hugetlb(folio)) |
178 | return; |
179 | |
180 | nr = folio_nr_pages(folio); |
181 | |
182 | __lruvec_stat_mod_folio(folio, idx: NR_FILE_PAGES, val: -nr); |
183 | if (folio_test_swapbacked(folio)) { |
184 | __lruvec_stat_mod_folio(folio, idx: NR_SHMEM, val: -nr); |
185 | if (folio_test_pmd_mappable(folio)) |
186 | __lruvec_stat_mod_folio(folio, idx: NR_SHMEM_THPS, val: -nr); |
187 | } else if (folio_test_pmd_mappable(folio)) { |
188 | __lruvec_stat_mod_folio(folio, idx: NR_FILE_THPS, val: -nr); |
189 | filemap_nr_thps_dec(mapping); |
190 | } |
191 | |
192 | /* |
193 | * At this point folio must be either written or cleaned by |
194 | * truncate. Dirty folio here signals a bug and loss of |
195 | * unwritten data - on ordinary filesystems. |
196 | * |
197 | * But it's harmless on in-memory filesystems like tmpfs; and can |
198 | * occur when a driver which did get_user_pages() sets page dirty |
199 | * before putting it, while the inode is being finally evicted. |
200 | * |
201 | * Below fixes dirty accounting after removing the folio entirely |
202 | * but leaves the dirty flag set: it has no effect for truncated |
203 | * folio and anyway will be cleared before returning folio to |
204 | * buddy allocator. |
205 | */ |
206 | if (WARN_ON_ONCE(folio_test_dirty(folio) && |
207 | mapping_can_writeback(mapping))) |
208 | folio_account_cleaned(folio, wb: inode_to_wb(inode: mapping->host)); |
209 | } |
210 | |
211 | /* |
212 | * Delete a page from the page cache and free it. Caller has to make |
213 | * sure the page is locked and that nobody else uses it - or that usage |
214 | * is safe. The caller must hold the i_pages lock. |
215 | */ |
216 | void __filemap_remove_folio(struct folio *folio, void *shadow) |
217 | { |
218 | struct address_space *mapping = folio->mapping; |
219 | |
220 | trace_mm_filemap_delete_from_page_cache(folio); |
221 | filemap_unaccount_folio(mapping, folio); |
222 | page_cache_delete(mapping, folio, shadow); |
223 | } |
224 | |
225 | void filemap_free_folio(struct address_space *mapping, struct folio *folio) |
226 | { |
227 | void (*free_folio)(struct folio *); |
228 | int refs = 1; |
229 | |
230 | free_folio = mapping->a_ops->free_folio; |
231 | if (free_folio) |
232 | free_folio(folio); |
233 | |
234 | if (folio_test_large(folio)) |
235 | refs = folio_nr_pages(folio); |
236 | folio_put_refs(folio, refs); |
237 | } |
238 | |
239 | /** |
240 | * filemap_remove_folio - Remove folio from page cache. |
241 | * @folio: The folio. |
242 | * |
243 | * This must be called only on folios that are locked and have been |
244 | * verified to be in the page cache. It will never put the folio into |
245 | * the free list because the caller has a reference on the page. |
246 | */ |
247 | void filemap_remove_folio(struct folio *folio) |
248 | { |
249 | struct address_space *mapping = folio->mapping; |
250 | |
251 | BUG_ON(!folio_test_locked(folio)); |
252 | spin_lock(lock: &mapping->host->i_lock); |
253 | xa_lock_irq(&mapping->i_pages); |
254 | __filemap_remove_folio(folio, NULL); |
255 | xa_unlock_irq(&mapping->i_pages); |
256 | if (mapping_shrinkable(mapping)) |
257 | inode_add_lru(inode: mapping->host); |
258 | spin_unlock(lock: &mapping->host->i_lock); |
259 | |
260 | filemap_free_folio(mapping, folio); |
261 | } |
262 | |
263 | /* |
264 | * page_cache_delete_batch - delete several folios from page cache |
265 | * @mapping: the mapping to which folios belong |
266 | * @fbatch: batch of folios to delete |
267 | * |
268 | * The function walks over mapping->i_pages and removes folios passed in |
269 | * @fbatch from the mapping. The function expects @fbatch to be sorted |
270 | * by page index and is optimised for it to be dense. |
271 | * It tolerates holes in @fbatch (mapping entries at those indices are not |
272 | * modified). |
273 | * |
274 | * The function expects the i_pages lock to be held. |
275 | */ |
276 | static void page_cache_delete_batch(struct address_space *mapping, |
277 | struct folio_batch *fbatch) |
278 | { |
279 | XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index); |
280 | long total_pages = 0; |
281 | int i = 0; |
282 | struct folio *folio; |
283 | |
284 | mapping_set_update(&xas, mapping); |
285 | xas_for_each(&xas, folio, ULONG_MAX) { |
286 | if (i >= folio_batch_count(fbatch)) |
287 | break; |
288 | |
289 | /* A swap/dax/shadow entry got inserted? Skip it. */ |
290 | if (xa_is_value(entry: folio)) |
291 | continue; |
292 | /* |
293 | * A page got inserted in our range? Skip it. We have our |
294 | * pages locked so they are protected from being removed. |
295 | * If we see a page whose index is higher than ours, it |
296 | * means our page has been removed, which shouldn't be |
297 | * possible because we're holding the PageLock. |
298 | */ |
299 | if (folio != fbatch->folios[i]) { |
300 | VM_BUG_ON_FOLIO(folio->index > |
301 | fbatch->folios[i]->index, folio); |
302 | continue; |
303 | } |
304 | |
305 | WARN_ON_ONCE(!folio_test_locked(folio)); |
306 | |
307 | folio->mapping = NULL; |
308 | /* Leave folio->index set: truncation lookup relies on it */ |
309 | |
310 | i++; |
311 | xas_store(&xas, NULL); |
312 | total_pages += folio_nr_pages(folio); |
313 | } |
314 | mapping->nrpages -= total_pages; |
315 | } |
316 | |
317 | void delete_from_page_cache_batch(struct address_space *mapping, |
318 | struct folio_batch *fbatch) |
319 | { |
320 | int i; |
321 | |
322 | if (!folio_batch_count(fbatch)) |
323 | return; |
324 | |
325 | spin_lock(lock: &mapping->host->i_lock); |
326 | xa_lock_irq(&mapping->i_pages); |
327 | for (i = 0; i < folio_batch_count(fbatch); i++) { |
328 | struct folio *folio = fbatch->folios[i]; |
329 | |
330 | trace_mm_filemap_delete_from_page_cache(folio); |
331 | filemap_unaccount_folio(mapping, folio); |
332 | } |
333 | page_cache_delete_batch(mapping, fbatch); |
334 | xa_unlock_irq(&mapping->i_pages); |
335 | if (mapping_shrinkable(mapping)) |
336 | inode_add_lru(inode: mapping->host); |
337 | spin_unlock(lock: &mapping->host->i_lock); |
338 | |
339 | for (i = 0; i < folio_batch_count(fbatch); i++) |
340 | filemap_free_folio(mapping, folio: fbatch->folios[i]); |
341 | } |
342 | |
343 | int filemap_check_errors(struct address_space *mapping) |
344 | { |
345 | int ret = 0; |
346 | /* Check for outstanding write errors */ |
347 | if (test_bit(AS_ENOSPC, &mapping->flags) && |
348 | test_and_clear_bit(nr: AS_ENOSPC, addr: &mapping->flags)) |
349 | ret = -ENOSPC; |
350 | if (test_bit(AS_EIO, &mapping->flags) && |
351 | test_and_clear_bit(nr: AS_EIO, addr: &mapping->flags)) |
352 | ret = -EIO; |
353 | return ret; |
354 | } |
355 | EXPORT_SYMBOL(filemap_check_errors); |
356 | |
357 | static int filemap_check_and_keep_errors(struct address_space *mapping) |
358 | { |
359 | /* Check for outstanding write errors */ |
360 | if (test_bit(AS_EIO, &mapping->flags)) |
361 | return -EIO; |
362 | if (test_bit(AS_ENOSPC, &mapping->flags)) |
363 | return -ENOSPC; |
364 | return 0; |
365 | } |
366 | |
367 | /** |
368 | * filemap_fdatawrite_wbc - start writeback on mapping dirty pages in range |
369 | * @mapping: address space structure to write |
370 | * @wbc: the writeback_control controlling the writeout |
371 | * |
372 | * Call writepages on the mapping using the provided wbc to control the |
373 | * writeout. |
374 | * |
375 | * Return: %0 on success, negative error code otherwise. |
376 | */ |
377 | int filemap_fdatawrite_wbc(struct address_space *mapping, |
378 | struct writeback_control *wbc) |
379 | { |
380 | int ret; |
381 | |
382 | if (!mapping_can_writeback(mapping) || |
383 | !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) |
384 | return 0; |
385 | |
386 | wbc_attach_fdatawrite_inode(wbc, inode: mapping->host); |
387 | ret = do_writepages(mapping, wbc); |
388 | wbc_detach_inode(wbc); |
389 | return ret; |
390 | } |
391 | EXPORT_SYMBOL(filemap_fdatawrite_wbc); |
392 | |
393 | /** |
394 | * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range |
395 | * @mapping: address space structure to write |
396 | * @start: offset in bytes where the range starts |
397 | * @end: offset in bytes where the range ends (inclusive) |
398 | * @sync_mode: enable synchronous operation |
399 | * |
400 | * Start writeback against all of a mapping's dirty pages that lie |
401 | * within the byte offsets <start, end> inclusive. |
402 | * |
403 | * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as |
404 | * opposed to a regular memory cleansing writeback. The difference between |
405 | * these two operations is that if a dirty page/buffer is encountered, it must |
406 | * be waited upon, and not just skipped over. |
407 | * |
408 | * Return: %0 on success, negative error code otherwise. |
409 | */ |
410 | int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, |
411 | loff_t end, int sync_mode) |
412 | { |
413 | struct writeback_control wbc = { |
414 | .sync_mode = sync_mode, |
415 | .nr_to_write = LONG_MAX, |
416 | .range_start = start, |
417 | .range_end = end, |
418 | }; |
419 | |
420 | return filemap_fdatawrite_wbc(mapping, &wbc); |
421 | } |
422 | |
423 | static inline int __filemap_fdatawrite(struct address_space *mapping, |
424 | int sync_mode) |
425 | { |
426 | return __filemap_fdatawrite_range(mapping, start: 0, LLONG_MAX, sync_mode); |
427 | } |
428 | |
429 | int filemap_fdatawrite(struct address_space *mapping) |
430 | { |
431 | return __filemap_fdatawrite(mapping, sync_mode: WB_SYNC_ALL); |
432 | } |
433 | EXPORT_SYMBOL(filemap_fdatawrite); |
434 | |
435 | int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, |
436 | loff_t end) |
437 | { |
438 | return __filemap_fdatawrite_range(mapping, start, end, sync_mode: WB_SYNC_ALL); |
439 | } |
440 | EXPORT_SYMBOL(filemap_fdatawrite_range); |
441 | |
442 | /** |
443 | * filemap_flush - mostly a non-blocking flush |
444 | * @mapping: target address_space |
445 | * |
446 | * This is a mostly non-blocking flush. Not suitable for data-integrity |
447 | * purposes - I/O may not be started against all dirty pages. |
448 | * |
449 | * Return: %0 on success, negative error code otherwise. |
450 | */ |
451 | int filemap_flush(struct address_space *mapping) |
452 | { |
453 | return __filemap_fdatawrite(mapping, sync_mode: WB_SYNC_NONE); |
454 | } |
455 | EXPORT_SYMBOL(filemap_flush); |
456 | |
457 | /** |
458 | * filemap_range_has_page - check if a page exists in range. |
459 | * @mapping: address space within which to check |
460 | * @start_byte: offset in bytes where the range starts |
461 | * @end_byte: offset in bytes where the range ends (inclusive) |
462 | * |
463 | * Find at least one page in the range supplied, usually used to check if |
464 | * direct writing in this range will trigger a writeback. |
465 | * |
466 | * Return: %true if at least one page exists in the specified range, |
467 | * %false otherwise. |
468 | */ |
469 | bool filemap_range_has_page(struct address_space *mapping, |
470 | loff_t start_byte, loff_t end_byte) |
471 | { |
472 | struct folio *folio; |
473 | XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); |
474 | pgoff_t max = end_byte >> PAGE_SHIFT; |
475 | |
476 | if (end_byte < start_byte) |
477 | return false; |
478 | |
479 | rcu_read_lock(); |
480 | for (;;) { |
481 | folio = xas_find(&xas, max); |
482 | if (xas_retry(xas: &xas, entry: folio)) |
483 | continue; |
484 | /* Shadow entries don't count */ |
485 | if (xa_is_value(entry: folio)) |
486 | continue; |
487 | /* |
488 | * We don't need to try to pin this page; we're about to |
489 | * release the RCU lock anyway. It is enough to know that |
490 | * there was a page here recently. |
491 | */ |
492 | break; |
493 | } |
494 | rcu_read_unlock(); |
495 | |
496 | return folio != NULL; |
497 | } |
498 | EXPORT_SYMBOL(filemap_range_has_page); |
499 | |
500 | static void __filemap_fdatawait_range(struct address_space *mapping, |
501 | loff_t start_byte, loff_t end_byte) |
502 | { |
503 | pgoff_t index = start_byte >> PAGE_SHIFT; |
504 | pgoff_t end = end_byte >> PAGE_SHIFT; |
505 | struct folio_batch fbatch; |
506 | unsigned nr_folios; |
507 | |
508 | folio_batch_init(fbatch: &fbatch); |
509 | |
510 | while (index <= end) { |
511 | unsigned i; |
512 | |
513 | nr_folios = filemap_get_folios_tag(mapping, start: &index, end, |
514 | PAGECACHE_TAG_WRITEBACK, fbatch: &fbatch); |
515 | |
516 | if (!nr_folios) |
517 | break; |
518 | |
519 | for (i = 0; i < nr_folios; i++) { |
520 | struct folio *folio = fbatch.folios[i]; |
521 | |
522 | folio_wait_writeback(folio); |
523 | folio_clear_error(folio); |
524 | } |
525 | folio_batch_release(fbatch: &fbatch); |
526 | cond_resched(); |
527 | } |
528 | } |
529 | |
530 | /** |
531 | * filemap_fdatawait_range - wait for writeback to complete |
532 | * @mapping: address space structure to wait for |
533 | * @start_byte: offset in bytes where the range starts |
534 | * @end_byte: offset in bytes where the range ends (inclusive) |
535 | * |
536 | * Walk the list of under-writeback pages of the given address space |
537 | * in the given range and wait for all of them. Check error status of |
538 | * the address space and return it. |
539 | * |
540 | * Since the error status of the address space is cleared by this function, |
541 | * callers are responsible for checking the return value and handling and/or |
542 | * reporting the error. |
543 | * |
544 | * Return: error status of the address space. |
545 | */ |
546 | int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte, |
547 | loff_t end_byte) |
548 | { |
549 | __filemap_fdatawait_range(mapping, start_byte, end_byte); |
550 | return filemap_check_errors(mapping); |
551 | } |
552 | EXPORT_SYMBOL(filemap_fdatawait_range); |
553 | |
554 | /** |
555 | * filemap_fdatawait_range_keep_errors - wait for writeback to complete |
556 | * @mapping: address space structure to wait for |
557 | * @start_byte: offset in bytes where the range starts |
558 | * @end_byte: offset in bytes where the range ends (inclusive) |
559 | * |
560 | * Walk the list of under-writeback pages of the given address space in the |
561 | * given range and wait for all of them. Unlike filemap_fdatawait_range(), |
562 | * this function does not clear error status of the address space. |
563 | * |
564 | * Use this function if callers don't handle errors themselves. Expected |
565 | * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2), |
566 | * fsfreeze(8) |
567 | */ |
568 | int filemap_fdatawait_range_keep_errors(struct address_space *mapping, |
569 | loff_t start_byte, loff_t end_byte) |
570 | { |
571 | __filemap_fdatawait_range(mapping, start_byte, end_byte); |
572 | return filemap_check_and_keep_errors(mapping); |
573 | } |
574 | EXPORT_SYMBOL(filemap_fdatawait_range_keep_errors); |
575 | |
576 | /** |
577 | * file_fdatawait_range - wait for writeback to complete |
578 | * @file: file pointing to address space structure to wait for |
579 | * @start_byte: offset in bytes where the range starts |
580 | * @end_byte: offset in bytes where the range ends (inclusive) |
581 | * |
582 | * Walk the list of under-writeback pages of the address space that file |
583 | * refers to, in the given range and wait for all of them. Check error |
584 | * status of the address space vs. the file->f_wb_err cursor and return it. |
585 | * |
586 | * Since the error status of the file is advanced by this function, |
587 | * callers are responsible for checking the return value and handling and/or |
588 | * reporting the error. |
589 | * |
590 | * Return: error status of the address space vs. the file->f_wb_err cursor. |
591 | */ |
592 | int file_fdatawait_range(struct file *file, loff_t start_byte, loff_t end_byte) |
593 | { |
594 | struct address_space *mapping = file->f_mapping; |
595 | |
596 | __filemap_fdatawait_range(mapping, start_byte, end_byte); |
597 | return file_check_and_advance_wb_err(file); |
598 | } |
599 | EXPORT_SYMBOL(file_fdatawait_range); |
600 | |
601 | /** |
602 | * filemap_fdatawait_keep_errors - wait for writeback without clearing errors |
603 | * @mapping: address space structure to wait for |
604 | * |
605 | * Walk the list of under-writeback pages of the given address space |
606 | * and wait for all of them. Unlike filemap_fdatawait(), this function |
607 | * does not clear error status of the address space. |
608 | * |
609 | * Use this function if callers don't handle errors themselves. Expected |
610 | * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2), |
611 | * fsfreeze(8) |
612 | * |
613 | * Return: error status of the address space. |
614 | */ |
615 | int filemap_fdatawait_keep_errors(struct address_space *mapping) |
616 | { |
617 | __filemap_fdatawait_range(mapping, start_byte: 0, LLONG_MAX); |
618 | return filemap_check_and_keep_errors(mapping); |
619 | } |
620 | EXPORT_SYMBOL(filemap_fdatawait_keep_errors); |
621 | |
622 | /* Returns true if writeback might be needed or already in progress. */ |
623 | static bool mapping_needs_writeback(struct address_space *mapping) |
624 | { |
625 | return mapping->nrpages; |
626 | } |
627 | |
628 | bool filemap_range_has_writeback(struct address_space *mapping, |
629 | loff_t start_byte, loff_t end_byte) |
630 | { |
631 | XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); |
632 | pgoff_t max = end_byte >> PAGE_SHIFT; |
633 | struct folio *folio; |
634 | |
635 | if (end_byte < start_byte) |
636 | return false; |
637 | |
638 | rcu_read_lock(); |
639 | xas_for_each(&xas, folio, max) { |
640 | if (xas_retry(xas: &xas, entry: folio)) |
641 | continue; |
642 | if (xa_is_value(entry: folio)) |
643 | continue; |
644 | if (folio_test_dirty(folio) || folio_test_locked(folio) || |
645 | folio_test_writeback(folio)) |
646 | break; |
647 | } |
648 | rcu_read_unlock(); |
649 | return folio != NULL; |
650 | } |
651 | EXPORT_SYMBOL_GPL(filemap_range_has_writeback); |
652 | |
653 | /** |
654 | * filemap_write_and_wait_range - write out & wait on a file range |
655 | * @mapping: the address_space for the pages |
656 | * @lstart: offset in bytes where the range starts |
657 | * @lend: offset in bytes where the range ends (inclusive) |
658 | * |
659 | * Write out and wait upon file offsets lstart->lend, inclusive. |
660 | * |
661 | * Note that @lend is inclusive (describes the last byte to be written) so |
662 | * that this function can be used to write to the very end-of-file (end = -1). |
663 | * |
664 | * Return: error status of the address space. |
665 | */ |
666 | int filemap_write_and_wait_range(struct address_space *mapping, |
667 | loff_t lstart, loff_t lend) |
668 | { |
669 | int err = 0, err2; |
670 | |
671 | if (lend < lstart) |
672 | return 0; |
673 | |
674 | if (mapping_needs_writeback(mapping)) { |
675 | err = __filemap_fdatawrite_range(mapping, start: lstart, end: lend, |
676 | sync_mode: WB_SYNC_ALL); |
677 | /* |
678 | * Even if the above returned error, the pages may be |
679 | * written partially (e.g. -ENOSPC), so we wait for it. |
680 | * But the -EIO is special case, it may indicate the worst |
681 | * thing (e.g. bug) happened, so we avoid waiting for it. |
682 | */ |
683 | if (err != -EIO) |
684 | __filemap_fdatawait_range(mapping, start_byte: lstart, end_byte: lend); |
685 | } |
686 | err2 = filemap_check_errors(mapping); |
687 | if (!err) |
688 | err = err2; |
689 | return err; |
690 | } |
691 | EXPORT_SYMBOL(filemap_write_and_wait_range); |
692 | |
693 | void __filemap_set_wb_err(struct address_space *mapping, int err) |
694 | { |
695 | errseq_t eseq = errseq_set(eseq: &mapping->wb_err, err); |
696 | |
697 | trace_filemap_set_wb_err(mapping, eseq); |
698 | } |
699 | EXPORT_SYMBOL(__filemap_set_wb_err); |
700 | |
701 | /** |
702 | * file_check_and_advance_wb_err - report wb error (if any) that was previously |
703 | * and advance wb_err to current one |
704 | * @file: struct file on which the error is being reported |
705 | * |
706 | * When userland calls fsync (or something like nfsd does the equivalent), we |
707 | * want to report any writeback errors that occurred since the last fsync (or |
708 | * since the file was opened if there haven't been any). |
709 | * |
710 | * Grab the wb_err from the mapping. If it matches what we have in the file, |
711 | * then just quickly return 0. The file is all caught up. |
712 | * |
713 | * If it doesn't match, then take the mapping value, set the "seen" flag in |
714 | * it and try to swap it into place. If it works, or another task beat us |
715 | * to it with the new value, then update the f_wb_err and return the error |
716 | * portion. The error at this point must be reported via proper channels |
717 | * (a'la fsync, or NFS COMMIT operation, etc.). |
718 | * |
719 | * While we handle mapping->wb_err with atomic operations, the f_wb_err |
720 | * value is protected by the f_lock since we must ensure that it reflects |
721 | * the latest value swapped in for this file descriptor. |
722 | * |
723 | * Return: %0 on success, negative error code otherwise. |
724 | */ |
725 | int file_check_and_advance_wb_err(struct file *file) |
726 | { |
727 | int err = 0; |
728 | errseq_t old = READ_ONCE(file->f_wb_err); |
729 | struct address_space *mapping = file->f_mapping; |
730 | |
731 | /* Locklessly handle the common case where nothing has changed */ |
732 | if (errseq_check(eseq: &mapping->wb_err, since: old)) { |
733 | /* Something changed, must use slow path */ |
734 | spin_lock(lock: &file->f_lock); |
735 | old = file->f_wb_err; |
736 | err = errseq_check_and_advance(eseq: &mapping->wb_err, |
737 | since: &file->f_wb_err); |
738 | trace_file_check_and_advance_wb_err(file, old); |
739 | spin_unlock(lock: &file->f_lock); |
740 | } |
741 | |
742 | /* |
743 | * We're mostly using this function as a drop in replacement for |
744 | * filemap_check_errors. Clear AS_EIO/AS_ENOSPC to emulate the effect |
745 | * that the legacy code would have had on these flags. |
746 | */ |
747 | clear_bit(nr: AS_EIO, addr: &mapping->flags); |
748 | clear_bit(nr: AS_ENOSPC, addr: &mapping->flags); |
749 | return err; |
750 | } |
751 | EXPORT_SYMBOL(file_check_and_advance_wb_err); |
752 | |
753 | /** |
754 | * file_write_and_wait_range - write out & wait on a file range |
755 | * @file: file pointing to address_space with pages |
756 | * @lstart: offset in bytes where the range starts |
757 | * @lend: offset in bytes where the range ends (inclusive) |
758 | * |
759 | * Write out and wait upon file offsets lstart->lend, inclusive. |
760 | * |
761 | * Note that @lend is inclusive (describes the last byte to be written) so |
762 | * that this function can be used to write to the very end-of-file (end = -1). |
763 | * |
764 | * After writing out and waiting on the data, we check and advance the |
765 | * f_wb_err cursor to the latest value, and return any errors detected there. |
766 | * |
767 | * Return: %0 on success, negative error code otherwise. |
768 | */ |
769 | int file_write_and_wait_range(struct file *file, loff_t lstart, loff_t lend) |
770 | { |
771 | int err = 0, err2; |
772 | struct address_space *mapping = file->f_mapping; |
773 | |
774 | if (lend < lstart) |
775 | return 0; |
776 | |
777 | if (mapping_needs_writeback(mapping)) { |
778 | err = __filemap_fdatawrite_range(mapping, start: lstart, end: lend, |
779 | sync_mode: WB_SYNC_ALL); |
780 | /* See comment of filemap_write_and_wait() */ |
781 | if (err != -EIO) |
782 | __filemap_fdatawait_range(mapping, start_byte: lstart, end_byte: lend); |
783 | } |
784 | err2 = file_check_and_advance_wb_err(file); |
785 | if (!err) |
786 | err = err2; |
787 | return err; |
788 | } |
789 | EXPORT_SYMBOL(file_write_and_wait_range); |
790 | |
791 | /** |
792 | * replace_page_cache_folio - replace a pagecache folio with a new one |
793 | * @old: folio to be replaced |
794 | * @new: folio to replace with |
795 | * |
796 | * This function replaces a folio in the pagecache with a new one. On |
797 | * success it acquires the pagecache reference for the new folio and |
798 | * drops it for the old folio. Both the old and new folios must be |
799 | * locked. This function does not add the new folio to the LRU, the |
800 | * caller must do that. |
801 | * |
802 | * The remove + add is atomic. This function cannot fail. |
803 | */ |
804 | void replace_page_cache_folio(struct folio *old, struct folio *new) |
805 | { |
806 | struct address_space *mapping = old->mapping; |
807 | void (*free_folio)(struct folio *) = mapping->a_ops->free_folio; |
808 | pgoff_t offset = old->index; |
809 | XA_STATE(xas, &mapping->i_pages, offset); |
810 | |
811 | VM_BUG_ON_FOLIO(!folio_test_locked(old), old); |
812 | VM_BUG_ON_FOLIO(!folio_test_locked(new), new); |
813 | VM_BUG_ON_FOLIO(new->mapping, new); |
814 | |
815 | folio_get(folio: new); |
816 | new->mapping = mapping; |
817 | new->index = offset; |
818 | |
819 | mem_cgroup_replace_folio(old, new); |
820 | |
821 | xas_lock_irq(&xas); |
822 | xas_store(&xas, entry: new); |
823 | |
824 | old->mapping = NULL; |
825 | /* hugetlb pages do not participate in page cache accounting. */ |
826 | if (!folio_test_hugetlb(folio: old)) |
827 | __lruvec_stat_sub_folio(folio: old, idx: NR_FILE_PAGES); |
828 | if (!folio_test_hugetlb(folio: new)) |
829 | __lruvec_stat_add_folio(folio: new, idx: NR_FILE_PAGES); |
830 | if (folio_test_swapbacked(folio: old)) |
831 | __lruvec_stat_sub_folio(folio: old, idx: NR_SHMEM); |
832 | if (folio_test_swapbacked(folio: new)) |
833 | __lruvec_stat_add_folio(folio: new, idx: NR_SHMEM); |
834 | xas_unlock_irq(&xas); |
835 | if (free_folio) |
836 | free_folio(old); |
837 | folio_put(folio: old); |
838 | } |
839 | EXPORT_SYMBOL_GPL(replace_page_cache_folio); |
840 | |
841 | noinline int __filemap_add_folio(struct address_space *mapping, |
842 | struct folio *folio, pgoff_t index, gfp_t gfp, void **shadowp) |
843 | { |
844 | XA_STATE(xas, &mapping->i_pages, index); |
845 | int huge = folio_test_hugetlb(folio); |
846 | bool charged = false; |
847 | long nr = 1; |
848 | |
849 | VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); |
850 | VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio); |
851 | mapping_set_update(&xas, mapping); |
852 | |
853 | if (!huge) { |
854 | int error = mem_cgroup_charge(folio, NULL, gfp); |
855 | if (error) |
856 | return error; |
857 | charged = true; |
858 | } |
859 | |
860 | VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio); |
861 | xas_set_order(xas: &xas, index, order: folio_order(folio)); |
862 | nr = folio_nr_pages(folio); |
863 | |
864 | gfp &= GFP_RECLAIM_MASK; |
865 | folio_ref_add(folio, nr); |
866 | folio->mapping = mapping; |
867 | folio->index = xas.xa_index; |
868 | |
869 | do { |
870 | unsigned int order = xa_get_order(xas.xa, index: xas.xa_index); |
871 | void *entry, *old = NULL; |
872 | |
873 | if (order > folio_order(folio)) |
874 | xas_split_alloc(&xas, entry: xa_load(xas.xa, index: xas.xa_index), |
875 | order, gfp); |
876 | xas_lock_irq(&xas); |
877 | xas_for_each_conflict(&xas, entry) { |
878 | old = entry; |
879 | if (!xa_is_value(entry)) { |
880 | xas_set_err(xas: &xas, err: -EEXIST); |
881 | goto unlock; |
882 | } |
883 | } |
884 | |
885 | if (old) { |
886 | if (shadowp) |
887 | *shadowp = old; |
888 | /* entry may have been split before we acquired lock */ |
889 | order = xa_get_order(xas.xa, index: xas.xa_index); |
890 | if (order > folio_order(folio)) { |
891 | /* How to handle large swap entries? */ |
892 | BUG_ON(shmem_mapping(mapping)); |
893 | xas_split(&xas, entry: old, order); |
894 | xas_reset(xas: &xas); |
895 | } |
896 | } |
897 | |
898 | xas_store(&xas, entry: folio); |
899 | if (xas_error(xas: &xas)) |
900 | goto unlock; |
901 | |
902 | mapping->nrpages += nr; |
903 | |
904 | /* hugetlb pages do not participate in page cache accounting */ |
905 | if (!huge) { |
906 | __lruvec_stat_mod_folio(folio, idx: NR_FILE_PAGES, val: nr); |
907 | if (folio_test_pmd_mappable(folio)) |
908 | __lruvec_stat_mod_folio(folio, |
909 | idx: NR_FILE_THPS, val: nr); |
910 | } |
911 | unlock: |
912 | xas_unlock_irq(&xas); |
913 | } while (xas_nomem(&xas, gfp)); |
914 | |
915 | if (xas_error(xas: &xas)) |
916 | goto error; |
917 | |
918 | trace_mm_filemap_add_to_page_cache(folio); |
919 | return 0; |
920 | error: |
921 | if (charged) |
922 | mem_cgroup_uncharge(folio); |
923 | folio->mapping = NULL; |
924 | /* Leave page->index set: truncation relies upon it */ |
925 | folio_put_refs(folio, refs: nr); |
926 | return xas_error(xas: &xas); |
927 | } |
928 | ALLOW_ERROR_INJECTION(__filemap_add_folio, ERRNO); |
929 | |
930 | int filemap_add_folio(struct address_space *mapping, struct folio *folio, |
931 | pgoff_t index, gfp_t gfp) |
932 | { |
933 | void *shadow = NULL; |
934 | int ret; |
935 | |
936 | __folio_set_locked(folio); |
937 | ret = __filemap_add_folio(mapping, folio, index, gfp, shadowp: &shadow); |
938 | if (unlikely(ret)) |
939 | __folio_clear_locked(folio); |
940 | else { |
941 | /* |
942 | * The folio might have been evicted from cache only |
943 | * recently, in which case it should be activated like |
944 | * any other repeatedly accessed folio. |
945 | * The exception is folios getting rewritten; evicting other |
946 | * data from the working set, only to cache data that will |
947 | * get overwritten with something else, is a waste of memory. |
948 | */ |
949 | WARN_ON_ONCE(folio_test_active(folio)); |
950 | if (!(gfp & __GFP_WRITE) && shadow) |
951 | workingset_refault(folio, shadow); |
952 | folio_add_lru(folio); |
953 | } |
954 | return ret; |
955 | } |
956 | EXPORT_SYMBOL_GPL(filemap_add_folio); |
957 | |
958 | #ifdef CONFIG_NUMA |
959 | struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order) |
960 | { |
961 | int n; |
962 | struct folio *folio; |
963 | |
964 | if (cpuset_do_page_mem_spread()) { |
965 | unsigned int cpuset_mems_cookie; |
966 | do { |
967 | cpuset_mems_cookie = read_mems_allowed_begin(); |
968 | n = cpuset_mem_spread_node(); |
969 | folio = __folio_alloc_node(gfp, order, nid: n); |
970 | } while (!folio && read_mems_allowed_retry(seq: cpuset_mems_cookie)); |
971 | |
972 | return folio; |
973 | } |
974 | return folio_alloc(gfp, order); |
975 | } |
976 | EXPORT_SYMBOL(filemap_alloc_folio); |
977 | #endif |
978 | |
979 | /* |
980 | * filemap_invalidate_lock_two - lock invalidate_lock for two mappings |
981 | * |
982 | * Lock exclusively invalidate_lock of any passed mapping that is not NULL. |
983 | * |
984 | * @mapping1: the first mapping to lock |
985 | * @mapping2: the second mapping to lock |
986 | */ |
987 | void filemap_invalidate_lock_two(struct address_space *mapping1, |
988 | struct address_space *mapping2) |
989 | { |
990 | if (mapping1 > mapping2) |
991 | swap(mapping1, mapping2); |
992 | if (mapping1) |
993 | down_write(sem: &mapping1->invalidate_lock); |
994 | if (mapping2 && mapping1 != mapping2) |
995 | down_write_nested(sem: &mapping2->invalidate_lock, subclass: 1); |
996 | } |
997 | EXPORT_SYMBOL(filemap_invalidate_lock_two); |
998 | |
999 | /* |
1000 | * filemap_invalidate_unlock_two - unlock invalidate_lock for two mappings |
1001 | * |
1002 | * Unlock exclusive invalidate_lock of any passed mapping that is not NULL. |
1003 | * |
1004 | * @mapping1: the first mapping to unlock |
1005 | * @mapping2: the second mapping to unlock |
1006 | */ |
1007 | void filemap_invalidate_unlock_two(struct address_space *mapping1, |
1008 | struct address_space *mapping2) |
1009 | { |
1010 | if (mapping1) |
1011 | up_write(sem: &mapping1->invalidate_lock); |
1012 | if (mapping2 && mapping1 != mapping2) |
1013 | up_write(sem: &mapping2->invalidate_lock); |
1014 | } |
1015 | EXPORT_SYMBOL(filemap_invalidate_unlock_two); |
1016 | |
1017 | /* |
1018 | * In order to wait for pages to become available there must be |
1019 | * waitqueues associated with pages. By using a hash table of |
1020 | * waitqueues where the bucket discipline is to maintain all |
1021 | * waiters on the same queue and wake all when any of the pages |
1022 | * become available, and for the woken contexts to check to be |
1023 | * sure the appropriate page became available, this saves space |
1024 | * at a cost of "thundering herd" phenomena during rare hash |
1025 | * collisions. |
1026 | */ |
1027 | #define PAGE_WAIT_TABLE_BITS 8 |
1028 | #define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS) |
1029 | static wait_queue_head_t folio_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned; |
1030 | |
1031 | static wait_queue_head_t *folio_waitqueue(struct folio *folio) |
1032 | { |
1033 | return &folio_wait_table[hash_ptr(ptr: folio, PAGE_WAIT_TABLE_BITS)]; |
1034 | } |
1035 | |
1036 | void __init pagecache_init(void) |
1037 | { |
1038 | int i; |
1039 | |
1040 | for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++) |
1041 | init_waitqueue_head(&folio_wait_table[i]); |
1042 | |
1043 | page_writeback_init(); |
1044 | } |
1045 | |
1046 | /* |
1047 | * The page wait code treats the "wait->flags" somewhat unusually, because |
1048 | * we have multiple different kinds of waits, not just the usual "exclusive" |
1049 | * one. |
1050 | * |
1051 | * We have: |
1052 | * |
1053 | * (a) no special bits set: |
1054 | * |
1055 | * We're just waiting for the bit to be released, and when a waker |
1056 | * calls the wakeup function, we set WQ_FLAG_WOKEN and wake it up, |
1057 | * and remove it from the wait queue. |
1058 | * |
1059 | * Simple and straightforward. |
1060 | * |
1061 | * (b) WQ_FLAG_EXCLUSIVE: |
1062 | * |
1063 | * The waiter is waiting to get the lock, and only one waiter should |
1064 | * be woken up to avoid any thundering herd behavior. We'll set the |
1065 | * WQ_FLAG_WOKEN bit, wake it up, and remove it from the wait queue. |
1066 | * |
1067 | * This is the traditional exclusive wait. |
1068 | * |
1069 | * (c) WQ_FLAG_EXCLUSIVE | WQ_FLAG_CUSTOM: |
1070 | * |
1071 | * The waiter is waiting to get the bit, and additionally wants the |
1072 | * lock to be transferred to it for fair lock behavior. If the lock |
1073 | * cannot be taken, we stop walking the wait queue without waking |
1074 | * the waiter. |
1075 | * |
1076 | * This is the "fair lock handoff" case, and in addition to setting |
1077 | * WQ_FLAG_WOKEN, we set WQ_FLAG_DONE to let the waiter easily see |
1078 | * that it now has the lock. |
1079 | */ |
1080 | static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg) |
1081 | { |
1082 | unsigned int flags; |
1083 | struct wait_page_key *key = arg; |
1084 | struct wait_page_queue *wait_page |
1085 | = container_of(wait, struct wait_page_queue, wait); |
1086 | |
1087 | if (!wake_page_match(wait_page, key)) |
1088 | return 0; |
1089 | |
1090 | /* |
1091 | * If it's a lock handoff wait, we get the bit for it, and |
1092 | * stop walking (and do not wake it up) if we can't. |
1093 | */ |
1094 | flags = wait->flags; |
1095 | if (flags & WQ_FLAG_EXCLUSIVE) { |
1096 | if (test_bit(key->bit_nr, &key->folio->flags)) |
1097 | return -1; |
1098 | if (flags & WQ_FLAG_CUSTOM) { |
1099 | if (test_and_set_bit(nr: key->bit_nr, addr: &key->folio->flags)) |
1100 | return -1; |
1101 | flags |= WQ_FLAG_DONE; |
1102 | } |
1103 | } |
1104 | |
1105 | /* |
1106 | * We are holding the wait-queue lock, but the waiter that |
1107 | * is waiting for this will be checking the flags without |
1108 | * any locking. |
1109 | * |
1110 | * So update the flags atomically, and wake up the waiter |
1111 | * afterwards to avoid any races. This store-release pairs |
1112 | * with the load-acquire in folio_wait_bit_common(). |
1113 | */ |
1114 | smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN); |
1115 | wake_up_state(tsk: wait->private, state: mode); |
1116 | |
1117 | /* |
1118 | * Ok, we have successfully done what we're waiting for, |
1119 | * and we can unconditionally remove the wait entry. |
1120 | * |
1121 | * Note that this pairs with the "finish_wait()" in the |
1122 | * waiter, and has to be the absolute last thing we do. |
1123 | * After this list_del_init(&wait->entry) the wait entry |
1124 | * might be de-allocated and the process might even have |
1125 | * exited. |
1126 | */ |
1127 | list_del_init_careful(entry: &wait->entry); |
1128 | return (flags & WQ_FLAG_EXCLUSIVE) != 0; |
1129 | } |
1130 | |
1131 | static void folio_wake_bit(struct folio *folio, int bit_nr) |
1132 | { |
1133 | wait_queue_head_t *q = folio_waitqueue(folio); |
1134 | struct wait_page_key key; |
1135 | unsigned long flags; |
1136 | |
1137 | key.folio = folio; |
1138 | key.bit_nr = bit_nr; |
1139 | key.page_match = 0; |
1140 | |
1141 | spin_lock_irqsave(&q->lock, flags); |
1142 | __wake_up_locked_key(wq_head: q, TASK_NORMAL, key: &key); |
1143 | |
1144 | /* |
1145 | * It's possible to miss clearing waiters here, when we woke our page |
1146 | * waiters, but the hashed waitqueue has waiters for other pages on it. |
1147 | * That's okay, it's a rare case. The next waker will clear it. |
1148 | * |
1149 | * Note that, depending on the page pool (buddy, hugetlb, ZONE_DEVICE, |
1150 | * other), the flag may be cleared in the course of freeing the page; |
1151 | * but that is not required for correctness. |
1152 | */ |
1153 | if (!waitqueue_active(wq_head: q) || !key.page_match) |
1154 | folio_clear_waiters(folio); |
1155 | |
1156 | spin_unlock_irqrestore(lock: &q->lock, flags); |
1157 | } |
1158 | |
1159 | /* |
1160 | * A choice of three behaviors for folio_wait_bit_common(): |
1161 | */ |
1162 | enum behavior { |
1163 | EXCLUSIVE, /* Hold ref to page and take the bit when woken, like |
1164 | * __folio_lock() waiting on then setting PG_locked. |
1165 | */ |
1166 | SHARED, /* Hold ref to page and check the bit when woken, like |
1167 | * folio_wait_writeback() waiting on PG_writeback. |
1168 | */ |
1169 | DROP, /* Drop ref to page before wait, no check when woken, |
1170 | * like folio_put_wait_locked() on PG_locked. |
1171 | */ |
1172 | }; |
1173 | |
1174 | /* |
1175 | * Attempt to check (or get) the folio flag, and mark us done |
1176 | * if successful. |
1177 | */ |
1178 | static inline bool folio_trylock_flag(struct folio *folio, int bit_nr, |
1179 | struct wait_queue_entry *wait) |
1180 | { |
1181 | if (wait->flags & WQ_FLAG_EXCLUSIVE) { |
1182 | if (test_and_set_bit(nr: bit_nr, addr: &folio->flags)) |
1183 | return false; |
1184 | } else if (test_bit(bit_nr, &folio->flags)) |
1185 | return false; |
1186 | |
1187 | wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE; |
1188 | return true; |
1189 | } |
1190 | |
1191 | /* How many times do we accept lock stealing from under a waiter? */ |
1192 | int sysctl_page_lock_unfairness = 5; |
1193 | |
1194 | static inline int folio_wait_bit_common(struct folio *folio, int bit_nr, |
1195 | int state, enum behavior behavior) |
1196 | { |
1197 | wait_queue_head_t *q = folio_waitqueue(folio); |
1198 | int unfairness = sysctl_page_lock_unfairness; |
1199 | struct wait_page_queue wait_page; |
1200 | wait_queue_entry_t *wait = &wait_page.wait; |
1201 | bool thrashing = false; |
1202 | unsigned long pflags; |
1203 | bool in_thrashing; |
1204 | |
1205 | if (bit_nr == PG_locked && |
1206 | !folio_test_uptodate(folio) && folio_test_workingset(folio)) { |
1207 | delayacct_thrashing_start(in_thrashing: &in_thrashing); |
1208 | psi_memstall_enter(flags: &pflags); |
1209 | thrashing = true; |
1210 | } |
1211 | |
1212 | init_wait(wait); |
1213 | wait->func = wake_page_function; |
1214 | wait_page.folio = folio; |
1215 | wait_page.bit_nr = bit_nr; |
1216 | |
1217 | repeat: |
1218 | wait->flags = 0; |
1219 | if (behavior == EXCLUSIVE) { |
1220 | wait->flags = WQ_FLAG_EXCLUSIVE; |
1221 | if (--unfairness < 0) |
1222 | wait->flags |= WQ_FLAG_CUSTOM; |
1223 | } |
1224 | |
1225 | /* |
1226 | * Do one last check whether we can get the |
1227 | * page bit synchronously. |
1228 | * |
1229 | * Do the folio_set_waiters() marking before that |
1230 | * to let any waker we _just_ missed know they |
1231 | * need to wake us up (otherwise they'll never |
1232 | * even go to the slow case that looks at the |
1233 | * page queue), and add ourselves to the wait |
1234 | * queue if we need to sleep. |
1235 | * |
1236 | * This part needs to be done under the queue |
1237 | * lock to avoid races. |
1238 | */ |
1239 | spin_lock_irq(lock: &q->lock); |
1240 | folio_set_waiters(folio); |
1241 | if (!folio_trylock_flag(folio, bit_nr, wait)) |
1242 | __add_wait_queue_entry_tail(wq_head: q, wq_entry: wait); |
1243 | spin_unlock_irq(lock: &q->lock); |
1244 | |
1245 | /* |
1246 | * From now on, all the logic will be based on |
1247 | * the WQ_FLAG_WOKEN and WQ_FLAG_DONE flag, to |
1248 | * see whether the page bit testing has already |
1249 | * been done by the wake function. |
1250 | * |
1251 | * We can drop our reference to the folio. |
1252 | */ |
1253 | if (behavior == DROP) |
1254 | folio_put(folio); |
1255 | |
1256 | /* |
1257 | * Note that until the "finish_wait()", or until |
1258 | * we see the WQ_FLAG_WOKEN flag, we need to |
1259 | * be very careful with the 'wait->flags', because |
1260 | * we may race with a waker that sets them. |
1261 | */ |
1262 | for (;;) { |
1263 | unsigned int flags; |
1264 | |
1265 | set_current_state(state); |
1266 | |
1267 | /* Loop until we've been woken or interrupted */ |
1268 | flags = smp_load_acquire(&wait->flags); |
1269 | if (!(flags & WQ_FLAG_WOKEN)) { |
1270 | if (signal_pending_state(state, current)) |
1271 | break; |
1272 | |
1273 | io_schedule(); |
1274 | continue; |
1275 | } |
1276 | |
1277 | /* If we were non-exclusive, we're done */ |
1278 | if (behavior != EXCLUSIVE) |
1279 | break; |
1280 | |
1281 | /* If the waker got the lock for us, we're done */ |
1282 | if (flags & WQ_FLAG_DONE) |
1283 | break; |
1284 | |
1285 | /* |
1286 | * Otherwise, if we're getting the lock, we need to |
1287 | * try to get it ourselves. |
1288 | * |
1289 | * And if that fails, we'll have to retry this all. |
1290 | */ |
1291 | if (unlikely(test_and_set_bit(bit_nr, folio_flags(folio, 0)))) |
1292 | goto repeat; |
1293 | |
1294 | wait->flags |= WQ_FLAG_DONE; |
1295 | break; |
1296 | } |
1297 | |
1298 | /* |
1299 | * If a signal happened, this 'finish_wait()' may remove the last |
1300 | * waiter from the wait-queues, but the folio waiters bit will remain |
1301 | * set. That's ok. The next wakeup will take care of it, and trying |
1302 | * to do it here would be difficult and prone to races. |
1303 | */ |
1304 | finish_wait(wq_head: q, wq_entry: wait); |
1305 | |
1306 | if (thrashing) { |
1307 | delayacct_thrashing_end(in_thrashing: &in_thrashing); |
1308 | psi_memstall_leave(flags: &pflags); |
1309 | } |
1310 | |
1311 | /* |
1312 | * NOTE! The wait->flags weren't stable until we've done the |
1313 | * 'finish_wait()', and we could have exited the loop above due |
1314 | * to a signal, and had a wakeup event happen after the signal |
1315 | * test but before the 'finish_wait()'. |
1316 | * |
1317 | * So only after the finish_wait() can we reliably determine |
1318 | * if we got woken up or not, so we can now figure out the final |
1319 | * return value based on that state without races. |
1320 | * |
1321 | * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive |
1322 | * waiter, but an exclusive one requires WQ_FLAG_DONE. |
1323 | */ |
1324 | if (behavior == EXCLUSIVE) |
1325 | return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR; |
1326 | |
1327 | return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR; |
1328 | } |
1329 | |
1330 | #ifdef CONFIG_MIGRATION |
1331 | /** |
1332 | * migration_entry_wait_on_locked - Wait for a migration entry to be removed |
1333 | * @entry: migration swap entry. |
1334 | * @ptl: already locked ptl. This function will drop the lock. |
1335 | * |
1336 | * Wait for a migration entry referencing the given page to be removed. This is |
1337 | * equivalent to put_and_wait_on_page_locked(page, TASK_UNINTERRUPTIBLE) except |
1338 | * this can be called without taking a reference on the page. Instead this |
1339 | * should be called while holding the ptl for the migration entry referencing |
1340 | * the page. |
1341 | * |
1342 | * Returns after unlocking the ptl. |
1343 | * |
1344 | * This follows the same logic as folio_wait_bit_common() so see the comments |
1345 | * there. |
1346 | */ |
1347 | void migration_entry_wait_on_locked(swp_entry_t entry, spinlock_t *ptl) |
1348 | __releases(ptl) |
1349 | { |
1350 | struct wait_page_queue wait_page; |
1351 | wait_queue_entry_t *wait = &wait_page.wait; |
1352 | bool thrashing = false; |
1353 | unsigned long pflags; |
1354 | bool in_thrashing; |
1355 | wait_queue_head_t *q; |
1356 | struct folio *folio = page_folio(pfn_swap_entry_to_page(entry)); |
1357 | |
1358 | q = folio_waitqueue(folio); |
1359 | if (!folio_test_uptodate(folio) && folio_test_workingset(folio)) { |
1360 | delayacct_thrashing_start(in_thrashing: &in_thrashing); |
1361 | psi_memstall_enter(flags: &pflags); |
1362 | thrashing = true; |
1363 | } |
1364 | |
1365 | init_wait(wait); |
1366 | wait->func = wake_page_function; |
1367 | wait_page.folio = folio; |
1368 | wait_page.bit_nr = PG_locked; |
1369 | wait->flags = 0; |
1370 | |
1371 | spin_lock_irq(lock: &q->lock); |
1372 | folio_set_waiters(folio); |
1373 | if (!folio_trylock_flag(folio, bit_nr: PG_locked, wait)) |
1374 | __add_wait_queue_entry_tail(wq_head: q, wq_entry: wait); |
1375 | spin_unlock_irq(lock: &q->lock); |
1376 | |
1377 | /* |
1378 | * If a migration entry exists for the page the migration path must hold |
1379 | * a valid reference to the page, and it must take the ptl to remove the |
1380 | * migration entry. So the page is valid until the ptl is dropped. |
1381 | */ |
1382 | spin_unlock(lock: ptl); |
1383 | |
1384 | for (;;) { |
1385 | unsigned int flags; |
1386 | |
1387 | set_current_state(TASK_UNINTERRUPTIBLE); |
1388 | |
1389 | /* Loop until we've been woken or interrupted */ |
1390 | flags = smp_load_acquire(&wait->flags); |
1391 | if (!(flags & WQ_FLAG_WOKEN)) { |
1392 | if (signal_pending_state(TASK_UNINTERRUPTIBLE, current)) |
1393 | break; |
1394 | |
1395 | io_schedule(); |
1396 | continue; |
1397 | } |
1398 | break; |
1399 | } |
1400 | |
1401 | finish_wait(wq_head: q, wq_entry: wait); |
1402 | |
1403 | if (thrashing) { |
1404 | delayacct_thrashing_end(in_thrashing: &in_thrashing); |
1405 | psi_memstall_leave(flags: &pflags); |
1406 | } |
1407 | } |
1408 | #endif |
1409 | |
1410 | void folio_wait_bit(struct folio *folio, int bit_nr) |
1411 | { |
1412 | folio_wait_bit_common(folio, bit_nr, TASK_UNINTERRUPTIBLE, behavior: SHARED); |
1413 | } |
1414 | EXPORT_SYMBOL(folio_wait_bit); |
1415 | |
1416 | int folio_wait_bit_killable(struct folio *folio, int bit_nr) |
1417 | { |
1418 | return folio_wait_bit_common(folio, bit_nr, TASK_KILLABLE, behavior: SHARED); |
1419 | } |
1420 | EXPORT_SYMBOL(folio_wait_bit_killable); |
1421 | |
1422 | /** |
1423 | * folio_put_wait_locked - Drop a reference and wait for it to be unlocked |
1424 | * @folio: The folio to wait for. |
1425 | * @state: The sleep state (TASK_KILLABLE, TASK_UNINTERRUPTIBLE, etc). |
1426 | * |
1427 | * The caller should hold a reference on @folio. They expect the page to |
1428 | * become unlocked relatively soon, but do not wish to hold up migration |
1429 | * (for example) by holding the reference while waiting for the folio to |
1430 | * come unlocked. After this function returns, the caller should not |
1431 | * dereference @folio. |
1432 | * |
1433 | * Return: 0 if the folio was unlocked or -EINTR if interrupted by a signal. |
1434 | */ |
1435 | static int folio_put_wait_locked(struct folio *folio, int state) |
1436 | { |
1437 | return folio_wait_bit_common(folio, bit_nr: PG_locked, state, behavior: DROP); |
1438 | } |
1439 | |
1440 | /** |
1441 | * folio_add_wait_queue - Add an arbitrary waiter to a folio's wait queue |
1442 | * @folio: Folio defining the wait queue of interest |
1443 | * @waiter: Waiter to add to the queue |
1444 | * |
1445 | * Add an arbitrary @waiter to the wait queue for the nominated @folio. |
1446 | */ |
1447 | void folio_add_wait_queue(struct folio *folio, wait_queue_entry_t *waiter) |
1448 | { |
1449 | wait_queue_head_t *q = folio_waitqueue(folio); |
1450 | unsigned long flags; |
1451 | |
1452 | spin_lock_irqsave(&q->lock, flags); |
1453 | __add_wait_queue_entry_tail(wq_head: q, wq_entry: waiter); |
1454 | folio_set_waiters(folio); |
1455 | spin_unlock_irqrestore(lock: &q->lock, flags); |
1456 | } |
1457 | EXPORT_SYMBOL_GPL(folio_add_wait_queue); |
1458 | |
1459 | /** |
1460 | * folio_unlock - Unlock a locked folio. |
1461 | * @folio: The folio. |
1462 | * |
1463 | * Unlocks the folio and wakes up any thread sleeping on the page lock. |
1464 | * |
1465 | * Context: May be called from interrupt or process context. May not be |
1466 | * called from NMI context. |
1467 | */ |
1468 | void folio_unlock(struct folio *folio) |
1469 | { |
1470 | /* Bit 7 allows x86 to check the byte's sign bit */ |
1471 | BUILD_BUG_ON(PG_waiters != 7); |
1472 | BUILD_BUG_ON(PG_locked > 7); |
1473 | VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); |
1474 | if (folio_xor_flags_has_waiters(folio, mask: 1 << PG_locked)) |
1475 | folio_wake_bit(folio, bit_nr: PG_locked); |
1476 | } |
1477 | EXPORT_SYMBOL(folio_unlock); |
1478 | |
1479 | /** |
1480 | * folio_end_read - End read on a folio. |
1481 | * @folio: The folio. |
1482 | * @success: True if all reads completed successfully. |
1483 | * |
1484 | * When all reads against a folio have completed, filesystems should |
1485 | * call this function to let the pagecache know that no more reads |
1486 | * are outstanding. This will unlock the folio and wake up any thread |
1487 | * sleeping on the lock. The folio will also be marked uptodate if all |
1488 | * reads succeeded. |
1489 | * |
1490 | * Context: May be called from interrupt or process context. May not be |
1491 | * called from NMI context. |
1492 | */ |
1493 | void folio_end_read(struct folio *folio, bool success) |
1494 | { |
1495 | unsigned long mask = 1 << PG_locked; |
1496 | |
1497 | /* Must be in bottom byte for x86 to work */ |
1498 | BUILD_BUG_ON(PG_uptodate > 7); |
1499 | VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); |
1500 | VM_BUG_ON_FOLIO(folio_test_uptodate(folio), folio); |
1501 | |
1502 | if (likely(success)) |
1503 | mask |= 1 << PG_uptodate; |
1504 | if (folio_xor_flags_has_waiters(folio, mask)) |
1505 | folio_wake_bit(folio, bit_nr: PG_locked); |
1506 | } |
1507 | EXPORT_SYMBOL(folio_end_read); |
1508 | |
1509 | /** |
1510 | * folio_end_private_2 - Clear PG_private_2 and wake any waiters. |
1511 | * @folio: The folio. |
1512 | * |
1513 | * Clear the PG_private_2 bit on a folio and wake up any sleepers waiting for |
1514 | * it. The folio reference held for PG_private_2 being set is released. |
1515 | * |
1516 | * This is, for example, used when a netfs folio is being written to a local |
1517 | * disk cache, thereby allowing writes to the cache for the same folio to be |
1518 | * serialised. |
1519 | */ |
1520 | void folio_end_private_2(struct folio *folio) |
1521 | { |
1522 | VM_BUG_ON_FOLIO(!folio_test_private_2(folio), folio); |
1523 | clear_bit_unlock(nr: PG_private_2, addr: folio_flags(folio, n: 0)); |
1524 | folio_wake_bit(folio, bit_nr: PG_private_2); |
1525 | folio_put(folio); |
1526 | } |
1527 | EXPORT_SYMBOL(folio_end_private_2); |
1528 | |
1529 | /** |
1530 | * folio_wait_private_2 - Wait for PG_private_2 to be cleared on a folio. |
1531 | * @folio: The folio to wait on. |
1532 | * |
1533 | * Wait for PG_private_2 (aka PG_fscache) to be cleared on a folio. |
1534 | */ |
1535 | void folio_wait_private_2(struct folio *folio) |
1536 | { |
1537 | while (folio_test_private_2(folio)) |
1538 | folio_wait_bit(folio, PG_private_2); |
1539 | } |
1540 | EXPORT_SYMBOL(folio_wait_private_2); |
1541 | |
1542 | /** |
1543 | * folio_wait_private_2_killable - Wait for PG_private_2 to be cleared on a folio. |
1544 | * @folio: The folio to wait on. |
1545 | * |
1546 | * Wait for PG_private_2 (aka PG_fscache) to be cleared on a folio or until a |
1547 | * fatal signal is received by the calling task. |
1548 | * |
1549 | * Return: |
1550 | * - 0 if successful. |
1551 | * - -EINTR if a fatal signal was encountered. |
1552 | */ |
1553 | int folio_wait_private_2_killable(struct folio *folio) |
1554 | { |
1555 | int ret = 0; |
1556 | |
1557 | while (folio_test_private_2(folio)) { |
1558 | ret = folio_wait_bit_killable(folio, PG_private_2); |
1559 | if (ret < 0) |
1560 | break; |
1561 | } |
1562 | |
1563 | return ret; |
1564 | } |
1565 | EXPORT_SYMBOL(folio_wait_private_2_killable); |
1566 | |
1567 | /** |
1568 | * folio_end_writeback - End writeback against a folio. |
1569 | * @folio: The folio. |
1570 | * |
1571 | * The folio must actually be under writeback. |
1572 | * |
1573 | * Context: May be called from process or interrupt context. |
1574 | */ |
1575 | void folio_end_writeback(struct folio *folio) |
1576 | { |
1577 | VM_BUG_ON_FOLIO(!folio_test_writeback(folio), folio); |
1578 | |
1579 | /* |
1580 | * folio_test_clear_reclaim() could be used here but it is an |
1581 | * atomic operation and overkill in this particular case. Failing |
1582 | * to shuffle a folio marked for immediate reclaim is too mild |
1583 | * a gain to justify taking an atomic operation penalty at the |
1584 | * end of every folio writeback. |
1585 | */ |
1586 | if (folio_test_reclaim(folio)) { |
1587 | folio_clear_reclaim(folio); |
1588 | folio_rotate_reclaimable(folio); |
1589 | } |
1590 | |
1591 | /* |
1592 | * Writeback does not hold a folio reference of its own, relying |
1593 | * on truncation to wait for the clearing of PG_writeback. |
1594 | * But here we must make sure that the folio is not freed and |
1595 | * reused before the folio_wake_bit(). |
1596 | */ |
1597 | folio_get(folio); |
1598 | if (__folio_end_writeback(folio)) |
1599 | folio_wake_bit(folio, bit_nr: PG_writeback); |
1600 | acct_reclaim_writeback(folio); |
1601 | folio_put(folio); |
1602 | } |
1603 | EXPORT_SYMBOL(folio_end_writeback); |
1604 | |
1605 | /** |
1606 | * __folio_lock - Get a lock on the folio, assuming we need to sleep to get it. |
1607 | * @folio: The folio to lock |
1608 | */ |
1609 | void __folio_lock(struct folio *folio) |
1610 | { |
1611 | folio_wait_bit_common(folio, bit_nr: PG_locked, TASK_UNINTERRUPTIBLE, |
1612 | behavior: EXCLUSIVE); |
1613 | } |
1614 | EXPORT_SYMBOL(__folio_lock); |
1615 | |
1616 | int __folio_lock_killable(struct folio *folio) |
1617 | { |
1618 | return folio_wait_bit_common(folio, bit_nr: PG_locked, TASK_KILLABLE, |
1619 | behavior: EXCLUSIVE); |
1620 | } |
1621 | EXPORT_SYMBOL_GPL(__folio_lock_killable); |
1622 | |
1623 | static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait) |
1624 | { |
1625 | struct wait_queue_head *q = folio_waitqueue(folio); |
1626 | int ret = 0; |
1627 | |
1628 | wait->folio = folio; |
1629 | wait->bit_nr = PG_locked; |
1630 | |
1631 | spin_lock_irq(lock: &q->lock); |
1632 | __add_wait_queue_entry_tail(wq_head: q, wq_entry: &wait->wait); |
1633 | folio_set_waiters(folio); |
1634 | ret = !folio_trylock(folio); |
1635 | /* |
1636 | * If we were successful now, we know we're still on the |
1637 | * waitqueue as we're still under the lock. This means it's |
1638 | * safe to remove and return success, we know the callback |
1639 | * isn't going to trigger. |
1640 | */ |
1641 | if (!ret) |
1642 | __remove_wait_queue(wq_head: q, wq_entry: &wait->wait); |
1643 | else |
1644 | ret = -EIOCBQUEUED; |
1645 | spin_unlock_irq(lock: &q->lock); |
1646 | return ret; |
1647 | } |
1648 | |
1649 | /* |
1650 | * Return values: |
1651 | * 0 - folio is locked. |
1652 | * non-zero - folio is not locked. |
1653 | * mmap_lock or per-VMA lock has been released (mmap_read_unlock() or |
1654 | * vma_end_read()), unless flags had both FAULT_FLAG_ALLOW_RETRY and |
1655 | * FAULT_FLAG_RETRY_NOWAIT set, in which case the lock is still held. |
1656 | * |
1657 | * If neither ALLOW_RETRY nor KILLABLE are set, will always return 0 |
1658 | * with the folio locked and the mmap_lock/per-VMA lock is left unperturbed. |
1659 | */ |
1660 | vm_fault_t __folio_lock_or_retry(struct folio *folio, struct vm_fault *vmf) |
1661 | { |
1662 | unsigned int flags = vmf->flags; |
1663 | |
1664 | if (fault_flag_allow_retry_first(flags)) { |
1665 | /* |
1666 | * CAUTION! In this case, mmap_lock/per-VMA lock is not |
1667 | * released even though returning VM_FAULT_RETRY. |
1668 | */ |
1669 | if (flags & FAULT_FLAG_RETRY_NOWAIT) |
1670 | return VM_FAULT_RETRY; |
1671 | |
1672 | release_fault_lock(vmf); |
1673 | if (flags & FAULT_FLAG_KILLABLE) |
1674 | folio_wait_locked_killable(folio); |
1675 | else |
1676 | folio_wait_locked(folio); |
1677 | return VM_FAULT_RETRY; |
1678 | } |
1679 | if (flags & FAULT_FLAG_KILLABLE) { |
1680 | bool ret; |
1681 | |
1682 | ret = __folio_lock_killable(folio); |
1683 | if (ret) { |
1684 | release_fault_lock(vmf); |
1685 | return VM_FAULT_RETRY; |
1686 | } |
1687 | } else { |
1688 | __folio_lock(folio); |
1689 | } |
1690 | |
1691 | return 0; |
1692 | } |
1693 | |
1694 | /** |
1695 | * page_cache_next_miss() - Find the next gap in the page cache. |
1696 | * @mapping: Mapping. |
1697 | * @index: Index. |
1698 | * @max_scan: Maximum range to search. |
1699 | * |
1700 | * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the |
1701 | * gap with the lowest index. |
1702 | * |
1703 | * This function may be called under the rcu_read_lock. However, this will |
1704 | * not atomically search a snapshot of the cache at a single point in time. |
1705 | * For example, if a gap is created at index 5, then subsequently a gap is |
1706 | * created at index 10, page_cache_next_miss covering both indices may |
1707 | * return 10 if called under the rcu_read_lock. |
1708 | * |
1709 | * Return: The index of the gap if found, otherwise an index outside the |
1710 | * range specified (in which case 'return - index >= max_scan' will be true). |
1711 | * In the rare case of index wrap-around, 0 will be returned. |
1712 | */ |
1713 | pgoff_t page_cache_next_miss(struct address_space *mapping, |
1714 | pgoff_t index, unsigned long max_scan) |
1715 | { |
1716 | XA_STATE(xas, &mapping->i_pages, index); |
1717 | |
1718 | while (max_scan--) { |
1719 | void *entry = xas_next(xas: &xas); |
1720 | if (!entry || xa_is_value(entry)) |
1721 | break; |
1722 | if (xas.xa_index == 0) |
1723 | break; |
1724 | } |
1725 | |
1726 | return xas.xa_index; |
1727 | } |
1728 | EXPORT_SYMBOL(page_cache_next_miss); |
1729 | |
1730 | /** |
1731 | * page_cache_prev_miss() - Find the previous gap in the page cache. |
1732 | * @mapping: Mapping. |
1733 | * @index: Index. |
1734 | * @max_scan: Maximum range to search. |
1735 | * |
1736 | * Search the range [max(index - max_scan + 1, 0), index] for the |
1737 | * gap with the highest index. |
1738 | * |
1739 | * This function may be called under the rcu_read_lock. However, this will |
1740 | * not atomically search a snapshot of the cache at a single point in time. |
1741 | * For example, if a gap is created at index 10, then subsequently a gap is |
1742 | * created at index 5, page_cache_prev_miss() covering both indices may |
1743 | * return 5 if called under the rcu_read_lock. |
1744 | * |
1745 | * Return: The index of the gap if found, otherwise an index outside the |
1746 | * range specified (in which case 'index - return >= max_scan' will be true). |
1747 | * In the rare case of wrap-around, ULONG_MAX will be returned. |
1748 | */ |
1749 | pgoff_t page_cache_prev_miss(struct address_space *mapping, |
1750 | pgoff_t index, unsigned long max_scan) |
1751 | { |
1752 | XA_STATE(xas, &mapping->i_pages, index); |
1753 | |
1754 | while (max_scan--) { |
1755 | void *entry = xas_prev(xas: &xas); |
1756 | if (!entry || xa_is_value(entry)) |
1757 | break; |
1758 | if (xas.xa_index == ULONG_MAX) |
1759 | break; |
1760 | } |
1761 | |
1762 | return xas.xa_index; |
1763 | } |
1764 | EXPORT_SYMBOL(page_cache_prev_miss); |
1765 | |
1766 | /* |
1767 | * Lockless page cache protocol: |
1768 | * On the lookup side: |
1769 | * 1. Load the folio from i_pages |
1770 | * 2. Increment the refcount if it's not zero |
1771 | * 3. If the folio is not found by xas_reload(), put the refcount and retry |
1772 | * |
1773 | * On the removal side: |
1774 | * A. Freeze the page (by zeroing the refcount if nobody else has a reference) |
1775 | * B. Remove the page from i_pages |
1776 | * C. Return the page to the page allocator |
1777 | * |
1778 | * This means that any page may have its reference count temporarily |
1779 | * increased by a speculative page cache (or fast GUP) lookup as it can |
1780 | * be allocated by another user before the RCU grace period expires. |
1781 | * Because the refcount temporarily acquired here may end up being the |
1782 | * last refcount on the page, any page allocation must be freeable by |
1783 | * folio_put(). |
1784 | */ |
1785 | |
1786 | /* |
1787 | * filemap_get_entry - Get a page cache entry. |
1788 | * @mapping: the address_space to search |
1789 | * @index: The page cache index. |
1790 | * |
1791 | * Looks up the page cache entry at @mapping & @index. If it is a folio, |
1792 | * it is returned with an increased refcount. If it is a shadow entry |
1793 | * of a previously evicted folio, or a swap entry from shmem/tmpfs, |
1794 | * it is returned without further action. |
1795 | * |
1796 | * Return: The folio, swap or shadow entry, %NULL if nothing is found. |
1797 | */ |
1798 | void *filemap_get_entry(struct address_space *mapping, pgoff_t index) |
1799 | { |
1800 | XA_STATE(xas, &mapping->i_pages, index); |
1801 | struct folio *folio; |
1802 | |
1803 | rcu_read_lock(); |
1804 | repeat: |
1805 | xas_reset(xas: &xas); |
1806 | folio = xas_load(&xas); |
1807 | if (xas_retry(xas: &xas, entry: folio)) |
1808 | goto repeat; |
1809 | /* |
1810 | * A shadow entry of a recently evicted page, or a swap entry from |
1811 | * shmem/tmpfs. Return it without attempting to raise page count. |
1812 | */ |
1813 | if (!folio || xa_is_value(entry: folio)) |
1814 | goto out; |
1815 | |
1816 | if (!folio_try_get_rcu(folio)) |
1817 | goto repeat; |
1818 | |
1819 | if (unlikely(folio != xas_reload(&xas))) { |
1820 | folio_put(folio); |
1821 | goto repeat; |
1822 | } |
1823 | out: |
1824 | rcu_read_unlock(); |
1825 | |
1826 | return folio; |
1827 | } |
1828 | |
1829 | /** |
1830 | * __filemap_get_folio - Find and get a reference to a folio. |
1831 | * @mapping: The address_space to search. |
1832 | * @index: The page index. |
1833 | * @fgp_flags: %FGP flags modify how the folio is returned. |
1834 | * @gfp: Memory allocation flags to use if %FGP_CREAT is specified. |
1835 | * |
1836 | * Looks up the page cache entry at @mapping & @index. |
1837 | * |
1838 | * If %FGP_LOCK or %FGP_CREAT are specified then the function may sleep even |
1839 | * if the %GFP flags specified for %FGP_CREAT are atomic. |
1840 | * |
1841 | * If this function returns a folio, it is returned with an increased refcount. |
1842 | * |
1843 | * Return: The found folio or an ERR_PTR() otherwise. |
1844 | */ |
1845 | struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, |
1846 | fgf_t fgp_flags, gfp_t gfp) |
1847 | { |
1848 | struct folio *folio; |
1849 | |
1850 | repeat: |
1851 | folio = filemap_get_entry(mapping, index); |
1852 | if (xa_is_value(entry: folio)) |
1853 | folio = NULL; |
1854 | if (!folio) |
1855 | goto no_page; |
1856 | |
1857 | if (fgp_flags & FGP_LOCK) { |
1858 | if (fgp_flags & FGP_NOWAIT) { |
1859 | if (!folio_trylock(folio)) { |
1860 | folio_put(folio); |
1861 | return ERR_PTR(error: -EAGAIN); |
1862 | } |
1863 | } else { |
1864 | folio_lock(folio); |
1865 | } |
1866 | |
1867 | /* Has the page been truncated? */ |
1868 | if (unlikely(folio->mapping != mapping)) { |
1869 | folio_unlock(folio); |
1870 | folio_put(folio); |
1871 | goto repeat; |
1872 | } |
1873 | VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); |
1874 | } |
1875 | |
1876 | if (fgp_flags & FGP_ACCESSED) |
1877 | folio_mark_accessed(folio); |
1878 | else if (fgp_flags & FGP_WRITE) { |
1879 | /* Clear idle flag for buffer write */ |
1880 | if (folio_test_idle(folio)) |
1881 | folio_clear_idle(folio); |
1882 | } |
1883 | |
1884 | if (fgp_flags & FGP_STABLE) |
1885 | folio_wait_stable(folio); |
1886 | no_page: |
1887 | if (!folio && (fgp_flags & FGP_CREAT)) { |
1888 | unsigned order = FGF_GET_ORDER(fgp_flags); |
1889 | int err; |
1890 | |
1891 | if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping)) |
1892 | gfp |= __GFP_WRITE; |
1893 | if (fgp_flags & FGP_NOFS) |
1894 | gfp &= ~__GFP_FS; |
1895 | if (fgp_flags & FGP_NOWAIT) { |
1896 | gfp &= ~GFP_KERNEL; |
1897 | gfp |= GFP_NOWAIT | __GFP_NOWARN; |
1898 | } |
1899 | if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP)))) |
1900 | fgp_flags |= FGP_LOCK; |
1901 | |
1902 | if (!mapping_large_folio_support(mapping)) |
1903 | order = 0; |
1904 | if (order > MAX_PAGECACHE_ORDER) |
1905 | order = MAX_PAGECACHE_ORDER; |
1906 | /* If we're not aligned, allocate a smaller folio */ |
1907 | if (index & ((1UL << order) - 1)) |
1908 | order = __ffs(index); |
1909 | |
1910 | do { |
1911 | gfp_t alloc_gfp = gfp; |
1912 | |
1913 | err = -ENOMEM; |
1914 | if (order == 1) |
1915 | order = 0; |
1916 | if (order > 0) |
1917 | alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN; |
1918 | folio = filemap_alloc_folio(alloc_gfp, order); |
1919 | if (!folio) |
1920 | continue; |
1921 | |
1922 | /* Init accessed so avoid atomic mark_page_accessed later */ |
1923 | if (fgp_flags & FGP_ACCESSED) |
1924 | __folio_set_referenced(folio); |
1925 | |
1926 | err = filemap_add_folio(mapping, folio, index, gfp); |
1927 | if (!err) |
1928 | break; |
1929 | folio_put(folio); |
1930 | folio = NULL; |
1931 | } while (order-- > 0); |
1932 | |
1933 | if (err == -EEXIST) |
1934 | goto repeat; |
1935 | if (err) |
1936 | return ERR_PTR(error: err); |
1937 | /* |
1938 | * filemap_add_folio locks the page, and for mmap |
1939 | * we expect an unlocked page. |
1940 | */ |
1941 | if (folio && (fgp_flags & FGP_FOR_MMAP)) |
1942 | folio_unlock(folio); |
1943 | } |
1944 | |
1945 | if (!folio) |
1946 | return ERR_PTR(error: -ENOENT); |
1947 | return folio; |
1948 | } |
1949 | EXPORT_SYMBOL(__filemap_get_folio); |
1950 | |
1951 | static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max, |
1952 | xa_mark_t mark) |
1953 | { |
1954 | struct folio *folio; |
1955 | |
1956 | retry: |
1957 | if (mark == XA_PRESENT) |
1958 | folio = xas_find(xas, max); |
1959 | else |
1960 | folio = xas_find_marked(xas, max, mark); |
1961 | |
1962 | if (xas_retry(xas, entry: folio)) |
1963 | goto retry; |
1964 | /* |
1965 | * A shadow entry of a recently evicted page, a swap |
1966 | * entry from shmem/tmpfs or a DAX entry. Return it |
1967 | * without attempting to raise page count. |
1968 | */ |
1969 | if (!folio || xa_is_value(entry: folio)) |
1970 | return folio; |
1971 | |
1972 | if (!folio_try_get_rcu(folio)) |
1973 | goto reset; |
1974 | |
1975 | if (unlikely(folio != xas_reload(xas))) { |
1976 | folio_put(folio); |
1977 | goto reset; |
1978 | } |
1979 | |
1980 | return folio; |
1981 | reset: |
1982 | xas_reset(xas); |
1983 | goto retry; |
1984 | } |
1985 | |
1986 | /** |
1987 | * find_get_entries - gang pagecache lookup |
1988 | * @mapping: The address_space to search |
1989 | * @start: The starting page cache index |
1990 | * @end: The final page index (inclusive). |
1991 | * @fbatch: Where the resulting entries are placed. |
1992 | * @indices: The cache indices corresponding to the entries in @entries |
1993 | * |
1994 | * find_get_entries() will search for and return a batch of entries in |
1995 | * the mapping. The entries are placed in @fbatch. find_get_entries() |
1996 | * takes a reference on any actual folios it returns. |
1997 | * |
1998 | * The entries have ascending indexes. The indices may not be consecutive |
1999 | * due to not-present entries or large folios. |
2000 | * |
2001 | * Any shadow entries of evicted folios, or swap entries from |
2002 | * shmem/tmpfs, are included in the returned array. |
2003 | * |
2004 | * Return: The number of entries which were found. |
2005 | */ |
2006 | unsigned find_get_entries(struct address_space *mapping, pgoff_t *start, |
2007 | pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices) |
2008 | { |
2009 | XA_STATE(xas, &mapping->i_pages, *start); |
2010 | struct folio *folio; |
2011 | |
2012 | rcu_read_lock(); |
2013 | while ((folio = find_get_entry(xas: &xas, max: end, XA_PRESENT)) != NULL) { |
2014 | indices[fbatch->nr] = xas.xa_index; |
2015 | if (!folio_batch_add(fbatch, folio)) |
2016 | break; |
2017 | } |
2018 | rcu_read_unlock(); |
2019 | |
2020 | if (folio_batch_count(fbatch)) { |
2021 | unsigned long nr = 1; |
2022 | int idx = folio_batch_count(fbatch) - 1; |
2023 | |
2024 | folio = fbatch->folios[idx]; |
2025 | if (!xa_is_value(entry: folio)) |
2026 | nr = folio_nr_pages(folio); |
2027 | *start = indices[idx] + nr; |
2028 | } |
2029 | return folio_batch_count(fbatch); |
2030 | } |
2031 | |
2032 | /** |
2033 | * find_lock_entries - Find a batch of pagecache entries. |
2034 | * @mapping: The address_space to search. |
2035 | * @start: The starting page cache index. |
2036 | * @end: The final page index (inclusive). |
2037 | * @fbatch: Where the resulting entries are placed. |
2038 | * @indices: The cache indices of the entries in @fbatch. |
2039 | * |
2040 | * find_lock_entries() will return a batch of entries from @mapping. |
2041 | * Swap, shadow and DAX entries are included. Folios are returned |
2042 | * locked and with an incremented refcount. Folios which are locked |
2043 | * by somebody else or under writeback are skipped. Folios which are |
2044 | * partially outside the range are not returned. |
2045 | * |
2046 | * The entries have ascending indexes. The indices may not be consecutive |
2047 | * due to not-present entries, large folios, folios which could not be |
2048 | * locked or folios under writeback. |
2049 | * |
2050 | * Return: The number of entries which were found. |
2051 | */ |
2052 | unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start, |
2053 | pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices) |
2054 | { |
2055 | XA_STATE(xas, &mapping->i_pages, *start); |
2056 | struct folio *folio; |
2057 | |
2058 | rcu_read_lock(); |
2059 | while ((folio = find_get_entry(xas: &xas, max: end, XA_PRESENT))) { |
2060 | if (!xa_is_value(entry: folio)) { |
2061 | if (folio->index < *start) |
2062 | goto put; |
2063 | if (folio_next_index(folio) - 1 > end) |
2064 | goto put; |
2065 | if (!folio_trylock(folio)) |
2066 | goto put; |
2067 | if (folio->mapping != mapping || |
2068 | folio_test_writeback(folio)) |
2069 | goto unlock; |
2070 | VM_BUG_ON_FOLIO(!folio_contains(folio, xas.xa_index), |
2071 | folio); |
2072 | } |
2073 | indices[fbatch->nr] = xas.xa_index; |
2074 | if (!folio_batch_add(fbatch, folio)) |
2075 | break; |
2076 | continue; |
2077 | unlock: |
2078 | folio_unlock(folio); |
2079 | put: |
2080 | folio_put(folio); |
2081 | } |
2082 | rcu_read_unlock(); |
2083 | |
2084 | if (folio_batch_count(fbatch)) { |
2085 | unsigned long nr = 1; |
2086 | int idx = folio_batch_count(fbatch) - 1; |
2087 | |
2088 | folio = fbatch->folios[idx]; |
2089 | if (!xa_is_value(entry: folio)) |
2090 | nr = folio_nr_pages(folio); |
2091 | *start = indices[idx] + nr; |
2092 | } |
2093 | return folio_batch_count(fbatch); |
2094 | } |
2095 | |
2096 | /** |
2097 | * filemap_get_folios - Get a batch of folios |
2098 | * @mapping: The address_space to search |
2099 | * @start: The starting page index |
2100 | * @end: The final page index (inclusive) |
2101 | * @fbatch: The batch to fill. |
2102 | * |
2103 | * Search for and return a batch of folios in the mapping starting at |
2104 | * index @start and up to index @end (inclusive). The folios are returned |
2105 | * in @fbatch with an elevated reference count. |
2106 | * |
2107 | * Return: The number of folios which were found. |
2108 | * We also update @start to index the next folio for the traversal. |
2109 | */ |
2110 | unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, |
2111 | pgoff_t end, struct folio_batch *fbatch) |
2112 | { |
2113 | return filemap_get_folios_tag(mapping, start, end, XA_PRESENT, fbatch); |
2114 | } |
2115 | EXPORT_SYMBOL(filemap_get_folios); |
2116 | |
2117 | /** |
2118 | * filemap_get_folios_contig - Get a batch of contiguous folios |
2119 | * @mapping: The address_space to search |
2120 | * @start: The starting page index |
2121 | * @end: The final page index (inclusive) |
2122 | * @fbatch: The batch to fill |
2123 | * |
2124 | * filemap_get_folios_contig() works exactly like filemap_get_folios(), |
2125 | * except the returned folios are guaranteed to be contiguous. This may |
2126 | * not return all contiguous folios if the batch gets filled up. |
2127 | * |
2128 | * Return: The number of folios found. |
2129 | * Also update @start to be positioned for traversal of the next folio. |
2130 | */ |
2131 | |
2132 | unsigned filemap_get_folios_contig(struct address_space *mapping, |
2133 | pgoff_t *start, pgoff_t end, struct folio_batch *fbatch) |
2134 | { |
2135 | XA_STATE(xas, &mapping->i_pages, *start); |
2136 | unsigned long nr; |
2137 | struct folio *folio; |
2138 | |
2139 | rcu_read_lock(); |
2140 | |
2141 | for (folio = xas_load(&xas); folio && xas.xa_index <= end; |
2142 | folio = xas_next(xas: &xas)) { |
2143 | if (xas_retry(xas: &xas, entry: folio)) |
2144 | continue; |
2145 | /* |
2146 | * If the entry has been swapped out, we can stop looking. |
2147 | * No current caller is looking for DAX entries. |
2148 | */ |
2149 | if (xa_is_value(entry: folio)) |
2150 | goto update_start; |
2151 | |
2152 | if (!folio_try_get_rcu(folio)) |
2153 | goto retry; |
2154 | |
2155 | if (unlikely(folio != xas_reload(&xas))) |
2156 | goto put_folio; |
2157 | |
2158 | if (!folio_batch_add(fbatch, folio)) { |
2159 | nr = folio_nr_pages(folio); |
2160 | *start = folio->index + nr; |
2161 | goto out; |
2162 | } |
2163 | continue; |
2164 | put_folio: |
2165 | folio_put(folio); |
2166 | |
2167 | retry: |
2168 | xas_reset(xas: &xas); |
2169 | } |
2170 | |
2171 | update_start: |
2172 | nr = folio_batch_count(fbatch); |
2173 | |
2174 | if (nr) { |
2175 | folio = fbatch->folios[nr - 1]; |
2176 | *start = folio->index + folio_nr_pages(folio); |
2177 | } |
2178 | out: |
2179 | rcu_read_unlock(); |
2180 | return folio_batch_count(fbatch); |
2181 | } |
2182 | EXPORT_SYMBOL(filemap_get_folios_contig); |
2183 | |
2184 | /** |
2185 | * filemap_get_folios_tag - Get a batch of folios matching @tag |
2186 | * @mapping: The address_space to search |
2187 | * @start: The starting page index |
2188 | * @end: The final page index (inclusive) |
2189 | * @tag: The tag index |
2190 | * @fbatch: The batch to fill |
2191 | * |
2192 | * The first folio may start before @start; if it does, it will contain |
2193 | * @start. The final folio may extend beyond @end; if it does, it will |
2194 | * contain @end. The folios have ascending indices. There may be gaps |
2195 | * between the folios if there are indices which have no folio in the |
2196 | * page cache. If folios are added to or removed from the page cache |
2197 | * while this is running, they may or may not be found by this call. |
2198 | * Only returns folios that are tagged with @tag. |
2199 | * |
2200 | * Return: The number of folios found. |
2201 | * Also update @start to index the next folio for traversal. |
2202 | */ |
2203 | unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start, |
2204 | pgoff_t end, xa_mark_t tag, struct folio_batch *fbatch) |
2205 | { |
2206 | XA_STATE(xas, &mapping->i_pages, *start); |
2207 | struct folio *folio; |
2208 | |
2209 | rcu_read_lock(); |
2210 | while ((folio = find_get_entry(xas: &xas, max: end, mark: tag)) != NULL) { |
2211 | /* |
2212 | * Shadow entries should never be tagged, but this iteration |
2213 | * is lockless so there is a window for page reclaim to evict |
2214 | * a page we saw tagged. Skip over it. |
2215 | */ |
2216 | if (xa_is_value(entry: folio)) |
2217 | continue; |
2218 | if (!folio_batch_add(fbatch, folio)) { |
2219 | unsigned long nr = folio_nr_pages(folio); |
2220 | *start = folio->index + nr; |
2221 | goto out; |
2222 | } |
2223 | } |
2224 | /* |
2225 | * We come here when there is no page beyond @end. We take care to not |
2226 | * overflow the index @start as it confuses some of the callers. This |
2227 | * breaks the iteration when there is a page at index -1 but that is |
2228 | * already broke anyway. |
2229 | */ |
2230 | if (end == (pgoff_t)-1) |
2231 | *start = (pgoff_t)-1; |
2232 | else |
2233 | *start = end + 1; |
2234 | out: |
2235 | rcu_read_unlock(); |
2236 | |
2237 | return folio_batch_count(fbatch); |
2238 | } |
2239 | EXPORT_SYMBOL(filemap_get_folios_tag); |
2240 | |
2241 | /* |
2242 | * CD/DVDs are error prone. When a medium error occurs, the driver may fail |
2243 | * a _large_ part of the i/o request. Imagine the worst scenario: |
2244 | * |
2245 | * ---R__________________________________________B__________ |
2246 | * ^ reading here ^ bad block(assume 4k) |
2247 | * |
2248 | * read(R) => miss => readahead(R...B) => media error => frustrating retries |
2249 | * => failing the whole request => read(R) => read(R+1) => |
2250 | * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) => |
2251 | * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) => |
2252 | * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ...... |
2253 | * |
2254 | * It is going insane. Fix it by quickly scaling down the readahead size. |
2255 | */ |
2256 | static void shrink_readahead_size_eio(struct file_ra_state *ra) |
2257 | { |
2258 | ra->ra_pages /= 4; |
2259 | } |
2260 | |
2261 | /* |
2262 | * filemap_get_read_batch - Get a batch of folios for read |
2263 | * |
2264 | * Get a batch of folios which represent a contiguous range of bytes in |
2265 | * the file. No exceptional entries will be returned. If @index is in |
2266 | * the middle of a folio, the entire folio will be returned. The last |
2267 | * folio in the batch may have the readahead flag set or the uptodate flag |
2268 | * clear so that the caller can take the appropriate action. |
2269 | */ |
2270 | static void filemap_get_read_batch(struct address_space *mapping, |
2271 | pgoff_t index, pgoff_t max, struct folio_batch *fbatch) |
2272 | { |
2273 | XA_STATE(xas, &mapping->i_pages, index); |
2274 | struct folio *folio; |
2275 | |
2276 | rcu_read_lock(); |
2277 | for (folio = xas_load(&xas); folio; folio = xas_next(xas: &xas)) { |
2278 | if (xas_retry(xas: &xas, entry: folio)) |
2279 | continue; |
2280 | if (xas.xa_index > max || xa_is_value(entry: folio)) |
2281 | break; |
2282 | if (xa_is_sibling(entry: folio)) |
2283 | break; |
2284 | if (!folio_try_get_rcu(folio)) |
2285 | goto retry; |
2286 | |
2287 | if (unlikely(folio != xas_reload(&xas))) |
2288 | goto put_folio; |
2289 | |
2290 | if (!folio_batch_add(fbatch, folio)) |
2291 | break; |
2292 | if (!folio_test_uptodate(folio)) |
2293 | break; |
2294 | if (folio_test_readahead(folio)) |
2295 | break; |
2296 | xas_advance(xas: &xas, index: folio_next_index(folio) - 1); |
2297 | continue; |
2298 | put_folio: |
2299 | folio_put(folio); |
2300 | retry: |
2301 | xas_reset(xas: &xas); |
2302 | } |
2303 | rcu_read_unlock(); |
2304 | } |
2305 | |
2306 | static int filemap_read_folio(struct file *file, filler_t filler, |
2307 | struct folio *folio) |
2308 | { |
2309 | bool workingset = folio_test_workingset(folio); |
2310 | unsigned long pflags; |
2311 | int error; |
2312 | |
2313 | /* |
2314 | * A previous I/O error may have been due to temporary failures, |
2315 | * eg. multipath errors. PG_error will be set again if read_folio |
2316 | * fails. |
2317 | */ |
2318 | folio_clear_error(folio); |
2319 | |
2320 | /* Start the actual read. The read will unlock the page. */ |
2321 | if (unlikely(workingset)) |
2322 | psi_memstall_enter(flags: &pflags); |
2323 | error = filler(file, folio); |
2324 | if (unlikely(workingset)) |
2325 | psi_memstall_leave(flags: &pflags); |
2326 | if (error) |
2327 | return error; |
2328 | |
2329 | error = folio_wait_locked_killable(folio); |
2330 | if (error) |
2331 | return error; |
2332 | if (folio_test_uptodate(folio)) |
2333 | return 0; |
2334 | if (file) |
2335 | shrink_readahead_size_eio(ra: &file->f_ra); |
2336 | return -EIO; |
2337 | } |
2338 | |
2339 | static bool filemap_range_uptodate(struct address_space *mapping, |
2340 | loff_t pos, size_t count, struct folio *folio, |
2341 | bool need_uptodate) |
2342 | { |
2343 | if (folio_test_uptodate(folio)) |
2344 | return true; |
2345 | /* pipes can't handle partially uptodate pages */ |
2346 | if (need_uptodate) |
2347 | return false; |
2348 | if (!mapping->a_ops->is_partially_uptodate) |
2349 | return false; |
2350 | if (mapping->host->i_blkbits >= folio_shift(folio)) |
2351 | return false; |
2352 | |
2353 | if (folio_pos(folio) > pos) { |
2354 | count -= folio_pos(folio) - pos; |
2355 | pos = 0; |
2356 | } else { |
2357 | pos -= folio_pos(folio); |
2358 | } |
2359 | |
2360 | return mapping->a_ops->is_partially_uptodate(folio, pos, count); |
2361 | } |
2362 | |
2363 | static int filemap_update_page(struct kiocb *iocb, |
2364 | struct address_space *mapping, size_t count, |
2365 | struct folio *folio, bool need_uptodate) |
2366 | { |
2367 | int error; |
2368 | |
2369 | if (iocb->ki_flags & IOCB_NOWAIT) { |
2370 | if (!filemap_invalidate_trylock_shared(mapping)) |
2371 | return -EAGAIN; |
2372 | } else { |
2373 | filemap_invalidate_lock_shared(mapping); |
2374 | } |
2375 | |
2376 | if (!folio_trylock(folio)) { |
2377 | error = -EAGAIN; |
2378 | if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) |
2379 | goto unlock_mapping; |
2380 | if (!(iocb->ki_flags & IOCB_WAITQ)) { |
2381 | filemap_invalidate_unlock_shared(mapping); |
2382 | /* |
2383 | * This is where we usually end up waiting for a |
2384 | * previously submitted readahead to finish. |
2385 | */ |
2386 | folio_put_wait_locked(folio, TASK_KILLABLE); |
2387 | return AOP_TRUNCATED_PAGE; |
2388 | } |
2389 | error = __folio_lock_async(folio, wait: iocb->ki_waitq); |
2390 | if (error) |
2391 | goto unlock_mapping; |
2392 | } |
2393 | |
2394 | error = AOP_TRUNCATED_PAGE; |
2395 | if (!folio->mapping) |
2396 | goto unlock; |
2397 | |
2398 | error = 0; |
2399 | if (filemap_range_uptodate(mapping, pos: iocb->ki_pos, count, folio, |
2400 | need_uptodate)) |
2401 | goto unlock; |
2402 | |
2403 | error = -EAGAIN; |
2404 | if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ)) |
2405 | goto unlock; |
2406 | |
2407 | error = filemap_read_folio(file: iocb->ki_filp, filler: mapping->a_ops->read_folio, |
2408 | folio); |
2409 | goto unlock_mapping; |
2410 | unlock: |
2411 | folio_unlock(folio); |
2412 | unlock_mapping: |
2413 | filemap_invalidate_unlock_shared(mapping); |
2414 | if (error == AOP_TRUNCATED_PAGE) |
2415 | folio_put(folio); |
2416 | return error; |
2417 | } |
2418 | |
2419 | static int filemap_create_folio(struct file *file, |
2420 | struct address_space *mapping, pgoff_t index, |
2421 | struct folio_batch *fbatch) |
2422 | { |
2423 | struct folio *folio; |
2424 | int error; |
2425 | |
2426 | folio = filemap_alloc_folio(mapping_gfp_mask(mapping), 0); |
2427 | if (!folio) |
2428 | return -ENOMEM; |
2429 | |
2430 | /* |
2431 | * Protect against truncate / hole punch. Grabbing invalidate_lock |
2432 | * here assures we cannot instantiate and bring uptodate new |
2433 | * pagecache folios after evicting page cache during truncate |
2434 | * and before actually freeing blocks. Note that we could |
2435 | * release invalidate_lock after inserting the folio into |
2436 | * the page cache as the locked folio would then be enough to |
2437 | * synchronize with hole punching. But there are code paths |
2438 | * such as filemap_update_page() filling in partially uptodate |
2439 | * pages or ->readahead() that need to hold invalidate_lock |
2440 | * while mapping blocks for IO so let's hold the lock here as |
2441 | * well to keep locking rules simple. |
2442 | */ |
2443 | filemap_invalidate_lock_shared(mapping); |
2444 | error = filemap_add_folio(mapping, folio, index, |
2445 | mapping_gfp_constraint(mapping, GFP_KERNEL)); |
2446 | if (error == -EEXIST) |
2447 | error = AOP_TRUNCATED_PAGE; |
2448 | if (error) |
2449 | goto error; |
2450 | |
2451 | error = filemap_read_folio(file, filler: mapping->a_ops->read_folio, folio); |
2452 | if (error) |
2453 | goto error; |
2454 | |
2455 | filemap_invalidate_unlock_shared(mapping); |
2456 | folio_batch_add(fbatch, folio); |
2457 | return 0; |
2458 | error: |
2459 | filemap_invalidate_unlock_shared(mapping); |
2460 | folio_put(folio); |
2461 | return error; |
2462 | } |
2463 | |
2464 | static int filemap_readahead(struct kiocb *iocb, struct file *file, |
2465 | struct address_space *mapping, struct folio *folio, |
2466 | pgoff_t last_index) |
2467 | { |
2468 | DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, folio->index); |
2469 | |
2470 | if (iocb->ki_flags & IOCB_NOIO) |
2471 | return -EAGAIN; |
2472 | page_cache_async_ra(&ractl, folio, req_count: last_index - folio->index); |
2473 | return 0; |
2474 | } |
2475 | |
2476 | static int filemap_get_pages(struct kiocb *iocb, size_t count, |
2477 | struct folio_batch *fbatch, bool need_uptodate) |
2478 | { |
2479 | struct file *filp = iocb->ki_filp; |
2480 | struct address_space *mapping = filp->f_mapping; |
2481 | struct file_ra_state *ra = &filp->f_ra; |
2482 | pgoff_t index = iocb->ki_pos >> PAGE_SHIFT; |
2483 | pgoff_t last_index; |
2484 | struct folio *folio; |
2485 | int err = 0; |
2486 | |
2487 | /* "last_index" is the index of the page beyond the end of the read */ |
2488 | last_index = DIV_ROUND_UP(iocb->ki_pos + count, PAGE_SIZE); |
2489 | retry: |
2490 | if (fatal_signal_pending(current)) |
2491 | return -EINTR; |
2492 | |
2493 | filemap_get_read_batch(mapping, index, max: last_index - 1, fbatch); |
2494 | if (!folio_batch_count(fbatch)) { |
2495 | if (iocb->ki_flags & IOCB_NOIO) |
2496 | return -EAGAIN; |
2497 | page_cache_sync_readahead(mapping, ra, file: filp, index, |
2498 | req_count: last_index - index); |
2499 | filemap_get_read_batch(mapping, index, max: last_index - 1, fbatch); |
2500 | } |
2501 | if (!folio_batch_count(fbatch)) { |
2502 | if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ)) |
2503 | return -EAGAIN; |
2504 | err = filemap_create_folio(file: filp, mapping, |
2505 | index: iocb->ki_pos >> PAGE_SHIFT, fbatch); |
2506 | if (err == AOP_TRUNCATED_PAGE) |
2507 | goto retry; |
2508 | return err; |
2509 | } |
2510 | |
2511 | folio = fbatch->folios[folio_batch_count(fbatch) - 1]; |
2512 | if (folio_test_readahead(folio)) { |
2513 | err = filemap_readahead(iocb, file: filp, mapping, folio, last_index); |
2514 | if (err) |
2515 | goto err; |
2516 | } |
2517 | if (!folio_test_uptodate(folio)) { |
2518 | if ((iocb->ki_flags & IOCB_WAITQ) && |
2519 | folio_batch_count(fbatch) > 1) |
2520 | iocb->ki_flags |= IOCB_NOWAIT; |
2521 | err = filemap_update_page(iocb, mapping, count, folio, |
2522 | need_uptodate); |
2523 | if (err) |
2524 | goto err; |
2525 | } |
2526 | |
2527 | return 0; |
2528 | err: |
2529 | if (err < 0) |
2530 | folio_put(folio); |
2531 | if (likely(--fbatch->nr)) |
2532 | return 0; |
2533 | if (err == AOP_TRUNCATED_PAGE) |
2534 | goto retry; |
2535 | return err; |
2536 | } |
2537 | |
2538 | static inline bool pos_same_folio(loff_t pos1, loff_t pos2, struct folio *folio) |
2539 | { |
2540 | unsigned int shift = folio_shift(folio); |
2541 | |
2542 | return (pos1 >> shift == pos2 >> shift); |
2543 | } |
2544 | |
2545 | /** |
2546 | * filemap_read - Read data from the page cache. |
2547 | * @iocb: The iocb to read. |
2548 | * @iter: Destination for the data. |
2549 | * @already_read: Number of bytes already read by the caller. |
2550 | * |
2551 | * Copies data from the page cache. If the data is not currently present, |
2552 | * uses the readahead and read_folio address_space operations to fetch it. |
2553 | * |
2554 | * Return: Total number of bytes copied, including those already read by |
2555 | * the caller. If an error happens before any bytes are copied, returns |
2556 | * a negative error number. |
2557 | */ |
2558 | ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter, |
2559 | ssize_t already_read) |
2560 | { |
2561 | struct file *filp = iocb->ki_filp; |
2562 | struct file_ra_state *ra = &filp->f_ra; |
2563 | struct address_space *mapping = filp->f_mapping; |
2564 | struct inode *inode = mapping->host; |
2565 | struct folio_batch fbatch; |
2566 | int i, error = 0; |
2567 | bool writably_mapped; |
2568 | loff_t isize, end_offset; |
2569 | loff_t last_pos = ra->prev_pos; |
2570 | |
2571 | if (unlikely(iocb->ki_pos >= inode->i_sb->s_maxbytes)) |
2572 | return 0; |
2573 | if (unlikely(!iov_iter_count(iter))) |
2574 | return 0; |
2575 | |
2576 | iov_iter_truncate(i: iter, count: inode->i_sb->s_maxbytes); |
2577 | folio_batch_init(fbatch: &fbatch); |
2578 | |
2579 | do { |
2580 | cond_resched(); |
2581 | |
2582 | /* |
2583 | * If we've already successfully copied some data, then we |
2584 | * can no longer safely return -EIOCBQUEUED. Hence mark |
2585 | * an async read NOWAIT at that point. |
2586 | */ |
2587 | if ((iocb->ki_flags & IOCB_WAITQ) && already_read) |
2588 | iocb->ki_flags |= IOCB_NOWAIT; |
2589 | |
2590 | if (unlikely(iocb->ki_pos >= i_size_read(inode))) |
2591 | break; |
2592 | |
2593 | error = filemap_get_pages(iocb, count: iter->count, fbatch: &fbatch, need_uptodate: false); |
2594 | if (error < 0) |
2595 | break; |
2596 | |
2597 | /* |
2598 | * i_size must be checked after we know the pages are Uptodate. |
2599 | * |
2600 | * Checking i_size after the check allows us to calculate |
2601 | * the correct value for "nr", which means the zero-filled |
2602 | * part of the page is not copied back to userspace (unless |
2603 | * another truncate extends the file - this is desired though). |
2604 | */ |
2605 | isize = i_size_read(inode); |
2606 | if (unlikely(iocb->ki_pos >= isize)) |
2607 | goto put_folios; |
2608 | end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count); |
2609 | |
2610 | /* |
2611 | * Once we start copying data, we don't want to be touching any |
2612 | * cachelines that might be contended: |
2613 | */ |
2614 | writably_mapped = mapping_writably_mapped(mapping); |
2615 | |
2616 | /* |
2617 | * When a read accesses the same folio several times, only |
2618 | * mark it as accessed the first time. |
2619 | */ |
2620 | if (!pos_same_folio(pos1: iocb->ki_pos, pos2: last_pos - 1, |
2621 | folio: fbatch.folios[0])) |
2622 | folio_mark_accessed(fbatch.folios[0]); |
2623 | |
2624 | for (i = 0; i < folio_batch_count(fbatch: &fbatch); i++) { |
2625 | struct folio *folio = fbatch.folios[i]; |
2626 | size_t fsize = folio_size(folio); |
2627 | size_t offset = iocb->ki_pos & (fsize - 1); |
2628 | size_t bytes = min_t(loff_t, end_offset - iocb->ki_pos, |
2629 | fsize - offset); |
2630 | size_t copied; |
2631 | |
2632 | if (end_offset < folio_pos(folio)) |
2633 | break; |
2634 | if (i > 0) |
2635 | folio_mark_accessed(folio); |
2636 | /* |
2637 | * If users can be writing to this folio using arbitrary |
2638 | * virtual addresses, take care of potential aliasing |
2639 | * before reading the folio on the kernel side. |
2640 | */ |
2641 | if (writably_mapped) |
2642 | flush_dcache_folio(folio); |
2643 | |
2644 | copied = copy_folio_to_iter(folio, offset, bytes, i: iter); |
2645 | |
2646 | already_read += copied; |
2647 | iocb->ki_pos += copied; |
2648 | last_pos = iocb->ki_pos; |
2649 | |
2650 | if (copied < bytes) { |
2651 | error = -EFAULT; |
2652 | break; |
2653 | } |
2654 | } |
2655 | put_folios: |
2656 | for (i = 0; i < folio_batch_count(fbatch: &fbatch); i++) |
2657 | folio_put(folio: fbatch.folios[i]); |
2658 | folio_batch_init(fbatch: &fbatch); |
2659 | } while (iov_iter_count(i: iter) && iocb->ki_pos < isize && !error); |
2660 | |
2661 | file_accessed(file: filp); |
2662 | ra->prev_pos = last_pos; |
2663 | return already_read ? already_read : error; |
2664 | } |
2665 | EXPORT_SYMBOL_GPL(filemap_read); |
2666 | |
2667 | int kiocb_write_and_wait(struct kiocb *iocb, size_t count) |
2668 | { |
2669 | struct address_space *mapping = iocb->ki_filp->f_mapping; |
2670 | loff_t pos = iocb->ki_pos; |
2671 | loff_t end = pos + count - 1; |
2672 | |
2673 | if (iocb->ki_flags & IOCB_NOWAIT) { |
2674 | if (filemap_range_needs_writeback(mapping, start_byte: pos, end_byte: end)) |
2675 | return -EAGAIN; |
2676 | return 0; |
2677 | } |
2678 | |
2679 | return filemap_write_and_wait_range(mapping, pos, end); |
2680 | } |
2681 | |
2682 | int kiocb_invalidate_pages(struct kiocb *iocb, size_t count) |
2683 | { |
2684 | struct address_space *mapping = iocb->ki_filp->f_mapping; |
2685 | loff_t pos = iocb->ki_pos; |
2686 | loff_t end = pos + count - 1; |
2687 | int ret; |
2688 | |
2689 | if (iocb->ki_flags & IOCB_NOWAIT) { |
2690 | /* we could block if there are any pages in the range */ |
2691 | if (filemap_range_has_page(mapping, pos, end)) |
2692 | return -EAGAIN; |
2693 | } else { |
2694 | ret = filemap_write_and_wait_range(mapping, pos, end); |
2695 | if (ret) |
2696 | return ret; |
2697 | } |
2698 | |
2699 | /* |
2700 | * After a write we want buffered reads to be sure to go to disk to get |
2701 | * the new data. We invalidate clean cached page from the region we're |
2702 | * about to write. We do this *before* the write so that we can return |
2703 | * without clobbering -EIOCBQUEUED from ->direct_IO(). |
2704 | */ |
2705 | return invalidate_inode_pages2_range(mapping, start: pos >> PAGE_SHIFT, |
2706 | end: end >> PAGE_SHIFT); |
2707 | } |
2708 | |
2709 | /** |
2710 | * generic_file_read_iter - generic filesystem read routine |
2711 | * @iocb: kernel I/O control block |
2712 | * @iter: destination for the data read |
2713 | * |
2714 | * This is the "read_iter()" routine for all filesystems |
2715 | * that can use the page cache directly. |
2716 | * |
2717 | * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall |
2718 | * be returned when no data can be read without waiting for I/O requests |
2719 | * to complete; it doesn't prevent readahead. |
2720 | * |
2721 | * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O |
2722 | * requests shall be made for the read or for readahead. When no data |
2723 | * can be read, -EAGAIN shall be returned. When readahead would be |
2724 | * triggered, a partial, possibly empty read shall be returned. |
2725 | * |
2726 | * Return: |
2727 | * * number of bytes copied, even for partial reads |
2728 | * * negative error code (or 0 if IOCB_NOIO) if nothing was read |
2729 | */ |
2730 | ssize_t |
2731 | generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) |
2732 | { |
2733 | size_t count = iov_iter_count(i: iter); |
2734 | ssize_t retval = 0; |
2735 | |
2736 | if (!count) |
2737 | return 0; /* skip atime */ |
2738 | |
2739 | if (iocb->ki_flags & IOCB_DIRECT) { |
2740 | struct file *file = iocb->ki_filp; |
2741 | struct address_space *mapping = file->f_mapping; |
2742 | struct inode *inode = mapping->host; |
2743 | |
2744 | retval = kiocb_write_and_wait(iocb, count); |
2745 | if (retval < 0) |
2746 | return retval; |
2747 | file_accessed(file); |
2748 | |
2749 | retval = mapping->a_ops->direct_IO(iocb, iter); |
2750 | if (retval >= 0) { |
2751 | iocb->ki_pos += retval; |
2752 | count -= retval; |
2753 | } |
2754 | if (retval != -EIOCBQUEUED) |
2755 | iov_iter_revert(i: iter, bytes: count - iov_iter_count(i: iter)); |
2756 | |
2757 | /* |
2758 | * Btrfs can have a short DIO read if we encounter |
2759 | * compressed extents, so if there was an error, or if |
2760 | * we've already read everything we wanted to, or if |
2761 | * there was a short read because we hit EOF, go ahead |
2762 | * and return. Otherwise fallthrough to buffered io for |
2763 | * the rest of the read. Buffered reads will not work for |
2764 | * DAX files, so don't bother trying. |
2765 | */ |
2766 | if (retval < 0 || !count || IS_DAX(inode)) |
2767 | return retval; |
2768 | if (iocb->ki_pos >= i_size_read(inode)) |
2769 | return retval; |
2770 | } |
2771 | |
2772 | return filemap_read(iocb, iter, retval); |
2773 | } |
2774 | EXPORT_SYMBOL(generic_file_read_iter); |
2775 | |
2776 | /* |
2777 | * Splice subpages from a folio into a pipe. |
2778 | */ |
2779 | size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, |
2780 | struct folio *folio, loff_t fpos, size_t size) |
2781 | { |
2782 | struct page *page; |
2783 | size_t spliced = 0, offset = offset_in_folio(folio, fpos); |
2784 | |
2785 | page = folio_page(folio, offset / PAGE_SIZE); |
2786 | size = min(size, folio_size(folio) - offset); |
2787 | offset %= PAGE_SIZE; |
2788 | |
2789 | while (spliced < size && |
2790 | !pipe_full(head: pipe->head, tail: pipe->tail, limit: pipe->max_usage)) { |
2791 | struct pipe_buffer *buf = pipe_head_buf(pipe); |
2792 | size_t part = min_t(size_t, PAGE_SIZE - offset, size - spliced); |
2793 | |
2794 | *buf = (struct pipe_buffer) { |
2795 | .ops = &page_cache_pipe_buf_ops, |
2796 | .page = page, |
2797 | .offset = offset, |
2798 | .len = part, |
2799 | }; |
2800 | folio_get(folio); |
2801 | pipe->head++; |
2802 | page++; |
2803 | spliced += part; |
2804 | offset = 0; |
2805 | } |
2806 | |
2807 | return spliced; |
2808 | } |
2809 | |
2810 | /** |
2811 | * filemap_splice_read - Splice data from a file's pagecache into a pipe |
2812 | * @in: The file to read from |
2813 | * @ppos: Pointer to the file position to read from |
2814 | * @pipe: The pipe to splice into |
2815 | * @len: The amount to splice |
2816 | * @flags: The SPLICE_F_* flags |
2817 | * |
2818 | * This function gets folios from a file's pagecache and splices them into the |
2819 | * pipe. Readahead will be called as necessary to fill more folios. This may |
2820 | * be used for blockdevs also. |
2821 | * |
2822 | * Return: On success, the number of bytes read will be returned and *@ppos |
2823 | * will be updated if appropriate; 0 will be returned if there is no more data |
2824 | * to be read; -EAGAIN will be returned if the pipe had no space, and some |
2825 | * other negative error code will be returned on error. A short read may occur |
2826 | * if the pipe has insufficient space, we reach the end of the data or we hit a |
2827 | * hole. |
2828 | */ |
2829 | ssize_t filemap_splice_read(struct file *in, loff_t *ppos, |
2830 | struct pipe_inode_info *pipe, |
2831 | size_t len, unsigned int flags) |
2832 | { |
2833 | struct folio_batch fbatch; |
2834 | struct kiocb iocb; |
2835 | size_t total_spliced = 0, used, npages; |
2836 | loff_t isize, end_offset; |
2837 | bool writably_mapped; |
2838 | int i, error = 0; |
2839 | |
2840 | if (unlikely(*ppos >= in->f_mapping->host->i_sb->s_maxbytes)) |
2841 | return 0; |
2842 | |
2843 | init_sync_kiocb(kiocb: &iocb, filp: in); |
2844 | iocb.ki_pos = *ppos; |
2845 | |
2846 | /* Work out how much data we can actually add into the pipe */ |
2847 | used = pipe_occupancy(head: pipe->head, tail: pipe->tail); |
2848 | npages = max_t(ssize_t, pipe->max_usage - used, 0); |
2849 | len = min_t(size_t, len, npages * PAGE_SIZE); |
2850 | |
2851 | folio_batch_init(fbatch: &fbatch); |
2852 | |
2853 | do { |
2854 | cond_resched(); |
2855 | |
2856 | if (*ppos >= i_size_read(inode: in->f_mapping->host)) |
2857 | break; |
2858 | |
2859 | iocb.ki_pos = *ppos; |
2860 | error = filemap_get_pages(iocb: &iocb, count: len, fbatch: &fbatch, need_uptodate: true); |
2861 | if (error < 0) |
2862 | break; |
2863 | |
2864 | /* |
2865 | * i_size must be checked after we know the pages are Uptodate. |
2866 | * |
2867 | * Checking i_size after the check allows us to calculate |
2868 | * the correct value for "nr", which means the zero-filled |
2869 | * part of the page is not copied back to userspace (unless |
2870 | * another truncate extends the file - this is desired though). |
2871 | */ |
2872 | isize = i_size_read(inode: in->f_mapping->host); |
2873 | if (unlikely(*ppos >= isize)) |
2874 | break; |
2875 | end_offset = min_t(loff_t, isize, *ppos + len); |
2876 | |
2877 | /* |
2878 | * Once we start copying data, we don't want to be touching any |
2879 | * cachelines that might be contended: |
2880 | */ |
2881 | writably_mapped = mapping_writably_mapped(mapping: in->f_mapping); |
2882 | |
2883 | for (i = 0; i < folio_batch_count(fbatch: &fbatch); i++) { |
2884 | struct folio *folio = fbatch.folios[i]; |
2885 | size_t n; |
2886 | |
2887 | if (folio_pos(folio) >= end_offset) |
2888 | goto out; |
2889 | folio_mark_accessed(folio); |
2890 | |
2891 | /* |
2892 | * If users can be writing to this folio using arbitrary |
2893 | * virtual addresses, take care of potential aliasing |
2894 | * before reading the folio on the kernel side. |
2895 | */ |
2896 | if (writably_mapped) |
2897 | flush_dcache_folio(folio); |
2898 | |
2899 | n = min_t(loff_t, len, isize - *ppos); |
2900 | n = splice_folio_into_pipe(pipe, folio, fpos: *ppos, size: n); |
2901 | if (!n) |
2902 | goto out; |
2903 | len -= n; |
2904 | total_spliced += n; |
2905 | *ppos += n; |
2906 | in->f_ra.prev_pos = *ppos; |
2907 | if (pipe_full(head: pipe->head, tail: pipe->tail, limit: pipe->max_usage)) |
2908 | goto out; |
2909 | } |
2910 | |
2911 | folio_batch_release(fbatch: &fbatch); |
2912 | } while (len); |
2913 | |
2914 | out: |
2915 | folio_batch_release(fbatch: &fbatch); |
2916 | file_accessed(file: in); |
2917 | |
2918 | return total_spliced ? total_spliced : error; |
2919 | } |
2920 | EXPORT_SYMBOL(filemap_splice_read); |
2921 | |
2922 | static inline loff_t folio_seek_hole_data(struct xa_state *xas, |
2923 | struct address_space *mapping, struct folio *folio, |
2924 | loff_t start, loff_t end, bool seek_data) |
2925 | { |
2926 | const struct address_space_operations *ops = mapping->a_ops; |
2927 | size_t offset, bsz = i_blocksize(node: mapping->host); |
2928 | |
2929 | if (xa_is_value(entry: folio) || folio_test_uptodate(folio)) |
2930 | return seek_data ? start : end; |
2931 | if (!ops->is_partially_uptodate) |
2932 | return seek_data ? end : start; |
2933 | |
2934 | xas_pause(xas); |
2935 | rcu_read_unlock(); |
2936 | folio_lock(folio); |
2937 | if (unlikely(folio->mapping != mapping)) |
2938 | goto unlock; |
2939 | |
2940 | offset = offset_in_folio(folio, start) & ~(bsz - 1); |
2941 | |
2942 | do { |
2943 | if (ops->is_partially_uptodate(folio, offset, bsz) == |
2944 | seek_data) |
2945 | break; |
2946 | start = (start + bsz) & ~(bsz - 1); |
2947 | offset += bsz; |
2948 | } while (offset < folio_size(folio)); |
2949 | unlock: |
2950 | folio_unlock(folio); |
2951 | rcu_read_lock(); |
2952 | return start; |
2953 | } |
2954 | |
2955 | static inline size_t seek_folio_size(struct xa_state *xas, struct folio *folio) |
2956 | { |
2957 | if (xa_is_value(entry: folio)) |
2958 | return PAGE_SIZE << xa_get_order(xas->xa, index: xas->xa_index); |
2959 | return folio_size(folio); |
2960 | } |
2961 | |
2962 | /** |
2963 | * mapping_seek_hole_data - Seek for SEEK_DATA / SEEK_HOLE in the page cache. |
2964 | * @mapping: Address space to search. |
2965 | * @start: First byte to consider. |
2966 | * @end: Limit of search (exclusive). |
2967 | * @whence: Either SEEK_HOLE or SEEK_DATA. |
2968 | * |
2969 | * If the page cache knows which blocks contain holes and which blocks |
2970 | * contain data, your filesystem can use this function to implement |
2971 | * SEEK_HOLE and SEEK_DATA. This is useful for filesystems which are |
2972 | * entirely memory-based such as tmpfs, and filesystems which support |
2973 | * unwritten extents. |
2974 | * |
2975 | * Return: The requested offset on success, or -ENXIO if @whence specifies |
2976 | * SEEK_DATA and there is no data after @start. There is an implicit hole |
2977 | * after @end - 1, so SEEK_HOLE returns @end if all the bytes between @start |
2978 | * and @end contain data. |
2979 | */ |
2980 | loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start, |
2981 | loff_t end, int whence) |
2982 | { |
2983 | XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT); |
2984 | pgoff_t max = (end - 1) >> PAGE_SHIFT; |
2985 | bool seek_data = (whence == SEEK_DATA); |
2986 | struct folio *folio; |
2987 | |
2988 | if (end <= start) |
2989 | return -ENXIO; |
2990 | |
2991 | rcu_read_lock(); |
2992 | while ((folio = find_get_entry(xas: &xas, max, XA_PRESENT))) { |
2993 | loff_t pos = (u64)xas.xa_index << PAGE_SHIFT; |
2994 | size_t seek_size; |
2995 | |
2996 | if (start < pos) { |
2997 | if (!seek_data) |
2998 | goto unlock; |
2999 | start = pos; |
3000 | } |
3001 | |
3002 | seek_size = seek_folio_size(xas: &xas, folio); |
3003 | pos = round_up((u64)pos + 1, seek_size); |
3004 | start = folio_seek_hole_data(xas: &xas, mapping, folio, start, end: pos, |
3005 | seek_data); |
3006 | if (start < pos) |
3007 | goto unlock; |
3008 | if (start >= end) |
3009 | break; |
3010 | if (seek_size > PAGE_SIZE) |
3011 | xas_set(xas: &xas, index: pos >> PAGE_SHIFT); |
3012 | if (!xa_is_value(entry: folio)) |
3013 | folio_put(folio); |
3014 | } |
3015 | if (seek_data) |
3016 | start = -ENXIO; |
3017 | unlock: |
3018 | rcu_read_unlock(); |
3019 | if (folio && !xa_is_value(entry: folio)) |
3020 | folio_put(folio); |
3021 | if (start > end) |
3022 | return end; |
3023 | return start; |
3024 | } |
3025 | |
3026 | #ifdef CONFIG_MMU |
3027 | #define MMAP_LOTSAMISS (100) |
3028 | /* |
3029 | * lock_folio_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock |
3030 | * @vmf - the vm_fault for this fault. |
3031 | * @folio - the folio to lock. |
3032 | * @fpin - the pointer to the file we may pin (or is already pinned). |
3033 | * |
3034 | * This works similar to lock_folio_or_retry in that it can drop the |
3035 | * mmap_lock. It differs in that it actually returns the folio locked |
3036 | * if it returns 1 and 0 if it couldn't lock the folio. If we did have |
3037 | * to drop the mmap_lock then fpin will point to the pinned file and |
3038 | * needs to be fput()'ed at a later point. |
3039 | */ |
3040 | static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio, |
3041 | struct file **fpin) |
3042 | { |
3043 | if (folio_trylock(folio)) |
3044 | return 1; |
3045 | |
3046 | /* |
3047 | * NOTE! This will make us return with VM_FAULT_RETRY, but with |
3048 | * the fault lock still held. That's how FAULT_FLAG_RETRY_NOWAIT |
3049 | * is supposed to work. We have way too many special cases.. |
3050 | */ |
3051 | if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) |
3052 | return 0; |
3053 | |
3054 | *fpin = maybe_unlock_mmap_for_io(vmf, fpin: *fpin); |
3055 | if (vmf->flags & FAULT_FLAG_KILLABLE) { |
3056 | if (__folio_lock_killable(folio)) { |
3057 | /* |
3058 | * We didn't have the right flags to drop the |
3059 | * fault lock, but all fault_handlers only check |
3060 | * for fatal signals if we return VM_FAULT_RETRY, |
3061 | * so we need to drop the fault lock here and |
3062 | * return 0 if we don't have a fpin. |
3063 | */ |
3064 | if (*fpin == NULL) |
3065 | release_fault_lock(vmf); |
3066 | return 0; |
3067 | } |
3068 | } else |
3069 | __folio_lock(folio); |
3070 | |
3071 | return 1; |
3072 | } |
3073 | |
3074 | /* |
3075 | * Synchronous readahead happens when we don't even find a page in the page |
3076 | * cache at all. We don't want to perform IO under the mmap sem, so if we have |
3077 | * to drop the mmap sem we return the file that was pinned in order for us to do |
3078 | * that. If we didn't pin a file then we return NULL. The file that is |
3079 | * returned needs to be fput()'ed when we're done with it. |
3080 | */ |
3081 | static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) |
3082 | { |
3083 | struct file *file = vmf->vma->vm_file; |
3084 | struct file_ra_state *ra = &file->f_ra; |
3085 | struct address_space *mapping = file->f_mapping; |
3086 | DEFINE_READAHEAD(ractl, file, ra, mapping, vmf->pgoff); |
3087 | struct file *fpin = NULL; |
3088 | unsigned long vm_flags = vmf->vma->vm_flags; |
3089 | unsigned int mmap_miss; |
3090 | |
3091 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
3092 | /* Use the readahead code, even if readahead is disabled */ |
3093 | if (vm_flags & VM_HUGEPAGE) { |
3094 | fpin = maybe_unlock_mmap_for_io(vmf, fpin); |
3095 | ractl._index &= ~((unsigned long)HPAGE_PMD_NR - 1); |
3096 | ra->size = HPAGE_PMD_NR; |
3097 | /* |
3098 | * Fetch two PMD folios, so we get the chance to actually |
3099 | * readahead, unless we've been told not to. |
3100 | */ |
3101 | if (!(vm_flags & VM_RAND_READ)) |
3102 | ra->size *= 2; |
3103 | ra->async_size = HPAGE_PMD_NR; |
3104 | page_cache_ra_order(&ractl, ra, HPAGE_PMD_ORDER); |
3105 | return fpin; |
3106 | } |
3107 | #endif |
3108 | |
3109 | /* If we don't want any read-ahead, don't bother */ |
3110 | if (vm_flags & VM_RAND_READ) |
3111 | return fpin; |
3112 | if (!ra->ra_pages) |
3113 | return fpin; |
3114 | |
3115 | if (vm_flags & VM_SEQ_READ) { |
3116 | fpin = maybe_unlock_mmap_for_io(vmf, fpin); |
3117 | page_cache_sync_ra(&ractl, req_count: ra->ra_pages); |
3118 | return fpin; |
3119 | } |
3120 | |
3121 | /* Avoid banging the cache line if not needed */ |
3122 | mmap_miss = READ_ONCE(ra->mmap_miss); |
3123 | if (mmap_miss < MMAP_LOTSAMISS * 10) |
3124 | WRITE_ONCE(ra->mmap_miss, ++mmap_miss); |
3125 | |
3126 | /* |
3127 | * Do we miss much more than hit in this file? If so, |
3128 | * stop bothering with read-ahead. It will only hurt. |
3129 | */ |
3130 | if (mmap_miss > MMAP_LOTSAMISS) |
3131 | return fpin; |
3132 | |
3133 | /* |
3134 | * mmap read-around |
3135 | */ |
3136 | fpin = maybe_unlock_mmap_for_io(vmf, fpin); |
3137 | ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2); |
3138 | ra->size = ra->ra_pages; |
3139 | ra->async_size = ra->ra_pages / 4; |
3140 | ractl._index = ra->start; |
3141 | page_cache_ra_order(&ractl, ra, order: 0); |
3142 | return fpin; |
3143 | } |
3144 | |
3145 | /* |
3146 | * Asynchronous readahead happens when we find the page and PG_readahead, |
3147 | * so we want to possibly extend the readahead further. We return the file that |
3148 | * was pinned if we have to drop the mmap_lock in order to do IO. |
3149 | */ |
3150 | static struct file *do_async_mmap_readahead(struct vm_fault *vmf, |
3151 | struct folio *folio) |
3152 | { |
3153 | struct file *file = vmf->vma->vm_file; |
3154 | struct file_ra_state *ra = &file->f_ra; |
3155 | DEFINE_READAHEAD(ractl, file, ra, file->f_mapping, vmf->pgoff); |
3156 | struct file *fpin = NULL; |
3157 | unsigned int mmap_miss; |
3158 | |
3159 | /* If we don't want any read-ahead, don't bother */ |
3160 | if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages) |
3161 | return fpin; |
3162 | |
3163 | mmap_miss = READ_ONCE(ra->mmap_miss); |
3164 | if (mmap_miss) |
3165 | WRITE_ONCE(ra->mmap_miss, --mmap_miss); |
3166 | |
3167 | if (folio_test_readahead(folio)) { |
3168 | fpin = maybe_unlock_mmap_for_io(vmf, fpin); |
3169 | page_cache_async_ra(&ractl, folio, req_count: ra->ra_pages); |
3170 | } |
3171 | return fpin; |
3172 | } |
3173 | |
3174 | /** |
3175 | * filemap_fault - read in file data for page fault handling |
3176 | * @vmf: struct vm_fault containing details of the fault |
3177 | * |
3178 | * filemap_fault() is invoked via the vma operations vector for a |
3179 | * mapped memory region to read in file data during a page fault. |
3180 | * |
3181 | * The goto's are kind of ugly, but this streamlines the normal case of having |
3182 | * it in the page cache, and handles the special cases reasonably without |
3183 | * having a lot of duplicated code. |
3184 | * |
3185 | * vma->vm_mm->mmap_lock must be held on entry. |
3186 | * |
3187 | * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock |
3188 | * may be dropped before doing I/O or by lock_folio_maybe_drop_mmap(). |
3189 | * |
3190 | * If our return value does not have VM_FAULT_RETRY set, the mmap_lock |
3191 | * has not been released. |
3192 | * |
3193 | * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set. |
3194 | * |
3195 | * Return: bitwise-OR of %VM_FAULT_ codes. |
3196 | */ |
3197 | vm_fault_t filemap_fault(struct vm_fault *vmf) |
3198 | { |
3199 | int error; |
3200 | struct file *file = vmf->vma->vm_file; |
3201 | struct file *fpin = NULL; |
3202 | struct address_space *mapping = file->f_mapping; |
3203 | struct inode *inode = mapping->host; |
3204 | pgoff_t max_idx, index = vmf->pgoff; |
3205 | struct folio *folio; |
3206 | vm_fault_t ret = 0; |
3207 | bool mapping_locked = false; |
3208 | |
3209 | max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); |
3210 | if (unlikely(index >= max_idx)) |
3211 | return VM_FAULT_SIGBUS; |
3212 | |
3213 | /* |
3214 | * Do we have something in the page cache already? |
3215 | */ |
3216 | folio = filemap_get_folio(mapping, index); |
3217 | if (likely(!IS_ERR(folio))) { |
3218 | /* |
3219 | * We found the page, so try async readahead before waiting for |
3220 | * the lock. |
3221 | */ |
3222 | if (!(vmf->flags & FAULT_FLAG_TRIED)) |
3223 | fpin = do_async_mmap_readahead(vmf, folio); |
3224 | if (unlikely(!folio_test_uptodate(folio))) { |
3225 | filemap_invalidate_lock_shared(mapping); |
3226 | mapping_locked = true; |
3227 | } |
3228 | } else { |
3229 | /* No page in the page cache at all */ |
3230 | count_vm_event(item: PGMAJFAULT); |
3231 | count_memcg_event_mm(mm: vmf->vma->vm_mm, idx: PGMAJFAULT); |
3232 | ret = VM_FAULT_MAJOR; |
3233 | fpin = do_sync_mmap_readahead(vmf); |
3234 | retry_find: |
3235 | /* |
3236 | * See comment in filemap_create_folio() why we need |
3237 | * invalidate_lock |
3238 | */ |
3239 | if (!mapping_locked) { |
3240 | filemap_invalidate_lock_shared(mapping); |
3241 | mapping_locked = true; |
3242 | } |
3243 | folio = __filemap_get_folio(mapping, index, |
3244 | FGP_CREAT|FGP_FOR_MMAP, |
3245 | vmf->gfp_mask); |
3246 | if (IS_ERR(ptr: folio)) { |
3247 | if (fpin) |
3248 | goto out_retry; |
3249 | filemap_invalidate_unlock_shared(mapping); |
3250 | return VM_FAULT_OOM; |
3251 | } |
3252 | } |
3253 | |
3254 | if (!lock_folio_maybe_drop_mmap(vmf, folio, fpin: &fpin)) |
3255 | goto out_retry; |
3256 | |
3257 | /* Did it get truncated? */ |
3258 | if (unlikely(folio->mapping != mapping)) { |
3259 | folio_unlock(folio); |
3260 | folio_put(folio); |
3261 | goto retry_find; |
3262 | } |
3263 | VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); |
3264 | |
3265 | /* |
3266 | * We have a locked folio in the page cache, now we need to check |
3267 | * that it's up-to-date. If not, it is going to be due to an error, |
3268 | * or because readahead was otherwise unable to retrieve it. |
3269 | */ |
3270 | if (unlikely(!folio_test_uptodate(folio))) { |
3271 | /* |
3272 | * If the invalidate lock is not held, the folio was in cache |
3273 | * and uptodate and now it is not. Strange but possible since we |
3274 | * didn't hold the page lock all the time. Let's drop |
3275 | * everything, get the invalidate lock and try again. |
3276 | */ |
3277 | if (!mapping_locked) { |
3278 | folio_unlock(folio); |
3279 | folio_put(folio); |
3280 | goto retry_find; |
3281 | } |
3282 | |
3283 | /* |
3284 | * OK, the folio is really not uptodate. This can be because the |
3285 | * VMA has the VM_RAND_READ flag set, or because an error |
3286 | * arose. Let's read it in directly. |
3287 | */ |
3288 | goto page_not_uptodate; |
3289 | } |
3290 | |
3291 | /* |
3292 | * We've made it this far and we had to drop our mmap_lock, now is the |
3293 | * time to return to the upper layer and have it re-find the vma and |
3294 | * redo the fault. |
3295 | */ |
3296 | if (fpin) { |
3297 | folio_unlock(folio); |
3298 | goto out_retry; |
3299 | } |
3300 | if (mapping_locked) |
3301 | filemap_invalidate_unlock_shared(mapping); |
3302 | |
3303 | /* |
3304 | * Found the page and have a reference on it. |
3305 | * We must recheck i_size under page lock. |
3306 | */ |
3307 | max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); |
3308 | if (unlikely(index >= max_idx)) { |
3309 | folio_unlock(folio); |
3310 | folio_put(folio); |
3311 | return VM_FAULT_SIGBUS; |
3312 | } |
3313 | |
3314 | vmf->page = folio_file_page(folio, index); |
3315 | return ret | VM_FAULT_LOCKED; |
3316 | |
3317 | page_not_uptodate: |
3318 | /* |
3319 | * Umm, take care of errors if the page isn't up-to-date. |
3320 | * Try to re-read it _once_. We do this synchronously, |
3321 | * because there really aren't any performance issues here |
3322 | * and we need to check for errors. |
3323 | */ |
3324 | fpin = maybe_unlock_mmap_for_io(vmf, fpin); |
3325 | error = filemap_read_folio(file, filler: mapping->a_ops->read_folio, folio); |
3326 | if (fpin) |
3327 | goto out_retry; |
3328 | folio_put(folio); |
3329 | |
3330 | if (!error || error == AOP_TRUNCATED_PAGE) |
3331 | goto retry_find; |
3332 | filemap_invalidate_unlock_shared(mapping); |
3333 | |
3334 | return VM_FAULT_SIGBUS; |
3335 | |
3336 | out_retry: |
3337 | /* |
3338 | * We dropped the mmap_lock, we need to return to the fault handler to |
3339 | * re-find the vma and come back and find our hopefully still populated |
3340 | * page. |
3341 | */ |
3342 | if (!IS_ERR(ptr: folio)) |
3343 | folio_put(folio); |
3344 | if (mapping_locked) |
3345 | filemap_invalidate_unlock_shared(mapping); |
3346 | if (fpin) |
3347 | fput(fpin); |
3348 | return ret | VM_FAULT_RETRY; |
3349 | } |
3350 | EXPORT_SYMBOL(filemap_fault); |
3351 | |
3352 | static bool filemap_map_pmd(struct vm_fault *vmf, struct folio *folio, |
3353 | pgoff_t start) |
3354 | { |
3355 | struct mm_struct *mm = vmf->vma->vm_mm; |
3356 | |
3357 | /* Huge page is mapped? No need to proceed. */ |
3358 | if (pmd_trans_huge(pmd: *vmf->pmd)) { |
3359 | folio_unlock(folio); |
3360 | folio_put(folio); |
3361 | return true; |
3362 | } |
3363 | |
3364 | if (pmd_none(pmd: *vmf->pmd) && folio_test_pmd_mappable(folio)) { |
3365 | struct page *page = folio_file_page(folio, index: start); |
3366 | vm_fault_t ret = do_set_pmd(vmf, page); |
3367 | if (!ret) { |
3368 | /* The page is mapped successfully, reference consumed. */ |
3369 | folio_unlock(folio); |
3370 | return true; |
3371 | } |
3372 | } |
3373 | |
3374 | if (pmd_none(pmd: *vmf->pmd)) |
3375 | pmd_install(mm, pmd: vmf->pmd, pte: &vmf->prealloc_pte); |
3376 | |
3377 | return false; |
3378 | } |
3379 | |
3380 | static struct folio *next_uptodate_folio(struct xa_state *xas, |
3381 | struct address_space *mapping, pgoff_t end_pgoff) |
3382 | { |
3383 | struct folio *folio = xas_next_entry(xas, max: end_pgoff); |
3384 | unsigned long max_idx; |
3385 | |
3386 | do { |
3387 | if (!folio) |
3388 | return NULL; |
3389 | if (xas_retry(xas, entry: folio)) |
3390 | continue; |
3391 | if (xa_is_value(entry: folio)) |
3392 | continue; |
3393 | if (folio_test_locked(folio)) |
3394 | continue; |
3395 | if (!folio_try_get_rcu(folio)) |
3396 | continue; |
3397 | /* Has the page moved or been split? */ |
3398 | if (unlikely(folio != xas_reload(xas))) |
3399 | goto skip; |
3400 | if (!folio_test_uptodate(folio) || folio_test_readahead(folio)) |
3401 | goto skip; |
3402 | if (!folio_trylock(folio)) |
3403 | goto skip; |
3404 | if (folio->mapping != mapping) |
3405 | goto unlock; |
3406 | if (!folio_test_uptodate(folio)) |
3407 | goto unlock; |
3408 | max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); |
3409 | if (xas->xa_index >= max_idx) |
3410 | goto unlock; |
3411 | return folio; |
3412 | unlock: |
3413 | folio_unlock(folio); |
3414 | skip: |
3415 | folio_put(folio); |
3416 | } while ((folio = xas_next_entry(xas, max: end_pgoff)) != NULL); |
3417 | |
3418 | return NULL; |
3419 | } |
3420 | |
3421 | /* |
3422 | * Map page range [start_page, start_page + nr_pages) of folio. |
3423 | * start_page is gotten from start by folio_page(folio, start) |
3424 | */ |
3425 | static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, |
3426 | struct folio *folio, unsigned long start, |
3427 | unsigned long addr, unsigned int nr_pages, |
3428 | unsigned int *mmap_miss) |
3429 | { |
3430 | vm_fault_t ret = 0; |
3431 | struct page *page = folio_page(folio, start); |
3432 | unsigned int count = 0; |
3433 | pte_t *old_ptep = vmf->pte; |
3434 | |
3435 | do { |
3436 | if (PageHWPoison(page: page + count)) |
3437 | goto skip; |
3438 | |
3439 | (*mmap_miss)++; |
3440 | |
3441 | /* |
3442 | * NOTE: If there're PTE markers, we'll leave them to be |
3443 | * handled in the specific fault path, and it'll prohibit the |
3444 | * fault-around logic. |
3445 | */ |
3446 | if (!pte_none(pte: vmf->pte[count])) |
3447 | goto skip; |
3448 | |
3449 | count++; |
3450 | continue; |
3451 | skip: |
3452 | if (count) { |
3453 | set_pte_range(vmf, folio, page, nr: count, addr); |
3454 | folio_ref_add(folio, nr: count); |
3455 | if (in_range(vmf->address, addr, count * PAGE_SIZE)) |
3456 | ret = VM_FAULT_NOPAGE; |
3457 | } |
3458 | |
3459 | count++; |
3460 | page += count; |
3461 | vmf->pte += count; |
3462 | addr += count * PAGE_SIZE; |
3463 | count = 0; |
3464 | } while (--nr_pages > 0); |
3465 | |
3466 | if (count) { |
3467 | set_pte_range(vmf, folio, page, nr: count, addr); |
3468 | folio_ref_add(folio, nr: count); |
3469 | if (in_range(vmf->address, addr, count * PAGE_SIZE)) |
3470 | ret = VM_FAULT_NOPAGE; |
3471 | } |
3472 | |
3473 | vmf->pte = old_ptep; |
3474 | |
3475 | return ret; |
3476 | } |
3477 | |
3478 | static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, |
3479 | struct folio *folio, unsigned long addr, |
3480 | unsigned int *mmap_miss) |
3481 | { |
3482 | vm_fault_t ret = 0; |
3483 | struct page *page = &folio->page; |
3484 | |
3485 | if (PageHWPoison(page)) |
3486 | return ret; |
3487 | |
3488 | (*mmap_miss)++; |
3489 | |
3490 | /* |
3491 | * NOTE: If there're PTE markers, we'll leave them to be |
3492 | * handled in the specific fault path, and it'll prohibit |
3493 | * the fault-around logic. |
3494 | */ |
3495 | if (!pte_none(pte: ptep_get(ptep: vmf->pte))) |
3496 | return ret; |
3497 | |
3498 | if (vmf->address == addr) |
3499 | ret = VM_FAULT_NOPAGE; |
3500 | |
3501 | set_pte_range(vmf, folio, page, nr: 1, addr); |
3502 | folio_ref_inc(folio); |
3503 | |
3504 | return ret; |
3505 | } |
3506 | |
3507 | vm_fault_t filemap_map_pages(struct vm_fault *vmf, |
3508 | pgoff_t start_pgoff, pgoff_t end_pgoff) |
3509 | { |
3510 | struct vm_area_struct *vma = vmf->vma; |
3511 | struct file *file = vma->vm_file; |
3512 | struct address_space *mapping = file->f_mapping; |
3513 | pgoff_t last_pgoff = start_pgoff; |
3514 | unsigned long addr; |
3515 | XA_STATE(xas, &mapping->i_pages, start_pgoff); |
3516 | struct folio *folio; |
3517 | vm_fault_t ret = 0; |
3518 | unsigned int nr_pages = 0, mmap_miss = 0, mmap_miss_saved; |
3519 | |
3520 | rcu_read_lock(); |
3521 | folio = next_uptodate_folio(xas: &xas, mapping, end_pgoff); |
3522 | if (!folio) |
3523 | goto out; |
3524 | |
3525 | if (filemap_map_pmd(vmf, folio, start: start_pgoff)) { |
3526 | ret = VM_FAULT_NOPAGE; |
3527 | goto out; |
3528 | } |
3529 | |
3530 | addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT); |
3531 | vmf->pte = pte_offset_map_lock(mm: vma->vm_mm, pmd: vmf->pmd, addr, ptlp: &vmf->ptl); |
3532 | if (!vmf->pte) { |
3533 | folio_unlock(folio); |
3534 | folio_put(folio); |
3535 | goto out; |
3536 | } |
3537 | do { |
3538 | unsigned long end; |
3539 | |
3540 | addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT; |
3541 | vmf->pte += xas.xa_index - last_pgoff; |
3542 | last_pgoff = xas.xa_index; |
3543 | end = folio_next_index(folio) - 1; |
3544 | nr_pages = min(end, end_pgoff) - xas.xa_index + 1; |
3545 | |
3546 | if (!folio_test_large(folio)) |
3547 | ret |= filemap_map_order0_folio(vmf, |
3548 | folio, addr, mmap_miss: &mmap_miss); |
3549 | else |
3550 | ret |= filemap_map_folio_range(vmf, folio, |
3551 | start: xas.xa_index - folio->index, addr, |
3552 | nr_pages, mmap_miss: &mmap_miss); |
3553 | |
3554 | folio_unlock(folio); |
3555 | folio_put(folio); |
3556 | } while ((folio = next_uptodate_folio(xas: &xas, mapping, end_pgoff)) != NULL); |
3557 | pte_unmap_unlock(vmf->pte, vmf->ptl); |
3558 | out: |
3559 | rcu_read_unlock(); |
3560 | |
3561 | mmap_miss_saved = READ_ONCE(file->f_ra.mmap_miss); |
3562 | if (mmap_miss >= mmap_miss_saved) |
3563 | WRITE_ONCE(file->f_ra.mmap_miss, 0); |
3564 | else |
3565 | WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss_saved - mmap_miss); |
3566 | |
3567 | return ret; |
3568 | } |
3569 | EXPORT_SYMBOL(filemap_map_pages); |
3570 | |
3571 | vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf) |
3572 | { |
3573 | struct address_space *mapping = vmf->vma->vm_file->f_mapping; |
3574 | struct folio *folio = page_folio(vmf->page); |
3575 | vm_fault_t ret = VM_FAULT_LOCKED; |
3576 | |
3577 | sb_start_pagefault(sb: mapping->host->i_sb); |
3578 | file_update_time(file: vmf->vma->vm_file); |
3579 | folio_lock(folio); |
3580 | if (folio->mapping != mapping) { |
3581 | folio_unlock(folio); |
3582 | ret = VM_FAULT_NOPAGE; |
3583 | goto out; |
3584 | } |
3585 | /* |
3586 | * We mark the folio dirty already here so that when freeze is in |
3587 | * progress, we are guaranteed that writeback during freezing will |
3588 | * see the dirty folio and writeprotect it again. |
3589 | */ |
3590 | folio_mark_dirty(folio); |
3591 | folio_wait_stable(folio); |
3592 | out: |
3593 | sb_end_pagefault(sb: mapping->host->i_sb); |
3594 | return ret; |
3595 | } |
3596 | |
3597 | const struct vm_operations_struct generic_file_vm_ops = { |
3598 | .fault = filemap_fault, |
3599 | .map_pages = filemap_map_pages, |
3600 | .page_mkwrite = filemap_page_mkwrite, |
3601 | }; |
3602 | |
3603 | /* This is used for a general mmap of a disk file */ |
3604 | |
3605 | int generic_file_mmap(struct file *file, struct vm_area_struct *vma) |
3606 | { |
3607 | struct address_space *mapping = file->f_mapping; |
3608 | |
3609 | if (!mapping->a_ops->read_folio) |
3610 | return -ENOEXEC; |
3611 | file_accessed(file); |
3612 | vma->vm_ops = &generic_file_vm_ops; |
3613 | return 0; |
3614 | } |
3615 | |
3616 | /* |
3617 | * This is for filesystems which do not implement ->writepage. |
3618 | */ |
3619 | int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) |
3620 | { |
3621 | if (vma_is_shared_maywrite(vma)) |
3622 | return -EINVAL; |
3623 | return generic_file_mmap(file, vma); |
3624 | } |
3625 | #else |
3626 | vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf) |
3627 | { |
3628 | return VM_FAULT_SIGBUS; |
3629 | } |
3630 | int generic_file_mmap(struct file *file, struct vm_area_struct *vma) |
3631 | { |
3632 | return -ENOSYS; |
3633 | } |
3634 | int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma) |
3635 | { |
3636 | return -ENOSYS; |
3637 | } |
3638 | #endif /* CONFIG_MMU */ |
3639 | |
3640 | EXPORT_SYMBOL(filemap_page_mkwrite); |
3641 | EXPORT_SYMBOL(generic_file_mmap); |
3642 | EXPORT_SYMBOL(generic_file_readonly_mmap); |
3643 | |
3644 | static struct folio *do_read_cache_folio(struct address_space *mapping, |
3645 | pgoff_t index, filler_t filler, struct file *file, gfp_t gfp) |
3646 | { |
3647 | struct folio *folio; |
3648 | int err; |
3649 | |
3650 | if (!filler) |
3651 | filler = mapping->a_ops->read_folio; |
3652 | repeat: |
3653 | folio = filemap_get_folio(mapping, index); |
3654 | if (IS_ERR(ptr: folio)) { |
3655 | folio = filemap_alloc_folio(gfp, 0); |
3656 | if (!folio) |
3657 | return ERR_PTR(error: -ENOMEM); |
3658 | err = filemap_add_folio(mapping, folio, index, gfp); |
3659 | if (unlikely(err)) { |
3660 | folio_put(folio); |
3661 | if (err == -EEXIST) |
3662 | goto repeat; |
3663 | /* Presumably ENOMEM for xarray node */ |
3664 | return ERR_PTR(error: err); |
3665 | } |
3666 | |
3667 | goto filler; |
3668 | } |
3669 | if (folio_test_uptodate(folio)) |
3670 | goto out; |
3671 | |
3672 | if (!folio_trylock(folio)) { |
3673 | folio_put_wait_locked(folio, TASK_UNINTERRUPTIBLE); |
3674 | goto repeat; |
3675 | } |
3676 | |
3677 | /* Folio was truncated from mapping */ |
3678 | if (!folio->mapping) { |
3679 | folio_unlock(folio); |
3680 | folio_put(folio); |
3681 | goto repeat; |
3682 | } |
3683 | |
3684 | /* Someone else locked and filled the page in a very small window */ |
3685 | if (folio_test_uptodate(folio)) { |
3686 | folio_unlock(folio); |
3687 | goto out; |
3688 | } |
3689 | |
3690 | filler: |
3691 | err = filemap_read_folio(file, filler, folio); |
3692 | if (err) { |
3693 | folio_put(folio); |
3694 | if (err == AOP_TRUNCATED_PAGE) |
3695 | goto repeat; |
3696 | return ERR_PTR(error: err); |
3697 | } |
3698 | |
3699 | out: |
3700 | folio_mark_accessed(folio); |
3701 | return folio; |
3702 | } |
3703 | |
3704 | /** |
3705 | * read_cache_folio - Read into page cache, fill it if needed. |
3706 | * @mapping: The address_space to read from. |
3707 | * @index: The index to read. |
3708 | * @filler: Function to perform the read, or NULL to use aops->read_folio(). |
3709 | * @file: Passed to filler function, may be NULL if not required. |
3710 | * |
3711 | * Read one page into the page cache. If it succeeds, the folio returned |
3712 | * will contain @index, but it may not be the first page of the folio. |
3713 | * |
3714 | * If the filler function returns an error, it will be returned to the |
3715 | * caller. |
3716 | * |
3717 | * Context: May sleep. Expects mapping->invalidate_lock to be held. |
3718 | * Return: An uptodate folio on success, ERR_PTR() on failure. |
3719 | */ |
3720 | struct folio *read_cache_folio(struct address_space *mapping, pgoff_t index, |
3721 | filler_t filler, struct file *file) |
3722 | { |
3723 | return do_read_cache_folio(mapping, index, filler, file, |
3724 | gfp: mapping_gfp_mask(mapping)); |
3725 | } |
3726 | EXPORT_SYMBOL(read_cache_folio); |
3727 | |
3728 | /** |
3729 | * mapping_read_folio_gfp - Read into page cache, using specified allocation flags. |
3730 | * @mapping: The address_space for the folio. |
3731 | * @index: The index that the allocated folio will contain. |
3732 | * @gfp: The page allocator flags to use if allocating. |
3733 | * |
3734 | * This is the same as "read_cache_folio(mapping, index, NULL, NULL)", but with |
3735 | * any new memory allocations done using the specified allocation flags. |
3736 | * |
3737 | * The most likely error from this function is EIO, but ENOMEM is |
3738 | * possible and so is EINTR. If ->read_folio returns another error, |
3739 | * that will be returned to the caller. |
3740 | * |
3741 | * The function expects mapping->invalidate_lock to be already held. |
3742 | * |
3743 | * Return: Uptodate folio on success, ERR_PTR() on failure. |
3744 | */ |
3745 | struct folio *mapping_read_folio_gfp(struct address_space *mapping, |
3746 | pgoff_t index, gfp_t gfp) |
3747 | { |
3748 | return do_read_cache_folio(mapping, index, NULL, NULL, gfp); |
3749 | } |
3750 | EXPORT_SYMBOL(mapping_read_folio_gfp); |
3751 | |
3752 | static struct page *do_read_cache_page(struct address_space *mapping, |
3753 | pgoff_t index, filler_t *filler, struct file *file, gfp_t gfp) |
3754 | { |
3755 | struct folio *folio; |
3756 | |
3757 | folio = do_read_cache_folio(mapping, index, filler, file, gfp); |
3758 | if (IS_ERR(ptr: folio)) |
3759 | return &folio->page; |
3760 | return folio_file_page(folio, index); |
3761 | } |
3762 | |
3763 | struct page *read_cache_page(struct address_space *mapping, |
3764 | pgoff_t index, filler_t *filler, struct file *file) |
3765 | { |
3766 | return do_read_cache_page(mapping, index, filler, file, |
3767 | gfp: mapping_gfp_mask(mapping)); |
3768 | } |
3769 | EXPORT_SYMBOL(read_cache_page); |
3770 | |
3771 | /** |
3772 | * read_cache_page_gfp - read into page cache, using specified page allocation flags. |
3773 | * @mapping: the page's address_space |
3774 | * @index: the page index |
3775 | * @gfp: the page allocator flags to use if allocating |
3776 | * |
3777 | * This is the same as "read_mapping_page(mapping, index, NULL)", but with |
3778 | * any new page allocations done using the specified allocation flags. |
3779 | * |
3780 | * If the page does not get brought uptodate, return -EIO. |
3781 | * |
3782 | * The function expects mapping->invalidate_lock to be already held. |
3783 | * |
3784 | * Return: up to date page on success, ERR_PTR() on failure. |
3785 | */ |
3786 | struct page *read_cache_page_gfp(struct address_space *mapping, |
3787 | pgoff_t index, |
3788 | gfp_t gfp) |
3789 | { |
3790 | return do_read_cache_page(mapping, index, NULL, NULL, gfp); |
3791 | } |
3792 | EXPORT_SYMBOL(read_cache_page_gfp); |
3793 | |
3794 | /* |
3795 | * Warn about a page cache invalidation failure during a direct I/O write. |
3796 | */ |
3797 | static void dio_warn_stale_pagecache(struct file *filp) |
3798 | { |
3799 | static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST); |
3800 | char pathname[128]; |
3801 | char *path; |
3802 | |
3803 | errseq_set(eseq: &filp->f_mapping->wb_err, err: -EIO); |
3804 | if (__ratelimit(&_rs)) { |
3805 | path = file_path(filp, pathname, sizeof(pathname)); |
3806 | if (IS_ERR(ptr: path)) |
3807 | path = "(unknown)" ; |
3808 | pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n" ); |
3809 | pr_crit("File: %s PID: %d Comm: %.20s\n" , path, current->pid, |
3810 | current->comm); |
3811 | } |
3812 | } |
3813 | |
3814 | void kiocb_invalidate_post_direct_write(struct kiocb *iocb, size_t count) |
3815 | { |
3816 | struct address_space *mapping = iocb->ki_filp->f_mapping; |
3817 | |
3818 | if (mapping->nrpages && |
3819 | invalidate_inode_pages2_range(mapping, |
3820 | start: iocb->ki_pos >> PAGE_SHIFT, |
3821 | end: (iocb->ki_pos + count - 1) >> PAGE_SHIFT)) |
3822 | dio_warn_stale_pagecache(filp: iocb->ki_filp); |
3823 | } |
3824 | |
3825 | ssize_t |
3826 | generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from) |
3827 | { |
3828 | struct address_space *mapping = iocb->ki_filp->f_mapping; |
3829 | size_t write_len = iov_iter_count(i: from); |
3830 | ssize_t written; |
3831 | |
3832 | /* |
3833 | * If a page can not be invalidated, return 0 to fall back |
3834 | * to buffered write. |
3835 | */ |
3836 | written = kiocb_invalidate_pages(iocb, count: write_len); |
3837 | if (written) { |
3838 | if (written == -EBUSY) |
3839 | return 0; |
3840 | return written; |
3841 | } |
3842 | |
3843 | written = mapping->a_ops->direct_IO(iocb, from); |
3844 | |
3845 | /* |
3846 | * Finally, try again to invalidate clean pages which might have been |
3847 | * cached by non-direct readahead, or faulted in by get_user_pages() |
3848 | * if the source of the write was an mmap'ed region of the file |
3849 | * we're writing. Either one is a pretty crazy thing to do, |
3850 | * so we don't support it 100%. If this invalidation |
3851 | * fails, tough, the write still worked... |
3852 | * |
3853 | * Most of the time we do not need this since dio_complete() will do |
3854 | * the invalidation for us. However there are some file systems that |
3855 | * do not end up with dio_complete() being called, so let's not break |
3856 | * them by removing it completely. |
3857 | * |
3858 | * Noticeable example is a blkdev_direct_IO(). |
3859 | * |
3860 | * Skip invalidation for async writes or if mapping has no pages. |
3861 | */ |
3862 | if (written > 0) { |
3863 | struct inode *inode = mapping->host; |
3864 | loff_t pos = iocb->ki_pos; |
3865 | |
3866 | kiocb_invalidate_post_direct_write(iocb, count: written); |
3867 | pos += written; |
3868 | write_len -= written; |
3869 | if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) { |
3870 | i_size_write(inode, i_size: pos); |
3871 | mark_inode_dirty(inode); |
3872 | } |
3873 | iocb->ki_pos = pos; |
3874 | } |
3875 | if (written != -EIOCBQUEUED) |
3876 | iov_iter_revert(i: from, bytes: write_len - iov_iter_count(i: from)); |
3877 | return written; |
3878 | } |
3879 | EXPORT_SYMBOL(generic_file_direct_write); |
3880 | |
3881 | ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i) |
3882 | { |
3883 | struct file *file = iocb->ki_filp; |
3884 | loff_t pos = iocb->ki_pos; |
3885 | struct address_space *mapping = file->f_mapping; |
3886 | const struct address_space_operations *a_ops = mapping->a_ops; |
3887 | long status = 0; |
3888 | ssize_t written = 0; |
3889 | |
3890 | do { |
3891 | struct page *page; |
3892 | unsigned long offset; /* Offset into pagecache page */ |
3893 | unsigned long bytes; /* Bytes to write to page */ |
3894 | size_t copied; /* Bytes copied from user */ |
3895 | void *fsdata = NULL; |
3896 | |
3897 | offset = (pos & (PAGE_SIZE - 1)); |
3898 | bytes = min_t(unsigned long, PAGE_SIZE - offset, |
3899 | iov_iter_count(i)); |
3900 | |
3901 | again: |
3902 | /* |
3903 | * Bring in the user page that we will copy from _first_. |
3904 | * Otherwise there's a nasty deadlock on copying from the |
3905 | * same page as we're writing to, without it being marked |
3906 | * up-to-date. |
3907 | */ |
3908 | if (unlikely(fault_in_iov_iter_readable(i, bytes) == bytes)) { |
3909 | status = -EFAULT; |
3910 | break; |
3911 | } |
3912 | |
3913 | if (fatal_signal_pending(current)) { |
3914 | status = -EINTR; |
3915 | break; |
3916 | } |
3917 | |
3918 | status = a_ops->write_begin(file, mapping, pos, bytes, |
3919 | &page, &fsdata); |
3920 | if (unlikely(status < 0)) |
3921 | break; |
3922 | |
3923 | if (mapping_writably_mapped(mapping)) |
3924 | flush_dcache_page(page); |
3925 | |
3926 | copied = copy_page_from_iter_atomic(page, offset, bytes, i); |
3927 | flush_dcache_page(page); |
3928 | |
3929 | status = a_ops->write_end(file, mapping, pos, bytes, copied, |
3930 | page, fsdata); |
3931 | if (unlikely(status != copied)) { |
3932 | iov_iter_revert(i, bytes: copied - max(status, 0L)); |
3933 | if (unlikely(status < 0)) |
3934 | break; |
3935 | } |
3936 | cond_resched(); |
3937 | |
3938 | if (unlikely(status == 0)) { |
3939 | /* |
3940 | * A short copy made ->write_end() reject the |
3941 | * thing entirely. Might be memory poisoning |
3942 | * halfway through, might be a race with munmap, |
3943 | * might be severe memory pressure. |
3944 | */ |
3945 | if (copied) |
3946 | bytes = copied; |
3947 | goto again; |
3948 | } |
3949 | pos += status; |
3950 | written += status; |
3951 | |
3952 | balance_dirty_pages_ratelimited(mapping); |
3953 | } while (iov_iter_count(i)); |
3954 | |
3955 | if (!written) |
3956 | return status; |
3957 | iocb->ki_pos += written; |
3958 | return written; |
3959 | } |
3960 | EXPORT_SYMBOL(generic_perform_write); |
3961 | |
3962 | /** |
3963 | * __generic_file_write_iter - write data to a file |
3964 | * @iocb: IO state structure (file, offset, etc.) |
3965 | * @from: iov_iter with data to write |
3966 | * |
3967 | * This function does all the work needed for actually writing data to a |
3968 | * file. It does all basic checks, removes SUID from the file, updates |
3969 | * modification times and calls proper subroutines depending on whether we |
3970 | * do direct IO or a standard buffered write. |
3971 | * |
3972 | * It expects i_rwsem to be grabbed unless we work on a block device or similar |
3973 | * object which does not need locking at all. |
3974 | * |
3975 | * This function does *not* take care of syncing data in case of O_SYNC write. |
3976 | * A caller has to handle it. This is mainly due to the fact that we want to |
3977 | * avoid syncing under i_rwsem. |
3978 | * |
3979 | * Return: |
3980 | * * number of bytes written, even for truncated writes |
3981 | * * negative error code if no data has been written at all |
3982 | */ |
3983 | ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) |
3984 | { |
3985 | struct file *file = iocb->ki_filp; |
3986 | struct address_space *mapping = file->f_mapping; |
3987 | struct inode *inode = mapping->host; |
3988 | ssize_t ret; |
3989 | |
3990 | ret = file_remove_privs(file); |
3991 | if (ret) |
3992 | return ret; |
3993 | |
3994 | ret = file_update_time(file); |
3995 | if (ret) |
3996 | return ret; |
3997 | |
3998 | if (iocb->ki_flags & IOCB_DIRECT) { |
3999 | ret = generic_file_direct_write(iocb, from); |
4000 | /* |
4001 | * If the write stopped short of completing, fall back to |
4002 | * buffered writes. Some filesystems do this for writes to |
4003 | * holes, for example. For DAX files, a buffered write will |
4004 | * not succeed (even if it did, DAX does not handle dirty |
4005 | * page-cache pages correctly). |
4006 | */ |
4007 | if (ret < 0 || !iov_iter_count(i: from) || IS_DAX(inode)) |
4008 | return ret; |
4009 | return direct_write_fallback(iocb, iter: from, direct_written: ret, |
4010 | buffered_written: generic_perform_write(iocb, from)); |
4011 | } |
4012 | |
4013 | return generic_perform_write(iocb, from); |
4014 | } |
4015 | EXPORT_SYMBOL(__generic_file_write_iter); |
4016 | |
4017 | /** |
4018 | * generic_file_write_iter - write data to a file |
4019 | * @iocb: IO state structure |
4020 | * @from: iov_iter with data to write |
4021 | * |
4022 | * This is a wrapper around __generic_file_write_iter() to be used by most |
4023 | * filesystems. It takes care of syncing the file in case of O_SYNC file |
4024 | * and acquires i_rwsem as needed. |
4025 | * Return: |
4026 | * * negative error code if no data has been written at all of |
4027 | * vfs_fsync_range() failed for a synchronous write |
4028 | * * number of bytes written, even for truncated writes |
4029 | */ |
4030 | ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) |
4031 | { |
4032 | struct file *file = iocb->ki_filp; |
4033 | struct inode *inode = file->f_mapping->host; |
4034 | ssize_t ret; |
4035 | |
4036 | inode_lock(inode); |
4037 | ret = generic_write_checks(iocb, from); |
4038 | if (ret > 0) |
4039 | ret = __generic_file_write_iter(iocb, from); |
4040 | inode_unlock(inode); |
4041 | |
4042 | if (ret > 0) |
4043 | ret = generic_write_sync(iocb, count: ret); |
4044 | return ret; |
4045 | } |
4046 | EXPORT_SYMBOL(generic_file_write_iter); |
4047 | |
4048 | /** |
4049 | * filemap_release_folio() - Release fs-specific metadata on a folio. |
4050 | * @folio: The folio which the kernel is trying to free. |
4051 | * @gfp: Memory allocation flags (and I/O mode). |
4052 | * |
4053 | * The address_space is trying to release any data attached to a folio |
4054 | * (presumably at folio->private). |
4055 | * |
4056 | * This will also be called if the private_2 flag is set on a page, |
4057 | * indicating that the folio has other metadata associated with it. |
4058 | * |
4059 | * The @gfp argument specifies whether I/O may be performed to release |
4060 | * this page (__GFP_IO), and whether the call may block |
4061 | * (__GFP_RECLAIM & __GFP_FS). |
4062 | * |
4063 | * Return: %true if the release was successful, otherwise %false. |
4064 | */ |
4065 | bool filemap_release_folio(struct folio *folio, gfp_t gfp) |
4066 | { |
4067 | struct address_space * const mapping = folio->mapping; |
4068 | |
4069 | BUG_ON(!folio_test_locked(folio)); |
4070 | if (!folio_needs_release(folio)) |
4071 | return true; |
4072 | if (folio_test_writeback(folio)) |
4073 | return false; |
4074 | |
4075 | if (mapping && mapping->a_ops->release_folio) |
4076 | return mapping->a_ops->release_folio(folio, gfp); |
4077 | return try_to_free_buffers(folio); |
4078 | } |
4079 | EXPORT_SYMBOL(filemap_release_folio); |
4080 | |
4081 | #ifdef CONFIG_CACHESTAT_SYSCALL |
4082 | /** |
4083 | * filemap_cachestat() - compute the page cache statistics of a mapping |
4084 | * @mapping: The mapping to compute the statistics for. |
4085 | * @first_index: The starting page cache index. |
4086 | * @last_index: The final page index (inclusive). |
4087 | * @cs: the cachestat struct to write the result to. |
4088 | * |
4089 | * This will query the page cache statistics of a mapping in the |
4090 | * page range of [first_index, last_index] (inclusive). The statistics |
4091 | * queried include: number of dirty pages, number of pages marked for |
4092 | * writeback, and the number of (recently) evicted pages. |
4093 | */ |
4094 | static void filemap_cachestat(struct address_space *mapping, |
4095 | pgoff_t first_index, pgoff_t last_index, struct cachestat *cs) |
4096 | { |
4097 | XA_STATE(xas, &mapping->i_pages, first_index); |
4098 | struct folio *folio; |
4099 | |
4100 | rcu_read_lock(); |
4101 | xas_for_each(&xas, folio, last_index) { |
4102 | unsigned long nr_pages; |
4103 | pgoff_t folio_first_index, folio_last_index; |
4104 | |
4105 | if (xas_retry(xas: &xas, entry: folio)) |
4106 | continue; |
4107 | |
4108 | if (xa_is_value(entry: folio)) { |
4109 | /* page is evicted */ |
4110 | void *shadow = (void *)folio; |
4111 | bool workingset; /* not used */ |
4112 | int order = xa_get_order(xas.xa, index: xas.xa_index); |
4113 | |
4114 | nr_pages = 1 << order; |
4115 | folio_first_index = round_down(xas.xa_index, 1 << order); |
4116 | folio_last_index = folio_first_index + nr_pages - 1; |
4117 | |
4118 | /* Folios might straddle the range boundaries, only count covered pages */ |
4119 | if (folio_first_index < first_index) |
4120 | nr_pages -= first_index - folio_first_index; |
4121 | |
4122 | if (folio_last_index > last_index) |
4123 | nr_pages -= folio_last_index - last_index; |
4124 | |
4125 | cs->nr_evicted += nr_pages; |
4126 | |
4127 | #ifdef CONFIG_SWAP /* implies CONFIG_MMU */ |
4128 | if (shmem_mapping(mapping)) { |
4129 | /* shmem file - in swap cache */ |
4130 | swp_entry_t swp = radix_to_swp_entry(arg: folio); |
4131 | |
4132 | shadow = get_shadow_from_swap_cache(entry: swp); |
4133 | } |
4134 | #endif |
4135 | if (workingset_test_recent(shadow, file: true, workingset: &workingset)) |
4136 | cs->nr_recently_evicted += nr_pages; |
4137 | |
4138 | goto resched; |
4139 | } |
4140 | |
4141 | nr_pages = folio_nr_pages(folio); |
4142 | folio_first_index = folio_pgoff(folio); |
4143 | folio_last_index = folio_first_index + nr_pages - 1; |
4144 | |
4145 | /* Folios might straddle the range boundaries, only count covered pages */ |
4146 | if (folio_first_index < first_index) |
4147 | nr_pages -= first_index - folio_first_index; |
4148 | |
4149 | if (folio_last_index > last_index) |
4150 | nr_pages -= folio_last_index - last_index; |
4151 | |
4152 | /* page is in cache */ |
4153 | cs->nr_cache += nr_pages; |
4154 | |
4155 | if (folio_test_dirty(folio)) |
4156 | cs->nr_dirty += nr_pages; |
4157 | |
4158 | if (folio_test_writeback(folio)) |
4159 | cs->nr_writeback += nr_pages; |
4160 | |
4161 | resched: |
4162 | if (need_resched()) { |
4163 | xas_pause(&xas); |
4164 | cond_resched_rcu(); |
4165 | } |
4166 | } |
4167 | rcu_read_unlock(); |
4168 | } |
4169 | |
4170 | /* |
4171 | * The cachestat(2) system call. |
4172 | * |
4173 | * cachestat() returns the page cache statistics of a file in the |
4174 | * bytes range specified by `off` and `len`: number of cached pages, |
4175 | * number of dirty pages, number of pages marked for writeback, |
4176 | * number of evicted pages, and number of recently evicted pages. |
4177 | * |
4178 | * An evicted page is a page that is previously in the page cache |
4179 | * but has been evicted since. A page is recently evicted if its last |
4180 | * eviction was recent enough that its reentry to the cache would |
4181 | * indicate that it is actively being used by the system, and that |
4182 | * there is memory pressure on the system. |
4183 | * |
4184 | * `off` and `len` must be non-negative integers. If `len` > 0, |
4185 | * the queried range is [`off`, `off` + `len`]. If `len` == 0, |
4186 | * we will query in the range from `off` to the end of the file. |
4187 | * |
4188 | * The `flags` argument is unused for now, but is included for future |
4189 | * extensibility. User should pass 0 (i.e no flag specified). |
4190 | * |
4191 | * Currently, hugetlbfs is not supported. |
4192 | * |
4193 | * Because the status of a page can change after cachestat() checks it |
4194 | * but before it returns to the application, the returned values may |
4195 | * contain stale information. |
4196 | * |
4197 | * return values: |
4198 | * zero - success |
4199 | * -EFAULT - cstat or cstat_range points to an illegal address |
4200 | * -EINVAL - invalid flags |
4201 | * -EBADF - invalid file descriptor |
4202 | * -EOPNOTSUPP - file descriptor is of a hugetlbfs file |
4203 | */ |
4204 | SYSCALL_DEFINE4(cachestat, unsigned int, fd, |
4205 | struct cachestat_range __user *, cstat_range, |
4206 | struct cachestat __user *, cstat, unsigned int, flags) |
4207 | { |
4208 | struct fd f = fdget(fd); |
4209 | struct address_space *mapping; |
4210 | struct cachestat_range csr; |
4211 | struct cachestat cs; |
4212 | pgoff_t first_index, last_index; |
4213 | |
4214 | if (!f.file) |
4215 | return -EBADF; |
4216 | |
4217 | if (copy_from_user(to: &csr, from: cstat_range, |
4218 | n: sizeof(struct cachestat_range))) { |
4219 | fdput(fd: f); |
4220 | return -EFAULT; |
4221 | } |
4222 | |
4223 | /* hugetlbfs is not supported */ |
4224 | if (is_file_hugepages(file: f.file)) { |
4225 | fdput(fd: f); |
4226 | return -EOPNOTSUPP; |
4227 | } |
4228 | |
4229 | if (flags != 0) { |
4230 | fdput(fd: f); |
4231 | return -EINVAL; |
4232 | } |
4233 | |
4234 | first_index = csr.off >> PAGE_SHIFT; |
4235 | last_index = |
4236 | csr.len == 0 ? ULONG_MAX : (csr.off + csr.len - 1) >> PAGE_SHIFT; |
4237 | memset(&cs, 0, sizeof(struct cachestat)); |
4238 | mapping = f.file->f_mapping; |
4239 | filemap_cachestat(mapping, first_index, last_index, cs: &cs); |
4240 | fdput(fd: f); |
4241 | |
4242 | if (copy_to_user(to: cstat, from: &cs, n: sizeof(struct cachestat))) |
4243 | return -EFAULT; |
4244 | |
4245 | return 0; |
4246 | } |
4247 | #endif /* CONFIG_CACHESTAT_SYSCALL */ |
4248 | |