1	// SPDX-License-Identifier: GPL-2.0
2
3	#include <linux/bitops.h>
4	#include <linux/slab.h>
5	#include <linux/bio.h>
6	#include <linux/mm.h>
7	#include <linux/pagemap.h>
8	#include <linux/page-flags.h>
9	#include <linux/sched/mm.h>
10	#include <linux/spinlock.h>
11	#include <linux/blkdev.h>
12	#include <linux/swap.h>
13	#include <linux/writeback.h>
14	#include <linux/pagevec.h>
15	#include <linux/prefetch.h>
16	#include <linux/fsverity.h>
17	#include "extent_io.h"
18	#include "extent-io-tree.h"
19	#include "extent_map.h"
20	#include "ctree.h"
21	#include "btrfs_inode.h"
22	#include "bio.h"
23	#include "locking.h"
24	#include "backref.h"
25	#include "disk-io.h"
26	#include "subpage.h"
27	#include "zoned.h"
28	#include "block-group.h"
29	#include "compression.h"
30	#include "fs.h"
31	#include "accessors.h"
32	#include "file-item.h"
33	#include "file.h"
34	#include "dev-replace.h"
35	#include "super.h"
36	#include "transaction.h"
37
38	static struct kmem_cache *extent_buffer_cache;
39
40	#ifdef CONFIG_BTRFS_DEBUG
41	static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb)
42	{
43	struct btrfs_fs_info *fs_info = eb->fs_info;
44	unsigned long flags;
45
46	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
47	list_add(new: &eb->leak_list, head: &fs_info->allocated_ebs);
48	spin_unlock_irqrestore(lock: &fs_info->eb_leak_lock, flags);
49	}
50
51	static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb)
52	{
53	struct btrfs_fs_info *fs_info = eb->fs_info;
54	unsigned long flags;
55
56	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
57	list_del(entry: &eb->leak_list);
58	spin_unlock_irqrestore(lock: &fs_info->eb_leak_lock, flags);
59	}
60
61	void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
62	{
63	struct extent_buffer *eb;
64	unsigned long flags;
65
66	/*
67	* If we didn't get into open_ctree our allocated_ebs will not be
68	* initialized, so just skip this.
69	*/
70	if (!fs_info->allocated_ebs.next)
71	return;
72
73	WARN_ON(!list_empty(&fs_info->allocated_ebs));
74	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
75	while (!list_empty(head: &fs_info->allocated_ebs)) {
76	eb = list_first_entry(&fs_info->allocated_ebs,
77	struct extent_buffer, leak_list);
78	pr_err(
79	"BTRFS: buffer leak start %llu len %u refs %d bflags %lu owner %llu\n",
80	eb->start, eb->len, atomic_read(&eb->refs), eb->bflags,
81	btrfs_header_owner(eb));
82	list_del(entry: &eb->leak_list);
83	WARN_ON_ONCE(1);
84	kmem_cache_free(s: extent_buffer_cache, objp: eb);
85	}
86	spin_unlock_irqrestore(lock: &fs_info->eb_leak_lock, flags);
87	}
88	#else
89	#define btrfs_leak_debug_add_eb(eb) do {} while (0)
90	#define btrfs_leak_debug_del_eb(eb) do {} while (0)
91	#endif
92
93	/*
94	* Structure to record info about the bio being assembled, and other info like
95	* how many bytes are there before stripe/ordered extent boundary.
96	*/
97	struct btrfs_bio_ctrl {
98	struct btrfs_bio *bbio;
99	enum btrfs_compression_type compress_type;
100	u32 len_to_oe_boundary;
101	blk_opf_t opf;
102	btrfs_bio_end_io_t end_io_func;
103	struct writeback_control *wbc;
104	};
105
106	static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl)
107	{
108	struct btrfs_bio *bbio = bio_ctrl->bbio;
109
110	if (!bbio)
111	return;
112
113	/* Caller should ensure the bio has at least some range added */
114	ASSERT(bbio->bio.bi_iter.bi_size);
115
116	if (btrfs_op(bio: &bbio->bio) == BTRFS_MAP_READ &&
117	bio_ctrl->compress_type != BTRFS_COMPRESS_NONE)
118	btrfs_submit_compressed_read(bbio);
119	else
120	btrfs_submit_bio(bbio, mirror_num: 0);
121
122	/* The bbio is owned by the end_io handler now */
123	bio_ctrl->bbio = NULL;
124	}
125
126	/*
127	* Submit or fail the current bio in the bio_ctrl structure.
128	*/
129	static void submit_write_bio(struct btrfs_bio_ctrl *bio_ctrl, int ret)
130	{
131	struct btrfs_bio *bbio = bio_ctrl->bbio;
132
133	if (!bbio)
134	return;
135
136	if (ret) {
137	ASSERT(ret < 0);
138	btrfs_bio_end_io(bbio, status: errno_to_blk_status(errno: ret));
139	/* The bio is owned by the end_io handler now */
140	bio_ctrl->bbio = NULL;
141	} else {
142	submit_one_bio(bio_ctrl);
143	}
144	}
145
146	int __init extent_buffer_init_cachep(void)
147	{
148	extent_buffer_cache = kmem_cache_create(name: "btrfs_extent_buffer",
149	size: sizeof(struct extent_buffer), align: 0, flags: 0,
150	NULL);
151	if (!extent_buffer_cache)
152	return -ENOMEM;
153
154	return 0;
155	}
156
157	void __cold extent_buffer_free_cachep(void)
158	{
159	/*
160	* Make sure all delayed rcu free are flushed before we
161	* destroy caches.
162	*/
163	rcu_barrier();
164	kmem_cache_destroy(s: extent_buffer_cache);
165	}
166
167	void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
168	{
169	unsigned long index = start >> PAGE_SHIFT;
170	unsigned long end_index = end >> PAGE_SHIFT;
171	struct page *page;
172
173	while (index <= end_index) {
174	page = find_get_page(mapping: inode->i_mapping, offset: index);
175	BUG_ON(!page); /* Pages should be in the extent_io_tree */
176	clear_page_dirty_for_io(page);
177	put_page(page);
178	index++;
179	}
180	}
181
182	static void process_one_page(struct btrfs_fs_info *fs_info,
183	struct page *page, struct page *locked_page,
184	unsigned long page_ops, u64 start, u64 end)
185	{
186	struct folio *folio = page_folio(page);
187	u32 len;
188
189	ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX);
190	len = end + 1 - start;
191
192	if (page_ops & PAGE_SET_ORDERED)
193	btrfs_folio_clamp_set_ordered(fs_info, folio, start, len);
194	if (page_ops & PAGE_START_WRITEBACK) {
195	btrfs_folio_clamp_clear_dirty(fs_info, folio, start, len);
196	btrfs_folio_clamp_set_writeback(fs_info, folio, start, len);
197	}
198	if (page_ops & PAGE_END_WRITEBACK)
199	btrfs_folio_clamp_clear_writeback(fs_info, folio, start, len);
200
201	if (page != locked_page && (page_ops & PAGE_UNLOCK))
202	btrfs_folio_end_writer_lock(fs_info, folio, start, len);
203	}
204
205	static void __process_pages_contig(struct address_space *mapping,
206	struct page *locked_page, u64 start, u64 end,
207	unsigned long page_ops)
208	{
209	struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host);
210	pgoff_t start_index = start >> PAGE_SHIFT;
211	pgoff_t end_index = end >> PAGE_SHIFT;
212	pgoff_t index = start_index;
213	struct folio_batch fbatch;
214	int i;
215
216	folio_batch_init(fbatch: &fbatch);
217	while (index <= end_index) {
218	int found_folios;
219
220	found_folios = filemap_get_folios_contig(mapping, start: &index,
221	end: end_index, fbatch: &fbatch);
222	for (i = 0; i < found_folios; i++) {
223	struct folio *folio = fbatch.folios[i];
224
225	process_one_page(fs_info, page: &folio->page, locked_page,
226	page_ops, start, end);
227	}
228	folio_batch_release(fbatch: &fbatch);
229	cond_resched();
230	}
231	}
232
233	static noinline void __unlock_for_delalloc(struct inode *inode,
234	struct page *locked_page,
235	u64 start, u64 end)
236	{
237	unsigned long index = start >> PAGE_SHIFT;
238	unsigned long end_index = end >> PAGE_SHIFT;
239
240	ASSERT(locked_page);
241	if (index == locked_page->index && end_index == index)
242	return;
243
244	__process_pages_contig(mapping: inode->i_mapping, locked_page, start, end,
245	page_ops: PAGE_UNLOCK);
246	}
247
248	static noinline int lock_delalloc_pages(struct inode *inode,
249	struct page *locked_page,
250	u64 start,
251	u64 end)
252	{
253	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
254	struct address_space *mapping = inode->i_mapping;
255	pgoff_t start_index = start >> PAGE_SHIFT;
256	pgoff_t end_index = end >> PAGE_SHIFT;
257	pgoff_t index = start_index;
258	u64 processed_end = start;
259	struct folio_batch fbatch;
260
261	if (index == locked_page->index && index == end_index)
262	return 0;
263
264	folio_batch_init(fbatch: &fbatch);
265	while (index <= end_index) {
266	unsigned int found_folios, i;
267
268	found_folios = filemap_get_folios_contig(mapping, start: &index,
269	end: end_index, fbatch: &fbatch);
270	if (found_folios == 0)
271	goto out;
272
273	for (i = 0; i < found_folios; i++) {
274	struct folio *folio = fbatch.folios[i];
275	struct page *page = folio_page(folio, 0);
276	u32 len = end + 1 - start;
277
278	if (page == locked_page)
279	continue;
280
281	if (btrfs_folio_start_writer_lock(fs_info, folio, start,
282	len))
283	goto out;
284
285	if (!PageDirty(page) || page->mapping != mapping) {
286	btrfs_folio_end_writer_lock(fs_info, folio, start,
287	len);
288	goto out;
289	}
290
291	processed_end = page_offset(page) + PAGE_SIZE - 1;
292	}
293	folio_batch_release(fbatch: &fbatch);
294	cond_resched();
295	}
296
297	return 0;
298	out:
299	folio_batch_release(fbatch: &fbatch);
300	if (processed_end > start)
301	__unlock_for_delalloc(inode, locked_page, start, end: processed_end);
302	return -EAGAIN;
303	}
304
305	/*
306	* Find and lock a contiguous range of bytes in the file marked as delalloc, no
307	* more than @max_bytes.
308	*
309	* @start: The original start bytenr to search.
310	* Will store the extent range start bytenr.
311	* @end: The original end bytenr of the search range
312	* Will store the extent range end bytenr.
313	*
314	* Return true if we find a delalloc range which starts inside the original
315	* range, and @start/@end will store the delalloc range start/end.
316	*
317	* Return false if we can't find any delalloc range which starts inside the
318	* original range, and @start/@end will be the non-delalloc range start/end.
319	*/
320	EXPORT_FOR_TESTS
321	noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
322	struct page *locked_page, u64 *start,
323	u64 *end)
324	{
325	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
326	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
327	const u64 orig_start = *start;
328	const u64 orig_end = *end;
329	/* The sanity tests may not set a valid fs_info. */
330	u64 max_bytes = fs_info ? fs_info->max_extent_size : BTRFS_MAX_EXTENT_SIZE;
331	u64 delalloc_start;
332	u64 delalloc_end;
333	bool found;
334	struct extent_state *cached_state = NULL;
335	int ret;
336	int loops = 0;
337
338	/* Caller should pass a valid @end to indicate the search range end */
339	ASSERT(orig_end > orig_start);
340
341	/* The range should at least cover part of the page */
342	ASSERT(!(orig_start >= page_offset(locked_page) + PAGE_SIZE ||
343	orig_end <= page_offset(locked_page)));
344	again:
345	/* step one, find a bunch of delalloc bytes starting at start */
346	delalloc_start = *start;
347	delalloc_end = 0;
348	found = btrfs_find_delalloc_range(tree, start: &delalloc_start, end: &delalloc_end,
349	max_bytes, cached_state: &cached_state);
350	if (!found || delalloc_end <= *start || delalloc_start > orig_end) {
351	*start = delalloc_start;
352
353	/* @delalloc_end can be -1, never go beyond @orig_end */
354	*end = min(delalloc_end, orig_end);
355	free_extent_state(state: cached_state);
356	return false;
357	}
358
359	/*
360	* start comes from the offset of locked_page. We have to lock
361	* pages in order, so we can't process delalloc bytes before
362	* locked_page
363	*/
364	if (delalloc_start < *start)
365	delalloc_start = *start;
366
367	/*
368	* make sure to limit the number of pages we try to lock down
369	*/
370	if (delalloc_end + 1 - delalloc_start > max_bytes)
371	delalloc_end = delalloc_start + max_bytes - 1;
372
373	/* step two, lock all the pages after the page that has start */
374	ret = lock_delalloc_pages(inode, locked_page,
375	start: delalloc_start, end: delalloc_end);
376	ASSERT(!ret || ret == -EAGAIN);
377	if (ret == -EAGAIN) {
378	/* some of the pages are gone, lets avoid looping by
379	* shortening the size of the delalloc range we're searching
380	*/
381	free_extent_state(state: cached_state);
382	cached_state = NULL;
383	if (!loops) {
384	max_bytes = PAGE_SIZE;
385	loops = 1;
386	goto again;
387	} else {
388	found = false;
389	goto out_failed;
390	}
391	}
392
393	/* step three, lock the state bits for the whole range */
394	lock_extent(tree, start: delalloc_start, end: delalloc_end, cached: &cached_state);
395
396	/* then test to make sure it is all still delalloc */
397	ret = test_range_bit(tree, start: delalloc_start, end: delalloc_end,
398	bit: EXTENT_DELALLOC, cached_state);
399	if (!ret) {
400	unlock_extent(tree, start: delalloc_start, end: delalloc_end,
401	cached: &cached_state);
402	__unlock_for_delalloc(inode, locked_page,
403	start: delalloc_start, end: delalloc_end);
404	cond_resched();
405	goto again;
406	}
407	free_extent_state(state: cached_state);
408	*start = delalloc_start;
409	*end = delalloc_end;
410	out_failed:
411	return found;
412	}
413
414	void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
415	struct page *locked_page,
416	u32 clear_bits, unsigned long page_ops)
417	{
418	clear_extent_bit(tree: &inode->io_tree, start, end, bits: clear_bits, NULL);
419
420	__process_pages_contig(mapping: inode->vfs_inode.i_mapping, locked_page,
421	start, end, page_ops);
422	}
423
424	static bool btrfs_verify_page(struct page *page, u64 start)
425	{
426	if (!fsverity_active(inode: page->mapping->host) ||
427	PageUptodate(page) ||
428	start >= i_size_read(inode: page->mapping->host))
429	return true;
430	return fsverity_verify_page(page);
431	}
432
433	static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len)
434	{
435	struct btrfs_fs_info *fs_info = page_to_fs_info(page);
436	struct folio *folio = page_folio(page);
437
438	ASSERT(page_offset(page) <= start &&
439	start + len <= page_offset(page) + PAGE_SIZE);
440
441	if (uptodate && btrfs_verify_page(page, start))
442	btrfs_folio_set_uptodate(fs_info, folio, start, len);
443	else
444	btrfs_folio_clear_uptodate(fs_info, folio, start, len);
445
446	if (!btrfs_is_subpage(fs_info, mapping: page->mapping))
447	unlock_page(page);
448	else
449	btrfs_subpage_end_reader(fs_info, folio, start, len);
450	}
451
452	/*
453	* After a write IO is done, we need to:
454	*
455	* - clear the uptodate bits on error
456	* - clear the writeback bits in the extent tree for the range
457	* - filio_end_writeback() if there is no more pending io for the folio
458	*
459	* Scheduling is not allowed, so the extent state tree is expected
460	* to have one and only one object corresponding to this IO.
461	*/
462	static void end_bbio_data_write(struct btrfs_bio *bbio)
463	{
464	struct btrfs_fs_info *fs_info = bbio->fs_info;
465	struct bio *bio = &bbio->bio;
466	int error = blk_status_to_errno(status: bio->bi_status);
467	struct folio_iter fi;
468	const u32 sectorsize = fs_info->sectorsize;
469
470	ASSERT(!bio_flagged(bio, BIO_CLONED));
471	bio_for_each_folio_all(fi, bio) {
472	struct folio *folio = fi.folio;
473	u64 start = folio_pos(folio) + fi.offset;
474	u32 len = fi.length;
475
476	/* Only order 0 (single page) folios are allowed for data. */
477	ASSERT(folio_order(folio) == 0);
478
479	/* Our read/write should always be sector aligned. */
480	if (!IS_ALIGNED(fi.offset, sectorsize))
481	btrfs_err(fs_info,
482	"partial page write in btrfs with offset %zu and length %zu",
483	fi.offset, fi.length);
484	else if (!IS_ALIGNED(fi.length, sectorsize))
485	btrfs_info(fs_info,
486	"incomplete page write with offset %zu and length %zu",
487	fi.offset, fi.length);
488
489	btrfs_finish_ordered_extent(ordered: bbio->ordered,
490	folio_page(folio, 0), file_offset: start, len, uptodate: !error);
491	if (error)
492	mapping_set_error(mapping: folio->mapping, error);
493	btrfs_folio_clear_writeback(fs_info, folio, start, len);
494	}
495
496	bio_put(bio);
497	}
498
499	/*
500	* Record previously processed extent range
501	*
502	* For endio_readpage_release_extent() to handle a full extent range, reducing
503	* the extent io operations.
504	*/
505	struct processed_extent {
506	struct btrfs_inode *inode;
507	/* Start of the range in @inode */
508	u64 start;
509	/* End of the range in @inode */
510	u64 end;
511	bool uptodate;
512	};
513
514	/*
515	* Try to release processed extent range
516	*
517	* May not release the extent range right now if the current range is
518	* contiguous to processed extent.
519	*
520	* Will release processed extent when any of @inode, @uptodate, the range is
521	* no longer contiguous to the processed range.
522	*
523	* Passing @inode == NULL will force processed extent to be released.
524	*/
525	static void endio_readpage_release_extent(struct processed_extent *processed,
526	struct btrfs_inode *inode, u64 start, u64 end,
527	bool uptodate)
528	{
529	struct extent_state *cached = NULL;
530	struct extent_io_tree *tree;
531
532	/* The first extent, initialize @processed */
533	if (!processed->inode)
534	goto update;
535
536	/*
537	* Contiguous to processed extent, just uptodate the end.
538	*
539	* Several things to notice:
540	*
541	* - bio can be merged as long as on-disk bytenr is contiguous
542	* This means we can have page belonging to other inodes, thus need to
543	* check if the inode still matches.
544	* - bvec can contain range beyond current page for multi-page bvec
545	* Thus we need to do processed->end + 1 >= start check
546	*/
547	if (processed->inode == inode && processed->uptodate == uptodate &&
548	processed->end + 1 >= start && end >= processed->end) {
549	processed->end = end;
550	return;
551	}
552
553	tree = &processed->inode->io_tree;
554	/*
555	* Now we don't have range contiguous to the processed range, release
556	* the processed range now.
557	*/
558	unlock_extent(tree, start: processed->start, end: processed->end, cached: &cached);
559
560	update:
561	/* Update processed to current range */
562	processed->inode = inode;
563	processed->start = start;
564	processed->end = end;
565	processed->uptodate = uptodate;
566	}
567
568	static void begin_page_read(struct btrfs_fs_info *fs_info, struct page *page)
569	{
570	struct folio *folio = page_folio(page);
571
572	ASSERT(folio_test_locked(folio));
573	if (!btrfs_is_subpage(fs_info, mapping: folio->mapping))
574	return;
575
576	ASSERT(folio_test_private(folio));
577	btrfs_subpage_start_reader(fs_info, folio, start: page_offset(page), PAGE_SIZE);
578	}
579
580	/*
581	* After a data read IO is done, we need to:
582	*
583	* - clear the uptodate bits on error
584	* - set the uptodate bits if things worked
585	* - set the folio up to date if all extents in the tree are uptodate
586	* - clear the lock bit in the extent tree
587	* - unlock the folio if there are no other extents locked for it
588	*
589	* Scheduling is not allowed, so the extent state tree is expected
590	* to have one and only one object corresponding to this IO.
591	*/
592	static void end_bbio_data_read(struct btrfs_bio *bbio)
593	{
594	struct btrfs_fs_info *fs_info = bbio->fs_info;
595	struct bio *bio = &bbio->bio;
596	struct processed_extent processed = { 0 };
597	struct folio_iter fi;
598	const u32 sectorsize = fs_info->sectorsize;
599
600	ASSERT(!bio_flagged(bio, BIO_CLONED));
601	bio_for_each_folio_all(fi, &bbio->bio) {
602	bool uptodate = !bio->bi_status;
603	struct folio *folio = fi.folio;
604	struct inode *inode = folio->mapping->host;
605	u64 start;
606	u64 end;
607	u32 len;
608
609	/* For now only order 0 folios are supported for data. */
610	ASSERT(folio_order(folio) == 0);
611	btrfs_debug(fs_info,
612	"%s: bi_sector=%llu, err=%d, mirror=%u",
613	__func__, bio->bi_iter.bi_sector, bio->bi_status,
614	bbio->mirror_num);
615
616	/*
617	* We always issue full-sector reads, but if some block in a
618	* folio fails to read, blk_update_request() will advance
619	* bv_offset and adjust bv_len to compensate. Print a warning
620	* for unaligned offsets, and an error if they don't add up to
621	* a full sector.
622	*/
623	if (!IS_ALIGNED(fi.offset, sectorsize))
624	btrfs_err(fs_info,
625	"partial page read in btrfs with offset %zu and length %zu",
626	fi.offset, fi.length);
627	else if (!IS_ALIGNED(fi.offset + fi.length, sectorsize))
628	btrfs_info(fs_info,
629	"incomplete page read with offset %zu and length %zu",
630	fi.offset, fi.length);
631
632	start = folio_pos(folio) + fi.offset;
633	end = start + fi.length - 1;
634	len = fi.length;
635
636	if (likely(uptodate)) {
637	loff_t i_size = i_size_read(inode);
638	pgoff_t end_index = i_size >> folio_shift(folio);
639
640	/*
641	* Zero out the remaining part if this range straddles
642	* i_size.
643	*
644	* Here we should only zero the range inside the folio,
645	* not touch anything else.
646	*
647	* NOTE: i_size is exclusive while end is inclusive.
648	*/
649	if (folio_index(folio) == end_index && i_size <= end) {
650	u32 zero_start = max(offset_in_folio(folio, i_size),
651	offset_in_folio(folio, start));
652	u32 zero_len = offset_in_folio(folio, end) + 1 -
653	zero_start;
654
655	folio_zero_range(folio, start: zero_start, length: zero_len);
656	}
657	}
658
659	/* Update page status and unlock. */
660	end_page_read(folio_page(folio, 0), uptodate, start, len);
661	endio_readpage_release_extent(processed: &processed, inode: BTRFS_I(inode),
662	start, end, uptodate);
663	}
664	/* Release the last extent */
665	endio_readpage_release_extent(processed: &processed, NULL, start: 0, end: 0, uptodate: false);
666	bio_put(bio);
667	}
668
669	/*
670	* Populate every free slot in a provided array with pages.
671	*
672	* @nr_pages: number of pages to allocate
673	* @page_array: the array to fill with pages; any existing non-null entries in
674	* the array will be skipped
675	* @extra_gfp: the extra GFP flags for the allocation.
676	*
677	* Return: 0 if all pages were able to be allocated;
678	* -ENOMEM otherwise, the partially allocated pages would be freed and
679	* the array slots zeroed
680	*/
681	int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array,
682	gfp_t extra_gfp)
683	{
684	const gfp_t gfp = GFP_NOFS | extra_gfp;
685	unsigned int allocated;
686
687	for (allocated = 0; allocated < nr_pages;) {
688	unsigned int last = allocated;
689
690	allocated = alloc_pages_bulk_array(gfp, nr_pages, page_array);
691	if (unlikely(allocated == last)) {
692	/* No progress, fail and do cleanup. */
693	for (int i = 0; i < allocated; i++) {
694	__free_page(page_array[i]);
695	page_array[i] = NULL;
696	}
697	return -ENOMEM;
698	}
699	}
700	return 0;
701	}
702
703	/*
704	* Populate needed folios for the extent buffer.
705	*
706	* For now, the folios populated are always in order 0 (aka, single page).
707	*/
708	static int alloc_eb_folio_array(struct extent_buffer *eb, gfp_t extra_gfp)
709	{
710	struct page *page_array[INLINE_EXTENT_BUFFER_PAGES] = { 0 };
711	int num_pages = num_extent_pages(eb);
712	int ret;
713
714	ret = btrfs_alloc_page_array(nr_pages: num_pages, page_array, extra_gfp);
715	if (ret < 0)
716	return ret;
717
718	for (int i = 0; i < num_pages; i++)
719	eb->folios[i] = page_folio(page_array[i]);
720	eb->folio_size = PAGE_SIZE;
721	eb->folio_shift = PAGE_SHIFT;
722	return 0;
723	}
724
725	static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl,
726	struct page *page, u64 disk_bytenr,
727	unsigned int pg_offset)
728	{
729	struct bio *bio = &bio_ctrl->bbio->bio;
730	struct bio_vec *bvec = bio_last_bvec_all(bio);
731	const sector_t sector = disk_bytenr >> SECTOR_SHIFT;
732
733	if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) {
734	/*
735	* For compression, all IO should have its logical bytenr set
736	* to the starting bytenr of the compressed extent.
737	*/
738	return bio->bi_iter.bi_sector == sector;
739	}
740
741	/*
742	* The contig check requires the following conditions to be met:
743	*
744	* 1) The pages are belonging to the same inode
745	* This is implied by the call chain.
746	*
747	* 2) The range has adjacent logical bytenr
748	*
749	* 3) The range has adjacent file offset
750	* This is required for the usage of btrfs_bio->file_offset.
751	*/
752	return bio_end_sector(bio) == sector &&
753	page_offset(page: bvec->bv_page) + bvec->bv_offset + bvec->bv_len ==
754	page_offset(page) + pg_offset;
755	}
756
757	static void alloc_new_bio(struct btrfs_inode *inode,
758	struct btrfs_bio_ctrl *bio_ctrl,
759	u64 disk_bytenr, u64 file_offset)
760	{
761	struct btrfs_fs_info *fs_info = inode->root->fs_info;
762	struct btrfs_bio *bbio;
763
764	bbio = btrfs_bio_alloc(BIO_MAX_VECS, opf: bio_ctrl->opf, fs_info,
765	end_io: bio_ctrl->end_io_func, NULL);
766	bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
767	bbio->inode = inode;
768	bbio->file_offset = file_offset;
769	bio_ctrl->bbio = bbio;
770	bio_ctrl->len_to_oe_boundary = U32_MAX;
771
772	/* Limit data write bios to the ordered boundary. */
773	if (bio_ctrl->wbc) {
774	struct btrfs_ordered_extent *ordered;
775
776	ordered = btrfs_lookup_ordered_extent(inode, file_offset);
777	if (ordered) {
778	bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX,
779	ordered->file_offset +
780	ordered->disk_num_bytes - file_offset);
781	bbio->ordered = ordered;
782	}
783
784	/*
785	* Pick the last added device to support cgroup writeback. For
786	* multi-device file systems this means blk-cgroup policies have
787	* to always be set on the last added/replaced device.
788	* This is a bit odd but has been like that for a long time.
789	*/
790	bio_set_dev(bio: &bbio->bio, bdev: fs_info->fs_devices->latest_dev->bdev);
791	wbc_init_bio(wbc: bio_ctrl->wbc, bio: &bbio->bio);
792	}
793	}
794
795	/*
796	* @disk_bytenr: logical bytenr where the write will be
797	* @page: page to add to the bio
798	* @size: portion of page that we want to write to
799	* @pg_offset: offset of the new bio or to check whether we are adding
800	* a contiguous page to the previous one
801	*
802	* The will either add the page into the existing @bio_ctrl->bbio, or allocate a
803	* new one in @bio_ctrl->bbio.
804	* The mirror number for this IO should already be initizlied in
805	* @bio_ctrl->mirror_num.
806	*/
807	static void submit_extent_page(struct btrfs_bio_ctrl *bio_ctrl,
808	u64 disk_bytenr, struct page *page,
809	size_t size, unsigned long pg_offset)
810	{
811	struct btrfs_inode *inode = page_to_inode(page);
812
813	ASSERT(pg_offset + size <= PAGE_SIZE);
814	ASSERT(bio_ctrl->end_io_func);
815
816	if (bio_ctrl->bbio &&
817	!btrfs_bio_is_contig(bio_ctrl, page, disk_bytenr, pg_offset))
818	submit_one_bio(bio_ctrl);
819
820	do {
821	u32 len = size;
822
823	/* Allocate new bio if needed */
824	if (!bio_ctrl->bbio) {
825	alloc_new_bio(inode, bio_ctrl, disk_bytenr,
826	file_offset: page_offset(page) + pg_offset);
827	}
828
829	/* Cap to the current ordered extent boundary if there is one. */
830	if (len > bio_ctrl->len_to_oe_boundary) {
831	ASSERT(bio_ctrl->compress_type == BTRFS_COMPRESS_NONE);
832	ASSERT(is_data_inode(&inode->vfs_inode));
833	len = bio_ctrl->len_to_oe_boundary;
834	}
835
836	if (bio_add_page(bio: &bio_ctrl->bbio->bio, page, len, off: pg_offset) != len) {
837	/* bio full: move on to a new one */
838	submit_one_bio(bio_ctrl);
839	continue;
840	}
841
842	if (bio_ctrl->wbc)
843	wbc_account_cgroup_owner(wbc: bio_ctrl->wbc, page, bytes: len);
844
845	size -= len;
846	pg_offset += len;
847	disk_bytenr += len;
848
849	/*
850	* len_to_oe_boundary defaults to U32_MAX, which isn't page or
851	* sector aligned. alloc_new_bio() then sets it to the end of
852	* our ordered extent for writes into zoned devices.
853	*
854	* When len_to_oe_boundary is tracking an ordered extent, we
855	* trust the ordered extent code to align things properly, and
856	* the check above to cap our write to the ordered extent
857	* boundary is correct.
858	*
859	* When len_to_oe_boundary is U32_MAX, the cap above would
860	* result in a 4095 byte IO for the last page right before
861	* we hit the bio limit of UINT_MAX. bio_add_page() has all
862	* the checks required to make sure we don't overflow the bio,
863	* and we should just ignore len_to_oe_boundary completely
864	* unless we're using it to track an ordered extent.
865	*
866	* It's pretty hard to make a bio sized U32_MAX, but it can
867	* happen when the page cache is able to feed us contiguous
868	* pages for large extents.
869	*/
870	if (bio_ctrl->len_to_oe_boundary != U32_MAX)
871	bio_ctrl->len_to_oe_boundary -= len;
872
873	/* Ordered extent boundary: move on to a new bio. */
874	if (bio_ctrl->len_to_oe_boundary == 0)
875	submit_one_bio(bio_ctrl);
876	} while (size);
877	}
878
879	static int attach_extent_buffer_folio(struct extent_buffer *eb,
880	struct folio *folio,
881	struct btrfs_subpage *prealloc)
882	{
883	struct btrfs_fs_info *fs_info = eb->fs_info;
884	int ret = 0;
885
886	/*
887	* If the page is mapped to btree inode, we should hold the private
888	* lock to prevent race.
889	* For cloned or dummy extent buffers, their pages are not mapped and
890	* will not race with any other ebs.
891	*/
892	if (folio->mapping)
893	lockdep_assert_held(&folio->mapping->i_private_lock);
894
895	if (fs_info->nodesize >= PAGE_SIZE) {
896	if (!folio_test_private(folio))
897	folio_attach_private(folio, data: eb);
898	else
899	WARN_ON(folio_get_private(folio) != eb);
900	return 0;
901	}
902
903	/* Already mapped, just free prealloc */
904	if (folio_test_private(folio)) {
905	btrfs_free_subpage(subpage: prealloc);
906	return 0;
907	}
908
909	if (prealloc)
910	/* Has preallocated memory for subpage */
911	folio_attach_private(folio, data: prealloc);
912	else
913	/* Do new allocation to attach subpage */
914	ret = btrfs_attach_subpage(fs_info, folio, type: BTRFS_SUBPAGE_METADATA);
915	return ret;
916	}
917
918	int set_page_extent_mapped(struct page *page)
919	{
920	return set_folio_extent_mapped(page_folio(page));
921	}
922
923	int set_folio_extent_mapped(struct folio *folio)
924	{
925	struct btrfs_fs_info *fs_info;
926
927	ASSERT(folio->mapping);
928
929	if (folio_test_private(folio))
930	return 0;
931
932	fs_info = folio_to_fs_info(folio);
933
934	if (btrfs_is_subpage(fs_info, mapping: folio->mapping))
935	return btrfs_attach_subpage(fs_info, folio, type: BTRFS_SUBPAGE_DATA);
936
937	folio_attach_private(folio, data: (void *)EXTENT_FOLIO_PRIVATE);
938	return 0;
939	}
940
941	void clear_page_extent_mapped(struct page *page)
942	{
943	struct folio *folio = page_folio(page);
944	struct btrfs_fs_info *fs_info;
945
946	ASSERT(page->mapping);
947
948	if (!folio_test_private(folio))
949	return;
950
951	fs_info = page_to_fs_info(page);
952	if (btrfs_is_subpage(fs_info, mapping: page->mapping))
953	return btrfs_detach_subpage(fs_info, folio);
954
955	folio_detach_private(folio);
956	}
957
958	static struct extent_map *__get_extent_map(struct inode *inode, struct page *page,
959	u64 start, u64 len, struct extent_map **em_cached)
960	{
961	struct extent_map *em;
962
963	ASSERT(em_cached);
964
965	if (*em_cached) {
966	em = *em_cached;
967	if (extent_map_in_tree(em) && start >= em->start &&
968	start < extent_map_end(em)) {
969	refcount_inc(r: &em->refs);
970	return em;
971	}
972
973	free_extent_map(em);
974	*em_cached = NULL;
975	}
976
977	em = btrfs_get_extent(inode: BTRFS_I(inode), page, start, len);
978	if (!IS_ERR(ptr: em)) {
979	BUG_ON(*em_cached);
980	refcount_inc(r: &em->refs);
981	*em_cached = em;
982	}
983	return em;
984	}
985	/*
986	* basic readpage implementation. Locked extent state structs are inserted
987	* into the tree that are removed when the IO is done (by the end_io
988	* handlers)
989	* XXX JDM: This needs looking at to ensure proper page locking
990	* return 0 on success, otherwise return error
991	*/
992	static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
993	struct btrfs_bio_ctrl *bio_ctrl, u64 *prev_em_start)
994	{
995	struct inode *inode = page->mapping->host;
996	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
997	u64 start = page_offset(page);
998	const u64 end = start + PAGE_SIZE - 1;
999	u64 cur = start;
1000	u64 extent_offset;
1001	u64 last_byte = i_size_read(inode);
1002	u64 block_start;
1003	struct extent_map *em;
1004	int ret = 0;
1005	size_t pg_offset = 0;
1006	size_t iosize;
1007	size_t blocksize = fs_info->sectorsize;
1008	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
1009
1010	ret = set_page_extent_mapped(page);
1011	if (ret < 0) {
1012	unlock_extent(tree, start, end, NULL);
1013	unlock_page(page);
1014	return ret;
1015	}
1016
1017	if (page->index == last_byte >> PAGE_SHIFT) {
1018	size_t zero_offset = offset_in_page(last_byte);
1019
1020	if (zero_offset) {
1021	iosize = PAGE_SIZE - zero_offset;
1022	memzero_page(page, offset: zero_offset, len: iosize);
1023	}
1024	}
1025	bio_ctrl->end_io_func = end_bbio_data_read;
1026	begin_page_read(fs_info, page);
1027	while (cur <= end) {
1028	enum btrfs_compression_type compress_type = BTRFS_COMPRESS_NONE;
1029	bool force_bio_submit = false;
1030	u64 disk_bytenr;
1031
1032	ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
1033	if (cur >= last_byte) {
1034	iosize = PAGE_SIZE - pg_offset;
1035	memzero_page(page, offset: pg_offset, len: iosize);
1036	unlock_extent(tree, start: cur, end: cur + iosize - 1, NULL);
1037	end_page_read(page, uptodate: true, start: cur, len: iosize);
1038	break;
1039	}
1040	em = __get_extent_map(inode, page, start: cur, len: end - cur + 1, em_cached);
1041	if (IS_ERR(ptr: em)) {
1042	unlock_extent(tree, start: cur, end, NULL);
1043	end_page_read(page, uptodate: false, start: cur, len: end + 1 - cur);
1044	return PTR_ERR(ptr: em);
1045	}
1046	extent_offset = cur - em->start;
1047	BUG_ON(extent_map_end(em) <= cur);
1048	BUG_ON(end < cur);
1049
1050	compress_type = extent_map_compression(em);
1051
1052	iosize = min(extent_map_end(em) - cur, end - cur + 1);
1053	iosize = ALIGN(iosize, blocksize);
1054	if (compress_type != BTRFS_COMPRESS_NONE)
1055	disk_bytenr = em->block_start;
1056	else
1057	disk_bytenr = em->block_start + extent_offset;
1058	block_start = em->block_start;
1059	if (em->flags & EXTENT_FLAG_PREALLOC)
1060	block_start = EXTENT_MAP_HOLE;
1061
1062	/*
1063	* If we have a file range that points to a compressed extent
1064	* and it's followed by a consecutive file range that points
1065	* to the same compressed extent (possibly with a different
1066	* offset and/or length, so it either points to the whole extent
1067	* or only part of it), we must make sure we do not submit a
1068	* single bio to populate the pages for the 2 ranges because
1069	* this makes the compressed extent read zero out the pages
1070	* belonging to the 2nd range. Imagine the following scenario:
1071	*
1072	* File layout
1073	* [0 - 8K] [8K - 24K]
1074	* | |
1075	* | |
1076	* points to extent X, points to extent X,
1077	* offset 4K, length of 8K offset 0, length 16K
1078	*
1079	* [extent X, compressed length = 4K uncompressed length = 16K]
1080	*
1081	* If the bio to read the compressed extent covers both ranges,
1082	* it will decompress extent X into the pages belonging to the
1083	* first range and then it will stop, zeroing out the remaining
1084	* pages that belong to the other range that points to extent X.
1085	* So here we make sure we submit 2 bios, one for the first
1086	* range and another one for the third range. Both will target
1087	* the same physical extent from disk, but we can't currently
1088	* make the compressed bio endio callback populate the pages
1089	* for both ranges because each compressed bio is tightly
1090	* coupled with a single extent map, and each range can have
1091	* an extent map with a different offset value relative to the
1092	* uncompressed data of our extent and different lengths. This
1093	* is a corner case so we prioritize correctness over
1094	* non-optimal behavior (submitting 2 bios for the same extent).
1095	*/
1096	if (compress_type != BTRFS_COMPRESS_NONE &&
1097	prev_em_start && *prev_em_start != (u64)-1 &&
1098	*prev_em_start != em->start)
1099	force_bio_submit = true;
1100
1101	if (prev_em_start)
1102	*prev_em_start = em->start;
1103
1104	free_extent_map(em);
1105	em = NULL;
1106
1107	/* we've found a hole, just zero and go on */
1108	if (block_start == EXTENT_MAP_HOLE) {
1109	memzero_page(page, offset: pg_offset, len: iosize);
1110
1111	unlock_extent(tree, start: cur, end: cur + iosize - 1, NULL);
1112	end_page_read(page, uptodate: true, start: cur, len: iosize);
1113	cur = cur + iosize;
1114	pg_offset += iosize;
1115	continue;
1116	}
1117	/* the get_extent function already copied into the page */
1118	if (block_start == EXTENT_MAP_INLINE) {
1119	unlock_extent(tree, start: cur, end: cur + iosize - 1, NULL);
1120	end_page_read(page, uptodate: true, start: cur, len: iosize);
1121	cur = cur + iosize;
1122	pg_offset += iosize;
1123	continue;
1124	}
1125
1126	if (bio_ctrl->compress_type != compress_type) {
1127	submit_one_bio(bio_ctrl);
1128	bio_ctrl->compress_type = compress_type;
1129	}
1130
1131	if (force_bio_submit)
1132	submit_one_bio(bio_ctrl);
1133	submit_extent_page(bio_ctrl, disk_bytenr, page, size: iosize,
1134	pg_offset);
1135	cur = cur + iosize;
1136	pg_offset += iosize;
1137	}
1138
1139	return 0;
1140	}
1141
1142	int btrfs_read_folio(struct file *file, struct folio *folio)
1143	{
1144	struct page *page = &folio->page;
1145	struct btrfs_inode *inode = page_to_inode(page);
1146	u64 start = page_offset(page);
1147	u64 end = start + PAGE_SIZE - 1;
1148	struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ };
1149	struct extent_map *em_cached = NULL;
1150	int ret;
1151
1152	btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
1153
1154	ret = btrfs_do_readpage(page, em_cached: &em_cached, bio_ctrl: &bio_ctrl, NULL);
1155	free_extent_map(em: em_cached);
1156
1157	/*
1158	* If btrfs_do_readpage() failed we will want to submit the assembled
1159	* bio to do the cleanup.
1160	*/
1161	submit_one_bio(bio_ctrl: &bio_ctrl);
1162	return ret;
1163	}
1164
1165	static inline void contiguous_readpages(struct page *pages[], int nr_pages,
1166	u64 start, u64 end,
1167	struct extent_map **em_cached,
1168	struct btrfs_bio_ctrl *bio_ctrl,
1169	u64 *prev_em_start)
1170	{
1171	struct btrfs_inode *inode = page_to_inode(pages[0]);
1172	int index;
1173
1174	ASSERT(em_cached);
1175
1176	btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
1177
1178	for (index = 0; index < nr_pages; index++) {
1179	btrfs_do_readpage(page: pages[index], em_cached, bio_ctrl,
1180	prev_em_start);
1181	put_page(page: pages[index]);
1182	}
1183	}
1184
1185	/*
1186	* helper for __extent_writepage, doing all of the delayed allocation setup.
1187	*
1188	* This returns 1 if btrfs_run_delalloc_range function did all the work required
1189	* to write the page (copy into inline extent). In this case the IO has
1190	* been started and the page is already unlocked.
1191	*
1192	* This returns 0 if all went well (page still locked)
1193	* This returns < 0 if there were errors (page still locked)
1194	*/
1195	static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
1196	struct page *page, struct writeback_control *wbc)
1197	{
1198	const u64 page_start = page_offset(page);
1199	const u64 page_end = page_start + PAGE_SIZE - 1;
1200	u64 delalloc_start = page_start;
1201	u64 delalloc_end = page_end;
1202	u64 delalloc_to_write = 0;
1203	int ret = 0;
1204
1205	while (delalloc_start < page_end) {
1206	delalloc_end = page_end;
1207	if (!find_lock_delalloc_range(inode: &inode->vfs_inode, locked_page: page,
1208	start: &delalloc_start, end: &delalloc_end)) {
1209	delalloc_start = delalloc_end + 1;
1210	continue;
1211	}
1212
1213	ret = btrfs_run_delalloc_range(inode, locked_page: page, start: delalloc_start,
1214	end: delalloc_end, wbc);
1215	if (ret < 0)
1216	return ret;
1217
1218	delalloc_start = delalloc_end + 1;
1219	}
1220
1221	/*
1222	* delalloc_end is already one less than the total length, so
1223	* we don't subtract one from PAGE_SIZE
1224	*/
1225	delalloc_to_write +=
1226	DIV_ROUND_UP(delalloc_end + 1 - page_start, PAGE_SIZE);
1227
1228	/*
1229	* If btrfs_run_dealloc_range() already started I/O and unlocked
1230	* the pages, we just need to account for them here.
1231	*/
1232	if (ret == 1) {
1233	wbc->nr_to_write -= delalloc_to_write;
1234	return 1;
1235	}
1236
1237	if (wbc->nr_to_write < delalloc_to_write) {
1238	int thresh = 8192;
1239
1240	if (delalloc_to_write < thresh * 2)
1241	thresh = delalloc_to_write;
1242	wbc->nr_to_write = min_t(u64, delalloc_to_write,
1243	thresh);
1244	}
1245
1246	return 0;
1247	}
1248
1249	/*
1250	* Find the first byte we need to write.
1251	*
1252	* For subpage, one page can contain several sectors, and
1253	* __extent_writepage_io() will just grab all extent maps in the page
1254	* range and try to submit all non-inline/non-compressed extents.
1255	*
1256	* This is a big problem for subpage, we shouldn't re-submit already written
1257	* data at all.
1258	* This function will lookup subpage dirty bit to find which range we really
1259	* need to submit.
1260	*
1261	* Return the next dirty range in [@start, @end).
1262	* If no dirty range is found, @start will be page_offset(page) + PAGE_SIZE.
1263	*/
1264	static void find_next_dirty_byte(struct btrfs_fs_info *fs_info,
1265	struct page *page, u64 *start, u64 *end)
1266	{
1267	struct folio *folio = page_folio(page);
1268	struct btrfs_subpage *subpage = folio_get_private(folio);
1269	struct btrfs_subpage_info *spi = fs_info->subpage_info;
1270	u64 orig_start = *start;
1271	/* Declare as unsigned long so we can use bitmap ops */
1272	unsigned long flags;
1273	int range_start_bit;
1274	int range_end_bit;
1275
1276	/*
1277	* For regular sector size == page size case, since one page only
1278	* contains one sector, we return the page offset directly.
1279	*/
1280	if (!btrfs_is_subpage(fs_info, mapping: page->mapping)) {
1281	*start = page_offset(page);
1282	*end = page_offset(page) + PAGE_SIZE;
1283	return;
1284	}
1285
1286	range_start_bit = spi->dirty_offset +
1287	(offset_in_page(orig_start) >> fs_info->sectorsize_bits);
1288
1289	/* We should have the page locked, but just in case */
1290	spin_lock_irqsave(&subpage->lock, flags);
1291	bitmap_next_set_region(bitmap: subpage->bitmaps, rs: &range_start_bit, re: &range_end_bit,
1292	end: spi->dirty_offset + spi->bitmap_nr_bits);
1293	spin_unlock_irqrestore(lock: &subpage->lock, flags);
1294
1295	range_start_bit -= spi->dirty_offset;
1296	range_end_bit -= spi->dirty_offset;
1297
1298	*start = page_offset(page) + range_start_bit * fs_info->sectorsize;
1299	*end = page_offset(page) + range_end_bit * fs_info->sectorsize;
1300	}
1301
1302	/*
1303	* helper for __extent_writepage. This calls the writepage start hooks,
1304	* and does the loop to map the page into extents and bios.
1305	*
1306	* We return 1 if the IO is started and the page is unlocked,
1307	* 0 if all went well (page still locked)
1308	* < 0 if there were errors (page still locked)
1309	*/
1310	static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode,
1311	struct page *page,
1312	struct btrfs_bio_ctrl *bio_ctrl,
1313	loff_t i_size,
1314	int *nr_ret)
1315	{
1316	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1317	u64 cur = page_offset(page);
1318	u64 end = cur + PAGE_SIZE - 1;
1319	u64 extent_offset;
1320	u64 block_start;
1321	struct extent_map *em;
1322	int ret = 0;
1323	int nr = 0;
1324
1325	ret = btrfs_writepage_cow_fixup(page);
1326	if (ret) {
1327	/* Fixup worker will requeue */
1328	redirty_page_for_writepage(bio_ctrl->wbc, page);
1329	unlock_page(page);
1330	return 1;
1331	}
1332
1333	bio_ctrl->end_io_func = end_bbio_data_write;
1334	while (cur <= end) {
1335	u32 len = end - cur + 1;
1336	u64 disk_bytenr;
1337	u64 em_end;
1338	u64 dirty_range_start = cur;
1339	u64 dirty_range_end;
1340	u32 iosize;
1341
1342	if (cur >= i_size) {
1343	btrfs_mark_ordered_io_finished(inode, page, file_offset: cur, num_bytes: len,
1344	uptodate: true);
1345	/*
1346	* This range is beyond i_size, thus we don't need to
1347	* bother writing back.
1348	* But we still need to clear the dirty subpage bit, or
1349	* the next time the page gets dirtied, we will try to
1350	* writeback the sectors with subpage dirty bits,
1351	* causing writeback without ordered extent.
1352	*/
1353	btrfs_folio_clear_dirty(fs_info, page_folio(page), start: cur, len);
1354	break;
1355	}
1356
1357	find_next_dirty_byte(fs_info, page, start: &dirty_range_start,
1358	end: &dirty_range_end);
1359	if (cur < dirty_range_start) {
1360	cur = dirty_range_start;
1361	continue;
1362	}
1363
1364	em = btrfs_get_extent(inode, NULL, start: cur, len);
1365	if (IS_ERR(ptr: em)) {
1366	ret = PTR_ERR_OR_ZERO(ptr: em);
1367	goto out_error;
1368	}
1369
1370	extent_offset = cur - em->start;
1371	em_end = extent_map_end(em);
1372	ASSERT(cur <= em_end);
1373	ASSERT(cur < end);
1374	ASSERT(IS_ALIGNED(em->start, fs_info->sectorsize));
1375	ASSERT(IS_ALIGNED(em->len, fs_info->sectorsize));
1376
1377	block_start = em->block_start;
1378	disk_bytenr = em->block_start + extent_offset;
1379
1380	ASSERT(!extent_map_is_compressed(em));
1381	ASSERT(block_start != EXTENT_MAP_HOLE);
1382	ASSERT(block_start != EXTENT_MAP_INLINE);
1383
1384	/*
1385	* Note that em_end from extent_map_end() and dirty_range_end from
1386	* find_next_dirty_byte() are all exclusive
1387	*/
1388	iosize = min(min(em_end, end + 1), dirty_range_end) - cur;
1389	free_extent_map(em);
1390	em = NULL;
1391
1392	btrfs_set_range_writeback(inode, start: cur, end: cur + iosize - 1);
1393	if (!PageWriteback(page)) {
1394	btrfs_err(inode->root->fs_info,
1395	"page %lu not writeback, cur %llu end %llu",
1396	page->index, cur, end);
1397	}
1398
1399	/*
1400	* Although the PageDirty bit is cleared before entering this
1401	* function, subpage dirty bit is not cleared.
1402	* So clear subpage dirty bit here so next time we won't submit
1403	* page for range already written to disk.
1404	*/
1405	btrfs_folio_clear_dirty(fs_info, page_folio(page), start: cur, len: iosize);
1406
1407	submit_extent_page(bio_ctrl, disk_bytenr, page, size: iosize,
1408	pg_offset: cur - page_offset(page));
1409	cur += iosize;
1410	nr++;
1411	}
1412
1413	btrfs_folio_assert_not_dirty(fs_info, page_folio(page));
1414	*nr_ret = nr;
1415	return 0;
1416
1417	out_error:
1418	/*
1419	* If we finish without problem, we should not only clear page dirty,
1420	* but also empty subpage dirty bits
1421	*/
1422	*nr_ret = nr;
1423	return ret;
1424	}
1425
1426	/*
1427	* the writepage semantics are similar to regular writepage. extent
1428	* records are inserted to lock ranges in the tree, and as dirty areas
1429	* are found, they are marked writeback. Then the lock bits are removed
1430	* and the end_io handler clears the writeback ranges
1431	*
1432	* Return 0 if everything goes well.
1433	* Return <0 for error.
1434	*/
1435	static int __extent_writepage(struct page *page, struct btrfs_bio_ctrl *bio_ctrl)
1436	{
1437	struct folio *folio = page_folio(page);
1438	struct inode *inode = page->mapping->host;
1439	const u64 page_start = page_offset(page);
1440	int ret;
1441	int nr = 0;
1442	size_t pg_offset;
1443	loff_t i_size = i_size_read(inode);
1444	unsigned long end_index = i_size >> PAGE_SHIFT;
1445
1446	trace___extent_writepage(page, inode, wbc: bio_ctrl->wbc);
1447
1448	WARN_ON(!PageLocked(page));
1449
1450	pg_offset = offset_in_page(i_size);
1451	if (page->index > end_index ||
1452	(page->index == end_index && !pg_offset)) {
1453	folio_invalidate(folio, offset: 0, length: folio_size(folio));
1454	folio_unlock(folio);
1455	return 0;
1456	}
1457
1458	if (page->index == end_index)
1459	memzero_page(page, offset: pg_offset, PAGE_SIZE - pg_offset);
1460
1461	ret = set_page_extent_mapped(page);
1462	if (ret < 0)
1463	goto done;
1464
1465	ret = writepage_delalloc(inode: BTRFS_I(inode), page, wbc: bio_ctrl->wbc);
1466	if (ret == 1)
1467	return 0;
1468	if (ret)
1469	goto done;
1470
1471	ret = __extent_writepage_io(inode: BTRFS_I(inode), page, bio_ctrl, i_size, nr_ret: &nr);
1472	if (ret == 1)
1473	return 0;
1474
1475	bio_ctrl->wbc->nr_to_write--;
1476
1477	done:
1478	if (nr == 0) {
1479	/* make sure the mapping tag for page dirty gets cleared */
1480	set_page_writeback(page);
1481	end_page_writeback(page);
1482	}
1483	if (ret) {
1484	btrfs_mark_ordered_io_finished(inode: BTRFS_I(inode), page, file_offset: page_start,
1485	PAGE_SIZE, uptodate: !ret);
1486	mapping_set_error(mapping: page->mapping, error: ret);
1487	}
1488	unlock_page(page);
1489	ASSERT(ret <= 0);
1490	return ret;
1491	}
1492
1493	void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
1494	{
1495	wait_on_bit_io(word: &eb->bflags, bit: EXTENT_BUFFER_WRITEBACK,
1496	TASK_UNINTERRUPTIBLE);
1497	}
1498
1499	/*
1500	* Lock extent buffer status and pages for writeback.
1501	*
1502	* Return %false if the extent buffer doesn't need to be submitted (e.g. the
1503	* extent buffer is not dirty)
1504	* Return %true is the extent buffer is submitted to bio.
1505	*/
1506	static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *eb,
1507	struct writeback_control *wbc)
1508	{
1509	struct btrfs_fs_info *fs_info = eb->fs_info;
1510	bool ret = false;
1511
1512	btrfs_tree_lock(eb);
1513	while (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
1514	btrfs_tree_unlock(eb);
1515	if (wbc->sync_mode != WB_SYNC_ALL)
1516	return false;
1517	wait_on_extent_buffer_writeback(eb);
1518	btrfs_tree_lock(eb);
1519	}
1520
1521	/*
1522	* We need to do this to prevent races in people who check if the eb is
1523	* under IO since we can end up having no IO bits set for a short period
1524	* of time.
1525	*/
1526	spin_lock(lock: &eb->refs_lock);
1527	if (test_and_clear_bit(nr: EXTENT_BUFFER_DIRTY, addr: &eb->bflags)) {
1528	set_bit(nr: EXTENT_BUFFER_WRITEBACK, addr: &eb->bflags);
1529	spin_unlock(lock: &eb->refs_lock);
1530	btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
1531	percpu_counter_add_batch(fbc: &fs_info->dirty_metadata_bytes,
1532	amount: -eb->len,
1533	batch: fs_info->dirty_metadata_batch);
1534	ret = true;
1535	} else {
1536	spin_unlock(lock: &eb->refs_lock);
1537	}
1538	btrfs_tree_unlock(eb);
1539	return ret;
1540	}
1541
1542	static void set_btree_ioerr(struct extent_buffer *eb)
1543	{
1544	struct btrfs_fs_info *fs_info = eb->fs_info;
1545
1546	set_bit(nr: EXTENT_BUFFER_WRITE_ERR, addr: &eb->bflags);
1547
1548	/*
1549	* A read may stumble upon this buffer later, make sure that it gets an
1550	* error and knows there was an error.
1551	*/
1552	clear_bit(nr: EXTENT_BUFFER_UPTODATE, addr: &eb->bflags);
1553
1554	/*
1555	* We need to set the mapping with the io error as well because a write
1556	* error will flip the file system readonly, and then syncfs() will
1557	* return a 0 because we are readonly if we don't modify the err seq for
1558	* the superblock.
1559	*/
1560	mapping_set_error(mapping: eb->fs_info->btree_inode->i_mapping, error: -EIO);
1561
1562	/*
1563	* If writeback for a btree extent that doesn't belong to a log tree
1564	* failed, increment the counter transaction->eb_write_errors.
1565	* We do this because while the transaction is running and before it's
1566	* committing (when we call filemap_fdata[write|wait]_range against
1567	* the btree inode), we might have
1568	* btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
1569	* returns an error or an error happens during writeback, when we're
1570	* committing the transaction we wouldn't know about it, since the pages
1571	* can be no longer dirty nor marked anymore for writeback (if a
1572	* subsequent modification to the extent buffer didn't happen before the
1573	* transaction commit), which makes filemap_fdata[write|wait]_range not
1574	* able to find the pages tagged with SetPageError at transaction
1575	* commit time. So if this happens we must abort the transaction,
1576	* otherwise we commit a super block with btree roots that point to
1577	* btree nodes/leafs whose content on disk is invalid - either garbage
1578	* or the content of some node/leaf from a past generation that got
1579	* cowed or deleted and is no longer valid.
1580	*
1581	* Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
1582	* not be enough - we need to distinguish between log tree extents vs
1583	* non-log tree extents, and the next filemap_fdatawait_range() call
1584	* will catch and clear such errors in the mapping - and that call might
1585	* be from a log sync and not from a transaction commit. Also, checking
1586	* for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
1587	* not done and would not be reliable - the eb might have been released
1588	* from memory and reading it back again means that flag would not be
1589	* set (since it's a runtime flag, not persisted on disk).
1590	*
1591	* Using the flags below in the btree inode also makes us achieve the
1592	* goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
1593	* writeback for all dirty pages and before filemap_fdatawait_range()
1594	* is called, the writeback for all dirty pages had already finished
1595	* with errors - because we were not using AS_EIO/AS_ENOSPC,
1596	* filemap_fdatawait_range() would return success, as it could not know
1597	* that writeback errors happened (the pages were no longer tagged for
1598	* writeback).
1599	*/
1600	switch (eb->log_index) {
1601	case -1:
1602	set_bit(nr: BTRFS_FS_BTREE_ERR, addr: &fs_info->flags);
1603	break;
1604	case 0:
1605	set_bit(nr: BTRFS_FS_LOG1_ERR, addr: &fs_info->flags);
1606	break;
1607	case 1:
1608	set_bit(nr: BTRFS_FS_LOG2_ERR, addr: &fs_info->flags);
1609	break;
1610	default:
1611	BUG(); /* unexpected, logic error */
1612	}
1613	}
1614
1615	/*
1616	* The endio specific version which won't touch any unsafe spinlock in endio
1617	* context.
1618	*/
1619	static struct extent_buffer *find_extent_buffer_nolock(
1620	struct btrfs_fs_info *fs_info, u64 start)
1621	{
1622	struct extent_buffer *eb;
1623
1624	rcu_read_lock();
1625	eb = radix_tree_lookup(&fs_info->buffer_radix,
1626	start >> fs_info->sectorsize_bits);
1627	if (eb && atomic_inc_not_zero(v: &eb->refs)) {
1628	rcu_read_unlock();
1629	return eb;
1630	}
1631	rcu_read_unlock();
1632	return NULL;
1633	}
1634
1635	static void end_bbio_meta_write(struct btrfs_bio *bbio)
1636	{
1637	struct extent_buffer *eb = bbio->private;
1638	struct btrfs_fs_info *fs_info = eb->fs_info;
1639	bool uptodate = !bbio->bio.bi_status;
1640	struct folio_iter fi;
1641	u32 bio_offset = 0;
1642
1643	if (!uptodate)
1644	set_btree_ioerr(eb);
1645
1646	bio_for_each_folio_all(fi, &bbio->bio) {
1647	u64 start = eb->start + bio_offset;
1648	struct folio *folio = fi.folio;
1649	u32 len = fi.length;
1650
1651	btrfs_folio_clear_writeback(fs_info, folio, start, len);
1652	bio_offset += len;
1653	}
1654
1655	clear_bit(nr: EXTENT_BUFFER_WRITEBACK, addr: &eb->bflags);
1656	smp_mb__after_atomic();
1657	wake_up_bit(word: &eb->bflags, bit: EXTENT_BUFFER_WRITEBACK);
1658
1659	bio_put(&bbio->bio);
1660	}
1661
1662	static void prepare_eb_write(struct extent_buffer *eb)
1663	{
1664	u32 nritems;
1665	unsigned long start;
1666	unsigned long end;
1667
1668	clear_bit(nr: EXTENT_BUFFER_WRITE_ERR, addr: &eb->bflags);
1669
1670	/* Set btree blocks beyond nritems with 0 to avoid stale content */
1671	nritems = btrfs_header_nritems(eb);
1672	if (btrfs_header_level(eb) > 0) {
1673	end = btrfs_node_key_ptr_offset(eb, nr: nritems);
1674	memzero_extent_buffer(eb, start: end, len: eb->len - end);
1675	} else {
1676	/*
1677	* Leaf:
1678	* header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
1679	*/
1680	start = btrfs_item_nr_offset(eb, nr: nritems);
1681	end = btrfs_item_nr_offset(eb, nr: 0);
1682	if (nritems == 0)
1683	end += BTRFS_LEAF_DATA_SIZE(info: eb->fs_info);
1684	else
1685	end += btrfs_item_offset(eb, slot: nritems - 1);
1686	memzero_extent_buffer(eb, start, len: end - start);
1687	}
1688	}
1689
1690	static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
1691	struct writeback_control *wbc)
1692	{
1693	struct btrfs_fs_info *fs_info = eb->fs_info;
1694	struct btrfs_bio *bbio;
1695
1696	prepare_eb_write(eb);
1697
1698	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
1699	opf: REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc),
1700	fs_info: eb->fs_info, end_io: end_bbio_meta_write, private: eb);
1701	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
1702	bio_set_dev(bio: &bbio->bio, bdev: fs_info->fs_devices->latest_dev->bdev);
1703	wbc_init_bio(wbc, bio: &bbio->bio);
1704	bbio->inode = BTRFS_I(inode: eb->fs_info->btree_inode);
1705	bbio->file_offset = eb->start;
1706	if (fs_info->nodesize < PAGE_SIZE) {
1707	struct folio *folio = eb->folios[0];
1708	bool ret;
1709
1710	folio_lock(folio);
1711	btrfs_subpage_set_writeback(fs_info, folio, start: eb->start, len: eb->len);
1712	if (btrfs_subpage_clear_and_test_dirty(fs_info, folio, start: eb->start,
1713	len: eb->len)) {
1714	folio_clear_dirty_for_io(folio);
1715	wbc->nr_to_write--;
1716	}
1717	ret = bio_add_folio(bio: &bbio->bio, folio, len: eb->len,
1718	off: eb->start - folio_pos(folio));
1719	ASSERT(ret);
1720	wbc_account_cgroup_owner(wbc, folio_page(folio, 0), bytes: eb->len);
1721	folio_unlock(folio);
1722	} else {
1723	int num_folios = num_extent_folios(eb);
1724
1725	for (int i = 0; i < num_folios; i++) {
1726	struct folio *folio = eb->folios[i];
1727	bool ret;
1728
1729	folio_lock(folio);
1730	folio_clear_dirty_for_io(folio);
1731	folio_start_writeback(folio);
1732	ret = bio_add_folio(bio: &bbio->bio, folio, len: eb->folio_size, off: 0);
1733	ASSERT(ret);
1734	wbc_account_cgroup_owner(wbc, folio_page(folio, 0),
1735	bytes: eb->folio_size);
1736	wbc->nr_to_write -= folio_nr_pages(folio);
1737	folio_unlock(folio);
1738	}
1739	}
1740	btrfs_submit_bio(bbio, mirror_num: 0);
1741	}
1742
1743	/*
1744	* Submit one subpage btree page.
1745	*
1746	* The main difference to submit_eb_page() is:
1747	* - Page locking
1748	* For subpage, we don't rely on page locking at all.
1749	*
1750	* - Flush write bio
1751	* We only flush bio if we may be unable to fit current extent buffers into
1752	* current bio.
1753	*
1754	* Return >=0 for the number of submitted extent buffers.
1755	* Return <0 for fatal error.
1756	*/
1757	static int submit_eb_subpage(struct page *page, struct writeback_control *wbc)
1758	{
1759	struct btrfs_fs_info *fs_info = page_to_fs_info(page);
1760	struct folio *folio = page_folio(page);
1761	int submitted = 0;
1762	u64 page_start = page_offset(page);
1763	int bit_start = 0;
1764	int sectors_per_node = fs_info->nodesize >> fs_info->sectorsize_bits;
1765
1766	/* Lock and write each dirty extent buffers in the range */
1767	while (bit_start < fs_info->subpage_info->bitmap_nr_bits) {
1768	struct btrfs_subpage *subpage = folio_get_private(folio);
1769	struct extent_buffer *eb;
1770	unsigned long flags;
1771	u64 start;
1772
1773	/*
1774	* Take private lock to ensure the subpage won't be detached
1775	* in the meantime.
1776	*/
1777	spin_lock(lock: &page->mapping->i_private_lock);
1778	if (!folio_test_private(folio)) {
1779	spin_unlock(lock: &page->mapping->i_private_lock);
1780	break;
1781	}
1782	spin_lock_irqsave(&subpage->lock, flags);
1783	if (!test_bit(bit_start + fs_info->subpage_info->dirty_offset,
1784	subpage->bitmaps)) {
1785	spin_unlock_irqrestore(lock: &subpage->lock, flags);
1786	spin_unlock(lock: &page->mapping->i_private_lock);
1787	bit_start++;
1788	continue;
1789	}
1790
1791	start = page_start + bit_start * fs_info->sectorsize;
1792	bit_start += sectors_per_node;
1793
1794	/*
1795	* Here we just want to grab the eb without touching extra
1796	* spin locks, so call find_extent_buffer_nolock().
1797	*/
1798	eb = find_extent_buffer_nolock(fs_info, start);
1799	spin_unlock_irqrestore(lock: &subpage->lock, flags);
1800	spin_unlock(lock: &page->mapping->i_private_lock);
1801
1802	/*
1803	* The eb has already reached 0 refs thus find_extent_buffer()
1804	* doesn't return it. We don't need to write back such eb
1805	* anyway.
1806	*/
1807	if (!eb)
1808	continue;
1809
1810	if (lock_extent_buffer_for_io(eb, wbc)) {
1811	write_one_eb(eb, wbc);
1812	submitted++;
1813	}
1814	free_extent_buffer(eb);
1815	}
1816	return submitted;
1817	}
1818
1819	/*
1820	* Submit all page(s) of one extent buffer.
1821	*
1822	* @page: the page of one extent buffer
1823	* @eb_context: to determine if we need to submit this page, if current page
1824	* belongs to this eb, we don't need to submit
1825	*
1826	* The caller should pass each page in their bytenr order, and here we use
1827	* @eb_context to determine if we have submitted pages of one extent buffer.
1828	*
1829	* If we have, we just skip until we hit a new page that doesn't belong to
1830	* current @eb_context.
1831	*
1832	* If not, we submit all the page(s) of the extent buffer.
1833	*
1834	* Return >0 if we have submitted the extent buffer successfully.
1835	* Return 0 if we don't need to submit the page, as it's already submitted by
1836	* previous call.
1837	* Return <0 for fatal error.
1838	*/
1839	static int submit_eb_page(struct page *page, struct btrfs_eb_write_context *ctx)
1840	{
1841	struct writeback_control *wbc = ctx->wbc;
1842	struct address_space *mapping = page->mapping;
1843	struct folio *folio = page_folio(page);
1844	struct extent_buffer *eb;
1845	int ret;
1846
1847	if (!folio_test_private(folio))
1848	return 0;
1849
1850	if (page_to_fs_info(page)->nodesize < PAGE_SIZE)
1851	return submit_eb_subpage(page, wbc);
1852
1853	spin_lock(lock: &mapping->i_private_lock);
1854	if (!folio_test_private(folio)) {
1855	spin_unlock(lock: &mapping->i_private_lock);
1856	return 0;
1857	}
1858
1859	eb = folio_get_private(folio);
1860
1861	/*
1862	* Shouldn't happen and normally this would be a BUG_ON but no point
1863	* crashing the machine for something we can survive anyway.
1864	*/
1865	if (WARN_ON(!eb)) {
1866	spin_unlock(lock: &mapping->i_private_lock);
1867	return 0;
1868	}
1869
1870	if (eb == ctx->eb) {
1871	spin_unlock(lock: &mapping->i_private_lock);
1872	return 0;
1873	}
1874	ret = atomic_inc_not_zero(v: &eb->refs);
1875	spin_unlock(lock: &mapping->i_private_lock);
1876	if (!ret)
1877	return 0;
1878
1879	ctx->eb = eb;
1880
1881	ret = btrfs_check_meta_write_pointer(fs_info: eb->fs_info, ctx);
1882	if (ret) {
1883	if (ret == -EBUSY)
1884	ret = 0;
1885	free_extent_buffer(eb);
1886	return ret;
1887	}
1888
1889	if (!lock_extent_buffer_for_io(eb, wbc)) {
1890	free_extent_buffer(eb);
1891	return 0;
1892	}
1893	/* Implies write in zoned mode. */
1894	if (ctx->zoned_bg) {
1895	/* Mark the last eb in the block group. */
1896	btrfs_schedule_zone_finish_bg(bg: ctx->zoned_bg, eb);
1897	ctx->zoned_bg->meta_write_pointer += eb->len;
1898	}
1899	write_one_eb(eb, wbc);
1900	free_extent_buffer(eb);
1901	return 1;
1902	}
1903
1904	int btree_write_cache_pages(struct address_space *mapping,
1905	struct writeback_control *wbc)
1906	{
1907	struct btrfs_eb_write_context ctx = { .wbc = wbc };
1908	struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host);
1909	int ret = 0;
1910	int done = 0;
1911	int nr_to_write_done = 0;
1912	struct folio_batch fbatch;
1913	unsigned int nr_folios;
1914	pgoff_t index;
1915	pgoff_t end; /* Inclusive */
1916	int scanned = 0;
1917	xa_mark_t tag;
1918
1919	folio_batch_init(fbatch: &fbatch);
1920	if (wbc->range_cyclic) {
1921	index = mapping->writeback_index; /* Start from prev offset */
1922	end = -1;
1923	/*
1924	* Start from the beginning does not need to cycle over the
1925	* range, mark it as scanned.
1926	*/
1927	scanned = (index == 0);
1928	} else {
1929	index = wbc->range_start >> PAGE_SHIFT;
1930	end = wbc->range_end >> PAGE_SHIFT;
1931	scanned = 1;
1932	}
1933	if (wbc->sync_mode == WB_SYNC_ALL)
1934	tag = PAGECACHE_TAG_TOWRITE;
1935	else
1936	tag = PAGECACHE_TAG_DIRTY;
1937	btrfs_zoned_meta_io_lock(fs_info);
1938	retry:
1939	if (wbc->sync_mode == WB_SYNC_ALL)
1940	tag_pages_for_writeback(mapping, start: index, end);
1941	while (!done && !nr_to_write_done && (index <= end) &&
1942	(nr_folios = filemap_get_folios_tag(mapping, start: &index, end,
1943	tag, fbatch: &fbatch))) {
1944	unsigned i;
1945
1946	for (i = 0; i < nr_folios; i++) {
1947	struct folio *folio = fbatch.folios[i];
1948
1949	ret = submit_eb_page(page: &folio->page, ctx: &ctx);
1950	if (ret == 0)
1951	continue;
1952	if (ret < 0) {
1953	done = 1;
1954	break;
1955	}
1956
1957	/*
1958	* the filesystem may choose to bump up nr_to_write.
1959	* We have to make sure to honor the new nr_to_write
1960	* at any time
1961	*/
1962	nr_to_write_done = wbc->nr_to_write <= 0;
1963	}
1964	folio_batch_release(fbatch: &fbatch);
1965	cond_resched();
1966	}
1967	if (!scanned && !done) {
1968	/*
1969	* We hit the last page and there is more work to be done: wrap
1970	* back to the start of the file
1971	*/
1972	scanned = 1;
1973	index = 0;
1974	goto retry;
1975	}
1976	/*
1977	* If something went wrong, don't allow any metadata write bio to be
1978	* submitted.
1979	*
1980	* This would prevent use-after-free if we had dirty pages not
1981	* cleaned up, which can still happen by fuzzed images.
1982	*
1983	* - Bad extent tree
1984	* Allowing existing tree block to be allocated for other trees.
1985	*
1986	* - Log tree operations
1987	* Exiting tree blocks get allocated to log tree, bumps its
1988	* generation, then get cleaned in tree re-balance.
1989	* Such tree block will not be written back, since it's clean,
1990	* thus no WRITTEN flag set.
1991	* And after log writes back, this tree block is not traced by
1992	* any dirty extent_io_tree.
1993	*
1994	* - Offending tree block gets re-dirtied from its original owner
1995	* Since it has bumped generation, no WRITTEN flag, it can be
1996	* reused without COWing. This tree block will not be traced
1997	* by btrfs_transaction::dirty_pages.
1998	*
1999	* Now such dirty tree block will not be cleaned by any dirty
2000	* extent io tree. Thus we don't want to submit such wild eb
2001	* if the fs already has error.
2002	*
2003	* We can get ret > 0 from submit_extent_page() indicating how many ebs
2004	* were submitted. Reset it to 0 to avoid false alerts for the caller.
2005	*/
2006	if (ret > 0)
2007	ret = 0;
2008	if (!ret && BTRFS_FS_ERROR(fs_info))
2009	ret = -EROFS;
2010
2011	if (ctx.zoned_bg)
2012	btrfs_put_block_group(cache: ctx.zoned_bg);
2013	btrfs_zoned_meta_io_unlock(fs_info);
2014	return ret;
2015	}
2016
2017	/*
2018	* Walk the list of dirty pages of the given address space and write all of them.
2019	*
2020	* @mapping: address space structure to write
2021	* @wbc: subtract the number of written pages from *@wbc->nr_to_write
2022	* @bio_ctrl: holds context for the write, namely the bio
2023	*
2024	* If a page is already under I/O, write_cache_pages() skips it, even
2025	* if it's dirty. This is desirable behaviour for memory-cleaning writeback,
2026	* but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
2027	* and msync() need to guarantee that all the data which was dirty at the time
2028	* the call was made get new I/O started against them. If wbc->sync_mode is
2029	* WB_SYNC_ALL then we were called for data integrity and we must wait for
2030	* existing IO to complete.
2031	*/
2032	static int extent_write_cache_pages(struct address_space *mapping,
2033	struct btrfs_bio_ctrl *bio_ctrl)
2034	{
2035	struct writeback_control *wbc = bio_ctrl->wbc;
2036	struct inode *inode = mapping->host;
2037	int ret = 0;
2038	int done = 0;
2039	int nr_to_write_done = 0;
2040	struct folio_batch fbatch;
2041	unsigned int nr_folios;
2042	pgoff_t index;
2043	pgoff_t end; /* Inclusive */
2044	pgoff_t done_index;
2045	int range_whole = 0;
2046	int scanned = 0;
2047	xa_mark_t tag;
2048
2049	/*
2050	* We have to hold onto the inode so that ordered extents can do their
2051	* work when the IO finishes. The alternative to this is failing to add
2052	* an ordered extent if the igrab() fails there and that is a huge pain
2053	* to deal with, so instead just hold onto the inode throughout the
2054	* writepages operation. If it fails here we are freeing up the inode
2055	* anyway and we'd rather not waste our time writing out stuff that is
2056	* going to be truncated anyway.
2057	*/
2058	if (!igrab(inode))
2059	return 0;
2060
2061	folio_batch_init(fbatch: &fbatch);
2062	if (wbc->range_cyclic) {
2063	index = mapping->writeback_index; /* Start from prev offset */
2064	end = -1;
2065	/*
2066	* Start from the beginning does not need to cycle over the
2067	* range, mark it as scanned.
2068	*/
2069	scanned = (index == 0);
2070	} else {
2071	index = wbc->range_start >> PAGE_SHIFT;
2072	end = wbc->range_end >> PAGE_SHIFT;
2073	if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2074	range_whole = 1;
2075	scanned = 1;
2076	}
2077
2078	/*
2079	* We do the tagged writepage as long as the snapshot flush bit is set
2080	* and we are the first one who do the filemap_flush() on this inode.
2081	*
2082	* The nr_to_write == LONG_MAX is needed to make sure other flushers do
2083	* not race in and drop the bit.
2084	*/
2085	if (range_whole && wbc->nr_to_write == LONG_MAX &&
2086	test_and_clear_bit(nr: BTRFS_INODE_SNAPSHOT_FLUSH,
2087	addr: &BTRFS_I(inode)->runtime_flags))
2088	wbc->tagged_writepages = 1;
2089
2090	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2091	tag = PAGECACHE_TAG_TOWRITE;
2092	else
2093	tag = PAGECACHE_TAG_DIRTY;
2094	retry:
2095	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2096	tag_pages_for_writeback(mapping, start: index, end);
2097	done_index = index;
2098	while (!done && !nr_to_write_done && (index <= end) &&
2099	(nr_folios = filemap_get_folios_tag(mapping, start: &index,
2100	end, tag, fbatch: &fbatch))) {
2101	unsigned i;
2102
2103	for (i = 0; i < nr_folios; i++) {
2104	struct folio *folio = fbatch.folios[i];
2105
2106	done_index = folio_next_index(folio);
2107	/*
2108	* At this point we hold neither the i_pages lock nor
2109	* the page lock: the page may be truncated or
2110	* invalidated (changing page->mapping to NULL),
2111	* or even swizzled back from swapper_space to
2112	* tmpfs file mapping
2113	*/
2114	if (!folio_trylock(folio)) {
2115	submit_write_bio(bio_ctrl, ret: 0);
2116	folio_lock(folio);
2117	}
2118
2119	if (unlikely(folio->mapping != mapping)) {
2120	folio_unlock(folio);
2121	continue;
2122	}
2123
2124	if (!folio_test_dirty(folio)) {
2125	/* Someone wrote it for us. */
2126	folio_unlock(folio);
2127	continue;
2128	}
2129
2130	if (wbc->sync_mode != WB_SYNC_NONE) {
2131	if (folio_test_writeback(folio))
2132	submit_write_bio(bio_ctrl, ret: 0);
2133	folio_wait_writeback(folio);
2134	}
2135
2136	if (folio_test_writeback(folio) ||
2137	!folio_clear_dirty_for_io(folio)) {
2138	folio_unlock(folio);
2139	continue;
2140	}
2141
2142	ret = __extent_writepage(page: &folio->page, bio_ctrl);
2143	if (ret < 0) {
2144	done = 1;
2145	break;
2146	}
2147
2148	/*
2149	* The filesystem may choose to bump up nr_to_write.
2150	* We have to make sure to honor the new nr_to_write
2151	* at any time.
2152	*/
2153	nr_to_write_done = (wbc->sync_mode == WB_SYNC_NONE &&
2154	wbc->nr_to_write <= 0);
2155	}
2156	folio_batch_release(fbatch: &fbatch);
2157	cond_resched();
2158	}
2159	if (!scanned && !done) {
2160	/*
2161	* We hit the last page and there is more work to be done: wrap
2162	* back to the start of the file
2163	*/
2164	scanned = 1;
2165	index = 0;
2166
2167	/*
2168	* If we're looping we could run into a page that is locked by a
2169	* writer and that writer could be waiting on writeback for a
2170	* page in our current bio, and thus deadlock, so flush the
2171	* write bio here.
2172	*/
2173	submit_write_bio(bio_ctrl, ret: 0);
2174	goto retry;
2175	}
2176
2177	if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
2178	mapping->writeback_index = done_index;
2179
2180	btrfs_add_delayed_iput(inode: BTRFS_I(inode));
2181	return ret;
2182	}
2183
2184	/*
2185	* Submit the pages in the range to bio for call sites which delalloc range has
2186	* already been ran (aka, ordered extent inserted) and all pages are still
2187	* locked.
2188	*/
2189	void extent_write_locked_range(struct inode *inode, struct page *locked_page,
2190	u64 start, u64 end, struct writeback_control *wbc,
2191	bool pages_dirty)
2192	{
2193	bool found_error = false;
2194	int ret = 0;
2195	struct address_space *mapping = inode->i_mapping;
2196	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
2197	const u32 sectorsize = fs_info->sectorsize;
2198	loff_t i_size = i_size_read(inode);
2199	u64 cur = start;
2200	struct btrfs_bio_ctrl bio_ctrl = {
2201	.wbc = wbc,
2202	.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2203	};
2204
2205	if (wbc->no_cgroup_owner)
2206	bio_ctrl.opf |= REQ_BTRFS_CGROUP_PUNT;
2207
2208	ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(end + 1, sectorsize));
2209
2210	while (cur <= end) {
2211	u64 cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end);
2212	u32 cur_len = cur_end + 1 - cur;
2213	struct page *page;
2214	int nr = 0;
2215
2216	page = find_get_page(mapping, offset: cur >> PAGE_SHIFT);
2217	ASSERT(PageLocked(page));
2218	if (pages_dirty && page != locked_page) {
2219	ASSERT(PageDirty(page));
2220	clear_page_dirty_for_io(page);
2221	}
2222
2223	ret = __extent_writepage_io(inode: BTRFS_I(inode), page, bio_ctrl: &bio_ctrl,
2224	i_size, nr_ret: &nr);
2225	if (ret == 1)
2226	goto next_page;
2227
2228	/* Make sure the mapping tag for page dirty gets cleared. */
2229	if (nr == 0) {
2230	set_page_writeback(page);
2231	end_page_writeback(page);
2232	}
2233	if (ret) {
2234	btrfs_mark_ordered_io_finished(inode: BTRFS_I(inode), page,
2235	file_offset: cur, num_bytes: cur_len, uptodate: !ret);
2236	mapping_set_error(mapping: page->mapping, error: ret);
2237	}
2238	btrfs_folio_unlock_writer(fs_info, page_folio(page), start: cur, len: cur_len);
2239	if (ret < 0)
2240	found_error = true;
2241	next_page:
2242	put_page(page);
2243	cur = cur_end + 1;
2244	}
2245
2246	submit_write_bio(bio_ctrl: &bio_ctrl, ret: found_error ? ret : 0);
2247	}
2248
2249	int extent_writepages(struct address_space *mapping,
2250	struct writeback_control *wbc)
2251	{
2252	struct inode *inode = mapping->host;
2253	int ret = 0;
2254	struct btrfs_bio_ctrl bio_ctrl = {
2255	.wbc = wbc,
2256	.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2257	};
2258
2259	/*
2260	* Allow only a single thread to do the reloc work in zoned mode to
2261	* protect the write pointer updates.
2262	*/
2263	btrfs_zoned_data_reloc_lock(inode: BTRFS_I(inode));
2264	ret = extent_write_cache_pages(mapping, bio_ctrl: &bio_ctrl);
2265	submit_write_bio(bio_ctrl: &bio_ctrl, ret);
2266	btrfs_zoned_data_reloc_unlock(inode: BTRFS_I(inode));
2267	return ret;
2268	}
2269
2270	void extent_readahead(struct readahead_control *rac)
2271	{
2272	struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ | REQ_RAHEAD };
2273	struct page *pagepool[16];
2274	struct extent_map *em_cached = NULL;
2275	u64 prev_em_start = (u64)-1;
2276	int nr;
2277
2278	while ((nr = readahead_page_batch(rac, pagepool))) {
2279	u64 contig_start = readahead_pos(rac);
2280	u64 contig_end = contig_start + readahead_batch_length(rac) - 1;
2281
2282	contiguous_readpages(pages: pagepool, nr_pages: nr, start: contig_start, end: contig_end,
2283	em_cached: &em_cached, bio_ctrl: &bio_ctrl, prev_em_start: &prev_em_start);
2284	}
2285
2286	if (em_cached)
2287	free_extent_map(em: em_cached);
2288	submit_one_bio(bio_ctrl: &bio_ctrl);
2289	}
2290
2291	/*
2292	* basic invalidate_folio code, this waits on any locked or writeback
2293	* ranges corresponding to the folio, and then deletes any extent state
2294	* records from the tree
2295	*/
2296	int extent_invalidate_folio(struct extent_io_tree *tree,
2297	struct folio *folio, size_t offset)
2298	{
2299	struct extent_state *cached_state = NULL;
2300	u64 start = folio_pos(folio);
2301	u64 end = start + folio_size(folio) - 1;
2302	size_t blocksize = folio_to_fs_info(folio)->sectorsize;
2303
2304	/* This function is only called for the btree inode */
2305	ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO);
2306
2307	start += ALIGN(offset, blocksize);
2308	if (start > end)
2309	return 0;
2310
2311	lock_extent(tree, start, end, cached: &cached_state);
2312	folio_wait_writeback(folio);
2313
2314	/*
2315	* Currently for btree io tree, only EXTENT_LOCKED is utilized,
2316	* so here we only need to unlock the extent range to free any
2317	* existing extent state.
2318	*/
2319	unlock_extent(tree, start, end, cached: &cached_state);
2320	return 0;
2321	}
2322
2323	/*
2324	* a helper for release_folio, this tests for areas of the page that
2325	* are locked or under IO and drops the related state bits if it is safe
2326	* to drop the page.
2327	*/
2328	static int try_release_extent_state(struct extent_io_tree *tree,
2329	struct page *page, gfp_t mask)
2330	{
2331	u64 start = page_offset(page);
2332	u64 end = start + PAGE_SIZE - 1;
2333	int ret = 1;
2334
2335	if (test_range_bit_exists(tree, start, end, bit: EXTENT_LOCKED)) {
2336	ret = 0;
2337	} else {
2338	u32 clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM |
2339	EXTENT_DELALLOC_NEW | EXTENT_CTLBITS |
2340	EXTENT_QGROUP_RESERVED);
2341
2342	/*
2343	* At this point we can safely clear everything except the
2344	* locked bit, the nodatasum bit and the delalloc new bit.
2345	* The delalloc new bit will be cleared by ordered extent
2346	* completion.
2347	*/
2348	ret = __clear_extent_bit(tree, start, end, bits: clear_bits, NULL, NULL);
2349
2350	/* if clear_extent_bit failed for enomem reasons,
2351	* we can't allow the release to continue.
2352	*/
2353	if (ret < 0)
2354	ret = 0;
2355	else
2356	ret = 1;
2357	}
2358	return ret;
2359	}
2360
2361	/*
2362	* a helper for release_folio. As long as there are no locked extents
2363	* in the range corresponding to the page, both state records and extent
2364	* map records are removed
2365	*/
2366	int try_release_extent_mapping(struct page *page, gfp_t mask)
2367	{
2368	struct extent_map *em;
2369	u64 start = page_offset(page);
2370	u64 end = start + PAGE_SIZE - 1;
2371	struct btrfs_inode *btrfs_inode = page_to_inode(page);
2372	struct extent_io_tree *tree = &btrfs_inode->io_tree;
2373	struct extent_map_tree *map = &btrfs_inode->extent_tree;
2374
2375	if (gfpflags_allow_blocking(gfp_flags: mask) &&
2376	page->mapping->host->i_size > SZ_16M) {
2377	u64 len;
2378	while (start <= end) {
2379	struct btrfs_fs_info *fs_info;
2380	u64 cur_gen;
2381
2382	len = end - start + 1;
2383	write_lock(&map->lock);
2384	em = lookup_extent_mapping(tree: map, start, len);
2385	if (!em) {
2386	write_unlock(&map->lock);
2387	break;
2388	}
2389	if ((em->flags & EXTENT_FLAG_PINNED) ||
2390	em->start != start) {
2391	write_unlock(&map->lock);
2392	free_extent_map(em);
2393	break;
2394	}
2395	if (test_range_bit_exists(tree, start: em->start,
2396	end: extent_map_end(em) - 1,
2397	bit: EXTENT_LOCKED))
2398	goto next;
2399	/*
2400	* If it's not in the list of modified extents, used
2401	* by a fast fsync, we can remove it. If it's being
2402	* logged we can safely remove it since fsync took an
2403	* extra reference on the em.
2404	*/
2405	if (list_empty(head: &em->list) ||
2406	(em->flags & EXTENT_FLAG_LOGGING))
2407	goto remove_em;
2408	/*
2409	* If it's in the list of modified extents, remove it
2410	* only if its generation is older then the current one,
2411	* in which case we don't need it for a fast fsync.
2412	* Otherwise don't remove it, we could be racing with an
2413	* ongoing fast fsync that could miss the new extent.
2414	*/
2415	fs_info = btrfs_inode->root->fs_info;
2416	spin_lock(lock: &fs_info->trans_lock);
2417	cur_gen = fs_info->generation;
2418	spin_unlock(lock: &fs_info->trans_lock);
2419	if (em->generation >= cur_gen)
2420	goto next;
2421	remove_em:
2422	/*
2423	* We only remove extent maps that are not in the list of
2424	* modified extents or that are in the list but with a
2425	* generation lower then the current generation, so there
2426	* is no need to set the full fsync flag on the inode (it
2427	* hurts the fsync performance for workloads with a data
2428	* size that exceeds or is close to the system's memory).
2429	*/
2430	remove_extent_mapping(tree: map, em);
2431	/* once for the rb tree */
2432	free_extent_map(em);
2433	next:
2434	start = extent_map_end(em);
2435	write_unlock(&map->lock);
2436
2437	/* once for us */
2438	free_extent_map(em);
2439
2440	cond_resched(); /* Allow large-extent preemption. */
2441	}
2442	}
2443	return try_release_extent_state(tree, page, mask);
2444	}
2445
2446	struct btrfs_fiemap_entry {
2447	u64 offset;
2448	u64 phys;
2449	u64 len;
2450	u32 flags;
2451	};
2452
2453	/*
2454	* Indicate the caller of emit_fiemap_extent() that it needs to unlock the file
2455	* range from the inode's io tree, unlock the subvolume tree search path, flush
2456	* the fiemap cache and relock the file range and research the subvolume tree.
2457	* The value here is something negative that can't be confused with a valid
2458	* errno value and different from 1 because that's also a return value from
2459	* fiemap_fill_next_extent() and also it's often used to mean some btree search
2460	* did not find a key, so make it some distinct negative value.
2461	*/
2462	#define BTRFS_FIEMAP_FLUSH_CACHE (-(MAX_ERRNO + 1))
2463
2464	/*
2465	* Used to:
2466	*
2467	* - Cache the next entry to be emitted to the fiemap buffer, so that we can
2468	* merge extents that are contiguous and can be grouped as a single one;
2469	*
2470	* - Store extents ready to be written to the fiemap buffer in an intermediary
2471	* buffer. This intermediary buffer is to ensure that in case the fiemap
2472	* buffer is memory mapped to the fiemap target file, we don't deadlock
2473	* during btrfs_page_mkwrite(). This is because during fiemap we are locking
2474	* an extent range in order to prevent races with delalloc flushing and
2475	* ordered extent completion, which is needed in order to reliably detect
2476	* delalloc in holes and prealloc extents. And this can lead to a deadlock
2477	* if the fiemap buffer is memory mapped to the file we are running fiemap
2478	* against (a silly, useless in practice scenario, but possible) because
2479	* btrfs_page_mkwrite() will try to lock the same extent range.
2480	*/
2481	struct fiemap_cache {
2482	/* An array of ready fiemap entries. */
2483	struct btrfs_fiemap_entry *entries;
2484	/* Number of entries in the entries array. */
2485	int entries_size;
2486	/* Index of the next entry in the entries array to write to. */
2487	int entries_pos;
2488	/*
2489	* Once the entries array is full, this indicates what's the offset for
2490	* the next file extent item we must search for in the inode's subvolume
2491	* tree after unlocking the extent range in the inode's io tree and
2492	* releasing the search path.
2493	*/
2494	u64 next_search_offset;
2495	/*
2496	* This matches struct fiemap_extent_info::fi_mapped_extents, we use it
2497	* to count ourselves emitted extents and stop instead of relying on
2498	* fiemap_fill_next_extent() because we buffer ready fiemap entries at
2499	* the @entries array, and we want to stop as soon as we hit the max
2500	* amount of extents to map, not just to save time but also to make the
2501	* logic at extent_fiemap() simpler.
2502	*/
2503	unsigned int extents_mapped;
2504	/* Fields for the cached extent (unsubmitted, not ready, extent). */
2505	u64 offset;
2506	u64 phys;
2507	u64 len;
2508	u32 flags;
2509	bool cached;
2510	};
2511
2512	static int flush_fiemap_cache(struct fiemap_extent_info *fieinfo,
2513	struct fiemap_cache *cache)
2514	{
2515	for (int i = 0; i < cache->entries_pos; i++) {
2516	struct btrfs_fiemap_entry *entry = &cache->entries[i];
2517	int ret;
2518
2519	ret = fiemap_fill_next_extent(info: fieinfo, logical: entry->offset,
2520	phys: entry->phys, len: entry->len,
2521	flags: entry->flags);
2522	/*
2523	* Ignore 1 (reached max entries) because we keep track of that
2524	* ourselves in emit_fiemap_extent().
2525	*/
2526	if (ret < 0)
2527	return ret;
2528	}
2529	cache->entries_pos = 0;
2530
2531	return 0;
2532	}
2533
2534	/*
2535	* Helper to submit fiemap extent.
2536	*
2537	* Will try to merge current fiemap extent specified by @offset, @phys,
2538	* @len and @flags with cached one.
2539	* And only when we fails to merge, cached one will be submitted as
2540	* fiemap extent.
2541	*
2542	* Return value is the same as fiemap_fill_next_extent().
2543	*/
2544	static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo,
2545	struct fiemap_cache *cache,
2546	u64 offset, u64 phys, u64 len, u32 flags)
2547	{
2548	struct btrfs_fiemap_entry *entry;
2549	u64 cache_end;
2550
2551	/* Set at the end of extent_fiemap(). */
2552	ASSERT((flags & FIEMAP_EXTENT_LAST) == 0);
2553
2554	if (!cache->cached)
2555	goto assign;
2556
2557	/*
2558	* When iterating the extents of the inode, at extent_fiemap(), we may
2559	* find an extent that starts at an offset behind the end offset of the
2560	* previous extent we processed. This happens if fiemap is called
2561	* without FIEMAP_FLAG_SYNC and there are ordered extents completing
2562	* after we had to unlock the file range, release the search path, emit
2563	* the fiemap extents stored in the buffer (cache->entries array) and
2564	* the lock the remainder of the range and re-search the btree.
2565	*
2566	* For example we are in leaf X processing its last item, which is the
2567	* file extent item for file range [512K, 1M[, and after
2568	* btrfs_next_leaf() releases the path, there's an ordered extent that
2569	* completes for the file range [768K, 2M[, and that results in trimming
2570	* the file extent item so that it now corresponds to the file range
2571	* [512K, 768K[ and a new file extent item is inserted for the file
2572	* range [768K, 2M[, which may end up as the last item of leaf X or as
2573	* the first item of the next leaf - in either case btrfs_next_leaf()
2574	* will leave us with a path pointing to the new extent item, for the
2575	* file range [768K, 2M[, since that's the first key that follows the
2576	* last one we processed. So in order not to report overlapping extents
2577	* to user space, we trim the length of the previously cached extent and
2578	* emit it.
2579	*
2580	* Upon calling btrfs_next_leaf() we may also find an extent with an
2581	* offset smaller than or equals to cache->offset, and this happens
2582	* when we had a hole or prealloc extent with several delalloc ranges in
2583	* it, but after btrfs_next_leaf() released the path, delalloc was
2584	* flushed and the resulting ordered extents were completed, so we can
2585	* now have found a file extent item for an offset that is smaller than
2586	* or equals to what we have in cache->offset. We deal with this as
2587	* described below.
2588	*/
2589	cache_end = cache->offset + cache->len;
2590	if (cache_end > offset) {
2591	if (offset == cache->offset) {
2592	/*
2593	* We cached a dealloc range (found in the io tree) for
2594	* a hole or prealloc extent and we have now found a
2595	* file extent item for the same offset. What we have
2596	* now is more recent and up to date, so discard what
2597	* we had in the cache and use what we have just found.
2598	*/
2599	goto assign;
2600	} else if (offset > cache->offset) {
2601	/*
2602	* The extent range we previously found ends after the
2603	* offset of the file extent item we found and that
2604	* offset falls somewhere in the middle of that previous
2605	* extent range. So adjust the range we previously found
2606	* to end at the offset of the file extent item we have
2607	* just found, since this extent is more up to date.
2608	* Emit that adjusted range and cache the file extent
2609	* item we have just found. This corresponds to the case
2610	* where a previously found file extent item was split
2611	* due to an ordered extent completing.
2612	*/
2613	cache->len = offset - cache->offset;
2614	goto emit;
2615	} else {
2616	const u64 range_end = offset + len;
2617
2618	/*
2619	* The offset of the file extent item we have just found
2620	* is behind the cached offset. This means we were
2621	* processing a hole or prealloc extent for which we
2622	* have found delalloc ranges (in the io tree), so what
2623	* we have in the cache is the last delalloc range we
2624	* found while the file extent item we found can be
2625	* either for a whole delalloc range we previously
2626	* emmitted or only a part of that range.
2627	*
2628	* We have two cases here:
2629	*
2630	* 1) The file extent item's range ends at or behind the
2631	* cached extent's end. In this case just ignore the
2632	* current file extent item because we don't want to
2633	* overlap with previous ranges that may have been
2634	* emmitted already;
2635	*
2636	* 2) The file extent item starts behind the currently
2637	* cached extent but its end offset goes beyond the
2638	* end offset of the cached extent. We don't want to
2639	* overlap with a previous range that may have been
2640	* emmitted already, so we emit the currently cached
2641	* extent and then partially store the current file
2642	* extent item's range in the cache, for the subrange
2643	* going the cached extent's end to the end of the
2644	* file extent item.
2645	*/
2646	if (range_end <= cache_end)
2647	return 0;
2648
2649	if (!(flags & (FIEMAP_EXTENT_ENCODED | FIEMAP_EXTENT_DELALLOC)))
2650	phys += cache_end - offset;
2651
2652	offset = cache_end;
2653	len = range_end - cache_end;
2654	goto emit;
2655	}
2656	}
2657
2658	/*
2659	* Only merges fiemap extents if
2660	* 1) Their logical addresses are continuous
2661	*
2662	* 2) Their physical addresses are continuous
2663	* So truly compressed (physical size smaller than logical size)
2664	* extents won't get merged with each other
2665	*
2666	* 3) Share same flags
2667	*/
2668	if (cache->offset + cache->len == offset &&
2669	cache->phys + cache->len == phys &&
2670	cache->flags == flags) {
2671	cache->len += len;
2672	return 0;
2673	}
2674
2675	emit:
2676	/* Not mergeable, need to submit cached one */
2677
2678	if (cache->entries_pos == cache->entries_size) {
2679	/*
2680	* We will need to research for the end offset of the last
2681	* stored extent and not from the current offset, because after
2682	* unlocking the range and releasing the path, if there's a hole
2683	* between that end offset and this current offset, a new extent
2684	* may have been inserted due to a new write, so we don't want
2685	* to miss it.
2686	*/
2687	entry = &cache->entries[cache->entries_size - 1];
2688	cache->next_search_offset = entry->offset + entry->len;
2689	cache->cached = false;
2690
2691	return BTRFS_FIEMAP_FLUSH_CACHE;
2692	}
2693
2694	entry = &cache->entries[cache->entries_pos];
2695	entry->offset = cache->offset;
2696	entry->phys = cache->phys;
2697	entry->len = cache->len;
2698	entry->flags = cache->flags;
2699	cache->entries_pos++;
2700	cache->extents_mapped++;
2701
2702	if (cache->extents_mapped == fieinfo->fi_extents_max) {
2703	cache->cached = false;
2704	return 1;
2705	}
2706	assign:
2707	cache->cached = true;
2708	cache->offset = offset;
2709	cache->phys = phys;
2710	cache->len = len;
2711	cache->flags = flags;
2712
2713	return 0;
2714	}
2715
2716	/*
2717	* Emit last fiemap cache
2718	*
2719	* The last fiemap cache may still be cached in the following case:
2720	* 0 4k 8k
2721	* |<- Fiemap range ->|
2722	* |<------------ First extent ----------->|
2723	*
2724	* In this case, the first extent range will be cached but not emitted.
2725	* So we must emit it before ending extent_fiemap().
2726	*/
2727	static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo,
2728	struct fiemap_cache *cache)
2729	{
2730	int ret;
2731
2732	if (!cache->cached)
2733	return 0;
2734
2735	ret = fiemap_fill_next_extent(info: fieinfo, logical: cache->offset, phys: cache->phys,
2736	len: cache->len, flags: cache->flags);
2737	cache->cached = false;
2738	if (ret > 0)
2739	ret = 0;
2740	return ret;
2741	}
2742
2743	static int fiemap_next_leaf_item(struct btrfs_inode *inode, struct btrfs_path *path)
2744	{
2745	struct extent_buffer *clone = path->nodes[0];
2746	struct btrfs_key key;
2747	int slot;
2748	int ret;
2749
2750	path->slots[0]++;
2751	if (path->slots[0] < btrfs_header_nritems(eb: path->nodes[0]))
2752	return 0;
2753
2754	/*
2755	* Add a temporary extra ref to an already cloned extent buffer to
2756	* prevent btrfs_next_leaf() freeing it, we want to reuse it to avoid
2757	* the cost of allocating a new one.
2758	*/
2759	ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED, &clone->bflags));
2760	atomic_inc(v: &clone->refs);
2761
2762	ret = btrfs_next_leaf(root: inode->root, path);
2763	if (ret != 0)
2764	goto out;
2765
2766	/*
2767	* Don't bother with cloning if there are no more file extent items for
2768	* our inode.
2769	*/
2770	btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &key, nr: path->slots[0]);
2771	if (key.objectid != btrfs_ino(inode) || key.type != BTRFS_EXTENT_DATA_KEY) {
2772	ret = 1;
2773	goto out;
2774	}
2775
2776	/* See the comment at fiemap_search_slot() about why we clone. */
2777	copy_extent_buffer_full(dst: clone, src: path->nodes[0]);
2778	/*
2779	* Important to preserve the start field, for the optimizations when
2780	* checking if extents are shared (see extent_fiemap()).
2781	*/
2782	clone->start = path->nodes[0]->start;
2783
2784	slot = path->slots[0];
2785	btrfs_release_path(p: path);
2786	path->nodes[0] = clone;
2787	path->slots[0] = slot;
2788	out:
2789	if (ret)
2790	free_extent_buffer(eb: clone);
2791
2792	return ret;
2793	}
2794
2795	/*
2796	* Search for the first file extent item that starts at a given file offset or
2797	* the one that starts immediately before that offset.
2798	* Returns: 0 on success, < 0 on error, 1 if not found.
2799	*/
2800	static int fiemap_search_slot(struct btrfs_inode *inode, struct btrfs_path *path,
2801	u64 file_offset)
2802	{
2803	const u64 ino = btrfs_ino(inode);
2804	struct btrfs_root *root = inode->root;
2805	struct extent_buffer *clone;
2806	struct btrfs_key key;
2807	int slot;
2808	int ret;
2809
2810	key.objectid = ino;
2811	key.type = BTRFS_EXTENT_DATA_KEY;
2812	key.offset = file_offset;
2813
2814	ret = btrfs_search_slot(NULL, root, key: &key, p: path, ins_len: 0, cow: 0);
2815	if (ret < 0)
2816	return ret;
2817
2818	if (ret > 0 && path->slots[0] > 0) {
2819	btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &key, nr: path->slots[0] - 1);
2820	if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY)
2821	path->slots[0]--;
2822	}
2823
2824	if (path->slots[0] >= btrfs_header_nritems(eb: path->nodes[0])) {
2825	ret = btrfs_next_leaf(root, path);
2826	if (ret != 0)
2827	return ret;
2828
2829	btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &key, nr: path->slots[0]);
2830	if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
2831	return 1;
2832	}
2833
2834	/*
2835	* We clone the leaf and use it during fiemap. This is because while
2836	* using the leaf we do expensive things like checking if an extent is
2837	* shared, which can take a long time. In order to prevent blocking
2838	* other tasks for too long, we use a clone of the leaf. We have locked
2839	* the file range in the inode's io tree, so we know none of our file
2840	* extent items can change. This way we avoid blocking other tasks that
2841	* want to insert items for other inodes in the same leaf or b+tree
2842	* rebalance operations (triggered for example when someone is trying
2843	* to push items into this leaf when trying to insert an item in a
2844	* neighbour leaf).
2845	* We also need the private clone because holding a read lock on an
2846	* extent buffer of the subvolume's b+tree will make lockdep unhappy
2847	* when we check if extents are shared, as backref walking may need to
2848	* lock the same leaf we are processing.
2849	*/
2850	clone = btrfs_clone_extent_buffer(src: path->nodes[0]);
2851	if (!clone)
2852	return -ENOMEM;
2853
2854	slot = path->slots[0];
2855	btrfs_release_path(p: path);
2856	path->nodes[0] = clone;
2857	path->slots[0] = slot;
2858
2859	return 0;
2860	}
2861
2862	/*
2863	* Process a range which is a hole or a prealloc extent in the inode's subvolume
2864	* btree. If @disk_bytenr is 0, we are dealing with a hole, otherwise a prealloc
2865	* extent. The end offset (@end) is inclusive.
2866	*/
2867	static int fiemap_process_hole(struct btrfs_inode *inode,
2868	struct fiemap_extent_info *fieinfo,
2869	struct fiemap_cache *cache,
2870	struct extent_state **delalloc_cached_state,
2871	struct btrfs_backref_share_check_ctx *backref_ctx,
2872	u64 disk_bytenr, u64 extent_offset,
2873	u64 extent_gen,
2874	u64 start, u64 end)
2875	{
2876	const u64 i_size = i_size_read(inode: &inode->vfs_inode);
2877	u64 cur_offset = start;
2878	u64 last_delalloc_end = 0;
2879	u32 prealloc_flags = FIEMAP_EXTENT_UNWRITTEN;
2880	bool checked_extent_shared = false;
2881	int ret;
2882
2883	/*
2884	* There can be no delalloc past i_size, so don't waste time looking for
2885	* it beyond i_size.
2886	*/
2887	while (cur_offset < end && cur_offset < i_size) {
2888	u64 delalloc_start;
2889	u64 delalloc_end;
2890	u64 prealloc_start;
2891	u64 prealloc_len = 0;
2892	bool delalloc;
2893
2894	delalloc = btrfs_find_delalloc_in_range(inode, start: cur_offset, end,
2895	cached_state: delalloc_cached_state,
2896	delalloc_start_ret: &delalloc_start,
2897	delalloc_end_ret: &delalloc_end);
2898	if (!delalloc)
2899	break;
2900
2901	/*
2902	* If this is a prealloc extent we have to report every section
2903	* of it that has no delalloc.
2904	*/
2905	if (disk_bytenr != 0) {
2906	if (last_delalloc_end == 0) {
2907	prealloc_start = start;
2908	prealloc_len = delalloc_start - start;
2909	} else {
2910	prealloc_start = last_delalloc_end + 1;
2911	prealloc_len = delalloc_start - prealloc_start;
2912	}
2913	}
2914
2915	if (prealloc_len > 0) {
2916	if (!checked_extent_shared && fieinfo->fi_extents_max) {
2917	ret = btrfs_is_data_extent_shared(inode,
2918	bytenr: disk_bytenr,
2919	extent_gen,
2920	ctx: backref_ctx);
2921	if (ret < 0)
2922	return ret;
2923	else if (ret > 0)
2924	prealloc_flags |= FIEMAP_EXTENT_SHARED;
2925
2926	checked_extent_shared = true;
2927	}
2928	ret = emit_fiemap_extent(fieinfo, cache, offset: prealloc_start,
2929	phys: disk_bytenr + extent_offset,
2930	len: prealloc_len, flags: prealloc_flags);
2931	if (ret)
2932	return ret;
2933	extent_offset += prealloc_len;
2934	}
2935
2936	ret = emit_fiemap_extent(fieinfo, cache, offset: delalloc_start, phys: 0,
2937	len: delalloc_end + 1 - delalloc_start,
2938	FIEMAP_EXTENT_DELALLOC |
2939	FIEMAP_EXTENT_UNKNOWN);
2940	if (ret)
2941	return ret;
2942
2943	last_delalloc_end = delalloc_end;
2944	cur_offset = delalloc_end + 1;
2945	extent_offset += cur_offset - delalloc_start;
2946	cond_resched();
2947	}
2948
2949	/*
2950	* Either we found no delalloc for the whole prealloc extent or we have
2951	* a prealloc extent that spans i_size or starts at or after i_size.
2952	*/
2953	if (disk_bytenr != 0 && last_delalloc_end < end) {
2954	u64 prealloc_start;
2955	u64 prealloc_len;
2956
2957	if (last_delalloc_end == 0) {
2958	prealloc_start = start;
2959	prealloc_len = end + 1 - start;
2960	} else {
2961	prealloc_start = last_delalloc_end + 1;
2962	prealloc_len = end + 1 - prealloc_start;
2963	}
2964
2965	if (!checked_extent_shared && fieinfo->fi_extents_max) {
2966	ret = btrfs_is_data_extent_shared(inode,
2967	bytenr: disk_bytenr,
2968	extent_gen,
2969	ctx: backref_ctx);
2970	if (ret < 0)
2971	return ret;
2972	else if (ret > 0)
2973	prealloc_flags |= FIEMAP_EXTENT_SHARED;
2974	}
2975	ret = emit_fiemap_extent(fieinfo, cache, offset: prealloc_start,
2976	phys: disk_bytenr + extent_offset,
2977	len: prealloc_len, flags: prealloc_flags);
2978	if (ret)
2979	return ret;
2980	}
2981
2982	return 0;
2983	}
2984
2985	static int fiemap_find_last_extent_offset(struct btrfs_inode *inode,
2986	struct btrfs_path *path,
2987	u64 *last_extent_end_ret)
2988	{
2989	const u64 ino = btrfs_ino(inode);
2990	struct btrfs_root *root = inode->root;
2991	struct extent_buffer *leaf;
2992	struct btrfs_file_extent_item *ei;
2993	struct btrfs_key key;
2994	u64 disk_bytenr;
2995	int ret;
2996
2997	/*
2998	* Lookup the last file extent. We're not using i_size here because
2999	* there might be preallocation past i_size.
3000	*/
3001	ret = btrfs_lookup_file_extent(NULL, root, path, objectid: ino, bytenr: (u64)-1, mod: 0);
3002	/* There can't be a file extent item at offset (u64)-1 */
3003	ASSERT(ret != 0);
3004	if (ret < 0)
3005	return ret;
3006
3007	/*
3008	* For a non-existing key, btrfs_search_slot() always leaves us at a
3009	* slot > 0, except if the btree is empty, which is impossible because
3010	* at least it has the inode item for this inode and all the items for
3011	* the root inode 256.
3012	*/
3013	ASSERT(path->slots[0] > 0);
3014	path->slots[0]--;
3015	leaf = path->nodes[0];
3016	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
3017	if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) {
3018	/* No file extent items in the subvolume tree. */
3019	*last_extent_end_ret = 0;
3020	return 0;
3021	}
3022
3023	/*
3024	* For an inline extent, the disk_bytenr is where inline data starts at,
3025	* so first check if we have an inline extent item before checking if we
3026	* have an implicit hole (disk_bytenr == 0).
3027	*/
3028	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
3029	if (btrfs_file_extent_type(eb: leaf, s: ei) == BTRFS_FILE_EXTENT_INLINE) {
3030	*last_extent_end_ret = btrfs_file_extent_end(path);
3031	return 0;
3032	}
3033
3034	/*
3035	* Find the last file extent item that is not a hole (when NO_HOLES is
3036	* not enabled). This should take at most 2 iterations in the worst
3037	* case: we have one hole file extent item at slot 0 of a leaf and
3038	* another hole file extent item as the last item in the previous leaf.
3039	* This is because we merge file extent items that represent holes.
3040	*/
3041	disk_bytenr = btrfs_file_extent_disk_bytenr(eb: leaf, s: ei);
3042	while (disk_bytenr == 0) {
3043	ret = btrfs_previous_item(root, path, min_objectid: ino, BTRFS_EXTENT_DATA_KEY);
3044	if (ret < 0) {
3045	return ret;
3046	} else if (ret > 0) {
3047	/* No file extent items that are not holes. */
3048	*last_extent_end_ret = 0;
3049	return 0;
3050	}
3051	leaf = path->nodes[0];
3052	ei = btrfs_item_ptr(leaf, path->slots[0],
3053	struct btrfs_file_extent_item);
3054	disk_bytenr = btrfs_file_extent_disk_bytenr(eb: leaf, s: ei);
3055	}
3056
3057	*last_extent_end_ret = btrfs_file_extent_end(path);
3058	return 0;
3059	}
3060
3061	int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo,
3062	u64 start, u64 len)
3063	{
3064	const u64 ino = btrfs_ino(inode);
3065	struct extent_state *cached_state = NULL;
3066	struct extent_state *delalloc_cached_state = NULL;
3067	struct btrfs_path *path;
3068	struct fiemap_cache cache = { 0 };
3069	struct btrfs_backref_share_check_ctx *backref_ctx;
3070	u64 last_extent_end;
3071	u64 prev_extent_end;
3072	u64 range_start;
3073	u64 range_end;
3074	const u64 sectorsize = inode->root->fs_info->sectorsize;
3075	bool stopped = false;
3076	int ret;
3077
3078	cache.entries_size = PAGE_SIZE / sizeof(struct btrfs_fiemap_entry);
3079	cache.entries = kmalloc_array(n: cache.entries_size,
3080	size: sizeof(struct btrfs_fiemap_entry),
3081	GFP_KERNEL);
3082	backref_ctx = btrfs_alloc_backref_share_check_ctx();
3083	path = btrfs_alloc_path();
3084	if (!cache.entries || !backref_ctx || !path) {
3085	ret = -ENOMEM;
3086	goto out;
3087	}
3088
3089	restart:
3090	range_start = round_down(start, sectorsize);
3091	range_end = round_up(start + len, sectorsize);
3092	prev_extent_end = range_start;
3093
3094	lock_extent(tree: &inode->io_tree, start: range_start, end: range_end, cached: &cached_state);
3095
3096	ret = fiemap_find_last_extent_offset(inode, path, last_extent_end_ret: &last_extent_end);
3097	if (ret < 0)
3098	goto out_unlock;
3099	btrfs_release_path(p: path);
3100
3101	path->reada = READA_FORWARD;
3102	ret = fiemap_search_slot(inode, path, file_offset: range_start);
3103	if (ret < 0) {
3104	goto out_unlock;
3105	} else if (ret > 0) {
3106	/*
3107	* No file extent item found, but we may have delalloc between
3108	* the current offset and i_size. So check for that.
3109	*/
3110	ret = 0;
3111	goto check_eof_delalloc;
3112	}
3113
3114	while (prev_extent_end < range_end) {
3115	struct extent_buffer *leaf = path->nodes[0];
3116	struct btrfs_file_extent_item *ei;
3117	struct btrfs_key key;
3118	u64 extent_end;
3119	u64 extent_len;
3120	u64 extent_offset = 0;
3121	u64 extent_gen;
3122	u64 disk_bytenr = 0;
3123	u64 flags = 0;
3124	int extent_type;
3125	u8 compression;
3126
3127	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
3128	if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
3129	break;
3130
3131	extent_end = btrfs_file_extent_end(path);
3132
3133	/*
3134	* The first iteration can leave us at an extent item that ends
3135	* before our range's start. Move to the next item.
3136	*/
3137	if (extent_end <= range_start)
3138	goto next_item;
3139
3140	backref_ctx->curr_leaf_bytenr = leaf->start;
3141
3142	/* We have in implicit hole (NO_HOLES feature enabled). */
3143	if (prev_extent_end < key.offset) {
3144	const u64 hole_end = min(key.offset, range_end) - 1;
3145
3146	ret = fiemap_process_hole(inode, fieinfo, cache: &cache,
3147	delalloc_cached_state: &delalloc_cached_state,
3148	backref_ctx, disk_bytenr: 0, extent_offset: 0, extent_gen: 0,
3149	start: prev_extent_end, end: hole_end);
3150	if (ret < 0) {
3151	goto out_unlock;
3152	} else if (ret > 0) {
3153	/* fiemap_fill_next_extent() told us to stop. */
3154	stopped = true;
3155	break;
3156	}
3157
3158	/* We've reached the end of the fiemap range, stop. */
3159	if (key.offset >= range_end) {
3160	stopped = true;
3161	break;
3162	}
3163	}
3164
3165	extent_len = extent_end - key.offset;
3166	ei = btrfs_item_ptr(leaf, path->slots[0],
3167	struct btrfs_file_extent_item);
3168	compression = btrfs_file_extent_compression(eb: leaf, s: ei);
3169	extent_type = btrfs_file_extent_type(eb: leaf, s: ei);
3170	extent_gen = btrfs_file_extent_generation(eb: leaf, s: ei);
3171
3172	if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
3173	disk_bytenr = btrfs_file_extent_disk_bytenr(eb: leaf, s: ei);
3174	if (compression == BTRFS_COMPRESS_NONE)
3175	extent_offset = btrfs_file_extent_offset(eb: leaf, s: ei);
3176	}
3177
3178	if (compression != BTRFS_COMPRESS_NONE)
3179	flags |= FIEMAP_EXTENT_ENCODED;
3180
3181	if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
3182	flags |= FIEMAP_EXTENT_DATA_INLINE;
3183	flags |= FIEMAP_EXTENT_NOT_ALIGNED;
3184	ret = emit_fiemap_extent(fieinfo, cache: &cache, offset: key.offset, phys: 0,
3185	len: extent_len, flags);
3186	} else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
3187	ret = fiemap_process_hole(inode, fieinfo, cache: &cache,
3188	delalloc_cached_state: &delalloc_cached_state,
3189	backref_ctx,
3190	disk_bytenr, extent_offset,
3191	extent_gen, start: key.offset,
3192	end: extent_end - 1);
3193	} else if (disk_bytenr == 0) {
3194	/* We have an explicit hole. */
3195	ret = fiemap_process_hole(inode, fieinfo, cache: &cache,
3196	delalloc_cached_state: &delalloc_cached_state,
3197	backref_ctx, disk_bytenr: 0, extent_offset: 0, extent_gen: 0,
3198	start: key.offset, end: extent_end - 1);
3199	} else {
3200	/* We have a regular extent. */
3201	if (fieinfo->fi_extents_max) {
3202	ret = btrfs_is_data_extent_shared(inode,
3203	bytenr: disk_bytenr,
3204	extent_gen,
3205	ctx: backref_ctx);
3206	if (ret < 0)
3207	goto out_unlock;
3208	else if (ret > 0)
3209	flags |= FIEMAP_EXTENT_SHARED;
3210	}
3211
3212	ret = emit_fiemap_extent(fieinfo, cache: &cache, offset: key.offset,
3213	phys: disk_bytenr + extent_offset,
3214	len: extent_len, flags);
3215	}
3216
3217	if (ret < 0) {
3218	goto out_unlock;
3219	} else if (ret > 0) {
3220	/* emit_fiemap_extent() told us to stop. */
3221	stopped = true;
3222	break;
3223	}
3224
3225	prev_extent_end = extent_end;
3226	next_item:
3227	if (fatal_signal_pending(current)) {
3228	ret = -EINTR;
3229	goto out_unlock;
3230	}
3231
3232	ret = fiemap_next_leaf_item(inode, path);
3233	if (ret < 0) {
3234	goto out_unlock;
3235	} else if (ret > 0) {
3236	/* No more file extent items for this inode. */
3237	break;
3238	}
3239	cond_resched();
3240	}
3241
3242	check_eof_delalloc:
3243	if (!stopped && prev_extent_end < range_end) {
3244	ret = fiemap_process_hole(inode, fieinfo, cache: &cache,
3245	delalloc_cached_state: &delalloc_cached_state, backref_ctx,
3246	disk_bytenr: 0, extent_offset: 0, extent_gen: 0, start: prev_extent_end, end: range_end - 1);
3247	if (ret < 0)
3248	goto out_unlock;
3249	prev_extent_end = range_end;
3250	}
3251
3252	if (cache.cached && cache.offset + cache.len >= last_extent_end) {
3253	const u64 i_size = i_size_read(inode: &inode->vfs_inode);
3254
3255	if (prev_extent_end < i_size) {
3256	u64 delalloc_start;
3257	u64 delalloc_end;
3258	bool delalloc;
3259
3260	delalloc = btrfs_find_delalloc_in_range(inode,
3261	start: prev_extent_end,
3262	end: i_size - 1,
3263	cached_state: &delalloc_cached_state,
3264	delalloc_start_ret: &delalloc_start,
3265	delalloc_end_ret: &delalloc_end);
3266	if (!delalloc)
3267	cache.flags |= FIEMAP_EXTENT_LAST;
3268	} else {
3269	cache.flags |= FIEMAP_EXTENT_LAST;
3270	}
3271	}
3272
3273	out_unlock:
3274	unlock_extent(tree: &inode->io_tree, start: range_start, end: range_end, cached: &cached_state);
3275
3276	if (ret == BTRFS_FIEMAP_FLUSH_CACHE) {
3277	btrfs_release_path(p: path);
3278	ret = flush_fiemap_cache(fieinfo, cache: &cache);
3279	if (ret)
3280	goto out;
3281	len -= cache.next_search_offset - start;
3282	start = cache.next_search_offset;
3283	goto restart;
3284	} else if (ret < 0) {
3285	goto out;
3286	}
3287
3288	/*
3289	* Must free the path before emitting to the fiemap buffer because we
3290	* may have a non-cloned leaf and if the fiemap buffer is memory mapped
3291	* to a file, a write into it (through btrfs_page_mkwrite()) may trigger
3292	* waiting for an ordered extent that in order to complete needs to
3293	* modify that leaf, therefore leading to a deadlock.
3294	*/
3295	btrfs_free_path(p: path);
3296	path = NULL;
3297
3298	ret = flush_fiemap_cache(fieinfo, cache: &cache);
3299	if (ret)
3300	goto out;
3301
3302	ret = emit_last_fiemap_cache(fieinfo, cache: &cache);
3303	out:
3304	free_extent_state(state: delalloc_cached_state);
3305	kfree(objp: cache.entries);
3306	btrfs_free_backref_share_ctx(ctx: backref_ctx);
3307	btrfs_free_path(p: path);
3308	return ret;
3309	}
3310
3311	static void __free_extent_buffer(struct extent_buffer *eb)
3312	{
3313	kmem_cache_free(s: extent_buffer_cache, objp: eb);
3314	}
3315
3316	static int extent_buffer_under_io(const struct extent_buffer *eb)
3317	{
3318	return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
3319	test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
3320	}
3321
3322	static bool folio_range_has_eb(struct btrfs_fs_info *fs_info, struct folio *folio)
3323	{
3324	struct btrfs_subpage *subpage;
3325
3326	lockdep_assert_held(&folio->mapping->i_private_lock);
3327
3328	if (folio_test_private(folio)) {
3329	subpage = folio_get_private(folio);
3330	if (atomic_read(v: &subpage->eb_refs))
3331	return true;
3332	/*
3333	* Even there is no eb refs here, we may still have
3334	* end_page_read() call relying on page::private.
3335	*/
3336	if (atomic_read(v: &subpage->readers))
3337	return true;
3338	}
3339	return false;
3340	}
3341
3342	static void detach_extent_buffer_folio(struct extent_buffer *eb, struct folio *folio)
3343	{
3344	struct btrfs_fs_info *fs_info = eb->fs_info;
3345	const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3346
3347	/*
3348	* For mapped eb, we're going to change the folio private, which should
3349	* be done under the i_private_lock.
3350	*/
3351	if (mapped)
3352	spin_lock(lock: &folio->mapping->i_private_lock);
3353
3354	if (!folio_test_private(folio)) {
3355	if (mapped)
3356	spin_unlock(lock: &folio->mapping->i_private_lock);
3357	return;
3358	}
3359
3360	if (fs_info->nodesize >= PAGE_SIZE) {
3361	/*
3362	* We do this since we'll remove the pages after we've
3363	* removed the eb from the radix tree, so we could race
3364	* and have this page now attached to the new eb. So
3365	* only clear folio if it's still connected to
3366	* this eb.
3367	*/
3368	if (folio_test_private(folio) && folio_get_private(folio) == eb) {
3369	BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
3370	BUG_ON(folio_test_dirty(folio));
3371	BUG_ON(folio_test_writeback(folio));
3372	/* We need to make sure we haven't be attached to a new eb. */
3373	folio_detach_private(folio);
3374	}
3375	if (mapped)
3376	spin_unlock(lock: &folio->mapping->i_private_lock);
3377	return;
3378	}
3379
3380	/*
3381	* For subpage, we can have dummy eb with folio private attached. In
3382	* this case, we can directly detach the private as such folio is only
3383	* attached to one dummy eb, no sharing.
3384	*/
3385	if (!mapped) {
3386	btrfs_detach_subpage(fs_info, folio);
3387	return;
3388	}
3389
3390	btrfs_folio_dec_eb_refs(fs_info, folio);
3391
3392	/*
3393	* We can only detach the folio private if there are no other ebs in the
3394	* page range and no unfinished IO.
3395	*/
3396	if (!folio_range_has_eb(fs_info, folio))
3397	btrfs_detach_subpage(fs_info, folio);
3398
3399	spin_unlock(lock: &folio->mapping->i_private_lock);
3400	}
3401
3402	/* Release all pages attached to the extent buffer */
3403	static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb)
3404	{
3405	ASSERT(!extent_buffer_under_io(eb));
3406
3407	for (int i = 0; i < INLINE_EXTENT_BUFFER_PAGES; i++) {
3408	struct folio *folio = eb->folios[i];
3409
3410	if (!folio)
3411	continue;
3412
3413	detach_extent_buffer_folio(eb, folio);
3414
3415	/* One for when we allocated the folio. */
3416	folio_put(folio);
3417	}
3418	}
3419
3420	/*
3421	* Helper for releasing the extent buffer.
3422	*/
3423	static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
3424	{
3425	btrfs_release_extent_buffer_pages(eb);
3426	btrfs_leak_debug_del_eb(eb);
3427	__free_extent_buffer(eb);
3428	}
3429
3430	static struct extent_buffer *
3431	__alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
3432	unsigned long len)
3433	{
3434	struct extent_buffer *eb = NULL;
3435
3436	eb = kmem_cache_zalloc(k: extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
3437	eb->start = start;
3438	eb->len = len;
3439	eb->fs_info = fs_info;
3440	init_rwsem(&eb->lock);
3441
3442	btrfs_leak_debug_add_eb(eb);
3443
3444	spin_lock_init(&eb->refs_lock);
3445	atomic_set(v: &eb->refs, i: 1);
3446
3447	ASSERT(len <= BTRFS_MAX_METADATA_BLOCKSIZE);
3448
3449	return eb;
3450	}
3451
3452	struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
3453	{
3454	struct extent_buffer *new;
3455	int num_folios = num_extent_folios(eb: src);
3456	int ret;
3457
3458	new = __alloc_extent_buffer(fs_info: src->fs_info, start: src->start, len: src->len);
3459	if (new == NULL)
3460	return NULL;
3461
3462	/*
3463	* Set UNMAPPED before calling btrfs_release_extent_buffer(), as
3464	* btrfs_release_extent_buffer() have different behavior for
3465	* UNMAPPED subpage extent buffer.
3466	*/
3467	set_bit(nr: EXTENT_BUFFER_UNMAPPED, addr: &new->bflags);
3468
3469	ret = alloc_eb_folio_array(eb: new, extra_gfp: 0);
3470	if (ret) {
3471	btrfs_release_extent_buffer(eb: new);
3472	return NULL;
3473	}
3474
3475	for (int i = 0; i < num_folios; i++) {
3476	struct folio *folio = new->folios[i];
3477	int ret;
3478
3479	ret = attach_extent_buffer_folio(eb: new, folio, NULL);
3480	if (ret < 0) {
3481	btrfs_release_extent_buffer(eb: new);
3482	return NULL;
3483	}
3484	WARN_ON(folio_test_dirty(folio));
3485	}
3486	copy_extent_buffer_full(dst: new, src);
3487	set_extent_buffer_uptodate(new);
3488
3489	return new;
3490	}
3491
3492	struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3493	u64 start, unsigned long len)
3494	{
3495	struct extent_buffer *eb;
3496	int num_folios = 0;
3497	int ret;
3498
3499	eb = __alloc_extent_buffer(fs_info, start, len);
3500	if (!eb)
3501	return NULL;
3502
3503	ret = alloc_eb_folio_array(eb, extra_gfp: 0);
3504	if (ret)
3505	goto err;
3506
3507	num_folios = num_extent_folios(eb);
3508	for (int i = 0; i < num_folios; i++) {
3509	ret = attach_extent_buffer_folio(eb, folio: eb->folios[i], NULL);
3510	if (ret < 0)
3511	goto err;
3512	}
3513
3514	set_extent_buffer_uptodate(eb);
3515	btrfs_set_header_nritems(eb, val: 0);
3516	set_bit(nr: EXTENT_BUFFER_UNMAPPED, addr: &eb->bflags);
3517
3518	return eb;
3519	err:
3520	for (int i = 0; i < num_folios; i++) {
3521	if (eb->folios[i]) {
3522	detach_extent_buffer_folio(eb, folio: eb->folios[i]);
3523	__folio_put(folio: eb->folios[i]);
3524	}
3525	}
3526	__free_extent_buffer(eb);
3527	return NULL;
3528	}
3529
3530	struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3531	u64 start)
3532	{
3533	return __alloc_dummy_extent_buffer(fs_info, start, len: fs_info->nodesize);
3534	}
3535
3536	static void check_buffer_tree_ref(struct extent_buffer *eb)
3537	{
3538	int refs;
3539	/*
3540	* The TREE_REF bit is first set when the extent_buffer is added
3541	* to the radix tree. It is also reset, if unset, when a new reference
3542	* is created by find_extent_buffer.
3543	*
3544	* It is only cleared in two cases: freeing the last non-tree
3545	* reference to the extent_buffer when its STALE bit is set or
3546	* calling release_folio when the tree reference is the only reference.
3547	*
3548	* In both cases, care is taken to ensure that the extent_buffer's
3549	* pages are not under io. However, release_folio can be concurrently
3550	* called with creating new references, which is prone to race
3551	* conditions between the calls to check_buffer_tree_ref in those
3552	* codepaths and clearing TREE_REF in try_release_extent_buffer.
3553	*
3554	* The actual lifetime of the extent_buffer in the radix tree is
3555	* adequately protected by the refcount, but the TREE_REF bit and
3556	* its corresponding reference are not. To protect against this
3557	* class of races, we call check_buffer_tree_ref from the codepaths
3558	* which trigger io. Note that once io is initiated, TREE_REF can no
3559	* longer be cleared, so that is the moment at which any such race is
3560	* best fixed.
3561	*/
3562	refs = atomic_read(v: &eb->refs);
3563	if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3564	return;
3565
3566	spin_lock(lock: &eb->refs_lock);
3567	if (!test_and_set_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags))
3568	atomic_inc(v: &eb->refs);
3569	spin_unlock(lock: &eb->refs_lock);
3570	}
3571
3572	static void mark_extent_buffer_accessed(struct extent_buffer *eb)
3573	{
3574	int num_folios= num_extent_folios(eb);
3575
3576	check_buffer_tree_ref(eb);
3577
3578	for (int i = 0; i < num_folios; i++)
3579	folio_mark_accessed(eb->folios[i]);
3580	}
3581
3582	struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
3583	u64 start)
3584	{
3585	struct extent_buffer *eb;
3586
3587	eb = find_extent_buffer_nolock(fs_info, start);
3588	if (!eb)
3589	return NULL;
3590	/*
3591	* Lock our eb's refs_lock to avoid races with free_extent_buffer().
3592	* When we get our eb it might be flagged with EXTENT_BUFFER_STALE and
3593	* another task running free_extent_buffer() might have seen that flag
3594	* set, eb->refs == 2, that the buffer isn't under IO (dirty and
3595	* writeback flags not set) and it's still in the tree (flag
3596	* EXTENT_BUFFER_TREE_REF set), therefore being in the process of
3597	* decrementing the extent buffer's reference count twice. So here we
3598	* could race and increment the eb's reference count, clear its stale
3599	* flag, mark it as dirty and drop our reference before the other task
3600	* finishes executing free_extent_buffer, which would later result in
3601	* an attempt to free an extent buffer that is dirty.
3602	*/
3603	if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
3604	spin_lock(lock: &eb->refs_lock);
3605	spin_unlock(lock: &eb->refs_lock);
3606	}
3607	mark_extent_buffer_accessed(eb);
3608	return eb;
3609	}
3610
3611	#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3612	struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
3613	u64 start)
3614	{
3615	struct extent_buffer *eb, *exists = NULL;
3616	int ret;
3617
3618	eb = find_extent_buffer(fs_info, start);
3619	if (eb)
3620	return eb;
3621	eb = alloc_dummy_extent_buffer(fs_info, start);
3622	if (!eb)
3623	return ERR_PTR(error: -ENOMEM);
3624	eb->fs_info = fs_info;
3625	again:
3626	ret = radix_tree_preload(GFP_NOFS);
3627	if (ret) {
3628	exists = ERR_PTR(error: ret);
3629	goto free_eb;
3630	}
3631	spin_lock(lock: &fs_info->buffer_lock);
3632	ret = radix_tree_insert(&fs_info->buffer_radix,
3633	index: start >> fs_info->sectorsize_bits, eb);
3634	spin_unlock(lock: &fs_info->buffer_lock);
3635	radix_tree_preload_end();
3636	if (ret == -EEXIST) {
3637	exists = find_extent_buffer(fs_info, start);
3638	if (exists)
3639	goto free_eb;
3640	else
3641	goto again;
3642	}
3643	check_buffer_tree_ref(eb);
3644	set_bit(nr: EXTENT_BUFFER_IN_TREE, addr: &eb->bflags);
3645
3646	return eb;
3647	free_eb:
3648	btrfs_release_extent_buffer(eb);
3649	return exists;
3650	}
3651	#endif
3652
3653	static struct extent_buffer *grab_extent_buffer(
3654	struct btrfs_fs_info *fs_info, struct page *page)
3655	{
3656	struct folio *folio = page_folio(page);
3657	struct extent_buffer *exists;
3658
3659	/*
3660	* For subpage case, we completely rely on radix tree to ensure we
3661	* don't try to insert two ebs for the same bytenr. So here we always
3662	* return NULL and just continue.
3663	*/
3664	if (fs_info->nodesize < PAGE_SIZE)
3665	return NULL;
3666
3667	/* Page not yet attached to an extent buffer */
3668	if (!folio_test_private(folio))
3669	return NULL;
3670
3671	/*
3672	* We could have already allocated an eb for this page and attached one
3673	* so lets see if we can get a ref on the existing eb, and if we can we
3674	* know it's good and we can just return that one, else we know we can
3675	* just overwrite folio private.
3676	*/
3677	exists = folio_get_private(folio);
3678	if (atomic_inc_not_zero(v: &exists->refs))
3679	return exists;
3680
3681	WARN_ON(PageDirty(page));
3682	folio_detach_private(folio);
3683	return NULL;
3684	}
3685
3686	static int check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start)
3687	{
3688	if (!IS_ALIGNED(start, fs_info->sectorsize)) {
3689	btrfs_err(fs_info, "bad tree block start %llu", start);
3690	return -EINVAL;
3691	}
3692
3693	if (fs_info->nodesize < PAGE_SIZE &&
3694	offset_in_page(start) + fs_info->nodesize > PAGE_SIZE) {
3695	btrfs_err(fs_info,
3696	"tree block crosses page boundary, start %llu nodesize %u",
3697	start, fs_info->nodesize);
3698	return -EINVAL;
3699	}
3700	if (fs_info->nodesize >= PAGE_SIZE &&
3701	!PAGE_ALIGNED(start)) {
3702	btrfs_err(fs_info,
3703	"tree block is not page aligned, start %llu nodesize %u",
3704	start, fs_info->nodesize);
3705	return -EINVAL;
3706	}
3707	if (!IS_ALIGNED(start, fs_info->nodesize) &&
3708	!test_and_set_bit(nr: BTRFS_FS_UNALIGNED_TREE_BLOCK, addr: &fs_info->flags)) {
3709	btrfs_warn(fs_info,
3710	"tree block not nodesize aligned, start %llu nodesize %u, can be resolved by a full metadata balance",
3711	start, fs_info->nodesize);
3712	}
3713	return 0;
3714	}
3715
3716
3717	/*
3718	* Return 0 if eb->folios[i] is attached to btree inode successfully.
3719	* Return >0 if there is already another extent buffer for the range,
3720	* and @found_eb_ret would be updated.
3721	* Return -EAGAIN if the filemap has an existing folio but with different size
3722	* than @eb.
3723	* The caller needs to free the existing folios and retry using the same order.
3724	*/
3725	static int attach_eb_folio_to_filemap(struct extent_buffer *eb, int i,
3726	struct extent_buffer **found_eb_ret)
3727	{
3728
3729	struct btrfs_fs_info *fs_info = eb->fs_info;
3730	struct address_space *mapping = fs_info->btree_inode->i_mapping;
3731	const unsigned long index = eb->start >> PAGE_SHIFT;
3732	struct folio *existing_folio;
3733	int ret;
3734
3735	ASSERT(found_eb_ret);
3736
3737	/* Caller should ensure the folio exists. */
3738	ASSERT(eb->folios[i]);
3739
3740	retry:
3741	ret = filemap_add_folio(mapping, folio: eb->folios[i], index: index + i,
3742	GFP_NOFS | __GFP_NOFAIL);
3743	if (!ret)
3744	return 0;
3745
3746	existing_folio = filemap_lock_folio(mapping, index: index + i);
3747	/* The page cache only exists for a very short time, just retry. */
3748	if (IS_ERR(ptr: existing_folio))
3749	goto retry;
3750
3751	/* For now, we should only have single-page folios for btree inode. */
3752	ASSERT(folio_nr_pages(existing_folio) == 1);
3753
3754	if (folio_size(folio: existing_folio) != eb->folio_size) {
3755	folio_unlock(folio: existing_folio);
3756	folio_put(folio: existing_folio);
3757	return -EAGAIN;
3758	}
3759
3760	if (fs_info->nodesize < PAGE_SIZE) {
3761	/*
3762	* We're going to reuse the existing page, can drop our page
3763	* and subpage structure now.
3764	*/
3765	__free_page(folio_page(eb->folios[i], 0));
3766	eb->folios[i] = existing_folio;
3767	} else {
3768	struct extent_buffer *existing_eb;
3769
3770	existing_eb = grab_extent_buffer(fs_info,
3771	folio_page(existing_folio, 0));
3772	if (existing_eb) {
3773	/* The extent buffer still exists, we can use it directly. */
3774	*found_eb_ret = existing_eb;
3775	folio_unlock(folio: existing_folio);
3776	folio_put(folio: existing_folio);
3777	return 1;
3778	}
3779	/* The extent buffer no longer exists, we can reuse the folio. */
3780	__free_page(folio_page(eb->folios[i], 0));
3781	eb->folios[i] = existing_folio;
3782	}
3783	return 0;
3784	}
3785
3786	struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
3787	u64 start, u64 owner_root, int level)
3788	{
3789	unsigned long len = fs_info->nodesize;
3790	int num_folios;
3791	int attached = 0;
3792	struct extent_buffer *eb;
3793	struct extent_buffer *existing_eb = NULL;
3794	struct address_space *mapping = fs_info->btree_inode->i_mapping;
3795	struct btrfs_subpage *prealloc = NULL;
3796	u64 lockdep_owner = owner_root;
3797	bool page_contig = true;
3798	int uptodate = 1;
3799	int ret;
3800
3801	if (check_eb_alignment(fs_info, start))
3802	return ERR_PTR(error: -EINVAL);
3803
3804	#if BITS_PER_LONG == 32
3805	if (start >= MAX_LFS_FILESIZE) {
3806	btrfs_err_rl(fs_info,
3807	"extent buffer %llu is beyond 32bit page cache limit", start);
3808	btrfs_err_32bit_limit(fs_info);
3809	return ERR_PTR(-EOVERFLOW);
3810	}
3811	if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD)
3812	btrfs_warn_32bit_limit(fs_info);
3813	#endif
3814
3815	eb = find_extent_buffer(fs_info, start);
3816	if (eb)
3817	return eb;
3818
3819	eb = __alloc_extent_buffer(fs_info, start, len);
3820	if (!eb)
3821	return ERR_PTR(error: -ENOMEM);
3822
3823	/*
3824	* The reloc trees are just snapshots, so we need them to appear to be
3825	* just like any other fs tree WRT lockdep.
3826	*/
3827	if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID)
3828	lockdep_owner = BTRFS_FS_TREE_OBJECTID;
3829
3830	btrfs_set_buffer_lockdep_class(objectid: lockdep_owner, eb, level);
3831
3832	/*
3833	* Preallocate folio private for subpage case, so that we won't
3834	* allocate memory with i_private_lock nor page lock hold.
3835	*
3836	* The memory will be freed by attach_extent_buffer_page() or freed
3837	* manually if we exit earlier.
3838	*/
3839	if (fs_info->nodesize < PAGE_SIZE) {
3840	prealloc = btrfs_alloc_subpage(fs_info, type: BTRFS_SUBPAGE_METADATA);
3841	if (IS_ERR(ptr: prealloc)) {
3842	ret = PTR_ERR(ptr: prealloc);
3843	goto out;
3844	}
3845	}
3846
3847	reallocate:
3848	/* Allocate all pages first. */
3849	ret = alloc_eb_folio_array(eb, __GFP_NOFAIL);
3850	if (ret < 0) {
3851	btrfs_free_subpage(subpage: prealloc);
3852	goto out;
3853	}
3854
3855	num_folios = num_extent_folios(eb);
3856	/* Attach all pages to the filemap. */
3857	for (int i = 0; i < num_folios; i++) {
3858	struct folio *folio;
3859
3860	ret = attach_eb_folio_to_filemap(eb, i, found_eb_ret: &existing_eb);
3861	if (ret > 0) {
3862	ASSERT(existing_eb);
3863	goto out;
3864	}
3865
3866	/*
3867	* TODO: Special handling for a corner case where the order of
3868	* folios mismatch between the new eb and filemap.
3869	*
3870	* This happens when:
3871	*
3872	* - the new eb is using higher order folio
3873	*
3874	* - the filemap is still using 0-order folios for the range
3875	* This can happen at the previous eb allocation, and we don't
3876	* have higher order folio for the call.
3877	*
3878	* - the existing eb has already been freed
3879	*
3880	* In this case, we have to free the existing folios first, and
3881	* re-allocate using the same order.
3882	* Thankfully this is not going to happen yet, as we're still
3883	* using 0-order folios.
3884	*/
3885	if (unlikely(ret == -EAGAIN)) {
3886	ASSERT(0);
3887	goto reallocate;
3888	}
3889	attached++;
3890
3891	/*
3892	* Only after attach_eb_folio_to_filemap(), eb->folios[] is
3893	* reliable, as we may choose to reuse the existing page cache
3894	* and free the allocated page.
3895	*/
3896	folio = eb->folios[i];
3897	eb->folio_size = folio_size(folio);
3898	eb->folio_shift = folio_shift(folio);
3899	spin_lock(lock: &mapping->i_private_lock);
3900	/* Should not fail, as we have preallocated the memory */
3901	ret = attach_extent_buffer_folio(eb, folio, prealloc);
3902	ASSERT(!ret);
3903	/*
3904	* To inform we have extra eb under allocation, so that
3905	* detach_extent_buffer_page() won't release the folio private
3906	* when the eb hasn't yet been inserted into radix tree.
3907	*
3908	* The ref will be decreased when the eb released the page, in
3909	* detach_extent_buffer_page().
3910	* Thus needs no special handling in error path.
3911	*/
3912	btrfs_folio_inc_eb_refs(fs_info, folio);
3913	spin_unlock(lock: &mapping->i_private_lock);
3914
3915	WARN_ON(btrfs_folio_test_dirty(fs_info, folio, eb->start, eb->len));
3916
3917	/*
3918	* Check if the current page is physically contiguous with previous eb
3919	* page.
3920	* At this stage, either we allocated a large folio, thus @i
3921	* would only be 0, or we fall back to per-page allocation.
3922	*/
3923	if (i && folio_page(eb->folios[i - 1], 0) + 1 != folio_page(folio, 0))
3924	page_contig = false;
3925
3926	if (!btrfs_folio_test_uptodate(fs_info, folio, start: eb->start, len: eb->len))
3927	uptodate = 0;
3928
3929	/*
3930	* We can't unlock the pages just yet since the extent buffer
3931	* hasn't been properly inserted in the radix tree, this
3932	* opens a race with btree_release_folio which can free a page
3933	* while we are still filling in all pages for the buffer and
3934	* we could crash.
3935	*/
3936	}
3937	if (uptodate)
3938	set_bit(nr: EXTENT_BUFFER_UPTODATE, addr: &eb->bflags);
3939	/* All pages are physically contiguous, can skip cross page handling. */
3940	if (page_contig)
3941	eb->addr = folio_address(folio: eb->folios[0]) + offset_in_page(eb->start);
3942	again:
3943	ret = radix_tree_preload(GFP_NOFS);
3944	if (ret)
3945	goto out;
3946
3947	spin_lock(lock: &fs_info->buffer_lock);
3948	ret = radix_tree_insert(&fs_info->buffer_radix,
3949	index: start >> fs_info->sectorsize_bits, eb);
3950	spin_unlock(lock: &fs_info->buffer_lock);
3951	radix_tree_preload_end();
3952	if (ret == -EEXIST) {
3953	ret = 0;
3954	existing_eb = find_extent_buffer(fs_info, start);
3955	if (existing_eb)
3956	goto out;
3957	else
3958	goto again;
3959	}
3960	/* add one reference for the tree */
3961	check_buffer_tree_ref(eb);
3962	set_bit(nr: EXTENT_BUFFER_IN_TREE, addr: &eb->bflags);
3963
3964	/*
3965	* Now it's safe to unlock the pages because any calls to
3966	* btree_release_folio will correctly detect that a page belongs to a
3967	* live buffer and won't free them prematurely.
3968	*/
3969	for (int i = 0; i < num_folios; i++)
3970	unlock_page(folio_page(eb->folios[i], 0));
3971	return eb;
3972
3973	out:
3974	WARN_ON(!atomic_dec_and_test(&eb->refs));
3975
3976	/*
3977	* Any attached folios need to be detached before we unlock them. This
3978	* is because when we're inserting our new folios into the mapping, and
3979	* then attaching our eb to that folio. If we fail to insert our folio
3980	* we'll lookup the folio for that index, and grab that EB. We do not
3981	* want that to grab this eb, as we're getting ready to free it. So we
3982	* have to detach it first and then unlock it.
3983	*
3984	* We have to drop our reference and NULL it out here because in the
3985	* subpage case detaching does a btrfs_folio_dec_eb_refs() for our eb.
3986	* Below when we call btrfs_release_extent_buffer() we will call
3987	* detach_extent_buffer_folio() on our remaining pages in the !subpage
3988	* case. If we left eb->folios[i] populated in the subpage case we'd
3989	* double put our reference and be super sad.
3990	*/
3991	for (int i = 0; i < attached; i++) {
3992	ASSERT(eb->folios[i]);
3993	detach_extent_buffer_folio(eb, folio: eb->folios[i]);
3994	unlock_page(folio_page(eb->folios[i], 0));
3995	folio_put(folio: eb->folios[i]);
3996	eb->folios[i] = NULL;
3997	}
3998	/*
3999	* Now all pages of that extent buffer is unmapped, set UNMAPPED flag,
4000	* so it can be cleaned up without utlizing page->mapping.
4001	*/
4002	set_bit(nr: EXTENT_BUFFER_UNMAPPED, addr: &eb->bflags);
4003
4004	btrfs_release_extent_buffer(eb);
4005	if (ret < 0)
4006	return ERR_PTR(error: ret);
4007	ASSERT(existing_eb);
4008	return existing_eb;
4009	}
4010
4011	static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4012	{
4013	struct extent_buffer *eb =
4014	container_of(head, struct extent_buffer, rcu_head);
4015
4016	__free_extent_buffer(eb);
4017	}
4018
4019	static int release_extent_buffer(struct extent_buffer *eb)
4020	__releases(&eb->refs_lock)
4021	{
4022	lockdep_assert_held(&eb->refs_lock);
4023
4024	WARN_ON(atomic_read(&eb->refs) == 0);
4025	if (atomic_dec_and_test(v: &eb->refs)) {
4026	if (test_and_clear_bit(nr: EXTENT_BUFFER_IN_TREE, addr: &eb->bflags)) {
4027	struct btrfs_fs_info *fs_info = eb->fs_info;
4028
4029	spin_unlock(lock: &eb->refs_lock);
4030
4031	spin_lock(lock: &fs_info->buffer_lock);
4032	radix_tree_delete(&fs_info->buffer_radix,
4033	eb->start >> fs_info->sectorsize_bits);
4034	spin_unlock(lock: &fs_info->buffer_lock);
4035	} else {
4036	spin_unlock(lock: &eb->refs_lock);
4037	}
4038
4039	btrfs_leak_debug_del_eb(eb);
4040	/* Should be safe to release our pages at this point */
4041	btrfs_release_extent_buffer_pages(eb);
4042	#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4043	if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
4044	__free_extent_buffer(eb);
4045	return 1;
4046	}
4047	#endif
4048	call_rcu(head: &eb->rcu_head, func: btrfs_release_extent_buffer_rcu);
4049	return 1;
4050	}
4051	spin_unlock(lock: &eb->refs_lock);
4052
4053	return 0;
4054	}
4055
4056	void free_extent_buffer(struct extent_buffer *eb)
4057	{
4058	int refs;
4059	if (!eb)
4060	return;
4061
4062	refs = atomic_read(v: &eb->refs);
4063	while (1) {
4064	if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
4065	|| (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
4066	refs == 1))
4067	break;
4068	if (atomic_try_cmpxchg(v: &eb->refs, old: &refs, new: refs - 1))
4069	return;
4070	}
4071
4072	spin_lock(lock: &eb->refs_lock);
4073	if (atomic_read(v: &eb->refs) == 2 &&
4074	test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
4075	!extent_buffer_under_io(eb) &&
4076	test_and_clear_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags))
4077	atomic_dec(v: &eb->refs);
4078
4079	/*
4080	* I know this is terrible, but it's temporary until we stop tracking
4081	* the uptodate bits and such for the extent buffers.
4082	*/
4083	release_extent_buffer(eb);
4084	}
4085
4086	void free_extent_buffer_stale(struct extent_buffer *eb)
4087	{
4088	if (!eb)
4089	return;
4090
4091	spin_lock(lock: &eb->refs_lock);
4092	set_bit(nr: EXTENT_BUFFER_STALE, addr: &eb->bflags);
4093
4094	if (atomic_read(v: &eb->refs) == 2 && !extent_buffer_under_io(eb) &&
4095	test_and_clear_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags))
4096	atomic_dec(v: &eb->refs);
4097	release_extent_buffer(eb);
4098	}
4099
4100	static void btree_clear_folio_dirty(struct folio *folio)
4101	{
4102	ASSERT(folio_test_dirty(folio));
4103	ASSERT(folio_test_locked(folio));
4104	folio_clear_dirty_for_io(folio);
4105	xa_lock_irq(&folio->mapping->i_pages);
4106	if (!folio_test_dirty(folio))
4107	__xa_clear_mark(&folio->mapping->i_pages,
4108	index: folio_index(folio), PAGECACHE_TAG_DIRTY);
4109	xa_unlock_irq(&folio->mapping->i_pages);
4110	}
4111
4112	static void clear_subpage_extent_buffer_dirty(const struct extent_buffer *eb)
4113	{
4114	struct btrfs_fs_info *fs_info = eb->fs_info;
4115	struct folio *folio = eb->folios[0];
4116	bool last;
4117
4118	/* btree_clear_folio_dirty() needs page locked. */
4119	folio_lock(folio);
4120	last = btrfs_subpage_clear_and_test_dirty(fs_info, folio, start: eb->start, len: eb->len);
4121	if (last)
4122	btree_clear_folio_dirty(folio);
4123	folio_unlock(folio);
4124	WARN_ON(atomic_read(&eb->refs) == 0);
4125	}
4126
4127	void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
4128	struct extent_buffer *eb)
4129	{
4130	struct btrfs_fs_info *fs_info = eb->fs_info;
4131	int num_folios;
4132
4133	btrfs_assert_tree_write_locked(eb);
4134
4135	if (trans && btrfs_header_generation(eb) != trans->transid)
4136	return;
4137
4138	/*
4139	* Instead of clearing the dirty flag off of the buffer, mark it as
4140	* EXTENT_BUFFER_ZONED_ZEROOUT. This allows us to preserve
4141	* write-ordering in zoned mode, without the need to later re-dirty
4142	* the extent_buffer.
4143	*
4144	* The actual zeroout of the buffer will happen later in
4145	* btree_csum_one_bio.
4146	*/
4147	if (btrfs_is_zoned(fs_info) && test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
4148	set_bit(nr: EXTENT_BUFFER_ZONED_ZEROOUT, addr: &eb->bflags);
4149	return;
4150	}
4151
4152	if (!test_and_clear_bit(nr: EXTENT_BUFFER_DIRTY, addr: &eb->bflags))
4153	return;
4154
4155	percpu_counter_add_batch(fbc: &fs_info->dirty_metadata_bytes, amount: -eb->len,
4156	batch: fs_info->dirty_metadata_batch);
4157
4158	if (eb->fs_info->nodesize < PAGE_SIZE)
4159	return clear_subpage_extent_buffer_dirty(eb);
4160
4161	num_folios = num_extent_folios(eb);
4162	for (int i = 0; i < num_folios; i++) {
4163	struct folio *folio = eb->folios[i];
4164
4165	if (!folio_test_dirty(folio))
4166	continue;
4167	folio_lock(folio);
4168	btree_clear_folio_dirty(folio);
4169	folio_unlock(folio);
4170	}
4171	WARN_ON(atomic_read(&eb->refs) == 0);
4172	}
4173
4174	void set_extent_buffer_dirty(struct extent_buffer *eb)
4175	{
4176	int num_folios;
4177	bool was_dirty;
4178
4179	check_buffer_tree_ref(eb);
4180
4181	was_dirty = test_and_set_bit(nr: EXTENT_BUFFER_DIRTY, addr: &eb->bflags);
4182
4183	num_folios = num_extent_folios(eb);
4184	WARN_ON(atomic_read(&eb->refs) == 0);
4185	WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
4186	WARN_ON(test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags));
4187
4188	if (!was_dirty) {
4189	bool subpage = eb->fs_info->nodesize < PAGE_SIZE;
4190
4191	/*
4192	* For subpage case, we can have other extent buffers in the
4193	* same page, and in clear_subpage_extent_buffer_dirty() we
4194	* have to clear page dirty without subpage lock held.
4195	* This can cause race where our page gets dirty cleared after
4196	* we just set it.
4197	*
4198	* Thankfully, clear_subpage_extent_buffer_dirty() has locked
4199	* its page for other reasons, we can use page lock to prevent
4200	* the above race.
4201	*/
4202	if (subpage)
4203	lock_page(folio_page(eb->folios[0], 0));
4204	for (int i = 0; i < num_folios; i++)
4205	btrfs_folio_set_dirty(fs_info: eb->fs_info, folio: eb->folios[i],
4206	start: eb->start, len: eb->len);
4207	if (subpage)
4208	unlock_page(folio_page(eb->folios[0], 0));
4209	percpu_counter_add_batch(fbc: &eb->fs_info->dirty_metadata_bytes,
4210	amount: eb->len,
4211	batch: eb->fs_info->dirty_metadata_batch);
4212	}
4213	#ifdef CONFIG_BTRFS_DEBUG
4214	for (int i = 0; i < num_folios; i++)
4215	ASSERT(folio_test_dirty(eb->folios[i]));
4216	#endif
4217	}
4218
4219	void clear_extent_buffer_uptodate(struct extent_buffer *eb)
4220	{
4221	struct btrfs_fs_info *fs_info = eb->fs_info;
4222	int num_folios = num_extent_folios(eb);
4223
4224	clear_bit(nr: EXTENT_BUFFER_UPTODATE, addr: &eb->bflags);
4225	for (int i = 0; i < num_folios; i++) {
4226	struct folio *folio = eb->folios[i];
4227
4228	if (!folio)
4229	continue;
4230
4231	/*
4232	* This is special handling for metadata subpage, as regular
4233	* btrfs_is_subpage() can not handle cloned/dummy metadata.
4234	*/
4235	if (fs_info->nodesize >= PAGE_SIZE)
4236	folio_clear_uptodate(folio);
4237	else
4238	btrfs_subpage_clear_uptodate(fs_info, folio,
4239	start: eb->start, len: eb->len);
4240	}
4241	}
4242
4243	void set_extent_buffer_uptodate(struct extent_buffer *eb)
4244	{
4245	struct btrfs_fs_info *fs_info = eb->fs_info;
4246	int num_folios = num_extent_folios(eb);
4247
4248	set_bit(nr: EXTENT_BUFFER_UPTODATE, addr: &eb->bflags);
4249	for (int i = 0; i < num_folios; i++) {
4250	struct folio *folio = eb->folios[i];
4251
4252	/*
4253	* This is special handling for metadata subpage, as regular
4254	* btrfs_is_subpage() can not handle cloned/dummy metadata.
4255	*/
4256	if (fs_info->nodesize >= PAGE_SIZE)
4257	folio_mark_uptodate(folio);
4258	else
4259	btrfs_subpage_set_uptodate(fs_info, folio,
4260	start: eb->start, len: eb->len);
4261	}
4262	}
4263
4264	static void end_bbio_meta_read(struct btrfs_bio *bbio)
4265	{
4266	struct extent_buffer *eb = bbio->private;
4267	struct btrfs_fs_info *fs_info = eb->fs_info;
4268	bool uptodate = !bbio->bio.bi_status;
4269	struct folio_iter fi;
4270	u32 bio_offset = 0;
4271
4272	eb->read_mirror = bbio->mirror_num;
4273
4274	if (uptodate &&
4275	btrfs_validate_extent_buffer(eb, check: &bbio->parent_check) < 0)
4276	uptodate = false;
4277
4278	if (uptodate) {
4279	set_extent_buffer_uptodate(eb);
4280	} else {
4281	clear_extent_buffer_uptodate(eb);
4282	set_bit(nr: EXTENT_BUFFER_READ_ERR, addr: &eb->bflags);
4283	}
4284
4285	bio_for_each_folio_all(fi, &bbio->bio) {
4286	struct folio *folio = fi.folio;
4287	u64 start = eb->start + bio_offset;
4288	u32 len = fi.length;
4289
4290	if (uptodate)
4291	btrfs_folio_set_uptodate(fs_info, folio, start, len);
4292	else
4293	btrfs_folio_clear_uptodate(fs_info, folio, start, len);
4294
4295	bio_offset += len;
4296	}
4297
4298	clear_bit(nr: EXTENT_BUFFER_READING, addr: &eb->bflags);
4299	smp_mb__after_atomic();
4300	wake_up_bit(word: &eb->bflags, bit: EXTENT_BUFFER_READING);
4301	free_extent_buffer(eb);
4302
4303	bio_put(&bbio->bio);
4304	}
4305
4306	int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num,
4307	struct btrfs_tree_parent_check *check)
4308	{
4309	struct btrfs_bio *bbio;
4310	bool ret;
4311
4312	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
4313	return 0;
4314
4315	/*
4316	* We could have had EXTENT_BUFFER_UPTODATE cleared by the write
4317	* operation, which could potentially still be in flight. In this case
4318	* we simply want to return an error.
4319	*/
4320	if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)))
4321	return -EIO;
4322
4323	/* Someone else is already reading the buffer, just wait for it. */
4324	if (test_and_set_bit(nr: EXTENT_BUFFER_READING, addr: &eb->bflags))
4325	goto done;
4326
4327	/*
4328	* Between the initial test_bit(EXTENT_BUFFER_UPTODATE) and the above
4329	* test_and_set_bit(EXTENT_BUFFER_READING), someone else could have
4330	* started and finished reading the same eb. In this case, UPTODATE
4331	* will now be set, and we shouldn't read it in again.
4332	*/
4333	if (unlikely(test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))) {
4334	clear_bit(nr: EXTENT_BUFFER_READING, addr: &eb->bflags);
4335	smp_mb__after_atomic();
4336	wake_up_bit(word: &eb->bflags, bit: EXTENT_BUFFER_READING);
4337	return 0;
4338	}
4339
4340	clear_bit(nr: EXTENT_BUFFER_READ_ERR, addr: &eb->bflags);
4341	eb->read_mirror = 0;
4342	check_buffer_tree_ref(eb);
4343	atomic_inc(v: &eb->refs);
4344
4345	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
4346	opf: REQ_OP_READ | REQ_META, fs_info: eb->fs_info,
4347	end_io: end_bbio_meta_read, private: eb);
4348	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
4349	bbio->inode = BTRFS_I(inode: eb->fs_info->btree_inode);
4350	bbio->file_offset = eb->start;
4351	memcpy(&bbio->parent_check, check, sizeof(*check));
4352	if (eb->fs_info->nodesize < PAGE_SIZE) {
4353	ret = bio_add_folio(bio: &bbio->bio, folio: eb->folios[0], len: eb->len,
4354	off: eb->start - folio_pos(folio: eb->folios[0]));
4355	ASSERT(ret);
4356	} else {
4357	int num_folios = num_extent_folios(eb);
4358
4359	for (int i = 0; i < num_folios; i++) {
4360	struct folio *folio = eb->folios[i];
4361
4362	ret = bio_add_folio(bio: &bbio->bio, folio, len: eb->folio_size, off: 0);
4363	ASSERT(ret);
4364	}
4365	}
4366	btrfs_submit_bio(bbio, mirror_num);
4367
4368	done:
4369	if (wait == WAIT_COMPLETE) {
4370	wait_on_bit_io(word: &eb->bflags, bit: EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE);
4371	if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
4372	return -EIO;
4373	}
4374
4375	return 0;
4376	}
4377
4378	static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
4379	unsigned long len)
4380	{
4381	btrfs_warn(eb->fs_info,
4382	"access to eb bytenr %llu len %u out of range start %lu len %lu",
4383	eb->start, eb->len, start, len);
4384	WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
4385
4386	return true;
4387	}
4388
4389	/*
4390	* Check if the [start, start + len) range is valid before reading/writing
4391	* the eb.
4392	* NOTE: @start and @len are offset inside the eb, not logical address.
4393	*
4394	* Caller should not touch the dst/src memory if this function returns error.
4395	*/
4396	static inline int check_eb_range(const struct extent_buffer *eb,
4397	unsigned long start, unsigned long len)
4398	{
4399	unsigned long offset;
4400
4401	/* start, start + len should not go beyond eb->len nor overflow */
4402	if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
4403	return report_eb_range(eb, start, len);
4404
4405	return false;
4406	}
4407
4408	void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
4409	unsigned long start, unsigned long len)
4410	{
4411	const int unit_size = eb->folio_size;
4412	size_t cur;
4413	size_t offset;
4414	char *dst = (char *)dstv;
4415	unsigned long i = get_eb_folio_index(eb, offset: start);
4416
4417	if (check_eb_range(eb, start, len)) {
4418	/*
4419	* Invalid range hit, reset the memory, so callers won't get
4420	* some random garbage for their uninitialized memory.
4421	*/
4422	memset(dstv, 0, len);
4423	return;
4424	}
4425
4426	if (eb->addr) {
4427	memcpy(dstv, eb->addr + start, len);
4428	return;
4429	}
4430
4431	offset = get_eb_offset_in_folio(eb, offset: start);
4432
4433	while (len > 0) {
4434	char *kaddr;
4435
4436	cur = min(len, unit_size - offset);
4437	kaddr = folio_address(folio: eb->folios[i]);
4438	memcpy(dst, kaddr + offset, cur);
4439
4440	dst += cur;
4441	len -= cur;
4442	offset = 0;
4443	i++;
4444	}
4445	}
4446
4447	int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
4448	void __user *dstv,
4449	unsigned long start, unsigned long len)
4450	{
4451	const int unit_size = eb->folio_size;
4452	size_t cur;
4453	size_t offset;
4454	char __user *dst = (char __user *)dstv;
4455	unsigned long i = get_eb_folio_index(eb, offset: start);
4456	int ret = 0;
4457
4458	WARN_ON(start > eb->len);
4459	WARN_ON(start + len > eb->start + eb->len);
4460
4461	if (eb->addr) {
4462	if (copy_to_user_nofault(dst: dstv, src: eb->addr + start, size: len))
4463	ret = -EFAULT;
4464	return ret;
4465	}
4466
4467	offset = get_eb_offset_in_folio(eb, offset: start);
4468
4469	while (len > 0) {
4470	char *kaddr;
4471
4472	cur = min(len, unit_size - offset);
4473	kaddr = folio_address(folio: eb->folios[i]);
4474	if (copy_to_user_nofault(dst, src: kaddr + offset, size: cur)) {
4475	ret = -EFAULT;
4476	break;
4477	}
4478
4479	dst += cur;
4480	len -= cur;
4481	offset = 0;
4482	i++;
4483	}
4484
4485	return ret;
4486	}
4487
4488	int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
4489	unsigned long start, unsigned long len)
4490	{
4491	const int unit_size = eb->folio_size;
4492	size_t cur;
4493	size_t offset;
4494	char *kaddr;
4495	char *ptr = (char *)ptrv;
4496	unsigned long i = get_eb_folio_index(eb, offset: start);
4497	int ret = 0;
4498
4499	if (check_eb_range(eb, start, len))
4500	return -EINVAL;
4501
4502	if (eb->addr)
4503	return memcmp(p: ptrv, q: eb->addr + start, size: len);
4504
4505	offset = get_eb_offset_in_folio(eb, offset: start);
4506
4507	while (len > 0) {
4508	cur = min(len, unit_size - offset);
4509	kaddr = folio_address(folio: eb->folios[i]);
4510	ret = memcmp(p: ptr, q: kaddr + offset, size: cur);
4511	if (ret)
4512	break;
4513
4514	ptr += cur;
4515	len -= cur;
4516	offset = 0;
4517	i++;
4518	}
4519	return ret;
4520	}
4521
4522	/*
4523	* Check that the extent buffer is uptodate.
4524	*
4525	* For regular sector size == PAGE_SIZE case, check if @page is uptodate.
4526	* For subpage case, check if the range covered by the eb has EXTENT_UPTODATE.
4527	*/
4528	static void assert_eb_folio_uptodate(const struct extent_buffer *eb, int i)
4529	{
4530	struct btrfs_fs_info *fs_info = eb->fs_info;
4531	struct folio *folio = eb->folios[i];
4532
4533	ASSERT(folio);
4534
4535	/*
4536	* If we are using the commit root we could potentially clear a page
4537	* Uptodate while we're using the extent buffer that we've previously
4538	* looked up. We don't want to complain in this case, as the page was
4539	* valid before, we just didn't write it out. Instead we want to catch
4540	* the case where we didn't actually read the block properly, which
4541	* would have !PageUptodate and !EXTENT_BUFFER_WRITE_ERR.
4542	*/
4543	if (test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
4544	return;
4545
4546	if (fs_info->nodesize < PAGE_SIZE) {
4547	struct folio *folio = eb->folios[0];
4548
4549	ASSERT(i == 0);
4550	if (WARN_ON(!btrfs_subpage_test_uptodate(fs_info, folio,
4551	eb->start, eb->len)))
4552	btrfs_subpage_dump_bitmap(fs_info, folio, start: eb->start, len: eb->len);
4553	} else {
4554	WARN_ON(!folio_test_uptodate(folio));
4555	}
4556	}
4557
4558	static void __write_extent_buffer(const struct extent_buffer *eb,
4559	const void *srcv, unsigned long start,
4560	unsigned long len, bool use_memmove)
4561	{
4562	const int unit_size = eb->folio_size;
4563	size_t cur;
4564	size_t offset;
4565	char *kaddr;
4566	char *src = (char *)srcv;
4567	unsigned long i = get_eb_folio_index(eb, offset: start);
4568	/* For unmapped (dummy) ebs, no need to check their uptodate status. */
4569	const bool check_uptodate = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
4570
4571	if (check_eb_range(eb, start, len))
4572	return;
4573
4574	if (eb->addr) {
4575	if (use_memmove)
4576	memmove(eb->addr + start, srcv, len);
4577	else
4578	memcpy(eb->addr + start, srcv, len);
4579	return;
4580	}
4581
4582	offset = get_eb_offset_in_folio(eb, offset: start);
4583
4584	while (len > 0) {
4585	if (check_uptodate)
4586	assert_eb_folio_uptodate(eb, i);
4587
4588	cur = min(len, unit_size - offset);
4589	kaddr = folio_address(folio: eb->folios[i]);
4590	if (use_memmove)
4591	memmove(kaddr + offset, src, cur);
4592	else
4593	memcpy(kaddr + offset, src, cur);
4594
4595	src += cur;
4596	len -= cur;
4597	offset = 0;
4598	i++;
4599	}
4600	}
4601
4602	void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
4603	unsigned long start, unsigned long len)
4604	{
4605	return __write_extent_buffer(eb, srcv, start, len, use_memmove: false);
4606	}
4607
4608	static void memset_extent_buffer(const struct extent_buffer *eb, int c,
4609	unsigned long start, unsigned long len)
4610	{
4611	const int unit_size = eb->folio_size;
4612	unsigned long cur = start;
4613
4614	if (eb->addr) {
4615	memset(eb->addr + start, c, len);
4616	return;
4617	}
4618
4619	while (cur < start + len) {
4620	unsigned long index = get_eb_folio_index(eb, offset: cur);
4621	unsigned int offset = get_eb_offset_in_folio(eb, offset: cur);
4622	unsigned int cur_len = min(start + len - cur, unit_size - offset);
4623
4624	assert_eb_folio_uptodate(eb, i: index);
4625	memset(folio_address(eb->folios[index]) + offset, c, cur_len);
4626
4627	cur += cur_len;
4628	}
4629	}
4630
4631	void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
4632	unsigned long len)
4633	{
4634	if (check_eb_range(eb, start, len))
4635	return;
4636	return memset_extent_buffer(eb, c: 0, start, len);
4637	}
4638
4639	void copy_extent_buffer_full(const struct extent_buffer *dst,
4640	const struct extent_buffer *src)
4641	{
4642	const int unit_size = src->folio_size;
4643	unsigned long cur = 0;
4644
4645	ASSERT(dst->len == src->len);
4646
4647	while (cur < src->len) {
4648	unsigned long index = get_eb_folio_index(eb: src, offset: cur);
4649	unsigned long offset = get_eb_offset_in_folio(eb: src, offset: cur);
4650	unsigned long cur_len = min(src->len, unit_size - offset);
4651	void *addr = folio_address(folio: src->folios[index]) + offset;
4652
4653	write_extent_buffer(eb: dst, srcv: addr, start: cur, len: cur_len);
4654
4655	cur += cur_len;
4656	}
4657	}
4658
4659	void copy_extent_buffer(const struct extent_buffer *dst,
4660	const struct extent_buffer *src,
4661	unsigned long dst_offset, unsigned long src_offset,
4662	unsigned long len)
4663	{
4664	const int unit_size = dst->folio_size;
4665	u64 dst_len = dst->len;
4666	size_t cur;
4667	size_t offset;
4668	char *kaddr;
4669	unsigned long i = get_eb_folio_index(eb: dst, offset: dst_offset);
4670
4671	if (check_eb_range(eb: dst, start: dst_offset, len) ||
4672	check_eb_range(eb: src, start: src_offset, len))
4673	return;
4674
4675	WARN_ON(src->len != dst_len);
4676
4677	offset = get_eb_offset_in_folio(eb: dst, offset: dst_offset);
4678
4679	while (len > 0) {
4680	assert_eb_folio_uptodate(eb: dst, i);
4681
4682	cur = min(len, (unsigned long)(unit_size - offset));
4683
4684	kaddr = folio_address(folio: dst->folios[i]);
4685	read_extent_buffer(eb: src, dstv: kaddr + offset, start: src_offset, len: cur);
4686
4687	src_offset += cur;
4688	len -= cur;
4689	offset = 0;
4690	i++;
4691	}
4692	}
4693
4694	/*
4695	* Calculate the folio and offset of the byte containing the given bit number.
4696	*
4697	* @eb: the extent buffer
4698	* @start: offset of the bitmap item in the extent buffer
4699	* @nr: bit number
4700	* @folio_index: return index of the folio in the extent buffer that contains
4701	* the given bit number
4702	* @folio_offset: return offset into the folio given by folio_index
4703	*
4704	* This helper hides the ugliness of finding the byte in an extent buffer which
4705	* contains a given bit.
4706	*/
4707	static inline void eb_bitmap_offset(const struct extent_buffer *eb,
4708	unsigned long start, unsigned long nr,
4709	unsigned long *folio_index,
4710	size_t *folio_offset)
4711	{
4712	size_t byte_offset = BIT_BYTE(nr);
4713	size_t offset;
4714
4715	/*
4716	* The byte we want is the offset of the extent buffer + the offset of
4717	* the bitmap item in the extent buffer + the offset of the byte in the
4718	* bitmap item.
4719	*/
4720	offset = start + offset_in_eb_folio(eb, start: eb->start) + byte_offset;
4721
4722	*folio_index = offset >> eb->folio_shift;
4723	*folio_offset = offset_in_eb_folio(eb, start: offset);
4724	}
4725
4726	/*
4727	* Determine whether a bit in a bitmap item is set.
4728	*
4729	* @eb: the extent buffer
4730	* @start: offset of the bitmap item in the extent buffer
4731	* @nr: bit number to test
4732	*/
4733	int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
4734	unsigned long nr)
4735	{
4736	unsigned long i;
4737	size_t offset;
4738	u8 *kaddr;
4739
4740	eb_bitmap_offset(eb, start, nr, folio_index: &i, folio_offset: &offset);
4741	assert_eb_folio_uptodate(eb, i);
4742	kaddr = folio_address(folio: eb->folios[i]);
4743	return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
4744	}
4745
4746	static u8 *extent_buffer_get_byte(const struct extent_buffer *eb, unsigned long bytenr)
4747	{
4748	unsigned long index = get_eb_folio_index(eb, offset: bytenr);
4749
4750	if (check_eb_range(eb, start: bytenr, len: 1))
4751	return NULL;
4752	return folio_address(folio: eb->folios[index]) + get_eb_offset_in_folio(eb, offset: bytenr);
4753	}
4754
4755	/*
4756	* Set an area of a bitmap to 1.
4757	*
4758	* @eb: the extent buffer
4759	* @start: offset of the bitmap item in the extent buffer
4760	* @pos: bit number of the first bit
4761	* @len: number of bits to set
4762	*/
4763	void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
4764	unsigned long pos, unsigned long len)
4765	{
4766	unsigned int first_byte = start + BIT_BYTE(pos);
4767	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4768	const bool same_byte = (first_byte == last_byte);
4769	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4770	u8 *kaddr;
4771
4772	if (same_byte)
4773	mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4774
4775	/* Handle the first byte. */
4776	kaddr = extent_buffer_get_byte(eb, bytenr: first_byte);
4777	*kaddr |= mask;
4778	if (same_byte)
4779	return;
4780
4781	/* Handle the byte aligned part. */
4782	ASSERT(first_byte + 1 <= last_byte);
4783	memset_extent_buffer(eb, c: 0xff, start: first_byte + 1, len: last_byte - first_byte - 1);
4784
4785	/* Handle the last byte. */
4786	kaddr = extent_buffer_get_byte(eb, bytenr: last_byte);
4787	*kaddr |= BITMAP_LAST_BYTE_MASK(pos + len);
4788	}
4789
4790
4791	/*
4792	* Clear an area of a bitmap.
4793	*
4794	* @eb: the extent buffer
4795	* @start: offset of the bitmap item in the extent buffer
4796	* @pos: bit number of the first bit
4797	* @len: number of bits to clear
4798	*/
4799	void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
4800	unsigned long start, unsigned long pos,
4801	unsigned long len)
4802	{
4803	unsigned int first_byte = start + BIT_BYTE(pos);
4804	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4805	const bool same_byte = (first_byte == last_byte);
4806	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4807	u8 *kaddr;
4808
4809	if (same_byte)
4810	mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4811
4812	/* Handle the first byte. */
4813	kaddr = extent_buffer_get_byte(eb, bytenr: first_byte);
4814	*kaddr &= ~mask;
4815	if (same_byte)
4816	return;
4817
4818	/* Handle the byte aligned part. */
4819	ASSERT(first_byte + 1 <= last_byte);
4820	memset_extent_buffer(eb, c: 0, start: first_byte + 1, len: last_byte - first_byte - 1);
4821
4822	/* Handle the last byte. */
4823	kaddr = extent_buffer_get_byte(eb, bytenr: last_byte);
4824	*kaddr &= ~BITMAP_LAST_BYTE_MASK(pos + len);
4825	}
4826
4827	static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4828	{
4829	unsigned long distance = (src > dst) ? src - dst : dst - src;
4830	return distance < len;
4831	}
4832
4833	void memcpy_extent_buffer(const struct extent_buffer *dst,
4834	unsigned long dst_offset, unsigned long src_offset,
4835	unsigned long len)
4836	{
4837	const int unit_size = dst->folio_size;
4838	unsigned long cur_off = 0;
4839
4840	if (check_eb_range(eb: dst, start: dst_offset, len) ||
4841	check_eb_range(eb: dst, start: src_offset, len))
4842	return;
4843
4844	if (dst->addr) {
4845	const bool use_memmove = areas_overlap(src: src_offset, dst: dst_offset, len);
4846
4847	if (use_memmove)
4848	memmove(dst->addr + dst_offset, dst->addr + src_offset, len);
4849	else
4850	memcpy(dst->addr + dst_offset, dst->addr + src_offset, len);
4851	return;
4852	}
4853
4854	while (cur_off < len) {
4855	unsigned long cur_src = cur_off + src_offset;
4856	unsigned long folio_index = get_eb_folio_index(eb: dst, offset: cur_src);
4857	unsigned long folio_off = get_eb_offset_in_folio(eb: dst, offset: cur_src);
4858	unsigned long cur_len = min(src_offset + len - cur_src,
4859	unit_size - folio_off);
4860	void *src_addr = folio_address(folio: dst->folios[folio_index]) + folio_off;
4861	const bool use_memmove = areas_overlap(src: src_offset + cur_off,
4862	dst: dst_offset + cur_off, len: cur_len);
4863
4864	__write_extent_buffer(eb: dst, srcv: src_addr, start: dst_offset + cur_off, len: cur_len,
4865	use_memmove);
4866	cur_off += cur_len;
4867	}
4868	}
4869
4870	void memmove_extent_buffer(const struct extent_buffer *dst,
4871	unsigned long dst_offset, unsigned long src_offset,
4872	unsigned long len)
4873	{
4874	unsigned long dst_end = dst_offset + len - 1;
4875	unsigned long src_end = src_offset + len - 1;
4876
4877	if (check_eb_range(eb: dst, start: dst_offset, len) ||
4878	check_eb_range(eb: dst, start: src_offset, len))
4879	return;
4880
4881	if (dst_offset < src_offset) {
4882	memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4883	return;
4884	}
4885
4886	if (dst->addr) {
4887	memmove(dst->addr + dst_offset, dst->addr + src_offset, len);
4888	return;
4889	}
4890
4891	while (len > 0) {
4892	unsigned long src_i;
4893	size_t cur;
4894	size_t dst_off_in_folio;
4895	size_t src_off_in_folio;
4896	void *src_addr;
4897	bool use_memmove;
4898
4899	src_i = get_eb_folio_index(eb: dst, offset: src_end);
4900
4901	dst_off_in_folio = get_eb_offset_in_folio(eb: dst, offset: dst_end);
4902	src_off_in_folio = get_eb_offset_in_folio(eb: dst, offset: src_end);
4903
4904	cur = min_t(unsigned long, len, src_off_in_folio + 1);
4905	cur = min(cur, dst_off_in_folio + 1);
4906
4907	src_addr = folio_address(folio: dst->folios[src_i]) + src_off_in_folio -
4908	cur + 1;
4909	use_memmove = areas_overlap(src: src_end - cur + 1, dst: dst_end - cur + 1,
4910	len: cur);
4911
4912	__write_extent_buffer(eb: dst, srcv: src_addr, start: dst_end - cur + 1, len: cur,
4913	use_memmove);
4914
4915	dst_end -= cur;
4916	src_end -= cur;
4917	len -= cur;
4918	}
4919	}
4920
4921	#define GANG_LOOKUP_SIZE 16
4922	static struct extent_buffer *get_next_extent_buffer(
4923	struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr)
4924	{
4925	struct extent_buffer *gang[GANG_LOOKUP_SIZE];
4926	struct extent_buffer *found = NULL;
4927	u64 page_start = page_offset(page);
4928	u64 cur = page_start;
4929
4930	ASSERT(in_range(bytenr, page_start, PAGE_SIZE));
4931	lockdep_assert_held(&fs_info->buffer_lock);
4932
4933	while (cur < page_start + PAGE_SIZE) {
4934	int ret;
4935	int i;
4936
4937	ret = radix_tree_gang_lookup(&fs_info->buffer_radix,
4938	results: (void **)gang, first_index: cur >> fs_info->sectorsize_bits,
4939	min_t(unsigned int, GANG_LOOKUP_SIZE,
4940	PAGE_SIZE / fs_info->nodesize));
4941	if (ret == 0)
4942	goto out;
4943	for (i = 0; i < ret; i++) {
4944	/* Already beyond page end */
4945	if (gang[i]->start >= page_start + PAGE_SIZE)
4946	goto out;
4947	/* Found one */
4948	if (gang[i]->start >= bytenr) {
4949	found = gang[i];
4950	goto out;
4951	}
4952	}
4953	cur = gang[ret - 1]->start + gang[ret - 1]->len;
4954	}
4955	out:
4956	return found;
4957	}
4958
4959	static int try_release_subpage_extent_buffer(struct page *page)
4960	{
4961	struct btrfs_fs_info *fs_info = page_to_fs_info(page);
4962	u64 cur = page_offset(page);
4963	const u64 end = page_offset(page) + PAGE_SIZE;
4964	int ret;
4965
4966	while (cur < end) {
4967	struct extent_buffer *eb = NULL;
4968
4969	/*
4970	* Unlike try_release_extent_buffer() which uses folio private
4971	* to grab buffer, for subpage case we rely on radix tree, thus
4972	* we need to ensure radix tree consistency.
4973	*
4974	* We also want an atomic snapshot of the radix tree, thus go
4975	* with spinlock rather than RCU.
4976	*/
4977	spin_lock(lock: &fs_info->buffer_lock);
4978	eb = get_next_extent_buffer(fs_info, page, bytenr: cur);
4979	if (!eb) {
4980	/* No more eb in the page range after or at cur */
4981	spin_unlock(lock: &fs_info->buffer_lock);
4982	break;
4983	}
4984	cur = eb->start + eb->len;
4985
4986	/*
4987	* The same as try_release_extent_buffer(), to ensure the eb
4988	* won't disappear out from under us.
4989	*/
4990	spin_lock(lock: &eb->refs_lock);
4991	if (atomic_read(v: &eb->refs) != 1 || extent_buffer_under_io(eb)) {
4992	spin_unlock(lock: &eb->refs_lock);
4993	spin_unlock(lock: &fs_info->buffer_lock);
4994	break;
4995	}
4996	spin_unlock(lock: &fs_info->buffer_lock);
4997
4998	/*
4999	* If tree ref isn't set then we know the ref on this eb is a
5000	* real ref, so just return, this eb will likely be freed soon
5001	* anyway.
5002	*/
5003	if (!test_and_clear_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags)) {
5004	spin_unlock(lock: &eb->refs_lock);
5005	break;
5006	}
5007
5008	/*
5009	* Here we don't care about the return value, we will always
5010	* check the folio private at the end. And
5011	* release_extent_buffer() will release the refs_lock.
5012	*/
5013	release_extent_buffer(eb);
5014	}
5015	/*
5016	* Finally to check if we have cleared folio private, as if we have
5017	* released all ebs in the page, the folio private should be cleared now.
5018	*/
5019	spin_lock(lock: &page->mapping->i_private_lock);
5020	if (!folio_test_private(page_folio(page)))
5021	ret = 1;
5022	else
5023	ret = 0;
5024	spin_unlock(lock: &page->mapping->i_private_lock);
5025	return ret;
5026
5027	}
5028
5029	int try_release_extent_buffer(struct page *page)
5030	{
5031	struct folio *folio = page_folio(page);
5032	struct extent_buffer *eb;
5033
5034	if (page_to_fs_info(page)->nodesize < PAGE_SIZE)
5035	return try_release_subpage_extent_buffer(page);
5036
5037	/*
5038	* We need to make sure nobody is changing folio private, as we rely on
5039	* folio private as the pointer to extent buffer.
5040	*/
5041	spin_lock(lock: &page->mapping->i_private_lock);
5042	if (!folio_test_private(folio)) {
5043	spin_unlock(lock: &page->mapping->i_private_lock);
5044	return 1;
5045	}
5046
5047	eb = folio_get_private(folio);
5048	BUG_ON(!eb);
5049
5050	/*
5051	* This is a little awful but should be ok, we need to make sure that
5052	* the eb doesn't disappear out from under us while we're looking at
5053	* this page.
5054	*/
5055	spin_lock(lock: &eb->refs_lock);
5056	if (atomic_read(v: &eb->refs) != 1 || extent_buffer_under_io(eb)) {
5057	spin_unlock(lock: &eb->refs_lock);
5058	spin_unlock(lock: &page->mapping->i_private_lock);
5059	return 0;
5060	}
5061	spin_unlock(lock: &page->mapping->i_private_lock);
5062
5063	/*
5064	* If tree ref isn't set then we know the ref on this eb is a real ref,
5065	* so just return, this page will likely be freed soon anyway.
5066	*/
5067	if (!test_and_clear_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags)) {
5068	spin_unlock(lock: &eb->refs_lock);
5069	return 0;
5070	}
5071
5072	return release_extent_buffer(eb);
5073	}
5074
5075	/*
5076	* Attempt to readahead a child block.
5077	*
5078	* @fs_info: the fs_info
5079	* @bytenr: bytenr to read
5080	* @owner_root: objectid of the root that owns this eb
5081	* @gen: generation for the uptodate check, can be 0
5082	* @level: level for the eb
5083	*
5084	* Attempt to readahead a tree block at @bytenr. If @gen is 0 then we do a
5085	* normal uptodate check of the eb, without checking the generation. If we have
5086	* to read the block we will not block on anything.
5087	*/
5088	void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
5089	u64 bytenr, u64 owner_root, u64 gen, int level)
5090	{
5091	struct btrfs_tree_parent_check check = {
5092	.has_first_key = 0,
5093	.level = level,
5094	.transid = gen
5095	};
5096	struct extent_buffer *eb;
5097	int ret;
5098
5099	eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
5100	if (IS_ERR(ptr: eb))
5101	return;
5102
5103	if (btrfs_buffer_uptodate(buf: eb, parent_transid: gen, atomic: 1)) {
5104	free_extent_buffer(eb);
5105	return;
5106	}
5107
5108	ret = read_extent_buffer_pages(eb, WAIT_NONE, mirror_num: 0, check: &check);
5109	if (ret < 0)
5110	free_extent_buffer_stale(eb);
5111	else
5112	free_extent_buffer(eb);
5113	}
5114
5115	/*
5116	* Readahead a node's child block.
5117	*
5118	* @node: parent node we're reading from
5119	* @slot: slot in the parent node for the child we want to read
5120	*
5121	* A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at
5122	* the slot in the node provided.
5123	*/
5124	void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
5125	{
5126	btrfs_readahead_tree_block(fs_info: node->fs_info,
5127	bytenr: btrfs_node_blockptr(eb: node, nr: slot),
5128	owner_root: btrfs_header_owner(eb: node),
5129	gen: btrfs_node_ptr_generation(eb: node, nr: slot),
5130	level: btrfs_header_level(eb: node) - 1);
5131	}
5132