1	// SPDX-License-Identifier: GPL-2.0
2
3	#include <linux/bitops.h>
4	#include <linux/slab.h>
5	#include <linux/bio.h>
6	#include <linux/mm.h>
7	#include <linux/pagemap.h>
8	#include <linux/page-flags.h>
9	#include <linux/sched/mm.h>
10	#include <linux/spinlock.h>
11	#include <linux/blkdev.h>
12	#include <linux/swap.h>
13	#include <linux/writeback.h>
14	#include <linux/pagevec.h>
15	#include <linux/prefetch.h>
16	#include <linux/fsverity.h>
17	#include "misc.h"
18	#include "extent_io.h"
19	#include "extent-io-tree.h"
20	#include "extent_map.h"
21	#include "ctree.h"
22	#include "btrfs_inode.h"
23	#include "bio.h"
24	#include "locking.h"
25	#include "rcu-string.h"
26	#include "backref.h"
27	#include "disk-io.h"
28	#include "subpage.h"
29	#include "zoned.h"
30	#include "block-group.h"
31	#include "compression.h"
32	#include "fs.h"
33	#include "accessors.h"
34	#include "file-item.h"
35	#include "file.h"
36	#include "dev-replace.h"
37	#include "super.h"
38	#include "transaction.h"
39
40	static struct kmem_cache *extent_buffer_cache;
41
42	#ifdef CONFIG_BTRFS_DEBUG
43	static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb)
44	{
45	struct btrfs_fs_info *fs_info = eb->fs_info;
46	unsigned long flags;
47
48	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
49	list_add(new: &eb->leak_list, head: &fs_info->allocated_ebs);
50	spin_unlock_irqrestore(lock: &fs_info->eb_leak_lock, flags);
51	}
52
53	static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb)
54	{
55	struct btrfs_fs_info *fs_info = eb->fs_info;
56	unsigned long flags;
57
58	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
59	list_del(entry: &eb->leak_list);
60	spin_unlock_irqrestore(lock: &fs_info->eb_leak_lock, flags);
61	}
62
63	void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
64	{
65	struct extent_buffer *eb;
66	unsigned long flags;
67
68	/*
69	* If we didn't get into open_ctree our allocated_ebs will not be
70	* initialized, so just skip this.
71	*/
72	if (!fs_info->allocated_ebs.next)
73	return;
74
75	WARN_ON(!list_empty(&fs_info->allocated_ebs));
76	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
77	while (!list_empty(head: &fs_info->allocated_ebs)) {
78	eb = list_first_entry(&fs_info->allocated_ebs,
79	struct extent_buffer, leak_list);
80	pr_err(
81	"BTRFS: buffer leak start %llu len %lu refs %d bflags %lu owner %llu\n",
82	eb->start, eb->len, atomic_read(&eb->refs), eb->bflags,
83	btrfs_header_owner(eb));
84	list_del(entry: &eb->leak_list);
85	kmem_cache_free(s: extent_buffer_cache, objp: eb);
86	}
87	spin_unlock_irqrestore(lock: &fs_info->eb_leak_lock, flags);
88	}
89	#else
90	#define btrfs_leak_debug_add_eb(eb) do {} while (0)
91	#define btrfs_leak_debug_del_eb(eb) do {} while (0)
92	#endif
93
94	/*
95	* Structure to record info about the bio being assembled, and other info like
96	* how many bytes are there before stripe/ordered extent boundary.
97	*/
98	struct btrfs_bio_ctrl {
99	struct btrfs_bio *bbio;
100	enum btrfs_compression_type compress_type;
101	u32 len_to_oe_boundary;
102	blk_opf_t opf;
103	btrfs_bio_end_io_t end_io_func;
104	struct writeback_control *wbc;
105	};
106
107	static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl)
108	{
109	struct btrfs_bio *bbio = bio_ctrl->bbio;
110
111	if (!bbio)
112	return;
113
114	/* Caller should ensure the bio has at least some range added */
115	ASSERT(bbio->bio.bi_iter.bi_size);
116
117	if (btrfs_op(bio: &bbio->bio) == BTRFS_MAP_READ &&
118	bio_ctrl->compress_type != BTRFS_COMPRESS_NONE)
119	btrfs_submit_compressed_read(bbio);
120	else
121	btrfs_submit_bio(bbio, mirror_num: 0);
122
123	/* The bbio is owned by the end_io handler now */
124	bio_ctrl->bbio = NULL;
125	}
126
127	/*
128	* Submit or fail the current bio in the bio_ctrl structure.
129	*/
130	static void submit_write_bio(struct btrfs_bio_ctrl *bio_ctrl, int ret)
131	{
132	struct btrfs_bio *bbio = bio_ctrl->bbio;
133
134	if (!bbio)
135	return;
136
137	if (ret) {
138	ASSERT(ret < 0);
139	btrfs_bio_end_io(bbio, status: errno_to_blk_status(errno: ret));
140	/* The bio is owned by the end_io handler now */
141	bio_ctrl->bbio = NULL;
142	} else {
143	submit_one_bio(bio_ctrl);
144	}
145	}
146
147	int __init extent_buffer_init_cachep(void)
148	{
149	extent_buffer_cache = kmem_cache_create(name: "btrfs_extent_buffer",
150	size: sizeof(struct extent_buffer), align: 0,
151	SLAB_MEM_SPREAD, NULL);
152	if (!extent_buffer_cache)
153	return -ENOMEM;
154
155	return 0;
156	}
157
158	void __cold extent_buffer_free_cachep(void)
159	{
160	/*
161	* Make sure all delayed rcu free are flushed before we
162	* destroy caches.
163	*/
164	rcu_barrier();
165	kmem_cache_destroy(s: extent_buffer_cache);
166	}
167
168	void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
169	{
170	unsigned long index = start >> PAGE_SHIFT;
171	unsigned long end_index = end >> PAGE_SHIFT;
172	struct page *page;
173
174	while (index <= end_index) {
175	page = find_get_page(mapping: inode->i_mapping, offset: index);
176	BUG_ON(!page); /* Pages should be in the extent_io_tree */
177	clear_page_dirty_for_io(page);
178	put_page(page);
179	index++;
180	}
181	}
182
183	static void process_one_page(struct btrfs_fs_info *fs_info,
184	struct page *page, struct page *locked_page,
185	unsigned long page_ops, u64 start, u64 end)
186	{
187	u32 len;
188
189	ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX);
190	len = end + 1 - start;
191
192	if (page_ops & PAGE_SET_ORDERED)
193	btrfs_page_clamp_set_ordered(fs_info, page, start, len);
194	if (page_ops & PAGE_START_WRITEBACK) {
195	btrfs_page_clamp_clear_dirty(fs_info, page, start, len);
196	btrfs_page_clamp_set_writeback(fs_info, page, start, len);
197	}
198	if (page_ops & PAGE_END_WRITEBACK)
199	btrfs_page_clamp_clear_writeback(fs_info, page, start, len);
200
201	if (page != locked_page && (page_ops & PAGE_UNLOCK))
202	btrfs_page_end_writer_lock(fs_info, page, start, len);
203	}
204
205	static void __process_pages_contig(struct address_space *mapping,
206	struct page *locked_page, u64 start, u64 end,
207	unsigned long page_ops)
208	{
209	struct btrfs_fs_info *fs_info = btrfs_sb(sb: mapping->host->i_sb);
210	pgoff_t start_index = start >> PAGE_SHIFT;
211	pgoff_t end_index = end >> PAGE_SHIFT;
212	pgoff_t index = start_index;
213	struct folio_batch fbatch;
214	int i;
215
216	folio_batch_init(fbatch: &fbatch);
217	while (index <= end_index) {
218	int found_folios;
219
220	found_folios = filemap_get_folios_contig(mapping, start: &index,
221	end: end_index, fbatch: &fbatch);
222	for (i = 0; i < found_folios; i++) {
223	struct folio *folio = fbatch.folios[i];
224
225	process_one_page(fs_info, page: &folio->page, locked_page,
226	page_ops, start, end);
227	}
228	folio_batch_release(fbatch: &fbatch);
229	cond_resched();
230	}
231	}
232
233	static noinline void __unlock_for_delalloc(struct inode *inode,
234	struct page *locked_page,
235	u64 start, u64 end)
236	{
237	unsigned long index = start >> PAGE_SHIFT;
238	unsigned long end_index = end >> PAGE_SHIFT;
239
240	ASSERT(locked_page);
241	if (index == locked_page->index && end_index == index)
242	return;
243
244	__process_pages_contig(mapping: inode->i_mapping, locked_page, start, end,
245	page_ops: PAGE_UNLOCK);
246	}
247
248	static noinline int lock_delalloc_pages(struct inode *inode,
249	struct page *locked_page,
250	u64 start,
251	u64 end)
252	{
253	struct btrfs_fs_info *fs_info = btrfs_sb(sb: inode->i_sb);
254	struct address_space *mapping = inode->i_mapping;
255	pgoff_t start_index = start >> PAGE_SHIFT;
256	pgoff_t end_index = end >> PAGE_SHIFT;
257	pgoff_t index = start_index;
258	u64 processed_end = start;
259	struct folio_batch fbatch;
260
261	if (index == locked_page->index && index == end_index)
262	return 0;
263
264	folio_batch_init(fbatch: &fbatch);
265	while (index <= end_index) {
266	unsigned int found_folios, i;
267
268	found_folios = filemap_get_folios_contig(mapping, start: &index,
269	end: end_index, fbatch: &fbatch);
270	if (found_folios == 0)
271	goto out;
272
273	for (i = 0; i < found_folios; i++) {
274	struct page *page = &fbatch.folios[i]->page;
275	u32 len = end + 1 - start;
276
277	if (page == locked_page)
278	continue;
279
280	if (btrfs_page_start_writer_lock(fs_info, page, start,
281	len))
282	goto out;
283
284	if (!PageDirty(page) || page->mapping != mapping) {
285	btrfs_page_end_writer_lock(fs_info, page, start,
286	len);
287	goto out;
288	}
289
290	processed_end = page_offset(page) + PAGE_SIZE - 1;
291	}
292	folio_batch_release(fbatch: &fbatch);
293	cond_resched();
294	}
295
296	return 0;
297	out:
298	folio_batch_release(fbatch: &fbatch);
299	if (processed_end > start)
300	__unlock_for_delalloc(inode, locked_page, start, end: processed_end);
301	return -EAGAIN;
302	}
303
304	/*
305	* Find and lock a contiguous range of bytes in the file marked as delalloc, no
306	* more than @max_bytes.
307	*
308	* @start: The original start bytenr to search.
309	* Will store the extent range start bytenr.
310	* @end: The original end bytenr of the search range
311	* Will store the extent range end bytenr.
312	*
313	* Return true if we find a delalloc range which starts inside the original
314	* range, and @start/@end will store the delalloc range start/end.
315	*
316	* Return false if we can't find any delalloc range which starts inside the
317	* original range, and @start/@end will be the non-delalloc range start/end.
318	*/
319	EXPORT_FOR_TESTS
320	noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
321	struct page *locked_page, u64 *start,
322	u64 *end)
323	{
324	struct btrfs_fs_info *fs_info = btrfs_sb(sb: inode->i_sb);
325	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
326	const u64 orig_start = *start;
327	const u64 orig_end = *end;
328	/* The sanity tests may not set a valid fs_info. */
329	u64 max_bytes = fs_info ? fs_info->max_extent_size : BTRFS_MAX_EXTENT_SIZE;
330	u64 delalloc_start;
331	u64 delalloc_end;
332	bool found;
333	struct extent_state *cached_state = NULL;
334	int ret;
335	int loops = 0;
336
337	/* Caller should pass a valid @end to indicate the search range end */
338	ASSERT(orig_end > orig_start);
339
340	/* The range should at least cover part of the page */
341	ASSERT(!(orig_start >= page_offset(locked_page) + PAGE_SIZE ||
342	orig_end <= page_offset(locked_page)));
343	again:
344	/* step one, find a bunch of delalloc bytes starting at start */
345	delalloc_start = *start;
346	delalloc_end = 0;
347	found = btrfs_find_delalloc_range(tree, start: &delalloc_start, end: &delalloc_end,
348	max_bytes, cached_state: &cached_state);
349	if (!found || delalloc_end <= *start || delalloc_start > orig_end) {
350	*start = delalloc_start;
351
352	/* @delalloc_end can be -1, never go beyond @orig_end */
353	*end = min(delalloc_end, orig_end);
354	free_extent_state(state: cached_state);
355	return false;
356	}
357
358	/*
359	* start comes from the offset of locked_page. We have to lock
360	* pages in order, so we can't process delalloc bytes before
361	* locked_page
362	*/
363	if (delalloc_start < *start)
364	delalloc_start = *start;
365
366	/*
367	* make sure to limit the number of pages we try to lock down
368	*/
369	if (delalloc_end + 1 - delalloc_start > max_bytes)
370	delalloc_end = delalloc_start + max_bytes - 1;
371
372	/* step two, lock all the pages after the page that has start */
373	ret = lock_delalloc_pages(inode, locked_page,
374	start: delalloc_start, end: delalloc_end);
375	ASSERT(!ret || ret == -EAGAIN);
376	if (ret == -EAGAIN) {
377	/* some of the pages are gone, lets avoid looping by
378	* shortening the size of the delalloc range we're searching
379	*/
380	free_extent_state(state: cached_state);
381	cached_state = NULL;
382	if (!loops) {
383	max_bytes = PAGE_SIZE;
384	loops = 1;
385	goto again;
386	} else {
387	found = false;
388	goto out_failed;
389	}
390	}
391
392	/* step three, lock the state bits for the whole range */
393	lock_extent(tree, start: delalloc_start, end: delalloc_end, cached: &cached_state);
394
395	/* then test to make sure it is all still delalloc */
396	ret = test_range_bit(tree, start: delalloc_start, end: delalloc_end,
397	bit: EXTENT_DELALLOC, cached_state);
398	if (!ret) {
399	unlock_extent(tree, start: delalloc_start, end: delalloc_end,
400	cached: &cached_state);
401	__unlock_for_delalloc(inode, locked_page,
402	start: delalloc_start, end: delalloc_end);
403	cond_resched();
404	goto again;
405	}
406	free_extent_state(state: cached_state);
407	*start = delalloc_start;
408	*end = delalloc_end;
409	out_failed:
410	return found;
411	}
412
413	void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
414	struct page *locked_page,
415	u32 clear_bits, unsigned long page_ops)
416	{
417	clear_extent_bit(tree: &inode->io_tree, start, end, bits: clear_bits, NULL);
418
419	__process_pages_contig(mapping: inode->vfs_inode.i_mapping, locked_page,
420	start, end, page_ops);
421	}
422
423	static bool btrfs_verify_page(struct page *page, u64 start)
424	{
425	if (!fsverity_active(inode: page->mapping->host) ||
426	PageUptodate(page) ||
427	start >= i_size_read(inode: page->mapping->host))
428	return true;
429	return fsverity_verify_page(page);
430	}
431
432	static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len)
433	{
434	struct btrfs_fs_info *fs_info = btrfs_sb(sb: page->mapping->host->i_sb);
435
436	ASSERT(page_offset(page) <= start &&
437	start + len <= page_offset(page) + PAGE_SIZE);
438
439	if (uptodate && btrfs_verify_page(page, start))
440	btrfs_page_set_uptodate(fs_info, page, start, len);
441	else
442	btrfs_page_clear_uptodate(fs_info, page, start, len);
443
444	if (!btrfs_is_subpage(fs_info, page))
445	unlock_page(page);
446	else
447	btrfs_subpage_end_reader(fs_info, page, start, len);
448	}
449
450	/*
451	* after a writepage IO is done, we need to:
452	* clear the uptodate bits on error
453	* clear the writeback bits in the extent tree for this IO
454	* end_page_writeback if the page has no more pending IO
455	*
456	* Scheduling is not allowed, so the extent state tree is expected
457	* to have one and only one object corresponding to this IO.
458	*/
459	static void end_bio_extent_writepage(struct btrfs_bio *bbio)
460	{
461	struct bio *bio = &bbio->bio;
462	int error = blk_status_to_errno(status: bio->bi_status);
463	struct bio_vec *bvec;
464	struct bvec_iter_all iter_all;
465
466	ASSERT(!bio_flagged(bio, BIO_CLONED));
467	bio_for_each_segment_all(bvec, bio, iter_all) {
468	struct page *page = bvec->bv_page;
469	struct inode *inode = page->mapping->host;
470	struct btrfs_fs_info *fs_info = btrfs_sb(sb: inode->i_sb);
471	const u32 sectorsize = fs_info->sectorsize;
472	u64 start = page_offset(page) + bvec->bv_offset;
473	u32 len = bvec->bv_len;
474
475	/* Our read/write should always be sector aligned. */
476	if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
477	btrfs_err(fs_info,
478	"partial page write in btrfs with offset %u and length %u",
479	bvec->bv_offset, bvec->bv_len);
480	else if (!IS_ALIGNED(bvec->bv_len, sectorsize))
481	btrfs_info(fs_info,
482	"incomplete page write with offset %u and length %u",
483	bvec->bv_offset, bvec->bv_len);
484
485	btrfs_finish_ordered_extent(ordered: bbio->ordered, page, file_offset: start, len, uptodate: !error);
486	if (error)
487	mapping_set_error(mapping: page->mapping, error);
488	btrfs_page_clear_writeback(fs_info, page, start, len);
489	}
490
491	bio_put(bio);
492	}
493
494	/*
495	* Record previously processed extent range
496	*
497	* For endio_readpage_release_extent() to handle a full extent range, reducing
498	* the extent io operations.
499	*/
500	struct processed_extent {
501	struct btrfs_inode *inode;
502	/* Start of the range in @inode */
503	u64 start;
504	/* End of the range in @inode */
505	u64 end;
506	bool uptodate;
507	};
508
509	/*
510	* Try to release processed extent range
511	*
512	* May not release the extent range right now if the current range is
513	* contiguous to processed extent.
514	*
515	* Will release processed extent when any of @inode, @uptodate, the range is
516	* no longer contiguous to the processed range.
517	*
518	* Passing @inode == NULL will force processed extent to be released.
519	*/
520	static void endio_readpage_release_extent(struct processed_extent *processed,
521	struct btrfs_inode *inode, u64 start, u64 end,
522	bool uptodate)
523	{
524	struct extent_state *cached = NULL;
525	struct extent_io_tree *tree;
526
527	/* The first extent, initialize @processed */
528	if (!processed->inode)
529	goto update;
530
531	/*
532	* Contiguous to processed extent, just uptodate the end.
533	*
534	* Several things to notice:
535	*
536	* - bio can be merged as long as on-disk bytenr is contiguous
537	* This means we can have page belonging to other inodes, thus need to
538	* check if the inode still matches.
539	* - bvec can contain range beyond current page for multi-page bvec
540	* Thus we need to do processed->end + 1 >= start check
541	*/
542	if (processed->inode == inode && processed->uptodate == uptodate &&
543	processed->end + 1 >= start && end >= processed->end) {
544	processed->end = end;
545	return;
546	}
547
548	tree = &processed->inode->io_tree;
549	/*
550	* Now we don't have range contiguous to the processed range, release
551	* the processed range now.
552	*/
553	unlock_extent(tree, start: processed->start, end: processed->end, cached: &cached);
554
555	update:
556	/* Update processed to current range */
557	processed->inode = inode;
558	processed->start = start;
559	processed->end = end;
560	processed->uptodate = uptodate;
561	}
562
563	static void begin_page_read(struct btrfs_fs_info *fs_info, struct page *page)
564	{
565	ASSERT(PageLocked(page));
566	if (!btrfs_is_subpage(fs_info, page))
567	return;
568
569	ASSERT(PagePrivate(page));
570	btrfs_subpage_start_reader(fs_info, page, start: page_offset(page), PAGE_SIZE);
571	}
572
573	/*
574	* after a readpage IO is done, we need to:
575	* clear the uptodate bits on error
576	* set the uptodate bits if things worked
577	* set the page up to date if all extents in the tree are uptodate
578	* clear the lock bit in the extent tree
579	* unlock the page if there are no other extents locked for it
580	*
581	* Scheduling is not allowed, so the extent state tree is expected
582	* to have one and only one object corresponding to this IO.
583	*/
584	static void end_bio_extent_readpage(struct btrfs_bio *bbio)
585	{
586	struct bio *bio = &bbio->bio;
587	struct bio_vec *bvec;
588	struct processed_extent processed = { 0 };
589	/*
590	* The offset to the beginning of a bio, since one bio can never be
591	* larger than UINT_MAX, u32 here is enough.
592	*/
593	u32 bio_offset = 0;
594	struct bvec_iter_all iter_all;
595
596	ASSERT(!bio_flagged(bio, BIO_CLONED));
597	bio_for_each_segment_all(bvec, bio, iter_all) {
598	bool uptodate = !bio->bi_status;
599	struct page *page = bvec->bv_page;
600	struct inode *inode = page->mapping->host;
601	struct btrfs_fs_info *fs_info = btrfs_sb(sb: inode->i_sb);
602	const u32 sectorsize = fs_info->sectorsize;
603	u64 start;
604	u64 end;
605	u32 len;
606
607	btrfs_debug(fs_info,
608	"end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
609	bio->bi_iter.bi_sector, bio->bi_status,
610	bbio->mirror_num);
611
612	/*
613	* We always issue full-sector reads, but if some block in a
614	* page fails to read, blk_update_request() will advance
615	* bv_offset and adjust bv_len to compensate. Print a warning
616	* for unaligned offsets, and an error if they don't add up to
617	* a full sector.
618	*/
619	if (!IS_ALIGNED(bvec->bv_offset, sectorsize))
620	btrfs_err(fs_info,
621	"partial page read in btrfs with offset %u and length %u",
622	bvec->bv_offset, bvec->bv_len);
623	else if (!IS_ALIGNED(bvec->bv_offset + bvec->bv_len,
624	sectorsize))
625	btrfs_info(fs_info,
626	"incomplete page read with offset %u and length %u",
627	bvec->bv_offset, bvec->bv_len);
628
629	start = page_offset(page) + bvec->bv_offset;
630	end = start + bvec->bv_len - 1;
631	len = bvec->bv_len;
632
633	if (likely(uptodate)) {
634	loff_t i_size = i_size_read(inode);
635	pgoff_t end_index = i_size >> PAGE_SHIFT;
636
637	/*
638	* Zero out the remaining part if this range straddles
639	* i_size.
640	*
641	* Here we should only zero the range inside the bvec,
642	* not touch anything else.
643	*
644	* NOTE: i_size is exclusive while end is inclusive.
645	*/
646	if (page->index == end_index && i_size <= end) {
647	u32 zero_start = max(offset_in_page(i_size),
648	offset_in_page(start));
649
650	zero_user_segment(page, start: zero_start,
651	offset_in_page(end) + 1);
652	}
653	}
654
655	/* Update page status and unlock. */
656	end_page_read(page, uptodate, start, len);
657	endio_readpage_release_extent(processed: &processed, inode: BTRFS_I(inode),
658	start, end, uptodate);
659
660	ASSERT(bio_offset + len > bio_offset);
661	bio_offset += len;
662
663	}
664	/* Release the last extent */
665	endio_readpage_release_extent(processed: &processed, NULL, start: 0, end: 0, uptodate: false);
666	bio_put(bio);
667	}
668
669	/*
670	* Populate every free slot in a provided array with pages.
671	*
672	* @nr_pages: number of pages to allocate
673	* @page_array: the array to fill with pages; any existing non-null entries in
674	* the array will be skipped
675	*
676	* Return: 0 if all pages were able to be allocated;
677	* -ENOMEM otherwise, and the caller is responsible for freeing all
678	* non-null page pointers in the array.
679	*/
680	int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array)
681	{
682	unsigned int allocated;
683
684	for (allocated = 0; allocated < nr_pages;) {
685	unsigned int last = allocated;
686
687	allocated = alloc_pages_bulk_array(GFP_NOFS, nr_pages, page_array);
688
689	if (allocated == nr_pages)
690	return 0;
691
692	/*
693	* During this iteration, no page could be allocated, even
694	* though alloc_pages_bulk_array() falls back to alloc_page()
695	* if it could not bulk-allocate. So we must be out of memory.
696	*/
697	if (allocated == last)
698	return -ENOMEM;
699
700	memalloc_retry_wait(GFP_NOFS);
701	}
702	return 0;
703	}
704
705	static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl,
706	struct page *page, u64 disk_bytenr,
707	unsigned int pg_offset)
708	{
709	struct bio *bio = &bio_ctrl->bbio->bio;
710	struct bio_vec *bvec = bio_last_bvec_all(bio);
711	const sector_t sector = disk_bytenr >> SECTOR_SHIFT;
712
713	if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) {
714	/*
715	* For compression, all IO should have its logical bytenr set
716	* to the starting bytenr of the compressed extent.
717	*/
718	return bio->bi_iter.bi_sector == sector;
719	}
720
721	/*
722	* The contig check requires the following conditions to be met:
723	*
724	* 1) The pages are belonging to the same inode
725	* This is implied by the call chain.
726	*
727	* 2) The range has adjacent logical bytenr
728	*
729	* 3) The range has adjacent file offset
730	* This is required for the usage of btrfs_bio->file_offset.
731	*/
732	return bio_end_sector(bio) == sector &&
733	page_offset(page: bvec->bv_page) + bvec->bv_offset + bvec->bv_len ==
734	page_offset(page) + pg_offset;
735	}
736
737	static void alloc_new_bio(struct btrfs_inode *inode,
738	struct btrfs_bio_ctrl *bio_ctrl,
739	u64 disk_bytenr, u64 file_offset)
740	{
741	struct btrfs_fs_info *fs_info = inode->root->fs_info;
742	struct btrfs_bio *bbio;
743
744	bbio = btrfs_bio_alloc(BIO_MAX_VECS, opf: bio_ctrl->opf, fs_info,
745	end_io: bio_ctrl->end_io_func, NULL);
746	bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
747	bbio->inode = inode;
748	bbio->file_offset = file_offset;
749	bio_ctrl->bbio = bbio;
750	bio_ctrl->len_to_oe_boundary = U32_MAX;
751
752	/* Limit data write bios to the ordered boundary. */
753	if (bio_ctrl->wbc) {
754	struct btrfs_ordered_extent *ordered;
755
756	ordered = btrfs_lookup_ordered_extent(inode, file_offset);
757	if (ordered) {
758	bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX,
759	ordered->file_offset +
760	ordered->disk_num_bytes - file_offset);
761	bbio->ordered = ordered;
762	}
763
764	/*
765	* Pick the last added device to support cgroup writeback. For
766	* multi-device file systems this means blk-cgroup policies have
767	* to always be set on the last added/replaced device.
768	* This is a bit odd but has been like that for a long time.
769	*/
770	bio_set_dev(bio: &bbio->bio, bdev: fs_info->fs_devices->latest_dev->bdev);
771	wbc_init_bio(wbc: bio_ctrl->wbc, bio: &bbio->bio);
772	}
773	}
774
775	/*
776	* @disk_bytenr: logical bytenr where the write will be
777	* @page: page to add to the bio
778	* @size: portion of page that we want to write to
779	* @pg_offset: offset of the new bio or to check whether we are adding
780	* a contiguous page to the previous one
781	*
782	* The will either add the page into the existing @bio_ctrl->bbio, or allocate a
783	* new one in @bio_ctrl->bbio.
784	* The mirror number for this IO should already be initizlied in
785	* @bio_ctrl->mirror_num.
786	*/
787	static void submit_extent_page(struct btrfs_bio_ctrl *bio_ctrl,
788	u64 disk_bytenr, struct page *page,
789	size_t size, unsigned long pg_offset)
790	{
791	struct btrfs_inode *inode = BTRFS_I(inode: page->mapping->host);
792
793	ASSERT(pg_offset + size <= PAGE_SIZE);
794	ASSERT(bio_ctrl->end_io_func);
795
796	if (bio_ctrl->bbio &&
797	!btrfs_bio_is_contig(bio_ctrl, page, disk_bytenr, pg_offset))
798	submit_one_bio(bio_ctrl);
799
800	do {
801	u32 len = size;
802
803	/* Allocate new bio if needed */
804	if (!bio_ctrl->bbio) {
805	alloc_new_bio(inode, bio_ctrl, disk_bytenr,
806	file_offset: page_offset(page) + pg_offset);
807	}
808
809	/* Cap to the current ordered extent boundary if there is one. */
810	if (len > bio_ctrl->len_to_oe_boundary) {
811	ASSERT(bio_ctrl->compress_type == BTRFS_COMPRESS_NONE);
812	ASSERT(is_data_inode(&inode->vfs_inode));
813	len = bio_ctrl->len_to_oe_boundary;
814	}
815
816	if (bio_add_page(bio: &bio_ctrl->bbio->bio, page, len, off: pg_offset) != len) {
817	/* bio full: move on to a new one */
818	submit_one_bio(bio_ctrl);
819	continue;
820	}
821
822	if (bio_ctrl->wbc)
823	wbc_account_cgroup_owner(wbc: bio_ctrl->wbc, page, bytes: len);
824
825	size -= len;
826	pg_offset += len;
827	disk_bytenr += len;
828
829	/*
830	* len_to_oe_boundary defaults to U32_MAX, which isn't page or
831	* sector aligned. alloc_new_bio() then sets it to the end of
832	* our ordered extent for writes into zoned devices.
833	*
834	* When len_to_oe_boundary is tracking an ordered extent, we
835	* trust the ordered extent code to align things properly, and
836	* the check above to cap our write to the ordered extent
837	* boundary is correct.
838	*
839	* When len_to_oe_boundary is U32_MAX, the cap above would
840	* result in a 4095 byte IO for the last page right before
841	* we hit the bio limit of UINT_MAX. bio_add_page() has all
842	* the checks required to make sure we don't overflow the bio,
843	* and we should just ignore len_to_oe_boundary completely
844	* unless we're using it to track an ordered extent.
845	*
846	* It's pretty hard to make a bio sized U32_MAX, but it can
847	* happen when the page cache is able to feed us contiguous
848	* pages for large extents.
849	*/
850	if (bio_ctrl->len_to_oe_boundary != U32_MAX)
851	bio_ctrl->len_to_oe_boundary -= len;
852
853	/* Ordered extent boundary: move on to a new bio. */
854	if (bio_ctrl->len_to_oe_boundary == 0)
855	submit_one_bio(bio_ctrl);
856	} while (size);
857	}
858
859	static int attach_extent_buffer_page(struct extent_buffer *eb,
860	struct page *page,
861	struct btrfs_subpage *prealloc)
862	{
863	struct btrfs_fs_info *fs_info = eb->fs_info;
864	int ret = 0;
865
866	/*
867	* If the page is mapped to btree inode, we should hold the private
868	* lock to prevent race.
869	* For cloned or dummy extent buffers, their pages are not mapped and
870	* will not race with any other ebs.
871	*/
872	if (page->mapping)
873	lockdep_assert_held(&page->mapping->private_lock);
874
875	if (fs_info->nodesize >= PAGE_SIZE) {
876	if (!PagePrivate(page))
877	attach_page_private(page, data: eb);
878	else
879	WARN_ON(page->private != (unsigned long)eb);
880	return 0;
881	}
882
883	/* Already mapped, just free prealloc */
884	if (PagePrivate(page)) {
885	btrfs_free_subpage(subpage: prealloc);
886	return 0;
887	}
888
889	if (prealloc)
890	/* Has preallocated memory for subpage */
891	attach_page_private(page, data: prealloc);
892	else
893	/* Do new allocation to attach subpage */
894	ret = btrfs_attach_subpage(fs_info, page,
895	type: BTRFS_SUBPAGE_METADATA);
896	return ret;
897	}
898
899	int set_page_extent_mapped(struct page *page)
900	{
901	struct btrfs_fs_info *fs_info;
902
903	ASSERT(page->mapping);
904
905	if (PagePrivate(page))
906	return 0;
907
908	fs_info = btrfs_sb(sb: page->mapping->host->i_sb);
909
910	if (btrfs_is_subpage(fs_info, page))
911	return btrfs_attach_subpage(fs_info, page, type: BTRFS_SUBPAGE_DATA);
912
913	attach_page_private(page, data: (void *)EXTENT_PAGE_PRIVATE);
914	return 0;
915	}
916
917	void clear_page_extent_mapped(struct page *page)
918	{
919	struct btrfs_fs_info *fs_info;
920
921	ASSERT(page->mapping);
922
923	if (!PagePrivate(page))
924	return;
925
926	fs_info = btrfs_sb(sb: page->mapping->host->i_sb);
927	if (btrfs_is_subpage(fs_info, page))
928	return btrfs_detach_subpage(fs_info, page);
929
930	detach_page_private(page);
931	}
932
933	static struct extent_map *
934	__get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
935	u64 start, u64 len, struct extent_map **em_cached)
936	{
937	struct extent_map *em;
938
939	if (em_cached && *em_cached) {
940	em = *em_cached;
941	if (extent_map_in_tree(em) && start >= em->start &&
942	start < extent_map_end(em)) {
943	refcount_inc(r: &em->refs);
944	return em;
945	}
946
947	free_extent_map(em);
948	*em_cached = NULL;
949	}
950
951	em = btrfs_get_extent(inode: BTRFS_I(inode), page, pg_offset, start, end: len);
952	if (em_cached && !IS_ERR(ptr: em)) {
953	BUG_ON(*em_cached);
954	refcount_inc(r: &em->refs);
955	*em_cached = em;
956	}
957	return em;
958	}
959	/*
960	* basic readpage implementation. Locked extent state structs are inserted
961	* into the tree that are removed when the IO is done (by the end_io
962	* handlers)
963	* XXX JDM: This needs looking at to ensure proper page locking
964	* return 0 on success, otherwise return error
965	*/
966	static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
967	struct btrfs_bio_ctrl *bio_ctrl, u64 *prev_em_start)
968	{
969	struct inode *inode = page->mapping->host;
970	struct btrfs_fs_info *fs_info = btrfs_sb(sb: inode->i_sb);
971	u64 start = page_offset(page);
972	const u64 end = start + PAGE_SIZE - 1;
973	u64 cur = start;
974	u64 extent_offset;
975	u64 last_byte = i_size_read(inode);
976	u64 block_start;
977	struct extent_map *em;
978	int ret = 0;
979	size_t pg_offset = 0;
980	size_t iosize;
981	size_t blocksize = inode->i_sb->s_blocksize;
982	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
983
984	ret = set_page_extent_mapped(page);
985	if (ret < 0) {
986	unlock_extent(tree, start, end, NULL);
987	unlock_page(page);
988	return ret;
989	}
990
991	if (page->index == last_byte >> PAGE_SHIFT) {
992	size_t zero_offset = offset_in_page(last_byte);
993
994	if (zero_offset) {
995	iosize = PAGE_SIZE - zero_offset;
996	memzero_page(page, offset: zero_offset, len: iosize);
997	}
998	}
999	bio_ctrl->end_io_func = end_bio_extent_readpage;
1000	begin_page_read(fs_info, page);
1001	while (cur <= end) {
1002	enum btrfs_compression_type compress_type = BTRFS_COMPRESS_NONE;
1003	bool force_bio_submit = false;
1004	u64 disk_bytenr;
1005
1006	ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
1007	if (cur >= last_byte) {
1008	iosize = PAGE_SIZE - pg_offset;
1009	memzero_page(page, offset: pg_offset, len: iosize);
1010	unlock_extent(tree, start: cur, end: cur + iosize - 1, NULL);
1011	end_page_read(page, uptodate: true, start: cur, len: iosize);
1012	break;
1013	}
1014	em = __get_extent_map(inode, page, pg_offset, start: cur,
1015	len: end - cur + 1, em_cached);
1016	if (IS_ERR(ptr: em)) {
1017	unlock_extent(tree, start: cur, end, NULL);
1018	end_page_read(page, uptodate: false, start: cur, len: end + 1 - cur);
1019	return PTR_ERR(ptr: em);
1020	}
1021	extent_offset = cur - em->start;
1022	BUG_ON(extent_map_end(em) <= cur);
1023	BUG_ON(end < cur);
1024
1025	if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
1026	compress_type = em->compress_type;
1027
1028	iosize = min(extent_map_end(em) - cur, end - cur + 1);
1029	iosize = ALIGN(iosize, blocksize);
1030	if (compress_type != BTRFS_COMPRESS_NONE)
1031	disk_bytenr = em->block_start;
1032	else
1033	disk_bytenr = em->block_start + extent_offset;
1034	block_start = em->block_start;
1035	if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
1036	block_start = EXTENT_MAP_HOLE;
1037
1038	/*
1039	* If we have a file range that points to a compressed extent
1040	* and it's followed by a consecutive file range that points
1041	* to the same compressed extent (possibly with a different
1042	* offset and/or length, so it either points to the whole extent
1043	* or only part of it), we must make sure we do not submit a
1044	* single bio to populate the pages for the 2 ranges because
1045	* this makes the compressed extent read zero out the pages
1046	* belonging to the 2nd range. Imagine the following scenario:
1047	*
1048	* File layout
1049	* [0 - 8K] [8K - 24K]
1050	* | |
1051	* | |
1052	* points to extent X, points to extent X,
1053	* offset 4K, length of 8K offset 0, length 16K
1054	*
1055	* [extent X, compressed length = 4K uncompressed length = 16K]
1056	*
1057	* If the bio to read the compressed extent covers both ranges,
1058	* it will decompress extent X into the pages belonging to the
1059	* first range and then it will stop, zeroing out the remaining
1060	* pages that belong to the other range that points to extent X.
1061	* So here we make sure we submit 2 bios, one for the first
1062	* range and another one for the third range. Both will target
1063	* the same physical extent from disk, but we can't currently
1064	* make the compressed bio endio callback populate the pages
1065	* for both ranges because each compressed bio is tightly
1066	* coupled with a single extent map, and each range can have
1067	* an extent map with a different offset value relative to the
1068	* uncompressed data of our extent and different lengths. This
1069	* is a corner case so we prioritize correctness over
1070	* non-optimal behavior (submitting 2 bios for the same extent).
1071	*/
1072	if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
1073	prev_em_start && *prev_em_start != (u64)-1 &&
1074	*prev_em_start != em->start)
1075	force_bio_submit = true;
1076
1077	if (prev_em_start)
1078	*prev_em_start = em->start;
1079
1080	free_extent_map(em);
1081	em = NULL;
1082
1083	/* we've found a hole, just zero and go on */
1084	if (block_start == EXTENT_MAP_HOLE) {
1085	memzero_page(page, offset: pg_offset, len: iosize);
1086
1087	unlock_extent(tree, start: cur, end: cur + iosize - 1, NULL);
1088	end_page_read(page, uptodate: true, start: cur, len: iosize);
1089	cur = cur + iosize;
1090	pg_offset += iosize;
1091	continue;
1092	}
1093	/* the get_extent function already copied into the page */
1094	if (block_start == EXTENT_MAP_INLINE) {
1095	unlock_extent(tree, start: cur, end: cur + iosize - 1, NULL);
1096	end_page_read(page, uptodate: true, start: cur, len: iosize);
1097	cur = cur + iosize;
1098	pg_offset += iosize;
1099	continue;
1100	}
1101
1102	if (bio_ctrl->compress_type != compress_type) {
1103	submit_one_bio(bio_ctrl);
1104	bio_ctrl->compress_type = compress_type;
1105	}
1106
1107	if (force_bio_submit)
1108	submit_one_bio(bio_ctrl);
1109	submit_extent_page(bio_ctrl, disk_bytenr, page, size: iosize,
1110	pg_offset);
1111	cur = cur + iosize;
1112	pg_offset += iosize;
1113	}
1114
1115	return 0;
1116	}
1117
1118	int btrfs_read_folio(struct file *file, struct folio *folio)
1119	{
1120	struct page *page = &folio->page;
1121	struct btrfs_inode *inode = BTRFS_I(inode: page->mapping->host);
1122	u64 start = page_offset(page);
1123	u64 end = start + PAGE_SIZE - 1;
1124	struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ };
1125	int ret;
1126
1127	btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
1128
1129	ret = btrfs_do_readpage(page, NULL, bio_ctrl: &bio_ctrl, NULL);
1130	/*
1131	* If btrfs_do_readpage() failed we will want to submit the assembled
1132	* bio to do the cleanup.
1133	*/
1134	submit_one_bio(bio_ctrl: &bio_ctrl);
1135	return ret;
1136	}
1137
1138	static inline void contiguous_readpages(struct page *pages[], int nr_pages,
1139	u64 start, u64 end,
1140	struct extent_map **em_cached,
1141	struct btrfs_bio_ctrl *bio_ctrl,
1142	u64 *prev_em_start)
1143	{
1144	struct btrfs_inode *inode = BTRFS_I(inode: pages[0]->mapping->host);
1145	int index;
1146
1147	btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
1148
1149	for (index = 0; index < nr_pages; index++) {
1150	btrfs_do_readpage(page: pages[index], em_cached, bio_ctrl,
1151	prev_em_start);
1152	put_page(page: pages[index]);
1153	}
1154	}
1155
1156	/*
1157	* helper for __extent_writepage, doing all of the delayed allocation setup.
1158	*
1159	* This returns 1 if btrfs_run_delalloc_range function did all the work required
1160	* to write the page (copy into inline extent). In this case the IO has
1161	* been started and the page is already unlocked.
1162	*
1163	* This returns 0 if all went well (page still locked)
1164	* This returns < 0 if there were errors (page still locked)
1165	*/
1166	static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
1167	struct page *page, struct writeback_control *wbc)
1168	{
1169	const u64 page_start = page_offset(page);
1170	const u64 page_end = page_start + PAGE_SIZE - 1;
1171	u64 delalloc_start = page_start;
1172	u64 delalloc_end = page_end;
1173	u64 delalloc_to_write = 0;
1174	int ret = 0;
1175
1176	while (delalloc_start < page_end) {
1177	delalloc_end = page_end;
1178	if (!find_lock_delalloc_range(inode: &inode->vfs_inode, locked_page: page,
1179	start: &delalloc_start, end: &delalloc_end)) {
1180	delalloc_start = delalloc_end + 1;
1181	continue;
1182	}
1183
1184	ret = btrfs_run_delalloc_range(inode, locked_page: page, start: delalloc_start,
1185	end: delalloc_end, wbc);
1186	if (ret < 0)
1187	return ret;
1188
1189	delalloc_start = delalloc_end + 1;
1190	}
1191
1192	/*
1193	* delalloc_end is already one less than the total length, so
1194	* we don't subtract one from PAGE_SIZE
1195	*/
1196	delalloc_to_write +=
1197	DIV_ROUND_UP(delalloc_end + 1 - page_start, PAGE_SIZE);
1198
1199	/*
1200	* If btrfs_run_dealloc_range() already started I/O and unlocked
1201	* the pages, we just need to account for them here.
1202	*/
1203	if (ret == 1) {
1204	wbc->nr_to_write -= delalloc_to_write;
1205	return 1;
1206	}
1207
1208	if (wbc->nr_to_write < delalloc_to_write) {
1209	int thresh = 8192;
1210
1211	if (delalloc_to_write < thresh * 2)
1212	thresh = delalloc_to_write;
1213	wbc->nr_to_write = min_t(u64, delalloc_to_write,
1214	thresh);
1215	}
1216
1217	return 0;
1218	}
1219
1220	/*
1221	* Find the first byte we need to write.
1222	*
1223	* For subpage, one page can contain several sectors, and
1224	* __extent_writepage_io() will just grab all extent maps in the page
1225	* range and try to submit all non-inline/non-compressed extents.
1226	*
1227	* This is a big problem for subpage, we shouldn't re-submit already written
1228	* data at all.
1229	* This function will lookup subpage dirty bit to find which range we really
1230	* need to submit.
1231	*
1232	* Return the next dirty range in [@start, @end).
1233	* If no dirty range is found, @start will be page_offset(page) + PAGE_SIZE.
1234	*/
1235	static void find_next_dirty_byte(struct btrfs_fs_info *fs_info,
1236	struct page *page, u64 *start, u64 *end)
1237	{
1238	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
1239	struct btrfs_subpage_info *spi = fs_info->subpage_info;
1240	u64 orig_start = *start;
1241	/* Declare as unsigned long so we can use bitmap ops */
1242	unsigned long flags;
1243	int range_start_bit;
1244	int range_end_bit;
1245
1246	/*
1247	* For regular sector size == page size case, since one page only
1248	* contains one sector, we return the page offset directly.
1249	*/
1250	if (!btrfs_is_subpage(fs_info, page)) {
1251	*start = page_offset(page);
1252	*end = page_offset(page) + PAGE_SIZE;
1253	return;
1254	}
1255
1256	range_start_bit = spi->dirty_offset +
1257	(offset_in_page(orig_start) >> fs_info->sectorsize_bits);
1258
1259	/* We should have the page locked, but just in case */
1260	spin_lock_irqsave(&subpage->lock, flags);
1261	bitmap_next_set_region(bitmap: subpage->bitmaps, rs: &range_start_bit, re: &range_end_bit,
1262	end: spi->dirty_offset + spi->bitmap_nr_bits);
1263	spin_unlock_irqrestore(lock: &subpage->lock, flags);
1264
1265	range_start_bit -= spi->dirty_offset;
1266	range_end_bit -= spi->dirty_offset;
1267
1268	*start = page_offset(page) + range_start_bit * fs_info->sectorsize;
1269	*end = page_offset(page) + range_end_bit * fs_info->sectorsize;
1270	}
1271
1272	/*
1273	* helper for __extent_writepage. This calls the writepage start hooks,
1274	* and does the loop to map the page into extents and bios.
1275	*
1276	* We return 1 if the IO is started and the page is unlocked,
1277	* 0 if all went well (page still locked)
1278	* < 0 if there were errors (page still locked)
1279	*/
1280	static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode,
1281	struct page *page,
1282	struct btrfs_bio_ctrl *bio_ctrl,
1283	loff_t i_size,
1284	int *nr_ret)
1285	{
1286	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1287	u64 cur = page_offset(page);
1288	u64 end = cur + PAGE_SIZE - 1;
1289	u64 extent_offset;
1290	u64 block_start;
1291	struct extent_map *em;
1292	int ret = 0;
1293	int nr = 0;
1294
1295	ret = btrfs_writepage_cow_fixup(page);
1296	if (ret) {
1297	/* Fixup worker will requeue */
1298	redirty_page_for_writepage(bio_ctrl->wbc, page);
1299	unlock_page(page);
1300	return 1;
1301	}
1302
1303	bio_ctrl->end_io_func = end_bio_extent_writepage;
1304	while (cur <= end) {
1305	u32 len = end - cur + 1;
1306	u64 disk_bytenr;
1307	u64 em_end;
1308	u64 dirty_range_start = cur;
1309	u64 dirty_range_end;
1310	u32 iosize;
1311
1312	if (cur >= i_size) {
1313	btrfs_mark_ordered_io_finished(inode, page, file_offset: cur, num_bytes: len,
1314	uptodate: true);
1315	/*
1316	* This range is beyond i_size, thus we don't need to
1317	* bother writing back.
1318	* But we still need to clear the dirty subpage bit, or
1319	* the next time the page gets dirtied, we will try to
1320	* writeback the sectors with subpage dirty bits,
1321	* causing writeback without ordered extent.
1322	*/
1323	btrfs_page_clear_dirty(fs_info, page, start: cur, len);
1324	break;
1325	}
1326
1327	find_next_dirty_byte(fs_info, page, start: &dirty_range_start,
1328	end: &dirty_range_end);
1329	if (cur < dirty_range_start) {
1330	cur = dirty_range_start;
1331	continue;
1332	}
1333
1334	em = btrfs_get_extent(inode, NULL, pg_offset: 0, start: cur, end: len);
1335	if (IS_ERR(ptr: em)) {
1336	ret = PTR_ERR_OR_ZERO(ptr: em);
1337	goto out_error;
1338	}
1339
1340	extent_offset = cur - em->start;
1341	em_end = extent_map_end(em);
1342	ASSERT(cur <= em_end);
1343	ASSERT(cur < end);
1344	ASSERT(IS_ALIGNED(em->start, fs_info->sectorsize));
1345	ASSERT(IS_ALIGNED(em->len, fs_info->sectorsize));
1346
1347	block_start = em->block_start;
1348	disk_bytenr = em->block_start + extent_offset;
1349
1350	ASSERT(!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags));
1351	ASSERT(block_start != EXTENT_MAP_HOLE);
1352	ASSERT(block_start != EXTENT_MAP_INLINE);
1353
1354	/*
1355	* Note that em_end from extent_map_end() and dirty_range_end from
1356	* find_next_dirty_byte() are all exclusive
1357	*/
1358	iosize = min(min(em_end, end + 1), dirty_range_end) - cur;
1359	free_extent_map(em);
1360	em = NULL;
1361
1362	btrfs_set_range_writeback(inode, start: cur, end: cur + iosize - 1);
1363	if (!PageWriteback(page)) {
1364	btrfs_err(inode->root->fs_info,
1365	"page %lu not writeback, cur %llu end %llu",
1366	page->index, cur, end);
1367	}
1368
1369	/*
1370	* Although the PageDirty bit is cleared before entering this
1371	* function, subpage dirty bit is not cleared.
1372	* So clear subpage dirty bit here so next time we won't submit
1373	* page for range already written to disk.
1374	*/
1375	btrfs_page_clear_dirty(fs_info, page, start: cur, len: iosize);
1376
1377	submit_extent_page(bio_ctrl, disk_bytenr, page, size: iosize,
1378	pg_offset: cur - page_offset(page));
1379	cur += iosize;
1380	nr++;
1381	}
1382
1383	btrfs_page_assert_not_dirty(fs_info, page);
1384	*nr_ret = nr;
1385	return 0;
1386
1387	out_error:
1388	/*
1389	* If we finish without problem, we should not only clear page dirty,
1390	* but also empty subpage dirty bits
1391	*/
1392	*nr_ret = nr;
1393	return ret;
1394	}
1395
1396	/*
1397	* the writepage semantics are similar to regular writepage. extent
1398	* records are inserted to lock ranges in the tree, and as dirty areas
1399	* are found, they are marked writeback. Then the lock bits are removed
1400	* and the end_io handler clears the writeback ranges
1401	*
1402	* Return 0 if everything goes well.
1403	* Return <0 for error.
1404	*/
1405	static int __extent_writepage(struct page *page, struct btrfs_bio_ctrl *bio_ctrl)
1406	{
1407	struct folio *folio = page_folio(page);
1408	struct inode *inode = page->mapping->host;
1409	const u64 page_start = page_offset(page);
1410	int ret;
1411	int nr = 0;
1412	size_t pg_offset;
1413	loff_t i_size = i_size_read(inode);
1414	unsigned long end_index = i_size >> PAGE_SHIFT;
1415
1416	trace___extent_writepage(page, inode, wbc: bio_ctrl->wbc);
1417
1418	WARN_ON(!PageLocked(page));
1419
1420	pg_offset = offset_in_page(i_size);
1421	if (page->index > end_index ||
1422	(page->index == end_index && !pg_offset)) {
1423	folio_invalidate(folio, offset: 0, length: folio_size(folio));
1424	folio_unlock(folio);
1425	return 0;
1426	}
1427
1428	if (page->index == end_index)
1429	memzero_page(page, offset: pg_offset, PAGE_SIZE - pg_offset);
1430
1431	ret = set_page_extent_mapped(page);
1432	if (ret < 0)
1433	goto done;
1434
1435	ret = writepage_delalloc(inode: BTRFS_I(inode), page, wbc: bio_ctrl->wbc);
1436	if (ret == 1)
1437	return 0;
1438	if (ret)
1439	goto done;
1440
1441	ret = __extent_writepage_io(inode: BTRFS_I(inode), page, bio_ctrl, i_size, nr_ret: &nr);
1442	if (ret == 1)
1443	return 0;
1444
1445	bio_ctrl->wbc->nr_to_write--;
1446
1447	done:
1448	if (nr == 0) {
1449	/* make sure the mapping tag for page dirty gets cleared */
1450	set_page_writeback(page);
1451	end_page_writeback(page);
1452	}
1453	if (ret) {
1454	btrfs_mark_ordered_io_finished(inode: BTRFS_I(inode), page, file_offset: page_start,
1455	PAGE_SIZE, uptodate: !ret);
1456	mapping_set_error(mapping: page->mapping, error: ret);
1457	}
1458	unlock_page(page);
1459	ASSERT(ret <= 0);
1460	return ret;
1461	}
1462
1463	void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
1464	{
1465	wait_on_bit_io(word: &eb->bflags, bit: EXTENT_BUFFER_WRITEBACK,
1466	TASK_UNINTERRUPTIBLE);
1467	}
1468
1469	/*
1470	* Lock extent buffer status and pages for writeback.
1471	*
1472	* Return %false if the extent buffer doesn't need to be submitted (e.g. the
1473	* extent buffer is not dirty)
1474	* Return %true is the extent buffer is submitted to bio.
1475	*/
1476	static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *eb,
1477	struct writeback_control *wbc)
1478	{
1479	struct btrfs_fs_info *fs_info = eb->fs_info;
1480	bool ret = false;
1481
1482	btrfs_tree_lock(eb);
1483	while (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
1484	btrfs_tree_unlock(eb);
1485	if (wbc->sync_mode != WB_SYNC_ALL)
1486	return false;
1487	wait_on_extent_buffer_writeback(eb);
1488	btrfs_tree_lock(eb);
1489	}
1490
1491	/*
1492	* We need to do this to prevent races in people who check if the eb is
1493	* under IO since we can end up having no IO bits set for a short period
1494	* of time.
1495	*/
1496	spin_lock(lock: &eb->refs_lock);
1497	if (test_and_clear_bit(nr: EXTENT_BUFFER_DIRTY, addr: &eb->bflags)) {
1498	set_bit(nr: EXTENT_BUFFER_WRITEBACK, addr: &eb->bflags);
1499	spin_unlock(lock: &eb->refs_lock);
1500	btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
1501	percpu_counter_add_batch(fbc: &fs_info->dirty_metadata_bytes,
1502	amount: -eb->len,
1503	batch: fs_info->dirty_metadata_batch);
1504	ret = true;
1505	} else {
1506	spin_unlock(lock: &eb->refs_lock);
1507	}
1508	btrfs_tree_unlock(eb);
1509	return ret;
1510	}
1511
1512	static void set_btree_ioerr(struct extent_buffer *eb)
1513	{
1514	struct btrfs_fs_info *fs_info = eb->fs_info;
1515
1516	set_bit(nr: EXTENT_BUFFER_WRITE_ERR, addr: &eb->bflags);
1517
1518	/*
1519	* A read may stumble upon this buffer later, make sure that it gets an
1520	* error and knows there was an error.
1521	*/
1522	clear_bit(nr: EXTENT_BUFFER_UPTODATE, addr: &eb->bflags);
1523
1524	/*
1525	* We need to set the mapping with the io error as well because a write
1526	* error will flip the file system readonly, and then syncfs() will
1527	* return a 0 because we are readonly if we don't modify the err seq for
1528	* the superblock.
1529	*/
1530	mapping_set_error(mapping: eb->fs_info->btree_inode->i_mapping, error: -EIO);
1531
1532	/*
1533	* If writeback for a btree extent that doesn't belong to a log tree
1534	* failed, increment the counter transaction->eb_write_errors.
1535	* We do this because while the transaction is running and before it's
1536	* committing (when we call filemap_fdata[write|wait]_range against
1537	* the btree inode), we might have
1538	* btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
1539	* returns an error or an error happens during writeback, when we're
1540	* committing the transaction we wouldn't know about it, since the pages
1541	* can be no longer dirty nor marked anymore for writeback (if a
1542	* subsequent modification to the extent buffer didn't happen before the
1543	* transaction commit), which makes filemap_fdata[write|wait]_range not
1544	* able to find the pages tagged with SetPageError at transaction
1545	* commit time. So if this happens we must abort the transaction,
1546	* otherwise we commit a super block with btree roots that point to
1547	* btree nodes/leafs whose content on disk is invalid - either garbage
1548	* or the content of some node/leaf from a past generation that got
1549	* cowed or deleted and is no longer valid.
1550	*
1551	* Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
1552	* not be enough - we need to distinguish between log tree extents vs
1553	* non-log tree extents, and the next filemap_fdatawait_range() call
1554	* will catch and clear such errors in the mapping - and that call might
1555	* be from a log sync and not from a transaction commit. Also, checking
1556	* for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
1557	* not done and would not be reliable - the eb might have been released
1558	* from memory and reading it back again means that flag would not be
1559	* set (since it's a runtime flag, not persisted on disk).
1560	*
1561	* Using the flags below in the btree inode also makes us achieve the
1562	* goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
1563	* writeback for all dirty pages and before filemap_fdatawait_range()
1564	* is called, the writeback for all dirty pages had already finished
1565	* with errors - because we were not using AS_EIO/AS_ENOSPC,
1566	* filemap_fdatawait_range() would return success, as it could not know
1567	* that writeback errors happened (the pages were no longer tagged for
1568	* writeback).
1569	*/
1570	switch (eb->log_index) {
1571	case -1:
1572	set_bit(nr: BTRFS_FS_BTREE_ERR, addr: &fs_info->flags);
1573	break;
1574	case 0:
1575	set_bit(nr: BTRFS_FS_LOG1_ERR, addr: &fs_info->flags);
1576	break;
1577	case 1:
1578	set_bit(nr: BTRFS_FS_LOG2_ERR, addr: &fs_info->flags);
1579	break;
1580	default:
1581	BUG(); /* unexpected, logic error */
1582	}
1583	}
1584
1585	/*
1586	* The endio specific version which won't touch any unsafe spinlock in endio
1587	* context.
1588	*/
1589	static struct extent_buffer *find_extent_buffer_nolock(
1590	struct btrfs_fs_info *fs_info, u64 start)
1591	{
1592	struct extent_buffer *eb;
1593
1594	rcu_read_lock();
1595	eb = radix_tree_lookup(&fs_info->buffer_radix,
1596	start >> fs_info->sectorsize_bits);
1597	if (eb && atomic_inc_not_zero(v: &eb->refs)) {
1598	rcu_read_unlock();
1599	return eb;
1600	}
1601	rcu_read_unlock();
1602	return NULL;
1603	}
1604
1605	static void extent_buffer_write_end_io(struct btrfs_bio *bbio)
1606	{
1607	struct extent_buffer *eb = bbio->private;
1608	struct btrfs_fs_info *fs_info = eb->fs_info;
1609	bool uptodate = !bbio->bio.bi_status;
1610	struct bvec_iter_all iter_all;
1611	struct bio_vec *bvec;
1612	u32 bio_offset = 0;
1613
1614	if (!uptodate)
1615	set_btree_ioerr(eb);
1616
1617	bio_for_each_segment_all(bvec, &bbio->bio, iter_all) {
1618	u64 start = eb->start + bio_offset;
1619	struct page *page = bvec->bv_page;
1620	u32 len = bvec->bv_len;
1621
1622	btrfs_page_clear_writeback(fs_info, page, start, len);
1623	bio_offset += len;
1624	}
1625
1626	clear_bit(nr: EXTENT_BUFFER_WRITEBACK, addr: &eb->bflags);
1627	smp_mb__after_atomic();
1628	wake_up_bit(word: &eb->bflags, bit: EXTENT_BUFFER_WRITEBACK);
1629
1630	bio_put(&bbio->bio);
1631	}
1632
1633	static void prepare_eb_write(struct extent_buffer *eb)
1634	{
1635	u32 nritems;
1636	unsigned long start;
1637	unsigned long end;
1638
1639	clear_bit(nr: EXTENT_BUFFER_WRITE_ERR, addr: &eb->bflags);
1640
1641	/* Set btree blocks beyond nritems with 0 to avoid stale content */
1642	nritems = btrfs_header_nritems(eb);
1643	if (btrfs_header_level(eb) > 0) {
1644	end = btrfs_node_key_ptr_offset(eb, nr: nritems);
1645	memzero_extent_buffer(eb, start: end, len: eb->len - end);
1646	} else {
1647	/*
1648	* Leaf:
1649	* header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
1650	*/
1651	start = btrfs_item_nr_offset(eb, nr: nritems);
1652	end = btrfs_item_nr_offset(eb, nr: 0);
1653	if (nritems == 0)
1654	end += BTRFS_LEAF_DATA_SIZE(info: eb->fs_info);
1655	else
1656	end += btrfs_item_offset(eb, slot: nritems - 1);
1657	memzero_extent_buffer(eb, start, len: end - start);
1658	}
1659	}
1660
1661	static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
1662	struct writeback_control *wbc)
1663	{
1664	struct btrfs_fs_info *fs_info = eb->fs_info;
1665	struct btrfs_bio *bbio;
1666
1667	prepare_eb_write(eb);
1668
1669	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
1670	opf: REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc),
1671	fs_info: eb->fs_info, end_io: extent_buffer_write_end_io, private: eb);
1672	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
1673	bio_set_dev(bio: &bbio->bio, bdev: fs_info->fs_devices->latest_dev->bdev);
1674	wbc_init_bio(wbc, bio: &bbio->bio);
1675	bbio->inode = BTRFS_I(inode: eb->fs_info->btree_inode);
1676	bbio->file_offset = eb->start;
1677	if (fs_info->nodesize < PAGE_SIZE) {
1678	struct page *p = eb->pages[0];
1679
1680	lock_page(page: p);
1681	btrfs_subpage_set_writeback(fs_info, page: p, start: eb->start, len: eb->len);
1682	if (btrfs_subpage_clear_and_test_dirty(fs_info, page: p, start: eb->start,
1683	len: eb->len)) {
1684	clear_page_dirty_for_io(page: p);
1685	wbc->nr_to_write--;
1686	}
1687	__bio_add_page(bio: &bbio->bio, page: p, len: eb->len, off: eb->start - page_offset(page: p));
1688	wbc_account_cgroup_owner(wbc, page: p, bytes: eb->len);
1689	unlock_page(page: p);
1690	} else {
1691	for (int i = 0; i < num_extent_pages(eb); i++) {
1692	struct page *p = eb->pages[i];
1693
1694	lock_page(page: p);
1695	clear_page_dirty_for_io(page: p);
1696	set_page_writeback(p);
1697	__bio_add_page(bio: &bbio->bio, page: p, PAGE_SIZE, off: 0);
1698	wbc_account_cgroup_owner(wbc, page: p, PAGE_SIZE);
1699	wbc->nr_to_write--;
1700	unlock_page(page: p);
1701	}
1702	}
1703	btrfs_submit_bio(bbio, mirror_num: 0);
1704	}
1705
1706	/*
1707	* Submit one subpage btree page.
1708	*
1709	* The main difference to submit_eb_page() is:
1710	* - Page locking
1711	* For subpage, we don't rely on page locking at all.
1712	*
1713	* - Flush write bio
1714	* We only flush bio if we may be unable to fit current extent buffers into
1715	* current bio.
1716	*
1717	* Return >=0 for the number of submitted extent buffers.
1718	* Return <0 for fatal error.
1719	*/
1720	static int submit_eb_subpage(struct page *page, struct writeback_control *wbc)
1721	{
1722	struct btrfs_fs_info *fs_info = btrfs_sb(sb: page->mapping->host->i_sb);
1723	int submitted = 0;
1724	u64 page_start = page_offset(page);
1725	int bit_start = 0;
1726	int sectors_per_node = fs_info->nodesize >> fs_info->sectorsize_bits;
1727
1728	/* Lock and write each dirty extent buffers in the range */
1729	while (bit_start < fs_info->subpage_info->bitmap_nr_bits) {
1730	struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
1731	struct extent_buffer *eb;
1732	unsigned long flags;
1733	u64 start;
1734
1735	/*
1736	* Take private lock to ensure the subpage won't be detached
1737	* in the meantime.
1738	*/
1739	spin_lock(lock: &page->mapping->private_lock);
1740	if (!PagePrivate(page)) {
1741	spin_unlock(lock: &page->mapping->private_lock);
1742	break;
1743	}
1744	spin_lock_irqsave(&subpage->lock, flags);
1745	if (!test_bit(bit_start + fs_info->subpage_info->dirty_offset,
1746	subpage->bitmaps)) {
1747	spin_unlock_irqrestore(lock: &subpage->lock, flags);
1748	spin_unlock(lock: &page->mapping->private_lock);
1749	bit_start++;
1750	continue;
1751	}
1752
1753	start = page_start + bit_start * fs_info->sectorsize;
1754	bit_start += sectors_per_node;
1755
1756	/*
1757	* Here we just want to grab the eb without touching extra
1758	* spin locks, so call find_extent_buffer_nolock().
1759	*/
1760	eb = find_extent_buffer_nolock(fs_info, start);
1761	spin_unlock_irqrestore(lock: &subpage->lock, flags);
1762	spin_unlock(lock: &page->mapping->private_lock);
1763
1764	/*
1765	* The eb has already reached 0 refs thus find_extent_buffer()
1766	* doesn't return it. We don't need to write back such eb
1767	* anyway.
1768	*/
1769	if (!eb)
1770	continue;
1771
1772	if (lock_extent_buffer_for_io(eb, wbc)) {
1773	write_one_eb(eb, wbc);
1774	submitted++;
1775	}
1776	free_extent_buffer(eb);
1777	}
1778	return submitted;
1779	}
1780
1781	/*
1782	* Submit all page(s) of one extent buffer.
1783	*
1784	* @page: the page of one extent buffer
1785	* @eb_context: to determine if we need to submit this page, if current page
1786	* belongs to this eb, we don't need to submit
1787	*
1788	* The caller should pass each page in their bytenr order, and here we use
1789	* @eb_context to determine if we have submitted pages of one extent buffer.
1790	*
1791	* If we have, we just skip until we hit a new page that doesn't belong to
1792	* current @eb_context.
1793	*
1794	* If not, we submit all the page(s) of the extent buffer.
1795	*
1796	* Return >0 if we have submitted the extent buffer successfully.
1797	* Return 0 if we don't need to submit the page, as it's already submitted by
1798	* previous call.
1799	* Return <0 for fatal error.
1800	*/
1801	static int submit_eb_page(struct page *page, struct btrfs_eb_write_context *ctx)
1802	{
1803	struct writeback_control *wbc = ctx->wbc;
1804	struct address_space *mapping = page->mapping;
1805	struct extent_buffer *eb;
1806	int ret;
1807
1808	if (!PagePrivate(page))
1809	return 0;
1810
1811	if (btrfs_sb(sb: page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
1812	return submit_eb_subpage(page, wbc);
1813
1814	spin_lock(lock: &mapping->private_lock);
1815	if (!PagePrivate(page)) {
1816	spin_unlock(lock: &mapping->private_lock);
1817	return 0;
1818	}
1819
1820	eb = (struct extent_buffer *)page->private;
1821
1822	/*
1823	* Shouldn't happen and normally this would be a BUG_ON but no point
1824	* crashing the machine for something we can survive anyway.
1825	*/
1826	if (WARN_ON(!eb)) {
1827	spin_unlock(lock: &mapping->private_lock);
1828	return 0;
1829	}
1830
1831	if (eb == ctx->eb) {
1832	spin_unlock(lock: &mapping->private_lock);
1833	return 0;
1834	}
1835	ret = atomic_inc_not_zero(v: &eb->refs);
1836	spin_unlock(lock: &mapping->private_lock);
1837	if (!ret)
1838	return 0;
1839
1840	ctx->eb = eb;
1841
1842	ret = btrfs_check_meta_write_pointer(fs_info: eb->fs_info, ctx);
1843	if (ret) {
1844	if (ret == -EBUSY)
1845	ret = 0;
1846	free_extent_buffer(eb);
1847	return ret;
1848	}
1849
1850	if (!lock_extent_buffer_for_io(eb, wbc)) {
1851	free_extent_buffer(eb);
1852	return 0;
1853	}
1854	/* Implies write in zoned mode. */
1855	if (ctx->zoned_bg) {
1856	/* Mark the last eb in the block group. */
1857	btrfs_schedule_zone_finish_bg(bg: ctx->zoned_bg, eb);
1858	ctx->zoned_bg->meta_write_pointer += eb->len;
1859	}
1860	write_one_eb(eb, wbc);
1861	free_extent_buffer(eb);
1862	return 1;
1863	}
1864
1865	int btree_write_cache_pages(struct address_space *mapping,
1866	struct writeback_control *wbc)
1867	{
1868	struct btrfs_eb_write_context ctx = { .wbc = wbc };
1869	struct btrfs_fs_info *fs_info = BTRFS_I(inode: mapping->host)->root->fs_info;
1870	int ret = 0;
1871	int done = 0;
1872	int nr_to_write_done = 0;
1873	struct folio_batch fbatch;
1874	unsigned int nr_folios;
1875	pgoff_t index;
1876	pgoff_t end; /* Inclusive */
1877	int scanned = 0;
1878	xa_mark_t tag;
1879
1880	folio_batch_init(fbatch: &fbatch);
1881	if (wbc->range_cyclic) {
1882	index = mapping->writeback_index; /* Start from prev offset */
1883	end = -1;
1884	/*
1885	* Start from the beginning does not need to cycle over the
1886	* range, mark it as scanned.
1887	*/
1888	scanned = (index == 0);
1889	} else {
1890	index = wbc->range_start >> PAGE_SHIFT;
1891	end = wbc->range_end >> PAGE_SHIFT;
1892	scanned = 1;
1893	}
1894	if (wbc->sync_mode == WB_SYNC_ALL)
1895	tag = PAGECACHE_TAG_TOWRITE;
1896	else
1897	tag = PAGECACHE_TAG_DIRTY;
1898	btrfs_zoned_meta_io_lock(fs_info);
1899	retry:
1900	if (wbc->sync_mode == WB_SYNC_ALL)
1901	tag_pages_for_writeback(mapping, start: index, end);
1902	while (!done && !nr_to_write_done && (index <= end) &&
1903	(nr_folios = filemap_get_folios_tag(mapping, start: &index, end,
1904	tag, fbatch: &fbatch))) {
1905	unsigned i;
1906
1907	for (i = 0; i < nr_folios; i++) {
1908	struct folio *folio = fbatch.folios[i];
1909
1910	ret = submit_eb_page(page: &folio->page, ctx: &ctx);
1911	if (ret == 0)
1912	continue;
1913	if (ret < 0) {
1914	done = 1;
1915	break;
1916	}
1917
1918	/*
1919	* the filesystem may choose to bump up nr_to_write.
1920	* We have to make sure to honor the new nr_to_write
1921	* at any time
1922	*/
1923	nr_to_write_done = wbc->nr_to_write <= 0;
1924	}
1925	folio_batch_release(fbatch: &fbatch);
1926	cond_resched();
1927	}
1928	if (!scanned && !done) {
1929	/*
1930	* We hit the last page and there is more work to be done: wrap
1931	* back to the start of the file
1932	*/
1933	scanned = 1;
1934	index = 0;
1935	goto retry;
1936	}
1937	/*
1938	* If something went wrong, don't allow any metadata write bio to be
1939	* submitted.
1940	*
1941	* This would prevent use-after-free if we had dirty pages not
1942	* cleaned up, which can still happen by fuzzed images.
1943	*
1944	* - Bad extent tree
1945	* Allowing existing tree block to be allocated for other trees.
1946	*
1947	* - Log tree operations
1948	* Exiting tree blocks get allocated to log tree, bumps its
1949	* generation, then get cleaned in tree re-balance.
1950	* Such tree block will not be written back, since it's clean,
1951	* thus no WRITTEN flag set.
1952	* And after log writes back, this tree block is not traced by
1953	* any dirty extent_io_tree.
1954	*
1955	* - Offending tree block gets re-dirtied from its original owner
1956	* Since it has bumped generation, no WRITTEN flag, it can be
1957	* reused without COWing. This tree block will not be traced
1958	* by btrfs_transaction::dirty_pages.
1959	*
1960	* Now such dirty tree block will not be cleaned by any dirty
1961	* extent io tree. Thus we don't want to submit such wild eb
1962	* if the fs already has error.
1963	*
1964	* We can get ret > 0 from submit_extent_page() indicating how many ebs
1965	* were submitted. Reset it to 0 to avoid false alerts for the caller.
1966	*/
1967	if (ret > 0)
1968	ret = 0;
1969	if (!ret && BTRFS_FS_ERROR(fs_info))
1970	ret = -EROFS;
1971
1972	if (ctx.zoned_bg)
1973	btrfs_put_block_group(cache: ctx.zoned_bg);
1974	btrfs_zoned_meta_io_unlock(fs_info);
1975	return ret;
1976	}
1977
1978	/*
1979	* Walk the list of dirty pages of the given address space and write all of them.
1980	*
1981	* @mapping: address space structure to write
1982	* @wbc: subtract the number of written pages from *@wbc->nr_to_write
1983	* @bio_ctrl: holds context for the write, namely the bio
1984	*
1985	* If a page is already under I/O, write_cache_pages() skips it, even
1986	* if it's dirty. This is desirable behaviour for memory-cleaning writeback,
1987	* but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
1988	* and msync() need to guarantee that all the data which was dirty at the time
1989	* the call was made get new I/O started against them. If wbc->sync_mode is
1990	* WB_SYNC_ALL then we were called for data integrity and we must wait for
1991	* existing IO to complete.
1992	*/
1993	static int extent_write_cache_pages(struct address_space *mapping,
1994	struct btrfs_bio_ctrl *bio_ctrl)
1995	{
1996	struct writeback_control *wbc = bio_ctrl->wbc;
1997	struct inode *inode = mapping->host;
1998	int ret = 0;
1999	int done = 0;
2000	int nr_to_write_done = 0;
2001	struct folio_batch fbatch;
2002	unsigned int nr_folios;
2003	pgoff_t index;
2004	pgoff_t end; /* Inclusive */
2005	pgoff_t done_index;
2006	int range_whole = 0;
2007	int scanned = 0;
2008	xa_mark_t tag;
2009
2010	/*
2011	* We have to hold onto the inode so that ordered extents can do their
2012	* work when the IO finishes. The alternative to this is failing to add
2013	* an ordered extent if the igrab() fails there and that is a huge pain
2014	* to deal with, so instead just hold onto the inode throughout the
2015	* writepages operation. If it fails here we are freeing up the inode
2016	* anyway and we'd rather not waste our time writing out stuff that is
2017	* going to be truncated anyway.
2018	*/
2019	if (!igrab(inode))
2020	return 0;
2021
2022	folio_batch_init(fbatch: &fbatch);
2023	if (wbc->range_cyclic) {
2024	index = mapping->writeback_index; /* Start from prev offset */
2025	end = -1;
2026	/*
2027	* Start from the beginning does not need to cycle over the
2028	* range, mark it as scanned.
2029	*/
2030	scanned = (index == 0);
2031	} else {
2032	index = wbc->range_start >> PAGE_SHIFT;
2033	end = wbc->range_end >> PAGE_SHIFT;
2034	if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2035	range_whole = 1;
2036	scanned = 1;
2037	}
2038
2039	/*
2040	* We do the tagged writepage as long as the snapshot flush bit is set
2041	* and we are the first one who do the filemap_flush() on this inode.
2042	*
2043	* The nr_to_write == LONG_MAX is needed to make sure other flushers do
2044	* not race in and drop the bit.
2045	*/
2046	if (range_whole && wbc->nr_to_write == LONG_MAX &&
2047	test_and_clear_bit(nr: BTRFS_INODE_SNAPSHOT_FLUSH,
2048	addr: &BTRFS_I(inode)->runtime_flags))
2049	wbc->tagged_writepages = 1;
2050
2051	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2052	tag = PAGECACHE_TAG_TOWRITE;
2053	else
2054	tag = PAGECACHE_TAG_DIRTY;
2055	retry:
2056	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2057	tag_pages_for_writeback(mapping, start: index, end);
2058	done_index = index;
2059	while (!done && !nr_to_write_done && (index <= end) &&
2060	(nr_folios = filemap_get_folios_tag(mapping, start: &index,
2061	end, tag, fbatch: &fbatch))) {
2062	unsigned i;
2063
2064	for (i = 0; i < nr_folios; i++) {
2065	struct folio *folio = fbatch.folios[i];
2066
2067	done_index = folio_next_index(folio);
2068	/*
2069	* At this point we hold neither the i_pages lock nor
2070	* the page lock: the page may be truncated or
2071	* invalidated (changing page->mapping to NULL),
2072	* or even swizzled back from swapper_space to
2073	* tmpfs file mapping
2074	*/
2075	if (!folio_trylock(folio)) {
2076	submit_write_bio(bio_ctrl, ret: 0);
2077	folio_lock(folio);
2078	}
2079
2080	if (unlikely(folio->mapping != mapping)) {
2081	folio_unlock(folio);
2082	continue;
2083	}
2084
2085	if (!folio_test_dirty(folio)) {
2086	/* Someone wrote it for us. */
2087	folio_unlock(folio);
2088	continue;
2089	}
2090
2091	if (wbc->sync_mode != WB_SYNC_NONE) {
2092	if (folio_test_writeback(folio))
2093	submit_write_bio(bio_ctrl, ret: 0);
2094	folio_wait_writeback(folio);
2095	}
2096
2097	if (folio_test_writeback(folio) ||
2098	!folio_clear_dirty_for_io(folio)) {
2099	folio_unlock(folio);
2100	continue;
2101	}
2102
2103	ret = __extent_writepage(page: &folio->page, bio_ctrl);
2104	if (ret < 0) {
2105	done = 1;
2106	break;
2107	}
2108
2109	/*
2110	* The filesystem may choose to bump up nr_to_write.
2111	* We have to make sure to honor the new nr_to_write
2112	* at any time.
2113	*/
2114	nr_to_write_done = (wbc->sync_mode == WB_SYNC_NONE &&
2115	wbc->nr_to_write <= 0);
2116	}
2117	folio_batch_release(fbatch: &fbatch);
2118	cond_resched();
2119	}
2120	if (!scanned && !done) {
2121	/*
2122	* We hit the last page and there is more work to be done: wrap
2123	* back to the start of the file
2124	*/
2125	scanned = 1;
2126	index = 0;
2127
2128	/*
2129	* If we're looping we could run into a page that is locked by a
2130	* writer and that writer could be waiting on writeback for a
2131	* page in our current bio, and thus deadlock, so flush the
2132	* write bio here.
2133	*/
2134	submit_write_bio(bio_ctrl, ret: 0);
2135	goto retry;
2136	}
2137
2138	if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
2139	mapping->writeback_index = done_index;
2140
2141	btrfs_add_delayed_iput(inode: BTRFS_I(inode));
2142	return ret;
2143	}
2144
2145	/*
2146	* Submit the pages in the range to bio for call sites which delalloc range has
2147	* already been ran (aka, ordered extent inserted) and all pages are still
2148	* locked.
2149	*/
2150	void extent_write_locked_range(struct inode *inode, struct page *locked_page,
2151	u64 start, u64 end, struct writeback_control *wbc,
2152	bool pages_dirty)
2153	{
2154	bool found_error = false;
2155	int ret = 0;
2156	struct address_space *mapping = inode->i_mapping;
2157	struct btrfs_fs_info *fs_info = btrfs_sb(sb: inode->i_sb);
2158	const u32 sectorsize = fs_info->sectorsize;
2159	loff_t i_size = i_size_read(inode);
2160	u64 cur = start;
2161	struct btrfs_bio_ctrl bio_ctrl = {
2162	.wbc = wbc,
2163	.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2164	};
2165
2166	if (wbc->no_cgroup_owner)
2167	bio_ctrl.opf |= REQ_BTRFS_CGROUP_PUNT;
2168
2169	ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(end + 1, sectorsize));
2170
2171	while (cur <= end) {
2172	u64 cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end);
2173	u32 cur_len = cur_end + 1 - cur;
2174	struct page *page;
2175	int nr = 0;
2176
2177	page = find_get_page(mapping, offset: cur >> PAGE_SHIFT);
2178	ASSERT(PageLocked(page));
2179	if (pages_dirty && page != locked_page) {
2180	ASSERT(PageDirty(page));
2181	clear_page_dirty_for_io(page);
2182	}
2183
2184	ret = __extent_writepage_io(inode: BTRFS_I(inode), page, bio_ctrl: &bio_ctrl,
2185	i_size, nr_ret: &nr);
2186	if (ret == 1)
2187	goto next_page;
2188
2189	/* Make sure the mapping tag for page dirty gets cleared. */
2190	if (nr == 0) {
2191	set_page_writeback(page);
2192	end_page_writeback(page);
2193	}
2194	if (ret) {
2195	btrfs_mark_ordered_io_finished(inode: BTRFS_I(inode), page,
2196	file_offset: cur, num_bytes: cur_len, uptodate: !ret);
2197	mapping_set_error(mapping: page->mapping, error: ret);
2198	}
2199	btrfs_page_unlock_writer(fs_info, page, start: cur, len: cur_len);
2200	if (ret < 0)
2201	found_error = true;
2202	next_page:
2203	put_page(page);
2204	cur = cur_end + 1;
2205	}
2206
2207	submit_write_bio(bio_ctrl: &bio_ctrl, ret: found_error ? ret : 0);
2208	}
2209
2210	int extent_writepages(struct address_space *mapping,
2211	struct writeback_control *wbc)
2212	{
2213	struct inode *inode = mapping->host;
2214	int ret = 0;
2215	struct btrfs_bio_ctrl bio_ctrl = {
2216	.wbc = wbc,
2217	.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2218	};
2219
2220	/*
2221	* Allow only a single thread to do the reloc work in zoned mode to
2222	* protect the write pointer updates.
2223	*/
2224	btrfs_zoned_data_reloc_lock(inode: BTRFS_I(inode));
2225	ret = extent_write_cache_pages(mapping, bio_ctrl: &bio_ctrl);
2226	submit_write_bio(bio_ctrl: &bio_ctrl, ret);
2227	btrfs_zoned_data_reloc_unlock(inode: BTRFS_I(inode));
2228	return ret;
2229	}
2230
2231	void extent_readahead(struct readahead_control *rac)
2232	{
2233	struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ | REQ_RAHEAD };
2234	struct page *pagepool[16];
2235	struct extent_map *em_cached = NULL;
2236	u64 prev_em_start = (u64)-1;
2237	int nr;
2238
2239	while ((nr = readahead_page_batch(rac, pagepool))) {
2240	u64 contig_start = readahead_pos(rac);
2241	u64 contig_end = contig_start + readahead_batch_length(rac) - 1;
2242
2243	contiguous_readpages(pages: pagepool, nr_pages: nr, start: contig_start, end: contig_end,
2244	em_cached: &em_cached, bio_ctrl: &bio_ctrl, prev_em_start: &prev_em_start);
2245	}
2246
2247	if (em_cached)
2248	free_extent_map(em: em_cached);
2249	submit_one_bio(bio_ctrl: &bio_ctrl);
2250	}
2251
2252	/*
2253	* basic invalidate_folio code, this waits on any locked or writeback
2254	* ranges corresponding to the folio, and then deletes any extent state
2255	* records from the tree
2256	*/
2257	int extent_invalidate_folio(struct extent_io_tree *tree,
2258	struct folio *folio, size_t offset)
2259	{
2260	struct extent_state *cached_state = NULL;
2261	u64 start = folio_pos(folio);
2262	u64 end = start + folio_size(folio) - 1;
2263	size_t blocksize = folio->mapping->host->i_sb->s_blocksize;
2264
2265	/* This function is only called for the btree inode */
2266	ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO);
2267
2268	start += ALIGN(offset, blocksize);
2269	if (start > end)
2270	return 0;
2271
2272	lock_extent(tree, start, end, cached: &cached_state);
2273	folio_wait_writeback(folio);
2274
2275	/*
2276	* Currently for btree io tree, only EXTENT_LOCKED is utilized,
2277	* so here we only need to unlock the extent range to free any
2278	* existing extent state.
2279	*/
2280	unlock_extent(tree, start, end, cached: &cached_state);
2281	return 0;
2282	}
2283
2284	/*
2285	* a helper for release_folio, this tests for areas of the page that
2286	* are locked or under IO and drops the related state bits if it is safe
2287	* to drop the page.
2288	*/
2289	static int try_release_extent_state(struct extent_io_tree *tree,
2290	struct page *page, gfp_t mask)
2291	{
2292	u64 start = page_offset(page);
2293	u64 end = start + PAGE_SIZE - 1;
2294	int ret = 1;
2295
2296	if (test_range_bit_exists(tree, start, end, bit: EXTENT_LOCKED)) {
2297	ret = 0;
2298	} else {
2299	u32 clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM |
2300	EXTENT_DELALLOC_NEW | EXTENT_CTLBITS);
2301
2302	/*
2303	* At this point we can safely clear everything except the
2304	* locked bit, the nodatasum bit and the delalloc new bit.
2305	* The delalloc new bit will be cleared by ordered extent
2306	* completion.
2307	*/
2308	ret = __clear_extent_bit(tree, start, end, bits: clear_bits, NULL, NULL);
2309
2310	/* if clear_extent_bit failed for enomem reasons,
2311	* we can't allow the release to continue.
2312	*/
2313	if (ret < 0)
2314	ret = 0;
2315	else
2316	ret = 1;
2317	}
2318	return ret;
2319	}
2320
2321	/*
2322	* a helper for release_folio. As long as there are no locked extents
2323	* in the range corresponding to the page, both state records and extent
2324	* map records are removed
2325	*/
2326	int try_release_extent_mapping(struct page *page, gfp_t mask)
2327	{
2328	struct extent_map *em;
2329	u64 start = page_offset(page);
2330	u64 end = start + PAGE_SIZE - 1;
2331	struct btrfs_inode *btrfs_inode = BTRFS_I(inode: page->mapping->host);
2332	struct extent_io_tree *tree = &btrfs_inode->io_tree;
2333	struct extent_map_tree *map = &btrfs_inode->extent_tree;
2334
2335	if (gfpflags_allow_blocking(gfp_flags: mask) &&
2336	page->mapping->host->i_size > SZ_16M) {
2337	u64 len;
2338	while (start <= end) {
2339	struct btrfs_fs_info *fs_info;
2340	u64 cur_gen;
2341
2342	len = end - start + 1;
2343	write_lock(&map->lock);
2344	em = lookup_extent_mapping(tree: map, start, len);
2345	if (!em) {
2346	write_unlock(&map->lock);
2347	break;
2348	}
2349	if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
2350	em->start != start) {
2351	write_unlock(&map->lock);
2352	free_extent_map(em);
2353	break;
2354	}
2355	if (test_range_bit_exists(tree, start: em->start,
2356	end: extent_map_end(em) - 1,
2357	bit: EXTENT_LOCKED))
2358	goto next;
2359	/*
2360	* If it's not in the list of modified extents, used
2361	* by a fast fsync, we can remove it. If it's being
2362	* logged we can safely remove it since fsync took an
2363	* extra reference on the em.
2364	*/
2365	if (list_empty(head: &em->list) ||
2366	test_bit(EXTENT_FLAG_LOGGING, &em->flags))
2367	goto remove_em;
2368	/*
2369	* If it's in the list of modified extents, remove it
2370	* only if its generation is older then the current one,
2371	* in which case we don't need it for a fast fsync.
2372	* Otherwise don't remove it, we could be racing with an
2373	* ongoing fast fsync that could miss the new extent.
2374	*/
2375	fs_info = btrfs_inode->root->fs_info;
2376	spin_lock(lock: &fs_info->trans_lock);
2377	cur_gen = fs_info->generation;
2378	spin_unlock(lock: &fs_info->trans_lock);
2379	if (em->generation >= cur_gen)
2380	goto next;
2381	remove_em:
2382	/*
2383	* We only remove extent maps that are not in the list of
2384	* modified extents or that are in the list but with a
2385	* generation lower then the current generation, so there
2386	* is no need to set the full fsync flag on the inode (it
2387	* hurts the fsync performance for workloads with a data
2388	* size that exceeds or is close to the system's memory).
2389	*/
2390	remove_extent_mapping(tree: map, em);
2391	/* once for the rb tree */
2392	free_extent_map(em);
2393	next:
2394	start = extent_map_end(em);
2395	write_unlock(&map->lock);
2396
2397	/* once for us */
2398	free_extent_map(em);
2399
2400	cond_resched(); /* Allow large-extent preemption. */
2401	}
2402	}
2403	return try_release_extent_state(tree, page, mask);
2404	}
2405
2406	/*
2407	* To cache previous fiemap extent
2408	*
2409	* Will be used for merging fiemap extent
2410	*/
2411	struct fiemap_cache {
2412	u64 offset;
2413	u64 phys;
2414	u64 len;
2415	u32 flags;
2416	bool cached;
2417	};
2418
2419	/*
2420	* Helper to submit fiemap extent.
2421	*
2422	* Will try to merge current fiemap extent specified by @offset, @phys,
2423	* @len and @flags with cached one.
2424	* And only when we fails to merge, cached one will be submitted as
2425	* fiemap extent.
2426	*
2427	* Return value is the same as fiemap_fill_next_extent().
2428	*/
2429	static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo,
2430	struct fiemap_cache *cache,
2431	u64 offset, u64 phys, u64 len, u32 flags)
2432	{
2433	int ret = 0;
2434
2435	/* Set at the end of extent_fiemap(). */
2436	ASSERT((flags & FIEMAP_EXTENT_LAST) == 0);
2437
2438	if (!cache->cached)
2439	goto assign;
2440
2441	/*
2442	* Sanity check, extent_fiemap() should have ensured that new
2443	* fiemap extent won't overlap with cached one.
2444	* Not recoverable.
2445	*
2446	* NOTE: Physical address can overlap, due to compression
2447	*/
2448	if (cache->offset + cache->len > offset) {
2449	WARN_ON(1);
2450	return -EINVAL;
2451	}
2452
2453	/*
2454	* Only merges fiemap extents if
2455	* 1) Their logical addresses are continuous
2456	*
2457	* 2) Their physical addresses are continuous
2458	* So truly compressed (physical size smaller than logical size)
2459	* extents won't get merged with each other
2460	*
2461	* 3) Share same flags
2462	*/
2463	if (cache->offset + cache->len == offset &&
2464	cache->phys + cache->len == phys &&
2465	cache->flags == flags) {
2466	cache->len += len;
2467	return 0;
2468	}
2469
2470	/* Not mergeable, need to submit cached one */
2471	ret = fiemap_fill_next_extent(info: fieinfo, logical: cache->offset, phys: cache->phys,
2472	len: cache->len, flags: cache->flags);
2473	cache->cached = false;
2474	if (ret)
2475	return ret;
2476	assign:
2477	cache->cached = true;
2478	cache->offset = offset;
2479	cache->phys = phys;
2480	cache->len = len;
2481	cache->flags = flags;
2482
2483	return 0;
2484	}
2485
2486	/*
2487	* Emit last fiemap cache
2488	*
2489	* The last fiemap cache may still be cached in the following case:
2490	* 0 4k 8k
2491	* |<- Fiemap range ->|
2492	* |<------------ First extent ----------->|
2493	*
2494	* In this case, the first extent range will be cached but not emitted.
2495	* So we must emit it before ending extent_fiemap().
2496	*/
2497	static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo,
2498	struct fiemap_cache *cache)
2499	{
2500	int ret;
2501
2502	if (!cache->cached)
2503	return 0;
2504
2505	ret = fiemap_fill_next_extent(info: fieinfo, logical: cache->offset, phys: cache->phys,
2506	len: cache->len, flags: cache->flags);
2507	cache->cached = false;
2508	if (ret > 0)
2509	ret = 0;
2510	return ret;
2511	}
2512
2513	static int fiemap_next_leaf_item(struct btrfs_inode *inode, struct btrfs_path *path)
2514	{
2515	struct extent_buffer *clone;
2516	struct btrfs_key key;
2517	int slot;
2518	int ret;
2519
2520	path->slots[0]++;
2521	if (path->slots[0] < btrfs_header_nritems(eb: path->nodes[0]))
2522	return 0;
2523
2524	ret = btrfs_next_leaf(root: inode->root, path);
2525	if (ret != 0)
2526	return ret;
2527
2528	/*
2529	* Don't bother with cloning if there are no more file extent items for
2530	* our inode.
2531	*/
2532	btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &key, nr: path->slots[0]);
2533	if (key.objectid != btrfs_ino(inode) || key.type != BTRFS_EXTENT_DATA_KEY)
2534	return 1;
2535
2536	/* See the comment at fiemap_search_slot() about why we clone. */
2537	clone = btrfs_clone_extent_buffer(src: path->nodes[0]);
2538	if (!clone)
2539	return -ENOMEM;
2540
2541	slot = path->slots[0];
2542	btrfs_release_path(p: path);
2543	path->nodes[0] = clone;
2544	path->slots[0] = slot;
2545
2546	return 0;
2547	}
2548
2549	/*
2550	* Search for the first file extent item that starts at a given file offset or
2551	* the one that starts immediately before that offset.
2552	* Returns: 0 on success, < 0 on error, 1 if not found.
2553	*/
2554	static int fiemap_search_slot(struct btrfs_inode *inode, struct btrfs_path *path,
2555	u64 file_offset)
2556	{
2557	const u64 ino = btrfs_ino(inode);
2558	struct btrfs_root *root = inode->root;
2559	struct extent_buffer *clone;
2560	struct btrfs_key key;
2561	int slot;
2562	int ret;
2563
2564	key.objectid = ino;
2565	key.type = BTRFS_EXTENT_DATA_KEY;
2566	key.offset = file_offset;
2567
2568	ret = btrfs_search_slot(NULL, root, key: &key, p: path, ins_len: 0, cow: 0);
2569	if (ret < 0)
2570	return ret;
2571
2572	if (ret > 0 && path->slots[0] > 0) {
2573	btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &key, nr: path->slots[0] - 1);
2574	if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY)
2575	path->slots[0]--;
2576	}
2577
2578	if (path->slots[0] >= btrfs_header_nritems(eb: path->nodes[0])) {
2579	ret = btrfs_next_leaf(root, path);
2580	if (ret != 0)
2581	return ret;
2582
2583	btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &key, nr: path->slots[0]);
2584	if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
2585	return 1;
2586	}
2587
2588	/*
2589	* We clone the leaf and use it during fiemap. This is because while
2590	* using the leaf we do expensive things like checking if an extent is
2591	* shared, which can take a long time. In order to prevent blocking
2592	* other tasks for too long, we use a clone of the leaf. We have locked
2593	* the file range in the inode's io tree, so we know none of our file
2594	* extent items can change. This way we avoid blocking other tasks that
2595	* want to insert items for other inodes in the same leaf or b+tree
2596	* rebalance operations (triggered for example when someone is trying
2597	* to push items into this leaf when trying to insert an item in a
2598	* neighbour leaf).
2599	* We also need the private clone because holding a read lock on an
2600	* extent buffer of the subvolume's b+tree will make lockdep unhappy
2601	* when we call fiemap_fill_next_extent(), because that may cause a page
2602	* fault when filling the user space buffer with fiemap data.
2603	*/
2604	clone = btrfs_clone_extent_buffer(src: path->nodes[0]);
2605	if (!clone)
2606	return -ENOMEM;
2607
2608	slot = path->slots[0];
2609	btrfs_release_path(p: path);
2610	path->nodes[0] = clone;
2611	path->slots[0] = slot;
2612
2613	return 0;
2614	}
2615
2616	/*
2617	* Process a range which is a hole or a prealloc extent in the inode's subvolume
2618	* btree. If @disk_bytenr is 0, we are dealing with a hole, otherwise a prealloc
2619	* extent. The end offset (@end) is inclusive.
2620	*/
2621	static int fiemap_process_hole(struct btrfs_inode *inode,
2622	struct fiemap_extent_info *fieinfo,
2623	struct fiemap_cache *cache,
2624	struct extent_state **delalloc_cached_state,
2625	struct btrfs_backref_share_check_ctx *backref_ctx,
2626	u64 disk_bytenr, u64 extent_offset,
2627	u64 extent_gen,
2628	u64 start, u64 end)
2629	{
2630	const u64 i_size = i_size_read(inode: &inode->vfs_inode);
2631	u64 cur_offset = start;
2632	u64 last_delalloc_end = 0;
2633	u32 prealloc_flags = FIEMAP_EXTENT_UNWRITTEN;
2634	bool checked_extent_shared = false;
2635	int ret;
2636
2637	/*
2638	* There can be no delalloc past i_size, so don't waste time looking for
2639	* it beyond i_size.
2640	*/
2641	while (cur_offset < end && cur_offset < i_size) {
2642	u64 delalloc_start;
2643	u64 delalloc_end;
2644	u64 prealloc_start;
2645	u64 prealloc_len = 0;
2646	bool delalloc;
2647
2648	delalloc = btrfs_find_delalloc_in_range(inode, start: cur_offset, end,
2649	cached_state: delalloc_cached_state,
2650	delalloc_start_ret: &delalloc_start,
2651	delalloc_end_ret: &delalloc_end);
2652	if (!delalloc)
2653	break;
2654
2655	/*
2656	* If this is a prealloc extent we have to report every section
2657	* of it that has no delalloc.
2658	*/
2659	if (disk_bytenr != 0) {
2660	if (last_delalloc_end == 0) {
2661	prealloc_start = start;
2662	prealloc_len = delalloc_start - start;
2663	} else {
2664	prealloc_start = last_delalloc_end + 1;
2665	prealloc_len = delalloc_start - prealloc_start;
2666	}
2667	}
2668
2669	if (prealloc_len > 0) {
2670	if (!checked_extent_shared && fieinfo->fi_extents_max) {
2671	ret = btrfs_is_data_extent_shared(inode,
2672	bytenr: disk_bytenr,
2673	extent_gen,
2674	ctx: backref_ctx);
2675	if (ret < 0)
2676	return ret;
2677	else if (ret > 0)
2678	prealloc_flags |= FIEMAP_EXTENT_SHARED;
2679
2680	checked_extent_shared = true;
2681	}
2682	ret = emit_fiemap_extent(fieinfo, cache, offset: prealloc_start,
2683	phys: disk_bytenr + extent_offset,
2684	len: prealloc_len, flags: prealloc_flags);
2685	if (ret)
2686	return ret;
2687	extent_offset += prealloc_len;
2688	}
2689
2690	ret = emit_fiemap_extent(fieinfo, cache, offset: delalloc_start, phys: 0,
2691	len: delalloc_end + 1 - delalloc_start,
2692	FIEMAP_EXTENT_DELALLOC |
2693	FIEMAP_EXTENT_UNKNOWN);
2694	if (ret)
2695	return ret;
2696
2697	last_delalloc_end = delalloc_end;
2698	cur_offset = delalloc_end + 1;
2699	extent_offset += cur_offset - delalloc_start;
2700	cond_resched();
2701	}
2702
2703	/*
2704	* Either we found no delalloc for the whole prealloc extent or we have
2705	* a prealloc extent that spans i_size or starts at or after i_size.
2706	*/
2707	if (disk_bytenr != 0 && last_delalloc_end < end) {
2708	u64 prealloc_start;
2709	u64 prealloc_len;
2710
2711	if (last_delalloc_end == 0) {
2712	prealloc_start = start;
2713	prealloc_len = end + 1 - start;
2714	} else {
2715	prealloc_start = last_delalloc_end + 1;
2716	prealloc_len = end + 1 - prealloc_start;
2717	}
2718
2719	if (!checked_extent_shared && fieinfo->fi_extents_max) {
2720	ret = btrfs_is_data_extent_shared(inode,
2721	bytenr: disk_bytenr,
2722	extent_gen,
2723	ctx: backref_ctx);
2724	if (ret < 0)
2725	return ret;
2726	else if (ret > 0)
2727	prealloc_flags |= FIEMAP_EXTENT_SHARED;
2728	}
2729	ret = emit_fiemap_extent(fieinfo, cache, offset: prealloc_start,
2730	phys: disk_bytenr + extent_offset,
2731	len: prealloc_len, flags: prealloc_flags);
2732	if (ret)
2733	return ret;
2734	}
2735
2736	return 0;
2737	}
2738
2739	static int fiemap_find_last_extent_offset(struct btrfs_inode *inode,
2740	struct btrfs_path *path,
2741	u64 *last_extent_end_ret)
2742	{
2743	const u64 ino = btrfs_ino(inode);
2744	struct btrfs_root *root = inode->root;
2745	struct extent_buffer *leaf;
2746	struct btrfs_file_extent_item *ei;
2747	struct btrfs_key key;
2748	u64 disk_bytenr;
2749	int ret;
2750
2751	/*
2752	* Lookup the last file extent. We're not using i_size here because
2753	* there might be preallocation past i_size.
2754	*/
2755	ret = btrfs_lookup_file_extent(NULL, root, path, objectid: ino, bytenr: (u64)-1, mod: 0);
2756	/* There can't be a file extent item at offset (u64)-1 */
2757	ASSERT(ret != 0);
2758	if (ret < 0)
2759	return ret;
2760
2761	/*
2762	* For a non-existing key, btrfs_search_slot() always leaves us at a
2763	* slot > 0, except if the btree is empty, which is impossible because
2764	* at least it has the inode item for this inode and all the items for
2765	* the root inode 256.
2766	*/
2767	ASSERT(path->slots[0] > 0);
2768	path->slots[0]--;
2769	leaf = path->nodes[0];
2770	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
2771	if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) {
2772	/* No file extent items in the subvolume tree. */
2773	*last_extent_end_ret = 0;
2774	return 0;
2775	}
2776
2777	/*
2778	* For an inline extent, the disk_bytenr is where inline data starts at,
2779	* so first check if we have an inline extent item before checking if we
2780	* have an implicit hole (disk_bytenr == 0).
2781	*/
2782	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
2783	if (btrfs_file_extent_type(eb: leaf, s: ei) == BTRFS_FILE_EXTENT_INLINE) {
2784	*last_extent_end_ret = btrfs_file_extent_end(path);
2785	return 0;
2786	}
2787
2788	/*
2789	* Find the last file extent item that is not a hole (when NO_HOLES is
2790	* not enabled). This should take at most 2 iterations in the worst
2791	* case: we have one hole file extent item at slot 0 of a leaf and
2792	* another hole file extent item as the last item in the previous leaf.
2793	* This is because we merge file extent items that represent holes.
2794	*/
2795	disk_bytenr = btrfs_file_extent_disk_bytenr(eb: leaf, s: ei);
2796	while (disk_bytenr == 0) {
2797	ret = btrfs_previous_item(root, path, min_objectid: ino, BTRFS_EXTENT_DATA_KEY);
2798	if (ret < 0) {
2799	return ret;
2800	} else if (ret > 0) {
2801	/* No file extent items that are not holes. */
2802	*last_extent_end_ret = 0;
2803	return 0;
2804	}
2805	leaf = path->nodes[0];
2806	ei = btrfs_item_ptr(leaf, path->slots[0],
2807	struct btrfs_file_extent_item);
2808	disk_bytenr = btrfs_file_extent_disk_bytenr(eb: leaf, s: ei);
2809	}
2810
2811	*last_extent_end_ret = btrfs_file_extent_end(path);
2812	return 0;
2813	}
2814
2815	int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo,
2816	u64 start, u64 len)
2817	{
2818	const u64 ino = btrfs_ino(inode);
2819	struct extent_state *cached_state = NULL;
2820	struct extent_state *delalloc_cached_state = NULL;
2821	struct btrfs_path *path;
2822	struct fiemap_cache cache = { 0 };
2823	struct btrfs_backref_share_check_ctx *backref_ctx;
2824	u64 last_extent_end;
2825	u64 prev_extent_end;
2826	u64 lockstart;
2827	u64 lockend;
2828	bool stopped = false;
2829	int ret;
2830
2831	backref_ctx = btrfs_alloc_backref_share_check_ctx();
2832	path = btrfs_alloc_path();
2833	if (!backref_ctx || !path) {
2834	ret = -ENOMEM;
2835	goto out;
2836	}
2837
2838	lockstart = round_down(start, inode->root->fs_info->sectorsize);
2839	lockend = round_up(start + len, inode->root->fs_info->sectorsize);
2840	prev_extent_end = lockstart;
2841
2842	btrfs_inode_lock(inode, ilock_flags: BTRFS_ILOCK_SHARED);
2843	lock_extent(tree: &inode->io_tree, start: lockstart, end: lockend, cached: &cached_state);
2844
2845	ret = fiemap_find_last_extent_offset(inode, path, last_extent_end_ret: &last_extent_end);
2846	if (ret < 0)
2847	goto out_unlock;
2848	btrfs_release_path(p: path);
2849
2850	path->reada = READA_FORWARD;
2851	ret = fiemap_search_slot(inode, path, file_offset: lockstart);
2852	if (ret < 0) {
2853	goto out_unlock;
2854	} else if (ret > 0) {
2855	/*
2856	* No file extent item found, but we may have delalloc between
2857	* the current offset and i_size. So check for that.
2858	*/
2859	ret = 0;
2860	goto check_eof_delalloc;
2861	}
2862
2863	while (prev_extent_end < lockend) {
2864	struct extent_buffer *leaf = path->nodes[0];
2865	struct btrfs_file_extent_item *ei;
2866	struct btrfs_key key;
2867	u64 extent_end;
2868	u64 extent_len;
2869	u64 extent_offset = 0;
2870	u64 extent_gen;
2871	u64 disk_bytenr = 0;
2872	u64 flags = 0;
2873	int extent_type;
2874	u8 compression;
2875
2876	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
2877	if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)
2878	break;
2879
2880	extent_end = btrfs_file_extent_end(path);
2881
2882	/*
2883	* The first iteration can leave us at an extent item that ends
2884	* before our range's start. Move to the next item.
2885	*/
2886	if (extent_end <= lockstart)
2887	goto next_item;
2888
2889	backref_ctx->curr_leaf_bytenr = leaf->start;
2890
2891	/* We have in implicit hole (NO_HOLES feature enabled). */
2892	if (prev_extent_end < key.offset) {
2893	const u64 range_end = min(key.offset, lockend) - 1;
2894
2895	ret = fiemap_process_hole(inode, fieinfo, cache: &cache,
2896	delalloc_cached_state: &delalloc_cached_state,
2897	backref_ctx, disk_bytenr: 0, extent_offset: 0, extent_gen: 0,
2898	start: prev_extent_end, end: range_end);
2899	if (ret < 0) {
2900	goto out_unlock;
2901	} else if (ret > 0) {
2902	/* fiemap_fill_next_extent() told us to stop. */
2903	stopped = true;
2904	break;
2905	}
2906
2907	/* We've reached the end of the fiemap range, stop. */
2908	if (key.offset >= lockend) {
2909	stopped = true;
2910	break;
2911	}
2912	}
2913
2914	extent_len = extent_end - key.offset;
2915	ei = btrfs_item_ptr(leaf, path->slots[0],
2916	struct btrfs_file_extent_item);
2917	compression = btrfs_file_extent_compression(eb: leaf, s: ei);
2918	extent_type = btrfs_file_extent_type(eb: leaf, s: ei);
2919	extent_gen = btrfs_file_extent_generation(eb: leaf, s: ei);
2920
2921	if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
2922	disk_bytenr = btrfs_file_extent_disk_bytenr(eb: leaf, s: ei);
2923	if (compression == BTRFS_COMPRESS_NONE)
2924	extent_offset = btrfs_file_extent_offset(eb: leaf, s: ei);
2925	}
2926
2927	if (compression != BTRFS_COMPRESS_NONE)
2928	flags |= FIEMAP_EXTENT_ENCODED;
2929
2930	if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
2931	flags |= FIEMAP_EXTENT_DATA_INLINE;
2932	flags |= FIEMAP_EXTENT_NOT_ALIGNED;
2933	ret = emit_fiemap_extent(fieinfo, cache: &cache, offset: key.offset, phys: 0,
2934	len: extent_len, flags);
2935	} else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
2936	ret = fiemap_process_hole(inode, fieinfo, cache: &cache,
2937	delalloc_cached_state: &delalloc_cached_state,
2938	backref_ctx,
2939	disk_bytenr, extent_offset,
2940	extent_gen, start: key.offset,
2941	end: extent_end - 1);
2942	} else if (disk_bytenr == 0) {
2943	/* We have an explicit hole. */
2944	ret = fiemap_process_hole(inode, fieinfo, cache: &cache,
2945	delalloc_cached_state: &delalloc_cached_state,
2946	backref_ctx, disk_bytenr: 0, extent_offset: 0, extent_gen: 0,
2947	start: key.offset, end: extent_end - 1);
2948	} else {
2949	/* We have a regular extent. */
2950	if (fieinfo->fi_extents_max) {
2951	ret = btrfs_is_data_extent_shared(inode,
2952	bytenr: disk_bytenr,
2953	extent_gen,
2954	ctx: backref_ctx);
2955	if (ret < 0)
2956	goto out_unlock;
2957	else if (ret > 0)
2958	flags |= FIEMAP_EXTENT_SHARED;
2959	}
2960
2961	ret = emit_fiemap_extent(fieinfo, cache: &cache, offset: key.offset,
2962	phys: disk_bytenr + extent_offset,
2963	len: extent_len, flags);
2964	}
2965
2966	if (ret < 0) {
2967	goto out_unlock;
2968	} else if (ret > 0) {
2969	/* fiemap_fill_next_extent() told us to stop. */
2970	stopped = true;
2971	break;
2972	}
2973
2974	prev_extent_end = extent_end;
2975	next_item:
2976	if (fatal_signal_pending(current)) {
2977	ret = -EINTR;
2978	goto out_unlock;
2979	}
2980
2981	ret = fiemap_next_leaf_item(inode, path);
2982	if (ret < 0) {
2983	goto out_unlock;
2984	} else if (ret > 0) {
2985	/* No more file extent items for this inode. */
2986	break;
2987	}
2988	cond_resched();
2989	}
2990
2991	check_eof_delalloc:
2992	/*
2993	* Release (and free) the path before emitting any final entries to
2994	* fiemap_fill_next_extent() to keep lockdep happy. This is because
2995	* once we find no more file extent items exist, we may have a
2996	* non-cloned leaf, and fiemap_fill_next_extent() can trigger page
2997	* faults when copying data to the user space buffer.
2998	*/
2999	btrfs_free_path(p: path);
3000	path = NULL;
3001
3002	if (!stopped && prev_extent_end < lockend) {
3003	ret = fiemap_process_hole(inode, fieinfo, cache: &cache,
3004	delalloc_cached_state: &delalloc_cached_state, backref_ctx,
3005	disk_bytenr: 0, extent_offset: 0, extent_gen: 0, start: prev_extent_end, end: lockend - 1);
3006	if (ret < 0)
3007	goto out_unlock;
3008	prev_extent_end = lockend;
3009	}
3010
3011	if (cache.cached && cache.offset + cache.len >= last_extent_end) {
3012	const u64 i_size = i_size_read(inode: &inode->vfs_inode);
3013
3014	if (prev_extent_end < i_size) {
3015	u64 delalloc_start;
3016	u64 delalloc_end;
3017	bool delalloc;
3018
3019	delalloc = btrfs_find_delalloc_in_range(inode,
3020	start: prev_extent_end,
3021	end: i_size - 1,
3022	cached_state: &delalloc_cached_state,
3023	delalloc_start_ret: &delalloc_start,
3024	delalloc_end_ret: &delalloc_end);
3025	if (!delalloc)
3026	cache.flags |= FIEMAP_EXTENT_LAST;
3027	} else {
3028	cache.flags |= FIEMAP_EXTENT_LAST;
3029	}
3030	}
3031
3032	ret = emit_last_fiemap_cache(fieinfo, cache: &cache);
3033
3034	out_unlock:
3035	unlock_extent(tree: &inode->io_tree, start: lockstart, end: lockend, cached: &cached_state);
3036	btrfs_inode_unlock(inode, ilock_flags: BTRFS_ILOCK_SHARED);
3037	out:
3038	free_extent_state(state: delalloc_cached_state);
3039	btrfs_free_backref_share_ctx(ctx: backref_ctx);
3040	btrfs_free_path(p: path);
3041	return ret;
3042	}
3043
3044	static void __free_extent_buffer(struct extent_buffer *eb)
3045	{
3046	kmem_cache_free(s: extent_buffer_cache, objp: eb);
3047	}
3048
3049	static int extent_buffer_under_io(const struct extent_buffer *eb)
3050	{
3051	return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
3052	test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
3053	}
3054
3055	static bool page_range_has_eb(struct btrfs_fs_info *fs_info, struct page *page)
3056	{
3057	struct btrfs_subpage *subpage;
3058
3059	lockdep_assert_held(&page->mapping->private_lock);
3060
3061	if (PagePrivate(page)) {
3062	subpage = (struct btrfs_subpage *)page->private;
3063	if (atomic_read(v: &subpage->eb_refs))
3064	return true;
3065	/*
3066	* Even there is no eb refs here, we may still have
3067	* end_page_read() call relying on page::private.
3068	*/
3069	if (atomic_read(v: &subpage->readers))
3070	return true;
3071	}
3072	return false;
3073	}
3074
3075	static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *page)
3076	{
3077	struct btrfs_fs_info *fs_info = eb->fs_info;
3078	const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3079
3080	/*
3081	* For mapped eb, we're going to change the page private, which should
3082	* be done under the private_lock.
3083	*/
3084	if (mapped)
3085	spin_lock(lock: &page->mapping->private_lock);
3086
3087	if (!PagePrivate(page)) {
3088	if (mapped)
3089	spin_unlock(lock: &page->mapping->private_lock);
3090	return;
3091	}
3092
3093	if (fs_info->nodesize >= PAGE_SIZE) {
3094	/*
3095	* We do this since we'll remove the pages after we've
3096	* removed the eb from the radix tree, so we could race
3097	* and have this page now attached to the new eb. So
3098	* only clear page_private if it's still connected to
3099	* this eb.
3100	*/
3101	if (PagePrivate(page) &&
3102	page->private == (unsigned long)eb) {
3103	BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
3104	BUG_ON(PageDirty(page));
3105	BUG_ON(PageWriteback(page));
3106	/*
3107	* We need to make sure we haven't be attached
3108	* to a new eb.
3109	*/
3110	detach_page_private(page);
3111	}
3112	if (mapped)
3113	spin_unlock(lock: &page->mapping->private_lock);
3114	return;
3115	}
3116
3117	/*
3118	* For subpage, we can have dummy eb with page private. In this case,
3119	* we can directly detach the private as such page is only attached to
3120	* one dummy eb, no sharing.
3121	*/
3122	if (!mapped) {
3123	btrfs_detach_subpage(fs_info, page);
3124	return;
3125	}
3126
3127	btrfs_page_dec_eb_refs(fs_info, page);
3128
3129	/*
3130	* We can only detach the page private if there are no other ebs in the
3131	* page range and no unfinished IO.
3132	*/
3133	if (!page_range_has_eb(fs_info, page))
3134	btrfs_detach_subpage(fs_info, page);
3135
3136	spin_unlock(lock: &page->mapping->private_lock);
3137	}
3138
3139	/* Release all pages attached to the extent buffer */
3140	static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb)
3141	{
3142	int i;
3143	int num_pages;
3144
3145	ASSERT(!extent_buffer_under_io(eb));
3146
3147	num_pages = num_extent_pages(eb);
3148	for (i = 0; i < num_pages; i++) {
3149	struct page *page = eb->pages[i];
3150
3151	if (!page)
3152	continue;
3153
3154	detach_extent_buffer_page(eb, page);
3155
3156	/* One for when we allocated the page */
3157	put_page(page);
3158	}
3159	}
3160
3161	/*
3162	* Helper for releasing the extent buffer.
3163	*/
3164	static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
3165	{
3166	btrfs_release_extent_buffer_pages(eb);
3167	btrfs_leak_debug_del_eb(eb);
3168	__free_extent_buffer(eb);
3169	}
3170
3171	static struct extent_buffer *
3172	__alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
3173	unsigned long len)
3174	{
3175	struct extent_buffer *eb = NULL;
3176
3177	eb = kmem_cache_zalloc(k: extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
3178	eb->start = start;
3179	eb->len = len;
3180	eb->fs_info = fs_info;
3181	init_rwsem(&eb->lock);
3182
3183	btrfs_leak_debug_add_eb(eb);
3184
3185	spin_lock_init(&eb->refs_lock);
3186	atomic_set(v: &eb->refs, i: 1);
3187
3188	ASSERT(len <= BTRFS_MAX_METADATA_BLOCKSIZE);
3189
3190	return eb;
3191	}
3192
3193	struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
3194	{
3195	int i;
3196	struct extent_buffer *new;
3197	int num_pages = num_extent_pages(eb: src);
3198	int ret;
3199
3200	new = __alloc_extent_buffer(fs_info: src->fs_info, start: src->start, len: src->len);
3201	if (new == NULL)
3202	return NULL;
3203
3204	/*
3205	* Set UNMAPPED before calling btrfs_release_extent_buffer(), as
3206	* btrfs_release_extent_buffer() have different behavior for
3207	* UNMAPPED subpage extent buffer.
3208	*/
3209	set_bit(nr: EXTENT_BUFFER_UNMAPPED, addr: &new->bflags);
3210
3211	ret = btrfs_alloc_page_array(nr_pages: num_pages, page_array: new->pages);
3212	if (ret) {
3213	btrfs_release_extent_buffer(eb: new);
3214	return NULL;
3215	}
3216
3217	for (i = 0; i < num_pages; i++) {
3218	int ret;
3219	struct page *p = new->pages[i];
3220
3221	ret = attach_extent_buffer_page(eb: new, page: p, NULL);
3222	if (ret < 0) {
3223	btrfs_release_extent_buffer(eb: new);
3224	return NULL;
3225	}
3226	WARN_ON(PageDirty(p));
3227	}
3228	copy_extent_buffer_full(dst: new, src);
3229	set_extent_buffer_uptodate(new);
3230
3231	return new;
3232	}
3233
3234	struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3235	u64 start, unsigned long len)
3236	{
3237	struct extent_buffer *eb;
3238	int num_pages;
3239	int i;
3240	int ret;
3241
3242	eb = __alloc_extent_buffer(fs_info, start, len);
3243	if (!eb)
3244	return NULL;
3245
3246	num_pages = num_extent_pages(eb);
3247	ret = btrfs_alloc_page_array(nr_pages: num_pages, page_array: eb->pages);
3248	if (ret)
3249	goto err;
3250
3251	for (i = 0; i < num_pages; i++) {
3252	struct page *p = eb->pages[i];
3253
3254	ret = attach_extent_buffer_page(eb, page: p, NULL);
3255	if (ret < 0)
3256	goto err;
3257	}
3258
3259	set_extent_buffer_uptodate(eb);
3260	btrfs_set_header_nritems(eb, val: 0);
3261	set_bit(nr: EXTENT_BUFFER_UNMAPPED, addr: &eb->bflags);
3262
3263	return eb;
3264	err:
3265	for (i = 0; i < num_pages; i++) {
3266	if (eb->pages[i]) {
3267	detach_extent_buffer_page(eb, page: eb->pages[i]);
3268	__free_page(eb->pages[i]);
3269	}
3270	}
3271	__free_extent_buffer(eb);
3272	return NULL;
3273	}
3274
3275	struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3276	u64 start)
3277	{
3278	return __alloc_dummy_extent_buffer(fs_info, start, len: fs_info->nodesize);
3279	}
3280
3281	static void check_buffer_tree_ref(struct extent_buffer *eb)
3282	{
3283	int refs;
3284	/*
3285	* The TREE_REF bit is first set when the extent_buffer is added
3286	* to the radix tree. It is also reset, if unset, when a new reference
3287	* is created by find_extent_buffer.
3288	*
3289	* It is only cleared in two cases: freeing the last non-tree
3290	* reference to the extent_buffer when its STALE bit is set or
3291	* calling release_folio when the tree reference is the only reference.
3292	*
3293	* In both cases, care is taken to ensure that the extent_buffer's
3294	* pages are not under io. However, release_folio can be concurrently
3295	* called with creating new references, which is prone to race
3296	* conditions between the calls to check_buffer_tree_ref in those
3297	* codepaths and clearing TREE_REF in try_release_extent_buffer.
3298	*
3299	* The actual lifetime of the extent_buffer in the radix tree is
3300	* adequately protected by the refcount, but the TREE_REF bit and
3301	* its corresponding reference are not. To protect against this
3302	* class of races, we call check_buffer_tree_ref from the codepaths
3303	* which trigger io. Note that once io is initiated, TREE_REF can no
3304	* longer be cleared, so that is the moment at which any such race is
3305	* best fixed.
3306	*/
3307	refs = atomic_read(v: &eb->refs);
3308	if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3309	return;
3310
3311	spin_lock(lock: &eb->refs_lock);
3312	if (!test_and_set_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags))
3313	atomic_inc(v: &eb->refs);
3314	spin_unlock(lock: &eb->refs_lock);
3315	}
3316
3317	static void mark_extent_buffer_accessed(struct extent_buffer *eb,
3318	struct page *accessed)
3319	{
3320	int num_pages, i;
3321
3322	check_buffer_tree_ref(eb);
3323
3324	num_pages = num_extent_pages(eb);
3325	for (i = 0; i < num_pages; i++) {
3326	struct page *p = eb->pages[i];
3327
3328	if (p != accessed)
3329	mark_page_accessed(p);
3330	}
3331	}
3332
3333	struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
3334	u64 start)
3335	{
3336	struct extent_buffer *eb;
3337
3338	eb = find_extent_buffer_nolock(fs_info, start);
3339	if (!eb)
3340	return NULL;
3341	/*
3342	* Lock our eb's refs_lock to avoid races with free_extent_buffer().
3343	* When we get our eb it might be flagged with EXTENT_BUFFER_STALE and
3344	* another task running free_extent_buffer() might have seen that flag
3345	* set, eb->refs == 2, that the buffer isn't under IO (dirty and
3346	* writeback flags not set) and it's still in the tree (flag
3347	* EXTENT_BUFFER_TREE_REF set), therefore being in the process of
3348	* decrementing the extent buffer's reference count twice. So here we
3349	* could race and increment the eb's reference count, clear its stale
3350	* flag, mark it as dirty and drop our reference before the other task
3351	* finishes executing free_extent_buffer, which would later result in
3352	* an attempt to free an extent buffer that is dirty.
3353	*/
3354	if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
3355	spin_lock(lock: &eb->refs_lock);
3356	spin_unlock(lock: &eb->refs_lock);
3357	}
3358	mark_extent_buffer_accessed(eb, NULL);
3359	return eb;
3360	}
3361
3362	#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3363	struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
3364	u64 start)
3365	{
3366	struct extent_buffer *eb, *exists = NULL;
3367	int ret;
3368
3369	eb = find_extent_buffer(fs_info, start);
3370	if (eb)
3371	return eb;
3372	eb = alloc_dummy_extent_buffer(fs_info, start);
3373	if (!eb)
3374	return ERR_PTR(error: -ENOMEM);
3375	eb->fs_info = fs_info;
3376	again:
3377	ret = radix_tree_preload(GFP_NOFS);
3378	if (ret) {
3379	exists = ERR_PTR(error: ret);
3380	goto free_eb;
3381	}
3382	spin_lock(lock: &fs_info->buffer_lock);
3383	ret = radix_tree_insert(&fs_info->buffer_radix,
3384	index: start >> fs_info->sectorsize_bits, eb);
3385	spin_unlock(lock: &fs_info->buffer_lock);
3386	radix_tree_preload_end();
3387	if (ret == -EEXIST) {
3388	exists = find_extent_buffer(fs_info, start);
3389	if (exists)
3390	goto free_eb;
3391	else
3392	goto again;
3393	}
3394	check_buffer_tree_ref(eb);
3395	set_bit(nr: EXTENT_BUFFER_IN_TREE, addr: &eb->bflags);
3396
3397	return eb;
3398	free_eb:
3399	btrfs_release_extent_buffer(eb);
3400	return exists;
3401	}
3402	#endif
3403
3404	static struct extent_buffer *grab_extent_buffer(
3405	struct btrfs_fs_info *fs_info, struct page *page)
3406	{
3407	struct extent_buffer *exists;
3408
3409	/*
3410	* For subpage case, we completely rely on radix tree to ensure we
3411	* don't try to insert two ebs for the same bytenr. So here we always
3412	* return NULL and just continue.
3413	*/
3414	if (fs_info->nodesize < PAGE_SIZE)
3415	return NULL;
3416
3417	/* Page not yet attached to an extent buffer */
3418	if (!PagePrivate(page))
3419	return NULL;
3420
3421	/*
3422	* We could have already allocated an eb for this page and attached one
3423	* so lets see if we can get a ref on the existing eb, and if we can we
3424	* know it's good and we can just return that one, else we know we can
3425	* just overwrite page->private.
3426	*/
3427	exists = (struct extent_buffer *)page->private;
3428	if (atomic_inc_not_zero(v: &exists->refs))
3429	return exists;
3430
3431	WARN_ON(PageDirty(page));
3432	detach_page_private(page);
3433	return NULL;
3434	}
3435
3436	static int check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start)
3437	{
3438	if (!IS_ALIGNED(start, fs_info->sectorsize)) {
3439	btrfs_err(fs_info, "bad tree block start %llu", start);
3440	return -EINVAL;
3441	}
3442
3443	if (fs_info->nodesize < PAGE_SIZE &&
3444	offset_in_page(start) + fs_info->nodesize > PAGE_SIZE) {
3445	btrfs_err(fs_info,
3446	"tree block crosses page boundary, start %llu nodesize %u",
3447	start, fs_info->nodesize);
3448	return -EINVAL;
3449	}
3450	if (fs_info->nodesize >= PAGE_SIZE &&
3451	!PAGE_ALIGNED(start)) {
3452	btrfs_err(fs_info,
3453	"tree block is not page aligned, start %llu nodesize %u",
3454	start, fs_info->nodesize);
3455	return -EINVAL;
3456	}
3457	if (!IS_ALIGNED(start, fs_info->nodesize) &&
3458	!test_and_set_bit(nr: BTRFS_FS_UNALIGNED_TREE_BLOCK, addr: &fs_info->flags)) {
3459	btrfs_warn(fs_info,
3460	"tree block not nodesize aligned, start %llu nodesize %u, can be resolved by a full metadata balance",
3461	start, fs_info->nodesize);
3462	}
3463	return 0;
3464	}
3465
3466	struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
3467	u64 start, u64 owner_root, int level)
3468	{
3469	unsigned long len = fs_info->nodesize;
3470	int num_pages;
3471	int i;
3472	unsigned long index = start >> PAGE_SHIFT;
3473	struct extent_buffer *eb;
3474	struct extent_buffer *exists = NULL;
3475	struct page *p;
3476	struct address_space *mapping = fs_info->btree_inode->i_mapping;
3477	struct btrfs_subpage *prealloc = NULL;
3478	u64 lockdep_owner = owner_root;
3479	int uptodate = 1;
3480	int ret;
3481
3482	if (check_eb_alignment(fs_info, start))
3483	return ERR_PTR(error: -EINVAL);
3484
3485	#if BITS_PER_LONG == 32
3486	if (start >= MAX_LFS_FILESIZE) {
3487	btrfs_err_rl(fs_info,
3488	"extent buffer %llu is beyond 32bit page cache limit", start);
3489	btrfs_err_32bit_limit(fs_info);
3490	return ERR_PTR(-EOVERFLOW);
3491	}
3492	if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD)
3493	btrfs_warn_32bit_limit(fs_info);
3494	#endif
3495
3496	eb = find_extent_buffer(fs_info, start);
3497	if (eb)
3498	return eb;
3499
3500	eb = __alloc_extent_buffer(fs_info, start, len);
3501	if (!eb)
3502	return ERR_PTR(error: -ENOMEM);
3503
3504	/*
3505	* The reloc trees are just snapshots, so we need them to appear to be
3506	* just like any other fs tree WRT lockdep.
3507	*/
3508	if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID)
3509	lockdep_owner = BTRFS_FS_TREE_OBJECTID;
3510
3511	btrfs_set_buffer_lockdep_class(objectid: lockdep_owner, eb, level);
3512
3513	num_pages = num_extent_pages(eb);
3514
3515	/*
3516	* Preallocate page->private for subpage case, so that we won't
3517	* allocate memory with private_lock nor page lock hold.
3518	*
3519	* The memory will be freed by attach_extent_buffer_page() or freed
3520	* manually if we exit earlier.
3521	*/
3522	if (fs_info->nodesize < PAGE_SIZE) {
3523	prealloc = btrfs_alloc_subpage(fs_info, type: BTRFS_SUBPAGE_METADATA);
3524	if (IS_ERR(ptr: prealloc)) {
3525	exists = ERR_CAST(ptr: prealloc);
3526	goto free_eb;
3527	}
3528	}
3529
3530	for (i = 0; i < num_pages; i++, index++) {
3531	p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
3532	if (!p) {
3533	exists = ERR_PTR(error: -ENOMEM);
3534	btrfs_free_subpage(subpage: prealloc);
3535	goto free_eb;
3536	}
3537
3538	spin_lock(lock: &mapping->private_lock);
3539	exists = grab_extent_buffer(fs_info, page: p);
3540	if (exists) {
3541	spin_unlock(lock: &mapping->private_lock);
3542	unlock_page(page: p);
3543	put_page(page: p);
3544	mark_extent_buffer_accessed(eb: exists, accessed: p);
3545	btrfs_free_subpage(subpage: prealloc);
3546	goto free_eb;
3547	}
3548	/* Should not fail, as we have preallocated the memory */
3549	ret = attach_extent_buffer_page(eb, page: p, prealloc);
3550	ASSERT(!ret);
3551	/*
3552	* To inform we have extra eb under allocation, so that
3553	* detach_extent_buffer_page() won't release the page private
3554	* when the eb hasn't yet been inserted into radix tree.
3555	*
3556	* The ref will be decreased when the eb released the page, in
3557	* detach_extent_buffer_page().
3558	* Thus needs no special handling in error path.
3559	*/
3560	btrfs_page_inc_eb_refs(fs_info, page: p);
3561	spin_unlock(lock: &mapping->private_lock);
3562
3563	WARN_ON(btrfs_page_test_dirty(fs_info, p, eb->start, eb->len));
3564	eb->pages[i] = p;
3565	if (!btrfs_page_test_uptodate(fs_info, page: p, start: eb->start, len: eb->len))
3566	uptodate = 0;
3567
3568	/*
3569	* We can't unlock the pages just yet since the extent buffer
3570	* hasn't been properly inserted in the radix tree, this
3571	* opens a race with btree_release_folio which can free a page
3572	* while we are still filling in all pages for the buffer and
3573	* we could crash.
3574	*/
3575	}
3576	if (uptodate)
3577	set_bit(nr: EXTENT_BUFFER_UPTODATE, addr: &eb->bflags);
3578	again:
3579	ret = radix_tree_preload(GFP_NOFS);
3580	if (ret) {
3581	exists = ERR_PTR(error: ret);
3582	goto free_eb;
3583	}
3584
3585	spin_lock(lock: &fs_info->buffer_lock);
3586	ret = radix_tree_insert(&fs_info->buffer_radix,
3587	index: start >> fs_info->sectorsize_bits, eb);
3588	spin_unlock(lock: &fs_info->buffer_lock);
3589	radix_tree_preload_end();
3590	if (ret == -EEXIST) {
3591	exists = find_extent_buffer(fs_info, start);
3592	if (exists)
3593	goto free_eb;
3594	else
3595	goto again;
3596	}
3597	/* add one reference for the tree */
3598	check_buffer_tree_ref(eb);
3599	set_bit(nr: EXTENT_BUFFER_IN_TREE, addr: &eb->bflags);
3600
3601	/*
3602	* Now it's safe to unlock the pages because any calls to
3603	* btree_release_folio will correctly detect that a page belongs to a
3604	* live buffer and won't free them prematurely.
3605	*/
3606	for (i = 0; i < num_pages; i++)
3607	unlock_page(page: eb->pages[i]);
3608	return eb;
3609
3610	free_eb:
3611	WARN_ON(!atomic_dec_and_test(&eb->refs));
3612	for (i = 0; i < num_pages; i++) {
3613	if (eb->pages[i])
3614	unlock_page(page: eb->pages[i]);
3615	}
3616
3617	btrfs_release_extent_buffer(eb);
3618	return exists;
3619	}
3620
3621	static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
3622	{
3623	struct extent_buffer *eb =
3624	container_of(head, struct extent_buffer, rcu_head);
3625
3626	__free_extent_buffer(eb);
3627	}
3628
3629	static int release_extent_buffer(struct extent_buffer *eb)
3630	__releases(&eb->refs_lock)
3631	{
3632	lockdep_assert_held(&eb->refs_lock);
3633
3634	WARN_ON(atomic_read(&eb->refs) == 0);
3635	if (atomic_dec_and_test(v: &eb->refs)) {
3636	if (test_and_clear_bit(nr: EXTENT_BUFFER_IN_TREE, addr: &eb->bflags)) {
3637	struct btrfs_fs_info *fs_info = eb->fs_info;
3638
3639	spin_unlock(lock: &eb->refs_lock);
3640
3641	spin_lock(lock: &fs_info->buffer_lock);
3642	radix_tree_delete(&fs_info->buffer_radix,
3643	eb->start >> fs_info->sectorsize_bits);
3644	spin_unlock(lock: &fs_info->buffer_lock);
3645	} else {
3646	spin_unlock(lock: &eb->refs_lock);
3647	}
3648
3649	btrfs_leak_debug_del_eb(eb);
3650	/* Should be safe to release our pages at this point */
3651	btrfs_release_extent_buffer_pages(eb);
3652	#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3653	if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
3654	__free_extent_buffer(eb);
3655	return 1;
3656	}
3657	#endif
3658	call_rcu(head: &eb->rcu_head, func: btrfs_release_extent_buffer_rcu);
3659	return 1;
3660	}
3661	spin_unlock(lock: &eb->refs_lock);
3662
3663	return 0;
3664	}
3665
3666	void free_extent_buffer(struct extent_buffer *eb)
3667	{
3668	int refs;
3669	if (!eb)
3670	return;
3671
3672	refs = atomic_read(v: &eb->refs);
3673	while (1) {
3674	if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
3675	|| (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
3676	refs == 1))
3677	break;
3678	if (atomic_try_cmpxchg(v: &eb->refs, old: &refs, new: refs - 1))
3679	return;
3680	}
3681
3682	spin_lock(lock: &eb->refs_lock);
3683	if (atomic_read(v: &eb->refs) == 2 &&
3684	test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
3685	!extent_buffer_under_io(eb) &&
3686	test_and_clear_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags))
3687	atomic_dec(v: &eb->refs);
3688
3689	/*
3690	* I know this is terrible, but it's temporary until we stop tracking
3691	* the uptodate bits and such for the extent buffers.
3692	*/
3693	release_extent_buffer(eb);
3694	}
3695
3696	void free_extent_buffer_stale(struct extent_buffer *eb)
3697	{
3698	if (!eb)
3699	return;
3700
3701	spin_lock(lock: &eb->refs_lock);
3702	set_bit(nr: EXTENT_BUFFER_STALE, addr: &eb->bflags);
3703
3704	if (atomic_read(v: &eb->refs) == 2 && !extent_buffer_under_io(eb) &&
3705	test_and_clear_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags))
3706	atomic_dec(v: &eb->refs);
3707	release_extent_buffer(eb);
3708	}
3709
3710	static void btree_clear_page_dirty(struct page *page)
3711	{
3712	ASSERT(PageDirty(page));
3713	ASSERT(PageLocked(page));
3714	clear_page_dirty_for_io(page);
3715	xa_lock_irq(&page->mapping->i_pages);
3716	if (!PageDirty(page))
3717	__xa_clear_mark(&page->mapping->i_pages,
3718	index: page_index(page), PAGECACHE_TAG_DIRTY);
3719	xa_unlock_irq(&page->mapping->i_pages);
3720	}
3721
3722	static void clear_subpage_extent_buffer_dirty(const struct extent_buffer *eb)
3723	{
3724	struct btrfs_fs_info *fs_info = eb->fs_info;
3725	struct page *page = eb->pages[0];
3726	bool last;
3727
3728	/* btree_clear_page_dirty() needs page locked */
3729	lock_page(page);
3730	last = btrfs_subpage_clear_and_test_dirty(fs_info, page, start: eb->start,
3731	len: eb->len);
3732	if (last)
3733	btree_clear_page_dirty(page);
3734	unlock_page(page);
3735	WARN_ON(atomic_read(&eb->refs) == 0);
3736	}
3737
3738	void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
3739	struct extent_buffer *eb)
3740	{
3741	struct btrfs_fs_info *fs_info = eb->fs_info;
3742	int i;
3743	int num_pages;
3744	struct page *page;
3745
3746	btrfs_assert_tree_write_locked(eb);
3747
3748	if (trans && btrfs_header_generation(eb) != trans->transid)
3749	return;
3750
3751	if (!test_and_clear_bit(nr: EXTENT_BUFFER_DIRTY, addr: &eb->bflags))
3752	return;
3753
3754	percpu_counter_add_batch(fbc: &fs_info->dirty_metadata_bytes, amount: -eb->len,
3755	batch: fs_info->dirty_metadata_batch);
3756
3757	if (eb->fs_info->nodesize < PAGE_SIZE)
3758	return clear_subpage_extent_buffer_dirty(eb);
3759
3760	num_pages = num_extent_pages(eb);
3761
3762	for (i = 0; i < num_pages; i++) {
3763	page = eb->pages[i];
3764	if (!PageDirty(page))
3765	continue;
3766	lock_page(page);
3767	btree_clear_page_dirty(page);
3768	unlock_page(page);
3769	}
3770	WARN_ON(atomic_read(&eb->refs) == 0);
3771	}
3772
3773	void set_extent_buffer_dirty(struct extent_buffer *eb)
3774	{
3775	int i;
3776	int num_pages;
3777	bool was_dirty;
3778
3779	check_buffer_tree_ref(eb);
3780
3781	was_dirty = test_and_set_bit(nr: EXTENT_BUFFER_DIRTY, addr: &eb->bflags);
3782
3783	num_pages = num_extent_pages(eb);
3784	WARN_ON(atomic_read(&eb->refs) == 0);
3785	WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
3786
3787	if (!was_dirty) {
3788	bool subpage = eb->fs_info->nodesize < PAGE_SIZE;
3789
3790	/*
3791	* For subpage case, we can have other extent buffers in the
3792	* same page, and in clear_subpage_extent_buffer_dirty() we
3793	* have to clear page dirty without subpage lock held.
3794	* This can cause race where our page gets dirty cleared after
3795	* we just set it.
3796	*
3797	* Thankfully, clear_subpage_extent_buffer_dirty() has locked
3798	* its page for other reasons, we can use page lock to prevent
3799	* the above race.
3800	*/
3801	if (subpage)
3802	lock_page(page: eb->pages[0]);
3803	for (i = 0; i < num_pages; i++)
3804	btrfs_page_set_dirty(fs_info: eb->fs_info, page: eb->pages[i],
3805	start: eb->start, len: eb->len);
3806	if (subpage)
3807	unlock_page(page: eb->pages[0]);
3808	percpu_counter_add_batch(fbc: &eb->fs_info->dirty_metadata_bytes,
3809	amount: eb->len,
3810	batch: eb->fs_info->dirty_metadata_batch);
3811	}
3812	#ifdef CONFIG_BTRFS_DEBUG
3813	for (i = 0; i < num_pages; i++)
3814	ASSERT(PageDirty(eb->pages[i]));
3815	#endif
3816	}
3817
3818	void clear_extent_buffer_uptodate(struct extent_buffer *eb)
3819	{
3820	struct btrfs_fs_info *fs_info = eb->fs_info;
3821	struct page *page;
3822	int num_pages;
3823	int i;
3824
3825	clear_bit(nr: EXTENT_BUFFER_UPTODATE, addr: &eb->bflags);
3826	num_pages = num_extent_pages(eb);
3827	for (i = 0; i < num_pages; i++) {
3828	page = eb->pages[i];
3829	if (!page)
3830	continue;
3831
3832	/*
3833	* This is special handling for metadata subpage, as regular
3834	* btrfs_is_subpage() can not handle cloned/dummy metadata.
3835	*/
3836	if (fs_info->nodesize >= PAGE_SIZE)
3837	ClearPageUptodate(page);
3838	else
3839	btrfs_subpage_clear_uptodate(fs_info, page, start: eb->start,
3840	len: eb->len);
3841	}
3842	}
3843
3844	void set_extent_buffer_uptodate(struct extent_buffer *eb)
3845	{
3846	struct btrfs_fs_info *fs_info = eb->fs_info;
3847	struct page *page;
3848	int num_pages;
3849	int i;
3850
3851	set_bit(nr: EXTENT_BUFFER_UPTODATE, addr: &eb->bflags);
3852	num_pages = num_extent_pages(eb);
3853	for (i = 0; i < num_pages; i++) {
3854	page = eb->pages[i];
3855
3856	/*
3857	* This is special handling for metadata subpage, as regular
3858	* btrfs_is_subpage() can not handle cloned/dummy metadata.
3859	*/
3860	if (fs_info->nodesize >= PAGE_SIZE)
3861	SetPageUptodate(page);
3862	else
3863	btrfs_subpage_set_uptodate(fs_info, page, start: eb->start,
3864	len: eb->len);
3865	}
3866	}
3867
3868	static void extent_buffer_read_end_io(struct btrfs_bio *bbio)
3869	{
3870	struct extent_buffer *eb = bbio->private;
3871	struct btrfs_fs_info *fs_info = eb->fs_info;
3872	bool uptodate = !bbio->bio.bi_status;
3873	struct bvec_iter_all iter_all;
3874	struct bio_vec *bvec;
3875	u32 bio_offset = 0;
3876
3877	eb->read_mirror = bbio->mirror_num;
3878
3879	if (uptodate &&
3880	btrfs_validate_extent_buffer(eb, check: &bbio->parent_check) < 0)
3881	uptodate = false;
3882
3883	if (uptodate) {
3884	set_extent_buffer_uptodate(eb);
3885	} else {
3886	clear_extent_buffer_uptodate(eb);
3887	set_bit(nr: EXTENT_BUFFER_READ_ERR, addr: &eb->bflags);
3888	}
3889
3890	bio_for_each_segment_all(bvec, &bbio->bio, iter_all) {
3891	u64 start = eb->start + bio_offset;
3892	struct page *page = bvec->bv_page;
3893	u32 len = bvec->bv_len;
3894
3895	if (uptodate)
3896	btrfs_page_set_uptodate(fs_info, page, start, len);
3897	else
3898	btrfs_page_clear_uptodate(fs_info, page, start, len);
3899
3900	bio_offset += len;
3901	}
3902
3903	clear_bit(nr: EXTENT_BUFFER_READING, addr: &eb->bflags);
3904	smp_mb__after_atomic();
3905	wake_up_bit(word: &eb->bflags, bit: EXTENT_BUFFER_READING);
3906	free_extent_buffer(eb);
3907
3908	bio_put(&bbio->bio);
3909	}
3910
3911	int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num,
3912	struct btrfs_tree_parent_check *check)
3913	{
3914	int num_pages = num_extent_pages(eb), i;
3915	struct btrfs_bio *bbio;
3916
3917	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
3918	return 0;
3919
3920	/*
3921	* We could have had EXTENT_BUFFER_UPTODATE cleared by the write
3922	* operation, which could potentially still be in flight. In this case
3923	* we simply want to return an error.
3924	*/
3925	if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)))
3926	return -EIO;
3927
3928	/* Someone else is already reading the buffer, just wait for it. */
3929	if (test_and_set_bit(nr: EXTENT_BUFFER_READING, addr: &eb->bflags))
3930	goto done;
3931
3932	clear_bit(nr: EXTENT_BUFFER_READ_ERR, addr: &eb->bflags);
3933	eb->read_mirror = 0;
3934	check_buffer_tree_ref(eb);
3935	atomic_inc(v: &eb->refs);
3936
3937	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
3938	opf: REQ_OP_READ | REQ_META, fs_info: eb->fs_info,
3939	end_io: extent_buffer_read_end_io, private: eb);
3940	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
3941	bbio->inode = BTRFS_I(inode: eb->fs_info->btree_inode);
3942	bbio->file_offset = eb->start;
3943	memcpy(&bbio->parent_check, check, sizeof(*check));
3944	if (eb->fs_info->nodesize < PAGE_SIZE) {
3945	__bio_add_page(bio: &bbio->bio, page: eb->pages[0], len: eb->len,
3946	off: eb->start - page_offset(page: eb->pages[0]));
3947	} else {
3948	for (i = 0; i < num_pages; i++)
3949	__bio_add_page(bio: &bbio->bio, page: eb->pages[i], PAGE_SIZE, off: 0);
3950	}
3951	btrfs_submit_bio(bbio, mirror_num);
3952
3953	done:
3954	if (wait == WAIT_COMPLETE) {
3955	wait_on_bit_io(word: &eb->bflags, bit: EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE);
3956	if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
3957	return -EIO;
3958	}
3959
3960	return 0;
3961	}
3962
3963	static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
3964	unsigned long len)
3965	{
3966	btrfs_warn(eb->fs_info,
3967	"access to eb bytenr %llu len %lu out of range start %lu len %lu",
3968	eb->start, eb->len, start, len);
3969	WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
3970
3971	return true;
3972	}
3973
3974	/*
3975	* Check if the [start, start + len) range is valid before reading/writing
3976	* the eb.
3977	* NOTE: @start and @len are offset inside the eb, not logical address.
3978	*
3979	* Caller should not touch the dst/src memory if this function returns error.
3980	*/
3981	static inline int check_eb_range(const struct extent_buffer *eb,
3982	unsigned long start, unsigned long len)
3983	{
3984	unsigned long offset;
3985
3986	/* start, start + len should not go beyond eb->len nor overflow */
3987	if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
3988	return report_eb_range(eb, start, len);
3989
3990	return false;
3991	}
3992
3993	void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
3994	unsigned long start, unsigned long len)
3995	{
3996	size_t cur;
3997	size_t offset;
3998	struct page *page;
3999	char *kaddr;
4000	char *dst = (char *)dstv;
4001	unsigned long i = get_eb_page_index(offset: start);
4002
4003	if (check_eb_range(eb, start, len)) {
4004	/*
4005	* Invalid range hit, reset the memory, so callers won't get
4006	* some random garbage for their uninitialzed memory.
4007	*/
4008	memset(dstv, 0, len);
4009	return;
4010	}
4011
4012	offset = get_eb_offset_in_page(eb, offset: start);
4013
4014	while (len > 0) {
4015	page = eb->pages[i];
4016
4017	cur = min(len, (PAGE_SIZE - offset));
4018	kaddr = page_address(page);
4019	memcpy(dst, kaddr + offset, cur);
4020
4021	dst += cur;
4022	len -= cur;
4023	offset = 0;
4024	i++;
4025	}
4026	}
4027
4028	int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
4029	void __user *dstv,
4030	unsigned long start, unsigned long len)
4031	{
4032	size_t cur;
4033	size_t offset;
4034	struct page *page;
4035	char *kaddr;
4036	char __user *dst = (char __user *)dstv;
4037	unsigned long i = get_eb_page_index(offset: start);
4038	int ret = 0;
4039
4040	WARN_ON(start > eb->len);
4041	WARN_ON(start + len > eb->start + eb->len);
4042
4043	offset = get_eb_offset_in_page(eb, offset: start);
4044
4045	while (len > 0) {
4046	page = eb->pages[i];
4047
4048	cur = min(len, (PAGE_SIZE - offset));
4049	kaddr = page_address(page);
4050	if (copy_to_user_nofault(dst, src: kaddr + offset, size: cur)) {
4051	ret = -EFAULT;
4052	break;
4053	}
4054
4055	dst += cur;
4056	len -= cur;
4057	offset = 0;
4058	i++;
4059	}
4060
4061	return ret;
4062	}
4063
4064	int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
4065	unsigned long start, unsigned long len)
4066	{
4067	size_t cur;
4068	size_t offset;
4069	struct page *page;
4070	char *kaddr;
4071	char *ptr = (char *)ptrv;
4072	unsigned long i = get_eb_page_index(offset: start);
4073	int ret = 0;
4074
4075	if (check_eb_range(eb, start, len))
4076	return -EINVAL;
4077
4078	offset = get_eb_offset_in_page(eb, offset: start);
4079
4080	while (len > 0) {
4081	page = eb->pages[i];
4082
4083	cur = min(len, (PAGE_SIZE - offset));
4084
4085	kaddr = page_address(page);
4086	ret = memcmp(p: ptr, q: kaddr + offset, size: cur);
4087	if (ret)
4088	break;
4089
4090	ptr += cur;
4091	len -= cur;
4092	offset = 0;
4093	i++;
4094	}
4095	return ret;
4096	}
4097
4098	/*
4099	* Check that the extent buffer is uptodate.
4100	*
4101	* For regular sector size == PAGE_SIZE case, check if @page is uptodate.
4102	* For subpage case, check if the range covered by the eb has EXTENT_UPTODATE.
4103	*/
4104	static void assert_eb_page_uptodate(const struct extent_buffer *eb,
4105	struct page *page)
4106	{
4107	struct btrfs_fs_info *fs_info = eb->fs_info;
4108
4109	/*
4110	* If we are using the commit root we could potentially clear a page
4111	* Uptodate while we're using the extent buffer that we've previously
4112	* looked up. We don't want to complain in this case, as the page was
4113	* valid before, we just didn't write it out. Instead we want to catch
4114	* the case where we didn't actually read the block properly, which
4115	* would have !PageUptodate and !EXTENT_BUFFER_WRITE_ERR.
4116	*/
4117	if (test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
4118	return;
4119
4120	if (fs_info->nodesize < PAGE_SIZE) {
4121	if (WARN_ON(!btrfs_subpage_test_uptodate(fs_info, page,
4122	eb->start, eb->len)))
4123	btrfs_subpage_dump_bitmap(fs_info, page, start: eb->start, len: eb->len);
4124	} else {
4125	WARN_ON(!PageUptodate(page));
4126	}
4127	}
4128
4129	static void __write_extent_buffer(const struct extent_buffer *eb,
4130	const void *srcv, unsigned long start,
4131	unsigned long len, bool use_memmove)
4132	{
4133	size_t cur;
4134	size_t offset;
4135	struct page *page;
4136	char *kaddr;
4137	char *src = (char *)srcv;
4138	unsigned long i = get_eb_page_index(offset: start);
4139	/* For unmapped (dummy) ebs, no need to check their uptodate status. */
4140	const bool check_uptodate = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
4141
4142	WARN_ON(test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags));
4143
4144	if (check_eb_range(eb, start, len))
4145	return;
4146
4147	offset = get_eb_offset_in_page(eb, offset: start);
4148
4149	while (len > 0) {
4150	page = eb->pages[i];
4151	if (check_uptodate)
4152	assert_eb_page_uptodate(eb, page);
4153
4154	cur = min(len, PAGE_SIZE - offset);
4155	kaddr = page_address(page);
4156	if (use_memmove)
4157	memmove(kaddr + offset, src, cur);
4158	else
4159	memcpy(kaddr + offset, src, cur);
4160
4161	src += cur;
4162	len -= cur;
4163	offset = 0;
4164	i++;
4165	}
4166	}
4167
4168	void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
4169	unsigned long start, unsigned long len)
4170	{
4171	return __write_extent_buffer(eb, srcv, start, len, use_memmove: false);
4172	}
4173
4174	static void memset_extent_buffer(const struct extent_buffer *eb, int c,
4175	unsigned long start, unsigned long len)
4176	{
4177	unsigned long cur = start;
4178
4179	while (cur < start + len) {
4180	unsigned long index = get_eb_page_index(offset: cur);
4181	unsigned int offset = get_eb_offset_in_page(eb, offset: cur);
4182	unsigned int cur_len = min(start + len - cur, PAGE_SIZE - offset);
4183	struct page *page = eb->pages[index];
4184
4185	assert_eb_page_uptodate(eb, page);
4186	memset(page_address(page) + offset, c, cur_len);
4187
4188	cur += cur_len;
4189	}
4190	}
4191
4192	void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
4193	unsigned long len)
4194	{
4195	if (check_eb_range(eb, start, len))
4196	return;
4197	return memset_extent_buffer(eb, c: 0, start, len);
4198	}
4199
4200	void copy_extent_buffer_full(const struct extent_buffer *dst,
4201	const struct extent_buffer *src)
4202	{
4203	unsigned long cur = 0;
4204
4205	ASSERT(dst->len == src->len);
4206
4207	while (cur < src->len) {
4208	unsigned long index = get_eb_page_index(offset: cur);
4209	unsigned long offset = get_eb_offset_in_page(eb: src, offset: cur);
4210	unsigned long cur_len = min(src->len, PAGE_SIZE - offset);
4211	void *addr = page_address(src->pages[index]) + offset;
4212
4213	write_extent_buffer(eb: dst, srcv: addr, start: cur, len: cur_len);
4214
4215	cur += cur_len;
4216	}
4217	}
4218
4219	void copy_extent_buffer(const struct extent_buffer *dst,
4220	const struct extent_buffer *src,
4221	unsigned long dst_offset, unsigned long src_offset,
4222	unsigned long len)
4223	{
4224	u64 dst_len = dst->len;
4225	size_t cur;
4226	size_t offset;
4227	struct page *page;
4228	char *kaddr;
4229	unsigned long i = get_eb_page_index(offset: dst_offset);
4230
4231	if (check_eb_range(eb: dst, start: dst_offset, len) ||
4232	check_eb_range(eb: src, start: src_offset, len))
4233	return;
4234
4235	WARN_ON(src->len != dst_len);
4236
4237	offset = get_eb_offset_in_page(eb: dst, offset: dst_offset);
4238
4239	while (len > 0) {
4240	page = dst->pages[i];
4241	assert_eb_page_uptodate(eb: dst, page);
4242
4243	cur = min(len, (unsigned long)(PAGE_SIZE - offset));
4244
4245	kaddr = page_address(page);
4246	read_extent_buffer(eb: src, dstv: kaddr + offset, start: src_offset, len: cur);
4247
4248	src_offset += cur;
4249	len -= cur;
4250	offset = 0;
4251	i++;
4252	}
4253	}
4254
4255	/*
4256	* Calculate the page and offset of the byte containing the given bit number.
4257	*
4258	* @eb: the extent buffer
4259	* @start: offset of the bitmap item in the extent buffer
4260	* @nr: bit number
4261	* @page_index: return index of the page in the extent buffer that contains
4262	* the given bit number
4263	* @page_offset: return offset into the page given by page_index
4264	*
4265	* This helper hides the ugliness of finding the byte in an extent buffer which
4266	* contains a given bit.
4267	*/
4268	static inline void eb_bitmap_offset(const struct extent_buffer *eb,
4269	unsigned long start, unsigned long nr,
4270	unsigned long *page_index,
4271	size_t *page_offset)
4272	{
4273	size_t byte_offset = BIT_BYTE(nr);
4274	size_t offset;
4275
4276	/*
4277	* The byte we want is the offset of the extent buffer + the offset of
4278	* the bitmap item in the extent buffer + the offset of the byte in the
4279	* bitmap item.
4280	*/
4281	offset = start + offset_in_page(eb->start) + byte_offset;
4282
4283	*page_index = offset >> PAGE_SHIFT;
4284	*page_offset = offset_in_page(offset);
4285	}
4286
4287	/*
4288	* Determine whether a bit in a bitmap item is set.
4289	*
4290	* @eb: the extent buffer
4291	* @start: offset of the bitmap item in the extent buffer
4292	* @nr: bit number to test
4293	*/
4294	int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
4295	unsigned long nr)
4296	{
4297	u8 *kaddr;
4298	struct page *page;
4299	unsigned long i;
4300	size_t offset;
4301
4302	eb_bitmap_offset(eb, start, nr, page_index: &i, page_offset: &offset);
4303	page = eb->pages[i];
4304	assert_eb_page_uptodate(eb, page);
4305	kaddr = page_address(page);
4306	return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
4307	}
4308
4309	static u8 *extent_buffer_get_byte(const struct extent_buffer *eb, unsigned long bytenr)
4310	{
4311	unsigned long index = get_eb_page_index(offset: bytenr);
4312
4313	if (check_eb_range(eb, start: bytenr, len: 1))
4314	return NULL;
4315	return page_address(eb->pages[index]) + get_eb_offset_in_page(eb, offset: bytenr);
4316	}
4317
4318	/*
4319	* Set an area of a bitmap to 1.
4320	*
4321	* @eb: the extent buffer
4322	* @start: offset of the bitmap item in the extent buffer
4323	* @pos: bit number of the first bit
4324	* @len: number of bits to set
4325	*/
4326	void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
4327	unsigned long pos, unsigned long len)
4328	{
4329	unsigned int first_byte = start + BIT_BYTE(pos);
4330	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4331	const bool same_byte = (first_byte == last_byte);
4332	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4333	u8 *kaddr;
4334
4335	if (same_byte)
4336	mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4337
4338	/* Handle the first byte. */
4339	kaddr = extent_buffer_get_byte(eb, bytenr: first_byte);
4340	*kaddr |= mask;
4341	if (same_byte)
4342	return;
4343
4344	/* Handle the byte aligned part. */
4345	ASSERT(first_byte + 1 <= last_byte);
4346	memset_extent_buffer(eb, c: 0xff, start: first_byte + 1, len: last_byte - first_byte - 1);
4347
4348	/* Handle the last byte. */
4349	kaddr = extent_buffer_get_byte(eb, bytenr: last_byte);
4350	*kaddr |= BITMAP_LAST_BYTE_MASK(pos + len);
4351	}
4352
4353
4354	/*
4355	* Clear an area of a bitmap.
4356	*
4357	* @eb: the extent buffer
4358	* @start: offset of the bitmap item in the extent buffer
4359	* @pos: bit number of the first bit
4360	* @len: number of bits to clear
4361	*/
4362	void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
4363	unsigned long start, unsigned long pos,
4364	unsigned long len)
4365	{
4366	unsigned int first_byte = start + BIT_BYTE(pos);
4367	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4368	const bool same_byte = (first_byte == last_byte);
4369	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4370	u8 *kaddr;
4371
4372	if (same_byte)
4373	mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4374
4375	/* Handle the first byte. */
4376	kaddr = extent_buffer_get_byte(eb, bytenr: first_byte);
4377	*kaddr &= ~mask;
4378	if (same_byte)
4379	return;
4380
4381	/* Handle the byte aligned part. */
4382	ASSERT(first_byte + 1 <= last_byte);
4383	memset_extent_buffer(eb, c: 0, start: first_byte + 1, len: last_byte - first_byte - 1);
4384
4385	/* Handle the last byte. */
4386	kaddr = extent_buffer_get_byte(eb, bytenr: last_byte);
4387	*kaddr &= ~BITMAP_LAST_BYTE_MASK(pos + len);
4388	}
4389
4390	static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4391	{
4392	unsigned long distance = (src > dst) ? src - dst : dst - src;
4393	return distance < len;
4394	}
4395
4396	void memcpy_extent_buffer(const struct extent_buffer *dst,
4397	unsigned long dst_offset, unsigned long src_offset,
4398	unsigned long len)
4399	{
4400	unsigned long cur_off = 0;
4401
4402	if (check_eb_range(eb: dst, start: dst_offset, len) ||
4403	check_eb_range(eb: dst, start: src_offset, len))
4404	return;
4405
4406	while (cur_off < len) {
4407	unsigned long cur_src = cur_off + src_offset;
4408	unsigned long pg_index = get_eb_page_index(offset: cur_src);
4409	unsigned long pg_off = get_eb_offset_in_page(eb: dst, offset: cur_src);
4410	unsigned long cur_len = min(src_offset + len - cur_src,
4411	PAGE_SIZE - pg_off);
4412	void *src_addr = page_address(dst->pages[pg_index]) + pg_off;
4413	const bool use_memmove = areas_overlap(src: src_offset + cur_off,
4414	dst: dst_offset + cur_off, len: cur_len);
4415
4416	__write_extent_buffer(eb: dst, srcv: src_addr, start: dst_offset + cur_off, len: cur_len,
4417	use_memmove);
4418	cur_off += cur_len;
4419	}
4420	}
4421
4422	void memmove_extent_buffer(const struct extent_buffer *dst,
4423	unsigned long dst_offset, unsigned long src_offset,
4424	unsigned long len)
4425	{
4426	unsigned long dst_end = dst_offset + len - 1;
4427	unsigned long src_end = src_offset + len - 1;
4428
4429	if (check_eb_range(eb: dst, start: dst_offset, len) ||
4430	check_eb_range(eb: dst, start: src_offset, len))
4431	return;
4432
4433	if (dst_offset < src_offset) {
4434	memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4435	return;
4436	}
4437
4438	while (len > 0) {
4439	unsigned long src_i;
4440	size_t cur;
4441	size_t dst_off_in_page;
4442	size_t src_off_in_page;
4443	void *src_addr;
4444	bool use_memmove;
4445
4446	src_i = get_eb_page_index(offset: src_end);
4447
4448	dst_off_in_page = get_eb_offset_in_page(eb: dst, offset: dst_end);
4449	src_off_in_page = get_eb_offset_in_page(eb: dst, offset: src_end);
4450
4451	cur = min_t(unsigned long, len, src_off_in_page + 1);
4452	cur = min(cur, dst_off_in_page + 1);
4453
4454	src_addr = page_address(dst->pages[src_i]) + src_off_in_page -
4455	cur + 1;
4456	use_memmove = areas_overlap(src: src_end - cur + 1, dst: dst_end - cur + 1,
4457	len: cur);
4458
4459	__write_extent_buffer(eb: dst, srcv: src_addr, start: dst_end - cur + 1, len: cur,
4460	use_memmove);
4461
4462	dst_end -= cur;
4463	src_end -= cur;
4464	len -= cur;
4465	}
4466	}
4467
4468	#define GANG_LOOKUP_SIZE 16
4469	static struct extent_buffer *get_next_extent_buffer(
4470	struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr)
4471	{
4472	struct extent_buffer *gang[GANG_LOOKUP_SIZE];
4473	struct extent_buffer *found = NULL;
4474	u64 page_start = page_offset(page);
4475	u64 cur = page_start;
4476
4477	ASSERT(in_range(bytenr, page_start, PAGE_SIZE));
4478	lockdep_assert_held(&fs_info->buffer_lock);
4479
4480	while (cur < page_start + PAGE_SIZE) {
4481	int ret;
4482	int i;
4483
4484	ret = radix_tree_gang_lookup(&fs_info->buffer_radix,
4485	results: (void **)gang, first_index: cur >> fs_info->sectorsize_bits,
4486	min_t(unsigned int, GANG_LOOKUP_SIZE,
4487	PAGE_SIZE / fs_info->nodesize));
4488	if (ret == 0)
4489	goto out;
4490	for (i = 0; i < ret; i++) {
4491	/* Already beyond page end */
4492	if (gang[i]->start >= page_start + PAGE_SIZE)
4493	goto out;
4494	/* Found one */
4495	if (gang[i]->start >= bytenr) {
4496	found = gang[i];
4497	goto out;
4498	}
4499	}
4500	cur = gang[ret - 1]->start + gang[ret - 1]->len;
4501	}
4502	out:
4503	return found;
4504	}
4505
4506	static int try_release_subpage_extent_buffer(struct page *page)
4507	{
4508	struct btrfs_fs_info *fs_info = btrfs_sb(sb: page->mapping->host->i_sb);
4509	u64 cur = page_offset(page);
4510	const u64 end = page_offset(page) + PAGE_SIZE;
4511	int ret;
4512
4513	while (cur < end) {
4514	struct extent_buffer *eb = NULL;
4515
4516	/*
4517	* Unlike try_release_extent_buffer() which uses page->private
4518	* to grab buffer, for subpage case we rely on radix tree, thus
4519	* we need to ensure radix tree consistency.
4520	*
4521	* We also want an atomic snapshot of the radix tree, thus go
4522	* with spinlock rather than RCU.
4523	*/
4524	spin_lock(lock: &fs_info->buffer_lock);
4525	eb = get_next_extent_buffer(fs_info, page, bytenr: cur);
4526	if (!eb) {
4527	/* No more eb in the page range after or at cur */
4528	spin_unlock(lock: &fs_info->buffer_lock);
4529	break;
4530	}
4531	cur = eb->start + eb->len;
4532
4533	/*
4534	* The same as try_release_extent_buffer(), to ensure the eb
4535	* won't disappear out from under us.
4536	*/
4537	spin_lock(lock: &eb->refs_lock);
4538	if (atomic_read(v: &eb->refs) != 1 || extent_buffer_under_io(eb)) {
4539	spin_unlock(lock: &eb->refs_lock);
4540	spin_unlock(lock: &fs_info->buffer_lock);
4541	break;
4542	}
4543	spin_unlock(lock: &fs_info->buffer_lock);
4544
4545	/*
4546	* If tree ref isn't set then we know the ref on this eb is a
4547	* real ref, so just return, this eb will likely be freed soon
4548	* anyway.
4549	*/
4550	if (!test_and_clear_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags)) {
4551	spin_unlock(lock: &eb->refs_lock);
4552	break;
4553	}
4554
4555	/*
4556	* Here we don't care about the return value, we will always
4557	* check the page private at the end. And
4558	* release_extent_buffer() will release the refs_lock.
4559	*/
4560	release_extent_buffer(eb);
4561	}
4562	/*
4563	* Finally to check if we have cleared page private, as if we have
4564	* released all ebs in the page, the page private should be cleared now.
4565	*/
4566	spin_lock(lock: &page->mapping->private_lock);
4567	if (!PagePrivate(page))
4568	ret = 1;
4569	else
4570	ret = 0;
4571	spin_unlock(lock: &page->mapping->private_lock);
4572	return ret;
4573
4574	}
4575
4576	int try_release_extent_buffer(struct page *page)
4577	{
4578	struct extent_buffer *eb;
4579
4580	if (btrfs_sb(sb: page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
4581	return try_release_subpage_extent_buffer(page);
4582
4583	/*
4584	* We need to make sure nobody is changing page->private, as we rely on
4585	* page->private as the pointer to extent buffer.
4586	*/
4587	spin_lock(lock: &page->mapping->private_lock);
4588	if (!PagePrivate(page)) {
4589	spin_unlock(lock: &page->mapping->private_lock);
4590	return 1;
4591	}
4592
4593	eb = (struct extent_buffer *)page->private;
4594	BUG_ON(!eb);
4595
4596	/*
4597	* This is a little awful but should be ok, we need to make sure that
4598	* the eb doesn't disappear out from under us while we're looking at
4599	* this page.
4600	*/
4601	spin_lock(lock: &eb->refs_lock);
4602	if (atomic_read(v: &eb->refs) != 1 || extent_buffer_under_io(eb)) {
4603	spin_unlock(lock: &eb->refs_lock);
4604	spin_unlock(lock: &page->mapping->private_lock);
4605	return 0;
4606	}
4607	spin_unlock(lock: &page->mapping->private_lock);
4608
4609	/*
4610	* If tree ref isn't set then we know the ref on this eb is a real ref,
4611	* so just return, this page will likely be freed soon anyway.
4612	*/
4613	if (!test_and_clear_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags)) {
4614	spin_unlock(lock: &eb->refs_lock);
4615	return 0;
4616	}
4617
4618	return release_extent_buffer(eb);
4619	}
4620
4621	/*
4622	* Attempt to readahead a child block.
4623	*
4624	* @fs_info: the fs_info
4625	* @bytenr: bytenr to read
4626	* @owner_root: objectid of the root that owns this eb
4627	* @gen: generation for the uptodate check, can be 0
4628	* @level: level for the eb
4629	*
4630	* Attempt to readahead a tree block at @bytenr. If @gen is 0 then we do a
4631	* normal uptodate check of the eb, without checking the generation. If we have
4632	* to read the block we will not block on anything.
4633	*/
4634	void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
4635	u64 bytenr, u64 owner_root, u64 gen, int level)
4636	{
4637	struct btrfs_tree_parent_check check = {
4638	.has_first_key = 0,
4639	.level = level,
4640	.transid = gen
4641	};
4642	struct extent_buffer *eb;
4643	int ret;
4644
4645	eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
4646	if (IS_ERR(ptr: eb))
4647	return;
4648
4649	if (btrfs_buffer_uptodate(buf: eb, parent_transid: gen, atomic: 1)) {
4650	free_extent_buffer(eb);
4651	return;
4652	}
4653
4654	ret = read_extent_buffer_pages(eb, WAIT_NONE, mirror_num: 0, check: &check);
4655	if (ret < 0)
4656	free_extent_buffer_stale(eb);
4657	else
4658	free_extent_buffer(eb);
4659	}
4660
4661	/*
4662	* Readahead a node's child block.
4663	*
4664	* @node: parent node we're reading from
4665	* @slot: slot in the parent node for the child we want to read
4666	*
4667	* A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at
4668	* the slot in the node provided.
4669	*/
4670	void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
4671	{
4672	btrfs_readahead_tree_block(fs_info: node->fs_info,
4673	bytenr: btrfs_node_blockptr(eb: node, nr: slot),
4674	owner_root: btrfs_header_owner(eb: node),
4675	gen: btrfs_node_ptr_generation(eb: node, nr: slot),
4676	level: btrfs_header_level(eb: node) - 1);
4677	}
4678