1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* fs/f2fs/data.c
4	*
5	* Copyright (c) 2012 Samsung Electronics Co., Ltd.
6	* http://www.samsung.com/
7	*/
8	#include <linux/fs.h>
9	#include <linux/f2fs_fs.h>
10	#include <linux/buffer_head.h>
11	#include <linux/sched/mm.h>
12	#include <linux/mpage.h>
13	#include <linux/writeback.h>
14	#include <linux/pagevec.h>
15	#include <linux/blkdev.h>
16	#include <linux/bio.h>
17	#include <linux/blk-crypto.h>
18	#include <linux/swap.h>
19	#include <linux/prefetch.h>
20	#include <linux/uio.h>
21	#include <linux/sched/signal.h>
22	#include <linux/fiemap.h>
23	#include <linux/iomap.h>
24
25	#include "f2fs.h"
26	#include "node.h"
27	#include "segment.h"
28	#include "iostat.h"
29	#include <trace/events/f2fs.h>
30
31	#define NUM_PREALLOC_POST_READ_CTXS 128
32
33	static struct kmem_cache *bio_post_read_ctx_cache;
34	static struct kmem_cache *bio_entry_slab;
35	static mempool_t *bio_post_read_ctx_pool;
36	static struct bio_set f2fs_bioset;
37
38	#define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE
39
40	int __init f2fs_init_bioset(void)
41	{
42	return bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE,
43	0, flags: BIOSET_NEED_BVECS);
44	}
45
46	void f2fs_destroy_bioset(void)
47	{
48	bioset_exit(&f2fs_bioset);
49	}
50
51	bool f2fs_is_cp_guaranteed(struct page *page)
52	{
53	struct address_space *mapping = page->mapping;
54	struct inode *inode;
55	struct f2fs_sb_info *sbi;
56
57	if (!mapping)
58	return false;
59
60	inode = mapping->host;
61	sbi = F2FS_I_SB(inode);
62
63	if (inode->i_ino == F2FS_META_INO(sbi) ||
64	inode->i_ino == F2FS_NODE_INO(sbi) ||
65	S_ISDIR(inode->i_mode))
66	return true;
67
68	if ((S_ISREG(inode->i_mode) && IS_NOQUOTA(inode)) ||
69	page_private_gcing(page))
70	return true;
71	return false;
72	}
73
74	static enum count_type __read_io_type(struct page *page)
75	{
76	struct address_space *mapping = page_file_mapping(page);
77
78	if (mapping) {
79	struct inode *inode = mapping->host;
80	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
81
82	if (inode->i_ino == F2FS_META_INO(sbi))
83	return F2FS_RD_META;
84
85	if (inode->i_ino == F2FS_NODE_INO(sbi))
86	return F2FS_RD_NODE;
87	}
88	return F2FS_RD_DATA;
89	}
90
91	/* postprocessing steps for read bios */
92	enum bio_post_read_step {
93	#ifdef CONFIG_FS_ENCRYPTION
94	STEP_DECRYPT = BIT(0),
95	#else
96	STEP_DECRYPT = 0, /* compile out the decryption-related code */
97	#endif
98	#ifdef CONFIG_F2FS_FS_COMPRESSION
99	STEP_DECOMPRESS = BIT(1),
100	#else
101	STEP_DECOMPRESS = 0, /* compile out the decompression-related code */
102	#endif
103	#ifdef CONFIG_FS_VERITY
104	STEP_VERITY = BIT(2),
105	#else
106	STEP_VERITY = 0, /* compile out the verity-related code */
107	#endif
108	};
109
110	struct bio_post_read_ctx {
111	struct bio *bio;
112	struct f2fs_sb_info *sbi;
113	struct work_struct work;
114	unsigned int enabled_steps;
115	/*
116	* decompression_attempted keeps track of whether
117	* f2fs_end_read_compressed_page() has been called on the pages in the
118	* bio that belong to a compressed cluster yet.
119	*/
120	bool decompression_attempted;
121	block_t fs_blkaddr;
122	};
123
124	/*
125	* Update and unlock a bio's pages, and free the bio.
126	*
127	* This marks pages up-to-date only if there was no error in the bio (I/O error,
128	* decryption error, or verity error), as indicated by bio->bi_status.
129	*
130	* "Compressed pages" (pagecache pages backed by a compressed cluster on-disk)
131	* aren't marked up-to-date here, as decompression is done on a per-compression-
132	* cluster basis rather than a per-bio basis. Instead, we only must do two
133	* things for each compressed page here: call f2fs_end_read_compressed_page()
134	* with failed=true if an error occurred before it would have normally gotten
135	* called (i.e., I/O error or decryption error, but *not* verity error), and
136	* release the bio's reference to the decompress_io_ctx of the page's cluster.
137	*/
138	static void f2fs_finish_read_bio(struct bio *bio, bool in_task)
139	{
140	struct bio_vec *bv;
141	struct bvec_iter_all iter_all;
142	struct bio_post_read_ctx *ctx = bio->bi_private;
143
144	bio_for_each_segment_all(bv, bio, iter_all) {
145	struct page *page = bv->bv_page;
146
147	if (f2fs_is_compressed_page(page)) {
148	if (ctx && !ctx->decompression_attempted)
149	f2fs_end_read_compressed_page(page, failed: true, blkaddr: 0,
150	in_task);
151	f2fs_put_page_dic(page, in_task);
152	continue;
153	}
154
155	if (bio->bi_status)
156	ClearPageUptodate(page);
157	else
158	SetPageUptodate(page);
159	dec_page_count(sbi: F2FS_P_SB(page), count_type: __read_io_type(page));
160	unlock_page(page);
161	}
162
163	if (ctx)
164	mempool_free(element: ctx, pool: bio_post_read_ctx_pool);
165	bio_put(bio);
166	}
167
168	static void f2fs_verify_bio(struct work_struct *work)
169	{
170	struct bio_post_read_ctx *ctx =
171	container_of(work, struct bio_post_read_ctx, work);
172	struct bio *bio = ctx->bio;
173	bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS);
174
175	/*
176	* fsverity_verify_bio() may call readahead() again, and while verity
177	* will be disabled for this, decryption and/or decompression may still
178	* be needed, resulting in another bio_post_read_ctx being allocated.
179	* So to prevent deadlocks we need to release the current ctx to the
180	* mempool first. This assumes that verity is the last post-read step.
181	*/
182	mempool_free(element: ctx, pool: bio_post_read_ctx_pool);
183	bio->bi_private = NULL;
184
185	/*
186	* Verify the bio's pages with fs-verity. Exclude compressed pages,
187	* as those were handled separately by f2fs_end_read_compressed_page().
188	*/
189	if (may_have_compressed_pages) {
190	struct bio_vec *bv;
191	struct bvec_iter_all iter_all;
192
193	bio_for_each_segment_all(bv, bio, iter_all) {
194	struct page *page = bv->bv_page;
195
196	if (!f2fs_is_compressed_page(page) &&
197	!fsverity_verify_page(page)) {
198	bio->bi_status = BLK_STS_IOERR;
199	break;
200	}
201	}
202	} else {
203	fsverity_verify_bio(bio);
204	}
205
206	f2fs_finish_read_bio(bio, in_task: true);
207	}
208
209	/*
210	* If the bio's data needs to be verified with fs-verity, then enqueue the
211	* verity work for the bio. Otherwise finish the bio now.
212	*
213	* Note that to avoid deadlocks, the verity work can't be done on the
214	* decryption/decompression workqueue. This is because verifying the data pages
215	* can involve reading verity metadata pages from the file, and these verity
216	* metadata pages may be encrypted and/or compressed.
217	*/
218	static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task)
219	{
220	struct bio_post_read_ctx *ctx = bio->bi_private;
221
222	if (ctx && (ctx->enabled_steps & STEP_VERITY)) {
223	INIT_WORK(&ctx->work, f2fs_verify_bio);
224	fsverity_enqueue_verify_work(work: &ctx->work);
225	} else {
226	f2fs_finish_read_bio(bio, in_task);
227	}
228	}
229
230	/*
231	* Handle STEP_DECOMPRESS by decompressing any compressed clusters whose last
232	* remaining page was read by @ctx->bio.
233	*
234	* Note that a bio may span clusters (even a mix of compressed and uncompressed
235	* clusters) or be for just part of a cluster. STEP_DECOMPRESS just indicates
236	* that the bio includes at least one compressed page. The actual decompression
237	* is done on a per-cluster basis, not a per-bio basis.
238	*/
239	static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx,
240	bool in_task)
241	{
242	struct bio_vec *bv;
243	struct bvec_iter_all iter_all;
244	bool all_compressed = true;
245	block_t blkaddr = ctx->fs_blkaddr;
246
247	bio_for_each_segment_all(bv, ctx->bio, iter_all) {
248	struct page *page = bv->bv_page;
249
250	if (f2fs_is_compressed_page(page))
251	f2fs_end_read_compressed_page(page, failed: false, blkaddr,
252	in_task);
253	else
254	all_compressed = false;
255
256	blkaddr++;
257	}
258
259	ctx->decompression_attempted = true;
260
261	/*
262	* Optimization: if all the bio's pages are compressed, then scheduling
263	* the per-bio verity work is unnecessary, as verity will be fully
264	* handled at the compression cluster level.
265	*/
266	if (all_compressed)
267	ctx->enabled_steps &= ~STEP_VERITY;
268	}
269
270	static void f2fs_post_read_work(struct work_struct *work)
271	{
272	struct bio_post_read_ctx *ctx =
273	container_of(work, struct bio_post_read_ctx, work);
274	struct bio *bio = ctx->bio;
275
276	if ((ctx->enabled_steps & STEP_DECRYPT) && !fscrypt_decrypt_bio(bio)) {
277	f2fs_finish_read_bio(bio, in_task: true);
278	return;
279	}
280
281	if (ctx->enabled_steps & STEP_DECOMPRESS)
282	f2fs_handle_step_decompress(ctx, in_task: true);
283
284	f2fs_verify_and_finish_bio(bio, in_task: true);
285	}
286
287	static void f2fs_read_end_io(struct bio *bio)
288	{
289	struct f2fs_sb_info *sbi = F2FS_P_SB(page: bio_first_page_all(bio));
290	struct bio_post_read_ctx *ctx;
291	bool intask = in_task();
292
293	iostat_update_and_unbind_ctx(bio);
294	ctx = bio->bi_private;
295
296	if (time_to_inject(sbi, FAULT_READ_IO))
297	bio->bi_status = BLK_STS_IOERR;
298
299	if (bio->bi_status) {
300	f2fs_finish_read_bio(bio, in_task: intask);
301	return;
302	}
303
304	if (ctx) {
305	unsigned int enabled_steps = ctx->enabled_steps &
306	(STEP_DECRYPT | STEP_DECOMPRESS);
307
308	/*
309	* If we have only decompression step between decompression and
310	* decrypt, we don't need post processing for this.
311	*/
312	if (enabled_steps == STEP_DECOMPRESS &&
313	!f2fs_low_mem_mode(sbi)) {
314	f2fs_handle_step_decompress(ctx, in_task: intask);
315	} else if (enabled_steps) {
316	INIT_WORK(&ctx->work, f2fs_post_read_work);
317	queue_work(wq: ctx->sbi->post_read_wq, work: &ctx->work);
318	return;
319	}
320	}
321
322	f2fs_verify_and_finish_bio(bio, in_task: intask);
323	}
324
325	static void f2fs_write_end_io(struct bio *bio)
326	{
327	struct f2fs_sb_info *sbi;
328	struct bio_vec *bvec;
329	struct bvec_iter_all iter_all;
330
331	iostat_update_and_unbind_ctx(bio);
332	sbi = bio->bi_private;
333
334	if (time_to_inject(sbi, FAULT_WRITE_IO))
335	bio->bi_status = BLK_STS_IOERR;
336
337	bio_for_each_segment_all(bvec, bio, iter_all) {
338	struct page *page = bvec->bv_page;
339	enum count_type type = WB_DATA_TYPE(page, false);
340
341	fscrypt_finalize_bounce_page(pagep: &page);
342
343	#ifdef CONFIG_F2FS_FS_COMPRESSION
344	if (f2fs_is_compressed_page(page)) {
345	f2fs_compress_write_end_io(bio, page);
346	continue;
347	}
348	#endif
349
350	if (unlikely(bio->bi_status)) {
351	mapping_set_error(mapping: page->mapping, error: -EIO);
352	if (type == F2FS_WB_CP_DATA)
353	f2fs_stop_checkpoint(sbi, end_io: true,
354	reason: STOP_CP_REASON_WRITE_FAIL);
355	}
356
357	f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
358	page->index != nid_of_node(page));
359
360	dec_page_count(sbi, count_type: type);
361	if (f2fs_in_warm_node_list(sbi, page))
362	f2fs_del_fsync_node_entry(sbi, page);
363	clear_page_private_gcing(page);
364	end_page_writeback(page);
365	}
366	if (!get_pages(sbi, count_type: F2FS_WB_CP_DATA) &&
367	wq_has_sleeper(wq_head: &sbi->cp_wait))
368	wake_up(&sbi->cp_wait);
369
370	bio_put(bio);
371	}
372
373	#ifdef CONFIG_BLK_DEV_ZONED
374	static void f2fs_zone_write_end_io(struct bio *bio)
375	{
376	struct f2fs_bio_info *io = (struct f2fs_bio_info *)bio->bi_private;
377
378	bio->bi_private = io->bi_private;
379	complete(&io->zone_wait);
380	f2fs_write_end_io(bio);
381	}
382	#endif
383
384	struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
385	block_t blk_addr, sector_t *sector)
386	{
387	struct block_device *bdev = sbi->sb->s_bdev;
388	int i;
389
390	if (f2fs_is_multi_device(sbi)) {
391	for (i = 0; i < sbi->s_ndevs; i++) {
392	if (FDEV(i).start_blk <= blk_addr &&
393	FDEV(i).end_blk >= blk_addr) {
394	blk_addr -= FDEV(i).start_blk;
395	bdev = FDEV(i).bdev;
396	break;
397	}
398	}
399	}
400
401	if (sector)
402	*sector = SECTOR_FROM_BLOCK(blk_addr);
403	return bdev;
404	}
405
406	int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
407	{
408	int i;
409
410	if (!f2fs_is_multi_device(sbi))
411	return 0;
412
413	for (i = 0; i < sbi->s_ndevs; i++)
414	if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
415	return i;
416	return 0;
417	}
418
419	static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio)
420	{
421	unsigned int temp_mask = GENMASK(NR_TEMP_TYPE - 1, 0);
422	unsigned int fua_flag, meta_flag, io_flag;
423	blk_opf_t op_flags = 0;
424
425	if (fio->op != REQ_OP_WRITE)
426	return 0;
427	if (fio->type == DATA)
428	io_flag = fio->sbi->data_io_flag;
429	else if (fio->type == NODE)
430	io_flag = fio->sbi->node_io_flag;
431	else
432	return 0;
433
434	fua_flag = io_flag & temp_mask;
435	meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask;
436
437	/*
438	* data/node io flag bits per temp:
439	* REQ_META | REQ_FUA |
440	* 5 | 4 | 3 | 2 | 1 | 0 |
441	* Cold | Warm | Hot | Cold | Warm | Hot |
442	*/
443	if (BIT(fio->temp) & meta_flag)
444	op_flags |= REQ_META;
445	if (BIT(fio->temp) & fua_flag)
446	op_flags |= REQ_FUA;
447	return op_flags;
448	}
449
450	static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)
451	{
452	struct f2fs_sb_info *sbi = fio->sbi;
453	struct block_device *bdev;
454	sector_t sector;
455	struct bio *bio;
456
457	bdev = f2fs_target_device(sbi, blk_addr: fio->new_blkaddr, sector: &sector);
458	bio = bio_alloc_bioset(bdev, nr_vecs: npages,
459	opf: fio->op | fio->op_flags | f2fs_io_flags(fio),
460	GFP_NOIO, bs: &f2fs_bioset);
461	bio->bi_iter.bi_sector = sector;
462	if (is_read_io(fio->op)) {
463	bio->bi_end_io = f2fs_read_end_io;
464	bio->bi_private = NULL;
465	} else {
466	bio->bi_end_io = f2fs_write_end_io;
467	bio->bi_private = sbi;
468	}
469	iostat_alloc_and_bind_ctx(sbi, bio, NULL);
470
471	if (fio->io_wbc)
472	wbc_init_bio(wbc: fio->io_wbc, bio);
473
474	return bio;
475	}
476
477	static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
478	pgoff_t first_idx,
479	const struct f2fs_io_info *fio,
480	gfp_t gfp_mask)
481	{
482	/*
483	* The f2fs garbage collector sets ->encrypted_page when it wants to
484	* read/write raw data without encryption.
485	*/
486	if (!fio || !fio->encrypted_page)
487	fscrypt_set_bio_crypt_ctx(bio, inode, first_lblk: first_idx, gfp_mask);
488	}
489
490	static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode,
491	pgoff_t next_idx,
492	const struct f2fs_io_info *fio)
493	{
494	/*
495	* The f2fs garbage collector sets ->encrypted_page when it wants to
496	* read/write raw data without encryption.
497	*/
498	if (fio && fio->encrypted_page)
499	return !bio_has_crypt_ctx(bio);
500
501	return fscrypt_mergeable_bio(bio, inode, next_lblk: next_idx);
502	}
503
504	void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio,
505	enum page_type type)
506	{
507	WARN_ON_ONCE(!is_read_io(bio_op(bio)));
508	trace_f2fs_submit_read_bio(sb: sbi->sb, type, bio);
509
510	iostat_update_submit_ctx(bio, type);
511	submit_bio(bio);
512	}
513
514	static void f2fs_submit_write_bio(struct f2fs_sb_info *sbi, struct bio *bio,
515	enum page_type type)
516	{
517	WARN_ON_ONCE(is_read_io(bio_op(bio)));
518
519	if (f2fs_lfs_mode(sbi) && current->plug && PAGE_TYPE_ON_MAIN(type))
520	blk_finish_plug(current->plug);
521
522	trace_f2fs_submit_write_bio(sb: sbi->sb, type, bio);
523	iostat_update_submit_ctx(bio, type);
524	submit_bio(bio);
525	}
526
527	static void __submit_merged_bio(struct f2fs_bio_info *io)
528	{
529	struct f2fs_io_info *fio = &io->fio;
530
531	if (!io->bio)
532	return;
533
534	if (is_read_io(fio->op)) {
535	trace_f2fs_prepare_read_bio(sb: io->sbi->sb, type: fio->type, bio: io->bio);
536	f2fs_submit_read_bio(sbi: io->sbi, bio: io->bio, type: fio->type);
537	} else {
538	trace_f2fs_prepare_write_bio(sb: io->sbi->sb, type: fio->type, bio: io->bio);
539	f2fs_submit_write_bio(sbi: io->sbi, bio: io->bio, type: fio->type);
540	}
541	io->bio = NULL;
542	}
543
544	static bool __has_merged_page(struct bio *bio, struct inode *inode,
545	struct page *page, nid_t ino)
546	{
547	struct bio_vec *bvec;
548	struct bvec_iter_all iter_all;
549
550	if (!bio)
551	return false;
552
553	if (!inode && !page && !ino)
554	return true;
555
556	bio_for_each_segment_all(bvec, bio, iter_all) {
557	struct page *target = bvec->bv_page;
558
559	if (fscrypt_is_bounce_page(page: target)) {
560	target = fscrypt_pagecache_page(bounce_page: target);
561	if (IS_ERR(ptr: target))
562	continue;
563	}
564	if (f2fs_is_compressed_page(page: target)) {
565	target = f2fs_compress_control_page(page: target);
566	if (IS_ERR(ptr: target))
567	continue;
568	}
569
570	if (inode && inode == target->mapping->host)
571	return true;
572	if (page && page == target)
573	return true;
574	if (ino && ino == ino_of_node(node_page: target))
575	return true;
576	}
577
578	return false;
579	}
580
581	int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi)
582	{
583	int i;
584
585	for (i = 0; i < NR_PAGE_TYPE; i++) {
586	int n = (i == META) ? 1 : NR_TEMP_TYPE;
587	int j;
588
589	sbi->write_io[i] = f2fs_kmalloc(sbi,
590	array_size(n, sizeof(struct f2fs_bio_info)),
591	GFP_KERNEL);
592	if (!sbi->write_io[i])
593	return -ENOMEM;
594
595	for (j = HOT; j < n; j++) {
596	init_f2fs_rwsem(&sbi->write_io[i][j].io_rwsem);
597	sbi->write_io[i][j].sbi = sbi;
598	sbi->write_io[i][j].bio = NULL;
599	spin_lock_init(&sbi->write_io[i][j].io_lock);
600	INIT_LIST_HEAD(list: &sbi->write_io[i][j].io_list);
601	INIT_LIST_HEAD(list: &sbi->write_io[i][j].bio_list);
602	init_f2fs_rwsem(&sbi->write_io[i][j].bio_list_lock);
603	#ifdef CONFIG_BLK_DEV_ZONED
604	init_completion(x: &sbi->write_io[i][j].zone_wait);
605	sbi->write_io[i][j].zone_pending_bio = NULL;
606	sbi->write_io[i][j].bi_private = NULL;
607	#endif
608	}
609	}
610
611	return 0;
612	}
613
614	static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
615	enum page_type type, enum temp_type temp)
616	{
617	enum page_type btype = PAGE_TYPE_OF_BIO(type);
618	struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
619
620	f2fs_down_write(sem: &io->io_rwsem);
621
622	if (!io->bio)
623	goto unlock_out;
624
625	/* change META to META_FLUSH in the checkpoint procedure */
626	if (type >= META_FLUSH) {
627	io->fio.type = META_FLUSH;
628	io->bio->bi_opf |= REQ_META | REQ_PRIO | REQ_SYNC;
629	if (!test_opt(sbi, NOBARRIER))
630	io->bio->bi_opf |= REQ_PREFLUSH | REQ_FUA;
631	}
632	__submit_merged_bio(io);
633	unlock_out:
634	f2fs_up_write(sem: &io->io_rwsem);
635	}
636
637	static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
638	struct inode *inode, struct page *page,
639	nid_t ino, enum page_type type, bool force)
640	{
641	enum temp_type temp;
642	bool ret = true;
643
644	for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
645	if (!force) {
646	enum page_type btype = PAGE_TYPE_OF_BIO(type);
647	struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
648
649	f2fs_down_read(sem: &io->io_rwsem);
650	ret = __has_merged_page(bio: io->bio, inode, page, ino);
651	f2fs_up_read(sem: &io->io_rwsem);
652	}
653	if (ret)
654	__f2fs_submit_merged_write(sbi, type, temp);
655
656	/* TODO: use HOT temp only for meta pages now. */
657	if (type >= META)
658	break;
659	}
660	}
661
662	void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
663	{
664	__submit_merged_write_cond(sbi, NULL, NULL, ino: 0, type, force: true);
665	}
666
667	void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
668	struct inode *inode, struct page *page,
669	nid_t ino, enum page_type type)
670	{
671	__submit_merged_write_cond(sbi, inode, page, ino, type, force: false);
672	}
673
674	void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
675	{
676	f2fs_submit_merged_write(sbi, type: DATA);
677	f2fs_submit_merged_write(sbi, type: NODE);
678	f2fs_submit_merged_write(sbi, type: META);
679	}
680
681	/*
682	* Fill the locked page with data located in the block address.
683	* A caller needs to unlock the page on failure.
684	*/
685	int f2fs_submit_page_bio(struct f2fs_io_info *fio)
686	{
687	struct bio *bio;
688	struct page *page = fio->encrypted_page ?
689	fio->encrypted_page : fio->page;
690
691	if (!f2fs_is_valid_blkaddr(sbi: fio->sbi, blkaddr: fio->new_blkaddr,
692	type: fio->is_por ? META_POR : (__is_meta_io(fio) ?
693	META_GENERIC : DATA_GENERIC_ENHANCE)))
694	return -EFSCORRUPTED;
695
696	trace_f2fs_submit_page_bio(page, fio);
697
698	/* Allocate a new bio */
699	bio = __bio_alloc(fio, npages: 1);
700
701	f2fs_set_bio_crypt_ctx(bio, inode: fio->page->mapping->host,
702	first_idx: fio->page->index, fio, GFP_NOIO);
703
704	if (bio_add_page(bio, page, PAGE_SIZE, off: 0) < PAGE_SIZE) {
705	bio_put(bio);
706	return -EFAULT;
707	}
708
709	if (fio->io_wbc && !is_read_io(fio->op))
710	wbc_account_cgroup_owner(wbc: fio->io_wbc, page: fio->page, PAGE_SIZE);
711
712	inc_page_count(sbi: fio->sbi, is_read_io(fio->op) ?
713	__read_io_type(page) : WB_DATA_TYPE(fio->page, false));
714
715	if (is_read_io(bio_op(bio)))
716	f2fs_submit_read_bio(sbi: fio->sbi, bio, type: fio->type);
717	else
718	f2fs_submit_write_bio(sbi: fio->sbi, bio, type: fio->type);
719	return 0;
720	}
721
722	static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
723	block_t last_blkaddr, block_t cur_blkaddr)
724	{
725	if (unlikely(sbi->max_io_bytes &&
726	bio->bi_iter.bi_size >= sbi->max_io_bytes))
727	return false;
728	if (last_blkaddr + 1 != cur_blkaddr)
729	return false;
730	return bio->bi_bdev == f2fs_target_device(sbi, blk_addr: cur_blkaddr, NULL);
731	}
732
733	static bool io_type_is_mergeable(struct f2fs_bio_info *io,
734	struct f2fs_io_info *fio)
735	{
736	if (io->fio.op != fio->op)
737	return false;
738	return io->fio.op_flags == fio->op_flags;
739	}
740
741	static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
742	struct f2fs_bio_info *io,
743	struct f2fs_io_info *fio,
744	block_t last_blkaddr,
745	block_t cur_blkaddr)
746	{
747	if (!page_is_mergeable(sbi, bio, last_blkaddr, cur_blkaddr))
748	return false;
749	return io_type_is_mergeable(io, fio);
750	}
751
752	static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio,
753	struct page *page, enum temp_type temp)
754	{
755	struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
756	struct bio_entry *be;
757
758	be = f2fs_kmem_cache_alloc(cachep: bio_entry_slab, GFP_NOFS, nofail: true, NULL);
759	be->bio = bio;
760	bio_get(bio);
761
762	if (bio_add_page(bio, page, PAGE_SIZE, off: 0) != PAGE_SIZE)
763	f2fs_bug_on(sbi, 1);
764
765	f2fs_down_write(sem: &io->bio_list_lock);
766	list_add_tail(new: &be->list, head: &io->bio_list);
767	f2fs_up_write(sem: &io->bio_list_lock);
768	}
769
770	static void del_bio_entry(struct bio_entry *be)
771	{
772	list_del(entry: &be->list);
773	kmem_cache_free(s: bio_entry_slab, objp: be);
774	}
775
776	static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio,
777	struct page *page)
778	{
779	struct f2fs_sb_info *sbi = fio->sbi;
780	enum temp_type temp;
781	bool found = false;
782	int ret = -EAGAIN;
783
784	for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
785	struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
786	struct list_head *head = &io->bio_list;
787	struct bio_entry *be;
788
789	f2fs_down_write(sem: &io->bio_list_lock);
790	list_for_each_entry(be, head, list) {
791	if (be->bio != *bio)
792	continue;
793
794	found = true;
795
796	f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio,
797	*fio->last_block,
798	fio->new_blkaddr));
799	if (f2fs_crypt_mergeable_bio(bio: *bio,
800	inode: fio->page->mapping->host,
801	next_idx: fio->page->index, fio) &&
802	bio_add_page(bio: *bio, page, PAGE_SIZE, off: 0) ==
803	PAGE_SIZE) {
804	ret = 0;
805	break;
806	}
807
808	/* page can't be merged into bio; submit the bio */
809	del_bio_entry(be);
810	f2fs_submit_write_bio(sbi, bio: *bio, type: DATA);
811	break;
812	}
813	f2fs_up_write(sem: &io->bio_list_lock);
814	}
815
816	if (ret) {
817	bio_put(*bio);
818	*bio = NULL;
819	}
820
821	return ret;
822	}
823
824	void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
825	struct bio **bio, struct page *page)
826	{
827	enum temp_type temp;
828	bool found = false;
829	struct bio *target = bio ? *bio : NULL;
830
831	f2fs_bug_on(sbi, !target && !page);
832
833	for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
834	struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
835	struct list_head *head = &io->bio_list;
836	struct bio_entry *be;
837
838	if (list_empty(head))
839	continue;
840
841	f2fs_down_read(sem: &io->bio_list_lock);
842	list_for_each_entry(be, head, list) {
843	if (target)
844	found = (target == be->bio);
845	else
846	found = __has_merged_page(bio: be->bio, NULL,
847	page, ino: 0);
848	if (found)
849	break;
850	}
851	f2fs_up_read(sem: &io->bio_list_lock);
852
853	if (!found)
854	continue;
855
856	found = false;
857
858	f2fs_down_write(sem: &io->bio_list_lock);
859	list_for_each_entry(be, head, list) {
860	if (target)
861	found = (target == be->bio);
862	else
863	found = __has_merged_page(bio: be->bio, NULL,
864	page, ino: 0);
865	if (found) {
866	target = be->bio;
867	del_bio_entry(be);
868	break;
869	}
870	}
871	f2fs_up_write(sem: &io->bio_list_lock);
872	}
873
874	if (found)
875	f2fs_submit_write_bio(sbi, bio: target, type: DATA);
876	if (bio && *bio) {
877	bio_put(*bio);
878	*bio = NULL;
879	}
880	}
881
882	int f2fs_merge_page_bio(struct f2fs_io_info *fio)
883	{
884	struct bio *bio = *fio->bio;
885	struct page *page = fio->encrypted_page ?
886	fio->encrypted_page : fio->page;
887
888	if (!f2fs_is_valid_blkaddr(sbi: fio->sbi, blkaddr: fio->new_blkaddr,
889	__is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))
890	return -EFSCORRUPTED;
891
892	trace_f2fs_submit_page_bio(page, fio);
893
894	if (bio && !page_is_mergeable(sbi: fio->sbi, bio, last_blkaddr: *fio->last_block,
895	cur_blkaddr: fio->new_blkaddr))
896	f2fs_submit_merged_ipu_write(sbi: fio->sbi, bio: &bio, NULL);
897	alloc_new:
898	if (!bio) {
899	bio = __bio_alloc(fio, BIO_MAX_VECS);
900	f2fs_set_bio_crypt_ctx(bio, inode: fio->page->mapping->host,
901	first_idx: fio->page->index, fio, GFP_NOIO);
902
903	add_bio_entry(sbi: fio->sbi, bio, page, temp: fio->temp);
904	} else {
905	if (add_ipu_page(fio, bio: &bio, page))
906	goto alloc_new;
907	}
908
909	if (fio->io_wbc)
910	wbc_account_cgroup_owner(wbc: fio->io_wbc, page: fio->page, PAGE_SIZE);
911
912	inc_page_count(sbi: fio->sbi, WB_DATA_TYPE(page, false));
913
914	*fio->last_block = fio->new_blkaddr;
915	*fio->bio = bio;
916
917	return 0;
918	}
919
920	#ifdef CONFIG_BLK_DEV_ZONED
921	static bool is_end_zone_blkaddr(struct f2fs_sb_info *sbi, block_t blkaddr)
922	{
923	int devi = 0;
924
925	if (f2fs_is_multi_device(sbi)) {
926	devi = f2fs_target_device_index(sbi, blkaddr);
927	if (blkaddr < FDEV(devi).start_blk ||
928	blkaddr > FDEV(devi).end_blk) {
929	f2fs_err(sbi, "Invalid block %x", blkaddr);
930	return false;
931	}
932	blkaddr -= FDEV(devi).start_blk;
933	}
934	return bdev_is_zoned(FDEV(devi).bdev) &&
935	f2fs_blkz_is_seq(sbi, devi, blkaddr) &&
936	(blkaddr % sbi->blocks_per_blkz == sbi->blocks_per_blkz - 1);
937	}
938	#endif
939
940	void f2fs_submit_page_write(struct f2fs_io_info *fio)
941	{
942	struct f2fs_sb_info *sbi = fio->sbi;
943	enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
944	struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
945	struct page *bio_page;
946	enum count_type type;
947
948	f2fs_bug_on(sbi, is_read_io(fio->op));
949
950	f2fs_down_write(sem: &io->io_rwsem);
951	next:
952	#ifdef CONFIG_BLK_DEV_ZONED
953	if (f2fs_sb_has_blkzoned(sbi) && btype < META && io->zone_pending_bio) {
954	wait_for_completion_io(&io->zone_wait);
955	bio_put(io->zone_pending_bio);
956	io->zone_pending_bio = NULL;
957	io->bi_private = NULL;
958	}
959	#endif
960
961	if (fio->in_list) {
962	spin_lock(lock: &io->io_lock);
963	if (list_empty(head: &io->io_list)) {
964	spin_unlock(lock: &io->io_lock);
965	goto out;
966	}
967	fio = list_first_entry(&io->io_list,
968	struct f2fs_io_info, list);
969	list_del(entry: &fio->list);
970	spin_unlock(lock: &io->io_lock);
971	}
972
973	verify_fio_blkaddr(fio);
974
975	if (fio->encrypted_page)
976	bio_page = fio->encrypted_page;
977	else if (fio->compressed_page)
978	bio_page = fio->compressed_page;
979	else
980	bio_page = fio->page;
981
982	/* set submitted = true as a return value */
983	fio->submitted = 1;
984
985	type = WB_DATA_TYPE(bio_page, fio->compressed_page);
986	inc_page_count(sbi, count_type: type);
987
988	if (io->bio &&
989	(!io_is_mergeable(sbi, bio: io->bio, io, fio, last_blkaddr: io->last_block_in_bio,
990	cur_blkaddr: fio->new_blkaddr) ||
991	!f2fs_crypt_mergeable_bio(bio: io->bio, inode: fio->page->mapping->host,
992	next_idx: bio_page->index, fio)))
993	__submit_merged_bio(io);
994	alloc_new:
995	if (io->bio == NULL) {
996	io->bio = __bio_alloc(fio, BIO_MAX_VECS);
997	f2fs_set_bio_crypt_ctx(bio: io->bio, inode: fio->page->mapping->host,
998	first_idx: bio_page->index, fio, GFP_NOIO);
999	io->fio = *fio;
1000	}
1001
1002	if (bio_add_page(bio: io->bio, page: bio_page, PAGE_SIZE, off: 0) < PAGE_SIZE) {
1003	__submit_merged_bio(io);
1004	goto alloc_new;
1005	}
1006
1007	if (fio->io_wbc)
1008	wbc_account_cgroup_owner(wbc: fio->io_wbc, page: fio->page, PAGE_SIZE);
1009
1010	io->last_block_in_bio = fio->new_blkaddr;
1011
1012	trace_f2fs_submit_page_write(page: fio->page, fio);
1013	#ifdef CONFIG_BLK_DEV_ZONED
1014	if (f2fs_sb_has_blkzoned(sbi) && btype < META &&
1015	is_end_zone_blkaddr(sbi, blkaddr: fio->new_blkaddr)) {
1016	bio_get(bio: io->bio);
1017	reinit_completion(x: &io->zone_wait);
1018	io->bi_private = io->bio->bi_private;
1019	io->bio->bi_private = io;
1020	io->bio->bi_end_io = f2fs_zone_write_end_io;
1021	io->zone_pending_bio = io->bio;
1022	__submit_merged_bio(io);
1023	}
1024	#endif
1025	if (fio->in_list)
1026	goto next;
1027	out:
1028	if (is_sbi_flag_set(sbi, type: SBI_IS_SHUTDOWN) ||
1029	!f2fs_is_checkpoint_ready(sbi))
1030	__submit_merged_bio(io);
1031	f2fs_up_write(sem: &io->io_rwsem);
1032	}
1033
1034	static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
1035	unsigned nr_pages, blk_opf_t op_flag,
1036	pgoff_t first_idx, bool for_write)
1037	{
1038	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1039	struct bio *bio;
1040	struct bio_post_read_ctx *ctx = NULL;
1041	unsigned int post_read_steps = 0;
1042	sector_t sector;
1043	struct block_device *bdev = f2fs_target_device(sbi, blk_addr: blkaddr, sector: &sector);
1044
1045	bio = bio_alloc_bioset(bdev, nr_vecs: bio_max_segs(nr_segs: nr_pages),
1046	opf: REQ_OP_READ | op_flag,
1047	gfp_mask: for_write ? GFP_NOIO : GFP_KERNEL, bs: &f2fs_bioset);
1048	if (!bio)
1049	return ERR_PTR(error: -ENOMEM);
1050	bio->bi_iter.bi_sector = sector;
1051	f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS);
1052	bio->bi_end_io = f2fs_read_end_io;
1053
1054	if (fscrypt_inode_uses_fs_layer_crypto(inode))
1055	post_read_steps |= STEP_DECRYPT;
1056
1057	if (f2fs_need_verity(inode, idx: first_idx))
1058	post_read_steps |= STEP_VERITY;
1059
1060	/*
1061	* STEP_DECOMPRESS is handled specially, since a compressed file might
1062	* contain both compressed and uncompressed clusters. We'll allocate a
1063	* bio_post_read_ctx if the file is compressed, but the caller is
1064	* responsible for enabling STEP_DECOMPRESS if it's actually needed.
1065	*/
1066
1067	if (post_read_steps || f2fs_compressed_file(inode)) {
1068	/* Due to the mempool, this never fails. */
1069	ctx = mempool_alloc(pool: bio_post_read_ctx_pool, GFP_NOFS);
1070	ctx->bio = bio;
1071	ctx->sbi = sbi;
1072	ctx->enabled_steps = post_read_steps;
1073	ctx->fs_blkaddr = blkaddr;
1074	ctx->decompression_attempted = false;
1075	bio->bi_private = ctx;
1076	}
1077	iostat_alloc_and_bind_ctx(sbi, bio, ctx);
1078
1079	return bio;
1080	}
1081
1082	/* This can handle encryption stuffs */
1083	static int f2fs_submit_page_read(struct inode *inode, struct page *page,
1084	block_t blkaddr, blk_opf_t op_flags,
1085	bool for_write)
1086	{
1087	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1088	struct bio *bio;
1089
1090	bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages: 1, op_flag: op_flags,
1091	first_idx: page->index, for_write);
1092	if (IS_ERR(ptr: bio))
1093	return PTR_ERR(ptr: bio);
1094
1095	/* wait for GCed page writeback via META_MAPPING */
1096	f2fs_wait_on_block_writeback(inode, blkaddr);
1097
1098	if (bio_add_page(bio, page, PAGE_SIZE, off: 0) < PAGE_SIZE) {
1099	iostat_update_and_unbind_ctx(bio);
1100	if (bio->bi_private)
1101	mempool_free(element: bio->bi_private, pool: bio_post_read_ctx_pool);
1102	bio_put(bio);
1103	return -EFAULT;
1104	}
1105	inc_page_count(sbi, count_type: F2FS_RD_DATA);
1106	f2fs_update_iostat(sbi, NULL, type: FS_DATA_READ_IO, F2FS_BLKSIZE);
1107	f2fs_submit_read_bio(sbi, bio, type: DATA);
1108	return 0;
1109	}
1110
1111	static void __set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
1112	{
1113	__le32 *addr = get_dnode_addr(inode: dn->inode, node_page: dn->node_page);
1114
1115	dn->data_blkaddr = blkaddr;
1116	addr[dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
1117	}
1118
1119	/*
1120	* Lock ordering for the change of data block address:
1121	* ->data_page
1122	* ->node_page
1123	* update block addresses in the node page
1124	*/
1125	void f2fs_set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
1126	{
1127	f2fs_wait_on_page_writeback(page: dn->node_page, type: NODE, ordered: true, locked: true);
1128	__set_data_blkaddr(dn, blkaddr);
1129	if (set_page_dirty(dn->node_page))
1130	dn->node_changed = true;
1131	}
1132
1133	void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
1134	{
1135	f2fs_set_data_blkaddr(dn, blkaddr);
1136	f2fs_update_read_extent_cache(dn);
1137	}
1138
1139	/* dn->ofs_in_node will be returned with up-to-date last block pointer */
1140	int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
1141	{
1142	struct f2fs_sb_info *sbi = F2FS_I_SB(inode: dn->inode);
1143	int err;
1144
1145	if (!count)
1146	return 0;
1147
1148	if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
1149	return -EPERM;
1150	err = inc_valid_block_count(sbi, inode: dn->inode, count: &count, partial: true);
1151	if (unlikely(err))
1152	return err;
1153
1154	trace_f2fs_reserve_new_blocks(inode: dn->inode, nid: dn->nid,
1155	ofs_in_node: dn->ofs_in_node, count);
1156
1157	f2fs_wait_on_page_writeback(page: dn->node_page, type: NODE, ordered: true, locked: true);
1158
1159	for (; count > 0; dn->ofs_in_node++) {
1160	block_t blkaddr = f2fs_data_blkaddr(dn);
1161
1162	if (blkaddr == NULL_ADDR) {
1163	__set_data_blkaddr(dn, NEW_ADDR);
1164	count--;
1165	}
1166	}
1167
1168	if (set_page_dirty(dn->node_page))
1169	dn->node_changed = true;
1170	return 0;
1171	}
1172
1173	/* Should keep dn->ofs_in_node unchanged */
1174	int f2fs_reserve_new_block(struct dnode_of_data *dn)
1175	{
1176	unsigned int ofs_in_node = dn->ofs_in_node;
1177	int ret;
1178
1179	ret = f2fs_reserve_new_blocks(dn, count: 1);
1180	dn->ofs_in_node = ofs_in_node;
1181	return ret;
1182	}
1183
1184	int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
1185	{
1186	bool need_put = dn->inode_page ? false : true;
1187	int err;
1188
1189	err = f2fs_get_dnode_of_data(dn, index, mode: ALLOC_NODE);
1190	if (err)
1191	return err;
1192
1193	if (dn->data_blkaddr == NULL_ADDR)
1194	err = f2fs_reserve_new_block(dn);
1195	if (err || need_put)
1196	f2fs_put_dnode(dn);
1197	return err;
1198	}
1199
1200	struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
1201	blk_opf_t op_flags, bool for_write,
1202	pgoff_t *next_pgofs)
1203	{
1204	struct address_space *mapping = inode->i_mapping;
1205	struct dnode_of_data dn;
1206	struct page *page;
1207	int err;
1208
1209	page = f2fs_grab_cache_page(mapping, index, for_write);
1210	if (!page)
1211	return ERR_PTR(error: -ENOMEM);
1212
1213	if (f2fs_lookup_read_extent_cache_block(inode, index,
1214	blkaddr: &dn.data_blkaddr)) {
1215	if (!f2fs_is_valid_blkaddr(sbi: F2FS_I_SB(inode), blkaddr: dn.data_blkaddr,
1216	type: DATA_GENERIC_ENHANCE_READ)) {
1217	err = -EFSCORRUPTED;
1218	goto put_err;
1219	}
1220	goto got_it;
1221	}
1222
1223	set_new_dnode(dn: &dn, inode, NULL, NULL, nid: 0);
1224	err = f2fs_get_dnode_of_data(dn: &dn, index, mode: LOOKUP_NODE);
1225	if (err) {
1226	if (err == -ENOENT && next_pgofs)
1227	*next_pgofs = f2fs_get_next_page_offset(dn: &dn, pgofs: index);
1228	goto put_err;
1229	}
1230	f2fs_put_dnode(dn: &dn);
1231
1232	if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
1233	err = -ENOENT;
1234	if (next_pgofs)
1235	*next_pgofs = index + 1;
1236	goto put_err;
1237	}
1238	if (dn.data_blkaddr != NEW_ADDR &&
1239	!f2fs_is_valid_blkaddr(sbi: F2FS_I_SB(inode),
1240	blkaddr: dn.data_blkaddr,
1241	type: DATA_GENERIC_ENHANCE)) {
1242	err = -EFSCORRUPTED;
1243	goto put_err;
1244	}
1245	got_it:
1246	if (PageUptodate(page)) {
1247	unlock_page(page);
1248	return page;
1249	}
1250
1251	/*
1252	* A new dentry page is allocated but not able to be written, since its
1253	* new inode page couldn't be allocated due to -ENOSPC.
1254	* In such the case, its blkaddr can be remained as NEW_ADDR.
1255	* see, f2fs_add_link -> f2fs_get_new_data_page ->
1256	* f2fs_init_inode_metadata.
1257	*/
1258	if (dn.data_blkaddr == NEW_ADDR) {
1259	zero_user_segment(page, start: 0, PAGE_SIZE);
1260	if (!PageUptodate(page))
1261	SetPageUptodate(page);
1262	unlock_page(page);
1263	return page;
1264	}
1265
1266	err = f2fs_submit_page_read(inode, page, blkaddr: dn.data_blkaddr,
1267	op_flags, for_write);
1268	if (err)
1269	goto put_err;
1270	return page;
1271
1272	put_err:
1273	f2fs_put_page(page, unlock: 1);
1274	return ERR_PTR(error: err);
1275	}
1276
1277	struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index,
1278	pgoff_t *next_pgofs)
1279	{
1280	struct address_space *mapping = inode->i_mapping;
1281	struct page *page;
1282
1283	page = find_get_page(mapping, offset: index);
1284	if (page && PageUptodate(page))
1285	return page;
1286	f2fs_put_page(page, unlock: 0);
1287
1288	page = f2fs_get_read_data_page(inode, index, op_flags: 0, for_write: false, next_pgofs);
1289	if (IS_ERR(ptr: page))
1290	return page;
1291
1292	if (PageUptodate(page))
1293	return page;
1294
1295	wait_on_page_locked(page);
1296	if (unlikely(!PageUptodate(page))) {
1297	f2fs_put_page(page, unlock: 0);
1298	return ERR_PTR(error: -EIO);
1299	}
1300	return page;
1301	}
1302
1303	/*
1304	* If it tries to access a hole, return an error.
1305	* Because, the callers, functions in dir.c and GC, should be able to know
1306	* whether this page exists or not.
1307	*/
1308	struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index,
1309	bool for_write)
1310	{
1311	struct address_space *mapping = inode->i_mapping;
1312	struct page *page;
1313
1314	page = f2fs_get_read_data_page(inode, index, op_flags: 0, for_write, NULL);
1315	if (IS_ERR(ptr: page))
1316	return page;
1317
1318	/* wait for read completion */
1319	lock_page(page);
1320	if (unlikely(page->mapping != mapping || !PageUptodate(page))) {
1321	f2fs_put_page(page, unlock: 1);
1322	return ERR_PTR(error: -EIO);
1323	}
1324	return page;
1325	}
1326
1327	/*
1328	* Caller ensures that this data page is never allocated.
1329	* A new zero-filled data page is allocated in the page cache.
1330	*
1331	* Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
1332	* f2fs_unlock_op().
1333	* Note that, ipage is set only by make_empty_dir, and if any error occur,
1334	* ipage should be released by this function.
1335	*/
1336	struct page *f2fs_get_new_data_page(struct inode *inode,
1337	struct page *ipage, pgoff_t index, bool new_i_size)
1338	{
1339	struct address_space *mapping = inode->i_mapping;
1340	struct page *page;
1341	struct dnode_of_data dn;
1342	int err;
1343
1344	page = f2fs_grab_cache_page(mapping, index, for_write: true);
1345	if (!page) {
1346	/*
1347	* before exiting, we should make sure ipage will be released
1348	* if any error occur.
1349	*/
1350	f2fs_put_page(page: ipage, unlock: 1);
1351	return ERR_PTR(error: -ENOMEM);
1352	}
1353
1354	set_new_dnode(dn: &dn, inode, ipage, NULL, nid: 0);
1355	err = f2fs_reserve_block(dn: &dn, index);
1356	if (err) {
1357	f2fs_put_page(page, unlock: 1);
1358	return ERR_PTR(error: err);
1359	}
1360	if (!ipage)
1361	f2fs_put_dnode(dn: &dn);
1362
1363	if (PageUptodate(page))
1364	goto got_it;
1365
1366	if (dn.data_blkaddr == NEW_ADDR) {
1367	zero_user_segment(page, start: 0, PAGE_SIZE);
1368	if (!PageUptodate(page))
1369	SetPageUptodate(page);
1370	} else {
1371	f2fs_put_page(page, unlock: 1);
1372
1373	/* if ipage exists, blkaddr should be NEW_ADDR */
1374	f2fs_bug_on(F2FS_I_SB(inode), ipage);
1375	page = f2fs_get_lock_data_page(inode, index, for_write: true);
1376	if (IS_ERR(ptr: page))
1377	return page;
1378	}
1379	got_it:
1380	if (new_i_size && i_size_read(inode) <
1381	((loff_t)(index + 1) << PAGE_SHIFT))
1382	f2fs_i_size_write(inode, i_size: ((loff_t)(index + 1) << PAGE_SHIFT));
1383	return page;
1384	}
1385
1386	static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
1387	{
1388	struct f2fs_sb_info *sbi = F2FS_I_SB(inode: dn->inode);
1389	struct f2fs_summary sum;
1390	struct node_info ni;
1391	block_t old_blkaddr;
1392	blkcnt_t count = 1;
1393	int err;
1394
1395	if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
1396	return -EPERM;
1397
1398	err = f2fs_get_node_info(sbi, nid: dn->nid, ni: &ni, checkpoint_context: false);
1399	if (err)
1400	return err;
1401
1402	dn->data_blkaddr = f2fs_data_blkaddr(dn);
1403	if (dn->data_blkaddr == NULL_ADDR) {
1404	err = inc_valid_block_count(sbi, inode: dn->inode, count: &count, partial: true);
1405	if (unlikely(err))
1406	return err;
1407	}
1408
1409	set_summary(sum: &sum, nid: dn->nid, ofs_in_node: dn->ofs_in_node, version: ni.version);
1410	old_blkaddr = dn->data_blkaddr;
1411	err = f2fs_allocate_data_block(sbi, NULL, old_blkaddr,
1412	new_blkaddr: &dn->data_blkaddr, sum: &sum, type: seg_type, NULL);
1413	if (err)
1414	return err;
1415
1416	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
1417	f2fs_invalidate_internal_cache(sbi, blkaddr: old_blkaddr);
1418
1419	f2fs_update_data_blkaddr(dn, blkaddr: dn->data_blkaddr);
1420	return 0;
1421	}
1422
1423	static void f2fs_map_lock(struct f2fs_sb_info *sbi, int flag)
1424	{
1425	if (flag == F2FS_GET_BLOCK_PRE_AIO)
1426	f2fs_down_read(sem: &sbi->node_change);
1427	else
1428	f2fs_lock_op(sbi);
1429	}
1430
1431	static void f2fs_map_unlock(struct f2fs_sb_info *sbi, int flag)
1432	{
1433	if (flag == F2FS_GET_BLOCK_PRE_AIO)
1434	f2fs_up_read(sem: &sbi->node_change);
1435	else
1436	f2fs_unlock_op(sbi);
1437	}
1438
1439	int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index)
1440	{
1441	struct f2fs_sb_info *sbi = F2FS_I_SB(inode: dn->inode);
1442	int err = 0;
1443
1444	f2fs_map_lock(sbi, flag: F2FS_GET_BLOCK_PRE_AIO);
1445	if (!f2fs_lookup_read_extent_cache_block(inode: dn->inode, index,
1446	blkaddr: &dn->data_blkaddr))
1447	err = f2fs_reserve_block(dn, index);
1448	f2fs_map_unlock(sbi, flag: F2FS_GET_BLOCK_PRE_AIO);
1449
1450	return err;
1451	}
1452
1453	static int f2fs_map_no_dnode(struct inode *inode,
1454	struct f2fs_map_blocks *map, struct dnode_of_data *dn,
1455	pgoff_t pgoff)
1456	{
1457	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1458
1459	/*
1460	* There is one exceptional case that read_node_page() may return
1461	* -ENOENT due to filesystem has been shutdown or cp_error, return
1462	* -EIO in that case.
1463	*/
1464	if (map->m_may_create &&
1465	(is_sbi_flag_set(sbi, type: SBI_IS_SHUTDOWN) || f2fs_cp_error(sbi)))
1466	return -EIO;
1467
1468	if (map->m_next_pgofs)
1469	*map->m_next_pgofs = f2fs_get_next_page_offset(dn, pgofs: pgoff);
1470	if (map->m_next_extent)
1471	*map->m_next_extent = f2fs_get_next_page_offset(dn, pgofs: pgoff);
1472	return 0;
1473	}
1474
1475	static bool f2fs_map_blocks_cached(struct inode *inode,
1476	struct f2fs_map_blocks *map, int flag)
1477	{
1478	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1479	unsigned int maxblocks = map->m_len;
1480	pgoff_t pgoff = (pgoff_t)map->m_lblk;
1481	struct extent_info ei = {};
1482
1483	if (!f2fs_lookup_read_extent_cache(inode, pgofs: pgoff, ei: &ei))
1484	return false;
1485
1486	map->m_pblk = ei.blk + pgoff - ei.fofs;
1487	map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgoff);
1488	map->m_flags = F2FS_MAP_MAPPED;
1489	if (map->m_next_extent)
1490	*map->m_next_extent = pgoff + map->m_len;
1491
1492	/* for hardware encryption, but to avoid potential issue in future */
1493	if (flag == F2FS_GET_BLOCK_DIO)
1494	f2fs_wait_on_block_writeback_range(inode,
1495	blkaddr: map->m_pblk, len: map->m_len);
1496
1497	if (f2fs_allow_multi_device_dio(sbi, flag)) {
1498	int bidx = f2fs_target_device_index(sbi, blkaddr: map->m_pblk);
1499	struct f2fs_dev_info *dev = &sbi->devs[bidx];
1500
1501	map->m_bdev = dev->bdev;
1502	map->m_pblk -= dev->start_blk;
1503	map->m_len = min(map->m_len, dev->end_blk + 1 - map->m_pblk);
1504	} else {
1505	map->m_bdev = inode->i_sb->s_bdev;
1506	}
1507	return true;
1508	}
1509
1510	/*
1511	* f2fs_map_blocks() tries to find or build mapping relationship which
1512	* maps continuous logical blocks to physical blocks, and return such
1513	* info via f2fs_map_blocks structure.
1514	*/
1515	int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag)
1516	{
1517	unsigned int maxblocks = map->m_len;
1518	struct dnode_of_data dn;
1519	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1520	int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE;
1521	pgoff_t pgofs, end_offset, end;
1522	int err = 0, ofs = 1;
1523	unsigned int ofs_in_node, last_ofs_in_node;
1524	blkcnt_t prealloc;
1525	block_t blkaddr;
1526	unsigned int start_pgofs;
1527	int bidx = 0;
1528	bool is_hole;
1529
1530	if (!maxblocks)
1531	return 0;
1532
1533	if (!map->m_may_create && f2fs_map_blocks_cached(inode, map, flag))
1534	goto out;
1535
1536	map->m_bdev = inode->i_sb->s_bdev;
1537	map->m_multidev_dio =
1538	f2fs_allow_multi_device_dio(sbi: F2FS_I_SB(inode), flag);
1539
1540	map->m_len = 0;
1541	map->m_flags = 0;
1542
1543	/* it only supports block size == page size */
1544	pgofs = (pgoff_t)map->m_lblk;
1545	end = pgofs + maxblocks;
1546
1547	next_dnode:
1548	if (map->m_may_create)
1549	f2fs_map_lock(sbi, flag);
1550
1551	/* When reading holes, we need its node page */
1552	set_new_dnode(dn: &dn, inode, NULL, NULL, nid: 0);
1553	err = f2fs_get_dnode_of_data(dn: &dn, index: pgofs, mode);
1554	if (err) {
1555	if (flag == F2FS_GET_BLOCK_BMAP)
1556	map->m_pblk = 0;
1557	if (err == -ENOENT)
1558	err = f2fs_map_no_dnode(inode, map, dn: &dn, pgoff: pgofs);
1559	goto unlock_out;
1560	}
1561
1562	start_pgofs = pgofs;
1563	prealloc = 0;
1564	last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
1565	end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
1566
1567	next_block:
1568	blkaddr = f2fs_data_blkaddr(dn: &dn);
1569	is_hole = !__is_valid_data_blkaddr(blkaddr);
1570	if (!is_hole &&
1571	!f2fs_is_valid_blkaddr(sbi, blkaddr, type: DATA_GENERIC_ENHANCE)) {
1572	err = -EFSCORRUPTED;
1573	goto sync_out;
1574	}
1575
1576	/* use out-place-update for direct IO under LFS mode */
1577	if (map->m_may_create &&
1578	(is_hole || (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO))) {
1579	if (unlikely(f2fs_cp_error(sbi))) {
1580	err = -EIO;
1581	goto sync_out;
1582	}
1583
1584	switch (flag) {
1585	case F2FS_GET_BLOCK_PRE_AIO:
1586	if (blkaddr == NULL_ADDR) {
1587	prealloc++;
1588	last_ofs_in_node = dn.ofs_in_node;
1589	}
1590	break;
1591	case F2FS_GET_BLOCK_PRE_DIO:
1592	case F2FS_GET_BLOCK_DIO:
1593	err = __allocate_data_block(dn: &dn, seg_type: map->m_seg_type);
1594	if (err)
1595	goto sync_out;
1596	if (flag == F2FS_GET_BLOCK_PRE_DIO)
1597	file_need_truncate(inode);
1598	set_inode_flag(inode, flag: FI_APPEND_WRITE);
1599	break;
1600	default:
1601	WARN_ON_ONCE(1);
1602	err = -EIO;
1603	goto sync_out;
1604	}
1605
1606	blkaddr = dn.data_blkaddr;
1607	if (is_hole)
1608	map->m_flags |= F2FS_MAP_NEW;
1609	} else if (is_hole) {
1610	if (f2fs_compressed_file(inode) &&
1611	f2fs_sanity_check_cluster(dn: &dn)) {
1612	err = -EFSCORRUPTED;
1613	f2fs_handle_error(sbi,
1614	error: ERROR_CORRUPTED_CLUSTER);
1615	goto sync_out;
1616	}
1617
1618	switch (flag) {
1619	case F2FS_GET_BLOCK_PRECACHE:
1620	goto sync_out;
1621	case F2FS_GET_BLOCK_BMAP:
1622	map->m_pblk = 0;
1623	goto sync_out;
1624	case F2FS_GET_BLOCK_FIEMAP:
1625	if (blkaddr == NULL_ADDR) {
1626	if (map->m_next_pgofs)
1627	*map->m_next_pgofs = pgofs + 1;
1628	goto sync_out;
1629	}
1630	break;
1631	default:
1632	/* for defragment case */
1633	if (map->m_next_pgofs)
1634	*map->m_next_pgofs = pgofs + 1;
1635	goto sync_out;
1636	}
1637	}
1638
1639	if (flag == F2FS_GET_BLOCK_PRE_AIO)
1640	goto skip;
1641
1642	if (map->m_multidev_dio)
1643	bidx = f2fs_target_device_index(sbi, blkaddr);
1644
1645	if (map->m_len == 0) {
1646	/* reserved delalloc block should be mapped for fiemap. */
1647	if (blkaddr == NEW_ADDR)
1648	map->m_flags |= F2FS_MAP_DELALLOC;
1649	map->m_flags |= F2FS_MAP_MAPPED;
1650
1651	map->m_pblk = blkaddr;
1652	map->m_len = 1;
1653
1654	if (map->m_multidev_dio)
1655	map->m_bdev = FDEV(bidx).bdev;
1656	} else if ((map->m_pblk != NEW_ADDR &&
1657	blkaddr == (map->m_pblk + ofs)) ||
1658	(map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
1659	flag == F2FS_GET_BLOCK_PRE_DIO) {
1660	if (map->m_multidev_dio && map->m_bdev != FDEV(bidx).bdev)
1661	goto sync_out;
1662	ofs++;
1663	map->m_len++;
1664	} else {
1665	goto sync_out;
1666	}
1667
1668	skip:
1669	dn.ofs_in_node++;
1670	pgofs++;
1671
1672	/* preallocate blocks in batch for one dnode page */
1673	if (flag == F2FS_GET_BLOCK_PRE_AIO &&
1674	(pgofs == end || dn.ofs_in_node == end_offset)) {
1675
1676	dn.ofs_in_node = ofs_in_node;
1677	err = f2fs_reserve_new_blocks(dn: &dn, count: prealloc);
1678	if (err)
1679	goto sync_out;
1680
1681	map->m_len += dn.ofs_in_node - ofs_in_node;
1682	if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
1683	err = -ENOSPC;
1684	goto sync_out;
1685	}
1686	dn.ofs_in_node = end_offset;
1687	}
1688
1689	if (pgofs >= end)
1690	goto sync_out;
1691	else if (dn.ofs_in_node < end_offset)
1692	goto next_block;
1693
1694	if (flag == F2FS_GET_BLOCK_PRECACHE) {
1695	if (map->m_flags & F2FS_MAP_MAPPED) {
1696	unsigned int ofs = start_pgofs - map->m_lblk;
1697
1698	f2fs_update_read_extent_cache_range(dn: &dn,
1699	fofs: start_pgofs, blkaddr: map->m_pblk + ofs,
1700	len: map->m_len - ofs);
1701	}
1702	}
1703
1704	f2fs_put_dnode(dn: &dn);
1705
1706	if (map->m_may_create) {
1707	f2fs_map_unlock(sbi, flag);
1708	f2fs_balance_fs(sbi, need: dn.node_changed);
1709	}
1710	goto next_dnode;
1711
1712	sync_out:
1713
1714	if (flag == F2FS_GET_BLOCK_DIO && map->m_flags & F2FS_MAP_MAPPED) {
1715	/*
1716	* for hardware encryption, but to avoid potential issue
1717	* in future
1718	*/
1719	f2fs_wait_on_block_writeback_range(inode,
1720	blkaddr: map->m_pblk, len: map->m_len);
1721
1722	if (map->m_multidev_dio) {
1723	block_t blk_addr = map->m_pblk;
1724
1725	bidx = f2fs_target_device_index(sbi, blkaddr: map->m_pblk);
1726
1727	map->m_bdev = FDEV(bidx).bdev;
1728	map->m_pblk -= FDEV(bidx).start_blk;
1729
1730	if (map->m_may_create)
1731	f2fs_update_device_state(sbi, ino: inode->i_ino,
1732	blkaddr: blk_addr, blkcnt: map->m_len);
1733
1734	f2fs_bug_on(sbi, blk_addr + map->m_len >
1735	FDEV(bidx).end_blk + 1);
1736	}
1737	}
1738
1739	if (flag == F2FS_GET_BLOCK_PRECACHE) {
1740	if (map->m_flags & F2FS_MAP_MAPPED) {
1741	unsigned int ofs = start_pgofs - map->m_lblk;
1742
1743	f2fs_update_read_extent_cache_range(dn: &dn,
1744	fofs: start_pgofs, blkaddr: map->m_pblk + ofs,
1745	len: map->m_len - ofs);
1746	}
1747	if (map->m_next_extent)
1748	*map->m_next_extent = pgofs + 1;
1749	}
1750	f2fs_put_dnode(dn: &dn);
1751	unlock_out:
1752	if (map->m_may_create) {
1753	f2fs_map_unlock(sbi, flag);
1754	f2fs_balance_fs(sbi, need: dn.node_changed);
1755	}
1756	out:
1757	trace_f2fs_map_blocks(inode, map, flag, ret: err);
1758	return err;
1759	}
1760
1761	bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len)
1762	{
1763	struct f2fs_map_blocks map;
1764	block_t last_lblk;
1765	int err;
1766
1767	if (pos + len > i_size_read(inode))
1768	return false;
1769
1770	map.m_lblk = F2FS_BYTES_TO_BLK(pos);
1771	map.m_next_pgofs = NULL;
1772	map.m_next_extent = NULL;
1773	map.m_seg_type = NO_CHECK_TYPE;
1774	map.m_may_create = false;
1775	last_lblk = F2FS_BLK_ALIGN(pos + len);
1776
1777	while (map.m_lblk < last_lblk) {
1778	map.m_len = last_lblk - map.m_lblk;
1779	err = f2fs_map_blocks(inode, map: &map, flag: F2FS_GET_BLOCK_DEFAULT);
1780	if (err || map.m_len == 0)
1781	return false;
1782	map.m_lblk += map.m_len;
1783	}
1784	return true;
1785	}
1786
1787	static inline u64 bytes_to_blks(struct inode *inode, u64 bytes)
1788	{
1789	return (bytes >> inode->i_blkbits);
1790	}
1791
1792	static inline u64 blks_to_bytes(struct inode *inode, u64 blks)
1793	{
1794	return (blks << inode->i_blkbits);
1795	}
1796
1797	static int f2fs_xattr_fiemap(struct inode *inode,
1798	struct fiemap_extent_info *fieinfo)
1799	{
1800	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1801	struct page *page;
1802	struct node_info ni;
1803	__u64 phys = 0, len;
1804	__u32 flags;
1805	nid_t xnid = F2FS_I(inode)->i_xattr_nid;
1806	int err = 0;
1807
1808	if (f2fs_has_inline_xattr(inode)) {
1809	int offset;
1810
1811	page = f2fs_grab_cache_page(mapping: NODE_MAPPING(sbi),
1812	index: inode->i_ino, for_write: false);
1813	if (!page)
1814	return -ENOMEM;
1815
1816	err = f2fs_get_node_info(sbi, nid: inode->i_ino, ni: &ni, checkpoint_context: false);
1817	if (err) {
1818	f2fs_put_page(page, unlock: 1);
1819	return err;
1820	}
1821
1822	phys = blks_to_bytes(inode, blks: ni.blk_addr);
1823	offset = offsetof(struct f2fs_inode, i_addr) +
1824	sizeof(__le32) * (DEF_ADDRS_PER_INODE -
1825	get_inline_xattr_addrs(inode));
1826
1827	phys += offset;
1828	len = inline_xattr_size(inode);
1829
1830	f2fs_put_page(page, unlock: 1);
1831
1832	flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
1833
1834	if (!xnid)
1835	flags |= FIEMAP_EXTENT_LAST;
1836
1837	err = fiemap_fill_next_extent(info: fieinfo, logical: 0, phys, len, flags);
1838	trace_f2fs_fiemap(inode, lblock: 0, pblock: phys, len, flags, ret: err);
1839	if (err)
1840	return err;
1841	}
1842
1843	if (xnid) {
1844	page = f2fs_grab_cache_page(mapping: NODE_MAPPING(sbi), index: xnid, for_write: false);
1845	if (!page)
1846	return -ENOMEM;
1847
1848	err = f2fs_get_node_info(sbi, nid: xnid, ni: &ni, checkpoint_context: false);
1849	if (err) {
1850	f2fs_put_page(page, unlock: 1);
1851	return err;
1852	}
1853
1854	phys = blks_to_bytes(inode, blks: ni.blk_addr);
1855	len = inode->i_sb->s_blocksize;
1856
1857	f2fs_put_page(page, unlock: 1);
1858
1859	flags = FIEMAP_EXTENT_LAST;
1860	}
1861
1862	if (phys) {
1863	err = fiemap_fill_next_extent(info: fieinfo, logical: 0, phys, len, flags);
1864	trace_f2fs_fiemap(inode, lblock: 0, pblock: phys, len, flags, ret: err);
1865	}
1866
1867	return (err < 0 ? err : 0);
1868	}
1869
1870	static loff_t max_inode_blocks(struct inode *inode)
1871	{
1872	loff_t result = ADDRS_PER_INODE(inode);
1873	loff_t leaf_count = ADDRS_PER_BLOCK(inode);
1874
1875	/* two direct node blocks */
1876	result += (leaf_count * 2);
1877
1878	/* two indirect node blocks */
1879	leaf_count *= NIDS_PER_BLOCK;
1880	result += (leaf_count * 2);
1881
1882	/* one double indirect node block */
1883	leaf_count *= NIDS_PER_BLOCK;
1884	result += leaf_count;
1885
1886	return result;
1887	}
1888
1889	int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1890	u64 start, u64 len)
1891	{
1892	struct f2fs_map_blocks map;
1893	sector_t start_blk, last_blk;
1894	pgoff_t next_pgofs;
1895	u64 logical = 0, phys = 0, size = 0;
1896	u32 flags = 0;
1897	int ret = 0;
1898	bool compr_cluster = false, compr_appended;
1899	unsigned int cluster_size = F2FS_I(inode)->i_cluster_size;
1900	unsigned int count_in_cluster = 0;
1901	loff_t maxbytes;
1902
1903	if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
1904	ret = f2fs_precache_extents(inode);
1905	if (ret)
1906	return ret;
1907	}
1908
1909	ret = fiemap_prep(inode, fieinfo, start, len: &len, FIEMAP_FLAG_XATTR);
1910	if (ret)
1911	return ret;
1912
1913	inode_lock_shared(inode);
1914
1915	maxbytes = max_file_blocks(inode) << F2FS_BLKSIZE_BITS;
1916	if (start > maxbytes) {
1917	ret = -EFBIG;
1918	goto out;
1919	}
1920
1921	if (len > maxbytes || (maxbytes - len) < start)
1922	len = maxbytes - start;
1923
1924	if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
1925	ret = f2fs_xattr_fiemap(inode, fieinfo);
1926	goto out;
1927	}
1928
1929	if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) {
1930	ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1931	if (ret != -EAGAIN)
1932	goto out;
1933	}
1934
1935	if (bytes_to_blks(inode, bytes: len) == 0)
1936	len = blks_to_bytes(inode, blks: 1);
1937
1938	start_blk = bytes_to_blks(inode, bytes: start);
1939	last_blk = bytes_to_blks(inode, bytes: start + len - 1);
1940
1941	next:
1942	memset(&map, 0, sizeof(map));
1943	map.m_lblk = start_blk;
1944	map.m_len = bytes_to_blks(inode, bytes: len);
1945	map.m_next_pgofs = &next_pgofs;
1946	map.m_seg_type = NO_CHECK_TYPE;
1947
1948	if (compr_cluster) {
1949	map.m_lblk += 1;
1950	map.m_len = cluster_size - count_in_cluster;
1951	}
1952
1953	ret = f2fs_map_blocks(inode, map: &map, flag: F2FS_GET_BLOCK_FIEMAP);
1954	if (ret)
1955	goto out;
1956
1957	/* HOLE */
1958	if (!compr_cluster && !(map.m_flags & F2FS_MAP_FLAGS)) {
1959	start_blk = next_pgofs;
1960
1961	if (blks_to_bytes(inode, blks: start_blk) < blks_to_bytes(inode,
1962	blks: max_inode_blocks(inode)))
1963	goto prep_next;
1964
1965	flags |= FIEMAP_EXTENT_LAST;
1966	}
1967
1968	compr_appended = false;
1969	/* In a case of compressed cluster, append this to the last extent */
1970	if (compr_cluster && ((map.m_flags & F2FS_MAP_DELALLOC) ||
1971	!(map.m_flags & F2FS_MAP_FLAGS))) {
1972	compr_appended = true;
1973	goto skip_fill;
1974	}
1975
1976	if (size) {
1977	flags |= FIEMAP_EXTENT_MERGED;
1978	if (IS_ENCRYPTED(inode))
1979	flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1980
1981	ret = fiemap_fill_next_extent(info: fieinfo, logical,
1982	phys, len: size, flags);
1983	trace_f2fs_fiemap(inode, lblock: logical, pblock: phys, len: size, flags, ret);
1984	if (ret)
1985	goto out;
1986	size = 0;
1987	}
1988
1989	if (start_blk > last_blk)
1990	goto out;
1991
1992	skip_fill:
1993	if (map.m_pblk == COMPRESS_ADDR) {
1994	compr_cluster = true;
1995	count_in_cluster = 1;
1996	} else if (compr_appended) {
1997	unsigned int appended_blks = cluster_size -
1998	count_in_cluster + 1;
1999	size += blks_to_bytes(inode, blks: appended_blks);
2000	start_blk += appended_blks;
2001	compr_cluster = false;
2002	} else {
2003	logical = blks_to_bytes(inode, blks: start_blk);
2004	phys = __is_valid_data_blkaddr(blkaddr: map.m_pblk) ?
2005	blks_to_bytes(inode, blks: map.m_pblk) : 0;
2006	size = blks_to_bytes(inode, blks: map.m_len);
2007	flags = 0;
2008
2009	if (compr_cluster) {
2010	flags = FIEMAP_EXTENT_ENCODED;
2011	count_in_cluster += map.m_len;
2012	if (count_in_cluster == cluster_size) {
2013	compr_cluster = false;
2014	size += blks_to_bytes(inode, blks: 1);
2015	}
2016	} else if (map.m_flags & F2FS_MAP_DELALLOC) {
2017	flags = FIEMAP_EXTENT_UNWRITTEN;
2018	}
2019
2020	start_blk += bytes_to_blks(inode, bytes: size);
2021	}
2022
2023	prep_next:
2024	cond_resched();
2025	if (fatal_signal_pending(current))
2026	ret = -EINTR;
2027	else
2028	goto next;
2029	out:
2030	if (ret == 1)
2031	ret = 0;
2032
2033	inode_unlock_shared(inode);
2034	return ret;
2035	}
2036
2037	static inline loff_t f2fs_readpage_limit(struct inode *inode)
2038	{
2039	if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
2040	return inode->i_sb->s_maxbytes;
2041
2042	return i_size_read(inode);
2043	}
2044
2045	static int f2fs_read_single_page(struct inode *inode, struct page *page,
2046	unsigned nr_pages,
2047	struct f2fs_map_blocks *map,
2048	struct bio **bio_ret,
2049	sector_t *last_block_in_bio,
2050	bool is_readahead)
2051	{
2052	struct bio *bio = *bio_ret;
2053	const unsigned blocksize = blks_to_bytes(inode, blks: 1);
2054	sector_t block_in_file;
2055	sector_t last_block;
2056	sector_t last_block_in_file;
2057	sector_t block_nr;
2058	int ret = 0;
2059
2060	block_in_file = (sector_t)page_index(page);
2061	last_block = block_in_file + nr_pages;
2062	last_block_in_file = bytes_to_blks(inode,
2063	bytes: f2fs_readpage_limit(inode) + blocksize - 1);
2064	if (last_block > last_block_in_file)
2065	last_block = last_block_in_file;
2066
2067	/* just zeroing out page which is beyond EOF */
2068	if (block_in_file >= last_block)
2069	goto zero_out;
2070	/*
2071	* Map blocks using the previous result first.
2072	*/
2073	if ((map->m_flags & F2FS_MAP_MAPPED) &&
2074	block_in_file > map->m_lblk &&
2075	block_in_file < (map->m_lblk + map->m_len))
2076	goto got_it;
2077
2078	/*
2079	* Then do more f2fs_map_blocks() calls until we are
2080	* done with this page.
2081	*/
2082	map->m_lblk = block_in_file;
2083	map->m_len = last_block - block_in_file;
2084
2085	ret = f2fs_map_blocks(inode, map, flag: F2FS_GET_BLOCK_DEFAULT);
2086	if (ret)
2087	goto out;
2088	got_it:
2089	if ((map->m_flags & F2FS_MAP_MAPPED)) {
2090	block_nr = map->m_pblk + block_in_file - map->m_lblk;
2091	SetPageMappedToDisk(page);
2092
2093	if (!f2fs_is_valid_blkaddr(sbi: F2FS_I_SB(inode), blkaddr: block_nr,
2094	type: DATA_GENERIC_ENHANCE_READ)) {
2095	ret = -EFSCORRUPTED;
2096	goto out;
2097	}
2098	} else {
2099	zero_out:
2100	zero_user_segment(page, start: 0, PAGE_SIZE);
2101	if (f2fs_need_verity(inode, idx: page->index) &&
2102	!fsverity_verify_page(page)) {
2103	ret = -EIO;
2104	goto out;
2105	}
2106	if (!PageUptodate(page))
2107	SetPageUptodate(page);
2108	unlock_page(page);
2109	goto out;
2110	}
2111
2112	/*
2113	* This page will go to BIO. Do we need to send this
2114	* BIO off first?
2115	*/
2116	if (bio && (!page_is_mergeable(sbi: F2FS_I_SB(inode), bio,
2117	last_blkaddr: *last_block_in_bio, cur_blkaddr: block_nr) ||
2118	!f2fs_crypt_mergeable_bio(bio, inode, next_idx: page->index, NULL))) {
2119	submit_and_realloc:
2120	f2fs_submit_read_bio(sbi: F2FS_I_SB(inode), bio, type: DATA);
2121	bio = NULL;
2122	}
2123	if (bio == NULL) {
2124	bio = f2fs_grab_read_bio(inode, blkaddr: block_nr, nr_pages,
2125	op_flag: is_readahead ? REQ_RAHEAD : 0, first_idx: page->index,
2126	for_write: false);
2127	if (IS_ERR(ptr: bio)) {
2128	ret = PTR_ERR(ptr: bio);
2129	bio = NULL;
2130	goto out;
2131	}
2132	}
2133
2134	/*
2135	* If the page is under writeback, we need to wait for
2136	* its completion to see the correct decrypted data.
2137	*/
2138	f2fs_wait_on_block_writeback(inode, blkaddr: block_nr);
2139
2140	if (bio_add_page(bio, page, len: blocksize, off: 0) < blocksize)
2141	goto submit_and_realloc;
2142
2143	inc_page_count(sbi: F2FS_I_SB(inode), count_type: F2FS_RD_DATA);
2144	f2fs_update_iostat(sbi: F2FS_I_SB(inode), NULL, type: FS_DATA_READ_IO,
2145	F2FS_BLKSIZE);
2146	*last_block_in_bio = block_nr;
2147	out:
2148	*bio_ret = bio;
2149	return ret;
2150	}
2151
2152	#ifdef CONFIG_F2FS_FS_COMPRESSION
2153	int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
2154	unsigned nr_pages, sector_t *last_block_in_bio,
2155	bool is_readahead, bool for_write)
2156	{
2157	struct dnode_of_data dn;
2158	struct inode *inode = cc->inode;
2159	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2160	struct bio *bio = *bio_ret;
2161	unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size;
2162	sector_t last_block_in_file;
2163	const unsigned blocksize = blks_to_bytes(inode, blks: 1);
2164	struct decompress_io_ctx *dic = NULL;
2165	struct extent_info ei = {};
2166	bool from_dnode = true;
2167	int i;
2168	int ret = 0;
2169
2170	f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc));
2171
2172	last_block_in_file = bytes_to_blks(inode,
2173	bytes: f2fs_readpage_limit(inode) + blocksize - 1);
2174
2175	/* get rid of pages beyond EOF */
2176	for (i = 0; i < cc->cluster_size; i++) {
2177	struct page *page = cc->rpages[i];
2178
2179	if (!page)
2180	continue;
2181	if ((sector_t)page->index >= last_block_in_file) {
2182	zero_user_segment(page, start: 0, PAGE_SIZE);
2183	if (!PageUptodate(page))
2184	SetPageUptodate(page);
2185	} else if (!PageUptodate(page)) {
2186	continue;
2187	}
2188	unlock_page(page);
2189	if (for_write)
2190	put_page(page);
2191	cc->rpages[i] = NULL;
2192	cc->nr_rpages--;
2193	}
2194
2195	/* we are done since all pages are beyond EOF */
2196	if (f2fs_cluster_is_empty(cc))
2197	goto out;
2198
2199	if (f2fs_lookup_read_extent_cache(inode, pgofs: start_idx, ei: &ei))
2200	from_dnode = false;
2201
2202	if (!from_dnode)
2203	goto skip_reading_dnode;
2204
2205	set_new_dnode(dn: &dn, inode, NULL, NULL, nid: 0);
2206	ret = f2fs_get_dnode_of_data(dn: &dn, index: start_idx, mode: LOOKUP_NODE);
2207	if (ret)
2208	goto out;
2209
2210	if (unlikely(f2fs_cp_error(sbi))) {
2211	ret = -EIO;
2212	goto out_put_dnode;
2213	}
2214	f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR);
2215
2216	skip_reading_dnode:
2217	for (i = 1; i < cc->cluster_size; i++) {
2218	block_t blkaddr;
2219
2220	blkaddr = from_dnode ? data_blkaddr(inode: dn.inode, node_page: dn.node_page,
2221	offset: dn.ofs_in_node + i) :
2222	ei.blk + i - 1;
2223
2224	if (!__is_valid_data_blkaddr(blkaddr))
2225	break;
2226
2227	if (!f2fs_is_valid_blkaddr(sbi, blkaddr, type: DATA_GENERIC)) {
2228	ret = -EFAULT;
2229	goto out_put_dnode;
2230	}
2231	cc->nr_cpages++;
2232
2233	if (!from_dnode && i >= ei.c_len)
2234	break;
2235	}
2236
2237	/* nothing to decompress */
2238	if (cc->nr_cpages == 0) {
2239	ret = 0;
2240	goto out_put_dnode;
2241	}
2242
2243	dic = f2fs_alloc_dic(cc);
2244	if (IS_ERR(ptr: dic)) {
2245	ret = PTR_ERR(ptr: dic);
2246	goto out_put_dnode;
2247	}
2248
2249	for (i = 0; i < cc->nr_cpages; i++) {
2250	struct page *page = dic->cpages[i];
2251	block_t blkaddr;
2252	struct bio_post_read_ctx *ctx;
2253
2254	blkaddr = from_dnode ? data_blkaddr(inode: dn.inode, node_page: dn.node_page,
2255	offset: dn.ofs_in_node + i + 1) :
2256	ei.blk + i;
2257
2258	f2fs_wait_on_block_writeback(inode, blkaddr);
2259
2260	if (f2fs_load_compressed_page(sbi, page, blkaddr)) {
2261	if (atomic_dec_and_test(v: &dic->remaining_pages)) {
2262	f2fs_decompress_cluster(dic, in_task: true);
2263	break;
2264	}
2265	continue;
2266	}
2267
2268	if (bio && (!page_is_mergeable(sbi, bio,
2269	last_blkaddr: *last_block_in_bio, cur_blkaddr: blkaddr) ||
2270	!f2fs_crypt_mergeable_bio(bio, inode, next_idx: page->index, NULL))) {
2271	submit_and_realloc:
2272	f2fs_submit_read_bio(sbi, bio, type: DATA);
2273	bio = NULL;
2274	}
2275
2276	if (!bio) {
2277	bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages,
2278	op_flag: is_readahead ? REQ_RAHEAD : 0,
2279	first_idx: page->index, for_write);
2280	if (IS_ERR(ptr: bio)) {
2281	ret = PTR_ERR(ptr: bio);
2282	f2fs_decompress_end_io(dic, failed: ret, in_task: true);
2283	f2fs_put_dnode(dn: &dn);
2284	*bio_ret = NULL;
2285	return ret;
2286	}
2287	}
2288
2289	if (bio_add_page(bio, page, len: blocksize, off: 0) < blocksize)
2290	goto submit_and_realloc;
2291
2292	ctx = get_post_read_ctx(bio);
2293	ctx->enabled_steps |= STEP_DECOMPRESS;
2294	refcount_inc(r: &dic->refcnt);
2295
2296	inc_page_count(sbi, count_type: F2FS_RD_DATA);
2297	f2fs_update_iostat(sbi, inode, type: FS_DATA_READ_IO, F2FS_BLKSIZE);
2298	*last_block_in_bio = blkaddr;
2299	}
2300
2301	if (from_dnode)
2302	f2fs_put_dnode(dn: &dn);
2303
2304	*bio_ret = bio;
2305	return 0;
2306
2307	out_put_dnode:
2308	if (from_dnode)
2309	f2fs_put_dnode(dn: &dn);
2310	out:
2311	for (i = 0; i < cc->cluster_size; i++) {
2312	if (cc->rpages[i]) {
2313	ClearPageUptodate(page: cc->rpages[i]);
2314	unlock_page(page: cc->rpages[i]);
2315	}
2316	}
2317	*bio_ret = bio;
2318	return ret;
2319	}
2320	#endif
2321
2322	/*
2323	* This function was originally taken from fs/mpage.c, and customized for f2fs.
2324	* Major change was from block_size == page_size in f2fs by default.
2325	*/
2326	static int f2fs_mpage_readpages(struct inode *inode,
2327	struct readahead_control *rac, struct page *page)
2328	{
2329	struct bio *bio = NULL;
2330	sector_t last_block_in_bio = 0;
2331	struct f2fs_map_blocks map;
2332	#ifdef CONFIG_F2FS_FS_COMPRESSION
2333	struct compress_ctx cc = {
2334	.inode = inode,
2335	.log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
2336	.cluster_size = F2FS_I(inode)->i_cluster_size,
2337	.cluster_idx = NULL_CLUSTER,
2338	.rpages = NULL,
2339	.cpages = NULL,
2340	.nr_rpages = 0,
2341	.nr_cpages = 0,
2342	};
2343	pgoff_t nc_cluster_idx = NULL_CLUSTER;
2344	#endif
2345	unsigned nr_pages = rac ? readahead_count(rac) : 1;
2346	unsigned max_nr_pages = nr_pages;
2347	int ret = 0;
2348
2349	map.m_pblk = 0;
2350	map.m_lblk = 0;
2351	map.m_len = 0;
2352	map.m_flags = 0;
2353	map.m_next_pgofs = NULL;
2354	map.m_next_extent = NULL;
2355	map.m_seg_type = NO_CHECK_TYPE;
2356	map.m_may_create = false;
2357
2358	for (; nr_pages; nr_pages--) {
2359	if (rac) {
2360	page = readahead_page(ractl: rac);
2361	prefetchw(x: &page->flags);
2362	}
2363
2364	#ifdef CONFIG_F2FS_FS_COMPRESSION
2365	if (f2fs_compressed_file(inode)) {
2366	/* there are remained compressed pages, submit them */
2367	if (!f2fs_cluster_can_merge_page(cc: &cc, index: page->index)) {
2368	ret = f2fs_read_multi_pages(cc: &cc, bio_ret: &bio,
2369	nr_pages: max_nr_pages,
2370	last_block_in_bio: &last_block_in_bio,
2371	is_readahead: rac != NULL, for_write: false);
2372	f2fs_destroy_compress_ctx(cc: &cc, reuse: false);
2373	if (ret)
2374	goto set_error_page;
2375	}
2376	if (cc.cluster_idx == NULL_CLUSTER) {
2377	if (nc_cluster_idx ==
2378	page->index >> cc.log_cluster_size) {
2379	goto read_single_page;
2380	}
2381
2382	ret = f2fs_is_compressed_cluster(inode, index: page->index);
2383	if (ret < 0)
2384	goto set_error_page;
2385	else if (!ret) {
2386	nc_cluster_idx =
2387	page->index >> cc.log_cluster_size;
2388	goto read_single_page;
2389	}
2390
2391	nc_cluster_idx = NULL_CLUSTER;
2392	}
2393	ret = f2fs_init_compress_ctx(cc: &cc);
2394	if (ret)
2395	goto set_error_page;
2396
2397	f2fs_compress_ctx_add_page(cc: &cc, page);
2398
2399	goto next_page;
2400	}
2401	read_single_page:
2402	#endif
2403
2404	ret = f2fs_read_single_page(inode, page, nr_pages: max_nr_pages, map: &map,
2405	bio_ret: &bio, last_block_in_bio: &last_block_in_bio, is_readahead: rac);
2406	if (ret) {
2407	#ifdef CONFIG_F2FS_FS_COMPRESSION
2408	set_error_page:
2409	#endif
2410	zero_user_segment(page, start: 0, PAGE_SIZE);
2411	unlock_page(page);
2412	}
2413	#ifdef CONFIG_F2FS_FS_COMPRESSION
2414	next_page:
2415	#endif
2416	if (rac)
2417	put_page(page);
2418
2419	#ifdef CONFIG_F2FS_FS_COMPRESSION
2420	if (f2fs_compressed_file(inode)) {
2421	/* last page */
2422	if (nr_pages == 1 && !f2fs_cluster_is_empty(cc: &cc)) {
2423	ret = f2fs_read_multi_pages(cc: &cc, bio_ret: &bio,
2424	nr_pages: max_nr_pages,
2425	last_block_in_bio: &last_block_in_bio,
2426	is_readahead: rac != NULL, for_write: false);
2427	f2fs_destroy_compress_ctx(cc: &cc, reuse: false);
2428	}
2429	}
2430	#endif
2431	}
2432	if (bio)
2433	f2fs_submit_read_bio(sbi: F2FS_I_SB(inode), bio, type: DATA);
2434	return ret;
2435	}
2436
2437	static int f2fs_read_data_folio(struct file *file, struct folio *folio)
2438	{
2439	struct page *page = &folio->page;
2440	struct inode *inode = page_file_mapping(page)->host;
2441	int ret = -EAGAIN;
2442
2443	trace_f2fs_readpage(page, type: DATA);
2444
2445	if (!f2fs_is_compress_backend_ready(inode)) {
2446	unlock_page(page);
2447	return -EOPNOTSUPP;
2448	}
2449
2450	/* If the file has inline data, try to read it directly */
2451	if (f2fs_has_inline_data(inode))
2452	ret = f2fs_read_inline_data(inode, page);
2453	if (ret == -EAGAIN)
2454	ret = f2fs_mpage_readpages(inode, NULL, page);
2455	return ret;
2456	}
2457
2458	static void f2fs_readahead(struct readahead_control *rac)
2459	{
2460	struct inode *inode = rac->mapping->host;
2461
2462	trace_f2fs_readpages(inode, start: readahead_index(rac), nrpage: readahead_count(rac));
2463
2464	if (!f2fs_is_compress_backend_ready(inode))
2465	return;
2466
2467	/* If the file has inline data, skip readahead */
2468	if (f2fs_has_inline_data(inode))
2469	return;
2470
2471	f2fs_mpage_readpages(inode, rac, NULL);
2472	}
2473
2474	int f2fs_encrypt_one_page(struct f2fs_io_info *fio)
2475	{
2476	struct inode *inode = fio->page->mapping->host;
2477	struct page *mpage, *page;
2478	gfp_t gfp_flags = GFP_NOFS;
2479
2480	if (!f2fs_encrypted_file(inode))
2481	return 0;
2482
2483	page = fio->compressed_page ? fio->compressed_page : fio->page;
2484
2485	if (fscrypt_inode_uses_inline_crypto(inode))
2486	return 0;
2487
2488	retry_encrypt:
2489	fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page,
2490	PAGE_SIZE, offs: 0, gfp_flags);
2491	if (IS_ERR(ptr: fio->encrypted_page)) {
2492	/* flush pending IOs and wait for a while in the ENOMEM case */
2493	if (PTR_ERR(ptr: fio->encrypted_page) == -ENOMEM) {
2494	f2fs_flush_merged_writes(sbi: fio->sbi);
2495	memalloc_retry_wait(GFP_NOFS);
2496	gfp_flags |= __GFP_NOFAIL;
2497	goto retry_encrypt;
2498	}
2499	return PTR_ERR(ptr: fio->encrypted_page);
2500	}
2501
2502	mpage = find_lock_page(mapping: META_MAPPING(sbi: fio->sbi), index: fio->old_blkaddr);
2503	if (mpage) {
2504	if (PageUptodate(page: mpage))
2505	memcpy(page_address(mpage),
2506	page_address(fio->encrypted_page), PAGE_SIZE);
2507	f2fs_put_page(page: mpage, unlock: 1);
2508	}
2509	return 0;
2510	}
2511
2512	static inline bool check_inplace_update_policy(struct inode *inode,
2513	struct f2fs_io_info *fio)
2514	{
2515	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2516
2517	if (IS_F2FS_IPU_HONOR_OPU_WRITE(sbi) &&
2518	is_inode_flag_set(inode, flag: FI_OPU_WRITE))
2519	return false;
2520	if (IS_F2FS_IPU_FORCE(sbi))
2521	return true;
2522	if (IS_F2FS_IPU_SSR(sbi) && f2fs_need_SSR(sbi))
2523	return true;
2524	if (IS_F2FS_IPU_UTIL(sbi) && utilization(sbi) > SM_I(sbi)->min_ipu_util)
2525	return true;
2526	if (IS_F2FS_IPU_SSR_UTIL(sbi) && f2fs_need_SSR(sbi) &&
2527	utilization(sbi) > SM_I(sbi)->min_ipu_util)
2528	return true;
2529
2530	/*
2531	* IPU for rewrite async pages
2532	*/
2533	if (IS_F2FS_IPU_ASYNC(sbi) && fio && fio->op == REQ_OP_WRITE &&
2534	!(fio->op_flags & REQ_SYNC) && !IS_ENCRYPTED(inode))
2535	return true;
2536
2537	/* this is only set during fdatasync */
2538	if (IS_F2FS_IPU_FSYNC(sbi) && is_inode_flag_set(inode, flag: FI_NEED_IPU))
2539	return true;
2540
2541	if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
2542	!f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
2543	return true;
2544
2545	return false;
2546	}
2547
2548	bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)
2549	{
2550	/* swap file is migrating in aligned write mode */
2551	if (is_inode_flag_set(inode, flag: FI_ALIGNED_WRITE))
2552	return false;
2553
2554	if (f2fs_is_pinned_file(inode))
2555	return true;
2556
2557	/* if this is cold file, we should overwrite to avoid fragmentation */
2558	if (file_is_cold(inode) && !is_inode_flag_set(inode, flag: FI_OPU_WRITE))
2559	return true;
2560
2561	return check_inplace_update_policy(inode, fio);
2562	}
2563
2564	bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
2565	{
2566	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2567
2568	/* The below cases were checked when setting it. */
2569	if (f2fs_is_pinned_file(inode))
2570	return false;
2571	if (fio && is_sbi_flag_set(sbi, type: SBI_NEED_FSCK))
2572	return true;
2573	if (f2fs_lfs_mode(sbi))
2574	return true;
2575	if (S_ISDIR(inode->i_mode))
2576	return true;
2577	if (IS_NOQUOTA(inode))
2578	return true;
2579	if (f2fs_is_atomic_file(inode))
2580	return true;
2581	/* rewrite low ratio compress data w/ OPU mode to avoid fragmentation */
2582	if (f2fs_compressed_file(inode) &&
2583	F2FS_OPTION(sbi).compress_mode == COMPR_MODE_USER &&
2584	is_inode_flag_set(inode, flag: FI_ENABLE_COMPRESS))
2585	return true;
2586
2587	/* swap file is migrating in aligned write mode */
2588	if (is_inode_flag_set(inode, flag: FI_ALIGNED_WRITE))
2589	return true;
2590
2591	if (is_inode_flag_set(inode, flag: FI_OPU_WRITE))
2592	return true;
2593
2594	if (fio) {
2595	if (page_private_gcing(page: fio->page))
2596	return true;
2597	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
2598	f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
2599	return true;
2600	}
2601	return false;
2602	}
2603
2604	static inline bool need_inplace_update(struct f2fs_io_info *fio)
2605	{
2606	struct inode *inode = fio->page->mapping->host;
2607
2608	if (f2fs_should_update_outplace(inode, fio))
2609	return false;
2610
2611	return f2fs_should_update_inplace(inode, fio);
2612	}
2613
2614	int f2fs_do_write_data_page(struct f2fs_io_info *fio)
2615	{
2616	struct page *page = fio->page;
2617	struct inode *inode = page->mapping->host;
2618	struct dnode_of_data dn;
2619	struct node_info ni;
2620	bool ipu_force = false;
2621	int err = 0;
2622
2623	/* Use COW inode to make dnode_of_data for atomic write */
2624	if (f2fs_is_atomic_file(inode))
2625	set_new_dnode(dn: &dn, inode: F2FS_I(inode)->cow_inode, NULL, NULL, nid: 0);
2626	else
2627	set_new_dnode(dn: &dn, inode, NULL, NULL, nid: 0);
2628
2629	if (need_inplace_update(fio) &&
2630	f2fs_lookup_read_extent_cache_block(inode, index: page->index,
2631	blkaddr: &fio->old_blkaddr)) {
2632	if (!f2fs_is_valid_blkaddr(sbi: fio->sbi, blkaddr: fio->old_blkaddr,
2633	type: DATA_GENERIC_ENHANCE))
2634	return -EFSCORRUPTED;
2635
2636	ipu_force = true;
2637	fio->need_lock = LOCK_DONE;
2638	goto got_it;
2639	}
2640
2641	/* Deadlock due to between page->lock and f2fs_lock_op */
2642	if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(sbi: fio->sbi))
2643	return -EAGAIN;
2644
2645	err = f2fs_get_dnode_of_data(dn: &dn, index: page->index, mode: LOOKUP_NODE);
2646	if (err)
2647	goto out;
2648
2649	fio->old_blkaddr = dn.data_blkaddr;
2650
2651	/* This page is already truncated */
2652	if (fio->old_blkaddr == NULL_ADDR) {
2653	ClearPageUptodate(page);
2654	clear_page_private_gcing(page);
2655	goto out_writepage;
2656	}
2657	got_it:
2658	if (__is_valid_data_blkaddr(blkaddr: fio->old_blkaddr) &&
2659	!f2fs_is_valid_blkaddr(sbi: fio->sbi, blkaddr: fio->old_blkaddr,
2660	type: DATA_GENERIC_ENHANCE)) {
2661	err = -EFSCORRUPTED;
2662	goto out_writepage;
2663	}
2664
2665	/* wait for GCed page writeback via META_MAPPING */
2666	if (fio->post_read)
2667	f2fs_wait_on_block_writeback(inode, blkaddr: fio->old_blkaddr);
2668
2669	/*
2670	* If current allocation needs SSR,
2671	* it had better in-place writes for updated data.
2672	*/
2673	if (ipu_force ||
2674	(__is_valid_data_blkaddr(blkaddr: fio->old_blkaddr) &&
2675	need_inplace_update(fio))) {
2676	err = f2fs_encrypt_one_page(fio);
2677	if (err)
2678	goto out_writepage;
2679
2680	set_page_writeback(page);
2681	f2fs_put_dnode(dn: &dn);
2682	if (fio->need_lock == LOCK_REQ)
2683	f2fs_unlock_op(sbi: fio->sbi);
2684	err = f2fs_inplace_write_data(fio);
2685	if (err) {
2686	if (fscrypt_inode_uses_fs_layer_crypto(inode))
2687	fscrypt_finalize_bounce_page(pagep: &fio->encrypted_page);
2688	if (PageWriteback(page))
2689	end_page_writeback(page);
2690	} else {
2691	set_inode_flag(inode, flag: FI_UPDATE_WRITE);
2692	}
2693	trace_f2fs_do_write_data_page(page: fio->page, type: IPU);
2694	return err;
2695	}
2696
2697	if (fio->need_lock == LOCK_RETRY) {
2698	if (!f2fs_trylock_op(sbi: fio->sbi)) {
2699	err = -EAGAIN;
2700	goto out_writepage;
2701	}
2702	fio->need_lock = LOCK_REQ;
2703	}
2704
2705	err = f2fs_get_node_info(sbi: fio->sbi, nid: dn.nid, ni: &ni, checkpoint_context: false);
2706	if (err)
2707	goto out_writepage;
2708
2709	fio->version = ni.version;
2710
2711	err = f2fs_encrypt_one_page(fio);
2712	if (err)
2713	goto out_writepage;
2714
2715	set_page_writeback(page);
2716
2717	if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR)
2718	f2fs_i_compr_blocks_update(inode, blocks: fio->compr_blocks - 1, add: false);
2719
2720	/* LFS mode write path */
2721	f2fs_outplace_write_data(dn: &dn, fio);
2722	trace_f2fs_do_write_data_page(page, type: OPU);
2723	set_inode_flag(inode, flag: FI_APPEND_WRITE);
2724	out_writepage:
2725	f2fs_put_dnode(dn: &dn);
2726	out:
2727	if (fio->need_lock == LOCK_REQ)
2728	f2fs_unlock_op(sbi: fio->sbi);
2729	return err;
2730	}
2731
2732	int f2fs_write_single_data_page(struct page *page, int *submitted,
2733	struct bio **bio,
2734	sector_t *last_block,
2735	struct writeback_control *wbc,
2736	enum iostat_type io_type,
2737	int compr_blocks,
2738	bool allow_balance)
2739	{
2740	struct inode *inode = page->mapping->host;
2741	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2742	loff_t i_size = i_size_read(inode);
2743	const pgoff_t end_index = ((unsigned long long)i_size)
2744	>> PAGE_SHIFT;
2745	loff_t psize = (loff_t)(page->index + 1) << PAGE_SHIFT;
2746	unsigned offset = 0;
2747	bool need_balance_fs = false;
2748	bool quota_inode = IS_NOQUOTA(inode);
2749	int err = 0;
2750	struct f2fs_io_info fio = {
2751	.sbi = sbi,
2752	.ino = inode->i_ino,
2753	.type = DATA,
2754	.op = REQ_OP_WRITE,
2755	.op_flags = wbc_to_write_flags(wbc),
2756	.old_blkaddr = NULL_ADDR,
2757	.page = page,
2758	.encrypted_page = NULL,
2759	.submitted = 0,
2760	.compr_blocks = compr_blocks,
2761	.need_lock = compr_blocks ? LOCK_DONE : LOCK_RETRY,
2762	.post_read = f2fs_post_read_required(inode) ? 1 : 0,
2763	.io_type = io_type,
2764	.io_wbc = wbc,
2765	.bio = bio,
2766	.last_block = last_block,
2767	};
2768
2769	trace_f2fs_writepage(page, type: DATA);
2770
2771	/* we should bypass data pages to proceed the kworker jobs */
2772	if (unlikely(f2fs_cp_error(sbi))) {
2773	mapping_set_error(mapping: page->mapping, error: -EIO);
2774	/*
2775	* don't drop any dirty dentry pages for keeping lastest
2776	* directory structure.
2777	*/
2778	if (S_ISDIR(inode->i_mode) &&
2779	!is_sbi_flag_set(sbi, type: SBI_IS_CLOSE))
2780	goto redirty_out;
2781
2782	/* keep data pages in remount-ro mode */
2783	if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY)
2784	goto redirty_out;
2785	goto out;
2786	}
2787
2788	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2789	goto redirty_out;
2790
2791	if (page->index < end_index ||
2792	f2fs_verity_in_progress(inode) ||
2793	compr_blocks)
2794	goto write;
2795
2796	/*
2797	* If the offset is out-of-range of file size,
2798	* this page does not have to be written to disk.
2799	*/
2800	offset = i_size & (PAGE_SIZE - 1);
2801	if ((page->index >= end_index + 1) || !offset)
2802	goto out;
2803
2804	zero_user_segment(page, start: offset, PAGE_SIZE);
2805	write:
2806	/* Dentry/quota blocks are controlled by checkpoint */
2807	if (S_ISDIR(inode->i_mode) || quota_inode) {
2808	/*
2809	* We need to wait for node_write to avoid block allocation during
2810	* checkpoint. This can only happen to quota writes which can cause
2811	* the below discard race condition.
2812	*/
2813	if (quota_inode)
2814	f2fs_down_read(sem: &sbi->node_write);
2815
2816	fio.need_lock = LOCK_DONE;
2817	err = f2fs_do_write_data_page(fio: &fio);
2818
2819	if (quota_inode)
2820	f2fs_up_read(sem: &sbi->node_write);
2821
2822	goto done;
2823	}
2824
2825	if (!wbc->for_reclaim)
2826	need_balance_fs = true;
2827	else if (has_not_enough_free_secs(sbi, freed: 0, needed: 0))
2828	goto redirty_out;
2829	else
2830	set_inode_flag(inode, flag: FI_HOT_DATA);
2831
2832	err = -EAGAIN;
2833	if (f2fs_has_inline_data(inode)) {
2834	err = f2fs_write_inline_data(inode, page);
2835	if (!err)
2836	goto out;
2837	}
2838
2839	if (err == -EAGAIN) {
2840	err = f2fs_do_write_data_page(fio: &fio);
2841	if (err == -EAGAIN) {
2842	f2fs_bug_on(sbi, compr_blocks);
2843	fio.need_lock = LOCK_REQ;
2844	err = f2fs_do_write_data_page(fio: &fio);
2845	}
2846	}
2847
2848	if (err) {
2849	file_set_keep_isize(inode);
2850	} else {
2851	spin_lock(lock: &F2FS_I(inode)->i_size_lock);
2852	if (F2FS_I(inode)->last_disk_size < psize)
2853	F2FS_I(inode)->last_disk_size = psize;
2854	spin_unlock(lock: &F2FS_I(inode)->i_size_lock);
2855	}
2856
2857	done:
2858	if (err && err != -ENOENT)
2859	goto redirty_out;
2860
2861	out:
2862	inode_dec_dirty_pages(inode);
2863	if (err) {
2864	ClearPageUptodate(page);
2865	clear_page_private_gcing(page);
2866	}
2867
2868	if (wbc->for_reclaim) {
2869	f2fs_submit_merged_write_cond(sbi, NULL, page, ino: 0, type: DATA);
2870	clear_inode_flag(inode, flag: FI_HOT_DATA);
2871	f2fs_remove_dirty_inode(inode);
2872	submitted = NULL;
2873	}
2874	unlock_page(page);
2875	if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
2876	!F2FS_I(inode)->wb_task && allow_balance)
2877	f2fs_balance_fs(sbi, need: need_balance_fs);
2878
2879	if (unlikely(f2fs_cp_error(sbi))) {
2880	f2fs_submit_merged_write(sbi, type: DATA);
2881	if (bio && *bio)
2882	f2fs_submit_merged_ipu_write(sbi, bio, NULL);
2883	submitted = NULL;
2884	}
2885
2886	if (submitted)
2887	*submitted = fio.submitted;
2888
2889	return 0;
2890
2891	redirty_out:
2892	redirty_page_for_writepage(wbc, page);
2893	/*
2894	* pageout() in MM translates EAGAIN, so calls handle_write_error()
2895	* -> mapping_set_error() -> set_bit(AS_EIO, ...).
2896	* file_write_and_wait_range() will see EIO error, which is critical
2897	* to return value of fsync() followed by atomic_write failure to user.
2898	*/
2899	if (!err || wbc->for_reclaim)
2900	return AOP_WRITEPAGE_ACTIVATE;
2901	unlock_page(page);
2902	return err;
2903	}
2904
2905	static int f2fs_write_data_page(struct page *page,
2906	struct writeback_control *wbc)
2907	{
2908	#ifdef CONFIG_F2FS_FS_COMPRESSION
2909	struct inode *inode = page->mapping->host;
2910
2911	if (unlikely(f2fs_cp_error(F2FS_I_SB(inode))))
2912	goto out;
2913
2914	if (f2fs_compressed_file(inode)) {
2915	if (f2fs_is_compressed_cluster(inode, index: page->index)) {
2916	redirty_page_for_writepage(wbc, page);
2917	return AOP_WRITEPAGE_ACTIVATE;
2918	}
2919	}
2920	out:
2921	#endif
2922
2923	return f2fs_write_single_data_page(page, NULL, NULL, NULL,
2924	wbc, io_type: FS_DATA_IO, compr_blocks: 0, allow_balance: true);
2925	}
2926
2927	/*
2928	* This function was copied from write_cache_pages from mm/page-writeback.c.
2929	* The major change is making write step of cold data page separately from
2930	* warm/hot data page.
2931	*/
2932	static int f2fs_write_cache_pages(struct address_space *mapping,
2933	struct writeback_control *wbc,
2934	enum iostat_type io_type)
2935	{
2936	int ret = 0;
2937	int done = 0, retry = 0;
2938	struct page *pages_local[F2FS_ONSTACK_PAGES];
2939	struct page **pages = pages_local;
2940	struct folio_batch fbatch;
2941	struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
2942	struct bio *bio = NULL;
2943	sector_t last_block;
2944	#ifdef CONFIG_F2FS_FS_COMPRESSION
2945	struct inode *inode = mapping->host;
2946	struct compress_ctx cc = {
2947	.inode = inode,
2948	.log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
2949	.cluster_size = F2FS_I(inode)->i_cluster_size,
2950	.cluster_idx = NULL_CLUSTER,
2951	.rpages = NULL,
2952	.nr_rpages = 0,
2953	.cpages = NULL,
2954	.valid_nr_cpages = 0,
2955	.rbuf = NULL,
2956	.cbuf = NULL,
2957	.rlen = PAGE_SIZE * F2FS_I(inode)->i_cluster_size,
2958	.private = NULL,
2959	};
2960	#endif
2961	int nr_folios, p, idx;
2962	int nr_pages;
2963	unsigned int max_pages = F2FS_ONSTACK_PAGES;
2964	pgoff_t index;
2965	pgoff_t end; /* Inclusive */
2966	pgoff_t done_index;
2967	int range_whole = 0;
2968	xa_mark_t tag;
2969	int nwritten = 0;
2970	int submitted = 0;
2971	int i;
2972
2973	#ifdef CONFIG_F2FS_FS_COMPRESSION
2974	if (f2fs_compressed_file(inode) &&
2975	1 << cc.log_cluster_size > F2FS_ONSTACK_PAGES) {
2976	pages = f2fs_kzalloc(sbi, size: sizeof(struct page *) <<
2977	cc.log_cluster_size, GFP_NOFS | __GFP_NOFAIL);
2978	max_pages = 1 << cc.log_cluster_size;
2979	}
2980	#endif
2981
2982	folio_batch_init(fbatch: &fbatch);
2983
2984	if (get_dirty_pages(inode: mapping->host) <=
2985	SM_I(sbi: F2FS_M_SB(mapping))->min_hot_blocks)
2986	set_inode_flag(inode: mapping->host, flag: FI_HOT_DATA);
2987	else
2988	clear_inode_flag(inode: mapping->host, flag: FI_HOT_DATA);
2989
2990	if (wbc->range_cyclic) {
2991	index = mapping->writeback_index; /* prev offset */
2992	end = -1;
2993	} else {
2994	index = wbc->range_start >> PAGE_SHIFT;
2995	end = wbc->range_end >> PAGE_SHIFT;
2996	if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2997	range_whole = 1;
2998	}
2999	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
3000	tag = PAGECACHE_TAG_TOWRITE;
3001	else
3002	tag = PAGECACHE_TAG_DIRTY;
3003	retry:
3004	retry = 0;
3005	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
3006	tag_pages_for_writeback(mapping, start: index, end);
3007	done_index = index;
3008	while (!done && !retry && (index <= end)) {
3009	nr_pages = 0;
3010	again:
3011	nr_folios = filemap_get_folios_tag(mapping, start: &index, end,
3012	tag, fbatch: &fbatch);
3013	if (nr_folios == 0) {
3014	if (nr_pages)
3015	goto write;
3016	break;
3017	}
3018
3019	for (i = 0; i < nr_folios; i++) {
3020	struct folio *folio = fbatch.folios[i];
3021
3022	idx = 0;
3023	p = folio_nr_pages(folio);
3024	add_more:
3025	pages[nr_pages] = folio_page(folio, idx);
3026	folio_get(folio);
3027	if (++nr_pages == max_pages) {
3028	index = folio->index + idx + 1;
3029	folio_batch_release(fbatch: &fbatch);
3030	goto write;
3031	}
3032	if (++idx < p)
3033	goto add_more;
3034	}
3035	folio_batch_release(fbatch: &fbatch);
3036	goto again;
3037	write:
3038	for (i = 0; i < nr_pages; i++) {
3039	struct page *page = pages[i];
3040	struct folio *folio = page_folio(page);
3041	bool need_readd;
3042	readd:
3043	need_readd = false;
3044	#ifdef CONFIG_F2FS_FS_COMPRESSION
3045	if (f2fs_compressed_file(inode)) {
3046	void *fsdata = NULL;
3047	struct page *pagep;
3048	int ret2;
3049
3050	ret = f2fs_init_compress_ctx(cc: &cc);
3051	if (ret) {
3052	done = 1;
3053	break;
3054	}
3055
3056	if (!f2fs_cluster_can_merge_page(cc: &cc,
3057	index: folio->index)) {
3058	ret = f2fs_write_multi_pages(cc: &cc,
3059	submitted: &submitted, wbc, io_type);
3060	if (!ret)
3061	need_readd = true;
3062	goto result;
3063	}
3064
3065	if (unlikely(f2fs_cp_error(sbi)))
3066	goto lock_folio;
3067
3068	if (!f2fs_cluster_is_empty(cc: &cc))
3069	goto lock_folio;
3070
3071	if (f2fs_all_cluster_page_ready(cc: &cc,
3072	pages, index: i, nr_pages, uptodate: true))
3073	goto lock_folio;
3074
3075	ret2 = f2fs_prepare_compress_overwrite(
3076	inode, pagep: &pagep,
3077	index: folio->index, fsdata: &fsdata);
3078	if (ret2 < 0) {
3079	ret = ret2;
3080	done = 1;
3081	break;
3082	} else if (ret2 &&
3083	(!f2fs_compress_write_end(inode,
3084	fsdata, index: folio->index, copied: 1) ||
3085	!f2fs_all_cluster_page_ready(cc: &cc,
3086	pages, index: i, nr_pages,
3087	uptodate: false))) {
3088	retry = 1;
3089	break;
3090	}
3091	}
3092	#endif
3093	/* give a priority to WB_SYNC threads */
3094	if (atomic_read(v: &sbi->wb_sync_req[DATA]) &&
3095	wbc->sync_mode == WB_SYNC_NONE) {
3096	done = 1;
3097	break;
3098	}
3099	#ifdef CONFIG_F2FS_FS_COMPRESSION
3100	lock_folio:
3101	#endif
3102	done_index = folio->index;
3103	retry_write:
3104	folio_lock(folio);
3105
3106	if (unlikely(folio->mapping != mapping)) {
3107	continue_unlock:
3108	folio_unlock(folio);
3109	continue;
3110	}
3111
3112	if (!folio_test_dirty(folio)) {
3113	/* someone wrote it for us */
3114	goto continue_unlock;
3115	}
3116
3117	if (folio_test_writeback(folio)) {
3118	if (wbc->sync_mode == WB_SYNC_NONE)
3119	goto continue_unlock;
3120	f2fs_wait_on_page_writeback(page: &folio->page, type: DATA, ordered: true, locked: true);
3121	}
3122
3123	if (!folio_clear_dirty_for_io(folio))
3124	goto continue_unlock;
3125
3126	#ifdef CONFIG_F2FS_FS_COMPRESSION
3127	if (f2fs_compressed_file(inode)) {
3128	folio_get(folio);
3129	f2fs_compress_ctx_add_page(cc: &cc, page: &folio->page);
3130	continue;
3131	}
3132	#endif
3133	ret = f2fs_write_single_data_page(page: &folio->page,
3134	submitted: &submitted, bio: &bio, last_block: &last_block,
3135	wbc, io_type, compr_blocks: 0, allow_balance: true);
3136	if (ret == AOP_WRITEPAGE_ACTIVATE)
3137	folio_unlock(folio);
3138	#ifdef CONFIG_F2FS_FS_COMPRESSION
3139	result:
3140	#endif
3141	nwritten += submitted;
3142	wbc->nr_to_write -= submitted;
3143
3144	if (unlikely(ret)) {
3145	/*
3146	* keep nr_to_write, since vfs uses this to
3147	* get # of written pages.
3148	*/
3149	if (ret == AOP_WRITEPAGE_ACTIVATE) {
3150	ret = 0;
3151	goto next;
3152	} else if (ret == -EAGAIN) {
3153	ret = 0;
3154	if (wbc->sync_mode == WB_SYNC_ALL) {
3155	f2fs_io_schedule_timeout(
3156	DEFAULT_IO_TIMEOUT);
3157	goto retry_write;
3158	}
3159	goto next;
3160	}
3161	done_index = folio_next_index(folio);
3162	done = 1;
3163	break;
3164	}
3165
3166	if (wbc->nr_to_write <= 0 &&
3167	wbc->sync_mode == WB_SYNC_NONE) {
3168	done = 1;
3169	break;
3170	}
3171	next:
3172	if (need_readd)
3173	goto readd;
3174	}
3175	release_pages(pages, nr: nr_pages);
3176	cond_resched();
3177	}
3178	#ifdef CONFIG_F2FS_FS_COMPRESSION
3179	/* flush remained pages in compress cluster */
3180	if (f2fs_compressed_file(inode) && !f2fs_cluster_is_empty(cc: &cc)) {
3181	ret = f2fs_write_multi_pages(cc: &cc, submitted: &submitted, wbc, io_type);
3182	nwritten += submitted;
3183	wbc->nr_to_write -= submitted;
3184	if (ret) {
3185	done = 1;
3186	retry = 0;
3187	}
3188	}
3189	if (f2fs_compressed_file(inode))
3190	f2fs_destroy_compress_ctx(cc: &cc, reuse: false);
3191	#endif
3192	if (retry) {
3193	index = 0;
3194	end = -1;
3195	goto retry;
3196	}
3197	if (wbc->range_cyclic && !done)
3198	done_index = 0;
3199	if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
3200	mapping->writeback_index = done_index;
3201
3202	if (nwritten)
3203	f2fs_submit_merged_write_cond(sbi: F2FS_M_SB(mapping), inode: mapping->host,
3204	NULL, ino: 0, type: DATA);
3205	/* submit cached bio of IPU write */
3206	if (bio)
3207	f2fs_submit_merged_ipu_write(sbi, bio: &bio, NULL);
3208
3209	#ifdef CONFIG_F2FS_FS_COMPRESSION
3210	if (pages != pages_local)
3211	kfree(objp: pages);
3212	#endif
3213
3214	return ret;
3215	}
3216
3217	static inline bool __should_serialize_io(struct inode *inode,
3218	struct writeback_control *wbc)
3219	{
3220	/* to avoid deadlock in path of data flush */
3221	if (F2FS_I(inode)->wb_task)
3222	return false;
3223
3224	if (!S_ISREG(inode->i_mode))
3225	return false;
3226	if (IS_NOQUOTA(inode))
3227	return false;
3228
3229	if (f2fs_need_compress_data(inode))
3230	return true;
3231	if (wbc->sync_mode != WB_SYNC_ALL)
3232	return true;
3233	if (get_dirty_pages(inode) >= SM_I(sbi: F2FS_I_SB(inode))->min_seq_blocks)
3234	return true;
3235	return false;
3236	}
3237
3238	static int __f2fs_write_data_pages(struct address_space *mapping,
3239	struct writeback_control *wbc,
3240	enum iostat_type io_type)
3241	{
3242	struct inode *inode = mapping->host;
3243	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3244	struct blk_plug plug;
3245	int ret;
3246	bool locked = false;
3247
3248	/* deal with chardevs and other special file */
3249	if (!mapping->a_ops->writepage)
3250	return 0;
3251
3252	/* skip writing if there is no dirty page in this inode */
3253	if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
3254	return 0;
3255
3256	/* during POR, we don't need to trigger writepage at all. */
3257	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
3258	goto skip_write;
3259
3260	if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) &&
3261	wbc->sync_mode == WB_SYNC_NONE &&
3262	get_dirty_pages(inode) < nr_pages_to_skip(sbi, type: DATA) &&
3263	f2fs_available_free_memory(sbi, type: DIRTY_DENTS))
3264	goto skip_write;
3265
3266	/* skip writing in file defragment preparing stage */
3267	if (is_inode_flag_set(inode, flag: FI_SKIP_WRITES))
3268	goto skip_write;
3269
3270	trace_f2fs_writepages(inode: mapping->host, wbc, type: DATA);
3271
3272	/* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
3273	if (wbc->sync_mode == WB_SYNC_ALL)
3274	atomic_inc(v: &sbi->wb_sync_req[DATA]);
3275	else if (atomic_read(v: &sbi->wb_sync_req[DATA])) {
3276	/* to avoid potential deadlock */
3277	if (current->plug)
3278	blk_finish_plug(current->plug);
3279	goto skip_write;
3280	}
3281
3282	if (__should_serialize_io(inode, wbc)) {
3283	mutex_lock(&sbi->writepages);
3284	locked = true;
3285	}
3286
3287	blk_start_plug(&plug);
3288	ret = f2fs_write_cache_pages(mapping, wbc, io_type);
3289	blk_finish_plug(&plug);
3290
3291	if (locked)
3292	mutex_unlock(lock: &sbi->writepages);
3293
3294	if (wbc->sync_mode == WB_SYNC_ALL)
3295	atomic_dec(v: &sbi->wb_sync_req[DATA]);
3296	/*
3297	* if some pages were truncated, we cannot guarantee its mapping->host
3298	* to detect pending bios.
3299	*/
3300
3301	f2fs_remove_dirty_inode(inode);
3302	return ret;
3303
3304	skip_write:
3305	wbc->pages_skipped += get_dirty_pages(inode);
3306	trace_f2fs_writepages(inode: mapping->host, wbc, type: DATA);
3307	return 0;
3308	}
3309
3310	static int f2fs_write_data_pages(struct address_space *mapping,
3311	struct writeback_control *wbc)
3312	{
3313	struct inode *inode = mapping->host;
3314
3315	return __f2fs_write_data_pages(mapping, wbc,
3316	io_type: F2FS_I(inode)->cp_task == current ?
3317	FS_CP_DATA_IO : FS_DATA_IO);
3318	}
3319
3320	void f2fs_write_failed(struct inode *inode, loff_t to)
3321	{
3322	loff_t i_size = i_size_read(inode);
3323
3324	if (IS_NOQUOTA(inode))
3325	return;
3326
3327	/* In the fs-verity case, f2fs_end_enable_verity() does the truncate */
3328	if (to > i_size && !f2fs_verity_in_progress(inode)) {
3329	f2fs_down_write(sem: &F2FS_I(inode)->i_gc_rwsem[WRITE]);
3330	filemap_invalidate_lock(mapping: inode->i_mapping);
3331
3332	truncate_pagecache(inode, new: i_size);
3333	f2fs_truncate_blocks(inode, from: i_size, lock: true);
3334
3335	filemap_invalidate_unlock(mapping: inode->i_mapping);
3336	f2fs_up_write(sem: &F2FS_I(inode)->i_gc_rwsem[WRITE]);
3337	}
3338	}
3339
3340	static int prepare_write_begin(struct f2fs_sb_info *sbi,
3341	struct page *page, loff_t pos, unsigned len,
3342	block_t *blk_addr, bool *node_changed)
3343	{
3344	struct inode *inode = page->mapping->host;
3345	pgoff_t index = page->index;
3346	struct dnode_of_data dn;
3347	struct page *ipage;
3348	bool locked = false;
3349	int flag = F2FS_GET_BLOCK_PRE_AIO;
3350	int err = 0;
3351
3352	/*
3353	* If a whole page is being written and we already preallocated all the
3354	* blocks, then there is no need to get a block address now.
3355	*/
3356	if (len == PAGE_SIZE && is_inode_flag_set(inode, flag: FI_PREALLOCATED_ALL))
3357	return 0;
3358
3359	/* f2fs_lock_op avoids race between write CP and convert_inline_page */
3360	if (f2fs_has_inline_data(inode)) {
3361	if (pos + len > MAX_INLINE_DATA(inode))
3362	flag = F2FS_GET_BLOCK_DEFAULT;
3363	f2fs_map_lock(sbi, flag);
3364	locked = true;
3365	} else if ((pos & PAGE_MASK) >= i_size_read(inode)) {
3366	f2fs_map_lock(sbi, flag);
3367	locked = true;
3368	}
3369
3370	restart:
3371	/* check inline_data */
3372	ipage = f2fs_get_node_page(sbi, nid: inode->i_ino);
3373	if (IS_ERR(ptr: ipage)) {
3374	err = PTR_ERR(ptr: ipage);
3375	goto unlock_out;
3376	}
3377
3378	set_new_dnode(dn: &dn, inode, ipage, npage: ipage, nid: 0);
3379
3380	if (f2fs_has_inline_data(inode)) {
3381	if (pos + len <= MAX_INLINE_DATA(inode)) {
3382	f2fs_do_read_inline_data(page, ipage);
3383	set_inode_flag(inode, flag: FI_DATA_EXIST);
3384	if (inode->i_nlink)
3385	set_page_private_inline(ipage);
3386	goto out;
3387	}
3388	err = f2fs_convert_inline_page(dn: &dn, page);
3389	if (err || dn.data_blkaddr != NULL_ADDR)
3390	goto out;
3391	}
3392
3393	if (!f2fs_lookup_read_extent_cache_block(inode, index,
3394	blkaddr: &dn.data_blkaddr)) {
3395	if (locked) {
3396	err = f2fs_reserve_block(dn: &dn, index);
3397	goto out;
3398	}
3399
3400	/* hole case */
3401	err = f2fs_get_dnode_of_data(dn: &dn, index, mode: LOOKUP_NODE);
3402	if (!err && dn.data_blkaddr != NULL_ADDR)
3403	goto out;
3404	f2fs_put_dnode(dn: &dn);
3405	f2fs_map_lock(sbi, flag: F2FS_GET_BLOCK_PRE_AIO);
3406	WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO);
3407	locked = true;
3408	goto restart;
3409	}
3410	out:
3411	if (!err) {
3412	/* convert_inline_page can make node_changed */
3413	*blk_addr = dn.data_blkaddr;
3414	*node_changed = dn.node_changed;
3415	}
3416	f2fs_put_dnode(dn: &dn);
3417	unlock_out:
3418	if (locked)
3419	f2fs_map_unlock(sbi, flag);
3420	return err;
3421	}
3422
3423	static int __find_data_block(struct inode *inode, pgoff_t index,
3424	block_t *blk_addr)
3425	{
3426	struct dnode_of_data dn;
3427	struct page *ipage;
3428	int err = 0;
3429
3430	ipage = f2fs_get_node_page(sbi: F2FS_I_SB(inode), nid: inode->i_ino);
3431	if (IS_ERR(ptr: ipage))
3432	return PTR_ERR(ptr: ipage);
3433
3434	set_new_dnode(dn: &dn, inode, ipage, npage: ipage, nid: 0);
3435
3436	if (!f2fs_lookup_read_extent_cache_block(inode, index,
3437	blkaddr: &dn.data_blkaddr)) {
3438	/* hole case */
3439	err = f2fs_get_dnode_of_data(dn: &dn, index, mode: LOOKUP_NODE);
3440	if (err) {
3441	dn.data_blkaddr = NULL_ADDR;
3442	err = 0;
3443	}
3444	}
3445	*blk_addr = dn.data_blkaddr;
3446	f2fs_put_dnode(dn: &dn);
3447	return err;
3448	}
3449
3450	static int __reserve_data_block(struct inode *inode, pgoff_t index,
3451	block_t *blk_addr, bool *node_changed)
3452	{
3453	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3454	struct dnode_of_data dn;
3455	struct page *ipage;
3456	int err = 0;
3457
3458	f2fs_map_lock(sbi, flag: F2FS_GET_BLOCK_PRE_AIO);
3459
3460	ipage = f2fs_get_node_page(sbi, nid: inode->i_ino);
3461	if (IS_ERR(ptr: ipage)) {
3462	err = PTR_ERR(ptr: ipage);
3463	goto unlock_out;
3464	}
3465	set_new_dnode(dn: &dn, inode, ipage, npage: ipage, nid: 0);
3466
3467	if (!f2fs_lookup_read_extent_cache_block(inode: dn.inode, index,
3468	blkaddr: &dn.data_blkaddr))
3469	err = f2fs_reserve_block(dn: &dn, index);
3470
3471	*blk_addr = dn.data_blkaddr;
3472	*node_changed = dn.node_changed;
3473	f2fs_put_dnode(dn: &dn);
3474
3475	unlock_out:
3476	f2fs_map_unlock(sbi, flag: F2FS_GET_BLOCK_PRE_AIO);
3477	return err;
3478	}
3479
3480	static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi,
3481	struct page *page, loff_t pos, unsigned int len,
3482	block_t *blk_addr, bool *node_changed, bool *use_cow)
3483	{
3484	struct inode *inode = page->mapping->host;
3485	struct inode *cow_inode = F2FS_I(inode)->cow_inode;
3486	pgoff_t index = page->index;
3487	int err = 0;
3488	block_t ori_blk_addr = NULL_ADDR;
3489
3490	/* If pos is beyond the end of file, reserve a new block in COW inode */
3491	if ((pos & PAGE_MASK) >= i_size_read(inode))
3492	goto reserve_block;
3493
3494	/* Look for the block in COW inode first */
3495	err = __find_data_block(inode: cow_inode, index, blk_addr);
3496	if (err) {
3497	return err;
3498	} else if (*blk_addr != NULL_ADDR) {
3499	*use_cow = true;
3500	return 0;
3501	}
3502
3503	if (is_inode_flag_set(inode, flag: FI_ATOMIC_REPLACE))
3504	goto reserve_block;
3505
3506	/* Look for the block in the original inode */
3507	err = __find_data_block(inode, index, blk_addr: &ori_blk_addr);
3508	if (err)
3509	return err;
3510
3511	reserve_block:
3512	/* Finally, we should reserve a new block in COW inode for the update */
3513	err = __reserve_data_block(inode: cow_inode, index, blk_addr, node_changed);
3514	if (err)
3515	return err;
3516	inc_atomic_write_cnt(inode);
3517
3518	if (ori_blk_addr != NULL_ADDR)
3519	*blk_addr = ori_blk_addr;
3520	return 0;
3521	}
3522
3523	static int f2fs_write_begin(struct file *file, struct address_space *mapping,
3524	loff_t pos, unsigned len, struct page **pagep, void **fsdata)
3525	{
3526	struct inode *inode = mapping->host;
3527	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3528	struct page *page = NULL;
3529	pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
3530	bool need_balance = false;
3531	bool use_cow = false;
3532	block_t blkaddr = NULL_ADDR;
3533	int err = 0;
3534
3535	trace_f2fs_write_begin(inode, pos, len);
3536
3537	if (!f2fs_is_checkpoint_ready(sbi)) {
3538	err = -ENOSPC;
3539	goto fail;
3540	}
3541
3542	/*
3543	* We should check this at this moment to avoid deadlock on inode page
3544	* and #0 page. The locking rule for inline_data conversion should be:
3545	* lock_page(page #0) -> lock_page(inode_page)
3546	*/
3547	if (index != 0) {
3548	err = f2fs_convert_inline_inode(inode);
3549	if (err)
3550	goto fail;
3551	}
3552
3553	#ifdef CONFIG_F2FS_FS_COMPRESSION
3554	if (f2fs_compressed_file(inode)) {
3555	int ret;
3556
3557	*fsdata = NULL;
3558
3559	if (len == PAGE_SIZE && !(f2fs_is_atomic_file(inode)))
3560	goto repeat;
3561
3562	ret = f2fs_prepare_compress_overwrite(inode, pagep,
3563	index, fsdata);
3564	if (ret < 0) {
3565	err = ret;
3566	goto fail;
3567	} else if (ret) {
3568	return 0;
3569	}
3570	}
3571	#endif
3572
3573	repeat:
3574	/*
3575	* Do not use grab_cache_page_write_begin() to avoid deadlock due to
3576	* wait_for_stable_page. Will wait that below with our IO control.
3577	*/
3578	page = f2fs_pagecache_get_page(mapping, index,
3579	FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
3580	if (!page) {
3581	err = -ENOMEM;
3582	goto fail;
3583	}
3584
3585	/* TODO: cluster can be compressed due to race with .writepage */
3586
3587	*pagep = page;
3588
3589	if (f2fs_is_atomic_file(inode))
3590	err = prepare_atomic_write_begin(sbi, page, pos, len,
3591	blk_addr: &blkaddr, node_changed: &need_balance, use_cow: &use_cow);
3592	else
3593	err = prepare_write_begin(sbi, page, pos, len,
3594	blk_addr: &blkaddr, node_changed: &need_balance);
3595	if (err)
3596	goto fail;
3597
3598	if (need_balance && !IS_NOQUOTA(inode) &&
3599	has_not_enough_free_secs(sbi, freed: 0, needed: 0)) {
3600	unlock_page(page);
3601	f2fs_balance_fs(sbi, need: true);
3602	lock_page(page);
3603	if (page->mapping != mapping) {
3604	/* The page got truncated from under us */
3605	f2fs_put_page(page, unlock: 1);
3606	goto repeat;
3607	}
3608	}
3609
3610	f2fs_wait_on_page_writeback(page, type: DATA, ordered: false, locked: true);
3611
3612	if (len == PAGE_SIZE || PageUptodate(page))
3613	return 0;
3614
3615	if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode) &&
3616	!f2fs_verity_in_progress(inode)) {
3617	zero_user_segment(page, start: len, PAGE_SIZE);
3618	return 0;
3619	}
3620
3621	if (blkaddr == NEW_ADDR) {
3622	zero_user_segment(page, start: 0, PAGE_SIZE);
3623	SetPageUptodate(page);
3624	} else {
3625	if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
3626	type: DATA_GENERIC_ENHANCE_READ)) {
3627	err = -EFSCORRUPTED;
3628	goto fail;
3629	}
3630	err = f2fs_submit_page_read(inode: use_cow ?
3631	F2FS_I(inode)->cow_inode : inode, page,
3632	blkaddr, op_flags: 0, for_write: true);
3633	if (err)
3634	goto fail;
3635
3636	lock_page(page);
3637	if (unlikely(page->mapping != mapping)) {
3638	f2fs_put_page(page, unlock: 1);
3639	goto repeat;
3640	}
3641	if (unlikely(!PageUptodate(page))) {
3642	err = -EIO;
3643	goto fail;
3644	}
3645	}
3646	return 0;
3647
3648	fail:
3649	f2fs_put_page(page, unlock: 1);
3650	f2fs_write_failed(inode, to: pos + len);
3651	return err;
3652	}
3653
3654	static int f2fs_write_end(struct file *file,
3655	struct address_space *mapping,
3656	loff_t pos, unsigned len, unsigned copied,
3657	struct page *page, void *fsdata)
3658	{
3659	struct inode *inode = page->mapping->host;
3660
3661	trace_f2fs_write_end(inode, pos, len, copied);
3662
3663	/*
3664	* This should be come from len == PAGE_SIZE, and we expect copied
3665	* should be PAGE_SIZE. Otherwise, we treat it with zero copied and
3666	* let generic_perform_write() try to copy data again through copied=0.
3667	*/
3668	if (!PageUptodate(page)) {
3669	if (unlikely(copied != len))
3670	copied = 0;
3671	else
3672	SetPageUptodate(page);
3673	}
3674
3675	#ifdef CONFIG_F2FS_FS_COMPRESSION
3676	/* overwrite compressed file */
3677	if (f2fs_compressed_file(inode) && fsdata) {
3678	f2fs_compress_write_end(inode, fsdata, index: page->index, copied);
3679	f2fs_update_time(sbi: F2FS_I_SB(inode), type: REQ_TIME);
3680
3681	if (pos + copied > i_size_read(inode) &&
3682	!f2fs_verity_in_progress(inode))
3683	f2fs_i_size_write(inode, i_size: pos + copied);
3684	return copied;
3685	}
3686	#endif
3687
3688	if (!copied)
3689	goto unlock_out;
3690
3691	set_page_dirty(page);
3692
3693	if (pos + copied > i_size_read(inode) &&
3694	!f2fs_verity_in_progress(inode)) {
3695	f2fs_i_size_write(inode, i_size: pos + copied);
3696	if (f2fs_is_atomic_file(inode))
3697	f2fs_i_size_write(inode: F2FS_I(inode)->cow_inode,
3698	i_size: pos + copied);
3699	}
3700	unlock_out:
3701	f2fs_put_page(page, unlock: 1);
3702	f2fs_update_time(sbi: F2FS_I_SB(inode), type: REQ_TIME);
3703	return copied;
3704	}
3705
3706	void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
3707	{
3708	struct inode *inode = folio->mapping->host;
3709	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3710
3711	if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
3712	(offset || length != folio_size(folio)))
3713	return;
3714
3715	if (folio_test_dirty(folio)) {
3716	if (inode->i_ino == F2FS_META_INO(sbi)) {
3717	dec_page_count(sbi, count_type: F2FS_DIRTY_META);
3718	} else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
3719	dec_page_count(sbi, count_type: F2FS_DIRTY_NODES);
3720	} else {
3721	inode_dec_dirty_pages(inode);
3722	f2fs_remove_dirty_inode(inode);
3723	}
3724	}
3725	clear_page_private_all(page: &folio->page);
3726	}
3727
3728	bool f2fs_release_folio(struct folio *folio, gfp_t wait)
3729	{
3730	/* If this is dirty folio, keep private data */
3731	if (folio_test_dirty(folio))
3732	return false;
3733
3734	clear_page_private_all(page: &folio->page);
3735	return true;
3736	}
3737
3738	static bool f2fs_dirty_data_folio(struct address_space *mapping,
3739	struct folio *folio)
3740	{
3741	struct inode *inode = mapping->host;
3742
3743	trace_f2fs_set_page_dirty(page: &folio->page, type: DATA);
3744
3745	if (!folio_test_uptodate(folio))
3746	folio_mark_uptodate(folio);
3747	BUG_ON(folio_test_swapcache(folio));
3748
3749	if (filemap_dirty_folio(mapping, folio)) {
3750	f2fs_update_dirty_folio(inode, folio);
3751	return true;
3752	}
3753	return false;
3754	}
3755
3756
3757	static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block)
3758	{
3759	#ifdef CONFIG_F2FS_FS_COMPRESSION
3760	struct dnode_of_data dn;
3761	sector_t start_idx, blknr = 0;
3762	int ret;
3763
3764	start_idx = round_down(block, F2FS_I(inode)->i_cluster_size);
3765
3766	set_new_dnode(dn: &dn, inode, NULL, NULL, nid: 0);
3767	ret = f2fs_get_dnode_of_data(dn: &dn, index: start_idx, mode: LOOKUP_NODE);
3768	if (ret)
3769	return 0;
3770
3771	if (dn.data_blkaddr != COMPRESS_ADDR) {
3772	dn.ofs_in_node += block - start_idx;
3773	blknr = f2fs_data_blkaddr(dn: &dn);
3774	if (!__is_valid_data_blkaddr(blkaddr: blknr))
3775	blknr = 0;
3776	}
3777
3778	f2fs_put_dnode(dn: &dn);
3779	return blknr;
3780	#else
3781	return 0;
3782	#endif
3783	}
3784
3785
3786	static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
3787	{
3788	struct inode *inode = mapping->host;
3789	sector_t blknr = 0;
3790
3791	if (f2fs_has_inline_data(inode))
3792	goto out;
3793
3794	/* make sure allocating whole blocks */
3795	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
3796	filemap_write_and_wait(mapping);
3797
3798	/* Block number less than F2FS MAX BLOCKS */
3799	if (unlikely(block >= max_file_blocks(inode)))
3800	goto out;
3801
3802	if (f2fs_compressed_file(inode)) {
3803	blknr = f2fs_bmap_compress(inode, block);
3804	} else {
3805	struct f2fs_map_blocks map;
3806
3807	memset(&map, 0, sizeof(map));
3808	map.m_lblk = block;
3809	map.m_len = 1;
3810	map.m_next_pgofs = NULL;
3811	map.m_seg_type = NO_CHECK_TYPE;
3812
3813	if (!f2fs_map_blocks(inode, map: &map, flag: F2FS_GET_BLOCK_BMAP))
3814	blknr = map.m_pblk;
3815	}
3816	out:
3817	trace_f2fs_bmap(inode, lblock: block, pblock: blknr);
3818	return blknr;
3819	}
3820
3821	#ifdef CONFIG_SWAP
3822	static int f2fs_migrate_blocks(struct inode *inode, block_t start_blk,
3823	unsigned int blkcnt)
3824	{
3825	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3826	unsigned int blkofs;
3827	unsigned int blk_per_sec = BLKS_PER_SEC(sbi);
3828	unsigned int end_blk = start_blk + blkcnt - 1;
3829	unsigned int secidx = start_blk / blk_per_sec;
3830	unsigned int end_sec;
3831	int ret = 0;
3832
3833	if (!blkcnt)
3834	return 0;
3835	end_sec = end_blk / blk_per_sec;
3836
3837	f2fs_down_write(sem: &F2FS_I(inode)->i_gc_rwsem[WRITE]);
3838	filemap_invalidate_lock(mapping: inode->i_mapping);
3839
3840	set_inode_flag(inode, flag: FI_ALIGNED_WRITE);
3841	set_inode_flag(inode, flag: FI_OPU_WRITE);
3842
3843	for (; secidx <= end_sec; secidx++) {
3844	unsigned int blkofs_end = secidx == end_sec ?
3845	end_blk % blk_per_sec : blk_per_sec - 1;
3846
3847	f2fs_down_write(sem: &sbi->pin_sem);
3848
3849	ret = f2fs_allocate_pinning_section(sbi);
3850	if (ret) {
3851	f2fs_up_write(sem: &sbi->pin_sem);
3852	break;
3853	}
3854
3855	set_inode_flag(inode, flag: FI_SKIP_WRITES);
3856
3857	for (blkofs = 0; blkofs <= blkofs_end; blkofs++) {
3858	struct page *page;
3859	unsigned int blkidx = secidx * blk_per_sec + blkofs;
3860
3861	page = f2fs_get_lock_data_page(inode, index: blkidx, for_write: true);
3862	if (IS_ERR(ptr: page)) {
3863	f2fs_up_write(sem: &sbi->pin_sem);
3864	ret = PTR_ERR(ptr: page);
3865	goto done;
3866	}
3867
3868	set_page_dirty(page);
3869	f2fs_put_page(page, unlock: 1);
3870	}
3871
3872	clear_inode_flag(inode, flag: FI_SKIP_WRITES);
3873
3874	ret = filemap_fdatawrite(inode->i_mapping);
3875
3876	f2fs_up_write(sem: &sbi->pin_sem);
3877
3878	if (ret)
3879	break;
3880	}
3881
3882	done:
3883	clear_inode_flag(inode, flag: FI_SKIP_WRITES);
3884	clear_inode_flag(inode, flag: FI_OPU_WRITE);
3885	clear_inode_flag(inode, flag: FI_ALIGNED_WRITE);
3886
3887	filemap_invalidate_unlock(mapping: inode->i_mapping);
3888	f2fs_up_write(sem: &F2FS_I(inode)->i_gc_rwsem[WRITE]);
3889
3890	return ret;
3891	}
3892
3893	static int check_swap_activate(struct swap_info_struct *sis,
3894	struct file *swap_file, sector_t *span)
3895	{
3896	struct address_space *mapping = swap_file->f_mapping;
3897	struct inode *inode = mapping->host;
3898	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3899	sector_t cur_lblock;
3900	sector_t last_lblock;
3901	sector_t pblock;
3902	sector_t lowest_pblock = -1;
3903	sector_t highest_pblock = 0;
3904	int nr_extents = 0;
3905	unsigned long nr_pblocks;
3906	unsigned int blks_per_sec = BLKS_PER_SEC(sbi);
3907	unsigned int sec_blks_mask = BLKS_PER_SEC(sbi) - 1;
3908	unsigned int not_aligned = 0;
3909	int ret = 0;
3910
3911	/*
3912	* Map all the blocks into the extent list. This code doesn't try
3913	* to be very smart.
3914	*/
3915	cur_lblock = 0;
3916	last_lblock = bytes_to_blks(inode, bytes: i_size_read(inode));
3917
3918	while (cur_lblock < last_lblock && cur_lblock < sis->max) {
3919	struct f2fs_map_blocks map;
3920	retry:
3921	cond_resched();
3922
3923	memset(&map, 0, sizeof(map));
3924	map.m_lblk = cur_lblock;
3925	map.m_len = last_lblock - cur_lblock;
3926	map.m_next_pgofs = NULL;
3927	map.m_next_extent = NULL;
3928	map.m_seg_type = NO_CHECK_TYPE;
3929	map.m_may_create = false;
3930
3931	ret = f2fs_map_blocks(inode, map: &map, flag: F2FS_GET_BLOCK_FIEMAP);
3932	if (ret)
3933	goto out;
3934
3935	/* hole */
3936	if (!(map.m_flags & F2FS_MAP_FLAGS)) {
3937	f2fs_err(sbi, "Swapfile has holes");
3938	ret = -EINVAL;
3939	goto out;
3940	}
3941
3942	pblock = map.m_pblk;
3943	nr_pblocks = map.m_len;
3944
3945	if ((pblock - SM_I(sbi)->main_blkaddr) & sec_blks_mask ||
3946	nr_pblocks & sec_blks_mask ||
3947	!f2fs_valid_pinned_area(sbi, blkaddr: pblock)) {
3948	bool last_extent = false;
3949
3950	not_aligned++;
3951
3952	nr_pblocks = roundup(nr_pblocks, blks_per_sec);
3953	if (cur_lblock + nr_pblocks > sis->max)
3954	nr_pblocks -= blks_per_sec;
3955
3956	/* this extent is last one */
3957	if (!nr_pblocks) {
3958	nr_pblocks = last_lblock - cur_lblock;
3959	last_extent = true;
3960	}
3961
3962	ret = f2fs_migrate_blocks(inode, start_blk: cur_lblock,
3963	blkcnt: nr_pblocks);
3964	if (ret) {
3965	if (ret == -ENOENT)
3966	ret = -EINVAL;
3967	goto out;
3968	}
3969
3970	if (!last_extent)
3971	goto retry;
3972	}
3973
3974	if (cur_lblock + nr_pblocks >= sis->max)
3975	nr_pblocks = sis->max - cur_lblock;
3976
3977	if (cur_lblock) { /* exclude the header page */
3978	if (pblock < lowest_pblock)
3979	lowest_pblock = pblock;
3980	if (pblock + nr_pblocks - 1 > highest_pblock)
3981	highest_pblock = pblock + nr_pblocks - 1;
3982	}
3983
3984	/*
3985	* We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
3986	*/
3987	ret = add_swap_extent(sis, start_page: cur_lblock, nr_pages: nr_pblocks, start_block: pblock);
3988	if (ret < 0)
3989	goto out;
3990	nr_extents += ret;
3991	cur_lblock += nr_pblocks;
3992	}
3993	ret = nr_extents;
3994	*span = 1 + highest_pblock - lowest_pblock;
3995	if (cur_lblock == 0)
3996	cur_lblock = 1; /* force Empty message */
3997	sis->max = cur_lblock;
3998	sis->pages = cur_lblock - 1;
3999	sis->highest_bit = cur_lblock - 1;
4000	out:
4001	if (not_aligned)
4002	f2fs_warn(sbi, "Swapfile (%u) is not align to section: 1) creat(), 2) ioctl(F2FS_IOC_SET_PIN_FILE), 3) fallocate(%lu * N)",
4003	not_aligned, blks_per_sec * F2FS_BLKSIZE);
4004	return ret;
4005	}
4006
4007	static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file,
4008	sector_t *span)
4009	{
4010	struct inode *inode = file_inode(f: file);
4011	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4012	int ret;
4013
4014	if (!S_ISREG(inode->i_mode))
4015	return -EINVAL;
4016
4017	if (f2fs_readonly(sb: sbi->sb))
4018	return -EROFS;
4019
4020	if (f2fs_lfs_mode(sbi) && !f2fs_sb_has_blkzoned(sbi)) {
4021	f2fs_err(sbi, "Swapfile not supported in LFS mode");
4022	return -EINVAL;
4023	}
4024
4025	ret = f2fs_convert_inline_inode(inode);
4026	if (ret)
4027	return ret;
4028
4029	if (!f2fs_disable_compressed_file(inode))
4030	return -EINVAL;
4031
4032	ret = filemap_fdatawrite(inode->i_mapping);
4033	if (ret < 0)
4034	return ret;
4035
4036	f2fs_precache_extents(inode);
4037
4038	ret = check_swap_activate(sis, swap_file: file, span);
4039	if (ret < 0)
4040	return ret;
4041
4042	stat_inc_swapfile_inode(inode);
4043	set_inode_flag(inode, flag: FI_PIN_FILE);
4044	f2fs_update_time(sbi, type: REQ_TIME);
4045	return ret;
4046	}
4047
4048	static void f2fs_swap_deactivate(struct file *file)
4049	{
4050	struct inode *inode = file_inode(f: file);
4051
4052	stat_dec_swapfile_inode(inode);
4053	clear_inode_flag(inode, flag: FI_PIN_FILE);
4054	}
4055	#else
4056	static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file,
4057	sector_t *span)
4058	{
4059	return -EOPNOTSUPP;
4060	}
4061
4062	static void f2fs_swap_deactivate(struct file *file)
4063	{
4064	}
4065	#endif
4066
4067	const struct address_space_operations f2fs_dblock_aops = {
4068	.read_folio = f2fs_read_data_folio,
4069	.readahead = f2fs_readahead,
4070	.writepage = f2fs_write_data_page,
4071	.writepages = f2fs_write_data_pages,
4072	.write_begin = f2fs_write_begin,
4073	.write_end = f2fs_write_end,
4074	.dirty_folio = f2fs_dirty_data_folio,
4075	.migrate_folio = filemap_migrate_folio,
4076	.invalidate_folio = f2fs_invalidate_folio,
4077	.release_folio = f2fs_release_folio,
4078	.bmap = f2fs_bmap,
4079	.swap_activate = f2fs_swap_activate,
4080	.swap_deactivate = f2fs_swap_deactivate,
4081	};
4082
4083	void f2fs_clear_page_cache_dirty_tag(struct page *page)
4084	{
4085	struct address_space *mapping = page_mapping(page);
4086	unsigned long flags;
4087
4088	xa_lock_irqsave(&mapping->i_pages, flags);
4089	__xa_clear_mark(&mapping->i_pages, index: page_index(page),
4090	PAGECACHE_TAG_DIRTY);
4091	xa_unlock_irqrestore(&mapping->i_pages, flags);
4092	}
4093
4094	int __init f2fs_init_post_read_processing(void)
4095	{
4096	bio_post_read_ctx_cache =
4097	kmem_cache_create(name: "f2fs_bio_post_read_ctx",
4098	size: sizeof(struct bio_post_read_ctx), align: 0, flags: 0, NULL);
4099	if (!bio_post_read_ctx_cache)
4100	goto fail;
4101	bio_post_read_ctx_pool =
4102	mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,
4103	kc: bio_post_read_ctx_cache);
4104	if (!bio_post_read_ctx_pool)
4105	goto fail_free_cache;
4106	return 0;
4107
4108	fail_free_cache:
4109	kmem_cache_destroy(s: bio_post_read_ctx_cache);
4110	fail:
4111	return -ENOMEM;
4112	}
4113
4114	void f2fs_destroy_post_read_processing(void)
4115	{
4116	mempool_destroy(pool: bio_post_read_ctx_pool);
4117	kmem_cache_destroy(s: bio_post_read_ctx_cache);
4118	}
4119
4120	int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi)
4121	{
4122	if (!f2fs_sb_has_encrypt(sbi) &&
4123	!f2fs_sb_has_verity(sbi) &&
4124	!f2fs_sb_has_compression(sbi))
4125	return 0;
4126
4127	sbi->post_read_wq = alloc_workqueue(fmt: "f2fs_post_read_wq",
4128	flags: WQ_UNBOUND | WQ_HIGHPRI,
4129	max_active: num_online_cpus());
4130	return sbi->post_read_wq ? 0 : -ENOMEM;
4131	}
4132
4133	void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi)
4134	{
4135	if (sbi->post_read_wq)
4136	destroy_workqueue(wq: sbi->post_read_wq);
4137	}
4138
4139	int __init f2fs_init_bio_entry_cache(void)
4140	{
4141	bio_entry_slab = f2fs_kmem_cache_create(name: "f2fs_bio_entry_slab",
4142	size: sizeof(struct bio_entry));
4143	return bio_entry_slab ? 0 : -ENOMEM;
4144	}
4145
4146	void f2fs_destroy_bio_entry_cache(void)
4147	{
4148	kmem_cache_destroy(s: bio_entry_slab);
4149	}
4150
4151	static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
4152	unsigned int flags, struct iomap *iomap,
4153	struct iomap *srcmap)
4154	{
4155	struct f2fs_map_blocks map = {};
4156	pgoff_t next_pgofs = 0;
4157	int err;
4158
4159	map.m_lblk = bytes_to_blks(inode, bytes: offset);
4160	map.m_len = bytes_to_blks(inode, bytes: offset + length - 1) - map.m_lblk + 1;
4161	map.m_next_pgofs = &next_pgofs;
4162	map.m_seg_type = f2fs_rw_hint_to_seg_type(hint: inode->i_write_hint);
4163	if (flags & IOMAP_WRITE)
4164	map.m_may_create = true;
4165
4166	err = f2fs_map_blocks(inode, map: &map, flag: F2FS_GET_BLOCK_DIO);
4167	if (err)
4168	return err;
4169
4170	iomap->offset = blks_to_bytes(inode, blks: map.m_lblk);
4171
4172	/*
4173	* When inline encryption is enabled, sometimes I/O to an encrypted file
4174	* has to be broken up to guarantee DUN contiguity. Handle this by
4175	* limiting the length of the mapping returned.
4176	*/
4177	map.m_len = fscrypt_limit_io_blocks(inode, lblk: map.m_lblk, nr_blocks: map.m_len);
4178
4179	/*
4180	* We should never see delalloc or compressed extents here based on
4181	* prior flushing and checks.
4182	*/
4183	if (WARN_ON_ONCE(map.m_pblk == NEW_ADDR))
4184	return -EINVAL;
4185	if (WARN_ON_ONCE(map.m_pblk == COMPRESS_ADDR))
4186	return -EINVAL;
4187
4188	if (map.m_pblk != NULL_ADDR) {
4189	iomap->length = blks_to_bytes(inode, blks: map.m_len);
4190	iomap->type = IOMAP_MAPPED;
4191	iomap->flags |= IOMAP_F_MERGED;
4192	iomap->bdev = map.m_bdev;
4193	iomap->addr = blks_to_bytes(inode, blks: map.m_pblk);
4194	} else {
4195	if (flags & IOMAP_WRITE)
4196	return -ENOTBLK;
4197	iomap->length = blks_to_bytes(inode, blks: next_pgofs) -
4198	iomap->offset;
4199	iomap->type = IOMAP_HOLE;
4200	iomap->addr = IOMAP_NULL_ADDR;
4201	}
4202
4203	if (map.m_flags & F2FS_MAP_NEW)
4204	iomap->flags |= IOMAP_F_NEW;
4205	if ((inode->i_state & I_DIRTY_DATASYNC) ||
4206	offset + length > i_size_read(inode))
4207	iomap->flags |= IOMAP_F_DIRTY;
4208
4209	return 0;
4210	}
4211
4212	const struct iomap_ops f2fs_iomap_ops = {
4213	.iomap_begin = f2fs_iomap_begin,
4214	};
4215