1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* linux/fs/ext4/inode.c
4	*
5	* Copyright (C) 1992, 1993, 1994, 1995
6	* Remy Card (card@masi.ibp.fr)
7	* Laboratoire MASI - Institut Blaise Pascal
8	* Universite Pierre et Marie Curie (Paris VI)
9	*
10	* from
11	*
12	* linux/fs/minix/inode.c
13	*
14	* Copyright (C) 1991, 1992 Linus Torvalds
15	*
16	* 64-bit file support on 64-bit platforms by Jakub Jelinek
17	* (jj@sunsite.ms.mff.cuni.cz)
18	*
19	* Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000
20	*/
21
22	#include <linux/fs.h>
23	#include <linux/mount.h>
24	#include <linux/time.h>
25	#include <linux/highuid.h>
26	#include <linux/pagemap.h>
27	#include <linux/dax.h>
28	#include <linux/quotaops.h>
29	#include <linux/string.h>
30	#include <linux/buffer_head.h>
31	#include <linux/writeback.h>
32	#include <linux/pagevec.h>
33	#include <linux/mpage.h>
34	#include <linux/namei.h>
35	#include <linux/uio.h>
36	#include <linux/bio.h>
37	#include <linux/workqueue.h>
38	#include <linux/kernel.h>
39	#include <linux/printk.h>
40	#include <linux/slab.h>
41	#include <linux/bitops.h>
42	#include <linux/iomap.h>
43	#include <linux/iversion.h>
44
45	#include "ext4_jbd2.h"
46	#include "xattr.h"
47	#include "acl.h"
48	#include "truncate.h"
49
50	#include <trace/events/ext4.h>
51
52	static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
53	struct ext4_inode_info *ei)
54	{
55	struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
56	__u32 csum;
57	__u16 dummy_csum = 0;
58	int offset = offsetof(struct ext4_inode, i_checksum_lo);
59	unsigned int csum_size = sizeof(dummy_csum);
60
61	csum = ext4_chksum(sbi, crc: ei->i_csum_seed, address: (__u8 *)raw, length: offset);
62	csum = ext4_chksum(sbi, crc: csum, address: (__u8 *)&dummy_csum, length: csum_size);
63	offset += csum_size;
64	csum = ext4_chksum(sbi, crc: csum, address: (__u8 *)raw + offset,
65	EXT4_GOOD_OLD_INODE_SIZE - offset);
66
67	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
68	offset = offsetof(struct ext4_inode, i_checksum_hi);
69	csum = ext4_chksum(sbi, crc: csum, address: (__u8 *)raw +
70	EXT4_GOOD_OLD_INODE_SIZE,
71	length: offset - EXT4_GOOD_OLD_INODE_SIZE);
72	if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {
73	csum = ext4_chksum(sbi, crc: csum, address: (__u8 *)&dummy_csum,
74	length: csum_size);
75	offset += csum_size;
76	}
77	csum = ext4_chksum(sbi, crc: csum, address: (__u8 *)raw + offset,
78	EXT4_INODE_SIZE(inode->i_sb) - offset);
79	}
80
81	return csum;
82	}
83
84	static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,
85	struct ext4_inode_info *ei)
86	{
87	__u32 provided, calculated;
88
89	if (EXT4_SB(sb: inode->i_sb)->s_es->s_creator_os !=
90	cpu_to_le32(EXT4_OS_LINUX) ||
91	!ext4_has_metadata_csum(sb: inode->i_sb))
92	return 1;
93
94	provided = le16_to_cpu(raw->i_checksum_lo);
95	calculated = ext4_inode_csum(inode, raw, ei);
96	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
97	EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
98	provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;
99	else
100	calculated &= 0xFFFF;
101
102	return provided == calculated;
103	}
104
105	void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,
106	struct ext4_inode_info *ei)
107	{
108	__u32 csum;
109
110	if (EXT4_SB(sb: inode->i_sb)->s_es->s_creator_os !=
111	cpu_to_le32(EXT4_OS_LINUX) ||
112	!ext4_has_metadata_csum(sb: inode->i_sb))
113	return;
114
115	csum = ext4_inode_csum(inode, raw, ei);
116	raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);
117	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
118	EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
119	raw->i_checksum_hi = cpu_to_le16(csum >> 16);
120	}
121
122	static inline int ext4_begin_ordered_truncate(struct inode *inode,
123	loff_t new_size)
124	{
125	trace_ext4_begin_ordered_truncate(inode, new_size);
126	/*
127	* If jinode is zero, then we never opened the file for
128	* writing, so there's no need to call
129	* jbd2_journal_begin_ordered_truncate() since there's no
130	* outstanding writes we need to flush.
131	*/
132	if (!EXT4_I(inode)->jinode)
133	return 0;
134	return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
135	EXT4_I(inode)->jinode,
136	new_size);
137	}
138
139	static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
140	int pextents);
141
142	/*
143	* Test whether an inode is a fast symlink.
144	* A fast symlink has its symlink data stored in ext4_inode_info->i_data.
145	*/
146	int ext4_inode_is_fast_symlink(struct inode *inode)
147	{
148	if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
149	int ea_blocks = EXT4_I(inode)->i_file_acl ?
150	EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;
151
152	if (ext4_has_inline_data(inode))
153	return 0;
154
155	return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
156	}
157	return S_ISLNK(inode->i_mode) && inode->i_size &&
158	(inode->i_size < EXT4_N_BLOCKS * 4);
159	}
160
161	/*
162	* Called at the last iput() if i_nlink is zero.
163	*/
164	void ext4_evict_inode(struct inode *inode)
165	{
166	handle_t *handle;
167	int err;
168	/*
169	* Credits for final inode cleanup and freeing:
170	* sb + inode (ext4_orphan_del()), block bitmap, group descriptor
171	* (xattr block freeing), bitmap, group descriptor (inode freeing)
172	*/
173	int extra_credits = 6;
174	struct ext4_xattr_inode_array *ea_inode_array = NULL;
175	bool freeze_protected = false;
176
177	trace_ext4_evict_inode(inode);
178
179	if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)
180	ext4_evict_ea_inode(inode);
181	if (inode->i_nlink) {
182	truncate_inode_pages_final(&inode->i_data);
183
184	goto no_delete;
185	}
186
187	if (is_bad_inode(inode))
188	goto no_delete;
189	dquot_initialize(inode);
190
191	if (ext4_should_order_data(inode))
192	ext4_begin_ordered_truncate(inode, new_size: 0);
193	truncate_inode_pages_final(&inode->i_data);
194
195	/*
196	* For inodes with journalled data, transaction commit could have
197	* dirtied the inode. And for inodes with dioread_nolock, unwritten
198	* extents converting worker could merge extents and also have dirtied
199	* the inode. Flush worker is ignoring it because of I_FREEING flag but
200	* we still need to remove the inode from the writeback lists.
201	*/
202	if (!list_empty_careful(head: &inode->i_io_list))
203	inode_io_list_del(inode);
204
205	/*
206	* Protect us against freezing - iput() caller didn't have to have any
207	* protection against it. When we are in a running transaction though,
208	* we are already protected against freezing and we cannot grab further
209	* protection due to lock ordering constraints.
210	*/
211	if (!ext4_journal_current_handle()) {
212	sb_start_intwrite(sb: inode->i_sb);
213	freeze_protected = true;
214	}
215
216	if (!IS_NOQUOTA(inode))
217	extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb);
218
219	/*
220	* Block bitmap, group descriptor, and inode are accounted in both
221	* ext4_blocks_for_truncate() and extra_credits. So subtract 3.
222	*/
223	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,
224	ext4_blocks_for_truncate(inode) + extra_credits - 3);
225	if (IS_ERR(ptr: handle)) {
226	ext4_std_error(inode->i_sb, PTR_ERR(handle));
227	/*
228	* If we're going to skip the normal cleanup, we still need to
229	* make sure that the in-core orphan linked list is properly
230	* cleaned up.
231	*/
232	ext4_orphan_del(NULL, inode);
233	if (freeze_protected)
234	sb_end_intwrite(sb: inode->i_sb);
235	goto no_delete;
236	}
237
238	if (IS_SYNC(inode))
239	ext4_handle_sync(handle);
240
241	/*
242	* Set inode->i_size to 0 before calling ext4_truncate(). We need
243	* special handling of symlinks here because i_size is used to
244	* determine whether ext4_inode_info->i_data contains symlink data or
245	* block mappings. Setting i_size to 0 will remove its fast symlink
246	* status. Erase i_data so that it becomes a valid empty block map.
247	*/
248	if (ext4_inode_is_fast_symlink(inode))
249	memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));
250	inode->i_size = 0;
251	err = ext4_mark_inode_dirty(handle, inode);
252	if (err) {
253	ext4_warning(inode->i_sb,
254	"couldn't mark inode dirty (err %d)", err);
255	goto stop_handle;
256	}
257	if (inode->i_blocks) {
258	err = ext4_truncate(inode);
259	if (err) {
260	ext4_error_err(inode->i_sb, -err,
261	"couldn't truncate inode %lu (err %d)",
262	inode->i_ino, err);
263	goto stop_handle;
264	}
265	}
266
267	/* Remove xattr references. */
268	err = ext4_xattr_delete_inode(handle, inode, array: &ea_inode_array,
269	extra_credits);
270	if (err) {
271	ext4_warning(inode->i_sb, "xattr delete (err %d)", err);
272	stop_handle:
273	ext4_journal_stop(handle);
274	ext4_orphan_del(NULL, inode);
275	if (freeze_protected)
276	sb_end_intwrite(sb: inode->i_sb);
277	ext4_xattr_inode_array_free(array: ea_inode_array);
278	goto no_delete;
279	}
280
281	/*
282	* Kill off the orphan record which ext4_truncate created.
283	* AKPM: I think this can be inside the above `if'.
284	* Note that ext4_orphan_del() has to be able to cope with the
285	* deletion of a non-existent orphan - this is because we don't
286	* know if ext4_truncate() actually created an orphan record.
287	* (Well, we could do this if we need to, but heck - it works)
288	*/
289	ext4_orphan_del(handle, inode);
290	EXT4_I(inode)->i_dtime = (__u32)ktime_get_real_seconds();
291
292	/*
293	* One subtle ordering requirement: if anything has gone wrong
294	* (transaction abort, IO errors, whatever), then we can still
295	* do these next steps (the fs will already have been marked as
296	* having errors), but we can't free the inode if the mark_dirty
297	* fails.
298	*/
299	if (ext4_mark_inode_dirty(handle, inode))
300	/* If that failed, just do the required in-core inode clear. */
301	ext4_clear_inode(inode);
302	else
303	ext4_free_inode(handle, inode);
304	ext4_journal_stop(handle);
305	if (freeze_protected)
306	sb_end_intwrite(sb: inode->i_sb);
307	ext4_xattr_inode_array_free(array: ea_inode_array);
308	return;
309	no_delete:
310	/*
311	* Check out some where else accidentally dirty the evicting inode,
312	* which may probably cause inode use-after-free issues later.
313	*/
314	WARN_ON_ONCE(!list_empty_careful(&inode->i_io_list));
315
316	if (!list_empty(head: &EXT4_I(inode)->i_fc_list))
317	ext4_fc_mark_ineligible(sb: inode->i_sb, reason: EXT4_FC_REASON_NOMEM, NULL);
318	ext4_clear_inode(inode); /* We must guarantee clearing of inode... */
319	}
320
321	#ifdef CONFIG_QUOTA
322	qsize_t *ext4_get_reserved_space(struct inode *inode)
323	{
324	return &EXT4_I(inode)->i_reserved_quota;
325	}
326	#endif
327
328	/*
329	* Called with i_data_sem down, which is important since we can call
330	* ext4_discard_preallocations() from here.
331	*/
332	void ext4_da_update_reserve_space(struct inode *inode,
333	int used, int quota_claim)
334	{
335	struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
336	struct ext4_inode_info *ei = EXT4_I(inode);
337
338	spin_lock(lock: &ei->i_block_reservation_lock);
339	trace_ext4_da_update_reserve_space(inode, used_blocks: used, quota_claim);
340	if (unlikely(used > ei->i_reserved_data_blocks)) {
341	ext4_warning(inode->i_sb, "%s: ino %lu, used %d "
342	"with only %d reserved data blocks",
343	__func__, inode->i_ino, used,
344	ei->i_reserved_data_blocks);
345	WARN_ON(1);
346	used = ei->i_reserved_data_blocks;
347	}
348
349	/* Update per-inode reservations */
350	ei->i_reserved_data_blocks -= used;
351	percpu_counter_sub(fbc: &sbi->s_dirtyclusters_counter, amount: used);
352
353	spin_unlock(lock: &ei->i_block_reservation_lock);
354
355	/* Update quota subsystem for data blocks */
356	if (quota_claim)
357	dquot_claim_block(inode, EXT4_C2B(sbi, used));
358	else {
359	/*
360	* We did fallocate with an offset that is already delayed
361	* allocated. So on delayed allocated writeback we should
362	* not re-claim the quota for fallocated blocks.
363	*/
364	dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
365	}
366
367	/*
368	* If we have done all the pending block allocations and if
369	* there aren't any writers on the inode, we can discard the
370	* inode's preallocations.
371	*/
372	if ((ei->i_reserved_data_blocks == 0) &&
373	!inode_is_open_for_write(inode))
374	ext4_discard_preallocations(inode, 0);
375	}
376
377	static int __check_block_validity(struct inode *inode, const char *func,
378	unsigned int line,
379	struct ext4_map_blocks *map)
380	{
381	if (ext4_has_feature_journal(sb: inode->i_sb) &&
382	(inode->i_ino ==
383	le32_to_cpu(EXT4_SB(inode->i_sb)->s_es->s_journal_inum)))
384	return 0;
385	if (!ext4_inode_block_valid(inode, start_blk: map->m_pblk, count: map->m_len)) {
386	ext4_error_inode(inode, func, line, map->m_pblk,
387	"lblock %lu mapped to illegal pblock %llu "
388	"(length %d)", (unsigned long) map->m_lblk,
389	map->m_pblk, map->m_len);
390	return -EFSCORRUPTED;
391	}
392	return 0;
393	}
394
395	int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
396	ext4_lblk_t len)
397	{
398	int ret;
399
400	if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))
401	return fscrypt_zeroout_range(inode, lblk, pblk, len);
402
403	ret = sb_issue_zeroout(sb: inode->i_sb, block: pblk, nr_blocks: len, GFP_NOFS);
404	if (ret > 0)
405	ret = 0;
406
407	return ret;
408	}
409
410	#define check_block_validity(inode, map) \
411	__check_block_validity((inode), __func__, __LINE__, (map))
412
413	#ifdef ES_AGGRESSIVE_TEST
414	static void ext4_map_blocks_es_recheck(handle_t *handle,
415	struct inode *inode,
416	struct ext4_map_blocks *es_map,
417	struct ext4_map_blocks *map,
418	int flags)
419	{
420	int retval;
421
422	map->m_flags = 0;
423	/*
424	* There is a race window that the result is not the same.
425	* e.g. xfstests #223 when dioread_nolock enables. The reason
426	* is that we lookup a block mapping in extent status tree with
427	* out taking i_data_sem. So at the time the unwritten extent
428	* could be converted.
429	*/
430	down_read(&EXT4_I(inode)->i_data_sem);
431	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
432	retval = ext4_ext_map_blocks(handle, inode, map, 0);
433	} else {
434	retval = ext4_ind_map_blocks(handle, inode, map, 0);
435	}
436	up_read((&EXT4_I(inode)->i_data_sem));
437
438	/*
439	* We don't check m_len because extent will be collpased in status
440	* tree. So the m_len might not equal.
441	*/
442	if (es_map->m_lblk != map->m_lblk ||
443	es_map->m_flags != map->m_flags ||
444	es_map->m_pblk != map->m_pblk) {
445	printk("ES cache assertion failed for inode: %lu "
446	"es_cached ex [%d/%d/%llu/%x] != "
447	"found ex [%d/%d/%llu/%x] retval %d flags %x\n",
448	inode->i_ino, es_map->m_lblk, es_map->m_len,
449	es_map->m_pblk, es_map->m_flags, map->m_lblk,
450	map->m_len, map->m_pblk, map->m_flags,
451	retval, flags);
452	}
453	}
454	#endif /* ES_AGGRESSIVE_TEST */
455
456	/*
457	* The ext4_map_blocks() function tries to look up the requested blocks,
458	* and returns if the blocks are already mapped.
459	*
460	* Otherwise it takes the write lock of the i_data_sem and allocate blocks
461	* and store the allocated blocks in the result buffer head and mark it
462	* mapped.
463	*
464	* If file type is extents based, it will call ext4_ext_map_blocks(),
465	* Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping
466	* based files
467	*
468	* On success, it returns the number of blocks being mapped or allocated. if
469	* create==0 and the blocks are pre-allocated and unwritten, the resulting @map
470	* is marked as unwritten. If the create == 1, it will mark @map as mapped.
471	*
472	* It returns 0 if plain look up failed (blocks have not been allocated), in
473	* that case, @map is returned as unmapped but we still do fill map->m_len to
474	* indicate the length of a hole starting at map->m_lblk.
475	*
476	* It returns the error in case of allocation failure.
477	*/
478	int ext4_map_blocks(handle_t *handle, struct inode *inode,
479	struct ext4_map_blocks *map, int flags)
480	{
481	struct extent_status es;
482	int retval;
483	int ret = 0;
484	#ifdef ES_AGGRESSIVE_TEST
485	struct ext4_map_blocks orig_map;
486
487	memcpy(&orig_map, map, sizeof(*map));
488	#endif
489
490	map->m_flags = 0;
491	ext_debug(inode, "flag 0x%x, max_blocks %u, logical block %lu\n",
492	flags, map->m_len, (unsigned long) map->m_lblk);
493
494	/*
495	* ext4_map_blocks returns an int, and m_len is an unsigned int
496	*/
497	if (unlikely(map->m_len > INT_MAX))
498	map->m_len = INT_MAX;
499
500	/* We can handle the block number less than EXT_MAX_BLOCKS */
501	if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))
502	return -EFSCORRUPTED;
503
504	/* Lookup extent status tree firstly */
505	if (!(EXT4_SB(sb: inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) &&
506	ext4_es_lookup_extent(inode, lblk: map->m_lblk, NULL, es: &es)) {
507	if (ext4_es_is_written(es: &es) || ext4_es_is_unwritten(es: &es)) {
508	map->m_pblk = ext4_es_pblock(es: &es) +
509	map->m_lblk - es.es_lblk;
510	map->m_flags |= ext4_es_is_written(es: &es) ?
511	EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;
512	retval = es.es_len - (map->m_lblk - es.es_lblk);
513	if (retval > map->m_len)
514	retval = map->m_len;
515	map->m_len = retval;
516	} else if (ext4_es_is_delayed(es: &es) || ext4_es_is_hole(es: &es)) {
517	map->m_pblk = 0;
518	retval = es.es_len - (map->m_lblk - es.es_lblk);
519	if (retval > map->m_len)
520	retval = map->m_len;
521	map->m_len = retval;
522	retval = 0;
523	} else {
524	BUG();
525	}
526
527	if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT)
528	return retval;
529	#ifdef ES_AGGRESSIVE_TEST
530	ext4_map_blocks_es_recheck(handle, inode, map,
531	&orig_map, flags);
532	#endif
533	goto found;
534	}
535	/*
536	* In the query cache no-wait mode, nothing we can do more if we
537	* cannot find extent in the cache.
538	*/
539	if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT)
540	return 0;
541
542	/*
543	* Try to see if we can get the block without requesting a new
544	* file system block.
545	*/
546	down_read(sem: &EXT4_I(inode)->i_data_sem);
547	if (ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS)) {
548	retval = ext4_ext_map_blocks(handle, inode, map, flags: 0);
549	} else {
550	retval = ext4_ind_map_blocks(handle, inode, map, flags: 0);
551	}
552	if (retval > 0) {
553	unsigned int status;
554
555	if (unlikely(retval != map->m_len)) {
556	ext4_warning(inode->i_sb,
557	"ES len assertion failed for inode "
558	"%lu: retval %d != map->m_len %d",
559	inode->i_ino, retval, map->m_len);
560	WARN_ON(1);
561	}
562
563	status = map->m_flags & EXT4_MAP_UNWRITTEN ?
564	EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
565	if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
566	!(status & EXTENT_STATUS_WRITTEN) &&
567	ext4_es_scan_range(inode, matching_fn: &ext4_es_is_delayed, lblk: map->m_lblk,
568	end: map->m_lblk + map->m_len - 1))
569	status |= EXTENT_STATUS_DELAYED;
570	ext4_es_insert_extent(inode, lblk: map->m_lblk, len: map->m_len,
571	pblk: map->m_pblk, status);
572	}
573	up_read(sem: (&EXT4_I(inode)->i_data_sem));
574
575	found:
576	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
577	ret = check_block_validity(inode, map);
578	if (ret != 0)
579	return ret;
580	}
581
582	/* If it is only a block(s) look up */
583	if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
584	return retval;
585
586	/*
587	* Returns if the blocks have already allocated
588	*
589	* Note that if blocks have been preallocated
590	* ext4_ext_get_block() returns the create = 0
591	* with buffer head unmapped.
592	*/
593	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
594	/*
595	* If we need to convert extent to unwritten
596	* we continue and do the actual work in
597	* ext4_ext_map_blocks()
598	*/
599	if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))
600	return retval;
601
602	/*
603	* Here we clear m_flags because after allocating an new extent,
604	* it will be set again.
605	*/
606	map->m_flags &= ~EXT4_MAP_FLAGS;
607
608	/*
609	* New blocks allocate and/or writing to unwritten extent
610	* will possibly result in updating i_data, so we take
611	* the write lock of i_data_sem, and call get_block()
612	* with create == 1 flag.
613	*/
614	down_write(sem: &EXT4_I(inode)->i_data_sem);
615
616	/*
617	* We need to check for EXT4 here because migrate
618	* could have changed the inode type in between
619	*/
620	if (ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS)) {
621	retval = ext4_ext_map_blocks(handle, inode, map, flags);
622	} else {
623	retval = ext4_ind_map_blocks(handle, inode, map, flags);
624
625	if (retval > 0 && map->m_flags & EXT4_MAP_NEW) {
626	/*
627	* We allocated new blocks which will result in
628	* i_data's format changing. Force the migrate
629	* to fail by clearing migrate flags
630	*/
631	ext4_clear_inode_state(inode, bit: EXT4_STATE_EXT_MIGRATE);
632	}
633	}
634
635	if (retval > 0) {
636	unsigned int status;
637
638	if (unlikely(retval != map->m_len)) {
639	ext4_warning(inode->i_sb,
640	"ES len assertion failed for inode "
641	"%lu: retval %d != map->m_len %d",
642	inode->i_ino, retval, map->m_len);
643	WARN_ON(1);
644	}
645
646	/*
647	* We have to zeroout blocks before inserting them into extent
648	* status tree. Otherwise someone could look them up there and
649	* use them before they are really zeroed. We also have to
650	* unmap metadata before zeroing as otherwise writeback can
651	* overwrite zeros with stale data from block device.
652	*/
653	if (flags & EXT4_GET_BLOCKS_ZERO &&
654	map->m_flags & EXT4_MAP_MAPPED &&
655	map->m_flags & EXT4_MAP_NEW) {
656	ret = ext4_issue_zeroout(inode, lblk: map->m_lblk,
657	pblk: map->m_pblk, len: map->m_len);
658	if (ret) {
659	retval = ret;
660	goto out_sem;
661	}
662	}
663
664	/*
665	* If the extent has been zeroed out, we don't need to update
666	* extent status tree.
667	*/
668	if ((flags & EXT4_GET_BLOCKS_PRE_IO) &&
669	ext4_es_lookup_extent(inode, lblk: map->m_lblk, NULL, es: &es)) {
670	if (ext4_es_is_written(es: &es))
671	goto out_sem;
672	}
673	status = map->m_flags & EXT4_MAP_UNWRITTEN ?
674	EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
675	if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
676	!(status & EXTENT_STATUS_WRITTEN) &&
677	ext4_es_scan_range(inode, matching_fn: &ext4_es_is_delayed, lblk: map->m_lblk,
678	end: map->m_lblk + map->m_len - 1))
679	status |= EXTENT_STATUS_DELAYED;
680	ext4_es_insert_extent(inode, lblk: map->m_lblk, len: map->m_len,
681	pblk: map->m_pblk, status);
682	}
683
684	out_sem:
685	up_write(sem: (&EXT4_I(inode)->i_data_sem));
686	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
687	ret = check_block_validity(inode, map);
688	if (ret != 0)
689	return ret;
690
691	/*
692	* Inodes with freshly allocated blocks where contents will be
693	* visible after transaction commit must be on transaction's
694	* ordered data list.
695	*/
696	if (map->m_flags & EXT4_MAP_NEW &&
697	!(map->m_flags & EXT4_MAP_UNWRITTEN) &&
698	!(flags & EXT4_GET_BLOCKS_ZERO) &&
699	!ext4_is_quota_file(inode) &&
700	ext4_should_order_data(inode)) {
701	loff_t start_byte =
702	(loff_t)map->m_lblk << inode->i_blkbits;
703	loff_t length = (loff_t)map->m_len << inode->i_blkbits;
704
705	if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
706	ret = ext4_jbd2_inode_add_wait(handle, inode,
707	start_byte, length);
708	else
709	ret = ext4_jbd2_inode_add_write(handle, inode,
710	start_byte, length);
711	if (ret)
712	return ret;
713	}
714	}
715	if (retval > 0 && (map->m_flags & EXT4_MAP_UNWRITTEN ||
716	map->m_flags & EXT4_MAP_MAPPED))
717	ext4_fc_track_range(handle, inode, start: map->m_lblk,
718	end: map->m_lblk + map->m_len - 1);
719	if (retval < 0)
720	ext_debug(inode, "failed with err %d\n", retval);
721	return retval;
722	}
723
724	/*
725	* Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages
726	* we have to be careful as someone else may be manipulating b_state as well.
727	*/
728	static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)
729	{
730	unsigned long old_state;
731	unsigned long new_state;
732
733	flags &= EXT4_MAP_FLAGS;
734
735	/* Dummy buffer_head? Set non-atomically. */
736	if (!bh->b_page) {
737	bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;
738	return;
739	}
740	/*
741	* Someone else may be modifying b_state. Be careful! This is ugly but
742	* once we get rid of using bh as a container for mapping information
743	* to pass to / from get_block functions, this can go away.
744	*/
745	old_state = READ_ONCE(bh->b_state);
746	do {
747	new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;
748	} while (unlikely(!try_cmpxchg(&bh->b_state, &old_state, new_state)));
749	}
750
751	static int _ext4_get_block(struct inode *inode, sector_t iblock,
752	struct buffer_head *bh, int flags)
753	{
754	struct ext4_map_blocks map;
755	int ret = 0;
756
757	if (ext4_has_inline_data(inode))
758	return -ERANGE;
759
760	map.m_lblk = iblock;
761	map.m_len = bh->b_size >> inode->i_blkbits;
762
763	ret = ext4_map_blocks(handle: ext4_journal_current_handle(), inode, map: &map,
764	flags);
765	if (ret > 0) {
766	map_bh(bh, sb: inode->i_sb, block: map.m_pblk);
767	ext4_update_bh_state(bh, flags: map.m_flags);
768	bh->b_size = inode->i_sb->s_blocksize * map.m_len;
769	ret = 0;
770	} else if (ret == 0) {
771	/* hole case, need to fill in bh->b_size */
772	bh->b_size = inode->i_sb->s_blocksize * map.m_len;
773	}
774	return ret;
775	}
776
777	int ext4_get_block(struct inode *inode, sector_t iblock,
778	struct buffer_head *bh, int create)
779	{
780	return _ext4_get_block(inode, iblock, bh,
781	flags: create ? EXT4_GET_BLOCKS_CREATE : 0);
782	}
783
784	/*
785	* Get block function used when preparing for buffered write if we require
786	* creating an unwritten extent if blocks haven't been allocated. The extent
787	* will be converted to written after the IO is complete.
788	*/
789	int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
790	struct buffer_head *bh_result, int create)
791	{
792	int ret = 0;
793
794	ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n",
795	inode->i_ino, create);
796	ret = _ext4_get_block(inode, iblock, bh: bh_result,
797	EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT);
798
799	/*
800	* If the buffer is marked unwritten, mark it as new to make sure it is
801	* zeroed out correctly in case of partial writes. Otherwise, there is
802	* a chance of stale data getting exposed.
803	*/
804	if (ret == 0 && buffer_unwritten(bh: bh_result))
805	set_buffer_new(bh_result);
806
807	return ret;
808	}
809
810	/* Maximum number of blocks we map for direct IO at once. */
811	#define DIO_MAX_BLOCKS 4096
812
813	/*
814	* `handle' can be NULL if create is zero
815	*/
816	struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
817	ext4_lblk_t block, int map_flags)
818	{
819	struct ext4_map_blocks map;
820	struct buffer_head *bh;
821	int create = map_flags & EXT4_GET_BLOCKS_CREATE;
822	bool nowait = map_flags & EXT4_GET_BLOCKS_CACHED_NOWAIT;
823	int err;
824
825	ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
826	|| handle != NULL || create == 0);
827	ASSERT(create == 0 || !nowait);
828
829	map.m_lblk = block;
830	map.m_len = 1;
831	err = ext4_map_blocks(handle, inode, map: &map, flags: map_flags);
832
833	if (err == 0)
834	return create ? ERR_PTR(error: -ENOSPC) : NULL;
835	if (err < 0)
836	return ERR_PTR(error: err);
837
838	if (nowait)
839	return sb_find_get_block(sb: inode->i_sb, block: map.m_pblk);
840
841	bh = sb_getblk(sb: inode->i_sb, block: map.m_pblk);
842	if (unlikely(!bh))
843	return ERR_PTR(error: -ENOMEM);
844	if (map.m_flags & EXT4_MAP_NEW) {
845	ASSERT(create != 0);
846	ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
847	|| (handle != NULL));
848
849	/*
850	* Now that we do not always journal data, we should
851	* keep in mind whether this should always journal the
852	* new buffer as metadata. For now, regular file
853	* writes use ext4_get_block instead, so it's not a
854	* problem.
855	*/
856	lock_buffer(bh);
857	BUFFER_TRACE(bh, "call get_create_access");
858	err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
859	EXT4_JTR_NONE);
860	if (unlikely(err)) {
861	unlock_buffer(bh);
862	goto errout;
863	}
864	if (!buffer_uptodate(bh)) {
865	memset(bh->b_data, 0, inode->i_sb->s_blocksize);
866	set_buffer_uptodate(bh);
867	}
868	unlock_buffer(bh);
869	BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
870	err = ext4_handle_dirty_metadata(handle, inode, bh);
871	if (unlikely(err))
872	goto errout;
873	} else
874	BUFFER_TRACE(bh, "not a new buffer");
875	return bh;
876	errout:
877	brelse(bh);
878	return ERR_PTR(error: err);
879	}
880
881	struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
882	ext4_lblk_t block, int map_flags)
883	{
884	struct buffer_head *bh;
885	int ret;
886
887	bh = ext4_getblk(handle, inode, block, map_flags);
888	if (IS_ERR(ptr: bh))
889	return bh;
890	if (!bh || ext4_buffer_uptodate(bh))
891	return bh;
892
893	ret = ext4_read_bh_lock(bh, REQ_META | REQ_PRIO, wait: true);
894	if (ret) {
895	put_bh(bh);
896	return ERR_PTR(error: ret);
897	}
898	return bh;
899	}
900
901	/* Read a contiguous batch of blocks. */
902	int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count,
903	bool wait, struct buffer_head **bhs)
904	{
905	int i, err;
906
907	for (i = 0; i < bh_count; i++) {
908	bhs[i] = ext4_getblk(NULL, inode, block: block + i, map_flags: 0 /* map_flags */);
909	if (IS_ERR(ptr: bhs[i])) {
910	err = PTR_ERR(ptr: bhs[i]);
911	bh_count = i;
912	goto out_brelse;
913	}
914	}
915
916	for (i = 0; i < bh_count; i++)
917	/* Note that NULL bhs[i] is valid because of holes. */
918	if (bhs[i] && !ext4_buffer_uptodate(bh: bhs[i]))
919	ext4_read_bh_lock(bh: bhs[i], REQ_META | REQ_PRIO, wait: false);
920
921	if (!wait)
922	return 0;
923
924	for (i = 0; i < bh_count; i++)
925	if (bhs[i])
926	wait_on_buffer(bh: bhs[i]);
927
928	for (i = 0; i < bh_count; i++) {
929	if (bhs[i] && !buffer_uptodate(bh: bhs[i])) {
930	err = -EIO;
931	goto out_brelse;
932	}
933	}
934	return 0;
935
936	out_brelse:
937	for (i = 0; i < bh_count; i++) {
938	brelse(bh: bhs[i]);
939	bhs[i] = NULL;
940	}
941	return err;
942	}
943
944	int ext4_walk_page_buffers(handle_t *handle, struct inode *inode,
945	struct buffer_head *head,
946	unsigned from,
947	unsigned to,
948	int *partial,
949	int (*fn)(handle_t *handle, struct inode *inode,
950	struct buffer_head *bh))
951	{
952	struct buffer_head *bh;
953	unsigned block_start, block_end;
954	unsigned blocksize = head->b_size;
955	int err, ret = 0;
956	struct buffer_head *next;
957
958	for (bh = head, block_start = 0;
959	ret == 0 && (bh != head || !block_start);
960	block_start = block_end, bh = next) {
961	next = bh->b_this_page;
962	block_end = block_start + blocksize;
963	if (block_end <= from || block_start >= to) {
964	if (partial && !buffer_uptodate(bh))
965	*partial = 1;
966	continue;
967	}
968	err = (*fn)(handle, inode, bh);
969	if (!ret)
970	ret = err;
971	}
972	return ret;
973	}
974
975	/*
976	* Helper for handling dirtying of journalled data. We also mark the folio as
977	* dirty so that writeback code knows about this page (and inode) contains
978	* dirty data. ext4_writepages() then commits appropriate transaction to
979	* make data stable.
980	*/
981	static int ext4_dirty_journalled_data(handle_t *handle, struct buffer_head *bh)
982	{
983	folio_mark_dirty(folio: bh->b_folio);
984	return ext4_handle_dirty_metadata(handle, NULL, bh);
985	}
986
987	int do_journal_get_write_access(handle_t *handle, struct inode *inode,
988	struct buffer_head *bh)
989	{
990	int dirty = buffer_dirty(bh);
991	int ret;
992
993	if (!buffer_mapped(bh) || buffer_freed(bh))
994	return 0;
995	/*
996	* __block_write_begin() could have dirtied some buffers. Clean
997	* the dirty bit as jbd2_journal_get_write_access() could complain
998	* otherwise about fs integrity issues. Setting of the dirty bit
999	* by __block_write_begin() isn't a real problem here as we clear
1000	* the bit before releasing a page lock and thus writeback cannot
1001	* ever write the buffer.
1002	*/
1003	if (dirty)
1004	clear_buffer_dirty(bh);
1005	BUFFER_TRACE(bh, "get write access");
1006	ret = ext4_journal_get_write_access(handle, inode->i_sb, bh,
1007	EXT4_JTR_NONE);
1008	if (!ret && dirty)
1009	ret = ext4_dirty_journalled_data(handle, bh);
1010	return ret;
1011	}
1012
1013	#ifdef CONFIG_FS_ENCRYPTION
1014	static int ext4_block_write_begin(struct folio *folio, loff_t pos, unsigned len,
1015	get_block_t *get_block)
1016	{
1017	unsigned from = pos & (PAGE_SIZE - 1);
1018	unsigned to = from + len;
1019	struct inode *inode = folio->mapping->host;
1020	unsigned block_start, block_end;
1021	sector_t block;
1022	int err = 0;
1023	unsigned blocksize = inode->i_sb->s_blocksize;
1024	unsigned bbits;
1025	struct buffer_head *bh, *head, *wait[2];
1026	int nr_wait = 0;
1027	int i;
1028
1029	BUG_ON(!folio_test_locked(folio));
1030	BUG_ON(from > PAGE_SIZE);
1031	BUG_ON(to > PAGE_SIZE);
1032	BUG_ON(from > to);
1033
1034	head = folio_buffers(folio);
1035	if (!head)
1036	head = create_empty_buffers(folio, blocksize, b_state: 0);
1037	bbits = ilog2(blocksize);
1038	block = (sector_t)folio->index << (PAGE_SHIFT - bbits);
1039
1040	for (bh = head, block_start = 0; bh != head || !block_start;
1041	block++, block_start = block_end, bh = bh->b_this_page) {
1042	block_end = block_start + blocksize;
1043	if (block_end <= from || block_start >= to) {
1044	if (folio_test_uptodate(folio)) {
1045	set_buffer_uptodate(bh);
1046	}
1047	continue;
1048	}
1049	if (buffer_new(bh))
1050	clear_buffer_new(bh);
1051	if (!buffer_mapped(bh)) {
1052	WARN_ON(bh->b_size != blocksize);
1053	err = get_block(inode, block, bh, 1);
1054	if (err)
1055	break;
1056	if (buffer_new(bh)) {
1057	if (folio_test_uptodate(folio)) {
1058	clear_buffer_new(bh);
1059	set_buffer_uptodate(bh);
1060	mark_buffer_dirty(bh);
1061	continue;
1062	}
1063	if (block_end > to || block_start < from)
1064	folio_zero_segments(folio, start1: to,
1065	xend1: block_end,
1066	start2: block_start, xend2: from);
1067	continue;
1068	}
1069	}
1070	if (folio_test_uptodate(folio)) {
1071	set_buffer_uptodate(bh);
1072	continue;
1073	}
1074	if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1075	!buffer_unwritten(bh) &&
1076	(block_start < from || block_end > to)) {
1077	ext4_read_bh_lock(bh, op_flags: 0, wait: false);
1078	wait[nr_wait++] = bh;
1079	}
1080	}
1081	/*
1082	* If we issued read requests, let them complete.
1083	*/
1084	for (i = 0; i < nr_wait; i++) {
1085	wait_on_buffer(bh: wait[i]);
1086	if (!buffer_uptodate(bh: wait[i]))
1087	err = -EIO;
1088	}
1089	if (unlikely(err)) {
1090	folio_zero_new_buffers(folio, from, to);
1091	} else if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
1092	for (i = 0; i < nr_wait; i++) {
1093	int err2;
1094
1095	err2 = fscrypt_decrypt_pagecache_blocks(folio,
1096	len: blocksize, offs: bh_offset(bh: wait[i]));
1097	if (err2) {
1098	clear_buffer_uptodate(bh: wait[i]);
1099	err = err2;
1100	}
1101	}
1102	}
1103
1104	return err;
1105	}
1106	#endif
1107
1108	/*
1109	* To preserve ordering, it is essential that the hole instantiation and
1110	* the data write be encapsulated in a single transaction. We cannot
1111	* close off a transaction and start a new one between the ext4_get_block()
1112	* and the ext4_write_end(). So doing the jbd2_journal_start at the start of
1113	* ext4_write_begin() is the right place.
1114	*/
1115	static int ext4_write_begin(struct file *file, struct address_space *mapping,
1116	loff_t pos, unsigned len,
1117	struct page **pagep, void **fsdata)
1118	{
1119	struct inode *inode = mapping->host;
1120	int ret, needed_blocks;
1121	handle_t *handle;
1122	int retries = 0;
1123	struct folio *folio;
1124	pgoff_t index;
1125	unsigned from, to;
1126
1127	if (unlikely(ext4_forced_shutdown(inode->i_sb)))
1128	return -EIO;
1129
1130	trace_ext4_write_begin(inode, pos, len);
1131	/*
1132	* Reserve one block more for addition to orphan list in case
1133	* we allocate blocks but write fails for some reason
1134	*/
1135	needed_blocks = ext4_writepage_trans_blocks(inode) + 1;
1136	index = pos >> PAGE_SHIFT;
1137	from = pos & (PAGE_SIZE - 1);
1138	to = from + len;
1139
1140	if (ext4_test_inode_state(inode, bit: EXT4_STATE_MAY_INLINE_DATA)) {
1141	ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,
1142	pagep);
1143	if (ret < 0)
1144	return ret;
1145	if (ret == 1)
1146	return 0;
1147	}
1148
1149	/*
1150	* __filemap_get_folio() can take a long time if the
1151	* system is thrashing due to memory pressure, or if the folio
1152	* is being written back. So grab it first before we start
1153	* the transaction handle. This also allows us to allocate
1154	* the folio (if needed) without using GFP_NOFS.
1155	*/
1156	retry_grab:
1157	folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
1158	gfp: mapping_gfp_mask(mapping));
1159	if (IS_ERR(ptr: folio))
1160	return PTR_ERR(ptr: folio);
1161	/*
1162	* The same as page allocation, we prealloc buffer heads before
1163	* starting the handle.
1164	*/
1165	if (!folio_buffers(folio))
1166	create_empty_buffers(folio, blocksize: inode->i_sb->s_blocksize, b_state: 0);
1167
1168	folio_unlock(folio);
1169
1170	retry_journal:
1171	handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
1172	if (IS_ERR(ptr: handle)) {
1173	folio_put(folio);
1174	return PTR_ERR(ptr: handle);
1175	}
1176
1177	folio_lock(folio);
1178	if (folio->mapping != mapping) {
1179	/* The folio got truncated from under us */
1180	folio_unlock(folio);
1181	folio_put(folio);
1182	ext4_journal_stop(handle);
1183	goto retry_grab;
1184	}
1185	/* In case writeback began while the folio was unlocked */
1186	folio_wait_stable(folio);
1187
1188	#ifdef CONFIG_FS_ENCRYPTION
1189	if (ext4_should_dioread_nolock(inode))
1190	ret = ext4_block_write_begin(folio, pos, len,
1191	get_block: ext4_get_block_unwritten);
1192	else
1193	ret = ext4_block_write_begin(folio, pos, len, get_block: ext4_get_block);
1194	#else
1195	if (ext4_should_dioread_nolock(inode))
1196	ret = __block_write_begin(&folio->page, pos, len,
1197	ext4_get_block_unwritten);
1198	else
1199	ret = __block_write_begin(&folio->page, pos, len, ext4_get_block);
1200	#endif
1201	if (!ret && ext4_should_journal_data(inode)) {
1202	ret = ext4_walk_page_buffers(handle, inode,
1203	folio_buffers(folio), from, to,
1204	NULL, fn: do_journal_get_write_access);
1205	}
1206
1207	if (ret) {
1208	bool extended = (pos + len > inode->i_size) &&
1209	!ext4_verity_in_progress(inode);
1210
1211	folio_unlock(folio);
1212	/*
1213	* __block_write_begin may have instantiated a few blocks
1214	* outside i_size. Trim these off again. Don't need
1215	* i_size_read because we hold i_rwsem.
1216	*
1217	* Add inode to orphan list in case we crash before
1218	* truncate finishes
1219	*/
1220	if (extended && ext4_can_truncate(inode))
1221	ext4_orphan_add(handle, inode);
1222
1223	ext4_journal_stop(handle);
1224	if (extended) {
1225	ext4_truncate_failed_write(inode);
1226	/*
1227	* If truncate failed early the inode might
1228	* still be on the orphan list; we need to
1229	* make sure the inode is removed from the
1230	* orphan list in that case.
1231	*/
1232	if (inode->i_nlink)
1233	ext4_orphan_del(NULL, inode);
1234	}
1235
1236	if (ret == -ENOSPC &&
1237	ext4_should_retry_alloc(sb: inode->i_sb, retries: &retries))
1238	goto retry_journal;
1239	folio_put(folio);
1240	return ret;
1241	}
1242	*pagep = &folio->page;
1243	return ret;
1244	}
1245
1246	/* For write_end() in data=journal mode */
1247	static int write_end_fn(handle_t *handle, struct inode *inode,
1248	struct buffer_head *bh)
1249	{
1250	int ret;
1251	if (!buffer_mapped(bh) || buffer_freed(bh))
1252	return 0;
1253	set_buffer_uptodate(bh);
1254	ret = ext4_dirty_journalled_data(handle, bh);
1255	clear_buffer_meta(bh);
1256	clear_buffer_prio(bh);
1257	return ret;
1258	}
1259
1260	/*
1261	* We need to pick up the new inode size which generic_commit_write gave us
1262	* `file' can be NULL - eg, when called from page_symlink().
1263	*
1264	* ext4 never places buffers on inode->i_mapping->private_list. metadata
1265	* buffers are managed internally.
1266	*/
1267	static int ext4_write_end(struct file *file,
1268	struct address_space *mapping,
1269	loff_t pos, unsigned len, unsigned copied,
1270	struct page *page, void *fsdata)
1271	{
1272	struct folio *folio = page_folio(page);
1273	handle_t *handle = ext4_journal_current_handle();
1274	struct inode *inode = mapping->host;
1275	loff_t old_size = inode->i_size;
1276	int ret = 0, ret2;
1277	int i_size_changed = 0;
1278	bool verity = ext4_verity_in_progress(inode);
1279
1280	trace_ext4_write_end(inode, pos, len, copied);
1281
1282	if (ext4_has_inline_data(inode) &&
1283	ext4_test_inode_state(inode, bit: EXT4_STATE_MAY_INLINE_DATA))
1284	return ext4_write_inline_data_end(inode, pos, len, copied,
1285	folio);
1286
1287	copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
1288	/*
1289	* it's important to update i_size while still holding folio lock:
1290	* page writeout could otherwise come in and zero beyond i_size.
1291	*
1292	* If FS_IOC_ENABLE_VERITY is running on this inode, then Merkle tree
1293	* blocks are being written past EOF, so skip the i_size update.
1294	*/
1295	if (!verity)
1296	i_size_changed = ext4_update_inode_size(inode, newsize: pos + copied);
1297	folio_unlock(folio);
1298	folio_put(folio);
1299
1300	if (old_size < pos && !verity)
1301	pagecache_isize_extended(inode, from: old_size, to: pos);
1302	/*
1303	* Don't mark the inode dirty under folio lock. First, it unnecessarily
1304	* makes the holding time of folio lock longer. Second, it forces lock
1305	* ordering of folio lock and transaction start for journaling
1306	* filesystems.
1307	*/
1308	if (i_size_changed)
1309	ret = ext4_mark_inode_dirty(handle, inode);
1310
1311	if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
1312	/* if we have allocated more blocks and copied
1313	* less. We will have blocks allocated outside
1314	* inode->i_size. So truncate them
1315	*/
1316	ext4_orphan_add(handle, inode);
1317
1318	ret2 = ext4_journal_stop(handle);
1319	if (!ret)
1320	ret = ret2;
1321
1322	if (pos + len > inode->i_size && !verity) {
1323	ext4_truncate_failed_write(inode);
1324	/*
1325	* If truncate failed early the inode might still be
1326	* on the orphan list; we need to make sure the inode
1327	* is removed from the orphan list in that case.
1328	*/
1329	if (inode->i_nlink)
1330	ext4_orphan_del(NULL, inode);
1331	}
1332
1333	return ret ? ret : copied;
1334	}
1335
1336	/*
1337	* This is a private version of folio_zero_new_buffers() which doesn't
1338	* set the buffer to be dirty, since in data=journalled mode we need
1339	* to call ext4_dirty_journalled_data() instead.
1340	*/
1341	static void ext4_journalled_zero_new_buffers(handle_t *handle,
1342	struct inode *inode,
1343	struct folio *folio,
1344	unsigned from, unsigned to)
1345	{
1346	unsigned int block_start = 0, block_end;
1347	struct buffer_head *head, *bh;
1348
1349	bh = head = folio_buffers(folio);
1350	do {
1351	block_end = block_start + bh->b_size;
1352	if (buffer_new(bh)) {
1353	if (block_end > from && block_start < to) {
1354	if (!folio_test_uptodate(folio)) {
1355	unsigned start, size;
1356
1357	start = max(from, block_start);
1358	size = min(to, block_end) - start;
1359
1360	folio_zero_range(folio, start, length: size);
1361	write_end_fn(handle, inode, bh);
1362	}
1363	clear_buffer_new(bh);
1364	}
1365	}
1366	block_start = block_end;
1367	bh = bh->b_this_page;
1368	} while (bh != head);
1369	}
1370
1371	static int ext4_journalled_write_end(struct file *file,
1372	struct address_space *mapping,
1373	loff_t pos, unsigned len, unsigned copied,
1374	struct page *page, void *fsdata)
1375	{
1376	struct folio *folio = page_folio(page);
1377	handle_t *handle = ext4_journal_current_handle();
1378	struct inode *inode = mapping->host;
1379	loff_t old_size = inode->i_size;
1380	int ret = 0, ret2;
1381	int partial = 0;
1382	unsigned from, to;
1383	int size_changed = 0;
1384	bool verity = ext4_verity_in_progress(inode);
1385
1386	trace_ext4_journalled_write_end(inode, pos, len, copied);
1387	from = pos & (PAGE_SIZE - 1);
1388	to = from + len;
1389
1390	BUG_ON(!ext4_handle_valid(handle));
1391
1392	if (ext4_has_inline_data(inode))
1393	return ext4_write_inline_data_end(inode, pos, len, copied,
1394	folio);
1395
1396	if (unlikely(copied < len) && !folio_test_uptodate(folio)) {
1397	copied = 0;
1398	ext4_journalled_zero_new_buffers(handle, inode, folio,
1399	from, to);
1400	} else {
1401	if (unlikely(copied < len))
1402	ext4_journalled_zero_new_buffers(handle, inode, folio,
1403	from: from + copied, to);
1404	ret = ext4_walk_page_buffers(handle, inode,
1405	folio_buffers(folio),
1406	from, to: from + copied, partial: &partial,
1407	fn: write_end_fn);
1408	if (!partial)
1409	folio_mark_uptodate(folio);
1410	}
1411	if (!verity)
1412	size_changed = ext4_update_inode_size(inode, newsize: pos + copied);
1413	EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1414	folio_unlock(folio);
1415	folio_put(folio);
1416
1417	if (old_size < pos && !verity)
1418	pagecache_isize_extended(inode, from: old_size, to: pos);
1419
1420	if (size_changed) {
1421	ret2 = ext4_mark_inode_dirty(handle, inode);
1422	if (!ret)
1423	ret = ret2;
1424	}
1425
1426	if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
1427	/* if we have allocated more blocks and copied
1428	* less. We will have blocks allocated outside
1429	* inode->i_size. So truncate them
1430	*/
1431	ext4_orphan_add(handle, inode);
1432
1433	ret2 = ext4_journal_stop(handle);
1434	if (!ret)
1435	ret = ret2;
1436	if (pos + len > inode->i_size && !verity) {
1437	ext4_truncate_failed_write(inode);
1438	/*
1439	* If truncate failed early the inode might still be
1440	* on the orphan list; we need to make sure the inode
1441	* is removed from the orphan list in that case.
1442	*/
1443	if (inode->i_nlink)
1444	ext4_orphan_del(NULL, inode);
1445	}
1446
1447	return ret ? ret : copied;
1448	}
1449
1450	/*
1451	* Reserve space for a single cluster
1452	*/
1453	static int ext4_da_reserve_space(struct inode *inode)
1454	{
1455	struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
1456	struct ext4_inode_info *ei = EXT4_I(inode);
1457	int ret;
1458
1459	/*
1460	* We will charge metadata quota at writeout time; this saves
1461	* us from metadata over-estimation, though we may go over by
1462	* a small amount in the end. Here we just reserve for data.
1463	*/
1464	ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1));
1465	if (ret)
1466	return ret;
1467
1468	spin_lock(lock: &ei->i_block_reservation_lock);
1469	if (ext4_claim_free_clusters(sbi, nclusters: 1, flags: 0)) {
1470	spin_unlock(lock: &ei->i_block_reservation_lock);
1471	dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1));
1472	return -ENOSPC;
1473	}
1474	ei->i_reserved_data_blocks++;
1475	trace_ext4_da_reserve_space(inode);
1476	spin_unlock(lock: &ei->i_block_reservation_lock);
1477
1478	return 0; /* success */
1479	}
1480
1481	void ext4_da_release_space(struct inode *inode, int to_free)
1482	{
1483	struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
1484	struct ext4_inode_info *ei = EXT4_I(inode);
1485
1486	if (!to_free)
1487	return; /* Nothing to release, exit */
1488
1489	spin_lock(lock: &EXT4_I(inode)->i_block_reservation_lock);
1490
1491	trace_ext4_da_release_space(inode, freed_blocks: to_free);
1492	if (unlikely(to_free > ei->i_reserved_data_blocks)) {
1493	/*
1494	* if there aren't enough reserved blocks, then the
1495	* counter is messed up somewhere. Since this
1496	* function is called from invalidate page, it's
1497	* harmless to return without any action.
1498	*/
1499	ext4_warning(inode->i_sb, "ext4_da_release_space: "
1500	"ino %lu, to_free %d with only %d reserved "
1501	"data blocks", inode->i_ino, to_free,
1502	ei->i_reserved_data_blocks);
1503	WARN_ON(1);
1504	to_free = ei->i_reserved_data_blocks;
1505	}
1506	ei->i_reserved_data_blocks -= to_free;
1507
1508	/* update fs dirty data blocks counter */
1509	percpu_counter_sub(fbc: &sbi->s_dirtyclusters_counter, amount: to_free);
1510
1511	spin_unlock(lock: &EXT4_I(inode)->i_block_reservation_lock);
1512
1513	dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
1514	}
1515
1516	/*
1517	* Delayed allocation stuff
1518	*/
1519
1520	struct mpage_da_data {
1521	/* These are input fields for ext4_do_writepages() */
1522	struct inode *inode;
1523	struct writeback_control *wbc;
1524	unsigned int can_map:1; /* Can writepages call map blocks? */
1525
1526	/* These are internal state of ext4_do_writepages() */
1527	pgoff_t first_page; /* The first page to write */
1528	pgoff_t next_page; /* Current page to examine */
1529	pgoff_t last_page; /* Last page to examine */
1530	/*
1531	* Extent to map - this can be after first_page because that can be
1532	* fully mapped. We somewhat abuse m_flags to store whether the extent
1533	* is delalloc or unwritten.
1534	*/
1535	struct ext4_map_blocks map;
1536	struct ext4_io_submit io_submit; /* IO submission data */
1537	unsigned int do_map:1;
1538	unsigned int scanned_until_end:1;
1539	unsigned int journalled_more_data:1;
1540	};
1541
1542	static void mpage_release_unused_pages(struct mpage_da_data *mpd,
1543	bool invalidate)
1544	{
1545	unsigned nr, i;
1546	pgoff_t index, end;
1547	struct folio_batch fbatch;
1548	struct inode *inode = mpd->inode;
1549	struct address_space *mapping = inode->i_mapping;
1550
1551	/* This is necessary when next_page == 0. */
1552	if (mpd->first_page >= mpd->next_page)
1553	return;
1554
1555	mpd->scanned_until_end = 0;
1556	index = mpd->first_page;
1557	end = mpd->next_page - 1;
1558	if (invalidate) {
1559	ext4_lblk_t start, last;
1560	start = index << (PAGE_SHIFT - inode->i_blkbits);
1561	last = end << (PAGE_SHIFT - inode->i_blkbits);
1562
1563	/*
1564	* avoid racing with extent status tree scans made by
1565	* ext4_insert_delayed_block()
1566	*/
1567	down_write(sem: &EXT4_I(inode)->i_data_sem);
1568	ext4_es_remove_extent(inode, lblk: start, len: last - start + 1);
1569	up_write(sem: &EXT4_I(inode)->i_data_sem);
1570	}
1571
1572	folio_batch_init(fbatch: &fbatch);
1573	while (index <= end) {
1574	nr = filemap_get_folios(mapping, start: &index, end, fbatch: &fbatch);
1575	if (nr == 0)
1576	break;
1577	for (i = 0; i < nr; i++) {
1578	struct folio *folio = fbatch.folios[i];
1579
1580	if (folio->index < mpd->first_page)
1581	continue;
1582	if (folio_next_index(folio) - 1 > end)
1583	continue;
1584	BUG_ON(!folio_test_locked(folio));
1585	BUG_ON(folio_test_writeback(folio));
1586	if (invalidate) {
1587	if (folio_mapped(folio))
1588	folio_clear_dirty_for_io(folio);
1589	block_invalidate_folio(folio, offset: 0,
1590	length: folio_size(folio));
1591	folio_clear_uptodate(folio);
1592	}
1593	folio_unlock(folio);
1594	}
1595	folio_batch_release(fbatch: &fbatch);
1596	}
1597	}
1598
1599	static void ext4_print_free_blocks(struct inode *inode)
1600	{
1601	struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
1602	struct super_block *sb = inode->i_sb;
1603	struct ext4_inode_info *ei = EXT4_I(inode);
1604
1605	ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
1606	EXT4_C2B(EXT4_SB(inode->i_sb),
1607	ext4_count_free_clusters(sb)));
1608	ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
1609	ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
1610	(long long) EXT4_C2B(EXT4_SB(sb),
1611	percpu_counter_sum(&sbi->s_freeclusters_counter)));
1612	ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
1613	(long long) EXT4_C2B(EXT4_SB(sb),
1614	percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
1615	ext4_msg(sb, KERN_CRIT, "Block reservation details");
1616	ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
1617	ei->i_reserved_data_blocks);
1618	return;
1619	}
1620
1621	/*
1622	* ext4_insert_delayed_block - adds a delayed block to the extents status
1623	* tree, incrementing the reserved cluster/block
1624	* count or making a pending reservation
1625	* where needed
1626	*
1627	* @inode - file containing the newly added block
1628	* @lblk - logical block to be added
1629	*
1630	* Returns 0 on success, negative error code on failure.
1631	*/
1632	static int ext4_insert_delayed_block(struct inode *inode, ext4_lblk_t lblk)
1633	{
1634	struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
1635	int ret;
1636	bool allocated = false;
1637
1638	/*
1639	* If the cluster containing lblk is shared with a delayed,
1640	* written, or unwritten extent in a bigalloc file system, it's
1641	* already been accounted for and does not need to be reserved.
1642	* A pending reservation must be made for the cluster if it's
1643	* shared with a written or unwritten extent and doesn't already
1644	* have one. Written and unwritten extents can be purged from the
1645	* extents status tree if the system is under memory pressure, so
1646	* it's necessary to examine the extent tree if a search of the
1647	* extents status tree doesn't get a match.
1648	*/
1649	if (sbi->s_cluster_ratio == 1) {
1650	ret = ext4_da_reserve_space(inode);
1651	if (ret != 0) /* ENOSPC */
1652	return ret;
1653	} else { /* bigalloc */
1654	if (!ext4_es_scan_clu(inode, matching_fn: &ext4_es_is_delonly, lblk)) {
1655	if (!ext4_es_scan_clu(inode,
1656	matching_fn: &ext4_es_is_mapped, lblk)) {
1657	ret = ext4_clu_mapped(inode,
1658	EXT4_B2C(sbi, lblk));
1659	if (ret < 0)
1660	return ret;
1661	if (ret == 0) {
1662	ret = ext4_da_reserve_space(inode);
1663	if (ret != 0) /* ENOSPC */
1664	return ret;
1665	} else {
1666	allocated = true;
1667	}
1668	} else {
1669	allocated = true;
1670	}
1671	}
1672	}
1673
1674	ext4_es_insert_delayed_block(inode, lblk, allocated);
1675	return 0;
1676	}
1677
1678	/*
1679	* This function is grabs code from the very beginning of
1680	* ext4_map_blocks, but assumes that the caller is from delayed write
1681	* time. This function looks up the requested blocks and sets the
1682	* buffer delay bit under the protection of i_data_sem.
1683	*/
1684	static int ext4_da_map_blocks(struct inode *inode, sector_t iblock,
1685	struct ext4_map_blocks *map,
1686	struct buffer_head *bh)
1687	{
1688	struct extent_status es;
1689	int retval;
1690	sector_t invalid_block = ~((sector_t) 0xffff);
1691	#ifdef ES_AGGRESSIVE_TEST
1692	struct ext4_map_blocks orig_map;
1693
1694	memcpy(&orig_map, map, sizeof(*map));
1695	#endif
1696
1697	if (invalid_block < ext4_blocks_count(es: EXT4_SB(sb: inode->i_sb)->s_es))
1698	invalid_block = ~0;
1699
1700	map->m_flags = 0;
1701	ext_debug(inode, "max_blocks %u, logical block %lu\n", map->m_len,
1702	(unsigned long) map->m_lblk);
1703
1704	/* Lookup extent status tree firstly */
1705	if (ext4_es_lookup_extent(inode, lblk: iblock, NULL, es: &es)) {
1706	if (ext4_es_is_hole(es: &es)) {
1707	retval = 0;
1708	down_read(sem: &EXT4_I(inode)->i_data_sem);
1709	goto add_delayed;
1710	}
1711
1712	/*
1713	* Delayed extent could be allocated by fallocate.
1714	* So we need to check it.
1715	*/
1716	if (ext4_es_is_delayed(es: &es) && !ext4_es_is_unwritten(es: &es)) {
1717	map_bh(bh, sb: inode->i_sb, block: invalid_block);
1718	set_buffer_new(bh);
1719	set_buffer_delay(bh);
1720	return 0;
1721	}
1722
1723	map->m_pblk = ext4_es_pblock(es: &es) + iblock - es.es_lblk;
1724	retval = es.es_len - (iblock - es.es_lblk);
1725	if (retval > map->m_len)
1726	retval = map->m_len;
1727	map->m_len = retval;
1728	if (ext4_es_is_written(es: &es))
1729	map->m_flags |= EXT4_MAP_MAPPED;
1730	else if (ext4_es_is_unwritten(es: &es))
1731	map->m_flags |= EXT4_MAP_UNWRITTEN;
1732	else
1733	BUG();
1734
1735	#ifdef ES_AGGRESSIVE_TEST
1736	ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);
1737	#endif
1738	return retval;
1739	}
1740
1741	/*
1742	* Try to see if we can get the block without requesting a new
1743	* file system block.
1744	*/
1745	down_read(sem: &EXT4_I(inode)->i_data_sem);
1746	if (ext4_has_inline_data(inode))
1747	retval = 0;
1748	else if (ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))
1749	retval = ext4_ext_map_blocks(NULL, inode, map, flags: 0);
1750	else
1751	retval = ext4_ind_map_blocks(NULL, inode, map, flags: 0);
1752
1753	add_delayed:
1754	if (retval == 0) {
1755	int ret;
1756
1757	/*
1758	* XXX: __block_prepare_write() unmaps passed block,
1759	* is it OK?
1760	*/
1761
1762	ret = ext4_insert_delayed_block(inode, lblk: map->m_lblk);
1763	if (ret != 0) {
1764	retval = ret;
1765	goto out_unlock;
1766	}
1767
1768	map_bh(bh, sb: inode->i_sb, block: invalid_block);
1769	set_buffer_new(bh);
1770	set_buffer_delay(bh);
1771	} else if (retval > 0) {
1772	unsigned int status;
1773
1774	if (unlikely(retval != map->m_len)) {
1775	ext4_warning(inode->i_sb,
1776	"ES len assertion failed for inode "
1777	"%lu: retval %d != map->m_len %d",
1778	inode->i_ino, retval, map->m_len);
1779	WARN_ON(1);
1780	}
1781
1782	status = map->m_flags & EXT4_MAP_UNWRITTEN ?
1783	EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
1784	ext4_es_insert_extent(inode, lblk: map->m_lblk, len: map->m_len,
1785	pblk: map->m_pblk, status);
1786	}
1787
1788	out_unlock:
1789	up_read(sem: (&EXT4_I(inode)->i_data_sem));
1790
1791	return retval;
1792	}
1793
1794	/*
1795	* This is a special get_block_t callback which is used by
1796	* ext4_da_write_begin(). It will either return mapped block or
1797	* reserve space for a single block.
1798	*
1799	* For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.
1800	* We also have b_blocknr = -1 and b_bdev initialized properly
1801	*
1802	* For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.
1803	* We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev
1804	* initialized properly.
1805	*/
1806	int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
1807	struct buffer_head *bh, int create)
1808	{
1809	struct ext4_map_blocks map;
1810	int ret = 0;
1811
1812	BUG_ON(create == 0);
1813	BUG_ON(bh->b_size != inode->i_sb->s_blocksize);
1814
1815	map.m_lblk = iblock;
1816	map.m_len = 1;
1817
1818	/*
1819	* first, we need to know whether the block is allocated already
1820	* preallocated blocks are unmapped but should treated
1821	* the same as allocated blocks.
1822	*/
1823	ret = ext4_da_map_blocks(inode, iblock, map: &map, bh);
1824	if (ret <= 0)
1825	return ret;
1826
1827	map_bh(bh, sb: inode->i_sb, block: map.m_pblk);
1828	ext4_update_bh_state(bh, flags: map.m_flags);
1829
1830	if (buffer_unwritten(bh)) {
1831	/* A delayed write to unwritten bh should be marked
1832	* new and mapped. Mapped ensures that we don't do
1833	* get_block multiple times when we write to the same
1834	* offset and new ensures that we do proper zero out
1835	* for partial write.
1836	*/
1837	set_buffer_new(bh);
1838	set_buffer_mapped(bh);
1839	}
1840	return 0;
1841	}
1842
1843	static void mpage_folio_done(struct mpage_da_data *mpd, struct folio *folio)
1844	{
1845	mpd->first_page += folio_nr_pages(folio);
1846	folio_unlock(folio);
1847	}
1848
1849	static int mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio)
1850	{
1851	size_t len;
1852	loff_t size;
1853	int err;
1854
1855	BUG_ON(folio->index != mpd->first_page);
1856	folio_clear_dirty_for_io(folio);
1857	/*
1858	* We have to be very careful here! Nothing protects writeback path
1859	* against i_size changes and the page can be writeably mapped into
1860	* page tables. So an application can be growing i_size and writing
1861	* data through mmap while writeback runs. folio_clear_dirty_for_io()
1862	* write-protects our page in page tables and the page cannot get
1863	* written to again until we release folio lock. So only after
1864	* folio_clear_dirty_for_io() we are safe to sample i_size for
1865	* ext4_bio_write_folio() to zero-out tail of the written page. We rely
1866	* on the barrier provided by folio_test_clear_dirty() in
1867	* folio_clear_dirty_for_io() to make sure i_size is really sampled only
1868	* after page tables are updated.
1869	*/
1870	size = i_size_read(inode: mpd->inode);
1871	len = folio_size(folio);
1872	if (folio_pos(folio) + len > size &&
1873	!ext4_verity_in_progress(inode: mpd->inode))
1874	len = size & ~PAGE_MASK;
1875	err = ext4_bio_write_folio(io: &mpd->io_submit, page: folio, len);
1876	if (!err)
1877	mpd->wbc->nr_to_write--;
1878
1879	return err;
1880	}
1881
1882	#define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay))
1883
1884	/*
1885	* mballoc gives us at most this number of blocks...
1886	* XXX: That seems to be only a limitation of ext4_mb_normalize_request().
1887	* The rest of mballoc seems to handle chunks up to full group size.
1888	*/
1889	#define MAX_WRITEPAGES_EXTENT_LEN 2048
1890
1891	/*
1892	* mpage_add_bh_to_extent - try to add bh to extent of blocks to map
1893	*
1894	* @mpd - extent of blocks
1895	* @lblk - logical number of the block in the file
1896	* @bh - buffer head we want to add to the extent
1897	*
1898	* The function is used to collect contig. blocks in the same state. If the
1899	* buffer doesn't require mapping for writeback and we haven't started the
1900	* extent of buffers to map yet, the function returns 'true' immediately - the
1901	* caller can write the buffer right away. Otherwise the function returns true
1902	* if the block has been added to the extent, false if the block couldn't be
1903	* added.
1904	*/
1905	static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
1906	struct buffer_head *bh)
1907	{
1908	struct ext4_map_blocks *map = &mpd->map;
1909
1910	/* Buffer that doesn't need mapping for writeback? */
1911	if (!buffer_dirty(bh) || !buffer_mapped(bh) ||
1912	(!buffer_delay(bh) && !buffer_unwritten(bh))) {
1913	/* So far no extent to map => we write the buffer right away */
1914	if (map->m_len == 0)
1915	return true;
1916	return false;
1917	}
1918
1919	/* First block in the extent? */
1920	if (map->m_len == 0) {
1921	/* We cannot map unless handle is started... */
1922	if (!mpd->do_map)
1923	return false;
1924	map->m_lblk = lblk;
1925	map->m_len = 1;
1926	map->m_flags = bh->b_state & BH_FLAGS;
1927	return true;
1928	}
1929
1930	/* Don't go larger than mballoc is willing to allocate */
1931	if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)
1932	return false;
1933
1934	/* Can we merge the block to our big extent? */
1935	if (lblk == map->m_lblk + map->m_len &&
1936	(bh->b_state & BH_FLAGS) == map->m_flags) {
1937	map->m_len++;
1938	return true;
1939	}
1940	return false;
1941	}
1942
1943	/*
1944	* mpage_process_page_bufs - submit page buffers for IO or add them to extent
1945	*
1946	* @mpd - extent of blocks for mapping
1947	* @head - the first buffer in the page
1948	* @bh - buffer we should start processing from
1949	* @lblk - logical number of the block in the file corresponding to @bh
1950	*
1951	* Walk through page buffers from @bh upto @head (exclusive) and either submit
1952	* the page for IO if all buffers in this page were mapped and there's no
1953	* accumulated extent of buffers to map or add buffers in the page to the
1954	* extent of buffers to map. The function returns 1 if the caller can continue
1955	* by processing the next page, 0 if it should stop adding buffers to the
1956	* extent to map because we cannot extend it anymore. It can also return value
1957	* < 0 in case of error during IO submission.
1958	*/
1959	static int mpage_process_page_bufs(struct mpage_da_data *mpd,
1960	struct buffer_head *head,
1961	struct buffer_head *bh,
1962	ext4_lblk_t lblk)
1963	{
1964	struct inode *inode = mpd->inode;
1965	int err;
1966	ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(node: inode) - 1)
1967	>> inode->i_blkbits;
1968
1969	if (ext4_verity_in_progress(inode))
1970	blocks = EXT_MAX_BLOCKS;
1971
1972	do {
1973	BUG_ON(buffer_locked(bh));
1974
1975	if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
1976	/* Found extent to map? */
1977	if (mpd->map.m_len)
1978	return 0;
1979	/* Buffer needs mapping and handle is not started? */
1980	if (!mpd->do_map)
1981	return 0;
1982	/* Everything mapped so far and we hit EOF */
1983	break;
1984	}
1985	} while (lblk++, (bh = bh->b_this_page) != head);
1986	/* So far everything mapped? Submit the page for IO. */
1987	if (mpd->map.m_len == 0) {
1988	err = mpage_submit_folio(mpd, folio: head->b_folio);
1989	if (err < 0)
1990	return err;
1991	mpage_folio_done(mpd, folio: head->b_folio);
1992	}
1993	if (lblk >= blocks) {
1994	mpd->scanned_until_end = 1;
1995	return 0;
1996	}
1997	return 1;
1998	}
1999
2000	/*
2001	* mpage_process_folio - update folio buffers corresponding to changed extent
2002	* and may submit fully mapped page for IO
2003	* @mpd: description of extent to map, on return next extent to map
2004	* @folio: Contains these buffers.
2005	* @m_lblk: logical block mapping.
2006	* @m_pblk: corresponding physical mapping.
2007	* @map_bh: determines on return whether this page requires any further
2008	* mapping or not.
2009	*
2010	* Scan given folio buffers corresponding to changed extent and update buffer
2011	* state according to new extent state.
2012	* We map delalloc buffers to their physical location, clear unwritten bits.
2013	* If the given folio is not fully mapped, we update @mpd to the next extent in
2014	* the given folio that needs mapping & return @map_bh as true.
2015	*/
2016	static int mpage_process_folio(struct mpage_da_data *mpd, struct folio *folio,
2017	ext4_lblk_t *m_lblk, ext4_fsblk_t *m_pblk,
2018	bool *map_bh)
2019	{
2020	struct buffer_head *head, *bh;
2021	ext4_io_end_t *io_end = mpd->io_submit.io_end;
2022	ext4_lblk_t lblk = *m_lblk;
2023	ext4_fsblk_t pblock = *m_pblk;
2024	int err = 0;
2025	int blkbits = mpd->inode->i_blkbits;
2026	ssize_t io_end_size = 0;
2027	struct ext4_io_end_vec *io_end_vec = ext4_last_io_end_vec(io_end);
2028
2029	bh = head = folio_buffers(folio);
2030	do {
2031	if (lblk < mpd->map.m_lblk)
2032	continue;
2033	if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
2034	/*
2035	* Buffer after end of mapped extent.
2036	* Find next buffer in the folio to map.
2037	*/
2038	mpd->map.m_len = 0;
2039	mpd->map.m_flags = 0;
2040	io_end_vec->size += io_end_size;
2041
2042	err = mpage_process_page_bufs(mpd, head, bh, lblk);
2043	if (err > 0)
2044	err = 0;
2045	if (!err && mpd->map.m_len && mpd->map.m_lblk > lblk) {
2046	io_end_vec = ext4_alloc_io_end_vec(io_end);
2047	if (IS_ERR(ptr: io_end_vec)) {
2048	err = PTR_ERR(ptr: io_end_vec);
2049	goto out;
2050	}
2051	io_end_vec->offset = (loff_t)mpd->map.m_lblk << blkbits;
2052	}
2053	*map_bh = true;
2054	goto out;
2055	}
2056	if (buffer_delay(bh)) {
2057	clear_buffer_delay(bh);
2058	bh->b_blocknr = pblock++;
2059	}
2060	clear_buffer_unwritten(bh);
2061	io_end_size += (1 << blkbits);
2062	} while (lblk++, (bh = bh->b_this_page) != head);
2063
2064	io_end_vec->size += io_end_size;
2065	*map_bh = false;
2066	out:
2067	*m_lblk = lblk;
2068	*m_pblk = pblock;
2069	return err;
2070	}
2071
2072	/*
2073	* mpage_map_buffers - update buffers corresponding to changed extent and
2074	* submit fully mapped pages for IO
2075	*
2076	* @mpd - description of extent to map, on return next extent to map
2077	*
2078	* Scan buffers corresponding to changed extent (we expect corresponding pages
2079	* to be already locked) and update buffer state according to new extent state.
2080	* We map delalloc buffers to their physical location, clear unwritten bits,
2081	* and mark buffers as uninit when we perform writes to unwritten extents
2082	* and do extent conversion after IO is finished. If the last page is not fully
2083	* mapped, we update @map to the next extent in the last page that needs
2084	* mapping. Otherwise we submit the page for IO.
2085	*/
2086	static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
2087	{
2088	struct folio_batch fbatch;
2089	unsigned nr, i;
2090	struct inode *inode = mpd->inode;
2091	int bpp_bits = PAGE_SHIFT - inode->i_blkbits;
2092	pgoff_t start, end;
2093	ext4_lblk_t lblk;
2094	ext4_fsblk_t pblock;
2095	int err;
2096	bool map_bh = false;
2097
2098	start = mpd->map.m_lblk >> bpp_bits;
2099	end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits;
2100	lblk = start << bpp_bits;
2101	pblock = mpd->map.m_pblk;
2102
2103	folio_batch_init(fbatch: &fbatch);
2104	while (start <= end) {
2105	nr = filemap_get_folios(mapping: inode->i_mapping, start: &start, end, fbatch: &fbatch);
2106	if (nr == 0)
2107	break;
2108	for (i = 0; i < nr; i++) {
2109	struct folio *folio = fbatch.folios[i];
2110
2111	err = mpage_process_folio(mpd, folio, m_lblk: &lblk, m_pblk: &pblock,
2112	map_bh: &map_bh);
2113	/*
2114	* If map_bh is true, means page may require further bh
2115	* mapping, or maybe the page was submitted for IO.
2116	* So we return to call further extent mapping.
2117	*/
2118	if (err < 0 || map_bh)
2119	goto out;
2120	/* Page fully mapped - let IO run! */
2121	err = mpage_submit_folio(mpd, folio);
2122	if (err < 0)
2123	goto out;
2124	mpage_folio_done(mpd, folio);
2125	}
2126	folio_batch_release(fbatch: &fbatch);
2127	}
2128	/* Extent fully mapped and matches with page boundary. We are done. */
2129	mpd->map.m_len = 0;
2130	mpd->map.m_flags = 0;
2131	return 0;
2132	out:
2133	folio_batch_release(fbatch: &fbatch);
2134	return err;
2135	}
2136
2137	static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
2138	{
2139	struct inode *inode = mpd->inode;
2140	struct ext4_map_blocks *map = &mpd->map;
2141	int get_blocks_flags;
2142	int err, dioread_nolock;
2143
2144	trace_ext4_da_write_pages_extent(inode, map);
2145	/*
2146	* Call ext4_map_blocks() to allocate any delayed allocation blocks, or
2147	* to convert an unwritten extent to be initialized (in the case
2148	* where we have written into one or more preallocated blocks). It is
2149	* possible that we're going to need more metadata blocks than
2150	* previously reserved. However we must not fail because we're in
2151	* writeback and there is nothing we can do about it so it might result
2152	* in data loss. So use reserved blocks to allocate metadata if
2153	* possible.
2154	*
2155	* We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE if
2156	* the blocks in question are delalloc blocks. This indicates
2157	* that the blocks and quotas has already been checked when
2158	* the data was copied into the page cache.
2159	*/
2160	get_blocks_flags = EXT4_GET_BLOCKS_CREATE |
2161	EXT4_GET_BLOCKS_METADATA_NOFAIL |
2162	EXT4_GET_BLOCKS_IO_SUBMIT;
2163	dioread_nolock = ext4_should_dioread_nolock(inode);
2164	if (dioread_nolock)
2165	get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;
2166	if (map->m_flags & BIT(BH_Delay))
2167	get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
2168
2169	err = ext4_map_blocks(handle, inode, map, flags: get_blocks_flags);
2170	if (err < 0)
2171	return err;
2172	if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {
2173	if (!mpd->io_submit.io_end->handle &&
2174	ext4_handle_valid(handle)) {
2175	mpd->io_submit.io_end->handle = handle->h_rsv_handle;
2176	handle->h_rsv_handle = NULL;
2177	}
2178	ext4_set_io_unwritten_flag(inode, io_end: mpd->io_submit.io_end);
2179	}
2180
2181	BUG_ON(map->m_len == 0);
2182	return 0;
2183	}
2184
2185	/*
2186	* mpage_map_and_submit_extent - map extent starting at mpd->lblk of length
2187	* mpd->len and submit pages underlying it for IO
2188	*
2189	* @handle - handle for journal operations
2190	* @mpd - extent to map
2191	* @give_up_on_write - we set this to true iff there is a fatal error and there
2192	* is no hope of writing the data. The caller should discard
2193	* dirty pages to avoid infinite loops.
2194	*
2195	* The function maps extent starting at mpd->lblk of length mpd->len. If it is
2196	* delayed, blocks are allocated, if it is unwritten, we may need to convert
2197	* them to initialized or split the described range from larger unwritten
2198	* extent. Note that we need not map all the described range since allocation
2199	* can return less blocks or the range is covered by more unwritten extents. We
2200	* cannot map more because we are limited by reserved transaction credits. On
2201	* the other hand we always make sure that the last touched page is fully
2202	* mapped so that it can be written out (and thus forward progress is
2203	* guaranteed). After mapping we submit all mapped pages for IO.
2204	*/
2205	static int mpage_map_and_submit_extent(handle_t *handle,
2206	struct mpage_da_data *mpd,
2207	bool *give_up_on_write)
2208	{
2209	struct inode *inode = mpd->inode;
2210	struct ext4_map_blocks *map = &mpd->map;
2211	int err;
2212	loff_t disksize;
2213	int progress = 0;
2214	ext4_io_end_t *io_end = mpd->io_submit.io_end;
2215	struct ext4_io_end_vec *io_end_vec;
2216
2217	io_end_vec = ext4_alloc_io_end_vec(io_end);
2218	if (IS_ERR(ptr: io_end_vec))
2219	return PTR_ERR(ptr: io_end_vec);
2220	io_end_vec->offset = ((loff_t)map->m_lblk) << inode->i_blkbits;
2221	do {
2222	err = mpage_map_one_extent(handle, mpd);
2223	if (err < 0) {
2224	struct super_block *sb = inode->i_sb;
2225
2226	if (ext4_forced_shutdown(sb))
2227	goto invalidate_dirty_pages;
2228	/*
2229	* Let the uper layers retry transient errors.
2230	* In the case of ENOSPC, if ext4_count_free_blocks()
2231	* is non-zero, a commit should free up blocks.
2232	*/
2233	if ((err == -ENOMEM) ||
2234	(err == -ENOSPC && ext4_count_free_clusters(sb))) {
2235	if (progress)
2236	goto update_disksize;
2237	return err;
2238	}
2239	ext4_msg(sb, KERN_CRIT,
2240	"Delayed block allocation failed for "
2241	"inode %lu at logical offset %llu with"
2242	" max blocks %u with error %d",
2243	inode->i_ino,
2244	(unsigned long long)map->m_lblk,
2245	(unsigned)map->m_len, -err);
2246	ext4_msg(sb, KERN_CRIT,
2247	"This should not happen!! Data will "
2248	"be lost\n");
2249	if (err == -ENOSPC)
2250	ext4_print_free_blocks(inode);
2251	invalidate_dirty_pages:
2252	*give_up_on_write = true;
2253	return err;
2254	}
2255	progress = 1;
2256	/*
2257	* Update buffer state, submit mapped pages, and get us new
2258	* extent to map
2259	*/
2260	err = mpage_map_and_submit_buffers(mpd);
2261	if (err < 0)
2262	goto update_disksize;
2263	} while (map->m_len);
2264
2265	update_disksize:
2266	/*
2267	* Update on-disk size after IO is submitted. Races with
2268	* truncate are avoided by checking i_size under i_data_sem.
2269	*/
2270	disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT;
2271	if (disksize > READ_ONCE(EXT4_I(inode)->i_disksize)) {
2272	int err2;
2273	loff_t i_size;
2274
2275	down_write(sem: &EXT4_I(inode)->i_data_sem);
2276	i_size = i_size_read(inode);
2277	if (disksize > i_size)
2278	disksize = i_size;
2279	if (disksize > EXT4_I(inode)->i_disksize)
2280	EXT4_I(inode)->i_disksize = disksize;
2281	up_write(sem: &EXT4_I(inode)->i_data_sem);
2282	err2 = ext4_mark_inode_dirty(handle, inode);
2283	if (err2) {
2284	ext4_error_err(inode->i_sb, -err2,
2285	"Failed to mark inode %lu dirty",
2286	inode->i_ino);
2287	}
2288	if (!err)
2289	err = err2;
2290	}
2291	return err;
2292	}
2293
2294	/*
2295	* Calculate the total number of credits to reserve for one writepages
2296	* iteration. This is called from ext4_writepages(). We map an extent of
2297	* up to MAX_WRITEPAGES_EXTENT_LEN blocks and then we go on and finish mapping
2298	* the last partial page. So in total we can map MAX_WRITEPAGES_EXTENT_LEN +
2299	* bpp - 1 blocks in bpp different extents.
2300	*/
2301	static int ext4_da_writepages_trans_blocks(struct inode *inode)
2302	{
2303	int bpp = ext4_journal_blocks_per_page(inode);
2304
2305	return ext4_meta_trans_blocks(inode,
2306	MAX_WRITEPAGES_EXTENT_LEN + bpp - 1, pextents: bpp);
2307	}
2308
2309	static int ext4_journal_folio_buffers(handle_t *handle, struct folio *folio,
2310	size_t len)
2311	{
2312	struct buffer_head *page_bufs = folio_buffers(folio);
2313	struct inode *inode = folio->mapping->host;
2314	int ret, err;
2315
2316	ret = ext4_walk_page_buffers(handle, inode, head: page_bufs, from: 0, to: len,
2317	NULL, fn: do_journal_get_write_access);
2318	err = ext4_walk_page_buffers(handle, inode, head: page_bufs, from: 0, to: len,
2319	NULL, fn: write_end_fn);
2320	if (ret == 0)
2321	ret = err;
2322	err = ext4_jbd2_inode_add_write(handle, inode, start_byte: folio_pos(folio), length: len);
2323	if (ret == 0)
2324	ret = err;
2325	EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
2326
2327	return ret;
2328	}
2329
2330	static int mpage_journal_page_buffers(handle_t *handle,
2331	struct mpage_da_data *mpd,
2332	struct folio *folio)
2333	{
2334	struct inode *inode = mpd->inode;
2335	loff_t size = i_size_read(inode);
2336	size_t len = folio_size(folio);
2337
2338	folio_clear_checked(folio);
2339	mpd->wbc->nr_to_write--;
2340
2341	if (folio_pos(folio) + len > size &&
2342	!ext4_verity_in_progress(inode))
2343	len = size - folio_pos(folio);
2344
2345	return ext4_journal_folio_buffers(handle, folio, len);
2346	}
2347
2348	/*
2349	* mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages
2350	* needing mapping, submit mapped pages
2351	*
2352	* @mpd - where to look for pages
2353	*
2354	* Walk dirty pages in the mapping. If they are fully mapped, submit them for
2355	* IO immediately. If we cannot map blocks, we submit just already mapped
2356	* buffers in the page for IO and keep page dirty. When we can map blocks and
2357	* we find a page which isn't mapped we start accumulating extent of buffers
2358	* underlying these pages that needs mapping (formed by either delayed or
2359	* unwritten buffers). We also lock the pages containing these buffers. The
2360	* extent found is returned in @mpd structure (starting at mpd->lblk with
2361	* length mpd->len blocks).
2362	*
2363	* Note that this function can attach bios to one io_end structure which are
2364	* neither logically nor physically contiguous. Although it may seem as an
2365	* unnecessary complication, it is actually inevitable in blocksize < pagesize
2366	* case as we need to track IO to all buffers underlying a page in one io_end.
2367	*/
2368	static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
2369	{
2370	struct address_space *mapping = mpd->inode->i_mapping;
2371	struct folio_batch fbatch;
2372	unsigned int nr_folios;
2373	pgoff_t index = mpd->first_page;
2374	pgoff_t end = mpd->last_page;
2375	xa_mark_t tag;
2376	int i, err = 0;
2377	int blkbits = mpd->inode->i_blkbits;
2378	ext4_lblk_t lblk;
2379	struct buffer_head *head;
2380	handle_t *handle = NULL;
2381	int bpp = ext4_journal_blocks_per_page(inode: mpd->inode);
2382
2383	if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages)
2384	tag = PAGECACHE_TAG_TOWRITE;
2385	else
2386	tag = PAGECACHE_TAG_DIRTY;
2387
2388	mpd->map.m_len = 0;
2389	mpd->next_page = index;
2390	if (ext4_should_journal_data(inode: mpd->inode)) {
2391	handle = ext4_journal_start(mpd->inode, EXT4_HT_WRITE_PAGE,
2392	bpp);
2393	if (IS_ERR(ptr: handle))
2394	return PTR_ERR(ptr: handle);
2395	}
2396	folio_batch_init(fbatch: &fbatch);
2397	while (index <= end) {
2398	nr_folios = filemap_get_folios_tag(mapping, start: &index, end,
2399	tag, fbatch: &fbatch);
2400	if (nr_folios == 0)
2401	break;
2402
2403	for (i = 0; i < nr_folios; i++) {
2404	struct folio *folio = fbatch.folios[i];
2405
2406	/*
2407	* Accumulated enough dirty pages? This doesn't apply
2408	* to WB_SYNC_ALL mode. For integrity sync we have to
2409	* keep going because someone may be concurrently
2410	* dirtying pages, and we might have synced a lot of
2411	* newly appeared dirty pages, but have not synced all
2412	* of the old dirty pages.
2413	*/
2414	if (mpd->wbc->sync_mode == WB_SYNC_NONE &&
2415	mpd->wbc->nr_to_write <=
2416	mpd->map.m_len >> (PAGE_SHIFT - blkbits))
2417	goto out;
2418
2419	/* If we can't merge this page, we are done. */
2420	if (mpd->map.m_len > 0 && mpd->next_page != folio->index)
2421	goto out;
2422
2423	if (handle) {
2424	err = ext4_journal_ensure_credits(handle, credits: bpp,
2425	revoke_creds: 0);
2426	if (err < 0)
2427	goto out;
2428	}
2429
2430	folio_lock(folio);
2431	/*
2432	* If the page is no longer dirty, or its mapping no
2433	* longer corresponds to inode we are writing (which
2434	* means it has been truncated or invalidated), or the
2435	* page is already under writeback and we are not doing
2436	* a data integrity writeback, skip the page
2437	*/
2438	if (!folio_test_dirty(folio) ||
2439	(folio_test_writeback(folio) &&
2440	(mpd->wbc->sync_mode == WB_SYNC_NONE)) ||
2441	unlikely(folio->mapping != mapping)) {
2442	folio_unlock(folio);
2443	continue;
2444	}
2445
2446	folio_wait_writeback(folio);
2447	BUG_ON(folio_test_writeback(folio));
2448
2449	/*
2450	* Should never happen but for buggy code in
2451	* other subsystems that call
2452	* set_page_dirty() without properly warning
2453	* the file system first. See [1] for more
2454	* information.
2455	*
2456	* [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz
2457	*/
2458	if (!folio_buffers(folio)) {
2459	ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", folio->index);
2460	folio_clear_dirty(folio);
2461	folio_unlock(folio);
2462	continue;
2463	}
2464
2465	if (mpd->map.m_len == 0)
2466	mpd->first_page = folio->index;
2467	mpd->next_page = folio_next_index(folio);
2468	/*
2469	* Writeout when we cannot modify metadata is simple.
2470	* Just submit the page. For data=journal mode we
2471	* first handle writeout of the page for checkpoint and
2472	* only after that handle delayed page dirtying. This
2473	* makes sure current data is checkpointed to the final
2474	* location before possibly journalling it again which
2475	* is desirable when the page is frequently dirtied
2476	* through a pin.
2477	*/
2478	if (!mpd->can_map) {
2479	err = mpage_submit_folio(mpd, folio);
2480	if (err < 0)
2481	goto out;
2482	/* Pending dirtying of journalled data? */
2483	if (folio_test_checked(folio)) {
2484	err = mpage_journal_page_buffers(handle,
2485	mpd, folio);
2486	if (err < 0)
2487	goto out;
2488	mpd->journalled_more_data = 1;
2489	}
2490	mpage_folio_done(mpd, folio);
2491	} else {
2492	/* Add all dirty buffers to mpd */
2493	lblk = ((ext4_lblk_t)folio->index) <<
2494	(PAGE_SHIFT - blkbits);
2495	head = folio_buffers(folio);
2496	err = mpage_process_page_bufs(mpd, head, bh: head,
2497	lblk);
2498	if (err <= 0)
2499	goto out;
2500	err = 0;
2501	}
2502	}
2503	folio_batch_release(fbatch: &fbatch);
2504	cond_resched();
2505	}
2506	mpd->scanned_until_end = 1;
2507	if (handle)
2508	ext4_journal_stop(handle);
2509	return 0;
2510	out:
2511	folio_batch_release(fbatch: &fbatch);
2512	if (handle)
2513	ext4_journal_stop(handle);
2514	return err;
2515	}
2516
2517	static int ext4_do_writepages(struct mpage_da_data *mpd)
2518	{
2519	struct writeback_control *wbc = mpd->wbc;
2520	pgoff_t writeback_index = 0;
2521	long nr_to_write = wbc->nr_to_write;
2522	int range_whole = 0;
2523	int cycled = 1;
2524	handle_t *handle = NULL;
2525	struct inode *inode = mpd->inode;
2526	struct address_space *mapping = inode->i_mapping;
2527	int needed_blocks, rsv_blocks = 0, ret = 0;
2528	struct ext4_sb_info *sbi = EXT4_SB(sb: mapping->host->i_sb);
2529	struct blk_plug plug;
2530	bool give_up_on_write = false;
2531
2532	trace_ext4_writepages(inode, wbc);
2533
2534	/*
2535	* No pages to write? This is mainly a kludge to avoid starting
2536	* a transaction for special inodes like journal inode on last iput()
2537	* because that could violate lock ordering on umount
2538	*/
2539	if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2540	goto out_writepages;
2541
2542	/*
2543	* If the filesystem has aborted, it is read-only, so return
2544	* right away instead of dumping stack traces later on that
2545	* will obscure the real source of the problem. We test
2546	* fs shutdown state instead of sb->s_flag's SB_RDONLY because
2547	* the latter could be true if the filesystem is mounted
2548	* read-only, and in that case, ext4_writepages should
2549	* *never* be called, so if that ever happens, we would want
2550	* the stack trace.
2551	*/
2552	if (unlikely(ext4_forced_shutdown(mapping->host->i_sb))) {
2553	ret = -EROFS;
2554	goto out_writepages;
2555	}
2556
2557	/*
2558	* If we have inline data and arrive here, it means that
2559	* we will soon create the block for the 1st page, so
2560	* we'd better clear the inline data here.
2561	*/
2562	if (ext4_has_inline_data(inode)) {
2563	/* Just inode will be modified... */
2564	handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
2565	if (IS_ERR(ptr: handle)) {
2566	ret = PTR_ERR(ptr: handle);
2567	goto out_writepages;
2568	}
2569	BUG_ON(ext4_test_inode_state(inode,
2570	EXT4_STATE_MAY_INLINE_DATA));
2571	ext4_destroy_inline_data(handle, inode);
2572	ext4_journal_stop(handle);
2573	}
2574
2575	/*
2576	* data=journal mode does not do delalloc so we just need to writeout /
2577	* journal already mapped buffers. On the other hand we need to commit
2578	* transaction to make data stable. We expect all the data to be
2579	* already in the journal (the only exception are DMA pinned pages
2580	* dirtied behind our back) so we commit transaction here and run the
2581	* writeback loop to checkpoint them. The checkpointing is not actually
2582	* necessary to make data persistent *but* quite a few places (extent
2583	* shifting operations, fsverity, ...) depend on being able to drop
2584	* pagecache pages after calling filemap_write_and_wait() and for that
2585	* checkpointing needs to happen.
2586	*/
2587	if (ext4_should_journal_data(inode)) {
2588	mpd->can_map = 0;
2589	if (wbc->sync_mode == WB_SYNC_ALL)
2590	ext4_fc_commit(journal: sbi->s_journal,
2591	EXT4_I(inode)->i_datasync_tid);
2592	}
2593	mpd->journalled_more_data = 0;
2594
2595	if (ext4_should_dioread_nolock(inode)) {
2596	/*
2597	* We may need to convert up to one extent per block in
2598	* the page and we may dirty the inode.
2599	*/
2600	rsv_blocks = 1 + ext4_chunk_trans_blocks(inode,
2601	PAGE_SIZE >> inode->i_blkbits);
2602	}
2603
2604	if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2605	range_whole = 1;
2606
2607	if (wbc->range_cyclic) {
2608	writeback_index = mapping->writeback_index;
2609	if (writeback_index)
2610	cycled = 0;
2611	mpd->first_page = writeback_index;
2612	mpd->last_page = -1;
2613	} else {
2614	mpd->first_page = wbc->range_start >> PAGE_SHIFT;
2615	mpd->last_page = wbc->range_end >> PAGE_SHIFT;
2616	}
2617
2618	ext4_io_submit_init(io: &mpd->io_submit, wbc);
2619	retry:
2620	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2621	tag_pages_for_writeback(mapping, start: mpd->first_page,
2622	end: mpd->last_page);
2623	blk_start_plug(&plug);
2624
2625	/*
2626	* First writeback pages that don't need mapping - we can avoid
2627	* starting a transaction unnecessarily and also avoid being blocked
2628	* in the block layer on device congestion while having transaction
2629	* started.
2630	*/
2631	mpd->do_map = 0;
2632	mpd->scanned_until_end = 0;
2633	mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2634	if (!mpd->io_submit.io_end) {
2635	ret = -ENOMEM;
2636	goto unplug;
2637	}
2638	ret = mpage_prepare_extent_to_map(mpd);
2639	/* Unlock pages we didn't use */
2640	mpage_release_unused_pages(mpd, invalidate: false);
2641	/* Submit prepared bio */
2642	ext4_io_submit(io: &mpd->io_submit);
2643	ext4_put_io_end_defer(io_end: mpd->io_submit.io_end);
2644	mpd->io_submit.io_end = NULL;
2645	if (ret < 0)
2646	goto unplug;
2647
2648	while (!mpd->scanned_until_end && wbc->nr_to_write > 0) {
2649	/* For each extent of pages we use new io_end */
2650	mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2651	if (!mpd->io_submit.io_end) {
2652	ret = -ENOMEM;
2653	break;
2654	}
2655
2656	WARN_ON_ONCE(!mpd->can_map);
2657	/*
2658	* We have two constraints: We find one extent to map and we
2659	* must always write out whole page (makes a difference when
2660	* blocksize < pagesize) so that we don't block on IO when we
2661	* try to write out the rest of the page. Journalled mode is
2662	* not supported by delalloc.
2663	*/
2664	BUG_ON(ext4_should_journal_data(inode));
2665	needed_blocks = ext4_da_writepages_trans_blocks(inode);
2666
2667	/* start a new transaction */
2668	handle = ext4_journal_start_with_reserve(inode,
2669	EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);
2670	if (IS_ERR(ptr: handle)) {
2671	ret = PTR_ERR(ptr: handle);
2672	ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
2673	"%ld pages, ino %lu; err %d", __func__,
2674	wbc->nr_to_write, inode->i_ino, ret);
2675	/* Release allocated io_end */
2676	ext4_put_io_end(io_end: mpd->io_submit.io_end);
2677	mpd->io_submit.io_end = NULL;
2678	break;
2679	}
2680	mpd->do_map = 1;
2681
2682	trace_ext4_da_write_pages(inode, first_page: mpd->first_page, wbc);
2683	ret = mpage_prepare_extent_to_map(mpd);
2684	if (!ret && mpd->map.m_len)
2685	ret = mpage_map_and_submit_extent(handle, mpd,
2686	give_up_on_write: &give_up_on_write);
2687	/*
2688	* Caution: If the handle is synchronous,
2689	* ext4_journal_stop() can wait for transaction commit
2690	* to finish which may depend on writeback of pages to
2691	* complete or on page lock to be released. In that
2692	* case, we have to wait until after we have
2693	* submitted all the IO, released page locks we hold,
2694	* and dropped io_end reference (for extent conversion
2695	* to be able to complete) before stopping the handle.
2696	*/
2697	if (!ext4_handle_valid(handle) || handle->h_sync == 0) {
2698	ext4_journal_stop(handle);
2699	handle = NULL;
2700	mpd->do_map = 0;
2701	}
2702	/* Unlock pages we didn't use */
2703	mpage_release_unused_pages(mpd, invalidate: give_up_on_write);
2704	/* Submit prepared bio */
2705	ext4_io_submit(io: &mpd->io_submit);
2706
2707	/*
2708	* Drop our io_end reference we got from init. We have
2709	* to be careful and use deferred io_end finishing if
2710	* we are still holding the transaction as we can
2711	* release the last reference to io_end which may end
2712	* up doing unwritten extent conversion.
2713	*/
2714	if (handle) {
2715	ext4_put_io_end_defer(io_end: mpd->io_submit.io_end);
2716	ext4_journal_stop(handle);
2717	} else
2718	ext4_put_io_end(io_end: mpd->io_submit.io_end);
2719	mpd->io_submit.io_end = NULL;
2720
2721	if (ret == -ENOSPC && sbi->s_journal) {
2722	/*
2723	* Commit the transaction which would
2724	* free blocks released in the transaction
2725	* and try again
2726	*/
2727	jbd2_journal_force_commit_nested(sbi->s_journal);
2728	ret = 0;
2729	continue;
2730	}
2731	/* Fatal error - ENOMEM, EIO... */
2732	if (ret)
2733	break;
2734	}
2735	unplug:
2736	blk_finish_plug(&plug);
2737	if (!ret && !cycled && wbc->nr_to_write > 0) {
2738	cycled = 1;
2739	mpd->last_page = writeback_index - 1;
2740	mpd->first_page = 0;
2741	goto retry;
2742	}
2743
2744	/* Update index */
2745	if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2746	/*
2747	* Set the writeback_index so that range_cyclic
2748	* mode will write it back later
2749	*/
2750	mapping->writeback_index = mpd->first_page;
2751
2752	out_writepages:
2753	trace_ext4_writepages_result(inode, wbc, ret,
2754	pages_written: nr_to_write - wbc->nr_to_write);
2755	return ret;
2756	}
2757
2758	static int ext4_writepages(struct address_space *mapping,
2759	struct writeback_control *wbc)
2760	{
2761	struct super_block *sb = mapping->host->i_sb;
2762	struct mpage_da_data mpd = {
2763	.inode = mapping->host,
2764	.wbc = wbc,
2765	.can_map = 1,
2766	};
2767	int ret;
2768	int alloc_ctx;
2769
2770	if (unlikely(ext4_forced_shutdown(sb)))
2771	return -EIO;
2772
2773	alloc_ctx = ext4_writepages_down_read(sb);
2774	ret = ext4_do_writepages(mpd: &mpd);
2775	/*
2776	* For data=journal writeback we could have come across pages marked
2777	* for delayed dirtying (PageChecked) which were just added to the
2778	* running transaction. Try once more to get them to stable storage.
2779	*/
2780	if (!ret && mpd.journalled_more_data)
2781	ret = ext4_do_writepages(mpd: &mpd);
2782	ext4_writepages_up_read(sb, ctx: alloc_ctx);
2783
2784	return ret;
2785	}
2786
2787	int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode)
2788	{
2789	struct writeback_control wbc = {
2790	.sync_mode = WB_SYNC_ALL,
2791	.nr_to_write = LONG_MAX,
2792	.range_start = jinode->i_dirty_start,
2793	.range_end = jinode->i_dirty_end,
2794	};
2795	struct mpage_da_data mpd = {
2796	.inode = jinode->i_vfs_inode,
2797	.wbc = &wbc,
2798	.can_map = 0,
2799	};
2800	return ext4_do_writepages(mpd: &mpd);
2801	}
2802
2803	static int ext4_dax_writepages(struct address_space *mapping,
2804	struct writeback_control *wbc)
2805	{
2806	int ret;
2807	long nr_to_write = wbc->nr_to_write;
2808	struct inode *inode = mapping->host;
2809	int alloc_ctx;
2810
2811	if (unlikely(ext4_forced_shutdown(inode->i_sb)))
2812	return -EIO;
2813
2814	alloc_ctx = ext4_writepages_down_read(sb: inode->i_sb);
2815	trace_ext4_writepages(inode, wbc);
2816
2817	ret = dax_writeback_mapping_range(mapping,
2818	dax_dev: EXT4_SB(sb: inode->i_sb)->s_daxdev, wbc);
2819	trace_ext4_writepages_result(inode, wbc, ret,
2820	pages_written: nr_to_write - wbc->nr_to_write);
2821	ext4_writepages_up_read(sb: inode->i_sb, ctx: alloc_ctx);
2822	return ret;
2823	}
2824
2825	static int ext4_nonda_switch(struct super_block *sb)
2826	{
2827	s64 free_clusters, dirty_clusters;
2828	struct ext4_sb_info *sbi = EXT4_SB(sb);
2829
2830	/*
2831	* switch to non delalloc mode if we are running low
2832	* on free block. The free block accounting via percpu
2833	* counters can get slightly wrong with percpu_counter_batch getting
2834	* accumulated on each CPU without updating global counters
2835	* Delalloc need an accurate free block accounting. So switch
2836	* to non delalloc when we are near to error range.
2837	*/
2838	free_clusters =
2839	percpu_counter_read_positive(fbc: &sbi->s_freeclusters_counter);
2840	dirty_clusters =
2841	percpu_counter_read_positive(fbc: &sbi->s_dirtyclusters_counter);
2842	/*
2843	* Start pushing delalloc when 1/2 of free blocks are dirty.
2844	*/
2845	if (dirty_clusters && (free_clusters < 2 * dirty_clusters))
2846	try_to_writeback_inodes_sb(sb, reason: WB_REASON_FS_FREE_SPACE);
2847
2848	if (2 * free_clusters < 3 * dirty_clusters ||
2849	free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {
2850	/*
2851	* free block count is less than 150% of dirty blocks
2852	* or free blocks is less than watermark
2853	*/
2854	return 1;
2855	}
2856	return 0;
2857	}
2858
2859	static int ext4_da_write_begin(struct file *file, struct address_space *mapping,
2860	loff_t pos, unsigned len,
2861	struct page **pagep, void **fsdata)
2862	{
2863	int ret, retries = 0;
2864	struct folio *folio;
2865	pgoff_t index;
2866	struct inode *inode = mapping->host;
2867
2868	if (unlikely(ext4_forced_shutdown(inode->i_sb)))
2869	return -EIO;
2870
2871	index = pos >> PAGE_SHIFT;
2872
2873	if (ext4_nonda_switch(sb: inode->i_sb) || ext4_verity_in_progress(inode)) {
2874	*fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
2875	return ext4_write_begin(file, mapping, pos,
2876	len, pagep, fsdata);
2877	}
2878	*fsdata = (void *)0;
2879	trace_ext4_da_write_begin(inode, pos, len);
2880
2881	if (ext4_test_inode_state(inode, bit: EXT4_STATE_MAY_INLINE_DATA)) {
2882	ret = ext4_da_write_inline_data_begin(mapping, inode, pos, len,
2883	pagep, fsdata);
2884	if (ret < 0)
2885	return ret;
2886	if (ret == 1)
2887	return 0;
2888	}
2889
2890	retry:
2891	folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
2892	gfp: mapping_gfp_mask(mapping));
2893	if (IS_ERR(ptr: folio))
2894	return PTR_ERR(ptr: folio);
2895
2896	/* In case writeback began while the folio was unlocked */
2897	folio_wait_stable(folio);
2898
2899	#ifdef CONFIG_FS_ENCRYPTION
2900	ret = ext4_block_write_begin(folio, pos, len, get_block: ext4_da_get_block_prep);
2901	#else
2902	ret = __block_write_begin(&folio->page, pos, len, ext4_da_get_block_prep);
2903	#endif
2904	if (ret < 0) {
2905	folio_unlock(folio);
2906	folio_put(folio);
2907	/*
2908	* block_write_begin may have instantiated a few blocks
2909	* outside i_size. Trim these off again. Don't need
2910	* i_size_read because we hold inode lock.
2911	*/
2912	if (pos + len > inode->i_size)
2913	ext4_truncate_failed_write(inode);
2914
2915	if (ret == -ENOSPC &&
2916	ext4_should_retry_alloc(sb: inode->i_sb, retries: &retries))
2917	goto retry;
2918	return ret;
2919	}
2920
2921	*pagep = &folio->page;
2922	return ret;
2923	}
2924
2925	/*
2926	* Check if we should update i_disksize
2927	* when write to the end of file but not require block allocation
2928	*/
2929	static int ext4_da_should_update_i_disksize(struct folio *folio,
2930	unsigned long offset)
2931	{
2932	struct buffer_head *bh;
2933	struct inode *inode = folio->mapping->host;
2934	unsigned int idx;
2935	int i;
2936
2937	bh = folio_buffers(folio);
2938	idx = offset >> inode->i_blkbits;
2939
2940	for (i = 0; i < idx; i++)
2941	bh = bh->b_this_page;
2942
2943	if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
2944	return 0;
2945	return 1;
2946	}
2947
2948	static int ext4_da_do_write_end(struct address_space *mapping,
2949	loff_t pos, unsigned len, unsigned copied,
2950	struct page *page)
2951	{
2952	struct inode *inode = mapping->host;
2953	loff_t old_size = inode->i_size;
2954	bool disksize_changed = false;
2955	loff_t new_i_size;
2956
2957	/*
2958	* block_write_end() will mark the inode as dirty with I_DIRTY_PAGES
2959	* flag, which all that's needed to trigger page writeback.
2960	*/
2961	copied = block_write_end(NULL, mapping, pos, len, copied, page, NULL);
2962	new_i_size = pos + copied;
2963
2964	/*
2965	* It's important to update i_size while still holding page lock,
2966	* because page writeout could otherwise come in and zero beyond
2967	* i_size.
2968	*
2969	* Since we are holding inode lock, we are sure i_disksize <=
2970	* i_size. We also know that if i_disksize < i_size, there are
2971	* delalloc writes pending in the range up to i_size. If the end of
2972	* the current write is <= i_size, there's no need to touch
2973	* i_disksize since writeback will push i_disksize up to i_size
2974	* eventually. If the end of the current write is > i_size and
2975	* inside an allocated block which ext4_da_should_update_i_disksize()
2976	* checked, we need to update i_disksize here as certain
2977	* ext4_writepages() paths not allocating blocks and update i_disksize.
2978	*/
2979	if (new_i_size > inode->i_size) {
2980	unsigned long end;
2981
2982	i_size_write(inode, i_size: new_i_size);
2983	end = (new_i_size - 1) & (PAGE_SIZE - 1);
2984	if (copied && ext4_da_should_update_i_disksize(page_folio(page), offset: end)) {
2985	ext4_update_i_disksize(inode, newsize: new_i_size);
2986	disksize_changed = true;
2987	}
2988	}
2989
2990	unlock_page(page);
2991	put_page(page);
2992
2993	if (old_size < pos)
2994	pagecache_isize_extended(inode, from: old_size, to: pos);
2995
2996	if (disksize_changed) {
2997	handle_t *handle;
2998
2999	handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
3000	if (IS_ERR(ptr: handle))
3001	return PTR_ERR(ptr: handle);
3002	ext4_mark_inode_dirty(handle, inode);
3003	ext4_journal_stop(handle);
3004	}
3005
3006	return copied;
3007	}
3008
3009	static int ext4_da_write_end(struct file *file,
3010	struct address_space *mapping,
3011	loff_t pos, unsigned len, unsigned copied,
3012	struct page *page, void *fsdata)
3013	{
3014	struct inode *inode = mapping->host;
3015	int write_mode = (int)(unsigned long)fsdata;
3016	struct folio *folio = page_folio(page);
3017
3018	if (write_mode == FALL_BACK_TO_NONDELALLOC)
3019	return ext4_write_end(file, mapping, pos,
3020	len, copied, page: &folio->page, fsdata);
3021
3022	trace_ext4_da_write_end(inode, pos, len, copied);
3023
3024	if (write_mode != CONVERT_INLINE_DATA &&
3025	ext4_test_inode_state(inode, bit: EXT4_STATE_MAY_INLINE_DATA) &&
3026	ext4_has_inline_data(inode))
3027	return ext4_write_inline_data_end(inode, pos, len, copied,
3028	folio);
3029
3030	if (unlikely(copied < len) && !PageUptodate(page))
3031	copied = 0;
3032
3033	return ext4_da_do_write_end(mapping, pos, len, copied, page: &folio->page);
3034	}
3035
3036	/*
3037	* Force all delayed allocation blocks to be allocated for a given inode.
3038	*/
3039	int ext4_alloc_da_blocks(struct inode *inode)
3040	{
3041	trace_ext4_alloc_da_blocks(inode);
3042
3043	if (!EXT4_I(inode)->i_reserved_data_blocks)
3044	return 0;
3045
3046	/*
3047	* We do something simple for now. The filemap_flush() will
3048	* also start triggering a write of the data blocks, which is
3049	* not strictly speaking necessary (and for users of
3050	* laptop_mode, not even desirable). However, to do otherwise
3051	* would require replicating code paths in:
3052	*
3053	* ext4_writepages() ->
3054	* write_cache_pages() ---> (via passed in callback function)
3055	* __mpage_da_writepage() -->
3056	* mpage_add_bh_to_extent()
3057	* mpage_da_map_blocks()
3058	*
3059	* The problem is that write_cache_pages(), located in
3060	* mm/page-writeback.c, marks pages clean in preparation for
3061	* doing I/O, which is not desirable if we're not planning on
3062	* doing I/O at all.
3063	*
3064	* We could call write_cache_pages(), and then redirty all of
3065	* the pages by calling redirty_page_for_writepage() but that
3066	* would be ugly in the extreme. So instead we would need to
3067	* replicate parts of the code in the above functions,
3068	* simplifying them because we wouldn't actually intend to
3069	* write out the pages, but rather only collect contiguous
3070	* logical block extents, call the multi-block allocator, and
3071	* then update the buffer heads with the block allocations.
3072	*
3073	* For now, though, we'll cheat by calling filemap_flush(),
3074	* which will map the blocks, and start the I/O, but not
3075	* actually wait for the I/O to complete.
3076	*/
3077	return filemap_flush(inode->i_mapping);
3078	}
3079
3080	/*
3081	* bmap() is special. It gets used by applications such as lilo and by
3082	* the swapper to find the on-disk block of a specific piece of data.
3083	*
3084	* Naturally, this is dangerous if the block concerned is still in the
3085	* journal. If somebody makes a swapfile on an ext4 data-journaling
3086	* filesystem and enables swap, then they may get a nasty shock when the
3087	* data getting swapped to that swapfile suddenly gets overwritten by
3088	* the original zero's written out previously to the journal and
3089	* awaiting writeback in the kernel's buffer cache.
3090	*
3091	* So, if we see any bmap calls here on a modified, data-journaled file,
3092	* take extra steps to flush any blocks which might be in the cache.
3093	*/
3094	static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
3095	{
3096	struct inode *inode = mapping->host;
3097	sector_t ret = 0;
3098
3099	inode_lock_shared(inode);
3100	/*
3101	* We can get here for an inline file via the FIBMAP ioctl
3102	*/
3103	if (ext4_has_inline_data(inode))
3104	goto out;
3105
3106	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
3107	(test_opt(inode->i_sb, DELALLOC) ||
3108	ext4_should_journal_data(inode))) {
3109	/*
3110	* With delalloc or journalled data we want to sync the file so
3111	* that we can make sure we allocate blocks for file and data
3112	* is in place for the user to see it
3113	*/
3114	filemap_write_and_wait(mapping);
3115	}
3116
3117	ret = iomap_bmap(mapping, bno: block, ops: &ext4_iomap_ops);
3118
3119	out:
3120	inode_unlock_shared(inode);
3121	return ret;
3122	}
3123
3124	static int ext4_read_folio(struct file *file, struct folio *folio)
3125	{
3126	int ret = -EAGAIN;
3127	struct inode *inode = folio->mapping->host;
3128
3129	trace_ext4_read_folio(inode, folio);
3130
3131	if (ext4_has_inline_data(inode))
3132	ret = ext4_readpage_inline(inode, folio);
3133
3134	if (ret == -EAGAIN)
3135	return ext4_mpage_readpages(inode, NULL, folio);
3136
3137	return ret;
3138	}
3139
3140	static void ext4_readahead(struct readahead_control *rac)
3141	{
3142	struct inode *inode = rac->mapping->host;
3143
3144	/* If the file has inline data, no need to do readahead. */
3145	if (ext4_has_inline_data(inode))
3146	return;
3147
3148	ext4_mpage_readpages(inode, rac, NULL);
3149	}
3150
3151	static void ext4_invalidate_folio(struct folio *folio, size_t offset,
3152	size_t length)
3153	{
3154	trace_ext4_invalidate_folio(folio, offset, length);
3155
3156	/* No journalling happens on data buffers when this function is used */
3157	WARN_ON(folio_buffers(folio) && buffer_jbd(folio_buffers(folio)));
3158
3159	block_invalidate_folio(folio, offset, length);
3160	}
3161
3162	static int __ext4_journalled_invalidate_folio(struct folio *folio,
3163	size_t offset, size_t length)
3164	{
3165	journal_t *journal = EXT4_JOURNAL(folio->mapping->host);
3166
3167	trace_ext4_journalled_invalidate_folio(folio, offset, length);
3168
3169	/*
3170	* If it's a full truncate we just forget about the pending dirtying
3171	*/
3172	if (offset == 0 && length == folio_size(folio))
3173	folio_clear_checked(folio);
3174
3175	return jbd2_journal_invalidate_folio(journal, folio, offset, length);
3176	}
3177
3178	/* Wrapper for aops... */
3179	static void ext4_journalled_invalidate_folio(struct folio *folio,
3180	size_t offset,
3181	size_t length)
3182	{
3183	WARN_ON(__ext4_journalled_invalidate_folio(folio, offset, length) < 0);
3184	}
3185
3186	static bool ext4_release_folio(struct folio *folio, gfp_t wait)
3187	{
3188	struct inode *inode = folio->mapping->host;
3189	journal_t *journal = EXT4_JOURNAL(inode);
3190
3191	trace_ext4_release_folio(inode, folio);
3192
3193	/* Page has dirty journalled data -> cannot release */
3194	if (folio_test_checked(folio))
3195	return false;
3196	if (journal)
3197	return jbd2_journal_try_to_free_buffers(journal, folio);
3198	else
3199	return try_to_free_buffers(folio);
3200	}
3201
3202	static bool ext4_inode_datasync_dirty(struct inode *inode)
3203	{
3204	journal_t *journal = EXT4_SB(sb: inode->i_sb)->s_journal;
3205
3206	if (journal) {
3207	if (jbd2_transaction_committed(journal,
3208	EXT4_I(inode)->i_datasync_tid))
3209	return false;
3210	if (test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT))
3211	return !list_empty(head: &EXT4_I(inode)->i_fc_list);
3212	return true;
3213	}
3214
3215	/* Any metadata buffers to write? */
3216	if (!list_empty(head: &inode->i_mapping->private_list))
3217	return true;
3218	return inode->i_state & I_DIRTY_DATASYNC;
3219	}
3220
3221	static void ext4_set_iomap(struct inode *inode, struct iomap *iomap,
3222	struct ext4_map_blocks *map, loff_t offset,
3223	loff_t length, unsigned int flags)
3224	{
3225	u8 blkbits = inode->i_blkbits;
3226
3227	/*
3228	* Writes that span EOF might trigger an I/O size update on completion,
3229	* so consider them to be dirty for the purpose of O_DSYNC, even if
3230	* there is no other metadata changes being made or are pending.
3231	*/
3232	iomap->flags = 0;
3233	if (ext4_inode_datasync_dirty(inode) ||
3234	offset + length > i_size_read(inode))
3235	iomap->flags |= IOMAP_F_DIRTY;
3236
3237	if (map->m_flags & EXT4_MAP_NEW)
3238	iomap->flags |= IOMAP_F_NEW;
3239
3240	if (flags & IOMAP_DAX)
3241	iomap->dax_dev = EXT4_SB(sb: inode->i_sb)->s_daxdev;
3242	else
3243	iomap->bdev = inode->i_sb->s_bdev;
3244	iomap->offset = (u64) map->m_lblk << blkbits;
3245	iomap->length = (u64) map->m_len << blkbits;
3246
3247	if ((map->m_flags & EXT4_MAP_MAPPED) &&
3248	!ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))
3249	iomap->flags |= IOMAP_F_MERGED;
3250
3251	/*
3252	* Flags passed to ext4_map_blocks() for direct I/O writes can result
3253	* in m_flags having both EXT4_MAP_MAPPED and EXT4_MAP_UNWRITTEN bits
3254	* set. In order for any allocated unwritten extents to be converted
3255	* into written extents correctly within the ->end_io() handler, we
3256	* need to ensure that the iomap->type is set appropriately. Hence, the
3257	* reason why we need to check whether the EXT4_MAP_UNWRITTEN bit has
3258	* been set first.
3259	*/
3260	if (map->m_flags & EXT4_MAP_UNWRITTEN) {
3261	iomap->type = IOMAP_UNWRITTEN;
3262	iomap->addr = (u64) map->m_pblk << blkbits;
3263	if (flags & IOMAP_DAX)
3264	iomap->addr += EXT4_SB(sb: inode->i_sb)->s_dax_part_off;
3265	} else if (map->m_flags & EXT4_MAP_MAPPED) {
3266	iomap->type = IOMAP_MAPPED;
3267	iomap->addr = (u64) map->m_pblk << blkbits;
3268	if (flags & IOMAP_DAX)
3269	iomap->addr += EXT4_SB(sb: inode->i_sb)->s_dax_part_off;
3270	} else {
3271	iomap->type = IOMAP_HOLE;
3272	iomap->addr = IOMAP_NULL_ADDR;
3273	}
3274	}
3275
3276	static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map,
3277	unsigned int flags)
3278	{
3279	handle_t *handle;
3280	u8 blkbits = inode->i_blkbits;
3281	int ret, dio_credits, m_flags = 0, retries = 0;
3282
3283	/*
3284	* Trim the mapping request to the maximum value that we can map at
3285	* once for direct I/O.
3286	*/
3287	if (map->m_len > DIO_MAX_BLOCKS)
3288	map->m_len = DIO_MAX_BLOCKS;
3289	dio_credits = ext4_chunk_trans_blocks(inode, nrblocks: map->m_len);
3290
3291	retry:
3292	/*
3293	* Either we allocate blocks and then don't get an unwritten extent, so
3294	* in that case we have reserved enough credits. Or, the blocks are
3295	* already allocated and unwritten. In that case, the extent conversion
3296	* fits into the credits as well.
3297	*/
3298	handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);
3299	if (IS_ERR(ptr: handle))
3300	return PTR_ERR(ptr: handle);
3301
3302	/*
3303	* DAX and direct I/O are the only two operations that are currently
3304	* supported with IOMAP_WRITE.
3305	*/
3306	WARN_ON(!(flags & (IOMAP_DAX | IOMAP_DIRECT)));
3307	if (flags & IOMAP_DAX)
3308	m_flags = EXT4_GET_BLOCKS_CREATE_ZERO;
3309	/*
3310	* We use i_size instead of i_disksize here because delalloc writeback
3311	* can complete at any point during the I/O and subsequently push the
3312	* i_disksize out to i_size. This could be beyond where direct I/O is
3313	* happening and thus expose allocated blocks to direct I/O reads.
3314	*/
3315	else if (((loff_t)map->m_lblk << blkbits) >= i_size_read(inode))
3316	m_flags = EXT4_GET_BLOCKS_CREATE;
3317	else if (ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))
3318	m_flags = EXT4_GET_BLOCKS_IO_CREATE_EXT;
3319
3320	ret = ext4_map_blocks(handle, inode, map, flags: m_flags);
3321
3322	/*
3323	* We cannot fill holes in indirect tree based inodes as that could
3324	* expose stale data in the case of a crash. Use the magic error code
3325	* to fallback to buffered I/O.
3326	*/
3327	if (!m_flags && !ret)
3328	ret = -ENOTBLK;
3329
3330	ext4_journal_stop(handle);
3331	if (ret == -ENOSPC && ext4_should_retry_alloc(sb: inode->i_sb, retries: &retries))
3332	goto retry;
3333
3334	return ret;
3335	}
3336
3337
3338	static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
3339	unsigned flags, struct iomap *iomap, struct iomap *srcmap)
3340	{
3341	int ret;
3342	struct ext4_map_blocks map;
3343	u8 blkbits = inode->i_blkbits;
3344
3345	if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3346	return -EINVAL;
3347
3348	if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
3349	return -ERANGE;
3350
3351	/*
3352	* Calculate the first and last logical blocks respectively.
3353	*/
3354	map.m_lblk = offset >> blkbits;
3355	map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3356	EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
3357
3358	if (flags & IOMAP_WRITE) {
3359	/*
3360	* We check here if the blocks are already allocated, then we
3361	* don't need to start a journal txn and we can directly return
3362	* the mapping information. This could boost performance
3363	* especially in multi-threaded overwrite requests.
3364	*/
3365	if (offset + length <= i_size_read(inode)) {
3366	ret = ext4_map_blocks(NULL, inode, map: &map, flags: 0);
3367	if (ret > 0 && (map.m_flags & EXT4_MAP_MAPPED))
3368	goto out;
3369	}
3370	ret = ext4_iomap_alloc(inode, map: &map, flags);
3371	} else {
3372	ret = ext4_map_blocks(NULL, inode, map: &map, flags: 0);
3373	}
3374
3375	if (ret < 0)
3376	return ret;
3377	out:
3378	/*
3379	* When inline encryption is enabled, sometimes I/O to an encrypted file
3380	* has to be broken up to guarantee DUN contiguity. Handle this by
3381	* limiting the length of the mapping returned.
3382	*/
3383	map.m_len = fscrypt_limit_io_blocks(inode, lblk: map.m_lblk, nr_blocks: map.m_len);
3384
3385	ext4_set_iomap(inode, iomap, map: &map, offset, length, flags);
3386
3387	return 0;
3388	}
3389
3390	static int ext4_iomap_overwrite_begin(struct inode *inode, loff_t offset,
3391	loff_t length, unsigned flags, struct iomap *iomap,
3392	struct iomap *srcmap)
3393	{
3394	int ret;
3395
3396	/*
3397	* Even for writes we don't need to allocate blocks, so just pretend
3398	* we are reading to save overhead of starting a transaction.
3399	*/
3400	flags &= ~IOMAP_WRITE;
3401	ret = ext4_iomap_begin(inode, offset, length, flags, iomap, srcmap);
3402	WARN_ON_ONCE(!ret && iomap->type != IOMAP_MAPPED);
3403	return ret;
3404	}
3405
3406	static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length,
3407	ssize_t written, unsigned flags, struct iomap *iomap)
3408	{
3409	/*
3410	* Check to see whether an error occurred while writing out the data to
3411	* the allocated blocks. If so, return the magic error code so that we
3412	* fallback to buffered I/O and attempt to complete the remainder of
3413	* the I/O. Any blocks that may have been allocated in preparation for
3414	* the direct I/O will be reused during buffered I/O.
3415	*/
3416	if (flags & (IOMAP_WRITE | IOMAP_DIRECT) && written == 0)
3417	return -ENOTBLK;
3418
3419	return 0;
3420	}
3421
3422	const struct iomap_ops ext4_iomap_ops = {
3423	.iomap_begin = ext4_iomap_begin,
3424	.iomap_end = ext4_iomap_end,
3425	};
3426
3427	const struct iomap_ops ext4_iomap_overwrite_ops = {
3428	.iomap_begin = ext4_iomap_overwrite_begin,
3429	.iomap_end = ext4_iomap_end,
3430	};
3431
3432	static bool ext4_iomap_is_delalloc(struct inode *inode,
3433	struct ext4_map_blocks *map)
3434	{
3435	struct extent_status es;
3436	ext4_lblk_t offset = 0, end = map->m_lblk + map->m_len - 1;
3437
3438	ext4_es_find_extent_range(inode, match_fn: &ext4_es_is_delayed,
3439	lblk: map->m_lblk, end, es: &es);
3440
3441	if (!es.es_len || es.es_lblk > end)
3442	return false;
3443
3444	if (es.es_lblk > map->m_lblk) {
3445	map->m_len = es.es_lblk - map->m_lblk;
3446	return false;
3447	}
3448
3449	offset = map->m_lblk - es.es_lblk;
3450	map->m_len = es.es_len - offset;
3451
3452	return true;
3453	}
3454
3455	static int ext4_iomap_begin_report(struct inode *inode, loff_t offset,
3456	loff_t length, unsigned int flags,
3457	struct iomap *iomap, struct iomap *srcmap)
3458	{
3459	int ret;
3460	bool delalloc = false;
3461	struct ext4_map_blocks map;
3462	u8 blkbits = inode->i_blkbits;
3463
3464	if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3465	return -EINVAL;
3466
3467	if (ext4_has_inline_data(inode)) {
3468	ret = ext4_inline_data_iomap(inode, iomap);
3469	if (ret != -EAGAIN) {
3470	if (ret == 0 && offset >= iomap->length)
3471	ret = -ENOENT;
3472	return ret;
3473	}
3474	}
3475
3476	/*
3477	* Calculate the first and last logical block respectively.
3478	*/
3479	map.m_lblk = offset >> blkbits;
3480	map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3481	EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
3482
3483	/*
3484	* Fiemap callers may call for offset beyond s_bitmap_maxbytes.
3485	* So handle it here itself instead of querying ext4_map_blocks().
3486	* Since ext4_map_blocks() will warn about it and will return
3487	* -EIO error.
3488	*/
3489	if (!(ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))) {
3490	struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
3491
3492	if (offset >= sbi->s_bitmap_maxbytes) {
3493	map.m_flags = 0;
3494	goto set_iomap;
3495	}
3496	}
3497
3498	ret = ext4_map_blocks(NULL, inode, map: &map, flags: 0);
3499	if (ret < 0)
3500	return ret;
3501	if (ret == 0)
3502	delalloc = ext4_iomap_is_delalloc(inode, map: &map);
3503
3504	set_iomap:
3505	ext4_set_iomap(inode, iomap, map: &map, offset, length, flags);
3506	if (delalloc && iomap->type == IOMAP_HOLE)
3507	iomap->type = IOMAP_DELALLOC;
3508
3509	return 0;
3510	}
3511
3512	const struct iomap_ops ext4_iomap_report_ops = {
3513	.iomap_begin = ext4_iomap_begin_report,
3514	};
3515
3516	/*
3517	* For data=journal mode, folio should be marked dirty only when it was
3518	* writeably mapped. When that happens, it was already attached to the
3519	* transaction and marked as jbddirty (we take care of this in
3520	* ext4_page_mkwrite()). On transaction commit, we writeprotect page mappings
3521	* so we should have nothing to do here, except for the case when someone
3522	* had the page pinned and dirtied the page through this pin (e.g. by doing
3523	* direct IO to it). In that case we'd need to attach buffers here to the
3524	* transaction but we cannot due to lock ordering. We cannot just dirty the
3525	* folio and leave attached buffers clean, because the buffers' dirty state is
3526	* "definitive". We cannot just set the buffers dirty or jbddirty because all
3527	* the journalling code will explode. So what we do is to mark the folio
3528	* "pending dirty" and next time ext4_writepages() is called, attach buffers
3529	* to the transaction appropriately.
3530	*/
3531	static bool ext4_journalled_dirty_folio(struct address_space *mapping,
3532	struct folio *folio)
3533	{
3534	WARN_ON_ONCE(!folio_buffers(folio));
3535	if (folio_maybe_dma_pinned(folio))
3536	folio_set_checked(folio);
3537	return filemap_dirty_folio(mapping, folio);
3538	}
3539
3540	static bool ext4_dirty_folio(struct address_space *mapping, struct folio *folio)
3541	{
3542	WARN_ON_ONCE(!folio_test_locked(folio) && !folio_test_dirty(folio));
3543	WARN_ON_ONCE(!folio_buffers(folio));
3544	return block_dirty_folio(mapping, folio);
3545	}
3546
3547	static int ext4_iomap_swap_activate(struct swap_info_struct *sis,
3548	struct file *file, sector_t *span)
3549	{
3550	return iomap_swapfile_activate(sis, swap_file: file, pagespan: span,
3551	ops: &ext4_iomap_report_ops);
3552	}
3553
3554	static const struct address_space_operations ext4_aops = {
3555	.read_folio = ext4_read_folio,
3556	.readahead = ext4_readahead,
3557	.writepages = ext4_writepages,
3558	.write_begin = ext4_write_begin,
3559	.write_end = ext4_write_end,
3560	.dirty_folio = ext4_dirty_folio,
3561	.bmap = ext4_bmap,
3562	.invalidate_folio = ext4_invalidate_folio,
3563	.release_folio = ext4_release_folio,
3564	.direct_IO = noop_direct_IO,
3565	.migrate_folio = buffer_migrate_folio,
3566	.is_partially_uptodate = block_is_partially_uptodate,
3567	.error_remove_page = generic_error_remove_page,
3568	.swap_activate = ext4_iomap_swap_activate,
3569	};
3570
3571	static const struct address_space_operations ext4_journalled_aops = {
3572	.read_folio = ext4_read_folio,
3573	.readahead = ext4_readahead,
3574	.writepages = ext4_writepages,
3575	.write_begin = ext4_write_begin,
3576	.write_end = ext4_journalled_write_end,
3577	.dirty_folio = ext4_journalled_dirty_folio,
3578	.bmap = ext4_bmap,
3579	.invalidate_folio = ext4_journalled_invalidate_folio,
3580	.release_folio = ext4_release_folio,
3581	.direct_IO = noop_direct_IO,
3582	.migrate_folio = buffer_migrate_folio_norefs,
3583	.is_partially_uptodate = block_is_partially_uptodate,
3584	.error_remove_page = generic_error_remove_page,
3585	.swap_activate = ext4_iomap_swap_activate,
3586	};
3587
3588	static const struct address_space_operations ext4_da_aops = {
3589	.read_folio = ext4_read_folio,
3590	.readahead = ext4_readahead,
3591	.writepages = ext4_writepages,
3592	.write_begin = ext4_da_write_begin,
3593	.write_end = ext4_da_write_end,
3594	.dirty_folio = ext4_dirty_folio,
3595	.bmap = ext4_bmap,
3596	.invalidate_folio = ext4_invalidate_folio,
3597	.release_folio = ext4_release_folio,
3598	.direct_IO = noop_direct_IO,
3599	.migrate_folio = buffer_migrate_folio,
3600	.is_partially_uptodate = block_is_partially_uptodate,
3601	.error_remove_page = generic_error_remove_page,
3602	.swap_activate = ext4_iomap_swap_activate,
3603	};
3604
3605	static const struct address_space_operations ext4_dax_aops = {
3606	.writepages = ext4_dax_writepages,
3607	.direct_IO = noop_direct_IO,
3608	.dirty_folio = noop_dirty_folio,
3609	.bmap = ext4_bmap,
3610	.swap_activate = ext4_iomap_swap_activate,
3611	};
3612
3613	void ext4_set_aops(struct inode *inode)
3614	{
3615	switch (ext4_inode_journal_mode(inode)) {
3616	case EXT4_INODE_ORDERED_DATA_MODE:
3617	case EXT4_INODE_WRITEBACK_DATA_MODE:
3618	break;
3619	case EXT4_INODE_JOURNAL_DATA_MODE:
3620	inode->i_mapping->a_ops = &ext4_journalled_aops;
3621	return;
3622	default:
3623	BUG();
3624	}
3625	if (IS_DAX(inode))
3626	inode->i_mapping->a_ops = &ext4_dax_aops;
3627	else if (test_opt(inode->i_sb, DELALLOC))
3628	inode->i_mapping->a_ops = &ext4_da_aops;
3629	else
3630	inode->i_mapping->a_ops = &ext4_aops;
3631	}
3632
3633	static int __ext4_block_zero_page_range(handle_t *handle,
3634	struct address_space *mapping, loff_t from, loff_t length)
3635	{
3636	ext4_fsblk_t index = from >> PAGE_SHIFT;
3637	unsigned offset = from & (PAGE_SIZE-1);
3638	unsigned blocksize, pos;
3639	ext4_lblk_t iblock;
3640	struct inode *inode = mapping->host;
3641	struct buffer_head *bh;
3642	struct folio *folio;
3643	int err = 0;
3644
3645	folio = __filemap_get_folio(mapping, index: from >> PAGE_SHIFT,
3646	FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
3647	gfp: mapping_gfp_constraint(mapping, gfp_mask: ~__GFP_FS));
3648	if (IS_ERR(ptr: folio))
3649	return PTR_ERR(ptr: folio);
3650
3651	blocksize = inode->i_sb->s_blocksize;
3652
3653	iblock = index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits);
3654
3655	bh = folio_buffers(folio);
3656	if (!bh)
3657	bh = create_empty_buffers(folio, blocksize, b_state: 0);
3658
3659	/* Find the buffer that contains "offset" */
3660	pos = blocksize;
3661	while (offset >= pos) {
3662	bh = bh->b_this_page;
3663	iblock++;
3664	pos += blocksize;
3665	}
3666	if (buffer_freed(bh)) {
3667	BUFFER_TRACE(bh, "freed: skip");
3668	goto unlock;
3669	}
3670	if (!buffer_mapped(bh)) {
3671	BUFFER_TRACE(bh, "unmapped");
3672	ext4_get_block(inode, iblock, bh, create: 0);
3673	/* unmapped? It's a hole - nothing to do */
3674	if (!buffer_mapped(bh)) {
3675	BUFFER_TRACE(bh, "still unmapped");
3676	goto unlock;
3677	}
3678	}
3679
3680	/* Ok, it's mapped. Make sure it's up-to-date */
3681	if (folio_test_uptodate(folio))
3682	set_buffer_uptodate(bh);
3683
3684	if (!buffer_uptodate(bh)) {
3685	err = ext4_read_bh_lock(bh, op_flags: 0, wait: true);
3686	if (err)
3687	goto unlock;
3688	if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
3689	/* We expect the key to be set. */
3690	BUG_ON(!fscrypt_has_encryption_key(inode));
3691	err = fscrypt_decrypt_pagecache_blocks(folio,
3692	len: blocksize,
3693	offs: bh_offset(bh));
3694	if (err) {
3695	clear_buffer_uptodate(bh);
3696	goto unlock;
3697	}
3698	}
3699	}
3700	if (ext4_should_journal_data(inode)) {
3701	BUFFER_TRACE(bh, "get write access");
3702	err = ext4_journal_get_write_access(handle, inode->i_sb, bh,
3703	EXT4_JTR_NONE);
3704	if (err)
3705	goto unlock;
3706	}
3707	folio_zero_range(folio, start: offset, length);
3708	BUFFER_TRACE(bh, "zeroed end of block");
3709
3710	if (ext4_should_journal_data(inode)) {
3711	err = ext4_dirty_journalled_data(handle, bh);
3712	} else {
3713	err = 0;
3714	mark_buffer_dirty(bh);
3715	if (ext4_should_order_data(inode))
3716	err = ext4_jbd2_inode_add_write(handle, inode, start_byte: from,
3717	length);
3718	}
3719
3720	unlock:
3721	folio_unlock(folio);
3722	folio_put(folio);
3723	return err;
3724	}
3725
3726	/*
3727	* ext4_block_zero_page_range() zeros out a mapping of length 'length'
3728	* starting from file offset 'from'. The range to be zero'd must
3729	* be contained with in one block. If the specified range exceeds
3730	* the end of the block it will be shortened to end of the block
3731	* that corresponds to 'from'
3732	*/
3733	static int ext4_block_zero_page_range(handle_t *handle,
3734	struct address_space *mapping, loff_t from, loff_t length)
3735	{
3736	struct inode *inode = mapping->host;
3737	unsigned offset = from & (PAGE_SIZE-1);
3738	unsigned blocksize = inode->i_sb->s_blocksize;
3739	unsigned max = blocksize - (offset & (blocksize - 1));
3740
3741	/*
3742	* correct length if it does not fall between
3743	* 'from' and the end of the block
3744	*/
3745	if (length > max || length < 0)
3746	length = max;
3747
3748	if (IS_DAX(inode)) {
3749	return dax_zero_range(inode, pos: from, len: length, NULL,
3750	ops: &ext4_iomap_ops);
3751	}
3752	return __ext4_block_zero_page_range(handle, mapping, from, length);
3753	}
3754
3755	/*
3756	* ext4_block_truncate_page() zeroes out a mapping from file offset `from'
3757	* up to the end of the block which corresponds to `from'.
3758	* This required during truncate. We need to physically zero the tail end
3759	* of that block so it doesn't yield old data if the file is later grown.
3760	*/
3761	static int ext4_block_truncate_page(handle_t *handle,
3762	struct address_space *mapping, loff_t from)
3763	{
3764	unsigned offset = from & (PAGE_SIZE-1);
3765	unsigned length;
3766	unsigned blocksize;
3767	struct inode *inode = mapping->host;
3768
3769	/* If we are processing an encrypted inode during orphan list handling */
3770	if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode))
3771	return 0;
3772
3773	blocksize = inode->i_sb->s_blocksize;
3774	length = blocksize - (offset & (blocksize - 1));
3775
3776	return ext4_block_zero_page_range(handle, mapping, from, length);
3777	}
3778
3779	int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode,
3780	loff_t lstart, loff_t length)
3781	{
3782	struct super_block *sb = inode->i_sb;
3783	struct address_space *mapping = inode->i_mapping;
3784	unsigned partial_start, partial_end;
3785	ext4_fsblk_t start, end;
3786	loff_t byte_end = (lstart + length - 1);
3787	int err = 0;
3788
3789	partial_start = lstart & (sb->s_blocksize - 1);
3790	partial_end = byte_end & (sb->s_blocksize - 1);
3791
3792	start = lstart >> sb->s_blocksize_bits;
3793	end = byte_end >> sb->s_blocksize_bits;
3794
3795	/* Handle partial zero within the single block */
3796	if (start == end &&
3797	(partial_start || (partial_end != sb->s_blocksize - 1))) {
3798	err = ext4_block_zero_page_range(handle, mapping,
3799	from: lstart, length);
3800	return err;
3801	}
3802	/* Handle partial zero out on the start of the range */
3803	if (partial_start) {
3804	err = ext4_block_zero_page_range(handle, mapping,
3805	from: lstart, length: sb->s_blocksize);
3806	if (err)
3807	return err;
3808	}
3809	/* Handle partial zero out on the end of the range */
3810	if (partial_end != sb->s_blocksize - 1)
3811	err = ext4_block_zero_page_range(handle, mapping,
3812	from: byte_end - partial_end,
3813	length: partial_end + 1);
3814	return err;
3815	}
3816
3817	int ext4_can_truncate(struct inode *inode)
3818	{
3819	if (S_ISREG(inode->i_mode))
3820	return 1;
3821	if (S_ISDIR(inode->i_mode))
3822	return 1;
3823	if (S_ISLNK(inode->i_mode))
3824	return !ext4_inode_is_fast_symlink(inode);
3825	return 0;
3826	}
3827
3828	/*
3829	* We have to make sure i_disksize gets properly updated before we truncate
3830	* page cache due to hole punching or zero range. Otherwise i_disksize update
3831	* can get lost as it may have been postponed to submission of writeback but
3832	* that will never happen after we truncate page cache.
3833	*/
3834	int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset,
3835	loff_t len)
3836	{
3837	handle_t *handle;
3838	int ret;
3839
3840	loff_t size = i_size_read(inode);
3841
3842	WARN_ON(!inode_is_locked(inode));
3843	if (offset > size || offset + len < size)
3844	return 0;
3845
3846	if (EXT4_I(inode)->i_disksize >= size)
3847	return 0;
3848
3849	handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
3850	if (IS_ERR(ptr: handle))
3851	return PTR_ERR(ptr: handle);
3852	ext4_update_i_disksize(inode, newsize: size);
3853	ret = ext4_mark_inode_dirty(handle, inode);
3854	ext4_journal_stop(handle);
3855
3856	return ret;
3857	}
3858
3859	static void ext4_wait_dax_page(struct inode *inode)
3860	{
3861	filemap_invalidate_unlock(mapping: inode->i_mapping);
3862	schedule();
3863	filemap_invalidate_lock(mapping: inode->i_mapping);
3864	}
3865
3866	int ext4_break_layouts(struct inode *inode)
3867	{
3868	struct page *page;
3869	int error;
3870
3871	if (WARN_ON_ONCE(!rwsem_is_locked(&inode->i_mapping->invalidate_lock)))
3872	return -EINVAL;
3873
3874	do {
3875	page = dax_layout_busy_page(mapping: inode->i_mapping);
3876	if (!page)
3877	return 0;
3878
3879	error = ___wait_var_event(&page->_refcount,
3880	atomic_read(&page->_refcount) == 1,
3881	TASK_INTERRUPTIBLE, 0, 0,
3882	ext4_wait_dax_page(inode));
3883	} while (error == 0);
3884
3885	return error;
3886	}
3887
3888	/*
3889	* ext4_punch_hole: punches a hole in a file by releasing the blocks
3890	* associated with the given offset and length
3891	*
3892	* @inode: File inode
3893	* @offset: The offset where the hole will begin
3894	* @len: The length of the hole
3895	*
3896	* Returns: 0 on success or negative on failure
3897	*/
3898
3899	int ext4_punch_hole(struct file *file, loff_t offset, loff_t length)
3900	{
3901	struct inode *inode = file_inode(f: file);
3902	struct super_block *sb = inode->i_sb;
3903	ext4_lblk_t first_block, stop_block;
3904	struct address_space *mapping = inode->i_mapping;
3905	loff_t first_block_offset, last_block_offset, max_length;
3906	struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
3907	handle_t *handle;
3908	unsigned int credits;
3909	int ret = 0, ret2 = 0;
3910
3911	trace_ext4_punch_hole(inode, offset, len: length, mode: 0);
3912
3913	/*
3914	* Write out all dirty pages to avoid race conditions
3915	* Then release them.
3916	*/
3917	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
3918	ret = filemap_write_and_wait_range(mapping, lstart: offset,
3919	lend: offset + length - 1);
3920	if (ret)
3921	return ret;
3922	}
3923
3924	inode_lock(inode);
3925
3926	/* No need to punch hole beyond i_size */
3927	if (offset >= inode->i_size)
3928	goto out_mutex;
3929
3930	/*
3931	* If the hole extends beyond i_size, set the hole
3932	* to end after the page that contains i_size
3933	*/
3934	if (offset + length > inode->i_size) {
3935	length = inode->i_size +
3936	PAGE_SIZE - (inode->i_size & (PAGE_SIZE - 1)) -
3937	offset;
3938	}
3939
3940	/*
3941	* For punch hole the length + offset needs to be within one block
3942	* before last range. Adjust the length if it goes beyond that limit.
3943	*/
3944	max_length = sbi->s_bitmap_maxbytes - inode->i_sb->s_blocksize;
3945	if (offset + length > max_length)
3946	length = max_length - offset;
3947
3948	if (offset & (sb->s_blocksize - 1) ||
3949	(offset + length) & (sb->s_blocksize - 1)) {
3950	/*
3951	* Attach jinode to inode for jbd2 if we do any zeroing of
3952	* partial block
3953	*/
3954	ret = ext4_inode_attach_jinode(inode);
3955	if (ret < 0)
3956	goto out_mutex;
3957
3958	}
3959
3960	/* Wait all existing dio workers, newcomers will block on i_rwsem */
3961	inode_dio_wait(inode);
3962
3963	ret = file_modified(file);
3964	if (ret)
3965	goto out_mutex;
3966
3967	/*
3968	* Prevent page faults from reinstantiating pages we have released from
3969	* page cache.
3970	*/
3971	filemap_invalidate_lock(mapping);
3972
3973	ret = ext4_break_layouts(inode);
3974	if (ret)
3975	goto out_dio;
3976
3977	first_block_offset = round_up(offset, sb->s_blocksize);
3978	last_block_offset = round_down((offset + length), sb->s_blocksize) - 1;
3979
3980	/* Now release the pages and zero block aligned part of pages*/
3981	if (last_block_offset > first_block_offset) {
3982	ret = ext4_update_disksize_before_punch(inode, offset, len: length);
3983	if (ret)
3984	goto out_dio;
3985	truncate_pagecache_range(inode, offset: first_block_offset,
3986	end: last_block_offset);
3987	}
3988
3989	if (ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))
3990	credits = ext4_writepage_trans_blocks(inode);
3991	else
3992	credits = ext4_blocks_for_truncate(inode);
3993	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
3994	if (IS_ERR(ptr: handle)) {
3995	ret = PTR_ERR(ptr: handle);
3996	ext4_std_error(sb, ret);
3997	goto out_dio;
3998	}
3999
4000	ret = ext4_zero_partial_blocks(handle, inode, lstart: offset,
4001	length);
4002	if (ret)
4003	goto out_stop;
4004
4005	first_block = (offset + sb->s_blocksize - 1) >>
4006	EXT4_BLOCK_SIZE_BITS(sb);
4007	stop_block = (offset + length) >> EXT4_BLOCK_SIZE_BITS(sb);
4008
4009	/* If there are blocks to remove, do it */
4010	if (stop_block > first_block) {
4011
4012	down_write(sem: &EXT4_I(inode)->i_data_sem);
4013	ext4_discard_preallocations(inode, 0);
4014
4015	ext4_es_remove_extent(inode, lblk: first_block,
4016	len: stop_block - first_block);
4017
4018	if (ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))
4019	ret = ext4_ext_remove_space(inode, start: first_block,
4020	end: stop_block - 1);
4021	else
4022	ret = ext4_ind_remove_space(handle, inode, start: first_block,
4023	end: stop_block);
4024
4025	up_write(sem: &EXT4_I(inode)->i_data_sem);
4026	}
4027	ext4_fc_track_range(handle, inode, start: first_block, end: stop_block);
4028	if (IS_SYNC(inode))
4029	ext4_handle_sync(handle);
4030
4031	inode_set_mtime_to_ts(inode, ts: inode_set_ctime_current(inode));
4032	ret2 = ext4_mark_inode_dirty(handle, inode);
4033	if (unlikely(ret2))
4034	ret = ret2;
4035	if (ret >= 0)
4036	ext4_update_inode_fsync_trans(handle, inode, datasync: 1);
4037	out_stop:
4038	ext4_journal_stop(handle);
4039	out_dio:
4040	filemap_invalidate_unlock(mapping);
4041	out_mutex:
4042	inode_unlock(inode);
4043	return ret;
4044	}
4045
4046	int ext4_inode_attach_jinode(struct inode *inode)
4047	{
4048	struct ext4_inode_info *ei = EXT4_I(inode);
4049	struct jbd2_inode *jinode;
4050
4051	if (ei->jinode || !EXT4_SB(sb: inode->i_sb)->s_journal)
4052	return 0;
4053
4054	jinode = jbd2_alloc_inode(GFP_KERNEL);
4055	spin_lock(lock: &inode->i_lock);
4056	if (!ei->jinode) {
4057	if (!jinode) {
4058	spin_unlock(lock: &inode->i_lock);
4059	return -ENOMEM;
4060	}
4061	ei->jinode = jinode;
4062	jbd2_journal_init_jbd_inode(jinode: ei->jinode, inode);
4063	jinode = NULL;
4064	}
4065	spin_unlock(lock: &inode->i_lock);
4066	if (unlikely(jinode != NULL))
4067	jbd2_free_inode(jinode);
4068	return 0;
4069	}
4070
4071	/*
4072	* ext4_truncate()
4073	*
4074	* We block out ext4_get_block() block instantiations across the entire
4075	* transaction, and VFS/VM ensures that ext4_truncate() cannot run
4076	* simultaneously on behalf of the same inode.
4077	*
4078	* As we work through the truncate and commit bits of it to the journal there
4079	* is one core, guiding principle: the file's tree must always be consistent on
4080	* disk. We must be able to restart the truncate after a crash.
4081	*
4082	* The file's tree may be transiently inconsistent in memory (although it
4083	* probably isn't), but whenever we close off and commit a journal transaction,
4084	* the contents of (the filesystem + the journal) must be consistent and
4085	* restartable. It's pretty simple, really: bottom up, right to left (although
4086	* left-to-right works OK too).
4087	*
4088	* Note that at recovery time, journal replay occurs *before* the restart of
4089	* truncate against the orphan inode list.
4090	*
4091	* The committed inode has the new, desired i_size (which is the same as
4092	* i_disksize in this case). After a crash, ext4_orphan_cleanup() will see
4093	* that this inode's truncate did not complete and it will again call
4094	* ext4_truncate() to have another go. So there will be instantiated blocks
4095	* to the right of the truncation point in a crashed ext4 filesystem. But
4096	* that's fine - as long as they are linked from the inode, the post-crash
4097	* ext4_truncate() run will find them and release them.
4098	*/
4099	int ext4_truncate(struct inode *inode)
4100	{
4101	struct ext4_inode_info *ei = EXT4_I(inode);
4102	unsigned int credits;
4103	int err = 0, err2;
4104	handle_t *handle;
4105	struct address_space *mapping = inode->i_mapping;
4106
4107	/*
4108	* There is a possibility that we're either freeing the inode
4109	* or it's a completely new inode. In those cases we might not
4110	* have i_rwsem locked because it's not necessary.
4111	*/
4112	if (!(inode->i_state & (I_NEW|I_FREEING)))
4113	WARN_ON(!inode_is_locked(inode));
4114	trace_ext4_truncate_enter(inode);
4115
4116	if (!ext4_can_truncate(inode))
4117	goto out_trace;
4118
4119	if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
4120	ext4_set_inode_state(inode, bit: EXT4_STATE_DA_ALLOC_CLOSE);
4121
4122	if (ext4_has_inline_data(inode)) {
4123	int has_inline = 1;
4124
4125	err = ext4_inline_data_truncate(inode, has_inline: &has_inline);
4126	if (err || has_inline)
4127	goto out_trace;
4128	}
4129
4130	/* If we zero-out tail of the page, we have to create jinode for jbd2 */
4131	if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {
4132	err = ext4_inode_attach_jinode(inode);
4133	if (err)
4134	goto out_trace;
4135	}
4136
4137	if (ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))
4138	credits = ext4_writepage_trans_blocks(inode);
4139	else
4140	credits = ext4_blocks_for_truncate(inode);
4141
4142	handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4143	if (IS_ERR(ptr: handle)) {
4144	err = PTR_ERR(ptr: handle);
4145	goto out_trace;
4146	}
4147
4148	if (inode->i_size & (inode->i_sb->s_blocksize - 1))
4149	ext4_block_truncate_page(handle, mapping, from: inode->i_size);
4150
4151	/*
4152	* We add the inode to the orphan list, so that if this
4153	* truncate spans multiple transactions, and we crash, we will
4154	* resume the truncate when the filesystem recovers. It also
4155	* marks the inode dirty, to catch the new size.
4156	*
4157	* Implication: the file must always be in a sane, consistent
4158	* truncatable state while each transaction commits.
4159	*/
4160	err = ext4_orphan_add(handle, inode);
4161	if (err)
4162	goto out_stop;
4163
4164	down_write(sem: &EXT4_I(inode)->i_data_sem);
4165
4166	ext4_discard_preallocations(inode, 0);
4167
4168	if (ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))
4169	err = ext4_ext_truncate(handle, inode);
4170	else
4171	ext4_ind_truncate(handle, inode);
4172
4173	up_write(sem: &ei->i_data_sem);
4174	if (err)
4175	goto out_stop;
4176
4177	if (IS_SYNC(inode))
4178	ext4_handle_sync(handle);
4179
4180	out_stop:
4181	/*
4182	* If this was a simple ftruncate() and the file will remain alive,
4183	* then we need to clear up the orphan record which we created above.
4184	* However, if this was a real unlink then we were called by
4185	* ext4_evict_inode(), and we allow that function to clean up the
4186	* orphan info for us.
4187	*/
4188	if (inode->i_nlink)
4189	ext4_orphan_del(handle, inode);
4190
4191	inode_set_mtime_to_ts(inode, ts: inode_set_ctime_current(inode));
4192	err2 = ext4_mark_inode_dirty(handle, inode);
4193	if (unlikely(err2 && !err))
4194	err = err2;
4195	ext4_journal_stop(handle);
4196
4197	out_trace:
4198	trace_ext4_truncate_exit(inode);
4199	return err;
4200	}
4201
4202	static inline u64 ext4_inode_peek_iversion(const struct inode *inode)
4203	{
4204	if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4205	return inode_peek_iversion_raw(inode);
4206	else
4207	return inode_peek_iversion(inode);
4208	}
4209
4210	static int ext4_inode_blocks_set(struct ext4_inode *raw_inode,
4211	struct ext4_inode_info *ei)
4212	{
4213	struct inode *inode = &(ei->vfs_inode);
4214	u64 i_blocks = READ_ONCE(inode->i_blocks);
4215	struct super_block *sb = inode->i_sb;
4216
4217	if (i_blocks <= ~0U) {
4218	/*
4219	* i_blocks can be represented in a 32 bit variable
4220	* as multiple of 512 bytes
4221	*/
4222	raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
4223	raw_inode->i_blocks_high = 0;
4224	ext4_clear_inode_flag(inode, bit: EXT4_INODE_HUGE_FILE);
4225	return 0;
4226	}
4227
4228	/*
4229	* This should never happen since sb->s_maxbytes should not have
4230	* allowed this, sb->s_maxbytes was set according to the huge_file
4231	* feature in ext4_fill_super().
4232	*/
4233	if (!ext4_has_feature_huge_file(sb))
4234	return -EFSCORRUPTED;
4235
4236	if (i_blocks <= 0xffffffffffffULL) {
4237	/*
4238	* i_blocks can be represented in a 48 bit variable
4239	* as multiple of 512 bytes
4240	*/
4241	raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
4242	raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
4243	ext4_clear_inode_flag(inode, bit: EXT4_INODE_HUGE_FILE);
4244	} else {
4245	ext4_set_inode_flag(inode, bit: EXT4_INODE_HUGE_FILE);
4246	/* i_block is stored in file system block size */
4247	i_blocks = i_blocks >> (inode->i_blkbits - 9);
4248	raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
4249	raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
4250	}
4251	return 0;
4252	}
4253
4254	static int ext4_fill_raw_inode(struct inode *inode, struct ext4_inode *raw_inode)
4255	{
4256	struct ext4_inode_info *ei = EXT4_I(inode);
4257	uid_t i_uid;
4258	gid_t i_gid;
4259	projid_t i_projid;
4260	int block;
4261	int err;
4262
4263	err = ext4_inode_blocks_set(raw_inode, ei);
4264
4265	raw_inode->i_mode = cpu_to_le16(inode->i_mode);
4266	i_uid = i_uid_read(inode);
4267	i_gid = i_gid_read(inode);
4268	i_projid = from_kprojid(to: &init_user_ns, projid: ei->i_projid);
4269	if (!(test_opt(inode->i_sb, NO_UID32))) {
4270	raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid));
4271	raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid));
4272	/*
4273	* Fix up interoperability with old kernels. Otherwise,
4274	* old inodes get re-used with the upper 16 bits of the
4275	* uid/gid intact.
4276	*/
4277	if (ei->i_dtime && list_empty(head: &ei->i_orphan)) {
4278	raw_inode->i_uid_high = 0;
4279	raw_inode->i_gid_high = 0;
4280	} else {
4281	raw_inode->i_uid_high =
4282	cpu_to_le16(high_16_bits(i_uid));
4283	raw_inode->i_gid_high =
4284	cpu_to_le16(high_16_bits(i_gid));
4285	}
4286	} else {
4287	raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid));
4288	raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid));
4289	raw_inode->i_uid_high = 0;
4290	raw_inode->i_gid_high = 0;
4291	}
4292	raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);
4293
4294	EXT4_INODE_SET_CTIME(inode, raw_inode);
4295	EXT4_INODE_SET_MTIME(inode, raw_inode);
4296	EXT4_INODE_SET_ATIME(inode, raw_inode);
4297	EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);
4298
4299	raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
4300	raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);
4301	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT)))
4302	raw_inode->i_file_acl_high =
4303	cpu_to_le16(ei->i_file_acl >> 32);
4304	raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);
4305	ext4_isize_set(raw_inode, i_size: ei->i_disksize);
4306
4307	raw_inode->i_generation = cpu_to_le32(inode->i_generation);
4308	if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
4309	if (old_valid_dev(dev: inode->i_rdev)) {
4310	raw_inode->i_block[0] =
4311	cpu_to_le32(old_encode_dev(inode->i_rdev));
4312	raw_inode->i_block[1] = 0;
4313	} else {
4314	raw_inode->i_block[0] = 0;
4315	raw_inode->i_block[1] =
4316	cpu_to_le32(new_encode_dev(inode->i_rdev));
4317	raw_inode->i_block[2] = 0;
4318	}
4319	} else if (!ext4_has_inline_data(inode)) {
4320	for (block = 0; block < EXT4_N_BLOCKS; block++)
4321	raw_inode->i_block[block] = ei->i_data[block];
4322	}
4323
4324	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
4325	u64 ivers = ext4_inode_peek_iversion(inode);
4326
4327	raw_inode->i_disk_version = cpu_to_le32(ivers);
4328	if (ei->i_extra_isize) {
4329	if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
4330	raw_inode->i_version_hi =
4331	cpu_to_le32(ivers >> 32);
4332	raw_inode->i_extra_isize =
4333	cpu_to_le16(ei->i_extra_isize);
4334	}
4335	}
4336
4337	if (i_projid != EXT4_DEF_PROJID &&
4338	!ext4_has_feature_project(sb: inode->i_sb))
4339	err = err ?: -EFSCORRUPTED;
4340
4341	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
4342	EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
4343	raw_inode->i_projid = cpu_to_le32(i_projid);
4344
4345	ext4_inode_csum_set(inode, raw: raw_inode, ei);
4346	return err;
4347	}
4348
4349	/*
4350	* ext4_get_inode_loc returns with an extra refcount against the inode's
4351	* underlying buffer_head on success. If we pass 'inode' and it does not
4352	* have in-inode xattr, we have all inode data in memory that is needed
4353	* to recreate the on-disk version of this inode.
4354	*/
4355	static int __ext4_get_inode_loc(struct super_block *sb, unsigned long ino,
4356	struct inode *inode, struct ext4_iloc *iloc,
4357	ext4_fsblk_t *ret_block)
4358	{
4359	struct ext4_group_desc *gdp;
4360	struct buffer_head *bh;
4361	ext4_fsblk_t block;
4362	struct blk_plug plug;
4363	int inodes_per_block, inode_offset;
4364
4365	iloc->bh = NULL;
4366	if (ino < EXT4_ROOT_INO ||
4367	ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))
4368	return -EFSCORRUPTED;
4369
4370	iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb);
4371	gdp = ext4_get_group_desc(sb, block_group: iloc->block_group, NULL);
4372	if (!gdp)
4373	return -EIO;
4374
4375	/*
4376	* Figure out the offset within the block group inode table
4377	*/
4378	inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
4379	inode_offset = ((ino - 1) %
4380	EXT4_INODES_PER_GROUP(sb));
4381	iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
4382
4383	block = ext4_inode_table(sb, bg: gdp);
4384	if ((block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) ||
4385	(block >= ext4_blocks_count(es: EXT4_SB(sb)->s_es))) {
4386	ext4_error(sb, "Invalid inode table block %llu in "
4387	"block_group %u", block, iloc->block_group);
4388	return -EFSCORRUPTED;
4389	}
4390	block += (inode_offset / inodes_per_block);
4391
4392	bh = sb_getblk(sb, block);
4393	if (unlikely(!bh))
4394	return -ENOMEM;
4395	if (ext4_buffer_uptodate(bh))
4396	goto has_buffer;
4397
4398	lock_buffer(bh);
4399	if (ext4_buffer_uptodate(bh)) {
4400	/* Someone brought it uptodate while we waited */
4401	unlock_buffer(bh);
4402	goto has_buffer;
4403	}
4404
4405	/*
4406	* If we have all information of the inode in memory and this
4407	* is the only valid inode in the block, we need not read the
4408	* block.
4409	*/
4410	if (inode && !ext4_test_inode_state(inode, bit: EXT4_STATE_XATTR)) {
4411	struct buffer_head *bitmap_bh;
4412	int i, start;
4413
4414	start = inode_offset & ~(inodes_per_block - 1);
4415
4416	/* Is the inode bitmap in cache? */
4417	bitmap_bh = sb_getblk(sb, block: ext4_inode_bitmap(sb, bg: gdp));
4418	if (unlikely(!bitmap_bh))
4419	goto make_io;
4420
4421	/*
4422	* If the inode bitmap isn't in cache then the
4423	* optimisation may end up performing two reads instead
4424	* of one, so skip it.
4425	*/
4426	if (!buffer_uptodate(bh: bitmap_bh)) {
4427	brelse(bh: bitmap_bh);
4428	goto make_io;
4429	}
4430	for (i = start; i < start + inodes_per_block; i++) {
4431	if (i == inode_offset)
4432	continue;
4433	if (ext4_test_bit(nr: i, addr: bitmap_bh->b_data))
4434	break;
4435	}
4436	brelse(bh: bitmap_bh);
4437	if (i == start + inodes_per_block) {
4438	struct ext4_inode *raw_inode =
4439	(struct ext4_inode *) (bh->b_data + iloc->offset);
4440
4441	/* all other inodes are free, so skip I/O */
4442	memset(bh->b_data, 0, bh->b_size);
4443	if (!ext4_test_inode_state(inode, bit: EXT4_STATE_NEW))
4444	ext4_fill_raw_inode(inode, raw_inode);
4445	set_buffer_uptodate(bh);
4446	unlock_buffer(bh);
4447	goto has_buffer;
4448	}
4449	}
4450
4451	make_io:
4452	/*
4453	* If we need to do any I/O, try to pre-readahead extra
4454	* blocks from the inode table.
4455	*/
4456	blk_start_plug(&plug);
4457	if (EXT4_SB(sb)->s_inode_readahead_blks) {
4458	ext4_fsblk_t b, end, table;
4459	unsigned num;
4460	__u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks;
4461
4462	table = ext4_inode_table(sb, bg: gdp);
4463	/* s_inode_readahead_blks is always a power of 2 */
4464	b = block & ~((ext4_fsblk_t) ra_blks - 1);
4465	if (table > b)
4466	b = table;
4467	end = b + ra_blks;
4468	num = EXT4_INODES_PER_GROUP(sb);
4469	if (ext4_has_group_desc_csum(sb))
4470	num -= ext4_itable_unused_count(sb, bg: gdp);
4471	table += num / inodes_per_block;
4472	if (end > table)
4473	end = table;
4474	while (b <= end)
4475	ext4_sb_breadahead_unmovable(sb, block: b++);
4476	}
4477
4478	/*
4479	* There are other valid inodes in the buffer, this inode
4480	* has in-inode xattrs, or we don't have this inode in memory.
4481	* Read the block from disk.
4482	*/
4483	trace_ext4_load_inode(sb, ino);
4484	ext4_read_bh_nowait(bh, REQ_META | REQ_PRIO, NULL);
4485	blk_finish_plug(&plug);
4486	wait_on_buffer(bh);
4487	ext4_simulate_fail_bh(sb, bh, EXT4_SIM_INODE_EIO);
4488	if (!buffer_uptodate(bh)) {
4489	if (ret_block)
4490	*ret_block = block;
4491	brelse(bh);
4492	return -EIO;
4493	}
4494	has_buffer:
4495	iloc->bh = bh;
4496	return 0;
4497	}
4498
4499	static int __ext4_get_inode_loc_noinmem(struct inode *inode,
4500	struct ext4_iloc *iloc)
4501	{
4502	ext4_fsblk_t err_blk = 0;
4503	int ret;
4504
4505	ret = __ext4_get_inode_loc(sb: inode->i_sb, ino: inode->i_ino, NULL, iloc,
4506	ret_block: &err_blk);
4507
4508	if (ret == -EIO)
4509	ext4_error_inode_block(inode, err_blk, EIO,
4510	"unable to read itable block");
4511
4512	return ret;
4513	}
4514
4515	int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
4516	{
4517	ext4_fsblk_t err_blk = 0;
4518	int ret;
4519
4520	ret = __ext4_get_inode_loc(sb: inode->i_sb, ino: inode->i_ino, inode, iloc,
4521	ret_block: &err_blk);
4522
4523	if (ret == -EIO)
4524	ext4_error_inode_block(inode, err_blk, EIO,
4525	"unable to read itable block");
4526
4527	return ret;
4528	}
4529
4530
4531	int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino,
4532	struct ext4_iloc *iloc)
4533	{
4534	return __ext4_get_inode_loc(sb, ino, NULL, iloc, NULL);
4535	}
4536
4537	static bool ext4_should_enable_dax(struct inode *inode)
4538	{
4539	struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
4540
4541	if (test_opt2(inode->i_sb, DAX_NEVER))
4542	return false;
4543	if (!S_ISREG(inode->i_mode))
4544	return false;
4545	if (ext4_should_journal_data(inode))
4546	return false;
4547	if (ext4_has_inline_data(inode))
4548	return false;
4549	if (ext4_test_inode_flag(inode, bit: EXT4_INODE_ENCRYPT))
4550	return false;
4551	if (ext4_test_inode_flag(inode, bit: EXT4_INODE_VERITY))
4552	return false;
4553	if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags))
4554	return false;
4555	if (test_opt(inode->i_sb, DAX_ALWAYS))
4556	return true;
4557
4558	return ext4_test_inode_flag(inode, bit: EXT4_INODE_DAX);
4559	}
4560
4561	void ext4_set_inode_flags(struct inode *inode, bool init)
4562	{
4563	unsigned int flags = EXT4_I(inode)->i_flags;
4564	unsigned int new_fl = 0;
4565
4566	WARN_ON_ONCE(IS_DAX(inode) && init);
4567
4568	if (flags & EXT4_SYNC_FL)
4569	new_fl |= S_SYNC;
4570	if (flags & EXT4_APPEND_FL)
4571	new_fl |= S_APPEND;
4572	if (flags & EXT4_IMMUTABLE_FL)
4573	new_fl |= S_IMMUTABLE;
4574	if (flags & EXT4_NOATIME_FL)
4575	new_fl |= S_NOATIME;
4576	if (flags & EXT4_DIRSYNC_FL)
4577	new_fl |= S_DIRSYNC;
4578
4579	/* Because of the way inode_set_flags() works we must preserve S_DAX
4580	* here if already set. */
4581	new_fl |= (inode->i_flags & S_DAX);
4582	if (init && ext4_should_enable_dax(inode))
4583	new_fl |= S_DAX;
4584
4585	if (flags & EXT4_ENCRYPT_FL)
4586	new_fl |= S_ENCRYPTED;
4587	if (flags & EXT4_CASEFOLD_FL)
4588	new_fl |= S_CASEFOLD;
4589	if (flags & EXT4_VERITY_FL)
4590	new_fl |= S_VERITY;
4591	inode_set_flags(inode, flags: new_fl,
4592	S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX|
4593	S_ENCRYPTED|S_CASEFOLD|S_VERITY);
4594	}
4595
4596	static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,
4597	struct ext4_inode_info *ei)
4598	{
4599	blkcnt_t i_blocks ;
4600	struct inode *inode = &(ei->vfs_inode);
4601	struct super_block *sb = inode->i_sb;
4602
4603	if (ext4_has_feature_huge_file(sb)) {
4604	/* we are using combined 48 bit field */
4605	i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |
4606	le32_to_cpu(raw_inode->i_blocks_lo);
4607	if (ext4_test_inode_flag(inode, bit: EXT4_INODE_HUGE_FILE)) {
4608	/* i_blocks represent file system block size */
4609	return i_blocks << (inode->i_blkbits - 9);
4610	} else {
4611	return i_blocks;
4612	}
4613	} else {
4614	return le32_to_cpu(raw_inode->i_blocks_lo);
4615	}
4616	}
4617
4618	static inline int ext4_iget_extra_inode(struct inode *inode,
4619	struct ext4_inode *raw_inode,
4620	struct ext4_inode_info *ei)
4621	{
4622	__le32 *magic = (void *)raw_inode +
4623	EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;
4624
4625	if (EXT4_INODE_HAS_XATTR_SPACE(inode) &&
4626	*magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
4627	int err;
4628
4629	ext4_set_inode_state(inode, bit: EXT4_STATE_XATTR);
4630	err = ext4_find_inline_data_nolock(inode);
4631	if (!err && ext4_has_inline_data(inode))
4632	ext4_set_inode_state(inode, bit: EXT4_STATE_MAY_INLINE_DATA);
4633	return err;
4634	} else
4635	EXT4_I(inode)->i_inline_off = 0;
4636	return 0;
4637	}
4638
4639	int ext4_get_projid(struct inode *inode, kprojid_t *projid)
4640	{
4641	if (!ext4_has_feature_project(sb: inode->i_sb))
4642	return -EOPNOTSUPP;
4643	*projid = EXT4_I(inode)->i_projid;
4644	return 0;
4645	}
4646
4647	/*
4648	* ext4 has self-managed i_version for ea inodes, it stores the lower 32bit of
4649	* refcount in i_version, so use raw values if inode has EXT4_EA_INODE_FL flag
4650	* set.
4651	*/
4652	static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val)
4653	{
4654	if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4655	inode_set_iversion_raw(inode, val);
4656	else
4657	inode_set_iversion_queried(inode, val);
4658	}
4659
4660	static const char *check_igot_inode(struct inode *inode, ext4_iget_flags flags)
4661
4662	{
4663	if (flags & EXT4_IGET_EA_INODE) {
4664	if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4665	return "missing EA_INODE flag";
4666	if (ext4_test_inode_state(inode, bit: EXT4_STATE_XATTR) ||
4667	EXT4_I(inode)->i_file_acl)
4668	return "ea_inode with extended attributes";
4669	} else {
4670	if ((EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4671	return "unexpected EA_INODE flag";
4672	}
4673	if (is_bad_inode(inode) && !(flags & EXT4_IGET_BAD))
4674	return "unexpected bad inode w/o EXT4_IGET_BAD";
4675	return NULL;
4676	}
4677
4678	struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
4679	ext4_iget_flags flags, const char *function,
4680	unsigned int line)
4681	{
4682	struct ext4_iloc iloc;
4683	struct ext4_inode *raw_inode;
4684	struct ext4_inode_info *ei;
4685	struct ext4_super_block *es = EXT4_SB(sb)->s_es;
4686	struct inode *inode;
4687	const char *err_str;
4688	journal_t *journal = EXT4_SB(sb)->s_journal;
4689	long ret;
4690	loff_t size;
4691	int block;
4692	uid_t i_uid;
4693	gid_t i_gid;
4694	projid_t i_projid;
4695
4696	if ((!(flags & EXT4_IGET_SPECIAL) &&
4697	((ino < EXT4_FIRST_INO(sb) && ino != EXT4_ROOT_INO) ||
4698	ino == le32_to_cpu(es->s_usr_quota_inum) ||
4699	ino == le32_to_cpu(es->s_grp_quota_inum) ||
4700	ino == le32_to_cpu(es->s_prj_quota_inum) ||
4701	ino == le32_to_cpu(es->s_orphan_file_inum))) ||
4702	(ino < EXT4_ROOT_INO) ||
4703	(ino > le32_to_cpu(es->s_inodes_count))) {
4704	if (flags & EXT4_IGET_HANDLE)
4705	return ERR_PTR(error: -ESTALE);
4706	__ext4_error(sb, function, line, false, EFSCORRUPTED, 0,
4707	"inode #%lu: comm %s: iget: illegal inode #",
4708	ino, current->comm);
4709	return ERR_PTR(error: -EFSCORRUPTED);
4710	}
4711
4712	inode = iget_locked(sb, ino);
4713	if (!inode)
4714	return ERR_PTR(error: -ENOMEM);
4715	if (!(inode->i_state & I_NEW)) {
4716	if ((err_str = check_igot_inode(inode, flags)) != NULL) {
4717	ext4_error_inode(inode, function, line, 0, err_str);
4718	iput(inode);
4719	return ERR_PTR(error: -EFSCORRUPTED);
4720	}
4721	return inode;
4722	}
4723
4724	ei = EXT4_I(inode);
4725	iloc.bh = NULL;
4726
4727	ret = __ext4_get_inode_loc_noinmem(inode, iloc: &iloc);
4728	if (ret < 0)
4729	goto bad_inode;
4730	raw_inode = ext4_raw_inode(iloc: &iloc);
4731
4732	if ((flags & EXT4_IGET_HANDLE) &&
4733	(raw_inode->i_links_count == 0) && (raw_inode->i_mode == 0)) {
4734	ret = -ESTALE;
4735	goto bad_inode;
4736	}
4737
4738	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4739	ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
4740	if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
4741	EXT4_INODE_SIZE(inode->i_sb) ||
4742	(ei->i_extra_isize & 3)) {
4743	ext4_error_inode(inode, function, line, 0,
4744	"iget: bad extra_isize %u "
4745	"(inode size %u)",
4746	ei->i_extra_isize,
4747	EXT4_INODE_SIZE(inode->i_sb));
4748	ret = -EFSCORRUPTED;
4749	goto bad_inode;
4750	}
4751	} else
4752	ei->i_extra_isize = 0;
4753
4754	/* Precompute checksum seed for inode metadata */
4755	if (ext4_has_metadata_csum(sb)) {
4756	struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
4757	__u32 csum;
4758	__le32 inum = cpu_to_le32(inode->i_ino);
4759	__le32 gen = raw_inode->i_generation;
4760	csum = ext4_chksum(sbi, crc: sbi->s_csum_seed, address: (__u8 *)&inum,
4761	length: sizeof(inum));
4762	ei->i_csum_seed = ext4_chksum(sbi, crc: csum, address: (__u8 *)&gen,
4763	length: sizeof(gen));
4764	}
4765
4766	if ((!ext4_inode_csum_verify(inode, raw: raw_inode, ei) ||
4767	ext4_simulate_fail(sb, EXT4_SIM_INODE_CRC)) &&
4768	(!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))) {
4769	ext4_error_inode_err(inode, function, line, 0,
4770	EFSBADCRC, "iget: checksum invalid");
4771	ret = -EFSBADCRC;
4772	goto bad_inode;
4773	}
4774
4775	inode->i_mode = le16_to_cpu(raw_inode->i_mode);
4776	i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);
4777	i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);
4778	if (ext4_has_feature_project(sb) &&
4779	EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE &&
4780	EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
4781	i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid);
4782	else
4783	i_projid = EXT4_DEF_PROJID;
4784
4785	if (!(test_opt(inode->i_sb, NO_UID32))) {
4786	i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;
4787	i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;
4788	}
4789	i_uid_write(inode, uid: i_uid);
4790	i_gid_write(inode, gid: i_gid);
4791	ei->i_projid = make_kprojid(from: &init_user_ns, projid: i_projid);
4792	set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));
4793
4794	ext4_clear_state_flags(ei); /* Only relevant on 32-bit archs */
4795	ei->i_inline_off = 0;
4796	ei->i_dir_start_lookup = 0;
4797	ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
4798	/* We now have enough fields to check if the inode was active or not.
4799	* This is needed because nfsd might try to access dead inodes
4800	* the test is that same one that e2fsck uses
4801	* NeilBrown 1999oct15
4802	*/
4803	if (inode->i_nlink == 0) {
4804	if ((inode->i_mode == 0 || flags & EXT4_IGET_SPECIAL ||
4805	!(EXT4_SB(sb: inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) &&
4806	ino != EXT4_BOOT_LOADER_INO) {
4807	/* this inode is deleted or unallocated */
4808	if (flags & EXT4_IGET_SPECIAL) {
4809	ext4_error_inode(inode, function, line, 0,
4810	"iget: special inode unallocated");
4811	ret = -EFSCORRUPTED;
4812	} else
4813	ret = -ESTALE;
4814	goto bad_inode;
4815	}
4816	/* The only unlinked inodes we let through here have
4817	* valid i_mode and are being read by the orphan
4818	* recovery code: that's fine, we're about to complete
4819	* the process of deleting those.
4820	* OR it is the EXT4_BOOT_LOADER_INO which is
4821	* not initialized on a new filesystem. */
4822	}
4823	ei->i_flags = le32_to_cpu(raw_inode->i_flags);
4824	ext4_set_inode_flags(inode, init: true);
4825	inode->i_blocks = ext4_inode_blocks(raw_inode, ei);
4826	ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);
4827	if (ext4_has_feature_64bit(sb))
4828	ei->i_file_acl |=
4829	((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
4830	inode->i_size = ext4_isize(sb, raw_inode);
4831	if ((size = i_size_read(inode)) < 0) {
4832	ext4_error_inode(inode, function, line, 0,
4833	"iget: bad i_size value: %lld", size);
4834	ret = -EFSCORRUPTED;
4835	goto bad_inode;
4836	}
4837	/*
4838	* If dir_index is not enabled but there's dir with INDEX flag set,
4839	* we'd normally treat htree data as empty space. But with metadata
4840	* checksumming that corrupts checksums so forbid that.
4841	*/
4842	if (!ext4_has_feature_dir_index(sb) && ext4_has_metadata_csum(sb) &&
4843	ext4_test_inode_flag(inode, bit: EXT4_INODE_INDEX)) {
4844	ext4_error_inode(inode, function, line, 0,
4845	"iget: Dir with htree data on filesystem without dir_index feature.");
4846	ret = -EFSCORRUPTED;
4847	goto bad_inode;
4848	}
4849	ei->i_disksize = inode->i_size;
4850	#ifdef CONFIG_QUOTA
4851	ei->i_reserved_quota = 0;
4852	#endif
4853	inode->i_generation = le32_to_cpu(raw_inode->i_generation);
4854	ei->i_block_group = iloc.block_group;
4855	ei->i_last_alloc_group = ~0;
4856	/*
4857	* NOTE! The in-memory inode i_data array is in little-endian order
4858	* even on big-endian machines: we do NOT byteswap the block numbers!
4859	*/
4860	for (block = 0; block < EXT4_N_BLOCKS; block++)
4861	ei->i_data[block] = raw_inode->i_block[block];
4862	INIT_LIST_HEAD(list: &ei->i_orphan);
4863	ext4_fc_init_inode(inode: &ei->vfs_inode);
4864
4865	/*
4866	* Set transaction id's of transactions that have to be committed
4867	* to finish f[data]sync. We set them to currently running transaction
4868	* as we cannot be sure that the inode or some of its metadata isn't
4869	* part of the transaction - the inode could have been reclaimed and
4870	* now it is reread from disk.
4871	*/
4872	if (journal) {
4873	transaction_t *transaction;
4874	tid_t tid;
4875
4876	read_lock(&journal->j_state_lock);
4877	if (journal->j_running_transaction)
4878	transaction = journal->j_running_transaction;
4879	else
4880	transaction = journal->j_committing_transaction;
4881	if (transaction)
4882	tid = transaction->t_tid;
4883	else
4884	tid = journal->j_commit_sequence;
4885	read_unlock(&journal->j_state_lock);
4886	ei->i_sync_tid = tid;
4887	ei->i_datasync_tid = tid;
4888	}
4889
4890	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4891	if (ei->i_extra_isize == 0) {
4892	/* The extra space is currently unused. Use it. */
4893	BUILD_BUG_ON(sizeof(struct ext4_inode) & 3);
4894	ei->i_extra_isize = sizeof(struct ext4_inode) -
4895	EXT4_GOOD_OLD_INODE_SIZE;
4896	} else {
4897	ret = ext4_iget_extra_inode(inode, raw_inode, ei);
4898	if (ret)
4899	goto bad_inode;
4900	}
4901	}
4902
4903	EXT4_INODE_GET_CTIME(inode, raw_inode);
4904	EXT4_INODE_GET_ATIME(inode, raw_inode);
4905	EXT4_INODE_GET_MTIME(inode, raw_inode);
4906	EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);
4907
4908	if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
4909	u64 ivers = le32_to_cpu(raw_inode->i_disk_version);
4910
4911	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4912	if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
4913	ivers |=
4914	(__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;
4915	}
4916	ext4_inode_set_iversion_queried(inode, val: ivers);
4917	}
4918
4919	ret = 0;
4920	if (ei->i_file_acl &&
4921	!ext4_inode_block_valid(inode, start_blk: ei->i_file_acl, count: 1)) {
4922	ext4_error_inode(inode, function, line, 0,
4923	"iget: bad extended attribute block %llu",
4924	ei->i_file_acl);
4925	ret = -EFSCORRUPTED;
4926	goto bad_inode;
4927	} else if (!ext4_has_inline_data(inode)) {
4928	/* validate the block references in the inode */
4929	if (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) &&
4930	(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
4931	(S_ISLNK(inode->i_mode) &&
4932	!ext4_inode_is_fast_symlink(inode)))) {
4933	if (ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))
4934	ret = ext4_ext_check_inode(inode);
4935	else
4936	ret = ext4_ind_check_inode(inode);
4937	}
4938	}
4939	if (ret)
4940	goto bad_inode;
4941
4942	if (S_ISREG(inode->i_mode)) {
4943	inode->i_op = &ext4_file_inode_operations;
4944	inode->i_fop = &ext4_file_operations;
4945	ext4_set_aops(inode);
4946	} else if (S_ISDIR(inode->i_mode)) {
4947	inode->i_op = &ext4_dir_inode_operations;
4948	inode->i_fop = &ext4_dir_operations;
4949	} else if (S_ISLNK(inode->i_mode)) {
4950	/* VFS does not allow setting these so must be corruption */
4951	if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) {
4952	ext4_error_inode(inode, function, line, 0,
4953	"iget: immutable or append flags "
4954	"not allowed on symlinks");
4955	ret = -EFSCORRUPTED;
4956	goto bad_inode;
4957	}
4958	if (IS_ENCRYPTED(inode)) {
4959	inode->i_op = &ext4_encrypted_symlink_inode_operations;
4960	} else if (ext4_inode_is_fast_symlink(inode)) {
4961	inode->i_link = (char *)ei->i_data;
4962	inode->i_op = &ext4_fast_symlink_inode_operations;
4963	nd_terminate_link(name: ei->i_data, len: inode->i_size,
4964	maxlen: sizeof(ei->i_data) - 1);
4965	} else {
4966	inode->i_op = &ext4_symlink_inode_operations;
4967	}
4968	} else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
4969	S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
4970	inode->i_op = &ext4_special_inode_operations;
4971	if (raw_inode->i_block[0])
4972	init_special_inode(inode, inode->i_mode,
4973	old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));
4974	else
4975	init_special_inode(inode, inode->i_mode,
4976	new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
4977	} else if (ino == EXT4_BOOT_LOADER_INO) {
4978	make_bad_inode(inode);
4979	} else {
4980	ret = -EFSCORRUPTED;
4981	ext4_error_inode(inode, function, line, 0,
4982	"iget: bogus i_mode (%o)", inode->i_mode);
4983	goto bad_inode;
4984	}
4985	if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(sb: inode->i_sb)) {
4986	ext4_error_inode(inode, function, line, 0,
4987	"casefold flag without casefold feature");
4988	ret = -EFSCORRUPTED;
4989	goto bad_inode;
4990	}
4991	if ((err_str = check_igot_inode(inode, flags)) != NULL) {
4992	ext4_error_inode(inode, function, line, 0, err_str);
4993	ret = -EFSCORRUPTED;
4994	goto bad_inode;
4995	}
4996
4997	brelse(bh: iloc.bh);
4998	unlock_new_inode(inode);
4999	return inode;
5000
5001	bad_inode:
5002	brelse(bh: iloc.bh);
5003	iget_failed(inode);
5004	return ERR_PTR(error: ret);
5005	}
5006
5007	static void __ext4_update_other_inode_time(struct super_block *sb,
5008	unsigned long orig_ino,
5009	unsigned long ino,
5010	struct ext4_inode *raw_inode)
5011	{
5012	struct inode *inode;
5013
5014	inode = find_inode_by_ino_rcu(sb, ino);
5015	if (!inode)
5016	return;
5017
5018	if (!inode_is_dirtytime_only(inode))
5019	return;
5020
5021	spin_lock(lock: &inode->i_lock);
5022	if (inode_is_dirtytime_only(inode)) {
5023	struct ext4_inode_info *ei = EXT4_I(inode);
5024
5025	inode->i_state &= ~I_DIRTY_TIME;
5026	spin_unlock(lock: &inode->i_lock);
5027
5028	spin_lock(lock: &ei->i_raw_lock);
5029	EXT4_INODE_SET_CTIME(inode, raw_inode);
5030	EXT4_INODE_SET_MTIME(inode, raw_inode);
5031	EXT4_INODE_SET_ATIME(inode, raw_inode);
5032	ext4_inode_csum_set(inode, raw: raw_inode, ei);
5033	spin_unlock(lock: &ei->i_raw_lock);
5034	trace_ext4_other_inode_update_time(inode, orig_ino);
5035	return;
5036	}
5037	spin_unlock(lock: &inode->i_lock);
5038	}
5039
5040	/*
5041	* Opportunistically update the other time fields for other inodes in
5042	* the same inode table block.
5043	*/
5044	static void ext4_update_other_inodes_time(struct super_block *sb,
5045	unsigned long orig_ino, char *buf)
5046	{
5047	unsigned long ino;
5048	int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
5049	int inode_size = EXT4_INODE_SIZE(sb);
5050
5051	/*
5052	* Calculate the first inode in the inode table block. Inode
5053	* numbers are one-based. That is, the first inode in a block
5054	* (assuming 4k blocks and 256 byte inodes) is (n*16 + 1).
5055	*/
5056	ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1;
5057	rcu_read_lock();
5058	for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) {
5059	if (ino == orig_ino)
5060	continue;
5061	__ext4_update_other_inode_time(sb, orig_ino, ino,
5062	raw_inode: (struct ext4_inode *)buf);
5063	}
5064	rcu_read_unlock();
5065	}
5066
5067	/*
5068	* Post the struct inode info into an on-disk inode location in the
5069	* buffer-cache. This gobbles the caller's reference to the
5070	* buffer_head in the inode location struct.
5071	*
5072	* The caller must have write access to iloc->bh.
5073	*/
5074	static int ext4_do_update_inode(handle_t *handle,
5075	struct inode *inode,
5076	struct ext4_iloc *iloc)
5077	{
5078	struct ext4_inode *raw_inode = ext4_raw_inode(iloc);
5079	struct ext4_inode_info *ei = EXT4_I(inode);
5080	struct buffer_head *bh = iloc->bh;
5081	struct super_block *sb = inode->i_sb;
5082	int err;
5083	int need_datasync = 0, set_large_file = 0;
5084
5085	spin_lock(lock: &ei->i_raw_lock);
5086
5087	/*
5088	* For fields not tracked in the in-memory inode, initialise them
5089	* to zero for new inodes.
5090	*/
5091	if (ext4_test_inode_state(inode, bit: EXT4_STATE_NEW))
5092	memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
5093
5094	if (READ_ONCE(ei->i_disksize) != ext4_isize(sb: inode->i_sb, raw_inode))
5095	need_datasync = 1;
5096	if (ei->i_disksize > 0x7fffffffULL) {
5097	if (!ext4_has_feature_large_file(sb) ||
5098	EXT4_SB(sb)->s_es->s_rev_level == cpu_to_le32(EXT4_GOOD_OLD_REV))
5099	set_large_file = 1;
5100	}
5101
5102	err = ext4_fill_raw_inode(inode, raw_inode);
5103	spin_unlock(lock: &ei->i_raw_lock);
5104	if (err) {
5105	EXT4_ERROR_INODE(inode, "corrupted inode contents");
5106	goto out_brelse;
5107	}
5108
5109	if (inode->i_sb->s_flags & SB_LAZYTIME)
5110	ext4_update_other_inodes_time(sb: inode->i_sb, orig_ino: inode->i_ino,
5111	buf: bh->b_data);
5112
5113	BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
5114	err = ext4_handle_dirty_metadata(handle, NULL, bh);
5115	if (err)
5116	goto out_error;
5117	ext4_clear_inode_state(inode, bit: EXT4_STATE_NEW);
5118	if (set_large_file) {
5119	BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");
5120	err = ext4_journal_get_write_access(handle, sb,
5121	EXT4_SB(sb)->s_sbh,
5122	EXT4_JTR_NONE);
5123	if (err)
5124	goto out_error;
5125	lock_buffer(bh: EXT4_SB(sb)->s_sbh);
5126	ext4_set_feature_large_file(sb);
5127	ext4_superblock_csum_set(sb);
5128	unlock_buffer(bh: EXT4_SB(sb)->s_sbh);
5129	ext4_handle_sync(handle);
5130	err = ext4_handle_dirty_metadata(handle, NULL,
5131	EXT4_SB(sb)->s_sbh);
5132	}
5133	ext4_update_inode_fsync_trans(handle, inode, datasync: need_datasync);
5134	out_error:
5135	ext4_std_error(inode->i_sb, err);
5136	out_brelse:
5137	brelse(bh);
5138	return err;
5139	}
5140
5141	/*
5142	* ext4_write_inode()
5143	*
5144	* We are called from a few places:
5145	*
5146	* - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files.
5147	* Here, there will be no transaction running. We wait for any running
5148	* transaction to commit.
5149	*
5150	* - Within flush work (sys_sync(), kupdate and such).
5151	* We wait on commit, if told to.
5152	*
5153	* - Within iput_final() -> write_inode_now()
5154	* We wait on commit, if told to.
5155	*
5156	* In all cases it is actually safe for us to return without doing anything,
5157	* because the inode has been copied into a raw inode buffer in
5158	* ext4_mark_inode_dirty(). This is a correctness thing for WB_SYNC_ALL
5159	* writeback.
5160	*
5161	* Note that we are absolutely dependent upon all inode dirtiers doing the
5162	* right thing: they *must* call mark_inode_dirty() after dirtying info in
5163	* which we are interested.
5164	*
5165	* It would be a bug for them to not do this. The code:
5166	*
5167	* mark_inode_dirty(inode)
5168	* stuff();
5169	* inode->i_size = expr;
5170	*
5171	* is in error because write_inode() could occur while `stuff()' is running,
5172	* and the new i_size will be lost. Plus the inode will no longer be on the
5173	* superblock's dirty inode list.
5174	*/
5175	int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
5176	{
5177	int err;
5178
5179	if (WARN_ON_ONCE(current->flags & PF_MEMALLOC))
5180	return 0;
5181
5182	if (unlikely(ext4_forced_shutdown(inode->i_sb)))
5183	return -EIO;
5184
5185	if (EXT4_SB(sb: inode->i_sb)->s_journal) {
5186	if (ext4_journal_current_handle()) {
5187	ext4_debug("called recursively, non-PF_MEMALLOC!\n");
5188	dump_stack();
5189	return -EIO;
5190	}
5191
5192	/*
5193	* No need to force transaction in WB_SYNC_NONE mode. Also
5194	* ext4_sync_fs() will force the commit after everything is
5195	* written.
5196	*/
5197	if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync)
5198	return 0;
5199
5200	err = ext4_fc_commit(journal: EXT4_SB(sb: inode->i_sb)->s_journal,
5201	EXT4_I(inode)->i_sync_tid);
5202	} else {
5203	struct ext4_iloc iloc;
5204
5205	err = __ext4_get_inode_loc_noinmem(inode, iloc: &iloc);
5206	if (err)
5207	return err;
5208	/*
5209	* sync(2) will flush the whole buffer cache. No need to do
5210	* it here separately for each inode.
5211	*/
5212	if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
5213	sync_dirty_buffer(bh: iloc.bh);
5214	if (buffer_req(bh: iloc.bh) && !buffer_uptodate(bh: iloc.bh)) {
5215	ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO,
5216	"IO error syncing inode");
5217	err = -EIO;
5218	}
5219	brelse(bh: iloc.bh);
5220	}
5221	return err;
5222	}
5223
5224	/*
5225	* In data=journal mode ext4_journalled_invalidate_folio() may fail to invalidate
5226	* buffers that are attached to a folio straddling i_size and are undergoing
5227	* commit. In that case we have to wait for commit to finish and try again.
5228	*/
5229	static void ext4_wait_for_tail_page_commit(struct inode *inode)
5230	{
5231	unsigned offset;
5232	journal_t *journal = EXT4_SB(sb: inode->i_sb)->s_journal;
5233	tid_t commit_tid = 0;
5234	int ret;
5235
5236	offset = inode->i_size & (PAGE_SIZE - 1);
5237	/*
5238	* If the folio is fully truncated, we don't need to wait for any commit
5239	* (and we even should not as __ext4_journalled_invalidate_folio() may
5240	* strip all buffers from the folio but keep the folio dirty which can then
5241	* confuse e.g. concurrent ext4_writepages() seeing dirty folio without
5242	* buffers). Also we don't need to wait for any commit if all buffers in
5243	* the folio remain valid. This is most beneficial for the common case of
5244	* blocksize == PAGESIZE.
5245	*/
5246	if (!offset || offset > (PAGE_SIZE - i_blocksize(node: inode)))
5247	return;
5248	while (1) {
5249	struct folio *folio = filemap_lock_folio(mapping: inode->i_mapping,
5250	index: inode->i_size >> PAGE_SHIFT);
5251	if (IS_ERR(ptr: folio))
5252	return;
5253	ret = __ext4_journalled_invalidate_folio(folio, offset,
5254	length: folio_size(folio) - offset);
5255	folio_unlock(folio);
5256	folio_put(folio);
5257	if (ret != -EBUSY)
5258	return;
5259	commit_tid = 0;
5260	read_lock(&journal->j_state_lock);
5261	if (journal->j_committing_transaction)
5262	commit_tid = journal->j_committing_transaction->t_tid;
5263	read_unlock(&journal->j_state_lock);
5264	if (commit_tid)
5265	jbd2_log_wait_commit(journal, tid: commit_tid);
5266	}
5267	}
5268
5269	/*
5270	* ext4_setattr()
5271	*
5272	* Called from notify_change.
5273	*
5274	* We want to trap VFS attempts to truncate the file as soon as
5275	* possible. In particular, we want to make sure that when the VFS
5276	* shrinks i_size, we put the inode on the orphan list and modify
5277	* i_disksize immediately, so that during the subsequent flushing of
5278	* dirty pages and freeing of disk blocks, we can guarantee that any
5279	* commit will leave the blocks being flushed in an unused state on
5280	* disk. (On recovery, the inode will get truncated and the blocks will
5281	* be freed, so we have a strong guarantee that no future commit will
5282	* leave these blocks visible to the user.)
5283	*
5284	* Another thing we have to assure is that if we are in ordered mode
5285	* and inode is still attached to the committing transaction, we must
5286	* we start writeout of all the dirty pages which are being truncated.
5287	* This way we are sure that all the data written in the previous
5288	* transaction are already on disk (truncate waits for pages under
5289	* writeback).
5290	*
5291	* Called with inode->i_rwsem down.
5292	*/
5293	int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
5294	struct iattr *attr)
5295	{
5296	struct inode *inode = d_inode(dentry);
5297	int error, rc = 0;
5298	int orphan = 0;
5299	const unsigned int ia_valid = attr->ia_valid;
5300	bool inc_ivers = true;
5301
5302	if (unlikely(ext4_forced_shutdown(inode->i_sb)))
5303	return -EIO;
5304
5305	if (unlikely(IS_IMMUTABLE(inode)))
5306	return -EPERM;
5307
5308	if (unlikely(IS_APPEND(inode) &&
5309	(ia_valid & (ATTR_MODE | ATTR_UID |
5310	ATTR_GID | ATTR_TIMES_SET))))
5311	return -EPERM;
5312
5313	error = setattr_prepare(idmap, dentry, attr);
5314	if (error)
5315	return error;
5316
5317	error = fscrypt_prepare_setattr(dentry, attr);
5318	if (error)
5319	return error;
5320
5321	error = fsverity_prepare_setattr(dentry, attr);
5322	if (error)
5323	return error;
5324
5325	if (is_quota_modification(idmap, inode, ia: attr)) {
5326	error = dquot_initialize(inode);
5327	if (error)
5328	return error;
5329	}
5330
5331	if (i_uid_needs_update(idmap, attr, inode) ||
5332	i_gid_needs_update(idmap, attr, inode)) {
5333	handle_t *handle;
5334
5335	/* (user+group)*(old+new) structure, inode write (sb,
5336	* inode block, ? - but truncate inode update has it) */
5337	handle = ext4_journal_start(inode, EXT4_HT_QUOTA,
5338	(EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) +
5339	EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3);
5340	if (IS_ERR(ptr: handle)) {
5341	error = PTR_ERR(ptr: handle);
5342	goto err_out;
5343	}
5344
5345	/* dquot_transfer() calls back ext4_get_inode_usage() which
5346	* counts xattr inode references.
5347	*/
5348	down_read(sem: &EXT4_I(inode)->xattr_sem);
5349	error = dquot_transfer(idmap, inode, iattr: attr);
5350	up_read(sem: &EXT4_I(inode)->xattr_sem);
5351
5352	if (error) {
5353	ext4_journal_stop(handle);
5354	return error;
5355	}
5356	/* Update corresponding info in inode so that everything is in
5357	* one transaction */
5358	i_uid_update(idmap, attr, inode);
5359	i_gid_update(idmap, attr, inode);
5360	error = ext4_mark_inode_dirty(handle, inode);
5361	ext4_journal_stop(handle);
5362	if (unlikely(error)) {
5363	return error;
5364	}
5365	}
5366
5367	if (attr->ia_valid & ATTR_SIZE) {
5368	handle_t *handle;
5369	loff_t oldsize = inode->i_size;
5370	loff_t old_disksize;
5371	int shrink = (attr->ia_size < inode->i_size);
5372
5373	if (!(ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS))) {
5374	struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
5375
5376	if (attr->ia_size > sbi->s_bitmap_maxbytes) {
5377	return -EFBIG;
5378	}
5379	}
5380	if (!S_ISREG(inode->i_mode)) {
5381	return -EINVAL;
5382	}
5383
5384	if (attr->ia_size == inode->i_size)
5385	inc_ivers = false;
5386
5387	if (shrink) {
5388	if (ext4_should_order_data(inode)) {
5389	error = ext4_begin_ordered_truncate(inode,
5390	new_size: attr->ia_size);
5391	if (error)
5392	goto err_out;
5393	}
5394	/*
5395	* Blocks are going to be removed from the inode. Wait
5396	* for dio in flight.
5397	*/
5398	inode_dio_wait(inode);
5399	}
5400
5401	filemap_invalidate_lock(mapping: inode->i_mapping);
5402
5403	rc = ext4_break_layouts(inode);
5404	if (rc) {
5405	filemap_invalidate_unlock(mapping: inode->i_mapping);
5406	goto err_out;
5407	}
5408
5409	if (attr->ia_size != inode->i_size) {
5410	handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);
5411	if (IS_ERR(ptr: handle)) {
5412	error = PTR_ERR(ptr: handle);
5413	goto out_mmap_sem;
5414	}
5415	if (ext4_handle_valid(handle) && shrink) {
5416	error = ext4_orphan_add(handle, inode);
5417	orphan = 1;
5418	}
5419	/*
5420	* Update c/mtime on truncate up, ext4_truncate() will
5421	* update c/mtime in shrink case below
5422	*/
5423	if (!shrink)
5424	inode_set_mtime_to_ts(inode,
5425	ts: inode_set_ctime_current(inode));
5426
5427	if (shrink)
5428	ext4_fc_track_range(handle, inode,
5429	start: (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
5430	inode->i_sb->s_blocksize_bits,
5431	EXT_MAX_BLOCKS - 1);
5432	else
5433	ext4_fc_track_range(
5434	handle, inode,
5435	start: (oldsize > 0 ? oldsize - 1 : oldsize) >>
5436	inode->i_sb->s_blocksize_bits,
5437	end: (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
5438	inode->i_sb->s_blocksize_bits);
5439
5440	down_write(sem: &EXT4_I(inode)->i_data_sem);
5441	old_disksize = EXT4_I(inode)->i_disksize;
5442	EXT4_I(inode)->i_disksize = attr->ia_size;
5443	rc = ext4_mark_inode_dirty(handle, inode);
5444	if (!error)
5445	error = rc;
5446	/*
5447	* We have to update i_size under i_data_sem together
5448	* with i_disksize to avoid races with writeback code
5449	* running ext4_wb_update_i_disksize().
5450	*/
5451	if (!error)
5452	i_size_write(inode, i_size: attr->ia_size);
5453	else
5454	EXT4_I(inode)->i_disksize = old_disksize;
5455	up_write(sem: &EXT4_I(inode)->i_data_sem);
5456	ext4_journal_stop(handle);
5457	if (error)
5458	goto out_mmap_sem;
5459	if (!shrink) {
5460	pagecache_isize_extended(inode, from: oldsize,
5461	to: inode->i_size);
5462	} else if (ext4_should_journal_data(inode)) {
5463	ext4_wait_for_tail_page_commit(inode);
5464	}
5465	}
5466
5467	/*
5468	* Truncate pagecache after we've waited for commit
5469	* in data=journal mode to make pages freeable.
5470	*/
5471	truncate_pagecache(inode, new: inode->i_size);
5472	/*
5473	* Call ext4_truncate() even if i_size didn't change to
5474	* truncate possible preallocated blocks.
5475	*/
5476	if (attr->ia_size <= oldsize) {
5477	rc = ext4_truncate(inode);
5478	if (rc)
5479	error = rc;
5480	}
5481	out_mmap_sem:
5482	filemap_invalidate_unlock(mapping: inode->i_mapping);
5483	}
5484
5485	if (!error) {
5486	if (inc_ivers)
5487	inode_inc_iversion(inode);
5488	setattr_copy(idmap, inode, attr);
5489	mark_inode_dirty(inode);
5490	}
5491
5492	/*
5493	* If the call to ext4_truncate failed to get a transaction handle at
5494	* all, we need to clean up the in-core orphan list manually.
5495	*/
5496	if (orphan && inode->i_nlink)
5497	ext4_orphan_del(NULL, inode);
5498
5499	if (!error && (ia_valid & ATTR_MODE))
5500	rc = posix_acl_chmod(idmap, dentry, inode->i_mode);
5501
5502	err_out:
5503	if (error)
5504	ext4_std_error(inode->i_sb, error);
5505	if (!error)
5506	error = rc;
5507	return error;
5508	}
5509
5510	u32 ext4_dio_alignment(struct inode *inode)
5511	{
5512	if (fsverity_active(inode))
5513	return 0;
5514	if (ext4_should_journal_data(inode))
5515	return 0;
5516	if (ext4_has_inline_data(inode))
5517	return 0;
5518	if (IS_ENCRYPTED(inode)) {
5519	if (!fscrypt_dio_supported(inode))
5520	return 0;
5521	return i_blocksize(node: inode);
5522	}
5523	return 1; /* use the iomap defaults */
5524	}
5525
5526	int ext4_getattr(struct mnt_idmap *idmap, const struct path *path,
5527	struct kstat *stat, u32 request_mask, unsigned int query_flags)
5528	{
5529	struct inode *inode = d_inode(dentry: path->dentry);
5530	struct ext4_inode *raw_inode;
5531	struct ext4_inode_info *ei = EXT4_I(inode);
5532	unsigned int flags;
5533
5534	if ((request_mask & STATX_BTIME) &&
5535	EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) {
5536	stat->result_mask |= STATX_BTIME;
5537	stat->btime.tv_sec = ei->i_crtime.tv_sec;
5538	stat->btime.tv_nsec = ei->i_crtime.tv_nsec;
5539	}
5540
5541	/*
5542	* Return the DIO alignment restrictions if requested. We only return
5543	* this information when requested, since on encrypted files it might
5544	* take a fair bit of work to get if the file wasn't opened recently.
5545	*/
5546	if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) {
5547	u32 dio_align = ext4_dio_alignment(inode);
5548
5549	stat->result_mask |= STATX_DIOALIGN;
5550	if (dio_align == 1) {
5551	struct block_device *bdev = inode->i_sb->s_bdev;
5552
5553	/* iomap defaults */
5554	stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;
5555	stat->dio_offset_align = bdev_logical_block_size(bdev);
5556	} else {
5557	stat->dio_mem_align = dio_align;
5558	stat->dio_offset_align = dio_align;
5559	}
5560	}
5561
5562	flags = ei->i_flags & EXT4_FL_USER_VISIBLE;
5563	if (flags & EXT4_APPEND_FL)
5564	stat->attributes |= STATX_ATTR_APPEND;
5565	if (flags & EXT4_COMPR_FL)
5566	stat->attributes |= STATX_ATTR_COMPRESSED;
5567	if (flags & EXT4_ENCRYPT_FL)
5568	stat->attributes |= STATX_ATTR_ENCRYPTED;
5569	if (flags & EXT4_IMMUTABLE_FL)
5570	stat->attributes |= STATX_ATTR_IMMUTABLE;
5571	if (flags & EXT4_NODUMP_FL)
5572	stat->attributes |= STATX_ATTR_NODUMP;
5573	if (flags & EXT4_VERITY_FL)
5574	stat->attributes |= STATX_ATTR_VERITY;
5575
5576	stat->attributes_mask |= (STATX_ATTR_APPEND |
5577	STATX_ATTR_COMPRESSED |
5578	STATX_ATTR_ENCRYPTED |
5579	STATX_ATTR_IMMUTABLE |
5580	STATX_ATTR_NODUMP |
5581	STATX_ATTR_VERITY);
5582
5583	generic_fillattr(idmap, request_mask, inode, stat);
5584	return 0;
5585	}
5586
5587	int ext4_file_getattr(struct mnt_idmap *idmap,
5588	const struct path *path, struct kstat *stat,
5589	u32 request_mask, unsigned int query_flags)
5590	{
5591	struct inode *inode = d_inode(dentry: path->dentry);
5592	u64 delalloc_blocks;
5593
5594	ext4_getattr(idmap, path, stat, request_mask, query_flags);
5595
5596	/*
5597	* If there is inline data in the inode, the inode will normally not
5598	* have data blocks allocated (it may have an external xattr block).
5599	* Report at least one sector for such files, so tools like tar, rsync,
5600	* others don't incorrectly think the file is completely sparse.
5601	*/
5602	if (unlikely(ext4_has_inline_data(inode)))
5603	stat->blocks += (stat->size + 511) >> 9;
5604
5605	/*
5606	* We can't update i_blocks if the block allocation is delayed
5607	* otherwise in the case of system crash before the real block
5608	* allocation is done, we will have i_blocks inconsistent with
5609	* on-disk file blocks.
5610	* We always keep i_blocks updated together with real
5611	* allocation. But to not confuse with user, stat
5612	* will return the blocks that include the delayed allocation
5613	* blocks for this file.
5614	*/
5615	delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb),
5616	EXT4_I(inode)->i_reserved_data_blocks);
5617	stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9);
5618	return 0;
5619	}
5620
5621	static int ext4_index_trans_blocks(struct inode *inode, int lblocks,
5622	int pextents)
5623	{
5624	if (!(ext4_test_inode_flag(inode, bit: EXT4_INODE_EXTENTS)))
5625	return ext4_ind_trans_blocks(inode, nrblocks: lblocks);
5626	return ext4_ext_index_trans_blocks(inode, extents: pextents);
5627	}
5628
5629	/*
5630	* Account for index blocks, block groups bitmaps and block group
5631	* descriptor blocks if modify datablocks and index blocks
5632	* worse case, the indexs blocks spread over different block groups
5633	*
5634	* If datablocks are discontiguous, they are possible to spread over
5635	* different block groups too. If they are contiguous, with flexbg,
5636	* they could still across block group boundary.
5637	*
5638	* Also account for superblock, inode, quota and xattr blocks
5639	*/
5640	static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
5641	int pextents)
5642	{
5643	ext4_group_t groups, ngroups = ext4_get_groups_count(sb: inode->i_sb);
5644	int gdpblocks;
5645	int idxblocks;
5646	int ret;
5647
5648	/*
5649	* How many index blocks need to touch to map @lblocks logical blocks
5650	* to @pextents physical extents?
5651	*/
5652	idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents);
5653
5654	ret = idxblocks;
5655
5656	/*
5657	* Now let's see how many group bitmaps and group descriptors need
5658	* to account
5659	*/
5660	groups = idxblocks + pextents;
5661	gdpblocks = groups;
5662	if (groups > ngroups)
5663	groups = ngroups;
5664	if (groups > EXT4_SB(sb: inode->i_sb)->s_gdb_count)
5665	gdpblocks = EXT4_SB(sb: inode->i_sb)->s_gdb_count;
5666
5667	/* bitmaps and block group descriptor blocks */
5668	ret += groups + gdpblocks;
5669
5670	/* Blocks for super block, inode, quota and xattr blocks */
5671	ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);
5672
5673	return ret;
5674	}
5675
5676	/*
5677	* Calculate the total number of credits to reserve to fit
5678	* the modification of a single pages into a single transaction,
5679	* which may include multiple chunks of block allocations.
5680	*
5681	* This could be called via ext4_write_begin()
5682	*
5683	* We need to consider the worse case, when
5684	* one new block per extent.
5685	*/
5686	int ext4_writepage_trans_blocks(struct inode *inode)
5687	{
5688	int bpp = ext4_journal_blocks_per_page(inode);
5689	int ret;
5690
5691	ret = ext4_meta_trans_blocks(inode, lblocks: bpp, pextents: bpp);
5692
5693	/* Account for data blocks for journalled mode */
5694	if (ext4_should_journal_data(inode))
5695	ret += bpp;
5696	return ret;
5697	}
5698
5699	/*
5700	* Calculate the journal credits for a chunk of data modification.
5701	*
5702	* This is called from DIO, fallocate or whoever calling
5703	* ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks.
5704	*
5705	* journal buffers for data blocks are not included here, as DIO
5706	* and fallocate do no need to journal data buffers.
5707	*/
5708	int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
5709	{
5710	return ext4_meta_trans_blocks(inode, lblocks: nrblocks, pextents: 1);
5711	}
5712
5713	/*
5714	* The caller must have previously called ext4_reserve_inode_write().
5715	* Give this, we know that the caller already has write access to iloc->bh.
5716	*/
5717	int ext4_mark_iloc_dirty(handle_t *handle,
5718	struct inode *inode, struct ext4_iloc *iloc)
5719	{
5720	int err = 0;
5721
5722	if (unlikely(ext4_forced_shutdown(inode->i_sb))) {
5723	put_bh(bh: iloc->bh);
5724	return -EIO;
5725	}
5726	ext4_fc_track_inode(handle, inode);
5727
5728	/* the do_update_inode consumes one bh->b_count */
5729	get_bh(bh: iloc->bh);
5730
5731	/* ext4_do_update_inode() does jbd2_journal_dirty_metadata */
5732	err = ext4_do_update_inode(handle, inode, iloc);
5733	put_bh(bh: iloc->bh);
5734	return err;
5735	}
5736
5737	/*
5738	* On success, We end up with an outstanding reference count against
5739	* iloc->bh. This _must_ be cleaned up later.
5740	*/
5741
5742	int
5743	ext4_reserve_inode_write(handle_t *handle, struct inode *inode,
5744	struct ext4_iloc *iloc)
5745	{
5746	int err;
5747
5748	if (unlikely(ext4_forced_shutdown(inode->i_sb)))
5749	return -EIO;
5750
5751	err = ext4_get_inode_loc(inode, iloc);
5752	if (!err) {
5753	BUFFER_TRACE(iloc->bh, "get_write_access");
5754	err = ext4_journal_get_write_access(handle, inode->i_sb,
5755	iloc->bh, EXT4_JTR_NONE);
5756	if (err) {
5757	brelse(bh: iloc->bh);
5758	iloc->bh = NULL;
5759	}
5760	}
5761	ext4_std_error(inode->i_sb, err);
5762	return err;
5763	}
5764
5765	static int __ext4_expand_extra_isize(struct inode *inode,
5766	unsigned int new_extra_isize,
5767	struct ext4_iloc *iloc,
5768	handle_t *handle, int *no_expand)
5769	{
5770	struct ext4_inode *raw_inode;
5771	struct ext4_xattr_ibody_header *header;
5772	unsigned int inode_size = EXT4_INODE_SIZE(inode->i_sb);
5773	struct ext4_inode_info *ei = EXT4_I(inode);
5774	int error;
5775
5776	/* this was checked at iget time, but double check for good measure */
5777	if ((EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > inode_size) ||
5778	(ei->i_extra_isize & 3)) {
5779	EXT4_ERROR_INODE(inode, "bad extra_isize %u (inode size %u)",
5780	ei->i_extra_isize,
5781	EXT4_INODE_SIZE(inode->i_sb));
5782	return -EFSCORRUPTED;
5783	}
5784	if ((new_extra_isize < ei->i_extra_isize) ||
5785	(new_extra_isize < 4) ||
5786	(new_extra_isize > inode_size - EXT4_GOOD_OLD_INODE_SIZE))
5787	return -EINVAL; /* Should never happen */
5788
5789	raw_inode = ext4_raw_inode(iloc);
5790
5791	header = IHDR(inode, raw_inode);
5792
5793	/* No extended attributes present */
5794	if (!ext4_test_inode_state(inode, bit: EXT4_STATE_XATTR) ||
5795	header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
5796	memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE +
5797	EXT4_I(inode)->i_extra_isize, 0,
5798	new_extra_isize - EXT4_I(inode)->i_extra_isize);
5799	EXT4_I(inode)->i_extra_isize = new_extra_isize;
5800	return 0;
5801	}
5802
5803	/*
5804	* We may need to allocate external xattr block so we need quotas
5805	* initialized. Here we can be called with various locks held so we
5806	* cannot affort to initialize quotas ourselves. So just bail.
5807	*/
5808	if (dquot_initialize_needed(inode))
5809	return -EAGAIN;
5810
5811	/* try to expand with EAs present */
5812	error = ext4_expand_extra_isize_ea(inode, new_extra_isize,
5813	raw_inode, handle);
5814	if (error) {
5815	/*
5816	* Inode size expansion failed; don't try again
5817	*/
5818	*no_expand = 1;
5819	}
5820
5821	return error;
5822	}
5823
5824	/*
5825	* Expand an inode by new_extra_isize bytes.
5826	* Returns 0 on success or negative error number on failure.
5827	*/
5828	static int ext4_try_to_expand_extra_isize(struct inode *inode,
5829	unsigned int new_extra_isize,
5830	struct ext4_iloc iloc,
5831	handle_t *handle)
5832	{
5833	int no_expand;
5834	int error;
5835
5836	if (ext4_test_inode_state(inode, bit: EXT4_STATE_NO_EXPAND))
5837	return -EOVERFLOW;
5838
5839	/*
5840	* In nojournal mode, we can immediately attempt to expand
5841	* the inode. When journaled, we first need to obtain extra
5842	* buffer credits since we may write into the EA block
5843	* with this same handle. If journal_extend fails, then it will
5844	* only result in a minor loss of functionality for that inode.
5845	* If this is felt to be critical, then e2fsck should be run to
5846	* force a large enough s_min_extra_isize.
5847	*/
5848	if (ext4_journal_extend(handle,
5849	EXT4_DATA_TRANS_BLOCKS(inode->i_sb), revoke: 0) != 0)
5850	return -ENOSPC;
5851
5852	if (ext4_write_trylock_xattr(inode, save: &no_expand) == 0)
5853	return -EBUSY;
5854
5855	error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc: &iloc,
5856	handle, no_expand: &no_expand);
5857	ext4_write_unlock_xattr(inode, save: &no_expand);
5858
5859	return error;
5860	}
5861
5862	int ext4_expand_extra_isize(struct inode *inode,
5863	unsigned int new_extra_isize,
5864	struct ext4_iloc *iloc)
5865	{
5866	handle_t *handle;
5867	int no_expand;
5868	int error, rc;
5869
5870	if (ext4_test_inode_state(inode, bit: EXT4_STATE_NO_EXPAND)) {
5871	brelse(bh: iloc->bh);
5872	return -EOVERFLOW;
5873	}
5874
5875	handle = ext4_journal_start(inode, EXT4_HT_INODE,
5876	EXT4_DATA_TRANS_BLOCKS(inode->i_sb));
5877	if (IS_ERR(ptr: handle)) {
5878	error = PTR_ERR(ptr: handle);
5879	brelse(bh: iloc->bh);
5880	return error;
5881	}
5882
5883	ext4_write_lock_xattr(inode, save: &no_expand);
5884
5885	BUFFER_TRACE(iloc->bh, "get_write_access");
5886	error = ext4_journal_get_write_access(handle, inode->i_sb, iloc->bh,
5887	EXT4_JTR_NONE);
5888	if (error) {
5889	brelse(bh: iloc->bh);
5890	goto out_unlock;
5891	}
5892
5893	error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc,
5894	handle, no_expand: &no_expand);
5895
5896	rc = ext4_mark_iloc_dirty(handle, inode, iloc);
5897	if (!error)
5898	error = rc;
5899
5900	out_unlock:
5901	ext4_write_unlock_xattr(inode, save: &no_expand);
5902	ext4_journal_stop(handle);
5903	return error;
5904	}
5905
5906	/*
5907	* What we do here is to mark the in-core inode as clean with respect to inode
5908	* dirtiness (it may still be data-dirty).
5909	* This means that the in-core inode may be reaped by prune_icache
5910	* without having to perform any I/O. This is a very good thing,
5911	* because *any* task may call prune_icache - even ones which
5912	* have a transaction open against a different journal.
5913	*
5914	* Is this cheating? Not really. Sure, we haven't written the
5915	* inode out, but prune_icache isn't a user-visible syncing function.
5916	* Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync)
5917	* we start and wait on commits.
5918	*/
5919	int __ext4_mark_inode_dirty(handle_t *handle, struct inode *inode,
5920	const char *func, unsigned int line)
5921	{
5922	struct ext4_iloc iloc;
5923	struct ext4_sb_info *sbi = EXT4_SB(sb: inode->i_sb);
5924	int err;
5925
5926	might_sleep();
5927	trace_ext4_mark_inode_dirty(inode, _RET_IP_);
5928	err = ext4_reserve_inode_write(handle, inode, iloc: &iloc);
5929	if (err)
5930	goto out;
5931
5932	if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize)
5933	ext4_try_to_expand_extra_isize(inode, new_extra_isize: sbi->s_want_extra_isize,
5934	iloc, handle);
5935
5936	err = ext4_mark_iloc_dirty(handle, inode, iloc: &iloc);
5937	out:
5938	if (unlikely(err))
5939	ext4_error_inode_err(inode, func, line, 0, err,
5940	"mark_inode_dirty error");
5941	return err;
5942	}
5943
5944	/*
5945	* ext4_dirty_inode() is called from __mark_inode_dirty()
5946	*
5947	* We're really interested in the case where a file is being extended.
5948	* i_size has been changed by generic_commit_write() and we thus need
5949	* to include the updated inode in the current transaction.
5950	*
5951	* Also, dquot_alloc_block() will always dirty the inode when blocks
5952	* are allocated to the file.
5953	*
5954	* If the inode is marked synchronous, we don't honour that here - doing
5955	* so would cause a commit on atime updates, which we don't bother doing.
5956	* We handle synchronous inodes at the highest possible level.
5957	*/
5958	void ext4_dirty_inode(struct inode *inode, int flags)
5959	{
5960	handle_t *handle;
5961
5962	handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
5963	if (IS_ERR(ptr: handle))
5964	return;
5965	ext4_mark_inode_dirty(handle, inode);
5966	ext4_journal_stop(handle);
5967	}
5968
5969	int ext4_change_inode_journal_flag(struct inode *inode, int val)
5970	{
5971	journal_t *journal;
5972	handle_t *handle;
5973	int err;
5974	int alloc_ctx;
5975
5976	/*
5977	* We have to be very careful here: changing a data block's
5978	* journaling status dynamically is dangerous. If we write a
5979	* data block to the journal, change the status and then delete
5980	* that block, we risk forgetting to revoke the old log record
5981	* from the journal and so a subsequent replay can corrupt data.
5982	* So, first we make sure that the journal is empty and that
5983	* nobody is changing anything.
5984	*/
5985
5986	journal = EXT4_JOURNAL(inode);
5987	if (!journal)
5988	return 0;
5989	if (is_journal_aborted(journal))
5990	return -EROFS;
5991
5992	/* Wait for all existing dio workers */
5993	inode_dio_wait(inode);
5994
5995	/*
5996	* Before flushing the journal and switching inode's aops, we have
5997	* to flush all dirty data the inode has. There can be outstanding
5998	* delayed allocations, there can be unwritten extents created by
5999	* fallocate or buffered writes in dioread_nolock mode covered by
6000	* dirty data which can be converted only after flushing the dirty
6001	* data (and journalled aops don't know how to handle these cases).
6002	*/
6003	if (val) {
6004	filemap_invalidate_lock(mapping: inode->i_mapping);
6005	err = filemap_write_and_wait(mapping: inode->i_mapping);
6006	if (err < 0) {
6007	filemap_invalidate_unlock(mapping: inode->i_mapping);
6008	return err;
6009	}
6010	}
6011
6012	alloc_ctx = ext4_writepages_down_write(sb: inode->i_sb);
6013	jbd2_journal_lock_updates(journal);
6014
6015	/*
6016	* OK, there are no updates running now, and all cached data is
6017	* synced to disk. We are now in a completely consistent state
6018	* which doesn't have anything in the journal, and we know that
6019	* no filesystem updates are running, so it is safe to modify
6020	* the inode's in-core data-journaling state flag now.
6021	*/
6022
6023	if (val)
6024	ext4_set_inode_flag(inode, bit: EXT4_INODE_JOURNAL_DATA);
6025	else {
6026	err = jbd2_journal_flush(journal, flags: 0);
6027	if (err < 0) {
6028	jbd2_journal_unlock_updates(journal);
6029	ext4_writepages_up_write(sb: inode->i_sb, ctx: alloc_ctx);
6030	return err;
6031	}
6032	ext4_clear_inode_flag(inode, bit: EXT4_INODE_JOURNAL_DATA);
6033	}
6034	ext4_set_aops(inode);
6035
6036	jbd2_journal_unlock_updates(journal);
6037	ext4_writepages_up_write(sb: inode->i_sb, ctx: alloc_ctx);
6038
6039	if (val)
6040	filemap_invalidate_unlock(mapping: inode->i_mapping);
6041
6042	/* Finally we can mark the inode as dirty. */
6043
6044	handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
6045	if (IS_ERR(ptr: handle))
6046	return PTR_ERR(ptr: handle);
6047
6048	ext4_fc_mark_ineligible(sb: inode->i_sb,
6049	reason: EXT4_FC_REASON_JOURNAL_FLAG_CHANGE, handle);
6050	err = ext4_mark_inode_dirty(handle, inode);
6051	ext4_handle_sync(handle);
6052	ext4_journal_stop(handle);
6053	ext4_std_error(inode->i_sb, err);
6054
6055	return err;
6056	}
6057
6058	static int ext4_bh_unmapped(handle_t *handle, struct inode *inode,
6059	struct buffer_head *bh)
6060	{
6061	return !buffer_mapped(bh);
6062	}
6063
6064	vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf)
6065	{
6066	struct vm_area_struct *vma = vmf->vma;
6067	struct folio *folio = page_folio(vmf->page);
6068	loff_t size;
6069	unsigned long len;
6070	int err;
6071	vm_fault_t ret;
6072	struct file *file = vma->vm_file;
6073	struct inode *inode = file_inode(f: file);
6074	struct address_space *mapping = inode->i_mapping;
6075	handle_t *handle;
6076	get_block_t *get_block;
6077	int retries = 0;
6078
6079	if (unlikely(IS_IMMUTABLE(inode)))
6080	return VM_FAULT_SIGBUS;
6081
6082	sb_start_pagefault(sb: inode->i_sb);
6083	file_update_time(file: vma->vm_file);
6084
6085	filemap_invalidate_lock_shared(mapping);
6086
6087	err = ext4_convert_inline_data(inode);
6088	if (err)
6089	goto out_ret;
6090
6091	/*
6092	* On data journalling we skip straight to the transaction handle:
6093	* there's no delalloc; page truncated will be checked later; the
6094	* early return w/ all buffers mapped (calculates size/len) can't
6095	* be used; and there's no dioread_nolock, so only ext4_get_block.
6096	*/
6097	if (ext4_should_journal_data(inode))
6098	goto retry_alloc;
6099
6100	/* Delalloc case is easy... */
6101	if (test_opt(inode->i_sb, DELALLOC) &&
6102	!ext4_nonda_switch(sb: inode->i_sb)) {
6103	do {
6104	err = block_page_mkwrite(vma, vmf,
6105	get_block: ext4_da_get_block_prep);
6106	} while (err == -ENOSPC &&
6107	ext4_should_retry_alloc(sb: inode->i_sb, retries: &retries));
6108	goto out_ret;
6109	}
6110
6111	folio_lock(folio);
6112	size = i_size_read(inode);
6113	/* Page got truncated from under us? */
6114	if (folio->mapping != mapping || folio_pos(folio) > size) {
6115	folio_unlock(folio);
6116	ret = VM_FAULT_NOPAGE;
6117	goto out;
6118	}
6119
6120	len = folio_size(folio);
6121	if (folio_pos(folio) + len > size)
6122	len = size - folio_pos(folio);
6123	/*
6124	* Return if we have all the buffers mapped. This avoids the need to do
6125	* journal_start/journal_stop which can block and take a long time
6126	*
6127	* This cannot be done for data journalling, as we have to add the
6128	* inode to the transaction's list to writeprotect pages on commit.
6129	*/
6130	if (folio_buffers(folio)) {
6131	if (!ext4_walk_page_buffers(NULL, inode, folio_buffers(folio),
6132	from: 0, to: len, NULL,
6133	fn: ext4_bh_unmapped)) {
6134	/* Wait so that we don't change page under IO */
6135	folio_wait_stable(folio);
6136	ret = VM_FAULT_LOCKED;
6137	goto out;
6138	}
6139	}
6140	folio_unlock(folio);
6141	/* OK, we need to fill the hole... */
6142	if (ext4_should_dioread_nolock(inode))
6143	get_block = ext4_get_block_unwritten;
6144	else
6145	get_block = ext4_get_block;
6146	retry_alloc:
6147	handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
6148	ext4_writepage_trans_blocks(inode));
6149	if (IS_ERR(ptr: handle)) {
6150	ret = VM_FAULT_SIGBUS;
6151	goto out;
6152	}
6153	/*
6154	* Data journalling can't use block_page_mkwrite() because it
6155	* will set_buffer_dirty() before do_journal_get_write_access()
6156	* thus might hit warning messages for dirty metadata buffers.
6157	*/
6158	if (!ext4_should_journal_data(inode)) {
6159	err = block_page_mkwrite(vma, vmf, get_block);
6160	} else {
6161	folio_lock(folio);
6162	size = i_size_read(inode);
6163	/* Page got truncated from under us? */
6164	if (folio->mapping != mapping || folio_pos(folio) > size) {
6165	ret = VM_FAULT_NOPAGE;
6166	goto out_error;
6167	}
6168
6169	len = folio_size(folio);
6170	if (folio_pos(folio) + len > size)
6171	len = size - folio_pos(folio);
6172
6173	err = __block_write_begin(page: &folio->page, pos: 0, len, get_block: ext4_get_block);
6174	if (!err) {
6175	ret = VM_FAULT_SIGBUS;
6176	if (ext4_journal_folio_buffers(handle, folio, len))
6177	goto out_error;
6178	} else {
6179	folio_unlock(folio);
6180	}
6181	}
6182	ext4_journal_stop(handle);
6183	if (err == -ENOSPC && ext4_should_retry_alloc(sb: inode->i_sb, retries: &retries))
6184	goto retry_alloc;
6185	out_ret:
6186	ret = vmf_fs_error(err);
6187	out:
6188	filemap_invalidate_unlock_shared(mapping);
6189	sb_end_pagefault(sb: inode->i_sb);
6190	return ret;
6191	out_error:
6192	folio_unlock(folio);
6193	ext4_journal_stop(handle);
6194	goto out;
6195	}
6196