1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/fs/ext4/inode.c
4 *
5 * Copyright (C) 1992, 1993, 1994, 1995
6 * Remy Card (card@masi.ibp.fr)
7 * Laboratoire MASI - Institut Blaise Pascal
8 * Universite Pierre et Marie Curie (Paris VI)
9 *
10 * from
11 *
12 * linux/fs/minix/inode.c
13 *
14 * Copyright (C) 1991, 1992 Linus Torvalds
15 *
16 * 64-bit file support on 64-bit platforms by Jakub Jelinek
17 * (jj@sunsite.ms.mff.cuni.cz)
18 *
19 * Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000
20 */
21
22#include <linux/fs.h>
23#include <linux/time.h>
24#include <linux/highuid.h>
25#include <linux/pagemap.h>
26#include <linux/dax.h>
27#include <linux/quotaops.h>
28#include <linux/string.h>
29#include <linux/buffer_head.h>
30#include <linux/writeback.h>
31#include <linux/pagevec.h>
32#include <linux/mpage.h>
33#include <linux/namei.h>
34#include <linux/uio.h>
35#include <linux/bio.h>
36#include <linux/workqueue.h>
37#include <linux/kernel.h>
38#include <linux/printk.h>
39#include <linux/slab.h>
40#include <linux/bitops.h>
41#include <linux/iomap.h>
42#include <linux/iversion.h>
43
44#include "ext4_jbd2.h"
45#include "xattr.h"
46#include "acl.h"
47#include "truncate.h"
48
49#include <trace/events/ext4.h>
50
51#define MPAGE_DA_EXTENT_TAIL 0x01
52
53static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
54 struct ext4_inode_info *ei)
55{
56 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
57 __u32 csum;
58 __u16 dummy_csum = 0;
59 int offset = offsetof(struct ext4_inode, i_checksum_lo);
60 unsigned int csum_size = sizeof(dummy_csum);
61
62 csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)raw, offset);
63 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, csum_size);
64 offset += csum_size;
65 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
66 EXT4_GOOD_OLD_INODE_SIZE - offset);
67
68 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
69 offset = offsetof(struct ext4_inode, i_checksum_hi);
70 csum = ext4_chksum(sbi, csum, (__u8 *)raw +
71 EXT4_GOOD_OLD_INODE_SIZE,
72 offset - EXT4_GOOD_OLD_INODE_SIZE);
73 if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {
74 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum,
75 csum_size);
76 offset += csum_size;
77 }
78 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
79 EXT4_INODE_SIZE(inode->i_sb) - offset);
80 }
81
82 return csum;
83}
84
85static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,
86 struct ext4_inode_info *ei)
87{
88 __u32 provided, calculated;
89
90 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
91 cpu_to_le32(EXT4_OS_LINUX) ||
92 !ext4_has_metadata_csum(inode->i_sb))
93 return 1;
94
95 provided = le16_to_cpu(raw->i_checksum_lo);
96 calculated = ext4_inode_csum(inode, raw, ei);
97 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
98 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
99 provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;
100 else
101 calculated &= 0xFFFF;
102
103 return provided == calculated;
104}
105
106static void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,
107 struct ext4_inode_info *ei)
108{
109 __u32 csum;
110
111 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
112 cpu_to_le32(EXT4_OS_LINUX) ||
113 !ext4_has_metadata_csum(inode->i_sb))
114 return;
115
116 csum = ext4_inode_csum(inode, raw, ei);
117 raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);
118 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
119 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
120 raw->i_checksum_hi = cpu_to_le16(csum >> 16);
121}
122
123static inline int ext4_begin_ordered_truncate(struct inode *inode,
124 loff_t new_size)
125{
126 trace_ext4_begin_ordered_truncate(inode, new_size);
127 /*
128 * If jinode is zero, then we never opened the file for
129 * writing, so there's no need to call
130 * jbd2_journal_begin_ordered_truncate() since there's no
131 * outstanding writes we need to flush.
132 */
133 if (!EXT4_I(inode)->jinode)
134 return 0;
135 return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
136 EXT4_I(inode)->jinode,
137 new_size);
138}
139
140static void ext4_invalidatepage(struct page *page, unsigned int offset,
141 unsigned int length);
142static int __ext4_journalled_writepage(struct page *page, unsigned int len);
143static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh);
144static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
145 int pextents);
146
147/*
148 * Test whether an inode is a fast symlink.
149 * A fast symlink has its symlink data stored in ext4_inode_info->i_data.
150 */
151int ext4_inode_is_fast_symlink(struct inode *inode)
152{
153 if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
154 int ea_blocks = EXT4_I(inode)->i_file_acl ?
155 EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;
156
157 if (ext4_has_inline_data(inode))
158 return 0;
159
160 return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
161 }
162 return S_ISLNK(inode->i_mode) && inode->i_size &&
163 (inode->i_size < EXT4_N_BLOCKS * 4);
164}
165
166/*
167 * Restart the transaction associated with *handle. This does a commit,
168 * so before we call here everything must be consistently dirtied against
169 * this transaction.
170 */
171int ext4_truncate_restart_trans(handle_t *handle, struct inode *inode,
172 int nblocks)
173{
174 int ret;
175
176 /*
177 * Drop i_data_sem to avoid deadlock with ext4_map_blocks. At this
178 * moment, get_block can be called only for blocks inside i_size since
179 * page cache has been already dropped and writes are blocked by
180 * i_mutex. So we can safely drop the i_data_sem here.
181 */
182 BUG_ON(EXT4_JOURNAL(inode) == NULL);
183 jbd_debug(2, "restarting handle %p\n", handle);
184 up_write(&EXT4_I(inode)->i_data_sem);
185 ret = ext4_journal_restart(handle, nblocks);
186 down_write(&EXT4_I(inode)->i_data_sem);
187 ext4_discard_preallocations(inode);
188
189 return ret;
190}
191
192/*
193 * Called at the last iput() if i_nlink is zero.
194 */
195void ext4_evict_inode(struct inode *inode)
196{
197 handle_t *handle;
198 int err;
199 int extra_credits = 3;
200 struct ext4_xattr_inode_array *ea_inode_array = NULL;
201
202 trace_ext4_evict_inode(inode);
203
204 if (inode->i_nlink) {
205 /*
206 * When journalling data dirty buffers are tracked only in the
207 * journal. So although mm thinks everything is clean and
208 * ready for reaping the inode might still have some pages to
209 * write in the running transaction or waiting to be
210 * checkpointed. Thus calling jbd2_journal_invalidatepage()
211 * (via truncate_inode_pages()) to discard these buffers can
212 * cause data loss. Also even if we did not discard these
213 * buffers, we would have no way to find them after the inode
214 * is reaped and thus user could see stale data if he tries to
215 * read them before the transaction is checkpointed. So be
216 * careful and force everything to disk here... We use
217 * ei->i_datasync_tid to store the newest transaction
218 * containing inode's data.
219 *
220 * Note that directories do not have this problem because they
221 * don't use page cache.
222 */
223 if (inode->i_ino != EXT4_JOURNAL_INO &&
224 ext4_should_journal_data(inode) &&
225 (S_ISLNK(inode->i_mode) || S_ISREG(inode->i_mode)) &&
226 inode->i_data.nrpages) {
227 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
228 tid_t commit_tid = EXT4_I(inode)->i_datasync_tid;
229
230 jbd2_complete_transaction(journal, commit_tid);
231 filemap_write_and_wait(&inode->i_data);
232 }
233 truncate_inode_pages_final(&inode->i_data);
234
235 goto no_delete;
236 }
237
238 if (is_bad_inode(inode))
239 goto no_delete;
240 dquot_initialize(inode);
241
242 if (ext4_should_order_data(inode))
243 ext4_begin_ordered_truncate(inode, 0);
244 truncate_inode_pages_final(&inode->i_data);
245
246 /*
247 * Protect us against freezing - iput() caller didn't have to have any
248 * protection against it
249 */
250 sb_start_intwrite(inode->i_sb);
251
252 if (!IS_NOQUOTA(inode))
253 extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb);
254
255 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,
256 ext4_blocks_for_truncate(inode)+extra_credits);
257 if (IS_ERR(handle)) {
258 ext4_std_error(inode->i_sb, PTR_ERR(handle));
259 /*
260 * If we're going to skip the normal cleanup, we still need to
261 * make sure that the in-core orphan linked list is properly
262 * cleaned up.
263 */
264 ext4_orphan_del(NULL, inode);
265 sb_end_intwrite(inode->i_sb);
266 goto no_delete;
267 }
268
269 if (IS_SYNC(inode))
270 ext4_handle_sync(handle);
271
272 /*
273 * Set inode->i_size to 0 before calling ext4_truncate(). We need
274 * special handling of symlinks here because i_size is used to
275 * determine whether ext4_inode_info->i_data contains symlink data or
276 * block mappings. Setting i_size to 0 will remove its fast symlink
277 * status. Erase i_data so that it becomes a valid empty block map.
278 */
279 if (ext4_inode_is_fast_symlink(inode))
280 memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));
281 inode->i_size = 0;
282 err = ext4_mark_inode_dirty(handle, inode);
283 if (err) {
284 ext4_warning(inode->i_sb,
285 "couldn't mark inode dirty (err %d)", err);
286 goto stop_handle;
287 }
288 if (inode->i_blocks) {
289 err = ext4_truncate(inode);
290 if (err) {
291 ext4_error(inode->i_sb,
292 "couldn't truncate inode %lu (err %d)",
293 inode->i_ino, err);
294 goto stop_handle;
295 }
296 }
297
298 /* Remove xattr references. */
299 err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array,
300 extra_credits);
301 if (err) {
302 ext4_warning(inode->i_sb, "xattr delete (err %d)", err);
303stop_handle:
304 ext4_journal_stop(handle);
305 ext4_orphan_del(NULL, inode);
306 sb_end_intwrite(inode->i_sb);
307 ext4_xattr_inode_array_free(ea_inode_array);
308 goto no_delete;
309 }
310
311 /*
312 * Kill off the orphan record which ext4_truncate created.
313 * AKPM: I think this can be inside the above `if'.
314 * Note that ext4_orphan_del() has to be able to cope with the
315 * deletion of a non-existent orphan - this is because we don't
316 * know if ext4_truncate() actually created an orphan record.
317 * (Well, we could do this if we need to, but heck - it works)
318 */
319 ext4_orphan_del(handle, inode);
320 EXT4_I(inode)->i_dtime = (__u32)ktime_get_real_seconds();
321
322 /*
323 * One subtle ordering requirement: if anything has gone wrong
324 * (transaction abort, IO errors, whatever), then we can still
325 * do these next steps (the fs will already have been marked as
326 * having errors), but we can't free the inode if the mark_dirty
327 * fails.
328 */
329 if (ext4_mark_inode_dirty(handle, inode))
330 /* If that failed, just do the required in-core inode clear. */
331 ext4_clear_inode(inode);
332 else
333 ext4_free_inode(handle, inode);
334 ext4_journal_stop(handle);
335 sb_end_intwrite(inode->i_sb);
336 ext4_xattr_inode_array_free(ea_inode_array);
337 return;
338no_delete:
339 ext4_clear_inode(inode); /* We must guarantee clearing of inode... */
340}
341
342#ifdef CONFIG_QUOTA
343qsize_t *ext4_get_reserved_space(struct inode *inode)
344{
345 return &EXT4_I(inode)->i_reserved_quota;
346}
347#endif
348
349/*
350 * Called with i_data_sem down, which is important since we can call
351 * ext4_discard_preallocations() from here.
352 */
353void ext4_da_update_reserve_space(struct inode *inode,
354 int used, int quota_claim)
355{
356 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
357 struct ext4_inode_info *ei = EXT4_I(inode);
358
359 spin_lock(&ei->i_block_reservation_lock);
360 trace_ext4_da_update_reserve_space(inode, used, quota_claim);
361 if (unlikely(used > ei->i_reserved_data_blocks)) {
362 ext4_warning(inode->i_sb, "%s: ino %lu, used %d "
363 "with only %d reserved data blocks",
364 __func__, inode->i_ino, used,
365 ei->i_reserved_data_blocks);
366 WARN_ON(1);
367 used = ei->i_reserved_data_blocks;
368 }
369
370 /* Update per-inode reservations */
371 ei->i_reserved_data_blocks -= used;
372 percpu_counter_sub(&sbi->s_dirtyclusters_counter, used);
373
374 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
375
376 /* Update quota subsystem for data blocks */
377 if (quota_claim)
378 dquot_claim_block(inode, EXT4_C2B(sbi, used));
379 else {
380 /*
381 * We did fallocate with an offset that is already delayed
382 * allocated. So on delayed allocated writeback we should
383 * not re-claim the quota for fallocated blocks.
384 */
385 dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
386 }
387
388 /*
389 * If we have done all the pending block allocations and if
390 * there aren't any writers on the inode, we can discard the
391 * inode's preallocations.
392 */
393 if ((ei->i_reserved_data_blocks == 0) &&
394 !inode_is_open_for_write(inode))
395 ext4_discard_preallocations(inode);
396}
397
398static int __check_block_validity(struct inode *inode, const char *func,
399 unsigned int line,
400 struct ext4_map_blocks *map)
401{
402 if (!ext4_data_block_valid(EXT4_SB(inode->i_sb), map->m_pblk,
403 map->m_len)) {
404 ext4_error_inode(inode, func, line, map->m_pblk,
405 "lblock %lu mapped to illegal pblock %llu "
406 "(length %d)", (unsigned long) map->m_lblk,
407 map->m_pblk, map->m_len);
408 return -EFSCORRUPTED;
409 }
410 return 0;
411}
412
413int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
414 ext4_lblk_t len)
415{
416 int ret;
417
418 if (IS_ENCRYPTED(inode))
419 return fscrypt_zeroout_range(inode, lblk, pblk, len);
420
421 ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);
422 if (ret > 0)
423 ret = 0;
424
425 return ret;
426}
427
428#define check_block_validity(inode, map) \
429 __check_block_validity((inode), __func__, __LINE__, (map))
430
431#ifdef ES_AGGRESSIVE_TEST
432static void ext4_map_blocks_es_recheck(handle_t *handle,
433 struct inode *inode,
434 struct ext4_map_blocks *es_map,
435 struct ext4_map_blocks *map,
436 int flags)
437{
438 int retval;
439
440 map->m_flags = 0;
441 /*
442 * There is a race window that the result is not the same.
443 * e.g. xfstests #223 when dioread_nolock enables. The reason
444 * is that we lookup a block mapping in extent status tree with
445 * out taking i_data_sem. So at the time the unwritten extent
446 * could be converted.
447 */
448 down_read(&EXT4_I(inode)->i_data_sem);
449 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
450 retval = ext4_ext_map_blocks(handle, inode, map, flags &
451 EXT4_GET_BLOCKS_KEEP_SIZE);
452 } else {
453 retval = ext4_ind_map_blocks(handle, inode, map, flags &
454 EXT4_GET_BLOCKS_KEEP_SIZE);
455 }
456 up_read((&EXT4_I(inode)->i_data_sem));
457
458 /*
459 * We don't check m_len because extent will be collpased in status
460 * tree. So the m_len might not equal.
461 */
462 if (es_map->m_lblk != map->m_lblk ||
463 es_map->m_flags != map->m_flags ||
464 es_map->m_pblk != map->m_pblk) {
465 printk("ES cache assertion failed for inode: %lu "
466 "es_cached ex [%d/%d/%llu/%x] != "
467 "found ex [%d/%d/%llu/%x] retval %d flags %x\n",
468 inode->i_ino, es_map->m_lblk, es_map->m_len,
469 es_map->m_pblk, es_map->m_flags, map->m_lblk,
470 map->m_len, map->m_pblk, map->m_flags,
471 retval, flags);
472 }
473}
474#endif /* ES_AGGRESSIVE_TEST */
475
476/*
477 * The ext4_map_blocks() function tries to look up the requested blocks,
478 * and returns if the blocks are already mapped.
479 *
480 * Otherwise it takes the write lock of the i_data_sem and allocate blocks
481 * and store the allocated blocks in the result buffer head and mark it
482 * mapped.
483 *
484 * If file type is extents based, it will call ext4_ext_map_blocks(),
485 * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping
486 * based files
487 *
488 * On success, it returns the number of blocks being mapped or allocated. if
489 * create==0 and the blocks are pre-allocated and unwritten, the resulting @map
490 * is marked as unwritten. If the create == 1, it will mark @map as mapped.
491 *
492 * It returns 0 if plain look up failed (blocks have not been allocated), in
493 * that case, @map is returned as unmapped but we still do fill map->m_len to
494 * indicate the length of a hole starting at map->m_lblk.
495 *
496 * It returns the error in case of allocation failure.
497 */
498int ext4_map_blocks(handle_t *handle, struct inode *inode,
499 struct ext4_map_blocks *map, int flags)
500{
501 struct extent_status es;
502 int retval;
503 int ret = 0;
504#ifdef ES_AGGRESSIVE_TEST
505 struct ext4_map_blocks orig_map;
506
507 memcpy(&orig_map, map, sizeof(*map));
508#endif
509
510 map->m_flags = 0;
511 ext_debug("ext4_map_blocks(): inode %lu, flag %d, max_blocks %u,"
512 "logical block %lu\n", inode->i_ino, flags, map->m_len,
513 (unsigned long) map->m_lblk);
514
515 /*
516 * ext4_map_blocks returns an int, and m_len is an unsigned int
517 */
518 if (unlikely(map->m_len > INT_MAX))
519 map->m_len = INT_MAX;
520
521 /* We can handle the block number less than EXT_MAX_BLOCKS */
522 if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))
523 return -EFSCORRUPTED;
524
525 /* Lookup extent status tree firstly */
526 if (ext4_es_lookup_extent(inode, map->m_lblk, &es)) {
527 if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) {
528 map->m_pblk = ext4_es_pblock(&es) +
529 map->m_lblk - es.es_lblk;
530 map->m_flags |= ext4_es_is_written(&es) ?
531 EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;
532 retval = es.es_len - (map->m_lblk - es.es_lblk);
533 if (retval > map->m_len)
534 retval = map->m_len;
535 map->m_len = retval;
536 } else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) {
537 map->m_pblk = 0;
538 retval = es.es_len - (map->m_lblk - es.es_lblk);
539 if (retval > map->m_len)
540 retval = map->m_len;
541 map->m_len = retval;
542 retval = 0;
543 } else {
544 BUG_ON(1);
545 }
546#ifdef ES_AGGRESSIVE_TEST
547 ext4_map_blocks_es_recheck(handle, inode, map,
548 &orig_map, flags);
549#endif
550 goto found;
551 }
552
553 /*
554 * Try to see if we can get the block without requesting a new
555 * file system block.
556 */
557 down_read(&EXT4_I(inode)->i_data_sem);
558 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
559 retval = ext4_ext_map_blocks(handle, inode, map, flags &
560 EXT4_GET_BLOCKS_KEEP_SIZE);
561 } else {
562 retval = ext4_ind_map_blocks(handle, inode, map, flags &
563 EXT4_GET_BLOCKS_KEEP_SIZE);
564 }
565 if (retval > 0) {
566 unsigned int status;
567
568 if (unlikely(retval != map->m_len)) {
569 ext4_warning(inode->i_sb,
570 "ES len assertion failed for inode "
571 "%lu: retval %d != map->m_len %d",
572 inode->i_ino, retval, map->m_len);
573 WARN_ON(1);
574 }
575
576 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
577 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
578 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
579 !(status & EXTENT_STATUS_WRITTEN) &&
580 ext4_es_scan_range(inode, &ext4_es_is_delayed, map->m_lblk,
581 map->m_lblk + map->m_len - 1))
582 status |= EXTENT_STATUS_DELAYED;
583 ret = ext4_es_insert_extent(inode, map->m_lblk,
584 map->m_len, map->m_pblk, status);
585 if (ret < 0)
586 retval = ret;
587 }
588 up_read((&EXT4_I(inode)->i_data_sem));
589
590found:
591 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
592 ret = check_block_validity(inode, map);
593 if (ret != 0)
594 return ret;
595 }
596
597 /* If it is only a block(s) look up */
598 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
599 return retval;
600
601 /*
602 * Returns if the blocks have already allocated
603 *
604 * Note that if blocks have been preallocated
605 * ext4_ext_get_block() returns the create = 0
606 * with buffer head unmapped.
607 */
608 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
609 /*
610 * If we need to convert extent to unwritten
611 * we continue and do the actual work in
612 * ext4_ext_map_blocks()
613 */
614 if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))
615 return retval;
616
617 /*
618 * Here we clear m_flags because after allocating an new extent,
619 * it will be set again.
620 */
621 map->m_flags &= ~EXT4_MAP_FLAGS;
622
623 /*
624 * New blocks allocate and/or writing to unwritten extent
625 * will possibly result in updating i_data, so we take
626 * the write lock of i_data_sem, and call get_block()
627 * with create == 1 flag.
628 */
629 down_write(&EXT4_I(inode)->i_data_sem);
630
631 /*
632 * We need to check for EXT4 here because migrate
633 * could have changed the inode type in between
634 */
635 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
636 retval = ext4_ext_map_blocks(handle, inode, map, flags);
637 } else {
638 retval = ext4_ind_map_blocks(handle, inode, map, flags);
639
640 if (retval > 0 && map->m_flags & EXT4_MAP_NEW) {
641 /*
642 * We allocated new blocks which will result in
643 * i_data's format changing. Force the migrate
644 * to fail by clearing migrate flags
645 */
646 ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
647 }
648
649 /*
650 * Update reserved blocks/metadata blocks after successful
651 * block allocation which had been deferred till now. We don't
652 * support fallocate for non extent files. So we can update
653 * reserve space here.
654 */
655 if ((retval > 0) &&
656 (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE))
657 ext4_da_update_reserve_space(inode, retval, 1);
658 }
659
660 if (retval > 0) {
661 unsigned int status;
662
663 if (unlikely(retval != map->m_len)) {
664 ext4_warning(inode->i_sb,
665 "ES len assertion failed for inode "
666 "%lu: retval %d != map->m_len %d",
667 inode->i_ino, retval, map->m_len);
668 WARN_ON(1);
669 }
670
671 /*
672 * We have to zeroout blocks before inserting them into extent
673 * status tree. Otherwise someone could look them up there and
674 * use them before they are really zeroed. We also have to
675 * unmap metadata before zeroing as otherwise writeback can
676 * overwrite zeros with stale data from block device.
677 */
678 if (flags & EXT4_GET_BLOCKS_ZERO &&
679 map->m_flags & EXT4_MAP_MAPPED &&
680 map->m_flags & EXT4_MAP_NEW) {
681 ret = ext4_issue_zeroout(inode, map->m_lblk,
682 map->m_pblk, map->m_len);
683 if (ret) {
684 retval = ret;
685 goto out_sem;
686 }
687 }
688
689 /*
690 * If the extent has been zeroed out, we don't need to update
691 * extent status tree.
692 */
693 if ((flags & EXT4_GET_BLOCKS_PRE_IO) &&
694 ext4_es_lookup_extent(inode, map->m_lblk, &es)) {
695 if (ext4_es_is_written(&es))
696 goto out_sem;
697 }
698 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
699 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
700 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
701 !(status & EXTENT_STATUS_WRITTEN) &&
702 ext4_es_scan_range(inode, &ext4_es_is_delayed, map->m_lblk,
703 map->m_lblk + map->m_len - 1))
704 status |= EXTENT_STATUS_DELAYED;
705 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
706 map->m_pblk, status);
707 if (ret < 0) {
708 retval = ret;
709 goto out_sem;
710 }
711 }
712
713out_sem:
714 up_write((&EXT4_I(inode)->i_data_sem));
715 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
716 ret = check_block_validity(inode, map);
717 if (ret != 0)
718 return ret;
719
720 /*
721 * Inodes with freshly allocated blocks where contents will be
722 * visible after transaction commit must be on transaction's
723 * ordered data list.
724 */
725 if (map->m_flags & EXT4_MAP_NEW &&
726 !(map->m_flags & EXT4_MAP_UNWRITTEN) &&
727 !(flags & EXT4_GET_BLOCKS_ZERO) &&
728 !ext4_is_quota_file(inode) &&
729 ext4_should_order_data(inode)) {
730 if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
731 ret = ext4_jbd2_inode_add_wait(handle, inode);
732 else
733 ret = ext4_jbd2_inode_add_write(handle, inode);
734 if (ret)
735 return ret;
736 }
737 }
738 return retval;
739}
740
741/*
742 * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages
743 * we have to be careful as someone else may be manipulating b_state as well.
744 */
745static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)
746{
747 unsigned long old_state;
748 unsigned long new_state;
749
750 flags &= EXT4_MAP_FLAGS;
751
752 /* Dummy buffer_head? Set non-atomically. */
753 if (!bh->b_page) {
754 bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;
755 return;
756 }
757 /*
758 * Someone else may be modifying b_state. Be careful! This is ugly but
759 * once we get rid of using bh as a container for mapping information
760 * to pass to / from get_block functions, this can go away.
761 */
762 do {
763 old_state = READ_ONCE(bh->b_state);
764 new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;
765 } while (unlikely(
766 cmpxchg(&bh->b_state, old_state, new_state) != old_state));
767}
768
769static int _ext4_get_block(struct inode *inode, sector_t iblock,
770 struct buffer_head *bh, int flags)
771{
772 struct ext4_map_blocks map;
773 int ret = 0;
774
775 if (ext4_has_inline_data(inode))
776 return -ERANGE;
777
778 map.m_lblk = iblock;
779 map.m_len = bh->b_size >> inode->i_blkbits;
780
781 ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map,
782 flags);
783 if (ret > 0) {
784 map_bh(bh, inode->i_sb, map.m_pblk);
785 ext4_update_bh_state(bh, map.m_flags);
786 bh->b_size = inode->i_sb->s_blocksize * map.m_len;
787 ret = 0;
788 } else if (ret == 0) {
789 /* hole case, need to fill in bh->b_size */
790 bh->b_size = inode->i_sb->s_blocksize * map.m_len;
791 }
792 return ret;
793}
794
795int ext4_get_block(struct inode *inode, sector_t iblock,
796 struct buffer_head *bh, int create)
797{
798 return _ext4_get_block(inode, iblock, bh,
799 create ? EXT4_GET_BLOCKS_CREATE : 0);
800}
801
802/*
803 * Get block function used when preparing for buffered write if we require
804 * creating an unwritten extent if blocks haven't been allocated. The extent
805 * will be converted to written after the IO is complete.
806 */
807int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
808 struct buffer_head *bh_result, int create)
809{
810 ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n",
811 inode->i_ino, create);
812 return _ext4_get_block(inode, iblock, bh_result,
813 EXT4_GET_BLOCKS_IO_CREATE_EXT);
814}
815
816/* Maximum number of blocks we map for direct IO at once. */
817#define DIO_MAX_BLOCKS 4096
818
819/*
820 * Get blocks function for the cases that need to start a transaction -
821 * generally difference cases of direct IO and DAX IO. It also handles retries
822 * in case of ENOSPC.
823 */
824static int ext4_get_block_trans(struct inode *inode, sector_t iblock,
825 struct buffer_head *bh_result, int flags)
826{
827 int dio_credits;
828 handle_t *handle;
829 int retries = 0;
830 int ret;
831
832 /* Trim mapping request to maximum we can map at once for DIO */
833 if (bh_result->b_size >> inode->i_blkbits > DIO_MAX_BLOCKS)
834 bh_result->b_size = DIO_MAX_BLOCKS << inode->i_blkbits;
835 dio_credits = ext4_chunk_trans_blocks(inode,
836 bh_result->b_size >> inode->i_blkbits);
837retry:
838 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);
839 if (IS_ERR(handle))
840 return PTR_ERR(handle);
841
842 ret = _ext4_get_block(inode, iblock, bh_result, flags);
843 ext4_journal_stop(handle);
844
845 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
846 goto retry;
847 return ret;
848}
849
850/* Get block function for DIO reads and writes to inodes without extents */
851int ext4_dio_get_block(struct inode *inode, sector_t iblock,
852 struct buffer_head *bh, int create)
853{
854 /* We don't expect handle for direct IO */
855 WARN_ON_ONCE(ext4_journal_current_handle());
856
857 if (!create)
858 return _ext4_get_block(inode, iblock, bh, 0);
859 return ext4_get_block_trans(inode, iblock, bh, EXT4_GET_BLOCKS_CREATE);
860}
861
862/*
863 * Get block function for AIO DIO writes when we create unwritten extent if
864 * blocks are not allocated yet. The extent will be converted to written
865 * after IO is complete.
866 */
867static int ext4_dio_get_block_unwritten_async(struct inode *inode,
868 sector_t iblock, struct buffer_head *bh_result, int create)
869{
870 int ret;
871
872 /* We don't expect handle for direct IO */
873 WARN_ON_ONCE(ext4_journal_current_handle());
874
875 ret = ext4_get_block_trans(inode, iblock, bh_result,
876 EXT4_GET_BLOCKS_IO_CREATE_EXT);
877
878 /*
879 * When doing DIO using unwritten extents, we need io_end to convert
880 * unwritten extents to written on IO completion. We allocate io_end
881 * once we spot unwritten extent and store it in b_private. Generic
882 * DIO code keeps b_private set and furthermore passes the value to
883 * our completion callback in 'private' argument.
884 */
885 if (!ret && buffer_unwritten(bh_result)) {
886 if (!bh_result->b_private) {
887 ext4_io_end_t *io_end;
888
889 io_end = ext4_init_io_end(inode, GFP_KERNEL);
890 if (!io_end)
891 return -ENOMEM;
892 bh_result->b_private = io_end;
893 ext4_set_io_unwritten_flag(inode, io_end);
894 }
895 set_buffer_defer_completion(bh_result);
896 }
897
898 return ret;
899}
900
901/*
902 * Get block function for non-AIO DIO writes when we create unwritten extent if
903 * blocks are not allocated yet. The extent will be converted to written
904 * after IO is complete by ext4_direct_IO_write().
905 */
906static int ext4_dio_get_block_unwritten_sync(struct inode *inode,
907 sector_t iblock, struct buffer_head *bh_result, int create)
908{
909 int ret;
910
911 /* We don't expect handle for direct IO */
912 WARN_ON_ONCE(ext4_journal_current_handle());
913
914 ret = ext4_get_block_trans(inode, iblock, bh_result,
915 EXT4_GET_BLOCKS_IO_CREATE_EXT);
916
917 /*
918 * Mark inode as having pending DIO writes to unwritten extents.
919 * ext4_direct_IO_write() checks this flag and converts extents to
920 * written.
921 */
922 if (!ret && buffer_unwritten(bh_result))
923 ext4_set_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN);
924
925 return ret;
926}
927
928static int ext4_dio_get_block_overwrite(struct inode *inode, sector_t iblock,
929 struct buffer_head *bh_result, int create)
930{
931 int ret;
932
933 ext4_debug("ext4_dio_get_block_overwrite: inode %lu, create flag %d\n",
934 inode->i_ino, create);
935 /* We don't expect handle for direct IO */
936 WARN_ON_ONCE(ext4_journal_current_handle());
937
938 ret = _ext4_get_block(inode, iblock, bh_result, 0);
939 /*
940 * Blocks should have been preallocated! ext4_file_write_iter() checks
941 * that.
942 */
943 WARN_ON_ONCE(!buffer_mapped(bh_result) || buffer_unwritten(bh_result));
944
945 return ret;
946}
947
948
949/*
950 * `handle' can be NULL if create is zero
951 */
952struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
953 ext4_lblk_t block, int map_flags)
954{
955 struct ext4_map_blocks map;
956 struct buffer_head *bh;
957 int create = map_flags & EXT4_GET_BLOCKS_CREATE;
958 int err;
959
960 J_ASSERT(handle != NULL || create == 0);
961
962 map.m_lblk = block;
963 map.m_len = 1;
964 err = ext4_map_blocks(handle, inode, &map, map_flags);
965
966 if (err == 0)
967 return create ? ERR_PTR(-ENOSPC) : NULL;
968 if (err < 0)
969 return ERR_PTR(err);
970
971 bh = sb_getblk(inode->i_sb, map.m_pblk);
972 if (unlikely(!bh))
973 return ERR_PTR(-ENOMEM);
974 if (map.m_flags & EXT4_MAP_NEW) {
975 J_ASSERT(create != 0);
976 J_ASSERT(handle != NULL);
977
978 /*
979 * Now that we do not always journal data, we should
980 * keep in mind whether this should always journal the
981 * new buffer as metadata. For now, regular file
982 * writes use ext4_get_block instead, so it's not a
983 * problem.
984 */
985 lock_buffer(bh);
986 BUFFER_TRACE(bh, "call get_create_access");
987 err = ext4_journal_get_create_access(handle, bh);
988 if (unlikely(err)) {
989 unlock_buffer(bh);
990 goto errout;
991 }
992 if (!buffer_uptodate(bh)) {
993 memset(bh->b_data, 0, inode->i_sb->s_blocksize);
994 set_buffer_uptodate(bh);
995 }
996 unlock_buffer(bh);
997 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
998 err = ext4_handle_dirty_metadata(handle, inode, bh);
999 if (unlikely(err))
1000 goto errout;
1001 } else
1002 BUFFER_TRACE(bh, "not a new buffer");
1003 return bh;
1004errout:
1005 brelse(bh);
1006 return ERR_PTR(err);
1007}
1008
1009struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
1010 ext4_lblk_t block, int map_flags)
1011{
1012 struct buffer_head *bh;
1013
1014 bh = ext4_getblk(handle, inode, block, map_flags);
1015 if (IS_ERR(bh))
1016 return bh;
1017 if (!bh || buffer_uptodate(bh))
1018 return bh;
1019 ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1, &bh);
1020 wait_on_buffer(bh);
1021 if (buffer_uptodate(bh))
1022 return bh;
1023 put_bh(bh);
1024 return ERR_PTR(-EIO);
1025}
1026
1027/* Read a contiguous batch of blocks. */
1028int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count,
1029 bool wait, struct buffer_head **bhs)
1030{
1031 int i, err;
1032
1033 for (i = 0; i < bh_count; i++) {
1034 bhs[i] = ext4_getblk(NULL, inode, block + i, 0 /* map_flags */);
1035 if (IS_ERR(bhs[i])) {
1036 err = PTR_ERR(bhs[i]);
1037 bh_count = i;
1038 goto out_brelse;
1039 }
1040 }
1041
1042 for (i = 0; i < bh_count; i++)
1043 /* Note that NULL bhs[i] is valid because of holes. */
1044 if (bhs[i] && !buffer_uptodate(bhs[i]))
1045 ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1,
1046 &bhs[i]);
1047
1048 if (!wait)
1049 return 0;
1050
1051 for (i = 0; i < bh_count; i++)
1052 if (bhs[i])
1053 wait_on_buffer(bhs[i]);
1054
1055 for (i = 0; i < bh_count; i++) {
1056 if (bhs[i] && !buffer_uptodate(bhs[i])) {
1057 err = -EIO;
1058 goto out_brelse;
1059 }
1060 }
1061 return 0;
1062
1063out_brelse:
1064 for (i = 0; i < bh_count; i++) {
1065 brelse(bhs[i]);
1066 bhs[i] = NULL;
1067 }
1068 return err;
1069}
1070
1071int ext4_walk_page_buffers(handle_t *handle,
1072 struct buffer_head *head,
1073 unsigned from,
1074 unsigned to,
1075 int *partial,
1076 int (*fn)(handle_t *handle,
1077 struct buffer_head *bh))
1078{
1079 struct buffer_head *bh;
1080 unsigned block_start, block_end;
1081 unsigned blocksize = head->b_size;
1082 int err, ret = 0;
1083 struct buffer_head *next;
1084
1085 for (bh = head, block_start = 0;
1086 ret == 0 && (bh != head || !block_start);
1087 block_start = block_end, bh = next) {
1088 next = bh->b_this_page;
1089 block_end = block_start + blocksize;
1090 if (block_end <= from || block_start >= to) {
1091 if (partial && !buffer_uptodate(bh))
1092 *partial = 1;
1093 continue;
1094 }
1095 err = (*fn)(handle, bh);
1096 if (!ret)
1097 ret = err;
1098 }
1099 return ret;
1100}
1101
1102/*
1103 * To preserve ordering, it is essential that the hole instantiation and
1104 * the data write be encapsulated in a single transaction. We cannot
1105 * close off a transaction and start a new one between the ext4_get_block()
1106 * and the commit_write(). So doing the jbd2_journal_start at the start of
1107 * prepare_write() is the right place.
1108 *
1109 * Also, this function can nest inside ext4_writepage(). In that case, we
1110 * *know* that ext4_writepage() has generated enough buffer credits to do the
1111 * whole page. So we won't block on the journal in that case, which is good,
1112 * because the caller may be PF_MEMALLOC.
1113 *
1114 * By accident, ext4 can be reentered when a transaction is open via
1115 * quota file writes. If we were to commit the transaction while thus
1116 * reentered, there can be a deadlock - we would be holding a quota
1117 * lock, and the commit would never complete if another thread had a
1118 * transaction open and was blocking on the quota lock - a ranking
1119 * violation.
1120 *
1121 * So what we do is to rely on the fact that jbd2_journal_stop/journal_start
1122 * will _not_ run commit under these circumstances because handle->h_ref
1123 * is elevated. We'll still have enough credits for the tiny quotafile
1124 * write.
1125 */
1126int do_journal_get_write_access(handle_t *handle,
1127 struct buffer_head *bh)
1128{
1129 int dirty = buffer_dirty(bh);
1130 int ret;
1131
1132 if (!buffer_mapped(bh) || buffer_freed(bh))
1133 return 0;
1134 /*
1135 * __block_write_begin() could have dirtied some buffers. Clean
1136 * the dirty bit as jbd2_journal_get_write_access() could complain
1137 * otherwise about fs integrity issues. Setting of the dirty bit
1138 * by __block_write_begin() isn't a real problem here as we clear
1139 * the bit before releasing a page lock and thus writeback cannot
1140 * ever write the buffer.
1141 */
1142 if (dirty)
1143 clear_buffer_dirty(bh);
1144 BUFFER_TRACE(bh, "get write access");
1145 ret = ext4_journal_get_write_access(handle, bh);
1146 if (!ret && dirty)
1147 ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1148 return ret;
1149}
1150
1151#ifdef CONFIG_FS_ENCRYPTION
1152static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
1153 get_block_t *get_block)
1154{
1155 unsigned from = pos & (PAGE_SIZE - 1);
1156 unsigned to = from + len;
1157 struct inode *inode = page->mapping->host;
1158 unsigned block_start, block_end;
1159 sector_t block;
1160 int err = 0;
1161 unsigned blocksize = inode->i_sb->s_blocksize;
1162 unsigned bbits;
1163 struct buffer_head *bh, *head, *wait[2], **wait_bh = wait;
1164 bool decrypt = false;
1165
1166 BUG_ON(!PageLocked(page));
1167 BUG_ON(from > PAGE_SIZE);
1168 BUG_ON(to > PAGE_SIZE);
1169 BUG_ON(from > to);
1170
1171 if (!page_has_buffers(page))
1172 create_empty_buffers(page, blocksize, 0);
1173 head = page_buffers(page);
1174 bbits = ilog2(blocksize);
1175 block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1176
1177 for (bh = head, block_start = 0; bh != head || !block_start;
1178 block++, block_start = block_end, bh = bh->b_this_page) {
1179 block_end = block_start + blocksize;
1180 if (block_end <= from || block_start >= to) {
1181 if (PageUptodate(page)) {
1182 if (!buffer_uptodate(bh))
1183 set_buffer_uptodate(bh);
1184 }
1185 continue;
1186 }
1187 if (buffer_new(bh))
1188 clear_buffer_new(bh);
1189 if (!buffer_mapped(bh)) {
1190 WARN_ON(bh->b_size != blocksize);
1191 err = get_block(inode, block, bh, 1);
1192 if (err)
1193 break;
1194 if (buffer_new(bh)) {
1195 if (PageUptodate(page)) {
1196 clear_buffer_new(bh);
1197 set_buffer_uptodate(bh);
1198 mark_buffer_dirty(bh);
1199 continue;
1200 }
1201 if (block_end > to || block_start < from)
1202 zero_user_segments(page, to, block_end,
1203 block_start, from);
1204 continue;
1205 }
1206 }
1207 if (PageUptodate(page)) {
1208 if (!buffer_uptodate(bh))
1209 set_buffer_uptodate(bh);
1210 continue;
1211 }
1212 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1213 !buffer_unwritten(bh) &&
1214 (block_start < from || block_end > to)) {
1215 ll_rw_block(REQ_OP_READ, 0, 1, &bh);
1216 *wait_bh++ = bh;
1217 decrypt = IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode);
1218 }
1219 }
1220 /*
1221 * If we issued read requests, let them complete.
1222 */
1223 while (wait_bh > wait) {
1224 wait_on_buffer(*--wait_bh);
1225 if (!buffer_uptodate(*wait_bh))
1226 err = -EIO;
1227 }
1228 if (unlikely(err))
1229 page_zero_new_buffers(page, from, to);
1230 else if (decrypt)
1231 err = fscrypt_decrypt_page(page->mapping->host, page,
1232 PAGE_SIZE, 0, page->index);
1233 return err;
1234}
1235#endif
1236
1237static int ext4_write_begin(struct file *file, struct address_space *mapping,
1238 loff_t pos, unsigned len, unsigned flags,
1239 struct page **pagep, void **fsdata)
1240{
1241 struct inode *inode = mapping->host;
1242 int ret, needed_blocks;
1243 handle_t *handle;
1244 int retries = 0;
1245 struct page *page;
1246 pgoff_t index;
1247 unsigned from, to;
1248
1249 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
1250 return -EIO;
1251
1252 trace_ext4_write_begin(inode, pos, len, flags);
1253 /*
1254 * Reserve one block more for addition to orphan list in case
1255 * we allocate blocks but write fails for some reason
1256 */
1257 needed_blocks = ext4_writepage_trans_blocks(inode) + 1;
1258 index = pos >> PAGE_SHIFT;
1259 from = pos & (PAGE_SIZE - 1);
1260 to = from + len;
1261
1262 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
1263 ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,
1264 flags, pagep);
1265 if (ret < 0)
1266 return ret;
1267 if (ret == 1)
1268 return 0;
1269 }
1270
1271 /*
1272 * grab_cache_page_write_begin() can take a long time if the
1273 * system is thrashing due to memory pressure, or if the page
1274 * is being written back. So grab it first before we start
1275 * the transaction handle. This also allows us to allocate
1276 * the page (if needed) without using GFP_NOFS.
1277 */
1278retry_grab:
1279 page = grab_cache_page_write_begin(mapping, index, flags);
1280 if (!page)
1281 return -ENOMEM;
1282 unlock_page(page);
1283
1284retry_journal:
1285 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
1286 if (IS_ERR(handle)) {
1287 put_page(page);
1288 return PTR_ERR(handle);
1289 }
1290
1291 lock_page(page);
1292 if (page->mapping != mapping) {
1293 /* The page got truncated from under us */
1294 unlock_page(page);
1295 put_page(page);
1296 ext4_journal_stop(handle);
1297 goto retry_grab;
1298 }
1299 /* In case writeback began while the page was unlocked */
1300 wait_for_stable_page(page);
1301
1302#ifdef CONFIG_FS_ENCRYPTION
1303 if (ext4_should_dioread_nolock(inode))
1304 ret = ext4_block_write_begin(page, pos, len,
1305 ext4_get_block_unwritten);
1306 else
1307 ret = ext4_block_write_begin(page, pos, len,
1308 ext4_get_block);
1309#else
1310 if (ext4_should_dioread_nolock(inode))
1311 ret = __block_write_begin(page, pos, len,
1312 ext4_get_block_unwritten);
1313 else
1314 ret = __block_write_begin(page, pos, len, ext4_get_block);
1315#endif
1316 if (!ret && ext4_should_journal_data(inode)) {
1317 ret = ext4_walk_page_buffers(handle, page_buffers(page),
1318 from, to, NULL,
1319 do_journal_get_write_access);
1320 }
1321
1322 if (ret) {
1323 unlock_page(page);
1324 /*
1325 * __block_write_begin may have instantiated a few blocks
1326 * outside i_size. Trim these off again. Don't need
1327 * i_size_read because we hold i_mutex.
1328 *
1329 * Add inode to orphan list in case we crash before
1330 * truncate finishes
1331 */
1332 if (pos + len > inode->i_size && ext4_can_truncate(inode))
1333 ext4_orphan_add(handle, inode);
1334
1335 ext4_journal_stop(handle);
1336 if (pos + len > inode->i_size) {
1337 ext4_truncate_failed_write(inode);
1338 /*
1339 * If truncate failed early the inode might
1340 * still be on the orphan list; we need to
1341 * make sure the inode is removed from the
1342 * orphan list in that case.
1343 */
1344 if (inode->i_nlink)
1345 ext4_orphan_del(NULL, inode);
1346 }
1347
1348 if (ret == -ENOSPC &&
1349 ext4_should_retry_alloc(inode->i_sb, &retries))
1350 goto retry_journal;
1351 put_page(page);
1352 return ret;
1353 }
1354 *pagep = page;
1355 return ret;
1356}
1357
1358/* For write_end() in data=journal mode */
1359static int write_end_fn(handle_t *handle, struct buffer_head *bh)
1360{
1361 int ret;
1362 if (!buffer_mapped(bh) || buffer_freed(bh))
1363 return 0;
1364 set_buffer_uptodate(bh);
1365 ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1366 clear_buffer_meta(bh);
1367 clear_buffer_prio(bh);
1368 return ret;
1369}
1370
1371/*
1372 * We need to pick up the new inode size which generic_commit_write gave us
1373 * `file' can be NULL - eg, when called from page_symlink().
1374 *
1375 * ext4 never places buffers on inode->i_mapping->private_list. metadata
1376 * buffers are managed internally.
1377 */
1378static int ext4_write_end(struct file *file,
1379 struct address_space *mapping,
1380 loff_t pos, unsigned len, unsigned copied,
1381 struct page *page, void *fsdata)
1382{
1383 handle_t *handle = ext4_journal_current_handle();
1384 struct inode *inode = mapping->host;
1385 loff_t old_size = inode->i_size;
1386 int ret = 0, ret2;
1387 int i_size_changed = 0;
1388 int inline_data = ext4_has_inline_data(inode);
1389
1390 trace_ext4_write_end(inode, pos, len, copied);
1391 if (inline_data) {
1392 ret = ext4_write_inline_data_end(inode, pos, len,
1393 copied, page);
1394 if (ret < 0) {
1395 unlock_page(page);
1396 put_page(page);
1397 goto errout;
1398 }
1399 copied = ret;
1400 } else
1401 copied = block_write_end(file, mapping, pos,
1402 len, copied, page, fsdata);
1403 /*
1404 * it's important to update i_size while still holding page lock:
1405 * page writeout could otherwise come in and zero beyond i_size.
1406 */
1407 i_size_changed = ext4_update_inode_size(inode, pos + copied);
1408 unlock_page(page);
1409 put_page(page);
1410
1411 if (old_size < pos)
1412 pagecache_isize_extended(inode, old_size, pos);
1413 /*
1414 * Don't mark the inode dirty under page lock. First, it unnecessarily
1415 * makes the holding time of page lock longer. Second, it forces lock
1416 * ordering of page lock and transaction start for journaling
1417 * filesystems.
1418 */
1419 if (i_size_changed || inline_data)
1420 ext4_mark_inode_dirty(handle, inode);
1421
1422 if (pos + len > inode->i_size && ext4_can_truncate(inode))
1423 /* if we have allocated more blocks and copied
1424 * less. We will have blocks allocated outside
1425 * inode->i_size. So truncate them
1426 */
1427 ext4_orphan_add(handle, inode);
1428errout:
1429 ret2 = ext4_journal_stop(handle);
1430 if (!ret)
1431 ret = ret2;
1432
1433 if (pos + len > inode->i_size) {
1434 ext4_truncate_failed_write(inode);
1435 /*
1436 * If truncate failed early the inode might still be
1437 * on the orphan list; we need to make sure the inode
1438 * is removed from the orphan list in that case.
1439 */
1440 if (inode->i_nlink)
1441 ext4_orphan_del(NULL, inode);
1442 }
1443
1444 return ret ? ret : copied;
1445}
1446
1447/*
1448 * This is a private version of page_zero_new_buffers() which doesn't
1449 * set the buffer to be dirty, since in data=journalled mode we need
1450 * to call ext4_handle_dirty_metadata() instead.
1451 */
1452static void ext4_journalled_zero_new_buffers(handle_t *handle,
1453 struct page *page,
1454 unsigned from, unsigned to)
1455{
1456 unsigned int block_start = 0, block_end;
1457 struct buffer_head *head, *bh;
1458
1459 bh = head = page_buffers(page);
1460 do {
1461 block_end = block_start + bh->b_size;
1462 if (buffer_new(bh)) {
1463 if (block_end > from && block_start < to) {
1464 if (!PageUptodate(page)) {
1465 unsigned start, size;
1466
1467 start = max(from, block_start);
1468 size = min(to, block_end) - start;
1469
1470 zero_user(page, start, size);
1471 write_end_fn(handle, bh);
1472 }
1473 clear_buffer_new(bh);
1474 }
1475 }
1476 block_start = block_end;
1477 bh = bh->b_this_page;
1478 } while (bh != head);
1479}
1480
1481static int ext4_journalled_write_end(struct file *file,
1482 struct address_space *mapping,
1483 loff_t pos, unsigned len, unsigned copied,
1484 struct page *page, void *fsdata)
1485{
1486 handle_t *handle = ext4_journal_current_handle();
1487 struct inode *inode = mapping->host;
1488 loff_t old_size = inode->i_size;
1489 int ret = 0, ret2;
1490 int partial = 0;
1491 unsigned from, to;
1492 int size_changed = 0;
1493 int inline_data = ext4_has_inline_data(inode);
1494
1495 trace_ext4_journalled_write_end(inode, pos, len, copied);
1496 from = pos & (PAGE_SIZE - 1);
1497 to = from + len;
1498
1499 BUG_ON(!ext4_handle_valid(handle));
1500
1501 if (inline_data) {
1502 ret = ext4_write_inline_data_end(inode, pos, len,
1503 copied, page);
1504 if (ret < 0) {
1505 unlock_page(page);
1506 put_page(page);
1507 goto errout;
1508 }
1509 copied = ret;
1510 } else if (unlikely(copied < len) && !PageUptodate(page)) {
1511 copied = 0;
1512 ext4_journalled_zero_new_buffers(handle, page, from, to);
1513 } else {
1514 if (unlikely(copied < len))
1515 ext4_journalled_zero_new_buffers(handle, page,
1516 from + copied, to);
1517 ret = ext4_walk_page_buffers(handle, page_buffers(page), from,
1518 from + copied, &partial,
1519 write_end_fn);
1520 if (!partial)
1521 SetPageUptodate(page);
1522 }
1523 size_changed = ext4_update_inode_size(inode, pos + copied);
1524 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
1525 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1526 unlock_page(page);
1527 put_page(page);
1528
1529 if (old_size < pos)
1530 pagecache_isize_extended(inode, old_size, pos);
1531
1532 if (size_changed || inline_data) {
1533 ret2 = ext4_mark_inode_dirty(handle, inode);
1534 if (!ret)
1535 ret = ret2;
1536 }
1537
1538 if (pos + len > inode->i_size && ext4_can_truncate(inode))
1539 /* if we have allocated more blocks and copied
1540 * less. We will have blocks allocated outside
1541 * inode->i_size. So truncate them
1542 */
1543 ext4_orphan_add(handle, inode);
1544
1545errout:
1546 ret2 = ext4_journal_stop(handle);
1547 if (!ret)
1548 ret = ret2;
1549 if (pos + len > inode->i_size) {
1550 ext4_truncate_failed_write(inode);
1551 /*
1552 * If truncate failed early the inode might still be
1553 * on the orphan list; we need to make sure the inode
1554 * is removed from the orphan list in that case.
1555 */
1556 if (inode->i_nlink)
1557 ext4_orphan_del(NULL, inode);
1558 }
1559
1560 return ret ? ret : copied;
1561}
1562
1563/*
1564 * Reserve space for a single cluster
1565 */
1566static int ext4_da_reserve_space(struct inode *inode)
1567{
1568 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1569 struct ext4_inode_info *ei = EXT4_I(inode);
1570 int ret;
1571
1572 /*
1573 * We will charge metadata quota at writeout time; this saves
1574 * us from metadata over-estimation, though we may go over by
1575 * a small amount in the end. Here we just reserve for data.
1576 */
1577 ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1));
1578 if (ret)
1579 return ret;
1580
1581 spin_lock(&ei->i_block_reservation_lock);
1582 if (ext4_claim_free_clusters(sbi, 1, 0)) {
1583 spin_unlock(&ei->i_block_reservation_lock);
1584 dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1));
1585 return -ENOSPC;
1586 }
1587 ei->i_reserved_data_blocks++;
1588 trace_ext4_da_reserve_space(inode);
1589 spin_unlock(&ei->i_block_reservation_lock);
1590
1591 return 0; /* success */
1592}
1593
1594void ext4_da_release_space(struct inode *inode, int to_free)
1595{
1596 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1597 struct ext4_inode_info *ei = EXT4_I(inode);
1598
1599 if (!to_free)
1600 return; /* Nothing to release, exit */
1601
1602 spin_lock(&EXT4_I(inode)->i_block_reservation_lock);
1603
1604 trace_ext4_da_release_space(inode, to_free);
1605 if (unlikely(to_free > ei->i_reserved_data_blocks)) {
1606 /*
1607 * if there aren't enough reserved blocks, then the
1608 * counter is messed up somewhere. Since this
1609 * function is called from invalidate page, it's
1610 * harmless to return without any action.
1611 */
1612 ext4_warning(inode->i_sb, "ext4_da_release_space: "
1613 "ino %lu, to_free %d with only %d reserved "
1614 "data blocks", inode->i_ino, to_free,
1615 ei->i_reserved_data_blocks);
1616 WARN_ON(1);
1617 to_free = ei->i_reserved_data_blocks;
1618 }
1619 ei->i_reserved_data_blocks -= to_free;
1620
1621 /* update fs dirty data blocks counter */
1622 percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);
1623
1624 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
1625
1626 dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
1627}
1628
1629static void ext4_da_page_release_reservation(struct page *page,
1630 unsigned int offset,
1631 unsigned int length)
1632{
1633 int contiguous_blks = 0;
1634 struct buffer_head *head, *bh;
1635 unsigned int curr_off = 0;
1636 struct inode *inode = page->mapping->host;
1637 unsigned int stop = offset + length;
1638 ext4_fsblk_t lblk;
1639
1640 BUG_ON(stop > PAGE_SIZE || stop < length);
1641
1642 head = page_buffers(page);
1643 bh = head;
1644 do {
1645 unsigned int next_off = curr_off + bh->b_size;
1646
1647 if (next_off > stop)
1648 break;
1649
1650 if ((offset <= curr_off) && (buffer_delay(bh))) {
1651 contiguous_blks++;
1652 clear_buffer_delay(bh);
1653 } else if (contiguous_blks) {
1654 lblk = page->index <<
1655 (PAGE_SHIFT - inode->i_blkbits);
1656 lblk += (curr_off >> inode->i_blkbits) -
1657 contiguous_blks;
1658 ext4_es_remove_blks(inode, lblk, contiguous_blks);
1659 contiguous_blks = 0;
1660 }
1661 curr_off = next_off;
1662 } while ((bh = bh->b_this_page) != head);
1663
1664 if (contiguous_blks) {
1665 lblk = page->index << (PAGE_SHIFT - inode->i_blkbits);
1666 lblk += (curr_off >> inode->i_blkbits) - contiguous_blks;
1667 ext4_es_remove_blks(inode, lblk, contiguous_blks);
1668 }
1669
1670}
1671
1672/*
1673 * Delayed allocation stuff
1674 */
1675
1676struct mpage_da_data {
1677 struct inode *inode;
1678 struct writeback_control *wbc;
1679
1680 pgoff_t first_page; /* The first page to write */
1681 pgoff_t next_page; /* Current page to examine */
1682 pgoff_t last_page; /* Last page to examine */
1683 /*
1684 * Extent to map - this can be after first_page because that can be
1685 * fully mapped. We somewhat abuse m_flags to store whether the extent
1686 * is delalloc or unwritten.
1687 */
1688 struct ext4_map_blocks map;
1689 struct ext4_io_submit io_submit; /* IO submission data */
1690 unsigned int do_map:1;
1691};
1692
1693static void mpage_release_unused_pages(struct mpage_da_data *mpd,
1694 bool invalidate)
1695{
1696 int nr_pages, i;
1697 pgoff_t index, end;
1698 struct pagevec pvec;
1699 struct inode *inode = mpd->inode;
1700 struct address_space *mapping = inode->i_mapping;
1701
1702 /* This is necessary when next_page == 0. */
1703 if (mpd->first_page >= mpd->next_page)
1704 return;
1705
1706 index = mpd->first_page;
1707 end = mpd->next_page - 1;
1708 if (invalidate) {
1709 ext4_lblk_t start, last;
1710 start = index << (PAGE_SHIFT - inode->i_blkbits);
1711 last = end << (PAGE_SHIFT - inode->i_blkbits);
1712 ext4_es_remove_extent(inode, start, last - start + 1);
1713 }
1714
1715 pagevec_init(&pvec);
1716 while (index <= end) {
1717 nr_pages = pagevec_lookup_range(&pvec, mapping, &index, end);
1718 if (nr_pages == 0)
1719 break;
1720 for (i = 0; i < nr_pages; i++) {
1721 struct page *page = pvec.pages[i];
1722
1723 BUG_ON(!PageLocked(page));
1724 BUG_ON(PageWriteback(page));
1725 if (invalidate) {
1726 if (page_mapped(page))
1727 clear_page_dirty_for_io(page);
1728 block_invalidatepage(page, 0, PAGE_SIZE);
1729 ClearPageUptodate(page);
1730 }
1731 unlock_page(page);
1732 }
1733 pagevec_release(&pvec);
1734 }
1735}
1736
1737static void ext4_print_free_blocks(struct inode *inode)
1738{
1739 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1740 struct super_block *sb = inode->i_sb;
1741 struct ext4_inode_info *ei = EXT4_I(inode);
1742
1743 ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
1744 EXT4_C2B(EXT4_SB(inode->i_sb),
1745 ext4_count_free_clusters(sb)));
1746 ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
1747 ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
1748 (long long) EXT4_C2B(EXT4_SB(sb),
1749 percpu_counter_sum(&sbi->s_freeclusters_counter)));
1750 ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
1751 (long long) EXT4_C2B(EXT4_SB(sb),
1752 percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
1753 ext4_msg(sb, KERN_CRIT, "Block reservation details");
1754 ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
1755 ei->i_reserved_data_blocks);
1756 return;
1757}
1758
1759static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh)
1760{
1761 return (buffer_delay(bh) || buffer_unwritten(bh)) && buffer_dirty(bh);
1762}
1763
1764/*
1765 * ext4_insert_delayed_block - adds a delayed block to the extents status
1766 * tree, incrementing the reserved cluster/block
1767 * count or making a pending reservation
1768 * where needed
1769 *
1770 * @inode - file containing the newly added block
1771 * @lblk - logical block to be added
1772 *
1773 * Returns 0 on success, negative error code on failure.
1774 */
1775static int ext4_insert_delayed_block(struct inode *inode, ext4_lblk_t lblk)
1776{
1777 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1778 int ret;
1779 bool allocated = false;
1780
1781 /*
1782 * If the cluster containing lblk is shared with a delayed,
1783 * written, or unwritten extent in a bigalloc file system, it's
1784 * already been accounted for and does not need to be reserved.
1785 * A pending reservation must be made for the cluster if it's
1786 * shared with a written or unwritten extent and doesn't already
1787 * have one. Written and unwritten extents can be purged from the
1788 * extents status tree if the system is under memory pressure, so
1789 * it's necessary to examine the extent tree if a search of the
1790 * extents status tree doesn't get a match.
1791 */
1792 if (sbi->s_cluster_ratio == 1) {
1793 ret = ext4_da_reserve_space(inode);
1794 if (ret != 0) /* ENOSPC */
1795 goto errout;
1796 } else { /* bigalloc */
1797 if (!ext4_es_scan_clu(inode, &ext4_es_is_delonly, lblk)) {
1798 if (!ext4_es_scan_clu(inode,
1799 &ext4_es_is_mapped, lblk)) {
1800 ret = ext4_clu_mapped(inode,
1801 EXT4_B2C(sbi, lblk));
1802 if (ret < 0)
1803 goto errout;
1804 if (ret == 0) {
1805 ret = ext4_da_reserve_space(inode);
1806 if (ret != 0) /* ENOSPC */
1807 goto errout;
1808 } else {
1809 allocated = true;
1810 }
1811 } else {
1812 allocated = true;
1813 }
1814 }
1815 }
1816
1817 ret = ext4_es_insert_delayed_block(inode, lblk, allocated);
1818
1819errout:
1820 return ret;
1821}
1822
1823/*
1824 * This function is grabs code from the very beginning of
1825 * ext4_map_blocks, but assumes that the caller is from delayed write
1826 * time. This function looks up the requested blocks and sets the
1827 * buffer delay bit under the protection of i_data_sem.
1828 */
1829static int ext4_da_map_blocks(struct inode *inode, sector_t iblock,
1830 struct ext4_map_blocks *map,
1831 struct buffer_head *bh)
1832{
1833 struct extent_status es;
1834 int retval;
1835 sector_t invalid_block = ~((sector_t) 0xffff);
1836#ifdef ES_AGGRESSIVE_TEST
1837 struct ext4_map_blocks orig_map;
1838
1839 memcpy(&orig_map, map, sizeof(*map));
1840#endif
1841
1842 if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))
1843 invalid_block = ~0;
1844
1845 map->m_flags = 0;
1846 ext_debug("ext4_da_map_blocks(): inode %lu, max_blocks %u,"
1847 "logical block %lu\n", inode->i_ino, map->m_len,
1848 (unsigned long) map->m_lblk);
1849
1850 /* Lookup extent status tree firstly */
1851 if (ext4_es_lookup_extent(inode, iblock, &es)) {
1852 if (ext4_es_is_hole(&es)) {
1853 retval = 0;
1854 down_read(&EXT4_I(inode)->i_data_sem);
1855 goto add_delayed;
1856 }
1857
1858 /*
1859 * Delayed extent could be allocated by fallocate.
1860 * So we need to check it.
1861 */
1862 if (ext4_es_is_delayed(&es) && !ext4_es_is_unwritten(&es)) {
1863 map_bh(bh, inode->i_sb, invalid_block);
1864 set_buffer_new(bh);
1865 set_buffer_delay(bh);
1866 return 0;
1867 }
1868
1869 map->m_pblk = ext4_es_pblock(&es) + iblock - es.es_lblk;
1870 retval = es.es_len - (iblock - es.es_lblk);
1871 if (retval > map->m_len)
1872 retval = map->m_len;
1873 map->m_len = retval;
1874 if (ext4_es_is_written(&es))
1875 map->m_flags |= EXT4_MAP_MAPPED;
1876 else if (ext4_es_is_unwritten(&es))
1877 map->m_flags |= EXT4_MAP_UNWRITTEN;
1878 else
1879 BUG_ON(1);
1880
1881#ifdef ES_AGGRESSIVE_TEST
1882 ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);
1883#endif
1884 return retval;
1885 }
1886
1887 /*
1888 * Try to see if we can get the block without requesting a new
1889 * file system block.
1890 */
1891 down_read(&EXT4_I(inode)->i_data_sem);
1892 if (ext4_has_inline_data(inode))
1893 retval = 0;
1894 else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
1895 retval = ext4_ext_map_blocks(NULL, inode, map, 0);
1896 else
1897 retval = ext4_ind_map_blocks(NULL, inode, map, 0);
1898
1899add_delayed:
1900 if (retval == 0) {
1901 int ret;
1902
1903 /*
1904 * XXX: __block_prepare_write() unmaps passed block,
1905 * is it OK?
1906 */
1907
1908 ret = ext4_insert_delayed_block(inode, map->m_lblk);
1909 if (ret != 0) {
1910 retval = ret;
1911 goto out_unlock;
1912 }
1913
1914 map_bh(bh, inode->i_sb, invalid_block);
1915 set_buffer_new(bh);
1916 set_buffer_delay(bh);
1917 } else if (retval > 0) {
1918 int ret;
1919 unsigned int status;
1920
1921 if (unlikely(retval != map->m_len)) {
1922 ext4_warning(inode->i_sb,
1923 "ES len assertion failed for inode "
1924 "%lu: retval %d != map->m_len %d",
1925 inode->i_ino, retval, map->m_len);
1926 WARN_ON(1);
1927 }
1928
1929 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
1930 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
1931 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
1932 map->m_pblk, status);
1933 if (ret != 0)
1934 retval = ret;
1935 }
1936
1937out_unlock:
1938 up_read((&EXT4_I(inode)->i_data_sem));
1939
1940 return retval;
1941}
1942
1943/*
1944 * This is a special get_block_t callback which is used by
1945 * ext4_da_write_begin(). It will either return mapped block or
1946 * reserve space for a single block.
1947 *
1948 * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.
1949 * We also have b_blocknr = -1 and b_bdev initialized properly
1950 *
1951 * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.
1952 * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev
1953 * initialized properly.
1954 */
1955int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
1956 struct buffer_head *bh, int create)
1957{
1958 struct ext4_map_blocks map;
1959 int ret = 0;
1960
1961 BUG_ON(create == 0);
1962 BUG_ON(bh->b_size != inode->i_sb->s_blocksize);
1963
1964 map.m_lblk = iblock;
1965 map.m_len = 1;
1966
1967 /*
1968 * first, we need to know whether the block is allocated already
1969 * preallocated blocks are unmapped but should treated
1970 * the same as allocated blocks.
1971 */
1972 ret = ext4_da_map_blocks(inode, iblock, &map, bh);
1973 if (ret <= 0)
1974 return ret;
1975
1976 map_bh(bh, inode->i_sb, map.m_pblk);
1977 ext4_update_bh_state(bh, map.m_flags);
1978
1979 if (buffer_unwritten(bh)) {
1980 /* A delayed write to unwritten bh should be marked
1981 * new and mapped. Mapped ensures that we don't do
1982 * get_block multiple times when we write to the same
1983 * offset and new ensures that we do proper zero out
1984 * for partial write.
1985 */
1986 set_buffer_new(bh);
1987 set_buffer_mapped(bh);
1988 }
1989 return 0;
1990}
1991
1992static int bget_one(handle_t *handle, struct buffer_head *bh)
1993{
1994 get_bh(bh);
1995 return 0;
1996}
1997
1998static int bput_one(handle_t *handle, struct buffer_head *bh)
1999{
2000 put_bh(bh);
2001 return 0;
2002}
2003
2004static int __ext4_journalled_writepage(struct page *page,
2005 unsigned int len)
2006{
2007 struct address_space *mapping = page->mapping;
2008 struct inode *inode = mapping->host;
2009 struct buffer_head *page_bufs = NULL;
2010 handle_t *handle = NULL;
2011 int ret = 0, err = 0;
2012 int inline_data = ext4_has_inline_data(inode);
2013 struct buffer_head *inode_bh = NULL;
2014
2015 ClearPageChecked(page);
2016
2017 if (inline_data) {
2018 BUG_ON(page->index != 0);
2019 BUG_ON(len > ext4_get_max_inline_size(inode));
2020 inode_bh = ext4_journalled_write_inline_data(inode, len, page);
2021 if (inode_bh == NULL)
2022 goto out;
2023 } else {
2024 page_bufs = page_buffers(page);
2025 if (!page_bufs) {
2026 BUG();
2027 goto out;
2028 }
2029 ext4_walk_page_buffers(handle, page_bufs, 0, len,
2030 NULL, bget_one);
2031 }
2032 /*
2033 * We need to release the page lock before we start the
2034 * journal, so grab a reference so the page won't disappear
2035 * out from under us.
2036 */
2037 get_page(page);
2038 unlock_page(page);
2039
2040 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
2041 ext4_writepage_trans_blocks(inode));
2042 if (IS_ERR(handle)) {
2043 ret = PTR_ERR(handle);
2044 put_page(page);
2045 goto out_no_pagelock;
2046 }
2047 BUG_ON(!ext4_handle_valid(handle));
2048
2049 lock_page(page);
2050 put_page(page);
2051 if (page->mapping != mapping) {
2052 /* The page got truncated from under us */
2053 ext4_journal_stop(handle);
2054 ret = 0;
2055 goto out;
2056 }
2057
2058 if (inline_data) {
2059 ret = ext4_mark_inode_dirty(handle, inode);
2060 } else {
2061 ret = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL,
2062 do_journal_get_write_access);
2063
2064 err = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL,
2065 write_end_fn);
2066 }
2067 if (ret == 0)
2068 ret = err;
2069 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
2070 err = ext4_journal_stop(handle);
2071 if (!ret)
2072 ret = err;
2073
2074 if (!ext4_has_inline_data(inode))
2075 ext4_walk_page_buffers(NULL, page_bufs, 0, len,
2076 NULL, bput_one);
2077 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
2078out:
2079 unlock_page(page);
2080out_no_pagelock:
2081 brelse(inode_bh);
2082 return ret;
2083}
2084
2085/*
2086 * Note that we don't need to start a transaction unless we're journaling data
2087 * because we should have holes filled from ext4_page_mkwrite(). We even don't
2088 * need to file the inode to the transaction's list in ordered mode because if
2089 * we are writing back data added by write(), the inode is already there and if
2090 * we are writing back data modified via mmap(), no one guarantees in which
2091 * transaction the data will hit the disk. In case we are journaling data, we
2092 * cannot start transaction directly because transaction start ranks above page
2093 * lock so we have to do some magic.
2094 *
2095 * This function can get called via...
2096 * - ext4_writepages after taking page lock (have journal handle)
2097 * - journal_submit_inode_data_buffers (no journal handle)
2098 * - shrink_page_list via the kswapd/direct reclaim (no journal handle)
2099 * - grab_page_cache when doing write_begin (have journal handle)
2100 *
2101 * We don't do any block allocation in this function. If we have page with
2102 * multiple blocks we need to write those buffer_heads that are mapped. This
2103 * is important for mmaped based write. So if we do with blocksize 1K
2104 * truncate(f, 1024);
2105 * a = mmap(f, 0, 4096);
2106 * a[0] = 'a';
2107 * truncate(f, 4096);
2108 * we have in the page first buffer_head mapped via page_mkwrite call back
2109 * but other buffer_heads would be unmapped but dirty (dirty done via the
2110 * do_wp_page). So writepage should write the first block. If we modify
2111 * the mmap area beyond 1024 we will again get a page_fault and the
2112 * page_mkwrite callback will do the block allocation and mark the
2113 * buffer_heads mapped.
2114 *
2115 * We redirty the page if we have any buffer_heads that is either delay or
2116 * unwritten in the page.
2117 *
2118 * We can get recursively called as show below.
2119 *
2120 * ext4_writepage() -> kmalloc() -> __alloc_pages() -> page_launder() ->
2121 * ext4_writepage()
2122 *
2123 * But since we don't do any block allocation we should not deadlock.
2124 * Page also have the dirty flag cleared so we don't get recurive page_lock.
2125 */
2126static int ext4_writepage(struct page *page,
2127 struct writeback_control *wbc)
2128{
2129 int ret = 0;
2130 loff_t size;
2131 unsigned int len;
2132 struct buffer_head *page_bufs = NULL;
2133 struct inode *inode = page->mapping->host;
2134 struct ext4_io_submit io_submit;
2135 bool keep_towrite = false;
2136
2137 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) {
2138 ext4_invalidatepage(page, 0, PAGE_SIZE);
2139 unlock_page(page);
2140 return -EIO;
2141 }
2142
2143 trace_ext4_writepage(page);
2144 size = i_size_read(inode);
2145 if (page->index == size >> PAGE_SHIFT)
2146 len = size & ~PAGE_MASK;
2147 else
2148 len = PAGE_SIZE;
2149
2150 page_bufs = page_buffers(page);
2151 /*
2152 * We cannot do block allocation or other extent handling in this
2153 * function. If there are buffers needing that, we have to redirty
2154 * the page. But we may reach here when we do a journal commit via
2155 * journal_submit_inode_data_buffers() and in that case we must write
2156 * allocated buffers to achieve data=ordered mode guarantees.
2157 *
2158 * Also, if there is only one buffer per page (the fs block
2159 * size == the page size), if one buffer needs block
2160 * allocation or needs to modify the extent tree to clear the
2161 * unwritten flag, we know that the page can't be written at
2162 * all, so we might as well refuse the write immediately.
2163 * Unfortunately if the block size != page size, we can't as
2164 * easily detect this case using ext4_walk_page_buffers(), but
2165 * for the extremely common case, this is an optimization that
2166 * skips a useless round trip through ext4_bio_write_page().
2167 */
2168 if (ext4_walk_page_buffers(NULL, page_bufs, 0, len, NULL,
2169 ext4_bh_delay_or_unwritten)) {
2170 redirty_page_for_writepage(wbc, page);
2171 if ((current->flags & PF_MEMALLOC) ||
2172 (inode->i_sb->s_blocksize == PAGE_SIZE)) {
2173 /*
2174 * For memory cleaning there's no point in writing only
2175 * some buffers. So just bail out. Warn if we came here
2176 * from direct reclaim.
2177 */
2178 WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD))
2179 == PF_MEMALLOC);
2180 unlock_page(page);
2181 return 0;
2182 }
2183 keep_towrite = true;
2184 }
2185
2186 if (PageChecked(page) && ext4_should_journal_data(inode))
2187 /*
2188 * It's mmapped pagecache. Add buffers and journal it. There
2189 * doesn't seem much point in redirtying the page here.
2190 */
2191 return __ext4_journalled_writepage(page, len);
2192
2193 ext4_io_submit_init(&io_submit, wbc);
2194 io_submit.io_end = ext4_init_io_end(inode, GFP_NOFS);
2195 if (!io_submit.io_end) {
2196 redirty_page_for_writepage(wbc, page);
2197 unlock_page(page);
2198 return -ENOMEM;
2199 }
2200 ret = ext4_bio_write_page(&io_submit, page, len, wbc, keep_towrite);
2201 ext4_io_submit(&io_submit);
2202 /* Drop io_end reference we got from init */
2203 ext4_put_io_end_defer(io_submit.io_end);
2204 return ret;
2205}
2206
2207static int mpage_submit_page(struct mpage_da_data *mpd, struct page *page)
2208{
2209 int len;
2210 loff_t size;
2211 int err;
2212
2213 BUG_ON(page->index != mpd->first_page);
2214 clear_page_dirty_for_io(page);
2215 /*
2216 * We have to be very careful here! Nothing protects writeback path
2217 * against i_size changes and the page can be writeably mapped into
2218 * page tables. So an application can be growing i_size and writing
2219 * data through mmap while writeback runs. clear_page_dirty_for_io()
2220 * write-protects our page in page tables and the page cannot get
2221 * written to again until we release page lock. So only after
2222 * clear_page_dirty_for_io() we are safe to sample i_size for
2223 * ext4_bio_write_page() to zero-out tail of the written page. We rely
2224 * on the barrier provided by TestClearPageDirty in
2225 * clear_page_dirty_for_io() to make sure i_size is really sampled only
2226 * after page tables are updated.
2227 */
2228 size = i_size_read(mpd->inode);
2229 if (page->index == size >> PAGE_SHIFT)
2230 len = size & ~PAGE_MASK;
2231 else
2232 len = PAGE_SIZE;
2233 err = ext4_bio_write_page(&mpd->io_submit, page, len, mpd->wbc, false);
2234 if (!err)
2235 mpd->wbc->nr_to_write--;
2236 mpd->first_page++;
2237
2238 return err;
2239}
2240
2241#define BH_FLAGS ((1 << BH_Unwritten) | (1 << BH_Delay))
2242
2243/*
2244 * mballoc gives us at most this number of blocks...
2245 * XXX: That seems to be only a limitation of ext4_mb_normalize_request().
2246 * The rest of mballoc seems to handle chunks up to full group size.
2247 */
2248#define MAX_WRITEPAGES_EXTENT_LEN 2048
2249
2250/*
2251 * mpage_add_bh_to_extent - try to add bh to extent of blocks to map
2252 *
2253 * @mpd - extent of blocks
2254 * @lblk - logical number of the block in the file
2255 * @bh - buffer head we want to add to the extent
2256 *
2257 * The function is used to collect contig. blocks in the same state. If the
2258 * buffer doesn't require mapping for writeback and we haven't started the
2259 * extent of buffers to map yet, the function returns 'true' immediately - the
2260 * caller can write the buffer right away. Otherwise the function returns true
2261 * if the block has been added to the extent, false if the block couldn't be
2262 * added.
2263 */
2264static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
2265 struct buffer_head *bh)
2266{
2267 struct ext4_map_blocks *map = &mpd->map;
2268
2269 /* Buffer that doesn't need mapping for writeback? */
2270 if (!buffer_dirty(bh) || !buffer_mapped(bh) ||
2271 (!buffer_delay(bh) && !buffer_unwritten(bh))) {
2272 /* So far no extent to map => we write the buffer right away */
2273 if (map->m_len == 0)
2274 return true;
2275 return false;
2276 }
2277
2278 /* First block in the extent? */
2279 if (map->m_len == 0) {
2280 /* We cannot map unless handle is started... */
2281 if (!mpd->do_map)
2282 return false;
2283 map->m_lblk = lblk;
2284 map->m_len = 1;
2285 map->m_flags = bh->b_state & BH_FLAGS;
2286 return true;
2287 }
2288
2289 /* Don't go larger than mballoc is willing to allocate */
2290 if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)
2291 return false;
2292
2293 /* Can we merge the block to our big extent? */
2294 if (lblk == map->m_lblk + map->m_len &&
2295 (bh->b_state & BH_FLAGS) == map->m_flags) {
2296 map->m_len++;
2297 return true;
2298 }
2299 return false;
2300}
2301
2302/*
2303 * mpage_process_page_bufs - submit page buffers for IO or add them to extent
2304 *
2305 * @mpd - extent of blocks for mapping
2306 * @head - the first buffer in the page
2307 * @bh - buffer we should start processing from
2308 * @lblk - logical number of the block in the file corresponding to @bh
2309 *
2310 * Walk through page buffers from @bh upto @head (exclusive) and either submit
2311 * the page for IO if all buffers in this page were mapped and there's no
2312 * accumulated extent of buffers to map or add buffers in the page to the
2313 * extent of buffers to map. The function returns 1 if the caller can continue
2314 * by processing the next page, 0 if it should stop adding buffers to the
2315 * extent to map because we cannot extend it anymore. It can also return value
2316 * < 0 in case of error during IO submission.
2317 */
2318static int mpage_process_page_bufs(struct mpage_da_data *mpd,
2319 struct buffer_head *head,
2320 struct buffer_head *bh,
2321 ext4_lblk_t lblk)
2322{
2323 struct inode *inode = mpd->inode;
2324 int err;
2325 ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1)
2326 >> inode->i_blkbits;
2327
2328 do {
2329 BUG_ON(buffer_locked(bh));
2330
2331 if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
2332 /* Found extent to map? */
2333 if (mpd->map.m_len)
2334 return 0;
2335 /* Buffer needs mapping and handle is not started? */
2336 if (!mpd->do_map)
2337 return 0;
2338 /* Everything mapped so far and we hit EOF */
2339 break;
2340 }
2341 } while (lblk++, (bh = bh->b_this_page) != head);
2342 /* So far everything mapped? Submit the page for IO. */
2343 if (mpd->map.m_len == 0) {
2344 err = mpage_submit_page(mpd, head->b_page);
2345 if (err < 0)
2346 return err;
2347 }
2348 return lblk < blocks;
2349}
2350
2351/*
2352 * mpage_map_buffers - update buffers corresponding to changed extent and
2353 * submit fully mapped pages for IO
2354 *
2355 * @mpd - description of extent to map, on return next extent to map
2356 *
2357 * Scan buffers corresponding to changed extent (we expect corresponding pages
2358 * to be already locked) and update buffer state according to new extent state.
2359 * We map delalloc buffers to their physical location, clear unwritten bits,
2360 * and mark buffers as uninit when we perform writes to unwritten extents
2361 * and do extent conversion after IO is finished. If the last page is not fully
2362 * mapped, we update @map to the next extent in the last page that needs
2363 * mapping. Otherwise we submit the page for IO.
2364 */
2365static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
2366{
2367 struct pagevec pvec;
2368 int nr_pages, i;
2369 struct inode *inode = mpd->inode;
2370 struct buffer_head *head, *bh;
2371 int bpp_bits = PAGE_SHIFT - inode->i_blkbits;
2372 pgoff_t start, end;
2373 ext4_lblk_t lblk;
2374 sector_t pblock;
2375 int err;
2376
2377 start = mpd->map.m_lblk >> bpp_bits;
2378 end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits;
2379 lblk = start << bpp_bits;
2380 pblock = mpd->map.m_pblk;
2381
2382 pagevec_init(&pvec);
2383 while (start <= end) {
2384 nr_pages = pagevec_lookup_range(&pvec, inode->i_mapping,
2385 &start, end);
2386 if (nr_pages == 0)
2387 break;
2388 for (i = 0; i < nr_pages; i++) {
2389 struct page *page = pvec.pages[i];
2390
2391 bh = head = page_buffers(page);
2392 do {
2393 if (lblk < mpd->map.m_lblk)
2394 continue;
2395 if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
2396 /*
2397 * Buffer after end of mapped extent.
2398 * Find next buffer in the page to map.
2399 */
2400 mpd->map.m_len = 0;
2401 mpd->map.m_flags = 0;
2402 /*
2403 * FIXME: If dioread_nolock supports
2404 * blocksize < pagesize, we need to make
2405 * sure we add size mapped so far to
2406 * io_end->size as the following call
2407 * can submit the page for IO.
2408 */
2409 err = mpage_process_page_bufs(mpd, head,
2410 bh, lblk);
2411 pagevec_release(&pvec);
2412 if (err > 0)
2413 err = 0;
2414 return err;
2415 }
2416 if (buffer_delay(bh)) {
2417 clear_buffer_delay(bh);
2418 bh->b_blocknr = pblock++;
2419 }
2420 clear_buffer_unwritten(bh);
2421 } while (lblk++, (bh = bh->b_this_page) != head);
2422
2423 /*
2424 * FIXME: This is going to break if dioread_nolock
2425 * supports blocksize < pagesize as we will try to
2426 * convert potentially unmapped parts of inode.
2427 */
2428 mpd->io_submit.io_end->size += PAGE_SIZE;
2429 /* Page fully mapped - let IO run! */
2430 err = mpage_submit_page(mpd, page);
2431 if (err < 0) {
2432 pagevec_release(&pvec);
2433 return err;
2434 }
2435 }
2436 pagevec_release(&pvec);
2437 }
2438 /* Extent fully mapped and matches with page boundary. We are done. */
2439 mpd->map.m_len = 0;
2440 mpd->map.m_flags = 0;
2441 return 0;
2442}
2443
2444static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
2445{
2446 struct inode *inode = mpd->inode;
2447 struct ext4_map_blocks *map = &mpd->map;
2448 int get_blocks_flags;
2449 int err, dioread_nolock;
2450
2451 trace_ext4_da_write_pages_extent(inode, map);
2452 /*
2453 * Call ext4_map_blocks() to allocate any delayed allocation blocks, or
2454 * to convert an unwritten extent to be initialized (in the case
2455 * where we have written into one or more preallocated blocks). It is
2456 * possible that we're going to need more metadata blocks than
2457 * previously reserved. However we must not fail because we're in
2458 * writeback and there is nothing we can do about it so it might result
2459 * in data loss. So use reserved blocks to allocate metadata if
2460 * possible.
2461 *
2462 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE if
2463 * the blocks in question are delalloc blocks. This indicates
2464 * that the blocks and quotas has already been checked when
2465 * the data was copied into the page cache.
2466 */
2467 get_blocks_flags = EXT4_GET_BLOCKS_CREATE |
2468 EXT4_GET_BLOCKS_METADATA_NOFAIL |
2469 EXT4_GET_BLOCKS_IO_SUBMIT;
2470 dioread_nolock = ext4_should_dioread_nolock(inode);
2471 if (dioread_nolock)
2472 get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;
2473 if (map->m_flags & (1 << BH_Delay))
2474 get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
2475
2476 err = ext4_map_blocks(handle, inode, map, get_blocks_flags);
2477 if (err < 0)
2478 return err;
2479 if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {
2480 if (!mpd->io_submit.io_end->handle &&
2481 ext4_handle_valid(handle)) {
2482 mpd->io_submit.io_end->handle = handle->h_rsv_handle;
2483 handle->h_rsv_handle = NULL;
2484 }
2485 ext4_set_io_unwritten_flag(inode, mpd->io_submit.io_end);
2486 }
2487
2488 BUG_ON(map->m_len == 0);
2489 return 0;
2490}
2491
2492/*
2493 * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length
2494 * mpd->len and submit pages underlying it for IO
2495 *
2496 * @handle - handle for journal operations
2497 * @mpd - extent to map
2498 * @give_up_on_write - we set this to true iff there is a fatal error and there
2499 * is no hope of writing the data. The caller should discard
2500 * dirty pages to avoid infinite loops.
2501 *
2502 * The function maps extent starting at mpd->lblk of length mpd->len. If it is
2503 * delayed, blocks are allocated, if it is unwritten, we may need to convert
2504 * them to initialized or split the described range from larger unwritten
2505 * extent. Note that we need not map all the described range since allocation
2506 * can return less blocks or the range is covered by more unwritten extents. We
2507 * cannot map more because we are limited by reserved transaction credits. On
2508 * the other hand we always make sure that the last touched page is fully
2509 * mapped so that it can be written out (and thus forward progress is
2510 * guaranteed). After mapping we submit all mapped pages for IO.
2511 */
2512static int mpage_map_and_submit_extent(handle_t *handle,
2513 struct mpage_da_data *mpd,
2514 bool *give_up_on_write)
2515{
2516 struct inode *inode = mpd->inode;
2517 struct ext4_map_blocks *map = &mpd->map;
2518 int err;
2519 loff_t disksize;
2520 int progress = 0;
2521
2522 mpd->io_submit.io_end->offset =
2523 ((loff_t)map->m_lblk) << inode->i_blkbits;
2524 do {
2525 err = mpage_map_one_extent(handle, mpd);
2526 if (err < 0) {
2527 struct super_block *sb = inode->i_sb;
2528
2529 if (ext4_forced_shutdown(EXT4_SB(sb)) ||
2530 EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED)
2531 goto invalidate_dirty_pages;
2532 /*
2533 * Let the uper layers retry transient errors.
2534 * In the case of ENOSPC, if ext4_count_free_blocks()
2535 * is non-zero, a commit should free up blocks.
2536 */
2537 if ((err == -ENOMEM) ||
2538 (err == -ENOSPC && ext4_count_free_clusters(sb))) {
2539 if (progress)
2540 goto update_disksize;
2541 return err;
2542 }
2543 ext4_msg(sb, KERN_CRIT,
2544 "Delayed block allocation failed for "
2545 "inode %lu at logical offset %llu with"
2546 " max blocks %u with error %d",
2547 inode->i_ino,
2548 (unsigned long long)map->m_lblk,
2549 (unsigned)map->m_len, -err);
2550 ext4_msg(sb, KERN_CRIT,
2551 "This should not happen!! Data will "
2552 "be lost\n");
2553 if (err == -ENOSPC)
2554 ext4_print_free_blocks(inode);
2555 invalidate_dirty_pages:
2556 *give_up_on_write = true;
2557 return err;
2558 }
2559 progress = 1;
2560 /*
2561 * Update buffer state, submit mapped pages, and get us new
2562 * extent to map
2563 */
2564 err = mpage_map_and_submit_buffers(mpd);
2565 if (err < 0)
2566 goto update_disksize;
2567 } while (map->m_len);
2568
2569update_disksize:
2570 /*
2571 * Update on-disk size after IO is submitted. Races with
2572 * truncate are avoided by checking i_size under i_data_sem.
2573 */
2574 disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT;
2575 if (disksize > EXT4_I(inode)->i_disksize) {
2576 int err2;
2577 loff_t i_size;
2578
2579 down_write(&EXT4_I(inode)->i_data_sem);
2580 i_size = i_size_read(inode);
2581 if (disksize > i_size)
2582 disksize = i_size;
2583 if (disksize > EXT4_I(inode)->i_disksize)
2584 EXT4_I(inode)->i_disksize = disksize;
2585 up_write(&EXT4_I(inode)->i_data_sem);
2586 err2 = ext4_mark_inode_dirty(handle, inode);
2587 if (err2)
2588 ext4_error(inode->i_sb,
2589 "Failed to mark inode %lu dirty",
2590 inode->i_ino);
2591 if (!err)
2592 err = err2;
2593 }
2594 return err;
2595}
2596
2597/*
2598 * Calculate the total number of credits to reserve for one writepages
2599 * iteration. This is called from ext4_writepages(). We map an extent of
2600 * up to MAX_WRITEPAGES_EXTENT_LEN blocks and then we go on and finish mapping
2601 * the last partial page. So in total we can map MAX_WRITEPAGES_EXTENT_LEN +
2602 * bpp - 1 blocks in bpp different extents.
2603 */
2604static int ext4_da_writepages_trans_blocks(struct inode *inode)
2605{
2606 int bpp = ext4_journal_blocks_per_page(inode);
2607
2608 return ext4_meta_trans_blocks(inode,
2609 MAX_WRITEPAGES_EXTENT_LEN + bpp - 1, bpp);
2610}
2611
2612/*
2613 * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages
2614 * and underlying extent to map
2615 *
2616 * @mpd - where to look for pages
2617 *
2618 * Walk dirty pages in the mapping. If they are fully mapped, submit them for
2619 * IO immediately. When we find a page which isn't mapped we start accumulating
2620 * extent of buffers underlying these pages that needs mapping (formed by
2621 * either delayed or unwritten buffers). We also lock the pages containing
2622 * these buffers. The extent found is returned in @mpd structure (starting at
2623 * mpd->lblk with length mpd->len blocks).
2624 *
2625 * Note that this function can attach bios to one io_end structure which are
2626 * neither logically nor physically contiguous. Although it may seem as an
2627 * unnecessary complication, it is actually inevitable in blocksize < pagesize
2628 * case as we need to track IO to all buffers underlying a page in one io_end.
2629 */
2630static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
2631{
2632 struct address_space *mapping = mpd->inode->i_mapping;
2633 struct pagevec pvec;
2634 unsigned int nr_pages;
2635 long left = mpd->wbc->nr_to_write;
2636 pgoff_t index = mpd->first_page;
2637 pgoff_t end = mpd->last_page;
2638 xa_mark_t tag;
2639 int i, err = 0;
2640 int blkbits = mpd->inode->i_blkbits;
2641 ext4_lblk_t lblk;
2642 struct buffer_head *head;
2643
2644 if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages)
2645 tag = PAGECACHE_TAG_TOWRITE;
2646 else
2647 tag = PAGECACHE_TAG_DIRTY;
2648
2649 pagevec_init(&pvec);
2650 mpd->map.m_len = 0;
2651 mpd->next_page = index;
2652 while (index <= end) {
2653 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
2654 tag);
2655 if (nr_pages == 0)
2656 goto out;
2657
2658 for (i = 0; i < nr_pages; i++) {
2659 struct page *page = pvec.pages[i];
2660
2661 /*
2662 * Accumulated enough dirty pages? This doesn't apply
2663 * to WB_SYNC_ALL mode. For integrity sync we have to
2664 * keep going because someone may be concurrently
2665 * dirtying pages, and we might have synced a lot of
2666 * newly appeared dirty pages, but have not synced all
2667 * of the old dirty pages.
2668 */
2669 if (mpd->wbc->sync_mode == WB_SYNC_NONE && left <= 0)
2670 goto out;
2671
2672 /* If we can't merge this page, we are done. */
2673 if (mpd->map.m_len > 0 && mpd->next_page != page->index)
2674 goto out;
2675
2676 lock_page(page);
2677 /*
2678 * If the page is no longer dirty, or its mapping no
2679 * longer corresponds to inode we are writing (which
2680 * means it has been truncated or invalidated), or the
2681 * page is already under writeback and we are not doing
2682 * a data integrity writeback, skip the page
2683 */
2684 if (!PageDirty(page) ||
2685 (PageWriteback(page) &&
2686 (mpd->wbc->sync_mode == WB_SYNC_NONE)) ||
2687 unlikely(page->mapping != mapping)) {
2688 unlock_page(page);
2689 continue;
2690 }
2691
2692 wait_on_page_writeback(page);
2693 BUG_ON(PageWriteback(page));
2694
2695 if (mpd->map.m_len == 0)
2696 mpd->first_page = page->index;
2697 mpd->next_page = page->index + 1;
2698 /* Add all dirty buffers to mpd */
2699 lblk = ((ext4_lblk_t)page->index) <<
2700 (PAGE_SHIFT - blkbits);
2701 head = page_buffers(page);
2702 err = mpage_process_page_bufs(mpd, head, head, lblk);
2703 if (err <= 0)
2704 goto out;
2705 err = 0;
2706 left--;
2707 }
2708 pagevec_release(&pvec);
2709 cond_resched();
2710 }
2711 return 0;
2712out:
2713 pagevec_release(&pvec);
2714 return err;
2715}
2716
2717static int ext4_writepages(struct address_space *mapping,
2718 struct writeback_control *wbc)
2719{
2720 pgoff_t writeback_index = 0;
2721 long nr_to_write = wbc->nr_to_write;
2722 int range_whole = 0;
2723 int cycled = 1;
2724 handle_t *handle = NULL;
2725 struct mpage_da_data mpd;
2726 struct inode *inode = mapping->host;
2727 int needed_blocks, rsv_blocks = 0, ret = 0;
2728 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2729 bool done;
2730 struct blk_plug plug;
2731 bool give_up_on_write = false;
2732
2733 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2734 return -EIO;
2735
2736 percpu_down_read(&sbi->s_journal_flag_rwsem);
2737 trace_ext4_writepages(inode, wbc);
2738
2739 /*
2740 * No pages to write? This is mainly a kludge to avoid starting
2741 * a transaction for special inodes like journal inode on last iput()
2742 * because that could violate lock ordering on umount
2743 */
2744 if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2745 goto out_writepages;
2746
2747 if (ext4_should_journal_data(inode)) {
2748 ret = generic_writepages(mapping, wbc);
2749 goto out_writepages;
2750 }
2751
2752 /*
2753 * If the filesystem has aborted, it is read-only, so return
2754 * right away instead of dumping stack traces later on that
2755 * will obscure the real source of the problem. We test
2756 * EXT4_MF_FS_ABORTED instead of sb->s_flag's SB_RDONLY because
2757 * the latter could be true if the filesystem is mounted
2758 * read-only, and in that case, ext4_writepages should
2759 * *never* be called, so if that ever happens, we would want
2760 * the stack trace.
2761 */
2762 if (unlikely(ext4_forced_shutdown(EXT4_SB(mapping->host->i_sb)) ||
2763 sbi->s_mount_flags & EXT4_MF_FS_ABORTED)) {
2764 ret = -EROFS;
2765 goto out_writepages;
2766 }
2767
2768 if (ext4_should_dioread_nolock(inode)) {
2769 /*
2770 * We may need to convert up to one extent per block in
2771 * the page and we may dirty the inode.
2772 */
2773 rsv_blocks = 1 + ext4_chunk_trans_blocks(inode,
2774 PAGE_SIZE >> inode->i_blkbits);
2775 }
2776
2777 /*
2778 * If we have inline data and arrive here, it means that
2779 * we will soon create the block for the 1st page, so
2780 * we'd better clear the inline data here.
2781 */
2782 if (ext4_has_inline_data(inode)) {
2783 /* Just inode will be modified... */
2784 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
2785 if (IS_ERR(handle)) {
2786 ret = PTR_ERR(handle);
2787 goto out_writepages;
2788 }
2789 BUG_ON(ext4_test_inode_state(inode,
2790 EXT4_STATE_MAY_INLINE_DATA));
2791 ext4_destroy_inline_data(handle, inode);
2792 ext4_journal_stop(handle);
2793 }
2794
2795 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2796 range_whole = 1;
2797
2798 if (wbc->range_cyclic) {
2799 writeback_index = mapping->writeback_index;
2800 if (writeback_index)
2801 cycled = 0;
2802 mpd.first_page = writeback_index;
2803 mpd.last_page = -1;
2804 } else {
2805 mpd.first_page = wbc->range_start >> PAGE_SHIFT;
2806 mpd.last_page = wbc->range_end >> PAGE_SHIFT;
2807 }
2808
2809 mpd.inode = inode;
2810 mpd.wbc = wbc;
2811 ext4_io_submit_init(&mpd.io_submit, wbc);
2812retry:
2813 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2814 tag_pages_for_writeback(mapping, mpd.first_page, mpd.last_page);
2815 done = false;
2816 blk_start_plug(&plug);
2817
2818 /*
2819 * First writeback pages that don't need mapping - we can avoid
2820 * starting a transaction unnecessarily and also avoid being blocked
2821 * in the block layer on device congestion while having transaction
2822 * started.
2823 */
2824 mpd.do_map = 0;
2825 mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2826 if (!mpd.io_submit.io_end) {
2827 ret = -ENOMEM;
2828 goto unplug;
2829 }
2830 ret = mpage_prepare_extent_to_map(&mpd);
2831 /* Unlock pages we didn't use */
2832 mpage_release_unused_pages(&mpd, false);
2833 /* Submit prepared bio */
2834 ext4_io_submit(&mpd.io_submit);
2835 ext4_put_io_end_defer(mpd.io_submit.io_end);
2836 mpd.io_submit.io_end = NULL;
2837 if (ret < 0)
2838 goto unplug;
2839
2840 while (!done && mpd.first_page <= mpd.last_page) {
2841 /* For each extent of pages we use new io_end */
2842 mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2843 if (!mpd.io_submit.io_end) {
2844 ret = -ENOMEM;
2845 break;
2846 }
2847
2848 /*
2849 * We have two constraints: We find one extent to map and we
2850 * must always write out whole page (makes a difference when
2851 * blocksize < pagesize) so that we don't block on IO when we
2852 * try to write out the rest of the page. Journalled mode is
2853 * not supported by delalloc.
2854 */
2855 BUG_ON(ext4_should_journal_data(inode));
2856 needed_blocks = ext4_da_writepages_trans_blocks(inode);
2857
2858 /* start a new transaction */
2859 handle = ext4_journal_start_with_reserve(inode,
2860 EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);
2861 if (IS_ERR(handle)) {
2862 ret = PTR_ERR(handle);
2863 ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
2864 "%ld pages, ino %lu; err %d", __func__,
2865 wbc->nr_to_write, inode->i_ino, ret);
2866 /* Release allocated io_end */
2867 ext4_put_io_end(mpd.io_submit.io_end);
2868 mpd.io_submit.io_end = NULL;
2869 break;
2870 }
2871 mpd.do_map = 1;
2872
2873 trace_ext4_da_write_pages(inode, mpd.first_page, mpd.wbc);
2874 ret = mpage_prepare_extent_to_map(&mpd);
2875 if (!ret) {
2876 if (mpd.map.m_len)
2877 ret = mpage_map_and_submit_extent(handle, &mpd,
2878 &give_up_on_write);
2879 else {
2880 /*
2881 * We scanned the whole range (or exhausted
2882 * nr_to_write), submitted what was mapped and
2883 * didn't find anything needing mapping. We are
2884 * done.
2885 */
2886 done = true;
2887 }
2888 }
2889 /*
2890 * Caution: If the handle is synchronous,
2891 * ext4_journal_stop() can wait for transaction commit
2892 * to finish which may depend on writeback of pages to
2893 * complete or on page lock to be released. In that
2894 * case, we have to wait until after after we have
2895 * submitted all the IO, released page locks we hold,
2896 * and dropped io_end reference (for extent conversion
2897 * to be able to complete) before stopping the handle.
2898 */
2899 if (!ext4_handle_valid(handle) || handle->h_sync == 0) {
2900 ext4_journal_stop(handle);
2901 handle = NULL;
2902 mpd.do_map = 0;
2903 }
2904 /* Unlock pages we didn't use */
2905 mpage_release_unused_pages(&mpd, give_up_on_write);
2906 /* Submit prepared bio */
2907 ext4_io_submit(&mpd.io_submit);
2908
2909 /*
2910 * Drop our io_end reference we got from init. We have
2911 * to be careful and use deferred io_end finishing if
2912 * we are still holding the transaction as we can
2913 * release the last reference to io_end which may end
2914 * up doing unwritten extent conversion.
2915 */
2916 if (handle) {
2917 ext4_put_io_end_defer(mpd.io_submit.io_end);
2918 ext4_journal_stop(handle);
2919 } else
2920 ext4_put_io_end(mpd.io_submit.io_end);
2921 mpd.io_submit.io_end = NULL;
2922
2923 if (ret == -ENOSPC && sbi->s_journal) {
2924 /*
2925 * Commit the transaction which would
2926 * free blocks released in the transaction
2927 * and try again
2928 */
2929 jbd2_journal_force_commit_nested(sbi->s_journal);
2930 ret = 0;
2931 continue;
2932 }
2933 /* Fatal error - ENOMEM, EIO... */
2934 if (ret)
2935 break;
2936 }
2937unplug:
2938 blk_finish_plug(&plug);
2939 if (!ret && !cycled && wbc->nr_to_write > 0) {
2940 cycled = 1;
2941 mpd.last_page = writeback_index - 1;
2942 mpd.first_page = 0;
2943 goto retry;
2944 }
2945
2946 /* Update index */
2947 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2948 /*
2949 * Set the writeback_index so that range_cyclic
2950 * mode will write it back later
2951 */
2952 mapping->writeback_index = mpd.first_page;
2953
2954out_writepages:
2955 trace_ext4_writepages_result(inode, wbc, ret,
2956 nr_to_write - wbc->nr_to_write);
2957 percpu_up_read(&sbi->s_journal_flag_rwsem);
2958 return ret;
2959}
2960
2961static int ext4_dax_writepages(struct address_space *mapping,
2962 struct writeback_control *wbc)
2963{
2964 int ret;
2965 long nr_to_write = wbc->nr_to_write;
2966 struct inode *inode = mapping->host;
2967 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2968
2969 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2970 return -EIO;
2971
2972 percpu_down_read(&sbi->s_journal_flag_rwsem);
2973 trace_ext4_writepages(inode, wbc);
2974
2975 ret = dax_writeback_mapping_range(mapping, inode->i_sb->s_bdev, wbc);
2976 trace_ext4_writepages_result(inode, wbc, ret,
2977 nr_to_write - wbc->nr_to_write);
2978 percpu_up_read(&sbi->s_journal_flag_rwsem);
2979 return ret;
2980}
2981
2982static int ext4_nonda_switch(struct super_block *sb)
2983{
2984 s64 free_clusters, dirty_clusters;
2985 struct ext4_sb_info *sbi = EXT4_SB(sb);
2986
2987 /*
2988 * switch to non delalloc mode if we are running low
2989 * on free block. The free block accounting via percpu
2990 * counters can get slightly wrong with percpu_counter_batch getting
2991 * accumulated on each CPU without updating global counters
2992 * Delalloc need an accurate free block accounting. So switch
2993 * to non delalloc when we are near to error range.
2994 */
2995 free_clusters =
2996 percpu_counter_read_positive(&sbi->s_freeclusters_counter);
2997 dirty_clusters =
2998 percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);
2999 /*
3000 * Start pushing delalloc when 1/2 of free blocks are dirty.
3001 */
3002 if (dirty_clusters && (free_clusters < 2 * dirty_clusters))
3003 try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);
3004
3005 if (2 * free_clusters < 3 * dirty_clusters ||
3006 free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {
3007 /*
3008 * free block count is less than 150% of dirty blocks
3009 * or free blocks is less than watermark
3010 */
3011 return 1;
3012 }
3013 return 0;
3014}
3015
3016/* We always reserve for an inode update; the superblock could be there too */
3017static int ext4_da_write_credits(struct inode *inode, loff_t pos, unsigned len)
3018{
3019 if (likely(ext4_has_feature_large_file(inode->i_sb)))
3020 return 1;
3021
3022 if (pos + len <= 0x7fffffffULL)
3023 return 1;
3024
3025 /* We might need to update the superblock to set LARGE_FILE */
3026 return 2;
3027}
3028
3029static int ext4_da_write_begin(struct file *file, struct address_space *mapping,
3030 loff_t pos, unsigned len, unsigned flags,
3031 struct page **pagep, void **fsdata)
3032{
3033 int ret, retries = 0;
3034 struct page *page;
3035 pgoff_t index;
3036 struct inode *inode = mapping->host;
3037 handle_t *handle;
3038
3039 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
3040 return -EIO;
3041
3042 index = pos >> PAGE_SHIFT;
3043
3044 if (ext4_nonda_switch(inode->i_sb) ||
3045 S_ISLNK(inode->i_mode)) {
3046 *fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
3047 return ext4_write_begin(file, mapping, pos,
3048 len, flags, pagep, fsdata);
3049 }
3050 *fsdata = (void *)0;
3051 trace_ext4_da_write_begin(inode, pos, len, flags);
3052
3053 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
3054 ret = ext4_da_write_inline_data_begin(mapping, inode,
3055 pos, len, flags,
3056 pagep, fsdata);
3057 if (ret < 0)
3058 return ret;
3059 if (ret == 1)
3060 return 0;
3061 }
3062
3063 /*
3064 * grab_cache_page_write_begin() can take a long time if the
3065 * system is thrashing due to memory pressure, or if the page
3066 * is being written back. So grab it first before we start
3067 * the transaction handle. This also allows us to allocate
3068 * the page (if needed) without using GFP_NOFS.
3069 */
3070retry_grab:
3071 page = grab_cache_page_write_begin(mapping, index, flags);
3072 if (!page)
3073 return -ENOMEM;
3074 unlock_page(page);
3075
3076 /*
3077 * With delayed allocation, we don't log the i_disksize update
3078 * if there is delayed block allocation. But we still need
3079 * to journalling the i_disksize update if writes to the end
3080 * of file which has an already mapped buffer.
3081 */
3082retry_journal:
3083 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
3084 ext4_da_write_credits(inode, pos, len));
3085 if (IS_ERR(handle)) {
3086 put_page(page);
3087 return PTR_ERR(handle);
3088 }
3089
3090 lock_page(page);
3091 if (page->mapping != mapping) {
3092 /* The page got truncated from under us */
3093 unlock_page(page);
3094 put_page(page);
3095 ext4_journal_stop(handle);
3096 goto retry_grab;
3097 }
3098 /* In case writeback began while the page was unlocked */
3099 wait_for_stable_page(page);
3100
3101#ifdef CONFIG_FS_ENCRYPTION
3102 ret = ext4_block_write_begin(page, pos, len,
3103 ext4_da_get_block_prep);
3104#else
3105 ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep);
3106#endif
3107 if (ret < 0) {
3108 unlock_page(page);
3109 ext4_journal_stop(handle);
3110 /*
3111 * block_write_begin may have instantiated a few blocks
3112 * outside i_size. Trim these off again. Don't need
3113 * i_size_read because we hold i_mutex.
3114 */
3115 if (pos + len > inode->i_size)
3116 ext4_truncate_failed_write(inode);
3117
3118 if (ret == -ENOSPC &&
3119 ext4_should_retry_alloc(inode->i_sb, &retries))
3120 goto retry_journal;
3121
3122 put_page(page);
3123 return ret;
3124 }
3125
3126 *pagep = page;
3127 return ret;
3128}
3129
3130/*
3131 * Check if we should update i_disksize
3132 * when write to the end of file but not require block allocation
3133 */
3134static int ext4_da_should_update_i_disksize(struct page *page,
3135 unsigned long offset)
3136{
3137 struct buffer_head *bh;
3138 struct inode *inode = page->mapping->host;
3139 unsigned int idx;
3140 int i;
3141
3142 bh = page_buffers(page);
3143 idx = offset >> inode->i_blkbits;
3144
3145 for (i = 0; i < idx; i++)
3146 bh = bh->b_this_page;
3147
3148 if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
3149 return 0;
3150 return 1;
3151}
3152
3153static int ext4_da_write_end(struct file *file,
3154 struct address_space *mapping,
3155 loff_t pos, unsigned len, unsigned copied,
3156 struct page *page, void *fsdata)
3157{
3158 struct inode *inode = mapping->host;
3159 int ret = 0, ret2;
3160 handle_t *handle = ext4_journal_current_handle();
3161 loff_t new_i_size;
3162 unsigned long start, end;
3163 int write_mode = (int)(unsigned long)fsdata;
3164
3165 if (write_mode == FALL_BACK_TO_NONDELALLOC)
3166 return ext4_write_end(file, mapping, pos,
3167 len, copied, page, fsdata);
3168
3169 trace_ext4_da_write_end(inode, pos, len, copied);
3170 start = pos & (PAGE_SIZE - 1);
3171 end = start + copied - 1;
3172
3173 /*
3174 * generic_write_end() will run mark_inode_dirty() if i_size
3175 * changes. So let's piggyback the i_disksize mark_inode_dirty
3176 * into that.
3177 */
3178 new_i_size = pos + copied;
3179 if (copied && new_i_size > EXT4_I(inode)->i_disksize) {
3180 if (ext4_has_inline_data(inode) ||
3181 ext4_da_should_update_i_disksize(page, end)) {
3182 ext4_update_i_disksize(inode, new_i_size);
3183 /* We need to mark inode dirty even if
3184 * new_i_size is less that inode->i_size
3185 * bu greater than i_disksize.(hint delalloc)
3186 */
3187 ext4_mark_inode_dirty(handle, inode);
3188 }
3189 }
3190
3191 if (write_mode != CONVERT_INLINE_DATA &&
3192 ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) &&
3193 ext4_has_inline_data(inode))
3194 ret2 = ext4_da_write_inline_data_end(inode, pos, len, copied,
3195 page);
3196 else
3197 ret2 = generic_write_end(file, mapping, pos, len, copied,
3198 page, fsdata);
3199
3200 copied = ret2;
3201 if (ret2 < 0)
3202 ret = ret2;
3203 ret2 = ext4_journal_stop(handle);
3204 if (!ret)
3205 ret = ret2;
3206
3207 return ret ? ret : copied;
3208}
3209
3210static void ext4_da_invalidatepage(struct page *page, unsigned int offset,
3211 unsigned int length)
3212{
3213 /*
3214 * Drop reserved blocks
3215 */
3216 BUG_ON(!PageLocked(page));
3217 if (!page_has_buffers(page))
3218 goto out;
3219
3220 ext4_da_page_release_reservation(page, offset, length);
3221
3222out:
3223 ext4_invalidatepage(page, offset, length);
3224
3225 return;
3226}
3227
3228/*
3229 * Force all delayed allocation blocks to be allocated for a given inode.
3230 */
3231int ext4_alloc_da_blocks(struct inode *inode)
3232{
3233 trace_ext4_alloc_da_blocks(inode);
3234
3235 if (!EXT4_I(inode)->i_reserved_data_blocks)
3236 return 0;
3237
3238 /*
3239 * We do something simple for now. The filemap_flush() will
3240 * also start triggering a write of the data blocks, which is
3241 * not strictly speaking necessary (and for users of
3242 * laptop_mode, not even desirable). However, to do otherwise
3243 * would require replicating code paths in:
3244 *
3245 * ext4_writepages() ->
3246 * write_cache_pages() ---> (via passed in callback function)
3247 * __mpage_da_writepage() -->
3248 * mpage_add_bh_to_extent()
3249 * mpage_da_map_blocks()
3250 *
3251 * The problem is that write_cache_pages(), located in
3252 * mm/page-writeback.c, marks pages clean in preparation for
3253 * doing I/O, which is not desirable if we're not planning on
3254 * doing I/O at all.
3255 *
3256 * We could call write_cache_pages(), and then redirty all of
3257 * the pages by calling redirty_page_for_writepage() but that
3258 * would be ugly in the extreme. So instead we would need to
3259 * replicate parts of the code in the above functions,
3260 * simplifying them because we wouldn't actually intend to
3261 * write out the pages, but rather only collect contiguous
3262 * logical block extents, call the multi-block allocator, and
3263 * then update the buffer heads with the block allocations.
3264 *
3265 * For now, though, we'll cheat by calling filemap_flush(),
3266 * which will map the blocks, and start the I/O, but not
3267 * actually wait for the I/O to complete.
3268 */
3269 return filemap_flush(inode->i_mapping);
3270}
3271
3272/*
3273 * bmap() is special. It gets used by applications such as lilo and by
3274 * the swapper to find the on-disk block of a specific piece of data.
3275 *
3276 * Naturally, this is dangerous if the block concerned is still in the
3277 * journal. If somebody makes a swapfile on an ext4 data-journaling
3278 * filesystem and enables swap, then they may get a nasty shock when the
3279 * data getting swapped to that swapfile suddenly gets overwritten by
3280 * the original zero's written out previously to the journal and
3281 * awaiting writeback in the kernel's buffer cache.
3282 *
3283 * So, if we see any bmap calls here on a modified, data-journaled file,
3284 * take extra steps to flush any blocks which might be in the cache.
3285 */
3286static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
3287{
3288 struct inode *inode = mapping->host;
3289 journal_t *journal;
3290 int err;
3291
3292 /*
3293 * We can get here for an inline file via the FIBMAP ioctl
3294 */
3295 if (ext4_has_inline_data(inode))
3296 return 0;
3297
3298 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
3299 test_opt(inode->i_sb, DELALLOC)) {
3300 /*
3301 * With delalloc we want to sync the file
3302 * so that we can make sure we allocate
3303 * blocks for file
3304 */
3305 filemap_write_and_wait(mapping);
3306 }
3307
3308 if (EXT4_JOURNAL(inode) &&
3309 ext4_test_inode_state(inode, EXT4_STATE_JDATA)) {
3310 /*
3311 * This is a REALLY heavyweight approach, but the use of
3312 * bmap on dirty files is expected to be extremely rare:
3313 * only if we run lilo or swapon on a freshly made file
3314 * do we expect this to happen.
3315 *
3316 * (bmap requires CAP_SYS_RAWIO so this does not
3317 * represent an unprivileged user DOS attack --- we'd be
3318 * in trouble if mortal users could trigger this path at
3319 * will.)
3320 *
3321 * NB. EXT4_STATE_JDATA is not set on files other than
3322 * regular files. If somebody wants to bmap a directory
3323 * or symlink and gets confused because the buffer
3324 * hasn't yet been flushed to disk, they deserve
3325 * everything they get.
3326 */
3327
3328 ext4_clear_inode_state(inode, EXT4_STATE_JDATA);
3329 journal = EXT4_JOURNAL(inode);
3330 jbd2_journal_lock_updates(journal);
3331 err = jbd2_journal_flush(journal);
3332 jbd2_journal_unlock_updates(journal);
3333
3334 if (err)
3335 return 0;
3336 }
3337
3338 return generic_block_bmap(mapping, block, ext4_get_block);
3339}
3340
3341static int ext4_readpage(struct file *file, struct page *page)
3342{
3343 int ret = -EAGAIN;
3344 struct inode *inode = page->mapping->host;
3345
3346 trace_ext4_readpage(page);
3347
3348 if (ext4_has_inline_data(inode))
3349 ret = ext4_readpage_inline(inode, page);
3350
3351 if (ret == -EAGAIN)
3352 return ext4_mpage_readpages(page->mapping, NULL, page, 1,
3353 false);
3354
3355 return ret;
3356}
3357
3358static int
3359ext4_readpages(struct file *file, struct address_space *mapping,
3360 struct list_head *pages, unsigned nr_pages)
3361{
3362 struct inode *inode = mapping->host;
3363
3364 /* If the file has inline data, no need to do readpages. */
3365 if (ext4_has_inline_data(inode))
3366 return 0;
3367
3368 return ext4_mpage_readpages(mapping, pages, NULL, nr_pages, true);
3369}
3370
3371static void ext4_invalidatepage(struct page *page, unsigned int offset,
3372 unsigned int length)
3373{
3374 trace_ext4_invalidatepage(page, offset, length);
3375
3376 /* No journalling happens on data buffers when this function is used */
3377 WARN_ON(page_has_buffers(page) && buffer_jbd(page_buffers(page)));
3378
3379 block_invalidatepage(page, offset, length);
3380}
3381
3382static int __ext4_journalled_invalidatepage(struct page *page,
3383 unsigned int offset,
3384 unsigned int length)
3385{
3386 journal_t *journal = EXT4_JOURNAL(page->mapping->host);
3387
3388 trace_ext4_journalled_invalidatepage(page, offset, length);
3389
3390 /*
3391 * If it's a full truncate we just forget about the pending dirtying
3392 */
3393 if (offset == 0 && length == PAGE_SIZE)
3394 ClearPageChecked(page);
3395
3396 return jbd2_journal_invalidatepage(journal, page, offset, length);
3397}
3398
3399/* Wrapper for aops... */
3400static void ext4_journalled_invalidatepage(struct page *page,
3401 unsigned int offset,
3402 unsigned int length)
3403{
3404 WARN_ON(__ext4_journalled_invalidatepage(page, offset, length) < 0);
3405}
3406
3407static int ext4_releasepage(struct page *page, gfp_t wait)
3408{
3409 journal_t *journal = EXT4_JOURNAL(page->mapping->host);
3410
3411 trace_ext4_releasepage(page);
3412
3413 /* Page has dirty journalled data -> cannot release */
3414 if (PageChecked(page))
3415 return 0;
3416 if (journal)
3417 return jbd2_journal_try_to_free_buffers(journal, page, wait);
3418 else
3419 return try_to_free_buffers(page);
3420}
3421
3422static bool ext4_inode_datasync_dirty(struct inode *inode)
3423{
3424 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
3425
3426 if (journal)
3427 return !jbd2_transaction_committed(journal,
3428 EXT4_I(inode)->i_datasync_tid);
3429 /* Any metadata buffers to write? */
3430 if (!list_empty(&inode->i_mapping->private_list))
3431 return true;
3432 return inode->i_state & I_DIRTY_DATASYNC;
3433}
3434
3435static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
3436 unsigned flags, struct iomap *iomap)
3437{
3438 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
3439 unsigned int blkbits = inode->i_blkbits;
3440 unsigned long first_block, last_block;
3441 struct ext4_map_blocks map;
3442 bool delalloc = false;
3443 int ret;
3444
3445 if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3446 return -EINVAL;
3447 first_block = offset >> blkbits;
3448 last_block = min_t(loff_t, (offset + length - 1) >> blkbits,
3449 EXT4_MAX_LOGICAL_BLOCK);
3450
3451 if (flags & IOMAP_REPORT) {
3452 if (ext4_has_inline_data(inode)) {
3453 ret = ext4_inline_data_iomap(inode, iomap);
3454 if (ret != -EAGAIN) {
3455 if (ret == 0 && offset >= iomap->length)
3456 ret = -ENOENT;
3457 return ret;
3458 }
3459 }
3460 } else {
3461 if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
3462 return -ERANGE;
3463 }
3464
3465 map.m_lblk = first_block;
3466 map.m_len = last_block - first_block + 1;
3467
3468 if (flags & IOMAP_REPORT) {
3469 ret = ext4_map_blocks(NULL, inode, &map, 0);
3470 if (ret < 0)
3471 return ret;
3472
3473 if (ret == 0) {
3474 ext4_lblk_t end = map.m_lblk + map.m_len - 1;
3475 struct extent_status es;
3476
3477 ext4_es_find_extent_range(inode, &ext4_es_is_delayed,
3478 map.m_lblk, end, &es);
3479
3480 if (!es.es_len || es.es_lblk > end) {
3481 /* entire range is a hole */
3482 } else if (es.es_lblk > map.m_lblk) {
3483 /* range starts with a hole */
3484 map.m_len = es.es_lblk - map.m_lblk;
3485 } else {
3486 ext4_lblk_t offs = 0;
3487
3488 if (es.es_lblk < map.m_lblk)
3489 offs = map.m_lblk - es.es_lblk;
3490 map.m_lblk = es.es_lblk + offs;
3491 map.m_len = es.es_len - offs;
3492 delalloc = true;
3493 }
3494 }
3495 } else if (flags & IOMAP_WRITE) {
3496 int dio_credits;
3497 handle_t *handle;
3498 int retries = 0;
3499
3500 /* Trim mapping request to maximum we can map at once for DIO */
3501 if (map.m_len > DIO_MAX_BLOCKS)
3502 map.m_len = DIO_MAX_BLOCKS;
3503 dio_credits = ext4_chunk_trans_blocks(inode, map.m_len);
3504retry:
3505 /*
3506 * Either we allocate blocks and then we don't get unwritten
3507 * extent so we have reserved enough credits, or the blocks
3508 * are already allocated and unwritten and in that case
3509 * extent conversion fits in the credits as well.
3510 */
3511 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
3512 dio_credits);
3513 if (IS_ERR(handle))
3514 return PTR_ERR(handle);
3515
3516 ret = ext4_map_blocks(handle, inode, &map,
3517 EXT4_GET_BLOCKS_CREATE_ZERO);
3518 if (ret < 0) {
3519 ext4_journal_stop(handle);
3520 if (ret == -ENOSPC &&
3521 ext4_should_retry_alloc(inode->i_sb, &retries))
3522 goto retry;
3523 return ret;
3524 }
3525
3526 /*
3527 * If we added blocks beyond i_size, we need to make sure they
3528 * will get truncated if we crash before updating i_size in
3529 * ext4_iomap_end(). For faults we don't need to do that (and
3530 * even cannot because for orphan list operations inode_lock is
3531 * required) - if we happen to instantiate block beyond i_size,
3532 * it is because we race with truncate which has already added
3533 * the inode to the orphan list.
3534 */
3535 if (!(flags & IOMAP_FAULT) && first_block + map.m_len >
3536 (i_size_read(inode) + (1 << blkbits) - 1) >> blkbits) {
3537 int err;
3538
3539 err = ext4_orphan_add(handle, inode);
3540 if (err < 0) {
3541 ext4_journal_stop(handle);
3542 return err;
3543 }
3544 }
3545 ext4_journal_stop(handle);
3546 } else {
3547 ret = ext4_map_blocks(NULL, inode, &map, 0);
3548 if (ret < 0)
3549 return ret;
3550 }
3551
3552 iomap->flags = 0;
3553 if (ext4_inode_datasync_dirty(inode))
3554 iomap->flags |= IOMAP_F_DIRTY;
3555 iomap->bdev = inode->i_sb->s_bdev;
3556 iomap->dax_dev = sbi->s_daxdev;
3557 iomap->offset = (u64)first_block << blkbits;
3558 iomap->length = (u64)map.m_len << blkbits;
3559
3560 if (ret == 0) {
3561 iomap->type = delalloc ? IOMAP_DELALLOC : IOMAP_HOLE;
3562 iomap->addr = IOMAP_NULL_ADDR;
3563 } else {
3564 if (map.m_flags & EXT4_MAP_MAPPED) {
3565 iomap->type = IOMAP_MAPPED;
3566 } else if (map.m_flags & EXT4_MAP_UNWRITTEN) {
3567 iomap->type = IOMAP_UNWRITTEN;
3568 } else {
3569 WARN_ON_ONCE(1);
3570 return -EIO;
3571 }
3572 iomap->addr = (u64)map.m_pblk << blkbits;
3573 }
3574
3575 if (map.m_flags & EXT4_MAP_NEW)
3576 iomap->flags |= IOMAP_F_NEW;
3577
3578 return 0;
3579}
3580
3581static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length,
3582 ssize_t written, unsigned flags, struct iomap *iomap)
3583{
3584 int ret = 0;
3585 handle_t *handle;
3586 int blkbits = inode->i_blkbits;
3587 bool truncate = false;
3588
3589 if (!(flags & IOMAP_WRITE) || (flags & IOMAP_FAULT))
3590 return 0;
3591
3592 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
3593 if (IS_ERR(handle)) {
3594 ret = PTR_ERR(handle);
3595 goto orphan_del;
3596 }
3597 if (ext4_update_inode_size(inode, offset + written))
3598 ext4_mark_inode_dirty(handle, inode);
3599 /*
3600 * We may need to truncate allocated but not written blocks beyond EOF.
3601 */
3602 if (iomap->offset + iomap->length >
3603 ALIGN(inode->i_size, 1 << blkbits)) {
3604 ext4_lblk_t written_blk, end_blk;
3605
3606 written_blk = (offset + written) >> blkbits;
3607 end_blk = (offset + length) >> blkbits;
3608 if (written_blk < end_blk && ext4_can_truncate(inode))
3609 truncate = true;
3610 }
3611 /*
3612 * Remove inode from orphan list if we were extending a inode and
3613 * everything went fine.
3614 */
3615 if (!truncate && inode->i_nlink &&
3616 !list_empty(&EXT4_I(inode)->i_orphan))
3617 ext4_orphan_del(handle, inode);
3618 ext4_journal_stop(handle);
3619 if (truncate) {
3620 ext4_truncate_failed_write(inode);
3621orphan_del:
3622 /*
3623 * If truncate failed early the inode might still be on the
3624 * orphan list; we need to make sure the inode is removed from
3625 * the orphan list in that case.
3626 */
3627 if (inode->i_nlink)
3628 ext4_orphan_del(NULL, inode);
3629 }
3630 return ret;
3631}
3632
3633const struct iomap_ops ext4_iomap_ops = {
3634 .iomap_begin = ext4_iomap_begin,
3635 .iomap_end = ext4_iomap_end,
3636};
3637
3638static int ext4_end_io_dio(struct kiocb *iocb, loff_t offset,
3639 ssize_t size, void *private)
3640{
3641 ext4_io_end_t *io_end = private;
3642
3643 /* if not async direct IO just return */
3644 if (!io_end)
3645 return 0;
3646
3647 ext_debug("ext4_end_io_dio(): io_end 0x%p "
3648 "for inode %lu, iocb 0x%p, offset %llu, size %zd\n",
3649 io_end, io_end->inode->i_ino, iocb, offset, size);
3650
3651 /*
3652 * Error during AIO DIO. We cannot convert unwritten extents as the
3653 * data was not written. Just clear the unwritten flag and drop io_end.
3654 */
3655 if (size <= 0) {
3656 ext4_clear_io_unwritten_flag(io_end);
3657 size = 0;
3658 }
3659 io_end->offset = offset;
3660 io_end->size = size;
3661 ext4_put_io_end(io_end);
3662
3663 return 0;
3664}
3665
3666/*
3667 * Handling of direct IO writes.
3668 *
3669 * For ext4 extent files, ext4 will do direct-io write even to holes,
3670 * preallocated extents, and those write extend the file, no need to
3671 * fall back to buffered IO.
3672 *
3673 * For holes, we fallocate those blocks, mark them as unwritten
3674 * If those blocks were preallocated, we mark sure they are split, but
3675 * still keep the range to write as unwritten.
3676 *
3677 * The unwritten extents will be converted to written when DIO is completed.
3678 * For async direct IO, since the IO may still pending when return, we
3679 * set up an end_io call back function, which will do the conversion
3680 * when async direct IO completed.
3681 *
3682 * If the O_DIRECT write will extend the file then add this inode to the
3683 * orphan list. So recovery will truncate it back to the original size
3684 * if the machine crashes during the write.
3685 *
3686 */
3687static ssize_t ext4_direct_IO_write(struct kiocb *iocb, struct iov_iter *iter)
3688{
3689 struct file *file = iocb->ki_filp;
3690 struct inode *inode = file->f_mapping->host;
3691 struct ext4_inode_info *ei = EXT4_I(inode);
3692 ssize_t ret;
3693 loff_t offset = iocb->ki_pos;
3694 size_t