1/*
2 * Copyright (C) 2010 Red Hat, Inc.
3 * Copyright (c) 2016-2018 Christoph Hellwig.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 */
14#include <linux/module.h>
15#include <linux/compiler.h>
16#include <linux/fs.h>
17#include <linux/iomap.h>
18#include <linux/uaccess.h>
19#include <linux/gfp.h>
20#include <linux/migrate.h>
21#include <linux/mm.h>
22#include <linux/mm_inline.h>
23#include <linux/swap.h>
24#include <linux/pagemap.h>
25#include <linux/pagevec.h>
26#include <linux/file.h>
27#include <linux/uio.h>
28#include <linux/backing-dev.h>
29#include <linux/buffer_head.h>
30#include <linux/task_io_accounting_ops.h>
31#include <linux/dax.h>
32#include <linux/sched/signal.h>
33
34#include "internal.h"
35
36/*
37 * Execute a iomap write on a segment of the mapping that spans a
38 * contiguous range of pages that have identical block mapping state.
39 *
40 * This avoids the need to map pages individually, do individual allocations
41 * for each page and most importantly avoid the need for filesystem specific
42 * locking per page. Instead, all the operations are amortised over the entire
43 * range of pages. It is assumed that the filesystems will lock whatever
44 * resources they require in the iomap_begin call, and release them in the
45 * iomap_end call.
46 */
47loff_t
48iomap_apply(struct inode *inode, loff_t pos, loff_t length, unsigned flags,
49 const struct iomap_ops *ops, void *data, iomap_actor_t actor)
50{
51 struct iomap iomap = { 0 };
52 loff_t written = 0, ret;
53
54 /*
55 * Need to map a range from start position for length bytes. This can
56 * span multiple pages - it is only guaranteed to return a range of a
57 * single type of pages (e.g. all into a hole, all mapped or all
58 * unwritten). Failure at this point has nothing to undo.
59 *
60 * If allocation is required for this range, reserve the space now so
61 * that the allocation is guaranteed to succeed later on. Once we copy
62 * the data into the page cache pages, then we cannot fail otherwise we
63 * expose transient stale data. If the reserve fails, we can safely
64 * back out at this point as there is nothing to undo.
65 */
66 ret = ops->iomap_begin(inode, pos, length, flags, &iomap);
67 if (ret)
68 return ret;
69 if (WARN_ON(iomap.offset > pos))
70 return -EIO;
71 if (WARN_ON(iomap.length == 0))
72 return -EIO;
73
74 /*
75 * Cut down the length to the one actually provided by the filesystem,
76 * as it might not be able to give us the whole size that we requested.
77 */
78 if (iomap.offset + iomap.length < pos + length)
79 length = iomap.offset + iomap.length - pos;
80
81 /*
82 * Now that we have guaranteed that the space allocation will succeed.
83 * we can do the copy-in page by page without having to worry about
84 * failures exposing transient data.
85 */
86 written = actor(inode, pos, length, data, &iomap);
87
88 /*
89 * Now the data has been copied, commit the range we've copied. This
90 * should not fail unless the filesystem has had a fatal error.
91 */
92 if (ops->iomap_end) {
93 ret = ops->iomap_end(inode, pos, length,
94 written > 0 ? written : 0,
95 flags, &iomap);
96 }
97
98 return written ? written : ret;
99}
100
101static sector_t
102iomap_sector(struct iomap *iomap, loff_t pos)
103{
104 return (iomap->addr + pos - iomap->offset) >> SECTOR_SHIFT;
105}
106
107static struct iomap_page *
108iomap_page_create(struct inode *inode, struct page *page)
109{
110 struct iomap_page *iop = to_iomap_page(page);
111
112 if (iop || i_blocksize(inode) == PAGE_SIZE)
113 return iop;
114
115 iop = kmalloc(sizeof(*iop), GFP_NOFS | __GFP_NOFAIL);
116 atomic_set(&iop->read_count, 0);
117 atomic_set(&iop->write_count, 0);
118 bitmap_zero(iop->uptodate, PAGE_SIZE / SECTOR_SIZE);
119
120 /*
121 * migrate_page_move_mapping() assumes that pages with private data have
122 * their count elevated by 1.
123 */
124 get_page(page);
125 set_page_private(page, (unsigned long)iop);
126 SetPagePrivate(page);
127 return iop;
128}
129
130static void
131iomap_page_release(struct page *page)
132{
133 struct iomap_page *iop = to_iomap_page(page);
134
135 if (!iop)
136 return;
137 WARN_ON_ONCE(atomic_read(&iop->read_count));
138 WARN_ON_ONCE(atomic_read(&iop->write_count));
139 ClearPagePrivate(page);
140 set_page_private(page, 0);
141 put_page(page);
142 kfree(iop);
143}
144
145/*
146 * Calculate the range inside the page that we actually need to read.
147 */
148static void
149iomap_adjust_read_range(struct inode *inode, struct iomap_page *iop,
150 loff_t *pos, loff_t length, unsigned *offp, unsigned *lenp)
151{
152 loff_t orig_pos = *pos;
153 loff_t isize = i_size_read(inode);
154 unsigned block_bits = inode->i_blkbits;
155 unsigned block_size = (1 << block_bits);
156 unsigned poff = offset_in_page(*pos);
157 unsigned plen = min_t(loff_t, PAGE_SIZE - poff, length);
158 unsigned first = poff >> block_bits;
159 unsigned last = (poff + plen - 1) >> block_bits;
160
161 /*
162 * If the block size is smaller than the page size we need to check the
163 * per-block uptodate status and adjust the offset and length if needed
164 * to avoid reading in already uptodate ranges.
165 */
166 if (iop) {
167 unsigned int i;
168
169 /* move forward for each leading block marked uptodate */
170 for (i = first; i <= last; i++) {
171 if (!test_bit(i, iop->uptodate))
172 break;
173 *pos += block_size;
174 poff += block_size;
175 plen -= block_size;
176 first++;
177 }
178
179 /* truncate len if we find any trailing uptodate block(s) */
180 for ( ; i <= last; i++) {
181 if (test_bit(i, iop->uptodate)) {
182 plen -= (last - i + 1) * block_size;
183 last = i - 1;
184 break;
185 }
186 }
187 }
188
189 /*
190 * If the extent spans the block that contains the i_size we need to
191 * handle both halves separately so that we properly zero data in the
192 * page cache for blocks that are entirely outside of i_size.
193 */
194 if (orig_pos <= isize && orig_pos + length > isize) {
195 unsigned end = offset_in_page(isize - 1) >> block_bits;
196
197 if (first <= end && last > end)
198 plen -= (last - end) * block_size;
199 }
200
201 *offp = poff;
202 *lenp = plen;
203}
204
205static void
206iomap_set_range_uptodate(struct page *page, unsigned off, unsigned len)
207{
208 struct iomap_page *iop = to_iomap_page(page);
209 struct inode *inode = page->mapping->host;
210 unsigned first = off >> inode->i_blkbits;
211 unsigned last = (off + len - 1) >> inode->i_blkbits;
212 unsigned int i;
213 bool uptodate = true;
214
215 if (iop) {
216 for (i = 0; i < PAGE_SIZE / i_blocksize(inode); i++) {
217 if (i >= first && i <= last)
218 set_bit(i, iop->uptodate);
219 else if (!test_bit(i, iop->uptodate))
220 uptodate = false;
221 }
222 }
223
224 if (uptodate && !PageError(page))
225 SetPageUptodate(page);
226}
227
228static void
229iomap_read_finish(struct iomap_page *iop, struct page *page)
230{
231 if (!iop || atomic_dec_and_test(&iop->read_count))
232 unlock_page(page);
233}
234
235static void
236iomap_read_page_end_io(struct bio_vec *bvec, int error)
237{
238 struct page *page = bvec->bv_page;
239 struct iomap_page *iop = to_iomap_page(page);
240
241 if (unlikely(error)) {
242 ClearPageUptodate(page);
243 SetPageError(page);
244 } else {
245 iomap_set_range_uptodate(page, bvec->bv_offset, bvec->bv_len);
246 }
247
248 iomap_read_finish(iop, page);
249}
250
251static void
252iomap_read_inline_data(struct inode *inode, struct page *page,
253 struct iomap *iomap)
254{
255 size_t size = i_size_read(inode);
256 void *addr;
257
258 if (PageUptodate(page))
259 return;
260
261 BUG_ON(page->index);
262 BUG_ON(size > PAGE_SIZE - offset_in_page(iomap->inline_data));
263
264 addr = kmap_atomic(page);
265 memcpy(addr, iomap->inline_data, size);
266 memset(addr + size, 0, PAGE_SIZE - size);
267 kunmap_atomic(addr);
268 SetPageUptodate(page);
269}
270
271static void
272iomap_read_end_io(struct bio *bio)
273{
274 int error = blk_status_to_errno(bio->bi_status);
275 struct bio_vec *bvec;
276 int i;
277 struct bvec_iter_all iter_all;
278
279 bio_for_each_segment_all(bvec, bio, i, iter_all)
280 iomap_read_page_end_io(bvec, error);
281 bio_put(bio);
282}
283
284struct iomap_readpage_ctx {
285 struct page *cur_page;
286 bool cur_page_in_bio;
287 bool is_readahead;
288 struct bio *bio;
289 struct list_head *pages;
290};
291
292static loff_t
293iomap_readpage_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
294 struct iomap *iomap)
295{
296 struct iomap_readpage_ctx *ctx = data;
297 struct page *page = ctx->cur_page;
298 struct iomap_page *iop = iomap_page_create(inode, page);
299 bool is_contig = false;
300 loff_t orig_pos = pos;
301 unsigned poff, plen;
302 sector_t sector;
303
304 if (iomap->type == IOMAP_INLINE) {
305 WARN_ON_ONCE(pos);
306 iomap_read_inline_data(inode, page, iomap);
307 return PAGE_SIZE;
308 }
309
310 /* zero post-eof blocks as the page may be mapped */
311 iomap_adjust_read_range(inode, iop, &pos, length, &poff, &plen);
312 if (plen == 0)
313 goto done;
314
315 if (iomap->type != IOMAP_MAPPED || pos >= i_size_read(inode)) {
316 zero_user(page, poff, plen);
317 iomap_set_range_uptodate(page, poff, plen);
318 goto done;
319 }
320
321 ctx->cur_page_in_bio = true;
322
323 /*
324 * Try to merge into a previous segment if we can.
325 */
326 sector = iomap_sector(iomap, pos);
327 if (ctx->bio && bio_end_sector(ctx->bio) == sector) {
328 if (__bio_try_merge_page(ctx->bio, page, plen, poff, true))
329 goto done;
330 is_contig = true;
331 }
332
333 /*
334 * If we start a new segment we need to increase the read count, and we
335 * need to do so before submitting any previous full bio to make sure
336 * that we don't prematurely unlock the page.
337 */
338 if (iop)
339 atomic_inc(&iop->read_count);
340
341 if (!ctx->bio || !is_contig || bio_full(ctx->bio)) {
342 gfp_t gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL);
343 int nr_vecs = (length + PAGE_SIZE - 1) >> PAGE_SHIFT;
344
345 if (ctx->bio)
346 submit_bio(ctx->bio);
347
348 if (ctx->is_readahead) /* same as readahead_gfp_mask */
349 gfp |= __GFP_NORETRY | __GFP_NOWARN;
350 ctx->bio = bio_alloc(gfp, min(BIO_MAX_PAGES, nr_vecs));
351 ctx->bio->bi_opf = REQ_OP_READ;
352 if (ctx->is_readahead)
353 ctx->bio->bi_opf |= REQ_RAHEAD;
354 ctx->bio->bi_iter.bi_sector = sector;
355 bio_set_dev(ctx->bio, iomap->bdev);
356 ctx->bio->bi_end_io = iomap_read_end_io;
357 }
358
359 bio_add_page(ctx->bio, page, plen, poff);
360done:
361 /*
362 * Move the caller beyond our range so that it keeps making progress.
363 * For that we have to include any leading non-uptodate ranges, but
364 * we can skip trailing ones as they will be handled in the next
365 * iteration.
366 */
367 return pos - orig_pos + plen;
368}
369
370int
371iomap_readpage(struct page *page, const struct iomap_ops *ops)
372{
373 struct iomap_readpage_ctx ctx = { .cur_page = page };
374 struct inode *inode = page->mapping->host;
375 unsigned poff;
376 loff_t ret;
377
378 for (poff = 0; poff < PAGE_SIZE; poff += ret) {
379 ret = iomap_apply(inode, page_offset(page) + poff,
380 PAGE_SIZE - poff, 0, ops, &ctx,
381 iomap_readpage_actor);
382 if (ret <= 0) {
383 WARN_ON_ONCE(ret == 0);
384 SetPageError(page);
385 break;
386 }
387 }
388
389 if (ctx.bio) {
390 submit_bio(ctx.bio);
391 WARN_ON_ONCE(!ctx.cur_page_in_bio);
392 } else {
393 WARN_ON_ONCE(ctx.cur_page_in_bio);
394 unlock_page(page);
395 }
396
397 /*
398 * Just like mpage_readpages and block_read_full_page we always
399 * return 0 and just mark the page as PageError on errors. This
400 * should be cleaned up all through the stack eventually.
401 */
402 return 0;
403}
404EXPORT_SYMBOL_GPL(iomap_readpage);
405
406static struct page *
407iomap_next_page(struct inode *inode, struct list_head *pages, loff_t pos,
408 loff_t length, loff_t *done)
409{
410 while (!list_empty(pages)) {
411 struct page *page = lru_to_page(pages);
412
413 if (page_offset(page) >= (u64)pos + length)
414 break;
415
416 list_del(&page->lru);
417 if (!add_to_page_cache_lru(page, inode->i_mapping, page->index,
418 GFP_NOFS))
419 return page;
420
421 /*
422 * If we already have a page in the page cache at index we are
423 * done. Upper layers don't care if it is uptodate after the
424 * readpages call itself as every page gets checked again once
425 * actually needed.
426 */
427 *done += PAGE_SIZE;
428 put_page(page);
429 }
430
431 return NULL;
432}
433
434static loff_t
435iomap_readpages_actor(struct inode *inode, loff_t pos, loff_t length,
436 void *data, struct iomap *iomap)
437{
438 struct iomap_readpage_ctx *ctx = data;
439 loff_t done, ret;
440
441 for (done = 0; done < length; done += ret) {
442 if (ctx->cur_page && offset_in_page(pos + done) == 0) {
443 if (!ctx->cur_page_in_bio)
444 unlock_page(ctx->cur_page);
445 put_page(ctx->cur_page);
446 ctx->cur_page = NULL;
447 }
448 if (!ctx->cur_page) {
449 ctx->cur_page = iomap_next_page(inode, ctx->pages,
450 pos, length, &done);
451 if (!ctx->cur_page)
452 break;
453 ctx->cur_page_in_bio = false;
454 }
455 ret = iomap_readpage_actor(inode, pos + done, length - done,
456 ctx, iomap);
457 }
458
459 return done;
460}
461
462int
463iomap_readpages(struct address_space *mapping, struct list_head *pages,
464 unsigned nr_pages, const struct iomap_ops *ops)
465{
466 struct iomap_readpage_ctx ctx = {
467 .pages = pages,
468 .is_readahead = true,
469 };
470 loff_t pos = page_offset(list_entry(pages->prev, struct page, lru));
471 loff_t last = page_offset(list_entry(pages->next, struct page, lru));
472 loff_t length = last - pos + PAGE_SIZE, ret = 0;
473
474 while (length > 0) {
475 ret = iomap_apply(mapping->host, pos, length, 0, ops,
476 &ctx, iomap_readpages_actor);
477 if (ret <= 0) {
478 WARN_ON_ONCE(ret == 0);
479 goto done;
480 }
481 pos += ret;
482 length -= ret;
483 }
484 ret = 0;
485done:
486 if (ctx.bio)
487 submit_bio(ctx.bio);
488 if (ctx.cur_page) {
489 if (!ctx.cur_page_in_bio)
490 unlock_page(ctx.cur_page);
491 put_page(ctx.cur_page);
492 }
493
494 /*
495 * Check that we didn't lose a page due to the arcance calling
496 * conventions..
497 */
498 WARN_ON_ONCE(!ret && !list_empty(ctx.pages));
499 return ret;
500}
501EXPORT_SYMBOL_GPL(iomap_readpages);
502
503/*
504 * iomap_is_partially_uptodate checks whether blocks within a page are
505 * uptodate or not.
506 *
507 * Returns true if all blocks which correspond to a file portion
508 * we want to read within the page are uptodate.
509 */
510int
511iomap_is_partially_uptodate(struct page *page, unsigned long from,
512 unsigned long count)
513{
514 struct iomap_page *iop = to_iomap_page(page);
515 struct inode *inode = page->mapping->host;
516 unsigned len, first, last;
517 unsigned i;
518
519 /* Limit range to one page */
520 len = min_t(unsigned, PAGE_SIZE - from, count);
521
522 /* First and last blocks in range within page */
523 first = from >> inode->i_blkbits;
524 last = (from + len - 1) >> inode->i_blkbits;
525
526 if (iop) {
527 for (i = first; i <= last; i++)
528 if (!test_bit(i, iop->uptodate))
529 return 0;
530 return 1;
531 }
532
533 return 0;
534}
535EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate);
536
537int
538iomap_releasepage(struct page *page, gfp_t gfp_mask)
539{
540 /*
541 * mm accommodates an old ext3 case where clean pages might not have had
542 * the dirty bit cleared. Thus, it can send actual dirty pages to
543 * ->releasepage() via shrink_active_list(), skip those here.
544 */
545 if (PageDirty(page) || PageWriteback(page))
546 return 0;
547 iomap_page_release(page);
548 return 1;
549}
550EXPORT_SYMBOL_GPL(iomap_releasepage);
551
552void
553iomap_invalidatepage(struct page *page, unsigned int offset, unsigned int len)
554{
555 /*
556 * If we are invalidating the entire page, clear the dirty state from it
557 * and release it to avoid unnecessary buildup of the LRU.
558 */
559 if (offset == 0 && len == PAGE_SIZE) {
560 WARN_ON_ONCE(PageWriteback(page));
561 cancel_dirty_page(page);
562 iomap_page_release(page);
563 }
564}
565EXPORT_SYMBOL_GPL(iomap_invalidatepage);
566
567#ifdef CONFIG_MIGRATION
568int
569iomap_migrate_page(struct address_space *mapping, struct page *newpage,
570 struct page *page, enum migrate_mode mode)
571{
572 int ret;
573
574 ret = migrate_page_move_mapping(mapping, newpage, page, mode, 0);
575 if (ret != MIGRATEPAGE_SUCCESS)
576 return ret;
577
578 if (page_has_private(page)) {
579 ClearPagePrivate(page);
580 get_page(newpage);
581 set_page_private(newpage, page_private(page));
582 set_page_private(page, 0);
583 put_page(page);
584 SetPagePrivate(newpage);
585 }
586
587 if (mode != MIGRATE_SYNC_NO_COPY)
588 migrate_page_copy(newpage, page);
589 else
590 migrate_page_states(newpage, page);
591 return MIGRATEPAGE_SUCCESS;
592}
593EXPORT_SYMBOL_GPL(iomap_migrate_page);
594#endif /* CONFIG_MIGRATION */
595
596static void
597iomap_write_failed(struct inode *inode, loff_t pos, unsigned len)
598{
599 loff_t i_size = i_size_read(inode);
600
601 /*
602 * Only truncate newly allocated pages beyoned EOF, even if the
603 * write started inside the existing inode size.
604 */
605 if (pos + len > i_size)
606 truncate_pagecache_range(inode, max(pos, i_size), pos + len);
607}
608
609static int
610iomap_read_page_sync(struct inode *inode, loff_t block_start, struct page *page,
611 unsigned poff, unsigned plen, unsigned from, unsigned to,
612 struct iomap *iomap)
613{
614 struct bio_vec bvec;
615 struct bio bio;
616
617 if (iomap->type != IOMAP_MAPPED || block_start >= i_size_read(inode)) {
618 zero_user_segments(page, poff, from, to, poff + plen);
619 iomap_set_range_uptodate(page, poff, plen);
620 return 0;
621 }
622
623 bio_init(&bio, &bvec, 1);
624 bio.bi_opf = REQ_OP_READ;
625 bio.bi_iter.bi_sector = iomap_sector(iomap, block_start);
626 bio_set_dev(&bio, iomap->bdev);
627 __bio_add_page(&bio, page, plen, poff);
628 return submit_bio_wait(&bio);
629}
630
631static int
632__iomap_write_begin(struct inode *inode, loff_t pos, unsigned len,
633 struct page *page, struct iomap *iomap)
634{
635 struct iomap_page *iop = iomap_page_create(inode, page);
636 loff_t block_size = i_blocksize(inode);
637 loff_t block_start = pos & ~(block_size - 1);
638 loff_t block_end = (pos + len + block_size - 1) & ~(block_size - 1);
639 unsigned from = offset_in_page(pos), to = from + len, poff, plen;
640 int status = 0;
641
642 if (PageUptodate(page))
643 return 0;
644
645 do {
646 iomap_adjust_read_range(inode, iop, &block_start,
647 block_end - block_start, &poff, &plen);
648 if (plen == 0)
649 break;
650
651 if ((from > poff && from < poff + plen) ||
652 (to > poff && to < poff + plen)) {
653 status = iomap_read_page_sync(inode, block_start, page,
654 poff, plen, from, to, iomap);
655 if (status)
656 break;
657 }
658
659 } while ((block_start += plen) < block_end);
660
661 return status;
662}
663
664static int
665iomap_write_begin(struct inode *inode, loff_t pos, unsigned len, unsigned flags,
666 struct page **pagep, struct iomap *iomap)
667{
668 pgoff_t index = pos >> PAGE_SHIFT;
669 struct page *page;
670 int status = 0;
671
672 BUG_ON(pos + len > iomap->offset + iomap->length);
673
674 if (fatal_signal_pending(current))
675 return -EINTR;
676
677 page = grab_cache_page_write_begin(inode->i_mapping, index, flags);
678 if (!page)
679 return -ENOMEM;
680
681 if (iomap->type == IOMAP_INLINE)
682 iomap_read_inline_data(inode, page, iomap);
683 else if (iomap->flags & IOMAP_F_BUFFER_HEAD)
684 status = __block_write_begin_int(page, pos, len, NULL, iomap);
685 else
686 status = __iomap_write_begin(inode, pos, len, page, iomap);
687 if (unlikely(status)) {
688 unlock_page(page);
689 put_page(page);
690 page = NULL;
691
692 iomap_write_failed(inode, pos, len);
693 }
694
695 *pagep = page;
696 return status;
697}
698
699int
700iomap_set_page_dirty(struct page *page)
701{
702 struct address_space *mapping = page_mapping(page);
703 int newly_dirty;
704
705 if (unlikely(!mapping))
706 return !TestSetPageDirty(page);
707
708 /*
709 * Lock out page->mem_cgroup migration to keep PageDirty
710 * synchronized with per-memcg dirty page counters.
711 */
712 lock_page_memcg(page);
713 newly_dirty = !TestSetPageDirty(page);
714 if (newly_dirty)
715 __set_page_dirty(page, mapping, 0);
716 unlock_page_memcg(page);
717
718 if (newly_dirty)
719 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
720 return newly_dirty;
721}
722EXPORT_SYMBOL_GPL(iomap_set_page_dirty);
723
724static int
725__iomap_write_end(struct inode *inode, loff_t pos, unsigned len,
726 unsigned copied, struct page *page, struct iomap *iomap)
727{
728 flush_dcache_page(page);
729
730 /*
731 * The blocks that were entirely written will now be uptodate, so we
732 * don't have to worry about a readpage reading them and overwriting a
733 * partial write. However if we have encountered a short write and only
734 * partially written into a block, it will not be marked uptodate, so a
735 * readpage might come in and destroy our partial write.
736 *
737 * Do the simplest thing, and just treat any short write to a non
738 * uptodate page as a zero-length write, and force the caller to redo
739 * the whole thing.
740 */
741 if (unlikely(copied < len && !PageUptodate(page))) {
742 copied = 0;
743 } else {
744 iomap_set_range_uptodate(page, offset_in_page(pos), len);
745 iomap_set_page_dirty(page);
746 }
747 return __generic_write_end(inode, pos, copied, page);
748}
749
750static int
751iomap_write_end_inline(struct inode *inode, struct page *page,
752 struct iomap *iomap, loff_t pos, unsigned copied)
753{
754 void *addr;
755
756 WARN_ON_ONCE(!PageUptodate(page));
757 BUG_ON(pos + copied > PAGE_SIZE - offset_in_page(iomap->inline_data));
758
759 addr = kmap_atomic(page);
760 memcpy(iomap->inline_data + pos, addr + pos, copied);
761 kunmap_atomic(addr);
762
763 mark_inode_dirty(inode);
764 __generic_write_end(inode, pos, copied, page);
765 return copied;
766}
767
768static int
769iomap_write_end(struct inode *inode, loff_t pos, unsigned len,
770 unsigned copied, struct page *page, struct iomap *iomap)
771{
772 int ret;
773
774 if (iomap->type == IOMAP_INLINE) {
775 ret = iomap_write_end_inline(inode, page, iomap, pos, copied);
776 } else if (iomap->flags & IOMAP_F_BUFFER_HEAD) {
777 ret = generic_write_end(NULL, inode->i_mapping, pos, len,
778 copied, page, NULL);
779 } else {
780 ret = __iomap_write_end(inode, pos, len, copied, page, iomap);
781 }
782
783 if (iomap->page_done)
784 iomap->page_done(inode, pos, copied, page, iomap);
785
786 if (ret < len)
787 iomap_write_failed(inode, pos, len);
788 return ret;
789}
790
791static loff_t
792iomap_write_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
793 struct iomap *iomap)
794{
795 struct iov_iter *i = data;
796 long status = 0;
797 ssize_t written = 0;
798 unsigned int flags = AOP_FLAG_NOFS;
799
800 do {
801 struct page *page;
802 unsigned long offset; /* Offset into pagecache page */
803 unsigned long bytes; /* Bytes to write to page */
804 size_t copied; /* Bytes copied from user */
805
806 offset = offset_in_page(pos);
807 bytes = min_t(unsigned long, PAGE_SIZE - offset,
808 iov_iter_count(i));
809again:
810 if (bytes > length)
811 bytes = length;
812
813 /*
814 * Bring in the user page that we will copy from _first_.
815 * Otherwise there's a nasty deadlock on copying from the
816 * same page as we're writing to, without it being marked
817 * up-to-date.
818 *
819 * Not only is this an optimisation, but it is also required
820 * to check that the address is actually valid, when atomic
821 * usercopies are used, below.
822 */
823 if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
824 status = -EFAULT;
825 break;
826 }
827
828 status = iomap_write_begin(inode, pos, bytes, flags, &page,
829 iomap);
830 if (unlikely(status))
831 break;
832
833 if (mapping_writably_mapped(inode->i_mapping))
834 flush_dcache_page(page);
835
836 copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
837
838 flush_dcache_page(page);
839
840 status = iomap_write_end(inode, pos, bytes, copied, page,
841 iomap);
842 if (unlikely(status < 0))
843 break;
844 copied = status;
845
846 cond_resched();
847
848 iov_iter_advance(i, copied);
849 if (unlikely(copied == 0)) {
850 /*
851 * If we were unable to copy any data at all, we must
852 * fall back to a single segment length write.
853 *
854 * If we didn't fallback here, we could livelock
855 * because not all segments in the iov can be copied at
856 * once without a pagefault.
857 */
858 bytes = min_t(unsigned long, PAGE_SIZE - offset,
859 iov_iter_single_seg_count(i));
860 goto again;
861 }
862 pos += copied;
863 written += copied;
864 length -= copied;
865
866 balance_dirty_pages_ratelimited(inode->i_mapping);
867 } while (iov_iter_count(i) && length);
868
869 return written ? written : status;
870}
871
872ssize_t
873iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *iter,
874 const struct iomap_ops *ops)
875{
876 struct inode *inode = iocb->ki_filp->f_mapping->host;
877 loff_t pos = iocb->ki_pos, ret = 0, written = 0;
878
879 while (iov_iter_count(iter)) {
880 ret = iomap_apply(inode, pos, iov_iter_count(iter),
881 IOMAP_WRITE, ops, iter, iomap_write_actor);
882 if (ret <= 0)
883 break;
884 pos += ret;
885 written += ret;
886 }
887
888 return written ? written : ret;
889}
890EXPORT_SYMBOL_GPL(iomap_file_buffered_write);
891
892static struct page *
893__iomap_read_page(struct inode *inode, loff_t offset)
894{
895 struct address_space *mapping = inode->i_mapping;
896 struct page *page;
897
898 page = read_mapping_page(mapping, offset >> PAGE_SHIFT, NULL);
899 if (IS_ERR(page))
900 return page;
901 if (!PageUptodate(page)) {
902 put_page(page);
903 return ERR_PTR(-EIO);
904 }
905 return page;
906}
907
908static loff_t
909iomap_dirty_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
910 struct iomap *iomap)
911{
912 long status = 0;
913 ssize_t written = 0;
914
915 do {
916 struct page *page, *rpage;
917 unsigned long offset; /* Offset into pagecache page */
918 unsigned long bytes; /* Bytes to write to page */
919
920 offset = offset_in_page(pos);
921 bytes = min_t(loff_t, PAGE_SIZE - offset, length);
922
923 rpage = __iomap_read_page(inode, pos);
924 if (IS_ERR(rpage))
925 return PTR_ERR(rpage);
926
927 status = iomap_write_begin(inode, pos, bytes,
928 AOP_FLAG_NOFS, &page, iomap);
929 put_page(rpage);
930 if (unlikely(status))
931 return status;
932
933 WARN_ON_ONCE(!PageUptodate(page));
934
935 status = iomap_write_end(inode, pos, bytes, bytes, page, iomap);
936 if (unlikely(status <= 0)) {
937 if (WARN_ON_ONCE(status == 0))
938 return -EIO;
939 return status;
940 }
941
942 cond_resched();
943
944 pos += status;
945 written += status;
946 length -= status;
947
948 balance_dirty_pages_ratelimited(inode->i_mapping);
949 } while (length);
950
951 return written;
952}
953
954int
955iomap_file_dirty(struct inode *inode, loff_t pos, loff_t len,
956 const struct iomap_ops *ops)
957{
958 loff_t ret;
959
960 while (len) {
961 ret = iomap_apply(inode, pos, len, IOMAP_WRITE, ops, NULL,
962 iomap_dirty_actor);
963 if (ret <= 0)
964 return ret;
965 pos += ret;
966 len -= ret;
967 }
968
969 return 0;
970}
971EXPORT_SYMBOL_GPL(iomap_file_dirty);
972
973static int iomap_zero(struct inode *inode, loff_t pos, unsigned offset,
974 unsigned bytes, struct iomap *iomap)
975{
976 struct page *page;
977 int status;
978
979 status = iomap_write_begin(inode, pos, bytes, AOP_FLAG_NOFS, &page,
980 iomap);
981 if (status)
982 return status;
983
984 zero_user(page, offset, bytes);
985 mark_page_accessed(page);
986
987 return iomap_write_end(inode, pos, bytes, bytes, page, iomap);
988}
989
990static int iomap_dax_zero(loff_t pos, unsigned offset, unsigned bytes,
991 struct iomap *iomap)
992{
993 return __dax_zero_page_range(iomap->bdev, iomap->dax_dev,
994 iomap_sector(iomap, pos & PAGE_MASK), offset, bytes);
995}
996
997static loff_t
998iomap_zero_range_actor(struct inode *inode, loff_t pos, loff_t count,
999 void *data, struct iomap *iomap)
1000{
1001 bool *did_zero = data;
1002 loff_t written = 0;
1003 int status;
1004
1005 /* already zeroed? we're done. */
1006 if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
1007 return count;
1008
1009 do {
1010 unsigned offset, bytes;
1011
1012 offset = offset_in_page(pos);
1013 bytes = min_t(loff_t, PAGE_SIZE - offset, count);
1014
1015 if (IS_DAX(inode))
1016 status = iomap_dax_zero(pos, offset, bytes, iomap);
1017 else
1018 status = iomap_zero(inode, pos, offset, bytes, iomap);
1019 if (status < 0)
1020 return status;
1021
1022 pos += bytes;
1023 count -= bytes;
1024 written += bytes;
1025 if (did_zero)
1026 *did_zero = true;
1027 } while (count > 0);
1028
1029 return written;
1030}
1031
1032int
1033iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
1034 const struct iomap_ops *ops)
1035{
1036 loff_t ret;
1037
1038 while (len > 0) {
1039 ret = iomap_apply(inode, pos, len, IOMAP_ZERO,
1040 ops, did_zero, iomap_zero_range_actor);
1041 if (ret <= 0)
1042 return ret;
1043
1044 pos += ret;
1045 len -= ret;
1046 }
1047
1048 return 0;
1049}
1050EXPORT_SYMBOL_GPL(iomap_zero_range);
1051
1052int
1053iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
1054 const struct iomap_ops *ops)
1055{
1056 unsigned int blocksize = i_blocksize(inode);
1057 unsigned int off = pos & (blocksize - 1);
1058
1059 /* Block boundary? Nothing to do */
1060 if (!off)
1061 return 0;
1062 return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops);
1063}
1064EXPORT_SYMBOL_GPL(iomap_truncate_page);
1065
1066static loff_t
1067iomap_page_mkwrite_actor(struct inode *inode, loff_t pos, loff_t length,
1068 void *data, struct iomap *iomap)
1069{
1070 struct page *page = data;
1071 int ret;
1072
1073 if (iomap->flags & IOMAP_F_BUFFER_HEAD) {
1074 ret = __block_write_begin_int(page, pos, length, NULL, iomap);
1075 if (ret)
1076 return ret;
1077 block_commit_write(page, 0, length);
1078 } else {
1079 WARN_ON_ONCE(!PageUptodate(page));
1080 iomap_page_create(inode, page);
1081 set_page_dirty(page);
1082 }
1083
1084 return length;
1085}
1086
1087vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops)
1088{
1089 struct page *page = vmf->page;
1090 struct inode *inode = file_inode(vmf->vma->vm_file);
1091 unsigned long length;
1092 loff_t offset, size;
1093 ssize_t ret;
1094
1095 lock_page(page);
1096 size = i_size_read(inode);
1097 if ((page->mapping != inode->i_mapping) ||
1098 (page_offset(page) > size)) {
1099 /* We overload EFAULT to mean page got truncated */
1100 ret = -EFAULT;
1101 goto out_unlock;
1102 }
1103
1104 /* page is wholly or partially inside EOF */
1105 if (((page->index + 1) << PAGE_SHIFT) > size)
1106 length = offset_in_page(size);
1107 else
1108 length = PAGE_SIZE;
1109
1110 offset = page_offset(page);
1111 while (length > 0) {
1112 ret = iomap_apply(inode, offset, length,
1113 IOMAP_WRITE | IOMAP_FAULT, ops, page,
1114 iomap_page_mkwrite_actor);
1115 if (unlikely(ret <= 0))
1116 goto out_unlock;
1117 offset += ret;
1118 length -= ret;
1119 }
1120
1121 wait_for_stable_page(page);
1122 return VM_FAULT_LOCKED;
1123out_unlock:
1124 unlock_page(page);
1125 return block_page_mkwrite_return(ret);
1126}
1127EXPORT_SYMBOL_GPL(iomap_page_mkwrite);
1128
1129struct fiemap_ctx {
1130 struct fiemap_extent_info *fi;
1131 struct iomap prev;
1132};
1133
1134static int iomap_to_fiemap(struct fiemap_extent_info *fi,
1135 struct iomap *iomap, u32 flags)
1136{
1137 switch (iomap->type) {
1138 case IOMAP_HOLE:
1139 /* skip holes */
1140 return 0;
1141 case IOMAP_DELALLOC:
1142 flags |= FIEMAP_EXTENT_DELALLOC | FIEMAP_EXTENT_UNKNOWN;
1143 break;
1144 case IOMAP_MAPPED:
1145 break;
1146 case IOMAP_UNWRITTEN:
1147 flags |= FIEMAP_EXTENT_UNWRITTEN;
1148 break;
1149 case IOMAP_INLINE:
1150 flags |= FIEMAP_EXTENT_DATA_INLINE;
1151 break;
1152 }
1153
1154 if (iomap->flags & IOMAP_F_MERGED)
1155 flags |= FIEMAP_EXTENT_MERGED;
1156 if (iomap->flags & IOMAP_F_SHARED)
1157 flags |= FIEMAP_EXTENT_SHARED;
1158
1159 return fiemap_fill_next_extent(fi, iomap->offset,
1160 iomap->addr != IOMAP_NULL_ADDR ? iomap->addr : 0,
1161 iomap->length, flags);
1162}
1163
1164static loff_t
1165iomap_fiemap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
1166 struct iomap *iomap)
1167{
1168 struct fiemap_ctx *ctx = data;
1169 loff_t ret = length;
1170
1171 if (iomap->type == IOMAP_HOLE)
1172 return length;
1173
1174 ret = iomap_to_fiemap(ctx->fi, &ctx->prev, 0);
1175 ctx->prev = *iomap;
1176 switch (ret) {
1177 case 0: /* success */
1178 return length;
1179 case 1: /* extent array full */
1180 return 0;
1181 default:
1182 return ret;
1183 }
1184}
1185
1186int iomap_fiemap(struct inode *inode, struct fiemap_extent_info *fi,
1187 loff_t start, loff_t len, const struct iomap_ops *ops)
1188{
1189 struct fiemap_ctx ctx;
1190 loff_t ret;
1191
1192 memset(&ctx, 0, sizeof(ctx));
1193 ctx.fi = fi;
1194 ctx.prev.type = IOMAP_HOLE;
1195
1196 ret = fiemap_check_flags(fi, FIEMAP_FLAG_SYNC);
1197 if (ret)
1198 return ret;
1199
1200 if (fi->fi_flags & FIEMAP_FLAG_SYNC) {
1201 ret = filemap_write_and_wait(inode->i_mapping);
1202 if (ret)
1203 return ret;
1204 }
1205
1206 while (len > 0) {
1207 ret = iomap_apply(inode, start, len, IOMAP_REPORT, ops, &ctx,
1208 iomap_fiemap_actor);
1209 /* inode with no (attribute) mapping will give ENOENT */
1210 if (ret == -ENOENT)
1211 break;
1212 if (ret < 0)
1213 return ret;
1214 if (ret == 0)
1215 break;
1216
1217 start += ret;
1218 len -= ret;
1219 }
1220
1221 if (ctx.prev.type != IOMAP_HOLE) {
1222 ret = iomap_to_fiemap(fi, &ctx.prev, FIEMAP_EXTENT_LAST);
1223 if (ret < 0)
1224 return ret;
1225 }
1226
1227 return 0;
1228}
1229EXPORT_SYMBOL_GPL(iomap_fiemap);
1230
1231/*
1232 * Seek for SEEK_DATA / SEEK_HOLE within @page, starting at @lastoff.
1233 * Returns true if found and updates @lastoff to the offset in file.
1234 */
1235static bool
1236page_seek_hole_data(struct inode *inode, struct page *page, loff_t *lastoff,
1237 int whence)
1238{
1239 const struct address_space_operations *ops = inode->i_mapping->a_ops;
1240 unsigned int bsize = i_blocksize(inode), off;
1241 bool seek_data = whence == SEEK_DATA;
1242 loff_t poff = page_offset(page);
1243
1244 if (WARN_ON_ONCE(*lastoff >= poff + PAGE_SIZE))
1245 return false;
1246
1247 if (*lastoff < poff) {
1248 /*
1249 * Last offset smaller than the start of the page means we found
1250 * a hole:
1251 */
1252 if (whence == SEEK_HOLE)
1253 return true;
1254 *lastoff = poff;
1255 }
1256
1257 /*
1258 * Just check the page unless we can and should check block ranges:
1259 */
1260 if (bsize == PAGE_SIZE || !ops->is_partially_uptodate)
1261 return PageUptodate(page) == seek_data;
1262
1263 lock_page(page);
1264 if (unlikely(page->mapping != inode->i_mapping))
1265 goto out_unlock_not_found;
1266
1267 for (off = 0; off < PAGE_SIZE; off += bsize) {
1268 if (offset_in_page(*lastoff) >= off + bsize)
1269 continue;
1270 if (ops->is_partially_uptodate(page, off, bsize) == seek_data) {
1271 unlock_page(page);
1272 return true;
1273 }
1274 *lastoff = poff + off + bsize;
1275 }
1276
1277out_unlock_not_found:
1278 unlock_page(page);
1279 return false;
1280}
1281
1282/*
1283 * Seek for SEEK_DATA / SEEK_HOLE in the page cache.
1284 *
1285 * Within unwritten extents, the page cache determines which parts are holes
1286 * and which are data: uptodate buffer heads count as data; everything else
1287 * counts as a hole.
1288 *
1289 * Returns the resulting offset on successs, and -ENOENT otherwise.
1290 */
1291static loff_t
1292page_cache_seek_hole_data(struct inode *inode, loff_t offset, loff_t length,
1293 int whence)
1294{
1295 pgoff_t index = offset >> PAGE_SHIFT;
1296 pgoff_t end = DIV_ROUND_UP(offset + length, PAGE_SIZE);
1297 loff_t lastoff = offset;
1298 struct pagevec pvec;
1299
1300 if (length <= 0)
1301 return -ENOENT;
1302
1303 pagevec_init(&pvec);
1304
1305 do {
1306 unsigned nr_pages, i;
1307
1308 nr_pages = pagevec_lookup_range(&pvec, inode->i_mapping, &index,
1309 end - 1);
1310 if (nr_pages == 0)
1311 break;
1312
1313 for (i = 0; i < nr_pages; i++) {
1314 struct page *page = pvec.pages[i];
1315
1316 if (page_seek_hole_data(inode, page, &lastoff, whence))
1317 goto check_range;
1318 lastoff = page_offset(page) + PAGE_SIZE;
1319 }
1320 pagevec_release(&pvec);
1321 } while (index < end);
1322
1323 /* When no page at lastoff and we are not done, we found a hole. */
1324 if (whence != SEEK_HOLE)
1325 goto not_found;
1326
1327check_range:
1328 if (lastoff < offset + length)
1329 goto out;
1330not_found:
1331 lastoff = -ENOENT;
1332out:
1333 pagevec_release(&pvec);
1334 return lastoff;
1335}
1336
1337
1338static loff_t
1339iomap_seek_hole_actor(struct inode *inode, loff_t offset, loff_t length,
1340 void *data, struct iomap *iomap)
1341{
1342 switch (iomap->type) {
1343 case IOMAP_UNWRITTEN:
1344 offset = page_cache_seek_hole_data(inode, offset, length,
1345 SEEK_HOLE);
1346 if (offset < 0)
1347 return length;
1348 /* fall through */
1349 case IOMAP_HOLE:
1350 *(loff_t *)data = offset;
1351 return 0;
1352 default:
1353 return length;
1354 }
1355}
1356
1357loff_t
1358iomap_seek_hole(struct inode *inode, loff_t offset, const struct iomap_ops *ops)
1359{
1360 loff_t size = i_size_read(inode);
1361 loff_t length = size - offset;
1362 loff_t ret;
1363
1364 /* Nothing to be found before or beyond the end of the file. */
1365 if (offset < 0 || offset >= size)
1366 return -ENXIO;
1367
1368 while (length > 0) {
1369 ret = iomap_apply(inode, offset, length, IOMAP_REPORT, ops,
1370 &offset, iomap_seek_hole_actor);
1371 if (ret < 0)
1372 return ret;
1373 if (ret == 0)
1374 break;
1375
1376 offset += ret;
1377 length -= ret;
1378 }
1379
1380 return offset;
1381}
1382EXPORT_SYMBOL_GPL(iomap_seek_hole);
1383
1384static loff_t
1385iomap_seek_data_actor(struct inode *inode, loff_t offset, loff_t length,
1386 void *data, struct iomap *iomap)
1387{
1388 switch (iomap->type) {
1389 case IOMAP_HOLE:
1390 return length;
1391 case IOMAP_UNWRITTEN:
1392 offset = page_cache_seek_hole_data(inode, offset, length,
1393 SEEK_DATA);
1394 if (offset < 0)
1395 return length;
1396 /*FALLTHRU*/
1397 default:
1398 *(loff_t *)data = offset;
1399 return 0;
1400 }
1401}
1402
1403loff_t
1404iomap_seek_data(struct inode *inode, loff_t offset, const struct iomap_ops *ops)
1405{
1406 loff_t size = i_size_read(inode);
1407 loff_t length = size - offset;
1408 loff_t ret;
1409
1410 /* Nothing to be found before or beyond the end of the file. */
1411 if (offset < 0 || offset >= size)
1412 return -ENXIO;
1413
1414 while (length > 0) {
1415 ret = iomap_apply(inode, offset, length, IOMAP_REPORT, ops,
1416 &offset, iomap_seek_data_actor);
1417 if (ret < 0)
1418 return ret;
1419 if (ret == 0)
1420 break;
1421
1422 offset += ret;
1423 length -= ret;
1424 }
1425
1426 if (length <= 0)
1427 return -ENXIO;
1428 return offset;
1429}
1430EXPORT_SYMBOL_GPL(iomap_seek_data);
1431
1432/*
1433 * Private flags for iomap_dio, must not overlap with the public ones in
1434 * iomap.h:
1435 */
1436#define IOMAP_DIO_WRITE_FUA (1 << 28)
1437#define IOMAP_DIO_NEED_SYNC (1 << 29)
1438#define IOMAP_DIO_WRITE (1 << 30)
1439#define IOMAP_DIO_DIRTY (1 << 31)
1440
1441struct iomap_dio {
1442 struct kiocb *iocb;
1443 iomap_dio_end_io_t *end_io;
1444 loff_t i_size;
1445 loff_t size;
1446 atomic_t ref;
1447 unsigned flags;
1448 int error;
1449 bool wait_for_completion;
1450
1451 union {
1452 /* used during submission and for synchronous completion: */
1453 struct {
1454 struct iov_iter *iter;
1455 struct task_struct *waiter;
1456 struct request_queue *last_queue;
1457 blk_qc_t cookie;
1458 } submit;
1459
1460 /* used for aio completion: */
1461 struct {
1462 struct work_struct work;
1463 } aio;
1464 };
1465};
1466
1467int iomap_dio_iopoll(struct kiocb *kiocb, bool spin)
1468{
1469 struct request_queue *q = READ_ONCE(kiocb->private);
1470
1471 if (!q)
1472 return 0;
1473 return blk_poll(q, READ_ONCE(kiocb->ki_cookie), spin);
1474}
1475EXPORT_SYMBOL_GPL(iomap_dio_iopoll);
1476
1477static void iomap_dio_submit_bio(struct iomap_dio *dio, struct iomap *iomap,
1478 struct bio *bio)
1479{
1480 atomic_inc(&dio->ref);
1481
1482 if (dio->iocb->ki_flags & IOCB_HIPRI)
1483 bio_set_polled(bio, dio->iocb);
1484
1485 dio->submit.last_queue = bdev_get_queue(iomap->bdev);
1486 dio->submit.cookie = submit_bio(bio);
1487}
1488
1489static ssize_t iomap_dio_complete(struct iomap_dio *dio)
1490{
1491 struct kiocb *iocb = dio->iocb;
1492 struct inode *inode = file_inode(iocb->ki_filp);
1493 loff_t offset = iocb->ki_pos;
1494 ssize_t ret;
1495
1496 if (dio->end_io) {
1497 ret = dio->end_io(iocb,
1498 dio->error ? dio->error : dio->size,
1499 dio->flags);
1500 } else {
1501 ret = dio->error;
1502 }
1503
1504 if (likely(!ret)) {
1505 ret = dio->size;
1506 /* check for short read */
1507 if (offset + ret > dio->i_size &&
1508 !(dio->flags & IOMAP_DIO_WRITE))
1509 ret = dio->i_size - offset;
1510 iocb->ki_pos += ret;
1511 }
1512
1513 /*
1514 * Try again to invalidate clean pages which might have been cached by
1515 * non-direct readahead, or faulted in by get_user_pages() if the source
1516 * of the write was an mmap'ed region of the file we're writing. Either
1517 * one is a pretty crazy thing to do, so we don't support it 100%. If
1518 * this invalidation fails, tough, the write still worked...
1519 *
1520 * And this page cache invalidation has to be after dio->end_io(), as
1521 * some filesystems convert unwritten extents to real allocations in
1522 * end_io() when necessary, otherwise a racing buffer read would cache
1523 * zeros from unwritten extents.
1524 */
1525 if (!dio->error &&
1526 (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) {
1527 int err;
1528 err = invalidate_inode_pages2_range(inode->i_mapping,
1529 offset >> PAGE_SHIFT,
1530 (offset + dio->size - 1) >> PAGE_SHIFT);
1531 if (err)
1532 dio_warn_stale_pagecache(iocb->ki_filp);
1533 }
1534
1535 /*
1536 * If this is a DSYNC write, make sure we push it to stable storage now
1537 * that we've written data.
1538 */
1539 if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC))
1540 ret = generic_write_sync(iocb, ret);
1541
1542 inode_dio_end(file_inode(iocb->ki_filp));
1543 kfree(dio);
1544
1545 return ret;
1546}
1547
1548static void iomap_dio_complete_work(struct work_struct *work)
1549{
1550 struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);
1551 struct kiocb *iocb = dio->iocb;
1552
1553 iocb->ki_complete(iocb, iomap_dio_complete(dio), 0);
1554}
1555
1556/*
1557 * Set an error in the dio if none is set yet. We have to use cmpxchg
1558 * as the submission context and the completion context(s) can race to
1559 * update the error.
1560 */
1561static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)
1562{
1563 cmpxchg(&dio->error, 0, ret);
1564}
1565
1566static void iomap_dio_bio_end_io(struct bio *bio)
1567{
1568 struct iomap_dio *dio = bio->bi_private;
1569 bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
1570
1571 if (bio->bi_status)
1572 iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
1573
1574 if (atomic_dec_and_test(&dio->ref)) {
1575 if (dio->wait_for_completion) {
1576 struct task_struct *waiter = dio->submit.waiter;
1577 WRITE_ONCE(dio->submit.waiter, NULL);
1578 blk_wake_io_task(waiter);
1579 } else if (dio->flags & IOMAP_DIO_WRITE) {
1580 struct inode *inode = file_inode(dio->iocb->ki_filp);
1581
1582 INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
1583 queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work);
1584 } else {
1585 iomap_dio_complete_work(&dio->aio.work);
1586 }
1587 }
1588
1589 if (should_dirty) {
1590 bio_check_pages_dirty(bio);
1591 } else {
1592 if (!bio_flagged(bio, BIO_NO_PAGE_REF)) {
1593 struct bvec_iter_all iter_all;
1594 struct bio_vec *bvec;
1595 int i;
1596
1597 bio_for_each_segment_all(bvec, bio, i, iter_all)
1598 put_page(bvec->bv_page);
1599 }
1600 bio_put(bio);
1601 }
1602}
1603
1604static void
1605iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos,
1606 unsigned len)
1607{
1608 struct page *page = ZERO_PAGE(0);
1609 int flags = REQ_SYNC | REQ_IDLE;
1610 struct bio *bio;
1611
1612 bio = bio_alloc(GFP_KERNEL, 1);
1613 bio_set_dev(bio, iomap->bdev);
1614 bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
1615 bio->bi_private = dio;
1616 bio->bi_end_io = iomap_dio_bio_end_io;
1617
1618 get_page(page);
1619 __bio_add_page(bio, page, len, 0);
1620 bio_set_op_attrs(bio, REQ_OP_WRITE, flags);
1621 iomap_dio_submit_bio(dio, iomap, bio);
1622}
1623
1624static loff_t
1625iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length,
1626 struct iomap_dio *dio, struct iomap *iomap)
1627{
1628 unsigned int blkbits = blksize_bits(bdev_logical_block_size(iomap->bdev));
1629 unsigned int fs_block_size = i_blocksize(inode), pad;
1630 unsigned int align = iov_iter_alignment(dio->submit.iter);
1631 struct iov_iter iter;
1632 struct bio *bio;
1633 bool need_zeroout = false;
1634 bool use_fua = false;
1635 int nr_pages, ret = 0;
1636 size_t copied = 0;
1637
1638 if ((pos | length | align) & ((1 << blkbits) - 1))
1639 return -EINVAL;
1640
1641 if (iomap->type == IOMAP_UNWRITTEN) {
1642 dio->flags |= IOMAP_DIO_UNWRITTEN;
1643 need_zeroout = true;
1644 }
1645
1646 if (iomap->flags & IOMAP_F_SHARED)
1647 dio->flags |= IOMAP_DIO_COW;
1648
1649 if (iomap->flags & IOMAP_F_NEW) {
1650 need_zeroout = true;
1651 } else if (iomap->type == IOMAP_MAPPED) {
1652 /*
1653 * Use a FUA write if we need datasync semantics, this is a pure
1654 * data IO that doesn't require any metadata updates (including
1655 * after IO completion such as unwritten extent conversion) and
1656 * the underlying device supports FUA. This allows us to avoid
1657 * cache flushes on IO completion.
1658 */
1659 if (!(iomap->flags & (IOMAP_F_SHARED|IOMAP_F_DIRTY)) &&
1660 (dio->flags & IOMAP_DIO_WRITE_FUA) &&
1661 blk_queue_fua(bdev_get_queue(iomap->bdev)))
1662 use_fua = true;
1663 }
1664
1665 /*
1666 * Operate on a partial iter trimmed to the extent we were called for.
1667 * We'll update the iter in the dio once we're done with this extent.
1668 */
1669 iter = *dio->submit.iter;
1670 iov_iter_truncate(&iter, length);
1671
1672 nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
1673 if (nr_pages <= 0)
1674 return nr_pages;
1675
1676 if (need_zeroout) {
1677 /* zero out from the start of the block to the write offset */
1678 pad = pos & (fs_block_size - 1);
1679 if (pad)
1680 iomap_dio_zero(dio, iomap, pos - pad, pad);
1681 }
1682
1683 do {
1684 size_t n;
1685 if (dio->error) {
1686 iov_iter_revert(dio->submit.iter, copied);
1687 return 0;
1688 }
1689
1690 bio = bio_alloc(GFP_KERNEL, nr_pages);
1691 bio_set_dev(bio, iomap->bdev);
1692 bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
1693 bio->bi_write_hint = dio->iocb->ki_hint;
1694 bio->bi_ioprio = dio->iocb->ki_ioprio;
1695 bio->bi_private = dio;
1696 bio->bi_end_io = iomap_dio_bio_end_io;
1697
1698 ret = bio_iov_iter_get_pages(bio, &iter);
1699 if (unlikely(ret)) {
1700 /*
1701 * We have to stop part way through an IO. We must fall
1702 * through to the sub-block tail zeroing here, otherwise
1703 * this short IO may expose stale data in the tail of
1704 * the block we haven't written data to.
1705 */
1706 bio_put(bio);
1707 goto zero_tail;
1708 }
1709
1710 n = bio->bi_iter.bi_size;
1711 if (dio->flags & IOMAP_DIO_WRITE) {
1712 bio->bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
1713 if (use_fua)
1714 bio->bi_opf |= REQ_FUA;
1715 else
1716 dio->flags &= ~IOMAP_DIO_WRITE_FUA;
1717 task_io_account_write(n);
1718 } else {
1719 bio->bi_opf = REQ_OP_READ;
1720 if (dio->flags & IOMAP_DIO_DIRTY)
1721 bio_set_pages_dirty(bio);
1722 }
1723
1724 iov_iter_advance(dio->submit.iter, n);
1725
1726 dio->size += n;
1727 pos += n;
1728 copied += n;
1729
1730 nr_pages = iov_iter_npages(&iter, BIO_MAX_PAGES);
1731 iomap_dio_submit_bio(dio, iomap, bio);
1732 } while (nr_pages);
1733
1734 /*
1735 * We need to zeroout the tail of a sub-block write if the extent type
1736 * requires zeroing or the write extends beyond EOF. If we don't zero
1737 * the block tail in the latter case, we can expose stale data via mmap
1738 * reads of the EOF block.
1739 */
1740zero_tail:
1741 if (need_zeroout ||
1742 ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
1743 /* zero out from the end of the write to the end of the block */
1744 pad = pos & (fs_block_size - 1);
1745 if (pad)
1746 iomap_dio_zero(dio, iomap, pos, fs_block_size - pad);
1747 }
1748 return copied ? copied : ret;
1749}
1750
1751static loff_t
1752iomap_dio_hole_actor(loff_t length, struct iomap_dio *dio)
1753{
1754 length = iov_iter_zero(length, dio->submit.iter);
1755 dio->size += length;
1756 return length;
1757}
1758
1759static loff_t
1760iomap_dio_inline_actor(struct inode *inode, loff_t pos, loff_t length,
1761 struct iomap_dio *dio, struct iomap *iomap)
1762{
1763 struct iov_iter *iter = dio->submit.iter;
1764 size_t copied;
1765
1766 BUG_ON(pos + length > PAGE_SIZE - offset_in_page(iomap->inline_data));
1767
1768 if (dio->flags & IOMAP_DIO_WRITE) {
1769 loff_t size = inode->i_size;
1770
1771 if (pos > size)
1772 memset(iomap->inline_data + size, 0, pos - size);
1773 copied = copy_from_iter(iomap->inline_data + pos, length, iter);
1774 if (copied) {
1775 if (pos + copied > size)
1776 i_size_write(inode, pos + copied);
1777 mark_inode_dirty(inode);
1778 }
1779 } else {
1780 copied = copy_to_iter(iomap->inline_data + pos, length, iter);
1781 }
1782 dio->size += copied;
1783 return copied;
1784}
1785
1786static loff_t
1787iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
1788 void *data, struct iomap *iomap)
1789{
1790 struct iomap_dio *dio = data;
1791
1792 switch (iomap->type) {
1793 case IOMAP_HOLE:
1794 if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))
1795 return -EIO;
1796 return iomap_dio_hole_actor(length, dio);
1797 case IOMAP_UNWRITTEN:
1798 if (!(dio->flags & IOMAP_DIO_WRITE))
1799 return iomap_dio_hole_actor(length, dio);
1800 return iomap_dio_bio_actor(inode, pos, length, dio, iomap);
1801 case IOMAP_MAPPED:
1802 return iomap_dio_bio_actor(inode, pos, length, dio, iomap);
1803 case IOMAP_INLINE:
1804 return iomap_dio_inline_actor(inode, pos, length, dio, iomap);
1805 default:
1806 WARN_ON_ONCE(1);
1807 return -EIO;
1808 }
1809}
1810
1811/*
1812 * iomap_dio_rw() always completes O_[D]SYNC writes regardless of whether the IO
1813 * is being issued as AIO or not. This allows us to optimise pure data writes
1814 * to use REQ_FUA rather than requiring generic_write_sync() to issue a
1815 * REQ_FLUSH post write. This is slightly tricky because a single request here
1816 * can be mapped into multiple disjoint IOs and only a subset of the IOs issued
1817 * may be pure data writes. In that case, we still need to do a full data sync
1818 * completion.
1819 */
1820ssize_t
1821iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
1822 const struct iomap_ops *ops, iomap_dio_end_io_t end_io)
1823{
1824 struct address_space *mapping = iocb->ki_filp->f_mapping;
1825 struct inode *inode = file_inode(iocb->ki_filp);
1826 size_t count = iov_iter_count(iter);
1827 loff_t pos = iocb->ki_pos, start = pos;
1828 loff_t end = iocb->ki_pos + count - 1, ret = 0;
1829 unsigned int flags = IOMAP_DIRECT;
1830 bool wait_for_completion = is_sync_kiocb(iocb);
1831 struct blk_plug plug;
1832 struct iomap_dio *dio;
1833
1834 lockdep_assert_held(&inode->i_rwsem);
1835
1836 if (!count)
1837 return 0;
1838
1839 dio = kmalloc(sizeof(*dio), GFP_KERNEL);
1840 if (!dio)
1841 return -ENOMEM;
1842
1843 dio->iocb = iocb;
1844 atomic_set(&dio->ref, 1);
1845 dio->size = 0;
1846 dio->i_size = i_size_read(inode);
1847 dio->end_io = end_io;
1848 dio->error = 0;
1849 dio->flags = 0;
1850
1851 dio->submit.iter = iter;
1852 dio->submit.waiter = current;
1853 dio->submit.cookie = BLK_QC_T_NONE;
1854 dio->submit.last_queue = NULL;
1855
1856 if (iov_iter_rw(iter) == READ) {
1857 if (pos >= dio->i_size)
1858 goto out_free_dio;
1859
1860 if (iter_is_iovec(iter) && iov_iter_rw(iter) == READ)
1861 dio->flags |= IOMAP_DIO_DIRTY;
1862 } else {
1863 flags |= IOMAP_WRITE;
1864 dio->flags |= IOMAP_DIO_WRITE;
1865
1866 /* for data sync or sync, we need sync completion processing */
1867 if (iocb->ki_flags & IOCB_DSYNC)
1868 dio->flags |= IOMAP_DIO_NEED_SYNC;
1869
1870 /*
1871 * For datasync only writes, we optimistically try using FUA for
1872 * this IO. Any non-FUA write that occurs will clear this flag,
1873 * hence we know before completion whether a cache flush is
1874 * necessary.
1875 */
1876 if ((iocb->ki_flags & (IOCB_DSYNC | IOCB_SYNC)) == IOCB_DSYNC)
1877 dio->flags |= IOMAP_DIO_WRITE_FUA;
1878 }
1879
1880 if (iocb->ki_flags & IOCB_NOWAIT) {
1881 if (filemap_range_has_page(mapping, start, end)) {
1882 ret = -EAGAIN;
1883 goto out_free_dio;
1884 }
1885 flags |= IOMAP_NOWAIT;
1886 }
1887
1888 ret = filemap_write_and_wait_range(mapping, start, end);
1889 if (ret)
1890 goto out_free_dio;
1891
1892 /*
1893 * Try to invalidate cache pages for the range we're direct
1894 * writing. If this invalidation fails, tough, the write will
1895 * still work, but racing two incompatible write paths is a
1896 * pretty crazy thing to do, so we don't support it 100%.
1897 */
1898 ret = invalidate_inode_pages2_range(mapping,
1899 start >> PAGE_SHIFT, end >> PAGE_SHIFT);
1900 if (ret)
1901 dio_warn_stale_pagecache(iocb->ki_filp);
1902 ret = 0;
1903
1904 if (iov_iter_rw(iter) == WRITE && !wait_for_completion &&
1905 !inode->i_sb->s_dio_done_wq) {
1906 ret = sb_init_dio_done_wq(inode->i_sb);
1907 if (ret < 0)
1908 goto out_free_dio;
1909 }
1910
1911 inode_dio_begin(inode);
1912
1913 blk_start_plug(&plug);
1914 do {
1915 ret = iomap_apply(inode, pos, count, flags, ops, dio,
1916 iomap_dio_actor);
1917 if (ret <= 0) {
1918 /* magic error code to fall back to buffered I/O */
1919 if (ret == -ENOTBLK) {
1920 wait_for_completion = true;
1921 ret = 0;
1922 }
1923 break;
1924 }
1925 pos += ret;
1926
1927 if (iov_iter_rw(iter) == READ && pos >= dio->i_size)
1928 break;
1929 } while ((count = iov_iter_count(iter)) > 0);
1930 blk_finish_plug(&plug);
1931
1932 if (ret < 0)
1933 iomap_dio_set_error(dio, ret);
1934
1935 /*
1936 * If all the writes we issued were FUA, we don't need to flush the
1937 * cache on IO completion. Clear the sync flag for this case.
1938 */
1939 if (dio->flags & IOMAP_DIO_WRITE_FUA)
1940 dio->flags &= ~IOMAP_DIO_NEED_SYNC;
1941
1942 WRITE_ONCE(iocb->ki_cookie, dio->submit.cookie);
1943 WRITE_ONCE(iocb->private, dio->submit.last_queue);
1944
1945 /*
1946 * We are about to drop our additional submission reference, which
1947 * might be the last reference to the dio. There are three three
1948 * different ways we can progress here:
1949 *
1950 * (a) If this is the last reference we will always complete and free
1951 * the dio ourselves.
1952 * (b) If this is not the last reference, and we serve an asynchronous
1953 * iocb, we must never touch the dio after the decrement, the
1954 * I/O completion handler will complete and free it.
1955 * (c) If this is not the last reference, but we serve a synchronous
1956 * iocb, the I/O completion handler will wake us up on the drop
1957 * of the final reference, and we will complete and free it here
1958 * after we got woken by the I/O completion handler.
1959 */
1960 dio->wait_for_completion = wait_for_completion;
1961 if (!atomic_dec_and_test(&dio->ref)) {
1962 if (!wait_for_completion)
1963 return -EIOCBQUEUED;
1964
1965 for (;;) {
1966 set_current_state(TASK_UNINTERRUPTIBLE);
1967 if (!READ_ONCE(dio->submit.waiter))
1968 break;
1969
1970 if (!(iocb->ki_flags & IOCB_HIPRI) ||
1971 !dio->submit.last_queue ||
1972 !blk_poll(dio->submit.last_queue,
1973 dio->submit.cookie, true))
1974 io_schedule();
1975 }
1976 __set_current_state(TASK_RUNNING);
1977 }
1978
1979 return iomap_dio_complete(dio);
1980
1981out_free_dio:
1982 kfree(dio);
1983 return ret;
1984}
1985EXPORT_SYMBOL_GPL(iomap_dio_rw);
1986
1987/* Swapfile activation */
1988
1989#ifdef CONFIG_SWAP
1990struct iomap_swapfile_info {
1991 struct iomap iomap; /* accumulated iomap */
1992 struct swap_info_struct *sis;
1993 uint64_t lowest_ppage; /* lowest physical addr seen (pages) */
1994 uint64_t highest_ppage; /* highest physical addr seen (pages) */
1995 unsigned long nr_pages; /* number of pages collected */
1996 int nr_extents; /* extent count */
1997};
1998
1999/*
2000 * Collect physical extents for this swap file. Physical extents reported to
2001 * the swap code must be trimmed to align to a page boundary. The logical
2002 * offset within the file is irrelevant since the swapfile code maps logical
2003 * page numbers of the swap device to the physical page-aligned extents.
2004 */
2005static int iomap_swapfile_add_extent(struct iomap_swapfile_info *isi)
2006{
2007 struct iomap *iomap = &isi->iomap;
2008 unsigned long nr_pages;
2009 uint64_t first_ppage;
2010 uint64_t first_ppage_reported;
2011 uint64_t next_ppage;
2012 int error;
2013
2014 /*
2015 * Round the start up and the end down so that the physical
2016 * extent aligns to a page boundary.
2017 */
2018 first_ppage = ALIGN(iomap->addr, PAGE_SIZE) >> PAGE_SHIFT;
2019 next_ppage = ALIGN_DOWN(iomap->addr + iomap->length, PAGE_SIZE) >>
2020 PAGE_SHIFT;
2021
2022 /* Skip too-short physical extents. */
2023 if (first_ppage >= next_ppage)
2024 return 0;
2025 nr_pages = next_ppage - first_ppage;
2026
2027 /*
2028 * Calculate how much swap space we're adding; the first page contains
2029 * the swap header and doesn't count. The mm still wants that first
2030 * page fed to add_swap_extent, however.
2031 */
2032 first_ppage_reported = first_ppage;
2033 if (iomap->offset == 0)
2034 first_ppage_reported++;
2035 if (isi->lowest_ppage > first_ppage_reported)
2036 isi->lowest_ppage = first_ppage_reported;
2037 if (isi->highest_ppage < (next_ppage - 1))
2038 isi->highest_ppage = next_ppage - 1;
2039
2040 /* Add extent, set up for the next call. */
2041 error = add_swap_extent(isi->sis, isi->nr_pages, nr_pages, first_ppage);
2042 if (error < 0)
2043 return error;
2044 isi->nr_extents += error;
2045 isi->nr_pages += nr_pages;
2046 return 0;
2047}
2048
2049/*
2050 * Accumulate iomaps for this swap file. We have to accumulate iomaps because
2051 * swap only cares about contiguous page-aligned physical extents and makes no
2052 * distinction between written and unwritten extents.
2053 */
2054static loff_t iomap_swapfile_activate_actor(struct inode *inode, loff_t pos,
2055 loff_t count, void *data, struct iomap *iomap)
2056{
2057 struct iomap_swapfile_info *isi = data;
2058 int error;
2059
2060 switch (iomap->type) {
2061 case IOMAP_MAPPED:
2062 case IOMAP_UNWRITTEN:
2063 /* Only real or unwritten extents. */
2064 break;
2065 case IOMAP_INLINE:
2066 /* No inline data. */
2067 pr_err("swapon: file is inline\n");
2068 return -EINVAL;
2069 default:
2070 pr_err("swapon: file has unallocated extents\n");
2071 return -EINVAL;
2072 }
2073
2074 /* No uncommitted metadata or shared blocks. */
2075 if (iomap->flags & IOMAP_F_DIRTY) {
2076 pr_err("swapon: file is not committed\n");
2077 return -EINVAL;
2078 }
2079 if (iomap->flags & IOMAP_F_SHARED) {
2080 pr_err("swapon: file has shared extents\n");
2081 return -EINVAL;
2082 }
2083
2084 /* Only one bdev per swap file. */
2085 if (iomap->bdev != isi->sis->bdev) {
2086 pr_err("swapon: file is on multiple devices\n");
2087 return -EINVAL;
2088 }
2089
2090 if (isi->iomap.length == 0) {
2091 /* No accumulated extent, so just store it. */
2092 memcpy(&isi->iomap, iomap, sizeof(isi->iomap));
2093 } else if (isi->iomap.addr + isi->iomap.length == iomap->addr) {
2094 /* Append this to the accumulated extent. */
2095 isi->iomap.length += iomap->length;
2096 } else {
2097 /* Otherwise, add the retained iomap and store this one. */
2098 error = iomap_swapfile_add_extent(isi);
2099 if (error)
2100 return error;
2101 memcpy(&isi->iomap, iomap, sizeof(isi->iomap));
2102 }
2103 return count;
2104}
2105
2106/*
2107 * Iterate a swap file's iomaps to construct physical extents that can be
2108 * passed to the swapfile subsystem.
2109 */
2110int iomap_swapfile_activate(struct swap_info_struct *sis,
2111 struct file *swap_file, sector_t *pagespan,
2112 const struct iomap_ops *ops)
2113{
2114 struct iomap_swapfile_info isi = {
2115 .sis = sis,
2116 .lowest_ppage = (sector_t)-1ULL,
2117 };
2118 struct address_space *mapping = swap_file->f_mapping;
2119 struct inode *inode = mapping->host;
2120 loff_t pos = 0;
2121 loff_t len = ALIGN_DOWN(i_size_read(inode), PAGE_SIZE);
2122 loff_t ret;
2123
2124 /*
2125 * Persist all file mapping metadata so that we won't have any
2126 * IOMAP_F_DIRTY iomaps.
2127 */
2128 ret = vfs_fsync(swap_file, 1);
2129 if (ret)
2130 return ret;
2131
2132 while (len > 0) {
2133 ret = iomap_apply(inode, pos, len, IOMAP_REPORT,
2134 ops, &isi, iomap_swapfile_activate_actor);
2135 if (ret <= 0)
2136 return ret;
2137
2138 pos += ret;
2139 len -= ret;
2140 }
2141
2142 if (isi.iomap.length) {
2143 ret = iomap_swapfile_add_extent(&isi);
2144 if (ret)
2145 return ret;
2146 }
2147
2148 *pagespan = 1 + isi.highest_ppage - isi.lowest_ppage;
2149 sis->max = isi.nr_pages;
2150 sis->pages = isi.nr_pages - 1;
2151 sis->highest_bit = isi.nr_pages - 1;
2152 return isi.nr_extents;
2153}
2154EXPORT_SYMBOL_GPL(iomap_swapfile_activate);
2155#endif /* CONFIG_SWAP */
2156
2157static loff_t
2158iomap_bmap_actor(struct inode *inode, loff_t pos, loff_t length,
2159 void *data, struct iomap *iomap)
2160{
2161 sector_t *bno = data, addr;
2162
2163 if (iomap->type == IOMAP_MAPPED) {
2164 addr = (pos - iomap->offset + iomap->addr) >> inode->i_blkbits;
2165 if (addr > INT_MAX)
2166 WARN(1, "would truncate bmap result\n");
2167 else
2168 *bno = addr;
2169 }
2170 return 0;
2171}
2172
2173/* legacy ->bmap interface. 0 is the error return (!) */
2174sector_t
2175iomap_bmap(struct address_space *mapping, sector_t bno,
2176 const struct iomap_ops *ops)
2177{
2178 struct inode *inode = mapping->host;
2179 loff_t pos = bno << inode->i_blkbits;
2180 unsigned blocksize = i_blocksize(inode);
2181
2182 if (filemap_write_and_wait(mapping))
2183 return 0;
2184
2185 bno = 0;
2186 iomap_apply(inode, pos, blocksize, 0, ops, &bno, iomap_bmap_actor);
2187 return bno;
2188}
2189EXPORT_SYMBOL_GPL(iomap_bmap);
2190