1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Maximum size of each resync request */ |
3 | #define RESYNC_BLOCK_SIZE (64*1024) |
4 | #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE) |
5 | |
6 | /* |
7 | * Number of guaranteed raid bios in case of extreme VM load: |
8 | */ |
9 | #define NR_RAID_BIOS 256 |
10 | |
11 | /* when we get a read error on a read-only array, we redirect to another |
12 | * device without failing the first device, or trying to over-write to |
13 | * correct the read error. To keep track of bad blocks on a per-bio |
14 | * level, we store IO_BLOCKED in the appropriate 'bios' pointer |
15 | */ |
16 | #define IO_BLOCKED ((struct bio *)1) |
17 | /* When we successfully write to a known bad-block, we need to remove the |
18 | * bad-block marking which must be done from process context. So we record |
19 | * the success by setting devs[n].bio to IO_MADE_GOOD |
20 | */ |
21 | #define IO_MADE_GOOD ((struct bio *)2) |
22 | |
23 | #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2) |
24 | #define MAX_PLUG_BIO 32 |
25 | |
26 | /* for managing resync I/O pages */ |
27 | struct resync_pages { |
28 | void *raid_bio; |
29 | struct page *pages[RESYNC_PAGES]; |
30 | }; |
31 | |
32 | struct raid1_plug_cb { |
33 | struct blk_plug_cb cb; |
34 | struct bio_list pending; |
35 | unsigned int count; |
36 | }; |
37 | |
38 | static void rbio_pool_free(void *rbio, void *data) |
39 | { |
40 | kfree(rbio); |
41 | } |
42 | |
43 | static inline int resync_alloc_pages(struct resync_pages *rp, |
44 | gfp_t gfp_flags) |
45 | { |
46 | int i; |
47 | |
48 | for (i = 0; i < RESYNC_PAGES; i++) { |
49 | rp->pages[i] = alloc_page(gfp_flags); |
50 | if (!rp->pages[i]) |
51 | goto out_free; |
52 | } |
53 | |
54 | return 0; |
55 | |
56 | out_free: |
57 | while (--i >= 0) |
58 | put_page(rp->pages[i]); |
59 | return -ENOMEM; |
60 | } |
61 | |
62 | static inline void resync_free_pages(struct resync_pages *rp) |
63 | { |
64 | int i; |
65 | |
66 | for (i = 0; i < RESYNC_PAGES; i++) |
67 | put_page(rp->pages[i]); |
68 | } |
69 | |
70 | static inline void resync_get_all_pages(struct resync_pages *rp) |
71 | { |
72 | int i; |
73 | |
74 | for (i = 0; i < RESYNC_PAGES; i++) |
75 | get_page(rp->pages[i]); |
76 | } |
77 | |
78 | static inline struct page *resync_fetch_page(struct resync_pages *rp, |
79 | unsigned idx) |
80 | { |
81 | if (WARN_ON_ONCE(idx >= RESYNC_PAGES)) |
82 | return NULL; |
83 | return rp->pages[idx]; |
84 | } |
85 | |
86 | /* |
87 | * 'strct resync_pages' stores actual pages used for doing the resync |
88 | * IO, and it is per-bio, so make .bi_private points to it. |
89 | */ |
90 | static inline struct resync_pages *get_resync_pages(struct bio *bio) |
91 | { |
92 | return bio->bi_private; |
93 | } |
94 | |
95 | /* generally called after bio_reset() for reseting bvec */ |
96 | static void md_bio_reset_resync_pages(struct bio *bio, struct resync_pages *rp, |
97 | int size) |
98 | { |
99 | int idx = 0; |
100 | |
101 | /* initialize bvec table again */ |
102 | do { |
103 | struct page *page = resync_fetch_page(rp, idx); |
104 | int len = min_t(int, size, PAGE_SIZE); |
105 | |
106 | if (WARN_ON(!bio_add_page(bio, page, len, 0))) { |
107 | bio->bi_status = BLK_STS_RESOURCE; |
108 | bio_endio(bio); |
109 | return; |
110 | } |
111 | |
112 | size -= len; |
113 | } while (idx++ < RESYNC_PAGES && size > 0); |
114 | } |
115 | |
116 | |
117 | static inline void raid1_submit_write(struct bio *bio) |
118 | { |
119 | struct md_rdev *rdev = (void *)bio->bi_bdev; |
120 | |
121 | bio->bi_next = NULL; |
122 | bio_set_dev(bio, rdev->bdev); |
123 | if (test_bit(Faulty, &rdev->flags)) |
124 | bio_io_error(bio); |
125 | else if (unlikely(bio_op(bio) == REQ_OP_DISCARD && |
126 | !bdev_max_discard_sectors(bio->bi_bdev))) |
127 | /* Just ignore it */ |
128 | bio_endio(bio); |
129 | else |
130 | submit_bio_noacct(bio); |
131 | } |
132 | |
133 | static inline bool raid1_add_bio_to_plug(struct mddev *mddev, struct bio *bio, |
134 | blk_plug_cb_fn unplug, int copies) |
135 | { |
136 | struct raid1_plug_cb *plug = NULL; |
137 | struct blk_plug_cb *cb; |
138 | |
139 | /* |
140 | * If bitmap is not enabled, it's safe to submit the io directly, and |
141 | * this can get optimal performance. |
142 | */ |
143 | if (!md_bitmap_enabled(mddev->bitmap)) { |
144 | raid1_submit_write(bio); |
145 | return true; |
146 | } |
147 | |
148 | cb = blk_check_plugged(unplug, mddev, sizeof(*plug)); |
149 | if (!cb) |
150 | return false; |
151 | |
152 | plug = container_of(cb, struct raid1_plug_cb, cb); |
153 | bio_list_add(&plug->pending, bio); |
154 | if (++plug->count / MAX_PLUG_BIO >= copies) { |
155 | list_del(&cb->list); |
156 | cb->callback(cb, false); |
157 | } |
158 | |
159 | |
160 | return true; |
161 | } |
162 | |
163 | /* |
164 | * current->bio_list will be set under submit_bio() context, in this case bitmap |
165 | * io will be added to the list and wait for current io submission to finish, |
166 | * while current io submission must wait for bitmap io to be done. In order to |
167 | * avoid such deadlock, submit bitmap io asynchronously. |
168 | */ |
169 | static inline void raid1_prepare_flush_writes(struct bitmap *bitmap) |
170 | { |
171 | if (current->bio_list) |
172 | md_bitmap_unplug_async(bitmap); |
173 | else |
174 | md_bitmap_unplug(bitmap); |
175 | } |
176 | |