1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* raid10.c : Multiple Devices driver for Linux
4	*
5	* Copyright (C) 2000-2004 Neil Brown
6	*
7	* RAID-10 support for md.
8	*
9	* Base on code in raid1.c. See raid1.c for further copyright information.
10	*/
11
12	#include <linux/slab.h>
13	#include <linux/delay.h>
14	#include <linux/blkdev.h>
15	#include <linux/module.h>
16	#include <linux/seq_file.h>
17	#include <linux/ratelimit.h>
18	#include <linux/kthread.h>
19	#include <linux/raid/md_p.h>
20	#include <trace/events/block.h>
21	#include "md.h"
22
23	#define RAID_1_10_NAME "raid10"
24	#include "raid10.h"
25	#include "raid0.h"
26	#include "md-bitmap.h"
27
28	/*
29	* RAID10 provides a combination of RAID0 and RAID1 functionality.
30	* The layout of data is defined by
31	* chunk_size
32	* raid_disks
33	* near_copies (stored in low byte of layout)
34	* far_copies (stored in second byte of layout)
35	* far_offset (stored in bit 16 of layout )
36	* use_far_sets (stored in bit 17 of layout )
37	* use_far_sets_bugfixed (stored in bit 18 of layout )
38	*
39	* The data to be stored is divided into chunks using chunksize. Each device
40	* is divided into far_copies sections. In each section, chunks are laid out
41	* in a style similar to raid0, but near_copies copies of each chunk is stored
42	* (each on a different drive). The starting device for each section is offset
43	* near_copies from the starting device of the previous section. Thus there
44	* are (near_copies * far_copies) of each chunk, and each is on a different
45	* drive. near_copies and far_copies must be at least one, and their product
46	* is at most raid_disks.
47	*
48	* If far_offset is true, then the far_copies are handled a bit differently.
49	* The copies are still in different stripes, but instead of being very far
50	* apart on disk, there are adjacent stripes.
51	*
52	* The far and offset algorithms are handled slightly differently if
53	* 'use_far_sets' is true. In this case, the array's devices are grouped into
54	* sets that are (near_copies * far_copies) in size. The far copied stripes
55	* are still shifted by 'near_copies' devices, but this shifting stays confined
56	* to the set rather than the entire array. This is done to improve the number
57	* of device combinations that can fail without causing the array to fail.
58	* Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk
59	* on a device):
60	* A B C D A B C D E
61	* ... ...
62	* D A B C E A B C D
63	* Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s):
64	* [A B] [C D] [A B] [C D E]
65	* |...| |...| |...| | ... |
66	* [B A] [D C] [B A] [E C D]
67	*/
68
69	static void allow_barrier(struct r10conf *conf);
70	static void lower_barrier(struct r10conf *conf);
71	static int _enough(struct r10conf *conf, int previous, int ignore);
72	static int enough(struct r10conf *conf, int ignore);
73	static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
74	int *skipped);
75	static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
76	static void end_reshape_write(struct bio *bio);
77	static void end_reshape(struct r10conf *conf);
78
79	#include "raid1-10.c"
80
81	#define NULL_CMD
82	#define cmd_before(conf, cmd) \
83	do { \
84	write_sequnlock_irq(&(conf)->resync_lock); \
85	cmd; \
86	} while (0)
87	#define cmd_after(conf) write_seqlock_irq(&(conf)->resync_lock)
88
89	#define wait_event_barrier_cmd(conf, cond, cmd) \
90	wait_event_cmd((conf)->wait_barrier, cond, cmd_before(conf, cmd), \
91	cmd_after(conf))
92
93	#define wait_event_barrier(conf, cond) \
94	wait_event_barrier_cmd(conf, cond, NULL_CMD)
95
96	/*
97	* for resync bio, r10bio pointer can be retrieved from the per-bio
98	* 'struct resync_pages'.
99	*/
100	static inline struct r10bio *get_resync_r10bio(struct bio *bio)
101	{
102	return get_resync_pages(bio)->raid_bio;
103	}
104
105	static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
106	{
107	struct r10conf *conf = data;
108	int size = offsetof(struct r10bio, devs[conf->geo.raid_disks]);
109
110	/* allocate a r10bio with room for raid_disks entries in the
111	* bios array */
112	return kzalloc(size, flags: gfp_flags);
113	}
114
115	#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
116	/* amount of memory to reserve for resync requests */
117	#define RESYNC_WINDOW (1024*1024)
118	/* maximum number of concurrent requests, memory permitting */
119	#define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
120	#define CLUSTER_RESYNC_WINDOW (32 * RESYNC_WINDOW)
121	#define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9)
122
123	/*
124	* When performing a resync, we need to read and compare, so
125	* we need as many pages are there are copies.
126	* When performing a recovery, we need 2 bios, one for read,
127	* one for write (we recover only one drive per r10buf)
128	*
129	*/
130	static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
131	{
132	struct r10conf *conf = data;
133	struct r10bio *r10_bio;
134	struct bio *bio;
135	int j;
136	int nalloc, nalloc_rp;
137	struct resync_pages *rps;
138
139	r10_bio = r10bio_pool_alloc(gfp_flags, data: conf);
140	if (!r10_bio)
141	return NULL;
142
143	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
144	test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
145	nalloc = conf->copies; /* resync */
146	else
147	nalloc = 2; /* recovery */
148
149	/* allocate once for all bios */
150	if (!conf->have_replacement)
151	nalloc_rp = nalloc;
152	else
153	nalloc_rp = nalloc * 2;
154	rps = kmalloc_array(n: nalloc_rp, size: sizeof(struct resync_pages), flags: gfp_flags);
155	if (!rps)
156	goto out_free_r10bio;
157
158	/*
159	* Allocate bios.
160	*/
161	for (j = nalloc ; j-- ; ) {
162	bio = bio_kmalloc(RESYNC_PAGES, gfp_mask: gfp_flags);
163	if (!bio)
164	goto out_free_bio;
165	bio_init(bio, NULL, table: bio->bi_inline_vecs, RESYNC_PAGES, opf: 0);
166	r10_bio->devs[j].bio = bio;
167	if (!conf->have_replacement)
168	continue;
169	bio = bio_kmalloc(RESYNC_PAGES, gfp_mask: gfp_flags);
170	if (!bio)
171	goto out_free_bio;
172	bio_init(bio, NULL, table: bio->bi_inline_vecs, RESYNC_PAGES, opf: 0);
173	r10_bio->devs[j].repl_bio = bio;
174	}
175	/*
176	* Allocate RESYNC_PAGES data pages and attach them
177	* where needed.
178	*/
179	for (j = 0; j < nalloc; j++) {
180	struct bio *rbio = r10_bio->devs[j].repl_bio;
181	struct resync_pages *rp, *rp_repl;
182
183	rp = &rps[j];
184	if (rbio)
185	rp_repl = &rps[nalloc + j];
186
187	bio = r10_bio->devs[j].bio;
188
189	if (!j || test_bit(MD_RECOVERY_SYNC,
190	&conf->mddev->recovery)) {
191	if (resync_alloc_pages(rp, gfp_flags))
192	goto out_free_pages;
193	} else {
194	memcpy(rp, &rps[0], sizeof(*rp));
195	resync_get_all_pages(rp);
196	}
197
198	rp->raid_bio = r10_bio;
199	bio->bi_private = rp;
200	if (rbio) {
201	memcpy(rp_repl, rp, sizeof(*rp));
202	rbio->bi_private = rp_repl;
203	}
204	}
205
206	return r10_bio;
207
208	out_free_pages:
209	while (--j >= 0)
210	resync_free_pages(rp: &rps[j]);
211
212	j = 0;
213	out_free_bio:
214	for ( ; j < nalloc; j++) {
215	if (r10_bio->devs[j].bio)
216	bio_uninit(r10_bio->devs[j].bio);
217	kfree(objp: r10_bio->devs[j].bio);
218	if (r10_bio->devs[j].repl_bio)
219	bio_uninit(r10_bio->devs[j].repl_bio);
220	kfree(objp: r10_bio->devs[j].repl_bio);
221	}
222	kfree(objp: rps);
223	out_free_r10bio:
224	rbio_pool_free(rbio: r10_bio, data: conf);
225	return NULL;
226	}
227
228	static void r10buf_pool_free(void *__r10_bio, void *data)
229	{
230	struct r10conf *conf = data;
231	struct r10bio *r10bio = __r10_bio;
232	int j;
233	struct resync_pages *rp = NULL;
234
235	for (j = conf->copies; j--; ) {
236	struct bio *bio = r10bio->devs[j].bio;
237
238	if (bio) {
239	rp = get_resync_pages(bio);
240	resync_free_pages(rp);
241	bio_uninit(bio);
242	kfree(objp: bio);
243	}
244
245	bio = r10bio->devs[j].repl_bio;
246	if (bio) {
247	bio_uninit(bio);
248	kfree(objp: bio);
249	}
250	}
251
252	/* resync pages array stored in the 1st bio's .bi_private */
253	kfree(objp: rp);
254
255	rbio_pool_free(rbio: r10bio, data: conf);
256	}
257
258	static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
259	{
260	int i;
261
262	for (i = 0; i < conf->geo.raid_disks; i++) {
263	struct bio **bio = & r10_bio->devs[i].bio;
264	if (!BIO_SPECIAL(*bio))
265	bio_put(*bio);
266	*bio = NULL;
267	bio = &r10_bio->devs[i].repl_bio;
268	if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
269	bio_put(*bio);
270	*bio = NULL;
271	}
272	}
273
274	static void free_r10bio(struct r10bio *r10_bio)
275	{
276	struct r10conf *conf = r10_bio->mddev->private;
277
278	put_all_bios(conf, r10_bio);
279	mempool_free(element: r10_bio, pool: &conf->r10bio_pool);
280	}
281
282	static void put_buf(struct r10bio *r10_bio)
283	{
284	struct r10conf *conf = r10_bio->mddev->private;
285
286	mempool_free(element: r10_bio, pool: &conf->r10buf_pool);
287
288	lower_barrier(conf);
289	}
290
291	static void wake_up_barrier(struct r10conf *conf)
292	{
293	if (wq_has_sleeper(wq_head: &conf->wait_barrier))
294	wake_up(&conf->wait_barrier);
295	}
296
297	static void reschedule_retry(struct r10bio *r10_bio)
298	{
299	unsigned long flags;
300	struct mddev *mddev = r10_bio->mddev;
301	struct r10conf *conf = mddev->private;
302
303	spin_lock_irqsave(&conf->device_lock, flags);
304	list_add(new: &r10_bio->retry_list, head: &conf->retry_list);
305	conf->nr_queued ++;
306	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
307
308	/* wake up frozen array... */
309	wake_up(&conf->wait_barrier);
310
311	md_wakeup_thread(thread: mddev->thread);
312	}
313
314	/*
315	* raid_end_bio_io() is called when we have finished servicing a mirrored
316	* operation and are ready to return a success/failure code to the buffer
317	* cache layer.
318	*/
319	static void raid_end_bio_io(struct r10bio *r10_bio)
320	{
321	struct bio *bio = r10_bio->master_bio;
322	struct r10conf *conf = r10_bio->mddev->private;
323
324	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
325	bio->bi_status = BLK_STS_IOERR;
326
327	bio_endio(bio);
328	/*
329	* Wake up any possible resync thread that waits for the device
330	* to go idle.
331	*/
332	allow_barrier(conf);
333
334	free_r10bio(r10_bio);
335	}
336
337	/*
338	* Update disk head position estimator based on IRQ completion info.
339	*/
340	static inline void update_head_pos(int slot, struct r10bio *r10_bio)
341	{
342	struct r10conf *conf = r10_bio->mddev->private;
343
344	conf->mirrors[r10_bio->devs[slot].devnum].head_position =
345	r10_bio->devs[slot].addr + (r10_bio->sectors);
346	}
347
348	/*
349	* Find the disk number which triggered given bio
350	*/
351	static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
352	struct bio *bio, int *slotp, int *replp)
353	{
354	int slot;
355	int repl = 0;
356
357	for (slot = 0; slot < conf->geo.raid_disks; slot++) {
358	if (r10_bio->devs[slot].bio == bio)
359	break;
360	if (r10_bio->devs[slot].repl_bio == bio) {
361	repl = 1;
362	break;
363	}
364	}
365
366	update_head_pos(slot, r10_bio);
367
368	if (slotp)
369	*slotp = slot;
370	if (replp)
371	*replp = repl;
372	return r10_bio->devs[slot].devnum;
373	}
374
375	static void raid10_end_read_request(struct bio *bio)
376	{
377	int uptodate = !bio->bi_status;
378	struct r10bio *r10_bio = bio->bi_private;
379	int slot;
380	struct md_rdev *rdev;
381	struct r10conf *conf = r10_bio->mddev->private;
382
383	slot = r10_bio->read_slot;
384	rdev = r10_bio->devs[slot].rdev;
385	/*
386	* this branch is our 'one mirror IO has finished' event handler:
387	*/
388	update_head_pos(slot, r10_bio);
389
390	if (uptodate) {
391	/*
392	* Set R10BIO_Uptodate in our master bio, so that
393	* we will return a good error code to the higher
394	* levels even if IO on some other mirrored buffer fails.
395	*
396	* The 'master' represents the composite IO operation to
397	* user-side. So if something waits for IO, then it will
398	* wait for the 'master' bio.
399	*/
400	set_bit(nr: R10BIO_Uptodate, addr: &r10_bio->state);
401	} else {
402	/* If all other devices that store this block have
403	* failed, we want to return the error upwards rather
404	* than fail the last device. Here we redefine
405	* "uptodate" to mean "Don't want to retry"
406	*/
407	if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state),
408	ignore: rdev->raid_disk))
409	uptodate = 1;
410	}
411	if (uptodate) {
412	raid_end_bio_io(r10_bio);
413	rdev_dec_pending(rdev, mddev: conf->mddev);
414	} else {
415	/*
416	* oops, read error - keep the refcount on the rdev
417	*/
418	pr_err_ratelimited("md/raid10:%s: %pg: rescheduling sector %llu\n",
419	mdname(conf->mddev),
420	rdev->bdev,
421	(unsigned long long)r10_bio->sector);
422	set_bit(nr: R10BIO_ReadError, addr: &r10_bio->state);
423	reschedule_retry(r10_bio);
424	}
425	}
426
427	static void close_write(struct r10bio *r10_bio)
428	{
429	/* clear the bitmap if all writes complete successfully */
430	md_bitmap_endwrite(bitmap: r10_bio->mddev->bitmap, offset: r10_bio->sector,
431	sectors: r10_bio->sectors,
432	success: !test_bit(R10BIO_Degraded, &r10_bio->state),
433	behind: 0);
434	md_write_end(mddev: r10_bio->mddev);
435	}
436
437	static void one_write_done(struct r10bio *r10_bio)
438	{
439	if (atomic_dec_and_test(v: &r10_bio->remaining)) {
440	if (test_bit(R10BIO_WriteError, &r10_bio->state))
441	reschedule_retry(r10_bio);
442	else {
443	close_write(r10_bio);
444	if (test_bit(R10BIO_MadeGood, &r10_bio->state))
445	reschedule_retry(r10_bio);
446	else
447	raid_end_bio_io(r10_bio);
448	}
449	}
450	}
451
452	static void raid10_end_write_request(struct bio *bio)
453	{
454	struct r10bio *r10_bio = bio->bi_private;
455	int dev;
456	int dec_rdev = 1;
457	struct r10conf *conf = r10_bio->mddev->private;
458	int slot, repl;
459	struct md_rdev *rdev = NULL;
460	struct bio *to_put = NULL;
461	bool discard_error;
462
463	discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD;
464
465	dev = find_bio_disk(conf, r10_bio, bio, slotp: &slot, replp: &repl);
466
467	if (repl)
468	rdev = conf->mirrors[dev].replacement;
469	if (!rdev) {
470	smp_rmb();
471	repl = 0;
472	rdev = conf->mirrors[dev].rdev;
473	}
474	/*
475	* this branch is our 'one mirror IO has finished' event handler:
476	*/
477	if (bio->bi_status && !discard_error) {
478	if (repl)
479	/* Never record new bad blocks to replacement,
480	* just fail it.
481	*/
482	md_error(mddev: rdev->mddev, rdev);
483	else {
484	set_bit(nr: WriteErrorSeen, addr: &rdev->flags);
485	if (!test_and_set_bit(nr: WantReplacement, addr: &rdev->flags))
486	set_bit(nr: MD_RECOVERY_NEEDED,
487	addr: &rdev->mddev->recovery);
488
489	dec_rdev = 0;
490	if (test_bit(FailFast, &rdev->flags) &&
491	(bio->bi_opf & MD_FAILFAST)) {
492	md_error(mddev: rdev->mddev, rdev);
493	}
494
495	/*
496	* When the device is faulty, it is not necessary to
497	* handle write error.
498	*/
499	if (!test_bit(Faulty, &rdev->flags))
500	set_bit(nr: R10BIO_WriteError, addr: &r10_bio->state);
501	else {
502	/* Fail the request */
503	set_bit(nr: R10BIO_Degraded, addr: &r10_bio->state);
504	r10_bio->devs[slot].bio = NULL;
505	to_put = bio;
506	dec_rdev = 1;
507	}
508	}
509	} else {
510	/*
511	* Set R10BIO_Uptodate in our master bio, so that
512	* we will return a good error code for to the higher
513	* levels even if IO on some other mirrored buffer fails.
514	*
515	* The 'master' represents the composite IO operation to
516	* user-side. So if something waits for IO, then it will
517	* wait for the 'master' bio.
518	*
519	* Do not set R10BIO_Uptodate if the current device is
520	* rebuilding or Faulty. This is because we cannot use
521	* such device for properly reading the data back (we could
522	* potentially use it, if the current write would have felt
523	* before rdev->recovery_offset, but for simplicity we don't
524	* check this here.
525	*/
526	if (test_bit(In_sync, &rdev->flags) &&
527	!test_bit(Faulty, &rdev->flags))
528	set_bit(nr: R10BIO_Uptodate, addr: &r10_bio->state);
529
530	/* Maybe we can clear some bad blocks. */
531	if (rdev_has_badblock(rdev, s: r10_bio->devs[slot].addr,
532	sectors: r10_bio->sectors) &&
533	!discard_error) {
534	bio_put(bio);
535	if (repl)
536	r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
537	else
538	r10_bio->devs[slot].bio = IO_MADE_GOOD;
539	dec_rdev = 0;
540	set_bit(nr: R10BIO_MadeGood, addr: &r10_bio->state);
541	}
542	}
543
544	/*
545	*
546	* Let's see if all mirrored write operations have finished
547	* already.
548	*/
549	one_write_done(r10_bio);
550	if (dec_rdev)
551	rdev_dec_pending(rdev, mddev: conf->mddev);
552	if (to_put)
553	bio_put(to_put);
554	}
555
556	/*
557	* RAID10 layout manager
558	* As well as the chunksize and raid_disks count, there are two
559	* parameters: near_copies and far_copies.
560	* near_copies * far_copies must be <= raid_disks.
561	* Normally one of these will be 1.
562	* If both are 1, we get raid0.
563	* If near_copies == raid_disks, we get raid1.
564	*
565	* Chunks are laid out in raid0 style with near_copies copies of the
566	* first chunk, followed by near_copies copies of the next chunk and
567	* so on.
568	* If far_copies > 1, then after 1/far_copies of the array has been assigned
569	* as described above, we start again with a device offset of near_copies.
570	* So we effectively have another copy of the whole array further down all
571	* the drives, but with blocks on different drives.
572	* With this layout, and block is never stored twice on the one device.
573	*
574	* raid10_find_phys finds the sector offset of a given virtual sector
575	* on each device that it is on.
576	*
577	* raid10_find_virt does the reverse mapping, from a device and a
578	* sector offset to a virtual address
579	*/
580
581	static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
582	{
583	int n,f;
584	sector_t sector;
585	sector_t chunk;
586	sector_t stripe;
587	int dev;
588	int slot = 0;
589	int last_far_set_start, last_far_set_size;
590
591	last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
592	last_far_set_start *= geo->far_set_size;
593
594	last_far_set_size = geo->far_set_size;
595	last_far_set_size += (geo->raid_disks % geo->far_set_size);
596
597	/* now calculate first sector/dev */
598	chunk = r10bio->sector >> geo->chunk_shift;
599	sector = r10bio->sector & geo->chunk_mask;
600
601	chunk *= geo->near_copies;
602	stripe = chunk;
603	dev = sector_div(stripe, geo->raid_disks);
604	if (geo->far_offset)
605	stripe *= geo->far_copies;
606
607	sector += stripe << geo->chunk_shift;
608
609	/* and calculate all the others */
610	for (n = 0; n < geo->near_copies; n++) {
611	int d = dev;
612	int set;
613	sector_t s = sector;
614	r10bio->devs[slot].devnum = d;
615	r10bio->devs[slot].addr = s;
616	slot++;
617
618	for (f = 1; f < geo->far_copies; f++) {
619	set = d / geo->far_set_size;
620	d += geo->near_copies;
621
622	if ((geo->raid_disks % geo->far_set_size) &&
623	(d > last_far_set_start)) {
624	d -= last_far_set_start;
625	d %= last_far_set_size;
626	d += last_far_set_start;
627	} else {
628	d %= geo->far_set_size;
629	d += geo->far_set_size * set;
630	}
631	s += geo->stride;
632	r10bio->devs[slot].devnum = d;
633	r10bio->devs[slot].addr = s;
634	slot++;
635	}
636	dev++;
637	if (dev >= geo->raid_disks) {
638	dev = 0;
639	sector += (geo->chunk_mask + 1);
640	}
641	}
642	}
643
644	static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
645	{
646	struct geom *geo = &conf->geo;
647
648	if (conf->reshape_progress != MaxSector &&
649	((r10bio->sector >= conf->reshape_progress) !=
650	conf->mddev->reshape_backwards)) {
651	set_bit(nr: R10BIO_Previous, addr: &r10bio->state);
652	geo = &conf->prev;
653	} else
654	clear_bit(nr: R10BIO_Previous, addr: &r10bio->state);
655
656	__raid10_find_phys(geo, r10bio);
657	}
658
659	static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
660	{
661	sector_t offset, chunk, vchunk;
662	/* Never use conf->prev as this is only called during resync
663	* or recovery, so reshape isn't happening
664	*/
665	struct geom *geo = &conf->geo;
666	int far_set_start = (dev / geo->far_set_size) * geo->far_set_size;
667	int far_set_size = geo->far_set_size;
668	int last_far_set_start;
669
670	if (geo->raid_disks % geo->far_set_size) {
671	last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
672	last_far_set_start *= geo->far_set_size;
673
674	if (dev >= last_far_set_start) {
675	far_set_size = geo->far_set_size;
676	far_set_size += (geo->raid_disks % geo->far_set_size);
677	far_set_start = last_far_set_start;
678	}
679	}
680
681	offset = sector & geo->chunk_mask;
682	if (geo->far_offset) {
683	int fc;
684	chunk = sector >> geo->chunk_shift;
685	fc = sector_div(chunk, geo->far_copies);
686	dev -= fc * geo->near_copies;
687	if (dev < far_set_start)
688	dev += far_set_size;
689	} else {
690	while (sector >= geo->stride) {
691	sector -= geo->stride;
692	if (dev < (geo->near_copies + far_set_start))
693	dev += far_set_size - geo->near_copies;
694	else
695	dev -= geo->near_copies;
696	}
697	chunk = sector >> geo->chunk_shift;
698	}
699	vchunk = chunk * geo->raid_disks + dev;
700	sector_div(vchunk, geo->near_copies);
701	return (vchunk << geo->chunk_shift) + offset;
702	}
703
704	/*
705	* This routine returns the disk from which the requested read should
706	* be done. There is a per-array 'next expected sequential IO' sector
707	* number - if this matches on the next IO then we use the last disk.
708	* There is also a per-disk 'last know head position' sector that is
709	* maintained from IRQ contexts, both the normal and the resync IO
710	* completion handlers update this position correctly. If there is no
711	* perfect sequential match then we pick the disk whose head is closest.
712	*
713	* If there are 2 mirrors in the same 2 devices, performance degrades
714	* because position is mirror, not device based.
715	*
716	* The rdev for the device selected will have nr_pending incremented.
717	*/
718
719	/*
720	* FIXME: possibly should rethink readbalancing and do it differently
721	* depending on near_copies / far_copies geometry.
722	*/
723	static struct md_rdev *read_balance(struct r10conf *conf,
724	struct r10bio *r10_bio,
725	int *max_sectors)
726	{
727	const sector_t this_sector = r10_bio->sector;
728	int disk, slot;
729	int sectors = r10_bio->sectors;
730	int best_good_sectors;
731	sector_t new_distance, best_dist;
732	struct md_rdev *best_dist_rdev, *best_pending_rdev, *rdev = NULL;
733	int do_balance;
734	int best_dist_slot, best_pending_slot;
735	bool has_nonrot_disk = false;
736	unsigned int min_pending;
737	struct geom *geo = &conf->geo;
738
739	raid10_find_phys(conf, r10bio: r10_bio);
740	best_dist_slot = -1;
741	min_pending = UINT_MAX;
742	best_dist_rdev = NULL;
743	best_pending_rdev = NULL;
744	best_dist = MaxSector;
745	best_good_sectors = 0;
746	do_balance = 1;
747	clear_bit(nr: R10BIO_FailFast, addr: &r10_bio->state);
748
749	if (raid1_should_read_first(mddev: conf->mddev, this_sector, len: sectors))
750	do_balance = 0;
751
752	for (slot = 0; slot < conf->copies ; slot++) {
753	sector_t first_bad;
754	int bad_sectors;
755	sector_t dev_sector;
756	unsigned int pending;
757	bool nonrot;
758
759	if (r10_bio->devs[slot].bio == IO_BLOCKED)
760	continue;
761	disk = r10_bio->devs[slot].devnum;
762	rdev = conf->mirrors[disk].replacement;
763	if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
764	r10_bio->devs[slot].addr + sectors >
765	rdev->recovery_offset)
766	rdev = conf->mirrors[disk].rdev;
767	if (rdev == NULL ||
768	test_bit(Faulty, &rdev->flags))
769	continue;
770	if (!test_bit(In_sync, &rdev->flags) &&
771	r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
772	continue;
773
774	dev_sector = r10_bio->devs[slot].addr;
775	if (is_badblock(rdev, s: dev_sector, sectors,
776	first_bad: &first_bad, bad_sectors: &bad_sectors)) {
777	if (best_dist < MaxSector)
778	/* Already have a better slot */
779	continue;
780	if (first_bad <= dev_sector) {
781	/* Cannot read here. If this is the
782	* 'primary' device, then we must not read
783	* beyond 'bad_sectors' from another device.
784	*/
785	bad_sectors -= (dev_sector - first_bad);
786	if (!do_balance && sectors > bad_sectors)
787	sectors = bad_sectors;
788	if (best_good_sectors > sectors)
789	best_good_sectors = sectors;
790	} else {
791	sector_t good_sectors =
792	first_bad - dev_sector;
793	if (good_sectors > best_good_sectors) {
794	best_good_sectors = good_sectors;
795	best_dist_slot = slot;
796	best_dist_rdev = rdev;
797	}
798	if (!do_balance)
799	/* Must read from here */
800	break;
801	}
802	continue;
803	} else
804	best_good_sectors = sectors;
805
806	if (!do_balance)
807	break;
808
809	nonrot = bdev_nonrot(bdev: rdev->bdev);
810	has_nonrot_disk |= nonrot;
811	pending = atomic_read(v: &rdev->nr_pending);
812	if (min_pending > pending && nonrot) {
813	min_pending = pending;
814	best_pending_slot = slot;
815	best_pending_rdev = rdev;
816	}
817
818	if (best_dist_slot >= 0)
819	/* At least 2 disks to choose from so failfast is OK */
820	set_bit(nr: R10BIO_FailFast, addr: &r10_bio->state);
821	/* This optimisation is debatable, and completely destroys
822	* sequential read speed for 'far copies' arrays. So only
823	* keep it for 'near' arrays, and review those later.
824	*/
825	if (geo->near_copies > 1 && !pending)
826	new_distance = 0;
827
828	/* for far > 1 always use the lowest address */
829	else if (geo->far_copies > 1)
830	new_distance = r10_bio->devs[slot].addr;
831	else
832	new_distance = abs(r10_bio->devs[slot].addr -
833	conf->mirrors[disk].head_position);
834
835	if (new_distance < best_dist) {
836	best_dist = new_distance;
837	best_dist_slot = slot;
838	best_dist_rdev = rdev;
839	}
840	}
841	if (slot >= conf->copies) {
842	if (has_nonrot_disk) {
843	slot = best_pending_slot;
844	rdev = best_pending_rdev;
845	} else {
846	slot = best_dist_slot;
847	rdev = best_dist_rdev;
848	}
849	}
850
851	if (slot >= 0) {
852	atomic_inc(v: &rdev->nr_pending);
853	r10_bio->read_slot = slot;
854	} else
855	rdev = NULL;
856	*max_sectors = best_good_sectors;
857
858	return rdev;
859	}
860
861	static void flush_pending_writes(struct r10conf *conf)
862	{
863	/* Any writes that have been queued but are awaiting
864	* bitmap updates get flushed here.
865	*/
866	spin_lock_irq(lock: &conf->device_lock);
867
868	if (conf->pending_bio_list.head) {
869	struct blk_plug plug;
870	struct bio *bio;
871
872	bio = bio_list_get(bl: &conf->pending_bio_list);
873	spin_unlock_irq(lock: &conf->device_lock);
874
875	/*
876	* As this is called in a wait_event() loop (see freeze_array),
877	* current->state might be TASK_UNINTERRUPTIBLE which will
878	* cause a warning when we prepare to wait again. As it is
879	* rare that this path is taken, it is perfectly safe to force
880	* us to go around the wait_event() loop again, so the warning
881	* is a false-positive. Silence the warning by resetting
882	* thread state
883	*/
884	__set_current_state(TASK_RUNNING);
885
886	blk_start_plug(&plug);
887	raid1_prepare_flush_writes(bitmap: conf->mddev->bitmap);
888	wake_up(&conf->wait_barrier);
889
890	while (bio) { /* submit pending writes */
891	struct bio *next = bio->bi_next;
892
893	raid1_submit_write(bio);
894	bio = next;
895	cond_resched();
896	}
897	blk_finish_plug(&plug);
898	} else
899	spin_unlock_irq(lock: &conf->device_lock);
900	}
901
902	/* Barriers....
903	* Sometimes we need to suspend IO while we do something else,
904	* either some resync/recovery, or reconfigure the array.
905	* To do this we raise a 'barrier'.
906	* The 'barrier' is a counter that can be raised multiple times
907	* to count how many activities are happening which preclude
908	* normal IO.
909	* We can only raise the barrier if there is no pending IO.
910	* i.e. if nr_pending == 0.
911	* We choose only to raise the barrier if no-one is waiting for the
912	* barrier to go down. This means that as soon as an IO request
913	* is ready, no other operations which require a barrier will start
914	* until the IO request has had a chance.
915	*
916	* So: regular IO calls 'wait_barrier'. When that returns there
917	* is no backgroup IO happening, It must arrange to call
918	* allow_barrier when it has finished its IO.
919	* backgroup IO calls must call raise_barrier. Once that returns
920	* there is no normal IO happeing. It must arrange to call
921	* lower_barrier when the particular background IO completes.
922	*/
923
924	static void raise_barrier(struct r10conf *conf, int force)
925	{
926	write_seqlock_irq(sl: &conf->resync_lock);
927
928	if (WARN_ON_ONCE(force && !conf->barrier))
929	force = false;
930
931	/* Wait until no block IO is waiting (unless 'force') */
932	wait_event_barrier(conf, force || !conf->nr_waiting);
933
934	/* block any new IO from starting */
935	WRITE_ONCE(conf->barrier, conf->barrier + 1);
936
937	/* Now wait for all pending IO to complete */
938	wait_event_barrier(conf, !atomic_read(&conf->nr_pending) &&
939	conf->barrier < RESYNC_DEPTH);
940
941	write_sequnlock_irq(sl: &conf->resync_lock);
942	}
943
944	static void lower_barrier(struct r10conf *conf)
945	{
946	unsigned long flags;
947
948	write_seqlock_irqsave(&conf->resync_lock, flags);
949	WRITE_ONCE(conf->barrier, conf->barrier - 1);
950	write_sequnlock_irqrestore(sl: &conf->resync_lock, flags);
951	wake_up(&conf->wait_barrier);
952	}
953
954	static bool stop_waiting_barrier(struct r10conf *conf)
955	{
956	struct bio_list *bio_list = current->bio_list;
957	struct md_thread *thread;
958
959	/* barrier is dropped */
960	if (!conf->barrier)
961	return true;
962
963	/*
964	* If there are already pending requests (preventing the barrier from
965	* rising completely), and the pre-process bio queue isn't empty, then
966	* don't wait, as we need to empty that queue to get the nr_pending
967	* count down.
968	*/
969	if (atomic_read(v: &conf->nr_pending) && bio_list &&
970	(!bio_list_empty(bl: &bio_list[0]) || !bio_list_empty(bl: &bio_list[1])))
971	return true;
972
973	/* daemon thread must exist while handling io */
974	thread = rcu_dereference_protected(conf->mddev->thread, true);
975	/*
976	* move on if io is issued from raid10d(), nr_pending is not released
977	* from original io(see handle_read_error()). All raise barrier is
978	* blocked until this io is done.
979	*/
980	if (thread->tsk == current) {
981	WARN_ON_ONCE(atomic_read(&conf->nr_pending) == 0);
982	return true;
983	}
984
985	return false;
986	}
987
988	static bool wait_barrier_nolock(struct r10conf *conf)
989	{
990	unsigned int seq = read_seqbegin(sl: &conf->resync_lock);
991
992	if (READ_ONCE(conf->barrier))
993	return false;
994
995	atomic_inc(v: &conf->nr_pending);
996	if (!read_seqretry(sl: &conf->resync_lock, start: seq))
997	return true;
998
999	if (atomic_dec_and_test(v: &conf->nr_pending))
1000	wake_up_barrier(conf);
1001
1002	return false;
1003	}
1004
1005	static bool wait_barrier(struct r10conf *conf, bool nowait)
1006	{
1007	bool ret = true;
1008
1009	if (wait_barrier_nolock(conf))
1010	return true;
1011
1012	write_seqlock_irq(sl: &conf->resync_lock);
1013	if (conf->barrier) {
1014	/* Return false when nowait flag is set */
1015	if (nowait) {
1016	ret = false;
1017	} else {
1018	conf->nr_waiting++;
1019	mddev_add_trace_msg(conf->mddev, "raid10 wait barrier");
1020	wait_event_barrier(conf, stop_waiting_barrier(conf));
1021	conf->nr_waiting--;
1022	}
1023	if (!conf->nr_waiting)
1024	wake_up(&conf->wait_barrier);
1025	}
1026	/* Only increment nr_pending when we wait */
1027	if (ret)
1028	atomic_inc(v: &conf->nr_pending);
1029	write_sequnlock_irq(sl: &conf->resync_lock);
1030	return ret;
1031	}
1032
1033	static void allow_barrier(struct r10conf *conf)
1034	{
1035	if ((atomic_dec_and_test(v: &conf->nr_pending)) ||
1036	(conf->array_freeze_pending))
1037	wake_up_barrier(conf);
1038	}
1039
1040	static void freeze_array(struct r10conf *conf, int extra)
1041	{
1042	/* stop syncio and normal IO and wait for everything to
1043	* go quiet.
1044	* We increment barrier and nr_waiting, and then
1045	* wait until nr_pending match nr_queued+extra
1046	* This is called in the context of one normal IO request
1047	* that has failed. Thus any sync request that might be pending
1048	* will be blocked by nr_pending, and we need to wait for
1049	* pending IO requests to complete or be queued for re-try.
1050	* Thus the number queued (nr_queued) plus this request (extra)
1051	* must match the number of pending IOs (nr_pending) before
1052	* we continue.
1053	*/
1054	write_seqlock_irq(sl: &conf->resync_lock);
1055	conf->array_freeze_pending++;
1056	WRITE_ONCE(conf->barrier, conf->barrier + 1);
1057	conf->nr_waiting++;
1058	wait_event_barrier_cmd(conf, atomic_read(&conf->nr_pending) ==
1059	conf->nr_queued + extra, flush_pending_writes(conf));
1060	conf->array_freeze_pending--;
1061	write_sequnlock_irq(sl: &conf->resync_lock);
1062	}
1063
1064	static void unfreeze_array(struct r10conf *conf)
1065	{
1066	/* reverse the effect of the freeze */
1067	write_seqlock_irq(sl: &conf->resync_lock);
1068	WRITE_ONCE(conf->barrier, conf->barrier - 1);
1069	conf->nr_waiting--;
1070	wake_up(&conf->wait_barrier);
1071	write_sequnlock_irq(sl: &conf->resync_lock);
1072	}
1073
1074	static sector_t choose_data_offset(struct r10bio *r10_bio,
1075	struct md_rdev *rdev)
1076	{
1077	if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
1078	test_bit(R10BIO_Previous, &r10_bio->state))
1079	return rdev->data_offset;
1080	else
1081	return rdev->new_data_offset;
1082	}
1083
1084	static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
1085	{
1086	struct raid1_plug_cb *plug = container_of(cb, struct raid1_plug_cb, cb);
1087	struct mddev *mddev = plug->cb.data;
1088	struct r10conf *conf = mddev->private;
1089	struct bio *bio;
1090
1091	if (from_schedule) {
1092	spin_lock_irq(lock: &conf->device_lock);
1093	bio_list_merge(bl: &conf->pending_bio_list, bl2: &plug->pending);
1094	spin_unlock_irq(lock: &conf->device_lock);
1095	wake_up_barrier(conf);
1096	md_wakeup_thread(thread: mddev->thread);
1097	kfree(objp: plug);
1098	return;
1099	}
1100
1101	/* we aren't scheduling, so we can do the write-out directly. */
1102	bio = bio_list_get(bl: &plug->pending);
1103	raid1_prepare_flush_writes(bitmap: mddev->bitmap);
1104	wake_up_barrier(conf);
1105
1106	while (bio) { /* submit pending writes */
1107	struct bio *next = bio->bi_next;
1108
1109	raid1_submit_write(bio);
1110	bio = next;
1111	cond_resched();
1112	}
1113	kfree(objp: plug);
1114	}
1115
1116	/*
1117	* 1. Register the new request and wait if the reconstruction thread has put
1118	* up a bar for new requests. Continue immediately if no resync is active
1119	* currently.
1120	* 2. If IO spans the reshape position. Need to wait for reshape to pass.
1121	*/
1122	static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf,
1123	struct bio *bio, sector_t sectors)
1124	{
1125	/* Bail out if REQ_NOWAIT is set for the bio */
1126	if (!wait_barrier(conf, nowait: bio->bi_opf & REQ_NOWAIT)) {
1127	bio_wouldblock_error(bio);
1128	return false;
1129	}
1130	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1131	bio->bi_iter.bi_sector < conf->reshape_progress &&
1132	bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
1133	allow_barrier(conf);
1134	if (bio->bi_opf & REQ_NOWAIT) {
1135	bio_wouldblock_error(bio);
1136	return false;
1137	}
1138	mddev_add_trace_msg(conf->mddev, "raid10 wait reshape");
1139	wait_event(conf->wait_barrier,
1140	conf->reshape_progress <= bio->bi_iter.bi_sector ||
1141	conf->reshape_progress >= bio->bi_iter.bi_sector +
1142	sectors);
1143	wait_barrier(conf, nowait: false);
1144	}
1145	return true;
1146	}
1147
1148	static void raid10_read_request(struct mddev *mddev, struct bio *bio,
1149	struct r10bio *r10_bio, bool io_accounting)
1150	{
1151	struct r10conf *conf = mddev->private;
1152	struct bio *read_bio;
1153	const enum req_op op = bio_op(bio);
1154	const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;
1155	int max_sectors;
1156	struct md_rdev *rdev;
1157	char b[BDEVNAME_SIZE];
1158	int slot = r10_bio->read_slot;
1159	struct md_rdev *err_rdev = NULL;
1160	gfp_t gfp = GFP_NOIO;
1161
1162	if (slot >= 0 && r10_bio->devs[slot].rdev) {
1163	/*
1164	* This is an error retry, but we cannot
1165	* safely dereference the rdev in the r10_bio,
1166	* we must use the one in conf.
1167	* If it has already been disconnected (unlikely)
1168	* we lose the device name in error messages.
1169	*/
1170	int disk;
1171	/*
1172	* As we are blocking raid10, it is a little safer to
1173	* use __GFP_HIGH.
1174	*/
1175	gfp = GFP_NOIO | __GFP_HIGH;
1176
1177	disk = r10_bio->devs[slot].devnum;
1178	err_rdev = conf->mirrors[disk].rdev;
1179	if (err_rdev)
1180	snprintf(buf: b, size: sizeof(b), fmt: "%pg", err_rdev->bdev);
1181	else {
1182	strcpy(p: b, q: "???");
1183	/* This never gets dereferenced */
1184	err_rdev = r10_bio->devs[slot].rdev;
1185	}
1186	}
1187
1188	if (!regular_request_wait(mddev, conf, bio, sectors: r10_bio->sectors))
1189	return;
1190	rdev = read_balance(conf, r10_bio, max_sectors: &max_sectors);
1191	if (!rdev) {
1192	if (err_rdev) {
1193	pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n",
1194	mdname(mddev), b,
1195	(unsigned long long)r10_bio->sector);
1196	}
1197	raid_end_bio_io(r10_bio);
1198	return;
1199	}
1200	if (err_rdev)
1201	pr_err_ratelimited("md/raid10:%s: %pg: redirecting sector %llu to another mirror\n",
1202	mdname(mddev),
1203	rdev->bdev,
1204	(unsigned long long)r10_bio->sector);
1205	if (max_sectors < bio_sectors(bio)) {
1206	struct bio *split = bio_split(bio, sectors: max_sectors,
1207	gfp, bs: &conf->bio_split);
1208	bio_chain(split, bio);
1209	allow_barrier(conf);
1210	submit_bio_noacct(bio);
1211	wait_barrier(conf, nowait: false);
1212	bio = split;
1213	r10_bio->master_bio = bio;
1214	r10_bio->sectors = max_sectors;
1215	}
1216	slot = r10_bio->read_slot;
1217
1218	if (io_accounting) {
1219	md_account_bio(mddev, bio: &bio);
1220	r10_bio->master_bio = bio;
1221	}
1222	read_bio = bio_alloc_clone(bdev: rdev->bdev, bio_src: bio, gfp, bs: &mddev->bio_set);
1223
1224	r10_bio->devs[slot].bio = read_bio;
1225	r10_bio->devs[slot].rdev = rdev;
1226
1227	read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr +
1228	choose_data_offset(r10_bio, rdev);
1229	read_bio->bi_end_io = raid10_end_read_request;
1230	read_bio->bi_opf = op | do_sync;
1231	if (test_bit(FailFast, &rdev->flags) &&
1232	test_bit(R10BIO_FailFast, &r10_bio->state))
1233	read_bio->bi_opf |= MD_FAILFAST;
1234	read_bio->bi_private = r10_bio;
1235	mddev_trace_remap(mddev, bio: read_bio, sector: r10_bio->sector);
1236	submit_bio_noacct(bio: read_bio);
1237	return;
1238	}
1239
1240	static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio,
1241	struct bio *bio, bool replacement,
1242	int n_copy)
1243	{
1244	const enum req_op op = bio_op(bio);
1245	const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;
1246	const blk_opf_t do_fua = bio->bi_opf & REQ_FUA;
1247	unsigned long flags;
1248	struct r10conf *conf = mddev->private;
1249	struct md_rdev *rdev;
1250	int devnum = r10_bio->devs[n_copy].devnum;
1251	struct bio *mbio;
1252
1253	rdev = replacement ? conf->mirrors[devnum].replacement :
1254	conf->mirrors[devnum].rdev;
1255
1256	mbio = bio_alloc_clone(bdev: rdev->bdev, bio_src: bio, GFP_NOIO, bs: &mddev->bio_set);
1257	if (replacement)
1258	r10_bio->devs[n_copy].repl_bio = mbio;
1259	else
1260	r10_bio->devs[n_copy].bio = mbio;
1261
1262	mbio->bi_iter.bi_sector = (r10_bio->devs[n_copy].addr +
1263	choose_data_offset(r10_bio, rdev));
1264	mbio->bi_end_io = raid10_end_write_request;
1265	mbio->bi_opf = op | do_sync | do_fua;
1266	if (!replacement && test_bit(FailFast,
1267	&conf->mirrors[devnum].rdev->flags)
1268	&& enough(conf, ignore: devnum))
1269	mbio->bi_opf |= MD_FAILFAST;
1270	mbio->bi_private = r10_bio;
1271	mddev_trace_remap(mddev, bio: mbio, sector: r10_bio->sector);
1272	/* flush_pending_writes() needs access to the rdev so...*/
1273	mbio->bi_bdev = (void *)rdev;
1274
1275	atomic_inc(v: &r10_bio->remaining);
1276
1277	if (!raid1_add_bio_to_plug(mddev, bio: mbio, unplug: raid10_unplug, copies: conf->copies)) {
1278	spin_lock_irqsave(&conf->device_lock, flags);
1279	bio_list_add(bl: &conf->pending_bio_list, bio: mbio);
1280	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
1281	md_wakeup_thread(thread: mddev->thread);
1282	}
1283	}
1284
1285	static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio)
1286	{
1287	int i;
1288	struct r10conf *conf = mddev->private;
1289	struct md_rdev *blocked_rdev;
1290
1291	retry_wait:
1292	blocked_rdev = NULL;
1293	for (i = 0; i < conf->copies; i++) {
1294	struct md_rdev *rdev, *rrdev;
1295
1296	rdev = conf->mirrors[i].rdev;
1297	rrdev = conf->mirrors[i].replacement;
1298	if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
1299	atomic_inc(v: &rdev->nr_pending);
1300	blocked_rdev = rdev;
1301	break;
1302	}
1303	if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
1304	atomic_inc(v: &rrdev->nr_pending);
1305	blocked_rdev = rrdev;
1306	break;
1307	}
1308
1309	if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1310	sector_t dev_sector = r10_bio->devs[i].addr;
1311
1312	/*
1313	* Discard request doesn't care the write result
1314	* so it doesn't need to wait blocked disk here.
1315	*/
1316	if (!r10_bio->sectors)
1317	continue;
1318
1319	if (rdev_has_badblock(rdev, s: dev_sector,
1320	sectors: r10_bio->sectors) < 0) {
1321	/*
1322	* Mustn't write here until the bad block
1323	* is acknowledged
1324	*/
1325	atomic_inc(v: &rdev->nr_pending);
1326	set_bit(nr: BlockedBadBlocks, addr: &rdev->flags);
1327	blocked_rdev = rdev;
1328	break;
1329	}
1330	}
1331	}
1332
1333	if (unlikely(blocked_rdev)) {
1334	/* Have to wait for this device to get unblocked, then retry */
1335	allow_barrier(conf);
1336	mddev_add_trace_msg(conf->mddev,
1337	"raid10 %s wait rdev %d blocked",
1338	__func__, blocked_rdev->raid_disk);
1339	md_wait_for_blocked_rdev(rdev: blocked_rdev, mddev);
1340	wait_barrier(conf, nowait: false);
1341	goto retry_wait;
1342	}
1343	}
1344
1345	static void raid10_write_request(struct mddev *mddev, struct bio *bio,
1346	struct r10bio *r10_bio)
1347	{
1348	struct r10conf *conf = mddev->private;
1349	int i;
1350	sector_t sectors;
1351	int max_sectors;
1352
1353	if ((mddev_is_clustered(mddev) &&
1354	md_cluster_ops->area_resyncing(mddev, WRITE,
1355	bio->bi_iter.bi_sector,
1356	bio_end_sector(bio)))) {
1357	DEFINE_WAIT(w);
1358	/* Bail out if REQ_NOWAIT is set for the bio */
1359	if (bio->bi_opf & REQ_NOWAIT) {
1360	bio_wouldblock_error(bio);
1361	return;
1362	}
1363	for (;;) {
1364	prepare_to_wait(wq_head: &conf->wait_barrier,
1365	wq_entry: &w, TASK_IDLE);
1366	if (!md_cluster_ops->area_resyncing(mddev, WRITE,
1367	bio->bi_iter.bi_sector, bio_end_sector(bio)))
1368	break;
1369	schedule();
1370	}
1371	finish_wait(wq_head: &conf->wait_barrier, wq_entry: &w);
1372	}
1373
1374	sectors = r10_bio->sectors;
1375	if (!regular_request_wait(mddev, conf, bio, sectors))
1376	return;
1377	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1378	(mddev->reshape_backwards
1379	? (bio->bi_iter.bi_sector < conf->reshape_safe &&
1380	bio->bi_iter.bi_sector + sectors > conf->reshape_progress)
1381	: (bio->bi_iter.bi_sector + sectors > conf->reshape_safe &&
1382	bio->bi_iter.bi_sector < conf->reshape_progress))) {
1383	/* Need to update reshape_position in metadata */
1384	mddev->reshape_position = conf->reshape_progress;
1385	set_mask_bits(&mddev->sb_flags, 0,
1386	BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1387	md_wakeup_thread(thread: mddev->thread);
1388	if (bio->bi_opf & REQ_NOWAIT) {
1389	allow_barrier(conf);
1390	bio_wouldblock_error(bio);
1391	return;
1392	}
1393	mddev_add_trace_msg(conf->mddev,
1394	"raid10 wait reshape metadata");
1395	wait_event(mddev->sb_wait,
1396	!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
1397
1398	conf->reshape_safe = mddev->reshape_position;
1399	}
1400
1401	/* first select target devices under rcu_lock and
1402	* inc refcount on their rdev. Record them by setting
1403	* bios[x] to bio
1404	* If there are known/acknowledged bad blocks on any device
1405	* on which we have seen a write error, we want to avoid
1406	* writing to those blocks. This potentially requires several
1407	* writes to write around the bad blocks. Each set of writes
1408	* gets its own r10_bio with a set of bios attached.
1409	*/
1410
1411	r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
1412	raid10_find_phys(conf, r10bio: r10_bio);
1413
1414	wait_blocked_dev(mddev, r10_bio);
1415
1416	max_sectors = r10_bio->sectors;
1417
1418	for (i = 0; i < conf->copies; i++) {
1419	int d = r10_bio->devs[i].devnum;
1420	struct md_rdev *rdev, *rrdev;
1421
1422	rdev = conf->mirrors[d].rdev;
1423	rrdev = conf->mirrors[d].replacement;
1424	if (rdev && (test_bit(Faulty, &rdev->flags)))
1425	rdev = NULL;
1426	if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1427	rrdev = NULL;
1428
1429	r10_bio->devs[i].bio = NULL;
1430	r10_bio->devs[i].repl_bio = NULL;
1431
1432	if (!rdev && !rrdev) {
1433	set_bit(nr: R10BIO_Degraded, addr: &r10_bio->state);
1434	continue;
1435	}
1436	if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1437	sector_t first_bad;
1438	sector_t dev_sector = r10_bio->devs[i].addr;
1439	int bad_sectors;
1440	int is_bad;
1441
1442	is_bad = is_badblock(rdev, s: dev_sector, sectors: max_sectors,
1443	first_bad: &first_bad, bad_sectors: &bad_sectors);
1444	if (is_bad && first_bad <= dev_sector) {
1445	/* Cannot write here at all */
1446	bad_sectors -= (dev_sector - first_bad);
1447	if (bad_sectors < max_sectors)
1448	/* Mustn't write more than bad_sectors
1449	* to other devices yet
1450	*/
1451	max_sectors = bad_sectors;
1452	/* We don't set R10BIO_Degraded as that
1453	* only applies if the disk is missing,
1454	* so it might be re-added, and we want to
1455	* know to recover this chunk.
1456	* In this case the device is here, and the
1457	* fact that this chunk is not in-sync is
1458	* recorded in the bad block log.
1459	*/
1460	continue;
1461	}
1462	if (is_bad) {
1463	int good_sectors = first_bad - dev_sector;
1464	if (good_sectors < max_sectors)
1465	max_sectors = good_sectors;
1466	}
1467	}
1468	if (rdev) {
1469	r10_bio->devs[i].bio = bio;
1470	atomic_inc(v: &rdev->nr_pending);
1471	}
1472	if (rrdev) {
1473	r10_bio->devs[i].repl_bio = bio;
1474	atomic_inc(v: &rrdev->nr_pending);
1475	}
1476	}
1477
1478	if (max_sectors < r10_bio->sectors)
1479	r10_bio->sectors = max_sectors;
1480
1481	if (r10_bio->sectors < bio_sectors(bio)) {
1482	struct bio *split = bio_split(bio, sectors: r10_bio->sectors,
1483	GFP_NOIO, bs: &conf->bio_split);
1484	bio_chain(split, bio);
1485	allow_barrier(conf);
1486	submit_bio_noacct(bio);
1487	wait_barrier(conf, nowait: false);
1488	bio = split;
1489	r10_bio->master_bio = bio;
1490	}
1491
1492	md_account_bio(mddev, bio: &bio);
1493	r10_bio->master_bio = bio;
1494	atomic_set(v: &r10_bio->remaining, i: 1);
1495	md_bitmap_startwrite(bitmap: mddev->bitmap, offset: r10_bio->sector, sectors: r10_bio->sectors, behind: 0);
1496
1497	for (i = 0; i < conf->copies; i++) {
1498	if (r10_bio->devs[i].bio)
1499	raid10_write_one_disk(mddev, r10_bio, bio, replacement: false, n_copy: i);
1500	if (r10_bio->devs[i].repl_bio)
1501	raid10_write_one_disk(mddev, r10_bio, bio, replacement: true, n_copy: i);
1502	}
1503	one_write_done(r10_bio);
1504	}
1505
1506	static void __make_request(struct mddev *mddev, struct bio *bio, int sectors)
1507	{
1508	struct r10conf *conf = mddev->private;
1509	struct r10bio *r10_bio;
1510
1511	r10_bio = mempool_alloc(pool: &conf->r10bio_pool, GFP_NOIO);
1512
1513	r10_bio->master_bio = bio;
1514	r10_bio->sectors = sectors;
1515
1516	r10_bio->mddev = mddev;
1517	r10_bio->sector = bio->bi_iter.bi_sector;
1518	r10_bio->state = 0;
1519	r10_bio->read_slot = -1;
1520	memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) *
1521	conf->geo.raid_disks);
1522
1523	if (bio_data_dir(bio) == READ)
1524	raid10_read_request(mddev, bio, r10_bio, io_accounting: true);
1525	else
1526	raid10_write_request(mddev, bio, r10_bio);
1527	}
1528
1529	static void raid_end_discard_bio(struct r10bio *r10bio)
1530	{
1531	struct r10conf *conf = r10bio->mddev->private;
1532	struct r10bio *first_r10bio;
1533
1534	while (atomic_dec_and_test(v: &r10bio->remaining)) {
1535
1536	allow_barrier(conf);
1537
1538	if (!test_bit(R10BIO_Discard, &r10bio->state)) {
1539	first_r10bio = (struct r10bio *)r10bio->master_bio;
1540	free_r10bio(r10_bio: r10bio);
1541	r10bio = first_r10bio;
1542	} else {
1543	md_write_end(mddev: r10bio->mddev);
1544	bio_endio(r10bio->master_bio);
1545	free_r10bio(r10_bio: r10bio);
1546	break;
1547	}
1548	}
1549	}
1550
1551	static void raid10_end_discard_request(struct bio *bio)
1552	{
1553	struct r10bio *r10_bio = bio->bi_private;
1554	struct r10conf *conf = r10_bio->mddev->private;
1555	struct md_rdev *rdev = NULL;
1556	int dev;
1557	int slot, repl;
1558
1559	/*
1560	* We don't care the return value of discard bio
1561	*/
1562	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
1563	set_bit(nr: R10BIO_Uptodate, addr: &r10_bio->state);
1564
1565	dev = find_bio_disk(conf, r10_bio, bio, slotp: &slot, replp: &repl);
1566	rdev = repl ? conf->mirrors[dev].replacement :
1567	conf->mirrors[dev].rdev;
1568
1569	raid_end_discard_bio(r10bio: r10_bio);
1570	rdev_dec_pending(rdev, mddev: conf->mddev);
1571	}
1572
1573	/*
1574	* There are some limitations to handle discard bio
1575	* 1st, the discard size is bigger than stripe_size*2.
1576	* 2st, if the discard bio spans reshape progress, we use the old way to
1577	* handle discard bio
1578	*/
1579	static int raid10_handle_discard(struct mddev *mddev, struct bio *bio)
1580	{
1581	struct r10conf *conf = mddev->private;
1582	struct geom *geo = &conf->geo;
1583	int far_copies = geo->far_copies;
1584	bool first_copy = true;
1585	struct r10bio *r10_bio, *first_r10bio;
1586	struct bio *split;
1587	int disk;
1588	sector_t chunk;
1589	unsigned int stripe_size;
1590	unsigned int stripe_data_disks;
1591	sector_t split_size;
1592	sector_t bio_start, bio_end;
1593	sector_t first_stripe_index, last_stripe_index;
1594	sector_t start_disk_offset;
1595	unsigned int start_disk_index;
1596	sector_t end_disk_offset;
1597	unsigned int end_disk_index;
1598	unsigned int remainder;
1599
1600	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
1601	return -EAGAIN;
1602
1603	if (WARN_ON_ONCE(bio->bi_opf & REQ_NOWAIT)) {
1604	bio_wouldblock_error(bio);
1605	return 0;
1606	}
1607	wait_barrier(conf, nowait: false);
1608
1609	/*
1610	* Check reshape again to avoid reshape happens after checking
1611	* MD_RECOVERY_RESHAPE and before wait_barrier
1612	*/
1613	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
1614	goto out;
1615
1616	if (geo->near_copies)
1617	stripe_data_disks = geo->raid_disks / geo->near_copies +
1618	geo->raid_disks % geo->near_copies;
1619	else
1620	stripe_data_disks = geo->raid_disks;
1621
1622	stripe_size = stripe_data_disks << geo->chunk_shift;
1623
1624	bio_start = bio->bi_iter.bi_sector;
1625	bio_end = bio_end_sector(bio);
1626
1627	/*
1628	* Maybe one discard bio is smaller than strip size or across one
1629	* stripe and discard region is larger than one stripe size. For far
1630	* offset layout, if the discard region is not aligned with stripe
1631	* size, there is hole when we submit discard bio to member disk.
1632	* For simplicity, we only handle discard bio which discard region
1633	* is bigger than stripe_size * 2
1634	*/
1635	if (bio_sectors(bio) < stripe_size*2)
1636	goto out;
1637
1638	/*
1639	* Keep bio aligned with strip size.
1640	*/
1641	div_u64_rem(dividend: bio_start, divisor: stripe_size, remainder: &remainder);
1642	if (remainder) {
1643	split_size = stripe_size - remainder;
1644	split = bio_split(bio, sectors: split_size, GFP_NOIO, bs: &conf->bio_split);
1645	bio_chain(split, bio);
1646	allow_barrier(conf);
1647	/* Resend the fist split part */
1648	submit_bio_noacct(bio: split);
1649	wait_barrier(conf, nowait: false);
1650	}
1651	div_u64_rem(dividend: bio_end, divisor: stripe_size, remainder: &remainder);
1652	if (remainder) {
1653	split_size = bio_sectors(bio) - remainder;
1654	split = bio_split(bio, sectors: split_size, GFP_NOIO, bs: &conf->bio_split);
1655	bio_chain(split, bio);
1656	allow_barrier(conf);
1657	/* Resend the second split part */
1658	submit_bio_noacct(bio);
1659	bio = split;
1660	wait_barrier(conf, nowait: false);
1661	}
1662
1663	bio_start = bio->bi_iter.bi_sector;
1664	bio_end = bio_end_sector(bio);
1665
1666	/*
1667	* Raid10 uses chunk as the unit to store data. It's similar like raid0.
1668	* One stripe contains the chunks from all member disk (one chunk from
1669	* one disk at the same HBA address). For layout detail, see 'man md 4'
1670	*/
1671	chunk = bio_start >> geo->chunk_shift;
1672	chunk *= geo->near_copies;
1673	first_stripe_index = chunk;
1674	start_disk_index = sector_div(first_stripe_index, geo->raid_disks);
1675	if (geo->far_offset)
1676	first_stripe_index *= geo->far_copies;
1677	start_disk_offset = (bio_start & geo->chunk_mask) +
1678	(first_stripe_index << geo->chunk_shift);
1679
1680	chunk = bio_end >> geo->chunk_shift;
1681	chunk *= geo->near_copies;
1682	last_stripe_index = chunk;
1683	end_disk_index = sector_div(last_stripe_index, geo->raid_disks);
1684	if (geo->far_offset)
1685	last_stripe_index *= geo->far_copies;
1686	end_disk_offset = (bio_end & geo->chunk_mask) +
1687	(last_stripe_index << geo->chunk_shift);
1688
1689	retry_discard:
1690	r10_bio = mempool_alloc(pool: &conf->r10bio_pool, GFP_NOIO);
1691	r10_bio->mddev = mddev;
1692	r10_bio->state = 0;
1693	r10_bio->sectors = 0;
1694	memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * geo->raid_disks);
1695	wait_blocked_dev(mddev, r10_bio);
1696
1697	/*
1698	* For far layout it needs more than one r10bio to cover all regions.
1699	* Inspired by raid10_sync_request, we can use the first r10bio->master_bio
1700	* to record the discard bio. Other r10bio->master_bio record the first
1701	* r10bio. The first r10bio only release after all other r10bios finish.
1702	* The discard bio returns only first r10bio finishes
1703	*/
1704	if (first_copy) {
1705	r10_bio->master_bio = bio;
1706	set_bit(nr: R10BIO_Discard, addr: &r10_bio->state);
1707	first_copy = false;
1708	first_r10bio = r10_bio;
1709	} else
1710	r10_bio->master_bio = (struct bio *)first_r10bio;
1711
1712	/*
1713	* first select target devices under rcu_lock and
1714	* inc refcount on their rdev. Record them by setting
1715	* bios[x] to bio
1716	*/
1717	for (disk = 0; disk < geo->raid_disks; disk++) {
1718	struct md_rdev *rdev, *rrdev;
1719
1720	rdev = conf->mirrors[disk].rdev;
1721	rrdev = conf->mirrors[disk].replacement;
1722	r10_bio->devs[disk].bio = NULL;
1723	r10_bio->devs[disk].repl_bio = NULL;
1724
1725	if (rdev && (test_bit(Faulty, &rdev->flags)))
1726	rdev = NULL;
1727	if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1728	rrdev = NULL;
1729	if (!rdev && !rrdev)
1730	continue;
1731
1732	if (rdev) {
1733	r10_bio->devs[disk].bio = bio;
1734	atomic_inc(v: &rdev->nr_pending);
1735	}
1736	if (rrdev) {
1737	r10_bio->devs[disk].repl_bio = bio;
1738	atomic_inc(v: &rrdev->nr_pending);
1739	}
1740	}
1741
1742	atomic_set(v: &r10_bio->remaining, i: 1);
1743	for (disk = 0; disk < geo->raid_disks; disk++) {
1744	sector_t dev_start, dev_end;
1745	struct bio *mbio, *rbio = NULL;
1746
1747	/*
1748	* Now start to calculate the start and end address for each disk.
1749	* The space between dev_start and dev_end is the discard region.
1750	*
1751	* For dev_start, it needs to consider three conditions:
1752	* 1st, the disk is before start_disk, you can imagine the disk in
1753	* the next stripe. So the dev_start is the start address of next
1754	* stripe.
1755	* 2st, the disk is after start_disk, it means the disk is at the
1756	* same stripe of first disk
1757	* 3st, the first disk itself, we can use start_disk_offset directly
1758	*/
1759	if (disk < start_disk_index)
1760	dev_start = (first_stripe_index + 1) * mddev->chunk_sectors;
1761	else if (disk > start_disk_index)
1762	dev_start = first_stripe_index * mddev->chunk_sectors;
1763	else
1764	dev_start = start_disk_offset;
1765
1766	if (disk < end_disk_index)
1767	dev_end = (last_stripe_index + 1) * mddev->chunk_sectors;
1768	else if (disk > end_disk_index)
1769	dev_end = last_stripe_index * mddev->chunk_sectors;
1770	else
1771	dev_end = end_disk_offset;
1772
1773	/*
1774	* It only handles discard bio which size is >= stripe size, so
1775	* dev_end > dev_start all the time.
1776	* It doesn't need to use rcu lock to get rdev here. We already
1777	* add rdev->nr_pending in the first loop.
1778	*/
1779	if (r10_bio->devs[disk].bio) {
1780	struct md_rdev *rdev = conf->mirrors[disk].rdev;
1781	mbio = bio_alloc_clone(bdev: bio->bi_bdev, bio_src: bio, GFP_NOIO,
1782	bs: &mddev->bio_set);
1783	mbio->bi_end_io = raid10_end_discard_request;
1784	mbio->bi_private = r10_bio;
1785	r10_bio->devs[disk].bio = mbio;
1786	r10_bio->devs[disk].devnum = disk;
1787	atomic_inc(v: &r10_bio->remaining);
1788	md_submit_discard_bio(mddev, rdev, bio: mbio,
1789	start: dev_start + choose_data_offset(r10_bio, rdev),
1790	size: dev_end - dev_start);
1791	bio_endio(mbio);
1792	}
1793	if (r10_bio->devs[disk].repl_bio) {
1794	struct md_rdev *rrdev = conf->mirrors[disk].replacement;
1795	rbio = bio_alloc_clone(bdev: bio->bi_bdev, bio_src: bio, GFP_NOIO,
1796	bs: &mddev->bio_set);
1797	rbio->bi_end_io = raid10_end_discard_request;
1798	rbio->bi_private = r10_bio;
1799	r10_bio->devs[disk].repl_bio = rbio;
1800	r10_bio->devs[disk].devnum = disk;
1801	atomic_inc(v: &r10_bio->remaining);
1802	md_submit_discard_bio(mddev, rdev: rrdev, bio: rbio,
1803	start: dev_start + choose_data_offset(r10_bio, rdev: rrdev),
1804	size: dev_end - dev_start);
1805	bio_endio(rbio);
1806	}
1807	}
1808
1809	if (!geo->far_offset && --far_copies) {
1810	first_stripe_index += geo->stride >> geo->chunk_shift;
1811	start_disk_offset += geo->stride;
1812	last_stripe_index += geo->stride >> geo->chunk_shift;
1813	end_disk_offset += geo->stride;
1814	atomic_inc(v: &first_r10bio->remaining);
1815	raid_end_discard_bio(r10bio: r10_bio);
1816	wait_barrier(conf, nowait: false);
1817	goto retry_discard;
1818	}
1819
1820	raid_end_discard_bio(r10bio: r10_bio);
1821
1822	return 0;
1823	out:
1824	allow_barrier(conf);
1825	return -EAGAIN;
1826	}
1827
1828	static bool raid10_make_request(struct mddev *mddev, struct bio *bio)
1829	{
1830	struct r10conf *conf = mddev->private;
1831	sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
1832	int chunk_sects = chunk_mask + 1;
1833	int sectors = bio_sectors(bio);
1834
1835	if (unlikely(bio->bi_opf & REQ_PREFLUSH)
1836	&& md_flush_request(mddev, bio))
1837	return true;
1838
1839	if (!md_write_start(mddev, bi: bio))
1840	return false;
1841
1842	if (unlikely(bio_op(bio) == REQ_OP_DISCARD))
1843	if (!raid10_handle_discard(mddev, bio))
1844	return true;
1845
1846	/*
1847	* If this request crosses a chunk boundary, we need to split
1848	* it.
1849	*/
1850	if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +
1851	sectors > chunk_sects
1852	&& (conf->geo.near_copies < conf->geo.raid_disks
1853	|| conf->prev.near_copies <
1854	conf->prev.raid_disks)))
1855	sectors = chunk_sects -
1856	(bio->bi_iter.bi_sector &
1857	(chunk_sects - 1));
1858	__make_request(mddev, bio, sectors);
1859
1860	/* In case raid10d snuck in to freeze_array */
1861	wake_up_barrier(conf);
1862	return true;
1863	}
1864
1865	static void raid10_status(struct seq_file *seq, struct mddev *mddev)
1866	{
1867	struct r10conf *conf = mddev->private;
1868	int i;
1869
1870	lockdep_assert_held(&mddev->lock);
1871
1872	if (conf->geo.near_copies < conf->geo.raid_disks)
1873	seq_printf(m: seq, fmt: " %dK chunks", mddev->chunk_sectors / 2);
1874	if (conf->geo.near_copies > 1)
1875	seq_printf(m: seq, fmt: " %d near-copies", conf->geo.near_copies);
1876	if (conf->geo.far_copies > 1) {
1877	if (conf->geo.far_offset)
1878	seq_printf(m: seq, fmt: " %d offset-copies", conf->geo.far_copies);
1879	else
1880	seq_printf(m: seq, fmt: " %d far-copies", conf->geo.far_copies);
1881	if (conf->geo.far_set_size != conf->geo.raid_disks)
1882	seq_printf(m: seq, fmt: " %d devices per set", conf->geo.far_set_size);
1883	}
1884	seq_printf(m: seq, fmt: " [%d/%d] [", conf->geo.raid_disks,
1885	conf->geo.raid_disks - mddev->degraded);
1886	for (i = 0; i < conf->geo.raid_disks; i++) {
1887	struct md_rdev *rdev = READ_ONCE(conf->mirrors[i].rdev);
1888
1889	seq_printf(m: seq, fmt: "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
1890	}
1891	seq_printf(m: seq, fmt: "]");
1892	}
1893
1894	/* check if there are enough drives for
1895	* every block to appear on atleast one.
1896	* Don't consider the device numbered 'ignore'
1897	* as we might be about to remove it.
1898	*/
1899	static int _enough(struct r10conf *conf, int previous, int ignore)
1900	{
1901	int first = 0;
1902	int has_enough = 0;
1903	int disks, ncopies;
1904	if (previous) {
1905	disks = conf->prev.raid_disks;
1906	ncopies = conf->prev.near_copies;
1907	} else {
1908	disks = conf->geo.raid_disks;
1909	ncopies = conf->geo.near_copies;
1910	}
1911
1912	do {
1913	int n = conf->copies;
1914	int cnt = 0;
1915	int this = first;
1916	while (n--) {
1917	struct md_rdev *rdev;
1918	if (this != ignore &&
1919	(rdev = conf->mirrors[this].rdev) &&
1920	test_bit(In_sync, &rdev->flags))
1921	cnt++;
1922	this = (this+1) % disks;
1923	}
1924	if (cnt == 0)
1925	goto out;
1926	first = (first + ncopies) % disks;
1927	} while (first != 0);
1928	has_enough = 1;
1929	out:
1930	return has_enough;
1931	}
1932
1933	static int enough(struct r10conf *conf, int ignore)
1934	{
1935	/* when calling 'enough', both 'prev' and 'geo' must
1936	* be stable.
1937	* This is ensured if ->reconfig_mutex or ->device_lock
1938	* is held.
1939	*/
1940	return _enough(conf, previous: 0, ignore) &&
1941	_enough(conf, previous: 1, ignore);
1942	}
1943
1944	/**
1945	* raid10_error() - RAID10 error handler.
1946	* @mddev: affected md device.
1947	* @rdev: member device to fail.
1948	*
1949	* The routine acknowledges &rdev failure and determines new @mddev state.
1950	* If it failed, then:
1951	* - &MD_BROKEN flag is set in &mddev->flags.
1952	* Otherwise, it must be degraded:
1953	* - recovery is interrupted.
1954	* - &mddev->degraded is bumped.
1955	*
1956	* @rdev is marked as &Faulty excluding case when array is failed and
1957	* &mddev->fail_last_dev is off.
1958	*/
1959	static void raid10_error(struct mddev *mddev, struct md_rdev *rdev)
1960	{
1961	struct r10conf *conf = mddev->private;
1962	unsigned long flags;
1963
1964	spin_lock_irqsave(&conf->device_lock, flags);
1965
1966	if (test_bit(In_sync, &rdev->flags) && !enough(conf, ignore: rdev->raid_disk)) {
1967	set_bit(nr: MD_BROKEN, addr: &mddev->flags);
1968
1969	if (!mddev->fail_last_dev) {
1970	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
1971	return;
1972	}
1973	}
1974	if (test_and_clear_bit(nr: In_sync, addr: &rdev->flags))
1975	mddev->degraded++;
1976
1977	set_bit(nr: MD_RECOVERY_INTR, addr: &mddev->recovery);
1978	set_bit(nr: Blocked, addr: &rdev->flags);
1979	set_bit(nr: Faulty, addr: &rdev->flags);
1980	set_mask_bits(&mddev->sb_flags, 0,
1981	BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1982	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
1983	pr_crit("md/raid10:%s: Disk failure on %pg, disabling device.\n"
1984	"md/raid10:%s: Operation continuing on %d devices.\n",
1985	mdname(mddev), rdev->bdev,
1986	mdname(mddev), conf->geo.raid_disks - mddev->degraded);
1987	}
1988
1989	static void print_conf(struct r10conf *conf)
1990	{
1991	int i;
1992	struct md_rdev *rdev;
1993
1994	pr_debug("RAID10 conf printout:\n");
1995	if (!conf) {
1996	pr_debug("(!conf)\n");
1997	return;
1998	}
1999	pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
2000	conf->geo.raid_disks);
2001
2002	lockdep_assert_held(&conf->mddev->reconfig_mutex);
2003	for (i = 0; i < conf->geo.raid_disks; i++) {
2004	rdev = conf->mirrors[i].rdev;
2005	if (rdev)
2006	pr_debug(" disk %d, wo:%d, o:%d, dev:%pg\n",
2007	i, !test_bit(In_sync, &rdev->flags),
2008	!test_bit(Faulty, &rdev->flags),
2009	rdev->bdev);
2010	}
2011	}
2012
2013	static void close_sync(struct r10conf *conf)
2014	{
2015	wait_barrier(conf, nowait: false);
2016	allow_barrier(conf);
2017
2018	mempool_exit(pool: &conf->r10buf_pool);
2019	}
2020
2021	static int raid10_spare_active(struct mddev *mddev)
2022	{
2023	int i;
2024	struct r10conf *conf = mddev->private;
2025	struct raid10_info *tmp;
2026	int count = 0;
2027	unsigned long flags;
2028
2029	/*
2030	* Find all non-in_sync disks within the RAID10 configuration
2031	* and mark them in_sync
2032	*/
2033	for (i = 0; i < conf->geo.raid_disks; i++) {
2034	tmp = conf->mirrors + i;
2035	if (tmp->replacement
2036	&& tmp->replacement->recovery_offset == MaxSector
2037	&& !test_bit(Faulty, &tmp->replacement->flags)
2038	&& !test_and_set_bit(nr: In_sync, addr: &tmp->replacement->flags)) {
2039	/* Replacement has just become active */
2040	if (!tmp->rdev
2041	|| !test_and_clear_bit(nr: In_sync, addr: &tmp->rdev->flags))
2042	count++;
2043	if (tmp->rdev) {
2044	/* Replaced device not technically faulty,
2045	* but we need to be sure it gets removed
2046	* and never re-added.
2047	*/
2048	set_bit(nr: Faulty, addr: &tmp->rdev->flags);
2049	sysfs_notify_dirent_safe(
2050	sd: tmp->rdev->sysfs_state);
2051	}
2052	sysfs_notify_dirent_safe(sd: tmp->replacement->sysfs_state);
2053	} else if (tmp->rdev
2054	&& tmp->rdev->recovery_offset == MaxSector
2055	&& !test_bit(Faulty, &tmp->rdev->flags)
2056	&& !test_and_set_bit(nr: In_sync, addr: &tmp->rdev->flags)) {
2057	count++;
2058	sysfs_notify_dirent_safe(sd: tmp->rdev->sysfs_state);
2059	}
2060	}
2061	spin_lock_irqsave(&conf->device_lock, flags);
2062	mddev->degraded -= count;
2063	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
2064
2065	print_conf(conf);
2066	return count;
2067	}
2068
2069	static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
2070	{
2071	struct r10conf *conf = mddev->private;
2072	int err = -EEXIST;
2073	int mirror, repl_slot = -1;
2074	int first = 0;
2075	int last = conf->geo.raid_disks - 1;
2076	struct raid10_info *p;
2077
2078	if (mddev->recovery_cp < MaxSector)
2079	/* only hot-add to in-sync arrays, as recovery is
2080	* very different from resync
2081	*/
2082	return -EBUSY;
2083	if (rdev->saved_raid_disk < 0 && !_enough(conf, previous: 1, ignore: -1))
2084	return -EINVAL;
2085
2086	if (md_integrity_add_rdev(rdev, mddev))
2087	return -ENXIO;
2088
2089	if (rdev->raid_disk >= 0)
2090	first = last = rdev->raid_disk;
2091
2092	if (rdev->saved_raid_disk >= first &&
2093	rdev->saved_raid_disk < conf->geo.raid_disks &&
2094	conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
2095	mirror = rdev->saved_raid_disk;
2096	else
2097	mirror = first;
2098	for ( ; mirror <= last ; mirror++) {
2099	p = &conf->mirrors[mirror];
2100	if (p->recovery_disabled == mddev->recovery_disabled)
2101	continue;
2102	if (p->rdev) {
2103	if (test_bit(WantReplacement, &p->rdev->flags) &&
2104	p->replacement == NULL && repl_slot < 0)
2105	repl_slot = mirror;
2106	continue;
2107	}
2108
2109	err = mddev_stack_new_rdev(mddev, rdev);
2110	if (err)
2111	return err;
2112	p->head_position = 0;
2113	p->recovery_disabled = mddev->recovery_disabled - 1;
2114	rdev->raid_disk = mirror;
2115	err = 0;
2116	if (rdev->saved_raid_disk != mirror)
2117	conf->fullsync = 1;
2118	WRITE_ONCE(p->rdev, rdev);
2119	break;
2120	}
2121
2122	if (err && repl_slot >= 0) {
2123	p = &conf->mirrors[repl_slot];
2124	clear_bit(nr: In_sync, addr: &rdev->flags);
2125	set_bit(nr: Replacement, addr: &rdev->flags);
2126	rdev->raid_disk = repl_slot;
2127	err = mddev_stack_new_rdev(mddev, rdev);
2128	if (err)
2129	return err;
2130	conf->fullsync = 1;
2131	WRITE_ONCE(p->replacement, rdev);
2132	}
2133
2134	print_conf(conf);
2135	return err;
2136	}
2137
2138	static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
2139	{
2140	struct r10conf *conf = mddev->private;
2141	int err = 0;
2142	int number = rdev->raid_disk;
2143	struct md_rdev **rdevp;
2144	struct raid10_info *p;
2145
2146	print_conf(conf);
2147	if (unlikely(number >= mddev->raid_disks))
2148	return 0;
2149	p = conf->mirrors + number;
2150	if (rdev == p->rdev)
2151	rdevp = &p->rdev;
2152	else if (rdev == p->replacement)
2153	rdevp = &p->replacement;
2154	else
2155	return 0;
2156
2157	if (test_bit(In_sync, &rdev->flags) ||
2158	atomic_read(v: &rdev->nr_pending)) {
2159	err = -EBUSY;
2160	goto abort;
2161	}
2162	/* Only remove non-faulty devices if recovery
2163	* is not possible.
2164	*/
2165	if (!test_bit(Faulty, &rdev->flags) &&
2166	mddev->recovery_disabled != p->recovery_disabled &&
2167	(!p->replacement || p->replacement == rdev) &&
2168	number < conf->geo.raid_disks &&
2169	enough(conf, ignore: -1)) {
2170	err = -EBUSY;
2171	goto abort;
2172	}
2173	WRITE_ONCE(*rdevp, NULL);
2174	if (p->replacement) {
2175	/* We must have just cleared 'rdev' */
2176	WRITE_ONCE(p->rdev, p->replacement);
2177	clear_bit(nr: Replacement, addr: &p->replacement->flags);
2178	WRITE_ONCE(p->replacement, NULL);
2179	}
2180
2181	clear_bit(nr: WantReplacement, addr: &rdev->flags);
2182	err = md_integrity_register(mddev);
2183
2184	abort:
2185
2186	print_conf(conf);
2187	return err;
2188	}
2189
2190	static void __end_sync_read(struct r10bio *r10_bio, struct bio *bio, int d)
2191	{
2192	struct r10conf *conf = r10_bio->mddev->private;
2193
2194	if (!bio->bi_status)
2195	set_bit(nr: R10BIO_Uptodate, addr: &r10_bio->state);
2196	else
2197	/* The write handler will notice the lack of
2198	* R10BIO_Uptodate and record any errors etc
2199	*/
2200	atomic_add(i: r10_bio->sectors,
2201	v: &conf->mirrors[d].rdev->corrected_errors);
2202
2203	/* for reconstruct, we always reschedule after a read.
2204	* for resync, only after all reads
2205	*/
2206	rdev_dec_pending(rdev: conf->mirrors[d].rdev, mddev: conf->mddev);
2207	if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
2208	atomic_dec_and_test(v: &r10_bio->remaining)) {
2209	/* we have read all the blocks,
2210	* do the comparison in process context in raid10d
2211	*/
2212	reschedule_retry(r10_bio);
2213	}
2214	}
2215
2216	static void end_sync_read(struct bio *bio)
2217	{
2218	struct r10bio *r10_bio = get_resync_r10bio(bio);
2219	struct r10conf *conf = r10_bio->mddev->private;
2220	int d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
2221
2222	__end_sync_read(r10_bio, bio, d);
2223	}
2224
2225	static void end_reshape_read(struct bio *bio)
2226	{
2227	/* reshape read bio isn't allocated from r10buf_pool */
2228	struct r10bio *r10_bio = bio->bi_private;
2229
2230	__end_sync_read(r10_bio, bio, d: r10_bio->read_slot);
2231	}
2232
2233	static void end_sync_request(struct r10bio *r10_bio)
2234	{
2235	struct mddev *mddev = r10_bio->mddev;
2236
2237	while (atomic_dec_and_test(v: &r10_bio->remaining)) {
2238	if (r10_bio->master_bio == NULL) {
2239	/* the primary of several recovery bios */
2240	sector_t s = r10_bio->sectors;
2241	if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2242	test_bit(R10BIO_WriteError, &r10_bio->state))
2243	reschedule_retry(r10_bio);
2244	else
2245	put_buf(r10_bio);
2246	md_done_sync(mddev, blocks: s, ok: 1);
2247	break;
2248	} else {
2249	struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
2250	if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2251	test_bit(R10BIO_WriteError, &r10_bio->state))
2252	reschedule_retry(r10_bio);
2253	else
2254	put_buf(r10_bio);
2255	r10_bio = r10_bio2;
2256	}
2257	}
2258	}
2259
2260	static void end_sync_write(struct bio *bio)
2261	{
2262	struct r10bio *r10_bio = get_resync_r10bio(bio);
2263	struct mddev *mddev = r10_bio->mddev;
2264	struct r10conf *conf = mddev->private;
2265	int d;
2266	int slot;
2267	int repl;
2268	struct md_rdev *rdev = NULL;
2269
2270	d = find_bio_disk(conf, r10_bio, bio, slotp: &slot, replp: &repl);
2271	if (repl)
2272	rdev = conf->mirrors[d].replacement;
2273	else
2274	rdev = conf->mirrors[d].rdev;
2275
2276	if (bio->bi_status) {
2277	if (repl)
2278	md_error(mddev, rdev);
2279	else {
2280	set_bit(nr: WriteErrorSeen, addr: &rdev->flags);
2281	if (!test_and_set_bit(nr: WantReplacement, addr: &rdev->flags))
2282	set_bit(nr: MD_RECOVERY_NEEDED,
2283	addr: &rdev->mddev->recovery);
2284	set_bit(nr: R10BIO_WriteError, addr: &r10_bio->state);
2285	}
2286	} else if (rdev_has_badblock(rdev, s: r10_bio->devs[slot].addr,
2287	sectors: r10_bio->sectors)) {
2288	set_bit(nr: R10BIO_MadeGood, addr: &r10_bio->state);
2289	}
2290
2291	rdev_dec_pending(rdev, mddev);
2292
2293	end_sync_request(r10_bio);
2294	}
2295
2296	/*
2297	* Note: sync and recover and handled very differently for raid10
2298	* This code is for resync.
2299	* For resync, we read through virtual addresses and read all blocks.
2300	* If there is any error, we schedule a write. The lowest numbered
2301	* drive is authoritative.
2302	* However requests come for physical address, so we need to map.
2303	* For every physical address there are raid_disks/copies virtual addresses,
2304	* which is always are least one, but is not necessarly an integer.
2305	* This means that a physical address can span multiple chunks, so we may
2306	* have to submit multiple io requests for a single sync request.
2307	*/
2308	/*
2309	* We check if all blocks are in-sync and only write to blocks that
2310	* aren't in sync
2311	*/
2312	static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2313	{
2314	struct r10conf *conf = mddev->private;
2315	int i, first;
2316	struct bio *tbio, *fbio;
2317	int vcnt;
2318	struct page **tpages, **fpages;
2319
2320	atomic_set(v: &r10_bio->remaining, i: 1);
2321
2322	/* find the first device with a block */
2323	for (i=0; i<conf->copies; i++)
2324	if (!r10_bio->devs[i].bio->bi_status)
2325	break;
2326
2327	if (i == conf->copies)
2328	goto done;
2329
2330	first = i;
2331	fbio = r10_bio->devs[i].bio;
2332	fbio->bi_iter.bi_size = r10_bio->sectors << 9;
2333	fbio->bi_iter.bi_idx = 0;
2334	fpages = get_resync_pages(bio: fbio)->pages;
2335
2336	vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
2337	/* now find blocks with errors */
2338	for (i=0 ; i < conf->copies ; i++) {
2339	int j, d;
2340	struct md_rdev *rdev;
2341	struct resync_pages *rp;
2342
2343	tbio = r10_bio->devs[i].bio;
2344
2345	if (tbio->bi_end_io != end_sync_read)
2346	continue;
2347	if (i == first)
2348	continue;
2349
2350	tpages = get_resync_pages(bio: tbio)->pages;
2351	d = r10_bio->devs[i].devnum;
2352	rdev = conf->mirrors[d].rdev;
2353	if (!r10_bio->devs[i].bio->bi_status) {
2354	/* We know that the bi_io_vec layout is the same for
2355	* both 'first' and 'i', so we just compare them.
2356	* All vec entries are PAGE_SIZE;
2357	*/
2358	int sectors = r10_bio->sectors;
2359	for (j = 0; j < vcnt; j++) {
2360	int len = PAGE_SIZE;
2361	if (sectors < (len / 512))
2362	len = sectors * 512;
2363	if (memcmp(page_address(fpages[j]),
2364	page_address(tpages[j]),
2365	size: len))
2366	break;
2367	sectors -= len/512;
2368	}
2369	if (j == vcnt)
2370	continue;
2371	atomic64_add(i: r10_bio->sectors, v: &mddev->resync_mismatches);
2372	if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
2373	/* Don't fix anything. */
2374	continue;
2375	} else if (test_bit(FailFast, &rdev->flags)) {
2376	/* Just give up on this device */
2377	md_error(mddev: rdev->mddev, rdev);
2378	continue;
2379	}
2380	/* Ok, we need to write this bio, either to correct an
2381	* inconsistency or to correct an unreadable block.
2382	* First we need to fixup bv_offset, bv_len and
2383	* bi_vecs, as the read request might have corrupted these
2384	*/
2385	rp = get_resync_pages(bio: tbio);
2386	bio_reset(bio: tbio, bdev: conf->mirrors[d].rdev->bdev, opf: REQ_OP_WRITE);
2387
2388	md_bio_reset_resync_pages(bio: tbio, rp, size: fbio->bi_iter.bi_size);
2389
2390	rp->raid_bio = r10_bio;
2391	tbio->bi_private = rp;
2392	tbio->bi_iter.bi_sector = r10_bio->devs[i].addr;
2393	tbio->bi_end_io = end_sync_write;
2394
2395	bio_copy_data(dst: tbio, src: fbio);
2396
2397	atomic_inc(v: &conf->mirrors[d].rdev->nr_pending);
2398	atomic_inc(v: &r10_bio->remaining);
2399	md_sync_acct(bdev: conf->mirrors[d].rdev->bdev, bio_sectors(tbio));
2400
2401	if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
2402	tbio->bi_opf |= MD_FAILFAST;
2403	tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;
2404	submit_bio_noacct(bio: tbio);
2405	}
2406
2407	/* Now write out to any replacement devices
2408	* that are active
2409	*/
2410	for (i = 0; i < conf->copies; i++) {
2411	int d;
2412
2413	tbio = r10_bio->devs[i].repl_bio;
2414	if (!tbio || !tbio->bi_end_io)
2415	continue;
2416	if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
2417	&& r10_bio->devs[i].bio != fbio)
2418	bio_copy_data(dst: tbio, src: fbio);
2419	d = r10_bio->devs[i].devnum;
2420	atomic_inc(v: &r10_bio->remaining);
2421	md_sync_acct(bdev: conf->mirrors[d].replacement->bdev,
2422	bio_sectors(tbio));
2423	submit_bio_noacct(bio: tbio);
2424	}
2425
2426	done:
2427	if (atomic_dec_and_test(v: &r10_bio->remaining)) {
2428	md_done_sync(mddev, blocks: r10_bio->sectors, ok: 1);
2429	put_buf(r10_bio);
2430	}
2431	}
2432
2433	/*
2434	* Now for the recovery code.
2435	* Recovery happens across physical sectors.
2436	* We recover all non-is_sync drives by finding the virtual address of
2437	* each, and then choose a working drive that also has that virt address.
2438	* There is a separate r10_bio for each non-in_sync drive.
2439	* Only the first two slots are in use. The first for reading,
2440	* The second for writing.
2441	*
2442	*/
2443	static void fix_recovery_read_error(struct r10bio *r10_bio)
2444	{
2445	/* We got a read error during recovery.
2446	* We repeat the read in smaller page-sized sections.
2447	* If a read succeeds, write it to the new device or record
2448	* a bad block if we cannot.
2449	* If a read fails, record a bad block on both old and
2450	* new devices.
2451	*/
2452	struct mddev *mddev = r10_bio->mddev;
2453	struct r10conf *conf = mddev->private;
2454	struct bio *bio = r10_bio->devs[0].bio;
2455	sector_t sect = 0;
2456	int sectors = r10_bio->sectors;
2457	int idx = 0;
2458	int dr = r10_bio->devs[0].devnum;
2459	int dw = r10_bio->devs[1].devnum;
2460	struct page **pages = get_resync_pages(bio)->pages;
2461
2462	while (sectors) {
2463	int s = sectors;
2464	struct md_rdev *rdev;
2465	sector_t addr;
2466	int ok;
2467
2468	if (s > (PAGE_SIZE>>9))
2469	s = PAGE_SIZE >> 9;
2470
2471	rdev = conf->mirrors[dr].rdev;
2472	addr = r10_bio->devs[0].addr + sect,
2473	ok = sync_page_io(rdev,
2474	sector: addr,
2475	size: s << 9,
2476	page: pages[idx],
2477	opf: REQ_OP_READ, metadata_op: false);
2478	if (ok) {
2479	rdev = conf->mirrors[dw].rdev;
2480	addr = r10_bio->devs[1].addr + sect;
2481	ok = sync_page_io(rdev,
2482	sector: addr,
2483	size: s << 9,
2484	page: pages[idx],
2485	opf: REQ_OP_WRITE, metadata_op: false);
2486	if (!ok) {
2487	set_bit(nr: WriteErrorSeen, addr: &rdev->flags);
2488	if (!test_and_set_bit(nr: WantReplacement,
2489	addr: &rdev->flags))
2490	set_bit(nr: MD_RECOVERY_NEEDED,
2491	addr: &rdev->mddev->recovery);
2492	}
2493	}
2494	if (!ok) {
2495	/* We don't worry if we cannot set a bad block -
2496	* it really is bad so there is no loss in not
2497	* recording it yet
2498	*/
2499	rdev_set_badblocks(rdev, s: addr, sectors: s, is_new: 0);
2500
2501	if (rdev != conf->mirrors[dw].rdev) {
2502	/* need bad block on destination too */
2503	struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2504	addr = r10_bio->devs[1].addr + sect;
2505	ok = rdev_set_badblocks(rdev: rdev2, s: addr, sectors: s, is_new: 0);
2506	if (!ok) {
2507	/* just abort the recovery */
2508	pr_notice("md/raid10:%s: recovery aborted due to read error\n",
2509	mdname(mddev));
2510
2511	conf->mirrors[dw].recovery_disabled
2512	= mddev->recovery_disabled;
2513	set_bit(nr: MD_RECOVERY_INTR,
2514	addr: &mddev->recovery);
2515	break;
2516	}
2517	}
2518	}
2519
2520	sectors -= s;
2521	sect += s;
2522	idx++;
2523	}
2524	}
2525
2526	static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2527	{
2528	struct r10conf *conf = mddev->private;
2529	int d;
2530	struct bio *wbio = r10_bio->devs[1].bio;
2531	struct bio *wbio2 = r10_bio->devs[1].repl_bio;
2532
2533	/* Need to test wbio2->bi_end_io before we call
2534	* submit_bio_noacct as if the former is NULL,
2535	* the latter is free to free wbio2.
2536	*/
2537	if (wbio2 && !wbio2->bi_end_io)
2538	wbio2 = NULL;
2539
2540	if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
2541	fix_recovery_read_error(r10_bio);
2542	if (wbio->bi_end_io)
2543	end_sync_request(r10_bio);
2544	if (wbio2)
2545	end_sync_request(r10_bio);
2546	return;
2547	}
2548
2549	/*
2550	* share the pages with the first bio
2551	* and submit the write request
2552	*/
2553	d = r10_bio->devs[1].devnum;
2554	if (wbio->bi_end_io) {
2555	atomic_inc(v: &conf->mirrors[d].rdev->nr_pending);
2556	md_sync_acct(bdev: conf->mirrors[d].rdev->bdev, bio_sectors(wbio));
2557	submit_bio_noacct(bio: wbio);
2558	}
2559	if (wbio2) {
2560	atomic_inc(v: &conf->mirrors[d].replacement->nr_pending);
2561	md_sync_acct(bdev: conf->mirrors[d].replacement->bdev,
2562	bio_sectors(wbio2));
2563	submit_bio_noacct(bio: wbio2);
2564	}
2565	}
2566
2567	static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2568	int sectors, struct page *page, enum req_op op)
2569	{
2570	if (rdev_has_badblock(rdev, s: sector, sectors) &&
2571	(op == REQ_OP_READ || test_bit(WriteErrorSeen, &rdev->flags)))
2572	return -1;
2573	if (sync_page_io(rdev, sector, size: sectors << 9, page, opf: op, metadata_op: false))
2574	/* success */
2575	return 1;
2576	if (op == REQ_OP_WRITE) {
2577	set_bit(nr: WriteErrorSeen, addr: &rdev->flags);
2578	if (!test_and_set_bit(nr: WantReplacement, addr: &rdev->flags))
2579	set_bit(nr: MD_RECOVERY_NEEDED,
2580	addr: &rdev->mddev->recovery);
2581	}
2582	/* need to record an error - either for the block or the device */
2583	if (!rdev_set_badblocks(rdev, s: sector, sectors, is_new: 0))
2584	md_error(mddev: rdev->mddev, rdev);
2585	return 0;
2586	}
2587
2588	/*
2589	* This is a kernel thread which:
2590	*
2591	* 1. Retries failed read operations on working mirrors.
2592	* 2. Updates the raid superblock when problems encounter.
2593	* 3. Performs writes following reads for array synchronising.
2594	*/
2595
2596	static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2597	{
2598	int sect = 0; /* Offset from r10_bio->sector */
2599	int sectors = r10_bio->sectors, slot = r10_bio->read_slot;
2600	struct md_rdev *rdev;
2601	int d = r10_bio->devs[slot].devnum;
2602
2603	/* still own a reference to this rdev, so it cannot
2604	* have been cleared recently.
2605	*/
2606	rdev = conf->mirrors[d].rdev;
2607
2608	if (test_bit(Faulty, &rdev->flags))
2609	/* drive has already been failed, just ignore any
2610	more fix_read_error() attempts */
2611	return;
2612
2613	if (exceed_read_errors(mddev, rdev)) {
2614	r10_bio->devs[slot].bio = IO_BLOCKED;
2615	return;
2616	}
2617
2618	while(sectors) {
2619	int s = sectors;
2620	int sl = slot;
2621	int success = 0;
2622	int start;
2623
2624	if (s > (PAGE_SIZE>>9))
2625	s = PAGE_SIZE >> 9;
2626
2627	do {
2628	d = r10_bio->devs[sl].devnum;
2629	rdev = conf->mirrors[d].rdev;
2630	if (rdev &&
2631	test_bit(In_sync, &rdev->flags) &&
2632	!test_bit(Faulty, &rdev->flags) &&
2633	rdev_has_badblock(rdev,
2634	s: r10_bio->devs[sl].addr + sect,
2635	sectors: s) == 0) {
2636	atomic_inc(v: &rdev->nr_pending);
2637	success = sync_page_io(rdev,
2638	sector: r10_bio->devs[sl].addr +
2639	sect,
2640	size: s<<9,
2641	page: conf->tmppage,
2642	opf: REQ_OP_READ, metadata_op: false);
2643	rdev_dec_pending(rdev, mddev);
2644	if (success)
2645	break;
2646	}
2647	sl++;
2648	if (sl == conf->copies)
2649	sl = 0;
2650	} while (sl != slot);
2651
2652	if (!success) {
2653	/* Cannot read from anywhere, just mark the block
2654	* as bad on the first device to discourage future
2655	* reads.
2656	*/
2657	int dn = r10_bio->devs[slot].devnum;
2658	rdev = conf->mirrors[dn].rdev;
2659
2660	if (!rdev_set_badblocks(
2661	rdev,
2662	s: r10_bio->devs[slot].addr
2663	+ sect,
2664	sectors: s, is_new: 0)) {
2665	md_error(mddev, rdev);
2666	r10_bio->devs[slot].bio
2667	= IO_BLOCKED;
2668	}
2669	break;
2670	}
2671
2672	start = sl;
2673	/* write it back and re-read */
2674	while (sl != slot) {
2675	if (sl==0)
2676	sl = conf->copies;
2677	sl--;
2678	d = r10_bio->devs[sl].devnum;
2679	rdev = conf->mirrors[d].rdev;
2680	if (!rdev ||
2681	test_bit(Faulty, &rdev->flags) ||
2682	!test_bit(In_sync, &rdev->flags))
2683	continue;
2684
2685	atomic_inc(v: &rdev->nr_pending);
2686	if (r10_sync_page_io(rdev,
2687	sector: r10_bio->devs[sl].addr +
2688	sect,
2689	sectors: s, page: conf->tmppage, op: REQ_OP_WRITE)
2690	== 0) {
2691	/* Well, this device is dead */
2692	pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %pg)\n",
2693	mdname(mddev), s,
2694	(unsigned long long)(
2695	sect +
2696	choose_data_offset(r10_bio,
2697	rdev)),
2698	rdev->bdev);
2699	pr_notice("md/raid10:%s: %pg: failing drive\n",
2700	mdname(mddev),
2701	rdev->bdev);
2702	}
2703	rdev_dec_pending(rdev, mddev);
2704	}
2705	sl = start;
2706	while (sl != slot) {
2707	if (sl==0)
2708	sl = conf->copies;
2709	sl--;
2710	d = r10_bio->devs[sl].devnum;
2711	rdev = conf->mirrors[d].rdev;
2712	if (!rdev ||
2713	test_bit(Faulty, &rdev->flags) ||
2714	!test_bit(In_sync, &rdev->flags))
2715	continue;
2716
2717	atomic_inc(v: &rdev->nr_pending);
2718	switch (r10_sync_page_io(rdev,
2719	sector: r10_bio->devs[sl].addr +
2720	sect,
2721	sectors: s, page: conf->tmppage, op: REQ_OP_READ)) {
2722	case 0:
2723	/* Well, this device is dead */
2724	pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %pg)\n",
2725	mdname(mddev), s,
2726	(unsigned long long)(
2727	sect +
2728	choose_data_offset(r10_bio, rdev)),
2729	rdev->bdev);
2730	pr_notice("md/raid10:%s: %pg: failing drive\n",
2731	mdname(mddev),
2732	rdev->bdev);
2733	break;
2734	case 1:
2735	pr_info("md/raid10:%s: read error corrected (%d sectors at %llu on %pg)\n",
2736	mdname(mddev), s,
2737	(unsigned long long)(
2738	sect +
2739	choose_data_offset(r10_bio, rdev)),
2740	rdev->bdev);
2741	atomic_add(i: s, v: &rdev->corrected_errors);
2742	}
2743
2744	rdev_dec_pending(rdev, mddev);
2745	}
2746
2747	sectors -= s;
2748	sect += s;
2749	}
2750	}
2751
2752	static int narrow_write_error(struct r10bio *r10_bio, int i)
2753	{
2754	struct bio *bio = r10_bio->master_bio;
2755	struct mddev *mddev = r10_bio->mddev;
2756	struct r10conf *conf = mddev->private;
2757	struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2758	/* bio has the data to be written to slot 'i' where
2759	* we just recently had a write error.
2760	* We repeatedly clone the bio and trim down to one block,
2761	* then try the write. Where the write fails we record
2762	* a bad block.
2763	* It is conceivable that the bio doesn't exactly align with
2764	* blocks. We must handle this.
2765	*
2766	* We currently own a reference to the rdev.
2767	*/
2768
2769	int block_sectors;
2770	sector_t sector;
2771	int sectors;
2772	int sect_to_write = r10_bio->sectors;
2773	int ok = 1;
2774
2775	if (rdev->badblocks.shift < 0)
2776	return 0;
2777
2778	block_sectors = roundup(1 << rdev->badblocks.shift,
2779	bdev_logical_block_size(rdev->bdev) >> 9);
2780	sector = r10_bio->sector;
2781	sectors = ((r10_bio->sector + block_sectors)
2782	& ~(sector_t)(block_sectors - 1))
2783	- sector;
2784
2785	while (sect_to_write) {
2786	struct bio *wbio;
2787	sector_t wsector;
2788	if (sectors > sect_to_write)
2789	sectors = sect_to_write;
2790	/* Write at 'sector' for 'sectors' */
2791	wbio = bio_alloc_clone(bdev: rdev->bdev, bio_src: bio, GFP_NOIO,
2792	bs: &mddev->bio_set);
2793	bio_trim(bio: wbio, offset: sector - bio->bi_iter.bi_sector, size: sectors);
2794	wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector);
2795	wbio->bi_iter.bi_sector = wsector +
2796	choose_data_offset(r10_bio, rdev);
2797	wbio->bi_opf = REQ_OP_WRITE;
2798
2799	if (submit_bio_wait(bio: wbio) < 0)
2800	/* Failure! */
2801	ok = rdev_set_badblocks(rdev, s: wsector,
2802	sectors, is_new: 0)
2803	&& ok;
2804
2805	bio_put(wbio);
2806	sect_to_write -= sectors;
2807	sector += sectors;
2808	sectors = block_sectors;
2809	}
2810	return ok;
2811	}
2812
2813	static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2814	{
2815	int slot = r10_bio->read_slot;
2816	struct bio *bio;
2817	struct r10conf *conf = mddev->private;
2818	struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2819
2820	/* we got a read error. Maybe the drive is bad. Maybe just
2821	* the block and we can fix it.
2822	* We freeze all other IO, and try reading the block from
2823	* other devices. When we find one, we re-write
2824	* and check it that fixes the read error.
2825	* This is all done synchronously while the array is
2826	* frozen.
2827	*/
2828	bio = r10_bio->devs[slot].bio;
2829	bio_put(bio);
2830	r10_bio->devs[slot].bio = NULL;
2831
2832	if (mddev->ro)
2833	r10_bio->devs[slot].bio = IO_BLOCKED;
2834	else if (!test_bit(FailFast, &rdev->flags)) {
2835	freeze_array(conf, extra: 1);
2836	fix_read_error(conf, mddev, r10_bio);
2837	unfreeze_array(conf);
2838	} else
2839	md_error(mddev, rdev);
2840
2841	rdev_dec_pending(rdev, mddev);
2842	r10_bio->state = 0;
2843	raid10_read_request(mddev, bio: r10_bio->master_bio, r10_bio, io_accounting: false);
2844	/*
2845	* allow_barrier after re-submit to ensure no sync io
2846	* can be issued while regular io pending.
2847	*/
2848	allow_barrier(conf);
2849	}
2850
2851	static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2852	{
2853	/* Some sort of write request has finished and it
2854	* succeeded in writing where we thought there was a
2855	* bad block. So forget the bad block.
2856	* Or possibly if failed and we need to record
2857	* a bad block.
2858	*/
2859	int m;
2860	struct md_rdev *rdev;
2861
2862	if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
2863	test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2864	for (m = 0; m < conf->copies; m++) {
2865	int dev = r10_bio->devs[m].devnum;
2866	rdev = conf->mirrors[dev].rdev;
2867	if (r10_bio->devs[m].bio == NULL ||
2868	r10_bio->devs[m].bio->bi_end_io == NULL)
2869	continue;
2870	if (!r10_bio->devs[m].bio->bi_status) {
2871	rdev_clear_badblocks(
2872	rdev,
2873	s: r10_bio->devs[m].addr,
2874	sectors: r10_bio->sectors, is_new: 0);
2875	} else {
2876	if (!rdev_set_badblocks(
2877	rdev,
2878	s: r10_bio->devs[m].addr,
2879	sectors: r10_bio->sectors, is_new: 0))
2880	md_error(mddev: conf->mddev, rdev);
2881	}
2882	rdev = conf->mirrors[dev].replacement;
2883	if (r10_bio->devs[m].repl_bio == NULL ||
2884	r10_bio->devs[m].repl_bio->bi_end_io == NULL)
2885	continue;
2886
2887	if (!r10_bio->devs[m].repl_bio->bi_status) {
2888	rdev_clear_badblocks(
2889	rdev,
2890	s: r10_bio->devs[m].addr,
2891	sectors: r10_bio->sectors, is_new: 0);
2892	} else {
2893	if (!rdev_set_badblocks(
2894	rdev,
2895	s: r10_bio->devs[m].addr,
2896	sectors: r10_bio->sectors, is_new: 0))
2897	md_error(mddev: conf->mddev, rdev);
2898	}
2899	}
2900	put_buf(r10_bio);
2901	} else {
2902	bool fail = false;
2903	for (m = 0; m < conf->copies; m++) {
2904	int dev = r10_bio->devs[m].devnum;
2905	struct bio *bio = r10_bio->devs[m].bio;
2906	rdev = conf->mirrors[dev].rdev;
2907	if (bio == IO_MADE_GOOD) {
2908	rdev_clear_badblocks(
2909	rdev,
2910	s: r10_bio->devs[m].addr,
2911	sectors: r10_bio->sectors, is_new: 0);
2912	rdev_dec_pending(rdev, mddev: conf->mddev);
2913	} else if (bio != NULL && bio->bi_status) {
2914	fail = true;
2915	if (!narrow_write_error(r10_bio, i: m)) {
2916	md_error(mddev: conf->mddev, rdev);
2917	set_bit(nr: R10BIO_Degraded,
2918	addr: &r10_bio->state);
2919	}
2920	rdev_dec_pending(rdev, mddev: conf->mddev);
2921	}
2922	bio = r10_bio->devs[m].repl_bio;
2923	rdev = conf->mirrors[dev].replacement;
2924	if (rdev && bio == IO_MADE_GOOD) {
2925	rdev_clear_badblocks(
2926	rdev,
2927	s: r10_bio->devs[m].addr,
2928	sectors: r10_bio->sectors, is_new: 0);
2929	rdev_dec_pending(rdev, mddev: conf->mddev);
2930	}
2931	}
2932	if (fail) {
2933	spin_lock_irq(lock: &conf->device_lock);
2934	list_add(new: &r10_bio->retry_list, head: &conf->bio_end_io_list);
2935	conf->nr_queued++;
2936	spin_unlock_irq(lock: &conf->device_lock);
2937	/*
2938	* In case freeze_array() is waiting for condition
2939	* nr_pending == nr_queued + extra to be true.
2940	*/
2941	wake_up(&conf->wait_barrier);
2942	md_wakeup_thread(thread: conf->mddev->thread);
2943	} else {
2944	if (test_bit(R10BIO_WriteError,
2945	&r10_bio->state))
2946	close_write(r10_bio);
2947	raid_end_bio_io(r10_bio);
2948	}
2949	}
2950	}
2951
2952	static void raid10d(struct md_thread *thread)
2953	{
2954	struct mddev *mddev = thread->mddev;
2955	struct r10bio *r10_bio;
2956	unsigned long flags;
2957	struct r10conf *conf = mddev->private;
2958	struct list_head *head = &conf->retry_list;
2959	struct blk_plug plug;
2960
2961	md_check_recovery(mddev);
2962
2963	if (!list_empty_careful(head: &conf->bio_end_io_list) &&
2964	!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2965	LIST_HEAD(tmp);
2966	spin_lock_irqsave(&conf->device_lock, flags);
2967	if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2968	while (!list_empty(head: &conf->bio_end_io_list)) {
2969	list_move(list: conf->bio_end_io_list.prev, head: &tmp);
2970	conf->nr_queued--;
2971	}
2972	}
2973	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
2974	while (!list_empty(head: &tmp)) {
2975	r10_bio = list_first_entry(&tmp, struct r10bio,
2976	retry_list);
2977	list_del(entry: &r10_bio->retry_list);
2978	if (mddev->degraded)
2979	set_bit(nr: R10BIO_Degraded, addr: &r10_bio->state);
2980
2981	if (test_bit(R10BIO_WriteError,
2982	&r10_bio->state))
2983	close_write(r10_bio);
2984	raid_end_bio_io(r10_bio);
2985	}
2986	}
2987
2988	blk_start_plug(&plug);
2989	for (;;) {
2990
2991	flush_pending_writes(conf);
2992
2993	spin_lock_irqsave(&conf->device_lock, flags);
2994	if (list_empty(head)) {
2995	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
2996	break;
2997	}
2998	r10_bio = list_entry(head->prev, struct r10bio, retry_list);
2999	list_del(entry: head->prev);
3000	conf->nr_queued--;
3001	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
3002
3003	mddev = r10_bio->mddev;
3004	conf = mddev->private;
3005	if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
3006	test_bit(R10BIO_WriteError, &r10_bio->state))
3007	handle_write_completed(conf, r10_bio);
3008	else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
3009	reshape_request_write(mddev, r10_bio);
3010	else if (test_bit(R10BIO_IsSync, &r10_bio->state))
3011	sync_request_write(mddev, r10_bio);
3012	else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
3013	recovery_request_write(mddev, r10_bio);
3014	else if (test_bit(R10BIO_ReadError, &r10_bio->state))
3015	handle_read_error(mddev, r10_bio);
3016	else
3017	WARN_ON_ONCE(1);
3018
3019	cond_resched();
3020	if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
3021	md_check_recovery(mddev);
3022	}
3023	blk_finish_plug(&plug);
3024	}
3025
3026	static int init_resync(struct r10conf *conf)
3027	{
3028	int ret, buffs, i;
3029
3030	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
3031	BUG_ON(mempool_initialized(&conf->r10buf_pool));
3032	conf->have_replacement = 0;
3033	for (i = 0; i < conf->geo.raid_disks; i++)
3034	if (conf->mirrors[i].replacement)
3035	conf->have_replacement = 1;
3036	ret = mempool_init(pool: &conf->r10buf_pool, min_nr: buffs,
3037	alloc_fn: r10buf_pool_alloc, free_fn: r10buf_pool_free, pool_data: conf);
3038	if (ret)
3039	return ret;
3040	conf->next_resync = 0;
3041	return 0;
3042	}
3043
3044	static struct r10bio *raid10_alloc_init_r10buf(struct r10conf *conf)
3045	{
3046	struct r10bio *r10bio = mempool_alloc(pool: &conf->r10buf_pool, GFP_NOIO);
3047	struct rsync_pages *rp;
3048	struct bio *bio;
3049	int nalloc;
3050	int i;
3051
3052	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
3053	test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
3054	nalloc = conf->copies; /* resync */
3055	else
3056	nalloc = 2; /* recovery */
3057
3058	for (i = 0; i < nalloc; i++) {
3059	bio = r10bio->devs[i].bio;
3060	rp = bio->bi_private;
3061	bio_reset(bio, NULL, opf: 0);
3062	bio->bi_private = rp;
3063	bio = r10bio->devs[i].repl_bio;
3064	if (bio) {
3065	rp = bio->bi_private;
3066	bio_reset(bio, NULL, opf: 0);
3067	bio->bi_private = rp;
3068	}
3069	}
3070	return r10bio;
3071	}
3072
3073	/*
3074	* Set cluster_sync_high since we need other nodes to add the
3075	* range [cluster_sync_low, cluster_sync_high] to suspend list.
3076	*/
3077	static void raid10_set_cluster_sync_high(struct r10conf *conf)
3078	{
3079	sector_t window_size;
3080	int extra_chunk, chunks;
3081
3082	/*
3083	* First, here we define "stripe" as a unit which across
3084	* all member devices one time, so we get chunks by use
3085	* raid_disks / near_copies. Otherwise, if near_copies is
3086	* close to raid_disks, then resync window could increases
3087	* linearly with the increase of raid_disks, which means
3088	* we will suspend a really large IO window while it is not
3089	* necessary. If raid_disks is not divisible by near_copies,
3090	* an extra chunk is needed to ensure the whole "stripe" is
3091	* covered.
3092	*/
3093
3094	chunks = conf->geo.raid_disks / conf->geo.near_copies;
3095	if (conf->geo.raid_disks % conf->geo.near_copies == 0)
3096	extra_chunk = 0;
3097	else
3098	extra_chunk = 1;
3099	window_size = (chunks + extra_chunk) * conf->mddev->chunk_sectors;
3100
3101	/*
3102	* At least use a 32M window to align with raid1's resync window
3103	*/
3104	window_size = (CLUSTER_RESYNC_WINDOW_SECTORS > window_size) ?
3105	CLUSTER_RESYNC_WINDOW_SECTORS : window_size;
3106
3107	conf->cluster_sync_high = conf->cluster_sync_low + window_size;
3108	}
3109
3110	/*
3111	* perform a "sync" on one "block"
3112	*
3113	* We need to make sure that no normal I/O request - particularly write
3114	* requests - conflict with active sync requests.
3115	*
3116	* This is achieved by tracking pending requests and a 'barrier' concept
3117	* that can be installed to exclude normal IO requests.
3118	*
3119	* Resync and recovery are handled very differently.
3120	* We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
3121	*
3122	* For resync, we iterate over virtual addresses, read all copies,
3123	* and update if there are differences. If only one copy is live,
3124	* skip it.
3125	* For recovery, we iterate over physical addresses, read a good
3126	* value for each non-in_sync drive, and over-write.
3127	*
3128	* So, for recovery we may have several outstanding complex requests for a
3129	* given address, one for each out-of-sync device. We model this by allocating
3130	* a number of r10_bio structures, one for each out-of-sync device.
3131	* As we setup these structures, we collect all bio's together into a list
3132	* which we then process collectively to add pages, and then process again
3133	* to pass to submit_bio_noacct.
3134	*
3135	* The r10_bio structures are linked using a borrowed master_bio pointer.
3136	* This link is counted in ->remaining. When the r10_bio that points to NULL
3137	* has its remaining count decremented to 0, the whole complex operation
3138	* is complete.
3139	*
3140	*/
3141
3142	static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
3143	int *skipped)
3144	{
3145	struct r10conf *conf = mddev->private;
3146	struct r10bio *r10_bio;
3147	struct bio *biolist = NULL, *bio;
3148	sector_t max_sector, nr_sectors;
3149	int i;
3150	int max_sync;
3151	sector_t sync_blocks;
3152	sector_t sectors_skipped = 0;
3153	int chunks_skipped = 0;
3154	sector_t chunk_mask = conf->geo.chunk_mask;
3155	int page_idx = 0;
3156	int error_disk = -1;
3157
3158	/*
3159	* Allow skipping a full rebuild for incremental assembly
3160	* of a clean array, like RAID1 does.
3161	*/
3162	if (mddev->bitmap == NULL &&
3163	mddev->recovery_cp == MaxSector &&
3164	mddev->reshape_position == MaxSector &&
3165	!test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
3166	!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
3167	!test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
3168	conf->fullsync == 0) {
3169	*skipped = 1;
3170	return mddev->dev_sectors - sector_nr;
3171	}
3172
3173	if (!mempool_initialized(pool: &conf->r10buf_pool))
3174	if (init_resync(conf))
3175	return 0;
3176
3177	skipped:
3178	max_sector = mddev->dev_sectors;
3179	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
3180	test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
3181	max_sector = mddev->resync_max_sectors;
3182	if (sector_nr >= max_sector) {
3183	conf->cluster_sync_low = 0;
3184	conf->cluster_sync_high = 0;
3185
3186	/* If we aborted, we need to abort the
3187	* sync on the 'current' bitmap chucks (there can
3188	* be several when recovering multiple devices).
3189	* as we may have started syncing it but not finished.
3190	* We can find the current address in
3191	* mddev->curr_resync, but for recovery,
3192	* we need to convert that to several
3193	* virtual addresses.
3194	*/
3195	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
3196	end_reshape(conf);
3197	close_sync(conf);
3198	return 0;
3199	}
3200
3201	if (mddev->curr_resync < max_sector) { /* aborted */
3202	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
3203	md_bitmap_end_sync(bitmap: mddev->bitmap, offset: mddev->curr_resync,
3204	blocks: &sync_blocks, aborted: 1);
3205	else for (i = 0; i < conf->geo.raid_disks; i++) {
3206	sector_t sect =
3207	raid10_find_virt(conf, sector: mddev->curr_resync, dev: i);
3208	md_bitmap_end_sync(bitmap: mddev->bitmap, offset: sect,
3209	blocks: &sync_blocks, aborted: 1);
3210	}
3211	} else {
3212	/* completed sync */
3213	if ((!mddev->bitmap || conf->fullsync)
3214	&& conf->have_replacement
3215	&& test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3216	/* Completed a full sync so the replacements
3217	* are now fully recovered.
3218	*/
3219	for (i = 0; i < conf->geo.raid_disks; i++) {
3220	struct md_rdev *rdev =
3221	conf->mirrors[i].replacement;
3222
3223	if (rdev)
3224	rdev->recovery_offset = MaxSector;
3225	}
3226	}
3227	conf->fullsync = 0;
3228	}
3229	md_bitmap_close_sync(bitmap: mddev->bitmap);
3230	close_sync(conf);
3231	*skipped = 1;
3232	return sectors_skipped;
3233	}
3234
3235	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
3236	return reshape_request(mddev, sector_nr, skipped);
3237
3238	if (chunks_skipped >= conf->geo.raid_disks) {
3239	pr_err("md/raid10:%s: %s fails\n", mdname(mddev),
3240	test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ? "resync" : "recovery");
3241	if (error_disk >= 0 &&
3242	!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3243	/*
3244	* recovery fails, set mirrors.recovery_disabled,
3245	* device shouldn't be added to there.
3246	*/
3247	conf->mirrors[error_disk].recovery_disabled =
3248	mddev->recovery_disabled;
3249	return 0;
3250	}
3251	/*
3252	* if there has been nothing to do on any drive,
3253	* then there is nothing to do at all.
3254	*/
3255	*skipped = 1;
3256	return (max_sector - sector_nr) + sectors_skipped;
3257	}
3258
3259	if (max_sector > mddev->resync_max)
3260	max_sector = mddev->resync_max; /* Don't do IO beyond here */
3261
3262	/* make sure whole request will fit in a chunk - if chunks
3263	* are meaningful
3264	*/
3265	if (conf->geo.near_copies < conf->geo.raid_disks &&
3266	max_sector > (sector_nr | chunk_mask))
3267	max_sector = (sector_nr | chunk_mask) + 1;
3268
3269	/*
3270	* If there is non-resync activity waiting for a turn, then let it
3271	* though before starting on this new sync request.
3272	*/
3273	if (conf->nr_waiting)
3274	schedule_timeout_uninterruptible(timeout: 1);
3275
3276	/* Again, very different code for resync and recovery.
3277	* Both must result in an r10bio with a list of bios that
3278	* have bi_end_io, bi_sector, bi_bdev set,
3279	* and bi_private set to the r10bio.
3280	* For recovery, we may actually create several r10bios
3281	* with 2 bios in each, that correspond to the bios in the main one.
3282	* In this case, the subordinate r10bios link back through a
3283	* borrowed master_bio pointer, and the counter in the master
3284	* includes a ref from each subordinate.
3285	*/
3286	/* First, we decide what to do and set ->bi_end_io
3287	* To end_sync_read if we want to read, and
3288	* end_sync_write if we will want to write.
3289	*/
3290
3291	max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
3292	if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3293	/* recovery... the complicated one */
3294	int j;
3295	r10_bio = NULL;
3296
3297	for (i = 0 ; i < conf->geo.raid_disks; i++) {
3298	int still_degraded;
3299	struct r10bio *rb2;
3300	sector_t sect;
3301	int must_sync;
3302	int any_working;
3303	struct raid10_info *mirror = &conf->mirrors[i];
3304	struct md_rdev *mrdev, *mreplace;
3305
3306	mrdev = mirror->rdev;
3307	mreplace = mirror->replacement;
3308
3309	if (mrdev && (test_bit(Faulty, &mrdev->flags) ||
3310	test_bit(In_sync, &mrdev->flags)))
3311	mrdev = NULL;
3312	if (mreplace && test_bit(Faulty, &mreplace->flags))
3313	mreplace = NULL;
3314
3315	if (!mrdev && !mreplace)
3316	continue;
3317
3318	still_degraded = 0;
3319	/* want to reconstruct this device */
3320	rb2 = r10_bio;
3321	sect = raid10_find_virt(conf, sector: sector_nr, dev: i);
3322	if (sect >= mddev->resync_max_sectors)
3323	/* last stripe is not complete - don't
3324	* try to recover this sector.
3325	*/
3326	continue;
3327	/* Unless we are doing a full sync, or a replacement
3328	* we only need to recover the block if it is set in
3329	* the bitmap
3330	*/
3331	must_sync = md_bitmap_start_sync(bitmap: mddev->bitmap, offset: sect,
3332	blocks: &sync_blocks, degraded: 1);
3333	if (sync_blocks < max_sync)
3334	max_sync = sync_blocks;
3335	if (!must_sync &&
3336	mreplace == NULL &&
3337	!conf->fullsync) {
3338	/* yep, skip the sync_blocks here, but don't assume
3339	* that there will never be anything to do here
3340	*/
3341	chunks_skipped = -1;
3342	continue;
3343	}
3344	if (mrdev)
3345	atomic_inc(v: &mrdev->nr_pending);
3346	if (mreplace)
3347	atomic_inc(v: &mreplace->nr_pending);
3348
3349	r10_bio = raid10_alloc_init_r10buf(conf);
3350	r10_bio->state = 0;
3351	raise_barrier(conf, force: rb2 != NULL);
3352	atomic_set(v: &r10_bio->remaining, i: 0);
3353
3354	r10_bio->master_bio = (struct bio*)rb2;
3355	if (rb2)
3356	atomic_inc(v: &rb2->remaining);
3357	r10_bio->mddev = mddev;
3358	set_bit(nr: R10BIO_IsRecover, addr: &r10_bio->state);
3359	r10_bio->sector = sect;
3360
3361	raid10_find_phys(conf, r10bio: r10_bio);
3362
3363	/* Need to check if the array will still be
3364	* degraded
3365	*/
3366	for (j = 0; j < conf->geo.raid_disks; j++) {
3367	struct md_rdev *rdev = conf->mirrors[j].rdev;
3368
3369	if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
3370	still_degraded = 1;
3371	break;
3372	}
3373	}
3374
3375	must_sync = md_bitmap_start_sync(bitmap: mddev->bitmap, offset: sect,
3376	blocks: &sync_blocks, degraded: still_degraded);
3377
3378	any_working = 0;
3379	for (j=0; j<conf->copies;j++) {
3380	int k;
3381	int d = r10_bio->devs[j].devnum;
3382	sector_t from_addr, to_addr;
3383	struct md_rdev *rdev = conf->mirrors[d].rdev;
3384	sector_t sector, first_bad;
3385	int bad_sectors;
3386	if (!rdev ||
3387	!test_bit(In_sync, &rdev->flags))
3388	continue;
3389	/* This is where we read from */
3390	any_working = 1;
3391	sector = r10_bio->devs[j].addr;
3392
3393	if (is_badblock(rdev, s: sector, sectors: max_sync,
3394	first_bad: &first_bad, bad_sectors: &bad_sectors)) {
3395	if (first_bad > sector)
3396	max_sync = first_bad - sector;
3397	else {
3398	bad_sectors -= (sector
3399	- first_bad);
3400	if (max_sync > bad_sectors)
3401	max_sync = bad_sectors;
3402	continue;
3403	}
3404	}
3405	bio = r10_bio->devs[0].bio;
3406	bio->bi_next = biolist;
3407	biolist = bio;
3408	bio->bi_end_io = end_sync_read;
3409	bio->bi_opf = REQ_OP_READ;
3410	if (test_bit(FailFast, &rdev->flags))
3411	bio->bi_opf |= MD_FAILFAST;
3412	from_addr = r10_bio->devs[j].addr;
3413	bio->bi_iter.bi_sector = from_addr +
3414	rdev->data_offset;
3415	bio_set_dev(bio, bdev: rdev->bdev);
3416	atomic_inc(v: &rdev->nr_pending);
3417	/* and we write to 'i' (if not in_sync) */
3418
3419	for (k=0; k<conf->copies; k++)
3420	if (r10_bio->devs[k].devnum == i)
3421	break;
3422	BUG_ON(k == conf->copies);
3423	to_addr = r10_bio->devs[k].addr;
3424	r10_bio->devs[0].devnum = d;
3425	r10_bio->devs[0].addr = from_addr;
3426	r10_bio->devs[1].devnum = i;
3427	r10_bio->devs[1].addr = to_addr;
3428
3429	if (mrdev) {
3430	bio = r10_bio->devs[1].bio;
3431	bio->bi_next = biolist;
3432	biolist = bio;
3433	bio->bi_end_io = end_sync_write;
3434	bio->bi_opf = REQ_OP_WRITE;
3435	bio->bi_iter.bi_sector = to_addr
3436	+ mrdev->data_offset;
3437	bio_set_dev(bio, bdev: mrdev->bdev);
3438	atomic_inc(v: &r10_bio->remaining);
3439	} else
3440	r10_bio->devs[1].bio->bi_end_io = NULL;
3441
3442	/* and maybe write to replacement */
3443	bio = r10_bio->devs[1].repl_bio;
3444	if (bio)
3445	bio->bi_end_io = NULL;
3446	/* Note: if replace is not NULL, then bio
3447	* cannot be NULL as r10buf_pool_alloc will
3448	* have allocated it.
3449	*/
3450	if (!mreplace)
3451	break;
3452	bio->bi_next = biolist;
3453	biolist = bio;
3454	bio->bi_end_io = end_sync_write;
3455	bio->bi_opf = REQ_OP_WRITE;
3456	bio->bi_iter.bi_sector = to_addr +
3457	mreplace->data_offset;
3458	bio_set_dev(bio, bdev: mreplace->bdev);
3459	atomic_inc(v: &r10_bio->remaining);
3460	break;
3461	}
3462	if (j == conf->copies) {
3463	/* Cannot recover, so abort the recovery or
3464	* record a bad block */
3465	if (any_working) {
3466	/* problem is that there are bad blocks
3467	* on other device(s)
3468	*/
3469	int k;
3470	for (k = 0; k < conf->copies; k++)
3471	if (r10_bio->devs[k].devnum == i)
3472	break;
3473	if (mrdev && !test_bit(In_sync,
3474	&mrdev->flags)
3475	&& !rdev_set_badblocks(
3476	rdev: mrdev,
3477	s: r10_bio->devs[k].addr,
3478	sectors: max_sync, is_new: 0))
3479	any_working = 0;
3480	if (mreplace &&
3481	!rdev_set_badblocks(
3482	rdev: mreplace,
3483	s: r10_bio->devs[k].addr,
3484	sectors: max_sync, is_new: 0))
3485	any_working = 0;
3486	}
3487	if (!any_working) {
3488	if (!test_and_set_bit(nr: MD_RECOVERY_INTR,
3489	addr: &mddev->recovery))
3490	pr_warn("md/raid10:%s: insufficient working devices for recovery.\n",
3491	mdname(mddev));
3492	mirror->recovery_disabled
3493	= mddev->recovery_disabled;
3494	} else {
3495	error_disk = i;
3496	}
3497	put_buf(r10_bio);
3498	if (rb2)
3499	atomic_dec(v: &rb2->remaining);
3500	r10_bio = rb2;
3501	if (mrdev)
3502	rdev_dec_pending(rdev: mrdev, mddev);
3503	if (mreplace)
3504	rdev_dec_pending(rdev: mreplace, mddev);
3505	break;
3506	}
3507	if (mrdev)
3508	rdev_dec_pending(rdev: mrdev, mddev);
3509	if (mreplace)
3510	rdev_dec_pending(rdev: mreplace, mddev);
3511	if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) {
3512	/* Only want this if there is elsewhere to
3513	* read from. 'j' is currently the first
3514	* readable copy.
3515	*/
3516	int targets = 1;
3517	for (; j < conf->copies; j++) {
3518	int d = r10_bio->devs[j].devnum;
3519	if (conf->mirrors[d].rdev &&
3520	test_bit(In_sync,
3521	&conf->mirrors[d].rdev->flags))
3522	targets++;
3523	}
3524	if (targets == 1)
3525	r10_bio->devs[0].bio->bi_opf
3526	&= ~MD_FAILFAST;
3527	}
3528	}
3529	if (biolist == NULL) {
3530	while (r10_bio) {
3531	struct r10bio *rb2 = r10_bio;
3532	r10_bio = (struct r10bio*) rb2->master_bio;
3533	rb2->master_bio = NULL;
3534	put_buf(r10_bio: rb2);
3535	}
3536	goto giveup;
3537	}
3538	} else {
3539	/* resync. Schedule a read for every block at this virt offset */
3540	int count = 0;
3541
3542	/*
3543	* Since curr_resync_completed could probably not update in
3544	* time, and we will set cluster_sync_low based on it.
3545	* Let's check against "sector_nr + 2 * RESYNC_SECTORS" for
3546	* safety reason, which ensures curr_resync_completed is
3547	* updated in bitmap_cond_end_sync.
3548	*/
3549	md_bitmap_cond_end_sync(bitmap: mddev->bitmap, sector: sector_nr,
3550	force: mddev_is_clustered(mddev) &&
3551	(sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high));
3552
3553	if (!md_bitmap_start_sync(bitmap: mddev->bitmap, offset: sector_nr,
3554	blocks: &sync_blocks, degraded: mddev->degraded) &&
3555	!conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
3556	&mddev->recovery)) {
3557	/* We can skip this block */
3558	*skipped = 1;
3559	return sync_blocks + sectors_skipped;
3560	}
3561	if (sync_blocks < max_sync)
3562	max_sync = sync_blocks;
3563	r10_bio = raid10_alloc_init_r10buf(conf);
3564	r10_bio->state = 0;
3565
3566	r10_bio->mddev = mddev;
3567	atomic_set(v: &r10_bio->remaining, i: 0);
3568	raise_barrier(conf, force: 0);
3569	conf->next_resync = sector_nr;
3570
3571	r10_bio->master_bio = NULL;
3572	r10_bio->sector = sector_nr;
3573	set_bit(nr: R10BIO_IsSync, addr: &r10_bio->state);
3574	raid10_find_phys(conf, r10bio: r10_bio);
3575	r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;
3576
3577	for (i = 0; i < conf->copies; i++) {
3578	int d = r10_bio->devs[i].devnum;
3579	sector_t first_bad, sector;
3580	int bad_sectors;
3581	struct md_rdev *rdev;
3582
3583	if (r10_bio->devs[i].repl_bio)
3584	r10_bio->devs[i].repl_bio->bi_end_io = NULL;
3585
3586	bio = r10_bio->devs[i].bio;
3587	bio->bi_status = BLK_STS_IOERR;
3588	rdev = conf->mirrors[d].rdev;
3589	if (rdev == NULL || test_bit(Faulty, &rdev->flags))
3590	continue;
3591
3592	sector = r10_bio->devs[i].addr;
3593	if (is_badblock(rdev, s: sector, sectors: max_sync,
3594	first_bad: &first_bad, bad_sectors: &bad_sectors)) {
3595	if (first_bad > sector)
3596	max_sync = first_bad - sector;
3597	else {
3598	bad_sectors -= (sector - first_bad);
3599	if (max_sync > bad_sectors)
3600	max_sync = bad_sectors;
3601	continue;
3602	}
3603	}
3604	atomic_inc(v: &rdev->nr_pending);
3605	atomic_inc(v: &r10_bio->remaining);
3606	bio->bi_next = biolist;
3607	biolist = bio;
3608	bio->bi_end_io = end_sync_read;
3609	bio->bi_opf = REQ_OP_READ;
3610	if (test_bit(FailFast, &rdev->flags))
3611	bio->bi_opf |= MD_FAILFAST;
3612	bio->bi_iter.bi_sector = sector + rdev->data_offset;
3613	bio_set_dev(bio, bdev: rdev->bdev);
3614	count++;
3615
3616	rdev = conf->mirrors[d].replacement;
3617	if (rdev == NULL || test_bit(Faulty, &rdev->flags))
3618	continue;
3619
3620	atomic_inc(v: &rdev->nr_pending);
3621
3622	/* Need to set up for writing to the replacement */
3623	bio = r10_bio->devs[i].repl_bio;
3624	bio->bi_status = BLK_STS_IOERR;
3625
3626	sector = r10_bio->devs[i].addr;
3627	bio->bi_next = biolist;
3628	biolist = bio;
3629	bio->bi_end_io = end_sync_write;
3630	bio->bi_opf = REQ_OP_WRITE;
3631	if (test_bit(FailFast, &rdev->flags))
3632	bio->bi_opf |= MD_FAILFAST;
3633	bio->bi_iter.bi_sector = sector + rdev->data_offset;
3634	bio_set_dev(bio, bdev: rdev->bdev);
3635	count++;
3636	}
3637
3638	if (count < 2) {
3639	for (i=0; i<conf->copies; i++) {
3640	int d = r10_bio->devs[i].devnum;
3641	if (r10_bio->devs[i].bio->bi_end_io)
3642	rdev_dec_pending(rdev: conf->mirrors[d].rdev,
3643	mddev);
3644	if (r10_bio->devs[i].repl_bio &&
3645	r10_bio->devs[i].repl_bio->bi_end_io)
3646	rdev_dec_pending(
3647	rdev: conf->mirrors[d].replacement,
3648	mddev);
3649	}
3650	put_buf(r10_bio);
3651	biolist = NULL;
3652	goto giveup;
3653	}
3654	}
3655
3656	nr_sectors = 0;
3657	if (sector_nr + max_sync < max_sector)
3658	max_sector = sector_nr + max_sync;
3659	do {
3660	struct page *page;
3661	int len = PAGE_SIZE;
3662	if (sector_nr + (len>>9) > max_sector)
3663	len = (max_sector - sector_nr) << 9;
3664	if (len == 0)
3665	break;
3666	for (bio= biolist ; bio ; bio=bio->bi_next) {
3667	struct resync_pages *rp = get_resync_pages(bio);
3668	page = resync_fetch_page(rp, idx: page_idx);
3669	if (WARN_ON(!bio_add_page(bio, page, len, 0))) {
3670	bio->bi_status = BLK_STS_RESOURCE;
3671	bio_endio(bio);
3672	goto giveup;
3673	}
3674	}
3675	nr_sectors += len>>9;
3676	sector_nr += len>>9;
3677	} while (++page_idx < RESYNC_PAGES);
3678	r10_bio->sectors = nr_sectors;
3679
3680	if (mddev_is_clustered(mddev) &&
3681	test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3682	/* It is resync not recovery */
3683	if (conf->cluster_sync_high < sector_nr + nr_sectors) {
3684	conf->cluster_sync_low = mddev->curr_resync_completed;
3685	raid10_set_cluster_sync_high(conf);
3686	/* Send resync message */
3687	md_cluster_ops->resync_info_update(mddev,
3688	conf->cluster_sync_low,
3689	conf->cluster_sync_high);
3690	}
3691	} else if (mddev_is_clustered(mddev)) {
3692	/* This is recovery not resync */
3693	sector_t sect_va1, sect_va2;
3694	bool broadcast_msg = false;
3695
3696	for (i = 0; i < conf->geo.raid_disks; i++) {
3697	/*
3698	* sector_nr is a device address for recovery, so we
3699	* need translate it to array address before compare
3700	* with cluster_sync_high.
3701	*/
3702	sect_va1 = raid10_find_virt(conf, sector: sector_nr, dev: i);
3703
3704	if (conf->cluster_sync_high < sect_va1 + nr_sectors) {
3705	broadcast_msg = true;
3706	/*
3707	* curr_resync_completed is similar as
3708	* sector_nr, so make the translation too.
3709	*/
3710	sect_va2 = raid10_find_virt(conf,
3711	sector: mddev->curr_resync_completed, dev: i);
3712
3713	if (conf->cluster_sync_low == 0 ||
3714	conf->cluster_sync_low > sect_va2)
3715	conf->cluster_sync_low = sect_va2;
3716	}
3717	}
3718	if (broadcast_msg) {
3719	raid10_set_cluster_sync_high(conf);
3720	md_cluster_ops->resync_info_update(mddev,
3721	conf->cluster_sync_low,
3722	conf->cluster_sync_high);
3723	}
3724	}
3725
3726	while (biolist) {
3727	bio = biolist;
3728	biolist = biolist->bi_next;
3729
3730	bio->bi_next = NULL;
3731	r10_bio = get_resync_r10bio(bio);
3732	r10_bio->sectors = nr_sectors;
3733
3734	if (bio->bi_end_io == end_sync_read) {
3735	md_sync_acct_bio(bio, nr_sectors);
3736	bio->bi_status = 0;
3737	submit_bio_noacct(bio);
3738	}
3739	}
3740
3741	if (sectors_skipped)
3742	/* pretend they weren't skipped, it makes
3743	* no important difference in this case
3744	*/
3745	md_done_sync(mddev, blocks: sectors_skipped, ok: 1);
3746
3747	return sectors_skipped + nr_sectors;
3748	giveup:
3749	/* There is nowhere to write, so all non-sync
3750	* drives must be failed or in resync, all drives
3751	* have a bad block, so try the next chunk...
3752	*/
3753	if (sector_nr + max_sync < max_sector)
3754	max_sector = sector_nr + max_sync;
3755
3756	sectors_skipped += (max_sector - sector_nr);
3757	chunks_skipped ++;
3758	sector_nr = max_sector;
3759	goto skipped;
3760	}
3761
3762	static sector_t
3763	raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks)
3764	{
3765	sector_t size;
3766	struct r10conf *conf = mddev->private;
3767
3768	if (!raid_disks)
3769	raid_disks = min(conf->geo.raid_disks,
3770	conf->prev.raid_disks);
3771	if (!sectors)
3772	sectors = conf->dev_sectors;
3773
3774	size = sectors >> conf->geo.chunk_shift;
3775	sector_div(size, conf->geo.far_copies);
3776	size = size * raid_disks;
3777	sector_div(size, conf->geo.near_copies);
3778
3779	return size << conf->geo.chunk_shift;
3780	}
3781
3782	static void calc_sectors(struct r10conf *conf, sector_t size)
3783	{
3784	/* Calculate the number of sectors-per-device that will
3785	* actually be used, and set conf->dev_sectors and
3786	* conf->stride
3787	*/
3788
3789	size = size >> conf->geo.chunk_shift;
3790	sector_div(size, conf->geo.far_copies);
3791	size = size * conf->geo.raid_disks;
3792	sector_div(size, conf->geo.near_copies);
3793	/* 'size' is now the number of chunks in the array */
3794	/* calculate "used chunks per device" */
3795	size = size * conf->copies;
3796
3797	/* We need to round up when dividing by raid_disks to
3798	* get the stride size.
3799	*/
3800	size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);
3801
3802	conf->dev_sectors = size << conf->geo.chunk_shift;
3803
3804	if (conf->geo.far_offset)
3805	conf->geo.stride = 1 << conf->geo.chunk_shift;
3806	else {
3807	sector_div(size, conf->geo.far_copies);
3808	conf->geo.stride = size << conf->geo.chunk_shift;
3809	}
3810	}
3811
3812	enum geo_type {geo_new, geo_old, geo_start};
3813	static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
3814	{
3815	int nc, fc, fo;
3816	int layout, chunk, disks;
3817	switch (new) {
3818	case geo_old:
3819	layout = mddev->layout;
3820	chunk = mddev->chunk_sectors;
3821	disks = mddev->raid_disks - mddev->delta_disks;
3822	break;
3823	case geo_new:
3824	layout = mddev->new_layout;
3825	chunk = mddev->new_chunk_sectors;
3826	disks = mddev->raid_disks;
3827	break;
3828	default: /* avoid 'may be unused' warnings */
3829	case geo_start: /* new when starting reshape - raid_disks not
3830	* updated yet. */
3831	layout = mddev->new_layout;
3832	chunk = mddev->new_chunk_sectors;
3833	disks = mddev->raid_disks + mddev->delta_disks;
3834	break;
3835	}
3836	if (layout >> 19)
3837	return -1;
3838	if (chunk < (PAGE_SIZE >> 9) ||
3839	!is_power_of_2(n: chunk))
3840	return -2;
3841	nc = layout & 255;
3842	fc = (layout >> 8) & 255;
3843	fo = layout & (1<<16);
3844	geo->raid_disks = disks;
3845	geo->near_copies = nc;
3846	geo->far_copies = fc;
3847	geo->far_offset = fo;
3848	switch (layout >> 17) {
3849	case 0: /* original layout. simple but not always optimal */
3850	geo->far_set_size = disks;
3851	break;
3852	case 1: /* "improved" layout which was buggy. Hopefully no-one is
3853	* actually using this, but leave code here just in case.*/
3854	geo->far_set_size = disks/fc;
3855	WARN(geo->far_set_size < fc,
3856	"This RAID10 layout does not provide data safety - please backup and create new array\n");
3857	break;
3858	case 2: /* "improved" layout fixed to match documentation */
3859	geo->far_set_size = fc * nc;
3860	break;
3861	default: /* Not a valid layout */
3862	return -1;
3863	}
3864	geo->chunk_mask = chunk - 1;
3865	geo->chunk_shift = ffz(~chunk);
3866	return nc*fc;
3867	}
3868
3869	static void raid10_free_conf(struct r10conf *conf)
3870	{
3871	if (!conf)
3872	return;
3873
3874	mempool_exit(pool: &conf->r10bio_pool);
3875	kfree(objp: conf->mirrors);
3876	kfree(objp: conf->mirrors_old);
3877	kfree(objp: conf->mirrors_new);
3878	safe_put_page(p: conf->tmppage);
3879	bioset_exit(&conf->bio_split);
3880	kfree(objp: conf);
3881	}
3882
3883	static struct r10conf *setup_conf(struct mddev *mddev)
3884	{
3885	struct r10conf *conf = NULL;
3886	int err = -EINVAL;
3887	struct geom geo;
3888	int copies;
3889
3890	copies = setup_geo(geo: &geo, mddev, new: geo_new);
3891
3892	if (copies == -2) {
3893	pr_warn("md/raid10:%s: chunk size must be at least PAGE_SIZE(%ld) and be a power of 2.\n",
3894	mdname(mddev), PAGE_SIZE);
3895	goto out;
3896	}
3897
3898	if (copies < 2 || copies > mddev->raid_disks) {
3899	pr_warn("md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
3900	mdname(mddev), mddev->new_layout);
3901	goto out;
3902	}
3903
3904	err = -ENOMEM;
3905	conf = kzalloc(size: sizeof(struct r10conf), GFP_KERNEL);
3906	if (!conf)
3907	goto out;
3908
3909	/* FIXME calc properly */
3910	conf->mirrors = kcalloc(n: mddev->raid_disks + max(0, -mddev->delta_disks),
3911	size: sizeof(struct raid10_info),
3912	GFP_KERNEL);
3913	if (!conf->mirrors)
3914	goto out;
3915
3916	conf->tmppage = alloc_page(GFP_KERNEL);
3917	if (!conf->tmppage)
3918	goto out;
3919
3920	conf->geo = geo;
3921	conf->copies = copies;
3922	err = mempool_init(pool: &conf->r10bio_pool, NR_RAID_BIOS, alloc_fn: r10bio_pool_alloc,
3923	free_fn: rbio_pool_free, pool_data: conf);
3924	if (err)
3925	goto out;
3926
3927	err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, flags: 0);
3928	if (err)
3929	goto out;
3930
3931	calc_sectors(conf, size: mddev->dev_sectors);
3932	if (mddev->reshape_position == MaxSector) {
3933	conf->prev = conf->geo;
3934	conf->reshape_progress = MaxSector;
3935	} else {
3936	if (setup_geo(geo: &conf->prev, mddev, new: geo_old) != conf->copies) {
3937	err = -EINVAL;
3938	goto out;
3939	}
3940	conf->reshape_progress = mddev->reshape_position;
3941	if (conf->prev.far_offset)
3942	conf->prev.stride = 1 << conf->prev.chunk_shift;
3943	else
3944	/* far_copies must be 1 */
3945	conf->prev.stride = conf->dev_sectors;
3946	}
3947	conf->reshape_safe = conf->reshape_progress;
3948	spin_lock_init(&conf->device_lock);
3949	INIT_LIST_HEAD(list: &conf->retry_list);
3950	INIT_LIST_HEAD(list: &conf->bio_end_io_list);
3951
3952	seqlock_init(&conf->resync_lock);
3953	init_waitqueue_head(&conf->wait_barrier);
3954	atomic_set(v: &conf->nr_pending, i: 0);
3955
3956	err = -ENOMEM;
3957	rcu_assign_pointer(conf->thread,
3958	md_register_thread(raid10d, mddev, "raid10"));
3959	if (!conf->thread)
3960	goto out;
3961
3962	conf->mddev = mddev;
3963	return conf;
3964
3965	out:
3966	raid10_free_conf(conf);
3967	return ERR_PTR(error: err);
3968	}
3969
3970	static unsigned int raid10_nr_stripes(struct r10conf *conf)
3971	{
3972	unsigned int raid_disks = conf->geo.raid_disks;
3973
3974	if (conf->geo.raid_disks % conf->geo.near_copies)
3975	return raid_disks;
3976	return raid_disks / conf->geo.near_copies;
3977	}
3978
3979	static int raid10_set_queue_limits(struct mddev *mddev)
3980	{
3981	struct r10conf *conf = mddev->private;
3982	struct queue_limits lim;
3983
3984	blk_set_stacking_limits(lim: &lim);
3985	lim.max_write_zeroes_sectors = 0;
3986	lim.io_min = mddev->chunk_sectors << 9;
3987	lim.io_opt = lim.io_min * raid10_nr_stripes(conf);
3988	mddev_stack_rdev_limits(mddev, lim: &lim);
3989	return queue_limits_set(q: mddev->gendisk->queue, lim: &lim);
3990	}
3991
3992	static int raid10_run(struct mddev *mddev)
3993	{
3994	struct r10conf *conf;
3995	int i, disk_idx;
3996	struct raid10_info *disk;
3997	struct md_rdev *rdev;
3998	sector_t size;
3999	sector_t min_offset_diff = 0;
4000	int first = 1;
4001	int ret = -EIO;
4002
4003	if (mddev->private == NULL) {
4004	conf = setup_conf(mddev);
4005	if (IS_ERR(ptr: conf))
4006	return PTR_ERR(ptr: conf);
4007	mddev->private = conf;
4008	}
4009	conf = mddev->private;
4010	if (!conf)
4011	goto out;
4012
4013	rcu_assign_pointer(mddev->thread, conf->thread);
4014	rcu_assign_pointer(conf->thread, NULL);
4015
4016	if (mddev_is_clustered(mddev: conf->mddev)) {
4017	int fc, fo;
4018
4019	fc = (mddev->layout >> 8) & 255;
4020	fo = mddev->layout & (1<<16);
4021	if (fc > 1 || fo > 0) {
4022	pr_err("only near layout is supported by clustered"
4023	" raid10\n");
4024	goto out_free_conf;
4025	}
4026	}
4027
4028	rdev_for_each(rdev, mddev) {
4029	long long diff;
4030
4031	disk_idx = rdev->raid_disk;
4032	if (disk_idx < 0)
4033	continue;
4034	if (disk_idx >= conf->geo.raid_disks &&
4035	disk_idx >= conf->prev.raid_disks)
4036	continue;
4037	disk = conf->mirrors + disk_idx;
4038
4039	if (test_bit(Replacement, &rdev->flags)) {
4040	if (disk->replacement)
4041	goto out_free_conf;
4042	disk->replacement = rdev;
4043	} else {
4044	if (disk->rdev)
4045	goto out_free_conf;
4046	disk->rdev = rdev;
4047	}
4048	diff = (rdev->new_data_offset - rdev->data_offset);
4049	if (!mddev->reshape_backwards)
4050	diff = -diff;
4051	if (diff < 0)
4052	diff = 0;
4053	if (first || diff < min_offset_diff)
4054	min_offset_diff = diff;
4055
4056	disk->head_position = 0;
4057	first = 0;
4058	}
4059
4060	if (!mddev_is_dm(mddev: conf->mddev)) {
4061	ret = raid10_set_queue_limits(mddev);
4062	if (ret)
4063	goto out_free_conf;
4064	}
4065
4066	/* need to check that every block has at least one working mirror */
4067	if (!enough(conf, ignore: -1)) {
4068	pr_err("md/raid10:%s: not enough operational mirrors.\n",
4069	mdname(mddev));
4070	goto out_free_conf;
4071	}
4072
4073	if (conf->reshape_progress != MaxSector) {
4074	/* must ensure that shape change is supported */
4075	if (conf->geo.far_copies != 1 &&
4076	conf->geo.far_offset == 0)
4077	goto out_free_conf;
4078	if (conf->prev.far_copies != 1 &&
4079	conf->prev.far_offset == 0)
4080	goto out_free_conf;
4081	}
4082
4083	mddev->degraded = 0;
4084	for (i = 0;
4085	i < conf->geo.raid_disks
4086	|| i < conf->prev.raid_disks;
4087	i++) {
4088
4089	disk = conf->mirrors + i;
4090
4091	if (!disk->rdev && disk->replacement) {
4092	/* The replacement is all we have - use it */
4093	disk->rdev = disk->replacement;
4094	disk->replacement = NULL;
4095	clear_bit(nr: Replacement, addr: &disk->rdev->flags);
4096	}
4097
4098	if (!disk->rdev ||
4099	!test_bit(In_sync, &disk->rdev->flags)) {
4100	disk->head_position = 0;
4101	mddev->degraded++;
4102	if (disk->rdev &&
4103	disk->rdev->saved_raid_disk < 0)
4104	conf->fullsync = 1;
4105	}
4106
4107	if (disk->replacement &&
4108	!test_bit(In_sync, &disk->replacement->flags) &&
4109	disk->replacement->saved_raid_disk < 0) {
4110	conf->fullsync = 1;
4111	}
4112
4113	disk->recovery_disabled = mddev->recovery_disabled - 1;
4114	}
4115
4116	if (mddev->recovery_cp != MaxSector)
4117	pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n",
4118	mdname(mddev));
4119	pr_info("md/raid10:%s: active with %d out of %d devices\n",
4120	mdname(mddev), conf->geo.raid_disks - mddev->degraded,
4121	conf->geo.raid_disks);
4122	/*
4123	* Ok, everything is just fine now
4124	*/
4125	mddev->dev_sectors = conf->dev_sectors;
4126	size = raid10_size(mddev, sectors: 0, raid_disks: 0);
4127	md_set_array_sectors(mddev, array_sectors: size);
4128	mddev->resync_max_sectors = size;
4129	set_bit(nr: MD_FAILFAST_SUPPORTED, addr: &mddev->flags);
4130
4131	if (md_integrity_register(mddev))
4132	goto out_free_conf;
4133
4134	if (conf->reshape_progress != MaxSector) {
4135	unsigned long before_length, after_length;
4136
4137	before_length = ((1 << conf->prev.chunk_shift) *
4138	conf->prev.far_copies);
4139	after_length = ((1 << conf->geo.chunk_shift) *
4140	conf->geo.far_copies);
4141
4142	if (max(before_length, after_length) > min_offset_diff) {
4143	/* This cannot work */
4144	pr_warn("md/raid10: offset difference not enough to continue reshape\n");
4145	goto out_free_conf;
4146	}
4147	conf->offset_diff = min_offset_diff;
4148
4149	clear_bit(nr: MD_RECOVERY_SYNC, addr: &mddev->recovery);
4150	clear_bit(nr: MD_RECOVERY_CHECK, addr: &mddev->recovery);
4151	set_bit(nr: MD_RECOVERY_RESHAPE, addr: &mddev->recovery);
4152	set_bit(nr: MD_RECOVERY_NEEDED, addr: &mddev->recovery);
4153	}
4154
4155	return 0;
4156
4157	out_free_conf:
4158	md_unregister_thread(mddev, threadp: &mddev->thread);
4159	raid10_free_conf(conf);
4160	mddev->private = NULL;
4161	out:
4162	return ret;
4163	}
4164
4165	static void raid10_free(struct mddev *mddev, void *priv)
4166	{
4167	raid10_free_conf(conf: priv);
4168	}
4169
4170	static void raid10_quiesce(struct mddev *mddev, int quiesce)
4171	{
4172	struct r10conf *conf = mddev->private;
4173
4174	if (quiesce)
4175	raise_barrier(conf, force: 0);
4176	else
4177	lower_barrier(conf);
4178	}
4179
4180	static int raid10_resize(struct mddev *mddev, sector_t sectors)
4181	{
4182	/* Resize of 'far' arrays is not supported.
4183	* For 'near' and 'offset' arrays we can set the
4184	* number of sectors used to be an appropriate multiple
4185	* of the chunk size.
4186	* For 'offset', this is far_copies*chunksize.
4187	* For 'near' the multiplier is the LCM of
4188	* near_copies and raid_disks.
4189	* So if far_copies > 1 && !far_offset, fail.
4190	* Else find LCM(raid_disks, near_copy)*far_copies and
4191	* multiply by chunk_size. Then round to this number.
4192	* This is mostly done by raid10_size()
4193	*/
4194	struct r10conf *conf = mddev->private;
4195	sector_t oldsize, size;
4196
4197	if (mddev->reshape_position != MaxSector)
4198	return -EBUSY;
4199
4200	if (conf->geo.far_copies > 1 && !conf->geo.far_offset)
4201	return -EINVAL;
4202
4203	oldsize = raid10_size(mddev, sectors: 0, raid_disks: 0);
4204	size = raid10_size(mddev, sectors, raid_disks: 0);
4205	if (mddev->external_size &&
4206	mddev->array_sectors > size)
4207	return -EINVAL;
4208	if (mddev->bitmap) {
4209	int ret = md_bitmap_resize(bitmap: mddev->bitmap, blocks: size, chunksize: 0, init: 0);
4210	if (ret)
4211	return ret;
4212	}
4213	md_set_array_sectors(mddev, array_sectors: size);
4214	if (sectors > mddev->dev_sectors &&
4215	mddev->recovery_cp > oldsize) {
4216	mddev->recovery_cp = oldsize;
4217	set_bit(nr: MD_RECOVERY_NEEDED, addr: &mddev->recovery);
4218	}
4219	calc_sectors(conf, size: sectors);
4220	mddev->dev_sectors = conf->dev_sectors;
4221	mddev->resync_max_sectors = size;
4222	return 0;
4223	}
4224
4225	static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs)
4226	{
4227	struct md_rdev *rdev;
4228	struct r10conf *conf;
4229
4230	if (mddev->degraded > 0) {
4231	pr_warn("md/raid10:%s: Error: degraded raid0!\n",
4232	mdname(mddev));
4233	return ERR_PTR(error: -EINVAL);
4234	}
4235	sector_div(size, devs);
4236
4237	/* Set new parameters */
4238	mddev->new_level = 10;
4239	/* new layout: far_copies = 1, near_copies = 2 */
4240	mddev->new_layout = (1<<8) + 2;
4241	mddev->new_chunk_sectors = mddev->chunk_sectors;
4242	mddev->delta_disks = mddev->raid_disks;
4243	mddev->raid_disks *= 2;
4244	/* make sure it will be not marked as dirty */
4245	mddev->recovery_cp = MaxSector;
4246	mddev->dev_sectors = size;
4247
4248	conf = setup_conf(mddev);
4249	if (!IS_ERR(ptr: conf)) {
4250	rdev_for_each(rdev, mddev)
4251	if (rdev->raid_disk >= 0) {
4252	rdev->new_raid_disk = rdev->raid_disk * 2;
4253	rdev->sectors = size;
4254	}
4255	}
4256
4257	return conf;
4258	}
4259
4260	static void *raid10_takeover(struct mddev *mddev)
4261	{
4262	struct r0conf *raid0_conf;
4263
4264	/* raid10 can take over:
4265	* raid0 - providing it has only two drives
4266	*/
4267	if (mddev->level == 0) {
4268	/* for raid0 takeover only one zone is supported */
4269	raid0_conf = mddev->private;
4270	if (raid0_conf->nr_strip_zones > 1) {
4271	pr_warn("md/raid10:%s: cannot takeover raid 0 with more than one zone.\n",
4272	mdname(mddev));
4273	return ERR_PTR(error: -EINVAL);
4274	}
4275	return raid10_takeover_raid0(mddev,
4276	size: raid0_conf->strip_zone->zone_end,
4277	devs: raid0_conf->strip_zone->nb_dev);
4278	}
4279	return ERR_PTR(error: -EINVAL);
4280	}
4281
4282	static int raid10_check_reshape(struct mddev *mddev)
4283	{
4284	/* Called when there is a request to change
4285	* - layout (to ->new_layout)
4286	* - chunk size (to ->new_chunk_sectors)
4287	* - raid_disks (by delta_disks)
4288	* or when trying to restart a reshape that was ongoing.
4289	*
4290	* We need to validate the request and possibly allocate
4291	* space if that might be an issue later.
4292	*
4293	* Currently we reject any reshape of a 'far' mode array,
4294	* allow chunk size to change if new is generally acceptable,
4295	* allow raid_disks to increase, and allow
4296	* a switch between 'near' mode and 'offset' mode.
4297	*/
4298	struct r10conf *conf = mddev->private;
4299	struct geom geo;
4300
4301	if (conf->geo.far_copies != 1 && !conf->geo.far_offset)
4302	return -EINVAL;
4303
4304	if (setup_geo(geo: &geo, mddev, new: geo_start) != conf->copies)
4305	/* mustn't change number of copies */
4306	return -EINVAL;
4307	if (geo.far_copies > 1 && !geo.far_offset)
4308	/* Cannot switch to 'far' mode */
4309	return -EINVAL;
4310
4311	if (mddev->array_sectors & geo.chunk_mask)
4312	/* not factor of array size */
4313	return -EINVAL;
4314
4315	if (!enough(conf, ignore: -1))
4316	return -EINVAL;
4317
4318	kfree(objp: conf->mirrors_new);
4319	conf->mirrors_new = NULL;
4320	if (mddev->delta_disks > 0) {
4321	/* allocate new 'mirrors' list */
4322	conf->mirrors_new =
4323	kcalloc(n: mddev->raid_disks + mddev->delta_disks,
4324	size: sizeof(struct raid10_info),
4325	GFP_KERNEL);
4326	if (!conf->mirrors_new)
4327	return -ENOMEM;
4328	}
4329	return 0;
4330	}
4331
4332	/*
4333	* Need to check if array has failed when deciding whether to:
4334	* - start an array
4335	* - remove non-faulty devices
4336	* - add a spare
4337	* - allow a reshape
4338	* This determination is simple when no reshape is happening.
4339	* However if there is a reshape, we need to carefully check
4340	* both the before and after sections.
4341	* This is because some failed devices may only affect one
4342	* of the two sections, and some non-in_sync devices may
4343	* be insync in the section most affected by failed devices.
4344	*/
4345	static int calc_degraded(struct r10conf *conf)
4346	{
4347	int degraded, degraded2;
4348	int i;
4349
4350	degraded = 0;
4351	/* 'prev' section first */
4352	for (i = 0; i < conf->prev.raid_disks; i++) {
4353	struct md_rdev *rdev = conf->mirrors[i].rdev;
4354
4355	if (!rdev || test_bit(Faulty, &rdev->flags))
4356	degraded++;
4357	else if (!test_bit(In_sync, &rdev->flags))
4358	/* When we can reduce the number of devices in
4359	* an array, this might not contribute to
4360	* 'degraded'. It does now.
4361	*/
4362	degraded++;
4363	}
4364	if (conf->geo.raid_disks == conf->prev.raid_disks)
4365	return degraded;
4366	degraded2 = 0;
4367	for (i = 0; i < conf->geo.raid_disks; i++) {
4368	struct md_rdev *rdev = conf->mirrors[i].rdev;
4369
4370	if (!rdev || test_bit(Faulty, &rdev->flags))
4371	degraded2++;
4372	else if (!test_bit(In_sync, &rdev->flags)) {
4373	/* If reshape is increasing the number of devices,
4374	* this section has already been recovered, so
4375	* it doesn't contribute to degraded.
4376	* else it does.
4377	*/
4378	if (conf->geo.raid_disks <= conf->prev.raid_disks)
4379	degraded2++;
4380	}
4381	}
4382	if (degraded2 > degraded)
4383	return degraded2;
4384	return degraded;
4385	}
4386
4387	static int raid10_start_reshape(struct mddev *mddev)
4388	{
4389	/* A 'reshape' has been requested. This commits
4390	* the various 'new' fields and sets MD_RECOVER_RESHAPE
4391	* This also checks if there are enough spares and adds them
4392	* to the array.
4393	* We currently require enough spares to make the final
4394	* array non-degraded. We also require that the difference
4395	* between old and new data_offset - on each device - is
4396	* enough that we never risk over-writing.
4397	*/
4398
4399	unsigned long before_length, after_length;
4400	sector_t min_offset_diff = 0;
4401	int first = 1;
4402	struct geom new;
4403	struct r10conf *conf = mddev->private;
4404	struct md_rdev *rdev;
4405	int spares = 0;
4406	int ret;
4407
4408	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
4409	return -EBUSY;
4410
4411	if (setup_geo(geo: &new, mddev, new: geo_start) != conf->copies)
4412	return -EINVAL;
4413
4414	before_length = ((1 << conf->prev.chunk_shift) *
4415	conf->prev.far_copies);
4416	after_length = ((1 << conf->geo.chunk_shift) *
4417	conf->geo.far_copies);
4418
4419	rdev_for_each(rdev, mddev) {
4420	if (!test_bit(In_sync, &rdev->flags)
4421	&& !test_bit(Faulty, &rdev->flags))
4422	spares++;
4423	if (rdev->raid_disk >= 0) {
4424	long long diff = (rdev->new_data_offset
4425	- rdev->data_offset);
4426	if (!mddev->reshape_backwards)
4427	diff = -diff;
4428	if (diff < 0)
4429	diff = 0;
4430	if (first || diff < min_offset_diff)
4431	min_offset_diff = diff;
4432	first = 0;
4433	}
4434	}
4435
4436	if (max(before_length, after_length) > min_offset_diff)
4437	return -EINVAL;
4438
4439	if (spares < mddev->delta_disks)
4440	return -EINVAL;
4441
4442	conf->offset_diff = min_offset_diff;
4443	spin_lock_irq(lock: &conf->device_lock);
4444	if (conf->mirrors_new) {
4445	memcpy(conf->mirrors_new, conf->mirrors,
4446	sizeof(struct raid10_info)*conf->prev.raid_disks);
4447	smp_mb();
4448	kfree(objp: conf->mirrors_old);
4449	conf->mirrors_old = conf->mirrors;
4450	conf->mirrors = conf->mirrors_new;
4451	conf->mirrors_new = NULL;
4452	}
4453	setup_geo(geo: &conf->geo, mddev, new: geo_start);
4454	smp_mb();
4455	if (mddev->reshape_backwards) {
4456	sector_t size = raid10_size(mddev, sectors: 0, raid_disks: 0);
4457	if (size < mddev->array_sectors) {
4458	spin_unlock_irq(lock: &conf->device_lock);
4459	pr_warn("md/raid10:%s: array size must be reduce before number of disks\n",
4460	mdname(mddev));
4461	return -EINVAL;
4462	}
4463	mddev->resync_max_sectors = size;
4464	conf->reshape_progress = size;
4465	} else
4466	conf->reshape_progress = 0;
4467	conf->reshape_safe = conf->reshape_progress;
4468	spin_unlock_irq(lock: &conf->device_lock);
4469
4470	if (mddev->delta_disks && mddev->bitmap) {
4471	struct mdp_superblock_1 *sb = NULL;
4472	sector_t oldsize, newsize;
4473
4474	oldsize = raid10_size(mddev, sectors: 0, raid_disks: 0);
4475	newsize = raid10_size(mddev, sectors: 0, raid_disks: conf->geo.raid_disks);
4476
4477	if (!mddev_is_clustered(mddev)) {
4478	ret = md_bitmap_resize(bitmap: mddev->bitmap, blocks: newsize, chunksize: 0, init: 0);
4479	if (ret)
4480	goto abort;
4481	else
4482	goto out;
4483	}
4484
4485	rdev_for_each(rdev, mddev) {
4486	if (rdev->raid_disk > -1 &&
4487	!test_bit(Faulty, &rdev->flags))
4488	sb = page_address(rdev->sb_page);
4489	}
4490
4491	/*
4492	* some node is already performing reshape, and no need to
4493	* call md_bitmap_resize again since it should be called when
4494	* receiving BITMAP_RESIZE msg
4495	*/
4496	if ((sb && (le32_to_cpu(sb->feature_map) &
4497	MD_FEATURE_RESHAPE_ACTIVE)) || (oldsize == newsize))
4498	goto out;
4499
4500	ret = md_bitmap_resize(bitmap: mddev->bitmap, blocks: newsize, chunksize: 0, init: 0);
4501	if (ret)
4502	goto abort;
4503
4504	ret = md_cluster_ops->resize_bitmaps(mddev, newsize, oldsize);
4505	if (ret) {
4506	md_bitmap_resize(bitmap: mddev->bitmap, blocks: oldsize, chunksize: 0, init: 0);
4507	goto abort;
4508	}
4509	}
4510	out:
4511	if (mddev->delta_disks > 0) {
4512	rdev_for_each(rdev, mddev)
4513	if (rdev->raid_disk < 0 &&
4514	!test_bit(Faulty, &rdev->flags)) {
4515	if (raid10_add_disk(mddev, rdev) == 0) {
4516	if (rdev->raid_disk >=
4517	conf->prev.raid_disks)
4518	set_bit(nr: In_sync, addr: &rdev->flags);
4519	else
4520	rdev->recovery_offset = 0;
4521
4522	/* Failure here is OK */
4523	sysfs_link_rdev(mddev, rdev);
4524	}
4525	} else if (rdev->raid_disk >= conf->prev.raid_disks
4526	&& !test_bit(Faulty, &rdev->flags)) {
4527	/* This is a spare that was manually added */
4528	set_bit(nr: In_sync, addr: &rdev->flags);
4529	}
4530	}
4531	/* When a reshape changes the number of devices,
4532	* ->degraded is measured against the larger of the
4533	* pre and post numbers.
4534	*/
4535	spin_lock_irq(lock: &conf->device_lock);
4536	mddev->degraded = calc_degraded(conf);
4537	spin_unlock_irq(lock: &conf->device_lock);
4538	mddev->raid_disks = conf->geo.raid_disks;
4539	mddev->reshape_position = conf->reshape_progress;
4540	set_bit(nr: MD_SB_CHANGE_DEVS, addr: &mddev->sb_flags);
4541
4542	clear_bit(nr: MD_RECOVERY_SYNC, addr: &mddev->recovery);
4543	clear_bit(nr: MD_RECOVERY_CHECK, addr: &mddev->recovery);
4544	clear_bit(nr: MD_RECOVERY_DONE, addr: &mddev->recovery);
4545	set_bit(nr: MD_RECOVERY_RESHAPE, addr: &mddev->recovery);
4546	set_bit(nr: MD_RECOVERY_NEEDED, addr: &mddev->recovery);
4547	conf->reshape_checkpoint = jiffies;
4548	md_new_event();
4549	return 0;
4550
4551	abort:
4552	mddev->recovery = 0;
4553	spin_lock_irq(lock: &conf->device_lock);
4554	conf->geo = conf->prev;
4555	mddev->raid_disks = conf->geo.raid_disks;
4556	rdev_for_each(rdev, mddev)
4557	rdev->new_data_offset = rdev->data_offset;
4558	smp_wmb();
4559	conf->reshape_progress = MaxSector;
4560	conf->reshape_safe = MaxSector;
4561	mddev->reshape_position = MaxSector;
4562	spin_unlock_irq(lock: &conf->device_lock);
4563	return ret;
4564	}
4565
4566	/* Calculate the last device-address that could contain
4567	* any block from the chunk that includes the array-address 's'
4568	* and report the next address.
4569	* i.e. the address returned will be chunk-aligned and after
4570	* any data that is in the chunk containing 's'.
4571	*/
4572	static sector_t last_dev_address(sector_t s, struct geom *geo)
4573	{
4574	s = (s | geo->chunk_mask) + 1;
4575	s >>= geo->chunk_shift;
4576	s *= geo->near_copies;
4577	s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks);
4578	s *= geo->far_copies;
4579	s <<= geo->chunk_shift;
4580	return s;
4581	}
4582
4583	/* Calculate the first device-address that could contain
4584	* any block from the chunk that includes the array-address 's'.
4585	* This too will be the start of a chunk
4586	*/
4587	static sector_t first_dev_address(sector_t s, struct geom *geo)
4588	{
4589	s >>= geo->chunk_shift;
4590	s *= geo->near_copies;
4591	sector_div(s, geo->raid_disks);
4592	s *= geo->far_copies;
4593	s <<= geo->chunk_shift;
4594	return s;
4595	}
4596
4597	static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
4598	int *skipped)
4599	{
4600	/* We simply copy at most one chunk (smallest of old and new)
4601	* at a time, possibly less if that exceeds RESYNC_PAGES,
4602	* or we hit a bad block or something.
4603	* This might mean we pause for normal IO in the middle of
4604	* a chunk, but that is not a problem as mddev->reshape_position
4605	* can record any location.
4606	*
4607	* If we will want to write to a location that isn't
4608	* yet recorded as 'safe' (i.e. in metadata on disk) then
4609	* we need to flush all reshape requests and update the metadata.
4610	*
4611	* When reshaping forwards (e.g. to more devices), we interpret
4612	* 'safe' as the earliest block which might not have been copied
4613	* down yet. We divide this by previous stripe size and multiply
4614	* by previous stripe length to get lowest device offset that we
4615	* cannot write to yet.
4616	* We interpret 'sector_nr' as an address that we want to write to.
4617	* From this we use last_device_address() to find where we might
4618	* write to, and first_device_address on the 'safe' position.
4619	* If this 'next' write position is after the 'safe' position,
4620	* we must update the metadata to increase the 'safe' position.
4621	*
4622	* When reshaping backwards, we round in the opposite direction
4623	* and perform the reverse test: next write position must not be
4624	* less than current safe position.
4625	*
4626	* In all this the minimum difference in data offsets
4627	* (conf->offset_diff - always positive) allows a bit of slack,
4628	* so next can be after 'safe', but not by more than offset_diff
4629	*
4630	* We need to prepare all the bios here before we start any IO
4631	* to ensure the size we choose is acceptable to all devices.
4632	* The means one for each copy for write-out and an extra one for
4633	* read-in.
4634	* We store the read-in bio in ->master_bio and the others in
4635	* ->devs[x].bio and ->devs[x].repl_bio.
4636	*/
4637	struct r10conf *conf = mddev->private;
4638	struct r10bio *r10_bio;
4639	sector_t next, safe, last;
4640	int max_sectors;
4641	int nr_sectors;
4642	int s;
4643	struct md_rdev *rdev;
4644	int need_flush = 0;
4645	struct bio *blist;
4646	struct bio *bio, *read_bio;
4647	int sectors_done = 0;
4648	struct page **pages;
4649
4650	if (sector_nr == 0) {
4651	/* If restarting in the middle, skip the initial sectors */
4652	if (mddev->reshape_backwards &&
4653	conf->reshape_progress < raid10_size(mddev, sectors: 0, raid_disks: 0)) {
4654	sector_nr = (raid10_size(mddev, sectors: 0, raid_disks: 0)
4655	- conf->reshape_progress);
4656	} else if (!mddev->reshape_backwards &&
4657	conf->reshape_progress > 0)
4658	sector_nr = conf->reshape_progress;
4659	if (sector_nr) {
4660	mddev->curr_resync_completed = sector_nr;
4661	sysfs_notify_dirent_safe(sd: mddev->sysfs_completed);
4662	*skipped = 1;
4663	return sector_nr;
4664	}
4665	}
4666
4667	/* We don't use sector_nr to track where we are up to
4668	* as that doesn't work well for ->reshape_backwards.
4669	* So just use ->reshape_progress.
4670	*/
4671	if (mddev->reshape_backwards) {
4672	/* 'next' is the earliest device address that we might
4673	* write to for this chunk in the new layout
4674	*/
4675	next = first_dev_address(s: conf->reshape_progress - 1,
4676	geo: &conf->geo);
4677
4678	/* 'safe' is the last device address that we might read from
4679	* in the old layout after a restart
4680	*/
4681	safe = last_dev_address(s: conf->reshape_safe - 1,
4682	geo: &conf->prev);
4683
4684	if (next + conf->offset_diff < safe)
4685	need_flush = 1;
4686
4687	last = conf->reshape_progress - 1;
4688	sector_nr = last & ~(sector_t)(conf->geo.chunk_mask
4689	& conf->prev.chunk_mask);
4690	if (sector_nr + RESYNC_SECTORS < last)
4691	sector_nr = last + 1 - RESYNC_SECTORS;
4692	} else {
4693	/* 'next' is after the last device address that we
4694	* might write to for this chunk in the new layout
4695	*/
4696	next = last_dev_address(s: conf->reshape_progress, geo: &conf->geo);
4697
4698	/* 'safe' is the earliest device address that we might
4699	* read from in the old layout after a restart
4700	*/
4701	safe = first_dev_address(s: conf->reshape_safe, geo: &conf->prev);
4702
4703	/* Need to update metadata if 'next' might be beyond 'safe'
4704	* as that would possibly corrupt data
4705	*/
4706	if (next > safe + conf->offset_diff)
4707	need_flush = 1;
4708
4709	sector_nr = conf->reshape_progress;
4710	last = sector_nr | (conf->geo.chunk_mask
4711	& conf->prev.chunk_mask);
4712
4713	if (sector_nr + RESYNC_SECTORS <= last)
4714	last = sector_nr + RESYNC_SECTORS - 1;
4715	}
4716
4717	if (need_flush ||
4718	time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4719	/* Need to update reshape_position in metadata */
4720	wait_barrier(conf, nowait: false);
4721	mddev->reshape_position = conf->reshape_progress;
4722	if (mddev->reshape_backwards)
4723	mddev->curr_resync_completed = raid10_size(mddev, sectors: 0, raid_disks: 0)
4724	- conf->reshape_progress;
4725	else
4726	mddev->curr_resync_completed = conf->reshape_progress;
4727	conf->reshape_checkpoint = jiffies;
4728	set_bit(nr: MD_SB_CHANGE_DEVS, addr: &mddev->sb_flags);
4729	md_wakeup_thread(thread: mddev->thread);
4730	wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||
4731	test_bit(MD_RECOVERY_INTR, &mddev->recovery));
4732	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
4733	allow_barrier(conf);
4734	return sectors_done;
4735	}
4736	conf->reshape_safe = mddev->reshape_position;
4737	allow_barrier(conf);
4738	}
4739
4740	raise_barrier(conf, force: 0);
4741	read_more:
4742	/* Now schedule reads for blocks from sector_nr to last */
4743	r10_bio = raid10_alloc_init_r10buf(conf);
4744	r10_bio->state = 0;
4745	raise_barrier(conf, force: 1);
4746	atomic_set(v: &r10_bio->remaining, i: 0);
4747	r10_bio->mddev = mddev;
4748	r10_bio->sector = sector_nr;
4749	set_bit(nr: R10BIO_IsReshape, addr: &r10_bio->state);
4750	r10_bio->sectors = last - sector_nr + 1;
4751	rdev = read_balance(conf, r10_bio, max_sectors: &max_sectors);
4752	BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state));
4753
4754	if (!rdev) {
4755	/* Cannot read from here, so need to record bad blocks
4756	* on all the target devices.
4757	*/
4758	// FIXME
4759	mempool_free(element: r10_bio, pool: &conf->r10buf_pool);
4760	set_bit(nr: MD_RECOVERY_INTR, addr: &mddev->recovery);
4761	return sectors_done;
4762	}
4763
4764	read_bio = bio_alloc_bioset(bdev: rdev->bdev, RESYNC_PAGES, opf: REQ_OP_READ,
4765	GFP_KERNEL, bs: &mddev->bio_set);
4766	read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
4767	+ rdev->data_offset);
4768	read_bio->bi_private = r10_bio;
4769	read_bio->bi_end_io = end_reshape_read;
4770	r10_bio->master_bio = read_bio;
4771	r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;
4772
4773	/*
4774	* Broadcast RESYNC message to other nodes, so all nodes would not
4775	* write to the region to avoid conflict.
4776	*/
4777	if (mddev_is_clustered(mddev) && conf->cluster_sync_high <= sector_nr) {
4778	struct mdp_superblock_1 *sb = NULL;
4779	int sb_reshape_pos = 0;
4780
4781	conf->cluster_sync_low = sector_nr;
4782	conf->cluster_sync_high = sector_nr + CLUSTER_RESYNC_WINDOW_SECTORS;
4783	sb = page_address(rdev->sb_page);
4784	if (sb) {
4785	sb_reshape_pos = le64_to_cpu(sb->reshape_position);
4786	/*
4787	* Set cluster_sync_low again if next address for array
4788	* reshape is less than cluster_sync_low. Since we can't
4789	* update cluster_sync_low until it has finished reshape.
4790	*/
4791	if (sb_reshape_pos < conf->cluster_sync_low)
4792	conf->cluster_sync_low = sb_reshape_pos;
4793	}
4794
4795	md_cluster_ops->resync_info_update(mddev, conf->cluster_sync_low,
4796	conf->cluster_sync_high);
4797	}
4798
4799	/* Now find the locations in the new layout */
4800	__raid10_find_phys(geo: &conf->geo, r10bio: r10_bio);
4801
4802	blist = read_bio;
4803	read_bio->bi_next = NULL;
4804
4805	for (s = 0; s < conf->copies*2; s++) {
4806	struct bio *b;
4807	int d = r10_bio->devs[s/2].devnum;
4808	struct md_rdev *rdev2;
4809	if (s&1) {
4810	rdev2 = conf->mirrors[d].replacement;
4811	b = r10_bio->devs[s/2].repl_bio;
4812	} else {
4813	rdev2 = conf->mirrors[d].rdev;
4814	b = r10_bio->devs[s/2].bio;
4815	}
4816	if (!rdev2 || test_bit(Faulty, &rdev2->flags))
4817	continue;
4818
4819	bio_set_dev(bio: b, bdev: rdev2->bdev);
4820	b->bi_iter.bi_sector = r10_bio->devs[s/2].addr +
4821	rdev2->new_data_offset;
4822	b->bi_end_io = end_reshape_write;
4823	b->bi_opf = REQ_OP_WRITE;
4824	b->bi_next = blist;
4825	blist = b;
4826	}
4827
4828	/* Now add as many pages as possible to all of these bios. */
4829
4830	nr_sectors = 0;
4831	pages = get_resync_pages(bio: r10_bio->devs[0].bio)->pages;
4832	for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) {
4833	struct page *page = pages[s / (PAGE_SIZE >> 9)];
4834	int len = (max_sectors - s) << 9;
4835	if (len > PAGE_SIZE)
4836	len = PAGE_SIZE;
4837	for (bio = blist; bio ; bio = bio->bi_next) {
4838	if (WARN_ON(!bio_add_page(bio, page, len, 0))) {
4839	bio->bi_status = BLK_STS_RESOURCE;
4840	bio_endio(bio);
4841	return sectors_done;
4842	}
4843	}
4844	sector_nr += len >> 9;
4845	nr_sectors += len >> 9;
4846	}
4847	r10_bio->sectors = nr_sectors;
4848
4849	/* Now submit the read */
4850	md_sync_acct_bio(bio: read_bio, nr_sectors: r10_bio->sectors);
4851	atomic_inc(v: &r10_bio->remaining);
4852	read_bio->bi_next = NULL;
4853	submit_bio_noacct(bio: read_bio);
4854	sectors_done += nr_sectors;
4855	if (sector_nr <= last)
4856	goto read_more;
4857
4858	lower_barrier(conf);
4859
4860	/* Now that we have done the whole section we can
4861	* update reshape_progress
4862	*/
4863	if (mddev->reshape_backwards)
4864	conf->reshape_progress -= sectors_done;
4865	else
4866	conf->reshape_progress += sectors_done;
4867
4868	return sectors_done;
4869	}
4870
4871	static void end_reshape_request(struct r10bio *r10_bio);
4872	static int handle_reshape_read_error(struct mddev *mddev,
4873	struct r10bio *r10_bio);
4874	static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
4875	{
4876	/* Reshape read completed. Hopefully we have a block
4877	* to write out.
4878	* If we got a read error then we do sync 1-page reads from
4879	* elsewhere until we find the data - or give up.
4880	*/
4881	struct r10conf *conf = mddev->private;
4882	int s;
4883
4884	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
4885	if (handle_reshape_read_error(mddev, r10_bio) < 0) {
4886	/* Reshape has been aborted */
4887	md_done_sync(mddev, blocks: r10_bio->sectors, ok: 0);
4888	return;
4889	}
4890
4891	/* We definitely have the data in the pages, schedule the
4892	* writes.
4893	*/
4894	atomic_set(v: &r10_bio->remaining, i: 1);
4895	for (s = 0; s < conf->copies*2; s++) {
4896	struct bio *b;
4897	int d = r10_bio->devs[s/2].devnum;
4898	struct md_rdev *rdev;
4899	if (s&1) {
4900	rdev = conf->mirrors[d].replacement;
4901	b = r10_bio->devs[s/2].repl_bio;
4902	} else {
4903	rdev = conf->mirrors[d].rdev;
4904	b = r10_bio->devs[s/2].bio;
4905	}
4906	if (!rdev || test_bit(Faulty, &rdev->flags))
4907	continue;
4908
4909	atomic_inc(v: &rdev->nr_pending);
4910	md_sync_acct_bio(bio: b, nr_sectors: r10_bio->sectors);
4911	atomic_inc(v: &r10_bio->remaining);
4912	b->bi_next = NULL;
4913	submit_bio_noacct(bio: b);
4914	}
4915	end_reshape_request(r10_bio);
4916	}
4917
4918	static void end_reshape(struct r10conf *conf)
4919	{
4920	if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))
4921	return;
4922
4923	spin_lock_irq(lock: &conf->device_lock);
4924	conf->prev = conf->geo;
4925	md_finish_reshape(mddev: conf->mddev);
4926	smp_wmb();
4927	conf->reshape_progress = MaxSector;
4928	conf->reshape_safe = MaxSector;
4929	spin_unlock_irq(lock: &conf->device_lock);
4930
4931	mddev_update_io_opt(mddev: conf->mddev, nr_stripes: raid10_nr_stripes(conf));
4932	conf->fullsync = 0;
4933	}
4934
4935	static void raid10_update_reshape_pos(struct mddev *mddev)
4936	{
4937	struct r10conf *conf = mddev->private;
4938	sector_t lo, hi;
4939
4940	md_cluster_ops->resync_info_get(mddev, &lo, &hi);
4941	if (((mddev->reshape_position <= hi) && (mddev->reshape_position >= lo))
4942	|| mddev->reshape_position == MaxSector)
4943	conf->reshape_progress = mddev->reshape_position;
4944	else
4945	WARN_ON_ONCE(1);
4946	}
4947
4948	static int handle_reshape_read_error(struct mddev *mddev,
4949	struct r10bio *r10_bio)
4950	{
4951	/* Use sync reads to get the blocks from somewhere else */
4952	int sectors = r10_bio->sectors;
4953	struct r10conf *conf = mddev->private;
4954	struct r10bio *r10b;
4955	int slot = 0;
4956	int idx = 0;
4957	struct page **pages;
4958
4959	r10b = kmalloc(struct_size(r10b, devs, conf->copies), GFP_NOIO);
4960	if (!r10b) {
4961	set_bit(nr: MD_RECOVERY_INTR, addr: &mddev->recovery);
4962	return -ENOMEM;
4963	}
4964
4965	/* reshape IOs share pages from .devs[0].bio */
4966	pages = get_resync_pages(bio: r10_bio->devs[0].bio)->pages;
4967
4968	r10b->sector = r10_bio->sector;
4969	__raid10_find_phys(geo: &conf->prev, r10bio: r10b);
4970
4971	while (sectors) {
4972	int s = sectors;
4973	int success = 0;
4974	int first_slot = slot;
4975
4976	if (s > (PAGE_SIZE >> 9))
4977	s = PAGE_SIZE >> 9;
4978
4979	while (!success) {
4980	int d = r10b->devs[slot].devnum;
4981	struct md_rdev *rdev = conf->mirrors[d].rdev;
4982	sector_t addr;
4983	if (rdev == NULL ||
4984	test_bit(Faulty, &rdev->flags) ||
4985	!test_bit(In_sync, &rdev->flags))
4986	goto failed;
4987
4988	addr = r10b->devs[slot].addr + idx * PAGE_SIZE;
4989	atomic_inc(v: &rdev->nr_pending);
4990	success = sync_page_io(rdev,
4991	sector: addr,
4992	size: s << 9,
4993	page: pages[idx],
4994	opf: REQ_OP_READ, metadata_op: false);
4995	rdev_dec_pending(rdev, mddev);
4996	if (success)
4997	break;
4998	failed:
4999	slot++;
5000	if (slot >= conf->copies)
5001	slot = 0;
5002	if (slot == first_slot)
5003	break;
5004	}
5005	if (!success) {
5006	/* couldn't read this block, must give up */
5007	set_bit(nr: MD_RECOVERY_INTR,
5008	addr: &mddev->recovery);
5009	kfree(objp: r10b);
5010	return -EIO;
5011	}
5012	sectors -= s;
5013	idx++;
5014	}
5015	kfree(objp: r10b);
5016	return 0;
5017	}
5018
5019	static void end_reshape_write(struct bio *bio)
5020	{
5021	struct r10bio *r10_bio = get_resync_r10bio(bio);
5022	struct mddev *mddev = r10_bio->mddev;
5023	struct r10conf *conf = mddev->private;
5024	int d;
5025	int slot;
5026	int repl;
5027	struct md_rdev *rdev = NULL;
5028
5029	d = find_bio_disk(conf, r10_bio, bio, slotp: &slot, replp: &repl);
5030	rdev = repl ? conf->mirrors[d].replacement :
5031	conf->mirrors[d].rdev;
5032
5033	if (bio->bi_status) {
5034	/* FIXME should record badblock */
5035	md_error(mddev, rdev);
5036	}
5037
5038	rdev_dec_pending(rdev, mddev);
5039	end_reshape_request(r10_bio);
5040	}
5041
5042	static void end_reshape_request(struct r10bio *r10_bio)
5043	{
5044	if (!atomic_dec_and_test(v: &r10_bio->remaining))
5045	return;
5046	md_done_sync(mddev: r10_bio->mddev, blocks: r10_bio->sectors, ok: 1);
5047	bio_put(r10_bio->master_bio);
5048	put_buf(r10_bio);
5049	}
5050
5051	static void raid10_finish_reshape(struct mddev *mddev)
5052	{
5053	struct r10conf *conf = mddev->private;
5054
5055	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
5056	return;
5057
5058	if (mddev->delta_disks > 0) {
5059	if (mddev->recovery_cp > mddev->resync_max_sectors) {
5060	mddev->recovery_cp = mddev->resync_max_sectors;
5061	set_bit(nr: MD_RECOVERY_NEEDED, addr: &mddev->recovery);
5062	}
5063	mddev->resync_max_sectors = mddev->array_sectors;
5064	} else {
5065	int d;
5066	for (d = conf->geo.raid_disks ;
5067	d < conf->geo.raid_disks - mddev->delta_disks;
5068	d++) {
5069	struct md_rdev *rdev = conf->mirrors[d].rdev;
5070	if (rdev)
5071	clear_bit(nr: In_sync, addr: &rdev->flags);
5072	rdev = conf->mirrors[d].replacement;
5073	if (rdev)
5074	clear_bit(nr: In_sync, addr: &rdev->flags);
5075	}
5076	}
5077	mddev->layout = mddev->new_layout;
5078	mddev->chunk_sectors = 1 << conf->geo.chunk_shift;
5079	mddev->reshape_position = MaxSector;
5080	mddev->delta_disks = 0;
5081	mddev->reshape_backwards = 0;
5082	}
5083
5084	static struct md_personality raid10_personality =
5085	{
5086	.name = "raid10",
5087	.level = 10,
5088	.owner = THIS_MODULE,
5089	.make_request = raid10_make_request,
5090	.run = raid10_run,
5091	.free = raid10_free,
5092	.status = raid10_status,
5093	.error_handler = raid10_error,
5094	.hot_add_disk = raid10_add_disk,
5095	.hot_remove_disk= raid10_remove_disk,
5096	.spare_active = raid10_spare_active,
5097	.sync_request = raid10_sync_request,
5098	.quiesce = raid10_quiesce,
5099	.size = raid10_size,
5100	.resize = raid10_resize,
5101	.takeover = raid10_takeover,
5102	.check_reshape = raid10_check_reshape,
5103	.start_reshape = raid10_start_reshape,
5104	.finish_reshape = raid10_finish_reshape,
5105	.update_reshape_pos = raid10_update_reshape_pos,
5106	};
5107
5108	static int __init raid_init(void)
5109	{
5110	return register_md_personality(p: &raid10_personality);
5111	}
5112
5113	static void raid_exit(void)
5114	{
5115	unregister_md_personality(p: &raid10_personality);
5116	}
5117
5118	module_init(raid_init);
5119	module_exit(raid_exit);
5120	MODULE_LICENSE("GPL");
5121	MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD");
5122	MODULE_ALIAS("md-personality-9"); /* RAID10 */
5123	MODULE_ALIAS("md-raid10");
5124	MODULE_ALIAS("md-level-10");
5125