1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) 2010-2011 Neil Brown
4	* Copyright (C) 2010-2018 Red Hat, Inc. All rights reserved.
5	*
6	* This file is released under the GPL.
7	*/
8
9	#include <linux/slab.h>
10	#include <linux/module.h>
11
12	#include "md.h"
13	#include "raid1.h"
14	#include "raid5.h"
15	#include "raid10.h"
16	#include "md-bitmap.h"
17
18	#include <linux/device-mapper.h>
19
20	#define DM_MSG_PREFIX "raid"
21	#define MAX_RAID_DEVICES 253 /* md-raid kernel limit */
22
23	/*
24	* Minimum sectors of free reshape space per raid device
25	*/
26	#define MIN_FREE_RESHAPE_SPACE to_sector(4*4096)
27
28	/*
29	* Minimum journal space 4 MiB in sectors.
30	*/
31	#define MIN_RAID456_JOURNAL_SPACE (4*2048)
32
33	static bool devices_handle_discard_safely;
34
35	/*
36	* The following flags are used by dm-raid to set up the array state.
37	* They must be cleared before md_run is called.
38	*/
39	#define FirstUse 10 /* rdev flag */
40
41	struct raid_dev {
42	/*
43	* Two DM devices, one to hold metadata and one to hold the
44	* actual data/parity. The reason for this is to not confuse
45	* ti->len and give more flexibility in altering size and
46	* characteristics.
47	*
48	* While it is possible for this device to be associated
49	* with a different physical device than the data_dev, it
50	* is intended for it to be the same.
51	* |--------- Physical Device ---------|
52	* |- meta_dev -|------ data_dev ------|
53	*/
54	struct dm_dev *meta_dev;
55	struct dm_dev *data_dev;
56	struct md_rdev rdev;
57	};
58
59	/*
60	* Bits for establishing rs->ctr_flags
61	*
62	* 1 = no flag value
63	* 2 = flag with value
64	*/
65	#define __CTR_FLAG_SYNC 0 /* 1 */ /* Not with raid0! */
66	#define __CTR_FLAG_NOSYNC 1 /* 1 */ /* Not with raid0! */
67	#define __CTR_FLAG_REBUILD 2 /* 2 */ /* Not with raid0! */
68	#define __CTR_FLAG_DAEMON_SLEEP 3 /* 2 */ /* Not with raid0! */
69	#define __CTR_FLAG_MIN_RECOVERY_RATE 4 /* 2 */ /* Not with raid0! */
70	#define __CTR_FLAG_MAX_RECOVERY_RATE 5 /* 2 */ /* Not with raid0! */
71	#define __CTR_FLAG_MAX_WRITE_BEHIND 6 /* 2 */ /* Only with raid1! */
72	#define __CTR_FLAG_WRITE_MOSTLY 7 /* 2 */ /* Only with raid1! */
73	#define __CTR_FLAG_STRIPE_CACHE 8 /* 2 */ /* Only with raid4/5/6! */
74	#define __CTR_FLAG_REGION_SIZE 9 /* 2 */ /* Not with raid0! */
75	#define __CTR_FLAG_RAID10_COPIES 10 /* 2 */ /* Only with raid10 */
76	#define __CTR_FLAG_RAID10_FORMAT 11 /* 2 */ /* Only with raid10 */
77	/* New for v1.9.0 */
78	#define __CTR_FLAG_DELTA_DISKS 12 /* 2 */ /* Only with reshapable raid1/4/5/6/10! */
79	#define __CTR_FLAG_DATA_OFFSET 13 /* 2 */ /* Only with reshapable raid4/5/6/10! */
80	#define __CTR_FLAG_RAID10_USE_NEAR_SETS 14 /* 2 */ /* Only with raid10! */
81
82	/* New for v1.10.0 */
83	#define __CTR_FLAG_JOURNAL_DEV 15 /* 2 */ /* Only with raid4/5/6 (journal device)! */
84
85	/* New for v1.11.1 */
86	#define __CTR_FLAG_JOURNAL_MODE 16 /* 2 */ /* Only with raid4/5/6 (journal mode)! */
87
88	/*
89	* Flags for rs->ctr_flags field.
90	*/
91	#define CTR_FLAG_SYNC (1 << __CTR_FLAG_SYNC)
92	#define CTR_FLAG_NOSYNC (1 << __CTR_FLAG_NOSYNC)
93	#define CTR_FLAG_REBUILD (1 << __CTR_FLAG_REBUILD)
94	#define CTR_FLAG_DAEMON_SLEEP (1 << __CTR_FLAG_DAEMON_SLEEP)
95	#define CTR_FLAG_MIN_RECOVERY_RATE (1 << __CTR_FLAG_MIN_RECOVERY_RATE)
96	#define CTR_FLAG_MAX_RECOVERY_RATE (1 << __CTR_FLAG_MAX_RECOVERY_RATE)
97	#define CTR_FLAG_MAX_WRITE_BEHIND (1 << __CTR_FLAG_MAX_WRITE_BEHIND)
98	#define CTR_FLAG_WRITE_MOSTLY (1 << __CTR_FLAG_WRITE_MOSTLY)
99	#define CTR_FLAG_STRIPE_CACHE (1 << __CTR_FLAG_STRIPE_CACHE)
100	#define CTR_FLAG_REGION_SIZE (1 << __CTR_FLAG_REGION_SIZE)
101	#define CTR_FLAG_RAID10_COPIES (1 << __CTR_FLAG_RAID10_COPIES)
102	#define CTR_FLAG_RAID10_FORMAT (1 << __CTR_FLAG_RAID10_FORMAT)
103	#define CTR_FLAG_DELTA_DISKS (1 << __CTR_FLAG_DELTA_DISKS)
104	#define CTR_FLAG_DATA_OFFSET (1 << __CTR_FLAG_DATA_OFFSET)
105	#define CTR_FLAG_RAID10_USE_NEAR_SETS (1 << __CTR_FLAG_RAID10_USE_NEAR_SETS)
106	#define CTR_FLAG_JOURNAL_DEV (1 << __CTR_FLAG_JOURNAL_DEV)
107	#define CTR_FLAG_JOURNAL_MODE (1 << __CTR_FLAG_JOURNAL_MODE)
108
109	/*
110	* Definitions of various constructor flags to
111	* be used in checks of valid / invalid flags
112	* per raid level.
113	*/
114	/* Define all any sync flags */
115	#define CTR_FLAGS_ANY_SYNC (CTR_FLAG_SYNC | CTR_FLAG_NOSYNC)
116
117	/* Define flags for options without argument (e.g. 'nosync') */
118	#define CTR_FLAG_OPTIONS_NO_ARGS (CTR_FLAGS_ANY_SYNC | \
119	CTR_FLAG_RAID10_USE_NEAR_SETS)
120
121	/* Define flags for options with one argument (e.g. 'delta_disks +2') */
122	#define CTR_FLAG_OPTIONS_ONE_ARG (CTR_FLAG_REBUILD | \
123	CTR_FLAG_WRITE_MOSTLY | \
124	CTR_FLAG_DAEMON_SLEEP | \
125	CTR_FLAG_MIN_RECOVERY_RATE | \
126	CTR_FLAG_MAX_RECOVERY_RATE | \
127	CTR_FLAG_MAX_WRITE_BEHIND | \
128	CTR_FLAG_STRIPE_CACHE | \
129	CTR_FLAG_REGION_SIZE | \
130	CTR_FLAG_RAID10_COPIES | \
131	CTR_FLAG_RAID10_FORMAT | \
132	CTR_FLAG_DELTA_DISKS | \
133	CTR_FLAG_DATA_OFFSET | \
134	CTR_FLAG_JOURNAL_DEV | \
135	CTR_FLAG_JOURNAL_MODE)
136
137	/* Valid options definitions per raid level... */
138
139	/* "raid0" does only accept data offset */
140	#define RAID0_VALID_FLAGS (CTR_FLAG_DATA_OFFSET)
141
142	/* "raid1" does not accept stripe cache, data offset, delta_disks or any raid10 options */
143	#define RAID1_VALID_FLAGS (CTR_FLAGS_ANY_SYNC | \
144	CTR_FLAG_REBUILD | \
145	CTR_FLAG_WRITE_MOSTLY | \
146	CTR_FLAG_DAEMON_SLEEP | \
147	CTR_FLAG_MIN_RECOVERY_RATE | \
148	CTR_FLAG_MAX_RECOVERY_RATE | \
149	CTR_FLAG_MAX_WRITE_BEHIND | \
150	CTR_FLAG_REGION_SIZE | \
151	CTR_FLAG_DELTA_DISKS | \
152	CTR_FLAG_DATA_OFFSET)
153
154	/* "raid10" does not accept any raid1 or stripe cache options */
155	#define RAID10_VALID_FLAGS (CTR_FLAGS_ANY_SYNC | \
156	CTR_FLAG_REBUILD | \
157	CTR_FLAG_DAEMON_SLEEP | \
158	CTR_FLAG_MIN_RECOVERY_RATE | \
159	CTR_FLAG_MAX_RECOVERY_RATE | \
160	CTR_FLAG_REGION_SIZE | \
161	CTR_FLAG_RAID10_COPIES | \
162	CTR_FLAG_RAID10_FORMAT | \
163	CTR_FLAG_DELTA_DISKS | \
164	CTR_FLAG_DATA_OFFSET | \
165	CTR_FLAG_RAID10_USE_NEAR_SETS)
166
167	/*
168	* "raid4/5/6" do not accept any raid1 or raid10 specific options
169	*
170	* "raid6" does not accept "nosync", because it is not guaranteed
171	* that both parity and q-syndrome are being written properly with
172	* any writes
173	*/
174	#define RAID45_VALID_FLAGS (CTR_FLAGS_ANY_SYNC | \
175	CTR_FLAG_REBUILD | \
176	CTR_FLAG_DAEMON_SLEEP | \
177	CTR_FLAG_MIN_RECOVERY_RATE | \
178	CTR_FLAG_MAX_RECOVERY_RATE | \
179	CTR_FLAG_STRIPE_CACHE | \
180	CTR_FLAG_REGION_SIZE | \
181	CTR_FLAG_DELTA_DISKS | \
182	CTR_FLAG_DATA_OFFSET | \
183	CTR_FLAG_JOURNAL_DEV | \
184	CTR_FLAG_JOURNAL_MODE)
185
186	#define RAID6_VALID_FLAGS (CTR_FLAG_SYNC | \
187	CTR_FLAG_REBUILD | \
188	CTR_FLAG_DAEMON_SLEEP | \
189	CTR_FLAG_MIN_RECOVERY_RATE | \
190	CTR_FLAG_MAX_RECOVERY_RATE | \
191	CTR_FLAG_STRIPE_CACHE | \
192	CTR_FLAG_REGION_SIZE | \
193	CTR_FLAG_DELTA_DISKS | \
194	CTR_FLAG_DATA_OFFSET | \
195	CTR_FLAG_JOURNAL_DEV | \
196	CTR_FLAG_JOURNAL_MODE)
197	/* ...valid options definitions per raid level */
198
199	/*
200	* Flags for rs->runtime_flags field
201	* (RT_FLAG prefix meaning "runtime flag")
202	*
203	* These are all internal and used to define runtime state,
204	* e.g. to prevent another resume from preresume processing
205	* the raid set all over again.
206	*/
207	#define RT_FLAG_RS_PRERESUMED 0
208	#define RT_FLAG_RS_RESUMED 1
209	#define RT_FLAG_RS_BITMAP_LOADED 2
210	#define RT_FLAG_UPDATE_SBS 3
211	#define RT_FLAG_RESHAPE_RS 4
212	#define RT_FLAG_RS_SUSPENDED 5
213	#define RT_FLAG_RS_IN_SYNC 6
214	#define RT_FLAG_RS_RESYNCING 7
215	#define RT_FLAG_RS_GROW 8
216
217	/* Array elements of 64 bit needed for rebuild/failed disk bits */
218	#define DISKS_ARRAY_ELEMS ((MAX_RAID_DEVICES + (sizeof(uint64_t) * 8 - 1)) / sizeof(uint64_t) / 8)
219
220	/*
221	* raid set level, layout and chunk sectors backup/restore
222	*/
223	struct rs_layout {
224	int new_level;
225	int new_layout;
226	int new_chunk_sectors;
227	};
228
229	struct raid_set {
230	struct dm_target *ti;
231
232	uint32_t stripe_cache_entries;
233	unsigned long ctr_flags;
234	unsigned long runtime_flags;
235
236	uint64_t rebuild_disks[DISKS_ARRAY_ELEMS];
237
238	int raid_disks;
239	int delta_disks;
240	int data_offset;
241	int raid10_copies;
242	int requested_bitmap_chunk_sectors;
243
244	struct mddev md;
245	struct raid_type *raid_type;
246
247	sector_t array_sectors;
248	sector_t dev_sectors;
249
250	/* Optional raid4/5/6 journal device */
251	struct journal_dev {
252	struct dm_dev *dev;
253	struct md_rdev rdev;
254	int mode;
255	} journal_dev;
256
257	struct raid_dev dev[] __counted_by(raid_disks);
258	};
259
260	static void rs_config_backup(struct raid_set *rs, struct rs_layout *l)
261	{
262	struct mddev *mddev = &rs->md;
263
264	l->new_level = mddev->new_level;
265	l->new_layout = mddev->new_layout;
266	l->new_chunk_sectors = mddev->new_chunk_sectors;
267	}
268
269	static void rs_config_restore(struct raid_set *rs, struct rs_layout *l)
270	{
271	struct mddev *mddev = &rs->md;
272
273	mddev->new_level = l->new_level;
274	mddev->new_layout = l->new_layout;
275	mddev->new_chunk_sectors = l->new_chunk_sectors;
276	}
277
278	/* raid10 algorithms (i.e. formats) */
279	#define ALGORITHM_RAID10_DEFAULT 0
280	#define ALGORITHM_RAID10_NEAR 1
281	#define ALGORITHM_RAID10_OFFSET 2
282	#define ALGORITHM_RAID10_FAR 3
283
284	/* Supported raid types and properties. */
285	static struct raid_type {
286	const char *name; /* RAID algorithm. */
287	const char *descr; /* Descriptor text for logging. */
288	const unsigned int parity_devs; /* # of parity devices. */
289	const unsigned int minimal_devs;/* minimal # of devices in set. */
290	const unsigned int level; /* RAID level. */
291	const unsigned int algorithm; /* RAID algorithm. */
292	} raid_types[] = {
293	{"raid0", "raid0 (striping)", 0, 2, 0, 0 /* NONE */},
294	{"raid1", "raid1 (mirroring)", 0, 2, 1, 0 /* NONE */},
295	{"raid10_far", "raid10 far (striped mirrors)", 0, 2, 10, ALGORITHM_RAID10_FAR},
296	{"raid10_offset", "raid10 offset (striped mirrors)", 0, 2, 10, ALGORITHM_RAID10_OFFSET},
297	{"raid10_near", "raid10 near (striped mirrors)", 0, 2, 10, ALGORITHM_RAID10_NEAR},
298	{"raid10", "raid10 (striped mirrors)", 0, 2, 10, ALGORITHM_RAID10_DEFAULT},
299	{"raid4", "raid4 (dedicated first parity disk)", 1, 2, 5, ALGORITHM_PARITY_0}, /* raid4 layout = raid5_0 */
300	{"raid5_n", "raid5 (dedicated last parity disk)", 1, 2, 5, ALGORITHM_PARITY_N},
301	{"raid5_ls", "raid5 (left symmetric)", 1, 2, 5, ALGORITHM_LEFT_SYMMETRIC},
302	{"raid5_rs", "raid5 (right symmetric)", 1, 2, 5, ALGORITHM_RIGHT_SYMMETRIC},
303	{"raid5_la", "raid5 (left asymmetric)", 1, 2, 5, ALGORITHM_LEFT_ASYMMETRIC},
304	{"raid5_ra", "raid5 (right asymmetric)", 1, 2, 5, ALGORITHM_RIGHT_ASYMMETRIC},
305	{"raid6_zr", "raid6 (zero restart)", 2, 4, 6, ALGORITHM_ROTATING_ZERO_RESTART},
306	{"raid6_nr", "raid6 (N restart)", 2, 4, 6, ALGORITHM_ROTATING_N_RESTART},
307	{"raid6_nc", "raid6 (N continue)", 2, 4, 6, ALGORITHM_ROTATING_N_CONTINUE},
308	{"raid6_n_6", "raid6 (dedicated parity/Q n/6)", 2, 4, 6, ALGORITHM_PARITY_N_6},
309	{"raid6_ls_6", "raid6 (left symmetric dedicated Q 6)", 2, 4, 6, ALGORITHM_LEFT_SYMMETRIC_6},
310	{"raid6_rs_6", "raid6 (right symmetric dedicated Q 6)", 2, 4, 6, ALGORITHM_RIGHT_SYMMETRIC_6},
311	{"raid6_la_6", "raid6 (left asymmetric dedicated Q 6)", 2, 4, 6, ALGORITHM_LEFT_ASYMMETRIC_6},
312	{"raid6_ra_6", "raid6 (right asymmetric dedicated Q 6)", 2, 4, 6, ALGORITHM_RIGHT_ASYMMETRIC_6}
313	};
314
315	/* True, if @v is in inclusive range [@min, @max] */
316	static bool __within_range(long v, long min, long max)
317	{
318	return v >= min && v <= max;
319	}
320
321	/* All table line arguments are defined here */
322	static struct arg_name_flag {
323	const unsigned long flag;
324	const char *name;
325	} __arg_name_flags[] = {
326	{ CTR_FLAG_SYNC, "sync"},
327	{ CTR_FLAG_NOSYNC, "nosync"},
328	{ CTR_FLAG_REBUILD, "rebuild"},
329	{ CTR_FLAG_DAEMON_SLEEP, "daemon_sleep"},
330	{ CTR_FLAG_MIN_RECOVERY_RATE, "min_recovery_rate"},
331	{ CTR_FLAG_MAX_RECOVERY_RATE, "max_recovery_rate"},
332	{ CTR_FLAG_MAX_WRITE_BEHIND, "max_write_behind"},
333	{ CTR_FLAG_WRITE_MOSTLY, "write_mostly"},
334	{ CTR_FLAG_STRIPE_CACHE, "stripe_cache"},
335	{ CTR_FLAG_REGION_SIZE, "region_size"},
336	{ CTR_FLAG_RAID10_COPIES, "raid10_copies"},
337	{ CTR_FLAG_RAID10_FORMAT, "raid10_format"},
338	{ CTR_FLAG_DATA_OFFSET, "data_offset"},
339	{ CTR_FLAG_DELTA_DISKS, "delta_disks"},
340	{ CTR_FLAG_RAID10_USE_NEAR_SETS, "raid10_use_near_sets"},
341	{ CTR_FLAG_JOURNAL_DEV, "journal_dev" },
342	{ CTR_FLAG_JOURNAL_MODE, "journal_mode" },
343	};
344
345	/* Return argument name string for given @flag */
346	static const char *dm_raid_arg_name_by_flag(const uint32_t flag)
347	{
348	if (hweight32(flag) == 1) {
349	struct arg_name_flag *anf = __arg_name_flags + ARRAY_SIZE(__arg_name_flags);
350
351	while (anf-- > __arg_name_flags)
352	if (flag & anf->flag)
353	return anf->name;
354
355	} else
356	DMERR("%s called with more than one flag!", __func__);
357
358	return NULL;
359	}
360
361	/* Define correlation of raid456 journal cache modes and dm-raid target line parameters */
362	static struct {
363	const int mode;
364	const char *param;
365	} _raid456_journal_mode[] = {
366	{ R5C_JOURNAL_MODE_WRITE_THROUGH, "writethrough" },
367	{ R5C_JOURNAL_MODE_WRITE_BACK, "writeback" }
368	};
369
370	/* Return MD raid4/5/6 journal mode for dm @journal_mode one */
371	static int dm_raid_journal_mode_to_md(const char *mode)
372	{
373	int m = ARRAY_SIZE(_raid456_journal_mode);
374
375	while (m--)
376	if (!strcasecmp(s1: mode, s2: _raid456_journal_mode[m].param))
377	return _raid456_journal_mode[m].mode;
378
379	return -EINVAL;
380	}
381
382	/* Return dm-raid raid4/5/6 journal mode string for @mode */
383	static const char *md_journal_mode_to_dm_raid(const int mode)
384	{
385	int m = ARRAY_SIZE(_raid456_journal_mode);
386
387	while (m--)
388	if (mode == _raid456_journal_mode[m].mode)
389	return _raid456_journal_mode[m].param;
390
391	return "unknown";
392	}
393
394	/*
395	* Bool helpers to test for various raid levels of a raid set.
396	* It's level as reported by the superblock rather than
397	* the requested raid_type passed to the constructor.
398	*/
399	/* Return true, if raid set in @rs is raid0 */
400	static bool rs_is_raid0(struct raid_set *rs)
401	{
402	return !rs->md.level;
403	}
404
405	/* Return true, if raid set in @rs is raid1 */
406	static bool rs_is_raid1(struct raid_set *rs)
407	{
408	return rs->md.level == 1;
409	}
410
411	/* Return true, if raid set in @rs is raid10 */
412	static bool rs_is_raid10(struct raid_set *rs)
413	{
414	return rs->md.level == 10;
415	}
416
417	/* Return true, if raid set in @rs is level 6 */
418	static bool rs_is_raid6(struct raid_set *rs)
419	{
420	return rs->md.level == 6;
421	}
422
423	/* Return true, if raid set in @rs is level 4, 5 or 6 */
424	static bool rs_is_raid456(struct raid_set *rs)
425	{
426	return __within_range(v: rs->md.level, min: 4, max: 6);
427	}
428
429	/* Return true, if raid set in @rs is reshapable */
430	static bool __is_raid10_far(int layout);
431	static bool rs_is_reshapable(struct raid_set *rs)
432	{
433	return rs_is_raid456(rs) ||
434	(rs_is_raid10(rs) && !__is_raid10_far(layout: rs->md.new_layout));
435	}
436
437	/* Return true, if raid set in @rs is recovering */
438	static bool rs_is_recovering(struct raid_set *rs)
439	{
440	return rs->md.recovery_cp < rs->md.dev_sectors;
441	}
442
443	/* Return true, if raid set in @rs is reshaping */
444	static bool rs_is_reshaping(struct raid_set *rs)
445	{
446	return rs->md.reshape_position != MaxSector;
447	}
448
449	/*
450	* bool helpers to test for various raid levels of a raid type @rt
451	*/
452
453	/* Return true, if raid type in @rt is raid0 */
454	static bool rt_is_raid0(struct raid_type *rt)
455	{
456	return !rt->level;
457	}
458
459	/* Return true, if raid type in @rt is raid1 */
460	static bool rt_is_raid1(struct raid_type *rt)
461	{
462	return rt->level == 1;
463	}
464
465	/* Return true, if raid type in @rt is raid10 */
466	static bool rt_is_raid10(struct raid_type *rt)
467	{
468	return rt->level == 10;
469	}
470
471	/* Return true, if raid type in @rt is raid4/5 */
472	static bool rt_is_raid45(struct raid_type *rt)
473	{
474	return __within_range(v: rt->level, min: 4, max: 5);
475	}
476
477	/* Return true, if raid type in @rt is raid6 */
478	static bool rt_is_raid6(struct raid_type *rt)
479	{
480	return rt->level == 6;
481	}
482
483	/* Return true, if raid type in @rt is raid4/5/6 */
484	static bool rt_is_raid456(struct raid_type *rt)
485	{
486	return __within_range(v: rt->level, min: 4, max: 6);
487	}
488	/* END: raid level bools */
489
490	/* Return valid ctr flags for the raid level of @rs */
491	static unsigned long __valid_flags(struct raid_set *rs)
492	{
493	if (rt_is_raid0(rt: rs->raid_type))
494	return RAID0_VALID_FLAGS;
495	else if (rt_is_raid1(rt: rs->raid_type))
496	return RAID1_VALID_FLAGS;
497	else if (rt_is_raid10(rt: rs->raid_type))
498	return RAID10_VALID_FLAGS;
499	else if (rt_is_raid45(rt: rs->raid_type))
500	return RAID45_VALID_FLAGS;
501	else if (rt_is_raid6(rt: rs->raid_type))
502	return RAID6_VALID_FLAGS;
503
504	return 0;
505	}
506
507	/*
508	* Check for valid flags set on @rs
509	*
510	* Has to be called after parsing of the ctr flags!
511	*/
512	static int rs_check_for_valid_flags(struct raid_set *rs)
513	{
514	if (rs->ctr_flags & ~__valid_flags(rs)) {
515	rs->ti->error = "Invalid flags combination";
516	return -EINVAL;
517	}
518
519	return 0;
520	}
521
522	/* MD raid10 bit definitions and helpers */
523	#define RAID10_OFFSET (1 << 16) /* stripes with data copies area adjacent on devices */
524	#define RAID10_BROCKEN_USE_FAR_SETS (1 << 17) /* Broken in raid10.c: use sets instead of whole stripe rotation */
525	#define RAID10_USE_FAR_SETS (1 << 18) /* Use sets instead of whole stripe rotation */
526	#define RAID10_FAR_COPIES_SHIFT 8 /* raid10 # far copies shift (2nd byte of layout) */
527
528	/* Return md raid10 near copies for @layout */
529	static unsigned int __raid10_near_copies(int layout)
530	{
531	return layout & 0xFF;
532	}
533
534	/* Return md raid10 far copies for @layout */
535	static unsigned int __raid10_far_copies(int layout)
536	{
537	return __raid10_near_copies(layout: layout >> RAID10_FAR_COPIES_SHIFT);
538	}
539
540	/* Return true if md raid10 offset for @layout */
541	static bool __is_raid10_offset(int layout)
542	{
543	return !!(layout & RAID10_OFFSET);
544	}
545
546	/* Return true if md raid10 near for @layout */
547	static bool __is_raid10_near(int layout)
548	{
549	return !__is_raid10_offset(layout) && __raid10_near_copies(layout) > 1;
550	}
551
552	/* Return true if md raid10 far for @layout */
553	static bool __is_raid10_far(int layout)
554	{
555	return !__is_raid10_offset(layout) && __raid10_far_copies(layout) > 1;
556	}
557
558	/* Return md raid10 layout string for @layout */
559	static const char *raid10_md_layout_to_format(int layout)
560	{
561	/*
562	* Bit 16 stands for "offset"
563	* (i.e. adjacent stripes hold copies)
564	*
565	* Refer to MD's raid10.c for details
566	*/
567	if (__is_raid10_offset(layout))
568	return "offset";
569
570	if (__raid10_near_copies(layout) > 1)
571	return "near";
572
573	if (__raid10_far_copies(layout) > 1)
574	return "far";
575
576	return "unknown";
577	}
578
579	/* Return md raid10 algorithm for @name */
580	static int raid10_name_to_format(const char *name)
581	{
582	if (!strcasecmp(s1: name, s2: "near"))
583	return ALGORITHM_RAID10_NEAR;
584	else if (!strcasecmp(s1: name, s2: "offset"))
585	return ALGORITHM_RAID10_OFFSET;
586	else if (!strcasecmp(s1: name, s2: "far"))
587	return ALGORITHM_RAID10_FAR;
588
589	return -EINVAL;
590	}
591
592	/* Return md raid10 copies for @layout */
593	static unsigned int raid10_md_layout_to_copies(int layout)
594	{
595	return max(__raid10_near_copies(layout), __raid10_far_copies(layout));
596	}
597
598	/* Return md raid10 format id for @format string */
599	static int raid10_format_to_md_layout(struct raid_set *rs,
600	unsigned int algorithm,
601	unsigned int copies)
602	{
603	unsigned int n = 1, f = 1, r = 0;
604
605	/*
606	* MD resilienece flaw:
607	*
608	* enabling use_far_sets for far/offset formats causes copies
609	* to be colocated on the same devs together with their origins!
610	*
611	* -> disable it for now in the definition above
612	*/
613	if (algorithm == ALGORITHM_RAID10_DEFAULT ||
614	algorithm == ALGORITHM_RAID10_NEAR)
615	n = copies;
616
617	else if (algorithm == ALGORITHM_RAID10_OFFSET) {
618	f = copies;
619	r = RAID10_OFFSET;
620	if (!test_bit(__CTR_FLAG_RAID10_USE_NEAR_SETS, &rs->ctr_flags))
621	r |= RAID10_USE_FAR_SETS;
622
623	} else if (algorithm == ALGORITHM_RAID10_FAR) {
624	f = copies;
625	if (!test_bit(__CTR_FLAG_RAID10_USE_NEAR_SETS, &rs->ctr_flags))
626	r |= RAID10_USE_FAR_SETS;
627
628	} else
629	return -EINVAL;
630
631	return r | (f << RAID10_FAR_COPIES_SHIFT) | n;
632	}
633	/* END: MD raid10 bit definitions and helpers */
634
635	/* Check for any of the raid10 algorithms */
636	static bool __got_raid10(struct raid_type *rtp, const int layout)
637	{
638	if (rtp->level == 10) {
639	switch (rtp->algorithm) {
640	case ALGORITHM_RAID10_DEFAULT:
641	case ALGORITHM_RAID10_NEAR:
642	return __is_raid10_near(layout);
643	case ALGORITHM_RAID10_OFFSET:
644	return __is_raid10_offset(layout);
645	case ALGORITHM_RAID10_FAR:
646	return __is_raid10_far(layout);
647	default:
648	break;
649	}
650	}
651
652	return false;
653	}
654
655	/* Return raid_type for @name */
656	static struct raid_type *get_raid_type(const char *name)
657	{
658	struct raid_type *rtp = raid_types + ARRAY_SIZE(raid_types);
659
660	while (rtp-- > raid_types)
661	if (!strcasecmp(s1: rtp->name, s2: name))
662	return rtp;
663
664	return NULL;
665	}
666
667	/* Return raid_type for @name based derived from @level and @layout */
668	static struct raid_type *get_raid_type_by_ll(const int level, const int layout)
669	{
670	struct raid_type *rtp = raid_types + ARRAY_SIZE(raid_types);
671
672	while (rtp-- > raid_types) {
673	/* RAID10 special checks based on @layout flags/properties */
674	if (rtp->level == level &&
675	(__got_raid10(rtp, layout) || rtp->algorithm == layout))
676	return rtp;
677	}
678
679	return NULL;
680	}
681
682	/* Adjust rdev sectors */
683	static void rs_set_rdev_sectors(struct raid_set *rs)
684	{
685	struct mddev *mddev = &rs->md;
686	struct md_rdev *rdev;
687
688	/*
689	* raid10 sets rdev->sector to the device size, which
690	* is unintended in case of out-of-place reshaping
691	*/
692	rdev_for_each(rdev, mddev)
693	if (!test_bit(Journal, &rdev->flags))
694	rdev->sectors = mddev->dev_sectors;
695	}
696
697	/*
698	* Change bdev capacity of @rs in case of a disk add/remove reshape
699	*/
700	static void rs_set_capacity(struct raid_set *rs)
701	{
702	struct gendisk *gendisk = dm_disk(md: dm_table_get_md(t: rs->ti->table));
703
704	set_capacity_and_notify(disk: gendisk, size: rs->md.array_sectors);
705	}
706
707	/*
708	* Set the mddev properties in @rs to the current
709	* ones retrieved from the freshest superblock
710	*/
711	static void rs_set_cur(struct raid_set *rs)
712	{
713	struct mddev *mddev = &rs->md;
714
715	mddev->new_level = mddev->level;
716	mddev->new_layout = mddev->layout;
717	mddev->new_chunk_sectors = mddev->chunk_sectors;
718	}
719
720	/*
721	* Set the mddev properties in @rs to the new
722	* ones requested by the ctr
723	*/
724	static void rs_set_new(struct raid_set *rs)
725	{
726	struct mddev *mddev = &rs->md;
727
728	mddev->level = mddev->new_level;
729	mddev->layout = mddev->new_layout;
730	mddev->chunk_sectors = mddev->new_chunk_sectors;
731	mddev->raid_disks = rs->raid_disks;
732	mddev->delta_disks = 0;
733	}
734
735	static struct raid_set *raid_set_alloc(struct dm_target *ti, struct raid_type *raid_type,
736	unsigned int raid_devs)
737	{
738	unsigned int i;
739	struct raid_set *rs;
740
741	if (raid_devs <= raid_type->parity_devs) {
742	ti->error = "Insufficient number of devices";
743	return ERR_PTR(error: -EINVAL);
744	}
745
746	rs = kzalloc(struct_size(rs, dev, raid_devs), GFP_KERNEL);
747	if (!rs) {
748	ti->error = "Cannot allocate raid context";
749	return ERR_PTR(error: -ENOMEM);
750	}
751
752	if (mddev_init(mddev: &rs->md)) {
753	kfree(objp: rs);
754	ti->error = "Cannot initialize raid context";
755	return ERR_PTR(error: -ENOMEM);
756	}
757
758	rs->raid_disks = raid_devs;
759	rs->delta_disks = 0;
760
761	rs->ti = ti;
762	rs->raid_type = raid_type;
763	rs->stripe_cache_entries = 256;
764	rs->md.raid_disks = raid_devs;
765	rs->md.level = raid_type->level;
766	rs->md.new_level = rs->md.level;
767	rs->md.layout = raid_type->algorithm;
768	rs->md.new_layout = rs->md.layout;
769	rs->md.delta_disks = 0;
770	rs->md.recovery_cp = MaxSector;
771
772	for (i = 0; i < raid_devs; i++)
773	md_rdev_init(rdev: &rs->dev[i].rdev);
774
775	/*
776	* Remaining items to be initialized by further RAID params:
777	* rs->md.persistent
778	* rs->md.external
779	* rs->md.chunk_sectors
780	* rs->md.new_chunk_sectors
781	* rs->md.dev_sectors
782	*/
783
784	return rs;
785	}
786
787	/* Free all @rs allocations */
788	static void raid_set_free(struct raid_set *rs)
789	{
790	int i;
791
792	if (rs->journal_dev.dev) {
793	md_rdev_clear(rdev: &rs->journal_dev.rdev);
794	dm_put_device(ti: rs->ti, d: rs->journal_dev.dev);
795	}
796
797	for (i = 0; i < rs->raid_disks; i++) {
798	if (rs->dev[i].meta_dev)
799	dm_put_device(ti: rs->ti, d: rs->dev[i].meta_dev);
800	md_rdev_clear(rdev: &rs->dev[i].rdev);
801	if (rs->dev[i].data_dev)
802	dm_put_device(ti: rs->ti, d: rs->dev[i].data_dev);
803	}
804
805	mddev_destroy(mddev: &rs->md);
806	kfree(objp: rs);
807	}
808
809	/*
810	* For every device we have two words
811	* <meta_dev>: meta device name or '-' if missing
812	* <data_dev>: data device name or '-' if missing
813	*
814	* The following are permitted:
815	* - -
816	* - <data_dev>
817	* <meta_dev> <data_dev>
818	*
819	* The following is not allowed:
820	* <meta_dev> -
821	*
822	* This code parses those words. If there is a failure,
823	* the caller must use raid_set_free() to unwind the operations.
824	*/
825	static int parse_dev_params(struct raid_set *rs, struct dm_arg_set *as)
826	{
827	int i;
828	int rebuild = 0;
829	int metadata_available = 0;
830	int r = 0;
831	const char *arg;
832
833	/* Put off the number of raid devices argument to get to dev pairs */
834	arg = dm_shift_arg(as);
835	if (!arg)
836	return -EINVAL;
837
838	for (i = 0; i < rs->raid_disks; i++) {
839	rs->dev[i].rdev.raid_disk = i;
840
841	rs->dev[i].meta_dev = NULL;
842	rs->dev[i].data_dev = NULL;
843
844	/*
845	* There are no offsets initially.
846	* Out of place reshape will set them accordingly.
847	*/
848	rs->dev[i].rdev.data_offset = 0;
849	rs->dev[i].rdev.new_data_offset = 0;
850	rs->dev[i].rdev.mddev = &rs->md;
851
852	arg = dm_shift_arg(as);
853	if (!arg)
854	return -EINVAL;
855
856	if (strcmp(arg, "-")) {
857	r = dm_get_device(ti: rs->ti, path: arg, mode: dm_table_get_mode(t: rs->ti->table),
858	result: &rs->dev[i].meta_dev);
859	if (r) {
860	rs->ti->error = "RAID metadata device lookup failure";
861	return r;
862	}
863
864	rs->dev[i].rdev.sb_page = alloc_page(GFP_KERNEL);
865	if (!rs->dev[i].rdev.sb_page) {
866	rs->ti->error = "Failed to allocate superblock page";
867	return -ENOMEM;
868	}
869	}
870
871	arg = dm_shift_arg(as);
872	if (!arg)
873	return -EINVAL;
874
875	if (!strcmp(arg, "-")) {
876	if (!test_bit(In_sync, &rs->dev[i].rdev.flags) &&
877	(!rs->dev[i].rdev.recovery_offset)) {
878	rs->ti->error = "Drive designated for rebuild not specified";
879	return -EINVAL;
880	}
881
882	if (rs->dev[i].meta_dev) {
883	rs->ti->error = "No data device supplied with metadata device";
884	return -EINVAL;
885	}
886
887	continue;
888	}
889
890	r = dm_get_device(ti: rs->ti, path: arg, mode: dm_table_get_mode(t: rs->ti->table),
891	result: &rs->dev[i].data_dev);
892	if (r) {
893	rs->ti->error = "RAID device lookup failure";
894	return r;
895	}
896
897	if (rs->dev[i].meta_dev) {
898	metadata_available = 1;
899	rs->dev[i].rdev.meta_bdev = rs->dev[i].meta_dev->bdev;
900	}
901	rs->dev[i].rdev.bdev = rs->dev[i].data_dev->bdev;
902	list_add_tail(new: &rs->dev[i].rdev.same_set, head: &rs->md.disks);
903	if (!test_bit(In_sync, &rs->dev[i].rdev.flags))
904	rebuild++;
905	}
906
907	if (rs->journal_dev.dev)
908	list_add_tail(new: &rs->journal_dev.rdev.same_set, head: &rs->md.disks);
909
910	if (metadata_available) {
911	rs->md.external = 0;
912	rs->md.persistent = 1;
913	rs->md.major_version = 2;
914	} else if (rebuild && !rs->md.recovery_cp) {
915	/*
916	* Without metadata, we will not be able to tell if the array
917	* is in-sync or not - we must assume it is not. Therefore,
918	* it is impossible to rebuild a drive.
919	*
920	* Even if there is metadata, the on-disk information may
921	* indicate that the array is not in-sync and it will then
922	* fail at that time.
923	*
924	* User could specify 'nosync' option if desperate.
925	*/
926	rs->ti->error = "Unable to rebuild drive while array is not in-sync";
927	return -EINVAL;
928	}
929
930	return 0;
931	}
932
933	/*
934	* validate_region_size
935	* @rs
936	* @region_size: region size in sectors. If 0, pick a size (4MiB default).
937	*
938	* Set rs->md.bitmap_info.chunksize (which really refers to 'region size').
939	* Ensure that (ti->len/region_size < 2^21) - required by MD bitmap.
940	*
941	* Returns: 0 on success, -EINVAL on failure.
942	*/
943	static int validate_region_size(struct raid_set *rs, unsigned long region_size)
944	{
945	unsigned long min_region_size = rs->ti->len / (1 << 21);
946
947	if (rs_is_raid0(rs))
948	return 0;
949
950	if (!region_size) {
951	/*
952	* Choose a reasonable default. All figures in sectors.
953	*/
954	if (min_region_size > (1 << 13)) {
955	/* If not a power of 2, make it the next power of 2 */
956	region_size = roundup_pow_of_two(min_region_size);
957	DMINFO("Choosing default region size of %lu sectors",
958	region_size);
959	} else {
960	DMINFO("Choosing default region size of 4MiB");
961	region_size = 1 << 13; /* sectors */
962	}
963	} else {
964	/*
965	* Validate user-supplied value.
966	*/
967	if (region_size > rs->ti->len) {
968	rs->ti->error = "Supplied region size is too large";
969	return -EINVAL;
970	}
971
972	if (region_size < min_region_size) {
973	DMERR("Supplied region_size (%lu sectors) below minimum (%lu)",
974	region_size, min_region_size);
975	rs->ti->error = "Supplied region size is too small";
976	return -EINVAL;
977	}
978
979	if (!is_power_of_2(n: region_size)) {
980	rs->ti->error = "Region size is not a power of 2";
981	return -EINVAL;
982	}
983
984	if (region_size < rs->md.chunk_sectors) {
985	rs->ti->error = "Region size is smaller than the chunk size";
986	return -EINVAL;
987	}
988	}
989
990	/*
991	* Convert sectors to bytes.
992	*/
993	rs->md.bitmap_info.chunksize = to_bytes(n: region_size);
994
995	return 0;
996	}
997
998	/*
999	* validate_raid_redundancy
1000	* @rs
1001	*
1002	* Determine if there are enough devices in the array that haven't
1003	* failed (or are being rebuilt) to form a usable array.
1004	*
1005	* Returns: 0 on success, -EINVAL on failure.
1006	*/
1007	static int validate_raid_redundancy(struct raid_set *rs)
1008	{
1009	unsigned int i, rebuild_cnt = 0;
1010	unsigned int rebuilds_per_group = 0, copies, raid_disks;
1011	unsigned int group_size, last_group_start;
1012
1013	for (i = 0; i < rs->raid_disks; i++)
1014	if (!test_bit(FirstUse, &rs->dev[i].rdev.flags) &&
1015	((!test_bit(In_sync, &rs->dev[i].rdev.flags) ||
1016	!rs->dev[i].rdev.sb_page)))
1017	rebuild_cnt++;
1018
1019	switch (rs->md.level) {
1020	case 0:
1021	break;
1022	case 1:
1023	if (rebuild_cnt >= rs->md.raid_disks)
1024	goto too_many;
1025	break;
1026	case 4:
1027	case 5:
1028	case 6:
1029	if (rebuild_cnt > rs->raid_type->parity_devs)
1030	goto too_many;
1031	break;
1032	case 10:
1033	copies = raid10_md_layout_to_copies(layout: rs->md.new_layout);
1034	if (copies < 2) {
1035	DMERR("Bogus raid10 data copies < 2!");
1036	return -EINVAL;
1037	}
1038
1039	if (rebuild_cnt < copies)
1040	break;
1041
1042	/*
1043	* It is possible to have a higher rebuild count for RAID10,
1044	* as long as the failed devices occur in different mirror
1045	* groups (i.e. different stripes).
1046	*
1047	* When checking "near" format, make sure no adjacent devices
1048	* have failed beyond what can be handled. In addition to the
1049	* simple case where the number of devices is a multiple of the
1050	* number of copies, we must also handle cases where the number
1051	* of devices is not a multiple of the number of copies.
1052	* E.g. dev1 dev2 dev3 dev4 dev5
1053	* A A B B C
1054	* C D D E E
1055	*/
1056	raid_disks = min(rs->raid_disks, rs->md.raid_disks);
1057	if (__is_raid10_near(layout: rs->md.new_layout)) {
1058	for (i = 0; i < raid_disks; i++) {
1059	if (!(i % copies))
1060	rebuilds_per_group = 0;
1061	if ((!rs->dev[i].rdev.sb_page ||
1062	!test_bit(In_sync, &rs->dev[i].rdev.flags)) &&
1063	(++rebuilds_per_group >= copies))
1064	goto too_many;
1065	}
1066	break;
1067	}
1068
1069	/*
1070	* When checking "far" and "offset" formats, we need to ensure
1071	* that the device that holds its copy is not also dead or
1072	* being rebuilt. (Note that "far" and "offset" formats only
1073	* support two copies right now. These formats also only ever
1074	* use the 'use_far_sets' variant.)
1075	*
1076	* This check is somewhat complicated by the need to account
1077	* for arrays that are not a multiple of (far) copies. This
1078	* results in the need to treat the last (potentially larger)
1079	* set differently.
1080	*/
1081	group_size = (raid_disks / copies);
1082	last_group_start = (raid_disks / group_size) - 1;
1083	last_group_start *= group_size;
1084	for (i = 0; i < raid_disks; i++) {
1085	if (!(i % copies) && !(i > last_group_start))
1086	rebuilds_per_group = 0;
1087	if ((!rs->dev[i].rdev.sb_page ||
1088	!test_bit(In_sync, &rs->dev[i].rdev.flags)) &&
1089	(++rebuilds_per_group >= copies))
1090	goto too_many;
1091	}
1092	break;
1093	default:
1094	if (rebuild_cnt)
1095	return -EINVAL;
1096	}
1097
1098	return 0;
1099
1100	too_many:
1101	return -EINVAL;
1102	}
1103
1104	/*
1105	* Possible arguments are...
1106	* <chunk_size> [optional_args]
1107	*
1108	* Argument definitions
1109	* <chunk_size> The number of sectors per disk that
1110	* will form the "stripe"
1111	* [[no]sync] Force or prevent recovery of the
1112	* entire array
1113	* [rebuild <idx>] Rebuild the drive indicated by the index
1114	* [daemon_sleep <ms>] Time between bitmap daemon work to
1115	* clear bits
1116	* [min_recovery_rate <kB/sec/disk>] Throttle RAID initialization
1117	* [max_recovery_rate <kB/sec/disk>] Throttle RAID initialization
1118	* [write_mostly <idx>] Indicate a write mostly drive via index
1119	* [max_write_behind <sectors>] See '-write-behind=' (man mdadm)
1120	* [stripe_cache <sectors>] Stripe cache size for higher RAIDs
1121	* [region_size <sectors>] Defines granularity of bitmap
1122	* [journal_dev <dev>] raid4/5/6 journaling deviice
1123	* (i.e. write hole closing log)
1124	*
1125	* RAID10-only options:
1126	* [raid10_copies <# copies>] Number of copies. (Default: 2)
1127	* [raid10_format <near|far|offset>] Layout algorithm. (Default: near)
1128	*/
1129	static int parse_raid_params(struct raid_set *rs, struct dm_arg_set *as,
1130	unsigned int num_raid_params)
1131	{
1132	int value, raid10_format = ALGORITHM_RAID10_DEFAULT;
1133	unsigned int raid10_copies = 2;
1134	unsigned int i, write_mostly = 0;
1135	unsigned int region_size = 0;
1136	sector_t max_io_len;
1137	const char *arg, *key;
1138	struct raid_dev *rd;
1139	struct raid_type *rt = rs->raid_type;
1140
1141	arg = dm_shift_arg(as);
1142	num_raid_params--; /* Account for chunk_size argument */
1143
1144	if (kstrtoint(s: arg, base: 10, res: &value) < 0) {
1145	rs->ti->error = "Bad numerical argument given for chunk_size";
1146	return -EINVAL;
1147	}
1148
1149	/*
1150	* First, parse the in-order required arguments
1151	* "chunk_size" is the only argument of this type.
1152	*/
1153	if (rt_is_raid1(rt)) {
1154	if (value)
1155	DMERR("Ignoring chunk size parameter for RAID 1");
1156	value = 0;
1157	} else if (!is_power_of_2(n: value)) {
1158	rs->ti->error = "Chunk size must be a power of 2";
1159	return -EINVAL;
1160	} else if (value < 8) {
1161	rs->ti->error = "Chunk size value is too small";
1162	return -EINVAL;
1163	}
1164
1165	rs->md.new_chunk_sectors = rs->md.chunk_sectors = value;
1166
1167	/*
1168	* We set each individual device as In_sync with a completed
1169	* 'recovery_offset'. If there has been a device failure or
1170	* replacement then one of the following cases applies:
1171	*
1172	* 1) User specifies 'rebuild'.
1173	* - Device is reset when param is read.
1174	* 2) A new device is supplied.
1175	* - No matching superblock found, resets device.
1176	* 3) Device failure was transient and returns on reload.
1177	* - Failure noticed, resets device for bitmap replay.
1178	* 4) Device hadn't completed recovery after previous failure.
1179	* - Superblock is read and overrides recovery_offset.
1180	*
1181	* What is found in the superblocks of the devices is always
1182	* authoritative, unless 'rebuild' or '[no]sync' was specified.
1183	*/
1184	for (i = 0; i < rs->raid_disks; i++) {
1185	set_bit(nr: In_sync, addr: &rs->dev[i].rdev.flags);
1186	rs->dev[i].rdev.recovery_offset = MaxSector;
1187	}
1188
1189	/*
1190	* Second, parse the unordered optional arguments
1191	*/
1192	for (i = 0; i < num_raid_params; i++) {
1193	key = dm_shift_arg(as);
1194	if (!key) {
1195	rs->ti->error = "Not enough raid parameters given";
1196	return -EINVAL;
1197	}
1198
1199	if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_NOSYNC))) {
1200	if (test_and_set_bit(__CTR_FLAG_NOSYNC, addr: &rs->ctr_flags)) {
1201	rs->ti->error = "Only one 'nosync' argument allowed";
1202	return -EINVAL;
1203	}
1204	continue;
1205	}
1206	if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_SYNC))) {
1207	if (test_and_set_bit(__CTR_FLAG_SYNC, addr: &rs->ctr_flags)) {
1208	rs->ti->error = "Only one 'sync' argument allowed";
1209	return -EINVAL;
1210	}
1211	continue;
1212	}
1213	if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_RAID10_USE_NEAR_SETS))) {
1214	if (test_and_set_bit(__CTR_FLAG_RAID10_USE_NEAR_SETS, addr: &rs->ctr_flags)) {
1215	rs->ti->error = "Only one 'raid10_use_new_sets' argument allowed";
1216	return -EINVAL;
1217	}
1218	continue;
1219	}
1220
1221	arg = dm_shift_arg(as);
1222	i++; /* Account for the argument pairs */
1223	if (!arg) {
1224	rs->ti->error = "Wrong number of raid parameters given";
1225	return -EINVAL;
1226	}
1227
1228	/*
1229	* Parameters that take a string value are checked here.
1230	*/
1231	/* "raid10_format {near|offset|far} */
1232	if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_RAID10_FORMAT))) {
1233	if (test_and_set_bit(__CTR_FLAG_RAID10_FORMAT, addr: &rs->ctr_flags)) {
1234	rs->ti->error = "Only one 'raid10_format' argument pair allowed";
1235	return -EINVAL;
1236	}
1237	if (!rt_is_raid10(rt)) {
1238	rs->ti->error = "'raid10_format' is an invalid parameter for this RAID type";
1239	return -EINVAL;
1240	}
1241	raid10_format = raid10_name_to_format(name: arg);
1242	if (raid10_format < 0) {
1243	rs->ti->error = "Invalid 'raid10_format' value given";
1244	return raid10_format;
1245	}
1246	continue;
1247	}
1248
1249	/* "journal_dev <dev>" */
1250	if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_JOURNAL_DEV))) {
1251	int r;
1252	struct md_rdev *jdev;
1253
1254	if (test_and_set_bit(__CTR_FLAG_JOURNAL_DEV, addr: &rs->ctr_flags)) {
1255	rs->ti->error = "Only one raid4/5/6 set journaling device allowed";
1256	return -EINVAL;
1257	}
1258	if (!rt_is_raid456(rt)) {
1259	rs->ti->error = "'journal_dev' is an invalid parameter for this RAID type";
1260	return -EINVAL;
1261	}
1262	r = dm_get_device(ti: rs->ti, path: arg, mode: dm_table_get_mode(t: rs->ti->table),
1263	result: &rs->journal_dev.dev);
1264	if (r) {
1265	rs->ti->error = "raid4/5/6 journal device lookup failure";
1266	return r;
1267	}
1268	jdev = &rs->journal_dev.rdev;
1269	md_rdev_init(rdev: jdev);
1270	jdev->mddev = &rs->md;
1271	jdev->bdev = rs->journal_dev.dev->bdev;
1272	jdev->sectors = bdev_nr_sectors(bdev: jdev->bdev);
1273	if (jdev->sectors < MIN_RAID456_JOURNAL_SPACE) {
1274	rs->ti->error = "No space for raid4/5/6 journal";
1275	return -ENOSPC;
1276	}
1277	rs->journal_dev.mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
1278	set_bit(nr: Journal, addr: &jdev->flags);
1279	continue;
1280	}
1281
1282	/* "journal_mode <mode>" ("journal_dev" mandatory!) */
1283	if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_JOURNAL_MODE))) {
1284	int r;
1285
1286	if (!test_bit(__CTR_FLAG_JOURNAL_DEV, &rs->ctr_flags)) {
1287	rs->ti->error = "raid4/5/6 'journal_mode' is invalid without 'journal_dev'";
1288	return -EINVAL;
1289	}
1290	if (test_and_set_bit(__CTR_FLAG_JOURNAL_MODE, addr: &rs->ctr_flags)) {
1291	rs->ti->error = "Only one raid4/5/6 'journal_mode' argument allowed";
1292	return -EINVAL;
1293	}
1294	r = dm_raid_journal_mode_to_md(mode: arg);
1295	if (r < 0) {
1296	rs->ti->error = "Invalid 'journal_mode' argument";
1297	return r;
1298	}
1299	rs->journal_dev.mode = r;
1300	continue;
1301	}
1302
1303	/*
1304	* Parameters with number values from here on.
1305	*/
1306	if (kstrtoint(s: arg, base: 10, res: &value) < 0) {
1307	rs->ti->error = "Bad numerical argument given in raid params";
1308	return -EINVAL;
1309	}
1310
1311	if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_REBUILD))) {
1312	/*
1313	* "rebuild" is being passed in by userspace to provide
1314	* indexes of replaced devices and to set up additional
1315	* devices on raid level takeover.
1316	*/
1317	if (!__within_range(v: value, min: 0, max: rs->raid_disks - 1)) {
1318	rs->ti->error = "Invalid rebuild index given";
1319	return -EINVAL;
1320	}
1321
1322	if (test_and_set_bit(nr: value, addr: (void *) rs->rebuild_disks)) {
1323	rs->ti->error = "rebuild for this index already given";
1324	return -EINVAL;
1325	}
1326
1327	rd = rs->dev + value;
1328	clear_bit(nr: In_sync, addr: &rd->rdev.flags);
1329	clear_bit(nr: Faulty, addr: &rd->rdev.flags);
1330	rd->rdev.recovery_offset = 0;
1331	set_bit(__CTR_FLAG_REBUILD, addr: &rs->ctr_flags);
1332	} else if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_WRITE_MOSTLY))) {
1333	if (!rt_is_raid1(rt)) {
1334	rs->ti->error = "write_mostly option is only valid for RAID1";
1335	return -EINVAL;
1336	}
1337
1338	if (!__within_range(v: value, min: 0, max: rs->md.raid_disks - 1)) {
1339	rs->ti->error = "Invalid write_mostly index given";
1340	return -EINVAL;
1341	}
1342
1343	write_mostly++;
1344	set_bit(nr: WriteMostly, addr: &rs->dev[value].rdev.flags);
1345	set_bit(__CTR_FLAG_WRITE_MOSTLY, addr: &rs->ctr_flags);
1346	} else if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_MAX_WRITE_BEHIND))) {
1347	if (!rt_is_raid1(rt)) {
1348	rs->ti->error = "max_write_behind option is only valid for RAID1";
1349	return -EINVAL;
1350	}
1351
1352	if (test_and_set_bit(__CTR_FLAG_MAX_WRITE_BEHIND, addr: &rs->ctr_flags)) {
1353	rs->ti->error = "Only one max_write_behind argument pair allowed";
1354	return -EINVAL;
1355	}
1356
1357	/*
1358	* In device-mapper, we specify things in sectors, but
1359	* MD records this value in kB
1360	*/
1361	if (value < 0 || value / 2 > COUNTER_MAX) {
1362	rs->ti->error = "Max write-behind limit out of range";
1363	return -EINVAL;
1364	}
1365
1366	rs->md.bitmap_info.max_write_behind = value / 2;
1367	} else if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_DAEMON_SLEEP))) {
1368	if (test_and_set_bit(__CTR_FLAG_DAEMON_SLEEP, addr: &rs->ctr_flags)) {
1369	rs->ti->error = "Only one daemon_sleep argument pair allowed";
1370	return -EINVAL;
1371	}
1372	if (value < 0) {
1373	rs->ti->error = "daemon sleep period out of range";
1374	return -EINVAL;
1375	}
1376	rs->md.bitmap_info.daemon_sleep = value;
1377	} else if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_DATA_OFFSET))) {
1378	/* Userspace passes new data_offset after having extended the data image LV */
1379	if (test_and_set_bit(__CTR_FLAG_DATA_OFFSET, addr: &rs->ctr_flags)) {
1380	rs->ti->error = "Only one data_offset argument pair allowed";
1381	return -EINVAL;
1382	}
1383	/* Ensure sensible data offset */
1384	if (value < 0 ||
1385	(value && (value < MIN_FREE_RESHAPE_SPACE || value % to_sector(PAGE_SIZE)))) {
1386	rs->ti->error = "Bogus data_offset value";
1387	return -EINVAL;
1388	}
1389	rs->data_offset = value;
1390	} else if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_DELTA_DISKS))) {
1391	/* Define the +/-# of disks to add to/remove from the given raid set */
1392	if (test_and_set_bit(__CTR_FLAG_DELTA_DISKS, addr: &rs->ctr_flags)) {
1393	rs->ti->error = "Only one delta_disks argument pair allowed";
1394	return -EINVAL;
1395	}
1396	/* Ensure MAX_RAID_DEVICES and raid type minimal_devs! */
1397	if (!__within_range(abs(value), min: 1, MAX_RAID_DEVICES - rt->minimal_devs)) {
1398	rs->ti->error = "Too many delta_disk requested";
1399	return -EINVAL;
1400	}
1401
1402	rs->delta_disks = value;
1403	} else if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_STRIPE_CACHE))) {
1404	if (test_and_set_bit(__CTR_FLAG_STRIPE_CACHE, addr: &rs->ctr_flags)) {
1405	rs->ti->error = "Only one stripe_cache argument pair allowed";
1406	return -EINVAL;
1407	}
1408
1409	if (!rt_is_raid456(rt)) {
1410	rs->ti->error = "Inappropriate argument: stripe_cache";
1411	return -EINVAL;
1412	}
1413
1414	if (value < 0) {
1415	rs->ti->error = "Bogus stripe cache entries value";
1416	return -EINVAL;
1417	}
1418	rs->stripe_cache_entries = value;
1419	} else if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_MIN_RECOVERY_RATE))) {
1420	if (test_and_set_bit(__CTR_FLAG_MIN_RECOVERY_RATE, addr: &rs->ctr_flags)) {
1421	rs->ti->error = "Only one min_recovery_rate argument pair allowed";
1422	return -EINVAL;
1423	}
1424
1425	if (value < 0) {
1426	rs->ti->error = "min_recovery_rate out of range";
1427	return -EINVAL;
1428	}
1429	rs->md.sync_speed_min = value;
1430	} else if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_MAX_RECOVERY_RATE))) {
1431	if (test_and_set_bit(__CTR_FLAG_MAX_RECOVERY_RATE, addr: &rs->ctr_flags)) {
1432	rs->ti->error = "Only one max_recovery_rate argument pair allowed";
1433	return -EINVAL;
1434	}
1435
1436	if (value < 0) {
1437	rs->ti->error = "max_recovery_rate out of range";
1438	return -EINVAL;
1439	}
1440	rs->md.sync_speed_max = value;
1441	} else if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_REGION_SIZE))) {
1442	if (test_and_set_bit(__CTR_FLAG_REGION_SIZE, addr: &rs->ctr_flags)) {
1443	rs->ti->error = "Only one region_size argument pair allowed";
1444	return -EINVAL;
1445	}
1446
1447	region_size = value;
1448	rs->requested_bitmap_chunk_sectors = value;
1449	} else if (!strcasecmp(s1: key, s2: dm_raid_arg_name_by_flag(CTR_FLAG_RAID10_COPIES))) {
1450	if (test_and_set_bit(__CTR_FLAG_RAID10_COPIES, addr: &rs->ctr_flags)) {
1451	rs->ti->error = "Only one raid10_copies argument pair allowed";
1452	return -EINVAL;
1453	}
1454
1455	if (!__within_range(v: value, min: 2, max: rs->md.raid_disks)) {
1456	rs->ti->error = "Bad value for 'raid10_copies'";
1457	return -EINVAL;
1458	}
1459
1460	raid10_copies = value;
1461	} else {
1462	DMERR("Unable to parse RAID parameter: %s", key);
1463	rs->ti->error = "Unable to parse RAID parameter";
1464	return -EINVAL;
1465	}
1466	}
1467
1468	if (test_bit(__CTR_FLAG_SYNC, &rs->ctr_flags) &&
1469	test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags)) {
1470	rs->ti->error = "sync and nosync are mutually exclusive";
1471	return -EINVAL;
1472	}
1473
1474	if (test_bit(__CTR_FLAG_REBUILD, &rs->ctr_flags) &&
1475	(test_bit(__CTR_FLAG_SYNC, &rs->ctr_flags) ||
1476	test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags))) {
1477	rs->ti->error = "sync/nosync and rebuild are mutually exclusive";
1478	return -EINVAL;
1479	}
1480
1481	if (write_mostly >= rs->md.raid_disks) {
1482	rs->ti->error = "Can't set all raid1 devices to write_mostly";
1483	return -EINVAL;
1484	}
1485
1486	if (rs->md.sync_speed_max &&
1487	rs->md.sync_speed_min > rs->md.sync_speed_max) {
1488	rs->ti->error = "Bogus recovery rates";
1489	return -EINVAL;
1490	}
1491
1492	if (validate_region_size(rs, region_size))
1493	return -EINVAL;
1494
1495	if (rs->md.chunk_sectors)
1496	max_io_len = rs->md.chunk_sectors;
1497	else
1498	max_io_len = region_size;
1499
1500	if (dm_set_target_max_io_len(ti: rs->ti, len: max_io_len))
1501	return -EINVAL;
1502
1503	if (rt_is_raid10(rt)) {
1504	if (raid10_copies > rs->md.raid_disks) {
1505	rs->ti->error = "Not enough devices to satisfy specification";
1506	return -EINVAL;
1507	}
1508
1509	rs->md.new_layout = raid10_format_to_md_layout(rs, algorithm: raid10_format, copies: raid10_copies);
1510	if (rs->md.new_layout < 0) {
1511	rs->ti->error = "Error getting raid10 format";
1512	return rs->md.new_layout;
1513	}
1514
1515	rt = get_raid_type_by_ll(level: 10, layout: rs->md.new_layout);
1516	if (!rt) {
1517	rs->ti->error = "Failed to recognize new raid10 layout";
1518	return -EINVAL;
1519	}
1520
1521	if ((rt->algorithm == ALGORITHM_RAID10_DEFAULT ||
1522	rt->algorithm == ALGORITHM_RAID10_NEAR) &&
1523	test_bit(__CTR_FLAG_RAID10_USE_NEAR_SETS, &rs->ctr_flags)) {
1524	rs->ti->error = "RAID10 format 'near' and 'raid10_use_near_sets' are incompatible";
1525	return -EINVAL;
1526	}
1527	}
1528
1529	rs->raid10_copies = raid10_copies;
1530
1531	/* Assume there are no metadata devices until the drives are parsed */
1532	rs->md.persistent = 0;
1533	rs->md.external = 1;
1534
1535	/* Check, if any invalid ctr arguments have been passed in for the raid level */
1536	return rs_check_for_valid_flags(rs);
1537	}
1538
1539	/* Set raid4/5/6 cache size */
1540	static int rs_set_raid456_stripe_cache(struct raid_set *rs)
1541	{
1542	int r;
1543	struct r5conf *conf;
1544	struct mddev *mddev = &rs->md;
1545	uint32_t min_stripes = max(mddev->chunk_sectors, mddev->new_chunk_sectors) / 2;
1546	uint32_t nr_stripes = rs->stripe_cache_entries;
1547
1548	if (!rt_is_raid456(rt: rs->raid_type)) {
1549	rs->ti->error = "Inappropriate raid level; cannot change stripe_cache size";
1550	return -EINVAL;
1551	}
1552
1553	if (nr_stripes < min_stripes) {
1554	DMINFO("Adjusting requested %u stripe cache entries to %u to suit stripe size",
1555	nr_stripes, min_stripes);
1556	nr_stripes = min_stripes;
1557	}
1558
1559	conf = mddev->private;
1560	if (!conf) {
1561	rs->ti->error = "Cannot change stripe_cache size on inactive RAID set";
1562	return -EINVAL;
1563	}
1564
1565	/* Try setting number of stripes in raid456 stripe cache */
1566	if (conf->min_nr_stripes != nr_stripes) {
1567	r = raid5_set_cache_size(mddev, size: nr_stripes);
1568	if (r) {
1569	rs->ti->error = "Failed to set raid4/5/6 stripe cache size";
1570	return r;
1571	}
1572
1573	DMINFO("%u stripe cache entries", nr_stripes);
1574	}
1575
1576	return 0;
1577	}
1578
1579	/* Return # of data stripes as kept in mddev as of @rs (i.e. as of superblock) */
1580	static unsigned int mddev_data_stripes(struct raid_set *rs)
1581	{
1582	return rs->md.raid_disks - rs->raid_type->parity_devs;
1583	}
1584
1585	/* Return # of data stripes of @rs (i.e. as of ctr) */
1586	static unsigned int rs_data_stripes(struct raid_set *rs)
1587	{
1588	return rs->raid_disks - rs->raid_type->parity_devs;
1589	}
1590
1591	/*
1592	* Retrieve rdev->sectors from any valid raid device of @rs
1593	* to allow userpace to pass in arbitray "- -" device tupples.
1594	*/
1595	static sector_t __rdev_sectors(struct raid_set *rs)
1596	{
1597	int i;
1598
1599	for (i = 0; i < rs->raid_disks; i++) {
1600	struct md_rdev *rdev = &rs->dev[i].rdev;
1601
1602	if (!test_bit(Journal, &rdev->flags) &&
1603	rdev->bdev && rdev->sectors)
1604	return rdev->sectors;
1605	}
1606
1607	return 0;
1608	}
1609
1610	/* Check that calculated dev_sectors fits all component devices. */
1611	static int _check_data_dev_sectors(struct raid_set *rs)
1612	{
1613	sector_t ds = ~0;
1614	struct md_rdev *rdev;
1615
1616	rdev_for_each(rdev, &rs->md)
1617	if (!test_bit(Journal, &rdev->flags) && rdev->bdev) {
1618	ds = min(ds, bdev_nr_sectors(rdev->bdev));
1619	if (ds < rs->md.dev_sectors) {
1620	rs->ti->error = "Component device(s) too small";
1621	return -EINVAL;
1622	}
1623	}
1624
1625	return 0;
1626	}
1627
1628	/* Calculate the sectors per device and per array used for @rs */
1629	static int rs_set_dev_and_array_sectors(struct raid_set *rs, sector_t sectors, bool use_mddev)
1630	{
1631	int delta_disks;
1632	unsigned int data_stripes;
1633	sector_t array_sectors = sectors, dev_sectors = sectors;
1634	struct mddev *mddev = &rs->md;
1635
1636	if (use_mddev) {
1637	delta_disks = mddev->delta_disks;
1638	data_stripes = mddev_data_stripes(rs);
1639	} else {
1640	delta_disks = rs->delta_disks;
1641	data_stripes = rs_data_stripes(rs);
1642	}
1643
1644	/* Special raid1 case w/o delta_disks support (yet) */
1645	if (rt_is_raid1(rt: rs->raid_type))
1646	;
1647	else if (rt_is_raid10(rt: rs->raid_type)) {
1648	if (rs->raid10_copies < 2 ||
1649	delta_disks < 0) {
1650	rs->ti->error = "Bogus raid10 data copies or delta disks";
1651	return -EINVAL;
1652	}
1653
1654	dev_sectors *= rs->raid10_copies;
1655	if (sector_div(dev_sectors, data_stripes))
1656	goto bad;
1657
1658	array_sectors = (data_stripes + delta_disks) * dev_sectors;
1659	if (sector_div(array_sectors, rs->raid10_copies))
1660	goto bad;
1661
1662	} else if (sector_div(dev_sectors, data_stripes))
1663	goto bad;
1664
1665	else
1666	/* Striped layouts */
1667	array_sectors = (data_stripes + delta_disks) * dev_sectors;
1668
1669	mddev->array_sectors = array_sectors;
1670	mddev->dev_sectors = dev_sectors;
1671	rs_set_rdev_sectors(rs);
1672
1673	return _check_data_dev_sectors(rs);
1674	bad:
1675	rs->ti->error = "Target length not divisible by number of data devices";
1676	return -EINVAL;
1677	}
1678
1679	/* Setup recovery on @rs */
1680	static void rs_setup_recovery(struct raid_set *rs, sector_t dev_sectors)
1681	{
1682	/* raid0 does not recover */
1683	if (rs_is_raid0(rs))
1684	rs->md.recovery_cp = MaxSector;
1685	/*
1686	* A raid6 set has to be recovered either
1687	* completely or for the grown part to
1688	* ensure proper parity and Q-Syndrome
1689	*/
1690	else if (rs_is_raid6(rs))
1691	rs->md.recovery_cp = dev_sectors;
1692	/*
1693	* Other raid set types may skip recovery
1694	* depending on the 'nosync' flag.
1695	*/
1696	else
1697	rs->md.recovery_cp = test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags)
1698	? MaxSector : dev_sectors;
1699	}
1700
1701	static void do_table_event(struct work_struct *ws)
1702	{
1703	struct raid_set *rs = container_of(ws, struct raid_set, md.event_work);
1704
1705	smp_rmb(); /* Make sure we access most actual mddev properties */
1706	if (!rs_is_reshaping(rs)) {
1707	if (rs_is_raid10(rs))
1708	rs_set_rdev_sectors(rs);
1709	rs_set_capacity(rs);
1710	}
1711	dm_table_event(t: rs->ti->table);
1712	}
1713
1714	/*
1715	* Make sure a valid takover (level switch) is being requested on @rs
1716	*
1717	* Conversions of raid sets from one MD personality to another
1718	* have to conform to restrictions which are enforced here.
1719	*/
1720	static int rs_check_takeover(struct raid_set *rs)
1721	{
1722	struct mddev *mddev = &rs->md;
1723	unsigned int near_copies;
1724
1725	if (rs->md.degraded) {
1726	rs->ti->error = "Can't takeover degraded raid set";
1727	return -EPERM;
1728	}
1729
1730	if (rs_is_reshaping(rs)) {
1731	rs->ti->error = "Can't takeover reshaping raid set";
1732	return -EPERM;
1733	}
1734
1735	switch (mddev->level) {
1736	case 0:
1737	/* raid0 -> raid1/5 with one disk */
1738	if ((mddev->new_level == 1 || mddev->new_level == 5) &&
1739	mddev->raid_disks == 1)
1740	return 0;
1741
1742	/* raid0 -> raid10 */
1743	if (mddev->new_level == 10 &&
1744	!(rs->raid_disks % mddev->raid_disks))
1745	return 0;
1746
1747	/* raid0 with multiple disks -> raid4/5/6 */
1748	if (__within_range(v: mddev->new_level, min: 4, max: 6) &&
1749	mddev->new_layout == ALGORITHM_PARITY_N &&
1750	mddev->raid_disks > 1)
1751	return 0;
1752
1753	break;
1754
1755	case 10:
1756	/* Can't takeover raid10_offset! */
1757	if (__is_raid10_offset(layout: mddev->layout))
1758	break;
1759
1760	near_copies = __raid10_near_copies(layout: mddev->layout);
1761
1762	/* raid10* -> raid0 */
1763	if (mddev->new_level == 0) {
1764	/* Can takeover raid10_near with raid disks divisable by data copies! */
1765	if (near_copies > 1 &&
1766	!(mddev->raid_disks % near_copies)) {
1767	mddev->raid_disks /= near_copies;
1768	mddev->delta_disks = mddev->raid_disks;
1769	return 0;
1770	}
1771
1772	/* Can takeover raid10_far */
1773	if (near_copies == 1 &&
1774	__raid10_far_copies(layout: mddev->layout) > 1)
1775	return 0;
1776
1777	break;
1778	}
1779
1780	/* raid10_{near,far} -> raid1 */
1781	if (mddev->new_level == 1 &&
1782	max(near_copies, __raid10_far_copies(mddev->layout)) == mddev->raid_disks)
1783	return 0;
1784
1785	/* raid10_{near,far} with 2 disks -> raid4/5 */
1786	if (__within_range(v: mddev->new_level, min: 4, max: 5) &&
1787	mddev->raid_disks == 2)
1788	return 0;
1789	break;
1790
1791	case 1:
1792	/* raid1 with 2 disks -> raid4/5 */
1793	if (__within_range(v: mddev->new_level, min: 4, max: 5) &&
1794	mddev->raid_disks == 2) {
1795	mddev->degraded = 1;
1796	return 0;
1797	}
1798
1799	/* raid1 -> raid0 */
1800	if (mddev->new_level == 0 &&
1801	mddev->raid_disks == 1)
1802	return 0;
1803
1804	/* raid1 -> raid10 */
1805	if (mddev->new_level == 10)
1806	return 0;
1807	break;
1808
1809	case 4:
1810	/* raid4 -> raid0 */
1811	if (mddev->new_level == 0)
1812	return 0;
1813
1814	/* raid4 -> raid1/5 with 2 disks */
1815	if ((mddev->new_level == 1 || mddev->new_level == 5) &&
1816	mddev->raid_disks == 2)
1817	return 0;
1818
1819	/* raid4 -> raid5/6 with parity N */
1820	if (__within_range(v: mddev->new_level, min: 5, max: 6) &&
1821	mddev->layout == ALGORITHM_PARITY_N)
1822	return 0;
1823	break;
1824
1825	case 5:
1826	/* raid5 with parity N -> raid0 */
1827	if (mddev->new_level == 0 &&
1828	mddev->layout == ALGORITHM_PARITY_N)
1829	return 0;
1830
1831	/* raid5 with parity N -> raid4 */
1832	if (mddev->new_level == 4 &&
1833	mddev->layout == ALGORITHM_PARITY_N)
1834	return 0;
1835
1836	/* raid5 with 2 disks -> raid1/4/10 */
1837	if ((mddev->new_level == 1 || mddev->new_level == 4 || mddev->new_level == 10) &&
1838	mddev->raid_disks == 2)
1839	return 0;
1840
1841	/* raid5_* -> raid6_*_6 with Q-Syndrome N (e.g. raid5_ra -> raid6_ra_6 */
1842	if (mddev->new_level == 6 &&
1843	((mddev->layout == ALGORITHM_PARITY_N && mddev->new_layout == ALGORITHM_PARITY_N) ||
1844	__within_range(v: mddev->new_layout, ALGORITHM_LEFT_ASYMMETRIC_6, ALGORITHM_RIGHT_SYMMETRIC_6)))
1845	return 0;
1846	break;
1847
1848	case 6:
1849	/* raid6 with parity N -> raid0 */
1850	if (mddev->new_level == 0 &&
1851	mddev->layout == ALGORITHM_PARITY_N)
1852	return 0;
1853
1854	/* raid6 with parity N -> raid4 */
1855	if (mddev->new_level == 4 &&
1856	mddev->layout == ALGORITHM_PARITY_N)
1857	return 0;
1858
1859	/* raid6_*_n with Q-Syndrome N -> raid5_* */
1860	if (mddev->new_level == 5 &&
1861	((mddev->layout == ALGORITHM_PARITY_N && mddev->new_layout == ALGORITHM_PARITY_N) ||
1862	__within_range(v: mddev->new_layout, ALGORITHM_LEFT_ASYMMETRIC, ALGORITHM_RIGHT_SYMMETRIC)))
1863	return 0;
1864	break;
1865
1866	default:
1867	break;
1868	}
1869
1870	rs->ti->error = "takeover not possible";
1871	return -EINVAL;
1872	}
1873
1874	/* True if @rs requested to be taken over */
1875	static bool rs_takeover_requested(struct raid_set *rs)
1876	{
1877	return rs->md.new_level != rs->md.level;
1878	}
1879
1880	/* True if layout is set to reshape. */
1881	static bool rs_is_layout_change(struct raid_set *rs, bool use_mddev)
1882	{
1883	return (use_mddev ? rs->md.delta_disks : rs->delta_disks) ||
1884	rs->md.new_layout != rs->md.layout ||
1885	rs->md.new_chunk_sectors != rs->md.chunk_sectors;
1886	}
1887
1888	/* True if @rs is requested to reshape by ctr */
1889	static bool rs_reshape_requested(struct raid_set *rs)
1890	{
1891	bool change;
1892	struct mddev *mddev = &rs->md;
1893
1894	if (rs_takeover_requested(rs))
1895	return false;
1896
1897	if (rs_is_raid0(rs))
1898	return false;
1899
1900	change = rs_is_layout_change(rs, use_mddev: false);
1901
1902	/* Historical case to support raid1 reshape without delta disks */
1903	if (rs_is_raid1(rs)) {
1904	if (rs->delta_disks)
1905	return !!rs->delta_disks;
1906
1907	return !change &&
1908	mddev->raid_disks != rs->raid_disks;
1909	}
1910
1911	if (rs_is_raid10(rs))
1912	return change &&
1913	!__is_raid10_far(layout: mddev->new_layout) &&
1914	rs->delta_disks >= 0;
1915
1916	return change;
1917	}
1918
1919	/* Features */
1920	#define FEATURE_FLAG_SUPPORTS_V190 0x1 /* Supports extended superblock */
1921
1922	/* State flags for sb->flags */
1923	#define SB_FLAG_RESHAPE_ACTIVE 0x1
1924	#define SB_FLAG_RESHAPE_BACKWARDS 0x2
1925
1926	/*
1927	* This structure is never routinely used by userspace, unlike md superblocks.
1928	* Devices with this superblock should only ever be accessed via device-mapper.
1929	*/
1930	#define DM_RAID_MAGIC 0x64526D44
1931	struct dm_raid_superblock {
1932	__le32 magic; /* "DmRd" */
1933	__le32 compat_features; /* Used to indicate compatible features (like 1.9.0 ondisk metadata extension) */
1934
1935	__le32 num_devices; /* Number of devices in this raid set. (Max 64) */
1936	__le32 array_position; /* The position of this drive in the raid set */
1937
1938	__le64 events; /* Incremented by md when superblock updated */
1939	__le64 failed_devices; /* Pre 1.9.0 part of bit field of devices to */
1940	/* indicate failures (see extension below) */
1941
1942	/*
1943	* This offset tracks the progress of the repair or replacement of
1944	* an individual drive.
1945	*/
1946	__le64 disk_recovery_offset;
1947
1948	/*
1949	* This offset tracks the progress of the initial raid set
1950	* synchronisation/parity calculation.
1951	*/
1952	__le64 array_resync_offset;
1953
1954	/*
1955	* raid characteristics
1956	*/
1957	__le32 level;
1958	__le32 layout;
1959	__le32 stripe_sectors;
1960
1961	/********************************************************************
1962	* BELOW FOLLOW V1.9.0 EXTENSIONS TO THE PRISTINE SUPERBLOCK FORMAT!!!
1963	*
1964	* FEATURE_FLAG_SUPPORTS_V190 in the compat_features member indicates that those exist
1965	*/
1966
1967	__le32 flags; /* Flags defining array states for reshaping */
1968
1969	/*
1970	* This offset tracks the progress of a raid
1971	* set reshape in order to be able to restart it
1972	*/
1973	__le64 reshape_position;
1974
1975	/*
1976	* These define the properties of the array in case of an interrupted reshape
1977	*/
1978	__le32 new_level;
1979	__le32 new_layout;
1980	__le32 new_stripe_sectors;
1981	__le32 delta_disks;
1982
1983	__le64 array_sectors; /* Array size in sectors */
1984
1985	/*
1986	* Sector offsets to data on devices (reshaping).
1987	* Needed to support out of place reshaping, thus
1988	* not writing over any stripes whilst converting
1989	* them from old to new layout
1990	*/
1991	__le64 data_offset;
1992	__le64 new_data_offset;
1993
1994	__le64 sectors; /* Used device size in sectors */
1995
1996	/*
1997	* Additional Bit field of devices indicating failures to support
1998	* up to 256 devices with the 1.9.0 on-disk metadata format
1999	*/
2000	__le64 extended_failed_devices[DISKS_ARRAY_ELEMS - 1];
2001
2002	__le32 incompat_features; /* Used to indicate any incompatible features */
2003
2004	/* Always set rest up to logical block size to 0 when writing (see get_metadata_device() below). */
2005	} __packed;
2006
2007	/*
2008	* Check for reshape constraints on raid set @rs:
2009	*
2010	* - reshape function non-existent
2011	* - degraded set
2012	* - ongoing recovery
2013	* - ongoing reshape
2014	*
2015	* Returns 0 if none or -EPERM if given constraint
2016	* and error message reference in @errmsg
2017	*/
2018	static int rs_check_reshape(struct raid_set *rs)
2019	{
2020	struct mddev *mddev = &rs->md;
2021
2022	if (!mddev->pers || !mddev->pers->check_reshape)
2023	rs->ti->error = "Reshape not supported";
2024	else if (mddev->degraded)
2025	rs->ti->error = "Can't reshape degraded raid set";
2026	else if (rs_is_recovering(rs))
2027	rs->ti->error = "Convert request on recovering raid set prohibited";
2028	else if (rs_is_reshaping(rs))
2029	rs->ti->error = "raid set already reshaping!";
2030	else if (!(rs_is_raid1(rs) || rs_is_raid10(rs) || rs_is_raid456(rs)))
2031	rs->ti->error = "Reshaping only supported for raid1/4/5/6/10";
2032	else
2033	return 0;
2034
2035	return -EPERM;
2036	}
2037
2038	static int read_disk_sb(struct md_rdev *rdev, int size, bool force_reload)
2039	{
2040	BUG_ON(!rdev->sb_page);
2041
2042	if (rdev->sb_loaded && !force_reload)
2043	return 0;
2044
2045	rdev->sb_loaded = 0;
2046
2047	if (!sync_page_io(rdev, sector: 0, size, page: rdev->sb_page, opf: REQ_OP_READ, metadata_op: true)) {
2048	DMERR("Failed to read superblock of device at position %d",
2049	rdev->raid_disk);
2050	md_error(mddev: rdev->mddev, rdev);
2051	set_bit(nr: Faulty, addr: &rdev->flags);
2052	return -EIO;
2053	}
2054
2055	rdev->sb_loaded = 1;
2056
2057	return 0;
2058	}
2059
2060	static void sb_retrieve_failed_devices(struct dm_raid_superblock *sb, uint64_t *failed_devices)
2061	{
2062	failed_devices[0] = le64_to_cpu(sb->failed_devices);
2063	memset(failed_devices + 1, 0, sizeof(sb->extended_failed_devices));
2064
2065	if (le32_to_cpu(sb->compat_features) & FEATURE_FLAG_SUPPORTS_V190) {
2066	int i = ARRAY_SIZE(sb->extended_failed_devices);
2067
2068	while (i--)
2069	failed_devices[i+1] = le64_to_cpu(sb->extended_failed_devices[i]);
2070	}
2071	}
2072
2073	static void sb_update_failed_devices(struct dm_raid_superblock *sb, uint64_t *failed_devices)
2074	{
2075	int i = ARRAY_SIZE(sb->extended_failed_devices);
2076
2077	sb->failed_devices = cpu_to_le64(failed_devices[0]);
2078	while (i--)
2079	sb->extended_failed_devices[i] = cpu_to_le64(failed_devices[i+1]);
2080	}
2081
2082	/*
2083	* Synchronize the superblock members with the raid set properties
2084	*
2085	* All superblock data is little endian.
2086	*/
2087	static void super_sync(struct mddev *mddev, struct md_rdev *rdev)
2088	{
2089	bool update_failed_devices = false;
2090	unsigned int i;
2091	uint64_t failed_devices[DISKS_ARRAY_ELEMS];
2092	struct dm_raid_superblock *sb;
2093	struct raid_set *rs = container_of(mddev, struct raid_set, md);
2094
2095	/* No metadata device, no superblock */
2096	if (!rdev->meta_bdev)
2097	return;
2098
2099	BUG_ON(!rdev->sb_page);
2100
2101	sb = page_address(rdev->sb_page);
2102
2103	sb_retrieve_failed_devices(sb, failed_devices);
2104
2105	for (i = 0; i < rs->raid_disks; i++)
2106	if (!rs->dev[i].data_dev || test_bit(Faulty, &rs->dev[i].rdev.flags)) {
2107	update_failed_devices = true;
2108	set_bit(nr: i, addr: (void *) failed_devices);
2109	}
2110
2111	if (update_failed_devices)
2112	sb_update_failed_devices(sb, failed_devices);
2113
2114	sb->magic = cpu_to_le32(DM_RAID_MAGIC);
2115	sb->compat_features = cpu_to_le32(FEATURE_FLAG_SUPPORTS_V190);
2116
2117	sb->num_devices = cpu_to_le32(mddev->raid_disks);
2118	sb->array_position = cpu_to_le32(rdev->raid_disk);
2119
2120	sb->events = cpu_to_le64(mddev->events);
2121
2122	sb->disk_recovery_offset = cpu_to_le64(rdev->recovery_offset);
2123	sb->array_resync_offset = cpu_to_le64(mddev->recovery_cp);
2124
2125	sb->level = cpu_to_le32(mddev->level);
2126	sb->layout = cpu_to_le32(mddev->layout);
2127	sb->stripe_sectors = cpu_to_le32(mddev->chunk_sectors);
2128
2129	/********************************************************************
2130	* BELOW FOLLOW V1.9.0 EXTENSIONS TO THE PRISTINE SUPERBLOCK FORMAT!!!
2131	*
2132	* FEATURE_FLAG_SUPPORTS_V190 in the compat_features member indicates that those exist
2133	*/
2134	sb->new_level = cpu_to_le32(mddev->new_level);
2135	sb->new_layout = cpu_to_le32(mddev->new_layout);
2136	sb->new_stripe_sectors = cpu_to_le32(mddev->new_chunk_sectors);
2137
2138	sb->delta_disks = cpu_to_le32(mddev->delta_disks);
2139
2140	smp_rmb(); /* Make sure we access most recent reshape position */
2141	sb->reshape_position = cpu_to_le64(mddev->reshape_position);
2142	if (le64_to_cpu(sb->reshape_position) != MaxSector) {
2143	/* Flag ongoing reshape */
2144	sb->flags |= cpu_to_le32(SB_FLAG_RESHAPE_ACTIVE);
2145
2146	if (mddev->delta_disks < 0 || mddev->reshape_backwards)
2147	sb->flags |= cpu_to_le32(SB_FLAG_RESHAPE_BACKWARDS);
2148	} else {
2149	/* Clear reshape flags */
2150	sb->flags &= ~(cpu_to_le32(SB_FLAG_RESHAPE_ACTIVE|SB_FLAG_RESHAPE_BACKWARDS));
2151	}
2152
2153	sb->array_sectors = cpu_to_le64(mddev->array_sectors);
2154	sb->data_offset = cpu_to_le64(rdev->data_offset);
2155	sb->new_data_offset = cpu_to_le64(rdev->new_data_offset);
2156	sb->sectors = cpu_to_le64(rdev->sectors);
2157	sb->incompat_features = cpu_to_le32(0);
2158
2159	/* Zero out the rest of the payload after the size of the superblock */
2160	memset(sb + 1, 0, rdev->sb_size - sizeof(*sb));
2161	}
2162
2163	/*
2164	* super_load
2165	*
2166	* This function creates a superblock if one is not found on the device
2167	* and will decide which superblock to use if there's a choice.
2168	*
2169	* Return: 1 if use rdev, 0 if use refdev, -Exxx otherwise
2170	*/
2171	static int super_load(struct md_rdev *rdev, struct md_rdev *refdev)
2172	{
2173	int r;
2174	struct dm_raid_superblock *sb;
2175	struct dm_raid_superblock *refsb;
2176	uint64_t events_sb, events_refsb;
2177
2178	r = read_disk_sb(rdev, size: rdev->sb_size, force_reload: false);
2179	if (r)
2180	return r;
2181
2182	sb = page_address(rdev->sb_page);
2183
2184	/*
2185	* Two cases that we want to write new superblocks and rebuild:
2186	* 1) New device (no matching magic number)
2187	* 2) Device specified for rebuild (!In_sync w/ offset == 0)
2188	*/
2189	if ((sb->magic != cpu_to_le32(DM_RAID_MAGIC)) ||
2190	(!test_bit(In_sync, &rdev->flags) && !rdev->recovery_offset)) {
2191	super_sync(mddev: rdev->mddev, rdev);
2192
2193	set_bit(FirstUse, addr: &rdev->flags);
2194	sb->compat_features = cpu_to_le32(FEATURE_FLAG_SUPPORTS_V190);
2195
2196	/* Force writing of superblocks to disk */
2197	set_bit(nr: MD_SB_CHANGE_DEVS, addr: &rdev->mddev->sb_flags);
2198
2199	/* Any superblock is better than none, choose that if given */
2200	return refdev ? 0 : 1;
2201	}
2202
2203	if (!refdev)
2204	return 1;
2205
2206	events_sb = le64_to_cpu(sb->events);
2207
2208	refsb = page_address(refdev->sb_page);
2209	events_refsb = le64_to_cpu(refsb->events);
2210
2211	return (events_sb > events_refsb) ? 1 : 0;
2212	}
2213
2214	static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev)
2215	{
2216	int role;
2217	struct mddev *mddev = &rs->md;
2218	uint64_t events_sb;
2219	uint64_t failed_devices[DISKS_ARRAY_ELEMS];
2220	struct dm_raid_superblock *sb;
2221	uint32_t new_devs = 0, rebuild_and_new = 0, rebuilds = 0;
2222	struct md_rdev *r;
2223	struct dm_raid_superblock *sb2;
2224
2225	sb = page_address(rdev->sb_page);
2226	events_sb = le64_to_cpu(sb->events);
2227
2228	/*
2229	* Initialise to 1 if this is a new superblock.
2230	*/
2231	mddev->events = events_sb ? : 1;
2232
2233	mddev->reshape_position = MaxSector;
2234
2235	mddev->raid_disks = le32_to_cpu(sb->num_devices);
2236	mddev->level = le32_to_cpu(sb->level);
2237	mddev->layout = le32_to_cpu(sb->layout);
2238	mddev->chunk_sectors = le32_to_cpu(sb->stripe_sectors);
2239
2240	/*
2241	* Reshaping is supported, e.g. reshape_position is valid
2242	* in superblock and superblock content is authoritative.
2243	*/
2244	if (le32_to_cpu(sb->compat_features) & FEATURE_FLAG_SUPPORTS_V190) {
2245	/* Superblock is authoritative wrt given raid set layout! */
2246	mddev->new_level = le32_to_cpu(sb->new_level);
2247	mddev->new_layout = le32_to_cpu(sb->new_layout);
2248	mddev->new_chunk_sectors = le32_to_cpu(sb->new_stripe_sectors);
2249	mddev->delta_disks = le32_to_cpu(sb->delta_disks);
2250	mddev->array_sectors = le64_to_cpu(sb->array_sectors);
2251
2252	/* raid was reshaping and got interrupted */
2253	if (le32_to_cpu(sb->flags) & SB_FLAG_RESHAPE_ACTIVE) {
2254	if (test_bit(__CTR_FLAG_DELTA_DISKS, &rs->ctr_flags)) {
2255	DMERR("Reshape requested but raid set is still reshaping");
2256	return -EINVAL;
2257	}
2258
2259	if (mddev->delta_disks < 0 ||
2260	(!mddev->delta_disks && (le32_to_cpu(sb->flags) & SB_FLAG_RESHAPE_BACKWARDS)))
2261	mddev->reshape_backwards = 1;
2262	else
2263	mddev->reshape_backwards = 0;
2264
2265	mddev->reshape_position = le64_to_cpu(sb->reshape_position);
2266	rs->raid_type = get_raid_type_by_ll(level: mddev->level, layout: mddev->layout);
2267	}
2268
2269	} else {
2270	/*
2271	* No takeover/reshaping, because we don't have the extended v1.9.0 metadata
2272	*/
2273	struct raid_type *rt_cur = get_raid_type_by_ll(level: mddev->level, layout: mddev->layout);
2274	struct raid_type *rt_new = get_raid_type_by_ll(level: mddev->new_level, layout: mddev->new_layout);
2275
2276	if (rs_takeover_requested(rs)) {
2277	if (rt_cur && rt_new)
2278	DMERR("Takeover raid sets from %s to %s not yet supported by metadata. (raid level change)",
2279	rt_cur->name, rt_new->name);
2280	else
2281	DMERR("Takeover raid sets not yet supported by metadata. (raid level change)");
2282	return -EINVAL;
2283	} else if (rs_reshape_requested(rs)) {
2284	DMERR("Reshaping raid sets not yet supported by metadata. (raid layout change keeping level)");
2285	if (mddev->layout != mddev->new_layout) {
2286	if (rt_cur && rt_new)
2287	DMERR(" current layout %s vs new layout %s",
2288	rt_cur->name, rt_new->name);
2289	else
2290	DMERR(" current layout 0x%X vs new layout 0x%X",
2291	le32_to_cpu(sb->layout), mddev->new_layout);
2292	}
2293	if (mddev->chunk_sectors != mddev->new_chunk_sectors)
2294	DMERR(" current stripe sectors %u vs new stripe sectors %u",
2295	mddev->chunk_sectors, mddev->new_chunk_sectors);
2296	if (rs->delta_disks)
2297	DMERR(" current %u disks vs new %u disks",
2298	mddev->raid_disks, mddev->raid_disks + rs->delta_disks);
2299	if (rs_is_raid10(rs)) {
2300	DMERR(" Old layout: %s w/ %u copies",
2301	raid10_md_layout_to_format(mddev->layout),
2302	raid10_md_layout_to_copies(mddev->layout));
2303	DMERR(" New layout: %s w/ %u copies",
2304	raid10_md_layout_to_format(mddev->new_layout),
2305	raid10_md_layout_to_copies(mddev->new_layout));
2306	}
2307	return -EINVAL;
2308	}
2309
2310	DMINFO("Discovered old metadata format; upgrading to extended metadata format");
2311	}
2312
2313	if (!test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags))
2314	mddev->recovery_cp = le64_to_cpu(sb->array_resync_offset);
2315
2316	/*
2317	* During load, we set FirstUse if a new superblock was written.
2318	* There are two reasons we might not have a superblock:
2319	* 1) The raid set is brand new - in which case, all of the
2320	* devices must have their In_sync bit set. Also,
2321	* recovery_cp must be 0, unless forced.
2322	* 2) This is a new device being added to an old raid set
2323	* and the new device needs to be rebuilt - in which
2324	* case the In_sync bit will /not/ be set and
2325	* recovery_cp must be MaxSector.
2326	* 3) This is/are a new device(s) being added to an old
2327	* raid set during takeover to a higher raid level
2328	* to provide capacity for redundancy or during reshape
2329	* to add capacity to grow the raid set.
2330	*/
2331	rdev_for_each(r, mddev) {
2332	if (test_bit(Journal, &rdev->flags))
2333	continue;
2334
2335	if (test_bit(FirstUse, &r->flags))
2336	new_devs++;
2337
2338	if (!test_bit(In_sync, &r->flags)) {
2339	DMINFO("Device %d specified for rebuild; clearing superblock",
2340	r->raid_disk);
2341	rebuilds++;
2342
2343	if (test_bit(FirstUse, &r->flags))
2344	rebuild_and_new++;
2345	}
2346	}
2347
2348	if (new_devs == rs->raid_disks || !rebuilds) {
2349	/* Replace a broken device */
2350	if (new_devs == rs->raid_disks) {
2351	DMINFO("Superblocks created for new raid set");
2352	set_bit(nr: MD_ARRAY_FIRST_USE, addr: &mddev->flags);
2353	} else if (new_devs != rebuilds &&
2354	new_devs != rs->delta_disks) {
2355	DMERR("New device injected into existing raid set without "
2356	"'delta_disks' or 'rebuild' parameter specified");
2357	return -EINVAL;
2358	}
2359	} else if (new_devs && new_devs != rebuilds) {
2360	DMERR("%u 'rebuild' devices cannot be injected into"
2361	" a raid set with %u other first-time devices",
2362	rebuilds, new_devs);
2363	return -EINVAL;
2364	} else if (rebuilds) {
2365	if (rebuild_and_new && rebuilds != rebuild_and_new) {
2366	DMERR("new device%s provided without 'rebuild'",
2367	new_devs > 1 ? "s" : "");
2368	return -EINVAL;
2369	} else if (!test_bit(__CTR_FLAG_REBUILD, &rs->ctr_flags) && rs_is_recovering(rs)) {
2370	DMERR("'rebuild' specified while raid set is not in-sync (recovery_cp=%llu)",
2371	(unsigned long long) mddev->recovery_cp);
2372	return -EINVAL;
2373	} else if (rs_is_reshaping(rs)) {
2374	DMERR("'rebuild' specified while raid set is being reshaped (reshape_position=%llu)",
2375	(unsigned long long) mddev->reshape_position);
2376	return -EINVAL;
2377	}
2378	}
2379
2380	/*
2381	* Now we set the Faulty bit for those devices that are
2382	* recorded in the superblock as failed.
2383	*/
2384	sb_retrieve_failed_devices(sb, failed_devices);
2385	rdev_for_each(r, mddev) {
2386	if (test_bit(Journal, &rdev->flags) ||
2387	!r->sb_page)
2388	continue;
2389	sb2 = page_address(r->sb_page);
2390	sb2->failed_devices = 0;
2391	memset(sb2->extended_failed_devices, 0, sizeof(sb2->extended_failed_devices));
2392
2393	/*
2394	* Check for any device re-ordering.
2395	*/
2396	if (!test_bit(FirstUse, &r->flags) && (r->raid_disk >= 0)) {
2397	role = le32_to_cpu(sb2->array_position);
2398	if (role < 0)
2399	continue;
2400
2401	if (role != r->raid_disk) {
2402	if (rs_is_raid10(rs) && __is_raid10_near(layout: mddev->layout)) {
2403	if (mddev->raid_disks % __raid10_near_copies(layout: mddev->layout) ||
2404	rs->raid_disks % rs->raid10_copies) {
2405	rs->ti->error =
2406	"Cannot change raid10 near set to odd # of devices!";
2407	return -EINVAL;
2408	}
2409
2410	sb2->array_position = cpu_to_le32(r->raid_disk);
2411
2412	} else if (!(rs_is_raid10(rs) && rt_is_raid0(rt: rs->raid_type)) &&
2413	!(rs_is_raid0(rs) && rt_is_raid10(rt: rs->raid_type)) &&
2414	!rt_is_raid1(rt: rs->raid_type)) {
2415	rs->ti->error = "Cannot change device positions in raid set";
2416	return -EINVAL;
2417	}
2418
2419	DMINFO("raid device #%d now at position #%d", role, r->raid_disk);
2420	}
2421
2422	/*
2423	* Partial recovery is performed on
2424	* returning failed devices.
2425	*/
2426	if (test_bit(role, (void *) failed_devices))
2427	set_bit(nr: Faulty, addr: &r->flags);
2428	}
2429	}
2430
2431	return 0;
2432	}
2433
2434	static int super_validate(struct raid_set *rs, struct md_rdev *rdev)
2435	{
2436	struct mddev *mddev = &rs->md;
2437	struct dm_raid_superblock *sb;
2438
2439	if (rs_is_raid0(rs) || !rdev->sb_page || rdev->raid_disk < 0)
2440	return 0;
2441
2442	sb = page_address(rdev->sb_page);
2443
2444	/*
2445	* If mddev->events is not set, we know we have not yet initialized
2446	* the array.
2447	*/
2448	if (!mddev->events && super_init_validation(rs, rdev))
2449	return -EINVAL;
2450
2451	if (le32_to_cpu(sb->compat_features) &&
2452	le32_to_cpu(sb->compat_features) != FEATURE_FLAG_SUPPORTS_V190) {
2453	rs->ti->error = "Unable to assemble array: Unknown flag(s) in compatible feature flags";
2454	return -EINVAL;
2455	}
2456
2457	if (sb->incompat_features) {
2458	rs->ti->error = "Unable to assemble array: No incompatible feature flags supported yet";
2459	return -EINVAL;
2460	}
2461
2462	/* Enable bitmap creation on @rs unless no metadevs or raid0 or journaled raid4/5/6 set. */
2463	mddev->bitmap_info.offset = (rt_is_raid0(rt: rs->raid_type) || rs->journal_dev.dev) ? 0 : to_sector(n: 4096);
2464	mddev->bitmap_info.default_offset = mddev->bitmap_info.offset;
2465
2466	if (!test_and_clear_bit(FirstUse, addr: &rdev->flags)) {
2467	/*
2468	* Retrieve rdev size stored in superblock to be prepared for shrink.
2469	* Check extended superblock members are present otherwise the size
2470	* will not be set!
2471	*/
2472	if (le32_to_cpu(sb->compat_features) & FEATURE_FLAG_SUPPORTS_V190)
2473	rdev->sectors = le64_to_cpu(sb->sectors);
2474
2475	rdev->recovery_offset = le64_to_cpu(sb->disk_recovery_offset);
2476	if (rdev->recovery_offset == MaxSector)
2477	set_bit(nr: In_sync, addr: &rdev->flags);
2478	/*
2479	* If no reshape in progress -> we're recovering single
2480	* disk(s) and have to set the device(s) to out-of-sync
2481	*/
2482	else if (!rs_is_reshaping(rs))
2483	clear_bit(nr: In_sync, addr: &rdev->flags); /* Mandatory for recovery */
2484	}
2485
2486	/*
2487	* If a device comes back, set it as not In_sync and no longer faulty.
2488	*/
2489	if (test_and_clear_bit(nr: Faulty, addr: &rdev->flags)) {
2490	rdev->recovery_offset = 0;
2491	clear_bit(nr: In_sync, addr: &rdev->flags);
2492	rdev->saved_raid_disk = rdev->raid_disk;
2493	}
2494
2495	/* Reshape support -> restore repective data offsets */
2496	rdev->data_offset = le64_to_cpu(sb->data_offset);
2497	rdev->new_data_offset = le64_to_cpu(sb->new_data_offset);
2498
2499	return 0;
2500	}
2501
2502	/*
2503	* Analyse superblocks and select the freshest.
2504	*/
2505	static int analyse_superblocks(struct dm_target *ti, struct raid_set *rs)
2506	{
2507	int r;
2508	struct md_rdev *rdev, *freshest;
2509	struct mddev *mddev = &rs->md;
2510
2511	freshest = NULL;
2512	rdev_for_each(rdev, mddev) {
2513	if (test_bit(Journal, &rdev->flags))
2514	continue;
2515
2516	if (!rdev->meta_bdev)
2517	continue;
2518
2519	/* Set superblock offset/size for metadata device. */
2520	rdev->sb_start = 0;
2521	rdev->sb_size = bdev_logical_block_size(bdev: rdev->meta_bdev);
2522	if (rdev->sb_size < sizeof(struct dm_raid_superblock) || rdev->sb_size > PAGE_SIZE) {
2523	DMERR("superblock size of a logical block is no longer valid");
2524	return -EINVAL;
2525	}
2526
2527	/*
2528	* Skipping super_load due to CTR_FLAG_SYNC will cause
2529	* the array to undergo initialization again as
2530	* though it were new. This is the intended effect
2531	* of the "sync" directive.
2532	*
2533	* With reshaping capability added, we must ensure that
2534	* the "sync" directive is disallowed during the reshape.
2535	*/
2536	if (test_bit(__CTR_FLAG_SYNC, &rs->ctr_flags))
2537	continue;
2538
2539	r = super_load(rdev, refdev: freshest);
2540
2541	switch (r) {
2542	case 1:
2543	freshest = rdev;
2544	break;
2545	case 0:
2546	break;
2547	default:
2548	/* This is a failure to read the superblock from the metadata device. */
2549	/*
2550	* We have to keep any raid0 data/metadata device pairs or
2551	* the MD raid0 personality will fail to start the array.
2552	*/
2553	if (rs_is_raid0(rs))
2554	continue;
2555
2556	/*
2557	* We keep the dm_devs to be able to emit the device tuple
2558	* properly on the table line in raid_status() (rather than
2559	* mistakenly acting as if '- -' got passed into the constructor).
2560	*
2561	* The rdev has to stay on the same_set list to allow for
2562	* the attempt to restore faulty devices on second resume.
2563	*/
2564	rdev->raid_disk = rdev->saved_raid_disk = -1;
2565	break;
2566	}
2567	}
2568
2569	if (!freshest)
2570	return 0;
2571
2572	/*
2573	* Validation of the freshest device provides the source of
2574	* validation for the remaining devices.
2575	*/
2576	rs->ti->error = "Unable to assemble array: Invalid superblocks";
2577	if (super_validate(rs, rdev: freshest))
2578	return -EINVAL;
2579
2580	if (validate_raid_redundancy(rs)) {
2581	rs->ti->error = "Insufficient redundancy to activate array";
2582	return -EINVAL;
2583	}
2584
2585	rdev_for_each(rdev, mddev)
2586	if (!test_bit(Journal, &rdev->flags) &&
2587	rdev != freshest &&
2588	super_validate(rs, rdev))
2589	return -EINVAL;
2590	return 0;
2591	}
2592
2593	/*
2594	* Adjust data_offset and new_data_offset on all disk members of @rs
2595	* for out of place reshaping if requested by constructor
2596	*
2597	* We need free space at the beginning of each raid disk for forward
2598	* and at the end for backward reshapes which userspace has to provide
2599	* via remapping/reordering of space.
2600	*/
2601	static int rs_adjust_data_offsets(struct raid_set *rs)
2602	{
2603	sector_t data_offset = 0, new_data_offset = 0;
2604	struct md_rdev *rdev;
2605
2606	/* Constructor did not request data offset change */
2607	if (!test_bit(__CTR_FLAG_DATA_OFFSET, &rs->ctr_flags)) {
2608	if (!rs_is_reshapable(rs))
2609	goto out;
2610
2611	return 0;
2612	}
2613
2614	/* HM FIXME: get In_Sync raid_dev? */
2615	rdev = &rs->dev[0].rdev;
2616
2617	if (rs->delta_disks < 0) {
2618	/*
2619	* Removing disks (reshaping backwards):
2620	*
2621	* - before reshape: data is at offset 0 and free space
2622	* is at end of each component LV
2623	*
2624	* - after reshape: data is at offset rs->data_offset != 0 on each component LV
2625	*/
2626	data_offset = 0;
2627	new_data_offset = rs->data_offset;
2628
2629	} else if (rs->delta_disks > 0) {
2630	/*
2631	* Adding disks (reshaping forwards):
2632	*
2633	* - before reshape: data is at offset rs->data_offset != 0 and
2634	* free space is at begin of each component LV
2635	*
2636	* - after reshape: data is at offset 0 on each component LV
2637	*/
2638	data_offset = rs->data_offset;
2639	new_data_offset = 0;
2640
2641	} else {
2642	/*
2643	* User space passes in 0 for data offset after having removed reshape space
2644	*
2645	* - or - (data offset != 0)
2646	*
2647	* Changing RAID layout or chunk size -> toggle offsets
2648	*
2649	* - before reshape: data is at offset rs->data_offset 0 and
2650	* free space is at end of each component LV
2651	* -or-
2652	* data is at offset rs->data_offset != 0 and
2653	* free space is at begin of each component LV
2654	*
2655	* - after reshape: data is at offset 0 if it was at offset != 0
2656	* or at offset != 0 if it was at offset 0
2657	* on each component LV
2658	*
2659	*/
2660	data_offset = rs->data_offset ? rdev->data_offset : 0;
2661	new_data_offset = data_offset ? 0 : rs->data_offset;
2662	set_bit(RT_FLAG_UPDATE_SBS, addr: &rs->runtime_flags);
2663	}
2664
2665	/*
2666	* Make sure we got a minimum amount of free sectors per device
2667	*/
2668	if (rs->data_offset &&
2669	bdev_nr_sectors(bdev: rdev->bdev) - rs->md.dev_sectors < MIN_FREE_RESHAPE_SPACE) {
2670	rs->ti->error = data_offset ? "No space for forward reshape" :
2671	"No space for backward reshape";
2672	return -ENOSPC;
2673	}
2674	out:
2675	/*
2676	* Raise recovery_cp in case data_offset != 0 to
2677	* avoid false recovery positives in the constructor.
2678	*/
2679	if (rs->md.recovery_cp < rs->md.dev_sectors)
2680	rs->md.recovery_cp += rs->dev[0].rdev.data_offset;
2681
2682	/* Adjust data offsets on all rdevs but on any raid4/5/6 journal device */
2683	rdev_for_each(rdev, &rs->md) {
2684	if (!test_bit(Journal, &rdev->flags)) {
2685	rdev->data_offset = data_offset;
2686	rdev->new_data_offset = new_data_offset;
2687	}
2688	}
2689
2690	return 0;
2691	}
2692
2693	/* Userpace reordered disks -> adjust raid_disk indexes in @rs */
2694	static void __reorder_raid_disk_indexes(struct raid_set *rs)
2695	{
2696	int i = 0;
2697	struct md_rdev *rdev;
2698
2699	rdev_for_each(rdev, &rs->md) {
2700	if (!test_bit(Journal, &rdev->flags)) {
2701	rdev->raid_disk = i++;
2702	rdev->saved_raid_disk = rdev->new_raid_disk = -1;
2703	}
2704	}
2705	}
2706
2707	/*
2708	* Setup @rs for takeover by a different raid level
2709	*/
2710	static int rs_setup_takeover(struct raid_set *rs)
2711	{
2712	struct mddev *mddev = &rs->md;
2713	struct md_rdev *rdev;
2714	unsigned int d = mddev->raid_disks = rs->raid_disks;
2715	sector_t new_data_offset = rs->dev[0].rdev.data_offset ? 0 : rs->data_offset;
2716
2717	if (rt_is_raid10(rt: rs->raid_type)) {
2718	if (rs_is_raid0(rs)) {
2719	/* Userpace reordered disks -> adjust raid_disk indexes */
2720	__reorder_raid_disk_indexes(rs);
2721
2722	/* raid0 -> raid10_far layout */
2723	mddev->layout = raid10_format_to_md_layout(rs, ALGORITHM_RAID10_FAR,
2724	copies: rs->raid10_copies);
2725	} else if (rs_is_raid1(rs))
2726	/* raid1 -> raid10_near layout */
2727	mddev->layout = raid10_format_to_md_layout(rs, ALGORITHM_RAID10_NEAR,
2728	copies: rs->raid_disks);
2729	else
2730	return -EINVAL;
2731
2732	}
2733
2734	clear_bit(nr: MD_ARRAY_FIRST_USE, addr: &mddev->flags);
2735	mddev->recovery_cp = MaxSector;
2736
2737	while (d--) {
2738	rdev = &rs->dev[d].rdev;
2739
2740	if (test_bit(d, (void *) rs->rebuild_disks)) {
2741	clear_bit(nr: In_sync, addr: &rdev->flags);
2742	clear_bit(nr: Faulty, addr: &rdev->flags);
2743	mddev->recovery_cp = rdev->recovery_offset = 0;
2744	/* Bitmap has to be created when we do an "up" takeover */
2745	set_bit(nr: MD_ARRAY_FIRST_USE, addr: &mddev->flags);
2746	}
2747
2748	rdev->new_data_offset = new_data_offset;
2749	}
2750
2751	return 0;
2752	}
2753
2754	/* Prepare @rs for reshape */
2755	static int rs_prepare_reshape(struct raid_set *rs)
2756	{
2757	bool reshape;
2758	struct mddev *mddev = &rs->md;
2759
2760	if (rs_is_raid10(rs)) {
2761	if (rs->raid_disks != mddev->raid_disks &&
2762	__is_raid10_near(layout: mddev->layout) &&
2763	rs->raid10_copies &&
2764	rs->raid10_copies != __raid10_near_copies(layout: mddev->layout)) {
2765	/*
2766	* raid disk have to be multiple of data copies to allow this conversion,
2767	*
2768	* This is actually not a reshape it is a
2769	* rebuild of any additional mirrors per group
2770	*/
2771	if (rs->raid_disks % rs->raid10_copies) {
2772	rs->ti->error = "Can't reshape raid10 mirror groups";
2773	return -EINVAL;
2774	}
2775
2776	/* Userpace reordered disks to add/remove mirrors -> adjust raid_disk indexes */
2777	__reorder_raid_disk_indexes(rs);
2778	mddev->layout = raid10_format_to_md_layout(rs, ALGORITHM_RAID10_NEAR,
2779	copies: rs->raid10_copies);
2780	mddev->new_layout = mddev->layout;
2781	reshape = false;
2782	} else
2783	reshape = true;
2784
2785	} else if (rs_is_raid456(rs))
2786	reshape = true;
2787
2788	else if (rs_is_raid1(rs)) {
2789	if (rs->delta_disks) {
2790	/* Process raid1 via delta_disks */
2791	mddev->degraded = rs->delta_disks < 0 ? -rs->delta_disks : rs->delta_disks;
2792	reshape = true;
2793	} else {
2794	/* Process raid1 without delta_disks */
2795	mddev->raid_disks = rs->raid_disks;
2796	reshape = false;
2797	}
2798	} else {
2799	rs->ti->error = "Called with bogus raid type";
2800	return -EINVAL;
2801	}
2802
2803	if (reshape) {
2804	set_bit(RT_FLAG_RESHAPE_RS, addr: &rs->runtime_flags);
2805	set_bit(RT_FLAG_UPDATE_SBS, addr: &rs->runtime_flags);
2806	} else if (mddev->raid_disks < rs->raid_disks)
2807	/* Create new superblocks and bitmaps, if any new disks */
2808	set_bit(RT_FLAG_UPDATE_SBS, addr: &rs->runtime_flags);
2809
2810	return 0;
2811	}
2812
2813	/* Get reshape sectors from data_offsets or raid set */
2814	static sector_t _get_reshape_sectors(struct raid_set *rs)
2815	{
2816	struct md_rdev *rdev;
2817	sector_t reshape_sectors = 0;
2818
2819	rdev_for_each(rdev, &rs->md)
2820	if (!test_bit(Journal, &rdev->flags)) {
2821	reshape_sectors = (rdev->data_offset > rdev->new_data_offset) ?
2822	rdev->data_offset - rdev->new_data_offset :
2823	rdev->new_data_offset - rdev->data_offset;
2824	break;
2825	}
2826
2827	return max(reshape_sectors, (sector_t) rs->data_offset);
2828	}
2829
2830	/*
2831	* Reshape:
2832	* - change raid layout
2833	* - change chunk size
2834	* - add disks
2835	* - remove disks
2836	*/
2837	static int rs_setup_reshape(struct raid_set *rs)
2838	{
2839	int r = 0;
2840	unsigned int cur_raid_devs, d;
2841	sector_t reshape_sectors = _get_reshape_sectors(rs);
2842	struct mddev *mddev = &rs->md;
2843	struct md_rdev *rdev;
2844
2845	mddev->delta_disks = rs->delta_disks;
2846	cur_raid_devs = mddev->raid_disks;
2847
2848	/* Ignore impossible layout change whilst adding/removing disks */
2849	if (mddev->delta_disks &&
2850	mddev->layout != mddev->new_layout) {
2851	DMINFO("Ignoring invalid layout change with delta_disks=%d", rs->delta_disks);
2852	mddev->new_layout = mddev->layout;
2853	}
2854
2855	/*
2856	* Adjust array size:
2857	*
2858	* - in case of adding disk(s), array size has
2859	* to grow after the disk adding reshape,
2860	* which'll happen in the event handler;
2861	* reshape will happen forward, so space has to
2862	* be available at the beginning of each disk
2863	*
2864	* - in case of removing disk(s), array size
2865	* has to shrink before starting the reshape,
2866	* which'll happen here;
2867	* reshape will happen backward, so space has to
2868	* be available at the end of each disk
2869	*
2870	* - data_offset and new_data_offset are
2871	* adjusted for aforementioned out of place
2872	* reshaping based on userspace passing in
2873	* the "data_offset <sectors>" key/value
2874	* pair via the constructor
2875	*/
2876
2877	/* Add disk(s) */
2878	if (rs->delta_disks > 0) {
2879	/* Prepare disks for check in raid4/5/6/10 {check|start}_reshape */
2880	for (d = cur_raid_devs; d < rs->raid_disks; d++) {
2881	rdev = &rs->dev[d].rdev;
2882	clear_bit(nr: In_sync, addr: &rdev->flags);
2883
2884	/*
2885	* save_raid_disk needs to be -1, or recovery_offset will be set to 0
2886	* by md, which'll store that erroneously in the superblock on reshape
2887	*/
2888	rdev->saved_raid_disk = -1;
2889	rdev->raid_disk = d;
2890
2891	rdev->sectors = mddev->dev_sectors;
2892	rdev->recovery_offset = rs_is_raid1(rs) ? 0 : MaxSector;
2893	}
2894
2895	mddev->reshape_backwards = 0; /* adding disk(s) -> forward reshape */
2896
2897	/* Remove disk(s) */
2898	} else if (rs->delta_disks < 0) {
2899	r = rs_set_dev_and_array_sectors(rs, sectors: rs->ti->len, use_mddev: true);
2900	mddev->reshape_backwards = 1; /* removing disk(s) -> backward reshape */
2901
2902	/* Change layout and/or chunk size */
2903	} else {
2904	/*
2905	* Reshape layout (e.g. raid5_ls -> raid5_n) and/or chunk size:
2906	*
2907	* keeping number of disks and do layout change ->
2908	*
2909	* toggle reshape_backward depending on data_offset:
2910	*
2911	* - free space upfront -> reshape forward
2912	*
2913	* - free space at the end -> reshape backward
2914	*
2915	*
2916	* This utilizes free reshape space avoiding the need
2917	* for userspace to move (parts of) LV segments in
2918	* case of layout/chunksize change (for disk
2919	* adding/removing reshape space has to be at
2920	* the proper address (see above with delta_disks):
2921	*
2922	* add disk(s) -> begin
2923	* remove disk(s)-> end
2924	*/
2925	mddev->reshape_backwards = rs->dev[0].rdev.data_offset ? 0 : 1;
2926	}
2927
2928	/*
2929	* Adjust device size for forward reshape
2930	* because md_finish_reshape() reduces it.
2931	*/
2932	if (!mddev->reshape_backwards)
2933	rdev_for_each(rdev, &rs->md)
2934	if (!test_bit(Journal, &rdev->flags))
2935	rdev->sectors += reshape_sectors;
2936
2937	return r;
2938	}
2939
2940	/*
2941	* If the md resync thread has updated superblock with max reshape position
2942	* at the end of a reshape but not (yet) reset the layout configuration
2943	* changes -> reset the latter.
2944	*/
2945	static void rs_reset_inconclusive_reshape(struct raid_set *rs)
2946	{
2947	if (!rs_is_reshaping(rs) && rs_is_layout_change(rs, use_mddev: true)) {
2948	rs_set_cur(rs);
2949	rs->md.delta_disks = 0;
2950	rs->md.reshape_backwards = 0;
2951	}
2952	}
2953
2954	/*
2955	* Enable/disable discard support on RAID set depending on
2956	* RAID level and discard properties of underlying RAID members.
2957	*/
2958	static void configure_discard_support(struct raid_set *rs)
2959	{
2960	int i;
2961	bool raid456;
2962	struct dm_target *ti = rs->ti;
2963
2964	/*
2965	* XXX: RAID level 4,5,6 require zeroing for safety.
2966	*/
2967	raid456 = rs_is_raid456(rs);
2968
2969	for (i = 0; i < rs->raid_disks; i++) {
2970	if (!rs->dev[i].rdev.bdev ||
2971	!bdev_max_discard_sectors(bdev: rs->dev[i].rdev.bdev))
2972	return;
2973
2974	if (raid456) {
2975	if (!devices_handle_discard_safely) {
2976	DMERR("raid456 discard support disabled due to discard_zeroes_data uncertainty.");
2977	DMERR("Set dm-raid.devices_handle_discard_safely=Y to override.");
2978	return;
2979	}
2980	}
2981	}
2982
2983	ti->num_discard_bios = 1;
2984	}
2985
2986	/*
2987	* Construct a RAID0/1/10/4/5/6 mapping:
2988	* Args:
2989	* <raid_type> <#raid_params> <raid_params>{0,} \
2990	* <#raid_devs> [<meta_dev1> <dev1>]{1,}
2991	*
2992	* <raid_params> varies by <raid_type>. See 'parse_raid_params' for
2993	* details on possible <raid_params>.
2994	*
2995	* Userspace is free to initialize the metadata devices, hence the superblocks to
2996	* enforce recreation based on the passed in table parameters.
2997	*
2998	*/
2999	static int raid_ctr(struct dm_target *ti, unsigned int argc, char **argv)
3000	{
3001	int r;
3002	bool resize = false;
3003	struct raid_type *rt;
3004	unsigned int num_raid_params, num_raid_devs;
3005	sector_t sb_array_sectors, rdev_sectors, reshape_sectors;
3006	struct raid_set *rs = NULL;
3007	const char *arg;
3008	struct rs_layout rs_layout;
3009	struct dm_arg_set as = { argc, argv }, as_nrd;
3010	struct dm_arg _args[] = {
3011	{ 0, as.argc, "Cannot understand number of raid parameters" },
3012	{ 1, 254, "Cannot understand number of raid devices parameters" }
3013	};
3014
3015	arg = dm_shift_arg(as: &as);
3016	if (!arg) {
3017	ti->error = "No arguments";
3018	return -EINVAL;
3019	}
3020
3021	rt = get_raid_type(name: arg);
3022	if (!rt) {
3023	ti->error = "Unrecognised raid_type";
3024	return -EINVAL;
3025	}
3026
3027	/* Must have <#raid_params> */
3028	if (dm_read_arg_group(arg: _args, arg_set: &as, num_args: &num_raid_params, error: &ti->error))
3029	return -EINVAL;
3030
3031	/* number of raid device tupples <meta_dev data_dev> */
3032	as_nrd = as;
3033	dm_consume_args(as: &as_nrd, num_args: num_raid_params);
3034	_args[1].max = (as_nrd.argc - 1) / 2;
3035	if (dm_read_arg(arg: _args + 1, arg_set: &as_nrd, value: &num_raid_devs, error: &ti->error))
3036	return -EINVAL;
3037
3038	if (!__within_range(v: num_raid_devs, min: 1, MAX_RAID_DEVICES)) {
3039	ti->error = "Invalid number of supplied raid devices";
3040	return -EINVAL;
3041	}
3042
3043	rs = raid_set_alloc(ti, raid_type: rt, raid_devs: num_raid_devs);
3044	if (IS_ERR(ptr: rs))
3045	return PTR_ERR(ptr: rs);
3046
3047	r = parse_raid_params(rs, as: &as, num_raid_params);
3048	if (r)
3049	goto bad;
3050
3051	r = parse_dev_params(rs, as: &as);
3052	if (r)
3053	goto bad;
3054
3055	rs->md.sync_super = super_sync;
3056
3057	/*
3058	* Calculate ctr requested array and device sizes to allow
3059	* for superblock analysis needing device sizes defined.
3060	*
3061	* Any existing superblock will overwrite the array and device sizes
3062	*/
3063	r = rs_set_dev_and_array_sectors(rs, sectors: rs->ti->len, use_mddev: false);
3064	if (r)
3065	goto bad;
3066
3067	/* Memorize just calculated, potentially larger sizes to grow the raid set in preresume */
3068	rs->array_sectors = rs->md.array_sectors;
3069	rs->dev_sectors = rs->md.dev_sectors;
3070
3071	/*
3072	* Backup any new raid set level, layout, ...
3073	* requested to be able to compare to superblock
3074	* members for conversion decisions.
3075	*/
3076	rs_config_backup(rs, l: &rs_layout);
3077
3078	r = analyse_superblocks(ti, rs);
3079	if (r)
3080	goto bad;
3081
3082	/* All in-core metadata now as of current superblocks after calling analyse_superblocks() */
3083	sb_array_sectors = rs->md.array_sectors;
3084	rdev_sectors = __rdev_sectors(rs);
3085	if (!rdev_sectors) {
3086	ti->error = "Invalid rdev size";
3087	r = -EINVAL;
3088	goto bad;
3089	}
3090
3091
3092	reshape_sectors = _get_reshape_sectors(rs);
3093	if (rs->dev_sectors != rdev_sectors) {
3094	resize = (rs->dev_sectors != rdev_sectors - reshape_sectors);
3095	if (rs->dev_sectors > rdev_sectors - reshape_sectors)
3096	set_bit(RT_FLAG_RS_GROW, addr: &rs->runtime_flags);
3097	}
3098
3099	INIT_WORK(&rs->md.event_work, do_table_event);
3100	ti->private = rs;
3101	ti->num_flush_bios = 1;
3102	ti->needs_bio_set_dev = true;
3103
3104	/* Restore any requested new layout for conversion decision */
3105	rs_config_restore(rs, l: &rs_layout);
3106
3107	/*
3108	* Now that we have any superblock metadata available,
3109	* check for new, recovering, reshaping, to be taken over,
3110	* to be reshaped or an existing, unchanged raid set to
3111	* run in sequence.
3112	*/
3113	if (test_bit(MD_ARRAY_FIRST_USE, &rs->md.flags)) {
3114	/* A new raid6 set has to be recovered to ensure proper parity and Q-Syndrome */
3115	if (rs_is_raid6(rs) &&
3116	test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags)) {
3117	ti->error = "'nosync' not allowed for new raid6 set";
3118	r = -EINVAL;
3119	goto bad;
3120	}
3121	rs_setup_recovery(rs, dev_sectors: 0);
3122	set_bit(RT_FLAG_UPDATE_SBS, addr: &rs->runtime_flags);
3123	rs_set_new(rs);
3124	} else if (rs_is_recovering(rs)) {
3125	/* A recovering raid set may be resized */
3126	goto size_check;
3127	} else if (rs_is_reshaping(rs)) {
3128	/* Have to reject size change request during reshape */
3129	if (resize) {
3130	ti->error = "Can't resize a reshaping raid set";
3131	r = -EPERM;
3132	goto bad;
3133	}
3134	/* skip setup rs */
3135	} else if (rs_takeover_requested(rs)) {
3136	if (rs_is_reshaping(rs)) {
3137	ti->error = "Can't takeover a reshaping raid set";
3138	r = -EPERM;
3139	goto bad;
3140	}
3141
3142	/* We can't takeover a journaled raid4/5/6 */
3143	if (test_bit(__CTR_FLAG_JOURNAL_DEV, &rs->ctr_flags)) {
3144	ti->error = "Can't takeover a journaled raid4/5/6 set";
3145	r = -EPERM;
3146	goto bad;
3147	}
3148
3149	/*
3150	* If a takeover is needed, userspace sets any additional
3151	* devices to rebuild and we can check for a valid request here.
3152	*
3153	* If acceptable, set the level to the new requested
3154	* one, prohibit requesting recovery, allow the raid
3155	* set to run and store superblocks during resume.
3156	*/
3157	r = rs_check_takeover(rs);
3158	if (r)
3159	goto bad;
3160
3161	r = rs_setup_takeover(rs);
3162	if (r)
3163	goto bad;
3164
3165	set_bit(RT_FLAG_UPDATE_SBS, addr: &rs->runtime_flags);
3166	/* Takeover ain't recovery, so disable recovery */
3167	rs_setup_recovery(rs, MaxSector);
3168	rs_set_new(rs);
3169	} else if (rs_reshape_requested(rs)) {
3170	/* Only request grow on raid set size extensions, not on reshapes. */
3171	clear_bit(RT_FLAG_RS_GROW, addr: &rs->runtime_flags);
3172
3173	/*
3174	* No need to check for 'ongoing' takeover here, because takeover
3175	* is an instant operation as oposed to an ongoing reshape.
3176	*/
3177
3178	/* We can't reshape a journaled raid4/5/6 */
3179	if (test_bit(__CTR_FLAG_JOURNAL_DEV, &rs->ctr_flags)) {
3180	ti->error = "Can't reshape a journaled raid4/5/6 set";
3181	r = -EPERM;
3182	goto bad;
3183	}
3184
3185	/* Out-of-place space has to be available to allow for a reshape unless raid1! */
3186	if (reshape_sectors || rs_is_raid1(rs)) {
3187	/*
3188	* We can only prepare for a reshape here, because the
3189	* raid set needs to run to provide the repective reshape
3190	* check functions via its MD personality instance.
3191	*
3192	* So do the reshape check after md_run() succeeded.
3193	*/
3194	r = rs_prepare_reshape(rs);
3195	if (r)
3196	goto bad;
3197
3198	/* Reshaping ain't recovery, so disable recovery */
3199	rs_setup_recovery(rs, MaxSector);
3200	}
3201	rs_set_cur(rs);
3202	} else {
3203	size_check:
3204	/* May not set recovery when a device rebuild is requested */
3205	if (test_bit(__CTR_FLAG_REBUILD, &rs->ctr_flags)) {
3206	clear_bit(RT_FLAG_RS_GROW, addr: &rs->runtime_flags);
3207	set_bit(RT_FLAG_UPDATE_SBS, addr: &rs->runtime_flags);
3208	rs_setup_recovery(rs, MaxSector);
3209	} else if (test_bit(RT_FLAG_RS_GROW, &rs->runtime_flags)) {
3210	/*
3211	* Set raid set to current size, i.e. size as of
3212	* superblocks to grow to larger size in preresume.
3213	*/
3214	r = rs_set_dev_and_array_sectors(rs, sectors: sb_array_sectors, use_mddev: false);
3215	if (r)
3216	goto bad;
3217
3218	rs_setup_recovery(rs, dev_sectors: rs->md.recovery_cp < rs->md.dev_sectors ? rs->md.recovery_cp : rs->md.dev_sectors);
3219	} else {
3220	/* This is no size change or it is shrinking, update size and record in superblocks */
3221	r = rs_set_dev_and_array_sectors(rs, sectors: rs->ti->len, use_mddev: false);
3222	if (r)
3223	goto bad;
3224
3225	if (sb_array_sectors > rs->array_sectors)
3226	set_bit(RT_FLAG_UPDATE_SBS, addr: &rs->runtime_flags);
3227	}
3228	rs_set_cur(rs);
3229	}
3230
3231	/* If constructor requested it, change data and new_data offsets */
3232	r = rs_adjust_data_offsets(rs);
3233	if (r)
3234	goto bad;
3235
3236	/* Catch any inconclusive reshape superblock content. */
3237	rs_reset_inconclusive_reshape(rs);
3238
3239	/* Start raid set read-only and assumed clean to change in raid_resume() */
3240	rs->md.ro = 1;
3241	rs->md.in_sync = 1;
3242
3243	/* Keep array frozen until resume. */
3244	set_bit(nr: MD_RECOVERY_FROZEN, addr: &rs->md.recovery);
3245
3246	/* Has to be held on running the array */
3247	mddev_suspend_and_lock_nointr(mddev: &rs->md);
3248	r = md_run(mddev: &rs->md);
3249	rs->md.in_sync = 0; /* Assume already marked dirty */
3250	if (r) {
3251	ti->error = "Failed to run raid array";
3252	mddev_unlock(mddev: &rs->md);
3253	goto bad;
3254	}
3255
3256	r = md_start(mddev: &rs->md);
3257	if (r) {
3258	ti->error = "Failed to start raid array";
3259	goto bad_unlock;
3260	}
3261
3262	/* If raid4/5/6 journal mode explicitly requested (only possible with journal dev) -> set it */
3263	if (test_bit(__CTR_FLAG_JOURNAL_MODE, &rs->ctr_flags)) {
3264	r = r5c_journal_mode_set(mddev: &rs->md, journal_mode: rs->journal_dev.mode);
3265	if (r) {
3266	ti->error = "Failed to set raid4/5/6 journal mode";
3267	goto bad_unlock;
3268	}
3269	}
3270
3271	set_bit(RT_FLAG_RS_SUSPENDED, addr: &rs->runtime_flags);
3272
3273	/* Try to adjust the raid4/5/6 stripe cache size to the stripe size */
3274	if (rs_is_raid456(rs)) {
3275	r = rs_set_raid456_stripe_cache(rs);
3276	if (r)
3277	goto bad_unlock;
3278	}
3279
3280	/* Now do an early reshape check */
3281	if (test_bit(RT_FLAG_RESHAPE_RS, &rs->runtime_flags)) {
3282	r = rs_check_reshape(rs);
3283	if (r)
3284	goto bad_unlock;
3285
3286	/* Restore new, ctr requested layout to perform check */
3287	rs_config_restore(rs, l: &rs_layout);
3288
3289	if (rs->md.pers->start_reshape) {
3290	r = rs->md.pers->check_reshape(&rs->md);
3291	if (r) {
3292	ti->error = "Reshape check failed";
3293	goto bad_unlock;
3294	}
3295	}
3296	}
3297
3298	/* Disable/enable discard support on raid set. */
3299	configure_discard_support(rs);
3300
3301	mddev_unlock(mddev: &rs->md);
3302	return 0;
3303
3304	bad_unlock:
3305	md_stop(mddev: &rs->md);
3306	mddev_unlock(mddev: &rs->md);
3307	bad:
3308	raid_set_free(rs);
3309
3310	return r;
3311	}
3312
3313	static void raid_dtr(struct dm_target *ti)
3314	{
3315	struct raid_set *rs = ti->private;
3316
3317	mddev_lock_nointr(mddev: &rs->md);
3318	md_stop(mddev: &rs->md);
3319	mddev_unlock(mddev: &rs->md);
3320	raid_set_free(rs);
3321	}
3322
3323	static int raid_map(struct dm_target *ti, struct bio *bio)
3324	{
3325	struct raid_set *rs = ti->private;
3326	struct mddev *mddev = &rs->md;
3327
3328	/*
3329	* If we're reshaping to add disk(s)), ti->len and
3330	* mddev->array_sectors will differ during the process
3331	* (ti->len > mddev->array_sectors), so we have to requeue
3332	* bios with addresses > mddev->array_sectors here or
3333	* there will occur accesses past EOD of the component
3334	* data images thus erroring the raid set.
3335	*/
3336	if (unlikely(bio_end_sector(bio) > mddev->array_sectors))
3337	return DM_MAPIO_REQUEUE;
3338
3339	md_handle_request(mddev, bio);
3340
3341	return DM_MAPIO_SUBMITTED;
3342	}
3343
3344	/* Return sync state string for @state */
3345	enum sync_state { st_frozen, st_reshape, st_resync, st_check, st_repair, st_recover, st_idle };
3346	static const char *sync_str(enum sync_state state)
3347	{
3348	/* Has to be in above sync_state order! */
3349	static const char *sync_strs[] = {
3350	"frozen",
3351	"reshape",
3352	"resync",
3353	"check",
3354	"repair",
3355	"recover",
3356	"idle"
3357	};
3358
3359	return __within_range(v: state, min: 0, ARRAY_SIZE(sync_strs) - 1) ? sync_strs[state] : "undef";
3360	};
3361
3362	/* Return enum sync_state for @mddev derived from @recovery flags */
3363	static enum sync_state decipher_sync_action(struct mddev *mddev, unsigned long recovery)
3364	{
3365	if (test_bit(MD_RECOVERY_FROZEN, &recovery))
3366	return st_frozen;
3367
3368	/* The MD sync thread can be done with io or be interrupted but still be running */
3369	if (!test_bit(MD_RECOVERY_DONE, &recovery) &&
3370	(test_bit(MD_RECOVERY_RUNNING, &recovery) ||
3371	(!mddev->ro && test_bit(MD_RECOVERY_NEEDED, &recovery)))) {
3372	if (test_bit(MD_RECOVERY_RESHAPE, &recovery))
3373	return st_reshape;
3374
3375	if (test_bit(MD_RECOVERY_SYNC, &recovery)) {
3376	if (!test_bit(MD_RECOVERY_REQUESTED, &recovery))
3377	return st_resync;
3378	if (test_bit(MD_RECOVERY_CHECK, &recovery))
3379	return st_check;
3380	return st_repair;
3381	}
3382
3383	if (test_bit(MD_RECOVERY_RECOVER, &recovery))
3384	return st_recover;
3385
3386	if (mddev->reshape_position != MaxSector)
3387	return st_reshape;
3388	}
3389
3390	return st_idle;
3391	}
3392
3393	/*
3394	* Return status string for @rdev
3395	*
3396	* Status characters:
3397	*
3398	* 'D' = Dead/Failed raid set component or raid4/5/6 journal device
3399	* 'a' = Alive but not in-sync raid set component _or_ alive raid4/5/6 'write_back' journal device
3400	* 'A' = Alive and in-sync raid set component _or_ alive raid4/5/6 'write_through' journal device
3401	* '-' = Non-existing device (i.e. uspace passed '- -' into the ctr)
3402	*/
3403	static const char *__raid_dev_status(struct raid_set *rs, struct md_rdev *rdev)
3404	{
3405	if (!rdev->bdev)
3406	return "-";
3407	else if (test_bit(Faulty, &rdev->flags))
3408	return "D";
3409	else if (test_bit(Journal, &rdev->flags))
3410	return (rs->journal_dev.mode == R5C_JOURNAL_MODE_WRITE_THROUGH) ? "A" : "a";
3411	else if (test_bit(RT_FLAG_RS_RESYNCING, &rs->runtime_flags) ||
3412	(!test_bit(RT_FLAG_RS_IN_SYNC, &rs->runtime_flags) &&
3413	!test_bit(In_sync, &rdev->flags)))
3414	return "a";
3415	else
3416	return "A";
3417	}
3418
3419	/* Helper to return resync/reshape progress for @rs and runtime flags for raid set in sync / resynching */
3420	static sector_t rs_get_progress(struct raid_set *rs, unsigned long recovery,
3421	enum sync_state state, sector_t resync_max_sectors)
3422	{
3423	sector_t r;
3424	struct mddev *mddev = &rs->md;
3425
3426	clear_bit(RT_FLAG_RS_IN_SYNC, addr: &rs->runtime_flags);
3427	clear_bit(RT_FLAG_RS_RESYNCING, addr: &rs->runtime_flags);
3428
3429	if (rs_is_raid0(rs)) {
3430	r = resync_max_sectors;
3431	set_bit(RT_FLAG_RS_IN_SYNC, addr: &rs->runtime_flags);
3432
3433	} else {
3434	if (state == st_idle && !test_bit(MD_RECOVERY_INTR, &recovery))
3435	r = mddev->recovery_cp;
3436	else
3437	r = mddev->curr_resync_completed;
3438
3439	if (state == st_idle && r >= resync_max_sectors) {
3440	/*
3441	* Sync complete.
3442	*/
3443	/* In case we have finished recovering, the array is in sync. */
3444	if (test_bit(MD_RECOVERY_RECOVER, &recovery))
3445	set_bit(RT_FLAG_RS_IN_SYNC, addr: &rs->runtime_flags);
3446
3447	} else if (state == st_recover)
3448	/*
3449	* In case we are recovering, the array is not in sync
3450	* and health chars should show the recovering legs.
3451	*
3452	* Already retrieved recovery offset from curr_resync_completed above.
3453	*/
3454	;
3455
3456	else if (state == st_resync || state == st_reshape)
3457	/*
3458	* If "resync/reshape" is occurring, the raid set
3459	* is or may be out of sync hence the health
3460	* characters shall be 'a'.
3461	*/
3462	set_bit(RT_FLAG_RS_RESYNCING, addr: &rs->runtime_flags);
3463
3464	else if (state == st_check || state == st_repair)
3465	/*
3466	* If "check" or "repair" is occurring, the raid set has
3467	* undergone an initial sync and the health characters
3468	* should not be 'a' anymore.
3469	*/
3470	set_bit(RT_FLAG_RS_IN_SYNC, addr: &rs->runtime_flags);
3471
3472	else if (test_bit(MD_RECOVERY_NEEDED, &recovery))
3473	/*
3474	* We are idle and recovery is needed, prevent 'A' chars race
3475	* caused by components still set to in-sync by constructor.
3476	*/
3477	set_bit(RT_FLAG_RS_RESYNCING, addr: &rs->runtime_flags);
3478
3479	else {
3480	/*
3481	* We are idle and the raid set may be doing an initial
3482	* sync, or it may be rebuilding individual components.
3483	* If all the devices are In_sync, then it is the raid set
3484	* that is being initialized.
3485	*/
3486	struct md_rdev *rdev;
3487
3488	set_bit(RT_FLAG_RS_IN_SYNC, addr: &rs->runtime_flags);
3489	rdev_for_each(rdev, mddev)
3490	if (!test_bit(Journal, &rdev->flags) &&
3491	!test_bit(In_sync, &rdev->flags)) {
3492	clear_bit(RT_FLAG_RS_IN_SYNC, addr: &rs->runtime_flags);
3493	break;
3494	}
3495	}
3496	}
3497
3498	return min(r, resync_max_sectors);
3499	}
3500
3501	/* Helper to return @dev name or "-" if !@dev */
3502	static const char *__get_dev_name(struct dm_dev *dev)
3503	{
3504	return dev ? dev->name : "-";
3505	}
3506
3507	static void raid_status(struct dm_target *ti, status_type_t type,
3508	unsigned int status_flags, char *result, unsigned int maxlen)
3509	{
3510	struct raid_set *rs = ti->private;
3511	struct mddev *mddev = &rs->md;
3512	struct r5conf *conf = rs_is_raid456(rs) ? mddev->private : NULL;
3513	int i, max_nr_stripes = conf ? conf->max_nr_stripes : 0;
3514	unsigned long recovery;
3515	unsigned int raid_param_cnt = 1; /* at least 1 for chunksize */
3516	unsigned int sz = 0;
3517	unsigned int rebuild_writemostly_count = 0;
3518	sector_t progress, resync_max_sectors, resync_mismatches;
3519	enum sync_state state;
3520	struct raid_type *rt;
3521
3522	switch (type) {
3523	case STATUSTYPE_INFO:
3524	/* *Should* always succeed */
3525	rt = get_raid_type_by_ll(level: mddev->new_level, layout: mddev->new_layout);
3526	if (!rt)
3527	return;
3528
3529	DMEMIT("%s %d ", rt->name, mddev->raid_disks);
3530
3531	/* Access most recent mddev properties for status output */
3532	smp_rmb();
3533	/* Get sensible max sectors even if raid set not yet started */
3534	resync_max_sectors = test_bit(RT_FLAG_RS_PRERESUMED, &rs->runtime_flags) ?
3535	mddev->resync_max_sectors : mddev->dev_sectors;
3536	recovery = rs->md.recovery;
3537	state = decipher_sync_action(mddev, recovery);
3538	progress = rs_get_progress(rs, recovery, state, resync_max_sectors);
3539	resync_mismatches = (mddev->last_sync_action && !strcasecmp(s1: mddev->last_sync_action, s2: "check")) ?
3540	atomic64_read(v: &mddev->resync_mismatches) : 0;
3541
3542	/* HM FIXME: do we want another state char for raid0? It shows 'D'/'A'/'-' now */
3543	for (i = 0; i < rs->raid_disks; i++)
3544	DMEMIT(__raid_dev_status(rs, &rs->dev[i].rdev));
3545
3546	/*
3547	* In-sync/Reshape ratio:
3548	* The in-sync ratio shows the progress of:
3549	* - Initializing the raid set
3550	* - Rebuilding a subset of devices of the raid set
3551	* The user can distinguish between the two by referring
3552	* to the status characters.
3553	*
3554	* The reshape ratio shows the progress of
3555	* changing the raid layout or the number of
3556	* disks of a raid set
3557	*/
3558	DMEMIT(" %llu/%llu", (unsigned long long) progress,
3559	(unsigned long long) resync_max_sectors);
3560
3561	/*
3562	* v1.5.0+:
3563	*
3564	* Sync action:
3565	* See Documentation/admin-guide/device-mapper/dm-raid.rst for
3566	* information on each of these states.
3567	*/
3568	DMEMIT(" %s", sync_str(state));
3569
3570	/*
3571	* v1.5.0+:
3572	*
3573	* resync_mismatches/mismatch_cnt
3574	* This field shows the number of discrepancies found when
3575	* performing a "check" of the raid set.
3576	*/
3577	DMEMIT(" %llu", (unsigned long long) resync_mismatches);
3578
3579	/*
3580	* v1.9.0+:
3581	*
3582	* data_offset (needed for out of space reshaping)
3583	* This field shows the data offset into the data
3584	* image LV where the first stripes data starts.
3585	*
3586	* We keep data_offset equal on all raid disks of the set,
3587	* so retrieving it from the first raid disk is sufficient.
3588	*/
3589	DMEMIT(" %llu", (unsigned long long) rs->dev[0].rdev.data_offset);
3590
3591	/*
3592	* v1.10.0+:
3593	*/
3594	DMEMIT(" %s", test_bit(__CTR_FLAG_JOURNAL_DEV, &rs->ctr_flags) ?
3595	__raid_dev_status(rs, &rs->journal_dev.rdev) : "-");
3596	break;
3597
3598	case STATUSTYPE_TABLE:
3599	/* Report the table line string you would use to construct this raid set */
3600
3601	/*
3602	* Count any rebuild or writemostly argument pairs and subtract the
3603	* hweight count being added below of any rebuild and writemostly ctr flags.
3604	*/
3605	for (i = 0; i < rs->raid_disks; i++) {
3606	rebuild_writemostly_count += (test_bit(i, (void *) rs->rebuild_disks) ? 2 : 0) +
3607	(test_bit(WriteMostly, &rs->dev[i].rdev.flags) ? 2 : 0);
3608	}
3609	rebuild_writemostly_count -= (test_bit(__CTR_FLAG_REBUILD, &rs->ctr_flags) ? 2 : 0) +
3610	(test_bit(__CTR_FLAG_WRITE_MOSTLY, &rs->ctr_flags) ? 2 : 0);
3611	/* Calculate raid parameter count based on ^ rebuild/writemostly argument counts and ctr flags set. */
3612	raid_param_cnt += rebuild_writemostly_count +
3613	hweight32(rs->ctr_flags & CTR_FLAG_OPTIONS_NO_ARGS) +
3614	hweight32(rs->ctr_flags & CTR_FLAG_OPTIONS_ONE_ARG) * 2;
3615	/* Emit table line */
3616	/* This has to be in the documented order for userspace! */
3617	DMEMIT("%s %u %u", rs->raid_type->name, raid_param_cnt, mddev->new_chunk_sectors);
3618	if (test_bit(__CTR_FLAG_SYNC, &rs->ctr_flags))
3619	DMEMIT(" %s", dm_raid_arg_name_by_flag(CTR_FLAG_SYNC));
3620	if (test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags))
3621	DMEMIT(" %s", dm_raid_arg_name_by_flag(CTR_FLAG_NOSYNC));
3622	if (test_bit(__CTR_FLAG_REBUILD, &rs->ctr_flags))
3623	for (i = 0; i < rs->raid_disks; i++)
3624	if (test_bit(i, (void *) rs->rebuild_disks))
3625	DMEMIT(" %s %u", dm_raid_arg_name_by_flag(CTR_FLAG_REBUILD), i);
3626	if (test_bit(__CTR_FLAG_DAEMON_SLEEP, &rs->ctr_flags))
3627	DMEMIT(" %s %lu", dm_raid_arg_name_by_flag(CTR_FLAG_DAEMON_SLEEP),
3628	mddev->bitmap_info.daemon_sleep);
3629	if (test_bit(__CTR_FLAG_MIN_RECOVERY_RATE, &rs->ctr_flags))
3630	DMEMIT(" %s %d", dm_raid_arg_name_by_flag(CTR_FLAG_MIN_RECOVERY_RATE),
3631	mddev->sync_speed_min);
3632	if (test_bit(__CTR_FLAG_MAX_RECOVERY_RATE, &rs->ctr_flags))
3633	DMEMIT(" %s %d", dm_raid_arg_name_by_flag(CTR_FLAG_MAX_RECOVERY_RATE),
3634	mddev->sync_speed_max);
3635	if (test_bit(__CTR_FLAG_WRITE_MOSTLY, &rs->ctr_flags))
3636	for (i = 0; i < rs->raid_disks; i++)
3637	if (test_bit(WriteMostly, &rs->dev[i].rdev.flags))
3638	DMEMIT(" %s %d", dm_raid_arg_name_by_flag(CTR_FLAG_WRITE_MOSTLY),
3639	rs->dev[i].rdev.raid_disk);
3640	if (test_bit(__CTR_FLAG_MAX_WRITE_BEHIND, &rs->ctr_flags))
3641	DMEMIT(" %s %lu", dm_raid_arg_name_by_flag(CTR_FLAG_MAX_WRITE_BEHIND),
3642	mddev->bitmap_info.max_write_behind);
3643	if (test_bit(__CTR_FLAG_STRIPE_CACHE, &rs->ctr_flags))
3644	DMEMIT(" %s %d", dm_raid_arg_name_by_flag(CTR_FLAG_STRIPE_CACHE),
3645	max_nr_stripes);
3646	if (test_bit(__CTR_FLAG_REGION_SIZE, &rs->ctr_flags))
3647	DMEMIT(" %s %llu", dm_raid_arg_name_by_flag(CTR_FLAG_REGION_SIZE),
3648	(unsigned long long) to_sector(mddev->bitmap_info.chunksize));
3649	if (test_bit(__CTR_FLAG_RAID10_COPIES, &rs->ctr_flags))
3650	DMEMIT(" %s %d", dm_raid_arg_name_by_flag(CTR_FLAG_RAID10_COPIES),
3651	raid10_md_layout_to_copies(mddev->layout));
3652	if (test_bit(__CTR_FLAG_RAID10_FORMAT, &rs->ctr_flags))
3653	DMEMIT(" %s %s", dm_raid_arg_name_by_flag(CTR_FLAG_RAID10_FORMAT),
3654	raid10_md_layout_to_format(mddev->layout));
3655	if (test_bit(__CTR_FLAG_DELTA_DISKS, &rs->ctr_flags))
3656	DMEMIT(" %s %d", dm_raid_arg_name_by_flag(CTR_FLAG_DELTA_DISKS),
3657	max(rs->delta_disks, mddev->delta_disks));
3658	if (test_bit(__CTR_FLAG_DATA_OFFSET, &rs->ctr_flags))
3659	DMEMIT(" %s %llu", dm_raid_arg_name_by_flag(CTR_FLAG_DATA_OFFSET),
3660	(unsigned long long) rs->data_offset);
3661	if (test_bit(__CTR_FLAG_JOURNAL_DEV, &rs->ctr_flags))
3662	DMEMIT(" %s %s", dm_raid_arg_name_by_flag(CTR_FLAG_JOURNAL_DEV),
3663	__get_dev_name(rs->journal_dev.dev));
3664	if (test_bit(__CTR_FLAG_JOURNAL_MODE, &rs->ctr_flags))
3665	DMEMIT(" %s %s", dm_raid_arg_name_by_flag(CTR_FLAG_JOURNAL_MODE),
3666	md_journal_mode_to_dm_raid(rs->journal_dev.mode));
3667	DMEMIT(" %d", rs->raid_disks);
3668	for (i = 0; i < rs->raid_disks; i++)
3669	DMEMIT(" %s %s", __get_dev_name(rs->dev[i].meta_dev),
3670	__get_dev_name(rs->dev[i].data_dev));
3671	break;
3672
3673	case STATUSTYPE_IMA:
3674	rt = get_raid_type_by_ll(level: mddev->new_level, layout: mddev->new_layout);
3675	if (!rt)
3676	return;
3677
3678	DMEMIT_TARGET_NAME_VERSION(ti->type);
3679	DMEMIT(",raid_type=%s,raid_disks=%d", rt->name, mddev->raid_disks);
3680
3681	/* Access most recent mddev properties for status output */
3682	smp_rmb();
3683	recovery = rs->md.recovery;
3684	state = decipher_sync_action(mddev, recovery);
3685	DMEMIT(",raid_state=%s", sync_str(state));
3686
3687	for (i = 0; i < rs->raid_disks; i++) {
3688	DMEMIT(",raid_device_%d_status=", i);
3689	DMEMIT(__raid_dev_status(rs, &rs->dev[i].rdev));
3690	}
3691
3692	if (rt_is_raid456(rt)) {
3693	DMEMIT(",journal_dev_mode=");
3694	switch (rs->journal_dev.mode) {
3695	case R5C_JOURNAL_MODE_WRITE_THROUGH:
3696	DMEMIT("%s",
3697	_raid456_journal_mode[R5C_JOURNAL_MODE_WRITE_THROUGH].param);
3698	break;
3699	case R5C_JOURNAL_MODE_WRITE_BACK:
3700	DMEMIT("%s",
3701	_raid456_journal_mode[R5C_JOURNAL_MODE_WRITE_BACK].param);
3702	break;
3703	default:
3704	DMEMIT("invalid");
3705	break;
3706	}
3707	}
3708	DMEMIT(";");
3709	break;
3710	}
3711	}
3712
3713	static int raid_message(struct dm_target *ti, unsigned int argc, char **argv,
3714	char *result, unsigned int maxlen)
3715	{
3716	struct raid_set *rs = ti->private;
3717	struct mddev *mddev = &rs->md;
3718
3719	if (!mddev->pers || !mddev->pers->sync_request)
3720	return -EINVAL;
3721
3722	if (!strcasecmp(s1: argv[0], s2: "frozen"))
3723	set_bit(nr: MD_RECOVERY_FROZEN, addr: &mddev->recovery);
3724	else
3725	clear_bit(nr: MD_RECOVERY_FROZEN, addr: &mddev->recovery);
3726
3727	if (!strcasecmp(s1: argv[0], s2: "idle") || !strcasecmp(s1: argv[0], s2: "frozen")) {
3728	if (mddev->sync_thread) {
3729	set_bit(nr: MD_RECOVERY_INTR, addr: &mddev->recovery);
3730	md_reap_sync_thread(mddev);
3731	}
3732	} else if (decipher_sync_action(mddev, recovery: mddev->recovery) != st_idle)
3733	return -EBUSY;
3734	else if (!strcasecmp(s1: argv[0], s2: "resync"))
3735	; /* MD_RECOVERY_NEEDED set below */
3736	else if (!strcasecmp(s1: argv[0], s2: "recover"))
3737	set_bit(nr: MD_RECOVERY_RECOVER, addr: &mddev->recovery);
3738	else {
3739	if (!strcasecmp(s1: argv[0], s2: "check")) {
3740	set_bit(nr: MD_RECOVERY_CHECK, addr: &mddev->recovery);
3741	set_bit(nr: MD_RECOVERY_REQUESTED, addr: &mddev->recovery);
3742	set_bit(nr: MD_RECOVERY_SYNC, addr: &mddev->recovery);
3743	} else if (!strcasecmp(s1: argv[0], s2: "repair")) {
3744	set_bit(nr: MD_RECOVERY_REQUESTED, addr: &mddev->recovery);
3745	set_bit(nr: MD_RECOVERY_SYNC, addr: &mddev->recovery);
3746	} else
3747	return -EINVAL;
3748	}
3749	if (mddev->ro == 2) {
3750	/* A write to sync_action is enough to justify
3751	* canceling read-auto mode
3752	*/
3753	mddev->ro = 0;
3754	if (!mddev->suspended)
3755	md_wakeup_thread(thread: mddev->sync_thread);
3756	}
3757	set_bit(nr: MD_RECOVERY_NEEDED, addr: &mddev->recovery);
3758	if (!mddev->suspended)
3759	md_wakeup_thread(thread: mddev->thread);
3760
3761	return 0;
3762	}
3763
3764	static int raid_iterate_devices(struct dm_target *ti,
3765	iterate_devices_callout_fn fn, void *data)
3766	{
3767	struct raid_set *rs = ti->private;
3768	unsigned int i;
3769	int r = 0;
3770
3771	for (i = 0; !r && i < rs->raid_disks; i++) {
3772	if (rs->dev[i].data_dev) {
3773	r = fn(ti, rs->dev[i].data_dev,
3774	0, /* No offset on data devs */
3775	rs->md.dev_sectors, data);
3776	}
3777	}
3778
3779	return r;
3780	}
3781
3782	static void raid_io_hints(struct dm_target *ti, struct queue_limits *limits)
3783	{
3784	struct raid_set *rs = ti->private;
3785	unsigned int chunk_size_bytes = to_bytes(n: rs->md.chunk_sectors);
3786
3787	blk_limits_io_min(limits, min: chunk_size_bytes);
3788	blk_limits_io_opt(limits, opt: chunk_size_bytes * mddev_data_stripes(rs));
3789	}
3790
3791	static void raid_postsuspend(struct dm_target *ti)
3792	{
3793	struct raid_set *rs = ti->private;
3794
3795	if (!test_and_set_bit(RT_FLAG_RS_SUSPENDED, addr: &rs->runtime_flags)) {
3796	/* Writes have to be stopped before suspending to avoid deadlocks. */
3797	if (!test_bit(MD_RECOVERY_FROZEN, &rs->md.recovery))
3798	md_stop_writes(mddev: &rs->md);
3799
3800	mddev_suspend(mddev: &rs->md, interruptible: false);
3801	}
3802	}
3803
3804	static void attempt_restore_of_faulty_devices(struct raid_set *rs)
3805	{
3806	int i;
3807	uint64_t cleared_failed_devices[DISKS_ARRAY_ELEMS];
3808	unsigned long flags;
3809	bool cleared = false;
3810	struct dm_raid_superblock *sb;
3811	struct mddev *mddev = &rs->md;
3812	struct md_rdev *r;
3813
3814	/* RAID personalities have to provide hot add/remove methods or we need to bail out. */
3815	if (!mddev->pers || !mddev->pers->hot_add_disk || !mddev->pers->hot_remove_disk)
3816	return;
3817
3818	memset(cleared_failed_devices, 0, sizeof(cleared_failed_devices));
3819
3820	for (i = 0; i < rs->raid_disks; i++) {
3821	r = &rs->dev[i].rdev;
3822	/* HM FIXME: enhance journal device recovery processing */
3823	if (test_bit(Journal, &r->flags))
3824	continue;
3825
3826	if (test_bit(Faulty, &r->flags) &&
3827	r->meta_bdev && !read_disk_sb(rdev: r, size: r->sb_size, force_reload: true)) {
3828	DMINFO("Faulty %s device #%d has readable super block."
3829	" Attempting to revive it.",
3830	rs->raid_type->name, i);
3831
3832	/*
3833	* Faulty bit may be set, but sometimes the array can
3834	* be suspended before the personalities can respond
3835	* by removing the device from the array (i.e. calling
3836	* 'hot_remove_disk'). If they haven't yet removed
3837	* the failed device, its 'raid_disk' number will be
3838	* '>= 0' - meaning we must call this function
3839	* ourselves.
3840	*/
3841	flags = r->flags;
3842	clear_bit(nr: In_sync, addr: &r->flags); /* Mandatory for hot remove. */
3843	if (r->raid_disk >= 0) {
3844	if (mddev->pers->hot_remove_disk(mddev, r)) {
3845	/* Failed to revive this device, try next */
3846	r->flags = flags;
3847	continue;
3848	}
3849	} else
3850	r->raid_disk = r->saved_raid_disk = i;
3851
3852	clear_bit(nr: Faulty, addr: &r->flags);
3853	clear_bit(nr: WriteErrorSeen, addr: &r->flags);
3854
3855	if (mddev->pers->hot_add_disk(mddev, r)) {
3856	/* Failed to revive this device, try next */
3857	r->raid_disk = r->saved_raid_disk = -1;
3858	r->flags = flags;
3859	} else {
3860	clear_bit(nr: In_sync, addr: &r->flags);
3861	r->recovery_offset = 0;
3862	set_bit(nr: i, addr: (void *) cleared_failed_devices);
3863	cleared = true;
3864	}
3865	}
3866	}
3867
3868	/* If any failed devices could be cleared, update all sbs failed_devices bits */
3869	if (cleared) {
3870	uint64_t failed_devices[DISKS_ARRAY_ELEMS];
3871
3872	rdev_for_each(r, &rs->md) {
3873	if (test_bit(Journal, &r->flags))
3874	continue;
3875
3876	sb = page_address(r->sb_page);
3877	sb_retrieve_failed_devices(sb, failed_devices);
3878
3879	for (i = 0; i < DISKS_ARRAY_ELEMS; i++)
3880	failed_devices[i] &= ~cleared_failed_devices[i];
3881
3882	sb_update_failed_devices(sb, failed_devices);
3883	}
3884	}
3885	}
3886
3887	static int __load_dirty_region_bitmap(struct raid_set *rs)
3888	{
3889	int r = 0;
3890
3891	/* Try loading the bitmap unless "raid0", which does not have one */
3892	if (!rs_is_raid0(rs) &&
3893	!test_and_set_bit(RT_FLAG_RS_BITMAP_LOADED, addr: &rs->runtime_flags)) {
3894	r = md_bitmap_load(mddev: &rs->md);
3895	if (r)
3896	DMERR("Failed to load bitmap");
3897	}
3898
3899	return r;
3900	}
3901
3902	/* Enforce updating all superblocks */
3903	static void rs_update_sbs(struct raid_set *rs)
3904	{
3905	struct mddev *mddev = &rs->md;
3906	int ro = mddev->ro;
3907
3908	set_bit(nr: MD_SB_CHANGE_DEVS, addr: &mddev->sb_flags);
3909	mddev->ro = 0;
3910	md_update_sb(mddev, force: 1);
3911	mddev->ro = ro;
3912	}
3913
3914	/*
3915	* Reshape changes raid algorithm of @rs to new one within personality
3916	* (e.g. raid6_zr -> raid6_nc), changes stripe size, adds/removes
3917	* disks from a raid set thus growing/shrinking it or resizes the set
3918	*
3919	* Call mddev_lock_nointr() before!
3920	*/
3921	static int rs_start_reshape(struct raid_set *rs)
3922	{
3923	int r;
3924	struct mddev *mddev = &rs->md;
3925	struct md_personality *pers = mddev->pers;
3926
3927	/* Don't allow the sync thread to work until the table gets reloaded. */
3928	set_bit(nr: MD_RECOVERY_WAIT, addr: &mddev->recovery);
3929
3930	r = rs_setup_reshape(rs);
3931	if (r)
3932	return r;
3933
3934	/*
3935	* Check any reshape constraints enforced by the personalility
3936	*
3937	* May as well already kick the reshape off so that * pers->start_reshape() becomes optional.
3938	*/
3939	r = pers->check_reshape(mddev);
3940	if (r) {
3941	rs->ti->error = "pers->check_reshape() failed";
3942	return r;
3943	}
3944
3945	/*
3946	* Personality may not provide start reshape method in which
3947	* case check_reshape above has already covered everything
3948	*/
3949	if (pers->start_reshape) {
3950	r = pers->start_reshape(mddev);
3951	if (r) {
3952	rs->ti->error = "pers->start_reshape() failed";
3953	return r;
3954	}
3955	}
3956
3957	/*
3958	* Now reshape got set up, update superblocks to
3959	* reflect the fact so that a table reload will
3960	* access proper superblock content in the ctr.
3961	*/
3962	rs_update_sbs(rs);
3963
3964	return 0;
3965	}
3966
3967	static int raid_preresume(struct dm_target *ti)
3968	{
3969	int r;
3970	struct raid_set *rs = ti->private;
3971	struct mddev *mddev = &rs->md;
3972
3973	/* This is a resume after a suspend of the set -> it's already started. */
3974	if (test_and_set_bit(RT_FLAG_RS_PRERESUMED, addr: &rs->runtime_flags))
3975	return 0;
3976
3977	/*
3978	* The superblocks need to be updated on disk if the
3979	* array is new or new devices got added (thus zeroed
3980	* out by userspace) or __load_dirty_region_bitmap
3981	* will overwrite them in core with old data or fail.
3982	*/
3983	if (test_bit(RT_FLAG_UPDATE_SBS, &rs->runtime_flags))
3984	rs_update_sbs(rs);
3985
3986	/* Load the bitmap from disk unless raid0 */
3987	r = __load_dirty_region_bitmap(rs);
3988	if (r)
3989	return r;
3990
3991	/* We are extending the raid set size, adjust mddev/md_rdev sizes and set capacity. */
3992	if (test_bit(RT_FLAG_RS_GROW, &rs->runtime_flags)) {
3993	mddev->array_sectors = rs->array_sectors;
3994	mddev->dev_sectors = rs->dev_sectors;
3995	rs_set_rdev_sectors(rs);
3996	rs_set_capacity(rs);
3997	}
3998
3999	/* Resize bitmap to adjust to changed region size (aka MD bitmap chunksize) or grown device size */
4000	if (test_bit(RT_FLAG_RS_BITMAP_LOADED, &rs->runtime_flags) && mddev->bitmap &&
4001	(test_bit(RT_FLAG_RS_GROW, &rs->runtime_flags) ||
4002	(rs->requested_bitmap_chunk_sectors &&
4003	mddev->bitmap_info.chunksize != to_bytes(n: rs->requested_bitmap_chunk_sectors)))) {
4004	int chunksize = to_bytes(n: rs->requested_bitmap_chunk_sectors) ?: mddev->bitmap_info.chunksize;
4005
4006	r = md_bitmap_resize(bitmap: mddev->bitmap, blocks: mddev->dev_sectors, chunksize, init: 0);
4007	if (r)
4008	DMERR("Failed to resize bitmap");
4009	}
4010
4011	/* Check for any resize/reshape on @rs and adjust/initiate */
4012	/* Be prepared for mddev_resume() in raid_resume() */
4013	set_bit(nr: MD_RECOVERY_FROZEN, addr: &mddev->recovery);
4014	if (mddev->recovery_cp && mddev->recovery_cp < MaxSector) {
4015	set_bit(nr: MD_RECOVERY_REQUESTED, addr: &mddev->recovery);
4016	mddev->resync_min = mddev->recovery_cp;
4017	if (test_bit(RT_FLAG_RS_GROW, &rs->runtime_flags))
4018	mddev->resync_max_sectors = mddev->dev_sectors;
4019	}
4020
4021	/* Check for any reshape request unless new raid set */
4022	if (test_bit(RT_FLAG_RESHAPE_RS, &rs->runtime_flags)) {
4023	/* Initiate a reshape. */
4024	rs_set_rdev_sectors(rs);
4025	mddev_lock_nointr(mddev);
4026	r = rs_start_reshape(rs);
4027	mddev_unlock(mddev);
4028	if (r)
4029	DMWARN("Failed to check/start reshape, continuing without change");
4030	r = 0;
4031	}
4032
4033	return r;
4034	}
4035
4036	static void raid_resume(struct dm_target *ti)
4037	{
4038	struct raid_set *rs = ti->private;
4039	struct mddev *mddev = &rs->md;
4040
4041	if (test_and_set_bit(RT_FLAG_RS_RESUMED, addr: &rs->runtime_flags)) {
4042	/*
4043	* A secondary resume while the device is active.
4044	* Take this opportunity to check whether any failed
4045	* devices are reachable again.
4046	*/
4047	attempt_restore_of_faulty_devices(rs);
4048	}
4049
4050	if (test_and_clear_bit(RT_FLAG_RS_SUSPENDED, addr: &rs->runtime_flags)) {
4051	/* Only reduce raid set size before running a disk removing reshape. */
4052	if (mddev->delta_disks < 0)
4053	rs_set_capacity(rs);
4054
4055	mddev_lock_nointr(mddev);
4056	clear_bit(nr: MD_RECOVERY_FROZEN, addr: &mddev->recovery);
4057	mddev->ro = 0;
4058	mddev->in_sync = 0;
4059	mddev_unlock_and_resume(mddev);
4060	}
4061	}
4062
4063	static struct target_type raid_target = {
4064	.name = "raid",
4065	.version = {1, 15, 1},
4066	.module = THIS_MODULE,
4067	.ctr = raid_ctr,
4068	.dtr = raid_dtr,
4069	.map = raid_map,
4070	.status = raid_status,
4071	.message = raid_message,
4072	.iterate_devices = raid_iterate_devices,
4073	.io_hints = raid_io_hints,
4074	.postsuspend = raid_postsuspend,
4075	.preresume = raid_preresume,
4076	.resume = raid_resume,
4077	};
4078	module_dm(raid);
4079
4080	module_param(devices_handle_discard_safely, bool, 0644);
4081	MODULE_PARM_DESC(devices_handle_discard_safely,
4082	"Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
4083
4084	MODULE_DESCRIPTION(DM_NAME " raid0/1/10/4/5/6 target");
4085	MODULE_ALIAS("dm-raid0");
4086	MODULE_ALIAS("dm-raid1");
4087	MODULE_ALIAS("dm-raid10");
4088	MODULE_ALIAS("dm-raid4");
4089	MODULE_ALIAS("dm-raid5");
4090	MODULE_ALIAS("dm-raid6");
4091	MODULE_AUTHOR("Neil Brown <dm-devel@redhat.com>");
4092	MODULE_AUTHOR("Heinz Mauelshagen <dm-devel@redhat.com>");
4093	MODULE_LICENSE("GPL");
4094