1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
4	*
5	* bitmap_create - sets up the bitmap structure
6	* bitmap_destroy - destroys the bitmap structure
7	*
8	* additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
9	* - added disk storage for bitmap
10	* - changes to allow various bitmap chunk sizes
11	*/
12
13	/*
14	* Still to do:
15	*
16	* flush after percent set rather than just time based. (maybe both).
17	*/
18
19	#include <linux/blkdev.h>
20	#include <linux/module.h>
21	#include <linux/errno.h>
22	#include <linux/slab.h>
23	#include <linux/init.h>
24	#include <linux/timer.h>
25	#include <linux/sched.h>
26	#include <linux/list.h>
27	#include <linux/file.h>
28	#include <linux/mount.h>
29	#include <linux/buffer_head.h>
30	#include <linux/seq_file.h>
31	#include <trace/events/block.h>
32	#include "md.h"
33	#include "md-bitmap.h"
34
35	static inline char *bmname(struct bitmap *bitmap)
36	{
37	return bitmap->mddev ? mdname(mddev: bitmap->mddev) : "mdX";
38	}
39
40	/*
41	* check a page and, if necessary, allocate it (or hijack it if the alloc fails)
42	*
43	* 1) check to see if this page is allocated, if it's not then try to alloc
44	* 2) if the alloc fails, set the page's hijacked flag so we'll use the
45	* page pointer directly as a counter
46	*
47	* if we find our page, we increment the page's refcount so that it stays
48	* allocated while we're using it
49	*/
50	static int md_bitmap_checkpage(struct bitmap_counts *bitmap,
51	unsigned long page, int create, int no_hijack)
52	__releases(bitmap->lock)
53	__acquires(bitmap->lock)
54	{
55	unsigned char *mappage;
56
57	WARN_ON_ONCE(page >= bitmap->pages);
58	if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
59	return 0;
60
61	if (bitmap->bp[page].map) /* page is already allocated, just return */
62	return 0;
63
64	if (!create)
65	return -ENOENT;
66
67	/* this page has not been allocated yet */
68
69	spin_unlock_irq(lock: &bitmap->lock);
70	/* It is possible that this is being called inside a
71	* prepare_to_wait/finish_wait loop from raid5c:make_request().
72	* In general it is not permitted to sleep in that context as it
73	* can cause the loop to spin freely.
74	* That doesn't apply here as we can only reach this point
75	* once with any loop.
76	* When this function completes, either bp[page].map or
77	* bp[page].hijacked. In either case, this function will
78	* abort before getting to this point again. So there is
79	* no risk of a free-spin, and so it is safe to assert
80	* that sleeping here is allowed.
81	*/
82	sched_annotate_sleep();
83	mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
84	spin_lock_irq(lock: &bitmap->lock);
85
86	if (mappage == NULL) {
87	pr_debug("md/bitmap: map page allocation failed, hijacking\n");
88	/* We don't support hijack for cluster raid */
89	if (no_hijack)
90	return -ENOMEM;
91	/* failed - set the hijacked flag so that we can use the
92	* pointer as a counter */
93	if (!bitmap->bp[page].map)
94	bitmap->bp[page].hijacked = 1;
95	} else if (bitmap->bp[page].map ||
96	bitmap->bp[page].hijacked) {
97	/* somebody beat us to getting the page */
98	kfree(objp: mappage);
99	} else {
100
101	/* no page was in place and we have one, so install it */
102
103	bitmap->bp[page].map = mappage;
104	bitmap->missing_pages--;
105	}
106	return 0;
107	}
108
109	/* if page is completely empty, put it back on the free list, or dealloc it */
110	/* if page was hijacked, unmark the flag so it might get alloced next time */
111	/* Note: lock should be held when calling this */
112	static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
113	{
114	char *ptr;
115
116	if (bitmap->bp[page].count) /* page is still busy */
117	return;
118
119	/* page is no longer in use, it can be released */
120
121	if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
122	bitmap->bp[page].hijacked = 0;
123	bitmap->bp[page].map = NULL;
124	} else {
125	/* normal case, free the page */
126	ptr = bitmap->bp[page].map;
127	bitmap->bp[page].map = NULL;
128	bitmap->missing_pages++;
129	kfree(objp: ptr);
130	}
131	}
132
133	/*
134	* bitmap file handling - read and write the bitmap file and its superblock
135	*/
136
137	/*
138	* basic page I/O operations
139	*/
140
141	/* IO operations when bitmap is stored near all superblocks */
142
143	/* choose a good rdev and read the page from there */
144	static int read_sb_page(struct mddev *mddev, loff_t offset,
145	struct page *page, unsigned long index, int size)
146	{
147
148	sector_t sector = mddev->bitmap_info.offset + offset +
149	index * (PAGE_SIZE / SECTOR_SIZE);
150	struct md_rdev *rdev;
151
152	rdev_for_each(rdev, mddev) {
153	u32 iosize = roundup(size, bdev_logical_block_size(rdev->bdev));
154
155	if (!test_bit(In_sync, &rdev->flags) ||
156	test_bit(Faulty, &rdev->flags) ||
157	test_bit(Bitmap_sync, &rdev->flags))
158	continue;
159
160	if (sync_page_io(rdev, sector, size: iosize, page, opf: REQ_OP_READ, metadata_op: true))
161	return 0;
162	}
163	return -EIO;
164	}
165
166	static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
167	{
168	/* Iterate the disks of an mddev, using rcu to protect access to the
169	* linked list, and raising the refcount of devices we return to ensure
170	* they don't disappear while in use.
171	* As devices are only added or removed when raid_disk is < 0 and
172	* nr_pending is 0 and In_sync is clear, the entries we return will
173	* still be in the same position on the list when we re-enter
174	* list_for_each_entry_continue_rcu.
175	*
176	* Note that if entered with 'rdev == NULL' to start at the
177	* beginning, we temporarily assign 'rdev' to an address which
178	* isn't really an rdev, but which can be used by
179	* list_for_each_entry_continue_rcu() to find the first entry.
180	*/
181	rcu_read_lock();
182	if (rdev == NULL)
183	/* start at the beginning */
184	rdev = list_entry(&mddev->disks, struct md_rdev, same_set);
185	else {
186	/* release the previous rdev and start from there. */
187	rdev_dec_pending(rdev, mddev);
188	}
189	list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
190	if (rdev->raid_disk >= 0 &&
191	!test_bit(Faulty, &rdev->flags)) {
192	/* this is a usable devices */
193	atomic_inc(v: &rdev->nr_pending);
194	rcu_read_unlock();
195	return rdev;
196	}
197	}
198	rcu_read_unlock();
199	return NULL;
200	}
201
202	static unsigned int optimal_io_size(struct block_device *bdev,
203	unsigned int last_page_size,
204	unsigned int io_size)
205	{
206	if (bdev_io_opt(bdev) > bdev_logical_block_size(bdev))
207	return roundup(last_page_size, bdev_io_opt(bdev));
208	return io_size;
209	}
210
211	static unsigned int bitmap_io_size(unsigned int io_size, unsigned int opt_size,
212	loff_t start, loff_t boundary)
213	{
214	if (io_size != opt_size &&
215	start + opt_size / SECTOR_SIZE <= boundary)
216	return opt_size;
217	if (start + io_size / SECTOR_SIZE <= boundary)
218	return io_size;
219
220	/* Overflows boundary */
221	return 0;
222	}
223
224	static int __write_sb_page(struct md_rdev *rdev, struct bitmap *bitmap,
225	unsigned long pg_index, struct page *page)
226	{
227	struct block_device *bdev;
228	struct mddev *mddev = bitmap->mddev;
229	struct bitmap_storage *store = &bitmap->storage;
230	loff_t sboff, offset = mddev->bitmap_info.offset;
231	sector_t ps = pg_index * PAGE_SIZE / SECTOR_SIZE;
232	unsigned int size = PAGE_SIZE;
233	unsigned int opt_size = PAGE_SIZE;
234	sector_t doff;
235
236	bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
237	/* we compare length (page numbers), not page offset. */
238	if ((pg_index - store->sb_index) == store->file_pages - 1) {
239	unsigned int last_page_size = store->bytes & (PAGE_SIZE - 1);
240
241	if (last_page_size == 0)
242	last_page_size = PAGE_SIZE;
243	size = roundup(last_page_size, bdev_logical_block_size(bdev));
244	opt_size = optimal_io_size(bdev, last_page_size, io_size: size);
245	}
246
247	sboff = rdev->sb_start + offset;
248	doff = rdev->data_offset;
249
250	/* Just make sure we aren't corrupting data or metadata */
251	if (mddev->external) {
252	/* Bitmap could be anywhere. */
253	if (sboff + ps > doff &&
254	sboff < (doff + mddev->dev_sectors + PAGE_SIZE / SECTOR_SIZE))
255	return -EINVAL;
256	} else if (offset < 0) {
257	/* DATA BITMAP METADATA */
258	size = bitmap_io_size(io_size: size, opt_size, start: offset + ps, boundary: 0);
259	if (size == 0)
260	/* bitmap runs in to metadata */
261	return -EINVAL;
262
263	if (doff + mddev->dev_sectors > sboff)
264	/* data runs in to bitmap */
265	return -EINVAL;
266	} else if (rdev->sb_start < rdev->data_offset) {
267	/* METADATA BITMAP DATA */
268	size = bitmap_io_size(io_size: size, opt_size, start: sboff + ps, boundary: doff);
269	if (size == 0)
270	/* bitmap runs in to data */
271	return -EINVAL;
272	} else {
273	/* DATA METADATA BITMAP - no problems */
274	}
275
276	md_super_write(mddev, rdev, sector: sboff + ps, size: (int) size, page);
277	return 0;
278	}
279
280	static void write_sb_page(struct bitmap *bitmap, unsigned long pg_index,
281	struct page *page, bool wait)
282	{
283	struct mddev *mddev = bitmap->mddev;
284
285	do {
286	struct md_rdev *rdev = NULL;
287
288	while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
289	if (__write_sb_page(rdev, bitmap, pg_index, page) < 0) {
290	set_bit(nr: BITMAP_WRITE_ERROR, addr: &bitmap->flags);
291	return;
292	}
293	}
294	} while (wait && md_super_wait(mddev) < 0);
295	}
296
297	static void md_bitmap_file_kick(struct bitmap *bitmap);
298
299	#ifdef CONFIG_MD_BITMAP_FILE
300	static void write_file_page(struct bitmap *bitmap, struct page *page, int wait)
301	{
302	struct buffer_head *bh = page_buffers(page);
303
304	while (bh && bh->b_blocknr) {
305	atomic_inc(v: &bitmap->pending_writes);
306	set_buffer_locked(bh);
307	set_buffer_mapped(bh);
308	submit_bh(REQ_OP_WRITE | REQ_SYNC, bh);
309	bh = bh->b_this_page;
310	}
311
312	if (wait)
313	wait_event(bitmap->write_wait,
314	atomic_read(&bitmap->pending_writes) == 0);
315	}
316
317	static void end_bitmap_write(struct buffer_head *bh, int uptodate)
318	{
319	struct bitmap *bitmap = bh->b_private;
320
321	if (!uptodate)
322	set_bit(nr: BITMAP_WRITE_ERROR, addr: &bitmap->flags);
323	if (atomic_dec_and_test(v: &bitmap->pending_writes))
324	wake_up(&bitmap->write_wait);
325	}
326
327	static void free_buffers(struct page *page)
328	{
329	struct buffer_head *bh;
330
331	if (!PagePrivate(page))
332	return;
333
334	bh = page_buffers(page);
335	while (bh) {
336	struct buffer_head *next = bh->b_this_page;
337	free_buffer_head(bh);
338	bh = next;
339	}
340	detach_page_private(page);
341	put_page(page);
342	}
343
344	/* read a page from a file.
345	* We both read the page, and attach buffers to the page to record the
346	* address of each block (using bmap). These addresses will be used
347	* to write the block later, completely bypassing the filesystem.
348	* This usage is similar to how swap files are handled, and allows us
349	* to write to a file with no concerns of memory allocation failing.
350	*/
351	static int read_file_page(struct file *file, unsigned long index,
352	struct bitmap *bitmap, unsigned long count, struct page *page)
353	{
354	int ret = 0;
355	struct inode *inode = file_inode(f: file);
356	struct buffer_head *bh;
357	sector_t block, blk_cur;
358	unsigned long blocksize = i_blocksize(node: inode);
359
360	pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
361	(unsigned long long)index << PAGE_SHIFT);
362
363	bh = alloc_page_buffers(page, size: blocksize, retry: false);
364	if (!bh) {
365	ret = -ENOMEM;
366	goto out;
367	}
368	attach_page_private(page, data: bh);
369	blk_cur = index << (PAGE_SHIFT - inode->i_blkbits);
370	while (bh) {
371	block = blk_cur;
372
373	if (count == 0)
374	bh->b_blocknr = 0;
375	else {
376	ret = bmap(inode, block: &block);
377	if (ret || !block) {
378	ret = -EINVAL;
379	bh->b_blocknr = 0;
380	goto out;
381	}
382
383	bh->b_blocknr = block;
384	bh->b_bdev = inode->i_sb->s_bdev;
385	if (count < blocksize)
386	count = 0;
387	else
388	count -= blocksize;
389
390	bh->b_end_io = end_bitmap_write;
391	bh->b_private = bitmap;
392	atomic_inc(v: &bitmap->pending_writes);
393	set_buffer_locked(bh);
394	set_buffer_mapped(bh);
395	submit_bh(REQ_OP_READ, bh);
396	}
397	blk_cur++;
398	bh = bh->b_this_page;
399	}
400
401	wait_event(bitmap->write_wait,
402	atomic_read(&bitmap->pending_writes)==0);
403	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
404	ret = -EIO;
405	out:
406	if (ret)
407	pr_err("md: bitmap read error: (%dB @ %llu): %d\n",
408	(int)PAGE_SIZE,
409	(unsigned long long)index << PAGE_SHIFT,
410	ret);
411	return ret;
412	}
413	#else /* CONFIG_MD_BITMAP_FILE */
414	static void write_file_page(struct bitmap *bitmap, struct page *page, int wait)
415	{
416	}
417	static int read_file_page(struct file *file, unsigned long index,
418	struct bitmap *bitmap, unsigned long count, struct page *page)
419	{
420	return -EIO;
421	}
422	static void free_buffers(struct page *page)
423	{
424	put_page(page);
425	}
426	#endif /* CONFIG_MD_BITMAP_FILE */
427
428	/*
429	* bitmap file superblock operations
430	*/
431
432	/*
433	* write out a page to a file
434	*/
435	static void filemap_write_page(struct bitmap *bitmap, unsigned long pg_index,
436	bool wait)
437	{
438	struct bitmap_storage *store = &bitmap->storage;
439	struct page *page = store->filemap[pg_index];
440
441	if (mddev_is_clustered(mddev: bitmap->mddev)) {
442	/* go to node bitmap area starting point */
443	pg_index += store->sb_index;
444	}
445
446	if (store->file)
447	write_file_page(bitmap, page, wait);
448	else
449	write_sb_page(bitmap, pg_index, page, wait);
450	}
451
452	/*
453	* md_bitmap_wait_writes() should be called before writing any bitmap
454	* blocks, to ensure previous writes, particularly from
455	* md_bitmap_daemon_work(), have completed.
456	*/
457	static void md_bitmap_wait_writes(struct bitmap *bitmap)
458	{
459	if (bitmap->storage.file)
460	wait_event(bitmap->write_wait,
461	atomic_read(&bitmap->pending_writes)==0);
462	else
463	/* Note that we ignore the return value. The writes
464	* might have failed, but that would just mean that
465	* some bits which should be cleared haven't been,
466	* which is safe. The relevant bitmap blocks will
467	* probably get written again, but there is no great
468	* loss if they aren't.
469	*/
470	md_super_wait(mddev: bitmap->mddev);
471	}
472
473
474	/* update the event counter and sync the superblock to disk */
475	void md_bitmap_update_sb(struct bitmap *bitmap)
476	{
477	bitmap_super_t *sb;
478
479	if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
480	return;
481	if (bitmap->mddev->bitmap_info.external)
482	return;
483	if (!bitmap->storage.sb_page) /* no superblock */
484	return;
485	sb = kmap_atomic(page: bitmap->storage.sb_page);
486	sb->events = cpu_to_le64(bitmap->mddev->events);
487	if (bitmap->mddev->events < bitmap->events_cleared)
488	/* rocking back to read-only */
489	bitmap->events_cleared = bitmap->mddev->events;
490	sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
491	/*
492	* clear BITMAP_WRITE_ERROR bit to protect against the case that
493	* a bitmap write error occurred but the later writes succeeded.
494	*/
495	sb->state = cpu_to_le32(bitmap->flags & ~BIT(BITMAP_WRITE_ERROR));
496	/* Just in case these have been changed via sysfs: */
497	sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
498	sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
499	/* This might have been changed by a reshape */
500	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
501	sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
502	sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
503	sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
504	bitmap_info.space);
505	kunmap_atomic(sb);
506
507	if (bitmap->storage.file)
508	write_file_page(bitmap, page: bitmap->storage.sb_page, wait: 1);
509	else
510	write_sb_page(bitmap, pg_index: bitmap->storage.sb_index,
511	page: bitmap->storage.sb_page, wait: 1);
512	}
513	EXPORT_SYMBOL(md_bitmap_update_sb);
514
515	/* print out the bitmap file superblock */
516	void md_bitmap_print_sb(struct bitmap *bitmap)
517	{
518	bitmap_super_t *sb;
519
520	if (!bitmap || !bitmap->storage.sb_page)
521	return;
522	sb = kmap_atomic(page: bitmap->storage.sb_page);
523	pr_debug("%s: bitmap file superblock:\n", bmname(bitmap));
524	pr_debug(" magic: %08x\n", le32_to_cpu(sb->magic));
525	pr_debug(" version: %u\n", le32_to_cpu(sb->version));
526	pr_debug(" uuid: %08x.%08x.%08x.%08x\n",
527	le32_to_cpu(*(__le32 *)(sb->uuid+0)),
528	le32_to_cpu(*(__le32 *)(sb->uuid+4)),
529	le32_to_cpu(*(__le32 *)(sb->uuid+8)),
530	le32_to_cpu(*(__le32 *)(sb->uuid+12)));
531	pr_debug(" events: %llu\n",
532	(unsigned long long) le64_to_cpu(sb->events));
533	pr_debug("events cleared: %llu\n",
534	(unsigned long long) le64_to_cpu(sb->events_cleared));
535	pr_debug(" state: %08x\n", le32_to_cpu(sb->state));
536	pr_debug(" chunksize: %u B\n", le32_to_cpu(sb->chunksize));
537	pr_debug(" daemon sleep: %us\n", le32_to_cpu(sb->daemon_sleep));
538	pr_debug(" sync size: %llu KB\n",
539	(unsigned long long)le64_to_cpu(sb->sync_size)/2);
540	pr_debug("max write behind: %u\n", le32_to_cpu(sb->write_behind));
541	kunmap_atomic(sb);
542	}
543
544	/*
545	* bitmap_new_disk_sb
546	* @bitmap
547	*
548	* This function is somewhat the reverse of bitmap_read_sb. bitmap_read_sb
549	* reads and verifies the on-disk bitmap superblock and populates bitmap_info.
550	* This function verifies 'bitmap_info' and populates the on-disk bitmap
551	* structure, which is to be written to disk.
552	*
553	* Returns: 0 on success, -Exxx on error
554	*/
555	static int md_bitmap_new_disk_sb(struct bitmap *bitmap)
556	{
557	bitmap_super_t *sb;
558	unsigned long chunksize, daemon_sleep, write_behind;
559
560	bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
561	if (bitmap->storage.sb_page == NULL)
562	return -ENOMEM;
563	bitmap->storage.sb_index = 0;
564
565	sb = kmap_atomic(page: bitmap->storage.sb_page);
566
567	sb->magic = cpu_to_le32(BITMAP_MAGIC);
568	sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
569
570	chunksize = bitmap->mddev->bitmap_info.chunksize;
571	BUG_ON(!chunksize);
572	if (!is_power_of_2(n: chunksize)) {
573	kunmap_atomic(sb);
574	pr_warn("bitmap chunksize not a power of 2\n");
575	return -EINVAL;
576	}
577	sb->chunksize = cpu_to_le32(chunksize);
578
579	daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
580	if (!daemon_sleep || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
581	pr_debug("Choosing daemon_sleep default (5 sec)\n");
582	daemon_sleep = 5 * HZ;
583	}
584	sb->daemon_sleep = cpu_to_le32(daemon_sleep);
585	bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
586
587	/*
588	* FIXME: write_behind for RAID1. If not specified, what
589	* is a good choice? We choose COUNTER_MAX / 2 arbitrarily.
590	*/
591	write_behind = bitmap->mddev->bitmap_info.max_write_behind;
592	if (write_behind > COUNTER_MAX)
593	write_behind = COUNTER_MAX / 2;
594	sb->write_behind = cpu_to_le32(write_behind);
595	bitmap->mddev->bitmap_info.max_write_behind = write_behind;
596
597	/* keep the array size field of the bitmap superblock up to date */
598	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
599
600	memcpy(sb->uuid, bitmap->mddev->uuid, 16);
601
602	set_bit(nr: BITMAP_STALE, addr: &bitmap->flags);
603	sb->state = cpu_to_le32(bitmap->flags);
604	bitmap->events_cleared = bitmap->mddev->events;
605	sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
606	bitmap->mddev->bitmap_info.nodes = 0;
607
608	kunmap_atomic(sb);
609
610	return 0;
611	}
612
613	/* read the superblock from the bitmap file and initialize some bitmap fields */
614	static int md_bitmap_read_sb(struct bitmap *bitmap)
615	{
616	char *reason = NULL;
617	bitmap_super_t *sb;
618	unsigned long chunksize, daemon_sleep, write_behind;
619	unsigned long long events;
620	int nodes = 0;
621	unsigned long sectors_reserved = 0;
622	int err = -EINVAL;
623	struct page *sb_page;
624	loff_t offset = 0;
625
626	if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
627	chunksize = 128 * 1024 * 1024;
628	daemon_sleep = 5 * HZ;
629	write_behind = 0;
630	set_bit(nr: BITMAP_STALE, addr: &bitmap->flags);
631	err = 0;
632	goto out_no_sb;
633	}
634	/* page 0 is the superblock, read it... */
635	sb_page = alloc_page(GFP_KERNEL);
636	if (!sb_page)
637	return -ENOMEM;
638	bitmap->storage.sb_page = sb_page;
639
640	re_read:
641	/* If cluster_slot is set, the cluster is setup */
642	if (bitmap->cluster_slot >= 0) {
643	sector_t bm_blocks = bitmap->mddev->resync_max_sectors;
644
645	bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks,
646	(bitmap->mddev->bitmap_info.chunksize >> 9));
647	/* bits to bytes */
648	bm_blocks = ((bm_blocks+7) >> 3) + sizeof(bitmap_super_t);
649	/* to 4k blocks */
650	bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks, 4096);
651	offset = bitmap->cluster_slot * (bm_blocks << 3);
652	pr_debug("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
653	bitmap->cluster_slot, offset);
654	}
655
656	if (bitmap->storage.file) {
657	loff_t isize = i_size_read(inode: bitmap->storage.file->f_mapping->host);
658	int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
659
660	err = read_file_page(file: bitmap->storage.file, index: 0,
661	bitmap, count: bytes, page: sb_page);
662	} else {
663	err = read_sb_page(mddev: bitmap->mddev, offset, page: sb_page, index: 0,
664	size: sizeof(bitmap_super_t));
665	}
666	if (err)
667	return err;
668
669	err = -EINVAL;
670	sb = kmap_atomic(page: sb_page);
671
672	chunksize = le32_to_cpu(sb->chunksize);
673	daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
674	write_behind = le32_to_cpu(sb->write_behind);
675	sectors_reserved = le32_to_cpu(sb->sectors_reserved);
676
677	/* verify that the bitmap-specific fields are valid */
678	if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
679	reason = "bad magic";
680	else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
681	le32_to_cpu(sb->version) > BITMAP_MAJOR_CLUSTERED)
682	reason = "unrecognized superblock version";
683	else if (chunksize < 512)
684	reason = "bitmap chunksize too small";
685	else if (!is_power_of_2(n: chunksize))
686	reason = "bitmap chunksize not a power of 2";
687	else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
688	reason = "daemon sleep period out of range";
689	else if (write_behind > COUNTER_MAX)
690	reason = "write-behind limit out of range (0 - 16383)";
691	if (reason) {
692	pr_warn("%s: invalid bitmap file superblock: %s\n",
693	bmname(bitmap), reason);
694	goto out;
695	}
696
697	/*
698	* Setup nodes/clustername only if bitmap version is
699	* cluster-compatible
700	*/
701	if (sb->version == cpu_to_le32(BITMAP_MAJOR_CLUSTERED)) {
702	nodes = le32_to_cpu(sb->nodes);
703	strscpy(bitmap->mddev->bitmap_info.cluster_name,
704	sb->cluster_name, 64);
705	}
706
707	/* keep the array size field of the bitmap superblock up to date */
708	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
709
710	if (bitmap->mddev->persistent) {
711	/*
712	* We have a persistent array superblock, so compare the
713	* bitmap's UUID and event counter to the mddev's
714	*/
715	if (memcmp(p: sb->uuid, q: bitmap->mddev->uuid, size: 16)) {
716	pr_warn("%s: bitmap superblock UUID mismatch\n",
717	bmname(bitmap));
718	goto out;
719	}
720	events = le64_to_cpu(sb->events);
721	if (!nodes && (events < bitmap->mddev->events)) {
722	pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n",
723	bmname(bitmap), events,
724	(unsigned long long) bitmap->mddev->events);
725	set_bit(nr: BITMAP_STALE, addr: &bitmap->flags);
726	}
727	}
728
729	/* assign fields using values from superblock */
730	bitmap->flags |= le32_to_cpu(sb->state);
731	if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
732	set_bit(nr: BITMAP_HOSTENDIAN, addr: &bitmap->flags);
733	bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
734	err = 0;
735
736	out:
737	kunmap_atomic(sb);
738	if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
739	/* Assigning chunksize is required for "re_read" */
740	bitmap->mddev->bitmap_info.chunksize = chunksize;
741	err = md_setup_cluster(mddev: bitmap->mddev, nodes);
742	if (err) {
743	pr_warn("%s: Could not setup cluster service (%d)\n",
744	bmname(bitmap), err);
745	goto out_no_sb;
746	}
747	bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
748	goto re_read;
749	}
750
751	out_no_sb:
752	if (err == 0) {
753	if (test_bit(BITMAP_STALE, &bitmap->flags))
754	bitmap->events_cleared = bitmap->mddev->events;
755	bitmap->mddev->bitmap_info.chunksize = chunksize;
756	bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
757	bitmap->mddev->bitmap_info.max_write_behind = write_behind;
758	bitmap->mddev->bitmap_info.nodes = nodes;
759	if (bitmap->mddev->bitmap_info.space == 0 ||
760	bitmap->mddev->bitmap_info.space > sectors_reserved)
761	bitmap->mddev->bitmap_info.space = sectors_reserved;
762	} else {
763	md_bitmap_print_sb(bitmap);
764	if (bitmap->cluster_slot < 0)
765	md_cluster_stop(mddev: bitmap->mddev);
766	}
767	return err;
768	}
769
770	/*
771	* general bitmap file operations
772	*/
773
774	/*
775	* on-disk bitmap:
776	*
777	* Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
778	* file a page at a time. There's a superblock at the start of the file.
779	*/
780	/* calculate the index of the page that contains this bit */
781	static inline unsigned long file_page_index(struct bitmap_storage *store,
782	unsigned long chunk)
783	{
784	if (store->sb_page)
785	chunk += sizeof(bitmap_super_t) << 3;
786	return chunk >> PAGE_BIT_SHIFT;
787	}
788
789	/* calculate the (bit) offset of this bit within a page */
790	static inline unsigned long file_page_offset(struct bitmap_storage *store,
791	unsigned long chunk)
792	{
793	if (store->sb_page)
794	chunk += sizeof(bitmap_super_t) << 3;
795	return chunk & (PAGE_BITS - 1);
796	}
797
798	/*
799	* return a pointer to the page in the filemap that contains the given bit
800	*
801	*/
802	static inline struct page *filemap_get_page(struct bitmap_storage *store,
803	unsigned long chunk)
804	{
805	if (file_page_index(store, chunk) >= store->file_pages)
806	return NULL;
807	return store->filemap[file_page_index(store, chunk)];
808	}
809
810	static int md_bitmap_storage_alloc(struct bitmap_storage *store,
811	unsigned long chunks, int with_super,
812	int slot_number)
813	{
814	int pnum, offset = 0;
815	unsigned long num_pages;
816	unsigned long bytes;
817
818	bytes = DIV_ROUND_UP(chunks, 8);
819	if (with_super)
820	bytes += sizeof(bitmap_super_t);
821
822	num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
823	offset = slot_number * num_pages;
824
825	store->filemap = kmalloc_array(n: num_pages, size: sizeof(struct page *),
826	GFP_KERNEL);
827	if (!store->filemap)
828	return -ENOMEM;
829
830	if (with_super && !store->sb_page) {
831	store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
832	if (store->sb_page == NULL)
833	return -ENOMEM;
834	}
835
836	pnum = 0;
837	if (store->sb_page) {
838	store->filemap[0] = store->sb_page;
839	pnum = 1;
840	store->sb_index = offset;
841	}
842
843	for ( ; pnum < num_pages; pnum++) {
844	store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
845	if (!store->filemap[pnum]) {
846	store->file_pages = pnum;
847	return -ENOMEM;
848	}
849	}
850	store->file_pages = pnum;
851
852	/* We need 4 bits per page, rounded up to a multiple
853	* of sizeof(unsigned long) */
854	store->filemap_attr = kzalloc(
855	roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
856	GFP_KERNEL);
857	if (!store->filemap_attr)
858	return -ENOMEM;
859
860	store->bytes = bytes;
861
862	return 0;
863	}
864
865	static void md_bitmap_file_unmap(struct bitmap_storage *store)
866	{
867	struct file *file = store->file;
868	struct page *sb_page = store->sb_page;
869	struct page **map = store->filemap;
870	int pages = store->file_pages;
871
872	while (pages--)
873	if (map[pages] != sb_page) /* 0 is sb_page, release it below */
874	free_buffers(page: map[pages]);
875	kfree(objp: map);
876	kfree(objp: store->filemap_attr);
877
878	if (sb_page)
879	free_buffers(page: sb_page);
880
881	if (file) {
882	struct inode *inode = file_inode(f: file);
883	invalidate_mapping_pages(mapping: inode->i_mapping, start: 0, end: -1);
884	fput(file);
885	}
886	}
887
888	/*
889	* bitmap_file_kick - if an error occurs while manipulating the bitmap file
890	* then it is no longer reliable, so we stop using it and we mark the file
891	* as failed in the superblock
892	*/
893	static void md_bitmap_file_kick(struct bitmap *bitmap)
894	{
895	if (!test_and_set_bit(nr: BITMAP_STALE, addr: &bitmap->flags)) {
896	md_bitmap_update_sb(bitmap);
897
898	if (bitmap->storage.file) {
899	pr_warn("%s: kicking failed bitmap file %pD4 from array!\n",
900	bmname(bitmap), bitmap->storage.file);
901
902	} else
903	pr_warn("%s: disabling internal bitmap due to errors\n",
904	bmname(bitmap));
905	}
906	}
907
908	enum bitmap_page_attr {
909	BITMAP_PAGE_DIRTY = 0, /* there are set bits that need to be synced */
910	BITMAP_PAGE_PENDING = 1, /* there are bits that are being cleaned.
911	* i.e. counter is 1 or 2. */
912	BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
913	};
914
915	static inline void set_page_attr(struct bitmap *bitmap, int pnum,
916	enum bitmap_page_attr attr)
917	{
918	set_bit(nr: (pnum<<2) + attr, addr: bitmap->storage.filemap_attr);
919	}
920
921	static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
922	enum bitmap_page_attr attr)
923	{
924	clear_bit(nr: (pnum<<2) + attr, addr: bitmap->storage.filemap_attr);
925	}
926
927	static inline int test_page_attr(struct bitmap *bitmap, int pnum,
928	enum bitmap_page_attr attr)
929	{
930	return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
931	}
932
933	static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
934	enum bitmap_page_attr attr)
935	{
936	return test_and_clear_bit(nr: (pnum<<2) + attr,
937	addr: bitmap->storage.filemap_attr);
938	}
939	/*
940	* bitmap_file_set_bit -- called before performing a write to the md device
941	* to set (and eventually sync) a particular bit in the bitmap file
942	*
943	* we set the bit immediately, then we record the page number so that
944	* when an unplug occurs, we can flush the dirty pages out to disk
945	*/
946	static void md_bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
947	{
948	unsigned long bit;
949	struct page *page;
950	void *kaddr;
951	unsigned long chunk = block >> bitmap->counts.chunkshift;
952	struct bitmap_storage *store = &bitmap->storage;
953	unsigned long index = file_page_index(store, chunk);
954	unsigned long node_offset = 0;
955
956	index += store->sb_index;
957	if (mddev_is_clustered(mddev: bitmap->mddev))
958	node_offset = bitmap->cluster_slot * store->file_pages;
959
960	page = filemap_get_page(store: &bitmap->storage, chunk);
961	if (!page)
962	return;
963	bit = file_page_offset(store: &bitmap->storage, chunk);
964
965	/* set the bit */
966	kaddr = kmap_atomic(page);
967	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
968	set_bit(nr: bit, addr: kaddr);
969	else
970	set_bit_le(nr: bit, addr: kaddr);
971	kunmap_atomic(kaddr);
972	pr_debug("set file bit %lu page %lu\n", bit, index);
973	/* record page number so it gets flushed to disk when unplug occurs */
974	set_page_attr(bitmap, pnum: index - node_offset, attr: BITMAP_PAGE_DIRTY);
975	}
976
977	static void md_bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
978	{
979	unsigned long bit;
980	struct page *page;
981	void *paddr;
982	unsigned long chunk = block >> bitmap->counts.chunkshift;
983	struct bitmap_storage *store = &bitmap->storage;
984	unsigned long index = file_page_index(store, chunk);
985	unsigned long node_offset = 0;
986
987	index += store->sb_index;
988	if (mddev_is_clustered(mddev: bitmap->mddev))
989	node_offset = bitmap->cluster_slot * store->file_pages;
990
991	page = filemap_get_page(store: &bitmap->storage, chunk);
992	if (!page)
993	return;
994	bit = file_page_offset(store: &bitmap->storage, chunk);
995	paddr = kmap_atomic(page);
996	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
997	clear_bit(nr: bit, addr: paddr);
998	else
999	clear_bit_le(nr: bit, addr: paddr);
1000	kunmap_atomic(paddr);
1001	if (!test_page_attr(bitmap, pnum: index - node_offset, attr: BITMAP_PAGE_NEEDWRITE)) {
1002	set_page_attr(bitmap, pnum: index - node_offset, attr: BITMAP_PAGE_PENDING);
1003	bitmap->allclean = 0;
1004	}
1005	}
1006
1007	static int md_bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
1008	{
1009	unsigned long bit;
1010	struct page *page;
1011	void *paddr;
1012	unsigned long chunk = block >> bitmap->counts.chunkshift;
1013	int set = 0;
1014
1015	page = filemap_get_page(store: &bitmap->storage, chunk);
1016	if (!page)
1017	return -EINVAL;
1018	bit = file_page_offset(store: &bitmap->storage, chunk);
1019	paddr = kmap_atomic(page);
1020	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1021	set = test_bit(bit, paddr);
1022	else
1023	set = test_bit_le(nr: bit, addr: paddr);
1024	kunmap_atomic(paddr);
1025	return set;
1026	}
1027
1028	/* this gets called when the md device is ready to unplug its underlying
1029	* (slave) device queues -- before we let any writes go down, we need to
1030	* sync the dirty pages of the bitmap file to disk */
1031	void md_bitmap_unplug(struct bitmap *bitmap)
1032	{
1033	unsigned long i;
1034	int dirty, need_write;
1035	int writing = 0;
1036
1037	if (!md_bitmap_enabled(bitmap))
1038	return;
1039
1040	/* look at each page to see if there are any set bits that need to be
1041	* flushed out to disk */
1042	for (i = 0; i < bitmap->storage.file_pages; i++) {
1043	dirty = test_and_clear_page_attr(bitmap, pnum: i, attr: BITMAP_PAGE_DIRTY);
1044	need_write = test_and_clear_page_attr(bitmap, pnum: i,
1045	attr: BITMAP_PAGE_NEEDWRITE);
1046	if (dirty || need_write) {
1047	if (!writing) {
1048	md_bitmap_wait_writes(bitmap);
1049	mddev_add_trace_msg(bitmap->mddev,
1050	"md bitmap_unplug");
1051	}
1052	clear_page_attr(bitmap, pnum: i, attr: BITMAP_PAGE_PENDING);
1053	filemap_write_page(bitmap, pg_index: i, wait: false);
1054	writing = 1;
1055	}
1056	}
1057	if (writing)
1058	md_bitmap_wait_writes(bitmap);
1059
1060	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1061	md_bitmap_file_kick(bitmap);
1062	}
1063	EXPORT_SYMBOL(md_bitmap_unplug);
1064
1065	struct bitmap_unplug_work {
1066	struct work_struct work;
1067	struct bitmap *bitmap;
1068	struct completion *done;
1069	};
1070
1071	static void md_bitmap_unplug_fn(struct work_struct *work)
1072	{
1073	struct bitmap_unplug_work *unplug_work =
1074	container_of(work, struct bitmap_unplug_work, work);
1075
1076	md_bitmap_unplug(unplug_work->bitmap);
1077	complete(unplug_work->done);
1078	}
1079
1080	void md_bitmap_unplug_async(struct bitmap *bitmap)
1081	{
1082	DECLARE_COMPLETION_ONSTACK(done);
1083	struct bitmap_unplug_work unplug_work;
1084
1085	INIT_WORK_ONSTACK(&unplug_work.work, md_bitmap_unplug_fn);
1086	unplug_work.bitmap = bitmap;
1087	unplug_work.done = &done;
1088
1089	queue_work(wq: md_bitmap_wq, work: &unplug_work.work);
1090	wait_for_completion(&done);
1091	}
1092	EXPORT_SYMBOL(md_bitmap_unplug_async);
1093
1094	static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1095
1096	/*
1097	* Initialize the in-memory bitmap from the on-disk bitmap and set up the memory
1098	* mapping of the bitmap file.
1099	*
1100	* Special case: If there's no bitmap file, or if the bitmap file had been
1101	* previously kicked from the array, we mark all the bits as 1's in order to
1102	* cause a full resync.
1103	*
1104	* We ignore all bits for sectors that end earlier than 'start'.
1105	* This is used when reading an out-of-date bitmap.
1106	*/
1107	static int md_bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1108	{
1109	bool outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
1110	struct mddev *mddev = bitmap->mddev;
1111	unsigned long chunks = bitmap->counts.chunks;
1112	struct bitmap_storage *store = &bitmap->storage;
1113	struct file *file = store->file;
1114	unsigned long node_offset = 0;
1115	unsigned long bit_cnt = 0;
1116	unsigned long i;
1117	int ret;
1118
1119	if (!file && !mddev->bitmap_info.offset) {
1120	/* No permanent bitmap - fill with '1s'. */
1121	store->filemap = NULL;
1122	store->file_pages = 0;
1123	for (i = 0; i < chunks ; i++) {
1124	/* if the disk bit is set, set the memory bit */
1125	int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
1126	>= start);
1127	md_bitmap_set_memory_bits(bitmap,
1128	offset: (sector_t)i << bitmap->counts.chunkshift,
1129	needed);
1130	}
1131	return 0;
1132	}
1133
1134	if (file && i_size_read(inode: file->f_mapping->host) < store->bytes) {
1135	pr_warn("%s: bitmap file too short %lu < %lu\n",
1136	bmname(bitmap),
1137	(unsigned long) i_size_read(file->f_mapping->host),
1138	store->bytes);
1139	ret = -ENOSPC;
1140	goto err;
1141	}
1142
1143	if (mddev_is_clustered(mddev))
1144	node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));
1145
1146	for (i = 0; i < store->file_pages; i++) {
1147	struct page *page = store->filemap[i];
1148	int count;
1149
1150	/* unmap the old page, we're done with it */
1151	if (i == store->file_pages - 1)
1152	count = store->bytes - i * PAGE_SIZE;
1153	else
1154	count = PAGE_SIZE;
1155
1156	if (file)
1157	ret = read_file_page(file, index: i, bitmap, count, page);
1158	else
1159	ret = read_sb_page(mddev, offset: 0, page, index: i + node_offset,
1160	size: count);
1161	if (ret)
1162	goto err;
1163	}
1164
1165	if (outofdate) {
1166	pr_warn("%s: bitmap file is out of date, doing full recovery\n",
1167	bmname(bitmap));
1168
1169	for (i = 0; i < store->file_pages; i++) {
1170	struct page *page = store->filemap[i];
1171	unsigned long offset = 0;
1172	void *paddr;
1173
1174	if (i == 0 && !mddev->bitmap_info.external)
1175	offset = sizeof(bitmap_super_t);
1176
1177	/*
1178	* If the bitmap is out of date, dirty the whole page
1179	* and write it out
1180	*/
1181	paddr = kmap_atomic(page);
1182	memset(paddr + offset, 0xff, PAGE_SIZE - offset);
1183	kunmap_atomic(paddr);
1184
1185	filemap_write_page(bitmap, pg_index: i, wait: true);
1186	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) {
1187	ret = -EIO;
1188	goto err;
1189	}
1190	}
1191	}
1192
1193	for (i = 0; i < chunks; i++) {
1194	struct page *page = filemap_get_page(store: &bitmap->storage, chunk: i);
1195	unsigned long bit = file_page_offset(store: &bitmap->storage, chunk: i);
1196	void *paddr;
1197	bool was_set;
1198
1199	paddr = kmap_atomic(page);
1200	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1201	was_set = test_bit(bit, paddr);
1202	else
1203	was_set = test_bit_le(nr: bit, addr: paddr);
1204	kunmap_atomic(paddr);
1205
1206	if (was_set) {
1207	/* if the disk bit is set, set the memory bit */
1208	int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1209	>= start);
1210	md_bitmap_set_memory_bits(bitmap,
1211	offset: (sector_t)i << bitmap->counts.chunkshift,
1212	needed);
1213	bit_cnt++;
1214	}
1215	}
1216
1217	pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n",
1218	bmname(bitmap), store->file_pages,
1219	bit_cnt, chunks);
1220
1221	return 0;
1222
1223	err:
1224	pr_warn("%s: bitmap initialisation failed: %d\n",
1225	bmname(bitmap), ret);
1226	return ret;
1227	}
1228
1229	void md_bitmap_write_all(struct bitmap *bitmap)
1230	{
1231	/* We don't actually write all bitmap blocks here,
1232	* just flag them as needing to be written
1233	*/
1234	int i;
1235
1236	if (!bitmap || !bitmap->storage.filemap)
1237	return;
1238	if (bitmap->storage.file)
1239	/* Only one copy, so nothing needed */
1240	return;
1241
1242	for (i = 0; i < bitmap->storage.file_pages; i++)
1243	set_page_attr(bitmap, pnum: i,
1244	attr: BITMAP_PAGE_NEEDWRITE);
1245	bitmap->allclean = 0;
1246	}
1247
1248	static void md_bitmap_count_page(struct bitmap_counts *bitmap,
1249	sector_t offset, int inc)
1250	{
1251	sector_t chunk = offset >> bitmap->chunkshift;
1252	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1253	bitmap->bp[page].count += inc;
1254	md_bitmap_checkfree(bitmap, page);
1255	}
1256
1257	static void md_bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1258	{
1259	sector_t chunk = offset >> bitmap->chunkshift;
1260	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1261	struct bitmap_page *bp = &bitmap->bp[page];
1262
1263	if (!bp->pending)
1264	bp->pending = 1;
1265	}
1266
1267	static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1268	sector_t offset, sector_t *blocks,
1269	int create);
1270
1271	static void mddev_set_timeout(struct mddev *mddev, unsigned long timeout,
1272	bool force)
1273	{
1274	struct md_thread *thread;
1275
1276	rcu_read_lock();
1277	thread = rcu_dereference(mddev->thread);
1278
1279	if (!thread)
1280	goto out;
1281
1282	if (force || thread->timeout < MAX_SCHEDULE_TIMEOUT)
1283	thread->timeout = timeout;
1284
1285	out:
1286	rcu_read_unlock();
1287	}
1288
1289	/*
1290	* bitmap daemon -- periodically wakes up to clean bits and flush pages
1291	* out to disk
1292	*/
1293	void md_bitmap_daemon_work(struct mddev *mddev)
1294	{
1295	struct bitmap *bitmap;
1296	unsigned long j;
1297	unsigned long nextpage;
1298	sector_t blocks;
1299	struct bitmap_counts *counts;
1300
1301	/* Use a mutex to guard daemon_work against
1302	* bitmap_destroy.
1303	*/
1304	mutex_lock(&mddev->bitmap_info.mutex);
1305	bitmap = mddev->bitmap;
1306	if (bitmap == NULL) {
1307	mutex_unlock(lock: &mddev->bitmap_info.mutex);
1308	return;
1309	}
1310	if (time_before(jiffies, bitmap->daemon_lastrun
1311	+ mddev->bitmap_info.daemon_sleep))
1312	goto done;
1313
1314	bitmap->daemon_lastrun = jiffies;
1315	if (bitmap->allclean) {
1316	mddev_set_timeout(mddev, MAX_SCHEDULE_TIMEOUT, force: true);
1317	goto done;
1318	}
1319	bitmap->allclean = 1;
1320
1321	mddev_add_trace_msg(bitmap->mddev, "md bitmap_daemon_work");
1322
1323	/* Any file-page which is PENDING now needs to be written.
1324	* So set NEEDWRITE now, then after we make any last-minute changes
1325	* we will write it.
1326	*/
1327	for (j = 0; j < bitmap->storage.file_pages; j++)
1328	if (test_and_clear_page_attr(bitmap, pnum: j,
1329	attr: BITMAP_PAGE_PENDING))
1330	set_page_attr(bitmap, pnum: j,
1331	attr: BITMAP_PAGE_NEEDWRITE);
1332
1333	if (bitmap->need_sync &&
1334	mddev->bitmap_info.external == 0) {
1335	/* Arrange for superblock update as well as
1336	* other changes */
1337	bitmap_super_t *sb;
1338	bitmap->need_sync = 0;
1339	if (bitmap->storage.filemap) {
1340	sb = kmap_atomic(page: bitmap->storage.sb_page);
1341	sb->events_cleared =
1342	cpu_to_le64(bitmap->events_cleared);
1343	kunmap_atomic(sb);
1344	set_page_attr(bitmap, pnum: 0,
1345	attr: BITMAP_PAGE_NEEDWRITE);
1346	}
1347	}
1348	/* Now look at the bitmap counters and if any are '2' or '1',
1349	* decrement and handle accordingly.
1350	*/
1351	counts = &bitmap->counts;
1352	spin_lock_irq(lock: &counts->lock);
1353	nextpage = 0;
1354	for (j = 0; j < counts->chunks; j++) {
1355	bitmap_counter_t *bmc;
1356	sector_t block = (sector_t)j << counts->chunkshift;
1357
1358	if (j == nextpage) {
1359	nextpage += PAGE_COUNTER_RATIO;
1360	if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
1361	j |= PAGE_COUNTER_MASK;
1362	continue;
1363	}
1364	counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1365	}
1366
1367	bmc = md_bitmap_get_counter(bitmap: counts, offset: block, blocks: &blocks, create: 0);
1368	if (!bmc) {
1369	j |= PAGE_COUNTER_MASK;
1370	continue;
1371	}
1372	if (*bmc == 1 && !bitmap->need_sync) {
1373	/* We can clear the bit */
1374	*bmc = 0;
1375	md_bitmap_count_page(bitmap: counts, offset: block, inc: -1);
1376	md_bitmap_file_clear_bit(bitmap, block);
1377	} else if (*bmc && *bmc <= 2) {
1378	*bmc = 1;
1379	md_bitmap_set_pending(bitmap: counts, offset: block);
1380	bitmap->allclean = 0;
1381	}
1382	}
1383	spin_unlock_irq(lock: &counts->lock);
1384
1385	md_bitmap_wait_writes(bitmap);
1386	/* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
1387	* DIRTY pages need to be written by bitmap_unplug so it can wait
1388	* for them.
1389	* If we find any DIRTY page we stop there and let bitmap_unplug
1390	* handle all the rest. This is important in the case where
1391	* the first blocking holds the superblock and it has been updated.
1392	* We mustn't write any other blocks before the superblock.
1393	*/
1394	for (j = 0;
1395	j < bitmap->storage.file_pages
1396	&& !test_bit(BITMAP_STALE, &bitmap->flags);
1397	j++) {
1398	if (test_page_attr(bitmap, pnum: j,
1399	attr: BITMAP_PAGE_DIRTY))
1400	/* bitmap_unplug will handle the rest */
1401	break;
1402	if (bitmap->storage.filemap &&
1403	test_and_clear_page_attr(bitmap, pnum: j,
1404	attr: BITMAP_PAGE_NEEDWRITE))
1405	filemap_write_page(bitmap, pg_index: j, wait: false);
1406	}
1407
1408	done:
1409	if (bitmap->allclean == 0)
1410	mddev_set_timeout(mddev, timeout: mddev->bitmap_info.daemon_sleep, force: true);
1411	mutex_unlock(lock: &mddev->bitmap_info.mutex);
1412	}
1413
1414	static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1415	sector_t offset, sector_t *blocks,
1416	int create)
1417	__releases(bitmap->lock)
1418	__acquires(bitmap->lock)
1419	{
1420	/* If 'create', we might release the lock and reclaim it.
1421	* The lock must have been taken with interrupts enabled.
1422	* If !create, we don't release the lock.
1423	*/
1424	sector_t chunk = offset >> bitmap->chunkshift;
1425	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1426	unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1427	sector_t csize;
1428	int err;
1429
1430	if (page >= bitmap->pages) {
1431	/*
1432	* This can happen if bitmap_start_sync goes beyond
1433	* End-of-device while looking for a whole page or
1434	* user set a huge number to sysfs bitmap_set_bits.
1435	*/
1436	return NULL;
1437	}
1438	err = md_bitmap_checkpage(bitmap, page, create, no_hijack: 0);
1439
1440	if (bitmap->bp[page].hijacked ||
1441	bitmap->bp[page].map == NULL)
1442	csize = ((sector_t)1) << (bitmap->chunkshift +
1443	PAGE_COUNTER_SHIFT);
1444	else
1445	csize = ((sector_t)1) << bitmap->chunkshift;
1446	*blocks = csize - (offset & (csize - 1));
1447
1448	if (err < 0)
1449	return NULL;
1450
1451	/* now locked ... */
1452
1453	if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1454	/* should we use the first or second counter field
1455	* of the hijacked pointer? */
1456	int hi = (pageoff > PAGE_COUNTER_MASK);
1457	return &((bitmap_counter_t *)
1458	&bitmap->bp[page].map)[hi];
1459	} else /* page is allocated */
1460	return (bitmap_counter_t *)
1461	&(bitmap->bp[page].map[pageoff]);
1462	}
1463
1464	int md_bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1465	{
1466	if (!bitmap)
1467	return 0;
1468
1469	if (behind) {
1470	int bw;
1471	atomic_inc(v: &bitmap->behind_writes);
1472	bw = atomic_read(v: &bitmap->behind_writes);
1473	if (bw > bitmap->behind_writes_used)
1474	bitmap->behind_writes_used = bw;
1475
1476	pr_debug("inc write-behind count %d/%lu\n",
1477	bw, bitmap->mddev->bitmap_info.max_write_behind);
1478	}
1479
1480	while (sectors) {
1481	sector_t blocks;
1482	bitmap_counter_t *bmc;
1483
1484	spin_lock_irq(lock: &bitmap->counts.lock);
1485	bmc = md_bitmap_get_counter(bitmap: &bitmap->counts, offset, blocks: &blocks, create: 1);
1486	if (!bmc) {
1487	spin_unlock_irq(lock: &bitmap->counts.lock);
1488	return 0;
1489	}
1490
1491	if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1492	DEFINE_WAIT(__wait);
1493	/* note that it is safe to do the prepare_to_wait
1494	* after the test as long as we do it before dropping
1495	* the spinlock.
1496	*/
1497	prepare_to_wait(wq_head: &bitmap->overflow_wait, wq_entry: &__wait,
1498	TASK_UNINTERRUPTIBLE);
1499	spin_unlock_irq(lock: &bitmap->counts.lock);
1500	schedule();
1501	finish_wait(wq_head: &bitmap->overflow_wait, wq_entry: &__wait);
1502	continue;
1503	}
1504
1505	switch (*bmc) {
1506	case 0:
1507	md_bitmap_file_set_bit(bitmap, block: offset);
1508	md_bitmap_count_page(bitmap: &bitmap->counts, offset, inc: 1);
1509	fallthrough;
1510	case 1:
1511	*bmc = 2;
1512	}
1513
1514	(*bmc)++;
1515
1516	spin_unlock_irq(lock: &bitmap->counts.lock);
1517
1518	offset += blocks;
1519	if (sectors > blocks)
1520	sectors -= blocks;
1521	else
1522	sectors = 0;
1523	}
1524	return 0;
1525	}
1526	EXPORT_SYMBOL(md_bitmap_startwrite);
1527
1528	void md_bitmap_endwrite(struct bitmap *bitmap, sector_t offset,
1529	unsigned long sectors, int success, int behind)
1530	{
1531	if (!bitmap)
1532	return;
1533	if (behind) {
1534	if (atomic_dec_and_test(v: &bitmap->behind_writes))
1535	wake_up(&bitmap->behind_wait);
1536	pr_debug("dec write-behind count %d/%lu\n",
1537	atomic_read(&bitmap->behind_writes),
1538	bitmap->mddev->bitmap_info.max_write_behind);
1539	}
1540
1541	while (sectors) {
1542	sector_t blocks;
1543	unsigned long flags;
1544	bitmap_counter_t *bmc;
1545
1546	spin_lock_irqsave(&bitmap->counts.lock, flags);
1547	bmc = md_bitmap_get_counter(bitmap: &bitmap->counts, offset, blocks: &blocks, create: 0);
1548	if (!bmc) {
1549	spin_unlock_irqrestore(lock: &bitmap->counts.lock, flags);
1550	return;
1551	}
1552
1553	if (success && !bitmap->mddev->degraded &&
1554	bitmap->events_cleared < bitmap->mddev->events) {
1555	bitmap->events_cleared = bitmap->mddev->events;
1556	bitmap->need_sync = 1;
1557	sysfs_notify_dirent_safe(sd: bitmap->sysfs_can_clear);
1558	}
1559
1560	if (!success && !NEEDED(*bmc))
1561	*bmc |= NEEDED_MASK;
1562
1563	if (COUNTER(*bmc) == COUNTER_MAX)
1564	wake_up(&bitmap->overflow_wait);
1565
1566	(*bmc)--;
1567	if (*bmc <= 2) {
1568	md_bitmap_set_pending(bitmap: &bitmap->counts, offset);
1569	bitmap->allclean = 0;
1570	}
1571	spin_unlock_irqrestore(lock: &bitmap->counts.lock, flags);
1572	offset += blocks;
1573	if (sectors > blocks)
1574	sectors -= blocks;
1575	else
1576	sectors = 0;
1577	}
1578	}
1579	EXPORT_SYMBOL(md_bitmap_endwrite);
1580
1581	static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1582	int degraded)
1583	{
1584	bitmap_counter_t *bmc;
1585	int rv;
1586	if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
1587	*blocks = 1024;
1588	return 1; /* always resync if no bitmap */
1589	}
1590	spin_lock_irq(lock: &bitmap->counts.lock);
1591	bmc = md_bitmap_get_counter(bitmap: &bitmap->counts, offset, blocks, create: 0);
1592	rv = 0;
1593	if (bmc) {
1594	/* locked */
1595	if (RESYNC(*bmc))
1596	rv = 1;
1597	else if (NEEDED(*bmc)) {
1598	rv = 1;
1599	if (!degraded) { /* don't set/clear bits if degraded */
1600	*bmc |= RESYNC_MASK;
1601	*bmc &= ~NEEDED_MASK;
1602	}
1603	}
1604	}
1605	spin_unlock_irq(lock: &bitmap->counts.lock);
1606	return rv;
1607	}
1608
1609	int md_bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1610	int degraded)
1611	{
1612	/* bitmap_start_sync must always report on multiples of whole
1613	* pages, otherwise resync (which is very PAGE_SIZE based) will
1614	* get confused.
1615	* So call __bitmap_start_sync repeatedly (if needed) until
1616	* At least PAGE_SIZE>>9 blocks are covered.
1617	* Return the 'or' of the result.
1618	*/
1619	int rv = 0;
1620	sector_t blocks1;
1621
1622	*blocks = 0;
1623	while (*blocks < (PAGE_SIZE>>9)) {
1624	rv |= __bitmap_start_sync(bitmap, offset,
1625	blocks: &blocks1, degraded);
1626	offset += blocks1;
1627	*blocks += blocks1;
1628	}
1629	return rv;
1630	}
1631	EXPORT_SYMBOL(md_bitmap_start_sync);
1632
1633	void md_bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1634	{
1635	bitmap_counter_t *bmc;
1636	unsigned long flags;
1637
1638	if (bitmap == NULL) {
1639	*blocks = 1024;
1640	return;
1641	}
1642	spin_lock_irqsave(&bitmap->counts.lock, flags);
1643	bmc = md_bitmap_get_counter(bitmap: &bitmap->counts, offset, blocks, create: 0);
1644	if (bmc == NULL)
1645	goto unlock;
1646	/* locked */
1647	if (RESYNC(*bmc)) {
1648	*bmc &= ~RESYNC_MASK;
1649
1650	if (!NEEDED(*bmc) && aborted)
1651	*bmc |= NEEDED_MASK;
1652	else {
1653	if (*bmc <= 2) {
1654	md_bitmap_set_pending(bitmap: &bitmap->counts, offset);
1655	bitmap->allclean = 0;
1656	}
1657	}
1658	}
1659	unlock:
1660	spin_unlock_irqrestore(lock: &bitmap->counts.lock, flags);
1661	}
1662	EXPORT_SYMBOL(md_bitmap_end_sync);
1663
1664	void md_bitmap_close_sync(struct bitmap *bitmap)
1665	{
1666	/* Sync has finished, and any bitmap chunks that weren't synced
1667	* properly have been aborted. It remains to us to clear the
1668	* RESYNC bit wherever it is still on
1669	*/
1670	sector_t sector = 0;
1671	sector_t blocks;
1672	if (!bitmap)
1673	return;
1674	while (sector < bitmap->mddev->resync_max_sectors) {
1675	md_bitmap_end_sync(bitmap, sector, &blocks, 0);
1676	sector += blocks;
1677	}
1678	}
1679	EXPORT_SYMBOL(md_bitmap_close_sync);
1680
1681	void md_bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force)
1682	{
1683	sector_t s = 0;
1684	sector_t blocks;
1685
1686	if (!bitmap)
1687	return;
1688	if (sector == 0) {
1689	bitmap->last_end_sync = jiffies;
1690	return;
1691	}
1692	if (!force && time_before(jiffies, (bitmap->last_end_sync
1693	+ bitmap->mddev->bitmap_info.daemon_sleep)))
1694	return;
1695	wait_event(bitmap->mddev->recovery_wait,
1696	atomic_read(&bitmap->mddev->recovery_active) == 0);
1697
1698	bitmap->mddev->curr_resync_completed = sector;
1699	set_bit(nr: MD_SB_CHANGE_CLEAN, addr: &bitmap->mddev->sb_flags);
1700	sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1701	s = 0;
1702	while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1703	md_bitmap_end_sync(bitmap, s, &blocks, 0);
1704	s += blocks;
1705	}
1706	bitmap->last_end_sync = jiffies;
1707	sysfs_notify_dirent_safe(sd: bitmap->mddev->sysfs_completed);
1708	}
1709	EXPORT_SYMBOL(md_bitmap_cond_end_sync);
1710
1711	void md_bitmap_sync_with_cluster(struct mddev *mddev,
1712	sector_t old_lo, sector_t old_hi,
1713	sector_t new_lo, sector_t new_hi)
1714	{
1715	struct bitmap *bitmap = mddev->bitmap;
1716	sector_t sector, blocks = 0;
1717
1718	for (sector = old_lo; sector < new_lo; ) {
1719	md_bitmap_end_sync(bitmap, sector, &blocks, 0);
1720	sector += blocks;
1721	}
1722	WARN((blocks > new_lo) && old_lo, "alignment is not correct for lo\n");
1723
1724	for (sector = old_hi; sector < new_hi; ) {
1725	md_bitmap_start_sync(bitmap, sector, &blocks, 0);
1726	sector += blocks;
1727	}
1728	WARN((blocks > new_hi) && old_hi, "alignment is not correct for hi\n");
1729	}
1730	EXPORT_SYMBOL(md_bitmap_sync_with_cluster);
1731
1732	static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1733	{
1734	/* For each chunk covered by any of these sectors, set the
1735	* counter to 2 and possibly set resync_needed. They should all
1736	* be 0 at this point
1737	*/
1738
1739	sector_t secs;
1740	bitmap_counter_t *bmc;
1741	spin_lock_irq(lock: &bitmap->counts.lock);
1742	bmc = md_bitmap_get_counter(bitmap: &bitmap->counts, offset, blocks: &secs, create: 1);
1743	if (!bmc) {
1744	spin_unlock_irq(lock: &bitmap->counts.lock);
1745	return;
1746	}
1747	if (!*bmc) {
1748	*bmc = 2;
1749	md_bitmap_count_page(bitmap: &bitmap->counts, offset, inc: 1);
1750	md_bitmap_set_pending(bitmap: &bitmap->counts, offset);
1751	bitmap->allclean = 0;
1752	}
1753	if (needed)
1754	*bmc |= NEEDED_MASK;
1755	spin_unlock_irq(lock: &bitmap->counts.lock);
1756	}
1757
1758	/* dirty the memory and file bits for bitmap chunks "s" to "e" */
1759	void md_bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1760	{
1761	unsigned long chunk;
1762
1763	for (chunk = s; chunk <= e; chunk++) {
1764	sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1765	md_bitmap_set_memory_bits(bitmap, offset: sec, needed: 1);
1766	md_bitmap_file_set_bit(bitmap, block: sec);
1767	if (sec < bitmap->mddev->recovery_cp)
1768	/* We are asserting that the array is dirty,
1769	* so move the recovery_cp address back so
1770	* that it is obvious that it is dirty
1771	*/
1772	bitmap->mddev->recovery_cp = sec;
1773	}
1774	}
1775
1776	/*
1777	* flush out any pending updates
1778	*/
1779	void md_bitmap_flush(struct mddev *mddev)
1780	{
1781	struct bitmap *bitmap = mddev->bitmap;
1782	long sleep;
1783
1784	if (!bitmap) /* there was no bitmap */
1785	return;
1786
1787	/* run the daemon_work three time to ensure everything is flushed
1788	* that can be
1789	*/
1790	sleep = mddev->bitmap_info.daemon_sleep * 2;
1791	bitmap->daemon_lastrun -= sleep;
1792	md_bitmap_daemon_work(mddev);
1793	bitmap->daemon_lastrun -= sleep;
1794	md_bitmap_daemon_work(mddev);
1795	bitmap->daemon_lastrun -= sleep;
1796	md_bitmap_daemon_work(mddev);
1797	if (mddev->bitmap_info.external)
1798	md_super_wait(mddev);
1799	md_bitmap_update_sb(bitmap);
1800	}
1801
1802	/*
1803	* free memory that was allocated
1804	*/
1805	void md_bitmap_free(struct bitmap *bitmap)
1806	{
1807	unsigned long k, pages;
1808	struct bitmap_page *bp;
1809
1810	if (!bitmap) /* there was no bitmap */
1811	return;
1812
1813	if (bitmap->sysfs_can_clear)
1814	sysfs_put(kn: bitmap->sysfs_can_clear);
1815
1816	if (mddev_is_clustered(mddev: bitmap->mddev) && bitmap->mddev->cluster_info &&
1817	bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev))
1818	md_cluster_stop(mddev: bitmap->mddev);
1819
1820	/* Shouldn't be needed - but just in case.... */
1821	wait_event(bitmap->write_wait,
1822	atomic_read(&bitmap->pending_writes) == 0);
1823
1824	/* release the bitmap file */
1825	md_bitmap_file_unmap(store: &bitmap->storage);
1826
1827	bp = bitmap->counts.bp;
1828	pages = bitmap->counts.pages;
1829
1830	/* free all allocated memory */
1831
1832	if (bp) /* deallocate the page memory */
1833	for (k = 0; k < pages; k++)
1834	if (bp[k].map && !bp[k].hijacked)
1835	kfree(objp: bp[k].map);
1836	kfree(objp: bp);
1837	kfree(objp: bitmap);
1838	}
1839	EXPORT_SYMBOL(md_bitmap_free);
1840
1841	void md_bitmap_wait_behind_writes(struct mddev *mddev)
1842	{
1843	struct bitmap *bitmap = mddev->bitmap;
1844
1845	/* wait for behind writes to complete */
1846	if (bitmap && atomic_read(v: &bitmap->behind_writes) > 0) {
1847	pr_debug("md:%s: behind writes in progress - waiting to stop.\n",
1848	mdname(mddev));
1849	/* need to kick something here to make sure I/O goes? */
1850	wait_event(bitmap->behind_wait,
1851	atomic_read(&bitmap->behind_writes) == 0);
1852	}
1853	}
1854
1855	void md_bitmap_destroy(struct mddev *mddev)
1856	{
1857	struct bitmap *bitmap = mddev->bitmap;
1858
1859	if (!bitmap) /* there was no bitmap */
1860	return;
1861
1862	md_bitmap_wait_behind_writes(mddev);
1863	if (!mddev->serialize_policy)
1864	mddev_destroy_serial_pool(mddev, NULL);
1865
1866	mutex_lock(&mddev->bitmap_info.mutex);
1867	spin_lock(lock: &mddev->lock);
1868	mddev->bitmap = NULL; /* disconnect from the md device */
1869	spin_unlock(lock: &mddev->lock);
1870	mutex_unlock(lock: &mddev->bitmap_info.mutex);
1871	mddev_set_timeout(mddev, MAX_SCHEDULE_TIMEOUT, force: true);
1872
1873	md_bitmap_free(bitmap);
1874	}
1875
1876	/*
1877	* initialize the bitmap structure
1878	* if this returns an error, bitmap_destroy must be called to do clean up
1879	* once mddev->bitmap is set
1880	*/
1881	struct bitmap *md_bitmap_create(struct mddev *mddev, int slot)
1882	{
1883	struct bitmap *bitmap;
1884	sector_t blocks = mddev->resync_max_sectors;
1885	struct file *file = mddev->bitmap_info.file;
1886	int err;
1887	struct kernfs_node *bm = NULL;
1888
1889	BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1890
1891	BUG_ON(file && mddev->bitmap_info.offset);
1892
1893	if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
1894	pr_notice("md/raid:%s: array with journal cannot have bitmap\n",
1895	mdname(mddev));
1896	return ERR_PTR(error: -EBUSY);
1897	}
1898
1899	bitmap = kzalloc(size: sizeof(*bitmap), GFP_KERNEL);
1900	if (!bitmap)
1901	return ERR_PTR(error: -ENOMEM);
1902
1903	spin_lock_init(&bitmap->counts.lock);
1904	atomic_set(v: &bitmap->pending_writes, i: 0);
1905	init_waitqueue_head(&bitmap->write_wait);
1906	init_waitqueue_head(&bitmap->overflow_wait);
1907	init_waitqueue_head(&bitmap->behind_wait);
1908
1909	bitmap->mddev = mddev;
1910	bitmap->cluster_slot = slot;
1911
1912	if (mddev->kobj.sd)
1913	bm = sysfs_get_dirent(parent: mddev->kobj.sd, name: "bitmap");
1914	if (bm) {
1915	bitmap->sysfs_can_clear = sysfs_get_dirent(parent: bm, name: "can_clear");
1916	sysfs_put(kn: bm);
1917	} else
1918	bitmap->sysfs_can_clear = NULL;
1919
1920	bitmap->storage.file = file;
1921	if (file) {
1922	get_file(f: file);
1923	/* As future accesses to this file will use bmap,
1924	* and bypass the page cache, we must sync the file
1925	* first.
1926	*/
1927	vfs_fsync(file, datasync: 1);
1928	}
1929	/* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1930	if (!mddev->bitmap_info.external) {
1931	/*
1932	* If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
1933	* instructing us to create a new on-disk bitmap instance.
1934	*/
1935	if (test_and_clear_bit(nr: MD_ARRAY_FIRST_USE, addr: &mddev->flags))
1936	err = md_bitmap_new_disk_sb(bitmap);
1937	else
1938	err = md_bitmap_read_sb(bitmap);
1939	} else {
1940	err = 0;
1941	if (mddev->bitmap_info.chunksize == 0 ||
1942	mddev->bitmap_info.daemon_sleep == 0)
1943	/* chunksize and time_base need to be
1944	* set first. */
1945	err = -EINVAL;
1946	}
1947	if (err)
1948	goto error;
1949
1950	bitmap->daemon_lastrun = jiffies;
1951	err = md_bitmap_resize(bitmap, blocks, chunksize: mddev->bitmap_info.chunksize, init: 1);
1952	if (err)
1953	goto error;
1954
1955	pr_debug("created bitmap (%lu pages) for device %s\n",
1956	bitmap->counts.pages, bmname(bitmap));
1957
1958	err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1959	if (err)
1960	goto error;
1961
1962	return bitmap;
1963	error:
1964	md_bitmap_free(bitmap);
1965	return ERR_PTR(error: err);
1966	}
1967
1968	int md_bitmap_load(struct mddev *mddev)
1969	{
1970	int err = 0;
1971	sector_t start = 0;
1972	sector_t sector = 0;
1973	struct bitmap *bitmap = mddev->bitmap;
1974	struct md_rdev *rdev;
1975
1976	if (!bitmap)
1977	goto out;
1978
1979	rdev_for_each(rdev, mddev)
1980	mddev_create_serial_pool(mddev, rdev);
1981
1982	if (mddev_is_clustered(mddev))
1983	md_cluster_ops->load_bitmaps(mddev, mddev->bitmap_info.nodes);
1984
1985	/* Clear out old bitmap info first: Either there is none, or we
1986	* are resuming after someone else has possibly changed things,
1987	* so we should forget old cached info.
1988	* All chunks should be clean, but some might need_sync.
1989	*/
1990	while (sector < mddev->resync_max_sectors) {
1991	sector_t blocks;
1992	md_bitmap_start_sync(bitmap, sector, &blocks, 0);
1993	sector += blocks;
1994	}
1995	md_bitmap_close_sync(bitmap);
1996
1997	if (mddev->degraded == 0
1998	|| bitmap->events_cleared == mddev->events)
1999	/* no need to keep dirty bits to optimise a
2000	* re-add of a missing device */
2001	start = mddev->recovery_cp;
2002
2003	mutex_lock(&mddev->bitmap_info.mutex);
2004	err = md_bitmap_init_from_disk(bitmap, start);
2005	mutex_unlock(lock: &mddev->bitmap_info.mutex);
2006
2007	if (err)
2008	goto out;
2009	clear_bit(nr: BITMAP_STALE, addr: &bitmap->flags);
2010
2011	/* Kick recovery in case any bits were set */
2012	set_bit(nr: MD_RECOVERY_NEEDED, addr: &bitmap->mddev->recovery);
2013
2014	mddev_set_timeout(mddev, timeout: mddev->bitmap_info.daemon_sleep, force: true);
2015	md_wakeup_thread(thread: mddev->thread);
2016
2017	md_bitmap_update_sb(bitmap);
2018
2019	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
2020	err = -EIO;
2021	out:
2022	return err;
2023	}
2024	EXPORT_SYMBOL_GPL(md_bitmap_load);
2025
2026	/* caller need to free returned bitmap with md_bitmap_free() */
2027	struct bitmap *get_bitmap_from_slot(struct mddev *mddev, int slot)
2028	{
2029	int rv = 0;
2030	struct bitmap *bitmap;
2031
2032	bitmap = md_bitmap_create(mddev, slot);
2033	if (IS_ERR(ptr: bitmap)) {
2034	rv = PTR_ERR(ptr: bitmap);
2035	return ERR_PTR(error: rv);
2036	}
2037
2038	rv = md_bitmap_init_from_disk(bitmap, start: 0);
2039	if (rv) {
2040	md_bitmap_free(bitmap);
2041	return ERR_PTR(error: rv);
2042	}
2043
2044	return bitmap;
2045	}
2046	EXPORT_SYMBOL(get_bitmap_from_slot);
2047
2048	/* Loads the bitmap associated with slot and copies the resync information
2049	* to our bitmap
2050	*/
2051	int md_bitmap_copy_from_slot(struct mddev *mddev, int slot,
2052	sector_t *low, sector_t *high, bool clear_bits)
2053	{
2054	int rv = 0, i, j;
2055	sector_t block, lo = 0, hi = 0;
2056	struct bitmap_counts *counts;
2057	struct bitmap *bitmap;
2058
2059	bitmap = get_bitmap_from_slot(mddev, slot);
2060	if (IS_ERR(ptr: bitmap)) {
2061	pr_err("%s can't get bitmap from slot %d\n", __func__, slot);
2062	return -1;
2063	}
2064
2065	counts = &bitmap->counts;
2066	for (j = 0; j < counts->chunks; j++) {
2067	block = (sector_t)j << counts->chunkshift;
2068	if (md_bitmap_file_test_bit(bitmap, block)) {
2069	if (!lo)
2070	lo = block;
2071	hi = block;
2072	md_bitmap_file_clear_bit(bitmap, block);
2073	md_bitmap_set_memory_bits(bitmap: mddev->bitmap, offset: block, needed: 1);
2074	md_bitmap_file_set_bit(bitmap: mddev->bitmap, block);
2075	}
2076	}
2077
2078	if (clear_bits) {
2079	md_bitmap_update_sb(bitmap);
2080	/* BITMAP_PAGE_PENDING is set, but bitmap_unplug needs
2081	* BITMAP_PAGE_DIRTY or _NEEDWRITE to write ... */
2082	for (i = 0; i < bitmap->storage.file_pages; i++)
2083	if (test_page_attr(bitmap, pnum: i, attr: BITMAP_PAGE_PENDING))
2084	set_page_attr(bitmap, pnum: i, attr: BITMAP_PAGE_NEEDWRITE);
2085	md_bitmap_unplug(bitmap);
2086	}
2087	md_bitmap_unplug(mddev->bitmap);
2088	*low = lo;
2089	*high = hi;
2090	md_bitmap_free(bitmap);
2091
2092	return rv;
2093	}
2094	EXPORT_SYMBOL_GPL(md_bitmap_copy_from_slot);
2095
2096
2097	void md_bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
2098	{
2099	unsigned long chunk_kb;
2100	struct bitmap_counts *counts;
2101
2102	if (!bitmap)
2103	return;
2104
2105	counts = &bitmap->counts;
2106
2107	chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
2108	seq_printf(m: seq, fmt: "bitmap: %lu/%lu pages [%luKB], "
2109	"%lu%s chunk",
2110	counts->pages - counts->missing_pages,
2111	counts->pages,
2112	(counts->pages - counts->missing_pages)
2113	<< (PAGE_SHIFT - 10),
2114	chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
2115	chunk_kb ? "KB" : "B");
2116	if (bitmap->storage.file) {
2117	seq_printf(m: seq, fmt: ", file: ");
2118	seq_file_path(seq, bitmap->storage.file, " \t\n");
2119	}
2120
2121	seq_printf(m: seq, fmt: "\n");
2122	}
2123
2124	int md_bitmap_resize(struct bitmap *bitmap, sector_t blocks,
2125	int chunksize, int init)
2126	{
2127	/* If chunk_size is 0, choose an appropriate chunk size.
2128	* Then possibly allocate new storage space.
2129	* Then quiesce, copy bits, replace bitmap, and re-start
2130	*
2131	* This function is called both to set up the initial bitmap
2132	* and to resize the bitmap while the array is active.
2133	* If this happens as a result of the array being resized,
2134	* chunksize will be zero, and we need to choose a suitable
2135	* chunksize, otherwise we use what we are given.
2136	*/
2137	struct bitmap_storage store;
2138	struct bitmap_counts old_counts;
2139	unsigned long chunks;
2140	sector_t block;
2141	sector_t old_blocks, new_blocks;
2142	int chunkshift;
2143	int ret = 0;
2144	long pages;
2145	struct bitmap_page *new_bp;
2146
2147	if (bitmap->storage.file && !init) {
2148	pr_info("md: cannot resize file-based bitmap\n");
2149	return -EINVAL;
2150	}
2151
2152	if (chunksize == 0) {
2153	/* If there is enough space, leave the chunk size unchanged,
2154	* else increase by factor of two until there is enough space.
2155	*/
2156	long bytes;
2157	long space = bitmap->mddev->bitmap_info.space;
2158
2159	if (space == 0) {
2160	/* We don't know how much space there is, so limit
2161	* to current size - in sectors.
2162	*/
2163	bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
2164	if (!bitmap->mddev->bitmap_info.external)
2165	bytes += sizeof(bitmap_super_t);
2166	space = DIV_ROUND_UP(bytes, 512);
2167	bitmap->mddev->bitmap_info.space = space;
2168	}
2169	chunkshift = bitmap->counts.chunkshift;
2170	chunkshift--;
2171	do {
2172	/* 'chunkshift' is shift from block size to chunk size */
2173	chunkshift++;
2174	chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
2175	bytes = DIV_ROUND_UP(chunks, 8);
2176	if (!bitmap->mddev->bitmap_info.external)
2177	bytes += sizeof(bitmap_super_t);
2178	} while (bytes > (space << 9) && (chunkshift + BITMAP_BLOCK_SHIFT) <
2179	(BITS_PER_BYTE * sizeof(((bitmap_super_t *)0)->chunksize) - 1));
2180	} else
2181	chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;
2182
2183	chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
2184	memset(&store, 0, sizeof(store));
2185	if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
2186	ret = md_bitmap_storage_alloc(store: &store, chunks,
2187	with_super: !bitmap->mddev->bitmap_info.external,
2188	slot_number: mddev_is_clustered(mddev: bitmap->mddev)
2189	? bitmap->cluster_slot : 0);
2190	if (ret) {
2191	md_bitmap_file_unmap(store: &store);
2192	goto err;
2193	}
2194
2195	pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
2196
2197	new_bp = kcalloc(n: pages, size: sizeof(*new_bp), GFP_KERNEL);
2198	ret = -ENOMEM;
2199	if (!new_bp) {
2200	md_bitmap_file_unmap(store: &store);
2201	goto err;
2202	}
2203
2204	if (!init)
2205	bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
2206
2207	store.file = bitmap->storage.file;
2208	bitmap->storage.file = NULL;
2209
2210	if (store.sb_page && bitmap->storage.sb_page)
2211	memcpy(page_address(store.sb_page),
2212	page_address(bitmap->storage.sb_page),
2213	sizeof(bitmap_super_t));
2214	spin_lock_irq(lock: &bitmap->counts.lock);
2215	md_bitmap_file_unmap(store: &bitmap->storage);
2216	bitmap->storage = store;
2217
2218	old_counts = bitmap->counts;
2219	bitmap->counts.bp = new_bp;
2220	bitmap->counts.pages = pages;
2221	bitmap->counts.missing_pages = pages;
2222	bitmap->counts.chunkshift = chunkshift;
2223	bitmap->counts.chunks = chunks;
2224	bitmap->mddev->bitmap_info.chunksize = 1UL << (chunkshift +
2225	BITMAP_BLOCK_SHIFT);
2226
2227	blocks = min(old_counts.chunks << old_counts.chunkshift,
2228	chunks << chunkshift);
2229
2230	/* For cluster raid, need to pre-allocate bitmap */
2231	if (mddev_is_clustered(mddev: bitmap->mddev)) {
2232	unsigned long page;
2233	for (page = 0; page < pages; page++) {
2234	ret = md_bitmap_checkpage(bitmap: &bitmap->counts, page, create: 1, no_hijack: 1);
2235	if (ret) {
2236	unsigned long k;
2237
2238	/* deallocate the page memory */
2239	for (k = 0; k < page; k++) {
2240	kfree(objp: new_bp[k].map);
2241	}
2242	kfree(objp: new_bp);
2243
2244	/* restore some fields from old_counts */
2245	bitmap->counts.bp = old_counts.bp;
2246	bitmap->counts.pages = old_counts.pages;
2247	bitmap->counts.missing_pages = old_counts.pages;
2248	bitmap->counts.chunkshift = old_counts.chunkshift;
2249	bitmap->counts.chunks = old_counts.chunks;
2250	bitmap->mddev->bitmap_info.chunksize =
2251	1UL << (old_counts.chunkshift + BITMAP_BLOCK_SHIFT);
2252	blocks = old_counts.chunks << old_counts.chunkshift;
2253	pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n");
2254	break;
2255	} else
2256	bitmap->counts.bp[page].count += 1;
2257	}
2258	}
2259
2260	for (block = 0; block < blocks; ) {
2261	bitmap_counter_t *bmc_old, *bmc_new;
2262	int set;
2263
2264	bmc_old = md_bitmap_get_counter(bitmap: &old_counts, offset: block, blocks: &old_blocks, create: 0);
2265	set = bmc_old && NEEDED(*bmc_old);
2266
2267	if (set) {
2268	bmc_new = md_bitmap_get_counter(bitmap: &bitmap->counts, offset: block, blocks: &new_blocks, create: 1);
2269	if (bmc_new) {
2270	if (*bmc_new == 0) {
2271	/* need to set on-disk bits too. */
2272	sector_t end = block + new_blocks;
2273	sector_t start = block >> chunkshift;
2274
2275	start <<= chunkshift;
2276	while (start < end) {
2277	md_bitmap_file_set_bit(bitmap, block);
2278	start += 1 << chunkshift;
2279	}
2280	*bmc_new = 2;
2281	md_bitmap_count_page(bitmap: &bitmap->counts, offset: block, inc: 1);
2282	md_bitmap_set_pending(bitmap: &bitmap->counts, offset: block);
2283	}
2284	*bmc_new |= NEEDED_MASK;
2285	}
2286	if (new_blocks < old_blocks)
2287	old_blocks = new_blocks;
2288	}
2289	block += old_blocks;
2290	}
2291
2292	if (bitmap->counts.bp != old_counts.bp) {
2293	unsigned long k;
2294	for (k = 0; k < old_counts.pages; k++)
2295	if (!old_counts.bp[k].hijacked)
2296	kfree(objp: old_counts.bp[k].map);
2297	kfree(objp: old_counts.bp);
2298	}
2299
2300	if (!init) {
2301	int i;
2302	while (block < (chunks << chunkshift)) {
2303	bitmap_counter_t *bmc;
2304	bmc = md_bitmap_get_counter(bitmap: &bitmap->counts, offset: block, blocks: &new_blocks, create: 1);
2305	if (bmc) {
2306	/* new space. It needs to be resynced, so
2307	* we set NEEDED_MASK.
2308	*/
2309	if (*bmc == 0) {
2310	*bmc = NEEDED_MASK | 2;
2311	md_bitmap_count_page(bitmap: &bitmap->counts, offset: block, inc: 1);
2312	md_bitmap_set_pending(bitmap: &bitmap->counts, offset: block);
2313	}
2314	}
2315	block += new_blocks;
2316	}
2317	for (i = 0; i < bitmap->storage.file_pages; i++)
2318	set_page_attr(bitmap, pnum: i, attr: BITMAP_PAGE_DIRTY);
2319	}
2320	spin_unlock_irq(lock: &bitmap->counts.lock);
2321
2322	if (!init) {
2323	md_bitmap_unplug(bitmap);
2324	bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
2325	}
2326	ret = 0;
2327	err:
2328	return ret;
2329	}
2330	EXPORT_SYMBOL_GPL(md_bitmap_resize);
2331
2332	static ssize_t
2333	location_show(struct mddev *mddev, char *page)
2334	{
2335	ssize_t len;
2336	if (mddev->bitmap_info.file)
2337	len = sprintf(buf: page, fmt: "file");
2338	else if (mddev->bitmap_info.offset)
2339	len = sprintf(buf: page, fmt: "%+lld", (long long)mddev->bitmap_info.offset);
2340	else
2341	len = sprintf(buf: page, fmt: "none");
2342	len += sprintf(buf: page+len, fmt: "\n");
2343	return len;
2344	}
2345
2346	static ssize_t
2347	location_store(struct mddev *mddev, const char *buf, size_t len)
2348	{
2349	int rv;
2350
2351	rv = mddev_suspend_and_lock(mddev);
2352	if (rv)
2353	return rv;
2354
2355	if (mddev->pers) {
2356	if (mddev->recovery || mddev->sync_thread) {
2357	rv = -EBUSY;
2358	goto out;
2359	}
2360	}
2361
2362	if (mddev->bitmap || mddev->bitmap_info.file ||
2363	mddev->bitmap_info.offset) {
2364	/* bitmap already configured. Only option is to clear it */
2365	if (strncmp(buf, "none", 4) != 0) {
2366	rv = -EBUSY;
2367	goto out;
2368	}
2369
2370	md_bitmap_destroy(mddev);
2371	mddev->bitmap_info.offset = 0;
2372	if (mddev->bitmap_info.file) {
2373	struct file *f = mddev->bitmap_info.file;
2374	mddev->bitmap_info.file = NULL;
2375	fput(f);
2376	}
2377	} else {
2378	/* No bitmap, OK to set a location */
2379	long long offset;
2380	struct bitmap *bitmap;
2381
2382	if (strncmp(buf, "none", 4) == 0)
2383	/* nothing to be done */;
2384	else if (strncmp(buf, "file:", 5) == 0) {
2385	/* Not supported yet */
2386	rv = -EINVAL;
2387	goto out;
2388	} else {
2389	if (buf[0] == '+')
2390	rv = kstrtoll(s: buf+1, base: 10, res: &offset);
2391	else
2392	rv = kstrtoll(s: buf, base: 10, res: &offset);
2393	if (rv)
2394	goto out;
2395	if (offset == 0) {
2396	rv = -EINVAL;
2397	goto out;
2398	}
2399	if (mddev->bitmap_info.external == 0 &&
2400	mddev->major_version == 0 &&
2401	offset != mddev->bitmap_info.default_offset) {
2402	rv = -EINVAL;
2403	goto out;
2404	}
2405
2406	mddev->bitmap_info.offset = offset;
2407	bitmap = md_bitmap_create(mddev, slot: -1);
2408	if (IS_ERR(ptr: bitmap)) {
2409	rv = PTR_ERR(ptr: bitmap);
2410	goto out;
2411	}
2412
2413	mddev->bitmap = bitmap;
2414	rv = md_bitmap_load(mddev);
2415	if (rv) {
2416	mddev->bitmap_info.offset = 0;
2417	md_bitmap_destroy(mddev);
2418	goto out;
2419	}
2420	}
2421	}
2422	if (!mddev->external) {
2423	/* Ensure new bitmap info is stored in
2424	* metadata promptly.
2425	*/
2426	set_bit(nr: MD_SB_CHANGE_DEVS, addr: &mddev->sb_flags);
2427	md_wakeup_thread(thread: mddev->thread);
2428	}
2429	rv = 0;
2430	out:
2431	mddev_unlock_and_resume(mddev);
2432	if (rv)
2433	return rv;
2434	return len;
2435	}
2436
2437	static struct md_sysfs_entry bitmap_location =
2438	__ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
2439
2440	/* 'bitmap/space' is the space available at 'location' for the
2441	* bitmap. This allows the kernel to know when it is safe to
2442	* resize the bitmap to match a resized array.
2443	*/
2444	static ssize_t
2445	space_show(struct mddev *mddev, char *page)
2446	{
2447	return sprintf(buf: page, fmt: "%lu\n", mddev->bitmap_info.space);
2448	}
2449
2450	static ssize_t
2451	space_store(struct mddev *mddev, const char *buf, size_t len)
2452	{
2453	unsigned long sectors;
2454	int rv;
2455
2456	rv = kstrtoul(s: buf, base: 10, res: &sectors);
2457	if (rv)
2458	return rv;
2459
2460	if (sectors == 0)
2461	return -EINVAL;
2462
2463	if (mddev->bitmap &&
2464	sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2465	return -EFBIG; /* Bitmap is too big for this small space */
2466
2467	/* could make sure it isn't too big, but that isn't really
2468	* needed - user-space should be careful.
2469	*/
2470	mddev->bitmap_info.space = sectors;
2471	return len;
2472	}
2473
2474	static struct md_sysfs_entry bitmap_space =
2475	__ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);
2476
2477	static ssize_t
2478	timeout_show(struct mddev *mddev, char *page)
2479	{
2480	ssize_t len;
2481	unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
2482	unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2483
2484	len = sprintf(buf: page, fmt: "%lu", secs);
2485	if (jifs)
2486	len += sprintf(buf: page+len, fmt: ".%03u", jiffies_to_msecs(j: jifs));
2487	len += sprintf(buf: page+len, fmt: "\n");
2488	return len;
2489	}
2490
2491	static ssize_t
2492	timeout_store(struct mddev *mddev, const char *buf, size_t len)
2493	{
2494	/* timeout can be set at any time */
2495	unsigned long timeout;
2496	int rv = strict_strtoul_scaled(cp: buf, res: &timeout, scale: 4);
2497	if (rv)
2498	return rv;
2499
2500	/* just to make sure we don't overflow... */
2501	if (timeout >= LONG_MAX / HZ)
2502	return -EINVAL;
2503
2504	timeout = timeout * HZ / 10000;
2505
2506	if (timeout >= MAX_SCHEDULE_TIMEOUT)
2507	timeout = MAX_SCHEDULE_TIMEOUT-1;
2508	if (timeout < 1)
2509	timeout = 1;
2510
2511	mddev->bitmap_info.daemon_sleep = timeout;
2512	mddev_set_timeout(mddev, timeout, force: false);
2513	md_wakeup_thread(thread: mddev->thread);
2514
2515	return len;
2516	}
2517
2518	static struct md_sysfs_entry bitmap_timeout =
2519	__ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);
2520
2521	static ssize_t
2522	backlog_show(struct mddev *mddev, char *page)
2523	{
2524	return sprintf(buf: page, fmt: "%lu\n", mddev->bitmap_info.max_write_behind);
2525	}
2526
2527	static ssize_t
2528	backlog_store(struct mddev *mddev, const char *buf, size_t len)
2529	{
2530	unsigned long backlog;
2531	unsigned long old_mwb = mddev->bitmap_info.max_write_behind;
2532	struct md_rdev *rdev;
2533	bool has_write_mostly = false;
2534	int rv = kstrtoul(s: buf, base: 10, res: &backlog);
2535	if (rv)
2536	return rv;
2537	if (backlog > COUNTER_MAX)
2538	return -EINVAL;
2539
2540	rv = mddev_suspend_and_lock(mddev);
2541	if (rv)
2542	return rv;
2543
2544	/*
2545	* Without write mostly device, it doesn't make sense to set
2546	* backlog for max_write_behind.
2547	*/
2548	rdev_for_each(rdev, mddev) {
2549	if (test_bit(WriteMostly, &rdev->flags)) {
2550	has_write_mostly = true;
2551	break;
2552	}
2553	}
2554	if (!has_write_mostly) {
2555	pr_warn_ratelimited("%s: can't set backlog, no write mostly device available\n",
2556	mdname(mddev));
2557	mddev_unlock(mddev);
2558	return -EINVAL;
2559	}
2560
2561	mddev->bitmap_info.max_write_behind = backlog;
2562	if (!backlog && mddev->serial_info_pool) {
2563	/* serial_info_pool is not needed if backlog is zero */
2564	if (!mddev->serialize_policy)
2565	mddev_destroy_serial_pool(mddev, NULL);
2566	} else if (backlog && !mddev->serial_info_pool) {
2567	/* serial_info_pool is needed since backlog is not zero */
2568	rdev_for_each(rdev, mddev)
2569	mddev_create_serial_pool(mddev, rdev);
2570	}
2571	if (old_mwb != backlog)
2572	md_bitmap_update_sb(mddev->bitmap);
2573
2574	mddev_unlock_and_resume(mddev);
2575	return len;
2576	}
2577
2578	static struct md_sysfs_entry bitmap_backlog =
2579	__ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);
2580
2581	static ssize_t
2582	chunksize_show(struct mddev *mddev, char *page)
2583	{
2584	return sprintf(buf: page, fmt: "%lu\n", mddev->bitmap_info.chunksize);
2585	}
2586
2587	static ssize_t
2588	chunksize_store(struct mddev *mddev, const char *buf, size_t len)
2589	{
2590	/* Can only be changed when no bitmap is active */
2591	int rv;
2592	unsigned long csize;
2593	if (mddev->bitmap)
2594	return -EBUSY;
2595	rv = kstrtoul(s: buf, base: 10, res: &csize);
2596	if (rv)
2597	return rv;
2598	if (csize < 512 ||
2599	!is_power_of_2(n: csize))
2600	return -EINVAL;
2601	if (BITS_PER_LONG > 32 && csize >= (1ULL << (BITS_PER_BYTE *
2602	sizeof(((bitmap_super_t *)0)->chunksize))))
2603	return -EOVERFLOW;
2604	mddev->bitmap_info.chunksize = csize;
2605	return len;
2606	}
2607
2608	static struct md_sysfs_entry bitmap_chunksize =
2609	__ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);
2610
2611	static ssize_t metadata_show(struct mddev *mddev, char *page)
2612	{
2613	if (mddev_is_clustered(mddev))
2614	return sprintf(buf: page, fmt: "clustered\n");
2615	return sprintf(buf: page, fmt: "%s\n", (mddev->bitmap_info.external
2616	? "external" : "internal"));
2617	}
2618
2619	static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
2620	{
2621	if (mddev->bitmap ||
2622	mddev->bitmap_info.file ||
2623	mddev->bitmap_info.offset)
2624	return -EBUSY;
2625	if (strncmp(buf, "external", 8) == 0)
2626	mddev->bitmap_info.external = 1;
2627	else if ((strncmp(buf, "internal", 8) == 0) ||
2628	(strncmp(buf, "clustered", 9) == 0))
2629	mddev->bitmap_info.external = 0;
2630	else
2631	return -EINVAL;
2632	return len;
2633	}
2634
2635	static struct md_sysfs_entry bitmap_metadata =
2636	__ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);
2637
2638	static ssize_t can_clear_show(struct mddev *mddev, char *page)
2639	{
2640	int len;
2641	spin_lock(lock: &mddev->lock);
2642	if (mddev->bitmap)
2643	len = sprintf(buf: page, fmt: "%s\n", (mddev->bitmap->need_sync ?
2644	"false" : "true"));
2645	else
2646	len = sprintf(buf: page, fmt: "\n");
2647	spin_unlock(lock: &mddev->lock);
2648	return len;
2649	}
2650
2651	static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
2652	{
2653	if (mddev->bitmap == NULL)
2654	return -ENOENT;
2655	if (strncmp(buf, "false", 5) == 0)
2656	mddev->bitmap->need_sync = 1;
2657	else if (strncmp(buf, "true", 4) == 0) {
2658	if (mddev->degraded)
2659	return -EBUSY;
2660	mddev->bitmap->need_sync = 0;
2661	} else
2662	return -EINVAL;
2663	return len;
2664	}
2665
2666	static struct md_sysfs_entry bitmap_can_clear =
2667	__ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
2668
2669	static ssize_t
2670	behind_writes_used_show(struct mddev *mddev, char *page)
2671	{
2672	ssize_t ret;
2673	spin_lock(lock: &mddev->lock);
2674	if (mddev->bitmap == NULL)
2675	ret = sprintf(buf: page, fmt: "0\n");
2676	else
2677	ret = sprintf(buf: page, fmt: "%lu\n",
2678	mddev->bitmap->behind_writes_used);
2679	spin_unlock(lock: &mddev->lock);
2680	return ret;
2681	}
2682
2683	static ssize_t
2684	behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
2685	{
2686	if (mddev->bitmap)
2687	mddev->bitmap->behind_writes_used = 0;
2688	return len;
2689	}
2690
2691	static struct md_sysfs_entry max_backlog_used =
2692	__ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
2693	behind_writes_used_show, behind_writes_used_reset);
2694
2695	static struct attribute *md_bitmap_attrs[] = {
2696	&bitmap_location.attr,
2697	&bitmap_space.attr,
2698	&bitmap_timeout.attr,
2699	&bitmap_backlog.attr,
2700	&bitmap_chunksize.attr,
2701	&bitmap_metadata.attr,
2702	&bitmap_can_clear.attr,
2703	&max_backlog_used.attr,
2704	NULL
2705	};
2706	const struct attribute_group md_bitmap_group = {
2707	.name = "bitmap",
2708	.attrs = md_bitmap_attrs,
2709	};
2710