1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* bcachefs setup/teardown code, and some metadata io - read a superblock and
4	* figure out what to do with it.
5	*
6	* Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
7	* Copyright 2012 Google, Inc.
8	*/
9
10	#include "bcachefs.h"
11	#include "alloc_background.h"
12	#include "alloc_foreground.h"
13	#include "bkey_sort.h"
14	#include "btree_cache.h"
15	#include "btree_gc.h"
16	#include "btree_journal_iter.h"
17	#include "btree_key_cache.h"
18	#include "btree_node_scan.h"
19	#include "btree_update_interior.h"
20	#include "btree_io.h"
21	#include "btree_write_buffer.h"
22	#include "buckets_waiting_for_journal.h"
23	#include "chardev.h"
24	#include "checksum.h"
25	#include "clock.h"
26	#include "compress.h"
27	#include "debug.h"
28	#include "disk_groups.h"
29	#include "ec.h"
30	#include "errcode.h"
31	#include "error.h"
32	#include "fs.h"
33	#include "fs-io.h"
34	#include "fs-io-buffered.h"
35	#include "fs-io-direct.h"
36	#include "fsck.h"
37	#include "inode.h"
38	#include "io_read.h"
39	#include "io_write.h"
40	#include "journal.h"
41	#include "journal_reclaim.h"
42	#include "journal_seq_blacklist.h"
43	#include "move.h"
44	#include "migrate.h"
45	#include "movinggc.h"
46	#include "nocow_locking.h"
47	#include "quota.h"
48	#include "rebalance.h"
49	#include "recovery.h"
50	#include "replicas.h"
51	#include "sb-clean.h"
52	#include "sb-counters.h"
53	#include "sb-errors.h"
54	#include "sb-members.h"
55	#include "snapshot.h"
56	#include "subvolume.h"
57	#include "super.h"
58	#include "super-io.h"
59	#include "sysfs.h"
60	#include "thread_with_file.h"
61	#include "trace.h"
62
63	#include <linux/backing-dev.h>
64	#include <linux/blkdev.h>
65	#include <linux/debugfs.h>
66	#include <linux/device.h>
67	#include <linux/idr.h>
68	#include <linux/module.h>
69	#include <linux/percpu.h>
70	#include <linux/random.h>
71	#include <linux/sysfs.h>
72	#include <crypto/hash.h>
73
74	MODULE_LICENSE("GPL");
75	MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>");
76	MODULE_DESCRIPTION("bcachefs filesystem");
77	MODULE_SOFTDEP("pre: crc32c");
78	MODULE_SOFTDEP("pre: crc64");
79	MODULE_SOFTDEP("pre: sha256");
80	MODULE_SOFTDEP("pre: chacha20");
81	MODULE_SOFTDEP("pre: poly1305");
82	MODULE_SOFTDEP("pre: xxhash");
83
84	const char * const bch2_fs_flag_strs[] = {
85	#define x(n) #n,
86	BCH_FS_FLAGS()
87	#undef x
88	NULL
89	};
90
91	__printf(2, 0)
92	static void bch2_print_maybe_redirect(struct stdio_redirect *stdio, const char *fmt, va_list args)
93	{
94	#ifdef __KERNEL__
95	if (unlikely(stdio)) {
96	if (fmt[0] == KERN_SOH[0])
97	fmt += 2;
98
99	bch2_stdio_redirect_vprintf(stdio, true, fmt, args);
100	return;
101	}
102	#endif
103	vprintk(fmt, args);
104	}
105
106	void bch2_print_opts(struct bch_opts *opts, const char *fmt, ...)
107	{
108	struct stdio_redirect *stdio = (void *)(unsigned long)opts->stdio;
109
110	va_list args;
111	va_start(args, fmt);
112	bch2_print_maybe_redirect(stdio, fmt, args);
113	va_end(args);
114	}
115
116	void __bch2_print(struct bch_fs *c, const char *fmt, ...)
117	{
118	struct stdio_redirect *stdio = bch2_fs_stdio_redirect(c);
119
120	va_list args;
121	va_start(args, fmt);
122	bch2_print_maybe_redirect(stdio, fmt, args);
123	va_end(args);
124	}
125
126	#define KTYPE(type) \
127	static const struct attribute_group type ## _group = { \
128	.attrs = type ## _files \
129	}; \
130	\
131	static const struct attribute_group *type ## _groups[] = { \
132	&type ## _group, \
133	NULL \
134	}; \
135	\
136	static const struct kobj_type type ## _ktype = { \
137	.release = type ## _release, \
138	.sysfs_ops = &type ## _sysfs_ops, \
139	.default_groups = type ## _groups \
140	}
141
142	static void bch2_fs_release(struct kobject *);
143	static void bch2_dev_release(struct kobject *);
144	static void bch2_fs_counters_release(struct kobject *k)
145	{
146	}
147
148	static void bch2_fs_internal_release(struct kobject *k)
149	{
150	}
151
152	static void bch2_fs_opts_dir_release(struct kobject *k)
153	{
154	}
155
156	static void bch2_fs_time_stats_release(struct kobject *k)
157	{
158	}
159
160	KTYPE(bch2_fs);
161	KTYPE(bch2_fs_counters);
162	KTYPE(bch2_fs_internal);
163	KTYPE(bch2_fs_opts_dir);
164	KTYPE(bch2_fs_time_stats);
165	KTYPE(bch2_dev);
166
167	static struct kset *bcachefs_kset;
168	static LIST_HEAD(bch_fs_list);
169	static DEFINE_MUTEX(bch_fs_list_lock);
170
171	DECLARE_WAIT_QUEUE_HEAD(bch2_read_only_wait);
172
173	static void bch2_dev_free(struct bch_dev *);
174	static int bch2_dev_alloc(struct bch_fs *, unsigned);
175	static int bch2_dev_sysfs_online(struct bch_fs *, struct bch_dev *);
176	static void __bch2_dev_read_only(struct bch_fs *, struct bch_dev *);
177
178	struct bch_fs *bch2_dev_to_fs(dev_t dev)
179	{
180	struct bch_fs *c;
181
182	mutex_lock(&bch_fs_list_lock);
183	rcu_read_lock();
184
185	list_for_each_entry(c, &bch_fs_list, list)
186	for_each_member_device_rcu(c, ca, NULL)
187	if (ca->disk_sb.bdev && ca->disk_sb.bdev->bd_dev == dev) {
188	closure_get(cl: &c->cl);
189	goto found;
190	}
191	c = NULL;
192	found:
193	rcu_read_unlock();
194	mutex_unlock(lock: &bch_fs_list_lock);
195
196	return c;
197	}
198
199	static struct bch_fs *__bch2_uuid_to_fs(__uuid_t uuid)
200	{
201	struct bch_fs *c;
202
203	lockdep_assert_held(&bch_fs_list_lock);
204
205	list_for_each_entry(c, &bch_fs_list, list)
206	if (!memcmp(p: &c->disk_sb.sb->uuid, q: &uuid, size: sizeof(uuid)))
207	return c;
208
209	return NULL;
210	}
211
212	struct bch_fs *bch2_uuid_to_fs(__uuid_t uuid)
213	{
214	struct bch_fs *c;
215
216	mutex_lock(&bch_fs_list_lock);
217	c = __bch2_uuid_to_fs(uuid);
218	if (c)
219	closure_get(cl: &c->cl);
220	mutex_unlock(lock: &bch_fs_list_lock);
221
222	return c;
223	}
224
225	static void bch2_dev_usage_journal_reserve(struct bch_fs *c)
226	{
227	unsigned nr = 0, u64s =
228	((sizeof(struct jset_entry_dev_usage) +
229	sizeof(struct jset_entry_dev_usage_type) * BCH_DATA_NR)) /
230	sizeof(u64);
231
232	rcu_read_lock();
233	for_each_member_device_rcu(c, ca, NULL)
234	nr++;
235	rcu_read_unlock();
236
237	bch2_journal_entry_res_resize(&c->journal,
238	&c->dev_usage_journal_res, u64s * nr);
239	}
240
241	/* Filesystem RO/RW: */
242
243	/*
244	* For startup/shutdown of RW stuff, the dependencies are:
245	*
246	* - foreground writes depend on copygc and rebalance (to free up space)
247	*
248	* - copygc and rebalance depend on mark and sweep gc (they actually probably
249	* don't because they either reserve ahead of time or don't block if
250	* allocations fail, but allocations can require mark and sweep gc to run
251	* because of generation number wraparound)
252	*
253	* - all of the above depends on the allocator threads
254	*
255	* - allocator depends on the journal (when it rewrites prios and gens)
256	*/
257
258	static void __bch2_fs_read_only(struct bch_fs *c)
259	{
260	unsigned clean_passes = 0;
261	u64 seq = 0;
262
263	bch2_fs_ec_stop(c);
264	bch2_open_buckets_stop(c, NULL, true);
265	bch2_rebalance_stop(c);
266	bch2_copygc_stop(c);
267	bch2_gc_thread_stop(c);
268	bch2_fs_ec_flush(c);
269
270	bch_verbose(c, "flushing journal and stopping allocators, journal seq %llu",
271	journal_cur_seq(&c->journal));
272
273	do {
274	clean_passes++;
275
276	if (bch2_btree_interior_updates_flush(c) ||
277	bch2_journal_flush_all_pins(j: &c->journal) ||
278	bch2_btree_flush_all_writes(c) ||
279	seq != atomic64_read(v: &c->journal.seq)) {
280	seq = atomic64_read(v: &c->journal.seq);
281	clean_passes = 0;
282	}
283	} while (clean_passes < 2);
284
285	bch_verbose(c, "flushing journal and stopping allocators complete, journal seq %llu",
286	journal_cur_seq(&c->journal));
287
288	if (test_bit(JOURNAL_REPLAY_DONE, &c->journal.flags) &&
289	!test_bit(BCH_FS_emergency_ro, &c->flags))
290	set_bit(nr: BCH_FS_clean_shutdown, addr: &c->flags);
291
292	bch2_fs_journal_stop(&c->journal);
293
294	bch_info(c, "%sshutdown complete, journal seq %llu",
295	test_bit(BCH_FS_clean_shutdown, &c->flags) ? "" : "un",
296	c->journal.seq_ondisk);
297
298	/*
299	* After stopping journal:
300	*/
301	for_each_member_device(c, ca)
302	bch2_dev_allocator_remove(c, ca);
303	}
304
305	#ifndef BCH_WRITE_REF_DEBUG
306	static void bch2_writes_disabled(struct percpu_ref *writes)
307	{
308	struct bch_fs *c = container_of(writes, struct bch_fs, writes);
309
310	set_bit(BCH_FS_write_disable_complete, &c->flags);
311	wake_up(&bch2_read_only_wait);
312	}
313	#endif
314
315	void bch2_fs_read_only(struct bch_fs *c)
316	{
317	if (!test_bit(BCH_FS_rw, &c->flags)) {
318	bch2_journal_reclaim_stop(&c->journal);
319	return;
320	}
321
322	BUG_ON(test_bit(BCH_FS_write_disable_complete, &c->flags));
323
324	bch_verbose(c, "going read-only");
325
326	/*
327	* Block new foreground-end write operations from starting - any new
328	* writes will return -EROFS:
329	*/
330	set_bit(nr: BCH_FS_going_ro, addr: &c->flags);
331	#ifndef BCH_WRITE_REF_DEBUG
332	percpu_ref_kill(&c->writes);
333	#else
334	for (unsigned i = 0; i < BCH_WRITE_REF_NR; i++)
335	bch2_write_ref_put(c, ref: i);
336	#endif
337
338	/*
339	* If we're not doing an emergency shutdown, we want to wait on
340	* outstanding writes to complete so they don't see spurious errors due
341	* to shutting down the allocator:
342	*
343	* If we are doing an emergency shutdown outstanding writes may
344	* hang until we shutdown the allocator so we don't want to wait
345	* on outstanding writes before shutting everything down - but
346	* we do need to wait on them before returning and signalling
347	* that going RO is complete:
348	*/
349	wait_event(bch2_read_only_wait,
350	test_bit(BCH_FS_write_disable_complete, &c->flags) ||
351	test_bit(BCH_FS_emergency_ro, &c->flags));
352
353	bool writes_disabled = test_bit(BCH_FS_write_disable_complete, &c->flags);
354	if (writes_disabled)
355	bch_verbose(c, "finished waiting for writes to stop");
356
357	__bch2_fs_read_only(c);
358
359	wait_event(bch2_read_only_wait,
360	test_bit(BCH_FS_write_disable_complete, &c->flags));
361
362	if (!writes_disabled)
363	bch_verbose(c, "finished waiting for writes to stop");
364
365	clear_bit(nr: BCH_FS_write_disable_complete, addr: &c->flags);
366	clear_bit(nr: BCH_FS_going_ro, addr: &c->flags);
367	clear_bit(nr: BCH_FS_rw, addr: &c->flags);
368
369	if (!bch2_journal_error(j: &c->journal) &&
370	!test_bit(BCH_FS_error, &c->flags) &&
371	!test_bit(BCH_FS_emergency_ro, &c->flags) &&
372	test_bit(BCH_FS_started, &c->flags) &&
373	test_bit(BCH_FS_clean_shutdown, &c->flags) &&
374	c->recovery_pass_done >= BCH_RECOVERY_PASS_journal_replay) {
375	BUG_ON(c->journal.last_empty_seq != journal_cur_seq(&c->journal));
376	BUG_ON(atomic_read(&c->btree_cache.dirty));
377	BUG_ON(atomic_long_read(&c->btree_key_cache.nr_dirty));
378	BUG_ON(c->btree_write_buffer.inc.keys.nr);
379	BUG_ON(c->btree_write_buffer.flushing.keys.nr);
380
381	bch_verbose(c, "marking filesystem clean");
382	bch2_fs_mark_clean(c);
383	} else {
384	bch_verbose(c, "done going read-only, filesystem not clean");
385	}
386	}
387
388	static void bch2_fs_read_only_work(struct work_struct *work)
389	{
390	struct bch_fs *c =
391	container_of(work, struct bch_fs, read_only_work);
392
393	down_write(sem: &c->state_lock);
394	bch2_fs_read_only(c);
395	up_write(sem: &c->state_lock);
396	}
397
398	static void bch2_fs_read_only_async(struct bch_fs *c)
399	{
400	queue_work(wq: system_long_wq, work: &c->read_only_work);
401	}
402
403	bool bch2_fs_emergency_read_only(struct bch_fs *c)
404	{
405	bool ret = !test_and_set_bit(nr: BCH_FS_emergency_ro, addr: &c->flags);
406
407	bch2_journal_halt(&c->journal);
408	bch2_fs_read_only_async(c);
409
410	wake_up(&bch2_read_only_wait);
411	return ret;
412	}
413
414	static int bch2_fs_read_write_late(struct bch_fs *c)
415	{
416	int ret;
417
418	/*
419	* Data move operations can't run until after check_snapshots has
420	* completed, and bch2_snapshot_is_ancestor() is available.
421	*
422	* Ideally we'd start copygc/rebalance earlier instead of waiting for
423	* all of recovery/fsck to complete:
424	*/
425	ret = bch2_copygc_start(c);
426	if (ret) {
427	bch_err(c, "error starting copygc thread");
428	return ret;
429	}
430
431	ret = bch2_rebalance_start(c);
432	if (ret) {
433	bch_err(c, "error starting rebalance thread");
434	return ret;
435	}
436
437	return 0;
438	}
439
440	static int __bch2_fs_read_write(struct bch_fs *c, bool early)
441	{
442	int ret;
443
444	if (test_bit(BCH_FS_initial_gc_unfixed, &c->flags)) {
445	bch_err(c, "cannot go rw, unfixed btree errors");
446	return -BCH_ERR_erofs_unfixed_errors;
447	}
448
449	if (test_bit(BCH_FS_rw, &c->flags))
450	return 0;
451
452	bch_info(c, "going read-write");
453
454	ret = bch2_sb_members_v2_init(c);
455	if (ret)
456	goto err;
457
458	ret = bch2_fs_mark_dirty(c);
459	if (ret)
460	goto err;
461
462	clear_bit(nr: BCH_FS_clean_shutdown, addr: &c->flags);
463
464	/*
465	* First journal write must be a flush write: after a clean shutdown we
466	* don't read the journal, so the first journal write may end up
467	* overwriting whatever was there previously, and there must always be
468	* at least one non-flush write in the journal or recovery will fail:
469	*/
470	set_bit(nr: JOURNAL_NEED_FLUSH_WRITE, addr: &c->journal.flags);
471
472	for_each_rw_member(c, ca)
473	bch2_dev_allocator_add(c, ca);
474	bch2_recalc_capacity(c);
475
476	set_bit(nr: BCH_FS_rw, addr: &c->flags);
477	set_bit(nr: BCH_FS_was_rw, addr: &c->flags);
478
479	#ifndef BCH_WRITE_REF_DEBUG
480	percpu_ref_reinit(&c->writes);
481	#else
482	for (unsigned i = 0; i < BCH_WRITE_REF_NR; i++) {
483	BUG_ON(atomic_long_read(&c->writes[i]));
484	atomic_long_inc(v: &c->writes[i]);
485	}
486	#endif
487
488	ret = bch2_gc_thread_start(c);
489	if (ret) {
490	bch_err(c, "error starting gc thread");
491	return ret;
492	}
493
494	ret = bch2_journal_reclaim_start(&c->journal);
495	if (ret)
496	goto err;
497
498	if (!early) {
499	ret = bch2_fs_read_write_late(c);
500	if (ret)
501	goto err;
502	}
503
504	bch2_do_discards(c);
505	bch2_do_invalidates(c);
506	bch2_do_stripe_deletes(c);
507	bch2_do_pending_node_rewrites(c);
508	return 0;
509	err:
510	if (test_bit(BCH_FS_rw, &c->flags))
511	bch2_fs_read_only(c);
512	else
513	__bch2_fs_read_only(c);
514	return ret;
515	}
516
517	int bch2_fs_read_write(struct bch_fs *c)
518	{
519	if (c->opts.recovery_pass_last &&
520	c->opts.recovery_pass_last < BCH_RECOVERY_PASS_journal_replay)
521	return -BCH_ERR_erofs_norecovery;
522
523	if (c->opts.nochanges)
524	return -BCH_ERR_erofs_nochanges;
525
526	return __bch2_fs_read_write(c, early: false);
527	}
528
529	int bch2_fs_read_write_early(struct bch_fs *c)
530	{
531	lockdep_assert_held(&c->state_lock);
532
533	return __bch2_fs_read_write(c, early: true);
534	}
535
536	/* Filesystem startup/shutdown: */
537
538	static void __bch2_fs_free(struct bch_fs *c)
539	{
540	unsigned i;
541
542	for (i = 0; i < BCH_TIME_STAT_NR; i++)
543	bch2_time_stats_exit(&c->times[i]);
544
545	bch2_find_btree_nodes_exit(&c->found_btree_nodes);
546	bch2_free_pending_node_rewrites(c);
547	bch2_fs_allocator_background_exit(c);
548	bch2_fs_sb_errors_exit(c);
549	bch2_fs_counters_exit(c);
550	bch2_fs_snapshots_exit(c);
551	bch2_fs_quota_exit(c);
552	bch2_fs_fs_io_direct_exit(c);
553	bch2_fs_fs_io_buffered_exit(c);
554	bch2_fs_fsio_exit(c);
555	bch2_fs_ec_exit(c);
556	bch2_fs_encryption_exit(c);
557	bch2_fs_nocow_locking_exit(c);
558	bch2_fs_io_write_exit(c);
559	bch2_fs_io_read_exit(c);
560	bch2_fs_buckets_waiting_for_journal_exit(c);
561	bch2_fs_btree_interior_update_exit(c);
562	bch2_fs_btree_iter_exit(c);
563	bch2_fs_btree_key_cache_exit(&c->btree_key_cache);
564	bch2_fs_btree_cache_exit(c);
565	bch2_fs_replicas_exit(c);
566	bch2_fs_journal_exit(&c->journal);
567	bch2_io_clock_exit(&c->io_clock[WRITE]);
568	bch2_io_clock_exit(&c->io_clock[READ]);
569	bch2_fs_compress_exit(c);
570	bch2_journal_keys_put_initial(c);
571	bch2_find_btree_nodes_exit(&c->found_btree_nodes);
572	BUG_ON(atomic_read(&c->journal_keys.ref));
573	bch2_fs_btree_write_buffer_exit(c);
574	percpu_free_rwsem(&c->mark_lock);
575	free_percpu(pdata: c->online_reserved);
576
577	darray_exit(&c->btree_roots_extra);
578	free_percpu(pdata: c->pcpu);
579	mempool_exit(pool: &c->large_bkey_pool);
580	mempool_exit(pool: &c->btree_bounce_pool);
581	bioset_exit(&c->btree_bio);
582	mempool_exit(pool: &c->fill_iter);
583	#ifndef BCH_WRITE_REF_DEBUG
584	percpu_ref_exit(&c->writes);
585	#endif
586	kfree(rcu_dereference_protected(c->disk_groups, 1));
587	kfree(objp: c->journal_seq_blacklist_table);
588	kfree(objp: c->unused_inode_hints);
589
590	if (c->write_ref_wq)
591	destroy_workqueue(wq: c->write_ref_wq);
592	if (c->io_complete_wq)
593	destroy_workqueue(wq: c->io_complete_wq);
594	if (c->copygc_wq)
595	destroy_workqueue(wq: c->copygc_wq);
596	if (c->btree_io_complete_wq)
597	destroy_workqueue(wq: c->btree_io_complete_wq);
598	if (c->btree_update_wq)
599	destroy_workqueue(wq: c->btree_update_wq);
600
601	bch2_free_super(&c->disk_sb);
602	kvfree(addr: c);
603	module_put(THIS_MODULE);
604	}
605
606	static void bch2_fs_release(struct kobject *kobj)
607	{
608	struct bch_fs *c = container_of(kobj, struct bch_fs, kobj);
609
610	__bch2_fs_free(c);
611	}
612
613	void __bch2_fs_stop(struct bch_fs *c)
614	{
615	bch_verbose(c, "shutting down");
616
617	set_bit(nr: BCH_FS_stopping, addr: &c->flags);
618
619	cancel_work_sync(work: &c->journal_seq_blacklist_gc_work);
620
621	down_write(sem: &c->state_lock);
622	bch2_fs_read_only(c);
623	up_write(sem: &c->state_lock);
624
625	for_each_member_device(c, ca)
626	if (ca->kobj.state_in_sysfs &&
627	ca->disk_sb.bdev)
628	sysfs_remove_link(bdev_kobj(ca->disk_sb.bdev), name: "bcachefs");
629
630	if (c->kobj.state_in_sysfs)
631	kobject_del(kobj: &c->kobj);
632
633	bch2_fs_debug_exit(c);
634	bch2_fs_chardev_exit(c);
635
636	bch2_ro_ref_put(c);
637	wait_event(c->ro_ref_wait, !refcount_read(&c->ro_ref));
638
639	kobject_put(kobj: &c->counters_kobj);
640	kobject_put(kobj: &c->time_stats);
641	kobject_put(kobj: &c->opts_dir);
642	kobject_put(kobj: &c->internal);
643
644	/* btree prefetch might have kicked off reads in the background: */
645	bch2_btree_flush_all_reads(c);
646
647	for_each_member_device(c, ca)
648	cancel_work_sync(work: &ca->io_error_work);
649
650	cancel_work_sync(work: &c->read_only_work);
651	}
652
653	void bch2_fs_free(struct bch_fs *c)
654	{
655	unsigned i;
656
657	mutex_lock(&bch_fs_list_lock);
658	list_del(entry: &c->list);
659	mutex_unlock(lock: &bch_fs_list_lock);
660
661	closure_sync(cl: &c->cl);
662	closure_debug_destroy(cl: &c->cl);
663
664	for (i = 0; i < c->sb.nr_devices; i++) {
665	struct bch_dev *ca = rcu_dereference_protected(c->devs[i], true);
666
667	if (ca) {
668	bch2_free_super(&ca->disk_sb);
669	bch2_dev_free(ca);
670	}
671	}
672
673	bch_verbose(c, "shutdown complete");
674
675	kobject_put(kobj: &c->kobj);
676	}
677
678	void bch2_fs_stop(struct bch_fs *c)
679	{
680	__bch2_fs_stop(c);
681	bch2_fs_free(c);
682	}
683
684	static int bch2_fs_online(struct bch_fs *c)
685	{
686	int ret = 0;
687
688	lockdep_assert_held(&bch_fs_list_lock);
689
690	if (__bch2_uuid_to_fs(uuid: c->sb.uuid)) {
691	bch_err(c, "filesystem UUID already open");
692	return -EINVAL;
693	}
694
695	ret = bch2_fs_chardev_init(c);
696	if (ret) {
697	bch_err(c, "error creating character device");
698	return ret;
699	}
700
701	bch2_fs_debug_init(c);
702
703	ret = kobject_add(kobj: &c->kobj, NULL, fmt: "%pU", c->sb.user_uuid.b) ?:
704	kobject_add(kobj: &c->internal, parent: &c->kobj, fmt: "internal") ?:
705	kobject_add(kobj: &c->opts_dir, parent: &c->kobj, fmt: "options") ?:
706	#ifndef CONFIG_BCACHEFS_NO_LATENCY_ACCT
707	kobject_add(&c->time_stats, &c->kobj, "time_stats") ?:
708	#endif
709	kobject_add(kobj: &c->counters_kobj, parent: &c->kobj, fmt: "counters") ?:
710	bch2_opts_create_sysfs_files(&c->opts_dir);
711	if (ret) {
712	bch_err(c, "error creating sysfs objects");
713	return ret;
714	}
715
716	down_write(sem: &c->state_lock);
717
718	for_each_member_device(c, ca) {
719	ret = bch2_dev_sysfs_online(c, ca);
720	if (ret) {
721	bch_err(c, "error creating sysfs objects");
722	percpu_ref_put(ref: &ca->ref);
723	goto err;
724	}
725	}
726
727	BUG_ON(!list_empty(&c->list));
728	list_add(new: &c->list, head: &bch_fs_list);
729	err:
730	up_write(sem: &c->state_lock);
731	return ret;
732	}
733
734	static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
735	{
736	struct bch_fs *c;
737	struct printbuf name = PRINTBUF;
738	unsigned i, iter_size;
739	int ret = 0;
740
741	c = kvmalloc(size: sizeof(struct bch_fs), GFP_KERNEL|__GFP_ZERO);
742	if (!c) {
743	c = ERR_PTR(error: -BCH_ERR_ENOMEM_fs_alloc);
744	goto out;
745	}
746
747	c->stdio = (void *)(unsigned long) opts.stdio;
748
749	__module_get(THIS_MODULE);
750
751	closure_init(cl: &c->cl, NULL);
752
753	c->kobj.kset = bcachefs_kset;
754	kobject_init(kobj: &c->kobj, ktype: &bch2_fs_ktype);
755	kobject_init(kobj: &c->internal, ktype: &bch2_fs_internal_ktype);
756	kobject_init(kobj: &c->opts_dir, ktype: &bch2_fs_opts_dir_ktype);
757	kobject_init(kobj: &c->time_stats, ktype: &bch2_fs_time_stats_ktype);
758	kobject_init(kobj: &c->counters_kobj, ktype: &bch2_fs_counters_ktype);
759
760	c->minor = -1;
761	c->disk_sb.fs_sb = true;
762
763	init_rwsem(&c->state_lock);
764	mutex_init(&c->sb_lock);
765	mutex_init(&c->replicas_gc_lock);
766	mutex_init(&c->btree_root_lock);
767	INIT_WORK(&c->read_only_work, bch2_fs_read_only_work);
768
769	refcount_set(r: &c->ro_ref, n: 1);
770	init_waitqueue_head(&c->ro_ref_wait);
771	sema_init(sem: &c->online_fsck_mutex, val: 1);
772
773	init_rwsem(&c->gc_lock);
774	mutex_init(&c->gc_gens_lock);
775	atomic_set(v: &c->journal_keys.ref, i: 1);
776	c->journal_keys.initial_ref_held = true;
777
778	for (i = 0; i < BCH_TIME_STAT_NR; i++)
779	bch2_time_stats_init(&c->times[i]);
780
781	bch2_fs_copygc_init(c);
782	bch2_fs_btree_key_cache_init_early(&c->btree_key_cache);
783	bch2_fs_btree_iter_init_early(c);
784	bch2_fs_btree_interior_update_init_early(c);
785	bch2_fs_allocator_background_init(c);
786	bch2_fs_allocator_foreground_init(c);
787	bch2_fs_rebalance_init(c);
788	bch2_fs_quota_init(c);
789	bch2_fs_ec_init_early(c);
790	bch2_fs_move_init(c);
791	bch2_fs_sb_errors_init_early(c);
792
793	INIT_LIST_HEAD(list: &c->list);
794
795	mutex_init(&c->usage_scratch_lock);
796
797	mutex_init(&c->bio_bounce_pages_lock);
798	mutex_init(&c->snapshot_table_lock);
799	init_rwsem(&c->snapshot_create_lock);
800
801	spin_lock_init(&c->btree_write_error_lock);
802
803	INIT_WORK(&c->journal_seq_blacklist_gc_work,
804	bch2_blacklist_entries_gc);
805
806	INIT_LIST_HEAD(list: &c->journal_iters);
807
808	INIT_LIST_HEAD(list: &c->fsck_error_msgs);
809	mutex_init(&c->fsck_error_msgs_lock);
810
811	seqcount_init(&c->gc_pos_lock);
812
813	seqcount_init(&c->usage_lock);
814
815	sema_init(sem: &c->io_in_flight, val: 128);
816
817	INIT_LIST_HEAD(list: &c->vfs_inodes_list);
818	mutex_init(&c->vfs_inodes_lock);
819
820	c->copy_gc_enabled = 1;
821	c->rebalance.enabled = 1;
822	c->promote_whole_extents = true;
823
824	c->journal.flush_write_time = &c->times[BCH_TIME_journal_flush_write];
825	c->journal.noflush_write_time = &c->times[BCH_TIME_journal_noflush_write];
826	c->journal.flush_seq_time = &c->times[BCH_TIME_journal_flush_seq];
827
828	bch2_fs_btree_cache_init_early(&c->btree_cache);
829
830	mutex_init(&c->sectors_available_lock);
831
832	ret = percpu_init_rwsem(&c->mark_lock);
833	if (ret)
834	goto err;
835
836	mutex_lock(&c->sb_lock);
837	ret = bch2_sb_to_fs(c, sb);
838	mutex_unlock(lock: &c->sb_lock);
839
840	if (ret)
841	goto err;
842
843	pr_uuid(out: &name, uuid: c->sb.user_uuid.b);
844	ret = name.allocation_failure ? -BCH_ERR_ENOMEM_fs_name_alloc : 0;
845	if (ret)
846	goto err;
847
848	strscpy(c->name, name.buf, sizeof(c->name));
849	printbuf_exit(&name);
850
851	/* Compat: */
852	if (le16_to_cpu(sb->version) <= bcachefs_metadata_version_inode_v2 &&
853	!BCH_SB_JOURNAL_FLUSH_DELAY(k: sb))
854	SET_BCH_SB_JOURNAL_FLUSH_DELAY(k: sb, v: 1000);
855
856	if (le16_to_cpu(sb->version) <= bcachefs_metadata_version_inode_v2 &&
857	!BCH_SB_JOURNAL_RECLAIM_DELAY(k: sb))
858	SET_BCH_SB_JOURNAL_RECLAIM_DELAY(k: sb, v: 100);
859
860	c->opts = bch2_opts_default;
861	ret = bch2_opts_from_sb(&c->opts, sb);
862	if (ret)
863	goto err;
864
865	bch2_opts_apply(&c->opts, opts);
866
867	c->btree_key_cache_btrees |= 1U << BTREE_ID_alloc;
868	if (c->opts.inodes_use_key_cache)
869	c->btree_key_cache_btrees |= 1U << BTREE_ID_inodes;
870	c->btree_key_cache_btrees |= 1U << BTREE_ID_logged_ops;
871
872	c->block_bits = ilog2(block_sectors(c));
873	c->btree_foreground_merge_threshold = BTREE_FOREGROUND_MERGE_THRESHOLD(c);
874
875	if (bch2_fs_init_fault("fs_alloc")) {
876	bch_err(c, "fs_alloc fault injected");
877	ret = -EFAULT;
878	goto err;
879	}
880
881	iter_size = sizeof(struct sort_iter) +
882	(btree_blocks(c) + 1) * 2 *
883	sizeof(struct sort_iter_set);
884
885	c->inode_shard_bits = ilog2(roundup_pow_of_two(num_possible_cpus()));
886
887	if (!(c->btree_update_wq = alloc_workqueue(fmt: "bcachefs",
888	flags: WQ_HIGHPRI|WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_UNBOUND, max_active: 512)) ||
889	!(c->btree_io_complete_wq = alloc_workqueue(fmt: "bcachefs_btree_io",
890	flags: WQ_HIGHPRI|WQ_FREEZABLE|WQ_MEM_RECLAIM, max_active: 1)) ||
891	!(c->copygc_wq = alloc_workqueue(fmt: "bcachefs_copygc",
892	flags: WQ_HIGHPRI|WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, max_active: 1)) ||
893	!(c->io_complete_wq = alloc_workqueue(fmt: "bcachefs_io",
894	flags: WQ_HIGHPRI|WQ_FREEZABLE|WQ_MEM_RECLAIM, max_active: 512)) ||
895	!(c->write_ref_wq = alloc_workqueue(fmt: "bcachefs_write_ref",
896	flags: WQ_FREEZABLE, max_active: 0)) ||
897	#ifndef BCH_WRITE_REF_DEBUG
898	percpu_ref_init(&c->writes, bch2_writes_disabled,
899	PERCPU_REF_INIT_DEAD, GFP_KERNEL) ||
900	#endif
901	mempool_init_kmalloc_pool(pool: &c->fill_iter, min_nr: 1, size: iter_size) ||
902	bioset_init(&c->btree_bio, 1,
903	max(offsetof(struct btree_read_bio, bio),
904	offsetof(struct btree_write_bio, wbio.bio)),
905	flags: BIOSET_NEED_BVECS) ||
906	!(c->pcpu = alloc_percpu(struct bch_fs_pcpu)) ||
907	!(c->online_reserved = alloc_percpu(u64)) ||
908	mempool_init_kvmalloc_pool(pool: &c->btree_bounce_pool, min_nr: 1,
909	size: c->opts.btree_node_size) ||
910	mempool_init_kmalloc_pool(pool: &c->large_bkey_pool, min_nr: 1, size: 2048) ||
911	!(c->unused_inode_hints = kcalloc(n: 1U << c->inode_shard_bits,
912	size: sizeof(u64), GFP_KERNEL))) {
913	ret = -BCH_ERR_ENOMEM_fs_other_alloc;
914	goto err;
915	}
916
917	ret = bch2_fs_counters_init(c) ?:
918	bch2_fs_sb_errors_init(c) ?:
919	bch2_io_clock_init(&c->io_clock[READ]) ?:
920	bch2_io_clock_init(&c->io_clock[WRITE]) ?:
921	bch2_fs_journal_init(&c->journal) ?:
922	bch2_fs_replicas_init(c) ?:
923	bch2_fs_btree_cache_init(c) ?:
924	bch2_fs_btree_key_cache_init(&c->btree_key_cache) ?:
925	bch2_fs_btree_iter_init(c) ?:
926	bch2_fs_btree_interior_update_init(c) ?:
927	bch2_fs_buckets_waiting_for_journal_init(c) ?:
928	bch2_fs_btree_write_buffer_init(c) ?:
929	bch2_fs_subvolumes_init(c) ?:
930	bch2_fs_io_read_init(c) ?:
931	bch2_fs_io_write_init(c) ?:
932	bch2_fs_nocow_locking_init(c) ?:
933	bch2_fs_encryption_init(c) ?:
934	bch2_fs_compress_init(c) ?:
935	bch2_fs_ec_init(c) ?:
936	bch2_fs_fsio_init(c) ?:
937	bch2_fs_fs_io_buffered_init(c) ?:
938	bch2_fs_fs_io_direct_init(c);
939	if (ret)
940	goto err;
941
942	for (i = 0; i < c->sb.nr_devices; i++)
943	if (bch2_dev_exists(sb: c->disk_sb.sb, dev: i) &&
944	bch2_dev_alloc(c, i)) {
945	ret = -EEXIST;
946	goto err;
947	}
948
949	bch2_journal_entry_res_resize(&c->journal,
950	&c->btree_root_journal_res,
951	BTREE_ID_NR * (JSET_KEYS_U64s + BKEY_BTREE_PTR_U64s_MAX));
952	bch2_dev_usage_journal_reserve(c);
953	bch2_journal_entry_res_resize(&c->journal,
954	&c->clock_journal_res,
955	(sizeof(struct jset_entry_clock) / sizeof(u64)) * 2);
956
957	mutex_lock(&bch_fs_list_lock);
958	ret = bch2_fs_online(c);
959	mutex_unlock(lock: &bch_fs_list_lock);
960
961	if (ret)
962	goto err;
963	out:
964	return c;
965	err:
966	bch2_fs_free(c);
967	c = ERR_PTR(error: ret);
968	goto out;
969	}
970
971	noinline_for_stack
972	static void print_mount_opts(struct bch_fs *c)
973	{
974	enum bch_opt_id i;
975	struct printbuf p = PRINTBUF;
976	bool first = true;
977
978	prt_str(out: &p, str: "mounting version ");
979	bch2_version_to_text(&p, c->sb.version);
980
981	if (c->opts.read_only) {
982	prt_str(out: &p, str: " opts=");
983	first = false;
984	prt_printf(&p, "ro");
985	}
986
987	for (i = 0; i < bch2_opts_nr; i++) {
988	const struct bch_option *opt = &bch2_opt_table[i];
989	u64 v = bch2_opt_get_by_id(&c->opts, i);
990
991	if (!(opt->flags & OPT_MOUNT))
992	continue;
993
994	if (v == bch2_opt_get_by_id(&bch2_opts_default, i))
995	continue;
996
997	prt_str(out: &p, str: first ? " opts=" : ",");
998	first = false;
999	bch2_opt_to_text(&p, c, c->disk_sb.sb, opt, v, OPT_SHOW_MOUNT_STYLE);
1000	}
1001
1002	bch_info(c, "%s", p.buf);
1003	printbuf_exit(&p);
1004	}
1005
1006	int bch2_fs_start(struct bch_fs *c)
1007	{
1008	time64_t now = ktime_get_real_seconds();
1009	int ret;
1010
1011	print_mount_opts(c);
1012
1013	down_write(sem: &c->state_lock);
1014
1015	BUG_ON(test_bit(BCH_FS_started, &c->flags));
1016
1017	mutex_lock(&c->sb_lock);
1018
1019	ret = bch2_sb_members_v2_init(c);
1020	if (ret) {
1021	mutex_unlock(lock: &c->sb_lock);
1022	goto err;
1023	}
1024
1025	for_each_online_member(c, ca)
1026	bch2_members_v2_get_mut(sb: c->disk_sb.sb, i: ca->dev_idx)->last_mount = cpu_to_le64(now);
1027
1028	struct bch_sb_field_ext *ext =
1029	bch2_sb_field_get_minsize(&c->disk_sb, ext, sizeof(*ext) / sizeof(u64));
1030	mutex_unlock(lock: &c->sb_lock);
1031
1032	if (!ext) {
1033	bch_err(c, "insufficient space in superblock for sb_field_ext");
1034	ret = -BCH_ERR_ENOSPC_sb;
1035	goto err;
1036	}
1037
1038	for_each_rw_member(c, ca)
1039	bch2_dev_allocator_add(c, ca);
1040	bch2_recalc_capacity(c);
1041
1042	ret = BCH_SB_INITIALIZED(k: c->disk_sb.sb)
1043	? bch2_fs_recovery(c)
1044	: bch2_fs_initialize(c);
1045	if (ret)
1046	goto err;
1047
1048	ret = bch2_opts_check_may_set(c);
1049	if (ret)
1050	goto err;
1051
1052	if (bch2_fs_init_fault("fs_start")) {
1053	bch_err(c, "fs_start fault injected");
1054	ret = -EINVAL;
1055	goto err;
1056	}
1057
1058	set_bit(nr: BCH_FS_started, addr: &c->flags);
1059
1060	if (c->opts.read_only) {
1061	bch2_fs_read_only(c);
1062	} else {
1063	ret = !test_bit(BCH_FS_rw, &c->flags)
1064	? bch2_fs_read_write(c)
1065	: bch2_fs_read_write_late(c);
1066	if (ret)
1067	goto err;
1068	}
1069
1070	ret = 0;
1071	err:
1072	if (ret)
1073	bch_err_msg(c, ret, "starting filesystem");
1074	else
1075	bch_verbose(c, "done starting filesystem");
1076	up_write(sem: &c->state_lock);
1077	return ret;
1078	}
1079
1080	static int bch2_dev_may_add(struct bch_sb *sb, struct bch_fs *c)
1081	{
1082	struct bch_member m = bch2_sb_member_get(sb, i: sb->dev_idx);
1083
1084	if (le16_to_cpu(sb->block_size) != block_sectors(c))
1085	return -BCH_ERR_mismatched_block_size;
1086
1087	if (le16_to_cpu(m.bucket_size) <
1088	BCH_SB_BTREE_NODE_SIZE(k: c->disk_sb.sb))
1089	return -BCH_ERR_bucket_size_too_small;
1090
1091	return 0;
1092	}
1093
1094	static int bch2_dev_in_fs(struct bch_sb_handle *fs,
1095	struct bch_sb_handle *sb,
1096	struct bch_opts *opts)
1097	{
1098	if (fs == sb)
1099	return 0;
1100
1101	if (!uuid_equal(u1: &fs->sb->uuid, u2: &sb->sb->uuid))
1102	return -BCH_ERR_device_not_a_member_of_filesystem;
1103
1104	if (!bch2_dev_exists(sb: fs->sb, dev: sb->sb->dev_idx))
1105	return -BCH_ERR_device_has_been_removed;
1106
1107	if (fs->sb->block_size != sb->sb->block_size)
1108	return -BCH_ERR_mismatched_block_size;
1109
1110	if (le16_to_cpu(fs->sb->version) < bcachefs_metadata_version_member_seq ||
1111	le16_to_cpu(sb->sb->version) < bcachefs_metadata_version_member_seq)
1112	return 0;
1113
1114	if (fs->sb->seq == sb->sb->seq &&
1115	fs->sb->write_time != sb->sb->write_time) {
1116	struct printbuf buf = PRINTBUF;
1117
1118	prt_str(out: &buf, str: "Split brain detected between ");
1119	prt_bdevname(out: &buf, bdev: sb->bdev);
1120	prt_str(out: &buf, str: " and ");
1121	prt_bdevname(out: &buf, bdev: fs->bdev);
1122	prt_char(out: &buf, c: ':');
1123	prt_newline(&buf);
1124	prt_printf(&buf, "seq=%llu but write_time different, got", le64_to_cpu(sb->sb->seq));
1125	prt_newline(&buf);
1126
1127	prt_bdevname(out: &buf, bdev: fs->bdev);
1128	prt_char(out: &buf, c: ' ');
1129	bch2_prt_datetime(&buf, le64_to_cpu(fs->sb->write_time));;
1130	prt_newline(&buf);
1131
1132	prt_bdevname(out: &buf, bdev: sb->bdev);
1133	prt_char(out: &buf, c: ' ');
1134	bch2_prt_datetime(&buf, le64_to_cpu(sb->sb->write_time));;
1135	prt_newline(&buf);
1136
1137	if (!opts->no_splitbrain_check)
1138	prt_printf(&buf, "Not using older sb");
1139
1140	pr_err("%s", buf.buf);
1141	printbuf_exit(&buf);
1142
1143	if (!opts->no_splitbrain_check)
1144	return -BCH_ERR_device_splitbrain;
1145	}
1146
1147	struct bch_member m = bch2_sb_member_get(sb: fs->sb, i: sb->sb->dev_idx);
1148	u64 seq_from_fs = le64_to_cpu(m.seq);
1149	u64 seq_from_member = le64_to_cpu(sb->sb->seq);
1150
1151	if (seq_from_fs && seq_from_fs < seq_from_member) {
1152	struct printbuf buf = PRINTBUF;
1153
1154	prt_str(out: &buf, str: "Split brain detected between ");
1155	prt_bdevname(out: &buf, bdev: sb->bdev);
1156	prt_str(out: &buf, str: " and ");
1157	prt_bdevname(out: &buf, bdev: fs->bdev);
1158	prt_char(out: &buf, c: ':');
1159	prt_newline(&buf);
1160
1161	prt_bdevname(out: &buf, bdev: fs->bdev);
1162	prt_str(out: &buf, str: " believes seq of ");
1163	prt_bdevname(out: &buf, bdev: sb->bdev);
1164	prt_printf(&buf, " to be %llu, but ", seq_from_fs);
1165	prt_bdevname(out: &buf, bdev: sb->bdev);
1166	prt_printf(&buf, " has %llu\n", seq_from_member);
1167
1168	if (!opts->no_splitbrain_check) {
1169	prt_str(out: &buf, str: "Not using ");
1170	prt_bdevname(out: &buf, bdev: sb->bdev);
1171	}
1172
1173	pr_err("%s", buf.buf);
1174	printbuf_exit(&buf);
1175
1176	if (!opts->no_splitbrain_check)
1177	return -BCH_ERR_device_splitbrain;
1178	}
1179
1180	return 0;
1181	}
1182
1183	/* Device startup/shutdown: */
1184
1185	static void bch2_dev_release(struct kobject *kobj)
1186	{
1187	struct bch_dev *ca = container_of(kobj, struct bch_dev, kobj);
1188
1189	kfree(objp: ca);
1190	}
1191
1192	static void bch2_dev_free(struct bch_dev *ca)
1193	{
1194	cancel_work_sync(work: &ca->io_error_work);
1195
1196	if (ca->kobj.state_in_sysfs &&
1197	ca->disk_sb.bdev)
1198	sysfs_remove_link(bdev_kobj(ca->disk_sb.bdev), name: "bcachefs");
1199
1200	if (ca->kobj.state_in_sysfs)
1201	kobject_del(kobj: &ca->kobj);
1202
1203	bch2_free_super(&ca->disk_sb);
1204	bch2_dev_journal_exit(ca);
1205
1206	free_percpu(pdata: ca->io_done);
1207	bioset_exit(&ca->replica_set);
1208	bch2_dev_buckets_free(ca);
1209	free_page((unsigned long) ca->sb_read_scratch);
1210
1211	bch2_time_stats_quantiles_exit(statq: &ca->io_latency[WRITE]);
1212	bch2_time_stats_quantiles_exit(statq: &ca->io_latency[READ]);
1213
1214	percpu_ref_exit(ref: &ca->io_ref);
1215	percpu_ref_exit(ref: &ca->ref);
1216	kobject_put(kobj: &ca->kobj);
1217	}
1218
1219	static void __bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca)
1220	{
1221
1222	lockdep_assert_held(&c->state_lock);
1223
1224	if (percpu_ref_is_zero(ref: &ca->io_ref))
1225	return;
1226
1227	__bch2_dev_read_only(c, ca);
1228
1229	reinit_completion(x: &ca->io_ref_completion);
1230	percpu_ref_kill(ref: &ca->io_ref);
1231	wait_for_completion(&ca->io_ref_completion);
1232
1233	if (ca->kobj.state_in_sysfs) {
1234	sysfs_remove_link(bdev_kobj(ca->disk_sb.bdev), name: "bcachefs");
1235	sysfs_remove_link(kobj: &ca->kobj, name: "block");
1236	}
1237
1238	bch2_free_super(&ca->disk_sb);
1239	bch2_dev_journal_exit(ca);
1240	}
1241
1242	static void bch2_dev_ref_complete(struct percpu_ref *ref)
1243	{
1244	struct bch_dev *ca = container_of(ref, struct bch_dev, ref);
1245
1246	complete(&ca->ref_completion);
1247	}
1248
1249	static void bch2_dev_io_ref_complete(struct percpu_ref *ref)
1250	{
1251	struct bch_dev *ca = container_of(ref, struct bch_dev, io_ref);
1252
1253	complete(&ca->io_ref_completion);
1254	}
1255
1256	static int bch2_dev_sysfs_online(struct bch_fs *c, struct bch_dev *ca)
1257	{
1258	int ret;
1259
1260	if (!c->kobj.state_in_sysfs)
1261	return 0;
1262
1263	if (!ca->kobj.state_in_sysfs) {
1264	ret = kobject_add(kobj: &ca->kobj, parent: &c->kobj,
1265	fmt: "dev-%u", ca->dev_idx);
1266	if (ret)
1267	return ret;
1268	}
1269
1270	if (ca->disk_sb.bdev) {
1271	struct kobject *block = bdev_kobj(ca->disk_sb.bdev);
1272
1273	ret = sysfs_create_link(kobj: block, target: &ca->kobj, name: "bcachefs");
1274	if (ret)
1275	return ret;
1276
1277	ret = sysfs_create_link(kobj: &ca->kobj, target: block, name: "block");
1278	if (ret)
1279	return ret;
1280	}
1281
1282	return 0;
1283	}
1284
1285	static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c,
1286	struct bch_member *member)
1287	{
1288	struct bch_dev *ca;
1289	unsigned i;
1290
1291	ca = kzalloc(size: sizeof(*ca), GFP_KERNEL);
1292	if (!ca)
1293	return NULL;
1294
1295	kobject_init(kobj: &ca->kobj, ktype: &bch2_dev_ktype);
1296	init_completion(x: &ca->ref_completion);
1297	init_completion(x: &ca->io_ref_completion);
1298
1299	init_rwsem(&ca->bucket_lock);
1300
1301	INIT_WORK(&ca->io_error_work, bch2_io_error_work);
1302
1303	bch2_time_stats_quantiles_init(statq: &ca->io_latency[READ]);
1304	bch2_time_stats_quantiles_init(statq: &ca->io_latency[WRITE]);
1305
1306	ca->mi = bch2_mi_to_cpu(mi: member);
1307
1308	for (i = 0; i < ARRAY_SIZE(member->errors); i++)
1309	atomic64_set(v: &ca->errors[i], le64_to_cpu(member->errors[i]));
1310
1311	ca->uuid = member->uuid;
1312
1313	ca->nr_btree_reserve = DIV_ROUND_UP(BTREE_NODE_RESERVE,
1314	ca->mi.bucket_size / btree_sectors(c));
1315
1316	if (percpu_ref_init(ref: &ca->ref, release: bch2_dev_ref_complete,
1317	flags: 0, GFP_KERNEL) ||
1318	percpu_ref_init(ref: &ca->io_ref, release: bch2_dev_io_ref_complete,
1319	flags: PERCPU_REF_INIT_DEAD, GFP_KERNEL) ||
1320	!(ca->sb_read_scratch = (void *) __get_free_page(GFP_KERNEL)) ||
1321	bch2_dev_buckets_alloc(c, ca) ||
1322	bioset_init(&ca->replica_set, 4,
1323	offsetof(struct bch_write_bio, bio), flags: 0) ||
1324	!(ca->io_done = alloc_percpu(*ca->io_done)))
1325	goto err;
1326
1327	return ca;
1328	err:
1329	bch2_dev_free(ca);
1330	return NULL;
1331	}
1332
1333	static void bch2_dev_attach(struct bch_fs *c, struct bch_dev *ca,
1334	unsigned dev_idx)
1335	{
1336	ca->dev_idx = dev_idx;
1337	__set_bit(ca->dev_idx, ca->self.d);
1338	scnprintf(buf: ca->name, size: sizeof(ca->name), fmt: "dev-%u", dev_idx);
1339
1340	ca->fs = c;
1341	rcu_assign_pointer(c->devs[ca->dev_idx], ca);
1342
1343	if (bch2_dev_sysfs_online(c, ca))
1344	pr_warn("error creating sysfs objects");
1345	}
1346
1347	static int bch2_dev_alloc(struct bch_fs *c, unsigned dev_idx)
1348	{
1349	struct bch_member member = bch2_sb_member_get(sb: c->disk_sb.sb, i: dev_idx);
1350	struct bch_dev *ca = NULL;
1351	int ret = 0;
1352
1353	if (bch2_fs_init_fault("dev_alloc"))
1354	goto err;
1355
1356	ca = __bch2_dev_alloc(c, member: &member);
1357	if (!ca)
1358	goto err;
1359
1360	ca->fs = c;
1361
1362	bch2_dev_attach(c, ca, dev_idx);
1363	return ret;
1364	err:
1365	if (ca)
1366	bch2_dev_free(ca);
1367	return -BCH_ERR_ENOMEM_dev_alloc;
1368	}
1369
1370	static int __bch2_dev_attach_bdev(struct bch_dev *ca, struct bch_sb_handle *sb)
1371	{
1372	unsigned ret;
1373
1374	if (bch2_dev_is_online(ca)) {
1375	bch_err(ca, "already have device online in slot %u",
1376	sb->sb->dev_idx);
1377	return -BCH_ERR_device_already_online;
1378	}
1379
1380	if (get_capacity(disk: sb->bdev->bd_disk) <
1381	ca->mi.bucket_size * ca->mi.nbuckets) {
1382	bch_err(ca, "cannot online: device too small");
1383	return -BCH_ERR_device_size_too_small;
1384	}
1385
1386	BUG_ON(!percpu_ref_is_zero(&ca->io_ref));
1387
1388	ret = bch2_dev_journal_init(ca, sb->sb);
1389	if (ret)
1390	return ret;
1391
1392	/* Commit: */
1393	ca->disk_sb = *sb;
1394	memset(sb, 0, sizeof(*sb));
1395
1396	ca->dev = ca->disk_sb.bdev->bd_dev;
1397
1398	percpu_ref_reinit(ref: &ca->io_ref);
1399
1400	return 0;
1401	}
1402
1403	static int bch2_dev_attach_bdev(struct bch_fs *c, struct bch_sb_handle *sb)
1404	{
1405	struct bch_dev *ca;
1406	int ret;
1407
1408	lockdep_assert_held(&c->state_lock);
1409
1410	if (le64_to_cpu(sb->sb->seq) >
1411	le64_to_cpu(c->disk_sb.sb->seq))
1412	bch2_sb_to_fs(c, sb->sb);
1413
1414	BUG_ON(sb->sb->dev_idx >= c->sb.nr_devices ||
1415	!c->devs[sb->sb->dev_idx]);
1416
1417	ca = bch_dev_locked(c, idx: sb->sb->dev_idx);
1418
1419	ret = __bch2_dev_attach_bdev(ca, sb);
1420	if (ret)
1421	return ret;
1422
1423	bch2_dev_sysfs_online(c, ca);
1424
1425	struct printbuf name = PRINTBUF;
1426	prt_bdevname(out: &name, bdev: ca->disk_sb.bdev);
1427
1428	if (c->sb.nr_devices == 1)
1429	strscpy(c->name, name.buf, sizeof(c->name));
1430	strscpy(ca->name, name.buf, sizeof(ca->name));
1431
1432	printbuf_exit(&name);
1433
1434	rebalance_wakeup(c);
1435	return 0;
1436	}
1437
1438	/* Device management: */
1439
1440	/*
1441	* Note: this function is also used by the error paths - when a particular
1442	* device sees an error, we call it to determine whether we can just set the
1443	* device RO, or - if this function returns false - we'll set the whole
1444	* filesystem RO:
1445	*
1446	* XXX: maybe we should be more explicit about whether we're changing state
1447	* because we got an error or what have you?
1448	*/
1449	bool bch2_dev_state_allowed(struct bch_fs *c, struct bch_dev *ca,
1450	enum bch_member_state new_state, int flags)
1451	{
1452	struct bch_devs_mask new_online_devs;
1453	int nr_rw = 0, required;
1454
1455	lockdep_assert_held(&c->state_lock);
1456
1457	switch (new_state) {
1458	case BCH_MEMBER_STATE_rw:
1459	return true;
1460	case BCH_MEMBER_STATE_ro:
1461	if (ca->mi.state != BCH_MEMBER_STATE_rw)
1462	return true;
1463
1464	/* do we have enough devices to write to? */
1465	for_each_member_device(c, ca2)
1466	if (ca2 != ca)
1467	nr_rw += ca2->mi.state == BCH_MEMBER_STATE_rw;
1468
1469	required = max(!(flags & BCH_FORCE_IF_METADATA_DEGRADED)
1470	? c->opts.metadata_replicas
1471	: metadata_replicas_required(c),
1472	!(flags & BCH_FORCE_IF_DATA_DEGRADED)
1473	? c->opts.data_replicas
1474	: data_replicas_required(c));
1475
1476	return nr_rw >= required;
1477	case BCH_MEMBER_STATE_failed:
1478	case BCH_MEMBER_STATE_spare:
1479	if (ca->mi.state != BCH_MEMBER_STATE_rw &&
1480	ca->mi.state != BCH_MEMBER_STATE_ro)
1481	return true;
1482
1483	/* do we have enough devices to read from? */
1484	new_online_devs = bch2_online_devs(c);
1485	__clear_bit(ca->dev_idx, new_online_devs.d);
1486
1487	return bch2_have_enough_devs(c, new_online_devs, flags, false);
1488	default:
1489	BUG();
1490	}
1491	}
1492
1493	static bool bch2_fs_may_start(struct bch_fs *c)
1494	{
1495	struct bch_dev *ca;
1496	unsigned i, flags = 0;
1497
1498	if (c->opts.very_degraded)
1499	flags |= BCH_FORCE_IF_DEGRADED|BCH_FORCE_IF_LOST;
1500
1501	if (c->opts.degraded)
1502	flags |= BCH_FORCE_IF_DEGRADED;
1503
1504	if (!c->opts.degraded &&
1505	!c->opts.very_degraded) {
1506	mutex_lock(&c->sb_lock);
1507
1508	for (i = 0; i < c->disk_sb.sb->nr_devices; i++) {
1509	if (!bch2_dev_exists(sb: c->disk_sb.sb, dev: i))
1510	continue;
1511
1512	ca = bch_dev_locked(c, idx: i);
1513
1514	if (!bch2_dev_is_online(ca) &&
1515	(ca->mi.state == BCH_MEMBER_STATE_rw ||
1516	ca->mi.state == BCH_MEMBER_STATE_ro)) {
1517	mutex_unlock(lock: &c->sb_lock);
1518	return false;
1519	}
1520	}
1521	mutex_unlock(lock: &c->sb_lock);
1522	}
1523
1524	return bch2_have_enough_devs(c, bch2_online_devs(c), flags, true);
1525	}
1526
1527	static void __bch2_dev_read_only(struct bch_fs *c, struct bch_dev *ca)
1528	{
1529	/*
1530	* The allocator thread itself allocates btree nodes, so stop it first:
1531	*/
1532	bch2_dev_allocator_remove(c, ca);
1533	bch2_dev_journal_stop(&c->journal, ca);
1534	}
1535
1536	static void __bch2_dev_read_write(struct bch_fs *c, struct bch_dev *ca)
1537	{
1538	lockdep_assert_held(&c->state_lock);
1539
1540	BUG_ON(ca->mi.state != BCH_MEMBER_STATE_rw);
1541
1542	bch2_dev_allocator_add(c, ca);
1543	bch2_recalc_capacity(c);
1544	}
1545
1546	int __bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca,
1547	enum bch_member_state new_state, int flags)
1548	{
1549	struct bch_member *m;
1550	int ret = 0;
1551
1552	if (ca->mi.state == new_state)
1553	return 0;
1554
1555	if (!bch2_dev_state_allowed(c, ca, new_state, flags))
1556	return -BCH_ERR_device_state_not_allowed;
1557
1558	if (new_state != BCH_MEMBER_STATE_rw)
1559	__bch2_dev_read_only(c, ca);
1560
1561	bch_notice(ca, "%s", bch2_member_states[new_state]);
1562
1563	mutex_lock(&c->sb_lock);
1564	m = bch2_members_v2_get_mut(sb: c->disk_sb.sb, i: ca->dev_idx);
1565	SET_BCH_MEMBER_STATE(k: m, v: new_state);
1566	bch2_write_super(c);
1567	mutex_unlock(lock: &c->sb_lock);
1568
1569	if (new_state == BCH_MEMBER_STATE_rw)
1570	__bch2_dev_read_write(c, ca);
1571
1572	rebalance_wakeup(c);
1573
1574	return ret;
1575	}
1576
1577	int bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca,
1578	enum bch_member_state new_state, int flags)
1579	{
1580	int ret;
1581
1582	down_write(sem: &c->state_lock);
1583	ret = __bch2_dev_set_state(c, ca, new_state, flags);
1584	up_write(sem: &c->state_lock);
1585
1586	return ret;
1587	}
1588
1589	/* Device add/removal: */
1590
1591	static int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca)
1592	{
1593	struct bpos start = POS(ca->dev_idx, 0);
1594	struct bpos end = POS(ca->dev_idx, U64_MAX);
1595	int ret;
1596
1597	/*
1598	* We clear the LRU and need_discard btrees first so that we don't race
1599	* with bch2_do_invalidates() and bch2_do_discards()
1600	*/
1601	ret = bch2_btree_delete_range(c, BTREE_ID_lru, start, end,
1602	BTREE_TRIGGER_NORUN, NULL) ?:
1603	bch2_btree_delete_range(c, BTREE_ID_need_discard, start, end,
1604	BTREE_TRIGGER_NORUN, NULL) ?:
1605	bch2_btree_delete_range(c, BTREE_ID_freespace, start, end,
1606	BTREE_TRIGGER_NORUN, NULL) ?:
1607	bch2_btree_delete_range(c, BTREE_ID_backpointers, start, end,
1608	BTREE_TRIGGER_NORUN, NULL) ?:
1609	bch2_btree_delete_range(c, BTREE_ID_alloc, start, end,
1610	BTREE_TRIGGER_NORUN, NULL) ?:
1611	bch2_btree_delete_range(c, BTREE_ID_bucket_gens, start, end,
1612	BTREE_TRIGGER_NORUN, NULL);
1613	bch_err_msg(c, ret, "removing dev alloc info");
1614	return ret;
1615	}
1616
1617	int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags)
1618	{
1619	struct bch_member *m;
1620	unsigned dev_idx = ca->dev_idx, data;
1621	int ret;
1622
1623	down_write(sem: &c->state_lock);
1624
1625	/*
1626	* We consume a reference to ca->ref, regardless of whether we succeed
1627	* or fail:
1628	*/
1629	percpu_ref_put(ref: &ca->ref);
1630
1631	if (!bch2_dev_state_allowed(c, ca, new_state: BCH_MEMBER_STATE_failed, flags)) {
1632	bch_err(ca, "Cannot remove without losing data");
1633	ret = -BCH_ERR_device_state_not_allowed;
1634	goto err;
1635	}
1636
1637	__bch2_dev_read_only(c, ca);
1638
1639	ret = bch2_dev_data_drop(c, ca->dev_idx, flags);
1640	bch_err_msg(ca, ret, "bch2_dev_data_drop()");
1641	if (ret)
1642	goto err;
1643
1644	ret = bch2_dev_remove_alloc(c, ca);
1645	bch_err_msg(ca, ret, "bch2_dev_remove_alloc()");
1646	if (ret)
1647	goto err;
1648
1649	ret = bch2_journal_flush_device_pins(&c->journal, ca->dev_idx);
1650	bch_err_msg(ca, ret, "bch2_journal_flush_device_pins()");
1651	if (ret)
1652	goto err;
1653
1654	ret = bch2_journal_flush(&c->journal);
1655	bch_err_msg(ca, ret, "bch2_journal_flush()");
1656	if (ret)
1657	goto err;
1658
1659	ret = bch2_replicas_gc2(c);
1660	bch_err_msg(ca, ret, "bch2_replicas_gc2()");
1661	if (ret)
1662	goto err;
1663
1664	data = bch2_dev_has_data(c, ca);
1665	if (data) {
1666	struct printbuf data_has = PRINTBUF;
1667
1668	prt_bitflags(&data_has, __bch2_data_types, data);
1669	bch_err(ca, "Remove failed, still has data (%s)", data_has.buf);
1670	printbuf_exit(&data_has);
1671	ret = -EBUSY;
1672	goto err;
1673	}
1674
1675	__bch2_dev_offline(c, ca);
1676
1677	mutex_lock(&c->sb_lock);
1678	rcu_assign_pointer(c->devs[ca->dev_idx], NULL);
1679	mutex_unlock(lock: &c->sb_lock);
1680
1681	percpu_ref_kill(ref: &ca->ref);
1682	wait_for_completion(&ca->ref_completion);
1683
1684	bch2_dev_free(ca);
1685
1686	/*
1687	* At this point the device object has been removed in-core, but the
1688	* on-disk journal might still refer to the device index via sb device
1689	* usage entries. Recovery fails if it sees usage information for an
1690	* invalid device. Flush journal pins to push the back of the journal
1691	* past now invalid device index references before we update the
1692	* superblock, but after the device object has been removed so any
1693	* further journal writes elide usage info for the device.
1694	*/
1695	bch2_journal_flush_all_pins(j: &c->journal);
1696
1697	/*
1698	* Free this device's slot in the bch_member array - all pointers to
1699	* this device must be gone:
1700	*/
1701	mutex_lock(&c->sb_lock);
1702	m = bch2_members_v2_get_mut(sb: c->disk_sb.sb, i: dev_idx);
1703	memset(&m->uuid, 0, sizeof(m->uuid));
1704
1705	bch2_write_super(c);
1706
1707	mutex_unlock(lock: &c->sb_lock);
1708	up_write(sem: &c->state_lock);
1709
1710	bch2_dev_usage_journal_reserve(c);
1711	return 0;
1712	err:
1713	if (ca->mi.state == BCH_MEMBER_STATE_rw &&
1714	!percpu_ref_is_zero(ref: &ca->io_ref))
1715	__bch2_dev_read_write(c, ca);
1716	up_write(sem: &c->state_lock);
1717	return ret;
1718	}
1719
1720	/* Add new device to running filesystem: */
1721	int bch2_dev_add(struct bch_fs *c, const char *path)
1722	{
1723	struct bch_opts opts = bch2_opts_empty();
1724	struct bch_sb_handle sb;
1725	struct bch_dev *ca = NULL;
1726	struct bch_sb_field_members_v2 *mi;
1727	struct bch_member dev_mi;
1728	unsigned dev_idx, nr_devices, u64s;
1729	struct printbuf errbuf = PRINTBUF;
1730	struct printbuf label = PRINTBUF;
1731	int ret;
1732
1733	ret = bch2_read_super(path, &opts, &sb);
1734	bch_err_msg(c, ret, "reading super");
1735	if (ret)
1736	goto err;
1737
1738	dev_mi = bch2_sb_member_get(sb: sb.sb, i: sb.sb->dev_idx);
1739
1740	if (BCH_MEMBER_GROUP(k: &dev_mi)) {
1741	bch2_disk_path_to_text_sb(&label, sb.sb, BCH_MEMBER_GROUP(k: &dev_mi) - 1);
1742	if (label.allocation_failure) {
1743	ret = -ENOMEM;
1744	goto err;
1745	}
1746	}
1747
1748	ret = bch2_dev_may_add(sb: sb.sb, c);
1749	if (ret)
1750	goto err;
1751
1752	ca = __bch2_dev_alloc(c, member: &dev_mi);
1753	if (!ca) {
1754	ret = -ENOMEM;
1755	goto err;
1756	}
1757
1758	bch2_dev_usage_init(ca);
1759
1760	ret = __bch2_dev_attach_bdev(ca, sb: &sb);
1761	if (ret)
1762	goto err;
1763
1764	ret = bch2_dev_journal_alloc(ca);
1765	bch_err_msg(c, ret, "allocating journal");
1766	if (ret)
1767	goto err;
1768
1769	down_write(sem: &c->state_lock);
1770	mutex_lock(&c->sb_lock);
1771
1772	ret = bch2_sb_from_fs(c, ca);
1773	bch_err_msg(c, ret, "setting up new superblock");
1774	if (ret)
1775	goto err_unlock;
1776
1777	if (dynamic_fault("bcachefs:add:no_slot"))
1778	goto no_slot;
1779
1780	for (dev_idx = 0; dev_idx < BCH_SB_MEMBERS_MAX; dev_idx++)
1781	if (!bch2_dev_exists(sb: c->disk_sb.sb, dev: dev_idx))
1782	goto have_slot;
1783	no_slot:
1784	ret = -BCH_ERR_ENOSPC_sb_members;
1785	bch_err_msg(c, ret, "setting up new superblock");
1786	goto err_unlock;
1787
1788	have_slot:
1789	nr_devices = max_t(unsigned, dev_idx + 1, c->sb.nr_devices);
1790
1791	mi = bch2_sb_field_get(c->disk_sb.sb, members_v2);
1792	u64s = DIV_ROUND_UP(sizeof(struct bch_sb_field_members_v2) +
1793	le16_to_cpu(mi->member_bytes) * nr_devices, sizeof(u64));
1794
1795	mi = bch2_sb_field_resize(&c->disk_sb, members_v2, u64s);
1796	if (!mi) {
1797	ret = -BCH_ERR_ENOSPC_sb_members;
1798	bch_err_msg(c, ret, "setting up new superblock");
1799	goto err_unlock;
1800	}
1801	struct bch_member *m = bch2_members_v2_get_mut(sb: c->disk_sb.sb, i: dev_idx);
1802
1803	/* success: */
1804
1805	*m = dev_mi;
1806	m->last_mount = cpu_to_le64(ktime_get_real_seconds());
1807	c->disk_sb.sb->nr_devices = nr_devices;
1808
1809	ca->disk_sb.sb->dev_idx = dev_idx;
1810	bch2_dev_attach(c, ca, dev_idx);
1811
1812	if (BCH_MEMBER_GROUP(k: &dev_mi)) {
1813	ret = __bch2_dev_group_set(c, ca, label.buf);
1814	bch_err_msg(c, ret, "creating new label");
1815	if (ret)
1816	goto err_unlock;
1817	}
1818
1819	bch2_write_super(c);
1820	mutex_unlock(lock: &c->sb_lock);
1821
1822	bch2_dev_usage_journal_reserve(c);
1823
1824	ret = bch2_trans_mark_dev_sb(c, ca);
1825	bch_err_msg(ca, ret, "marking new superblock");
1826	if (ret)
1827	goto err_late;
1828
1829	ret = bch2_fs_freespace_init(c);
1830	bch_err_msg(ca, ret, "initializing free space");
1831	if (ret)
1832	goto err_late;
1833
1834	ca->new_fs_bucket_idx = 0;
1835
1836	if (ca->mi.state == BCH_MEMBER_STATE_rw)
1837	__bch2_dev_read_write(c, ca);
1838
1839	up_write(sem: &c->state_lock);
1840	return 0;
1841
1842	err_unlock:
1843	mutex_unlock(lock: &c->sb_lock);
1844	up_write(sem: &c->state_lock);
1845	err:
1846	if (ca)
1847	bch2_dev_free(ca);
1848	bch2_free_super(&sb);
1849	printbuf_exit(&label);
1850	printbuf_exit(&errbuf);
1851	bch_err_fn(c, ret);
1852	return ret;
1853	err_late:
1854	up_write(sem: &c->state_lock);
1855	ca = NULL;
1856	goto err;
1857	}
1858
1859	/* Hot add existing device to running filesystem: */
1860	int bch2_dev_online(struct bch_fs *c, const char *path)
1861	{
1862	struct bch_opts opts = bch2_opts_empty();
1863	struct bch_sb_handle sb = { NULL };
1864	struct bch_dev *ca;
1865	unsigned dev_idx;
1866	int ret;
1867
1868	down_write(sem: &c->state_lock);
1869
1870	ret = bch2_read_super(path, &opts, &sb);
1871	if (ret) {
1872	up_write(sem: &c->state_lock);
1873	return ret;
1874	}
1875
1876	dev_idx = sb.sb->dev_idx;
1877
1878	ret = bch2_dev_in_fs(fs: &c->disk_sb, sb: &sb, opts: &c->opts);
1879	bch_err_msg(c, ret, "bringing %s online", path);
1880	if (ret)
1881	goto err;
1882
1883	ret = bch2_dev_attach_bdev(c, sb: &sb);
1884	if (ret)
1885	goto err;
1886
1887	ca = bch_dev_locked(c, idx: dev_idx);
1888
1889	ret = bch2_trans_mark_dev_sb(c, ca);
1890	bch_err_msg(c, ret, "bringing %s online: error from bch2_trans_mark_dev_sb", path);
1891	if (ret)
1892	goto err;
1893
1894	if (ca->mi.state == BCH_MEMBER_STATE_rw)
1895	__bch2_dev_read_write(c, ca);
1896
1897	if (!ca->mi.freespace_initialized) {
1898	ret = bch2_dev_freespace_init(c, ca, 0, ca->mi.nbuckets);
1899	bch_err_msg(ca, ret, "initializing free space");
1900	if (ret)
1901	goto err;
1902	}
1903
1904	if (!ca->journal.nr) {
1905	ret = bch2_dev_journal_alloc(ca);
1906	bch_err_msg(ca, ret, "allocating journal");
1907	if (ret)
1908	goto err;
1909	}
1910
1911	mutex_lock(&c->sb_lock);
1912	bch2_members_v2_get_mut(sb: c->disk_sb.sb, i: ca->dev_idx)->last_mount =
1913	cpu_to_le64(ktime_get_real_seconds());
1914	bch2_write_super(c);
1915	mutex_unlock(lock: &c->sb_lock);
1916
1917	up_write(sem: &c->state_lock);
1918	return 0;
1919	err:
1920	up_write(sem: &c->state_lock);
1921	bch2_free_super(&sb);
1922	return ret;
1923	}
1924
1925	int bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca, int flags)
1926	{
1927	down_write(sem: &c->state_lock);
1928
1929	if (!bch2_dev_is_online(ca)) {
1930	bch_err(ca, "Already offline");
1931	up_write(sem: &c->state_lock);
1932	return 0;
1933	}
1934
1935	if (!bch2_dev_state_allowed(c, ca, new_state: BCH_MEMBER_STATE_failed, flags)) {
1936	bch_err(ca, "Cannot offline required disk");
1937	up_write(sem: &c->state_lock);
1938	return -BCH_ERR_device_state_not_allowed;
1939	}
1940
1941	__bch2_dev_offline(c, ca);
1942
1943	up_write(sem: &c->state_lock);
1944	return 0;
1945	}
1946
1947	int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
1948	{
1949	struct bch_member *m;
1950	u64 old_nbuckets;
1951	int ret = 0;
1952
1953	down_write(sem: &c->state_lock);
1954	old_nbuckets = ca->mi.nbuckets;
1955
1956	if (nbuckets < ca->mi.nbuckets) {
1957	bch_err(ca, "Cannot shrink yet");
1958	ret = -EINVAL;
1959	goto err;
1960	}
1961
1962	if (bch2_dev_is_online(ca) &&
1963	get_capacity(disk: ca->disk_sb.bdev->bd_disk) <
1964	ca->mi.bucket_size * nbuckets) {
1965	bch_err(ca, "New size larger than device");
1966	ret = -BCH_ERR_device_size_too_small;
1967	goto err;
1968	}
1969
1970	ret = bch2_dev_buckets_resize(c, ca, nbuckets);
1971	bch_err_msg(ca, ret, "resizing buckets");
1972	if (ret)
1973	goto err;
1974
1975	ret = bch2_trans_mark_dev_sb(c, ca);
1976	if (ret)
1977	goto err;
1978
1979	mutex_lock(&c->sb_lock);
1980	m = bch2_members_v2_get_mut(sb: c->disk_sb.sb, i: ca->dev_idx);
1981	m->nbuckets = cpu_to_le64(nbuckets);
1982
1983	bch2_write_super(c);
1984	mutex_unlock(lock: &c->sb_lock);
1985
1986	if (ca->mi.freespace_initialized) {
1987	ret = bch2_dev_freespace_init(c, ca, old_nbuckets, nbuckets);
1988	if (ret)
1989	goto err;
1990
1991	/*
1992	* XXX: this is all wrong transactionally - we'll be able to do
1993	* this correctly after the disk space accounting rewrite
1994	*/
1995	ca->usage_base->d[BCH_DATA_free].buckets += nbuckets - old_nbuckets;
1996	}
1997
1998	bch2_recalc_capacity(c);
1999	err:
2000	up_write(sem: &c->state_lock);
2001	return ret;
2002	}
2003
2004	/* return with ref on ca->ref: */
2005	struct bch_dev *bch2_dev_lookup(struct bch_fs *c, const char *name)
2006	{
2007	rcu_read_lock();
2008	for_each_member_device_rcu(c, ca, NULL)
2009	if (!strcmp(name, ca->name)) {
2010	rcu_read_unlock();
2011	return ca;
2012	}
2013	rcu_read_unlock();
2014	return ERR_PTR(error: -BCH_ERR_ENOENT_dev_not_found);
2015	}
2016
2017	/* Filesystem open: */
2018
2019	static inline int sb_cmp(struct bch_sb *l, struct bch_sb *r)
2020	{
2021	return cmp_int(le64_to_cpu(l->seq), le64_to_cpu(r->seq)) ?:
2022	cmp_int(le64_to_cpu(l->write_time), le64_to_cpu(r->write_time));
2023	}
2024
2025	struct bch_fs *bch2_fs_open(char * const *devices, unsigned nr_devices,
2026	struct bch_opts opts)
2027	{
2028	DARRAY(struct bch_sb_handle) sbs = { 0 };
2029	struct bch_fs *c = NULL;
2030	struct bch_sb_handle *best = NULL;
2031	struct printbuf errbuf = PRINTBUF;
2032	int ret = 0;
2033
2034	if (!try_module_get(THIS_MODULE))
2035	return ERR_PTR(error: -ENODEV);
2036
2037	if (!nr_devices) {
2038	ret = -EINVAL;
2039	goto err;
2040	}
2041
2042	ret = darray_make_room(&sbs, nr_devices);
2043	if (ret)
2044	goto err;
2045
2046	for (unsigned i = 0; i < nr_devices; i++) {
2047	struct bch_sb_handle sb = { NULL };
2048
2049	ret = bch2_read_super(devices[i], &opts, &sb);
2050	if (ret)
2051	goto err;
2052
2053	BUG_ON(darray_push(&sbs, sb));
2054	}
2055
2056	if (opts.nochanges && !opts.read_only) {
2057	ret = -BCH_ERR_erofs_nochanges;
2058	goto err_print;
2059	}
2060
2061	darray_for_each(sbs, sb)
2062	if (!best || sb_cmp(l: sb->sb, r: best->sb) > 0)
2063	best = sb;
2064
2065	darray_for_each_reverse(sbs, sb) {
2066	ret = bch2_dev_in_fs(fs: best, sb, opts: &opts);
2067
2068	if (ret == -BCH_ERR_device_has_been_removed ||
2069	ret == -BCH_ERR_device_splitbrain) {
2070	bch2_free_super(sb);
2071	darray_remove_item(&sbs, sb);
2072	best -= best > sb;
2073	ret = 0;
2074	continue;
2075	}
2076
2077	if (ret)
2078	goto err_print;
2079	}
2080
2081	c = bch2_fs_alloc(sb: best->sb, opts);
2082	ret = PTR_ERR_OR_ZERO(ptr: c);
2083	if (ret)
2084	goto err;
2085
2086	down_write(sem: &c->state_lock);
2087	darray_for_each(sbs, sb) {
2088	ret = bch2_dev_attach_bdev(c, sb);
2089	if (ret) {
2090	up_write(sem: &c->state_lock);
2091	goto err;
2092	}
2093	}
2094	up_write(sem: &c->state_lock);
2095
2096	if (!bch2_fs_may_start(c)) {
2097	ret = -BCH_ERR_insufficient_devices_to_start;
2098	goto err_print;
2099	}
2100
2101	if (!c->opts.nostart) {
2102	ret = bch2_fs_start(c);
2103	if (ret)
2104	goto err;
2105	}
2106	out:
2107	darray_for_each(sbs, sb)
2108	bch2_free_super(sb);
2109	darray_exit(&sbs);
2110	printbuf_exit(&errbuf);
2111	module_put(THIS_MODULE);
2112	return c;
2113	err_print:
2114	pr_err("bch_fs_open err opening %s: %s",
2115	devices[0], bch2_err_str(ret));
2116	err:
2117	if (!IS_ERR_OR_NULL(ptr: c))
2118	bch2_fs_stop(c);
2119	c = ERR_PTR(error: ret);
2120	goto out;
2121	}
2122
2123	/* Global interfaces/init */
2124
2125	static void bcachefs_exit(void)
2126	{
2127	bch2_debug_exit();
2128	bch2_vfs_exit();
2129	bch2_chardev_exit();
2130	bch2_btree_key_cache_exit();
2131	if (bcachefs_kset)
2132	kset_unregister(kset: bcachefs_kset);
2133	}
2134
2135	static int __init bcachefs_init(void)
2136	{
2137	bch2_bkey_pack_test();
2138
2139	if (!(bcachefs_kset = kset_create_and_add(name: "bcachefs", NULL, parent_kobj: fs_kobj)) ||
2140	bch2_btree_key_cache_init() ||
2141	bch2_chardev_init() ||
2142	bch2_vfs_init() ||
2143	bch2_debug_init())
2144	goto err;
2145
2146	return 0;
2147	err:
2148	bcachefs_exit();
2149	return -ENOMEM;
2150	}
2151
2152	#define BCH_DEBUG_PARAM(name, description) \
2153	bool bch2_##name; \
2154	module_param_named(name, bch2_##name, bool, 0644); \
2155	MODULE_PARM_DESC(name, description);
2156	BCH_DEBUG_PARAMS()
2157	#undef BCH_DEBUG_PARAM
2158
2159	__maybe_unused
2160	static unsigned bch2_metadata_version = bcachefs_metadata_version_current;
2161	module_param_named(version, bch2_metadata_version, uint, 0400);
2162
2163	module_exit(bcachefs_exit);
2164	module_init(bcachefs_init);
2165