1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
4	* Shaohua Li <shli@fb.com>
5	*/
6	#include <linux/module.h>
7
8	#include <linux/moduleparam.h>
9	#include <linux/sched.h>
10	#include <linux/fs.h>
11	#include <linux/init.h>
12	#include "null_blk.h"
13
14	#undef pr_fmt
15	#define pr_fmt(fmt) "null_blk: " fmt
16
17	#define FREE_BATCH 16
18
19	#define TICKS_PER_SEC 50ULL
20	#define TIMER_INTERVAL (NSEC_PER_SEC / TICKS_PER_SEC)
21
22	#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
23	static DECLARE_FAULT_ATTR(null_timeout_attr);
24	static DECLARE_FAULT_ATTR(null_requeue_attr);
25	static DECLARE_FAULT_ATTR(null_init_hctx_attr);
26	#endif
27
28	static inline u64 mb_per_tick(int mbps)
29	{
30	return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
31	}
32
33	/*
34	* Status flags for nullb_device.
35	*
36	* CONFIGURED: Device has been configured and turned on. Cannot reconfigure.
37	* UP: Device is currently on and visible in userspace.
38	* THROTTLED: Device is being throttled.
39	* CACHE: Device is using a write-back cache.
40	*/
41	enum nullb_device_flags {
42	NULLB_DEV_FL_CONFIGURED = 0,
43	NULLB_DEV_FL_UP = 1,
44	NULLB_DEV_FL_THROTTLED = 2,
45	NULLB_DEV_FL_CACHE = 3,
46	};
47
48	#define MAP_SZ ((PAGE_SIZE >> SECTOR_SHIFT) + 2)
49	/*
50	* nullb_page is a page in memory for nullb devices.
51	*
52	* @page: The page holding the data.
53	* @bitmap: The bitmap represents which sector in the page has data.
54	* Each bit represents one block size. For example, sector 8
55	* will use the 7th bit
56	* The highest 2 bits of bitmap are for special purpose. LOCK means the cache
57	* page is being flushing to storage. FREE means the cache page is freed and
58	* should be skipped from flushing to storage. Please see
59	* null_make_cache_space
60	*/
61	struct nullb_page {
62	struct page *page;
63	DECLARE_BITMAP(bitmap, MAP_SZ);
64	};
65	#define NULLB_PAGE_LOCK (MAP_SZ - 1)
66	#define NULLB_PAGE_FREE (MAP_SZ - 2)
67
68	static LIST_HEAD(nullb_list);
69	static struct mutex lock;
70	static int null_major;
71	static DEFINE_IDA(nullb_indexes);
72	static struct blk_mq_tag_set tag_set;
73
74	enum {
75	NULL_IRQ_NONE = 0,
76	NULL_IRQ_SOFTIRQ = 1,
77	NULL_IRQ_TIMER = 2,
78	};
79
80	static bool g_virt_boundary = false;
81	module_param_named(virt_boundary, g_virt_boundary, bool, 0444);
82	MODULE_PARM_DESC(virt_boundary, "Require a virtual boundary for the device. Default: False");
83
84	static int g_no_sched;
85	module_param_named(no_sched, g_no_sched, int, 0444);
86	MODULE_PARM_DESC(no_sched, "No io scheduler");
87
88	static int g_submit_queues = 1;
89	module_param_named(submit_queues, g_submit_queues, int, 0444);
90	MODULE_PARM_DESC(submit_queues, "Number of submission queues");
91
92	static int g_poll_queues = 1;
93	module_param_named(poll_queues, g_poll_queues, int, 0444);
94	MODULE_PARM_DESC(poll_queues, "Number of IOPOLL submission queues");
95
96	static int g_home_node = NUMA_NO_NODE;
97	module_param_named(home_node, g_home_node, int, 0444);
98	MODULE_PARM_DESC(home_node, "Home node for the device");
99
100	#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
101	/*
102	* For more details about fault injection, please refer to
103	* Documentation/fault-injection/fault-injection.rst.
104	*/
105	static char g_timeout_str[80];
106	module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), 0444);
107	MODULE_PARM_DESC(timeout, "Fault injection. timeout=<interval>,<probability>,<space>,<times>");
108
109	static char g_requeue_str[80];
110	module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), 0444);
111	MODULE_PARM_DESC(requeue, "Fault injection. requeue=<interval>,<probability>,<space>,<times>");
112
113	static char g_init_hctx_str[80];
114	module_param_string(init_hctx, g_init_hctx_str, sizeof(g_init_hctx_str), 0444);
115	MODULE_PARM_DESC(init_hctx, "Fault injection to fail hctx init. init_hctx=<interval>,<probability>,<space>,<times>");
116	#endif
117
118	/*
119	* Historic queue modes.
120	*
121	* These days nothing but NULL_Q_MQ is actually supported, but we keep it the
122	* enum for error reporting.
123	*/
124	enum {
125	NULL_Q_BIO = 0,
126	NULL_Q_RQ = 1,
127	NULL_Q_MQ = 2,
128	};
129
130	static int g_queue_mode = NULL_Q_MQ;
131
132	static int null_param_store_val(const char *str, int *val, int min, int max)
133	{
134	int ret, new_val;
135
136	ret = kstrtoint(s: str, base: 10, res: &new_val);
137	if (ret)
138	return -EINVAL;
139
140	if (new_val < min || new_val > max)
141	return -EINVAL;
142
143	*val = new_val;
144	return 0;
145	}
146
147	static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
148	{
149	return null_param_store_val(str, val: &g_queue_mode, min: NULL_Q_BIO, max: NULL_Q_MQ);
150	}
151
152	static const struct kernel_param_ops null_queue_mode_param_ops = {
153	.set = null_set_queue_mode,
154	.get = param_get_int,
155	};
156
157	device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, 0444);
158	MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
159
160	static int g_gb = 250;
161	module_param_named(gb, g_gb, int, 0444);
162	MODULE_PARM_DESC(gb, "Size in GB");
163
164	static int g_bs = 512;
165	module_param_named(bs, g_bs, int, 0444);
166	MODULE_PARM_DESC(bs, "Block size (in bytes)");
167
168	static int g_max_sectors;
169	module_param_named(max_sectors, g_max_sectors, int, 0444);
170	MODULE_PARM_DESC(max_sectors, "Maximum size of a command (in 512B sectors)");
171
172	static unsigned int nr_devices = 1;
173	module_param(nr_devices, uint, 0444);
174	MODULE_PARM_DESC(nr_devices, "Number of devices to register");
175
176	static bool g_blocking;
177	module_param_named(blocking, g_blocking, bool, 0444);
178	MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
179
180	static bool g_shared_tags;
181	module_param_named(shared_tags, g_shared_tags, bool, 0444);
182	MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
183
184	static bool g_shared_tag_bitmap;
185	module_param_named(shared_tag_bitmap, g_shared_tag_bitmap, bool, 0444);
186	MODULE_PARM_DESC(shared_tag_bitmap, "Use shared tag bitmap for all submission queues for blk-mq");
187
188	static int g_irqmode = NULL_IRQ_SOFTIRQ;
189
190	static int null_set_irqmode(const char *str, const struct kernel_param *kp)
191	{
192	return null_param_store_val(str, val: &g_irqmode, min: NULL_IRQ_NONE,
193	max: NULL_IRQ_TIMER);
194	}
195
196	static const struct kernel_param_ops null_irqmode_param_ops = {
197	.set = null_set_irqmode,
198	.get = param_get_int,
199	};
200
201	device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, 0444);
202	MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
203
204	static unsigned long g_completion_nsec = 10000;
205	module_param_named(completion_nsec, g_completion_nsec, ulong, 0444);
206	MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
207
208	static int g_hw_queue_depth = 64;
209	module_param_named(hw_queue_depth, g_hw_queue_depth, int, 0444);
210	MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
211
212	static bool g_use_per_node_hctx;
213	module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, 0444);
214	MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
215
216	static bool g_memory_backed;
217	module_param_named(memory_backed, g_memory_backed, bool, 0444);
218	MODULE_PARM_DESC(memory_backed, "Create a memory-backed block device. Default: false");
219
220	static bool g_discard;
221	module_param_named(discard, g_discard, bool, 0444);
222	MODULE_PARM_DESC(discard, "Support discard operations (requires memory-backed null_blk device). Default: false");
223
224	static unsigned long g_cache_size;
225	module_param_named(cache_size, g_cache_size, ulong, 0444);
226	MODULE_PARM_DESC(mbps, "Cache size in MiB for memory-backed device. Default: 0 (none)");
227
228	static unsigned int g_mbps;
229	module_param_named(mbps, g_mbps, uint, 0444);
230	MODULE_PARM_DESC(mbps, "Limit maximum bandwidth (in MiB/s). Default: 0 (no limit)");
231
232	static bool g_zoned;
233	module_param_named(zoned, g_zoned, bool, S_IRUGO);
234	MODULE_PARM_DESC(zoned, "Make device as a host-managed zoned block device. Default: false");
235
236	static unsigned long g_zone_size = 256;
237	module_param_named(zone_size, g_zone_size, ulong, S_IRUGO);
238	MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256");
239
240	static unsigned long g_zone_capacity;
241	module_param_named(zone_capacity, g_zone_capacity, ulong, 0444);
242	MODULE_PARM_DESC(zone_capacity, "Zone capacity in MB when block device is zoned. Can be less than or equal to zone size. Default: Zone size");
243
244	static unsigned int g_zone_nr_conv;
245	module_param_named(zone_nr_conv, g_zone_nr_conv, uint, 0444);
246	MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones when block device is zoned. Default: 0");
247
248	static unsigned int g_zone_max_open;
249	module_param_named(zone_max_open, g_zone_max_open, uint, 0444);
250	MODULE_PARM_DESC(zone_max_open, "Maximum number of open zones when block device is zoned. Default: 0 (no limit)");
251
252	static unsigned int g_zone_max_active;
253	module_param_named(zone_max_active, g_zone_max_active, uint, 0444);
254	MODULE_PARM_DESC(zone_max_active, "Maximum number of active zones when block device is zoned. Default: 0 (no limit)");
255
256	static struct nullb_device *null_alloc_dev(void);
257	static void null_free_dev(struct nullb_device *dev);
258	static void null_del_dev(struct nullb *nullb);
259	static int null_add_dev(struct nullb_device *dev);
260	static struct nullb *null_find_dev_by_name(const char *name);
261	static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
262
263	static inline struct nullb_device *to_nullb_device(struct config_item *item)
264	{
265	return item ? container_of(to_config_group(item), struct nullb_device, group) : NULL;
266	}
267
268	static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
269	{
270	return snprintf(buf: page, PAGE_SIZE, fmt: "%u\n", val);
271	}
272
273	static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
274	char *page)
275	{
276	return snprintf(buf: page, PAGE_SIZE, fmt: "%lu\n", val);
277	}
278
279	static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
280	{
281	return snprintf(buf: page, PAGE_SIZE, fmt: "%u\n", val);
282	}
283
284	static ssize_t nullb_device_uint_attr_store(unsigned int *val,
285	const char *page, size_t count)
286	{
287	unsigned int tmp;
288	int result;
289
290	result = kstrtouint(s: page, base: 0, res: &tmp);
291	if (result < 0)
292	return result;
293
294	*val = tmp;
295	return count;
296	}
297
298	static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
299	const char *page, size_t count)
300	{
301	int result;
302	unsigned long tmp;
303
304	result = kstrtoul(s: page, base: 0, res: &tmp);
305	if (result < 0)
306	return result;
307
308	*val = tmp;
309	return count;
310	}
311
312	static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
313	size_t count)
314	{
315	bool tmp;
316	int result;
317
318	result = kstrtobool(s: page, res: &tmp);
319	if (result < 0)
320	return result;
321
322	*val = tmp;
323	return count;
324	}
325
326	/* The following macro should only be used with TYPE = {uint, ulong, bool}. */
327	#define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY) \
328	static ssize_t \
329	nullb_device_##NAME##_show(struct config_item *item, char *page) \
330	{ \
331	return nullb_device_##TYPE##_attr_show( \
332	to_nullb_device(item)->NAME, page); \
333	} \
334	static ssize_t \
335	nullb_device_##NAME##_store(struct config_item *item, const char *page, \
336	size_t count) \
337	{ \
338	int (*apply_fn)(struct nullb_device *dev, TYPE new_value) = APPLY;\
339	struct nullb_device *dev = to_nullb_device(item); \
340	TYPE new_value = 0; \
341	int ret; \
342	\
343	ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\
344	if (ret < 0) \
345	return ret; \
346	if (apply_fn) \
347	ret = apply_fn(dev, new_value); \
348	else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) \
349	ret = -EBUSY; \
350	if (ret < 0) \
351	return ret; \
352	dev->NAME = new_value; \
353	return count; \
354	} \
355	CONFIGFS_ATTR(nullb_device_, NAME);
356
357	static int nullb_update_nr_hw_queues(struct nullb_device *dev,
358	unsigned int submit_queues,
359	unsigned int poll_queues)
360
361	{
362	struct blk_mq_tag_set *set;
363	int ret, nr_hw_queues;
364
365	if (!dev->nullb)
366	return 0;
367
368	/*
369	* Make sure at least one submit queue exists.
370	*/
371	if (!submit_queues)
372	return -EINVAL;
373
374	/*
375	* Make sure that null_init_hctx() does not access nullb->queues[] past
376	* the end of that array.
377	*/
378	if (submit_queues > nr_cpu_ids || poll_queues > g_poll_queues)
379	return -EINVAL;
380
381	/*
382	* Keep previous and new queue numbers in nullb_device for reference in
383	* the call back function null_map_queues().
384	*/
385	dev->prev_submit_queues = dev->submit_queues;
386	dev->prev_poll_queues = dev->poll_queues;
387	dev->submit_queues = submit_queues;
388	dev->poll_queues = poll_queues;
389
390	set = dev->nullb->tag_set;
391	nr_hw_queues = submit_queues + poll_queues;
392	blk_mq_update_nr_hw_queues(set, nr_hw_queues);
393	ret = set->nr_hw_queues == nr_hw_queues ? 0 : -ENOMEM;
394
395	if (ret) {
396	/* on error, revert the queue numbers */
397	dev->submit_queues = dev->prev_submit_queues;
398	dev->poll_queues = dev->prev_poll_queues;
399	}
400
401	return ret;
402	}
403
404	static int nullb_apply_submit_queues(struct nullb_device *dev,
405	unsigned int submit_queues)
406	{
407	return nullb_update_nr_hw_queues(dev, submit_queues, poll_queues: dev->poll_queues);
408	}
409
410	static int nullb_apply_poll_queues(struct nullb_device *dev,
411	unsigned int poll_queues)
412	{
413	return nullb_update_nr_hw_queues(dev, submit_queues: dev->submit_queues, poll_queues);
414	}
415
416	NULLB_DEVICE_ATTR(size, ulong, NULL);
417	NULLB_DEVICE_ATTR(completion_nsec, ulong, NULL);
418	NULLB_DEVICE_ATTR(submit_queues, uint, nullb_apply_submit_queues);
419	NULLB_DEVICE_ATTR(poll_queues, uint, nullb_apply_poll_queues);
420	NULLB_DEVICE_ATTR(home_node, uint, NULL);
421	NULLB_DEVICE_ATTR(queue_mode, uint, NULL);
422	NULLB_DEVICE_ATTR(blocksize, uint, NULL);
423	NULLB_DEVICE_ATTR(max_sectors, uint, NULL);
424	NULLB_DEVICE_ATTR(irqmode, uint, NULL);
425	NULLB_DEVICE_ATTR(hw_queue_depth, uint, NULL);
426	NULLB_DEVICE_ATTR(index, uint, NULL);
427	NULLB_DEVICE_ATTR(blocking, bool, NULL);
428	NULLB_DEVICE_ATTR(use_per_node_hctx, bool, NULL);
429	NULLB_DEVICE_ATTR(memory_backed, bool, NULL);
430	NULLB_DEVICE_ATTR(discard, bool, NULL);
431	NULLB_DEVICE_ATTR(mbps, uint, NULL);
432	NULLB_DEVICE_ATTR(cache_size, ulong, NULL);
433	NULLB_DEVICE_ATTR(zoned, bool, NULL);
434	NULLB_DEVICE_ATTR(zone_size, ulong, NULL);
435	NULLB_DEVICE_ATTR(zone_capacity, ulong, NULL);
436	NULLB_DEVICE_ATTR(zone_nr_conv, uint, NULL);
437	NULLB_DEVICE_ATTR(zone_max_open, uint, NULL);
438	NULLB_DEVICE_ATTR(zone_max_active, uint, NULL);
439	NULLB_DEVICE_ATTR(virt_boundary, bool, NULL);
440	NULLB_DEVICE_ATTR(no_sched, bool, NULL);
441	NULLB_DEVICE_ATTR(shared_tags, bool, NULL);
442	NULLB_DEVICE_ATTR(shared_tag_bitmap, bool, NULL);
443
444	static ssize_t nullb_device_power_show(struct config_item *item, char *page)
445	{
446	return nullb_device_bool_attr_show(val: to_nullb_device(item)->power, page);
447	}
448
449	static ssize_t nullb_device_power_store(struct config_item *item,
450	const char *page, size_t count)
451	{
452	struct nullb_device *dev = to_nullb_device(item);
453	bool newp = false;
454	ssize_t ret;
455
456	ret = nullb_device_bool_attr_store(val: &newp, page, count);
457	if (ret < 0)
458	return ret;
459
460	if (!dev->power && newp) {
461	if (test_and_set_bit(nr: NULLB_DEV_FL_UP, addr: &dev->flags))
462	return count;
463	ret = null_add_dev(dev);
464	if (ret) {
465	clear_bit(nr: NULLB_DEV_FL_UP, addr: &dev->flags);
466	return ret;
467	}
468
469	set_bit(nr: NULLB_DEV_FL_CONFIGURED, addr: &dev->flags);
470	dev->power = newp;
471	} else if (dev->power && !newp) {
472	if (test_and_clear_bit(nr: NULLB_DEV_FL_UP, addr: &dev->flags)) {
473	mutex_lock(&lock);
474	dev->power = newp;
475	null_del_dev(nullb: dev->nullb);
476	mutex_unlock(lock: &lock);
477	}
478	clear_bit(nr: NULLB_DEV_FL_CONFIGURED, addr: &dev->flags);
479	}
480
481	return count;
482	}
483
484	CONFIGFS_ATTR(nullb_device_, power);
485
486	static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
487	{
488	struct nullb_device *t_dev = to_nullb_device(item);
489
490	return badblocks_show(bb: &t_dev->badblocks, page, unack: 0);
491	}
492
493	static ssize_t nullb_device_badblocks_store(struct config_item *item,
494	const char *page, size_t count)
495	{
496	struct nullb_device *t_dev = to_nullb_device(item);
497	char *orig, *buf, *tmp;
498	u64 start, end;
499	int ret;
500
501	orig = kstrndup(s: page, len: count, GFP_KERNEL);
502	if (!orig)
503	return -ENOMEM;
504
505	buf = strstrip(str: orig);
506
507	ret = -EINVAL;
508	if (buf[0] != '+' && buf[0] != '-')
509	goto out;
510	tmp = strchr(&buf[1], '-');
511	if (!tmp)
512	goto out;
513	*tmp = '\0';
514	ret = kstrtoull(s: buf + 1, base: 0, res: &start);
515	if (ret)
516	goto out;
517	ret = kstrtoull(s: tmp + 1, base: 0, res: &end);
518	if (ret)
519	goto out;
520	ret = -EINVAL;
521	if (start > end)
522	goto out;
523	/* enable badblocks */
524	cmpxchg(&t_dev->badblocks.shift, -1, 0);
525	if (buf[0] == '+')
526	ret = badblocks_set(bb: &t_dev->badblocks, s: start,
527	sectors: end - start + 1, acknowledged: 1);
528	else
529	ret = badblocks_clear(bb: &t_dev->badblocks, s: start,
530	sectors: end - start + 1);
531	if (ret == 0)
532	ret = count;
533	out:
534	kfree(objp: orig);
535	return ret;
536	}
537	CONFIGFS_ATTR(nullb_device_, badblocks);
538
539	static ssize_t nullb_device_zone_readonly_store(struct config_item *item,
540	const char *page, size_t count)
541	{
542	struct nullb_device *dev = to_nullb_device(item);
543
544	return zone_cond_store(dev, page, count, cond: BLK_ZONE_COND_READONLY);
545	}
546	CONFIGFS_ATTR_WO(nullb_device_, zone_readonly);
547
548	static ssize_t nullb_device_zone_offline_store(struct config_item *item,
549	const char *page, size_t count)
550	{
551	struct nullb_device *dev = to_nullb_device(item);
552
553	return zone_cond_store(dev, page, count, cond: BLK_ZONE_COND_OFFLINE);
554	}
555	CONFIGFS_ATTR_WO(nullb_device_, zone_offline);
556
557	static struct configfs_attribute *nullb_device_attrs[] = {
558	&nullb_device_attr_size,
559	&nullb_device_attr_completion_nsec,
560	&nullb_device_attr_submit_queues,
561	&nullb_device_attr_poll_queues,
562	&nullb_device_attr_home_node,
563	&nullb_device_attr_queue_mode,
564	&nullb_device_attr_blocksize,
565	&nullb_device_attr_max_sectors,
566	&nullb_device_attr_irqmode,
567	&nullb_device_attr_hw_queue_depth,
568	&nullb_device_attr_index,
569	&nullb_device_attr_blocking,
570	&nullb_device_attr_use_per_node_hctx,
571	&nullb_device_attr_power,
572	&nullb_device_attr_memory_backed,
573	&nullb_device_attr_discard,
574	&nullb_device_attr_mbps,
575	&nullb_device_attr_cache_size,
576	&nullb_device_attr_badblocks,
577	&nullb_device_attr_zoned,
578	&nullb_device_attr_zone_size,
579	&nullb_device_attr_zone_capacity,
580	&nullb_device_attr_zone_nr_conv,
581	&nullb_device_attr_zone_max_open,
582	&nullb_device_attr_zone_max_active,
583	&nullb_device_attr_zone_readonly,
584	&nullb_device_attr_zone_offline,
585	&nullb_device_attr_virt_boundary,
586	&nullb_device_attr_no_sched,
587	&nullb_device_attr_shared_tags,
588	&nullb_device_attr_shared_tag_bitmap,
589	NULL,
590	};
591
592	static void nullb_device_release(struct config_item *item)
593	{
594	struct nullb_device *dev = to_nullb_device(item);
595
596	null_free_device_storage(dev, is_cache: false);
597	null_free_dev(dev);
598	}
599
600	static struct configfs_item_operations nullb_device_ops = {
601	.release = nullb_device_release,
602	};
603
604	static const struct config_item_type nullb_device_type = {
605	.ct_item_ops = &nullb_device_ops,
606	.ct_attrs = nullb_device_attrs,
607	.ct_owner = THIS_MODULE,
608	};
609
610	#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
611
612	static void nullb_add_fault_config(struct nullb_device *dev)
613	{
614	fault_config_init(config: &dev->timeout_config, name: "timeout_inject");
615	fault_config_init(config: &dev->requeue_config, name: "requeue_inject");
616	fault_config_init(config: &dev->init_hctx_fault_config, name: "init_hctx_fault_inject");
617
618	configfs_add_default_group(new_group: &dev->timeout_config.group, group: &dev->group);
619	configfs_add_default_group(new_group: &dev->requeue_config.group, group: &dev->group);
620	configfs_add_default_group(new_group: &dev->init_hctx_fault_config.group, group: &dev->group);
621	}
622
623	#else
624
625	static void nullb_add_fault_config(struct nullb_device *dev)
626	{
627	}
628
629	#endif
630
631	static struct
632	config_group *nullb_group_make_group(struct config_group *group, const char *name)
633	{
634	struct nullb_device *dev;
635
636	if (null_find_dev_by_name(name))
637	return ERR_PTR(error: -EEXIST);
638
639	dev = null_alloc_dev();
640	if (!dev)
641	return ERR_PTR(error: -ENOMEM);
642
643	config_group_init_type_name(group: &dev->group, name, type: &nullb_device_type);
644	nullb_add_fault_config(dev);
645
646	return &dev->group;
647	}
648
649	static void
650	nullb_group_drop_item(struct config_group *group, struct config_item *item)
651	{
652	struct nullb_device *dev = to_nullb_device(item);
653
654	if (test_and_clear_bit(nr: NULLB_DEV_FL_UP, addr: &dev->flags)) {
655	mutex_lock(&lock);
656	dev->power = false;
657	null_del_dev(nullb: dev->nullb);
658	mutex_unlock(lock: &lock);
659	}
660
661	config_item_put(item);
662	}
663
664	static ssize_t memb_group_features_show(struct config_item *item, char *page)
665	{
666	return snprintf(buf: page, PAGE_SIZE,
667	fmt: "badblocks,blocking,blocksize,cache_size,"
668	"completion_nsec,discard,home_node,hw_queue_depth,"
669	"irqmode,max_sectors,mbps,memory_backed,no_sched,"
670	"poll_queues,power,queue_mode,shared_tag_bitmap,"
671	"shared_tags,size,submit_queues,use_per_node_hctx,"
672	"virt_boundary,zoned,zone_capacity,zone_max_active,"
673	"zone_max_open,zone_nr_conv,zone_offline,zone_readonly,"
674	"zone_size\n");
675	}
676
677	CONFIGFS_ATTR_RO(memb_group_, features);
678
679	static struct configfs_attribute *nullb_group_attrs[] = {
680	&memb_group_attr_features,
681	NULL,
682	};
683
684	static struct configfs_group_operations nullb_group_ops = {
685	.make_group = nullb_group_make_group,
686	.drop_item = nullb_group_drop_item,
687	};
688
689	static const struct config_item_type nullb_group_type = {
690	.ct_group_ops = &nullb_group_ops,
691	.ct_attrs = nullb_group_attrs,
692	.ct_owner = THIS_MODULE,
693	};
694
695	static struct configfs_subsystem nullb_subsys = {
696	.su_group = {
697	.cg_item = {
698	.ci_namebuf = "nullb",
699	.ci_type = &nullb_group_type,
700	},
701	},
702	};
703
704	static inline int null_cache_active(struct nullb *nullb)
705	{
706	return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
707	}
708
709	static struct nullb_device *null_alloc_dev(void)
710	{
711	struct nullb_device *dev;
712
713	dev = kzalloc(size: sizeof(*dev), GFP_KERNEL);
714	if (!dev)
715	return NULL;
716
717	#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
718	dev->timeout_config.attr = null_timeout_attr;
719	dev->requeue_config.attr = null_requeue_attr;
720	dev->init_hctx_fault_config.attr = null_init_hctx_attr;
721	#endif
722
723	INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
724	INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
725	if (badblocks_init(bb: &dev->badblocks, enable: 0)) {
726	kfree(objp: dev);
727	return NULL;
728	}
729
730	dev->size = g_gb * 1024;
731	dev->completion_nsec = g_completion_nsec;
732	dev->submit_queues = g_submit_queues;
733	dev->prev_submit_queues = g_submit_queues;
734	dev->poll_queues = g_poll_queues;
735	dev->prev_poll_queues = g_poll_queues;
736	dev->home_node = g_home_node;
737	dev->queue_mode = g_queue_mode;
738	dev->blocksize = g_bs;
739	dev->max_sectors = g_max_sectors;
740	dev->irqmode = g_irqmode;
741	dev->hw_queue_depth = g_hw_queue_depth;
742	dev->blocking = g_blocking;
743	dev->memory_backed = g_memory_backed;
744	dev->discard = g_discard;
745	dev->cache_size = g_cache_size;
746	dev->mbps = g_mbps;
747	dev->use_per_node_hctx = g_use_per_node_hctx;
748	dev->zoned = g_zoned;
749	dev->zone_size = g_zone_size;
750	dev->zone_capacity = g_zone_capacity;
751	dev->zone_nr_conv = g_zone_nr_conv;
752	dev->zone_max_open = g_zone_max_open;
753	dev->zone_max_active = g_zone_max_active;
754	dev->virt_boundary = g_virt_boundary;
755	dev->no_sched = g_no_sched;
756	dev->shared_tags = g_shared_tags;
757	dev->shared_tag_bitmap = g_shared_tag_bitmap;
758	return dev;
759	}
760
761	static void null_free_dev(struct nullb_device *dev)
762	{
763	if (!dev)
764	return;
765
766	null_free_zoned_dev(dev);
767	badblocks_exit(bb: &dev->badblocks);
768	kfree(objp: dev);
769	}
770
771	static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
772	{
773	struct nullb_cmd *cmd = container_of(timer, struct nullb_cmd, timer);
774
775	blk_mq_end_request(rq: blk_mq_rq_from_pdu(pdu: cmd), error: cmd->error);
776	return HRTIMER_NORESTART;
777	}
778
779	static void null_cmd_end_timer(struct nullb_cmd *cmd)
780	{
781	ktime_t kt = cmd->nq->dev->completion_nsec;
782
783	hrtimer_start(timer: &cmd->timer, tim: kt, mode: HRTIMER_MODE_REL);
784	}
785
786	static void null_complete_rq(struct request *rq)
787	{
788	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
789
790	blk_mq_end_request(rq, error: cmd->error);
791	}
792
793	static struct nullb_page *null_alloc_page(void)
794	{
795	struct nullb_page *t_page;
796
797	t_page = kmalloc(size: sizeof(struct nullb_page), GFP_NOIO);
798	if (!t_page)
799	return NULL;
800
801	t_page->page = alloc_pages(GFP_NOIO, order: 0);
802	if (!t_page->page) {
803	kfree(objp: t_page);
804	return NULL;
805	}
806
807	memset(t_page->bitmap, 0, sizeof(t_page->bitmap));
808	return t_page;
809	}
810
811	static void null_free_page(struct nullb_page *t_page)
812	{
813	__set_bit(NULLB_PAGE_FREE, t_page->bitmap);
814	if (test_bit(NULLB_PAGE_LOCK, t_page->bitmap))
815	return;
816	__free_page(t_page->page);
817	kfree(objp: t_page);
818	}
819
820	static bool null_page_empty(struct nullb_page *page)
821	{
822	int size = MAP_SZ - 2;
823
824	return find_first_bit(addr: page->bitmap, size) == size;
825	}
826
827	static void null_free_sector(struct nullb *nullb, sector_t sector,
828	bool is_cache)
829	{
830	unsigned int sector_bit;
831	u64 idx;
832	struct nullb_page *t_page, *ret;
833	struct radix_tree_root *root;
834
835	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
836	idx = sector >> PAGE_SECTORS_SHIFT;
837	sector_bit = (sector & SECTOR_MASK);
838
839	t_page = radix_tree_lookup(root, idx);
840	if (t_page) {
841	__clear_bit(sector_bit, t_page->bitmap);
842
843	if (null_page_empty(page: t_page)) {
844	ret = radix_tree_delete_item(root, idx, t_page);
845	WARN_ON(ret != t_page);
846	null_free_page(t_page: ret);
847	if (is_cache)
848	nullb->dev->curr_cache -= PAGE_SIZE;
849	}
850	}
851	}
852
853	static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
854	struct nullb_page *t_page, bool is_cache)
855	{
856	struct radix_tree_root *root;
857
858	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
859
860	if (radix_tree_insert(root, index: idx, t_page)) {
861	null_free_page(t_page);
862	t_page = radix_tree_lookup(root, idx);
863	WARN_ON(!t_page || t_page->page->index != idx);
864	} else if (is_cache)
865	nullb->dev->curr_cache += PAGE_SIZE;
866
867	return t_page;
868	}
869
870	static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
871	{
872	unsigned long pos = 0;
873	int nr_pages;
874	struct nullb_page *ret, *t_pages[FREE_BATCH];
875	struct radix_tree_root *root;
876
877	root = is_cache ? &dev->cache : &dev->data;
878
879	do {
880	int i;
881
882	nr_pages = radix_tree_gang_lookup(root,
883	results: (void **)t_pages, first_index: pos, FREE_BATCH);
884
885	for (i = 0; i < nr_pages; i++) {
886	pos = t_pages[i]->page->index;
887	ret = radix_tree_delete_item(root, pos, t_pages[i]);
888	WARN_ON(ret != t_pages[i]);
889	null_free_page(t_page: ret);
890	}
891
892	pos++;
893	} while (nr_pages == FREE_BATCH);
894
895	if (is_cache)
896	dev->curr_cache = 0;
897	}
898
899	static struct nullb_page *__null_lookup_page(struct nullb *nullb,
900	sector_t sector, bool for_write, bool is_cache)
901	{
902	unsigned int sector_bit;
903	u64 idx;
904	struct nullb_page *t_page;
905	struct radix_tree_root *root;
906
907	idx = sector >> PAGE_SECTORS_SHIFT;
908	sector_bit = (sector & SECTOR_MASK);
909
910	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
911	t_page = radix_tree_lookup(root, idx);
912	WARN_ON(t_page && t_page->page->index != idx);
913
914	if (t_page && (for_write || test_bit(sector_bit, t_page->bitmap)))
915	return t_page;
916
917	return NULL;
918	}
919
920	static struct nullb_page *null_lookup_page(struct nullb *nullb,
921	sector_t sector, bool for_write, bool ignore_cache)
922	{
923	struct nullb_page *page = NULL;
924
925	if (!ignore_cache)
926	page = __null_lookup_page(nullb, sector, for_write, is_cache: true);
927	if (page)
928	return page;
929	return __null_lookup_page(nullb, sector, for_write, is_cache: false);
930	}
931
932	static struct nullb_page *null_insert_page(struct nullb *nullb,
933	sector_t sector, bool ignore_cache)
934	__releases(&nullb->lock)
935	__acquires(&nullb->lock)
936	{
937	u64 idx;
938	struct nullb_page *t_page;
939
940	t_page = null_lookup_page(nullb, sector, for_write: true, ignore_cache);
941	if (t_page)
942	return t_page;
943
944	spin_unlock_irq(lock: &nullb->lock);
945
946	t_page = null_alloc_page();
947	if (!t_page)
948	goto out_lock;
949
950	if (radix_tree_preload(GFP_NOIO))
951	goto out_freepage;
952
953	spin_lock_irq(lock: &nullb->lock);
954	idx = sector >> PAGE_SECTORS_SHIFT;
955	t_page->page->index = idx;
956	t_page = null_radix_tree_insert(nullb, idx, t_page, is_cache: !ignore_cache);
957	radix_tree_preload_end();
958
959	return t_page;
960	out_freepage:
961	null_free_page(t_page);
962	out_lock:
963	spin_lock_irq(lock: &nullb->lock);
964	return null_lookup_page(nullb, sector, for_write: true, ignore_cache);
965	}
966
967	static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
968	{
969	int i;
970	unsigned int offset;
971	u64 idx;
972	struct nullb_page *t_page, *ret;
973	void *dst, *src;
974
975	idx = c_page->page->index;
976
977	t_page = null_insert_page(nullb, sector: idx << PAGE_SECTORS_SHIFT, ignore_cache: true);
978
979	__clear_bit(NULLB_PAGE_LOCK, c_page->bitmap);
980	if (test_bit(NULLB_PAGE_FREE, c_page->bitmap)) {
981	null_free_page(t_page: c_page);
982	if (t_page && null_page_empty(page: t_page)) {
983	ret = radix_tree_delete_item(&nullb->dev->data,
984	idx, t_page);
985	null_free_page(t_page);
986	}
987	return 0;
988	}
989
990	if (!t_page)
991	return -ENOMEM;
992
993	src = kmap_local_page(page: c_page->page);
994	dst = kmap_local_page(page: t_page->page);
995
996	for (i = 0; i < PAGE_SECTORS;
997	i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
998	if (test_bit(i, c_page->bitmap)) {
999	offset = (i << SECTOR_SHIFT);
1000	memcpy(dst + offset, src + offset,
1001	nullb->dev->blocksize);
1002	__set_bit(i, t_page->bitmap);
1003	}
1004	}
1005
1006	kunmap_local(dst);
1007	kunmap_local(src);
1008
1009	ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
1010	null_free_page(t_page: ret);
1011	nullb->dev->curr_cache -= PAGE_SIZE;
1012
1013	return 0;
1014	}
1015
1016	static int null_make_cache_space(struct nullb *nullb, unsigned long n)
1017	{
1018	int i, err, nr_pages;
1019	struct nullb_page *c_pages[FREE_BATCH];
1020	unsigned long flushed = 0, one_round;
1021
1022	again:
1023	if ((nullb->dev->cache_size * 1024 * 1024) >
1024	nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
1025	return 0;
1026
1027	nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
1028	results: (void **)c_pages, first_index: nullb->cache_flush_pos, FREE_BATCH);
1029	/*
1030	* nullb_flush_cache_page could unlock before using the c_pages. To
1031	* avoid race, we don't allow page free
1032	*/
1033	for (i = 0; i < nr_pages; i++) {
1034	nullb->cache_flush_pos = c_pages[i]->page->index;
1035	/*
1036	* We found the page which is being flushed to disk by other
1037	* threads
1038	*/
1039	if (test_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap))
1040	c_pages[i] = NULL;
1041	else
1042	__set_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap);
1043	}
1044
1045	one_round = 0;
1046	for (i = 0; i < nr_pages; i++) {
1047	if (c_pages[i] == NULL)
1048	continue;
1049	err = null_flush_cache_page(nullb, c_page: c_pages[i]);
1050	if (err)
1051	return err;
1052	one_round++;
1053	}
1054	flushed += one_round << PAGE_SHIFT;
1055
1056	if (n > flushed) {
1057	if (nr_pages == 0)
1058	nullb->cache_flush_pos = 0;
1059	if (one_round == 0) {
1060	/* give other threads a chance */
1061	spin_unlock_irq(lock: &nullb->lock);
1062	spin_lock_irq(lock: &nullb->lock);
1063	}
1064	goto again;
1065	}
1066	return 0;
1067	}
1068
1069	static int copy_to_nullb(struct nullb *nullb, struct page *source,
1070	unsigned int off, sector_t sector, size_t n, bool is_fua)
1071	{
1072	size_t temp, count = 0;
1073	unsigned int offset;
1074	struct nullb_page *t_page;
1075
1076	while (count < n) {
1077	temp = min_t(size_t, nullb->dev->blocksize, n - count);
1078
1079	if (null_cache_active(nullb) && !is_fua)
1080	null_make_cache_space(nullb, PAGE_SIZE);
1081
1082	offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
1083	t_page = null_insert_page(nullb, sector,
1084	ignore_cache: !null_cache_active(nullb) || is_fua);
1085	if (!t_page)
1086	return -ENOSPC;
1087
1088	memcpy_page(dst_page: t_page->page, dst_off: offset, src_page: source, src_off: off + count, len: temp);
1089
1090	__set_bit(sector & SECTOR_MASK, t_page->bitmap);
1091
1092	if (is_fua)
1093	null_free_sector(nullb, sector, is_cache: true);
1094
1095	count += temp;
1096	sector += temp >> SECTOR_SHIFT;
1097	}
1098	return 0;
1099	}
1100
1101	static int copy_from_nullb(struct nullb *nullb, struct page *dest,
1102	unsigned int off, sector_t sector, size_t n)
1103	{
1104	size_t temp, count = 0;
1105	unsigned int offset;
1106	struct nullb_page *t_page;
1107
1108	while (count < n) {
1109	temp = min_t(size_t, nullb->dev->blocksize, n - count);
1110
1111	offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
1112	t_page = null_lookup_page(nullb, sector, for_write: false,
1113	ignore_cache: !null_cache_active(nullb));
1114
1115	if (t_page)
1116	memcpy_page(dst_page: dest, dst_off: off + count, src_page: t_page->page, src_off: offset,
1117	len: temp);
1118	else
1119	zero_user(page: dest, start: off + count, size: temp);
1120
1121	count += temp;
1122	sector += temp >> SECTOR_SHIFT;
1123	}
1124	return 0;
1125	}
1126
1127	static void nullb_fill_pattern(struct nullb *nullb, struct page *page,
1128	unsigned int len, unsigned int off)
1129	{
1130	memset_page(page, offset: off, val: 0xff, len);
1131	}
1132
1133	blk_status_t null_handle_discard(struct nullb_device *dev,
1134	sector_t sector, sector_t nr_sectors)
1135	{
1136	struct nullb *nullb = dev->nullb;
1137	size_t n = nr_sectors << SECTOR_SHIFT;
1138	size_t temp;
1139
1140	spin_lock_irq(lock: &nullb->lock);
1141	while (n > 0) {
1142	temp = min_t(size_t, n, dev->blocksize);
1143	null_free_sector(nullb, sector, is_cache: false);
1144	if (null_cache_active(nullb))
1145	null_free_sector(nullb, sector, is_cache: true);
1146	sector += temp >> SECTOR_SHIFT;
1147	n -= temp;
1148	}
1149	spin_unlock_irq(lock: &nullb->lock);
1150
1151	return BLK_STS_OK;
1152	}
1153
1154	static int null_handle_flush(struct nullb *nullb)
1155	{
1156	int err;
1157
1158	if (!null_cache_active(nullb))
1159	return 0;
1160
1161	spin_lock_irq(lock: &nullb->lock);
1162	while (true) {
1163	err = null_make_cache_space(nullb,
1164	n: nullb->dev->cache_size * 1024 * 1024);
1165	if (err || nullb->dev->curr_cache == 0)
1166	break;
1167	}
1168
1169	WARN_ON(!radix_tree_empty(&nullb->dev->cache));
1170	spin_unlock_irq(lock: &nullb->lock);
1171	return err;
1172	}
1173
1174	static int null_transfer(struct nullb *nullb, struct page *page,
1175	unsigned int len, unsigned int off, bool is_write, sector_t sector,
1176	bool is_fua)
1177	{
1178	struct nullb_device *dev = nullb->dev;
1179	unsigned int valid_len = len;
1180	int err = 0;
1181
1182	if (!is_write) {
1183	if (dev->zoned)
1184	valid_len = null_zone_valid_read_len(nullb,
1185	sector, len);
1186
1187	if (valid_len) {
1188	err = copy_from_nullb(nullb, dest: page, off,
1189	sector, n: valid_len);
1190	off += valid_len;
1191	len -= valid_len;
1192	}
1193
1194	if (len)
1195	nullb_fill_pattern(nullb, page, len, off);
1196	flush_dcache_page(page);
1197	} else {
1198	flush_dcache_page(page);
1199	err = copy_to_nullb(nullb, source: page, off, sector, n: len, is_fua);
1200	}
1201
1202	return err;
1203	}
1204
1205	static int null_handle_rq(struct nullb_cmd *cmd)
1206	{
1207	struct request *rq = blk_mq_rq_from_pdu(pdu: cmd);
1208	struct nullb *nullb = cmd->nq->dev->nullb;
1209	int err;
1210	unsigned int len;
1211	sector_t sector = blk_rq_pos(rq);
1212	struct req_iterator iter;
1213	struct bio_vec bvec;
1214
1215	spin_lock_irq(lock: &nullb->lock);
1216	rq_for_each_segment(bvec, rq, iter) {
1217	len = bvec.bv_len;
1218	err = null_transfer(nullb, page: bvec.bv_page, len, off: bvec.bv_offset,
1219	is_write: op_is_write(op: req_op(req: rq)), sector,
1220	is_fua: rq->cmd_flags & REQ_FUA);
1221	if (err) {
1222	spin_unlock_irq(lock: &nullb->lock);
1223	return err;
1224	}
1225	sector += len >> SECTOR_SHIFT;
1226	}
1227	spin_unlock_irq(lock: &nullb->lock);
1228
1229	return 0;
1230	}
1231
1232	static inline blk_status_t null_handle_throttled(struct nullb_cmd *cmd)
1233	{
1234	struct nullb_device *dev = cmd->nq->dev;
1235	struct nullb *nullb = dev->nullb;
1236	blk_status_t sts = BLK_STS_OK;
1237	struct request *rq = blk_mq_rq_from_pdu(pdu: cmd);
1238
1239	if (!hrtimer_active(timer: &nullb->bw_timer))
1240	hrtimer_restart(timer: &nullb->bw_timer);
1241
1242	if (atomic_long_sub_return(i: blk_rq_bytes(rq), v: &nullb->cur_bytes) < 0) {
1243	blk_mq_stop_hw_queues(q: nullb->q);
1244	/* race with timer */
1245	if (atomic_long_read(v: &nullb->cur_bytes) > 0)
1246	blk_mq_start_stopped_hw_queues(q: nullb->q, async: true);
1247	/* requeue request */
1248	sts = BLK_STS_DEV_RESOURCE;
1249	}
1250	return sts;
1251	}
1252
1253	static inline blk_status_t null_handle_badblocks(struct nullb_cmd *cmd,
1254	sector_t sector,
1255	sector_t nr_sectors)
1256	{
1257	struct badblocks *bb = &cmd->nq->dev->badblocks;
1258	sector_t first_bad;
1259	int bad_sectors;
1260
1261	if (badblocks_check(bb, s: sector, sectors: nr_sectors, first_bad: &first_bad, bad_sectors: &bad_sectors))
1262	return BLK_STS_IOERR;
1263
1264	return BLK_STS_OK;
1265	}
1266
1267	static inline blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd,
1268	enum req_op op,
1269	sector_t sector,
1270	sector_t nr_sectors)
1271	{
1272	struct nullb_device *dev = cmd->nq->dev;
1273
1274	if (op == REQ_OP_DISCARD)
1275	return null_handle_discard(dev, sector, nr_sectors);
1276	return errno_to_blk_status(errno: null_handle_rq(cmd));
1277
1278	}
1279
1280	static void nullb_zero_read_cmd_buffer(struct nullb_cmd *cmd)
1281	{
1282	struct request *rq = blk_mq_rq_from_pdu(pdu: cmd);
1283	struct nullb_device *dev = cmd->nq->dev;
1284	struct bio *bio;
1285
1286	if (!dev->memory_backed && req_op(req: rq) == REQ_OP_READ) {
1287	__rq_for_each_bio(bio, rq)
1288	zero_fill_bio(bio);
1289	}
1290	}
1291
1292	static inline void nullb_complete_cmd(struct nullb_cmd *cmd)
1293	{
1294	struct request *rq = blk_mq_rq_from_pdu(pdu: cmd);
1295
1296	/*
1297	* Since root privileges are required to configure the null_blk
1298	* driver, it is fine that this driver does not initialize the
1299	* data buffers of read commands. Zero-initialize these buffers
1300	* anyway if KMSAN is enabled to prevent that KMSAN complains
1301	* about null_blk not initializing read data buffers.
1302	*/
1303	if (IS_ENABLED(CONFIG_KMSAN))
1304	nullb_zero_read_cmd_buffer(cmd);
1305
1306	/* Complete IO by inline, softirq or timer */
1307	switch (cmd->nq->dev->irqmode) {
1308	case NULL_IRQ_SOFTIRQ:
1309	blk_mq_complete_request(rq);
1310	break;
1311	case NULL_IRQ_NONE:
1312	blk_mq_end_request(rq, error: cmd->error);
1313	break;
1314	case NULL_IRQ_TIMER:
1315	null_cmd_end_timer(cmd);
1316	break;
1317	}
1318	}
1319
1320	blk_status_t null_process_cmd(struct nullb_cmd *cmd, enum req_op op,
1321	sector_t sector, unsigned int nr_sectors)
1322	{
1323	struct nullb_device *dev = cmd->nq->dev;
1324	blk_status_t ret;
1325
1326	if (dev->badblocks.shift != -1) {
1327	ret = null_handle_badblocks(cmd, sector, nr_sectors);
1328	if (ret != BLK_STS_OK)
1329	return ret;
1330	}
1331
1332	if (dev->memory_backed)
1333	return null_handle_memory_backed(cmd, op, sector, nr_sectors);
1334
1335	return BLK_STS_OK;
1336	}
1337
1338	static void null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
1339	sector_t nr_sectors, enum req_op op)
1340	{
1341	struct nullb_device *dev = cmd->nq->dev;
1342	struct nullb *nullb = dev->nullb;
1343	blk_status_t sts;
1344
1345	if (op == REQ_OP_FLUSH) {
1346	cmd->error = errno_to_blk_status(errno: null_handle_flush(nullb));
1347	goto out;
1348	}
1349
1350	if (dev->zoned)
1351	sts = null_process_zoned_cmd(cmd, op, sector, nr_sectors);
1352	else
1353	sts = null_process_cmd(cmd, op, sector, nr_sectors);
1354
1355	/* Do not overwrite errors (e.g. timeout errors) */
1356	if (cmd->error == BLK_STS_OK)
1357	cmd->error = sts;
1358
1359	out:
1360	nullb_complete_cmd(cmd);
1361	}
1362
1363	static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
1364	{
1365	struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
1366	ktime_t timer_interval = ktime_set(secs: 0, TIMER_INTERVAL);
1367	unsigned int mbps = nullb->dev->mbps;
1368
1369	if (atomic_long_read(v: &nullb->cur_bytes) == mb_per_tick(mbps))
1370	return HRTIMER_NORESTART;
1371
1372	atomic_long_set(v: &nullb->cur_bytes, i: mb_per_tick(mbps));
1373	blk_mq_start_stopped_hw_queues(q: nullb->q, async: true);
1374
1375	hrtimer_forward_now(timer: &nullb->bw_timer, interval: timer_interval);
1376
1377	return HRTIMER_RESTART;
1378	}
1379
1380	static void nullb_setup_bwtimer(struct nullb *nullb)
1381	{
1382	ktime_t timer_interval = ktime_set(secs: 0, TIMER_INTERVAL);
1383
1384	hrtimer_init(timer: &nullb->bw_timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
1385	nullb->bw_timer.function = nullb_bwtimer_fn;
1386	atomic_long_set(v: &nullb->cur_bytes, i: mb_per_tick(mbps: nullb->dev->mbps));
1387	hrtimer_start(timer: &nullb->bw_timer, tim: timer_interval, mode: HRTIMER_MODE_REL);
1388	}
1389
1390	#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1391
1392	static bool should_timeout_request(struct request *rq)
1393	{
1394	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1395	struct nullb_device *dev = cmd->nq->dev;
1396
1397	return should_fail(attr: &dev->timeout_config.attr, size: 1);
1398	}
1399
1400	static bool should_requeue_request(struct request *rq)
1401	{
1402	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1403	struct nullb_device *dev = cmd->nq->dev;
1404
1405	return should_fail(attr: &dev->requeue_config.attr, size: 1);
1406	}
1407
1408	static bool should_init_hctx_fail(struct nullb_device *dev)
1409	{
1410	return should_fail(attr: &dev->init_hctx_fault_config.attr, size: 1);
1411	}
1412
1413	#else
1414
1415	static bool should_timeout_request(struct request *rq)
1416	{
1417	return false;
1418	}
1419
1420	static bool should_requeue_request(struct request *rq)
1421	{
1422	return false;
1423	}
1424
1425	static bool should_init_hctx_fail(struct nullb_device *dev)
1426	{
1427	return false;
1428	}
1429
1430	#endif
1431
1432	static void null_map_queues(struct blk_mq_tag_set *set)
1433	{
1434	struct nullb *nullb = set->driver_data;
1435	int i, qoff;
1436	unsigned int submit_queues = g_submit_queues;
1437	unsigned int poll_queues = g_poll_queues;
1438
1439	if (nullb) {
1440	struct nullb_device *dev = nullb->dev;
1441
1442	/*
1443	* Refer nr_hw_queues of the tag set to check if the expected
1444	* number of hardware queues are prepared. If block layer failed
1445	* to prepare them, use previous numbers of submit queues and
1446	* poll queues to map queues.
1447	*/
1448	if (set->nr_hw_queues ==
1449	dev->submit_queues + dev->poll_queues) {
1450	submit_queues = dev->submit_queues;
1451	poll_queues = dev->poll_queues;
1452	} else if (set->nr_hw_queues ==
1453	dev->prev_submit_queues + dev->prev_poll_queues) {
1454	submit_queues = dev->prev_submit_queues;
1455	poll_queues = dev->prev_poll_queues;
1456	} else {
1457	pr_warn("tag set has unexpected nr_hw_queues: %d\n",
1458	set->nr_hw_queues);
1459	WARN_ON_ONCE(true);
1460	submit_queues = 1;
1461	poll_queues = 0;
1462	}
1463	}
1464
1465	for (i = 0, qoff = 0; i < set->nr_maps; i++) {
1466	struct blk_mq_queue_map *map = &set->map[i];
1467
1468	switch (i) {
1469	case HCTX_TYPE_DEFAULT:
1470	map->nr_queues = submit_queues;
1471	break;
1472	case HCTX_TYPE_READ:
1473	map->nr_queues = 0;
1474	continue;
1475	case HCTX_TYPE_POLL:
1476	map->nr_queues = poll_queues;
1477	break;
1478	}
1479	map->queue_offset = qoff;
1480	qoff += map->nr_queues;
1481	blk_mq_map_queues(qmap: map);
1482	}
1483	}
1484
1485	static int null_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
1486	{
1487	struct nullb_queue *nq = hctx->driver_data;
1488	LIST_HEAD(list);
1489	int nr = 0;
1490	struct request *rq;
1491
1492	spin_lock(lock: &nq->poll_lock);
1493	list_splice_init(list: &nq->poll_list, head: &list);
1494	list_for_each_entry(rq, &list, queuelist)
1495	blk_mq_set_request_complete(rq);
1496	spin_unlock(lock: &nq->poll_lock);
1497
1498	while (!list_empty(head: &list)) {
1499	struct nullb_cmd *cmd;
1500	struct request *req;
1501
1502	req = list_first_entry(&list, struct request, queuelist);
1503	list_del_init(entry: &req->queuelist);
1504	cmd = blk_mq_rq_to_pdu(rq: req);
1505	cmd->error = null_process_cmd(cmd, op: req_op(req), sector: blk_rq_pos(rq: req),
1506	nr_sectors: blk_rq_sectors(rq: req));
1507	if (!blk_mq_add_to_batch(req, iob, ioerror: (__force int) cmd->error,
1508	complete: blk_mq_end_request_batch))
1509	blk_mq_end_request(rq: req, error: cmd->error);
1510	nr++;
1511	}
1512
1513	return nr;
1514	}
1515
1516	static enum blk_eh_timer_return null_timeout_rq(struct request *rq)
1517	{
1518	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1519	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1520
1521	if (hctx->type == HCTX_TYPE_POLL) {
1522	struct nullb_queue *nq = hctx->driver_data;
1523
1524	spin_lock(lock: &nq->poll_lock);
1525	/* The request may have completed meanwhile. */
1526	if (blk_mq_request_completed(rq)) {
1527	spin_unlock(lock: &nq->poll_lock);
1528	return BLK_EH_DONE;
1529	}
1530	list_del_init(entry: &rq->queuelist);
1531	spin_unlock(lock: &nq->poll_lock);
1532	}
1533
1534	pr_info("rq %p timed out\n", rq);
1535
1536	/*
1537	* If the device is marked as blocking (i.e. memory backed or zoned
1538	* device), the submission path may be blocked waiting for resources
1539	* and cause real timeouts. For these real timeouts, the submission
1540	* path will complete the request using blk_mq_complete_request().
1541	* Only fake timeouts need to execute blk_mq_complete_request() here.
1542	*/
1543	cmd->error = BLK_STS_TIMEOUT;
1544	if (cmd->fake_timeout || hctx->type == HCTX_TYPE_POLL)
1545	blk_mq_complete_request(rq);
1546	return BLK_EH_DONE;
1547	}
1548
1549	static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
1550	const struct blk_mq_queue_data *bd)
1551	{
1552	struct request *rq = bd->rq;
1553	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1554	struct nullb_queue *nq = hctx->driver_data;
1555	sector_t nr_sectors = blk_rq_sectors(rq);
1556	sector_t sector = blk_rq_pos(rq);
1557	const bool is_poll = hctx->type == HCTX_TYPE_POLL;
1558
1559	might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1560
1561	if (!is_poll && nq->dev->irqmode == NULL_IRQ_TIMER) {
1562	hrtimer_init(timer: &cmd->timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
1563	cmd->timer.function = null_cmd_timer_expired;
1564	}
1565	cmd->error = BLK_STS_OK;
1566	cmd->nq = nq;
1567	cmd->fake_timeout = should_timeout_request(rq) ||
1568	blk_should_fake_timeout(q: rq->q);
1569
1570	if (should_requeue_request(rq)) {
1571	/*
1572	* Alternate between hitting the core BUSY path, and the
1573	* driver driven requeue path
1574	*/
1575	nq->requeue_selection++;
1576	if (nq->requeue_selection & 1)
1577	return BLK_STS_RESOURCE;
1578	blk_mq_requeue_request(rq, kick_requeue_list: true);
1579	return BLK_STS_OK;
1580	}
1581
1582	if (test_bit(NULLB_DEV_FL_THROTTLED, &nq->dev->flags)) {
1583	blk_status_t sts = null_handle_throttled(cmd);
1584
1585	if (sts != BLK_STS_OK)
1586	return sts;
1587	}
1588
1589	blk_mq_start_request(rq);
1590
1591	if (is_poll) {
1592	spin_lock(lock: &nq->poll_lock);
1593	list_add_tail(new: &rq->queuelist, head: &nq->poll_list);
1594	spin_unlock(lock: &nq->poll_lock);
1595	return BLK_STS_OK;
1596	}
1597	if (cmd->fake_timeout)
1598	return BLK_STS_OK;
1599
1600	null_handle_cmd(cmd, sector, nr_sectors, op: req_op(req: rq));
1601	return BLK_STS_OK;
1602	}
1603
1604	static void null_queue_rqs(struct request **rqlist)
1605	{
1606	struct request *requeue_list = NULL;
1607	struct request **requeue_lastp = &requeue_list;
1608	struct blk_mq_queue_data bd = { };
1609	blk_status_t ret;
1610
1611	do {
1612	struct request *rq = rq_list_pop(rqlist);
1613
1614	bd.rq = rq;
1615	ret = null_queue_rq(hctx: rq->mq_hctx, bd: &bd);
1616	if (ret != BLK_STS_OK)
1617	rq_list_add_tail(&requeue_lastp, rq);
1618	} while (!rq_list_empty(*rqlist));
1619
1620	*rqlist = requeue_list;
1621	}
1622
1623	static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
1624	{
1625	nq->dev = nullb->dev;
1626	INIT_LIST_HEAD(list: &nq->poll_list);
1627	spin_lock_init(&nq->poll_lock);
1628	}
1629
1630	static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
1631	unsigned int hctx_idx)
1632	{
1633	struct nullb *nullb = hctx->queue->queuedata;
1634	struct nullb_queue *nq;
1635
1636	if (should_init_hctx_fail(dev: nullb->dev))
1637	return -EFAULT;
1638
1639	nq = &nullb->queues[hctx_idx];
1640	hctx->driver_data = nq;
1641	null_init_queue(nullb, nq);
1642
1643	return 0;
1644	}
1645
1646	static const struct blk_mq_ops null_mq_ops = {
1647	.queue_rq = null_queue_rq,
1648	.queue_rqs = null_queue_rqs,
1649	.complete = null_complete_rq,
1650	.timeout = null_timeout_rq,
1651	.poll = null_poll,
1652	.map_queues = null_map_queues,
1653	.init_hctx = null_init_hctx,
1654	};
1655
1656	static void null_del_dev(struct nullb *nullb)
1657	{
1658	struct nullb_device *dev;
1659
1660	if (!nullb)
1661	return;
1662
1663	dev = nullb->dev;
1664
1665	ida_free(&nullb_indexes, id: nullb->index);
1666
1667	list_del_init(entry: &nullb->list);
1668
1669	del_gendisk(gp: nullb->disk);
1670
1671	if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
1672	hrtimer_cancel(timer: &nullb->bw_timer);
1673	atomic_long_set(v: &nullb->cur_bytes, LONG_MAX);
1674	blk_mq_start_stopped_hw_queues(q: nullb->q, async: true);
1675	}
1676
1677	put_disk(disk: nullb->disk);
1678	if (nullb->tag_set == &nullb->__tag_set)
1679	blk_mq_free_tag_set(set: nullb->tag_set);
1680	kfree(objp: nullb->queues);
1681	if (null_cache_active(nullb))
1682	null_free_device_storage(dev: nullb->dev, is_cache: true);
1683	kfree(objp: nullb);
1684	dev->nullb = NULL;
1685	}
1686
1687	static void null_config_discard(struct nullb *nullb, struct queue_limits *lim)
1688	{
1689	if (nullb->dev->discard == false)
1690	return;
1691
1692	if (!nullb->dev->memory_backed) {
1693	nullb->dev->discard = false;
1694	pr_info("discard option is ignored without memory backing\n");
1695	return;
1696	}
1697
1698	if (nullb->dev->zoned) {
1699	nullb->dev->discard = false;
1700	pr_info("discard option is ignored in zoned mode\n");
1701	return;
1702	}
1703
1704	lim->max_hw_discard_sectors = UINT_MAX >> 9;
1705	}
1706
1707	static const struct block_device_operations null_ops = {
1708	.owner = THIS_MODULE,
1709	.report_zones = null_report_zones,
1710	};
1711
1712	static int setup_queues(struct nullb *nullb)
1713	{
1714	int nqueues = nr_cpu_ids;
1715
1716	if (g_poll_queues)
1717	nqueues += g_poll_queues;
1718
1719	nullb->queues = kcalloc(n: nqueues, size: sizeof(struct nullb_queue),
1720	GFP_KERNEL);
1721	if (!nullb->queues)
1722	return -ENOMEM;
1723
1724	return 0;
1725	}
1726
1727	static int null_init_tag_set(struct blk_mq_tag_set *set, int poll_queues)
1728	{
1729	set->ops = &null_mq_ops;
1730	set->cmd_size = sizeof(struct nullb_cmd);
1731	set->timeout = 5 * HZ;
1732	set->nr_maps = 1;
1733	if (poll_queues) {
1734	set->nr_hw_queues += poll_queues;
1735	set->nr_maps += 2;
1736	}
1737	return blk_mq_alloc_tag_set(set);
1738	}
1739
1740	static int null_init_global_tag_set(void)
1741	{
1742	int error;
1743
1744	if (tag_set.ops)
1745	return 0;
1746
1747	tag_set.nr_hw_queues = g_submit_queues;
1748	tag_set.queue_depth = g_hw_queue_depth;
1749	tag_set.numa_node = g_home_node;
1750	tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
1751	if (g_no_sched)
1752	tag_set.flags |= BLK_MQ_F_NO_SCHED;
1753	if (g_shared_tag_bitmap)
1754	tag_set.flags |= BLK_MQ_F_TAG_HCTX_SHARED;
1755	if (g_blocking)
1756	tag_set.flags |= BLK_MQ_F_BLOCKING;
1757
1758	error = null_init_tag_set(set: &tag_set, poll_queues: g_poll_queues);
1759	if (error)
1760	tag_set.ops = NULL;
1761	return error;
1762	}
1763
1764	static int null_setup_tagset(struct nullb *nullb)
1765	{
1766	if (nullb->dev->shared_tags) {
1767	nullb->tag_set = &tag_set;
1768	return null_init_global_tag_set();
1769	}
1770
1771	nullb->tag_set = &nullb->__tag_set;
1772	nullb->tag_set->driver_data = nullb;
1773	nullb->tag_set->nr_hw_queues = nullb->dev->submit_queues;
1774	nullb->tag_set->queue_depth = nullb->dev->hw_queue_depth;
1775	nullb->tag_set->numa_node = nullb->dev->home_node;
1776	nullb->tag_set->flags = BLK_MQ_F_SHOULD_MERGE;
1777	if (nullb->dev->no_sched)
1778	nullb->tag_set->flags |= BLK_MQ_F_NO_SCHED;
1779	if (nullb->dev->shared_tag_bitmap)
1780	nullb->tag_set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;
1781	if (nullb->dev->blocking)
1782	nullb->tag_set->flags |= BLK_MQ_F_BLOCKING;
1783	return null_init_tag_set(set: nullb->tag_set, poll_queues: nullb->dev->poll_queues);
1784	}
1785
1786	static int null_validate_conf(struct nullb_device *dev)
1787	{
1788	if (dev->queue_mode == NULL_Q_RQ) {
1789	pr_err("legacy IO path is no longer available\n");
1790	return -EINVAL;
1791	}
1792	if (dev->queue_mode == NULL_Q_BIO) {
1793	pr_err("BIO-based IO path is no longer available, using blk-mq instead.\n");
1794	dev->queue_mode = NULL_Q_MQ;
1795	}
1796
1797	dev->blocksize = round_down(dev->blocksize, 512);
1798	dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
1799
1800	if (dev->use_per_node_hctx) {
1801	if (dev->submit_queues != nr_online_nodes)
1802	dev->submit_queues = nr_online_nodes;
1803	} else if (dev->submit_queues > nr_cpu_ids)
1804	dev->submit_queues = nr_cpu_ids;
1805	else if (dev->submit_queues == 0)
1806	dev->submit_queues = 1;
1807	dev->prev_submit_queues = dev->submit_queues;
1808
1809	if (dev->poll_queues > g_poll_queues)
1810	dev->poll_queues = g_poll_queues;
1811	dev->prev_poll_queues = dev->poll_queues;
1812	dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
1813
1814	/* Do memory allocation, so set blocking */
1815	if (dev->memory_backed)
1816	dev->blocking = true;
1817	else /* cache is meaningless */
1818	dev->cache_size = 0;
1819	dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
1820	dev->cache_size);
1821	dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
1822
1823	if (dev->zoned &&
1824	(!dev->zone_size || !is_power_of_2(n: dev->zone_size))) {
1825	pr_err("zone_size must be power-of-two\n");
1826	return -EINVAL;
1827	}
1828
1829	return 0;
1830	}
1831
1832	#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1833	static bool __null_setup_fault(struct fault_attr *attr, char *str)
1834	{
1835	if (!str[0])
1836	return true;
1837
1838	if (!setup_fault_attr(attr, str))
1839	return false;
1840
1841	attr->verbose = 0;
1842	return true;
1843	}
1844	#endif
1845
1846	static bool null_setup_fault(void)
1847	{
1848	#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1849	if (!__null_setup_fault(attr: &null_timeout_attr, str: g_timeout_str))
1850	return false;
1851	if (!__null_setup_fault(attr: &null_requeue_attr, str: g_requeue_str))
1852	return false;
1853	if (!__null_setup_fault(attr: &null_init_hctx_attr, str: g_init_hctx_str))
1854	return false;
1855	#endif
1856	return true;
1857	}
1858
1859	static int null_add_dev(struct nullb_device *dev)
1860	{
1861	struct queue_limits lim = {
1862	.logical_block_size = dev->blocksize,
1863	.physical_block_size = dev->blocksize,
1864	.max_hw_sectors = dev->max_sectors,
1865	};
1866
1867	struct nullb *nullb;
1868	int rv;
1869
1870	rv = null_validate_conf(dev);
1871	if (rv)
1872	return rv;
1873
1874	nullb = kzalloc_node(size: sizeof(*nullb), GFP_KERNEL, node: dev->home_node);
1875	if (!nullb) {
1876	rv = -ENOMEM;
1877	goto out;
1878	}
1879	nullb->dev = dev;
1880	dev->nullb = nullb;
1881
1882	spin_lock_init(&nullb->lock);
1883
1884	rv = setup_queues(nullb);
1885	if (rv)
1886	goto out_free_nullb;
1887
1888	rv = null_setup_tagset(nullb);
1889	if (rv)
1890	goto out_cleanup_queues;
1891
1892	if (dev->virt_boundary)
1893	lim.virt_boundary_mask = PAGE_SIZE - 1;
1894	null_config_discard(nullb, lim: &lim);
1895	if (dev->zoned) {
1896	rv = null_init_zoned_dev(dev, lim: &lim);
1897	if (rv)
1898	goto out_cleanup_tags;
1899	}
1900
1901	nullb->disk = blk_mq_alloc_disk(nullb->tag_set, &lim, nullb);
1902	if (IS_ERR(ptr: nullb->disk)) {
1903	rv = PTR_ERR(ptr: nullb->disk);
1904	goto out_cleanup_zone;
1905	}
1906	nullb->q = nullb->disk->queue;
1907
1908	if (dev->mbps) {
1909	set_bit(nr: NULLB_DEV_FL_THROTTLED, addr: &dev->flags);
1910	nullb_setup_bwtimer(nullb);
1911	}
1912
1913	if (dev->cache_size > 0) {
1914	set_bit(nr: NULLB_DEV_FL_CACHE, addr: &nullb->dev->flags);
1915	blk_queue_write_cache(q: nullb->q, enabled: true, fua: true);
1916	}
1917
1918	nullb->q->queuedata = nullb;
1919	blk_queue_flag_set(QUEUE_FLAG_NONROT, q: nullb->q);
1920
1921	mutex_lock(&lock);
1922	rv = ida_alloc(ida: &nullb_indexes, GFP_KERNEL);
1923	if (rv < 0) {
1924	mutex_unlock(lock: &lock);
1925	goto out_cleanup_disk;
1926	}
1927	nullb->index = rv;
1928	dev->index = rv;
1929	mutex_unlock(lock: &lock);
1930
1931	if (config_item_name(item: &dev->group.cg_item)) {
1932	/* Use configfs dir name as the device name */
1933	snprintf(buf: nullb->disk_name, size: sizeof(nullb->disk_name),
1934	fmt: "%s", config_item_name(item: &dev->group.cg_item));
1935	} else {
1936	sprintf(buf: nullb->disk_name, fmt: "nullb%d", nullb->index);
1937	}
1938
1939	set_capacity(disk: nullb->disk,
1940	size: ((sector_t)nullb->dev->size * SZ_1M) >> SECTOR_SHIFT);
1941	nullb->disk->major = null_major;
1942	nullb->disk->first_minor = nullb->index;
1943	nullb->disk->minors = 1;
1944	nullb->disk->fops = &null_ops;
1945	nullb->disk->private_data = nullb;
1946	strscpy_pad(nullb->disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
1947
1948	if (nullb->dev->zoned) {
1949	rv = null_register_zoned_dev(nullb);
1950	if (rv)
1951	goto out_ida_free;
1952	}
1953
1954	rv = add_disk(disk: nullb->disk);
1955	if (rv)
1956	goto out_ida_free;
1957
1958	mutex_lock(&lock);
1959	list_add_tail(new: &nullb->list, head: &nullb_list);
1960	mutex_unlock(lock: &lock);
1961
1962	pr_info("disk %s created\n", nullb->disk_name);
1963
1964	return 0;
1965
1966	out_ida_free:
1967	ida_free(&nullb_indexes, id: nullb->index);
1968	out_cleanup_disk:
1969	put_disk(disk: nullb->disk);
1970	out_cleanup_zone:
1971	null_free_zoned_dev(dev);
1972	out_cleanup_tags:
1973	if (nullb->tag_set == &nullb->__tag_set)
1974	blk_mq_free_tag_set(set: nullb->tag_set);
1975	out_cleanup_queues:
1976	kfree(objp: nullb->queues);
1977	out_free_nullb:
1978	kfree(objp: nullb);
1979	dev->nullb = NULL;
1980	out:
1981	return rv;
1982	}
1983
1984	static struct nullb *null_find_dev_by_name(const char *name)
1985	{
1986	struct nullb *nullb = NULL, *nb;
1987
1988	mutex_lock(&lock);
1989	list_for_each_entry(nb, &nullb_list, list) {
1990	if (strcmp(nb->disk_name, name) == 0) {
1991	nullb = nb;
1992	break;
1993	}
1994	}
1995	mutex_unlock(lock: &lock);
1996
1997	return nullb;
1998	}
1999
2000	static int null_create_dev(void)
2001	{
2002	struct nullb_device *dev;
2003	int ret;
2004
2005	dev = null_alloc_dev();
2006	if (!dev)
2007	return -ENOMEM;
2008
2009	ret = null_add_dev(dev);
2010	if (ret) {
2011	null_free_dev(dev);
2012	return ret;
2013	}
2014
2015	return 0;
2016	}
2017
2018	static void null_destroy_dev(struct nullb *nullb)
2019	{
2020	struct nullb_device *dev = nullb->dev;
2021
2022	null_del_dev(nullb);
2023	null_free_device_storage(dev, is_cache: false);
2024	null_free_dev(dev);
2025	}
2026
2027	static int __init null_init(void)
2028	{
2029	int ret = 0;
2030	unsigned int i;
2031	struct nullb *nullb;
2032
2033	if (g_bs > PAGE_SIZE) {
2034	pr_warn("invalid block size\n");
2035	pr_warn("defaults block size to %lu\n", PAGE_SIZE);
2036	g_bs = PAGE_SIZE;
2037	}
2038
2039	if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) {
2040	pr_err("invalid home_node value\n");
2041	g_home_node = NUMA_NO_NODE;
2042	}
2043
2044	if (!null_setup_fault())
2045	return -EINVAL;
2046
2047	if (g_queue_mode == NULL_Q_RQ) {
2048	pr_err("legacy IO path is no longer available\n");
2049	return -EINVAL;
2050	}
2051
2052	if (g_use_per_node_hctx) {
2053	if (g_submit_queues != nr_online_nodes) {
2054	pr_warn("submit_queues param is set to %u.\n",
2055	nr_online_nodes);
2056	g_submit_queues = nr_online_nodes;
2057	}
2058	} else if (g_submit_queues > nr_cpu_ids) {
2059	g_submit_queues = nr_cpu_ids;
2060	} else if (g_submit_queues <= 0) {
2061	g_submit_queues = 1;
2062	}
2063
2064	config_group_init(group: &nullb_subsys.su_group);
2065	mutex_init(&nullb_subsys.su_mutex);
2066
2067	ret = configfs_register_subsystem(subsys: &nullb_subsys);
2068	if (ret)
2069	return ret;
2070
2071	mutex_init(&lock);
2072
2073	null_major = register_blkdev(0, "nullb");
2074	if (null_major < 0) {
2075	ret = null_major;
2076	goto err_conf;
2077	}
2078
2079	for (i = 0; i < nr_devices; i++) {
2080	ret = null_create_dev();
2081	if (ret)
2082	goto err_dev;
2083	}
2084
2085	pr_info("module loaded\n");
2086	return 0;
2087
2088	err_dev:
2089	while (!list_empty(head: &nullb_list)) {
2090	nullb = list_entry(nullb_list.next, struct nullb, list);
2091	null_destroy_dev(nullb);
2092	}
2093	unregister_blkdev(major: null_major, name: "nullb");
2094	err_conf:
2095	configfs_unregister_subsystem(subsys: &nullb_subsys);
2096	return ret;
2097	}
2098
2099	static void __exit null_exit(void)
2100	{
2101	struct nullb *nullb;
2102
2103	configfs_unregister_subsystem(subsys: &nullb_subsys);
2104
2105	unregister_blkdev(major: null_major, name: "nullb");
2106
2107	mutex_lock(&lock);
2108	while (!list_empty(head: &nullb_list)) {
2109	nullb = list_entry(nullb_list.next, struct nullb, list);
2110	null_destroy_dev(nullb);
2111	}
2112	mutex_unlock(lock: &lock);
2113
2114	if (tag_set.ops)
2115	blk_mq_free_tag_set(set: &tag_set);
2116	}
2117
2118	module_init(null_init);
2119	module_exit(null_exit);
2120
2121	MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
2122	MODULE_LICENSE("GPL");
2123