1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Author(s)......: Holger Smolinski <Holger.Smolinski@de.ibm.com>
4	* Horst Hummel <Horst.Hummel@de.ibm.com>
5	* Carsten Otte <Cotte@de.ibm.com>
6	* Martin Schwidefsky <schwidefsky@de.ibm.com>
7	* Bugreports.to..: <Linux390@de.ibm.com>
8	* Copyright IBM Corp. 1999, 2009
9	*/
10
11	#define KMSG_COMPONENT "dasd"
12	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
13
14	#include <linux/kmod.h>
15	#include <linux/init.h>
16	#include <linux/interrupt.h>
17	#include <linux/ctype.h>
18	#include <linux/major.h>
19	#include <linux/slab.h>
20	#include <linux/hdreg.h>
21	#include <linux/async.h>
22	#include <linux/mutex.h>
23	#include <linux/debugfs.h>
24	#include <linux/seq_file.h>
25	#include <linux/vmalloc.h>
26
27	#include <asm/ccwdev.h>
28	#include <asm/ebcdic.h>
29	#include <asm/idals.h>
30	#include <asm/itcw.h>
31	#include <asm/diag.h>
32
33	/* This is ugly... */
34	#define PRINTK_HEADER "dasd:"
35
36	#include "dasd_int.h"
37	/*
38	* SECTION: Constant definitions to be used within this file
39	*/
40	#define DASD_CHANQ_MAX_SIZE 4
41
42	#define DASD_DIAG_MOD "dasd_diag_mod"
43
44	/*
45	* SECTION: exported variables of dasd.c
46	*/
47	debug_info_t *dasd_debug_area;
48	EXPORT_SYMBOL(dasd_debug_area);
49	static struct dentry *dasd_debugfs_root_entry;
50	struct dasd_discipline *dasd_diag_discipline_pointer;
51	EXPORT_SYMBOL(dasd_diag_discipline_pointer);
52	void dasd_int_handler(struct ccw_device *, unsigned long, struct irb *);
53
54	MODULE_AUTHOR("Holger Smolinski <Holger.Smolinski@de.ibm.com>");
55	MODULE_DESCRIPTION("Linux on S/390 DASD device driver,"
56	" Copyright IBM Corp. 2000");
57	MODULE_LICENSE("GPL");
58
59	/*
60	* SECTION: prototypes for static functions of dasd.c
61	*/
62	static int dasd_flush_block_queue(struct dasd_block *);
63	static void dasd_device_tasklet(unsigned long);
64	static void dasd_block_tasklet(unsigned long);
65	static void do_kick_device(struct work_struct *);
66	static void do_reload_device(struct work_struct *);
67	static void do_requeue_requests(struct work_struct *);
68	static void dasd_return_cqr_cb(struct dasd_ccw_req *, void *);
69	static void dasd_device_timeout(struct timer_list *);
70	static void dasd_block_timeout(struct timer_list *);
71	static void __dasd_process_erp(struct dasd_device *, struct dasd_ccw_req *);
72	static void dasd_profile_init(struct dasd_profile *, struct dentry *);
73	static void dasd_profile_exit(struct dasd_profile *);
74	static void dasd_hosts_init(struct dentry *, struct dasd_device *);
75	static void dasd_hosts_exit(struct dasd_device *);
76	static int dasd_handle_autoquiesce(struct dasd_device *, struct dasd_ccw_req *,
77	unsigned int);
78	/*
79	* SECTION: Operations on the device structure.
80	*/
81	static wait_queue_head_t dasd_init_waitq;
82	static wait_queue_head_t dasd_flush_wq;
83	static wait_queue_head_t generic_waitq;
84	static wait_queue_head_t shutdown_waitq;
85
86	/*
87	* Allocate memory for a new device structure.
88	*/
89	struct dasd_device *dasd_alloc_device(void)
90	{
91	struct dasd_device *device;
92
93	device = kzalloc(size: sizeof(struct dasd_device), GFP_ATOMIC);
94	if (!device)
95	return ERR_PTR(error: -ENOMEM);
96
97	/* Get two pages for normal block device operations. */
98	device->ccw_mem = (void *) __get_free_pages(GFP_ATOMIC | GFP_DMA, order: 1);
99	if (!device->ccw_mem) {
100	kfree(objp: device);
101	return ERR_PTR(error: -ENOMEM);
102	}
103	/* Get one page for error recovery. */
104	device->erp_mem = (void *) get_zeroed_page(GFP_ATOMIC | GFP_DMA);
105	if (!device->erp_mem) {
106	free_pages(addr: (unsigned long) device->ccw_mem, order: 1);
107	kfree(objp: device);
108	return ERR_PTR(error: -ENOMEM);
109	}
110	/* Get two pages for ese format. */
111	device->ese_mem = (void *)__get_free_pages(GFP_ATOMIC | GFP_DMA, order: 1);
112	if (!device->ese_mem) {
113	free_page((unsigned long) device->erp_mem);
114	free_pages(addr: (unsigned long) device->ccw_mem, order: 1);
115	kfree(objp: device);
116	return ERR_PTR(error: -ENOMEM);
117	}
118
119	dasd_init_chunklist(chunk_list: &device->ccw_chunks, mem: device->ccw_mem, PAGE_SIZE*2);
120	dasd_init_chunklist(chunk_list: &device->erp_chunks, mem: device->erp_mem, PAGE_SIZE);
121	dasd_init_chunklist(chunk_list: &device->ese_chunks, mem: device->ese_mem, PAGE_SIZE * 2);
122	spin_lock_init(&device->mem_lock);
123	atomic_set(v: &device->tasklet_scheduled, i: 0);
124	tasklet_init(t: &device->tasklet, func: dasd_device_tasklet,
125	data: (unsigned long) device);
126	INIT_LIST_HEAD(list: &device->ccw_queue);
127	timer_setup(&device->timer, dasd_device_timeout, 0);
128	INIT_WORK(&device->kick_work, do_kick_device);
129	INIT_WORK(&device->reload_device, do_reload_device);
130	INIT_WORK(&device->requeue_requests, do_requeue_requests);
131	device->state = DASD_STATE_NEW;
132	device->target = DASD_STATE_NEW;
133	mutex_init(&device->state_mutex);
134	spin_lock_init(&device->profile.lock);
135	return device;
136	}
137
138	/*
139	* Free memory of a device structure.
140	*/
141	void dasd_free_device(struct dasd_device *device)
142	{
143	kfree(objp: device->private);
144	free_pages(addr: (unsigned long) device->ese_mem, order: 1);
145	free_page((unsigned long) device->erp_mem);
146	free_pages(addr: (unsigned long) device->ccw_mem, order: 1);
147	kfree(objp: device);
148	}
149
150	/*
151	* Allocate memory for a new device structure.
152	*/
153	struct dasd_block *dasd_alloc_block(void)
154	{
155	struct dasd_block *block;
156
157	block = kzalloc(size: sizeof(*block), GFP_ATOMIC);
158	if (!block)
159	return ERR_PTR(error: -ENOMEM);
160	/* open_count = 0 means device online but not in use */
161	atomic_set(v: &block->open_count, i: -1);
162
163	atomic_set(v: &block->tasklet_scheduled, i: 0);
164	tasklet_init(t: &block->tasklet, func: dasd_block_tasklet,
165	data: (unsigned long) block);
166	INIT_LIST_HEAD(list: &block->ccw_queue);
167	spin_lock_init(&block->queue_lock);
168	INIT_LIST_HEAD(list: &block->format_list);
169	spin_lock_init(&block->format_lock);
170	timer_setup(&block->timer, dasd_block_timeout, 0);
171	spin_lock_init(&block->profile.lock);
172
173	return block;
174	}
175	EXPORT_SYMBOL_GPL(dasd_alloc_block);
176
177	/*
178	* Free memory of a device structure.
179	*/
180	void dasd_free_block(struct dasd_block *block)
181	{
182	kfree(objp: block);
183	}
184	EXPORT_SYMBOL_GPL(dasd_free_block);
185
186	/*
187	* Make a new device known to the system.
188	*/
189	static int dasd_state_new_to_known(struct dasd_device *device)
190	{
191	/*
192	* As long as the device is not in state DASD_STATE_NEW we want to
193	* keep the reference count > 0.
194	*/
195	dasd_get_device(device);
196	device->state = DASD_STATE_KNOWN;
197	return 0;
198	}
199
200	/*
201	* Let the system forget about a device.
202	*/
203	static int dasd_state_known_to_new(struct dasd_device *device)
204	{
205	/* Disable extended error reporting for this device. */
206	dasd_eer_disable(device);
207	device->state = DASD_STATE_NEW;
208
209	/* Give up reference we took in dasd_state_new_to_known. */
210	dasd_put_device(device);
211	return 0;
212	}
213
214	static struct dentry *dasd_debugfs_setup(const char *name,
215	struct dentry *base_dentry)
216	{
217	struct dentry *pde;
218
219	if (!base_dentry)
220	return NULL;
221	pde = debugfs_create_dir(name, parent: base_dentry);
222	if (!pde || IS_ERR(ptr: pde))
223	return NULL;
224	return pde;
225	}
226
227	/*
228	* Request the irq line for the device.
229	*/
230	static int dasd_state_known_to_basic(struct dasd_device *device)
231	{
232	struct dasd_block *block = device->block;
233	int rc = 0;
234
235	/* Allocate and register gendisk structure. */
236	if (block) {
237	rc = dasd_gendisk_alloc(block);
238	if (rc)
239	return rc;
240	block->debugfs_dentry =
241	dasd_debugfs_setup(name: block->gdp->disk_name,
242	base_dentry: dasd_debugfs_root_entry);
243	dasd_profile_init(&block->profile, block->debugfs_dentry);
244	if (dasd_global_profile_level == DASD_PROFILE_ON)
245	dasd_profile_on(&device->block->profile);
246	}
247	device->debugfs_dentry =
248	dasd_debugfs_setup(name: dev_name(dev: &device->cdev->dev),
249	base_dentry: dasd_debugfs_root_entry);
250	dasd_profile_init(&device->profile, device->debugfs_dentry);
251	dasd_hosts_init(device->debugfs_dentry, device);
252
253	/* register 'device' debug area, used for all DBF_DEV_XXX calls */
254	device->debug_area = debug_register(dev_name(dev: &device->cdev->dev), 4, 1,
255	8 * sizeof(long));
256	debug_register_view(device->debug_area, &debug_sprintf_view);
257	debug_set_level(device->debug_area, DBF_WARNING);
258	DBF_DEV_EVENT(DBF_EMERG, device, "%s", "debug area created");
259
260	device->state = DASD_STATE_BASIC;
261
262	return rc;
263	}
264
265	/*
266	* Release the irq line for the device. Terminate any running i/o.
267	*/
268	static int dasd_state_basic_to_known(struct dasd_device *device)
269	{
270	int rc;
271
272	if (device->discipline->basic_to_known) {
273	rc = device->discipline->basic_to_known(device);
274	if (rc)
275	return rc;
276	}
277
278	if (device->block) {
279	dasd_profile_exit(&device->block->profile);
280	debugfs_remove(dentry: device->block->debugfs_dentry);
281	dasd_gendisk_free(device->block);
282	dasd_block_clear_timer(device->block);
283	}
284	rc = dasd_flush_device_queue(device);
285	if (rc)
286	return rc;
287	dasd_device_clear_timer(device);
288	dasd_profile_exit(&device->profile);
289	dasd_hosts_exit(device);
290	debugfs_remove(dentry: device->debugfs_dentry);
291	DBF_DEV_EVENT(DBF_EMERG, device, "%p debug area deleted", device);
292	if (device->debug_area != NULL) {
293	debug_unregister(device->debug_area);
294	device->debug_area = NULL;
295	}
296	device->state = DASD_STATE_KNOWN;
297	return 0;
298	}
299
300	/*
301	* Do the initial analysis. The do_analysis function may return
302	* -EAGAIN in which case the device keeps the state DASD_STATE_BASIC
303	* until the discipline decides to continue the startup sequence
304	* by calling the function dasd_change_state. The eckd disciplines
305	* uses this to start a ccw that detects the format. The completion
306	* interrupt for this detection ccw uses the kernel event daemon to
307	* trigger the call to dasd_change_state. All this is done in the
308	* discipline code, see dasd_eckd.c.
309	* After the analysis ccw is done (do_analysis returned 0) the block
310	* device is setup.
311	* In case the analysis returns an error, the device setup is stopped
312	* (a fake disk was already added to allow formatting).
313	*/
314	static int dasd_state_basic_to_ready(struct dasd_device *device)
315	{
316	int rc;
317	struct dasd_block *block;
318	struct gendisk *disk;
319
320	rc = 0;
321	block = device->block;
322	/* make disk known with correct capacity */
323	if (block) {
324	if (block->base->discipline->do_analysis != NULL)
325	rc = block->base->discipline->do_analysis(block);
326	if (rc) {
327	if (rc != -EAGAIN) {
328	device->state = DASD_STATE_UNFMT;
329	disk = device->block->gdp;
330	kobject_uevent(kobj: &disk_to_dev(disk)->kobj,
331	action: KOBJ_CHANGE);
332	goto out;
333	}
334	return rc;
335	}
336	if (device->discipline->setup_blk_queue)
337	device->discipline->setup_blk_queue(block);
338	set_capacity(disk: block->gdp,
339	size: block->blocks << block->s2b_shift);
340	device->state = DASD_STATE_READY;
341	rc = dasd_scan_partitions(block);
342	if (rc) {
343	device->state = DASD_STATE_BASIC;
344	return rc;
345	}
346	} else {
347	device->state = DASD_STATE_READY;
348	}
349	out:
350	if (device->discipline->basic_to_ready)
351	rc = device->discipline->basic_to_ready(device);
352	return rc;
353	}
354
355	static inline
356	int _wait_for_empty_queues(struct dasd_device *device)
357	{
358	if (device->block)
359	return list_empty(head: &device->ccw_queue) &&
360	list_empty(head: &device->block->ccw_queue);
361	else
362	return list_empty(head: &device->ccw_queue);
363	}
364
365	/*
366	* Remove device from block device layer. Destroy dirty buffers.
367	* Forget format information. Check if the target level is basic
368	* and if it is create fake disk for formatting.
369	*/
370	static int dasd_state_ready_to_basic(struct dasd_device *device)
371	{
372	int rc;
373
374	device->state = DASD_STATE_BASIC;
375	if (device->block) {
376	struct dasd_block *block = device->block;
377	rc = dasd_flush_block_queue(block);
378	if (rc) {
379	device->state = DASD_STATE_READY;
380	return rc;
381	}
382	dasd_destroy_partitions(block);
383	block->blocks = 0;
384	block->bp_block = 0;
385	block->s2b_shift = 0;
386	}
387	return 0;
388	}
389
390	/*
391	* Back to basic.
392	*/
393	static int dasd_state_unfmt_to_basic(struct dasd_device *device)
394	{
395	device->state = DASD_STATE_BASIC;
396	return 0;
397	}
398
399	/*
400	* Make the device online and schedule the bottom half to start
401	* the requeueing of requests from the linux request queue to the
402	* ccw queue.
403	*/
404	static int
405	dasd_state_ready_to_online(struct dasd_device * device)
406	{
407	device->state = DASD_STATE_ONLINE;
408	if (device->block) {
409	dasd_schedule_block_bh(device->block);
410	if ((device->features & DASD_FEATURE_USERAW)) {
411	kobject_uevent(kobj: &disk_to_dev(device->block->gdp)->kobj,
412	action: KOBJ_CHANGE);
413	return 0;
414	}
415	disk_uevent(disk: device->block->bdev_handle->bdev->bd_disk,
416	action: KOBJ_CHANGE);
417	}
418	return 0;
419	}
420
421	/*
422	* Stop the requeueing of requests again.
423	*/
424	static int dasd_state_online_to_ready(struct dasd_device *device)
425	{
426	int rc;
427
428	if (device->discipline->online_to_ready) {
429	rc = device->discipline->online_to_ready(device);
430	if (rc)
431	return rc;
432	}
433
434	device->state = DASD_STATE_READY;
435	if (device->block && !(device->features & DASD_FEATURE_USERAW))
436	disk_uevent(disk: device->block->bdev_handle->bdev->bd_disk,
437	action: KOBJ_CHANGE);
438	return 0;
439	}
440
441	/*
442	* Device startup state changes.
443	*/
444	static int dasd_increase_state(struct dasd_device *device)
445	{
446	int rc;
447
448	rc = 0;
449	if (device->state == DASD_STATE_NEW &&
450	device->target >= DASD_STATE_KNOWN)
451	rc = dasd_state_new_to_known(device);
452
453	if (!rc &&
454	device->state == DASD_STATE_KNOWN &&
455	device->target >= DASD_STATE_BASIC)
456	rc = dasd_state_known_to_basic(device);
457
458	if (!rc &&
459	device->state == DASD_STATE_BASIC &&
460	device->target >= DASD_STATE_READY)
461	rc = dasd_state_basic_to_ready(device);
462
463	if (!rc &&
464	device->state == DASD_STATE_UNFMT &&
465	device->target > DASD_STATE_UNFMT)
466	rc = -EPERM;
467
468	if (!rc &&
469	device->state == DASD_STATE_READY &&
470	device->target >= DASD_STATE_ONLINE)
471	rc = dasd_state_ready_to_online(device);
472
473	return rc;
474	}
475
476	/*
477	* Device shutdown state changes.
478	*/
479	static int dasd_decrease_state(struct dasd_device *device)
480	{
481	int rc;
482
483	rc = 0;
484	if (device->state == DASD_STATE_ONLINE &&
485	device->target <= DASD_STATE_READY)
486	rc = dasd_state_online_to_ready(device);
487
488	if (!rc &&
489	device->state == DASD_STATE_READY &&
490	device->target <= DASD_STATE_BASIC)
491	rc = dasd_state_ready_to_basic(device);
492
493	if (!rc &&
494	device->state == DASD_STATE_UNFMT &&
495	device->target <= DASD_STATE_BASIC)
496	rc = dasd_state_unfmt_to_basic(device);
497
498	if (!rc &&
499	device->state == DASD_STATE_BASIC &&
500	device->target <= DASD_STATE_KNOWN)
501	rc = dasd_state_basic_to_known(device);
502
503	if (!rc &&
504	device->state == DASD_STATE_KNOWN &&
505	device->target <= DASD_STATE_NEW)
506	rc = dasd_state_known_to_new(device);
507
508	return rc;
509	}
510
511	/*
512	* This is the main startup/shutdown routine.
513	*/
514	static void dasd_change_state(struct dasd_device *device)
515	{
516	int rc;
517
518	if (device->state == device->target)
519	/* Already where we want to go today... */
520	return;
521	if (device->state < device->target)
522	rc = dasd_increase_state(device);
523	else
524	rc = dasd_decrease_state(device);
525	if (rc == -EAGAIN)
526	return;
527	if (rc)
528	device->target = device->state;
529
530	/* let user-space know that the device status changed */
531	kobject_uevent(kobj: &device->cdev->dev.kobj, action: KOBJ_CHANGE);
532
533	if (device->state == device->target)
534	wake_up(&dasd_init_waitq);
535	}
536
537	/*
538	* Kick starter for devices that did not complete the startup/shutdown
539	* procedure or were sleeping because of a pending state.
540	* dasd_kick_device will schedule a call do do_kick_device to the kernel
541	* event daemon.
542	*/
543	static void do_kick_device(struct work_struct *work)
544	{
545	struct dasd_device *device = container_of(work, struct dasd_device, kick_work);
546	mutex_lock(&device->state_mutex);
547	dasd_change_state(device);
548	mutex_unlock(lock: &device->state_mutex);
549	dasd_schedule_device_bh(device);
550	dasd_put_device(device);
551	}
552
553	void dasd_kick_device(struct dasd_device *device)
554	{
555	dasd_get_device(device);
556	/* queue call to dasd_kick_device to the kernel event daemon. */
557	if (!schedule_work(work: &device->kick_work))
558	dasd_put_device(device);
559	}
560	EXPORT_SYMBOL(dasd_kick_device);
561
562	/*
563	* dasd_reload_device will schedule a call do do_reload_device to the kernel
564	* event daemon.
565	*/
566	static void do_reload_device(struct work_struct *work)
567	{
568	struct dasd_device *device = container_of(work, struct dasd_device,
569	reload_device);
570	device->discipline->reload(device);
571	dasd_put_device(device);
572	}
573
574	void dasd_reload_device(struct dasd_device *device)
575	{
576	dasd_get_device(device);
577	/* queue call to dasd_reload_device to the kernel event daemon. */
578	if (!schedule_work(work: &device->reload_device))
579	dasd_put_device(device);
580	}
581	EXPORT_SYMBOL(dasd_reload_device);
582
583	/*
584	* Set the target state for a device and starts the state change.
585	*/
586	void dasd_set_target_state(struct dasd_device *device, int target)
587	{
588	dasd_get_device(device);
589	mutex_lock(&device->state_mutex);
590	/* If we are in probeonly mode stop at DASD_STATE_READY. */
591	if (dasd_probeonly && target > DASD_STATE_READY)
592	target = DASD_STATE_READY;
593	if (device->target != target) {
594	if (device->state == target)
595	wake_up(&dasd_init_waitq);
596	device->target = target;
597	}
598	if (device->state != device->target)
599	dasd_change_state(device);
600	mutex_unlock(lock: &device->state_mutex);
601	dasd_put_device(device);
602	}
603
604	/*
605	* Enable devices with device numbers in [from..to].
606	*/
607	static inline int _wait_for_device(struct dasd_device *device)
608	{
609	return (device->state == device->target);
610	}
611
612	void dasd_enable_device(struct dasd_device *device)
613	{
614	dasd_set_target_state(device, DASD_STATE_ONLINE);
615	if (device->state <= DASD_STATE_KNOWN)
616	/* No discipline for device found. */
617	dasd_set_target_state(device, DASD_STATE_NEW);
618	/* Now wait for the devices to come up. */
619	wait_event(dasd_init_waitq, _wait_for_device(device));
620
621	dasd_reload_device(device);
622	if (device->discipline->kick_validate)
623	device->discipline->kick_validate(device);
624	}
625	EXPORT_SYMBOL(dasd_enable_device);
626
627	/*
628	* SECTION: device operation (interrupt handler, start i/o, term i/o ...)
629	*/
630
631	unsigned int dasd_global_profile_level = DASD_PROFILE_OFF;
632
633	#ifdef CONFIG_DASD_PROFILE
634	struct dasd_profile dasd_global_profile = {
635	.lock = __SPIN_LOCK_UNLOCKED(dasd_global_profile.lock),
636	};
637	static struct dentry *dasd_debugfs_global_entry;
638
639	/*
640	* Add profiling information for cqr before execution.
641	*/
642	static void dasd_profile_start(struct dasd_block *block,
643	struct dasd_ccw_req *cqr,
644	struct request *req)
645	{
646	struct list_head *l;
647	unsigned int counter;
648	struct dasd_device *device;
649
650	/* count the length of the chanq for statistics */
651	counter = 0;
652	if (dasd_global_profile_level || block->profile.data)
653	list_for_each(l, &block->ccw_queue)
654	if (++counter >= 31)
655	break;
656
657	spin_lock(&dasd_global_profile.lock);
658	if (dasd_global_profile.data) {
659	dasd_global_profile.data->dasd_io_nr_req[counter]++;
660	if (rq_data_dir(req) == READ)
661	dasd_global_profile.data->dasd_read_nr_req[counter]++;
662	}
663	spin_unlock(&dasd_global_profile.lock);
664
665	spin_lock(&block->profile.lock);
666	if (block->profile.data) {
667	block->profile.data->dasd_io_nr_req[counter]++;
668	if (rq_data_dir(req) == READ)
669	block->profile.data->dasd_read_nr_req[counter]++;
670	}
671	spin_unlock(&block->profile.lock);
672
673	/*
674	* We count the request for the start device, even though it may run on
675	* some other device due to error recovery. This way we make sure that
676	* we count each request only once.
677	*/
678	device = cqr->startdev;
679	if (device->profile.data) {
680	counter = 1; /* request is not yet queued on the start device */
681	list_for_each(l, &device->ccw_queue)
682	if (++counter >= 31)
683	break;
684	}
685	spin_lock(&device->profile.lock);
686	if (device->profile.data) {
687	device->profile.data->dasd_io_nr_req[counter]++;
688	if (rq_data_dir(req) == READ)
689	device->profile.data->dasd_read_nr_req[counter]++;
690	}
691	spin_unlock(&device->profile.lock);
692	}
693
694	/*
695	* Add profiling information for cqr after execution.
696	*/
697
698	#define dasd_profile_counter(value, index) \
699	{ \
700	for (index = 0; index < 31 && value >> (2+index); index++) \
701	; \
702	}
703
704	static void dasd_profile_end_add_data(struct dasd_profile_info *data,
705	int is_alias,
706	int is_tpm,
707	int is_read,
708	long sectors,
709	int sectors_ind,
710	int tottime_ind,
711	int tottimeps_ind,
712	int strtime_ind,
713	int irqtime_ind,
714	int irqtimeps_ind,
715	int endtime_ind)
716	{
717	/* in case of an overflow, reset the whole profile */
718	if (data->dasd_io_reqs == UINT_MAX) {
719	memset(data, 0, sizeof(*data));
720	ktime_get_real_ts64(&data->starttod);
721	}
722	data->dasd_io_reqs++;
723	data->dasd_io_sects += sectors;
724	if (is_alias)
725	data->dasd_io_alias++;
726	if (is_tpm)
727	data->dasd_io_tpm++;
728
729	data->dasd_io_secs[sectors_ind]++;
730	data->dasd_io_times[tottime_ind]++;
731	data->dasd_io_timps[tottimeps_ind]++;
732	data->dasd_io_time1[strtime_ind]++;
733	data->dasd_io_time2[irqtime_ind]++;
734	data->dasd_io_time2ps[irqtimeps_ind]++;
735	data->dasd_io_time3[endtime_ind]++;
736
737	if (is_read) {
738	data->dasd_read_reqs++;
739	data->dasd_read_sects += sectors;
740	if (is_alias)
741	data->dasd_read_alias++;
742	if (is_tpm)
743	data->dasd_read_tpm++;
744	data->dasd_read_secs[sectors_ind]++;
745	data->dasd_read_times[tottime_ind]++;
746	data->dasd_read_time1[strtime_ind]++;
747	data->dasd_read_time2[irqtime_ind]++;
748	data->dasd_read_time3[endtime_ind]++;
749	}
750	}
751
752	static void dasd_profile_end(struct dasd_block *block,
753	struct dasd_ccw_req *cqr,
754	struct request *req)
755	{
756	unsigned long strtime, irqtime, endtime, tottime;
757	unsigned long tottimeps, sectors;
758	struct dasd_device *device;
759	int sectors_ind, tottime_ind, tottimeps_ind, strtime_ind;
760	int irqtime_ind, irqtimeps_ind, endtime_ind;
761	struct dasd_profile_info *data;
762
763	device = cqr->startdev;
764	if (!(dasd_global_profile_level ||
765	block->profile.data ||
766	device->profile.data))
767	return;
768
769	sectors = blk_rq_sectors(req);
770	if (!cqr->buildclk || !cqr->startclk ||
771	!cqr->stopclk || !cqr->endclk ||
772	!sectors)
773	return;
774
775	strtime = ((cqr->startclk - cqr->buildclk) >> 12);
776	irqtime = ((cqr->stopclk - cqr->startclk) >> 12);
777	endtime = ((cqr->endclk - cqr->stopclk) >> 12);
778	tottime = ((cqr->endclk - cqr->buildclk) >> 12);
779	tottimeps = tottime / sectors;
780
781	dasd_profile_counter(sectors, sectors_ind);
782	dasd_profile_counter(tottime, tottime_ind);
783	dasd_profile_counter(tottimeps, tottimeps_ind);
784	dasd_profile_counter(strtime, strtime_ind);
785	dasd_profile_counter(irqtime, irqtime_ind);
786	dasd_profile_counter(irqtime / sectors, irqtimeps_ind);
787	dasd_profile_counter(endtime, endtime_ind);
788
789	spin_lock(&dasd_global_profile.lock);
790	if (dasd_global_profile.data) {
791	data = dasd_global_profile.data;
792	data->dasd_sum_times += tottime;
793	data->dasd_sum_time_str += strtime;
794	data->dasd_sum_time_irq += irqtime;
795	data->dasd_sum_time_end += endtime;
796	dasd_profile_end_add_data(dasd_global_profile.data,
797	cqr->startdev != block->base,
798	cqr->cpmode == 1,
799	rq_data_dir(req) == READ,
800	sectors, sectors_ind, tottime_ind,
801	tottimeps_ind, strtime_ind,
802	irqtime_ind, irqtimeps_ind,
803	endtime_ind);
804	}
805	spin_unlock(&dasd_global_profile.lock);
806
807	spin_lock(&block->profile.lock);
808	if (block->profile.data) {
809	data = block->profile.data;
810	data->dasd_sum_times += tottime;
811	data->dasd_sum_time_str += strtime;
812	data->dasd_sum_time_irq += irqtime;
813	data->dasd_sum_time_end += endtime;
814	dasd_profile_end_add_data(block->profile.data,
815	cqr->startdev != block->base,
816	cqr->cpmode == 1,
817	rq_data_dir(req) == READ,
818	sectors, sectors_ind, tottime_ind,
819	tottimeps_ind, strtime_ind,
820	irqtime_ind, irqtimeps_ind,
821	endtime_ind);
822	}
823	spin_unlock(&block->profile.lock);
824
825	spin_lock(&device->profile.lock);
826	if (device->profile.data) {
827	data = device->profile.data;
828	data->dasd_sum_times += tottime;
829	data->dasd_sum_time_str += strtime;
830	data->dasd_sum_time_irq += irqtime;
831	data->dasd_sum_time_end += endtime;
832	dasd_profile_end_add_data(device->profile.data,
833	cqr->startdev != block->base,
834	cqr->cpmode == 1,
835	rq_data_dir(req) == READ,
836	sectors, sectors_ind, tottime_ind,
837	tottimeps_ind, strtime_ind,
838	irqtime_ind, irqtimeps_ind,
839	endtime_ind);
840	}
841	spin_unlock(&device->profile.lock);
842	}
843
844	void dasd_profile_reset(struct dasd_profile *profile)
845	{
846	struct dasd_profile_info *data;
847
848	spin_lock_bh(&profile->lock);
849	data = profile->data;
850	if (!data) {
851	spin_unlock_bh(&profile->lock);
852	return;
853	}
854	memset(data, 0, sizeof(*data));
855	ktime_get_real_ts64(&data->starttod);
856	spin_unlock_bh(&profile->lock);
857	}
858
859	int dasd_profile_on(struct dasd_profile *profile)
860	{
861	struct dasd_profile_info *data;
862
863	data = kzalloc(sizeof(*data), GFP_KERNEL);
864	if (!data)
865	return -ENOMEM;
866	spin_lock_bh(&profile->lock);
867	if (profile->data) {
868	spin_unlock_bh(&profile->lock);
869	kfree(data);
870	return 0;
871	}
872	ktime_get_real_ts64(&data->starttod);
873	profile->data = data;
874	spin_unlock_bh(&profile->lock);
875	return 0;
876	}
877
878	void dasd_profile_off(struct dasd_profile *profile)
879	{
880	spin_lock_bh(&profile->lock);
881	kfree(profile->data);
882	profile->data = NULL;
883	spin_unlock_bh(&profile->lock);
884	}
885
886	char *dasd_get_user_string(const char __user *user_buf, size_t user_len)
887	{
888	char *buffer;
889
890	buffer = vmalloc(user_len + 1);
891	if (buffer == NULL)
892	return ERR_PTR(-ENOMEM);
893	if (copy_from_user(buffer, user_buf, user_len) != 0) {
894	vfree(buffer);
895	return ERR_PTR(-EFAULT);
896	}
897	/* got the string, now strip linefeed. */
898	if (buffer[user_len - 1] == '\n')
899	buffer[user_len - 1] = 0;
900	else
901	buffer[user_len] = 0;
902	return buffer;
903	}
904
905	static ssize_t dasd_stats_write(struct file *file,
906	const char __user *user_buf,
907	size_t user_len, loff_t *pos)
908	{
909	char *buffer, *str;
910	int rc;
911	struct seq_file *m = (struct seq_file *)file->private_data;
912	struct dasd_profile *prof = m->private;
913
914	if (user_len > 65536)
915	user_len = 65536;
916	buffer = dasd_get_user_string(user_buf, user_len);
917	if (IS_ERR(buffer))
918	return PTR_ERR(buffer);
919
920	str = skip_spaces(buffer);
921	rc = user_len;
922	if (strncmp(str, "reset", 5) == 0) {
923	dasd_profile_reset(prof);
924	} else if (strncmp(str, "on", 2) == 0) {
925	rc = dasd_profile_on(prof);
926	if (rc)
927	goto out;
928	rc = user_len;
929	if (prof == &dasd_global_profile) {
930	dasd_profile_reset(prof);
931	dasd_global_profile_level = DASD_PROFILE_GLOBAL_ONLY;
932	}
933	} else if (strncmp(str, "off", 3) == 0) {
934	if (prof == &dasd_global_profile)
935	dasd_global_profile_level = DASD_PROFILE_OFF;
936	dasd_profile_off(prof);
937	} else
938	rc = -EINVAL;
939	out:
940	vfree(buffer);
941	return rc;
942	}
943
944	static void dasd_stats_array(struct seq_file *m, unsigned int *array)
945	{
946	int i;
947
948	for (i = 0; i < 32; i++)
949	seq_printf(m, "%u ", array[i]);
950	seq_putc(m, '\n');
951	}
952
953	static void dasd_stats_seq_print(struct seq_file *m,
954	struct dasd_profile_info *data)
955	{
956	seq_printf(m, "start_time %lld.%09ld\n",
957	(s64)data->starttod.tv_sec, data->starttod.tv_nsec);
958	seq_printf(m, "total_requests %u\n", data->dasd_io_reqs);
959	seq_printf(m, "total_sectors %u\n", data->dasd_io_sects);
960	seq_printf(m, "total_pav %u\n", data->dasd_io_alias);
961	seq_printf(m, "total_hpf %u\n", data->dasd_io_tpm);
962	seq_printf(m, "avg_total %lu\n", data->dasd_io_reqs ?
963	data->dasd_sum_times / data->dasd_io_reqs : 0UL);
964	seq_printf(m, "avg_build_to_ssch %lu\n", data->dasd_io_reqs ?
965	data->dasd_sum_time_str / data->dasd_io_reqs : 0UL);
966	seq_printf(m, "avg_ssch_to_irq %lu\n", data->dasd_io_reqs ?
967	data->dasd_sum_time_irq / data->dasd_io_reqs : 0UL);
968	seq_printf(m, "avg_irq_to_end %lu\n", data->dasd_io_reqs ?
969	data->dasd_sum_time_end / data->dasd_io_reqs : 0UL);
970	seq_puts(m, "histogram_sectors ");
971	dasd_stats_array(m, data->dasd_io_secs);
972	seq_puts(m, "histogram_io_times ");
973	dasd_stats_array(m, data->dasd_io_times);
974	seq_puts(m, "histogram_io_times_weighted ");
975	dasd_stats_array(m, data->dasd_io_timps);
976	seq_puts(m, "histogram_time_build_to_ssch ");
977	dasd_stats_array(m, data->dasd_io_time1);
978	seq_puts(m, "histogram_time_ssch_to_irq ");
979	dasd_stats_array(m, data->dasd_io_time2);
980	seq_puts(m, "histogram_time_ssch_to_irq_weighted ");
981	dasd_stats_array(m, data->dasd_io_time2ps);
982	seq_puts(m, "histogram_time_irq_to_end ");
983	dasd_stats_array(m, data->dasd_io_time3);
984	seq_puts(m, "histogram_ccw_queue_length ");
985	dasd_stats_array(m, data->dasd_io_nr_req);
986	seq_printf(m, "total_read_requests %u\n", data->dasd_read_reqs);
987	seq_printf(m, "total_read_sectors %u\n", data->dasd_read_sects);
988	seq_printf(m, "total_read_pav %u\n", data->dasd_read_alias);
989	seq_printf(m, "total_read_hpf %u\n", data->dasd_read_tpm);
990	seq_puts(m, "histogram_read_sectors ");
991	dasd_stats_array(m, data->dasd_read_secs);
992	seq_puts(m, "histogram_read_times ");
993	dasd_stats_array(m, data->dasd_read_times);
994	seq_puts(m, "histogram_read_time_build_to_ssch ");
995	dasd_stats_array(m, data->dasd_read_time1);
996	seq_puts(m, "histogram_read_time_ssch_to_irq ");
997	dasd_stats_array(m, data->dasd_read_time2);
998	seq_puts(m, "histogram_read_time_irq_to_end ");
999	dasd_stats_array(m, data->dasd_read_time3);
1000	seq_puts(m, "histogram_read_ccw_queue_length ");
1001	dasd_stats_array(m, data->dasd_read_nr_req);
1002	}
1003
1004	static int dasd_stats_show(struct seq_file *m, void *v)
1005	{
1006	struct dasd_profile *profile;
1007	struct dasd_profile_info *data;
1008
1009	profile = m->private;
1010	spin_lock_bh(&profile->lock);
1011	data = profile->data;
1012	if (!data) {
1013	spin_unlock_bh(&profile->lock);
1014	seq_puts(m, "disabled\n");
1015	return 0;
1016	}
1017	dasd_stats_seq_print(m, data);
1018	spin_unlock_bh(&profile->lock);
1019	return 0;
1020	}
1021
1022	static int dasd_stats_open(struct inode *inode, struct file *file)
1023	{
1024	struct dasd_profile *profile = inode->i_private;
1025	return single_open(file, dasd_stats_show, profile);
1026	}
1027
1028	static const struct file_operations dasd_stats_raw_fops = {
1029	.owner = THIS_MODULE,
1030	.open = dasd_stats_open,
1031	.read = seq_read,
1032	.llseek = seq_lseek,
1033	.release = single_release,
1034	.write = dasd_stats_write,
1035	};
1036
1037	static void dasd_profile_init(struct dasd_profile *profile,
1038	struct dentry *base_dentry)
1039	{
1040	umode_t mode;
1041	struct dentry *pde;
1042
1043	if (!base_dentry)
1044	return;
1045	profile->dentry = NULL;
1046	profile->data = NULL;
1047	mode = (S_IRUSR | S_IWUSR | S_IFREG);
1048	pde = debugfs_create_file("statistics", mode, base_dentry,
1049	profile, &dasd_stats_raw_fops);
1050	if (pde && !IS_ERR(pde))
1051	profile->dentry = pde;
1052	return;
1053	}
1054
1055	static void dasd_profile_exit(struct dasd_profile *profile)
1056	{
1057	dasd_profile_off(profile);
1058	debugfs_remove(profile->dentry);
1059	profile->dentry = NULL;
1060	}
1061
1062	static void dasd_statistics_removeroot(void)
1063	{
1064	dasd_global_profile_level = DASD_PROFILE_OFF;
1065	dasd_profile_exit(&dasd_global_profile);
1066	debugfs_remove(dasd_debugfs_global_entry);
1067	debugfs_remove(dasd_debugfs_root_entry);
1068	}
1069
1070	static void dasd_statistics_createroot(void)
1071	{
1072	struct dentry *pde;
1073
1074	dasd_debugfs_root_entry = NULL;
1075	pde = debugfs_create_dir("dasd", NULL);
1076	if (!pde || IS_ERR(pde))
1077	goto error;
1078	dasd_debugfs_root_entry = pde;
1079	pde = debugfs_create_dir("global", dasd_debugfs_root_entry);
1080	if (!pde || IS_ERR(pde))
1081	goto error;
1082	dasd_debugfs_global_entry = pde;
1083	dasd_profile_init(&dasd_global_profile, dasd_debugfs_global_entry);
1084	return;
1085
1086	error:
1087	DBF_EVENT(DBF_ERR, "%s",
1088	"Creation of the dasd debugfs interface failed");
1089	dasd_statistics_removeroot();
1090	return;
1091	}
1092
1093	#else
1094	#define dasd_profile_start(block, cqr, req) do {} while (0)
1095	#define dasd_profile_end(block, cqr, req) do {} while (0)
1096
1097	static void dasd_statistics_createroot(void)
1098	{
1099	return;
1100	}
1101
1102	static void dasd_statistics_removeroot(void)
1103	{
1104	return;
1105	}
1106
1107	int dasd_stats_generic_show(struct seq_file *m, void *v)
1108	{
1109	seq_puts(m, s: "Statistics are not activated in this kernel\n");
1110	return 0;
1111	}
1112
1113	static void dasd_profile_init(struct dasd_profile *profile,
1114	struct dentry *base_dentry)
1115	{
1116	return;
1117	}
1118
1119	static void dasd_profile_exit(struct dasd_profile *profile)
1120	{
1121	return;
1122	}
1123
1124	int dasd_profile_on(struct dasd_profile *profile)
1125	{
1126	return 0;
1127	}
1128
1129	#endif /* CONFIG_DASD_PROFILE */
1130
1131	static int dasd_hosts_show(struct seq_file *m, void *v)
1132	{
1133	struct dasd_device *device;
1134	int rc = -EOPNOTSUPP;
1135
1136	device = m->private;
1137	dasd_get_device(device);
1138
1139	if (device->discipline->hosts_print)
1140	rc = device->discipline->hosts_print(device, m);
1141
1142	dasd_put_device(device);
1143	return rc;
1144	}
1145
1146	DEFINE_SHOW_ATTRIBUTE(dasd_hosts);
1147
1148	static void dasd_hosts_exit(struct dasd_device *device)
1149	{
1150	debugfs_remove(dentry: device->hosts_dentry);
1151	device->hosts_dentry = NULL;
1152	}
1153
1154	static void dasd_hosts_init(struct dentry *base_dentry,
1155	struct dasd_device *device)
1156	{
1157	struct dentry *pde;
1158	umode_t mode;
1159
1160	if (!base_dentry)
1161	return;
1162
1163	mode = S_IRUSR | S_IFREG;
1164	pde = debugfs_create_file(name: "host_access_list", mode, parent: base_dentry,
1165	data: device, fops: &dasd_hosts_fops);
1166	if (pde && !IS_ERR(ptr: pde))
1167	device->hosts_dentry = pde;
1168	}
1169
1170	struct dasd_ccw_req *dasd_smalloc_request(int magic, int cplength, int datasize,
1171	struct dasd_device *device,
1172	struct dasd_ccw_req *cqr)
1173	{
1174	unsigned long flags;
1175	char *data, *chunk;
1176	int size = 0;
1177
1178	if (cplength > 0)
1179	size += cplength * sizeof(struct ccw1);
1180	if (datasize > 0)
1181	size += datasize;
1182	if (!cqr)
1183	size += (sizeof(*cqr) + 7L) & -8L;
1184
1185	spin_lock_irqsave(&device->mem_lock, flags);
1186	data = chunk = dasd_alloc_chunk(chunk_list: &device->ccw_chunks, size);
1187	spin_unlock_irqrestore(lock: &device->mem_lock, flags);
1188	if (!chunk)
1189	return ERR_PTR(error: -ENOMEM);
1190	if (!cqr) {
1191	cqr = (void *) data;
1192	data += (sizeof(*cqr) + 7L) & -8L;
1193	}
1194	memset(cqr, 0, sizeof(*cqr));
1195	cqr->mem_chunk = chunk;
1196	if (cplength > 0) {
1197	cqr->cpaddr = data;
1198	data += cplength * sizeof(struct ccw1);
1199	memset(cqr->cpaddr, 0, cplength * sizeof(struct ccw1));
1200	}
1201	if (datasize > 0) {
1202	cqr->data = data;
1203	memset(cqr->data, 0, datasize);
1204	}
1205	cqr->magic = magic;
1206	set_bit(DASD_CQR_FLAGS_USE_ERP, addr: &cqr->flags);
1207	dasd_get_device(device);
1208	return cqr;
1209	}
1210	EXPORT_SYMBOL(dasd_smalloc_request);
1211
1212	struct dasd_ccw_req *dasd_fmalloc_request(int magic, int cplength,
1213	int datasize,
1214	struct dasd_device *device)
1215	{
1216	struct dasd_ccw_req *cqr;
1217	unsigned long flags;
1218	int size, cqr_size;
1219	char *data;
1220
1221	cqr_size = (sizeof(*cqr) + 7L) & -8L;
1222	size = cqr_size;
1223	if (cplength > 0)
1224	size += cplength * sizeof(struct ccw1);
1225	if (datasize > 0)
1226	size += datasize;
1227
1228	spin_lock_irqsave(&device->mem_lock, flags);
1229	cqr = dasd_alloc_chunk(chunk_list: &device->ese_chunks, size);
1230	spin_unlock_irqrestore(lock: &device->mem_lock, flags);
1231	if (!cqr)
1232	return ERR_PTR(error: -ENOMEM);
1233	memset(cqr, 0, sizeof(*cqr));
1234	data = (char *)cqr + cqr_size;
1235	cqr->cpaddr = NULL;
1236	if (cplength > 0) {
1237	cqr->cpaddr = data;
1238	data += cplength * sizeof(struct ccw1);
1239	memset(cqr->cpaddr, 0, cplength * sizeof(struct ccw1));
1240	}
1241	cqr->data = NULL;
1242	if (datasize > 0) {
1243	cqr->data = data;
1244	memset(cqr->data, 0, datasize);
1245	}
1246
1247	cqr->magic = magic;
1248	set_bit(DASD_CQR_FLAGS_USE_ERP, addr: &cqr->flags);
1249	dasd_get_device(device);
1250
1251	return cqr;
1252	}
1253	EXPORT_SYMBOL(dasd_fmalloc_request);
1254
1255	void dasd_sfree_request(struct dasd_ccw_req *cqr, struct dasd_device *device)
1256	{
1257	unsigned long flags;
1258
1259	spin_lock_irqsave(&device->mem_lock, flags);
1260	dasd_free_chunk(chunk_list: &device->ccw_chunks, mem: cqr->mem_chunk);
1261	spin_unlock_irqrestore(lock: &device->mem_lock, flags);
1262	dasd_put_device(device);
1263	}
1264	EXPORT_SYMBOL(dasd_sfree_request);
1265
1266	void dasd_ffree_request(struct dasd_ccw_req *cqr, struct dasd_device *device)
1267	{
1268	unsigned long flags;
1269
1270	spin_lock_irqsave(&device->mem_lock, flags);
1271	dasd_free_chunk(chunk_list: &device->ese_chunks, mem: cqr);
1272	spin_unlock_irqrestore(lock: &device->mem_lock, flags);
1273	dasd_put_device(device);
1274	}
1275	EXPORT_SYMBOL(dasd_ffree_request);
1276
1277	/*
1278	* Check discipline magic in cqr.
1279	*/
1280	static inline int dasd_check_cqr(struct dasd_ccw_req *cqr)
1281	{
1282	struct dasd_device *device;
1283
1284	if (cqr == NULL)
1285	return -EINVAL;
1286	device = cqr->startdev;
1287	if (strncmp((char *) &cqr->magic, device->discipline->ebcname, 4)) {
1288	DBF_DEV_EVENT(DBF_WARNING, device,
1289	" dasd_ccw_req 0x%08x magic doesn't match"
1290	" discipline 0x%08x",
1291	cqr->magic,
1292	*(unsigned int *) device->discipline->name);
1293	return -EINVAL;
1294	}
1295	return 0;
1296	}
1297
1298	/*
1299	* Terminate the current i/o and set the request to clear_pending.
1300	* Timer keeps device runnig.
1301	* ccw_device_clear can fail if the i/o subsystem
1302	* is in a bad mood.
1303	*/
1304	int dasd_term_IO(struct dasd_ccw_req *cqr)
1305	{
1306	struct dasd_device *device;
1307	int retries, rc;
1308	char errorstring[ERRORLENGTH];
1309
1310	/* Check the cqr */
1311	rc = dasd_check_cqr(cqr);
1312	if (rc)
1313	return rc;
1314	retries = 0;
1315	device = (struct dasd_device *) cqr->startdev;
1316	while ((retries < 5) && (cqr->status == DASD_CQR_IN_IO)) {
1317	rc = ccw_device_clear(device->cdev, (long) cqr);
1318	switch (rc) {
1319	case 0: /* termination successful */
1320	cqr->status = DASD_CQR_CLEAR_PENDING;
1321	cqr->stopclk = get_tod_clock();
1322	cqr->starttime = 0;
1323	DBF_DEV_EVENT(DBF_DEBUG, device,
1324	"terminate cqr %p successful",
1325	cqr);
1326	break;
1327	case -ENODEV:
1328	DBF_DEV_EVENT(DBF_ERR, device, "%s",
1329	"device gone, retry");
1330	break;
1331	case -EINVAL:
1332	/*
1333	* device not valid so no I/O could be running
1334	* handle CQR as termination successful
1335	*/
1336	cqr->status = DASD_CQR_CLEARED;
1337	cqr->stopclk = get_tod_clock();
1338	cqr->starttime = 0;
1339	/* no retries for invalid devices */
1340	cqr->retries = -1;
1341	DBF_DEV_EVENT(DBF_ERR, device, "%s",
1342	"EINVAL, handle as terminated");
1343	/* fake rc to success */
1344	rc = 0;
1345	break;
1346	default:
1347	/* internal error 10 - unknown rc*/
1348	snprintf(buf: errorstring, ERRORLENGTH, fmt: "10 %d", rc);
1349	dev_err(&device->cdev->dev, "An error occurred in the "
1350	"DASD device driver, reason=%s\n", errorstring);
1351	BUG();
1352	break;
1353	}
1354	retries++;
1355	}
1356	dasd_schedule_device_bh(device);
1357	return rc;
1358	}
1359	EXPORT_SYMBOL(dasd_term_IO);
1360
1361	/*
1362	* Start the i/o. This start_IO can fail if the channel is really busy.
1363	* In that case set up a timer to start the request later.
1364	*/
1365	int dasd_start_IO(struct dasd_ccw_req *cqr)
1366	{
1367	struct dasd_device *device;
1368	int rc;
1369	char errorstring[ERRORLENGTH];
1370
1371	/* Check the cqr */
1372	rc = dasd_check_cqr(cqr);
1373	if (rc) {
1374	cqr->intrc = rc;
1375	return rc;
1376	}
1377	device = (struct dasd_device *) cqr->startdev;
1378	if (((cqr->block &&
1379	test_bit(DASD_FLAG_LOCK_STOLEN, &cqr->block->base->flags)) ||
1380	test_bit(DASD_FLAG_LOCK_STOLEN, &device->flags)) &&
1381	!test_bit(DASD_CQR_ALLOW_SLOCK, &cqr->flags)) {
1382	DBF_DEV_EVENT(DBF_DEBUG, device, "start_IO: return request %p "
1383	"because of stolen lock", cqr);
1384	cqr->status = DASD_CQR_ERROR;
1385	cqr->intrc = -EPERM;
1386	return -EPERM;
1387	}
1388	if (cqr->retries < 0) {
1389	/* internal error 14 - start_IO run out of retries */
1390	sprintf(buf: errorstring, fmt: "14 %p", cqr);
1391	dev_err(&device->cdev->dev, "An error occurred in the DASD "
1392	"device driver, reason=%s\n", errorstring);
1393	cqr->status = DASD_CQR_ERROR;
1394	return -EIO;
1395	}
1396	cqr->startclk = get_tod_clock();
1397	cqr->starttime = jiffies;
1398	cqr->retries--;
1399	if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) {
1400	cqr->lpm &= dasd_path_get_opm(device);
1401	if (!cqr->lpm)
1402	cqr->lpm = dasd_path_get_opm(device);
1403	}
1404	/*
1405	* remember the amount of formatted tracks to prevent double format on
1406	* ESE devices
1407	*/
1408	if (cqr->block)
1409	cqr->trkcount = atomic_read(v: &cqr->block->trkcount);
1410
1411	if (cqr->cpmode == 1) {
1412	rc = ccw_device_tm_start(device->cdev, cqr->cpaddr,
1413	(long) cqr, cqr->lpm);
1414	} else {
1415	rc = ccw_device_start(device->cdev, cqr->cpaddr,
1416	(long) cqr, cqr->lpm, 0);
1417	}
1418	switch (rc) {
1419	case 0:
1420	cqr->status = DASD_CQR_IN_IO;
1421	break;
1422	case -EBUSY:
1423	DBF_DEV_EVENT(DBF_WARNING, device, "%s",
1424	"start_IO: device busy, retry later");
1425	break;
1426	case -EACCES:
1427	/* -EACCES indicates that the request used only a subset of the
1428	* available paths and all these paths are gone. If the lpm of
1429	* this request was only a subset of the opm (e.g. the ppm) then
1430	* we just do a retry with all available paths.
1431	* If we already use the full opm, something is amiss, and we
1432	* need a full path verification.
1433	*/
1434	if (test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) {
1435	DBF_DEV_EVENT(DBF_WARNING, device,
1436	"start_IO: selected paths gone (%x)",
1437	cqr->lpm);
1438	} else if (cqr->lpm != dasd_path_get_opm(device)) {
1439	cqr->lpm = dasd_path_get_opm(device);
1440	DBF_DEV_EVENT(DBF_DEBUG, device, "%s",
1441	"start_IO: selected paths gone,"
1442	" retry on all paths");
1443	} else {
1444	DBF_DEV_EVENT(DBF_WARNING, device, "%s",
1445	"start_IO: all paths in opm gone,"
1446	" do path verification");
1447	dasd_generic_last_path_gone(device);
1448	dasd_path_no_path(device);
1449	dasd_path_set_tbvpm(device,
1450	pm: ccw_device_get_path_mask(
1451	device->cdev));
1452	}
1453	break;
1454	case -ENODEV:
1455	DBF_DEV_EVENT(DBF_WARNING, device, "%s",
1456	"start_IO: -ENODEV device gone, retry");
1457	/* this is equivalent to CC=3 for SSCH report this to EER */
1458	dasd_handle_autoquiesce(device, cqr, DASD_EER_STARTIO);
1459	break;
1460	case -EIO:
1461	DBF_DEV_EVENT(DBF_WARNING, device, "%s",
1462	"start_IO: -EIO device gone, retry");
1463	break;
1464	case -EINVAL:
1465	DBF_DEV_EVENT(DBF_WARNING, device, "%s",
1466	"start_IO: -EINVAL device currently "
1467	"not accessible");
1468	break;
1469	default:
1470	/* internal error 11 - unknown rc */
1471	snprintf(buf: errorstring, ERRORLENGTH, fmt: "11 %d", rc);
1472	dev_err(&device->cdev->dev,
1473	"An error occurred in the DASD device driver, "
1474	"reason=%s\n", errorstring);
1475	BUG();
1476	break;
1477	}
1478	cqr->intrc = rc;
1479	return rc;
1480	}
1481	EXPORT_SYMBOL(dasd_start_IO);
1482
1483	/*
1484	* Timeout function for dasd devices. This is used for different purposes
1485	* 1) missing interrupt handler for normal operation
1486	* 2) delayed start of request where start_IO failed with -EBUSY
1487	* 3) timeout for missing state change interrupts
1488	* The head of the ccw queue will have status DASD_CQR_IN_IO for 1),
1489	* DASD_CQR_QUEUED for 2) and 3).
1490	*/
1491	static void dasd_device_timeout(struct timer_list *t)
1492	{
1493	unsigned long flags;
1494	struct dasd_device *device;
1495
1496	device = from_timer(device, t, timer);
1497	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
1498	/* re-activate request queue */
1499	dasd_device_remove_stop_bits(device, DASD_STOPPED_PENDING);
1500	spin_unlock_irqrestore(lock: get_ccwdev_lock(device->cdev), flags);
1501	dasd_schedule_device_bh(device);
1502	}
1503
1504	/*
1505	* Setup timeout for a device in jiffies.
1506	*/
1507	void dasd_device_set_timer(struct dasd_device *device, int expires)
1508	{
1509	if (expires == 0)
1510	del_timer(timer: &device->timer);
1511	else
1512	mod_timer(timer: &device->timer, expires: jiffies + expires);
1513	}
1514	EXPORT_SYMBOL(dasd_device_set_timer);
1515
1516	/*
1517	* Clear timeout for a device.
1518	*/
1519	void dasd_device_clear_timer(struct dasd_device *device)
1520	{
1521	del_timer(timer: &device->timer);
1522	}
1523	EXPORT_SYMBOL(dasd_device_clear_timer);
1524
1525	static void dasd_handle_killed_request(struct ccw_device *cdev,
1526	unsigned long intparm)
1527	{
1528	struct dasd_ccw_req *cqr;
1529	struct dasd_device *device;
1530
1531	if (!intparm)
1532	return;
1533	cqr = (struct dasd_ccw_req *) intparm;
1534	if (cqr->status != DASD_CQR_IN_IO) {
1535	DBF_EVENT_DEVID(DBF_DEBUG, cdev,
1536	"invalid status in handle_killed_request: "
1537	"%02x", cqr->status);
1538	return;
1539	}
1540
1541	device = dasd_device_from_cdev_locked(cdev);
1542	if (IS_ERR(ptr: device)) {
1543	DBF_EVENT_DEVID(DBF_DEBUG, cdev, "%s",
1544	"unable to get device from cdev");
1545	return;
1546	}
1547
1548	if (!cqr->startdev ||
1549	device != cqr->startdev ||
1550	strncmp(cqr->startdev->discipline->ebcname,
1551	(char *) &cqr->magic, 4)) {
1552	DBF_EVENT_DEVID(DBF_DEBUG, cdev, "%s",
1553	"invalid device in request");
1554	dasd_put_device(device);
1555	return;
1556	}
1557
1558	/* Schedule request to be retried. */
1559	cqr->status = DASD_CQR_QUEUED;
1560
1561	dasd_device_clear_timer(device);
1562	dasd_schedule_device_bh(device);
1563	dasd_put_device(device);
1564	}
1565
1566	void dasd_generic_handle_state_change(struct dasd_device *device)
1567	{
1568	/* First of all start sense subsystem status request. */
1569	dasd_eer_snss(device);
1570
1571	dasd_device_remove_stop_bits(device, DASD_STOPPED_PENDING);
1572	dasd_schedule_device_bh(device);
1573	if (device->block) {
1574	dasd_schedule_block_bh(device->block);
1575	if (device->block->gdp)
1576	blk_mq_run_hw_queues(q: device->block->gdp->queue, async: true);
1577	}
1578	}
1579	EXPORT_SYMBOL_GPL(dasd_generic_handle_state_change);
1580
1581	static int dasd_check_hpf_error(struct irb *irb)
1582	{
1583	return (scsw_tm_is_valid_schxs(&irb->scsw) &&
1584	(irb->scsw.tm.sesq == SCSW_SESQ_DEV_NOFCX ||
1585	irb->scsw.tm.sesq == SCSW_SESQ_PATH_NOFCX));
1586	}
1587
1588	static int dasd_ese_needs_format(struct dasd_block *block, struct irb *irb)
1589	{
1590	struct dasd_device *device = NULL;
1591	u8 *sense = NULL;
1592
1593	if (!block)
1594	return 0;
1595	device = block->base;
1596	if (!device || !device->discipline->is_ese)
1597	return 0;
1598	if (!device->discipline->is_ese(device))
1599	return 0;
1600
1601	sense = dasd_get_sense(irb);
1602	if (!sense)
1603	return 0;
1604
1605	return !!(sense[1] & SNS1_NO_REC_FOUND) ||
1606	!!(sense[1] & SNS1_FILE_PROTECTED) ||
1607	scsw_cstat(&irb->scsw) == SCHN_STAT_INCORR_LEN;
1608	}
1609
1610	static int dasd_ese_oos_cond(u8 *sense)
1611	{
1612	return sense[0] & SNS0_EQUIPMENT_CHECK &&
1613	sense[1] & SNS1_PERM_ERR &&
1614	sense[1] & SNS1_WRITE_INHIBITED &&
1615	sense[25] == 0x01;
1616	}
1617
1618	/*
1619	* Interrupt handler for "normal" ssch-io based dasd devices.
1620	*/
1621	void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm,
1622	struct irb *irb)
1623	{
1624	struct dasd_ccw_req *cqr, *next, *fcqr;
1625	struct dasd_device *device;
1626	unsigned long now;
1627	int nrf_suppressed = 0;
1628	int fp_suppressed = 0;
1629	struct request *req;
1630	u8 *sense = NULL;
1631	int expires;
1632
1633	cqr = (struct dasd_ccw_req *) intparm;
1634	if (IS_ERR(ptr: irb)) {
1635	switch (PTR_ERR(ptr: irb)) {
1636	case -EIO:
1637	if (cqr && cqr->status == DASD_CQR_CLEAR_PENDING) {
1638	device = cqr->startdev;
1639	cqr->status = DASD_CQR_CLEARED;
1640	dasd_device_clear_timer(device);
1641	wake_up(&dasd_flush_wq);
1642	dasd_schedule_device_bh(device);
1643	return;
1644	}
1645	break;
1646	case -ETIMEDOUT:
1647	DBF_EVENT_DEVID(DBF_WARNING, cdev, "%s: "
1648	"request timed out\n", __func__);
1649	break;
1650	default:
1651	DBF_EVENT_DEVID(DBF_WARNING, cdev, "%s: "
1652	"unknown error %ld\n", __func__,
1653	PTR_ERR(irb));
1654	}
1655	dasd_handle_killed_request(cdev, intparm);
1656	return;
1657	}
1658
1659	now = get_tod_clock();
1660	/* check for conditions that should be handled immediately */
1661	if (!cqr ||
1662	!(scsw_dstat(&irb->scsw) == (DEV_STAT_CHN_END | DEV_STAT_DEV_END) &&
1663	scsw_cstat(&irb->scsw) == 0)) {
1664	if (cqr)
1665	memcpy(&cqr->irb, irb, sizeof(*irb));
1666	device = dasd_device_from_cdev_locked(cdev);
1667	if (IS_ERR(ptr: device))
1668	return;
1669	/* ignore unsolicited interrupts for DIAG discipline */
1670	if (device->discipline == dasd_diag_discipline_pointer) {
1671	dasd_put_device(device);
1672	return;
1673	}
1674
1675	/*
1676	* In some cases 'File Protected' or 'No Record Found' errors
1677	* might be expected and debug log messages for the
1678	* corresponding interrupts shouldn't be written then.
1679	* Check if either of the according suppress bits is set.
1680	*/
1681	sense = dasd_get_sense(irb);
1682	if (sense) {
1683	fp_suppressed = (sense[1] & SNS1_FILE_PROTECTED) &&
1684	test_bit(DASD_CQR_SUPPRESS_FP, &cqr->flags);
1685	nrf_suppressed = (sense[1] & SNS1_NO_REC_FOUND) &&
1686	test_bit(DASD_CQR_SUPPRESS_NRF, &cqr->flags);
1687
1688	/*
1689	* Extent pool probably out-of-space.
1690	* Stop device and check exhaust level.
1691	*/
1692	if (dasd_ese_oos_cond(sense)) {
1693	dasd_generic_space_exhaust(device, cqr);
1694	device->discipline->ext_pool_exhaust(device, cqr);
1695	dasd_put_device(device);
1696	return;
1697	}
1698	}
1699	if (!(fp_suppressed || nrf_suppressed))
1700	device->discipline->dump_sense_dbf(device, irb, "int");
1701
1702	if (device->features & DASD_FEATURE_ERPLOG)
1703	device->discipline->dump_sense(device, cqr, irb);
1704	device->discipline->check_for_device_change(device, cqr, irb);
1705	dasd_put_device(device);
1706	}
1707
1708	/* check for attention message */
1709	if (scsw_dstat(&irb->scsw) & DEV_STAT_ATTENTION) {
1710	device = dasd_device_from_cdev_locked(cdev);
1711	if (!IS_ERR(ptr: device)) {
1712	device->discipline->check_attention(device,
1713	irb->esw.esw1.lpum);
1714	dasd_put_device(device);
1715	}
1716	}
1717
1718	if (!cqr)
1719	return;
1720
1721	device = (struct dasd_device *) cqr->startdev;
1722	if (!device ||
1723	strncmp(device->discipline->ebcname, (char *) &cqr->magic, 4)) {
1724	DBF_EVENT_DEVID(DBF_DEBUG, cdev, "%s",
1725	"invalid device in request");
1726	return;
1727	}
1728
1729	if (dasd_ese_needs_format(block: cqr->block, irb)) {
1730	req = dasd_get_callback_data(cqr);
1731	if (!req) {
1732	cqr->status = DASD_CQR_ERROR;
1733	return;
1734	}
1735	if (rq_data_dir(req) == READ) {
1736	device->discipline->ese_read(cqr, irb);
1737	cqr->status = DASD_CQR_SUCCESS;
1738	cqr->stopclk = now;
1739	dasd_device_clear_timer(device);
1740	dasd_schedule_device_bh(device);
1741	return;
1742	}
1743	fcqr = device->discipline->ese_format(device, cqr, irb);
1744	if (IS_ERR(ptr: fcqr)) {
1745	if (PTR_ERR(ptr: fcqr) == -EINVAL) {
1746	cqr->status = DASD_CQR_ERROR;
1747	return;
1748	}
1749	/*
1750	* If we can't format now, let the request go
1751	* one extra round. Maybe we can format later.
1752	*/
1753	cqr->status = DASD_CQR_QUEUED;
1754	dasd_schedule_device_bh(device);
1755	return;
1756	} else {
1757	fcqr->status = DASD_CQR_QUEUED;
1758	cqr->status = DASD_CQR_QUEUED;
1759	list_add(new: &fcqr->devlist, head: &device->ccw_queue);
1760	dasd_schedule_device_bh(device);
1761	return;
1762	}
1763	}
1764
1765	/* Check for clear pending */
1766	if (cqr->status == DASD_CQR_CLEAR_PENDING &&
1767	scsw_fctl(&irb->scsw) & SCSW_FCTL_CLEAR_FUNC) {
1768	cqr->status = DASD_CQR_CLEARED;
1769	dasd_device_clear_timer(device);
1770	wake_up(&dasd_flush_wq);
1771	dasd_schedule_device_bh(device);
1772	return;
1773	}
1774
1775	/* check status - the request might have been killed by dyn detach */
1776	if (cqr->status != DASD_CQR_IN_IO) {
1777	DBF_DEV_EVENT(DBF_DEBUG, device, "invalid status: bus_id %s, "
1778	"status %02x", dev_name(&cdev->dev), cqr->status);
1779	return;
1780	}
1781
1782	next = NULL;
1783	expires = 0;
1784	if (scsw_dstat(&irb->scsw) == (DEV_STAT_CHN_END | DEV_STAT_DEV_END) &&
1785	scsw_cstat(&irb->scsw) == 0) {
1786	/* request was completed successfully */
1787	cqr->status = DASD_CQR_SUCCESS;
1788	cqr->stopclk = now;
1789	/* Start first request on queue if possible -> fast_io. */
1790	if (cqr->devlist.next != &device->ccw_queue) {
1791	next = list_entry(cqr->devlist.next,
1792	struct dasd_ccw_req, devlist);
1793	}
1794	} else { /* error */
1795	/* check for HPF error
1796	* call discipline function to requeue all requests
1797	* and disable HPF accordingly
1798	*/
1799	if (cqr->cpmode && dasd_check_hpf_error(irb) &&
1800	device->discipline->handle_hpf_error)
1801	device->discipline->handle_hpf_error(device, irb);
1802	/*
1803	* If we don't want complex ERP for this request, then just
1804	* reset this and retry it in the fastpath
1805	*/
1806	if (!test_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags) &&
1807	cqr->retries > 0) {
1808	if (cqr->lpm == dasd_path_get_opm(device))
1809	DBF_DEV_EVENT(DBF_DEBUG, device,
1810	"default ERP in fastpath "
1811	"(%i retries left)",
1812	cqr->retries);
1813	if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags))
1814	cqr->lpm = dasd_path_get_opm(device);
1815	cqr->status = DASD_CQR_QUEUED;
1816	next = cqr;
1817	} else
1818	cqr->status = DASD_CQR_ERROR;
1819	}
1820	if (next && (next->status == DASD_CQR_QUEUED) &&
1821	(!device->stopped)) {
1822	if (device->discipline->start_IO(next) == 0)
1823	expires = next->expires;
1824	}
1825	if (expires != 0)
1826	dasd_device_set_timer(device, expires);
1827	else
1828	dasd_device_clear_timer(device);
1829	dasd_schedule_device_bh(device);
1830	}
1831	EXPORT_SYMBOL(dasd_int_handler);
1832
1833	enum uc_todo dasd_generic_uc_handler(struct ccw_device *cdev, struct irb *irb)
1834	{
1835	struct dasd_device *device;
1836
1837	device = dasd_device_from_cdev_locked(cdev);
1838
1839	if (IS_ERR(ptr: device))
1840	goto out;
1841	if (test_bit(DASD_FLAG_OFFLINE, &device->flags) ||
1842	device->state != device->target ||
1843	!device->discipline->check_for_device_change){
1844	dasd_put_device(device);
1845	goto out;
1846	}
1847	if (device->discipline->dump_sense_dbf)
1848	device->discipline->dump_sense_dbf(device, irb, "uc");
1849	device->discipline->check_for_device_change(device, NULL, irb);
1850	dasd_put_device(device);
1851	out:
1852	return UC_TODO_RETRY;
1853	}
1854	EXPORT_SYMBOL_GPL(dasd_generic_uc_handler);
1855
1856	/*
1857	* If we have an error on a dasd_block layer request then we cancel
1858	* and return all further requests from the same dasd_block as well.
1859	*/
1860	static void __dasd_device_recovery(struct dasd_device *device,
1861	struct dasd_ccw_req *ref_cqr)
1862	{
1863	struct list_head *l, *n;
1864	struct dasd_ccw_req *cqr;
1865
1866	/*
1867	* only requeue request that came from the dasd_block layer
1868	*/
1869	if (!ref_cqr->block)
1870	return;
1871
1872	list_for_each_safe(l, n, &device->ccw_queue) {
1873	cqr = list_entry(l, struct dasd_ccw_req, devlist);
1874	if (cqr->status == DASD_CQR_QUEUED &&
1875	ref_cqr->block == cqr->block) {
1876	cqr->status = DASD_CQR_CLEARED;
1877	}
1878	}
1879	};
1880
1881	/*
1882	* Remove those ccw requests from the queue that need to be returned
1883	* to the upper layer.
1884	*/
1885	static void __dasd_device_process_ccw_queue(struct dasd_device *device,
1886	struct list_head *final_queue)
1887	{
1888	struct list_head *l, *n;
1889	struct dasd_ccw_req *cqr;
1890
1891	/* Process request with final status. */
1892	list_for_each_safe(l, n, &device->ccw_queue) {
1893	cqr = list_entry(l, struct dasd_ccw_req, devlist);
1894
1895	/* Skip any non-final request. */
1896	if (cqr->status == DASD_CQR_QUEUED ||
1897	cqr->status == DASD_CQR_IN_IO ||
1898	cqr->status == DASD_CQR_CLEAR_PENDING)
1899	continue;
1900	if (cqr->status == DASD_CQR_ERROR) {
1901	__dasd_device_recovery(device, ref_cqr: cqr);
1902	}
1903	/* Rechain finished requests to final queue */
1904	list_move_tail(list: &cqr->devlist, head: final_queue);
1905	}
1906	}
1907
1908	static void __dasd_process_cqr(struct dasd_device *device,
1909	struct dasd_ccw_req *cqr)
1910	{
1911	char errorstring[ERRORLENGTH];
1912
1913	switch (cqr->status) {
1914	case DASD_CQR_SUCCESS:
1915	cqr->status = DASD_CQR_DONE;
1916	break;
1917	case DASD_CQR_ERROR:
1918	cqr->status = DASD_CQR_NEED_ERP;
1919	break;
1920	case DASD_CQR_CLEARED:
1921	cqr->status = DASD_CQR_TERMINATED;
1922	break;
1923	default:
1924	/* internal error 12 - wrong cqr status*/
1925	snprintf(buf: errorstring, ERRORLENGTH, fmt: "12 %p %x02", cqr, cqr->status);
1926	dev_err(&device->cdev->dev,
1927	"An error occurred in the DASD device driver, "
1928	"reason=%s\n", errorstring);
1929	BUG();
1930	}
1931	if (cqr->callback)
1932	cqr->callback(cqr, cqr->callback_data);
1933	}
1934
1935	/*
1936	* the cqrs from the final queue are returned to the upper layer
1937	* by setting a dasd_block state and calling the callback function
1938	*/
1939	static void __dasd_device_process_final_queue(struct dasd_device *device,
1940	struct list_head *final_queue)
1941	{
1942	struct list_head *l, *n;
1943	struct dasd_ccw_req *cqr;
1944	struct dasd_block *block;
1945
1946	list_for_each_safe(l, n, final_queue) {
1947	cqr = list_entry(l, struct dasd_ccw_req, devlist);
1948	list_del_init(entry: &cqr->devlist);
1949	block = cqr->block;
1950	if (!block) {
1951	__dasd_process_cqr(device, cqr);
1952	} else {
1953	spin_lock_bh(lock: &block->queue_lock);
1954	__dasd_process_cqr(device, cqr);
1955	spin_unlock_bh(lock: &block->queue_lock);
1956	}
1957	}
1958	}
1959
1960	/*
1961	* check if device should be autoquiesced due to too many timeouts
1962	*/
1963	static void __dasd_device_check_autoquiesce_timeout(struct dasd_device *device,
1964	struct dasd_ccw_req *cqr)
1965	{
1966	if ((device->default_retries - cqr->retries) >= device->aq_timeouts)
1967	dasd_handle_autoquiesce(device, cqr, DASD_EER_TIMEOUTS);
1968	}
1969
1970	/*
1971	* Take a look at the first request on the ccw queue and check
1972	* if it reached its expire time. If so, terminate the IO.
1973	*/
1974	static void __dasd_device_check_expire(struct dasd_device *device)
1975	{
1976	struct dasd_ccw_req *cqr;
1977
1978	if (list_empty(head: &device->ccw_queue))
1979	return;
1980	cqr = list_entry(device->ccw_queue.next, struct dasd_ccw_req, devlist);
1981	if ((cqr->status == DASD_CQR_IN_IO && cqr->expires != 0) &&
1982	(time_after_eq(jiffies, cqr->expires + cqr->starttime))) {
1983	if (test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) {
1984	/*
1985	* IO in safe offline processing should not
1986	* run out of retries
1987	*/
1988	cqr->retries++;
1989	}
1990	if (device->discipline->term_IO(cqr) != 0) {
1991	/* Hmpf, try again in 5 sec */
1992	dev_err(&device->cdev->dev,
1993	"cqr %p timed out (%lus) but cannot be "
1994	"ended, retrying in 5 s\n",
1995	cqr, (cqr->expires/HZ));
1996	cqr->expires += 5*HZ;
1997	dasd_device_set_timer(device, 5*HZ);
1998	} else {
1999	dev_err(&device->cdev->dev,
2000	"cqr %p timed out (%lus), %i retries "
2001	"remaining\n", cqr, (cqr->expires/HZ),
2002	cqr->retries);
2003	}
2004	__dasd_device_check_autoquiesce_timeout(device, cqr);
2005	}
2006	}
2007
2008	/*
2009	* return 1 when device is not eligible for IO
2010	*/
2011	static int __dasd_device_is_unusable(struct dasd_device *device,
2012	struct dasd_ccw_req *cqr)
2013	{
2014	int mask = ~(DASD_STOPPED_DC_WAIT | DASD_STOPPED_NOSPC);
2015
2016	if (test_bit(DASD_FLAG_OFFLINE, &device->flags) &&
2017	!test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) {
2018	/*
2019	* dasd is being set offline
2020	* but it is no safe offline where we have to allow I/O
2021	*/
2022	return 1;
2023	}
2024	if (device->stopped) {
2025	if (device->stopped & mask) {
2026	/* stopped and CQR will not change that. */
2027	return 1;
2028	}
2029	if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) {
2030	/* CQR is not able to change device to
2031	* operational. */
2032	return 1;
2033	}
2034	/* CQR required to get device operational. */
2035	}
2036	return 0;
2037	}
2038
2039	/*
2040	* Take a look at the first request on the ccw queue and check
2041	* if it needs to be started.
2042	*/
2043	static void __dasd_device_start_head(struct dasd_device *device)
2044	{
2045	struct dasd_ccw_req *cqr;
2046	int rc;
2047
2048	if (list_empty(head: &device->ccw_queue))
2049	return;
2050	cqr = list_entry(device->ccw_queue.next, struct dasd_ccw_req, devlist);
2051	if (cqr->status != DASD_CQR_QUEUED)
2052	return;
2053	/* if device is not usable return request to upper layer */
2054	if (__dasd_device_is_unusable(device, cqr)) {
2055	cqr->intrc = -EAGAIN;
2056	cqr->status = DASD_CQR_CLEARED;
2057	dasd_schedule_device_bh(device);
2058	return;
2059	}
2060
2061	rc = device->discipline->start_IO(cqr);
2062	if (rc == 0)
2063	dasd_device_set_timer(device, cqr->expires);
2064	else if (rc == -EACCES) {
2065	dasd_schedule_device_bh(device);
2066	} else
2067	/* Hmpf, try again in 1/2 sec */
2068	dasd_device_set_timer(device, 50);
2069	}
2070
2071	static void __dasd_device_check_path_events(struct dasd_device *device)
2072	{
2073	__u8 tbvpm, fcsecpm;
2074	int rc;
2075
2076	tbvpm = dasd_path_get_tbvpm(device);
2077	fcsecpm = dasd_path_get_fcsecpm(device);
2078
2079	if (!tbvpm && !fcsecpm)
2080	return;
2081
2082	if (device->stopped & ~(DASD_STOPPED_DC_WAIT))
2083	return;
2084
2085	dasd_path_clear_all_verify(device);
2086	dasd_path_clear_all_fcsec(device);
2087
2088	rc = device->discipline->pe_handler(device, tbvpm, fcsecpm);
2089	if (rc) {
2090	dasd_path_add_tbvpm(device, pm: tbvpm);
2091	dasd_path_add_fcsecpm(device, pm: fcsecpm);
2092	dasd_device_set_timer(device, 50);
2093	}
2094	};
2095
2096	/*
2097	* Go through all request on the dasd_device request queue,
2098	* terminate them on the cdev if necessary, and return them to the
2099	* submitting layer via callback.
2100	* Note:
2101	* Make sure that all 'submitting layers' still exist when
2102	* this function is called!. In other words, when 'device' is a base
2103	* device then all block layer requests must have been removed before
2104	* via dasd_flush_block_queue.
2105	*/
2106	int dasd_flush_device_queue(struct dasd_device *device)
2107	{
2108	struct dasd_ccw_req *cqr, *n;
2109	int rc;
2110	struct list_head flush_queue;
2111
2112	INIT_LIST_HEAD(list: &flush_queue);
2113	spin_lock_irq(lock: get_ccwdev_lock(device->cdev));
2114	rc = 0;
2115	list_for_each_entry_safe(cqr, n, &device->ccw_queue, devlist) {
2116	/* Check status and move request to flush_queue */
2117	switch (cqr->status) {
2118	case DASD_CQR_IN_IO:
2119	rc = device->discipline->term_IO(cqr);
2120	if (rc) {
2121	/* unable to terminate requeust */
2122	dev_err(&device->cdev->dev,
2123	"Flushing the DASD request queue "
2124	"failed for request %p\n", cqr);
2125	/* stop flush processing */
2126	goto finished;
2127	}
2128	break;
2129	case DASD_CQR_QUEUED:
2130	cqr->stopclk = get_tod_clock();
2131	cqr->status = DASD_CQR_CLEARED;
2132	break;
2133	default: /* no need to modify the others */
2134	break;
2135	}
2136	list_move_tail(list: &cqr->devlist, head: &flush_queue);
2137	}
2138	finished:
2139	spin_unlock_irq(lock: get_ccwdev_lock(device->cdev));
2140	/*
2141	* After this point all requests must be in state CLEAR_PENDING,
2142	* CLEARED, SUCCESS or ERROR. Now wait for CLEAR_PENDING to become
2143	* one of the others.
2144	*/
2145	list_for_each_entry_safe(cqr, n, &flush_queue, devlist)
2146	wait_event(dasd_flush_wq,
2147	(cqr->status != DASD_CQR_CLEAR_PENDING));
2148	/*
2149	* Now set each request back to TERMINATED, DONE or NEED_ERP
2150	* and call the callback function of flushed requests
2151	*/
2152	__dasd_device_process_final_queue(device, final_queue: &flush_queue);
2153	return rc;
2154	}
2155	EXPORT_SYMBOL_GPL(dasd_flush_device_queue);
2156
2157	/*
2158	* Acquire the device lock and process queues for the device.
2159	*/
2160	static void dasd_device_tasklet(unsigned long data)
2161	{
2162	struct dasd_device *device = (struct dasd_device *) data;
2163	struct list_head final_queue;
2164
2165	atomic_set (v: &device->tasklet_scheduled, i: 0);
2166	INIT_LIST_HEAD(list: &final_queue);
2167	spin_lock_irq(lock: get_ccwdev_lock(device->cdev));
2168	/* Check expire time of first request on the ccw queue. */
2169	__dasd_device_check_expire(device);
2170	/* find final requests on ccw queue */
2171	__dasd_device_process_ccw_queue(device, final_queue: &final_queue);
2172	__dasd_device_check_path_events(device);
2173	spin_unlock_irq(lock: get_ccwdev_lock(device->cdev));
2174	/* Now call the callback function of requests with final status */
2175	__dasd_device_process_final_queue(device, final_queue: &final_queue);
2176	spin_lock_irq(lock: get_ccwdev_lock(device->cdev));
2177	/* Now check if the head of the ccw queue needs to be started. */
2178	__dasd_device_start_head(device);
2179	spin_unlock_irq(lock: get_ccwdev_lock(device->cdev));
2180	if (waitqueue_active(wq_head: &shutdown_waitq))
2181	wake_up(&shutdown_waitq);
2182	dasd_put_device(device);
2183	}
2184
2185	/*
2186	* Schedules a call to dasd_tasklet over the device tasklet.
2187	*/
2188	void dasd_schedule_device_bh(struct dasd_device *device)
2189	{
2190	/* Protect against rescheduling. */
2191	if (atomic_cmpxchg (v: &device->tasklet_scheduled, old: 0, new: 1) != 0)
2192	return;
2193	dasd_get_device(device);
2194	tasklet_hi_schedule(t: &device->tasklet);
2195	}
2196	EXPORT_SYMBOL(dasd_schedule_device_bh);
2197
2198	void dasd_device_set_stop_bits(struct dasd_device *device, int bits)
2199	{
2200	device->stopped |= bits;
2201	}
2202	EXPORT_SYMBOL_GPL(dasd_device_set_stop_bits);
2203
2204	void dasd_device_remove_stop_bits(struct dasd_device *device, int bits)
2205	{
2206	device->stopped &= ~bits;
2207	if (!device->stopped)
2208	wake_up(&generic_waitq);
2209	}
2210	EXPORT_SYMBOL_GPL(dasd_device_remove_stop_bits);
2211
2212	/*
2213	* Queue a request to the head of the device ccw_queue.
2214	* Start the I/O if possible.
2215	*/
2216	void dasd_add_request_head(struct dasd_ccw_req *cqr)
2217	{
2218	struct dasd_device *device;
2219	unsigned long flags;
2220
2221	device = cqr->startdev;
2222	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
2223	cqr->status = DASD_CQR_QUEUED;
2224	list_add(new: &cqr->devlist, head: &device->ccw_queue);
2225	/* let the bh start the request to keep them in order */
2226	dasd_schedule_device_bh(device);
2227	spin_unlock_irqrestore(lock: get_ccwdev_lock(device->cdev), flags);
2228	}
2229	EXPORT_SYMBOL(dasd_add_request_head);
2230
2231	/*
2232	* Queue a request to the tail of the device ccw_queue.
2233	* Start the I/O if possible.
2234	*/
2235	void dasd_add_request_tail(struct dasd_ccw_req *cqr)
2236	{
2237	struct dasd_device *device;
2238	unsigned long flags;
2239
2240	device = cqr->startdev;
2241	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
2242	cqr->status = DASD_CQR_QUEUED;
2243	list_add_tail(new: &cqr->devlist, head: &device->ccw_queue);
2244	/* let the bh start the request to keep them in order */
2245	dasd_schedule_device_bh(device);
2246	spin_unlock_irqrestore(lock: get_ccwdev_lock(device->cdev), flags);
2247	}
2248	EXPORT_SYMBOL(dasd_add_request_tail);
2249
2250	/*
2251	* Wakeup helper for the 'sleep_on' functions.
2252	*/
2253	void dasd_wakeup_cb(struct dasd_ccw_req *cqr, void *data)
2254	{
2255	spin_lock_irq(lock: get_ccwdev_lock(cqr->startdev->cdev));
2256	cqr->callback_data = DASD_SLEEPON_END_TAG;
2257	spin_unlock_irq(lock: get_ccwdev_lock(cqr->startdev->cdev));
2258	wake_up(&generic_waitq);
2259	}
2260	EXPORT_SYMBOL_GPL(dasd_wakeup_cb);
2261
2262	static inline int _wait_for_wakeup(struct dasd_ccw_req *cqr)
2263	{
2264	struct dasd_device *device;
2265	int rc;
2266
2267	device = cqr->startdev;
2268	spin_lock_irq(lock: get_ccwdev_lock(device->cdev));
2269	rc = (cqr->callback_data == DASD_SLEEPON_END_TAG);
2270	spin_unlock_irq(lock: get_ccwdev_lock(device->cdev));
2271	return rc;
2272	}
2273
2274	/*
2275	* checks if error recovery is necessary, returns 1 if yes, 0 otherwise.
2276	*/
2277	static int __dasd_sleep_on_erp(struct dasd_ccw_req *cqr)
2278	{
2279	struct dasd_device *device;
2280	dasd_erp_fn_t erp_fn;
2281
2282	if (cqr->status == DASD_CQR_FILLED)
2283	return 0;
2284	device = cqr->startdev;
2285	if (test_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags)) {
2286	if (cqr->status == DASD_CQR_TERMINATED) {
2287	device->discipline->handle_terminated_request(cqr);
2288	return 1;
2289	}
2290	if (cqr->status == DASD_CQR_NEED_ERP) {
2291	erp_fn = device->discipline->erp_action(cqr);
2292	erp_fn(cqr);
2293	return 1;
2294	}
2295	if (cqr->status == DASD_CQR_FAILED)
2296	dasd_log_sense(cqr, &cqr->irb);
2297	if (cqr->refers) {
2298	__dasd_process_erp(device, cqr);
2299	return 1;
2300	}
2301	}
2302	return 0;
2303	}
2304
2305	static int __dasd_sleep_on_loop_condition(struct dasd_ccw_req *cqr)
2306	{
2307	if (test_bit(DASD_CQR_FLAGS_USE_ERP, &cqr->flags)) {
2308	if (cqr->refers) /* erp is not done yet */
2309	return 1;
2310	return ((cqr->status != DASD_CQR_DONE) &&
2311	(cqr->status != DASD_CQR_FAILED));
2312	} else
2313	return (cqr->status == DASD_CQR_FILLED);
2314	}
2315
2316	static int _dasd_sleep_on(struct dasd_ccw_req *maincqr, int interruptible)
2317	{
2318	struct dasd_device *device;
2319	int rc;
2320	struct list_head ccw_queue;
2321	struct dasd_ccw_req *cqr;
2322
2323	INIT_LIST_HEAD(list: &ccw_queue);
2324	maincqr->status = DASD_CQR_FILLED;
2325	device = maincqr->startdev;
2326	list_add(new: &maincqr->blocklist, head: &ccw_queue);
2327	for (cqr = maincqr; __dasd_sleep_on_loop_condition(cqr);
2328	cqr = list_first_entry(&ccw_queue,
2329	struct dasd_ccw_req, blocklist)) {
2330
2331	if (__dasd_sleep_on_erp(cqr))
2332	continue;
2333	if (cqr->status != DASD_CQR_FILLED) /* could be failed */
2334	continue;
2335	if (test_bit(DASD_FLAG_LOCK_STOLEN, &device->flags) &&
2336	!test_bit(DASD_CQR_ALLOW_SLOCK, &cqr->flags)) {
2337	cqr->status = DASD_CQR_FAILED;
2338	cqr->intrc = -EPERM;
2339	continue;
2340	}
2341	/* Non-temporary stop condition will trigger fail fast */
2342	if (device->stopped & ~DASD_STOPPED_PENDING &&
2343	test_bit(DASD_CQR_FLAGS_FAILFAST, &cqr->flags) &&
2344	!dasd_eer_enabled(device) && device->aq_mask == 0) {
2345	cqr->status = DASD_CQR_FAILED;
2346	cqr->intrc = -ENOLINK;
2347	continue;
2348	}
2349	/*
2350	* Don't try to start requests if device is in
2351	* offline processing, it might wait forever
2352	*/
2353	if (test_bit(DASD_FLAG_OFFLINE, &device->flags)) {
2354	cqr->status = DASD_CQR_FAILED;
2355	cqr->intrc = -ENODEV;
2356	continue;
2357	}
2358	/*
2359	* Don't try to start requests if device is stopped
2360	* except path verification requests
2361	*/
2362	if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) {
2363	if (interruptible) {
2364	rc = wait_event_interruptible(
2365	generic_waitq, !(device->stopped));
2366	if (rc == -ERESTARTSYS) {
2367	cqr->status = DASD_CQR_FAILED;
2368	maincqr->intrc = rc;
2369	continue;
2370	}
2371	} else
2372	wait_event(generic_waitq, !(device->stopped));
2373	}
2374	if (!cqr->callback)
2375	cqr->callback = dasd_wakeup_cb;
2376
2377	cqr->callback_data = DASD_SLEEPON_START_TAG;
2378	dasd_add_request_tail(cqr);
2379	if (interruptible) {
2380	rc = wait_event_interruptible(
2381	generic_waitq, _wait_for_wakeup(cqr));
2382	if (rc == -ERESTARTSYS) {
2383	dasd_cancel_req(cqr);
2384	/* wait (non-interruptible) for final status */
2385	wait_event(generic_waitq,
2386	_wait_for_wakeup(cqr));
2387	cqr->status = DASD_CQR_FAILED;
2388	maincqr->intrc = rc;
2389	continue;
2390	}
2391	} else
2392	wait_event(generic_waitq, _wait_for_wakeup(cqr));
2393	}
2394
2395	maincqr->endclk = get_tod_clock();
2396	if ((maincqr->status != DASD_CQR_DONE) &&
2397	(maincqr->intrc != -ERESTARTSYS))
2398	dasd_log_sense(maincqr, &maincqr->irb);
2399	if (maincqr->status == DASD_CQR_DONE)
2400	rc = 0;
2401	else if (maincqr->intrc)
2402	rc = maincqr->intrc;
2403	else
2404	rc = -EIO;
2405	return rc;
2406	}
2407
2408	static inline int _wait_for_wakeup_queue(struct list_head *ccw_queue)
2409	{
2410	struct dasd_ccw_req *cqr;
2411
2412	list_for_each_entry(cqr, ccw_queue, blocklist) {
2413	if (cqr->callback_data != DASD_SLEEPON_END_TAG)
2414	return 0;
2415	}
2416
2417	return 1;
2418	}
2419
2420	static int _dasd_sleep_on_queue(struct list_head *ccw_queue, int interruptible)
2421	{
2422	struct dasd_device *device;
2423	struct dasd_ccw_req *cqr, *n;
2424	u8 *sense = NULL;
2425	int rc;
2426
2427	retry:
2428	list_for_each_entry_safe(cqr, n, ccw_queue, blocklist) {
2429	device = cqr->startdev;
2430	if (cqr->status != DASD_CQR_FILLED) /*could be failed*/
2431	continue;
2432
2433	if (test_bit(DASD_FLAG_LOCK_STOLEN, &device->flags) &&
2434	!test_bit(DASD_CQR_ALLOW_SLOCK, &cqr->flags)) {
2435	cqr->status = DASD_CQR_FAILED;
2436	cqr->intrc = -EPERM;
2437	continue;
2438	}
2439	/*Non-temporary stop condition will trigger fail fast*/
2440	if (device->stopped & ~DASD_STOPPED_PENDING &&
2441	test_bit(DASD_CQR_FLAGS_FAILFAST, &cqr->flags) &&
2442	!dasd_eer_enabled(device)) {
2443	cqr->status = DASD_CQR_FAILED;
2444	cqr->intrc = -EAGAIN;
2445	continue;
2446	}
2447
2448	/*Don't try to start requests if device is stopped*/
2449	if (interruptible) {
2450	rc = wait_event_interruptible(
2451	generic_waitq, !device->stopped);
2452	if (rc == -ERESTARTSYS) {
2453	cqr->status = DASD_CQR_FAILED;
2454	cqr->intrc = rc;
2455	continue;
2456	}
2457	} else
2458	wait_event(generic_waitq, !(device->stopped));
2459
2460	if (!cqr->callback)
2461	cqr->callback = dasd_wakeup_cb;
2462	cqr->callback_data = DASD_SLEEPON_START_TAG;
2463	dasd_add_request_tail(cqr);
2464	}
2465
2466	wait_event(generic_waitq, _wait_for_wakeup_queue(ccw_queue));
2467
2468	rc = 0;
2469	list_for_each_entry_safe(cqr, n, ccw_queue, blocklist) {
2470	/*
2471	* In some cases the 'File Protected' or 'Incorrect Length'
2472	* error might be expected and error recovery would be
2473	* unnecessary in these cases. Check if the according suppress
2474	* bit is set.
2475	*/
2476	sense = dasd_get_sense(&cqr->irb);
2477	if (sense && sense[1] & SNS1_FILE_PROTECTED &&
2478	test_bit(DASD_CQR_SUPPRESS_FP, &cqr->flags))
2479	continue;
2480	if (scsw_cstat(&cqr->irb.scsw) == 0x40 &&
2481	test_bit(DASD_CQR_SUPPRESS_IL, &cqr->flags))
2482	continue;
2483
2484	/*
2485	* for alias devices simplify error recovery and
2486	* return to upper layer
2487	* do not skip ERP requests
2488	*/
2489	if (cqr->startdev != cqr->basedev && !cqr->refers &&
2490	(cqr->status == DASD_CQR_TERMINATED ||
2491	cqr->status == DASD_CQR_NEED_ERP))
2492	return -EAGAIN;
2493
2494	/* normal recovery for basedev IO */
2495	if (__dasd_sleep_on_erp(cqr))
2496	/* handle erp first */
2497	goto retry;
2498	}
2499
2500	return 0;
2501	}
2502
2503	/*
2504	* Queue a request to the tail of the device ccw_queue and wait for
2505	* it's completion.
2506	*/
2507	int dasd_sleep_on(struct dasd_ccw_req *cqr)
2508	{
2509	return _dasd_sleep_on(maincqr: cqr, interruptible: 0);
2510	}
2511	EXPORT_SYMBOL(dasd_sleep_on);
2512
2513	/*
2514	* Start requests from a ccw_queue and wait for their completion.
2515	*/
2516	int dasd_sleep_on_queue(struct list_head *ccw_queue)
2517	{
2518	return _dasd_sleep_on_queue(ccw_queue, interruptible: 0);
2519	}
2520	EXPORT_SYMBOL(dasd_sleep_on_queue);
2521
2522	/*
2523	* Start requests from a ccw_queue and wait interruptible for their completion.
2524	*/
2525	int dasd_sleep_on_queue_interruptible(struct list_head *ccw_queue)
2526	{
2527	return _dasd_sleep_on_queue(ccw_queue, interruptible: 1);
2528	}
2529	EXPORT_SYMBOL(dasd_sleep_on_queue_interruptible);
2530
2531	/*
2532	* Queue a request to the tail of the device ccw_queue and wait
2533	* interruptible for it's completion.
2534	*/
2535	int dasd_sleep_on_interruptible(struct dasd_ccw_req *cqr)
2536	{
2537	return _dasd_sleep_on(maincqr: cqr, interruptible: 1);
2538	}
2539	EXPORT_SYMBOL(dasd_sleep_on_interruptible);
2540
2541	/*
2542	* Whoa nelly now it gets really hairy. For some functions (e.g. steal lock
2543	* for eckd devices) the currently running request has to be terminated
2544	* and be put back to status queued, before the special request is added
2545	* to the head of the queue. Then the special request is waited on normally.
2546	*/
2547	static inline int _dasd_term_running_cqr(struct dasd_device *device)
2548	{
2549	struct dasd_ccw_req *cqr;
2550	int rc;
2551
2552	if (list_empty(head: &device->ccw_queue))
2553	return 0;
2554	cqr = list_entry(device->ccw_queue.next, struct dasd_ccw_req, devlist);
2555	rc = device->discipline->term_IO(cqr);
2556	if (!rc)
2557	/*
2558	* CQR terminated because a more important request is pending.
2559	* Undo decreasing of retry counter because this is
2560	* not an error case.
2561	*/
2562	cqr->retries++;
2563	return rc;
2564	}
2565
2566	int dasd_sleep_on_immediatly(struct dasd_ccw_req *cqr)
2567	{
2568	struct dasd_device *device;
2569	int rc;
2570
2571	device = cqr->startdev;
2572	if (test_bit(DASD_FLAG_LOCK_STOLEN, &device->flags) &&
2573	!test_bit(DASD_CQR_ALLOW_SLOCK, &cqr->flags)) {
2574	cqr->status = DASD_CQR_FAILED;
2575	cqr->intrc = -EPERM;
2576	return -EIO;
2577	}
2578	spin_lock_irq(lock: get_ccwdev_lock(device->cdev));
2579	rc = _dasd_term_running_cqr(device);
2580	if (rc) {
2581	spin_unlock_irq(lock: get_ccwdev_lock(device->cdev));
2582	return rc;
2583	}
2584	cqr->callback = dasd_wakeup_cb;
2585	cqr->callback_data = DASD_SLEEPON_START_TAG;
2586	cqr->status = DASD_CQR_QUEUED;
2587	/*
2588	* add new request as second
2589	* first the terminated cqr needs to be finished
2590	*/
2591	list_add(new: &cqr->devlist, head: device->ccw_queue.next);
2592
2593	/* let the bh start the request to keep them in order */
2594	dasd_schedule_device_bh(device);
2595
2596	spin_unlock_irq(lock: get_ccwdev_lock(device->cdev));
2597
2598	wait_event(generic_waitq, _wait_for_wakeup(cqr));
2599
2600	if (cqr->status == DASD_CQR_DONE)
2601	rc = 0;
2602	else if (cqr->intrc)
2603	rc = cqr->intrc;
2604	else
2605	rc = -EIO;
2606
2607	/* kick tasklets */
2608	dasd_schedule_device_bh(device);
2609	if (device->block)
2610	dasd_schedule_block_bh(device->block);
2611
2612	return rc;
2613	}
2614	EXPORT_SYMBOL(dasd_sleep_on_immediatly);
2615
2616	/*
2617	* Cancels a request that was started with dasd_sleep_on_req.
2618	* This is useful to timeout requests. The request will be
2619	* terminated if it is currently in i/o.
2620	* Returns 0 if request termination was successful
2621	* negative error code if termination failed
2622	* Cancellation of a request is an asynchronous operation! The calling
2623	* function has to wait until the request is properly returned via callback.
2624	*/
2625	static int __dasd_cancel_req(struct dasd_ccw_req *cqr)
2626	{
2627	struct dasd_device *device = cqr->startdev;
2628	int rc = 0;
2629
2630	switch (cqr->status) {
2631	case DASD_CQR_QUEUED:
2632	/* request was not started - just set to cleared */
2633	cqr->status = DASD_CQR_CLEARED;
2634	break;
2635	case DASD_CQR_IN_IO:
2636	/* request in IO - terminate IO and release again */
2637	rc = device->discipline->term_IO(cqr);
2638	if (rc) {
2639	dev_err(&device->cdev->dev,
2640	"Cancelling request %p failed with rc=%d\n",
2641	cqr, rc);
2642	} else {
2643	cqr->stopclk = get_tod_clock();
2644	}
2645	break;
2646	default: /* already finished or clear pending - do nothing */
2647	break;
2648	}
2649	dasd_schedule_device_bh(device);
2650	return rc;
2651	}
2652
2653	int dasd_cancel_req(struct dasd_ccw_req *cqr)
2654	{
2655	struct dasd_device *device = cqr->startdev;
2656	unsigned long flags;
2657	int rc;
2658
2659	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
2660	rc = __dasd_cancel_req(cqr);
2661	spin_unlock_irqrestore(lock: get_ccwdev_lock(device->cdev), flags);
2662	return rc;
2663	}
2664
2665	/*
2666	* SECTION: Operations of the dasd_block layer.
2667	*/
2668
2669	/*
2670	* Timeout function for dasd_block. This is used when the block layer
2671	* is waiting for something that may not come reliably, (e.g. a state
2672	* change interrupt)
2673	*/
2674	static void dasd_block_timeout(struct timer_list *t)
2675	{
2676	unsigned long flags;
2677	struct dasd_block *block;
2678
2679	block = from_timer(block, t, timer);
2680	spin_lock_irqsave(get_ccwdev_lock(block->base->cdev), flags);
2681	/* re-activate request queue */
2682	dasd_device_remove_stop_bits(block->base, DASD_STOPPED_PENDING);
2683	spin_unlock_irqrestore(lock: get_ccwdev_lock(block->base->cdev), flags);
2684	dasd_schedule_block_bh(block);
2685	blk_mq_run_hw_queues(q: block->gdp->queue, async: true);
2686	}
2687
2688	/*
2689	* Setup timeout for a dasd_block in jiffies.
2690	*/
2691	void dasd_block_set_timer(struct dasd_block *block, int expires)
2692	{
2693	if (expires == 0)
2694	del_timer(timer: &block->timer);
2695	else
2696	mod_timer(timer: &block->timer, expires: jiffies + expires);
2697	}
2698	EXPORT_SYMBOL(dasd_block_set_timer);
2699
2700	/*
2701	* Clear timeout for a dasd_block.
2702	*/
2703	void dasd_block_clear_timer(struct dasd_block *block)
2704	{
2705	del_timer(timer: &block->timer);
2706	}
2707	EXPORT_SYMBOL(dasd_block_clear_timer);
2708
2709	/*
2710	* Process finished error recovery ccw.
2711	*/
2712	static void __dasd_process_erp(struct dasd_device *device,
2713	struct dasd_ccw_req *cqr)
2714	{
2715	dasd_erp_fn_t erp_fn;
2716
2717	if (cqr->status == DASD_CQR_DONE)
2718	DBF_DEV_EVENT(DBF_NOTICE, device, "%s", "ERP successful");
2719	else
2720	dev_err(&device->cdev->dev, "ERP failed for the DASD\n");
2721	erp_fn = device->discipline->erp_postaction(cqr);
2722	erp_fn(cqr);
2723	}
2724
2725	static void __dasd_cleanup_cqr(struct dasd_ccw_req *cqr)
2726	{
2727	struct request *req;
2728	blk_status_t error = BLK_STS_OK;
2729	unsigned int proc_bytes;
2730	int status;
2731
2732	req = (struct request *) cqr->callback_data;
2733	dasd_profile_end(cqr->block, cqr, req);
2734
2735	proc_bytes = cqr->proc_bytes;
2736	status = cqr->block->base->discipline->free_cp(cqr, req);
2737	if (status < 0)
2738	error = errno_to_blk_status(errno: status);
2739	else if (status == 0) {
2740	switch (cqr->intrc) {
2741	case -EPERM:
2742	/*
2743	* DASD doesn't implement SCSI/NVMe reservations, but it
2744	* implements a locking scheme similar to them. We
2745	* return this error when we no longer have the lock.
2746	*/
2747	error = BLK_STS_RESV_CONFLICT;
2748	break;
2749	case -ENOLINK:
2750	error = BLK_STS_TRANSPORT;
2751	break;
2752	case -ETIMEDOUT:
2753	error = BLK_STS_TIMEOUT;
2754	break;
2755	default:
2756	error = BLK_STS_IOERR;
2757	break;
2758	}
2759	}
2760
2761	/*
2762	* We need to take care for ETIMEDOUT errors here since the
2763	* complete callback does not get called in this case.
2764	* Take care of all errors here and avoid additional code to
2765	* transfer the error value to the complete callback.
2766	*/
2767	if (error) {
2768	blk_mq_end_request(rq: req, error);
2769	blk_mq_run_hw_queues(q: req->q, async: true);
2770	} else {
2771	/*
2772	* Partial completed requests can happen with ESE devices.
2773	* During read we might have gotten a NRF error and have to
2774	* complete a request partially.
2775	*/
2776	if (proc_bytes) {
2777	blk_update_request(rq: req, BLK_STS_OK, nr_bytes: proc_bytes);
2778	blk_mq_requeue_request(rq: req, kick_requeue_list: true);
2779	} else if (likely(!blk_should_fake_timeout(req->q))) {
2780	blk_mq_complete_request(rq: req);
2781	}
2782	}
2783	}
2784
2785	/*
2786	* Process ccw request queue.
2787	*/
2788	static void __dasd_process_block_ccw_queue(struct dasd_block *block,
2789	struct list_head *final_queue)
2790	{
2791	struct list_head *l, *n;
2792	struct dasd_ccw_req *cqr;
2793	dasd_erp_fn_t erp_fn;
2794	unsigned long flags;
2795	struct dasd_device *base = block->base;
2796
2797	restart:
2798	/* Process request with final status. */
2799	list_for_each_safe(l, n, &block->ccw_queue) {
2800	cqr = list_entry(l, struct dasd_ccw_req, blocklist);
2801	if (cqr->status != DASD_CQR_DONE &&
2802	cqr->status != DASD_CQR_FAILED &&
2803	cqr->status != DASD_CQR_NEED_ERP &&
2804	cqr->status != DASD_CQR_TERMINATED)
2805	continue;
2806
2807	if (cqr->status == DASD_CQR_TERMINATED) {
2808	base->discipline->handle_terminated_request(cqr);
2809	goto restart;
2810	}
2811
2812	/* Process requests that may be recovered */
2813	if (cqr->status == DASD_CQR_NEED_ERP) {
2814	erp_fn = base->discipline->erp_action(cqr);
2815	if (IS_ERR(ptr: erp_fn(cqr)))
2816	continue;
2817	goto restart;
2818	}
2819
2820	/* log sense for fatal error */
2821	if (cqr->status == DASD_CQR_FAILED) {
2822	dasd_log_sense(cqr, &cqr->irb);
2823	}
2824
2825	/*
2826	* First call extended error reporting and check for autoquiesce
2827	*/
2828	spin_lock_irqsave(get_ccwdev_lock(base->cdev), flags);
2829	if (cqr->status == DASD_CQR_FAILED &&
2830	dasd_handle_autoquiesce(base, cqr, DASD_EER_FATALERROR)) {
2831	cqr->status = DASD_CQR_FILLED;
2832	cqr->retries = 255;
2833	spin_unlock_irqrestore(lock: get_ccwdev_lock(base->cdev), flags);
2834	goto restart;
2835	}
2836	spin_unlock_irqrestore(lock: get_ccwdev_lock(base->cdev), flags);
2837
2838	/* Process finished ERP request. */
2839	if (cqr->refers) {
2840	__dasd_process_erp(device: base, cqr);
2841	goto restart;
2842	}
2843
2844	/* Rechain finished requests to final queue */
2845	cqr->endclk = get_tod_clock();
2846	list_move_tail(list: &cqr->blocklist, head: final_queue);
2847	}
2848	}
2849
2850	static void dasd_return_cqr_cb(struct dasd_ccw_req *cqr, void *data)
2851	{
2852	dasd_schedule_block_bh(cqr->block);
2853	}
2854
2855	static void __dasd_block_start_head(struct dasd_block *block)
2856	{
2857	struct dasd_ccw_req *cqr;
2858
2859	if (list_empty(head: &block->ccw_queue))
2860	return;
2861	/* We allways begin with the first requests on the queue, as some
2862	* of previously started requests have to be enqueued on a
2863	* dasd_device again for error recovery.
2864	*/
2865	list_for_each_entry(cqr, &block->ccw_queue, blocklist) {
2866	if (cqr->status != DASD_CQR_FILLED)
2867	continue;
2868	if (test_bit(DASD_FLAG_LOCK_STOLEN, &block->base->flags) &&
2869	!test_bit(DASD_CQR_ALLOW_SLOCK, &cqr->flags)) {
2870	cqr->status = DASD_CQR_FAILED;
2871	cqr->intrc = -EPERM;
2872	dasd_schedule_block_bh(block);
2873	continue;
2874	}
2875	/* Non-temporary stop condition will trigger fail fast */
2876	if (block->base->stopped & ~DASD_STOPPED_PENDING &&
2877	test_bit(DASD_CQR_FLAGS_FAILFAST, &cqr->flags) &&
2878	!dasd_eer_enabled(block->base) && block->base->aq_mask == 0) {
2879	cqr->status = DASD_CQR_FAILED;
2880	cqr->intrc = -ENOLINK;
2881	dasd_schedule_block_bh(block);
2882	continue;
2883	}
2884	/* Don't try to start requests if device is stopped */
2885	if (block->base->stopped)
2886	return;
2887
2888	/* just a fail safe check, should not happen */
2889	if (!cqr->startdev)
2890	cqr->startdev = block->base;
2891
2892	/* make sure that the requests we submit find their way back */
2893	cqr->callback = dasd_return_cqr_cb;
2894
2895	dasd_add_request_tail(cqr);
2896	}
2897	}
2898
2899	/*
2900	* Central dasd_block layer routine. Takes requests from the generic
2901	* block layer request queue, creates ccw requests, enqueues them on
2902	* a dasd_device and processes ccw requests that have been returned.
2903	*/
2904	static void dasd_block_tasklet(unsigned long data)
2905	{
2906	struct dasd_block *block = (struct dasd_block *) data;
2907	struct list_head final_queue;
2908	struct list_head *l, *n;
2909	struct dasd_ccw_req *cqr;
2910	struct dasd_queue *dq;
2911
2912	atomic_set(v: &block->tasklet_scheduled, i: 0);
2913	INIT_LIST_HEAD(list: &final_queue);
2914	spin_lock_irq(lock: &block->queue_lock);
2915	/* Finish off requests on ccw queue */
2916	__dasd_process_block_ccw_queue(block, final_queue: &final_queue);
2917	spin_unlock_irq(lock: &block->queue_lock);
2918
2919	/* Now call the callback function of requests with final status */
2920	list_for_each_safe(l, n, &final_queue) {
2921	cqr = list_entry(l, struct dasd_ccw_req, blocklist);
2922	dq = cqr->dq;
2923	spin_lock_irq(lock: &dq->lock);
2924	list_del_init(entry: &cqr->blocklist);
2925	__dasd_cleanup_cqr(cqr);
2926	spin_unlock_irq(lock: &dq->lock);
2927	}
2928
2929	spin_lock_irq(lock: &block->queue_lock);
2930	/* Now check if the head of the ccw queue needs to be started. */
2931	__dasd_block_start_head(block);
2932	spin_unlock_irq(lock: &block->queue_lock);
2933
2934	if (waitqueue_active(wq_head: &shutdown_waitq))
2935	wake_up(&shutdown_waitq);
2936	dasd_put_device(device: block->base);
2937	}
2938
2939	static void _dasd_wake_block_flush_cb(struct dasd_ccw_req *cqr, void *data)
2940	{
2941	wake_up(&dasd_flush_wq);
2942	}
2943
2944	/*
2945	* Requeue a request back to the block request queue
2946	* only works for block requests
2947	*/
2948	static void _dasd_requeue_request(struct dasd_ccw_req *cqr)
2949	{
2950	struct request *req;
2951
2952	/*
2953	* If the request is an ERP request there is nothing to requeue.
2954	* This will be done with the remaining original request.
2955	*/
2956	if (cqr->refers)
2957	return;
2958	spin_lock_irq(lock: &cqr->dq->lock);
2959	req = (struct request *) cqr->callback_data;
2960	blk_mq_requeue_request(rq: req, kick_requeue_list: true);
2961	spin_unlock_irq(lock: &cqr->dq->lock);
2962
2963	return;
2964	}
2965
2966	static int _dasd_requests_to_flushqueue(struct dasd_block *block,
2967	struct list_head *flush_queue)
2968	{
2969	struct dasd_ccw_req *cqr, *n;
2970	unsigned long flags;
2971	int rc, i;
2972
2973	spin_lock_irqsave(&block->queue_lock, flags);
2974	rc = 0;
2975	restart:
2976	list_for_each_entry_safe(cqr, n, &block->ccw_queue, blocklist) {
2977	/* if this request currently owned by a dasd_device cancel it */
2978	if (cqr->status >= DASD_CQR_QUEUED)
2979	rc = dasd_cancel_req(cqr);
2980	if (rc < 0)
2981	break;
2982	/* Rechain request (including erp chain) so it won't be
2983	* touched by the dasd_block_tasklet anymore.
2984	* Replace the callback so we notice when the request
2985	* is returned from the dasd_device layer.
2986	*/
2987	cqr->callback = _dasd_wake_block_flush_cb;
2988	for (i = 0; cqr; cqr = cqr->refers, i++)
2989	list_move_tail(list: &cqr->blocklist, head: flush_queue);
2990	if (i > 1)
2991	/* moved more than one request - need to restart */
2992	goto restart;
2993	}
2994	spin_unlock_irqrestore(lock: &block->queue_lock, flags);
2995
2996	return rc;
2997	}
2998
2999	/*
3000	* Go through all request on the dasd_block request queue, cancel them
3001	* on the respective dasd_device, and return them to the generic
3002	* block layer.
3003	*/
3004	static int dasd_flush_block_queue(struct dasd_block *block)
3005	{
3006	struct dasd_ccw_req *cqr, *n;
3007	struct list_head flush_queue;
3008	unsigned long flags;
3009	int rc;
3010
3011	INIT_LIST_HEAD(list: &flush_queue);
3012	rc = _dasd_requests_to_flushqueue(block, flush_queue: &flush_queue);
3013
3014	/* Now call the callback function of flushed requests */
3015	restart_cb:
3016	list_for_each_entry_safe(cqr, n, &flush_queue, blocklist) {
3017	wait_event(dasd_flush_wq, (cqr->status < DASD_CQR_QUEUED));
3018	/* Process finished ERP request. */
3019	if (cqr->refers) {
3020	spin_lock_bh(lock: &block->queue_lock);
3021	__dasd_process_erp(device: block->base, cqr);
3022	spin_unlock_bh(lock: &block->queue_lock);
3023	/* restart list_for_xx loop since dasd_process_erp
3024	* might remove multiple elements */
3025	goto restart_cb;
3026	}
3027	/* call the callback function */
3028	spin_lock_irqsave(&cqr->dq->lock, flags);
3029	cqr->endclk = get_tod_clock();
3030	list_del_init(entry: &cqr->blocklist);
3031	__dasd_cleanup_cqr(cqr);
3032	spin_unlock_irqrestore(lock: &cqr->dq->lock, flags);
3033	}
3034	return rc;
3035	}
3036
3037	/*
3038	* Schedules a call to dasd_tasklet over the device tasklet.
3039	*/
3040	void dasd_schedule_block_bh(struct dasd_block *block)
3041	{
3042	/* Protect against rescheduling. */
3043	if (atomic_cmpxchg(v: &block->tasklet_scheduled, old: 0, new: 1) != 0)
3044	return;
3045	/* life cycle of block is bound to it's base device */
3046	dasd_get_device(device: block->base);
3047	tasklet_hi_schedule(t: &block->tasklet);
3048	}
3049	EXPORT_SYMBOL(dasd_schedule_block_bh);
3050
3051
3052	/*
3053	* SECTION: external block device operations
3054	* (request queue handling, open, release, etc.)
3055	*/
3056
3057	/*
3058	* Dasd request queue function. Called from ll_rw_blk.c
3059	*/
3060	static blk_status_t do_dasd_request(struct blk_mq_hw_ctx *hctx,
3061	const struct blk_mq_queue_data *qd)
3062	{
3063	struct dasd_block *block = hctx->queue->queuedata;
3064	struct dasd_queue *dq = hctx->driver_data;
3065	struct request *req = qd->rq;
3066	struct dasd_device *basedev;
3067	struct dasd_ccw_req *cqr;
3068	blk_status_t rc = BLK_STS_OK;
3069
3070	basedev = block->base;
3071	spin_lock_irq(lock: &dq->lock);
3072	if (basedev->state < DASD_STATE_READY ||
3073	test_bit(DASD_FLAG_OFFLINE, &basedev->flags)) {
3074	DBF_DEV_EVENT(DBF_ERR, basedev,
3075	"device not ready for request %p", req);
3076	rc = BLK_STS_IOERR;
3077	goto out;
3078	}
3079
3080	/*
3081	* if device is stopped do not fetch new requests
3082	* except failfast is active which will let requests fail
3083	* immediately in __dasd_block_start_head()
3084	*/
3085	if (basedev->stopped && !(basedev->features & DASD_FEATURE_FAILFAST)) {
3086	DBF_DEV_EVENT(DBF_ERR, basedev,
3087	"device stopped request %p", req);
3088	rc = BLK_STS_RESOURCE;
3089	goto out;
3090	}
3091
3092	if (basedev->features & DASD_FEATURE_READONLY &&
3093	rq_data_dir(req) == WRITE) {
3094	DBF_DEV_EVENT(DBF_ERR, basedev,
3095	"Rejecting write request %p", req);
3096	rc = BLK_STS_IOERR;
3097	goto out;
3098	}
3099
3100	if (test_bit(DASD_FLAG_ABORTALL, &basedev->flags) &&
3101	(basedev->features & DASD_FEATURE_FAILFAST ||
3102	blk_noretry_request(req))) {
3103	DBF_DEV_EVENT(DBF_ERR, basedev,
3104	"Rejecting failfast request %p", req);
3105	rc = BLK_STS_IOERR;
3106	goto out;
3107	}
3108
3109	cqr = basedev->discipline->build_cp(basedev, block, req);
3110	if (IS_ERR(ptr: cqr)) {
3111	if (PTR_ERR(ptr: cqr) == -EBUSY ||
3112	PTR_ERR(ptr: cqr) == -ENOMEM ||
3113	PTR_ERR(ptr: cqr) == -EAGAIN) {
3114	rc = BLK_STS_RESOURCE;
3115	goto out;
3116	}
3117	DBF_DEV_EVENT(DBF_ERR, basedev,
3118	"CCW creation failed (rc=%ld) on request %p",
3119	PTR_ERR(cqr), req);
3120	rc = BLK_STS_IOERR;
3121	goto out;
3122	}
3123	/*
3124	* Note: callback is set to dasd_return_cqr_cb in
3125	* __dasd_block_start_head to cover erp requests as well
3126	*/
3127	cqr->callback_data = req;
3128	cqr->status = DASD_CQR_FILLED;
3129	cqr->dq = dq;
3130
3131	blk_mq_start_request(rq: req);
3132	spin_lock(lock: &block->queue_lock);
3133	list_add_tail(new: &cqr->blocklist, head: &block->ccw_queue);
3134	INIT_LIST_HEAD(list: &cqr->devlist);
3135	dasd_profile_start(block, cqr, req);
3136	dasd_schedule_block_bh(block);
3137	spin_unlock(lock: &block->queue_lock);
3138
3139	out:
3140	spin_unlock_irq(lock: &dq->lock);
3141	return rc;
3142	}
3143
3144	/*
3145	* Block timeout callback, called from the block layer
3146	*
3147	* Return values:
3148	* BLK_EH_RESET_TIMER if the request should be left running
3149	* BLK_EH_DONE if the request is handled or terminated
3150	* by the driver.
3151	*/
3152	enum blk_eh_timer_return dasd_times_out(struct request *req)
3153	{
3154	struct dasd_block *block = req->q->queuedata;
3155	struct dasd_device *device;
3156	struct dasd_ccw_req *cqr;
3157	unsigned long flags;
3158	int rc = 0;
3159
3160	cqr = blk_mq_rq_to_pdu(rq: req);
3161	if (!cqr)
3162	return BLK_EH_DONE;
3163
3164	spin_lock_irqsave(&cqr->dq->lock, flags);
3165	device = cqr->startdev ? cqr->startdev : block->base;
3166	if (!device->blk_timeout) {
3167	spin_unlock_irqrestore(lock: &cqr->dq->lock, flags);
3168	return BLK_EH_RESET_TIMER;
3169	}
3170	DBF_DEV_EVENT(DBF_WARNING, device,
3171	" dasd_times_out cqr %p status %x",
3172	cqr, cqr->status);
3173
3174	spin_lock(lock: &block->queue_lock);
3175	spin_lock(lock: get_ccwdev_lock(device->cdev));
3176	cqr->retries = -1;
3177	cqr->intrc = -ETIMEDOUT;
3178	if (cqr->status >= DASD_CQR_QUEUED) {
3179	rc = __dasd_cancel_req(cqr);
3180	} else if (cqr->status == DASD_CQR_FILLED ||
3181	cqr->status == DASD_CQR_NEED_ERP) {
3182	cqr->status = DASD_CQR_TERMINATED;
3183	} else if (cqr->status == DASD_CQR_IN_ERP) {
3184	struct dasd_ccw_req *searchcqr, *nextcqr, *tmpcqr;
3185
3186	list_for_each_entry_safe(searchcqr, nextcqr,
3187	&block->ccw_queue, blocklist) {
3188	tmpcqr = searchcqr;
3189	while (tmpcqr->refers)
3190	tmpcqr = tmpcqr->refers;
3191	if (tmpcqr != cqr)
3192	continue;
3193	/* searchcqr is an ERP request for cqr */
3194	searchcqr->retries = -1;
3195	searchcqr->intrc = -ETIMEDOUT;
3196	if (searchcqr->status >= DASD_CQR_QUEUED) {
3197	rc = __dasd_cancel_req(cqr: searchcqr);
3198	} else if ((searchcqr->status == DASD_CQR_FILLED) ||
3199	(searchcqr->status == DASD_CQR_NEED_ERP)) {
3200	searchcqr->status = DASD_CQR_TERMINATED;
3201	rc = 0;
3202	} else if (searchcqr->status == DASD_CQR_IN_ERP) {
3203	/*
3204	* Shouldn't happen; most recent ERP
3205	* request is at the front of queue
3206	*/
3207	continue;
3208	}
3209	break;
3210	}
3211	}
3212	spin_unlock(lock: get_ccwdev_lock(device->cdev));
3213	dasd_schedule_block_bh(block);
3214	spin_unlock(lock: &block->queue_lock);
3215	spin_unlock_irqrestore(lock: &cqr->dq->lock, flags);
3216
3217	return rc ? BLK_EH_RESET_TIMER : BLK_EH_DONE;
3218	}
3219
3220	static int dasd_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
3221	unsigned int idx)
3222	{
3223	struct dasd_queue *dq = kzalloc(size: sizeof(*dq), GFP_KERNEL);
3224
3225	if (!dq)
3226	return -ENOMEM;
3227
3228	spin_lock_init(&dq->lock);
3229	hctx->driver_data = dq;
3230
3231	return 0;
3232	}
3233
3234	static void dasd_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int idx)
3235	{
3236	kfree(objp: hctx->driver_data);
3237	hctx->driver_data = NULL;
3238	}
3239
3240	static void dasd_request_done(struct request *req)
3241	{
3242	blk_mq_end_request(rq: req, error: 0);
3243	blk_mq_run_hw_queues(q: req->q, async: true);
3244	}
3245
3246	struct blk_mq_ops dasd_mq_ops = {
3247	.queue_rq = do_dasd_request,
3248	.complete = dasd_request_done,
3249	.timeout = dasd_times_out,
3250	.init_hctx = dasd_init_hctx,
3251	.exit_hctx = dasd_exit_hctx,
3252	};
3253
3254	static int dasd_open(struct gendisk *disk, blk_mode_t mode)
3255	{
3256	struct dasd_device *base;
3257	int rc;
3258
3259	base = dasd_device_from_gendisk(disk);
3260	if (!base)
3261	return -ENODEV;
3262
3263	atomic_inc(v: &base->block->open_count);
3264	if (test_bit(DASD_FLAG_OFFLINE, &base->flags)) {
3265	rc = -ENODEV;
3266	goto unlock;
3267	}
3268
3269	if (!try_module_get(module: base->discipline->owner)) {
3270	rc = -EINVAL;
3271	goto unlock;
3272	}
3273
3274	if (dasd_probeonly) {
3275	dev_info(&base->cdev->dev,
3276	"Accessing the DASD failed because it is in "
3277	"probeonly mode\n");
3278	rc = -EPERM;
3279	goto out;
3280	}
3281
3282	if (base->state <= DASD_STATE_BASIC) {
3283	DBF_DEV_EVENT(DBF_ERR, base, " %s",
3284	" Cannot open unrecognized device");
3285	rc = -ENODEV;
3286	goto out;
3287	}
3288	if ((mode & BLK_OPEN_WRITE) &&
3289	(test_bit(DASD_FLAG_DEVICE_RO, &base->flags) ||
3290	(base->features & DASD_FEATURE_READONLY))) {
3291	rc = -EROFS;
3292	goto out;
3293	}
3294	dasd_put_device(device: base);
3295	return 0;
3296
3297	out:
3298	module_put(module: base->discipline->owner);
3299	unlock:
3300	atomic_dec(v: &base->block->open_count);
3301	dasd_put_device(device: base);
3302	return rc;
3303	}
3304
3305	static void dasd_release(struct gendisk *disk)
3306	{
3307	struct dasd_device *base = dasd_device_from_gendisk(disk);
3308	if (base) {
3309	atomic_dec(v: &base->block->open_count);
3310	module_put(module: base->discipline->owner);
3311	dasd_put_device(device: base);
3312	}
3313	}
3314
3315	/*
3316	* Return disk geometry.
3317	*/
3318	static int dasd_getgeo(struct block_device *bdev, struct hd_geometry *geo)
3319	{
3320	struct dasd_device *base;
3321
3322	base = dasd_device_from_gendisk(bdev->bd_disk);
3323	if (!base)
3324	return -ENODEV;
3325
3326	if (!base->discipline ||
3327	!base->discipline->fill_geometry) {
3328	dasd_put_device(device: base);
3329	return -EINVAL;
3330	}
3331	base->discipline->fill_geometry(base->block, geo);
3332	geo->start = get_start_sect(bdev) >> base->block->s2b_shift;
3333	dasd_put_device(device: base);
3334	return 0;
3335	}
3336
3337	const struct block_device_operations
3338	dasd_device_operations = {
3339	.owner = THIS_MODULE,
3340	.open = dasd_open,
3341	.release = dasd_release,
3342	.ioctl = dasd_ioctl,
3343	.compat_ioctl = dasd_ioctl,
3344	.getgeo = dasd_getgeo,
3345	.set_read_only = dasd_set_read_only,
3346	};
3347
3348	/*******************************************************************************
3349	* end of block device operations
3350	*/
3351
3352	static void
3353	dasd_exit(void)
3354	{
3355	#ifdef CONFIG_PROC_FS
3356	dasd_proc_exit();
3357	#endif
3358	dasd_eer_exit();
3359	kmem_cache_destroy(s: dasd_page_cache);
3360	dasd_page_cache = NULL;
3361	dasd_gendisk_exit();
3362	dasd_devmap_exit();
3363	if (dasd_debug_area != NULL) {
3364	debug_unregister(dasd_debug_area);
3365	dasd_debug_area = NULL;
3366	}
3367	dasd_statistics_removeroot();
3368	}
3369
3370	/*
3371	* SECTION: common functions for ccw_driver use
3372	*/
3373
3374	/*
3375	* Is the device read-only?
3376	* Note that this function does not report the setting of the
3377	* readonly device attribute, but how it is configured in z/VM.
3378	*/
3379	int dasd_device_is_ro(struct dasd_device *device)
3380	{
3381	struct ccw_dev_id dev_id;
3382	struct diag210 diag_data;
3383	int rc;
3384
3385	if (!MACHINE_IS_VM)
3386	return 0;
3387	ccw_device_get_id(device->cdev, &dev_id);
3388	memset(&diag_data, 0, sizeof(diag_data));
3389	diag_data.vrdcdvno = dev_id.devno;
3390	diag_data.vrdclen = sizeof(diag_data);
3391	rc = diag210(&diag_data);
3392	if (rc == 0 || rc == 2) {
3393	return diag_data.vrdcvfla & 0x80;
3394	} else {
3395	DBF_EVENT(DBF_WARNING, "diag210 failed for dev=%04x with rc=%d",
3396	dev_id.devno, rc);
3397	return 0;
3398	}
3399	}
3400	EXPORT_SYMBOL_GPL(dasd_device_is_ro);
3401
3402	static void dasd_generic_auto_online(void *data, async_cookie_t cookie)
3403	{
3404	struct ccw_device *cdev = data;
3405	int ret;
3406
3407	ret = ccw_device_set_online(cdev);
3408	if (ret)
3409	pr_warn("%s: Setting the DASD online failed with rc=%d\n",
3410	dev_name(&cdev->dev), ret);
3411	}
3412
3413	/*
3414	* Initial attempt at a probe function. this can be simplified once
3415	* the other detection code is gone.
3416	*/
3417	int dasd_generic_probe(struct ccw_device *cdev)
3418	{
3419	cdev->handler = &dasd_int_handler;
3420
3421	/*
3422	* Automatically online either all dasd devices (dasd_autodetect)
3423	* or all devices specified with dasd= parameters during
3424	* initial probe.
3425	*/
3426	if ((dasd_get_feature(cdev, DASD_FEATURE_INITIAL_ONLINE) > 0 ) ||
3427	(dasd_autodetect && dasd_busid_known(dev_name(&cdev->dev)) != 0))
3428	async_schedule(func: dasd_generic_auto_online, data: cdev);
3429	return 0;
3430	}
3431	EXPORT_SYMBOL_GPL(dasd_generic_probe);
3432
3433	void dasd_generic_free_discipline(struct dasd_device *device)
3434	{
3435	/* Forget the discipline information. */
3436	if (device->discipline) {
3437	if (device->discipline->uncheck_device)
3438	device->discipline->uncheck_device(device);
3439	module_put(module: device->discipline->owner);
3440	device->discipline = NULL;
3441	}
3442	if (device->base_discipline) {
3443	module_put(module: device->base_discipline->owner);
3444	device->base_discipline = NULL;
3445	}
3446	}
3447	EXPORT_SYMBOL_GPL(dasd_generic_free_discipline);
3448
3449	/*
3450	* This will one day be called from a global not_oper handler.
3451	* It is also used by driver_unregister during module unload.
3452	*/
3453	void dasd_generic_remove(struct ccw_device *cdev)
3454	{
3455	struct dasd_device *device;
3456	struct dasd_block *block;
3457
3458	device = dasd_device_from_cdev(cdev);
3459	if (IS_ERR(ptr: device))
3460	return;
3461
3462	if (test_and_set_bit(DASD_FLAG_OFFLINE, addr: &device->flags) &&
3463	!test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) {
3464	/* Already doing offline processing */
3465	dasd_put_device(device);
3466	return;
3467	}
3468	/*
3469	* This device is removed unconditionally. Set offline
3470	* flag to prevent dasd_open from opening it while it is
3471	* no quite down yet.
3472	*/
3473	dasd_set_target_state(device, DASD_STATE_NEW);
3474	cdev->handler = NULL;
3475	/* dasd_delete_device destroys the device reference. */
3476	block = device->block;
3477	dasd_delete_device(device);
3478	/*
3479	* life cycle of block is bound to device, so delete it after
3480	* device was safely removed
3481	*/
3482	if (block)
3483	dasd_free_block(block);
3484	}
3485	EXPORT_SYMBOL_GPL(dasd_generic_remove);
3486
3487	/*
3488	* Activate a device. This is called from dasd_{eckd,fba}_probe() when either
3489	* the device is detected for the first time and is supposed to be used
3490	* or the user has started activation through sysfs.
3491	*/
3492	int dasd_generic_set_online(struct ccw_device *cdev,
3493	struct dasd_discipline *base_discipline)
3494	{
3495	struct dasd_discipline *discipline;
3496	struct dasd_device *device;
3497	int rc;
3498
3499	/* first online clears initial online feature flag */
3500	dasd_set_feature(cdev, DASD_FEATURE_INITIAL_ONLINE, 0);
3501	device = dasd_create_device(cdev);
3502	if (IS_ERR(ptr: device))
3503	return PTR_ERR(ptr: device);
3504
3505	discipline = base_discipline;
3506	if (device->features & DASD_FEATURE_USEDIAG) {
3507	if (!dasd_diag_discipline_pointer) {
3508	/* Try to load the required module. */
3509	rc = request_module(DASD_DIAG_MOD);
3510	if (rc) {
3511	pr_warn("%s Setting the DASD online failed "
3512	"because the required module %s "
3513	"could not be loaded (rc=%d)\n",
3514	dev_name(&cdev->dev), DASD_DIAG_MOD,
3515	rc);
3516	dasd_delete_device(device);
3517	return -ENODEV;
3518	}
3519	}
3520	/* Module init could have failed, so check again here after
3521	* request_module(). */
3522	if (!dasd_diag_discipline_pointer) {
3523	pr_warn("%s Setting the DASD online failed because of missing DIAG discipline\n",
3524	dev_name(&cdev->dev));
3525	dasd_delete_device(device);
3526	return -ENODEV;
3527	}
3528	discipline = dasd_diag_discipline_pointer;
3529	}
3530	if (!try_module_get(module: base_discipline->owner)) {
3531	dasd_delete_device(device);
3532	return -EINVAL;
3533	}
3534	if (!try_module_get(module: discipline->owner)) {
3535	module_put(module: base_discipline->owner);
3536	dasd_delete_device(device);
3537	return -EINVAL;
3538	}
3539	device->base_discipline = base_discipline;
3540	device->discipline = discipline;
3541
3542	/* check_device will allocate block device if necessary */
3543	rc = discipline->check_device(device);
3544	if (rc) {
3545	pr_warn("%s Setting the DASD online with discipline %s failed with rc=%i\n",
3546	dev_name(&cdev->dev), discipline->name, rc);
3547	module_put(module: discipline->owner);
3548	module_put(module: base_discipline->owner);
3549	dasd_delete_device(device);
3550	return rc;
3551	}
3552
3553	dasd_set_target_state(device, DASD_STATE_ONLINE);
3554	if (device->state <= DASD_STATE_KNOWN) {
3555	pr_warn("%s Setting the DASD online failed because of a missing discipline\n",
3556	dev_name(&cdev->dev));
3557	rc = -ENODEV;
3558	dasd_set_target_state(device, DASD_STATE_NEW);
3559	if (device->block)
3560	dasd_free_block(device->block);
3561	dasd_delete_device(device);
3562	} else
3563	pr_debug("dasd_generic device %s found\n",
3564	dev_name(&cdev->dev));
3565
3566	wait_event(dasd_init_waitq, _wait_for_device(device));
3567
3568	dasd_put_device(device);
3569	return rc;
3570	}
3571	EXPORT_SYMBOL_GPL(dasd_generic_set_online);
3572
3573	int dasd_generic_set_offline(struct ccw_device *cdev)
3574	{
3575	struct dasd_device *device;
3576	struct dasd_block *block;
3577	int max_count, open_count, rc;
3578	unsigned long flags;
3579
3580	rc = 0;
3581	spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
3582	device = dasd_device_from_cdev_locked(cdev);
3583	if (IS_ERR(ptr: device)) {
3584	spin_unlock_irqrestore(lock: get_ccwdev_lock(cdev), flags);
3585	return PTR_ERR(ptr: device);
3586	}
3587
3588	/*
3589	* We must make sure that this device is currently not in use.
3590	* The open_count is increased for every opener, that includes
3591	* the blkdev_get in dasd_scan_partitions. We are only interested
3592	* in the other openers.
3593	*/
3594	if (device->block) {
3595	max_count = device->block->bdev_handle ? 0 : -1;
3596	open_count = atomic_read(v: &device->block->open_count);
3597	if (open_count > max_count) {
3598	if (open_count > 0)
3599	pr_warn("%s: The DASD cannot be set offline with open count %i\n",
3600	dev_name(&cdev->dev), open_count);
3601	else
3602	pr_warn("%s: The DASD cannot be set offline while it is in use\n",
3603	dev_name(&cdev->dev));
3604	rc = -EBUSY;
3605	goto out_err;
3606	}
3607	}
3608
3609	/*
3610	* Test if the offline processing is already running and exit if so.
3611	* If a safe offline is being processed this could only be a normal
3612	* offline that should be able to overtake the safe offline and
3613	* cancel any I/O we do not want to wait for any longer
3614	*/
3615	if (test_bit(DASD_FLAG_OFFLINE, &device->flags)) {
3616	if (test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) {
3617	clear_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING,
3618	addr: &device->flags);
3619	} else {
3620	rc = -EBUSY;
3621	goto out_err;
3622	}
3623	}
3624	set_bit(DASD_FLAG_OFFLINE, addr: &device->flags);
3625
3626	/*
3627	* if safe_offline is called set safe_offline_running flag and
3628	* clear safe_offline so that a call to normal offline
3629	* can overrun safe_offline processing
3630	*/
3631	if (test_and_clear_bit(DASD_FLAG_SAFE_OFFLINE, addr: &device->flags) &&
3632	!test_and_set_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, addr: &device->flags)) {
3633	/* need to unlock here to wait for outstanding I/O */
3634	spin_unlock_irqrestore(lock: get_ccwdev_lock(cdev), flags);
3635	/*
3636	* If we want to set the device safe offline all IO operations
3637	* should be finished before continuing the offline process
3638	* so sync bdev first and then wait for our queues to become
3639	* empty
3640	*/
3641	if (device->block && device->block->bdev_handle)
3642	bdev_mark_dead(bdev: device->block->bdev_handle->bdev, surprise: false);
3643	dasd_schedule_device_bh(device);
3644	rc = wait_event_interruptible(shutdown_waitq,
3645	_wait_for_empty_queues(device));
3646	if (rc != 0)
3647	goto interrupted;
3648
3649	/*
3650	* check if a normal offline process overtook the offline
3651	* processing in this case simply do nothing beside returning
3652	* that we got interrupted
3653	* otherwise mark safe offline as not running any longer and
3654	* continue with normal offline
3655	*/
3656	spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
3657	if (!test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) {
3658	rc = -ERESTARTSYS;
3659	goto out_err;
3660	}
3661	clear_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, addr: &device->flags);
3662	}
3663	spin_unlock_irqrestore(lock: get_ccwdev_lock(cdev), flags);
3664
3665	dasd_set_target_state(device, DASD_STATE_NEW);
3666	/* dasd_delete_device destroys the device reference. */
3667	block = device->block;
3668	dasd_delete_device(device);
3669	/*
3670	* life cycle of block is bound to device, so delete it after
3671	* device was safely removed
3672	*/
3673	if (block)
3674	dasd_free_block(block);
3675
3676	return 0;
3677
3678	interrupted:
3679	/* interrupted by signal */
3680	spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
3681	clear_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, addr: &device->flags);
3682	clear_bit(DASD_FLAG_OFFLINE, addr: &device->flags);
3683	out_err:
3684	dasd_put_device(device);
3685	spin_unlock_irqrestore(lock: get_ccwdev_lock(cdev), flags);
3686	return rc;
3687	}
3688	EXPORT_SYMBOL_GPL(dasd_generic_set_offline);
3689
3690	int dasd_generic_last_path_gone(struct dasd_device *device)
3691	{
3692	struct dasd_ccw_req *cqr;
3693
3694	dev_warn(&device->cdev->dev, "No operational channel path is left "
3695	"for the device\n");
3696	DBF_DEV_EVENT(DBF_WARNING, device, "%s", "last path gone");
3697	/* First call extended error reporting and check for autoquiesce. */
3698	dasd_handle_autoquiesce(device, NULL, DASD_EER_NOPATH);
3699
3700	if (device->state < DASD_STATE_BASIC)
3701	return 0;
3702	/* Device is active. We want to keep it. */
3703	list_for_each_entry(cqr, &device->ccw_queue, devlist)
3704	if ((cqr->status == DASD_CQR_IN_IO) ||
3705	(cqr->status == DASD_CQR_CLEAR_PENDING)) {
3706	cqr->status = DASD_CQR_QUEUED;
3707	cqr->retries++;
3708	}
3709	dasd_device_set_stop_bits(device, DASD_STOPPED_DC_WAIT);
3710	dasd_device_clear_timer(device);
3711	dasd_schedule_device_bh(device);
3712	return 1;
3713	}
3714	EXPORT_SYMBOL_GPL(dasd_generic_last_path_gone);
3715
3716	int dasd_generic_path_operational(struct dasd_device *device)
3717	{
3718	dev_info(&device->cdev->dev, "A channel path to the device has become "
3719	"operational\n");
3720	DBF_DEV_EVENT(DBF_WARNING, device, "%s", "path operational");
3721	dasd_device_remove_stop_bits(device, DASD_STOPPED_DC_WAIT);
3722	dasd_schedule_device_bh(device);
3723	if (device->block) {
3724	dasd_schedule_block_bh(device->block);
3725	if (device->block->gdp)
3726	blk_mq_run_hw_queues(q: device->block->gdp->queue, async: true);
3727	}
3728
3729	if (!device->stopped)
3730	wake_up(&generic_waitq);
3731
3732	return 1;
3733	}
3734	EXPORT_SYMBOL_GPL(dasd_generic_path_operational);
3735
3736	int dasd_generic_notify(struct ccw_device *cdev, int event)
3737	{
3738	struct dasd_device *device;
3739	int ret;
3740
3741	device = dasd_device_from_cdev_locked(cdev);
3742	if (IS_ERR(ptr: device))
3743	return 0;
3744	ret = 0;
3745	switch (event) {
3746	case CIO_GONE:
3747	case CIO_BOXED:
3748	case CIO_NO_PATH:
3749	dasd_path_no_path(device);
3750	ret = dasd_generic_last_path_gone(device);
3751	break;
3752	case CIO_OPER:
3753	ret = 1;
3754	if (dasd_path_get_opm(device))
3755	ret = dasd_generic_path_operational(device);
3756	break;
3757	}
3758	dasd_put_device(device);
3759	return ret;
3760	}
3761	EXPORT_SYMBOL_GPL(dasd_generic_notify);
3762
3763	void dasd_generic_path_event(struct ccw_device *cdev, int *path_event)
3764	{
3765	struct dasd_device *device;
3766	int chp, oldopm, hpfpm, ifccpm;
3767
3768	device = dasd_device_from_cdev_locked(cdev);
3769	if (IS_ERR(ptr: device))
3770	return;
3771
3772	oldopm = dasd_path_get_opm(device);
3773	for (chp = 0; chp < 8; chp++) {
3774	if (path_event[chp] & PE_PATH_GONE) {
3775	dasd_path_notoper(device, chp);
3776	}
3777	if (path_event[chp] & PE_PATH_AVAILABLE) {
3778	dasd_path_available(device, chp);
3779	dasd_schedule_device_bh(device);
3780	}
3781	if (path_event[chp] & PE_PATHGROUP_ESTABLISHED) {
3782	if (!dasd_path_is_operational(device, chp) &&
3783	!dasd_path_need_verify(device, chp)) {
3784	/*
3785	* we can not establish a pathgroup on an
3786	* unavailable path, so trigger a path
3787	* verification first
3788	*/
3789	dasd_path_available(device, chp);
3790	dasd_schedule_device_bh(device);
3791	}
3792	DBF_DEV_EVENT(DBF_WARNING, device, "%s",
3793	"Pathgroup re-established\n");
3794	if (device->discipline->kick_validate)
3795	device->discipline->kick_validate(device);
3796	}
3797	if (path_event[chp] & PE_PATH_FCES_EVENT) {
3798	dasd_path_fcsec_update(device, chp);
3799	dasd_schedule_device_bh(device);
3800	}
3801	}
3802	hpfpm = dasd_path_get_hpfpm(device);
3803	ifccpm = dasd_path_get_ifccpm(device);
3804	if (!dasd_path_get_opm(device) && hpfpm) {
3805	/*
3806	* device has no operational paths but at least one path is
3807	* disabled due to HPF errors
3808	* disable HPF at all and use the path(s) again
3809	*/
3810	if (device->discipline->disable_hpf)
3811	device->discipline->disable_hpf(device);
3812	dasd_device_set_stop_bits(device, DASD_STOPPED_NOT_ACC);
3813	dasd_path_set_tbvpm(device, pm: hpfpm);
3814	dasd_schedule_device_bh(device);
3815	dasd_schedule_requeue(device);
3816	} else if (!dasd_path_get_opm(device) && ifccpm) {
3817	/*
3818	* device has no operational paths but at least one path is
3819	* disabled due to IFCC errors
3820	* trigger path verification on paths with IFCC errors
3821	*/
3822	dasd_path_set_tbvpm(device, pm: ifccpm);
3823	dasd_schedule_device_bh(device);
3824	}
3825	if (oldopm && !dasd_path_get_opm(device) && !hpfpm && !ifccpm) {
3826	dev_warn(&device->cdev->dev,
3827	"No verified channel paths remain for the device\n");
3828	DBF_DEV_EVENT(DBF_WARNING, device,
3829	"%s", "last verified path gone");
3830	/* First call extended error reporting and check for autoquiesce. */
3831	dasd_handle_autoquiesce(device, NULL, DASD_EER_NOPATH);
3832	dasd_device_set_stop_bits(device,
3833	DASD_STOPPED_DC_WAIT);
3834	}
3835	dasd_put_device(device);
3836	}
3837	EXPORT_SYMBOL_GPL(dasd_generic_path_event);
3838
3839	int dasd_generic_verify_path(struct dasd_device *device, __u8 lpm)
3840	{
3841	if (!dasd_path_get_opm(device) && lpm) {
3842	dasd_path_set_opm(device, pm: lpm);
3843	dasd_generic_path_operational(device);
3844	} else
3845	dasd_path_add_opm(device, pm: lpm);
3846	return 0;
3847	}
3848	EXPORT_SYMBOL_GPL(dasd_generic_verify_path);
3849
3850	void dasd_generic_space_exhaust(struct dasd_device *device,
3851	struct dasd_ccw_req *cqr)
3852	{
3853	/* First call extended error reporting and check for autoquiesce. */
3854	dasd_handle_autoquiesce(device, NULL, DASD_EER_NOSPC);
3855
3856	if (device->state < DASD_STATE_BASIC)
3857	return;
3858
3859	if (cqr->status == DASD_CQR_IN_IO ||
3860	cqr->status == DASD_CQR_CLEAR_PENDING) {
3861	cqr->status = DASD_CQR_QUEUED;
3862	cqr->retries++;
3863	}
3864	dasd_device_set_stop_bits(device, DASD_STOPPED_NOSPC);
3865	dasd_device_clear_timer(device);
3866	dasd_schedule_device_bh(device);
3867	}
3868	EXPORT_SYMBOL_GPL(dasd_generic_space_exhaust);
3869
3870	void dasd_generic_space_avail(struct dasd_device *device)
3871	{
3872	dev_info(&device->cdev->dev, "Extent pool space is available\n");
3873	DBF_DEV_EVENT(DBF_WARNING, device, "%s", "space available");
3874
3875	dasd_device_remove_stop_bits(device, DASD_STOPPED_NOSPC);
3876	dasd_schedule_device_bh(device);
3877
3878	if (device->block) {
3879	dasd_schedule_block_bh(device->block);
3880	if (device->block->gdp)
3881	blk_mq_run_hw_queues(q: device->block->gdp->queue, async: true);
3882	}
3883	if (!device->stopped)
3884	wake_up(&generic_waitq);
3885	}
3886	EXPORT_SYMBOL_GPL(dasd_generic_space_avail);
3887
3888	/*
3889	* clear active requests and requeue them to block layer if possible
3890	*/
3891	int dasd_generic_requeue_all_requests(struct dasd_device *device)
3892	{
3893	struct dasd_block *block = device->block;
3894	struct list_head requeue_queue;
3895	struct dasd_ccw_req *cqr, *n;
3896	int rc;
3897
3898	if (!block)
3899	return 0;
3900
3901	INIT_LIST_HEAD(list: &requeue_queue);
3902	rc = _dasd_requests_to_flushqueue(block, flush_queue: &requeue_queue);
3903
3904	/* Now call the callback function of flushed requests */
3905	restart_cb:
3906	list_for_each_entry_safe(cqr, n, &requeue_queue, blocklist) {
3907	wait_event(dasd_flush_wq, (cqr->status < DASD_CQR_QUEUED));
3908	/* Process finished ERP request. */
3909	if (cqr->refers) {
3910	spin_lock_bh(lock: &block->queue_lock);
3911	__dasd_process_erp(device: block->base, cqr);
3912	spin_unlock_bh(lock: &block->queue_lock);
3913	/* restart list_for_xx loop since dasd_process_erp
3914	* might remove multiple elements
3915	*/
3916	goto restart_cb;
3917	}
3918	_dasd_requeue_request(cqr);
3919	list_del_init(entry: &cqr->blocklist);
3920	cqr->block->base->discipline->free_cp(
3921	cqr, (struct request *) cqr->callback_data);
3922	}
3923	dasd_schedule_device_bh(device);
3924	return rc;
3925	}
3926	EXPORT_SYMBOL_GPL(dasd_generic_requeue_all_requests);
3927
3928	static void do_requeue_requests(struct work_struct *work)
3929	{
3930	struct dasd_device *device = container_of(work, struct dasd_device,
3931	requeue_requests);
3932	dasd_generic_requeue_all_requests(device);
3933	dasd_device_remove_stop_bits(device, DASD_STOPPED_NOT_ACC);
3934	if (device->block)
3935	dasd_schedule_block_bh(device->block);
3936	dasd_put_device(device);
3937	}
3938
3939	void dasd_schedule_requeue(struct dasd_device *device)
3940	{
3941	dasd_get_device(device);
3942	/* queue call to dasd_reload_device to the kernel event daemon. */
3943	if (!schedule_work(work: &device->requeue_requests))
3944	dasd_put_device(device);
3945	}
3946	EXPORT_SYMBOL(dasd_schedule_requeue);
3947
3948	static int dasd_handle_autoquiesce(struct dasd_device *device,
3949	struct dasd_ccw_req *cqr,
3950	unsigned int reason)
3951	{
3952	/* in any case write eer message with reason */
3953	if (dasd_eer_enabled(device))
3954	dasd_eer_write(device, cqr, reason);
3955
3956	if (!test_bit(reason, &device->aq_mask))
3957	return 0;
3958
3959	/* notify eer about autoquiesce */
3960	if (dasd_eer_enabled(device))
3961	dasd_eer_write(device, NULL, DASD_EER_AUTOQUIESCE);
3962
3963	pr_info("%s: The DASD has been put in the quiesce state\n",
3964	dev_name(&device->cdev->dev));
3965	dasd_device_set_stop_bits(device, DASD_STOPPED_QUIESCE);
3966
3967	if (device->features & DASD_FEATURE_REQUEUEQUIESCE)
3968	dasd_schedule_requeue(device);
3969
3970	return 1;
3971	}
3972
3973	static struct dasd_ccw_req *dasd_generic_build_rdc(struct dasd_device *device,
3974	int rdc_buffer_size,
3975	int magic)
3976	{
3977	struct dasd_ccw_req *cqr;
3978	struct ccw1 *ccw;
3979
3980	cqr = dasd_smalloc_request(magic, 1 /* RDC */, rdc_buffer_size, device,
3981	NULL);
3982
3983	if (IS_ERR(ptr: cqr)) {
3984	/* internal error 13 - Allocating the RDC request failed*/
3985	dev_err(&device->cdev->dev,
3986	"An error occurred in the DASD device driver, "
3987	"reason=%s\n", "13");
3988	return cqr;
3989	}
3990
3991	ccw = cqr->cpaddr;
3992	ccw->cmd_code = CCW_CMD_RDC;
3993	ccw->cda = (__u32)virt_to_phys(address: cqr->data);
3994	ccw->flags = 0;
3995	ccw->count = rdc_buffer_size;
3996	cqr->startdev = device;
3997	cqr->memdev = device;
3998	cqr->expires = 10*HZ;
3999	cqr->retries = 256;
4000	cqr->buildclk = get_tod_clock();
4001	cqr->status = DASD_CQR_FILLED;
4002	return cqr;
4003	}
4004
4005
4006	int dasd_generic_read_dev_chars(struct dasd_device *device, int magic,
4007	void *rdc_buffer, int rdc_buffer_size)
4008	{
4009	int ret;
4010	struct dasd_ccw_req *cqr;
4011
4012	cqr = dasd_generic_build_rdc(device, rdc_buffer_size, magic);
4013	if (IS_ERR(ptr: cqr))
4014	return PTR_ERR(ptr: cqr);
4015
4016	ret = dasd_sleep_on(cqr);
4017	if (ret == 0)
4018	memcpy(rdc_buffer, cqr->data, rdc_buffer_size);
4019	dasd_sfree_request(cqr, cqr->memdev);
4020	return ret;
4021	}
4022	EXPORT_SYMBOL_GPL(dasd_generic_read_dev_chars);
4023
4024	/*
4025	* In command mode and transport mode we need to look for sense
4026	* data in different places. The sense data itself is allways
4027	* an array of 32 bytes, so we can unify the sense data access
4028	* for both modes.
4029	*/
4030	char *dasd_get_sense(struct irb *irb)
4031	{
4032	struct tsb *tsb = NULL;
4033	char *sense = NULL;
4034
4035	if (scsw_is_tm(&irb->scsw) && (irb->scsw.tm.fcxs == 0x01)) {
4036	if (irb->scsw.tm.tcw)
4037	tsb = tcw_get_tsb(phys_to_virt(address: irb->scsw.tm.tcw));
4038	if (tsb && tsb->length == 64 && tsb->flags)
4039	switch (tsb->flags & 0x07) {
4040	case 1: /* tsa_iostat */
4041	sense = tsb->tsa.iostat.sense;
4042	break;
4043	case 2: /* tsa_ddpc */
4044	sense = tsb->tsa.ddpc.sense;
4045	break;
4046	default:
4047	/* currently we don't use interrogate data */
4048	break;
4049	}
4050	} else if (irb->esw.esw0.erw.cons) {
4051	sense = irb->ecw;
4052	}
4053	return sense;
4054	}
4055	EXPORT_SYMBOL_GPL(dasd_get_sense);
4056
4057	void dasd_generic_shutdown(struct ccw_device *cdev)
4058	{
4059	struct dasd_device *device;
4060
4061	device = dasd_device_from_cdev(cdev);
4062	if (IS_ERR(ptr: device))
4063	return;
4064
4065	if (device->block)
4066	dasd_schedule_block_bh(device->block);
4067
4068	dasd_schedule_device_bh(device);
4069
4070	wait_event(shutdown_waitq, _wait_for_empty_queues(device));
4071	}
4072	EXPORT_SYMBOL_GPL(dasd_generic_shutdown);
4073
4074	static int __init dasd_init(void)
4075	{
4076	int rc;
4077
4078	init_waitqueue_head(&dasd_init_waitq);
4079	init_waitqueue_head(&dasd_flush_wq);
4080	init_waitqueue_head(&generic_waitq);
4081	init_waitqueue_head(&shutdown_waitq);
4082
4083	/* register 'common' DASD debug area, used for all DBF_XXX calls */
4084	dasd_debug_area = debug_register("dasd", 1, 1, 8 * sizeof(long));
4085	if (dasd_debug_area == NULL) {
4086	rc = -ENOMEM;
4087	goto failed;
4088	}
4089	debug_register_view(dasd_debug_area, &debug_sprintf_view);
4090	debug_set_level(dasd_debug_area, DBF_WARNING);
4091
4092	DBF_EVENT(DBF_EMERG, "%s", "debug area created");
4093
4094	dasd_diag_discipline_pointer = NULL;
4095
4096	dasd_statistics_createroot();
4097
4098	rc = dasd_devmap_init();
4099	if (rc)
4100	goto failed;
4101	rc = dasd_gendisk_init();
4102	if (rc)
4103	goto failed;
4104	rc = dasd_parse();
4105	if (rc)
4106	goto failed;
4107	rc = dasd_eer_init();
4108	if (rc)
4109	goto failed;
4110	#ifdef CONFIG_PROC_FS
4111	rc = dasd_proc_init();
4112	if (rc)
4113	goto failed;
4114	#endif
4115
4116	return 0;
4117	failed:
4118	pr_info("The DASD device driver could not be initialized\n");
4119	dasd_exit();
4120	return rc;
4121	}
4122
4123	module_init(dasd_init);
4124	module_exit(dasd_exit);
4125