1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* libata-eh.c - libata error handling
4	*
5	* Copyright 2006 Tejun Heo <htejun@gmail.com>
6	*
7	* libata documentation is available via 'make {ps|pdf}docs',
8	* as Documentation/driver-api/libata.rst
9	*
10	* Hardware documentation available from http://www.t13.org/ and
11	* http://www.sata-io.org/
12	*/
13
14	#include <linux/kernel.h>
15	#include <linux/blkdev.h>
16	#include <linux/export.h>
17	#include <linux/pci.h>
18	#include <scsi/scsi.h>
19	#include <scsi/scsi_host.h>
20	#include <scsi/scsi_eh.h>
21	#include <scsi/scsi_device.h>
22	#include <scsi/scsi_cmnd.h>
23	#include <scsi/scsi_dbg.h>
24	#include "../scsi/scsi_transport_api.h"
25
26	#include <linux/libata.h>
27
28	#include <trace/events/libata.h>
29	#include "libata.h"
30
31	enum {
32	/* speed down verdicts */
33	ATA_EH_SPDN_NCQ_OFF = (1 << 0),
34	ATA_EH_SPDN_SPEED_DOWN = (1 << 1),
35	ATA_EH_SPDN_FALLBACK_TO_PIO = (1 << 2),
36	ATA_EH_SPDN_KEEP_ERRORS = (1 << 3),
37
38	/* error flags */
39	ATA_EFLAG_IS_IO = (1 << 0),
40	ATA_EFLAG_DUBIOUS_XFER = (1 << 1),
41	ATA_EFLAG_OLD_ER = (1 << 31),
42
43	/* error categories */
44	ATA_ECAT_NONE = 0,
45	ATA_ECAT_ATA_BUS = 1,
46	ATA_ECAT_TOUT_HSM = 2,
47	ATA_ECAT_UNK_DEV = 3,
48	ATA_ECAT_DUBIOUS_NONE = 4,
49	ATA_ECAT_DUBIOUS_ATA_BUS = 5,
50	ATA_ECAT_DUBIOUS_TOUT_HSM = 6,
51	ATA_ECAT_DUBIOUS_UNK_DEV = 7,
52	ATA_ECAT_NR = 8,
53
54	ATA_EH_CMD_DFL_TIMEOUT = 5000,
55
56	/* always put at least this amount of time between resets */
57	ATA_EH_RESET_COOL_DOWN = 5000,
58
59	/* Waiting in ->prereset can never be reliable. It's
60	* sometimes nice to wait there but it can't be depended upon;
61	* otherwise, we wouldn't be resetting. Just give it enough
62	* time for most drives to spin up.
63	*/
64	ATA_EH_PRERESET_TIMEOUT = 10000,
65	ATA_EH_FASTDRAIN_INTERVAL = 3000,
66
67	ATA_EH_UA_TRIES = 5,
68
69	/* probe speed down parameters, see ata_eh_schedule_probe() */
70	ATA_EH_PROBE_TRIAL_INTERVAL = 60000, /* 1 min */
71	ATA_EH_PROBE_TRIALS = 2,
72	};
73
74	/* The following table determines how we sequence resets. Each entry
75	* represents timeout for that try. The first try can be soft or
76	* hardreset. All others are hardreset if available. In most cases
77	* the first reset w/ 10sec timeout should succeed. Following entries
78	* are mostly for error handling, hotplug and those outlier devices that
79	* take an exceptionally long time to recover from reset.
80	*/
81	static const unsigned int ata_eh_reset_timeouts[] = {
82	10000, /* most drives spin up by 10sec */
83	10000, /* > 99% working drives spin up before 20sec */
84	35000, /* give > 30 secs of idleness for outlier devices */
85	5000, /* and sweet one last chance */
86	UINT_MAX, /* > 1 min has elapsed, give up */
87	};
88
89	static const unsigned int ata_eh_identify_timeouts[] = {
90	5000, /* covers > 99% of successes and not too boring on failures */
91	10000, /* combined time till here is enough even for media access */
92	30000, /* for true idiots */
93	UINT_MAX,
94	};
95
96	static const unsigned int ata_eh_revalidate_timeouts[] = {
97	15000, /* Some drives are slow to read log pages when waking-up */
98	15000, /* combined time till here is enough even for media access */
99	UINT_MAX,
100	};
101
102	static const unsigned int ata_eh_flush_timeouts[] = {
103	15000, /* be generous with flush */
104	15000, /* ditto */
105	30000, /* and even more generous */
106	UINT_MAX,
107	};
108
109	static const unsigned int ata_eh_other_timeouts[] = {
110	5000, /* same rationale as identify timeout */
111	10000, /* ditto */
112	/* but no merciful 30sec for other commands, it just isn't worth it */
113	UINT_MAX,
114	};
115
116	struct ata_eh_cmd_timeout_ent {
117	const u8 *commands;
118	const unsigned int *timeouts;
119	};
120
121	/* The following table determines timeouts to use for EH internal
122	* commands. Each table entry is a command class and matches the
123	* commands the entry applies to and the timeout table to use.
124	*
125	* On the retry after a command timed out, the next timeout value from
126	* the table is used. If the table doesn't contain further entries,
127	* the last value is used.
128	*
129	* ehc->cmd_timeout_idx keeps track of which timeout to use per
130	* command class, so if SET_FEATURES times out on the first try, the
131	* next try will use the second timeout value only for that class.
132	*/
133	#define CMDS(cmds...) (const u8 []){ cmds, 0 }
134	static const struct ata_eh_cmd_timeout_ent
135	ata_eh_cmd_timeout_table[ATA_EH_CMD_TIMEOUT_TABLE_SIZE] = {
136	{ .commands = CMDS(ATA_CMD_ID_ATA, ATA_CMD_ID_ATAPI),
137	.timeouts = ata_eh_identify_timeouts, },
138	{ .commands = CMDS(ATA_CMD_READ_LOG_EXT, ATA_CMD_READ_LOG_DMA_EXT),
139	.timeouts = ata_eh_revalidate_timeouts, },
140	{ .commands = CMDS(ATA_CMD_READ_NATIVE_MAX, ATA_CMD_READ_NATIVE_MAX_EXT),
141	.timeouts = ata_eh_other_timeouts, },
142	{ .commands = CMDS(ATA_CMD_SET_MAX, ATA_CMD_SET_MAX_EXT),
143	.timeouts = ata_eh_other_timeouts, },
144	{ .commands = CMDS(ATA_CMD_SET_FEATURES),
145	.timeouts = ata_eh_other_timeouts, },
146	{ .commands = CMDS(ATA_CMD_INIT_DEV_PARAMS),
147	.timeouts = ata_eh_other_timeouts, },
148	{ .commands = CMDS(ATA_CMD_FLUSH, ATA_CMD_FLUSH_EXT),
149	.timeouts = ata_eh_flush_timeouts },
150	{ .commands = CMDS(ATA_CMD_VERIFY),
151	.timeouts = ata_eh_reset_timeouts },
152	};
153	#undef CMDS
154
155	static void __ata_port_freeze(struct ata_port *ap);
156	static int ata_eh_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
157	struct ata_device **r_failed_dev);
158	#ifdef CONFIG_PM
159	static void ata_eh_handle_port_suspend(struct ata_port *ap);
160	static void ata_eh_handle_port_resume(struct ata_port *ap);
161	#else /* CONFIG_PM */
162	static void ata_eh_handle_port_suspend(struct ata_port *ap)
163	{ }
164
165	static void ata_eh_handle_port_resume(struct ata_port *ap)
166	{ }
167	#endif /* CONFIG_PM */
168
169	static __printf(2, 0) void __ata_ehi_pushv_desc(struct ata_eh_info *ehi,
170	const char *fmt, va_list args)
171	{
172	ehi->desc_len += vscnprintf(buf: ehi->desc + ehi->desc_len,
173	size: ATA_EH_DESC_LEN - ehi->desc_len,
174	fmt, args);
175	}
176
177	/**
178	* __ata_ehi_push_desc - push error description without adding separator
179	* @ehi: target EHI
180	* @fmt: printf format string
181	*
182	* Format string according to @fmt and append it to @ehi->desc.
183	*
184	* LOCKING:
185	* spin_lock_irqsave(host lock)
186	*/
187	void __ata_ehi_push_desc(struct ata_eh_info *ehi, const char *fmt, ...)
188	{
189	va_list args;
190
191	va_start(args, fmt);
192	__ata_ehi_pushv_desc(ehi, fmt, args);
193	va_end(args);
194	}
195	EXPORT_SYMBOL_GPL(__ata_ehi_push_desc);
196
197	/**
198	* ata_ehi_push_desc - push error description with separator
199	* @ehi: target EHI
200	* @fmt: printf format string
201	*
202	* Format string according to @fmt and append it to @ehi->desc.
203	* If @ehi->desc is not empty, ", " is added in-between.
204	*
205	* LOCKING:
206	* spin_lock_irqsave(host lock)
207	*/
208	void ata_ehi_push_desc(struct ata_eh_info *ehi, const char *fmt, ...)
209	{
210	va_list args;
211
212	if (ehi->desc_len)
213	__ata_ehi_push_desc(ehi, ", ");
214
215	va_start(args, fmt);
216	__ata_ehi_pushv_desc(ehi, fmt, args);
217	va_end(args);
218	}
219	EXPORT_SYMBOL_GPL(ata_ehi_push_desc);
220
221	/**
222	* ata_ehi_clear_desc - clean error description
223	* @ehi: target EHI
224	*
225	* Clear @ehi->desc.
226	*
227	* LOCKING:
228	* spin_lock_irqsave(host lock)
229	*/
230	void ata_ehi_clear_desc(struct ata_eh_info *ehi)
231	{
232	ehi->desc[0] = '\0';
233	ehi->desc_len = 0;
234	}
235	EXPORT_SYMBOL_GPL(ata_ehi_clear_desc);
236
237	/**
238	* ata_port_desc - append port description
239	* @ap: target ATA port
240	* @fmt: printf format string
241	*
242	* Format string according to @fmt and append it to port
243	* description. If port description is not empty, " " is added
244	* in-between. This function is to be used while initializing
245	* ata_host. The description is printed on host registration.
246	*
247	* LOCKING:
248	* None.
249	*/
250	void ata_port_desc(struct ata_port *ap, const char *fmt, ...)
251	{
252	va_list args;
253
254	WARN_ON(!(ap->pflags & ATA_PFLAG_INITIALIZING));
255
256	if (ap->link.eh_info.desc_len)
257	__ata_ehi_push_desc(&ap->link.eh_info, " ");
258
259	va_start(args, fmt);
260	__ata_ehi_pushv_desc(ehi: &ap->link.eh_info, fmt, args);
261	va_end(args);
262	}
263	EXPORT_SYMBOL_GPL(ata_port_desc);
264
265	#ifdef CONFIG_PCI
266	/**
267	* ata_port_pbar_desc - append PCI BAR description
268	* @ap: target ATA port
269	* @bar: target PCI BAR
270	* @offset: offset into PCI BAR
271	* @name: name of the area
272	*
273	* If @offset is negative, this function formats a string which
274	* contains the name, address, size and type of the BAR and
275	* appends it to the port description. If @offset is zero or
276	* positive, only name and offsetted address is appended.
277	*
278	* LOCKING:
279	* None.
280	*/
281	void ata_port_pbar_desc(struct ata_port *ap, int bar, ssize_t offset,
282	const char *name)
283	{
284	struct pci_dev *pdev = to_pci_dev(ap->host->dev);
285	char *type = "";
286	unsigned long long start, len;
287
288	if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM)
289	type = "m";
290	else if (pci_resource_flags(pdev, bar) & IORESOURCE_IO)
291	type = "i";
292
293	start = (unsigned long long)pci_resource_start(pdev, bar);
294	len = (unsigned long long)pci_resource_len(pdev, bar);
295
296	if (offset < 0)
297	ata_port_desc(ap, "%s %s%llu@0x%llx", name, type, len, start);
298	else
299	ata_port_desc(ap, "%s 0x%llx", name,
300	start + (unsigned long long)offset);
301	}
302	EXPORT_SYMBOL_GPL(ata_port_pbar_desc);
303	#endif /* CONFIG_PCI */
304
305	static int ata_lookup_timeout_table(u8 cmd)
306	{
307	int i;
308
309	for (i = 0; i < ATA_EH_CMD_TIMEOUT_TABLE_SIZE; i++) {
310	const u8 *cur;
311
312	for (cur = ata_eh_cmd_timeout_table[i].commands; *cur; cur++)
313	if (*cur == cmd)
314	return i;
315	}
316
317	return -1;
318	}
319
320	/**
321	* ata_internal_cmd_timeout - determine timeout for an internal command
322	* @dev: target device
323	* @cmd: internal command to be issued
324	*
325	* Determine timeout for internal command @cmd for @dev.
326	*
327	* LOCKING:
328	* EH context.
329	*
330	* RETURNS:
331	* Determined timeout.
332	*/
333	unsigned int ata_internal_cmd_timeout(struct ata_device *dev, u8 cmd)
334	{
335	struct ata_eh_context *ehc = &dev->link->eh_context;
336	int ent = ata_lookup_timeout_table(cmd);
337	int idx;
338
339	if (ent < 0)
340	return ATA_EH_CMD_DFL_TIMEOUT;
341
342	idx = ehc->cmd_timeout_idx[dev->devno][ent];
343	return ata_eh_cmd_timeout_table[ent].timeouts[idx];
344	}
345
346	/**
347	* ata_internal_cmd_timed_out - notification for internal command timeout
348	* @dev: target device
349	* @cmd: internal command which timed out
350	*
351	* Notify EH that internal command @cmd for @dev timed out. This
352	* function should be called only for commands whose timeouts are
353	* determined using ata_internal_cmd_timeout().
354	*
355	* LOCKING:
356	* EH context.
357	*/
358	void ata_internal_cmd_timed_out(struct ata_device *dev, u8 cmd)
359	{
360	struct ata_eh_context *ehc = &dev->link->eh_context;
361	int ent = ata_lookup_timeout_table(cmd);
362	int idx;
363
364	if (ent < 0)
365	return;
366
367	idx = ehc->cmd_timeout_idx[dev->devno][ent];
368	if (ata_eh_cmd_timeout_table[ent].timeouts[idx + 1] != UINT_MAX)
369	ehc->cmd_timeout_idx[dev->devno][ent]++;
370	}
371
372	static void ata_ering_record(struct ata_ering *ering, unsigned int eflags,
373	unsigned int err_mask)
374	{
375	struct ata_ering_entry *ent;
376
377	WARN_ON(!err_mask);
378
379	ering->cursor++;
380	ering->cursor %= ATA_ERING_SIZE;
381
382	ent = &ering->ring[ering->cursor];
383	ent->eflags = eflags;
384	ent->err_mask = err_mask;
385	ent->timestamp = get_jiffies_64();
386	}
387
388	static struct ata_ering_entry *ata_ering_top(struct ata_ering *ering)
389	{
390	struct ata_ering_entry *ent = &ering->ring[ering->cursor];
391
392	if (ent->err_mask)
393	return ent;
394	return NULL;
395	}
396
397	int ata_ering_map(struct ata_ering *ering,
398	int (*map_fn)(struct ata_ering_entry *, void *),
399	void *arg)
400	{
401	int idx, rc = 0;
402	struct ata_ering_entry *ent;
403
404	idx = ering->cursor;
405	do {
406	ent = &ering->ring[idx];
407	if (!ent->err_mask)
408	break;
409	rc = map_fn(ent, arg);
410	if (rc)
411	break;
412	idx = (idx - 1 + ATA_ERING_SIZE) % ATA_ERING_SIZE;
413	} while (idx != ering->cursor);
414
415	return rc;
416	}
417
418	static int ata_ering_clear_cb(struct ata_ering_entry *ent, void *void_arg)
419	{
420	ent->eflags |= ATA_EFLAG_OLD_ER;
421	return 0;
422	}
423
424	static void ata_ering_clear(struct ata_ering *ering)
425	{
426	ata_ering_map(ering, map_fn: ata_ering_clear_cb, NULL);
427	}
428
429	static unsigned int ata_eh_dev_action(struct ata_device *dev)
430	{
431	struct ata_eh_context *ehc = &dev->link->eh_context;
432
433	return ehc->i.action | ehc->i.dev_action[dev->devno];
434	}
435
436	static void ata_eh_clear_action(struct ata_link *link, struct ata_device *dev,
437	struct ata_eh_info *ehi, unsigned int action)
438	{
439	struct ata_device *tdev;
440
441	if (!dev) {
442	ehi->action &= ~action;
443	ata_for_each_dev(tdev, link, ALL)
444	ehi->dev_action[tdev->devno] &= ~action;
445	} else {
446	/* doesn't make sense for port-wide EH actions */
447	WARN_ON(!(action & ATA_EH_PERDEV_MASK));
448
449	/* break ehi->action into ehi->dev_action */
450	if (ehi->action & action) {
451	ata_for_each_dev(tdev, link, ALL)
452	ehi->dev_action[tdev->devno] |=
453	ehi->action & action;
454	ehi->action &= ~action;
455	}
456
457	/* turn off the specified per-dev action */
458	ehi->dev_action[dev->devno] &= ~action;
459	}
460	}
461
462	/**
463	* ata_eh_acquire - acquire EH ownership
464	* @ap: ATA port to acquire EH ownership for
465	*
466	* Acquire EH ownership for @ap. This is the basic exclusion
467	* mechanism for ports sharing a host. Only one port hanging off
468	* the same host can claim the ownership of EH.
469	*
470	* LOCKING:
471	* EH context.
472	*/
473	void ata_eh_acquire(struct ata_port *ap)
474	{
475	mutex_lock(&ap->host->eh_mutex);
476	WARN_ON_ONCE(ap->host->eh_owner);
477	ap->host->eh_owner = current;
478	}
479
480	/**
481	* ata_eh_release - release EH ownership
482	* @ap: ATA port to release EH ownership for
483	*
484	* Release EH ownership for @ap if the caller. The caller must
485	* have acquired EH ownership using ata_eh_acquire() previously.
486	*
487	* LOCKING:
488	* EH context.
489	*/
490	void ata_eh_release(struct ata_port *ap)
491	{
492	WARN_ON_ONCE(ap->host->eh_owner != current);
493	ap->host->eh_owner = NULL;
494	mutex_unlock(lock: &ap->host->eh_mutex);
495	}
496
497	static void ata_eh_dev_disable(struct ata_device *dev)
498	{
499	ata_acpi_on_disable(dev);
500	ata_down_xfermask_limit(dev, sel: ATA_DNXFER_FORCE_PIO0 | ATA_DNXFER_QUIET);
501	dev->class++;
502
503	/* From now till the next successful probe, ering is used to
504	* track probe failures. Clear accumulated device error info.
505	*/
506	ata_ering_clear(ering: &dev->ering);
507	}
508
509	static void ata_eh_unload(struct ata_port *ap)
510	{
511	struct ata_link *link;
512	struct ata_device *dev;
513	unsigned long flags;
514
515	/*
516	* Unless we are restarting, transition all enabled devices to
517	* standby power mode.
518	*/
519	if (system_state != SYSTEM_RESTART) {
520	ata_for_each_link(link, ap, PMP_FIRST) {
521	ata_for_each_dev(dev, link, ENABLED)
522	ata_dev_power_set_standby(dev);
523	}
524	}
525
526	/*
527	* Restore SControl IPM and SPD for the next driver and
528	* disable attached devices.
529	*/
530	ata_for_each_link(link, ap, PMP_FIRST) {
531	sata_scr_write(link, reg: SCR_CONTROL, val: link->saved_scontrol & 0xff0);
532	ata_for_each_dev(dev, link, ENABLED)
533	ata_eh_dev_disable(dev);
534	}
535
536	/* freeze and set UNLOADED */
537	spin_lock_irqsave(ap->lock, flags);
538
539	ata_port_freeze(ap); /* won't be thawed */
540	ap->pflags &= ~ATA_PFLAG_EH_PENDING; /* clear pending from freeze */
541	ap->pflags |= ATA_PFLAG_UNLOADED;
542
543	spin_unlock_irqrestore(lock: ap->lock, flags);
544	}
545
546	/**
547	* ata_scsi_error - SCSI layer error handler callback
548	* @host: SCSI host on which error occurred
549	*
550	* Handles SCSI-layer-thrown error events.
551	*
552	* LOCKING:
553	* Inherited from SCSI layer (none, can sleep)
554	*
555	* RETURNS:
556	* Zero.
557	*/
558	void ata_scsi_error(struct Scsi_Host *host)
559	{
560	struct ata_port *ap = ata_shost_to_port(host);
561	unsigned long flags;
562	LIST_HEAD(eh_work_q);
563
564	spin_lock_irqsave(host->host_lock, flags);
565	list_splice_init(list: &host->eh_cmd_q, head: &eh_work_q);
566	spin_unlock_irqrestore(lock: host->host_lock, flags);
567
568	ata_scsi_cmd_error_handler(host, ap, eh_q: &eh_work_q);
569
570	/* If we timed raced normal completion and there is nothing to
571	recover nr_timedout == 0 why exactly are we doing error recovery ? */
572	ata_scsi_port_error_handler(host, ap);
573
574	/* finish or retry handled scmd's and clean up */
575	WARN_ON(!list_empty(&eh_work_q));
576
577	}
578
579	/**
580	* ata_scsi_cmd_error_handler - error callback for a list of commands
581	* @host: scsi host containing the port
582	* @ap: ATA port within the host
583	* @eh_work_q: list of commands to process
584	*
585	* process the given list of commands and return those finished to the
586	* ap->eh_done_q. This function is the first part of the libata error
587	* handler which processes a given list of failed commands.
588	*/
589	void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,
590	struct list_head *eh_work_q)
591	{
592	int i;
593	unsigned long flags;
594	struct scsi_cmnd *scmd, *tmp;
595	int nr_timedout = 0;
596
597	/* make sure sff pio task is not running */
598	ata_sff_flush_pio_task(ap);
599
600	/* synchronize with host lock and sort out timeouts */
601
602	/*
603	* For EH, all qcs are finished in one of three ways -
604	* normal completion, error completion, and SCSI timeout.
605	* Both completions can race against SCSI timeout. When normal
606	* completion wins, the qc never reaches EH. When error
607	* completion wins, the qc has ATA_QCFLAG_EH set.
608	*
609	* When SCSI timeout wins, things are a bit more complex.
610	* Normal or error completion can occur after the timeout but
611	* before this point. In such cases, both types of
612	* completions are honored. A scmd is determined to have
613	* timed out iff its associated qc is active and not failed.
614	*/
615	spin_lock_irqsave(ap->lock, flags);
616
617	/*
618	* This must occur under the ap->lock as we don't want
619	* a polled recovery to race the real interrupt handler
620	*
621	* The lost_interrupt handler checks for any completed but
622	* non-notified command and completes much like an IRQ handler.
623	*
624	* We then fall into the error recovery code which will treat
625	* this as if normal completion won the race
626	*/
627	if (ap->ops->lost_interrupt)
628	ap->ops->lost_interrupt(ap);
629
630	list_for_each_entry_safe(scmd, tmp, eh_work_q, eh_entry) {
631	struct ata_queued_cmd *qc;
632
633	ata_qc_for_each_raw(ap, qc, i) {
634	if (qc->flags & ATA_QCFLAG_ACTIVE &&
635	qc->scsicmd == scmd)
636	break;
637	}
638
639	if (i < ATA_MAX_QUEUE) {
640	/* the scmd has an associated qc */
641	if (!(qc->flags & ATA_QCFLAG_EH)) {
642	/* which hasn't failed yet, timeout */
643	qc->err_mask |= AC_ERR_TIMEOUT;
644	qc->flags |= ATA_QCFLAG_EH;
645	nr_timedout++;
646	}
647	} else {
648	/* Normal completion occurred after
649	* SCSI timeout but before this point.
650	* Successfully complete it.
651	*/
652	scmd->retries = scmd->allowed;
653	scsi_eh_finish_cmd(scmd, done_q: &ap->eh_done_q);
654	}
655	}
656
657	/*
658	* If we have timed out qcs. They belong to EH from
659	* this point but the state of the controller is
660	* unknown. Freeze the port to make sure the IRQ
661	* handler doesn't diddle with those qcs. This must
662	* be done atomically w.r.t. setting ATA_QCFLAG_EH.
663	*/
664	if (nr_timedout)
665	__ata_port_freeze(ap);
666
667	/* initialize eh_tries */
668	ap->eh_tries = ATA_EH_MAX_TRIES;
669
670	spin_unlock_irqrestore(lock: ap->lock, flags);
671	}
672	EXPORT_SYMBOL(ata_scsi_cmd_error_handler);
673
674	/**
675	* ata_scsi_port_error_handler - recover the port after the commands
676	* @host: SCSI host containing the port
677	* @ap: the ATA port
678	*
679	* Handle the recovery of the port @ap after all the commands
680	* have been recovered.
681	*/
682	void ata_scsi_port_error_handler(struct Scsi_Host *host, struct ata_port *ap)
683	{
684	unsigned long flags;
685	struct ata_link *link;
686
687	/* acquire EH ownership */
688	ata_eh_acquire(ap);
689	repeat:
690	/* kill fast drain timer */
691	del_timer_sync(timer: &ap->fastdrain_timer);
692
693	/* process port resume request */
694	ata_eh_handle_port_resume(ap);
695
696	/* fetch & clear EH info */
697	spin_lock_irqsave(ap->lock, flags);
698
699	ata_for_each_link(link, ap, HOST_FIRST) {
700	struct ata_eh_context *ehc = &link->eh_context;
701	struct ata_device *dev;
702
703	memset(&link->eh_context, 0, sizeof(link->eh_context));
704	link->eh_context.i = link->eh_info;
705	memset(&link->eh_info, 0, sizeof(link->eh_info));
706
707	ata_for_each_dev(dev, link, ENABLED) {
708	int devno = dev->devno;
709
710	ehc->saved_xfer_mode[devno] = dev->xfer_mode;
711	if (ata_ncq_enabled(dev))
712	ehc->saved_ncq_enabled |= 1 << devno;
713
714	/* If we are resuming, wake up the device */
715	if (ap->pflags & ATA_PFLAG_RESUMING)
716	ehc->i.dev_action[devno] |= ATA_EH_SET_ACTIVE;
717	}
718	}
719
720	ap->pflags |= ATA_PFLAG_EH_IN_PROGRESS;
721	ap->pflags &= ~ATA_PFLAG_EH_PENDING;
722	ap->excl_link = NULL; /* don't maintain exclusion over EH */
723
724	spin_unlock_irqrestore(lock: ap->lock, flags);
725
726	/* invoke EH, skip if unloading or suspended */
727	if (!(ap->pflags & (ATA_PFLAG_UNLOADING | ATA_PFLAG_SUSPENDED)))
728	ap->ops->error_handler(ap);
729	else {
730	/* if unloading, commence suicide */
731	if ((ap->pflags & ATA_PFLAG_UNLOADING) &&
732	!(ap->pflags & ATA_PFLAG_UNLOADED))
733	ata_eh_unload(ap);
734	ata_eh_finish(ap);
735	}
736
737	/* process port suspend request */
738	ata_eh_handle_port_suspend(ap);
739
740	/*
741	* Exception might have happened after ->error_handler recovered the
742	* port but before this point. Repeat EH in such case.
743	*/
744	spin_lock_irqsave(ap->lock, flags);
745
746	if (ap->pflags & ATA_PFLAG_EH_PENDING) {
747	if (--ap->eh_tries) {
748	spin_unlock_irqrestore(lock: ap->lock, flags);
749	goto repeat;
750	}
751	ata_port_err(ap,
752	"EH pending after %d tries, giving up\n",
753	ATA_EH_MAX_TRIES);
754	ap->pflags &= ~ATA_PFLAG_EH_PENDING;
755	}
756
757	/* this run is complete, make sure EH info is clear */
758	ata_for_each_link(link, ap, HOST_FIRST)
759	memset(&link->eh_info, 0, sizeof(link->eh_info));
760
761	/*
762	* end eh (clear host_eh_scheduled) while holding ap->lock such that if
763	* exception occurs after this point but before EH completion, SCSI
764	* midlayer will re-initiate EH.
765	*/
766	ap->ops->end_eh(ap);
767
768	spin_unlock_irqrestore(lock: ap->lock, flags);
769	ata_eh_release(ap);
770
771	scsi_eh_flush_done_q(done_q: &ap->eh_done_q);
772
773	/* clean up */
774	spin_lock_irqsave(ap->lock, flags);
775
776	ap->pflags &= ~ATA_PFLAG_RESUMING;
777
778	if (ap->pflags & ATA_PFLAG_LOADING)
779	ap->pflags &= ~ATA_PFLAG_LOADING;
780	else if ((ap->pflags & ATA_PFLAG_SCSI_HOTPLUG) &&
781	!(ap->flags & ATA_FLAG_SAS_HOST))
782	schedule_delayed_work(dwork: &ap->hotplug_task, delay: 0);
783
784	if (ap->pflags & ATA_PFLAG_RECOVERED)
785	ata_port_info(ap, "EH complete\n");
786
787	ap->pflags &= ~(ATA_PFLAG_SCSI_HOTPLUG | ATA_PFLAG_RECOVERED);
788
789	/* tell wait_eh that we're done */
790	ap->pflags &= ~ATA_PFLAG_EH_IN_PROGRESS;
791	wake_up_all(&ap->eh_wait_q);
792
793	spin_unlock_irqrestore(lock: ap->lock, flags);
794	}
795	EXPORT_SYMBOL_GPL(ata_scsi_port_error_handler);
796
797	/**
798	* ata_port_wait_eh - Wait for the currently pending EH to complete
799	* @ap: Port to wait EH for
800	*
801	* Wait until the currently pending EH is complete.
802	*
803	* LOCKING:
804	* Kernel thread context (may sleep).
805	*/
806	void ata_port_wait_eh(struct ata_port *ap)
807	{
808	unsigned long flags;
809	DEFINE_WAIT(wait);
810
811	retry:
812	spin_lock_irqsave(ap->lock, flags);
813
814	while (ap->pflags & (ATA_PFLAG_EH_PENDING | ATA_PFLAG_EH_IN_PROGRESS)) {
815	prepare_to_wait(wq_head: &ap->eh_wait_q, wq_entry: &wait, TASK_UNINTERRUPTIBLE);
816	spin_unlock_irqrestore(lock: ap->lock, flags);
817	schedule();
818	spin_lock_irqsave(ap->lock, flags);
819	}
820	finish_wait(wq_head: &ap->eh_wait_q, wq_entry: &wait);
821
822	spin_unlock_irqrestore(lock: ap->lock, flags);
823
824	/* make sure SCSI EH is complete */
825	if (scsi_host_in_recovery(shost: ap->scsi_host)) {
826	ata_msleep(ap, msecs: 10);
827	goto retry;
828	}
829	}
830	EXPORT_SYMBOL_GPL(ata_port_wait_eh);
831
832	static unsigned int ata_eh_nr_in_flight(struct ata_port *ap)
833	{
834	struct ata_queued_cmd *qc;
835	unsigned int tag;
836	unsigned int nr = 0;
837
838	/* count only non-internal commands */
839	ata_qc_for_each(ap, qc, tag) {
840	if (qc)
841	nr++;
842	}
843
844	return nr;
845	}
846
847	void ata_eh_fastdrain_timerfn(struct timer_list *t)
848	{
849	struct ata_port *ap = from_timer(ap, t, fastdrain_timer);
850	unsigned long flags;
851	unsigned int cnt;
852
853	spin_lock_irqsave(ap->lock, flags);
854
855	cnt = ata_eh_nr_in_flight(ap);
856
857	/* are we done? */
858	if (!cnt)
859	goto out_unlock;
860
861	if (cnt == ap->fastdrain_cnt) {
862	struct ata_queued_cmd *qc;
863	unsigned int tag;
864
865	/* No progress during the last interval, tag all
866	* in-flight qcs as timed out and freeze the port.
867	*/
868	ata_qc_for_each(ap, qc, tag) {
869	if (qc)
870	qc->err_mask |= AC_ERR_TIMEOUT;
871	}
872
873	ata_port_freeze(ap);
874	} else {
875	/* some qcs have finished, give it another chance */
876	ap->fastdrain_cnt = cnt;
877	ap->fastdrain_timer.expires =
878	ata_deadline(from_jiffies: jiffies, timeout_msecs: ATA_EH_FASTDRAIN_INTERVAL);
879	add_timer(timer: &ap->fastdrain_timer);
880	}
881
882	out_unlock:
883	spin_unlock_irqrestore(lock: ap->lock, flags);
884	}
885
886	/**
887	* ata_eh_set_pending - set ATA_PFLAG_EH_PENDING and activate fast drain
888	* @ap: target ATA port
889	* @fastdrain: activate fast drain
890	*
891	* Set ATA_PFLAG_EH_PENDING and activate fast drain if @fastdrain
892	* is non-zero and EH wasn't pending before. Fast drain ensures
893	* that EH kicks in in timely manner.
894	*
895	* LOCKING:
896	* spin_lock_irqsave(host lock)
897	*/
898	static void ata_eh_set_pending(struct ata_port *ap, int fastdrain)
899	{
900	unsigned int cnt;
901
902	/* already scheduled? */
903	if (ap->pflags & ATA_PFLAG_EH_PENDING)
904	return;
905
906	ap->pflags |= ATA_PFLAG_EH_PENDING;
907
908	if (!fastdrain)
909	return;
910
911	/* do we have in-flight qcs? */
912	cnt = ata_eh_nr_in_flight(ap);
913	if (!cnt)
914	return;
915
916	/* activate fast drain */
917	ap->fastdrain_cnt = cnt;
918	ap->fastdrain_timer.expires =
919	ata_deadline(from_jiffies: jiffies, timeout_msecs: ATA_EH_FASTDRAIN_INTERVAL);
920	add_timer(timer: &ap->fastdrain_timer);
921	}
922
923	/**
924	* ata_qc_schedule_eh - schedule qc for error handling
925	* @qc: command to schedule error handling for
926	*
927	* Schedule error handling for @qc. EH will kick in as soon as
928	* other commands are drained.
929	*
930	* LOCKING:
931	* spin_lock_irqsave(host lock)
932	*/
933	void ata_qc_schedule_eh(struct ata_queued_cmd *qc)
934	{
935	struct ata_port *ap = qc->ap;
936
937	qc->flags |= ATA_QCFLAG_EH;
938	ata_eh_set_pending(ap, fastdrain: 1);
939
940	/* The following will fail if timeout has already expired.
941	* ata_scsi_error() takes care of such scmds on EH entry.
942	* Note that ATA_QCFLAG_EH is unconditionally set after
943	* this function completes.
944	*/
945	blk_abort_request(scsi_cmd_to_rq(scmd: qc->scsicmd));
946	}
947
948	/**
949	* ata_std_sched_eh - non-libsas ata_ports issue eh with this common routine
950	* @ap: ATA port to schedule EH for
951	*
952	* LOCKING: inherited from ata_port_schedule_eh
953	* spin_lock_irqsave(host lock)
954	*/
955	void ata_std_sched_eh(struct ata_port *ap)
956	{
957	if (ap->pflags & ATA_PFLAG_INITIALIZING)
958	return;
959
960	ata_eh_set_pending(ap, fastdrain: 1);
961	scsi_schedule_eh(shost: ap->scsi_host);
962
963	trace_ata_std_sched_eh(ap);
964	}
965	EXPORT_SYMBOL_GPL(ata_std_sched_eh);
966
967	/**
968	* ata_std_end_eh - non-libsas ata_ports complete eh with this common routine
969	* @ap: ATA port to end EH for
970	*
971	* In the libata object model there is a 1:1 mapping of ata_port to
972	* shost, so host fields can be directly manipulated under ap->lock, in
973	* the libsas case we need to hold a lock at the ha->level to coordinate
974	* these events.
975	*
976	* LOCKING:
977	* spin_lock_irqsave(host lock)
978	*/
979	void ata_std_end_eh(struct ata_port *ap)
980	{
981	struct Scsi_Host *host = ap->scsi_host;
982
983	host->host_eh_scheduled = 0;
984	}
985	EXPORT_SYMBOL(ata_std_end_eh);
986
987
988	/**
989	* ata_port_schedule_eh - schedule error handling without a qc
990	* @ap: ATA port to schedule EH for
991	*
992	* Schedule error handling for @ap. EH will kick in as soon as
993	* all commands are drained.
994	*
995	* LOCKING:
996	* spin_lock_irqsave(host lock)
997	*/
998	void ata_port_schedule_eh(struct ata_port *ap)
999	{
1000	/* see: ata_std_sched_eh, unless you know better */
1001	ap->ops->sched_eh(ap);
1002	}
1003	EXPORT_SYMBOL_GPL(ata_port_schedule_eh);
1004
1005	static int ata_do_link_abort(struct ata_port *ap, struct ata_link *link)
1006	{
1007	struct ata_queued_cmd *qc;
1008	int tag, nr_aborted = 0;
1009
1010	/* we're gonna abort all commands, no need for fast drain */
1011	ata_eh_set_pending(ap, fastdrain: 0);
1012
1013	/* include internal tag in iteration */
1014	ata_qc_for_each_with_internal(ap, qc, tag) {
1015	if (qc && (!link || qc->dev->link == link)) {
1016	qc->flags |= ATA_QCFLAG_EH;
1017	ata_qc_complete(qc);
1018	nr_aborted++;
1019	}
1020	}
1021
1022	if (!nr_aborted)
1023	ata_port_schedule_eh(ap);
1024
1025	return nr_aborted;
1026	}
1027
1028	/**
1029	* ata_link_abort - abort all qc's on the link
1030	* @link: ATA link to abort qc's for
1031	*
1032	* Abort all active qc's active on @link and schedule EH.
1033	*
1034	* LOCKING:
1035	* spin_lock_irqsave(host lock)
1036	*
1037	* RETURNS:
1038	* Number of aborted qc's.
1039	*/
1040	int ata_link_abort(struct ata_link *link)
1041	{
1042	return ata_do_link_abort(ap: link->ap, link);
1043	}
1044	EXPORT_SYMBOL_GPL(ata_link_abort);
1045
1046	/**
1047	* ata_port_abort - abort all qc's on the port
1048	* @ap: ATA port to abort qc's for
1049	*
1050	* Abort all active qc's of @ap and schedule EH.
1051	*
1052	* LOCKING:
1053	* spin_lock_irqsave(host_set lock)
1054	*
1055	* RETURNS:
1056	* Number of aborted qc's.
1057	*/
1058	int ata_port_abort(struct ata_port *ap)
1059	{
1060	return ata_do_link_abort(ap, NULL);
1061	}
1062	EXPORT_SYMBOL_GPL(ata_port_abort);
1063
1064	/**
1065	* __ata_port_freeze - freeze port
1066	* @ap: ATA port to freeze
1067	*
1068	* This function is called when HSM violation or some other
1069	* condition disrupts normal operation of the port. Frozen port
1070	* is not allowed to perform any operation until the port is
1071	* thawed, which usually follows a successful reset.
1072	*
1073	* ap->ops->freeze() callback can be used for freezing the port
1074	* hardware-wise (e.g. mask interrupt and stop DMA engine). If a
1075	* port cannot be frozen hardware-wise, the interrupt handler
1076	* must ack and clear interrupts unconditionally while the port
1077	* is frozen.
1078	*
1079	* LOCKING:
1080	* spin_lock_irqsave(host lock)
1081	*/
1082	static void __ata_port_freeze(struct ata_port *ap)
1083	{
1084	if (ap->ops->freeze)
1085	ap->ops->freeze(ap);
1086
1087	ap->pflags |= ATA_PFLAG_FROZEN;
1088
1089	trace_ata_port_freeze(ap);
1090	}
1091
1092	/**
1093	* ata_port_freeze - abort & freeze port
1094	* @ap: ATA port to freeze
1095	*
1096	* Abort and freeze @ap. The freeze operation must be called
1097	* first, because some hardware requires special operations
1098	* before the taskfile registers are accessible.
1099	*
1100	* LOCKING:
1101	* spin_lock_irqsave(host lock)
1102	*
1103	* RETURNS:
1104	* Number of aborted commands.
1105	*/
1106	int ata_port_freeze(struct ata_port *ap)
1107	{
1108	__ata_port_freeze(ap);
1109
1110	return ata_port_abort(ap);
1111	}
1112	EXPORT_SYMBOL_GPL(ata_port_freeze);
1113
1114	/**
1115	* ata_eh_freeze_port - EH helper to freeze port
1116	* @ap: ATA port to freeze
1117	*
1118	* Freeze @ap.
1119	*
1120	* LOCKING:
1121	* None.
1122	*/
1123	void ata_eh_freeze_port(struct ata_port *ap)
1124	{
1125	unsigned long flags;
1126
1127	spin_lock_irqsave(ap->lock, flags);
1128	__ata_port_freeze(ap);
1129	spin_unlock_irqrestore(lock: ap->lock, flags);
1130	}
1131	EXPORT_SYMBOL_GPL(ata_eh_freeze_port);
1132
1133	/**
1134	* ata_eh_thaw_port - EH helper to thaw port
1135	* @ap: ATA port to thaw
1136	*
1137	* Thaw frozen port @ap.
1138	*
1139	* LOCKING:
1140	* None.
1141	*/
1142	void ata_eh_thaw_port(struct ata_port *ap)
1143	{
1144	unsigned long flags;
1145
1146	spin_lock_irqsave(ap->lock, flags);
1147
1148	ap->pflags &= ~ATA_PFLAG_FROZEN;
1149
1150	if (ap->ops->thaw)
1151	ap->ops->thaw(ap);
1152
1153	spin_unlock_irqrestore(lock: ap->lock, flags);
1154
1155	trace_ata_port_thaw(ap);
1156	}
1157
1158	static void ata_eh_scsidone(struct scsi_cmnd *scmd)
1159	{
1160	/* nada */
1161	}
1162
1163	static void __ata_eh_qc_complete(struct ata_queued_cmd *qc)
1164	{
1165	struct ata_port *ap = qc->ap;
1166	struct scsi_cmnd *scmd = qc->scsicmd;
1167	unsigned long flags;
1168
1169	spin_lock_irqsave(ap->lock, flags);
1170	qc->scsidone = ata_eh_scsidone;
1171	__ata_qc_complete(qc);
1172	WARN_ON(ata_tag_valid(qc->tag));
1173	spin_unlock_irqrestore(lock: ap->lock, flags);
1174
1175	scsi_eh_finish_cmd(scmd, done_q: &ap->eh_done_q);
1176	}
1177
1178	/**
1179	* ata_eh_qc_complete - Complete an active ATA command from EH
1180	* @qc: Command to complete
1181	*
1182	* Indicate to the mid and upper layers that an ATA command has
1183	* completed. To be used from EH.
1184	*/
1185	void ata_eh_qc_complete(struct ata_queued_cmd *qc)
1186	{
1187	struct scsi_cmnd *scmd = qc->scsicmd;
1188	scmd->retries = scmd->allowed;
1189	__ata_eh_qc_complete(qc);
1190	}
1191
1192	/**
1193	* ata_eh_qc_retry - Tell midlayer to retry an ATA command after EH
1194	* @qc: Command to retry
1195	*
1196	* Indicate to the mid and upper layers that an ATA command
1197	* should be retried. To be used from EH.
1198	*
1199	* SCSI midlayer limits the number of retries to scmd->allowed.
1200	* scmd->allowed is incremented for commands which get retried
1201	* due to unrelated failures (qc->err_mask is zero).
1202	*/
1203	void ata_eh_qc_retry(struct ata_queued_cmd *qc)
1204	{
1205	struct scsi_cmnd *scmd = qc->scsicmd;
1206	if (!qc->err_mask)
1207	scmd->allowed++;
1208	__ata_eh_qc_complete(qc);
1209	}
1210
1211	/**
1212	* ata_dev_disable - disable ATA device
1213	* @dev: ATA device to disable
1214	*
1215	* Disable @dev.
1216	*
1217	* Locking:
1218	* EH context.
1219	*/
1220	void ata_dev_disable(struct ata_device *dev)
1221	{
1222	if (!ata_dev_enabled(dev))
1223	return;
1224
1225	ata_dev_warn(dev, "disable device\n");
1226
1227	ata_eh_dev_disable(dev);
1228	}
1229	EXPORT_SYMBOL_GPL(ata_dev_disable);
1230
1231	/**
1232	* ata_eh_detach_dev - detach ATA device
1233	* @dev: ATA device to detach
1234	*
1235	* Detach @dev.
1236	*
1237	* LOCKING:
1238	* None.
1239	*/
1240	void ata_eh_detach_dev(struct ata_device *dev)
1241	{
1242	struct ata_link *link = dev->link;
1243	struct ata_port *ap = link->ap;
1244	struct ata_eh_context *ehc = &link->eh_context;
1245	unsigned long flags;
1246
1247	/*
1248	* If the device is still enabled, transition it to standby power mode
1249	* (i.e. spin down HDDs) and disable it.
1250	*/
1251	if (ata_dev_enabled(dev)) {
1252	ata_dev_power_set_standby(dev);
1253	ata_eh_dev_disable(dev);
1254	}
1255
1256	spin_lock_irqsave(ap->lock, flags);
1257
1258	dev->flags &= ~ATA_DFLAG_DETACH;
1259
1260	if (ata_scsi_offline_dev(dev)) {
1261	dev->flags |= ATA_DFLAG_DETACHED;
1262	ap->pflags |= ATA_PFLAG_SCSI_HOTPLUG;
1263	}
1264
1265	/* clear per-dev EH info */
1266	ata_eh_clear_action(link, dev, ehi: &link->eh_info, action: ATA_EH_PERDEV_MASK);
1267	ata_eh_clear_action(link, dev, ehi: &link->eh_context.i, action: ATA_EH_PERDEV_MASK);
1268	ehc->saved_xfer_mode[dev->devno] = 0;
1269	ehc->saved_ncq_enabled &= ~(1 << dev->devno);
1270
1271	spin_unlock_irqrestore(lock: ap->lock, flags);
1272	}
1273
1274	/**
1275	* ata_eh_about_to_do - about to perform eh_action
1276	* @link: target ATA link
1277	* @dev: target ATA dev for per-dev action (can be NULL)
1278	* @action: action about to be performed
1279	*
1280	* Called just before performing EH actions to clear related bits
1281	* in @link->eh_info such that eh actions are not unnecessarily
1282	* repeated.
1283	*
1284	* LOCKING:
1285	* None.
1286	*/
1287	void ata_eh_about_to_do(struct ata_link *link, struct ata_device *dev,
1288	unsigned int action)
1289	{
1290	struct ata_port *ap = link->ap;
1291	struct ata_eh_info *ehi = &link->eh_info;
1292	struct ata_eh_context *ehc = &link->eh_context;
1293	unsigned long flags;
1294
1295	trace_ata_eh_about_to_do(link, devno: dev ? dev->devno : 0, eh_action: action);
1296
1297	spin_lock_irqsave(ap->lock, flags);
1298
1299	ata_eh_clear_action(link, dev, ehi, action);
1300
1301	/* About to take EH action, set RECOVERED. Ignore actions on
1302	* slave links as master will do them again.
1303	*/
1304	if (!(ehc->i.flags & ATA_EHI_QUIET) && link != ap->slave_link)
1305	ap->pflags |= ATA_PFLAG_RECOVERED;
1306
1307	spin_unlock_irqrestore(lock: ap->lock, flags);
1308	}
1309
1310	/**
1311	* ata_eh_done - EH action complete
1312	* @link: ATA link for which EH actions are complete
1313	* @dev: target ATA dev for per-dev action (can be NULL)
1314	* @action: action just completed
1315	*
1316	* Called right after performing EH actions to clear related bits
1317	* in @link->eh_context.
1318	*
1319	* LOCKING:
1320	* None.
1321	*/
1322	void ata_eh_done(struct ata_link *link, struct ata_device *dev,
1323	unsigned int action)
1324	{
1325	struct ata_eh_context *ehc = &link->eh_context;
1326
1327	trace_ata_eh_done(link, devno: dev ? dev->devno : 0, eh_action: action);
1328
1329	ata_eh_clear_action(link, dev, ehi: &ehc->i, action);
1330	}
1331
1332	/**
1333	* ata_err_string - convert err_mask to descriptive string
1334	* @err_mask: error mask to convert to string
1335	*
1336	* Convert @err_mask to descriptive string. Errors are
1337	* prioritized according to severity and only the most severe
1338	* error is reported.
1339	*
1340	* LOCKING:
1341	* None.
1342	*
1343	* RETURNS:
1344	* Descriptive string for @err_mask
1345	*/
1346	static const char *ata_err_string(unsigned int err_mask)
1347	{
1348	if (err_mask & AC_ERR_HOST_BUS)
1349	return "host bus error";
1350	if (err_mask & AC_ERR_ATA_BUS)
1351	return "ATA bus error";
1352	if (err_mask & AC_ERR_TIMEOUT)
1353	return "timeout";
1354	if (err_mask & AC_ERR_HSM)
1355	return "HSM violation";
1356	if (err_mask & AC_ERR_SYSTEM)
1357	return "internal error";
1358	if (err_mask & AC_ERR_MEDIA)
1359	return "media error";
1360	if (err_mask & AC_ERR_INVALID)
1361	return "invalid argument";
1362	if (err_mask & AC_ERR_DEV)
1363	return "device error";
1364	if (err_mask & AC_ERR_NCQ)
1365	return "NCQ error";
1366	if (err_mask & AC_ERR_NODEV_HINT)
1367	return "Polling detection error";
1368	return "unknown error";
1369	}
1370
1371	/**
1372	* atapi_eh_tur - perform ATAPI TEST_UNIT_READY
1373	* @dev: target ATAPI device
1374	* @r_sense_key: out parameter for sense_key
1375	*
1376	* Perform ATAPI TEST_UNIT_READY.
1377	*
1378	* LOCKING:
1379	* EH context (may sleep).
1380	*
1381	* RETURNS:
1382	* 0 on success, AC_ERR_* mask on failure.
1383	*/
1384	unsigned int atapi_eh_tur(struct ata_device *dev, u8 *r_sense_key)
1385	{
1386	u8 cdb[ATAPI_CDB_LEN] = { TEST_UNIT_READY, 0, 0, 0, 0, 0 };
1387	struct ata_taskfile tf;
1388	unsigned int err_mask;
1389
1390	ata_tf_init(dev, tf: &tf);
1391
1392	tf.flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_DEVICE;
1393	tf.command = ATA_CMD_PACKET;
1394	tf.protocol = ATAPI_PROT_NODATA;
1395
1396	err_mask = ata_exec_internal(dev, tf: &tf, cdb, dma_dir: DMA_NONE, NULL, buflen: 0, timeout: 0);
1397	if (err_mask == AC_ERR_DEV)
1398	*r_sense_key = tf.error >> 4;
1399	return err_mask;
1400	}
1401
1402	/**
1403	* ata_eh_request_sense - perform REQUEST_SENSE_DATA_EXT
1404	* @qc: qc to perform REQUEST_SENSE_SENSE_DATA_EXT to
1405	*
1406	* Perform REQUEST_SENSE_DATA_EXT after the device reported CHECK
1407	* SENSE. This function is an EH helper.
1408	*
1409	* LOCKING:
1410	* Kernel thread context (may sleep).
1411	*
1412	* RETURNS:
1413	* true if sense data could be fetched, false otherwise.
1414	*/
1415	static bool ata_eh_request_sense(struct ata_queued_cmd *qc)
1416	{
1417	struct scsi_cmnd *cmd = qc->scsicmd;
1418	struct ata_device *dev = qc->dev;
1419	struct ata_taskfile tf;
1420	unsigned int err_mask;
1421
1422	if (ata_port_is_frozen(ap: qc->ap)) {
1423	ata_dev_warn(dev, "sense data available but port frozen\n");
1424	return false;
1425	}
1426
1427	if (!ata_id_sense_reporting_enabled(id: dev->id)) {
1428	ata_dev_warn(qc->dev, "sense data reporting disabled\n");
1429	return false;
1430	}
1431
1432	ata_tf_init(dev, tf: &tf);
1433	tf.flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_DEVICE;
1434	tf.flags |= ATA_TFLAG_LBA | ATA_TFLAG_LBA48;
1435	tf.command = ATA_CMD_REQ_SENSE_DATA;
1436	tf.protocol = ATA_PROT_NODATA;
1437
1438	err_mask = ata_exec_internal(dev, tf: &tf, NULL, dma_dir: DMA_NONE, NULL, buflen: 0, timeout: 0);
1439	/* Ignore err_mask; ATA_ERR might be set */
1440	if (tf.status & ATA_SENSE) {
1441	if (ata_scsi_sense_is_valid(sk: tf.lbah, asc: tf.lbam, ascq: tf.lbal)) {
1442	/* Set sense without also setting scsicmd->result */
1443	scsi_build_sense_buffer(desc: dev->flags & ATA_DFLAG_D_SENSE,
1444	buf: cmd->sense_buffer, key: tf.lbah,
1445	asc: tf.lbam, ascq: tf.lbal);
1446	qc->flags |= ATA_QCFLAG_SENSE_VALID;
1447	return true;
1448	}
1449	} else {
1450	ata_dev_warn(dev, "request sense failed stat %02x emask %x\n",
1451	tf.status, err_mask);
1452	}
1453
1454	return false;
1455	}
1456
1457	/**
1458	* atapi_eh_request_sense - perform ATAPI REQUEST_SENSE
1459	* @dev: device to perform REQUEST_SENSE to
1460	* @sense_buf: result sense data buffer (SCSI_SENSE_BUFFERSIZE bytes long)
1461	* @dfl_sense_key: default sense key to use
1462	*
1463	* Perform ATAPI REQUEST_SENSE after the device reported CHECK
1464	* SENSE. This function is EH helper.
1465	*
1466	* LOCKING:
1467	* Kernel thread context (may sleep).
1468	*
1469	* RETURNS:
1470	* 0 on success, AC_ERR_* mask on failure
1471	*/
1472	unsigned int atapi_eh_request_sense(struct ata_device *dev,
1473	u8 *sense_buf, u8 dfl_sense_key)
1474	{
1475	u8 cdb[ATAPI_CDB_LEN] =
1476	{ REQUEST_SENSE, 0, 0, 0, SCSI_SENSE_BUFFERSIZE, 0 };
1477	struct ata_port *ap = dev->link->ap;
1478	struct ata_taskfile tf;
1479
1480	memset(sense_buf, 0, SCSI_SENSE_BUFFERSIZE);
1481
1482	/* initialize sense_buf with the error register,
1483	* for the case where they are -not- overwritten
1484	*/
1485	sense_buf[0] = 0x70;
1486	sense_buf[2] = dfl_sense_key;
1487
1488	/* some devices time out if garbage left in tf */
1489	ata_tf_init(dev, tf: &tf);
1490
1491	tf.flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_DEVICE;
1492	tf.command = ATA_CMD_PACKET;
1493
1494	/* is it pointless to prefer PIO for "safety reasons"? */
1495	if (ap->flags & ATA_FLAG_PIO_DMA) {
1496	tf.protocol = ATAPI_PROT_DMA;
1497	tf.feature |= ATAPI_PKT_DMA;
1498	} else {
1499	tf.protocol = ATAPI_PROT_PIO;
1500	tf.lbam = SCSI_SENSE_BUFFERSIZE;
1501	tf.lbah = 0;
1502	}
1503
1504	return ata_exec_internal(dev, tf: &tf, cdb, dma_dir: DMA_FROM_DEVICE,
1505	buf: sense_buf, SCSI_SENSE_BUFFERSIZE, timeout: 0);
1506	}
1507
1508	/**
1509	* ata_eh_analyze_serror - analyze SError for a failed port
1510	* @link: ATA link to analyze SError for
1511	*
1512	* Analyze SError if available and further determine cause of
1513	* failure.
1514	*
1515	* LOCKING:
1516	* None.
1517	*/
1518	static void ata_eh_analyze_serror(struct ata_link *link)
1519	{
1520	struct ata_eh_context *ehc = &link->eh_context;
1521	u32 serror = ehc->i.serror;
1522	unsigned int err_mask = 0, action = 0;
1523	u32 hotplug_mask;
1524
1525	if (serror & (SERR_PERSISTENT | SERR_DATA)) {
1526	err_mask |= AC_ERR_ATA_BUS;
1527	action |= ATA_EH_RESET;
1528	}
1529	if (serror & SERR_PROTOCOL) {
1530	err_mask |= AC_ERR_HSM;
1531	action |= ATA_EH_RESET;
1532	}
1533	if (serror & SERR_INTERNAL) {
1534	err_mask |= AC_ERR_SYSTEM;
1535	action |= ATA_EH_RESET;
1536	}
1537
1538	/* Determine whether a hotplug event has occurred. Both
1539	* SError.N/X are considered hotplug events for enabled or
1540	* host links. For disabled PMP links, only N bit is
1541	* considered as X bit is left at 1 for link plugging.
1542	*/
1543	if (link->lpm_policy > ATA_LPM_MAX_POWER)
1544	hotplug_mask = 0; /* hotplug doesn't work w/ LPM */
1545	else if (!(link->flags & ATA_LFLAG_DISABLED) || ata_is_host_link(link))
1546	hotplug_mask = SERR_PHYRDY_CHG | SERR_DEV_XCHG;
1547	else
1548	hotplug_mask = SERR_PHYRDY_CHG;
1549
1550	if (serror & hotplug_mask)
1551	ata_ehi_hotplugged(ehi: &ehc->i);
1552
1553	ehc->i.err_mask |= err_mask;
1554	ehc->i.action |= action;
1555	}
1556
1557	/**
1558	* ata_eh_analyze_tf - analyze taskfile of a failed qc
1559	* @qc: qc to analyze
1560	*
1561	* Analyze taskfile of @qc and further determine cause of
1562	* failure. This function also requests ATAPI sense data if
1563	* available.
1564	*
1565	* LOCKING:
1566	* Kernel thread context (may sleep).
1567	*
1568	* RETURNS:
1569	* Determined recovery action
1570	*/
1571	static unsigned int ata_eh_analyze_tf(struct ata_queued_cmd *qc)
1572	{
1573	const struct ata_taskfile *tf = &qc->result_tf;
1574	unsigned int tmp, action = 0;
1575	u8 stat = tf->status, err = tf->error;
1576
1577	if ((stat & (ATA_BUSY | ATA_DRQ | ATA_DRDY)) != ATA_DRDY) {
1578	qc->err_mask |= AC_ERR_HSM;
1579	return ATA_EH_RESET;
1580	}
1581
1582	if (stat & (ATA_ERR | ATA_DF)) {
1583	qc->err_mask |= AC_ERR_DEV;
1584	/*
1585	* Sense data reporting does not work if the
1586	* device fault bit is set.
1587	*/
1588	if (stat & ATA_DF)
1589	stat &= ~ATA_SENSE;
1590	} else {
1591	return 0;
1592	}
1593
1594	switch (qc->dev->class) {
1595	case ATA_DEV_ATA:
1596	case ATA_DEV_ZAC:
1597	/*
1598	* Fetch the sense data explicitly if:
1599	* -It was a non-NCQ command that failed, or
1600	* -It was a NCQ command that failed, but the sense data
1601	* was not included in the NCQ command error log
1602	* (i.e. NCQ autosense is not supported by the device).
1603	*/
1604	if (!(qc->flags & ATA_QCFLAG_SENSE_VALID) &&
1605	(stat & ATA_SENSE) && ata_eh_request_sense(qc))
1606	set_status_byte(cmd: qc->scsicmd, status: SAM_STAT_CHECK_CONDITION);
1607	if (err & ATA_ICRC)
1608	qc->err_mask |= AC_ERR_ATA_BUS;
1609	if (err & (ATA_UNC | ATA_AMNF))
1610	qc->err_mask |= AC_ERR_MEDIA;
1611	if (err & ATA_IDNF)
1612	qc->err_mask |= AC_ERR_INVALID;
1613	break;
1614
1615	case ATA_DEV_ATAPI:
1616	if (!ata_port_is_frozen(ap: qc->ap)) {
1617	tmp = atapi_eh_request_sense(dev: qc->dev,
1618	sense_buf: qc->scsicmd->sense_buffer,
1619	dfl_sense_key: qc->result_tf.error >> 4);
1620	if (!tmp)
1621	qc->flags |= ATA_QCFLAG_SENSE_VALID;
1622	else
1623	qc->err_mask |= tmp;
1624	}
1625	}
1626
1627	if (qc->flags & ATA_QCFLAG_SENSE_VALID) {
1628	enum scsi_disposition ret = scsi_check_sense(qc->scsicmd);
1629	/*
1630	* SUCCESS here means that the sense code could be
1631	* evaluated and should be passed to the upper layers
1632	* for correct evaluation.
1633	* FAILED means the sense code could not be interpreted
1634	* and the device would need to be reset.
1635	* NEEDS_RETRY and ADD_TO_MLQUEUE means that the
1636	* command would need to be retried.
1637	*/
1638	if (ret == NEEDS_RETRY || ret == ADD_TO_MLQUEUE) {
1639	qc->flags |= ATA_QCFLAG_RETRY;
1640	qc->err_mask |= AC_ERR_OTHER;
1641	} else if (ret != SUCCESS) {
1642	qc->err_mask |= AC_ERR_HSM;
1643	}
1644	}
1645	if (qc->err_mask & (AC_ERR_HSM | AC_ERR_TIMEOUT | AC_ERR_ATA_BUS))
1646	action |= ATA_EH_RESET;
1647
1648	return action;
1649	}
1650
1651	static int ata_eh_categorize_error(unsigned int eflags, unsigned int err_mask,
1652	int *xfer_ok)
1653	{
1654	int base = 0;
1655
1656	if (!(eflags & ATA_EFLAG_DUBIOUS_XFER))
1657	*xfer_ok = 1;
1658
1659	if (!*xfer_ok)
1660	base = ATA_ECAT_DUBIOUS_NONE;
1661
1662	if (err_mask & AC_ERR_ATA_BUS)
1663	return base + ATA_ECAT_ATA_BUS;
1664
1665	if (err_mask & AC_ERR_TIMEOUT)
1666	return base + ATA_ECAT_TOUT_HSM;
1667
1668	if (eflags & ATA_EFLAG_IS_IO) {
1669	if (err_mask & AC_ERR_HSM)
1670	return base + ATA_ECAT_TOUT_HSM;
1671	if ((err_mask &
1672	(AC_ERR_DEV|AC_ERR_MEDIA|AC_ERR_INVALID)) == AC_ERR_DEV)
1673	return base + ATA_ECAT_UNK_DEV;
1674	}
1675
1676	return 0;
1677	}
1678
1679	struct speed_down_verdict_arg {
1680	u64 since;
1681	int xfer_ok;
1682	int nr_errors[ATA_ECAT_NR];
1683	};
1684
1685	static int speed_down_verdict_cb(struct ata_ering_entry *ent, void *void_arg)
1686	{
1687	struct speed_down_verdict_arg *arg = void_arg;
1688	int cat;
1689
1690	if ((ent->eflags & ATA_EFLAG_OLD_ER) || (ent->timestamp < arg->since))
1691	return -1;
1692
1693	cat = ata_eh_categorize_error(eflags: ent->eflags, err_mask: ent->err_mask,
1694	xfer_ok: &arg->xfer_ok);
1695	arg->nr_errors[cat]++;
1696
1697	return 0;
1698	}
1699
1700	/**
1701	* ata_eh_speed_down_verdict - Determine speed down verdict
1702	* @dev: Device of interest
1703	*
1704	* This function examines error ring of @dev and determines
1705	* whether NCQ needs to be turned off, transfer speed should be
1706	* stepped down, or falling back to PIO is necessary.
1707	*
1708	* ECAT_ATA_BUS : ATA_BUS error for any command
1709	*
1710	* ECAT_TOUT_HSM : TIMEOUT for any command or HSM violation for
1711	* IO commands
1712	*
1713	* ECAT_UNK_DEV : Unknown DEV error for IO commands
1714	*
1715	* ECAT_DUBIOUS_* : Identical to above three but occurred while
1716	* data transfer hasn't been verified.
1717	*
1718	* Verdicts are
1719	*
1720	* NCQ_OFF : Turn off NCQ.
1721	*
1722	* SPEED_DOWN : Speed down transfer speed but don't fall back
1723	* to PIO.
1724	*
1725	* FALLBACK_TO_PIO : Fall back to PIO.
1726	*
1727	* Even if multiple verdicts are returned, only one action is
1728	* taken per error. An action triggered by non-DUBIOUS errors
1729	* clears ering, while one triggered by DUBIOUS_* errors doesn't.
1730	* This is to expedite speed down decisions right after device is
1731	* initially configured.
1732	*
1733	* The following are speed down rules. #1 and #2 deal with
1734	* DUBIOUS errors.
1735	*
1736	* 1. If more than one DUBIOUS_ATA_BUS or DUBIOUS_TOUT_HSM errors
1737	* occurred during last 5 mins, SPEED_DOWN and FALLBACK_TO_PIO.
1738	*
1739	* 2. If more than one DUBIOUS_TOUT_HSM or DUBIOUS_UNK_DEV errors
1740	* occurred during last 5 mins, NCQ_OFF.
1741	*
1742	* 3. If more than 8 ATA_BUS, TOUT_HSM or UNK_DEV errors
1743	* occurred during last 5 mins, FALLBACK_TO_PIO
1744	*
1745	* 4. If more than 3 TOUT_HSM or UNK_DEV errors occurred
1746	* during last 10 mins, NCQ_OFF.
1747	*
1748	* 5. If more than 3 ATA_BUS or TOUT_HSM errors, or more than 6
1749	* UNK_DEV errors occurred during last 10 mins, SPEED_DOWN.
1750	*
1751	* LOCKING:
1752	* Inherited from caller.
1753	*
1754	* RETURNS:
1755	* OR of ATA_EH_SPDN_* flags.
1756	*/
1757	static unsigned int ata_eh_speed_down_verdict(struct ata_device *dev)
1758	{
1759	const u64 j5mins = 5LLU * 60 * HZ, j10mins = 10LLU * 60 * HZ;
1760	u64 j64 = get_jiffies_64();
1761	struct speed_down_verdict_arg arg;
1762	unsigned int verdict = 0;
1763
1764	/* scan past 5 mins of error history */
1765	memset(&arg, 0, sizeof(arg));
1766	arg.since = j64 - min(j64, j5mins);
1767	ata_ering_map(ering: &dev->ering, map_fn: speed_down_verdict_cb, arg: &arg);
1768
1769	if (arg.nr_errors[ATA_ECAT_DUBIOUS_ATA_BUS] +
1770	arg.nr_errors[ATA_ECAT_DUBIOUS_TOUT_HSM] > 1)
1771	verdict |= ATA_EH_SPDN_SPEED_DOWN |
1772	ATA_EH_SPDN_FALLBACK_TO_PIO | ATA_EH_SPDN_KEEP_ERRORS;
1773
1774	if (arg.nr_errors[ATA_ECAT_DUBIOUS_TOUT_HSM] +
1775	arg.nr_errors[ATA_ECAT_DUBIOUS_UNK_DEV] > 1)
1776	verdict |= ATA_EH_SPDN_NCQ_OFF | ATA_EH_SPDN_KEEP_ERRORS;
1777
1778	if (arg.nr_errors[ATA_ECAT_ATA_BUS] +
1779	arg.nr_errors[ATA_ECAT_TOUT_HSM] +
1780	arg.nr_errors[ATA_ECAT_UNK_DEV] > 6)
1781	verdict |= ATA_EH_SPDN_FALLBACK_TO_PIO;
1782
1783	/* scan past 10 mins of error history */
1784	memset(&arg, 0, sizeof(arg));
1785	arg.since = j64 - min(j64, j10mins);
1786	ata_ering_map(ering: &dev->ering, map_fn: speed_down_verdict_cb, arg: &arg);
1787
1788	if (arg.nr_errors[ATA_ECAT_TOUT_HSM] +
1789	arg.nr_errors[ATA_ECAT_UNK_DEV] > 3)
1790	verdict |= ATA_EH_SPDN_NCQ_OFF;
1791
1792	if (arg.nr_errors[ATA_ECAT_ATA_BUS] +
1793	arg.nr_errors[ATA_ECAT_TOUT_HSM] > 3 ||
1794	arg.nr_errors[ATA_ECAT_UNK_DEV] > 6)
1795	verdict |= ATA_EH_SPDN_SPEED_DOWN;
1796
1797	return verdict;
1798	}
1799
1800	/**
1801	* ata_eh_speed_down - record error and speed down if necessary
1802	* @dev: Failed device
1803	* @eflags: mask of ATA_EFLAG_* flags
1804	* @err_mask: err_mask of the error
1805	*
1806	* Record error and examine error history to determine whether
1807	* adjusting transmission speed is necessary. It also sets
1808	* transmission limits appropriately if such adjustment is
1809	* necessary.
1810	*
1811	* LOCKING:
1812	* Kernel thread context (may sleep).
1813	*
1814	* RETURNS:
1815	* Determined recovery action.
1816	*/
1817	static unsigned int ata_eh_speed_down(struct ata_device *dev,
1818	unsigned int eflags, unsigned int err_mask)
1819	{
1820	struct ata_link *link = ata_dev_phys_link(dev);
1821	int xfer_ok = 0;
1822	unsigned int verdict;
1823	unsigned int action = 0;
1824
1825	/* don't bother if Cat-0 error */
1826	if (ata_eh_categorize_error(eflags, err_mask, xfer_ok: &xfer_ok) == 0)
1827	return 0;
1828
1829	/* record error and determine whether speed down is necessary */
1830	ata_ering_record(ering: &dev->ering, eflags, err_mask);
1831	verdict = ata_eh_speed_down_verdict(dev);
1832
1833	/* turn off NCQ? */
1834	if ((verdict & ATA_EH_SPDN_NCQ_OFF) && ata_ncq_enabled(dev)) {
1835	dev->flags |= ATA_DFLAG_NCQ_OFF;
1836	ata_dev_warn(dev, "NCQ disabled due to excessive errors\n");
1837	goto done;
1838	}
1839
1840	/* speed down? */
1841	if (verdict & ATA_EH_SPDN_SPEED_DOWN) {
1842	/* speed down SATA link speed if possible */
1843	if (sata_down_spd_limit(link, spd_limit: 0) == 0) {
1844	action |= ATA_EH_RESET;
1845	goto done;
1846	}
1847
1848	/* lower transfer mode */
1849	if (dev->spdn_cnt < 2) {
1850	static const int dma_dnxfer_sel[] =
1851	{ ATA_DNXFER_DMA, ATA_DNXFER_40C };
1852	static const int pio_dnxfer_sel[] =
1853	{ ATA_DNXFER_PIO, ATA_DNXFER_FORCE_PIO0 };
1854	int sel;
1855
1856	if (dev->xfer_shift != ATA_SHIFT_PIO)
1857	sel = dma_dnxfer_sel[dev->spdn_cnt];
1858	else
1859	sel = pio_dnxfer_sel[dev->spdn_cnt];
1860
1861	dev->spdn_cnt++;
1862
1863	if (ata_down_xfermask_limit(dev, sel) == 0) {
1864	action |= ATA_EH_RESET;
1865	goto done;
1866	}
1867	}
1868	}
1869
1870	/* Fall back to PIO? Slowing down to PIO is meaningless for
1871	* SATA ATA devices. Consider it only for PATA and SATAPI.
1872	*/
1873	if ((verdict & ATA_EH_SPDN_FALLBACK_TO_PIO) && (dev->spdn_cnt >= 2) &&
1874	(link->ap->cbl != ATA_CBL_SATA || dev->class == ATA_DEV_ATAPI) &&
1875	(dev->xfer_shift != ATA_SHIFT_PIO)) {
1876	if (ata_down_xfermask_limit(dev, sel: ATA_DNXFER_FORCE_PIO) == 0) {
1877	dev->spdn_cnt = 0;
1878	action |= ATA_EH_RESET;
1879	goto done;
1880	}
1881	}
1882
1883	return 0;
1884	done:
1885	/* device has been slowed down, blow error history */
1886	if (!(verdict & ATA_EH_SPDN_KEEP_ERRORS))
1887	ata_ering_clear(ering: &dev->ering);
1888	return action;
1889	}
1890
1891	/**
1892	* ata_eh_worth_retry - analyze error and decide whether to retry
1893	* @qc: qc to possibly retry
1894	*
1895	* Look at the cause of the error and decide if a retry
1896	* might be useful or not. We don't want to retry media errors
1897	* because the drive itself has probably already taken 10-30 seconds
1898	* doing its own internal retries before reporting the failure.
1899	*/
1900	static inline int ata_eh_worth_retry(struct ata_queued_cmd *qc)
1901	{
1902	if (qc->err_mask & AC_ERR_MEDIA)
1903	return 0; /* don't retry media errors */
1904	if (qc->flags & ATA_QCFLAG_IO)
1905	return 1; /* otherwise retry anything from fs stack */
1906	if (qc->err_mask & AC_ERR_INVALID)
1907	return 0; /* don't retry these */
1908	return qc->err_mask != AC_ERR_DEV; /* retry if not dev error */
1909	}
1910
1911	/**
1912	* ata_eh_quiet - check if we need to be quiet about a command error
1913	* @qc: qc to check
1914	*
1915	* Look at the qc flags anbd its scsi command request flags to determine
1916	* if we need to be quiet about the command failure.
1917	*/
1918	static inline bool ata_eh_quiet(struct ata_queued_cmd *qc)
1919	{
1920	if (qc->scsicmd && scsi_cmd_to_rq(scmd: qc->scsicmd)->rq_flags & RQF_QUIET)
1921	qc->flags |= ATA_QCFLAG_QUIET;
1922	return qc->flags & ATA_QCFLAG_QUIET;
1923	}
1924
1925	static int ata_eh_read_sense_success_non_ncq(struct ata_link *link)
1926	{
1927	struct ata_port *ap = link->ap;
1928	struct ata_queued_cmd *qc;
1929
1930	qc = __ata_qc_from_tag(ap, tag: link->active_tag);
1931	if (!qc)
1932	return -EIO;
1933
1934	if (!(qc->flags & ATA_QCFLAG_EH) ||
1935	!(qc->flags & ATA_QCFLAG_EH_SUCCESS_CMD) ||
1936	qc->err_mask)
1937	return -EIO;
1938
1939	if (!ata_eh_request_sense(qc))
1940	return -EIO;
1941
1942	/*
1943	* If we have sense data, call scsi_check_sense() in order to set the
1944	* correct SCSI ML byte (if any). No point in checking the return value,
1945	* since the command has already completed successfully.
1946	*/
1947	scsi_check_sense(qc->scsicmd);
1948
1949	return 0;
1950	}
1951
1952	static void ata_eh_get_success_sense(struct ata_link *link)
1953	{
1954	struct ata_eh_context *ehc = &link->eh_context;
1955	struct ata_device *dev = link->device;
1956	struct ata_port *ap = link->ap;
1957	struct ata_queued_cmd *qc;
1958	int tag, ret = 0;
1959
1960	if (!(ehc->i.dev_action[dev->devno] & ATA_EH_GET_SUCCESS_SENSE))
1961	return;
1962
1963	/* if frozen, we can't do much */
1964	if (ata_port_is_frozen(ap)) {
1965	ata_dev_warn(dev,
1966	"successful sense data available but port frozen\n");
1967	goto out;
1968	}
1969
1970	/*
1971	* If the link has sactive set, then we have outstanding NCQ commands
1972	* and have to read the Successful NCQ Commands log to get the sense
1973	* data. Otherwise, we are dealing with a non-NCQ command and use
1974	* request sense ext command to retrieve the sense data.
1975	*/
1976	if (link->sactive)
1977	ret = ata_eh_read_sense_success_ncq_log(link);
1978	else
1979	ret = ata_eh_read_sense_success_non_ncq(link);
1980	if (ret)
1981	goto out;
1982
1983	ata_eh_done(link, dev, action: ATA_EH_GET_SUCCESS_SENSE);
1984	return;
1985
1986	out:
1987	/*
1988	* If we failed to get sense data for a successful command that ought to
1989	* have sense data, we cannot simply return BLK_STS_OK to user space.
1990	* This is because we can't know if the sense data that we couldn't get
1991	* was actually "DATA CURRENTLY UNAVAILABLE". Reporting such a command
1992	* as success to user space would result in a silent data corruption.
1993	* Thus, add a bogus ABORTED_COMMAND sense data to such commands, such
1994	* that SCSI will report these commands as BLK_STS_IOERR to user space.
1995	*/
1996	ata_qc_for_each_raw(ap, qc, tag) {
1997	if (!(qc->flags & ATA_QCFLAG_EH) ||
1998	!(qc->flags & ATA_QCFLAG_EH_SUCCESS_CMD) ||
1999	qc->err_mask ||
2000	ata_dev_phys_link(dev: qc->dev) != link)
2001	continue;
2002
2003	/* We managed to get sense for this success command, skip. */
2004	if (qc->flags & ATA_QCFLAG_SENSE_VALID)
2005	continue;
2006
2007	/* This success command did not have any sense data, skip. */
2008	if (!(qc->result_tf.status & ATA_SENSE))
2009	continue;
2010
2011	/* This success command had sense data, but we failed to get. */
2012	ata_scsi_set_sense(dev, cmd: qc->scsicmd, ABORTED_COMMAND, asc: 0, ascq: 0);
2013	qc->flags |= ATA_QCFLAG_SENSE_VALID;
2014	}
2015	ata_eh_done(link, dev, action: ATA_EH_GET_SUCCESS_SENSE);
2016	}
2017
2018	/**
2019	* ata_eh_link_autopsy - analyze error and determine recovery action
2020	* @link: host link to perform autopsy on
2021	*
2022	* Analyze why @link failed and determine which recovery actions
2023	* are needed. This function also sets more detailed AC_ERR_*
2024	* values and fills sense data for ATAPI CHECK SENSE.
2025	*
2026	* LOCKING:
2027	* Kernel thread context (may sleep).
2028	*/
2029	static void ata_eh_link_autopsy(struct ata_link *link)
2030	{
2031	struct ata_port *ap = link->ap;
2032	struct ata_eh_context *ehc = &link->eh_context;
2033	struct ata_queued_cmd *qc;
2034	struct ata_device *dev;
2035	unsigned int all_err_mask = 0, eflags = 0;
2036	int tag, nr_failed = 0, nr_quiet = 0;
2037	u32 serror;
2038	int rc;
2039
2040	if (ehc->i.flags & ATA_EHI_NO_AUTOPSY)
2041	return;
2042
2043	/* obtain and analyze SError */
2044	rc = sata_scr_read(link, reg: SCR_ERROR, val: &serror);
2045	if (rc == 0) {
2046	ehc->i.serror |= serror;
2047	ata_eh_analyze_serror(link);
2048	} else if (rc != -EOPNOTSUPP) {
2049	/* SError read failed, force reset and probing */
2050	ehc->i.probe_mask |= ATA_ALL_DEVICES;
2051	ehc->i.action |= ATA_EH_RESET;
2052	ehc->i.err_mask |= AC_ERR_OTHER;
2053	}
2054
2055	/* analyze NCQ failure */
2056	ata_eh_analyze_ncq_error(link);
2057
2058	/*
2059	* Check if this was a successful command that simply needs sense data.
2060	* Since the sense data is not part of the completion, we need to fetch
2061	* it using an additional command. Since this can't be done from irq
2062	* context, the sense data for successful commands are fetched by EH.
2063	*/
2064	ata_eh_get_success_sense(link);
2065
2066	/* any real error trumps AC_ERR_OTHER */
2067	if (ehc->i.err_mask & ~AC_ERR_OTHER)
2068	ehc->i.err_mask &= ~AC_ERR_OTHER;
2069
2070	all_err_mask |= ehc->i.err_mask;
2071
2072	ata_qc_for_each_raw(ap, qc, tag) {
2073	if (!(qc->flags & ATA_QCFLAG_EH) ||
2074	qc->flags & ATA_QCFLAG_RETRY ||
2075	qc->flags & ATA_QCFLAG_EH_SUCCESS_CMD ||
2076	ata_dev_phys_link(dev: qc->dev) != link)
2077	continue;
2078
2079	/* inherit upper level err_mask */
2080	qc->err_mask |= ehc->i.err_mask;
2081
2082	/* analyze TF */
2083	ehc->i.action |= ata_eh_analyze_tf(qc);
2084
2085	/* DEV errors are probably spurious in case of ATA_BUS error */
2086	if (qc->err_mask & AC_ERR_ATA_BUS)
2087	qc->err_mask &= ~(AC_ERR_DEV | AC_ERR_MEDIA |
2088	AC_ERR_INVALID);
2089
2090	/* any real error trumps unknown error */
2091	if (qc->err_mask & ~AC_ERR_OTHER)
2092	qc->err_mask &= ~AC_ERR_OTHER;
2093
2094	/*
2095	* SENSE_VALID trumps dev/unknown error and revalidation. Upper
2096	* layers will determine whether the command is worth retrying
2097	* based on the sense data and device class/type. Otherwise,
2098	* determine directly if the command is worth retrying using its
2099	* error mask and flags.
2100	*/
2101	if (qc->flags & ATA_QCFLAG_SENSE_VALID)
2102	qc->err_mask &= ~(AC_ERR_DEV | AC_ERR_OTHER);
2103	else if (ata_eh_worth_retry(qc))
2104	qc->flags |= ATA_QCFLAG_RETRY;
2105
2106	/* accumulate error info */
2107	ehc->i.dev = qc->dev;
2108	all_err_mask |= qc->err_mask;
2109	if (qc->flags & ATA_QCFLAG_IO)
2110	eflags |= ATA_EFLAG_IS_IO;
2111	trace_ata_eh_link_autopsy_qc(qc);
2112
2113	/* Count quiet errors */
2114	if (ata_eh_quiet(qc))
2115	nr_quiet++;
2116	nr_failed++;
2117	}
2118
2119	/* If all failed commands requested silence, then be quiet */
2120	if (nr_quiet == nr_failed)
2121	ehc->i.flags |= ATA_EHI_QUIET;
2122
2123	/* enforce default EH actions */
2124	if (ata_port_is_frozen(ap) ||
2125	all_err_mask & (AC_ERR_HSM | AC_ERR_TIMEOUT))
2126	ehc->i.action |= ATA_EH_RESET;
2127	else if (((eflags & ATA_EFLAG_IS_IO) && all_err_mask) ||
2128	(!(eflags & ATA_EFLAG_IS_IO) && (all_err_mask & ~AC_ERR_DEV)))
2129	ehc->i.action |= ATA_EH_REVALIDATE;
2130
2131	/* If we have offending qcs and the associated failed device,
2132	* perform per-dev EH action only on the offending device.
2133	*/
2134	if (ehc->i.dev) {
2135	ehc->i.dev_action[ehc->i.dev->devno] |=
2136	ehc->i.action & ATA_EH_PERDEV_MASK;
2137	ehc->i.action &= ~ATA_EH_PERDEV_MASK;
2138	}
2139
2140	/* propagate timeout to host link */
2141	if ((all_err_mask & AC_ERR_TIMEOUT) && !ata_is_host_link(link))
2142	ap->link.eh_context.i.err_mask |= AC_ERR_TIMEOUT;
2143
2144	/* record error and consider speeding down */
2145	dev = ehc->i.dev;
2146	if (!dev && ((ata_link_max_devices(link) == 1 &&
2147	ata_dev_enabled(dev: link->device))))
2148	dev = link->device;
2149
2150	if (dev) {
2151	if (dev->flags & ATA_DFLAG_DUBIOUS_XFER)
2152	eflags |= ATA_EFLAG_DUBIOUS_XFER;
2153	ehc->i.action |= ata_eh_speed_down(dev, eflags, err_mask: all_err_mask);
2154	trace_ata_eh_link_autopsy(dev, eh_action: ehc->i.action, eh_err_mask: all_err_mask);
2155	}
2156	}
2157
2158	/**
2159	* ata_eh_autopsy - analyze error and determine recovery action
2160	* @ap: host port to perform autopsy on
2161	*
2162	* Analyze all links of @ap and determine why they failed and
2163	* which recovery actions are needed.
2164	*
2165	* LOCKING:
2166	* Kernel thread context (may sleep).
2167	*/
2168	void ata_eh_autopsy(struct ata_port *ap)
2169	{
2170	struct ata_link *link;
2171
2172	ata_for_each_link(link, ap, EDGE)
2173	ata_eh_link_autopsy(link);
2174
2175	/* Handle the frigging slave link. Autopsy is done similarly
2176	* but actions and flags are transferred over to the master
2177	* link and handled from there.
2178	*/
2179	if (ap->slave_link) {
2180	struct ata_eh_context *mehc = &ap->link.eh_context;
2181	struct ata_eh_context *sehc = &ap->slave_link->eh_context;
2182
2183	/* transfer control flags from master to slave */
2184	sehc->i.flags |= mehc->i.flags & ATA_EHI_TO_SLAVE_MASK;
2185
2186	/* perform autopsy on the slave link */
2187	ata_eh_link_autopsy(link: ap->slave_link);
2188
2189	/* transfer actions from slave to master and clear slave */
2190	ata_eh_about_to_do(link: ap->slave_link, NULL, action: ATA_EH_ALL_ACTIONS);
2191	mehc->i.action |= sehc->i.action;
2192	mehc->i.dev_action[1] |= sehc->i.dev_action[1];
2193	mehc->i.flags |= sehc->i.flags;
2194	ata_eh_done(link: ap->slave_link, NULL, action: ATA_EH_ALL_ACTIONS);
2195	}
2196
2197	/* Autopsy of fanout ports can affect host link autopsy.
2198	* Perform host link autopsy last.
2199	*/
2200	if (sata_pmp_attached(ap))
2201	ata_eh_link_autopsy(link: &ap->link);
2202	}
2203
2204	/**
2205	* ata_get_cmd_name - get name for ATA command
2206	* @command: ATA command code to get name for
2207	*
2208	* Return a textual name of the given command or "unknown"
2209	*
2210	* LOCKING:
2211	* None
2212	*/
2213	const char *ata_get_cmd_name(u8 command)
2214	{
2215	#ifdef CONFIG_ATA_VERBOSE_ERROR
2216	static const struct
2217	{
2218	u8 command;
2219	const char *text;
2220	} cmd_descr[] = {
2221	{ ATA_CMD_DEV_RESET, "DEVICE RESET" },
2222	{ ATA_CMD_CHK_POWER, "CHECK POWER MODE" },
2223	{ ATA_CMD_STANDBY, "STANDBY" },
2224	{ ATA_CMD_IDLE, "IDLE" },
2225	{ ATA_CMD_EDD, "EXECUTE DEVICE DIAGNOSTIC" },
2226	{ ATA_CMD_DOWNLOAD_MICRO, "DOWNLOAD MICROCODE" },
2227	{ ATA_CMD_DOWNLOAD_MICRO_DMA, "DOWNLOAD MICROCODE DMA" },
2228	{ ATA_CMD_NOP, "NOP" },
2229	{ ATA_CMD_FLUSH, "FLUSH CACHE" },
2230	{ ATA_CMD_FLUSH_EXT, "FLUSH CACHE EXT" },
2231	{ ATA_CMD_ID_ATA, "IDENTIFY DEVICE" },
2232	{ ATA_CMD_ID_ATAPI, "IDENTIFY PACKET DEVICE" },
2233	{ ATA_CMD_SERVICE, "SERVICE" },
2234	{ ATA_CMD_READ, "READ DMA" },
2235	{ ATA_CMD_READ_EXT, "READ DMA EXT" },
2236	{ ATA_CMD_READ_QUEUED, "READ DMA QUEUED" },
2237	{ ATA_CMD_READ_STREAM_EXT, "READ STREAM EXT" },
2238	{ ATA_CMD_READ_STREAM_DMA_EXT, "READ STREAM DMA EXT" },
2239	{ ATA_CMD_WRITE, "WRITE DMA" },
2240	{ ATA_CMD_WRITE_EXT, "WRITE DMA EXT" },
2241	{ ATA_CMD_WRITE_QUEUED, "WRITE DMA QUEUED EXT" },
2242	{ ATA_CMD_WRITE_STREAM_EXT, "WRITE STREAM EXT" },
2243	{ ATA_CMD_WRITE_STREAM_DMA_EXT, "WRITE STREAM DMA EXT" },
2244	{ ATA_CMD_WRITE_FUA_EXT, "WRITE DMA FUA EXT" },
2245	{ ATA_CMD_WRITE_QUEUED_FUA_EXT, "WRITE DMA QUEUED FUA EXT" },
2246	{ ATA_CMD_FPDMA_READ, "READ FPDMA QUEUED" },
2247	{ ATA_CMD_FPDMA_WRITE, "WRITE FPDMA QUEUED" },
2248	{ ATA_CMD_NCQ_NON_DATA, "NCQ NON-DATA" },
2249	{ ATA_CMD_FPDMA_SEND, "SEND FPDMA QUEUED" },
2250	{ ATA_CMD_FPDMA_RECV, "RECEIVE FPDMA QUEUED" },
2251	{ ATA_CMD_PIO_READ, "READ SECTOR(S)" },
2252	{ ATA_CMD_PIO_READ_EXT, "READ SECTOR(S) EXT" },
2253	{ ATA_CMD_PIO_WRITE, "WRITE SECTOR(S)" },
2254	{ ATA_CMD_PIO_WRITE_EXT, "WRITE SECTOR(S) EXT" },
2255	{ ATA_CMD_READ_MULTI, "READ MULTIPLE" },
2256	{ ATA_CMD_READ_MULTI_EXT, "READ MULTIPLE EXT" },
2257	{ ATA_CMD_WRITE_MULTI, "WRITE MULTIPLE" },
2258	{ ATA_CMD_WRITE_MULTI_EXT, "WRITE MULTIPLE EXT" },
2259	{ ATA_CMD_WRITE_MULTI_FUA_EXT, "WRITE MULTIPLE FUA EXT" },
2260	{ ATA_CMD_SET_FEATURES, "SET FEATURES" },
2261	{ ATA_CMD_SET_MULTI, "SET MULTIPLE MODE" },
2262	{ ATA_CMD_VERIFY, "READ VERIFY SECTOR(S)" },
2263	{ ATA_CMD_VERIFY_EXT, "READ VERIFY SECTOR(S) EXT" },
2264	{ ATA_CMD_WRITE_UNCORR_EXT, "WRITE UNCORRECTABLE EXT" },
2265	{ ATA_CMD_STANDBYNOW1, "STANDBY IMMEDIATE" },
2266	{ ATA_CMD_IDLEIMMEDIATE, "IDLE IMMEDIATE" },
2267	{ ATA_CMD_SLEEP, "SLEEP" },
2268	{ ATA_CMD_INIT_DEV_PARAMS, "INITIALIZE DEVICE PARAMETERS" },
2269	{ ATA_CMD_READ_NATIVE_MAX, "READ NATIVE MAX ADDRESS" },
2270	{ ATA_CMD_READ_NATIVE_MAX_EXT, "READ NATIVE MAX ADDRESS EXT" },
2271	{ ATA_CMD_SET_MAX, "SET MAX ADDRESS" },
2272	{ ATA_CMD_SET_MAX_EXT, "SET MAX ADDRESS EXT" },
2273	{ ATA_CMD_READ_LOG_EXT, "READ LOG EXT" },
2274	{ ATA_CMD_WRITE_LOG_EXT, "WRITE LOG EXT" },
2275	{ ATA_CMD_READ_LOG_DMA_EXT, "READ LOG DMA EXT" },
2276	{ ATA_CMD_WRITE_LOG_DMA_EXT, "WRITE LOG DMA EXT" },
2277	{ ATA_CMD_TRUSTED_NONDATA, "TRUSTED NON-DATA" },
2278	{ ATA_CMD_TRUSTED_RCV, "TRUSTED RECEIVE" },
2279	{ ATA_CMD_TRUSTED_RCV_DMA, "TRUSTED RECEIVE DMA" },
2280	{ ATA_CMD_TRUSTED_SND, "TRUSTED SEND" },
2281	{ ATA_CMD_TRUSTED_SND_DMA, "TRUSTED SEND DMA" },
2282	{ ATA_CMD_PMP_READ, "READ BUFFER" },
2283	{ ATA_CMD_PMP_READ_DMA, "READ BUFFER DMA" },
2284	{ ATA_CMD_PMP_WRITE, "WRITE BUFFER" },
2285	{ ATA_CMD_PMP_WRITE_DMA, "WRITE BUFFER DMA" },
2286	{ ATA_CMD_CONF_OVERLAY, "DEVICE CONFIGURATION OVERLAY" },
2287	{ ATA_CMD_SEC_SET_PASS, "SECURITY SET PASSWORD" },
2288	{ ATA_CMD_SEC_UNLOCK, "SECURITY UNLOCK" },
2289	{ ATA_CMD_SEC_ERASE_PREP, "SECURITY ERASE PREPARE" },
2290	{ ATA_CMD_SEC_ERASE_UNIT, "SECURITY ERASE UNIT" },
2291	{ ATA_CMD_SEC_FREEZE_LOCK, "SECURITY FREEZE LOCK" },
2292	{ ATA_CMD_SEC_DISABLE_PASS, "SECURITY DISABLE PASSWORD" },
2293	{ ATA_CMD_CONFIG_STREAM, "CONFIGURE STREAM" },
2294	{ ATA_CMD_SMART, "SMART" },
2295	{ ATA_CMD_MEDIA_LOCK, "DOOR LOCK" },
2296	{ ATA_CMD_MEDIA_UNLOCK, "DOOR UNLOCK" },
2297	{ ATA_CMD_DSM, "DATA SET MANAGEMENT" },
2298	{ ATA_CMD_CHK_MED_CRD_TYP, "CHECK MEDIA CARD TYPE" },
2299	{ ATA_CMD_CFA_REQ_EXT_ERR, "CFA REQUEST EXTENDED ERROR" },
2300	{ ATA_CMD_CFA_WRITE_NE, "CFA WRITE SECTORS WITHOUT ERASE" },
2301	{ ATA_CMD_CFA_TRANS_SECT, "CFA TRANSLATE SECTOR" },
2302	{ ATA_CMD_CFA_ERASE, "CFA ERASE SECTORS" },
2303	{ ATA_CMD_CFA_WRITE_MULT_NE, "CFA WRITE MULTIPLE WITHOUT ERASE" },
2304	{ ATA_CMD_REQ_SENSE_DATA, "REQUEST SENSE DATA EXT" },
2305	{ ATA_CMD_SANITIZE_DEVICE, "SANITIZE DEVICE" },
2306	{ ATA_CMD_ZAC_MGMT_IN, "ZAC MANAGEMENT IN" },
2307	{ ATA_CMD_ZAC_MGMT_OUT, "ZAC MANAGEMENT OUT" },
2308	{ ATA_CMD_READ_LONG, "READ LONG (with retries)" },
2309	{ ATA_CMD_READ_LONG_ONCE, "READ LONG (without retries)" },
2310	{ ATA_CMD_WRITE_LONG, "WRITE LONG (with retries)" },
2311	{ ATA_CMD_WRITE_LONG_ONCE, "WRITE LONG (without retries)" },
2312	{ ATA_CMD_RESTORE, "RECALIBRATE" },
2313	{ 0, NULL } /* terminate list */
2314	};
2315
2316	unsigned int i;
2317	for (i = 0; cmd_descr[i].text; i++)
2318	if (cmd_descr[i].command == command)
2319	return cmd_descr[i].text;
2320	#endif
2321
2322	return "unknown";
2323	}
2324	EXPORT_SYMBOL_GPL(ata_get_cmd_name);
2325
2326	/**
2327	* ata_eh_link_report - report error handling to user
2328	* @link: ATA link EH is going on
2329	*
2330	* Report EH to user.
2331	*
2332	* LOCKING:
2333	* None.
2334	*/
2335	static void ata_eh_link_report(struct ata_link *link)
2336	{
2337	struct ata_port *ap = link->ap;
2338	struct ata_eh_context *ehc = &link->eh_context;
2339	struct ata_queued_cmd *qc;
2340	const char *frozen, *desc;
2341	char tries_buf[16] = "";
2342	int tag, nr_failed = 0;
2343
2344	if (ehc->i.flags & ATA_EHI_QUIET)
2345	return;
2346
2347	desc = NULL;
2348	if (ehc->i.desc[0] != '\0')
2349	desc = ehc->i.desc;
2350
2351	ata_qc_for_each_raw(ap, qc, tag) {
2352	if (!(qc->flags & ATA_QCFLAG_EH) ||
2353	ata_dev_phys_link(dev: qc->dev) != link ||
2354	((qc->flags & ATA_QCFLAG_QUIET) &&
2355	qc->err_mask == AC_ERR_DEV))
2356	continue;
2357	if (qc->flags & ATA_QCFLAG_SENSE_VALID && !qc->err_mask)
2358	continue;
2359
2360	nr_failed++;
2361	}
2362
2363	if (!nr_failed && !ehc->i.err_mask)
2364	return;
2365
2366	frozen = "";
2367	if (ata_port_is_frozen(ap))
2368	frozen = " frozen";
2369
2370	if (ap->eh_tries < ATA_EH_MAX_TRIES)
2371	snprintf(buf: tries_buf, size: sizeof(tries_buf), fmt: " t%d",
2372	ap->eh_tries);
2373
2374	if (ehc->i.dev) {
2375	ata_dev_err(ehc->i.dev, "exception Emask 0x%x "
2376	"SAct 0x%x SErr 0x%x action 0x%x%s%s\n",
2377	ehc->i.err_mask, link->sactive, ehc->i.serror,
2378	ehc->i.action, frozen, tries_buf);
2379	if (desc)
2380	ata_dev_err(ehc->i.dev, "%s\n", desc);
2381	} else {
2382	ata_link_err(link, "exception Emask 0x%x "
2383	"SAct 0x%x SErr 0x%x action 0x%x%s%s\n",
2384	ehc->i.err_mask, link->sactive, ehc->i.serror,
2385	ehc->i.action, frozen, tries_buf);
2386	if (desc)
2387	ata_link_err(link, "%s\n", desc);
2388	}
2389
2390	#ifdef CONFIG_ATA_VERBOSE_ERROR
2391	if (ehc->i.serror)
2392	ata_link_err(link,
2393	"SError: { %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s}\n",
2394	ehc->i.serror & SERR_DATA_RECOVERED ? "RecovData " : "",
2395	ehc->i.serror & SERR_COMM_RECOVERED ? "RecovComm " : "",
2396	ehc->i.serror & SERR_DATA ? "UnrecovData " : "",
2397	ehc->i.serror & SERR_PERSISTENT ? "Persist " : "",
2398	ehc->i.serror & SERR_PROTOCOL ? "Proto " : "",
2399	ehc->i.serror & SERR_INTERNAL ? "HostInt " : "",
2400	ehc->i.serror & SERR_PHYRDY_CHG ? "PHYRdyChg " : "",
2401	ehc->i.serror & SERR_PHY_INT_ERR ? "PHYInt " : "",
2402	ehc->i.serror & SERR_COMM_WAKE ? "CommWake " : "",
2403	ehc->i.serror & SERR_10B_8B_ERR ? "10B8B " : "",
2404	ehc->i.serror & SERR_DISPARITY ? "Dispar " : "",
2405	ehc->i.serror & SERR_CRC ? "BadCRC " : "",
2406	ehc->i.serror & SERR_HANDSHAKE ? "Handshk " : "",
2407	ehc->i.serror & SERR_LINK_SEQ_ERR ? "LinkSeq " : "",
2408	ehc->i.serror & SERR_TRANS_ST_ERROR ? "TrStaTrns " : "",
2409	ehc->i.serror & SERR_UNRECOG_FIS ? "UnrecFIS " : "",
2410	ehc->i.serror & SERR_DEV_XCHG ? "DevExch " : "");
2411	#endif
2412
2413	ata_qc_for_each_raw(ap, qc, tag) {
2414	struct ata_taskfile *cmd = &qc->tf, *res = &qc->result_tf;
2415	char data_buf[20] = "";
2416	char cdb_buf[70] = "";
2417
2418	if (!(qc->flags & ATA_QCFLAG_EH) ||
2419	ata_dev_phys_link(dev: qc->dev) != link || !qc->err_mask)
2420	continue;
2421
2422	if (qc->dma_dir != DMA_NONE) {
2423	static const char *dma_str[] = {
2424	[DMA_BIDIRECTIONAL] = "bidi",
2425	[DMA_TO_DEVICE] = "out",
2426	[DMA_FROM_DEVICE] = "in",
2427	};
2428	const char *prot_str = NULL;
2429
2430	switch (qc->tf.protocol) {
2431	case ATA_PROT_UNKNOWN:
2432	prot_str = "unknown";
2433	break;
2434	case ATA_PROT_NODATA:
2435	prot_str = "nodata";
2436	break;
2437	case ATA_PROT_PIO:
2438	prot_str = "pio";
2439	break;
2440	case ATA_PROT_DMA:
2441	prot_str = "dma";
2442	break;
2443	case ATA_PROT_NCQ:
2444	prot_str = "ncq dma";
2445	break;
2446	case ATA_PROT_NCQ_NODATA:
2447	prot_str = "ncq nodata";
2448	break;
2449	case ATAPI_PROT_NODATA:
2450	prot_str = "nodata";
2451	break;
2452	case ATAPI_PROT_PIO:
2453	prot_str = "pio";
2454	break;
2455	case ATAPI_PROT_DMA:
2456	prot_str = "dma";
2457	break;
2458	}
2459	snprintf(buf: data_buf, size: sizeof(data_buf), fmt: " %s %u %s",
2460	prot_str, qc->nbytes, dma_str[qc->dma_dir]);
2461	}
2462
2463	if (ata_is_atapi(prot: qc->tf.protocol)) {
2464	const u8 *cdb = qc->cdb;
2465	size_t cdb_len = qc->dev->cdb_len;
2466
2467	if (qc->scsicmd) {
2468	cdb = qc->scsicmd->cmnd;
2469	cdb_len = qc->scsicmd->cmd_len;
2470	}
2471	__scsi_format_command(cdb_buf, sizeof(cdb_buf),
2472	cdb, cdb_len);
2473	} else
2474	ata_dev_err(qc->dev, "failed command: %s\n",
2475	ata_get_cmd_name(cmd->command));
2476
2477	ata_dev_err(qc->dev,
2478	"cmd %02x/%02x:%02x:%02x:%02x:%02x/%02x:%02x:%02x:%02x:%02x/%02x "
2479	"tag %d%s\n %s"
2480	"res %02x/%02x:%02x:%02x:%02x:%02x/%02x:%02x:%02x:%02x:%02x/%02x "
2481	"Emask 0x%x (%s)%s\n",
2482	cmd->command, cmd->feature, cmd->nsect,
2483	cmd->lbal, cmd->lbam, cmd->lbah,
2484	cmd->hob_feature, cmd->hob_nsect,
2485	cmd->hob_lbal, cmd->hob_lbam, cmd->hob_lbah,
2486	cmd->device, qc->tag, data_buf, cdb_buf,
2487	res->status, res->error, res->nsect,
2488	res->lbal, res->lbam, res->lbah,
2489	res->hob_feature, res->hob_nsect,
2490	res->hob_lbal, res->hob_lbam, res->hob_lbah,
2491	res->device, qc->err_mask, ata_err_string(qc->err_mask),
2492	qc->err_mask & AC_ERR_NCQ ? " <F>" : "");
2493
2494	#ifdef CONFIG_ATA_VERBOSE_ERROR
2495	if (res->status & (ATA_BUSY | ATA_DRDY | ATA_DF | ATA_DRQ |
2496	ATA_SENSE | ATA_ERR)) {
2497	if (res->status & ATA_BUSY)
2498	ata_dev_err(qc->dev, "status: { Busy }\n");
2499	else
2500	ata_dev_err(qc->dev, "status: { %s%s%s%s%s}\n",
2501	res->status & ATA_DRDY ? "DRDY " : "",
2502	res->status & ATA_DF ? "DF " : "",
2503	res->status & ATA_DRQ ? "DRQ " : "",
2504	res->status & ATA_SENSE ? "SENSE " : "",
2505	res->status & ATA_ERR ? "ERR " : "");
2506	}
2507
2508	if (cmd->command != ATA_CMD_PACKET &&
2509	(res->error & (ATA_ICRC | ATA_UNC | ATA_AMNF | ATA_IDNF |
2510	ATA_ABORTED)))
2511	ata_dev_err(qc->dev, "error: { %s%s%s%s%s}\n",
2512	res->error & ATA_ICRC ? "ICRC " : "",
2513	res->error & ATA_UNC ? "UNC " : "",
2514	res->error & ATA_AMNF ? "AMNF " : "",
2515	res->error & ATA_IDNF ? "IDNF " : "",
2516	res->error & ATA_ABORTED ? "ABRT " : "");
2517	#endif
2518	}
2519	}
2520
2521	/**
2522	* ata_eh_report - report error handling to user
2523	* @ap: ATA port to report EH about
2524	*
2525	* Report EH to user.
2526	*
2527	* LOCKING:
2528	* None.
2529	*/
2530	void ata_eh_report(struct ata_port *ap)
2531	{
2532	struct ata_link *link;
2533
2534	ata_for_each_link(link, ap, HOST_FIRST)
2535	ata_eh_link_report(link);
2536	}
2537
2538	static int ata_do_reset(struct ata_link *link, ata_reset_fn_t reset,
2539	unsigned int *classes, unsigned long deadline,
2540	bool clear_classes)
2541	{
2542	struct ata_device *dev;
2543
2544	if (clear_classes)
2545	ata_for_each_dev(dev, link, ALL)
2546	classes[dev->devno] = ATA_DEV_UNKNOWN;
2547
2548	return reset(link, classes, deadline);
2549	}
2550
2551	static int ata_eh_followup_srst_needed(struct ata_link *link, int rc)
2552	{
2553	if ((link->flags & ATA_LFLAG_NO_SRST) || ata_link_offline(link))
2554	return 0;
2555	if (rc == -EAGAIN)
2556	return 1;
2557	if (sata_pmp_supported(ap: link->ap) && ata_is_host_link(link))
2558	return 1;
2559	return 0;
2560	}
2561
2562	int ata_eh_reset(struct ata_link *link, int classify,
2563	ata_prereset_fn_t prereset, ata_reset_fn_t softreset,
2564	ata_reset_fn_t hardreset, ata_postreset_fn_t postreset)
2565	{
2566	struct ata_port *ap = link->ap;
2567	struct ata_link *slave = ap->slave_link;
2568	struct ata_eh_context *ehc = &link->eh_context;
2569	struct ata_eh_context *sehc = slave ? &slave->eh_context : NULL;
2570	unsigned int *classes = ehc->classes;
2571	unsigned int lflags = link->flags;
2572	int verbose = !(ehc->i.flags & ATA_EHI_QUIET);
2573	int max_tries = 0, try = 0;
2574	struct ata_link *failed_link;
2575	struct ata_device *dev;
2576	unsigned long deadline, now;
2577	ata_reset_fn_t reset;
2578	unsigned long flags;
2579	u32 sstatus;
2580	int nr_unknown, rc;
2581
2582	/*
2583	* Prepare to reset
2584	*/
2585	while (ata_eh_reset_timeouts[max_tries] != UINT_MAX)
2586	max_tries++;
2587	if (link->flags & ATA_LFLAG_RST_ONCE)
2588	max_tries = 1;
2589	if (link->flags & ATA_LFLAG_NO_HRST)
2590	hardreset = NULL;
2591	if (link->flags & ATA_LFLAG_NO_SRST)
2592	softreset = NULL;
2593
2594	/* make sure each reset attempt is at least COOL_DOWN apart */
2595	if (ehc->i.flags & ATA_EHI_DID_RESET) {
2596	now = jiffies;
2597	WARN_ON(time_after(ehc->last_reset, now));
2598	deadline = ata_deadline(from_jiffies: ehc->last_reset,
2599	timeout_msecs: ATA_EH_RESET_COOL_DOWN);
2600	if (time_before(now, deadline))
2601	schedule_timeout_uninterruptible(timeout: deadline - now);
2602	}
2603
2604	spin_lock_irqsave(ap->lock, flags);
2605	ap->pflags |= ATA_PFLAG_RESETTING;
2606	spin_unlock_irqrestore(lock: ap->lock, flags);
2607
2608	ata_eh_about_to_do(link, NULL, action: ATA_EH_RESET);
2609
2610	ata_for_each_dev(dev, link, ALL) {
2611	/* If we issue an SRST then an ATA drive (not ATAPI)
2612	* may change configuration and be in PIO0 timing. If
2613	* we do a hard reset (or are coming from power on)
2614	* this is true for ATA or ATAPI. Until we've set a
2615	* suitable controller mode we should not touch the
2616	* bus as we may be talking too fast.
2617	*/
2618	dev->pio_mode = XFER_PIO_0;
2619	dev->dma_mode = 0xff;
2620
2621	/* If the controller has a pio mode setup function
2622	* then use it to set the chipset to rights. Don't
2623	* touch the DMA setup as that will be dealt with when
2624	* configuring devices.
2625	*/
2626	if (ap->ops->set_piomode)
2627	ap->ops->set_piomode(ap, dev);
2628	}
2629
2630	/* prefer hardreset */
2631	reset = NULL;
2632	ehc->i.action &= ~ATA_EH_RESET;
2633	if (hardreset) {
2634	reset = hardreset;
2635	ehc->i.action |= ATA_EH_HARDRESET;
2636	} else if (softreset) {
2637	reset = softreset;
2638	ehc->i.action |= ATA_EH_SOFTRESET;
2639	}
2640
2641	if (prereset) {
2642	unsigned long deadline = ata_deadline(from_jiffies: jiffies,
2643	timeout_msecs: ATA_EH_PRERESET_TIMEOUT);
2644
2645	if (slave) {
2646	sehc->i.action &= ~ATA_EH_RESET;
2647	sehc->i.action |= ehc->i.action;
2648	}
2649
2650	rc = prereset(link, deadline);
2651
2652	/* If present, do prereset on slave link too. Reset
2653	* is skipped iff both master and slave links report
2654	* -ENOENT or clear ATA_EH_RESET.
2655	*/
2656	if (slave && (rc == 0 || rc == -ENOENT)) {
2657	int tmp;
2658
2659	tmp = prereset(slave, deadline);
2660	if (tmp != -ENOENT)
2661	rc = tmp;
2662
2663	ehc->i.action |= sehc->i.action;
2664	}
2665
2666	if (rc) {
2667	if (rc == -ENOENT) {
2668	ata_link_dbg(link, "port disabled--ignoring\n");
2669	ehc->i.action &= ~ATA_EH_RESET;
2670
2671	ata_for_each_dev(dev, link, ALL)
2672	classes[dev->devno] = ATA_DEV_NONE;
2673
2674	rc = 0;
2675	} else
2676	ata_link_err(link,
2677	"prereset failed (errno=%d)\n",
2678	rc);
2679	goto out;
2680	}
2681
2682	/* prereset() might have cleared ATA_EH_RESET. If so,
2683	* bang classes, thaw and return.
2684	*/
2685	if (reset && !(ehc->i.action & ATA_EH_RESET)) {
2686	ata_for_each_dev(dev, link, ALL)
2687	classes[dev->devno] = ATA_DEV_NONE;
2688	if (ata_port_is_frozen(ap) && ata_is_host_link(link))
2689	ata_eh_thaw_port(ap);
2690	rc = 0;
2691	goto out;
2692	}
2693	}
2694
2695	retry:
2696	/*
2697	* Perform reset
2698	*/
2699	if (ata_is_host_link(link))
2700	ata_eh_freeze_port(ap);
2701
2702	deadline = ata_deadline(from_jiffies: jiffies, timeout_msecs: ata_eh_reset_timeouts[try++]);
2703
2704	if (reset) {
2705	if (verbose)
2706	ata_link_info(link, "%s resetting link\n",
2707	reset == softreset ? "soft" : "hard");
2708
2709	/* mark that this EH session started with reset */
2710	ehc->last_reset = jiffies;
2711	if (reset == hardreset) {
2712	ehc->i.flags |= ATA_EHI_DID_HARDRESET;
2713	trace_ata_link_hardreset_begin(link, class: classes, deadline);
2714	} else {
2715	ehc->i.flags |= ATA_EHI_DID_SOFTRESET;
2716	trace_ata_link_softreset_begin(link, class: classes, deadline);
2717	}
2718
2719	rc = ata_do_reset(link, reset, classes, deadline, clear_classes: true);
2720	if (reset == hardreset)
2721	trace_ata_link_hardreset_end(link, class: classes, rc);
2722	else
2723	trace_ata_link_softreset_end(link, class: classes, rc);
2724	if (rc && rc != -EAGAIN) {
2725	failed_link = link;
2726	goto fail;
2727	}
2728
2729	/* hardreset slave link if existent */
2730	if (slave && reset == hardreset) {
2731	int tmp;
2732
2733	if (verbose)
2734	ata_link_info(slave, "hard resetting link\n");
2735
2736	ata_eh_about_to_do(link: slave, NULL, action: ATA_EH_RESET);
2737	trace_ata_slave_hardreset_begin(link: slave, class: classes,
2738	deadline);
2739	tmp = ata_do_reset(link: slave, reset, classes, deadline,
2740	clear_classes: false);
2741	trace_ata_slave_hardreset_end(link: slave, class: classes, rc: tmp);
2742	switch (tmp) {
2743	case -EAGAIN:
2744	rc = -EAGAIN;
2745	break;
2746	case 0:
2747	break;
2748	default:
2749	failed_link = slave;
2750	rc = tmp;
2751	goto fail;
2752	}
2753	}
2754
2755	/* perform follow-up SRST if necessary */
2756	if (reset == hardreset &&
2757	ata_eh_followup_srst_needed(link, rc)) {
2758	reset = softreset;
2759
2760	if (!reset) {
2761	ata_link_err(link,
2762	"follow-up softreset required but no softreset available\n");
2763	failed_link = link;
2764	rc = -EINVAL;
2765	goto fail;
2766	}
2767
2768	ata_eh_about_to_do(link, NULL, action: ATA_EH_RESET);
2769	trace_ata_link_softreset_begin(link, class: classes, deadline);
2770	rc = ata_do_reset(link, reset, classes, deadline, clear_classes: true);
2771	trace_ata_link_softreset_end(link, class: classes, rc);
2772	if (rc) {
2773	failed_link = link;
2774	goto fail;
2775	}
2776	}
2777	} else {
2778	if (verbose)
2779	ata_link_info(link,
2780	"no reset method available, skipping reset\n");
2781	if (!(lflags & ATA_LFLAG_ASSUME_CLASS))
2782	lflags |= ATA_LFLAG_ASSUME_ATA;
2783	}
2784
2785	/*
2786	* Post-reset processing
2787	*/
2788	ata_for_each_dev(dev, link, ALL) {
2789	/* After the reset, the device state is PIO 0 and the
2790	* controller state is undefined. Reset also wakes up
2791	* drives from sleeping mode.
2792	*/
2793	dev->pio_mode = XFER_PIO_0;
2794	dev->flags &= ~ATA_DFLAG_SLEEPING;
2795
2796	if (ata_phys_link_offline(link: ata_dev_phys_link(dev)))
2797	continue;
2798
2799	/* apply class override */
2800	if (lflags & ATA_LFLAG_ASSUME_ATA)
2801	classes[dev->devno] = ATA_DEV_ATA;
2802	else if (lflags & ATA_LFLAG_ASSUME_SEMB)
2803	classes[dev->devno] = ATA_DEV_SEMB_UNSUP;
2804	}
2805
2806	/* record current link speed */
2807	if (sata_scr_read(link, reg: SCR_STATUS, val: &sstatus) == 0)
2808	link->sata_spd = (sstatus >> 4) & 0xf;
2809	if (slave && sata_scr_read(link: slave, reg: SCR_STATUS, val: &sstatus) == 0)
2810	slave->sata_spd = (sstatus >> 4) & 0xf;
2811
2812	/* thaw the port */
2813	if (ata_is_host_link(link))
2814	ata_eh_thaw_port(ap);
2815
2816	/* postreset() should clear hardware SError. Although SError
2817	* is cleared during link resume, clearing SError here is
2818	* necessary as some PHYs raise hotplug events after SRST.
2819	* This introduces race condition where hotplug occurs between
2820	* reset and here. This race is mediated by cross checking
2821	* link onlineness and classification result later.
2822	*/
2823	if (postreset) {
2824	postreset(link, classes);
2825	trace_ata_link_postreset(link, class: classes, rc);
2826	if (slave) {
2827	postreset(slave, classes);
2828	trace_ata_slave_postreset(link: slave, class: classes, rc);
2829	}
2830	}
2831
2832	/* clear cached SError */
2833	spin_lock_irqsave(link->ap->lock, flags);
2834	link->eh_info.serror = 0;
2835	if (slave)
2836	slave->eh_info.serror = 0;
2837	spin_unlock_irqrestore(lock: link->ap->lock, flags);
2838
2839	/*
2840	* Make sure onlineness and classification result correspond.
2841	* Hotplug could have happened during reset and some
2842	* controllers fail to wait while a drive is spinning up after
2843	* being hotplugged causing misdetection. By cross checking
2844	* link on/offlineness and classification result, those
2845	* conditions can be reliably detected and retried.
2846	*/
2847	nr_unknown = 0;
2848	ata_for_each_dev(dev, link, ALL) {
2849	if (ata_phys_link_online(link: ata_dev_phys_link(dev))) {
2850	if (classes[dev->devno] == ATA_DEV_UNKNOWN) {
2851	ata_dev_dbg(dev, "link online but device misclassified\n");
2852	classes[dev->devno] = ATA_DEV_NONE;
2853	nr_unknown++;
2854	}
2855	} else if (ata_phys_link_offline(link: ata_dev_phys_link(dev))) {
2856	if (ata_class_enabled(class: classes[dev->devno]))
2857	ata_dev_dbg(dev,
2858	"link offline, clearing class %d to NONE\n",
2859	classes[dev->devno]);
2860	classes[dev->devno] = ATA_DEV_NONE;
2861	} else if (classes[dev->devno] == ATA_DEV_UNKNOWN) {
2862	ata_dev_dbg(dev,
2863	"link status unknown, clearing UNKNOWN to NONE\n");
2864	classes[dev->devno] = ATA_DEV_NONE;
2865	}
2866	}
2867
2868	if (classify && nr_unknown) {
2869	if (try < max_tries) {
2870	ata_link_warn(link,
2871	"link online but %d devices misclassified, retrying\n",
2872	nr_unknown);
2873	failed_link = link;
2874	rc = -EAGAIN;
2875	goto fail;
2876	}
2877	ata_link_warn(link,
2878	"link online but %d devices misclassified, "
2879	"device detection might fail\n", nr_unknown);
2880	}
2881
2882	/* reset successful, schedule revalidation */
2883	ata_eh_done(link, NULL, action: ATA_EH_RESET);
2884	if (slave)
2885	ata_eh_done(link: slave, NULL, action: ATA_EH_RESET);
2886	ehc->last_reset = jiffies; /* update to completion time */
2887	ehc->i.action |= ATA_EH_REVALIDATE;
2888	link->lpm_policy = ATA_LPM_UNKNOWN; /* reset LPM state */
2889
2890	rc = 0;
2891	out:
2892	/* clear hotplug flag */
2893	ehc->i.flags &= ~ATA_EHI_HOTPLUGGED;
2894	if (slave)
2895	sehc->i.flags &= ~ATA_EHI_HOTPLUGGED;
2896
2897	spin_lock_irqsave(ap->lock, flags);
2898	ap->pflags &= ~ATA_PFLAG_RESETTING;
2899	spin_unlock_irqrestore(lock: ap->lock, flags);
2900
2901	return rc;
2902
2903	fail:
2904	/* if SCR isn't accessible on a fan-out port, PMP needs to be reset */
2905	if (!ata_is_host_link(link) &&
2906	sata_scr_read(link, reg: SCR_STATUS, val: &sstatus))
2907	rc = -ERESTART;
2908
2909	if (try >= max_tries) {
2910	/*
2911	* Thaw host port even if reset failed, so that the port
2912	* can be retried on the next phy event. This risks
2913	* repeated EH runs but seems to be a better tradeoff than
2914	* shutting down a port after a botched hotplug attempt.
2915	*/
2916	if (ata_is_host_link(link))
2917	ata_eh_thaw_port(ap);
2918	ata_link_warn(link, "%s failed\n",
2919	reset == hardreset ? "hardreset" : "softreset");
2920	goto out;
2921	}
2922
2923	now = jiffies;
2924	if (time_before(now, deadline)) {
2925	unsigned long delta = deadline - now;
2926
2927	ata_link_warn(failed_link,
2928	"reset failed (errno=%d), retrying in %u secs\n",
2929	rc, DIV_ROUND_UP(jiffies_to_msecs(delta), 1000));
2930
2931	ata_eh_release(ap);
2932	while (delta)
2933	delta = schedule_timeout_uninterruptible(timeout: delta);
2934	ata_eh_acquire(ap);
2935	}
2936
2937	/*
2938	* While disks spinup behind PMP, some controllers fail sending SRST.
2939	* They need to be reset - as well as the PMP - before retrying.
2940	*/
2941	if (rc == -ERESTART) {
2942	if (ata_is_host_link(link))
2943	ata_eh_thaw_port(ap);
2944	goto out;
2945	}
2946
2947	if (try == max_tries - 1) {
2948	sata_down_spd_limit(link, spd_limit: 0);
2949	if (slave)
2950	sata_down_spd_limit(link: slave, spd_limit: 0);
2951	} else if (rc == -EPIPE)
2952	sata_down_spd_limit(link: failed_link, spd_limit: 0);
2953
2954	if (hardreset)
2955	reset = hardreset;
2956	goto retry;
2957	}
2958
2959	static inline void ata_eh_pull_park_action(struct ata_port *ap)
2960	{
2961	struct ata_link *link;
2962	struct ata_device *dev;
2963	unsigned long flags;
2964
2965	/*
2966	* This function can be thought of as an extended version of
2967	* ata_eh_about_to_do() specially crafted to accommodate the
2968	* requirements of ATA_EH_PARK handling. Since the EH thread
2969	* does not leave the do {} while () loop in ata_eh_recover as
2970	* long as the timeout for a park request to *one* device on
2971	* the port has not expired, and since we still want to pick
2972	* up park requests to other devices on the same port or
2973	* timeout updates for the same device, we have to pull
2974	* ATA_EH_PARK actions from eh_info into eh_context.i
2975	* ourselves at the beginning of each pass over the loop.
2976	*
2977	* Additionally, all write accesses to &ap->park_req_pending
2978	* through reinit_completion() (see below) or complete_all()
2979	* (see ata_scsi_park_store()) are protected by the host lock.
2980	* As a result we have that park_req_pending.done is zero on
2981	* exit from this function, i.e. when ATA_EH_PARK actions for
2982	* *all* devices on port ap have been pulled into the
2983	* respective eh_context structs. If, and only if,
2984	* park_req_pending.done is non-zero by the time we reach
2985	* wait_for_completion_timeout(), another ATA_EH_PARK action
2986	* has been scheduled for at least one of the devices on port
2987	* ap and we have to cycle over the do {} while () loop in
2988	* ata_eh_recover() again.
2989	*/
2990
2991	spin_lock_irqsave(ap->lock, flags);
2992	reinit_completion(x: &ap->park_req_pending);
2993	ata_for_each_link(link, ap, EDGE) {
2994	ata_for_each_dev(dev, link, ALL) {
2995	struct ata_eh_info *ehi = &link->eh_info;
2996
2997	link->eh_context.i.dev_action[dev->devno] |=
2998	ehi->dev_action[dev->devno] & ATA_EH_PARK;
2999	ata_eh_clear_action(link, dev, ehi, action: ATA_EH_PARK);
3000	}
3001	}
3002	spin_unlock_irqrestore(lock: ap->lock, flags);
3003	}
3004
3005	static void ata_eh_park_issue_cmd(struct ata_device *dev, int park)
3006	{
3007	struct ata_eh_context *ehc = &dev->link->eh_context;
3008	struct ata_taskfile tf;
3009	unsigned int err_mask;
3010
3011	ata_tf_init(dev, tf: &tf);
3012	if (park) {
3013	ehc->unloaded_mask |= 1 << dev->devno;
3014	tf.command = ATA_CMD_IDLEIMMEDIATE;
3015	tf.feature = 0x44;
3016	tf.lbal = 0x4c;
3017	tf.lbam = 0x4e;
3018	tf.lbah = 0x55;
3019	} else {
3020	ehc->unloaded_mask &= ~(1 << dev->devno);
3021	tf.command = ATA_CMD_CHK_POWER;
3022	}
3023
3024	tf.flags |= ATA_TFLAG_DEVICE | ATA_TFLAG_ISADDR;
3025	tf.protocol = ATA_PROT_NODATA;
3026	err_mask = ata_exec_internal(dev, tf: &tf, NULL, dma_dir: DMA_NONE, NULL, buflen: 0, timeout: 0);
3027	if (park && (err_mask || tf.lbal != 0xc4)) {
3028	ata_dev_err(dev, "head unload failed!\n");
3029	ehc->unloaded_mask &= ~(1 << dev->devno);
3030	}
3031	}
3032
3033	static int ata_eh_revalidate_and_attach(struct ata_link *link,
3034	struct ata_device **r_failed_dev)
3035	{
3036	struct ata_port *ap = link->ap;
3037	struct ata_eh_context *ehc = &link->eh_context;
3038	struct ata_device *dev;
3039	unsigned int new_mask = 0;
3040	unsigned long flags;
3041	int rc = 0;
3042
3043	/* For PATA drive side cable detection to work, IDENTIFY must
3044	* be done backwards such that PDIAG- is released by the slave
3045	* device before the master device is identified.
3046	*/
3047	ata_for_each_dev(dev, link, ALL_REVERSE) {
3048	unsigned int action = ata_eh_dev_action(dev);
3049	unsigned int readid_flags = 0;
3050
3051	if (ehc->i.flags & ATA_EHI_DID_RESET)
3052	readid_flags |= ATA_READID_POSTRESET;
3053
3054	if ((action & ATA_EH_REVALIDATE) && ata_dev_enabled(dev)) {
3055	WARN_ON(dev->class == ATA_DEV_PMP);
3056
3057	/*
3058	* The link may be in a deep sleep, wake it up.
3059	*
3060	* If the link is in deep sleep, ata_phys_link_offline()
3061	* will return true, causing the revalidation to fail,
3062	* which leads to a (potentially) needless hard reset.
3063	*
3064	* ata_eh_recover() will later restore the link policy
3065	* to ap->target_lpm_policy after revalidation is done.
3066	*/
3067	if (link->lpm_policy > ATA_LPM_MAX_POWER) {
3068	rc = ata_eh_set_lpm(link, policy: ATA_LPM_MAX_POWER,
3069	r_failed_dev);
3070	if (rc)
3071	goto err;
3072	}
3073
3074	if (ata_phys_link_offline(link: ata_dev_phys_link(dev))) {
3075	rc = -EIO;
3076	goto err;
3077	}
3078
3079	ata_eh_about_to_do(link, dev, action: ATA_EH_REVALIDATE);
3080	rc = ata_dev_revalidate(dev, new_class: ehc->classes[dev->devno],
3081	readid_flags);
3082	if (rc)
3083	goto err;
3084
3085	ata_eh_done(link, dev, action: ATA_EH_REVALIDATE);
3086
3087	/* Configuration may have changed, reconfigure
3088	* transfer mode.
3089	*/
3090	ehc->i.flags |= ATA_EHI_SETMODE;
3091
3092	/* schedule the scsi_rescan_device() here */
3093	schedule_delayed_work(dwork: &ap->scsi_rescan_task, delay: 0);
3094	} else if (dev->class == ATA_DEV_UNKNOWN &&
3095	ehc->tries[dev->devno] &&
3096	ata_class_enabled(class: ehc->classes[dev->devno])) {
3097	/* Temporarily set dev->class, it will be
3098	* permanently set once all configurations are
3099	* complete. This is necessary because new
3100	* device configuration is done in two
3101	* separate loops.
3102	*/
3103	dev->class = ehc->classes[dev->devno];
3104
3105	if (dev->class == ATA_DEV_PMP)
3106	rc = sata_pmp_attach(dev);
3107	else
3108	rc = ata_dev_read_id(dev, p_class: &dev->class,
3109	flags: readid_flags, id: dev->id);
3110
3111	/* read_id might have changed class, store and reset */
3112	ehc->classes[dev->devno] = dev->class;
3113	dev->class = ATA_DEV_UNKNOWN;
3114
3115	switch (rc) {
3116	case 0:
3117	/* clear error info accumulated during probe */
3118	ata_ering_clear(ering: &dev->ering);
3119	new_mask |= 1 << dev->devno;
3120	break;
3121	case -ENOENT:
3122	/* IDENTIFY was issued to non-existent
3123	* device. No need to reset. Just
3124	* thaw and ignore the device.
3125	*/
3126	ata_eh_thaw_port(ap);
3127	break;
3128	default:
3129	goto err;
3130	}
3131	}
3132	}
3133
3134	/* PDIAG- should have been released, ask cable type if post-reset */
3135	if ((ehc->i.flags & ATA_EHI_DID_RESET) && ata_is_host_link(link)) {
3136	if (ap->ops->cable_detect)
3137	ap->cbl = ap->ops->cable_detect(ap);
3138	ata_force_cbl(ap);
3139	}
3140
3141	/* Configure new devices forward such that user doesn't see
3142	* device detection messages backwards.
3143	*/
3144	ata_for_each_dev(dev, link, ALL) {
3145	if (!(new_mask & (1 << dev->devno)))
3146	continue;
3147
3148	dev->class = ehc->classes[dev->devno];
3149
3150	if (dev->class == ATA_DEV_PMP)
3151	continue;
3152
3153	ehc->i.flags |= ATA_EHI_PRINTINFO;
3154	rc = ata_dev_configure(dev);
3155	ehc->i.flags &= ~ATA_EHI_PRINTINFO;
3156	if (rc) {
3157	dev->class = ATA_DEV_UNKNOWN;
3158	goto err;
3159	}
3160
3161	spin_lock_irqsave(ap->lock, flags);
3162	ap->pflags |= ATA_PFLAG_SCSI_HOTPLUG;
3163	spin_unlock_irqrestore(lock: ap->lock, flags);
3164
3165	/* new device discovered, configure xfermode */
3166	ehc->i.flags |= ATA_EHI_SETMODE;
3167	}
3168
3169	return 0;
3170
3171	err:
3172	*r_failed_dev = dev;
3173	return rc;
3174	}
3175
3176	/**
3177	* ata_set_mode - Program timings and issue SET FEATURES - XFER
3178	* @link: link on which timings will be programmed
3179	* @r_failed_dev: out parameter for failed device
3180	*
3181	* Set ATA device disk transfer mode (PIO3, UDMA6, etc.). If
3182	* ata_set_mode() fails, pointer to the failing device is
3183	* returned in @r_failed_dev.
3184	*
3185	* LOCKING:
3186	* PCI/etc. bus probe sem.
3187	*
3188	* RETURNS:
3189	* 0 on success, negative errno otherwise
3190	*/
3191	int ata_set_mode(struct ata_link *link, struct ata_device **r_failed_dev)
3192	{
3193	struct ata_port *ap = link->ap;
3194	struct ata_device *dev;
3195	int rc;
3196
3197	/* if data transfer is verified, clear DUBIOUS_XFER on ering top */
3198	ata_for_each_dev(dev, link, ENABLED) {
3199	if (!(dev->flags & ATA_DFLAG_DUBIOUS_XFER)) {
3200	struct ata_ering_entry *ent;
3201
3202	ent = ata_ering_top(ering: &dev->ering);
3203	if (ent)
3204	ent->eflags &= ~ATA_EFLAG_DUBIOUS_XFER;
3205	}
3206	}
3207
3208	/* has private set_mode? */
3209	if (ap->ops->set_mode)
3210	rc = ap->ops->set_mode(link, r_failed_dev);
3211	else
3212	rc = ata_do_set_mode(link, r_failed_dev);
3213
3214	/* if transfer mode has changed, set DUBIOUS_XFER on device */
3215	ata_for_each_dev(dev, link, ENABLED) {
3216	struct ata_eh_context *ehc = &link->eh_context;
3217	u8 saved_xfer_mode = ehc->saved_xfer_mode[dev->devno];
3218	u8 saved_ncq = !!(ehc->saved_ncq_enabled & (1 << dev->devno));
3219
3220	if (dev->xfer_mode != saved_xfer_mode ||
3221	ata_ncq_enabled(dev) != saved_ncq)
3222	dev->flags |= ATA_DFLAG_DUBIOUS_XFER;
3223	}
3224
3225	return rc;
3226	}
3227
3228	/**
3229	* atapi_eh_clear_ua - Clear ATAPI UNIT ATTENTION after reset
3230	* @dev: ATAPI device to clear UA for
3231	*
3232	* Resets and other operations can make an ATAPI device raise
3233	* UNIT ATTENTION which causes the next operation to fail. This
3234	* function clears UA.
3235	*
3236	* LOCKING:
3237	* EH context (may sleep).
3238	*
3239	* RETURNS:
3240	* 0 on success, -errno on failure.
3241	*/
3242	static int atapi_eh_clear_ua(struct ata_device *dev)
3243	{
3244	int i;
3245
3246	for (i = 0; i < ATA_EH_UA_TRIES; i++) {
3247	u8 *sense_buffer = dev->link->ap->sector_buf;
3248	u8 sense_key = 0;
3249	unsigned int err_mask;
3250
3251	err_mask = atapi_eh_tur(dev, r_sense_key: &sense_key);
3252	if (err_mask != 0 && err_mask != AC_ERR_DEV) {
3253	ata_dev_warn(dev,
3254	"TEST_UNIT_READY failed (err_mask=0x%x)\n",
3255	err_mask);
3256	return -EIO;
3257	}
3258
3259	if (!err_mask || sense_key != UNIT_ATTENTION)
3260	return 0;
3261
3262	err_mask = atapi_eh_request_sense(dev, sense_buf: sense_buffer, dfl_sense_key: sense_key);
3263	if (err_mask) {
3264	ata_dev_warn(dev, "failed to clear "
3265	"UNIT ATTENTION (err_mask=0x%x)\n", err_mask);
3266	return -EIO;
3267	}
3268	}
3269
3270	ata_dev_warn(dev, "UNIT ATTENTION persists after %d tries\n",
3271	ATA_EH_UA_TRIES);
3272
3273	return 0;
3274	}
3275
3276	/**
3277	* ata_eh_maybe_retry_flush - Retry FLUSH if necessary
3278	* @dev: ATA device which may need FLUSH retry
3279	*
3280	* If @dev failed FLUSH, it needs to be reported upper layer
3281	* immediately as it means that @dev failed to remap and already
3282	* lost at least a sector and further FLUSH retrials won't make
3283	* any difference to the lost sector. However, if FLUSH failed
3284	* for other reasons, for example transmission error, FLUSH needs
3285	* to be retried.
3286	*
3287	* This function determines whether FLUSH failure retry is
3288	* necessary and performs it if so.
3289	*
3290	* RETURNS:
3291	* 0 if EH can continue, -errno if EH needs to be repeated.
3292	*/
3293	static int ata_eh_maybe_retry_flush(struct ata_device *dev)
3294	{
3295	struct ata_link *link = dev->link;
3296	struct ata_port *ap = link->ap;
3297	struct ata_queued_cmd *qc;
3298	struct ata_taskfile tf;
3299	unsigned int err_mask;
3300	int rc = 0;
3301
3302	/* did flush fail for this device? */
3303	if (!ata_tag_valid(tag: link->active_tag))
3304	return 0;
3305
3306	qc = __ata_qc_from_tag(ap, tag: link->active_tag);
3307	if (qc->dev != dev || (qc->tf.command != ATA_CMD_FLUSH_EXT &&
3308	qc->tf.command != ATA_CMD_FLUSH))
3309	return 0;
3310
3311	/* if the device failed it, it should be reported to upper layers */
3312	if (qc->err_mask & AC_ERR_DEV)
3313	return 0;
3314
3315	/* flush failed for some other reason, give it another shot */
3316	ata_tf_init(dev, tf: &tf);
3317
3318	tf.command = qc->tf.command;
3319	tf.flags |= ATA_TFLAG_DEVICE;
3320	tf.protocol = ATA_PROT_NODATA;
3321
3322	ata_dev_warn(dev, "retrying FLUSH 0x%x Emask 0x%x\n",
3323	tf.command, qc->err_mask);
3324
3325	err_mask = ata_exec_internal(dev, tf: &tf, NULL, dma_dir: DMA_NONE, NULL, buflen: 0, timeout: 0);
3326	if (!err_mask) {
3327	/*
3328	* FLUSH is complete but there's no way to
3329	* successfully complete a failed command from EH.
3330	* Making sure retry is allowed at least once and
3331	* retrying it should do the trick - whatever was in
3332	* the cache is already on the platter and this won't
3333	* cause infinite loop.
3334	*/
3335	qc->scsicmd->allowed = max(qc->scsicmd->allowed, 1);
3336	} else {
3337	ata_dev_warn(dev, "FLUSH failed Emask 0x%x\n",
3338	err_mask);
3339	rc = -EIO;
3340
3341	/* if device failed it, report it to upper layers */
3342	if (err_mask & AC_ERR_DEV) {
3343	qc->err_mask |= AC_ERR_DEV;
3344	qc->result_tf = tf;
3345	if (!ata_port_is_frozen(ap))
3346	rc = 0;
3347	}
3348	}
3349	return rc;
3350	}
3351
3352	/**
3353	* ata_eh_set_lpm - configure SATA interface power management
3354	* @link: link to configure power management
3355	* @policy: the link power management policy
3356	* @r_failed_dev: out parameter for failed device
3357	*
3358	* Enable SATA Interface power management. This will enable
3359	* Device Interface Power Management (DIPM) for min_power and
3360	* medium_power_with_dipm policies, and then call driver specific
3361	* callbacks for enabling Host Initiated Power management.
3362	*
3363	* LOCKING:
3364	* EH context.
3365	*
3366	* RETURNS:
3367	* 0 on success, -errno on failure.
3368	*/
3369	static int ata_eh_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
3370	struct ata_device **r_failed_dev)
3371	{
3372	struct ata_port *ap = ata_is_host_link(link) ? link->ap : NULL;
3373	struct ata_eh_context *ehc = &link->eh_context;
3374	struct ata_device *dev, *link_dev = NULL, *lpm_dev = NULL;
3375	enum ata_lpm_policy old_policy = link->lpm_policy;
3376	bool no_dipm = link->ap->flags & ATA_FLAG_NO_DIPM;
3377	unsigned int hints = ATA_LPM_EMPTY | ATA_LPM_HIPM;
3378	unsigned int err_mask;
3379	int rc;
3380
3381	/* if the link or host doesn't do LPM, noop */
3382	if (!IS_ENABLED(CONFIG_SATA_HOST) ||
3383	(link->flags & ATA_LFLAG_NO_LPM) || (ap && !ap->ops->set_lpm))
3384	return 0;
3385
3386	/*
3387	* DIPM is enabled only for MIN_POWER as some devices
3388	* misbehave when the host NACKs transition to SLUMBER. Order
3389	* device and link configurations such that the host always
3390	* allows DIPM requests.
3391	*/
3392	ata_for_each_dev(dev, link, ENABLED) {
3393	bool hipm = ata_id_has_hipm(id: dev->id);
3394	bool dipm = ata_id_has_dipm(dev->id) && !no_dipm;
3395
3396	/* find the first enabled and LPM enabled devices */
3397	if (!link_dev)
3398	link_dev = dev;
3399
3400	if (!lpm_dev && (hipm || dipm))
3401	lpm_dev = dev;
3402
3403	hints &= ~ATA_LPM_EMPTY;
3404	if (!hipm)
3405	hints &= ~ATA_LPM_HIPM;
3406
3407	/* disable DIPM before changing link config */
3408	if (policy < ATA_LPM_MED_POWER_WITH_DIPM && dipm) {
3409	err_mask = ata_dev_set_feature(dev,
3410	subcmd: SETFEATURES_SATA_DISABLE, action: SATA_DIPM);
3411	if (err_mask && err_mask != AC_ERR_DEV) {
3412	ata_dev_warn(dev,
3413	"failed to disable DIPM, Emask 0x%x\n",
3414	err_mask);
3415	rc = -EIO;
3416	goto fail;
3417	}
3418	}
3419	}
3420
3421	if (ap) {
3422	rc = ap->ops->set_lpm(link, policy, hints);
3423	if (!rc && ap->slave_link)
3424	rc = ap->ops->set_lpm(ap->slave_link, policy, hints);
3425	} else
3426	rc = sata_pmp_set_lpm(link, policy, hints);
3427
3428	/*
3429	* Attribute link config failure to the first (LPM) enabled
3430	* device on the link.
3431	*/
3432	if (rc) {
3433	if (rc == -EOPNOTSUPP) {
3434	link->flags |= ATA_LFLAG_NO_LPM;
3435	return 0;
3436	}
3437	dev = lpm_dev ? lpm_dev : link_dev;
3438	goto fail;
3439	}
3440
3441	/*
3442	* Low level driver acked the transition. Issue DIPM command
3443	* with the new policy set.
3444	*/
3445	link->lpm_policy = policy;
3446	if (ap && ap->slave_link)
3447	ap->slave_link->lpm_policy = policy;
3448
3449	/* host config updated, enable DIPM if transitioning to MIN_POWER */
3450	ata_for_each_dev(dev, link, ENABLED) {
3451	if (policy >= ATA_LPM_MED_POWER_WITH_DIPM && !no_dipm &&
3452	ata_id_has_dipm(dev->id)) {
3453	err_mask = ata_dev_set_feature(dev,
3454	subcmd: SETFEATURES_SATA_ENABLE, action: SATA_DIPM);
3455	if (err_mask && err_mask != AC_ERR_DEV) {
3456	ata_dev_warn(dev,
3457	"failed to enable DIPM, Emask 0x%x\n",
3458	err_mask);
3459	rc = -EIO;
3460	goto fail;
3461	}
3462	}
3463	}
3464
3465	link->last_lpm_change = jiffies;
3466	link->flags |= ATA_LFLAG_CHANGED;
3467
3468	return 0;
3469
3470	fail:
3471	/* restore the old policy */
3472	link->lpm_policy = old_policy;
3473	if (ap && ap->slave_link)
3474	ap->slave_link->lpm_policy = old_policy;
3475
3476	/* if no device or only one more chance is left, disable LPM */
3477	if (!dev || ehc->tries[dev->devno] <= 2) {
3478	ata_link_warn(link, "disabling LPM on the link\n");
3479	link->flags |= ATA_LFLAG_NO_LPM;
3480	}
3481	if (r_failed_dev)
3482	*r_failed_dev = dev;
3483	return rc;
3484	}
3485
3486	int ata_link_nr_enabled(struct ata_link *link)
3487	{
3488	struct ata_device *dev;
3489	int cnt = 0;
3490
3491	ata_for_each_dev(dev, link, ENABLED)
3492	cnt++;
3493	return cnt;
3494	}
3495
3496	static int ata_link_nr_vacant(struct ata_link *link)
3497	{
3498	struct ata_device *dev;
3499	int cnt = 0;
3500
3501	ata_for_each_dev(dev, link, ALL)
3502	if (dev->class == ATA_DEV_UNKNOWN)
3503	cnt++;
3504	return cnt;
3505	}
3506
3507	static int ata_eh_skip_recovery(struct ata_link *link)
3508	{
3509	struct ata_port *ap = link->ap;
3510	struct ata_eh_context *ehc = &link->eh_context;
3511	struct ata_device *dev;
3512
3513	/* skip disabled links */
3514	if (link->flags & ATA_LFLAG_DISABLED)
3515	return 1;
3516
3517	/* skip if explicitly requested */
3518	if (ehc->i.flags & ATA_EHI_NO_RECOVERY)
3519	return 1;
3520
3521	/* thaw frozen port and recover failed devices */
3522	if (ata_port_is_frozen(ap) || ata_link_nr_enabled(link))
3523	return 0;
3524
3525	/* reset at least once if reset is requested */
3526	if ((ehc->i.action & ATA_EH_RESET) &&
3527	!(ehc->i.flags & ATA_EHI_DID_RESET))
3528	return 0;
3529
3530	/* skip if class codes for all vacant slots are ATA_DEV_NONE */
3531	ata_for_each_dev(dev, link, ALL) {
3532	if (dev->class == ATA_DEV_UNKNOWN &&
3533	ehc->classes[dev->devno] != ATA_DEV_NONE)
3534	return 0;
3535	}
3536
3537	return 1;
3538	}
3539
3540	static int ata_count_probe_trials_cb(struct ata_ering_entry *ent, void *void_arg)
3541	{
3542	u64 interval = msecs_to_jiffies(m: ATA_EH_PROBE_TRIAL_INTERVAL);
3543	u64 now = get_jiffies_64();
3544	int *trials = void_arg;
3545
3546	if ((ent->eflags & ATA_EFLAG_OLD_ER) ||
3547	(ent->timestamp < now - min(now, interval)))
3548	return -1;
3549
3550	(*trials)++;
3551	return 0;
3552	}
3553
3554	static int ata_eh_schedule_probe(struct ata_device *dev)
3555	{
3556	struct ata_eh_context *ehc = &dev->link->eh_context;
3557	struct ata_link *link = ata_dev_phys_link(dev);
3558	int trials = 0;
3559
3560	if (!(ehc->i.probe_mask & (1 << dev->devno)) ||
3561	(ehc->did_probe_mask & (1 << dev->devno)))
3562	return 0;
3563
3564	ata_eh_detach_dev(dev);
3565	ata_dev_init(dev);
3566	ehc->did_probe_mask |= (1 << dev->devno);
3567	ehc->i.action |= ATA_EH_RESET;
3568	ehc->saved_xfer_mode[dev->devno] = 0;
3569	ehc->saved_ncq_enabled &= ~(1 << dev->devno);
3570
3571	/* the link maybe in a deep sleep, wake it up */
3572	if (link->lpm_policy > ATA_LPM_MAX_POWER) {
3573	if (ata_is_host_link(link))
3574	link->ap->ops->set_lpm(link, ATA_LPM_MAX_POWER,
3575	ATA_LPM_EMPTY);
3576	else
3577	sata_pmp_set_lpm(link, policy: ATA_LPM_MAX_POWER,
3578	hints: ATA_LPM_EMPTY);
3579	}
3580
3581	/* Record and count probe trials on the ering. The specific
3582	* error mask used is irrelevant. Because a successful device
3583	* detection clears the ering, this count accumulates only if
3584	* there are consecutive failed probes.
3585	*
3586	* If the count is equal to or higher than ATA_EH_PROBE_TRIALS
3587	* in the last ATA_EH_PROBE_TRIAL_INTERVAL, link speed is
3588	* forced to 1.5Gbps.
3589	*
3590	* This is to work around cases where failed link speed
3591	* negotiation results in device misdetection leading to
3592	* infinite DEVXCHG or PHRDY CHG events.
3593	*/
3594	ata_ering_record(ering: &dev->ering, eflags: 0, err_mask: AC_ERR_OTHER);
3595	ata_ering_map(ering: &dev->ering, map_fn: ata_count_probe_trials_cb, arg: &trials);
3596
3597	if (trials > ATA_EH_PROBE_TRIALS)
3598	sata_down_spd_limit(link, spd_limit: 1);
3599
3600	return 1;
3601	}
3602
3603	static int ata_eh_handle_dev_fail(struct ata_device *dev, int err)
3604	{
3605	struct ata_eh_context *ehc = &dev->link->eh_context;
3606
3607	/* -EAGAIN from EH routine indicates retry without prejudice.
3608	* The requester is responsible for ensuring forward progress.
3609	*/
3610	if (err != -EAGAIN)
3611	ehc->tries[dev->devno]--;
3612
3613	switch (err) {
3614	case -ENODEV:
3615	/* device missing or wrong IDENTIFY data, schedule probing */
3616	ehc->i.probe_mask |= (1 << dev->devno);
3617	fallthrough;
3618	case -EINVAL:
3619	/* give it just one more chance */
3620	ehc->tries[dev->devno] = min(ehc->tries[dev->devno], 1);
3621	fallthrough;
3622	case -EIO:
3623	if (ehc->tries[dev->devno] == 1) {
3624	/* This is the last chance, better to slow
3625	* down than lose it.
3626	*/
3627	sata_down_spd_limit(link: ata_dev_phys_link(dev), spd_limit: 0);
3628	if (dev->pio_mode > XFER_PIO_0)
3629	ata_down_xfermask_limit(dev, sel: ATA_DNXFER_PIO);
3630	}
3631	}
3632
3633	if (ata_dev_enabled(dev) && !ehc->tries[dev->devno]) {
3634	/* disable device if it has used up all its chances */
3635	ata_dev_disable(dev);
3636
3637	/* detach if offline */
3638	if (ata_phys_link_offline(link: ata_dev_phys_link(dev)))
3639	ata_eh_detach_dev(dev);
3640
3641	/* schedule probe if necessary */
3642	if (ata_eh_schedule_probe(dev)) {
3643	ehc->tries[dev->devno] = ATA_EH_DEV_TRIES;
3644	memset(ehc->cmd_timeout_idx[dev->devno], 0,
3645	sizeof(ehc->cmd_timeout_idx[dev->devno]));
3646	}
3647
3648	return 1;
3649	} else {
3650	ehc->i.action |= ATA_EH_RESET;
3651	return 0;
3652	}
3653	}
3654
3655	/**
3656	* ata_eh_recover - recover host port after error
3657	* @ap: host port to recover
3658	* @prereset: prereset method (can be NULL)
3659	* @softreset: softreset method (can be NULL)
3660	* @hardreset: hardreset method (can be NULL)
3661	* @postreset: postreset method (can be NULL)
3662	* @r_failed_link: out parameter for failed link
3663	*
3664	* This is the alpha and omega, eum and yang, heart and soul of
3665	* libata exception handling. On entry, actions required to
3666	* recover each link and hotplug requests are recorded in the
3667	* link's eh_context. This function executes all the operations
3668	* with appropriate retrials and fallbacks to resurrect failed
3669	* devices, detach goners and greet newcomers.
3670	*
3671	* LOCKING:
3672	* Kernel thread context (may sleep).
3673	*
3674	* RETURNS:
3675	* 0 on success, -errno on failure.
3676	*/
3677	int ata_eh_recover(struct ata_port *ap, ata_prereset_fn_t prereset,
3678	ata_reset_fn_t softreset, ata_reset_fn_t hardreset,
3679	ata_postreset_fn_t postreset,
3680	struct ata_link **r_failed_link)
3681	{
3682	struct ata_link *link;
3683	struct ata_device *dev;
3684	int rc, nr_fails;
3685	unsigned long flags, deadline;
3686
3687	/* prep for recovery */
3688	ata_for_each_link(link, ap, EDGE) {
3689	struct ata_eh_context *ehc = &link->eh_context;
3690
3691	/* re-enable link? */
3692	if (ehc->i.action & ATA_EH_ENABLE_LINK) {
3693	ata_eh_about_to_do(link, NULL, action: ATA_EH_ENABLE_LINK);
3694	spin_lock_irqsave(ap->lock, flags);
3695	link->flags &= ~ATA_LFLAG_DISABLED;
3696	spin_unlock_irqrestore(lock: ap->lock, flags);
3697	ata_eh_done(link, NULL, action: ATA_EH_ENABLE_LINK);
3698	}
3699
3700	ata_for_each_dev(dev, link, ALL) {
3701	if (link->flags & ATA_LFLAG_NO_RETRY)
3702	ehc->tries[dev->devno] = 1;
3703	else
3704	ehc->tries[dev->devno] = ATA_EH_DEV_TRIES;
3705
3706	/* collect port action mask recorded in dev actions */
3707	ehc->i.action |= ehc->i.dev_action[dev->devno] &
3708	~ATA_EH_PERDEV_MASK;
3709	ehc->i.dev_action[dev->devno] &= ATA_EH_PERDEV_MASK;
3710
3711	/* process hotplug request */
3712	if (dev->flags & ATA_DFLAG_DETACH)
3713	ata_eh_detach_dev(dev);
3714
3715	/* schedule probe if necessary */
3716	if (!ata_dev_enabled(dev))
3717	ata_eh_schedule_probe(dev);
3718	}
3719	}
3720
3721	retry:
3722	rc = 0;
3723
3724	/* if UNLOADING, finish immediately */
3725	if (ap->pflags & ATA_PFLAG_UNLOADING)
3726	goto out;
3727
3728	/* prep for EH */
3729	ata_for_each_link(link, ap, EDGE) {
3730	struct ata_eh_context *ehc = &link->eh_context;
3731
3732	/* skip EH if possible. */
3733	if (ata_eh_skip_recovery(link))
3734	ehc->i.action = 0;
3735
3736	ata_for_each_dev(dev, link, ALL)
3737	ehc->classes[dev->devno] = ATA_DEV_UNKNOWN;
3738	}
3739
3740	/* reset */
3741	ata_for_each_link(link, ap, EDGE) {
3742	struct ata_eh_context *ehc = &link->eh_context;
3743
3744	if (!(ehc->i.action & ATA_EH_RESET))
3745	continue;
3746
3747	rc = ata_eh_reset(link, classify: ata_link_nr_vacant(link),
3748	prereset, softreset, hardreset, postreset);
3749	if (rc) {
3750	ata_link_err(link, "reset failed, giving up\n");
3751	goto out;
3752	}
3753	}
3754
3755	do {
3756	unsigned long now;
3757
3758	/*
3759	* clears ATA_EH_PARK in eh_info and resets
3760	* ap->park_req_pending
3761	*/
3762	ata_eh_pull_park_action(ap);
3763
3764	deadline = jiffies;
3765	ata_for_each_link(link, ap, EDGE) {
3766	ata_for_each_dev(dev, link, ALL) {
3767	struct ata_eh_context *ehc = &link->eh_context;
3768	unsigned long tmp;
3769
3770	if (dev->class != ATA_DEV_ATA &&
3771	dev->class != ATA_DEV_ZAC)
3772	continue;
3773	if (!(ehc->i.dev_action[dev->devno] &
3774	ATA_EH_PARK))
3775	continue;
3776	tmp = dev->unpark_deadline;
3777	if (time_before(deadline, tmp))
3778	deadline = tmp;
3779	else if (time_before_eq(tmp, jiffies))
3780	continue;
3781	if (ehc->unloaded_mask & (1 << dev->devno))
3782	continue;
3783
3784	ata_eh_park_issue_cmd(dev, park: 1);
3785	}
3786	}
3787
3788	now = jiffies;
3789	if (time_before_eq(deadline, now))
3790	break;
3791
3792	ata_eh_release(ap);
3793	deadline = wait_for_completion_timeout(x: &ap->park_req_pending,
3794	timeout: deadline - now);
3795	ata_eh_acquire(ap);
3796	} while (deadline);
3797	ata_for_each_link(link, ap, EDGE) {
3798	ata_for_each_dev(dev, link, ALL) {
3799	if (!(link->eh_context.unloaded_mask &
3800	(1 << dev->devno)))
3801	continue;
3802
3803	ata_eh_park_issue_cmd(dev, park: 0);
3804	ata_eh_done(link, dev, action: ATA_EH_PARK);
3805	}
3806	}
3807
3808	/* the rest */
3809	nr_fails = 0;
3810	ata_for_each_link(link, ap, PMP_FIRST) {
3811	struct ata_eh_context *ehc = &link->eh_context;
3812
3813	if (sata_pmp_attached(ap) && ata_is_host_link(link))
3814	goto config_lpm;
3815
3816	/* revalidate existing devices and attach new ones */
3817	rc = ata_eh_revalidate_and_attach(link, r_failed_dev: &dev);
3818	if (rc)
3819	goto rest_fail;
3820
3821	/* if PMP got attached, return, pmp EH will take care of it */
3822	if (link->device->class == ATA_DEV_PMP) {
3823	ehc->i.action = 0;
3824	return 0;
3825	}
3826
3827	/* configure transfer mode if necessary */
3828	if (ehc->i.flags & ATA_EHI_SETMODE) {
3829	rc = ata_set_mode(link, r_failed_dev: &dev);
3830	if (rc)
3831	goto rest_fail;
3832	ehc->i.flags &= ~ATA_EHI_SETMODE;
3833	}
3834
3835	/* If reset has been issued, clear UA to avoid
3836	* disrupting the current users of the device.
3837	*/
3838	if (ehc->i.flags & ATA_EHI_DID_RESET) {
3839	ata_for_each_dev(dev, link, ALL) {
3840	if (dev->class != ATA_DEV_ATAPI)
3841	continue;
3842	rc = atapi_eh_clear_ua(dev);
3843	if (rc)
3844	goto rest_fail;
3845	if (zpodd_dev_enabled(dev))
3846	zpodd_post_poweron(dev);
3847	}
3848	}
3849
3850	/*
3851	* Make sure to transition devices to the active power mode
3852	* if needed (e.g. if we were scheduled on system resume).
3853	*/
3854	ata_for_each_dev(dev, link, ENABLED) {
3855	if (ehc->i.dev_action[dev->devno] & ATA_EH_SET_ACTIVE) {
3856	ata_dev_power_set_active(dev);
3857	ata_eh_done(link, dev, action: ATA_EH_SET_ACTIVE);
3858	}
3859	}
3860
3861	/* retry flush if necessary */
3862	ata_for_each_dev(dev, link, ALL) {
3863	if (dev->class != ATA_DEV_ATA &&
3864	dev->class != ATA_DEV_ZAC)
3865	continue;
3866	rc = ata_eh_maybe_retry_flush(dev);
3867	if (rc)
3868	goto rest_fail;
3869	}
3870
3871	config_lpm:
3872	/* configure link power saving */
3873	if (link->lpm_policy != ap->target_lpm_policy) {
3874	rc = ata_eh_set_lpm(link, policy: ap->target_lpm_policy, r_failed_dev: &dev);
3875	if (rc)
3876	goto rest_fail;
3877	}
3878
3879	/* this link is okay now */
3880	ehc->i.flags = 0;
3881	continue;
3882
3883	rest_fail:
3884	nr_fails++;
3885	if (dev)
3886	ata_eh_handle_dev_fail(dev, err: rc);
3887
3888	if (ata_port_is_frozen(ap)) {
3889	/* PMP reset requires working host port.
3890	* Can't retry if it's frozen.
3891	*/
3892	if (sata_pmp_attached(ap))
3893	goto out;
3894	break;
3895	}
3896	}
3897
3898	if (nr_fails)
3899	goto retry;
3900
3901	out:
3902	if (rc && r_failed_link)
3903	*r_failed_link = link;
3904
3905	return rc;
3906	}
3907
3908	/**
3909	* ata_eh_finish - finish up EH
3910	* @ap: host port to finish EH for
3911	*
3912	* Recovery is complete. Clean up EH states and retry or finish
3913	* failed qcs.
3914	*
3915	* LOCKING:
3916	* None.
3917	*/
3918	void ata_eh_finish(struct ata_port *ap)
3919	{
3920	struct ata_queued_cmd *qc;
3921	int tag;
3922
3923	/* retry or finish qcs */
3924	ata_qc_for_each_raw(ap, qc, tag) {
3925	if (!(qc->flags & ATA_QCFLAG_EH))
3926	continue;
3927
3928	if (qc->err_mask) {
3929	/* FIXME: Once EH migration is complete,
3930	* generate sense data in this function,
3931	* considering both err_mask and tf.
3932	*/
3933	if (qc->flags & ATA_QCFLAG_RETRY) {
3934	/*
3935	* Since qc->err_mask is set, ata_eh_qc_retry()
3936	* will not increment scmd->allowed, so upper
3937	* layer will only retry the command if it has
3938	* not already been retried too many times.
3939	*/
3940	ata_eh_qc_retry(qc);
3941	} else {
3942	ata_eh_qc_complete(qc);
3943	}
3944	} else {
3945	if (qc->flags & ATA_QCFLAG_SENSE_VALID ||
3946	qc->flags & ATA_QCFLAG_EH_SUCCESS_CMD) {
3947	ata_eh_qc_complete(qc);
3948	} else {
3949	/* feed zero TF to sense generation */
3950	memset(&qc->result_tf, 0, sizeof(qc->result_tf));
3951	/*
3952	* Since qc->err_mask is not set,
3953	* ata_eh_qc_retry() will increment
3954	* scmd->allowed, so upper layer is guaranteed
3955	* to retry the command.
3956	*/
3957	ata_eh_qc_retry(qc);
3958	}
3959	}
3960	}
3961
3962	/* make sure nr_active_links is zero after EH */
3963	WARN_ON(ap->nr_active_links);
3964	ap->nr_active_links = 0;
3965	}
3966
3967	/**
3968	* ata_do_eh - do standard error handling
3969	* @ap: host port to handle error for
3970	*
3971	* @prereset: prereset method (can be NULL)
3972	* @softreset: softreset method (can be NULL)
3973	* @hardreset: hardreset method (can be NULL)
3974	* @postreset: postreset method (can be NULL)
3975	*
3976	* Perform standard error handling sequence.
3977	*
3978	* LOCKING:
3979	* Kernel thread context (may sleep).
3980	*/
3981	void ata_do_eh(struct ata_port *ap, ata_prereset_fn_t prereset,
3982	ata_reset_fn_t softreset, ata_reset_fn_t hardreset,
3983	ata_postreset_fn_t postreset)
3984	{
3985	struct ata_device *dev;
3986	int rc;
3987
3988	ata_eh_autopsy(ap);
3989	ata_eh_report(ap);
3990
3991	rc = ata_eh_recover(ap, prereset, softreset, hardreset, postreset,
3992	NULL);
3993	if (rc) {
3994	ata_for_each_dev(dev, &ap->link, ALL)
3995	ata_dev_disable(dev);
3996	}
3997
3998	ata_eh_finish(ap);
3999	}
4000
4001	/**
4002	* ata_std_error_handler - standard error handler
4003	* @ap: host port to handle error for
4004	*
4005	* Standard error handler
4006	*
4007	* LOCKING:
4008	* Kernel thread context (may sleep).
4009	*/
4010	void ata_std_error_handler(struct ata_port *ap)
4011	{
4012	struct ata_port_operations *ops = ap->ops;
4013	ata_reset_fn_t hardreset = ops->hardreset;
4014
4015	/* ignore built-in hardreset if SCR access is not available */
4016	if (hardreset == sata_std_hardreset && !sata_scr_valid(link: &ap->link))
4017	hardreset = NULL;
4018
4019	ata_do_eh(ap, prereset: ops->prereset, softreset: ops->softreset, hardreset, postreset: ops->postreset);
4020	}
4021	EXPORT_SYMBOL_GPL(ata_std_error_handler);
4022
4023	#ifdef CONFIG_PM
4024	/**
4025	* ata_eh_handle_port_suspend - perform port suspend operation
4026	* @ap: port to suspend
4027	*
4028	* Suspend @ap.
4029	*
4030	* LOCKING:
4031	* Kernel thread context (may sleep).
4032	*/
4033	static void ata_eh_handle_port_suspend(struct ata_port *ap)
4034	{
4035	unsigned long flags;
4036	int rc = 0;
4037	struct ata_device *dev;
4038	struct ata_link *link;
4039
4040	/* are we suspending? */
4041	spin_lock_irqsave(ap->lock, flags);
4042	if (!(ap->pflags & ATA_PFLAG_PM_PENDING) ||
4043	ap->pm_mesg.event & PM_EVENT_RESUME) {
4044	spin_unlock_irqrestore(lock: ap->lock, flags);
4045	return;
4046	}
4047	spin_unlock_irqrestore(lock: ap->lock, flags);
4048
4049	WARN_ON(ap->pflags & ATA_PFLAG_SUSPENDED);
4050
4051	/* Set all devices attached to the port in standby mode */
4052	ata_for_each_link(link, ap, HOST_FIRST) {
4053	ata_for_each_dev(dev, link, ENABLED)
4054	ata_dev_power_set_standby(dev);
4055	}
4056
4057	/*
4058	* If we have a ZPODD attached, check its zero
4059	* power ready status before the port is frozen.
4060	* Only needed for runtime suspend.
4061	*/
4062	if (PMSG_IS_AUTO(ap->pm_mesg)) {
4063	ata_for_each_dev(dev, &ap->link, ENABLED) {
4064	if (zpodd_dev_enabled(dev))
4065	zpodd_on_suspend(dev);
4066	}
4067	}
4068
4069	/* suspend */
4070	ata_eh_freeze_port(ap);
4071
4072	if (ap->ops->port_suspend)
4073	rc = ap->ops->port_suspend(ap, ap->pm_mesg);
4074
4075	ata_acpi_set_state(ap, state: ap->pm_mesg);
4076
4077	/* update the flags */
4078	spin_lock_irqsave(ap->lock, flags);
4079
4080	ap->pflags &= ~ATA_PFLAG_PM_PENDING;
4081	if (rc == 0)
4082	ap->pflags |= ATA_PFLAG_SUSPENDED;
4083	else if (ata_port_is_frozen(ap))
4084	ata_port_schedule_eh(ap);
4085
4086	spin_unlock_irqrestore(lock: ap->lock, flags);
4087
4088	return;
4089	}
4090
4091	/**
4092	* ata_eh_handle_port_resume - perform port resume operation
4093	* @ap: port to resume
4094	*
4095	* Resume @ap.
4096	*
4097	* LOCKING:
4098	* Kernel thread context (may sleep).
4099	*/
4100	static void ata_eh_handle_port_resume(struct ata_port *ap)
4101	{
4102	struct ata_link *link;
4103	struct ata_device *dev;
4104	unsigned long flags;
4105
4106	/* are we resuming? */
4107	spin_lock_irqsave(ap->lock, flags);
4108	if (!(ap->pflags & ATA_PFLAG_PM_PENDING) ||
4109	!(ap->pm_mesg.event & PM_EVENT_RESUME)) {
4110	spin_unlock_irqrestore(lock: ap->lock, flags);
4111	return;
4112	}
4113	spin_unlock_irqrestore(lock: ap->lock, flags);
4114
4115	WARN_ON(!(ap->pflags & ATA_PFLAG_SUSPENDED));
4116
4117	/*
4118	* Error timestamps are in jiffies which doesn't run while
4119	* suspended and PHY events during resume isn't too uncommon.
4120	* When the two are combined, it can lead to unnecessary speed
4121	* downs if the machine is suspended and resumed repeatedly.
4122	* Clear error history.
4123	*/
4124	ata_for_each_link(link, ap, HOST_FIRST)
4125	ata_for_each_dev(dev, link, ALL)
4126	ata_ering_clear(ering: &dev->ering);
4127
4128	ata_acpi_set_state(ap, state: ap->pm_mesg);
4129
4130	if (ap->ops->port_resume)
4131	ap->ops->port_resume(ap);
4132
4133	/* tell ACPI that we're resuming */
4134	ata_acpi_on_resume(ap);
4135
4136	/* update the flags */
4137	spin_lock_irqsave(ap->lock, flags);
4138	ap->pflags &= ~(ATA_PFLAG_PM_PENDING | ATA_PFLAG_SUSPENDED);
4139	ap->pflags |= ATA_PFLAG_RESUMING;
4140	spin_unlock_irqrestore(lock: ap->lock, flags);
4141	}
4142	#endif /* CONFIG_PM */
4143