1	/*
2	* Scsi Host Layer for MPT (Message Passing Technology) based controllers
3	*
4	* This code is based on drivers/scsi/mpt3sas/mpt3sas_scsih.c
5	* Copyright (C) 2012-2014 LSI Corporation
6	* Copyright (C) 2013-2014 Avago Technologies
7	* (mailto: MPT-FusionLinux.pdl@avagotech.com)
8	*
9	* This program is free software; you can redistribute it and/or
10	* modify it under the terms of the GNU General Public License
11	* as published by the Free Software Foundation; either version 2
12	* of the License, or (at your option) any later version.
13	*
14	* This program is distributed in the hope that it will be useful,
15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17	* GNU General Public License for more details.
18	*
19	* NO WARRANTY
20	* THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
21	* CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
22	* LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
23	* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
24	* solely responsible for determining the appropriateness of using and
25	* distributing the Program and assumes all risks associated with its
26	* exercise of rights under this Agreement, including but not limited to
27	* the risks and costs of program errors, damage to or loss of data,
28	* programs or equipment, and unavailability or interruption of operations.
29
30	* DISCLAIMER OF LIABILITY
31	* NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
32	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33	* DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
34	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
35	* TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
36	* USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
37	* HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
38
39	* You should have received a copy of the GNU General Public License
40	* along with this program; if not, write to the Free Software
41	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
42	* USA.
43	*/
44
45	#include <linux/module.h>
46	#include <linux/kernel.h>
47	#include <linux/init.h>
48	#include <linux/errno.h>
49	#include <linux/blkdev.h>
50	#include <linux/sched.h>
51	#include <linux/workqueue.h>
52	#include <linux/delay.h>
53	#include <linux/pci.h>
54	#include <linux/interrupt.h>
55	#include <linux/raid_class.h>
56	#include <linux/blk-mq-pci.h>
57	#include <asm/unaligned.h>
58
59	#include "mpt3sas_base.h"
60
61	#define RAID_CHANNEL 1
62
63	#define PCIE_CHANNEL 2
64
65	/* forward proto's */
66	static void _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,
67	struct _sas_node *sas_expander);
68	static void _firmware_event_work(struct work_struct *work);
69
70	static void _scsih_remove_device(struct MPT3SAS_ADAPTER *ioc,
71	struct _sas_device *sas_device);
72	static int _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle,
73	u8 retry_count, u8 is_pd);
74	static int _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle);
75	static void _scsih_pcie_device_remove_from_sml(struct MPT3SAS_ADAPTER *ioc,
76	struct _pcie_device *pcie_device);
77	static void
78	_scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle);
79	static u8 _scsih_check_for_pending_tm(struct MPT3SAS_ADAPTER *ioc, u16 smid);
80	static void _scsih_complete_devices_scanning(struct MPT3SAS_ADAPTER *ioc);
81
82	/* global parameters */
83	LIST_HEAD(mpt3sas_ioc_list);
84	/* global ioc lock for list operations */
85	DEFINE_SPINLOCK(gioc_lock);
86
87	MODULE_AUTHOR(MPT3SAS_AUTHOR);
88	MODULE_DESCRIPTION(MPT3SAS_DESCRIPTION);
89	MODULE_LICENSE("GPL");
90	MODULE_VERSION(MPT3SAS_DRIVER_VERSION);
91	MODULE_ALIAS("mpt2sas");
92
93	/* local parameters */
94	static u8 scsi_io_cb_idx = -1;
95	static u8 tm_cb_idx = -1;
96	static u8 ctl_cb_idx = -1;
97	static u8 base_cb_idx = -1;
98	static u8 port_enable_cb_idx = -1;
99	static u8 transport_cb_idx = -1;
100	static u8 scsih_cb_idx = -1;
101	static u8 config_cb_idx = -1;
102	static int mpt2_ids;
103	static int mpt3_ids;
104
105	static u8 tm_tr_cb_idx = -1 ;
106	static u8 tm_tr_volume_cb_idx = -1 ;
107	static u8 tm_sas_control_cb_idx = -1;
108
109	/* command line options */
110	static u32 logging_level;
111	MODULE_PARM_DESC(logging_level,
112	" bits for enabling additional logging info (default=0)");
113
114
115	static ushort max_sectors = 0xFFFF;
116	module_param(max_sectors, ushort, 0444);
117	MODULE_PARM_DESC(max_sectors, "max sectors, range 64 to 32767 default=32767");
118
119
120	static int missing_delay[2] = {-1, -1};
121	module_param_array(missing_delay, int, NULL, 0444);
122	MODULE_PARM_DESC(missing_delay, " device missing delay , io missing delay");
123
124	/* scsi-mid layer global parmeter is max_report_luns, which is 511 */
125	#define MPT3SAS_MAX_LUN (16895)
126	static u64 max_lun = MPT3SAS_MAX_LUN;
127	module_param(max_lun, ullong, 0444);
128	MODULE_PARM_DESC(max_lun, " max lun, default=16895 ");
129
130	static ushort hbas_to_enumerate;
131	module_param(hbas_to_enumerate, ushort, 0444);
132	MODULE_PARM_DESC(hbas_to_enumerate,
133	" 0 - enumerates both SAS 2.0 & SAS 3.0 generation HBAs\n \
134	1 - enumerates only SAS 2.0 generation HBAs\n \
135	2 - enumerates only SAS 3.0 generation HBAs (default=0)");
136
137	/* diag_buffer_enable is bitwise
138	* bit 0 set = TRACE
139	* bit 1 set = SNAPSHOT
140	* bit 2 set = EXTENDED
141	*
142	* Either bit can be set, or both
143	*/
144	static int diag_buffer_enable = -1;
145	module_param(diag_buffer_enable, int, 0444);
146	MODULE_PARM_DESC(diag_buffer_enable,
147	" post diag buffers (TRACE=1/SNAPSHOT=2/EXTENDED=4/default=0)");
148	static int disable_discovery = -1;
149	module_param(disable_discovery, int, 0444);
150	MODULE_PARM_DESC(disable_discovery, " disable discovery ");
151
152
153	/* permit overriding the host protection capabilities mask (EEDP/T10 PI) */
154	static int prot_mask = -1;
155	module_param(prot_mask, int, 0444);
156	MODULE_PARM_DESC(prot_mask, " host protection capabilities mask, def=7 ");
157
158	static bool enable_sdev_max_qd;
159	module_param(enable_sdev_max_qd, bool, 0444);
160	MODULE_PARM_DESC(enable_sdev_max_qd,
161	"Enable sdev max qd as can_queue, def=disabled(0)");
162
163	static int multipath_on_hba = -1;
164	module_param(multipath_on_hba, int, 0);
165	MODULE_PARM_DESC(multipath_on_hba,
166	"Multipath support to add same target device\n\t\t"
167	"as many times as it is visible to HBA from various paths\n\t\t"
168	"(by default:\n\t\t"
169	"\t SAS 2.0 & SAS 3.0 HBA - This will be disabled,\n\t\t"
170	"\t SAS 3.5 HBA - This will be enabled)");
171
172	static int host_tagset_enable = 1;
173	module_param(host_tagset_enable, int, 0444);
174	MODULE_PARM_DESC(host_tagset_enable,
175	"Shared host tagset enable/disable Default: enable(1)");
176
177	/* raid transport support */
178	static struct raid_template *mpt3sas_raid_template;
179	static struct raid_template *mpt2sas_raid_template;
180
181
182	/**
183	* struct sense_info - common structure for obtaining sense keys
184	* @skey: sense key
185	* @asc: additional sense code
186	* @ascq: additional sense code qualifier
187	*/
188	struct sense_info {
189	u8 skey;
190	u8 asc;
191	u8 ascq;
192	};
193
194	#define MPT3SAS_PROCESS_TRIGGER_DIAG (0xFFFB)
195	#define MPT3SAS_TURN_ON_PFA_LED (0xFFFC)
196	#define MPT3SAS_PORT_ENABLE_COMPLETE (0xFFFD)
197	#define MPT3SAS_ABRT_TASK_SET (0xFFFE)
198	#define MPT3SAS_REMOVE_UNRESPONDING_DEVICES (0xFFFF)
199	/**
200	* struct fw_event_work - firmware event struct
201	* @list: link list framework
202	* @work: work object (ioc->fault_reset_work_q)
203	* @ioc: per adapter object
204	* @device_handle: device handle
205	* @VF_ID: virtual function id
206	* @VP_ID: virtual port id
207	* @ignore: flag meaning this event has been marked to ignore
208	* @event: firmware event MPI2_EVENT_XXX defined in mpi2_ioc.h
209	* @refcount: kref for this event
210	* @event_data: reply event data payload follows
211	*
212	* This object stored on ioc->fw_event_list.
213	*/
214	struct fw_event_work {
215	struct list_head list;
216	struct work_struct work;
217
218	struct MPT3SAS_ADAPTER *ioc;
219	u16 device_handle;
220	u8 VF_ID;
221	u8 VP_ID;
222	u8 ignore;
223	u16 event;
224	struct kref refcount;
225	char event_data[] __aligned(4);
226	};
227
228	static void fw_event_work_free(struct kref *r)
229	{
230	kfree(container_of(r, struct fw_event_work, refcount));
231	}
232
233	static void fw_event_work_get(struct fw_event_work *fw_work)
234	{
235	kref_get(kref: &fw_work->refcount);
236	}
237
238	static void fw_event_work_put(struct fw_event_work *fw_work)
239	{
240	kref_put(kref: &fw_work->refcount, release: fw_event_work_free);
241	}
242
243	static struct fw_event_work *alloc_fw_event_work(int len)
244	{
245	struct fw_event_work *fw_event;
246
247	fw_event = kzalloc(size: sizeof(*fw_event) + len, GFP_ATOMIC);
248	if (!fw_event)
249	return NULL;
250
251	kref_init(kref: &fw_event->refcount);
252	return fw_event;
253	}
254
255	/**
256	* struct _scsi_io_transfer - scsi io transfer
257	* @handle: sas device handle (assigned by firmware)
258	* @is_raid: flag set for hidden raid components
259	* @dir: DMA_TO_DEVICE, DMA_FROM_DEVICE,
260	* @data_length: data transfer length
261	* @data_dma: dma pointer to data
262	* @sense: sense data
263	* @lun: lun number
264	* @cdb_length: cdb length
265	* @cdb: cdb contents
266	* @timeout: timeout for this command
267	* @VF_ID: virtual function id
268	* @VP_ID: virtual port id
269	* @valid_reply: flag set for reply message
270	* @sense_length: sense length
271	* @ioc_status: ioc status
272	* @scsi_state: scsi state
273	* @scsi_status: scsi staus
274	* @log_info: log information
275	* @transfer_length: data length transfer when there is a reply message
276	*
277	* Used for sending internal scsi commands to devices within this module.
278	* Refer to _scsi_send_scsi_io().
279	*/
280	struct _scsi_io_transfer {
281	u16 handle;
282	u8 is_raid;
283	enum dma_data_direction dir;
284	u32 data_length;
285	dma_addr_t data_dma;
286	u8 sense[SCSI_SENSE_BUFFERSIZE];
287	u32 lun;
288	u8 cdb_length;
289	u8 cdb[32];
290	u8 timeout;
291	u8 VF_ID;
292	u8 VP_ID;
293	u8 valid_reply;
294	/* the following bits are only valid when 'valid_reply = 1' */
295	u32 sense_length;
296	u16 ioc_status;
297	u8 scsi_state;
298	u8 scsi_status;
299	u32 log_info;
300	u32 transfer_length;
301	};
302
303	/**
304	* _scsih_set_debug_level - global setting of ioc->logging_level.
305	* @val: ?
306	* @kp: ?
307	*
308	* Note: The logging levels are defined in mpt3sas_debug.h.
309	*/
310	static int
311	_scsih_set_debug_level(const char *val, const struct kernel_param *kp)
312	{
313	int ret = param_set_int(val, kp);
314	struct MPT3SAS_ADAPTER *ioc;
315
316	if (ret)
317	return ret;
318
319	pr_info("setting logging_level(0x%08x)\n", logging_level);
320	spin_lock(lock: &gioc_lock);
321	list_for_each_entry(ioc, &mpt3sas_ioc_list, list)
322	ioc->logging_level = logging_level;
323	spin_unlock(lock: &gioc_lock);
324	return 0;
325	}
326	module_param_call(logging_level, _scsih_set_debug_level, param_get_int,
327	&logging_level, 0644);
328
329	/**
330	* _scsih_srch_boot_sas_address - search based on sas_address
331	* @sas_address: sas address
332	* @boot_device: boot device object from bios page 2
333	*
334	* Return: 1 when there's a match, 0 means no match.
335	*/
336	static inline int
337	_scsih_srch_boot_sas_address(u64 sas_address,
338	Mpi2BootDeviceSasWwid_t *boot_device)
339	{
340	return (sas_address == le64_to_cpu(boot_device->SASAddress)) ? 1 : 0;
341	}
342
343	/**
344	* _scsih_srch_boot_device_name - search based on device name
345	* @device_name: device name specified in INDENTIFY fram
346	* @boot_device: boot device object from bios page 2
347	*
348	* Return: 1 when there's a match, 0 means no match.
349	*/
350	static inline int
351	_scsih_srch_boot_device_name(u64 device_name,
352	Mpi2BootDeviceDeviceName_t *boot_device)
353	{
354	return (device_name == le64_to_cpu(boot_device->DeviceName)) ? 1 : 0;
355	}
356
357	/**
358	* _scsih_srch_boot_encl_slot - search based on enclosure_logical_id/slot
359	* @enclosure_logical_id: enclosure logical id
360	* @slot_number: slot number
361	* @boot_device: boot device object from bios page 2
362	*
363	* Return: 1 when there's a match, 0 means no match.
364	*/
365	static inline int
366	_scsih_srch_boot_encl_slot(u64 enclosure_logical_id, u16 slot_number,
367	Mpi2BootDeviceEnclosureSlot_t *boot_device)
368	{
369	return (enclosure_logical_id == le64_to_cpu(boot_device->
370	EnclosureLogicalID) && slot_number == le16_to_cpu(boot_device->
371	SlotNumber)) ? 1 : 0;
372	}
373
374	/**
375	* mpt3sas_get_port_by_id - get hba port entry corresponding to provided
376	* port number from port list
377	* @ioc: per adapter object
378	* @port_id: port number
379	* @bypass_dirty_port_flag: when set look the matching hba port entry even
380	* if hba port entry is marked as dirty.
381	*
382	* Search for hba port entry corresponding to provided port number,
383	* if available return port object otherwise return NULL.
384	*/
385	struct hba_port *
386	mpt3sas_get_port_by_id(struct MPT3SAS_ADAPTER *ioc,
387	u8 port_id, u8 bypass_dirty_port_flag)
388	{
389	struct hba_port *port, *port_next;
390
391	/*
392	* When multipath_on_hba is disabled then
393	* search the hba_port entry using default
394	* port id i.e. 255
395	*/
396	if (!ioc->multipath_on_hba)
397	port_id = MULTIPATH_DISABLED_PORT_ID;
398
399	list_for_each_entry_safe(port, port_next,
400	&ioc->port_table_list, list) {
401	if (port->port_id != port_id)
402	continue;
403	if (bypass_dirty_port_flag)
404	return port;
405	if (port->flags & HBA_PORT_FLAG_DIRTY_PORT)
406	continue;
407	return port;
408	}
409
410	/*
411	* Allocate hba_port object for default port id (i.e. 255)
412	* when multipath_on_hba is disabled for the HBA.
413	* And add this object to port_table_list.
414	*/
415	if (!ioc->multipath_on_hba) {
416	port = kzalloc(size: sizeof(struct hba_port), GFP_ATOMIC);
417	if (!port)
418	return NULL;
419
420	port->port_id = port_id;
421	ioc_info(ioc,
422	"hba_port entry: %p, port: %d is added to hba_port list\n",
423	port, port->port_id);
424	list_add_tail(new: &port->list,
425	head: &ioc->port_table_list);
426	return port;
427	}
428	return NULL;
429	}
430
431	/**
432	* mpt3sas_get_vphy_by_phy - get virtual_phy object corresponding to phy number
433	* @ioc: per adapter object
434	* @port: hba_port object
435	* @phy: phy number
436	*
437	* Return virtual_phy object corresponding to phy number.
438	*/
439	struct virtual_phy *
440	mpt3sas_get_vphy_by_phy(struct MPT3SAS_ADAPTER *ioc,
441	struct hba_port *port, u32 phy)
442	{
443	struct virtual_phy *vphy, *vphy_next;
444
445	if (!port->vphys_mask)
446	return NULL;
447
448	list_for_each_entry_safe(vphy, vphy_next, &port->vphys_list, list) {
449	if (vphy->phy_mask & (1 << phy))
450	return vphy;
451	}
452	return NULL;
453	}
454
455	/**
456	* _scsih_is_boot_device - search for matching boot device.
457	* @sas_address: sas address
458	* @device_name: device name specified in INDENTIFY fram
459	* @enclosure_logical_id: enclosure logical id
460	* @slot: slot number
461	* @form: specifies boot device form
462	* @boot_device: boot device object from bios page 2
463	*
464	* Return: 1 when there's a match, 0 means no match.
465	*/
466	static int
467	_scsih_is_boot_device(u64 sas_address, u64 device_name,
468	u64 enclosure_logical_id, u16 slot, u8 form,
469	Mpi2BiosPage2BootDevice_t *boot_device)
470	{
471	int rc = 0;
472
473	switch (form) {
474	case MPI2_BIOSPAGE2_FORM_SAS_WWID:
475	if (!sas_address)
476	break;
477	rc = _scsih_srch_boot_sas_address(
478	sas_address, boot_device: &boot_device->SasWwid);
479	break;
480	case MPI2_BIOSPAGE2_FORM_ENCLOSURE_SLOT:
481	if (!enclosure_logical_id)
482	break;
483	rc = _scsih_srch_boot_encl_slot(
484	enclosure_logical_id,
485	slot_number: slot, boot_device: &boot_device->EnclosureSlot);
486	break;
487	case MPI2_BIOSPAGE2_FORM_DEVICE_NAME:
488	if (!device_name)
489	break;
490	rc = _scsih_srch_boot_device_name(
491	device_name, boot_device: &boot_device->DeviceName);
492	break;
493	case MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED:
494	break;
495	}
496
497	return rc;
498	}
499
500	/**
501	* _scsih_get_sas_address - set the sas_address for given device handle
502	* @ioc: ?
503	* @handle: device handle
504	* @sas_address: sas address
505	*
506	* Return: 0 success, non-zero when failure
507	*/
508	static int
509	_scsih_get_sas_address(struct MPT3SAS_ADAPTER *ioc, u16 handle,
510	u64 *sas_address)
511	{
512	Mpi2SasDevicePage0_t sas_device_pg0;
513	Mpi2ConfigReply_t mpi_reply;
514	u32 ioc_status;
515
516	*sas_address = 0;
517
518	if ((mpt3sas_config_get_sas_device_pg0(ioc, mpi_reply: &mpi_reply, config_page: &sas_device_pg0,
519	MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
520	ioc_err(ioc, "failure at %s:%d/%s()!\n",
521	__FILE__, __LINE__, __func__);
522	return -ENXIO;
523	}
524
525	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
526	if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
527	/* For HBA, vSES doesn't return HBA SAS address. Instead return
528	* vSES's sas address.
529	*/
530	if ((handle <= ioc->sas_hba.num_phys) &&
531	(!(le32_to_cpu(sas_device_pg0.DeviceInfo) &
532	MPI2_SAS_DEVICE_INFO_SEP)))
533	*sas_address = ioc->sas_hba.sas_address;
534	else
535	*sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
536	return 0;
537	}
538
539	/* we hit this because the given parent handle doesn't exist */
540	if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE)
541	return -ENXIO;
542
543	/* else error case */
544	ioc_err(ioc, "handle(0x%04x), ioc_status(0x%04x), failure at %s:%d/%s()!\n",
545	handle, ioc_status, __FILE__, __LINE__, __func__);
546	return -EIO;
547	}
548
549	/**
550	* _scsih_determine_boot_device - determine boot device.
551	* @ioc: per adapter object
552	* @device: sas_device or pcie_device object
553	* @channel: SAS or PCIe channel
554	*
555	* Determines whether this device should be first reported device to
556	* to scsi-ml or sas transport, this purpose is for persistent boot device.
557	* There are primary, alternate, and current entries in bios page 2. The order
558	* priority is primary, alternate, then current. This routine saves
559	* the corresponding device object.
560	* The saved data to be used later in _scsih_probe_boot_devices().
561	*/
562	static void
563	_scsih_determine_boot_device(struct MPT3SAS_ADAPTER *ioc, void *device,
564	u32 channel)
565	{
566	struct _sas_device *sas_device;
567	struct _pcie_device *pcie_device;
568	struct _raid_device *raid_device;
569	u64 sas_address;
570	u64 device_name;
571	u64 enclosure_logical_id;
572	u16 slot;
573
574	/* only process this function when driver loads */
575	if (!ioc->is_driver_loading)
576	return;
577
578	/* no Bios, return immediately */
579	if (!ioc->bios_pg3.BiosVersion)
580	return;
581
582	if (channel == RAID_CHANNEL) {
583	raid_device = device;
584	sas_address = raid_device->wwid;
585	device_name = 0;
586	enclosure_logical_id = 0;
587	slot = 0;
588	} else if (channel == PCIE_CHANNEL) {
589	pcie_device = device;
590	sas_address = pcie_device->wwid;
591	device_name = 0;
592	enclosure_logical_id = 0;
593	slot = 0;
594	} else {
595	sas_device = device;
596	sas_address = sas_device->sas_address;
597	device_name = sas_device->device_name;
598	enclosure_logical_id = sas_device->enclosure_logical_id;
599	slot = sas_device->slot;
600	}
601
602	if (!ioc->req_boot_device.device) {
603	if (_scsih_is_boot_device(sas_address, device_name,
604	enclosure_logical_id, slot,
605	form: (ioc->bios_pg2.ReqBootDeviceForm &
606	MPI2_BIOSPAGE2_FORM_MASK),
607	boot_device: &ioc->bios_pg2.RequestedBootDevice)) {
608	dinitprintk(ioc,
609	ioc_info(ioc, "%s: req_boot_device(0x%016llx)\n",
610	__func__, (u64)sas_address));
611	ioc->req_boot_device.device = device;
612	ioc->req_boot_device.channel = channel;
613	}
614	}
615
616	if (!ioc->req_alt_boot_device.device) {
617	if (_scsih_is_boot_device(sas_address, device_name,
618	enclosure_logical_id, slot,
619	form: (ioc->bios_pg2.ReqAltBootDeviceForm &
620	MPI2_BIOSPAGE2_FORM_MASK),
621	boot_device: &ioc->bios_pg2.RequestedAltBootDevice)) {
622	dinitprintk(ioc,
623	ioc_info(ioc, "%s: req_alt_boot_device(0x%016llx)\n",
624	__func__, (u64)sas_address));
625	ioc->req_alt_boot_device.device = device;
626	ioc->req_alt_boot_device.channel = channel;
627	}
628	}
629
630	if (!ioc->current_boot_device.device) {
631	if (_scsih_is_boot_device(sas_address, device_name,
632	enclosure_logical_id, slot,
633	form: (ioc->bios_pg2.CurrentBootDeviceForm &
634	MPI2_BIOSPAGE2_FORM_MASK),
635	boot_device: &ioc->bios_pg2.CurrentBootDevice)) {
636	dinitprintk(ioc,
637	ioc_info(ioc, "%s: current_boot_device(0x%016llx)\n",
638	__func__, (u64)sas_address));
639	ioc->current_boot_device.device = device;
640	ioc->current_boot_device.channel = channel;
641	}
642	}
643	}
644
645	static struct _sas_device *
646	__mpt3sas_get_sdev_from_target(struct MPT3SAS_ADAPTER *ioc,
647	struct MPT3SAS_TARGET *tgt_priv)
648	{
649	struct _sas_device *ret;
650
651	assert_spin_locked(&ioc->sas_device_lock);
652
653	ret = tgt_priv->sas_dev;
654	if (ret)
655	sas_device_get(s: ret);
656
657	return ret;
658	}
659
660	static struct _sas_device *
661	mpt3sas_get_sdev_from_target(struct MPT3SAS_ADAPTER *ioc,
662	struct MPT3SAS_TARGET *tgt_priv)
663	{
664	struct _sas_device *ret;
665	unsigned long flags;
666
667	spin_lock_irqsave(&ioc->sas_device_lock, flags);
668	ret = __mpt3sas_get_sdev_from_target(ioc, tgt_priv);
669	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
670
671	return ret;
672	}
673
674	static struct _pcie_device *
675	__mpt3sas_get_pdev_from_target(struct MPT3SAS_ADAPTER *ioc,
676	struct MPT3SAS_TARGET *tgt_priv)
677	{
678	struct _pcie_device *ret;
679
680	assert_spin_locked(&ioc->pcie_device_lock);
681
682	ret = tgt_priv->pcie_dev;
683	if (ret)
684	pcie_device_get(p: ret);
685
686	return ret;
687	}
688
689	/**
690	* mpt3sas_get_pdev_from_target - pcie device search
691	* @ioc: per adapter object
692	* @tgt_priv: starget private object
693	*
694	* Context: This function will acquire ioc->pcie_device_lock and will release
695	* before returning the pcie_device object.
696	*
697	* This searches for pcie_device from target, then return pcie_device object.
698	*/
699	static struct _pcie_device *
700	mpt3sas_get_pdev_from_target(struct MPT3SAS_ADAPTER *ioc,
701	struct MPT3SAS_TARGET *tgt_priv)
702	{
703	struct _pcie_device *ret;
704	unsigned long flags;
705
706	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
707	ret = __mpt3sas_get_pdev_from_target(ioc, tgt_priv);
708	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
709
710	return ret;
711	}
712
713
714	/**
715	* __mpt3sas_get_sdev_by_rphy - sas device search
716	* @ioc: per adapter object
717	* @rphy: sas_rphy pointer
718	*
719	* Context: This function will acquire ioc->sas_device_lock and will release
720	* before returning the sas_device object.
721	*
722	* This searches for sas_device from rphy object
723	* then return sas_device object.
724	*/
725	struct _sas_device *
726	__mpt3sas_get_sdev_by_rphy(struct MPT3SAS_ADAPTER *ioc,
727	struct sas_rphy *rphy)
728	{
729	struct _sas_device *sas_device;
730
731	assert_spin_locked(&ioc->sas_device_lock);
732
733	list_for_each_entry(sas_device, &ioc->sas_device_list, list) {
734	if (sas_device->rphy != rphy)
735	continue;
736	sas_device_get(s: sas_device);
737	return sas_device;
738	}
739
740	sas_device = NULL;
741	list_for_each_entry(sas_device, &ioc->sas_device_init_list, list) {
742	if (sas_device->rphy != rphy)
743	continue;
744	sas_device_get(s: sas_device);
745	return sas_device;
746	}
747
748	return NULL;
749	}
750
751	/**
752	* __mpt3sas_get_sdev_by_addr - get _sas_device object corresponding to provided
753	* sas address from sas_device_list list
754	* @ioc: per adapter object
755	* @sas_address: device sas address
756	* @port: port number
757	*
758	* Search for _sas_device object corresponding to provided sas address,
759	* if available return _sas_device object address otherwise return NULL.
760	*/
761	struct _sas_device *
762	__mpt3sas_get_sdev_by_addr(struct MPT3SAS_ADAPTER *ioc,
763	u64 sas_address, struct hba_port *port)
764	{
765	struct _sas_device *sas_device;
766
767	if (!port)
768	return NULL;
769
770	assert_spin_locked(&ioc->sas_device_lock);
771
772	list_for_each_entry(sas_device, &ioc->sas_device_list, list) {
773	if (sas_device->sas_address != sas_address)
774	continue;
775	if (sas_device->port != port)
776	continue;
777	sas_device_get(s: sas_device);
778	return sas_device;
779	}
780
781	list_for_each_entry(sas_device, &ioc->sas_device_init_list, list) {
782	if (sas_device->sas_address != sas_address)
783	continue;
784	if (sas_device->port != port)
785	continue;
786	sas_device_get(s: sas_device);
787	return sas_device;
788	}
789
790	return NULL;
791	}
792
793	/**
794	* mpt3sas_get_sdev_by_addr - sas device search
795	* @ioc: per adapter object
796	* @sas_address: sas address
797	* @port: hba port entry
798	* Context: Calling function should acquire ioc->sas_device_lock
799	*
800	* This searches for sas_device based on sas_address & port number,
801	* then return sas_device object.
802	*/
803	struct _sas_device *
804	mpt3sas_get_sdev_by_addr(struct MPT3SAS_ADAPTER *ioc,
805	u64 sas_address, struct hba_port *port)
806	{
807	struct _sas_device *sas_device;
808	unsigned long flags;
809
810	spin_lock_irqsave(&ioc->sas_device_lock, flags);
811	sas_device = __mpt3sas_get_sdev_by_addr(ioc,
812	sas_address, port);
813	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
814
815	return sas_device;
816	}
817
818	static struct _sas_device *
819	__mpt3sas_get_sdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
820	{
821	struct _sas_device *sas_device;
822
823	assert_spin_locked(&ioc->sas_device_lock);
824
825	list_for_each_entry(sas_device, &ioc->sas_device_list, list)
826	if (sas_device->handle == handle)
827	goto found_device;
828
829	list_for_each_entry(sas_device, &ioc->sas_device_init_list, list)
830	if (sas_device->handle == handle)
831	goto found_device;
832
833	return NULL;
834
835	found_device:
836	sas_device_get(s: sas_device);
837	return sas_device;
838	}
839
840	/**
841	* mpt3sas_get_sdev_by_handle - sas device search
842	* @ioc: per adapter object
843	* @handle: sas device handle (assigned by firmware)
844	* Context: Calling function should acquire ioc->sas_device_lock
845	*
846	* This searches for sas_device based on sas_address, then return sas_device
847	* object.
848	*/
849	struct _sas_device *
850	mpt3sas_get_sdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
851	{
852	struct _sas_device *sas_device;
853	unsigned long flags;
854
855	spin_lock_irqsave(&ioc->sas_device_lock, flags);
856	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
857	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
858
859	return sas_device;
860	}
861
862	/**
863	* _scsih_display_enclosure_chassis_info - display device location info
864	* @ioc: per adapter object
865	* @sas_device: per sas device object
866	* @sdev: scsi device struct
867	* @starget: scsi target struct
868	*/
869	static void
870	_scsih_display_enclosure_chassis_info(struct MPT3SAS_ADAPTER *ioc,
871	struct _sas_device *sas_device, struct scsi_device *sdev,
872	struct scsi_target *starget)
873	{
874	if (sdev) {
875	if (sas_device->enclosure_handle != 0)
876	sdev_printk(KERN_INFO, sdev,
877	"enclosure logical id (0x%016llx), slot(%d) \n",
878	(unsigned long long)
879	sas_device->enclosure_logical_id,
880	sas_device->slot);
881	if (sas_device->connector_name[0] != '\0')
882	sdev_printk(KERN_INFO, sdev,
883	"enclosure level(0x%04x), connector name( %s)\n",
884	sas_device->enclosure_level,
885	sas_device->connector_name);
886	if (sas_device->is_chassis_slot_valid)
887	sdev_printk(KERN_INFO, sdev, "chassis slot(0x%04x)\n",
888	sas_device->chassis_slot);
889	} else if (starget) {
890	if (sas_device->enclosure_handle != 0)
891	starget_printk(KERN_INFO, starget,
892	"enclosure logical id(0x%016llx), slot(%d) \n",
893	(unsigned long long)
894	sas_device->enclosure_logical_id,
895	sas_device->slot);
896	if (sas_device->connector_name[0] != '\0')
897	starget_printk(KERN_INFO, starget,
898	"enclosure level(0x%04x), connector name( %s)\n",
899	sas_device->enclosure_level,
900	sas_device->connector_name);
901	if (sas_device->is_chassis_slot_valid)
902	starget_printk(KERN_INFO, starget,
903	"chassis slot(0x%04x)\n",
904	sas_device->chassis_slot);
905	} else {
906	if (sas_device->enclosure_handle != 0)
907	ioc_info(ioc, "enclosure logical id(0x%016llx), slot(%d)\n",
908	(u64)sas_device->enclosure_logical_id,
909	sas_device->slot);
910	if (sas_device->connector_name[0] != '\0')
911	ioc_info(ioc, "enclosure level(0x%04x), connector name( %s)\n",
912	sas_device->enclosure_level,
913	sas_device->connector_name);
914	if (sas_device->is_chassis_slot_valid)
915	ioc_info(ioc, "chassis slot(0x%04x)\n",
916	sas_device->chassis_slot);
917	}
918	}
919
920	/**
921	* _scsih_sas_device_remove - remove sas_device from list.
922	* @ioc: per adapter object
923	* @sas_device: the sas_device object
924	* Context: This function will acquire ioc->sas_device_lock.
925	*
926	* If sas_device is on the list, remove it and decrement its reference count.
927	*/
928	static void
929	_scsih_sas_device_remove(struct MPT3SAS_ADAPTER *ioc,
930	struct _sas_device *sas_device)
931	{
932	unsigned long flags;
933
934	if (!sas_device)
935	return;
936	ioc_info(ioc, "removing handle(0x%04x), sas_addr(0x%016llx)\n",
937	sas_device->handle, (u64)sas_device->sas_address);
938
939	_scsih_display_enclosure_chassis_info(ioc, sas_device, NULL, NULL);
940
941	/*
942	* The lock serializes access to the list, but we still need to verify
943	* that nobody removed the entry while we were waiting on the lock.
944	*/
945	spin_lock_irqsave(&ioc->sas_device_lock, flags);
946	if (!list_empty(head: &sas_device->list)) {
947	list_del_init(entry: &sas_device->list);
948	sas_device_put(s: sas_device);
949	}
950	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
951	}
952
953	/**
954	* _scsih_device_remove_by_handle - removing device object by handle
955	* @ioc: per adapter object
956	* @handle: device handle
957	*/
958	static void
959	_scsih_device_remove_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
960	{
961	struct _sas_device *sas_device;
962	unsigned long flags;
963
964	if (ioc->shost_recovery)
965	return;
966
967	spin_lock_irqsave(&ioc->sas_device_lock, flags);
968	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
969	if (sas_device) {
970	list_del_init(entry: &sas_device->list);
971	sas_device_put(s: sas_device);
972	}
973	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
974	if (sas_device) {
975	_scsih_remove_device(ioc, sas_device);
976	sas_device_put(s: sas_device);
977	}
978	}
979
980	/**
981	* mpt3sas_device_remove_by_sas_address - removing device object by
982	* sas address & port number
983	* @ioc: per adapter object
984	* @sas_address: device sas_address
985	* @port: hba port entry
986	*
987	* Return nothing.
988	*/
989	void
990	mpt3sas_device_remove_by_sas_address(struct MPT3SAS_ADAPTER *ioc,
991	u64 sas_address, struct hba_port *port)
992	{
993	struct _sas_device *sas_device;
994	unsigned long flags;
995
996	if (ioc->shost_recovery)
997	return;
998
999	spin_lock_irqsave(&ioc->sas_device_lock, flags);
1000	sas_device = __mpt3sas_get_sdev_by_addr(ioc, sas_address, port);
1001	if (sas_device) {
1002	list_del_init(entry: &sas_device->list);
1003	sas_device_put(s: sas_device);
1004	}
1005	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
1006	if (sas_device) {
1007	_scsih_remove_device(ioc, sas_device);
1008	sas_device_put(s: sas_device);
1009	}
1010	}
1011
1012	/**
1013	* _scsih_sas_device_add - insert sas_device to the list.
1014	* @ioc: per adapter object
1015	* @sas_device: the sas_device object
1016	* Context: This function will acquire ioc->sas_device_lock.
1017	*
1018	* Adding new object to the ioc->sas_device_list.
1019	*/
1020	static void
1021	_scsih_sas_device_add(struct MPT3SAS_ADAPTER *ioc,
1022	struct _sas_device *sas_device)
1023	{
1024	unsigned long flags;
1025
1026	dewtprintk(ioc,
1027	ioc_info(ioc, "%s: handle(0x%04x), sas_addr(0x%016llx)\n",
1028	__func__, sas_device->handle,
1029	(u64)sas_device->sas_address));
1030
1031	dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,
1032	NULL, NULL));
1033
1034	spin_lock_irqsave(&ioc->sas_device_lock, flags);
1035	sas_device_get(s: sas_device);
1036	list_add_tail(new: &sas_device->list, head: &ioc->sas_device_list);
1037	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
1038
1039	if (ioc->hide_drives) {
1040	clear_bit(nr: sas_device->handle, addr: ioc->pend_os_device_add);
1041	return;
1042	}
1043
1044	if (!mpt3sas_transport_port_add(ioc, handle: sas_device->handle,
1045	sas_address: sas_device->sas_address_parent, port: sas_device->port)) {
1046	_scsih_sas_device_remove(ioc, sas_device);
1047	} else if (!sas_device->starget) {
1048	/*
1049	* When asyn scanning is enabled, its not possible to remove
1050	* devices while scanning is turned on due to an oops in
1051	* scsi_sysfs_add_sdev()->add_device()->sysfs_addrm_start()
1052	*/
1053	if (!ioc->is_driver_loading) {
1054	mpt3sas_transport_port_remove(ioc,
1055	sas_address: sas_device->sas_address,
1056	sas_address_parent: sas_device->sas_address_parent,
1057	port: sas_device->port);
1058	_scsih_sas_device_remove(ioc, sas_device);
1059	}
1060	} else
1061	clear_bit(nr: sas_device->handle, addr: ioc->pend_os_device_add);
1062	}
1063
1064	/**
1065	* _scsih_sas_device_init_add - insert sas_device to the list.
1066	* @ioc: per adapter object
1067	* @sas_device: the sas_device object
1068	* Context: This function will acquire ioc->sas_device_lock.
1069	*
1070	* Adding new object at driver load time to the ioc->sas_device_init_list.
1071	*/
1072	static void
1073	_scsih_sas_device_init_add(struct MPT3SAS_ADAPTER *ioc,
1074	struct _sas_device *sas_device)
1075	{
1076	unsigned long flags;
1077
1078	dewtprintk(ioc,
1079	ioc_info(ioc, "%s: handle(0x%04x), sas_addr(0x%016llx)\n",
1080	__func__, sas_device->handle,
1081	(u64)sas_device->sas_address));
1082
1083	dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,
1084	NULL, NULL));
1085
1086	spin_lock_irqsave(&ioc->sas_device_lock, flags);
1087	sas_device_get(s: sas_device);
1088	list_add_tail(new: &sas_device->list, head: &ioc->sas_device_init_list);
1089	_scsih_determine_boot_device(ioc, device: sas_device, channel: 0);
1090	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
1091	}
1092
1093
1094	static struct _pcie_device *
1095	__mpt3sas_get_pdev_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)
1096	{
1097	struct _pcie_device *pcie_device;
1098
1099	assert_spin_locked(&ioc->pcie_device_lock);
1100
1101	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)
1102	if (pcie_device->wwid == wwid)
1103	goto found_device;
1104
1105	list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)
1106	if (pcie_device->wwid == wwid)
1107	goto found_device;
1108
1109	return NULL;
1110
1111	found_device:
1112	pcie_device_get(p: pcie_device);
1113	return pcie_device;
1114	}
1115
1116
1117	/**
1118	* mpt3sas_get_pdev_by_wwid - pcie device search
1119	* @ioc: per adapter object
1120	* @wwid: wwid
1121	*
1122	* Context: This function will acquire ioc->pcie_device_lock and will release
1123	* before returning the pcie_device object.
1124	*
1125	* This searches for pcie_device based on wwid, then return pcie_device object.
1126	*/
1127	static struct _pcie_device *
1128	mpt3sas_get_pdev_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)
1129	{
1130	struct _pcie_device *pcie_device;
1131	unsigned long flags;
1132
1133	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1134	pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);
1135	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
1136
1137	return pcie_device;
1138	}
1139
1140
1141	static struct _pcie_device *
1142	__mpt3sas_get_pdev_by_idchannel(struct MPT3SAS_ADAPTER *ioc, int id,
1143	int channel)
1144	{
1145	struct _pcie_device *pcie_device;
1146
1147	assert_spin_locked(&ioc->pcie_device_lock);
1148
1149	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)
1150	if (pcie_device->id == id && pcie_device->channel == channel)
1151	goto found_device;
1152
1153	list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)
1154	if (pcie_device->id == id && pcie_device->channel == channel)
1155	goto found_device;
1156
1157	return NULL;
1158
1159	found_device:
1160	pcie_device_get(p: pcie_device);
1161	return pcie_device;
1162	}
1163
1164	static struct _pcie_device *
1165	__mpt3sas_get_pdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1166	{
1167	struct _pcie_device *pcie_device;
1168
1169	assert_spin_locked(&ioc->pcie_device_lock);
1170
1171	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)
1172	if (pcie_device->handle == handle)
1173	goto found_device;
1174
1175	list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)
1176	if (pcie_device->handle == handle)
1177	goto found_device;
1178
1179	return NULL;
1180
1181	found_device:
1182	pcie_device_get(p: pcie_device);
1183	return pcie_device;
1184	}
1185
1186
1187	/**
1188	* mpt3sas_get_pdev_by_handle - pcie device search
1189	* @ioc: per adapter object
1190	* @handle: Firmware device handle
1191	*
1192	* Context: This function will acquire ioc->pcie_device_lock and will release
1193	* before returning the pcie_device object.
1194	*
1195	* This searches for pcie_device based on handle, then return pcie_device
1196	* object.
1197	*/
1198	struct _pcie_device *
1199	mpt3sas_get_pdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1200	{
1201	struct _pcie_device *pcie_device;
1202	unsigned long flags;
1203
1204	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1205	pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);
1206	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
1207
1208	return pcie_device;
1209	}
1210
1211	/**
1212	* _scsih_set_nvme_max_shutdown_latency - Update max_shutdown_latency.
1213	* @ioc: per adapter object
1214	* Context: This function will acquire ioc->pcie_device_lock
1215	*
1216	* Update ioc->max_shutdown_latency to that NVMe drives RTD3 Entry Latency
1217	* which has reported maximum among all available NVMe drives.
1218	* Minimum max_shutdown_latency will be six seconds.
1219	*/
1220	static void
1221	_scsih_set_nvme_max_shutdown_latency(struct MPT3SAS_ADAPTER *ioc)
1222	{
1223	struct _pcie_device *pcie_device;
1224	unsigned long flags;
1225	u16 shutdown_latency = IO_UNIT_CONTROL_SHUTDOWN_TIMEOUT;
1226
1227	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1228	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list) {
1229	if (pcie_device->shutdown_latency) {
1230	if (shutdown_latency < pcie_device->shutdown_latency)
1231	shutdown_latency =
1232	pcie_device->shutdown_latency;
1233	}
1234	}
1235	ioc->max_shutdown_latency = shutdown_latency;
1236	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
1237	}
1238
1239	/**
1240	* _scsih_pcie_device_remove - remove pcie_device from list.
1241	* @ioc: per adapter object
1242	* @pcie_device: the pcie_device object
1243	* Context: This function will acquire ioc->pcie_device_lock.
1244	*
1245	* If pcie_device is on the list, remove it and decrement its reference count.
1246	*/
1247	static void
1248	_scsih_pcie_device_remove(struct MPT3SAS_ADAPTER *ioc,
1249	struct _pcie_device *pcie_device)
1250	{
1251	unsigned long flags;
1252	int was_on_pcie_device_list = 0;
1253	u8 update_latency = 0;
1254
1255	if (!pcie_device)
1256	return;
1257	ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",
1258	pcie_device->handle, (u64)pcie_device->wwid);
1259	if (pcie_device->enclosure_handle != 0)
1260	ioc_info(ioc, "removing enclosure logical id(0x%016llx), slot(%d)\n",
1261	(u64)pcie_device->enclosure_logical_id,
1262	pcie_device->slot);
1263	if (pcie_device->connector_name[0] != '\0')
1264	ioc_info(ioc, "removing enclosure level(0x%04x), connector name( %s)\n",
1265	pcie_device->enclosure_level,
1266	pcie_device->connector_name);
1267
1268	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1269	if (!list_empty(head: &pcie_device->list)) {
1270	list_del_init(entry: &pcie_device->list);
1271	was_on_pcie_device_list = 1;
1272	}
1273	if (pcie_device->shutdown_latency == ioc->max_shutdown_latency)
1274	update_latency = 1;
1275	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
1276	if (was_on_pcie_device_list) {
1277	kfree(objp: pcie_device->serial_number);
1278	pcie_device_put(p: pcie_device);
1279	}
1280
1281	/*
1282	* This device's RTD3 Entry Latency matches IOC's
1283	* max_shutdown_latency. Recalculate IOC's max_shutdown_latency
1284	* from the available drives as current drive is getting removed.
1285	*/
1286	if (update_latency)
1287	_scsih_set_nvme_max_shutdown_latency(ioc);
1288	}
1289
1290
1291	/**
1292	* _scsih_pcie_device_remove_by_handle - removing pcie device object by handle
1293	* @ioc: per adapter object
1294	* @handle: device handle
1295	*/
1296	static void
1297	_scsih_pcie_device_remove_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1298	{
1299	struct _pcie_device *pcie_device;
1300	unsigned long flags;
1301	int was_on_pcie_device_list = 0;
1302	u8 update_latency = 0;
1303
1304	if (ioc->shost_recovery)
1305	return;
1306
1307	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1308	pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);
1309	if (pcie_device) {
1310	if (!list_empty(head: &pcie_device->list)) {
1311	list_del_init(entry: &pcie_device->list);
1312	was_on_pcie_device_list = 1;
1313	pcie_device_put(p: pcie_device);
1314	}
1315	if (pcie_device->shutdown_latency == ioc->max_shutdown_latency)
1316	update_latency = 1;
1317	}
1318	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
1319	if (was_on_pcie_device_list) {
1320	_scsih_pcie_device_remove_from_sml(ioc, pcie_device);
1321	pcie_device_put(p: pcie_device);
1322	}
1323
1324	/*
1325	* This device's RTD3 Entry Latency matches IOC's
1326	* max_shutdown_latency. Recalculate IOC's max_shutdown_latency
1327	* from the available drives as current drive is getting removed.
1328	*/
1329	if (update_latency)
1330	_scsih_set_nvme_max_shutdown_latency(ioc);
1331	}
1332
1333	/**
1334	* _scsih_pcie_device_add - add pcie_device object
1335	* @ioc: per adapter object
1336	* @pcie_device: pcie_device object
1337	*
1338	* This is added to the pcie_device_list link list.
1339	*/
1340	static void
1341	_scsih_pcie_device_add(struct MPT3SAS_ADAPTER *ioc,
1342	struct _pcie_device *pcie_device)
1343	{
1344	unsigned long flags;
1345
1346	dewtprintk(ioc,
1347	ioc_info(ioc, "%s: handle (0x%04x), wwid(0x%016llx)\n",
1348	__func__,
1349	pcie_device->handle, (u64)pcie_device->wwid));
1350	if (pcie_device->enclosure_handle != 0)
1351	dewtprintk(ioc,
1352	ioc_info(ioc, "%s: enclosure logical id(0x%016llx), slot( %d)\n",
1353	__func__,
1354	(u64)pcie_device->enclosure_logical_id,
1355	pcie_device->slot));
1356	if (pcie_device->connector_name[0] != '\0')
1357	dewtprintk(ioc,
1358	ioc_info(ioc, "%s: enclosure level(0x%04x), connector name( %s)\n",
1359	__func__, pcie_device->enclosure_level,
1360	pcie_device->connector_name));
1361
1362	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1363	pcie_device_get(p: pcie_device);
1364	list_add_tail(new: &pcie_device->list, head: &ioc->pcie_device_list);
1365	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
1366
1367	if (pcie_device->access_status ==
1368	MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED) {
1369	clear_bit(nr: pcie_device->handle, addr: ioc->pend_os_device_add);
1370	return;
1371	}
1372	if (scsi_add_device(host: ioc->shost, PCIE_CHANNEL, target: pcie_device->id, lun: 0)) {
1373	_scsih_pcie_device_remove(ioc, pcie_device);
1374	} else if (!pcie_device->starget) {
1375	if (!ioc->is_driver_loading) {
1376	/*TODO-- Need to find out whether this condition will occur or not*/
1377	clear_bit(nr: pcie_device->handle, addr: ioc->pend_os_device_add);
1378	}
1379	} else
1380	clear_bit(nr: pcie_device->handle, addr: ioc->pend_os_device_add);
1381	}
1382
1383	/*
1384	* _scsih_pcie_device_init_add - insert pcie_device to the init list.
1385	* @ioc: per adapter object
1386	* @pcie_device: the pcie_device object
1387	* Context: This function will acquire ioc->pcie_device_lock.
1388	*
1389	* Adding new object at driver load time to the ioc->pcie_device_init_list.
1390	*/
1391	static void
1392	_scsih_pcie_device_init_add(struct MPT3SAS_ADAPTER *ioc,
1393	struct _pcie_device *pcie_device)
1394	{
1395	unsigned long flags;
1396
1397	dewtprintk(ioc,
1398	ioc_info(ioc, "%s: handle (0x%04x), wwid(0x%016llx)\n",
1399	__func__,
1400	pcie_device->handle, (u64)pcie_device->wwid));
1401	if (pcie_device->enclosure_handle != 0)
1402	dewtprintk(ioc,
1403	ioc_info(ioc, "%s: enclosure logical id(0x%016llx), slot( %d)\n",
1404	__func__,
1405	(u64)pcie_device->enclosure_logical_id,
1406	pcie_device->slot));
1407	if (pcie_device->connector_name[0] != '\0')
1408	dewtprintk(ioc,
1409	ioc_info(ioc, "%s: enclosure level(0x%04x), connector name( %s)\n",
1410	__func__, pcie_device->enclosure_level,
1411	pcie_device->connector_name));
1412
1413	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1414	pcie_device_get(p: pcie_device);
1415	list_add_tail(new: &pcie_device->list, head: &ioc->pcie_device_init_list);
1416	if (pcie_device->access_status !=
1417	MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED)
1418	_scsih_determine_boot_device(ioc, device: pcie_device, PCIE_CHANNEL);
1419	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
1420	}
1421	/**
1422	* _scsih_raid_device_find_by_id - raid device search
1423	* @ioc: per adapter object
1424	* @id: sas device target id
1425	* @channel: sas device channel
1426	* Context: Calling function should acquire ioc->raid_device_lock
1427	*
1428	* This searches for raid_device based on target id, then return raid_device
1429	* object.
1430	*/
1431	static struct _raid_device *
1432	_scsih_raid_device_find_by_id(struct MPT3SAS_ADAPTER *ioc, int id, int channel)
1433	{
1434	struct _raid_device *raid_device, *r;
1435
1436	r = NULL;
1437	list_for_each_entry(raid_device, &ioc->raid_device_list, list) {
1438	if (raid_device->id == id && raid_device->channel == channel) {
1439	r = raid_device;
1440	goto out;
1441	}
1442	}
1443
1444	out:
1445	return r;
1446	}
1447
1448	/**
1449	* mpt3sas_raid_device_find_by_handle - raid device search
1450	* @ioc: per adapter object
1451	* @handle: sas device handle (assigned by firmware)
1452	* Context: Calling function should acquire ioc->raid_device_lock
1453	*
1454	* This searches for raid_device based on handle, then return raid_device
1455	* object.
1456	*/
1457	struct _raid_device *
1458	mpt3sas_raid_device_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1459	{
1460	struct _raid_device *raid_device, *r;
1461
1462	r = NULL;
1463	list_for_each_entry(raid_device, &ioc->raid_device_list, list) {
1464	if (raid_device->handle != handle)
1465	continue;
1466	r = raid_device;
1467	goto out;
1468	}
1469
1470	out:
1471	return r;
1472	}
1473
1474	/**
1475	* _scsih_raid_device_find_by_wwid - raid device search
1476	* @ioc: per adapter object
1477	* @wwid: ?
1478	* Context: Calling function should acquire ioc->raid_device_lock
1479	*
1480	* This searches for raid_device based on wwid, then return raid_device
1481	* object.
1482	*/
1483	static struct _raid_device *
1484	_scsih_raid_device_find_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)
1485	{
1486	struct _raid_device *raid_device, *r;
1487
1488	r = NULL;
1489	list_for_each_entry(raid_device, &ioc->raid_device_list, list) {
1490	if (raid_device->wwid != wwid)
1491	continue;
1492	r = raid_device;
1493	goto out;
1494	}
1495
1496	out:
1497	return r;
1498	}
1499
1500	/**
1501	* _scsih_raid_device_add - add raid_device object
1502	* @ioc: per adapter object
1503	* @raid_device: raid_device object
1504	*
1505	* This is added to the raid_device_list link list.
1506	*/
1507	static void
1508	_scsih_raid_device_add(struct MPT3SAS_ADAPTER *ioc,
1509	struct _raid_device *raid_device)
1510	{
1511	unsigned long flags;
1512
1513	dewtprintk(ioc,
1514	ioc_info(ioc, "%s: handle(0x%04x), wwid(0x%016llx)\n",
1515	__func__,
1516	raid_device->handle, (u64)raid_device->wwid));
1517
1518	spin_lock_irqsave(&ioc->raid_device_lock, flags);
1519	list_add_tail(new: &raid_device->list, head: &ioc->raid_device_list);
1520	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
1521	}
1522
1523	/**
1524	* _scsih_raid_device_remove - delete raid_device object
1525	* @ioc: per adapter object
1526	* @raid_device: raid_device object
1527	*
1528	*/
1529	static void
1530	_scsih_raid_device_remove(struct MPT3SAS_ADAPTER *ioc,
1531	struct _raid_device *raid_device)
1532	{
1533	unsigned long flags;
1534
1535	spin_lock_irqsave(&ioc->raid_device_lock, flags);
1536	list_del(entry: &raid_device->list);
1537	kfree(objp: raid_device);
1538	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
1539	}
1540
1541	/**
1542	* mpt3sas_scsih_expander_find_by_handle - expander device search
1543	* @ioc: per adapter object
1544	* @handle: expander handle (assigned by firmware)
1545	* Context: Calling function should acquire ioc->sas_device_lock
1546	*
1547	* This searches for expander device based on handle, then returns the
1548	* sas_node object.
1549	*/
1550	struct _sas_node *
1551	mpt3sas_scsih_expander_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1552	{
1553	struct _sas_node *sas_expander, *r;
1554
1555	r = NULL;
1556	list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {
1557	if (sas_expander->handle != handle)
1558	continue;
1559	r = sas_expander;
1560	goto out;
1561	}
1562	out:
1563	return r;
1564	}
1565
1566	/**
1567	* mpt3sas_scsih_enclosure_find_by_handle - exclosure device search
1568	* @ioc: per adapter object
1569	* @handle: enclosure handle (assigned by firmware)
1570	* Context: Calling function should acquire ioc->sas_device_lock
1571	*
1572	* This searches for enclosure device based on handle, then returns the
1573	* enclosure object.
1574	*/
1575	static struct _enclosure_node *
1576	mpt3sas_scsih_enclosure_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1577	{
1578	struct _enclosure_node *enclosure_dev, *r;
1579
1580	r = NULL;
1581	list_for_each_entry(enclosure_dev, &ioc->enclosure_list, list) {
1582	if (le16_to_cpu(enclosure_dev->pg0.EnclosureHandle) != handle)
1583	continue;
1584	r = enclosure_dev;
1585	goto out;
1586	}
1587	out:
1588	return r;
1589	}
1590	/**
1591	* mpt3sas_scsih_expander_find_by_sas_address - expander device search
1592	* @ioc: per adapter object
1593	* @sas_address: sas address
1594	* @port: hba port entry
1595	* Context: Calling function should acquire ioc->sas_node_lock.
1596	*
1597	* This searches for expander device based on sas_address & port number,
1598	* then returns the sas_node object.
1599	*/
1600	struct _sas_node *
1601	mpt3sas_scsih_expander_find_by_sas_address(struct MPT3SAS_ADAPTER *ioc,
1602	u64 sas_address, struct hba_port *port)
1603	{
1604	struct _sas_node *sas_expander, *r = NULL;
1605
1606	if (!port)
1607	return r;
1608
1609	list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {
1610	if (sas_expander->sas_address != sas_address)
1611	continue;
1612	if (sas_expander->port != port)
1613	continue;
1614	r = sas_expander;
1615	goto out;
1616	}
1617	out:
1618	return r;
1619	}
1620
1621	/**
1622	* _scsih_expander_node_add - insert expander device to the list.
1623	* @ioc: per adapter object
1624	* @sas_expander: the sas_device object
1625	* Context: This function will acquire ioc->sas_node_lock.
1626	*
1627	* Adding new object to the ioc->sas_expander_list.
1628	*/
1629	static void
1630	_scsih_expander_node_add(struct MPT3SAS_ADAPTER *ioc,
1631	struct _sas_node *sas_expander)
1632	{
1633	unsigned long flags;
1634
1635	spin_lock_irqsave(&ioc->sas_node_lock, flags);
1636	list_add_tail(new: &sas_expander->list, head: &ioc->sas_expander_list);
1637	spin_unlock_irqrestore(lock: &ioc->sas_node_lock, flags);
1638	}
1639
1640	/**
1641	* _scsih_is_end_device - determines if device is an end device
1642	* @device_info: bitfield providing information about the device.
1643	* Context: none
1644	*
1645	* Return: 1 if end device.
1646	*/
1647	static int
1648	_scsih_is_end_device(u32 device_info)
1649	{
1650	if (device_info & MPI2_SAS_DEVICE_INFO_END_DEVICE &&
1651	((device_info & MPI2_SAS_DEVICE_INFO_SSP_TARGET) |
1652	(device_info & MPI2_SAS_DEVICE_INFO_STP_TARGET) |
1653	(device_info & MPI2_SAS_DEVICE_INFO_SATA_DEVICE)))
1654	return 1;
1655	else
1656	return 0;
1657	}
1658
1659	/**
1660	* _scsih_is_nvme_pciescsi_device - determines if
1661	* device is an pcie nvme/scsi device
1662	* @device_info: bitfield providing information about the device.
1663	* Context: none
1664	*
1665	* Returns 1 if device is pcie device type nvme/scsi.
1666	*/
1667	static int
1668	_scsih_is_nvme_pciescsi_device(u32 device_info)
1669	{
1670	if (((device_info & MPI26_PCIE_DEVINFO_MASK_DEVICE_TYPE)
1671	== MPI26_PCIE_DEVINFO_NVME) ||
1672	((device_info & MPI26_PCIE_DEVINFO_MASK_DEVICE_TYPE)
1673	== MPI26_PCIE_DEVINFO_SCSI))
1674	return 1;
1675	else
1676	return 0;
1677	}
1678
1679	/**
1680	* _scsih_scsi_lookup_find_by_target - search for matching channel:id
1681	* @ioc: per adapter object
1682	* @id: target id
1683	* @channel: channel
1684	* Context: This function will acquire ioc->scsi_lookup_lock.
1685	*
1686	* This will search for a matching channel:id in the scsi_lookup array,
1687	* returning 1 if found.
1688	*/
1689	static u8
1690	_scsih_scsi_lookup_find_by_target(struct MPT3SAS_ADAPTER *ioc, int id,
1691	int channel)
1692	{
1693	int smid;
1694	struct scsi_cmnd *scmd;
1695
1696	for (smid = 1;
1697	smid <= ioc->shost->can_queue; smid++) {
1698	scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
1699	if (!scmd)
1700	continue;
1701	if (scmd->device->id == id &&
1702	scmd->device->channel == channel)
1703	return 1;
1704	}
1705	return 0;
1706	}
1707
1708	/**
1709	* _scsih_scsi_lookup_find_by_lun - search for matching channel:id:lun
1710	* @ioc: per adapter object
1711	* @id: target id
1712	* @lun: lun number
1713	* @channel: channel
1714	* Context: This function will acquire ioc->scsi_lookup_lock.
1715	*
1716	* This will search for a matching channel:id:lun in the scsi_lookup array,
1717	* returning 1 if found.
1718	*/
1719	static u8
1720	_scsih_scsi_lookup_find_by_lun(struct MPT3SAS_ADAPTER *ioc, int id,
1721	unsigned int lun, int channel)
1722	{
1723	int smid;
1724	struct scsi_cmnd *scmd;
1725
1726	for (smid = 1; smid <= ioc->shost->can_queue; smid++) {
1727
1728	scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
1729	if (!scmd)
1730	continue;
1731	if (scmd->device->id == id &&
1732	scmd->device->channel == channel &&
1733	scmd->device->lun == lun)
1734	return 1;
1735	}
1736	return 0;
1737	}
1738
1739	/**
1740	* mpt3sas_scsih_scsi_lookup_get - returns scmd entry
1741	* @ioc: per adapter object
1742	* @smid: system request message index
1743	*
1744	* Return: the smid stored scmd pointer.
1745	* Then will dereference the stored scmd pointer.
1746	*/
1747	struct scsi_cmnd *
1748	mpt3sas_scsih_scsi_lookup_get(struct MPT3SAS_ADAPTER *ioc, u16 smid)
1749	{
1750	struct scsi_cmnd *scmd = NULL;
1751	struct scsiio_tracker *st;
1752	Mpi25SCSIIORequest_t *mpi_request;
1753	u16 tag = smid - 1;
1754
1755	if (smid > 0 &&
1756	smid <= ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT) {
1757	u32 unique_tag =
1758	ioc->io_queue_num[tag] << BLK_MQ_UNIQUE_TAG_BITS | tag;
1759
1760	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
1761
1762	/*
1763	* If SCSI IO request is outstanding at driver level then
1764	* DevHandle filed must be non-zero. If DevHandle is zero
1765	* then it means that this smid is free at driver level,
1766	* so return NULL.
1767	*/
1768	if (!mpi_request->DevHandle)
1769	return scmd;
1770
1771	scmd = scsi_host_find_tag(shost: ioc->shost, tag: unique_tag);
1772	if (scmd) {
1773	st = scsi_cmd_priv(cmd: scmd);
1774	if (st->cb_idx == 0xFF || st->smid == 0)
1775	scmd = NULL;
1776	}
1777	}
1778	return scmd;
1779	}
1780
1781	/**
1782	* scsih_change_queue_depth - setting device queue depth
1783	* @sdev: scsi device struct
1784	* @qdepth: requested queue depth
1785	*
1786	* Return: queue depth.
1787	*/
1788	static int
1789	scsih_change_queue_depth(struct scsi_device *sdev, int qdepth)
1790	{
1791	struct Scsi_Host *shost = sdev->host;
1792	int max_depth;
1793	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
1794	struct MPT3SAS_DEVICE *sas_device_priv_data;
1795	struct MPT3SAS_TARGET *sas_target_priv_data;
1796	struct _sas_device *sas_device;
1797	unsigned long flags;
1798
1799	max_depth = shost->can_queue;
1800
1801	/*
1802	* limit max device queue for SATA to 32 if enable_sdev_max_qd
1803	* is disabled.
1804	*/
1805	if (ioc->enable_sdev_max_qd || ioc->is_gen35_ioc)
1806	goto not_sata;
1807
1808	sas_device_priv_data = sdev->hostdata;
1809	if (!sas_device_priv_data)
1810	goto not_sata;
1811	sas_target_priv_data = sas_device_priv_data->sas_target;
1812	if (!sas_target_priv_data)
1813	goto not_sata;
1814	if ((sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME))
1815	goto not_sata;
1816
1817	spin_lock_irqsave(&ioc->sas_device_lock, flags);
1818	sas_device = __mpt3sas_get_sdev_from_target(ioc, tgt_priv: sas_target_priv_data);
1819	if (sas_device) {
1820	if (sas_device->device_info & MPI2_SAS_DEVICE_INFO_SATA_DEVICE)
1821	max_depth = MPT3SAS_SATA_QUEUE_DEPTH;
1822
1823	sas_device_put(s: sas_device);
1824	}
1825	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
1826
1827	not_sata:
1828
1829	if (!sdev->tagged_supported)
1830	max_depth = 1;
1831	if (qdepth > max_depth)
1832	qdepth = max_depth;
1833	scsi_change_queue_depth(sdev, qdepth);
1834	sdev_printk(KERN_INFO, sdev,
1835	"qdepth(%d), tagged(%d), scsi_level(%d), cmd_que(%d)\n",
1836	sdev->queue_depth, sdev->tagged_supported,
1837	sdev->scsi_level, ((sdev->inquiry[7] & 2) >> 1));
1838	return sdev->queue_depth;
1839	}
1840
1841	/**
1842	* mpt3sas_scsih_change_queue_depth - setting device queue depth
1843	* @sdev: scsi device struct
1844	* @qdepth: requested queue depth
1845	*
1846	* Returns nothing.
1847	*/
1848	void
1849	mpt3sas_scsih_change_queue_depth(struct scsi_device *sdev, int qdepth)
1850	{
1851	struct Scsi_Host *shost = sdev->host;
1852	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
1853
1854	if (ioc->enable_sdev_max_qd)
1855	qdepth = shost->can_queue;
1856
1857	scsih_change_queue_depth(sdev, qdepth);
1858	}
1859
1860	/**
1861	* scsih_target_alloc - target add routine
1862	* @starget: scsi target struct
1863	*
1864	* Return: 0 if ok. Any other return is assumed to be an error and
1865	* the device is ignored.
1866	*/
1867	static int
1868	scsih_target_alloc(struct scsi_target *starget)
1869	{
1870	struct Scsi_Host *shost = dev_to_shost(dev: &starget->dev);
1871	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
1872	struct MPT3SAS_TARGET *sas_target_priv_data;
1873	struct _sas_device *sas_device;
1874	struct _raid_device *raid_device;
1875	struct _pcie_device *pcie_device;
1876	unsigned long flags;
1877	struct sas_rphy *rphy;
1878
1879	sas_target_priv_data = kzalloc(size: sizeof(*sas_target_priv_data),
1880	GFP_KERNEL);
1881	if (!sas_target_priv_data)
1882	return -ENOMEM;
1883
1884	starget->hostdata = sas_target_priv_data;
1885	sas_target_priv_data->starget = starget;
1886	sas_target_priv_data->handle = MPT3SAS_INVALID_DEVICE_HANDLE;
1887
1888	/* RAID volumes */
1889	if (starget->channel == RAID_CHANNEL) {
1890	spin_lock_irqsave(&ioc->raid_device_lock, flags);
1891	raid_device = _scsih_raid_device_find_by_id(ioc, id: starget->id,
1892	channel: starget->channel);
1893	if (raid_device) {
1894	sas_target_priv_data->handle = raid_device->handle;
1895	sas_target_priv_data->sas_address = raid_device->wwid;
1896	sas_target_priv_data->flags |= MPT_TARGET_FLAGS_VOLUME;
1897	if (ioc->is_warpdrive)
1898	sas_target_priv_data->raid_device = raid_device;
1899	raid_device->starget = starget;
1900	}
1901	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
1902	return 0;
1903	}
1904
1905	/* PCIe devices */
1906	if (starget->channel == PCIE_CHANNEL) {
1907	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1908	pcie_device = __mpt3sas_get_pdev_by_idchannel(ioc, id: starget->id,
1909	channel: starget->channel);
1910	if (pcie_device) {
1911	sas_target_priv_data->handle = pcie_device->handle;
1912	sas_target_priv_data->sas_address = pcie_device->wwid;
1913	sas_target_priv_data->port = NULL;
1914	sas_target_priv_data->pcie_dev = pcie_device;
1915	pcie_device->starget = starget;
1916	pcie_device->id = starget->id;
1917	pcie_device->channel = starget->channel;
1918	sas_target_priv_data->flags |=
1919	MPT_TARGET_FLAGS_PCIE_DEVICE;
1920	if (pcie_device->fast_path)
1921	sas_target_priv_data->flags |=
1922	MPT_TARGET_FASTPATH_IO;
1923	}
1924	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
1925	return 0;
1926	}
1927
1928	/* sas/sata devices */
1929	spin_lock_irqsave(&ioc->sas_device_lock, flags);
1930	rphy = dev_to_rphy(starget->dev.parent);
1931	sas_device = __mpt3sas_get_sdev_by_rphy(ioc, rphy);
1932
1933	if (sas_device) {
1934	sas_target_priv_data->handle = sas_device->handle;
1935	sas_target_priv_data->sas_address = sas_device->sas_address;
1936	sas_target_priv_data->port = sas_device->port;
1937	sas_target_priv_data->sas_dev = sas_device;
1938	sas_device->starget = starget;
1939	sas_device->id = starget->id;
1940	sas_device->channel = starget->channel;
1941	if (test_bit(sas_device->handle, ioc->pd_handles))
1942	sas_target_priv_data->flags |=
1943	MPT_TARGET_FLAGS_RAID_COMPONENT;
1944	if (sas_device->fast_path)
1945	sas_target_priv_data->flags |=
1946	MPT_TARGET_FASTPATH_IO;
1947	}
1948	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
1949
1950	return 0;
1951	}
1952
1953	/**
1954	* scsih_target_destroy - target destroy routine
1955	* @starget: scsi target struct
1956	*/
1957	static void
1958	scsih_target_destroy(struct scsi_target *starget)
1959	{
1960	struct Scsi_Host *shost = dev_to_shost(dev: &starget->dev);
1961	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
1962	struct MPT3SAS_TARGET *sas_target_priv_data;
1963	struct _sas_device *sas_device;
1964	struct _raid_device *raid_device;
1965	struct _pcie_device *pcie_device;
1966	unsigned long flags;
1967
1968	sas_target_priv_data = starget->hostdata;
1969	if (!sas_target_priv_data)
1970	return;
1971
1972	if (starget->channel == RAID_CHANNEL) {
1973	spin_lock_irqsave(&ioc->raid_device_lock, flags);
1974	raid_device = _scsih_raid_device_find_by_id(ioc, id: starget->id,
1975	channel: starget->channel);
1976	if (raid_device) {
1977	raid_device->starget = NULL;
1978	raid_device->sdev = NULL;
1979	}
1980	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
1981	goto out;
1982	}
1983
1984	if (starget->channel == PCIE_CHANNEL) {
1985	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1986	pcie_device = __mpt3sas_get_pdev_from_target(ioc,
1987	tgt_priv: sas_target_priv_data);
1988	if (pcie_device && (pcie_device->starget == starget) &&
1989	(pcie_device->id == starget->id) &&
1990	(pcie_device->channel == starget->channel))
1991	pcie_device->starget = NULL;
1992
1993	if (pcie_device) {
1994	/*
1995	* Corresponding get() is in _scsih_target_alloc()
1996	*/
1997	sas_target_priv_data->pcie_dev = NULL;
1998	pcie_device_put(p: pcie_device);
1999	pcie_device_put(p: pcie_device);
2000	}
2001	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
2002	goto out;
2003	}
2004
2005	spin_lock_irqsave(&ioc->sas_device_lock, flags);
2006	sas_device = __mpt3sas_get_sdev_from_target(ioc, tgt_priv: sas_target_priv_data);
2007	if (sas_device && (sas_device->starget == starget) &&
2008	(sas_device->id == starget->id) &&
2009	(sas_device->channel == starget->channel))
2010	sas_device->starget = NULL;
2011
2012	if (sas_device) {
2013	/*
2014	* Corresponding get() is in _scsih_target_alloc()
2015	*/
2016	sas_target_priv_data->sas_dev = NULL;
2017	sas_device_put(s: sas_device);
2018
2019	sas_device_put(s: sas_device);
2020	}
2021	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
2022
2023	out:
2024	kfree(objp: sas_target_priv_data);
2025	starget->hostdata = NULL;
2026	}
2027
2028	/**
2029	* scsih_slave_alloc - device add routine
2030	* @sdev: scsi device struct
2031	*
2032	* Return: 0 if ok. Any other return is assumed to be an error and
2033	* the device is ignored.
2034	*/
2035	static int
2036	scsih_slave_alloc(struct scsi_device *sdev)
2037	{
2038	struct Scsi_Host *shost;
2039	struct MPT3SAS_ADAPTER *ioc;
2040	struct MPT3SAS_TARGET *sas_target_priv_data;
2041	struct MPT3SAS_DEVICE *sas_device_priv_data;
2042	struct scsi_target *starget;
2043	struct _raid_device *raid_device;
2044	struct _sas_device *sas_device;
2045	struct _pcie_device *pcie_device;
2046	unsigned long flags;
2047
2048	sas_device_priv_data = kzalloc(size: sizeof(*sas_device_priv_data),
2049	GFP_KERNEL);
2050	if (!sas_device_priv_data)
2051	return -ENOMEM;
2052
2053	sas_device_priv_data->lun = sdev->lun;
2054	sas_device_priv_data->flags = MPT_DEVICE_FLAGS_INIT;
2055
2056	starget = scsi_target(sdev);
2057	sas_target_priv_data = starget->hostdata;
2058	sas_target_priv_data->num_luns++;
2059	sas_device_priv_data->sas_target = sas_target_priv_data;
2060	sdev->hostdata = sas_device_priv_data;
2061	if ((sas_target_priv_data->flags & MPT_TARGET_FLAGS_RAID_COMPONENT))
2062	sdev->no_uld_attach = 1;
2063
2064	shost = dev_to_shost(dev: &starget->dev);
2065	ioc = shost_priv(shost);
2066	if (starget->channel == RAID_CHANNEL) {
2067	spin_lock_irqsave(&ioc->raid_device_lock, flags);
2068	raid_device = _scsih_raid_device_find_by_id(ioc,
2069	id: starget->id, channel: starget->channel);
2070	if (raid_device)
2071	raid_device->sdev = sdev; /* raid is single lun */
2072	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
2073	}
2074	if (starget->channel == PCIE_CHANNEL) {
2075	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
2076	pcie_device = __mpt3sas_get_pdev_by_wwid(ioc,
2077	wwid: sas_target_priv_data->sas_address);
2078	if (pcie_device && (pcie_device->starget == NULL)) {
2079	sdev_printk(KERN_INFO, sdev,
2080	"%s : pcie_device->starget set to starget @ %d\n",
2081	__func__, __LINE__);
2082	pcie_device->starget = starget;
2083	}
2084
2085	if (pcie_device)
2086	pcie_device_put(p: pcie_device);
2087	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
2088
2089	} else if (!(sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME)) {
2090	spin_lock_irqsave(&ioc->sas_device_lock, flags);
2091	sas_device = __mpt3sas_get_sdev_by_addr(ioc,
2092	sas_address: sas_target_priv_data->sas_address,
2093	port: sas_target_priv_data->port);
2094	if (sas_device && (sas_device->starget == NULL)) {
2095	sdev_printk(KERN_INFO, sdev,
2096	"%s : sas_device->starget set to starget @ %d\n",
2097	__func__, __LINE__);
2098	sas_device->starget = starget;
2099	}
2100
2101	if (sas_device)
2102	sas_device_put(s: sas_device);
2103
2104	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
2105	}
2106
2107	return 0;
2108	}
2109
2110	/**
2111	* scsih_slave_destroy - device destroy routine
2112	* @sdev: scsi device struct
2113	*/
2114	static void
2115	scsih_slave_destroy(struct scsi_device *sdev)
2116	{
2117	struct MPT3SAS_TARGET *sas_target_priv_data;
2118	struct scsi_target *starget;
2119	struct Scsi_Host *shost;
2120	struct MPT3SAS_ADAPTER *ioc;
2121	struct _sas_device *sas_device;
2122	struct _pcie_device *pcie_device;
2123	unsigned long flags;
2124
2125	if (!sdev->hostdata)
2126	return;
2127
2128	starget = scsi_target(sdev);
2129	sas_target_priv_data = starget->hostdata;
2130	sas_target_priv_data->num_luns--;
2131
2132	shost = dev_to_shost(dev: &starget->dev);
2133	ioc = shost_priv(shost);
2134
2135	if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_PCIE_DEVICE) {
2136	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
2137	pcie_device = __mpt3sas_get_pdev_from_target(ioc,
2138	tgt_priv: sas_target_priv_data);
2139	if (pcie_device && !sas_target_priv_data->num_luns)
2140	pcie_device->starget = NULL;
2141
2142	if (pcie_device)
2143	pcie_device_put(p: pcie_device);
2144
2145	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
2146
2147	} else if (!(sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME)) {
2148	spin_lock_irqsave(&ioc->sas_device_lock, flags);
2149	sas_device = __mpt3sas_get_sdev_from_target(ioc,
2150	tgt_priv: sas_target_priv_data);
2151	if (sas_device && !sas_target_priv_data->num_luns)
2152	sas_device->starget = NULL;
2153
2154	if (sas_device)
2155	sas_device_put(s: sas_device);
2156	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
2157	}
2158
2159	kfree(objp: sdev->hostdata);
2160	sdev->hostdata = NULL;
2161	}
2162
2163	/**
2164	* _scsih_display_sata_capabilities - sata capabilities
2165	* @ioc: per adapter object
2166	* @handle: device handle
2167	* @sdev: scsi device struct
2168	*/
2169	static void
2170	_scsih_display_sata_capabilities(struct MPT3SAS_ADAPTER *ioc,
2171	u16 handle, struct scsi_device *sdev)
2172	{
2173	Mpi2ConfigReply_t mpi_reply;
2174	Mpi2SasDevicePage0_t sas_device_pg0;
2175	u32 ioc_status;
2176	u16 flags;
2177	u32 device_info;
2178
2179	if ((mpt3sas_config_get_sas_device_pg0(ioc, mpi_reply: &mpi_reply, config_page: &sas_device_pg0,
2180	MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
2181	ioc_err(ioc, "failure at %s:%d/%s()!\n",
2182	__FILE__, __LINE__, __func__);
2183	return;
2184	}
2185
2186	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
2187	MPI2_IOCSTATUS_MASK;
2188	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
2189	ioc_err(ioc, "failure at %s:%d/%s()!\n",
2190	__FILE__, __LINE__, __func__);
2191	return;
2192	}
2193
2194	flags = le16_to_cpu(sas_device_pg0.Flags);
2195	device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
2196
2197	sdev_printk(KERN_INFO, sdev,
2198	"atapi(%s), ncq(%s), asyn_notify(%s), smart(%s), fua(%s), "
2199	"sw_preserve(%s)\n",
2200	(device_info & MPI2_SAS_DEVICE_INFO_ATAPI_DEVICE) ? "y" : "n",
2201	(flags & MPI2_SAS_DEVICE0_FLAGS_SATA_NCQ_SUPPORTED) ? "y" : "n",
2202	(flags & MPI2_SAS_DEVICE0_FLAGS_SATA_ASYNCHRONOUS_NOTIFY) ? "y" :
2203	"n",
2204	(flags & MPI2_SAS_DEVICE0_FLAGS_SATA_SMART_SUPPORTED) ? "y" : "n",
2205	(flags & MPI2_SAS_DEVICE0_FLAGS_SATA_FUA_SUPPORTED) ? "y" : "n",
2206	(flags & MPI2_SAS_DEVICE0_FLAGS_SATA_SW_PRESERVE) ? "y" : "n");
2207	}
2208
2209	/*
2210	* raid transport support -
2211	* Enabled for SLES11 and newer, in older kernels the driver will panic when
2212	* unloading the driver followed by a load - I believe that the subroutine
2213	* raid_class_release() is not cleaning up properly.
2214	*/
2215
2216	/**
2217	* scsih_is_raid - return boolean indicating device is raid volume
2218	* @dev: the device struct object
2219	*/
2220	static int
2221	scsih_is_raid(struct device *dev)
2222	{
2223	struct scsi_device *sdev = to_scsi_device(dev);
2224	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost: sdev->host);
2225
2226	if (ioc->is_warpdrive)
2227	return 0;
2228	return (sdev->channel == RAID_CHANNEL) ? 1 : 0;
2229	}
2230
2231	static int
2232	scsih_is_nvme(struct device *dev)
2233	{
2234	struct scsi_device *sdev = to_scsi_device(dev);
2235
2236	return (sdev->channel == PCIE_CHANNEL) ? 1 : 0;
2237	}
2238
2239	/**
2240	* scsih_get_resync - get raid volume resync percent complete
2241	* @dev: the device struct object
2242	*/
2243	static void
2244	scsih_get_resync(struct device *dev)
2245	{
2246	struct scsi_device *sdev = to_scsi_device(dev);
2247	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost: sdev->host);
2248	static struct _raid_device *raid_device;
2249	unsigned long flags;
2250	Mpi2RaidVolPage0_t vol_pg0;
2251	Mpi2ConfigReply_t mpi_reply;
2252	u32 volume_status_flags;
2253	u8 percent_complete;
2254	u16 handle;
2255
2256	percent_complete = 0;
2257	handle = 0;
2258	if (ioc->is_warpdrive)
2259	goto out;
2260
2261	spin_lock_irqsave(&ioc->raid_device_lock, flags);
2262	raid_device = _scsih_raid_device_find_by_id(ioc, id: sdev->id,
2263	channel: sdev->channel);
2264	if (raid_device) {
2265	handle = raid_device->handle;
2266	percent_complete = raid_device->percent_complete;
2267	}
2268	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
2269
2270	if (!handle)
2271	goto out;
2272
2273	if (mpt3sas_config_get_raid_volume_pg0(ioc, mpi_reply: &mpi_reply, config_page: &vol_pg0,
2274	MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
2275	sz: sizeof(Mpi2RaidVolPage0_t))) {
2276	ioc_err(ioc, "failure at %s:%d/%s()!\n",
2277	__FILE__, __LINE__, __func__);
2278	percent_complete = 0;
2279	goto out;
2280	}
2281
2282	volume_status_flags = le32_to_cpu(vol_pg0.VolumeStatusFlags);
2283	if (!(volume_status_flags &
2284	MPI2_RAIDVOL0_STATUS_FLAG_RESYNC_IN_PROGRESS))
2285	percent_complete = 0;
2286
2287	out:
2288
2289	switch (ioc->hba_mpi_version_belonged) {
2290	case MPI2_VERSION:
2291	raid_set_resync(r: mpt2sas_raid_template, dev, value: percent_complete);
2292	break;
2293	case MPI25_VERSION:
2294	case MPI26_VERSION:
2295	raid_set_resync(r: mpt3sas_raid_template, dev, value: percent_complete);
2296	break;
2297	}
2298	}
2299
2300	/**
2301	* scsih_get_state - get raid volume level
2302	* @dev: the device struct object
2303	*/
2304	static void
2305	scsih_get_state(struct device *dev)
2306	{
2307	struct scsi_device *sdev = to_scsi_device(dev);
2308	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost: sdev->host);
2309	static struct _raid_device *raid_device;
2310	unsigned long flags;
2311	Mpi2RaidVolPage0_t vol_pg0;
2312	Mpi2ConfigReply_t mpi_reply;
2313	u32 volstate;
2314	enum raid_state state = RAID_STATE_UNKNOWN;
2315	u16 handle = 0;
2316
2317	spin_lock_irqsave(&ioc->raid_device_lock, flags);
2318	raid_device = _scsih_raid_device_find_by_id(ioc, id: sdev->id,
2319	channel: sdev->channel);
2320	if (raid_device)
2321	handle = raid_device->handle;
2322	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
2323
2324	if (!raid_device)
2325	goto out;
2326
2327	if (mpt3sas_config_get_raid_volume_pg0(ioc, mpi_reply: &mpi_reply, config_page: &vol_pg0,
2328	MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
2329	sz: sizeof(Mpi2RaidVolPage0_t))) {
2330	ioc_err(ioc, "failure at %s:%d/%s()!\n",
2331	__FILE__, __LINE__, __func__);
2332	goto out;
2333	}
2334
2335	volstate = le32_to_cpu(vol_pg0.VolumeStatusFlags);
2336	if (volstate & MPI2_RAIDVOL0_STATUS_FLAG_RESYNC_IN_PROGRESS) {
2337	state = RAID_STATE_RESYNCING;
2338	goto out;
2339	}
2340
2341	switch (vol_pg0.VolumeState) {
2342	case MPI2_RAID_VOL_STATE_OPTIMAL:
2343	case MPI2_RAID_VOL_STATE_ONLINE:
2344	state = RAID_STATE_ACTIVE;
2345	break;
2346	case MPI2_RAID_VOL_STATE_DEGRADED:
2347	state = RAID_STATE_DEGRADED;
2348	break;
2349	case MPI2_RAID_VOL_STATE_FAILED:
2350	case MPI2_RAID_VOL_STATE_MISSING:
2351	state = RAID_STATE_OFFLINE;
2352	break;
2353	}
2354	out:
2355	switch (ioc->hba_mpi_version_belonged) {
2356	case MPI2_VERSION:
2357	raid_set_state(r: mpt2sas_raid_template, dev, value: state);
2358	break;
2359	case MPI25_VERSION:
2360	case MPI26_VERSION:
2361	raid_set_state(r: mpt3sas_raid_template, dev, value: state);
2362	break;
2363	}
2364	}
2365
2366	/**
2367	* _scsih_set_level - set raid level
2368	* @ioc: ?
2369	* @sdev: scsi device struct
2370	* @volume_type: volume type
2371	*/
2372	static void
2373	_scsih_set_level(struct MPT3SAS_ADAPTER *ioc,
2374	struct scsi_device *sdev, u8 volume_type)
2375	{
2376	enum raid_level level = RAID_LEVEL_UNKNOWN;
2377
2378	switch (volume_type) {
2379	case MPI2_RAID_VOL_TYPE_RAID0:
2380	level = RAID_LEVEL_0;
2381	break;
2382	case MPI2_RAID_VOL_TYPE_RAID10:
2383	level = RAID_LEVEL_10;
2384	break;
2385	case MPI2_RAID_VOL_TYPE_RAID1E:
2386	level = RAID_LEVEL_1E;
2387	break;
2388	case MPI2_RAID_VOL_TYPE_RAID1:
2389	level = RAID_LEVEL_1;
2390	break;
2391	}
2392
2393	switch (ioc->hba_mpi_version_belonged) {
2394	case MPI2_VERSION:
2395	raid_set_level(r: mpt2sas_raid_template,
2396	dev: &sdev->sdev_gendev, value: level);
2397	break;
2398	case MPI25_VERSION:
2399	case MPI26_VERSION:
2400	raid_set_level(r: mpt3sas_raid_template,
2401	dev: &sdev->sdev_gendev, value: level);
2402	break;
2403	}
2404	}
2405
2406
2407	/**
2408	* _scsih_get_volume_capabilities - volume capabilities
2409	* @ioc: per adapter object
2410	* @raid_device: the raid_device object
2411	*
2412	* Return: 0 for success, else 1
2413	*/
2414	static int
2415	_scsih_get_volume_capabilities(struct MPT3SAS_ADAPTER *ioc,
2416	struct _raid_device *raid_device)
2417	{
2418	Mpi2RaidVolPage0_t *vol_pg0;
2419	Mpi2RaidPhysDiskPage0_t pd_pg0;
2420	Mpi2SasDevicePage0_t sas_device_pg0;
2421	Mpi2ConfigReply_t mpi_reply;
2422	u16 sz;
2423	u8 num_pds;
2424
2425	if ((mpt3sas_config_get_number_pds(ioc, handle: raid_device->handle,
2426	num_pds: &num_pds)) || !num_pds) {
2427	dfailprintk(ioc,
2428	ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2429	__FILE__, __LINE__, __func__));
2430	return 1;
2431	}
2432
2433	raid_device->num_pds = num_pds;
2434	sz = offsetof(Mpi2RaidVolPage0_t, PhysDisk) + (num_pds *
2435	sizeof(Mpi2RaidVol0PhysDisk_t));
2436	vol_pg0 = kzalloc(size: sz, GFP_KERNEL);
2437	if (!vol_pg0) {
2438	dfailprintk(ioc,
2439	ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2440	__FILE__, __LINE__, __func__));
2441	return 1;
2442	}
2443
2444	if ((mpt3sas_config_get_raid_volume_pg0(ioc, mpi_reply: &mpi_reply, config_page: vol_pg0,
2445	MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle: raid_device->handle, sz))) {
2446	dfailprintk(ioc,
2447	ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2448	__FILE__, __LINE__, __func__));
2449	kfree(objp: vol_pg0);
2450	return 1;
2451	}
2452
2453	raid_device->volume_type = vol_pg0->VolumeType;
2454
2455	/* figure out what the underlying devices are by
2456	* obtaining the device_info bits for the 1st device
2457	*/
2458	if (!(mpt3sas_config_get_phys_disk_pg0(ioc, mpi_reply: &mpi_reply,
2459	config_page: &pd_pg0, MPI2_PHYSDISK_PGAD_FORM_PHYSDISKNUM,
2460	form_specific: vol_pg0->PhysDisk[0].PhysDiskNum))) {
2461	if (!(mpt3sas_config_get_sas_device_pg0(ioc, mpi_reply: &mpi_reply,
2462	config_page: &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE,
2463	le16_to_cpu(pd_pg0.DevHandle)))) {
2464	raid_device->device_info =
2465	le32_to_cpu(sas_device_pg0.DeviceInfo);
2466	}
2467	}
2468
2469	kfree(objp: vol_pg0);
2470	return 0;
2471	}
2472
2473	/**
2474	* _scsih_enable_tlr - setting TLR flags
2475	* @ioc: per adapter object
2476	* @sdev: scsi device struct
2477	*
2478	* Enabling Transaction Layer Retries for tape devices when
2479	* vpd page 0x90 is present
2480	*
2481	*/
2482	static void
2483	_scsih_enable_tlr(struct MPT3SAS_ADAPTER *ioc, struct scsi_device *sdev)
2484	{
2485
2486	/* only for TAPE */
2487	if (sdev->type != TYPE_TAPE)
2488	return;
2489
2490	if (!(ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR))
2491	return;
2492
2493	sas_enable_tlr(sdev);
2494	sdev_printk(KERN_INFO, sdev, "TLR %s\n",
2495	sas_is_tlr_enabled(sdev) ? "Enabled" : "Disabled");
2496	return;
2497
2498	}
2499
2500	/**
2501	* scsih_slave_configure - device configure routine.
2502	* @sdev: scsi device struct
2503	*
2504	* Return: 0 if ok. Any other return is assumed to be an error and
2505	* the device is ignored.
2506	*/
2507	static int
2508	scsih_slave_configure(struct scsi_device *sdev)
2509	{
2510	struct Scsi_Host *shost = sdev->host;
2511	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
2512	struct MPT3SAS_DEVICE *sas_device_priv_data;
2513	struct MPT3SAS_TARGET *sas_target_priv_data;
2514	struct _sas_device *sas_device;
2515	struct _pcie_device *pcie_device;
2516	struct _raid_device *raid_device;
2517	unsigned long flags;
2518	int qdepth;
2519	u8 ssp_target = 0;
2520	char *ds = "";
2521	char *r_level = "";
2522	u16 handle, volume_handle = 0;
2523	u64 volume_wwid = 0;
2524
2525	qdepth = 1;
2526	sas_device_priv_data = sdev->hostdata;
2527	sas_device_priv_data->configured_lun = 1;
2528	sas_device_priv_data->flags &= ~MPT_DEVICE_FLAGS_INIT;
2529	sas_target_priv_data = sas_device_priv_data->sas_target;
2530	handle = sas_target_priv_data->handle;
2531
2532	/* raid volume handling */
2533	if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME) {
2534
2535	spin_lock_irqsave(&ioc->raid_device_lock, flags);
2536	raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
2537	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
2538	if (!raid_device) {
2539	dfailprintk(ioc,
2540	ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2541	__FILE__, __LINE__, __func__));
2542	return 1;
2543	}
2544
2545	if (_scsih_get_volume_capabilities(ioc, raid_device)) {
2546	dfailprintk(ioc,
2547	ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2548	__FILE__, __LINE__, __func__));
2549	return 1;
2550	}
2551
2552	/*
2553	* WARPDRIVE: Initialize the required data for Direct IO
2554	*/
2555	mpt3sas_init_warpdrive_properties(ioc, raid_device);
2556
2557	/* RAID Queue Depth Support
2558	* IS volume = underlying qdepth of drive type, either
2559	* MPT3SAS_SAS_QUEUE_DEPTH or MPT3SAS_SATA_QUEUE_DEPTH
2560	* IM/IME/R10 = 128 (MPT3SAS_RAID_QUEUE_DEPTH)
2561	*/
2562	if (raid_device->device_info &
2563	MPI2_SAS_DEVICE_INFO_SSP_TARGET) {
2564	qdepth = MPT3SAS_SAS_QUEUE_DEPTH;
2565	ds = "SSP";
2566	} else {
2567	qdepth = MPT3SAS_SATA_QUEUE_DEPTH;
2568	if (raid_device->device_info &
2569	MPI2_SAS_DEVICE_INFO_SATA_DEVICE)
2570	ds = "SATA";
2571	else
2572	ds = "STP";
2573	}
2574
2575	switch (raid_device->volume_type) {
2576	case MPI2_RAID_VOL_TYPE_RAID0:
2577	r_level = "RAID0";
2578	break;
2579	case MPI2_RAID_VOL_TYPE_RAID1E:
2580	qdepth = MPT3SAS_RAID_QUEUE_DEPTH;
2581	if (ioc->manu_pg10.OEMIdentifier &&
2582	(le32_to_cpu(ioc->manu_pg10.GenericFlags0) &
2583	MFG10_GF0_R10_DISPLAY) &&
2584	!(raid_device->num_pds % 2))
2585	r_level = "RAID10";
2586	else
2587	r_level = "RAID1E";
2588	break;
2589	case MPI2_RAID_VOL_TYPE_RAID1:
2590	qdepth = MPT3SAS_RAID_QUEUE_DEPTH;
2591	r_level = "RAID1";
2592	break;
2593	case MPI2_RAID_VOL_TYPE_RAID10:
2594	qdepth = MPT3SAS_RAID_QUEUE_DEPTH;
2595	r_level = "RAID10";
2596	break;
2597	case MPI2_RAID_VOL_TYPE_UNKNOWN:
2598	default:
2599	qdepth = MPT3SAS_RAID_QUEUE_DEPTH;
2600	r_level = "RAIDX";
2601	break;
2602	}
2603
2604	if (!ioc->hide_ir_msg)
2605	sdev_printk(KERN_INFO, sdev,
2606	"%s: handle(0x%04x), wwid(0x%016llx),"
2607	" pd_count(%d), type(%s)\n",
2608	r_level, raid_device->handle,
2609	(unsigned long long)raid_device->wwid,
2610	raid_device->num_pds, ds);
2611
2612	if (shost->max_sectors > MPT3SAS_RAID_MAX_SECTORS) {
2613	blk_queue_max_hw_sectors(sdev->request_queue,
2614	MPT3SAS_RAID_MAX_SECTORS);
2615	sdev_printk(KERN_INFO, sdev,
2616	"Set queue's max_sector to: %u\n",
2617	MPT3SAS_RAID_MAX_SECTORS);
2618	}
2619
2620	mpt3sas_scsih_change_queue_depth(sdev, qdepth);
2621
2622	/* raid transport support */
2623	if (!ioc->is_warpdrive)
2624	_scsih_set_level(ioc, sdev, volume_type: raid_device->volume_type);
2625	return 0;
2626	}
2627
2628	/* non-raid handling */
2629	if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_RAID_COMPONENT) {
2630	if (mpt3sas_config_get_volume_handle(ioc, pd_handle: handle,
2631	volume_handle: &volume_handle)) {
2632	dfailprintk(ioc,
2633	ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2634	__FILE__, __LINE__, __func__));
2635	return 1;
2636	}
2637	if (volume_handle && mpt3sas_config_get_volume_wwid(ioc,
2638	volume_handle, wwid: &volume_wwid)) {
2639	dfailprintk(ioc,
2640	ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2641	__FILE__, __LINE__, __func__));
2642	return 1;
2643	}
2644	}
2645
2646	/* PCIe handling */
2647	if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_PCIE_DEVICE) {
2648	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
2649	pcie_device = __mpt3sas_get_pdev_by_wwid(ioc,
2650	wwid: sas_device_priv_data->sas_target->sas_address);
2651	if (!pcie_device) {
2652	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
2653	dfailprintk(ioc,
2654	ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2655	__FILE__, __LINE__, __func__));
2656	return 1;
2657	}
2658
2659	qdepth = ioc->max_nvme_qd;
2660	ds = "NVMe";
2661	sdev_printk(KERN_INFO, sdev,
2662	"%s: handle(0x%04x), wwid(0x%016llx), port(%d)\n",
2663	ds, handle, (unsigned long long)pcie_device->wwid,
2664	pcie_device->port_num);
2665	if (pcie_device->enclosure_handle != 0)
2666	sdev_printk(KERN_INFO, sdev,
2667	"%s: enclosure logical id(0x%016llx), slot(%d)\n",
2668	ds,
2669	(unsigned long long)pcie_device->enclosure_logical_id,
2670	pcie_device->slot);
2671	if (pcie_device->connector_name[0] != '\0')
2672	sdev_printk(KERN_INFO, sdev,
2673	"%s: enclosure level(0x%04x),"
2674	"connector name( %s)\n", ds,
2675	pcie_device->enclosure_level,
2676	pcie_device->connector_name);
2677
2678	if (pcie_device->nvme_mdts)
2679	blk_queue_max_hw_sectors(sdev->request_queue,
2680	pcie_device->nvme_mdts/512);
2681
2682	pcie_device_put(p: pcie_device);
2683	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
2684	mpt3sas_scsih_change_queue_depth(sdev, qdepth);
2685	/* Enable QUEUE_FLAG_NOMERGES flag, so that IOs won't be
2686	** merged and can eliminate holes created during merging
2687	** operation.
2688	**/
2689	blk_queue_flag_set(QUEUE_FLAG_NOMERGES,
2690	q: sdev->request_queue);
2691	blk_queue_virt_boundary(sdev->request_queue,
2692	ioc->page_size - 1);
2693	return 0;
2694	}
2695
2696	spin_lock_irqsave(&ioc->sas_device_lock, flags);
2697	sas_device = __mpt3sas_get_sdev_by_addr(ioc,
2698	sas_address: sas_device_priv_data->sas_target->sas_address,
2699	port: sas_device_priv_data->sas_target->port);
2700	if (!sas_device) {
2701	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
2702	dfailprintk(ioc,
2703	ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2704	__FILE__, __LINE__, __func__));
2705	return 1;
2706	}
2707
2708	sas_device->volume_handle = volume_handle;
2709	sas_device->volume_wwid = volume_wwid;
2710	if (sas_device->device_info & MPI2_SAS_DEVICE_INFO_SSP_TARGET) {
2711	qdepth = (sas_device->port_type > 1) ?
2712	ioc->max_wideport_qd : ioc->max_narrowport_qd;
2713	ssp_target = 1;
2714	if (sas_device->device_info &
2715	MPI2_SAS_DEVICE_INFO_SEP) {
2716	sdev_printk(KERN_WARNING, sdev,
2717	"set ignore_delay_remove for handle(0x%04x)\n",
2718	sas_device_priv_data->sas_target->handle);
2719	sas_device_priv_data->ignore_delay_remove = 1;
2720	ds = "SES";
2721	} else
2722	ds = "SSP";
2723	} else {
2724	qdepth = ioc->max_sata_qd;
2725	if (sas_device->device_info & MPI2_SAS_DEVICE_INFO_STP_TARGET)
2726	ds = "STP";
2727	else if (sas_device->device_info &
2728	MPI2_SAS_DEVICE_INFO_SATA_DEVICE)
2729	ds = "SATA";
2730	}
2731
2732	sdev_printk(KERN_INFO, sdev, "%s: handle(0x%04x), " \
2733	"sas_addr(0x%016llx), phy(%d), device_name(0x%016llx)\n",
2734	ds, handle, (unsigned long long)sas_device->sas_address,
2735	sas_device->phy, (unsigned long long)sas_device->device_name);
2736
2737	_scsih_display_enclosure_chassis_info(NULL, sas_device, sdev, NULL);
2738
2739	sas_device_put(s: sas_device);
2740	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
2741
2742	if (!ssp_target)
2743	_scsih_display_sata_capabilities(ioc, handle, sdev);
2744
2745
2746	mpt3sas_scsih_change_queue_depth(sdev, qdepth);
2747
2748	if (ssp_target) {
2749	sas_read_port_mode_page(sdev);
2750	_scsih_enable_tlr(ioc, sdev);
2751	}
2752
2753	return 0;
2754	}
2755
2756	/**
2757	* scsih_bios_param - fetch head, sector, cylinder info for a disk
2758	* @sdev: scsi device struct
2759	* @bdev: pointer to block device context
2760	* @capacity: device size (in 512 byte sectors)
2761	* @params: three element array to place output:
2762	* params[0] number of heads (max 255)
2763	* params[1] number of sectors (max 63)
2764	* params[2] number of cylinders
2765	*/
2766	static int
2767	scsih_bios_param(struct scsi_device *sdev, struct block_device *bdev,
2768	sector_t capacity, int params[])
2769	{
2770	int heads;
2771	int sectors;
2772	sector_t cylinders;
2773	ulong dummy;
2774
2775	heads = 64;
2776	sectors = 32;
2777
2778	dummy = heads * sectors;
2779	cylinders = capacity;
2780	sector_div(cylinders, dummy);
2781
2782	/*
2783	* Handle extended translation size for logical drives
2784	* > 1Gb
2785	*/
2786	if ((ulong)capacity >= 0x200000) {
2787	heads = 255;
2788	sectors = 63;
2789	dummy = heads * sectors;
2790	cylinders = capacity;
2791	sector_div(cylinders, dummy);
2792	}
2793
2794	/* return result */
2795	params[0] = heads;
2796	params[1] = sectors;
2797	params[2] = cylinders;
2798
2799	return 0;
2800	}
2801
2802	/**
2803	* _scsih_response_code - translation of device response code
2804	* @ioc: per adapter object
2805	* @response_code: response code returned by the device
2806	*/
2807	static void
2808	_scsih_response_code(struct MPT3SAS_ADAPTER *ioc, u8 response_code)
2809	{
2810	char *desc;
2811
2812	switch (response_code) {
2813	case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
2814	desc = "task management request completed";
2815	break;
2816	case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
2817	desc = "invalid frame";
2818	break;
2819	case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
2820	desc = "task management request not supported";
2821	break;
2822	case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
2823	desc = "task management request failed";
2824	break;
2825	case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
2826	desc = "task management request succeeded";
2827	break;
2828	case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
2829	desc = "invalid lun";
2830	break;
2831	case 0xA:
2832	desc = "overlapped tag attempted";
2833	break;
2834	case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
2835	desc = "task queued, however not sent to target";
2836	break;
2837	default:
2838	desc = "unknown";
2839	break;
2840	}
2841	ioc_warn(ioc, "response_code(0x%01x): %s\n", response_code, desc);
2842	}
2843
2844	/**
2845	* _scsih_tm_done - tm completion routine
2846	* @ioc: per adapter object
2847	* @smid: system request message index
2848	* @msix_index: MSIX table index supplied by the OS
2849	* @reply: reply message frame(lower 32bit addr)
2850	* Context: none.
2851	*
2852	* The callback handler when using scsih_issue_tm.
2853	*
2854	* Return: 1 meaning mf should be freed from _base_interrupt
2855	* 0 means the mf is freed from this function.
2856	*/
2857	static u8
2858	_scsih_tm_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
2859	{
2860	MPI2DefaultReply_t *mpi_reply;
2861
2862	if (ioc->tm_cmds.status == MPT3_CMD_NOT_USED)
2863	return 1;
2864	if (ioc->tm_cmds.smid != smid)
2865	return 1;
2866	ioc->tm_cmds.status |= MPT3_CMD_COMPLETE;
2867	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, phys_addr: reply);
2868	if (mpi_reply) {
2869	memcpy(ioc->tm_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
2870	ioc->tm_cmds.status |= MPT3_CMD_REPLY_VALID;
2871	}
2872	ioc->tm_cmds.status &= ~MPT3_CMD_PENDING;
2873	complete(&ioc->tm_cmds.done);
2874	return 1;
2875	}
2876
2877	/**
2878	* mpt3sas_scsih_set_tm_flag - set per target tm_busy
2879	* @ioc: per adapter object
2880	* @handle: device handle
2881	*
2882	* During taskmangement request, we need to freeze the device queue.
2883	*/
2884	void
2885	mpt3sas_scsih_set_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
2886	{
2887	struct MPT3SAS_DEVICE *sas_device_priv_data;
2888	struct scsi_device *sdev;
2889	u8 skip = 0;
2890
2891	shost_for_each_device(sdev, ioc->shost) {
2892	if (skip)
2893	continue;
2894	sas_device_priv_data = sdev->hostdata;
2895	if (!sas_device_priv_data)
2896	continue;
2897	if (sas_device_priv_data->sas_target->handle == handle) {
2898	sas_device_priv_data->sas_target->tm_busy = 1;
2899	skip = 1;
2900	ioc->ignore_loginfos = 1;
2901	}
2902	}
2903	}
2904
2905	/**
2906	* mpt3sas_scsih_clear_tm_flag - clear per target tm_busy
2907	* @ioc: per adapter object
2908	* @handle: device handle
2909	*
2910	* During taskmangement request, we need to freeze the device queue.
2911	*/
2912	void
2913	mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
2914	{
2915	struct MPT3SAS_DEVICE *sas_device_priv_data;
2916	struct scsi_device *sdev;
2917	u8 skip = 0;
2918
2919	shost_for_each_device(sdev, ioc->shost) {
2920	if (skip)
2921	continue;
2922	sas_device_priv_data = sdev->hostdata;
2923	if (!sas_device_priv_data)
2924	continue;
2925	if (sas_device_priv_data->sas_target->handle == handle) {
2926	sas_device_priv_data->sas_target->tm_busy = 0;
2927	skip = 1;
2928	ioc->ignore_loginfos = 0;
2929	}
2930	}
2931	}
2932
2933	/**
2934	* scsih_tm_cmd_map_status - map the target reset & LUN reset TM status
2935	* @ioc: per adapter object
2936	* @channel: the channel assigned by the OS
2937	* @id: the id assigned by the OS
2938	* @lun: lun number
2939	* @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in mpi2_init.h)
2940	* @smid_task: smid assigned to the task
2941	*
2942	* Look whether TM has aborted the timed out SCSI command, if
2943	* TM has aborted the IO then return SUCCESS else return FAILED.
2944	*/
2945	static int
2946	scsih_tm_cmd_map_status(struct MPT3SAS_ADAPTER *ioc, uint channel,
2947	uint id, uint lun, u8 type, u16 smid_task)
2948	{
2949
2950	if (smid_task <= ioc->shost->can_queue) {
2951	switch (type) {
2952	case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
2953	if (!(_scsih_scsi_lookup_find_by_target(ioc,
2954	id, channel)))
2955	return SUCCESS;
2956	break;
2957	case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
2958	case MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
2959	if (!(_scsih_scsi_lookup_find_by_lun(ioc, id,
2960	lun, channel)))
2961	return SUCCESS;
2962	break;
2963	default:
2964	return SUCCESS;
2965	}
2966	} else if (smid_task == ioc->scsih_cmds.smid) {
2967	if ((ioc->scsih_cmds.status & MPT3_CMD_COMPLETE) ||
2968	(ioc->scsih_cmds.status & MPT3_CMD_NOT_USED))
2969	return SUCCESS;
2970	} else if (smid_task == ioc->ctl_cmds.smid) {
2971	if ((ioc->ctl_cmds.status & MPT3_CMD_COMPLETE) ||
2972	(ioc->ctl_cmds.status & MPT3_CMD_NOT_USED))
2973	return SUCCESS;
2974	}
2975
2976	return FAILED;
2977	}
2978
2979	/**
2980	* scsih_tm_post_processing - post processing of target & LUN reset
2981	* @ioc: per adapter object
2982	* @handle: device handle
2983	* @channel: the channel assigned by the OS
2984	* @id: the id assigned by the OS
2985	* @lun: lun number
2986	* @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in mpi2_init.h)
2987	* @smid_task: smid assigned to the task
2988	*
2989	* Post processing of target & LUN reset. Due to interrupt latency
2990	* issue it possible that interrupt for aborted IO might not be
2991	* received yet. So before returning failure status, poll the
2992	* reply descriptor pools for the reply of timed out SCSI command.
2993	* Return FAILED status if reply for timed out is not received
2994	* otherwise return SUCCESS.
2995	*/
2996	static int
2997	scsih_tm_post_processing(struct MPT3SAS_ADAPTER *ioc, u16 handle,
2998	uint channel, uint id, uint lun, u8 type, u16 smid_task)
2999	{
3000	int rc;
3001
3002	rc = scsih_tm_cmd_map_status(ioc, channel, id, lun, type, smid_task);
3003	if (rc == SUCCESS)
3004	return rc;
3005
3006	ioc_info(ioc,
3007	"Poll ReplyDescriptor queues for completion of"
3008	" smid(%d), task_type(0x%02x), handle(0x%04x)\n",
3009	smid_task, type, handle);
3010
3011	/*
3012	* Due to interrupt latency issues, driver may receive interrupt for
3013	* TM first and then for aborted SCSI IO command. So, poll all the
3014	* ReplyDescriptor pools before returning the FAILED status to SML.
3015	*/
3016	mpt3sas_base_mask_interrupts(ioc);
3017	mpt3sas_base_sync_reply_irqs(ioc, poll: 1);
3018	mpt3sas_base_unmask_interrupts(ioc);
3019
3020	return scsih_tm_cmd_map_status(ioc, channel, id, lun, type, smid_task);
3021	}
3022
3023	/**
3024	* mpt3sas_scsih_issue_tm - main routine for sending tm requests
3025	* @ioc: per adapter struct
3026	* @handle: device handle
3027	* @channel: the channel assigned by the OS
3028	* @id: the id assigned by the OS
3029	* @lun: lun number
3030	* @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in mpi2_init.h)
3031	* @smid_task: smid assigned to the task
3032	* @msix_task: MSIX table index supplied by the OS
3033	* @timeout: timeout in seconds
3034	* @tr_method: Target Reset Method
3035	* Context: user
3036	*
3037	* A generic API for sending task management requests to firmware.
3038	*
3039	* The callback index is set inside `ioc->tm_cb_idx`.
3040	* The caller is responsible to check for outstanding commands.
3041	*
3042	* Return: SUCCESS or FAILED.
3043	*/
3044	int
3045	mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, uint channel,
3046	uint id, u64 lun, u8 type, u16 smid_task, u16 msix_task,
3047	u8 timeout, u8 tr_method)
3048	{
3049	Mpi2SCSITaskManagementRequest_t *mpi_request;
3050	Mpi2SCSITaskManagementReply_t *mpi_reply;
3051	Mpi25SCSIIORequest_t *request;
3052	u16 smid = 0;
3053	u32 ioc_state;
3054	int rc;
3055	u8 issue_reset = 0;
3056
3057	lockdep_assert_held(&ioc->tm_cmds.mutex);
3058
3059	if (ioc->tm_cmds.status != MPT3_CMD_NOT_USED) {
3060	ioc_info(ioc, "%s: tm_cmd busy!!!\n", __func__);
3061	return FAILED;
3062	}
3063
3064	if (ioc->shost_recovery || ioc->remove_host ||
3065	ioc->pci_error_recovery) {
3066	ioc_info(ioc, "%s: host reset in progress!\n", __func__);
3067	return FAILED;
3068	}
3069
3070	ioc_state = mpt3sas_base_get_iocstate(ioc, cooked: 0);
3071	if (ioc_state & MPI2_DOORBELL_USED) {
3072	dhsprintk(ioc, ioc_info(ioc, "unexpected doorbell active!\n"));
3073	rc = mpt3sas_base_hard_reset_handler(ioc, type: FORCE_BIG_HAMMER);
3074	return (!rc) ? SUCCESS : FAILED;
3075	}
3076
3077	if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
3078	mpt3sas_print_fault_code(ioc, ioc_state &
3079	MPI2_DOORBELL_DATA_MASK);
3080	rc = mpt3sas_base_hard_reset_handler(ioc, type: FORCE_BIG_HAMMER);
3081	return (!rc) ? SUCCESS : FAILED;
3082	} else if ((ioc_state & MPI2_IOC_STATE_MASK) ==
3083	MPI2_IOC_STATE_COREDUMP) {
3084	mpt3sas_print_coredump_info(ioc, ioc_state &
3085	MPI2_DOORBELL_DATA_MASK);
3086	rc = mpt3sas_base_hard_reset_handler(ioc, type: FORCE_BIG_HAMMER);
3087	return (!rc) ? SUCCESS : FAILED;
3088	}
3089
3090	smid = mpt3sas_base_get_smid_hpr(ioc, cb_idx: ioc->tm_cb_idx);
3091	if (!smid) {
3092	ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
3093	return FAILED;
3094	}
3095
3096	dtmprintk(ioc,
3097	ioc_info(ioc, "sending tm: handle(0x%04x), task_type(0x%02x), smid(%d), timeout(%d), tr_method(0x%x)\n",
3098	handle, type, smid_task, timeout, tr_method));
3099	ioc->tm_cmds.status = MPT3_CMD_PENDING;
3100	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
3101	ioc->tm_cmds.smid = smid;
3102	memset(mpi_request, 0, sizeof(Mpi2SCSITaskManagementRequest_t));
3103	memset(ioc->tm_cmds.reply, 0, sizeof(Mpi2SCSITaskManagementReply_t));
3104	mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
3105	mpi_request->DevHandle = cpu_to_le16(handle);
3106	mpi_request->TaskType = type;
3107	if (type == MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK ||
3108	type == MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK)
3109	mpi_request->MsgFlags = tr_method;
3110	mpi_request->TaskMID = cpu_to_le16(smid_task);
3111	int_to_scsilun(lun, (struct scsi_lun *)mpi_request->LUN);
3112	mpt3sas_scsih_set_tm_flag(ioc, handle);
3113	init_completion(x: &ioc->tm_cmds.done);
3114	ioc->put_smid_hi_priority(ioc, smid, msix_task);
3115	wait_for_completion_timeout(x: &ioc->tm_cmds.done, timeout: timeout*HZ);
3116	if (!(ioc->tm_cmds.status & MPT3_CMD_COMPLETE)) {
3117	mpt3sas_check_cmd_timeout(ioc,
3118	ioc->tm_cmds.status, mpi_request,
3119	sizeof(Mpi2SCSITaskManagementRequest_t)/4, issue_reset);
3120	if (issue_reset) {
3121	rc = mpt3sas_base_hard_reset_handler(ioc,
3122	type: FORCE_BIG_HAMMER);
3123	rc = (!rc) ? SUCCESS : FAILED;
3124	goto out;
3125	}
3126	}
3127
3128	/* sync IRQs in case those were busy during flush. */
3129	mpt3sas_base_sync_reply_irqs(ioc, poll: 0);
3130
3131	if (ioc->tm_cmds.status & MPT3_CMD_REPLY_VALID) {
3132	mpt3sas_trigger_master(ioc, MASTER_TRIGGER_TASK_MANAGMENT);
3133	mpi_reply = ioc->tm_cmds.reply;
3134	dtmprintk(ioc,
3135	ioc_info(ioc, "complete tm: ioc_status(0x%04x), loginfo(0x%08x), term_count(0x%08x)\n",
3136	le16_to_cpu(mpi_reply->IOCStatus),
3137	le32_to_cpu(mpi_reply->IOCLogInfo),
3138	le32_to_cpu(mpi_reply->TerminationCount)));
3139	if (ioc->logging_level & MPT_DEBUG_TM) {
3140	_scsih_response_code(ioc, response_code: mpi_reply->ResponseCode);
3141	if (mpi_reply->IOCStatus)
3142	_debug_dump_mf(mpi_request,
3143	sz: sizeof(Mpi2SCSITaskManagementRequest_t)/4);
3144	}
3145	}
3146
3147	switch (type) {
3148	case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
3149	rc = SUCCESS;
3150	/*
3151	* If DevHandle filed in smid_task's entry of request pool
3152	* doesn't match with device handle on which this task abort
3153	* TM is received then it means that TM has successfully
3154	* aborted the timed out command. Since smid_task's entry in
3155	* request pool will be memset to zero once the timed out
3156	* command is returned to the SML. If the command is not
3157	* aborted then smid_task’s entry won’t be cleared and it
3158	* will have same DevHandle value on which this task abort TM
3159	* is received and driver will return the TM status as FAILED.
3160	*/
3161	request = mpt3sas_base_get_msg_frame(ioc, smid: smid_task);
3162	if (le16_to_cpu(request->DevHandle) != handle)
3163	break;
3164
3165	ioc_info(ioc, "Task abort tm failed: handle(0x%04x),"
3166	"timeout(%d) tr_method(0x%x) smid(%d) msix_index(%d)\n",
3167	handle, timeout, tr_method, smid_task, msix_task);
3168	rc = FAILED;
3169	break;
3170
3171	case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
3172	case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
3173	case MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
3174	rc = scsih_tm_post_processing(ioc, handle, channel, id, lun,
3175	type, smid_task);
3176	break;
3177	case MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK:
3178	rc = SUCCESS;
3179	break;
3180	default:
3181	rc = FAILED;
3182	break;
3183	}
3184
3185	out:
3186	mpt3sas_scsih_clear_tm_flag(ioc, handle);
3187	ioc->tm_cmds.status = MPT3_CMD_NOT_USED;
3188	return rc;
3189	}
3190
3191	int mpt3sas_scsih_issue_locked_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
3192	uint channel, uint id, u64 lun, u8 type, u16 smid_task,
3193	u16 msix_task, u8 timeout, u8 tr_method)
3194	{
3195	int ret;
3196
3197	mutex_lock(&ioc->tm_cmds.mutex);
3198	ret = mpt3sas_scsih_issue_tm(ioc, handle, channel, id, lun, type,
3199	smid_task, msix_task, timeout, tr_method);
3200	mutex_unlock(lock: &ioc->tm_cmds.mutex);
3201
3202	return ret;
3203	}
3204
3205	/**
3206	* _scsih_tm_display_info - displays info about the device
3207	* @ioc: per adapter struct
3208	* @scmd: pointer to scsi command object
3209	*
3210	* Called by task management callback handlers.
3211	*/
3212	static void
3213	_scsih_tm_display_info(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd)
3214	{
3215	struct scsi_target *starget = scmd->device->sdev_target;
3216	struct MPT3SAS_TARGET *priv_target = starget->hostdata;
3217	struct _sas_device *sas_device = NULL;
3218	struct _pcie_device *pcie_device = NULL;
3219	unsigned long flags;
3220	char *device_str = NULL;
3221
3222	if (!priv_target)
3223	return;
3224	if (ioc->hide_ir_msg)
3225	device_str = "WarpDrive";
3226	else
3227	device_str = "volume";
3228
3229	scsi_print_command(scmd);
3230	if (priv_target->flags & MPT_TARGET_FLAGS_VOLUME) {
3231	starget_printk(KERN_INFO, starget,
3232	"%s handle(0x%04x), %s wwid(0x%016llx)\n",
3233	device_str, priv_target->handle,
3234	device_str, (unsigned long long)priv_target->sas_address);
3235
3236	} else if (priv_target->flags & MPT_TARGET_FLAGS_PCIE_DEVICE) {
3237	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
3238	pcie_device = __mpt3sas_get_pdev_from_target(ioc, tgt_priv: priv_target);
3239	if (pcie_device) {
3240	starget_printk(KERN_INFO, starget,
3241	"handle(0x%04x), wwid(0x%016llx), port(%d)\n",
3242	pcie_device->handle,
3243	(unsigned long long)pcie_device->wwid,
3244	pcie_device->port_num);
3245	if (pcie_device->enclosure_handle != 0)
3246	starget_printk(KERN_INFO, starget,
3247	"enclosure logical id(0x%016llx), slot(%d)\n",
3248	(unsigned long long)
3249	pcie_device->enclosure_logical_id,
3250	pcie_device->slot);
3251	if (pcie_device->connector_name[0] != '\0')
3252	starget_printk(KERN_INFO, starget,
3253	"enclosure level(0x%04x), connector name( %s)\n",
3254	pcie_device->enclosure_level,
3255	pcie_device->connector_name);
3256	pcie_device_put(p: pcie_device);
3257	}
3258	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
3259
3260	} else {
3261	spin_lock_irqsave(&ioc->sas_device_lock, flags);
3262	sas_device = __mpt3sas_get_sdev_from_target(ioc, tgt_priv: priv_target);
3263	if (sas_device) {
3264	if (priv_target->flags &
3265	MPT_TARGET_FLAGS_RAID_COMPONENT) {
3266	starget_printk(KERN_INFO, starget,
3267	"volume handle(0x%04x), "
3268	"volume wwid(0x%016llx)\n",
3269	sas_device->volume_handle,
3270	(unsigned long long)sas_device->volume_wwid);
3271	}
3272	starget_printk(KERN_INFO, starget,
3273	"handle(0x%04x), sas_address(0x%016llx), phy(%d)\n",
3274	sas_device->handle,
3275	(unsigned long long)sas_device->sas_address,
3276	sas_device->phy);
3277
3278	_scsih_display_enclosure_chassis_info(NULL, sas_device,
3279	NULL, starget);
3280
3281	sas_device_put(s: sas_device);
3282	}
3283	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
3284	}
3285	}
3286
3287	/**
3288	* scsih_abort - eh threads main abort routine
3289	* @scmd: pointer to scsi command object
3290	*
3291	* Return: SUCCESS if command aborted else FAILED
3292	*/
3293	static int
3294	scsih_abort(struct scsi_cmnd *scmd)
3295	{
3296	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost: scmd->device->host);
3297	struct MPT3SAS_DEVICE *sas_device_priv_data;
3298	struct scsiio_tracker *st = scsi_cmd_priv(cmd: scmd);
3299	u16 handle;
3300	int r;
3301
3302	u8 timeout = 30;
3303	struct _pcie_device *pcie_device = NULL;
3304	sdev_printk(KERN_INFO, scmd->device, "attempting task abort!"
3305	"scmd(0x%p), outstanding for %u ms & timeout %u ms\n",
3306	scmd, jiffies_to_msecs(jiffies - scmd->jiffies_at_alloc),
3307	(scsi_cmd_to_rq(scmd)->timeout / HZ) * 1000);
3308	_scsih_tm_display_info(ioc, scmd);
3309
3310	sas_device_priv_data = scmd->device->hostdata;
3311	if (!sas_device_priv_data || !sas_device_priv_data->sas_target ||
3312	ioc->remove_host) {
3313	sdev_printk(KERN_INFO, scmd->device,
3314	"device been deleted! scmd(0x%p)\n", scmd);
3315	scmd->result = DID_NO_CONNECT << 16;
3316	scsi_done(cmd: scmd);
3317	r = SUCCESS;
3318	goto out;
3319	}
3320
3321	/* check for completed command */
3322	if (st == NULL || st->cb_idx == 0xFF) {
3323	sdev_printk(KERN_INFO, scmd->device, "No reference found at "
3324	"driver, assuming scmd(0x%p) might have completed\n", scmd);
3325	scmd->result = DID_RESET << 16;
3326	r = SUCCESS;
3327	goto out;
3328	}
3329
3330	/* for hidden raid components and volumes this is not supported */
3331	if (sas_device_priv_data->sas_target->flags &
3332	MPT_TARGET_FLAGS_RAID_COMPONENT ||
3333	sas_device_priv_data->sas_target->flags & MPT_TARGET_FLAGS_VOLUME) {
3334	scmd->result = DID_RESET << 16;
3335	r = FAILED;
3336	goto out;
3337	}
3338
3339	mpt3sas_halt_firmware(ioc);
3340
3341	handle = sas_device_priv_data->sas_target->handle;
3342	pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
3343	if (pcie_device && (!ioc->tm_custom_handling) &&
3344	(!(mpt3sas_scsih_is_pcie_scsi_device(device_info: pcie_device->device_info))))
3345	timeout = ioc->nvme_abort_timeout;
3346	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, channel: scmd->device->channel,
3347	id: scmd->device->id, lun: scmd->device->lun,
3348	MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
3349	smid_task: st->smid, msix_task: st->msix_io, timeout, tr_method: 0);
3350	/* Command must be cleared after abort */
3351	if (r == SUCCESS && st->cb_idx != 0xFF)
3352	r = FAILED;
3353	out:
3354	sdev_printk(KERN_INFO, scmd->device, "task abort: %s scmd(0x%p)\n",
3355	((r == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
3356	if (pcie_device)
3357	pcie_device_put(p: pcie_device);
3358	return r;
3359	}
3360
3361	/**
3362	* scsih_dev_reset - eh threads main device reset routine
3363	* @scmd: pointer to scsi command object
3364	*
3365	* Return: SUCCESS if command aborted else FAILED
3366	*/
3367	static int
3368	scsih_dev_reset(struct scsi_cmnd *scmd)
3369	{
3370	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost: scmd->device->host);
3371	struct MPT3SAS_DEVICE *sas_device_priv_data;
3372	struct _sas_device *sas_device = NULL;
3373	struct _pcie_device *pcie_device = NULL;
3374	u16 handle;
3375	u8 tr_method = 0;
3376	u8 tr_timeout = 30;
3377	int r;
3378
3379	struct scsi_target *starget = scmd->device->sdev_target;
3380	struct MPT3SAS_TARGET *target_priv_data = starget->hostdata;
3381
3382	sdev_printk(KERN_INFO, scmd->device,
3383	"attempting device reset! scmd(0x%p)\n", scmd);
3384	_scsih_tm_display_info(ioc, scmd);
3385
3386	sas_device_priv_data = scmd->device->hostdata;
3387	if (!sas_device_priv_data || !sas_device_priv_data->sas_target ||
3388	ioc->remove_host) {
3389	sdev_printk(KERN_INFO, scmd->device,
3390	"device been deleted! scmd(0x%p)\n", scmd);
3391	scmd->result = DID_NO_CONNECT << 16;
3392	scsi_done(cmd: scmd);
3393	r = SUCCESS;
3394	goto out;
3395	}
3396
3397	/* for hidden raid components obtain the volume_handle */
3398	handle = 0;
3399	if (sas_device_priv_data->sas_target->flags &
3400	MPT_TARGET_FLAGS_RAID_COMPONENT) {
3401	sas_device = mpt3sas_get_sdev_from_target(ioc,
3402	tgt_priv: target_priv_data);
3403	if (sas_device)
3404	handle = sas_device->volume_handle;
3405	} else
3406	handle = sas_device_priv_data->sas_target->handle;
3407
3408	if (!handle) {
3409	scmd->result = DID_RESET << 16;
3410	r = FAILED;
3411	goto out;
3412	}
3413
3414	pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
3415
3416	if (pcie_device && (!ioc->tm_custom_handling) &&
3417	(!(mpt3sas_scsih_is_pcie_scsi_device(device_info: pcie_device->device_info)))) {
3418	tr_timeout = pcie_device->reset_timeout;
3419	tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
3420	} else
3421	tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3422
3423	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, channel: scmd->device->channel,
3424	id: scmd->device->id, lun: scmd->device->lun,
3425	MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET, smid_task: 0, msix_task: 0,
3426	timeout: tr_timeout, tr_method);
3427	/* Check for busy commands after reset */
3428	if (r == SUCCESS && scsi_device_busy(sdev: scmd->device))
3429	r = FAILED;
3430	out:
3431	sdev_printk(KERN_INFO, scmd->device, "device reset: %s scmd(0x%p)\n",
3432	((r == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
3433
3434	if (sas_device)
3435	sas_device_put(s: sas_device);
3436	if (pcie_device)
3437	pcie_device_put(p: pcie_device);
3438
3439	return r;
3440	}
3441
3442	/**
3443	* scsih_target_reset - eh threads main target reset routine
3444	* @scmd: pointer to scsi command object
3445	*
3446	* Return: SUCCESS if command aborted else FAILED
3447	*/
3448	static int
3449	scsih_target_reset(struct scsi_cmnd *scmd)
3450	{
3451	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost: scmd->device->host);
3452	struct MPT3SAS_DEVICE *sas_device_priv_data;
3453	struct _sas_device *sas_device = NULL;
3454	struct _pcie_device *pcie_device = NULL;
3455	u16 handle;
3456	u8 tr_method = 0;
3457	u8 tr_timeout = 30;
3458	int r;
3459	struct scsi_target *starget = scmd->device->sdev_target;
3460	struct MPT3SAS_TARGET *target_priv_data = starget->hostdata;
3461
3462	starget_printk(KERN_INFO, starget,
3463	"attempting target reset! scmd(0x%p)\n", scmd);
3464	_scsih_tm_display_info(ioc, scmd);
3465
3466	sas_device_priv_data = scmd->device->hostdata;
3467	if (!sas_device_priv_data || !sas_device_priv_data->sas_target ||
3468	ioc->remove_host) {
3469	starget_printk(KERN_INFO, starget,
3470	"target been deleted! scmd(0x%p)\n", scmd);
3471	scmd->result = DID_NO_CONNECT << 16;
3472	scsi_done(cmd: scmd);
3473	r = SUCCESS;
3474	goto out;
3475	}
3476
3477	/* for hidden raid components obtain the volume_handle */
3478	handle = 0;
3479	if (sas_device_priv_data->sas_target->flags &
3480	MPT_TARGET_FLAGS_RAID_COMPONENT) {
3481	sas_device = mpt3sas_get_sdev_from_target(ioc,
3482	tgt_priv: target_priv_data);
3483	if (sas_device)
3484	handle = sas_device->volume_handle;
3485	} else
3486	handle = sas_device_priv_data->sas_target->handle;
3487
3488	if (!handle) {
3489	scmd->result = DID_RESET << 16;
3490	r = FAILED;
3491	goto out;
3492	}
3493
3494	pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
3495
3496	if (pcie_device && (!ioc->tm_custom_handling) &&
3497	(!(mpt3sas_scsih_is_pcie_scsi_device(device_info: pcie_device->device_info)))) {
3498	tr_timeout = pcie_device->reset_timeout;
3499	tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
3500	} else
3501	tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3502	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, channel: scmd->device->channel,
3503	id: scmd->device->id, lun: 0,
3504	MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, smid_task: 0, msix_task: 0,
3505	timeout: tr_timeout, tr_method);
3506	/* Check for busy commands after reset */
3507	if (r == SUCCESS && atomic_read(v: &starget->target_busy))
3508	r = FAILED;
3509	out:
3510	starget_printk(KERN_INFO, starget, "target reset: %s scmd(0x%p)\n",
3511	((r == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
3512
3513	if (sas_device)
3514	sas_device_put(s: sas_device);
3515	if (pcie_device)
3516	pcie_device_put(p: pcie_device);
3517	return r;
3518	}
3519
3520
3521	/**
3522	* scsih_host_reset - eh threads main host reset routine
3523	* @scmd: pointer to scsi command object
3524	*
3525	* Return: SUCCESS if command aborted else FAILED
3526	*/
3527	static int
3528	scsih_host_reset(struct scsi_cmnd *scmd)
3529	{
3530	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost: scmd->device->host);
3531	int r, retval;
3532
3533	ioc_info(ioc, "attempting host reset! scmd(0x%p)\n", scmd);
3534	scsi_print_command(scmd);
3535
3536	if (ioc->is_driver_loading || ioc->remove_host) {
3537	ioc_info(ioc, "Blocking the host reset\n");
3538	r = FAILED;
3539	goto out;
3540	}
3541
3542	retval = mpt3sas_base_hard_reset_handler(ioc, type: FORCE_BIG_HAMMER);
3543	r = (retval < 0) ? FAILED : SUCCESS;
3544	out:
3545	ioc_info(ioc, "host reset: %s scmd(0x%p)\n",
3546	r == SUCCESS ? "SUCCESS" : "FAILED", scmd);
3547
3548	return r;
3549	}
3550
3551	/**
3552	* _scsih_fw_event_add - insert and queue up fw_event
3553	* @ioc: per adapter object
3554	* @fw_event: object describing the event
3555	* Context: This function will acquire ioc->fw_event_lock.
3556	*
3557	* This adds the firmware event object into link list, then queues it up to
3558	* be processed from user context.
3559	*/
3560	static void
3561	_scsih_fw_event_add(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)
3562	{
3563	unsigned long flags;
3564
3565	if (ioc->firmware_event_thread == NULL)
3566	return;
3567
3568	spin_lock_irqsave(&ioc->fw_event_lock, flags);
3569	fw_event_work_get(fw_work: fw_event);
3570	INIT_LIST_HEAD(list: &fw_event->list);
3571	list_add_tail(new: &fw_event->list, head: &ioc->fw_event_list);
3572	INIT_WORK(&fw_event->work, _firmware_event_work);
3573	fw_event_work_get(fw_work: fw_event);
3574	queue_work(wq: ioc->firmware_event_thread, work: &fw_event->work);
3575	spin_unlock_irqrestore(lock: &ioc->fw_event_lock, flags);
3576	}
3577
3578	/**
3579	* _scsih_fw_event_del_from_list - delete fw_event from the list
3580	* @ioc: per adapter object
3581	* @fw_event: object describing the event
3582	* Context: This function will acquire ioc->fw_event_lock.
3583	*
3584	* If the fw_event is on the fw_event_list, remove it and do a put.
3585	*/
3586	static void
3587	_scsih_fw_event_del_from_list(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work
3588	*fw_event)
3589	{
3590	unsigned long flags;
3591
3592	spin_lock_irqsave(&ioc->fw_event_lock, flags);
3593	if (!list_empty(head: &fw_event->list)) {
3594	list_del_init(entry: &fw_event->list);
3595	fw_event_work_put(fw_work: fw_event);
3596	}
3597	spin_unlock_irqrestore(lock: &ioc->fw_event_lock, flags);
3598	}
3599
3600
3601	/**
3602	* mpt3sas_send_trigger_data_event - send event for processing trigger data
3603	* @ioc: per adapter object
3604	* @event_data: trigger event data
3605	*/
3606	void
3607	mpt3sas_send_trigger_data_event(struct MPT3SAS_ADAPTER *ioc,
3608	struct SL_WH_TRIGGERS_EVENT_DATA_T *event_data)
3609	{
3610	struct fw_event_work *fw_event;
3611	u16 sz;
3612
3613	if (ioc->is_driver_loading)
3614	return;
3615	sz = sizeof(*event_data);
3616	fw_event = alloc_fw_event_work(len: sz);
3617	if (!fw_event)
3618	return;
3619	fw_event->event = MPT3SAS_PROCESS_TRIGGER_DIAG;
3620	fw_event->ioc = ioc;
3621	memcpy(fw_event->event_data, event_data, sizeof(*event_data));
3622	_scsih_fw_event_add(ioc, fw_event);
3623	fw_event_work_put(fw_work: fw_event);
3624	}
3625
3626	/**
3627	* _scsih_error_recovery_delete_devices - remove devices not responding
3628	* @ioc: per adapter object
3629	*/
3630	static void
3631	_scsih_error_recovery_delete_devices(struct MPT3SAS_ADAPTER *ioc)
3632	{
3633	struct fw_event_work *fw_event;
3634
3635	fw_event = alloc_fw_event_work(len: 0);
3636	if (!fw_event)
3637	return;
3638	fw_event->event = MPT3SAS_REMOVE_UNRESPONDING_DEVICES;
3639	fw_event->ioc = ioc;
3640	_scsih_fw_event_add(ioc, fw_event);
3641	fw_event_work_put(fw_work: fw_event);
3642	}
3643
3644	/**
3645	* mpt3sas_port_enable_complete - port enable completed (fake event)
3646	* @ioc: per adapter object
3647	*/
3648	void
3649	mpt3sas_port_enable_complete(struct MPT3SAS_ADAPTER *ioc)
3650	{
3651	struct fw_event_work *fw_event;
3652
3653	fw_event = alloc_fw_event_work(len: 0);
3654	if (!fw_event)
3655	return;
3656	fw_event->event = MPT3SAS_PORT_ENABLE_COMPLETE;
3657	fw_event->ioc = ioc;
3658	_scsih_fw_event_add(ioc, fw_event);
3659	fw_event_work_put(fw_work: fw_event);
3660	}
3661
3662	static struct fw_event_work *dequeue_next_fw_event(struct MPT3SAS_ADAPTER *ioc)
3663	{
3664	unsigned long flags;
3665	struct fw_event_work *fw_event = NULL;
3666
3667	spin_lock_irqsave(&ioc->fw_event_lock, flags);
3668	if (!list_empty(head: &ioc->fw_event_list)) {
3669	fw_event = list_first_entry(&ioc->fw_event_list,
3670	struct fw_event_work, list);
3671	list_del_init(entry: &fw_event->list);
3672	fw_event_work_put(fw_work: fw_event);
3673	}
3674	spin_unlock_irqrestore(lock: &ioc->fw_event_lock, flags);
3675
3676	return fw_event;
3677	}
3678
3679	/**
3680	* _scsih_fw_event_cleanup_queue - cleanup event queue
3681	* @ioc: per adapter object
3682	*
3683	* Walk the firmware event queue, either killing timers, or waiting
3684	* for outstanding events to complete
3685	*
3686	* Context: task, can sleep
3687	*/
3688	static void
3689	_scsih_fw_event_cleanup_queue(struct MPT3SAS_ADAPTER *ioc)
3690	{
3691	struct fw_event_work *fw_event;
3692
3693	if ((list_empty(head: &ioc->fw_event_list) && !ioc->current_event) ||
3694	!ioc->firmware_event_thread)
3695	return;
3696	/*
3697	* Set current running event as ignore, so that
3698	* current running event will exit quickly.
3699	* As diag reset has occurred it is of no use
3700	* to process remaining stale event data entries.
3701	*/
3702	if (ioc->shost_recovery && ioc->current_event)
3703	ioc->current_event->ignore = 1;
3704
3705	ioc->fw_events_cleanup = 1;
3706	while ((fw_event = dequeue_next_fw_event(ioc)) ||
3707	(fw_event = ioc->current_event)) {
3708
3709	/*
3710	* Don't call cancel_work_sync() for current_event
3711	* other than MPT3SAS_REMOVE_UNRESPONDING_DEVICES;
3712	* otherwise we may observe deadlock if current
3713	* hard reset issued as part of processing the current_event.
3714	*
3715	* Orginal logic of cleaning the current_event is added
3716	* for handling the back to back host reset issued by the user.
3717	* i.e. during back to back host reset, driver use to process
3718	* the two instances of MPT3SAS_REMOVE_UNRESPONDING_DEVICES
3719	* event back to back and this made the drives to unregister
3720	* the devices from SML.
3721	*/
3722
3723	if (fw_event == ioc->current_event &&
3724	ioc->current_event->event !=
3725	MPT3SAS_REMOVE_UNRESPONDING_DEVICES) {
3726	ioc->current_event = NULL;
3727	continue;
3728	}
3729
3730	/*
3731	* Driver has to clear ioc->start_scan flag when
3732	* it is cleaning up MPT3SAS_PORT_ENABLE_COMPLETE,
3733	* otherwise scsi_scan_host() API waits for the
3734	* 5 minute timer to expire. If we exit from
3735	* scsi_scan_host() early then we can issue the
3736	* new port enable request as part of current diag reset.
3737	*/
3738	if (fw_event->event == MPT3SAS_PORT_ENABLE_COMPLETE) {
3739	ioc->port_enable_cmds.status |= MPT3_CMD_RESET;
3740	ioc->start_scan = 0;
3741	}
3742
3743	/*
3744	* Wait on the fw_event to complete. If this returns 1, then
3745	* the event was never executed, and we need a put for the
3746	* reference the work had on the fw_event.
3747	*
3748	* If it did execute, we wait for it to finish, and the put will
3749	* happen from _firmware_event_work()
3750	*/
3751	if (cancel_work_sync(work: &fw_event->work))
3752	fw_event_work_put(fw_work: fw_event);
3753
3754	}
3755	ioc->fw_events_cleanup = 0;
3756	}
3757
3758	/**
3759	* _scsih_internal_device_block - block the sdev device
3760	* @sdev: per device object
3761	* @sas_device_priv_data : per device driver private data
3762	*
3763	* make sure device is blocked without error, if not
3764	* print an error
3765	*/
3766	static void
3767	_scsih_internal_device_block(struct scsi_device *sdev,
3768	struct MPT3SAS_DEVICE *sas_device_priv_data)
3769	{
3770	int r = 0;
3771
3772	sdev_printk(KERN_INFO, sdev, "device_block, handle(0x%04x)\n",
3773	sas_device_priv_data->sas_target->handle);
3774	sas_device_priv_data->block = 1;
3775
3776	r = scsi_internal_device_block_nowait(sdev);
3777	if (r == -EINVAL)
3778	sdev_printk(KERN_WARNING, sdev,
3779	"device_block failed with return(%d) for handle(0x%04x)\n",
3780	r, sas_device_priv_data->sas_target->handle);
3781	}
3782
3783	/**
3784	* _scsih_internal_device_unblock - unblock the sdev device
3785	* @sdev: per device object
3786	* @sas_device_priv_data : per device driver private data
3787	* make sure device is unblocked without error, if not retry
3788	* by blocking and then unblocking
3789	*/
3790
3791	static void
3792	_scsih_internal_device_unblock(struct scsi_device *sdev,
3793	struct MPT3SAS_DEVICE *sas_device_priv_data)
3794	{
3795	int r = 0;
3796
3797	sdev_printk(KERN_WARNING, sdev, "device_unblock and setting to running, "
3798	"handle(0x%04x)\n", sas_device_priv_data->sas_target->handle);
3799	sas_device_priv_data->block = 0;
3800	r = scsi_internal_device_unblock_nowait(sdev, new_state: SDEV_RUNNING);
3801	if (r == -EINVAL) {
3802	/* The device has been set to SDEV_RUNNING by SD layer during
3803	* device addition but the request queue is still stopped by
3804	* our earlier block call. We need to perform a block again
3805	* to get the device to SDEV_BLOCK and then to SDEV_RUNNING */
3806
3807	sdev_printk(KERN_WARNING, sdev,
3808	"device_unblock failed with return(%d) for handle(0x%04x) "
3809	"performing a block followed by an unblock\n",
3810	r, sas_device_priv_data->sas_target->handle);
3811	sas_device_priv_data->block = 1;
3812	r = scsi_internal_device_block_nowait(sdev);
3813	if (r)
3814	sdev_printk(KERN_WARNING, sdev, "retried device_block "
3815	"failed with return(%d) for handle(0x%04x)\n",
3816	r, sas_device_priv_data->sas_target->handle);
3817
3818	sas_device_priv_data->block = 0;
3819	r = scsi_internal_device_unblock_nowait(sdev, new_state: SDEV_RUNNING);
3820	if (r)
3821	sdev_printk(KERN_WARNING, sdev, "retried device_unblock"
3822	" failed with return(%d) for handle(0x%04x)\n",
3823	r, sas_device_priv_data->sas_target->handle);
3824	}
3825	}
3826
3827	/**
3828	* _scsih_ublock_io_all_device - unblock every device
3829	* @ioc: per adapter object
3830	*
3831	* change the device state from block to running
3832	*/
3833	static void
3834	_scsih_ublock_io_all_device(struct MPT3SAS_ADAPTER *ioc)
3835	{
3836	struct MPT3SAS_DEVICE *sas_device_priv_data;
3837	struct scsi_device *sdev;
3838
3839	shost_for_each_device(sdev, ioc->shost) {
3840	sas_device_priv_data = sdev->hostdata;
3841	if (!sas_device_priv_data)
3842	continue;
3843	if (!sas_device_priv_data->block)
3844	continue;
3845
3846	dewtprintk(ioc, sdev_printk(KERN_INFO, sdev,
3847	"device_running, handle(0x%04x)\n",
3848	sas_device_priv_data->sas_target->handle));
3849	_scsih_internal_device_unblock(sdev, sas_device_priv_data);
3850	}
3851	}
3852
3853
3854	/**
3855	* _scsih_ublock_io_device - prepare device to be deleted
3856	* @ioc: per adapter object
3857	* @sas_address: sas address
3858	* @port: hba port entry
3859	*
3860	* unblock then put device in offline state
3861	*/
3862	static void
3863	_scsih_ublock_io_device(struct MPT3SAS_ADAPTER *ioc,
3864	u64 sas_address, struct hba_port *port)
3865	{
3866	struct MPT3SAS_DEVICE *sas_device_priv_data;
3867	struct scsi_device *sdev;
3868
3869	shost_for_each_device(sdev, ioc->shost) {
3870	sas_device_priv_data = sdev->hostdata;
3871	if (!sas_device_priv_data || !sas_device_priv_data->sas_target)
3872	continue;
3873	if (sas_device_priv_data->sas_target->sas_address
3874	!= sas_address)
3875	continue;
3876	if (sas_device_priv_data->sas_target->port != port)
3877	continue;
3878	if (sas_device_priv_data->block)
3879	_scsih_internal_device_unblock(sdev,
3880	sas_device_priv_data);
3881	}
3882	}
3883
3884	/**
3885	* _scsih_block_io_all_device - set the device state to SDEV_BLOCK
3886	* @ioc: per adapter object
3887	*
3888	* During device pull we need to appropriately set the sdev state.
3889	*/
3890	static void
3891	_scsih_block_io_all_device(struct MPT3SAS_ADAPTER *ioc)
3892	{
3893	struct MPT3SAS_DEVICE *sas_device_priv_data;
3894	struct scsi_device *sdev;
3895
3896	shost_for_each_device(sdev, ioc->shost) {
3897	sas_device_priv_data = sdev->hostdata;
3898	if (!sas_device_priv_data)
3899	continue;
3900	if (sas_device_priv_data->block)
3901	continue;
3902	if (sas_device_priv_data->ignore_delay_remove) {
3903	sdev_printk(KERN_INFO, sdev,
3904	"%s skip device_block for SES handle(0x%04x)\n",
3905	__func__, sas_device_priv_data->sas_target->handle);
3906	continue;
3907	}
3908	_scsih_internal_device_block(sdev, sas_device_priv_data);
3909	}
3910	}
3911
3912	/**
3913	* _scsih_block_io_device - set the device state to SDEV_BLOCK
3914	* @ioc: per adapter object
3915	* @handle: device handle
3916	*
3917	* During device pull we need to appropriately set the sdev state.
3918	*/
3919	static void
3920	_scsih_block_io_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
3921	{
3922	struct MPT3SAS_DEVICE *sas_device_priv_data;
3923	struct scsi_device *sdev;
3924	struct _sas_device *sas_device;
3925
3926	sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
3927
3928	shost_for_each_device(sdev, ioc->shost) {
3929	sas_device_priv_data = sdev->hostdata;
3930	if (!sas_device_priv_data)
3931	continue;
3932	if (sas_device_priv_data->sas_target->handle != handle)
3933	continue;
3934	if (sas_device_priv_data->block)
3935	continue;
3936	if (sas_device && sas_device->pend_sas_rphy_add)
3937	continue;
3938	if (sas_device_priv_data->ignore_delay_remove) {
3939	sdev_printk(KERN_INFO, sdev,
3940	"%s skip device_block for SES handle(0x%04x)\n",
3941	__func__, sas_device_priv_data->sas_target->handle);
3942	continue;
3943	}
3944	_scsih_internal_device_block(sdev, sas_device_priv_data);
3945	}
3946
3947	if (sas_device)
3948	sas_device_put(s: sas_device);
3949	}
3950
3951	/**
3952	* _scsih_block_io_to_children_attached_to_ex
3953	* @ioc: per adapter object
3954	* @sas_expander: the sas_device object
3955	*
3956	* This routine set sdev state to SDEV_BLOCK for all devices
3957	* attached to this expander. This function called when expander is
3958	* pulled.
3959	*/
3960	static void
3961	_scsih_block_io_to_children_attached_to_ex(struct MPT3SAS_ADAPTER *ioc,
3962	struct _sas_node *sas_expander)
3963	{
3964	struct _sas_port *mpt3sas_port;
3965	struct _sas_device *sas_device;
3966	struct _sas_node *expander_sibling;
3967	unsigned long flags;
3968
3969	if (!sas_expander)
3970	return;
3971
3972	list_for_each_entry(mpt3sas_port,
3973	&sas_expander->sas_port_list, port_list) {
3974	if (mpt3sas_port->remote_identify.device_type ==
3975	SAS_END_DEVICE) {
3976	spin_lock_irqsave(&ioc->sas_device_lock, flags);
3977	sas_device = __mpt3sas_get_sdev_by_addr(ioc,
3978	sas_address: mpt3sas_port->remote_identify.sas_address,
3979	port: mpt3sas_port->hba_port);
3980	if (sas_device) {
3981	set_bit(nr: sas_device->handle,
3982	addr: ioc->blocking_handles);
3983	sas_device_put(s: sas_device);
3984	}
3985	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
3986	}
3987	}
3988
3989	list_for_each_entry(mpt3sas_port,
3990	&sas_expander->sas_port_list, port_list) {
3991
3992	if (mpt3sas_port->remote_identify.device_type ==
3993	SAS_EDGE_EXPANDER_DEVICE ||
3994	mpt3sas_port->remote_identify.device_type ==
3995	SAS_FANOUT_EXPANDER_DEVICE) {
3996	expander_sibling =
3997	mpt3sas_scsih_expander_find_by_sas_address(
3998	ioc, sas_address: mpt3sas_port->remote_identify.sas_address,
3999	port: mpt3sas_port->hba_port);
4000	_scsih_block_io_to_children_attached_to_ex(ioc,
4001	sas_expander: expander_sibling);
4002	}
4003	}
4004	}
4005
4006	/**
4007	* _scsih_block_io_to_children_attached_directly
4008	* @ioc: per adapter object
4009	* @event_data: topology change event data
4010	*
4011	* This routine set sdev state to SDEV_BLOCK for all devices
4012	* direct attached during device pull.
4013	*/
4014	static void
4015	_scsih_block_io_to_children_attached_directly(struct MPT3SAS_ADAPTER *ioc,
4016	Mpi2EventDataSasTopologyChangeList_t *event_data)
4017	{
4018	int i;
4019	u16 handle;
4020	u16 reason_code;
4021
4022	for (i = 0; i < event_data->NumEntries; i++) {
4023	handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
4024	if (!handle)
4025	continue;
4026	reason_code = event_data->PHY[i].PhyStatus &
4027	MPI2_EVENT_SAS_TOPO_RC_MASK;
4028	if (reason_code == MPI2_EVENT_SAS_TOPO_RC_DELAY_NOT_RESPONDING)
4029	_scsih_block_io_device(ioc, handle);
4030	}
4031	}
4032
4033	/**
4034	* _scsih_block_io_to_pcie_children_attached_directly
4035	* @ioc: per adapter object
4036	* @event_data: topology change event data
4037	*
4038	* This routine set sdev state to SDEV_BLOCK for all devices
4039	* direct attached during device pull/reconnect.
4040	*/
4041	static void
4042	_scsih_block_io_to_pcie_children_attached_directly(struct MPT3SAS_ADAPTER *ioc,
4043	Mpi26EventDataPCIeTopologyChangeList_t *event_data)
4044	{
4045	int i;
4046	u16 handle;
4047	u16 reason_code;
4048
4049	for (i = 0; i < event_data->NumEntries; i++) {
4050	handle =
4051	le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
4052	if (!handle)
4053	continue;
4054	reason_code = event_data->PortEntry[i].PortStatus;
4055	if (reason_code ==
4056	MPI26_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING)
4057	_scsih_block_io_device(ioc, handle);
4058	}
4059	}
4060	/**
4061	* _scsih_tm_tr_send - send task management request
4062	* @ioc: per adapter object
4063	* @handle: device handle
4064	* Context: interrupt time.
4065	*
4066	* This code is to initiate the device removal handshake protocol
4067	* with controller firmware. This function will issue target reset
4068	* using high priority request queue. It will send a sas iounit
4069	* control request (MPI2_SAS_OP_REMOVE_DEVICE) from this completion.
4070	*
4071	* This is designed to send muliple task management request at the same
4072	* time to the fifo. If the fifo is full, we will append the request,
4073	* and process it in a future completion.
4074	*/
4075	static void
4076	_scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
4077	{
4078	Mpi2SCSITaskManagementRequest_t *mpi_request;
4079	u16 smid;
4080	struct _sas_device *sas_device = NULL;
4081	struct _pcie_device *pcie_device = NULL;
4082	struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
4083	u64 sas_address = 0;
4084	unsigned long flags;
4085	struct _tr_list *delayed_tr;
4086	u32 ioc_state;
4087	u8 tr_method = 0;
4088	struct hba_port *port = NULL;
4089
4090	if (ioc->pci_error_recovery) {
4091	dewtprintk(ioc,
4092	ioc_info(ioc, "%s: host in pci error recovery: handle(0x%04x)\n",
4093	__func__, handle));
4094	return;
4095	}
4096	ioc_state = mpt3sas_base_get_iocstate(ioc, cooked: 1);
4097	if (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
4098	dewtprintk(ioc,
4099	ioc_info(ioc, "%s: host is not operational: handle(0x%04x)\n",
4100	__func__, handle));
4101	return;
4102	}
4103
4104	/* if PD, then return */
4105	if (test_bit(handle, ioc->pd_handles))
4106	return;
4107
4108	clear_bit(nr: handle, addr: ioc->pend_os_device_add);
4109
4110	spin_lock_irqsave(&ioc->sas_device_lock, flags);
4111	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
4112	if (sas_device && sas_device->starget &&
4113	sas_device->starget->hostdata) {
4114	sas_target_priv_data = sas_device->starget->hostdata;
4115	sas_target_priv_data->deleted = 1;
4116	sas_address = sas_device->sas_address;
4117	port = sas_device->port;
4118	}
4119	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
4120	if (!sas_device) {
4121	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
4122	pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);
4123	if (pcie_device && pcie_device->starget &&
4124	pcie_device->starget->hostdata) {
4125	sas_target_priv_data = pcie_device->starget->hostdata;
4126	sas_target_priv_data->deleted = 1;
4127	sas_address = pcie_device->wwid;
4128	}
4129	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
4130	if (pcie_device && (!ioc->tm_custom_handling) &&
4131	(!(mpt3sas_scsih_is_pcie_scsi_device(
4132	device_info: pcie_device->device_info))))
4133	tr_method =
4134	MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
4135	else
4136	tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
4137	}
4138	if (sas_target_priv_data) {
4139	dewtprintk(ioc,
4140	ioc_info(ioc, "setting delete flag: handle(0x%04x), sas_addr(0x%016llx)\n",
4141	handle, (u64)sas_address));
4142	if (sas_device) {
4143	if (sas_device->enclosure_handle != 0)
4144	dewtprintk(ioc,
4145	ioc_info(ioc, "setting delete flag:enclosure logical id(0x%016llx), slot(%d)\n",
4146	(u64)sas_device->enclosure_logical_id,
4147	sas_device->slot));
4148	if (sas_device->connector_name[0] != '\0')
4149	dewtprintk(ioc,
4150	ioc_info(ioc, "setting delete flag: enclosure level(0x%04x), connector name( %s)\n",
4151	sas_device->enclosure_level,
4152	sas_device->connector_name));
4153	} else if (pcie_device) {
4154	if (pcie_device->enclosure_handle != 0)
4155	dewtprintk(ioc,
4156	ioc_info(ioc, "setting delete flag: logical id(0x%016llx), slot(%d)\n",
4157	(u64)pcie_device->enclosure_logical_id,
4158	pcie_device->slot));
4159	if (pcie_device->connector_name[0] != '\0')
4160	dewtprintk(ioc,
4161	ioc_info(ioc, "setting delete flag:, enclosure level(0x%04x), connector name( %s)\n",
4162	pcie_device->enclosure_level,
4163	pcie_device->connector_name));
4164	}
4165	_scsih_ublock_io_device(ioc, sas_address, port);
4166	sas_target_priv_data->handle = MPT3SAS_INVALID_DEVICE_HANDLE;
4167	}
4168
4169	smid = mpt3sas_base_get_smid_hpr(ioc, cb_idx: ioc->tm_tr_cb_idx);
4170	if (!smid) {
4171	delayed_tr = kzalloc(size: sizeof(*delayed_tr), GFP_ATOMIC);
4172	if (!delayed_tr)
4173	goto out;
4174	INIT_LIST_HEAD(list: &delayed_tr->list);
4175	delayed_tr->handle = handle;
4176	list_add_tail(new: &delayed_tr->list, head: &ioc->delayed_tr_list);
4177	dewtprintk(ioc,
4178	ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",
4179	handle));
4180	goto out;
4181	}
4182
4183	dewtprintk(ioc,
4184	ioc_info(ioc, "tr_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
4185	handle, smid, ioc->tm_tr_cb_idx));
4186	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4187	memset(mpi_request, 0, sizeof(Mpi2SCSITaskManagementRequest_t));
4188	mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
4189	mpi_request->DevHandle = cpu_to_le16(handle);
4190	mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
4191	mpi_request->MsgFlags = tr_method;
4192	set_bit(nr: handle, addr: ioc->device_remove_in_progress);
4193	ioc->put_smid_hi_priority(ioc, smid, 0);
4194	mpt3sas_trigger_master(ioc, MASTER_TRIGGER_DEVICE_REMOVAL);
4195
4196	out:
4197	if (sas_device)
4198	sas_device_put(s: sas_device);
4199	if (pcie_device)
4200	pcie_device_put(p: pcie_device);
4201	}
4202
4203	/**
4204	* _scsih_tm_tr_complete -
4205	* @ioc: per adapter object
4206	* @smid: system request message index
4207	* @msix_index: MSIX table index supplied by the OS
4208	* @reply: reply message frame(lower 32bit addr)
4209	* Context: interrupt time.
4210	*
4211	* This is the target reset completion routine.
4212	* This code is part of the code to initiate the device removal
4213	* handshake protocol with controller firmware.
4214	* It will send a sas iounit control request (MPI2_SAS_OP_REMOVE_DEVICE)
4215	*
4216	* Return: 1 meaning mf should be freed from _base_interrupt
4217	* 0 means the mf is freed from this function.
4218	*/
4219	static u8
4220	_scsih_tm_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
4221	u32 reply)
4222	{
4223	u16 handle;
4224	Mpi2SCSITaskManagementRequest_t *mpi_request_tm;
4225	Mpi2SCSITaskManagementReply_t *mpi_reply =
4226	mpt3sas_base_get_reply_virt_addr(ioc, phys_addr: reply);
4227	Mpi2SasIoUnitControlRequest_t *mpi_request;
4228	u16 smid_sas_ctrl;
4229	u32 ioc_state;
4230	struct _sc_list *delayed_sc;
4231
4232	if (ioc->pci_error_recovery) {
4233	dewtprintk(ioc,
4234	ioc_info(ioc, "%s: host in pci error recovery\n",
4235	__func__));
4236	return 1;
4237	}
4238	ioc_state = mpt3sas_base_get_iocstate(ioc, cooked: 1);
4239	if (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
4240	dewtprintk(ioc,
4241	ioc_info(ioc, "%s: host is not operational\n",
4242	__func__));
4243	return 1;
4244	}
4245	if (unlikely(!mpi_reply)) {
4246	ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
4247	__FILE__, __LINE__, __func__);
4248	return 1;
4249	}
4250	mpi_request_tm = mpt3sas_base_get_msg_frame(ioc, smid);
4251	handle = le16_to_cpu(mpi_request_tm->DevHandle);
4252	if (handle != le16_to_cpu(mpi_reply->DevHandle)) {
4253	dewtprintk(ioc,
4254	ioc_err(ioc, "spurious interrupt: handle(0x%04x:0x%04x), smid(%d)!!!\n",
4255	handle,
4256	le16_to_cpu(mpi_reply->DevHandle), smid));
4257	return 0;
4258	}
4259
4260	mpt3sas_trigger_master(ioc, MASTER_TRIGGER_TASK_MANAGMENT);
4261	dewtprintk(ioc,
4262	ioc_info(ioc, "tr_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x), completed(%d)\n",
4263	handle, smid, le16_to_cpu(mpi_reply->IOCStatus),
4264	le32_to_cpu(mpi_reply->IOCLogInfo),
4265	le32_to_cpu(mpi_reply->TerminationCount)));
4266
4267	smid_sas_ctrl = mpt3sas_base_get_smid(ioc, cb_idx: ioc->tm_sas_control_cb_idx);
4268	if (!smid_sas_ctrl) {
4269	delayed_sc = kzalloc(size: sizeof(*delayed_sc), GFP_ATOMIC);
4270	if (!delayed_sc)
4271	return _scsih_check_for_pending_tm(ioc, smid);
4272	INIT_LIST_HEAD(list: &delayed_sc->list);
4273	delayed_sc->handle = le16_to_cpu(mpi_request_tm->DevHandle);
4274	list_add_tail(new: &delayed_sc->list, head: &ioc->delayed_sc_list);
4275	dewtprintk(ioc,
4276	ioc_info(ioc, "DELAYED:sc:handle(0x%04x), (open)\n",
4277	handle));
4278	return _scsih_check_for_pending_tm(ioc, smid);
4279	}
4280
4281	dewtprintk(ioc,
4282	ioc_info(ioc, "sc_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
4283	handle, smid_sas_ctrl, ioc->tm_sas_control_cb_idx));
4284	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid: smid_sas_ctrl);
4285	memset(mpi_request, 0, sizeof(Mpi2SasIoUnitControlRequest_t));
4286	mpi_request->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
4287	mpi_request->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
4288	mpi_request->DevHandle = mpi_request_tm->DevHandle;
4289	ioc->put_smid_default(ioc, smid_sas_ctrl);
4290
4291	return _scsih_check_for_pending_tm(ioc, smid);
4292	}
4293
4294	/** _scsih_allow_scmd_to_device - check whether scmd needs to
4295	* issue to IOC or not.
4296	* @ioc: per adapter object
4297	* @scmd: pointer to scsi command object
4298	*
4299	* Returns true if scmd can be issued to IOC otherwise returns false.
4300	*/
4301	inline bool _scsih_allow_scmd_to_device(struct MPT3SAS_ADAPTER *ioc,
4302	struct scsi_cmnd *scmd)
4303	{
4304
4305	if (ioc->pci_error_recovery)
4306	return false;
4307
4308	if (ioc->hba_mpi_version_belonged == MPI2_VERSION) {
4309	if (ioc->remove_host)
4310	return false;
4311
4312	return true;
4313	}
4314
4315	if (ioc->remove_host) {
4316
4317	switch (scmd->cmnd[0]) {
4318	case SYNCHRONIZE_CACHE:
4319	case START_STOP:
4320	return true;
4321	default:
4322	return false;
4323	}
4324	}
4325
4326	return true;
4327	}
4328
4329	/**
4330	* _scsih_sas_control_complete - completion routine
4331	* @ioc: per adapter object
4332	* @smid: system request message index
4333	* @msix_index: MSIX table index supplied by the OS
4334	* @reply: reply message frame(lower 32bit addr)
4335	* Context: interrupt time.
4336	*
4337	* This is the sas iounit control completion routine.
4338	* This code is part of the code to initiate the device removal
4339	* handshake protocol with controller firmware.
4340	*
4341	* Return: 1 meaning mf should be freed from _base_interrupt
4342	* 0 means the mf is freed from this function.
4343	*/
4344	static u8
4345	_scsih_sas_control_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4346	u8 msix_index, u32 reply)
4347	{
4348	Mpi2SasIoUnitControlReply_t *mpi_reply =
4349	mpt3sas_base_get_reply_virt_addr(ioc, phys_addr: reply);
4350
4351	if (likely(mpi_reply)) {
4352	dewtprintk(ioc,
4353	ioc_info(ioc, "sc_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x)\n",
4354	le16_to_cpu(mpi_reply->DevHandle), smid,
4355	le16_to_cpu(mpi_reply->IOCStatus),
4356	le32_to_cpu(mpi_reply->IOCLogInfo)));
4357	if (le16_to_cpu(mpi_reply->IOCStatus) ==
4358	MPI2_IOCSTATUS_SUCCESS) {
4359	clear_bit(le16_to_cpu(mpi_reply->DevHandle),
4360	addr: ioc->device_remove_in_progress);
4361	}
4362	} else {
4363	ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
4364	__FILE__, __LINE__, __func__);
4365	}
4366	return mpt3sas_check_for_pending_internal_cmds(ioc, smid);
4367	}
4368
4369	/**
4370	* _scsih_tm_tr_volume_send - send target reset request for volumes
4371	* @ioc: per adapter object
4372	* @handle: device handle
4373	* Context: interrupt time.
4374	*
4375	* This is designed to send muliple task management request at the same
4376	* time to the fifo. If the fifo is full, we will append the request,
4377	* and process it in a future completion.
4378	*/
4379	static void
4380	_scsih_tm_tr_volume_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
4381	{
4382	Mpi2SCSITaskManagementRequest_t *mpi_request;
4383	u16 smid;
4384	struct _tr_list *delayed_tr;
4385
4386	if (ioc->pci_error_recovery) {
4387	dewtprintk(ioc,
4388	ioc_info(ioc, "%s: host reset in progress!\n",
4389	__func__));
4390	return;
4391	}
4392
4393	smid = mpt3sas_base_get_smid_hpr(ioc, cb_idx: ioc->tm_tr_volume_cb_idx);
4394	if (!smid) {
4395	delayed_tr = kzalloc(size: sizeof(*delayed_tr), GFP_ATOMIC);
4396	if (!delayed_tr)
4397	return;
4398	INIT_LIST_HEAD(list: &delayed_tr->list);
4399	delayed_tr->handle = handle;
4400	list_add_tail(new: &delayed_tr->list, head: &ioc->delayed_tr_volume_list);
4401	dewtprintk(ioc,
4402	ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",
4403	handle));
4404	return;
4405	}
4406
4407	dewtprintk(ioc,
4408	ioc_info(ioc, "tr_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
4409	handle, smid, ioc->tm_tr_volume_cb_idx));
4410	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4411	memset(mpi_request, 0, sizeof(Mpi2SCSITaskManagementRequest_t));
4412	mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
4413	mpi_request->DevHandle = cpu_to_le16(handle);
4414	mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
4415	ioc->put_smid_hi_priority(ioc, smid, 0);
4416	}
4417
4418	/**
4419	* _scsih_tm_volume_tr_complete - target reset completion
4420	* @ioc: per adapter object
4421	* @smid: system request message index
4422	* @msix_index: MSIX table index supplied by the OS
4423	* @reply: reply message frame(lower 32bit addr)
4424	* Context: interrupt time.
4425	*
4426	* Return: 1 meaning mf should be freed from _base_interrupt
4427	* 0 means the mf is freed from this function.
4428	*/
4429	static u8
4430	_scsih_tm_volume_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4431	u8 msix_index, u32 reply)
4432	{
4433	u16 handle;
4434	Mpi2SCSITaskManagementRequest_t *mpi_request_tm;
4435	Mpi2SCSITaskManagementReply_t *mpi_reply =
4436	mpt3sas_base_get_reply_virt_addr(ioc, phys_addr: reply);
4437
4438	if (ioc->shost_recovery || ioc->pci_error_recovery) {
4439	dewtprintk(ioc,
4440	ioc_info(ioc, "%s: host reset in progress!\n",
4441	__func__));
4442	return 1;
4443	}
4444	if (unlikely(!mpi_reply)) {
4445	ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
4446	__FILE__, __LINE__, __func__);
4447	return 1;
4448	}
4449
4450	mpi_request_tm = mpt3sas_base_get_msg_frame(ioc, smid);
4451	handle = le16_to_cpu(mpi_request_tm->DevHandle);
4452	if (handle != le16_to_cpu(mpi_reply->DevHandle)) {
4453	dewtprintk(ioc,
4454	ioc_err(ioc, "spurious interrupt: handle(0x%04x:0x%04x), smid(%d)!!!\n",
4455	handle, le16_to_cpu(mpi_reply->DevHandle),
4456	smid));
4457	return 0;
4458	}
4459
4460	dewtprintk(ioc,
4461	ioc_info(ioc, "tr_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x), completed(%d)\n",
4462	handle, smid, le16_to_cpu(mpi_reply->IOCStatus),
4463	le32_to_cpu(mpi_reply->IOCLogInfo),
4464	le32_to_cpu(mpi_reply->TerminationCount)));
4465
4466	return _scsih_check_for_pending_tm(ioc, smid);
4467	}
4468
4469	/**
4470	* _scsih_issue_delayed_event_ack - issue delayed Event ACK messages
4471	* @ioc: per adapter object
4472	* @smid: system request message index
4473	* @event: Event ID
4474	* @event_context: used to track events uniquely
4475	*
4476	* Context - processed in interrupt context.
4477	*/
4478	static void
4479	_scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, U16 event,
4480	U32 event_context)
4481	{
4482	Mpi2EventAckRequest_t *ack_request;
4483	int i = smid - ioc->internal_smid;
4484	unsigned long flags;
4485
4486	/* Without releasing the smid just update the
4487	* call back index and reuse the same smid for
4488	* processing this delayed request
4489	*/
4490	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
4491	ioc->internal_lookup[i].cb_idx = ioc->base_cb_idx;
4492	spin_unlock_irqrestore(lock: &ioc->scsi_lookup_lock, flags);
4493
4494	dewtprintk(ioc,
4495	ioc_info(ioc, "EVENT ACK: event(0x%04x), smid(%d), cb(%d)\n",
4496	le16_to_cpu(event), smid, ioc->base_cb_idx));
4497	ack_request = mpt3sas_base_get_msg_frame(ioc, smid);
4498	memset(ack_request, 0, sizeof(Mpi2EventAckRequest_t));
4499	ack_request->Function = MPI2_FUNCTION_EVENT_ACK;
4500	ack_request->Event = event;
4501	ack_request->EventContext = event_context;
4502	ack_request->VF_ID = 0; /* TODO */
4503	ack_request->VP_ID = 0;
4504	ioc->put_smid_default(ioc, smid);
4505	}
4506
4507	/**
4508	* _scsih_issue_delayed_sas_io_unit_ctrl - issue delayed
4509	* sas_io_unit_ctrl messages
4510	* @ioc: per adapter object
4511	* @smid: system request message index
4512	* @handle: device handle
4513	*
4514	* Context - processed in interrupt context.
4515	*/
4516	static void
4517	_scsih_issue_delayed_sas_io_unit_ctrl(struct MPT3SAS_ADAPTER *ioc,
4518	u16 smid, u16 handle)
4519	{
4520	Mpi2SasIoUnitControlRequest_t *mpi_request;
4521	u32 ioc_state;
4522	int i = smid - ioc->internal_smid;
4523	unsigned long flags;
4524
4525	if (ioc->remove_host) {
4526	dewtprintk(ioc,
4527	ioc_info(ioc, "%s: host has been removed\n",
4528	__func__));
4529	return;
4530	} else if (ioc->pci_error_recovery) {
4531	dewtprintk(ioc,
4532	ioc_info(ioc, "%s: host in pci error recovery\n",
4533	__func__));
4534	return;
4535	}
4536	ioc_state = mpt3sas_base_get_iocstate(ioc, cooked: 1);
4537	if (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
4538	dewtprintk(ioc,
4539	ioc_info(ioc, "%s: host is not operational\n",
4540	__func__));
4541	return;
4542	}
4543
4544	/* Without releasing the smid just update the
4545	* call back index and reuse the same smid for
4546	* processing this delayed request
4547	*/
4548	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
4549	ioc->internal_lookup[i].cb_idx = ioc->tm_sas_control_cb_idx;
4550	spin_unlock_irqrestore(lock: &ioc->scsi_lookup_lock, flags);
4551
4552	dewtprintk(ioc,
4553	ioc_info(ioc, "sc_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
4554	handle, smid, ioc->tm_sas_control_cb_idx));
4555	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4556	memset(mpi_request, 0, sizeof(Mpi2SasIoUnitControlRequest_t));
4557	mpi_request->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
4558	mpi_request->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
4559	mpi_request->DevHandle = cpu_to_le16(handle);
4560	ioc->put_smid_default(ioc, smid);
4561	}
4562
4563	/**
4564	* mpt3sas_check_for_pending_internal_cmds - check for pending internal messages
4565	* @ioc: per adapter object
4566	* @smid: system request message index
4567	*
4568	* Context: Executed in interrupt context
4569	*
4570	* This will check delayed internal messages list, and process the
4571	* next request.
4572	*
4573	* Return: 1 meaning mf should be freed from _base_interrupt
4574	* 0 means the mf is freed from this function.
4575	*/
4576	u8
4577	mpt3sas_check_for_pending_internal_cmds(struct MPT3SAS_ADAPTER *ioc, u16 smid)
4578	{
4579	struct _sc_list *delayed_sc;
4580	struct _event_ack_list *delayed_event_ack;
4581
4582	if (!list_empty(head: &ioc->delayed_event_ack_list)) {
4583	delayed_event_ack = list_entry(ioc->delayed_event_ack_list.next,
4584	struct _event_ack_list, list);
4585	_scsih_issue_delayed_event_ack(ioc, smid,
4586	event: delayed_event_ack->Event, event_context: delayed_event_ack->EventContext);
4587	list_del(entry: &delayed_event_ack->list);
4588	kfree(objp: delayed_event_ack);
4589	return 0;
4590	}
4591
4592	if (!list_empty(head: &ioc->delayed_sc_list)) {
4593	delayed_sc = list_entry(ioc->delayed_sc_list.next,
4594	struct _sc_list, list);
4595	_scsih_issue_delayed_sas_io_unit_ctrl(ioc, smid,
4596	handle: delayed_sc->handle);
4597	list_del(entry: &delayed_sc->list);
4598	kfree(objp: delayed_sc);
4599	return 0;
4600	}
4601	return 1;
4602	}
4603
4604	/**
4605	* _scsih_check_for_pending_tm - check for pending task management
4606	* @ioc: per adapter object
4607	* @smid: system request message index
4608	*
4609	* This will check delayed target reset list, and feed the
4610	* next reqeust.
4611	*
4612	* Return: 1 meaning mf should be freed from _base_interrupt
4613	* 0 means the mf is freed from this function.
4614	*/
4615	static u8
4616	_scsih_check_for_pending_tm(struct MPT3SAS_ADAPTER *ioc, u16 smid)
4617	{
4618	struct _tr_list *delayed_tr;
4619
4620	if (!list_empty(head: &ioc->delayed_tr_volume_list)) {
4621	delayed_tr = list_entry(ioc->delayed_tr_volume_list.next,
4622	struct _tr_list, list);
4623	mpt3sas_base_free_smid(ioc, smid);
4624	_scsih_tm_tr_volume_send(ioc, handle: delayed_tr->handle);
4625	list_del(entry: &delayed_tr->list);
4626	kfree(objp: delayed_tr);
4627	return 0;
4628	}
4629
4630	if (!list_empty(head: &ioc->delayed_tr_list)) {
4631	delayed_tr = list_entry(ioc->delayed_tr_list.next,
4632	struct _tr_list, list);
4633	mpt3sas_base_free_smid(ioc, smid);
4634	_scsih_tm_tr_send(ioc, handle: delayed_tr->handle);
4635	list_del(entry: &delayed_tr->list);
4636	kfree(objp: delayed_tr);
4637	return 0;
4638	}
4639
4640	return 1;
4641	}
4642
4643	/**
4644	* _scsih_check_topo_delete_events - sanity check on topo events
4645	* @ioc: per adapter object
4646	* @event_data: the event data payload
4647	*
4648	* This routine added to better handle cable breaker.
4649	*
4650	* This handles the case where driver receives multiple expander
4651	* add and delete events in a single shot. When there is a delete event
4652	* the routine will void any pending add events waiting in the event queue.
4653	*/
4654	static void
4655	_scsih_check_topo_delete_events(struct MPT3SAS_ADAPTER *ioc,
4656	Mpi2EventDataSasTopologyChangeList_t *event_data)
4657	{
4658	struct fw_event_work *fw_event;
4659	Mpi2EventDataSasTopologyChangeList_t *local_event_data;
4660	u16 expander_handle;
4661	struct _sas_node *sas_expander;
4662	unsigned long flags;
4663	int i, reason_code;
4664	u16 handle;
4665
4666	for (i = 0 ; i < event_data->NumEntries; i++) {
4667	handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
4668	if (!handle)
4669	continue;
4670	reason_code = event_data->PHY[i].PhyStatus &
4671	MPI2_EVENT_SAS_TOPO_RC_MASK;
4672	if (reason_code == MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING)
4673	_scsih_tm_tr_send(ioc, handle);
4674	}
4675
4676	expander_handle = le16_to_cpu(event_data->ExpanderDevHandle);
4677	if (expander_handle < ioc->sas_hba.num_phys) {
4678	_scsih_block_io_to_children_attached_directly(ioc, event_data);
4679	return;
4680	}
4681	if (event_data->ExpStatus ==
4682	MPI2_EVENT_SAS_TOPO_ES_DELAY_NOT_RESPONDING) {
4683	/* put expander attached devices into blocking state */
4684	spin_lock_irqsave(&ioc->sas_node_lock, flags);
4685	sas_expander = mpt3sas_scsih_expander_find_by_handle(ioc,
4686	handle: expander_handle);
4687	_scsih_block_io_to_children_attached_to_ex(ioc, sas_expander);
4688	spin_unlock_irqrestore(lock: &ioc->sas_node_lock, flags);
4689	do {
4690	handle = find_first_bit(addr: ioc->blocking_handles,
4691	size: ioc->facts.MaxDevHandle);
4692	if (handle < ioc->facts.MaxDevHandle)
4693	_scsih_block_io_device(ioc, handle);
4694	} while (test_and_clear_bit(nr: handle, addr: ioc->blocking_handles));
4695	} else if (event_data->ExpStatus == MPI2_EVENT_SAS_TOPO_ES_RESPONDING)
4696	_scsih_block_io_to_children_attached_directly(ioc, event_data);
4697
4698	if (event_data->ExpStatus != MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING)
4699	return;
4700
4701	/* mark ignore flag for pending events */
4702	spin_lock_irqsave(&ioc->fw_event_lock, flags);
4703	list_for_each_entry(fw_event, &ioc->fw_event_list, list) {
4704	if (fw_event->event != MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST ||
4705	fw_event->ignore)
4706	continue;
4707	local_event_data = (Mpi2EventDataSasTopologyChangeList_t *)
4708	fw_event->event_data;
4709	if (local_event_data->ExpStatus ==
4710	MPI2_EVENT_SAS_TOPO_ES_ADDED ||
4711	local_event_data->ExpStatus ==
4712	MPI2_EVENT_SAS_TOPO_ES_RESPONDING) {
4713	if (le16_to_cpu(local_event_data->ExpanderDevHandle) ==
4714	expander_handle) {
4715	dewtprintk(ioc,
4716	ioc_info(ioc, "setting ignoring flag\n"));
4717	fw_event->ignore = 1;
4718	}
4719	}
4720	}
4721	spin_unlock_irqrestore(lock: &ioc->fw_event_lock, flags);
4722	}
4723
4724	/**
4725	* _scsih_check_pcie_topo_remove_events - sanity check on topo
4726	* events
4727	* @ioc: per adapter object
4728	* @event_data: the event data payload
4729	*
4730	* This handles the case where driver receives multiple switch
4731	* or device add and delete events in a single shot. When there
4732	* is a delete event the routine will void any pending add
4733	* events waiting in the event queue.
4734	*/
4735	static void
4736	_scsih_check_pcie_topo_remove_events(struct MPT3SAS_ADAPTER *ioc,
4737	Mpi26EventDataPCIeTopologyChangeList_t *event_data)
4738	{
4739	struct fw_event_work *fw_event;
4740	Mpi26EventDataPCIeTopologyChangeList_t *local_event_data;
4741	unsigned long flags;
4742	int i, reason_code;
4743	u16 handle, switch_handle;
4744
4745	for (i = 0; i < event_data->NumEntries; i++) {
4746	handle =
4747	le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
4748	if (!handle)
4749	continue;
4750	reason_code = event_data->PortEntry[i].PortStatus;
4751	if (reason_code == MPI26_EVENT_PCIE_TOPO_PS_NOT_RESPONDING)
4752	_scsih_tm_tr_send(ioc, handle);
4753	}
4754
4755	switch_handle = le16_to_cpu(event_data->SwitchDevHandle);
4756	if (!switch_handle) {
4757	_scsih_block_io_to_pcie_children_attached_directly(
4758	ioc, event_data);
4759	return;
4760	}
4761	/* TODO We are not supporting cascaded PCIe Switch removal yet*/
4762	if ((event_data->SwitchStatus
4763	== MPI26_EVENT_PCIE_TOPO_SS_DELAY_NOT_RESPONDING) ||
4764	(event_data->SwitchStatus ==
4765	MPI26_EVENT_PCIE_TOPO_SS_RESPONDING))
4766	_scsih_block_io_to_pcie_children_attached_directly(
4767	ioc, event_data);
4768
4769	if (event_data->SwitchStatus != MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING)
4770	return;
4771
4772	/* mark ignore flag for pending events */
4773	spin_lock_irqsave(&ioc->fw_event_lock, flags);
4774	list_for_each_entry(fw_event, &ioc->fw_event_list, list) {
4775	if (fw_event->event != MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST ||
4776	fw_event->ignore)
4777	continue;
4778	local_event_data =
4779	(Mpi26EventDataPCIeTopologyChangeList_t *)
4780	fw_event->event_data;
4781	if (local_event_data->SwitchStatus ==
4782	MPI2_EVENT_SAS_TOPO_ES_ADDED ||
4783	local_event_data->SwitchStatus ==
4784	MPI2_EVENT_SAS_TOPO_ES_RESPONDING) {
4785	if (le16_to_cpu(local_event_data->SwitchDevHandle) ==
4786	switch_handle) {
4787	dewtprintk(ioc,
4788	ioc_info(ioc, "setting ignoring flag for switch event\n"));
4789	fw_event->ignore = 1;
4790	}
4791	}
4792	}
4793	spin_unlock_irqrestore(lock: &ioc->fw_event_lock, flags);
4794	}
4795
4796	/**
4797	* _scsih_set_volume_delete_flag - setting volume delete flag
4798	* @ioc: per adapter object
4799	* @handle: device handle
4800	*
4801	* This returns nothing.
4802	*/
4803	static void
4804	_scsih_set_volume_delete_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
4805	{
4806	struct _raid_device *raid_device;
4807	struct MPT3SAS_TARGET *sas_target_priv_data;
4808	unsigned long flags;
4809
4810	spin_lock_irqsave(&ioc->raid_device_lock, flags);
4811	raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
4812	if (raid_device && raid_device->starget &&
4813	raid_device->starget->hostdata) {
4814	sas_target_priv_data =
4815	raid_device->starget->hostdata;
4816	sas_target_priv_data->deleted = 1;
4817	dewtprintk(ioc,
4818	ioc_info(ioc, "setting delete flag: handle(0x%04x), wwid(0x%016llx)\n",
4819	handle, (u64)raid_device->wwid));
4820	}
4821	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
4822	}
4823
4824	/**
4825	* _scsih_set_volume_handle_for_tr - set handle for target reset to volume
4826	* @handle: input handle
4827	* @a: handle for volume a
4828	* @b: handle for volume b
4829	*
4830	* IR firmware only supports two raid volumes. The purpose of this
4831	* routine is to set the volume handle in either a or b. When the given
4832	* input handle is non-zero, or when a and b have not been set before.
4833	*/
4834	static void
4835	_scsih_set_volume_handle_for_tr(u16 handle, u16 *a, u16 *b)
4836	{
4837	if (!handle || handle == *a || handle == *b)
4838	return;
4839	if (!*a)
4840	*a = handle;
4841	else if (!*b)
4842	*b = handle;
4843	}
4844
4845	/**
4846	* _scsih_check_ir_config_unhide_events - check for UNHIDE events
4847	* @ioc: per adapter object
4848	* @event_data: the event data payload
4849	* Context: interrupt time.
4850	*
4851	* This routine will send target reset to volume, followed by target
4852	* resets to the PDs. This is called when a PD has been removed, or
4853	* volume has been deleted or removed. When the target reset is sent
4854	* to volume, the PD target resets need to be queued to start upon
4855	* completion of the volume target reset.
4856	*/
4857	static void
4858	_scsih_check_ir_config_unhide_events(struct MPT3SAS_ADAPTER *ioc,
4859	Mpi2EventDataIrConfigChangeList_t *event_data)
4860	{
4861	Mpi2EventIrConfigElement_t *element;
4862	int i;
4863	u16 handle, volume_handle, a, b;
4864	struct _tr_list *delayed_tr;
4865
4866	a = 0;
4867	b = 0;
4868
4869	if (ioc->is_warpdrive)
4870	return;
4871
4872	/* Volume Resets for Deleted or Removed */
4873	element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
4874	for (i = 0; i < event_data->NumElements; i++, element++) {
4875	if (le32_to_cpu(event_data->Flags) &
4876	MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG)
4877	continue;
4878	if (element->ReasonCode ==
4879	MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED ||
4880	element->ReasonCode ==
4881	MPI2_EVENT_IR_CHANGE_RC_REMOVED) {
4882	volume_handle = le16_to_cpu(element->VolDevHandle);
4883	_scsih_set_volume_delete_flag(ioc, handle: volume_handle);
4884	_scsih_set_volume_handle_for_tr(handle: volume_handle, a: &a, b: &b);
4885	}
4886	}
4887
4888	/* Volume Resets for UNHIDE events */
4889	element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
4890	for (i = 0; i < event_data->NumElements; i++, element++) {
4891	if (le32_to_cpu(event_data->Flags) &
4892	MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG)
4893	continue;
4894	if (element->ReasonCode == MPI2_EVENT_IR_CHANGE_RC_UNHIDE) {
4895	volume_handle = le16_to_cpu(element->VolDevHandle);
4896	_scsih_set_volume_handle_for_tr(handle: volume_handle, a: &a, b: &b);
4897	}
4898	}
4899
4900	if (a)
4901	_scsih_tm_tr_volume_send(ioc, handle: a);
4902	if (b)
4903	_scsih_tm_tr_volume_send(ioc, handle: b);
4904
4905	/* PD target resets */
4906	element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
4907	for (i = 0; i < event_data->NumElements; i++, element++) {
4908	if (element->ReasonCode != MPI2_EVENT_IR_CHANGE_RC_UNHIDE)
4909	continue;
4910	handle = le16_to_cpu(element->PhysDiskDevHandle);
4911	volume_handle = le16_to_cpu(element->VolDevHandle);
4912	clear_bit(nr: handle, addr: ioc->pd_handles);
4913	if (!volume_handle)
4914	_scsih_tm_tr_send(ioc, handle);
4915	else if (volume_handle == a || volume_handle == b) {
4916	delayed_tr = kzalloc(size: sizeof(*delayed_tr), GFP_ATOMIC);
4917	BUG_ON(!delayed_tr);
4918	INIT_LIST_HEAD(list: &delayed_tr->list);
4919	delayed_tr->handle = handle;
4920	list_add_tail(new: &delayed_tr->list, head: &ioc->delayed_tr_list);
4921	dewtprintk(ioc,
4922	ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",
4923	handle));
4924	} else
4925	_scsih_tm_tr_send(ioc, handle);
4926	}
4927	}
4928
4929
4930	/**
4931	* _scsih_check_volume_delete_events - set delete flag for volumes
4932	* @ioc: per adapter object
4933	* @event_data: the event data payload
4934	* Context: interrupt time.
4935	*
4936	* This will handle the case when the cable connected to entire volume is
4937	* pulled. We will take care of setting the deleted flag so normal IO will
4938	* not be sent.
4939	*/
4940	static void
4941	_scsih_check_volume_delete_events(struct MPT3SAS_ADAPTER *ioc,
4942	Mpi2EventDataIrVolume_t *event_data)
4943	{
4944	u32 state;
4945
4946	if (event_data->ReasonCode != MPI2_EVENT_IR_VOLUME_RC_STATE_CHANGED)
4947	return;
4948	state = le32_to_cpu(event_data->NewValue);
4949	if (state == MPI2_RAID_VOL_STATE_MISSING || state ==
4950	MPI2_RAID_VOL_STATE_FAILED)
4951	_scsih_set_volume_delete_flag(ioc,
4952	le16_to_cpu(event_data->VolDevHandle));
4953	}
4954
4955	/**
4956	* _scsih_temp_threshold_events - display temperature threshold exceeded events
4957	* @ioc: per adapter object
4958	* @event_data: the temp threshold event data
4959	* Context: interrupt time.
4960	*/
4961	static void
4962	_scsih_temp_threshold_events(struct MPT3SAS_ADAPTER *ioc,
4963	Mpi2EventDataTemperature_t *event_data)
4964	{
4965	u32 doorbell;
4966	if (ioc->temp_sensors_count >= event_data->SensorNum) {
4967	ioc_err(ioc, "Temperature Threshold flags %s%s%s%s exceeded for Sensor: %d !!!\n",
4968	le16_to_cpu(event_data->Status) & 0x1 ? "0 " : " ",
4969	le16_to_cpu(event_data->Status) & 0x2 ? "1 " : " ",
4970	le16_to_cpu(event_data->Status) & 0x4 ? "2 " : " ",
4971	le16_to_cpu(event_data->Status) & 0x8 ? "3 " : " ",
4972	event_data->SensorNum);
4973	ioc_err(ioc, "Current Temp In Celsius: %d\n",
4974	event_data->CurrentTemperature);
4975	if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
4976	doorbell = mpt3sas_base_get_iocstate(ioc, cooked: 0);
4977	if ((doorbell & MPI2_IOC_STATE_MASK) ==
4978	MPI2_IOC_STATE_FAULT) {
4979	mpt3sas_print_fault_code(ioc,
4980	doorbell & MPI2_DOORBELL_DATA_MASK);
4981	} else if ((doorbell & MPI2_IOC_STATE_MASK) ==
4982	MPI2_IOC_STATE_COREDUMP) {
4983	mpt3sas_print_coredump_info(ioc,
4984	doorbell & MPI2_DOORBELL_DATA_MASK);
4985	}
4986	}
4987	}
4988	}
4989
4990	static int _scsih_set_satl_pending(struct scsi_cmnd *scmd, bool pending)
4991	{
4992	struct MPT3SAS_DEVICE *priv = scmd->device->hostdata;
4993
4994	if (scmd->cmnd[0] != ATA_12 && scmd->cmnd[0] != ATA_16)
4995	return 0;
4996
4997	if (pending)
4998	return test_and_set_bit(nr: 0, addr: &priv->ata_command_pending);
4999
5000	clear_bit(nr: 0, addr: &priv->ata_command_pending);
5001	return 0;
5002	}
5003
5004	/**
5005	* _scsih_flush_running_cmds - completing outstanding commands.
5006	* @ioc: per adapter object
5007	*
5008	* The flushing out of all pending scmd commands following host reset,
5009	* where all IO is dropped to the floor.
5010	*/
5011	static void
5012	_scsih_flush_running_cmds(struct MPT3SAS_ADAPTER *ioc)
5013	{
5014	struct scsi_cmnd *scmd;
5015	struct scsiio_tracker *st;
5016	u16 smid;
5017	int count = 0;
5018
5019	for (smid = 1; smid <= ioc->scsiio_depth; smid++) {
5020	scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
5021	if (!scmd)
5022	continue;
5023	count++;
5024	_scsih_set_satl_pending(scmd, pending: false);
5025	st = scsi_cmd_priv(cmd: scmd);
5026	mpt3sas_base_clear_st(ioc, st);
5027	scsi_dma_unmap(cmd: scmd);
5028	if (ioc->pci_error_recovery || ioc->remove_host)
5029	scmd->result = DID_NO_CONNECT << 16;
5030	else
5031	scmd->result = DID_RESET << 16;
5032	scsi_done(cmd: scmd);
5033	}
5034	dtmprintk(ioc, ioc_info(ioc, "completing %d cmds\n", count));
5035	}
5036
5037	/**
5038	* _scsih_setup_eedp - setup MPI request for EEDP transfer
5039	* @ioc: per adapter object
5040	* @scmd: pointer to scsi command object
5041	* @mpi_request: pointer to the SCSI_IO request message frame
5042	*
5043	* Supporting protection 1 and 3.
5044	*/
5045	static void
5046	_scsih_setup_eedp(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,
5047	Mpi25SCSIIORequest_t *mpi_request)
5048	{
5049	u16 eedp_flags;
5050	Mpi25SCSIIORequest_t *mpi_request_3v =
5051	(Mpi25SCSIIORequest_t *)mpi_request;
5052
5053	switch (scsi_get_prot_op(scmd)) {
5054	case SCSI_PROT_READ_STRIP:
5055	eedp_flags = MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP;
5056	break;
5057	case SCSI_PROT_WRITE_INSERT:
5058	eedp_flags = MPI2_SCSIIO_EEDPFLAGS_INSERT_OP;
5059	break;
5060	default:
5061	return;
5062	}
5063
5064	if (scmd->prot_flags & SCSI_PROT_GUARD_CHECK)
5065	eedp_flags |= MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD;
5066
5067	if (scmd->prot_flags & SCSI_PROT_REF_CHECK)
5068	eedp_flags |= MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG;
5069
5070	if (scmd->prot_flags & SCSI_PROT_REF_INCREMENT) {
5071	eedp_flags |= MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG;
5072
5073	mpi_request->CDB.EEDP32.PrimaryReferenceTag =
5074	cpu_to_be32(scsi_prot_ref_tag(scmd));
5075	}
5076
5077	mpi_request_3v->EEDPBlockSize = cpu_to_le16(scsi_prot_interval(scmd));
5078
5079	if (ioc->is_gen35_ioc)
5080	eedp_flags |= MPI25_SCSIIO_EEDPFLAGS_APPTAG_DISABLE_MODE;
5081	mpi_request->EEDPFlags = cpu_to_le16(eedp_flags);
5082	}
5083
5084	/**
5085	* _scsih_eedp_error_handling - return sense code for EEDP errors
5086	* @scmd: pointer to scsi command object
5087	* @ioc_status: ioc status
5088	*/
5089	static void
5090	_scsih_eedp_error_handling(struct scsi_cmnd *scmd, u16 ioc_status)
5091	{
5092	u8 ascq;
5093
5094	switch (ioc_status) {
5095	case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
5096	ascq = 0x01;
5097	break;
5098	case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
5099	ascq = 0x02;
5100	break;
5101	case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
5102	ascq = 0x03;
5103	break;
5104	default:
5105	ascq = 0x00;
5106	break;
5107	}
5108	scsi_build_sense(scmd, desc: 0, ILLEGAL_REQUEST, asc: 0x10, ascq);
5109	set_host_byte(cmd: scmd, status: DID_ABORT);
5110	}
5111
5112	/**
5113	* scsih_qcmd - main scsi request entry point
5114	* @shost: SCSI host pointer
5115	* @scmd: pointer to scsi command object
5116	*
5117	* The callback index is set inside `ioc->scsi_io_cb_idx`.
5118	*
5119	* Return: 0 on success. If there's a failure, return either:
5120	* SCSI_MLQUEUE_DEVICE_BUSY if the device queue is full, or
5121	* SCSI_MLQUEUE_HOST_BUSY if the entire host queue is full
5122	*/
5123	static int
5124	scsih_qcmd(struct Scsi_Host *shost, struct scsi_cmnd *scmd)
5125	{
5126	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
5127	struct MPT3SAS_DEVICE *sas_device_priv_data;
5128	struct MPT3SAS_TARGET *sas_target_priv_data;
5129	struct _raid_device *raid_device;
5130	struct request *rq = scsi_cmd_to_rq(scmd);
5131	int class;
5132	Mpi25SCSIIORequest_t *mpi_request;
5133	struct _pcie_device *pcie_device = NULL;
5134	u32 mpi_control;
5135	u16 smid;
5136	u16 handle;
5137
5138	if (ioc->logging_level & MPT_DEBUG_SCSI)
5139	scsi_print_command(scmd);
5140
5141	sas_device_priv_data = scmd->device->hostdata;
5142	if (!sas_device_priv_data || !sas_device_priv_data->sas_target) {
5143	scmd->result = DID_NO_CONNECT << 16;
5144	scsi_done(cmd: scmd);
5145	return 0;
5146	}
5147
5148	if (!(_scsih_allow_scmd_to_device(ioc, scmd))) {
5149	scmd->result = DID_NO_CONNECT << 16;
5150	scsi_done(cmd: scmd);
5151	return 0;
5152	}
5153
5154	sas_target_priv_data = sas_device_priv_data->sas_target;
5155
5156	/* invalid device handle */
5157	handle = sas_target_priv_data->handle;
5158
5159	/*
5160	* Avoid error handling escallation when device is disconnected
5161	*/
5162	if (handle == MPT3SAS_INVALID_DEVICE_HANDLE || sas_device_priv_data->block) {
5163	if (scmd->device->host->shost_state == SHOST_RECOVERY &&
5164	scmd->cmnd[0] == TEST_UNIT_READY) {
5165	scsi_build_sense(scmd, desc: 0, UNIT_ATTENTION, asc: 0x29, ascq: 0x07);
5166	scsi_done(cmd: scmd);
5167	return 0;
5168	}
5169	}
5170
5171	if (handle == MPT3SAS_INVALID_DEVICE_HANDLE) {
5172	scmd->result = DID_NO_CONNECT << 16;
5173	scsi_done(cmd: scmd);
5174	return 0;
5175	}
5176
5177
5178	if (ioc->shost_recovery || ioc->ioc_link_reset_in_progress) {
5179	/* host recovery or link resets sent via IOCTLs */
5180	return SCSI_MLQUEUE_HOST_BUSY;
5181	} else if (sas_target_priv_data->deleted) {
5182	/* device has been deleted */
5183	scmd->result = DID_NO_CONNECT << 16;
5184	scsi_done(cmd: scmd);
5185	return 0;
5186	} else if (sas_target_priv_data->tm_busy ||
5187	sas_device_priv_data->block) {
5188	/* device busy with task management */
5189	return SCSI_MLQUEUE_DEVICE_BUSY;
5190	}
5191
5192	/*
5193	* Bug work around for firmware SATL handling. The loop
5194	* is based on atomic operations and ensures consistency
5195	* since we're lockless at this point
5196	*/
5197	do {
5198	if (test_bit(0, &sas_device_priv_data->ata_command_pending))
5199	return SCSI_MLQUEUE_DEVICE_BUSY;
5200	} while (_scsih_set_satl_pending(scmd, pending: true));
5201
5202	if (scmd->sc_data_direction == DMA_FROM_DEVICE)
5203	mpi_control = MPI2_SCSIIO_CONTROL_READ;
5204	else if (scmd->sc_data_direction == DMA_TO_DEVICE)
5205	mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
5206	else
5207	mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
5208
5209	/* set tags */
5210	mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
5211	/* NCQ Prio supported, make sure control indicated high priority */
5212	if (sas_device_priv_data->ncq_prio_enable) {
5213	class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
5214	if (class == IOPRIO_CLASS_RT)
5215	mpi_control |= 1 << MPI2_SCSIIO_CONTROL_CMDPRI_SHIFT;
5216	}
5217	/* Make sure Device is not raid volume.
5218	* We do not expose raid functionality to upper layer for warpdrive.
5219	*/
5220	if (((!ioc->is_warpdrive && !scsih_is_raid(dev: &scmd->device->sdev_gendev))
5221	&& !scsih_is_nvme(dev: &scmd->device->sdev_gendev))
5222	&& sas_is_tlr_enabled(scmd->device) && scmd->cmd_len != 32)
5223	mpi_control |= MPI2_SCSIIO_CONTROL_TLR_ON;
5224
5225	smid = mpt3sas_base_get_smid_scsiio(ioc, cb_idx: ioc->scsi_io_cb_idx, scmd);
5226	if (!smid) {
5227	ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
5228	_scsih_set_satl_pending(scmd, pending: false);
5229	goto out;
5230	}
5231	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5232	memset(mpi_request, 0, ioc->request_sz);
5233	_scsih_setup_eedp(ioc, scmd, mpi_request);
5234
5235	if (scmd->cmd_len == 32)
5236	mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
5237	mpi_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
5238	if (sas_device_priv_data->sas_target->flags &
5239	MPT_TARGET_FLAGS_RAID_COMPONENT)
5240	mpi_request->Function = MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH;
5241	else
5242	mpi_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
5243	mpi_request->DevHandle = cpu_to_le16(handle);
5244	mpi_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
5245	mpi_request->Control = cpu_to_le32(mpi_control);
5246	mpi_request->IoFlags = cpu_to_le16(scmd->cmd_len);
5247	mpi_request->MsgFlags = MPI2_SCSIIO_MSGFLAGS_SYSTEM_SENSE_ADDR;
5248	mpi_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE;
5249	mpi_request->SenseBufferLowAddress =
5250	mpt3sas_base_get_sense_buffer_dma(ioc, smid);
5251	mpi_request->SGLOffset0 = offsetof(Mpi25SCSIIORequest_t, SGL) / 4;
5252	int_to_scsilun(sas_device_priv_data->lun, (struct scsi_lun *)
5253	mpi_request->LUN);
5254	memcpy(mpi_request->CDB.CDB32, scmd->cmnd, scmd->cmd_len);
5255
5256	if (mpi_request->DataLength) {
5257	pcie_device = sas_target_priv_data->pcie_dev;
5258	if (ioc->build_sg_scmd(ioc, scmd, smid, pcie_device)) {
5259	mpt3sas_base_free_smid(ioc, smid);
5260	_scsih_set_satl_pending(scmd, pending: false);
5261	goto out;
5262	}
5263	} else
5264	ioc->build_zero_len_sge(ioc, &mpi_request->SGL);
5265
5266	raid_device = sas_target_priv_data->raid_device;
5267	if (raid_device && raid_device->direct_io_enabled)
5268	mpt3sas_setup_direct_io(ioc, scmd,
5269	raid_device, mpi_request);
5270
5271	if (likely(mpi_request->Function == MPI2_FUNCTION_SCSI_IO_REQUEST)) {
5272	if (sas_target_priv_data->flags & MPT_TARGET_FASTPATH_IO) {
5273	mpi_request->IoFlags = cpu_to_le16(scmd->cmd_len |
5274	MPI25_SCSIIO_IOFLAGS_FAST_PATH);
5275	ioc->put_smid_fast_path(ioc, smid, handle);
5276	} else
5277	ioc->put_smid_scsi_io(ioc, smid,
5278	le16_to_cpu(mpi_request->DevHandle));
5279	} else
5280	ioc->put_smid_default(ioc, smid);
5281	return 0;
5282
5283	out:
5284	return SCSI_MLQUEUE_HOST_BUSY;
5285	}
5286
5287	/**
5288	* _scsih_normalize_sense - normalize descriptor and fixed format sense data
5289	* @sense_buffer: sense data returned by target
5290	* @data: normalized skey/asc/ascq
5291	*/
5292	static void
5293	_scsih_normalize_sense(char *sense_buffer, struct sense_info *data)
5294	{
5295	if ((sense_buffer[0] & 0x7F) >= 0x72) {
5296	/* descriptor format */
5297	data->skey = sense_buffer[1] & 0x0F;
5298	data->asc = sense_buffer[2];
5299	data->ascq = sense_buffer[3];
5300	} else {
5301	/* fixed format */
5302	data->skey = sense_buffer[2] & 0x0F;
5303	data->asc = sense_buffer[12];
5304	data->ascq = sense_buffer[13];
5305	}
5306	}
5307
5308	/**
5309	* _scsih_scsi_ioc_info - translated non-successful SCSI_IO request
5310	* @ioc: per adapter object
5311	* @scmd: pointer to scsi command object
5312	* @mpi_reply: reply mf payload returned from firmware
5313	* @smid: ?
5314	*
5315	* scsi_status - SCSI Status code returned from target device
5316	* scsi_state - state info associated with SCSI_IO determined by ioc
5317	* ioc_status - ioc supplied status info
5318	*/
5319	static void
5320	_scsih_scsi_ioc_info(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,
5321	Mpi2SCSIIOReply_t *mpi_reply, u16 smid)
5322	{
5323	u32 response_info;
5324	u8 *response_bytes;
5325	u16 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) &
5326	MPI2_IOCSTATUS_MASK;
5327	u8 scsi_state = mpi_reply->SCSIState;
5328	u8 scsi_status = mpi_reply->SCSIStatus;
5329	char *desc_ioc_state = NULL;
5330	char *desc_scsi_status = NULL;
5331	char *desc_scsi_state = ioc->tmp_string;
5332	u32 log_info = le32_to_cpu(mpi_reply->IOCLogInfo);
5333	struct _sas_device *sas_device = NULL;
5334	struct _pcie_device *pcie_device = NULL;
5335	struct scsi_target *starget = scmd->device->sdev_target;
5336	struct MPT3SAS_TARGET *priv_target = starget->hostdata;
5337	char *device_str = NULL;
5338
5339	if (!priv_target)
5340	return;
5341	if (ioc->hide_ir_msg)
5342	device_str = "WarpDrive";
5343	else
5344	device_str = "volume";
5345
5346	if (log_info == 0x31170000)
5347	return;
5348
5349	switch (ioc_status) {
5350	case MPI2_IOCSTATUS_SUCCESS:
5351	desc_ioc_state = "success";
5352	break;
5353	case MPI2_IOCSTATUS_INVALID_FUNCTION:
5354	desc_ioc_state = "invalid function";
5355	break;
5356	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
5357	desc_ioc_state = "scsi recovered error";
5358	break;
5359	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
5360	desc_ioc_state = "scsi invalid dev handle";
5361	break;
5362	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
5363	desc_ioc_state = "scsi device not there";
5364	break;
5365	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
5366	desc_ioc_state = "scsi data overrun";
5367	break;
5368	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
5369	desc_ioc_state = "scsi data underrun";
5370	break;
5371	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
5372	desc_ioc_state = "scsi io data error";
5373	break;
5374	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
5375	desc_ioc_state = "scsi protocol error";
5376	break;
5377	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
5378	desc_ioc_state = "scsi task terminated";
5379	break;
5380	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
5381	desc_ioc_state = "scsi residual mismatch";
5382	break;
5383	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
5384	desc_ioc_state = "scsi task mgmt failed";
5385	break;
5386	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
5387	desc_ioc_state = "scsi ioc terminated";
5388	break;
5389	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
5390	desc_ioc_state = "scsi ext terminated";
5391	break;
5392	case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
5393	desc_ioc_state = "eedp guard error";
5394	break;
5395	case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
5396	desc_ioc_state = "eedp ref tag error";
5397	break;
5398	case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
5399	desc_ioc_state = "eedp app tag error";
5400	break;
5401	case MPI2_IOCSTATUS_INSUFFICIENT_POWER:
5402	desc_ioc_state = "insufficient power";
5403	break;
5404	default:
5405	desc_ioc_state = "unknown";
5406	break;
5407	}
5408
5409	switch (scsi_status) {
5410	case MPI2_SCSI_STATUS_GOOD:
5411	desc_scsi_status = "good";
5412	break;
5413	case MPI2_SCSI_STATUS_CHECK_CONDITION:
5414	desc_scsi_status = "check condition";
5415	break;
5416	case MPI2_SCSI_STATUS_CONDITION_MET:
5417	desc_scsi_status = "condition met";
5418	break;
5419	case MPI2_SCSI_STATUS_BUSY:
5420	desc_scsi_status = "busy";
5421	break;
5422	case MPI2_SCSI_STATUS_INTERMEDIATE:
5423	desc_scsi_status = "intermediate";
5424	break;
5425	case MPI2_SCSI_STATUS_INTERMEDIATE_CONDMET:
5426	desc_scsi_status = "intermediate condmet";
5427	break;
5428	case MPI2_SCSI_STATUS_RESERVATION_CONFLICT:
5429	desc_scsi_status = "reservation conflict";
5430	break;
5431	case MPI2_SCSI_STATUS_COMMAND_TERMINATED:
5432	desc_scsi_status = "command terminated";
5433	break;
5434	case MPI2_SCSI_STATUS_TASK_SET_FULL:
5435	desc_scsi_status = "task set full";
5436	break;
5437	case MPI2_SCSI_STATUS_ACA_ACTIVE:
5438	desc_scsi_status = "aca active";
5439	break;
5440	case MPI2_SCSI_STATUS_TASK_ABORTED:
5441	desc_scsi_status = "task aborted";
5442	break;
5443	default:
5444	desc_scsi_status = "unknown";
5445	break;
5446	}
5447
5448	desc_scsi_state[0] = '\0';
5449	if (!scsi_state)
5450	desc_scsi_state = " ";
5451	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID)
5452	strcat(p: desc_scsi_state, q: "response info ");
5453	if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
5454	strcat(p: desc_scsi_state, q: "state terminated ");
5455	if (scsi_state & MPI2_SCSI_STATE_NO_SCSI_STATUS)
5456	strcat(p: desc_scsi_state, q: "no status ");
5457	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_FAILED)
5458	strcat(p: desc_scsi_state, q: "autosense failed ");
5459	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID)
5460	strcat(p: desc_scsi_state, q: "autosense valid ");
5461
5462	scsi_print_command(scmd);
5463
5464	if (priv_target->flags & MPT_TARGET_FLAGS_VOLUME) {
5465	ioc_warn(ioc, "\t%s wwid(0x%016llx)\n",
5466	device_str, (u64)priv_target->sas_address);
5467	} else if (priv_target->flags & MPT_TARGET_FLAGS_PCIE_DEVICE) {
5468	pcie_device = mpt3sas_get_pdev_from_target(ioc, tgt_priv: priv_target);
5469	if (pcie_device) {
5470	ioc_info(ioc, "\twwid(0x%016llx), port(%d)\n",
5471	(u64)pcie_device->wwid, pcie_device->port_num);
5472	if (pcie_device->enclosure_handle != 0)
5473	ioc_info(ioc, "\tenclosure logical id(0x%016llx), slot(%d)\n",
5474	(u64)pcie_device->enclosure_logical_id,
5475	pcie_device->slot);
5476	if (pcie_device->connector_name[0])
5477	ioc_info(ioc, "\tenclosure level(0x%04x), connector name( %s)\n",
5478	pcie_device->enclosure_level,
5479	pcie_device->connector_name);
5480	pcie_device_put(p: pcie_device);
5481	}
5482	} else {
5483	sas_device = mpt3sas_get_sdev_from_target(ioc, tgt_priv: priv_target);
5484	if (sas_device) {
5485	ioc_warn(ioc, "\tsas_address(0x%016llx), phy(%d)\n",
5486	(u64)sas_device->sas_address, sas_device->phy);
5487
5488	_scsih_display_enclosure_chassis_info(ioc, sas_device,
5489	NULL, NULL);
5490
5491	sas_device_put(s: sas_device);
5492	}
5493	}
5494
5495	ioc_warn(ioc, "\thandle(0x%04x), ioc_status(%s)(0x%04x), smid(%d)\n",
5496	le16_to_cpu(mpi_reply->DevHandle),
5497	desc_ioc_state, ioc_status, smid);
5498	ioc_warn(ioc, "\trequest_len(%d), underflow(%d), resid(%d)\n",
5499	scsi_bufflen(scmd), scmd->underflow, scsi_get_resid(scmd));
5500	ioc_warn(ioc, "\ttag(%d), transfer_count(%d), sc->result(0x%08x)\n",
5501	le16_to_cpu(mpi_reply->TaskTag),
5502	le32_to_cpu(mpi_reply->TransferCount), scmd->result);
5503	ioc_warn(ioc, "\tscsi_status(%s)(0x%02x), scsi_state(%s)(0x%02x)\n",
5504	desc_scsi_status, scsi_status, desc_scsi_state, scsi_state);
5505
5506	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
5507	struct sense_info data;
5508	_scsih_normalize_sense(sense_buffer: scmd->sense_buffer, data: &data);
5509	ioc_warn(ioc, "\t[sense_key,asc,ascq]: [0x%02x,0x%02x,0x%02x], count(%d)\n",
5510	data.skey, data.asc, data.ascq,
5511	le32_to_cpu(mpi_reply->SenseCount));
5512	}
5513	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
5514	response_info = le32_to_cpu(mpi_reply->ResponseInfo);
5515	response_bytes = (u8 *)&response_info;
5516	_scsih_response_code(ioc, response_code: response_bytes[0]);
5517	}
5518	}
5519
5520	/**
5521	* _scsih_turn_on_pfa_led - illuminate PFA LED
5522	* @ioc: per adapter object
5523	* @handle: device handle
5524	* Context: process
5525	*/
5526	static void
5527	_scsih_turn_on_pfa_led(struct MPT3SAS_ADAPTER *ioc, u16 handle)
5528	{
5529	Mpi2SepReply_t mpi_reply;
5530	Mpi2SepRequest_t mpi_request;
5531	struct _sas_device *sas_device;
5532
5533	sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
5534	if (!sas_device)
5535	return;
5536
5537	memset(&mpi_request, 0, sizeof(Mpi2SepRequest_t));
5538	mpi_request.Function = MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR;
5539	mpi_request.Action = MPI2_SEP_REQ_ACTION_WRITE_STATUS;
5540	mpi_request.SlotStatus =
5541	cpu_to_le32(MPI2_SEP_REQ_SLOTSTATUS_PREDICTED_FAULT);
5542	mpi_request.DevHandle = cpu_to_le16(handle);
5543	mpi_request.Flags = MPI2_SEP_REQ_FLAGS_DEVHANDLE_ADDRESS;
5544	if ((mpt3sas_base_scsi_enclosure_processor(ioc, mpi_reply: &mpi_reply,
5545	mpi_request: &mpi_request)) != 0) {
5546	ioc_err(ioc, "failure at %s:%d/%s()!\n",
5547	__FILE__, __LINE__, __func__);
5548	goto out;
5549	}
5550	sas_device->pfa_led_on = 1;
5551
5552	if (mpi_reply.IOCStatus || mpi_reply.IOCLogInfo) {
5553	dewtprintk(ioc,
5554	ioc_info(ioc, "enclosure_processor: ioc_status (0x%04x), loginfo(0x%08x)\n",
5555	le16_to_cpu(mpi_reply.IOCStatus),
5556	le32_to_cpu(mpi_reply.IOCLogInfo)));
5557	goto out;
5558	}
5559	out:
5560	sas_device_put(s: sas_device);
5561	}
5562
5563	/**
5564	* _scsih_turn_off_pfa_led - turn off Fault LED
5565	* @ioc: per adapter object
5566	* @sas_device: sas device whose PFA LED has to turned off
5567	* Context: process
5568	*/
5569	static void
5570	_scsih_turn_off_pfa_led(struct MPT3SAS_ADAPTER *ioc,
5571	struct _sas_device *sas_device)
5572	{
5573	Mpi2SepReply_t mpi_reply;
5574	Mpi2SepRequest_t mpi_request;
5575
5576	memset(&mpi_request, 0, sizeof(Mpi2SepRequest_t));
5577	mpi_request.Function = MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR;
5578	mpi_request.Action = MPI2_SEP_REQ_ACTION_WRITE_STATUS;
5579	mpi_request.SlotStatus = 0;
5580	mpi_request.Slot = cpu_to_le16(sas_device->slot);
5581	mpi_request.DevHandle = 0;
5582	mpi_request.EnclosureHandle = cpu_to_le16(sas_device->enclosure_handle);
5583	mpi_request.Flags = MPI2_SEP_REQ_FLAGS_ENCLOSURE_SLOT_ADDRESS;
5584	if ((mpt3sas_base_scsi_enclosure_processor(ioc, mpi_reply: &mpi_reply,
5585	mpi_request: &mpi_request)) != 0) {
5586	ioc_err(ioc, "failure at %s:%d/%s()!\n",
5587	__FILE__, __LINE__, __func__);
5588	return;
5589	}
5590
5591	if (mpi_reply.IOCStatus || mpi_reply.IOCLogInfo) {
5592	dewtprintk(ioc,
5593	ioc_info(ioc, "enclosure_processor: ioc_status (0x%04x), loginfo(0x%08x)\n",
5594	le16_to_cpu(mpi_reply.IOCStatus),
5595	le32_to_cpu(mpi_reply.IOCLogInfo)));
5596	return;
5597	}
5598	}
5599
5600	/**
5601	* _scsih_send_event_to_turn_on_pfa_led - fire delayed event
5602	* @ioc: per adapter object
5603	* @handle: device handle
5604	* Context: interrupt.
5605	*/
5606	static void
5607	_scsih_send_event_to_turn_on_pfa_led(struct MPT3SAS_ADAPTER *ioc, u16 handle)
5608	{
5609	struct fw_event_work *fw_event;
5610
5611	fw_event = alloc_fw_event_work(len: 0);
5612	if (!fw_event)
5613	return;
5614	fw_event->event = MPT3SAS_TURN_ON_PFA_LED;
5615	fw_event->device_handle = handle;
5616	fw_event->ioc = ioc;
5617	_scsih_fw_event_add(ioc, fw_event);
5618	fw_event_work_put(fw_work: fw_event);
5619	}
5620
5621	/**
5622	* _scsih_smart_predicted_fault - process smart errors
5623	* @ioc: per adapter object
5624	* @handle: device handle
5625	* Context: interrupt.
5626	*/
5627	static void
5628	_scsih_smart_predicted_fault(struct MPT3SAS_ADAPTER *ioc, u16 handle)
5629	{
5630	struct scsi_target *starget;
5631	struct MPT3SAS_TARGET *sas_target_priv_data;
5632	Mpi2EventNotificationReply_t *event_reply;
5633	Mpi2EventDataSasDeviceStatusChange_t *event_data;
5634	struct _sas_device *sas_device;
5635	ssize_t sz;
5636	unsigned long flags;
5637
5638	/* only handle non-raid devices */
5639	spin_lock_irqsave(&ioc->sas_device_lock, flags);
5640	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
5641	if (!sas_device)
5642	goto out_unlock;
5643
5644	starget = sas_device->starget;
5645	sas_target_priv_data = starget->hostdata;
5646
5647	if ((sas_target_priv_data->flags & MPT_TARGET_FLAGS_RAID_COMPONENT) ||
5648	((sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME)))
5649	goto out_unlock;
5650
5651	_scsih_display_enclosure_chassis_info(NULL, sas_device, NULL, starget);
5652
5653	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
5654
5655	if (ioc->pdev->subsystem_vendor == PCI_VENDOR_ID_IBM)
5656	_scsih_send_event_to_turn_on_pfa_led(ioc, handle);
5657
5658	/* insert into event log */
5659	sz = offsetof(Mpi2EventNotificationReply_t, EventData) +
5660	sizeof(Mpi2EventDataSasDeviceStatusChange_t);
5661	event_reply = kzalloc(size: sz, GFP_ATOMIC);
5662	if (!event_reply) {
5663	ioc_err(ioc, "failure at %s:%d/%s()!\n",
5664	__FILE__, __LINE__, __func__);
5665	goto out;
5666	}
5667
5668	event_reply->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
5669	event_reply->Event =
5670	cpu_to_le16(MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
5671	event_reply->MsgLength = sz/4;
5672	event_reply->EventDataLength =
5673	cpu_to_le16(sizeof(Mpi2EventDataSasDeviceStatusChange_t)/4);
5674	event_data = (Mpi2EventDataSasDeviceStatusChange_t *)
5675	event_reply->EventData;
5676	event_data->ReasonCode = MPI2_EVENT_SAS_DEV_STAT_RC_SMART_DATA;
5677	event_data->ASC = 0x5D;
5678	event_data->DevHandle = cpu_to_le16(handle);
5679	event_data->SASAddress = cpu_to_le64(sas_target_priv_data->sas_address);
5680	mpt3sas_ctl_add_to_event_log(ioc, mpi_reply: event_reply);
5681	kfree(objp: event_reply);
5682	out:
5683	if (sas_device)
5684	sas_device_put(s: sas_device);
5685	return;
5686
5687	out_unlock:
5688	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
5689	goto out;
5690	}
5691
5692	/**
5693	* _scsih_io_done - scsi request callback
5694	* @ioc: per adapter object
5695	* @smid: system request message index
5696	* @msix_index: MSIX table index supplied by the OS
5697	* @reply: reply message frame(lower 32bit addr)
5698	*
5699	* Callback handler when using _scsih_qcmd.
5700	*
5701	* Return: 1 meaning mf should be freed from _base_interrupt
5702	* 0 means the mf is freed from this function.
5703	*/
5704	static u8
5705	_scsih_io_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
5706	{
5707	Mpi25SCSIIORequest_t *mpi_request;
5708	Mpi2SCSIIOReply_t *mpi_reply;
5709	struct scsi_cmnd *scmd;
5710	struct scsiio_tracker *st;
5711	u16 ioc_status;
5712	u32 xfer_cnt;
5713	u8 scsi_state;
5714	u8 scsi_status;
5715	u32 log_info;
5716	struct MPT3SAS_DEVICE *sas_device_priv_data;
5717	u32 response_code = 0;
5718
5719	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, phys_addr: reply);
5720
5721	scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
5722	if (scmd == NULL)
5723	return 1;
5724
5725	_scsih_set_satl_pending(scmd, pending: false);
5726
5727	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5728
5729	if (mpi_reply == NULL) {
5730	scmd->result = DID_OK << 16;
5731	goto out;
5732	}
5733
5734	sas_device_priv_data = scmd->device->hostdata;
5735	if (!sas_device_priv_data || !sas_device_priv_data->sas_target ||
5736	sas_device_priv_data->sas_target->deleted) {
5737	scmd->result = DID_NO_CONNECT << 16;
5738	goto out;
5739	}
5740	ioc_status = le16_to_cpu(mpi_reply->IOCStatus);
5741
5742	/*
5743	* WARPDRIVE: If direct_io is set then it is directIO,
5744	* the failed direct I/O should be redirected to volume
5745	*/
5746	st = scsi_cmd_priv(cmd: scmd);
5747	if (st->direct_io &&
5748	((ioc_status & MPI2_IOCSTATUS_MASK)
5749	!= MPI2_IOCSTATUS_SCSI_TASK_TERMINATED)) {
5750	st->direct_io = 0;
5751	st->scmd = scmd;
5752	memcpy(mpi_request->CDB.CDB32, scmd->cmnd, scmd->cmd_len);
5753	mpi_request->DevHandle =
5754	cpu_to_le16(sas_device_priv_data->sas_target->handle);
5755	ioc->put_smid_scsi_io(ioc, smid,
5756	sas_device_priv_data->sas_target->handle);
5757	return 0;
5758	}
5759	/* turning off TLR */
5760	scsi_state = mpi_reply->SCSIState;
5761	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID)
5762	response_code =
5763	le32_to_cpu(mpi_reply->ResponseInfo) & 0xFF;
5764	if (!sas_device_priv_data->tlr_snoop_check) {
5765	sas_device_priv_data->tlr_snoop_check++;
5766	if ((!ioc->is_warpdrive &&
5767	!scsih_is_raid(dev: &scmd->device->sdev_gendev) &&
5768	!scsih_is_nvme(dev: &scmd->device->sdev_gendev))
5769	&& sas_is_tlr_enabled(scmd->device) &&
5770	response_code == MPI2_SCSITASKMGMT_RSP_INVALID_FRAME) {
5771	sas_disable_tlr(scmd->device);
5772	sdev_printk(KERN_INFO, scmd->device, "TLR disabled\n");
5773	}
5774	}
5775
5776	xfer_cnt = le32_to_cpu(mpi_reply->TransferCount);
5777	scsi_set_resid(cmd: scmd, resid: scsi_bufflen(cmd: scmd) - xfer_cnt);
5778	if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE)
5779	log_info = le32_to_cpu(mpi_reply->IOCLogInfo);
5780	else
5781	log_info = 0;
5782	ioc_status &= MPI2_IOCSTATUS_MASK;
5783	scsi_status = mpi_reply->SCSIStatus;
5784
5785	if (ioc_status == MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN && xfer_cnt == 0 &&
5786	(scsi_status == MPI2_SCSI_STATUS_BUSY ||
5787	scsi_status == MPI2_SCSI_STATUS_RESERVATION_CONFLICT ||
5788	scsi_status == MPI2_SCSI_STATUS_TASK_SET_FULL)) {
5789	ioc_status = MPI2_IOCSTATUS_SUCCESS;
5790	}
5791
5792	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
5793	struct sense_info data;
5794	const void *sense_data = mpt3sas_base_get_sense_buffer(ioc,
5795	smid);
5796	u32 sz = min_t(u32, SCSI_SENSE_BUFFERSIZE,
5797	le32_to_cpu(mpi_reply->SenseCount));
5798	memcpy(scmd->sense_buffer, sense_data, sz);
5799	_scsih_normalize_sense(sense_buffer: scmd->sense_buffer, data: &data);
5800	/* failure prediction threshold exceeded */
5801	if (data.asc == 0x5D)
5802	_scsih_smart_predicted_fault(ioc,
5803	le16_to_cpu(mpi_reply->DevHandle));
5804	mpt3sas_trigger_scsi(ioc, sense_key: data.skey, asc: data.asc, ascq: data.ascq);
5805
5806	if ((ioc->logging_level & MPT_DEBUG_REPLY) &&
5807	((scmd->sense_buffer[2] == UNIT_ATTENTION) ||
5808	(scmd->sense_buffer[2] == MEDIUM_ERROR) ||
5809	(scmd->sense_buffer[2] == HARDWARE_ERROR)))
5810	_scsih_scsi_ioc_info(ioc, scmd, mpi_reply, smid);
5811	}
5812	switch (ioc_status) {
5813	case MPI2_IOCSTATUS_BUSY:
5814	case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES:
5815	scmd->result = SAM_STAT_BUSY;
5816	break;
5817
5818	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
5819	scmd->result = DID_NO_CONNECT << 16;
5820	break;
5821
5822	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
5823	if (sas_device_priv_data->block) {
5824	scmd->result = DID_TRANSPORT_DISRUPTED << 16;
5825	goto out;
5826	}
5827	if (log_info == 0x31110630) {
5828	if (scmd->retries > 2) {
5829	scmd->result = DID_NO_CONNECT << 16;
5830	scsi_device_set_state(sdev: scmd->device,
5831	state: SDEV_OFFLINE);
5832	} else {
5833	scmd->result = DID_SOFT_ERROR << 16;
5834	scmd->device->expecting_cc_ua = 1;
5835	}
5836	break;
5837	} else if (log_info == VIRTUAL_IO_FAILED_RETRY) {
5838	scmd->result = DID_RESET << 16;
5839	break;
5840	} else if ((scmd->device->channel == RAID_CHANNEL) &&
5841	(scsi_state == (MPI2_SCSI_STATE_TERMINATED |
5842	MPI2_SCSI_STATE_NO_SCSI_STATUS))) {
5843	scmd->result = DID_RESET << 16;
5844	break;
5845	}
5846	scmd->result = DID_SOFT_ERROR << 16;
5847	break;
5848	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
5849	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
5850	scmd->result = DID_RESET << 16;
5851	break;
5852
5853	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
5854	if ((xfer_cnt == 0) || (scmd->underflow > xfer_cnt))
5855	scmd->result = DID_SOFT_ERROR << 16;
5856	else
5857	scmd->result = (DID_OK << 16) | scsi_status;
5858	break;
5859
5860	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
5861	scmd->result = (DID_OK << 16) | scsi_status;
5862
5863	if ((scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID))
5864	break;
5865
5866	if (xfer_cnt < scmd->underflow) {
5867	if (scsi_status == SAM_STAT_BUSY)
5868	scmd->result = SAM_STAT_BUSY;
5869	else
5870	scmd->result = DID_SOFT_ERROR << 16;
5871	} else if (scsi_state & (MPI2_SCSI_STATE_AUTOSENSE_FAILED |
5872	MPI2_SCSI_STATE_NO_SCSI_STATUS))
5873	scmd->result = DID_SOFT_ERROR << 16;
5874	else if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
5875	scmd->result = DID_RESET << 16;
5876	else if (!xfer_cnt && scmd->cmnd[0] == REPORT_LUNS) {
5877	mpi_reply->SCSIState = MPI2_SCSI_STATE_AUTOSENSE_VALID;
5878	mpi_reply->SCSIStatus = SAM_STAT_CHECK_CONDITION;
5879	scsi_build_sense(scmd, desc: 0, ILLEGAL_REQUEST,
5880	asc: 0x20, ascq: 0);
5881	}
5882	break;
5883
5884	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
5885	scsi_set_resid(cmd: scmd, resid: 0);
5886	fallthrough;
5887	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
5888	case MPI2_IOCSTATUS_SUCCESS:
5889	scmd->result = (DID_OK << 16) | scsi_status;
5890	if (response_code ==
5891	MPI2_SCSITASKMGMT_RSP_INVALID_FRAME ||
5892	(scsi_state & (MPI2_SCSI_STATE_AUTOSENSE_FAILED |
5893	MPI2_SCSI_STATE_NO_SCSI_STATUS)))
5894	scmd->result = DID_SOFT_ERROR << 16;
5895	else if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
5896	scmd->result = DID_RESET << 16;
5897	break;
5898
5899	case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
5900	case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
5901	case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
5902	_scsih_eedp_error_handling(scmd, ioc_status);
5903	break;
5904
5905	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
5906	case MPI2_IOCSTATUS_INVALID_FUNCTION:
5907	case MPI2_IOCSTATUS_INVALID_SGL:
5908	case MPI2_IOCSTATUS_INTERNAL_ERROR:
5909	case MPI2_IOCSTATUS_INVALID_FIELD:
5910	case MPI2_IOCSTATUS_INVALID_STATE:
5911	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
5912	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
5913	case MPI2_IOCSTATUS_INSUFFICIENT_POWER:
5914	default:
5915	scmd->result = DID_SOFT_ERROR << 16;
5916	break;
5917
5918	}
5919
5920	if (scmd->result && (ioc->logging_level & MPT_DEBUG_REPLY))
5921	_scsih_scsi_ioc_info(ioc , scmd, mpi_reply, smid);
5922
5923	out:
5924
5925	scsi_dma_unmap(cmd: scmd);
5926	mpt3sas_base_free_smid(ioc, smid);
5927	scsi_done(cmd: scmd);
5928	return 0;
5929	}
5930
5931	/**
5932	* _scsih_update_vphys_after_reset - update the Port's
5933	* vphys_list after reset
5934	* @ioc: per adapter object
5935	*
5936	* Returns nothing.
5937	*/
5938	static void
5939	_scsih_update_vphys_after_reset(struct MPT3SAS_ADAPTER *ioc)
5940	{
5941	u16 sz, ioc_status;
5942	int i;
5943	Mpi2ConfigReply_t mpi_reply;
5944	Mpi2SasIOUnitPage0_t *sas_iounit_pg0 = NULL;
5945	u16 attached_handle;
5946	u64 attached_sas_addr;
5947	u8 found = 0, port_id;
5948	Mpi2SasPhyPage0_t phy_pg0;
5949	struct hba_port *port, *port_next, *mport;
5950	struct virtual_phy *vphy, *vphy_next;
5951	struct _sas_device *sas_device;
5952
5953	/*
5954	* Mark all the vphys objects as dirty.
5955	*/
5956	list_for_each_entry_safe(port, port_next,
5957	&ioc->port_table_list, list) {
5958	if (!port->vphys_mask)
5959	continue;
5960	list_for_each_entry_safe(vphy, vphy_next,
5961	&port->vphys_list, list) {
5962	vphy->flags |= MPT_VPHY_FLAG_DIRTY_PHY;
5963	}
5964	}
5965
5966	/*
5967	* Read SASIOUnitPage0 to get each HBA Phy's data.
5968	*/
5969	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) +
5970	(ioc->sas_hba.num_phys * sizeof(Mpi2SasIOUnit0PhyData_t));
5971	sas_iounit_pg0 = kzalloc(size: sz, GFP_KERNEL);
5972	if (!sas_iounit_pg0) {
5973	ioc_err(ioc, "failure at %s:%d/%s()!\n",
5974	__FILE__, __LINE__, __func__);
5975	return;
5976	}
5977	if ((mpt3sas_config_get_sas_iounit_pg0(ioc, mpi_reply: &mpi_reply,
5978	config_page: sas_iounit_pg0, sz)) != 0)
5979	goto out;
5980	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
5981	if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
5982	goto out;
5983	/*
5984	* Loop over each HBA Phy.
5985	*/
5986	for (i = 0; i < ioc->sas_hba.num_phys; i++) {
5987	/*
5988	* Check whether Phy's Negotiation Link Rate is > 1.5G or not.
5989	*/
5990	if ((sas_iounit_pg0->PhyData[i].NegotiatedLinkRate >> 4) <
5991	MPI2_SAS_NEG_LINK_RATE_1_5)
5992	continue;
5993	/*
5994	* Check whether Phy is connected to SEP device or not,
5995	* if it is SEP device then read the Phy's SASPHYPage0 data to
5996	* determine whether Phy is a virtual Phy or not. if it is
5997	* virtual phy then it is conformed that the attached remote
5998	* device is a HBA's vSES device.
5999	*/
6000	if (!(le32_to_cpu(
6001	sas_iounit_pg0->PhyData[i].ControllerPhyDeviceInfo) &
6002	MPI2_SAS_DEVICE_INFO_SEP))
6003	continue;
6004
6005	if ((mpt3sas_config_get_phy_pg0(ioc, mpi_reply: &mpi_reply, config_page: &phy_pg0,
6006	phy_number: i))) {
6007	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6008	__FILE__, __LINE__, __func__);
6009	continue;
6010	}
6011
6012	if (!(le32_to_cpu(phy_pg0.PhyInfo) &
6013	MPI2_SAS_PHYINFO_VIRTUAL_PHY))
6014	continue;
6015	/*
6016	* Get the vSES device's SAS Address.
6017	*/
6018	attached_handle = le16_to_cpu(
6019	sas_iounit_pg0->PhyData[i].AttachedDevHandle);
6020	if (_scsih_get_sas_address(ioc, handle: attached_handle,
6021	sas_address: &attached_sas_addr) != 0) {
6022	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6023	__FILE__, __LINE__, __func__);
6024	continue;
6025	}
6026
6027	found = 0;
6028	port = port_next = NULL;
6029	/*
6030	* Loop over each virtual_phy object from
6031	* each port's vphys_list.
6032	*/
6033	list_for_each_entry_safe(port,
6034	port_next, &ioc->port_table_list, list) {
6035	if (!port->vphys_mask)
6036	continue;
6037	list_for_each_entry_safe(vphy, vphy_next,
6038	&port->vphys_list, list) {
6039	/*
6040	* Continue with next virtual_phy object
6041	* if the object is not marked as dirty.
6042	*/
6043	if (!(vphy->flags & MPT_VPHY_FLAG_DIRTY_PHY))
6044	continue;
6045
6046	/*
6047	* Continue with next virtual_phy object
6048	* if the object's SAS Address is not equals
6049	* to current Phy's vSES device SAS Address.
6050	*/
6051	if (vphy->sas_address != attached_sas_addr)
6052	continue;
6053	/*
6054	* Enable current Phy number bit in object's
6055	* phy_mask field.
6056	*/
6057	if (!(vphy->phy_mask & (1 << i)))
6058	vphy->phy_mask = (1 << i);
6059	/*
6060	* Get hba_port object from hba_port table
6061	* corresponding to current phy's Port ID.
6062	* if there is no hba_port object corresponding
6063	* to Phy's Port ID then create a new hba_port
6064	* object & add to hba_port table.
6065	*/
6066	port_id = sas_iounit_pg0->PhyData[i].Port;
6067	mport = mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 1);
6068	if (!mport) {
6069	mport = kzalloc(
6070	size: sizeof(struct hba_port), GFP_KERNEL);
6071	if (!mport)
6072	break;
6073	mport->port_id = port_id;
6074	ioc_info(ioc,
6075	"%s: hba_port entry: %p, port: %d is added to hba_port list\n",
6076	__func__, mport, mport->port_id);
6077	list_add_tail(new: &mport->list,
6078	head: &ioc->port_table_list);
6079	}
6080	/*
6081	* If mport & port pointers are not pointing to
6082	* same hba_port object then it means that vSES
6083	* device's Port ID got changed after reset and
6084	* hence move current virtual_phy object from
6085	* port's vphys_list to mport's vphys_list.
6086	*/
6087	if (port != mport) {
6088	if (!mport->vphys_mask)
6089	INIT_LIST_HEAD(
6090	list: &mport->vphys_list);
6091	mport->vphys_mask |= (1 << i);
6092	port->vphys_mask &= ~(1 << i);
6093	list_move(list: &vphy->list,
6094	head: &mport->vphys_list);
6095	sas_device = mpt3sas_get_sdev_by_addr(
6096	ioc, sas_address: attached_sas_addr, port);
6097	if (sas_device)
6098	sas_device->port = mport;
6099	}
6100	/*
6101	* Earlier while updating the hba_port table,
6102	* it is determined that there is no other
6103	* direct attached device with mport's Port ID,
6104	* Hence mport was marked as dirty. Only vSES
6105	* device has this Port ID, so unmark the mport
6106	* as dirt.
6107	*/
6108	if (mport->flags & HBA_PORT_FLAG_DIRTY_PORT) {
6109	mport->sas_address = 0;
6110	mport->phy_mask = 0;
6111	mport->flags &=
6112	~HBA_PORT_FLAG_DIRTY_PORT;
6113	}
6114	/*
6115	* Unmark current virtual_phy object as dirty.
6116	*/
6117	vphy->flags &= ~MPT_VPHY_FLAG_DIRTY_PHY;
6118	found = 1;
6119	break;
6120	}
6121	if (found)
6122	break;
6123	}
6124	}
6125	out:
6126	kfree(objp: sas_iounit_pg0);
6127	}
6128
6129	/**
6130	* _scsih_get_port_table_after_reset - Construct temporary port table
6131	* @ioc: per adapter object
6132	* @port_table: address where port table needs to be constructed
6133	*
6134	* return number of HBA port entries available after reset.
6135	*/
6136	static int
6137	_scsih_get_port_table_after_reset(struct MPT3SAS_ADAPTER *ioc,
6138	struct hba_port *port_table)
6139	{
6140	u16 sz, ioc_status;
6141	int i, j;
6142	Mpi2ConfigReply_t mpi_reply;
6143	Mpi2SasIOUnitPage0_t *sas_iounit_pg0 = NULL;
6144	u16 attached_handle;
6145	u64 attached_sas_addr;
6146	u8 found = 0, port_count = 0, port_id;
6147
6148	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys
6149	* sizeof(Mpi2SasIOUnit0PhyData_t));
6150	sas_iounit_pg0 = kzalloc(size: sz, GFP_KERNEL);
6151	if (!sas_iounit_pg0) {
6152	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6153	__FILE__, __LINE__, __func__);
6154	return port_count;
6155	}
6156
6157	if ((mpt3sas_config_get_sas_iounit_pg0(ioc, mpi_reply: &mpi_reply,
6158	config_page: sas_iounit_pg0, sz)) != 0)
6159	goto out;
6160	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
6161	if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
6162	goto out;
6163	for (i = 0; i < ioc->sas_hba.num_phys; i++) {
6164	found = 0;
6165	if ((sas_iounit_pg0->PhyData[i].NegotiatedLinkRate >> 4) <
6166	MPI2_SAS_NEG_LINK_RATE_1_5)
6167	continue;
6168	attached_handle =
6169	le16_to_cpu(sas_iounit_pg0->PhyData[i].AttachedDevHandle);
6170	if (_scsih_get_sas_address(
6171	ioc, handle: attached_handle, sas_address: &attached_sas_addr) != 0) {
6172	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6173	__FILE__, __LINE__, __func__);
6174	continue;
6175	}
6176
6177	for (j = 0; j < port_count; j++) {
6178	port_id = sas_iounit_pg0->PhyData[i].Port;
6179	if (port_table[j].port_id == port_id &&
6180	port_table[j].sas_address == attached_sas_addr) {
6181	port_table[j].phy_mask |= (1 << i);
6182	found = 1;
6183	break;
6184	}
6185	}
6186
6187	if (found)
6188	continue;
6189
6190	port_id = sas_iounit_pg0->PhyData[i].Port;
6191	port_table[port_count].port_id = port_id;
6192	port_table[port_count].phy_mask = (1 << i);
6193	port_table[port_count].sas_address = attached_sas_addr;
6194	port_count++;
6195	}
6196	out:
6197	kfree(objp: sas_iounit_pg0);
6198	return port_count;
6199	}
6200
6201	enum hba_port_matched_codes {
6202	NOT_MATCHED = 0,
6203	MATCHED_WITH_ADDR_AND_PHYMASK,
6204	MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT,
6205	MATCHED_WITH_ADDR_AND_SUBPHYMASK,
6206	MATCHED_WITH_ADDR,
6207	};
6208
6209	/**
6210	* _scsih_look_and_get_matched_port_entry - Get matched hba port entry
6211	* from HBA port table
6212	* @ioc: per adapter object
6213	* @port_entry: hba port entry from temporary port table which needs to be
6214	* searched for matched entry in the HBA port table
6215	* @matched_port_entry: save matched hba port entry here
6216	* @count: count of matched entries
6217	*
6218	* return type of matched entry found.
6219	*/
6220	static enum hba_port_matched_codes
6221	_scsih_look_and_get_matched_port_entry(struct MPT3SAS_ADAPTER *ioc,
6222	struct hba_port *port_entry,
6223	struct hba_port **matched_port_entry, int *count)
6224	{
6225	struct hba_port *port_table_entry, *matched_port = NULL;
6226	enum hba_port_matched_codes matched_code = NOT_MATCHED;
6227	int lcount = 0;
6228	*matched_port_entry = NULL;
6229
6230	list_for_each_entry(port_table_entry, &ioc->port_table_list, list) {
6231	if (!(port_table_entry->flags & HBA_PORT_FLAG_DIRTY_PORT))
6232	continue;
6233
6234	if ((port_table_entry->sas_address == port_entry->sas_address)
6235	&& (port_table_entry->phy_mask == port_entry->phy_mask)) {
6236	matched_code = MATCHED_WITH_ADDR_AND_PHYMASK;
6237	matched_port = port_table_entry;
6238	break;
6239	}
6240
6241	if ((port_table_entry->sas_address == port_entry->sas_address)
6242	&& (port_table_entry->phy_mask & port_entry->phy_mask)
6243	&& (port_table_entry->port_id == port_entry->port_id)) {
6244	matched_code = MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT;
6245	matched_port = port_table_entry;
6246	continue;
6247	}
6248
6249	if ((port_table_entry->sas_address == port_entry->sas_address)
6250	&& (port_table_entry->phy_mask & port_entry->phy_mask)) {
6251	if (matched_code ==
6252	MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT)
6253	continue;
6254	matched_code = MATCHED_WITH_ADDR_AND_SUBPHYMASK;
6255	matched_port = port_table_entry;
6256	continue;
6257	}
6258
6259	if (port_table_entry->sas_address == port_entry->sas_address) {
6260	if (matched_code ==
6261	MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT)
6262	continue;
6263	if (matched_code == MATCHED_WITH_ADDR_AND_SUBPHYMASK)
6264	continue;
6265	matched_code = MATCHED_WITH_ADDR;
6266	matched_port = port_table_entry;
6267	lcount++;
6268	}
6269	}
6270
6271	*matched_port_entry = matched_port;
6272	if (matched_code == MATCHED_WITH_ADDR)
6273	*count = lcount;
6274	return matched_code;
6275	}
6276
6277	/**
6278	* _scsih_del_phy_part_of_anther_port - remove phy if it
6279	* is a part of anther port
6280	*@ioc: per adapter object
6281	*@port_table: port table after reset
6282	*@index: hba port entry index
6283	*@port_count: number of ports available after host reset
6284	*@offset: HBA phy bit offset
6285	*
6286	*/
6287	static void
6288	_scsih_del_phy_part_of_anther_port(struct MPT3SAS_ADAPTER *ioc,
6289	struct hba_port *port_table,
6290	int index, u8 port_count, int offset)
6291	{
6292	struct _sas_node *sas_node = &ioc->sas_hba;
6293	u32 i, found = 0;
6294
6295	for (i = 0; i < port_count; i++) {
6296	if (i == index)
6297	continue;
6298
6299	if (port_table[i].phy_mask & (1 << offset)) {
6300	mpt3sas_transport_del_phy_from_an_existing_port(
6301	ioc, sas_node, mpt3sas_phy: &sas_node->phy[offset]);
6302	found = 1;
6303	break;
6304	}
6305	}
6306	if (!found)
6307	port_table[index].phy_mask |= (1 << offset);
6308	}
6309
6310	/**
6311	* _scsih_add_or_del_phys_from_existing_port - add/remove phy to/from
6312	* right port
6313	*@ioc: per adapter object
6314	*@hba_port_entry: hba port table entry
6315	*@port_table: temporary port table
6316	*@index: hba port entry index
6317	*@port_count: number of ports available after host reset
6318	*
6319	*/
6320	static void
6321	_scsih_add_or_del_phys_from_existing_port(struct MPT3SAS_ADAPTER *ioc,
6322	struct hba_port *hba_port_entry, struct hba_port *port_table,
6323	int index, int port_count)
6324	{
6325	u32 phy_mask, offset = 0;
6326	struct _sas_node *sas_node = &ioc->sas_hba;
6327
6328	phy_mask = hba_port_entry->phy_mask ^ port_table[index].phy_mask;
6329
6330	for (offset = 0; offset < ioc->sas_hba.num_phys; offset++) {
6331	if (phy_mask & (1 << offset)) {
6332	if (!(port_table[index].phy_mask & (1 << offset))) {
6333	_scsih_del_phy_part_of_anther_port(
6334	ioc, port_table, index, port_count,
6335	offset);
6336	continue;
6337	}
6338	if (sas_node->phy[offset].phy_belongs_to_port)
6339	mpt3sas_transport_del_phy_from_an_existing_port(
6340	ioc, sas_node, mpt3sas_phy: &sas_node->phy[offset]);
6341	mpt3sas_transport_add_phy_to_an_existing_port(
6342	ioc, sas_node, mpt3sas_phy: &sas_node->phy[offset],
6343	sas_address: hba_port_entry->sas_address,
6344	port: hba_port_entry);
6345	}
6346	}
6347	}
6348
6349	/**
6350	* _scsih_del_dirty_vphy - delete virtual_phy objects marked as dirty.
6351	* @ioc: per adapter object
6352	*
6353	* Returns nothing.
6354	*/
6355	static void
6356	_scsih_del_dirty_vphy(struct MPT3SAS_ADAPTER *ioc)
6357	{
6358	struct hba_port *port, *port_next;
6359	struct virtual_phy *vphy, *vphy_next;
6360
6361	list_for_each_entry_safe(port, port_next,
6362	&ioc->port_table_list, list) {
6363	if (!port->vphys_mask)
6364	continue;
6365	list_for_each_entry_safe(vphy, vphy_next,
6366	&port->vphys_list, list) {
6367	if (vphy->flags & MPT_VPHY_FLAG_DIRTY_PHY) {
6368	drsprintk(ioc, ioc_info(ioc,
6369	"Deleting vphy %p entry from port id: %d\t, Phy_mask 0x%08x\n",
6370	vphy, port->port_id,
6371	vphy->phy_mask));
6372	port->vphys_mask &= ~vphy->phy_mask;
6373	list_del(entry: &vphy->list);
6374	kfree(objp: vphy);
6375	}
6376	}
6377	if (!port->vphys_mask && !port->sas_address)
6378	port->flags |= HBA_PORT_FLAG_DIRTY_PORT;
6379	}
6380	}
6381
6382	/**
6383	* _scsih_del_dirty_port_entries - delete dirty port entries from port list
6384	* after host reset
6385	*@ioc: per adapter object
6386	*
6387	*/
6388	static void
6389	_scsih_del_dirty_port_entries(struct MPT3SAS_ADAPTER *ioc)
6390	{
6391	struct hba_port *port, *port_next;
6392
6393	list_for_each_entry_safe(port, port_next,
6394	&ioc->port_table_list, list) {
6395	if (!(port->flags & HBA_PORT_FLAG_DIRTY_PORT) ||
6396	port->flags & HBA_PORT_FLAG_NEW_PORT)
6397	continue;
6398
6399	drsprintk(ioc, ioc_info(ioc,
6400	"Deleting port table entry %p having Port: %d\t Phy_mask 0x%08x\n",
6401	port, port->port_id, port->phy_mask));
6402	list_del(entry: &port->list);
6403	kfree(objp: port);
6404	}
6405	}
6406
6407	/**
6408	* _scsih_sas_port_refresh - Update HBA port table after host reset
6409	* @ioc: per adapter object
6410	*/
6411	static void
6412	_scsih_sas_port_refresh(struct MPT3SAS_ADAPTER *ioc)
6413	{
6414	u32 port_count = 0;
6415	struct hba_port *port_table;
6416	struct hba_port *port_table_entry;
6417	struct hba_port *port_entry = NULL;
6418	int i, j, count = 0, lcount = 0;
6419	int ret;
6420	u64 sas_addr;
6421	u8 num_phys;
6422
6423	drsprintk(ioc, ioc_info(ioc,
6424	"updating ports for sas_host(0x%016llx)\n",
6425	(unsigned long long)ioc->sas_hba.sas_address));
6426
6427	mpt3sas_config_get_number_hba_phys(ioc, num_phys: &num_phys);
6428	if (!num_phys) {
6429	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6430	__FILE__, __LINE__, __func__);
6431	return;
6432	}
6433
6434	if (num_phys > ioc->sas_hba.nr_phys_allocated) {
6435	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6436	__FILE__, __LINE__, __func__);
6437	return;
6438	}
6439	ioc->sas_hba.num_phys = num_phys;
6440
6441	port_table = kcalloc(n: ioc->sas_hba.num_phys,
6442	size: sizeof(struct hba_port), GFP_KERNEL);
6443	if (!port_table)
6444	return;
6445
6446	port_count = _scsih_get_port_table_after_reset(ioc, port_table);
6447	if (!port_count)
6448	return;
6449
6450	drsprintk(ioc, ioc_info(ioc, "New Port table\n"));
6451	for (j = 0; j < port_count; j++)
6452	drsprintk(ioc, ioc_info(ioc,
6453	"Port: %d\t Phy_mask 0x%08x\t sas_addr(0x%016llx)\n",
6454	port_table[j].port_id,
6455	port_table[j].phy_mask, port_table[j].sas_address));
6456
6457	list_for_each_entry(port_table_entry, &ioc->port_table_list, list)
6458	port_table_entry->flags |= HBA_PORT_FLAG_DIRTY_PORT;
6459
6460	drsprintk(ioc, ioc_info(ioc, "Old Port table\n"));
6461	port_table_entry = NULL;
6462	list_for_each_entry(port_table_entry, &ioc->port_table_list, list) {
6463	drsprintk(ioc, ioc_info(ioc,
6464	"Port: %d\t Phy_mask 0x%08x\t sas_addr(0x%016llx)\n",
6465	port_table_entry->port_id,
6466	port_table_entry->phy_mask,
6467	port_table_entry->sas_address));
6468	}
6469
6470	for (j = 0; j < port_count; j++) {
6471	ret = _scsih_look_and_get_matched_port_entry(ioc,
6472	port_entry: &port_table[j], matched_port_entry: &port_entry, count: &count);
6473	if (!port_entry) {
6474	drsprintk(ioc, ioc_info(ioc,
6475	"No Matched entry for sas_addr(0x%16llx), Port:%d\n",
6476	port_table[j].sas_address,
6477	port_table[j].port_id));
6478	continue;
6479	}
6480
6481	switch (ret) {
6482	case MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT:
6483	case MATCHED_WITH_ADDR_AND_SUBPHYMASK:
6484	_scsih_add_or_del_phys_from_existing_port(ioc,
6485	hba_port_entry: port_entry, port_table, index: j, port_count);
6486	break;
6487	case MATCHED_WITH_ADDR:
6488	sas_addr = port_table[j].sas_address;
6489	for (i = 0; i < port_count; i++) {
6490	if (port_table[i].sas_address == sas_addr)
6491	lcount++;
6492	}
6493
6494	if (count > 1 || lcount > 1)
6495	port_entry = NULL;
6496	else
6497	_scsih_add_or_del_phys_from_existing_port(ioc,
6498	hba_port_entry: port_entry, port_table, index: j, port_count);
6499	}
6500
6501	if (!port_entry)
6502	continue;
6503
6504	if (port_entry->port_id != port_table[j].port_id)
6505	port_entry->port_id = port_table[j].port_id;
6506	port_entry->flags &= ~HBA_PORT_FLAG_DIRTY_PORT;
6507	port_entry->phy_mask = port_table[j].phy_mask;
6508	}
6509
6510	port_table_entry = NULL;
6511	}
6512
6513	/**
6514	* _scsih_alloc_vphy - allocate virtual_phy object
6515	* @ioc: per adapter object
6516	* @port_id: Port ID number
6517	* @phy_num: HBA Phy number
6518	*
6519	* Returns allocated virtual_phy object.
6520	*/
6521	static struct virtual_phy *
6522	_scsih_alloc_vphy(struct MPT3SAS_ADAPTER *ioc, u8 port_id, u8 phy_num)
6523	{
6524	struct virtual_phy *vphy;
6525	struct hba_port *port;
6526
6527	port = mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0);
6528	if (!port)
6529	return NULL;
6530
6531	vphy = mpt3sas_get_vphy_by_phy(ioc, port, phy: phy_num);
6532	if (!vphy) {
6533	vphy = kzalloc(size: sizeof(struct virtual_phy), GFP_KERNEL);
6534	if (!vphy)
6535	return NULL;
6536
6537	if (!port->vphys_mask)
6538	INIT_LIST_HEAD(list: &port->vphys_list);
6539
6540	/*
6541	* Enable bit corresponding to HBA phy number on its
6542	* parent hba_port object's vphys_mask field.
6543	*/
6544	port->vphys_mask |= (1 << phy_num);
6545	vphy->phy_mask |= (1 << phy_num);
6546
6547	list_add_tail(new: &vphy->list, head: &port->vphys_list);
6548
6549	ioc_info(ioc,
6550	"vphy entry: %p, port id: %d, phy:%d is added to port's vphys_list\n",
6551	vphy, port->port_id, phy_num);
6552	}
6553	return vphy;
6554	}
6555
6556	/**
6557	* _scsih_sas_host_refresh - refreshing sas host object contents
6558	* @ioc: per adapter object
6559	* Context: user
6560	*
6561	* During port enable, fw will send topology events for every device. Its
6562	* possible that the handles may change from the previous setting, so this
6563	* code keeping handles updating if changed.
6564	*/
6565	static void
6566	_scsih_sas_host_refresh(struct MPT3SAS_ADAPTER *ioc)
6567	{
6568	u16 sz;
6569	u16 ioc_status;
6570	int i;
6571	Mpi2ConfigReply_t mpi_reply;
6572	Mpi2SasIOUnitPage0_t *sas_iounit_pg0 = NULL;
6573	u16 attached_handle;
6574	u8 link_rate, port_id;
6575	struct hba_port *port;
6576	Mpi2SasPhyPage0_t phy_pg0;
6577
6578	dtmprintk(ioc,
6579	ioc_info(ioc, "updating handles for sas_host(0x%016llx)\n",
6580	(u64)ioc->sas_hba.sas_address));
6581
6582	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys
6583	* sizeof(Mpi2SasIOUnit0PhyData_t));
6584	sas_iounit_pg0 = kzalloc(size: sz, GFP_KERNEL);
6585	if (!sas_iounit_pg0) {
6586	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6587	__FILE__, __LINE__, __func__);
6588	return;
6589	}
6590
6591	if ((mpt3sas_config_get_sas_iounit_pg0(ioc, mpi_reply: &mpi_reply,
6592	config_page: sas_iounit_pg0, sz)) != 0)
6593	goto out;
6594	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
6595	if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
6596	goto out;
6597	for (i = 0; i < ioc->sas_hba.num_phys ; i++) {
6598	link_rate = sas_iounit_pg0->PhyData[i].NegotiatedLinkRate >> 4;
6599	if (i == 0)
6600	ioc->sas_hba.handle = le16_to_cpu(
6601	sas_iounit_pg0->PhyData[0].ControllerDevHandle);
6602	port_id = sas_iounit_pg0->PhyData[i].Port;
6603	if (!(mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0))) {
6604	port = kzalloc(size: sizeof(struct hba_port), GFP_KERNEL);
6605	if (!port)
6606	goto out;
6607
6608	port->port_id = port_id;
6609	ioc_info(ioc,
6610	"hba_port entry: %p, port: %d is added to hba_port list\n",
6611	port, port->port_id);
6612	if (ioc->shost_recovery)
6613	port->flags = HBA_PORT_FLAG_NEW_PORT;
6614	list_add_tail(new: &port->list, head: &ioc->port_table_list);
6615	}
6616	/*
6617	* Check whether current Phy belongs to HBA vSES device or not.
6618	*/
6619	if (le32_to_cpu(sas_iounit_pg0->PhyData[i].ControllerPhyDeviceInfo) &
6620	MPI2_SAS_DEVICE_INFO_SEP &&
6621	(link_rate >= MPI2_SAS_NEG_LINK_RATE_1_5)) {
6622	if ((mpt3sas_config_get_phy_pg0(ioc, mpi_reply: &mpi_reply,
6623	config_page: &phy_pg0, phy_number: i))) {
6624	ioc_err(ioc,
6625	"failure at %s:%d/%s()!\n",
6626	__FILE__, __LINE__, __func__);
6627	goto out;
6628	}
6629	if (!(le32_to_cpu(phy_pg0.PhyInfo) &
6630	MPI2_SAS_PHYINFO_VIRTUAL_PHY))
6631	continue;
6632	/*
6633	* Allocate a virtual_phy object for vSES device, if
6634	* this vSES device is hot added.
6635	*/
6636	if (!_scsih_alloc_vphy(ioc, port_id, phy_num: i))
6637	goto out;
6638	ioc->sas_hba.phy[i].hba_vphy = 1;
6639	}
6640
6641	/*
6642	* Add new HBA phys to STL if these new phys got added as part
6643	* of HBA Firmware upgrade/downgrade operation.
6644	*/
6645	if (!ioc->sas_hba.phy[i].phy) {
6646	if ((mpt3sas_config_get_phy_pg0(ioc, mpi_reply: &mpi_reply,
6647	config_page: &phy_pg0, phy_number: i))) {
6648	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6649	__FILE__, __LINE__, __func__);
6650	continue;
6651	}
6652	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6653	MPI2_IOCSTATUS_MASK;
6654	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6655	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6656	__FILE__, __LINE__, __func__);
6657	continue;
6658	}
6659	ioc->sas_hba.phy[i].phy_id = i;
6660	mpt3sas_transport_add_host_phy(ioc,
6661	mpt3sas_phy: &ioc->sas_hba.phy[i], phy_pg0,
6662	parent_dev: ioc->sas_hba.parent_dev);
6663	continue;
6664	}
6665	ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;
6666	attached_handle = le16_to_cpu(sas_iounit_pg0->PhyData[i].
6667	AttachedDevHandle);
6668	if (attached_handle && link_rate < MPI2_SAS_NEG_LINK_RATE_1_5)
6669	link_rate = MPI2_SAS_NEG_LINK_RATE_1_5;
6670	ioc->sas_hba.phy[i].port =
6671	mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0);
6672	mpt3sas_transport_update_links(ioc, sas_address: ioc->sas_hba.sas_address,
6673	handle: attached_handle, phy_number: i, link_rate,
6674	port: ioc->sas_hba.phy[i].port);
6675	}
6676	/*
6677	* Clear the phy details if this phy got disabled as part of
6678	* HBA Firmware upgrade/downgrade operation.
6679	*/
6680	for (i = ioc->sas_hba.num_phys;
6681	i < ioc->sas_hba.nr_phys_allocated; i++) {
6682	if (ioc->sas_hba.phy[i].phy &&
6683	ioc->sas_hba.phy[i].phy->negotiated_linkrate >=
6684	SAS_LINK_RATE_1_5_GBPS)
6685	mpt3sas_transport_update_links(ioc,
6686	sas_address: ioc->sas_hba.sas_address, handle: 0, phy_number: i,
6687	MPI2_SAS_NEG_LINK_RATE_PHY_DISABLED, NULL);
6688	}
6689	out:
6690	kfree(objp: sas_iounit_pg0);
6691	}
6692
6693	/**
6694	* _scsih_sas_host_add - create sas host object
6695	* @ioc: per adapter object
6696	*
6697	* Creating host side data object, stored in ioc->sas_hba
6698	*/
6699	static void
6700	_scsih_sas_host_add(struct MPT3SAS_ADAPTER *ioc)
6701	{
6702	int i;
6703	Mpi2ConfigReply_t mpi_reply;
6704	Mpi2SasIOUnitPage0_t *sas_iounit_pg0 = NULL;
6705	Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL;
6706	Mpi2SasPhyPage0_t phy_pg0;
6707	Mpi2SasDevicePage0_t sas_device_pg0;
6708	Mpi2SasEnclosurePage0_t enclosure_pg0;
6709	u16 ioc_status;
6710	u16 sz;
6711	u8 device_missing_delay;
6712	u8 num_phys, port_id;
6713	struct hba_port *port;
6714
6715	mpt3sas_config_get_number_hba_phys(ioc, num_phys: &num_phys);
6716	if (!num_phys) {
6717	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6718	__FILE__, __LINE__, __func__);
6719	return;
6720	}
6721
6722	ioc->sas_hba.nr_phys_allocated = max_t(u8,
6723	MPT_MAX_HBA_NUM_PHYS, num_phys);
6724	ioc->sas_hba.phy = kcalloc(n: ioc->sas_hba.nr_phys_allocated,
6725	size: sizeof(struct _sas_phy), GFP_KERNEL);
6726	if (!ioc->sas_hba.phy) {
6727	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6728	__FILE__, __LINE__, __func__);
6729	goto out;
6730	}
6731	ioc->sas_hba.num_phys = num_phys;
6732
6733	/* sas_iounit page 0 */
6734	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys *
6735	sizeof(Mpi2SasIOUnit0PhyData_t));
6736	sas_iounit_pg0 = kzalloc(size: sz, GFP_KERNEL);
6737	if (!sas_iounit_pg0) {
6738	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6739	__FILE__, __LINE__, __func__);
6740	return;
6741	}
6742	if ((mpt3sas_config_get_sas_iounit_pg0(ioc, mpi_reply: &mpi_reply,
6743	config_page: sas_iounit_pg0, sz))) {
6744	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6745	__FILE__, __LINE__, __func__);
6746	goto out;
6747	}
6748	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6749	MPI2_IOCSTATUS_MASK;
6750	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6751	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6752	__FILE__, __LINE__, __func__);
6753	goto out;
6754	}
6755
6756	/* sas_iounit page 1 */
6757	sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (ioc->sas_hba.num_phys *
6758	sizeof(Mpi2SasIOUnit1PhyData_t));
6759	sas_iounit_pg1 = kzalloc(size: sz, GFP_KERNEL);
6760	if (!sas_iounit_pg1) {
6761	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6762	__FILE__, __LINE__, __func__);
6763	goto out;
6764	}
6765	if ((mpt3sas_config_get_sas_iounit_pg1(ioc, mpi_reply: &mpi_reply,
6766	config_page: sas_iounit_pg1, sz))) {
6767	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6768	__FILE__, __LINE__, __func__);
6769	goto out;
6770	}
6771	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6772	MPI2_IOCSTATUS_MASK;
6773	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6774	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6775	__FILE__, __LINE__, __func__);
6776	goto out;
6777	}
6778
6779	ioc->io_missing_delay =
6780	sas_iounit_pg1->IODeviceMissingDelay;
6781	device_missing_delay =
6782	sas_iounit_pg1->ReportDeviceMissingDelay;
6783	if (device_missing_delay & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
6784	ioc->device_missing_delay = (device_missing_delay &
6785	MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
6786	else
6787	ioc->device_missing_delay = device_missing_delay &
6788	MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
6789
6790	ioc->sas_hba.parent_dev = &ioc->shost->shost_gendev;
6791	for (i = 0; i < ioc->sas_hba.num_phys ; i++) {
6792	if ((mpt3sas_config_get_phy_pg0(ioc, mpi_reply: &mpi_reply, config_page: &phy_pg0,
6793	phy_number: i))) {
6794	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6795	__FILE__, __LINE__, __func__);
6796	goto out;
6797	}
6798	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6799	MPI2_IOCSTATUS_MASK;
6800	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6801	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6802	__FILE__, __LINE__, __func__);
6803	goto out;
6804	}
6805
6806	if (i == 0)
6807	ioc->sas_hba.handle = le16_to_cpu(sas_iounit_pg0->
6808	PhyData[0].ControllerDevHandle);
6809
6810	port_id = sas_iounit_pg0->PhyData[i].Port;
6811	if (!(mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0))) {
6812	port = kzalloc(size: sizeof(struct hba_port), GFP_KERNEL);
6813	if (!port)
6814	goto out;
6815
6816	port->port_id = port_id;
6817	ioc_info(ioc,
6818	"hba_port entry: %p, port: %d is added to hba_port list\n",
6819	port, port->port_id);
6820	list_add_tail(new: &port->list,
6821	head: &ioc->port_table_list);
6822	}
6823
6824	/*
6825	* Check whether current Phy belongs to HBA vSES device or not.
6826	*/
6827	if ((le32_to_cpu(phy_pg0.PhyInfo) &
6828	MPI2_SAS_PHYINFO_VIRTUAL_PHY) &&
6829	(phy_pg0.NegotiatedLinkRate >> 4) >=
6830	MPI2_SAS_NEG_LINK_RATE_1_5) {
6831	/*
6832	* Allocate a virtual_phy object for vSES device.
6833	*/
6834	if (!_scsih_alloc_vphy(ioc, port_id, phy_num: i))
6835	goto out;
6836	ioc->sas_hba.phy[i].hba_vphy = 1;
6837	}
6838
6839	ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;
6840	ioc->sas_hba.phy[i].phy_id = i;
6841	ioc->sas_hba.phy[i].port =
6842	mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0);
6843	mpt3sas_transport_add_host_phy(ioc, mpt3sas_phy: &ioc->sas_hba.phy[i],
6844	phy_pg0, parent_dev: ioc->sas_hba.parent_dev);
6845	}
6846	if ((mpt3sas_config_get_sas_device_pg0(ioc, mpi_reply: &mpi_reply, config_page: &sas_device_pg0,
6847	MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle: ioc->sas_hba.handle))) {
6848	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6849	__FILE__, __LINE__, __func__);
6850	goto out;
6851	}
6852	ioc->sas_hba.enclosure_handle =
6853	le16_to_cpu(sas_device_pg0.EnclosureHandle);
6854	ioc->sas_hba.sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
6855	ioc_info(ioc, "host_add: handle(0x%04x), sas_addr(0x%016llx), phys(%d)\n",
6856	ioc->sas_hba.handle,
6857	(u64)ioc->sas_hba.sas_address,
6858	ioc->sas_hba.num_phys);
6859
6860	if (ioc->sas_hba.enclosure_handle) {
6861	if (!(mpt3sas_config_get_enclosure_pg0(ioc, mpi_reply: &mpi_reply,
6862	config_page: &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
6863	handle: ioc->sas_hba.enclosure_handle)))
6864	ioc->sas_hba.enclosure_logical_id =
6865	le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
6866	}
6867
6868	out:
6869	kfree(objp: sas_iounit_pg1);
6870	kfree(objp: sas_iounit_pg0);
6871	}
6872
6873	/**
6874	* _scsih_expander_add - creating expander object
6875	* @ioc: per adapter object
6876	* @handle: expander handle
6877	*
6878	* Creating expander object, stored in ioc->sas_expander_list.
6879	*
6880	* Return: 0 for success, else error.
6881	*/
6882	static int
6883	_scsih_expander_add(struct MPT3SAS_ADAPTER *ioc, u16 handle)
6884	{
6885	struct _sas_node *sas_expander;
6886	struct _enclosure_node *enclosure_dev;
6887	Mpi2ConfigReply_t mpi_reply;
6888	Mpi2ExpanderPage0_t expander_pg0;
6889	Mpi2ExpanderPage1_t expander_pg1;
6890	u32 ioc_status;
6891	u16 parent_handle;
6892	u64 sas_address, sas_address_parent = 0;
6893	int i;
6894	unsigned long flags;
6895	struct _sas_port *mpt3sas_port = NULL;
6896	u8 port_id;
6897
6898	int rc = 0;
6899
6900	if (!handle)
6901	return -1;
6902
6903	if (ioc->shost_recovery || ioc->pci_error_recovery)
6904	return -1;
6905
6906	if ((mpt3sas_config_get_expander_pg0(ioc, mpi_reply: &mpi_reply, config_page: &expander_pg0,
6907	MPI2_SAS_EXPAND_PGAD_FORM_HNDL, handle))) {
6908	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6909	__FILE__, __LINE__, __func__);
6910	return -1;
6911	}
6912
6913	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6914	MPI2_IOCSTATUS_MASK;
6915	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6916	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6917	__FILE__, __LINE__, __func__);
6918	return -1;
6919	}
6920
6921	/* handle out of order topology events */
6922	parent_handle = le16_to_cpu(expander_pg0.ParentDevHandle);
6923	if (_scsih_get_sas_address(ioc, handle: parent_handle, sas_address: &sas_address_parent)
6924	!= 0) {
6925	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6926	__FILE__, __LINE__, __func__);
6927	return -1;
6928	}
6929
6930	port_id = expander_pg0.PhysicalPort;
6931	if (sas_address_parent != ioc->sas_hba.sas_address) {
6932	spin_lock_irqsave(&ioc->sas_node_lock, flags);
6933	sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,
6934	sas_address: sas_address_parent,
6935	port: mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0));
6936	spin_unlock_irqrestore(lock: &ioc->sas_node_lock, flags);
6937	if (!sas_expander) {
6938	rc = _scsih_expander_add(ioc, handle: parent_handle);
6939	if (rc != 0)
6940	return rc;
6941	}
6942	}
6943
6944	spin_lock_irqsave(&ioc->sas_node_lock, flags);
6945	sas_address = le64_to_cpu(expander_pg0.SASAddress);
6946	sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,
6947	sas_address, port: mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0));
6948	spin_unlock_irqrestore(lock: &ioc->sas_node_lock, flags);
6949
6950	if (sas_expander)
6951	return 0;
6952
6953	sas_expander = kzalloc(size: sizeof(struct _sas_node),
6954	GFP_KERNEL);
6955	if (!sas_expander) {
6956	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6957	__FILE__, __LINE__, __func__);
6958	return -1;
6959	}
6960
6961	sas_expander->handle = handle;
6962	sas_expander->num_phys = expander_pg0.NumPhys;
6963	sas_expander->sas_address_parent = sas_address_parent;
6964	sas_expander->sas_address = sas_address;
6965	sas_expander->port = mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0);
6966	if (!sas_expander->port) {
6967	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6968	__FILE__, __LINE__, __func__);
6969	rc = -1;
6970	goto out_fail;
6971	}
6972
6973	ioc_info(ioc, "expander_add: handle(0x%04x), parent(0x%04x), sas_addr(0x%016llx), phys(%d)\n",
6974	handle, parent_handle,
6975	(u64)sas_expander->sas_address, sas_expander->num_phys);
6976
6977	if (!sas_expander->num_phys) {
6978	rc = -1;
6979	goto out_fail;
6980	}
6981	sas_expander->phy = kcalloc(n: sas_expander->num_phys,
6982	size: sizeof(struct _sas_phy), GFP_KERNEL);
6983	if (!sas_expander->phy) {
6984	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6985	__FILE__, __LINE__, __func__);
6986	rc = -1;
6987	goto out_fail;
6988	}
6989
6990	INIT_LIST_HEAD(list: &sas_expander->sas_port_list);
6991	mpt3sas_port = mpt3sas_transport_port_add(ioc, handle,
6992	sas_address: sas_address_parent, port: sas_expander->port);
6993	if (!mpt3sas_port) {
6994	ioc_err(ioc, "failure at %s:%d/%s()!\n",
6995	__FILE__, __LINE__, __func__);
6996	rc = -1;
6997	goto out_fail;
6998	}
6999	sas_expander->parent_dev = &mpt3sas_port->rphy->dev;
7000	sas_expander->rphy = mpt3sas_port->rphy;
7001
7002	for (i = 0 ; i < sas_expander->num_phys ; i++) {
7003	if ((mpt3sas_config_get_expander_pg1(ioc, mpi_reply: &mpi_reply,
7004	config_page: &expander_pg1, phy_number: i, handle))) {
7005	ioc_err(ioc, "failure at %s:%d/%s()!\n",
7006	__FILE__, __LINE__, __func__);
7007	rc = -1;
7008	goto out_fail;
7009	}
7010	sas_expander->phy[i].handle = handle;
7011	sas_expander->phy[i].phy_id = i;
7012	sas_expander->phy[i].port =
7013	mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0);
7014
7015	if ((mpt3sas_transport_add_expander_phy(ioc,
7016	mpt3sas_phy: &sas_expander->phy[i], expander_pg1,
7017	parent_dev: sas_expander->parent_dev))) {
7018	ioc_err(ioc, "failure at %s:%d/%s()!\n",
7019	__FILE__, __LINE__, __func__);
7020	rc = -1;
7021	goto out_fail;
7022	}
7023	}
7024
7025	if (sas_expander->enclosure_handle) {
7026	enclosure_dev =
7027	mpt3sas_scsih_enclosure_find_by_handle(ioc,
7028	handle: sas_expander->enclosure_handle);
7029	if (enclosure_dev)
7030	sas_expander->enclosure_logical_id =
7031	le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
7032	}
7033
7034	_scsih_expander_node_add(ioc, sas_expander);
7035	return 0;
7036
7037	out_fail:
7038
7039	if (mpt3sas_port)
7040	mpt3sas_transport_port_remove(ioc, sas_address: sas_expander->sas_address,
7041	sas_address_parent, port: sas_expander->port);
7042	kfree(objp: sas_expander);
7043	return rc;
7044	}
7045
7046	/**
7047	* mpt3sas_expander_remove - removing expander object
7048	* @ioc: per adapter object
7049	* @sas_address: expander sas_address
7050	* @port: hba port entry
7051	*/
7052	void
7053	mpt3sas_expander_remove(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
7054	struct hba_port *port)
7055	{
7056	struct _sas_node *sas_expander;
7057	unsigned long flags;
7058
7059	if (ioc->shost_recovery)
7060	return;
7061
7062	if (!port)
7063	return;
7064
7065	spin_lock_irqsave(&ioc->sas_node_lock, flags);
7066	sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,
7067	sas_address, port);
7068	spin_unlock_irqrestore(lock: &ioc->sas_node_lock, flags);
7069	if (sas_expander)
7070	_scsih_expander_node_remove(ioc, sas_expander);
7071	}
7072
7073	/**
7074	* _scsih_done - internal SCSI_IO callback handler.
7075	* @ioc: per adapter object
7076	* @smid: system request message index
7077	* @msix_index: MSIX table index supplied by the OS
7078	* @reply: reply message frame(lower 32bit addr)
7079	*
7080	* Callback handler when sending internal generated SCSI_IO.
7081	* The callback index passed is `ioc->scsih_cb_idx`
7082	*
7083	* Return: 1 meaning mf should be freed from _base_interrupt
7084	* 0 means the mf is freed from this function.
7085	*/
7086	static u8
7087	_scsih_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
7088	{
7089	MPI2DefaultReply_t *mpi_reply;
7090
7091	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, phys_addr: reply);
7092	if (ioc->scsih_cmds.status == MPT3_CMD_NOT_USED)
7093	return 1;
7094	if (ioc->scsih_cmds.smid != smid)
7095	return 1;
7096	ioc->scsih_cmds.status |= MPT3_CMD_COMPLETE;
7097	if (mpi_reply) {
7098	memcpy(ioc->scsih_cmds.reply, mpi_reply,
7099	mpi_reply->MsgLength*4);
7100	ioc->scsih_cmds.status |= MPT3_CMD_REPLY_VALID;
7101	}
7102	ioc->scsih_cmds.status &= ~MPT3_CMD_PENDING;
7103	complete(&ioc->scsih_cmds.done);
7104	return 1;
7105	}
7106
7107
7108
7109
7110	#define MPT3_MAX_LUNS (255)
7111
7112
7113	/**
7114	* _scsih_check_access_status - check access flags
7115	* @ioc: per adapter object
7116	* @sas_address: sas address
7117	* @handle: sas device handle
7118	* @access_status: errors returned during discovery of the device
7119	*
7120	* Return: 0 for success, else failure
7121	*/
7122	static u8
7123	_scsih_check_access_status(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
7124	u16 handle, u8 access_status)
7125	{
7126	u8 rc = 1;
7127	char *desc = NULL;
7128
7129	switch (access_status) {
7130	case MPI2_SAS_DEVICE0_ASTATUS_NO_ERRORS:
7131	case MPI2_SAS_DEVICE0_ASTATUS_SATA_NEEDS_INITIALIZATION:
7132	rc = 0;
7133	break;
7134	case MPI2_SAS_DEVICE0_ASTATUS_SATA_CAPABILITY_FAILED:
7135	desc = "sata capability failed";
7136	break;
7137	case MPI2_SAS_DEVICE0_ASTATUS_SATA_AFFILIATION_CONFLICT:
7138	desc = "sata affiliation conflict";
7139	break;
7140	case MPI2_SAS_DEVICE0_ASTATUS_ROUTE_NOT_ADDRESSABLE:
7141	desc = "route not addressable";
7142	break;
7143	case MPI2_SAS_DEVICE0_ASTATUS_SMP_ERROR_NOT_ADDRESSABLE:
7144	desc = "smp error not addressable";
7145	break;
7146	case MPI2_SAS_DEVICE0_ASTATUS_DEVICE_BLOCKED:
7147	desc = "device blocked";
7148	break;
7149	case MPI2_SAS_DEVICE0_ASTATUS_SATA_INIT_FAILED:
7150	case MPI2_SAS_DEVICE0_ASTATUS_SIF_UNKNOWN:
7151	case MPI2_SAS_DEVICE0_ASTATUS_SIF_AFFILIATION_CONFLICT:
7152	case MPI2_SAS_DEVICE0_ASTATUS_SIF_DIAG:
7153	case MPI2_SAS_DEVICE0_ASTATUS_SIF_IDENTIFICATION:
7154	case MPI2_SAS_DEVICE0_ASTATUS_SIF_CHECK_POWER:
7155	case MPI2_SAS_DEVICE0_ASTATUS_SIF_PIO_SN:
7156	case MPI2_SAS_DEVICE0_ASTATUS_SIF_MDMA_SN:
7157	case MPI2_SAS_DEVICE0_ASTATUS_SIF_UDMA_SN:
7158	case MPI2_SAS_DEVICE0_ASTATUS_SIF_ZONING_VIOLATION:
7159	case MPI2_SAS_DEVICE0_ASTATUS_SIF_NOT_ADDRESSABLE:
7160	case MPI2_SAS_DEVICE0_ASTATUS_SIF_MAX:
7161	desc = "sata initialization failed";
7162	break;
7163	default:
7164	desc = "unknown";
7165	break;
7166	}
7167
7168	if (!rc)
7169	return 0;
7170
7171	ioc_err(ioc, "discovery errors(%s): sas_address(0x%016llx), handle(0x%04x)\n",
7172	desc, (u64)sas_address, handle);
7173	return rc;
7174	}
7175
7176	/**
7177	* _scsih_check_device - checking device responsiveness
7178	* @ioc: per adapter object
7179	* @parent_sas_address: sas address of parent expander or sas host
7180	* @handle: attached device handle
7181	* @phy_number: phy number
7182	* @link_rate: new link rate
7183	*/
7184	static void
7185	_scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
7186	u64 parent_sas_address, u16 handle, u8 phy_number, u8 link_rate)
7187	{
7188	Mpi2ConfigReply_t mpi_reply;
7189	Mpi2SasDevicePage0_t sas_device_pg0;
7190	struct _sas_device *sas_device = NULL;
7191	struct _enclosure_node *enclosure_dev = NULL;
7192	u32 ioc_status;
7193	unsigned long flags;
7194	u64 sas_address;
7195	struct scsi_target *starget;
7196	struct MPT3SAS_TARGET *sas_target_priv_data;
7197	u32 device_info;
7198	struct hba_port *port;
7199
7200	if ((mpt3sas_config_get_sas_device_pg0(ioc, mpi_reply: &mpi_reply, config_page: &sas_device_pg0,
7201	MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle)))
7202	return;
7203
7204	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
7205	if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
7206	return;
7207
7208	/* wide port handling ~ we need only handle device once for the phy that
7209	* is matched in sas device page zero
7210	*/
7211	if (phy_number != sas_device_pg0.PhyNum)
7212	return;
7213
7214	/* check if this is end device */
7215	device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
7216	if (!(_scsih_is_end_device(device_info)))
7217	return;
7218
7219	spin_lock_irqsave(&ioc->sas_device_lock, flags);
7220	sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
7221	port = mpt3sas_get_port_by_id(ioc, port_id: sas_device_pg0.PhysicalPort, bypass_dirty_port_flag: 0);
7222	if (!port)
7223	goto out_unlock;
7224	sas_device = __mpt3sas_get_sdev_by_addr(ioc,
7225	sas_address, port);
7226
7227	if (!sas_device)
7228	goto out_unlock;
7229
7230	if (unlikely(sas_device->handle != handle)) {
7231	starget = sas_device->starget;
7232	sas_target_priv_data = starget->hostdata;
7233	starget_printk(KERN_INFO, starget,
7234	"handle changed from(0x%04x) to (0x%04x)!!!\n",
7235	sas_device->handle, handle);
7236	sas_target_priv_data->handle = handle;
7237	sas_device->handle = handle;
7238	if (le16_to_cpu(sas_device_pg0.Flags) &
7239	MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
7240	sas_device->enclosure_level =
7241	sas_device_pg0.EnclosureLevel;
7242	memcpy(sas_device->connector_name,
7243	sas_device_pg0.ConnectorName, 4);
7244	sas_device->connector_name[4] = '\0';
7245	} else {
7246	sas_device->enclosure_level = 0;
7247	sas_device->connector_name[0] = '\0';
7248	}
7249
7250	sas_device->enclosure_handle =
7251	le16_to_cpu(sas_device_pg0.EnclosureHandle);
7252	sas_device->is_chassis_slot_valid = 0;
7253	enclosure_dev = mpt3sas_scsih_enclosure_find_by_handle(ioc,
7254	handle: sas_device->enclosure_handle);
7255	if (enclosure_dev) {
7256	sas_device->enclosure_logical_id =
7257	le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
7258	if (le16_to_cpu(enclosure_dev->pg0.Flags) &
7259	MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
7260	sas_device->is_chassis_slot_valid = 1;
7261	sas_device->chassis_slot =
7262	enclosure_dev->pg0.ChassisSlot;
7263	}
7264	}
7265	}
7266
7267	/* check if device is present */
7268	if (!(le16_to_cpu(sas_device_pg0.Flags) &
7269	MPI2_SAS_DEVICE0_FLAGS_DEVICE_PRESENT)) {
7270	ioc_err(ioc, "device is not present handle(0x%04x), flags!!!\n",
7271	handle);
7272	goto out_unlock;
7273	}
7274
7275	/* check if there were any issues with discovery */
7276	if (_scsih_check_access_status(ioc, sas_address, handle,
7277	access_status: sas_device_pg0.AccessStatus))
7278	goto out_unlock;
7279
7280	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
7281	_scsih_ublock_io_device(ioc, sas_address, port);
7282
7283	if (sas_device)
7284	sas_device_put(s: sas_device);
7285	return;
7286
7287	out_unlock:
7288	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
7289	if (sas_device)
7290	sas_device_put(s: sas_device);
7291	}
7292
7293	/**
7294	* _scsih_add_device - creating sas device object
7295	* @ioc: per adapter object
7296	* @handle: sas device handle
7297	* @phy_num: phy number end device attached to
7298	* @is_pd: is this hidden raid component
7299	*
7300	* Creating end device object, stored in ioc->sas_device_list.
7301	*
7302	* Return: 0 for success, non-zero for failure.
7303	*/
7304	static int
7305	_scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
7306	u8 is_pd)
7307	{
7308	Mpi2ConfigReply_t mpi_reply;
7309	Mpi2SasDevicePage0_t sas_device_pg0;
7310	struct _sas_device *sas_device;
7311	struct _enclosure_node *enclosure_dev = NULL;
7312	u32 ioc_status;
7313	u64 sas_address;
7314	u32 device_info;
7315	u8 port_id;
7316
7317	if ((mpt3sas_config_get_sas_device_pg0(ioc, mpi_reply: &mpi_reply, config_page: &sas_device_pg0,
7318	MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
7319	ioc_err(ioc, "failure at %s:%d/%s()!\n",
7320	__FILE__, __LINE__, __func__);
7321	return -1;
7322	}
7323
7324	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
7325	MPI2_IOCSTATUS_MASK;
7326	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
7327	ioc_err(ioc, "failure at %s:%d/%s()!\n",
7328	__FILE__, __LINE__, __func__);
7329	return -1;
7330	}
7331
7332	/* check if this is end device */
7333	device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
7334	if (!(_scsih_is_end_device(device_info)))
7335	return -1;
7336	set_bit(nr: handle, addr: ioc->pend_os_device_add);
7337	sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
7338
7339	/* check if device is present */
7340	if (!(le16_to_cpu(sas_device_pg0.Flags) &
7341	MPI2_SAS_DEVICE0_FLAGS_DEVICE_PRESENT)) {
7342	ioc_err(ioc, "device is not present handle(0x04%x)!!!\n",
7343	handle);
7344	return -1;
7345	}
7346
7347	/* check if there were any issues with discovery */
7348	if (_scsih_check_access_status(ioc, sas_address, handle,
7349	access_status: sas_device_pg0.AccessStatus))
7350	return -1;
7351
7352	port_id = sas_device_pg0.PhysicalPort;
7353	sas_device = mpt3sas_get_sdev_by_addr(ioc,
7354	sas_address, port: mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0));
7355	if (sas_device) {
7356	clear_bit(nr: handle, addr: ioc->pend_os_device_add);
7357	sas_device_put(s: sas_device);
7358	return -1;
7359	}
7360
7361	if (sas_device_pg0.EnclosureHandle) {
7362	enclosure_dev =
7363	mpt3sas_scsih_enclosure_find_by_handle(ioc,
7364	le16_to_cpu(sas_device_pg0.EnclosureHandle));
7365	if (enclosure_dev == NULL)
7366	ioc_info(ioc, "Enclosure handle(0x%04x) doesn't match with enclosure device!\n",
7367	sas_device_pg0.EnclosureHandle);
7368	}
7369
7370	sas_device = kzalloc(size: sizeof(struct _sas_device),
7371	GFP_KERNEL);
7372	if (!sas_device) {
7373	ioc_err(ioc, "failure at %s:%d/%s()!\n",
7374	__FILE__, __LINE__, __func__);
7375	return 0;
7376	}
7377
7378	kref_init(kref: &sas_device->refcount);
7379	sas_device->handle = handle;
7380	if (_scsih_get_sas_address(ioc,
7381	le16_to_cpu(sas_device_pg0.ParentDevHandle),
7382	sas_address: &sas_device->sas_address_parent) != 0)
7383	ioc_err(ioc, "failure at %s:%d/%s()!\n",
7384	__FILE__, __LINE__, __func__);
7385	sas_device->enclosure_handle =
7386	le16_to_cpu(sas_device_pg0.EnclosureHandle);
7387	if (sas_device->enclosure_handle != 0)
7388	sas_device->slot =
7389	le16_to_cpu(sas_device_pg0.Slot);
7390	sas_device->device_info = device_info;
7391	sas_device->sas_address = sas_address;
7392	sas_device->phy = sas_device_pg0.PhyNum;
7393	sas_device->fast_path = (le16_to_cpu(sas_device_pg0.Flags) &
7394	MPI25_SAS_DEVICE0_FLAGS_FAST_PATH_CAPABLE) ? 1 : 0;
7395	sas_device->port = mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0);
7396	if (!sas_device->port) {
7397	ioc_err(ioc, "failure at %s:%d/%s()!\n",
7398	__FILE__, __LINE__, __func__);
7399	goto out;
7400	}
7401
7402	if (le16_to_cpu(sas_device_pg0.Flags)
7403	& MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
7404	sas_device->enclosure_level =
7405	sas_device_pg0.EnclosureLevel;
7406	memcpy(sas_device->connector_name,
7407	sas_device_pg0.ConnectorName, 4);
7408	sas_device->connector_name[4] = '\0';
7409	} else {
7410	sas_device->enclosure_level = 0;
7411	sas_device->connector_name[0] = '\0';
7412	}
7413	/* get enclosure_logical_id & chassis_slot*/
7414	sas_device->is_chassis_slot_valid = 0;
7415	if (enclosure_dev) {
7416	sas_device->enclosure_logical_id =
7417	le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
7418	if (le16_to_cpu(enclosure_dev->pg0.Flags) &
7419	MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
7420	sas_device->is_chassis_slot_valid = 1;
7421	sas_device->chassis_slot =
7422	enclosure_dev->pg0.ChassisSlot;
7423	}
7424	}
7425
7426	/* get device name */
7427	sas_device->device_name = le64_to_cpu(sas_device_pg0.DeviceName);
7428	sas_device->port_type = sas_device_pg0.MaxPortConnections;
7429	ioc_info(ioc,
7430	"handle(0x%0x) sas_address(0x%016llx) port_type(0x%0x)\n",
7431	handle, sas_device->sas_address, sas_device->port_type);
7432
7433	if (ioc->wait_for_discovery_to_complete)
7434	_scsih_sas_device_init_add(ioc, sas_device);
7435	else
7436	_scsih_sas_device_add(ioc, sas_device);
7437
7438	out:
7439	sas_device_put(s: sas_device);
7440	return 0;
7441	}
7442
7443	/**
7444	* _scsih_remove_device - removing sas device object
7445	* @ioc: per adapter object
7446	* @sas_device: the sas_device object
7447	*/
7448	static void
7449	_scsih_remove_device(struct MPT3SAS_ADAPTER *ioc,
7450	struct _sas_device *sas_device)
7451	{
7452	struct MPT3SAS_TARGET *sas_target_priv_data;
7453
7454	if ((ioc->pdev->subsystem_vendor == PCI_VENDOR_ID_IBM) &&
7455	(sas_device->pfa_led_on)) {
7456	_scsih_turn_off_pfa_led(ioc, sas_device);
7457	sas_device->pfa_led_on = 0;
7458	}
7459
7460	dewtprintk(ioc,
7461	ioc_info(ioc, "%s: enter: handle(0x%04x), sas_addr(0x%016llx)\n",
7462	__func__,
7463	sas_device->handle, (u64)sas_device->sas_address));
7464
7465	dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,
7466	NULL, NULL));
7467
7468	if (sas_device->starget && sas_device->starget->hostdata) {
7469	sas_target_priv_data = sas_device->starget->hostdata;
7470	sas_target_priv_data->deleted = 1;
7471	_scsih_ublock_io_device(ioc, sas_address: sas_device->sas_address,
7472	port: sas_device->port);
7473	sas_target_priv_data->handle =
7474	MPT3SAS_INVALID_DEVICE_HANDLE;
7475	}
7476
7477	if (!ioc->hide_drives)
7478	mpt3sas_transport_port_remove(ioc,
7479	sas_address: sas_device->sas_address,
7480	sas_address_parent: sas_device->sas_address_parent,
7481	port: sas_device->port);
7482
7483	ioc_info(ioc, "removing handle(0x%04x), sas_addr(0x%016llx)\n",
7484	sas_device->handle, (u64)sas_device->sas_address);
7485
7486	_scsih_display_enclosure_chassis_info(ioc, sas_device, NULL, NULL);
7487
7488	dewtprintk(ioc,
7489	ioc_info(ioc, "%s: exit: handle(0x%04x), sas_addr(0x%016llx)\n",
7490	__func__,
7491	sas_device->handle, (u64)sas_device->sas_address));
7492	dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,
7493	NULL, NULL));
7494	}
7495
7496	/**
7497	* _scsih_sas_topology_change_event_debug - debug for topology event
7498	* @ioc: per adapter object
7499	* @event_data: event data payload
7500	* Context: user.
7501	*/
7502	static void
7503	_scsih_sas_topology_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
7504	Mpi2EventDataSasTopologyChangeList_t *event_data)
7505	{
7506	int i;
7507	u16 handle;
7508	u16 reason_code;
7509	u8 phy_number;
7510	char *status_str = NULL;
7511	u8 link_rate, prev_link_rate;
7512
7513	switch (event_data->ExpStatus) {
7514	case MPI2_EVENT_SAS_TOPO_ES_ADDED:
7515	status_str = "add";
7516	break;
7517	case MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING:
7518	status_str = "remove";
7519	break;
7520	case MPI2_EVENT_SAS_TOPO_ES_RESPONDING:
7521	case 0:
7522	status_str = "responding";
7523	break;
7524	case MPI2_EVENT_SAS_TOPO_ES_DELAY_NOT_RESPONDING:
7525	status_str = "remove delay";
7526	break;
7527	default:
7528	status_str = "unknown status";
7529	break;
7530	}
7531	ioc_info(ioc, "sas topology change: (%s)\n", status_str);
7532	pr_info("\thandle(0x%04x), enclosure_handle(0x%04x) " \
7533	"start_phy(%02d), count(%d)\n",
7534	le16_to_cpu(event_data->ExpanderDevHandle),
7535	le16_to_cpu(event_data->EnclosureHandle),
7536	event_data->StartPhyNum, event_data->NumEntries);
7537	for (i = 0; i < event_data->NumEntries; i++) {
7538	handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
7539	if (!handle)
7540	continue;
7541	phy_number = event_data->StartPhyNum + i;
7542	reason_code = event_data->PHY[i].PhyStatus &
7543	MPI2_EVENT_SAS_TOPO_RC_MASK;
7544	switch (reason_code) {
7545	case MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED:
7546	status_str = "target add";
7547	break;
7548	case MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING:
7549	status_str = "target remove";
7550	break;
7551	case MPI2_EVENT_SAS_TOPO_RC_DELAY_NOT_RESPONDING:
7552	status_str = "delay target remove";
7553	break;
7554	case MPI2_EVENT_SAS_TOPO_RC_PHY_CHANGED:
7555	status_str = "link rate change";
7556	break;
7557	case MPI2_EVENT_SAS_TOPO_RC_NO_CHANGE:
7558	status_str = "target responding";
7559	break;
7560	default:
7561	status_str = "unknown";
7562	break;
7563	}
7564	link_rate = event_data->PHY[i].LinkRate >> 4;
7565	prev_link_rate = event_data->PHY[i].LinkRate & 0xF;
7566	pr_info("\tphy(%02d), attached_handle(0x%04x): %s:" \
7567	" link rate: new(0x%02x), old(0x%02x)\n", phy_number,
7568	handle, status_str, link_rate, prev_link_rate);
7569
7570	}
7571	}
7572
7573	/**
7574	* _scsih_sas_topology_change_event - handle topology changes
7575	* @ioc: per adapter object
7576	* @fw_event: The fw_event_work object
7577	* Context: user.
7578	*
7579	*/
7580	static int
7581	_scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
7582	struct fw_event_work *fw_event)
7583	{
7584	int i;
7585	u16 parent_handle, handle;
7586	u16 reason_code;
7587	u8 phy_number, max_phys;
7588	struct _sas_node *sas_expander;
7589	u64 sas_address;
7590	unsigned long flags;
7591	u8 link_rate, prev_link_rate;
7592	struct hba_port *port;
7593	Mpi2EventDataSasTopologyChangeList_t *event_data =
7594	(Mpi2EventDataSasTopologyChangeList_t *)
7595	fw_event->event_data;
7596
7597	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
7598	_scsih_sas_topology_change_event_debug(ioc, event_data);
7599
7600	if (ioc->shost_recovery || ioc->remove_host || ioc->pci_error_recovery)
7601	return 0;
7602
7603	if (!ioc->sas_hba.num_phys)
7604	_scsih_sas_host_add(ioc);
7605	else
7606	_scsih_sas_host_refresh(ioc);
7607
7608	if (fw_event->ignore) {
7609	dewtprintk(ioc, ioc_info(ioc, "ignoring expander event\n"));
7610	return 0;
7611	}
7612
7613	parent_handle = le16_to_cpu(event_data->ExpanderDevHandle);
7614	port = mpt3sas_get_port_by_id(ioc, port_id: event_data->PhysicalPort, bypass_dirty_port_flag: 0);
7615
7616	/* handle expander add */
7617	if (event_data->ExpStatus == MPI2_EVENT_SAS_TOPO_ES_ADDED)
7618	if (_scsih_expander_add(ioc, handle: parent_handle) != 0)
7619	return 0;
7620
7621	spin_lock_irqsave(&ioc->sas_node_lock, flags);
7622	sas_expander = mpt3sas_scsih_expander_find_by_handle(ioc,
7623	handle: parent_handle);
7624	if (sas_expander) {
7625	sas_address = sas_expander->sas_address;
7626	max_phys = sas_expander->num_phys;
7627	port = sas_expander->port;
7628	} else if (parent_handle < ioc->sas_hba.num_phys) {
7629	sas_address = ioc->sas_hba.sas_address;
7630	max_phys = ioc->sas_hba.num_phys;
7631	} else {
7632	spin_unlock_irqrestore(lock: &ioc->sas_node_lock, flags);
7633	return 0;
7634	}
7635	spin_unlock_irqrestore(lock: &ioc->sas_node_lock, flags);
7636
7637	/* handle siblings events */
7638	for (i = 0; i < event_data->NumEntries; i++) {
7639	if (fw_event->ignore) {
7640	dewtprintk(ioc,
7641	ioc_info(ioc, "ignoring expander event\n"));
7642	return 0;
7643	}
7644	if (ioc->remove_host || ioc->pci_error_recovery)
7645	return 0;
7646	phy_number = event_data->StartPhyNum + i;
7647	if (phy_number >= max_phys)
7648	continue;
7649	reason_code = event_data->PHY[i].PhyStatus &
7650	MPI2_EVENT_SAS_TOPO_RC_MASK;
7651	if ((event_data->PHY[i].PhyStatus &
7652	MPI2_EVENT_SAS_TOPO_PHYSTATUS_VACANT) && (reason_code !=
7653	MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING))
7654	continue;
7655	handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
7656	if (!handle)
7657	continue;
7658	link_rate = event_data->PHY[i].LinkRate >> 4;
7659	prev_link_rate = event_data->PHY[i].LinkRate & 0xF;
7660	switch (reason_code) {
7661	case MPI2_EVENT_SAS_TOPO_RC_PHY_CHANGED:
7662
7663	if (ioc->shost_recovery)
7664	break;
7665
7666	if (link_rate == prev_link_rate)
7667	break;
7668
7669	mpt3sas_transport_update_links(ioc, sas_address,
7670	handle, phy_number, link_rate, port);
7671
7672	if (link_rate < MPI2_SAS_NEG_LINK_RATE_1_5)
7673	break;
7674
7675	_scsih_check_device(ioc, parent_sas_address: sas_address, handle,
7676	phy_number, link_rate);
7677
7678	if (!test_bit(handle, ioc->pend_os_device_add))
7679	break;
7680
7681	fallthrough;
7682
7683	case MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED:
7684
7685	if (ioc->shost_recovery)
7686	break;
7687
7688	mpt3sas_transport_update_links(ioc, sas_address,
7689	handle, phy_number, link_rate, port);
7690
7691	_scsih_add_device(ioc, handle, phy_num: phy_number, is_pd: 0);
7692
7693	break;
7694	case MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING:
7695
7696	_scsih_device_remove_by_handle(ioc, handle);
7697	break;
7698	}
7699	}
7700
7701	/* handle expander removal */
7702	if (event_data->ExpStatus == MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING &&
7703	sas_expander)
7704	mpt3sas_expander_remove(ioc, sas_address, port);
7705
7706	return 0;
7707	}
7708
7709	/**
7710	* _scsih_sas_device_status_change_event_debug - debug for device event
7711	* @ioc: ?
7712	* @event_data: event data payload
7713	* Context: user.
7714	*/
7715	static void
7716	_scsih_sas_device_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
7717	Mpi2EventDataSasDeviceStatusChange_t *event_data)
7718	{
7719	char *reason_str = NULL;
7720
7721	switch (event_data->ReasonCode) {
7722	case MPI2_EVENT_SAS_DEV_STAT_RC_SMART_DATA:
7723	reason_str = "smart data";
7724	break;
7725	case MPI2_EVENT_SAS_DEV_STAT_RC_UNSUPPORTED:
7726	reason_str = "unsupported device discovered";
7727	break;
7728	case MPI2_EVENT_SAS_DEV_STAT_RC_INTERNAL_DEVICE_RESET:
7729	reason_str = "internal device reset";
7730	break;
7731	case MPI2_EVENT_SAS_DEV_STAT_RC_TASK_ABORT_INTERNAL:
7732	reason_str = "internal task abort";
7733	break;
7734	case MPI2_EVENT_SAS_DEV_STAT_RC_ABORT_TASK_SET_INTERNAL:
7735	reason_str = "internal task abort set";
7736	break;
7737	case MPI2_EVENT_SAS_DEV_STAT_RC_CLEAR_TASK_SET_INTERNAL:
7738	reason_str = "internal clear task set";
7739	break;
7740	case MPI2_EVENT_SAS_DEV_STAT_RC_QUERY_TASK_INTERNAL:
7741	reason_str = "internal query task";
7742	break;
7743	case MPI2_EVENT_SAS_DEV_STAT_RC_SATA_INIT_FAILURE:
7744	reason_str = "sata init failure";
7745	break;
7746	case MPI2_EVENT_SAS_DEV_STAT_RC_CMP_INTERNAL_DEV_RESET:
7747	reason_str = "internal device reset complete";
7748	break;
7749	case MPI2_EVENT_SAS_DEV_STAT_RC_CMP_TASK_ABORT_INTERNAL:
7750	reason_str = "internal task abort complete";
7751	break;
7752	case MPI2_EVENT_SAS_DEV_STAT_RC_ASYNC_NOTIFICATION:
7753	reason_str = "internal async notification";
7754	break;
7755	case MPI2_EVENT_SAS_DEV_STAT_RC_EXPANDER_REDUCED_FUNCTIONALITY:
7756	reason_str = "expander reduced functionality";
7757	break;
7758	case MPI2_EVENT_SAS_DEV_STAT_RC_CMP_EXPANDER_REDUCED_FUNCTIONALITY:
7759	reason_str = "expander reduced functionality complete";
7760	break;
7761	default:
7762	reason_str = "unknown reason";
7763	break;
7764	}
7765	ioc_info(ioc, "device status change: (%s)\thandle(0x%04x), sas address(0x%016llx), tag(%d)",
7766	reason_str, le16_to_cpu(event_data->DevHandle),
7767	(u64)le64_to_cpu(event_data->SASAddress),
7768	le16_to_cpu(event_data->TaskTag));
7769	if (event_data->ReasonCode == MPI2_EVENT_SAS_DEV_STAT_RC_SMART_DATA)
7770	pr_cont(", ASC(0x%x), ASCQ(0x%x)\n",
7771	event_data->ASC, event_data->ASCQ);
7772	pr_cont("\n");
7773	}
7774
7775	/**
7776	* _scsih_sas_device_status_change_event - handle device status change
7777	* @ioc: per adapter object
7778	* @event_data: The fw event
7779	* Context: user.
7780	*/
7781	static void
7782	_scsih_sas_device_status_change_event(struct MPT3SAS_ADAPTER *ioc,
7783	Mpi2EventDataSasDeviceStatusChange_t *event_data)
7784	{
7785	struct MPT3SAS_TARGET *target_priv_data;
7786	struct _sas_device *sas_device;
7787	u64 sas_address;
7788	unsigned long flags;
7789
7790	/* In MPI Revision K (0xC), the internal device reset complete was
7791	* implemented, so avoid setting tm_busy flag for older firmware.
7792	*/
7793	if ((ioc->facts.HeaderVersion >> 8) < 0xC)
7794	return;
7795
7796	if (event_data->ReasonCode !=
7797	MPI2_EVENT_SAS_DEV_STAT_RC_INTERNAL_DEVICE_RESET &&
7798	event_data->ReasonCode !=
7799	MPI2_EVENT_SAS_DEV_STAT_RC_CMP_INTERNAL_DEV_RESET)
7800	return;
7801
7802	spin_lock_irqsave(&ioc->sas_device_lock, flags);
7803	sas_address = le64_to_cpu(event_data->SASAddress);
7804	sas_device = __mpt3sas_get_sdev_by_addr(ioc,
7805	sas_address,
7806	port: mpt3sas_get_port_by_id(ioc, port_id: event_data->PhysicalPort, bypass_dirty_port_flag: 0));
7807
7808	if (!sas_device || !sas_device->starget)
7809	goto out;
7810
7811	target_priv_data = sas_device->starget->hostdata;
7812	if (!target_priv_data)
7813	goto out;
7814
7815	if (event_data->ReasonCode ==
7816	MPI2_EVENT_SAS_DEV_STAT_RC_INTERNAL_DEVICE_RESET)
7817	target_priv_data->tm_busy = 1;
7818	else
7819	target_priv_data->tm_busy = 0;
7820
7821	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
7822	ioc_info(ioc,
7823	"%s tm_busy flag for handle(0x%04x)\n",
7824	(target_priv_data->tm_busy == 1) ? "Enable" : "Disable",
7825	target_priv_data->handle);
7826
7827	out:
7828	if (sas_device)
7829	sas_device_put(s: sas_device);
7830
7831	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
7832	}
7833
7834
7835	/**
7836	* _scsih_check_pcie_access_status - check access flags
7837	* @ioc: per adapter object
7838	* @wwid: wwid
7839	* @handle: sas device handle
7840	* @access_status: errors returned during discovery of the device
7841	*
7842	* Return: 0 for success, else failure
7843	*/
7844	static u8
7845	_scsih_check_pcie_access_status(struct MPT3SAS_ADAPTER *ioc, u64 wwid,
7846	u16 handle, u8 access_status)
7847	{
7848	u8 rc = 1;
7849	char *desc = NULL;
7850
7851	switch (access_status) {
7852	case MPI26_PCIEDEV0_ASTATUS_NO_ERRORS:
7853	case MPI26_PCIEDEV0_ASTATUS_NEEDS_INITIALIZATION:
7854	rc = 0;
7855	break;
7856	case MPI26_PCIEDEV0_ASTATUS_CAPABILITY_FAILED:
7857	desc = "PCIe device capability failed";
7858	break;
7859	case MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED:
7860	desc = "PCIe device blocked";
7861	ioc_info(ioc,
7862	"Device with Access Status (%s): wwid(0x%016llx), "
7863	"handle(0x%04x)\n ll only be added to the internal list",
7864	desc, (u64)wwid, handle);
7865	rc = 0;
7866	break;
7867	case MPI26_PCIEDEV0_ASTATUS_MEMORY_SPACE_ACCESS_FAILED:
7868	desc = "PCIe device mem space access failed";
7869	break;
7870	case MPI26_PCIEDEV0_ASTATUS_UNSUPPORTED_DEVICE:
7871	desc = "PCIe device unsupported";
7872	break;
7873	case MPI26_PCIEDEV0_ASTATUS_MSIX_REQUIRED:
7874	desc = "PCIe device MSIx Required";
7875	break;
7876	case MPI26_PCIEDEV0_ASTATUS_INIT_FAIL_MAX:
7877	desc = "PCIe device init fail max";
7878	break;
7879	case MPI26_PCIEDEV0_ASTATUS_UNKNOWN:
7880	desc = "PCIe device status unknown";
7881	break;
7882	case MPI26_PCIEDEV0_ASTATUS_NVME_READY_TIMEOUT:
7883	desc = "nvme ready timeout";
7884	break;
7885	case MPI26_PCIEDEV0_ASTATUS_NVME_DEVCFG_UNSUPPORTED:
7886	desc = "nvme device configuration unsupported";
7887	break;
7888	case MPI26_PCIEDEV0_ASTATUS_NVME_IDENTIFY_FAILED:
7889	desc = "nvme identify failed";
7890	break;
7891	case MPI26_PCIEDEV0_ASTATUS_NVME_QCONFIG_FAILED:
7892	desc = "nvme qconfig failed";
7893	break;
7894	case MPI26_PCIEDEV0_ASTATUS_NVME_QCREATION_FAILED:
7895	desc = "nvme qcreation failed";
7896	break;
7897	case MPI26_PCIEDEV0_ASTATUS_NVME_EVENTCFG_FAILED:
7898	desc = "nvme eventcfg failed";
7899	break;
7900	case MPI26_PCIEDEV0_ASTATUS_NVME_GET_FEATURE_STAT_FAILED:
7901	desc = "nvme get feature stat failed";
7902	break;
7903	case MPI26_PCIEDEV0_ASTATUS_NVME_IDLE_TIMEOUT:
7904	desc = "nvme idle timeout";
7905	break;
7906	case MPI26_PCIEDEV0_ASTATUS_NVME_FAILURE_STATUS:
7907	desc = "nvme failure status";
7908	break;
7909	default:
7910	ioc_err(ioc, "NVMe discovery error(0x%02x): wwid(0x%016llx), handle(0x%04x)\n",
7911	access_status, (u64)wwid, handle);
7912	return rc;
7913	}
7914
7915	if (!rc)
7916	return rc;
7917
7918	ioc_info(ioc, "NVMe discovery error(%s): wwid(0x%016llx), handle(0x%04x)\n",
7919	desc, (u64)wwid, handle);
7920	return rc;
7921	}
7922
7923	/**
7924	* _scsih_pcie_device_remove_from_sml - removing pcie device
7925	* from SML and free up associated memory
7926	* @ioc: per adapter object
7927	* @pcie_device: the pcie_device object
7928	*/
7929	static void
7930	_scsih_pcie_device_remove_from_sml(struct MPT3SAS_ADAPTER *ioc,
7931	struct _pcie_device *pcie_device)
7932	{
7933	struct MPT3SAS_TARGET *sas_target_priv_data;
7934
7935	dewtprintk(ioc,
7936	ioc_info(ioc, "%s: enter: handle(0x%04x), wwid(0x%016llx)\n",
7937	__func__,
7938	pcie_device->handle, (u64)pcie_device->wwid));
7939	if (pcie_device->enclosure_handle != 0)
7940	dewtprintk(ioc,
7941	ioc_info(ioc, "%s: enter: enclosure logical id(0x%016llx), slot(%d)\n",
7942	__func__,
7943	(u64)pcie_device->enclosure_logical_id,
7944	pcie_device->slot));
7945	if (pcie_device->connector_name[0] != '\0')
7946	dewtprintk(ioc,
7947	ioc_info(ioc, "%s: enter: enclosure level(0x%04x), connector name(%s)\n",
7948	__func__,
7949	pcie_device->enclosure_level,
7950	pcie_device->connector_name));
7951
7952	if (pcie_device->starget && pcie_device->starget->hostdata) {
7953	sas_target_priv_data = pcie_device->starget->hostdata;
7954	sas_target_priv_data->deleted = 1;
7955	_scsih_ublock_io_device(ioc, sas_address: pcie_device->wwid, NULL);
7956	sas_target_priv_data->handle = MPT3SAS_INVALID_DEVICE_HANDLE;
7957	}
7958
7959	ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",
7960	pcie_device->handle, (u64)pcie_device->wwid);
7961	if (pcie_device->enclosure_handle != 0)
7962	ioc_info(ioc, "removing : enclosure logical id(0x%016llx), slot(%d)\n",
7963	(u64)pcie_device->enclosure_logical_id,
7964	pcie_device->slot);
7965	if (pcie_device->connector_name[0] != '\0')
7966	ioc_info(ioc, "removing: enclosure level(0x%04x), connector name( %s)\n",
7967	pcie_device->enclosure_level,
7968	pcie_device->connector_name);
7969
7970	if (pcie_device->starget && (pcie_device->access_status !=
7971	MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED))
7972	scsi_remove_target(&pcie_device->starget->dev);
7973	dewtprintk(ioc,
7974	ioc_info(ioc, "%s: exit: handle(0x%04x), wwid(0x%016llx)\n",
7975	__func__,
7976	pcie_device->handle, (u64)pcie_device->wwid));
7977	if (pcie_device->enclosure_handle != 0)
7978	dewtprintk(ioc,
7979	ioc_info(ioc, "%s: exit: enclosure logical id(0x%016llx), slot(%d)\n",
7980	__func__,
7981	(u64)pcie_device->enclosure_logical_id,
7982	pcie_device->slot));
7983	if (pcie_device->connector_name[0] != '\0')
7984	dewtprintk(ioc,
7985	ioc_info(ioc, "%s: exit: enclosure level(0x%04x), connector name( %s)\n",
7986	__func__,
7987	pcie_device->enclosure_level,
7988	pcie_device->connector_name));
7989
7990	kfree(objp: pcie_device->serial_number);
7991	}
7992
7993
7994	/**
7995	* _scsih_pcie_check_device - checking device responsiveness
7996	* @ioc: per adapter object
7997	* @handle: attached device handle
7998	*/
7999	static void
8000	_scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
8001	{
8002	Mpi2ConfigReply_t mpi_reply;
8003	Mpi26PCIeDevicePage0_t pcie_device_pg0;
8004	u32 ioc_status;
8005	struct _pcie_device *pcie_device;
8006	u64 wwid;
8007	unsigned long flags;
8008	struct scsi_target *starget;
8009	struct MPT3SAS_TARGET *sas_target_priv_data;
8010	u32 device_info;
8011
8012	if ((mpt3sas_config_get_pcie_device_pg0(ioc, mpi_reply: &mpi_reply,
8013	config_page: &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_HANDLE, handle)))
8014	return;
8015
8016	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
8017	if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
8018	return;
8019
8020	/* check if this is end device */
8021	device_info = le32_to_cpu(pcie_device_pg0.DeviceInfo);
8022	if (!(_scsih_is_nvme_pciescsi_device(device_info)))
8023	return;
8024
8025	wwid = le64_to_cpu(pcie_device_pg0.WWID);
8026	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
8027	pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);
8028
8029	if (!pcie_device) {
8030	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
8031	return;
8032	}
8033
8034	if (unlikely(pcie_device->handle != handle)) {
8035	starget = pcie_device->starget;
8036	sas_target_priv_data = starget->hostdata;
8037	pcie_device->access_status = pcie_device_pg0.AccessStatus;
8038	starget_printk(KERN_INFO, starget,
8039	"handle changed from(0x%04x) to (0x%04x)!!!\n",
8040	pcie_device->handle, handle);
8041	sas_target_priv_data->handle = handle;
8042	pcie_device->handle = handle;
8043
8044	if (le32_to_cpu(pcie_device_pg0.Flags) &
8045	MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID) {
8046	pcie_device->enclosure_level =
8047	pcie_device_pg0.EnclosureLevel;
8048	memcpy(&pcie_device->connector_name[0],
8049	&pcie_device_pg0.ConnectorName[0], 4);
8050	} else {
8051	pcie_device->enclosure_level = 0;
8052	pcie_device->connector_name[0] = '\0';
8053	}
8054	}
8055
8056	/* check if device is present */
8057	if (!(le32_to_cpu(pcie_device_pg0.Flags) &
8058	MPI26_PCIEDEV0_FLAGS_DEVICE_PRESENT)) {
8059	ioc_info(ioc, "device is not present handle(0x%04x), flags!!!\n",
8060	handle);
8061	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
8062	pcie_device_put(p: pcie_device);
8063	return;
8064	}
8065
8066	/* check if there were any issues with discovery */
8067	if (_scsih_check_pcie_access_status(ioc, wwid, handle,
8068	access_status: pcie_device_pg0.AccessStatus)) {
8069	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
8070	pcie_device_put(p: pcie_device);
8071	return;
8072	}
8073
8074	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
8075	pcie_device_put(p: pcie_device);
8076
8077	_scsih_ublock_io_device(ioc, sas_address: wwid, NULL);
8078
8079	return;
8080	}
8081
8082	/**
8083	* _scsih_pcie_add_device - creating pcie device object
8084	* @ioc: per adapter object
8085	* @handle: pcie device handle
8086	*
8087	* Creating end device object, stored in ioc->pcie_device_list.
8088	*
8089	* Return: 1 means queue the event later, 0 means complete the event
8090	*/
8091	static int
8092	_scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
8093	{
8094	Mpi26PCIeDevicePage0_t pcie_device_pg0;
8095	Mpi26PCIeDevicePage2_t pcie_device_pg2;
8096	Mpi2ConfigReply_t mpi_reply;
8097	struct _pcie_device *pcie_device;
8098	struct _enclosure_node *enclosure_dev;
8099	u32 ioc_status;
8100	u64 wwid;
8101
8102	if ((mpt3sas_config_get_pcie_device_pg0(ioc, mpi_reply: &mpi_reply,
8103	config_page: &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_HANDLE, handle))) {
8104	ioc_err(ioc, "failure at %s:%d/%s()!\n",
8105	__FILE__, __LINE__, __func__);
8106	return 0;
8107	}
8108	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
8109	MPI2_IOCSTATUS_MASK;
8110	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
8111	ioc_err(ioc, "failure at %s:%d/%s()!\n",
8112	__FILE__, __LINE__, __func__);
8113	return 0;
8114	}
8115
8116	set_bit(nr: handle, addr: ioc->pend_os_device_add);
8117	wwid = le64_to_cpu(pcie_device_pg0.WWID);
8118
8119	/* check if device is present */
8120	if (!(le32_to_cpu(pcie_device_pg0.Flags) &
8121	MPI26_PCIEDEV0_FLAGS_DEVICE_PRESENT)) {
8122	ioc_err(ioc, "device is not present handle(0x04%x)!!!\n",
8123	handle);
8124	return 0;
8125	}
8126
8127	/* check if there were any issues with discovery */
8128	if (_scsih_check_pcie_access_status(ioc, wwid, handle,
8129	access_status: pcie_device_pg0.AccessStatus))
8130	return 0;
8131
8132	if (!(_scsih_is_nvme_pciescsi_device(le32_to_cpu
8133	(pcie_device_pg0.DeviceInfo))))
8134	return 0;
8135
8136	pcie_device = mpt3sas_get_pdev_by_wwid(ioc, wwid);
8137	if (pcie_device) {
8138	clear_bit(nr: handle, addr: ioc->pend_os_device_add);
8139	pcie_device_put(p: pcie_device);
8140	return 0;
8141	}
8142
8143	/* PCIe Device Page 2 contains read-only information about a
8144	* specific NVMe device; therefore, this page is only
8145	* valid for NVMe devices and skip for pcie devices of type scsi.
8146	*/
8147	if (!(mpt3sas_scsih_is_pcie_scsi_device(
8148	le32_to_cpu(pcie_device_pg0.DeviceInfo)))) {
8149	if (mpt3sas_config_get_pcie_device_pg2(ioc, mpi_reply: &mpi_reply,
8150	config_page: &pcie_device_pg2, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE,
8151	handle)) {
8152	ioc_err(ioc,
8153	"failure at %s:%d/%s()!\n", __FILE__,
8154	__LINE__, __func__);
8155	return 0;
8156	}
8157
8158	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
8159	MPI2_IOCSTATUS_MASK;
8160	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
8161	ioc_err(ioc,
8162	"failure at %s:%d/%s()!\n", __FILE__,
8163	__LINE__, __func__);
8164	return 0;
8165	}
8166	}
8167
8168	pcie_device = kzalloc(size: sizeof(struct _pcie_device), GFP_KERNEL);
8169	if (!pcie_device) {
8170	ioc_err(ioc, "failure at %s:%d/%s()!\n",
8171	__FILE__, __LINE__, __func__);
8172	return 0;
8173	}
8174
8175	kref_init(kref: &pcie_device->refcount);
8176	pcie_device->id = ioc->pcie_target_id++;
8177	pcie_device->channel = PCIE_CHANNEL;
8178	pcie_device->handle = handle;
8179	pcie_device->access_status = pcie_device_pg0.AccessStatus;
8180	pcie_device->device_info = le32_to_cpu(pcie_device_pg0.DeviceInfo);
8181	pcie_device->wwid = wwid;
8182	pcie_device->port_num = pcie_device_pg0.PortNum;
8183	pcie_device->fast_path = (le32_to_cpu(pcie_device_pg0.Flags) &
8184	MPI26_PCIEDEV0_FLAGS_FAST_PATH_CAPABLE) ? 1 : 0;
8185
8186	pcie_device->enclosure_handle =
8187	le16_to_cpu(pcie_device_pg0.EnclosureHandle);
8188	if (pcie_device->enclosure_handle != 0)
8189	pcie_device->slot = le16_to_cpu(pcie_device_pg0.Slot);
8190
8191	if (le32_to_cpu(pcie_device_pg0.Flags) &
8192	MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID) {
8193	pcie_device->enclosure_level = pcie_device_pg0.EnclosureLevel;
8194	memcpy(&pcie_device->connector_name[0],
8195	&pcie_device_pg0.ConnectorName[0], 4);
8196	} else {
8197	pcie_device->enclosure_level = 0;
8198	pcie_device->connector_name[0] = '\0';
8199	}
8200
8201	/* get enclosure_logical_id */
8202	if (pcie_device->enclosure_handle) {
8203	enclosure_dev =
8204	mpt3sas_scsih_enclosure_find_by_handle(ioc,
8205	handle: pcie_device->enclosure_handle);
8206	if (enclosure_dev)
8207	pcie_device->enclosure_logical_id =
8208	le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
8209	}
8210	/* TODO -- Add device name once FW supports it */
8211	if (!(mpt3sas_scsih_is_pcie_scsi_device(
8212	le32_to_cpu(pcie_device_pg0.DeviceInfo)))) {
8213	pcie_device->nvme_mdts =
8214	le32_to_cpu(pcie_device_pg2.MaximumDataTransferSize);
8215	pcie_device->shutdown_latency =
8216	le16_to_cpu(pcie_device_pg2.ShutdownLatency);
8217	/*
8218	* Set IOC's max_shutdown_latency to drive's RTD3 Entry Latency
8219	* if drive's RTD3 Entry Latency is greater then IOC's
8220	* max_shutdown_latency.
8221	*/
8222	if (pcie_device->shutdown_latency > ioc->max_shutdown_latency)
8223	ioc->max_shutdown_latency =
8224	pcie_device->shutdown_latency;
8225	if (pcie_device_pg2.ControllerResetTO)
8226	pcie_device->reset_timeout =
8227	pcie_device_pg2.ControllerResetTO;
8228	else
8229	pcie_device->reset_timeout = 30;
8230	} else
8231	pcie_device->reset_timeout = 30;
8232
8233	if (ioc->wait_for_discovery_to_complete)
8234	_scsih_pcie_device_init_add(ioc, pcie_device);
8235	else
8236	_scsih_pcie_device_add(ioc, pcie_device);
8237
8238	pcie_device_put(p: pcie_device);
8239	return 0;
8240	}
8241
8242	/**
8243	* _scsih_pcie_topology_change_event_debug - debug for topology
8244	* event
8245	* @ioc: per adapter object
8246	* @event_data: event data payload
8247	* Context: user.
8248	*/
8249	static void
8250	_scsih_pcie_topology_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
8251	Mpi26EventDataPCIeTopologyChangeList_t *event_data)
8252	{
8253	int i;
8254	u16 handle;
8255	u16 reason_code;
8256	u8 port_number;
8257	char *status_str = NULL;
8258	u8 link_rate, prev_link_rate;
8259
8260	switch (event_data->SwitchStatus) {
8261	case MPI26_EVENT_PCIE_TOPO_SS_ADDED:
8262	status_str = "add";
8263	break;
8264	case MPI26_EVENT_PCIE_TOPO_SS_NOT_RESPONDING:
8265	status_str = "remove";
8266	break;
8267	case MPI26_EVENT_PCIE_TOPO_SS_RESPONDING:
8268	case 0:
8269	status_str = "responding";
8270	break;
8271	case MPI26_EVENT_PCIE_TOPO_SS_DELAY_NOT_RESPONDING:
8272	status_str = "remove delay";
8273	break;
8274	default:
8275	status_str = "unknown status";
8276	break;
8277	}
8278	ioc_info(ioc, "pcie topology change: (%s)\n", status_str);
8279	pr_info("\tswitch_handle(0x%04x), enclosure_handle(0x%04x)"
8280	"start_port(%02d), count(%d)\n",
8281	le16_to_cpu(event_data->SwitchDevHandle),
8282	le16_to_cpu(event_data->EnclosureHandle),
8283	event_data->StartPortNum, event_data->NumEntries);
8284	for (i = 0; i < event_data->NumEntries; i++) {
8285	handle =
8286	le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
8287	if (!handle)
8288	continue;
8289	port_number = event_data->StartPortNum + i;
8290	reason_code = event_data->PortEntry[i].PortStatus;
8291	switch (reason_code) {
8292	case MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED:
8293	status_str = "target add";
8294	break;
8295	case MPI26_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
8296	status_str = "target remove";
8297	break;
8298	case MPI26_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING:
8299	status_str = "delay target remove";
8300	break;
8301	case MPI26_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
8302	status_str = "link rate change";
8303	break;
8304	case MPI26_EVENT_PCIE_TOPO_PS_NO_CHANGE:
8305	status_str = "target responding";
8306	break;
8307	default:
8308	status_str = "unknown";
8309	break;
8310	}
8311	link_rate = event_data->PortEntry[i].CurrentPortInfo &
8312	MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
8313	prev_link_rate = event_data->PortEntry[i].PreviousPortInfo &
8314	MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
8315	pr_info("\tport(%02d), attached_handle(0x%04x): %s:"
8316	" link rate: new(0x%02x), old(0x%02x)\n", port_number,
8317	handle, status_str, link_rate, prev_link_rate);
8318	}
8319	}
8320
8321	/**
8322	* _scsih_pcie_topology_change_event - handle PCIe topology
8323	* changes
8324	* @ioc: per adapter object
8325	* @fw_event: The fw_event_work object
8326	* Context: user.
8327	*
8328	*/
8329	static void
8330	_scsih_pcie_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
8331	struct fw_event_work *fw_event)
8332	{
8333	int i;
8334	u16 handle;
8335	u16 reason_code;
8336	u8 link_rate, prev_link_rate;
8337	unsigned long flags;
8338	int rc;
8339	Mpi26EventDataPCIeTopologyChangeList_t *event_data =
8340	(Mpi26EventDataPCIeTopologyChangeList_t *) fw_event->event_data;
8341	struct _pcie_device *pcie_device;
8342
8343	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
8344	_scsih_pcie_topology_change_event_debug(ioc, event_data);
8345
8346	if (ioc->shost_recovery || ioc->remove_host ||
8347	ioc->pci_error_recovery)
8348	return;
8349
8350	if (fw_event->ignore) {
8351	dewtprintk(ioc, ioc_info(ioc, "ignoring switch event\n"));
8352	return;
8353	}
8354
8355	/* handle siblings events */
8356	for (i = 0; i < event_data->NumEntries; i++) {
8357	if (fw_event->ignore) {
8358	dewtprintk(ioc,
8359	ioc_info(ioc, "ignoring switch event\n"));
8360	return;
8361	}
8362	if (ioc->remove_host || ioc->pci_error_recovery)
8363	return;
8364	reason_code = event_data->PortEntry[i].PortStatus;
8365	handle =
8366	le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
8367	if (!handle)
8368	continue;
8369
8370	link_rate = event_data->PortEntry[i].CurrentPortInfo
8371	& MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
8372	prev_link_rate = event_data->PortEntry[i].PreviousPortInfo
8373	& MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
8374
8375	switch (reason_code) {
8376	case MPI26_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
8377	if (ioc->shost_recovery)
8378	break;
8379	if (link_rate == prev_link_rate)
8380	break;
8381	if (link_rate < MPI26_EVENT_PCIE_TOPO_PI_RATE_2_5)
8382	break;
8383
8384	_scsih_pcie_check_device(ioc, handle);
8385
8386	/* This code after this point handles the test case
8387	* where a device has been added, however its returning
8388	* BUSY for sometime. Then before the Device Missing
8389	* Delay expires and the device becomes READY, the
8390	* device is removed and added back.
8391	*/
8392	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
8393	pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);
8394	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
8395
8396	if (pcie_device) {
8397	pcie_device_put(p: pcie_device);
8398	break;
8399	}
8400
8401	if (!test_bit(handle, ioc->pend_os_device_add))
8402	break;
8403
8404	dewtprintk(ioc,
8405	ioc_info(ioc, "handle(0x%04x) device not found: convert event to a device add\n",
8406	handle));
8407	event_data->PortEntry[i].PortStatus &= 0xF0;
8408	event_data->PortEntry[i].PortStatus |=
8409	MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED;
8410	fallthrough;
8411	case MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED:
8412	if (ioc->shost_recovery)
8413	break;
8414	if (link_rate < MPI26_EVENT_PCIE_TOPO_PI_RATE_2_5)
8415	break;
8416
8417	rc = _scsih_pcie_add_device(ioc, handle);
8418	if (!rc) {
8419	/* mark entry vacant */
8420	/* TODO This needs to be reviewed and fixed,
8421	* we dont have an entry
8422	* to make an event void like vacant
8423	*/
8424	event_data->PortEntry[i].PortStatus |=
8425	MPI26_EVENT_PCIE_TOPO_PS_NO_CHANGE;
8426	}
8427	break;
8428	case MPI26_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
8429	_scsih_pcie_device_remove_by_handle(ioc, handle);
8430	break;
8431	}
8432	}
8433	}
8434
8435	/**
8436	* _scsih_pcie_device_status_change_event_debug - debug for device event
8437	* @ioc: ?
8438	* @event_data: event data payload
8439	* Context: user.
8440	*/
8441	static void
8442	_scsih_pcie_device_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
8443	Mpi26EventDataPCIeDeviceStatusChange_t *event_data)
8444	{
8445	char *reason_str = NULL;
8446
8447	switch (event_data->ReasonCode) {
8448	case MPI26_EVENT_PCIDEV_STAT_RC_SMART_DATA:
8449	reason_str = "smart data";
8450	break;
8451	case MPI26_EVENT_PCIDEV_STAT_RC_UNSUPPORTED:
8452	reason_str = "unsupported device discovered";
8453	break;
8454	case MPI26_EVENT_PCIDEV_STAT_RC_INTERNAL_DEVICE_RESET:
8455	reason_str = "internal device reset";
8456	break;
8457	case MPI26_EVENT_PCIDEV_STAT_RC_TASK_ABORT_INTERNAL:
8458	reason_str = "internal task abort";
8459	break;
8460	case MPI26_EVENT_PCIDEV_STAT_RC_ABORT_TASK_SET_INTERNAL:
8461	reason_str = "internal task abort set";
8462	break;
8463	case MPI26_EVENT_PCIDEV_STAT_RC_CLEAR_TASK_SET_INTERNAL:
8464	reason_str = "internal clear task set";
8465	break;
8466	case MPI26_EVENT_PCIDEV_STAT_RC_QUERY_TASK_INTERNAL:
8467	reason_str = "internal query task";
8468	break;
8469	case MPI26_EVENT_PCIDEV_STAT_RC_DEV_INIT_FAILURE:
8470	reason_str = "device init failure";
8471	break;
8472	case MPI26_EVENT_PCIDEV_STAT_RC_CMP_INTERNAL_DEV_RESET:
8473	reason_str = "internal device reset complete";
8474	break;
8475	case MPI26_EVENT_PCIDEV_STAT_RC_CMP_TASK_ABORT_INTERNAL:
8476	reason_str = "internal task abort complete";
8477	break;
8478	case MPI26_EVENT_PCIDEV_STAT_RC_ASYNC_NOTIFICATION:
8479	reason_str = "internal async notification";
8480	break;
8481	case MPI26_EVENT_PCIDEV_STAT_RC_PCIE_HOT_RESET_FAILED:
8482	reason_str = "pcie hot reset failed";
8483	break;
8484	default:
8485	reason_str = "unknown reason";
8486	break;
8487	}
8488
8489	ioc_info(ioc, "PCIE device status change: (%s)\n"
8490	"\thandle(0x%04x), WWID(0x%016llx), tag(%d)",
8491	reason_str, le16_to_cpu(event_data->DevHandle),
8492	(u64)le64_to_cpu(event_data->WWID),
8493	le16_to_cpu(event_data->TaskTag));
8494	if (event_data->ReasonCode == MPI26_EVENT_PCIDEV_STAT_RC_SMART_DATA)
8495	pr_cont(", ASC(0x%x), ASCQ(0x%x)\n",
8496	event_data->ASC, event_data->ASCQ);
8497	pr_cont("\n");
8498	}
8499
8500	/**
8501	* _scsih_pcie_device_status_change_event - handle device status
8502	* change
8503	* @ioc: per adapter object
8504	* @fw_event: The fw_event_work object
8505	* Context: user.
8506	*/
8507	static void
8508	_scsih_pcie_device_status_change_event(struct MPT3SAS_ADAPTER *ioc,
8509	struct fw_event_work *fw_event)
8510	{
8511	struct MPT3SAS_TARGET *target_priv_data;
8512	struct _pcie_device *pcie_device;
8513	u64 wwid;
8514	unsigned long flags;
8515	Mpi26EventDataPCIeDeviceStatusChange_t *event_data =
8516	(Mpi26EventDataPCIeDeviceStatusChange_t *)fw_event->event_data;
8517	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
8518	_scsih_pcie_device_status_change_event_debug(ioc,
8519	event_data);
8520
8521	if (event_data->ReasonCode !=
8522	MPI26_EVENT_PCIDEV_STAT_RC_INTERNAL_DEVICE_RESET &&
8523	event_data->ReasonCode !=
8524	MPI26_EVENT_PCIDEV_STAT_RC_CMP_INTERNAL_DEV_RESET)
8525	return;
8526
8527	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
8528	wwid = le64_to_cpu(event_data->WWID);
8529	pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);
8530
8531	if (!pcie_device || !pcie_device->starget)
8532	goto out;
8533
8534	target_priv_data = pcie_device->starget->hostdata;
8535	if (!target_priv_data)
8536	goto out;
8537
8538	if (event_data->ReasonCode ==
8539	MPI26_EVENT_PCIDEV_STAT_RC_INTERNAL_DEVICE_RESET)
8540	target_priv_data->tm_busy = 1;
8541	else
8542	target_priv_data->tm_busy = 0;
8543	out:
8544	if (pcie_device)
8545	pcie_device_put(p: pcie_device);
8546
8547	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
8548	}
8549
8550	/**
8551	* _scsih_sas_enclosure_dev_status_change_event_debug - debug for enclosure
8552	* event
8553	* @ioc: per adapter object
8554	* @event_data: event data payload
8555	* Context: user.
8556	*/
8557	static void
8558	_scsih_sas_enclosure_dev_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
8559	Mpi2EventDataSasEnclDevStatusChange_t *event_data)
8560	{
8561	char *reason_str = NULL;
8562
8563	switch (event_data->ReasonCode) {
8564	case MPI2_EVENT_SAS_ENCL_RC_ADDED:
8565	reason_str = "enclosure add";
8566	break;
8567	case MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING:
8568	reason_str = "enclosure remove";
8569	break;
8570	default:
8571	reason_str = "unknown reason";
8572	break;
8573	}
8574
8575	ioc_info(ioc, "enclosure status change: (%s)\n"
8576	"\thandle(0x%04x), enclosure logical id(0x%016llx) number slots(%d)\n",
8577	reason_str,
8578	le16_to_cpu(event_data->EnclosureHandle),
8579	(u64)le64_to_cpu(event_data->EnclosureLogicalID),
8580	le16_to_cpu(event_data->StartSlot));
8581	}
8582
8583	/**
8584	* _scsih_sas_enclosure_dev_status_change_event - handle enclosure events
8585	* @ioc: per adapter object
8586	* @fw_event: The fw_event_work object
8587	* Context: user.
8588	*/
8589	static void
8590	_scsih_sas_enclosure_dev_status_change_event(struct MPT3SAS_ADAPTER *ioc,
8591	struct fw_event_work *fw_event)
8592	{
8593	Mpi2ConfigReply_t mpi_reply;
8594	struct _enclosure_node *enclosure_dev = NULL;
8595	Mpi2EventDataSasEnclDevStatusChange_t *event_data =
8596	(Mpi2EventDataSasEnclDevStatusChange_t *)fw_event->event_data;
8597	int rc;
8598	u16 enclosure_handle = le16_to_cpu(event_data->EnclosureHandle);
8599
8600	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
8601	_scsih_sas_enclosure_dev_status_change_event_debug(ioc,
8602	event_data: (Mpi2EventDataSasEnclDevStatusChange_t *)
8603	fw_event->event_data);
8604	if (ioc->shost_recovery)
8605	return;
8606
8607	if (enclosure_handle)
8608	enclosure_dev =
8609	mpt3sas_scsih_enclosure_find_by_handle(ioc,
8610	handle: enclosure_handle);
8611	switch (event_data->ReasonCode) {
8612	case MPI2_EVENT_SAS_ENCL_RC_ADDED:
8613	if (!enclosure_dev) {
8614	enclosure_dev =
8615	kzalloc(size: sizeof(struct _enclosure_node),
8616	GFP_KERNEL);
8617	if (!enclosure_dev) {
8618	ioc_info(ioc, "failure at %s:%d/%s()!\n",
8619	__FILE__, __LINE__, __func__);
8620	return;
8621	}
8622	rc = mpt3sas_config_get_enclosure_pg0(ioc, mpi_reply: &mpi_reply,
8623	config_page: &enclosure_dev->pg0,
8624	MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
8625	handle: enclosure_handle);
8626
8627	if (rc || (le16_to_cpu(mpi_reply.IOCStatus) &
8628	MPI2_IOCSTATUS_MASK)) {
8629	kfree(objp: enclosure_dev);
8630	return;
8631	}
8632
8633	list_add_tail(new: &enclosure_dev->list,
8634	head: &ioc->enclosure_list);
8635	}
8636	break;
8637	case MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING:
8638	if (enclosure_dev) {
8639	list_del(entry: &enclosure_dev->list);
8640	kfree(objp: enclosure_dev);
8641	}
8642	break;
8643	default:
8644	break;
8645	}
8646	}
8647
8648	/**
8649	* _scsih_sas_broadcast_primitive_event - handle broadcast events
8650	* @ioc: per adapter object
8651	* @fw_event: The fw_event_work object
8652	* Context: user.
8653	*/
8654	static void
8655	_scsih_sas_broadcast_primitive_event(struct MPT3SAS_ADAPTER *ioc,
8656	struct fw_event_work *fw_event)
8657	{
8658	struct scsi_cmnd *scmd;
8659	struct scsi_device *sdev;
8660	struct scsiio_tracker *st;
8661	u16 smid, handle;
8662	u32 lun;
8663	struct MPT3SAS_DEVICE *sas_device_priv_data;
8664	u32 termination_count;
8665	u32 query_count;
8666	Mpi2SCSITaskManagementReply_t *mpi_reply;
8667	Mpi2EventDataSasBroadcastPrimitive_t *event_data =
8668	(Mpi2EventDataSasBroadcastPrimitive_t *)
8669	fw_event->event_data;
8670	u16 ioc_status;
8671	unsigned long flags;
8672	int r;
8673	u8 max_retries = 0;
8674	u8 task_abort_retries;
8675
8676	mutex_lock(&ioc->tm_cmds.mutex);
8677	ioc_info(ioc, "%s: enter: phy number(%d), width(%d)\n",
8678	__func__, event_data->PhyNum, event_data->PortWidth);
8679
8680	_scsih_block_io_all_device(ioc);
8681
8682	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8683	mpi_reply = ioc->tm_cmds.reply;
8684	broadcast_aen_retry:
8685
8686	/* sanity checks for retrying this loop */
8687	if (max_retries++ == 5) {
8688	dewtprintk(ioc, ioc_info(ioc, "%s: giving up\n", __func__));
8689	goto out;
8690	} else if (max_retries > 1)
8691	dewtprintk(ioc,
8692	ioc_info(ioc, "%s: %d retry\n",
8693	__func__, max_retries - 1));
8694
8695	termination_count = 0;
8696	query_count = 0;
8697	for (smid = 1; smid <= ioc->scsiio_depth; smid++) {
8698	if (ioc->shost_recovery)
8699	goto out;
8700	scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
8701	if (!scmd)
8702	continue;
8703	st = scsi_cmd_priv(cmd: scmd);
8704	sdev = scmd->device;
8705	sas_device_priv_data = sdev->hostdata;
8706	if (!sas_device_priv_data || !sas_device_priv_data->sas_target)
8707	continue;
8708	/* skip hidden raid components */
8709	if (sas_device_priv_data->sas_target->flags &
8710	MPT_TARGET_FLAGS_RAID_COMPONENT)
8711	continue;
8712	/* skip volumes */
8713	if (sas_device_priv_data->sas_target->flags &
8714	MPT_TARGET_FLAGS_VOLUME)
8715	continue;
8716	/* skip PCIe devices */
8717	if (sas_device_priv_data->sas_target->flags &
8718	MPT_TARGET_FLAGS_PCIE_DEVICE)
8719	continue;
8720
8721	handle = sas_device_priv_data->sas_target->handle;
8722	lun = sas_device_priv_data->lun;
8723	query_count++;
8724
8725	if (ioc->shost_recovery)
8726	goto out;
8727
8728	spin_unlock_irqrestore(lock: &ioc->scsi_lookup_lock, flags);
8729	r = mpt3sas_scsih_issue_tm(ioc, handle, channel: 0, id: 0, lun,
8730	MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK, smid_task: st->smid,
8731	msix_task: st->msix_io, timeout: 30, tr_method: 0);
8732	if (r == FAILED) {
8733	sdev_printk(KERN_WARNING, sdev,
8734	"mpt3sas_scsih_issue_tm: FAILED when sending "
8735	"QUERY_TASK: scmd(%p)\n", scmd);
8736	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8737	goto broadcast_aen_retry;
8738	}
8739	ioc_status = le16_to_cpu(mpi_reply->IOCStatus)
8740	& MPI2_IOCSTATUS_MASK;
8741	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
8742	sdev_printk(KERN_WARNING, sdev,
8743	"query task: FAILED with IOCSTATUS(0x%04x), scmd(%p)\n",
8744	ioc_status, scmd);
8745	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8746	goto broadcast_aen_retry;
8747	}
8748
8749	/* see if IO is still owned by IOC and target */
8750	if (mpi_reply->ResponseCode ==
8751	MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED ||
8752	mpi_reply->ResponseCode ==
8753	MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC) {
8754	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8755	continue;
8756	}
8757	task_abort_retries = 0;
8758	tm_retry:
8759	if (task_abort_retries++ == 60) {
8760	dewtprintk(ioc,
8761	ioc_info(ioc, "%s: ABORT_TASK: giving up\n",
8762	__func__));
8763	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8764	goto broadcast_aen_retry;
8765	}
8766
8767	if (ioc->shost_recovery)
8768	goto out_no_lock;
8769
8770	r = mpt3sas_scsih_issue_tm(ioc, handle, channel: sdev->channel, id: sdev->id,
8771	lun: sdev->lun, MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
8772	smid_task: st->smid, msix_task: st->msix_io, timeout: 30, tr_method: 0);
8773	if (r == FAILED || st->cb_idx != 0xFF) {
8774	sdev_printk(KERN_WARNING, sdev,
8775	"mpt3sas_scsih_issue_tm: ABORT_TASK: FAILED : "
8776	"scmd(%p)\n", scmd);
8777	goto tm_retry;
8778	}
8779
8780	if (task_abort_retries > 1)
8781	sdev_printk(KERN_WARNING, sdev,
8782	"mpt3sas_scsih_issue_tm: ABORT_TASK: RETRIES (%d):"
8783	" scmd(%p)\n",
8784	task_abort_retries - 1, scmd);
8785
8786	termination_count += le32_to_cpu(mpi_reply->TerminationCount);
8787	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8788	}
8789
8790	if (ioc->broadcast_aen_pending) {
8791	dewtprintk(ioc,
8792	ioc_info(ioc,
8793	"%s: loop back due to pending AEN\n",
8794	__func__));
8795	ioc->broadcast_aen_pending = 0;
8796	goto broadcast_aen_retry;
8797	}
8798
8799	out:
8800	spin_unlock_irqrestore(lock: &ioc->scsi_lookup_lock, flags);
8801	out_no_lock:
8802
8803	dewtprintk(ioc,
8804	ioc_info(ioc, "%s - exit, query_count = %d termination_count = %d\n",
8805	__func__, query_count, termination_count));
8806
8807	ioc->broadcast_aen_busy = 0;
8808	if (!ioc->shost_recovery)
8809	_scsih_ublock_io_all_device(ioc);
8810	mutex_unlock(lock: &ioc->tm_cmds.mutex);
8811	}
8812
8813	/**
8814	* _scsih_sas_discovery_event - handle discovery events
8815	* @ioc: per adapter object
8816	* @fw_event: The fw_event_work object
8817	* Context: user.
8818	*/
8819	static void
8820	_scsih_sas_discovery_event(struct MPT3SAS_ADAPTER *ioc,
8821	struct fw_event_work *fw_event)
8822	{
8823	Mpi2EventDataSasDiscovery_t *event_data =
8824	(Mpi2EventDataSasDiscovery_t *) fw_event->event_data;
8825
8826	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) {
8827	ioc_info(ioc, "discovery event: (%s)",
8828	event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED ?
8829	"start" : "stop");
8830	if (event_data->DiscoveryStatus)
8831	pr_cont("discovery_status(0x%08x)",
8832	le32_to_cpu(event_data->DiscoveryStatus));
8833	pr_cont("\n");
8834	}
8835
8836	if (event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED &&
8837	!ioc->sas_hba.num_phys) {
8838	if (disable_discovery > 0 && ioc->shost_recovery) {
8839	/* Wait for the reset to complete */
8840	while (ioc->shost_recovery)
8841	ssleep(seconds: 1);
8842	}
8843	_scsih_sas_host_add(ioc);
8844	}
8845	}
8846
8847	/**
8848	* _scsih_sas_device_discovery_error_event - display SAS device discovery error
8849	* events
8850	* @ioc: per adapter object
8851	* @fw_event: The fw_event_work object
8852	* Context: user.
8853	*/
8854	static void
8855	_scsih_sas_device_discovery_error_event(struct MPT3SAS_ADAPTER *ioc,
8856	struct fw_event_work *fw_event)
8857	{
8858	Mpi25EventDataSasDeviceDiscoveryError_t *event_data =
8859	(Mpi25EventDataSasDeviceDiscoveryError_t *)fw_event->event_data;
8860
8861	switch (event_data->ReasonCode) {
8862	case MPI25_EVENT_SAS_DISC_ERR_SMP_FAILED:
8863	ioc_warn(ioc, "SMP command sent to the expander (handle:0x%04x, sas_address:0x%016llx, physical_port:0x%02x) has failed\n",
8864	le16_to_cpu(event_data->DevHandle),
8865	(u64)le64_to_cpu(event_data->SASAddress),
8866	event_data->PhysicalPort);
8867	break;
8868	case MPI25_EVENT_SAS_DISC_ERR_SMP_TIMEOUT:
8869	ioc_warn(ioc, "SMP command sent to the expander (handle:0x%04x, sas_address:0x%016llx, physical_port:0x%02x) has timed out\n",
8870	le16_to_cpu(event_data->DevHandle),
8871	(u64)le64_to_cpu(event_data->SASAddress),
8872	event_data->PhysicalPort);
8873	break;
8874	default:
8875	break;
8876	}
8877	}
8878
8879	/**
8880	* _scsih_pcie_enumeration_event - handle enumeration events
8881	* @ioc: per adapter object
8882	* @fw_event: The fw_event_work object
8883	* Context: user.
8884	*/
8885	static void
8886	_scsih_pcie_enumeration_event(struct MPT3SAS_ADAPTER *ioc,
8887	struct fw_event_work *fw_event)
8888	{
8889	Mpi26EventDataPCIeEnumeration_t *event_data =
8890	(Mpi26EventDataPCIeEnumeration_t *)fw_event->event_data;
8891
8892	if (!(ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK))
8893	return;
8894
8895	ioc_info(ioc, "pcie enumeration event: (%s) Flag 0x%02x",
8896	(event_data->ReasonCode == MPI26_EVENT_PCIE_ENUM_RC_STARTED) ?
8897	"started" : "completed",
8898	event_data->Flags);
8899	if (event_data->EnumerationStatus)
8900	pr_cont("enumeration_status(0x%08x)",
8901	le32_to_cpu(event_data->EnumerationStatus));
8902	pr_cont("\n");
8903	}
8904
8905	/**
8906	* _scsih_ir_fastpath - turn on fastpath for IR physdisk
8907	* @ioc: per adapter object
8908	* @handle: device handle for physical disk
8909	* @phys_disk_num: physical disk number
8910	*
8911	* Return: 0 for success, else failure.
8912	*/
8913	static int
8914	_scsih_ir_fastpath(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phys_disk_num)
8915	{
8916	Mpi2RaidActionRequest_t *mpi_request;
8917	Mpi2RaidActionReply_t *mpi_reply;
8918	u16 smid;
8919	u8 issue_reset = 0;
8920	int rc = 0;
8921	u16 ioc_status;
8922	u32 log_info;
8923
8924	if (ioc->hba_mpi_version_belonged == MPI2_VERSION)
8925	return rc;
8926
8927	mutex_lock(&ioc->scsih_cmds.mutex);
8928
8929	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {
8930	ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);
8931	rc = -EAGAIN;
8932	goto out;
8933	}
8934	ioc->scsih_cmds.status = MPT3_CMD_PENDING;
8935
8936	smid = mpt3sas_base_get_smid(ioc, cb_idx: ioc->scsih_cb_idx);
8937	if (!smid) {
8938	ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
8939	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
8940	rc = -EAGAIN;
8941	goto out;
8942	}
8943
8944	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
8945	ioc->scsih_cmds.smid = smid;
8946	memset(mpi_request, 0, sizeof(Mpi2RaidActionRequest_t));
8947
8948	mpi_request->Function = MPI2_FUNCTION_RAID_ACTION;
8949	mpi_request->Action = MPI2_RAID_ACTION_PHYSDISK_HIDDEN;
8950	mpi_request->PhysDiskNum = phys_disk_num;
8951
8952	dewtprintk(ioc,
8953	ioc_info(ioc, "IR RAID_ACTION: turning fast path on for handle(0x%04x), phys_disk_num (0x%02x)\n",
8954	handle, phys_disk_num));
8955
8956	init_completion(x: &ioc->scsih_cmds.done);
8957	ioc->put_smid_default(ioc, smid);
8958	wait_for_completion_timeout(x: &ioc->scsih_cmds.done, timeout: 10*HZ);
8959
8960	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
8961	mpt3sas_check_cmd_timeout(ioc,
8962	ioc->scsih_cmds.status, mpi_request,
8963	sizeof(Mpi2RaidActionRequest_t)/4, issue_reset);
8964	rc = -EFAULT;
8965	goto out;
8966	}
8967
8968	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {
8969
8970	mpi_reply = ioc->scsih_cmds.reply;
8971	ioc_status = le16_to_cpu(mpi_reply->IOCStatus);
8972	if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE)
8973	log_info = le32_to_cpu(mpi_reply->IOCLogInfo);
8974	else
8975	log_info = 0;
8976	ioc_status &= MPI2_IOCSTATUS_MASK;
8977	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
8978	dewtprintk(ioc,
8979	ioc_info(ioc, "IR RAID_ACTION: failed: ioc_status(0x%04x), loginfo(0x%08x)!!!\n",
8980	ioc_status, log_info));
8981	rc = -EFAULT;
8982	} else
8983	dewtprintk(ioc,
8984	ioc_info(ioc, "IR RAID_ACTION: completed successfully\n"));
8985	}
8986
8987	out:
8988	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
8989	mutex_unlock(lock: &ioc->scsih_cmds.mutex);
8990
8991	if (issue_reset)
8992	mpt3sas_base_hard_reset_handler(ioc, type: FORCE_BIG_HAMMER);
8993	return rc;
8994	}
8995
8996	/**
8997	* _scsih_reprobe_lun - reprobing lun
8998	* @sdev: scsi device struct
8999	* @no_uld_attach: sdev->no_uld_attach flag setting
9000	*
9001	**/
9002	static void
9003	_scsih_reprobe_lun(struct scsi_device *sdev, void *no_uld_attach)
9004	{
9005	sdev->no_uld_attach = no_uld_attach ? 1 : 0;
9006	sdev_printk(KERN_INFO, sdev, "%s raid component\n",
9007	sdev->no_uld_attach ? "hiding" : "exposing");
9008	WARN_ON(scsi_device_reprobe(sdev));
9009	}
9010
9011	/**
9012	* _scsih_sas_volume_add - add new volume
9013	* @ioc: per adapter object
9014	* @element: IR config element data
9015	* Context: user.
9016	*/
9017	static void
9018	_scsih_sas_volume_add(struct MPT3SAS_ADAPTER *ioc,
9019	Mpi2EventIrConfigElement_t *element)
9020	{
9021	struct _raid_device *raid_device;
9022	unsigned long flags;
9023	u64 wwid;
9024	u16 handle = le16_to_cpu(element->VolDevHandle);
9025	int rc;
9026
9027	mpt3sas_config_get_volume_wwid(ioc, volume_handle: handle, wwid: &wwid);
9028	if (!wwid) {
9029	ioc_err(ioc, "failure at %s:%d/%s()!\n",
9030	__FILE__, __LINE__, __func__);
9031	return;
9032	}
9033
9034	spin_lock_irqsave(&ioc->raid_device_lock, flags);
9035	raid_device = _scsih_raid_device_find_by_wwid(ioc, wwid);
9036	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
9037
9038	if (raid_device)
9039	return;
9040
9041	raid_device = kzalloc(size: sizeof(struct _raid_device), GFP_KERNEL);
9042	if (!raid_device) {
9043	ioc_err(ioc, "failure at %s:%d/%s()!\n",
9044	__FILE__, __LINE__, __func__);
9045	return;
9046	}
9047
9048	raid_device->id = ioc->sas_id++;
9049	raid_device->channel = RAID_CHANNEL;
9050	raid_device->handle = handle;
9051	raid_device->wwid = wwid;
9052	_scsih_raid_device_add(ioc, raid_device);
9053	if (!ioc->wait_for_discovery_to_complete) {
9054	rc = scsi_add_device(host: ioc->shost, RAID_CHANNEL,
9055	target: raid_device->id, lun: 0);
9056	if (rc)
9057	_scsih_raid_device_remove(ioc, raid_device);
9058	} else {
9059	spin_lock_irqsave(&ioc->raid_device_lock, flags);
9060	_scsih_determine_boot_device(ioc, device: raid_device, channel: 1);
9061	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
9062	}
9063	}
9064
9065	/**
9066	* _scsih_sas_volume_delete - delete volume
9067	* @ioc: per adapter object
9068	* @handle: volume device handle
9069	* Context: user.
9070	*/
9071	static void
9072	_scsih_sas_volume_delete(struct MPT3SAS_ADAPTER *ioc, u16 handle)
9073	{
9074	struct _raid_device *raid_device;
9075	unsigned long flags;
9076	struct MPT3SAS_TARGET *sas_target_priv_data;
9077	struct scsi_target *starget = NULL;
9078
9079	spin_lock_irqsave(&ioc->raid_device_lock, flags);
9080	raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
9081	if (raid_device) {
9082	if (raid_device->starget) {
9083	starget = raid_device->starget;
9084	sas_target_priv_data = starget->hostdata;
9085	sas_target_priv_data->deleted = 1;
9086	}
9087	ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",
9088	raid_device->handle, (u64)raid_device->wwid);
9089	list_del(entry: &raid_device->list);
9090	kfree(objp: raid_device);
9091	}
9092	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
9093	if (starget)
9094	scsi_remove_target(&starget->dev);
9095	}
9096
9097	/**
9098	* _scsih_sas_pd_expose - expose pd component to /dev/sdX
9099	* @ioc: per adapter object
9100	* @element: IR config element data
9101	* Context: user.
9102	*/
9103	static void
9104	_scsih_sas_pd_expose(struct MPT3SAS_ADAPTER *ioc,
9105	Mpi2EventIrConfigElement_t *element)
9106	{
9107	struct _sas_device *sas_device;
9108	struct scsi_target *starget = NULL;
9109	struct MPT3SAS_TARGET *sas_target_priv_data;
9110	unsigned long flags;
9111	u16 handle = le16_to_cpu(element->PhysDiskDevHandle);
9112
9113	spin_lock_irqsave(&ioc->sas_device_lock, flags);
9114	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
9115	if (sas_device) {
9116	sas_device->volume_handle = 0;
9117	sas_device->volume_wwid = 0;
9118	clear_bit(nr: handle, addr: ioc->pd_handles);
9119	if (sas_device->starget && sas_device->starget->hostdata) {
9120	starget = sas_device->starget;
9121	sas_target_priv_data = starget->hostdata;
9122	sas_target_priv_data->flags &=
9123	~MPT_TARGET_FLAGS_RAID_COMPONENT;
9124	}
9125	}
9126	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
9127	if (!sas_device)
9128	return;
9129
9130	/* exposing raid component */
9131	if (starget)
9132	starget_for_each_device(starget, NULL, fn: _scsih_reprobe_lun);
9133
9134	sas_device_put(s: sas_device);
9135	}
9136
9137	/**
9138	* _scsih_sas_pd_hide - hide pd component from /dev/sdX
9139	* @ioc: per adapter object
9140	* @element: IR config element data
9141	* Context: user.
9142	*/
9143	static void
9144	_scsih_sas_pd_hide(struct MPT3SAS_ADAPTER *ioc,
9145	Mpi2EventIrConfigElement_t *element)
9146	{
9147	struct _sas_device *sas_device;
9148	struct scsi_target *starget = NULL;
9149	struct MPT3SAS_TARGET *sas_target_priv_data;
9150	unsigned long flags;
9151	u16 handle = le16_to_cpu(element->PhysDiskDevHandle);
9152	u16 volume_handle = 0;
9153	u64 volume_wwid = 0;
9154
9155	mpt3sas_config_get_volume_handle(ioc, pd_handle: handle, volume_handle: &volume_handle);
9156	if (volume_handle)
9157	mpt3sas_config_get_volume_wwid(ioc, volume_handle,
9158	wwid: &volume_wwid);
9159
9160	spin_lock_irqsave(&ioc->sas_device_lock, flags);
9161	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
9162	if (sas_device) {
9163	set_bit(nr: handle, addr: ioc->pd_handles);
9164	if (sas_device->starget && sas_device->starget->hostdata) {
9165	starget = sas_device->starget;
9166	sas_target_priv_data = starget->hostdata;
9167	sas_target_priv_data->flags |=
9168	MPT_TARGET_FLAGS_RAID_COMPONENT;
9169	sas_device->volume_handle = volume_handle;
9170	sas_device->volume_wwid = volume_wwid;
9171	}
9172	}
9173	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
9174	if (!sas_device)
9175	return;
9176
9177	/* hiding raid component */
9178	_scsih_ir_fastpath(ioc, handle, phys_disk_num: element->PhysDiskNum);
9179
9180	if (starget)
9181	starget_for_each_device(starget, (void *)1, fn: _scsih_reprobe_lun);
9182
9183	sas_device_put(s: sas_device);
9184	}
9185
9186	/**
9187	* _scsih_sas_pd_delete - delete pd component
9188	* @ioc: per adapter object
9189	* @element: IR config element data
9190	* Context: user.
9191	*/
9192	static void
9193	_scsih_sas_pd_delete(struct MPT3SAS_ADAPTER *ioc,
9194	Mpi2EventIrConfigElement_t *element)
9195	{
9196	u16 handle = le16_to_cpu(element->PhysDiskDevHandle);
9197
9198	_scsih_device_remove_by_handle(ioc, handle);
9199	}
9200
9201	/**
9202	* _scsih_sas_pd_add - remove pd component
9203	* @ioc: per adapter object
9204	* @element: IR config element data
9205	* Context: user.
9206	*/
9207	static void
9208	_scsih_sas_pd_add(struct MPT3SAS_ADAPTER *ioc,
9209	Mpi2EventIrConfigElement_t *element)
9210	{
9211	struct _sas_device *sas_device;
9212	u16 handle = le16_to_cpu(element->PhysDiskDevHandle);
9213	Mpi2ConfigReply_t mpi_reply;
9214	Mpi2SasDevicePage0_t sas_device_pg0;
9215	u32 ioc_status;
9216	u64 sas_address;
9217	u16 parent_handle;
9218
9219	set_bit(nr: handle, addr: ioc->pd_handles);
9220
9221	sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
9222	if (sas_device) {
9223	_scsih_ir_fastpath(ioc, handle, phys_disk_num: element->PhysDiskNum);
9224	sas_device_put(s: sas_device);
9225	return;
9226	}
9227
9228	if ((mpt3sas_config_get_sas_device_pg0(ioc, mpi_reply: &mpi_reply, config_page: &sas_device_pg0,
9229	MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
9230	ioc_err(ioc, "failure at %s:%d/%s()!\n",
9231	__FILE__, __LINE__, __func__);
9232	return;
9233	}
9234
9235	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
9236	MPI2_IOCSTATUS_MASK;
9237	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
9238	ioc_err(ioc, "failure at %s:%d/%s()!\n",
9239	__FILE__, __LINE__, __func__);
9240	return;
9241	}
9242
9243	parent_handle = le16_to_cpu(sas_device_pg0.ParentDevHandle);
9244	if (!_scsih_get_sas_address(ioc, handle: parent_handle, sas_address: &sas_address))
9245	mpt3sas_transport_update_links(ioc, sas_address, handle,
9246	phy_number: sas_device_pg0.PhyNum, MPI2_SAS_NEG_LINK_RATE_1_5,
9247	port: mpt3sas_get_port_by_id(ioc,
9248	port_id: sas_device_pg0.PhysicalPort, bypass_dirty_port_flag: 0));
9249
9250	_scsih_ir_fastpath(ioc, handle, phys_disk_num: element->PhysDiskNum);
9251	_scsih_add_device(ioc, handle, phy_num: 0, is_pd: 1);
9252	}
9253
9254	/**
9255	* _scsih_sas_ir_config_change_event_debug - debug for IR Config Change events
9256	* @ioc: per adapter object
9257	* @event_data: event data payload
9258	* Context: user.
9259	*/
9260	static void
9261	_scsih_sas_ir_config_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
9262	Mpi2EventDataIrConfigChangeList_t *event_data)
9263	{
9264	Mpi2EventIrConfigElement_t *element;
9265	u8 element_type;
9266	int i;
9267	char *reason_str = NULL, *element_str = NULL;
9268
9269	element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
9270
9271	ioc_info(ioc, "raid config change: (%s), elements(%d)\n",
9272	le32_to_cpu(event_data->Flags) & MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG ?
9273	"foreign" : "native",
9274	event_data->NumElements);
9275	for (i = 0; i < event_data->NumElements; i++, element++) {
9276	switch (element->ReasonCode) {
9277	case MPI2_EVENT_IR_CHANGE_RC_ADDED:
9278	reason_str = "add";
9279	break;
9280	case MPI2_EVENT_IR_CHANGE_RC_REMOVED:
9281	reason_str = "remove";
9282	break;
9283	case MPI2_EVENT_IR_CHANGE_RC_NO_CHANGE:
9284	reason_str = "no change";
9285	break;
9286	case MPI2_EVENT_IR_CHANGE_RC_HIDE:
9287	reason_str = "hide";
9288	break;
9289	case MPI2_EVENT_IR_CHANGE_RC_UNHIDE:
9290	reason_str = "unhide";
9291	break;
9292	case MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED:
9293	reason_str = "volume_created";
9294	break;
9295	case MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED:
9296	reason_str = "volume_deleted";
9297	break;
9298	case MPI2_EVENT_IR_CHANGE_RC_PD_CREATED:
9299	reason_str = "pd_created";
9300	break;
9301	case MPI2_EVENT_IR_CHANGE_RC_PD_DELETED:
9302	reason_str = "pd_deleted";
9303	break;
9304	default:
9305	reason_str = "unknown reason";
9306	break;
9307	}
9308	element_type = le16_to_cpu(element->ElementFlags) &
9309	MPI2_EVENT_IR_CHANGE_EFLAGS_ELEMENT_TYPE_MASK;
9310	switch (element_type) {
9311	case MPI2_EVENT_IR_CHANGE_EFLAGS_VOLUME_ELEMENT:
9312	element_str = "volume";
9313	break;
9314	case MPI2_EVENT_IR_CHANGE_EFLAGS_VOLPHYSDISK_ELEMENT:
9315	element_str = "phys disk";
9316	break;
9317	case MPI2_EVENT_IR_CHANGE_EFLAGS_HOTSPARE_ELEMENT:
9318	element_str = "hot spare";
9319	break;
9320	default:
9321	element_str = "unknown element";
9322	break;
9323	}
9324	pr_info("\t(%s:%s), vol handle(0x%04x), " \
9325	"pd handle(0x%04x), pd num(0x%02x)\n", element_str,
9326	reason_str, le16_to_cpu(element->VolDevHandle),
9327	le16_to_cpu(element->PhysDiskDevHandle),
9328	element->PhysDiskNum);
9329	}
9330	}
9331
9332	/**
9333	* _scsih_sas_ir_config_change_event - handle ir configuration change events
9334	* @ioc: per adapter object
9335	* @fw_event: The fw_event_work object
9336	* Context: user.
9337	*/
9338	static void
9339	_scsih_sas_ir_config_change_event(struct MPT3SAS_ADAPTER *ioc,
9340	struct fw_event_work *fw_event)
9341	{
9342	Mpi2EventIrConfigElement_t *element;
9343	int i;
9344	u8 foreign_config;
9345	Mpi2EventDataIrConfigChangeList_t *event_data =
9346	(Mpi2EventDataIrConfigChangeList_t *)
9347	fw_event->event_data;
9348
9349	if ((ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) &&
9350	(!ioc->hide_ir_msg))
9351	_scsih_sas_ir_config_change_event_debug(ioc, event_data);
9352
9353	foreign_config = (le32_to_cpu(event_data->Flags) &
9354	MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG) ? 1 : 0;
9355
9356	element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
9357	if (ioc->shost_recovery &&
9358	ioc->hba_mpi_version_belonged != MPI2_VERSION) {
9359	for (i = 0; i < event_data->NumElements; i++, element++) {
9360	if (element->ReasonCode == MPI2_EVENT_IR_CHANGE_RC_HIDE)
9361	_scsih_ir_fastpath(ioc,
9362	le16_to_cpu(element->PhysDiskDevHandle),
9363	phys_disk_num: element->PhysDiskNum);
9364	}
9365	return;
9366	}
9367
9368	for (i = 0; i < event_data->NumElements; i++, element++) {
9369
9370	switch (element->ReasonCode) {
9371	case MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED:
9372	case MPI2_EVENT_IR_CHANGE_RC_ADDED:
9373	if (!foreign_config)
9374	_scsih_sas_volume_add(ioc, element);
9375	break;
9376	case MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED:
9377	case MPI2_EVENT_IR_CHANGE_RC_REMOVED:
9378	if (!foreign_config)
9379	_scsih_sas_volume_delete(ioc,
9380	le16_to_cpu(element->VolDevHandle));
9381	break;
9382	case MPI2_EVENT_IR_CHANGE_RC_PD_CREATED:
9383	if (!ioc->is_warpdrive)
9384	_scsih_sas_pd_hide(ioc, element);
9385	break;
9386	case MPI2_EVENT_IR_CHANGE_RC_PD_DELETED:
9387	if (!ioc->is_warpdrive)
9388	_scsih_sas_pd_expose(ioc, element);
9389	break;
9390	case MPI2_EVENT_IR_CHANGE_RC_HIDE:
9391	if (!ioc->is_warpdrive)
9392	_scsih_sas_pd_add(ioc, element);
9393	break;
9394	case MPI2_EVENT_IR_CHANGE_RC_UNHIDE:
9395	if (!ioc->is_warpdrive)
9396	_scsih_sas_pd_delete(ioc, element);
9397	break;
9398	}
9399	}
9400	}
9401
9402	/**
9403	* _scsih_sas_ir_volume_event - IR volume event
9404	* @ioc: per adapter object
9405	* @fw_event: The fw_event_work object
9406	* Context: user.
9407	*/
9408	static void
9409	_scsih_sas_ir_volume_event(struct MPT3SAS_ADAPTER *ioc,
9410	struct fw_event_work *fw_event)
9411	{
9412	u64 wwid;
9413	unsigned long flags;
9414	struct _raid_device *raid_device;
9415	u16 handle;
9416	u32 state;
9417	int rc;
9418	Mpi2EventDataIrVolume_t *event_data =
9419	(Mpi2EventDataIrVolume_t *) fw_event->event_data;
9420
9421	if (ioc->shost_recovery)
9422	return;
9423
9424	if (event_data->ReasonCode != MPI2_EVENT_IR_VOLUME_RC_STATE_CHANGED)
9425	return;
9426
9427	handle = le16_to_cpu(event_data->VolDevHandle);
9428	state = le32_to_cpu(event_data->NewValue);
9429	if (!ioc->hide_ir_msg)
9430	dewtprintk(ioc,
9431	ioc_info(ioc, "%s: handle(0x%04x), old(0x%08x), new(0x%08x)\n",
9432	__func__, handle,
9433	le32_to_cpu(event_data->PreviousValue),
9434	state));
9435	switch (state) {
9436	case MPI2_RAID_VOL_STATE_MISSING:
9437	case MPI2_RAID_VOL_STATE_FAILED:
9438	_scsih_sas_volume_delete(ioc, handle);
9439	break;
9440
9441	case MPI2_RAID_VOL_STATE_ONLINE:
9442	case MPI2_RAID_VOL_STATE_DEGRADED:
9443	case MPI2_RAID_VOL_STATE_OPTIMAL:
9444
9445	spin_lock_irqsave(&ioc->raid_device_lock, flags);
9446	raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
9447	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
9448
9449	if (raid_device)
9450	break;
9451
9452	mpt3sas_config_get_volume_wwid(ioc, volume_handle: handle, wwid: &wwid);
9453	if (!wwid) {
9454	ioc_err(ioc, "failure at %s:%d/%s()!\n",
9455	__FILE__, __LINE__, __func__);
9456	break;
9457	}
9458
9459	raid_device = kzalloc(size: sizeof(struct _raid_device), GFP_KERNEL);
9460	if (!raid_device) {
9461	ioc_err(ioc, "failure at %s:%d/%s()!\n",
9462	__FILE__, __LINE__, __func__);
9463	break;
9464	}
9465
9466	raid_device->id = ioc->sas_id++;
9467	raid_device->channel = RAID_CHANNEL;
9468	raid_device->handle = handle;
9469	raid_device->wwid = wwid;
9470	_scsih_raid_device_add(ioc, raid_device);
9471	rc = scsi_add_device(host: ioc->shost, RAID_CHANNEL,
9472	target: raid_device->id, lun: 0);
9473	if (rc)
9474	_scsih_raid_device_remove(ioc, raid_device);
9475	break;
9476
9477	case MPI2_RAID_VOL_STATE_INITIALIZING:
9478	default:
9479	break;
9480	}
9481	}
9482
9483	/**
9484	* _scsih_sas_ir_physical_disk_event - PD event
9485	* @ioc: per adapter object
9486	* @fw_event: The fw_event_work object
9487	* Context: user.
9488	*/
9489	static void
9490	_scsih_sas_ir_physical_disk_event(struct MPT3SAS_ADAPTER *ioc,
9491	struct fw_event_work *fw_event)
9492	{
9493	u16 handle, parent_handle;
9494	u32 state;
9495	struct _sas_device *sas_device;
9496	Mpi2ConfigReply_t mpi_reply;
9497	Mpi2SasDevicePage0_t sas_device_pg0;
9498	u32 ioc_status;
9499	Mpi2EventDataIrPhysicalDisk_t *event_data =
9500	(Mpi2EventDataIrPhysicalDisk_t *) fw_event->event_data;
9501	u64 sas_address;
9502
9503	if (ioc->shost_recovery)
9504	return;
9505
9506	if (event_data->ReasonCode != MPI2_EVENT_IR_PHYSDISK_RC_STATE_CHANGED)
9507	return;
9508
9509	handle = le16_to_cpu(event_data->PhysDiskDevHandle);
9510	state = le32_to_cpu(event_data->NewValue);
9511
9512	if (!ioc->hide_ir_msg)
9513	dewtprintk(ioc,
9514	ioc_info(ioc, "%s: handle(0x%04x), old(0x%08x), new(0x%08x)\n",
9515	__func__, handle,
9516	le32_to_cpu(event_data->PreviousValue),
9517	state));
9518
9519	switch (state) {
9520	case MPI2_RAID_PD_STATE_ONLINE:
9521	case MPI2_RAID_PD_STATE_DEGRADED:
9522	case MPI2_RAID_PD_STATE_REBUILDING:
9523	case MPI2_RAID_PD_STATE_OPTIMAL:
9524	case MPI2_RAID_PD_STATE_HOT_SPARE:
9525
9526	if (!ioc->is_warpdrive)
9527	set_bit(nr: handle, addr: ioc->pd_handles);
9528
9529	sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
9530	if (sas_device) {
9531	sas_device_put(s: sas_device);
9532	return;
9533	}
9534
9535	if ((mpt3sas_config_get_sas_device_pg0(ioc, mpi_reply: &mpi_reply,
9536	config_page: &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE,
9537	handle))) {
9538	ioc_err(ioc, "failure at %s:%d/%s()!\n",
9539	__FILE__, __LINE__, __func__);
9540	return;
9541	}
9542
9543	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
9544	MPI2_IOCSTATUS_MASK;
9545	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
9546	ioc_err(ioc, "failure at %s:%d/%s()!\n",
9547	__FILE__, __LINE__, __func__);
9548	return;
9549	}
9550
9551	parent_handle = le16_to_cpu(sas_device_pg0.ParentDevHandle);
9552	if (!_scsih_get_sas_address(ioc, handle: parent_handle, sas_address: &sas_address))
9553	mpt3sas_transport_update_links(ioc, sas_address, handle,
9554	phy_number: sas_device_pg0.PhyNum, MPI2_SAS_NEG_LINK_RATE_1_5,
9555	port: mpt3sas_get_port_by_id(ioc,
9556	port_id: sas_device_pg0.PhysicalPort, bypass_dirty_port_flag: 0));
9557
9558	_scsih_add_device(ioc, handle, phy_num: 0, is_pd: 1);
9559
9560	break;
9561
9562	case MPI2_RAID_PD_STATE_OFFLINE:
9563	case MPI2_RAID_PD_STATE_NOT_CONFIGURED:
9564	case MPI2_RAID_PD_STATE_NOT_COMPATIBLE:
9565	default:
9566	break;
9567	}
9568	}
9569
9570	/**
9571	* _scsih_sas_ir_operation_status_event_debug - debug for IR op event
9572	* @ioc: per adapter object
9573	* @event_data: event data payload
9574	* Context: user.
9575	*/
9576	static void
9577	_scsih_sas_ir_operation_status_event_debug(struct MPT3SAS_ADAPTER *ioc,
9578	Mpi2EventDataIrOperationStatus_t *event_data)
9579	{
9580	char *reason_str = NULL;
9581
9582	switch (event_data->RAIDOperation) {
9583	case MPI2_EVENT_IR_RAIDOP_RESYNC:
9584	reason_str = "resync";
9585	break;
9586	case MPI2_EVENT_IR_RAIDOP_ONLINE_CAP_EXPANSION:
9587	reason_str = "online capacity expansion";
9588	break;
9589	case MPI2_EVENT_IR_RAIDOP_CONSISTENCY_CHECK:
9590	reason_str = "consistency check";
9591	break;
9592	case MPI2_EVENT_IR_RAIDOP_BACKGROUND_INIT:
9593	reason_str = "background init";
9594	break;
9595	case MPI2_EVENT_IR_RAIDOP_MAKE_DATA_CONSISTENT:
9596	reason_str = "make data consistent";
9597	break;
9598	}
9599
9600	if (!reason_str)
9601	return;
9602
9603	ioc_info(ioc, "raid operational status: (%s)\thandle(0x%04x), percent complete(%d)\n",
9604	reason_str,
9605	le16_to_cpu(event_data->VolDevHandle),
9606	event_data->PercentComplete);
9607	}
9608
9609	/**
9610	* _scsih_sas_ir_operation_status_event - handle RAID operation events
9611	* @ioc: per adapter object
9612	* @fw_event: The fw_event_work object
9613	* Context: user.
9614	*/
9615	static void
9616	_scsih_sas_ir_operation_status_event(struct MPT3SAS_ADAPTER *ioc,
9617	struct fw_event_work *fw_event)
9618	{
9619	Mpi2EventDataIrOperationStatus_t *event_data =
9620	(Mpi2EventDataIrOperationStatus_t *)
9621	fw_event->event_data;
9622	static struct _raid_device *raid_device;
9623	unsigned long flags;
9624	u16 handle;
9625
9626	if ((ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) &&
9627	(!ioc->hide_ir_msg))
9628	_scsih_sas_ir_operation_status_event_debug(ioc,
9629	event_data);
9630
9631	/* code added for raid transport support */
9632	if (event_data->RAIDOperation == MPI2_EVENT_IR_RAIDOP_RESYNC) {
9633
9634	spin_lock_irqsave(&ioc->raid_device_lock, flags);
9635	handle = le16_to_cpu(event_data->VolDevHandle);
9636	raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
9637	if (raid_device)
9638	raid_device->percent_complete =
9639	event_data->PercentComplete;
9640	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
9641	}
9642	}
9643
9644	/**
9645	* _scsih_prep_device_scan - initialize parameters prior to device scan
9646	* @ioc: per adapter object
9647	*
9648	* Set the deleted flag prior to device scan. If the device is found during
9649	* the scan, then we clear the deleted flag.
9650	*/
9651	static void
9652	_scsih_prep_device_scan(struct MPT3SAS_ADAPTER *ioc)
9653	{
9654	struct MPT3SAS_DEVICE *sas_device_priv_data;
9655	struct scsi_device *sdev;
9656
9657	shost_for_each_device(sdev, ioc->shost) {
9658	sas_device_priv_data = sdev->hostdata;
9659	if (sas_device_priv_data && sas_device_priv_data->sas_target)
9660	sas_device_priv_data->sas_target->deleted = 1;
9661	}
9662	}
9663
9664	/**
9665	* _scsih_update_device_qdepth - Update QD during Reset.
9666	* @ioc: per adapter object
9667	*
9668	*/
9669	static void
9670	_scsih_update_device_qdepth(struct MPT3SAS_ADAPTER *ioc)
9671	{
9672	struct MPT3SAS_DEVICE *sas_device_priv_data;
9673	struct MPT3SAS_TARGET *sas_target_priv_data;
9674	struct _sas_device *sas_device;
9675	struct scsi_device *sdev;
9676	u16 qdepth;
9677
9678	ioc_info(ioc, "Update devices with firmware reported queue depth\n");
9679	shost_for_each_device(sdev, ioc->shost) {
9680	sas_device_priv_data = sdev->hostdata;
9681	if (sas_device_priv_data && sas_device_priv_data->sas_target) {
9682	sas_target_priv_data = sas_device_priv_data->sas_target;
9683	sas_device = sas_device_priv_data->sas_target->sas_dev;
9684	if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_PCIE_DEVICE)
9685	qdepth = ioc->max_nvme_qd;
9686	else if (sas_device &&
9687	sas_device->device_info & MPI2_SAS_DEVICE_INFO_SSP_TARGET)
9688	qdepth = (sas_device->port_type > 1) ?
9689	ioc->max_wideport_qd : ioc->max_narrowport_qd;
9690	else if (sas_device &&
9691	sas_device->device_info & MPI2_SAS_DEVICE_INFO_SATA_DEVICE)
9692	qdepth = ioc->max_sata_qd;
9693	else
9694	continue;
9695	mpt3sas_scsih_change_queue_depth(sdev, qdepth);
9696	}
9697	}
9698	}
9699
9700	/**
9701	* _scsih_mark_responding_sas_device - mark a sas_devices as responding
9702	* @ioc: per adapter object
9703	* @sas_device_pg0: SAS Device page 0
9704	*
9705	* After host reset, find out whether devices are still responding.
9706	* Used in _scsih_remove_unresponsive_sas_devices.
9707	*/
9708	static void
9709	_scsih_mark_responding_sas_device(struct MPT3SAS_ADAPTER *ioc,
9710	Mpi2SasDevicePage0_t *sas_device_pg0)
9711	{
9712	struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
9713	struct scsi_target *starget;
9714	struct _sas_device *sas_device = NULL;
9715	struct _enclosure_node *enclosure_dev = NULL;
9716	unsigned long flags;
9717	struct hba_port *port = mpt3sas_get_port_by_id(
9718	ioc, port_id: sas_device_pg0->PhysicalPort, bypass_dirty_port_flag: 0);
9719
9720	if (sas_device_pg0->EnclosureHandle) {
9721	enclosure_dev =
9722	mpt3sas_scsih_enclosure_find_by_handle(ioc,
9723	le16_to_cpu(sas_device_pg0->EnclosureHandle));
9724	if (enclosure_dev == NULL)
9725	ioc_info(ioc, "Enclosure handle(0x%04x) doesn't match with enclosure device!\n",
9726	sas_device_pg0->EnclosureHandle);
9727	}
9728	spin_lock_irqsave(&ioc->sas_device_lock, flags);
9729	list_for_each_entry(sas_device, &ioc->sas_device_list, list) {
9730	if (sas_device->sas_address != le64_to_cpu(
9731	sas_device_pg0->SASAddress))
9732	continue;
9733	if (sas_device->slot != le16_to_cpu(sas_device_pg0->Slot))
9734	continue;
9735	if (sas_device->port != port)
9736	continue;
9737	sas_device->responding = 1;
9738	starget = sas_device->starget;
9739	if (starget && starget->hostdata) {
9740	sas_target_priv_data = starget->hostdata;
9741	sas_target_priv_data->tm_busy = 0;
9742	sas_target_priv_data->deleted = 0;
9743	} else
9744	sas_target_priv_data = NULL;
9745	if (starget) {
9746	starget_printk(KERN_INFO, starget,
9747	"handle(0x%04x), sas_addr(0x%016llx)\n",
9748	le16_to_cpu(sas_device_pg0->DevHandle),
9749	(unsigned long long)
9750	sas_device->sas_address);
9751
9752	if (sas_device->enclosure_handle != 0)
9753	starget_printk(KERN_INFO, starget,
9754	"enclosure logical id(0x%016llx), slot(%d)\n",
9755	(unsigned long long)
9756	sas_device->enclosure_logical_id,
9757	sas_device->slot);
9758	}
9759	if (le16_to_cpu(sas_device_pg0->Flags) &
9760	MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
9761	sas_device->enclosure_level =
9762	sas_device_pg0->EnclosureLevel;
9763	memcpy(&sas_device->connector_name[0],
9764	&sas_device_pg0->ConnectorName[0], 4);
9765	} else {
9766	sas_device->enclosure_level = 0;
9767	sas_device->connector_name[0] = '\0';
9768	}
9769
9770	sas_device->enclosure_handle =
9771	le16_to_cpu(sas_device_pg0->EnclosureHandle);
9772	sas_device->is_chassis_slot_valid = 0;
9773	if (enclosure_dev) {
9774	sas_device->enclosure_logical_id = le64_to_cpu(
9775	enclosure_dev->pg0.EnclosureLogicalID);
9776	if (le16_to_cpu(enclosure_dev->pg0.Flags) &
9777	MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
9778	sas_device->is_chassis_slot_valid = 1;
9779	sas_device->chassis_slot =
9780	enclosure_dev->pg0.ChassisSlot;
9781	}
9782	}
9783
9784	if (sas_device->handle == le16_to_cpu(
9785	sas_device_pg0->DevHandle))
9786	goto out;
9787	pr_info("\thandle changed from(0x%04x)!!!\n",
9788	sas_device->handle);
9789	sas_device->handle = le16_to_cpu(
9790	sas_device_pg0->DevHandle);
9791	if (sas_target_priv_data)
9792	sas_target_priv_data->handle =
9793	le16_to_cpu(sas_device_pg0->DevHandle);
9794	goto out;
9795	}
9796	out:
9797	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
9798	}
9799
9800	/**
9801	* _scsih_create_enclosure_list_after_reset - Free Existing list,
9802	* And create enclosure list by scanning all Enclosure Page(0)s
9803	* @ioc: per adapter object
9804	*/
9805	static void
9806	_scsih_create_enclosure_list_after_reset(struct MPT3SAS_ADAPTER *ioc)
9807	{
9808	struct _enclosure_node *enclosure_dev;
9809	Mpi2ConfigReply_t mpi_reply;
9810	u16 enclosure_handle;
9811	int rc;
9812
9813	/* Free existing enclosure list */
9814	mpt3sas_free_enclosure_list(ioc);
9815
9816	/* Re constructing enclosure list after reset*/
9817	enclosure_handle = 0xFFFF;
9818	do {
9819	enclosure_dev =
9820	kzalloc(size: sizeof(struct _enclosure_node), GFP_KERNEL);
9821	if (!enclosure_dev) {
9822	ioc_err(ioc, "failure at %s:%d/%s()!\n",
9823	__FILE__, __LINE__, __func__);
9824	return;
9825	}
9826	rc = mpt3sas_config_get_enclosure_pg0(ioc, mpi_reply: &mpi_reply,
9827	config_page: &enclosure_dev->pg0,
9828	MPI2_SAS_ENCLOS_PGAD_FORM_GET_NEXT_HANDLE,
9829	handle: enclosure_handle);
9830
9831	if (rc || (le16_to_cpu(mpi_reply.IOCStatus) &
9832	MPI2_IOCSTATUS_MASK)) {
9833	kfree(objp: enclosure_dev);
9834	return;
9835	}
9836	list_add_tail(new: &enclosure_dev->list,
9837	head: &ioc->enclosure_list);
9838	enclosure_handle =
9839	le16_to_cpu(enclosure_dev->pg0.EnclosureHandle);
9840	} while (1);
9841	}
9842
9843	/**
9844	* _scsih_search_responding_sas_devices -
9845	* @ioc: per adapter object
9846	*
9847	* After host reset, find out whether devices are still responding.
9848	* If not remove.
9849	*/
9850	static void
9851	_scsih_search_responding_sas_devices(struct MPT3SAS_ADAPTER *ioc)
9852	{
9853	Mpi2SasDevicePage0_t sas_device_pg0;
9854	Mpi2ConfigReply_t mpi_reply;
9855	u16 ioc_status;
9856	u16 handle;
9857	u32 device_info;
9858
9859	ioc_info(ioc, "search for end-devices: start\n");
9860
9861	if (list_empty(head: &ioc->sas_device_list))
9862	goto out;
9863
9864	handle = 0xFFFF;
9865	while (!(mpt3sas_config_get_sas_device_pg0(ioc, mpi_reply: &mpi_reply,
9866	config_page: &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_GET_NEXT_HANDLE,
9867	handle))) {
9868	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
9869	MPI2_IOCSTATUS_MASK;
9870	if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
9871	break;
9872	handle = le16_to_cpu(sas_device_pg0.DevHandle);
9873	device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
9874	if (!(_scsih_is_end_device(device_info)))
9875	continue;
9876	_scsih_mark_responding_sas_device(ioc, sas_device_pg0: &sas_device_pg0);
9877	}
9878
9879	out:
9880	ioc_info(ioc, "search for end-devices: complete\n");
9881	}
9882
9883	/**
9884	* _scsih_mark_responding_pcie_device - mark a pcie_device as responding
9885	* @ioc: per adapter object
9886	* @pcie_device_pg0: PCIe Device page 0
9887	*
9888	* After host reset, find out whether devices are still responding.
9889	* Used in _scsih_remove_unresponding_devices.
9890	*/
9891	static void
9892	_scsih_mark_responding_pcie_device(struct MPT3SAS_ADAPTER *ioc,
9893	Mpi26PCIeDevicePage0_t *pcie_device_pg0)
9894	{
9895	struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
9896	struct scsi_target *starget;
9897	struct _pcie_device *pcie_device;
9898	unsigned long flags;
9899
9900	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
9901	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list) {
9902	if ((pcie_device->wwid == le64_to_cpu(pcie_device_pg0->WWID))
9903	&& (pcie_device->slot == le16_to_cpu(
9904	pcie_device_pg0->Slot))) {
9905	pcie_device->access_status =
9906	pcie_device_pg0->AccessStatus;
9907	pcie_device->responding = 1;
9908	starget = pcie_device->starget;
9909	if (starget && starget->hostdata) {
9910	sas_target_priv_data = starget->hostdata;
9911	sas_target_priv_data->tm_busy = 0;
9912	sas_target_priv_data->deleted = 0;
9913	} else
9914	sas_target_priv_data = NULL;
9915	if (starget) {
9916	starget_printk(KERN_INFO, starget,
9917	"handle(0x%04x), wwid(0x%016llx) ",
9918	pcie_device->handle,
9919	(unsigned long long)pcie_device->wwid);
9920	if (pcie_device->enclosure_handle != 0)
9921	starget_printk(KERN_INFO, starget,
9922	"enclosure logical id(0x%016llx), "
9923	"slot(%d)\n",
9924	(unsigned long long)
9925	pcie_device->enclosure_logical_id,
9926	pcie_device->slot);
9927	}
9928
9929	if (((le32_to_cpu(pcie_device_pg0->Flags)) &
9930	MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID) &&
9931	(ioc->hba_mpi_version_belonged != MPI2_VERSION)) {
9932	pcie_device->enclosure_level =
9933	pcie_device_pg0->EnclosureLevel;
9934	memcpy(&pcie_device->connector_name[0],
9935	&pcie_device_pg0->ConnectorName[0], 4);
9936	} else {
9937	pcie_device->enclosure_level = 0;
9938	pcie_device->connector_name[0] = '\0';
9939	}
9940
9941	if (pcie_device->handle == le16_to_cpu(
9942	pcie_device_pg0->DevHandle))
9943	goto out;
9944	pr_info("\thandle changed from(0x%04x)!!!\n",
9945	pcie_device->handle);
9946	pcie_device->handle = le16_to_cpu(
9947	pcie_device_pg0->DevHandle);
9948	if (sas_target_priv_data)
9949	sas_target_priv_data->handle =
9950	le16_to_cpu(pcie_device_pg0->DevHandle);
9951	goto out;
9952	}
9953	}
9954
9955	out:
9956	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
9957	}
9958
9959	/**
9960	* _scsih_search_responding_pcie_devices -
9961	* @ioc: per adapter object
9962	*
9963	* After host reset, find out whether devices are still responding.
9964	* If not remove.
9965	*/
9966	static void
9967	_scsih_search_responding_pcie_devices(struct MPT3SAS_ADAPTER *ioc)
9968	{
9969	Mpi26PCIeDevicePage0_t pcie_device_pg0;
9970	Mpi2ConfigReply_t mpi_reply;
9971	u16 ioc_status;
9972	u16 handle;
9973	u32 device_info;
9974
9975	ioc_info(ioc, "search for end-devices: start\n");
9976
9977	if (list_empty(head: &ioc->pcie_device_list))
9978	goto out;
9979
9980	handle = 0xFFFF;
9981	while (!(mpt3sas_config_get_pcie_device_pg0(ioc, mpi_reply: &mpi_reply,
9982	config_page: &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_GET_NEXT_HANDLE,
9983	handle))) {
9984	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
9985	MPI2_IOCSTATUS_MASK;
9986	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
9987	ioc_info(ioc, "\tbreak from %s: ioc_status(0x%04x), loginfo(0x%08x)\n",
9988	__func__, ioc_status,
9989	le32_to_cpu(mpi_reply.IOCLogInfo));
9990	break;
9991	}
9992	handle = le16_to_cpu(pcie_device_pg0.DevHandle);
9993	device_info = le32_to_cpu(pcie_device_pg0.DeviceInfo);
9994	if (!(_scsih_is_nvme_pciescsi_device(device_info)))
9995	continue;
9996	_scsih_mark_responding_pcie_device(ioc, pcie_device_pg0: &pcie_device_pg0);
9997	}
9998	out:
9999	ioc_info(ioc, "search for PCIe end-devices: complete\n");
10000	}
10001
10002	/**
10003	* _scsih_mark_responding_raid_device - mark a raid_device as responding
10004	* @ioc: per adapter object
10005	* @wwid: world wide identifier for raid volume
10006	* @handle: device handle
10007	*
10008	* After host reset, find out whether devices are still responding.
10009	* Used in _scsih_remove_unresponsive_raid_devices.
10010	*/
10011	static void
10012	_scsih_mark_responding_raid_device(struct MPT3SAS_ADAPTER *ioc, u64 wwid,
10013	u16 handle)
10014	{
10015	struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
10016	struct scsi_target *starget;
10017	struct _raid_device *raid_device;
10018	unsigned long flags;
10019
10020	spin_lock_irqsave(&ioc->raid_device_lock, flags);
10021	list_for_each_entry(raid_device, &ioc->raid_device_list, list) {
10022	if (raid_device->wwid == wwid && raid_device->starget) {
10023	starget = raid_device->starget;
10024	if (starget && starget->hostdata) {
10025	sas_target_priv_data = starget->hostdata;
10026	sas_target_priv_data->deleted = 0;
10027	} else
10028	sas_target_priv_data = NULL;
10029	raid_device->responding = 1;
10030	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
10031	starget_printk(KERN_INFO, raid_device->starget,
10032	"handle(0x%04x), wwid(0x%016llx)\n", handle,
10033	(unsigned long long)raid_device->wwid);
10034
10035	/*
10036	* WARPDRIVE: The handles of the PDs might have changed
10037	* across the host reset so re-initialize the
10038	* required data for Direct IO
10039	*/
10040	mpt3sas_init_warpdrive_properties(ioc, raid_device);
10041	spin_lock_irqsave(&ioc->raid_device_lock, flags);
10042	if (raid_device->handle == handle) {
10043	spin_unlock_irqrestore(lock: &ioc->raid_device_lock,
10044	flags);
10045	return;
10046	}
10047	pr_info("\thandle changed from(0x%04x)!!!\n",
10048	raid_device->handle);
10049	raid_device->handle = handle;
10050	if (sas_target_priv_data)
10051	sas_target_priv_data->handle = handle;
10052	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
10053	return;
10054	}
10055	}
10056	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
10057	}
10058
10059	/**
10060	* _scsih_search_responding_raid_devices -
10061	* @ioc: per adapter object
10062	*
10063	* After host reset, find out whether devices are still responding.
10064	* If not remove.
10065	*/
10066	static void
10067	_scsih_search_responding_raid_devices(struct MPT3SAS_ADAPTER *ioc)
10068	{
10069	Mpi2RaidVolPage1_t volume_pg1;
10070	Mpi2RaidVolPage0_t volume_pg0;
10071	Mpi2RaidPhysDiskPage0_t pd_pg0;
10072	Mpi2ConfigReply_t mpi_reply;
10073	u16 ioc_status;
10074	u16 handle;
10075	u8 phys_disk_num;
10076
10077	if (!ioc->ir_firmware)
10078	return;
10079
10080	ioc_info(ioc, "search for raid volumes: start\n");
10081
10082	if (list_empty(head: &ioc->raid_device_list))
10083	goto out;
10084
10085	handle = 0xFFFF;
10086	while (!(mpt3sas_config_get_raid_volume_pg1(ioc, mpi_reply: &mpi_reply,
10087	config_page: &volume_pg1, MPI2_RAID_VOLUME_PGAD_FORM_GET_NEXT_HANDLE, handle))) {
10088	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10089	MPI2_IOCSTATUS_MASK;
10090	if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
10091	break;
10092	handle = le16_to_cpu(volume_pg1.DevHandle);
10093
10094	if (mpt3sas_config_get_raid_volume_pg0(ioc, mpi_reply: &mpi_reply,
10095	config_page: &volume_pg0, MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
10096	sz: sizeof(Mpi2RaidVolPage0_t)))
10097	continue;
10098
10099	if (volume_pg0.VolumeState == MPI2_RAID_VOL_STATE_OPTIMAL ||
10100	volume_pg0.VolumeState == MPI2_RAID_VOL_STATE_ONLINE ||
10101	volume_pg0.VolumeState == MPI2_RAID_VOL_STATE_DEGRADED)
10102	_scsih_mark_responding_raid_device(ioc,
10103	le64_to_cpu(volume_pg1.WWID), handle);
10104	}
10105
10106	/* refresh the pd_handles */
10107	if (!ioc->is_warpdrive) {
10108	phys_disk_num = 0xFF;
10109	memset(ioc->pd_handles, 0, ioc->pd_handles_sz);
10110	while (!(mpt3sas_config_get_phys_disk_pg0(ioc, mpi_reply: &mpi_reply,
10111	config_page: &pd_pg0, MPI2_PHYSDISK_PGAD_FORM_GET_NEXT_PHYSDISKNUM,
10112	form_specific: phys_disk_num))) {
10113	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10114	MPI2_IOCSTATUS_MASK;
10115	if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
10116	break;
10117	phys_disk_num = pd_pg0.PhysDiskNum;
10118	handle = le16_to_cpu(pd_pg0.DevHandle);
10119	set_bit(nr: handle, addr: ioc->pd_handles);
10120	}
10121	}
10122	out:
10123	ioc_info(ioc, "search for responding raid volumes: complete\n");
10124	}
10125
10126	/**
10127	* _scsih_mark_responding_expander - mark a expander as responding
10128	* @ioc: per adapter object
10129	* @expander_pg0:SAS Expander Config Page0
10130	*
10131	* After host reset, find out whether devices are still responding.
10132	* Used in _scsih_remove_unresponsive_expanders.
10133	*/
10134	static void
10135	_scsih_mark_responding_expander(struct MPT3SAS_ADAPTER *ioc,
10136	Mpi2ExpanderPage0_t *expander_pg0)
10137	{
10138	struct _sas_node *sas_expander = NULL;
10139	unsigned long flags;
10140	int i;
10141	struct _enclosure_node *enclosure_dev = NULL;
10142	u16 handle = le16_to_cpu(expander_pg0->DevHandle);
10143	u16 enclosure_handle = le16_to_cpu(expander_pg0->EnclosureHandle);
10144	u64 sas_address = le64_to_cpu(expander_pg0->SASAddress);
10145	struct hba_port *port = mpt3sas_get_port_by_id(
10146	ioc, port_id: expander_pg0->PhysicalPort, bypass_dirty_port_flag: 0);
10147
10148	if (enclosure_handle)
10149	enclosure_dev =
10150	mpt3sas_scsih_enclosure_find_by_handle(ioc,
10151	handle: enclosure_handle);
10152
10153	spin_lock_irqsave(&ioc->sas_node_lock, flags);
10154	list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {
10155	if (sas_expander->sas_address != sas_address)
10156	continue;
10157	if (sas_expander->port != port)
10158	continue;
10159	sas_expander->responding = 1;
10160
10161	if (enclosure_dev) {
10162	sas_expander->enclosure_logical_id =
10163	le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
10164	sas_expander->enclosure_handle =
10165	le16_to_cpu(expander_pg0->EnclosureHandle);
10166	}
10167
10168	if (sas_expander->handle == handle)
10169	goto out;
10170	pr_info("\texpander(0x%016llx): handle changed" \
10171	" from(0x%04x) to (0x%04x)!!!\n",
10172	(unsigned long long)sas_expander->sas_address,
10173	sas_expander->handle, handle);
10174	sas_expander->handle = handle;
10175	for (i = 0 ; i < sas_expander->num_phys ; i++)
10176	sas_expander->phy[i].handle = handle;
10177	goto out;
10178	}
10179	out:
10180	spin_unlock_irqrestore(lock: &ioc->sas_node_lock, flags);
10181	}
10182
10183	/**
10184	* _scsih_search_responding_expanders -
10185	* @ioc: per adapter object
10186	*
10187	* After host reset, find out whether devices are still responding.
10188	* If not remove.
10189	*/
10190	static void
10191	_scsih_search_responding_expanders(struct MPT3SAS_ADAPTER *ioc)
10192	{
10193	Mpi2ExpanderPage0_t expander_pg0;
10194	Mpi2ConfigReply_t mpi_reply;
10195	u16 ioc_status;
10196	u64 sas_address;
10197	u16 handle;
10198	u8 port;
10199
10200	ioc_info(ioc, "search for expanders: start\n");
10201
10202	if (list_empty(head: &ioc->sas_expander_list))
10203	goto out;
10204
10205	handle = 0xFFFF;
10206	while (!(mpt3sas_config_get_expander_pg0(ioc, mpi_reply: &mpi_reply, config_page: &expander_pg0,
10207	MPI2_SAS_EXPAND_PGAD_FORM_GET_NEXT_HNDL, handle))) {
10208
10209	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10210	MPI2_IOCSTATUS_MASK;
10211	if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
10212	break;
10213
10214	handle = le16_to_cpu(expander_pg0.DevHandle);
10215	sas_address = le64_to_cpu(expander_pg0.SASAddress);
10216	port = expander_pg0.PhysicalPort;
10217	pr_info(
10218	"\texpander present: handle(0x%04x), sas_addr(0x%016llx), port:%d\n",
10219	handle, (unsigned long long)sas_address,
10220	(ioc->multipath_on_hba ?
10221	port : MULTIPATH_DISABLED_PORT_ID));
10222	_scsih_mark_responding_expander(ioc, expander_pg0: &expander_pg0);
10223	}
10224
10225	out:
10226	ioc_info(ioc, "search for expanders: complete\n");
10227	}
10228
10229	/**
10230	* _scsih_remove_unresponding_devices - removing unresponding devices
10231	* @ioc: per adapter object
10232	*/
10233	static void
10234	_scsih_remove_unresponding_devices(struct MPT3SAS_ADAPTER *ioc)
10235	{
10236	struct _sas_device *sas_device, *sas_device_next;
10237	struct _sas_node *sas_expander, *sas_expander_next;
10238	struct _raid_device *raid_device, *raid_device_next;
10239	struct _pcie_device *pcie_device, *pcie_device_next;
10240	struct list_head tmp_list;
10241	unsigned long flags;
10242	LIST_HEAD(head);
10243
10244	ioc_info(ioc, "removing unresponding devices: start\n");
10245
10246	/* removing unresponding end devices */
10247	ioc_info(ioc, "removing unresponding devices: end-devices\n");
10248	/*
10249	* Iterate, pulling off devices marked as non-responding. We become the
10250	* owner for the reference the list had on any object we prune.
10251	*/
10252	spin_lock_irqsave(&ioc->sas_device_lock, flags);
10253
10254	/*
10255	* Clean up the sas_device_init_list list as
10256	* driver goes for fresh scan as part of diag reset.
10257	*/
10258	list_for_each_entry_safe(sas_device, sas_device_next,
10259	&ioc->sas_device_init_list, list) {
10260	list_del_init(entry: &sas_device->list);
10261	sas_device_put(s: sas_device);
10262	}
10263
10264	list_for_each_entry_safe(sas_device, sas_device_next,
10265	&ioc->sas_device_list, list) {
10266	if (!sas_device->responding)
10267	list_move_tail(list: &sas_device->list, head: &head);
10268	else
10269	sas_device->responding = 0;
10270	}
10271	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
10272
10273	/*
10274	* Now, uninitialize and remove the unresponding devices we pruned.
10275	*/
10276	list_for_each_entry_safe(sas_device, sas_device_next, &head, list) {
10277	_scsih_remove_device(ioc, sas_device);
10278	list_del_init(entry: &sas_device->list);
10279	sas_device_put(s: sas_device);
10280	}
10281
10282	ioc_info(ioc, "Removing unresponding devices: pcie end-devices\n");
10283	INIT_LIST_HEAD(list: &head);
10284	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
10285	/*
10286	* Clean up the pcie_device_init_list list as
10287	* driver goes for fresh scan as part of diag reset.
10288	*/
10289	list_for_each_entry_safe(pcie_device, pcie_device_next,
10290	&ioc->pcie_device_init_list, list) {
10291	list_del_init(entry: &pcie_device->list);
10292	pcie_device_put(p: pcie_device);
10293	}
10294
10295	list_for_each_entry_safe(pcie_device, pcie_device_next,
10296	&ioc->pcie_device_list, list) {
10297	if (!pcie_device->responding)
10298	list_move_tail(list: &pcie_device->list, head: &head);
10299	else
10300	pcie_device->responding = 0;
10301	}
10302	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
10303
10304	list_for_each_entry_safe(pcie_device, pcie_device_next, &head, list) {
10305	_scsih_pcie_device_remove_from_sml(ioc, pcie_device);
10306	list_del_init(entry: &pcie_device->list);
10307	pcie_device_put(p: pcie_device);
10308	}
10309
10310	/* removing unresponding volumes */
10311	if (ioc->ir_firmware) {
10312	ioc_info(ioc, "removing unresponding devices: volumes\n");
10313	list_for_each_entry_safe(raid_device, raid_device_next,
10314	&ioc->raid_device_list, list) {
10315	if (!raid_device->responding)
10316	_scsih_sas_volume_delete(ioc,
10317	handle: raid_device->handle);
10318	else
10319	raid_device->responding = 0;
10320	}
10321	}
10322
10323	/* removing unresponding expanders */
10324	ioc_info(ioc, "removing unresponding devices: expanders\n");
10325	spin_lock_irqsave(&ioc->sas_node_lock, flags);
10326	INIT_LIST_HEAD(list: &tmp_list);
10327	list_for_each_entry_safe(sas_expander, sas_expander_next,
10328	&ioc->sas_expander_list, list) {
10329	if (!sas_expander->responding)
10330	list_move_tail(list: &sas_expander->list, head: &tmp_list);
10331	else
10332	sas_expander->responding = 0;
10333	}
10334	spin_unlock_irqrestore(lock: &ioc->sas_node_lock, flags);
10335	list_for_each_entry_safe(sas_expander, sas_expander_next, &tmp_list,
10336	list) {
10337	_scsih_expander_node_remove(ioc, sas_expander);
10338	}
10339
10340	ioc_info(ioc, "removing unresponding devices: complete\n");
10341
10342	/* unblock devices */
10343	_scsih_ublock_io_all_device(ioc);
10344	}
10345
10346	static void
10347	_scsih_refresh_expander_links(struct MPT3SAS_ADAPTER *ioc,
10348	struct _sas_node *sas_expander, u16 handle)
10349	{
10350	Mpi2ExpanderPage1_t expander_pg1;
10351	Mpi2ConfigReply_t mpi_reply;
10352	int i;
10353
10354	for (i = 0 ; i < sas_expander->num_phys ; i++) {
10355	if ((mpt3sas_config_get_expander_pg1(ioc, mpi_reply: &mpi_reply,
10356	config_page: &expander_pg1, phy_number: i, handle))) {
10357	ioc_err(ioc, "failure at %s:%d/%s()!\n",
10358	__FILE__, __LINE__, __func__);
10359	return;
10360	}
10361
10362	mpt3sas_transport_update_links(ioc, sas_address: sas_expander->sas_address,
10363	le16_to_cpu(expander_pg1.AttachedDevHandle), phy_number: i,
10364	link_rate: expander_pg1.NegotiatedLinkRate >> 4,
10365	port: sas_expander->port);
10366	}
10367	}
10368
10369	/**
10370	* _scsih_scan_for_devices_after_reset - scan for devices after host reset
10371	* @ioc: per adapter object
10372	*/
10373	static void
10374	_scsih_scan_for_devices_after_reset(struct MPT3SAS_ADAPTER *ioc)
10375	{
10376	Mpi2ExpanderPage0_t expander_pg0;
10377	Mpi2SasDevicePage0_t sas_device_pg0;
10378	Mpi26PCIeDevicePage0_t pcie_device_pg0;
10379	Mpi2RaidVolPage1_t *volume_pg1;
10380	Mpi2RaidVolPage0_t *volume_pg0;
10381	Mpi2RaidPhysDiskPage0_t pd_pg0;
10382	Mpi2EventIrConfigElement_t element;
10383	Mpi2ConfigReply_t mpi_reply;
10384	u8 phys_disk_num, port_id;
10385	u16 ioc_status;
10386	u16 handle, parent_handle;
10387	u64 sas_address;
10388	struct _sas_device *sas_device;
10389	struct _pcie_device *pcie_device;
10390	struct _sas_node *expander_device;
10391	static struct _raid_device *raid_device;
10392	u8 retry_count;
10393	unsigned long flags;
10394
10395	volume_pg0 = kzalloc(size: sizeof(*volume_pg0), GFP_KERNEL);
10396	if (!volume_pg0)
10397	return;
10398
10399	volume_pg1 = kzalloc(size: sizeof(*volume_pg1), GFP_KERNEL);
10400	if (!volume_pg1) {
10401	kfree(objp: volume_pg0);
10402	return;
10403	}
10404
10405	ioc_info(ioc, "scan devices: start\n");
10406
10407	_scsih_sas_host_refresh(ioc);
10408
10409	ioc_info(ioc, "\tscan devices: expanders start\n");
10410
10411	/* expanders */
10412	handle = 0xFFFF;
10413	while (!(mpt3sas_config_get_expander_pg0(ioc, mpi_reply: &mpi_reply, config_page: &expander_pg0,
10414	MPI2_SAS_EXPAND_PGAD_FORM_GET_NEXT_HNDL, handle))) {
10415	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10416	MPI2_IOCSTATUS_MASK;
10417	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10418	ioc_info(ioc, "\tbreak from expander scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10419	ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10420	break;
10421	}
10422	handle = le16_to_cpu(expander_pg0.DevHandle);
10423	spin_lock_irqsave(&ioc->sas_node_lock, flags);
10424	port_id = expander_pg0.PhysicalPort;
10425	expander_device = mpt3sas_scsih_expander_find_by_sas_address(
10426	ioc, le64_to_cpu(expander_pg0.SASAddress),
10427	port: mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0));
10428	spin_unlock_irqrestore(lock: &ioc->sas_node_lock, flags);
10429	if (expander_device)
10430	_scsih_refresh_expander_links(ioc, sas_expander: expander_device,
10431	handle);
10432	else {
10433	ioc_info(ioc, "\tBEFORE adding expander: handle (0x%04x), sas_addr(0x%016llx)\n",
10434	handle,
10435	(u64)le64_to_cpu(expander_pg0.SASAddress));
10436	_scsih_expander_add(ioc, handle);
10437	ioc_info(ioc, "\tAFTER adding expander: handle (0x%04x), sas_addr(0x%016llx)\n",
10438	handle,
10439	(u64)le64_to_cpu(expander_pg0.SASAddress));
10440	}
10441	}
10442
10443	ioc_info(ioc, "\tscan devices: expanders complete\n");
10444
10445	if (!ioc->ir_firmware)
10446	goto skip_to_sas;
10447
10448	ioc_info(ioc, "\tscan devices: phys disk start\n");
10449
10450	/* phys disk */
10451	phys_disk_num = 0xFF;
10452	while (!(mpt3sas_config_get_phys_disk_pg0(ioc, mpi_reply: &mpi_reply,
10453	config_page: &pd_pg0, MPI2_PHYSDISK_PGAD_FORM_GET_NEXT_PHYSDISKNUM,
10454	form_specific: phys_disk_num))) {
10455	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10456	MPI2_IOCSTATUS_MASK;
10457	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10458	ioc_info(ioc, "\tbreak from phys disk scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10459	ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10460	break;
10461	}
10462	phys_disk_num = pd_pg0.PhysDiskNum;
10463	handle = le16_to_cpu(pd_pg0.DevHandle);
10464	sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
10465	if (sas_device) {
10466	sas_device_put(s: sas_device);
10467	continue;
10468	}
10469	if (mpt3sas_config_get_sas_device_pg0(ioc, mpi_reply: &mpi_reply,
10470	config_page: &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE,
10471	handle) != 0)
10472	continue;
10473	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10474	MPI2_IOCSTATUS_MASK;
10475	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10476	ioc_info(ioc, "\tbreak from phys disk scan ioc_status(0x%04x), loginfo(0x%08x)\n",
10477	ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10478	break;
10479	}
10480	parent_handle = le16_to_cpu(sas_device_pg0.ParentDevHandle);
10481	if (!_scsih_get_sas_address(ioc, handle: parent_handle,
10482	sas_address: &sas_address)) {
10483	ioc_info(ioc, "\tBEFORE adding phys disk: handle (0x%04x), sas_addr(0x%016llx)\n",
10484	handle,
10485	(u64)le64_to_cpu(sas_device_pg0.SASAddress));
10486	port_id = sas_device_pg0.PhysicalPort;
10487	mpt3sas_transport_update_links(ioc, sas_address,
10488	handle, phy_number: sas_device_pg0.PhyNum,
10489	MPI2_SAS_NEG_LINK_RATE_1_5,
10490	port: mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0));
10491	set_bit(nr: handle, addr: ioc->pd_handles);
10492	retry_count = 0;
10493	/* This will retry adding the end device.
10494	* _scsih_add_device() will decide on retries and
10495	* return "1" when it should be retried
10496	*/
10497	while (_scsih_add_device(ioc, handle, phy_num: retry_count++,
10498	is_pd: 1)) {
10499	ssleep(seconds: 1);
10500	}
10501	ioc_info(ioc, "\tAFTER adding phys disk: handle (0x%04x), sas_addr(0x%016llx)\n",
10502	handle,
10503	(u64)le64_to_cpu(sas_device_pg0.SASAddress));
10504	}
10505	}
10506
10507	ioc_info(ioc, "\tscan devices: phys disk complete\n");
10508
10509	ioc_info(ioc, "\tscan devices: volumes start\n");
10510
10511	/* volumes */
10512	handle = 0xFFFF;
10513	while (!(mpt3sas_config_get_raid_volume_pg1(ioc, mpi_reply: &mpi_reply,
10514	config_page: volume_pg1, MPI2_RAID_VOLUME_PGAD_FORM_GET_NEXT_HANDLE, handle))) {
10515	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10516	MPI2_IOCSTATUS_MASK;
10517	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10518	ioc_info(ioc, "\tbreak from volume scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10519	ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10520	break;
10521	}
10522	handle = le16_to_cpu(volume_pg1->DevHandle);
10523	spin_lock_irqsave(&ioc->raid_device_lock, flags);
10524	raid_device = _scsih_raid_device_find_by_wwid(ioc,
10525	le64_to_cpu(volume_pg1->WWID));
10526	spin_unlock_irqrestore(lock: &ioc->raid_device_lock, flags);
10527	if (raid_device)
10528	continue;
10529	if (mpt3sas_config_get_raid_volume_pg0(ioc, mpi_reply: &mpi_reply,
10530	config_page: volume_pg0, MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
10531	sz: sizeof(Mpi2RaidVolPage0_t)))
10532	continue;
10533	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10534	MPI2_IOCSTATUS_MASK;
10535	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10536	ioc_info(ioc, "\tbreak from volume scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10537	ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10538	break;
10539	}
10540	if (volume_pg0->VolumeState == MPI2_RAID_VOL_STATE_OPTIMAL ||
10541	volume_pg0->VolumeState == MPI2_RAID_VOL_STATE_ONLINE ||
10542	volume_pg0->VolumeState == MPI2_RAID_VOL_STATE_DEGRADED) {
10543	memset(&element, 0, sizeof(Mpi2EventIrConfigElement_t));
10544	element.ReasonCode = MPI2_EVENT_IR_CHANGE_RC_ADDED;
10545	element.VolDevHandle = volume_pg1->DevHandle;
10546	ioc_info(ioc, "\tBEFORE adding volume: handle (0x%04x)\n",
10547	volume_pg1->DevHandle);
10548	_scsih_sas_volume_add(ioc, element: &element);
10549	ioc_info(ioc, "\tAFTER adding volume: handle (0x%04x)\n",
10550	volume_pg1->DevHandle);
10551	}
10552	}
10553
10554	ioc_info(ioc, "\tscan devices: volumes complete\n");
10555
10556	skip_to_sas:
10557
10558	ioc_info(ioc, "\tscan devices: end devices start\n");
10559
10560	/* sas devices */
10561	handle = 0xFFFF;
10562	while (!(mpt3sas_config_get_sas_device_pg0(ioc, mpi_reply: &mpi_reply,
10563	config_page: &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_GET_NEXT_HANDLE,
10564	handle))) {
10565	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10566	MPI2_IOCSTATUS_MASK;
10567	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10568	ioc_info(ioc, "\tbreak from end device scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10569	ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10570	break;
10571	}
10572	handle = le16_to_cpu(sas_device_pg0.DevHandle);
10573	if (!(_scsih_is_end_device(
10574	le32_to_cpu(sas_device_pg0.DeviceInfo))))
10575	continue;
10576	port_id = sas_device_pg0.PhysicalPort;
10577	sas_device = mpt3sas_get_sdev_by_addr(ioc,
10578	le64_to_cpu(sas_device_pg0.SASAddress),
10579	port: mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0));
10580	if (sas_device) {
10581	sas_device_put(s: sas_device);
10582	continue;
10583	}
10584	parent_handle = le16_to_cpu(sas_device_pg0.ParentDevHandle);
10585	if (!_scsih_get_sas_address(ioc, handle: parent_handle, sas_address: &sas_address)) {
10586	ioc_info(ioc, "\tBEFORE adding end device: handle (0x%04x), sas_addr(0x%016llx)\n",
10587	handle,
10588	(u64)le64_to_cpu(sas_device_pg0.SASAddress));
10589	mpt3sas_transport_update_links(ioc, sas_address, handle,
10590	phy_number: sas_device_pg0.PhyNum, MPI2_SAS_NEG_LINK_RATE_1_5,
10591	port: mpt3sas_get_port_by_id(ioc, port_id, bypass_dirty_port_flag: 0));
10592	retry_count = 0;
10593	/* This will retry adding the end device.
10594	* _scsih_add_device() will decide on retries and
10595	* return "1" when it should be retried
10596	*/
10597	while (_scsih_add_device(ioc, handle, phy_num: retry_count++,
10598	is_pd: 0)) {
10599	ssleep(seconds: 1);
10600	}
10601	ioc_info(ioc, "\tAFTER adding end device: handle (0x%04x), sas_addr(0x%016llx)\n",
10602	handle,
10603	(u64)le64_to_cpu(sas_device_pg0.SASAddress));
10604	}
10605	}
10606	ioc_info(ioc, "\tscan devices: end devices complete\n");
10607	ioc_info(ioc, "\tscan devices: pcie end devices start\n");
10608
10609	/* pcie devices */
10610	handle = 0xFFFF;
10611	while (!(mpt3sas_config_get_pcie_device_pg0(ioc, mpi_reply: &mpi_reply,
10612	config_page: &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_GET_NEXT_HANDLE,
10613	handle))) {
10614	ioc_status = le16_to_cpu(mpi_reply.IOCStatus)
10615	& MPI2_IOCSTATUS_MASK;
10616	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10617	ioc_info(ioc, "\tbreak from pcie end device scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10618	ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10619	break;
10620	}
10621	handle = le16_to_cpu(pcie_device_pg0.DevHandle);
10622	if (!(_scsih_is_nvme_pciescsi_device(
10623	le32_to_cpu(pcie_device_pg0.DeviceInfo))))
10624	continue;
10625	pcie_device = mpt3sas_get_pdev_by_wwid(ioc,
10626	le64_to_cpu(pcie_device_pg0.WWID));
10627	if (pcie_device) {
10628	pcie_device_put(p: pcie_device);
10629	continue;
10630	}
10631	retry_count = 0;
10632	parent_handle = le16_to_cpu(pcie_device_pg0.ParentDevHandle);
10633	_scsih_pcie_add_device(ioc, handle);
10634
10635	ioc_info(ioc, "\tAFTER adding pcie end device: handle (0x%04x), wwid(0x%016llx)\n",
10636	handle, (u64)le64_to_cpu(pcie_device_pg0.WWID));
10637	}
10638
10639	kfree(objp: volume_pg0);
10640	kfree(objp: volume_pg1);
10641
10642	ioc_info(ioc, "\tpcie devices: pcie end devices complete\n");
10643	ioc_info(ioc, "scan devices: complete\n");
10644	}
10645
10646	/**
10647	* mpt3sas_scsih_pre_reset_handler - reset callback handler (for scsih)
10648	* @ioc: per adapter object
10649	*
10650	* The handler for doing any required cleanup or initialization.
10651	*/
10652	void mpt3sas_scsih_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc)
10653	{
10654	dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_PRE_RESET\n", __func__));
10655	}
10656
10657	/**
10658	* mpt3sas_scsih_clear_outstanding_scsi_tm_commands - clears outstanding
10659	* scsi & tm cmds.
10660	* @ioc: per adapter object
10661	*
10662	* The handler for doing any required cleanup or initialization.
10663	*/
10664	void
10665	mpt3sas_scsih_clear_outstanding_scsi_tm_commands(struct MPT3SAS_ADAPTER *ioc)
10666	{
10667	dtmprintk(ioc,
10668	ioc_info(ioc, "%s: clear outstanding scsi & tm cmds\n", __func__));
10669	if (ioc->scsih_cmds.status & MPT3_CMD_PENDING) {
10670	ioc->scsih_cmds.status |= MPT3_CMD_RESET;
10671	mpt3sas_base_free_smid(ioc, smid: ioc->scsih_cmds.smid);
10672	complete(&ioc->scsih_cmds.done);
10673	}
10674	if (ioc->tm_cmds.status & MPT3_CMD_PENDING) {
10675	ioc->tm_cmds.status |= MPT3_CMD_RESET;
10676	mpt3sas_base_free_smid(ioc, smid: ioc->tm_cmds.smid);
10677	complete(&ioc->tm_cmds.done);
10678	}
10679
10680	memset(ioc->pend_os_device_add, 0, ioc->pend_os_device_add_sz);
10681	memset(ioc->device_remove_in_progress, 0,
10682	ioc->device_remove_in_progress_sz);
10683	_scsih_fw_event_cleanup_queue(ioc);
10684	_scsih_flush_running_cmds(ioc);
10685	}
10686
10687	/**
10688	* mpt3sas_scsih_reset_done_handler - reset callback handler (for scsih)
10689	* @ioc: per adapter object
10690	*
10691	* The handler for doing any required cleanup or initialization.
10692	*/
10693	void
10694	mpt3sas_scsih_reset_done_handler(struct MPT3SAS_ADAPTER *ioc)
10695	{
10696	dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_DONE_RESET\n", __func__));
10697	if (!(disable_discovery > 0 && !ioc->sas_hba.num_phys)) {
10698	if (ioc->multipath_on_hba) {
10699	_scsih_sas_port_refresh(ioc);
10700	_scsih_update_vphys_after_reset(ioc);
10701	}
10702	_scsih_prep_device_scan(ioc);
10703	_scsih_create_enclosure_list_after_reset(ioc);
10704	_scsih_search_responding_sas_devices(ioc);
10705	_scsih_search_responding_pcie_devices(ioc);
10706	_scsih_search_responding_raid_devices(ioc);
10707	_scsih_search_responding_expanders(ioc);
10708	_scsih_error_recovery_delete_devices(ioc);
10709	}
10710	}
10711
10712	/**
10713	* _mpt3sas_fw_work - delayed task for processing firmware events
10714	* @ioc: per adapter object
10715	* @fw_event: The fw_event_work object
10716	* Context: user.
10717	*/
10718	static void
10719	_mpt3sas_fw_work(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)
10720	{
10721	ioc->current_event = fw_event;
10722	_scsih_fw_event_del_from_list(ioc, fw_event);
10723
10724	/* the queue is being flushed so ignore this event */
10725	if (ioc->remove_host || ioc->pci_error_recovery) {
10726	fw_event_work_put(fw_work: fw_event);
10727	ioc->current_event = NULL;
10728	return;
10729	}
10730
10731	switch (fw_event->event) {
10732	case MPT3SAS_PROCESS_TRIGGER_DIAG:
10733	mpt3sas_process_trigger_data(ioc,
10734	event_data: (struct SL_WH_TRIGGERS_EVENT_DATA_T *)
10735	fw_event->event_data);
10736	break;
10737	case MPT3SAS_REMOVE_UNRESPONDING_DEVICES:
10738	while (scsi_host_in_recovery(shost: ioc->shost) ||
10739	ioc->shost_recovery) {
10740	/*
10741	* If we're unloading or cancelling the work, bail.
10742	* Otherwise, this can become an infinite loop.
10743	*/
10744	if (ioc->remove_host || ioc->fw_events_cleanup)
10745	goto out;
10746	ssleep(seconds: 1);
10747	}
10748	_scsih_remove_unresponding_devices(ioc);
10749	_scsih_del_dirty_vphy(ioc);
10750	_scsih_del_dirty_port_entries(ioc);
10751	if (ioc->is_gen35_ioc)
10752	_scsih_update_device_qdepth(ioc);
10753	_scsih_scan_for_devices_after_reset(ioc);
10754	/*
10755	* If diag reset has occurred during the driver load
10756	* then driver has to complete the driver load operation
10757	* by executing the following items:
10758	*- Register the devices from sas_device_init_list to SML
10759	*- clear is_driver_loading flag,
10760	*- start the watchdog thread.
10761	* In happy driver load path, above things are taken care of when
10762	* driver executes scsih_scan_finished().
10763	*/
10764	if (ioc->is_driver_loading)
10765	_scsih_complete_devices_scanning(ioc);
10766	_scsih_set_nvme_max_shutdown_latency(ioc);
10767	break;
10768	case MPT3SAS_PORT_ENABLE_COMPLETE:
10769	ioc->start_scan = 0;
10770	if (missing_delay[0] != -1 && missing_delay[1] != -1)
10771	mpt3sas_base_update_missing_delay(ioc, device_missing_delay: missing_delay[0],
10772	io_missing_delay: missing_delay[1]);
10773	dewtprintk(ioc,
10774	ioc_info(ioc, "port enable: complete from worker thread\n"));
10775	break;
10776	case MPT3SAS_TURN_ON_PFA_LED:
10777	_scsih_turn_on_pfa_led(ioc, handle: fw_event->device_handle);
10778	break;
10779	case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
10780	_scsih_sas_topology_change_event(ioc, fw_event);
10781	break;
10782	case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
10783	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
10784	_scsih_sas_device_status_change_event_debug(ioc,
10785	event_data: (Mpi2EventDataSasDeviceStatusChange_t *)
10786	fw_event->event_data);
10787	break;
10788	case MPI2_EVENT_SAS_DISCOVERY:
10789	_scsih_sas_discovery_event(ioc, fw_event);
10790	break;
10791	case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
10792	_scsih_sas_device_discovery_error_event(ioc, fw_event);
10793	break;
10794	case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
10795	_scsih_sas_broadcast_primitive_event(ioc, fw_event);
10796	break;
10797	case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
10798	_scsih_sas_enclosure_dev_status_change_event(ioc,
10799	fw_event);
10800	break;
10801	case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
10802	_scsih_sas_ir_config_change_event(ioc, fw_event);
10803	break;
10804	case MPI2_EVENT_IR_VOLUME:
10805	_scsih_sas_ir_volume_event(ioc, fw_event);
10806	break;
10807	case MPI2_EVENT_IR_PHYSICAL_DISK:
10808	_scsih_sas_ir_physical_disk_event(ioc, fw_event);
10809	break;
10810	case MPI2_EVENT_IR_OPERATION_STATUS:
10811	_scsih_sas_ir_operation_status_event(ioc, fw_event);
10812	break;
10813	case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
10814	_scsih_pcie_device_status_change_event(ioc, fw_event);
10815	break;
10816	case MPI2_EVENT_PCIE_ENUMERATION:
10817	_scsih_pcie_enumeration_event(ioc, fw_event);
10818	break;
10819	case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
10820	_scsih_pcie_topology_change_event(ioc, fw_event);
10821	ioc->current_event = NULL;
10822	return;
10823	}
10824	out:
10825	fw_event_work_put(fw_work: fw_event);
10826	ioc->current_event = NULL;
10827	}
10828
10829	/**
10830	* _firmware_event_work
10831	* @work: The fw_event_work object
10832	* Context: user.
10833	*
10834	* wrappers for the work thread handling firmware events
10835	*/
10836
10837	static void
10838	_firmware_event_work(struct work_struct *work)
10839	{
10840	struct fw_event_work *fw_event = container_of(work,
10841	struct fw_event_work, work);
10842
10843	_mpt3sas_fw_work(ioc: fw_event->ioc, fw_event);
10844	}
10845
10846	/**
10847	* mpt3sas_scsih_event_callback - firmware event handler (called at ISR time)
10848	* @ioc: per adapter object
10849	* @msix_index: MSIX table index supplied by the OS
10850	* @reply: reply message frame(lower 32bit addr)
10851	* Context: interrupt.
10852	*
10853	* This function merely adds a new work task into ioc->firmware_event_thread.
10854	* The tasks are worked from _firmware_event_work in user context.
10855	*
10856	* Return: 1 meaning mf should be freed from _base_interrupt
10857	* 0 means the mf is freed from this function.
10858	*/
10859	u8
10860	mpt3sas_scsih_event_callback(struct MPT3SAS_ADAPTER *ioc, u8 msix_index,
10861	u32 reply)
10862	{
10863	struct fw_event_work *fw_event;
10864	Mpi2EventNotificationReply_t *mpi_reply;
10865	u16 event;
10866	u16 sz;
10867	Mpi26EventDataActiveCableExcept_t *ActiveCableEventData;
10868
10869	/* events turned off due to host reset */
10870	if (ioc->pci_error_recovery)
10871	return 1;
10872
10873	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, phys_addr: reply);
10874
10875	if (unlikely(!mpi_reply)) {
10876	ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
10877	__FILE__, __LINE__, __func__);
10878	return 1;
10879	}
10880
10881	event = le16_to_cpu(mpi_reply->Event);
10882
10883	if (event != MPI2_EVENT_LOG_ENTRY_ADDED)
10884	mpt3sas_trigger_event(ioc, event, log_entry_qualifier: 0);
10885
10886	switch (event) {
10887	/* handle these */
10888	case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
10889	{
10890	Mpi2EventDataSasBroadcastPrimitive_t *baen_data =
10891	(Mpi2EventDataSasBroadcastPrimitive_t *)
10892	mpi_reply->EventData;
10893
10894	if (baen_data->Primitive !=
10895	MPI2_EVENT_PRIMITIVE_ASYNCHRONOUS_EVENT)
10896	return 1;
10897
10898	if (ioc->broadcast_aen_busy) {
10899	ioc->broadcast_aen_pending++;
10900	return 1;
10901	} else
10902	ioc->broadcast_aen_busy = 1;
10903	break;
10904	}
10905
10906	case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
10907	_scsih_check_topo_delete_events(ioc,
10908	event_data: (Mpi2EventDataSasTopologyChangeList_t *)
10909	mpi_reply->EventData);
10910	/*
10911	* No need to add the topology change list
10912	* event to fw event work queue when
10913	* diag reset is going on. Since during diag
10914	* reset driver scan the devices by reading
10915	* sas device page0's not by processing the
10916	* events.
10917	*/
10918	if (ioc->shost_recovery)
10919	return 1;
10920	break;
10921	case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
10922	_scsih_check_pcie_topo_remove_events(ioc,
10923	event_data: (Mpi26EventDataPCIeTopologyChangeList_t *)
10924	mpi_reply->EventData);
10925	if (ioc->shost_recovery)
10926	return 1;
10927	break;
10928	case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
10929	_scsih_check_ir_config_unhide_events(ioc,
10930	event_data: (Mpi2EventDataIrConfigChangeList_t *)
10931	mpi_reply->EventData);
10932	break;
10933	case MPI2_EVENT_IR_VOLUME:
10934	_scsih_check_volume_delete_events(ioc,
10935	event_data: (Mpi2EventDataIrVolume_t *)
10936	mpi_reply->EventData);
10937	break;
10938	case MPI2_EVENT_LOG_ENTRY_ADDED:
10939	{
10940	Mpi2EventDataLogEntryAdded_t *log_entry;
10941	u32 log_code;
10942
10943	if (!ioc->is_warpdrive)
10944	break;
10945
10946	log_entry = (Mpi2EventDataLogEntryAdded_t *)
10947	mpi_reply->EventData;
10948	log_code = le32_to_cpu(*(__le32 *)log_entry->LogData);
10949
10950	if (le16_to_cpu(log_entry->LogEntryQualifier)
10951	!= MPT2_WARPDRIVE_LOGENTRY)
10952	break;
10953
10954	switch (log_code) {
10955	case MPT2_WARPDRIVE_LC_SSDT:
10956	ioc_warn(ioc, "WarpDrive Warning: IO Throttling has occurred in the WarpDrive subsystem. Check WarpDrive documentation for additional details.\n");
10957	break;
10958	case MPT2_WARPDRIVE_LC_SSDLW:
10959	ioc_warn(ioc, "WarpDrive Warning: Program/Erase Cycles for the WarpDrive subsystem in degraded range. Check WarpDrive documentation for additional details.\n");
10960	break;
10961	case MPT2_WARPDRIVE_LC_SSDLF:
10962	ioc_err(ioc, "WarpDrive Fatal Error: There are no Program/Erase Cycles for the WarpDrive subsystem. The storage device will be in read-only mode. Check WarpDrive documentation for additional details.\n");
10963	break;
10964	case MPT2_WARPDRIVE_LC_BRMF:
10965	ioc_err(ioc, "WarpDrive Fatal Error: The Backup Rail Monitor has failed on the WarpDrive subsystem. Check WarpDrive documentation for additional details.\n");
10966	break;
10967	}
10968
10969	break;
10970	}
10971	case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
10972	_scsih_sas_device_status_change_event(ioc,
10973	event_data: (Mpi2EventDataSasDeviceStatusChange_t *)
10974	mpi_reply->EventData);
10975	break;
10976	case MPI2_EVENT_IR_OPERATION_STATUS:
10977	case MPI2_EVENT_SAS_DISCOVERY:
10978	case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
10979	case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
10980	case MPI2_EVENT_IR_PHYSICAL_DISK:
10981	case MPI2_EVENT_PCIE_ENUMERATION:
10982	case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
10983	break;
10984
10985	case MPI2_EVENT_TEMP_THRESHOLD:
10986	_scsih_temp_threshold_events(ioc,
10987	event_data: (Mpi2EventDataTemperature_t *)
10988	mpi_reply->EventData);
10989	break;
10990	case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION:
10991	ActiveCableEventData =
10992	(Mpi26EventDataActiveCableExcept_t *) mpi_reply->EventData;
10993	switch (ActiveCableEventData->ReasonCode) {
10994	case MPI26_EVENT_ACTIVE_CABLE_INSUFFICIENT_POWER:
10995	ioc_notice(ioc, "Currently an active cable with ReceptacleID %d\n",
10996	ActiveCableEventData->ReceptacleID);
10997	pr_notice("cannot be powered and devices connected\n");
10998	pr_notice("to this active cable will not be seen\n");
10999	pr_notice("This active cable requires %d mW of power\n",
11000	le32_to_cpu(
11001	ActiveCableEventData->ActiveCablePowerRequirement));
11002	break;
11003
11004	case MPI26_EVENT_ACTIVE_CABLE_DEGRADED:
11005	ioc_notice(ioc, "Currently a cable with ReceptacleID %d\n",
11006	ActiveCableEventData->ReceptacleID);
11007	pr_notice(
11008	"is not running at optimal speed(12 Gb/s rate)\n");
11009	break;
11010	}
11011
11012	break;
11013
11014	default: /* ignore the rest */
11015	return 1;
11016	}
11017
11018	sz = le16_to_cpu(mpi_reply->EventDataLength) * 4;
11019	fw_event = alloc_fw_event_work(len: sz);
11020	if (!fw_event) {
11021	ioc_err(ioc, "failure at %s:%d/%s()!\n",
11022	__FILE__, __LINE__, __func__);
11023	return 1;
11024	}
11025
11026	memcpy(fw_event->event_data, mpi_reply->EventData, sz);
11027	fw_event->ioc = ioc;
11028	fw_event->VF_ID = mpi_reply->VF_ID;
11029	fw_event->VP_ID = mpi_reply->VP_ID;
11030	fw_event->event = event;
11031	_scsih_fw_event_add(ioc, fw_event);
11032	fw_event_work_put(fw_work: fw_event);
11033	return 1;
11034	}
11035
11036	/**
11037	* _scsih_expander_node_remove - removing expander device from list.
11038	* @ioc: per adapter object
11039	* @sas_expander: the sas_device object
11040	*
11041	* Removing object and freeing associated memory from the
11042	* ioc->sas_expander_list.
11043	*/
11044	static void
11045	_scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,
11046	struct _sas_node *sas_expander)
11047	{
11048	struct _sas_port *mpt3sas_port, *next;
11049	unsigned long flags;
11050	int port_id;
11051
11052	/* remove sibling ports attached to this expander */
11053	list_for_each_entry_safe(mpt3sas_port, next,
11054	&sas_expander->sas_port_list, port_list) {
11055	if (ioc->shost_recovery)
11056	return;
11057	if (mpt3sas_port->remote_identify.device_type ==
11058	SAS_END_DEVICE)
11059	mpt3sas_device_remove_by_sas_address(ioc,
11060	sas_address: mpt3sas_port->remote_identify.sas_address,
11061	port: mpt3sas_port->hba_port);
11062	else if (mpt3sas_port->remote_identify.device_type ==
11063	SAS_EDGE_EXPANDER_DEVICE ||
11064	mpt3sas_port->remote_identify.device_type ==
11065	SAS_FANOUT_EXPANDER_DEVICE)
11066	mpt3sas_expander_remove(ioc,
11067	sas_address: mpt3sas_port->remote_identify.sas_address,
11068	port: mpt3sas_port->hba_port);
11069	}
11070
11071	port_id = sas_expander->port->port_id;
11072
11073	mpt3sas_transport_port_remove(ioc, sas_address: sas_expander->sas_address,
11074	sas_address_parent: sas_expander->sas_address_parent, port: sas_expander->port);
11075
11076	ioc_info(ioc,
11077	"expander_remove: handle(0x%04x), sas_addr(0x%016llx), port:%d\n",
11078	sas_expander->handle, (unsigned long long)
11079	sas_expander->sas_address,
11080	port_id);
11081
11082	spin_lock_irqsave(&ioc->sas_node_lock, flags);
11083	list_del(entry: &sas_expander->list);
11084	spin_unlock_irqrestore(lock: &ioc->sas_node_lock, flags);
11085
11086	kfree(objp: sas_expander->phy);
11087	kfree(objp: sas_expander);
11088	}
11089
11090	/**
11091	* _scsih_nvme_shutdown - NVMe shutdown notification
11092	* @ioc: per adapter object
11093	*
11094	* Sending IoUnitControl request with shutdown operation code to alert IOC that
11095	* the host system is shutting down so that IOC can issue NVMe shutdown to
11096	* NVMe drives attached to it.
11097	*/
11098	static void
11099	_scsih_nvme_shutdown(struct MPT3SAS_ADAPTER *ioc)
11100	{
11101	Mpi26IoUnitControlRequest_t *mpi_request;
11102	Mpi26IoUnitControlReply_t *mpi_reply;
11103	u16 smid;
11104
11105	/* are there any NVMe devices ? */
11106	if (list_empty(head: &ioc->pcie_device_list))
11107	return;
11108
11109	mutex_lock(&ioc->scsih_cmds.mutex);
11110
11111	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {
11112	ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);
11113	goto out;
11114	}
11115
11116	ioc->scsih_cmds.status = MPT3_CMD_PENDING;
11117
11118	smid = mpt3sas_base_get_smid(ioc, cb_idx: ioc->scsih_cb_idx);
11119	if (!smid) {
11120	ioc_err(ioc,
11121	"%s: failed obtaining a smid\n", __func__);
11122	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
11123	goto out;
11124	}
11125
11126	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
11127	ioc->scsih_cmds.smid = smid;
11128	memset(mpi_request, 0, sizeof(Mpi26IoUnitControlRequest_t));
11129	mpi_request->Function = MPI2_FUNCTION_IO_UNIT_CONTROL;
11130	mpi_request->Operation = MPI26_CTRL_OP_SHUTDOWN;
11131
11132	init_completion(x: &ioc->scsih_cmds.done);
11133	ioc->put_smid_default(ioc, smid);
11134	/* Wait for max_shutdown_latency seconds */
11135	ioc_info(ioc,
11136	"Io Unit Control shutdown (sending), Shutdown latency %d sec\n",
11137	ioc->max_shutdown_latency);
11138	wait_for_completion_timeout(x: &ioc->scsih_cmds.done,
11139	timeout: ioc->max_shutdown_latency*HZ);
11140
11141	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
11142	ioc_err(ioc, "%s: timeout\n", __func__);
11143	goto out;
11144	}
11145
11146	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {
11147	mpi_reply = ioc->scsih_cmds.reply;
11148	ioc_info(ioc, "Io Unit Control shutdown (complete):"
11149	"ioc_status(0x%04x), loginfo(0x%08x)\n",
11150	le16_to_cpu(mpi_reply->IOCStatus),
11151	le32_to_cpu(mpi_reply->IOCLogInfo));
11152	}
11153	out:
11154	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
11155	mutex_unlock(lock: &ioc->scsih_cmds.mutex);
11156	}
11157
11158
11159	/**
11160	* _scsih_ir_shutdown - IR shutdown notification
11161	* @ioc: per adapter object
11162	*
11163	* Sending RAID Action to alert the Integrated RAID subsystem of the IOC that
11164	* the host system is shutting down.
11165	*/
11166	static void
11167	_scsih_ir_shutdown(struct MPT3SAS_ADAPTER *ioc)
11168	{
11169	Mpi2RaidActionRequest_t *mpi_request;
11170	Mpi2RaidActionReply_t *mpi_reply;
11171	u16 smid;
11172
11173	/* is IR firmware build loaded ? */
11174	if (!ioc->ir_firmware)
11175	return;
11176
11177	/* are there any volumes ? */
11178	if (list_empty(head: &ioc->raid_device_list))
11179	return;
11180
11181	mutex_lock(&ioc->scsih_cmds.mutex);
11182
11183	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {
11184	ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);
11185	goto out;
11186	}
11187	ioc->scsih_cmds.status = MPT3_CMD_PENDING;
11188
11189	smid = mpt3sas_base_get_smid(ioc, cb_idx: ioc->scsih_cb_idx);
11190	if (!smid) {
11191	ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
11192	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
11193	goto out;
11194	}
11195
11196	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
11197	ioc->scsih_cmds.smid = smid;
11198	memset(mpi_request, 0, sizeof(Mpi2RaidActionRequest_t));
11199
11200	mpi_request->Function = MPI2_FUNCTION_RAID_ACTION;
11201	mpi_request->Action = MPI2_RAID_ACTION_SYSTEM_SHUTDOWN_INITIATED;
11202
11203	if (!ioc->hide_ir_msg)
11204	ioc_info(ioc, "IR shutdown (sending)\n");
11205	init_completion(x: &ioc->scsih_cmds.done);
11206	ioc->put_smid_default(ioc, smid);
11207	wait_for_completion_timeout(x: &ioc->scsih_cmds.done, timeout: 10*HZ);
11208
11209	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
11210	ioc_err(ioc, "%s: timeout\n", __func__);
11211	goto out;
11212	}
11213
11214	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {
11215	mpi_reply = ioc->scsih_cmds.reply;
11216	if (!ioc->hide_ir_msg)
11217	ioc_info(ioc, "IR shutdown (complete): ioc_status(0x%04x), loginfo(0x%08x)\n",
11218	le16_to_cpu(mpi_reply->IOCStatus),
11219	le32_to_cpu(mpi_reply->IOCLogInfo));
11220	}
11221
11222	out:
11223	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
11224	mutex_unlock(lock: &ioc->scsih_cmds.mutex);
11225	}
11226
11227	/**
11228	* _scsih_get_shost_and_ioc - get shost and ioc
11229	* and verify whether they are NULL or not
11230	* @pdev: PCI device struct
11231	* @shost: address of scsi host pointer
11232	* @ioc: address of HBA adapter pointer
11233	*
11234	* Return zero if *shost and *ioc are not NULL otherwise return error number.
11235	*/
11236	static int
11237	_scsih_get_shost_and_ioc(struct pci_dev *pdev,
11238	struct Scsi_Host **shost, struct MPT3SAS_ADAPTER **ioc)
11239	{
11240	*shost = pci_get_drvdata(pdev);
11241	if (*shost == NULL) {
11242	dev_err(&pdev->dev, "pdev's driver data is null\n");
11243	return -ENXIO;
11244	}
11245
11246	*ioc = shost_priv(shost: *shost);
11247	if (*ioc == NULL) {
11248	dev_err(&pdev->dev, "shost's private data is null\n");
11249	return -ENXIO;
11250	}
11251
11252	return 0;
11253	}
11254
11255	/**
11256	* scsih_remove - detach and remove add host
11257	* @pdev: PCI device struct
11258	*
11259	* Routine called when unloading the driver.
11260	*/
11261	static void scsih_remove(struct pci_dev *pdev)
11262	{
11263	struct Scsi_Host *shost;
11264	struct MPT3SAS_ADAPTER *ioc;
11265	struct _sas_port *mpt3sas_port, *next_port;
11266	struct _raid_device *raid_device, *next;
11267	struct MPT3SAS_TARGET *sas_target_priv_data;
11268	struct _pcie_device *pcie_device, *pcienext;
11269	struct workqueue_struct *wq;
11270	unsigned long flags;
11271	Mpi2ConfigReply_t mpi_reply;
11272	struct hba_port *port, *port_next;
11273
11274	if (_scsih_get_shost_and_ioc(pdev, shost: &shost, ioc: &ioc))
11275	return;
11276
11277	ioc->remove_host = 1;
11278
11279	if (!pci_device_is_present(pdev)) {
11280	mpt3sas_base_pause_mq_polling(ioc);
11281	_scsih_flush_running_cmds(ioc);
11282	}
11283
11284	_scsih_fw_event_cleanup_queue(ioc);
11285
11286	spin_lock_irqsave(&ioc->fw_event_lock, flags);
11287	wq = ioc->firmware_event_thread;
11288	ioc->firmware_event_thread = NULL;
11289	spin_unlock_irqrestore(lock: &ioc->fw_event_lock, flags);
11290	if (wq)
11291	destroy_workqueue(wq);
11292	/*
11293	* Copy back the unmodified ioc page1. so that on next driver load,
11294	* current modified changes on ioc page1 won't take effect.
11295	*/
11296	if (ioc->is_aero_ioc)
11297	mpt3sas_config_set_ioc_pg1(ioc, mpi_reply: &mpi_reply,
11298	config_page: &ioc->ioc_pg1_copy);
11299	/* release all the volumes */
11300	_scsih_ir_shutdown(ioc);
11301	mpt3sas_destroy_debugfs(ioc);
11302	sas_remove_host(shost);
11303	list_for_each_entry_safe(raid_device, next, &ioc->raid_device_list,
11304	list) {
11305	if (raid_device->starget) {
11306	sas_target_priv_data =
11307	raid_device->starget->hostdata;
11308	sas_target_priv_data->deleted = 1;
11309	scsi_remove_target(&raid_device->starget->dev);
11310	}
11311	ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",
11312	raid_device->handle, (u64)raid_device->wwid);
11313	_scsih_raid_device_remove(ioc, raid_device);
11314	}
11315	list_for_each_entry_safe(pcie_device, pcienext, &ioc->pcie_device_list,
11316	list) {
11317	_scsih_pcie_device_remove_from_sml(ioc, pcie_device);
11318	list_del_init(entry: &pcie_device->list);
11319	pcie_device_put(p: pcie_device);
11320	}
11321
11322	/* free ports attached to the sas_host */
11323	list_for_each_entry_safe(mpt3sas_port, next_port,
11324	&ioc->sas_hba.sas_port_list, port_list) {
11325	if (mpt3sas_port->remote_identify.device_type ==
11326	SAS_END_DEVICE)
11327	mpt3sas_device_remove_by_sas_address(ioc,
11328	sas_address: mpt3sas_port->remote_identify.sas_address,
11329	port: mpt3sas_port->hba_port);
11330	else if (mpt3sas_port->remote_identify.device_type ==
11331	SAS_EDGE_EXPANDER_DEVICE ||
11332	mpt3sas_port->remote_identify.device_type ==
11333	SAS_FANOUT_EXPANDER_DEVICE)
11334	mpt3sas_expander_remove(ioc,
11335	sas_address: mpt3sas_port->remote_identify.sas_address,
11336	port: mpt3sas_port->hba_port);
11337	}
11338
11339	list_for_each_entry_safe(port, port_next,
11340	&ioc->port_table_list, list) {
11341	list_del(entry: &port->list);
11342	kfree(objp: port);
11343	}
11344
11345	/* free phys attached to the sas_host */
11346	if (ioc->sas_hba.num_phys) {
11347	kfree(objp: ioc->sas_hba.phy);
11348	ioc->sas_hba.phy = NULL;
11349	ioc->sas_hba.num_phys = 0;
11350	}
11351
11352	mpt3sas_base_detach(ioc);
11353	spin_lock(lock: &gioc_lock);
11354	list_del(entry: &ioc->list);
11355	spin_unlock(lock: &gioc_lock);
11356	scsi_host_put(t: shost);
11357	}
11358
11359	/**
11360	* scsih_shutdown - routine call during system shutdown
11361	* @pdev: PCI device struct
11362	*/
11363	static void
11364	scsih_shutdown(struct pci_dev *pdev)
11365	{
11366	struct Scsi_Host *shost;
11367	struct MPT3SAS_ADAPTER *ioc;
11368	struct workqueue_struct *wq;
11369	unsigned long flags;
11370	Mpi2ConfigReply_t mpi_reply;
11371
11372	if (_scsih_get_shost_and_ioc(pdev, shost: &shost, ioc: &ioc))
11373	return;
11374
11375	ioc->remove_host = 1;
11376
11377	if (!pci_device_is_present(pdev)) {
11378	mpt3sas_base_pause_mq_polling(ioc);
11379	_scsih_flush_running_cmds(ioc);
11380	}
11381
11382	_scsih_fw_event_cleanup_queue(ioc);
11383
11384	spin_lock_irqsave(&ioc->fw_event_lock, flags);
11385	wq = ioc->firmware_event_thread;
11386	ioc->firmware_event_thread = NULL;
11387	spin_unlock_irqrestore(lock: &ioc->fw_event_lock, flags);
11388	if (wq)
11389	destroy_workqueue(wq);
11390	/*
11391	* Copy back the unmodified ioc page1 so that on next driver load,
11392	* current modified changes on ioc page1 won't take effect.
11393	*/
11394	if (ioc->is_aero_ioc)
11395	mpt3sas_config_set_ioc_pg1(ioc, mpi_reply: &mpi_reply,
11396	config_page: &ioc->ioc_pg1_copy);
11397
11398	_scsih_ir_shutdown(ioc);
11399	_scsih_nvme_shutdown(ioc);
11400	mpt3sas_base_mask_interrupts(ioc);
11401	mpt3sas_base_stop_watchdog(ioc);
11402	ioc->shost_recovery = 1;
11403	mpt3sas_base_make_ioc_ready(ioc, type: SOFT_RESET);
11404	ioc->shost_recovery = 0;
11405	mpt3sas_base_free_irq(ioc);
11406	mpt3sas_base_disable_msix(ioc);
11407	}
11408
11409
11410	/**
11411	* _scsih_probe_boot_devices - reports 1st device
11412	* @ioc: per adapter object
11413	*
11414	* If specified in bios page 2, this routine reports the 1st
11415	* device scsi-ml or sas transport for persistent boot device
11416	* purposes. Please refer to function _scsih_determine_boot_device()
11417	*/
11418	static void
11419	_scsih_probe_boot_devices(struct MPT3SAS_ADAPTER *ioc)
11420	{
11421	u32 channel;
11422	void *device;
11423	struct _sas_device *sas_device;
11424	struct _raid_device *raid_device;
11425	struct _pcie_device *pcie_device;
11426	u16 handle;
11427	u64 sas_address_parent;
11428	u64 sas_address;
11429	unsigned long flags;
11430	int rc;
11431	int tid;
11432	struct hba_port *port;
11433
11434	/* no Bios, return immediately */
11435	if (!ioc->bios_pg3.BiosVersion)
11436	return;
11437
11438	device = NULL;
11439	if (ioc->req_boot_device.device) {
11440	device = ioc->req_boot_device.device;
11441	channel = ioc->req_boot_device.channel;
11442	} else if (ioc->req_alt_boot_device.device) {
11443	device = ioc->req_alt_boot_device.device;
11444	channel = ioc->req_alt_boot_device.channel;
11445	} else if (ioc->current_boot_device.device) {
11446	device = ioc->current_boot_device.device;
11447	channel = ioc->current_boot_device.channel;
11448	}
11449
11450	if (!device)
11451	return;
11452
11453	if (channel == RAID_CHANNEL) {
11454	raid_device = device;
11455	/*
11456	* If this boot vd is already registered with SML then
11457	* no need to register it again as part of device scanning
11458	* after diag reset during driver load operation.
11459	*/
11460	if (raid_device->starget)
11461	return;
11462	rc = scsi_add_device(host: ioc->shost, RAID_CHANNEL,
11463	target: raid_device->id, lun: 0);
11464	if (rc)
11465	_scsih_raid_device_remove(ioc, raid_device);
11466	} else if (channel == PCIE_CHANNEL) {
11467	pcie_device = device;
11468	/*
11469	* If this boot NVMe device is already registered with SML then
11470	* no need to register it again as part of device scanning
11471	* after diag reset during driver load operation.
11472	*/
11473	if (pcie_device->starget)
11474	return;
11475	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
11476	tid = pcie_device->id;
11477	list_move_tail(list: &pcie_device->list, head: &ioc->pcie_device_list);
11478	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
11479	rc = scsi_add_device(host: ioc->shost, PCIE_CHANNEL, target: tid, lun: 0);
11480	if (rc)
11481	_scsih_pcie_device_remove(ioc, pcie_device);
11482	} else {
11483	sas_device = device;
11484	/*
11485	* If this boot sas/sata device is already registered with SML
11486	* then no need to register it again as part of device scanning
11487	* after diag reset during driver load operation.
11488	*/
11489	if (sas_device->starget)
11490	return;
11491	spin_lock_irqsave(&ioc->sas_device_lock, flags);
11492	handle = sas_device->handle;
11493	sas_address_parent = sas_device->sas_address_parent;
11494	sas_address = sas_device->sas_address;
11495	port = sas_device->port;
11496	list_move_tail(list: &sas_device->list, head: &ioc->sas_device_list);
11497	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
11498
11499	if (ioc->hide_drives)
11500	return;
11501
11502	if (!port)
11503	return;
11504
11505	if (!mpt3sas_transport_port_add(ioc, handle,
11506	sas_address: sas_address_parent, port)) {
11507	_scsih_sas_device_remove(ioc, sas_device);
11508	} else if (!sas_device->starget) {
11509	if (!ioc->is_driver_loading) {
11510	mpt3sas_transport_port_remove(ioc,
11511	sas_address,
11512	sas_address_parent, port);
11513	_scsih_sas_device_remove(ioc, sas_device);
11514	}
11515	}
11516	}
11517	}
11518
11519	/**
11520	* _scsih_probe_raid - reporting raid volumes to scsi-ml
11521	* @ioc: per adapter object
11522	*
11523	* Called during initial loading of the driver.
11524	*/
11525	static void
11526	_scsih_probe_raid(struct MPT3SAS_ADAPTER *ioc)
11527	{
11528	struct _raid_device *raid_device, *raid_next;
11529	int rc;
11530
11531	list_for_each_entry_safe(raid_device, raid_next,
11532	&ioc->raid_device_list, list) {
11533	if (raid_device->starget)
11534	continue;
11535	rc = scsi_add_device(host: ioc->shost, RAID_CHANNEL,
11536	target: raid_device->id, lun: 0);
11537	if (rc)
11538	_scsih_raid_device_remove(ioc, raid_device);
11539	}
11540	}
11541
11542	static struct _sas_device *get_next_sas_device(struct MPT3SAS_ADAPTER *ioc)
11543	{
11544	struct _sas_device *sas_device = NULL;
11545	unsigned long flags;
11546
11547	spin_lock_irqsave(&ioc->sas_device_lock, flags);
11548	if (!list_empty(head: &ioc->sas_device_init_list)) {
11549	sas_device = list_first_entry(&ioc->sas_device_init_list,
11550	struct _sas_device, list);
11551	sas_device_get(s: sas_device);
11552	}
11553	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
11554
11555	return sas_device;
11556	}
11557
11558	static void sas_device_make_active(struct MPT3SAS_ADAPTER *ioc,
11559	struct _sas_device *sas_device)
11560	{
11561	unsigned long flags;
11562
11563	spin_lock_irqsave(&ioc->sas_device_lock, flags);
11564
11565	/*
11566	* Since we dropped the lock during the call to port_add(), we need to
11567	* be careful here that somebody else didn't move or delete this item
11568	* while we were busy with other things.
11569	*
11570	* If it was on the list, we need a put() for the reference the list
11571	* had. Either way, we need a get() for the destination list.
11572	*/
11573	if (!list_empty(head: &sas_device->list)) {
11574	list_del_init(entry: &sas_device->list);
11575	sas_device_put(s: sas_device);
11576	}
11577
11578	sas_device_get(s: sas_device);
11579	list_add_tail(new: &sas_device->list, head: &ioc->sas_device_list);
11580
11581	spin_unlock_irqrestore(lock: &ioc->sas_device_lock, flags);
11582	}
11583
11584	/**
11585	* _scsih_probe_sas - reporting sas devices to sas transport
11586	* @ioc: per adapter object
11587	*
11588	* Called during initial loading of the driver.
11589	*/
11590	static void
11591	_scsih_probe_sas(struct MPT3SAS_ADAPTER *ioc)
11592	{
11593	struct _sas_device *sas_device;
11594
11595	if (ioc->hide_drives)
11596	return;
11597
11598	while ((sas_device = get_next_sas_device(ioc))) {
11599	if (!mpt3sas_transport_port_add(ioc, handle: sas_device->handle,
11600	sas_address: sas_device->sas_address_parent, port: sas_device->port)) {
11601	_scsih_sas_device_remove(ioc, sas_device);
11602	sas_device_put(s: sas_device);
11603	continue;
11604	} else if (!sas_device->starget) {
11605	/*
11606	* When asyn scanning is enabled, its not possible to
11607	* remove devices while scanning is turned on due to an
11608	* oops in scsi_sysfs_add_sdev()->add_device()->
11609	* sysfs_addrm_start()
11610	*/
11611	if (!ioc->is_driver_loading) {
11612	mpt3sas_transport_port_remove(ioc,
11613	sas_address: sas_device->sas_address,
11614	sas_address_parent: sas_device->sas_address_parent,
11615	port: sas_device->port);
11616	_scsih_sas_device_remove(ioc, sas_device);
11617	sas_device_put(s: sas_device);
11618	continue;
11619	}
11620	}
11621	sas_device_make_active(ioc, sas_device);
11622	sas_device_put(s: sas_device);
11623	}
11624	}
11625
11626	/**
11627	* get_next_pcie_device - Get the next pcie device
11628	* @ioc: per adapter object
11629	*
11630	* Get the next pcie device from pcie_device_init_list list.
11631	*
11632	* Return: pcie device structure if pcie_device_init_list list is not empty
11633	* otherwise returns NULL
11634	*/
11635	static struct _pcie_device *get_next_pcie_device(struct MPT3SAS_ADAPTER *ioc)
11636	{
11637	struct _pcie_device *pcie_device = NULL;
11638	unsigned long flags;
11639
11640	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
11641	if (!list_empty(head: &ioc->pcie_device_init_list)) {
11642	pcie_device = list_first_entry(&ioc->pcie_device_init_list,
11643	struct _pcie_device, list);
11644	pcie_device_get(p: pcie_device);
11645	}
11646	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
11647
11648	return pcie_device;
11649	}
11650
11651	/**
11652	* pcie_device_make_active - Add pcie device to pcie_device_list list
11653	* @ioc: per adapter object
11654	* @pcie_device: pcie device object
11655	*
11656	* Add the pcie device which has registered with SCSI Transport Later to
11657	* pcie_device_list list
11658	*/
11659	static void pcie_device_make_active(struct MPT3SAS_ADAPTER *ioc,
11660	struct _pcie_device *pcie_device)
11661	{
11662	unsigned long flags;
11663
11664	spin_lock_irqsave(&ioc->pcie_device_lock, flags);
11665
11666	if (!list_empty(head: &pcie_device->list)) {
11667	list_del_init(entry: &pcie_device->list);
11668	pcie_device_put(p: pcie_device);
11669	}
11670	pcie_device_get(p: pcie_device);
11671	list_add_tail(new: &pcie_device->list, head: &ioc->pcie_device_list);
11672
11673	spin_unlock_irqrestore(lock: &ioc->pcie_device_lock, flags);
11674	}
11675
11676	/**
11677	* _scsih_probe_pcie - reporting PCIe devices to scsi-ml
11678	* @ioc: per adapter object
11679	*
11680	* Called during initial loading of the driver.
11681	*/
11682	static void
11683	_scsih_probe_pcie(struct MPT3SAS_ADAPTER *ioc)
11684	{
11685	struct _pcie_device *pcie_device;
11686	int rc;
11687
11688	/* PCIe Device List */
11689	while ((pcie_device = get_next_pcie_device(ioc))) {
11690	if (pcie_device->starget) {
11691	pcie_device_put(p: pcie_device);
11692	continue;
11693	}
11694	if (pcie_device->access_status ==
11695	MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED) {
11696	pcie_device_make_active(ioc, pcie_device);
11697	pcie_device_put(p: pcie_device);
11698	continue;
11699	}
11700	rc = scsi_add_device(host: ioc->shost, PCIE_CHANNEL,
11701	target: pcie_device->id, lun: 0);
11702	if (rc) {
11703	_scsih_pcie_device_remove(ioc, pcie_device);
11704	pcie_device_put(p: pcie_device);
11705	continue;
11706	} else if (!pcie_device->starget) {
11707	/*
11708	* When async scanning is enabled, its not possible to
11709	* remove devices while scanning is turned on due to an
11710	* oops in scsi_sysfs_add_sdev()->add_device()->
11711	* sysfs_addrm_start()
11712	*/
11713	if (!ioc->is_driver_loading) {
11714	/* TODO-- Need to find out whether this condition will
11715	* occur or not
11716	*/
11717	_scsih_pcie_device_remove(ioc, pcie_device);
11718	pcie_device_put(p: pcie_device);
11719	continue;
11720	}
11721	}
11722	pcie_device_make_active(ioc, pcie_device);
11723	pcie_device_put(p: pcie_device);
11724	}
11725	}
11726
11727	/**
11728	* _scsih_probe_devices - probing for devices
11729	* @ioc: per adapter object
11730	*
11731	* Called during initial loading of the driver.
11732	*/
11733	static void
11734	_scsih_probe_devices(struct MPT3SAS_ADAPTER *ioc)
11735	{
11736	u16 volume_mapping_flags;
11737
11738	if (!(ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR))
11739	return; /* return when IOC doesn't support initiator mode */
11740
11741	_scsih_probe_boot_devices(ioc);
11742
11743	if (ioc->ir_firmware) {
11744	volume_mapping_flags =
11745	le16_to_cpu(ioc->ioc_pg8.IRVolumeMappingFlags) &
11746	MPI2_IOCPAGE8_IRFLAGS_MASK_VOLUME_MAPPING_MODE;
11747	if (volume_mapping_flags ==
11748	MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING) {
11749	_scsih_probe_raid(ioc);
11750	_scsih_probe_sas(ioc);
11751	} else {
11752	_scsih_probe_sas(ioc);
11753	_scsih_probe_raid(ioc);
11754	}
11755	} else {
11756	_scsih_probe_sas(ioc);
11757	_scsih_probe_pcie(ioc);
11758	}
11759	}
11760
11761	/**
11762	* scsih_scan_start - scsi lld callback for .scan_start
11763	* @shost: SCSI host pointer
11764	*
11765	* The shost has the ability to discover targets on its own instead
11766	* of scanning the entire bus. In our implemention, we will kick off
11767	* firmware discovery.
11768	*/
11769	static void
11770	scsih_scan_start(struct Scsi_Host *shost)
11771	{
11772	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
11773	int rc;
11774	if (diag_buffer_enable != -1 && diag_buffer_enable != 0)
11775	mpt3sas_enable_diag_buffer(ioc, bits_to_register: diag_buffer_enable);
11776	else if (ioc->manu_pg11.HostTraceBufferMaxSizeKB != 0)
11777	mpt3sas_enable_diag_buffer(ioc, bits_to_register: 1);
11778
11779	if (disable_discovery > 0)
11780	return;
11781
11782	ioc->start_scan = 1;
11783	rc = mpt3sas_port_enable(ioc);
11784
11785	if (rc != 0)
11786	ioc_info(ioc, "port enable: FAILED\n");
11787	}
11788
11789	/**
11790	* _scsih_complete_devices_scanning - add the devices to sml and
11791	* complete ioc initialization.
11792	* @ioc: per adapter object
11793	*
11794	* Return nothing.
11795	*/
11796	static void _scsih_complete_devices_scanning(struct MPT3SAS_ADAPTER *ioc)
11797	{
11798
11799	if (ioc->wait_for_discovery_to_complete) {
11800	ioc->wait_for_discovery_to_complete = 0;
11801	_scsih_probe_devices(ioc);
11802	}
11803
11804	mpt3sas_base_start_watchdog(ioc);
11805	ioc->is_driver_loading = 0;
11806	}
11807
11808	/**
11809	* scsih_scan_finished - scsi lld callback for .scan_finished
11810	* @shost: SCSI host pointer
11811	* @time: elapsed time of the scan in jiffies
11812	*
11813	* This function will be called periodicallyn until it returns 1 with the
11814	* scsi_host and the elapsed time of the scan in jiffies. In our implemention,
11815	* we wait for firmware discovery to complete, then return 1.
11816	*/
11817	static int
11818	scsih_scan_finished(struct Scsi_Host *shost, unsigned long time)
11819	{
11820	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
11821	u32 ioc_state;
11822	int issue_hard_reset = 0;
11823
11824	if (disable_discovery > 0) {
11825	ioc->is_driver_loading = 0;
11826	ioc->wait_for_discovery_to_complete = 0;
11827	return 1;
11828	}
11829
11830	if (time >= (300 * HZ)) {
11831	ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
11832	ioc_info(ioc, "port enable: FAILED with timeout (timeout=300s)\n");
11833	ioc->is_driver_loading = 0;
11834	return 1;
11835	}
11836
11837	if (ioc->start_scan) {
11838	ioc_state = mpt3sas_base_get_iocstate(ioc, cooked: 0);
11839	if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
11840	mpt3sas_print_fault_code(ioc, ioc_state &
11841	MPI2_DOORBELL_DATA_MASK);
11842	issue_hard_reset = 1;
11843	goto out;
11844	} else if ((ioc_state & MPI2_IOC_STATE_MASK) ==
11845	MPI2_IOC_STATE_COREDUMP) {
11846	mpt3sas_base_coredump_info(ioc, fault_code: ioc_state &
11847	MPI2_DOORBELL_DATA_MASK);
11848	mpt3sas_base_wait_for_coredump_completion(ioc, caller: __func__);
11849	issue_hard_reset = 1;
11850	goto out;
11851	}
11852	return 0;
11853	}
11854
11855	if (ioc->port_enable_cmds.status & MPT3_CMD_RESET) {
11856	ioc_info(ioc,
11857	"port enable: aborted due to diag reset\n");
11858	ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
11859	goto out;
11860	}
11861	if (ioc->start_scan_failed) {
11862	ioc_info(ioc, "port enable: FAILED with (ioc_status=0x%08x)\n",
11863	ioc->start_scan_failed);
11864	ioc->is_driver_loading = 0;
11865	ioc->wait_for_discovery_to_complete = 0;
11866	ioc->remove_host = 1;
11867	return 1;
11868	}
11869
11870	ioc_info(ioc, "port enable: SUCCESS\n");
11871	ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
11872	_scsih_complete_devices_scanning(ioc);
11873
11874	out:
11875	if (issue_hard_reset) {
11876	ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
11877	if (mpt3sas_base_hard_reset_handler(ioc, type: SOFT_RESET))
11878	ioc->is_driver_loading = 0;
11879	}
11880	return 1;
11881	}
11882
11883	/**
11884	* scsih_map_queues - map reply queues with request queues
11885	* @shost: SCSI host pointer
11886	*/
11887	static void scsih_map_queues(struct Scsi_Host *shost)
11888	{
11889	struct MPT3SAS_ADAPTER *ioc =
11890	(struct MPT3SAS_ADAPTER *)shost->hostdata;
11891	struct blk_mq_queue_map *map;
11892	int i, qoff, offset;
11893	int nr_msix_vectors = ioc->iopoll_q_start_index;
11894	int iopoll_q_count = ioc->reply_queue_count - nr_msix_vectors;
11895
11896	if (shost->nr_hw_queues == 1)
11897	return;
11898
11899	for (i = 0, qoff = 0; i < shost->nr_maps; i++) {
11900	map = &shost->tag_set.map[i];
11901	map->nr_queues = 0;
11902	offset = 0;
11903	if (i == HCTX_TYPE_DEFAULT) {
11904	map->nr_queues =
11905	nr_msix_vectors - ioc->high_iops_queues;
11906	offset = ioc->high_iops_queues;
11907	} else if (i == HCTX_TYPE_POLL)
11908	map->nr_queues = iopoll_q_count;
11909
11910	if (!map->nr_queues)
11911	BUG_ON(i == HCTX_TYPE_DEFAULT);
11912
11913	/*
11914	* The poll queue(s) doesn't have an IRQ (and hence IRQ
11915	* affinity), so use the regular blk-mq cpu mapping
11916	*/
11917	map->queue_offset = qoff;
11918	if (i != HCTX_TYPE_POLL)
11919	blk_mq_pci_map_queues(qmap: map, pdev: ioc->pdev, offset);
11920	else
11921	blk_mq_map_queues(qmap: map);
11922
11923	qoff += map->nr_queues;
11924	}
11925	}
11926
11927	/* shost template for SAS 2.0 HBA devices */
11928	static const struct scsi_host_template mpt2sas_driver_template = {
11929	.module = THIS_MODULE,
11930	.name = "Fusion MPT SAS Host",
11931	.proc_name = MPT2SAS_DRIVER_NAME,
11932	.queuecommand = scsih_qcmd,
11933	.target_alloc = scsih_target_alloc,
11934	.slave_alloc = scsih_slave_alloc,
11935	.slave_configure = scsih_slave_configure,
11936	.target_destroy = scsih_target_destroy,
11937	.slave_destroy = scsih_slave_destroy,
11938	.scan_finished = scsih_scan_finished,
11939	.scan_start = scsih_scan_start,
11940	.change_queue_depth = scsih_change_queue_depth,
11941	.eh_abort_handler = scsih_abort,
11942	.eh_device_reset_handler = scsih_dev_reset,
11943	.eh_target_reset_handler = scsih_target_reset,
11944	.eh_host_reset_handler = scsih_host_reset,
11945	.bios_param = scsih_bios_param,
11946	.can_queue = 1,
11947	.this_id = -1,
11948	.sg_tablesize = MPT2SAS_SG_DEPTH,
11949	.max_sectors = 32767,
11950	.cmd_per_lun = 7,
11951	.shost_groups = mpt3sas_host_groups,
11952	.sdev_groups = mpt3sas_dev_groups,
11953	.track_queue_depth = 1,
11954	.cmd_size = sizeof(struct scsiio_tracker),
11955	};
11956
11957	/* raid transport support for SAS 2.0 HBA devices */
11958	static struct raid_function_template mpt2sas_raid_functions = {
11959	.cookie = &mpt2sas_driver_template,
11960	.is_raid = scsih_is_raid,
11961	.get_resync = scsih_get_resync,
11962	.get_state = scsih_get_state,
11963	};
11964
11965	/* shost template for SAS 3.0 HBA devices */
11966	static const struct scsi_host_template mpt3sas_driver_template = {
11967	.module = THIS_MODULE,
11968	.name = "Fusion MPT SAS Host",
11969	.proc_name = MPT3SAS_DRIVER_NAME,
11970	.queuecommand = scsih_qcmd,
11971	.target_alloc = scsih_target_alloc,
11972	.slave_alloc = scsih_slave_alloc,
11973	.slave_configure = scsih_slave_configure,
11974	.target_destroy = scsih_target_destroy,
11975	.slave_destroy = scsih_slave_destroy,
11976	.scan_finished = scsih_scan_finished,
11977	.scan_start = scsih_scan_start,
11978	.change_queue_depth = scsih_change_queue_depth,
11979	.eh_abort_handler = scsih_abort,
11980	.eh_device_reset_handler = scsih_dev_reset,
11981	.eh_target_reset_handler = scsih_target_reset,
11982	.eh_host_reset_handler = scsih_host_reset,
11983	.bios_param = scsih_bios_param,
11984	.can_queue = 1,
11985	.this_id = -1,
11986	.sg_tablesize = MPT3SAS_SG_DEPTH,
11987	.max_sectors = 32767,
11988	.max_segment_size = 0xffffffff,
11989	.cmd_per_lun = 128,
11990	.shost_groups = mpt3sas_host_groups,
11991	.sdev_groups = mpt3sas_dev_groups,
11992	.track_queue_depth = 1,
11993	.cmd_size = sizeof(struct scsiio_tracker),
11994	.map_queues = scsih_map_queues,
11995	.mq_poll = mpt3sas_blk_mq_poll,
11996	};
11997
11998	/* raid transport support for SAS 3.0 HBA devices */
11999	static struct raid_function_template mpt3sas_raid_functions = {
12000	.cookie = &mpt3sas_driver_template,
12001	.is_raid = scsih_is_raid,
12002	.get_resync = scsih_get_resync,
12003	.get_state = scsih_get_state,
12004	};
12005
12006	/**
12007	* _scsih_determine_hba_mpi_version - determine in which MPI version class
12008	* this device belongs to.
12009	* @pdev: PCI device struct
12010	*
12011	* return MPI2_VERSION for SAS 2.0 HBA devices,
12012	* MPI25_VERSION for SAS 3.0 HBA devices, and
12013	* MPI26 VERSION for Cutlass & Invader SAS 3.0 HBA devices
12014	*/
12015	static u16
12016	_scsih_determine_hba_mpi_version(struct pci_dev *pdev)
12017	{
12018
12019	switch (pdev->device) {
12020	case MPI2_MFGPAGE_DEVID_SSS6200:
12021	case MPI2_MFGPAGE_DEVID_SAS2004:
12022	case MPI2_MFGPAGE_DEVID_SAS2008:
12023	case MPI2_MFGPAGE_DEVID_SAS2108_1:
12024	case MPI2_MFGPAGE_DEVID_SAS2108_2:
12025	case MPI2_MFGPAGE_DEVID_SAS2108_3:
12026	case MPI2_MFGPAGE_DEVID_SAS2116_1:
12027	case MPI2_MFGPAGE_DEVID_SAS2116_2:
12028	case MPI2_MFGPAGE_DEVID_SAS2208_1:
12029	case MPI2_MFGPAGE_DEVID_SAS2208_2:
12030	case MPI2_MFGPAGE_DEVID_SAS2208_3:
12031	case MPI2_MFGPAGE_DEVID_SAS2208_4:
12032	case MPI2_MFGPAGE_DEVID_SAS2208_5:
12033	case MPI2_MFGPAGE_DEVID_SAS2208_6:
12034	case MPI2_MFGPAGE_DEVID_SAS2308_1:
12035	case MPI2_MFGPAGE_DEVID_SAS2308_2:
12036	case MPI2_MFGPAGE_DEVID_SAS2308_3:
12037	case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP:
12038	case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1:
12039	return MPI2_VERSION;
12040	case MPI25_MFGPAGE_DEVID_SAS3004:
12041	case MPI25_MFGPAGE_DEVID_SAS3008:
12042	case MPI25_MFGPAGE_DEVID_SAS3108_1:
12043	case MPI25_MFGPAGE_DEVID_SAS3108_2:
12044	case MPI25_MFGPAGE_DEVID_SAS3108_5:
12045	case MPI25_MFGPAGE_DEVID_SAS3108_6:
12046	return MPI25_VERSION;
12047	case MPI26_MFGPAGE_DEVID_SAS3216:
12048	case MPI26_MFGPAGE_DEVID_SAS3224:
12049	case MPI26_MFGPAGE_DEVID_SAS3316_1:
12050	case MPI26_MFGPAGE_DEVID_SAS3316_2:
12051	case MPI26_MFGPAGE_DEVID_SAS3316_3:
12052	case MPI26_MFGPAGE_DEVID_SAS3316_4:
12053	case MPI26_MFGPAGE_DEVID_SAS3324_1:
12054	case MPI26_MFGPAGE_DEVID_SAS3324_2:
12055	case MPI26_MFGPAGE_DEVID_SAS3324_3:
12056	case MPI26_MFGPAGE_DEVID_SAS3324_4:
12057	case MPI26_MFGPAGE_DEVID_SAS3508:
12058	case MPI26_MFGPAGE_DEVID_SAS3508_1:
12059	case MPI26_MFGPAGE_DEVID_SAS3408:
12060	case MPI26_MFGPAGE_DEVID_SAS3516:
12061	case MPI26_MFGPAGE_DEVID_SAS3516_1:
12062	case MPI26_MFGPAGE_DEVID_SAS3416:
12063	case MPI26_MFGPAGE_DEVID_SAS3616:
12064	case MPI26_ATLAS_PCIe_SWITCH_DEVID:
12065	case MPI26_MFGPAGE_DEVID_CFG_SEC_3916:
12066	case MPI26_MFGPAGE_DEVID_HARD_SEC_3916:
12067	case MPI26_MFGPAGE_DEVID_CFG_SEC_3816:
12068	case MPI26_MFGPAGE_DEVID_HARD_SEC_3816:
12069	case MPI26_MFGPAGE_DEVID_INVALID0_3916:
12070	case MPI26_MFGPAGE_DEVID_INVALID1_3916:
12071	case MPI26_MFGPAGE_DEVID_INVALID0_3816:
12072	case MPI26_MFGPAGE_DEVID_INVALID1_3816:
12073	return MPI26_VERSION;
12074	}
12075	return 0;
12076	}
12077
12078	/**
12079	* _scsih_probe - attach and add scsi host
12080	* @pdev: PCI device struct
12081	* @id: pci device id
12082	*
12083	* Return: 0 success, anything else error.
12084	*/
12085	static int
12086	_scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id)
12087	{
12088	struct MPT3SAS_ADAPTER *ioc;
12089	struct Scsi_Host *shost = NULL;
12090	int rv;
12091	u16 hba_mpi_version;
12092	int iopoll_q_count = 0;
12093
12094	/* Determine in which MPI version class this pci device belongs */
12095	hba_mpi_version = _scsih_determine_hba_mpi_version(pdev);
12096	if (hba_mpi_version == 0)
12097	return -ENODEV;
12098
12099	/* Enumerate only SAS 2.0 HBA's if hbas_to_enumerate is one,
12100	* for other generation HBA's return with -ENODEV
12101	*/
12102	if ((hbas_to_enumerate == 1) && (hba_mpi_version != MPI2_VERSION))
12103	return -ENODEV;
12104
12105	/* Enumerate only SAS 3.0 HBA's if hbas_to_enumerate is two,
12106	* for other generation HBA's return with -ENODEV
12107	*/
12108	if ((hbas_to_enumerate == 2) && (!(hba_mpi_version == MPI25_VERSION
12109	|| hba_mpi_version == MPI26_VERSION)))
12110	return -ENODEV;
12111
12112	switch (hba_mpi_version) {
12113	case MPI2_VERSION:
12114	pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S |
12115	PCIE_LINK_STATE_L1 | PCIE_LINK_STATE_CLKPM);
12116	/* Use mpt2sas driver host template for SAS 2.0 HBA's */
12117	shost = scsi_host_alloc(&mpt2sas_driver_template,
12118	sizeof(struct MPT3SAS_ADAPTER));
12119	if (!shost)
12120	return -ENODEV;
12121	ioc = shost_priv(shost);
12122	memset(ioc, 0, sizeof(struct MPT3SAS_ADAPTER));
12123	ioc->hba_mpi_version_belonged = hba_mpi_version;
12124	ioc->id = mpt2_ids++;
12125	sprintf(buf: ioc->driver_name, fmt: "%s", MPT2SAS_DRIVER_NAME);
12126	switch (pdev->device) {
12127	case MPI2_MFGPAGE_DEVID_SSS6200:
12128	ioc->is_warpdrive = 1;
12129	ioc->hide_ir_msg = 1;
12130	break;
12131	case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP:
12132	case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1:
12133	ioc->is_mcpu_endpoint = 1;
12134	break;
12135	default:
12136	ioc->mfg_pg10_hide_flag = MFG_PAGE10_EXPOSE_ALL_DISKS;
12137	break;
12138	}
12139
12140	if (multipath_on_hba == -1 || multipath_on_hba == 0)
12141	ioc->multipath_on_hba = 0;
12142	else
12143	ioc->multipath_on_hba = 1;
12144
12145	break;
12146	case MPI25_VERSION:
12147	case MPI26_VERSION:
12148	/* Use mpt3sas driver host template for SAS 3.0 HBA's */
12149	shost = scsi_host_alloc(&mpt3sas_driver_template,
12150	sizeof(struct MPT3SAS_ADAPTER));
12151	if (!shost)
12152	return -ENODEV;
12153	ioc = shost_priv(shost);
12154	memset(ioc, 0, sizeof(struct MPT3SAS_ADAPTER));
12155	ioc->hba_mpi_version_belonged = hba_mpi_version;
12156	ioc->id = mpt3_ids++;
12157	sprintf(buf: ioc->driver_name, fmt: "%s", MPT3SAS_DRIVER_NAME);
12158	switch (pdev->device) {
12159	case MPI26_MFGPAGE_DEVID_SAS3508:
12160	case MPI26_MFGPAGE_DEVID_SAS3508_1:
12161	case MPI26_MFGPAGE_DEVID_SAS3408:
12162	case MPI26_MFGPAGE_DEVID_SAS3516:
12163	case MPI26_MFGPAGE_DEVID_SAS3516_1:
12164	case MPI26_MFGPAGE_DEVID_SAS3416:
12165	case MPI26_MFGPAGE_DEVID_SAS3616:
12166	case MPI26_ATLAS_PCIe_SWITCH_DEVID:
12167	ioc->is_gen35_ioc = 1;
12168	break;
12169	case MPI26_MFGPAGE_DEVID_INVALID0_3816:
12170	case MPI26_MFGPAGE_DEVID_INVALID0_3916:
12171	dev_err(&pdev->dev,
12172	"HBA with DeviceId 0x%04x, sub VendorId 0x%04x, sub DeviceId 0x%04x is Invalid",
12173	pdev->device, pdev->subsystem_vendor,
12174	pdev->subsystem_device);
12175	return 1;
12176	case MPI26_MFGPAGE_DEVID_INVALID1_3816:
12177	case MPI26_MFGPAGE_DEVID_INVALID1_3916:
12178	dev_err(&pdev->dev,
12179	"HBA with DeviceId 0x%04x, sub VendorId 0x%04x, sub DeviceId 0x%04x is Tampered",
12180	pdev->device, pdev->subsystem_vendor,
12181	pdev->subsystem_device);
12182	return 1;
12183	case MPI26_MFGPAGE_DEVID_CFG_SEC_3816:
12184	case MPI26_MFGPAGE_DEVID_CFG_SEC_3916:
12185	dev_info(&pdev->dev,
12186	"HBA is in Configurable Secure mode\n");
12187	fallthrough;
12188	case MPI26_MFGPAGE_DEVID_HARD_SEC_3816:
12189	case MPI26_MFGPAGE_DEVID_HARD_SEC_3916:
12190	ioc->is_aero_ioc = ioc->is_gen35_ioc = 1;
12191	break;
12192	default:
12193	ioc->is_gen35_ioc = ioc->is_aero_ioc = 0;
12194	}
12195	if ((ioc->hba_mpi_version_belonged == MPI25_VERSION &&
12196	pdev->revision >= SAS3_PCI_DEVICE_C0_REVISION) ||
12197	(ioc->hba_mpi_version_belonged == MPI26_VERSION)) {
12198	ioc->combined_reply_queue = 1;
12199	if (ioc->is_gen35_ioc)
12200	ioc->combined_reply_index_count =
12201	MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G35;
12202	else
12203	ioc->combined_reply_index_count =
12204	MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G3;
12205	}
12206
12207	switch (ioc->is_gen35_ioc) {
12208	case 0:
12209	if (multipath_on_hba == -1 || multipath_on_hba == 0)
12210	ioc->multipath_on_hba = 0;
12211	else
12212	ioc->multipath_on_hba = 1;
12213	break;
12214	case 1:
12215	if (multipath_on_hba == -1 || multipath_on_hba > 0)
12216	ioc->multipath_on_hba = 1;
12217	else
12218	ioc->multipath_on_hba = 0;
12219	break;
12220	default:
12221	break;
12222	}
12223
12224	break;
12225	default:
12226	return -ENODEV;
12227	}
12228
12229	INIT_LIST_HEAD(list: &ioc->list);
12230	spin_lock(lock: &gioc_lock);
12231	list_add_tail(new: &ioc->list, head: &mpt3sas_ioc_list);
12232	spin_unlock(lock: &gioc_lock);
12233	ioc->shost = shost;
12234	ioc->pdev = pdev;
12235	ioc->scsi_io_cb_idx = scsi_io_cb_idx;
12236	ioc->tm_cb_idx = tm_cb_idx;
12237	ioc->ctl_cb_idx = ctl_cb_idx;
12238	ioc->base_cb_idx = base_cb_idx;
12239	ioc->port_enable_cb_idx = port_enable_cb_idx;
12240	ioc->transport_cb_idx = transport_cb_idx;
12241	ioc->scsih_cb_idx = scsih_cb_idx;
12242	ioc->config_cb_idx = config_cb_idx;
12243	ioc->tm_tr_cb_idx = tm_tr_cb_idx;
12244	ioc->tm_tr_volume_cb_idx = tm_tr_volume_cb_idx;
12245	ioc->tm_sas_control_cb_idx = tm_sas_control_cb_idx;
12246	ioc->logging_level = logging_level;
12247	ioc->schedule_dead_ioc_flush_running_cmds = &_scsih_flush_running_cmds;
12248	/* Host waits for minimum of six seconds */
12249	ioc->max_shutdown_latency = IO_UNIT_CONTROL_SHUTDOWN_TIMEOUT;
12250	/*
12251	* Enable MEMORY MOVE support flag.
12252	*/
12253	ioc->drv_support_bitmap |= MPT_DRV_SUPPORT_BITMAP_MEMMOVE;
12254	/* Enable ADDITIONAL QUERY support flag. */
12255	ioc->drv_support_bitmap |= MPT_DRV_SUPPORT_BITMAP_ADDNLQUERY;
12256
12257	ioc->enable_sdev_max_qd = enable_sdev_max_qd;
12258
12259	/* misc semaphores and spin locks */
12260	mutex_init(&ioc->reset_in_progress_mutex);
12261	/* initializing pci_access_mutex lock */
12262	mutex_init(&ioc->pci_access_mutex);
12263	spin_lock_init(&ioc->ioc_reset_in_progress_lock);
12264	spin_lock_init(&ioc->scsi_lookup_lock);
12265	spin_lock_init(&ioc->sas_device_lock);
12266	spin_lock_init(&ioc->sas_node_lock);
12267	spin_lock_init(&ioc->fw_event_lock);
12268	spin_lock_init(&ioc->raid_device_lock);
12269	spin_lock_init(&ioc->pcie_device_lock);
12270	spin_lock_init(&ioc->diag_trigger_lock);
12271
12272	INIT_LIST_HEAD(list: &ioc->sas_device_list);
12273	INIT_LIST_HEAD(list: &ioc->sas_device_init_list);
12274	INIT_LIST_HEAD(list: &ioc->sas_expander_list);
12275	INIT_LIST_HEAD(list: &ioc->enclosure_list);
12276	INIT_LIST_HEAD(list: &ioc->pcie_device_list);
12277	INIT_LIST_HEAD(list: &ioc->pcie_device_init_list);
12278	INIT_LIST_HEAD(list: &ioc->fw_event_list);
12279	INIT_LIST_HEAD(list: &ioc->raid_device_list);
12280	INIT_LIST_HEAD(list: &ioc->sas_hba.sas_port_list);
12281	INIT_LIST_HEAD(list: &ioc->delayed_tr_list);
12282	INIT_LIST_HEAD(list: &ioc->delayed_sc_list);
12283	INIT_LIST_HEAD(list: &ioc->delayed_event_ack_list);
12284	INIT_LIST_HEAD(list: &ioc->delayed_tr_volume_list);
12285	INIT_LIST_HEAD(list: &ioc->reply_queue_list);
12286	INIT_LIST_HEAD(list: &ioc->port_table_list);
12287
12288	sprintf(buf: ioc->name, fmt: "%s_cm%d", ioc->driver_name, ioc->id);
12289
12290	/* init shost parameters */
12291	shost->max_cmd_len = 32;
12292	shost->max_lun = max_lun;
12293	shost->transportt = mpt3sas_transport_template;
12294	shost->unique_id = ioc->id;
12295
12296	if (ioc->is_mcpu_endpoint) {
12297	/* mCPU MPI support 64K max IO */
12298	shost->max_sectors = 128;
12299	ioc_info(ioc, "The max_sectors value is set to %d\n",
12300	shost->max_sectors);
12301	} else {
12302	if (max_sectors != 0xFFFF) {
12303	if (max_sectors < 64) {
12304	shost->max_sectors = 64;
12305	ioc_warn(ioc, "Invalid value %d passed for max_sectors, range is 64 to 32767. Assigning value of 64.\n",
12306	max_sectors);
12307	} else if (max_sectors > 32767) {
12308	shost->max_sectors = 32767;
12309	ioc_warn(ioc, "Invalid value %d passed for max_sectors, range is 64 to 32767.Assigning default value of 32767.\n",
12310	max_sectors);
12311	} else {
12312	shost->max_sectors = max_sectors & 0xFFFE;
12313	ioc_info(ioc, "The max_sectors value is set to %d\n",
12314	shost->max_sectors);
12315	}
12316	}
12317	}
12318	/* register EEDP capabilities with SCSI layer */
12319	if (prot_mask >= 0)
12320	scsi_host_set_prot(shost, mask: (prot_mask & 0x07));
12321	else
12322	scsi_host_set_prot(shost, mask: SHOST_DIF_TYPE1_PROTECTION
12323	| SHOST_DIF_TYPE2_PROTECTION
12324	| SHOST_DIF_TYPE3_PROTECTION);
12325
12326	scsi_host_set_guard(shost, type: SHOST_DIX_GUARD_CRC);
12327
12328	/* event thread */
12329	snprintf(buf: ioc->firmware_event_name, size: sizeof(ioc->firmware_event_name),
12330	fmt: "fw_event_%s%d", ioc->driver_name, ioc->id);
12331	ioc->firmware_event_thread = alloc_ordered_workqueue(
12332	ioc->firmware_event_name, 0);
12333	if (!ioc->firmware_event_thread) {
12334	ioc_err(ioc, "failure at %s:%d/%s()!\n",
12335	__FILE__, __LINE__, __func__);
12336	rv = -ENODEV;
12337	goto out_thread_fail;
12338	}
12339
12340	shost->host_tagset = 0;
12341
12342	if (ioc->is_gen35_ioc && host_tagset_enable)
12343	shost->host_tagset = 1;
12344
12345	ioc->is_driver_loading = 1;
12346	if ((mpt3sas_base_attach(ioc))) {
12347	ioc_err(ioc, "failure at %s:%d/%s()!\n",
12348	__FILE__, __LINE__, __func__);
12349	rv = -ENODEV;
12350	goto out_attach_fail;
12351	}
12352
12353	if (ioc->is_warpdrive) {
12354	if (ioc->mfg_pg10_hide_flag == MFG_PAGE10_EXPOSE_ALL_DISKS)
12355	ioc->hide_drives = 0;
12356	else if (ioc->mfg_pg10_hide_flag == MFG_PAGE10_HIDE_ALL_DISKS)
12357	ioc->hide_drives = 1;
12358	else {
12359	if (mpt3sas_get_num_volumes(ioc))
12360	ioc->hide_drives = 1;
12361	else
12362	ioc->hide_drives = 0;
12363	}
12364	} else
12365	ioc->hide_drives = 0;
12366
12367	shost->nr_hw_queues = 1;
12368
12369	if (shost->host_tagset) {
12370	shost->nr_hw_queues =
12371	ioc->reply_queue_count - ioc->high_iops_queues;
12372
12373	iopoll_q_count =
12374	ioc->reply_queue_count - ioc->iopoll_q_start_index;
12375
12376	shost->nr_maps = iopoll_q_count ? 3 : 1;
12377
12378	dev_info(&ioc->pdev->dev,
12379	"Max SCSIIO MPT commands: %d shared with nr_hw_queues = %d\n",
12380	shost->can_queue, shost->nr_hw_queues);
12381	}
12382
12383	rv = scsi_add_host(host: shost, dev: &pdev->dev);
12384	if (rv) {
12385	ioc_err(ioc, "failure at %s:%d/%s()!\n",
12386	__FILE__, __LINE__, __func__);
12387	goto out_add_shost_fail;
12388	}
12389
12390	scsi_scan_host(shost);
12391	mpt3sas_setup_debugfs(ioc);
12392	return 0;
12393	out_add_shost_fail:
12394	mpt3sas_base_detach(ioc);
12395	out_attach_fail:
12396	destroy_workqueue(wq: ioc->firmware_event_thread);
12397	out_thread_fail:
12398	spin_lock(lock: &gioc_lock);
12399	list_del(entry: &ioc->list);
12400	spin_unlock(lock: &gioc_lock);
12401	scsi_host_put(t: shost);
12402	return rv;
12403	}
12404
12405	/**
12406	* scsih_suspend - power management suspend main entry point
12407	* @dev: Device struct
12408	*
12409	* Return: 0 success, anything else error.
12410	*/
12411	static int __maybe_unused
12412	scsih_suspend(struct device *dev)
12413	{
12414	struct pci_dev *pdev = to_pci_dev(dev);
12415	struct Scsi_Host *shost;
12416	struct MPT3SAS_ADAPTER *ioc;
12417	int rc;
12418
12419	rc = _scsih_get_shost_and_ioc(pdev, shost: &shost, ioc: &ioc);
12420	if (rc)
12421	return rc;
12422
12423	mpt3sas_base_stop_watchdog(ioc);
12424	scsi_block_requests(shost);
12425	_scsih_nvme_shutdown(ioc);
12426	ioc_info(ioc, "pdev=0x%p, slot=%s, entering operating state\n",
12427	pdev, pci_name(pdev));
12428
12429	mpt3sas_base_free_resources(ioc);
12430	return 0;
12431	}
12432
12433	/**
12434	* scsih_resume - power management resume main entry point
12435	* @dev: Device struct
12436	*
12437	* Return: 0 success, anything else error.
12438	*/
12439	static int __maybe_unused
12440	scsih_resume(struct device *dev)
12441	{
12442	struct pci_dev *pdev = to_pci_dev(dev);
12443	struct Scsi_Host *shost;
12444	struct MPT3SAS_ADAPTER *ioc;
12445	pci_power_t device_state = pdev->current_state;
12446	int r;
12447
12448	r = _scsih_get_shost_and_ioc(pdev, shost: &shost, ioc: &ioc);
12449	if (r)
12450	return r;
12451
12452	ioc_info(ioc, "pdev=0x%p, slot=%s, previous operating state [D%d]\n",
12453	pdev, pci_name(pdev), device_state);
12454
12455	ioc->pdev = pdev;
12456	r = mpt3sas_base_map_resources(ioc);
12457	if (r)
12458	return r;
12459	ioc_info(ioc, "Issuing Hard Reset as part of OS Resume\n");
12460	mpt3sas_base_hard_reset_handler(ioc, type: SOFT_RESET);
12461	scsi_unblock_requests(shost);
12462	mpt3sas_base_start_watchdog(ioc);
12463	return 0;
12464	}
12465
12466	/**
12467	* scsih_pci_error_detected - Called when a PCI error is detected.
12468	* @pdev: PCI device struct
12469	* @state: PCI channel state
12470	*
12471	* Description: Called when a PCI error is detected.
12472	*
12473	* Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
12474	*/
12475	static pci_ers_result_t
12476	scsih_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
12477	{
12478	struct Scsi_Host *shost;
12479	struct MPT3SAS_ADAPTER *ioc;
12480
12481	if (_scsih_get_shost_and_ioc(pdev, shost: &shost, ioc: &ioc))
12482	return PCI_ERS_RESULT_DISCONNECT;
12483
12484	ioc_info(ioc, "PCI error: detected callback, state(%d)!!\n", state);
12485
12486	switch (state) {
12487	case pci_channel_io_normal:
12488	return PCI_ERS_RESULT_CAN_RECOVER;
12489	case pci_channel_io_frozen:
12490	/* Fatal error, prepare for slot reset */
12491	ioc->pci_error_recovery = 1;
12492	scsi_block_requests(ioc->shost);
12493	mpt3sas_base_stop_watchdog(ioc);
12494	mpt3sas_base_free_resources(ioc);
12495	return PCI_ERS_RESULT_NEED_RESET;
12496	case pci_channel_io_perm_failure:
12497	/* Permanent error, prepare for device removal */
12498	ioc->pci_error_recovery = 1;
12499	mpt3sas_base_stop_watchdog(ioc);
12500	mpt3sas_base_pause_mq_polling(ioc);
12501	_scsih_flush_running_cmds(ioc);
12502	return PCI_ERS_RESULT_DISCONNECT;
12503	}
12504	return PCI_ERS_RESULT_NEED_RESET;
12505	}
12506
12507	/**
12508	* scsih_pci_slot_reset - Called when PCI slot has been reset.
12509	* @pdev: PCI device struct
12510	*
12511	* Description: This routine is called by the pci error recovery
12512	* code after the PCI slot has been reset, just before we
12513	* should resume normal operations.
12514	*/
12515	static pci_ers_result_t
12516	scsih_pci_slot_reset(struct pci_dev *pdev)
12517	{
12518	struct Scsi_Host *shost;
12519	struct MPT3SAS_ADAPTER *ioc;
12520	int rc;
12521
12522	if (_scsih_get_shost_and_ioc(pdev, shost: &shost, ioc: &ioc))
12523	return PCI_ERS_RESULT_DISCONNECT;
12524
12525	ioc_info(ioc, "PCI error: slot reset callback!!\n");
12526
12527	ioc->pci_error_recovery = 0;
12528	ioc->pdev = pdev;
12529	pci_restore_state(dev: pdev);
12530	rc = mpt3sas_base_map_resources(ioc);
12531	if (rc)
12532	return PCI_ERS_RESULT_DISCONNECT;
12533
12534	ioc_info(ioc, "Issuing Hard Reset as part of PCI Slot Reset\n");
12535	rc = mpt3sas_base_hard_reset_handler(ioc, type: FORCE_BIG_HAMMER);
12536
12537	ioc_warn(ioc, "hard reset: %s\n",
12538	(rc == 0) ? "success" : "failed");
12539
12540	if (!rc)
12541	return PCI_ERS_RESULT_RECOVERED;
12542	else
12543	return PCI_ERS_RESULT_DISCONNECT;
12544	}
12545
12546	/**
12547	* scsih_pci_resume() - resume normal ops after PCI reset
12548	* @pdev: pointer to PCI device
12549	*
12550	* Called when the error recovery driver tells us that its
12551	* OK to resume normal operation. Use completion to allow
12552	* halted scsi ops to resume.
12553	*/
12554	static void
12555	scsih_pci_resume(struct pci_dev *pdev)
12556	{
12557	struct Scsi_Host *shost;
12558	struct MPT3SAS_ADAPTER *ioc;
12559
12560	if (_scsih_get_shost_and_ioc(pdev, shost: &shost, ioc: &ioc))
12561	return;
12562
12563	ioc_info(ioc, "PCI error: resume callback!!\n");
12564
12565	mpt3sas_base_start_watchdog(ioc);
12566	scsi_unblock_requests(ioc->shost);
12567	}
12568
12569	/**
12570	* scsih_pci_mmio_enabled - Enable MMIO and dump debug registers
12571	* @pdev: pointer to PCI device
12572	*/
12573	static pci_ers_result_t
12574	scsih_pci_mmio_enabled(struct pci_dev *pdev)
12575	{
12576	struct Scsi_Host *shost;
12577	struct MPT3SAS_ADAPTER *ioc;
12578
12579	if (_scsih_get_shost_and_ioc(pdev, shost: &shost, ioc: &ioc))
12580	return PCI_ERS_RESULT_DISCONNECT;
12581
12582	ioc_info(ioc, "PCI error: mmio enabled callback!!\n");
12583
12584	/* TODO - dump whatever for debugging purposes */
12585
12586	/* This called only if scsih_pci_error_detected returns
12587	* PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
12588	* works, no need to reset slot.
12589	*/
12590	return PCI_ERS_RESULT_RECOVERED;
12591	}
12592
12593	/**
12594	* scsih_ncq_prio_supp - Check for NCQ command priority support
12595	* @sdev: scsi device struct
12596	*
12597	* This is called when a user indicates they would like to enable
12598	* ncq command priorities. This works only on SATA devices.
12599	*/
12600	bool scsih_ncq_prio_supp(struct scsi_device *sdev)
12601	{
12602	struct scsi_vpd *vpd;
12603	bool ncq_prio_supp = false;
12604
12605	rcu_read_lock();
12606	vpd = rcu_dereference(sdev->vpd_pg89);
12607	if (!vpd || vpd->len < 214)
12608	goto out;
12609
12610	ncq_prio_supp = (vpd->data[213] >> 4) & 1;
12611	out:
12612	rcu_read_unlock();
12613
12614	return ncq_prio_supp;
12615	}
12616	/*
12617	* The pci device ids are defined in mpi/mpi2_cnfg.h.
12618	*/
12619	static const struct pci_device_id mpt3sas_pci_table[] = {
12620	/* Spitfire ~ 2004 */
12621	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2004,
12622	PCI_ANY_ID, PCI_ANY_ID },
12623	/* Falcon ~ 2008 */
12624	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2008,
12625	PCI_ANY_ID, PCI_ANY_ID },
12626	/* Liberator ~ 2108 */
12627	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2108_1,
12628	PCI_ANY_ID, PCI_ANY_ID },
12629	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2108_2,
12630	PCI_ANY_ID, PCI_ANY_ID },
12631	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2108_3,
12632	PCI_ANY_ID, PCI_ANY_ID },
12633	/* Meteor ~ 2116 */
12634	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2116_1,
12635	PCI_ANY_ID, PCI_ANY_ID },
12636	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2116_2,
12637	PCI_ANY_ID, PCI_ANY_ID },
12638	/* Thunderbolt ~ 2208 */
12639	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_1,
12640	PCI_ANY_ID, PCI_ANY_ID },
12641	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_2,
12642	PCI_ANY_ID, PCI_ANY_ID },
12643	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_3,
12644	PCI_ANY_ID, PCI_ANY_ID },
12645	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_4,
12646	PCI_ANY_ID, PCI_ANY_ID },
12647	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_5,
12648	PCI_ANY_ID, PCI_ANY_ID },
12649	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_6,
12650	PCI_ANY_ID, PCI_ANY_ID },
12651	/* Mustang ~ 2308 */
12652	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2308_1,
12653	PCI_ANY_ID, PCI_ANY_ID },
12654	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2308_2,
12655	PCI_ANY_ID, PCI_ANY_ID },
12656	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2308_3,
12657	PCI_ANY_ID, PCI_ANY_ID },
12658	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP,
12659	PCI_ANY_ID, PCI_ANY_ID },
12660	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1,
12661	PCI_ANY_ID, PCI_ANY_ID },
12662	/* SSS6200 */
12663	{ MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SSS6200,
12664	PCI_ANY_ID, PCI_ANY_ID },
12665	/* Fury ~ 3004 and 3008 */
12666	{ MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3004,
12667	PCI_ANY_ID, PCI_ANY_ID },
12668	{ MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3008,
12669	PCI_ANY_ID, PCI_ANY_ID },
12670	/* Invader ~ 3108 */
12671	{ MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_1,
12672	PCI_ANY_ID, PCI_ANY_ID },
12673	{ MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_2,
12674	PCI_ANY_ID, PCI_ANY_ID },
12675	{ MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_5,
12676	PCI_ANY_ID, PCI_ANY_ID },
12677	{ MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_6,
12678	PCI_ANY_ID, PCI_ANY_ID },
12679	/* Cutlass ~ 3216 and 3224 */
12680	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3216,
12681	PCI_ANY_ID, PCI_ANY_ID },
12682	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3224,
12683	PCI_ANY_ID, PCI_ANY_ID },
12684	/* Intruder ~ 3316 and 3324 */
12685	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_1,
12686	PCI_ANY_ID, PCI_ANY_ID },
12687	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_2,
12688	PCI_ANY_ID, PCI_ANY_ID },
12689	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_3,
12690	PCI_ANY_ID, PCI_ANY_ID },
12691	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_4,
12692	PCI_ANY_ID, PCI_ANY_ID },
12693	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_1,
12694	PCI_ANY_ID, PCI_ANY_ID },
12695	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_2,
12696	PCI_ANY_ID, PCI_ANY_ID },
12697	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_3,
12698	PCI_ANY_ID, PCI_ANY_ID },
12699	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_4,
12700	PCI_ANY_ID, PCI_ANY_ID },
12701	/* Ventura, Crusader, Harpoon & Tomcat ~ 3516, 3416, 3508 & 3408*/
12702	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3508,
12703	PCI_ANY_ID, PCI_ANY_ID },
12704	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3508_1,
12705	PCI_ANY_ID, PCI_ANY_ID },
12706	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3408,
12707	PCI_ANY_ID, PCI_ANY_ID },
12708	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3516,
12709	PCI_ANY_ID, PCI_ANY_ID },
12710	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3516_1,
12711	PCI_ANY_ID, PCI_ANY_ID },
12712	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3416,
12713	PCI_ANY_ID, PCI_ANY_ID },
12714	/* Mercator ~ 3616*/
12715	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3616,
12716	PCI_ANY_ID, PCI_ANY_ID },
12717
12718	/* Aero SI 0x00E1 Configurable Secure
12719	* 0x00E2 Hard Secure
12720	*/
12721	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_CFG_SEC_3916,
12722	PCI_ANY_ID, PCI_ANY_ID },
12723	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_HARD_SEC_3916,
12724	PCI_ANY_ID, PCI_ANY_ID },
12725
12726	/*
12727	* Aero SI –> 0x00E0 Invalid, 0x00E3 Tampered
12728	*/
12729	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_INVALID0_3916,
12730	PCI_ANY_ID, PCI_ANY_ID },
12731	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_INVALID1_3916,
12732	PCI_ANY_ID, PCI_ANY_ID },
12733
12734	/* Atlas PCIe Switch Management Port */
12735	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_ATLAS_PCIe_SWITCH_DEVID,
12736	PCI_ANY_ID, PCI_ANY_ID },
12737
12738	/* Sea SI 0x00E5 Configurable Secure
12739	* 0x00E6 Hard Secure
12740	*/
12741	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_CFG_SEC_3816,
12742	PCI_ANY_ID, PCI_ANY_ID },
12743	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_HARD_SEC_3816,
12744	PCI_ANY_ID, PCI_ANY_ID },
12745
12746	/*
12747	* ATTO Branded ExpressSAS H12xx GT
12748	*/
12749	{ MPI2_MFGPAGE_VENDORID_ATTO, MPI26_MFGPAGE_DEVID_HARD_SEC_3816,
12750	PCI_ANY_ID, PCI_ANY_ID },
12751
12752	/*
12753	* Sea SI –> 0x00E4 Invalid, 0x00E7 Tampered
12754	*/
12755	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_INVALID0_3816,
12756	PCI_ANY_ID, PCI_ANY_ID },
12757	{ MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_INVALID1_3816,
12758	PCI_ANY_ID, PCI_ANY_ID },
12759
12760	{0} /* Terminating entry */
12761	};
12762	MODULE_DEVICE_TABLE(pci, mpt3sas_pci_table);
12763
12764	static struct pci_error_handlers _mpt3sas_err_handler = {
12765	.error_detected = scsih_pci_error_detected,
12766	.mmio_enabled = scsih_pci_mmio_enabled,
12767	.slot_reset = scsih_pci_slot_reset,
12768	.resume = scsih_pci_resume,
12769	};
12770
12771	static SIMPLE_DEV_PM_OPS(scsih_pm_ops, scsih_suspend, scsih_resume);
12772
12773	static struct pci_driver mpt3sas_driver = {
12774	.name = MPT3SAS_DRIVER_NAME,
12775	.id_table = mpt3sas_pci_table,
12776	.probe = _scsih_probe,
12777	.remove = scsih_remove,
12778	.shutdown = scsih_shutdown,
12779	.err_handler = &_mpt3sas_err_handler,
12780	.driver.pm = &scsih_pm_ops,
12781	};
12782
12783	/**
12784	* scsih_init - main entry point for this driver.
12785	*
12786	* Return: 0 success, anything else error.
12787	*/
12788	static int
12789	scsih_init(void)
12790	{
12791	mpt2_ids = 0;
12792	mpt3_ids = 0;
12793
12794	mpt3sas_base_initialize_callback_handler();
12795
12796	/* queuecommand callback hander */
12797	scsi_io_cb_idx = mpt3sas_base_register_callback_handler(cb_func: _scsih_io_done);
12798
12799	/* task management callback handler */
12800	tm_cb_idx = mpt3sas_base_register_callback_handler(cb_func: _scsih_tm_done);
12801
12802	/* base internal commands callback handler */
12803	base_cb_idx = mpt3sas_base_register_callback_handler(cb_func: mpt3sas_base_done);
12804	port_enable_cb_idx = mpt3sas_base_register_callback_handler(
12805	cb_func: mpt3sas_port_enable_done);
12806
12807	/* transport internal commands callback handler */
12808	transport_cb_idx = mpt3sas_base_register_callback_handler(
12809	cb_func: mpt3sas_transport_done);
12810
12811	/* scsih internal commands callback handler */
12812	scsih_cb_idx = mpt3sas_base_register_callback_handler(cb_func: _scsih_done);
12813
12814	/* configuration page API internal commands callback handler */
12815	config_cb_idx = mpt3sas_base_register_callback_handler(
12816	cb_func: mpt3sas_config_done);
12817
12818	/* ctl module callback handler */
12819	ctl_cb_idx = mpt3sas_base_register_callback_handler(cb_func: mpt3sas_ctl_done);
12820
12821	tm_tr_cb_idx = mpt3sas_base_register_callback_handler(
12822	cb_func: _scsih_tm_tr_complete);
12823
12824	tm_tr_volume_cb_idx = mpt3sas_base_register_callback_handler(
12825	cb_func: _scsih_tm_volume_tr_complete);
12826
12827	tm_sas_control_cb_idx = mpt3sas_base_register_callback_handler(
12828	cb_func: _scsih_sas_control_complete);
12829
12830	mpt3sas_init_debugfs();
12831	return 0;
12832	}
12833
12834	/**
12835	* scsih_exit - exit point for this driver (when it is a module).
12836	*
12837	* Return: 0 success, anything else error.
12838	*/
12839	static void
12840	scsih_exit(void)
12841	{
12842
12843	mpt3sas_base_release_callback_handler(cb_idx: scsi_io_cb_idx);
12844	mpt3sas_base_release_callback_handler(cb_idx: tm_cb_idx);
12845	mpt3sas_base_release_callback_handler(cb_idx: base_cb_idx);
12846	mpt3sas_base_release_callback_handler(cb_idx: port_enable_cb_idx);
12847	mpt3sas_base_release_callback_handler(cb_idx: transport_cb_idx);
12848	mpt3sas_base_release_callback_handler(cb_idx: scsih_cb_idx);
12849	mpt3sas_base_release_callback_handler(cb_idx: config_cb_idx);
12850	mpt3sas_base_release_callback_handler(cb_idx: ctl_cb_idx);
12851
12852	mpt3sas_base_release_callback_handler(cb_idx: tm_tr_cb_idx);
12853	mpt3sas_base_release_callback_handler(cb_idx: tm_tr_volume_cb_idx);
12854	mpt3sas_base_release_callback_handler(cb_idx: tm_sas_control_cb_idx);
12855
12856	/* raid transport support */
12857	if (hbas_to_enumerate != 1)
12858	raid_class_release(mpt3sas_raid_template);
12859	if (hbas_to_enumerate != 2)
12860	raid_class_release(mpt2sas_raid_template);
12861	sas_release_transport(mpt3sas_transport_template);
12862	mpt3sas_exit_debugfs();
12863	}
12864
12865	/**
12866	* _mpt3sas_init - main entry point for this driver.
12867	*
12868	* Return: 0 success, anything else error.
12869	*/
12870	static int __init
12871	_mpt3sas_init(void)
12872	{
12873	int error;
12874
12875	pr_info("%s version %s loaded\n", MPT3SAS_DRIVER_NAME,
12876	MPT3SAS_DRIVER_VERSION);
12877
12878	mpt3sas_transport_template =
12879	sas_attach_transport(&mpt3sas_transport_functions);
12880	if (!mpt3sas_transport_template)
12881	return -ENODEV;
12882
12883	/* No need attach mpt3sas raid functions template
12884	* if hbas_to_enumarate value is one.
12885	*/
12886	if (hbas_to_enumerate != 1) {
12887	mpt3sas_raid_template =
12888	raid_class_attach(&mpt3sas_raid_functions);
12889	if (!mpt3sas_raid_template) {
12890	sas_release_transport(mpt3sas_transport_template);
12891	return -ENODEV;
12892	}
12893	}
12894
12895	/* No need to attach mpt2sas raid functions template
12896	* if hbas_to_enumarate value is two
12897	*/
12898	if (hbas_to_enumerate != 2) {
12899	mpt2sas_raid_template =
12900	raid_class_attach(&mpt2sas_raid_functions);
12901	if (!mpt2sas_raid_template) {
12902	sas_release_transport(mpt3sas_transport_template);
12903	return -ENODEV;
12904	}
12905	}
12906
12907	error = scsih_init();
12908	if (error) {
12909	scsih_exit();
12910	return error;
12911	}
12912
12913	mpt3sas_ctl_init(hbas_to_enumerate);
12914
12915	error = pci_register_driver(&mpt3sas_driver);
12916	if (error) {
12917	mpt3sas_ctl_exit(hbas_to_enumerate);
12918	scsih_exit();
12919	}
12920
12921	return error;
12922	}
12923
12924	/**
12925	* _mpt3sas_exit - exit point for this driver (when it is a module).
12926	*
12927	*/
12928	static void __exit
12929	_mpt3sas_exit(void)
12930	{
12931	pr_info("mpt3sas version %s unloading\n",
12932	MPT3SAS_DRIVER_VERSION);
12933
12934	pci_unregister_driver(dev: &mpt3sas_driver);
12935
12936	mpt3sas_ctl_exit(hbas_to_enumerate);
12937
12938	scsih_exit();
12939	}
12940
12941	module_init(_mpt3sas_init);
12942	module_exit(_mpt3sas_exit);
12943